From 23b9cfda5b73873ac6d0c27df44bea3f4f452c38 Mon Sep 17 00:00:00 2001 From: zhjc1124 Date: Mon, 12 Aug 2024 03:42:49 +0000 Subject: [PATCH 001/249] fix bug of check tenant_id in init_pkt --- deps/oblib/src/rpc/obrpc/ob_rpc_proxy.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_proxy.cpp b/deps/oblib/src/rpc/obrpc/ob_rpc_proxy.cpp index 1382bab3c..16788450a 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_proxy.cpp +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_proxy.cpp @@ -184,8 +184,7 @@ int ObRpcProxy::init_pkt( if (0 != get_group_id()) { pkt->set_group_id(get_group_id()); } else if (this_worker().get_group_id() == OBCG_LQ || - (is_user_group(this_worker().get_group_id()) && - (is_virtual_tenant_id(ob_get_tenant_id()) || ob_get_tenant_id() != tenant_id_))) { + (is_user_group(this_worker().get_group_id()) && ob_get_tenant_id() != tenant_id_)) { pkt->set_group_id(0); } else { pkt->set_group_id(this_worker().get_group_id()); From ca9ad2c6a49eef5b8ff8ae2c24ce5b165badcdc6 Mon Sep 17 00:00:00 2001 From: obdev Date: Mon, 12 Aug 2024 03:48:40 +0000 Subject: [PATCH 002/249] Add placeholder for parquet related item type --- src/objit/include/objit/common/ob_item_type.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/objit/include/objit/common/ob_item_type.h b/src/objit/include/objit/common/ob_item_type.h index 70840d3b7..8bf59e422 100755 --- a/src/objit/include/objit/common/ob_item_type.h +++ b/src/objit/include/objit/common/ob_item_type.h @@ -2577,6 +2577,10 @@ typedef enum ObItemType T_REBUILD_TABLET = 4727, T_MICRO_INDEX_CLUSTERED = 4728, + + // Parquet related + T_PER_ROW_GROUP_SIZE = 4729, + T_COMPRESSION_ALGORITHM = 4730, T_MAX //Attention: add a new type before T_MAX } ObItemType; From 2040575bb05a08aabc14cba36d40bea0999ffbb3 Mon Sep 17 00:00:00 2001 From: chinaxing Date: Mon, 12 Aug 2024 04:07:08 +0000 Subject: [PATCH 003/249] fix push callback-list-0's checksum_scn for parallel redo repeated repaly --- src/storage/memtable/mvcc/ob_mvcc_trans_ctx.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.cpp b/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.cpp index 0dd7d0c01..f44cb308a 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.cpp +++ b/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.cpp @@ -385,7 +385,7 @@ int ObTransCallbackMgr::append(ObITransCallback *node) } else { // need to push up the callback_list.0's checksum_scn // to avoid it calculate checksum includes those callbacks - callback_list_.inc_update_checksum_scn(serial_final_scn_); + callback_list_.inc_update_checksum_scn(SCN::scn_inc(serial_final_scn_)); ATOMIC_STORE(&has_branch_replayed_into_first_list_, true); TRANS_LOG(INFO, "replay log before serial final when reach serial final", KPC(this), KPC(get_trans_ctx()), KPC(node)); From 885df541724164708dad64e50156a2f3ace3db71 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Mon, 12 Aug 2024 05:04:06 +0000 Subject: [PATCH 004/249] placeholder for redis inner table --- src/share/inner_table/ob_inner_table_schema_def.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index daf86175a..478c7bab8 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -7310,6 +7310,7 @@ def_table_schema( # 524 : __all_pkg_coll_type # 525: __wr_sql_plan # 526: __wr_res_mgr_sysstat +# 527: __all_kv_redis_table # 余留位置（此行之前占位） # 本区域占位建议：采用真实表名进行占位 @@ -14613,6 +14614,7 @@ def_table_schema(**gen_iterate_virtual_table_def( # 12500: __all_virtual_kv_client_info # 12501: __all_virtual_wr_sql_plan # 12502: __all_virtual_wr_res_mgr_sysstat +# 12503: __all_virtual_kv_redis_table # 余留位置（此行之前占位） # 本区域占位建议：采用真实表名进行占位 ################################################################################ @@ -35746,6 +35748,8 @@ def_table_schema( # 21615: CDB_WR_RES_MGR_SYSSTAT # 21616: DBA_OB_SPM_EVO_RESULT # 21617: CDB_OB_SPM_EVO_RESULT +# 21618: DBA_OB_KV_REDIS_TABLE +# 21619: CDB_OB_KV_REDIS_TABLE # 余留位置（此行之前占位） # 本区域占位建议：采用真实视图名进行占位 ################################################################################ From 8df78306dba0bb857ebadbb870c72ec66c3216ea Mon Sep 17 00:00:00 2001 From: haohao022 Date: Mon, 12 Aug 2024 08:12:28 +0000 Subject: [PATCH 005/249] [CP] [to #2024072000103865484] fix: erroneously unpacking the char type field of the complex type into varchar type --- src/observer/mysql/obmp_stmt_execute.cpp | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/observer/mysql/obmp_stmt_execute.cpp b/src/observer/mysql/obmp_stmt_execute.cpp index aeda98832..4c611f71c 100644 --- a/src/observer/mysql/obmp_stmt_execute.cpp +++ b/src/observer/mysql/obmp_stmt_execute.cpp @@ -2394,7 +2394,7 @@ int ObMPStmtExecute::parse_basic_param_value(ObIAllocator &allocator, param.set_collation_type(ncs_type); } LOG_DEBUG("recieve Nchar param", K(ret), K(str), K(dst)); - } else if (ObURowIDType == type) { + } else if (MYSQL_TYPE_OB_UROWID == type) { // decode bae64 str and get urowid content ObURowIDData urowid_data; if (OB_FAIL(ObURowIDData::decode2urowid(str.ptr(), str.length(), @@ -2540,16 +2540,22 @@ int ObMPStmtExecute::parse_basic_param_value(ObIAllocator &allocator, LOG_WARN("Fail to convert plain lob data to templob",K(ret)); } } - } else { + } else if (MYSQL_TYPE_STRING == type + || MYSQL_TYPE_VARCHAR == type + || MYSQL_TYPE_VAR_STRING == type) { param.set_collation_type(cs_type); - if (is_oracle_mode() && !is_complex_element) { - param.set_char(dst); - } else { - if (is_complex_element && dst.length()== 0) { + if (is_complex_element) { + if (dst.length()== 0) { param.set_null(); + } else if (MYSQL_TYPE_STRING == type) { // ObCharType + param.set_char(dst); } else { param.set_varchar(dst); } + } else if (is_oracle_mode()) { + param.set_char(dst); + } else { + param.set_varchar(dst); } } } From 4e1728cb0119e963193c7e36ae8beb3abee1ebb1 Mon Sep 17 00:00:00 2001 From: haohao022 Date: Mon, 12 Aug 2024 08:18:16 +0000 Subject: [PATCH 006/249] [CP] [to #2024072000103865240] fix: response -9747 error packet and disconnect session when send piece protocol failed --- src/observer/mysql/obmp_stmt_send_piece_data.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/observer/mysql/obmp_stmt_send_piece_data.cpp b/src/observer/mysql/obmp_stmt_send_piece_data.cpp index 5b2bf7bea..43d40e53c 100644 --- a/src/observer/mysql/obmp_stmt_send_piece_data.cpp +++ b/src/observer/mysql/obmp_stmt_send_piece_data.cpp @@ -81,7 +81,7 @@ int ObMPStmtSendPieceData::before_process() is_null_ = (1 == is_null); ObMySQLUtil::get_int8(pos, buffer_len_); if (stmt_id_ < 1 || buffer_len_ < 0) { - ret = OB_ERR_PARAM_INVALID; + ret = OB_ERR_MALFORMED_PS_PACKET; LOG_WARN("send_piece receive unexpected params", K(ret), K(stmt_id_), K(buffer_len_)); } else if (param_id_ >= OB_PARAM_ID_OVERFLOW_RISK_THRESHOLD) { LOG_WARN("param_id_ has the risk of overflow", K(ret), K(stmt_id_), K(param_id_)); @@ -96,6 +96,13 @@ int ObMPStmtSendPieceData::before_process() LOG_INFO("resolve send_piece protocol packet", K(ret), K(stmt_id_), K(param_id_), K(buffer_len_), K(piece_mode_), K(is_null_)); } + + if (OB_FAIL(ret)) { + send_error_packet(ret, nullptr); + force_disconnect(); + LOG_WARN("force disconnect connection", K(ret)); + } + return ret; } From 61a961ec9ed8c742d62f612e440945d4812b5893 Mon Sep 17 00:00:00 2001 From: haohao022 Date: Mon, 12 Aug 2024 08:24:11 +0000 Subject: [PATCH 007/249] [CP] [to #2024071600103735497] fix: ObRecordType deserialize --- src/pl/ob_pl_user_type.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pl/ob_pl_user_type.cpp b/src/pl/ob_pl_user_type.cpp index fa81bb249..3e7293f96 100644 --- a/src/pl/ob_pl_user_type.cpp +++ b/src/pl/ob_pl_user_type.cpp @@ -2012,8 +2012,8 @@ int ObRecordType::deserialize(ObSchemaGetterGuard &schema_guard, } } else { value->set_null(); - new_dst_pos += sizeof(ObObj); } + OX (new_dst_pos += sizeof(ObObj)); } else if (OB_FAIL(type->deserialize(schema_guard, allocator, charset, cs_type, ncs_type, tz_info, src, new_dst, new_dst_len, new_dst_pos))) { LOG_WARN("deserialize record element type failed", K(i), K(*this), KP(src), KP(dst), K(dst_len), K(dst_pos), K(ret)); From 0964fd9f9e4ab783982719f4eb42c17063315ef9 Mon Sep 17 00:00:00 2001 From: zhjc1124 Date: Mon, 12 Aug 2024 08:30:03 +0000 Subject: [PATCH 008/249] [CP] fix io_clocks flush late --- src/share/io/io_schedule/ob_io_mclock.cpp | 21 ++++++++++----------- src/share/io/ob_io_struct.cpp | 12 ++++++------ 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/share/io/io_schedule/ob_io_mclock.cpp b/src/share/io/io_schedule/ob_io_mclock.cpp index 99c9e391d..f1445d657 100644 --- a/src/share/io/io_schedule/ob_io_mclock.cpp +++ b/src/share/io/io_schedule/ob_io_mclock.cpp @@ -288,6 +288,8 @@ void ObTenantIOClock::destroy() int ObTenantIOClock::calc_phyqueue_clock(ObPhyQueue *phy_queue, ObIORequest &req) { int ret = OB_SUCCESS; + const int64_t current_ts = ObTimeUtility::fast_current_time(); + bool is_unlimited = false; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret), K(is_inited_)); @@ -298,13 +300,8 @@ int ObTenantIOClock::calc_phyqueue_clock(ObPhyQueue *phy_queue, ObIORequest &req ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(req)); } else if (req.get_flag().is_unlimited()) { - const int64_t current_ts = ObTimeUtility::fast_current_time(); - phy_queue->reservation_ts_ = current_ts; - phy_queue->group_limitation_ts_ = current_ts; - phy_queue->tenant_limitation_ts_ = current_ts; - phy_queue->proportion_ts_ = current_ts; + is_unlimited = true; } else { - const int64_t current_ts = ObTimeUtility::fast_current_time(); uint64_t cur_queue_index = phy_queue->queue_index_; if (cur_queue_index < 0 || (cur_queue_index >= group_clocks_.count() && cur_queue_index != INT64_MAX)) { ret = OB_INVALID_ARGUMENT; @@ -324,11 +321,7 @@ int ObTenantIOClock::calc_phyqueue_clock(ObPhyQueue *phy_queue, ObIORequest &req LOG_WARN("get iops scale failed", K(ret), K(req)); } else if (OB_UNLIKELY(is_io_ability_valid == false)) { //unlimited - const int64_t current_ts = ObTimeUtility::fast_current_time(); - phy_queue->reservation_ts_ = current_ts; - phy_queue->group_limitation_ts_ = current_ts; - phy_queue->tenant_limitation_ts_ = current_ts; - phy_queue->proportion_ts_ = current_ts; + is_unlimited = true; } else if (OB_FAIL(mclock.calc_phy_clock(current_ts, iops_scale, weight_scale, phy_queue))) { LOG_WARN("calculate clock of the queue failed", K(ret), K(mclock), K(weight_scale)); } else { @@ -337,6 +330,12 @@ int ObTenantIOClock::calc_phyqueue_clock(ObPhyQueue *phy_queue, ObIORequest &req } } } + if (OB_FAIL(ret) || is_unlimited) { + phy_queue->reservation_ts_ = current_ts; + phy_queue->group_limitation_ts_ = current_ts; + phy_queue->tenant_limitation_ts_ = current_ts; + phy_queue->proportion_ts_ = current_ts; + } return ret; } diff --git a/src/share/io/ob_io_struct.cpp b/src/share/io/ob_io_struct.cpp index 1eeaffc86..8059b5f49 100644 --- a/src/share/io/ob_io_struct.cpp +++ b/src/share/io/ob_io_struct.cpp @@ -1251,25 +1251,25 @@ int ObIOSender::enqueue_request(ObIORequest &req) if (OB_NOT_NULL(req.io_result_)) { req.io_result_->time_log_.enqueue_ts_ = ObTimeUtility::fast_current_time(); } + } + int tmp_ret = OB_SUCCESS; + if (OB_SUCC(ret)) { //calc ts_ if (OB_NOT_NULL(req.tenant_io_mgr_.get_ptr())) { ObTenantIOClock *io_clock = static_cast(req.tenant_io_mgr_.get_ptr()->get_io_clock()); //phy_queue from idle to active and reach max_clock_adjust_wait_ts - int tmp_ret = OB_SUCCESS; if (tmp_phy_queue->reach_adjust_interval()) { tmp_ret = io_clock->sync_tenant_clock(io_clock); } - if (OB_FAIL(io_clock->calc_phyqueue_clock(tmp_phy_queue, req))) { + if (OB_TMP_FAIL(io_clock->calc_phyqueue_clock(tmp_phy_queue, req))) { LOG_WARN("calc phyqueue clock failed", K(ret), K(tmp_phy_queue->queue_index_)); } else if (OB_UNLIKELY(OB_SUCCESS != tmp_ret)) { LOG_WARN("sync tenant clock failed", K(tmp_ret)); } } } - int tmp_ret = io_queue_->push_phyqueue(tmp_phy_queue); - if (OB_UNLIKELY(OB_SUCCESS != tmp_ret)) { - LOG_WARN("re_into heap failed", K(tmp_ret)); - abort(); + if (OB_UNLIKELY(OB_TMP_FAIL(io_queue_->push_phyqueue(tmp_phy_queue)))) { + LOG_ERROR("re_into heap failed", K(tmp_ret)); } } } else { From e4c13a11e836b1cdba03106ade1714936fcb5736 Mon Sep 17 00:00:00 2001 From: sunzhoujia Date: Mon, 12 Aug 2024 09:10:58 +0000 Subject: [PATCH 009/249] occupy for erase ss_micro_cache --- src/objit/include/objit/common/ob_item_type.h | 3 +++ src/sql/resolver/ob_stmt_type.h | 1 + 2 files changed, 4 insertions(+) diff --git a/src/objit/include/objit/common/ob_item_type.h b/src/objit/include/objit/common/ob_item_type.h index 8bf59e422..019eeda7f 100755 --- a/src/objit/include/objit/common/ob_item_type.h +++ b/src/objit/include/objit/common/ob_item_type.h @@ -2581,6 +2581,9 @@ typedef enum ObItemType // Parquet related T_PER_ROW_GROUP_SIZE = 4729, T_COMPRESSION_ALGORITHM = 4730, + + // Erase micro cache + T_FLUSH_SS_MICRO_CACHE = 4731, T_MAX //Attention: add a new type before T_MAX } ObItemType; diff --git a/src/sql/resolver/ob_stmt_type.h b/src/sql/resolver/ob_stmt_type.h index a75b03d0e..81e751200 100644 --- a/src/sql/resolver/ob_stmt_type.h +++ b/src/sql/resolver/ob_stmt_type.h @@ -361,6 +361,7 @@ OB_STMT_TYPE_DEF_UNKNOWN_AT(T_SHOW_CHECK_TABLE, err_stmt_type_priv, 360) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_OLAP_ASYNC_JOB_SUBMIT, no_priv_needed, 361) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_OLAP_ASYNC_JOB_CANCEL, no_priv_needed, 362) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_SHOW_OLAP_ASYNC_JOB_STATUS, no_priv_needed, 363) +OB_STMT_TYPE_DEF_UNKNOWN_AT(T_FLUSH_SS_MICRO_CACHE, get_sys_tenant_alter_system_priv, 364) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_MAX, err_stmt_type_priv, 500) #endif From 8c0c36c10a8a3d6da3a1152e8d70dbf38bd9e1aa Mon Sep 17 00:00:00 2001 From: AnimationFan <3067477338@qq.com> Date: Mon, 12 Aug 2024 09:17:06 +0000 Subject: [PATCH 010/249] fix ddl struct place holder conflict --- src/storage/blocksstable/ob_macro_block_id.h | 1 + src/storage/ddl/ob_ddl_redo_log_writer.cpp | 6 ++++++ src/storage/ddl/ob_ddl_struct.cpp | 13 +++++++++++-- src/storage/ddl/ob_ddl_struct.h | 10 +++++++++- 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/storage/blocksstable/ob_macro_block_id.h b/src/storage/blocksstable/ob_macro_block_id.h index f171bd90c..160604f29 100644 --- a/src/storage/blocksstable/ob_macro_block_id.h +++ b/src/storage/blocksstable/ob_macro_block_id.h @@ -89,6 +89,7 @@ public: bool operator <(const MacroBlockId &other) const; NEED_SERIALIZE_AND_DESERIALIZE; + static MacroBlockId mock_valid_macro_id() { return MacroBlockId(0, AUTONOMIC_BLOCK_INDEX, 0); } public: static const int64_t MACRO_BLOCK_ID_VERSION = 0; diff --git a/src/storage/ddl/ob_ddl_redo_log_writer.cpp b/src/storage/ddl/ob_ddl_redo_log_writer.cpp index 3f4dd2af3..2cf674f44 100644 --- a/src/storage/ddl/ob_ddl_redo_log_writer.cpp +++ b/src/storage/ddl/ob_ddl_redo_log_writer.cpp @@ -1608,6 +1608,12 @@ int ObDDLRedoLogWriterCallback::write(ObMacroBlockHandle ¯o_handle, redo_info_.start_scn_ = start_scn_; redo_info_.data_format_version_ = data_format_version_; redo_info_.type_ = direct_load_type_; + + redo_info_.parallel_cnt_ = 0; // TODO @zhuoran.zzr, place holder for shared storage + redo_info_.cg_cnt_ = 0; + + redo_info_.macro_block_id_ = MacroBlockId::mock_valid_macro_id(); + if (is_column_group_info_valid()) { redo_info_.end_row_id_ = row_id_offset_ + row_count - 1; row_id_offset_ += row_count; diff --git a/src/storage/ddl/ob_ddl_struct.cpp b/src/storage/ddl/ob_ddl_struct.cpp index 1538c4825..8282e849a 100644 --- a/src/storage/ddl/ob_ddl_struct.cpp +++ b/src/storage/ddl/ob_ddl_struct.cpp @@ -322,7 +322,10 @@ ObDDLMacroBlockRedoInfo::ObDDLMacroBlockRedoInfo() end_row_id_(-1), type_(ObDirectLoadType::DIRECT_LOAD_INVALID), trans_id_(), - with_cs_replica_(false) + with_cs_replica_(false), + macro_block_id_(), + parallel_cnt_(0), + cg_cnt_(0) { } @@ -338,6 +341,9 @@ void ObDDLMacroBlockRedoInfo::reset() type_ = ObDirectLoadType::DIRECT_LOAD_INVALID; trans_id_.reset(); with_cs_replica_ = false; + macro_block_id_.reset(); + parallel_cnt_ = 0; + cg_cnt_ = 0; } bool ObDDLMacroBlockRedoInfo::is_valid() const @@ -363,7 +369,10 @@ OB_SERIALIZE_MEMBER(ObDDLMacroBlockRedoInfo, end_row_id_, type_, trans_id_, - with_cs_replica_); + with_cs_replica_, + macro_block_id_, + parallel_cnt_, + cg_cnt_); ObTabletDirectLoadMgrHandle::ObTabletDirectLoadMgrHandle() : tablet_mgr_(nullptr) diff --git a/src/storage/ddl/ob_ddl_struct.h b/src/storage/ddl/ob_ddl_struct.h index 8a221fffe..cae64144d 100644 --- a/src/storage/ddl/ob_ddl_struct.h +++ b/src/storage/ddl/ob_ddl_struct.h @@ -186,7 +186,10 @@ public: K_(end_row_id), K_(type), K_(trans_id), - K_(with_cs_replica)); + K_(with_cs_replica), + K_(macro_block_id), + K_(parallel_cnt), + K_(cg_cnt)); public: storage::ObITable::TableKey table_key_; ObString data_buffer_; @@ -198,6 +201,11 @@ public: storage::ObDirectLoadType type_; transaction::ObTransID trans_id_; // for incremental direct load only bool with_cs_replica_; + + blocksstable::MacroBlockId macro_block_id_; // for shared storage mode + // for shared storage gc occupy info + int64_t parallel_cnt_; + int64_t cg_cnt_; }; class ObTabletDirectLoadMgr; From 66938d14e8d61d18e234a9aec826b7d4310ae8eb Mon Sep 17 00:00:00 2001 From: hnwyllmm Date: Mon, 12 Aug 2024 09:41:53 +0000 Subject: [PATCH 011/249] fix load data file reader memory leak --- src/sql/engine/cmd/ob_load_data_direct_impl.cpp | 7 +++++-- src/sql/engine/cmd/ob_load_data_file_reader.cpp | 7 +++++++ src/sql/engine/cmd/ob_load_data_file_reader.h | 1 + src/sql/engine/cmd/ob_load_data_impl.cpp | 7 +++++-- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/sql/engine/cmd/ob_load_data_direct_impl.cpp b/src/sql/engine/cmd/ob_load_data_direct_impl.cpp index a36552935..751c56999 100644 --- a/src/sql/engine/cmd/ob_load_data_direct_impl.cpp +++ b/src/sql/engine/cmd/ob_load_data_direct_impl.cpp @@ -466,7 +466,7 @@ ObLoadDataDirectImpl::DataReader::DataReader() ObLoadDataDirectImpl::DataReader::~DataReader() { if (OB_NOT_NULL(file_reader_)) { - file_reader_->~ObFileReader(); + ObFileReader::destroy(file_reader_); file_reader_ = nullptr; } } @@ -502,7 +502,10 @@ int ObLoadDataDirectImpl::DataReader::init(const DataAccessParam &data_access_pa file_read_param.filename_ = data_desc.filename_; file_read_param.compression_format_ = data_access_param.compression_format_; file_read_param.access_info_ = data_access_param.access_info_; - file_read_param.packet_handle_ = &execute_ctx.exec_ctx_.get_session_info()->get_pl_query_sender()->get_packet_sender(); + file_read_param.packet_handle_ = nullptr; + if (OB_NOT_NULL(execute_ctx.exec_ctx_.get_session_info()) && OB_NOT_NULL(execute_ctx.exec_ctx_.get_session_info()->get_pl_query_sender())) { + file_read_param.packet_handle_ = &execute_ctx.exec_ctx_.get_session_info()->get_pl_query_sender()->get_packet_sender(); + } file_read_param.session_ = execute_ctx.exec_ctx_.get_session_info(); file_read_param.timeout_ts_ = THIS_WORKER.get_timeout_ts(); diff --git a/src/sql/engine/cmd/ob_load_data_file_reader.cpp b/src/sql/engine/cmd/ob_load_data_file_reader.cpp index 32861094e..ac9384871 100644 --- a/src/sql/engine/cmd/ob_load_data_file_reader.cpp +++ b/src/sql/engine/cmd/ob_load_data_file_reader.cpp @@ -132,6 +132,13 @@ int ObFileReader::open(const ObFileReadParam ¶m, ObIAllocator &allocator, Ob return ret; } +void ObFileReader::destroy(ObFileReader *file_reader) +{ + if (OB_NOT_NULL(file_reader)) { + OB_DELETE(ObFileReader, MEMORY_ATTR, file_reader); + } +} + int ObFileReader::open_decompress_reader(const ObFileReadParam ¶m, ObIAllocator &allocator, ObFileReader *source_reader, diff --git a/src/sql/engine/cmd/ob_load_data_file_reader.h b/src/sql/engine/cmd/ob_load_data_file_reader.h index 506b17c3e..b95050285 100644 --- a/src/sql/engine/cmd/ob_load_data_file_reader.h +++ b/src/sql/engine/cmd/ob_load_data_file_reader.h @@ -92,6 +92,7 @@ public: * A file reader factory */ static int open(const ObFileReadParam ¶m, ObIAllocator &allocator, ObFileReader *& file_reader); + static void destroy(ObFileReader *file_reader); private: static int open_decompress_reader(const ObFileReadParam ¶m, diff --git a/src/sql/engine/cmd/ob_load_data_impl.cpp b/src/sql/engine/cmd/ob_load_data_impl.cpp index 472ca6c73..89c37364d 100644 --- a/src/sql/engine/cmd/ob_load_data_impl.cpp +++ b/src/sql/engine/cmd/ob_load_data_impl.cpp @@ -2468,7 +2468,7 @@ int ObLoadDataSPImpl::ToolBox::release_resources() //release file reader if (OB_NOT_NULL(file_reader)) { - file_reader->~ObFileReader(); + ObFileReader::destroy(file_reader); file_reader = NULL; } @@ -2776,7 +2776,10 @@ int ObLoadDataSPImpl::ToolBox::init(ObExecContext &ctx, ObLoadDataStmt &load_stm file_read_param.filename_ = load_args.file_name_; file_read_param.compression_format_ = load_args.compression_format_; file_read_param.access_info_ = load_args.access_info_; - file_read_param.packet_handle_ = &ctx.get_my_session()->get_pl_query_sender()->get_packet_sender(); + file_read_param.packet_handle_ = nullptr; + if (OB_NOT_NULL(ctx.get_my_session()) && OB_NOT_NULL(ctx.get_my_session()->get_pl_query_sender())) { + file_read_param.packet_handle_ = &ctx.get_my_session()->get_pl_query_sender()->get_packet_sender(); + } file_read_param.session_ = ctx.get_my_session(); file_read_param.timeout_ts_ = THIS_WORKER.get_timeout_ts(); From 1a0f638cdf2016125cce087c4c55a147e202e17b Mon Sep 17 00:00:00 2001 From: obdev Date: Mon, 12 Aug 2024 09:47:38 +0000 Subject: [PATCH 012/249] Fix the correctness issue of tables with the same name and synonyms --- src/sql/plan_cache/ob_plan_cache_value.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/sql/plan_cache/ob_plan_cache_value.cpp b/src/sql/plan_cache/ob_plan_cache_value.cpp index 4d147cb3c..808c0a611 100644 --- a/src/sql/plan_cache/ob_plan_cache_value.cpp +++ b/src/sql/plan_cache/ob_plan_cache_value.cpp @@ -1954,13 +1954,21 @@ int ObPlanCacheValue::get_all_dep_schema(ObPlanCacheCtx &pc_ctx, tenant_id = get_tenant_id_by_object_id(stored_schema_objs_.at(i)->schema_id_); } else if (SYNONYM_SCHEMA == pcv_schema->schema_type_) { const ObSimpleSynonymSchema *synonym_schema = nullptr; + const ObSimpleTableSchemaV2 *sn_table_schema = nullptr; // table with the same name uint64_t synonym_database_id = OB_PUBLIC_SCHEMA_ID == pcv_schema->database_id_ ? database_id : pcv_schema->database_id_; - if (OB_FAIL(schema_guard.get_synonym_info(tenant_id, synonym_database_id, - pcv_schema->table_name_, synonym_schema))) { + if (OB_FAIL(schema_guard.get_simple_table_schema( + tenant_id, synonym_database_id, pcv_schema->table_name_, false, sn_table_schema))) { + LOG_WARN("failed to get table schema", K(pcv_schema->schema_id_), K(ret)); + } else if (nullptr != sn_table_schema) { + ret = OB_OLD_SCHEMA_VERSION; + LOG_INFO("a table with the same name exists. regenerate the plan", K(ret), + K(synonym_database_id), K(pcv_schema->table_name_)); + } else if (OB_FAIL(schema_guard.get_synonym_info( + tenant_id, synonym_database_id, pcv_schema->table_name_, synonym_schema))) { LOG_WARN("failed to get private synonym", K(ret)); } else if (OB_ISNULL(synonym_schema) - && OB_FAIL(schema_guard.get_synonym_info(tenant_id, OB_PUBLIC_SCHEMA_ID, + && OB_FAIL(schema_guard.get_synonym_info(tenant_id, OB_PUBLIC_SCHEMA_ID, pcv_schema->table_name_, synonym_schema))) { LOG_WARN("failed to get public synonym", K(ret)); From 267e78078763ea33acfb179249079ff3bcb5ccb3 Mon Sep 17 00:00:00 2001 From: zhangzhenyuyu Date: Mon, 12 Aug 2024 10:05:52 +0000 Subject: [PATCH 013/249] =?UTF-8?q?[FEAT=20MERGE]=20=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E6=B5=81=E5=89=AF=E6=9C=AC=E8=BF=90=E7=BB=B4=E5=91=BD=E4=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/observer/ob_rpc_processor_simple.cpp | 25 + src/observer/ob_rpc_processor_simple.h | 1 + src/observer/ob_srv_xlator_rootserver.cpp | 1 + src/observer/ob_srv_xlator_storage.cpp | 1 + src/rootserver/CMakeLists.txt | 1 + src/rootserver/ob_admin_drtask_util.cpp | 2 +- src/rootserver/ob_disaster_recovery_info.cpp | 64 +- src/rootserver/ob_disaster_recovery_info.h | 16 +- src/rootserver/ob_disaster_recovery_task.cpp | 690 +++++++--- src/rootserver/ob_disaster_recovery_task.h | 109 +- .../ob_disaster_recovery_task_mgr.cpp | 131 +- .../ob_disaster_recovery_task_mgr.h | 11 +- ..._disaster_recovery_task_table_operator.cpp | 273 ++++ ...ob_disaster_recovery_task_table_operator.h | 87 ++ ...b_disaster_recovery_task_table_updater.cpp | 29 +- .../ob_disaster_recovery_worker.cpp | 1152 ++++++++++++++++- src/rootserver/ob_disaster_recovery_worker.h | 133 ++ src/rootserver/ob_root_service.cpp | 89 ++ src/rootserver/ob_root_service.h | 2 + src/rootserver/ob_rs_rpc_processor.h | 1 + src/rootserver/ob_system_admin_util.cpp | 70 + src/rootserver/ob_system_admin_util.h | 11 + .../ob_inner_table_schema.12451_12500.cpp | 468 +++++++ .../ob_inner_table_schema.15401_15450.cpp | 434 +++++++ .../ob_inner_table_schema.21301_21350.cpp | 4 +- .../ob_inner_table_schema.21501_21550.cpp | 100 ++ .../ob_inner_table_schema.25201_25250.cpp | 2 +- .../ob_inner_table_schema.25251_25300.cpp | 50 + .../ob_inner_table_schema.501_550.cpp | 484 +++++++ .../ob_inner_table_schema.50501_50550.cpp | 135 ++ .../ob_inner_table_schema.60501_60550.cpp | 90 ++ src/share/inner_table/ob_inner_table_schema.h | 49 +- .../inner_table/ob_inner_table_schema.lob.cpp | 2 +- .../ob_inner_table_schema_constants.h | 16 + .../inner_table/ob_inner_table_schema_def.py | 253 +++- .../ob_inner_table_schema_misc.ipp | 24 +- src/share/inner_table/table_id_to_name | 9 + src/share/ob_common_rpc_proxy.h | 1 + src/share/ob_debug_sync_point.h | 5 + src/share/ob_rpc_struct.cpp | 293 +++++ src/share/ob_rpc_struct.h | 191 +++ src/share/ob_srv_rpc_proxy.h | 1 + src/share/unit/ob_unit_info.cpp | 1 + .../engine/cmd/ob_alter_system_executor.cpp | 20 + src/sql/engine/cmd/ob_alter_system_executor.h | 2 + src/sql/executor/ob_cmd_executor.cpp | 4 + .../parser/non_reserved_keywords_mysql_mode.c | 3 + src/sql/parser/sql_parser_mysql_mode.y | 82 +- .../privilege_check/ob_privilege_check.cpp | 3 +- .../resolver/cmd/ob_alter_system_resolver.cpp | 532 ++++++++ .../resolver/cmd/ob_alter_system_resolver.h | 23 + src/sql/resolver/cmd/ob_alter_system_stmt.h | 12 + src/sql/resolver/ob_resolver.cpp | 24 + src/sql/resolver/ob_stmt_type.h | 2 +- .../ob_ls_complete_migration.cpp | 16 +- .../high_availability/ob_ls_migration.cpp | 57 +- .../ob_ls_migration_handler.cpp | 213 ++- .../ob_ls_migration_handler.h | 6 + .../ob_ls_prepare_migration.cpp | 26 +- .../high_availability/ob_ls_restore.cpp | 19 +- .../high_availability/ob_storage_ha_dag.cpp | 38 + .../high_availability/ob_storage_ha_dag.h | 8 +- .../ob_storage_ha_tablet_builder.cpp | 52 +- .../ob_storage_ha_tablet_builder.h | 6 +- .../ob_tablet_group_restore.cpp | 17 +- .../r/mysql/information_schema.result | 8 + .../r/mysql/desc_sys_views_in_mysql.result | 32 + .../r/mysql/desc_sys_views_in_sys.result | 65 + .../mysql/desc_virtual_table_in_mysql.result | 31 + .../r/mysql/desc_virtual_table_in_sys.result | 31 + .../r/mysql/inner_table_overall.result | 4 + unittest/sql/parser/print_parser_tree.result | 199 +++ unittest/sql/parser/test_parser.result | 862 ++++++++++++ unittest/sql/parser/test_parser.test | 25 + 74 files changed, 7531 insertions(+), 402 deletions(-) create mode 100644 src/rootserver/ob_disaster_recovery_task_table_operator.cpp create mode 100644 src/rootserver/ob_disaster_recovery_task_table_operator.h diff --git a/src/observer/ob_rpc_processor_simple.cpp b/src/observer/ob_rpc_processor_simple.cpp index 22cf0d85d..ab56b6d19 100644 --- a/src/observer/ob_rpc_processor_simple.cpp +++ b/src/observer/ob_rpc_processor_simple.cpp @@ -141,6 +141,31 @@ int ObRpcCheckBackupSchuedulerWorkingP::process() return ret; } +int ObRpcLSCancelReplicaP::process() +{ + int ret = OB_SUCCESS; + bool is_exist = false; + uint64_t tenant_id = arg_.get_tenant_id(); + MAKE_TENANT_SWITCH_SCOPE_GUARD(guard); + if (tenant_id != MTL_ID()) { + if (OB_FAIL(guard.switch_to(tenant_id))) { + LOG_WARN("failed to switch to tenant", K(ret), K(tenant_id)); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(ObStorageHACancelDagNetUtils::cancel_task(arg_.get_ls_id(), arg_.get_task_id()))) { + LOG_WARN("failed to cancel task", K(ret), K(arg_)); + } + } + if (OB_FAIL(ret)) { + SERVER_EVENT_ADD("storage_ha", "cancel storage ha task failed", + "tenant_id", tenant_id, + "ls_id", arg_.get_ls_id().id(), + "task_id", arg_.get_task_id(), + "result", ret); + } + return ret; +} int ObRpcLSMigrateReplicaP::process() { diff --git a/src/observer/ob_rpc_processor_simple.h b/src/observer/ob_rpc_processor_simple.h index 0317bdb7a..7424f13f6 100644 --- a/src/observer/ob_rpc_processor_simple.h +++ b/src/observer/ob_rpc_processor_simple.h @@ -164,6 +164,7 @@ OB_DEFINE_PROCESSOR_S(Srv, OB_DELETE_BACKUP_LS_TASK_RES, ObRpcBackupCleanLSResP) OB_DEFINE_PROCESSOR_S(Srv, OB_BACKUP_LS_DATA_RES, ObRpcBackupLSDataResP); OB_DEFINE_PROCESSOR_S(Srv, OB_NOTIFY_ARCHIVE, ObRpcNotifyArchiveP); +OB_DEFINE_PROCESSOR_S(Srv, OB_LS_CANCEL_REPLICA_TASK, ObRpcLSCancelReplicaP); OB_DEFINE_PROCESSOR_S(Srv, OB_LS_MIGRATE_REPLICA, ObRpcLSMigrateReplicaP); OB_DEFINE_PROCESSOR_S(Srv, OB_LS_ADD_REPLICA, ObRpcLSAddReplicaP); OB_DEFINE_PROCESSOR_S(Srv, OB_LS_TYPE_TRANSFORM, ObRpcLSTypeTransformP); diff --git a/src/observer/ob_srv_xlator_rootserver.cpp b/src/observer/ob_srv_xlator_rootserver.cpp index 8a1fe117e..e21e510e8 100644 --- a/src/observer/ob_srv_xlator_rootserver.cpp +++ b/src/observer/ob_srv_xlator_rootserver.cpp @@ -230,6 +230,7 @@ void oceanbase::observer::init_srv_xlator_for_rootserver(ObSrvRpcXlator *xlator) RPC_PROCESSOR(rootserver::ObRpcAdminReloadZoneP, *gctx_.root_service_); RPC_PROCESSOR(rootserver::ObRpcAdminClearMergeErrorP, *gctx_.root_service_); RPC_PROCESSOR(rootserver::ObRpcAdminMigrateUnitP, *gctx_.root_service_); + RPC_PROCESSOR(rootserver::ObRpcAdminAlterLSReplicaP, *gctx_.root_service_); RPC_PROCESSOR(rootserver::ObRpcAdminUpgradeVirtualSchemaP, *gctx_.root_service_); RPC_PROCESSOR(rootserver::ObRpcRunJobP, *gctx_.root_service_); RPC_PROCESSOR(rootserver::ObRpcAdminRefreshIOCalibrationP, *gctx_.root_service_); diff --git a/src/observer/ob_srv_xlator_storage.cpp b/src/observer/ob_srv_xlator_storage.cpp index cef197777..a975ed8f7 100644 --- a/src/observer/ob_srv_xlator_storage.cpp +++ b/src/observer/ob_srv_xlator_storage.cpp @@ -108,6 +108,7 @@ void oceanbase::observer::init_srv_xlator_for_storage(ObSrvRpcXlator *xlator) { RPC_PROCESSOR(ObRpcRemoteWriteDDLRedoLogP, gctx_); RPC_PROCESSOR(ObRpcRemoteWriteDDLCommitLogP, gctx_); RPC_PROCESSOR(ObRpcRemoteWriteDDLIncCommitLogP, gctx_); + RPC_PROCESSOR(ObRpcLSCancelReplicaP, gctx_); RPC_PROCESSOR(ObRpcLSMigrateReplicaP, gctx_); RPC_PROCESSOR(ObRpcLSAddReplicaP, gctx_); RPC_PROCESSOR(ObRpcLSTypeTransformP, gctx_); diff --git a/src/rootserver/CMakeLists.txt b/src/rootserver/CMakeLists.txt index 60e0cac88..4aa3609ae 100644 --- a/src/rootserver/CMakeLists.txt +++ b/src/rootserver/CMakeLists.txt @@ -27,6 +27,7 @@ ob_set_subtarget(ob_rootserver common ob_bootstrap.cpp ob_admin_drtask_util.cpp ob_disaster_recovery_task_table_updater.cpp + ob_disaster_recovery_task_table_operator.cpp ob_balance_group_ls_stat_operator.cpp ob_disaster_recovery_info.cpp ob_disaster_recovery_worker.cpp diff --git a/src/rootserver/ob_admin_drtask_util.cpp b/src/rootserver/ob_admin_drtask_util.cpp index ac1e6e292..fdea3695b 100644 --- a/src/rootserver/ob_admin_drtask_util.cpp +++ b/src/rootserver/ob_admin_drtask_util.cpp @@ -517,7 +517,7 @@ int ObAdminDRTaskUtil::execute_remove_nonpaxos_task_( } if (OB_SUCC(ret)) { // rpc is send, log task start, task finish will be recorded later - ROOTSERVICE_EVENT_ADD("disaster_recovery", drtasklog::START_REMOVE_LS_PAXOS_REPLICA_STR, + ROOTSERVICE_EVENT_ADD("disaster_recovery", drtasklog::START_REMOVE_LS_NON_PAXOS_REPLICA_STR, "tenant_id", remove_non_paxos_arg.tenant_id_, "ls_id", remove_non_paxos_arg.ls_id_.id(), "task_id", ObCurTraceId::get_trace_id_str(), diff --git a/src/rootserver/ob_disaster_recovery_info.cpp b/src/rootserver/ob_disaster_recovery_info.cpp index faefa1b71..2fa660c24 100644 --- a/src/rootserver/ob_disaster_recovery_info.cpp +++ b/src/rootserver/ob_disaster_recovery_info.cpp @@ -547,7 +547,7 @@ int DRLSInfo::get_leader( int DRLSInfo::get_leader_and_member_list( common::ObAddr &leader_addr, common::ObMemberList &member_list, - GlobalLearnerList &learner_list) + GlobalLearnerList &learner_list) const { int ret = OB_SUCCESS; const ObLSReplica *leader_replica = nullptr; @@ -606,4 +606,66 @@ int DRLSInfo::get_default_data_source( leader_replica->get_memstore_percent()); } return ret; +} + +int DRLSInfo::get_member_by_server( + const common::ObAddr& server_addr, + ObMember &member) const +{ + int ret = OB_SUCCESS; + member.reset(); + common::ObAddr leader_addr; // not used + GlobalLearnerList learner_list; + common::ObMemberList member_list; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (OB_UNLIKELY(!server_addr.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(server_addr)); + } else if (OB_FAIL(get_leader_and_member_list(leader_addr, member_list, learner_list))) { + LOG_WARN("fail to get leader and member list", KR(ret), K(server_addr)); + } else if (member_list.contains(server_addr)) { + if (OB_FAIL(member_list.get_member_by_addr(server_addr, member))) { + LOG_WARN("fail to get member by addr", KR(ret), K(server_addr), K(member_list)); + } + } else if (learner_list.contains(server_addr)) { + if (OB_FAIL(learner_list.get_learner_by_addr(server_addr, member))) { + LOG_WARN("fail to get member by addr", KR(ret), K(server_addr), K(learner_list)); + } + } else { + ret = OB_ENTRY_NOT_EXIST; + LOG_WARN("fail to find server in leader member list and learner list", + KR(ret), K(server_addr), K(learner_list), K(member_list)); + } + return ret; +} + +int DRLSInfo::check_replica_exist_and_get_ls_replica( + const common::ObAddr& server_addr, + share::ObLSReplica& ls_replica) const +{ + int ret = OB_SUCCESS; + const share::ObLSReplica *ls_replica_ptr = nullptr; + ls_replica.reset(); + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (OB_UNLIKELY(!server_addr.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(server_addr)); + } else if (OB_FAIL(inner_ls_info_.find(server_addr, ls_replica_ptr))) { + if (OB_ENTRY_NOT_EXIST != ret) { + LOG_WARN("fail to find replica by server", KR(ret), K(server_addr), K(inner_ls_info_)); + } else { + LOG_INFO("dose not have replica", KR(ret), K(server_addr), K(inner_ls_info_)); + ret = OB_SUCCESS; + } + } else if (OB_ISNULL(ls_replica_ptr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls_replica_ptr is null", KR(ret), K(server_addr), KP(ls_replica_ptr), K(inner_ls_info_)); + } else if (OB_FAIL(ls_replica.assign(*ls_replica_ptr))) { + LOG_WARN("ls_replica assign failed", KR(ret), K(server_addr), KP(ls_replica_ptr)); + } + return ret; } \ No newline at end of file diff --git a/src/rootserver/ob_disaster_recovery_info.h b/src/rootserver/ob_disaster_recovery_info.h index 7340f4cca..9d3cfaba1 100644 --- a/src/rootserver/ob_disaster_recovery_info.h +++ b/src/rootserver/ob_disaster_recovery_info.h @@ -204,7 +204,7 @@ public: int get_leader_and_member_list( common::ObAddr &leader_addr, common::ObMemberList &member_list, - GlobalLearnerList &learner_list); + GlobalLearnerList &learner_list) const; // get data_source from leader replcia // @param [out] data_source, leader replica @@ -212,6 +212,20 @@ public: int get_default_data_source( ObReplicaMember &data_source, int64_t &data_size) const; + + // get member by server address in leader's learner list and member list + // @param [in] server_addr, which server the member in + // @param [out] member, target member + int get_member_by_server( + const common::ObAddr& server_addr, + ObMember &member) const; + + // check and get if there is a replica on the target server + // @param [in] server_addr, which server the replica in + // @param [out] ls_replica, target replic + int check_replica_exist_and_get_ls_replica( + const common::ObAddr& server_addr, + share::ObLSReplica& ls_replica) const; private: int construct_filtered_ls_info_to_use_( const share::ObLSInfo &input_ls_info, diff --git a/src/rootserver/ob_disaster_recovery_task.cpp b/src/rootserver/ob_disaster_recovery_task.cpp index b39b58e25..1ba8c83b3 100644 --- a/src/rootserver/ob_disaster_recovery_task.cpp +++ b/src/rootserver/ob_disaster_recovery_task.cpp @@ -43,6 +43,96 @@ using namespace share::schema; namespace rootserver { +OB_SERIALIZE_MEMBER( + ObDRLSReplicaTaskStatus, + status_); + +static const char* dr_ls_replica_task_status_strs[] = { + "INPROGRESS", + "COMPLETED", + "FAILED", + "CANCELED", +}; + +const char* ObDRLSReplicaTaskStatus::get_status_str() const { + STATIC_ASSERT(ARRAYSIZEOF(dr_ls_replica_task_status_strs) == (int64_t)MAX_STATUS, + "dr_ls_replica_task_status_strs string array size mismatch enum DRLSReplicaTaskStatus count"); + const char *str = NULL; + if (status_ >= INPROGRESS && status_ < MAX_STATUS) { + str = dr_ls_replica_task_status_strs[status_]; + } else { + LOG_WARN_RET(OB_ERR_UNEXPECTED, "invalid DRLSReplicaTaskStatus", K_(status)); + } + return str; +} + +int64_t ObDRLSReplicaTaskStatus::to_string(char *buf, const int64_t buf_len) const +{ + int64_t pos = 0; + J_OBJ_START(); + J_KV(K_(status), "status", get_status_str()); + J_OBJ_END(); + return pos; +} + +void ObDRLSReplicaTaskStatus::assign(const ObDRLSReplicaTaskStatus &other) +{ + if (this != &other) { + status_ = other.status_; + } +} + +int ObDRLSReplicaTaskStatus::parse_from_string(const ObString &status) +{ + int ret = OB_SUCCESS; + bool found = false; + STATIC_ASSERT(ARRAYSIZEOF(dr_ls_replica_task_status_strs) == (int64_t)MAX_STATUS, + "dr_ls_replica_task_status_strs string array size mismatch enum DRLSReplicaTaskStatus count"); + for (int64_t i = 0; i < ARRAYSIZEOF(dr_ls_replica_task_status_strs) && !found; i++) { + if (0 == status.case_compare(dr_ls_replica_task_status_strs[i])) { + status_ = static_cast(i); + found = true; + break; + } + } + if (!found) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("fail to parse status from string", KR(ret), K(status), K_(status)); + } + return ret; +} + +int build_execute_result( + const int ret_code, + const ObDRTaskRetComment &ret_comment, + const int64_t start_time, + ObSqlString &execute_result) +{ + int ret = OB_SUCCESS; + const int64_t now = ObTimeUtility::current_time(); + const int64_t elapsed = now - start_time; + execute_result.reset(); + if (OB_UNLIKELY(ObDRTaskRetComment::MAX == ret_comment || start_time <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(ret_comment), K(start_time)); + } else if (OB_FAIL(execute_result.append_fmt( + "ret:%d, %s; elapsed:%ld;", ret_code, common::ob_error_name(ret_code), elapsed))) { + LOG_WARN("fail to append to execute_result", KR(ret), K(ret_code), K(elapsed)); + } else if (OB_SUCCESS != ret_code + && OB_FAIL(execute_result.append_fmt(" comment:%s;", + ob_disaster_recovery_task_ret_comment_strs(ret_comment)))) { + LOG_WARN("fail to append ret comment to execute result", KR(ret), K(ret_comment)); + } + return ret; +} + +bool is_manual_dr_task_data_version_match(uint64_t tenant_data_version) +{ + return ((tenant_data_version >= DATA_VERSION_4_3_3_0) + || (tenant_data_version >= MOCK_DATA_VERSION_4_2_3_0 && tenant_data_version < DATA_VERSION_4_3_0_0) + || (tenant_data_version >= MOCK_DATA_VERSION_4_2_1_8 && tenant_data_version < DATA_VERSION_4_2_2_0)); +} + int ObDstReplica::assign( const uint64_t unit_id, const uint64_t unit_group_id, @@ -275,34 +365,82 @@ uint64_t ObDRTaskKey::inner_hash() const return hash_val; } +int ObDRTask::fill_dml_splicer( + share::ObDMLSqlSplicer &dml_splicer) const +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid task", KR(ret)); + } else if (OB_FAIL(dml_splicer.add_pk_column("tenant_id", tenant_id_))) { + LOG_WARN("add column failed", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(dml_splicer.add_pk_column("ls_id", ls_id_.id()))) { + LOG_WARN("add column failed", KR(ret), K(ls_id_)); + } else if (OB_FAIL(dml_splicer.add_pk_column("task_id", task_id_))) { + LOG_WARN("add column failed", KR(ret), K(task_id_)); + } else if (OB_FAIL(dml_splicer.add_column("task_status", ObDRLSReplicaTaskStatus(ObDRLSReplicaTaskStatus::INPROGRESS).get_status_str()))) { + // it will only be called when the task is started + LOG_WARN("add column failed", KR(ret)); + } else if (OB_FAIL(dml_splicer.add_column("priority", static_cast(priority_)))) { + LOG_WARN("add column failed", KR(ret), K(priority_)); + } else if (OB_FAIL(dml_splicer.add_time_column("generate_time", generate_time_))) { + LOG_WARN("add column failed", KR(ret), K(generate_time_)); + } else if (OB_FAIL(dml_splicer.add_time_column("schedule_time", schedule_time_))) { + LOG_WARN("add column failed", KR(ret), K(schedule_time_)); + } else if (OB_FAIL(dml_splicer.add_column("comment", comment_.ptr()))) { + LOG_WARN("add column failed", KR(ret), K(comment_)); + } + return ret; +} + +int ObDRTask::fill_dml_splicer_for_new_column( + share::ObDMLSqlSplicer &dml_splicer, + const common::ObAddr &force_data_src) const +{ + // force_data_src may be invalid + int ret = OB_SUCCESS; + uint64_t tenant_data_version = 0; + char force_data_source_ip[OB_MAX_SERVER_ADDR_SIZE] = ""; + if (OB_UNLIKELY(!is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid task", KR(ret)); + } else if (ObDRTaskType::MAX_TYPE == get_disaster_recovery_task_type()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected task type", KR(ret)); + } else if (OB_FAIL(GET_MIN_DATA_VERSION(gen_meta_tenant_id(tenant_id_), tenant_data_version))) { + LOG_WARN("fail to get min data version", KR(ret), K(tenant_id_)); + } else if (!is_manual_dr_task_data_version_match(tenant_data_version) + && (is_manual_task() || force_data_src.is_valid())) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("manual operation is not suppported when tenant's data version is not match", + KR(ret), K(tenant_data_version), K(is_manual_task()), K(force_data_src)); + } else if (is_manual_dr_task_data_version_match(tenant_data_version)) { + if (!is_manual_task() && force_data_src.is_valid()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("task invoke and data_source is not match", KR(ret), K(is_manual_task()), K(force_data_src)); + } else if (OB_FAIL(dml_splicer.add_column("is_manual", is_manual_task()))) { + LOG_WARN("add column failed", KR(ret), K(is_manual_task())); + } else if (ObDRTaskType::LS_ADD_REPLICA == get_disaster_recovery_task_type() + || ObDRTaskType::LS_MIGRATE_REPLICA == get_disaster_recovery_task_type()) { + if (false == force_data_src.ip_to_string(force_data_source_ip, sizeof(force_data_source_ip))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("convert data_src_server ip to string failed", KR(ret), K(force_data_src)); + } else if (OB_FAIL(dml_splicer.add_column("data_source_svr_ip", force_data_source_ip))) { + LOG_WARN("add column failed", KR(ret), K(force_data_source_ip)); + } else if (OB_FAIL(dml_splicer.add_column("data_source_svr_port", force_data_src.get_port()))) { + LOG_WARN("add column failed", KR(ret), K(force_data_src)); + } + } + } + return ret; +} + bool ObDRTask::is_already_timeout() const { int64_t now = ObTimeUtility::current_time(); return schedule_time_ + GCONF.balancer_task_timeout < now; } -int ObDRTask::build_execute_result( - const int ret_code, - const ObDRTaskRetComment &ret_comment, - ObSqlString &execute_result) const -{ - int ret = OB_SUCCESS; - const int64_t now = ObTimeUtility::current_time(); - const int64_t elapsed = (get_execute_time() > 0) - ? (now - get_execute_time()) - : (now - get_schedule_time()); - execute_result.reset(); - if (OB_FAIL(execute_result.append_fmt( - "ret:%d, %s; elapsed:%ld;", ret_code, common::ob_error_name(ret_code), elapsed))) { - LOG_WARN("fail to append to execute_result", KR(ret), K(ret_code), K(elapsed)); - } else if (OB_SUCCESS != ret_code - && OB_FAIL(execute_result.append_fmt(" ret_comment:%s;", - ob_disaster_recovery_task_ret_comment_strs(ret_comment)))) { - LOG_WARN("fail to append ret comment to execute result", KR(ret), K(ret_comment)); - } - return ret; -} - int ObDRTask::set_task_key( const ObDRTaskKey &task_key) { @@ -485,7 +623,9 @@ int ObMigrateLSReplicaTask::log_execute_result( { int ret = OB_SUCCESS; ObSqlString execute_result; - if (OB_FAIL(build_execute_result(ret_code, ret_comment, execute_result))) { + int64_t start_time = execute_time_ > 0 ? execute_time_ : schedule_time_; + // when rs change leader, execute_time_ is 0, only schedule_time_ is valid + if (OB_FAIL(build_execute_result(ret_code, ret_comment, start_time, execute_result))) { LOG_WARN("fail to build execute result", KR(ret), K(ret_code), K(ret_comment)); } else { ROOTSERVICE_EVENT_ADD("disaster_recovery", get_log_finish_str(), @@ -562,8 +702,6 @@ int ObMigrateLSReplicaTask::fill_dml_splicer( char dest_ip[OB_MAX_SERVER_ADDR_SIZE] = ""; char task_id[OB_TRACE_STAT_BUFFER_SIZE] = ""; char task_type[MAX_DISASTER_RECOVERY_TASK_TYPE_LENGTH] = "MIGRATE REPLICA"; - int64_t transmit_data_size = 0; - if (OB_UNLIKELY(!is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid task", KR(ret)); @@ -573,30 +711,22 @@ int ObMigrateLSReplicaTask::fill_dml_splicer( } else if (false == get_dst_server().ip_to_string(dest_ip, sizeof(dest_ip))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("convert dest_server ip to string failed", KR(ret), "dest_server", get_dst_server()); - } else if (OB_FAIL(get_execute_transmit_size(transmit_data_size))) { - LOG_WARN("fail to get transmit_data_size", KR(ret), K(transmit_data_size)); - } else { - if (OB_FAIL(dml_splicer.add_pk_column("tenant_id", tenant_id_)) - || OB_FAIL(dml_splicer.add_pk_column("ls_id", ls_id_.id())) - || OB_FAIL(dml_splicer.add_pk_column("task_type", task_type)) - || OB_FAIL(dml_splicer.add_pk_column("task_id", task_id_)) - || OB_FAIL(dml_splicer.add_column("task_status", TASK_STATUS)) - || OB_FAIL(dml_splicer.add_column("priority", static_cast(ObDRTaskPriority::HIGH_PRI))) - || OB_FAIL(dml_splicer.add_column("target_replica_svr_ip", dest_ip)) - || OB_FAIL(dml_splicer.add_column("target_replica_svr_port", get_dst_server().get_port())) - || OB_FAIL(dml_splicer.add_column("target_paxos_replica_number", get_paxos_replica_number())) - || OB_FAIL(dml_splicer.add_column("target_replica_type", ob_replica_type_strs(get_dst_replica().get_member().get_replica_type()))) - || OB_FAIL(dml_splicer.add_column("source_replica_svr_ip", src_ip)) - || OB_FAIL(dml_splicer.add_column("source_replica_svr_port", get_src_member().get_server().get_port())) - || OB_FAIL(dml_splicer.add_column("source_paxos_replica_number", get_paxos_replica_number())) - || OB_FAIL(dml_splicer.add_column("source_replica_type", ob_replica_type_strs(get_src_member().get_replica_type()))) - || OB_FAIL(dml_splicer.add_column("task_exec_svr_ip", dest_ip)) - || OB_FAIL(dml_splicer.add_column("task_exec_svr_port", get_dst_server().get_port())) - || OB_FAIL(dml_splicer.add_time_column("generate_time", generate_time_)) - || OB_FAIL(dml_splicer.add_time_column("schedule_time", schedule_time_)) - || OB_FAIL(dml_splicer.add_column("comment", comment_.ptr()))) { - LOG_WARN("add column failed", KR(ret)); - } + } else if (OB_FAIL(ObDRTask::fill_dml_splicer(dml_splicer))) { + LOG_WARN("ObDRTask fill dml splicer failed", KR(ret)); + } else if (OB_FAIL(dml_splicer.add_pk_column("task_type", task_type)) + || OB_FAIL(dml_splicer.add_column("target_replica_svr_ip", dest_ip)) + || OB_FAIL(dml_splicer.add_column("target_replica_svr_port", get_dst_server().get_port())) + || OB_FAIL(dml_splicer.add_column("target_paxos_replica_number", get_paxos_replica_number())) + || OB_FAIL(dml_splicer.add_column("target_replica_type", ob_replica_type_strs(get_dst_replica().get_member().get_replica_type()))) + || OB_FAIL(dml_splicer.add_column("source_replica_svr_ip", src_ip)) + || OB_FAIL(dml_splicer.add_column("source_replica_svr_port", get_src_member().get_server().get_port())) + || OB_FAIL(dml_splicer.add_column("source_paxos_replica_number", get_paxos_replica_number())) + || OB_FAIL(dml_splicer.add_column("source_replica_type", ob_replica_type_strs(get_src_member().get_replica_type()))) + || OB_FAIL(dml_splicer.add_column("task_exec_svr_ip", dest_ip)) + || OB_FAIL(dml_splicer.add_column("task_exec_svr_port", get_dst_server().get_port()))) { + LOG_WARN("add column failed", KR(ret)); + } else if (OB_FAIL(fill_dml_splicer_for_new_column(dml_splicer, get_force_data_src_member().get_server()))) { + LOG_WARN("fill dml_splicer for new column failed", KR(ret)); } return ret; } @@ -704,6 +834,7 @@ int ObMigrateLSReplicaTask::clone( } else { my_task->set_src_member(get_src_member()); my_task->set_data_src_member(get_data_src_member()); + my_task->set_force_data_src_member(get_force_data_src_member()); my_task->set_paxos_replica_number(get_paxos_replica_number()); output_task = my_task; } @@ -775,6 +906,52 @@ int ObMigrateLSReplicaTask::build( return ret; } +int ObMigrateLSReplicaTask::simple_build( + const uint64_t tenant_id, + const share::ObLSID &ls_id, + const share::ObTaskId &task_id, + const ObDstReplica &dst_replica, + const common::ObReplicaMember &src_member, + const common::ObReplicaMember &data_src_member, + const common::ObReplicaMember &force_data_src_member, + const int64_t paxos_replica_number) +{ + int ret = OB_SUCCESS; + ObDRTaskKey task_key; + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) + || !ls_id.is_valid_with_tenant(tenant_id) + || !task_id.is_valid() + || !dst_replica.is_valid() + || !src_member.is_valid() + || paxos_replica_number <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(ls_id), K(task_id), K(dst_replica), + K(src_member), K(paxos_replica_number)); + } else if (OB_FAIL(task_key.init(tenant_id, ls_id.id(), 0, 0, + ObDRTaskKeyType::FORMAL_DR_KEY))) { + LOG_WARN("fail to init task key", KR(ret), K(tenant_id), K(ls_id)); + } else if (OB_FAIL(ObDRTask::build( + task_key, + tenant_id, + ls_id, + task_id, + 0,/*schedule_time_us*/ 0,/*generate_time_us*/ + GCONF.cluster_id, 0,/*transmit_data_size*/ + obrpc::ObAdminClearDRTaskArg::TaskType::MANUAL, + ObDRTaskPriority::HIGH_PRI, + ObString(drtask::ALTER_SYSTEM_COMMAND_MIGRATE_REPLICA)))) { + LOG_WARN("fail to build ObDRTask", KR(ret), K(task_key), K(tenant_id), K(ls_id), K(task_id)); + } else if (OB_FAIL(dst_replica_.assign(dst_replica))) { + LOG_WARN("fail to assign dst replica", KR(ret), K(dst_replica)); + } else { + src_member_ = src_member; + data_src_member_ = data_src_member; + force_data_src_member_ = force_data_src_member; + paxos_replica_number_ = paxos_replica_number; + } + return ret; +} + int ObMigrateLSReplicaTask::build_task_from_sql_result( const sqlclient::ObMySQLResult &res) { @@ -792,6 +969,9 @@ int ObMigrateLSReplicaTask::build_task_from_sql_result( int64_t schedule_time_us = 0; int64_t generate_time_us = 0; common::ObString comment; + int64_t data_source_port = 0; + common::ObString data_source_ip; + bool is_manual = false; //STEP1_0: read certain members from sql result EXTRACT_INT_FIELD_MYSQL(res, "tenant_id", tenant_id, uint64_t); { @@ -811,6 +991,11 @@ int ObMigrateLSReplicaTask::build_task_from_sql_result( (void)GET_COL_IGNORE_NULL(res.get_int, "target_replica_svr_port", dest_port); (void)GET_COL_IGNORE_NULL(res.get_int, "source_paxos_replica_number", src_paxos_replica_number); (void)GET_COL_IGNORE_NULL(res.get_varchar, "comment", comment); + EXTRACT_INT_FIELD_MYSQL_WITH_DEFAULT_VALUE(res, "data_source_svr_port", data_source_port, + int64_t, true/*skip null error*/, true/*skip column error*/, 0); + EXTRACT_VARCHAR_FIELD_MYSQL_WITH_DEFAULT_VALUE(res, "data_source_svr_ip", data_source_ip, + true/*skip null error*/, true/*skip column error*/, "0.0.0.0"); + EXTRACT_BOOL_FIELD_MYSQL_SKIP_RET(res, "is_manual", is_manual); //STEP2_0: make necessary members to build a task ObDRTaskKey task_key; common::ObAddr src_server; @@ -821,6 +1006,7 @@ int ObMigrateLSReplicaTask::build_task_from_sql_result( share::ObTaskId task_id_to_set; ObSqlString comment_to_set; ObSqlString task_id_sqlstring_format; + common::ObAddr force_data_source; if (OB_FAIL(ret)) { } else if (OB_FAIL(comment_to_set.assign(comment))) { @@ -842,6 +1028,9 @@ int ObMigrateLSReplicaTask::build_task_from_sql_result( } else if (false == dest_server.set_ip_addr(dest_ip, static_cast(dest_port))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid server address", K(dest_ip), K(dest_port)); + } else if (false == force_data_source.set_ip_addr(data_source_ip, static_cast(data_source_port))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid server address", K(data_source_ip), K(data_source_port)); } else if (OB_FAIL(dst_replica.assign( 0/*unit id*/, 0/*unit group id*/, @@ -869,13 +1058,13 @@ int ObMigrateLSReplicaTask::build_task_from_sql_result( generate_time_us, //(in used) GCONF.cluster_id, //(not used)cluster_id transmit_data_size, //(not used) - obrpc::ObAdminClearDRTaskArg::TaskType::AUTO,//(not used)invoked_source + is_manual ? obrpc::ObAdminClearDRTaskArg::TaskType::MANUAL : obrpc::ObAdminClearDRTaskArg::TaskType::AUTO,//(not used)invoked_source priority_to_set, //(not used) comment_to_set.ptr(), //comment dst_replica, //(in used)dest_server ObReplicaMember(src_server, 0), //(in used)src_server ObReplicaMember(src_server, 0), //(not used)data_src_member - ObReplicaMember(src_server, 0), //(not used)data_src_member + ObReplicaMember(force_data_source, 0), //(not used)force_data_source src_paxos_replica_number))) { //(not used) LOG_WARN("fail to build a ObMigrateLSReplicaTask", KR(ret)); } else { @@ -939,7 +1128,8 @@ int ObAddLSReplicaTask::log_execute_result( { int ret = OB_SUCCESS; ObSqlString execute_result; - if (OB_FAIL(build_execute_result(ret_code, ret_comment, execute_result))) { + int64_t start_time = execute_time_ > 0 ? execute_time_ : schedule_time_; + if (OB_FAIL(build_execute_result(ret_code, ret_comment, start_time, execute_result))) { LOG_WARN("fail to build execute result", KR(ret), K(ret_code), K(ret_comment)); } else { ROOTSERVICE_EVENT_ADD("disaster_recovery", get_log_finish_str(), @@ -1019,8 +1209,6 @@ int ObAddLSReplicaTask::fill_dml_splicer( char dest_ip[OB_MAX_SERVER_ADDR_SIZE] = ""; char task_id[OB_TRACE_STAT_BUFFER_SIZE] = ""; char task_type[MAX_DISASTER_RECOVERY_TASK_TYPE_LENGTH] = "ADD REPLICA"; - int64_t transmit_data_size = 0; - if (OB_UNLIKELY(!is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid task", KR(ret)); @@ -1030,30 +1218,22 @@ int ObAddLSReplicaTask::fill_dml_splicer( } else if (false == get_dst_server().ip_to_string(dest_ip, sizeof(dest_ip))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("convert dest_server ip to string failed", KR(ret), "dest_server", get_dst_server()); - } else if (OB_FAIL(get_execute_transmit_size(transmit_data_size))) { - LOG_WARN("fail to get transmit_data_size", KR(ret), K(transmit_data_size)); - } else { - if (OB_FAIL(dml_splicer.add_pk_column("tenant_id", tenant_id_)) - || OB_FAIL(dml_splicer.add_pk_column("ls_id", ls_id_.id())) - || OB_FAIL(dml_splicer.add_pk_column("task_type", task_type)) - || OB_FAIL(dml_splicer.add_pk_column("task_id", task_id_)) - || OB_FAIL(dml_splicer.add_column("task_status", TASK_STATUS)) - || OB_FAIL(dml_splicer.add_column("priority", static_cast(ObDRTaskPriority::HIGH_PRI))) - || OB_FAIL(dml_splicer.add_column("target_replica_svr_ip", dest_ip)) - || OB_FAIL(dml_splicer.add_column("target_replica_svr_port", get_dst_server().get_port())) - || OB_FAIL(dml_splicer.add_column("target_paxos_replica_number", get_paxos_replica_number())) - || OB_FAIL(dml_splicer.add_column("target_replica_type", ob_replica_type_strs(get_dst_replica().get_member().get_replica_type()))) - || OB_FAIL(dml_splicer.add_column("source_replica_svr_ip", src_ip)) - || OB_FAIL(dml_splicer.add_column("source_replica_svr_port", get_data_src_member().get_server().get_port())) - || OB_FAIL(dml_splicer.add_column("source_paxos_replica_number", get_orig_paxos_replica_number())) - || OB_FAIL(dml_splicer.add_column("source_replica_type", ob_replica_type_strs(get_dst_replica().get_member().get_replica_type()))) - || OB_FAIL(dml_splicer.add_column("task_exec_svr_ip", dest_ip)) - || OB_FAIL(dml_splicer.add_column("task_exec_svr_port", get_dst_server().get_port())) - || OB_FAIL(dml_splicer.add_time_column("generate_time", generate_time_)) - || OB_FAIL(dml_splicer.add_time_column("schedule_time", schedule_time_)) - || OB_FAIL(dml_splicer.add_column("comment", comment_.ptr()))) { - LOG_WARN("add column failed", KR(ret)); - } + } else if (OB_FAIL(ObDRTask::fill_dml_splicer(dml_splicer))) { + LOG_WARN("ObDRTask fill dml splicer failed", KR(ret)); + } else if (OB_FAIL(dml_splicer.add_pk_column("task_type", task_type)) + || OB_FAIL(dml_splicer.add_column("target_replica_svr_ip", dest_ip)) + || OB_FAIL(dml_splicer.add_column("target_replica_svr_port", get_dst_server().get_port())) + || OB_FAIL(dml_splicer.add_column("target_paxos_replica_number", get_paxos_replica_number())) + || OB_FAIL(dml_splicer.add_column("target_replica_type", ob_replica_type_strs(get_dst_replica().get_member().get_replica_type()))) + || OB_FAIL(dml_splicer.add_column("source_replica_svr_ip", src_ip)) + || OB_FAIL(dml_splicer.add_column("source_replica_svr_port", get_data_src_member().get_server().get_port())) + || OB_FAIL(dml_splicer.add_column("source_paxos_replica_number", get_orig_paxos_replica_number())) + || OB_FAIL(dml_splicer.add_column("source_replica_type", ob_replica_type_strs(get_dst_replica().get_member().get_replica_type()))) + || OB_FAIL(dml_splicer.add_column("task_exec_svr_ip", dest_ip)) + || OB_FAIL(dml_splicer.add_column("task_exec_svr_port", get_dst_server().get_port()))) { + LOG_WARN("add column failed", KR(ret)); + } else if (OB_FAIL(fill_dml_splicer_for_new_column(dml_splicer, get_force_data_src_member().get_server()))) { + LOG_WARN("fill dml_splicer for new column failed", KR(ret)); } return ret; } @@ -1125,7 +1305,8 @@ int ObAddLSReplicaTask::check_paxos_member( LOG_WARN("get invalid replica", K(ret), K(ls_info)); } else if (r->get_server() == dst_replica_.get_server()) { // already check in check online - } else if (r->get_zone() == dst_zone + } else if ((!is_manual_task() && r->get_zone() == dst_zone) + // manual operation allowed mutiple replica in same zone && r->is_in_service() && ObReplicaTypeCheck::is_paxos_replica_V2(r->get_replica_type()) && ObReplicaTypeCheck::is_paxos_replica_V2(dst_replica_.get_replica_type())) { @@ -1165,6 +1346,7 @@ int ObAddLSReplicaTask::clone( LOG_WARN("fail to set dst replica", KR(ret)); } else { my_task->set_data_src_member(get_data_src_member()); + my_task->set_force_data_src_member(get_force_data_src_member()); my_task->set_orig_paxos_replica_number(get_orig_paxos_replica_number()); my_task->set_paxos_replica_number(get_paxos_replica_number()); output_task = my_task; @@ -1235,6 +1417,53 @@ int ObAddLSReplicaTask::build( return ret; } +int ObAddLSReplicaTask::simple_build( + const uint64_t tenant_id, + const share::ObLSID &ls_id, + const share::ObTaskId &task_id, + const ObDstReplica &dst_replica, + const common::ObReplicaMember &data_src_member, + const common::ObReplicaMember &force_data_src_member, + const int64_t orig_paxos_replica_number, + const int64_t paxos_replica_number) +{ + int ret = OB_SUCCESS; + ObDRTaskKey task_key; + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) + || !ls_id.is_valid_with_tenant(tenant_id) + || !task_id.is_valid() + || !dst_replica.is_valid() + || paxos_replica_number <= 0 + || orig_paxos_replica_number <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(ls_id), K(task_id), K(dst_replica), + K(orig_paxos_replica_number), K(paxos_replica_number)); + } else if (OB_FAIL(task_key.init(tenant_id, ls_id.id(), 0, 0, + ObDRTaskKeyType::FORMAL_DR_KEY))) { + LOG_WARN("fail to init task key", KR(ret), K(tenant_id), K(ls_id)); + } else if (OB_FAIL(ObDRTask::build( + task_key, + tenant_id, + ls_id, + task_id, + 0,/*schedule_time_us*/ 0,/*generate_time_us*/ + GCONF.cluster_id, 0,/*transmit_data_size*/ + obrpc::ObAdminClearDRTaskArg::TaskType::MANUAL, + ObDRTaskPriority::HIGH_PRI, + ObString(drtask::ALTER_SYSTEM_COMMAND_ADD_REPLICA)))) { + LOG_WARN("fail to build ObDRTask", KR(ret), K(task_key), K(tenant_id), K(ls_id), K(task_id)); + } else if (OB_FAIL(dst_replica_.assign(dst_replica))) { + LOG_WARN("fail to assign dst replica", KR(ret), K(dst_replica)); + } else { + data_src_member_ = data_src_member; + force_data_src_member_ = force_data_src_member; + orig_paxos_replica_number_ = orig_paxos_replica_number; + paxos_replica_number_ = paxos_replica_number; + } + return ret; +} + + int ObAddLSReplicaTask::build_task_from_sql_result( const sqlclient::ObMySQLResult &res) { @@ -1253,6 +1482,9 @@ int ObAddLSReplicaTask::build_task_from_sql_result( int64_t schedule_time_us = 0; int64_t generate_time_us = 0; common::ObString comment; + int64_t data_source_port = 0; + common::ObString data_source_ip; + bool is_manual = false; //STEP1_0: read certain members from sql result EXTRACT_INT_FIELD_MYSQL(res, "tenant_id", tenant_id, uint64_t); { @@ -1273,6 +1505,11 @@ int ObAddLSReplicaTask::build_task_from_sql_result( (void)GET_COL_IGNORE_NULL(res.get_int, "source_paxos_replica_number", src_paxos_replica_number); (void)GET_COL_IGNORE_NULL(res.get_int, "target_paxos_replica_number", dest_paxos_replica_number); (void)GET_COL_IGNORE_NULL(res.get_varchar, "comment", comment); + EXTRACT_INT_FIELD_MYSQL_WITH_DEFAULT_VALUE(res, "data_source_svr_port", data_source_port, + int64_t, true/*skip null error*/, true/*skip column error*/, 0); + EXTRACT_VARCHAR_FIELD_MYSQL_WITH_DEFAULT_VALUE(res, "data_source_svr_ip", data_source_ip, + true/*skip null error*/, true/*skip column error*/, "0.0.0.0"); + EXTRACT_BOOL_FIELD_MYSQL_SKIP_RET(res, "is_manual", is_manual); //STEP2_0: make necessary members to build a task ObDRTaskKey task_key; common::ObAddr src_server; @@ -1283,6 +1520,7 @@ int ObAddLSReplicaTask::build_task_from_sql_result( share::ObTaskId task_id_to_set; ObSqlString comment_to_set; ObSqlString task_id_sqlstring_format; + common::ObAddr force_data_source; if (OB_FAIL(ret)) { } else if (OB_FAIL(comment_to_set.assign(comment))) { @@ -1304,6 +1542,9 @@ int ObAddLSReplicaTask::build_task_from_sql_result( } else if (false == dest_server.set_ip_addr(dest_ip, static_cast(dest_port))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid server address", K(dest_ip), K(dest_port)); + } else if (false == force_data_source.set_ip_addr(data_source_ip, static_cast(data_source_port))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid server address", K(data_source_ip), K(data_source_port)); } else if (OB_FAIL(dst_replica.assign( 0/*unit id*/, 0/*unit group id*/, @@ -1331,12 +1572,12 @@ int ObAddLSReplicaTask::build_task_from_sql_result( generate_time_us, GCONF.cluster_id, //(not used)cluster_id transmit_data_size, //(not used) - obrpc::ObAdminClearDRTaskArg::TaskType::AUTO,//(not used)invoked_source + is_manual ? obrpc::ObAdminClearDRTaskArg::TaskType::MANUAL : obrpc::ObAdminClearDRTaskArg::TaskType::AUTO,//(not used)invoked_source priority_to_set, //(not used) comment_to_set.ptr(), //comments dst_replica, //(in used)dest_server ObReplicaMember(src_server, 0), //(in used)src_server - ObReplicaMember(src_server, 0), //(in used)src_server + ObReplicaMember(force_data_source, 0), //(in used)force_data_source src_paxos_replica_number, //(in used) dest_paxos_replica_number))) { //(in used) LOG_WARN("fail to build a ObAddLSReplicaTask", KR(ret)); @@ -1417,7 +1658,8 @@ int ObLSTypeTransformTask::log_execute_result( { int ret = OB_SUCCESS; ObSqlString execute_result; - if (OB_FAIL(build_execute_result(ret_code, ret_comment, execute_result))) { + int64_t start_time = execute_time_ > 0 ? execute_time_ : schedule_time_; + if (OB_FAIL(build_execute_result(ret_code, ret_comment, start_time, execute_result))) { LOG_WARN("fail to build execute result", KR(ret), K(ret_code), K(ret_comment)); } else { ROOTSERVICE_EVENT_ADD("disaster_recovery", get_log_finish_str(), @@ -1494,8 +1736,6 @@ int ObLSTypeTransformTask::fill_dml_splicer( char target_ip[OB_MAX_SERVER_ADDR_SIZE] = ""; char task_id[OB_TRACE_STAT_BUFFER_SIZE] = ""; char task_type[MAX_DISASTER_RECOVERY_TASK_TYPE_LENGTH] = "TYPE TRANSFORM"; - int64_t transmit_data_size = 0; - if (OB_UNLIKELY(!is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid task", KR(ret)); @@ -1505,30 +1745,22 @@ int ObLSTypeTransformTask::fill_dml_splicer( } else if (false == get_dst_server().ip_to_string(dest_ip, sizeof(dest_ip))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("convert dest_server ip to string failed", KR(ret), "dest_server", get_dst_server()); - } else if (OB_FAIL(get_execute_transmit_size(transmit_data_size))) { - LOG_WARN("fail to get transmit_data_size", KR(ret), K(transmit_data_size)); - } else { - if (OB_FAIL(dml_splicer.add_pk_column("tenant_id", tenant_id_)) - || OB_FAIL(dml_splicer.add_pk_column("ls_id", ls_id_.id())) - || OB_FAIL(dml_splicer.add_pk_column("task_type", task_type)) - || OB_FAIL(dml_splicer.add_pk_column("task_id", task_id_)) - || OB_FAIL(dml_splicer.add_column("task_status", TASK_STATUS)) - || OB_FAIL(dml_splicer.add_column("priority", static_cast(ObDRTaskPriority::HIGH_PRI))) - || OB_FAIL(dml_splicer.add_column("target_replica_svr_ip", dest_ip)) - || OB_FAIL(dml_splicer.add_column("target_replica_svr_port", get_dst_server().get_port())) - || OB_FAIL(dml_splicer.add_column("target_paxos_replica_number", get_paxos_replica_number())) - || OB_FAIL(dml_splicer.add_column("target_replica_type", ob_replica_type_strs(get_dst_replica().get_member().get_replica_type()))) - || OB_FAIL(dml_splicer.add_column("source_replica_svr_ip", src_ip)) - || OB_FAIL(dml_splicer.add_column("source_replica_svr_port", get_src_member().get_server().get_port())) - || OB_FAIL(dml_splicer.add_column("source_paxos_replica_number", get_orig_paxos_replica_number())) - || OB_FAIL(dml_splicer.add_column("source_replica_type", ob_replica_type_strs(get_src_member().get_replica_type()))) - || OB_FAIL(dml_splicer.add_column("task_exec_svr_ip", dest_ip)) - || OB_FAIL(dml_splicer.add_column("task_exec_svr_port", get_dst_server().get_port())) - || OB_FAIL(dml_splicer.add_time_column("generate_time", generate_time_)) - || OB_FAIL(dml_splicer.add_time_column("schedule_time", schedule_time_)) - || OB_FAIL(dml_splicer.add_column("comment", comment_.ptr()))) { - LOG_WARN("add column failed", KR(ret)); - } + } else if (OB_FAIL(ObDRTask::fill_dml_splicer(dml_splicer))) { + LOG_WARN("ObDRTask fill dml splicer failed", KR(ret)); + } else if (OB_FAIL(dml_splicer.add_pk_column("task_type", task_type)) + || OB_FAIL(dml_splicer.add_column("target_replica_svr_ip", dest_ip)) + || OB_FAIL(dml_splicer.add_column("target_replica_svr_port", get_dst_server().get_port())) + || OB_FAIL(dml_splicer.add_column("target_paxos_replica_number", get_paxos_replica_number())) + || OB_FAIL(dml_splicer.add_column("target_replica_type", ob_replica_type_strs(get_dst_replica().get_member().get_replica_type()))) + || OB_FAIL(dml_splicer.add_column("source_replica_svr_ip", src_ip)) + || OB_FAIL(dml_splicer.add_column("source_replica_svr_port", get_src_member().get_server().get_port())) + || OB_FAIL(dml_splicer.add_column("source_paxos_replica_number", get_orig_paxos_replica_number())) + || OB_FAIL(dml_splicer.add_column("source_replica_type", ob_replica_type_strs(get_src_member().get_replica_type()))) + || OB_FAIL(dml_splicer.add_column("task_exec_svr_ip", dest_ip)) + || OB_FAIL(dml_splicer.add_column("task_exec_svr_port", get_dst_server().get_port()))) { + LOG_WARN("add column failed", KR(ret)); + } else if (OB_FAIL(fill_dml_splicer_for_new_column(dml_splicer, common::ObAddr()))) { + LOG_WARN("fill dml_splicer for new column failed", KR(ret)); } return ret; } @@ -1691,6 +1923,53 @@ int ObLSTypeTransformTask::build( return ret; } +int ObLSTypeTransformTask::simple_build( + const uint64_t tenant_id, + const share::ObLSID &ls_id, + const share::ObTaskId &task_id, + const ObDstReplica &dst_replica, + const common::ObReplicaMember &src_member, + const common::ObReplicaMember &data_src_member, + const int64_t orig_paxos_replica_number, + const int64_t paxos_replica_number) +{ + int ret = OB_SUCCESS; + ObDRTaskKey task_key; + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) + || !ls_id.is_valid_with_tenant(tenant_id) + || !task_id.is_valid() + || !dst_replica.is_valid() + || !src_member.is_valid() + || paxos_replica_number <= 0 + || orig_paxos_replica_number <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(ls_id), K(task_id), K(dst_replica), K(src_member), + K(orig_paxos_replica_number), K(paxos_replica_number)); + } else if (OB_FAIL(task_key.init(tenant_id, ls_id.id(), 0, 0, + ObDRTaskKeyType::FORMAL_DR_KEY))) { + LOG_WARN("fail to init task key", KR(ret), K(tenant_id), K(ls_id)); + } else if (OB_FAIL(ObDRTask::build( + task_key, + tenant_id, + ls_id, + task_id, + 0,/*schedule_time_us*/ 0,/*generate_time_us*/ + GCONF.cluster_id, 0,/*transmit_data_size*/ + obrpc::ObAdminClearDRTaskArg::TaskType::MANUAL, + ObDRTaskPriority::HIGH_PRI, + ObString(drtask::ALTER_SYSTEM_COMMAND_MODIFY_REPLICA_TYPE)))) { + LOG_WARN("fail to build ObDRTask", KR(ret), K(task_key), K(tenant_id), K(ls_id), K(task_id)); + } else if (OB_FAIL(dst_replica_.assign(dst_replica))) { + LOG_WARN("fail to assign dst replica", KR(ret), K(dst_replica)); + } else { + src_member_ = src_member; + data_src_member_ = data_src_member; + orig_paxos_replica_number_ = orig_paxos_replica_number; + paxos_replica_number_ = paxos_replica_number; + } + return ret; +} + int ObLSTypeTransformTask::build_task_from_sql_result( const sqlclient::ObMySQLResult &res) { @@ -1711,6 +1990,7 @@ int ObLSTypeTransformTask::build_task_from_sql_result( int64_t schedule_time_us = 0; int64_t generate_time_us = 0; common::ObString comment; + bool is_manual = false; //STEP1_0: read certain members from sql result EXTRACT_INT_FIELD_MYSQL(res, "tenant_id", tenant_id, uint64_t); { @@ -1733,6 +2013,7 @@ int ObLSTypeTransformTask::build_task_from_sql_result( (void)GET_COL_IGNORE_NULL(res.get_varchar, "source_replica_type", src_type); (void)GET_COL_IGNORE_NULL(res.get_varchar, "target_replica_type", dest_type); (void)GET_COL_IGNORE_NULL(res.get_varchar, "comment", comment); + EXTRACT_BOOL_FIELD_MYSQL_SKIP_RET(res, "is_manual", is_manual); //STEP2_0: make necessary members to build a task ObDRTaskKey task_key; common::ObAddr src_server; @@ -1811,7 +2092,7 @@ int ObLSTypeTransformTask::build_task_from_sql_result( generate_time_us, GCONF.cluster_id, //(not used)cluster_id transmit_data_size, //(not used) - obrpc::ObAdminClearDRTaskArg::TaskType::AUTO,//(not used)invoked_source + is_manual ? obrpc::ObAdminClearDRTaskArg::TaskType::MANUAL : obrpc::ObAdminClearDRTaskArg::TaskType::AUTO,//(not used)invoked_source priority_to_set, //(not used) comment_to_set.ptr(), //comment dst_replica, //(in used)dest_server @@ -1873,7 +2154,8 @@ int ObRemoveLSReplicaTask::log_execute_result( { int ret = OB_SUCCESS; ObSqlString execute_result; - if (OB_FAIL(build_execute_result(ret_code, ret_comment, execute_result))) { + int64_t start_time = execute_time_ > 0 ? execute_time_ : schedule_time_; + if (OB_FAIL(build_execute_result(ret_code, ret_comment, start_time, execute_result))) { LOG_WARN("fail to build execute result", KR(ret), K(ret_code), K(ret_comment)); } else { ROOTSERVICE_EVENT_ADD("disaster_recovery", get_log_finish_str(), @@ -1952,7 +2234,6 @@ int ObRemoveLSReplicaTask::fill_dml_splicer( char dest_ip[OB_MAX_SERVER_ADDR_SIZE] = ""; char target_ip[OB_MAX_SERVER_ADDR_SIZE] = ""; char task_id[OB_TRACE_STAT_BUFFER_SIZE] = ""; - int64_t transmit_data_size = 0; const char *task_type_to_set = ob_disaster_recovery_task_type_strs(get_disaster_recovery_task_type()); if (OB_UNLIKELY(!is_valid())) { @@ -1964,30 +2245,22 @@ int ObRemoveLSReplicaTask::fill_dml_splicer( } else if (false == get_remove_server().get_server().ip_to_string(target_ip, sizeof(target_ip))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("convert target_server ip to string failed", KR(ret), "target_server", get_remove_server().get_server()); - } else if (OB_FAIL(get_execute_transmit_size(transmit_data_size))) { - LOG_WARN("fail to get transmit_data_size", KR(ret), K(transmit_data_size)); - } else { - if (OB_FAIL(dml_splicer.add_pk_column("tenant_id", tenant_id_)) - || OB_FAIL(dml_splicer.add_pk_column("ls_id", ls_id_.id())) - || OB_FAIL(dml_splicer.add_pk_column("task_type", task_type_to_set)) - || OB_FAIL(dml_splicer.add_pk_column("task_id", task_id_)) - || OB_FAIL(dml_splicer.add_column("task_status", TASK_STATUS)) - || OB_FAIL(dml_splicer.add_column("priority", static_cast(ObDRTaskPriority::HIGH_PRI))) - || OB_FAIL(dml_splicer.add_column("target_replica_svr_ip", target_ip)) - || OB_FAIL(dml_splicer.add_column("target_replica_svr_port", get_remove_server().get_server().get_port())) - || OB_FAIL(dml_splicer.add_column("target_paxos_replica_number", get_paxos_replica_number())) - || OB_FAIL(dml_splicer.add_column("target_replica_type", ob_replica_type_strs(get_remove_server().get_replica_type()))) - || OB_FAIL(dml_splicer.add_column("source_replica_svr_ip", src_ip)) - || OB_FAIL(dml_splicer.add_column("source_replica_svr_port", 0)) - || OB_FAIL(dml_splicer.add_column("source_paxos_replica_number", get_orig_paxos_replica_number())) - || OB_FAIL(dml_splicer.add_column("source_replica_type", "")) - || OB_FAIL(dml_splicer.add_column("task_exec_svr_ip", dest_ip)) - || OB_FAIL(dml_splicer.add_column("task_exec_svr_port", get_leader().get_port())) - || OB_FAIL(dml_splicer.add_time_column("generate_time", generate_time_)) - || OB_FAIL(dml_splicer.add_time_column("schedule_time", schedule_time_)) - || OB_FAIL(dml_splicer.add_column("comment", comment_.ptr()))) { - LOG_WARN("add column failed", KR(ret)); - } + } else if (OB_FAIL(ObDRTask::fill_dml_splicer(dml_splicer))) { + LOG_WARN("ObDRTask fill dml splicer failed", KR(ret)); + } else if (OB_FAIL(dml_splicer.add_pk_column("task_type", task_type_to_set)) + || OB_FAIL(dml_splicer.add_column("target_replica_svr_ip", target_ip)) + || OB_FAIL(dml_splicer.add_column("target_replica_svr_port", get_remove_server().get_server().get_port())) + || OB_FAIL(dml_splicer.add_column("target_paxos_replica_number", get_paxos_replica_number())) + || OB_FAIL(dml_splicer.add_column("target_replica_type", ob_replica_type_strs(get_remove_server().get_replica_type()))) + || OB_FAIL(dml_splicer.add_column("source_replica_svr_ip", src_ip)) + || OB_FAIL(dml_splicer.add_column("source_replica_svr_port", 0)) + || OB_FAIL(dml_splicer.add_column("source_paxos_replica_number", get_orig_paxos_replica_number())) + || OB_FAIL(dml_splicer.add_column("source_replica_type", "")) + || OB_FAIL(dml_splicer.add_column("task_exec_svr_ip", dest_ip)) + || OB_FAIL(dml_splicer.add_column("task_exec_svr_port", get_leader().get_port()))) { + LOG_WARN("add column failed", KR(ret)); + } else if (OB_FAIL(fill_dml_splicer_for_new_column(dml_splicer, common::ObAddr()))) { + LOG_WARN("fill dml_splicer for new column failed", KR(ret)); } return ret; } @@ -2082,6 +2355,54 @@ int ObRemoveLSReplicaTask::build( return ret; } +int ObRemoveLSReplicaTask::simple_build( + const uint64_t tenant_id, + const share::ObLSID &ls_id, + const share::ObTaskId &task_id, + const common::ObAddr &leader, + const common::ObReplicaMember &remove_server, + const int64_t orig_paxos_replica_number, + const int64_t paxos_replica_number, + const ObReplicaType &replica_type) +{ + int ret = OB_SUCCESS; + ObDRTaskKey task_key; + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) + || !ls_id.is_valid_with_tenant(tenant_id) + || !task_id.is_valid() + || !leader.is_valid() + || !remove_server.is_valid() + || orig_paxos_replica_number <= 0 + || paxos_replica_number <= 0 + || REPLICA_TYPE_MAX == replica_type)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(ls_id), K(task_id), + K(leader), K(remove_server), K(orig_paxos_replica_number), + K(paxos_replica_number), K(replica_type)); + } else if (OB_FAIL(task_key.init(tenant_id, ls_id.id(), 0, 0, + ObDRTaskKeyType::FORMAL_DR_KEY))) { + LOG_WARN("fail to init task key", KR(ret), K(tenant_id), K(ls_id)); + } else if (OB_FAIL(ObDRTask::build( + task_key, + tenant_id, + ls_id, + task_id, + 0,/*schedule_time_us*/ 0,/*generate_time_us*/ + GCONF.cluster_id, 0,/*transmit_data_size*/ + obrpc::ObAdminClearDRTaskArg::TaskType::MANUAL, + ObDRTaskPriority::HIGH_PRI, + ObString(drtask::ALTER_SYSTEM_COMMAND_REMOVE_REPLICA)))) { + LOG_WARN("fail to build ObDRTask", KR(ret), K(task_key), K(tenant_id), K(ls_id), K(task_id)); + } else { + leader_ = leader; + remove_server_ = remove_server; + orig_paxos_replica_number_ = orig_paxos_replica_number; + paxos_replica_number_ = paxos_replica_number; + replica_type_ = replica_type; + } + return ret; +} + int ObRemoveLSReplicaTask::build_task_from_sql_result( const sqlclient::ObMySQLResult &res) { @@ -2102,6 +2423,7 @@ int ObRemoveLSReplicaTask::build_task_from_sql_result( int64_t generate_time_us = 0; common::ObString comment; ObReplicaType replica_type = REPLICA_TYPE_MAX; + bool is_manual = false; //STEP1_0: read certain members from sql result EXTRACT_INT_FIELD_MYSQL(res, "tenant_id", tenant_id, uint64_t); { @@ -2123,6 +2445,7 @@ int ObRemoveLSReplicaTask::build_task_from_sql_result( (void)GET_COL_IGNORE_NULL(res.get_int, "source_paxos_replica_number", src_paxos_replica_number); (void)GET_COL_IGNORE_NULL(res.get_int, "target_paxos_replica_number", dest_paxos_replica_number); (void)GET_COL_IGNORE_NULL(res.get_varchar, "comment", comment); + EXTRACT_BOOL_FIELD_MYSQL_SKIP_RET(res, "is_manual", is_manual); //STEP2_0: make necessary members to build a task ObDRTaskKey task_key; common::ObAddr dest_server; @@ -2181,7 +2504,7 @@ int ObRemoveLSReplicaTask::build_task_from_sql_result( generate_time_us, GCONF.cluster_id, //(not used)cluster_id transmit_data_size, //(not used) - obrpc::ObAdminClearDRTaskArg::TaskType::AUTO,//(not used)invoked_source + is_manual ? obrpc::ObAdminClearDRTaskArg::TaskType::MANUAL : obrpc::ObAdminClearDRTaskArg::TaskType::AUTO,//(not used)invoked_source priority_to_set, //(not used) comment_to_set.ptr(), //comment dest_server, //(in used)leader @@ -2247,7 +2570,8 @@ int ObLSModifyPaxosReplicaNumberTask::log_execute_result( { int ret = OB_SUCCESS; ObSqlString execute_result; - if (OB_FAIL(build_execute_result(ret_code, ret_comment, execute_result))) { + int64_t start_time = execute_time_ > 0 ? execute_time_ : schedule_time_; + if (OB_FAIL(build_execute_result(ret_code, ret_comment, start_time, execute_result))) { LOG_WARN("fail to build execute result", KR(ret), K(ret_code), K(ret_comment)); } else { ROOTSERVICE_EVENT_ADD("disaster_recovery", get_log_finish_str(), @@ -2306,38 +2630,28 @@ int ObLSModifyPaxosReplicaNumberTask::fill_dml_splicer( char target_ip[OB_MAX_SERVER_ADDR_SIZE] = ""; char task_id[OB_TRACE_STAT_BUFFER_SIZE] = ""; char task_type[MAX_DISASTER_RECOVERY_TASK_TYPE_LENGTH] = "MODIFY PAXOS REPLICA NUMBER"; - int64_t transmit_data_size = 0; - if (OB_UNLIKELY(!is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid task", KR(ret)); } else if (false == get_dst_server().ip_to_string(dest_ip, sizeof(dest_ip))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("convert dest_server ip to string failed", KR(ret), "dest_server", get_dst_server()); - } else if (OB_FAIL(get_execute_transmit_size(transmit_data_size))) { - LOG_WARN("fail to get transmit_data_size", KR(ret), K(transmit_data_size)); - } else { - if (OB_FAIL(dml_splicer.add_pk_column("tenant_id", tenant_id_)) - || OB_FAIL(dml_splicer.add_pk_column("ls_id", ls_id_.id())) - || OB_FAIL(dml_splicer.add_pk_column("task_type", task_type)) - || OB_FAIL(dml_splicer.add_pk_column("task_id", task_id_)) - || OB_FAIL(dml_splicer.add_column("task_status", TASK_STATUS)) - || OB_FAIL(dml_splicer.add_column("priority", static_cast(ObDRTaskPriority::HIGH_PRI))) - || OB_FAIL(dml_splicer.add_column("target_replica_svr_ip", dest_ip)) - || OB_FAIL(dml_splicer.add_column("target_replica_svr_port", get_dst_server().get_port())) - || OB_FAIL(dml_splicer.add_column("target_paxos_replica_number", get_paxos_replica_number())) - || OB_FAIL(dml_splicer.add_column("target_replica_type", "")) - || OB_FAIL(dml_splicer.add_column("source_replica_svr_ip", src_ip)) - || OB_FAIL(dml_splicer.add_column("source_replica_svr_port", 0)) - || OB_FAIL(dml_splicer.add_column("source_paxos_replica_number", get_orig_paxos_replica_number())) - || OB_FAIL(dml_splicer.add_column("source_replica_type", "")) - || OB_FAIL(dml_splicer.add_column("task_exec_svr_ip", dest_ip)) - || OB_FAIL(dml_splicer.add_column("task_exec_svr_port", get_dst_server().get_port())) - || OB_FAIL(dml_splicer.add_time_column("generate_time", generate_time_)) - || OB_FAIL(dml_splicer.add_time_column("schedule_time", schedule_time_)) - || OB_FAIL(dml_splicer.add_column("comment", comment_.ptr()))) { - LOG_WARN("add column failed", KR(ret)); - } + } else if (OB_FAIL(ObDRTask::fill_dml_splicer(dml_splicer))) { + LOG_WARN("ObDRTask fill dml splicer failed", KR(ret)); + } else if (OB_FAIL(dml_splicer.add_pk_column("task_type", task_type)) + || OB_FAIL(dml_splicer.add_column("target_replica_svr_ip", dest_ip)) + || OB_FAIL(dml_splicer.add_column("target_replica_svr_port", get_dst_server().get_port())) + || OB_FAIL(dml_splicer.add_column("target_paxos_replica_number", get_paxos_replica_number())) + || OB_FAIL(dml_splicer.add_column("target_replica_type", "")) + || OB_FAIL(dml_splicer.add_column("source_replica_svr_ip", src_ip)) + || OB_FAIL(dml_splicer.add_column("source_replica_svr_port", 0)) + || OB_FAIL(dml_splicer.add_column("source_paxos_replica_number", get_orig_paxos_replica_number())) + || OB_FAIL(dml_splicer.add_column("source_replica_type", "")) + || OB_FAIL(dml_splicer.add_column("task_exec_svr_ip", dest_ip)) + || OB_FAIL(dml_splicer.add_column("task_exec_svr_port", get_dst_server().get_port()))) { + LOG_WARN("add column failed", KR(ret)); + } else if (OB_FAIL(fill_dml_splicer_for_new_column(dml_splicer, common::ObAddr()))) { + LOG_WARN("fill dml_splicer for new column failed", KR(ret)); } return ret; } @@ -2429,6 +2743,52 @@ int ObLSModifyPaxosReplicaNumberTask::build( return ret; } +int ObLSModifyPaxosReplicaNumberTask::simple_build( + const uint64_t tenant_id, + const share::ObLSID &ls_id, + const share::ObTaskId &task_id, + const common::ObAddr &dst_server, + const int64_t orig_paxos_replica_number, + const int64_t paxos_replica_number, + const common::ObMemberList &member_list) +{ + int ret = OB_SUCCESS; + ObDRTaskKey task_key; + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) + || !ls_id.is_valid_with_tenant(tenant_id) + || !task_id.is_valid() + || !dst_server.is_valid() + || orig_paxos_replica_number <= 0 + || paxos_replica_number <= 0 + || !member_list.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(ls_id), K(task_id), + K(dst_server), K(orig_paxos_replica_number), + K(paxos_replica_number), K(member_list)); + } else if (OB_FAIL(task_key.init(tenant_id, ls_id.id(), 0, 0, + ObDRTaskKeyType::FORMAL_DR_KEY))) { + LOG_WARN("fail to init task key", KR(ret), K(tenant_id), K(ls_id)); + } else if (OB_FAIL(ObDRTask::build( + task_key, + tenant_id, + ls_id, + task_id, + 0,/*schedule_time_us*/ 0,/*generate_time_us*/ + GCONF.cluster_id, 0,/*transmit_data_size*/ + obrpc::ObAdminClearDRTaskArg::TaskType::MANUAL, + ObDRTaskPriority::HIGH_PRI, + ObString(drtask::ALTER_SYSTEM_COMMAND_MODIFY_PAXOS_REPLICA_NUM)))) { + LOG_WARN("fail to build ObDRTask", KR(ret), + K(task_key), K(tenant_id), K(ls_id), K(task_id)); + } else { + orig_paxos_replica_number_ = orig_paxos_replica_number; + paxos_replica_number_ = paxos_replica_number; + server_ = dst_server; + member_list_ = member_list; + } + return ret; +} + int ObLSModifyPaxosReplicaNumberTask::build_task_from_sql_result( const sqlclient::ObMySQLResult &res) { @@ -2445,6 +2805,7 @@ int ObLSModifyPaxosReplicaNumberTask::build_task_from_sql_result( int64_t schedule_time_us = 0; int64_t generate_time_us = 0; common::ObString comment; + bool is_manual = false; //STEP1_0: read certain members from sql result EXTRACT_INT_FIELD_MYSQL(res, "tenant_id", tenant_id, uint64_t); { @@ -2463,6 +2824,7 @@ int ObLSModifyPaxosReplicaNumberTask::build_task_from_sql_result( (void)GET_COL_IGNORE_NULL(res.get_int, "source_paxos_replica_number", src_paxos_replica_number); (void)GET_COL_IGNORE_NULL(res.get_int, "target_paxos_replica_number", dest_paxos_replica_number); (void)GET_COL_IGNORE_NULL(res.get_varchar, "comment", comment); + EXTRACT_BOOL_FIELD_MYSQL_SKIP_RET(res, "is_manual", is_manual); //STEP2_0: make necessary members to build a task ObDRTaskKey task_key; common::ObAddr dest_server; @@ -2514,7 +2876,7 @@ int ObLSModifyPaxosReplicaNumberTask::build_task_from_sql_result( generate_time_us, GCONF.cluster_id, //(not used)cluster_id transmit_data_size, //(not used) - obrpc::ObAdminClearDRTaskArg::TaskType::AUTO,//(not used)invoked_source + is_manual ? obrpc::ObAdminClearDRTaskArg::TaskType::MANUAL : obrpc::ObAdminClearDRTaskArg::TaskType::AUTO,//(not used)invoked_source priority_to_set, //(not used) comment_to_set.ptr(), //comment dest_server, //(in used)leader diff --git a/src/rootserver/ob_disaster_recovery_task.h b/src/rootserver/ob_disaster_recovery_task.h index 796bb80f4..cdee1242a 100644 --- a/src/rootserver/ob_disaster_recovery_task.h +++ b/src/rootserver/ob_disaster_recovery_task.h @@ -52,6 +52,11 @@ namespace drtask const static char * const CANCEL_MIGRATE_UNIT_WITH_NON_PAXOS_REPLICA = "cancel migrate unit remove non-paxos replica"; const static char * const MIGRATE_REPLICA_DUE_TO_UNIT_GROUP_NOT_MATCH = "migrate replica due to unit group not match"; const static char * const MIGRATE_REPLICA_DUE_TO_UNIT_NOT_MATCH = "migrate replica due to unit not match"; + const static char * const ALTER_SYSTEM_COMMAND_ADD_REPLICA = "add replica by manual"; + const static char * const ALTER_SYSTEM_COMMAND_REMOVE_REPLICA = "remove replica by manual"; + const static char * const ALTER_SYSTEM_COMMAND_MODIFY_REPLICA_TYPE = "modify replica type by manual"; + const static char * const ALTER_SYSTEM_COMMAND_MIGRATE_REPLICA = "migrate replica by manual"; + const static char * const ALTER_SYSTEM_COMMAND_MODIFY_PAXOS_REPLICA_NUM = "modify paxos_replica_num by manual"; }; namespace drtasklog @@ -70,6 +75,36 @@ namespace drtasklog const static char * const FINISH_MODIFY_PAXOS_REPLICA_NUMBER_STR = "finish_modify_paxos_replica_number"; } +class ObDRLSReplicaTaskStatus +{ + OB_UNIS_VERSION(1); +public: + enum DRLSReplicaTaskStatus + { + INPROGRESS = 0, + COMPLETED, + FAILED, + CANCELED, + MAX_STATUS, + }; +public: + ObDRLSReplicaTaskStatus() : status_(MAX_STATUS) {} + ObDRLSReplicaTaskStatus(DRLSReplicaTaskStatus status) : status_(status) {} + + ObDRLSReplicaTaskStatus &operator=(const DRLSReplicaTaskStatus status) { status_ = status; return *this; } + ObDRLSReplicaTaskStatus &operator=(const ObDRLSReplicaTaskStatus &other) { status_ = other.status_; return *this; } + void reset() { status_ = MAX_STATUS; } + void assign(const ObDRLSReplicaTaskStatus &other); + bool is_valid() const { return MAX_STATUS != status_; } + const DRLSReplicaTaskStatus &get_status() const { return status_; } + int parse_from_string(const ObString &status); + int64_t to_string(char *buf, const int64_t buf_len) const; + const char* get_status_str() const; + +private: + DRLSReplicaTaskStatus status_; +}; + enum class ObDRTaskType : int64_t; enum class ObDRTaskPriority : int64_t; @@ -94,6 +129,12 @@ const char *ob_disaster_recovery_task_type_strs(const rootserver::ObDRTaskType t const char *ob_disaster_recovery_task_priority_strs(const rootserver::ObDRTaskPriority task_priority); const char* ob_disaster_recovery_task_ret_comment_strs(const rootserver::ObDRTaskRetComment ret_comment); const char *ob_replica_type_strs(const ObReplicaType type); +bool is_manual_dr_task_data_version_match(uint64_t tenant_data_version); +int build_execute_result( + const int ret_code, + const ObDRTaskRetComment &ret_comment, + const int64_t start_time, + ObSqlString &execute_result); class ObDstReplica { @@ -207,8 +248,6 @@ enum class ObDRTaskPriority : int64_t class ObDRTask : public common::ObDLinkBase { -public: - const char *const TASK_STATUS = "INPROGRESS"; public: ObDRTask() : task_key_(), tenant_id_(common::OB_INVALID_ID), @@ -247,10 +286,6 @@ public: const ObDRTaskPriority priority, const ObString &comment); - int build_execute_result( - const int ret_code, - const ObDRTaskRetComment &ret_comment, - ObSqlString &execute_result) const; public: virtual const common::ObAddr &get_dst_server() const = 0; @@ -285,8 +320,11 @@ public: ObDRTaskRetComment &ret_comment) const = 0; virtual int fill_dml_splicer( - share::ObDMLSqlSplicer &dml_splicer) const = 0; + share::ObDMLSqlSplicer &dml_splicer) const; + int fill_dml_splicer_for_new_column( + share::ObDMLSqlSplicer &dml_splicer, + const common::ObAddr &force_data_src) const; // to string virtual TO_STRING_KV(K_(task_key), K_(tenant_id), @@ -413,6 +451,17 @@ public: const int64_t paxos_replica_number ); + // only use some necessary information build a ObMigrateLSReplicaTask + // Specifically, this method is only used when manually executing operation and maintenance commands + int simple_build( + const uint64_t tenant_id, + const share::ObLSID &ls_id, + const share::ObTaskId &task_id, + const ObDstReplica &dst_replica, + const common::ObReplicaMember &src_member, + const common::ObReplicaMember &data_src_member, + const common::ObReplicaMember &force_data_src_member, + const int64_t paxos_replica_number); // build a ObMigrateLSReplicaTask from sql result read from inner table // @param [in] res, sql result read from inner table int build_task_from_sql_result(const sqlclient::ObMySQLResult &res); @@ -528,7 +577,18 @@ public: const common::ObReplicaMember &force_data_src_member, const int64_t orig_paxos_replica_number, const int64_t paxos_replica_number); - + + // only use some necessary information build a ObAddLSReplicaTask + // Specifically, this method is only used when manually executing operation and maintenance commands + int simple_build( + const uint64_t tenant_id, + const share::ObLSID &ls_id, + const share::ObTaskId &task_id, + const ObDstReplica &dst_replica, + const common::ObReplicaMember &data_src_member, + const common::ObReplicaMember &force_data_src_member, + const int64_t orig_paxos_replica_number, + const int64_t paxos_replica_number); // build a ObAddLSReplicaTask from sql result read from inner table // @param [in] res, sql result read from inner table int build_task_from_sql_result(const sqlclient::ObMySQLResult &res); @@ -644,6 +704,17 @@ public: const int64_t orig_paxos_replica_number, const int64_t paxos_replica_number); + // only use some necessary information build a ObLSTypeTransformTask + // Specifically, this method is only used when manually executing operation and maintenance commands + int simple_build( + const uint64_t tenant_id, + const share::ObLSID &ls_id, + const share::ObTaskId &task_id, + const ObDstReplica &dst_replica, + const common::ObReplicaMember &src_member, + const common::ObReplicaMember &data_src_member, + const int64_t orig_paxos_replica_number, + const int64_t paxos_replica_number); // build a ObLSTypeTransformTask from sql result read from inner table // @param [in] res, sql result read from inner table int build_task_from_sql_result(const sqlclient::ObMySQLResult &res); @@ -760,6 +831,18 @@ public: const int64_t paxos_replica_number, const ObReplicaType &replica_type); + // only use some necessary information build a ObRemoveLSReplicaTask + // Specifically, this method is only used when manually executing operation and maintenance commands + int simple_build( + const uint64_t tenant_id, + const share::ObLSID &ls_id, + const share::ObTaskId &task_id, + const common::ObAddr &leader, + const common::ObReplicaMember &remove_server, + const int64_t orig_paxos_replica_number, + const int64_t paxos_replica_number, + const ObReplicaType &replica_type); + // build a ObRemoveLSReplicaTask from sql result read from inner table // @param [in] res, sql result read from inner table int build_task_from_sql_result(const sqlclient::ObMySQLResult &res); @@ -874,6 +957,16 @@ public: const int64_t paxos_replica_number, const common::ObMemberList &member_list); + // only use some necessary information build a ObLSModifyPaxosReplicaNumberTask + // Specifically, this method is only used when manually executing operation and maintenance commands + int simple_build( + const uint64_t tenant_id, + const share::ObLSID &ls_id, + const share::ObTaskId &task_id, + const common::ObAddr &dst_server, + const int64_t orig_paxos_replica_number, + const int64_t paxos_replica_number, + const common::ObMemberList &member_list); // build a ObLSModifyPaxosReplicaNumberTask from sql result read from inner table // @param [in] res, sql result read from inner table int build_task_from_sql_result(const sqlclient::ObMySQLResult &res); diff --git a/src/rootserver/ob_disaster_recovery_task_mgr.cpp b/src/rootserver/ob_disaster_recovery_task_mgr.cpp index df8628988..f4aa22cf4 100644 --- a/src/rootserver/ob_disaster_recovery_task_mgr.cpp +++ b/src/rootserver/ob_disaster_recovery_task_mgr.cpp @@ -14,28 +14,29 @@ #include "ob_disaster_recovery_task_mgr.h" -#include "lib/lock/ob_mutex.h" -#include "lib/stat/ob_diagnose_info.h" -#include "lib/profile/ob_trace_id.h" #include "lib/alloc/ob_malloc_allocator.h" -#include "share/ob_debug_sync.h" -#include "share/ob_srv_rpc_proxy.h" -#include "share/config/ob_server_config.h" -#include "ob_disaster_recovery_task_executor.h" -#include "rootserver/ob_root_balancer.h" -#include "ob_rs_event_history_table_operator.h" -#include "share/ob_rpc_struct.h" -#include "observer/ob_server_struct.h" -#include "sql/executor/ob_executor_rpc_proxy.h" -#include "rootserver/ob_disaster_recovery_task.h" // for ObDRTaskType -#include "share/ob_share_util.h" // for ObShareUtil +#include "lib/lock/ob_mutex.h" #include "lib/lock/ob_tc_rwlock.h" // for common::RWLock +#include "lib/profile/ob_trace_id.h" +#include "lib/stat/ob_diagnose_info.h" +#include "observer/ob_server_struct.h" +#include "ob_disaster_recovery_task_executor.h" +#include "ob_disaster_recovery_task_table_operator.h" +#include "ob_rs_event_history_table_operator.h" +#include "rootserver/ob_disaster_recovery_task.h" // for ObDRTaskType #include "rootserver/ob_disaster_recovery_task.h" #include "rootserver/tenant_snapshot/ob_tenant_snapshot_util.h" // for ObTenantSnapshotUtil -#include "share/inner_table/ob_inner_table_schema_constants.h" -#include "share/ob_all_server_tracer.h" #include "storage/tablelock/ob_lock_inner_connection_util.h" // for ObInnerConnectionLockUtil #include "observer/ob_inner_sql_connection.h" +#include "rootserver/ob_root_balancer.h" +#include "share/config/ob_server_config.h" +#include "share/inner_table/ob_inner_table_schema_constants.h" +#include "share/ob_all_server_tracer.h" +#include "share/ob_debug_sync.h" +#include "share/ob_rpc_struct.h" +#include "share/ob_share_util.h" // for ObShareUtil +#include "share/ob_srv_rpc_proxy.h" +#include "sql/executor/ob_executor_rpc_proxy.h" namespace oceanbase { @@ -220,7 +221,7 @@ int ObDRTaskQueue::push_task_in_wait_list( } int ObDRTaskQueue::push_task_in_schedule_list( - ObDRTask &task) + const ObDRTask &task) { // STEP 1: push task into schedule list // STEP 2: push task into task_map @@ -815,6 +816,55 @@ int ObDRTaskMgr::check_task_exist( return ret; } +int ObDRTaskMgr::add_task_in_queue_and_execute(const ObDRTask &task) +{ + int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + if (OB_FAIL(check_inner_stat_())) { + LOG_WARN("fail to check inner stat", KR(ret), K_(inited), K_(loaded), K_(stopped)); + } else if (OB_UNLIKELY(!task.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid dr task", KR(ret), K(task)); + } else { + ObThreadCondGuard guard(cond_); + bool task_exist = false; + bool sibling_in_schedule = false; + ObDRTaskQueue &queue = task.is_high_priority_task() ? high_task_queue_ : low_task_queue_; + ObDRTaskQueue &sibling_queue = task.is_high_priority_task() ? low_task_queue_ : high_task_queue_; + if (OB_UNLIKELY(queue.task_cnt() >= TASK_QUEUE_LIMIT)) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("disaster recovery task queue is full", KR(ret), "task_cnt", queue.task_cnt()); + } else if (OB_FAIL(queue.check_task_exist(task.get_task_key(), task_exist))) { + LOG_WARN("fail to check task in scheduling", KR(ret), K(task)); + } else if (task_exist) { + ret = OB_ENTRY_EXIST; + LOG_WARN("ls disaster recovery task has existed in queue", KR(ret), K(task), K(task_exist)); + } else if (OB_FAIL(sibling_queue.check_task_in_scheduling(task.get_task_key(), sibling_in_schedule))) { + LOG_WARN("fail to check task in scheduling", KR(ret), K(task)); + } else if (sibling_in_schedule) { + ret = OB_ENTRY_EXIST; + LOG_WARN("ls disaster recovery task has existed in sibling_queue", + KR(ret), K(task), K(sibling_in_schedule)); + } else if (OB_FAIL(queue.push_task_in_schedule_list(task))) { + LOG_WARN("fail to add task to schedule list", KR(ret), K(task)); + } else if (OB_FAIL(set_sibling_in_schedule(task, true/*in_schedule*/))) { + //after successfully adding the scheduling queue, + //need mark in both queues that the task has been scheduled in the queue. + //if there is a task in the waiting queue of another queue, need update its mark + LOG_WARN("set sibling in schedule failed", KR(ret), K(task)); + } else { + if (OB_SUCCESS != (tmp_ret = task.log_execute_start())) { + LOG_WARN("fail to log task start", KR(tmp_ret), K(task)); + } + if (OB_FAIL(execute_manual_task_(task))) { + //must under cond + LOG_WARN("fail to execute manual task", KR(ret), K(task)); + } + } + } + return ret; +} + int ObDRTaskMgr::add_task( const ObDRTask &task) { @@ -1207,9 +1257,7 @@ int ObDRTaskMgr::persist_task_info_( { int ret = OB_SUCCESS; ret_comment = ObDRTaskRetComment::MAX; - share::ObDMLSqlSplicer dml; - ObSqlString sql; - int64_t affected_rows = 0; + ObLSReplicaTaskTableOperator task_table_operator; const uint64_t sql_tenant_id = gen_meta_tenant_id(task.get_tenant_id()); ObMySQLTransaction trans; const int64_t timeout = GCONF.internal_sql_execute_timeout; @@ -1223,10 +1271,6 @@ int ObDRTaskMgr::persist_task_info_( LOG_WARN("invalid argument", KR(ret)); } else if (OB_FAIL(trans.start(sql_proxy_, sql_tenant_id))) { LOG_WARN("failed to start trans", KR(ret), K(sql_tenant_id)); - } else if (OB_FAIL(task.fill_dml_splicer(dml))) { - LOG_WARN("fill dml splicer failed", KR(ret)); - } else if (OB_FAIL(dml.splice_insert_sql(share::OB_ALL_LS_REPLICA_TASK_TNAME, sql))) { - LOG_WARN("fail to splice batch insert update sql", KR(ret), K(sql)); } else if (OB_ISNULL(conn = static_cast(trans.get_connection()))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("conn_ is NULL", KR(ret)); @@ -1239,8 +1283,8 @@ int ObDRTaskMgr::persist_task_info_( } else if (OB_FAIL(ObTenantSnapshotUtil::check_tenant_not_in_cloning_procedure(task.get_tenant_id(), case_to_check))) { LOG_WARN("fail to check whether tenant is in cloning procedure", KR(ret)); ret_comment = CANNOT_PERSIST_TASK_DUE_TO_CLONE_CONFLICT; - } else if (OB_FAIL(trans.write(sql_tenant_id, sql.ptr(), affected_rows))) { - LOG_WARN("execute sql failed", KR(ret), "tenant_id",task.get_tenant_id(), K(sql_tenant_id), K(sql)); + } else if (OB_FAIL(task_table_operator.insert_task(trans, task))) { + LOG_WARN("task_table_operator insert_task failed", KR(ret), K(task)); } if (trans.is_started()) { int tmp_ret = OB_SUCCESS; @@ -1448,6 +1492,41 @@ int ObDRTaskMgr::pop_task( return ret; } +int ObDRTaskMgr::execute_manual_task_( + const ObDRTask &task) +{ + int ret = OB_SUCCESS; + FLOG_INFO("execute manual disaster recovery task", K(task)); + int dummy_ret = OB_SUCCESS; + ObDRTaskRetComment ret_comment = ObDRTaskRetComment::MAX; + DEBUG_SYNC(BEFORE_ADD_MANUAL_REPLICA_TASK_IN_INNER_TABLE); + if (OB_FAIL(check_inner_stat_())) { + LOG_WARN("fail to check inner stat", KR(ret), K_(inited), K_(stopped), K_(loaded)); + } else if (OB_UNLIKELY(!task.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(task)); + } else if (OB_ISNULL(task_executor_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("task_executor_ is nullptr", KR(ret), K(task)); + } else if (OB_FAIL(persist_task_info_(task, ret_comment))) { + LOG_WARN("fail to persist task info into table", KR(ret), K(task)); + } else if (OB_FAIL(task_executor_->execute(task, dummy_ret, ret_comment))) { + LOG_WARN("fail to execute disaster recovery task", KR(ret), K(task)); + } + if (OB_FAIL(ret)) { + (void)log_task_result(task, ret, ret_comment); + const bool data_in_limit = (OB_REACH_SERVER_DATA_COPY_IN_CONCURRENCY_LIMIT == ret); + if (OB_SUCCESS != async_add_cleaning_task_to_updater( + task.get_task_id(), + task.get_task_key(), + ret, false/*need_record_event*/, ret_comment, + !data_in_limit)) { + LOG_WARN("fail to do execute over", KR(ret), K(task)); + } + } + return ret; +} + int ObDRTaskMgr::execute_task( const ObDRTask &task) { diff --git a/src/rootserver/ob_disaster_recovery_task_mgr.h b/src/rootserver/ob_disaster_recovery_task_mgr.h index 33b57ce95..26514f347 100644 --- a/src/rootserver/ob_disaster_recovery_task_mgr.h +++ b/src/rootserver/ob_disaster_recovery_task_mgr.h @@ -87,7 +87,7 @@ public: // push a task into this queue's schedule_list // @param [in] task, the task to push in int push_task_in_schedule_list( - ObDRTask &task); + const ObDRTask &task); // pop a task and move it from wait_list to schedule_list // @param [out] task, the task to pop @@ -261,6 +261,10 @@ public: const ObDRTaskPriority priority, bool &task_exist); + // add task in schedule list and execute task + // @param [in] task, target task + virtual int add_task_in_queue_and_execute( + const ObDRTask &task); // add a task into queue // @param [in] task, the task to push in virtual int add_task( @@ -399,6 +403,11 @@ private: int execute_task( const ObDRTask &task); + // try to persist and execute a manual task + // @param [in] task, the task to execute + int execute_manual_task_( + const ObDRTask &task); + // set sibling in schedule // @param [in] task, which task to deal with // @param [in] in_schedule, whether in schedule diff --git a/src/rootserver/ob_disaster_recovery_task_table_operator.cpp b/src/rootserver/ob_disaster_recovery_task_table_operator.cpp new file mode 100644 index 000000000..40d467e5b --- /dev/null +++ b/src/rootserver/ob_disaster_recovery_task_table_operator.cpp @@ -0,0 +1,273 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#define USING_LOG_PREFIX RS +#include "ob_disaster_recovery_task_table_operator.h" +#include "lib/container/ob_se_array.h" // ObSEArray +#include "lib/mysqlclient/ob_isql_client.h" // for ObISQLClient +#include "lib/oblog/ob_log_module.h" // for LOG_WARN +#include "lib/string/ob_sql_string.h" // for ObSqlString +#include "share/inner_table/ob_inner_table_schema_constants.h" // for xxx_TNAME + +namespace oceanbase +{ +namespace rootserver +{ + +int ObLSReplicaTaskTableOperator::delete_task( + common::ObMySQLTransaction &trans, + const uint64_t tenant_id, + const share::ObLSID& ls_id, + const ObDRTaskType& task_type, + const share::ObTaskId& task_id, + int64_t &affected_rows) +{ + int ret = OB_SUCCESS; + affected_rows = 0; + ObSqlString sql; + char task_id_to_set[OB_TRACE_STAT_BUFFER_SIZE] = ""; + const uint64_t sql_tenant_id = gen_meta_tenant_id(tenant_id); + if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id + || !ls_id.is_valid() + || ObDRTaskType::MAX_TYPE == task_type + || task_id.is_invalid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(ls_id), K(task_type), K(task_id)); + } else if (false == task_id.to_string(task_id_to_set, sizeof(task_id_to_set))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("convert task id to string failed", KR(ret), K(task_id)); + } else if (OB_FAIL(sql.assign_fmt("DELETE FROM %s WHERE tenant_id = %lu AND ls_id = %lu " + "AND task_type = '%s' AND task_id = '%s'", + share::OB_ALL_LS_REPLICA_TASK_TNAME, + tenant_id, + ls_id.id(), + ob_disaster_recovery_task_type_strs(task_type), + task_id_to_set))) { + LOG_WARN("assign sql string failed", KR(ret), K(tenant_id), + K(ls_id), K(task_type), K(task_id_to_set)); + } else if (OB_FAIL(trans.write(sql_tenant_id, sql.ptr(), affected_rows))) { + LOG_WARN("execute sql failed", KR(ret), K(sql), K(sql_tenant_id)); + } + return ret; +} + +int ObLSReplicaTaskTableOperator::insert_task( + common::ObISQLClient &sql_proxy, + const ObDRTask &task) +{ + int ret = OB_SUCCESS; + share::ObDMLSqlSplicer dml; + ObSqlString sql; + int64_t affected_rows = 0; + const uint64_t sql_tenant_id = gen_meta_tenant_id(task.get_tenant_id()); + if (OB_UNLIKELY(!task.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(task)); + } else if (OB_FAIL(task.fill_dml_splicer(dml))) { + LOG_WARN("fill dml splicer failed", KR(ret), K(task)); + } else if (OB_FAIL(dml.splice_insert_sql(share::OB_ALL_LS_REPLICA_TASK_TNAME, sql))) { + LOG_WARN("fail to splice insert update sql", KR(ret), K(task)); + } else if (OB_FAIL(sql_proxy.write(sql_tenant_id, sql.ptr(), affected_rows))) { + LOG_WARN("execute sql failed", KR(ret), K(task.get_tenant_id()), K(sql_tenant_id), K(sql)); + } else if (!is_single_row(affected_rows)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("insert is not single row", KR(ret), K(affected_rows)); + } + return ret; +} + +int ObLSReplicaTaskTableOperator::finish_task( + common::ObMySQLTransaction& trans, + const ObDRTaskTableUpdateTask& task) +{ + int ret = OB_SUCCESS; + uint64_t tenant_data_version = 0; + int64_t insert_rows = 0; + int64_t delete_rows = 0; + if (OB_UNLIKELY(!task.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(task)); + } else if (OB_FAIL(GET_MIN_DATA_VERSION(gen_meta_tenant_id(task.get_tenant_id()), tenant_data_version))) { + LOG_WARN("fail to get min data version", KR(ret), K(task)); + } else if (is_manual_dr_task_data_version_match(tenant_data_version)) { + char task_id_to_set[OB_TRACE_STAT_BUFFER_SIZE] = ""; + int64_t schedule_time = 0; + ObSqlString execute_result; + ObSqlString condition_sql; + ObSqlString sql; + uint64_t sql_tenant_id = gen_meta_tenant_id(task.get_tenant_id()); + const char* table_column = "tenant_id, ls_id, task_type, task_id, priority, target_replica_svr_ip, target_replica_svr_port, target_paxos_replica_number," + "target_replica_type, source_replica_svr_ip, source_replica_svr_port, source_paxos_replica_number, source_replica_type, task_exec_svr_ip, task_exec_svr_port," + "generate_time, schedule_time, comment, data_source_svr_ip, data_source_svr_port, is_manual"; + // no task_status + ObDRLSReplicaTaskStatus task_status(ObDRLSReplicaTaskStatus::COMPLETED); + if (OB_CANCELED == task.get_ret_code()) { + task_status = ObDRLSReplicaTaskStatus::CANCELED; + } else if (OB_SUCCESS != task.get_ret_code()) { + task_status = ObDRLSReplicaTaskStatus::FAILED; + } + if (false == task.get_task_id().to_string(task_id_to_set, sizeof(task_id_to_set))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("convert task id to string failed", KR(ret), K(task)); + } else if (OB_FAIL(get_task_schedule_time_(trans, task, schedule_time))) { + if (OB_ENTRY_NOT_EXIST == ret) { + ret = OB_SUCCESS; + LOG_INFO("task count is 0", KR(ret), K(task)); + } else { + LOG_WARN("faild to get task schedule_time", KR(ret), K(task)); + } + } else if (OB_FAIL(build_execute_result(task.get_ret_code(), + task.get_ret_comment(), + schedule_time, + execute_result))) { + LOG_WARN("build_execute_result failed", KR(ret), K(task), K(schedule_time)); + } else if (OB_FAIL(condition_sql.assign_fmt("tenant_id = %lu AND ls_id = %lu AND task_type = '%s' AND task_id = '%s'", + task.get_tenant_id(), task.get_ls_id().id(), + ob_disaster_recovery_task_type_strs(task.get_task_type()), task_id_to_set))) { + LOG_WARN("failed to append sql", KR(ret), K(task), K(task_id_to_set)); + } else if (OB_FAIL(sql.assign_fmt("insert into %s (%s, task_status, execute_result, finish_time) " + " select %s, '%s', '%s', now() from %s where %s", + share::OB_ALL_LS_REPLICA_TASK_HISTORY_TNAME, table_column, table_column, + task_status.get_status_str(), execute_result.ptr(), + share::OB_ALL_LS_REPLICA_TASK_TNAME, condition_sql.ptr()))) { + LOG_WARN("failed to assign sql", KR(ret), K(task_status), K(execute_result), K(condition_sql)); + } else if (OB_FAIL(trans.write(sql_tenant_id, sql.ptr(), insert_rows))) { + LOG_WARN("execute sql failed", KR(ret), K(sql_tenant_id), K(sql)); + } else if (insert_rows != 1) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql insert error", KR(ret), K(insert_rows), K(task)); + } + } + if (FAILEDx(delete_task(trans, task.get_tenant_id(), task.get_ls_id(), task.get_task_type(), + task.get_task_id(), delete_rows))) { + LOG_WARN("delete_task failed", KR(ret), K(task)); + } else if (!is_single_row(delete_rows)) { + // ignore affected row check for task not exist + LOG_INFO("expected deleted single row", K(delete_rows), K(task)); + }// during the upgrade process, it is possible that insert_rows is 0 and delete_rows is 1. + return ret; +} + +int ObLSReplicaTaskTableOperator::get_task_schedule_time_( + common::ObMySQLTransaction& trans, + const ObDRTaskTableUpdateTask &task, + int64_t &schedule_time) +{ + int ret = OB_SUCCESS; + schedule_time = 0; + uint64_t sql_tenant_id = gen_meta_tenant_id(task.get_tenant_id()); + char task_id_to_set[OB_TRACE_STAT_BUFFER_SIZE] = ""; + if (OB_UNLIKELY(!task.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(task)); + } else if (false == task.get_task_id().to_string(task_id_to_set, sizeof(task_id_to_set))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("convert task id to string failed", KR(ret), K(task)); + } else { + ObSqlString sql; + SMART_VAR(ObISQLClient::ReadResult, res) { + sqlclient::ObMySQLResult* result = nullptr; + if (OB_FAIL(sql.assign_fmt("SELECT time_to_usec(schedule_time) AS schedule_time FROM %s WHERE " + "tenant_id = %lu AND ls_id = %lu AND task_type = '%s' AND task_id = '%s'", + share::OB_ALL_LS_REPLICA_TASK_TNAME, + task.get_tenant_id(), + task.get_ls_id().id(), + ob_disaster_recovery_task_type_strs(task.get_task_type()), + task_id_to_set))) { + LOG_WARN("fail to assign sql", KR(ret), K(task_id_to_set), K(task)); + } else if (OB_FAIL(trans.read(res, sql_tenant_id, sql.ptr()))) { + LOG_WARN("execute sql failed", KR(ret), K(sql_tenant_id), K(sql)); + } else if (OB_ISNULL(result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get mysql result failed", KR(ret), K(sql)); + } else if (OB_FAIL(result->next())) { + if (OB_ITER_END == ret) { + ret = OB_ENTRY_NOT_EXIST; + } + LOG_WARN("fail to get next result", KR(ret), K(sql)); + } else { + EXTRACT_INT_FIELD_MYSQL(*result, "schedule_time", schedule_time, int64_t); + int tmp_ret = OB_SUCCESS; + if (OB_SUCC(ret) && (OB_ITER_END != (tmp_ret = result->next()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get more row than one", KR(ret), KR(tmp_ret), K(sql)); + } + } + } + } + return ret; +} + +int ObLSReplicaTaskTableOperator::get_task_info_for_cancel( + common::ObISQLClient &sql_proxy, + const uint64_t tenant_id, + const share::ObTaskId &task_id, + common::ObAddr &task_execute_server, + share::ObLSID &ls_id) +{ + int ret = OB_SUCCESS; + task_execute_server.reset(); + ls_id.reset(); + int64_t ls_id_res = share::ObLSID::INVALID_LS_ID; + common::ObString server_ip; + int64_t server_port = OB_INVALID_INDEX; + char task_id_to_set[OB_TRACE_STAT_BUFFER_SIZE] = ""; + if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || !task_id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(task_id)); + } else if (false == task_id.to_string(task_id_to_set, sizeof(task_id_to_set))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("convert task id to string failed", KR(ret), K(task_id)); + } else { + uint64_t sql_tenant_id = gen_meta_tenant_id(tenant_id); + ObSqlString sql; + SMART_VAR(ObISQLClient::ReadResult, res) { + sqlclient::ObMySQLResult* result = nullptr; + if (OB_FAIL(sql.assign_fmt("SELECT ls_id, task_exec_svr_ip, task_exec_svr_port FROM %s WHERE " + "tenant_id = %lu AND task_id = '%s'", share::OB_ALL_LS_REPLICA_TASK_TNAME, tenant_id, task_id_to_set))) { + LOG_WARN("fail to assign sql", KR(ret), K(task_id_to_set), K(tenant_id)); + } else if (OB_FAIL(sql_proxy.read(res, sql_tenant_id, sql.ptr()))) { + LOG_WARN("execute sql failed", KR(ret), K(sql_tenant_id), K(sql)); + } else if (OB_ISNULL(result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get mysql result failed", KR(ret), K(sql)); + } else if (OB_FAIL(result->next())) { + if (OB_ITER_END == ret) { + ret = OB_ENTRY_NOT_EXIST; + } + LOG_WARN("fail to get next result", KR(ret), K(sql)); + } else { + EXTRACT_INT_FIELD_MYSQL(*result, "ls_id", ls_id_res, int64_t); + (void)GET_COL_IGNORE_NULL(result->get_varchar, "task_exec_svr_ip", server_ip); + (void)GET_COL_IGNORE_NULL(result->get_int, "task_exec_svr_port", server_port); + if (OB_FAIL(ret)) { + } else if (false == task_execute_server.set_ip_addr(server_ip, static_cast(server_port))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid server address", K(server_ip), K(server_port)); + } else if (OB_UNLIKELY(!task_execute_server.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid task_execute_server", KR(ret), K(task_execute_server)); + } else { + ls_id = ls_id_res; + } + int tmp_ret = OB_SUCCESS; + if (OB_SUCC(ret) && (OB_ITER_END != (tmp_ret = result->next()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get more row than one", KR(ret), KR(tmp_ret), K(sql)); + } + } + } + } + return ret; +} + +} // end namespace rootserver +} // end namespace oceanbase \ No newline at end of file diff --git a/src/rootserver/ob_disaster_recovery_task_table_operator.h b/src/rootserver/ob_disaster_recovery_task_table_operator.h new file mode 100644 index 000000000..888d5e826 --- /dev/null +++ b/src/rootserver/ob_disaster_recovery_task_table_operator.h @@ -0,0 +1,87 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#ifndef OCEANBASE_ROOTSERVER_OB_DISASTER_RECOVERY_TASK_TABLE_OPERATOR_H_ +#define OCEANBASE_ROOTSERVER_OB_DISASTER_RECOVERY_TASK_TABLE_OPERATOR_H_ +#include "lib/mysqlclient/ob_mysql_proxy.h" +#include "lib/ob_define.h" // for ObTaskId ObReplicaType .. +#include "lib/string/ob_string.h" // for ObString +#include "ob_disaster_recovery_task.h" // for ObDRTaskPriority +#include "ob_disaster_recovery_task_table_updater.h" // for ObDRTaskTableUpdateTask +#include "share/ob_ls_id.h" // for ObLSID + +namespace oceanbase +{ +namespace rootserver +{ + +class ObLSReplicaTaskTableOperator +{ + OB_UNIS_VERSION(1); +public: + ObLSReplicaTaskTableOperator() {} + virtual ~ObLSReplicaTaskTableOperator() {} + + // through tenant_id, ls_id, task_type and task_id, delete task from __all_ls_replica_task + // @params[in] trans, trans proxy to use + // @params[in] tenant_id, which tenant's task to delete + // @params[in] ls_id, which log stream's task to delete + // @params[in] task_type, type of target task + // @params[in] task_id, task_id of target task + // @params[out] affected_rows, number of rows to delete tasks + int delete_task( + common::ObMySQLTransaction &trans, + const uint64_t tenant_id, + const share::ObLSID& ls_id, + const ObDRTaskType& task_type, + const share::ObTaskId& task_id, + int64_t &affected_rows); + + // insert task into __all_ls_replica_task + // @params[in] trans, trans proxy to use + // @params[in] task, target task to insert + int insert_task( + common::ObISQLClient &sql_proxy, + const ObDRTask &task); + + // read task info and delete task from __all_ls_replica_task, insert task into __all_ls_replica_task_history + // @params[in] sql_proxy, proxy to use + // @params[in] task, contains task info which to finish + int finish_task( + common::ObMySQLTransaction& trans, + const ObDRTaskTableUpdateTask &task); + + // read task info for cancel replica task + // @params[in] tenant_id, tenant task to get + // @params[in] task_id, task_id to get + // @params[out] task_execute_server, which server the task execute + // @params[out] ls_id, which ls this task belong to + int get_task_info_for_cancel( + common::ObISQLClient &sql_proxy, + const uint64_t tenant_id, + const share::ObTaskId &task_id, + common::ObAddr &task_execute_server, + share::ObLSID &ls_id); + +private: + + int get_task_schedule_time_( + common::ObMySQLTransaction& trans, + const ObDRTaskTableUpdateTask &task, + int64_t &schedule_time); + +private: + DISALLOW_COPY_AND_ASSIGN(ObLSReplicaTaskTableOperator); +}; + +} // end namespace rootserver +} // end namespace oceanbase +#endif // OCEANBASE_ROOTSERVER_OB_DISASTER_RECOVERY_TASK_TABLE_OPERATOR_H_ diff --git a/src/rootserver/ob_disaster_recovery_task_table_updater.cpp b/src/rootserver/ob_disaster_recovery_task_table_updater.cpp index 32fd9cd9c..029564a04 100644 --- a/src/rootserver/ob_disaster_recovery_task_table_updater.cpp +++ b/src/rootserver/ob_disaster_recovery_task_table_updater.cpp @@ -16,6 +16,7 @@ #include "share/ob_define.h" #include "share/inner_table/ob_inner_table_schema_constants.h" // for OB_ALL_LS_REPLICA_TASK_TNAME +#include "ob_disaster_recovery_task_table_operator.h" //for ObLSReplicaTaskTableOperator #include "rootserver/ob_disaster_recovery_task_mgr.h" // for ObDRTaskMgr #include "share/schema/ob_multi_version_schema_service.h" // for GSCHEMASERVICE @@ -271,12 +272,9 @@ int ObDRTaskTableUpdater::process_task_( int ret = OB_SUCCESS; DEBUG_SYNC(BEFORE_DELETE_DRTASK_FROM_INNER_TABLE); common::ObMySQLTransaction trans; - int64_t affected_rows = 0; const uint64_t sql_tenant_id = gen_meta_tenant_id(task.get_tenant_id()); - char task_id_to_set[OB_TRACE_STAT_BUFFER_SIZE] = ""; - ObSqlString sql; bool has_dropped = false; - + ObLSReplicaTaskTableOperator task_operator; if (OB_FAIL(check_inner_stat_())) { LOG_WARN("fail to check inner stat", KR(ret), K_(inited), K_(stopped)); } else if (OB_ISNULL(sql_proxy_) || OB_ISNULL(task_mgr_)) { @@ -285,9 +283,6 @@ int ObDRTaskTableUpdater::process_task_( } else if (!task.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(task)); - } else if (false == task.get_task_id().to_string(task_id_to_set, sizeof(task_id_to_set))) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("convert task id to string failed", KR(ret), "task_id", task.get_task_id()); } else if (OB_UNLIKELY(!is_valid_tenant_id(task.get_tenant_id()))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), "tenant_id", task.get_tenant_id()); @@ -298,22 +293,8 @@ int ObDRTaskTableUpdater::process_task_( if (has_dropped) { } else if (OB_FAIL(trans.start(sql_proxy_, sql_tenant_id))) { LOG_WARN("start transaction failed", KR(ret), K(sql_tenant_id)); - } else { - if (OB_FAIL(sql.assign_fmt("DELETE FROM %s WHERE tenant_id = %lu AND ls_id = %lu " - "AND task_type = '%s' AND task_id = '%s'", - share::OB_ALL_LS_REPLICA_TASK_TNAME, - task.get_tenant_id(), - task.get_ls_id().id(), - ob_disaster_recovery_task_type_strs(task.get_task_type()), - task_id_to_set))) { - LOG_WARN("assign sql string failed", KR(ret), K(task)); - } else if (OB_FAIL(sql_proxy_->write(sql_tenant_id, sql.ptr(), affected_rows))) { - LOG_WARN("execute sql failed", KR(ret), "sql", sql.ptr(), K(task), K(sql_tenant_id)); - } else if (!is_single_row(affected_rows)) { - // ignore affected row check for task not exist - LOG_INFO("expected deleted single row", - K(affected_rows), K(sql), K(task), K(sql_tenant_id)); - } + } else if (OB_FAIL(task_operator.finish_task(trans, task))) { + LOG_WARN("task_operator get_task failed", KR(ret), K(task)); } if (FAILEDx(task_mgr_->do_cleaning( task.get_task_id(), @@ -325,7 +306,7 @@ int ObDRTaskTableUpdater::process_task_( LOG_WARN("fail to clean task info inside memory", KR(ret), K(task)); } else { LOG_INFO("success to delete row from ls disaster task table and do cleaning", - K(affected_rows), K(sql), K(task), K(sql_tenant_id)); + K(task), K(sql_tenant_id)); } if (trans.is_started()) { int trans_ret = trans.end(OB_SUCCESS == ret); diff --git a/src/rootserver/ob_disaster_recovery_worker.cpp b/src/rootserver/ob_disaster_recovery_worker.cpp index b5313b246..678796cc0 100755 --- a/src/rootserver/ob_disaster_recovery_worker.cpp +++ b/src/rootserver/ob_disaster_recovery_worker.cpp @@ -29,6 +29,7 @@ #include "storage/ls/ob_ls.h" #include "storage/tx_storage/ob_ls_handle.h" #include "observer/ob_server_struct.h" +#include "ob_disaster_recovery_task_table_operator.h" #include "rootserver/ob_disaster_recovery_task.h" #include "rootserver/ob_disaster_recovery_info.h" #include "rootserver/ob_disaster_recovery_task_mgr.h" @@ -2162,6 +2163,989 @@ int ObDRWorker::try_ls_disaster_recovery( return ret; } +int ObDRWorker::do_add_ls_replica_task( + const obrpc::ObAdminAlterLSReplicaArg &arg) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("DRWorker not init", KR(ret), K(arg)); + } else if (OB_UNLIKELY(!arg.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(arg)); + } else { + DRLSInfo dr_ls_info(gen_user_tenant_id(arg.get_tenant_id()), + zone_mgr_, schema_service_); + ObAddLSReplicaTask add_replica_task; + if (OB_FAIL(check_and_init_info_for_alter_ls_(arg, dr_ls_info))) { + LOG_WARN("fail to check and init info for alter ls", KR(ret), K(arg), K(dr_ls_info)); + } else if (OB_FAIL(build_add_replica_task_(arg, dr_ls_info, add_replica_task))) { + LOG_WARN("fail to build add replica task parameters", KR(ret), K(arg), K(dr_ls_info)); + } else if (OB_FAIL(add_task_in_queue_and_execute_(add_replica_task))) { + LOG_WARN("failed to add task in schedule list", KR(ret), K(add_replica_task)); + } + } + FLOG_INFO("ObDRWorker do add ls replica task", KR(ret), K(arg)); + return ret; +} + +int ObDRWorker::do_remove_ls_replica_task( + const obrpc::ObAdminAlterLSReplicaArg &arg) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("DRWorker not init", KR(ret), K(arg)); + } else if (OB_UNLIKELY(!arg.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(arg)); + } else { + DRLSInfo dr_ls_info(gen_user_tenant_id(arg.get_tenant_id()), + zone_mgr_, schema_service_); + ObRemoveLSReplicaTask remove_replica_task; + if (OB_FAIL(check_and_init_info_for_alter_ls_(arg, dr_ls_info))) { + LOG_WARN("fail to check and init info for alter ls", KR(ret), K(arg), K(dr_ls_info)); + } else if (OB_FAIL(build_remove_replica_task_(arg, dr_ls_info, remove_replica_task))) { + LOG_WARN("fail to build remove replica task parameters", KR(ret), K(arg), K(dr_ls_info)); + } else if (OB_FAIL(add_task_in_queue_and_execute_(remove_replica_task))) { + LOG_WARN("failed to add task in schedule list", KR(ret), K(remove_replica_task)); + } + } + FLOG_INFO("ObDRWorker do remove ls replica task", KR(ret), K(arg)); + return ret; +} + +int ObDRWorker::do_modify_ls_replica_type_task( + const obrpc::ObAdminAlterLSReplicaArg &arg) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("DRWorker not init", KR(ret), K(arg)); + } else if (OB_UNLIKELY(!arg.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(arg)); + } else { + DRLSInfo dr_ls_info(gen_user_tenant_id(arg.get_tenant_id()), + zone_mgr_, schema_service_); + ObLSTypeTransformTask modify_replica_type_task; + if (OB_FAIL(check_and_init_info_for_alter_ls_(arg, dr_ls_info))) { + LOG_WARN("fail to check and init info for alter ls", KR(ret), K(arg), K(dr_ls_info)); + } else if (OB_FAIL(build_modify_replica_type_task_(arg, dr_ls_info, modify_replica_type_task))) { + LOG_WARN("fail to build modify replica task parameters", KR(ret), K(arg), K(dr_ls_info)); + } else if (OB_FAIL(add_task_in_queue_and_execute_(modify_replica_type_task))) { + LOG_WARN("failed to add task in schedule list", KR(ret), K(modify_replica_type_task)); + } + } + FLOG_INFO("ObDRWorker do modify ls replica task", KR(ret), K(arg)); + return ret; +} + +int ObDRWorker::do_migrate_ls_replica_task( + const obrpc::ObAdminAlterLSReplicaArg &arg) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("DRWorker not init", KR(ret), K(arg)); + } else if (OB_UNLIKELY(!arg.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(arg)); + } else { + DRLSInfo dr_ls_info(gen_user_tenant_id(arg.get_tenant_id()), + zone_mgr_, schema_service_); + ObMigrateLSReplicaTask migrate_replica_task; + if (OB_FAIL(check_and_init_info_for_alter_ls_(arg, dr_ls_info))) { + LOG_WARN("fail to check and init info for alter ls", KR(ret), K(arg), K(dr_ls_info)); + } else if (OB_FAIL(build_migrate_replica_task_(arg, dr_ls_info, migrate_replica_task))) { + LOG_WARN("fail to build migrate replica task", KR(ret), K(arg), K(dr_ls_info)); + } else if (OB_FAIL(add_task_in_queue_and_execute_(migrate_replica_task))) { + LOG_WARN("failed to add task in schedule list", KR(ret), K(migrate_replica_task)); + } + } + FLOG_INFO("ObDRWorker do migrate ls replica task", KR(ret), K(arg)); + return ret; +} + +int ObDRWorker::do_modify_ls_paxos_replica_num_task( + const obrpc::ObAdminAlterLSReplicaArg &arg) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("DRWorker not init", KR(ret), K(arg)); + } else if (OB_UNLIKELY(!arg.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(arg)); + } else { + DRLSInfo dr_ls_info(gen_user_tenant_id(arg.get_tenant_id()), + zone_mgr_, schema_service_); + ObLSModifyPaxosReplicaNumberTask modify_paxos_replica_number_task; + if (OB_FAIL(check_and_init_info_for_alter_ls_(arg, dr_ls_info))) { + LOG_WARN("fail to check and init info for alter ls", KR(ret), K(arg), K(dr_ls_info)); + } else if (OB_FAIL(build_modify_paxos_replica_num_task_(arg, dr_ls_info, modify_paxos_replica_number_task))) { + LOG_WARN("fail to build modify paxos_replica_num task parameters", KR(ret), K(arg), K(dr_ls_info)); + } else if (OB_FAIL(add_task_in_queue_and_execute_(modify_paxos_replica_number_task))) { + LOG_WARN("failed to add task in schedule list", KR(ret), K(modify_paxos_replica_number_task)); + } + } + FLOG_INFO("ObDRWorker do modify ls paxos_replica_num task", KR(ret), K(arg)); + return ret; +} + +int ObDRWorker::do_cancel_ls_replica_task( + const obrpc::ObAdminAlterLSReplicaArg &arg) +{ + int ret = OB_SUCCESS; + common::ObAddr task_execute_server; + share::ObLSID ls_id; + share::ObTaskId task_id; + ObLSCancelReplicaTaskArg rpc_arg; + ObLSReplicaTaskTableOperator task_table_operator; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("DRWorker not init", KR(ret), K(arg)); + } else if (OB_UNLIKELY(!arg.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(arg)); + } else if (OB_ISNULL(rpc_proxy_) || OB_ISNULL(sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("some ptr is null", KR(ret), KP(rpc_proxy_), KP(sql_proxy_)); + } else if (OB_FAIL(task_id.set(arg.get_task_id().ptr()))) { + LOG_WARN("fail to set task_id", KR(ret), K(arg)); + } else if (OB_FAIL(task_table_operator.get_task_info_for_cancel( + *sql_proxy_, arg.get_tenant_id(), task_id, task_execute_server, ls_id))) { + if (OB_ENTRY_NOT_EXIST == ret) { + LOG_USER_ERROR(OB_ENTRY_NOT_EXIST, "Task not exist"); + } + LOG_WARN("get task info failed", KR(ret), K(task_id), K(arg)); + } else if (OB_FAIL(rpc_arg.init(task_id, ls_id, arg.get_tenant_id()))) { + LOG_WARN("fail to init arg", KR(ret), K(task_id), K(ls_id), K(arg)); + } else if (OB_FAIL(rpc_proxy_->to(task_execute_server).by(arg.get_tenant_id()).timeout(GCONF.rpc_timeout) + .ls_cancel_replica_task(rpc_arg))) { + if (OB_ENTRY_NOT_EXIST == ret) { + LOG_USER_ERROR(OB_ENTRY_NOT_EXIST, "Task not exist"); + } + LOG_WARN("fail to execute cancel", + KR(ret), K(arg), K(rpc_arg), K(task_execute_server), K(ls_id)); + } + FLOG_INFO("ObDRWorker do cancel ls replica task over", KR(ret), K(arg)); + return ret; +} + +int ObDRWorker::add_task_in_queue_and_execute_(const ObDRTask &task) +{ + int ret = OB_SUCCESS; + FLOG_INFO("add task in schedule list and execute", K(task)); + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("DRWorker not init", KR(ret)); + } else if (OB_UNLIKELY(!task.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(task)); + } else if (OB_ISNULL(disaster_recovery_task_mgr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("disaster_recovery_task_mgr_ null", KR(ret), KP(disaster_recovery_task_mgr_)); + } else if (OB_FAIL(disaster_recovery_task_mgr_->add_task_in_queue_and_execute(task))) { + if (OB_ENTRY_EXIST == ret) { + LOG_USER_ERROR(OB_ENTRY_EXIST, "LS has task executing, current operation is not allowed"); + LOG_WARN("task already exist in queue", KR(ret), K(task)); + } else { + LOG_WARN("push task in schedule list failed, unknow error", KR(ret), K(task)); + } + } + LOG_INFO("task has push in queue and execute over", KR(ret), K(task)); + return ret; +} + +int ObDRWorker::check_and_init_info_for_alter_ls_( + const obrpc::ObAdminAlterLSReplicaArg& arg, + DRLSInfo& dr_ls_info) +{ + int ret = OB_SUCCESS; + share::ObLSInfo ls_info; + share::ObLSStatusInfo ls_status_info; + const share::ObLSReplica *leader_replica = nullptr; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("DRWorker not init", KR(ret)); + } else if (OB_UNLIKELY(!arg.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(arg)); + } else if (OB_FAIL(check_ls_exist_and_get_ls_info_(arg.get_ls_id(), + arg.get_tenant_id(), ls_info, ls_status_info))) { + LOG_WARN("fail to check tenent ls", KR(ret), K(arg)); + } else if (OB_FAIL(ls_info.find_leader(leader_replica))) { + LOG_WARN("fail to find leader", KR(ret), K(ls_info), K(arg)); + if (OB_ENTRY_NOT_EXIST == ret) { + ret = OB_LEADER_NOT_EXIST; + LOG_USER_ERROR(OB_LEADER_NOT_EXIST); + } + } else if (OB_FAIL(dr_ls_info.init())) { + LOG_WARN("fail to init dr log stream info", KR(ret), K(dr_ls_info)); + } else if (OB_FAIL(dr_ls_info.build_disaster_ls_info( + ls_info, ls_status_info, false/*filter_readonly_replicas_with_flag*/))) { + LOG_WARN("fail to generate dr log stream info", KR(ret), K(ls_info), K(ls_status_info)); + } + return ret; +} + +int ObDRWorker::check_ls_exist_and_get_ls_info_( + const share::ObLSID& ls_id, + const int64_t tenant_id, + share::ObLSInfo& ls_info, + share::ObLSStatusInfo& ls_status_info) +{ + int ret = OB_SUCCESS; + share::ObLSStatusOperator ls_status_operator; + ls_info.reset(); + ls_status_info.reset(); + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("DRWorker not init", KR(ret)); + } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id)) || OB_UNLIKELY(!ls_id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(ls_id)); + } else if (!ls_id.is_valid_with_tenant(tenant_id)) { + ret = OB_ENTRY_NOT_EXIST; + LOG_USER_ERROR(OB_ENTRY_NOT_EXIST, "LS does not exist"); + LOG_WARN("check ls_id is_valid_with_tenant failed", KR(ret), K(tenant_id), K(ls_id)); + } else if (OB_ISNULL(lst_operator_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("lst_operator_ is null", KR(ret), KP(lst_operator_)); + } else if (OB_FAIL(lst_operator_->get(GCONF.cluster_id, tenant_id, ls_id, + share::ObLSTable::COMPOSITE_MODE, ls_info))) { + LOG_WARN("get ls info failed", KR(ret), K(tenant_id), K(ls_id)); + } else if (ls_info.get_replicas().count() == 0) { + ret = OB_ENTRY_NOT_EXIST; + LOG_USER_ERROR(OB_ENTRY_NOT_EXIST, "LS does not exist"); + LOG_WARN("ls_info.get_replicas().count() == 0", KR(ret), K(tenant_id), K(ls_id), K(ls_info)); + } else if (OB_ISNULL(sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql_proxy_ is null", KR(ret), KP(sql_proxy_)); + } else if (OB_FAIL(ls_status_operator.get_ls_status_info( + tenant_id, ls_id, ls_status_info, *sql_proxy_))) { + if (OB_ENTRY_NOT_EXIST == ret) { + LOG_USER_ERROR(OB_ENTRY_NOT_EXIST, "LS does not exist"); + } + LOG_WARN("fail to get all ls status", KR(ret), K(tenant_id), K(ls_id), K(ls_info), KP(sql_proxy_)); + } else if (ls_status_info.ls_is_creating() || ls_status_info.ls_is_create_abort()) { + ret = OB_OP_NOT_ALLOW; + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "LS is in CREATING or CREATE_ABORT status, current operation is"); + LOG_WARN("LS is creating, current operation is", KR(ret), K(tenant_id), K(ls_id), K(ls_info)); + } + return ret; +} + +int ObDRWorker::check_unit_exist_and_get_unit_( + const common::ObAddr &task_execute_server, + const uint64_t tenant_id, + const bool is_migrate_source_valid, + share::ObUnit& unit) +{ + int ret = OB_SUCCESS; + unit.reset(); + ObUnitTableOperator unit_operator; + common::ObArray unit_info_array; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("DRWorker not init", KR(ret)); + } else if (OB_UNLIKELY(!task_execute_server.is_valid() + || OB_INVALID_TENANT_ID == tenant_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(task_execute_server), K(tenant_id)); + } else if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql_proxy_ is null", KR(ret), KP(GCTX.sql_proxy_)); + } else if (OB_FAIL(unit_operator.init(*GCTX.sql_proxy_))) { + LOG_WARN("unit operator init failed", KR(ret), KP(GCTX.sql_proxy_)); + } else if (OB_FAIL(unit_operator.get_units_by_tenant( + gen_user_tenant_id(tenant_id), unit_info_array))) { + LOG_WARN("fail to get unit info array", KR(ret), K(tenant_id)); + } else { + bool found = false; + for (int64_t i = 0; OB_SUCC(ret) && !found && i < unit_info_array.count(); ++i) { + if ((unit_info_array.at(i).server_ == task_execute_server) + || (is_migrate_source_valid && unit_info_array.at(i).migrate_from_server_ == task_execute_server)) { + if (OB_FAIL(unit.assign(unit_info_array.at(i)))) { + LOG_WARN("fail to assign unit", KR(ret), K(unit_info_array.at(i))); + } else { + found = true; + } + } + } + if (OB_SUCC(ret) && !found) { + ret = OB_ENTRY_NOT_EXIST; + LOG_USER_ERROR(OB_ENTRY_NOT_EXIST, "Tenant has no unit on the server"); + LOG_WARN("this tenant has no unit on the server", + KR(ret), K(tenant_id), K(task_execute_server), K(found)); + } + } + return ret; +} + +int ObDRWorker::check_task_execute_server_status_( + const common::ObAddr &task_execute_server, + const bool need_check_can_migrate_in) +{ + int ret = OB_SUCCESS; + ObServerInfoInTable server_info; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("DRWorker not init", KR(ret)); + } else if (OB_UNLIKELY(!task_execute_server.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(task_execute_server)); + } else { + char err_msg[OB_MAX_ERROR_MSG_LEN] = {0}; + char addr_str_buf[128] = {0}; + if (OB_FAIL(task_execute_server.ip_port_to_string(addr_str_buf, 128))) { + LOG_WARN("fail to get server addr string", KR(ret), K(task_execute_server)); + } else if (OB_FAIL(SVR_TRACER.get_server_info(task_execute_server, server_info))) { + LOG_WARN("fail to check server active", KR(ret), K(task_execute_server)); + } else if (!server_info.is_alive()) { + snprintf(err_msg, sizeof(err_msg), + "The task needs to be executed on %s which status is not alive and the current operation is", addr_str_buf); + ret = OB_OP_NOT_ALLOW; + LOG_USER_ERROR(OB_OP_NOT_ALLOW, err_msg); + LOG_WARN("server is not active", KR(ret), K(task_execute_server), K(server_info)); + } else if (need_check_can_migrate_in && !server_info.can_migrate_in()) { + snprintf(err_msg, sizeof(err_msg), + "The task needs to be executed on %s which can not migrate in and the current operation is", addr_str_buf); + ret = OB_OP_NOT_ALLOW; + LOG_USER_ERROR(OB_OP_NOT_ALLOW, err_msg); + LOG_WARN("server can not migrate in now", KR(ret), K(task_execute_server), K(server_info)); + } + } + return ret; +} + +int ObDRWorker::get_replica_type_by_leader_( + const common::ObAddr& server_addr, + const DRLSInfo &dr_ls_info, + common::ObReplicaType& replica_type) +{ + // not leader replica get replica type may not right. when remove or modify replica, + // replica type wrong may result in fatal error, so get it by leader + int ret = OB_SUCCESS; + replica_type = REPLICA_TYPE_MAX; + common::ObAddr leader_addr; // not used + GlobalLearnerList learner_list; + common::ObMemberList member_list; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("DRWorker not init", KR(ret)); + } else if (OB_UNLIKELY(!server_addr.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(server_addr)); + } else if (OB_UNLIKELY(0 >= dr_ls_info.get_member_list_cnt())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("leader member list has no member", KR(ret), K(dr_ls_info)); + } else if (OB_FAIL(dr_ls_info.get_leader_and_member_list(leader_addr, member_list, learner_list))) { + LOG_WARN("fail to get leader and member list", KR(ret), K(server_addr), K(dr_ls_info)); + } else if (member_list.contains(server_addr)) { + replica_type = REPLICA_TYPE_FULL; + } else if (learner_list.contains(server_addr)) { + replica_type = REPLICA_TYPE_READONLY; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to find server in leader member list and learner list", + KR(ret), K(server_addr), K(dr_ls_info), K(learner_list), K(member_list)); + } + return ret; +} + +int ObDRWorker::build_add_replica_task_( + const obrpc::ObAdminAlterLSReplicaArg &arg, + const DRLSInfo &dr_ls_info, + ObAddLSReplicaTask &add_replica_task) +{ + int ret = OB_SUCCESS; + share::ObUnit unit; + share::ObLSReplica ls_replica; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("DRWorker not init", KR(ret)); + } else if (OB_UNLIKELY(!arg.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(arg)); + } else if (OB_FAIL(dr_ls_info.check_replica_exist_and_get_ls_replica(arg.get_server_addr(), ls_replica))) { + LOG_WARN("fail to check and get replica by server", KR(ret), K(arg)); + } else if (ls_replica.is_valid()) { + ret = OB_ENTRY_EXIST; + LOG_USER_ERROR(OB_ENTRY_EXIST, "Target server already has a replica"); + LOG_WARN("server already has a replica of this type, cannot add more", + KR(ret), K(arg), K(dr_ls_info), K(ls_replica)); + } else if (OB_FAIL(check_unit_exist_and_get_unit_( + arg.get_server_addr(), arg.get_tenant_id(), false/*is_migrate_source_valid*/, unit))) { + LOG_WARN("fail to check unit exist and get unit", KR(ret), K(arg)); + } else if (OB_FAIL(check_task_execute_server_status_(arg.get_server_addr(), true/*need_check_can_migrate_in*/))) { + LOG_WARN("fail to check server status", KR(ret), K(arg), K(dr_ls_info)); + } else if (REPLICA_TYPE_FULL == arg.get_replica_type() + && share::ObLSReplica::DEFAULT_REPLICA_COUNT == dr_ls_info.get_member_list_cnt()) { + ret = OB_OP_NOT_ALLOW; + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "Member list count has reached the limit, alter ls replica is"); + LOG_WARN("number of F replica has reached the limit", KR(ret), K(arg), K(dr_ls_info)); + } else { + share::ObTaskId task_id; + ObDstReplica dst_replica; + ObReplicaMember data_source; + int64_t data_size = 0; + ObReplicaMember force_data_source; + int64_t new_paxos_replica_number = 0; + ObReplicaMember dst_member(arg.get_server_addr(), + ObTimeUtility::current_time(), + arg.get_replica_type()); + if (FALSE_IT(task_id.init(self_addr_))) { + } else if (OB_FAIL(dst_replica.assign(unit.unit_id_, unit.unit_group_id_, unit.zone_, dst_member))) { + LOG_WARN("fail to assign dst replica", KR(ret), K(unit), K(dst_member)); + } else if (OB_FAIL(dr_ls_info.get_default_data_source(data_source, data_size))) { + LOG_WARN("fail to get data_size", KR(ret), K(dr_ls_info)); + } else if (OB_FAIL(check_data_source_available_and_init_(arg, arg.get_replica_type(), dr_ls_info, force_data_source))) { + LOG_WARN("fail to check and get data source", KR(ret), K(arg), K(dr_ls_info)); + } else if (OB_FAIL(check_and_generate_new_paxos_replica_num_( + arg, arg.get_replica_type(), dr_ls_info, new_paxos_replica_number))) { + LOG_WARN("fail to check and generate new paxos replica num", KR(ret), K(arg), K(dr_ls_info)); + } else if (OB_FAIL(add_replica_task.simple_build( + arg.get_tenant_id(), + arg.get_ls_id(), + task_id, + dst_replica, + data_source, + force_data_source, + dr_ls_info.get_paxos_replica_number(), + new_paxos_replica_number))) { + LOG_WARN("fail to build add replica task", KR(ret), K(arg), K(task_id), + K(dst_replica), K(data_source), K(force_data_source), K(dr_ls_info), K(new_paxos_replica_number)); + } + } + return ret; +} + +int ObDRWorker::build_remove_replica_task_( + const obrpc::ObAdminAlterLSReplicaArg &arg, + DRLSInfo &dr_ls_info, + ObRemoveLSReplicaTask &remove_replica_task) +{ + int ret = OB_SUCCESS; + common::ObReplicaType replica_type = REPLICA_TYPE_MAX; + common::ObAddr leader_addr; + ObMember member_to_remove; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("DRWorker not init", KR(ret)); + } else if (OB_UNLIKELY(!arg.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(arg)); + } else if (OB_FAIL(dr_ls_info.get_member_by_server(arg.get_server_addr(), member_to_remove))) { + if (OB_ENTRY_NOT_EXIST == ret) { + LOG_USER_ERROR(OB_ENTRY_NOT_EXIST, "Target server does not have a replica of this LS"); + } + LOG_WARN("server does not have a replica of this log stream", KR(ret), K(arg), K(dr_ls_info)); + } else if (OB_FAIL(dr_ls_info.get_leader(leader_addr))) { + LOG_WARN("fail to get leader", KR(ret), K(dr_ls_info)); + } else if (OB_FAIL(check_task_execute_server_status_(leader_addr, false/*need_check_can_migrate_in*/))) { + LOG_WARN("fail to check server status", KR(ret), K(leader_addr), K(dr_ls_info)); + } else if (OB_FAIL(get_replica_type_by_leader_(arg.get_server_addr(), dr_ls_info, replica_type))) { + LOG_WARN("fail to get_replica_type_by_leader", KR(ret), K(dr_ls_info), K(arg)); + } else { + share::ObTaskId task_id; + int64_t new_paxos_replica_number = 0; + bool has_leader = false; + ObReplicaMember remove_member(member_to_remove); + if (FALSE_IT(task_id.init(self_addr_))) { + } else if (OB_FAIL(remove_member.set_replica_type(replica_type))) { + LOG_WARN("fail to set replica type", KR(ret), K(replica_type), K(remove_member)); + } else if (OB_FAIL(check_and_generate_new_paxos_replica_num_( + arg, replica_type, dr_ls_info, new_paxos_replica_number))) { + LOG_WARN("fail to check and generate new paxos replica num", KR(ret), K(arg), K(replica_type), K(dr_ls_info)); + } else if (REPLICA_TYPE_FULL == replica_type + && OB_FAIL(check_majority_for_remove_(arg.get_server_addr(), dr_ls_info, new_paxos_replica_number))) { + LOG_WARN("check provided paxos_replica_num failed.", KR(ret), + K(arg), K(dr_ls_info), K(new_paxos_replica_number)); + } else if (OB_FAIL(remove_replica_task.simple_build( + arg.get_tenant_id(), + arg.get_ls_id(), + task_id, + leader_addr, + remove_member, + dr_ls_info.get_paxos_replica_number(), + new_paxos_replica_number, + replica_type))) { + LOG_WARN("fail to build task", KR(ret), K(arg), K(task_id), K(leader_addr), K(remove_member), + K(dr_ls_info), K(new_paxos_replica_number), K(replica_type)); + } + } + return ret; +} + +int ObDRWorker::build_modify_replica_type_task_( + const obrpc::ObAdminAlterLSReplicaArg &arg, + DRLSInfo &dr_ls_info, + ObLSTypeTransformTask &modify_replica_task) +{ + int ret = OB_SUCCESS; + share::ObUnit unit; + share::ObLSReplica ls_replica; + common::ObReplicaType replica_type = REPLICA_TYPE_MAX; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("DRWorker not init", KR(ret)); + } else if (OB_UNLIKELY(!arg.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(arg)); + } else if (OB_FAIL(dr_ls_info.check_replica_exist_and_get_ls_replica(arg.get_server_addr(), ls_replica))) { + LOG_WARN("fail to check and get replica by server", KR(ret), K(arg)); + } else if (!ls_replica.is_valid() || (ls_replica.is_valid() && !ls_replica.is_in_service())) { + ret = OB_ENTRY_NOT_EXIST; + LOG_USER_ERROR(OB_ENTRY_NOT_EXIST, "Target server does not have a replica of this LS"); + LOG_WARN("server does not have a replica of this log stream", + KR(ret), K(arg), K(dr_ls_info), K(ls_replica)); + } else if (OB_FAIL(check_unit_exist_and_get_unit_( + arg.get_server_addr(), arg.get_tenant_id(), true/*is_migrate_source_valid*/, unit))) { + LOG_WARN("fail to check unit exist and get unit", KR(ret), K(arg)); + } else if (OB_FAIL(check_task_execute_server_status_(arg.get_server_addr(), false/*need_check_can_migrate_in*/))) { + LOG_WARN("fail to check server status", KR(ret), K(arg), K(dr_ls_info)); + } else if (OB_FAIL(get_replica_type_by_leader_(arg.get_server_addr(), dr_ls_info, replica_type))) { + LOG_WARN("fail to get_replica_type_by_leader", KR(ret), K(dr_ls_info), K(arg)); + } else if (arg.get_replica_type() == replica_type) { + ret = OB_ENTRY_EXIST; + LOG_USER_ERROR(OB_ENTRY_EXIST, "Current replica type is same as the target type, no need to modify"); + LOG_WARN("replica type is the same as the target type, no need to modify type", + KR(ret), K(arg), K(replica_type)); + } else if (REPLICA_TYPE_FULL == arg.get_replica_type() + && share::ObLSReplica::DEFAULT_REPLICA_COUNT == dr_ls_info.get_member_list_cnt()) { + ret = OB_OP_NOT_ALLOW; + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "Member list count has reached the limit, alter ls replica is"); + LOG_WARN("number of F replica has reached the limit", KR(ret), K(replica_type), K(dr_ls_info)); + } else { + bool has_leader = false; + int64_t new_paxos_replica_number = 0; + share::ObTaskId task_id; + ObDstReplica dst_replica; + int64_t data_size = 0; + ObReplicaMember data_source; + ObReplicaMember src_member(ls_replica.get_server(), + ls_replica.get_member_time_us(), + replica_type, + ls_replica.get_memstore_percent()); + ObReplicaMember dst_member(ls_replica.get_server(), + ObTimeUtility::current_time(), + arg.get_replica_type(), + ls_replica.get_memstore_percent()); + if (FALSE_IT(task_id.init(self_addr_))) { + } else if (OB_FAIL(dst_replica.assign(unit.unit_id_, unit.unit_group_id_, + unit.zone_, dst_member))) { + LOG_WARN("fail to assign dst replica", KR(ret), K(unit), K(dst_member)); + } else if (OB_FAIL(check_and_generate_new_paxos_replica_num_( + arg, arg.get_replica_type(), dr_ls_info, new_paxos_replica_number))) { + LOG_WARN("fail to check and generate new paxos replica num", KR(ret), K(arg), K(dr_ls_info)); + } else if (REPLICA_TYPE_READONLY == arg.get_replica_type() + && OB_FAIL(check_majority_for_remove_(arg.get_server_addr(), dr_ls_info, new_paxos_replica_number))) { + LOG_WARN("check provided paxos_replica_num failed.", KR(ret), + K(arg), K(dr_ls_info), K(new_paxos_replica_number)); + } else if (OB_FAIL(dr_ls_info.get_default_data_source(data_source, data_size))) { + LOG_WARN("fail to get data_size", KR(ret), K(dr_ls_info)); + } else if (OB_FAIL(modify_replica_task.simple_build( + arg.get_tenant_id(), + arg.get_ls_id(), + task_id, + dst_replica, + src_member, + data_source, + dr_ls_info.get_paxos_replica_number(), + new_paxos_replica_number))) { + LOG_WARN("fail to build type transform task", KR(ret), K(arg), K(task_id), K(dst_replica), + K(src_member), K(data_source), K(dr_ls_info), K(new_paxos_replica_number)); + } + } + return ret; +} + +int ObDRWorker::build_migrate_replica_task_( + const obrpc::ObAdminAlterLSReplicaArg &arg, + const DRLSInfo &dr_ls_info, + ObMigrateLSReplicaTask &migrate_replica_task) +{ + int ret = OB_SUCCESS; + share::ObUnit destination_unit; + common::ObReplicaType replica_type = REPLICA_TYPE_MAX; + share::ObLSReplica desti_ls_replica; + share::ObLSReplica source_ls_replica; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("DRWorker not init", KR(ret)); + } else if (OB_UNLIKELY(!arg.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(arg)); + } else if (OB_FAIL(dr_ls_info.check_replica_exist_and_get_ls_replica( + arg.get_server_addr(), source_ls_replica))) { + LOG_WARN("fail to check and get replica by server", KR(ret), K(arg)); + } else if (!source_ls_replica.is_valid() || (source_ls_replica.is_valid() && !source_ls_replica.is_in_service())) { + ret = OB_ENTRY_NOT_EXIST; + LOG_USER_ERROR(OB_ENTRY_NOT_EXIST, "Source server does not have a replica of this LS"); + LOG_WARN("source server does not have a replica of this LS", + KR(ret), K(arg), K(dr_ls_info), K(source_ls_replica)); + } else if (OB_FAIL(dr_ls_info.check_replica_exist_and_get_ls_replica( + arg.get_destination_addr(), desti_ls_replica))) { + LOG_WARN("fail to check and get replica by server", KR(ret), K(arg)); + } else if (desti_ls_replica.is_valid()) { + ret = OB_ENTRY_EXIST; + LOG_USER_ERROR(OB_ENTRY_EXIST, "The destination server already has a replica"); + LOG_WARN("target server already has a replica, no need migrate", KR(ret), K(arg), K(dr_ls_info)); + } else if (OB_FAIL(check_unit_exist_and_get_unit_( + arg.get_destination_addr(), arg.get_tenant_id(), false/*is_migrate_source_valid*/, destination_unit))) { + LOG_WARN("fail to check unit exist and get unit", KR(ret), K(arg)); + } else if (OB_FAIL(check_task_execute_server_status_(arg.get_destination_addr(), true/*need_check_can_migrate_in*/))) { + LOG_WARN("fail to check server status", KR(ret), K(arg), K(dr_ls_info)); + } else if (OB_FAIL(get_replica_type_by_leader_(arg.get_server_addr(), dr_ls_info, replica_type))) { + LOG_WARN("fail to get_replica_type_by_leader", KR(ret), K(dr_ls_info), K(arg)); + } else if (destination_unit.zone_ != source_ls_replica.get_zone()) { + ret = OB_OP_NOT_ALLOW; + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "Migrate replica can only be in the same zone, current operation"); + LOG_WARN("migration replica can only be in the same zone", KR(ret), + K(destination_unit), K(source_ls_replica.get_zone())); + } else if (REPLICA_TYPE_FULL == replica_type + && share::ObLSReplica::DEFAULT_REPLICA_COUNT == dr_ls_info.get_member_list_cnt()) { + ret = OB_OP_NOT_ALLOW; + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "Member list count has reached the limit, alter ls replica is"); + LOG_WARN("number of F replica has reached the limit", KR(ret), K(replica_type), K(dr_ls_info)); + } else { + share::ObTaskId task_id; + ObDstReplica dst_replica; + ObReplicaMember data_source; + int64_t data_size = 0; + ObReplicaMember force_data_source; + ObReplicaMember src_member(source_ls_replica.get_server(), + source_ls_replica.get_member_time_us(), + replica_type, + source_ls_replica.get_memstore_percent()); + ObReplicaMember dst_member(arg.get_destination_addr(), + ObTimeUtility::current_time(), + replica_type, + source_ls_replica.get_memstore_percent()); + if (FALSE_IT(task_id.init(self_addr_))) { + } else if (OB_FAIL(dst_replica.assign(destination_unit.unit_id_, destination_unit.unit_group_id_, + source_ls_replica.get_zone(), dst_member))) { + LOG_WARN("fail to assign dst replica", + KR(ret), K(destination_unit), K(dst_member), K(source_ls_replica)); + } else if (OB_FAIL(check_data_source_available_and_init_(arg, replica_type, dr_ls_info, force_data_source))) { + LOG_WARN("fail to check and get data source", KR(ret), K(arg), K(replica_type), K(dr_ls_info)); + } else if (OB_FAIL(dr_ls_info.get_default_data_source(data_source, data_size))) { + LOG_WARN("fail to get data_size", KR(ret), K(dr_ls_info)); + } else if (OB_FAIL(migrate_replica_task.simple_build( + arg.get_tenant_id(), + arg.get_ls_id(), + task_id, + dst_replica, + src_member, + data_source, + force_data_source, + dr_ls_info.get_paxos_replica_number()))) { + LOG_WARN("fail to build migrate task", KR(ret), K(arg), K(task_id), K(dst_replica), + K(src_member), K(data_source), K(force_data_source), K(dr_ls_info)); + } + } + return ret; +} + +int ObDRWorker::build_modify_paxos_replica_num_task_( + const obrpc::ObAdminAlterLSReplicaArg &arg, + DRLSInfo &dr_ls_info, + ObLSModifyPaxosReplicaNumberTask &modify_paxos_replica_number_task) +{ + int ret = OB_SUCCESS; + share::ObTaskId task_id; + common::ObAddr leader_addr; + GlobalLearnerList learner_list; + common::ObMemberList member_list; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("DRWorker not init", KR(ret)); + } else if (OB_UNLIKELY(!arg.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(arg)); + } else if (FALSE_IT(task_id.init(self_addr_))) { + } else if (OB_FAIL(dr_ls_info.get_leader_and_member_list(leader_addr, member_list, learner_list))) { + LOG_WARN("fail to get leader and member list", KR(ret), K(arg), K(dr_ls_info)); + } else if (dr_ls_info.get_paxos_replica_number() <= arg.get_paxos_replica_num()) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "paxos_replica_num which should be less than current paxos_replica_num"); + LOG_WARN("paxos_replica_num invalid", KR(ret), K(arg), K(dr_ls_info)); + } else if (member_list.get_member_number() < majority(arg.get_paxos_replica_num())) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "paxos_replica_num which should be satisfy majority"); + LOG_WARN("number of replicas and paxos_replica_num do not satisfy majority", + KR(ret), K(arg), K(dr_ls_info), K(member_list)); + } else if (member_list.get_member_number() > arg.get_paxos_replica_num()) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "paxos_replica_num which should be greater or equal with member list count"); + LOG_WARN("member_list.get_member_number() > arg.get_paxos_replica_num()", + KR(ret), K(arg), K(dr_ls_info), K(member_list)); + } else if (OB_FAIL(check_task_execute_server_status_(leader_addr, false/*need_check_can_migrate_in*/))) { + LOG_WARN("fail to check server status", KR(ret), K(arg), K(dr_ls_info)); + } else if (OB_FAIL(modify_paxos_replica_number_task.simple_build( + arg.get_tenant_id(), + arg.get_ls_id(), + task_id, + leader_addr, + dr_ls_info.get_paxos_replica_number(), + arg.get_paxos_replica_num(), + member_list))) { + LOG_WARN("fail to build a modify paxos replica number task", + KR(ret), K(arg), K(task_id), K(leader_addr), K(dr_ls_info), K(member_list)); + } + return ret; +} + +int ObDRWorker::check_data_source_available_and_init_( + const obrpc::ObAdminAlterLSReplicaArg &arg, + const common::ObReplicaType &replica_type, + const DRLSInfo &dr_ls_info, + ObReplicaMember &data_source) +{ + /* + check if data_source is available, check the following conditions: + 1. replica exist and in service + 2. ls replica not restore failed + 3. F replica can be used as the data source of R replica and F replica, + R replica can only used as the data source of R replica. + 4. server status is alive and not stopped + */ + int ret = OB_SUCCESS; + data_source.reset(); + share::ObLSReplica ls_replica; + ObServerInfoInTable server_info; + common::ObReplicaType provide_replica_type = REPLICA_TYPE_MAX; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("DRWorker not init", KR(ret)); + } else if (!arg.get_data_source().is_valid()) { + // passed + LOG_INFO("data_source is not valid", KR(ret), K(arg)); + } else if (OB_UNLIKELY(!arg.is_valid() + || OB_UNLIKELY(replica_type != REPLICA_TYPE_FULL && replica_type != REPLICA_TYPE_READONLY))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(arg), K(replica_type)); + } else { + ObDataSourceCandidateChecker type_checker(replica_type); + if (OB_FAIL(dr_ls_info.check_replica_exist_and_get_ls_replica(arg.get_data_source(), ls_replica))) { + LOG_WARN("fail to get ls replica", KR(ret), K(dr_ls_info)); + } else if (!ls_replica.is_valid() || (ls_replica.is_valid() && !ls_replica.is_in_service())) { + ret = OB_OP_NOT_ALLOW; + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "The data source server has no replica, which is"); + LOG_WARN("source server has no replica", KR(ret), K(arg), K(ls_replica)); + } else if (OB_FAIL(get_replica_type_by_leader_(arg.get_data_source(), dr_ls_info, provide_replica_type))) { + LOG_WARN("get replica type by leader error", KR(ret), K(arg), K(dr_ls_info)); + } else if (ls_replica.get_restore_status().is_failed()) { + ret = OB_OP_NOT_ALLOW; + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "Data source replica restore or clone failed, which is"); + LOG_WARN("ls replica restore failed", KR(ret), K(arg), K(ls_replica)); + } else if (!type_checker.is_candidate(provide_replica_type)) { + ret = OB_OP_NOT_ALLOW; + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "R replica is not supported as the source of F replica, which is"); + LOG_WARN("type_checker failed", KR(ret), K(arg), K(provide_replica_type)); + } else if (OB_FAIL(SVR_TRACER.get_server_info(arg.get_data_source(), server_info))) { + LOG_WARN("fail to get server info", KR(ret), K(arg)); + } else if (!server_info.is_alive()) { + ret = OB_OP_NOT_ALLOW; + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "The data source server is not alive, which is"); + LOG_WARN("data source server is not alive", KR(ret), K(arg), K(server_info)); + } else if (server_info.is_stopped()) { + ret = OB_OP_NOT_ALLOW; + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "The data source server is stopped, which is"); + LOG_WARN("data source server is stopped", KR(ret), K(arg), K(server_info)); + } else { + data_source = ObReplicaMember(ls_replica.get_server(), + ls_replica.get_member_time_us(), + provide_replica_type, // attention + ls_replica.get_memstore_percent()); + } + } + return ret; +} + +int ObDRWorker::check_for_alter_full_replica_( + const int64_t member_list_count, + const int64_t new_p) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("DRWorker not init", KR(ret)); + } else if (OB_UNLIKELY(member_list_count <= 0 || new_p <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(member_list_count), K(new_p)); + } else if (member_list_count < majority(new_p)) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "paxos_replica_num which does not satisfy majority"); + LOG_WARN("paxos_replica_num is wrong", KR(ret), K(member_list_count), K(new_p)); + } else if (member_list_count > new_p) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "paxos_replica_num which should be greater or equal with member list count"); + LOG_WARN("paxos_replica_num is wrong", KR(ret), K(member_list_count), K(new_p)); + } + return ret; +} + +int ObDRWorker::check_and_generate_new_paxos_replica_num_( + const obrpc::ObAdminAlterLSReplicaArg &arg, + const common::ObReplicaType &replica_type, + const DRLSInfo &dr_ls_info, + int64_t &new_p) +{ + /* + If the user provides paxos_replica_num, use the user-provided paxos_replica_num, + otherwise keep paxos_replica_num unchanged. + The change range of paxos_replica_num is limited to 1. + If the change is too large, an error will be reported to the user. + Different member_change_types are generated based on the task type and the copy type of the operation. + Special: modify R->F is equivalent to MEMBER_CHANGE_ADD, modify F->R is equivalent to MEMBER_CHANGE_SUB + */ + int ret = OB_SUCCESS; + int64_t curr_p = dr_ls_info.get_paxos_replica_number(); + int64_t provided_p = arg.get_paxos_replica_num(); + int64_t member_list_count = dr_ls_info.get_member_list_cnt(); + new_p = provided_p > 0 ? provided_p : curr_p; + obrpc::ObAlterLSReplicaTaskType task_type = arg.get_alter_task_type(); + MemberChangeType member_change_type = MEMBER_CHANGE_NOP; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("DRWorker not init", KR(ret)); + } else if (OB_UNLIKELY(!arg.is_valid() + || (REPLICA_TYPE_FULL != replica_type && REPLICA_TYPE_READONLY != replica_type))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(arg), K(replica_type)); + } else if (std::abs(new_p - curr_p) > 1) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "paxos_replica_num which change cannot be greater than 1"); + LOG_WARN("paxos_replica_num is wrong", KR(ret), K(arg), K(new_p), K(curr_p)); + } else if (task_type.is_add_task()) { + if (REPLICA_TYPE_FULL == replica_type) { + member_change_type = MEMBER_CHANGE_ADD; + } else if (REPLICA_TYPE_READONLY == replica_type) { + member_change_type = MEMBER_CHANGE_NOP; + } + } else if (task_type.is_remove_task()) { + if (REPLICA_TYPE_FULL == replica_type) { + member_change_type = MEMBER_CHANGE_SUB; + } else if (REPLICA_TYPE_READONLY == replica_type) { + member_change_type = MEMBER_CHANGE_NOP; + } + } else if (task_type.is_modify_replica_task()) { + if (REPLICA_TYPE_FULL == replica_type) { + member_change_type = MEMBER_CHANGE_ADD; + } else if (REPLICA_TYPE_READONLY == replica_type) { + member_change_type = MEMBER_CHANGE_SUB; + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("task type unexpected", KR(ret), K(arg), K(dr_ls_info), K(replica_type)); + } + if (OB_FAIL(ret)) { + } else if ((MEMBER_CHANGE_ADD == member_change_type)) { + if (OB_FAIL(check_for_alter_full_replica_(member_list_count + 1, new_p))) { + LOG_WARN("check failed", KR(ret), K(arg), K(replica_type), K(dr_ls_info)); + } + } else if ((MEMBER_CHANGE_SUB == member_change_type)) { + if (OB_FAIL(check_for_alter_full_replica_(member_list_count - 1, new_p))) { + LOG_WARN("check failed", KR(ret), K(arg), K(replica_type), K(dr_ls_info)); + } + } else if ((MEMBER_CHANGE_NOP == member_change_type)) { + if (new_p != curr_p) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "paxos_replica_num which should remain unchanged"); + LOG_WARN("paxos_replica_num is wrong", KR(ret), K(arg), K(curr_p), K(new_p)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("member change type unexpected", KR(ret), K(arg), K(dr_ls_info), K(member_change_type)); + } + LOG_INFO("check and generate new paxos_replica_num over", + KR(ret), K(arg), K(replica_type), K(new_p), K(curr_p), K(member_list_count)); + return ret; +} + +int ObDRWorker::check_majority_for_remove_( + const common::ObAddr& server_addr, + const DRLSInfo &dr_ls_info, + const int64_t new_p) +{ + int ret = OB_SUCCESS; + int64_t inactive_count = 0; + int64_t arb_replica_number = 0; + ObLSID ls_id; + uint64_t tenant_id = OB_INVALID_TENANT_ID; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("DRWorker not init", KR(ret)); + } else if (OB_UNLIKELY(!server_addr.is_valid()) || OB_UNLIKELY(new_p <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(server_addr), K(new_p)); + } else if (OB_FAIL(dr_ls_info.get_ls_id(tenant_id, ls_id))) { + LOG_WARN("fail to get tenant and ls id", KR(ret), K(dr_ls_info)); + } else if (OB_FAIL(ObShareUtil::generate_arb_replica_num( + tenant_id, ls_id, arb_replica_number))) { + LOG_WARN("fail to generate arb replica number", KR(ret), K(tenant_id), K(ls_id)); + } else if (OB_FAIL(check_other_inactive_server_count_( + server_addr, dr_ls_info, inactive_count))) { + LOG_WARN("fail to check other permanent offline server", + KR(ret), K(server_addr), K(dr_ls_info)); + } else if ((0 == dr_ls_info.get_member_list_cnt() - 1 - inactive_count) // no member + || (dr_ls_info.get_member_list_cnt() - 1 - inactive_count + arb_replica_number < majority(new_p))) { + ret = OB_OP_NOT_ALLOW; + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "Current operation may result in no leader, while is"); + LOG_WARN("not satisfy majority", KR(ret), K(new_p), K(arb_replica_number), K(inactive_count)); + } + LOG_INFO("check majority for remove over", KR(ret), K(server_addr), K(new_p), K(arb_replica_number)); + return ret; +} + +int ObDRWorker::check_other_inactive_server_count_( + const common::ObAddr& desti_server_addr, + const DRLSInfo &dr_ls_info, + int64_t& other_inactive_server_count) +{ + /* + When removing replicas, need to judge the majority and consider the number of replicas + that have been inactive but are still in the member list. example: current paxos_replica_num = 3, + member_list_count = 3 (a b c), a is down but not permanently offline yet, now a remove operation + want remove another normal replica b, set paxos_replica_num = 2, at this time, + there is only one available replica of the underlying layer. which may result in no leader. + */ + int ret = OB_SUCCESS; + bool active = false; + common::ObAddr leader_addr; // not used + GlobalLearnerList learner_list; // not used + common::ObMemberList member_list; + other_inactive_server_count = 0; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("DRWorker not init", KR(ret)); + } else if (OB_UNLIKELY(!desti_server_addr.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(desti_server_addr)); + } else if (OB_FAIL(dr_ls_info.get_leader_and_member_list(leader_addr, member_list, learner_list))) { + LOG_WARN("fail to get leader and member list", KR(ret), K(desti_server_addr), K(dr_ls_info)); + } else { + for (int64_t index = 0; OB_SUCC(ret) && index < member_list.get_member_number(); ++index) { + ObMember member; + active = false; + if (OB_FAIL(member_list.get_member_by_index(index, member))) { + LOG_WARN("fail to get member", KR(ret), K(index), K(dr_ls_info)); + } else if (OB_FAIL(SVR_TRACER.check_server_active(member.get_server(), active))) { + LOG_WARN("fail to check server permanent offline", KR(ret), K(member.get_server())); + } else if (!active && desti_server_addr != member.get_server()) { + other_inactive_server_count = other_inactive_server_count + 1; + // other_inactive_server_count not include desti_server_addr + } + } + } + LOG_INFO("check other_inactive_server_count over", KR(ret), K(dr_ls_info), + K(desti_server_addr), K(other_inactive_server_count)); + return ret; +} + int ObDRWorker::generate_task_key( const DRLSInfo &dr_ls_info, ObDRTaskKey &task_key) const @@ -2205,7 +3189,6 @@ int ObDRWorker::check_has_leader_while_remove_replica( LOG_WARN("fail to get replica cnt", KR(ret), K(dr_ls_info)); } else { int64_t full_replica_count = 0; - int64_t paxos_replica_num = 0; int64_t arb_replica_num = 0; uint64_t tenant_id = OB_INVALID_TENANT_ID; ObLSID ls_id; @@ -2238,9 +3221,6 @@ int ObDRWorker::check_has_leader_while_remove_replica( if (server_stat_info->get_server() == server) { replica_type = ls_replica->get_replica_type(); } - if (ObReplicaTypeCheck::is_paxos_replica_V2(ls_replica->get_replica_type())) { - ++paxos_replica_num; - } if (REPLICA_TYPE_FULL == ls_replica->get_replica_type()) { ++full_replica_count; } @@ -2259,14 +3239,40 @@ int ObDRWorker::check_has_leader_while_remove_replica( has_leader = false; } else if (!ObReplicaTypeCheck::is_paxos_replica_V2(replica_type)) { has_leader = true; + } else if (1 == dr_ls_info.get_paxos_replica_number()) { + // member_list count should be always less than paxos_replica_number + // so member_list should have only one member (when 1 == paxos_replica_number) + // we can not remove the only member + has_leader = false; + } else if (2 == dr_ls_info.get_paxos_replica_number()) { + // member_list count should be always less than paxos_replica_number + // although member_list count can be less than paxos_replica_number + // member_list count can not be 1, because: + // 1. 2F can not reduce to 1F with paxos_replica_number = 2 + // 2. 2F1A permanent offline 1F then paxos_replica_number will change from 2 to 1 + // so member_list should have 2 members (when 2 == paxos_replica_number) + if (2 >= dr_ls_info.get_schema_replica_cnt()) { + // if 2 == schema_replica_count, it means locality has 2F, we specialy support 2F to 1F without locality changes + // if 2 > schema_replica_count, it means locality has 1F, but migration leads to 2F, we have to remove 1F + has_leader = true; + } else { + // if 2 != schema_replica_count, it means locality is changing, we can not reduce 2F to 1F + // Consider this case: + // tenant's initial locality is 3F(z1,z2,z3) + // member_list is reduced to 2F(z1,z2) paxos_replica_number reduced to 2 by using "alter system remove replica" command + // tenant locality is changing from 3F(z1,z2,z3) to 5F(z1,z2,z3,z4,z5), current member_list is (z1,z2) and paxos_replica_number = 2 + // before add z3 replica, server in z2 is permanent offline, we do not want to remove z2 replica in this case + has_leader = false; + } } else { - has_leader = true; - if (REPLICA_TYPE_FULL == replica_type) { - has_leader = full_replica_count >= 2; - } - if (has_leader) { - has_leader = (paxos_replica_num - 1 + arb_replica_num) >= majority(dr_ls_info.get_schema_replica_cnt()); - } + // we do not reduce member_list count less than majority of schema replica count + // consider this case: + // tenant's locality is changing from 3F to 4F + // before add replica in z4, replica in z3 is permanent offline + // if we remove replica in z3 first, member_list will have only 2 members, and + // schema is 4F requiring majority is 3 members which is not good + // So we prohibit removing replica in this case + has_leader = (full_replica_count - 1 + arb_replica_num) >= majority(dr_ls_info.get_schema_replica_cnt()); } } return ret; @@ -2527,10 +3533,6 @@ int ObDRWorker::do_single_replica_permanent_offline_( ObDRTaskKey task_key; bool can_generate = false; ObReplicaMember remove_member(member_to_remove); - //ObReplicaMember remove_member(member_to_remove.get_server(), - // member_to_remove.get_timestamp(), - // replica_type, - // memstore_percent); ObDRTaskType task_type = ObReplicaTypeCheck::is_paxos_replica_V2(replica_type) ? ObDRTaskType::LS_REMOVE_PAXOS_REPLICA : ObDRTaskType::LS_REMOVE_NON_PAXOS_REPLICA; @@ -2543,7 +3545,9 @@ int ObDRWorker::do_single_replica_permanent_offline_( old_paxos_replica_number, leader_addr, replica_type))) { - LOG_WARN("fail to construct extra infos to build remove replica task"); + LOG_WARN("fail to construct extra infos to build remove replica task", + KR(ret), K(dr_ls_info), K(task_id), K(new_paxos_replica_number), + K(old_paxos_replica_number), K(leader_addr), K(replica_type)); } else if (only_for_display) { // only for display, no need to execute this task ObLSReplicaTaskDisplayInfo display_info; @@ -4314,13 +5318,6 @@ int ObDRWorker::check_need_generate_migrate_to_unit_task( KP(server_stat_info), KP(unit_stat_info), KP(unit_in_group_stat_info)); - } else if (REPLICA_STATUS_NORMAL == ls_replica->get_replica_status() - && unit_in_group_stat_info->is_in_pool() - && server_stat_info->get_server() != unit_in_group_stat_info->get_unit().server_ - && unit_in_group_stat_info->get_server_stat()->is_alive() - && !unit_in_group_stat_info->get_server_stat()->is_block()) { - need_generate = true; - is_unit_in_group_related = true; } else if (REPLICA_STATUS_NORMAL == ls_replica->get_replica_status() && unit_stat_info->is_in_pool() && server_stat_info->get_server() != unit_stat_info->get_unit().server_ @@ -4328,6 +5325,13 @@ int ObDRWorker::check_need_generate_migrate_to_unit_task( && !unit_stat_info->get_server_stat()->is_block()) { need_generate = true; is_unit_in_group_related = false; + } else if (REPLICA_STATUS_NORMAL == ls_replica->get_replica_status() + && unit_in_group_stat_info->is_in_pool() + && server_stat_info->get_server() != unit_in_group_stat_info->get_unit().server_ + && unit_in_group_stat_info->get_server_stat()->is_alive() + && !unit_in_group_stat_info->get_server_stat()->is_block()) { + need_generate = true; + is_unit_in_group_related = true; } return ret; } @@ -4621,32 +5625,34 @@ int ObDRWorker::generate_disaster_recovery_paxos_replica_number( || curr_paxos_replica_number <= 0 || locality_paxos_replica_number <= 0)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), - K(member_list_cnt), - K(curr_paxos_replica_number), + LOG_WARN("invalid argument", KR(ret), K(member_list_cnt), K(curr_paxos_replica_number), K(locality_paxos_replica_number)); } else if (MEMBER_CHANGE_ADD == member_change_type) { + // 1. ADD MEMBER_LIST operation + // When current paxos_replica_number >= locality paxos_replica_number + // we do not change paxos_replica_number and ensure that paxos_replica_number no less than new member_list count + // When current paxos_replica_number < locality paxos_replica_number + // we try to increase paxos_replica_number towards locality and ensure that majority is satisfied const int64_t member_list_cnt_after = member_list_cnt + 1; - if (curr_paxos_replica_number == locality_paxos_replica_number) { - if (locality_paxos_replica_number >= member_list_cnt_after) { - new_paxos_replica_number = curr_paxos_replica_number; - found = true; - } else if (locality_paxos_replica_number + 1 == member_list_cnt_after) { - new_paxos_replica_number = curr_paxos_replica_number + 1; - found = true; - } - } else if (curr_paxos_replica_number > locality_paxos_replica_number) { + if (curr_paxos_replica_number >= locality_paxos_replica_number) { if (curr_paxos_replica_number >= member_list_cnt_after) { new_paxos_replica_number = curr_paxos_replica_number; found = true; - } else {} // new member cnt greater than paxos_replica_number, not good - } else { // curr_paxos_replica_number < locality_paxos_replica_number + } + } else { if (majority(curr_paxos_replica_number + 1) <= member_list_cnt_after) { new_paxos_replica_number = curr_paxos_replica_number + 1; found = true; - } else {} // majority not satisfied + } } } else if (MEMBER_CHANGE_NOP == member_change_type) { + // 2. MEMBER_LIST not changed operation + // When current paxos_replica_number == locality paxos_replica_number + // we do not change paxos_replica_number + // When current paxos_replica_number > locality paxos_replica_number + // we try to reduce paxos_replica_number towards locality + // When current paxos_replica_number < locality paxos_replica_number + // we try to increase paxos_replica_number towards locality if (curr_paxos_replica_number == locality_paxos_replica_number) { new_paxos_replica_number = curr_paxos_replica_number; found = true; @@ -4655,7 +5661,7 @@ int ObDRWorker::generate_disaster_recovery_paxos_replica_number( new_paxos_replica_number = curr_paxos_replica_number - 1; found = true; } - } else { // curr_paxos_replica_number < locality_paxos_replica_number + } else { if (member_list_cnt > majority(curr_paxos_replica_number + 1)) { new_paxos_replica_number = curr_paxos_replica_number + 1; found = true; @@ -4664,31 +5670,53 @@ int ObDRWorker::generate_disaster_recovery_paxos_replica_number( } else if (MEMBER_CHANGE_SUB == member_change_type) { int64_t member_list_cnt_after = 0; int64_t arb_replica_number = 0; - if (OB_FAIL(dr_ls_info.get_ls_id(tenant_id, ls_id))) { - LOG_WARN("fail to get tenant and ls id", KR(ret), K(dr_ls_info)); - } else if (OB_FAIL(ObShareUtil::generate_arb_replica_num( - tenant_id, - ls_id, - arb_replica_number))) { - LOG_WARN("fail to generate arb replica number", KR(ret), K(tenant_id), K(ls_id)); - } - member_list_cnt_after = member_list_cnt - 1 + arb_replica_number; - if (OB_FAIL(ret)) { - } else if (curr_paxos_replica_number == locality_paxos_replica_number) { - if (majority(curr_paxos_replica_number) <= member_list_cnt_after) { - new_paxos_replica_number = curr_paxos_replica_number; - found = true; - } else {} // majority not satisfied - } else if (curr_paxos_replica_number > locality_paxos_replica_number) { - if (majority(curr_paxos_replica_number - 1) <= member_list_cnt_after) { + // 3. REMOVE MEMBER_LIST operation + // When current paxos_replica_number == locality paxos_replica_number + // we do not change paxos_replica_number and ensure majority is satisfied + // When current paxos_replica_number > locality paxos_replica_number + // we try to reduce paxos_replica_number towards locality and ensure majority is satisfied + // When current paxos_replica_number < locality paxos_replica_number + // we do not change paxos_replica_number and ensure majority is satisfied + // SPECIALLY, we support 2F reduce to 1F aotumatically + if (2 == curr_paxos_replica_number) { + if (2 == member_list_cnt) { + // Specially, we support 2F to 1F to solve these cases: + // CASE 1: + // tenant's locality is 2F1A, F-replica in zone1 migrate and leads to source replica remains in member_list, + // member_list like (F-z1, F-z1, F-z2) and paxos_replica_number = 3 + // then F-replica in zone2 permanent offline, + // member_list like (F-z1, F-z1) and paxos_replica_number = 2 + // Under this case, we have to support remove one F in zone1 and later add one F in zone2 + // CASE 2: + // tenant's locality is 1F, F-replica in zone1 migrate and leads to source replica remains in member_list, + // member_list like (F-z1, F-z1) and paxos_replica_number = 2 + // Under this case, we have to support remove one F in zone1 new_paxos_replica_number = curr_paxos_replica_number - 1; found = true; - } else {} // majority not satisfied - } else { // curr_paxos_replica_number < locality_paxos_replica_number - if (majority(curr_paxos_replica_number) <= member_list_cnt_after) { - new_paxos_replica_number = curr_paxos_replica_number; - found = true; - } else {} // majority not satisfied + } else { + // do nothing + // When current paxos_replica_number = 2 + // member_list_cnt can not be less than 2, because it will leads to no-leader + // member_list_cnt can not be larger than 2, because member_list_cnt should <= paxos_replica_number + // we do not raise error and remain found = false + } + } else if (OB_FAIL(dr_ls_info.get_ls_id(tenant_id, ls_id))) { + LOG_WARN("fail to get tenant and ls id", KR(ret), K(dr_ls_info)); + } else if (OB_FAIL(ObShareUtil::generate_arb_replica_num(tenant_id, ls_id, arb_replica_number))) { + LOG_WARN("fail to generate arb replica number", KR(ret), K(tenant_id), K(ls_id)); + } else { + member_list_cnt_after = member_list_cnt - 1 + arb_replica_number; + if (curr_paxos_replica_number <= locality_paxos_replica_number) { + if (majority(curr_paxos_replica_number) <= member_list_cnt_after) { + new_paxos_replica_number = curr_paxos_replica_number; + found = true; + } + } else if (curr_paxos_replica_number > locality_paxos_replica_number) { + if (majority(curr_paxos_replica_number - 1) <= member_list_cnt_after) { + new_paxos_replica_number = curr_paxos_replica_number - 1; + found = true; + } + } } } else { ret = OB_INVALID_ARGUMENT; diff --git a/src/rootserver/ob_disaster_recovery_worker.h b/src/rootserver/ob_disaster_recovery_worker.h index 397b27bee..ec162fd32 100755 --- a/src/rootserver/ob_disaster_recovery_worker.h +++ b/src/rootserver/ob_disaster_recovery_worker.h @@ -119,6 +119,12 @@ public: const uint64_t tenant_id, const bool only_for_display, int64_t &acc_dr_task); + int do_add_ls_replica_task(const obrpc::ObAdminAlterLSReplicaArg &arg); + int do_remove_ls_replica_task(const obrpc::ObAdminAlterLSReplicaArg &arg); + int do_migrate_ls_replica_task(const obrpc::ObAdminAlterLSReplicaArg &arg); + int do_modify_ls_replica_type_task(const obrpc::ObAdminAlterLSReplicaArg &arg); + int do_modify_ls_paxos_replica_num_task(const obrpc::ObAdminAlterLSReplicaArg &arg); + int do_cancel_ls_replica_task(const obrpc::ObAdminAlterLSReplicaArg &arg); static int check_tenant_locality_match( const uint64_t tenant_id, ObZoneManager &zone_mgr, @@ -129,6 +135,133 @@ public: common::ObSArray &task_plan); private: + + // add task in queue in mgr and execute task + // @param [in] task, the task to execute + int add_task_in_queue_and_execute_(const ObDRTask &task); + // check ls exist and init dr_ls_info + // @param [in] arg, task info + // @param [out] dr_ls_info, target dr_ls_info to init + int check_and_init_info_for_alter_ls_( + const obrpc::ObAdminAlterLSReplicaArg& arg, + DRLSInfo& dr_ls_info); + // check ls exist and get ls_info and ls_status_info + // @param [in] ls_id, which ls to check + // @param [in] tenant_id, which user does the ls to check belong to + // @param [out] ls_info, target ls_info + // @param [out] ls_status_info, target ls_status_info + int check_ls_exist_and_get_ls_info_( + const share::ObLSID& ls_id, + const int64_t tenant_id, + share::ObLSInfo& ls_info, + share::ObLSStatusInfo& ls_status_info); + // check unit exist and get unit + // @param [in] task_execute_server, the unit in which server + // @param [in] tenant_id, which user does the unit to check belong to + // @param [in] is_migrate_source_valid, is unit migration valid on the source server + // @param [out] unit, target unit + int check_unit_exist_and_get_unit_( + const common::ObAddr &task_execute_server, + const uint64_t tenant_id, + const bool is_migrate_source_valid, + share::ObUnit& unit); + // check task execute server status + // @param [in] task_execute_server, the task execute in which server + // @param [in] need_check_can_migrate_in, if need to check can_migrate_in flag + int check_task_execute_server_status_( + const common::ObAddr &task_execute_server, + const bool need_check_can_migrate_in); + // get replica type by leader + // @param [in] server_addr, the replica in which server + // @param [in] dr_ls_info, dr_ls_info + // @param [out] replica_type, the replica_type of replica in server_addr + int get_replica_type_by_leader_( + const common::ObAddr& server_addr, + const DRLSInfo &dr_ls_info, + common::ObReplicaType& replica_type); + // build a add replica task by task info + // @param [in] arg, the task info + // @param [in] dr_ls_info, dr_ls_info + // @param [out] add_replica_task, target task + int build_add_replica_task_( + const obrpc::ObAdminAlterLSReplicaArg &arg, + const DRLSInfo &dr_ls_info, + ObAddLSReplicaTask &add_replica_task); + // build a remove replica task by task info + // @param [in] arg, the task info + // @param [in] dr_ls_info, dr_ls_info + // @param [out] remove_replica_task, target task + int build_remove_replica_task_( + const obrpc::ObAdminAlterLSReplicaArg &arg, + DRLSInfo &dr_ls_info, + ObRemoveLSReplicaTask &remove_replica_task); + // build a modify replica task by task info + // @param [in] arg, the task info + // @param [in] dr_ls_info, dr_ls_info + // @param [out] modify_replica_task, target task + int build_modify_replica_type_task_( + const obrpc::ObAdminAlterLSReplicaArg &arg, + DRLSInfo &dr_ls_info, + ObLSTypeTransformTask &modify_replica_task); + // build a migrate replica task by task info + // @param [in] arg, the task info + // @param [in] dr_ls_info, dr_ls_info + // @param [out] migrate_replica_task, target task + int build_migrate_replica_task_( + const obrpc::ObAdminAlterLSReplicaArg &arg, + const DRLSInfo &dr_ls_info, + ObMigrateLSReplicaTask &migrate_replica_task); + // build a modify paxos_replica_num task by task info + // @param [in] arg, the task info + // @param [in] dr_ls_info, dr_ls_info + // @param [out] modify_paxos_replica_number_task, target task + int build_modify_paxos_replica_num_task_( + const obrpc::ObAdminAlterLSReplicaArg &arg, + DRLSInfo &dr_ls_info, + ObLSModifyPaxosReplicaNumberTask &modify_paxos_replica_number_task); + // check if provide data source available and init data_source + // @param [in] arg, the task info + // @param [in] replica_type, source replica type + // @param [in] dr_ls_info, dr_ls_info + // @param [out] data_source, target data_source + int check_data_source_available_and_init_( + const obrpc::ObAdminAlterLSReplicaArg &arg, + const common::ObReplicaType &replica_type, + const DRLSInfo &dr_ls_info, + ObReplicaMember &data_source); + // if provided paxos_replica_num is valid, check it. otherwise generate new paxos_replica_num + // @param [in] arg, the task info + // @param [in] replica_type, target replica's type + // @param [in] dr_ls_info, dr_ls_info + // @param [out] new_p, new paxos_replica_num + int check_and_generate_new_paxos_replica_num_( + const obrpc::ObAdminAlterLSReplicaArg &arg, + const common::ObReplicaType &replica_type, + const DRLSInfo &dr_ls_info, + int64_t &new_p); + // check whether the values of member list count and new paxs_replica_num are legal in alter full replica + // @param [in] member_list_count, count of leader member list after alter full replica + // @param [in] new_p, new paxos_replica_num + int check_for_alter_full_replica_( + const int64_t member_list_count, + const int64_t new_p); + // check if majority is satisfied when remove replica + // @param [in] server_addr, target replica to remove in which server + // @param [in] dr_ls_info, dr_ls_info + // @param [in] new_p, new paxos_replica_num + int check_majority_for_remove_( + const common::ObAddr& server_addr, + const DRLSInfo &dr_ls_info, + const int64_t new_p); + // check the count of inactive server except desti_server_addr + // @param [in] desti_server_addr, except desti_server_addr + // @param [in] dr_ls_info, dr_ls_info + // @param [out] other_inactive_server_count, except desti_server_addr inactive server count + int check_other_inactive_server_count_( + const common::ObAddr& desti_server_addr, + const DRLSInfo &dr_ls_info, + int64_t& other_inactive_server_count); + struct TaskCountStatistic { public: diff --git a/src/rootserver/ob_root_service.cpp b/src/rootserver/ob_root_service.cpp index 78ef4739a..80b342bb5 100755 --- a/src/rootserver/ob_root_service.cpp +++ b/src/rootserver/ob_root_service.cpp @@ -9342,6 +9342,95 @@ int ObRootService::admin_migrate_unit(const obrpc::ObAdminMigrateUnitArg &arg) return ret; } +#define ADD_EVENT_FOR_ALTER_LS_REPLICA \ + "admin_alter_ls_replica", arg.get_alter_task_type().get_type_str(), \ + "ret_code", ret_val, \ + "tenant_id", arg.get_tenant_id() \ + +int ObRootService::add_rs_event_for_alter_ls_replica_( + const obrpc::ObAdminAlterLSReplicaArg &arg, + const int ret_val) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (OB_UNLIKELY(!arg.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg", KR(ret), K(arg)); + } else { + char extra_info[MSG_SIZE] = {0}; + char addr_str_buf[OB_SERVER_ADDR_STR_LEN]; + if (OB_FAIL(arg.get_data_source().ip_port_to_string(addr_str_buf, OB_SERVER_ADDR_STR_LEN))) { + LOG_WARN("data source to string failed", KR(ret), K(arg)); + } else { + snprintf(extra_info, sizeof(extra_info), + "data_source: %s, paxos_replica_num: %ld", addr_str_buf, arg.get_paxos_replica_num()); + } + if (OB_FAIL(ret)) { + } else if (arg.get_alter_task_type().is_add_task() + || arg.get_alter_task_type().is_modify_replica_task() + || arg.get_alter_task_type().is_remove_task()) { + ROOTSERVICE_EVENT_ADD(ADD_EVENT_FOR_ALTER_LS_REPLICA, + "ls_id", arg.get_ls_id().id(), + "target_replica", arg.get_server_addr(), + "replica_type", replica_type_to_str(arg.get_replica_type()), + "", NULL, + extra_info); + } else if (arg.get_alter_task_type().is_migrate_task()) { + ROOTSERVICE_EVENT_ADD(ADD_EVENT_FOR_ALTER_LS_REPLICA, + "ls_id", arg.get_ls_id().id(), + "source_replica", arg.get_server_addr(), + "target_replica", arg.get_destination_addr(), + "", NULL, + extra_info); + } else if (arg.get_alter_task_type().is_modify_paxos_replica_num_task()) { + ROOTSERVICE_EVENT_ADD(ADD_EVENT_FOR_ALTER_LS_REPLICA, + "ls_id", arg.get_ls_id().id(), + "paxos_replica_num", arg.get_paxos_replica_num()); + } else if (arg.get_alter_task_type().is_cancel_task()) { + ROOTSERVICE_EVENT_ADD(ADD_EVENT_FOR_ALTER_LS_REPLICA, + "task_id", arg.get_task_id()); + } else { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(arg)); + } + } + return ret; +} + +int ObRootService::admin_alter_ls_replica(const obrpc::ObAdminAlterLSReplicaArg &arg) +{ + int ret = OB_SUCCESS; + FLOG_INFO("receive alter ls replica request", K(arg)); + int64_t start_time = ObTimeUtility::current_time(); + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (OB_UNLIKELY(!arg.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg", KR(ret), K(arg)); + } else { + ObSystemAdminCtx ctx; + if (OB_FAIL(init_sys_admin_ctx(ctx))) { + LOG_WARN("init_sys_admin_ctx failed", KR(ret)); + } else { + ObAdminAlterLSReplica admin_util(ctx); + if (OB_FAIL(admin_util.execute(arg))) { + LOG_WARN("execute alter ls replica failed", KR(ret), K(arg)); + } + } + } + int64_t cost_time = ObTimeUtility::current_time() - start_time; + FLOG_INFO("alter ls replica over", KR(ret), K(arg), K(cost_time)); + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = add_rs_event_for_alter_ls_replica_(arg, ret))) { + // ignore + LOG_WARN("add rs event for alter ls replica failed", KR(ret), KR(tmp_ret), K(arg)); + } + return ret; +} + int ObRootService::admin_upgrade_virtual_schema() { int ret = OB_SUCCESS; diff --git a/src/rootserver/ob_root_service.h b/src/rootserver/ob_root_service.h index 5586f9851..4bcfdd8cf 100644 --- a/src/rootserver/ob_root_service.h +++ b/src/rootserver/ob_root_service.h @@ -740,6 +740,7 @@ public: int admin_switch_replica_role(const obrpc::ObAdminSwitchReplicaRoleArg &arg); int admin_switch_rs_role(const obrpc::ObAdminSwitchRSRoleArg &arg); int admin_drop_replica(const obrpc::ObAdminDropReplicaArg &arg); + int admin_alter_ls_replica(const obrpc::ObAdminAlterLSReplicaArg &arg); int admin_change_replica(const obrpc::ObAdminChangeReplicaArg &arg); int admin_migrate_replica(const obrpc::ObAdminMigrateReplicaArg &arg); int admin_report_replica(const obrpc::ObAdminReportReplicaArg &arg); @@ -961,6 +962,7 @@ private: int check_mds_memory_limit_(obrpc::ObAdminSetConfigItem &item); int check_freeze_trigger_percentage_(obrpc::ObAdminSetConfigItem &item); int check_write_throttle_trigger_percentage(obrpc::ObAdminSetConfigItem &item); + int add_rs_event_for_alter_ls_replica_(const obrpc::ObAdminAlterLSReplicaArg &arg, const int ret_val); int check_data_disk_write_limit_(obrpc::ObAdminSetConfigItem &item); int check_data_disk_usage_limit_(obrpc::ObAdminSetConfigItem &item); private: diff --git a/src/rootserver/ob_rs_rpc_processor.h b/src/rootserver/ob_rs_rpc_processor.h index 1ad5a48ca..1d68251cd 100644 --- a/src/rootserver/ob_rs_rpc_processor.h +++ b/src/rootserver/ob_rs_rpc_processor.h @@ -479,6 +479,7 @@ DEFINE_RS_RPC_PROCESSOR(obrpc::OB_ADMIN_RELOAD_SERVER, ObRpcAdminReloadServerP, DEFINE_RS_RPC_PROCESSOR(obrpc::OB_ADMIN_RELOAD_ZONE, ObRpcAdminReloadZoneP, admin_reload_zone()); DEFINE_RS_RPC_PROCESSOR(obrpc::OB_ADMIN_CLEAR_MERGE_ERROR, ObRpcAdminClearMergeErrorP, admin_clear_merge_error(arg_)); DEFINE_RS_RPC_PROCESSOR(obrpc::OB_ADMIN_MIGRATE_UNIT, ObRpcAdminMigrateUnitP, admin_migrate_unit(arg_)); +DEFINE_RS_RPC_PROCESSOR(obrpc::OB_ADMIN_ALTER_LS_REPLICA, ObRpcAdminAlterLSReplicaP, admin_alter_ls_replica(arg_)); #ifdef OB_BUILD_ARBITRATION DEFINE_RS_RPC_PROCESSOR(obrpc::OB_ADMIN_ADD_ARBITRATION_SERVICE, ObRpcAdminAddArbitrationServiceP, admin_add_arbitration_service(arg_)); DEFINE_RS_RPC_PROCESSOR(obrpc::OB_ADMIN_REMOVE_ARBITRATION_SERVICE, ObRpcAdminRemoveArbitrationServiceP, admin_remove_arbitration_service(arg_)); diff --git a/src/rootserver/ob_system_admin_util.cpp b/src/rootserver/ob_system_admin_util.cpp index 7311c5bc1..0f69e0b5a 100644 --- a/src/rootserver/ob_system_admin_util.cpp +++ b/src/rootserver/ob_system_admin_util.cpp @@ -1355,6 +1355,76 @@ int ObAdminMigrateUnit::execute(const ObAdminMigrateUnitArg &arg) return ret; } +int ObAdminAlterLSReplica::execute(const obrpc::ObAdminAlterLSReplicaArg &arg) +{ + FLOG_INFO("execute alter ls replica request", K(arg)); + int ret = OB_SUCCESS; + int64_t start_time = ObTimeUtility::current_time(); + if (OB_UNLIKELY(!ctx_.is_inited())) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (OB_ISNULL(ctx_.root_balancer_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("root_balancer_ is null", KR(ret), K(arg), KP(ctx_.root_balancer_)); + } else if (OB_UNLIKELY(!arg.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg", KR(ret), K(arg)); + } else { + switch (arg.get_alter_task_type().get_type()) { + case ObAlterLSReplicaTaskType::AddLSReplicaTask: { + if (OB_FAIL(ctx_.root_balancer_->get_disaster_recovery_worker() + .do_add_ls_replica_task(arg))) { + LOG_WARN("add ls replica task failed", KR(ret), K(arg)); + } + break; + } + case ObAlterLSReplicaTaskType::RemoveLSReplicaTask: { + if (OB_FAIL(ctx_.root_balancer_->get_disaster_recovery_worker() + .do_remove_ls_replica_task(arg))) { + LOG_WARN("remove ls replica task failed", KR(ret), K(arg)); + } + break; + } + case ObAlterLSReplicaTaskType::MigrateLSReplicaTask: { + if (OB_FAIL(ctx_.root_balancer_->get_disaster_recovery_worker() + .do_migrate_ls_replica_task(arg))) { + LOG_WARN("migrate ls replica task failed", KR(ret), K(arg)); + } + break; + } + case ObAlterLSReplicaTaskType::ModifyLSReplicaTypeTask: { + if (OB_FAIL(ctx_.root_balancer_->get_disaster_recovery_worker() + .do_modify_ls_replica_type_task(arg))) { + LOG_WARN("modify ls replica task failed", KR(ret), K(arg)); + } + break; + } + case ObAlterLSReplicaTaskType::ModifyLSPaxosReplicaNumTask: { + if (OB_FAIL(ctx_.root_balancer_->get_disaster_recovery_worker() + .do_modify_ls_paxos_replica_num_task(arg))) { + LOG_WARN("modify ls paxos_replica_num task failed", KR(ret), K(arg)); + } + break; + } + case ObAlterLSReplicaTaskType::CancelLSReplicaTask: { + if (OB_FAIL(ctx_.root_balancer_->get_disaster_recovery_worker() + .do_cancel_ls_replica_task(arg))) { + LOG_WARN("cancel ls replica task failed", KR(ret), K(arg)); + } + break; + } + default: { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("task type unexpected", KR(ret), K(arg)); + break; + } + } + } + int64_t cost_time = ObTimeUtility::current_time() - start_time; + FLOG_INFO("execute alter ls replica request over", KR(ret), K(arg), K(cost_time)); + return ret; +} + int ObAdminUpgradeVirtualSchema::execute() { int ret = OB_SUCCESS; diff --git a/src/rootserver/ob_system_admin_util.h b/src/rootserver/ob_system_admin_util.h index 898e681d6..97646d074 100644 --- a/src/rootserver/ob_system_admin_util.h +++ b/src/rootserver/ob_system_admin_util.h @@ -371,6 +371,17 @@ private: DISALLOW_COPY_AND_ASSIGN(ObAdminMigrateUnit); }; +class ObAdminAlterLSReplica : public ObSystemAdminUtil +{ +public: + explicit ObAdminAlterLSReplica(const ObSystemAdminCtx &ctx) : ObSystemAdminUtil(ctx) {} + virtual ~ObAdminAlterLSReplica() {} + + int execute(const obrpc::ObAdminAlterLSReplicaArg &arg); +private: + DISALLOW_COPY_AND_ASSIGN(ObAdminAlterLSReplica); +}; + class ObAdminUpgradeVirtualSchema : public ObSystemAdminUtil { public: diff --git a/src/share/inner_table/ob_inner_table_schema.12451_12500.cpp b/src/share/inner_table/ob_inner_table_schema.12451_12500.cpp index f60b9e34d..72b383d92 100644 --- a/src/share/inner_table/ob_inner_table_schema.12451_12500.cpp +++ b/src/share/inner_table/ob_inner_table_schema.12451_12500.cpp @@ -2295,6 +2295,474 @@ int ObInnerTableSchema::enabled_roles_schema(ObTableSchema &table_schema) return ret; } +int ObInnerTableSchema::all_virtual_ls_replica_task_history_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(VIRTUAL_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ls_id", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_type", //column_name + ++column_id, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_DISASTER_RECOVERY_TASK_TYPE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_id", //column_name + ++column_id, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_TRACE_STAT_BUFFER_SIZE, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("gmt_create", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("gmt_modified", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_status", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_COLUMN_COMMENT_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ObObj priority_default; + priority_default.set_int(1); + ADD_COLUMN_SCHEMA_T("priority", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + priority_default, + priority_default); //default_value + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("target_replica_svr_ip", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_IP_ADDR_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("target_replica_svr_port", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("target_paxos_replica_number", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("target_replica_type", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_REPLICA_TYPE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("source_replica_svr_ip", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_IP_ADDR_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("source_replica_svr_port", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("source_paxos_replica_number", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("source_replica_type", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_REPLICA_TYPE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("data_source_svr_ip", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_IP_ADDR_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("data_source_svr_port", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ObObj is_manual_default; + is_manual_default.set_tinyint(0); + ADD_COLUMN_SCHEMA_T("is_manual", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTinyIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + 1, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false, //is_autoincrement + is_manual_default, + is_manual_default); //default_value + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_exec_svr_ip", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_IP_ADDR_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_exec_svr_port", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ObObj gmt_default; + ObObj gmt_default_null; + + gmt_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("generate_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + 6, //column_scale + false, //is_nullable + false, //is_autoincrement + false, //is_on_update_for_timestamp + gmt_default_null, + gmt_default) + } + + if (OB_SUCC(ret)) { + ObObj gmt_default; + ObObj gmt_default_null; + + gmt_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("schedule_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + 6, //column_scale + false, //is_nullable + false, //is_autoincrement + false, //is_on_update_for_timestamp + gmt_default_null, + gmt_default) + } + + if (OB_SUCC(ret)) { + ObObj gmt_default; + ObObj gmt_default_null; + + gmt_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("finish_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + 6, //column_scale + false, //is_nullable + false, //is_autoincrement + false, //is_on_update_for_timestamp + gmt_default_null, + gmt_default) + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("execute_result", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_COLUMN_COMMENT_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("comment", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_COLUMN_COMMENT_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::all_virtual_session_ps_info_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.15401_15450.cpp b/src/share/inner_table/ob_inner_table_schema.15401_15450.cpp index 5e3e6d80d..818ee9021 100644 --- a/src/share/inner_table/ob_inner_table_schema.15401_15450.cpp +++ b/src/share/inner_table/ob_inner_table_schema.15401_15450.cpp @@ -6780,6 +6780,440 @@ int ObInnerTableSchema::all_virtual_index_usage_info_real_agent_ora_schema(ObTab return ret; } +int ObInnerTableSchema::all_virtual_ls_replica_task_history_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(VIRTUAL_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("LS_ID", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TASK_TYPE", //column_name + ++column_id, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + MAX_DISASTER_RECOVERY_TASK_TYPE_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TASK_ID", //column_name + ++column_id, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_TRACE_STAT_BUFFER_SIZE, //column_length + 2, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("GMT_CREATE", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampLTZType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("GMT_MODIFIED", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampLTZType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TASK_STATUS", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + MAX_COLUMN_COMMENT_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("PRIORITY", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TARGET_REPLICA_SVR_IP", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + MAX_IP_ADDR_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TARGET_REPLICA_SVR_PORT", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TARGET_PAXOS_REPLICA_NUMBER", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TARGET_REPLICA_TYPE", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + MAX_REPLICA_TYPE_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SOURCE_REPLICA_SVR_IP", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + MAX_IP_ADDR_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SOURCE_REPLICA_SVR_PORT", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SOURCE_PAXOS_REPLICA_NUMBER", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SOURCE_REPLICA_TYPE", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + MAX_REPLICA_TYPE_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DATA_SOURCE_SVR_IP", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + MAX_IP_ADDR_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DATA_SOURCE_SVR_PORT", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("IS_MANUAL", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TASK_EXEC_SVR_IP", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + MAX_IP_ADDR_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TASK_EXEC_SVR_PORT", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("GENERATE_TIME", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampLTZType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SCHEDULE_TIME", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampLTZType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("FINISH_TIME", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampLTZType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("EXECUTE_RESULT", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + MAX_COLUMN_COMMENT_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("COMMENT", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + MAX_COLUMN_COMMENT_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::all_virtual_session_ps_info_ora_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.21301_21350.cpp b/src/share/inner_table/ob_inner_table_schema.21301_21350.cpp index 7dd2d5b88..50e2a1891 100644 --- a/src/share/inner_table/ob_inner_table_schema.21301_21350.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21301_21350.cpp @@ -1217,7 +1217,7 @@ int ObInnerTableSchema::dba_ob_ls_replica_tasks_schema(ObTableSchema &table_sche table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( ( SELECT LS_ID, TASK_TYPE, TASK_ID, TASK_STATUS, CAST(CASE PRIORITY WHEN 0 THEN 'HIGH' WHEN 1 THEN 'LOW' ELSE NULL END AS CHAR(5)) AS PRIORITY, TARGET_REPLICA_SVR_IP, TARGET_REPLICA_SVR_PORT, TARGET_PAXOS_REPLICA_NUMBER, TARGET_REPLICA_TYPE, (CASE SOURCE_REPLICA_SVR_IP WHEN "" THEN NULL ELSE SOURCE_REPLICA_SVR_IP END) AS SOURCE_REPLICA_SVR_IP, SOURCE_REPLICA_SVR_PORT, SOURCE_PAXOS_REPLICA_NUMBER, (CASE SOURCE_REPLICA_TYPE WHEN "" THEN NULL ELSE SOURCE_REPLICA_TYPE END) AS SOURCE_REPLICA_TYPE, TASK_EXEC_SVR_IP, TASK_EXEC_SVR_PORT, CAST(GMT_CREATE AS DATETIME) AS CREATE_TIME, CAST(SCHEDULE_TIME AS DATETIME) AS START_TIME, CAST(GMT_MODIFIED AS DATETIME) AS MODIFY_TIME, COMMENT FROM OCEANBASE.__ALL_VIRTUAL_LS_REPLICA_TASK WHERE TENANT_ID = EFFECTIVE_TENANT_ID() ) )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( ( SELECT LS_ID, TASK_TYPE, TASK_ID, TASK_STATUS, CAST(CASE PRIORITY WHEN 0 THEN 'HIGH' WHEN 1 THEN 'LOW' ELSE NULL END AS CHAR(5)) AS PRIORITY, TARGET_REPLICA_SVR_IP, TARGET_REPLICA_SVR_PORT, TARGET_PAXOS_REPLICA_NUMBER, TARGET_REPLICA_TYPE, (CASE SOURCE_REPLICA_SVR_IP WHEN "" THEN NULL ELSE SOURCE_REPLICA_SVR_IP END) AS SOURCE_REPLICA_SVR_IP, SOURCE_REPLICA_SVR_PORT, SOURCE_PAXOS_REPLICA_NUMBER, (CASE SOURCE_REPLICA_TYPE WHEN "" THEN NULL ELSE SOURCE_REPLICA_TYPE END) AS SOURCE_REPLICA_TYPE, (CASE DATA_SOURCE_SVR_IP WHEN "" THEN NULL ELSE DATA_SOURCE_SVR_IP END) AS DATA_SOURCE_SVR_IP, DATA_SOURCE_SVR_PORT, CAST(CASE IS_MANUAL WHEN 0 THEN 'FALSE' WHEN 1 THEN 'TRUE' ELSE NULL END AS CHAR(6)) AS IS_MANUAL, TASK_EXEC_SVR_IP, TASK_EXEC_SVR_PORT, CAST(GMT_CREATE AS DATETIME) AS CREATE_TIME, CAST(SCHEDULE_TIME AS DATETIME) AS START_TIME, CAST(GMT_MODIFIED AS DATETIME) AS MODIFY_TIME, COMMENT FROM OCEANBASE.__ALL_VIRTUAL_LS_REPLICA_TASK WHERE TENANT_ID = EFFECTIVE_TENANT_ID() ) )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -1267,7 +1267,7 @@ int ObInnerTableSchema::cdb_ob_ls_replica_tasks_schema(ObTableSchema &table_sche table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( ( SELECT TENANT_ID, LS_ID, TASK_TYPE, TASK_ID, TASK_STATUS, CAST(CASE PRIORITY WHEN 0 THEN 'HIGH' WHEN 1 THEN 'LOW' ELSE NULL END AS CHAR(5)) AS PRIORITY, TARGET_REPLICA_SVR_IP, TARGET_REPLICA_SVR_PORT, TARGET_PAXOS_REPLICA_NUMBER, TARGET_REPLICA_TYPE, (CASE SOURCE_REPLICA_SVR_IP WHEN "" THEN NULL ELSE SOURCE_REPLICA_SVR_IP END) AS SOURCE_REPLICA_SVR_IP, SOURCE_REPLICA_SVR_PORT, SOURCE_PAXOS_REPLICA_NUMBER, (CASE SOURCE_REPLICA_TYPE WHEN "" THEN NULL ELSE SOURCE_REPLICA_TYPE END) AS SOURCE_REPLICA_TYPE, TASK_EXEC_SVR_IP, TASK_EXEC_SVR_PORT, CAST(GMT_CREATE AS DATETIME) AS CREATE_TIME, CAST(SCHEDULE_TIME AS DATETIME) AS START_TIME, CAST(GMT_MODIFIED AS DATETIME) AS MODIFY_TIME, COMMENT FROM OCEANBASE.__ALL_VIRTUAL_LS_REPLICA_TASK ) )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( ( SELECT TENANT_ID, LS_ID, TASK_TYPE, TASK_ID, TASK_STATUS, CAST(CASE PRIORITY WHEN 0 THEN 'HIGH' WHEN 1 THEN 'LOW' ELSE NULL END AS CHAR(5)) AS PRIORITY, TARGET_REPLICA_SVR_IP, TARGET_REPLICA_SVR_PORT, TARGET_PAXOS_REPLICA_NUMBER, TARGET_REPLICA_TYPE, (CASE SOURCE_REPLICA_SVR_IP WHEN "" THEN NULL ELSE SOURCE_REPLICA_SVR_IP END) AS SOURCE_REPLICA_SVR_IP, SOURCE_REPLICA_SVR_PORT, SOURCE_PAXOS_REPLICA_NUMBER, (CASE SOURCE_REPLICA_TYPE WHEN "" THEN NULL ELSE SOURCE_REPLICA_TYPE END) AS SOURCE_REPLICA_TYPE, (CASE DATA_SOURCE_SVR_IP WHEN "" THEN NULL ELSE DATA_SOURCE_SVR_IP END) AS DATA_SOURCE_SVR_IP, DATA_SOURCE_SVR_PORT, CAST(CASE IS_MANUAL WHEN 0 THEN 'FALSE' WHEN 1 THEN 'TRUE' ELSE NULL END AS CHAR(6)) AS IS_MANUAL, TASK_EXEC_SVR_IP, TASK_EXEC_SVR_PORT, CAST(GMT_CREATE AS DATETIME) AS CREATE_TIME, CAST(SCHEDULE_TIME AS DATETIME) AS START_TIME, CAST(GMT_MODIFIED AS DATETIME) AS MODIFY_TIME, COMMENT FROM OCEANBASE.__ALL_VIRTUAL_LS_REPLICA_TASK ) )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema.21501_21550.cpp b/src/share/inner_table/ob_inner_table_schema.21501_21550.cpp index c259b3fb0..625cddb4f 100644 --- a/src/share/inner_table/ob_inner_table_schema.21501_21550.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21501_21550.cpp @@ -725,6 +725,106 @@ int ObInnerTableSchema::dba_ob_clone_history_schema(ObTableSchema &table_schema) return ret; } +int ObInnerTableSchema::dba_ob_ls_replica_task_history_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_OB_LS_REPLICA_TASK_HISTORY_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_OB_LS_REPLICA_TASK_HISTORY_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( ( SELECT LS_ID, TASK_TYPE, TASK_ID, TASK_STATUS, CAST(CASE PRIORITY WHEN 0 THEN 'HIGH' WHEN 1 THEN 'LOW' ELSE NULL END AS CHAR(5)) AS PRIORITY, TARGET_REPLICA_SVR_IP, TARGET_REPLICA_SVR_PORT, TARGET_PAXOS_REPLICA_NUMBER, TARGET_REPLICA_TYPE, (CASE SOURCE_REPLICA_SVR_IP WHEN "" THEN NULL ELSE SOURCE_REPLICA_SVR_IP END) AS SOURCE_REPLICA_SVR_IP, SOURCE_REPLICA_SVR_PORT, SOURCE_PAXOS_REPLICA_NUMBER, (CASE SOURCE_REPLICA_TYPE WHEN "" THEN NULL ELSE SOURCE_REPLICA_TYPE END) AS SOURCE_REPLICA_TYPE, (CASE DATA_SOURCE_SVR_IP WHEN "" THEN NULL ELSE DATA_SOURCE_SVR_IP END) AS DATA_SOURCE_SVR_IP, DATA_SOURCE_SVR_PORT, CAST(CASE IS_MANUAL WHEN 0 THEN 'FALSE' WHEN 1 THEN 'TRUE' ELSE NULL END AS CHAR(6)) AS IS_MANUAL, TASK_EXEC_SVR_IP, TASK_EXEC_SVR_PORT, CAST(GMT_CREATE AS DATETIME) AS CREATE_TIME, CAST(SCHEDULE_TIME AS DATETIME) AS START_TIME, CAST(GMT_MODIFIED AS DATETIME) AS MODIFY_TIME, CAST(FINISH_TIME AS DATETIME) AS FINISH_TIME, (CASE EXECUTE_RESULT WHEN "" THEN NULL ELSE EXECUTE_RESULT END) AS EXECUTE_RESULT, COMMENT FROM OCEANBASE.__ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY WHERE TENANT_ID = EFFECTIVE_TENANT_ID() ) )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::cdb_ob_ls_replica_task_history_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_CDB_OB_LS_REPLICA_TASK_HISTORY_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_CDB_OB_LS_REPLICA_TASK_HISTORY_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( ( SELECT TENANT_ID, LS_ID, TASK_TYPE, TASK_ID, TASK_STATUS, CAST(CASE PRIORITY WHEN 0 THEN 'HIGH' WHEN 1 THEN 'LOW' ELSE NULL END AS CHAR(5)) AS PRIORITY, TARGET_REPLICA_SVR_IP, TARGET_REPLICA_SVR_PORT, TARGET_PAXOS_REPLICA_NUMBER, TARGET_REPLICA_TYPE, (CASE SOURCE_REPLICA_SVR_IP WHEN "" THEN NULL ELSE SOURCE_REPLICA_SVR_IP END) AS SOURCE_REPLICA_SVR_IP, SOURCE_REPLICA_SVR_PORT, SOURCE_PAXOS_REPLICA_NUMBER, (CASE SOURCE_REPLICA_TYPE WHEN "" THEN NULL ELSE SOURCE_REPLICA_TYPE END) AS SOURCE_REPLICA_TYPE, (CASE DATA_SOURCE_SVR_IP WHEN "" THEN NULL ELSE DATA_SOURCE_SVR_IP END) AS DATA_SOURCE_SVR_IP, DATA_SOURCE_SVR_PORT, CAST(CASE IS_MANUAL WHEN 0 THEN 'FALSE' WHEN 1 THEN 'TRUE' ELSE NULL END AS CHAR(6)) AS IS_MANUAL, TASK_EXEC_SVR_IP, TASK_EXEC_SVR_PORT, CAST(GMT_CREATE AS DATETIME) AS CREATE_TIME, CAST(SCHEDULE_TIME AS DATETIME) AS START_TIME, CAST(GMT_MODIFIED AS DATETIME) AS MODIFY_TIME, CAST(FINISH_TIME AS DATETIME) AS FINISH_TIME, (CASE EXECUTE_RESULT WHEN "" THEN NULL ELSE EXECUTE_RESULT END) AS EXECUTE_RESULT, COMMENT FROM OCEANBASE.__ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY ) )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::cdb_mview_logs_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.25201_25250.cpp b/src/share/inner_table/ob_inner_table_schema.25201_25250.cpp index 618a57f6d..e08bda956 100644 --- a/src/share/inner_table/ob_inner_table_schema.25201_25250.cpp +++ b/src/share/inner_table/ob_inner_table_schema.25201_25250.cpp @@ -110,7 +110,7 @@ int ObInnerTableSchema::dba_ob_ls_replica_tasks_ora_schema(ObTableSchema &table_ table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( ( SELECT LS_ID, TASK_TYPE, TASK_ID, TASK_STATUS, CAST(CASE PRIORITY WHEN 1 THEN 'HIGH' WHEN 2 THEN 'LOW' ELSE NULL END AS CHAR(5)) AS PRIORITY, TARGET_REPLICA_SVR_IP, TARGET_REPLICA_SVR_PORT, TARGET_PAXOS_REPLICA_NUMBER, TARGET_REPLICA_TYPE, CASE SOURCE_REPLICA_SVR_IP WHEN '' THEN NULL ELSE SOURCE_REPLICA_SVR_IP END AS SOURCE_REPLICA_SVR_IP, SOURCE_REPLICA_SVR_PORT, SOURCE_PAXOS_REPLICA_NUMBER, CASE SOURCE_REPLICA_TYPE WHEN '' THEN NULL ELSE SOURCE_REPLICA_TYPE END AS SOURCE_REPLICA_TYPE, TASK_EXEC_SVR_IP, TASK_EXEC_SVR_PORT, CAST(GMT_CREATE AS TIMESTAMP(6)) AS CREATE_TIME, CAST(SCHEDULE_TIME AS TIMESTAMP(6)) AS START_TIME, CAST(GMT_MODIFIED AS TIMESTAMP(6)) AS MODIFY_TIME, "COMMENT" FROM SYS.ALL_VIRTUAL_LS_REPLICA_TASK WHERE TENANT_ID = EFFECTIVE_TENANT_ID() ) )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( ( SELECT LS_ID, TASK_TYPE, TASK_ID, TASK_STATUS, CAST(CASE PRIORITY WHEN 0 THEN 'HIGH' WHEN 1 THEN 'LOW' ELSE NULL END AS CHAR(5)) AS PRIORITY, TARGET_REPLICA_SVR_IP, TARGET_REPLICA_SVR_PORT, TARGET_PAXOS_REPLICA_NUMBER, TARGET_REPLICA_TYPE, CASE SOURCE_REPLICA_SVR_IP WHEN '' THEN NULL ELSE SOURCE_REPLICA_SVR_IP END AS SOURCE_REPLICA_SVR_IP, SOURCE_REPLICA_SVR_PORT, SOURCE_PAXOS_REPLICA_NUMBER, CASE SOURCE_REPLICA_TYPE WHEN '' THEN NULL ELSE SOURCE_REPLICA_TYPE END AS SOURCE_REPLICA_TYPE, CASE DATA_SOURCE_SVR_IP WHEN '' THEN NULL ELSE DATA_SOURCE_SVR_IP END AS DATA_SOURCE_SVR_IP, DATA_SOURCE_SVR_PORT, CAST(CASE IS_MANUAL WHEN 0 THEN 'FALSE' WHEN 1 THEN 'TRUE' ELSE NULL END AS CHAR(6)) AS IS_MANUAL, TASK_EXEC_SVR_IP, TASK_EXEC_SVR_PORT, CAST(GMT_CREATE AS TIMESTAMP(6)) AS CREATE_TIME, CAST(SCHEDULE_TIME AS TIMESTAMP(6)) AS START_TIME, CAST(GMT_MODIFIED AS TIMESTAMP(6)) AS MODIFY_TIME, "COMMENT" FROM SYS.ALL_VIRTUAL_LS_REPLICA_TASK WHERE TENANT_ID = EFFECTIVE_TENANT_ID() ) )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema.25251_25300.cpp b/src/share/inner_table/ob_inner_table_schema.25251_25300.cpp index de5dce0dd..809593437 100644 --- a/src/share/inner_table/ob_inner_table_schema.25251_25300.cpp +++ b/src/share/inner_table/ob_inner_table_schema.25251_25300.cpp @@ -775,6 +775,56 @@ int ObInnerTableSchema::user_users_schema(ObTableSchema &table_schema) return ret; } +int ObInnerTableSchema::dba_ob_ls_replica_task_history_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_OB_LS_REPLICA_TASK_HISTORY_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_OB_LS_REPLICA_TASK_HISTORY_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( ( SELECT LS_ID, TASK_TYPE, TASK_ID, TASK_STATUS, CAST(CASE PRIORITY WHEN 0 THEN 'HIGH' WHEN 1 THEN 'LOW' ELSE NULL END AS CHAR(5)) AS PRIORITY, TARGET_REPLICA_SVR_IP, TARGET_REPLICA_SVR_PORT, TARGET_PAXOS_REPLICA_NUMBER, TARGET_REPLICA_TYPE, CASE SOURCE_REPLICA_SVR_IP WHEN '' THEN NULL ELSE SOURCE_REPLICA_SVR_IP END AS SOURCE_REPLICA_SVR_IP, SOURCE_REPLICA_SVR_PORT, SOURCE_PAXOS_REPLICA_NUMBER, CASE SOURCE_REPLICA_TYPE WHEN '' THEN NULL ELSE SOURCE_REPLICA_TYPE END AS SOURCE_REPLICA_TYPE, CASE DATA_SOURCE_SVR_IP WHEN '' THEN NULL ELSE DATA_SOURCE_SVR_IP END AS DATA_SOURCE_SVR_IP, DATA_SOURCE_SVR_PORT, CAST(CASE IS_MANUAL WHEN 0 THEN 'FALSE' WHEN 1 THEN 'TRUE' ELSE NULL END AS CHAR(6)) AS IS_MANUAL, TASK_EXEC_SVR_IP, TASK_EXEC_SVR_PORT, CAST(GMT_CREATE AS TIMESTAMP(6)) AS CREATE_TIME, CAST(SCHEDULE_TIME AS TIMESTAMP(6)) AS START_TIME, CAST(GMT_MODIFIED AS TIMESTAMP(6)) AS MODIFY_TIME, CAST(FINISH_TIME AS TIMESTAMP(6)) AS FINISH_TIME, CASE EXECUTE_RESULT WHEN '' THEN NULL ELSE EXECUTE_RESULT END AS EXECUTE_RESULT, "COMMENT" FROM SYS.ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY WHERE TENANT_ID = EFFECTIVE_TENANT_ID() ) )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::dba_mview_logs_ora_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.501_550.cpp b/src/share/inner_table/ob_inner_table_schema.501_550.cpp index 0ccf20f71..21590a612 100644 --- a/src/share/inner_table/ob_inner_table_schema.501_550.cpp +++ b/src/share/inner_table/ob_inner_table_schema.501_550.cpp @@ -1035,6 +1035,490 @@ int ObInnerTableSchema::all_tenant_snapshot_ls_replica_history_schema(ObTableSch return ret; } +int ObInnerTableSchema::all_ls_replica_task_history_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_LS_REPLICA_TASK_HISTORY_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_LS_REPLICA_TASK_HISTORY_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ObObj gmt_create_default; + ObObj gmt_create_default_null; + + gmt_create_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_create_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("gmt_create", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_BINARY,//collation_type + 0, //column length + -1, //column_precision + 6, //column_scale + true,//is nullable + false, //is_autoincrement + false, //is_on_update_for_timestamp + gmt_create_default_null, + gmt_create_default) + } + + if (OB_SUCC(ret)) { + ObObj gmt_modified_default; + ObObj gmt_modified_default_null; + + gmt_modified_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_modified_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("gmt_modified", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_BINARY,//collation_type + 0, //column length + -1, //column_precision + 6, //column_scale + true,//is nullable + false, //is_autoincrement + true, //is_on_update_for_timestamp + gmt_modified_default_null, + gmt_modified_default) + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ls_id", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_type", //column_name + ++column_id, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_DISASTER_RECOVERY_TASK_TYPE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_id", //column_name + ++column_id, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_TRACE_STAT_BUFFER_SIZE, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_status", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_COLUMN_COMMENT_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ObObj priority_default; + priority_default.set_int(1); + ADD_COLUMN_SCHEMA_T("priority", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + priority_default, + priority_default); //default_value + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("target_replica_svr_ip", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_IP_ADDR_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("target_replica_svr_port", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("target_paxos_replica_number", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("target_replica_type", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_REPLICA_TYPE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("source_replica_svr_ip", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_IP_ADDR_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("source_replica_svr_port", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("source_paxos_replica_number", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("source_replica_type", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_REPLICA_TYPE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("data_source_svr_ip", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_IP_ADDR_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("data_source_svr_port", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ObObj is_manual_default; + is_manual_default.set_tinyint(0); + ADD_COLUMN_SCHEMA_T("is_manual", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTinyIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + 1, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false, //is_autoincrement + is_manual_default, + is_manual_default); //default_value + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_exec_svr_ip", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_IP_ADDR_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_exec_svr_port", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ObObj gmt_default; + ObObj gmt_default_null; + + gmt_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("generate_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + 6, //column_scale + false, //is_nullable + false, //is_autoincrement + false, //is_on_update_for_timestamp + gmt_default_null, + gmt_default) + } + + if (OB_SUCC(ret)) { + ObObj gmt_default; + ObObj gmt_default_null; + + gmt_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("schedule_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + 6, //column_scale + false, //is_nullable + false, //is_autoincrement + false, //is_on_update_for_timestamp + gmt_default_null, + gmt_default) + } + + if (OB_SUCC(ret)) { + ObObj gmt_default; + ObObj gmt_default_null; + + gmt_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("finish_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + 6, //column_scale + false, //is_nullable + false, //is_autoincrement + false, //is_on_update_for_timestamp + gmt_default_null, + gmt_default) + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("execute_result", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_COLUMN_COMMENT_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("comment", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_COLUMN_COMMENT_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_LS_REPLICA_TASK_HISTORY_TID); + table_schema.set_aux_lob_meta_tid(OB_ALL_LS_REPLICA_TASK_HISTORY_AUX_LOB_META_TID); + table_schema.set_aux_lob_piece_tid(OB_ALL_LS_REPLICA_TASK_HISTORY_AUX_LOB_PIECE_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::all_user_proxy_info_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.50501_50550.cpp b/src/share/inner_table/ob_inner_table_schema.50501_50550.cpp index 416ada242..d60eb1b9a 100644 --- a/src/share/inner_table/ob_inner_table_schema.50501_50550.cpp +++ b/src/share/inner_table/ob_inner_table_schema.50501_50550.cpp @@ -700,6 +700,141 @@ int ObInnerTableSchema::all_tenant_snapshot_ls_replica_history_aux_lob_meta_sche return ret; } +int ObInnerTableSchema::all_ls_replica_task_history_aux_lob_meta_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_LS_REPLICA_TASK_HISTORY_AUX_LOB_META_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(AUX_LOB_META); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_LS_REPLICA_TASK_HISTORY_AUX_LOB_META_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 16, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("seq_id", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 8192, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("binary_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("char_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("piece_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_data", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 262144, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_LS_REPLICA_TASK_HISTORY_AUX_LOB_META_TID); + table_schema.set_data_table_id(OB_ALL_LS_REPLICA_TASK_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::all_user_proxy_info_aux_lob_meta_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.60501_60550.cpp b/src/share/inner_table/ob_inner_table_schema.60501_60550.cpp index 5e2488a05..04d44f0e6 100644 --- a/src/share/inner_table/ob_inner_table_schema.60501_60550.cpp +++ b/src/share/inner_table/ob_inner_table_schema.60501_60550.cpp @@ -475,6 +475,96 @@ int ObInnerTableSchema::all_tenant_snapshot_ls_replica_history_aux_lob_piece_sch return ret; } +int ObInnerTableSchema::all_ls_replica_task_history_aux_lob_piece_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_LS_REPLICA_TASK_HISTORY_AUX_LOB_PIECE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(1); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(AUX_LOB_PIECE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_LS_REPLICA_TASK_HISTORY_AUX_LOB_PIECE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("piece_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("data_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_data", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 32, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_LS_REPLICA_TASK_HISTORY_AUX_LOB_PIECE_TID); + table_schema.set_data_table_id(OB_ALL_LS_REPLICA_TASK_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::all_user_proxy_info_aux_lob_piece_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.h b/src/share/inner_table/ob_inner_table_schema.h index 2b112a4bc..16492e107 100644 --- a/src/share/inner_table/ob_inner_table_schema.h +++ b/src/share/inner_table/ob_inner_table_schema.h @@ -621,6 +621,7 @@ public: static int all_column_privilege_schema(share::schema::ObTableSchema &table_schema); static int all_column_privilege_history_schema(share::schema::ObTableSchema &table_schema); static int all_tenant_snapshot_ls_replica_history_schema(share::schema::ObTableSchema &table_schema); + static int all_ls_replica_task_history_schema(share::schema::ObTableSchema &table_schema); static int all_user_proxy_info_schema(share::schema::ObTableSchema &table_schema); static int all_user_proxy_info_history_schema(share::schema::ObTableSchema &table_schema); static int all_user_proxy_role_info_schema(share::schema::ObTableSchema &table_schema); @@ -1070,6 +1071,7 @@ public: static int all_virtual_column_privilege_history_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_tenant_snapshot_ls_replica_history_schema(share::schema::ObTableSchema &table_schema); static int enabled_roles_schema(share::schema::ObTableSchema &table_schema); + static int all_virtual_ls_replica_task_history_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_session_ps_info_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_tracepoint_info_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_compatibility_control_schema(share::schema::ObTableSchema &table_schema); @@ -1347,6 +1349,7 @@ public: static int all_virtual_transfer_partition_task_history_real_agent_ora_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_ls_snapshot_ora_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_index_usage_info_real_agent_ora_schema(share::schema::ObTableSchema &table_schema); + static int all_virtual_ls_replica_task_history_ora_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_session_ps_info_ora_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_tracepoint_info_ora_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_user_proxy_info_real_agent_ora_schema(share::schema::ObTableSchema &table_schema); @@ -1752,6 +1755,8 @@ public: static int gv_ob_ls_snapshots_schema(share::schema::ObTableSchema &table_schema); static int v_ob_ls_snapshots_schema(share::schema::ObTableSchema &table_schema); static int dba_ob_clone_history_schema(share::schema::ObTableSchema &table_schema); + static int dba_ob_ls_replica_task_history_schema(share::schema::ObTableSchema &table_schema); + static int cdb_ob_ls_replica_task_history_schema(share::schema::ObTableSchema &table_schema); static int cdb_mview_logs_schema(share::schema::ObTableSchema &table_schema); static int dba_mview_logs_schema(share::schema::ObTableSchema &table_schema); static int cdb_mviews_schema(share::schema::ObTableSchema &table_schema); @@ -2060,6 +2065,7 @@ public: static int dba_ob_transfer_partition_tasks_ora_schema(share::schema::ObTableSchema &table_schema); static int dba_ob_transfer_partition_task_history_ora_schema(share::schema::ObTableSchema &table_schema); static int user_users_schema(share::schema::ObTableSchema &table_schema); + static int dba_ob_ls_replica_task_history_ora_schema(share::schema::ObTableSchema &table_schema); static int dba_mview_logs_ora_schema(share::schema::ObTableSchema &table_schema); static int all_mview_logs_ora_schema(share::schema::ObTableSchema &table_schema); static int user_mview_logs_ora_schema(share::schema::ObTableSchema &table_schema); @@ -2561,6 +2567,7 @@ public: static int all_column_privilege_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_column_privilege_history_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_tenant_snapshot_ls_replica_history_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); + static int all_ls_replica_task_history_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_user_proxy_info_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_user_proxy_info_history_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_user_proxy_role_info_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); @@ -2860,6 +2867,7 @@ public: static int all_column_privilege_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_column_privilege_history_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_tenant_snapshot_ls_replica_history_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); + static int all_ls_replica_task_history_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_user_proxy_info_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_user_proxy_info_history_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_user_proxy_role_info_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); @@ -3378,6 +3386,7 @@ const schema_create_func sys_table_schema_creators [] = { ObInnerTableSchema::all_column_privilege_schema, ObInnerTableSchema::all_column_privilege_history_schema, ObInnerTableSchema::all_tenant_snapshot_ls_replica_history_schema, + ObInnerTableSchema::all_ls_replica_task_history_schema, ObInnerTableSchema::all_user_proxy_info_schema, ObInnerTableSchema::all_user_proxy_info_history_schema, ObInnerTableSchema::all_user_proxy_role_info_schema, @@ -3830,6 +3839,7 @@ const schema_create_func virtual_table_schema_creators [] = { ObInnerTableSchema::all_virtual_column_privilege_history_schema, ObInnerTableSchema::all_virtual_tenant_snapshot_ls_replica_history_schema, ObInnerTableSchema::enabled_roles_schema, + ObInnerTableSchema::all_virtual_ls_replica_task_history_schema, ObInnerTableSchema::all_virtual_session_ps_info_schema, ObInnerTableSchema::all_virtual_tracepoint_info_schema, ObInnerTableSchema::all_virtual_compatibility_control_schema, @@ -4117,6 +4127,7 @@ const schema_create_func virtual_table_schema_creators [] = { ObInnerTableSchema::all_virtual_transfer_partition_task_history_real_agent_ora_schema, ObInnerTableSchema::all_virtual_ls_snapshot_ora_schema, ObInnerTableSchema::all_virtual_index_usage_info_real_agent_ora_schema, + ObInnerTableSchema::all_virtual_ls_replica_task_history_ora_schema, ObInnerTableSchema::all_virtual_session_ps_info_ora_schema, ObInnerTableSchema::all_virtual_tracepoint_info_ora_schema, ObInnerTableSchema::all_virtual_user_proxy_info_real_agent_ora_schema, @@ -4609,6 +4620,8 @@ const schema_create_func sys_view_schema_creators [] = { ObInnerTableSchema::gv_ob_ls_snapshots_schema, ObInnerTableSchema::v_ob_ls_snapshots_schema, ObInnerTableSchema::dba_ob_clone_history_schema, + ObInnerTableSchema::dba_ob_ls_replica_task_history_schema, + ObInnerTableSchema::cdb_ob_ls_replica_task_history_schema, ObInnerTableSchema::cdb_mview_logs_schema, ObInnerTableSchema::dba_mview_logs_schema, ObInnerTableSchema::cdb_mviews_schema, @@ -4917,6 +4930,7 @@ const schema_create_func sys_view_schema_creators [] = { ObInnerTableSchema::dba_ob_transfer_partition_tasks_ora_schema, ObInnerTableSchema::dba_ob_transfer_partition_task_history_ora_schema, ObInnerTableSchema::user_users_schema, + ObInnerTableSchema::dba_ob_ls_replica_task_history_ora_schema, ObInnerTableSchema::dba_mview_logs_ora_schema, ObInnerTableSchema::all_mview_logs_ora_schema, ObInnerTableSchema::user_mview_logs_ora_schema, @@ -5520,6 +5534,7 @@ const uint64_t tenant_space_tables [] = { OB_ALL_COLUMN_PRIVILEGE_TID, OB_ALL_COLUMN_PRIVILEGE_HISTORY_TID, OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_TID, + OB_ALL_LS_REPLICA_TASK_HISTORY_TID, OB_ALL_USER_PROXY_INFO_TID, OB_ALL_USER_PROXY_INFO_HISTORY_TID, OB_ALL_USER_PROXY_ROLE_INFO_TID, @@ -5753,6 +5768,7 @@ const uint64_t tenant_space_tables [] = { OB_ALL_VIRTUAL_LS_SNAPSHOT_TID, OB_ALL_VIRTUAL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_TID, OB_ENABLED_ROLES_TID, + OB_ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY_TID, OB_ALL_VIRTUAL_SESSION_PS_INFO_TID, OB_ALL_VIRTUAL_TRACEPOINT_INFO_TID, OB_ALL_VIRTUAL_COMPATIBILITY_CONTROL_TID, @@ -6033,6 +6049,7 @@ const uint64_t tenant_space_tables [] = { OB_ALL_VIRTUAL_TRANSFER_PARTITION_TASK_HISTORY_REAL_AGENT_ORA_TID, OB_ALL_VIRTUAL_LS_SNAPSHOT_ORA_TID, OB_ALL_VIRTUAL_INDEX_USAGE_INFO_REAL_AGENT_ORA_TID, + OB_ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY_ORA_TID, OB_ALL_VIRTUAL_SESSION_PS_INFO_ORA_TID, OB_ALL_VIRTUAL_TRACEPOINT_INFO_ORA_TID, OB_ALL_VIRTUAL_USER_PROXY_INFO_REAL_AGENT_ORA_TID, @@ -6327,6 +6344,7 @@ const uint64_t tenant_space_tables [] = { OB_COLUMNS_PRIV_TID, OB_GV_OB_LS_SNAPSHOTS_TID, OB_V_OB_LS_SNAPSHOTS_TID, + OB_DBA_OB_LS_REPLICA_TASK_HISTORY_TID, OB_DBA_MVIEW_LOGS_TID, OB_DBA_MVIEWS_TID, OB_DBA_MVREF_STATS_SYS_DEFAULTS_TID, @@ -6626,6 +6644,7 @@ const uint64_t tenant_space_tables [] = { OB_DBA_OB_TRANSFER_PARTITION_TASKS_ORA_TID, OB_DBA_OB_TRANSFER_PARTITION_TASK_HISTORY_ORA_TID, OB_USER_USERS_TID, + OB_DBA_OB_LS_REPLICA_TASK_HISTORY_ORA_TID, OB_DBA_MVIEW_LOGS_ORA_TID, OB_ALL_MVIEW_LOGS_ORA_TID, OB_USER_MVIEW_LOGS_ORA_TID, @@ -7282,6 +7301,7 @@ const uint64_t tenant_space_tables [] = { OB_ALL_COLUMN_PRIVILEGE_AUX_LOB_META_TID, OB_ALL_COLUMN_PRIVILEGE_HISTORY_AUX_LOB_META_TID, OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_AUX_LOB_META_TID, + OB_ALL_LS_REPLICA_TASK_HISTORY_AUX_LOB_META_TID, OB_ALL_USER_PROXY_INFO_AUX_LOB_META_TID, OB_ALL_USER_PROXY_INFO_HISTORY_AUX_LOB_META_TID, OB_ALL_USER_PROXY_ROLE_INFO_AUX_LOB_META_TID, @@ -7557,6 +7577,7 @@ const uint64_t tenant_space_tables [] = { OB_ALL_COLUMN_PRIVILEGE_AUX_LOB_PIECE_TID, OB_ALL_COLUMN_PRIVILEGE_HISTORY_AUX_LOB_PIECE_TID, OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_AUX_LOB_PIECE_TID, + OB_ALL_LS_REPLICA_TASK_HISTORY_AUX_LOB_PIECE_TID, OB_ALL_USER_PROXY_INFO_AUX_LOB_PIECE_TID, OB_ALL_USER_PROXY_INFO_HISTORY_AUX_LOB_PIECE_TID, OB_ALL_USER_PROXY_ROLE_INFO_AUX_LOB_PIECE_TID, @@ -7706,6 +7727,7 @@ const uint64_t all_ora_mapping_virtual_table_org_tables [] = { OB_ALL_VIRTUAL_CGROUP_CONFIG_TID, OB_ALL_VIRTUAL_SYS_VARIABLE_DEFAULT_VALUE_TID, OB_ALL_VIRTUAL_LS_SNAPSHOT_TID, + OB_ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY_TID, OB_ALL_VIRTUAL_SESSION_PS_INFO_TID, OB_ALL_VIRTUAL_TRACEPOINT_INFO_TID, OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_TID, @@ -7853,6 +7875,7 @@ const uint64_t all_ora_mapping_virtual_tables [] = { OB_ALL_VIRTUAL_SQL_AUDIT_O , OB_ALL_VIRTUAL_CGROUP_CONFIG_ORA_TID , OB_ALL_VIRTUAL_SYS_VARIABLE_DEFAULT_VALUE_ORA_TID , OB_ALL_VIRTUAL_LS_SNAPSHOT_ORA_TID +, OB_ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY_ORA_TID , OB_ALL_VIRTUAL_SESSION_PS_INFO_ORA_TID , OB_ALL_VIRTUAL_TRACEPOINT_INFO_ORA_TID , OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_ORA_TID @@ -8142,6 +8165,7 @@ const char* const tenant_space_table_names [] = { OB_ALL_COLUMN_PRIVILEGE_TNAME, OB_ALL_COLUMN_PRIVILEGE_HISTORY_TNAME, OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_TNAME, + OB_ALL_LS_REPLICA_TASK_HISTORY_TNAME, OB_ALL_USER_PROXY_INFO_TNAME, OB_ALL_USER_PROXY_INFO_HISTORY_TNAME, OB_ALL_USER_PROXY_ROLE_INFO_TNAME, @@ -8375,6 +8399,7 @@ const char* const tenant_space_table_names [] = { OB_ALL_VIRTUAL_LS_SNAPSHOT_TNAME, OB_ALL_VIRTUAL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_TNAME, OB_ENABLED_ROLES_TNAME, + OB_ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY_TNAME, OB_ALL_VIRTUAL_SESSION_PS_INFO_TNAME, OB_ALL_VIRTUAL_TRACEPOINT_INFO_TNAME, OB_ALL_VIRTUAL_COMPATIBILITY_CONTROL_TNAME, @@ -8655,6 +8680,7 @@ const char* const tenant_space_table_names [] = { OB_ALL_VIRTUAL_TRANSFER_PARTITION_TASK_HISTORY_REAL_AGENT_ORA_TNAME, OB_ALL_VIRTUAL_LS_SNAPSHOT_ORA_TNAME, OB_ALL_VIRTUAL_INDEX_USAGE_INFO_REAL_AGENT_ORA_TNAME, + OB_ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY_ORA_TNAME, OB_ALL_VIRTUAL_SESSION_PS_INFO_ORA_TNAME, OB_ALL_VIRTUAL_TRACEPOINT_INFO_ORA_TNAME, OB_ALL_VIRTUAL_USER_PROXY_INFO_REAL_AGENT_ORA_TNAME, @@ -8949,6 +8975,7 @@ const char* const tenant_space_table_names [] = { OB_COLUMNS_PRIV_TNAME, OB_GV_OB_LS_SNAPSHOTS_TNAME, OB_V_OB_LS_SNAPSHOTS_TNAME, + OB_DBA_OB_LS_REPLICA_TASK_HISTORY_TNAME, OB_DBA_MVIEW_LOGS_TNAME, OB_DBA_MVIEWS_TNAME, OB_DBA_MVREF_STATS_SYS_DEFAULTS_TNAME, @@ -9248,6 +9275,7 @@ const char* const tenant_space_table_names [] = { OB_DBA_OB_TRANSFER_PARTITION_TASKS_ORA_TNAME, OB_DBA_OB_TRANSFER_PARTITION_TASK_HISTORY_ORA_TNAME, OB_USER_USERS_TNAME, + OB_DBA_OB_LS_REPLICA_TASK_HISTORY_ORA_TNAME, OB_DBA_MVIEW_LOGS_ORA_TNAME, OB_ALL_MVIEW_LOGS_ORA_TNAME, OB_USER_MVIEW_LOGS_ORA_TNAME, @@ -9904,6 +9932,7 @@ const char* const tenant_space_table_names [] = { OB_ALL_COLUMN_PRIVILEGE_AUX_LOB_META_TNAME, OB_ALL_COLUMN_PRIVILEGE_HISTORY_AUX_LOB_META_TNAME, OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_AUX_LOB_META_TNAME, + OB_ALL_LS_REPLICA_TASK_HISTORY_AUX_LOB_META_TNAME, OB_ALL_USER_PROXY_INFO_AUX_LOB_META_TNAME, OB_ALL_USER_PROXY_INFO_HISTORY_AUX_LOB_META_TNAME, OB_ALL_USER_PROXY_ROLE_INFO_AUX_LOB_META_TNAME, @@ -10179,6 +10208,7 @@ const char* const tenant_space_table_names [] = { OB_ALL_COLUMN_PRIVILEGE_AUX_LOB_PIECE_TNAME, OB_ALL_COLUMN_PRIVILEGE_HISTORY_AUX_LOB_PIECE_TNAME, OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_AUX_LOB_PIECE_TNAME, + OB_ALL_LS_REPLICA_TASK_HISTORY_AUX_LOB_PIECE_TNAME, OB_ALL_USER_PROXY_INFO_AUX_LOB_PIECE_TNAME, OB_ALL_USER_PROXY_INFO_HISTORY_AUX_LOB_PIECE_TNAME, OB_ALL_USER_PROXY_ROLE_INFO_AUX_LOB_PIECE_TNAME, @@ -10542,6 +10572,7 @@ const uint64_t restrict_access_virtual_tables[] = { OB_ALL_VIRTUAL_TRANSFER_PARTITION_TASK_HISTORY_REAL_AGENT_ORA_TID, OB_ALL_VIRTUAL_LS_SNAPSHOT_ORA_TID, OB_ALL_VIRTUAL_INDEX_USAGE_INFO_REAL_AGENT_ORA_TID, + OB_ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY_ORA_TID, OB_ALL_VIRTUAL_SESSION_PS_INFO_ORA_TID, OB_ALL_VIRTUAL_TRACEPOINT_INFO_ORA_TID, OB_ALL_VIRTUAL_USER_PROXY_INFO_REAL_AGENT_ORA_TID, @@ -13040,6 +13071,14 @@ LOBMapping const lob_aux_table_mappings [] = { ObInnerTableSchema::all_tenant_snapshot_ls_replica_history_aux_lob_piece_schema }, + { + OB_ALL_LS_REPLICA_TASK_HISTORY_TID, + OB_ALL_LS_REPLICA_TASK_HISTORY_AUX_LOB_META_TID, + OB_ALL_LS_REPLICA_TASK_HISTORY_AUX_LOB_PIECE_TID, + ObInnerTableSchema::all_ls_replica_task_history_aux_lob_meta_schema, + ObInnerTableSchema::all_ls_replica_task_history_aux_lob_piece_schema + }, + { OB_ALL_USER_PROXY_INFO_TID, OB_ALL_USER_PROXY_INFO_AUX_LOB_META_TID, @@ -13125,12 +13164,12 @@ static inline int get_sys_table_lob_aux_schema(const uint64_t tid, } const int64_t OB_CORE_TABLE_COUNT = 4; -const int64_t OB_SYS_TABLE_COUNT = 296; -const int64_t OB_VIRTUAL_TABLE_COUNT = 823; -const int64_t OB_SYS_VIEW_COUNT = 912; -const int64_t OB_SYS_TENANT_TABLE_COUNT = 2036; +const int64_t OB_SYS_TABLE_COUNT = 297; +const int64_t OB_VIRTUAL_TABLE_COUNT = 825; +const int64_t OB_SYS_VIEW_COUNT = 915; +const int64_t OB_SYS_TENANT_TABLE_COUNT = 2042; const int64_t OB_CORE_SCHEMA_VERSION = 1; -const int64_t OB_BOOTSTRAP_SCHEMA_VERSION = 2039; +const int64_t OB_BOOTSTRAP_SCHEMA_VERSION = 2045; } // end namespace share } // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema.lob.cpp b/src/share/inner_table/ob_inner_table_schema.lob.cpp index 95a758380..9d142bcc0 100644 --- a/src/share/inner_table/ob_inner_table_schema.lob.cpp +++ b/src/share/inner_table/ob_inner_table_schema.lob.cpp @@ -21,7 +21,7 @@ inner_lob_map_t inner_lob_map; bool lob_mapping_init() { int ret = OB_SUCCESS; - if (OB_FAIL(inner_lob_map.create(299, ObModIds::OB_INNER_LOB_HASH_SET))) { + if (OB_FAIL(inner_lob_map.create(300, ObModIds::OB_INNER_LOB_HASH_SET))) { SERVER_LOG(WARN, "fail to create inner lob map", K(ret)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < ARRAYSIZEOF(lob_aux_table_mappings); ++i) { diff --git a/src/share/inner_table/ob_inner_table_schema_constants.h b/src/share/inner_table/ob_inner_table_schema_constants.h index 41cbb5947..06953678d 100644 --- a/src/share/inner_table/ob_inner_table_schema_constants.h +++ b/src/share/inner_table/ob_inner_table_schema_constants.h @@ -321,6 +321,7 @@ const uint64_t OB_ALL_TRUSTED_ROOT_CERTIFICATE_TID = 502; // "__all_trusted_root const uint64_t OB_ALL_COLUMN_PRIVILEGE_TID = 505; // "__all_column_privilege" const uint64_t OB_ALL_COLUMN_PRIVILEGE_HISTORY_TID = 506; // "__all_column_privilege_history" const uint64_t OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_TID = 507; // "__all_tenant_snapshot_ls_replica_history" +const uint64_t OB_ALL_LS_REPLICA_TASK_HISTORY_TID = 508; // "__all_ls_replica_task_history" const uint64_t OB_ALL_USER_PROXY_INFO_TID = 512; // "__all_user_proxy_info" const uint64_t OB_ALL_USER_PROXY_INFO_HISTORY_TID = 513; // "__all_user_proxy_info_history" const uint64_t OB_ALL_USER_PROXY_ROLE_INFO_TID = 514; // "__all_user_proxy_role_info" @@ -770,6 +771,7 @@ const uint64_t OB_ALL_VIRTUAL_COLUMN_PRIVILEGE_TID = 12462; // "__all_virtual_co const uint64_t OB_ALL_VIRTUAL_COLUMN_PRIVILEGE_HISTORY_TID = 12463; // "__all_virtual_column_privilege_history" const uint64_t OB_ALL_VIRTUAL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_TID = 12464; // "__all_virtual_tenant_snapshot_ls_replica_history" const uint64_t OB_ENABLED_ROLES_TID = 12466; // "ENABLED_ROLES" +const uint64_t OB_ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY_TID = 12467; // "__all_virtual_ls_replica_task_history" const uint64_t OB_ALL_VIRTUAL_SESSION_PS_INFO_TID = 12468; // "__all_virtual_session_ps_info" const uint64_t OB_ALL_VIRTUAL_TRACEPOINT_INFO_TID = 12469; // "__all_virtual_tracepoint_info" const uint64_t OB_ALL_VIRTUAL_COMPATIBILITY_CONTROL_TID = 12473; // "__all_virtual_compatibility_control" @@ -1047,6 +1049,7 @@ const uint64_t OB_ALL_VIRTUAL_TRANSFER_PARTITION_TASK_REAL_AGENT_ORA_TID = 15430 const uint64_t OB_ALL_VIRTUAL_TRANSFER_PARTITION_TASK_HISTORY_REAL_AGENT_ORA_TID = 15431; // "ALL_VIRTUAL_TRANSFER_PARTITION_TASK_HISTORY_REAL_AGENT_ORA" const uint64_t OB_ALL_VIRTUAL_LS_SNAPSHOT_ORA_TID = 15439; // "ALL_VIRTUAL_LS_SNAPSHOT_ORA" const uint64_t OB_ALL_VIRTUAL_INDEX_USAGE_INFO_REAL_AGENT_ORA_TID = 15440; // "ALL_VIRTUAL_INDEX_USAGE_INFO_REAL_AGENT_ORA" +const uint64_t OB_ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY_ORA_TID = 15443; // "ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY_ORA" const uint64_t OB_ALL_VIRTUAL_SESSION_PS_INFO_ORA_TID = 15444; // "ALL_VIRTUAL_SESSION_PS_INFO_ORA" const uint64_t OB_ALL_VIRTUAL_TRACEPOINT_INFO_ORA_TID = 15445; // "ALL_VIRTUAL_TRACEPOINT_INFO_ORA" const uint64_t OB_ALL_VIRTUAL_USER_PROXY_INFO_REAL_AGENT_ORA_TID = 15446; // "ALL_VIRTUAL_USER_PROXY_INFO_REAL_AGENT_ORA" @@ -1452,6 +1455,8 @@ const uint64_t OB_COLUMNS_PRIV_TID = 21516; // "columns_priv" const uint64_t OB_GV_OB_LS_SNAPSHOTS_TID = 21517; // "GV$OB_LS_SNAPSHOTS" const uint64_t OB_V_OB_LS_SNAPSHOTS_TID = 21518; // "V$OB_LS_SNAPSHOTS" const uint64_t OB_DBA_OB_CLONE_HISTORY_TID = 21519; // "DBA_OB_CLONE_HISTORY" +const uint64_t OB_DBA_OB_LS_REPLICA_TASK_HISTORY_TID = 21523; // "DBA_OB_LS_REPLICA_TASK_HISTORY" +const uint64_t OB_CDB_OB_LS_REPLICA_TASK_HISTORY_TID = 21524; // "CDB_OB_LS_REPLICA_TASK_HISTORY" const uint64_t OB_CDB_MVIEW_LOGS_TID = 21525; // "CDB_MVIEW_LOGS" const uint64_t OB_DBA_MVIEW_LOGS_TID = 21526; // "DBA_MVIEW_LOGS" const uint64_t OB_CDB_MVIEWS_TID = 21527; // "CDB_MVIEWS" @@ -1760,6 +1765,7 @@ const uint64_t OB_DBA_OB_IMPORT_TABLE_TASK_HISTORY_ORA_TID = 25267; // "DBA_OB_I const uint64_t OB_DBA_OB_TRANSFER_PARTITION_TASKS_ORA_TID = 25275; // "DBA_OB_TRANSFER_PARTITION_TASKS_ORA" const uint64_t OB_DBA_OB_TRANSFER_PARTITION_TASK_HISTORY_ORA_TID = 25276; // "DBA_OB_TRANSFER_PARTITION_TASK_HISTORY_ORA" const uint64_t OB_USER_USERS_TID = 25278; // "USER_USERS" +const uint64_t OB_DBA_OB_LS_REPLICA_TASK_HISTORY_ORA_TID = 25279; // "DBA_OB_LS_REPLICA_TASK_HISTORY_ORA" const uint64_t OB_DBA_MVIEW_LOGS_ORA_TID = 25283; // "DBA_MVIEW_LOGS_ORA" const uint64_t OB_ALL_MVIEW_LOGS_ORA_TID = 25284; // "ALL_MVIEW_LOGS_ORA" const uint64_t OB_USER_MVIEW_LOGS_ORA_TID = 25285; // "USER_MVIEW_LOGS_ORA" @@ -2261,6 +2267,7 @@ const uint64_t OB_ALL_TRUSTED_ROOT_CERTIFICATE_AUX_LOB_META_TID = 50502; // "__a const uint64_t OB_ALL_COLUMN_PRIVILEGE_AUX_LOB_META_TID = 50505; // "__all_column_privilege_aux_lob_meta" const uint64_t OB_ALL_COLUMN_PRIVILEGE_HISTORY_AUX_LOB_META_TID = 50506; // "__all_column_privilege_history_aux_lob_meta" const uint64_t OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_AUX_LOB_META_TID = 50507; // "__all_tenant_snapshot_ls_replica_history_aux_lob_meta" +const uint64_t OB_ALL_LS_REPLICA_TASK_HISTORY_AUX_LOB_META_TID = 50508; // "__all_ls_replica_task_history_aux_lob_meta" const uint64_t OB_ALL_USER_PROXY_INFO_AUX_LOB_META_TID = 50512; // "__all_user_proxy_info_aux_lob_meta" const uint64_t OB_ALL_USER_PROXY_INFO_HISTORY_AUX_LOB_META_TID = 50513; // "__all_user_proxy_info_history_aux_lob_meta" const uint64_t OB_ALL_USER_PROXY_ROLE_INFO_AUX_LOB_META_TID = 50514; // "__all_user_proxy_role_info_aux_lob_meta" @@ -2560,6 +2567,7 @@ const uint64_t OB_ALL_TRUSTED_ROOT_CERTIFICATE_AUX_LOB_PIECE_TID = 60502; // "__ const uint64_t OB_ALL_COLUMN_PRIVILEGE_AUX_LOB_PIECE_TID = 60505; // "__all_column_privilege_aux_lob_piece" const uint64_t OB_ALL_COLUMN_PRIVILEGE_HISTORY_AUX_LOB_PIECE_TID = 60506; // "__all_column_privilege_history_aux_lob_piece" const uint64_t OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_AUX_LOB_PIECE_TID = 60507; // "__all_tenant_snapshot_ls_replica_history_aux_lob_piece" +const uint64_t OB_ALL_LS_REPLICA_TASK_HISTORY_AUX_LOB_PIECE_TID = 60508; // "__all_ls_replica_task_history_aux_lob_piece" const uint64_t OB_ALL_USER_PROXY_INFO_AUX_LOB_PIECE_TID = 60512; // "__all_user_proxy_info_aux_lob_piece" const uint64_t OB_ALL_USER_PROXY_INFO_HISTORY_AUX_LOB_PIECE_TID = 60513; // "__all_user_proxy_info_history_aux_lob_piece" const uint64_t OB_ALL_USER_PROXY_ROLE_INFO_AUX_LOB_PIECE_TID = 60514; // "__all_user_proxy_role_info_aux_lob_piece" @@ -3065,6 +3073,7 @@ const char *const OB_ALL_TRUSTED_ROOT_CERTIFICATE_TNAME = "__all_trusted_root_ce const char *const OB_ALL_COLUMN_PRIVILEGE_TNAME = "__all_column_privilege"; const char *const OB_ALL_COLUMN_PRIVILEGE_HISTORY_TNAME = "__all_column_privilege_history"; const char *const OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_TNAME = "__all_tenant_snapshot_ls_replica_history"; +const char *const OB_ALL_LS_REPLICA_TASK_HISTORY_TNAME = "__all_ls_replica_task_history"; const char *const OB_ALL_USER_PROXY_INFO_TNAME = "__all_user_proxy_info"; const char *const OB_ALL_USER_PROXY_INFO_HISTORY_TNAME = "__all_user_proxy_info_history"; const char *const OB_ALL_USER_PROXY_ROLE_INFO_TNAME = "__all_user_proxy_role_info"; @@ -3514,6 +3523,7 @@ const char *const OB_ALL_VIRTUAL_COLUMN_PRIVILEGE_TNAME = "__all_virtual_column_ const char *const OB_ALL_VIRTUAL_COLUMN_PRIVILEGE_HISTORY_TNAME = "__all_virtual_column_privilege_history"; const char *const OB_ALL_VIRTUAL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_TNAME = "__all_virtual_tenant_snapshot_ls_replica_history"; const char *const OB_ENABLED_ROLES_TNAME = "ENABLED_ROLES"; +const char *const OB_ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY_TNAME = "__all_virtual_ls_replica_task_history"; const char *const OB_ALL_VIRTUAL_SESSION_PS_INFO_TNAME = "__all_virtual_session_ps_info"; const char *const OB_ALL_VIRTUAL_TRACEPOINT_INFO_TNAME = "__all_virtual_tracepoint_info"; const char *const OB_ALL_VIRTUAL_COMPATIBILITY_CONTROL_TNAME = "__all_virtual_compatibility_control"; @@ -3791,6 +3801,7 @@ const char *const OB_ALL_VIRTUAL_TRANSFER_PARTITION_TASK_REAL_AGENT_ORA_TNAME = const char *const OB_ALL_VIRTUAL_TRANSFER_PARTITION_TASK_HISTORY_REAL_AGENT_ORA_TNAME = "ALL_VIRTUAL_TRANSFER_PARTITION_TASK_HISTORY_REAL_AGENT"; const char *const OB_ALL_VIRTUAL_LS_SNAPSHOT_ORA_TNAME = "ALL_VIRTUAL_LS_SNAPSHOT"; const char *const OB_ALL_VIRTUAL_INDEX_USAGE_INFO_REAL_AGENT_ORA_TNAME = "ALL_VIRTUAL_INDEX_USAGE_INFO_REAL_AGENT"; +const char *const OB_ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY_ORA_TNAME = "ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY"; const char *const OB_ALL_VIRTUAL_SESSION_PS_INFO_ORA_TNAME = "ALL_VIRTUAL_SESSION_PS_INFO"; const char *const OB_ALL_VIRTUAL_TRACEPOINT_INFO_ORA_TNAME = "ALL_VIRTUAL_TRACEPOINT_INFO"; const char *const OB_ALL_VIRTUAL_USER_PROXY_INFO_REAL_AGENT_ORA_TNAME = "ALL_VIRTUAL_USER_PROXY_INFO_REAL_AGENT"; @@ -4196,6 +4207,8 @@ const char *const OB_COLUMNS_PRIV_TNAME = "columns_priv"; const char *const OB_GV_OB_LS_SNAPSHOTS_TNAME = "GV$OB_LS_SNAPSHOTS"; const char *const OB_V_OB_LS_SNAPSHOTS_TNAME = "V$OB_LS_SNAPSHOTS"; const char *const OB_DBA_OB_CLONE_HISTORY_TNAME = "DBA_OB_CLONE_HISTORY"; +const char *const OB_DBA_OB_LS_REPLICA_TASK_HISTORY_TNAME = "DBA_OB_LS_REPLICA_TASK_HISTORY"; +const char *const OB_CDB_OB_LS_REPLICA_TASK_HISTORY_TNAME = "CDB_OB_LS_REPLICA_TASK_HISTORY"; const char *const OB_CDB_MVIEW_LOGS_TNAME = "CDB_MVIEW_LOGS"; const char *const OB_DBA_MVIEW_LOGS_TNAME = "DBA_MVIEW_LOGS"; const char *const OB_CDB_MVIEWS_TNAME = "CDB_MVIEWS"; @@ -4504,6 +4517,7 @@ const char *const OB_DBA_OB_IMPORT_TABLE_TASK_HISTORY_ORA_TNAME = "DBA_OB_IMPORT const char *const OB_DBA_OB_TRANSFER_PARTITION_TASKS_ORA_TNAME = "DBA_OB_TRANSFER_PARTITION_TASKS"; const char *const OB_DBA_OB_TRANSFER_PARTITION_TASK_HISTORY_ORA_TNAME = "DBA_OB_TRANSFER_PARTITION_TASK_HISTORY"; const char *const OB_USER_USERS_TNAME = "USER_USERS"; +const char *const OB_DBA_OB_LS_REPLICA_TASK_HISTORY_ORA_TNAME = "DBA_OB_LS_REPLICA_TASK_HISTORY"; const char *const OB_DBA_MVIEW_LOGS_ORA_TNAME = "DBA_MVIEW_LOGS"; const char *const OB_ALL_MVIEW_LOGS_ORA_TNAME = "ALL_MVIEW_LOGS"; const char *const OB_USER_MVIEW_LOGS_ORA_TNAME = "USER_MVIEW_LOGS"; @@ -5005,6 +5019,7 @@ const char *const OB_ALL_TRUSTED_ROOT_CERTIFICATE_AUX_LOB_META_TNAME = "__all_tr const char *const OB_ALL_COLUMN_PRIVILEGE_AUX_LOB_META_TNAME = "__all_column_privilege_aux_lob_meta"; const char *const OB_ALL_COLUMN_PRIVILEGE_HISTORY_AUX_LOB_META_TNAME = "__all_column_privilege_history_aux_lob_meta"; const char *const OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_AUX_LOB_META_TNAME = "__all_tenant_snapshot_ls_replica_history_aux_lob_meta"; +const char *const OB_ALL_LS_REPLICA_TASK_HISTORY_AUX_LOB_META_TNAME = "__all_ls_replica_task_history_aux_lob_meta"; const char *const OB_ALL_USER_PROXY_INFO_AUX_LOB_META_TNAME = "__all_user_proxy_info_aux_lob_meta"; const char *const OB_ALL_USER_PROXY_INFO_HISTORY_AUX_LOB_META_TNAME = "__all_user_proxy_info_history_aux_lob_meta"; const char *const OB_ALL_USER_PROXY_ROLE_INFO_AUX_LOB_META_TNAME = "__all_user_proxy_role_info_aux_lob_meta"; @@ -5304,6 +5319,7 @@ const char *const OB_ALL_TRUSTED_ROOT_CERTIFICATE_AUX_LOB_PIECE_TNAME = "__all_t const char *const OB_ALL_COLUMN_PRIVILEGE_AUX_LOB_PIECE_TNAME = "__all_column_privilege_aux_lob_piece"; const char *const OB_ALL_COLUMN_PRIVILEGE_HISTORY_AUX_LOB_PIECE_TNAME = "__all_column_privilege_history_aux_lob_piece"; const char *const OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_AUX_LOB_PIECE_TNAME = "__all_tenant_snapshot_ls_replica_history_aux_lob_piece"; +const char *const OB_ALL_LS_REPLICA_TASK_HISTORY_AUX_LOB_PIECE_TNAME = "__all_ls_replica_task_history_aux_lob_piece"; const char *const OB_ALL_USER_PROXY_INFO_AUX_LOB_PIECE_TNAME = "__all_user_proxy_info_aux_lob_piece"; const char *const OB_ALL_USER_PROXY_INFO_HISTORY_AUX_LOB_PIECE_TNAME = "__all_user_proxy_info_history_aux_lob_piece"; const char *const OB_ALL_USER_PROXY_ROLE_INFO_AUX_LOB_PIECE_TNAME = "__all_user_proxy_role_info_aux_lob_piece"; diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index 478c7bab8..786ba625f 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -7189,6 +7189,45 @@ all_tenant_snapshot_ls_replica_history_def = dict( ) def_table_schema(**all_tenant_snapshot_ls_replica_history_def) +def_table_schema( + owner = 'jinqian.zzy', + table_name = '__all_ls_replica_task_history', + table_id = '508', + table_type = 'SYSTEM_TABLE', + gm_columns = ['gmt_create', 'gmt_modified'], + rowkey_columns = [ + ('tenant_id', 'int'), + ('ls_id', 'int'), + ('task_type', 'varchar:MAX_DISASTER_RECOVERY_TASK_TYPE_LENGTH'), + ('task_id', 'varchar:OB_TRACE_STAT_BUFFER_SIZE'), + ], + in_tenant_space = True, + is_cluster_private = True, + meta_record_in_sys = False, + normal_columns = [ + ('task_status', 'varchar:MAX_COLUMN_COMMENT_LENGTH', 'true'), + ('priority', 'int', 'false', 1), + ('target_replica_svr_ip', 'varchar:MAX_IP_ADDR_LENGTH', 'true'), + ('target_replica_svr_port', 'int', 'true'), + ('target_paxos_replica_number', 'int', 'true'), + ('target_replica_type', 'varchar:MAX_REPLICA_TYPE_LENGTH', 'true'), + ('source_replica_svr_ip', 'varchar:MAX_IP_ADDR_LENGTH', 'true'), + ('source_replica_svr_port', 'int', 'true'), + ('source_paxos_replica_number', 'int', 'true'), + ('source_replica_type', 'varchar:MAX_REPLICA_TYPE_LENGTH', 'true'), + ('data_source_svr_ip', 'varchar:MAX_IP_ADDR_LENGTH', 'true'), + ('data_source_svr_port', 'int', 'true'), + ('is_manual', 'bool', 'true', '0'), + ('task_exec_svr_ip', 'varchar:MAX_IP_ADDR_LENGTH', 'true'), + ('task_exec_svr_port', 'int', 'true'), + ('generate_time', 'timestamp:6', 'false', 0), + ('schedule_time', 'timestamp:6', 'false', 0), + ('finish_time', 'timestamp:6', 'false', 0), + ('execute_result', 'varchar:MAX_COLUMN_COMMENT_LENGTH', 'true'), + ('comment', 'varchar:MAX_COLUMN_COMMENT_LENGTH', 'true'), + ], +) + all_user_proxy_info_def = dict( owner = 'mingye.swj', table_name = '__all_user_proxy_info', @@ -14419,7 +14458,12 @@ def_table_schema( ], ) -# 12467: __all_virtual_ls_replica_task_history +def_table_schema(**gen_iterate_private_virtual_table_def( + table_id = '12467', + table_name = '__all_virtual_ls_replica_task_history', + in_tenant_space = True, + keywords = all_def_keywords['__all_ls_replica_task_history'])) + def_table_schema( owner = 'gongyusen.gys', table_name = '__all_virtual_session_ps_info', @@ -15074,13 +15118,12 @@ def_table_schema(**no_direct_access(gen_oracle_mapping_real_virtual_table_def('1 # 15438: abandoned def_table_schema(**no_direct_access(gen_oracle_mapping_virtual_table_def('15439', all_def_keywords['__all_virtual_ls_snapshot']))) def_table_schema(**no_direct_access(gen_oracle_mapping_real_virtual_table_def('15440', all_def_keywords['__all_index_usage_info']))) -def_table_schema(**no_direct_access(gen_oracle_mapping_virtual_table_def('15444', all_def_keywords['__all_virtual_session_ps_info']))) # 余留位置 # 15441: __all_virtual_shared_storage_quota # 15442: __all_virtual_column_group -# 15443: __all_virtual_ls_replica_task_history -# 15444: __all_virtual_session_ps_info +def_table_schema(**no_direct_access(gen_oracle_mapping_virtual_table_def('15443', all_def_keywords['__all_virtual_ls_replica_task_history']))) +def_table_schema(**no_direct_access(gen_oracle_mapping_virtual_table_def('15444', all_def_keywords['__all_virtual_session_ps_info']))) def_table_schema(**no_direct_access(gen_oracle_mapping_virtual_table_def('15445', all_def_keywords['__all_virtual_tracepoint_info']))) def_table_schema(**no_direct_access(gen_oracle_mapping_real_virtual_table_def('15446', all_def_keywords['__all_user_proxy_info']))) def_table_schema(**no_direct_access(gen_oracle_mapping_real_virtual_table_def('15447', all_def_keywords['__all_user_proxy_role_info']))) @@ -28545,6 +28588,14 @@ def_table_schema( (CASE SOURCE_REPLICA_TYPE WHEN "" THEN NULL ELSE SOURCE_REPLICA_TYPE END) AS SOURCE_REPLICA_TYPE, + (CASE DATA_SOURCE_SVR_IP + WHEN "" THEN NULL + ELSE DATA_SOURCE_SVR_IP END) AS DATA_SOURCE_SVR_IP, + DATA_SOURCE_SVR_PORT, + CAST(CASE IS_MANUAL + WHEN 0 THEN 'FALSE' + WHEN 1 THEN 'TRUE' + ELSE NULL END AS CHAR(6)) AS IS_MANUAL, TASK_EXEC_SVR_IP, TASK_EXEC_SVR_PORT, CAST(GMT_CREATE AS DATETIME) AS CREATE_TIME, @@ -28590,6 +28641,14 @@ def_table_schema( (CASE SOURCE_REPLICA_TYPE WHEN "" THEN NULL ELSE SOURCE_REPLICA_TYPE END) AS SOURCE_REPLICA_TYPE, + (CASE DATA_SOURCE_SVR_IP + WHEN "" THEN NULL + ELSE DATA_SOURCE_SVR_IP END) AS DATA_SOURCE_SVR_IP, + DATA_SOURCE_SVR_PORT, + CAST(CASE IS_MANUAL + WHEN 0 THEN 'FALSE' + WHEN 1 THEN 'TRUE' + ELSE NULL END AS CHAR(6)) AS IS_MANUAL, TASK_EXEC_SVR_IP, TASK_EXEC_SVR_PORT, CAST(GMT_CREATE AS DATETIME) AS CREATE_TIME, @@ -33777,8 +33836,117 @@ FROM oceanbase.__all_clone_job_history ORDER BY CLONE_START_TIME #21520: GV$OB_SHARED_STORAGE_QUOTA #21521: V$OB_SHARED_STORAGE_QUOTA #21522: CDB_UNUSED_COL_TABS -#21523: DBA_OB_LS_REPLICA_TASK_HISTORY -#21524: CDB_OB_LS_REPLICA_TASK_HISTORY + +def_table_schema( + owner = 'jinqian.zzy', + table_name = 'DBA_OB_LS_REPLICA_TASK_HISTORY', + table_id = '21523', + table_type = 'SYSTEM_VIEW', + gm_columns = [], + rowkey_columns = [], + normal_columns = [], + in_tenant_space = True, + view_definition = + """ + ( + SELECT LS_ID, + TASK_TYPE, + TASK_ID, + TASK_STATUS, + CAST(CASE PRIORITY + WHEN 0 THEN 'HIGH' + WHEN 1 THEN 'LOW' + ELSE NULL END AS CHAR(5)) AS PRIORITY, + TARGET_REPLICA_SVR_IP, + TARGET_REPLICA_SVR_PORT, + TARGET_PAXOS_REPLICA_NUMBER, + TARGET_REPLICA_TYPE, + (CASE SOURCE_REPLICA_SVR_IP + WHEN "" THEN NULL + ELSE SOURCE_REPLICA_SVR_IP END) AS SOURCE_REPLICA_SVR_IP, + SOURCE_REPLICA_SVR_PORT, + SOURCE_PAXOS_REPLICA_NUMBER, + (CASE SOURCE_REPLICA_TYPE + WHEN "" THEN NULL + ELSE SOURCE_REPLICA_TYPE END) AS SOURCE_REPLICA_TYPE, + (CASE DATA_SOURCE_SVR_IP + WHEN "" THEN NULL + ELSE DATA_SOURCE_SVR_IP END) AS DATA_SOURCE_SVR_IP, + DATA_SOURCE_SVR_PORT, + CAST(CASE IS_MANUAL + WHEN 0 THEN 'FALSE' + WHEN 1 THEN 'TRUE' + ELSE NULL END AS CHAR(6)) AS IS_MANUAL, + TASK_EXEC_SVR_IP, + TASK_EXEC_SVR_PORT, + CAST(GMT_CREATE AS DATETIME) AS CREATE_TIME, + CAST(SCHEDULE_TIME AS DATETIME) AS START_TIME, + CAST(GMT_MODIFIED AS DATETIME) AS MODIFY_TIME, + CAST(FINISH_TIME AS DATETIME) AS FINISH_TIME, + (CASE EXECUTE_RESULT + WHEN "" THEN NULL + ELSE EXECUTE_RESULT END) AS EXECUTE_RESULT, + COMMENT + FROM OCEANBASE.__ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY + WHERE TENANT_ID = EFFECTIVE_TENANT_ID() + ) + """.replace("\n", " "), +) + +def_table_schema( + owner = 'jinqian.zzy', + table_name = 'CDB_OB_LS_REPLICA_TASK_HISTORY', + table_id = '21524', + table_type = 'SYSTEM_VIEW', + gm_columns = [], + rowkey_columns = [], + normal_columns = [], + view_definition = + """ + ( + SELECT TENANT_ID, + LS_ID, + TASK_TYPE, + TASK_ID, + TASK_STATUS, + CAST(CASE PRIORITY + WHEN 0 THEN 'HIGH' + WHEN 1 THEN 'LOW' + ELSE NULL END AS CHAR(5)) AS PRIORITY, + TARGET_REPLICA_SVR_IP, + TARGET_REPLICA_SVR_PORT, + TARGET_PAXOS_REPLICA_NUMBER, + TARGET_REPLICA_TYPE, + (CASE SOURCE_REPLICA_SVR_IP + WHEN "" THEN NULL + ELSE SOURCE_REPLICA_SVR_IP END) AS SOURCE_REPLICA_SVR_IP, + SOURCE_REPLICA_SVR_PORT, + SOURCE_PAXOS_REPLICA_NUMBER, + (CASE SOURCE_REPLICA_TYPE + WHEN "" THEN NULL + ELSE SOURCE_REPLICA_TYPE END) AS SOURCE_REPLICA_TYPE, + (CASE DATA_SOURCE_SVR_IP + WHEN "" THEN NULL + ELSE DATA_SOURCE_SVR_IP END) AS DATA_SOURCE_SVR_IP, + DATA_SOURCE_SVR_PORT, + CAST(CASE IS_MANUAL + WHEN 0 THEN 'FALSE' + WHEN 1 THEN 'TRUE' + ELSE NULL END AS CHAR(6)) AS IS_MANUAL, + TASK_EXEC_SVR_IP, + TASK_EXEC_SVR_PORT, + CAST(GMT_CREATE AS DATETIME) AS CREATE_TIME, + CAST(SCHEDULE_TIME AS DATETIME) AS START_TIME, + CAST(GMT_MODIFIED AS DATETIME) AS MODIFY_TIME, + CAST(FINISH_TIME AS DATETIME) AS FINISH_TIME, + (CASE EXECUTE_RESULT + WHEN "" THEN NULL + ELSE EXECUTE_RESULT END) AS EXECUTE_RESULT, + COMMENT + FROM OCEANBASE.__ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY + ) + """.replace("\n", " "), +) def_table_schema( owner = 'suzhi.yt', @@ -52363,8 +52531,8 @@ def_table_schema( TASK_ID, TASK_STATUS, CAST(CASE PRIORITY - WHEN 1 THEN 'HIGH' - WHEN 2 THEN 'LOW' + WHEN 0 THEN 'HIGH' + WHEN 1 THEN 'LOW' ELSE NULL END AS CHAR(5)) AS PRIORITY, TARGET_REPLICA_SVR_IP, TARGET_REPLICA_SVR_PORT, @@ -52378,6 +52546,14 @@ def_table_schema( CASE SOURCE_REPLICA_TYPE WHEN '' THEN NULL ELSE SOURCE_REPLICA_TYPE END AS SOURCE_REPLICA_TYPE, + CASE DATA_SOURCE_SVR_IP + WHEN '' THEN NULL + ELSE DATA_SOURCE_SVR_IP END AS DATA_SOURCE_SVR_IP, + DATA_SOURCE_SVR_PORT, + CAST(CASE IS_MANUAL + WHEN 0 THEN 'FALSE' + WHEN 1 THEN 'TRUE' + ELSE NULL END AS CHAR(6)) AS IS_MANUAL, TASK_EXEC_SVR_IP, TASK_EXEC_SVR_PORT, CAST(GMT_CREATE AS TIMESTAMP(6)) AS CREATE_TIME, @@ -54796,7 +54972,66 @@ def_table_schema( AND B.TENANT_ID = EFFECTIVE_TENANT_ID() """.replace("\n", " ") ) -# 25279: DBA_OB_LS_REPLICA_TASK_HISTORY + +def_table_schema( + owner = 'jinqian.zzy', + table_name = 'DBA_OB_LS_REPLICA_TASK_HISTORY', + name_postfix = '_ORA', + database_id = 'OB_ORA_SYS_DATABASE_ID', + table_id = '25279', + table_type = 'SYSTEM_VIEW', + gm_columns = [], + rowkey_columns = [], + normal_columns = [], + in_tenant_space = True, + view_definition = + """ + ( + SELECT LS_ID, + TASK_TYPE, + TASK_ID, + TASK_STATUS, + CAST(CASE PRIORITY + WHEN 0 THEN 'HIGH' + WHEN 1 THEN 'LOW' + ELSE NULL END AS CHAR(5)) AS PRIORITY, + TARGET_REPLICA_SVR_IP, + TARGET_REPLICA_SVR_PORT, + TARGET_PAXOS_REPLICA_NUMBER, + TARGET_REPLICA_TYPE, + CASE SOURCE_REPLICA_SVR_IP + WHEN '' THEN NULL + ELSE SOURCE_REPLICA_SVR_IP END AS SOURCE_REPLICA_SVR_IP, + SOURCE_REPLICA_SVR_PORT, + SOURCE_PAXOS_REPLICA_NUMBER, + CASE SOURCE_REPLICA_TYPE + WHEN '' THEN NULL + ELSE SOURCE_REPLICA_TYPE END AS SOURCE_REPLICA_TYPE, + CASE DATA_SOURCE_SVR_IP + WHEN '' THEN NULL + ELSE DATA_SOURCE_SVR_IP END AS DATA_SOURCE_SVR_IP, + DATA_SOURCE_SVR_PORT, + CAST(CASE IS_MANUAL + WHEN 0 THEN 'FALSE' + WHEN 1 THEN 'TRUE' + ELSE NULL END AS CHAR(6)) AS IS_MANUAL, + TASK_EXEC_SVR_IP, + TASK_EXEC_SVR_PORT, + CAST(GMT_CREATE AS TIMESTAMP(6)) AS CREATE_TIME, + CAST(SCHEDULE_TIME AS TIMESTAMP(6)) AS START_TIME, + CAST(GMT_MODIFIED AS TIMESTAMP(6)) AS MODIFY_TIME, + CAST(FINISH_TIME AS TIMESTAMP(6)) AS FINISH_TIME, + CASE EXECUTE_RESULT + WHEN '' THEN NULL + ELSE EXECUTE_RESULT END AS EXECUTE_RESULT, + "COMMENT" + FROM SYS.ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY + WHERE + TENANT_ID = EFFECTIVE_TENANT_ID() + ) + """.replace("\n", " "), +) + # 25280: ALL_UNUSED_COL_TABS # 25281: DBA_UNUSED_COL_TABS # 25282: USER_UNUSED_COL_TABS diff --git a/src/share/inner_table/ob_inner_table_schema_misc.ipp b/src/share/inner_table/ob_inner_table_schema_misc.ipp index ccf0d3982..8145ce45e 100644 --- a/src/share/inner_table/ob_inner_table_schema_misc.ipp +++ b/src/share/inner_table/ob_inner_table_schema_misc.ipp @@ -475,6 +475,7 @@ case OB_ALL_VIRTUAL_LS_LOG_ARCHIVE_PROGRESS_TID: case OB_ALL_VIRTUAL_LS_META_TABLE_TID: case OB_ALL_VIRTUAL_LS_RECOVERY_STAT_TID: case OB_ALL_VIRTUAL_LS_REPLICA_TASK_TID: +case OB_ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY_TID: case OB_ALL_VIRTUAL_LS_RESTORE_HISTORY_TID: case OB_ALL_VIRTUAL_LS_RESTORE_PROGRESS_TID: case OB_ALL_VIRTUAL_LS_STATUS_TID: @@ -1283,6 +1284,22 @@ case OB_ALL_VIRTUAL_ZONE_MERGE_INFO_TID: break; } + case OB_ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY_TID: { + ObIteratePrivateVirtualTable *iter = NULL; + const bool meta_record_in_sys = false; + if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIteratePrivateVirtualTable, iter))) { + SERVER_LOG(WARN, "create iterate private virtual table iterator failed", KR(ret)); + } else if (OB_FAIL(iter->init(OB_ALL_LS_REPLICA_TASK_HISTORY_TID, meta_record_in_sys, index_schema, params))) { + SERVER_LOG(WARN, "iterate private virtual table iter init failed", KR(ret)); + iter->~ObIteratePrivateVirtualTable(); + allocator.free(iter); + iter = NULL; + } else { + vt_iter = iter; + } + break; + } + case OB_ALL_VIRTUAL_LS_RESTORE_HISTORY_TID: { ObIteratePrivateVirtualTable *iter = NULL; const bool meta_record_in_sys = false; @@ -1458,7 +1475,9 @@ case OB_ALL_VIRTUAL_ZONE_MERGE_INFO_TID: } break; } + END_CREATE_VT_ITER_SWITCH_LAMBDA + BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA case OB_ALL_VIRTUAL_TABLET_META_TABLE_TID: { ObIteratePrivateVirtualTable *iter = NULL; const bool meta_record_in_sys = false; @@ -1474,9 +1493,7 @@ case OB_ALL_VIRTUAL_ZONE_MERGE_INFO_TID: } break; } - END_CREATE_VT_ITER_SWITCH_LAMBDA - BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA case OB_ALL_VIRTUAL_TABLET_REPLICA_CHECKSUM_TID: { ObIteratePrivateVirtualTable *iter = NULL; const bool meta_record_in_sys = false; @@ -4793,6 +4810,9 @@ case OB_ALL_LS_RECOVERY_STAT_AUX_LOB_PIECE_TID: case OB_ALL_LS_REPLICA_TASK_TID: case OB_ALL_LS_REPLICA_TASK_AUX_LOB_META_TID: case OB_ALL_LS_REPLICA_TASK_AUX_LOB_PIECE_TID: +case OB_ALL_LS_REPLICA_TASK_HISTORY_TID: +case OB_ALL_LS_REPLICA_TASK_HISTORY_AUX_LOB_META_TID: +case OB_ALL_LS_REPLICA_TASK_HISTORY_AUX_LOB_PIECE_TID: case OB_ALL_LS_RESTORE_HISTORY_TID: case OB_ALL_LS_RESTORE_HISTORY_AUX_LOB_META_TID: case OB_ALL_LS_RESTORE_HISTORY_AUX_LOB_PIECE_TID: diff --git a/src/share/inner_table/table_id_to_name b/src/share/inner_table/table_id_to_name index f0ab9066f..01fc57da9 100644 --- a/src/share/inner_table/table_id_to_name +++ b/src/share/inner_table/table_id_to_name @@ -348,6 +348,7 @@ # 506: __all_column_privilege_history # 506: __all_column_privilege # BASE_TABLE_NAME # 507: __all_tenant_snapshot_ls_replica_history +# 508: __all_ls_replica_task_history # 512: __all_user_proxy_info # 513: __all_user_proxy_info_history # 513: __all_user_proxy_info # BASE_TABLE_NAME @@ -1105,6 +1106,8 @@ # 12464: __all_virtual_tenant_snapshot_ls_replica_history # 12464: __all_tenant_snapshot_ls_replica_history # BASE_TABLE_NAME # 12466: ENABLED_ROLES +# 12467: __all_virtual_ls_replica_task_history +# 12467: __all_ls_replica_task_history # BASE_TABLE_NAME # 12468: __all_virtual_session_ps_info # 12469: __all_virtual_tracepoint_info # 12473: __all_virtual_compatibility_control @@ -1712,6 +1715,9 @@ # 15439: __all_virtual_ls_snapshot # BASE_TABLE_NAME # 15440: ALL_VIRTUAL_INDEX_USAGE_INFO_REAL_AGENT # 15440: __all_index_usage_info # BASE_TABLE_NAME +# 15443: ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY +# 15443: __all_ls_replica_task_history # BASE_TABLE_NAME +# 15443: __all_virtual_ls_replica_task_history # BASE_TABLE_NAME1 # 15444: ALL_VIRTUAL_SESSION_PS_INFO # 15444: __all_virtual_session_ps_info # BASE_TABLE_NAME # 15445: ALL_VIRTUAL_TRACEPOINT_INFO @@ -2126,6 +2132,8 @@ # 21517: GV$OB_LS_SNAPSHOTS # 21518: V$OB_LS_SNAPSHOTS # 21519: DBA_OB_CLONE_HISTORY +# 21523: DBA_OB_LS_REPLICA_TASK_HISTORY +# 21524: CDB_OB_LS_REPLICA_TASK_HISTORY # 21525: CDB_MVIEW_LOGS # 21526: DBA_MVIEW_LOGS # 21527: CDB_MVIEWS @@ -2434,6 +2442,7 @@ # 25275: DBA_OB_TRANSFER_PARTITION_TASKS # 25276: DBA_OB_TRANSFER_PARTITION_TASK_HISTORY # 25278: USER_USERS +# 25279: DBA_OB_LS_REPLICA_TASK_HISTORY # 25283: DBA_MVIEW_LOGS # 25284: ALL_MVIEW_LOGS # 25285: USER_MVIEW_LOGS diff --git a/src/share/ob_common_rpc_proxy.h b/src/share/ob_common_rpc_proxy.h index a5c598a0b..35f0ada41 100644 --- a/src/share/ob_common_rpc_proxy.h +++ b/src/share/ob_common_rpc_proxy.h @@ -230,6 +230,7 @@ public: RPC_S(PR5 admin_reload_zone, obrpc::OB_ADMIN_RELOAD_ZONE); RPC_S(PR5 admin_clear_merge_error, obrpc::OB_ADMIN_CLEAR_MERGE_ERROR, (ObAdminMergeArg)); RPC_S(PR5 admin_migrate_unit, obrpc::OB_ADMIN_MIGRATE_UNIT, (ObAdminMigrateUnitArg)); + RPC_S(PR5 admin_alter_ls_replica, obrpc::OB_ADMIN_ALTER_LS_REPLICA, (ObAdminAlterLSReplicaArg)); RPC_S(PRD admin_upgrade_virtual_schema, obrpc::OB_ADMIN_UPGRADE_VIRTUAL_SCHEMA); RPC_S(PRD run_job, obrpc::OB_RUN_JOB, (ObRunJobArg)); RPC_S(PRD run_upgrade_job, obrpc::OB_RUN_UPGRADE_JOB, (ObUpgradeJobArg)); diff --git a/src/share/ob_debug_sync_point.h b/src/share/ob_debug_sync_point.h index 898fe47d1..52b1e7eeb 100755 --- a/src/share/ob_debug_sync_point.h +++ b/src/share/ob_debug_sync_point.h @@ -552,6 +552,7 @@ class ObString; ACT(BEFORE_PARELLEL_TRUNCATE,)\ ACT(END_DDL_IN_PX_SUBCOORD,)\ ACT(BEFORE_SEND_ADD_REPLICA_DRTASK,)\ + ACT(BEFORE_ADD_MANUAL_REPLICA_TASK_IN_INNER_TABLE,)\ ACT(BETWEEN_INSERT_LOCK_INFO_AND_TRY_LOCK_CONFIG_CHANGE,)\ ACT(BEFORE_CHECK_SHRINK_RESOURCE_POOL,)\ ACT(STOP_RECOVERY_LS_THREAD0,)\ @@ -599,6 +600,10 @@ class ObString; ACT(HANG_IN_CLONE_SYS_FAILED_STATUS,)\ ACT(BEFORE_BACKUP_PREFETCH_TASK,)\ ACT(BEFORE_BACKUP_DATA_TASK,)\ + ACT(BEFORE_WAIT_TRANSFER_OUT_TABLET_READY,)\ + ACT(BEFORE_CHECK_TABLET_READY,)\ + ACT(BEFORE_CHECK_TABLET_TRANSFER_TABLE_READY,)\ + ACT(BEFORE_LOG_REPLAY_TO_MAX_MINOR_END_SCN,)\ ACT(HOLD_DDL_COMPLEMENT_DAG_WHEN_APPEND_ROW,)\ ACT(HOLD_DDL_COMPLEMENT_DAG_BEFORE_REPORT_FINISH,)\ ACT(HOLD_DDL_COMPLEMENT_DAG_AFTER_REPORT_FINISH,)\ diff --git a/src/share/ob_rpc_struct.cpp b/src/share/ob_rpc_struct.cpp index f45a61333..7ed0b2753 100644 --- a/src/share/ob_rpc_struct.cpp +++ b/src/share/ob_rpc_struct.cpp @@ -3982,6 +3982,299 @@ void ObCalcColumnChecksumResponseArg::reset() tenant_id_ = OB_INVALID_TENANT_ID; } +OB_SERIALIZE_MEMBER( + ObAlterLSReplicaTaskType, + type_); + +static const char* alter_ls_replica_task_type_strs[] = { + "ADD_LS_REPLICA", + "REMOVE_LS_REPLICA", + "MIGRATE_LS_REPLICA", + "MODIFY_LS_REPLICA_TYPE", + "MODIFY_LS_PAXOS_REPLICA_NUM", + "CANCEL_LS_REPLICA_TASK", +}; + +const char* ObAlterLSReplicaTaskType::get_type_str() const { + STATIC_ASSERT(ARRAYSIZEOF(alter_ls_replica_task_type_strs) == (int64_t)LSReplicaTaskMax, + "alter_ls_replica_task_type string array size mismatch enum AlterLSReplicaTaskType count"); + const char *str = NULL; + if (type_ != LSReplicaTaskMax) { + str = alter_ls_replica_task_type_strs[static_cast(type_)]; + } else { + LOG_WARN_RET(OB_ERR_UNEXPECTED, "invalid AlterLSReplicaTaskType", K_(type)); + } + return str; +} + +int64_t ObAlterLSReplicaTaskType::to_string(char *buf, const int64_t buf_len) const +{ + int64_t pos = 0; + J_OBJ_START(); + J_KV(K_(type), "type", get_type_str()); + J_OBJ_END(); + return pos; +} + +int ObAlterLSReplicaTaskType::parse_from_string(const ObString &type) +{ + int ret = OB_SUCCESS; + bool found = false; + STATIC_ASSERT(ARRAYSIZEOF(alter_ls_replica_task_type_strs) == (int64_t)LSReplicaTaskMax, + "alter_ls_replica_task_type string array size mismatch enum AlterLSReplicaTaskType count"); + for (int64_t i = 0; i < ARRAYSIZEOF(alter_ls_replica_task_type_strs) && !found; i++) { + if (0 == type.case_compare(alter_ls_replica_task_type_strs[i])) { + type_ = static_cast(i); + found = true; + break; + } + } + if (!found) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("fail to parse type from string", KR(ret), K(type), K_(type)); + } + return ret; +} + +OB_SERIALIZE_MEMBER(ObAdminAlterLSReplicaArg, + ls_id_, + server_addr_, + destination_addr_, + replica_type_, + tenant_id_, + task_id_, + data_source_, + paxos_replica_num_, + alter_task_type_); + +int ObAdminAlterLSReplicaArg::assign(const ObAdminAlterLSReplicaArg &that) +{ + int ret = OB_SUCCESS; + if (this == &that) { + //pass + } else if (OB_FAIL(task_id_.assign(that.task_id_))) { + LOG_WARN("task_id_ assign failed", KR(ret), K(that.task_id_)); + } else { + ls_id_ = that.ls_id_; + server_addr_ = that.server_addr_; + destination_addr_ = that.destination_addr_; + replica_type_ = that.replica_type_; + data_source_ = that.data_source_; + paxos_replica_num_ = that.paxos_replica_num_; + tenant_id_ = that.tenant_id_; + alter_task_type_ = that.alter_task_type_; + } + return ret; +} + +int ObAdminAlterLSReplicaArg::init_add( + const share::ObLSID& ls_id, + const common::ObAddr& server_addr, + const common::ObReplicaType& replica_type, + const common::ObAddr& data_source, + const int64_t paxos_replica_num, + const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!ls_id.is_valid()) + || OB_UNLIKELY(!server_addr.is_valid()) + || OB_UNLIKELY(replica_type != REPLICA_TYPE_FULL && replica_type != REPLICA_TYPE_READONLY) + || OB_UNLIKELY(paxos_replica_num < 0) + || OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { + //data_source and paxos_replica_num is optional parameter + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(ls_id), K(server_addr), K(replica_type), + K(paxos_replica_num), K(tenant_id)); + } else { + ls_id_ = ls_id; + server_addr_ = server_addr; + replica_type_ = replica_type; + data_source_ = data_source; + paxos_replica_num_ = paxos_replica_num; + tenant_id_ = tenant_id; + alter_task_type_ = ObAlterLSReplicaTaskType(ObAlterLSReplicaTaskType::AddLSReplicaTask); + } + return ret; +} + +int ObAdminAlterLSReplicaArg::init_remove( + const share::ObLSID& ls_id, + const common::ObAddr& server_addr, + const int64_t paxos_replica_num, + const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!ls_id.is_valid()) + || OB_UNLIKELY(!server_addr.is_valid()) + || OB_UNLIKELY(paxos_replica_num < 0) + || OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(ls_id), K(server_addr), + K(paxos_replica_num), K(tenant_id)); + } else { + ls_id_ = ls_id; + server_addr_ = server_addr; + paxos_replica_num_ = paxos_replica_num; + tenant_id_ = tenant_id; + alter_task_type_ = ObAlterLSReplicaTaskType(ObAlterLSReplicaTaskType::RemoveLSReplicaTask); + } + return ret; +} + +int ObAdminAlterLSReplicaArg::init_migrate( + const share::ObLSID& ls_id, + const common::ObAddr& server_addr, + const common::ObAddr& destination_addr, + const common::ObAddr& data_source, + const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!ls_id.is_valid()) + || OB_UNLIKELY(!server_addr.is_valid()) + || OB_UNLIKELY(!destination_addr.is_valid()) + || OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { + ret = OB_INVALID_ARGUMENT; //data_surce is optional parameter + LOG_WARN("invalid argument", KR(ret), K(ls_id), + K(server_addr), K(destination_addr), K(tenant_id)); + } else { + ls_id_ = ls_id; + server_addr_ = server_addr; + destination_addr_ = destination_addr; + data_source_ = data_source; + tenant_id_ = tenant_id; + alter_task_type_ = ObAlterLSReplicaTaskType(ObAlterLSReplicaTaskType::MigrateLSReplicaTask); + } + return ret; +} + +int ObAdminAlterLSReplicaArg::init_modify_replica( + const share::ObLSID& ls_id, + const common::ObAddr& server_addr, + const common::ObReplicaType& replica_type, + const int64_t paxos_replica_num, + const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!ls_id.is_valid()) + || OB_UNLIKELY(!server_addr.is_valid()) + || OB_UNLIKELY(replica_type != REPLICA_TYPE_FULL && replica_type != REPLICA_TYPE_READONLY) + || OB_UNLIKELY(paxos_replica_num < 0) + || OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(ls_id), K(server_addr), K(replica_type), + K(paxos_replica_num), K(tenant_id)); + } else { + ls_id_ = ls_id; + server_addr_ = server_addr; + replica_type_ = replica_type; + paxos_replica_num_ = paxos_replica_num; + tenant_id_ = tenant_id; + alter_task_type_ = ObAlterLSReplicaTaskType(ObAlterLSReplicaTaskType::ModifyLSReplicaTypeTask); + } + return ret; +} + +int ObAdminAlterLSReplicaArg::init_modify_paxos_replica_num( + const share::ObLSID& ls_id, + const int64_t paxos_replica_num, + const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!ls_id.is_valid()) + || OB_UNLIKELY(paxos_replica_num <= 0) + || OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(ls_id), + K(paxos_replica_num), K(tenant_id)); + } else { + ls_id_ = ls_id; + paxos_replica_num_ = paxos_replica_num; + tenant_id_ = tenant_id; + alter_task_type_ = ObAlterLSReplicaTaskType(ObAlterLSReplicaTaskType::ModifyLSPaxosReplicaNumTask); + } + return ret; +} + +int ObAdminAlterLSReplicaArg::init_cancel( + const common::ObFixedLengthString& task_id, + const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id)) + || OB_UNLIKELY(task_id.is_empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(task_id)); + } else if (OB_FAIL(task_id_.assign(task_id))) { + LOG_WARN("task_id_ assign failed", KR(ret), K(task_id)); + } else { + tenant_id_ = tenant_id; + alter_task_type_ = ObAlterLSReplicaTaskType(ObAlterLSReplicaTaskType::CancelLSReplicaTask); + } + return ret; +} + +void ObAdminAlterLSReplicaArg::reset() +{ + ls_id_.reset(); + server_addr_.reset(); + destination_addr_.reset(); + replica_type_ = common::REPLICA_TYPE_MAX; + tenant_id_ = OB_INVALID_TENANT_ID; + task_id_.reset(); + data_source_.reset(); + paxos_replica_num_ = 0; + alter_task_type_.reset(); +} + +OB_SERIALIZE_MEMBER(ObLSCancelReplicaTaskArg, + task_id_, + ls_id_, + tenant_id_); + +int ObLSCancelReplicaTaskArg::assign(const ObLSCancelReplicaTaskArg &that) +{ + int ret = OB_SUCCESS; + if (this != &that) { + task_id_ = that.task_id_; + ls_id_ = that.ls_id_; + tenant_id_ = that.tenant_id_; + } + return ret; +} + +int ObLSCancelReplicaTaskArg::init( + const share::ObTaskId &task_id, + const share::ObLSID &ls_id, + const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!task_id.is_valid() + || !ls_id.is_valid() + || OB_INVALID_TENANT_ID == tenant_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(task_id), K(ls_id), K(tenant_id)); + } else { + task_id_ = task_id; + ls_id_ = ls_id; + tenant_id_ = tenant_id; + } + return ret; +} + +void ObLSCancelReplicaTaskArg::reset() +{ + task_id_.reset(); + ls_id_.reset(); + tenant_id_ = OB_INVALID_TENANT_ID; +} + +bool ObLSCancelReplicaTaskArg::is_valid() const +{ + return task_id_.is_valid() + && ls_id_.is_valid() + && OB_INVALID_TENANT_ID != tenant_id_; +} + OB_SERIALIZE_MEMBER(ObLSMigrateReplicaArg, task_id_, tenant_id_, diff --git a/src/share/ob_rpc_struct.h b/src/share/ob_rpc_struct.h index 4a08a8dee..ad8be3f22 100644 --- a/src/share/ob_rpc_struct.h +++ b/src/share/ob_rpc_struct.h @@ -4290,6 +4290,197 @@ public: OB_UNIS_VERSION(3); }; +class ObAlterLSReplicaTaskType +{ + OB_UNIS_VERSION(1); +public: + enum AlterLSReplicaTaskType + { + AddLSReplicaTask = 0, + RemoveLSReplicaTask, + MigrateLSReplicaTask, + ModifyLSReplicaTypeTask, + ModifyLSPaxosReplicaNumTask, + CancelLSReplicaTask, + LSReplicaTaskMax + }; +public: + ObAlterLSReplicaTaskType() : type_(LSReplicaTaskMax) {} + ObAlterLSReplicaTaskType(AlterLSReplicaTaskType type) : type_(type) {} + + ObAlterLSReplicaTaskType &operator=(const AlterLSReplicaTaskType type) { type_ = type; return *this; } + ObAlterLSReplicaTaskType &operator=(const ObAlterLSReplicaTaskType &other) { type_ = other.type_; return *this; } + bool operator==(const ObAlterLSReplicaTaskType &other) const { return other.type_ == type_; } + bool operator!=(const ObAlterLSReplicaTaskType &other) const { return other.type_ != type_; } + + void reset() { type_ = LSReplicaTaskMax; } + int64_t to_string(char *buf, const int64_t buf_len) const; + void assign(const ObAlterLSReplicaTaskType &other) { type_ = other.type_; } + bool is_valid() const { return LSReplicaTaskMax != type_; } + bool is_add_task() const { return AddLSReplicaTask == type_; } + bool is_remove_task() const { return RemoveLSReplicaTask == type_; } + bool is_migrate_task() const { return MigrateLSReplicaTask == type_; } + bool is_modify_replica_task() const { return ModifyLSReplicaTypeTask == type_; } + bool is_modify_paxos_replica_num_task() const { return ModifyLSPaxosReplicaNumTask == type_; } + bool is_cancel_task() const { return CancelLSReplicaTask == type_; } + int parse_from_string(const ObString &type); + const AlterLSReplicaTaskType &get_type() const { return type_; } + const char* get_type_str() const; +private: + AlterLSReplicaTaskType type_; +}; + +struct ObAdminAlterLSReplicaArg +{ +public: + OB_UNIS_VERSION(1); +public: + ObAdminAlterLSReplicaArg() + : ls_id_(), + server_addr_(), + destination_addr_(), + replica_type_(common::REPLICA_TYPE_MAX), + tenant_id_(OB_INVALID_TENANT_ID), + task_id_(), + data_source_(), + paxos_replica_num_(0), + alter_task_type_(ObAlterLSReplicaTaskType::LSReplicaTaskMax) {} +public: + int assign(const ObAdminAlterLSReplicaArg &that); + int init_add( + const share::ObLSID& ls_id, + const common::ObAddr& server_addr, + const common::ObReplicaType& replica_type, + const common::ObAddr& data_source, + const int64_t paxos_replica_num, + const uint64_t tenant_id); + int init_remove( + const share::ObLSID& ls_id, + const common::ObAddr& server_addr, + const int64_t paxos_replica_num, + const uint64_t tenant_id); + int init_migrate( + const share::ObLSID& ls_id, + const common::ObAddr& server_addr, + const common::ObAddr& destination_addr, + const common::ObAddr& data_source, + const uint64_t tenant_id); + int init_modify_replica( + const share::ObLSID& ls_id, + const common::ObAddr& server_addr, + const common::ObReplicaType& replica_type, + const int64_t paxos_replica_num, + const uint64_t tenant_id); + int init_modify_paxos_replica_num( + const share::ObLSID& ls_id, + const int64_t paxos_replica_num, + const uint64_t tenant_id); + int init_cancel( + const common::ObFixedLengthString& task_id, + const uint64_t tenant_id); + void reset(); + TO_STRING_KV(K_(ls_id), + K_(server_addr), + K_(destination_addr), + K_(replica_type), + K_(tenant_id), + K_(task_id), + K_(data_source), + K_(paxos_replica_num), + K_(alter_task_type)); + bool is_valid() const { + return alter_task_type_.is_valid() && + ((alter_task_type_.is_add_task() && is_add_valid_()) + || (alter_task_type_.is_remove_task() && is_remove_valid_()) + || (alter_task_type_.is_migrate_task() && is_migrate_valid_()) + || (alter_task_type_.is_modify_replica_task() && is_modify_replica_valid_()) + || (alter_task_type_.is_modify_paxos_replica_num_task() && is_modify_paxos_replica_num_valid_()) + || (alter_task_type_.is_cancel_task() && is_cancel_valid_())); + } + const share::ObLSID &get_ls_id() const { return ls_id_; } + const common::ObAddr &get_server_addr() const { return server_addr_; } + const common::ObAddr &get_destination_addr() const { return destination_addr_; } + const common::ObReplicaType& get_replica_type() const { return replica_type_; } + const uint64_t& get_tenant_id() const { return tenant_id_; } + const common::ObFixedLengthString &get_task_id() const { return task_id_; } + const common::ObAddr &get_data_source() const { return data_source_; } + const int64_t& get_paxos_replica_num() const { return paxos_replica_num_; } + const ObAlterLSReplicaTaskType& get_alter_task_type() const { return alter_task_type_; } + +private: + bool is_add_valid_() const { + return ls_id_.is_valid() + && server_addr_.is_valid() + && REPLICA_TYPE_MAX != replica_type_ + && is_valid_tenant_id(tenant_id_) + && paxos_replica_num_ >= 0; + } + bool is_remove_valid_() const { + return ls_id_.is_valid() + && server_addr_.is_valid() + && is_valid_tenant_id(tenant_id_) + && paxos_replica_num_ >= 0; + } + bool is_migrate_valid_() const { + return ls_id_.is_valid() + && server_addr_.is_valid() + && destination_addr_.is_valid() + && is_valid_tenant_id(tenant_id_); + } + bool is_modify_replica_valid_() const { + return ls_id_.is_valid() + && server_addr_.is_valid() + && REPLICA_TYPE_MAX != replica_type_ + && is_valid_tenant_id(tenant_id_) + && paxos_replica_num_ >= 0; + } + bool is_modify_paxos_replica_num_valid_() const { + return ls_id_.is_valid() + && paxos_replica_num_ > 0 + && is_valid_tenant_id(tenant_id_); + } + bool is_cancel_valid_() const { + return !task_id_.is_empty() && is_valid_tenant_id(tenant_id_); + } + + share::ObLSID ls_id_; + common::ObAddr server_addr_; + common::ObAddr destination_addr_; + common::ObReplicaType replica_type_; + uint64_t tenant_id_; + common::ObFixedLengthString task_id_; + common::ObAddr data_source_; + int64_t paxos_replica_num_; + ObAlterLSReplicaTaskType alter_task_type_; +}; + +struct ObLSCancelReplicaTaskArg +{ + OB_UNIS_VERSION(1); +public: + ObLSCancelReplicaTaskArg() + : task_id_(), + ls_id_(), + tenant_id_(OB_INVALID_TENANT_ID) {} +public: + int assign(const ObLSCancelReplicaTaskArg &that); + int init(const share::ObTaskId &task_id, + const share::ObLSID &ls_id, + const uint64_t tenant_id); + void reset(); + TO_STRING_KV(K_(task_id), + K_(ls_id), + K_(tenant_id)); + bool is_valid() const; + const share::ObTaskId &get_task_id() const { return task_id_; } + const share::ObLSID &get_ls_id() const { return ls_id_; } + const uint64_t &get_tenant_id() const { return tenant_id_; } +private: + share::ObTaskId task_id_; + share::ObLSID ls_id_; + uint64_t tenant_id_; +}; + struct ObLSMigrateReplicaArg { public: diff --git a/src/share/ob_srv_rpc_proxy.h b/src/share/ob_srv_rpc_proxy.h index 125cd5a85..6880a2c86 100644 --- a/src/share/ob_srv_rpc_proxy.h +++ b/src/share/ob_srv_rpc_proxy.h @@ -77,6 +77,7 @@ public: RPC_S(PR5 notify_archive, OB_NOTIFY_ARCHIVE, (ObNotifyArchiveArg)); // ls disaster recovery rpc + RPC_S(PR5 ls_cancel_replica_task, OB_LS_CANCEL_REPLICA_TASK, (ObLSCancelReplicaTaskArg)); RPC_S(PR5 ls_migrate_replica, OB_LS_MIGRATE_REPLICA, (ObLSMigrateReplicaArg)); RPC_S(PR5 ls_add_replica, OB_LS_ADD_REPLICA, (ObLSAddReplicaArg)); RPC_S(PR5 ls_type_transform, OB_LS_TYPE_TRANSFORM, (ObLSChangeReplicaArg)); diff --git a/src/share/unit/ob_unit_info.cpp b/src/share/unit/ob_unit_info.cpp index 845f88844..3722f4ded 100644 --- a/src/share/unit/ob_unit_info.cpp +++ b/src/share/unit/ob_unit_info.cpp @@ -13,6 +13,7 @@ #include #include "ob_unit_info.h" +#include "share/ob_errno.h" namespace oceanbase { diff --git a/src/sql/engine/cmd/ob_alter_system_executor.cpp b/src/sql/engine/cmd/ob_alter_system_executor.cpp index f18a985b6..63b99f498 100644 --- a/src/sql/engine/cmd/ob_alter_system_executor.cpp +++ b/src/sql/engine/cmd/ob_alter_system_executor.cpp @@ -1322,6 +1322,26 @@ int ObMigrateUnitExecutor::execute(ObExecContext &ctx, ObMigrateUnitStmt &stmt) return ret; } +int ObAlterLSReplicaExecutor::execute(ObExecContext &ctx, ObAlterLSReplicaStmt &stmt) +{ + int ret = OB_SUCCESS; + ObTaskExecutorCtx *task_exec_ctx = GET_TASK_EXECUTOR_CTX(ctx); + obrpc::ObCommonRpcProxy *common_rpc = NULL; + if (OB_UNLIKELY(!stmt.get_rpc_arg().is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("rpc args is invalid", KR(ret), K(stmt)); + } else if (OB_ISNULL(task_exec_ctx)) { + ret = OB_NOT_INIT; + LOG_WARN("get task executor context failed", KR(ret)); + } else if (OB_ISNULL(common_rpc = task_exec_ctx->get_common_rpc())) { + ret = OB_NOT_INIT; + LOG_WARN("get common rpc proxy failed", KR(ret), KP(task_exec_ctx)); + } else if (OB_FAIL(common_rpc->admin_alter_ls_replica(stmt.get_rpc_arg()))) { + LOG_WARN("add ls replica rpc failed", KR(ret), K(stmt.get_rpc_arg())); + } + return ret; +} + int ObAddArbitrationServiceExecutor::execute(ObExecContext &ctx, ObAddArbitrationServiceStmt &stmt) { int ret = OB_SUCCESS; diff --git a/src/sql/engine/cmd/ob_alter_system_executor.h b/src/sql/engine/cmd/ob_alter_system_executor.h index 5005ca479..08cfdd67f 100644 --- a/src/sql/engine/cmd/ob_alter_system_executor.h +++ b/src/sql/engine/cmd/ob_alter_system_executor.h @@ -84,6 +84,8 @@ DEF_SIMPLE_EXECUTOR(ObClearMergeError); DEF_SIMPLE_EXECUTOR(ObMigrateUnit); +DEF_SIMPLE_EXECUTOR(ObAlterLSReplica); + DEF_SIMPLE_EXECUTOR(ObAddArbitrationService); DEF_SIMPLE_EXECUTOR(ObRemoveArbitrationService); diff --git a/src/sql/executor/ob_cmd_executor.cpp b/src/sql/executor/ob_cmd_executor.cpp index b2423e238..317a6a9e5 100644 --- a/src/sql/executor/ob_cmd_executor.cpp +++ b/src/sql/executor/ob_cmd_executor.cpp @@ -654,6 +654,10 @@ int ObCmdExecutor::execute(ObExecContext &ctx, ObICmd &cmd) DEFINE_EXECUTE_CMD(ObMigrateUnitStmt, ObMigrateUnitExecutor); break; } + case stmt::T_ALTER_LS_REPLICA: { + DEFINE_EXECUTE_CMD(ObAlterLSReplicaStmt, ObAlterLSReplicaExecutor); + break; + } case stmt::T_ADD_ARBITRATION_SERVICE: { DEFINE_EXECUTE_CMD(ObAddArbitrationServiceStmt, ObAddArbitrationServiceExecutor); break; diff --git a/src/sql/parser/non_reserved_keywords_mysql_mode.c b/src/sql/parser/non_reserved_keywords_mysql_mode.c index 0becfb89a..9f846978b 100644 --- a/src/sql/parser/non_reserved_keywords_mysql_mode.c +++ b/src/sql/parser/non_reserved_keywords_mysql_mode.c @@ -192,6 +192,7 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] = {"data_table_id", DATA_TABLE_ID}, {"database", DATABASE}, {"databases", DATABASES}, + {"data_source", DATA_SOURCE}, {"date", DATE}, {"date_add", DATE_ADD}, {"date_sub", DATE_SUB}, @@ -635,6 +636,7 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] = {"path", PATH}, {"pattern", PATTERN}, {"pause", PAUSE}, + {"paxos_replica_num", PAXOS_REPLICA_NUM}, {"percentage", PERCENTAGE}, {"percent_rank", PERCENT_RANK}, {"performance", PERFORMANCE}, @@ -907,6 +909,7 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] = {"tablet_id", TABLET_ID}, {"tablet_max_size", TABLET_MAX_SIZE}, {"task", TASK}, + {"task_id", TASK_ID}, {"template", TEMPLATE}, {"temporary", TEMPORARY}, {"temptable", TEMPTABLE}, diff --git a/src/sql/parser/sql_parser_mysql_mode.y b/src/sql/parser/sql_parser_mysql_mode.y index b57aca142..ab87f163f 100644 --- a/src/sql/parser/sql_parser_mysql_mode.y +++ b/src/sql/parser/sql_parser_mysql_mode.y @@ -278,7 +278,7 @@ END_P SET_VAR DELIMITER CONSTRAINT_NAME CONSTRAINT_SCHEMA CONTAINS CONTEXT CONTRIBUTORS COPY COUNT CPU CREATE_TIMESTAMP CTXCAT CTX_ID CUBE CURDATE CURRENT STACKED CURTIME CURSOR_NAME CUME_DIST CYCLE CALC_PARTITION_ID CONNECT - DAG DATA DATAFILE DATA_TABLE_ID DATE DATE_ADD DATE_SUB DATETIME DAY DEALLOCATE DECRYPTION + DAG DATA DATAFILE DATA_TABLE_ID DATA_SOURCE DATE DATE_ADD DATE_SUB DATETIME DAY DEALLOCATE DECRYPTION DEFAULT_AUTH DEFAULT_LOB_INROW_THRESHOLD DEFINER DELAY DELAY_KEY_WRITE DEPTH DES_KEY_FILE DENSE_RANK DESCRIPTION DESTINATION DIAGNOSTICS DIRECTORY DISABLE DISALLOW DISCARD DISK DISKGROUP DO DOT DUMP DUMPFILE DUPLICATE DUPLICATE_SCOPE DYNAMIC DATABASE_ID DEFAULT_TABLEGROUP DISCONNECT DEMAND @@ -328,7 +328,7 @@ END_P SET_VAR DELIMITER OBCONFIG_URL OJ OBJECT_ID - PACK_KEYS PAGE PARALLEL PARAMETERS PARSER PARTIAL PARTITION_ID PARTITIONING PARTITIONS PASSWORD PATH PAUSE PERCENTAGE + PACK_KEYS PAGE PARALLEL PARAMETERS PARSER PARTIAL PARTITION_ID PARTITIONING PARTITIONS PASSWORD PATH PAUSE PAXOS_REPLICA_NUM PERCENTAGE PERCENT_RANK PHASE PLAN PHYSICAL PLANREGRESS PLUGIN PLUGIN_DIR PLUGINS POINT POLYGON PERFORMANCE PROTECTION PRIORITY PL POLICY POOL PORT POSITION PREPARE PRESERVE PRETTY PRETTY_COLOR PREV PRIMARY_ZONE PRIVILEGES PROCESS PROCESSLIST PROFILE PROFILES PROXY PRECEDING PCTFREE P_ENTITY P_CHUNK @@ -356,7 +356,7 @@ END_P SET_VAR DELIMITER SUPER SUSPEND SWAPS SWITCH SWITCHES SWITCHOVER SYSTEM SYSTEM_USER SYSDATE SESSION_ALIAS SIZE SKEWONLY SEQUENCE SLOG STATEMENT_ID SKIP_HEADER SKIP_BLANK_LINES STATEMENT SUM_OPNSIZE - TABLE_CHECKSUM TABLE_MODE TABLE_ID TABLE_NAME TABLEGROUPS TABLES TABLESPACE TABLET TABLET_ID TABLET_MAX_SIZE + TABLE_CHECKSUM TABLE_MODE TABLE_ID TABLE_NAME TABLEGROUPS TABLES TABLESPACE TABLET TABLET_ID TABLET_MAX_SIZE TASK_ID TEMPLATE TEMPORARY TEMPTABLE TENANT TEXT THAN TIME TIMESTAMP TIMESTAMPADD TIMESTAMPDIFF TP_NO TP_NAME TRACE TRADITIONAL TRANSACTION TRIGGERS TRIM TRUNCATE TYPE TYPES TASK TABLET_SIZE TABLEGROUP_ID TENANT_ID THROTTLE TIME_ZONE_INFO TOP_K_FRE_HIST TIMES TRIM_SPACE TTL @@ -488,7 +488,7 @@ END_P SET_VAR DELIMITER %type partition_role ls_role zone_desc opt_zone_desc server_or_zone opt_server_or_zone opt_partitions opt_subpartitions add_or_alter_zone_options alter_or_change_or_modify %type ls opt_tenant_list_or_ls_or_tablet_id ls_server_or_server_or_zone_or_tenant add_or_alter_zone_option %type opt_tenant_list_v2 -%type suspend_or_resume tenant_name opt_tenant_name cache_name opt_cache_name file_id opt_file_id cancel_task_type +%type suspend_or_resume tenant_name opt_tenant_name cache_name opt_cache_name file_id opt_file_id cancel_task_type opt_data_source opt_paxos_replica_num %type sql_id_or_schema_id_expr opt_sql_id_or_schema_id %type namespace_expr opt_namespace %type server_action server_list opt_server_list @@ -18025,6 +18025,51 @@ alter_with_opt_hint SYSTEM CANCEL MIGRATE UNIT INTNUM malloc_non_terminal_node($$, result->malloc_pool_, T_MIGRATE_UNIT, 2, $6, NULL); } | +alter_with_opt_hint SYSTEM ADD REPLICA ls SERVER opt_equal_mark STRING_VALUE REPLICA_TYPE opt_equal_mark STRING_VALUE opt_data_source opt_paxos_replica_num opt_tenant_name +{ + (void)($1); + (void)($7); + (void)($10); + malloc_non_terminal_node($$, result->malloc_pool_, T_ADD_LS_REPLICA, 6, $5, $8, $11, $12, $13, $14); +} +| +alter_with_opt_hint SYSTEM REMOVE REPLICA ls SERVER opt_equal_mark STRING_VALUE opt_paxos_replica_num opt_tenant_name +{ + (void)($1); + (void)($7); + malloc_non_terminal_node($$, result->malloc_pool_, T_REMOVE_LS_REPLICA, 4, $5, $8, $9, $10); +} +| +alter_with_opt_hint SYSTEM MIGRATE REPLICA ls SOURCE opt_equal_mark STRING_VALUE DESTINATION opt_equal_mark STRING_VALUE opt_data_source opt_tenant_name +{ + (void)($1); + (void)($7); + (void)($10); + malloc_non_terminal_node($$, result->malloc_pool_, T_MIGRATE_LS_REPLICA, 5, $5, $8, $11, $12, $13); +} +| +alter_with_opt_hint SYSTEM MODIFY REPLICA ls SERVER opt_equal_mark STRING_VALUE REPLICA_TYPE opt_equal_mark STRING_VALUE opt_paxos_replica_num opt_tenant_name +{ + (void)($1); + (void)($7); + (void)($10); + malloc_non_terminal_node($$, result->malloc_pool_, T_MODIFY_LS_REPLICA_TYPE, 5, $5, $8, $11, $12, $13); +} +| +alter_with_opt_hint SYSTEM MODIFY ls PAXOS_REPLICA_NUM opt_equal_mark INTNUM opt_tenant_name +{ + (void)($1); + (void)($6); + malloc_non_terminal_node($$, result->malloc_pool_, T_MODIFY_LS_PAXOS_REPLICA_NUM, 3, $4, $7, $8); +} +| +alter_with_opt_hint SYSTEM CANCEL REPLICA TASK TASK_ID opt_equal_mark STRING_VALUE opt_tenant_name +{ + (void)($1); + (void)($7); + malloc_non_terminal_node($$, result->malloc_pool_, T_CANCEL_LS_REPLICA_TASK, 2, $8, $9); +} +| alter_with_opt_hint SYSTEM UPGRADE VIRTUAL SCHEMA { (void)($1); @@ -18956,6 +19001,30 @@ RT COMP_EQ int_or_decimal } ; +opt_data_source: +DATA_SOURCE opt_equal_mark STRING_VALUE +{ + (void)($2); + $$ = $3; +} +| /*EMPTY*/ +{ + $$ = NULL; +} +; + +opt_paxos_replica_num: +PAXOS_REPLICA_NUM opt_equal_mark INTNUM +{ + (void)($2); + $$ = $3; +} +| /*EMPTY*/ +{ + $$ = NULL; +} +; + opt_disk_alias: NAME opt_equal_mark relation_name_or_string { @@ -21996,6 +22065,7 @@ ACCOUNT | DATABASE_ID | DATAFILE | DATA_TABLE_ID +| DATA_SOURCE | DATE | DATE_ADD | DATE_SUB @@ -22307,6 +22377,7 @@ ACCOUNT | PARTITIONS | PARTITION_TYPE | PATTERN +| PAXOS_REPLICA_NUM | PERCENT_RANK | PAUSE | PERCENTAGE @@ -22522,7 +22593,8 @@ ACCOUNT | TABLET_ID | TABLET_SIZE | TABLET_MAX_SIZE -| TASK +| TASK +| TASK_ID | TEMPLATE | TEMPORARY | TEMPTABLE diff --git a/src/sql/privilege_check/ob_privilege_check.cpp b/src/sql/privilege_check/ob_privilege_check.cpp index d2f5184d3..3158640fa 100644 --- a/src/sql/privilege_check/ob_privilege_check.cpp +++ b/src/sql/privilege_check/ob_privilege_check.cpp @@ -2551,7 +2551,8 @@ int get_sys_tenant_alter_system_priv( stmt::T_RECOVER != basic_stmt->get_stmt_type() && stmt::T_TABLE_TTL != basic_stmt->get_stmt_type() && stmt::T_ALTER_SYSTEM_RESET_PARAMETER != basic_stmt->get_stmt_type() && - stmt::T_TRANSFER_PARTITION != basic_stmt->get_stmt_type()) { + stmt::T_TRANSFER_PARTITION != basic_stmt->get_stmt_type() && + stmt::T_ALTER_LS_REPLICA != basic_stmt->get_stmt_type()) { ret = OB_ERR_NO_PRIVILEGE; LOG_WARN("Only sys tenant can do this operation", K(ret), "stmt type", basic_stmt->get_stmt_type()); diff --git a/src/sql/resolver/cmd/ob_alter_system_resolver.cpp b/src/sql/resolver/cmd/ob_alter_system_resolver.cpp index 85f0d8114..85a41fa94 100644 --- a/src/sql/resolver/cmd/ob_alter_system_resolver.cpp +++ b/src/sql/resolver/cmd/ob_alter_system_resolver.cpp @@ -2779,6 +2779,538 @@ int ObMigrateUnitResolver::resolve(const ParseNode &parse_tree) return ret; } +int ObAlterSystemResolverUtil::get_and_verify_tenant_name( + const ParseNode* tenant_name_node, + const uint64_t exec_tenant_id, + uint64_t &target_tenant_id) +{ + // get tenant id + int ret = OB_SUCCESS; + target_tenant_id = OB_INVALID_TENANT_ID; + ObString tenant_name; + ObSchemaGetterGuard schema_guard; + const ObSimpleTenantSchema *tenant_schema = NULL; + if (OB_UNLIKELY(!is_valid_tenant_id(exec_tenant_id))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("exec tenant id is invalid", KR(ret), K(exec_tenant_id)); + } else if (OB_ISNULL(GCTX.schema_service_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("GCTX.schema_service_ is null", KR(ret), KP(GCTX.schema_service_)); + } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, schema_guard))) { + LOG_WARN("failed to get_tenant_schema_guard", KR(ret)); + } else if (NULL == tenant_name_node) { + target_tenant_id = exec_tenant_id; + } else if (OB_FAIL(resolve_tenant_name(tenant_name_node, exec_tenant_id, tenant_name))) { + LOG_WARN("fail to resolve target tenant id", KR(ret)); + } else if (OB_FAIL(schema_guard.get_tenant_id(tenant_name, target_tenant_id))) { + LOG_WARN("failed to get tenant id from schema guard", KR(ret), K(tenant_name)); + if (OB_TENANT_NOT_EXIST == ret || OB_ERR_INVALID_TENANT_NAME == ret) { + ret = OB_TENANT_NOT_EXIST; + LOG_USER_ERROR(OB_TENANT_NOT_EXIST, tenant_name.length(), tenant_name.ptr()); + } + } else if (OB_SYS_TENANT_ID != exec_tenant_id && target_tenant_id != exec_tenant_id) { + ret = OB_ERR_NO_PRIVILEGE; + LOG_USER_ERROR(OB_ERR_NO_PRIVILEGE, "sys tenant"); + LOG_WARN("no support operating other user tenants", KR(ret), K(target_tenant_id), K(exec_tenant_id)); + } + // check tenant status + if (OB_FAIL(ret)) { + } else if (OB_FAIL(schema_guard.get_tenant_info(target_tenant_id, tenant_schema))) { + LOG_WARN("fail to get tenant schema", KR(ret), K(target_tenant_id)); + } else if (OB_ISNULL(tenant_schema)) { + ret = OB_TENANT_NOT_EXIST; + LOG_USER_ERROR(OB_TENANT_NOT_EXIST, tenant_name.length(), tenant_name.ptr()); + LOG_WARN("tenant not exist", KR(ret), KP(tenant_schema), K(tenant_name), K(target_tenant_id)); + } else if (tenant_schema->is_creating() || tenant_schema->is_dropping()) { + ret = OB_OP_NOT_ALLOW; + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "Tenant is creating or dropping, current operation is"); + LOG_WARN("tenant status not normal", KR(ret), + K(tenant_schema->is_creating()), K(tenant_schema->is_dropping())); + } + return ret; +} + +int ObAlterSystemResolverUtil::check_and_get_data_source( + const ParseNode* data_source_node, + common::ObAddr& data_source) +{ + int ret = OB_SUCCESS; + data_source.reset(); + if (NULL == data_source_node) { + // pass + } else if (OB_FAIL(Util::resolve_server_value(data_source_node, data_source))) { + LOG_WARN("resolve data_source failed", KR(ret), KP(data_source_node)); + if (OB_INVALID_ARGUMENT == ret) { + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "data_source"); + } + } else if (OB_UNLIKELY(!data_source.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "data_source"); + LOG_WARN("data_source is invalid", KR(ret), K(data_source)); + } + return ret; +} + +int ObAlterSystemResolverUtil::check_and_get_server_addr( + const ParseNode* server_addr_node, + common::ObAddr& server_addr) +{ + int ret = OB_SUCCESS; + server_addr.reset(); + if (OB_ISNULL(server_addr_node)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), KP(server_addr_node)); + } else if (OB_FAIL(Util::resolve_server_value(server_addr_node, server_addr))) { + LOG_WARN("resolve server failed", KR(ret), KP(server_addr_node)); + if (OB_INVALID_ARGUMENT == ret) { + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "server"); + } + } else if (OB_UNLIKELY(!server_addr.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "server"); + LOG_WARN("server_addr is invalid", KR(ret), K(server_addr)); + } + return ret; +} + +int ObAlterSystemResolverUtil::check_and_get_paxos_replica_num( + const ParseNode* paxos_replica_num_node, + int64_t& paxos_replica_num) +{ + int ret = OB_SUCCESS; + paxos_replica_num = 0; + if (NULL == paxos_replica_num_node) { + // pass + } else { + paxos_replica_num = paxos_replica_num_node->value_; + if (paxos_replica_num <= 0) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "paxos_replica_num which should not be less than or equal to 0"); + LOG_WARN("not allowed to set paxos_replica_num less than or equal to 0", KR(ret), K(paxos_replica_num)); + } + } + return ret; +} + +int ObAlterSystemResolverUtil::check_compatibility_for_alter_ls_replica( + const uint64_t cur_tenant_id) +{ + int ret = OB_SUCCESS; + uint64_t tenant_data_version = 0; + if (OB_UNLIKELY(OB_INVALID_TENANT_ID == cur_tenant_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(cur_tenant_id)); + } else if (OB_FAIL(GET_MIN_DATA_VERSION(cur_tenant_id, tenant_data_version))) { + //The internal tables involved are under the tenant (meta) and do not involve sys tenants. + LOG_WARN("get tenant data version failed", KR(ret), K(cur_tenant_id)); + } else if (!((tenant_data_version >= DATA_VERSION_4_3_3_0) + || (tenant_data_version >= MOCK_DATA_VERSION_4_2_3_0 && tenant_data_version < DATA_VERSION_4_3_0_0) + || (tenant_data_version >= MOCK_DATA_VERSION_4_2_1_8 && tenant_data_version < DATA_VERSION_4_2_2_0))) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("Tenant data version does not match, alter LS replica command is not allowed", + KR(ret), K(cur_tenant_id), K(tenant_data_version)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "Tenant data version does not match, alter LS replica command is"); + } + return ret; +} + +int ObAlterSystemResolverUtil::do_check_for_alter_ls_replica( + const ParseNode *tenant_name_node, + ObSchemaChecker *schema_checker, + ObSQLSessionInfo *session_info, + uint64_t &target_tenant_id) +{ + int ret = OB_SUCCESS; + target_tenant_id = OB_INVALID_TENANT_ID; + if(OB_ISNULL(schema_checker) || OB_ISNULL(session_info)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), KP(schema_checker), KP(session_info)); + } else if (OB_FAIL(Util::get_and_verify_tenant_name(tenant_name_node, + session_info->get_effective_tenant_id(), + target_tenant_id))) { + LOG_WARN("get and verify tenant_name failed", KR(ret), + KP(tenant_name_node), K(session_info->get_effective_tenant_id())); + } else if (OB_FAIL(Util::check_compatibility_for_alter_ls_replica(target_tenant_id))) { + LOG_WARN("check compatibility for alter ls replica failed", KR(ret), K(target_tenant_id)); + } else if (ObSchemaChecker::is_ora_priv_check()) { + if (OB_FAIL(schema_checker->check_ora_ddl_priv( + session_info->get_effective_tenant_id(), + session_info->get_priv_user_id(), ObString(""), + // why use T_ALTER_SYSTEM_SET_PARAMETER? + // because T_ALTER_SYSTEM_SET_PARAMETER has following + // traits: T_ALTER_SYSTEM_SET_PARAMETER can allow dba to + // do an operation and prohibit other user to do this + // operation so we reuse this. + stmt::T_ALTER_SYSTEM_SET_PARAMETER, + session_info->get_enable_role_array()))) { + LOG_WARN("failed to check privilege", KR(ret), K(session_info->get_effective_tenant_id()), + K(session_info->get_user_id())); + } + } + return ret; +} + +int ObAddLSReplicaResolver::resolve(const ParseNode &parse_tree) +{ + int ret = OB_SUCCESS; + ObAlterLSReplicaStmt *stmt = NULL; + if (OB_UNLIKELY(T_ADD_LS_REPLICA != parse_tree.type_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("type is not T_ADD_LS_REPLICA", KR(ret), "type", get_type_name(parse_tree.type_)); + } else if (OB_ISNULL(parse_tree.children_) || OB_UNLIKELY(parse_tree.num_child_ != 6)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid argument", KR(ret), "type", get_type_name(parse_tree.type_), + "child_num", parse_tree.num_child_); + } else if (OB_ISNULL(parse_tree.children_[0]) + || OB_ISNULL(parse_tree.children_[1]) + || OB_ISNULL(parse_tree.children_[2])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid parse tree", KR(ret), KP(parse_tree.children_[0]), + KP(parse_tree.children_[1]), KP(parse_tree.children_[2])); + } else if (OB_ISNULL(stmt = create_stmt())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("create ObAlterLSReplicaStmt failed", KR(ret)); + } else if (OB_ISNULL(session_info_) || OB_ISNULL(schema_checker_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("session_info_ is null", KR(ret), KP(session_info_), KP(schema_checker_)); + } else { + stmt_ = stmt; + ParseNode *ls_id_node = parse_tree.children_[0]; + ParseNode *server_addr_node = parse_tree.children_[1]; + ParseNode *replica_type_node = parse_tree.children_[2]; + ParseNode *data_source_node = parse_tree.children_[3]; + ParseNode *paxos_replica_num_node = parse_tree.children_[4]; + ParseNode *tenant_name_node = parse_tree.children_[5]; + + int64_t ls_id = 0; + common::ObAddr server_addr; + common::ObReplicaType replica_type = REPLICA_TYPE_MAX; + common::ObAddr data_source; + int64_t paxos_replica_num = 0; + uint64_t tenant_id = OB_INVALID_TENANT_ID; + if (OB_FAIL(Util::do_check_for_alter_ls_replica(tenant_name_node, + schema_checker_, + session_info_, + tenant_id))) { + LOG_WARN("do check for alter ls replica failed", KR(ret), + KP(tenant_name_node), KP(schema_checker_), KP(session_info_)); + } else if (OB_FAIL(Util::resolve_ls_id(ls_id_node, ls_id))) { + LOG_WARN("resolve ls id failed", KR(ret), KP(ls_id_node)); + } else if (OB_FAIL(Util::check_and_get_server_addr(server_addr_node, server_addr))) { + LOG_WARN("resolve server failed", KR(ret), KP(server_addr_node)); + } else if (OB_FAIL(Util::resolve_replica_type(replica_type_node, replica_type))) { + LOG_WARN("resolve replica type failed", KR(ret), KP(replica_type_node)); + } else if (OB_FAIL(Util::check_and_get_data_source(data_source_node, data_source))) { + LOG_WARN("check and get data source failed", KR(ret), KP(data_source_node)); + } else if (OB_FAIL(Util::check_and_get_paxos_replica_num(paxos_replica_num_node, paxos_replica_num))) { + LOG_WARN("check and get paxos replica num failed", KR(ret), KP(paxos_replica_num_node)); + } + if (OB_SUCC(ret)) { + share::ObLSID id(ls_id); + if (OB_FAIL(stmt->get_rpc_arg().init_add(id, server_addr, replica_type, + data_source, paxos_replica_num, tenant_id))) { + LOG_WARN("init rpc arg failed", KR(ret), K(id), K(server_addr), + K(replica_type), K(data_source), K(paxos_replica_num), K(tenant_id)); + } + } + } + FLOG_INFO("resolve add replica parse tree over", KR(ret)); + return ret; +} + +int ObRemoveLSReplicaResolver::resolve(const ParseNode &parse_tree) +{ + int ret = OB_SUCCESS; + ObAlterLSReplicaStmt *stmt = NULL; + if (OB_UNLIKELY(T_REMOVE_LS_REPLICA != parse_tree.type_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("type is not T_REMOVE_LS_REPLICA", KR(ret), "type", get_type_name(parse_tree.type_)); + } else if (OB_ISNULL(parse_tree.children_) || OB_UNLIKELY(parse_tree.num_child_ != 4)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid argument", KR(ret), "type", get_type_name(parse_tree.type_), + "child_num", parse_tree.num_child_); + } else if (OB_ISNULL(parse_tree.children_[0]) || OB_ISNULL(parse_tree.children_[1])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid parse tree", KR(ret), KP(parse_tree.children_[0]), KP(parse_tree.children_[1])); + } else if (OB_ISNULL(stmt = create_stmt())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("create ObAlterLSReplicaStmt failed", KR(ret)); + } else if (OB_ISNULL(session_info_) || OB_ISNULL(schema_checker_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("session_info_ is null", KR(ret), KP(session_info_), KP(schema_checker_)); + } else { + stmt_ = stmt; + ParseNode *ls_id_node = parse_tree.children_[0]; + ParseNode *server_addr_node = parse_tree.children_[1]; + ParseNode *paxos_replica_num_node = parse_tree.children_[2]; + ParseNode *tenant_name_node = parse_tree.children_[3]; + int64_t ls_id = 0; + common::ObAddr server_addr; + int64_t paxos_replica_num = 0; + uint64_t tenant_id = OB_INVALID_TENANT_ID; + if (OB_FAIL(Util::do_check_for_alter_ls_replica(tenant_name_node, + schema_checker_, + session_info_, + tenant_id))) { + LOG_WARN("do check for alter ls replica failed", KR(ret), + KP(tenant_name_node), KP(schema_checker_), KP(session_info_)); + } else if (OB_FAIL(Util::resolve_ls_id(ls_id_node, ls_id))) { + LOG_WARN("resolve ls id failed", KR(ret), KP(ls_id_node)); + } else if (OB_FAIL(Util::check_and_get_server_addr(server_addr_node, server_addr))) { + LOG_WARN("resolve server failed", KR(ret), KP(server_addr_node)); + } else if (OB_FAIL(Util::check_and_get_paxos_replica_num(paxos_replica_num_node, paxos_replica_num))) { + LOG_WARN("check and get paxos replica num failed", KR(ret), KP(paxos_replica_num_node)); + } + if (OB_SUCC(ret)) { + share::ObLSID id(ls_id); + if (OB_FAIL(stmt->get_rpc_arg().init_remove(id, server_addr, paxos_replica_num, tenant_id))) { + LOG_WARN("init rpc arg failed", KR(ret), K(id), K(server_addr), + K(paxos_replica_num), K(tenant_id)); + } + } + } + FLOG_INFO("resolve remove replica parse tree over", KR(ret)); + return ret; +} + +int ObMigrateLSReplicaResolver::resolve(const ParseNode &parse_tree) +{ + int ret = OB_SUCCESS; + ObAlterLSReplicaStmt *stmt = NULL; + if (OB_UNLIKELY(T_MIGRATE_LS_REPLICA != parse_tree.type_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("type is not T_MIGRATE_LS_REPLICA", KR(ret), "type", get_type_name(parse_tree.type_)); + } else if (OB_ISNULL(parse_tree.children_) || OB_UNLIKELY(parse_tree.num_child_ != 5)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid argument", KR(ret), "type", get_type_name(parse_tree.type_), + "child_num", parse_tree.num_child_); + } else if (OB_ISNULL(parse_tree.children_[0]) + || OB_ISNULL(parse_tree.children_[1]) + || OB_ISNULL(parse_tree.children_[2])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid parse tree", KR(ret), KP(parse_tree.children_[0]), + KP(parse_tree.children_[1]), KP(parse_tree.children_[2])); + } else if (OB_ISNULL(stmt = create_stmt())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("create ObAlterLSReplicaStmt failed", KR(ret)); + } else if (OB_ISNULL(session_info_) || OB_ISNULL(schema_checker_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("session_info_ is null", KR(ret), KP(session_info_), KP(schema_checker_)); + } else { + stmt_ = stmt; + ParseNode *ls_id_node = parse_tree.children_[0]; + ParseNode *source_addr_node = parse_tree.children_[1]; + ParseNode *destination_addr_node = parse_tree.children_[2]; + ParseNode *data_source_node = parse_tree.children_[3]; + ParseNode *tenant_name_node = parse_tree.children_[4]; + int64_t ls_id = 0; + common::ObAddr source_addr; + common::ObAddr destination_addr; + common::ObAddr data_source; + uint64_t tenant_id = OB_INVALID_TENANT_ID; + if (OB_FAIL(Util::do_check_for_alter_ls_replica(tenant_name_node, + schema_checker_, + session_info_, + tenant_id))) { + LOG_WARN("do check for alter ls replica failed", KR(ret), + KP(tenant_name_node), KP(schema_checker_), KP(session_info_)); + } else if (OB_FAIL(Util::resolve_ls_id(ls_id_node, ls_id))) { + LOG_WARN("resolve ls id failed", KR(ret), KP(ls_id_node)); + } else if (OB_FAIL(Util::resolve_server_value(source_addr_node, source_addr))) { + LOG_WARN("resolve server failed", KR(ret), KP(source_addr_node)); + if (OB_INVALID_ARGUMENT == ret) { + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "source"); + } + } else if (OB_FAIL(Util::resolve_server_value(destination_addr_node, destination_addr))) { + LOG_WARN("resolve server failed", KR(ret), KP(destination_addr_node)); + if (OB_INVALID_ARGUMENT == ret) { + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "destination"); + } + } else if (OB_UNLIKELY(!source_addr.is_valid()) || OB_UNLIKELY(!destination_addr.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "source or destination"); + LOG_WARN("source_addr or destination_addr is invalid", KR(ret), K(source_addr), K(destination_addr)); + } else if (OB_FAIL(Util::check_and_get_data_source(data_source_node, data_source))) { + LOG_WARN("check and get data source failed", KR(ret), KP(data_source_node)); + } + if (OB_SUCC(ret)) { + share::ObLSID id(ls_id); + if (OB_FAIL(stmt->get_rpc_arg().init_migrate(id, source_addr, destination_addr, + data_source, tenant_id))) { + LOG_WARN("init rpc arg failed", KR(ret), K(id), K(source_addr), K(destination_addr), + K(data_source), K(tenant_id)); + } + } + } + FLOG_INFO("resolve migrate replica parse tree over", KR(ret)); + return ret; +} + +int ObModifyLSReplicaResolver::resolve(const ParseNode &parse_tree) +{ + int ret = OB_SUCCESS; + ObAlterLSReplicaStmt *stmt = NULL; + if (OB_UNLIKELY(T_MODIFY_LS_REPLICA_TYPE != parse_tree.type_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("type is not T_MODIFY_LS_REPLICA_TYPE", KR(ret), "type", get_type_name(parse_tree.type_)); + } else if (OB_ISNULL(parse_tree.children_) || OB_UNLIKELY(parse_tree.num_child_ != 5)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid argument", KR(ret), "type", get_type_name(parse_tree.type_), + "child_num", parse_tree.num_child_); + } else if (OB_ISNULL(parse_tree.children_[0]) + || OB_ISNULL(parse_tree.children_[1]) + || OB_ISNULL(parse_tree.children_[2])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid parse tree", KR(ret), KP(parse_tree.children_[0]), + KP(parse_tree.children_[1]), KP(parse_tree.children_[2])); + } else if (OB_ISNULL(stmt = create_stmt())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("create ObAlterLSReplicaStmt failed", KR(ret)); + } else if (OB_ISNULL(session_info_) || OB_ISNULL(schema_checker_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("session_info_ is null", KR(ret), KP(session_info_), KP(schema_checker_)); + } else { + stmt_ = stmt; + ParseNode *ls_id_node = parse_tree.children_[0]; + ParseNode *server_addr_node = parse_tree.children_[1]; + ParseNode *replica_type_node = parse_tree.children_[2]; + ParseNode *paxos_replica_num_node = parse_tree.children_[3]; + ParseNode *tenant_name_node = parse_tree.children_[4]; + int64_t ls_id = 0; + common::ObAddr server_addr; + common::ObReplicaType replica_type = REPLICA_TYPE_MAX; + int64_t paxos_replica_num = 0; + uint64_t tenant_id = OB_INVALID_TENANT_ID; + if (OB_FAIL(Util::do_check_for_alter_ls_replica(tenant_name_node, + schema_checker_, + session_info_, + tenant_id))) { + LOG_WARN("do check for alter ls replica failed", KR(ret), + KP(tenant_name_node), KP(schema_checker_), KP(session_info_)); + } else if (OB_FAIL(Util::resolve_ls_id(ls_id_node, ls_id))) { + LOG_WARN("resolve ls id failed", KR(ret), KP(ls_id_node)); + } else if (OB_FAIL(Util::check_and_get_server_addr(server_addr_node, server_addr))) { + LOG_WARN("resolve server failed", KR(ret), KP(server_addr_node)); + } else if (OB_FAIL(Util::resolve_replica_type(replica_type_node, replica_type))) { + LOG_WARN("resolve replica type failed", KR(ret), KP(replica_type_node)); + } else if (OB_FAIL(Util::check_and_get_paxos_replica_num(paxos_replica_num_node, paxos_replica_num))) { + LOG_WARN("check and get paxos replica num failed", KR(ret), KP(paxos_replica_num_node)); + } + if (OB_SUCC(ret)) { + share::ObLSID id(ls_id); + if (OB_FAIL(stmt->get_rpc_arg().init_modify_replica(id, server_addr, replica_type, + paxos_replica_num, tenant_id))) { + LOG_WARN("init rpc arg failed", KR(ret), K(id), K(server_addr), + K(replica_type), K(paxos_replica_num), K(tenant_id)); + } + } + } + FLOG_INFO("resolve modify replica type parse tree over", KR(ret)); + return ret; +} + +int ObModifyLSPaxosReplicaNumResolver::resolve(const ParseNode &parse_tree) +{ + int ret = OB_SUCCESS; + ObAlterLSReplicaStmt *stmt = NULL; + if (OB_UNLIKELY(T_MODIFY_LS_PAXOS_REPLICA_NUM != parse_tree.type_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("type is not T_MODIFY_LS_PAXOS_REPLICA_NUM", KR(ret), "type", get_type_name(parse_tree.type_)); + } else if (OB_ISNULL(parse_tree.children_) || OB_UNLIKELY(parse_tree.num_child_ != 3)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid argument", KR(ret), "type", get_type_name(parse_tree.type_), + "child_num", parse_tree.num_child_); + } else if (OB_ISNULL(parse_tree.children_[0]) || OB_ISNULL(parse_tree.children_[1])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid parse tree", KR(ret), KP(parse_tree.children_[0]), + KP(parse_tree.children_[1])); + } else if (OB_ISNULL(stmt = create_stmt())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("create ObAlterLSReplicaStmt failed", KR(ret)); + } else if (OB_ISNULL(session_info_) || OB_ISNULL(schema_checker_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("session_info_ is null", KR(ret), KP(session_info_), KP(schema_checker_)); + } else { + stmt_ = stmt; + ParseNode *ls_id_node = parse_tree.children_[0]; + ParseNode *paxos_replica_num_node = parse_tree.children_[1]; + ParseNode *tenant_name_node = parse_tree.children_[2]; + int64_t ls_id = 0; + int64_t paxos_replica_num = 0; + uint64_t tenant_id = OB_INVALID_TENANT_ID; + if (OB_FAIL(Util::do_check_for_alter_ls_replica(tenant_name_node, + schema_checker_, + session_info_, + tenant_id))) { + LOG_WARN("do check for alter ls replica failed", KR(ret), + KP(tenant_name_node), KP(schema_checker_), KP(session_info_)); + } else if (OB_FAIL(Util::resolve_ls_id(ls_id_node, ls_id))) { + LOG_WARN("resolve ls id failed", KR(ret), KP(ls_id_node)); + } + if (OB_SUCC(ret)) { + paxos_replica_num = paxos_replica_num_node->value_; + if (paxos_replica_num <= 0) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "paxos_replica_num which should not be less than or equal to 0"); + LOG_WARN("not allowed to set paxos_replica_num less than or equal to 0", KR(ret), K(paxos_replica_num)); + } + } + if (OB_SUCC(ret)) { + share::ObLSID id(ls_id); + if (OB_FAIL(stmt->get_rpc_arg().init_modify_paxos_replica_num(id, paxos_replica_num, tenant_id))) { + LOG_WARN("init rpc arg failed", KR(ret), K(id), K(paxos_replica_num), K(tenant_id)); + } + } + } + FLOG_INFO("resolve modify paxos_replica_num parse tree over", KR(ret)); + return ret; +} + +int ObCancelLSReplicaTaskResolver::resolve(const ParseNode &parse_tree) +{ + int ret = OB_SUCCESS; + ObAlterLSReplicaStmt *stmt = NULL; + if (OB_UNLIKELY(T_CANCEL_LS_REPLICA_TASK != parse_tree.type_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("type is not T_CANCEL_LS_REPLICA_TASK", KR(ret), "type", get_type_name(parse_tree.type_)); + } else if (OB_ISNULL(parse_tree.children_) || OB_UNLIKELY(parse_tree.num_child_ != 2)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid argument", KR(ret), "type", get_type_name(parse_tree.type_), + "child_num", parse_tree.num_child_); + } else if (OB_ISNULL(parse_tree.children_[0])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid parse tree", KR(ret), KP(parse_tree.children_[0])); + } else if (OB_ISNULL(stmt = create_stmt())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("create ObAlterLSReplicaStmt failed", KR(ret)); + } else if (OB_ISNULL(session_info_) || OB_ISNULL(schema_checker_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("session_info_ is null", KR(ret), KP(session_info_), KP(schema_checker_)); + } else { + stmt_ = stmt; + ParseNode *task_id_node = parse_tree.children_[0]; + ParseNode *tenant_name_node = parse_tree.children_[1]; + Task_Id task_id; + uint64_t tenant_id = OB_INVALID_TENANT_ID; + ObString task_id_str; + if (OB_FAIL(Util::do_check_for_alter_ls_replica(tenant_name_node, + schema_checker_, + session_info_, + tenant_id))) { + LOG_WARN("do check for alter ls replica failed", KR(ret), + KP(tenant_name_node), KP(schema_checker_), KP(session_info_)); + } else if (OB_FAIL(Util::resolve_string(task_id_node, task_id_str))) { + LOG_WARN("tenant name assign failed", KR(ret), KP(task_id_node)); + } else if (OB_FAIL(task_id.assign(task_id_str))) { + LOG_WARN("task id assign failed", KR(ret), K(task_id_str)); + } else if (OB_FAIL(stmt->get_rpc_arg().init_cancel(task_id, tenant_id))) { + LOG_WARN("init rpc arg failed", KR(ret), K(task_id), K(tenant_id)); + } + } + FLOG_INFO("resolve cancel parse tree over", KR(ret)); + return ret; +} + int ObAddArbitrationServiceResolver::resolve(const ParseNode &parse_tree) { int ret = OB_SUCCESS; diff --git a/src/sql/resolver/cmd/ob_alter_system_resolver.h b/src/sql/resolver/cmd/ob_alter_system_resolver.h index 18e23707a..682350bb1 100644 --- a/src/sql/resolver/cmd/ob_alter_system_resolver.h +++ b/src/sql/resolver/cmd/ob_alter_system_resolver.h @@ -65,8 +65,24 @@ public: bool &affect_all, bool &affect_all_user, bool &affect_all_meta); + static int get_and_verify_tenant_name(const ParseNode* tenant_name_node, + const uint64_t exec_tenant_id, + uint64_t &target_tenant_id); + static int check_and_get_data_source(const ParseNode* data_source_node, + common::ObAddr& data_source); + static int check_and_get_server_addr(const ParseNode* server_addr_node, + common::ObAddr& server_addr); + static int check_and_get_paxos_replica_num(const ParseNode* paxos_replica_num_node, + int64_t& paxos_replica_num); + static int check_compatibility_for_alter_ls_replica(const uint64_t cur_tenant_id); + static int do_check_for_alter_ls_replica(const ParseNode *tenant_name_node, + ObSchemaChecker *schema_checker, + ObSQLSessionInfo *session_info, + uint64_t &target_tenant_id); }; +typedef common::ObFixedLengthString Task_Id; + #define DEF_SIMPLE_CMD_RESOLVER(name) \ class name : public ObSystemCmdResolver \ { \ @@ -128,6 +144,13 @@ DEF_SIMPLE_CMD_RESOLVER(ObReplaceArbitrationServiceResolver); DEF_SIMPLE_CMD_RESOLVER(ObMigrateUnitResolver); +DEF_SIMPLE_CMD_RESOLVER(ObAddLSReplicaResolver); +DEF_SIMPLE_CMD_RESOLVER(ObRemoveLSReplicaResolver); +DEF_SIMPLE_CMD_RESOLVER(ObMigrateLSReplicaResolver); +DEF_SIMPLE_CMD_RESOLVER(ObModifyLSReplicaResolver); +DEF_SIMPLE_CMD_RESOLVER(ObModifyLSPaxosReplicaNumResolver); +DEF_SIMPLE_CMD_RESOLVER(ObCancelLSReplicaTaskResolver); + DEF_SIMPLE_CMD_RESOLVER(ObUpgradeVirtualSchemaResolver); DEF_SIMPLE_CMD_RESOLVER(ObRunJobResolver); diff --git a/src/sql/resolver/cmd/ob_alter_system_stmt.h b/src/sql/resolver/cmd/ob_alter_system_stmt.h index 35ff6d895..dd0a4add5 100644 --- a/src/sql/resolver/cmd/ob_alter_system_stmt.h +++ b/src/sql/resolver/cmd/ob_alter_system_stmt.h @@ -402,6 +402,18 @@ private: obrpc::ObAdminMigrateUnitArg rpc_arg_; }; +class ObAlterLSReplicaStmt : public ObSystemCmdStmt +{ +public: + ObAlterLSReplicaStmt() : ObSystemCmdStmt(stmt::T_ALTER_LS_REPLICA) {} + virtual ~ObAlterLSReplicaStmt() {} + + obrpc::ObAdminAlterLSReplicaArg &get_rpc_arg() { return rpc_arg_; } + TO_STRING_KV(N_STMT_TYPE, ((int)stmt_type_), K_(rpc_arg)); +private: + obrpc::ObAdminAlterLSReplicaArg rpc_arg_; +}; + class ObAddArbitrationServiceStmt : public ObSystemCmdStmt { public: diff --git a/src/sql/resolver/ob_resolver.cpp b/src/sql/resolver/ob_resolver.cpp index 1cad9426b..7eb90e741 100644 --- a/src/sql/resolver/ob_resolver.cpp +++ b/src/sql/resolver/ob_resolver.cpp @@ -533,6 +533,30 @@ int ObResolver::resolve(IsPrepared if_prepared, const ParseNode &parse_tree, ObS REGISTER_STMT_RESOLVER(MigrateUnit); break; } + case T_ADD_LS_REPLICA: { + REGISTER_STMT_RESOLVER(AddLSReplica); + break; + } + case T_REMOVE_LS_REPLICA: { + REGISTER_STMT_RESOLVER(RemoveLSReplica); + break; + } + case T_MIGRATE_LS_REPLICA: { + REGISTER_STMT_RESOLVER(MigrateLSReplica); + break; + } + case T_MODIFY_LS_REPLICA_TYPE: { + REGISTER_STMT_RESOLVER(ModifyLSReplica); + break; + } + case T_MODIFY_LS_PAXOS_REPLICA_NUM: { + REGISTER_STMT_RESOLVER(ModifyLSPaxosReplicaNum); + break; + } + case T_CANCEL_LS_REPLICA_TASK: { + REGISTER_STMT_RESOLVER(CancelLSReplicaTask); + break; + } case T_ADD_ARBITRATION_SERVICE: { REGISTER_STMT_RESOLVER(AddArbitrationService); break; diff --git a/src/sql/resolver/ob_stmt_type.h b/src/sql/resolver/ob_stmt_type.h index 81e751200..5a24fa63d 100644 --- a/src/sql/resolver/ob_stmt_type.h +++ b/src/sql/resolver/ob_stmt_type.h @@ -296,7 +296,7 @@ OB_STMT_TYPE_DEF_UNKNOWN_AT(T_CREATE_MLOG, get_create_mlog_stmt_need_privs, 295) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_DROP_MLOG, get_drop_mlog_stmt_need_privs, 296) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_TRANSFER_PARTITION, get_sys_tenant_alter_system_priv, 297) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_FLUSH_PRIVILEGES, no_priv_needed, 298) -// OB_STMT_TYPE_DEF_UNKNOWN_AT(T_ALTER_LS_REPLICA, get_sys_tenant_alter_system_priv, 299) +OB_STMT_TYPE_DEF_UNKNOWN_AT(T_ALTER_LS_REPLICA, get_sys_tenant_alter_system_priv, 299) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_SHOW_PROCEDURE_CODE, err_stmt_type_priv, 300) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_SHOW_FUNCTION_CODE, err_stmt_type_priv, 301) OB_STMT_TYPE_DEF(T_CHANGE_EXTERNAL_STORAGE_DEST, no_priv_needed, 302, ACTION_TYPE_ALTER_SYSTEM) diff --git a/src/storage/high_availability/ob_ls_complete_migration.cpp b/src/storage/high_availability/ob_ls_complete_migration.cpp index a4fbb607f..75f3510da 100644 --- a/src/storage/high_availability/ob_ls_complete_migration.cpp +++ b/src/storage/high_availability/ob_ls_complete_migration.cpp @@ -1740,6 +1740,7 @@ int ObStartCompleteMigrationTask::check_tablet_ready_( ret = OB_INVALID_ARGUMENT; LOG_WARN("check tablet ready get invalid argument", K(ret), K(tablet_id), KP(ls)); } else { + DEBUG_SYNC(BEFORE_CHECK_TABLET_READY); const int64_t wait_tablet_start_ts = ObTimeUtility::current_time(); while (OB_SUCC(ret)) { @@ -1825,6 +1826,7 @@ int ObStartCompleteMigrationTask::check_tablet_transfer_table_ready_( ret = OB_ERR_UNEXPECTED; LOG_WARN("transfer service should not be NULL", K(ret), KP(transfer_service)); } else { + DEBUG_SYNC(BEFORE_CHECK_TABLET_TRANSFER_TABLE_READY); const int64_t wait_tablet_start_ts = ObTimeUtility::current_time(); bool need_check_again = false; @@ -1973,6 +1975,7 @@ int ObStartCompleteMigrationTask::wait_log_replay_to_max_minor_end_scn_() } else if (OB_FAIL(init_timeout_ctx_(timeout, timeout_ctx))) { LOG_WARN("failed to init timeout ctx", K(ret)); } else { + DEBUG_SYNC(BEFORE_LOG_REPLAY_TO_MAX_MINOR_END_SCN); const int64_t wait_replay_start_ts = ObTimeUtility::current_time(); while (OB_SUCC(ret)) { if (timeout_ctx.is_timeouted()) { @@ -2026,6 +2029,7 @@ int ObStartCompleteMigrationTask::check_ls_and_task_status_( bool is_cancel = false; bool is_ls_deleted = true; int32_t result = OB_SUCCESS; + ObIDagNet *dag_net = nullptr; if (OB_ISNULL(ls)) { ret = OB_INVALID_ARGUMENT; @@ -2036,11 +2040,15 @@ int ObStartCompleteMigrationTask::check_ls_and_task_status_( } else if (ls->is_stopped()) { ret = OB_NOT_RUNNING; LOG_WARN("ls is not running, stop migration dag net", K(ret), KPC(ctx_)); - } else if (OB_FAIL(SYS_TASK_STATUS_MGR.is_task_cancel(get_dag()->get_dag_id(), is_cancel))) { - STORAGE_LOG(ERROR, "failed to check is task canceled", K(ret), K(*this)); - } else if (is_cancel) { + } else if (OB_ISNULL(this->get_dag())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag should not be nullptr", K(ret), KP(this->get_dag())); + } else if (OB_ISNULL(dag_net = this->get_dag()->get_dag_net())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag net should not be nullptr", K(ret), KP(dag_net)); + } else if (dag_net->is_cancel()) { ret = OB_CANCELED; - STORAGE_LOG(WARN, "task is cancelled", K(ret), K(*this)); + LOG_WARN("task is cancelled", K(ret), K(*this)); } else if (OB_FAIL(ObStorageHAUtils::check_ls_deleted(ls->get_ls_id(), is_ls_deleted))) { LOG_WARN("failed to get ls status from inner table", K(ret)); } else if (is_ls_deleted) { diff --git a/src/storage/high_availability/ob_ls_migration.cpp b/src/storage/high_availability/ob_ls_migration.cpp index 1824399e5..b773059db 100644 --- a/src/storage/high_availability/ob_ls_migration.cpp +++ b/src/storage/high_availability/ob_ls_migration.cpp @@ -1544,6 +1544,7 @@ int ObStartMigrationTask::create_all_tablets_( ObLS *ls = nullptr; ObArray tablet_id_array; bool need_check_tablet_limit = false; + ObIDagNet *dag_net = nullptr; if (!is_inited_) { ret = OB_NOT_INIT; @@ -1551,6 +1552,12 @@ int ObStartMigrationTask::create_all_tablets_( } else if (OB_ISNULL(ob_reader)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("create all tablets get ivnalid argument", K(ret)); + } else if (OB_ISNULL(this->get_dag())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag should not be nullptr", K(ret), KP(this->get_dag())); + } else if (OB_ISNULL(dag_net = this->get_dag()->get_dag_net())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag net should not be nullptr", K(ret), KP(dag_net)); } else if (FALSE_IT(need_check_tablet_limit = ctx_->arg_.type_ != ObMigrationOpType::REBUILD_LS_OP)) { } else if (OB_FAIL(ObStorageHADagUtils::get_ls(ctx_->arg_.ls_id_, ls_handle))) { LOG_WARN("failed to get ls", K(ret), KPC(ctx_)); @@ -1561,7 +1568,7 @@ int ObStartMigrationTask::create_all_tablets_( ctx_->tenant_id_, tablet_id_array, ctx_->minor_src_, ctx_->local_rebuild_seq_, ctx_->arg_.type_, ls, &ctx_->ha_table_info_mgr_, ha_tablets_builder))) { LOG_WARN("failed to init ha tablets builder", K(ret), KPC(ctx_)); - } else if (OB_FAIL(ha_tablets_builder.create_all_tablets(need_check_tablet_limit, ob_reader, + } else if (OB_FAIL(ha_tablets_builder.create_all_tablets(need_check_tablet_limit, ob_reader, dag_net, ctx_->sys_tablet_id_array_, ctx_->data_tablet_id_array_, ctx_->tablet_simple_info_map_))) { LOG_WARN("failed to create all tablets", K(ret), KPC(ctx_)); @@ -1641,10 +1648,17 @@ int ObStartMigrationTask::create_all_tablets_with_4_1_rpc_() ObLSHandle ls_handle; ObLS *ls = nullptr; ObArray tablet_id_array; + ObIDagNet *dag_net = nullptr; if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("start migration task do not init", K(ret)); + } else if (OB_ISNULL(this->get_dag())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag should not be nullptr", K(ret), KP(this->get_dag())); + } else if (OB_ISNULL(dag_net = this->get_dag()->get_dag_net())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag net should not be nullptr", K(ret), KP(dag_net)); } else if (OB_FAIL(ObStorageHAUtils::append_tablet_list(ctx_->sys_tablet_id_array_, tablet_id_array))) { LOG_WARN("failed to append sys tablet id array", K(ret), KPC(ctx_)); } else if (OB_FAIL(ObStorageHAUtils::append_tablet_list(ctx_->data_tablet_id_array_, tablet_id_array))) { @@ -1658,7 +1672,7 @@ int ObStartMigrationTask::create_all_tablets_with_4_1_rpc_() ctx_->tenant_id_, tablet_id_array, ctx_->minor_src_, ctx_->local_rebuild_seq_, ctx_->arg_.type_, ls, &ctx_->ha_table_info_mgr_, ha_tablets_builder))) { LOG_WARN("failed to init ha tablets builder", K(ret), KPC(ctx_)); - } else if (OB_FAIL(ha_tablets_builder.create_all_tablets_with_4_1_rpc( + } else if (OB_FAIL(ha_tablets_builder.create_all_tablets_with_4_1_rpc(dag_net, ctx_->tablet_simple_info_map_, ctx_->sys_tablet_id_array_, ctx_->data_tablet_id_array_))) { LOG_WARN("failed to create all tablets", K(ret), KPC(ctx_)); } @@ -1895,12 +1909,20 @@ int ObSysTabletsMigrationTask::process() int ObSysTabletsMigrationTask::build_tablets_sstable_info_() { int ret = OB_SUCCESS; + ObIDagNet *dag_net = nullptr; if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("sys tablets migration task do not init", K(ret), KPC(ctx_)); - } else if (OB_FAIL(ha_tablets_builder_.build_tablets_sstable_info())) { + } else if (OB_ISNULL(this->get_dag())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag should not be nullptr", K(ret), KP(this->get_dag())); + } else if (OB_ISNULL(dag_net = this->get_dag()->get_dag_net())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag net should not be nullptr", K(ret), KP(dag_net)); + } else if (OB_FAIL(ha_tablets_builder_.build_tablets_sstable_info(dag_net))) { LOG_WARN("failed to build tablets sstable info", K(ret), KPC(ctx_)); } + return ret; } @@ -2904,6 +2926,7 @@ int ObTabletMigrationTask::try_update_tablet_() ObLS *ls = nullptr; bool is_exist = false; ObCopyTabletStatus::STATUS status = ObCopyTabletStatus::MAX_STATUS; + ObIDagNet *dag_net = nullptr; if (!is_inited_) { ret = OB_NOT_INIT; @@ -2937,9 +2960,12 @@ int ObTabletMigrationTask::try_update_tablet_() } if (OB_FAIL(ret)) { - } else if (copy_tablet_ctx_->tablet_id_.is_ls_inner_tablet() && OB_FAIL(ha_tablets_builder.create_or_update_tablets())) { + } else if (OB_ISNULL(dag_net = dag->get_dag_net())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag net should not be nullptr", K(ret), KP(dag_net)); + } else if (copy_tablet_ctx_->tablet_id_.is_ls_inner_tablet() && OB_FAIL(ha_tablets_builder.create_or_update_tablets(dag_net))) { LOG_WARN("failed to create or update inner tablet", K(ret), KPC(ctx_)); - } else if (OB_FAIL(ha_tablets_builder.build_tablets_sstable_info())) { + } else if (OB_FAIL(ha_tablets_builder.build_tablets_sstable_info(dag_net))) { LOG_WARN("failed to build tablets sstable info", K(ret), KPC(ctx_), KPC(copy_tablet_ctx_)); } else if (OB_FAIL(ctx_->ha_table_info_mgr_.check_tablet_table_info_exist(copy_tablet_ctx_->tablet_id_, is_exist))) { LOG_WARN("failed to check tablet table info exist", K(ret), KPC(copy_tablet_ctx_)); @@ -3745,6 +3771,7 @@ int ObDataTabletsMigrationTask::try_remove_unneeded_tablets_() ObArray tablet_id_array; const int64_t MAX_BUCKET_NUM = 1024; const bool need_initial_state = true; + ObIDagNet *dag_net = nullptr; if (!is_inited_) { ret = OB_NOT_INIT; @@ -3780,7 +3807,16 @@ int ObDataTabletsMigrationTask::try_remove_unneeded_tablets_() LOG_WARN("failed to build tablet iter", K(ret), KPC(ctx_)); } else { while (OB_SUCC(ret)) { - if (OB_FAIL(iter.get_next_tablet_id(tablet_id))) { + if (OB_ISNULL(this->get_dag())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag should not be nullptr", K(ret), KP(this->get_dag())); + } else if (OB_ISNULL(dag_net = this->get_dag()->get_dag_net())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag net should not be nullptr", K(ret), KP(dag_net)); + } else if (dag_net->is_cancel()) { + ret = OB_CANCELED; + LOG_WARN("task is cancelled", K(ret), K(*this)); + } else if (OB_FAIL(iter.get_next_tablet_id(tablet_id))) { if (OB_ITER_END == ret) { ret = OB_SUCCESS; break; @@ -4301,6 +4337,7 @@ int ObTabletGroupMigrationTask::build_tablets_sstable_info_() { int ret = OB_SUCCESS; bool has_inner_table = false; + ObIDagNet *dag_net = nullptr; ObArray tablet_id_array; if (!is_inited_) { @@ -4315,7 +4352,13 @@ int ObTabletGroupMigrationTask::build_tablets_sstable_info_() DEBUG_SYNC(BEFORE_MIGRATION_BUILD_TABLET_SSTABLE_INFO); } - if (OB_FAIL(ha_tablets_builder_.build_tablets_sstable_info())) { + if (OB_ISNULL(this->get_dag())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag should not be nullptr", K(ret), KP(this->get_dag())); + } else if (OB_ISNULL(dag_net = this->get_dag()->get_dag_net())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag net should not be nullptr", K(ret), KP(dag_net)); + } else if (OB_FAIL(ha_tablets_builder_.build_tablets_sstable_info(dag_net))) { LOG_WARN("failed to build tablets sstable info", K(ret), KPC(ctx_)); } } diff --git a/src/storage/high_availability/ob_ls_migration_handler.cpp b/src/storage/high_availability/ob_ls_migration_handler.cpp index d4a8ef690..ec3dd6a64 100644 --- a/src/storage/high_availability/ob_ls_migration_handler.cpp +++ b/src/storage/high_availability/ob_ls_migration_handler.cpp @@ -44,7 +44,7 @@ int ObLSMigrationHandlerStatusHelper::check_can_change_status( if (!is_valid(curr_status) || !is_valid(change_status)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("check can change status get invalid argument", K(ret), K(curr_status), K(change_status)); - }else { + } else { switch (curr_status) { case ObLSMigrationHandlerStatus::INIT: { if (ObLSMigrationHandlerStatus::INIT == change_status @@ -164,7 +164,8 @@ ObLSMigrationHandler::ObLSMigrationHandler() lock_(), status_(ObLSMigrationHandlerStatus::INIT), result_(OB_SUCCESS), - is_stop_(false) + is_stop_(false), + is_cancel_(false) { } @@ -297,6 +298,7 @@ void ObLSMigrationHandler::reuse_() task_list_.reset(); status_ = ObLSMigrationHandlerStatus::INIT; result_ = OB_SUCCESS; + is_cancel_ = false; } void ObLSMigrationHandler::wakeup_() @@ -320,14 +322,8 @@ int ObLSMigrationHandler::get_ls_migration_task_(ObLSMigrationTask &task) LOG_WARN("ls migration handler do not init", K(ret)); } else { common::SpinRLockGuard guard(lock_); - if (task_list_.empty()) { - ret = OB_ENTRY_NOT_EXIST; - LOG_WARN("migration task is empty", K(ret), KPC(ls_)); - } else if (task_list_.count() > 1) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls migration task count should not more than 1", K(ret), K(task_list_), KPC(ls_)); - } else { - task = task_list_.at(0); + if (OB_FAIL(get_ls_migration_task_with_nolock_(task))) { + LOG_WARN("failed to get ls migration task", K(ret)); } } return ret; @@ -419,30 +415,15 @@ int ObLSMigrationHandler::add_ls_migration_task( int ObLSMigrationHandler::switch_next_stage(const int32_t result) { int ret = OB_SUCCESS; - ObLSMigrationHandlerStatus next_status = ObLSMigrationHandlerStatus::MAX_STATUS; - bool can_change = false; - int32_t new_result = OB_SUCCESS; if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("ls migration handler do not init", K(ret)); } else { common::SpinWLockGuard guard(lock_); - new_result = OB_SUCCESS != result_ ? result_ : result; - - if (OB_FAIL(ObLSMigrationHandlerStatusHelper::get_next_change_status(status_, new_result, next_status))) { - LOG_WARN("failed to get next change status", K(ret), K(status_), K(result), K(new_result)); - } else if (OB_FAIL(ObLSMigrationHandlerStatusHelper::check_can_change_status(status_, next_status, can_change))) { - LOG_WARN("failed to check can change status", K(ret), K(status_), K(next_status)); - } else if (!can_change) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("can not change ls migration handler status", K(ret), K(status_), K(next_status)); - } else { - FLOG_INFO("report result", K(result), K(new_result), K(result_), K(status_), K(next_status)); - result_ = new_result; - status_ = next_status; + if (OB_FAIL(switch_next_stage_with_nolock_(result))) { + LOG_WARN("failed to switch next stage", K(ret), K(result)); } - wakeup_(); } return ret; } @@ -541,6 +522,43 @@ int ObLSMigrationHandler::process() return ret; } +int ObLSMigrationHandler::cancel_task(const share::ObTaskId &task_id, bool &is_exist) +{ + int ret = OB_SUCCESS; + ObTenantDagScheduler *scheduler = nullptr; + is_exist = false; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ls migration handle do not init", K(ret)); + } else if (!task_id.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(task_id)); + } else { + common::SpinWLockGuard guard(lock_); + if (OB_FAIL(check_task_exist_with_nolock_(task_id, is_exist))) { + LOG_WARN("fail to check task exist", K(ret), K(task_id)); + } else if (!is_exist) { + LOG_INFO("task is not exist in migration task", K(task_id)); + } else if (OB_ISNULL(scheduler = MTL(ObTenantDagScheduler*))) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("failed to get ObTenantDagScheduler from MTL", K(ret)); + } + // If task not exist, cancel_dag_net return OB_SUCCESS + else if (OB_FAIL(scheduler->cancel_dag_net(task_id))) { + LOG_WARN("failed to cancel dag net", K(ret), K(this), K(task_id)); + } else { + is_cancel_ = true; + } + } + return ret; +} + +bool ObLSMigrationHandler::is_cancel() const +{ + common::SpinRLockGuard guard(lock_); + return is_cancel_; +} + int ObLSMigrationHandler::do_init_status_() { int ret = OB_SUCCESS; @@ -555,6 +573,9 @@ int ObLSMigrationHandler::do_init_status_() LOG_WARN("ls migration handler do not init", K(ret)); } else if (is_migration_failed_()) { //do nothing + } else if (is_cancel()) { + ret = OB_CANCELED; + LOG_WARN("task is canceled", K(ret)); } else { // this lock make sure the ls creating is not scheduled to migrate. ObLSLockGuard lock_ls(ls_, true /* read lock */); @@ -740,23 +761,39 @@ int ObLSMigrationHandler::schedule_build_ls_dag_net_( ret = OB_INVALID_ARGUMENT; LOG_WARN("schedule build ls dag net get invalid argument", K(ret), K(task)); } else { + const int32_t cancel_result = OB_CANCELED; +#ifdef ERRSIM + SERVER_EVENT_ADD("storage_ha", "build_ls_migration_dag_net_before", + "tenant_id", ls_->get_tenant_id(), + "ls_id", ls_->get_ls_id().id(), + "src", task.arg_.src_.get_server(), + "dst", task.arg_.dst_.get_server(), + "task_id", task.task_id_); +#endif DEBUG_SYNC(BEFORE_BUILD_LS_MIGRATION_DAG_NET); - ObTenantDagScheduler *scheduler = nullptr; - ObMigrationDagNetInitParam param; - param.arg_ = task.arg_; - param.task_id_ = task.task_id_; - param.bandwidth_throttle_ = bandwidth_throttle_; - param.storage_rpc_ = storage_rpc_; - param.svr_rpc_proxy_ = svr_rpc_proxy_; - param.sql_proxy_ = sql_proxy_; - - if (OB_ISNULL(scheduler = MTL(ObTenantDagScheduler*))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to get ObTenantDagScheduler from MTL", K(ret), KP(scheduler)); - } else if (OB_FAIL(scheduler->create_and_add_dag_net(¶m))) { - LOG_WARN("failed to create and add migration dag net", K(ret), K(task), KPC(ls_)); + common::SpinWLockGuard guard(lock_); + if (is_cancel_) { + if (OB_FAIL(switch_next_stage_with_nolock_(cancel_result))) { + LOG_WARN("failed to swicth next stage cancel", K(ret)); + } } else { - LOG_INFO("success to create migration dag net", K(ret), K(task)); + ObTenantDagScheduler *scheduler = nullptr; + ObMigrationDagNetInitParam param; + param.arg_ = task.arg_; + param.task_id_ = task.task_id_; + param.bandwidth_throttle_ = bandwidth_throttle_; + param.storage_rpc_ = storage_rpc_; + param.svr_rpc_proxy_ = svr_rpc_proxy_; + param.sql_proxy_ = sql_proxy_; + + if (OB_ISNULL(scheduler = MTL(ObTenantDagScheduler*))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get ObTenantDagScheduler from MTL", K(ret), KP(scheduler)); + } else if (OB_FAIL(scheduler->create_and_add_dag_net(¶m))) { + LOG_WARN("failed to create and add migration dag net", K(ret), K(task), KPC(ls_)); + } else { + LOG_INFO("success to create migration dag net", K(ret), K(task)); + } } } return ret; @@ -792,18 +829,26 @@ int ObLSMigrationHandler::schedule_prepare_ls_dag_net_( ret = OB_INVALID_ARGUMENT; LOG_WARN("schedule prepare ls dag net get invalid argument", K(ret), K(task)); } else { - ObTenantDagScheduler *scheduler = nullptr; - ObLSPrepareMigrationParam param; - param.arg_ = task.arg_; - param.task_id_ = task.task_id_; - - if (OB_ISNULL(scheduler = MTL(ObTenantDagScheduler*))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to get ObTenantDagScheduler from MTL", K(ret), KP(scheduler)); - } else if (OB_FAIL(scheduler->create_and_add_dag_net(¶m))) { - LOG_WARN("failed to create and add migration dag net", K(ret), K(task), KPC(ls_)); + const int32_t cancel_result = OB_CANCELED; + common::SpinWLockGuard guard(lock_); + if (is_cancel_) { + if (OB_FAIL(switch_next_stage_with_nolock_(cancel_result))) { + LOG_WARN("failed to swicth next stage cancel", K(ret)); + } } else { - LOG_INFO("success to create ls prepare migration dag net", K(ret), K(task)); + ObTenantDagScheduler *scheduler = nullptr; + ObLSPrepareMigrationParam param; + param.arg_ = task.arg_; + param.task_id_ = task.task_id_; + + if (OB_ISNULL(scheduler = MTL(ObTenantDagScheduler*))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get ObTenantDagScheduler from MTL", K(ret), KP(scheduler)); + } else if (OB_FAIL(scheduler->create_and_add_dag_net(¶m))) { + LOG_WARN("failed to create and add migration dag net", K(ret), K(task), KPC(ls_)); + } else { + LOG_INFO("success to create ls prepare migration dag net", K(ret), K(task)); + } } } return ret; @@ -1171,7 +1216,69 @@ void ObLSMigrationHandler::wait(bool &wait_finished) } } +int ObLSMigrationHandler::get_ls_migration_task_with_nolock_(ObLSMigrationTask &task) const +{ + int ret = OB_SUCCESS; + task.reset(); + if (task_list_.empty()) { + ret = OB_ENTRY_NOT_EXIST; + LOG_WARN("migration task is empty", K(ret), KPC(ls_)); + } else if (task_list_.count() > 1) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls migration task count should not more than 1", K(ret), K(task_list_), KPC(ls_)); + } else { + task = task_list_.at(0); + } + return ret; +} +int ObLSMigrationHandler::check_task_exist_with_nolock_(const share::ObTaskId &task_id, bool &is_exist) const +{ + int ret = OB_SUCCESS; + is_exist = false; + ObLSMigrationTask task; + if (task_id.is_invalid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(task_id)); + } else if (OB_FAIL(get_ls_migration_task_with_nolock_(task))) { + if (OB_ENTRY_NOT_EXIST == ret) { + is_exist = false; + ret = OB_SUCCESS; + } else { + LOG_WARN("failed to get ls migration task", K(ret), KPC(ls_)); + } + } else if (task_id == task.task_id_) { + is_exist = true; + } else { + is_exist = false; + } + return ret; +} + +int ObLSMigrationHandler::switch_next_stage_with_nolock_(const int32_t result) +{ + int ret = OB_SUCCESS; + ObLSMigrationHandlerStatus next_status = ObLSMigrationHandlerStatus::MAX_STATUS; + bool can_change = false; + int32_t new_result = OB_SUCCESS; + + new_result = OB_SUCCESS != result_ ? result_ : result; + + if (OB_FAIL(ObLSMigrationHandlerStatusHelper::get_next_change_status(status_, new_result, next_status))) { + LOG_WARN("failed to get next change status", K(ret), K(status_), K(result), K(new_result)); + } else if (OB_FAIL(ObLSMigrationHandlerStatusHelper::check_can_change_status(status_, next_status, can_change))) { + LOG_WARN("failed to check can change status", K(ret), K(status_), K(next_status)); + } else if (!can_change) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("can not change ls migration handler status", K(ret), K(status_), K(next_status)); + } else { + FLOG_INFO("report result", K(result), K(new_result), K(result_), K(status_), K(next_status)); + result_ = new_result; + status_ = next_status; + } + wakeup_(); + return ret; +} } } diff --git a/src/storage/high_availability/ob_ls_migration_handler.h b/src/storage/high_availability/ob_ls_migration_handler.h index 172dab351..0344c8388 100644 --- a/src/storage/high_availability/ob_ls_migration_handler.h +++ b/src/storage/high_availability/ob_ls_migration_handler.h @@ -84,6 +84,8 @@ public: void destroy(); void stop(); void wait(bool &wait_finished); + int cancel_task(const share::ObTaskId &task_id, bool &is_exist); + bool is_cancel() const; private: void reuse_(); @@ -130,6 +132,9 @@ private: const uint64_t tenant_id, const share::ObLSID &ls_id, share::ObLSInfo &ls_info); + int get_ls_migration_task_with_nolock_(ObLSMigrationTask &task) const; + int check_task_exist_with_nolock_(const share::ObTaskId &task_id, bool &is_exist) const; + int switch_next_stage_with_nolock_(const int32_t result); private: bool is_inited_; @@ -146,6 +151,7 @@ private: ObLSMigrationHandlerStatus status_; int32_t result_; bool is_stop_; + bool is_cancel_; DISALLOW_COPY_AND_ASSIGN(ObLSMigrationHandler); }; diff --git a/src/storage/high_availability/ob_ls_prepare_migration.cpp b/src/storage/high_availability/ob_ls_prepare_migration.cpp index 9042092cf..8e977835a 100644 --- a/src/storage/high_availability/ob_ls_prepare_migration.cpp +++ b/src/storage/high_availability/ob_ls_prepare_migration.cpp @@ -828,11 +828,22 @@ int ObStartPrepareMigrationTask::wait_transfer_tablets_ready_() } else if (OB_FAIL(ls->build_tablet_iter(tablet_iterator))) { LOG_WARN("failed to build ls tablet iter", K(ret), KPC(ctx_)); } else { + DEBUG_SYNC(BEFORE_WAIT_TRANSFER_OUT_TABLET_READY); + ObIDagNet *dag_net = nullptr; while (OB_SUCC(ret)) { ObTabletHandle tablet_handle; ObTablet *tablet = nullptr; user_data.reset(); - if (OB_FAIL(tablet_iterator.get_next_tablet(tablet_handle))) { + if (OB_ISNULL(this->get_dag())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag should not be nullptr", K(ret), KP(this->get_dag())); + } else if (OB_ISNULL(dag_net = this->get_dag()->get_dag_net())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag net should not be nullptr", K(ret), KP(dag_net)); + } else if (dag_net->is_cancel()) { + ret = OB_CANCELED; + LOG_WARN("task is cancelled", K(ret), K(*this)); + } else if (OB_FAIL(tablet_iterator.get_next_tablet(tablet_handle))) { if (OB_ITER_END == ret) { ret = OB_SUCCESS; break; @@ -897,6 +908,7 @@ int ObStartPrepareMigrationTask::wait_transfer_out_tablet_ready_( LOG_WARN("tablet status is unexpected", K(ret), K(user_data), KPC(tablet)); } else { const int64_t wait_transfer_tablet_ready_ts = ObTimeUtility::current_time(); + ObIDagNet *dag_net = nullptr; while (OB_SUCC(ret)) { if (ctx_->is_failed()) { ret = OB_CANCELED; @@ -904,11 +916,15 @@ int ObStartPrepareMigrationTask::wait_transfer_out_tablet_ready_( } else if (ls->is_stopped()) { ret = OB_NOT_RUNNING; LOG_WARN("ls is not running, stop migration dag net", K(ret), K(ctx_)); - } else if (OB_FAIL(SYS_TASK_STATUS_MGR.is_task_cancel(get_dag()->get_dag_id(), is_cancel))) { - STORAGE_LOG(ERROR, "failed to check is task canceled", K(ret), K(*this)); - } else if (is_cancel) { + } else if (OB_ISNULL(this->get_dag())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag should not be nullptr", K(ret), KP(this->get_dag())); + } else if (OB_ISNULL(dag_net = this->get_dag()->get_dag_net())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag net should not be nullptr", K(ret), KP(dag_net)); + } else if (dag_net->is_cancel()) { ret = OB_CANCELED; - STORAGE_LOG(WARN, "task is cancelled", K(ret), K(*this)); + LOG_WARN("task is cancelled", K(ret), K(*this)); } else if (OB_FAIL(ObStorageHADagUtils::get_ls(user_data.transfer_ls_id_, dest_ls_handle))) { if (OB_LS_NOT_EXIST == ret) { ret = OB_SUCCESS; diff --git a/src/storage/high_availability/ob_ls_restore.cpp b/src/storage/high_availability/ob_ls_restore.cpp index fcb3c97dc..1225c448b 100644 --- a/src/storage/high_availability/ob_ls_restore.cpp +++ b/src/storage/high_availability/ob_ls_restore.cpp @@ -1353,10 +1353,17 @@ int ObSysTabletsRestoreTask::process() int ObSysTabletsRestoreTask::create_or_update_tablets_() { int ret = OB_SUCCESS; + ObIDagNet *dag_net = nullptr; if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("sys tablets restore task do not init", K(ret)); - } else if (OB_FAIL(ha_tablets_builder_.create_or_update_tablets())) { + } else if (OB_ISNULL(this->get_dag())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag should not be nullptr", K(ret), KP(this->get_dag())); + } else if (OB_ISNULL(dag_net = this->get_dag()->get_dag_net())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag net should not be nullptr", K(ret), KP(dag_net)); + } else if (OB_FAIL(ha_tablets_builder_.create_or_update_tablets(dag_net))) { LOG_WARN("failed to create or update tablets", K(ret), KPC(ctx_)); } return ret; @@ -1365,11 +1372,17 @@ int ObSysTabletsRestoreTask::create_or_update_tablets_() int ObSysTabletsRestoreTask::build_tablets_sstable_info_() { int ret = OB_SUCCESS; - + ObIDagNet *dag_net = nullptr; if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("sys tablets restore task do not init", K(ret), KPC(ctx_)); - } else if (OB_FAIL(ha_tablets_builder_.build_tablets_sstable_info())) { + } else if (OB_ISNULL(this->get_dag())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag should not be nullptr", K(ret), KP(this->get_dag())); + } else if (OB_ISNULL(dag_net = this->get_dag()->get_dag_net())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag net should not be nullptr", K(ret), KP(dag_net)); + } else if (OB_FAIL(ha_tablets_builder_.build_tablets_sstable_info(dag_net))) { LOG_WARN("failed to build tablets sstable info", K(ret), KPC(ctx_)); } return ret; diff --git a/src/storage/high_availability/ob_storage_ha_dag.cpp b/src/storage/high_availability/ob_storage_ha_dag.cpp index 01d5252ce..2a86eeb42 100644 --- a/src/storage/high_availability/ob_storage_ha_dag.cpp +++ b/src/storage/high_availability/ob_storage_ha_dag.cpp @@ -823,7 +823,45 @@ int ObStorageHATaskUtils::check_ddl_sstable_need_copy_( return ret; } +int ObStorageHACancelDagNetUtils::cancel_task(const share::ObLSID &ls_id, const share::ObTaskId &task_id) +{ + int ret = OB_SUCCESS; + ObLSHandle ls_handle; + if (!ls_id.is_valid() || !task_id.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(task_id), K(task_id)); + } else if (OB_FAIL(ObStorageHADagUtils::get_ls(ls_id, ls_handle))) { + LOG_WARN("failed to get ls", K(ret), K(ls_id)); + } else { + bool is_exist = false; + if (OB_FAIL(cancel_migration_task_(task_id, ls_handle, is_exist))) { + LOG_WARN("failed to cancel migration task.", K(ret), K(ls_id), K(task_id), K(ls_handle)); + } else if (is_exist) { + } else { + ret = OB_ENTRY_NOT_EXIST; + LOG_WARN("task is not exist", K(ret), K(ls_id), K(task_id)); + } + } + return ret; +} +int ObStorageHACancelDagNetUtils::cancel_migration_task_(const share::ObTaskId &task_id, + const ObLSHandle &ls_handle, bool &is_exist) +{ + int ret = OB_SUCCESS; + ObLS *ls = nullptr; + is_exist = false; + if (!task_id.is_valid() || !ls_handle.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(task_id), K(ls_handle)); + } else if (OB_ISNULL(ls = ls_handle.get_ls())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("log stream should not be nullptr", K(ret), KP(ls)); + } else if (OB_FAIL(ls->get_ls_migration_handler()->cancel_task(task_id, is_exist))) { + LOG_WARN("failed to cancel migration task", K(ret), K(task_id), K(ls_handle)); + } + return ret; +} } } diff --git a/src/storage/high_availability/ob_storage_ha_dag.h b/src/storage/high_availability/ob_storage_ha_dag.h index 7eb156184..75397dd3f 100644 --- a/src/storage/high_availability/ob_storage_ha_dag.h +++ b/src/storage/high_availability/ob_storage_ha_dag.h @@ -197,7 +197,13 @@ private: }; - +class ObStorageHACancelDagNetUtils +{ +public: + static int cancel_task(const share::ObLSID &ls_id, const share::ObTaskId &task_id); +private: + static int cancel_migration_task_(const share::ObTaskId &task_id, const ObLSHandle &ls_handle, bool &is_exist); +}; } } #endif diff --git a/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp b/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp index aa225c19c..578e84222 100644 --- a/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp +++ b/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp @@ -148,7 +148,7 @@ int ObStorageHATabletsBuilder::init(const ObStorageHATabletsBuilderParam ¶m) return ret; } -int ObStorageHATabletsBuilder::create_or_update_tablets() +int ObStorageHATabletsBuilder::create_or_update_tablets(ObIDagNet *dag_net) { int ret = OB_SUCCESS; ObLS *ls = nullptr; @@ -160,6 +160,9 @@ int ObStorageHATabletsBuilder::create_or_update_tablets() if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("storage ha tablets builder do not init", K(ret)); + } else if (OB_ISNULL(dag_net)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argumnet", K(ret), KP(dag_net)); } else if (OB_ISNULL(ls = param_.ls_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("log stream should not be NULL", K(ret), KP(ls), K(param_)); @@ -168,7 +171,13 @@ int ObStorageHATabletsBuilder::create_or_update_tablets() } else { while (OB_SUCC(ret)) { tablet_info.reset(); - if (OB_FAIL(reader->fetch_tablet_info(tablet_info))) { + if (OB_ISNULL(dag_net)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argumnet", K(ret), KP(dag_net)); + } else if (dag_net->is_cancel()) { + ret = OB_CANCELED; + LOG_WARN("task is cancelled", K(ret)); + } else if (OB_FAIL(reader->fetch_tablet_info(tablet_info))) { if (OB_ITER_END == ret) { ret = OB_SUCCESS; break; @@ -202,6 +211,7 @@ int ObStorageHATabletsBuilder::create_or_update_tablets() int ObStorageHATabletsBuilder::create_all_tablets( const bool need_check_tablet_limit, ObICopyLSViewInfoReader *reader, + ObIDagNet *dag_net, common::ObIArray &sys_tablet_id_list, common::ObIArray &data_tablet_id_list, CopyTabletSimpleInfoMap &simple_info_map) @@ -218,9 +228,9 @@ int ObStorageHATabletsBuilder::create_all_tablets( if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("storage ha tablets builder do not init", K(ret)); - } else if (OB_ISNULL(reader)) { + } else if (OB_ISNULL(reader) || OB_ISNULL(dag_net)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("create all tablets get invalid argument", K(ret), KP(reader)); + LOG_WARN("create all tablets get invalid argument", K(ret), KP(reader), KP(dag_net)); } else if (OB_ISNULL(ls = param_.ls_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("log stream should not be NULL", K(ret), KP(ls), K(param_)); @@ -231,7 +241,13 @@ int ObStorageHATabletsBuilder::create_all_tablets( tablet_info.reset(); tablet_simple_info.reset(); logic_tablet_id.reset(); - if (OB_FAIL(reader->get_next_tablet_info(tablet_info))) { + if (OB_ISNULL(dag_net)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag net should not be nullptr", K(ret), KP(dag_net)); + } else if (dag_net->is_cancel()) { + ret = OB_CANCELED; + LOG_WARN("task is cancelled", K(ret)); + } else if (OB_FAIL(reader->get_next_tablet_info(tablet_info))) { if (OB_ITER_END == ret) { ret = OB_SUCCESS; break; @@ -280,6 +296,7 @@ int ObStorageHATabletsBuilder::create_all_tablets( } int ObStorageHATabletsBuilder::create_all_tablets_with_4_1_rpc( + ObIDagNet *dag_net, CopyTabletSimpleInfoMap &simple_info_map, common::ObIArray &sys_tablet_id_list, common::ObIArray &data_tablet_id_list) @@ -298,6 +315,9 @@ int ObStorageHATabletsBuilder::create_all_tablets_with_4_1_rpc( if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("storage ha tablets builder do not init", K(ret)); + } else if (OB_ISNULL(dag_net)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("create all tablets get invalid argument", K(ret), KP(dag_net)); } else if (OB_ISNULL(ls = param_.ls_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("log stream should not be NULL", K(ret), KP(ls), K(param_)); @@ -307,7 +327,13 @@ int ObStorageHATabletsBuilder::create_all_tablets_with_4_1_rpc( while (OB_SUCC(ret)) { tablet_info.reset(); logic_tablet_id.reset(); - if (OB_FAIL(reader->fetch_tablet_info(tablet_info))) { + if (OB_ISNULL(dag_net)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag net should not be nullptr", K(ret), KP(dag_net)); + } else if (dag_net->is_cancel()) { + ret = OB_CANCELED; + LOG_WARN("task is cancelled", K(ret)); + } else if (OB_FAIL(reader->fetch_tablet_info(tablet_info))) { if (OB_ITER_END == ret) { ret = OB_SUCCESS; break; @@ -591,7 +617,7 @@ int ObStorageHATabletsBuilder::create_or_update_tablet_( return ret; } -int ObStorageHATabletsBuilder::build_tablets_sstable_info() +int ObStorageHATabletsBuilder::build_tablets_sstable_info(ObIDagNet *dag_net) { int ret = OB_SUCCESS; ObICopySSTableInfoReader *reader = nullptr; @@ -603,6 +629,9 @@ int ObStorageHATabletsBuilder::build_tablets_sstable_info() if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("storage ha tablets builder do not init", K(ret)); + } else if (OB_ISNULL(dag_net)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("build_tablets_sstable_info get invalid argument", K(ret), KP(dag_net)); } else if (OB_ISNULL(ls = param_.ls_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("log stream should not be NULL", K(ret), KP(ls)); @@ -617,8 +646,13 @@ int ObStorageHATabletsBuilder::build_tablets_sstable_info() while (OB_SUCC(ret)) { sstable_info.reset(); copy_header.reset(); - - if (OB_FAIL(reader->get_next_tablet_sstable_header(copy_header))) { + if (OB_ISNULL(dag_net)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag net should not be nullptr", K(ret), KP(dag_net)); + } else if (dag_net->is_cancel()) { + ret = OB_CANCELED; + LOG_WARN("task is cancelled", K(ret)); + } else if (OB_FAIL(reader->get_next_tablet_sstable_header(copy_header))) { if (OB_ITER_END == ret) { ret = OB_SUCCESS; break; diff --git a/src/storage/high_availability/ob_storage_ha_tablet_builder.h b/src/storage/high_availability/ob_storage_ha_tablet_builder.h index b057ba107..74b03739c 100644 --- a/src/storage/high_availability/ob_storage_ha_tablet_builder.h +++ b/src/storage/high_availability/ob_storage_ha_tablet_builder.h @@ -71,10 +71,11 @@ public: virtual ~ObStorageHATabletsBuilder(); int init(const ObStorageHATabletsBuilderParam ¶m); // Create all tablets with remote tablet meta. - int create_or_update_tablets(); + int create_or_update_tablets(ObIDagNet *dag_net); int create_all_tablets( const bool need_check_tablet_limit, ObICopyLSViewInfoReader *reader, + ObIDagNet *dag_net, common::ObIArray &sys_tablet_id_list, common::ObIArray &data_tablet_id_list, CopyTabletSimpleInfoMap &simple_info_map); @@ -83,8 +84,9 @@ public: // If that tablet meta identified uniquely by transfer sequence exists, replace and update the restore status to EMPTY. // Otherwise, just update it to UNDEFINED. int update_pending_tablets_with_remote(); - int build_tablets_sstable_info(); + int build_tablets_sstable_info(ObIDagNet *dag_net); int create_all_tablets_with_4_1_rpc( + ObIDagNet *dag_net, CopyTabletSimpleInfoMap &simple_info_map, common::ObIArray &sys_tablet_id_list, common::ObIArray &data_tablet_id_list); diff --git a/src/storage/high_availability/ob_tablet_group_restore.cpp b/src/storage/high_availability/ob_tablet_group_restore.cpp index c7b9d09bb..ed4fd80fe 100644 --- a/src/storage/high_availability/ob_tablet_group_restore.cpp +++ b/src/storage/high_availability/ob_tablet_group_restore.cpp @@ -1395,6 +1395,7 @@ int ObStartTabletGroupRestoreTask::generate_tablet_restore_dag_() int ObStartTabletGroupRestoreTask::create_tablets_sstable_() { int ret = OB_SUCCESS; + ObIDagNet *dag_net = nullptr; #ifdef ERRSIM if (ctx_->arg_.ls_id_.is_user_ls()) { @@ -1412,7 +1413,13 @@ int ObStartTabletGroupRestoreTask::create_tablets_sstable_() LOG_WARN("start tablet group restore task do not init", K(ret), KPC(ctx_)); } else if (ObTabletRestoreAction::is_restore_tablet_meta(ctx_->arg_.action_)) { //do nothing - } else if (OB_FAIL(ha_tablets_builder_.build_tablets_sstable_info())) { + } else if (OB_ISNULL(this->get_dag())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag should not be nullptr", K(ret), KP(this->get_dag())); + } else if (OB_ISNULL(dag_net = this->get_dag()->get_dag_net())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag net should not be nullptr", K(ret), KP(dag_net)); + } else if (OB_FAIL(ha_tablets_builder_.build_tablets_sstable_info(dag_net))) { LOG_WARN("failed to build tablets sstable info", K(ret), KPC(ctx_)); } return ret; @@ -2534,6 +2541,7 @@ int ObTabletRestoreTask::try_update_tablet_() ObLS *ls = nullptr; bool is_exist = false; ObCopyTabletStatus::STATUS status = ObCopyTabletStatus::MAX_STATUS; + ObIDagNet *dag_net = nullptr; if (!is_inited_) { ret = OB_NOT_INIT; @@ -2572,9 +2580,12 @@ int ObTabletRestoreTask::try_update_tablet_() } if (OB_FAIL(ret)) { - } else if (tablet_restore_ctx_->tablet_id_.is_ls_inner_tablet() && OB_FAIL(ha_tablets_builder.create_or_update_tablets())) { + } else if (OB_ISNULL(dag_net = dag->get_dag_net())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag net should not be nullptr", K(ret), KP(dag_net)); + } else if (tablet_restore_ctx_->tablet_id_.is_ls_inner_tablet() && OB_FAIL(ha_tablets_builder.create_or_update_tablets(dag_net))) { LOG_WARN("failed to create or update inner tablet", K(ret), KPC(tablet_restore_ctx_)); - } else if (OB_FAIL(ha_tablets_builder.build_tablets_sstable_info())) { + } else if (OB_FAIL(ha_tablets_builder.build_tablets_sstable_info(dag_net))) { LOG_WARN("failed to build tablets sstable info", K(ret), KPC(tablet_restore_ctx_)); } else if (OB_FAIL(tablet_restore_ctx_->ha_table_info_mgr_->check_tablet_table_info_exist( tablet_restore_ctx_->tablet_id_, is_exist))) { diff --git a/tools/deploy/mysql_test/r/mysql/information_schema.result b/tools/deploy/mysql_test/r/mysql/information_schema.result index de38b876c..3d35af953 100644 --- a/tools/deploy/mysql_test/r/mysql/information_schema.result +++ b/tools/deploy/mysql_test/r/mysql/information_schema.result @@ -320,6 +320,7 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | DBA_OB_LS_LOCATIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_LS_LOG_ARCHIVE_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_LS_REPLICA_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_LS_REPLICA_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_MAJOR_COMPACTION | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_OUTLINES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_OUTLINE_CONCURRENT_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | @@ -795,6 +796,7 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | __all_virtual_ls_meta_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_ls_recovery_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_ls_replica_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_replica_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_ls_replica_task_plan | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_ls_restore_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_ls_restore_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | @@ -1826,6 +1828,7 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | DBA_OB_LS_LOCATIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_LS_LOG_ARCHIVE_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_LS_REPLICA_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_LS_REPLICA_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_MAJOR_COMPACTION | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_OUTLINES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_OUTLINE_CONCURRENT_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | @@ -2301,6 +2304,7 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | __all_virtual_ls_meta_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_ls_recovery_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_ls_replica_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_replica_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_ls_replica_task_plan | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_ls_restore_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_ls_restore_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | @@ -2862,6 +2866,10 @@ select * from information_schema.statistics where table_schema in ('oceanbase', | def | oceanbase | __all_ls_replica_task | 0 | oceanbase | PRIMARY | 2 | ls_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_ls_replica_task | 0 | oceanbase | PRIMARY | 3 | task_type | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_ls_replica_task | 0 | oceanbase | PRIMARY | 4 | task_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_ls_replica_task_history | 0 | oceanbase | PRIMARY | 1 | tenant_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_ls_replica_task_history | 0 | oceanbase | PRIMARY | 2 | ls_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_ls_replica_task_history | 0 | oceanbase | PRIMARY | 3 | task_type | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_ls_replica_task_history | 0 | oceanbase | PRIMARY | 4 | task_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_ls_restore_history | 0 | oceanbase | PRIMARY | 1 | tenant_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_ls_restore_history | 0 | oceanbase | PRIMARY | 2 | job_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_ls_restore_history | 0 | oceanbase | PRIMARY | 3 | ls_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result index 7de9dd0b6..30434c887 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result @@ -4325,6 +4325,9 @@ SOURCE_REPLICA_SVR_IP varchar(46) YES NULL SOURCE_REPLICA_SVR_PORT bigint(20) YES NULL SOURCE_PAXOS_REPLICA_NUMBER bigint(20) YES NULL SOURCE_REPLICA_TYPE varchar(16) YES NULL +DATA_SOURCE_SVR_IP varchar(46) YES NULL +DATA_SOURCE_SVR_PORT bigint(20) YES NULL +IS_MANUAL varchar(6) NO TASK_EXEC_SVR_IP varchar(46) YES NULL TASK_EXEC_SVR_PORT bigint(20) YES NULL CREATE_TIME datetime NO NULL @@ -6021,6 +6024,35 @@ TSNAP_META_EXISTED varchar(3) NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_LS_SNAPSHOTS limit 1); cnt 1 +desc oceanbase.DBA_OB_LS_REPLICA_TASK_HISTORY; +Field Type Null Key Default Extra +LS_ID bigint(20) NO NULL +TASK_TYPE varchar(64) NO NULL +TASK_ID varchar(200) NO NULL +TASK_STATUS varchar(2048) YES NULL +PRIORITY varchar(5) NO +TARGET_REPLICA_SVR_IP varchar(46) YES NULL +TARGET_REPLICA_SVR_PORT bigint(20) YES NULL +TARGET_PAXOS_REPLICA_NUMBER bigint(20) YES NULL +TARGET_REPLICA_TYPE varchar(16) YES NULL +SOURCE_REPLICA_SVR_IP varchar(46) YES NULL +SOURCE_REPLICA_SVR_PORT bigint(20) YES NULL +SOURCE_PAXOS_REPLICA_NUMBER bigint(20) YES NULL +SOURCE_REPLICA_TYPE varchar(16) YES NULL +DATA_SOURCE_SVR_IP varchar(46) YES NULL +DATA_SOURCE_SVR_PORT bigint(20) YES NULL +IS_MANUAL varchar(6) NO +TASK_EXEC_SVR_IP varchar(46) YES NULL +TASK_EXEC_SVR_PORT bigint(20) YES NULL +CREATE_TIME datetime NO NULL +START_TIME datetime NO +MODIFY_TIME datetime NO NULL +FINISH_TIME datetime NO +EXECUTE_RESULT varchar(2048) YES NULL +COMMENT varchar(2048) YES NULL +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_OB_LS_REPLICA_TASK_HISTORY limit 1); +cnt +1 desc oceanbase.DBA_MVIEW_LOGS; Field Type Null Key Default Extra LOG_OWNER varchar(128) NO diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result index 0b6a55c22..9fe9f06f7 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result @@ -6032,6 +6032,9 @@ SOURCE_REPLICA_SVR_IP varchar(46) YES NULL SOURCE_REPLICA_SVR_PORT bigint(20) YES NULL SOURCE_PAXOS_REPLICA_NUMBER bigint(20) YES NULL SOURCE_REPLICA_TYPE varchar(16) YES NULL +DATA_SOURCE_SVR_IP varchar(46) YES NULL +DATA_SOURCE_SVR_PORT bigint(20) YES NULL +IS_MANUAL varchar(6) NO TASK_EXEC_SVR_IP varchar(46) YES NULL TASK_EXEC_SVR_PORT bigint(20) YES NULL CREATE_TIME datetime NO NULL @@ -6057,6 +6060,9 @@ SOURCE_REPLICA_SVR_IP varchar(46) YES NULL SOURCE_REPLICA_SVR_PORT bigint(20) YES NULL SOURCE_PAXOS_REPLICA_NUMBER bigint(20) YES NULL SOURCE_REPLICA_TYPE varchar(16) YES NULL +DATA_SOURCE_SVR_IP varchar(46) YES NULL +DATA_SOURCE_SVR_PORT bigint(20) YES NULL +IS_MANUAL varchar(6) NO TASK_EXEC_SVR_IP varchar(46) YES NULL TASK_EXEC_SVR_PORT bigint(20) YES NULL CREATE_TIME datetime NO NULL @@ -8618,6 +8624,65 @@ ERROR_MESSAGE varchar(512) YES NULL select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_OB_CLONE_HISTORY limit 1); cnt 1 +desc oceanbase.DBA_OB_LS_REPLICA_TASK_HISTORY; +Field Type Null Key Default Extra +LS_ID bigint(20) NO NULL +TASK_TYPE varchar(64) NO NULL +TASK_ID varchar(200) NO NULL +TASK_STATUS varchar(2048) YES NULL +PRIORITY varchar(5) NO +TARGET_REPLICA_SVR_IP varchar(46) YES NULL +TARGET_REPLICA_SVR_PORT bigint(20) YES NULL +TARGET_PAXOS_REPLICA_NUMBER bigint(20) YES NULL +TARGET_REPLICA_TYPE varchar(16) YES NULL +SOURCE_REPLICA_SVR_IP varchar(46) YES NULL +SOURCE_REPLICA_SVR_PORT bigint(20) YES NULL +SOURCE_PAXOS_REPLICA_NUMBER bigint(20) YES NULL +SOURCE_REPLICA_TYPE varchar(16) YES NULL +DATA_SOURCE_SVR_IP varchar(46) YES NULL +DATA_SOURCE_SVR_PORT bigint(20) YES NULL +IS_MANUAL varchar(6) NO +TASK_EXEC_SVR_IP varchar(46) YES NULL +TASK_EXEC_SVR_PORT bigint(20) YES NULL +CREATE_TIME datetime NO NULL +START_TIME datetime NO +MODIFY_TIME datetime NO NULL +FINISH_TIME datetime NO +EXECUTE_RESULT varchar(2048) YES NULL +COMMENT varchar(2048) YES NULL +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_OB_LS_REPLICA_TASK_HISTORY limit 1); +cnt +1 +desc oceanbase.CDB_OB_LS_REPLICA_TASK_HISTORY; +Field Type Null Key Default Extra +TENANT_ID bigint(20) NO NULL +LS_ID bigint(20) NO NULL +TASK_TYPE varchar(64) NO NULL +TASK_ID varchar(200) NO NULL +TASK_STATUS varchar(2048) YES NULL +PRIORITY varchar(5) NO +TARGET_REPLICA_SVR_IP varchar(46) YES NULL +TARGET_REPLICA_SVR_PORT bigint(20) YES NULL +TARGET_PAXOS_REPLICA_NUMBER bigint(20) YES NULL +TARGET_REPLICA_TYPE varchar(16) YES NULL +SOURCE_REPLICA_SVR_IP varchar(46) YES NULL +SOURCE_REPLICA_SVR_PORT bigint(20) YES NULL +SOURCE_PAXOS_REPLICA_NUMBER bigint(20) YES NULL +SOURCE_REPLICA_TYPE varchar(16) YES NULL +DATA_SOURCE_SVR_IP varchar(46) YES NULL +DATA_SOURCE_SVR_PORT bigint(20) YES NULL +IS_MANUAL varchar(6) NO +TASK_EXEC_SVR_IP varchar(46) YES NULL +TASK_EXEC_SVR_PORT bigint(20) YES NULL +CREATE_TIME datetime NO NULL +START_TIME datetime NO +MODIFY_TIME datetime NO NULL +FINISH_TIME datetime NO +EXECUTE_RESULT varchar(2048) YES NULL +COMMENT varchar(2048) YES NULL +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.CDB_OB_LS_REPLICA_TASK_HISTORY limit 1); +cnt +1 desc oceanbase.CDB_MVIEW_LOGS; Field Type Null Key Default Extra TENANT_ID bigint(20) NO NULL diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result index d9934d903..9ac59d3c2 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result @@ -4905,6 +4905,37 @@ IS_MANDATORY varchar(1024) NO select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from information_schema.ENABLED_ROLES; IF(count(*) >= 0, 1, 0) 1 +desc oceanbase.__all_virtual_ls_replica_task_history; +Field Type Null Key Default Extra +tenant_id bigint(20) NO PRI NULL +ls_id bigint(20) NO PRI NULL +task_type varchar(64) NO PRI NULL +task_id varchar(200) NO PRI NULL +gmt_create timestamp(6) NO NULL +gmt_modified timestamp(6) NO NULL +task_status varchar(2048) YES NULL +priority bigint(20) NO 1 +target_replica_svr_ip varchar(46) YES NULL +target_replica_svr_port bigint(20) YES NULL +target_paxos_replica_number bigint(20) YES NULL +target_replica_type varchar(16) YES NULL +source_replica_svr_ip varchar(46) YES NULL +source_replica_svr_port bigint(20) YES NULL +source_paxos_replica_number bigint(20) YES NULL +source_replica_type varchar(16) YES NULL +data_source_svr_ip varchar(46) YES NULL +data_source_svr_port bigint(20) YES NULL +is_manual tinyint(4) YES 0 +task_exec_svr_ip varchar(46) YES NULL +task_exec_svr_port bigint(20) YES NULL +generate_time timestamp(6) NO CURRENT_TIMESTAMP(6) +schedule_time timestamp(6) NO CURRENT_TIMESTAMP(6) +finish_time timestamp(6) NO CURRENT_TIMESTAMP(6) +execute_result varchar(2048) YES NULL +comment varchar(2048) YES NULL +select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_ls_replica_task_history; +IF(count(*) >= 0, 1, 0) +1 desc oceanbase.__all_virtual_session_ps_info; Field Type Null Key Default Extra svr_ip varchar(46) NO NULL diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result index d0c0d5c51..6f009a4f9 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result @@ -9487,6 +9487,37 @@ IS_MANDATORY varchar(1024) NO select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from information_schema.ENABLED_ROLES; IF(count(*) >= 0, 1, 0) 1 +desc oceanbase.__all_virtual_ls_replica_task_history; +Field Type Null Key Default Extra +tenant_id bigint(20) NO PRI NULL +ls_id bigint(20) NO PRI NULL +task_type varchar(64) NO PRI NULL +task_id varchar(200) NO PRI NULL +gmt_create timestamp(6) NO NULL +gmt_modified timestamp(6) NO NULL +task_status varchar(2048) YES NULL +priority bigint(20) NO 1 +target_replica_svr_ip varchar(46) YES NULL +target_replica_svr_port bigint(20) YES NULL +target_paxos_replica_number bigint(20) YES NULL +target_replica_type varchar(16) YES NULL +source_replica_svr_ip varchar(46) YES NULL +source_replica_svr_port bigint(20) YES NULL +source_paxos_replica_number bigint(20) YES NULL +source_replica_type varchar(16) YES NULL +data_source_svr_ip varchar(46) YES NULL +data_source_svr_port bigint(20) YES NULL +is_manual tinyint(4) YES 0 +task_exec_svr_ip varchar(46) YES NULL +task_exec_svr_port bigint(20) YES NULL +generate_time timestamp(6) NO CURRENT_TIMESTAMP(6) +schedule_time timestamp(6) NO CURRENT_TIMESTAMP(6) +finish_time timestamp(6) NO CURRENT_TIMESTAMP(6) +execute_result varchar(2048) YES NULL +comment varchar(2048) YES NULL +select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_ls_replica_task_history; +IF(count(*) >= 0, 1, 0) +1 desc oceanbase.__all_virtual_session_ps_info; Field Type Null Key Default Extra svr_ip varchar(46) NO NULL diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result index 2d1fcbd88..f7b93ef3d 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result @@ -293,6 +293,7 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr 505 __all_column_privilege 0 201001 1 506 __all_column_privilege_history 0 201001 1 507 __all_tenant_snapshot_ls_replica_history 0 201001 1 +508 __all_ls_replica_task_history 0 201001 1 512 __all_user_proxy_info 0 201001 1 513 __all_user_proxy_info_history 0 201001 1 514 __all_user_proxy_role_info 0 201001 1 @@ -742,6 +743,7 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr 12463 __all_virtual_column_privilege_history 2 201001 1 12464 __all_virtual_tenant_snapshot_ls_replica_history 2 201001 1 12466 ENABLED_ROLES 2 201002 1 +12467 __all_virtual_ls_replica_task_history 2 201001 1 12468 __all_virtual_session_ps_info 2 201001 1 12469 __all_virtual_tracepoint_info 2 201001 1 12473 __all_virtual_compatibility_control 2 201001 1 @@ -1150,6 +1152,8 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr 21517 GV$OB_LS_SNAPSHOTS 1 201001 1 21518 V$OB_LS_SNAPSHOTS 1 201001 1 21519 DBA_OB_CLONE_HISTORY 1 201001 1 +21523 DBA_OB_LS_REPLICA_TASK_HISTORY 1 201001 1 +21524 CDB_OB_LS_REPLICA_TASK_HISTORY 1 201001 1 21525 CDB_MVIEW_LOGS 1 201001 1 21526 DBA_MVIEW_LOGS 1 201001 1 21527 CDB_MVIEWS 1 201001 1 diff --git a/unittest/sql/parser/print_parser_tree.result b/unittest/sql/parser/print_parser_tree.result index f34396099..65a69f1b5 100644 --- a/unittest/sql/parser/print_parser_tree.result +++ b/unittest/sql/parser/print_parser_tree.result @@ -3709,3 +3709,202 @@ question_mask_size: 0 |--[0],[T_CANCEL_BALANCE_JOB], str_value_=[], value=[0] |--[0],[T_TENANT_NAME], str_value_=[], value=[9223372036854775807] |--[0],[T_VARCHAR], str_value_=[mysql], value=[9223372036854775807] + +************** Case 204 *************** +alter system add replica ls=100 server='100.88.107.212:5000' replica_type='F' data_source='100.88.107.212:5001' paxos_replica_num=3 tenant='mysql_tenant'; +question_mask_size: 0 + +|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] + |--[0],[T_ADD_LS_REPLICA], str_value_=[], value=[0] + |--[0],[T_LS], str_value_=[], value=[9223372036854775807] + |--[0],[T_INT], str_value_=[100], value=[100] + |--[1],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] + |--[2],[T_VARCHAR], str_value_=[F], value=[9223372036854775807] + |--[3],[T_VARCHAR], str_value_=["100.88.107.212":5001], value=[9223372036854775807] + |--[4],[T_INT], str_value_=[3], value=[3] + |--[5],[T_TENANT_NAME], str_value_=[], value=[9223372036854775807] + |--[0],[T_VARCHAR], str_value_=[mysql_tenant], value=[9223372036854775807] + +************** Case 205 *************** +alter system add replica ls=100 server='100.88.107.212:5000' replica_type='F' data_source='100.88.107.212:5001' paxos_replica_num=3; +question_mask_size: 0 + +|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] + |--[0],[T_ADD_LS_REPLICA], str_value_=[], value=[0] + |--[0],[T_LS], str_value_=[], value=[9223372036854775807] + |--[0],[T_INT], str_value_=[100], value=[100] + |--[1],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] + |--[2],[T_VARCHAR], str_value_=[F], value=[9223372036854775807] + |--[3],[T_VARCHAR], str_value_=["100.88.107.212":5001], value=[9223372036854775807] + |--[4],[T_INT], str_value_=[3], value=[3] + +************** Case 206 *************** +alter system add replica ls=100 server='100.88.107.212:5000' replica_type='F' data_source='100.88.107.212:5001'; +question_mask_size: 0 + +|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] + |--[0],[T_ADD_LS_REPLICA], str_value_=[], value=[0] + |--[0],[T_LS], str_value_=[], value=[9223372036854775807] + |--[0],[T_INT], str_value_=[100], value=[100] + |--[1],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] + |--[2],[T_VARCHAR], str_value_=[F], value=[9223372036854775807] + |--[3],[T_VARCHAR], str_value_=["100.88.107.212":5001], value=[9223372036854775807] + +************** Case 207 *************** +alter system add replica ls=100 server='100.88.107.212:5000' replica_type='F'; +question_mask_size: 0 + +|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] + |--[0],[T_ADD_LS_REPLICA], str_value_=[], value=[0] + |--[0],[T_LS], str_value_=[], value=[9223372036854775807] + |--[0],[T_INT], str_value_=[100], value=[100] + |--[1],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] + |--[2],[T_VARCHAR], str_value_=[F], value=[9223372036854775807] + +************** Case 208 *************** +alter system remove replica ls=100 server='100.88.107.212:5000' paxos_replica_num=3 tenant='mysql_tenant'; +question_mask_size: 0 + +|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] + |--[0],[T_REMOVE_LS_REPLICA], str_value_=[], value=[0] + |--[0],[T_LS], str_value_=[], value=[9223372036854775807] + |--[0],[T_INT], str_value_=[100], value=[100] + |--[1],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] + |--[2],[T_INT], str_value_=[3], value=[3] + |--[3],[T_TENANT_NAME], str_value_=[], value=[9223372036854775807] + |--[0],[T_VARCHAR], str_value_=[mysql_tenant], value=[9223372036854775807] + +************** Case 209 *************** +alter system remove replica ls=100 server='100.88.107.212:5000' paxos_replica_num=3; +question_mask_size: 0 + +|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] + |--[0],[T_REMOVE_LS_REPLICA], str_value_=[], value=[0] + |--[0],[T_LS], str_value_=[], value=[9223372036854775807] + |--[0],[T_INT], str_value_=[100], value=[100] + |--[1],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] + |--[2],[T_INT], str_value_=[3], value=[3] + +************** Case 210 *************** +alter system remove replica ls=100 server='100.88.107.212:5000'; +question_mask_size: 0 + +|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] + |--[0],[T_REMOVE_LS_REPLICA], str_value_=[], value=[0] + |--[0],[T_LS], str_value_=[], value=[9223372036854775807] + |--[0],[T_INT], str_value_=[100], value=[100] + |--[1],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] + +************** Case 211 *************** +alter system modify replica ls=100 server='100.88.107.212:5000' replica_type='F' paxos_replica_num=3 tenant='mysql_tenant'; +question_mask_size: 0 + +|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] + |--[0],[T_MODIFY_LS_REPLICA_TYPE], str_value_=[], value=[0] + |--[0],[T_LS], str_value_=[], value=[9223372036854775807] + |--[0],[T_INT], str_value_=[100], value=[100] + |--[1],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] + |--[2],[T_VARCHAR], str_value_=[F], value=[9223372036854775807] + |--[3],[T_INT], str_value_=[3], value=[3] + |--[4],[T_TENANT_NAME], str_value_=[], value=[9223372036854775807] + |--[0],[T_VARCHAR], str_value_=[mysql_tenant], value=[9223372036854775807] + +************** Case 212 *************** +alter system modify replica ls=100 server='100.88.107.212:5000' replica_type='F' paxos_replica_num=3; +question_mask_size: 0 + +|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] + |--[0],[T_MODIFY_LS_REPLICA_TYPE], str_value_=[], value=[0] + |--[0],[T_LS], str_value_=[], value=[9223372036854775807] + |--[0],[T_INT], str_value_=[100], value=[100] + |--[1],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] + |--[2],[T_VARCHAR], str_value_=[F], value=[9223372036854775807] + |--[3],[T_INT], str_value_=[3], value=[3] + +************** Case 213 *************** +alter system modify replica ls=100 server='100.88.107.212:5000' replica_type='F'; +question_mask_size: 0 + +|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] + |--[0],[T_MODIFY_LS_REPLICA_TYPE], str_value_=[], value=[0] + |--[0],[T_LS], str_value_=[], value=[9223372036854775807] + |--[0],[T_INT], str_value_=[100], value=[100] + |--[1],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] + |--[2],[T_VARCHAR], str_value_=[F], value=[9223372036854775807] + +************** Case 214 *************** +alter system modify ls=100 paxos_replica_num=3 tenant='mysql_tenant'; +question_mask_size: 0 + +|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] + |--[0],[T_MODIFY_LS_PAXOS_REPLICA_NUM], str_value_=[], value=[0] + |--[0],[T_LS], str_value_=[], value=[9223372036854775807] + |--[0],[T_INT], str_value_=[100], value=[100] + |--[1],[T_INT], str_value_=[3], value=[3] + |--[2],[T_TENANT_NAME], str_value_=[], value=[9223372036854775807] + |--[0],[T_VARCHAR], str_value_=[mysql_tenant], value=[9223372036854775807] + +************** Case 215 *************** +alter system modify ls=100 paxos_replica_num=3; +question_mask_size: 0 + +|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] + |--[0],[T_MODIFY_LS_PAXOS_REPLICA_NUM], str_value_=[], value=[0] + |--[0],[T_LS], str_value_=[], value=[9223372036854775807] + |--[0],[T_INT], str_value_=[100], value=[100] + |--[1],[T_INT], str_value_=[3], value=[3] + +************** Case 216 *************** +alter system migrate replica ls=100 source='100.88.107.212:5000' destination='100.88.107.212:5000' data_source='100.88.107.212:5001' tenant='mysql_tenant'; +question_mask_size: 0 + +|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] + |--[0],[T_MIGRATE_LS_REPLICA], str_value_=[], value=[0] + |--[0],[T_LS], str_value_=[], value=[9223372036854775807] + |--[0],[T_INT], str_value_=[100], value=[100] + |--[1],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] + |--[2],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] + |--[3],[T_VARCHAR], str_value_=["100.88.107.212":5001], value=[9223372036854775807] + |--[4],[T_TENANT_NAME], str_value_=[], value=[9223372036854775807] + |--[0],[T_VARCHAR], str_value_=[mysql_tenant], value=[9223372036854775807] + +************** Case 217 *************** +alter system migrate replica ls=100 source='100.88.107.212:5000' destination='100.88.107.212:5000' data_source='100.88.107.212:5001'; +question_mask_size: 0 + +|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] + |--[0],[T_MIGRATE_LS_REPLICA], str_value_=[], value=[0] + |--[0],[T_LS], str_value_=[], value=[9223372036854775807] + |--[0],[T_INT], str_value_=[100], value=[100] + |--[1],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] + |--[2],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] + |--[3],[T_VARCHAR], str_value_=["100.88.107.212":5001], value=[9223372036854775807] + +************** Case 218 *************** +alter system migrate replica ls=100 source='100.88.107.212:5000' destination='100.88.107.212:5000'; +question_mask_size: 0 + +|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] + |--[0],[T_MIGRATE_LS_REPLICA], str_value_=[], value=[0] + |--[0],[T_LS], str_value_=[], value=[9223372036854775807] + |--[0],[T_INT], str_value_=[100], value=[100] + |--[1],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] + |--[2],[T_VARCHAR], str_value_=["100.88.107.212":5000], value=[9223372036854775807] + +************** Case 219 *************** +alter system cancel replica task task_id = 'xxx' tenant='mysql_tenant'; +question_mask_size: 0 + +|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] + |--[0],[T_CANCEL_LS_REPLICA_TASK], str_value_=[], value=[0] + |--[0],[T_VARCHAR], str_value_=[xxx], value=[9223372036854775807] + |--[1],[T_TENANT_NAME], str_value_=[], value=[9223372036854775807] + |--[0],[T_VARCHAR], str_value_=[mysql_tenant], value=[9223372036854775807] + +************** Case 220 *************** +alter system cancel replica task task_id = 'xxx'; +question_mask_size: 0 + +|--[0],[T_STMT_LIST], str_value_=[], value=[9223372036854775807] + |--[0],[T_CANCEL_LS_REPLICA_TASK], str_value_=[], value=[0] + |--[0],[T_VARCHAR], str_value_=[xxx], value=[9223372036854775807] diff --git a/unittest/sql/parser/test_parser.result b/unittest/sql/parser/test_parser.result index 62c36ff44..569acc869 100644 --- a/unittest/sql/parser/test_parser.result +++ b/unittest/sql/parser/test_parser.result @@ -24165,3 +24165,865 @@ question_mask_size: 0 } ] } +************** Case 204 *************** +alter system add replica ls=100 server='100.88.107.212:5000' replica_type='F' data_source='100.88.107.212:5001' paxos_replica_num=3 tenant='mysql_tenant'; +question_mask_size: 0 +{ + "type":"T_STMT_LIST", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_ADD_LS_REPLICA", + "int_val":0, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_LS", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_INT", + "int_val":100, + "str_len":3, + "str_val":"100" + } + ] + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":19, + "str_val":"100.88.107.212:5000" + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":1, + "str_val":"F" + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":19, + "str_val":"100.88.107.212:5001" + }, + { + "type":"T_INT", + "int_val":3, + "str_len":1, + "str_val":"3" + }, + { + "type":"T_TENANT_NAME", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":12, + "str_val":"mysql_tenant" + } + ] + } + ] + } + ] +} +************** Case 205 *************** +alter system add replica ls=100 server='100.88.107.212:5000' replica_type='F' data_source='100.88.107.212:5001' paxos_replica_num=3; +question_mask_size: 0 +{ + "type":"T_STMT_LIST", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_ADD_LS_REPLICA", + "int_val":0, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_LS", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_INT", + "int_val":100, + "str_len":3, + "str_val":"100" + } + ] + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":19, + "str_val":"100.88.107.212:5000" + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":1, + "str_val":"F" + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":19, + "str_val":"100.88.107.212:5001" + }, + { + "type":"T_INT", + "int_val":3, + "str_len":1, + "str_val":"3" + }, + { } + ] + } + ] +} +************** Case 206 *************** +alter system add replica ls=100 server='100.88.107.212:5000' replica_type='F' data_source='100.88.107.212:5001'; +question_mask_size: 0 +{ + "type":"T_STMT_LIST", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_ADD_LS_REPLICA", + "int_val":0, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_LS", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_INT", + "int_val":100, + "str_len":3, + "str_val":"100" + } + ] + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":19, + "str_val":"100.88.107.212:5000" + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":1, + "str_val":"F" + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":19, + "str_val":"100.88.107.212:5001" + }, + { }, + { } + ] + } + ] +} +************** Case 207 *************** +alter system add replica ls=100 server='100.88.107.212:5000' replica_type='F'; +question_mask_size: 0 +{ + "type":"T_STMT_LIST", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_ADD_LS_REPLICA", + "int_val":0, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_LS", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_INT", + "int_val":100, + "str_len":3, + "str_val":"100" + } + ] + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":19, + "str_val":"100.88.107.212:5000" + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":1, + "str_val":"F" + }, + { }, + { }, + { } + ] + } + ] +} +************** Case 208 *************** +alter system remove replica ls=100 server='100.88.107.212:5000' paxos_replica_num=3 tenant='mysql_tenant'; +question_mask_size: 0 +{ + "type":"T_STMT_LIST", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_REMOVE_LS_REPLICA", + "int_val":0, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_LS", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_INT", + "int_val":100, + "str_len":3, + "str_val":"100" + } + ] + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":19, + "str_val":"100.88.107.212:5000" + }, + { + "type":"T_INT", + "int_val":3, + "str_len":1, + "str_val":"3" + }, + { + "type":"T_TENANT_NAME", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":12, + "str_val":"mysql_tenant" + } + ] + } + ] + } + ] +} +************** Case 209 *************** +alter system remove replica ls=100 server='100.88.107.212:5000' paxos_replica_num=3; +question_mask_size: 0 +{ + "type":"T_STMT_LIST", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_REMOVE_LS_REPLICA", + "int_val":0, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_LS", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_INT", + "int_val":100, + "str_len":3, + "str_val":"100" + } + ] + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":19, + "str_val":"100.88.107.212:5000" + }, + { + "type":"T_INT", + "int_val":3, + "str_len":1, + "str_val":"3" + }, + { } + ] + } + ] +} +************** Case 210 *************** +alter system remove replica ls=100 server='100.88.107.212:5000'; +question_mask_size: 0 +{ + "type":"T_STMT_LIST", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_REMOVE_LS_REPLICA", + "int_val":0, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_LS", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_INT", + "int_val":100, + "str_len":3, + "str_val":"100" + } + ] + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":19, + "str_val":"100.88.107.212:5000" + }, + { }, + { } + ] + } + ] +} +************** Case 211 *************** +alter system modify replica ls=100 server='100.88.107.212:5000' replica_type='F' paxos_replica_num=3 tenant='mysql_tenant'; +question_mask_size: 0 +{ + "type":"T_STMT_LIST", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_MODIFY_LS_REPLICA_TYPE", + "int_val":0, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_LS", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_INT", + "int_val":100, + "str_len":3, + "str_val":"100" + } + ] + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":19, + "str_val":"100.88.107.212:5000" + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":1, + "str_val":"F" + }, + { + "type":"T_INT", + "int_val":3, + "str_len":1, + "str_val":"3" + }, + { + "type":"T_TENANT_NAME", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":12, + "str_val":"mysql_tenant" + } + ] + } + ] + } + ] +} +************** Case 212 *************** +alter system modify replica ls=100 server='100.88.107.212:5000' replica_type='F' paxos_replica_num=3; +question_mask_size: 0 +{ + "type":"T_STMT_LIST", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_MODIFY_LS_REPLICA_TYPE", + "int_val":0, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_LS", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_INT", + "int_val":100, + "str_len":3, + "str_val":"100" + } + ] + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":19, + "str_val":"100.88.107.212:5000" + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":1, + "str_val":"F" + }, + { + "type":"T_INT", + "int_val":3, + "str_len":1, + "str_val":"3" + }, + { } + ] + } + ] +} +************** Case 213 *************** +alter system modify replica ls=100 server='100.88.107.212:5000' replica_type='F'; +question_mask_size: 0 +{ + "type":"T_STMT_LIST", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_MODIFY_LS_REPLICA_TYPE", + "int_val":0, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_LS", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_INT", + "int_val":100, + "str_len":3, + "str_val":"100" + } + ] + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":19, + "str_val":"100.88.107.212:5000" + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":1, + "str_val":"F" + }, + { }, + { } + ] + } + ] +} +************** Case 214 *************** +alter system modify ls=100 paxos_replica_num=3 tenant='mysql_tenant'; +question_mask_size: 0 +{ + "type":"T_STMT_LIST", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_MODIFY_LS_PAXOS_REPLICA_NUM", + "int_val":0, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_LS", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_INT", + "int_val":100, + "str_len":3, + "str_val":"100" + } + ] + }, + { + "type":"T_INT", + "int_val":3, + "str_len":1, + "str_val":"3" + }, + { + "type":"T_TENANT_NAME", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":12, + "str_val":"mysql_tenant" + } + ] + } + ] + } + ] +} +************** Case 215 *************** +alter system modify ls=100 paxos_replica_num=3; +question_mask_size: 0 +{ + "type":"T_STMT_LIST", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_MODIFY_LS_PAXOS_REPLICA_NUM", + "int_val":0, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_LS", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_INT", + "int_val":100, + "str_len":3, + "str_val":"100" + } + ] + }, + { + "type":"T_INT", + "int_val":3, + "str_len":1, + "str_val":"3" + }, + { } + ] + } + ] +} +************** Case 216 *************** +alter system migrate replica ls=100 source='100.88.107.212:5000' destination='100.88.107.212:5000' data_source='100.88.107.212:5001' tenant='mysql_tenant'; +question_mask_size: 0 +{ + "type":"T_STMT_LIST", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_MIGRATE_LS_REPLICA", + "int_val":0, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_LS", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_INT", + "int_val":100, + "str_len":3, + "str_val":"100" + } + ] + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":19, + "str_val":"100.88.107.212:5000" + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":19, + "str_val":"100.88.107.212:5000" + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":19, + "str_val":"100.88.107.212:5001" + }, + { + "type":"T_TENANT_NAME", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":12, + "str_val":"mysql_tenant" + } + ] + } + ] + } + ] +} +************** Case 217 *************** +alter system migrate replica ls=100 source='100.88.107.212:5000' destination='100.88.107.212:5000' data_source='100.88.107.212:5001'; +question_mask_size: 0 +{ + "type":"T_STMT_LIST", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_MIGRATE_LS_REPLICA", + "int_val":0, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_LS", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_INT", + "int_val":100, + "str_len":3, + "str_val":"100" + } + ] + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":19, + "str_val":"100.88.107.212:5000" + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":19, + "str_val":"100.88.107.212:5000" + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":19, + "str_val":"100.88.107.212:5001" + }, + { } + ] + } + ] +} +************** Case 218 *************** +alter system migrate replica ls=100 source='100.88.107.212:5000' destination='100.88.107.212:5000'; +question_mask_size: 0 +{ + "type":"T_STMT_LIST", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_MIGRATE_LS_REPLICA", + "int_val":0, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_LS", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_INT", + "int_val":100, + "str_len":3, + "str_val":"100" + } + ] + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":19, + "str_val":"100.88.107.212:5000" + }, + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":19, + "str_val":"100.88.107.212:5000" + }, + { }, + { } + ] + } + ] +} +************** Case 219 *************** +alter system cancel replica task task_id = 'xxx' tenant='mysql_tenant'; +question_mask_size: 0 +{ + "type":"T_STMT_LIST", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_CANCEL_LS_REPLICA_TASK", + "int_val":0, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":3, + "str_val":"xxx" + }, + { + "type":"T_TENANT_NAME", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":12, + "str_val":"mysql_tenant" + } + ] + } + ] + } + ] +} +************** Case 220 *************** +alter system cancel replica task task_id = 'xxx'; +question_mask_size: 0 +{ + "type":"T_STMT_LIST", + "int_val":9223372036854775807, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_CANCEL_LS_REPLICA_TASK", + "int_val":0, + "str_len":0, + "str_val":"", + "children": [ + { + "type":"T_VARCHAR", + "int_val":9223372036854775807, + "str_len":3, + "str_val":"xxx" + }, + { } + ] + } + ] +} diff --git a/unittest/sql/parser/test_parser.test b/unittest/sql/parser/test_parser.test index e05feca20..c3add7f5a 100644 --- a/unittest/sql/parser/test_parser.test +++ b/unittest/sql/parser/test_parser.test @@ -313,3 +313,28 @@ alter system cancel transfer partition ALL; alter system cancel transfer partition ALL tenant 'mysql'; alter system cancel balance job; alter system cancel balance job tenant 'mysql'; + + + +alter system add replica ls=100 server='100.88.107.212:5000' replica_type='F' data_source='100.88.107.212:5001' paxos_replica_num=3 tenant='mysql_tenant'; +alter system add replica ls=100 server='100.88.107.212:5000' replica_type='F' data_source='100.88.107.212:5001' paxos_replica_num=3; +alter system add replica ls=100 server='100.88.107.212:5000' replica_type='F' data_source='100.88.107.212:5001'; +alter system add replica ls=100 server='100.88.107.212:5000' replica_type='F'; + +alter system remove replica ls=100 server='100.88.107.212:5000' paxos_replica_num=3 tenant='mysql_tenant'; +alter system remove replica ls=100 server='100.88.107.212:5000' paxos_replica_num=3; +alter system remove replica ls=100 server='100.88.107.212:5000'; + +alter system modify replica ls=100 server='100.88.107.212:5000' replica_type='F' paxos_replica_num=3 tenant='mysql_tenant'; +alter system modify replica ls=100 server='100.88.107.212:5000' replica_type='F' paxos_replica_num=3; +alter system modify replica ls=100 server='100.88.107.212:5000' replica_type='F'; + +alter system modify ls=100 paxos_replica_num=3 tenant='mysql_tenant'; +alter system modify ls=100 paxos_replica_num=3; + +alter system migrate replica ls=100 source='100.88.107.212:5000' destination='100.88.107.212:5000' data_source='100.88.107.212:5001' tenant='mysql_tenant'; +alter system migrate replica ls=100 source='100.88.107.212:5000' destination='100.88.107.212:5000' data_source='100.88.107.212:5001'; +alter system migrate replica ls=100 source='100.88.107.212:5000' destination='100.88.107.212:5000'; + +alter system cancel replica task task_id = 'xxx' tenant='mysql_tenant'; +alter system cancel replica task task_id = 'xxx'; From 18bf5eb5cbf8eec9f01add198eb50be17acb0bef Mon Sep 17 00:00:00 2001 From: obdev Date: Mon, 12 Aug 2024 10:28:46 +0000 Subject: [PATCH 014/249] rpc packet code placeholder. --- deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h | 1 + src/share/ob_ddl_common.h | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h index 2719bf455..09dbf33e8 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h @@ -876,6 +876,7 @@ PCODE_DEF(OB_REMOTE_WRITE_DDL_INC_COMMIT_LOG, 0x967) //PCODE_DEF(OB_PREPARE_TABLET_SPLIT_TASK_RANGES, 0x96C) PCODE_DEF(OB_BATCH_GET_TABLET_BINDING, 0x96D) //PCODE_DEF(OB_BATCH_UPGRADE_TABLE_SCHEMA, 0x96E) +//PCODE_DEF(OB_SPLIT_TABLET_DATA_START_REQUEST, 0x96F) // Depedency Detector PCODE_DEF(OB_DETECTOR_LCL_MESSAGE, 0x9F0) diff --git a/src/share/ob_ddl_common.h b/src/share/ob_ddl_common.h index d5c01315c..e8a27fa23 100644 --- a/src/share/ob_ddl_common.h +++ b/src/share/ob_ddl_common.h @@ -181,6 +181,7 @@ enum ObDDLTaskStatus { WAIT_VID_ROWKEY_TABLE_COMPLEMENT = 34, REBUILD_SCHEMA = 35, SWITCH_INDEX_NAME = 36, + WRITE_SPLIT_START_LOG = 37, FAIL = 99, SUCCESS = 100 }; @@ -324,6 +325,9 @@ static const char* ddl_task_status_to_str(const ObDDLTaskStatus &task_status) { case ObDDLTaskStatus::SWITCH_INDEX_NAME: str = "SWITCH_INDEX_NAME"; break; + case ObDDLTaskStatus::WRITE_SPLIT_START_LOG: + str = "WRITE_SPLIT_START_LOG"; + break; case ObDDLTaskStatus::FAIL: str = "FAIL"; break; From ef2c80ec84d094eebd2dd42968a6fbe7600880b2 Mon Sep 17 00:00:00 2001 From: LoLolobster <949673574@qq.com> Date: Mon, 12 Aug 2024 11:18:08 +0000 Subject: [PATCH 015/249] add item_type T_RESTORE_WITH_CONFIG_LIST,T_STS_CREDENTIAL on master --- src/objit/include/objit/common/ob_item_type.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/objit/include/objit/common/ob_item_type.h b/src/objit/include/objit/common/ob_item_type.h index 019eeda7f..fe8c733c0 100755 --- a/src/objit/include/objit/common/ob_item_type.h +++ b/src/objit/include/objit/common/ob_item_type.h @@ -2584,6 +2584,10 @@ typedef enum ObItemType // Erase micro cache T_FLUSH_SS_MICRO_CACHE = 4731, + + //restore sts + T_RESTORE_WITH_CONFIG_LIST = 4732, + T_STS_CREDENTIAL = 4733, T_MAX //Attention: add a new type before T_MAX } ObItemType; From a3d74f559a0688fe01ee5dab7a1f3154505d13f9 Mon Sep 17 00:00:00 2001 From: JLY2015 <1623359870@qq.com> Date: Mon, 12 Aug 2024 11:47:45 +0000 Subject: [PATCH 016/249] [vector index] modify relative variable name --- src/share/ob_rpc_struct.cpp | 4 ++-- src/share/ob_rpc_struct.h | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/share/ob_rpc_struct.cpp b/src/share/ob_rpc_struct.cpp index 7ed0b2753..7687b87e5 100644 --- a/src/share/ob_rpc_struct.cpp +++ b/src/share/ob_rpc_struct.cpp @@ -3352,7 +3352,7 @@ DEF_TO_STRING(ObDropIndexArg) { K_(is_in_recyclebin), K_(is_hidden), K_(is_inner), - K_(is_rebuild_drop)); + K_(is_vec_inner_drop)); J_OBJ_END(); return pos; } @@ -3367,7 +3367,7 @@ OB_SERIALIZE_MEMBER((ObDropIndexArg, ObIndexArg), is_in_recyclebin_, is_hidden_, is_inner_, - is_rebuild_drop_); + is_vec_inner_drop_); OB_SERIALIZE_MEMBER(ObDropIndexRes, tenant_id_, index_table_id_, schema_version_, task_id_); diff --git a/src/share/ob_rpc_struct.h b/src/share/ob_rpc_struct.h index ad8be3f22..d3d919dd8 100644 --- a/src/share/ob_rpc_struct.h +++ b/src/share/ob_rpc_struct.h @@ -1389,7 +1389,7 @@ public: is_hidden_ = false; is_in_recyclebin_ = false; is_inner_ = false; - is_rebuild_drop_ = false; + is_vec_inner_drop_ = false; } virtual ~ObDropIndexArg() {} void reset() @@ -1400,7 +1400,7 @@ public: is_hidden_ = false; is_in_recyclebin_ = false; is_inner_ = false; - is_rebuild_drop_ = false; + is_vec_inner_drop_ = false; } bool is_valid() const { return ObIndexArg::is_valid(); } uint64_t index_table_id_; @@ -1408,7 +1408,7 @@ public: bool is_hidden_; bool is_in_recyclebin_; bool is_inner_; - bool is_rebuild_drop_; + bool is_vec_inner_drop_; DECLARE_VIRTUAL_TO_STRING; }; From 9125f39f5691f3143891be288309a7ea41a3f352 Mon Sep 17 00:00:00 2001 From: zhjc1124 Date: Mon, 12 Aug 2024 11:53:36 +0000 Subject: [PATCH 017/249] check if req is null before free --- src/share/io/ob_io_manager.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/share/io/ob_io_manager.cpp b/src/share/io/ob_io_manager.cpp index e383459a7..a181a67c7 100644 --- a/src/share/io/ob_io_manager.cpp +++ b/src/share/io/ob_io_manager.cpp @@ -896,7 +896,9 @@ int ObTenantIOManager::inner_aio(const ObIOInfo &info, ObIOHandle &handle) LOG_WARN("schedule request failed", K(ret), KPC(req)); } if (OB_FAIL(ret)) { - req->free(); + if (OB_NOT_NULL(req)) { + req->free(); + } handle.reset(); } return ret; From 1f1d547b7b14e56aa4ba3773f690daafaaaa7bee Mon Sep 17 00:00:00 2001 From: bit-dance <2634358021@qq.com> Date: Mon, 12 Aug 2024 11:59:34 +0000 Subject: [PATCH 018/249] [to #2024072000103866815]Fix case when oracle compatible problems --- src/sql/resolver/expr/ob_raw_expr_util.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/sql/resolver/expr/ob_raw_expr_util.cpp b/src/sql/resolver/expr/ob_raw_expr_util.cpp index f73ee721b..2a6e90e19 100644 --- a/src/sql/resolver/expr/ob_raw_expr_util.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_util.cpp @@ -4122,8 +4122,10 @@ int ObRawExprUtils::try_add_cast_expr_above(ObRawExprFactory *expr_factory, if (T_FUN_SYS_CAST == expr.get_expr_type() && expr.has_flag(IS_OP_OPERAND_IMPLICIT_CAST) && !ignore_dup_cast_error - && !((src_type.is_user_defined_sql_type()|| src_type.is_collection_sql_type()) - && (dst_type.is_character_type() || dst_type.is_null()))) { + && !(((src_type.is_user_defined_sql_type()|| src_type.is_collection_sql_type()) + && (dst_type.is_character_type() || dst_type.is_null())) + || (src_type.is_character_type() && dst_type.is_character_type()))) + { // cases like: select xmltype(var)||xmltype(var) as "res1" from t1 t; // xmltype is a lp constructor, an implicit cast is added to cast PL xmltype to SQL xmltype // when deduce concat, another cast is needed to cast SQL xmltype to string From d21904fa68eb47a02d7f6690b3c3fe390fd475c9 Mon Sep 17 00:00:00 2001 From: obdev Date: Mon, 12 Aug 2024 12:17:16 +0000 Subject: [PATCH 019/249] fix: fix core when gathering palf stat --- src/logservice/palf/log_block_handler.cpp | 20 ++++++++++++-------- src/logservice/palf/log_reader.cpp | 12 ++++++------ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/logservice/palf/log_block_handler.cpp b/src/logservice/palf/log_block_handler.cpp index 6df820b4a..64bf80d27 100644 --- a/src/logservice/palf/log_block_handler.cpp +++ b/src/logservice/palf/log_block_handler.cpp @@ -401,15 +401,19 @@ int LogBlockHandler::inner_write_once_(const offset_t offset, K(offset), K(buf_len)); } else { dio_aligned_buf_.truncate_buf(); - total_write_size_ += buf_len; - total_write_size_after_dio_ += aligned_buf_len; - count_++; + const int64_t total_write_size = ATOMIC_AAF(&total_write_size_, buf_len); + const int64_t total_write_size_after_dio = ATOMIC_AAF(&total_write_size_after_dio_, aligned_buf_len); + const int64_t count = ATOMIC_AAF(&count_, 1); + const int64_t ob_pwrite_used_ts = ATOMIC_LOAD(&ob_pwrite_used_ts_); if (palf_reach_time_interval(PALF_IO_STAT_PRINT_INTERVAL_US, trace_time_)) { - const int64_t each_pwrite_cost = ob_pwrite_used_ts_ / count_; + const int64_t each_pwrite_cost = ob_pwrite_used_ts / count; PALF_LOG(INFO, "[PALF STAT WRITE LOG INFO TO DISK]", K(ret), K(offset), KPC(this), K(aligned_buf_len), - K(aligned_block_offset), K(buf_len), K(total_write_size_), - K(total_write_size_after_dio_), K_(ob_pwrite_used_ts), K_(count), K(each_pwrite_cost)); - total_write_size_ = total_write_size_after_dio_ = count_ = ob_pwrite_used_ts_ = 0; + K(aligned_block_offset), K(buf_len), K(total_write_size), + K(total_write_size_after_dio), K(ob_pwrite_used_ts), K(count), K(each_pwrite_cost)); + ATOMIC_STORE(&total_write_size_, 0); + ATOMIC_STORE(&total_write_size_after_dio_, 0); + ATOMIC_STORE(&count_, 0); + ATOMIC_STORE(&ob_pwrite_used_ts_, 0); } } return ret; @@ -460,7 +464,7 @@ int LogBlockHandler::inner_write_impl_(const int fd, const char *buf, const int6 EVENT_TENANT_INC(ObStatEventIds::PALF_WRITE_IO_COUNT, MTL_ID()); EVENT_ADD(ObStatEventIds::PALF_WRITE_SIZE, count); EVENT_ADD(ObStatEventIds::PALF_WRITE_TIME, cost_ts); - ob_pwrite_used_ts_ += cost_ts; + ATOMIC_AAF(&ob_pwrite_used_ts_, cost_ts); return ret; } } // end of logservice diff --git a/src/logservice/palf/log_reader.cpp b/src/logservice/palf/log_reader.cpp index 860bc1087..55da2402a 100644 --- a/src/logservice/palf/log_reader.cpp +++ b/src/logservice/palf/log_reader.cpp @@ -117,13 +117,13 @@ int LogReader::pread(const block_id_t block_id, EVENT_TENANT_INC(ObStatEventIds::PALF_READ_IO_COUNT_FROM_DISK, MTL_ID()); EVENT_ADD(ObStatEventIds::PALF_READ_SIZE_FROM_DISK, out_read_size); EVENT_ADD(ObStatEventIds::PALF_READ_TIME_FROM_DISK, cost_ts); - ATOMIC_INC(&accum_read_io_count_); - ATOMIC_AAF(&accum_read_log_size_, out_read_size); - ATOMIC_AAF(&accum_read_cost_ts_, cost_ts); + const int64_t accum_read_io_count = ATOMIC_AAF(&accum_read_io_count_, 1); + const int64_t accum_read_log_size = ATOMIC_AAF(&accum_read_log_size_, out_read_size); + const int64_t accum_read_cost_ts = ATOMIC_AAF(&accum_read_cost_ts_, cost_ts); if (palf_reach_time_interval(PALF_IO_STAT_PRINT_INTERVAL_US, last_accum_read_statistic_time_)) { - const int64_t avg_pread_cost = accum_read_cost_ts_ / accum_read_io_count_; - PALF_LOG(INFO, "[PALF STAT READ LOG INFO FROM DISK]", K_(accum_read_io_count), - K_(accum_read_log_size), K(avg_pread_cost)); + const int64_t avg_pread_cost = accum_read_cost_ts / accum_read_io_count; + PALF_LOG(INFO, "[PALF STAT READ LOG INFO FROM DISK]", K(accum_read_io_count), + K(accum_read_log_size), K(avg_pread_cost)); ATOMIC_STORE(&accum_read_io_count_, 0); ATOMIC_STORE(&accum_read_log_size_, 0); ATOMIC_STORE(&accum_read_cost_ts_, 0); From 11f38cff7c59024451e275dc285b5369842221f7 Mon Sep 17 00:00:00 2001 From: LoLolobster <949673574@qq.com> Date: Mon, 12 Aug 2024 14:59:02 +0000 Subject: [PATCH 020/249] fix object storage checksum --- deps/oblib/src/lib/restore/ob_storage_cos_base.cpp | 5 +++++ deps/oblib/src/lib/restore/ob_storage_info.cpp | 6 +++--- deps/oblib/src/lib/restore/ob_storage_oss_base.cpp | 5 +++++ deps/oblib/src/lib/restore/ob_storage_s3_base.cpp | 5 +++++ deps/oblib/src/lib/utility/ob_tracepoint_def.h | 1 + deps/oblib/unittest/lib/restore/test_storage_info.cpp | 8 ++++---- unittest/share/backup/test_backup_struct.cpp | 8 ++++---- 7 files changed, 27 insertions(+), 11 deletions(-) diff --git a/deps/oblib/src/lib/restore/ob_storage_cos_base.cpp b/deps/oblib/src/lib/restore/ob_storage_cos_base.cpp index 37d203e7c..20ce6d87f 100644 --- a/deps/oblib/src/lib/restore/ob_storage_cos_base.cpp +++ b/deps/oblib/src/lib/restore/ob_storage_cos_base.cpp @@ -551,6 +551,11 @@ int ObStorageCosBase::open( } else if (OB_FAIL(handle_.build_bucket_and_object_name(uri))) { OB_LOG(WARN, "failed to build bucket and object name", K(ret), K(uri)); } +#ifdef ERRSIM + if (OB_NOT_NULL(storage_info) && (OB_SUCCESS != EventTable::EN_ENABLE_LOG_OBJECT_STORAGE_CHECKSUM_TYPE)) { + OB_LOG(ERROR, "errsim backup io with checksum type", "checksum_type", storage_info->get_checksum_type_str()); + } +#endif return ret; } diff --git a/deps/oblib/src/lib/restore/ob_storage_info.cpp b/deps/oblib/src/lib/restore/ob_storage_info.cpp index d15d877f6..591c55412 100644 --- a/deps/oblib/src/lib/restore/ob_storage_info.cpp +++ b/deps/oblib/src/lib/restore/ob_storage_info.cpp @@ -221,6 +221,8 @@ int ObObjectStorageInfo::parse_storage_info_(const char *storage_info, bool &has const char *checksum_type_str = token + strlen(CHECKSUM_TYPE); if (OB_FAIL(set_checksum_type_(checksum_type_str))) { OB_LOG(WARN, "fail to set checksum type", K(ret), K(checksum_type_str)); + } else if (OB_FAIL(set_storage_info_field_(token, extension_, sizeof(extension_)))) { + LOG_WARN("fail to set checksum type into extension", K(ret), K(token)); } } else { } @@ -346,9 +348,7 @@ int ObObjectStorageInfo::get_storage_info_str(char *storage_info, const int64_t } else if (OB_STORAGE_FILE != device_type_) { if (OB_FAIL(get_access_key_(key, sizeof(key)))) { LOG_WARN("failed to get access key", K(ret)); - } else if (OB_FAIL(databuff_printf(storage_info, info_len, "%s&%s&%s&%s%s", - endpoint_, access_id_, key, - CHECKSUM_TYPE, get_checksum_type_str()))) { + } else if (OB_FAIL(databuff_printf(storage_info, info_len, "%s&%s&%s", endpoint_, access_id_, key))) { LOG_WARN("failed to set storage info", K(ret), K(info_len)); } } diff --git a/deps/oblib/src/lib/restore/ob_storage_oss_base.cpp b/deps/oblib/src/lib/restore/ob_storage_oss_base.cpp index edc3ea571..af0202085 100644 --- a/deps/oblib/src/lib/restore/ob_storage_oss_base.cpp +++ b/deps/oblib/src/lib/restore/ob_storage_oss_base.cpp @@ -452,6 +452,11 @@ int ObStorageOssBase::init_with_storage_info(common::ObObjectStorageInfo *storag OB_LOG(WARN, "aos pool or oss option is NULL", K(aos_pool_), K(oss_option_)); } else { checksum_type_ = storage_info->get_checksum_type(); +#ifdef ERRSIM + if (OB_NOT_NULL(storage_info) && (OB_SUCCESS != EventTable::EN_ENABLE_LOG_OBJECT_STORAGE_CHECKSUM_TYPE)) { + OB_LOG(ERROR, "errsim backup io with checksum type", "checksum_type", storage_info->get_checksum_type_str()); + } +#endif if (OB_UNLIKELY(!is_oss_supported_checksum(checksum_type_))) { ret = OB_CHECKSUM_TYPE_NOT_SUPPORTED; OB_LOG(WARN, "that checksum algorithm is not supported for oss", K(ret), K_(checksum_type)); diff --git a/deps/oblib/src/lib/restore/ob_storage_s3_base.cpp b/deps/oblib/src/lib/restore/ob_storage_s3_base.cpp index 6dde6308f..04f6678c5 100644 --- a/deps/oblib/src/lib/restore/ob_storage_s3_base.cpp +++ b/deps/oblib/src/lib/restore/ob_storage_s3_base.cpp @@ -992,6 +992,11 @@ int ObStorageS3Base::open(const ObString &uri, ObObjectStorageInfo *storage_info OB_LOG(WARN, "faied to get s3 client", K(ret)); } else { checksum_type_ = storage_info->get_checksum_type(); +#ifdef ERRSIM + if (OB_NOT_NULL(storage_info) && (OB_SUCCESS != EventTable::EN_ENABLE_LOG_OBJECT_STORAGE_CHECKSUM_TYPE)) { + OB_LOG(ERROR, "errsim backup io with checksum type", "checksum_type", storage_info->get_checksum_type_str()); + } +#endif if (OB_UNLIKELY(!is_s3_supported_checksum(checksum_type_))) { ret = OB_CHECKSUM_TYPE_NOT_SUPPORTED; OB_LOG(WARN, "that checksum algorithm is not supported for s3", K(ret), K_(checksum_type)); diff --git a/deps/oblib/src/lib/utility/ob_tracepoint_def.h b/deps/oblib/src/lib/utility/ob_tracepoint_def.h index 395030d68..6dd6c2767 100644 --- a/deps/oblib/src/lib/utility/ob_tracepoint_def.h +++ b/deps/oblib/src/lib/utility/ob_tracepoint_def.h @@ -495,6 +495,7 @@ GLOBAL_ERRSIM_POINT_DEF(1113, EN_RESTORE_TABLET_TASK_FAILED, ""); GLOBAL_ERRSIM_POINT_DEF(1114, EN_INSERT_USER_RECOVER_JOB_FAILED, ""); GLOBAL_ERRSIM_POINT_DEF(1115, EN_INSERT_AUX_TENANT_RESTORE_JOB_FAILED, ""); GLOBAL_ERRSIM_POINT_DEF(1116, EN_RESTORE_CREATE_LS_FAILED, ""); +GLOBAL_ERRSIM_POINT_DEF(1117, EN_ENABLE_LOG_OBJECT_STORAGE_CHECKSUM_TYPE, ""); // END OF STORAGE HA - 1101 - 2000 // sql parameterization 1170-1180 diff --git a/deps/oblib/unittest/lib/restore/test_storage_info.cpp b/deps/oblib/unittest/lib/restore/test_storage_info.cpp index 497adb02d..085cc5285 100644 --- a/deps/oblib/unittest/lib/restore/test_storage_info.cpp +++ b/deps/oblib/unittest/lib/restore/test_storage_info.cpp @@ -111,7 +111,7 @@ TEST(ObObjectStorageInfo, cos) storage_info = "host=xxx.com&access_id=111&access_key=222"; ASSERT_EQ(OB_INVALID_BACKUP_DEST, info1.set(uri, storage_info)); - storage_info = "host=xxx.com&access_id=111&access_key=222&checksum_type=md5&appid=333"; + storage_info = "host=xxx.com&access_id=111&access_key=222&appid=333&checksum_type=md5"; ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info)); char buf[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 }; @@ -153,15 +153,15 @@ TEST(ObObjectStorageInfo, s3) ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info)); info1.reset(); - storage_info = "host=xxx.com&access_id=111&access_key=222&checksum_type=md5&s3_region=333"; + storage_info = "host=xxx.com&access_id=111&access_key=222&s3_region=333&checksum_type=md5"; ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info)); - ASSERT_EQ(0, ::strcmp("s3_region=333", info1.extension_)); + ASSERT_EQ(0, ::strcmp("s3_region=333&checksum_type=md5", info1.extension_)); char buf[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 }; ASSERT_EQ(OB_SUCCESS, info1.get_storage_info_str(buf, sizeof(buf))); ASSERT_STREQ(storage_info, buf); - storage_info = "host=xxx.com&access_id=111&access_key=222&checksum_type=md5&s3_region=333&delete_mode=delete"; + storage_info = "host=xxx.com&access_id=111&access_key=222&s3_region=333&delete_mode=delete&checksum_type=md5"; info1.reset(); ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info)); ASSERT_EQ(OB_SUCCESS, info1.get_storage_info_str(buf, sizeof(buf))); diff --git a/unittest/share/backup/test_backup_struct.cpp b/unittest/share/backup/test_backup_struct.cpp index f4d63ef7e..9fe239981 100644 --- a/unittest/share/backup/test_backup_struct.cpp +++ b/unittest/share/backup/test_backup_struct.cpp @@ -98,7 +98,7 @@ TEST(ObBackupDest, oss) EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().set_root_key(OB_SYS_TENANT_ID, obrpc::RootKeyType::DEFAULT, ObString())); ASSERT_EQ(OB_SUCCESS, dest.get_backup_dest_str(backup_dest_str, sizeof(backup_dest_str))); - ASSERT_EQ(0, strcmp(backup_dest_str, "oss://backup_dir?host=xxx.com&access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F&checksum_type=md5&delete_mode=tagging")); + ASSERT_EQ(0, strcmp(backup_dest_str, "oss://backup_dir?host=xxx.com&access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F&delete_mode=tagging")); ASSERT_EQ(OB_SUCCESS, dest.get_backup_path_str(backup_path_str, sizeof(backup_path_str))); ASSERT_EQ(0, strcmp(backup_path_str, "oss://backup_dir?host=xxx.com")); ASSERT_TRUE(dest.is_root_path_equal(dest1)); @@ -139,7 +139,7 @@ TEST(ObBackupDest, oss_encrypt) char backup_dest_str[OB_MAX_BACKUP_DEST_LENGTH] = { 0 }; char backup_path_str[OB_MAX_BACKUP_DEST_LENGTH] = { 0 }; ASSERT_EQ(OB_SUCCESS, dest.get_backup_dest_str(backup_dest_str, sizeof(backup_dest_str))); - ASSERT_EQ(0, strcmp(backup_dest_str, "oss://backup_dir?host=xxx.com&access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F&checksum_type=md5")); + ASSERT_EQ(0, strcmp(backup_dest_str, "oss://backup_dir?host=xxx.com&access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F")); ASSERT_EQ(OB_SUCCESS, dest.get_backup_path_str(backup_path_str, sizeof(backup_path_str))); ASSERT_EQ(0, strcmp(backup_path_str, "oss://backup_dir?host=xxx.com")); @@ -168,7 +168,7 @@ TEST(ObBackupDest, cos) EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().set_root_key(OB_SYS_TENANT_ID, obrpc::RootKeyType::DEFAULT, ObString())); ASSERT_EQ(OB_SUCCESS, dest.get_backup_dest_str(backup_dest_str, sizeof(backup_dest_str))); - ASSERT_EQ(0, strcmp(backup_dest_str, "cos://backup_dir?host=xxx.com&access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F&checksum_type=md5&delete_mode=tagging&appid=333")); + ASSERT_EQ(0, strcmp(backup_dest_str, "cos://backup_dir?host=xxx.com&access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F&delete_mode=tagging&appid=333")); ASSERT_EQ(OB_SUCCESS, dest.get_backup_path_str(backup_path_str, sizeof(backup_path_str))); ASSERT_EQ(0, strcmp(backup_path_str, "cos://backup_dir?host=xxx.com")); ASSERT_TRUE(dest.is_root_path_equal(dest1)); @@ -209,7 +209,7 @@ TEST(ObBackupDest, cos_encrypt) char backup_dest_str[OB_MAX_BACKUP_DEST_LENGTH] = { 0 }; char backup_path_str[OB_MAX_BACKUP_DEST_LENGTH] = { 0 }; ASSERT_EQ(OB_SUCCESS, dest.get_backup_dest_str(backup_dest_str, sizeof(backup_dest_str))); - ASSERT_EQ(0, strcmp(backup_dest_str, "cos://backup_dir?host=xxx.com&access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F&checksum_type=md5&appid=333")); + ASSERT_EQ(0, strcmp(backup_dest_str, "cos://backup_dir?host=xxx.com&access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F&appid=333")); ASSERT_EQ(OB_SUCCESS, dest.get_backup_path_str(backup_path_str, sizeof(backup_path_str))); ASSERT_EQ(0, strcmp(backup_path_str, "cos://backup_dir?host=xxx.com")); From 3d3db4c67f0d99d66908f937efefdd09f3908ef6 Mon Sep 17 00:00:00 2001 From: Handora Date: Mon, 12 Aug 2024 15:38:53 +0000 Subject: [PATCH 021/249] order the data checkpoint and memtable mgr lock order --- deps/oblib/src/lib/lock/ob_qsync_lock.cpp | 18 +++++++++++ deps/oblib/src/lib/lock/ob_qsync_lock.h | 1 + src/storage/memtable/ob_memtable.cpp | 8 +++-- src/storage/ob_i_memtable_mgr.h | 24 +++++++++++++- src/storage/ob_i_tablet_memtable.cpp | 13 ++++---- src/storage/ob_i_tablet_memtable.h | 2 +- src/storage/tablet/ob_tablet_memtable_mgr.cpp | 32 ++++++++++++++++++- src/storage/tablet/ob_tablet_memtable_mgr.h | 2 ++ 8 files changed, 88 insertions(+), 12 deletions(-) diff --git a/deps/oblib/src/lib/lock/ob_qsync_lock.cpp b/deps/oblib/src/lib/lock/ob_qsync_lock.cpp index 465bb2853..5672164d2 100644 --- a/deps/oblib/src/lib/lock/ob_qsync_lock.cpp +++ b/deps/oblib/src/lib/lock/ob_qsync_lock.cpp @@ -86,5 +86,23 @@ int ObQSyncLock::try_rdlock() } return ret; } + +int ObQSyncLock::try_wrlock() +{ + int ret = OB_SUCCESS; + if (!ATOMIC_BCAS(&write_flag_, 0, 1)) { + ret = OB_EAGAIN; + } else { + bool sync_success = false; + for (int64_t i = 0; !sync_success && i < TRY_SYNC_COUNT; i++) { + sync_success = qsync_.try_sync(); + } + if (!sync_success) { + ATOMIC_STORE(&write_flag_, 0); + ret = OB_EAGAIN; + } + } + return ret; +} } } diff --git a/deps/oblib/src/lib/lock/ob_qsync_lock.h b/deps/oblib/src/lib/lock/ob_qsync_lock.h index df5bae1fb..e0c9559fd 100644 --- a/deps/oblib/src/lib/lock/ob_qsync_lock.h +++ b/deps/oblib/src/lib/lock/ob_qsync_lock.h @@ -32,6 +32,7 @@ public: void rdunlock(); int wrlock(); void wrunlock(); + int try_wrlock(); int try_rdlock(); private: static const int64_t TRY_SYNC_COUNT = 16; diff --git a/src/storage/memtable/ob_memtable.cpp b/src/storage/memtable/ob_memtable.cpp index dcfe66a06..89c25d234 100644 --- a/src/storage/memtable/ob_memtable.cpp +++ b/src/storage/memtable/ob_memtable.cpp @@ -1914,11 +1914,15 @@ bool ObMemtable::ready_for_flush_() TRANS_LOG(WARN, "fail to get current right boundary", K(ret)); } if ((bool_ret = (current_right_boundary >= get_max_end_scn()))) { + int tmp_ret = OB_SUCCESS; resolve_right_boundary(); if (!get_resolved_active_memtable_left_boundary()) { - resolve_left_boundary_for_active_memtable_(); + if (OB_TMP_FAIL(resolve_left_boundary_for_active_memtable_())) { + TRANS_LOG(WARN, "fail to resolve left boundary for active memtable", + K(tmp_ret), KPC(this)); + } } - bool_ret = (is_empty() || get_resolved_active_memtable_left_boundary()); + bool_ret = get_resolved_active_memtable_left_boundary(); } if (bool_ret) { set_freeze_state(TabletMemtableFreezeState::READY_FOR_FLUSH); diff --git a/src/storage/ob_i_memtable_mgr.h b/src/storage/ob_i_memtable_mgr.h index bc5ee1b22..b3ee0ffb3 100644 --- a/src/storage/ob_i_memtable_mgr.h +++ b/src/storage/ob_i_memtable_mgr.h @@ -96,8 +96,30 @@ public: if (!is_valid()) { ret = OB_ERR_UNEXPECTED; STORAGE_LOG(ERROR, "unexpected lock or lock_type", K(ret), K_(lock_type), KP_(lock)); + } else if (lock_type_ == OB_QSYNC_LOCK) { + ret = static_cast(lock_)->try_wrlock(); } else if (lock_type_ == LockType::OB_SPIN_RWLOCK) { - ret = static_cast(lock_)->try_wrlock(); + ret = static_cast(lock_)->try_wrlock() + ? OB_SUCCESS : OB_EAGAIN; + } else { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(ERROR, "unexpected lock_type", K(ret), K_(lock_type)); + } + return ret; + } + + int try_rdlock() + { + int ret = OB_SUCCESS; + + if (!is_valid()) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(ERROR, "unexpected lock or lock_type", K(ret), K_(lock_type), KP_(lock)); + } else if (lock_type_ == OB_QSYNC_LOCK) { + ret = static_cast(lock_)->try_rdlock(); + } else if (lock_type_ == LockType::OB_SPIN_RWLOCK) { + ret = static_cast(lock_)->try_rdlock() + ? OB_SUCCESS : OB_EAGAIN; } else { ret = OB_ERR_UNEXPECTED; STORAGE_LOG(ERROR, "unexpected lock_type", K(ret), K_(lock_type)); diff --git a/src/storage/ob_i_tablet_memtable.cpp b/src/storage/ob_i_tablet_memtable.cpp index 1e31ed79c..9e55f582e 100644 --- a/src/storage/ob_i_tablet_memtable.cpp +++ b/src/storage/ob_i_tablet_memtable.cpp @@ -91,20 +91,19 @@ void ObITabletMemtable::unset_logging_blocked_for_active_memtable_() } } -void ObITabletMemtable::resolve_left_boundary_for_active_memtable_() +int ObITabletMemtable::resolve_left_boundary_for_active_memtable_() { int ret = OB_SUCCESS; storage::ObTabletMemtableMgr *memtable_mgr = get_memtable_mgr(); const SCN new_start_scn = MAX(get_end_scn(), get_migration_clog_checkpoint_scn()); if (OB_NOT_NULL(memtable_mgr)) { - do { - if (OB_FAIL(memtable_mgr->resolve_left_boundary_for_active_memtable(this, new_start_scn))) { - TRANS_LOG(ERROR, "fail to set start log ts for active memtable", K(ret), K(ls_id_), KPC(this)); - ob_usleep(100); - } - } while (OB_FAIL(ret)); + if (OB_FAIL(memtable_mgr->resolve_left_boundary_for_active_memtable(this, new_start_scn))) { + TRANS_LOG(WARN, "fail to resolve left boundary for active memtable", K(ret), K(ls_id_), KPC(this)); + } } + + return ret; } int ObITabletMemtable::get_ls_current_right_boundary_(SCN ¤t_right_boundary) diff --git a/src/storage/ob_i_tablet_memtable.h b/src/storage/ob_i_tablet_memtable.h index 26d455d51..06c4a7f80 100644 --- a/src/storage/ob_i_tablet_memtable.h +++ b/src/storage/ob_i_tablet_memtable.h @@ -417,7 +417,7 @@ protected: // ************* memtable flag inner operator ************* protected: - void resolve_left_boundary_for_active_memtable_(); + int resolve_left_boundary_for_active_memtable_(); int get_ls_current_right_boundary_(share::SCN ¤t_right_boundary); int set_memtable_mgr_(ObTabletMemtableMgr *mgr); int64_t inc_unsubmitted_cnt_(); diff --git a/src/storage/tablet/ob_tablet_memtable_mgr.cpp b/src/storage/tablet/ob_tablet_memtable_mgr.cpp index 74feb26a4..6041e6d3e 100644 --- a/src/storage/tablet/ob_tablet_memtable_mgr.cpp +++ b/src/storage/tablet/ob_tablet_memtable_mgr.cpp @@ -501,6 +501,36 @@ int ObTabletMemtableMgr::get_active_memtable(ObTableHandleV2 &handle) const return ret; } +int ObTabletMemtableMgr::try_get_active_memtable(ObTableHandleV2 &handle, + const int64_t timeout) +{ + int ret = OB_SUCCESS; + const int64_t start_ts = ObTimeUtility::current_time(); + + handle.reset(); + + while (OB_SUCC(ret)) { + if (ObTimeUtility::current_time() - start_ts > timeout) { + ret = OB_EAGAIN; + } else if (OB_SUCCESS == lock_.try_rdlock()) { + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", K(ret), K_(is_inited)); + } else if (OB_FAIL(get_active_memtable_(handle))) { + if (OB_ENTRY_NOT_EXIST != ret) { + LOG_WARN("fail to get active memtable", K(ret)); + } + } + lock_.rdunlock(); + break; + } else { + ob_usleep(10 * 1000); + } + } + + return ret; +} + int ObTabletMemtableMgr::get_active_memtable_(ObTableHandleV2 &handle) const { int ret = OB_SUCCESS; @@ -585,7 +615,7 @@ int ObTabletMemtableMgr::resolve_left_boundary_for_active_memtable(ObITabletMemt if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not inited", K(ret), K_(is_inited)); - } else if (OB_FAIL(get_active_memtable(handle))) { + } else if (OB_FAIL(try_get_active_memtable(handle, 1_s/*timeout*/))) { if (OB_ENTRY_NOT_EXIST != ret) { LOG_WARN("fail to get active memtable", K(ret)); } diff --git a/src/storage/tablet/ob_tablet_memtable_mgr.h b/src/storage/tablet/ob_tablet_memtable_mgr.h index 2368120c8..2c06fdf86 100644 --- a/src/storage/tablet/ob_tablet_memtable_mgr.h +++ b/src/storage/tablet/ob_tablet_memtable_mgr.h @@ -58,6 +58,8 @@ public: int freeze_direct_load_memtable(ObITabletMemtable *tablet_memtable); int get_direct_load_memtables_for_write(ObTableHdlArray &handles); + int try_get_active_memtable(ObTableHandleV2 &handle, const int64_t timeout); + public: // derived from ObIMemtableMgr virtual int init(const common::ObTabletID &tablet_id, const share::ObLSID &ls_id, From 5c56b88b5b37127891f09ab2ba304a8de6593b3e Mon Sep 17 00:00:00 2001 From: XIAO-HOU <372060054@qq.com> Date: Mon, 12 Aug 2024 15:44:42 +0000 Subject: [PATCH 022/249] Add tp to force group by pushdown to storage layer --- src/sql/optimizer/ob_log_plan.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/sql/optimizer/ob_log_plan.cpp b/src/sql/optimizer/ob_log_plan.cpp index 710fe9195..cd17f38b9 100644 --- a/src/sql/optimizer/ob_log_plan.cpp +++ b/src/sql/optimizer/ob_log_plan.cpp @@ -91,6 +91,9 @@ using share::schema::ObColumnSchemaV2; using share::schema::ObSchemaGetterGuard; #include "sql/optimizer/ob_join_property.map" + +ERRSIM_POINT_DEF(EN_FORCE_GBY_PUSHDOWN_STORAGE, "force pushdown group by to storage layer"); + static const char *ExplainColumnName[] = { "ID", @@ -5933,7 +5936,7 @@ int ObLogPlan::check_storage_groupby_pushdown(const ObIArray can_push = false; } else if (group_exprs.count() != 1) { can_push = false; - } else if (aggrs.count() > 5) { + } else if (OB_LIKELY(!EN_FORCE_GBY_PUSHDOWN_STORAGE) && aggrs.count() > 5) { can_push = false; } else if (OB_ISNULL(groupby_column = group_exprs.at(0))) { ret = OB_ERR_UNEXPECTED; @@ -5994,6 +5997,9 @@ int ObLogPlan::check_table_columns_can_storage_pushdown(const uint64_t tenant_id OB_UNLIKELY(!pushdown_groupby_columns.at(0)->is_column_ref_expr())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret)); + } else if (OB_UNLIKELY(EN_FORCE_GBY_PUSHDOWN_STORAGE)) { + can_push = true; + LOG_TRACE("force pushdown group by to storage layer", K(ret), K(can_push)); } else if (OB_FALSE_IT(column = static_cast(pushdown_groupby_columns.at(0)))) { } else if (!ObColumnStatParam::is_valid_opt_col_type(column->get_data_type())) { can_push = false; From 6c7d149173565bbb0384608e018afcec06f56cbb Mon Sep 17 00:00:00 2001 From: wxhwang Date: Mon, 12 Aug 2024 15:50:35 +0000 Subject: [PATCH 023/249] [CP] let copy task cannot retry --- src/storage/high_availability/ob_physical_copy_task.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/storage/high_availability/ob_physical_copy_task.h b/src/storage/high_availability/ob_physical_copy_task.h index 035e525ba..2d5106c74 100644 --- a/src/storage/high_availability/ob_physical_copy_task.h +++ b/src/storage/high_availability/ob_physical_copy_task.h @@ -146,7 +146,8 @@ private: int record_server_event_(); private: - static const int64_t MAX_RETRY_TIMES = 3; + // For rebuilder can not retry, define MAX_RETRY_TIMES as 1. + static const int64_t MAX_RETRY_TIMES = 1; static const int64_t OB_FETCH_MAJOR_BLOCK_RETRY_INTERVAL = 1 * 1000 * 1000L;// 1s bool is_inited_; ObPhysicalCopyCtx *copy_ctx_; From f2cc96fbeb3eca41d6039ee09b370ad552b34346 Mon Sep 17 00:00:00 2001 From: HaHaJeff Date: Tue, 13 Aug 2024 06:02:43 +0000 Subject: [PATCH 024/249] fixed create arbration instance failed because concurrent create --- ...st_ob_simple_arb_server_single_replica.cpp | 69 ++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/mittest/logservice/test_ob_simple_arb_server_single_replica.cpp b/mittest/logservice/test_ob_simple_arb_server_single_replica.cpp index 5685bb266..9cd1cc39d 100755 --- a/mittest/logservice/test_ob_simple_arb_server_single_replica.cpp +++ b/mittest/logservice/test_ob_simple_arb_server_single_replica.cpp @@ -13,7 +13,7 @@ #define private public #include "env/ob_simple_log_cluster_env.h" #undef private - +#include const std::string TEST_NAME = "single_arb_server"; using namespace oceanbase::common; @@ -327,6 +327,73 @@ TEST_F(TestObSimpleMutilArbServer, restart_arb) EXPECT_EQ(OB_SUCCESS, restart_server(0)); } +TEST_F(TestObSimpleMutilArbServer, multi_thread) +{ + SET_CASE_LOG_FILE(TEST_NAME, "restart_arb"); + OB_LOGGER.set_log_level("TRACE"); + ObISimpleLogServer *iserver = get_cluster()[0]; + EXPECT_EQ(true, iserver->is_arb_server()); + ObSimpleArbServer *arb_server = dynamic_cast(iserver); + palflite::PalfEnvLiteMgr *palf_env_mgr = &arb_server->palf_env_mgr_; + int64_t cluster_id = 100; + arbserver::GCMsgEpoch epoch = arbserver::GCMsgEpoch(1, 1); + + // test add tenant without cluster, generate placeholder + EXPECT_EQ(OB_SUCCESS, palf_env_mgr->create_palf_env_lite(palflite::PalfEnvKey(cluster_id, 1))); + EXPECT_TRUE(palf_env_mgr->is_cluster_placeholder_exists(cluster_id)); + + std::vector ls_ids = {1001, 1002, 1003, 1004, 1005, 1006, 1007}; + int64_t create_success_count = 0; + auto create_func = [&]() { + for (auto ls_id : ls_ids) { + int ret = palf_env_mgr->create_arbitration_instance( + palflite::PalfEnvKey(cluster_id, 1), arb_server->self_, 1001, + ObTenantRole(ObTenantRole::PRIMARY_TENANT)); + if (OB_SUCCESS == ret) { + ATOMIC_INC(&create_success_count); + } + if (OB_SUCCESS != ret) { + ASSERT_EQ(false, true); + } else { + } + } + }; + int64_t remove_success_count = 0; + auto remove_func = [&] () { + for (auto ls_id : ls_ids) { + int ret = arb_server->palf_env_mgr_.delete_arbitration_instance( + palflite::PalfEnvKey(cluster_id, 1), arb_server->self_, ls_id); + if (OB_SUCCESS == ret) { + ATOMIC_INC(&remove_success_count); + } + if (OB_SUCCESS != ret) { + ASSERT_EQ(false, true); + } else { + } + } + }; + int64_t thread_count = 8; + std::vector create_threads; + create_threads.reserve(thread_count); + for (int i = 0; i < thread_count; i++) { + create_threads.emplace_back(std::thread(create_func)); + } + for (int i = 0; i < thread_count; i++) { + create_threads[i].join(); + } + ASSERT_EQ(thread_count*ls_ids.size(), create_success_count); + std::vector remove_threads; + remove_threads.reserve(thread_count); + for (int i = 0; i < thread_count; i++) { + remove_threads.emplace_back(std::thread(remove_func)); + } + for (int i = 0; i < thread_count; i++) { + remove_threads[i].join(); + } + ASSERT_EQ(thread_count*ls_ids.size(), remove_success_count); + +} + } // end unittest } // end oceanbase From 4747d70a68c5cbcc63625fc6f72ead6efd0b1bfd Mon Sep 17 00:00:00 2001 From: ZenoWang Date: Tue, 13 Aug 2024 07:17:33 +0000 Subject: [PATCH 025/249] Async freeze task count opt --- src/storage/ls/ob_freezer.cpp | 160 +++++++++++++++++++++++----------- src/storage/ls/ob_freezer.h | 11 +-- src/storage/ls/ob_ls.cpp | 19 ++-- 3 files changed, 125 insertions(+), 65 deletions(-) diff --git a/src/storage/ls/ob_freezer.cpp b/src/storage/ls/ob_freezer.cpp index 0d72eba11..77e77657b 100644 --- a/src/storage/ls/ob_freezer.cpp +++ b/src/storage/ls/ob_freezer.cpp @@ -389,7 +389,8 @@ ObFreezer::ObFreezer() need_resubmit_log_(false), enable_(false), is_inited_(false), - is_async_tablet_freeze_task_running_(false), + is_async_tablet_freeze_task_existing_(false), + is_async_ls_freeze_task_existing_(false), throttle_is_skipping_(false), tenant_replay_is_pending_(false), async_freeze_tablets_() {} @@ -408,7 +409,8 @@ ObFreezer::ObFreezer(ObLS *ls) need_resubmit_log_(false), enable_(false), is_inited_(false), - is_async_tablet_freeze_task_running_(false), + is_async_tablet_freeze_task_existing_(false), + is_async_ls_freeze_task_existing_(false), throttle_is_skipping_(false), tenant_replay_is_pending_(false), async_freeze_tablets_() {} @@ -433,7 +435,8 @@ void ObFreezer::reset() enable_ = false; async_freeze_tablets_.reuse(); is_inited_ = false; - is_async_tablet_freeze_task_running_ = false; + is_async_tablet_freeze_task_existing_ = false; + is_async_ls_freeze_task_existing_ = false; throttle_is_skipping_ = false; tenant_replay_is_pending_ = false; } @@ -458,7 +461,8 @@ int ObFreezer::init(ObLS *ls) low_priority_freeze_cnt_ = 0; pend_replay_cnt_ = 0; need_resubmit_log_ = false; - is_async_tablet_freeze_task_running_ = false; + is_async_tablet_freeze_task_existing_ = false; + is_async_ls_freeze_task_existing_ = false; throttle_is_skipping_ = false; tenant_replay_is_pending_ = false; @@ -658,27 +662,27 @@ void ObFreezer::resubmit_log_if_needed_(const int64_t start_time, struct AsyncFreezeFunctor { const int64_t trace_id_; const bool is_ls_freeze_; + const ObLSID ls_id_; ObFreezer *freezer_; // hold ls handle to avoid logstream being destroyed - ObLSHandle ls_handle_; AsyncFreezeFunctor(const int64_t trace_id, const bool is_ls_freeze, ObFreezer *freezer, ObLSHandle &ls_handle) - : trace_id_(trace_id), is_ls_freeze_(is_ls_freeze), freezer_(freezer), ls_handle_(ls_handle) {} + : trace_id_(trace_id), + is_ls_freeze_(is_ls_freeze), + ls_id_(freezer->get_ls_id()), + freezer_(freezer) {} int operator()() { int ret = OB_SUCCESS; - ObLS *ls = nullptr; - if (OB_ISNULL(freezer_)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(ERROR, "unexpected nullptr ", KR(ret), KP(this)); - } else if (FALSE_IT(ls = freezer_->ls_)) { - } else if (OB_ISNULL(ls)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(ERROR, "unexpected nullptr ", KR(ret), KP(this)); + + ObLSHandle ls_handle; + if (OB_FAIL(MTL(ObLSService *)->get_ls(ls_id_, ls_handle, ObLSGetMod::STORAGE_MOD))) { + STORAGE_LOG(WARN, "get ls handle failed. stop async freeze task", KR(ret), K(ls_id_)); } else { - share::ObLSID ls_id = freezer_->get_ls_id(); - STORAGE_LOG(INFO, "[Freezer] An Async Freeze Task Start", K(trace_id_), K(ls_id), K(is_ls_freeze_), KP(freezer_)); + // freezer_ cannot be nullptr because AsyncFreezeFunctor is constructed by ObFreezer::this pointer + STORAGE_LOG( + INFO, "[Freezer] An Async Freeze Task Start", K(trace_id_), K(ls_id_), K(is_ls_freeze_), KP(freezer_)); if (is_ls_freeze_) { - (void)ls->logstream_freeze_task(trace_id_, INT64_MAX); + (void)freezer_->async_ls_freeze_consumer(trace_id_); } else { (void)freezer_->async_tablet_freeze_consumer(trace_id_); } @@ -687,26 +691,40 @@ struct AsyncFreezeFunctor { } }; -void ObFreezer::commit_an_async_freeze_task(const int64_t trace_id, const bool is_ls_freeze) +/** + * @brief Try pushing back an async freeze task into Tenant Freeze Task Queue. + * To minimize the number of tasks in the async task queue as much as possible, + * we control the task count by async task existing flag. + * + * 1. First, we check task existing flag(async_freeze_task_already_exists_() function) + * and skip submit task if flag is true + * 2. Then, we use ATOMIC_CAS(acquired_exec_async_task_permission_() function) to avoid + * more than one thread set flag succ + * + */ +void ObFreezer::submit_an_async_freeze_task(const int64_t trace_id, const bool is_ls_freeze) { int ret = OB_SUCCESS; ObTenantFreezer *tenant_freezer = nullptr; share::ObLSID ls_id = get_ls_id(); const int64_t start_time = ObClockGenerator::getClock(); + bool submit_succ = false; - if (OB_UNLIKELY(!enable_)) { + if (OB_UNLIKELY(!enable_) || OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_RUNNING; LOG_WARN("freezer is offline, can not freeze now", K(ret), K(ls_id)); } else if (OB_ISNULL(tenant_freezer = MTL(storage::ObTenantFreezer *))) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "ObTenantFreezer is null", K(ret), K(ls_id)); + } else if (async_freeze_task_already_exists_(is_ls_freeze)) { + submit_succ = false; } else { ObSpinLockGuard freeze_thread_pool(tenant_freezer->freeze_thread_pool_lock_); ObLSHandle ls_handle; if (OB_FAIL(MTL(ObLSService *)->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { STORAGE_LOG(WARN, "get ls handle failed. stop async freeze task", KR(ret), K(ls_id)); - } else { + } else if (acquired_exec_async_task_permission_(is_ls_freeze)) { AsyncFreezeFunctor async_freeze_functor(trace_id, is_ls_freeze, this, ls_handle); do { ret = tenant_freezer->freeze_thread_pool_.commit_task_ignore_ret(async_freeze_functor); @@ -714,48 +732,88 @@ void ObFreezer::commit_an_async_freeze_task(const int64_t trace_id, const bool i STORAGE_LOG(WARN, "commit task to freeze thread pool failed", KR(ret), K(ls_id)); } } while (OB_FAIL(ret)); - STORAGE_LOG(INFO, "finish commit async freeze task", KR(ret), K(is_ls_freeze)); + submit_succ = true; + STORAGE_LOG(INFO, "finish submit async freeze task", KR(ret), K(ls_id), K(is_ls_freeze), K(submit_succ)); } } + + if (!submit_succ) { + TRANS_LOG(INFO, + "async freeze task already exists, skip submitting one task", + K(ls_id), + K(is_ls_freeze), + K(is_async_tablet_freeze_task_existing_), + K(is_async_ls_freeze_task_existing_)); + } +} + +bool ObFreezer::async_freeze_task_already_exists_(const bool is_ls_freeze) +{ + bool already_exists = false; + if (is_ls_freeze) { + already_exists = ATOMIC_LOAD(&is_async_ls_freeze_task_existing_); + } else { + already_exists = ATOMIC_LOAD(&is_async_tablet_freeze_task_existing_); + } + return already_exists; +} + +bool ObFreezer::acquired_exec_async_task_permission_(const bool is_ls_freeze) +{ + bool acquired = false; + if (is_ls_freeze) { + acquired = ATOMIC_BCAS(&is_async_ls_freeze_task_existing_, false, true); + } else { + acquired = ATOMIC_BCAS(&is_async_tablet_freeze_task_existing_, false, true); + } + return acquired; +} + +void ObFreezer::async_ls_freeze_consumer(const int64_t trace_id) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ls_->logstream_freeze_task(trace_id, INT64_MAX))) { + STORAGE_LOG(WARN, "async ls freeze failed", KR(ret), K(trace_id), K(get_ls_id())); + } + + // reset task existing flag + ATOMIC_STORE(&is_async_ls_freeze_task_existing_, false); } void ObFreezer::async_tablet_freeze_consumer(const int64_t trace_id) { - if (OB_ISNULL(ls_)) { - STORAGE_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "unexpected nullptr of ls", KP(this)); - } else if (set_async_tablet_freeze_task_start_succ()) { - const int64_t start_time = ObClockGenerator::getClock(); - STORAGE_LOG(INFO, "Async Tablet Freeze Task Start", K(get_ls_id())); + const int64_t start_time = ObClockGenerator::getClock(); + STORAGE_LOG(INFO, "Async Tablet Freeze Task Start", K(get_ls_id())); - ObSEArray tablet_ids; - tablet_ids.reuse(); - const int64_t current_epoch = ls_->get_switch_epoch(); - (void)get_all_async_freeze_tablets(current_epoch, tablet_ids); + ObSEArray tablet_ids; + tablet_ids.reuse(); + const int64_t current_epoch = ls_->get_switch_epoch(); + (void)get_all_async_freeze_tablets(current_epoch, tablet_ids); - int ret = OB_SUCCESS; - bool need_resubmit_task = false; - if (tablet_ids.empty()) { - // no tablet need freeze - } else { - const int64_t abs_timeout_ts = start_time + (3600LL * 1000LL * 1000LL/*an hour*/); - const bool need_rewrite_meta = false; - const bool is_sync = false; - if (OB_FAIL(ls_->tablet_freeze_task( - trace_id, tablet_ids, need_rewrite_meta, is_sync, abs_timeout_ts, current_epoch))) { - need_resubmit_task = true; - } + int ret = OB_SUCCESS; + bool need_resubmit_task = false; + if (tablet_ids.empty()) { + // no tablet need freeze + } else { + const int64_t abs_timeout_ts = start_time + (3600LL * 1000LL * 1000LL /*an hour*/); + const bool need_rewrite_meta = false; + const bool is_sync = false; + if (OB_FAIL( + ls_->tablet_freeze_task(trace_id, tablet_ids, need_rewrite_meta, is_sync, abs_timeout_ts, current_epoch))) { + need_resubmit_task = true; } + } - const int64_t end_time = ObClockGenerator::getClock(); - const int64_t spend_time_ms = (end_time - start_time) / 1000; - STORAGE_LOG(INFO, "Async Tablet Freeze Task finish", K(get_ls_id()), K(spend_time_ms)); - (void)set_async_freeze_task_stop(); + // print some debug info + const int64_t end_time = ObClockGenerator::getClock(); + const int64_t spend_time_ms = (end_time - start_time) / 1000; + STORAGE_LOG(INFO, "Async Tablet Freeze Task finish", K(get_ls_id()), K(spend_time_ms)); - // NOTE : must check is_async_freeze_tablets_empty() after set_async_freeze_task_stop() - if (need_resubmit_task || !is_async_freeze_tablets_empty()) { - const bool is_ls_freeze = false; - (void)commit_an_async_freeze_task(trace_id, is_ls_freeze); - } + // NOTE : reset task existing flag before submit another task + ATOMIC_STORE(&is_async_tablet_freeze_task_existing_, false); + if (need_resubmit_task || !is_async_freeze_tablets_empty()) { + const bool is_ls_freeze = false; + (void)submit_an_async_freeze_task(trace_id, is_ls_freeze); } } diff --git a/src/storage/ls/ob_freezer.h b/src/storage/ls/ob_freezer.h index 2b902441a..df7537f80 100644 --- a/src/storage/ls/ob_freezer.h +++ b/src/storage/ls/ob_freezer.h @@ -211,12 +211,10 @@ public: ObIArray &freeze_failed_tablets); int get_all_async_freeze_tablets(const int64_t ls_epoch, ObIArray &tablet_ids); bool is_async_freeze_tablets_empty() const { return async_freeze_tablets_.empty(); } - bool is_async_tablet_freeze_task_running() { return ATOMIC_LOAD(&is_async_tablet_freeze_task_running_); } - bool set_async_tablet_freeze_task_start_succ() { return ATOMIC_BCAS(&is_async_tablet_freeze_task_running_, false, true); } - void set_async_freeze_task_stop() { ATOMIC_STORE(&is_async_tablet_freeze_task_running_, false); } void record_async_freeze_tablet(const AsyncFreezeTabletInfo &async_freeze_tablet_info); void erase_async_freeze_tablet(const AsyncFreezeTabletInfo &async_freeze_tablet_info); - void commit_an_async_freeze_task(const int64_t trace_id, const bool is_ls_freeze); + void submit_an_async_freeze_task(const int64_t trace_id, const bool is_ls_freeze); + void async_ls_freeze_consumer(const int64_t trace_id); void async_tablet_freeze_consumer(const int64_t trace_id); common::hash::ObHashSet &get_async_freeze_tablets() { return async_freeze_tablets_; } /********************** freeze **********************/ @@ -369,6 +367,8 @@ private: ObIArray &freeze_failed_tablets); int inner_wait_memtable_freeze_finish_(ObTableHandleV2 &memtable_handle); void submit_checkpoint_task(); + bool async_freeze_task_already_exists_(const bool is_ls_freeze); + bool acquired_exec_async_task_permission_(const bool is_ls_freeze); private: // flag whether the logsteram is freezing // the first bit: 1, freeze; 0, not freeze @@ -394,7 +394,8 @@ private: bool need_resubmit_log_; bool enable_; // whether we can do freeze now bool is_inited_; - bool is_async_tablet_freeze_task_running_; + bool is_async_tablet_freeze_task_existing_; + bool is_async_ls_freeze_task_existing_; bool throttle_is_skipping_; bool tenant_replay_is_pending_; common::hash::ObHashSet async_freeze_tablets_; diff --git a/src/storage/ls/ob_ls.cpp b/src/storage/ls/ob_ls.cpp index 78564a359..8daaa75f1 100755 --- a/src/storage/ls/ob_ls.cpp +++ b/src/storage/ls/ob_ls.cpp @@ -1858,10 +1858,15 @@ int ObLS::logstream_freeze(const int64_t trace_id, const bool is_sync, const int const int64_t abs_timeout_ts = (0 == input_abs_timeout_ts) ? ObClockGenerator::getClock() + ObFreezer::SYNC_FREEZE_DEFAULT_RETRY_TIME : input_abs_timeout_ts; - ret = logstream_freeze_task(trace_id, abs_timeout_ts); + ObLSHandle ls_handle; + if (OB_FAIL(MTL(ObLSService *)->get_ls(ls_meta_.ls_id_, ls_handle, ObLSGetMod::STORAGE_MOD))) { + STORAGE_LOG(WARN, "get ls handle failed. stop async freeze task", KR(ret), K(ls_meta_.ls_id_)); + } else { + ret = logstream_freeze_task(trace_id, abs_timeout_ts); + } } else { const bool is_ls_freeze = true; - (void)ls_freezer_.commit_an_async_freeze_task(trace_id, is_ls_freeze); + (void)ls_freezer_.submit_an_async_freeze_task(trace_id, is_ls_freeze); } return ret; } @@ -1963,14 +1968,10 @@ int ObLS::tablet_freeze(const int64_t trace_id, is_not_timeout = current_time < abs_timeout_ts; } while (is_retry_code && is_not_timeout); } else { + //Async tablet freeze. Must record tablet ids before submit task + const bool is_ls_freeze = false; (void)record_async_freeze_tablets_(tablet_ids, freeze_epoch); - - if (ls_freezer_.is_async_tablet_freeze_task_running()) { - // do not need another async batch freeze task - } else { - const bool is_ls_freeze = false; - (void)ls_freezer_.commit_an_async_freeze_task(trace_id, is_ls_freeze); - } + (void)ls_freezer_.submit_an_async_freeze_task(trace_id, is_ls_freeze); } return ret; } From 0b27cf2bbd41684811e17c04539aedc50517d567 Mon Sep 17 00:00:00 2001 From: wanghangQ <493666756@qq.com> Date: Tue, 13 Aug 2024 09:48:54 +0000 Subject: [PATCH 026/249] [Place Holder] Vector/Array Type CDC Support --- deps/oblib/src/rpc/obmysql/ob_mysql_global.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deps/oblib/src/rpc/obmysql/ob_mysql_global.h b/deps/oblib/src/rpc/obmysql/ob_mysql_global.h index a50f66d96..57a26d659 100644 --- a/deps/oblib/src/rpc/obmysql/ob_mysql_global.h +++ b/deps/oblib/src/rpc/obmysql/ob_mysql_global.h @@ -261,6 +261,8 @@ enum EMySQLFieldType MYSQL_TYPE_ORA_BLOB = 210, MYSQL_TYPE_ORA_CLOB = 211, MYSQL_TYPE_ROARINGBITMAP = 215, + MYSQL_TYPE_OB_VECTOR = 216, // used in cdc/oms not used in client + MYSQL_TYPE_OB_ARRAY = 217, // used in cdc/oms not used in client MYSQL_TYPE_JSON = 245, MYSQL_TYPE_NEWDECIMAL = 246, MYSQL_TYPE_ENUM = 247, From 4ee3c2e81113fe3cbf8e052fbd24535dae30bd65 Mon Sep 17 00:00:00 2001 From: AA-tuliwei-BB <1123152962@qq.com> Date: Tue, 13 Aug 2024 10:29:02 +0000 Subject: [PATCH 027/249] fix bugs of unpivot log plan --- src/sql/optimizer/ob_log_unpivot.cpp | 203 +++++++++++++++++---------- src/sql/optimizer/ob_log_unpivot.h | 5 + 2 files changed, 133 insertions(+), 75 deletions(-) diff --git a/src/sql/optimizer/ob_log_unpivot.cpp b/src/sql/optimizer/ob_log_unpivot.cpp index 0b1f7cc96..d677810cd 100644 --- a/src/sql/optimizer/ob_log_unpivot.cpp +++ b/src/sql/optimizer/ob_log_unpivot.cpp @@ -57,6 +57,25 @@ int ObLogUnpivot::generate_access_exprs() return ret; } +bool ObLogUnpivot::is_in_old_columns(const ObSelectStmt &select_stmt, + const ObUnpivotInfo &unpivot_info, + ObRawExpr *expr) +{ + int ret = OB_SUCCESS; + bool is_in = false; + if (0 == unpivot_info.old_column_count_) { + //do nothing + } else { + for (int64_t j = 0; OB_SUCC(ret) && j < unpivot_info_.old_column_count_; ++j) { + if (expr == select_stmt.get_select_item(j).expr_) { + is_in = true; + break; + } + } + } + return is_in; +} + int ObLogUnpivot::allocate_expr_post(ObAllocExprContext &ctx) { int ret = OB_SUCCESS; @@ -239,7 +258,7 @@ int ObLogUnpivot::est_cost() int ObLogUnpivot::compute_op_ordering() { int ret = OB_SUCCESS; - ObArray select_exprs; + ObSEArray ordering_exprs; if (OB_ISNULL(get_stmt()) || OB_UNLIKELY(!get_stmt()->is_select_stmt()) || OB_UNLIKELY(!get_stmt()->is_unpivot_select())) { @@ -250,103 +269,137 @@ int ObLogUnpivot::compute_op_ordering() if (OB_ISNULL(child= get_child(ObLogicalOperator::first_child))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("operator does not have child", K(ret)); - } else if (OB_FAIL(set_op_ordering(child->get_op_ordering()))) { - LOG_WARN("failed to set op ordering", K(ret)); } + bool found = true; const ObSelectStmt &select_stmt = *static_cast(get_stmt()); - const ObUnpivotInfo unpivot_info = select_stmt.get_unpivot_info(); - if (unpivot_info.old_column_count_ > 0) { - if (OB_FAIL(select_stmt.get_select_exprs(select_exprs, true))) { - LOG_WARN("failed to get select exprs", K(ret)); + const ObUnpivotInfo &unpivot_info = select_stmt.get_unpivot_info(); + for (int64_t j = 0; OB_SUCC(ret) && found && j < child->get_op_ordering().count(); ++j) { + if (is_in_old_columns(select_stmt, unpivot_info, child->get_op_ordering().at(j).expr_)) { + if (OB_FAIL(ordering_exprs.push_back(child->get_op_ordering().at(j).expr_))) { + LOG_WARN("failed to push back", K(ret)); + } } else { - int64_t expr_count = select_exprs.count(); - for (int64_t i = expr_count - 1; OB_SUCC(ret)&& i >= unpivot_info.old_column_count_; i--) { - ret = select_exprs.remove(i); - } - expr_count = select_exprs.count(); - for (int64_t i = expr_count - 1; OB_SUCC(ret) && i >= 0; i--) { - bool found = false; - for (int64_t j = 0; !found && j < child->get_op_ordering().count(); ++j) { - found = select_exprs.at(i) == child->get_op_ordering().at(j).expr_; - } - if (!found) { - ret = select_exprs.remove(i); - } - } + found = false; } - - if (OB_SUCC(ret)) { - if (OB_FAIL(ObOptimizerUtil::make_sort_keys(select_exprs, get_op_ordering()))) { - LOG_WARN("failed to copy sort keys", K(ret)); - } - } - LOG_DEBUG("finish compute_op_ordering", K(select_exprs.count()), K(unpivot_info), - K(get_op_ordering()), K(select_exprs)); - } else { - //do nothing } + + if (OB_SUCC(ret)) { + if (OB_FAIL(ObOptimizerUtil::make_sort_keys(ordering_exprs, get_op_ordering()))) { + LOG_WARN("failed to copy sort keys", K(ret)); + } + } + LOG_DEBUG("finish compute_op_ordering", K(unpivot_info), K(get_op_ordering())); } return ret; } int ObLogUnpivot::compute_fd_item_set() { + int ret = OB_SUCCESS; - const ObLogicalOperator *child = NULL; - if (OB_ISNULL(child = get_child(ObLogicalOperator::first_child)) - || OB_ISNULL(my_plan_) - || OB_ISNULL(get_stmt()) - || OB_UNLIKELY(!get_stmt()->is_select_stmt()) - || OB_UNLIKELY(!get_stmt()->is_unpivot_select())) { + set_fd_item_set(NULL); + return ret; +} + +int ObLogUnpivot::compute_const_exprs() +{ + // output_const_exprs should be the intersection of 'const_expr of child' and 'old_columns' + int ret = OB_SUCCESS; + ObLogicalOperator *child = NULL; + if (OB_ISNULL(my_plan_) || OB_UNLIKELY(get_num_of_child() < 0) || + OB_ISNULL(get_stmt()) || OB_UNLIKELY(!get_stmt()->is_select_stmt()) || + OB_UNLIKELY(!get_stmt()->is_unpivot_select())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpect null", K(ret), K(child), K(my_plan_), K(get_stmt())); + LOG_WARN("operator is invalid", K(ret), K(get_num_of_child()), K(my_plan_)); + } else if (OB_UNLIKELY(get_num_of_child() == 0)) { + /*do nothing*/ + } else if (OB_ISNULL(child = get_child(0))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("child is null", K(ret), K(child)); } else { const ObSelectStmt &select_stmt = static_cast(*get_stmt()); - const ObUnpivotInfo unpivot_info = select_stmt.get_unpivot_info(); - ObFdItemSet *fd_item_set = NULL; - const ObFdItemSet &child_fd_item_set = child->get_fd_item_set(); - if (0 == unpivot_info.old_column_count_ || child_fd_item_set.empty()) { + const ObUnpivotInfo &unpivot_info = select_stmt.get_unpivot_info(); + const ObIArray &child_output_const_exprs= child->get_output_const_exprs(); + if (0 == unpivot_info.old_column_count_ || child_output_const_exprs.empty()) { //do nothing } else { - ObSEArray value_exprs; - for (int64_t i = 0; OB_SUCC(ret) && i < child_fd_item_set.count(); i++) { - ObRawExpr *expr = NULL; - if (OB_NOT_NULL(child_fd_item_set[i]) - && OB_NOT_NULL(child_fd_item_set[i]->get_parent_exprs()) - && child_fd_item_set[i]->get_parent_exprs()->count() == 1 - && OB_NOT_NULL(expr = child_fd_item_set[i]->get_parent_exprs()->at(0))) { - for (int64_t i = 0; OB_SUCC(ret) && i < unpivot_info.old_column_count_; ++i) { - if (expr == select_stmt.get_select_item(i).expr_) { - ObTableFdItem *fd_item = NULL; - value_exprs.reuse(); - if (OB_FAIL(value_exprs.push_back(expr))) { - LOG_WARN("failed to push back expr", K(ret)); - } else if (OB_FAIL(my_plan_->get_fd_item_factory().create_table_fd_item(fd_item, - true, value_exprs, get_table_set()))) { - LOG_WARN("failed to create fd item", K(ret)); - } else if (NULL == fd_item_set) { - if (OB_FAIL(my_plan_->get_fd_item_factory().create_fd_item_set(fd_item_set))) { - LOG_WARN("failed to create fd item set", K(ret)); - } - } - - if (OB_FAIL(ret)) { - } else if (OB_FAIL(fd_item_set->push_back(fd_item))) { - LOG_WARN("failed to push back fd item", K(ret)); - } - } + for (int64_t i = 0; OB_SUCC(ret) && i < child_output_const_exprs.count(); ++i) { + if (is_in_old_columns(select_stmt, unpivot_info, child_output_const_exprs.at(i))) { + if (OB_FAIL(output_const_exprs_.push_back(child_output_const_exprs.at(i)))) { + LOG_WARN("failed to push back", K(ret)); } } } } - - if (OB_FAIL(ret)) { + if (OB_FAIL(ret) || filter_exprs_.empty()) { /*do nothing*/ - } else if (OB_NOT_NULL(fd_item_set) && // rollup 时 fd_item_set is null - OB_FAIL(deduce_const_exprs_and_ft_item_set(*fd_item_set))) { - LOG_WARN("failed to deduce fd item set", K(ret)); + } else if (OB_FAIL(ObOptimizerUtil::compute_const_exprs(filter_exprs_, output_const_exprs_))) { + LOG_WARN("failed to compute const conditionexprs", K(ret)); + } else {/*do nothing*/} + } + return ret; +} + +int ObLogUnpivot::compute_equal_set() +{ + int ret = OB_SUCCESS; + ObLogicalOperator *child = NULL; + EqualSets *ordering_esets = NULL; + EqualSets *old_col_esets = NULL; + if (OB_ISNULL(my_plan_) || OB_UNLIKELY(get_num_of_child() < 0) || + OB_ISNULL(get_stmt()) || OB_UNLIKELY(!get_stmt()->is_select_stmt()) || + OB_UNLIKELY(!get_stmt()->is_unpivot_select())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("operator is invalid", K(ret), K(get_num_of_child()), K(my_plan_)); + } else if (OB_UNLIKELY(get_num_of_child() == 0)) { + // do nothing + } else if (OB_ISNULL(child = get_child(0))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("child is null", K(ret), K(child)); + } else if (OB_ISNULL(old_col_esets = get_plan()->create_equal_sets())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to create equal sets", K(ret)); + } else { + // get old_col_esets (equal set of old columns) + const EqualSets &child_esets = child->get_output_equal_sets(); + const ObSelectStmt &select_stmt = static_cast(*get_stmt()); + const ObUnpivotInfo &unpivot_info = select_stmt.get_unpivot_info(); + for (int64_t i = 0; OB_SUCC(ret) && i < child_esets.count(); ++i) { + ObSEArray eset_builder; + EqualSet *child_eset = child_esets.at(i); + if (OB_ISNULL(child_eset)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(child_eset)); + } + for (int64_t j = 0; OB_SUCC(ret) && j < child_eset->count(); ++j) { + if (is_in_old_columns(select_stmt, unpivot_info, child_eset->at(j))) { + if (OB_FAIL(eset_builder.push_back(child_eset->at(j)))) { + LOG_WARN("failed to push back", K(ret)); + } + } + } + if (OB_SUCC(ret) && OB_FAIL(ObRawExprSetUtils::add_expr_set( + &get_plan()->get_allocator(), eset_builder, *old_col_esets))) { + LOG_WARN("failed to add expr set", K(ret)); + } + } + + // compute equal set of unpivot log plan + if (OB_FAIL(ret)) { + } else if (filter_exprs_.empty()) { + // inherit equal sets from old_col_esets + set_output_equal_sets(old_col_esets); + } else if (OB_ISNULL(ordering_esets = get_plan()->create_equal_sets())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to create equal sets", K(ret)); + } else if (OB_FAIL(ObEqualAnalysis::compute_equal_set( + &my_plan_->get_allocator(), + filter_exprs_, + *old_col_esets, + *ordering_esets))) { + LOG_WARN("failed to compute ordering output equal set", K(ret)); } else { - set_fd_item_set(fd_item_set); + set_output_equal_sets(ordering_esets); } } return ret; diff --git a/src/sql/optimizer/ob_log_unpivot.h b/src/sql/optimizer/ob_log_unpivot.h index e117916a7..808112d29 100644 --- a/src/sql/optimizer/ob_log_unpivot.h +++ b/src/sql/optimizer/ob_log_unpivot.h @@ -44,11 +44,16 @@ public: inline common::ObIArray &get_access_exprs() { return access_exprs_; } virtual int compute_op_ordering() override; virtual int compute_fd_item_set() override; + virtual int compute_const_exprs() override; + virtual int compute_equal_set() override; virtual int compute_one_row_info() override; virtual int get_plan_item_info(PlanText &plan_text, ObSqlPlanItem &plan_item) override; private: int generate_access_exprs(); + bool is_in_old_columns(const ObSelectStmt &select_stmt, + const ObUnpivotInfo &unpivot_info, + ObRawExpr *expr); public: uint64_t subquery_id_; common::ObString subquery_name_; From 378625cd343385ccdbeb7780a425a2d07895f67d Mon Sep 17 00:00:00 2001 From: chaser-ch Date: Tue, 13 Aug 2024 10:35:24 +0000 Subject: [PATCH 028/249] sync wash mbs when specifed cache exceeds limit --- src/share/cache/ob_kvcache_store.cpp | 48 ++++++++++++++++------------ src/share/cache/ob_kvcache_store.h | 10 ++++-- 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/src/share/cache/ob_kvcache_store.cpp b/src/share/cache/ob_kvcache_store.cpp index de3c25a61..9cdb88755 100644 --- a/src/share/cache/ob_kvcache_store.cpp +++ b/src/share/cache/ob_kvcache_store.cpp @@ -454,7 +454,7 @@ int ObKVCacheStore::flush_washable_mbs(const uint64_t tenant_id, const bool forc void ObKVCacheStore::flush_washable_mbs(const int64_t cache_id) { int ret = OB_SUCCESS; - + if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; COMMON_LOG(WARN, "The ObKVCacheStore has not been inited", K(ret)); @@ -495,8 +495,10 @@ void ObKVCacheStore::flush_washable_mbs(const uint64_t tenant_id, const int64_t } } -int ObKVCacheStore::sync_wash_mbs(const uint64_t tenant_id, const int64_t size_need_washed, - ObICacheWasher::ObCacheMemBlock *&wash_blocks) +int ObKVCacheStore::sync_wash_mbs(const uint64_t tenant_id, + const int64_t size_need_washed, + ObICacheWasher::ObCacheMemBlock *&wash_blocks, + const int64_t wash_cache_id) { int ret = OB_SUCCESS; @@ -507,19 +509,19 @@ int ObKVCacheStore::sync_wash_mbs(const uint64_t tenant_id, const int64_t size_n ret = OB_INVALID_ARGUMENT; COMMON_LOG(WARN, "invalid arguments", K(ret), K(tenant_id), K(size_need_washed), K_(aligned_block_size)); - } else if (OB_FAIL(try_flush_washable_mb(tenant_id, wash_blocks, -1, size_need_washed))) { + } else if (OB_FAIL(try_flush_washable_mb(tenant_id, wash_blocks, wash_cache_id, size_need_washed))) { if (ret != OB_CACHE_FREE_BLOCK_NOT_ENOUGH) { COMMON_LOG(WARN, "Fail to try flush mb", K(ret), K(tenant_id)); } } - + return ret; } int ObKVCacheStore::try_flush_washable_mb( - const uint64_t tenant_id, - ObICacheWasher::ObCacheMemBlock *&wash_blocks, - const int64_t cache_id, + const uint64_t tenant_id, + ObICacheWasher::ObCacheMemBlock *&wash_blocks, + const int64_t cache_id, const int64_t size_need_washed, const bool force_flush) { @@ -884,21 +886,22 @@ int ObKVCacheStore::alloc_mbhandle( const uint64_t tenant_id = inst.tenant_id_; const int64_t memory_limit_pct = inst.get_memory_limit_pct(); const int64_t cache_store_size = ATOMIC_AAF(&inst.status_.store_size_, block_size); + int64_t wash_cache_id = -1; if (!inst.mb_list_handle_.is_valid()) { ret = OB_ERR_UNEXPECTED; COMMON_LOG(ERROR, "mb_list_handle is invalid", K(ret)); } else if (memory_limit_pct < 100 && cache_store_size > (inst.mb_list_handle_.get_resource_handle()->get_memory_mgr()->get_limit() * memory_limit_pct / 100)) { - ret = OB_SIZE_OVERFLOW; - COMMON_LOG(INFO, "Fail to allocate memory, ", K(ret), K(block_size), K(cache_store_size), K(memory_limit_pct)); - } else if (NULL == (buf = static_cast(alloc_mb( - *inst.mb_list_handle_.get_resource_handle(), tenant_id, block_size)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - COMMON_LOG(WARN, "Fail to allocate memory, ", K(block_size), K(ret)); + wash_cache_id = inst.cache_id_; } - if (NULL != buf) { + if (OB_FAIL(ret)) { + } else if (NULL == (buf = static_cast(alloc_mb( + *inst.mb_list_handle_.get_resource_handle(), tenant_id, block_size, wash_cache_id)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + COMMON_LOG(WARN, "Fail to allocate memory, ", K(block_size), K(ret)); + } else { mem_block = new (buf) ObKVStoreMemBlock(buf + sizeof(ObKVStoreMemBlock), block_size - sizeof(ObKVStoreMemBlock)); while (OB_FAIL(mb_handles_pool_.pop(mb_handle))) { @@ -1306,7 +1309,9 @@ void ObKVCacheStore::destroy_wash_structs() } void *ObKVCacheStore::alloc_mb(ObTenantResourceMgrHandle &resource_handle, - const uint64_t tenant_id, const int64_t block_size) + const uint64_t tenant_id, + const int64_t block_size, + const int64_t wash_cache_id) { void *ptr = NULL; int ret = OB_SUCCESS; @@ -1314,13 +1319,14 @@ void *ObKVCacheStore::alloc_mb(ObTenantResourceMgrHandle &resource_handle, ret = OB_INVALID_ARGUMENT; COMMON_LOG(WARN, "invalid arguments", K(ret), "handle valid", resource_handle.is_valid(), K(tenant_id), K(block_size), K_(block_size)); - } else if (NULL == (ptr = resource_handle.get_memory_mgr()->alloc_cache_mb(block_size))) { + } else if (-1 != wash_cache_id + || NULL == (ptr = resource_handle.get_memory_mgr()->alloc_cache_mb(block_size))) { if (block_size == block_size_) { ObICacheWasher::ObCacheMemBlock *washed_blocks = NULL; const int64_t wash_size = aligned_block_size_; - if (OB_FAIL(sync_wash_mbs(tenant_id, wash_size, washed_blocks))) { + if (OB_FAIL(sync_wash_mbs(tenant_id, wash_size, washed_blocks, wash_cache_id))) { COMMON_LOG(WARN, "sync_wash_mbs failed", K(ret), - K(tenant_id), K(wash_size)); + K(tenant_id), K(wash_size), K(wash_cache_id)); } else { ptr = reinterpret_cast(washed_blocks); } @@ -1333,8 +1339,8 @@ void *ObKVCacheStore::alloc_mb(ObTenantResourceMgrHandle &resource_handle, if (wash_size > aligned_block_size_) { wash_size += 2 * aligned_block_size_; } - if (OB_FAIL(sync_wash_mbs(tenant_id, wash_size, washed_blocks))) { - COMMON_LOG(WARN, "sync_wash_mbs failed", K(ret), K(tenant_id), K(wash_size)); + if (OB_FAIL(sync_wash_mbs(tenant_id, wash_size, washed_blocks, wash_cache_id))) { + COMMON_LOG(WARN, "sync_wash_mbs failed", K(ret), K(tenant_id), K(wash_size), K(wash_cache_id)); } else { ObICacheWasher::ObCacheMemBlock *wash_block = washed_blocks; ObICacheWasher::ObCacheMemBlock *next = NULL; diff --git a/src/share/cache/ob_kvcache_store.h b/src/share/cache/ob_kvcache_store.h index 1ad257f5f..c0161b09c 100644 --- a/src/share/cache/ob_kvcache_store.h +++ b/src/share/cache/ob_kvcache_store.h @@ -84,8 +84,10 @@ public: void flush_washable_mbs(const int64_t cache_id); void flush_washable_mbs(const uint64_t tenant_id, const int64_t cache_id); - int sync_wash_mbs(const uint64_t tenant_id, const int64_t wash_size, - lib::ObICacheWasher::ObCacheMemBlock *&wash_blocks); + int sync_wash_mbs(const uint64_t tenant_id, + const int64_t wash_size, + lib::ObICacheWasher::ObCacheMemBlock *&wash_blocks, + const int64_t wash_cache_id = -1); virtual int alloc_mbhandle(ObKVCacheInst &inst, const int64_t block_size, ObKVMemBlockHandle *&mb_handle); @@ -207,7 +209,9 @@ private: void destroy_wash_structs(); void *alloc_mb(lib::ObTenantResourceMgrHandle &resource_handle, - const uint64_t tenant_id, const int64_t block_size); + const uint64_t tenant_id, + const int64_t block_size, + const int64_t wash_cache_id = -1); void free_mb(lib::ObTenantResourceMgrHandle &resource_handle, const uint64_t tenant_id, void *ptr); From a133d85a4854484aa8d89190c682664a7fc00e77 Mon Sep 17 00:00:00 2001 From: XIAO-HOU <372060054@qq.com> Date: Tue, 13 Aug 2024 10:41:28 +0000 Subject: [PATCH 029/249] Support all null shortcut judge in oracle mode --- src/sql/engine/basic/ob_pushdown_filter.cpp | 129 ++++++++++++++++--- src/sql/engine/basic/ob_pushdown_filter.h | 18 +-- src/storage/access/ob_block_row_store.cpp | 23 +++- src/storage/access/ob_block_row_store.h | 2 +- src/storage/access/ob_sample_filter.h | 7 +- src/storage/access/ob_table_access_param.cpp | 32 +++-- src/storage/access/ob_table_access_param.h | 2 + 7 files changed, 163 insertions(+), 50 deletions(-) diff --git a/src/sql/engine/basic/ob_pushdown_filter.cpp b/src/sql/engine/basic/ob_pushdown_filter.cpp index cc188f5ea..ad0c089a4 100644 --- a/src/sql/engine/basic/ob_pushdown_filter.cpp +++ b/src/sql/engine/basic/ob_pushdown_filter.cpp @@ -963,6 +963,57 @@ int ObPushdownFilterFactory::alloc( return ret; } +int ObPushdownFilterFactory::convert_white_filter_to_black(ObPushdownFilterExecutor *&filter) +{ + int ret = OB_SUCCESS; + ObPushdownFilterNode *new_node = nullptr; + ObPushdownFilterExecutor *new_filter = nullptr; + if (OB_UNLIKELY(nullptr == filter || !filter->is_filter_white_node() || nullptr == alloc_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid filter", K(ret), KP_(alloc), KPC(filter)); + } else if (OB_FAIL(alloc(BLACK_FILTER, 0, new_node))) { + LOG_WARN("Failed to alloc pushdown black filter node", K(ret)); + } else if (OB_FAIL(alloc(BLACK_FILTER_EXECUTOR, 0, *new_node, new_filter, filter->get_op()))) { + LOG_WARN("Failed to alloc pushdown black filter executor", K(ret)); + } else { + ObWhiteFilterExecutor *white_filter = static_cast(filter); + ObPushdownWhiteFilterNode &white_node = white_filter->get_filter_node(); + ObPushdownBlackFilterNode &black_node = *static_cast(new_node); + if (OB_FAIL(black_node.col_ids_.init(white_node.col_ids_.count()))) { + LOG_WARN("Failed to init col id array", K(ret), K(white_node.col_ids_.count())); + } else if (OB_FAIL(black_node.col_ids_.assign(white_node.col_ids_))) { + LOG_WARN("Failed to assign col id array", K(ret)); + } else if (OB_FAIL(black_node.column_exprs_.init(white_node.column_exprs_.count()))) { + LOG_WARN("Failed to init column expr array", K(ret), K(white_node.column_exprs_.count())); + } else if (OB_FAIL(black_node.column_exprs_.assign(white_node.column_exprs_))) { + LOG_WARN("Failed to assign column expr array", K(ret)); + } else if (OB_FAIL(black_node.filter_exprs_.init(1))) { + LOG_WARN("Failed to init filter expr array", K(ret)); + } else if (OB_FAIL(black_node.filter_exprs_.push_back(white_node.expr_))) { + LOG_WARN("Failed to push back expr", K(ret)); + } else { + white_filter->clear_in_datums(); + filter = new_filter; + LOG_TRACE("convert white filter to black filter", K(ret), KPC(filter), KPC(white_filter)); + } + } + if (OB_FAIL(ret)) { + if (OB_NOT_NULL(alloc_)) { + if (nullptr != new_filter) { + new_filter->~ObPushdownFilterExecutor(); + alloc_->free(new_filter); + new_filter = nullptr; + } + if (nullptr != new_node) { + new_node->~ObPushdownFilterNode(); + alloc_->free(new_node); + new_node = nullptr; + } + } + } + return ret; +} + int ObPushdownFilter::serialize_pushdown_filter( char *buf, int64_t buf_len, @@ -1695,25 +1746,39 @@ int ObPushdownFilterExecutor::prepare_skip_filter() } // 初始化需要被清理的标记 -int ObAndFilterExecutor::init_evaluated_datums() +int ObAndFilterExecutor::init_evaluated_datums(common::ObIAllocator *allocator, bool &need_convert) { int ret = OB_SUCCESS; for (uint32_t i = 0; i < n_child_ && OB_SUCC(ret); ++i) { - if (OB_FAIL(childs_[i]->init_evaluated_datums())) { + need_convert = false; + if (OB_FAIL(childs_[i]->init_evaluated_datums(allocator, need_convert))) { LOG_WARN("failed to filter child", K(ret)); + } else if (OB_UNLIKELY(need_convert)) { + ObPushdownFilterFactory filter_factory(allocator); + if (OB_FAIL(filter_factory.convert_white_filter_to_black(childs_[i]))) { + LOG_WARN("Failed to convert white filter to black filter", K(ret), KPC(childs_[i])); + } } } + need_convert = false; return ret; } -int ObOrFilterExecutor::init_evaluated_datums() +int ObOrFilterExecutor::init_evaluated_datums(common::ObIAllocator *allocator, bool &need_convert) { int ret = OB_SUCCESS; for (uint32_t i = 0; i < n_child_ && OB_SUCC(ret); ++i) { - if (OB_FAIL(childs_[i]->init_evaluated_datums())) { + need_convert = false; + if (OB_FAIL(childs_[i]->init_evaluated_datums(allocator, need_convert))) { LOG_WARN("failed to filter child", K(ret)); + } else if (OB_UNLIKELY(need_convert)) { + ObPushdownFilterFactory filter_factory(allocator); + if (OB_FAIL(filter_factory.convert_white_filter_to_black(childs_[i]))) { + LOG_WARN("Failed to convert white filter to black filter", K(ret), KPC(childs_[i])); + } } } + need_convert = false; return ret; } @@ -1754,9 +1819,11 @@ int ObPhysicalFilterExecutor::filter(blocksstable::ObStorageDatum *datums, int64 // 根据calc expr来设置每个列（空集）对应的清理Datum // 这里将clear的datum放在filter node是为了更精准处理，其实只有涉及到的表达式清理即可，其他不需要清理 // 还有类似空集需要清理 -int ObPhysicalFilterExecutor::init_evaluated_datums() +int ObPhysicalFilterExecutor::init_evaluated_datums(common::ObIAllocator *allocator, bool &need_convert) { + UNUSED(allocator); int ret = OB_SUCCESS; + need_convert = false; const int32_t cur_eval_info_cnt = n_eval_infos_; n_eval_infos_ = 0; n_datum_eval_flags_ = 0; @@ -1843,24 +1910,38 @@ void ObPhysicalFilterExecutor::clear_evaluated_infos() } } -int ObWhiteFilterExecutor::init_evaluated_datums() +int ObWhiteFilterExecutor::init_evaluated_datums(common::ObIAllocator *allocator, bool &need_convert) { + UNUSED(allocator); int ret = OB_SUCCESS; + need_convert = false; if (OB_ISNULL(filter_.expr_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("Unexpected filter expr", K(ret), KPC(filter_.expr_)); } else if (WHITE_OP_IN == filter_.get_op_type()) { - if (OB_FAIL(init_in_eval_datums())) { - LOG_WARN("Failed to init eval datums for WHITE_OP_IN filter", K(ret)); + if (OB_FAIL(init_in_eval_datums(need_convert))) { + if (OB_UNLIKELY(need_convert)) { + ret = OB_SUCCESS; + } else { + LOG_WARN("Failed to init eval datums for WHITE_OP_IN filter", K(ret)); + } + } + } else if (OB_FAIL(init_compare_eval_datums(need_convert))) { + if (OB_UNLIKELY(need_convert)) { + ret = OB_SUCCESS; + } else { + LOG_WARN("Failed to init eval datums for compare white filter", K(ret)); } - } else if (OB_FAIL(init_compare_eval_datums())) { - LOG_WARN("Failed to init eval datums for compare white filter", K(ret)); } - LOG_DEBUG("[PUSHDOWN], white pushdown filter inited datum params", K(datum_params_)); + LOG_DEBUG("[PUSHDOWN], white pushdown filter inited datum params", K(need_convert), K(datum_params_)); return ret; } -int ObWhiteFilterExecutor::init_compare_eval_datums() +// In oracle mode, when the values in one column are all null, +// the result should be empty set even though the expr.eval() is not valid (e.g., c1 < 1/0). +// We do not handle this situation at the storage layer, +// but convert it into a black filter and hand it over to the sql layer to process. +int ObWhiteFilterExecutor::init_compare_eval_datums(bool &need_convert) { int ret = OB_SUCCESS; ObEvalCtx &eval_ctx = op_.get_eval_ctx(); @@ -1870,7 +1951,7 @@ int ObWhiteFilterExecutor::init_compare_eval_datums() if (OB_UNLIKELY(filter_.expr_->arg_cnt_ < 2)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("Unexpected filter expr", K(ret), KPC(filter_.expr_)); - } else if (OB_FAIL(ObPhysicalFilterExecutor::init_evaluated_datums())) { + } else if (OB_FAIL(ObPhysicalFilterExecutor::init_evaluated_datums(nullptr, need_convert))) { LOG_WARN("Failed to init evaluated datums", K(ret)); } else if (OB_FAIL(init_array_param(datum_params_, filter_.expr_->arg_cnt_))) { LOG_WARN("Failed to alloc params", K(ret)); @@ -1888,7 +1969,11 @@ int ObWhiteFilterExecutor::init_compare_eval_datums() } else { ObDatum *datum = NULL; if (OB_FAIL(filter_.expr_->args_[i]->eval(eval_ctx, datum))) { - LOG_WARN("evaluate filter arg expr failed", K(ret), K(i)); + if (lib::is_oracle_mode()) { + need_convert = true; + } else { + LOG_WARN("evaluate filter arg expr failed", K(ret), K(i)); + } } else if (OB_FAIL(datum_params_.push_back(*datum))) { LOG_WARN("Failed to push back datum", K(ret)); } else if (is_null_param(*datum, param_obj_meta)) { @@ -1911,7 +1996,7 @@ int ObWhiteFilterExecutor::init_compare_eval_datums() return ret; } -int ObWhiteFilterExecutor::init_in_eval_datums() +int ObWhiteFilterExecutor::init_in_eval_datums(bool &need_convert) { int ret = OB_SUCCESS; ObEvalCtx &eval_ctx = op_.get_eval_ctx(); @@ -1926,7 +2011,7 @@ int ObWhiteFilterExecutor::init_in_eval_datums() 0 >= filter_.expr_->inner_func_cnt_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("Unexpected filter expr", K(ret), KPC(filter_.expr_), KP(filter_.expr_->args_[0]), KP(filter_.expr_->args_[1])); - } else if (OB_FAIL(ObPhysicalFilterExecutor::init_evaluated_datums())) { + } else if (OB_FAIL(ObPhysicalFilterExecutor::init_evaluated_datums(nullptr, need_convert))) { LOG_WARN("Failed to init evaluated datums", K(ret)); } else if (OB_FAIL(init_array_param(datum_params_, filter_.expr_->inner_func_cnt_))) { LOG_WARN("Failed to alloc params", K(ret)); @@ -1946,7 +2031,11 @@ int ObWhiteFilterExecutor::init_in_eval_datums() } if (OB_SUCC(ret)) { if (OB_FAIL(cur_arg->eval(eval_ctx, datum))) { - LOG_WARN("Evaluate filter arg expr failed", K(ret), K(i)); + if (lib::is_oracle_mode()) { + need_convert = true; + } else { + LOG_WARN("Evaluate filter arg expr failed", K(ret), K(i)); + } } else if (is_null_param(*datum, param_obj_meta)) { // skip null in filter IN } else if (OB_FAIL(add_to_param_set_and_array(*datum, cur_arg))) { @@ -2306,7 +2395,7 @@ int ObBlackFilterExecutor::filter_batch( } if (OB_SUCC(ret)) { if (OB_FAIL(eval_exprs_batch(*skip_bit_, bsize))) { - LOG_WARN("failed to eval batch or", K(ret)); + LOG_WARN("failed to eval batch", K(ret)); } else if (FALSE_IT(skip_bit_->bit_not(bsize))) { } else if (OB_FAIL(result_bitmap.from_bits_mask(start, end, reinterpret_cast(skip_bit_->data_)))) { LOG_WARN("failed to set filter result bitmap", K(start), K(end)); @@ -2375,9 +2464,11 @@ void ObDynamicFilterExecutor::filter_on_success(ObPushdownFilterExecutor* parent } } -int ObDynamicFilterExecutor::init_evaluated_datums() +int ObDynamicFilterExecutor::init_evaluated_datums(common::ObIAllocator *allocator, bool &need_convert) { + UNUSED(allocator); int ret = OB_SUCCESS; + need_convert = false; if (is_data_prepared_ && OB_NOT_NULL(runtime_filter_ctx_) && runtime_filter_ctx_->need_reset_in_rescan()) { is_data_prepared_ = false; diff --git a/src/sql/engine/basic/ob_pushdown_filter.h b/src/sql/engine/basic/ob_pushdown_filter.h index 0110c4ee6..05235c28e 100644 --- a/src/sql/engine/basic/ob_pushdown_filter.h +++ b/src/sql/engine/basic/ob_pushdown_filter.h @@ -248,6 +248,7 @@ public: : alloc_(alloc), type_(type), n_child_(0), childs_(nullptr), col_ids_(alloc) {} + virtual ~ObPushdownFilterNode() {} PushdownFilterType get_type() const { return type_; } common::ObIArray &get_col_ids() { return col_ids_; } void set_type(PushdownFilterType type) { type_ = type; } @@ -458,6 +459,7 @@ public: ObPushdownFilterNode &filter_node, ObPushdownFilterExecutor *&filter_executor, ObPushdownOperator &op); + int convert_white_filter_to_black(ObPushdownFilterExecutor *&filter); private: // pushdown filter @@ -692,7 +694,7 @@ public: int pull_up_common_node( const common::ObIArray &filter_indexes, ObPushdownFilterExecutor *&common_filter_executor); - virtual int init_evaluated_datums() { return common::OB_NOT_SUPPORTED; } + virtual int init_evaluated_datums(common::ObIAllocator *allocator, bool &need_convert) { return common::OB_NOT_SUPPORTED; } int execute( ObPushdownFilterExecutor *parent, PushdownFilterInfo &filter_info, @@ -758,7 +760,7 @@ public: {} virtual ~ObPhysicalFilterExecutor(); int filter(blocksstable::ObStorageDatum *datums, int64_t col_cnt, const sql::ObBitVector &skip_bit, bool &ret_val); - virtual int init_evaluated_datums() override; + virtual int init_evaluated_datums(common::ObIAllocator *allocator, bool &need_convert) override; virtual int filter(ObEvalCtx &eval_ctx, const sql::ObBitVector &skip_bit, bool &filtered) = 0; INHERIT_TO_STRING_KV("ObPhysicalFilterExecutor", ObPushdownFilterExecutor, K_(n_eval_infos), KP_(eval_infos)); @@ -943,7 +945,7 @@ public: OB_INLINE virtual common::ObIArray &get_col_ids() override { return filter_.get_col_ids(); } virtual const common::ObIArray *get_cg_col_exprs() const override { return &filter_.column_exprs_; } - virtual int init_evaluated_datums() override; + virtual int init_evaluated_datums(common::ObIAllocator *allocator, bool &need_convert) override; OB_INLINE const common::ObIArray &get_datums() const { return datum_params_; } OB_INLINE const common::ObDatum &get_min_param() const @@ -972,8 +974,8 @@ protected: { return datum.is_null() || (lib::is_oracle_mode() && obj_meta.is_character_type() && (0 == datum.len_)); } - int init_compare_eval_datums(); - int init_in_eval_datums(); + int init_compare_eval_datums(bool &need_convert); + int init_in_eval_datums(bool &need_convert); int init_param_set(const int64_t count, const ObExpr *cur_arg); int add_to_param_set_and_array(const ObDatum &datum, const ObExpr *cur_arg); public: @@ -997,7 +999,7 @@ public: OB_INLINE ObPushdownAndFilterNode &get_filter_node() { return filter_; } OB_INLINE virtual common::ObIArray &get_col_ids() override { return filter_.get_col_ids(); } - virtual int init_evaluated_datums() override; + virtual int init_evaluated_datums(common::ObIAllocator *allocator, bool &need_convert) override; INHERIT_TO_STRING_KV("ObPushdownAndFilterExecutor", ObPushdownFilterExecutor, K_(filter)); private: ObPushdownAndFilterNode &filter_; @@ -1015,7 +1017,7 @@ public: OB_INLINE ObPushdownOrFilterNode &get_filter_node() { return filter_; } OB_INLINE virtual common::ObIArray &get_col_ids() override { return filter_.get_col_ids(); } - virtual int init_evaluated_datums() override; + virtual int init_evaluated_datums(common::ObIAllocator *allocator, bool &need_convert) override; INHERIT_TO_STRING_KV("ObPushdownOrFilterExecutor", ObPushdownFilterExecutor, K_(filter)); private: ObPushdownOrFilterNode &filter_; @@ -1045,7 +1047,7 @@ public: { return static_cast(filter_); } - virtual int init_evaluated_datums() override; + virtual int init_evaluated_datums(common::ObIAllocator *allocator, bool &need_convert) override; int check_runtime_filter(ObPushdownFilterExecutor* parent_filter, bool &is_needed); void filter_on_bypass(ObPushdownFilterExecutor* parent_filter); void filter_on_success(ObPushdownFilterExecutor* parent_filter); diff --git a/src/storage/access/ob_block_row_store.cpp b/src/storage/access/ob_block_row_store.cpp index 01d153e45..03efbc5bf 100644 --- a/src/storage/access/ob_block_row_store.cpp +++ b/src/storage/access/ob_block_row_store.cpp @@ -145,10 +145,11 @@ int ObBlockRowStore::get_filter_result(ObFilterResult &res) return ret; } -int ObBlockRowStore::open(const ObTableIterParam &iter_param) +int ObBlockRowStore::open(ObTableIterParam &iter_param) { int ret = OB_SUCCESS; const bool need_padding = is_pad_char_to_full_length(context_.sql_mode_); + bool need_convert = false; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("Not init", K(ret)); @@ -159,20 +160,28 @@ int ObBlockRowStore::open(const ObTableIterParam &iter_param) LOG_WARN("Invalid argument to init store pushdown filter", K(ret), K(iter_param)); } else if (nullptr == pd_filter_info_.filter_) { // nothing to do + } else if (OB_FAIL(pd_filter_info_.filter_->init_evaluated_datums(context_.stmt_allocator_, need_convert))) { + LOG_WARN("Failed to init pushdown filter evaluated datums", K(ret)); } else { - if (iter_param.is_use_column_store()) { + if (OB_UNLIKELY(need_convert)) { + sql::ObPushdownFilterFactory filter_factory(context_.stmt_allocator_); + if (OB_FAIL(filter_factory.convert_white_filter_to_black(pd_filter_info_.filter_))) { + LOG_WARN("Failed to convert white filter to black filter", K(ret), KPC_(pd_filter_info_.filter)); + } else { + iter_param.pushdown_filter_ = pd_filter_info_.filter_; + } + } + if (OB_FAIL(ret)) { + } else if (iter_param.is_use_column_store()) { if (OB_FAIL(pd_filter_info_.filter_->init_co_filter_param(iter_param, need_padding))) { LOG_WARN("Failed to init pushdown filter executor", K(ret)); } + } else if (OB_FAIL(iter_param.build_index_filter_for_row_store(context_.allocator_))) { + LOG_WARN("Failed to build skip index for row store", K(ret)); } else if (OB_FAIL(pd_filter_info_.filter_->init_filter_param( *iter_param.get_col_params(), *iter_param.out_cols_project_, need_padding))) { LOG_WARN("Failed to init pushdown filter executor", K(ret)); } - - if (OB_FAIL(ret)) { - } else if (OB_FAIL(pd_filter_info_.filter_->init_evaluated_datums())) { - LOG_WARN("Failed to init pushdown filter evaluated datums", K(ret)); - } } return ret; } diff --git a/src/storage/access/ob_block_row_store.h b/src/storage/access/ob_block_row_store.h index 39ea6e62c..0d7ed78cc 100644 --- a/src/storage/access/ob_block_row_store.h +++ b/src/storage/access/ob_block_row_store.h @@ -58,7 +58,7 @@ public: virtual void reset(); virtual void reuse(); virtual int init(const ObTableAccessParam ¶m); - int open(const ObTableIterParam &iter_param); + int open(ObTableIterParam &iter_param); OB_INLINE bool is_valid() const { return is_inited_; } OB_INLINE bool is_disabled() const { return disabled_; } OB_INLINE void disable() { disabled_ = true; } diff --git a/src/storage/access/ob_sample_filter.h b/src/storage/access/ob_sample_filter.h index 4da1c55a2..f90a3ac96 100644 --- a/src/storage/access/ob_sample_filter.h +++ b/src/storage/access/ob_sample_filter.h @@ -234,7 +234,12 @@ public: } } OB_INLINE common::ObIAllocator *get_allocator() { return allocator_; } - OB_INLINE int init_evaluated_datums() { return OB_SUCCESS; } + OB_INLINE int init_evaluated_datums(common::ObIAllocator *allocator, bool &need_convert) + { + UNUSED(allocator); + need_convert = false; + return OB_SUCCESS; + } TO_STRING_KV(K_(is_inited), K_(is_reverse_scan), K_(row_num), K_(interval_infos), K_(percent), K_(seed), K_(pd_row_range), K_(block_row_range), K_(index_tree_height), K_(index_prefetch_depth), K_(data_prefetch_depth), diff --git a/src/storage/access/ob_table_access_param.cpp b/src/storage/access/ob_table_access_param.cpp index 76b67a4bf..30c51b682 100644 --- a/src/storage/access/ob_table_access_param.cpp +++ b/src/storage/access/ob_table_access_param.cpp @@ -14,7 +14,6 @@ #include "ob_table_access_param.h" #include "ob_dml_param.h" -#include "ob_sstable_index_filter.h" #include "storage/ob_relative_table.h" #include "storage/tablet/ob_tablet.h" #include "share/schema/ob_table_dml_param.h" @@ -169,6 +168,23 @@ int ObTableIterParam::get_cg_column_param(const share::schema::ObColumnParam *&c return ret; } +int ObTableIterParam::build_index_filter_for_row_store(common::ObIAllocator *allocator) +{ + int ret = OB_SUCCESS; + if (!is_use_column_store() && enable_pd_blockscan() + && enable_pd_filter() && enable_skip_index() && nullptr != pushdown_filter_) { + if (OB_FAIL(ObSSTableIndexFilterFactory::build_sstable_index_filter( + false, + get_read_info(), + *pushdown_filter_, + allocator, + sstable_index_filter_))) { + STORAGE_LOG(WARN, "Failed to build sstable index filter", K(ret), KPC(this)); + } + } + return ret; +} + DEF_TO_STRING(ObTableIterParam) { int64_t pos = 0; @@ -328,19 +344,7 @@ int ObTableAccessParam::init( iter_param_.set_use_stmt_iter_pool(); } - if (!iter_param_.is_use_column_store() - && iter_param_.enable_pd_blockscan() - && iter_param_.enable_pd_filter() - && iter_param_.enable_skip_index() - && nullptr != iter_param_.pushdown_filter_ - && OB_FAIL(ObSSTableIndexFilterFactory::build_sstable_index_filter( - false, - iter_param_.get_read_info(), - *iter_param_.pushdown_filter_, - scan_param.scan_allocator_, - iter_param_.sstable_index_filter_))) { - STORAGE_LOG(WARN, "Failed to build sstable index filter", K(ret), K(iter_param_)); - } else if (OB_FAIL(iter_param_.refresh_lob_column_out_status())) { + if (OB_FAIL(iter_param_.refresh_lob_column_out_status())) { STORAGE_LOG(WARN, "Failed to refresh lob column out status", K(ret), K(iter_param_)); } else if (scan_param.use_index_skip_scan() && OB_FAIL(get_prefix_cnt_for_skip_scan(scan_param, iter_param_))) { diff --git a/src/storage/access/ob_table_access_param.h b/src/storage/access/ob_table_access_param.h index 577504195..f7358902c 100644 --- a/src/storage/access/ob_table_access_param.h +++ b/src/storage/access/ob_table_access_param.h @@ -15,6 +15,7 @@ #include "storage/ob_i_store.h" #include "storage/blocksstable/ob_datum_range.h" +#include "ob_sstable_index_filter.h" #include "common/ob_tablet_id.h" #include "share/ob_i_tablet_scan.h" #include "share/schema/ob_table_param.h" @@ -51,6 +52,7 @@ public: bool enable_fuse_row_cache(const ObQueryFlag &query_flag) const; //temp solution int get_cg_column_param(const share::schema::ObColumnParam *&column_param) const; + int build_index_filter_for_row_store(common::ObIAllocator *allocator); const ObITableReadInfo *get_read_info(const bool is_get = false) const { return is_get ? rowkey_read_info_ : read_info_; From 1116be93e8e52f2d6455a70d809e0cd31ce83b4b Mon Sep 17 00:00:00 2001 From: yangqise7en <877793735@qq.com> Date: Tue, 13 Aug 2024 10:47:12 +0000 Subject: [PATCH 030/249] add latch_id for lock & fix clear_checksum_error case --- deps/oblib/src/lib/stat/ob_latch_define.h | 13 +++++++++---- .../freeze/ob_major_merge_progress_checker.h | 3 --- .../compaction/ob_compaction_locality_cache.cpp | 11 ++++++----- src/share/scheduler/ob_tenant_dag_scheduler.cpp | 3 +++ src/share/scheduler/ob_tenant_dag_scheduler.h | 6 ++++-- src/storage/blocksstable/ob_sstable_meta.cpp | 2 +- src/storage/blocksstable/ob_sstable_meta.h | 8 ++++++++ src/storage/column_store/ob_co_merge_dag.cpp | 2 +- .../column_store/ob_column_oriented_sstable.h | 1 - src/storage/compaction/ob_tenant_medium_checker.h | 1 - src/storage/ls/ob_ls_reserved_snapshot_mgr.cpp | 2 +- 11 files changed, 33 insertions(+), 19 deletions(-) diff --git a/deps/oblib/src/lib/stat/ob_latch_define.h b/deps/oblib/src/lib/stat/ob_latch_define.h index 6a11574f3..89486c5b5 100644 --- a/deps/oblib/src/lib/stat/ob_latch_define.h +++ b/deps/oblib/src/lib/stat/ob_latch_define.h @@ -290,9 +290,9 @@ LATCH_DEF(ARCHIVE_ROUND_MGR_LOCK, 263, "archive round mgr lock", LATCH_FIFO, 200 LATCH_DEF(ARCHIVE_PERSIST_MGR_LOCK, 264, "archive persist mgr lock", LATCH_FIFO, 2000, 0, true) LATCH_DEF(ARCHIVE_TASK_QUEUE_LOCK, 265, "archive task queue lock", LATCH_FIFO, 2000, 0, true) LATCH_DEF(EXT_SVR_BLACKLIST_LOCK, 266, "external server blacklist lock", LATCH_FIFO, 2000, 0, true) -LATCH_DEF(SSTABLE_INSERT_TABLE_CONTEXT_LOCK, 267, "direct insert table context lock", LATCH_FIFO, 2000, 0, true) -LATCH_DEF(SSTABLE_INSERT_TABLET_CONTEXT_LOCK, 268, "direct insert tablet context lock", LATCH_FIFO, 2000, 0, true) -LATCH_DEF(SSTABLE_INSERT_TABLE_MANAGER_LOCK, 269, "direct insert table manager lock", LATCH_FIFO, 2000, 0, true) +LATCH_DEF(SSTABLE_INSERT_TABLE_CONTEXT_LOCK, 267, "direct insert table context lock", LATCH_FIFO, 2000, 0, false) +LATCH_DEF(SSTABLE_INSERT_TABLET_CONTEXT_LOCK, 268, "direct insert tablet context lock", LATCH_FIFO, 2000, 0, false) +LATCH_DEF(SSTABLE_INSERT_TABLE_MANAGER_LOCK, 269, "direct insert table manager lock", LATCH_FIFO, 2000, 0, false) LATCH_DEF(COMPLEMENT_DATA_CONTEXT_LOCK, 270, "complement data context lock", LATCH_FIFO, 2000, 0, true) LATCH_DEF(TABLET_DDL_KV_MGR_LOCK, 271, "tablet ddl kv mgr lock", LATCH_FIFO, 2000, 0, true) LATCH_DEF(TABLET_AUTO_INCREMENT_MGR_LOCK, 272, "tablet auto increment mgr lock", LATCH_FIFO, 2000, 0, true) @@ -374,7 +374,12 @@ LATCH_DEF(DI_SUMMARY_LOCK, 340, "diagnostic info summary lock", LATCH_FIFO, 2000 LATCH_DEF(DI_ALLOCATE_LOCK, 341, "diagnostic info allocator lock", LATCH_FIFO, 2000, 0, false) LATCH_DEF(DI_COLLECTOR_LOCK, 342, "diagnostic info collector lock", LATCH_FIFO, 2000, 0, false) -LATCH_DEF(LATCH_END, 343, "latch end", LATCH_FIFO, 2000, 0, true) +LATCH_DEF(DAG_NET_SCHEDULER, 343, "dag net scheduler lock", LATCH_FIFO, 2000, 0, true) +LATCH_DEF(DAG_PRIO_SCHEDULER, 344, "dag prio scheduler lock", LATCH_FIFO, 2000, 0, true) +LATCH_DEF(LS_RESERVED_SNAPSHOT_LOCK, 345, "ls reserved snapshot lock", LATCH_FIFO, 2000, 0, true) + +LATCH_DEF(LATCH_END, 346, "latch end", LATCH_FIFO, 2000, 0, true) + #endif diff --git a/src/rootserver/freeze/ob_major_merge_progress_checker.h b/src/rootserver/freeze/ob_major_merge_progress_checker.h index a19474973..f8a02da91 100644 --- a/src/rootserver/freeze/ob_major_merge_progress_checker.h +++ b/src/rootserver/freeze/ob_major_merge_progress_checker.h @@ -29,7 +29,6 @@ namespace oceanbase namespace share { class ObIServerTrace; -class ObCompactionTabletMetaIterator; namespace schema { class ObSchemaGetterGuard; @@ -110,7 +109,6 @@ private: const uint64_t data_table_id, const int64_t finish_index_cnt, const compaction::ObTableCkmItems &data_table_ckm); - int deal_with_validated_data_table(const uint64_t data_table_id); bool should_ignore_cur_table(const ObSimpleTableSchemaV2 *simple_schema); int deal_with_rest_data_table(); bool is_extra_check_round() const { return 0 == (loop_cnt_ % 8); } // check every 8 rounds @@ -127,7 +125,6 @@ private: int handle_fts_checksum(); private: static const int64_t ADD_RS_EVENT_INTERVAL = 10L * 60 * 1000 * 1000; // 10m - static const int64_t PRINT_LOG_INTERVAL = 2 * 60 * 1000 * 1000; // 2m static const int64_t DEAL_REST_TABLE_CNT_THRESHOLD = 100; static const int64_t DEAL_REST_TABLE_INTERVAL = 10 * 60 * 1000 * 1000L; // 10m static const int64_t ASSGIN_FAILURE_RETRY_TIMES = 10; diff --git a/src/share/compaction/ob_compaction_locality_cache.cpp b/src/share/compaction/ob_compaction_locality_cache.cpp index a66e92a9d..5d5f3a257 100644 --- a/src/share/compaction/ob_compaction_locality_cache.cpp +++ b/src/share/compaction/ob_compaction_locality_cache.cpp @@ -94,11 +94,8 @@ int ObCompactionLocalityCache::inner_refresh_ls_locality() LOG_WARN("failed to get zone list", K(ret), K_(tenant_id)); } else if (zone_list.empty()) { LOG_INFO("zone list is empty, skip get ls locality", K(ret), K_(tenant_id)); - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(MTL(compaction::ObDiagnoseTabletMgr *)->add_diagnose_tablet(UNKNOW_LS_ID, UNKNOW_TABLET_ID, - share::ObDiagnoseTabletType::TYPE_MEDIUM_MERGE))) { - LOG_WARN("failed to add diagnose tablet for locality cache", K(ret)); - } + (void) MTL(compaction::ObDiagnoseTabletMgr *)->add_diagnose_tablet(UNKNOW_LS_ID, UNKNOW_TABLET_ID, + share::ObDiagnoseTabletType::TYPE_MEDIUM_MERGE); } if (OB_SUCC(ret)) { // 1. clear ls_infos cached in memory @@ -122,6 +119,10 @@ int ObCompactionLocalityCache::inner_refresh_ls_locality() } } } + if (OB_SUCC(ret)) { + (void) MTL(compaction::ObDiagnoseTabletMgr *)->delete_diagnose_tablet(UNKNOW_LS_ID, UNKNOW_TABLET_ID, + share::ObDiagnoseTabletType::TYPE_MEDIUM_MERGE); + } cost_ts = ObTimeUtility::fast_current_time() - cost_ts; LOG_INFO("finish to refresh ls locality cache", KR(ret), K_(tenant_id), K(cost_ts), K(zone_list)); } diff --git a/src/share/scheduler/ob_tenant_dag_scheduler.cpp b/src/share/scheduler/ob_tenant_dag_scheduler.cpp index db7db6228..82d909b9a 100644 --- a/src/share/scheduler/ob_tenant_dag_scheduler.cpp +++ b/src/share/scheduler/ob_tenant_dag_scheduler.cpp @@ -866,6 +866,9 @@ int ObIDag::reset_status_for_retry() } if (OB_FAIL(inner_reset_status_for_retry())) { // will call alloc_task() COMMON_LOG(WARN, "failed to inner reset status", K(ret)); + } else if (OB_UNLIKELY(!is_valid())) { + ret = OB_ERR_UNEXPECTED; + COMMON_LOG(WARN, "dag after retry is invalid", K(ret), KPC(this)); } else { set_dag_status(ObIDag::DAG_STATUS_RETRY); start_time_ = ObTimeUtility::fast_current_time(); diff --git a/src/share/scheduler/ob_tenant_dag_scheduler.h b/src/share/scheduler/ob_tenant_dag_scheduler.h index 12aa31be8..47eab7598 100644 --- a/src/share/scheduler/ob_tenant_dag_scheduler.h +++ b/src/share/scheduler/ob_tenant_dag_scheduler.h @@ -776,7 +776,8 @@ public: ObDagNetScheduler() : allocator_(nullptr), ha_allocator_(nullptr), - scheduler_(nullptr) + scheduler_(nullptr), + dag_net_map_lock_(ObLatchIds::DAG_NET_SCHEDULER) {} ~ObDagNetScheduler() { destroy(); } void destroy(); @@ -857,7 +858,8 @@ class ObDagPrioScheduler public: typedef common::ObDList WorkerList; ObDagPrioScheduler() - : allocator_(nullptr), + : prio_lock_(ObLatchIds::DAG_PRIO_SCHEDULER), + allocator_(nullptr), ha_allocator_(nullptr), scheduler_(nullptr), priority_(ObDagPrio::DAG_PRIO_MAX), diff --git a/src/storage/blocksstable/ob_sstable_meta.cpp b/src/storage/blocksstable/ob_sstable_meta.cpp index a22361250..72e0874f6 100644 --- a/src/storage/blocksstable/ob_sstable_meta.cpp +++ b/src/storage/blocksstable/ob_sstable_meta.cpp @@ -744,7 +744,7 @@ int ObSSTableMeta::init( } if (OB_SUCC(ret) && transaction::ObTransID(param.uncommitted_tx_id_).is_valid()) { - if (OB_FAIL(prepare_tx_context({param.uncommitted_tx_id_, 0}, allocator))) { + if (OB_FAIL(prepare_tx_context(ObTxContext::ObTxDesc(param.uncommitted_tx_id_, 0), allocator))) { LOG_WARN("failed to alloc memory for tx_ids_", K(ret), K(param)); } } diff --git a/src/storage/blocksstable/ob_sstable_meta.h b/src/storage/blocksstable/ob_sstable_meta.h index 5b8429d84..8f48fcd77 100644 --- a/src/storage/blocksstable/ob_sstable_meta.h +++ b/src/storage/blocksstable/ob_sstable_meta.h @@ -36,6 +36,14 @@ class ObTxContext final public: struct ObTxDesc final { + ObTxDesc() + : tx_id_(0), + row_count_(0) + {} + ObTxDesc(const int64_t tx_id, const int64_t row_count) + : tx_id_(tx_id), + row_count_(row_count) + {} int64_t tx_id_; int64_t row_count_; int serialize(char *buf, const int64_t buf_len, int64_t &pos) const; diff --git a/src/storage/column_store/ob_co_merge_dag.cpp b/src/storage/column_store/ob_co_merge_dag.cpp index 46541abaa..0db2340e7 100644 --- a/src/storage/column_store/ob_co_merge_dag.cpp +++ b/src/storage/column_store/ob_co_merge_dag.cpp @@ -1248,7 +1248,7 @@ int ObCOMergeDagNet::inner_create_and_schedule_dags(ObIDag *parent_dag) && (co_merge_ctx_->is_build_row_store() || max_cg_idx < DELAY_SCHEDULE_FINISH_DAG_CG_CNT)) { // add into dag_scheduler after parent-child relation generated if (OB_FAIL(create_dag(0, 0, finish_dag_, parent_dag/*parent*/, false/*add_scheduler_flag*/))) { - LOG_WARN("failed to create finish dag", K(ret)); + LOG_WARN("failed to create finish dag", K(ret), K_(finish_dag)); } } // refine merge_batch_size_ with tenant memory diff --git a/src/storage/column_store/ob_column_oriented_sstable.h b/src/storage/column_store/ob_column_oriented_sstable.h index 6871aac6c..fb2ce5c89 100644 --- a/src/storage/column_store/ob_column_oriented_sstable.h +++ b/src/storage/column_store/ob_column_oriented_sstable.h @@ -92,7 +92,6 @@ public: uint32_t full_column_cnt_; }; - enum ObCOSSTableBaseType : int32_t { INVALID_TYPE = 0, diff --git a/src/storage/compaction/ob_tenant_medium_checker.h b/src/storage/compaction/ob_tenant_medium_checker.h index e3935a516..12cc6f2f9 100644 --- a/src/storage/compaction/ob_tenant_medium_checker.h +++ b/src/storage/compaction/ob_tenant_medium_checker.h @@ -113,7 +113,6 @@ public: static const int64_t CHECK_LS_LOCALITY_INTERVAL = 5 * 60 * 1000 * 1000L; // 5m #endif static const int64_t DEFAULT_MAP_BUCKET = 1024; - static const int64_t MAX_BATCH_CHECK_NUM = 3000; typedef common::ObArray TabletLSArray; typedef hash::ObHashSet TabletLSSet; typedef hash::ObHashMap LSInfoMap; diff --git a/src/storage/ls/ob_ls_reserved_snapshot_mgr.cpp b/src/storage/ls/ob_ls_reserved_snapshot_mgr.cpp index 5b262f2ec..0acae1932 100644 --- a/src/storage/ls/ob_ls_reserved_snapshot_mgr.cpp +++ b/src/storage/ls/ob_ls_reserved_snapshot_mgr.cpp @@ -26,7 +26,7 @@ ObLSReservedSnapshotMgr::ObLSReservedSnapshotMgr() allocator_("ResvSnapMgr"), min_reserved_snapshot_(0), next_reserved_snapshot_(0), - snapshot_lock_(), + snapshot_lock_(ObLatchIds::LS_RESERVED_SNAPSHOT_LOCK), sync_clog_lock_(), ls_(nullptr), ls_handle_(), From c53bcf5c28bb32eb638652ccd6315e9a66cb3b81 Mon Sep 17 00:00:00 2001 From: leftgeek <1094669802@qq.com> Date: Tue, 13 Aug 2024 10:52:58 +0000 Subject: [PATCH 031/249] fix the bug of parsing insert overwrite --- src/sql/parser/sql_parser_mysql_mode.y | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sql/parser/sql_parser_mysql_mode.y b/src/sql/parser/sql_parser_mysql_mode.y index ab87f163f..695d2bc71 100644 --- a/src/sql/parser/sql_parser_mysql_mode.y +++ b/src/sql/parser/sql_parser_mysql_mode.y @@ -9386,8 +9386,9 @@ insert_with_opt_hint opt_priority opt_ignore opt_into single_table_insert opt_on } ParseNode *overwrite_node; - malloc_terminal_node(overwrite_node, result->malloc_pool_, T_BOOL); + malloc_terminal_node(overwrite_node, result->malloc_pool_, T_INT); overwrite_node->value_ = 1; + overwrite_node->is_hidden_const_ = 1; $5->children_[2] = NULL; /*duplicate key node is null*/ malloc_non_terminal_node($$, result->malloc_pool_, T_INSERT, 5, From ac6160c366754501da66a745163af39c2ee84e2b Mon Sep 17 00:00:00 2001 From: jingtaoye35 <1255153887@qq.com> Date: Tue, 13 Aug 2024 11:40:34 +0000 Subject: [PATCH 032/249] [CP] fix or_expansion bug --- src/sql/rewrite/ob_transform_or_expansion.cpp | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/sql/rewrite/ob_transform_or_expansion.cpp b/src/sql/rewrite/ob_transform_or_expansion.cpp index 8299cc836..9b7868a9b 100644 --- a/src/sql/rewrite/ob_transform_or_expansion.cpp +++ b/src/sql/rewrite/ob_transform_or_expansion.cpp @@ -655,17 +655,19 @@ int ObTransformOrExpansion::get_joined_table_pushdown_conditions(const TableItem LOG_WARN("failed to get table rel ids", K(ret)); } } - for (int64_t i = 0; OB_SUCC(ret) && i < trans_stmt->get_condition_exprs().count(); i++) { - ObRawExpr *cond = NULL; - if (OB_ISNULL(cond = trans_stmt->get_condition_exprs().at(i))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", K(ret)); - } else if (!cond->get_relation_ids().is_subset(table_set) || - cond->has_flag(CNT_SUB_QUERY)) { - /* do not push down */ - } else if (OB_FAIL(pushdown_conds.push_back(cond))) { - LOG_WARN("failed to push cond", K(ret)); - } else { /* do nothing */ } + if (OB_SUCC(ret) && !table_set.is_empty()) { + for (int64_t i = 0; OB_SUCC(ret) && i < trans_stmt->get_condition_exprs().count(); i++) { + ObRawExpr *cond = NULL; + if (OB_ISNULL(cond = trans_stmt->get_condition_exprs().at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (!cond->get_relation_ids().is_subset(table_set) || + cond->has_flag(CNT_SUB_QUERY)) { + /* do not push down */ + } else if (OB_FAIL(pushdown_conds.push_back(cond))) { + LOG_WARN("failed to push cond", K(ret)); + } else { /* do nothing */ } + } } } return ret; From 19beac29ce34a69774552b75c8558a0694070c79 Mon Sep 17 00:00:00 2001 From: jingtaoye35 <1255153887@qq.com> Date: Tue, 13 Aug 2024 11:57:28 +0000 Subject: [PATCH 033/249] [CP] disable some context as members of array directly --- src/sql/resolver/expr/ob_raw_expr.h | 6 + .../resolver/expr/ob_shared_expr_resolver.h | 3 + src/sql/rewrite/ob_stmt_comparer.cpp | 21 +- src/sql/rewrite/ob_stmt_comparer.h | 17 +- .../ob_transform_predicate_move_around.h | 3 + src/sql/rewrite/ob_transform_temp_table.cpp | 196 +++++++++--------- src/sql/rewrite/ob_transform_win_magic.cpp | 4 +- 7 files changed, 138 insertions(+), 112 deletions(-) diff --git a/src/sql/resolver/expr/ob_raw_expr.h b/src/sql/resolver/expr/ob_raw_expr.h index 97d400ce2..150650b62 100644 --- a/src/sql/resolver/expr/ob_raw_expr.h +++ b/src/sql/resolver/expr/ob_raw_expr.h @@ -1530,6 +1530,9 @@ struct ObExprEqualCheckContext bool ignore_param_; // only compare structure of expr bool ora_numeric_compare_; int64_t error_code_; //error code to return + +private: + DISABLE_COPY_ASSIGN(ObExprEqualCheckContext); }; struct ObExprParamCheckContext : ObExprEqualCheckContext @@ -1568,6 +1571,9 @@ struct ObExprParamCheckContext : ObExprEqualCheckContext const ObIArray *calculable_items_; // from query context const common::ObIArray *equal_param_constraints_; EqualSets *equal_sets_; + +private: + DISABLE_COPY_ASSIGN(ObExprParamCheckContext); }; enum ObVarType diff --git a/src/sql/resolver/expr/ob_shared_expr_resolver.h b/src/sql/resolver/expr/ob_shared_expr_resolver.h index e5d45cf8f..111b43d6e 100644 --- a/src/sql/resolver/expr/ob_shared_expr_resolver.h +++ b/src/sql/resolver/expr/ob_shared_expr_resolver.h @@ -32,6 +32,9 @@ struct ObQuestionmarkEqualCtx : public ObExprEqualCheckContext const ObConstRawExpr &right); ObSEArray equal_pairs_; + +private: + DISABLE_COPY_ASSIGN(ObQuestionmarkEqualCtx); }; struct ObRawExprEntry diff --git a/src/sql/rewrite/ob_stmt_comparer.cpp b/src/sql/rewrite/ob_stmt_comparer.cpp index 131d43c10..618331619 100644 --- a/src/sql/rewrite/ob_stmt_comparer.cpp +++ b/src/sql/rewrite/ob_stmt_comparer.cpp @@ -99,15 +99,20 @@ void ObStmtCompareContext::init(const ObIArray *calculable_i calculable_items_ = calculable_items; } -void ObStmtCompareContext::init(const ObDMLStmt *inner, - const ObDMLStmt *outer, - const ObStmtMapInfo &map_info, - const ObIArray *calculable_items) +int ObStmtCompareContext::init(const ObDMLStmt *inner, + const ObDMLStmt *outer, + const ObStmtMapInfo &map_info, + const ObIArray *calculable_items) { - inner_ = inner; - outer_ = outer; - map_info_ = map_info; - calculable_items_ = calculable_items; + int ret = OB_SUCCESS; + if (OB_FAIL(map_info_.assign(map_info))) { + LOG_WARN("failed to assign", K(ret)); + } else { + inner_ = inner; + outer_ = outer; + calculable_items_ = calculable_items; + } + return ret; } int ObStmtCompareContext::get_table_map_idx(uint64_t l_table_id, uint64_t r_table_id) diff --git a/src/sql/rewrite/ob_stmt_comparer.h b/src/sql/rewrite/ob_stmt_comparer.h index dcbae0d82..a219947c9 100644 --- a/src/sql/rewrite/ob_stmt_comparer.h +++ b/src/sql/rewrite/ob_stmt_comparer.h @@ -34,7 +34,8 @@ enum QueryRelation QUERY_EQUAL, QUERY_UNCOMPARABLE }; - struct ObStmtMapInfo { + +struct ObStmtMapInfo { common::ObSEArray, 4> view_select_item_map_; common::ObSEArray expr_cons_map_; common::ObSEArray const_param_map_; @@ -113,6 +114,9 @@ struct StmtCompareHelper { ObSEArray similar_stmts_; QbNameList hint_force_stmt_set_; ObSelectStmt *stmt_; + +private: + DISABLE_COPY_ASSIGN(StmtCompareHelper); }; // NOTE (link.zt) remember to de-construct the struct @@ -185,10 +189,10 @@ struct ObStmtCompareContext : ObExprEqualCheckContext void init(const ObIArray *calculable_items); // for win_magic rewrite - void init(const ObDMLStmt *inner, - const ObDMLStmt *outer, - const ObStmtMapInfo &map_info, - const ObIArray *calculable_items); + int init(const ObDMLStmt *inner, + const ObDMLStmt *outer, + const ObStmtMapInfo &map_info, + const ObIArray *calculable_items); int get_table_map_idx(uint64_t l_table_id, uint64_t r_table_id); @@ -216,6 +220,9 @@ struct ObStmtCompareContext : ObExprEqualCheckContext common::ObSEArray expr_cons_info_; common::ObSEArray const_param_info_; bool is_in_same_stmt_; // only if the two stmts are in the same parent stmt, can we compare table id and column id directly + +private: + DISABLE_COPY_ASSIGN(ObStmtCompareContext); }; class ObStmtComparer diff --git a/src/sql/rewrite/ob_transform_predicate_move_around.h b/src/sql/rewrite/ob_transform_predicate_move_around.h index 392973415..a9afd708d 100644 --- a/src/sql/rewrite/ob_transform_predicate_move_around.h +++ b/src/sql/rewrite/ob_transform_predicate_move_around.h @@ -39,6 +39,9 @@ struct ObTempTableColumnCheckContext : public ObStmtCompareContext { } int64_t first_temp_table_id_; int64_t second_temp_table_id_; + +private: + DISABLE_COPY_ASSIGN(ObTempTableColumnCheckContext); }; class ObTransformPredicateMoveAround : public ObTransformRule diff --git a/src/sql/rewrite/ob_transform_temp_table.cpp b/src/sql/rewrite/ob_transform_temp_table.cpp index 375cd91b7..4ac67b07d 100644 --- a/src/sql/rewrite/ob_transform_temp_table.cpp +++ b/src/sql/rewrite/ob_transform_temp_table.cpp @@ -495,106 +495,108 @@ int ObTransformTempTable::inner_extract_common_subquery_as_cte(ObDMLStmt &root_s int ret = OB_SUCCESS; ObStmtMapInfo map_info; QueryRelation relation; - typedef ObSEArray StmtCompareHelperArray; - SMART_VAR(StmtCompareHelperArray, compare_info) { - //计算相似stmt分组 - if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->allocator_)) { + ObSEArray compare_infos; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->allocator_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect null param", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < stmts.count(); ++i) { + bool find_similar = false; + ObSelectStmt *stmt = stmts.at(i); + if (OB_ISNULL(stmt)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect null param", K(ret)); + LOG_WARN("unexpect null stmt ", K(ret)); } - for (int64_t i = 0; OB_SUCC(ret) && i < stmts.count(); ++i) { - bool find_similar = false; - ObSelectStmt *stmt = stmts.at(i); - if (OB_ISNULL(stmt)) { + for (int64_t j = 0; OB_SUCC(ret) && !find_similar && j < compare_infos.count(); ++j) { + bool has_stmt = false; + bool is_valid = false; + StmtCompareHelper *helper = compare_infos.at(j); + map_info.reset(); + if (OB_ISNULL(helper)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect null stmt ", K(ret)); - } - for (int64_t j = 0; OB_SUCC(ret) && !find_similar && j < compare_info.count(); ++j) { - map_info.reset(); - bool has_stmt = false; - bool is_valid = false; - StmtCompareHelper &helper = compare_info.at(j); - bool check_basic_similarity = !helper.hint_force_stmt_set_.empty() && - helper.hint_force_stmt_set_.has_qb_name(stmt); - if (OB_FAIL(check_has_stmt(helper.similar_stmts_, - stmt, - parent_map, - has_stmt))) { - LOG_WARN("failed to check has stmt", K(ret)); - } else if (has_stmt) { - //do nothing - } else if (OB_FAIL(ObStmtComparer::check_stmt_containment(helper.stmt_, - stmt, - map_info, - relation))) { - LOG_WARN("failed to check stmt containment", K(ret)); - } else if (OB_FAIL(check_stmt_can_extract_temp_table(helper.stmt_, - stmt, - map_info, - relation, - check_basic_similarity, - is_valid))) { - LOG_WARN("failed to check is similar stmt"); - } else if (!is_valid) { - //do nothing - } else if (OB_FAIL(helper.similar_stmts_.push_back(stmt))) { - LOG_WARN("failed to push back stmt", K(ret)); - } else if (OB_FAIL(helper.stmt_map_infos_.push_back(map_info))) { - LOG_WARN("failed to push back map info", K(ret)); - } else { - find_similar = true; - } - } - if (OB_SUCC(ret) && !find_similar) { - SMART_VAR(StmtCompareHelper, helper) { - map_info.reset(); - bool force_no_trans = false; - QbNameList qb_names; - if (OB_FAIL(get_hint_force_set(root_stmt, - *stmt, - qb_names, - force_no_trans))) { - LOG_WARN("failed to get hint set", K(ret)); - } else if (force_no_trans) { - //do nothing - OPT_TRACE("hint reject materialize:", stmt); - } else if (OB_FAIL(ObStmtComparer::check_stmt_containment(stmt, - stmt, - map_info, - relation))) { - LOG_WARN("failed to check stmt containment", K(ret)); - } else if (OB_FAIL(helper.similar_stmts_.push_back(stmt))) { - LOG_WARN("failed to push back stmt", K(ret)); - } else if (OB_FAIL(helper.stmt_map_infos_.push_back(map_info))) { - LOG_WARN("failed to push back map info", K(ret)); - } else if (OB_FAIL(helper.hint_force_stmt_set_.assign(qb_names))) { - LOG_WARN("failed to assign qb names", K(ret)); - } else if (OB_FALSE_IT(helper.stmt_ = stmt)) { - } else if (OB_FAIL(compare_info.push_back(helper))) { - LOG_WARN("failed to push back compare info", K(ret)); - } - } + LOG_WARN("got unexpected NULL ptr", K(ret)); + } else if (OB_FAIL(check_has_stmt(helper->similar_stmts_, stmt, parent_map, has_stmt))) { + LOG_WARN("failed to check has stmt", K(ret)); + } else if (has_stmt) { + //do nothing + } else if (OB_FAIL(ObStmtComparer::check_stmt_containment(helper->stmt_, + stmt, + map_info, + relation))) { + LOG_WARN("failed to check stmt containment", K(ret)); + } else if (OB_FAIL(check_stmt_can_extract_temp_table(helper->stmt_, + stmt, + map_info, + relation, + !helper->hint_force_stmt_set_.empty() && + helper->hint_force_stmt_set_.has_qb_name(stmt), + is_valid))) { + LOG_WARN("failed to check is similar stmt"); + } else if (!is_valid) { + //do nothing + } else if (OB_FAIL(helper->similar_stmts_.push_back(stmt))) { + LOG_WARN("failed to push back stmt", K(ret)); + } else if (OB_FAIL(helper->stmt_map_infos_.push_back(map_info))) { + LOG_WARN("failed to push back map info", K(ret)); + } else { + find_similar = true; } } - //对每组相似stmt创建temp table - for (int64_t i = 0; OB_SUCC(ret) && i < compare_info.count(); ++i) { - StmtCompareHelper &helper = compare_info.at(i); - bool is_happened = false; - OPT_TRACE("try to materialize:", helper.stmt_); - if (!helper.hint_force_stmt_set_.empty() && - !helper.hint_force_stmt_set_.is_subset(helper.similar_stmts_)) { + if (OB_SUCC(ret) && !find_similar) { + StmtCompareHelper *new_helper = NULL; + bool force_no_trans = false; + QbNameList qb_names; + map_info.reset(); + if (OB_FAIL(get_hint_force_set(root_stmt, *stmt, qb_names, force_no_trans))) { + LOG_WARN("failed to get hint set", K(ret)); + } else if (force_no_trans) { + //do nothing + OPT_TRACE("hint reject materialize:", stmt); + } else if (OB_FAIL(StmtCompareHelper::alloc_compare_helper(*ctx_->allocator_, new_helper))) { + LOG_WARN("failed to alloc compare helper", K(ret)); + } else if (OB_ISNULL(new_helper)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect null compare helper", K(ret)); + } else if (OB_FAIL(ObStmtComparer::check_stmt_containment(stmt, stmt, map_info, relation))) { + LOG_WARN("failed to check stmt containment", K(ret)); + } else if (OB_FAIL(new_helper->similar_stmts_.push_back(stmt))) { + LOG_WARN("failed to push back stmt", K(ret)); + } else if (OB_FAIL(new_helper->stmt_map_infos_.push_back(map_info))) { + LOG_WARN("failed to push back map info", K(ret)); + } else if (OB_FAIL(new_helper->hint_force_stmt_set_.assign(qb_names))) { + LOG_WARN("failed to assign qb names", K(ret)); + } else if (OB_FALSE_IT(new_helper->stmt_ = stmt)) { + } else if (OB_FAIL(compare_infos.push_back(new_helper))) { + LOG_WARN("failed to push back compare info", K(ret)); + } + } + } + //对每组相似stmt创建temp table + for (int64_t i = 0; OB_SUCC(ret) && i < compare_infos.count(); ++i) { + StmtCompareHelper *helper = compare_infos.at(i); + bool is_happened = false; + if (OB_ISNULL(helper)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("got unexpected NULL ptr", K(ret)); + } else { + OPT_TRACE("try to materialize:", helper->stmt_); + if (!helper->hint_force_stmt_set_.empty() && + !helper->hint_force_stmt_set_.is_subset(helper->similar_stmts_)) { //hint forbid, do nothing OPT_TRACE("hint reject transform"); - } else if (helper.hint_force_stmt_set_.empty() && - (helper.similar_stmts_.count() < 2)) { - OPT_TRACE("no other similar stmts"); + } else if (helper->hint_force_stmt_set_.empty() && helper->similar_stmts_.count() < 2) { //do nothing - } else if (OB_FAIL(create_temp_table(root_stmt, helper, parent_map, is_happened))) { + OPT_TRACE("no other similar stmts"); + } else if (OB_FAIL(create_temp_table(root_stmt, *helper, parent_map, is_happened))) { LOG_WARN("failed to create temp table", K(ret)); } else { trans_happened |= is_happened; } } + if (NULL != compare_infos.at(i)) { + compare_infos.at(i)->~StmtCompareHelper(); + compare_infos.at(i) = NULL; + } } return ret; } @@ -1598,16 +1600,16 @@ int ObTransformTempTable::apply_temp_table(ObSelectStmt *parent_stmt, OB_ISNULL(view = view_table->ref_query_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpect null param", K(ret), KP(parent_stmt), KP(temp_table_query), KP(view_table)); - } else { - context.init(temp_table_query, view, map_info, &parent_stmt->get_query_ctx()->calculable_items_); - if (OB_FAIL(apply_temp_table_columns(context, map_info, temp_table_query, view, - old_view_columns, new_temp_columns))) { - LOG_WARN("failed to apply temp table columns", K(ret)); - } else if (OB_FAIL(apply_temp_table_select_list(context, map_info, parent_stmt, - temp_table_query, view, view_table, - old_view_columns, new_temp_columns))) { - LOG_WARN("failed to apply temp table select list", K(ret)); - } + } else if (OB_FAIL(context.init(temp_table_query, view, map_info, + &parent_stmt->get_query_ctx()->calculable_items_))) { + LOG_WARN("failed to init context", K(ret)); + } else if (OB_FAIL(apply_temp_table_columns(context, map_info, temp_table_query, view, + old_view_columns, new_temp_columns))) { + LOG_WARN("failed to apply temp table columns", K(ret)); + } else if (OB_FAIL(apply_temp_table_select_list(context, map_info, parent_stmt, + temp_table_query, view, view_table, + old_view_columns, new_temp_columns))) { + LOG_WARN("failed to apply temp table select list", K(ret)); } } // end smart var return ret; diff --git a/src/sql/rewrite/ob_transform_win_magic.cpp b/src/sql/rewrite/ob_transform_win_magic.cpp index a471a20b8..97752ad1e 100644 --- a/src/sql/rewrite/ob_transform_win_magic.cpp +++ b/src/sql/rewrite/ob_transform_win_magic.cpp @@ -515,8 +515,8 @@ int ObTransformWinMagic::sanity_check_and_init(ObDMLStmt *stmt, } else if (view->get_table_size() != map_info.table_map_.count()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("table size does not match to table map count", K(ret)); - } else if (FALSE_IT(context.init(view, stmt, map_info, nullptr))) { - //nvr + } else if (OB_FAIL(context.init(view, stmt, map_info, nullptr))) { + LOG_WARN("failed to init", K(ret)); } return ret; } From a19e44ab4b16ab9393653233d1fb7655f926e70c Mon Sep 17 00:00:00 2001 From: qingsuijiu <642782632@qq.com> Date: Tue, 13 Aug 2024 12:48:01 +0000 Subject: [PATCH 034/249] [adaptive join] HJ's left_join_row_ placeholder --- src/sql/engine/join/ob_hash_join_op.cpp | 6 ++++-- src/sql/engine/join/ob_hash_join_op.h | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/sql/engine/join/ob_hash_join_op.cpp b/src/sql/engine/join/ob_hash_join_op.cpp index 9e45a707d..3331d27d6 100644 --- a/src/sql/engine/join/ob_hash_join_op.cpp +++ b/src/sql/engine/join/ob_hash_join_op.cpp @@ -110,7 +110,8 @@ ObHashJoinSpec::ObHashJoinSpec(common::ObIAllocator &alloc, const ObPhyOperatorT is_ns_equal_cond_(alloc), adaptive_hj_scan_cols_(alloc), adaptive_nlj_scan_cols_(alloc), - is_adaptive_(false) + is_adaptive_(false), + left_join_row_(alloc) { } @@ -125,7 +126,8 @@ OB_SERIALIZE_MEMBER((ObHashJoinSpec, ObJoinSpec), is_ns_equal_cond_, adaptive_hj_scan_cols_, adaptive_nlj_scan_cols_, - is_adaptive_); + is_adaptive_, + left_join_row_); int ObHashJoinOp::PartHashJoinTable::init(ObIAllocator &alloc) { diff --git a/src/sql/engine/join/ob_hash_join_op.h b/src/sql/engine/join/ob_hash_join_op.h index 0c32a83f2..54c1ffc17 100644 --- a/src/sql/engine/join/ob_hash_join_op.h +++ b/src/sql/engine/join/ob_hash_join_op.h @@ -278,6 +278,8 @@ public: ExprFixedArray adaptive_hj_scan_cols_; ExprFixedArray adaptive_nlj_scan_cols_; bool is_adaptive_; + // placeholder + ExprFixedArray left_join_row_; }; // hash join has no expression result overwrite problem: From 39265e53238cbc443ed394b4a7e5427667c41395 Mon Sep 17 00:00:00 2001 From: hanr881 <1741282579@qq.com> Date: Tue, 13 Aug 2024 13:17:32 +0000 Subject: [PATCH 035/249] [CP] to issue<2024072000103866576>:fix udf out param core --- src/sql/engine/expr/ob_expr_udf.cpp | 29 ++++++++++++++++------ src/sql/engine/expr/ob_expr_udf.h | 2 +- src/sql/resolver/expr/ob_raw_expr.h | 6 +++-- src/sql/resolver/expr/ob_raw_expr_util.cpp | 3 ++- 4 files changed, 29 insertions(+), 11 deletions(-) diff --git a/src/sql/engine/expr/ob_expr_udf.cpp b/src/sql/engine/expr/ob_expr_udf.cpp index c04fbc8ec..ddee4c1b4 100644 --- a/src/sql/engine/expr/ob_expr_udf.cpp +++ b/src/sql/engine/expr/ob_expr_udf.cpp @@ -256,9 +256,16 @@ int ObExprUDF::process_in_params(const ObObj *objs_stack, objs_stack[i].copy_value_or_obj(param, true); } else { if (params_type.at(i).get_type() == ObExtendType) { - param.set_extend(objs_stack[i].get_ext(), - objs_stack[i].get_meta().get_extend_type(), objs_stack[i].get_val_len()); - param.set_param_meta(); + if (params_desc.at(i).is_obj_access_pure_out()) { + OZ (pl::ObUserDefinedType::deep_copy_obj(allocator, objs_stack[i], param)); + if (OB_NOT_NULL(deep_in_objs)) { + OZ (deep_in_objs->push_back(param)); + } + } else { + param.set_extend(objs_stack[i].get_ext(), + objs_stack[i].get_meta().get_extend_type(), objs_stack[i].get_val_len()); + param.set_param_meta(); + } } else { void *ptr = NULL; ObObj *obj = NULL; @@ -429,6 +436,13 @@ int ObExprUDF::process_singal_out_param(int64_t i, OX (result.copy_value_or_obj(*obj, true)); OX (result.set_param_meta()); OX (dones.at(i) = true); + } else if (params_desc.at(i).is_obj_access_pure_out()) { // objaccess complex type pure out sence + ObObj &obj = iparams.at(i); + ObObj dst = objs_stack[i]; + if (obj.is_ext() && obj.get_meta().get_extend_type() != pl::PL_REF_CURSOR_TYPE) { + OZ (pl::ObUserDefinedType::deep_copy_obj(alloc, obj, dst, true)); + } + OX (dones.at(i) = true); } return ret; } @@ -449,7 +463,7 @@ int ObExprUDF::process_package_out_param(int64_t idx, for (int64_t i = idx + 1; OB_SUCC(ret) && i < iparams.count(); ++i) { if (!dones.at(i) && iparams.at(i).is_ext()) { bool is_child = false; - OZ (is_child_of(iparams.at(idx), iparams.at(i), is_child)); + OZ (is_child_of(objs_stack[idx], objs_stack[i], is_child)); if (OB_SUCC(ret) && is_child) { OZ (SMART_CALL(process_singal_out_param( i, dones, objs_stack, param_num, iparams, alloc, exec_ctx, nocopy_params, params_desc, params_type))); @@ -480,11 +494,12 @@ int ObExprUDF::process_package_out_param(int64_t idx, OX (result.set_param_meta()); } else { ObObj &obj = iparams.at(idx); + ObObj dst = objs_stack[idx]; if (OB_SUCC(ret) && nullptr != pkg_allocator) { if (obj.is_ext() && obj.get_meta().get_extend_type() != pl::PL_REF_CURSOR_TYPE) { - OZ (pl::ObUserDefinedType::deep_copy_obj(*pkg_allocator, obj, obj, true)); + OZ (pl::ObUserDefinedType::deep_copy_obj(*pkg_allocator, obj, dst, true)); } else { - OZ (deep_copy_obj(*pkg_allocator, obj, obj)); + OZ (deep_copy_obj(*pkg_allocator, obj, dst)); } } if (OB_FAIL(ret)) { @@ -500,7 +515,7 @@ int ObExprUDF::process_package_out_param(int64_t idx, return ret; } -int ObExprUDF::is_child_of(ObObj &parent, ObObj &child, bool &is_child) +int ObExprUDF::is_child_of(const ObObj &parent, const ObObj &child, bool &is_child) { int ret = OB_SUCCESS; if (parent.is_ext() && child.is_ext() && parent.get_ext() == child.get_ext()) { diff --git a/src/sql/engine/expr/ob_expr_udf.h b/src/sql/engine/expr/ob_expr_udf.h index 7dae8aea0..2f06997f9 100644 --- a/src/sql/engine/expr/ob_expr_udf.h +++ b/src/sql/engine/expr/ob_expr_udf.h @@ -161,7 +161,7 @@ public: const common::ObIArray ¶ms_desc, const common::ObIArray ¶ms_type); - static int is_child_of(ObObj &parent, ObObj &child, bool &is_child); + static int is_child_of(const ObObj &parent, const ObObj &child, bool &is_child); static int process_singal_out_param(int64_t i, ObIArray &dones, const ObObj *objs_stack, diff --git a/src/sql/resolver/expr/ob_raw_expr.h b/src/sql/resolver/expr/ob_raw_expr.h index 150650b62..0f29e2ea4 100644 --- a/src/sql/resolver/expr/ob_raw_expr.h +++ b/src/sql/resolver/expr/ob_raw_expr.h @@ -3912,7 +3912,8 @@ public: OBJ_ACCESS_OUT, LOCAL_OUT, PACKAGE_VAR_OUT, - SUBPROGRAM_VAR_OUT + SUBPROGRAM_VAR_OUT, + OBJ_ACCESS_INOUT }; OutType type_; @@ -3934,7 +3935,8 @@ public: OB_INLINE bool is_local_out() const { return OutType::LOCAL_OUT == type_; } OB_INLINE bool is_package_var_out() const { return OutType::PACKAGE_VAR_OUT == type_; } OB_INLINE bool is_subprogram_var_out() const { return OutType::SUBPROGRAM_VAR_OUT == type_; } - OB_INLINE bool is_obj_access_out() const { return OutType::OBJ_ACCESS_OUT == type_; } + OB_INLINE bool is_obj_access_out() const { return OutType::OBJ_ACCESS_OUT == type_ || OutType::OBJ_ACCESS_INOUT == type_; } + OB_INLINE bool is_obj_access_pure_out() const { return OutType::OBJ_ACCESS_OUT == type_; } OB_INLINE int64_t get_index() const { return id1_; } OB_INLINE int64_t get_subprogram_id() const { return id2_; } diff --git a/src/sql/resolver/expr/ob_raw_expr_util.cpp b/src/sql/resolver/expr/ob_raw_expr_util.cpp index 2a6e90e19..25b1ec17b 100644 --- a/src/sql/resolver/expr/ob_raw_expr_util.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_util.cpp @@ -1181,7 +1181,8 @@ do { GET_CONST_EXPR_VALUE(f_expr->get_param_expr(1), var_id); } OZ (udf_raw_expr->add_param_desc( - ObUDFParamDesc(ObUDFParamDesc::OBJ_ACCESS_OUT, var_id, OB_INVALID_ID, pkg_id))); + ObUDFParamDesc(pl::ObPLRoutineParamMode::PL_PARAM_OUT == mode ? ObUDFParamDesc::OBJ_ACCESS_OUT : ObUDFParamDesc::OBJ_ACCESS_INOUT, + var_id, OB_INVALID_ID, pkg_id))); } else if (T_QUESTIONMARK == iexpr->get_expr_type()) { ObConstRawExpr *c_expr = static_cast(iexpr); pl::ObPLDataType param_type; From 7ad58cd3a2751750ca6f604bb287549d4f92a02a Mon Sep 17 00:00:00 2001 From: fengdeyiji <546976189@qq.com> Date: Tue, 13 Aug 2024 13:47:44 +0000 Subject: [PATCH 036/249] [CP] [DeadLock] reset hash holder in elr path --- src/storage/memtable/mvcc/ob_mvcc_row.cpp | 9 +-- src/storage/memtable/ob_memtable.cpp | 73 +++++++---------------- 2 files changed, 24 insertions(+), 58 deletions(-) diff --git a/src/storage/memtable/mvcc/ob_mvcc_row.cpp b/src/storage/memtable/mvcc/ob_mvcc_row.cpp index 7a59dfea2..cae8e1db4 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_row.cpp +++ b/src/storage/memtable/mvcc/ob_mvcc_row.cpp @@ -636,14 +636,9 @@ int ObMvccRow::elr(const ObTransID &tx_id, // TODO shanyan.g if (NULL != key) { wakeup_waiter(tablet_id, *key); + MTL(ObLockWaitMgr*)->reset_hash_holder(tablet_id, *key, tx_id); } else { - ObLockWaitMgr *lwm = NULL; - if (OB_ISNULL(lwm = MTL(ObLockWaitMgr*))) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "MTL(LockWaitMgr) is null", K(ret), KPC(this)); - } else { - lwm->wakeup(tx_id); - } + MTL(ObLockWaitMgr*)->wakeup(tx_id); } } return ret; diff --git a/src/storage/memtable/ob_memtable.cpp b/src/storage/memtable/ob_memtable.cpp index 89c25d234..649282dcf 100644 --- a/src/storage/memtable/ob_memtable.cpp +++ b/src/storage/memtable/ob_memtable.cpp @@ -430,18 +430,6 @@ int ObMemtable::multi_set( if (OB_TMP_FAIL(try_report_dml_stat_(param.table_id_))) { TRANS_LOG_RET(WARN, tmp_ret, "fail to report dml stat", K_(reported_dml_stat)); } - /*****[for deadlock]*****/ - // recored this row is hold by this trans for deadlock detector - if (param.is_non_unique_local_index_) { - // no need to detect deadlock for non-unique local index table - } else { - for (int64_t idx = 0; idx < mtk_generator.count(); ++idx) { - MTL(ObLockWaitMgr*)->set_hash_holder(key_.get_tablet_id(), - mtk_generator[idx], - context.store_ctx_->mvcc_acc_ctx_.get_mem_ctx()->get_tx_id()); - } - } - /***********************/ } return ret; } @@ -595,17 +583,6 @@ int ObMemtable::set( if (OB_TMP_FAIL(try_report_dml_stat_(param.table_id_))) { TRANS_LOG_RET(WARN, tmp_ret, "fail to report dml stat", K_(reported_dml_stat)); } - - /*****[for deadlock]*****/ - // recored this row is hold by this trans for deadlock detector - if (param.is_non_unique_local_index_) { - // no need to detect deadlock for non-unique local index table - } else { - MTL(ObLockWaitMgr*)->set_hash_holder(key_.get_tablet_id(), - mtk_generator[0], - context.store_ctx_->mvcc_acc_ctx_.get_mem_ctx()->get_tx_id()); - } - /***********************/ } return ret; } @@ -667,16 +644,6 @@ int ObMemtable::set( if (OB_TMP_FAIL(try_report_dml_stat_(param.table_id_))) { TRANS_LOG_RET(WARN, tmp_ret, "fail to report dml stat", K_(reported_dml_stat)); } - /*****[for deadlock]*****/ - // recored this row is hold by this trans for deadlock detector - if (param.is_non_unique_local_index_) { - // no need to detect deadlock for local index table - } else { - MTL(ObLockWaitMgr*)->set_hash_holder(key_.get_tablet_id(), - mtk_generator[0], - context.store_ctx_->mvcc_acc_ctx_.get_mem_ctx()->get_tx_id()); - } - /***********************/ } return ret; } @@ -725,15 +692,6 @@ int ObMemtable::lock( if (OB_FAIL(ret) && (OB_TRY_LOCK_ROW_CONFLICT != ret)) { TRANS_LOG(WARN, "lock fail", K(ret), K(row), K(mtk)); } - - if (OB_SUCC(ret)) { - /*****[for deadlock]*****/ - // recored this row is hold by this trans for deadlock detector - MTL(ObLockWaitMgr*)->set_hash_holder(key_.get_tablet_id(), - mtk, - context.store_ctx_->mvcc_acc_ctx_.get_mem_ctx()->get_tx_id()); - /***********************/ - } return ret; } @@ -777,15 +735,6 @@ int ObMemtable::lock( if (OB_FAIL(ret) && (OB_TRY_LOCK_ROW_CONFLICT != ret) && (OB_TRANSACTION_SET_VIOLATION != ret)) { TRANS_LOG(WARN, "lock fail", K(ret), K(rowkey)); } - - if (OB_SUCC(ret)) { - /*****[for deadlock]*****/ - // recored this row is hold by this trans for deadlock detector - MTL(ObLockWaitMgr*)->set_hash_holder(key_.get_tablet_id(), - mtk, - context.store_ctx_->mvcc_acc_ctx_.get_mem_ctx()->get_tx_id()); - /***********************/ - } return ret; } @@ -2667,6 +2616,18 @@ int ObMemtable::multi_set_( if (OB_SUCC(ret)) { mvcc_engine_.finish_kvs(mvcc_rows); + /*****[for deadlock]*****/ + // recored this row is hold by this trans for deadlock detector + if (param.is_non_unique_local_index_) { + // no need to detect deadlock for non-unique local index table + } else { + for (int64_t idx = 0; idx < memtable_keys.count(); ++idx) { + MTL(ObLockWaitMgr*)->set_hash_holder(key_.get_tablet_id(), + memtable_keys[idx], + context.store_ctx_->mvcc_acc_ctx_.get_mem_ctx()->get_tx_id()); + } + } + /***********************/ } if (OB_TRANSACTION_SET_VIOLATION == ret) { @@ -2957,6 +2918,16 @@ int ObMemtable::mvcc_write_( TRANS_LOG(WARN, "register row commit failed", K(ret)); } else if (nullptr == mvcc_row && res.has_insert()) { (void)mvcc_engine_.finish_kv(res); + /*****[for deadlock]*****/ + // recored this row is hold by this trans for deadlock detector + if (param.is_non_unique_local_index_) { + // no need to detect deadlock for non-unique local index table + } else { + MTL(ObLockWaitMgr*)->set_hash_holder(key_.get_tablet_id(), + *key, + context.store_ctx_->mvcc_acc_ctx_.get_mem_ctx()->get_tx_id()); + } + /***********************/ } // cannot be serializable when transaction set violation From 9c97e86f099e18176faeeedcb7a7b43fe04016d1 Mon Sep 17 00:00:00 2001 From: AnimationFan <3067477338@qq.com> Date: Tue, 13 Aug 2024 13:53:30 +0000 Subject: [PATCH 037/249] fix bug, inc direct load error due to not set empty valid macro_block_id --- src/storage/ddl/ob_ddl_inc_redo_log_writer.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/storage/ddl/ob_ddl_inc_redo_log_writer.cpp b/src/storage/ddl/ob_ddl_inc_redo_log_writer.cpp index 3ee0348d5..1a783dc9c 100644 --- a/src/storage/ddl/ob_ddl_inc_redo_log_writer.cpp +++ b/src/storage/ddl/ob_ddl_inc_redo_log_writer.cpp @@ -739,6 +739,9 @@ int ObDDLIncRedoLogWriterCallback::write( redo_info_.type_ = direct_load_type_; redo_info_.trans_id_ = trans_id_; redo_info_.with_cs_replica_ = false; // TODO(chengkong): placeholder for column store replica feature + redo_info_.parallel_cnt_ = 0; // TODO @zhuoran.zzr, place holder for shared storage + redo_info_.cg_cnt_ = 0; + redo_info_.macro_block_id_ = MacroBlockId::mock_valid_macro_id(); if (OB_FAIL(ddl_inc_writer_.write_inc_redo_log_with_retry(redo_info_, macro_block_id_, task_id_, tx_desc_))) { LOG_WARN("write ddl inc redo log fail", K(ret)); } From 91021d27068d0d24b9906fe2046cb411d54f4981 Mon Sep 17 00:00:00 2001 From: KyrielightWei Date: Wed, 14 Aug 2024 02:47:11 +0000 Subject: [PATCH 038/249] restore tx ls state after switch_to_follower_gracefully failed --- src/storage/tx/ob_trans_ctx_mgr_v4.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/storage/tx/ob_trans_ctx_mgr_v4.cpp b/src/storage/tx/ob_trans_ctx_mgr_v4.cpp index 7f84fe754..8e8200223 100644 --- a/src/storage/tx/ob_trans_ctx_mgr_v4.cpp +++ b/src/storage/tx/ob_trans_ctx_mgr_v4.cpp @@ -889,9 +889,11 @@ int ObLSTxCtxMgr::switch_to_follower_gracefully() TRANS_LOG(WARN, "switch state error", KR(ret), K(ls_id_), K(tx_ls_state_mgr_)); } else if (OB_TMP_FAIL(submit_start_working_log_())) { TRANS_LOG(WARN, "submit start working log failed", KR(tmp_ret), K(*this)); - tx_ls_state_mgr_.restore_tx_ls_state(); } if (OB_SUCCESS != tmp_ret) { + //Use a processing method that is compatible with the old code, + //treating the situation as the on_failure of a start working log. + tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::SWL_CB_FAIL); ret = OB_LS_NEED_REVOKE; } TRANS_LOG(WARN, "switch to follower failed", KR(ret), KR(tmp_ret), K(*this)); From 35e02e8f07aa7fe9dea7c2887659481f6d562b36 Mon Sep 17 00:00:00 2001 From: 0xacc Date: Wed, 14 Aug 2024 03:17:17 +0000 Subject: [PATCH 039/249] [to #2024072000103865507] fix: fix memory leak in ALTER routine COMPILE DEBUG --- src/sql/resolver/ddl/ob_alter_routine_resolver.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sql/resolver/ddl/ob_alter_routine_resolver.cpp b/src/sql/resolver/ddl/ob_alter_routine_resolver.cpp index c2c302284..ea5f0c56d 100644 --- a/src/sql/resolver/ddl/ob_alter_routine_resolver.cpp +++ b/src/sql/resolver/ddl/ob_alter_routine_resolver.cpp @@ -413,7 +413,7 @@ int ObAlterRoutineResolver::register_debug_info(const share::schema::ObRoutineIn ObArray subprogram_path; // empty array ObSqlString const_name; - common::ObIArray *pl_dwarf_helpers = nullptr; + common::ObIArray *pl_dwarf_helpers = nullptr; pl::debugger::ObPLDebugger *old_debugger = session_info_->get_pl_debugger(); session_info_->set_pl_debugger(pl_debugger); @@ -455,7 +455,7 @@ int ObAlterRoutineResolver::register_debug_info(const share::schema::ObRoutineIn int64_t line = 1; int64_t address = -1; for (int64_t i = 0; OB_SUCC(ret) && address == -1 && i < pl_dwarf_helpers->count(); ++i) { - jit::ObDWARFHelper* dwarf_helper = pl_dwarf_helpers->at(i); + jit::ObDWARFHelper* dwarf_helper = pl_dwarf_helpers->at(i).first; if (OB_ISNULL(dwarf_helper)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected NULL dwarf helper", KPC(pl_dwarf_helpers)); @@ -501,7 +501,7 @@ int ObAlterRoutineResolver::register_debug_info(const share::schema::ObRoutineIn LOG_WARN("failed to placement new ObDWARFHelper"); } else if (OB_FAIL(dwarf_helper->init())) { LOG_WARN("failed to init dwarf helper", K(ret)); - } else if (OB_FAIL(pl_debugger->get_debugger_ctrl().register_debug_info(dwarf_helper))) { + } else if (OB_FAIL(pl_debugger->get_debugger_ctrl().register_debug_info(dwarf_helper, true))) { LOG_WARN("failed to register debug info", K(ret)); } } From d1cb7a5aeec359398856ed5b43b6f95f4b5726e1 Mon Sep 17 00:00:00 2001 From: haohao022 Date: Wed, 14 Aug 2024 03:23:17 +0000 Subject: [PATCH 040/249] [CP] [to #2024072000103864631] chore: alter get_cursor error message --- src/sql/session/ob_sql_session_info.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sql/session/ob_sql_session_info.cpp b/src/sql/session/ob_sql_session_info.cpp index 2ce377c8e..df0916591 100644 --- a/src/sql/session/ob_sql_session_info.cpp +++ b/src/sql/session/ob_sql_session_info.cpp @@ -1498,7 +1498,7 @@ ObPLCursorInfo *ObSQLSessionInfo::get_cursor(int64_t cursor_id) { ObPLCursorInfo *cursor = NULL; if (OB_SUCCESS != pl_cursor_cache_.pl_cursor_map_.get_refactored(cursor_id, cursor)) { - LOG_WARN_RET(OB_ERR_UNEXPECTED, "get cursor info failed", K(cursor_id), K(get_sessid())); + LOG_INFO("get cursor info failed", K(cursor_id), K(get_sessid())); } return cursor; } From d308c1e016deb4058d51a909ce0185e0a2408d55 Mon Sep 17 00:00:00 2001 From: simonjoylet Date: Wed, 14 Aug 2024 06:55:25 +0000 Subject: [PATCH 041/249] occupy ddl_slice_expr in px_transmit_op --- docs/coding_standard.md | 2571 +++++++++++++++++ docs/logging.md | 364 +++ docs/memory.md | 151 + .../engine/px/exchange/ob_px_transmit_op.cpp | 5 +- .../engine/px/exchange/ob_px_transmit_op.h | 2 + 5 files changed, 3091 insertions(+), 2 deletions(-) create mode 100644 docs/coding_standard.md create mode 100644 docs/logging.md create mode 100644 docs/memory.md diff --git a/docs/coding_standard.md b/docs/coding_standard.md new file mode 100644 index 000000000..6aa5c64f0 --- /dev/null +++ b/docs/coding_standard.md @@ -0,0 +1,2571 @@ + +| Number | Document Version | Revised Chapter | Reason for Revision | Revision Date | +| -------| ---------------- | --------------- | ------------------- | ------------- | +| 1 | 1.0 | | New | June 15th, 2023 | + +# 1 Introduction +This coding standard is applicable to the OceanBase project of Ant Group. It provides some coding constraints and defines coding styles. In the OceanBase project, the kernel code must comply with the coding style of this document, the test code is recommended to comply with the coding constraints of this document, and other codes must also comply with the coding constraints and coding style of this document. + +This coding standard is committed to writing C/C++ code that is easy to understand, reduces traps, and has a unified format. Therefore: + +- The most common and understandable way is used to write the code; +- Avoid using any obscure ways, such as "foo(int x = 1)"; +- Avoid very technical ways, such as "a += b; b = a-b; a -= b;" or "a ^= b; b ^= a; a ^= b;" to exchange the values of variables a and b. + +Finally, this document summarizes the coding constraints for quick reference. +This coding standard will be continuously supplemented and improved as needed. + +# 2 Directory and Files +## 2.1 Directory Structure + +The subdirectories of the OceanBase system are as follows: +- src: contains source code, including header files and implementation files +- unittest: contains unit test code and small-scale integration test code written by developers +- tools: contains external tools +- docs: contains documentation +- rpm: contains RPM spec files +- script: contains operation and maintenance scripts for OceanBase. + +Implementation files for C code are named ".c", header files are named ".h", implementation files for C++ code are named ".cpp", and header files are named ".h". In principle, header files and implementation files must correspond one-to-one, and directories under "src" and "unittest" must correspond one-to-one. All file names are written in lowercase English letters, with words separated by underscores ('_'). + +For example, under the "src/common" directory, there is a header file named "ob_schema.h" and an implementation file named "ob_schema.cpp". Correspondingly, under the "unittest/common" directory, there is a unit test file named "test_schema.cpp". + +Of course, developers may also perform module-level or multi-module integration testing. These testing codes are also placed under the "unittest" directory, but subdirectories and file names are not required to correspond one-to-one with those under "src". For example, integration testing code for the Baseline Storage Engine is placed under the "unittest/storagetest" directory. + +## 2.2 Copyright Information + +Currently (as of May 2023), all source code files in Observer must include the following copyright information in the file header: + +```cpp +Copyright (c) 2021 OceanBase +OceanBase is licensed under Mulan PubL v2. +You can use this software according to the terms and conditions of the Mulan PubL v2. +You may obtain a copy of Mulan PubL v2 at: + http://license.coscl.org.cn/MulanPubL-2.0 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PubL v2 for more details. +``` + +## 2.3 Header File Code + +Header files should not contain implementation code, with the exception of inline functions or C++ templates. Additionally, if the template code is too long, some or all of the template functions can be extracted to a dedicated .ipp file, as demonstrated in "common/ob_vector.h" and "common/ob_vector.ipp". + +Header files should be as concise and clear as possible, making them easy for users to understand. If the implementation code of some functions is short and you wish to place them directly in the header file, the functions must be declared as inline functions and comply with the standards for inline functions. + +## 2.4 Define Protection + +All header files should use `#define` to prevent multiple inclusions of the same file. The naming format should be: \_. For example, the header file "common/ob_schema.h" in the common module should be protected in the following way: + +```cpp +#ifndef OCEANBASE_COMMON_OB_SCHEMA_ +#define OCEANBASE_COMMON_OB_SCHEMA_ +… +#endif // OCEANBASE_COMMON_OB_SCHEMA_ +``` + +## 2.5 Header File Dependencies + +When using other classes in a header file, try to use forward declaration instead of `#include`. + +When a header file is included, new dependencies may be introduced. If this header file is modified, the code will be recompiled. If this header file also includes other header files, any changes to those header files will cause all code that includes that header file to be recompiled. Therefore, we tend to reduce the number of included header files, especially in header files that include other header files. + +Using forward declaration can significantly reduce the number of header files needed. For example, if a header file uses the class "ObFoo" but does not need to access the declaration of "ObFoo", the header file only needs to include the forward declaration `class ObFoo;`, without needing to `#include "ob_foo.h"`. + +Without being able to access the class definition, what operations can we perform on a class "ObFoo" in a header file? +- We can declare the data member type as `ObFoo *` or `ObFoo &`. +- We can declare the function parameter/return value type as ObFoo (but cannot define the implementation). +- We can declare the type of a static data member as ObFoo, because the definition of the static data member is outside the class definition. + +On the other hand, if your class is a subclass of ObFoo or contains a non-static data member of type `ObFoo`, you must include the "ob_foo.h" header file. +Of course, if using pointer members instead of object members reduces code readability or execution efficiency, do not do so just to avoid `#include`. + +## 2.6 Inline Functions + +In order to improve execution efficiency, sometimes we need to use inline functions, but it is important to understand how inline functions work. It is recommended to only use inline functions in performance-critical areas, with the executed code being less than 10 lines and not including loops or switch statements, and without using recursive mechanisms. + +Inline functions are most commonly used in C++ classes to define access functions. On one hand, inlining the function can avoid function call overhead, making the target code more efficient; on the other hand, each inline function call will copy the code, leading to an increase in the total program code. If the code inside the function body is relatively long, or there are loops in the function body, the time to execute the code in the function body will be greater than the overhead of the function call, making it unsuitable for inlining. + +Constructors and destructors of classes can be misleading. They may appear to be short, but be careful as they may hide some behavior, such as "secretly" executing the constructors and destructors of base classes or member objects. + +## 2.7 #include Path and Order + +Header files within a project should be imported according to the project directory tree structure and should not use special paths such as ".", "..", etc. It is recommended to include header files in the following order: the header file corresponding to the current file, system C header files, system C++ header files, other library header files (libeasy, tbsys), and other internal header files of OceanBase, to avoid multiple inclusions. The system C header files use angle brackets and end with ".h", system C++ header files use angle brackets and do not end with ".h", and other cases use quotes. For example: + +```cpp +#include +#include +#include "common/ob_schema.h" +``` + +The reason for placing the header file corresponding to the current file in the priority position is to reduce hidden dependencies. We hope that each header file can be compiled independently. The simplest way to achieve this is to include it as the first .h file in the corresponding .cpp file. + +For example, the include order of "ob_schema.cpp" is as follows: + +```cpp +#include "common/ob_schema.h" + +#include +#include +#include + +#include + +#include "config.h" +#include "tblog.h" + +#include "common/utility.h" +#include "common/ob_obj_type.h" +#include "common/ob_schema_helper.h" +#include "common/ob_define.h" +#include "common/file_directory_utils.h" +``` + +## 2.8 Summary + +The subdirectories in src and unittest correspond to each other, and "tests" is used to store test code. + +Header files should not include implementation code, with the exception of inline functions and templates. + +Use `#define` to protect header files from being included multiple times. + +Reduce compilation dependencies through forward declaration to prevent a change in one file triggering a domino effect. + +Proper use of inline functions can improve execution efficiency. + +The include paths for files within a project should use relative paths, and the include order should be: the header file corresponding to the current file, system C header files, system C++ header files, other library header files (Libeasy, tbsys), and other internal header files of OceanBase. + +# 3 Scope +## 3.1 Namespace +All variables, functions, and classes in the OceanBase source code are distinguished by namespaces, with namespaces corresponding to the directories where the code is located. For example, the namespace corresponding to "ob_schema.h" in the "src/common" directory is "oceanbase::common". + +```cpp +// .h file +namespace oceanbase +{ +// Please do not indent +namespace common +{ +// All declarations should be placed in namespaces, and please do not indent. +class ObSchemaManager +{ +public: + int func(); +}; +} // namespace common +} // namespace oceanbase + +// .cpp file +namespace oceanbase +{ +namespace common +{ +// All function implementations should also be placed in namespaces. +int ObSchemaManager::func() +{ + ... +} + +} // namespace common +} // namespace oceanbase +``` + +**It is prohibited to use anonymous namespaces** because the compiler will assign a random name string to the anonymous namespace, which can affect GDB debugging. + +Both header and implementation files may include references to classes in other namespaces. For example, declare classes in other namespaces in header files, as follows: + +```cpp +namespace oceanbase +{ +namespace common +{ +class ObConfigManager; // Forward declaration of the class common::ObConfigManager +} + +namespace chunkserver +{ + +class ObChunkServer +{ +public: + int func(); +}; + +} // namespace chunkserver +} // namespace oceanbase +``` + +C++ allows the use of using, which can be divided into two categories: + +1. Using directive: For example, using namespace common, which allows the compiler to automatically search for symbols in the common namespace from now on. + +2. Using declaration: For example, using `common::ObSchemaManager`, which makes `ObSchemaManager` equivalent to `common::ObSchemaManager` from now on. + +Because using directive is likely to pollute the scope, **it is prohibited to use it in header files**, but using declaration is allowed. In .cpp files, using directive is allowed, for example, when implementing `ObChunkServer`, it may need to use classes from the common namespace. However, it is important to note that only other namespaces can be introduced using directives in .cpp files. The code in the .cpp file itself still needs to be put in its own namespace. For example: + +```cpp +// incorrect ways of using +// The implementation code should be put in the chunkserver namespace +// instead of using the using namespace chunkserver directive. +namespace oceanbase +{ +using namespace common; +using namespace chunkserver; + +// using symbols from the common namespace +int ObChunkServer::func() +{ + ... +} + +} // namespace oceanbase + +// The correct way is to put the implementation code in the chunkserver namespace. +namespace oceanbase +{ +using namespace common; + +namespace chunkserver +{ +// Using symbols from the common namespace +int ObChunkServer::func() +{ + ... +} + +} // namespace chunkserver +} // namespace oceanbase +``` +## 3.2 Nested Classes +If a class is a member of another class, it can be defined as a nested class. Nested classes are also known as member classes. + +```cpp +class ObFoo +{ +private: + // ObBar is a nested class/member class inside ObFoo, + // and ObFoo is referred to as the host class/outer class." + class ObBar + { + ... + }; +}; +``` + +When a nested class is only used by the outer class, it is recommended to place it within the scope of the outer class to avoid polluting other scopes with the same class name. It is also recommended to forward declare the nested class in the outer class's .h file, and define the nested class's implementation in the .cpp file, to improve readability by avoiding the inclusion of the nested class's implementation in the outer class's .h file. + +Additionally, it is generally advised to avoid defining nested classes as `public`, unless they are part of the external interface. + +## 3.3 Global Variables and Functions +The use of global variables or functions should be strictly limited. New global variables and functions should not be added, except for those that already exist. + +If it is necessary to violate this guideline, please discuss and obtain approval beforehand, and provide detailed comments explaining the reason. + +Global variables and functions can cause a range of issues, such as naming conflicts and uncertainties in the initialization order for global objects. If it is necessary to share a variable globally, it should be placed in a server singleton, such as `ObUpdateServerMain` in `UpdateServer`. + +Global constants should be defined in `ob_define.h`, and global functions should be defined in the `common/ob_define.h` and utility methods (`common/utility.h`, `common/ob_print_utils.h`). + +**It is prohibited to define global const variables in header files.** + +Similar to the reason for prohibiting the use of static variables in header files, global const variables (including constexpr) without explicit extern have internal linkage, and multiple copies will be generated in the binary program. + +**Experimental analysis** +```cpp +// "a.h" +const int zzz = 1000; +extern const int bbb; +``` + +```cpp +// "a.cpp" +#include "a.h" +#include +const int bbb = 2000; +void func1() +{ + printf("a.cpp &zzz=%p\n", &zzz); + printf("a.cpp &bbb=%p\n", &bbb); +} +``` + +```cpp +// "b.cpp" +#include "a.h" +#include +void func2() +{ + printf("b.cpp &zzz=%p\n", &zzz); + printf("b.cpp &bbb=%p\n", &bbb); +} +``` + +```cpp +// "main.cpp" +void func2(); +void func1(); +int main(int argc, char *argv[]) +{ + func1(); + func2(); + return 0; +} +``` + +The compiled and executed program shows that multiple instances of the variable "zzz" are created, while only one instance of the variable "bbb" exists. + +```cpp +[OceanBase224004 tmp]$ ./a.out +a.cpp &zzz=0x4007e8 +a.cpp &bbb=0x400798 +b.cpp &zzz=0x400838 +b.cpp &bbb=0x400798 +``` + +## 3.4 Local Variables +**It is recommended to declare variables at the beginning of a statement block.** + +Simple variable declarations should be initialized when declared. + +OceanBase believes that declaring variables at the beginning of each statement block leads to more readable code. Additionally, OceanBase allows for code such as `for (int i = 0; i < 10; ++i)` where the variable 'i' is declared at the beginning of the loop statement block. If the declaration and use of a variable are far apart, it indicates that the statement block contains too much code, which often means that the code needs to be refactored. + +Declaring variables inside a loop body can be inefficient, as the constructor and destructor of an object will be called each time the loop iterates, and the variable will need to be pushed and popped from the stack each time. Therefore, it is recommended to extract such variables from the loop body to improve efficiency. **It is prohibited to declare complex variables** (e.g. class variables) inside a loop body, but if it is necessary to do so, approval from the team leader must be obtained, and a detailed comment explaining the reason must be provided. For the sake of code readability, declaring references inside a loop body is allowed. + +```cpp +// Inefficient implementation +for (int i = 0; i < 100000; ++i) { + ObFoo f; // The constructor and destructor are called every time the loop is entered + f.do_something(); +} + +// Efficient implementation +ObFoo f; +for (int i = 0; i < 100000; ++i) { + f.do_something(); +} + +// For readability, references can be declared inside the loop +for(int i = 0; i < N; ++i) { + const T &t = very_long_variable_name.at(i); + t.f1(); + t.f2(); + ... +} +``` + +In addition, OceanBase sets limits on the size of local variables and does not recommend defining excessively large local variables. +1. The function stack should not exceed 32K. +2. A single local variable should not exceed 8K. + +## 3.5 Static Variables +**Defining static variables in header files is prohibited** +Initializing static variables (whether const or not) is not allowed in .h header files, except for the following one exception. Otherwise, such static variables will produce a static stored variable in each compilation unit (.o file) and result in multiple instances of static variables after linking. If it is a const variable, it will cause the binary program file to bloat. If it is not a const variable, it may cause severe bugs. + +Note that defining (define) is prohibited, not declaring (declare). + +**[Exception] Static const/constexpr static member variables** + +Static member variables such as const int (including `int32_t`, `int64_t`, `uint32_t`, `uint64_t`, etc.), `static constexpr double`, etc. are often used to define hardcode array lengths. They do not occupy storage, do not have addresses (can be regarded as `#define` macro constants), and are allowed to be initialized in header files. + +Does that mean the following form (pseudocode) is allowed. + +```cpp +class Foo { + static const/constexpr xxx = yyy; +}; +``` + +The explanation for this exception is as follows: In C++98, it is allowed to define the value of a static const integer variable when it is declared. + +```cpp +class ObBar +{ +public: + static const int CONST_V = 1; +}; +``` + +The fact is that the C++ compiler considers the following code equivalent to the previous one. + +```cpp +class ObBar +{ + enum { CONST_V = 1 }; +} +``` + +If the address of this type of variable is taken in the program, an "Undefined reference" error will occur during linking. In such cases, the correct approach is to place the definition of the variable in the .cpp file. + +```cpp +// in the header file +class ObBar +{ + static const int CONST_V; +} +// in the implementation file +const int ObBar::CONST_V = 1; +``` + +Before C\+\+11, the C\+\+98 standard only allowed static const variables of integral type to be initialized with definitions included in the class declaration. In C++11, constexpr is introduced, and static constexpr member variables (including types such as double) can also be initialized in the declaration. This kind of variable will not generate static area storage after compilation. + +> Before C++11, the values of variables could be used in constant expressions only if the variables are declared const, have an initializer which is a constant expression, and are of integral or enumeration type. C++11 removes the restriction that the variables must be of integral or enumeration type if they are defined with the constexpr keyword: +> +> constexpr double earth_gravitational_acceleration = 9.8; +> constexpr double moon_gravitational_acceleration = earth_gravitational_acceleration / 6.0; + +> Such data variables are implicitly const, and must have an initializer which must be a constant expression. + +**Case 1** + +According to the current code style of OceanBase, we will define static variables (such as `ob_define.h`) in the header file, so that each cpp file will generate a declaration and definition of this variable when including this header file. In particular, some large objects (latch, wait event, etc.) generate a static definition in the header file, resulting in the generation of binary and memory expansion. + +Simply move the definition of several static variables from the header file to the cpp file, and change the header file to extern definition, the effect is quite obvious: +binary size: 2.6G->2.4G, reduce 200M. +Observer initial running memory: 6.3G->5.9G, reduced by 400M. + +**Case 2** +In the example below, different cpps see different copies of global variables. It was originally expected to communicate through global static, but it turned out to be different. This will also result in a "false" singleton implementation. + +**Analysis of behavior of static variables** +Let's write a small program to verify the performance of static variable definitions in .h. + +```cpp +// "a.h" +static unsigned char xxx[256]= +{ + 1, 2, 3 +}; +static unsigned char yyy = 10; +static const unsigned char ccc = 100; +``` + +```cpp +// "a.cpp" +#include "a.h" +#include +void func1() +{ + printf("a.cpp &xxx=%p\n", xxx); + printf("a.cpp &yyy=%p\n", &yyy); + printf("a.cpp &ccc=%p\n", &ccc); +} +``` + +```cpp +// "b.cpp" +#include "a.h" +#include +void func2() +{ + printf("b.cpp xxx=%p\n", xxx); + printf("b.cpp &yyy=%p\n", &yyy); + printf("b.cpp &ccc=%p\n", &ccc); +} +``` + +```cpp +// "main.cpp" +void func2(); +void func1(); +int main(int argc, char *argv[]) +{ + func1(); + func2(); + return 0; +} +``` + +Compile and execute, and you can see that whether it is a static integer or an array, whether there is const or not, multiple instances are generated. + +```txt +[OceanBase224004 tmp]$ g++ a.cpp b.cpp main.cpp +[OceanBase224004 tmp]$ ./a.out +a.cpp &xxx=0x601060 +a.cpp &yyy=0x601160 +a.cpp &ccc=0x400775 +b.cpp xxx=0x601180 +b.cpp &yyy=0x601280 +b.cpp &ccc=0x4007a2 +``` + +## 3.6 Resource recovery and parameter recovery +Resource management follows the principle of "who applies for release" and releases resources uniformly at the end of the statement block. If it must be violated, please obtain the consent of the group leader in advance, and explain the reason in detail. +The code structure of each statement block is as follows: +1. Variable definition +2. Resource application +3. Business logic +4. Resource release + +```cpp +// wrong +void *ptr = ob_malloc(sizeof(ObStruct), ObModIds::OB_COMMON_ARRAY); +if (NULL == ptr) { + // print error log +} else { + if (OB_SUCCESS != (ret = do_something1(ptr))) { + // print error log + ob_tc_free(ptr, ObModIds::OB_COMMON_ARRAY); + ptr = NULL; + } else if (OB_SUCCESS != (ret = do_something2(ptr))) { + // print error log + ob_free(ptr, ObModIds::OB_COMMON_ARRAY); + ptr = NULL; + } else { } +} +``` + +```cpp +// correct +void *ptr = ob_malloc(100, ObModIds::OB_COMMON_ARRAY); +if (NULL == ptr) { + // print error log +} else { + if (OB_SUCCESS != (ret = do_something1(ptr))) { + // print error log + } else if (OB_SUCCESS != (ret = do_something2(ptr))) { + // print error log + } else { } +} +// release resource +if (NULL != ptr) { + ob_free(ptr, ObModIds::OB_COMMON_ARRAY); + ptr = NULL; +} +``` + +In the above example, the outermost `if` branch only judges the failure of resource application, and the `else` branch handles the business logic. Therefore, the code for resource release can also be placed at the end of the outermost `else` branch. + +```cpp +// Another correct way of writing requires the if branch +// to simply handle resource application failures +void *ptr = ob_malloc(100, ObModIds::OB_COMMON_ARRAY); +if (NULL == ptr) { + // print error log +} else { + if (OB_SUCCESS != (ret = do_something1(ptr))) { + // print error log + } else if (OB_SUCCESS != (ret = do_something2(ptr))) { + // print error log + } else { } + // release resources + ob_free(ptr, ObModIds::OB_COMMON_ARRAY); + ptr = NULL; +} +``` + +Therefore, if resources need to be released, the resources should be released uniformly before the function returns or at the end of the outermost else branch. + +In some cases, it is necessary to save the input parameters at the beginning of the statement block and restore the parameters in case of an exception. Similar to resource reclamation, parameters can only be restored at the end of a statement block. The most typical example is the serialization function, such as: + +```cpp +// wrong +int serialize(char *buf, const int64_t buf_len, int64_t &pos) +{ + int ret = OB_SUCCESS; + const int64_t ori_pos = pos; + + if (OB_SUCCESS != (ret = serialize_one(buf, buf_len, pos)) { + pos = ori_pos; + ... + } else if (OB_SUCCESS != (ret = serialize_two(buf, buf_len, pos)) { + pos = ori_pos; + ... + } else { + ... + } + return ret; +} +``` + +The problem with this usage is that it is likely to forget to restore the value of pos in a certain branch. The correct way to write it is as follows. + +```cpp +// Correct +int serialize(char *buf, const int64_t buf_len, int64_t &pos) +{ + int ret = OB_SUCCESS; + const int64_t ori_pos = pos; + + if (OB_SUCCESS != (ret = serialize_one(buf, buf_len, pos)) { + ... + } else if (OB_SUCCESS != (ret = serialize_two(buf, buf_len, pos)) { + ... + } else { + ... + } + + if (OB_SUCCESS != ret) { + pos = ori_pos; + } + return ret; +} +``` + +So if you need to restore the input parameters, do so before the function returns. + +## 3.7 Summary + +1. Namespaces correspond to directories. **Anonymous namespaces are prohibited**. **Using directives are prohibited in .h files, and only using declarations are allowed**. +2. Nested classes are suitable for scenarios that are only used by external classes. It is recommended to pre-declare in .h files and implement them in .cpp files. Try not to use public. +3. **In addition to the existing global variables and global functions, no new global variables and global functions shall be added**. If it must be violated, please obtain the consent of the group leader in advance, and explain the reason in detail. +4. **Local variables are declared at the beginning of the statement block, and it is mandatory to initialize simple variables when they are declared.** +5. **It is forbidden to declare non-simple variables in the loop body**. If it must be violated, please obtain the consent of the group leader in advance, and explain the reason in detail. +6. **Resource management follows the principle of "who applies for release"**. If resources need to be released, release them before the function returns or at the end of the outermost else branch. So if you need to restore the input parameters, do so before the function returns. If it must be violated, please obtain the consent of the group leader in advance, and explain the reason in detail. + +# 4 Class +## 4.1 Constructors and Destructors +The constructor only performs trivial initialization work, such as initializing pointers to `NULL` and variables to 0 or -1. Non-trivial initialization operations are not allowed in the constructor. If necessary, define a separate `int init()` method and add an `is_inited_` variable to identify whether the object has been initialized successfully. This is because, if object construction fails, an indeterminate state may occur. + +Every class (including interface classes) is required to define a constructor, even if the class does not have any member variables, it also needs to define an empty default constructor. This is because, if no constructor is defined, the compiler will automatically generate a default constructor, which often has some side effects, such as initializing some member variables to random values. + +Every class (including interface classes) is required to define a destructor, even if the class does not have any member variables, it also needs to define an empty destructor. In addition, if there is no special reason (performance is particularly critical, it will not be inherited and does not contain virtual functions), the destructor of the class should be declared as virtual. + +## 4.2 explicit keyword +Use the C++ keyword `explicit` for single-argument constructors. + +Usually, a constructor with only one parameter can be used for conversion. For example, if `ObFoo::ObFoo(ObString name)` is defined, when an `ObString` is passed to a function that needs to pass in an `ObFoo` object, the constructor `ObFoo::ObFoo( ObString name)` will be automatically called and the string will be converted to a temporary `ObFoo` object passed to the calling function. This implicit conversion always brings some potential bugs. + +## 4.3 Copy Constructor +**In principle, the copy constructor should not be used (except for the base classes that have already been defined)**. If it must be violated, please obtain the consent of the group leader in advance, and explain the reason in detail. In addition, classes that do not need a copy constructor should use the `DISALLOW_COPY_AND_ASSIGN` macro (`ob_define.h`), except for interface classes. + +```cpp +#define DISALLOW_COPY_AND_ASSIGN(type_name) \ + type_name(const type_name&) \ + void operator=(const type_name&) + +class ObFoo +{ +public: + ObFoo(); + ~ObFoo(); + +private: + DISALLOW_COPY_AND_ASSIGN(ObFoo); +}; +``` + +## 4.4 reuse & reset & clear + +**`reset` is used to reset the object, `reuse` is used to reuse the object, and clear is prohibited**. described as follows. + +1. Both `reset` and `reuse` are used for object reuse. This `reuse` is often introduced in order to optimize performance because the memory allocation and construction of objects of certain key classes is too time-consuming. +2. The meaning of reset is to restore the state of the object to the initial state after the execution of the constructor or init function. Refer to `ObRow::reset`; +3. Use `reuse` in other situations except `reset`. Unlike `reset`, `reuse` often does not release some resources that are time-consuming to reapply, such as memory. Refer to `PageArena::reuse`; +4. `clear` is widely used in the container class of C++ STL, and often means to clear the size of the container class to 0, but it does not clear the internal objects or release the memory. The difference between `clear` and `reset/reuse` is very subtle. In order to simplify understanding, the use of `clear` is prohibited, and the used ones are gradually removed. + +## 4.5 Member Initialization +**All members must be initialized, and the order in which member variables are initialized is consistent with the order in which they are defined.** + +The `constructor`, `init` method, and `reset/reuse` method of each class may perform some initialization operations on the class, and it is necessary to ensure that all members have been initialized. If the constructor only initializes some members, then the `is_inited_` variable must be set to `false`, and the `init` method will continue to complete the initialization of other members. The members of the struct type can be initialized by the `reset` method (if the initialization is only to clear the struct members to 0, they can also be initialized by using `memset`); the members of the class type can be initialized by `init/reset/reuse` and other methods. +The initialization order of member variables needs to be consistent with the definition order. The advantage of this is that it is easy to find out whether you have forgotten which members have been initialized. + +## 4.6 Structures and Classes +Use `struct` only when there is only data, and use `class` for everything else. + +Struct is used on passive objects that only contain data, which may include associated constants, and `reset/is_valid`, serialization/deserialization these general functions. If you need more functions, `class` is more suitable. If in doubt, use class directly. + +If combined with STL, you can use `struct` instead of `class` for functor and traits. + +It should be noted that the data members inside the `class` can only be defined as `private` (private, except for static members), and can be accessed through the access functions `get_xxx` and `set_xxx`. + +## 4.7 Common Functions +The common functions contained in each class must adopt standard prototypes, and the serialization/deserialization functions must be implemented using macros. +The general functions contained in each class include: `init`, `destroy`, `reuse`, `reset`, `deep_copy`, `shallow_copy`, `to_string`, `is_valid`. The prototypes of these functions are as follows: + +```cpp +class ObFoo +{ +public: + int init(init_param_list); + bool is_inited(); + void destroy(); + + void reset(); + void reuse(); + + int deep_copy(const ObFoo &src); + int shallow_copy(const ObFoo &src); + + bool is_valid(); + + int64_t to_string(char *buf, const int64_t buf_len) const; + + NEED_SERIALIZE_AND_DESERIALIZE; +}; +``` + +It should be noted that `to_string` will always add '\0' at the end, and the function returns the actual printed byte length (excluding '\0'). It is implemented internally by calling `databuff_printf` related functions, please refer to `common/ob_print_utils.h` for details. + +Serialization and deserialization functions need to be implemented through macros, for example: + +```cpp +class ObSort +{ +public: + NEED_SERIALIZE_AND_DESERIALIZE; +private: + common::ObSArray sort_columns_; + int64_t mem_size_limit_; + int64_t sort_row_count_; +}; +``` + +For class `ObSort`, the three fields that need to be serialized are +`sort_columns_`, `mem_size_limit_`, `sort_row_count_`. Just write in `ob_sort.cpp`: + +```cpp +DEFINE_SERIALIZE_AND_DESERIALIZE(ObSort, sort_columns_, mem_size_limit_, sort_row_count_); +``` + +The serialization and deserialization can be completed and the realization of the three functions that have calculated the length after serialization can be completed. +The generic functions of structures are the same as the generic functions of classes. + +## 4.8 Common Macro +For the convenience of coding, some defined macros can be used in OceanBase, but it is not recommended for students to add macros by themselves. If it is really necessary to add macros, please confirm with the team leader before adding them. + +Here are some commonly used macros: + +1. `OB_SUCC` + + It is usually used to judge whether the return value is `OB_SUCCESS`, which is equivalent to `OB_SUCCESS == (ret = func())`. Note that ret needs to be pre-defined in the function when using `OB_SUCC`, for example, the following writing method. + + ```cpp + ret = OB_SUCCESS; + if (OB_SUCC(func())) { + // do something + } + ``` + +2. `OB_FAIL` + + It is usually used to judge whether the return value is not `OB_SUCCESS`, which is equivalent to `OB_SUCCESS != (ret = func())`. Note that ret needs to be pre-defined in the function when using `OB_FAIL`, for example, the following writing method. + + ```cpp + ret = OB_SUCCESS; + if (OB_FAIL(func())) { + // do something + } + ``` + +3. `OB_ISNULL` + + It is usually used to judge whether the pointer is empty, which is equivalent to nullptr ==, for example, the following writing method. + + ```cpp + if (OB_ISNULL(ptr)) { + // do something + } + ``` + +4. `OB_NOT_NULL` + + It is usually used to judge whether the pointer is not empty, which is equivalent to nullptr !=, for example, the following writing method + ```cpp + if (OB_NOT_NULL(ptr)) { + // do something + } + ``` + +5. `IS_INIT` + + It is usually used to judge whether the class has been initialized, which is equivalent to `is_inited_`. Note that the member `is_inited_` needs to exist in the class, for example, the following writing method. + ```cpp + if (IS_INIT) { + // do something + } + ``` + +6. `IS_NOT_INIT` + + It is usually used to judge whether the class has been initialized, which is equivalent to `!is_inited_`. Note that the member `is_inited_` needs to exist in the class, for example, the following writing method. + + ```cpp + if (IS_NOT_INIT) { + // do something + } + ``` + +7. `REACH_TIME_INTERVAL` + + It is used to judge whether a certain time interval has been exceeded. The parameter is `us`. Note that there will be a static variable to record the time inside the macro, so the judgment of time is global. It is usually used to control the log output frequency. For example, the following writing method will Let the system do some actions after more than 1s interval. + ```cpp + if (REACH_TIME_INTERVAL(1000 * 1000)) { + // do something + } + ``` + +8. `OZ` + + It is used to simplify the log output after `OB_FAIL`. When you only need to simply output the log after an error is reported, you can use `OZ`. Note that when using `OZ`, you need to define `USING_LOG_PREFIX` at the beginning of the cpp file. For example, the following writing method. + + ```cpp + OZ(func()); + ``` + + Equivalent to + + ```cpp + if (OB_FAIL(func())) { + LOG_WARN("fail to exec func, ", K(ret)); + } + ``` + +9. `K` + + Usually used for log output, output variable name and variable value, such as the following writing. + ```cpp + if (OB_FAIL(ret)) { + LOG_WARN("fail to exec func, ", K(ret)); + } + ``` + +10. `KP` + + Usually used for log output, output variable names and pointers, such as the following writing method. + ```cpp + if (OB_FAIL(ret)) { + LOG_WARN("fail to exec func, ", K(ret), KP(ptr)); + } + ``` + +## 4.9 Inherit + +All inheritance must be `public`, and inheritance must be used with care: use inheritance only if it "is one", use composition if it "has one". + +When a subclass inherits from a parent class, the subclass contains all the data and operation definitions of the parent class. In C++ practice, inheritance is mainly used in two scenarios: implementation inheritance, where the subclass inherits the implementation code of the parent class; interface inheritance, where the subclass inherits the method name of the parent class. For implementation inheritance, because the code that implements the subclass is extended between the parent class and the subclass, it becomes more difficult to understand its implementation, and it should be used with caution. + +Multiple inheritance is also used in OceanBase. The scenario is rare, and requires at most one base class to contain implementation, and other base classes are pure interface classes. + +## 4.10 Operator Overloading +**Except for container classes, custom data types (`ObString`, `ObNumber`, etc.) and a few global basic classes such as `ObRowkey`, `ObObj`, `ObRange`, etc., do not overload operators (except simple structure assignment operations)**. If it must be violated, please discuss and approve it in advance, and explain the reasons in detail. + +C++ STL template classes have a large number of overloaded operators, such as comparison functions, four operators, self-increment, and self-decrement. Such codes seem to be more intuitive, but in fact they often confuse the caller, such as making the caller mistaken for some time-consuming Operations are as efficient as built-in operations. + +Avoid overloading the assignment operator (operator=) for anything but simple constructs. If necessary, copy functions such as `deep_copy`, `shallow_copy`, etc. can be defined. Among them, `deep_copy` indicates that all members need deep copy, and `shallow_copy` indicates other situations. If some members need a shallow copy and some need a `deep copy`, then use `shallow_copy`. + +## 4.11 Declaration Order +Use a specific declaration order in the header file, public before private, and member functions before data members. + +The order of definition is as follows: `public` block, `protected` block, `private` block, and the internal order of each block is as follows: +1. typedefs and enums; +2. constant; +3. constructor; +4. destructor; +5. For member functions, static member functions come first, and ordinary member functions follow; +6. For data members, static data members come first, and ordinary data members follow; +The macro `DISALLOW_COPY_AND_ASSIGN` is placed after the `private:` block as the last part of the class. + +The function definitions in the .cpp file should be as consistent as possible with the declaration order in the .h. + +The reason why the constant definition should be placed in front of the function definition (constructor/destructor, member function) instead of in the data member is because the constant may be referenced by the function. + +## 4.12 Summary +1. The constructor only does trivial initialization. Each class needs to define at least one constructor, and the destructor with virtual function or subclass is declared as virtual. +2. In order to avoid implicit type conversion, the single-argument constructor needs to be declared as explicit. +3. **In principle, the copy constructor shall not be used (except for the base classes that have been defined and used)**. If it must be violated, please discuss and approve it in advance, and explain the reasons in detail. +4. Use `DISALLOW_COPY_AND_ASSIGN` to avoid abuse of copy constructor and assignment operation; +5. **Use reset for class reset, reuse for reuse, and clear for prohibition.** +6. **It is necessary to ensure that all members are initialized, and the initialization sequence of member variables is consistent with the definition sequence.** +7. Use struct only when there is only data, and use class in all other cases. +8. The common functions contained in each class must use standard prototypes, and the serialization/deserialization functions must be implemented using macros. +9. Prefer composition and only use inheritance for "is-a" relationships. Avoid private inheritance and multiple inheritance. When multiple inheritance is used, it is required that except for one base class with implementation, the other base classes are pure interface classes. +10. **Except for existing container classes, custom types, and a small number of global basic classes, overloading of operators is not allowed (except for simple structure assignment operations)**. If it must be violated, please obtain the consent of the group leader in advance, and explain the reason in detail. +11. Declaration order: `public`, `protected`, `private`. + +# 5 Function +## 5.1 Single Entry and Single Exit +**It is mandatory for all functions to return at the end, and it is forbidden to call global jump instructions such as `return`, `goto`, and exit halfway. If it must be violated, please discuss it in advance and explain the reason in detail.** + +OceanBase believes that large-scale project development should give priority to avoiding common pitfalls, and it is worthwhile to sacrifice programming complexity. The single entry and single exit can make it difficult for developers to forget to release resources or restore function input parameters. At any time, the function is required to have only one exit. + +## 5.2 Function Return Value +**Except for the following exceptions, the function must return the ret error code:** +1. Simple access function `set_xxx()/get_xxx()`. If the `set/get` function is complex or error-prone, it must still return an error code. +2. The `at(i)` function that has been defined and used (to define and use a new one, please obtain the consent of the team leader in advance, and explain the reason in detail). +3. Operator overloading that has been defined and used (to define and use new ones, please obtain the consent of the team leader in advance, and explain the reasons in detail). +4. For other small amount of functions, such as general function `void reset(); void reuse();` etc., refer to section 4.7 general function. + +**The function caller must check the return value (error code) of the function and handle it.** + +**Only ret variables of type int can be used to represent errors, and ret can only represent errors (except for iterator functions due to historical reasons).** If you need to return other types of values, such as the compare function returning a value of type bool, you need to use other variable names, such as `bool_ret`. For example: + +```cpp +// wrong +bool operator()(const RowRun &r1, const RowRun &r2) const +{ + bool ret = false; + int err = do_something(); + return ret; +} + +// correct +bool operator()(const RowRun &r1, const RowRun &r2) const +{ + bool bool_ret = false; + int ret = do_something(); + return bool_ret; +} +``` + +If some error codes need to be temporarily saved during function execution, try to use variables with clear meanings, such as `hash_ret` and `alloc_ret`. If the meaning is not clear, then `ret1` and `ret2` can also be used in sequence to avoid confusion caused by using err to represent error codes. For example: + +```cpp +int func() +{ + int ret = OB_SUCCESS; + + ret = do_something(); + if (OB_SUCCESS != ret) { + int alloc_ret = clear_some_resource (); + if (OB_SUCCESS != alloc_ret) { + // print error log + } + } else { + ... + } + return ret; +} +``` + +## 5.3 Sequential Statement +If multiple sequential statements are doing the same thing, condensed writing can be used in some cases. +This makes sequential code tedious due to the need to judge errors during function execution. For example: + +```cpp +// verbose code +int ret = OB_SUCCESS; + +ret = do_something1(); +if (OB_SUCCESS != ret) { + // print error log +} + +if (OB_SUCCESS == ret) { + ret = do_something2(); + if (OB_SUCCESS != ret) { + // print error log + } +} +// more code ... +``` + +It can be seen that there are only two lines of code that are really effective, but the overall code size is several times that of the effective code. This will make a screen contain too little valid code, affecting readability. +If each step in the sequence statement requires only one line of code, it is recommended to simplify the code in the following way: + +```cpp +// Use shorthand when there is only one line of code per step in sequential logic +int ret = OB_SUCCESS; + +if (OB_FAIL(do_something1())) { + // print error log +} else if (OB_FAIL(do_something2())) { + // print error log +} else if (OB_SUCCESS != (ret = do_something3())) { + // print error log +} else { } +``` + +If some steps of the sequence statement take more than one line of code, then some changes are required: + +```cpp +// When some steps in the sequential logic exceed one line of code, +// use simplified writing and make certain changes +int ret = OB_SUCCESS; + +if (OB_SUCCESS != (ret = do_something1())) { + // print error log +} else if (OB_SUCCESS != (ret = do_something2())) { + // print error log +} else { + // Step 3 executes more than one line of code + if (OB_SUCCESS != (ret = do_something3_1())) { + // print error log + } else if (OB_SUCCESS != (ret = do_something3_2()) { + // print error log +} else { } +} + +if (OB_SUCCESS == ret) { // start a new logic + if (OB_SUCCESS != (ret = do_something4())) { + // print error log + } else if (OB_SUCCESS != (ret = do_something5())) { + // print error log + } else { } +} +``` + +In the actual coding process, when should concise writing be used? OceanBase believes that when each step of a sequential statement has only one line of statement, and these steps are logically coupled tightly, the concise writing method should be used as much as possible. However, if it logically belongs to multiple sections, each of which does a different thing, then brevity should only be used within each section, not brevity for the sake of brevity. +It should be noted that if the sequential statement is followed by a conditional statement. If sequential statements are reduced to conditional statements, then they cannot be combined into one large conditional statement, but should be separated in code structure. For example: + +```cpp +// wrong +if (OB_SUCCESS != (ret = do_something1())) { + // print error log +} else if (OB_SUCCESS != (ret = do_something2())) { + // print error log +} else if (cond) { + // do something if cond +} else { + // do something if !cond +} + +// The first correct way +if (OB_SUCCESS != (ret = do_something1())) { + // print error log +} else if (OB_SUCCESS != (ret = do_something2())) { + // print error log +} else { + if (cond) { + // do something if cond + } else { + // do something if !cond + } +} + +// The second correct way +if (OB_SUCCESS != (ret = do_something1())) { + // print error log +} else if (OB_SUCCESS != (ret = do_something2())) { + // print error log +} else { } + +if (OB_SUCCESS == ret) { + if (cond) { + // do something if cond + } else { + // do something if !cond + } +} +``` +## 5.4 Loop Statement +Judge `OB_SUCCESS == ret` in the loop condition to prevent the error code from being overwritten. +OceanBase has discovered a large number of problems where error codes are covered. These problems often lead to serious consequences, such as data inconsistency, and are very difficult to find. For example: +```cpp +// error code is overwritten +for (int i = 0; i < 100; ++i) { + ret = do_something(); + if (OB_SUCCESS != ret) { + // print error log + } else { + ... + } +} +``` +In the above example, the `if` branch in the for loop is wrong, but forget to break, so that the code will enter the next loop, and the error code of the previous execution will be overwritten. +Therefore, the standard for loop statement is written as follows: +```cpp +for (int i = 0; OB_SUCCESS == ret && i < xxx; ++i) { + ... +} +``` +In addition, the standard `while` loop statement is written as follows: +```cpp +while (OB_SUCCESS == ret && other_cond) { + ... +} +``` +A `break` or `continue` may be used in a loop statement to change the execution path. OceanBase believes that it should be used as little as possible, which is the same as the principle of single entry and single exit of a function. It is equivalent to the input source of the subsequent code of the loop statement being multiple entries, which increases the complexity of the code. If it is really necessary to use `break` and `continue`, it is required to explain the reason in detail through comments, and you need to pay special attention when writing code or reviewing code. In addition, considering that the input source of the follow-up code is multiple entries, it is necessary to ensure that it is clear what conditions the input of the follow-up code satisfies. + +## 5.5 Conditional Statements +Conditional statements need to follow the MECE principle. +The word MECE comes from the McKinsey analysis method, which means mutually independent and completely exhaustive (Mutually Exclusive Collectively Exhaustive). In principle, every `if/else` branch of a conditional statement needs to fully exhaust all possibilities. +Some bad programming style, such as: +```cpp +// bad programming style +if (OB_SUCCESS != ret && x > 0) { + // do something +} + +if (OB_SUCCESS == ret && x < 0 ) { + // do something +} + +if (OB_SUCCESS == ret && x == 0) { + // do something +} +``` + +Such code does not conform to the MECE principle, it is difficult to analyze whether all possibilities are exhausted, and it is easy to miss some scenarios. +If there is only one condition, the correct way to write it is: +```cpp +// The correct way to write a judgment condition +if (cond) { + // do something +} else { + // do something +} +``` +In principle, every `if/else` branch is complete, even if the last else branch does nothing. However, there is one exception. If the if condition is just some error judgment or parameter checking, and there is no other logic, then the else branch can be omitted. +```cpp +// The if statement is only to judge the error code, the else branch can be omitted +if (OB_SUCCESS != ret) { + // handle errors +} +``` +If two judgment conditions are included, compare the following two possible writing methods: +```cpp +// The first way of writing the two judgment conditions (correct) +if (cond1) { + if (cond2) { + // do something + } else { + // do something + } +} else { + if (cond2) { + // do something + } else { + // do something + } +} + +// The second way of writing the two judgment conditions (wrong) +if (cond1 && cond2) { + // do something +} else if (cond1 && !cond2) { + // do something +} else if (!cond1 && cond2) { + // do something +} else { + // do something +} +``` + +The first writing method is divided into two layers, and the second writing method is divided into one layer. OceanBase only allows the first writing method. Of course, `cond1` and `cond2` here are from the perspective of business logic, referring to two independent business logics, rather than saying that `cond1` and `cond2` cannot contain `&&` or `||` operators. For example: + +```cpp +// Whether app_name is empty, including || +if (NULL == app_name || app_name[0] == '\0') { + ... +} + +// Judging whether table_name or column_name is empty, it is considered a business logic +if (NULL != table_name || NULL != column_name) { + ... +} +``` + +In any case, the number of `if/else` branches in each layer should not exceed 5. Why choose 5? This number also comes from the McKinsey analysis method. Generally speaking, the branch logic of the same level is generally between 3 and 5. If it exceeds, the division is often unreasonable. + +## 5.6 Const Declaration +Declare function parameters that do not change as const. In addition, if the function does not modify member variables, it should also be declared as a const function. +Declaring parameters as const can avoid some unnecessary errors, such as constant parameters being changed due to code errors. For simple data type value transfer, many people have disputes about whether to declare const, because in this case, declaring const has no effect. +Considering that most of the existing code in OceanBase has been declared as const, and it is easier to operate this way, as long as the function parameters do not change, they are uniformly declared as const. +## 5.7 Function Parameters +The number of function parameters should not exceed 7. The recommended order is: input parameters first, output parameters last. If some parameters are both input parameters and output parameters, they will be treated as input parameters and placed at the front like other input parameters. Add a new The parameters also need to follow this principle. + +**The principle of coding: don't trust anyone in the code! Every function (whether `public` or `private`, except `inline` functions) must check the legality of each input parameter, and it is strongly recommended that inline functions also perform these checks (unless there are serious performance problems)**. All functions (whether `public` or `private`) must check the legality of values obtained from class member variables or through function calls (such as get return values or output parameters), even if the return value is successful, the legality of output parameters must still be checked . Variable (parameter) check, only needs to be checked once in a function (if the value obtained by calling one or several functions multiple times, then check each time). +These checks include but are not limited to: + +1. Whether the pointer is `NULL`, and whether the string is empty +2. Whether the parameter value of the numeric type exceeds the value range. In particular, whether the subscript of the `array/string/buffer` is out of bounds +3. Whether the parameter of the object type is valid. Generally, the object can define a `bool is_valid()` method (refer to `common::TableSchema`) + +If an implicit check has been made within the function, for example by a check function, it should be stated where the variable is assigned. For example: + +```cpp +// Variables that have been implicitly checked should be explained +// where the variable is assigned: +if (!param.is_valid() || !context.is_valid()) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Invalid argument", K(ret), K(param), K(param)); + } else { + // block_cache_ not empty has been checked in the previous context.is_valid() + ObMicroBlockCache *block_cache = context.cache_context_.block_cache_; + ... +} +``` + +Use the `if` statement to check the validity of the input parameters (the function itself) and the output parameters (the function caller), and prohibit the use of `assert` and the previously defined `OB_ASSERT` macro at any time. +Examples are as follows: + +```cpp +// function that needs to return an error +int _func(void *ptr) +{ + int ret = OB_SUCCESS; + + if (NULL == ptr) { + // print error log + ret = OB_INVALID_ARGUMENT; + } + else { + // Execute business logic + } + return ret; +} +``` + +## 5.8 Function Call +When calling the function, you should try to avoid passing in some meaningless special values, such as `NULL`, `true/false`, `0/-1`, etc., and use constants instead. If you must pass in a special value, you need to use annotation instructions. + +For example: + +```cpp +// wrong +int ret = do_something(param1, 100, NULL); + +// Correct +ObCallback *null_callback = NULL; +int ret = do_something(param1, NUM_TIMES, null_callback); +``` + +## 5.9 Pointer or Reference +Function parameters can choose pointers or references. Use more references while respecting idioms. + +Pointer parameters and reference parameters can often achieve the same effect. Considering that the OceanBase coding specification has strict requirements for error judgment, more references are used to reduce some redundant error judgment codes. Provided, of course, that idioms are followed, such as: + +1. The method of applying for an object often returns a pointer, and the corresponding release method also passes in a pointer. +2. If the member of the object is a pointer, the corresponding `set_xxx` input is also a pointer. + +## 5.10 Function Length +It is mandatory that a single function does not exceed 120 lines. If it must be violated, please obtain the consent of the group leader in advance, and explain the reasons in detail. + +Most open source projects limit the number of lines of a single function. Generally speaking, functions with more than 80 lines are often inappropriate. Considering that OceanBase has a lot of redundant error judgment codes, a single function is limited to no more than 120 lines. If the function is too long, consider splitting it into several smaller and more manageable functions, or re-examine the design and modify the structure of the class. + +## 5.11 Summary +1. **Strictly abide by the single-entry single-exit function. If it must be violated, please obtain the consent of the project leader and project architect in advance, and explain the reasons in detail.** +2. **Except for the simple access functions `set_xxx()/get_xxx()` and a few exceptions (such as operator overloading, existing at(i) functions, general function `reset()/reuse()` of classes, etc.), all functions (public and private) should use `ret` to return the error code. If the set/get is complicated or may make an error, `ret` should still be used to return the error code.** Only ret variables of type int can be used to represent errors, and ret can only represent errors (except for iterator functions due to historical reasons). +3. If multiple sequential statements are doing the same thing, then, in some cases, you can use simplified writing. +4. Judge `OB_SUCCESS == ret` in the loop condition to prevent the error code from being overwritten. +5. The conditional statement needs to abide by the MECE principle: each condition is independent of each other and completely exhausted, and the number of branches of a single `if/else` should not exceed 5 as far as possible. +6. Declare functions/function parameters as const whenever possible. +7. **The principle of coding: do not trust anyone in the code! Every function (whether public or private, except inline functions) must check the legality of each input parameter, and it is strongly recommended that inline functions also perform these checks (unless there are serious performance problems). All functions (whether public or private) must check the legality of values obtained from class member variables or through function calls (such as get return values or output parameters), even if the return value is successful, the legality of output parameters must still be checked . Variable (parameter) check, only needs to be checked once in a function (if the value obtained by calling one or several functions multiple times, then check each time).** When defining functions, the recommended order is: input parameters first, output parameters last. +8. **Prohibit the use of assert and OB_ASSERT.** +9. When calling the function, you should try to avoid passing in some meaningless special values, and use constants instead. +10. On the premise of respecting idiomatic usage, use more references. +11. It is mandatory that a single function does not exceed 120 lines. If it must be violated, please obtain the consent of the group leader in advance, and explain the reasons in detail. + +# 6 C&C++ Features +The advantage of C++ is flexibility, and the disadvantage is also flexibility. For many functions of C++, OceanBase is conservative, and this section describes some of them. There are two principles for choosing these features: +1. Principle of caution: This feature is relatively "safe", even for beginners, there are not too many "pitfalls" +2. Necessity: It has "sufficient" benefits to improve the coding quality of OceanBase + +## 6.1 Smart Pointers and Resource Guard + +Smart pointers are not allowed, allowing automatic release of resources through the Guard class. +The boost library supports smart pointers, including `scoped_ptr`, `shared_ptr`, and `auto_ptr`. Many people think that smart pointers can be used safely, especially `scoped_ptr`. However, most of OceanBase's existing code releases resources manually, and smart pointers are prone to side effects if they are not used well. Therefore, smart pointers are not allowed. +Users are allowed to write some Guard classes by hand. The methods of these classes will apply for some resources, and these resources will be released automatically when the class is destroyed, such as LockGuard and SessionGuard. + +## 6.2 Memory Allocation and Release +It is required to use the memory allocator to allocate memory, and immediately set the pointer to NULL after the memory is released. + +The methods OceanBase can use for memory allocation include `ob_malloc` and various memory allocators. It is required to use the memory allocator to allocate memory, and specify the module it belongs to when allocating. The advantage of this is that it is convenient for the system to manage memory. If there is a memory leak, it is easy to see which module it is. In addition, it is necessary to prevent reference to the memory space that has been released, and it is required to set the pointer to `NULL` immediately after free. + +```cpp +void *ptr = ob_malloc(100, ObModIds::OB_COMMON_ARRAY); + +// do something + +if (NULL != ptr) { + // Release resources + ob_free(ptr, ObModIds::OB_COMMON_ARRAY); + ptr = NULL; // empty the pointer immediately after free +} +``` + +## 6.3 String +The `std::string` class is prohibited, and `ObString` is used instead. In addition, when manipulating C strings, it is required to use length-limited string functions. + +C++'s `std::string` class is very convenient to use. The problem is that it is impossible to figure out its internal behavior, such as copying and implicit conversion. OceanBase requires the use of `ObString` as much as possible, and the memory used in it needs to be manually managed by developers. + +Sometimes C strings are used. Be careful not to use string manipulation functions with unlimited length, including `strcpy/strcat/strdup/sprintf/strncpy`, but use the corresponding string manipulation functions with limited length `strncat/strndup/snprintf/memcpy`. You can use strlen to get the length of a string. The reason why `strncpy` is not used is that if the incoming buffer is not enough, it will not automatically '\0', and there are performance problems, so it needs to be replaced by `memcpy/snprintf`. + +## 6.4 Array/String/Buffer Access +**When a function passes an array/string/buffer as a parameter, the length of the array/string/buffer must be passed at the same time. When accessing the contents of an array/string/buffer, you must check whether the subscript is out of bounds.** + +## 6.5 Friend +Friend can only be used in the same file. If it must be violated, please obtain the consent of the group leader in advance and explain the reason in detail. Declaring unit test classes as friends is an exception, but should be used with caution. +Friend is usually defined in the same file to prevent code readers from going to other files to find their use of a class's `private` members. Scenarios where friend is often used include: + +1. Iterator: The iterator class is often declared as a friend, for example, `ObQueryEngine` declares `ObQueryEngineIterator` as a `friend` (friend class `ObQueryEngineIterator`). +2. Factory mode: For example, declare `ObFooBuilder` as a friend of `ObFoo` so that `ObFooBuilder` can access the internal state of `ObFoo`. + +In some cases, it may be convenient to declare a unit test class as a friend of the class under test in order to improve test coverage. However, this approach needs to be approached with caution. In most cases, we should indirectly test private functions through various input combinations of public functions, otherwise, these unit test codes will be difficult to maintain. + +## 6.6 Exception +C++ exceptions are prohibited. +Some programming languages encourage the use of exceptions, such as Java. Exceptions do make writing code more convenient, but only in the code writing stage, subsequent debugging and bug correction will be very inconvenient. Exceptions make the program control flow more complicated, and it is easy to forget to catch some exceptions. Therefore, it is forbidden to use ret error codes to return errors. +## 6.7 Runtime Type Identification +The use of Run-Time Type Identification (RTTI) is prohibited. +Runtime type recognition often indicates a problem with the design itself, and if it must be used, it usually indicates that the design of the class needs to be reconsidered. + +## 6.8 Type Conversion +Use `static_cast<>` and other C++ type conversions, prohibiting the use of int-like C cast of `y = (int) x`. +C++-style type conversions include: +1. `static_cast`: Similar to the C style, it can do value cast, or a clear upcast from the subclass of the pointer to the parent class. +2. `const_cast`: Remove the const attribute. +3. `reinterpret_cast`: Unsafe conversion between pointer types and integers or other pointers, so be **careful** when using it. +4. `dynamic_cast`: Except for test code, it is **forbidden** to use. + +`const_cast` needs to be used with **caution**. In particular, for an input parameter declared as const. In principle, it is **forbidden** to use `const_cast` to remove const. +`const_cast` will cause cognitive difficulties for code readers: for a const input parameter of a function, when analyzing the code logic, it will be considered that this parameter is generated outside the function and will not be modified inside the function; using `const_cast` will destroy this assumption, resulting in code readers cannot notice the modification of const input parameters inside the function. For example, `const_cast` in the following code fragment is prohibited. + +```cpp +int foo(const char* bar, int64_t len) +{ + ... + memcpy(const_cast(bar), src, len); + ... + return OB_SUCCESS; +} +``` + +## 6.9 Output +Try to use `to_cstring` output. + +In principle, every class that supports printing needs to implement `to_string`, +An example of using `to_cstring` is as follows: + +```cpp +FILL_TRACE_LOG("cur_trans_id=%s", to_cstring(my_session->get_trans_id())); +FILL_TRACE_LOG("session_trans_id=%s", to_cstring(physical_plan->get_trans_id())); +``` + +## 6.10 Integers +Use `int` for the returned ret error code, and use `int64_t` for function parameters and loop times as much as possible. In other cases, use a signed number with a specified length, such as `int32_t`, `int64_t`. Avoid unsigned numbers, except for a few idioms. + +The reason why function parameters and loop times use `int64_t` as much as possible is to avoid a large number of data type conversions in function calls and loop statements. Of course, idioms can be exceptions, such as ports being `int32_t`. In a structure such as struct, there is often a need for 8-byte alignment or memory saving, so a signed number of a specified length can be used. + +Except for idioms such as bit sets or numbers (such as `table_id`), the use of unsigned numbers should be avoided. Unsigned numbers may bring some hidden dangers, such as: +```cpp +for (unsigned int i = foo. length() – 1; i >= 0; --i) +``` + +The above code never terminates. +For numbering, some current codes use 0 as an illegal value, and some use `MAX_UINT64` as an illegal value. In the future, it will be uniformly stipulated that both 0 and `MAX_UINT64` are illegal values, and the inline function `is_valid_id` is provided in the utility for checking. In addition, uniformly initialize the number value to the macro `OB_INVALID_ID`, and adjust the initial value of the macro `OB_INVALID_ID` to 0. +## 6.11 sizeof +Try to use `sizeof(var_name)` instead of `sizeof(type)`. +This is because, if the type of `var_name` changes, `sizeof(var_name)` will automatically synchronize, but `sizeof(type)` will not, which may bring some hidden dangers. + +```cpp +ObStruct data; +memset(&data, 0, sizeof(data)); // correct way +memset(&data, 0, sizeof(ObStruct)); // Wrong way +``` + +It should be noted that instead of using sizeof to calculate the length of a string, use strlen instead. For example: + +```cpp +char *p = "abcdefg"; +// sizeof(p) indicates the pointer size, which is equal to 8 on a 64-bit machine +int64_t nsize = sizeof(p); +``` + +## 6.12 0 and nullptr +Use 0 for integers, 0.0 for real numbers, `nullptr` for pointers (replacing the previous `NULL`), and '\0' for strings. + +## 6.13 Preprocessing Macros +**In addition to existing macros, no new macros shall be defined, and inline functions, enumerations, and constants shall be substituted**. If it is necessary to define a new macro, please obtain the agreement of the group leader in advance, and explain the reason in detail. +Macros can do things that other techniques cannot, such as stringifying (using #), concatenation (using ##). Macros are often used for output and serialization, such as `common/ob_print_utils.h`, rpc-related classes. However, in many cases, other methods can be used instead of macros: macro inline efficiency-critical code can be replaced by inline functions, and macro storage constants can be used const variable substitution. +The principle of judgment is: in addition to output and serialization, as long as macros can be used, try not to use macros. + +## 6.14 Boost and STL +**In STL, only algorithm functions defined in the header file are allowed, such as std_sort, and other STL or boost functions are prohibited. If it must be violated, please obtain the consent of the project leader and project architect in advance, and explain the reasons in detail.** + +OceanBase has a conservative attitude towards libraries like boost and STL, and we believe that writing code correctly is far more important than writing code conveniently. Except for the algorithm class functions defined by STL , other functions should not be used. + +## 6.15 auto +**What is** +The specific type is omitted when declaring the variable, and the compiler automatically deduces the type according to the initialization expression. + +**Example** +```cpp +auto i = 42; // i is an int +auto l = 42LL; // l is a long long +auto p = new foo(); // p is a foo* +``` + +**Is it allowed** +**Prohibited.** +Although it is possible to make the declaration of some template types shorter, we hope that the type declaration matches the user's intention. For example, in the above examples 1 and 2, the type should be explicitly declared. + +## 6.16 Range-based for Loops +**What is** +The new for loop syntax is used to easily traverse the container that provides `begin()`, `end()`. + +**Example** +```cpp +for(const auto& kvp : map) { + std::cout << kvp. first << std::endl; +} +``` +**Is it allowed** +**Prohibited.** +This feature is just a syntactic sugar. The `FOREACH` macro defined by ourselves has been widely used in the previous OceanBase code, which can achieve similar effects. + +## 6.17 Override and Final +**What is** +`override` is used to indicate that a virtual function is an overload of a virtual function in the base class; `final` indicates that a virtual function cannot be overloaded by a derived class. + +**Example** +```cpp +class B +{ +public: + virtual void f(short) {std::cout << "B::f" << std::endl;} + virtual void f2(short) override final {std::cout << "B::f2" << std::endl;} +}; +class D : public B +{ +public: + virtual void f(int) override {std::cout << "D::f" << std::endl;} +}; +class F : public B +{ +public: + virtual void f2(int) override {std::cout << "D::f2" << std::endl;} // compiler error +}; +``` + +**Is it allowed** +**Allow.** `override` and `final` are not only allowed, but strongly recommended, and should be added wherever they can be used. + +According to previous experience, the overloading of the virtual function in the OceanBase missed the `const`, resulting in endless errors of overloading errors. It is required that in the new code, all overloads must be added with override to avoid this wrong overload situation. + +In addition to being used for virtual functions, when a class is added with the final keyword, it means that it cannot be further derived, which is conducive to compiler optimization. When such a class has no parent class, the destructor does not need to add virtual. + +```cpp +class ObLinearRetry final: public ObIRetryPolicy +{ + //... +}; + +class ObCData final +{ + ~ObCData(); +} +``` + +## 6.18 Strongly-typed Enums +**What is** +Traditional enumerated types have too many shortcomings to be true types. For example, it is implicitly converted to an integer; the enumeration value is in the same scope as the place where its type is defined. +**Example** + +```cpp +enum class Options {None, One, All}; +Options o = Options::All; +``` + +**Is it allowed** +**Allow.** The original enumeration type is a bug in the C++ language. The new enumeration type makes the compiler's inspection more strict, and uses new keyword definitions, which do not conflict with the original enum. + +## 6.19 Lambdas +**What is** +A concept borrowed from functional programming for conveniently writing anonymous functions. +**Example** +```cpp +std::function lfib = [&lfib](int n) {return n < 2 ? 1 : lfib(n-1) + lfib(n-2);}; +``` +**Is it allowed** +**prohibited**. The novel syntax of lambda makes C code look like a new language, and most people don't have enough understanding of functional programming, so the cost of learning is relatively high. Does not meet principle (1). In addition, lambda is essentially equivalent to defining a functor, which is a syntactic sugar that does not increase the abstraction ability of C. Does not meet principle (2). +## 6.20 Non-member begin() and end() +**What is** +The global functions `std::begin()` and `std::end()` are used to conveniently abstract operations on containers. + +**Example** +```cpp +int arr[] = {1,2,3}; +std::for_each(std::begin(arr), std::end(arr), [](int n) {std::cout << n << std::endl;}); +``` + +**Is it allowed** +**Prohibited**. This feature is mainly to make STL easier to use, but OceanBase prohibits the use of STL containers. + +## 6.21 static_assert and Type Traits +**What is** +The compile-time `assert` and compile-time constraint checking supported by the compiler. +**Example** +```cpp +template +class Vector +{ + static_assert(Size < 3, "Size is too small"); + T_points[Size]; +}; +``` +**Is it allowed** +**allow**. Although the OB code has defined `STATIC_ASSERT` by itself, it is only a simulation of the compiler check, and the error report is not friendly. And `type_traits` brings great benefits to the use of templates. +## 6.22 Move Semantics +**What is** +The move constructor and move assignment operator are one of the most important new features of C++11. Along with it, the concept of rvalue is introduced. Move semantics can make many container implementations much more efficient than before. +**Example** +```cpp +// move constructor +Buffer(Buffer&& temp): + name(std::move(temp.name)), + size(temp. size), + buffer(std::move(temp.buffer)) +{ + temp._buffer = nullptr; + temp._size = 0; +} +``` +**Is it allowed** +**Prohibited**. Banning it may bring some controversy. Mainly based on the following considerations: +1. OceanBase does not use STL containers, so the optimization of the standard library using move semantics does not bring us benefits. +2. The semantics of move semantic and rvalue are more complicated, and it is easy to introduce pitfalls +3. Using it to transform some existing containers of OceanBase can indeed improve performance. However, the memory management method of OceanBase has made the use of move semantics smaller. In many cases, we have optimized it during implementation, and only store pointers in the container, not large objects. + +It is recommended to consider other C++11 features after a period of familiarity, when the coding standard is revised next time. + +## 6.23 constexpr +**What is** +More standardized compile-time constant expression evaluation support, no longer need to use various template tricks to achieve the effect of compile-time evaluation. +**Example** +```cpp +constexpr int getDefaultArraySize (int multiplier) +{ + return 10 * multiplier; +} + +int my_array[ getDefaultArraySize( 3 ) ]; +``` + +**Is it allowed** +**allow**. Constants are always more friendly to compiler optimization. In the above example, the use of macros is also avoided. In addition, constexpr supports floating-point calculations, which cannot be replaced by static const. +## 6.23 Uniform Initialization Syntax and Semantics +**What is** +The initialization of variables of any type in any context can use the unified {} syntax. +**Example** +```cpp +X x1 = X{1,2}; +X x2 = {1,2}; // the = is optional +X x3{1,2}; +X* p = new X{1,2}; +``` + +**Is it allowed** +**Prohibited**. Syntactically more uniform, but again without any significant benefit. At the same time, it will significantly affect the style of OceanBase code and affect readability. +## 6.24 Right Angle Brackets +**What is** +Fix a common syntax problem in original C. It turns out that when the templates of C-defined templates are nested, the ending >> must be separated by spaces, which is no longer needed. +**Example** +```cpp +typedef std::vector> Flags; +``` + +**Is it allowed** +**allow.** + +## 6.25 Variadic Templates +**What is** +Variadic templates. + +**Example** +```cpp +template +void func(const Arg1& arg1, const Args&... args) +{ + process( arg1 ); + func(args...); // note: arg1 does not appear here! +} +``` +**Is it allowed** +**Allow**. This is a key feature for template programming. Because there is no variable-length template parameter, some basic libraries of OceanBase, such as `to_string`, `to_yson`, RPC framework, log library, etc., need to be implemented with some tricks and macros. And more type safe. + +## 6.26 Unrestricted Unions +**What is** +Before, union could not contain classes with constructors as members, but now it can. +**Example** +```cpp +struct Point { + Point() {} + Point(int x, int y): x(x), y(y) {} + int x, y; +}; +union_{ + int z; + double w; + Point p; // Illegal in C03; legal in C11. + U() {} // Due to the Point member, a constructor definition is now needed. + U(const Point& pt) : p(pt) {} // Construct Point object using initializer list. + U& operator=(const Point& pt) { new(&p) Point(pt); return *this; } // Assign Point object using placement 'new'. +}; +``` +**Is it allowed** +**Allow**. There are many places in the OceanBase code that have to define redundant domains because of this limitation, or use tricky methods to bypass (define char array placeholders). See `sql::ObPostExprItem` for a miserable example. + +## 6.27 Explicitly Defaulted and Deleted Special Member Functions +**What is** +One of the most disturbing things about C++ before is that the compiler implicitly and automatically generates constructors, copy constructors, assignment operators, destructors, etc. for you. They can now be explicitly required or disallowed. +**Example** +```cpp +struct NonCopyable { + NonCopyable() = default; + NonCopyable(const NonCopyable&) = delete; + NonCopyable& operator=(const NonCopyable&) = delete; +}; +struct NoInt { + void f(double i); + void f(int) = delete; +}; +``` + +**Is it allowed** +**Allowed**. This feature is like tailor-made for OceanBase; the function of disabling a certain function is also very useful. +## 6.28 Type Alias (Alias Declaration) +**What is** +Use the new alias declaration syntax to define an alias of a type, similar to the previous typedef; moreover, you can also define an alias template. +**Example** +```cpp +// C++11 +using func = void(*)(int); +// C++03 equivalent: +// typedef void (*func)(int); +template using ptr = T*; +// the name 'ptr' is now an alias for pointer to T +ptr ptr_int; +``` + +**Is it allowed** +**Prohibited**. For the time being, there is no need for alias templates, and the same effect can be achieved by using typedef for non-template aliases. + +## 6.29 Summary + +1. **Smart pointers are not allowed**, and resources are allowed to be released automatically through the Guard class. +2. It is required to use the memory allocator to allocate memory, and immediately set the pointer to `NULL` after the memory is released. +3. **Prohibit the use of std::string class, use ObString instead**. In addition, when manipulating C strings, it is required to use length-limited string functions. +4. **When passing an array/string/buffer as a parameter, the length must be passed at the same time. When reading and writing the content of the array/string/buffer, check whether the subscript is out of bounds.** +5. `friend` can only be used in the same file. If it must be violated, please obtain the consent of the code owner in advance and explain the reason in detail. Declaring unit test classes as `friend` is an exception, but should be used with caution. +6. **C++ exceptions are prohibited.** +7. Prohibit the use of runtime type identification (RTTI). +8. Use C++ type conversions such as `static_cast<>`, and prohibit the use of int C cast of `y = (int) x`. +9. Try to use `to_cstring` output. +10. Use int for the returned ret error code, and use `int64_t` for function parameters and loop times as much as possible. In other cases, use a signed number with a specified length, such as `int32_t`, `int64_t`. Try to avoid using unsigned numbers. +11. Try to use `sizeof(var_name)` instead of `sizeof(type)`. +12. Use 0 for integers, 0.0 for real numbers, `NULL` for pointers, and '\0' for strings. +13. **In addition to the existing macros, no new macros shall be defined, and inline functions, enumerations, and constants shall be used instead**. If it must be violated, please obtain the consent of the group leader in advance, and explain the reasons in detail. +14. **Except for the algorithm class functions defined in the header file in STL, the use of STL and boost is prohibited. If it must be violated, please obtain the consent of the project leader and project architect in advance, and explain the reasons in detail.** + +# 7 Naming Rules +## 7.1 General Rules +Function naming, variable naming, file naming should be descriptive and not overly abbreviated, types and variables should be nouns, and functions can use "imperative" verbs. +Identifier naming sometimes uses some common abbreviations, but it is not allowed to use too professional or unpopular ones. For example we can use the following ranges: +1. temp can be abbreviated as tmp; +2. statistic can be abbreviated as stat; +3. increment can be abbreviated as inc; +4. message can be abbreviated as msg; +5. count can be abbreviated as cnt; +6. buffer can be abbreviated as buf instead of buff; +7. current can be abbreviated as cur instead of curr; +When using abbreviations, consider whether each project team member can understand them. Avoid abbreviations if you're not sure. + +Types and variables are generally nouns, for example, `ObFileReader`, `num_errors`. +Function names are usually imperative, eg `open_file()`, `set_num_errors()`. + +## 7.2 File Naming +Self-describing well composed of all lowercase words separated by '_', for example +`ob_update_server.h` and `ob_update_server.cpp`. +The .h file and the .cpp file correspond to each other. If the template class code is long, it can be placed in the .ipp file, such as `ob_vector.h` and `ob_vector.ipp`. + +## 7.3 Type Naming +Use self-describing well-formed words. In order to distinguish it from variables, it is recommended to use the first letter of the word capitalized and no separator in the middle. Nested classes do not need to add "Ob" prefix, other classes need to add "Ob" prefix. For example: +```cpp +// class and structs +class ObArenaAllocator +{ + ... +}; +struct ObUpsInfo +{ + ... +}; + +// typedefs +typedef ObStringBufT<> ObStringBuf; + +// enums +enum ObPacketCode +{ +}; + +// inner class +class ObOuterClass +{ +private: + class InnerClass + { + }; +}; +``` + +The interface class needs to be preceded by an "I" modifier, and other classes should not be added, for example: + +```cpp +class ObIAllocator +{ + ... +}; +``` + +## 7.4 Variable Naming +### 7.4.1 Intra-class Variable Naming +Self-describing good all lowercase words, separated by '\_', in order to avoid confusion with other variables, it is required to add '\_' in the back end to distinguish, for example: +```cpp +class ObArenaAllocator +{ +private: + ModuleArena arena_; +}; + +struct ObUpsInfo +{ + common::ObServer addr_; + int32_t inner_port_; +}; +``` + +### 7.4.2 Common Variable Naming +Self-describing well-formed all-lowercase words separated by '_'. + +### 7.4.3 Global Variable Naming +New global variables must not be used in addition to existing global variables. If it must be violated, please obtain the consent of the code owner in advance, and explain the reasons in detail. Global variables are composed of self-describing all-lowercase words separated by '\_'. In order to mark the global nature, it is required to add the 'g\_' modifier at the front end. For example: + +```cpp +// globe thread number +int64_t g_thread_number; +``` + +## 7.5 Function Naming +### 7.5.1 Function Naming within a Class +Self-describing well-formed all-lowercase words separated by '_', for example: +```cpp +class ObArenaAllocator +{ +public: + int64_t used() const; + int64_t total() const; +}; +``` + +### 7.5.2 Access Function Naming +The name of the access function needs to correspond to the class member variable. If the class member variable is `xxx`, then the access function is `set_xxx` and `get_xxx` respectively. + +### 7.5.3 Ordinary Function Naming +Self-describing well-formed all-lowercase words separated by '_'. + +## 7.6 Constant Naming +All compile-time constants, whether local, global or in a class, are required to be composed of all uppercase letters, separated by '_' between words. For example: +```cpp +static const int NUM_TEST_CASES = 6; +``` + +# 7.7 Macro Naming +Try to avoid using macros. Macro names are all composed of uppercase letters, and words are separated by '_'. Note that parameters must be enclosed in parentheses when defining a macro. For example: +```cpp +// Correct spelling +#define MAX(a, b) (((a) > (b)) ? (a) : (b)) +// wrong wording +#define MAX(a, b) ((a > b) ? a : b) +``` + +## 7.8 Precautions +There are a few points that are easy to forget, as follows: +1. Try not to use abbreviations unless they are clear enough and widely accepted by project team members. +2. In addition to the exception that the name of the interface class needs to be modified with I, other classes, structures, and enumeration types do not need modifiers +3. The variable names in the struct also need to be underlined + +# 8 Typography Style +## 8.1 Code Indentation +Do not use the Tab key for indentation, you can use spaces instead, and different coding tools can be set, requiring the use of two spaces for indentation (4 spaces for indentation will appear a bit compact when the single-line code is relatively long). + +## 8.2 Empty Lines +Minimize unnecessary blank lines and only do so when the logic of the code is clearly divided into multiple parts. + +The internal code of the function body is determined by the visual code. Generally speaking, only when the code is logically divided into multiple parts, a blank line needs to be added between each part. + +None of the following code should have blank lines: +```cpp +// There should be no blank lines at the beginning and end of the function +void function() +{ + int ret = OB_SUCCESS; + +} + +// Do not have blank lines at the beginning and end of the code block +while (cond) { + // do_something(); + +} +if (cond) { + + // do_something() +} +``` +Empty lines below are reasonable. + +```cpp +// Function initialization and business logic are two parts, +// with a blank line in between +void function(const char *buf, const int64_t buf_len, int64_t &pos) +{ + int ret = OB_SUCCESS; + int64_t ori_pos = pos; + if (NULL == buf || buf_len <= 0 || pos >= buf_len) { + // print error log + ret = OB_INVALID_ARGUMENT; + } else { + ori_pos = pos; + } + + // Execute business logic + return ret; +} +``` +## 8.3 Line Length +The length of each line shall not exceed 100 characters, and one Chinese character is equivalent to two characters. +100 characters is the maximum value of a single line, and the maximum limit can be increased to 120 characters in the following situations: +1. If a line of comments contains commands or URLs exceeding 100 characters. +2. Include long paths. +## 8.4 Function Declaration +The return type and the function name are on the same line, and the parameters are also placed on the same line as much as possible. +The function looks like this: +```cpp +int ClassName::function_name(Type par_name1, Type par_name2) +{ + int ret = OB_SUCCESS; + do_something(); + ... + return ret; +} +``` +If the same line of text is too much to accommodate all parameters, you can split the parameters into multiple lines, with one parameter per line: +```cpp +int ClassName::really_long_function_name(Type par_name1, + Type par_name2, Type par_name3) // empty 4 spaces +{ + int ret = OB_SUCCESS; + do_something(); + ... + return ret; +} +``` +You can also put each parameter on a separate line, and each subsequent parameter is aligned with the first parameter, as follows: +```cpp +// The following parameters are aligned with the first parameter +int ClassName::really_long_function_name(Type par_name1, +Type par_name2, // align with the first parameter +Type par_name3) +{ + int ret = OB_SUCCESS; + do_something(); + ... + return ret; +} +``` +If you can't even fit the first parameter: +```cpp +// Start a new line for each parameter, with 4 spaces +int ClassName::really_really_long_function_name( + Type par_name1, // empty 4 spaces + Type par_name2, + Type par_name3) +{ + int ret = OB_SUCCESS; + do_something(); + ... + return ret; +} +``` +Note the following points: +- The return value is always on the same line as the function name; +- The opening parenthesis is always on the same line as the function name; +- There is no space between the function name and the opening parenthesis; +- There is no space between parentheses and parameters; +- The opening curly brace is always alone on the first line of the function (starting a new line); +- The closing curly brace is always alone on the last line of the function; +- All formal parameter names at function declaration and implementation must be consistent; +- All formal parameters should be aligned as much as possible; +- default indentation is 2 spaces; +- The parameters after the line break keep the indentation of 4 spaces; +- If the function is declared const, the keyword const should be on the same line as the last parameter +Some parameters are not used, and these parameter names are annotated when the function is defined: +```cpp +// Correct +int ObCircle::rotate(double /*radians*/) +{ +} + +// wrong +int ObCircle::rotate(double) +{ +} +``` +## 8.5 Function Calls +Try to put it on the same line. If you can't fit it, you can cut it into multiple lines. The splitting method is similar to the function declaration. +The form of a function call is often like this (no spaces after the opening parenthesis and before the closing parenthesis): +```cpp +int ret = function(argument1, argument2, argument3); +``` +If it is divided into multiple lines, the following parameters can be split into the next line, as follows: +```cpp +int ret = really_long_function(argument1, + argument2, argument3); // empty 4 spaces +``` +You can also put each parameter on a separate line, and each subsequent line is aligned with the first parameter, as follows: +``` +int ret = really_long_function(argument1, + argument2, // align with the first argument + argument3); +``` +If the function name is too long, all parameters can be separated into separate lines, as follows: +```cpp +int ret = really_really_long_function( + argument1, + argument2, // empty 4 spaces + argument3); +``` +For placement new, a space needs to be added between new and the pointer variable, as follows: +```cpp +new (ptr) ObArray();// There is a space between new and '(' +``` +## 8.6 Conditional Statements +`{` and if or else on the same line, `}` start a new line. In addition, between if and "(", ")" and `{` are guaranteed to contain a space. +Conditional statements tend to look like this: + +```cpp +if (cond) { // There is no space between (and cond, cond and) + ... +} else { // } and else, there is a space between else and { + ... +} +``` + +In any case, both if and else statements need to have `{` and `}`, even if the branch is only one line statement. In principle, `}` always start a new line, but there is one exception. If the else branch does nothing, `}` does not need a new line, as follows: +```cpp +if (OB_SUCCESS == (ret = do_something1())) { + ... +} else if (OB_SUCCESS == (ret = do_somethng2())) { + ... +} else { } // else branch does nothing, } does not require a new line +``` +For the comparison statement, if it is `=`, `!=`, then the constant needs to be written in front; while `>`, `>=`, `<`, `<=`, there is no such restriction. For example: +```cpp +// Correct +if (NULL == p) { + ... +} + +// wrong +if (p == NULL) { + ... +} +``` + +## 8.7 Expressions +There is a space between the expression operator and the preceding and following variables, as follows: +```cpp +a = b; // There is a space before and after = +a > b; +a & b; +``` +For boolean expressions, if the maximum length of the line is exceeded, the line break format needs to be taken care of. In addition, complex expressions need to use parentheses to clarify the order of operations of the expression to avoid using the default priority. +When breaking a line, the logical operator is always at the beginning of the next line, with 4 spaces: +```cpp +if ((condition1 && condition2) + || (condition3 && condition4) // && operator is at the beginning of the line, with 4 spaces + || (condition5 && condition6)) { + do_something(); + ... +} else { + do_another_thing(); + ... +} +``` +If the expression is complex, parentheses should be added to clarify the order of operations of the expression. +```cpp +// correct +word = (high << 8) | low; +if ((a && b) || (c && d)) { + ... +} else { + ... +} + +// wrong +word = high << 8 | low; +if (a && b || c && d) { + ... +} else { + ... +} +``` + +The ternary operator should be written in one line as much as possible. If it exceeds one line, it needs to be written in three lines. as follows: +```cpp +// The ternary operator is written in one line +int64_t length = (0 == digit_idx_) ? digit_pos_ : (digit_pos_ + 1); + +// The ternary operator is written in three lines +int64_t length = (0 == digit_idx_) + ? (ObNumber::MAX_CALC_LEN - digit_pos_ - 1) // 4 spaces + : (ObNumber::MAX_CALC_LEN - digit_pos_); + +// Error: Breaking into two lines is not allowed +int64_t length = (0 == digit_idx_) ? (ObNumber::MAX_CALC_LEN – digit_pos_ - 1) + : (ObNumber::MAX_CALC_LEN – digit_pos_); +``` + +## 8.8 Loops and Switch Selection Statements +Both the `switch` statement and the `case` block in it need to use `{}`. In addition, each case branch must add a break statement. Even if you can ensure that you will not go to the default branch, you need to write the default branch. +```cpp +switch (var) { +case OB_TYPE_ONE: { // top case + // empty 4 spaces relative to case, empty 4 spaces relative to switch + break; + } +case OB_TYPE_TWO: { + ... + break; + } +default: { + perform error handling; + } +} +``` +An empty loop body needs to write an empty comment instead of a simple semicolon. For example: +```cpp +// correct way +while (cond) { + //empty +} + +for (int64_t i = 0; i < num; ++i) { + //empty +} + +// wrong way +while (cond) ; +for (int64_t i = 0; i < num; ++i) ; +``` + +## 8.9 Variable Declaration +Only one variable is declared per line, and the variable must be initialized when it is declared. When declaring a pointer variable or parameter, `(*, &)` next to the variable name. The same is true for pointers or references `(*, &)` when a function type is declared. +```cpp +// correct way +int64_t *ptr1 = NULL; +int64_t *ptr2 = NULL; + +// wrong way + +int64_t *ptr1 = NULL, ptr2 = NULL; // error, declare only one variable per line +int64_t *ptr3; // Error, variable must be initialized when declared +int64_t* ptr = NULL; // error, * is next to the variable name, not next to the data type + +char* get_buf(); // error, * is next to the variable name, not next to the data type +char *get_buf(); // correct + +int set_buf(char* ptr); // error, * is next to the variable name, not next to the data type +int set_buf(char *ptr); // correct +``` + + +## 8.10 Variable References +For references and pointers, you need to pay attention: there should be no spaces before and after periods `(.)` or arrows `(->)`. There can be no spaces after the pointer `(*)` and the address operator `(&)`, and the address operator is next to the variable name. +```cpp +// correct way +p = &x; +x = *p; +x = r->y; +x = r.y; +``` + +## 8.11 Preprocessing Directives +Do not indent preprocessing directives, start at the beginning of the line. Even if a preprocessing directive is in an indented code block, the directive should start at the beginning of the line. +```cpp +// Correct way of writing, preprocessing directive is at the beginning of the line +#if !defined(_OB_VERSION) || _OB_VERSION<=300 + do_something1(); +#elif _OB_VERSION>300 + do_something2(); +#endif +``` + +## 8.12 Class Format +The order of declarations is `public`, `protected`, and `private`. These three keywords are in the top case and are not indented. +The basic format of a class declaration is as follows: + +```cpp +class ObMyClass : public ObOtherClass // : there is a space before and after +{ // { start a new line +public: // top grid + ObMyClass(); // Indent 2 spaces relative to public + ~ObMyClass(); + explicit ObMyClass(int var); + + int some_function1(); // first class function function + int some_function2(); + + inline void set_some_var(int64_t var) {some_var_ = var;} // the second type of function + inline int64_t get_some_var() const {return some_var_;} + + inline int some_inline_func(); // The third type of function + +private: + int some_internal_function(); // function defined first + + int64_t some_var_; // variables are defined after + DISALLOW_COPY_AND_ASSIGN(ObMyClass); +}; + +int ObMyClass::some_inline_func() +{ + ... +} +``` +For the declaration order of classes, please refer to Chapter 4 Declaration Order. It should be noted that only inline functions whose implementation code is one line can be placed in the class definition, and other inline functions can be placed outside the class definition in the .h file. In the above example, `set_some_var` and `get_some_var` have only one line of implementation code, so they are placed inside the class definition; the implementation code of `some_inline_func` exceeds one line, and need to be placed outside the class definition. This has the advantage of making class definitions more compact. +## 8.13 Initialization Lists +The constructor initialization list is placed on the same line or indented according to 4 spaces and lined up in several lines, and the following parameters are aligned with the first parameter. In addition, if the initialization list needs to wrap, it must start from the first parameter. +Two acceptable initializer list formats are: +```cpp +// initializer list on the same line +ObMyClass::ObMyClass(int var) : some_var_(var), other_var_(var+1) +{ + ... +} + +// The initialization list is placed on multiple lines, indented by 4 spaces +ObMyClass::ObMyClass(int var) + : some_var_(var), + some_other_var_(var+1) // The second parameter is aligned with the first parameter +{ + ... +} +``` +## 8.14 Namespaces +Namespace contents are not indented. +```cpp +namespace oceanbase +{ +namespace common +{ +class ObMyClass // ObMyClass do not indent +{ + ... +} +} // namespace common +} // namespace oceanbase +``` +## 8.15 Constants Instead of Numbers +Avoid confusing numbers and use meaningful symbols instead. Constants that involve physical states or have physical meanings should not use numbers directly, but must be replaced by meaningful enumerations or constants. +```cpp +const int64_t OB_MAX_HOST_NAME_LENGTH = 128; +const int64_t OB_MAX_HOST_NUM = 128; +``` +## 8.16 Precautions +1. The `{` of the `if&else`, `for&while` and `switch&case` statements are placed at the end of the line instead of starting a new line; +2. Define the `public`, `protected` and `private` keywords of the class with 2 spaces, and pay attention to the declaration order of the class. +3. When cutting a line into multiple lines, you need to pay attention to the format. +4. Minimize unnecessary blank lines as much as possible, and only do this when the code logic is clearly divided into multiple parts. +5. Do not indent the contents of the namespace. + + +# 9 Notes +Comments are written for others to understand the code, the following rules describe what should be commented and where. +## 9.1 Comment Language and Style +The comment language is required to use English, and Chinese cannot be used, and the comment style adopts `//`. The purpose of comments is to make your code easier for others to understand. +The comment style can use `//` or `/* */`, except for the comments of the header file, `//` is used in other cases. +## 9.2 Document Comments +Add a copyright notice at the beginning of each file, see Section 2.2 for the copyright notice. +For key algorithms and business logic, it should be clearly described here, and the file header should be defined. +## 9.3 Class Annotations +Each class definition must be accompanied by a comment describing the function and usage of the class. For example: +```cpp +// memtable iterator: the following four requirements all use MemTableGetIter iteration +// 1. [General get/scan] need to construct RNE cell, and construct mtime/ctime cell +// according to create_time, if there is column filtering, it will also construct NOP cell +// 2. [dump2text of QueryEngine] There is no column filtering and transaction id +// filtering, no NOP will be constructed, but RNE/mtime/ctime will be constructed +// 3. [Dump] Without column filtering and transaction id filtering, empty rows will be +// skipped in QueryEngine, RNE and NOP will not be constructed, but mtime/ctime +// will be constructed +// 4. [Single-line merge] Before merging, it is necessary to determine whether there +// is still data after the transaction id is filtered. If not, the GetIter +// iteration is not called to prevent the RNE from being constructed and written +// back to the memtable; in addition, RowCompaction is required to ensure that the +// order is not adjusted to prevent the mtime representing the transaction ID from +// being adjusted to the common column. +// 5. [update and return] is similar to regular get/scan, but without transaction +// id filtering +class MemTableGetIter : public common::ObIterator +{ +}; +``` + +Please note that the things that need to be paid attention to when using the class are indicated here, especially whether it is thread-safe, how resources are released, and so on. +## 9.4 Function Annotations +### 9.4.1 Function Declaration Comments +The function declaration comment is placed before the function declaration, and mainly describes the function declaration itself rather than how the function is completed. The content to be described includes: +- The input and output of the function. +- If the function allocated space, it needs to be freed by the caller. +- Whether the parameter can be NULL. +- Whether there are performance risks in function usage. +- Is the function reentrant and what are its synchronization prerequisites +```cpp +// Returns an iterator for this table. +//Note: +// It's the client's responsibility to delete the iterator +// when it's done with it. +// +// The method is equivalent to: +// ObMyIterator *iter = table->new_iterator(); +// iter->seek_to_front(); +// return iter; +ObMyIterator *get_iterator() const; +``` +Generally speaking, the external interface functions of each class need to be annotated. Of course, self-describing functions such as constructors, destructors, and accessor functions do not need comments. +If the comment needs to describe the input, output parameters or return value, the format is as follows: +```cpp +// Gets the value according to the specified key. +// +// @param [in] key the specified key. +// @param [in] value the result value. +// @return the error code. +int get(const ObKey &key, ObValue &value); +``` + +Examples of annotations for function reentrancy are as follows: +```cpp +// This function is not thread safe, but it will be called by only one xxx thread. +int thread_unsafe_func(); +``` +### 9.4.2 Function Implementation Comments +If the function implementation algorithm is unique or has some bright spots, you can add function implementation comments in the .cpp file. For example, the programming skills used, the general steps of implementation, or the reasons for this implementation, such as explaining why the first half needs to be locked and the second half does not. Note that the focus here is on how to implement, rather than copying the function declaration comments in the .h file. +## 9.5 Variable Annotations +Local variables do not need to write comments. Member variables and global variables generally need to write comments, unless the project team members recognize that the variable is self-describing. If some values of the variable have special meaning, such as NULL, -1, then it must be stated in the comment. + +Use good and unambiguous language to indicate the purpose, point of use, and scope of variables. Comments can appear on the right side of the variable definition or on the top line of the variable definition according to the number of characters in the line, for example: + +```cpp +// comments appear on the top line +private: + // Keeps track of the total number of entries in the table. + // -1 means that we don't yet know how many entries the table has. + int num_total_entries_; + + // Comments appear to the right of the variable + static const int NUM_TEST_CASES = 6; // the total number of test cases. +``` + +## 9.6 Implementation Notes +Similarly, you must make detailed comments on business-critical points, sophisticated algorithms, and poorly readable parts in the internal implementation of the function. It can also appear at the top of a code snippet or to the right of a line of code. +```cpp +// it may fail, but the caller will retry until success. +ret = try_recycle_schema(); +``` +Be careful not to write comments in pseudo-code, that is too cumbersome and of little value. +## 9.7 TODO Comments +For functions that have not been implemented or are not perfectly implemented, sometimes we need to add TODO comments. All TODO comments must reflect the worker and the completion time. Of course, if the completion time is undecided, you can mark it clearly. For example: +```cpp +// TODO(somebody): needs another network roundtrip, will be solved by 2014/12. +``` +## 9.8 Precautions +1. The comment language can be English or Chinese, and the comment style adopts // +2. Comments are often used to describe classes, function interfaces, and key implementation points. Comments are encouraged unless it is self-describing code. +3. Be sure not to forget TODO comments. + +# 10 Multithreading +## 10.1 Starting and Stopping Threads +1. Except for very special cases, it is forbidden to dynamically start and stop threads. Once the server is initialized, the number of threads is fixed. Special cases such as: a backdoor reserved for the server, when all threads of the server are occupied, a worker thread is added. +2. In order to ensure that a thread will not be busy waiting in an infinite loop when exiting, the loop generally needs to judge the stop flag. +## 10.2 pthread_key +1. There are only 1024 `pthread_key` at most, and this limit cannot be increased, so special attention should be paid when using it. +2. If you want to use a large number of thread-local variables, it is recommended to use the thread number as an array subscript to obtain a thread-private variable. An `itid()` function is encapsulated in the OceanBase to obtain continuously increasing thread numbers. + +```cpp +void *get_thread_local_variable() +{ + return global_array_[itid()]; +} +``` +## 10.3 Timers +Time-consuming tasks cannot be completed in the timer, and time-consuming tasks need to be submitted to the thread pool for execution. +## 10.4 Locking and Unlocking +It is recommended to use the Guard method to use locks +```cpp +// scope is the whole function +int foo() +{ + SpinLockGuardguard(lock_); + ... +} +// scope is a clause +while(...) { + SpinLockGuardguard(lock_); + ... +} +``` +If the scope of the lock is not the entire function or a certain clause, such as locking in the middle of function execution and unlocking before the function exits, manual locking and unlocking are allowed in this case: +```cpp +int foo() +{ + int ret = OB_SUCCESS; + bool lock_succ = false; + if (OB_SUCCESS != (ret = lock_.lock())) { + lock_succ = false; + } else { + lock_succ = true; + } + ... // some statements are executed + if (lock_succ) { + lock_.unlock(); + } + return ret; +} +``` +## 10.5 Standard Usage of Cond/Signal +1. Use cond/signal through `CThreadcond` encapsulated by tbsys +2. Prohibit the use of `cond_wait()` without a timeout +3. Use cond/signal in the following idioms + +```cpp +// waiting logic +cond. lock(); +while(need_wait()) +{ + cond.wait(timeout); +} +cond.unlock(); +// wake up logic +cond.lock(); +cond.signal(); +cond.unlock(); +``` +## 10.6 Atomic Operations +Uniformly use the macros defined in ob_define.h to do atomic operations, you need to pay attention to: +1. Atomic reads and writes also need to be done with `ATOMIC_LOAD()` and `ATOMIC_STORE()` +2. Use `ATOMIC_FAA()` and `ATOMIC_AAF()` to distinguish between `fetch_and_add` and `add_and_fetch` +3. Use `ATOMIC_VCAS()` and `ATOMIC_BCAS()` to distinguish between `CAS` operations returning value or `bool` +## 10.7 Compiler Barriers +Generally, memory barriers are also required where compiler barriers are to be used, and memory barriers contain compiler barriers, so there should be no place where compiler barriers are required. +## 10.8 Memory Barriers +1. Although there are various memory barriers, we only recommend using the full barrier. Because finer barriers are very error-prone, currently in the engineering practice of OceanBase, there is no code that must use finer barriers to meet performance requirements. How complicated are the various barriers, you can refer to this document [memory-barriers](https://www.kernel.org/doc/Documentation/memory-barriers.txt) +2. The atomic operation comes with a barrier, so it is generally not necessary to manually add a barrier. +3. If you need to manually add a barrier, use a macro: +```cpp +#define MEM_BARRIER() __sync_synchronize() +``` +## 10.9 Reference Counting and shared_ptr +First of all, `shared_ptr` must not be used, because `shared_ptr` is just syntactic sugar and does not solve the problem we hope to solve with reference counting. +1. To put it simply: it is safe for multiple threads to operate different `shared_ptr` at the same time, but it is not safe for multiple threads to operate the same `shared_ptr` at the same time. When we consider reference counting, it is often necessary to operate the same shared_ptr with multiple threads. +2. For details, please refer to [shared_ptr](http://en.cppreference.com/w/cpp/memory/shared_ptr) +Secondly, reference counting seems simple, but it is actually not easy to implement correctly. It is not recommended to use reference counting unless you think about it very clearly. +To use reference counting, you must first consider the following questions: How to ensure that the object is not recycled or reused before adding 1 to the reference count? +There are currently 2 methods in OceanBase that use reference counting, you can refer to: +The following simple scenarios can use reference counting: + a. Single-threaded object construction, the initial reference count of the object is 1 + b. After that, the single thread increases the reference count, and passes the object to the rest of the threads for use, and the rest of the threads decrement the reference count after use. + c. Finally, the single thread decides to release the object and decrements the reference count by 1. +This is the usage of `FifoAllocator` in OceanBase. +If the above simple scenario is not satisfied, a global lock is required to ensure security: + a. Add a read lock in step 1 in Example 1 + b. Add a write lock in step 3 in Example 1 +This is what UPS does when it manages `schema_mgr` +## 10.10 Alignment +In order to avoid cache false sharing, if a variable will be frequently accessed by multiple threads, it is recommended to align it with the cache line when defining the variable. +```cpp +int64_t foo CACHE_ALIGNED; +``` +But if there are a large number of objects, in order to save memory, it is allowed not to align by cache line. +If it is an object constructed by dynamically applying for memory, you need to pay attention that at least the starting address of the object is 8-byte aligned. For example. If you use `page_arena`, you can allocate 8-byte aligned memory through `alloc_aligned()`. +```cpp +struct_A *p = page_arena_.alloc_aligned(sizeof(*p)); +``` +## 10.11 volatile +Generally speaking, it is not recommended to use volatile variables, for reasons refer to this document [volatile-considered-harmful](https://www.kernel.org/doc/Documentation/volatile-considered-harmful.txt). +Use `ATOMIC_LOAD()/ATOMIC_STORE()` instead to ensure that reads and writes to variables are not optimized away. +```cpp +// Wrong +volatile int64_ti = 0; +x = i; +i = y; +// recommended practice +int64_t i = 0; +x = ATOMIC_LOAD(&i); +ATOMIC_STORE(&i, y); +``` +It is still reasonable to use volatile in a few cases, such as to indicate the state, but this state change has no strict timing meaning: such as a flag variable indicating thread exit. +```cpp +volatile bool stop_CACHE_ALIGNED; +``` +Or certain monitoring items. +```cpp +volatile int64_t counter_CACHE_ALIGNED; +``` +## 10.12 How to Use CAS +Because `ATOMIC_VCAS` returns the latest value of *addr in the event of an operation failure, it is not necessary to use `ATOMIC_LOAD` to read again every retry. +For example, to achieve atomic plus 1, use the CAS operation as follows: +```cpp +int64_t tmp_val = 0; +int64_told_val = ATOMIC_LOAD(addr) +while(old_val != (tmp_val = ATOMIC_VCAS(addr, old_val, old_val + 1))) +{ + old_val = tmp_val; +} +``` +## 10.13 Spin Wait and PAUSE +Add `PAUSE()` to the spin wait cycle. On some CPUs, `PAUSE()` can improve performance, and generally speaking, `PAUSE()` can reduce CPU power consumption. +```cpp +while (need_retry()) { + PAUSE(); +} +``` +The role of PAUSE can be seen in this answer: [What is the purpose of the "PAUSE" instruction in x86](http://stackoverflow.com/questions/12894078/pause-instruction-in-x86/12904645#12904645) +## 10.14 Critical Sections +Do not perform time-consuming or complicated operations in the critical section, such as opening/closing files, reading and writing files, etc. +## 10.15 Avoid Program Core or Exit +The restart time of the database system is often measured in hours. A large area of core or exit will cause the interruption of database services and may be exploited by malicious attackers. Therefore, the program core or exit must be avoided, such as accessing the address pointed to by a null pointer (except for temporary modification for locating bugs), or calling abort (unless an external instruction is received), etc. If it must be violated, please obtain the consent of the project leader and project architect in advance, and explain the reasons in detail. + +# 11 Log Specification +Version 1.0 of the logging module has two major improvements: +**Support multi-dimensional, fine-grained printing level settings** +Compared with the previous version, which only supported the global uniform setting of the log level, the 1.0 version supports four different scopes of printing log settings: statement, session, tenant and global (or server). The setting methods of different ranges are as follows: +- SQL statement hint +- Set the session log level variable +- Set tenant log level variable +- Set syslog level variable +In addition, version 1.0 also supports the concepts of log modules and submodules. When printing logs in the program, it is necessary to indicate the module (or module + submodule) to which the log belongs and the log printing level to which the log belongs. The system supports users to set different printing levels for each module and sub-module. +**Stricter log printing format** +In version 0.5, there is a problem that the print log format is not uniform and difficult to read. For example, when printing the variable `m` whose value is 5, there are many different printing formats: "m = 5", "m=5", "m(5)", +"m is 5", "m:5", etc. The new log module allows users to print the values of required variables in the form of key-value pairs. +## 11.1 Log Printing Level + +| Log Level | User | Level Definition | +| --------- | ---- | -------- | +| ERROR | DBA | Any unexpected, unrecoverable error requiring human intervention. The observer cannot provide normal service exceptions, such as the disk is full and the listening port is occupied. It can also be some internal inspection errors after our productization, such as our 4377 (dml defensive check error), 4103 (data checksum error), etc., which require DBA intervention to restore | +| WARN | DBA | In an unexpected scenario, the observer can provide services, but the behavior may not meet expectations, such as our write current limit | +| INFO | DBA | (Startup default level). A small amount of flagged information about system state changes. For example, a user, a table is added, the system enters daily merge, partition migration, etc. | +| EDIAG | RD | Error Diagnosis, diagnostic information to assist in troubleshooting, unexpected logical errors, such as function parameters that do not meet expectations, etc., usually OceanBase program BUG | +| WDIAG | RD | Warning Diagnosis, diagnostic information to assist in troubleshooting, expected errors, such as function return failure | +| TRACE | RD | Requests granular debugging information, such as printing a TRACE log at different stages of executing a SQL statement | +| DEBUG | RD | General and detailed debugging information to track the internal state and data structure of the system. | + +It should be noted that DEBUG logs are often used for integration testing or online system debugging, and cannot be used as a substitute for unit testing. + +## 11.2 Division of Printing Modules (Example) +| Module | Submodule Definition | +| ----------- | --------------------------------------------------- | +| SQL | Parser, transformer, optimizer, executor, scheduler | +| STORAGE | TBD | +| TRANSACTION | TBD | +| ROOTSERVER | TBD | +| COMMON | TBD | +| DML | TBD | + +The definition of sub-modules under each module will be further refined by each group. The definitions of modules and submodules are placed in the file ob_log_module.h. +## 11.3 Setting of Print Range +Version 1.0 supports users to set the printing level separately by statement, session and global (system) scope. The priority of reference in the system is +1. statement +2. session +3. For system global (or server), only when the previous item is not set or the setting is invalid, the system will refer to the subsequent level settings. +### 11.3.1 Statement Scope Printing Level Setting +**Set format** +Add `/*+ ... log_level=[log_level_statement]...*/` to the statement hint +(For the format of log_level_statement, see the following chapters) +**Scope of action** +The processing and execution process of the entire statement, including statement analysis, optimization, execution, etc. After the execution of the statement ends, this setting becomes invalid automatically. +### 11.3.2 Session Scope Printing Level Setting +**Set format** +```sql +sql> set @@session.log_level = '[log_level_statement]'; +``` +**Scope of action** +From the setting to the end of the session. +### 11.3.3 Tenant-wide Printing Level Settings +**Set format** +```sql +sql>set @@global.log_level = '[log_level_statement]'; +``` +**Scope of action** +It will take effect for all user sessions from the user setting until all user sessions exit. +### 11.3.4 System (or server) Wide Printing Level Setting +**Set format** + +```sql +sql>alter system set log_level = '[log_level_statement]{,server_ip=xxx.xxx.xxx.xxx}'; +``` +**Scope of action** +When the user specifies `server_ip`, the setting takes effect only for the server, and remains valid until the server exits or restarts. When the user does not specify `server_ip`, the setting takes effect for all servers in the entire system, and remains until the entire system reboots (newly launched servers also need to obey this setting). +### 11.3.5 log_level_statement Format +``` +log_level_statement = +mod_level_statement {, mod_level_statement } +mod_level_statement= +[mod[.[submod|*]]:][ERROR|WARNING|INFO|TRACE|DEBUG] +``` + +The definitions of mod and submod refer to section 12.2. If no mod or submod is specified, this setting will take effect for all mods. If multiple `mod_level_statement` settings conflict, the last valid setting shall prevail. +User settings do not guarantee atomicity: for example, when there are multiple settings, if the nth item setting is unsuccessful (syntax error or module does not exist), if it is a session or system-level setting, the statement will report an error, but before The effective item will not be rolled back. If it occurs in the statement hint, no error will be reported and the previous effective item will not be rolled back. +## 11.4 Unification of Log Format +Version 1.0 uses the "key=value" format to print logs uniformly. The log module uniformly provides an interface similar to the following: +```cpp +OB_MOD_LOG(mod,submod, level, "info_string", var1_name, var1, var2, 2.3, current_range, + range, ...); +``` +The corresponding print information is + +```txt +[2014-10-09 10:23:54.639198] DEBUG ob_tbnet_callback.cpp:203 [12530][Ytrace_id] info_string(var1_name=5, var2=2.3, current_range= "table_id:50,(MIN;MAX)" ) +``` +Among them, `info_string` is a summary of the main information of the log, which should be concise, clear, and easy to read. Avoid "operator failed" and other non-informative character strings. +The log header of each line (including information such as file name and line number) is automatically generated by the log printing module. For ease of use, the log module header file (`ob_log_module.h`) will also provide macros defined in units of modules and submodules, making the program print statements in a certain file or folder more concise, for example: +```cpp +#define OB_SQL_PARSER_LOG(level, ...) OB_MOD_LOG(sql, parser, level, ...) +``` +The choice of the name of the printed variable should consider the needs of different occasions. If you do not use the variable name itself, you should consider whether there is already a variable name with the same meaning in use in the system (for example, whether the version number is printed as "data_version" or "version", or "data version" should be as uniform as possible), so as to facilitate future debugging and monitor. +In case of return due to unsuccessful operation, the log must be printed, and the error code must be printed. +Since the new log supports module and range settings, it will be more effective in filtering the printed information. In principle, the necessary debug log information should be further enriched to facilitate future troubleshooting and debugging. + +# 12 Coding Constraint Summary +## 12.1 Scope +1. Namespaces correspond to directories. **Anonymous namespaces are prohibited**. **Using directives are prohibited in .h files, and only using declarations are allowed**. +2. Nested classes are suitable for scenarios that are only used by external classes. It is recommended to pre-declare in .h files and implement them in .cpp files. Try not to use `public`. +3. **In addition to the existing global variables and global functions, no new global variables and global functions shall be added**. If it must be violated, please obtain the consent of the code owner in advance, and explain the reason in detail. +4. **Local variables are declared at the beginning of the statement block, and it is mandatory to initialize simple variables when they are declared.** +5. **It is forbidden to declare non-simple variables in the loop body**. If it must be violated, please obtain the consent of the group leader in advance, and explain the reason in detail. +6. **Resource management follows the principle of "who applies for release"**. If resources need to be released, release them before the function returns or at the end of the outermost else branch. So if you need to restore the input parameters, do so before the function returns. If it must be violated, please obtain the consent of the group leader in advance, and explain the reason in detail. +## 12.2 Class +1. The constructor only does trivial initialization. Each class needs to define at least one constructor, and the destructor with virtual functions or subclasses is declared as virtual. +2. In order to avoid implicit type conversion, the single-parameter constructor needs to be declared as explicit. +3. **In principle, the copy constructor must not be used (except for the basic classes that have been defined and used)**. If it must be violated, please obtain the consent of the group leader in advance, and explain the reason in detail. +4. Use `DISALLOW_COPY_AND_ASSIGN` to avoid abuse of copy constructor and assignment operation; +5. **Class reset uses reset, reuse uses reuse, and clear is prohibited.** +6. **It is necessary to ensure that all members are initialized, and the initialization order of member variables is consistent with the order of definition.** +7. Use struct only when there is only data, and use class in all other cases. +8. The common functions contained in each class must use standard prototypes, and the serialization/deserialization functions must be implemented using macros. +9. Prioritize composition and only use inheritance for "is-a" relationships. Avoid private inheritance and multiple inheritance. When multiple inheritance is used, it is required that except for one base class with implementation, the other base classes are pure interface classes. +10. **Except for existing container classes, custom types, and a small number of global basic classes, overloading of operators is not allowed (except for simple structure assignment operations)**. If it must be violated, please obtain the consent of the group leader in advance, and explain the reasons in detail. +11. Declaration order: `public`, `protected`, `private`. +## 12.3 Functions +1. **Strictly abide by the single entry and single exit of the function. If it must be violated, please obtain the consent of the code owner and project architect in advance, and explain the reasons in detail.** +2. **Except for simple access functions set_xxx()/get_xxx() and a few exceptions (such as operator overloading, existing at(i) functions, general function reset()/reuse() of classes, etc.), all functions (public and private) should use ret to return the error code**. If the `set/get` is complicated or may make an error, ret should still be used to return the error code. Only ret variables of type int can be used to represent errors, and ret can only represent errors (except for iterator functions due to historical reasons). +3. If multiple sequential statements are doing the same thing, then, in some cases, you can use simplified writing. +4. Judging `OB_SUCCESS == ret` in the loop condition to prevent the error code from being overwritten. +5. Conditional statements need to abide by the MECE principle: each condition is independent of each other and completely exhausted, and the number of branches of a single if/else should not exceed 5 as far as possible. +6. Declare functions/function parameters as const whenever possible. +7. **The principle of coding: do not trust anyone in the code! Every function (whether public or private, except inline functions) must check the legality of each input parameter, and it is strongly recommended that inline functions also perform these checks (unless there are serious performance problems). All functions (whether public or private) must check the legality of values obtained from class member variables or through function calls (such as get return values or output parameters), even if the return value is successful, the legality of output parameters must still be checked . Variable (parameter) check, only needs to be checked once in a function (if the value obtained by calling one or several functions multiple times, then check each time)**. When defining functions, the recommended order is: input parameters first, output parameters last. +8. **Prohibit the use of assert and OB_ASSERT**. +9. Try to avoid passing in some meaningless special values when calling the function, and use constants instead. +10. On the premise of adhering to idioms, use more references. +11. It is mandatory that a single function does not exceed 120 lines. If it must be violated, please obtain the consent of the group leader in advance, and explain the reason in detail. +## 12.4 C&C++ Features +1. **Smart pointers are not allowed**, and resources are allowed to be released automatically through the Guard class. +2. It is required to use the memory allocator to apply for memory, and immediately set the pointer to NULL after the memory is released. +3. **Prohibit the use of std::string class, use ObString instead**. In addition, when manipulating C strings, it is required to use length-limited string functions. +4. **When passing an array/string/buffer as a parameter, the length must be passed at the same time. When reading and writing the content of the array/string/buffer, check whether the subscript is out of bounds.** +5. Friends can only be used in the same file. If it must be violated, please obtain the consent of the group leader in advance and explain the reason in detail. Declaring unit test classes as friends is an exception, but should be used with caution. +6. **Prohibit the use of C++ exceptions.** +7. Prohibit the use of runtime type identification (RTTI). +8. Use C++ type conversions such as `static_cast<>`, and prohibit the use of C-type conversions like int `y = (int) x`. +9. Try to use `to_cstring` output. +10. Use int for the returned ret error code, and use `int64_t` for function parameters and loop times as much as possible. In other cases, use a signed number with a specified length, such as `int32_t`, `int64_t`. Try to avoid using unsigned numbers. +11. Try to use `sizeof(var_name)` instead of `sizeof(type)`. +12. Use 0 for integers, 0.0 for real numbers, `NULL` for pointers, and '\0' for strings. +13. **In addition to the existing macros, no new macros shall be defined, and inline functions, enumerations and constants shall be used instead**. If it must be violated, please obtain the consent of the group leader in advance, and explain the reason in detail. +14. **Except for the algorithm class functions defined in the header file in STL, the use of STL and boost is prohibited. If it must be violated, please obtain the consent of the code owner and project architect in advance, and explain the reasons in detail.** +## 12.5 Others +- **Do not perform time-consuming or complex operations in the critical section, such as opening/closing files, reading and writing files, etc.** +- **Do not use shared_ptr and strictly limit the use of reference counting.** +- **It is necessary to avoid program core or exit, such as accessing the address pointed to by a null pointer (except temporary modification for locating bugs), or calling abort (unless an external instruction is received). If it must be violated, please obtain the consent of the code owner and project architect in advance, and explain the reasons in detail.** diff --git a/docs/logging.md b/docs/logging.md new file mode 100644 index 000000000..07fb090e9 --- /dev/null +++ b/docs/logging.md @@ -0,0 +1,364 @@ +# OceanBase System Log Introduction + +## Introduction + +This document mainly introduces the system logs of Oceanbase, including the classification and level of the log, how to output logs in the program, and the details of some log implementation. + + +## System Log Introduction + +Similar to common application systems, system logs are one of the important means for Oceanbase developers to investigate problems. +Oceanbase's system log is stored under the log directory under the observer installation path. The system log is mainly divided into two categories: + +1. Ordinary logs: with ".log" suffix, printed all logs (including warning logs) of a certain module. + +2. Warning log: with ".log.wf" suffix, only printed the warn level of a module and above. + +| log file name | record information | +| ------------------ | ----------------------------- | +| observer.log[.wf] | General logs (warning logs, general query logs, other logs) | +| rootservice.log[.wf] | rootservice module log (including global DDL log) | +| election.log[.wf] | Election related logs | +| trace.log | Full link tracking log | + +Especially, trace.log does not have the corresponding ".wf" log. + +In addition to output ordinary logs, `wf` logs also have a special `info` log, that is, every time the log file is created, some current systems and processes information will be recorded. + +### Log Parameters + +There are 7 parameters related to syslog, which are dynamically effective, that is, it can be adjusted dynamically during runtime. + +| Configuration Item | Type | Range | Default Value | Describtion | +| --------------------------- | ---- | ----------------------------------------- | ------ | ------------------------------------- | +| enable_syslog_recycle | Boolean | | False | Whether to recycle the old log files | +| enable_syslog_wf | Boolean | | True | Whether to print the WARN log level and above to a separate WF file | +| enable_async_syslog | Boolean | | True | Whether to print the log asynchronous | +| max_syslog_file_count | Integer | \[0, +∞) | 0 | The maximum number of each log file | +| syslog_io_bandwidth_limit | String | 0, Other legal size | "30MB" | Log IO bandwidth limit | +| syslog_level | String | DEBUG, TRACE, WDIAG, EDIAG, INFO, WARN, ERROR | WDIAG | Log level| +| diag_syslog_per_error_limit | Integer | \[0, +∞) | 200 | The maximum number of each error code of DIAG system log per second. | + +> All the parameters here are cluster-level and dynamic effect. +> Refer to ob_parameter_seed.ipp file for more details. + +## Log File Recycle + +OceanBase's log can be configured with the upper limit of the number of files to prevent the log file from occupying too much disk space. + +If `enable_syslog_recycle = true` and `max_syslog_file_count > 0`, the number of each type log files can not exceed `max_syslog_file_count`. OceanBase will detect and delete old log files periodically. + +The new log files will print a special log at the beginning. The information contains the IP and ports of the current node, version number, and some system information. Refer to `ObLogger::log_new_file_info` for more details. + +``` +[2023-12-26 13:15:58.612579] INFO New syslog file info: [address: "127.0.0.1:2882", observer version: OceanBase_CE 4.2.1.1, revision: 101010012023111012-2f6924cd5a576f09d6e7f212fac83f1a15ff531a, sysname: Linux, os release: 3.10.0-327.ali2019.alios7.x86_64, machine: x86_64, tz GMT offset: 08:00] +``` + +## Log Level + +Similar to the common system, Oceanbase also provides log macro to print different levels of logs: + +| Level | Macro | Describtion | +| ----- | --------- | ---- | +| DEBUG | LOG_DEBUG | Developers debug logs | +| TRACE | LOG_TRACE | Incident tracking logs are usually viewed by developers | +| INFO | LOG_INFO | System state change log | +| WARN | LOG_DBA_WARN | For DBA. observer can provide services, but the behavior not meet expectations | +| ERROR | LOG_DBA_ERROR | For DBA. observer cannot provide services, such as the disk full of monitoring ports occupied. Need DBA intervention to restore service | +| WDIAG | LOG_WARN | Warning Diagnosis. Assisting the diagnostic information of fault investigation, and the errors in the expected expectations, if the function returns failure. The level is the same as WARN | +| EDIAG | LOG_ERROR | Error Diagnosis. Assisting the diagnostic information of faulty investigation, unexpected logical errors, such as the function parameters do not meet the expected, are usually Oceanbase program bugs. The level is the same as ERROR | + + +> Only the most commonly used log levels are introduced here. For more detailed information, please refer to the configuration of syslog_level in `ob_parameter_seed.ipp`, and macro definitions such as `LOG_ERROR` in the `ob_log_module.h` file. + +**How to set up log level?** + +There are three ways to adjust the log level: + +- When the OceanBase process starts, it reads the log level config from configuration file or command line parameters. The configuration item name is `syslog_level`; +- After startup, you can also connect through the MySQL client and execute the SQL command `alter system set syslog_level='DEBUG'`; +- Modify the log level when the request is executed through the SQL Hint. For example `select /*+ log_level("ERROR") */ * from foo;`. This method is only effective for the current SQL request related logs. + +You can refer to the code of dynamic modification log settings `ObReloadConfig::reload_ob_logger_set`。 + +```cpp +if (OB_FAIL(OB_LOGGER.parse_set(conf_->syslog_level, + static_cast(STRLEN(conf_->syslog_level)), + (conf_->syslog_level).version()))) { + OB_LOG(ERROR, "fail to parse_set syslog_level", + K(conf_->syslog_level.str()), K((conf_->syslog_level).version()), K(ret)); +``` + +## How to Print Logs + +Common systems use C ++ Stream mode or C fprintf style printing log, but Oceanbase is slightly different. Let's start with the example to see how to print logs. + +### An Example of Printing Log + +Unlike `fprintf`, Oceanbase's system log does not have a format string, but only "info" parameter, and each parameter information. For example: + +```cpp +LOG_INFO("start stmt", K(ret), + K(auto_commit), + K(session_id), + K(snapshot), + K(savepoint), + KPC(tx_desc), + K(plan_type), + K(stmt_type), + K(has_for_update), + K(query_start_time), + K(use_das), + K(nested_level), + KPC(session), + K(plan), + "consistency_level_in_plan_ctx", plan_ctx->get_consistency_level(), + K(trans_result)); +``` + +Among the example, "start stmt" is the `INFO` information, and we uses the `K` macro to print objects. + +### Log Field Introduction + +A output of the example code above: + +```text +[2023-12-11 18:00:55.711877] INFO [SQL.EXE] start_stmt (ob_sql_trans_control.cpp:619) +[99178][T1004_TeRec][T1003][YD9F97F000001-00060C36119D4757-0-0] [lt=15] +start stmt(ret=0, auto_commit=true, session_id=1, +snapshot={this:0x7f3184fca0e8, valid:true, source:2, +core:{version:{val:1702288855549635029, v:0}, tx_id:{txid:167035}, +scn:1702288855704049}, uncertain_bound:0, snapshot_lsid:{id:1}, +snapshot_ls_role:0, parts:[{left:{id:1}, right:491146514786417}]}, +savepoint=1702288855704049, tx_desc={this:0x7f31df697420, +tx_id:{txid:167035}, state:2, addr:"127.0.0.1:55801", tenant_id:1003, +session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", +xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, +isolation:1, snapshot_version:{val:18446744073709551615, v:3}, +snapshot_scn:0, active_scn:1702288855704040, op_sn:6, alloc_ts:1702288855706134, +active_ts:1702288855706134, commit_ts:-1, finish_ts:-1, timeout_us:29999942, +lock_timeout_us:-1, expire_ts:1702288885706076, coord_id:{id:-1}, +parts:[{id:{id:1}, addr:"127.0.0.1:55801", epoch:491146514786417, +first_scn:1702288855704043, last_scn:1702288855704048, last_touch_ts:1702288855704044}], +exec_info_reap_ts:1702288855704043, commit_version:{val:18446744073709551615, v:3}, +commit_times:0, commit_cb:null, cluster_id:1, cluster_version:17180065792, +flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, +flags_.REPLICA:false, can_elr:true, cflict_txs:[], abort_cause:0, +commit_expire_ts:0, commit_task_.is_registered():false, ref:2}, +plan_type=1, stmt_type=5, has_for_update=false, query_start_time=1702288855711692, +use_das=false, nested_level=0, session={this:0x7f31de2521a0, id:1, +deser:false, tenant:"sys", tenant_id:1, effective_tenant:"sys", +effective_tenant_id:1003, database:"oceanbase", user:"root@%", +consistency_level:3, session_state:0, autocommit:true, tx:0x7f31df697420}, +plan=0x7f31565ba050, consistency_level_in_plan_ctx=3, +trans_result={incomplete:false, parts:[], touched_ls_list:[], +cflict_txs:[]}) +``` + +> NOTE: The log output is wrapped for readability. + +A log mainly contains the following parts: + +| field | example | description | +| -------- | ------------------------------------ | ----------------------------------- | +| time | [2023-12-11 18:00:55.711877] | The time of printing this log | +| level | INFO | The log level | +| module | [SQL.EXE] | The module printing the log | +| function name | start_stmt | The function printing the log | +| code location | (ob_sql_trans_control.cpp:619) | The location of code, including file name and line | +| thread identifier | [99178][T1004_TeRec] | The thread ID and name | +| tenant id | [T1003] | The tenant ID | +| Trace ID | [YD9F97F000001-00060C36119D4757-0-0] | The global ID of a specific request. You can usually get all logs related one request according the trace ID | +| The cost of printing log | [lt=15] | The cost in microsecond of printing last log | +| information | start stmt(...) | The log information | + +### Commonly Used Log Parameters Macro Introduction + +For developers, we only need to care about how to output our object information. Usually we write `K(obj)` to output the information we want in the log. Below are some details。 + +In order to avoid some errors in format string, OceanBase uses automatic recognition of types and then serialization to solve this problem. Any parameter in the log will be identified as multiple Key Value pairs, where Key is the name of the field to be printed and Value is the value of the field. For example, `"consistency_level_in_plan_ctx", plan_ctx->get_consistency_level()` in the above example prints the name and value of a field. OceanBase automatically recognizes the type of Value and converts it to a string. The final output in the log may be "consistency_level_in_plan_ctx=3". + +Because most logs print the original name and value of the specified object, OceanBase provides some macros to simplify the operation of printing logs. The most commonly used one is `K`. Taking the above example `K(ret)`, its expansion in the code is: + +```cpp +"ret", ret +``` + +The final information in the log is: +```cpp +ret=-5595 +``` + +OceanBase also provides some other macros, which are used in different scenarios. + +> Log parameter macro definitions can be found in the `ob_log_module.h` file. + +| macro | example | description | +| ------ | --------------- | ----------- | +| K | K(ret) | After expansion, it is `"ret", ret`. The parameter can be a simple value or an ordinary object | +| K_ | K_(consistency_level) | After expansion, it is `"consistency_level", consistency_level_`. Different from K, the `_` suffix will be automatically added after the expanded Value, which is used for printing class member variables. | +| KR | KR(ret) | After expansion, it is `"ret", ret, "ret", common::ob_error_name(ret)`. This macro is for the convenience of printing error code and error code name. In OceanBase, `ret` is usually used as the return value of a function, and each return value has a corresponding string description. `ob_error_name` can get the string description corresponding to the error code. Note that this macro can only be used in non-lib code | +| KCSTRING/
KCSTRING_ | KCSTRING(consistency_level_name) | After expansion, it is `"consistency_level_name", consistency_level_name`. This macro is used to print C-formatted strings. Since a variable of type `const char *` does not necessarily represent a string in C++, such as a binary buffer, when printing the value of this variable, if it is printed as a C string, an illegal memory access error will occur, so this macro has been added to explicitly print C strings | +| KP/KP_ | KP(plan) | After expansion, it is `"plan", plan`, where `plan` is a pointer. This macro will print out the hexadecimal value of a pointer | +| KPC/KPC_ | KPC(session) | The input parameters are object pointers. If it is NULL, "NULL" will be output. Otherwise, the `to_string` method of the pointer will be called to output the string. | +| KTIME | KTIME(cur_time) | Convert timestamp converted to string. Timestamp unit microseconds | +| KTIMERANGE/
KTIMERANGE_ | KTIMERANGE(cur_time, HOUR, SECOND) | Convert the timestamp to a string and only obtain the specified range, such as the hour to second period in the example | +| KPHEX/KPHEX_ | KPHEX(buf, 20) | Print buf content in hexadecimal | +| KERRMSG | KERRMSG | Output system error code information | +| KERRNOMSG | KERRNOMSG(2) | Specify error code to output system error information | + +## Some Implementation Details in the Log + +### How to Convert Value to String + +OceanBase automatically identifies the type of value you want to print in the log and converts it to a string. For example, in the above example, `ret` is an `int` type variable, and `plan_ctx->get_consistency_level()` returns an `enum` type variable. Both variables will be converted to strings. + +However, since OceanBase does not know how to convert an ordinary object into a string, the user needs to implement a `TO_STRING_KV` function to convert the object into a string. For example, in the above example, `snapshot` is an object of type `ObTxReadSnapshot`. This object implements the `TO_STRING_KV` function, so it can be printed directly. + +**Convert normal value to string** + +OceanBase can automatically identify simple type values, such as `int`, `int64_t`, `double`, `bool`, `const char *`, etc., and convert them into strings. For enumeration types, they will be treated as numbers. For pointers, the pointer value will be output in hexadecimal format. + +**Convert class object to string** + +Since C++ does not have a reflection mechanism, it cannot automatically identify the member variables of a class object and convert them into strings. Therefore, the user needs to implement a `TO_STRING_KV` function to convert the object into a string. For example, in the above example, `snapshot` is an object of type `ObTxReadSnapshot`. This object implements the `TO_STRING_KV` function. You can refer to the implementation code as follows: + +```cpp +class ObTxReadSnapshot { + ... + TO_STRING_KV(KP(this), + K_(valid), + K_(source), + K_(core), + K_(uncertain_bound), + K_(snapshot_lsid), + K_(parts)); +}; +``` + +As you can see, in `TO_STRING_KV`, you can directly use a macro similar to printing logs to "list" the member variable names you want to output. + +> NOTE: TO_STRING_KV is actually a macro definition. For specific implementation, please refer to `ob_print_utils.h`. TO_STRING_KV converts input parameters into strings and outputs them to a buffer. + + + +### Log Module + +OceanBase's logs are module-specific and can support sub-modules. For example, in the above example, `[SQL.EXE]` is a module, `SQL` is a main module, and `EXE` is a submodule. For the definition of the log module, please refer to the `LOG_MOD_BEGIN` and `DEFINE_LOG_SUB_MOD` related codes in the `ob_log_module.h` file. + +**How does the log module output to the log?** + +Normally, we just use macros like `LOG_WARN` to print logs, and different modules will be output, which is also achieved through macro definitions. Still taking the above log as an example, you can see a macro definition `#define USING_LOG_PREFIX SQL_EXE` at the beginning of the `ob_sql_trans_control.cpp` file. This macro defines the log module of the current file, that is, all logs in the current file the module `[SQL.EXE]` will be printed. + +> There is also an issue here, that is, the header file introduced in the current implementation file will also use this module to print logs by default. + + +**How to specify module name explicitly?** + +The above method is indeed a bit inflexible. OceanBase has another way to specify the module name, which is to use the macro `OB_MOD_LOG` or `OB_SUB_MOD_LOG`. The usage of these two macros is similar to `LOG_WARN`, except that there are additional module parameters and log levels: + +```cpp +OB_MOD_LOG(parMod, level, info_string, args...) +OB_SUB_MOD_LOG(parMod, subMod, level, info_string, args...) +``` + +**Set the module's log level** + +In addition to setting the global and current thread log levels, OceanBase can also adjust the log level of a certain module. Currently, you can use `SQL HINT` to modify the log level of a module when executing a request, for example: + +```sql +select /*+ log_level("SHARE.SCHEMA:ERROR") */ * from foo; +``` + +Where `SHARE` is the main module, `SCHEMA` is the submodule, and `ERROR` is the log level. The function of this SQL HINT is to set the log level of the `SHARE.SCHEMA` module to `ERROR`, and is only valid for the current request. + +### Log Time + +OceanBase's log time is the number of microseconds in the current local time. +Since converting a timestamp into a string is a time-consuming task, OceanBase caches the timestamp conversion to speed up the process. For details, please refer to the `ob_fast_localtime` function. + +### Thread Identifier + +Currently, two information related to thread will be recorded: + +- Thread ID: the information returned by the system call `__NR_gettid` (the system call is relatively inefficient, and this value will be cached); +- Thread name: The thread name field may contain the tenant ID, thread pool type, and thread pool index. The thread name of OceanBase is set through the `set_thread_name` function and will also be displayed in the `top` command. + +> NOTE：The thread name is determined by the created thread. Since the tenant of the created thread may be different from the tenant of subsequent runs of this thread, the tenant in the thread name may be incorrect. + +### Log Rate Limit + +OceanBase supports two log rate limits: a common system log disk IO bandwidth limit and a WDIAG system log limit. + + +**System log bandwidth rate limit** + +OceanBase will limit log output according to disk bandwidth. The log bandwidth rate limit does not limit the rate for different log levels. If the log rate is limited, the rate limit log may be printed with the keyword `REACH SYSLOG RATE LIMIT`. + +Rate limit log example: + +```txt +[2023-12-26 09:46:04.621435] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [35675][VTblLocAsyncUp0][T0][YB427F000001-00060D52A9614571-0-0] [lt=0] REACH SYSLOG RATE LIMIT [bandwidth] +``` + +The rate limit can be adjusted through the configuration item `syslog_io_bandwidth_limit`. + +Please refer to the `check_tl_log_limiter` function for rate limiting code details. + +**WDIAG log rate limit** + +OceanBase has implemented a current limit for WARN level logs. Each error code is limited to 200 logs per second by default. If the limit is exceeded, the current limiting log will be output, keyword `Throttled WDIAG logs in last second`. The current limiting threshold can be adjusted through the configuration item `diag_syslog_per_error_limit`. + +Limiting log example: + +```txt +[2023-12-25 18:01:15.527519] WDIAG [SHARE] refresh (ob_task_define.cpp:402) [35585][LogLimiterRefre][T0][Y0-0000000000000000-0-0] [lt=8][errcode=0] Throttled WDIAG logs in last second(details {error code, dropped logs, earliest tid}=[{errcode:-4006, dropped:31438, tid:35585}]) +``` + +Limiting code reference `ObSyslogPerErrLimiter::do_acquire`。 + + +## Some Other Details + +### Logs for DBA + +There are also two types of special logs in OceanBase, LOG_DBA_WARN and LOG_DBA_ERROR, which correspond to WARN and ERROR logs respectively. Since the volume of OceanBase logs is extremely large, and most of them can only be understood by R&D personnel, it brings a certain burden to DBA operation and maintenance troubleshooting problems. Therefore, these two types of logs are added, hoping that the DBA can only focus on a small amount of these two types of logs to troubleshoot system problems. The logs output using LOG_WARN and LOG_ERROR are converted into WDIAG and EDIAG logs to help developers troubleshoot problems. + +### Output Prompt Information to the User Terminal + +Sometimes we want to output the error message directly to the user's terminal, so that it can be more convenient for users to understand what error is currently occurring. At this time we can use `LOG_USER_ERROR`, `LOG_USER_WARN`, `LOG_USER_INFO` and other macros to print logs. Each error code has a corresponding `USER_ERROR_MSG`. If this `USER_ERROR_MSG` requires input parameters, then we also need to provide the corresponding parameters when printing the log. For example, the error code `OB_NOT_SINGLE_RESOURCE_POOL` has the corresponding `OB_NOT_SINGLE_RESOURCE_POOL__USER_ERROR_MSG`, and it's message is "create tenant only support single resource pool now, but pool list is %s", we just need to provide a string. + +The LOG_USER_ERROR macro is defined as follows： + +```cpp +#define LOG_USER_ERROR(errcode, args...) +``` + +The usage of other macros is similar. + +> Error code definitions can be found in `src/share/ob_errno.h`. + +Since `LOG_USER_XXX` provides fixed error information, if we want to output some customized information, we can use `FORWARD_USER_XXX`, such as `FORWARD_USER_ERROR`, `FORWARD_USER_WARN`, etc. Taking `FORWARD_USER_ERROR` as an example, its definition is as follows: + +```cpp +#define FORWARD_USER_ERROR(errcode, args...) +``` + +### Health Log + +OceanBase will periodically output some internal status information, such as the memory information of each module and tenant, to the log to facilitate problem finding. This kind of log usually outputs multiple lines of data in one log, such as: + +```txt +[2023-12-26 13:15:58.608131] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [35582][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=116] +[MEMORY] tenant_id= 500 ctx_id= GLIBC hold= 4,194,304 used= 1,209,328 limit= 9,223,372,036,854,775,807 +[MEMORY] idle_size= 0 free_size= 0 +[MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 +[MEMORY] hold= 858,240 used= 575,033 count= 3,043 avg_used= 188 block_cnt= 93 chunk_cnt= 2 mod=glibc_malloc +[MEMORY] hold= 351,088 used= 104,389 count= 3,290 avg_used= 31 block_cnt= 51 chunk_cnt= 1 mod=Buffer +[MEMORY] hold= 1,209,328 used= 679,422 count= 6,333 avg_used= 107 mod=SUMMARY +``` + +This kind of data can be helpful for finding historical issues. + +### ERROR Log +For general errors that occur in the system, such as an exception when processing a certain request, logs will be output at WARN level. Only when the normal operation of the OceanBase process is affected, or if there is a serious problem, the log will be output at the ERROR level. Therefore, if a process exits abnormally or cannot be started, searching the ERROR log will more effectively find the cause of the problem. diff --git a/docs/memory.md b/docs/memory.md new file mode 100644 index 000000000..3c4c0f16c --- /dev/null +++ b/docs/memory.md @@ -0,0 +1,151 @@ +# Introduction +Memory management is one of the most important modules in any large C++ project. Since OceanBase also needs to deal with the issue of multi-tenant memory resource isolation, OceanBase's memory management is more complicated than ordinary C++ projects. Generally, a good memory management module needs to consider the following issues: + +- Easy to use. The designed interface must be understood and used by the container, otherwise the code will be difficult to read and maintain, and memory errors will be more likely to occur; +- Efficient. An efficient memory allocator has a crucial impact on performance, especially in high-concurrency scenarios; +- Diagnosis. As the amount of code increases, bugs are inevitable. Common memory errors, such as memory leaks, memory out-of-bounds, wild pointers and other problems cause headaches for development and operation and maintenance. How to write a function that can help us avoid or troubleshoot these problems is also an important indicator to measure the quality of the memory management module. + +For the multi-tenant model, the impact of memory management design mainly includes the following aspects: +- Transparent interface design. How to make developers have no awareness or little need to care about the memory management of different tenants; +- Efficient and accurate. Sufficient memory must be applied successfully, and tenant memory exhaustion must be detected in time, which is the most basic condition for multi-tenant memory management. + +This article will introduce the commonly used memory allocation interfaces and memory management related idioms in OceanBase. For technical details of memory management, please refer to [Memory Management](https://open.oceanbase.com/blog/8501613072)( In Chinese). + +# Common Interfaces and Methods of OceanBase Memory Management +OceanBase provides different memory allocators for different scenarios. In addition, in order to improve program execution efficiency, there are some conventional implementations, such as reset/reuse, etc. + +## ob_malloc + +OceanBase has developed a set of libc-style interface functions ob_malloc/ob_free/ob_realloc. This set of interfaces will dynamically apply for memory blocks of size based on tenant_id, ctx_id, label and other attributes, and mark the memory blocks to determine ownership. This not only facilitates multi-tenant resource management, but is also very helpful in diagnosing memory problems. +ob_malloc will index to the corresponding ObTenantCtxAllocator based on tenant_id and ctx_id, and ObTenantCtxAllocator will allocate memory according to the current tenant context. + +ob_free uses offset operation to find the object allocator corresponding to the memory to be released, and then returns the memory to the memory pool. + +ob_realloc is different from libc's realloc. It does not expand the original address, but first copies the data to another memory through ob_malloc+memcpy, and then calls ob_free to release the original memory. + +```cpp +inline void *ob_malloc(const int64_t nbyte, const ObMemAttr &attr = default_memattr); +inline void ob_free(void *ptr); +inline void *ob_realloc(void *ptr, const int64_t nbyte, const ObMemAttr &attr); +``` + +## OB_NEW/OB_NEWx +Similar to ob_malloc, OB_NEW provides a set of "C++" interfaces that call the object's constructor and destructor when allocating and releasing memory. + +```cpp +/// T is the type, label is the memory label and it can be a const string +#define OB_NEW(T, label, ...) +#define OB_NEW_ALIGN32(T, label, ...) +#define OB_DELETE(T, label, ptr) +#define OB_DELETE_ALIGN32(T, label, ptr) + +/// T is the type, pool is the memory pool allocator +#define OB_NEWx(T, pool, ...) +``` + +> There is no OB_DELETEx, but you can release the memory by yourself. + +## ObArenaAllocator + +The design feature is to allocate release multiple times and only release once. Only reset or destruction can truly release the memory. The memory allocated before will not have any effect even if `free` is actively called. + +ObArenaAllocator is suitable for scenarios where many small memory allocates are released in a short period of time. For example, in a SQL request, many small block memories will be frequently allocated, and the life cycle of these small memories will last for the entire request period. Usually, the processing time of an SQL request is also very short. This memory allocation method is very effective for small memory and avoiding memory leaks. In OceanBase's code, don't be surprised if you see there is only apply for memory but cannot find a place to release it. + +> Code reference `page_arena.h` + +## ObMemAttr Introduction + +OceanBase uses `ObMemAttr` to mark a section of memory. + +```cpp +struct ObMemAttr +{ + uint64_t tenant_id_; // tenant + ObLabel label_; // label or module + uint64_t ctx_id_; // refer to ob_mod_define.h, each ctx id is corresponding to a ObTenantCtxAllocator + uint64_t sub_ctx_id_; // please ignore it + ObAllocPrio prio_; // priority +}; +``` + +> reference file alloc_struct.h + +**tenant_id** + +Memory allocation management perform resource statistics and restrictions based on tenant maintenance. + +**label** + +At the beginning, OceanBase uses a predefined method to create memory labels for each module. However, as the amount of code increases, the method of predefined labels is not suitable. Currently, ObLabel is constructed directly using constant strings. When using ob_malloc, you can also directly pass in a constant string as the ObLabel parameter, such as `buf = ob_malloc(disk_addr.size_, "ReadBuf");`. + +**ctx_id** + +ctx id is predefined, please refer to `alloc_struct.h`. Each ctx_id of each tenant will create an `ObTenantCtxAllocator` object, which can separately count the related memory allocation usage. Normally use `DEFAULT_CTX_ID` as ctx id. For some special modules, for example, if we want to more conveniently observe memory usage or troubleshoot problems, we define special ctx ids for them, such as libeasy communication library (LIBEASY) and Plan Cache cache usage (PLAN_CACHE_CTX_ID). We can see periodic memory statistics in log files, such as: + +```txt +[2024-01-02 20:05:50.375549] INFO [LIB] operator() (ob_malloc_allocator.cpp:537) [47814][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=10] [MEMORY] tenant: 500, limit: 9,223,372,036,854,775,807 hold: 800,768,000 rpc_hold: 0 cache_hold: 0 cache_used: 0 cache_item_count: 0 +[MEMORY] ctx_id= DEFAULT_CTX_ID hold_bytes= 270,385,152 limit= 2,147,483,648 +[MEMORY] ctx_id= GLIBC hold_bytes= 8,388,608 limit= 9,223,372,036,854,775,807 +[MEMORY] ctx_id= CO_STACK hold_bytes= 106,954,752 limit= 9,223,372,036,854,775,807 +[MEMORY] ctx_id= LIBEASY hold_bytes= 4,194,304 limit= 9,223,372,036,854,775,807 +[MEMORY] ctx_id= LOGGER_CTX_ID hold_bytes= 12,582,912 limit= 9,223,372,036,854,775,807 +[MEMORY] ctx_id= PKT_NIO hold_bytes= 17,969,152 limit= 9,223,372,036,854,775,807 +[MEMORY] ctx_id= SCHEMA_SERVICE hold_bytes= 135,024,640 limit= 9,223,372,036,854,775,807 +[MEMORY] ctx_id= UNEXPECTED_IN_500 hold_bytes= 245,268,480 limit= 9,223,372,036,854,775,807 +``` + +**prio** + +Currently, two memory allocation priorities are supported, Normal and High. The default is Normal. For definition, please refer to the `enum ObAllocPrio` in file `alloc_struct.h`. High priority memory can allocate memory from urgent (memory_reserved) memory, otherwise it cannot. Refer to `AChunkMgr::update_hold` implementation. + +> You can use the configuration item `memory_reserved` to view the reserved memory size. + +## init/destroy/reset/reuse + +Caching is one of the important methods to improve program performance. Object reuse is also a way of caching. On the one hand, it reduces the frequency of memory allocate and release, and on the other hand, it can reduce some construction and destruction overhead. There is a lot of object reuse in OceanBase, and some conventions have been formed, such as the reset and reuse functions. + +**reset** + +Used to reset objects. Restore the object's state to the state after the constructor or init function was executed. For example `ObNewRow::reset`. + +**reuse** + +Compared with reset, it is more lightweight. Try not to release some expensive resources, such as `PageArena::reuse`. + +**init/destroy** + +There are two other common interfaces in OceanBase: `init` and `destroy`. `init` is used to initizalize object and `destory` to release resources. Only do some very lightweight initialization work in the constructor, such as initializing the pointer to `nullptr`. + +## SMART_VAR/HEAP_VAR + +SMART_VAR is an auxiliary interface for defining local variables. Variables using this interface are always allocated from the stack first. When the stack memory is insufficient, they will be allocated from the heap. For those large local variables (>8K) that are not easy to optimize, this interface not only ensures the performance of regular scenarios, but also safely reduces the stack capacity. The interface is defined as follows: + +```cpp +SMART_VAR(Type, Name, Args...) { + // do... +} +``` + +It allocate from the stack when the following conditions are met, otherwise allocate from the heap +```cpp +sizeof(T) < 8K || (stack_used < 256K && stack_free > sizeof(T) + 64K) +``` + +> SMART_VAR was created to solve historical problems. It try to reduce the amount of stack memory occupied by large memory objects. + +HEAP_VAR is similar to SMART_VAR, except that it must allocate memory on the heap. + +## SMART_CALL + +SMART_CALL is used to "quasi-transparently" resolve recursive function calls that may explode the stack on threads with very small stacks. This interface accepts a function call as a parameter. It will automatically check the current stack usage before calling the function. Once it is found that the available stack space is insufficient, a new stack execution function will be created on this thread immediately. After the function ends, it will continue to return to the original stack. This ensures performance when the stack is sufficient, and can also avoid stack explosion scenarios. + +```cpp +SMART_CALL(func(args...)) +``` + +Notice: +1. The return value of func must be an int type representing the error code. +2. SMART_CALL will return an error code. This may be an internal mechanism or a func call. +3. It supports stack cascade expansion, each time a 2M stack is expanded (there is a hard-coded total upper limit of 10M) + +Compared with direct calling, SMART_CALL only call `check_stack_overflow` to check stack. diff --git a/src/sql/engine/px/exchange/ob_px_transmit_op.cpp b/src/sql/engine/px/exchange/ob_px_transmit_op.cpp index dbb1b036a..b7e72c46e 100644 --- a/src/sql/engine/px/exchange/ob_px_transmit_op.cpp +++ b/src/sql/engine/px/exchange/ob_px_transmit_op.cpp @@ -105,7 +105,7 @@ int ObPxTransmitOpInput::get_data_ch(ObPxTaskChSet &task_ch_set, int64_t timeout OB_SERIALIZE_MEMBER((ObPxTransmitSpec, ObTransmitSpec), sample_type_, need_null_aware_shuffle_, tablet_id_expr_, random_expr_, sampling_saving_row_, repartition_table_id_, - wf_hybrid_aggr_status_expr_, wf_hybrid_pby_exprs_cnt_array_); + wf_hybrid_aggr_status_expr_, wf_hybrid_pby_exprs_cnt_array_, ddl_slice_id_expr_); ObPxTransmitSpec::ObPxTransmitSpec(ObIAllocator &alloc, const ObPhyOperatorType type) : ObTransmitSpec(alloc, type), @@ -116,7 +116,8 @@ ObPxTransmitSpec::ObPxTransmitSpec(ObIAllocator &alloc, const ObPhyOperatorType sampling_saving_row_(alloc), repartition_table_id_(0), wf_hybrid_aggr_status_expr_(NULL), - wf_hybrid_pby_exprs_cnt_array_(alloc) + wf_hybrid_pby_exprs_cnt_array_(alloc), + ddl_slice_id_expr_(NULL) { } diff --git a/src/sql/engine/px/exchange/ob_px_transmit_op.h b/src/sql/engine/px/exchange/ob_px_transmit_op.h index 65dc2af86..a2de8b61d 100644 --- a/src/sql/engine/px/exchange/ob_px_transmit_op.h +++ b/src/sql/engine/px/exchange/ob_px_transmit_op.h @@ -91,6 +91,8 @@ public: int64_t repartition_table_id_; // for pkey, target table location id ObExpr *wf_hybrid_aggr_status_expr_; common::ObFixedArray wf_hybrid_pby_exprs_cnt_array_; + // for calc ddl sice idx + ObExpr *ddl_slice_id_expr_; }; class ObPxTransmitOp : public ObTransmitOp From df87f4f90a5d74789ddff13417a351f9579a40ae Mon Sep 17 00:00:00 2001 From: obdev Date: Wed, 14 Aug 2024 07:08:08 +0000 Subject: [PATCH 042/249] add 4.3.2.1 --- tools/upgrade/oceanbase_upgrade_dep.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/upgrade/oceanbase_upgrade_dep.yml b/tools/upgrade/oceanbase_upgrade_dep.yml index 70f8b71ef..36844c346 100644 --- a/tools/upgrade/oceanbase_upgrade_dep.yml +++ b/tools/upgrade/oceanbase_upgrade_dep.yml @@ -78,10 +78,14 @@ - version: 4.3.1.0 can_be_upgraded_to: - - 4.3.2.0 + - 4.3.2.1 - version: 4.3.2.0 can_be_upgraded_to: - 4.3.3.0 -- version: 4.3.3.0 \ No newline at end of file +- version: 4.3.2.1 + can_be_upgraded_to: + - 4.3.3.0 + +- version: 4.3.3.0 From 0881d89dd1975d170e1e8c26537aa55c4c3c99c1 Mon Sep 17 00:00:00 2001 From: zhjc1124 Date: Wed, 14 Aug 2024 07:25:47 +0000 Subject: [PATCH 043/249] fix wrong set of IO_TUNING is_inited_ --- src/share/io/ob_io_struct.cpp | 69 +++++++++++++++-------------------- 1 file changed, 30 insertions(+), 39 deletions(-) diff --git a/src/share/io/ob_io_struct.cpp b/src/share/io/ob_io_struct.cpp index 8059b5f49..5842545cf 100644 --- a/src/share/io/ob_io_struct.cpp +++ b/src/share/io/ob_io_struct.cpp @@ -843,28 +843,23 @@ void ObIOTuner::destroy() void ObIOTuner::run1() { int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret), K(is_inited_)); - } else { - const int64_t thread_id = get_thread_idx(); - set_thread_name("IO_TUNING", thread_id); - LOG_INFO("io tuner thread started"); - while (!has_set_stop()) { - //try to update callback_thread_count. - (void) try_release_thread(); - // print interval must <= 1s, for ensuring real_iops >= 1 in gv$ob_io_quota. - if (REACH_TIME_INTERVAL(1000L * 1000L * 1L)) { - print_io_status(); - print_sender_status(); - if (OB_FAIL(send_detect_task())) { - LOG_WARN("fail to send detect task", K(ret)); - } + const int64_t thread_id = get_thread_idx(); + set_thread_name("IO_TUNING", thread_id); + LOG_INFO("io tuner thread started"); + while (!has_set_stop()) { + //try to update callback_thread_count. + (void) try_release_thread(); + // print interval must <= 1s, for ensuring real_iops >= 1 in gv$ob_io_quota. + if (REACH_TIME_INTERVAL(1000L * 1000L * 1L)) { + print_io_status(); + print_sender_status(); + if (OB_FAIL(send_detect_task())) { + LOG_WARN("fail to send detect task", K(ret)); } - ob_usleep(100 * 1000); // 100ms } - LOG_INFO("io tuner thread stopped"); + ob_usleep(100 * 1000); // 100ms } + LOG_INFO("io tuner thread stopped"); } int64_t ObIOTuner::to_string(char *buf, const int64_t len) const @@ -2763,9 +2758,10 @@ int ObIORunner::init(const int64_t queue_capacity, ObIAllocator &allocator) LOG_WARN("init queue failed", K(ret), K(queue_capacity), KP(buf)); } else if (OB_FAIL(TG_CREATE_TENANT(lib::TGDefIDs::IO_CALLBACK, tg_id_))) { LOG_WARN("create runner thread failed", K(ret)); - } else if (FALSE_IT(is_inited_ = true)) { } else if (OB_FAIL(TG_SET_RUNNABLE_AND_START(tg_id_, *this))) { LOG_WARN("start runner thread failed", K(ret), K(tg_id_)); + } else { + is_inited_ = true; } if (OB_UNLIKELY(!is_inited_)) { destroy(); @@ -2811,29 +2807,24 @@ void ObIORunner::destroy() void ObIORunner::run1() { int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret), K(is_inited_)); - } else { - lib::set_thread_name("DiskCB"); - LOG_INFO("io callback thread started"); - while (!has_set_stop()) { - ObIORequest *req = nullptr; - if (OB_FAIL(pop(req))) { - if (OB_ENTRY_NOT_EXIST == ret) { - ret = OB_SUCCESS; - } else { - LOG_WARN("pop request failed", K(ret)); - } + lib::set_thread_name("DiskCB"); + LOG_INFO("io callback thread started"); + while (!has_set_stop()) { + ObIORequest *req = nullptr; + if (OB_FAIL(pop(req))) { + if (OB_ENTRY_NOT_EXIST == ret) { + ret = OB_SUCCESS; } else { - RequestHolder holder(req); - if (OB_FAIL(handle(req))) { - LOG_WARN("handle request failed", K(ret), KPC(req)); - } + LOG_WARN("pop request failed", K(ret)); + } + } else { + RequestHolder holder(req); + if (OB_FAIL(handle(req))) { + LOG_WARN("handle request failed", K(ret), KPC(req)); } } - LOG_INFO("io callback thread stopped"); } + LOG_INFO("io callback thread stopped"); } int ObIORunner::push(ObIORequest &req) From c4dea4a3d8d6ca6922d402b2c371688a755a5d60 Mon Sep 17 00:00:00 2001 From: godyangfight Date: Wed, 14 Aug 2024 08:47:49 +0000 Subject: [PATCH 044/249] Fix storage ha reader using 500 tenant memory. --- .../ob_storage_ha_reader.cpp | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/src/storage/high_availability/ob_storage_ha_reader.cpp b/src/storage/high_availability/ob_storage_ha_reader.cpp index 5f8075f0f..d0c7829b6 100644 --- a/src/storage/high_availability/ob_storage_ha_reader.cpp +++ b/src/storage/high_availability/ob_storage_ha_reader.cpp @@ -187,11 +187,13 @@ ObCopyMacroBlockObReader::ObCopyMacroBlockObReader() data_buffer_(), rpc_buffer_(), rpc_buffer_parse_pos_(0), - allocator_("CMBObReader"), + allocator_(), macro_block_mem_context_(), last_send_time_(0), data_size_(0) { + ObMemAttr attr(MTL_ID(), "CMBObReader"); + allocator_.set_attr(attr); } ObCopyMacroBlockObReader::~ObCopyMacroBlockObReader() @@ -624,16 +626,20 @@ ObCopyMacroBlockObProducer::ObCopyMacroBlockObProducer() macro_idx_(0), handle_idx_(0), prefetch_meta_time_(0), - tablet_allocator_("HaTabletHdl"), + tablet_allocator_(), tablet_handle_(), sstable_handle_(), sstable_(nullptr), datum_range_(), - allocator_("CopyMacroBlock"), + allocator_(), second_meta_iterator_(), io_allocator_("CMBP_IOUB", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), meta_row_buf_("CopyMacroMetaRow") { + ObMemAttr attr_tablet_alloc(MTL_ID(), "HaTabletHdl"); + tablet_allocator_.set_attr(attr_tablet_alloc); + ObMemAttr attr_copy_macro_block(MTL_ID(), "CopyMacroBlock"); + allocator_.set_attr(attr_copy_macro_block); } ObCopyMacroBlockObProducer::~ObCopyMacroBlockObProducer() @@ -1172,11 +1178,13 @@ int ObCopyTabletInfoObProducer::get_next_tablet_info(obrpc::ObCopyTabletInfo &ta ObCopySSTableInfoObReader::ObCopySSTableInfoObReader() : is_inited_(false), rpc_reader_(), - allocator_("CSSTObReader"), + allocator_(), is_sstable_iter_end_(true), sstable_index_(0), sstable_count_(0) { + ObMemAttr attr(MTL_ID(), "CSSTObReader"); + allocator_.set_attr(attr); } int ObCopySSTableInfoObReader::init( @@ -1292,11 +1300,13 @@ ObCopySSTableInfoRestoreReader::ObCopySSTableInfoRestoreReader() sstable_index_(0), is_sstable_iter_end_(true), backup_sstable_meta_array_(), - allocator_("CSSTREReader"), + allocator_(), ls_id_(), ls_handle_(), remote_sstable_producer_() { + ObMemAttr attr(MTL_ID(), "CSSTREReader"); + allocator_.set_attr(attr); } int ObCopySSTableInfoRestoreReader::init( @@ -2173,8 +2183,10 @@ int ObCopySSTableInfoObProducer::fake_deleted_tablet_meta_( ObCopySSTableMacroObReader::ObCopySSTableMacroObReader() : is_inited_(false), rpc_reader_(), - allocator_("CSSTMBObReader") + allocator_() { + ObMemAttr attr(MTL_ID(), "CSSTMBObReader"); + allocator_.set_attr(attr); } int ObCopySSTableMacroObReader::init( @@ -2637,9 +2649,11 @@ ObCopySSTableMacroRangeObProducer::ObCopySSTableMacroRangeObProducer() table_handle_(), tablet_handle_(), datum_range_(), - allocator_("CopySSTMacro"), + allocator_(), second_meta_iterator_() { + ObMemAttr attr(MTL_ID(), "CopySSTMacro"); + allocator_.set_attr(attr); } int ObCopySSTableMacroRangeObProducer::init( @@ -2770,8 +2784,10 @@ int ObCopySSTableMacroRangeObProducer::get_next_macro_range_info( ObCopyLSViewInfoObReader::ObCopyLSViewInfoObReader() : is_inited_(false), rpc_reader_(), - allocator_("CPLSVObReader") + allocator_() { + ObMemAttr attr(MTL_ID(), "CPLSVObReader"); + allocator_.set_attr(attr); } int ObCopyLSViewInfoObReader::init( From 995d01af9b61182abaa8d3dcc64b18830d49d00d Mon Sep 17 00:00:00 2001 From: suz-yang Date: Wed, 14 Aug 2024 09:18:38 +0000 Subject: [PATCH 045/249] direct load support specifying any number of columns and column order --- .../ob_table_direct_load_rpc_executor.cpp | 108 ++- .../ob_table_direct_load_rpc_executor.h | 4 +- .../ob_table_direct_load_rpc_struct.cpp | 3 +- .../client/ob_table_direct_load_rpc_struct.h | 4 +- .../ob_table_load_autoinc_nextval.cpp | 2 +- .../ob_table_load_client_service.cpp | 30 +- .../table_load/ob_table_load_client_service.h | 5 +- .../table_load/ob_table_load_client_task.cpp | 663 ++++++++++-------- .../table_load/ob_table_load_client_task.h | 110 +-- .../table_load/ob_table_load_coordinator.cpp | 15 +- .../ob_table_load_coordinator_ctx.cpp | 17 +- .../table_load/ob_table_load_exec_ctx.cpp | 23 +- .../table_load/ob_table_load_exec_ctx.h | 12 +- .../table_load/ob_table_load_instance.cpp | 8 +- .../ob_table_load_mem_compactor.cpp | 4 +- ...ble_load_multiple_heap_table_compactor.cpp | 4 +- .../table_load/ob_table_load_obj_cast.cpp | 59 +- .../ob_table_load_partition_calc.cpp | 10 +- .../table_load/ob_table_load_schema.cpp | 171 +++-- .../table_load/ob_table_load_schema.h | 23 +- .../table_load/ob_table_load_service.cpp | 81 ++- .../table_load/ob_table_load_service.h | 9 +- .../table_load/ob_table_load_store_ctx.cpp | 4 +- .../table_load/ob_table_load_table_ctx.cpp | 5 - .../ob_table_load_trans_bucket_writer.cpp | 61 +- .../ob_table_load_trans_bucket_writer.h | 1 + .../table_load/ob_table_load_trans_store.cpp | 40 +- src/share/table/ob_table_load_row.h | 19 +- .../engine/cmd/ob_load_data_direct_impl.cpp | 63 +- .../engine/cmd/ob_table_direct_insert_ctx.cpp | 69 +- .../engine/cmd/ob_table_direct_insert_ctx.h | 7 - .../data/misc/specify_column_0.csv | 3 + 32 files changed, 982 insertions(+), 655 deletions(-) create mode 100644 tools/deploy/mysql_test/test_suite/direct_load_data/data/misc/specify_column_0.csv diff --git a/src/observer/table_load/client/ob_table_direct_load_rpc_executor.cpp b/src/observer/table_load/client/ob_table_direct_load_rpc_executor.cpp index a9d17cd3e..7562d3a28 100644 --- a/src/observer/table_load/client/ob_table_direct_load_rpc_executor.cpp +++ b/src/observer/table_load/client/ob_table_direct_load_rpc_executor.cpp @@ -17,10 +17,7 @@ #include "observer/omt/ob_multi_tenant.h" #include "observer/table_load/ob_table_load_client_service.h" #include "observer/table_load/ob_table_load_client_task.h" -#include "observer/table_load/ob_table_load_exec_ctx.h" -#include "observer/table_load/ob_table_load_schema.h" #include "observer/table_load/ob_table_load_service.h" -#include "sql/resolver/dml/ob_hint.h" namespace oceanbase { @@ -37,7 +34,7 @@ using namespace table; int ObTableDirectLoadBeginExecutor::check_args() { int ret = OB_SUCCESS; - if (OB_UNLIKELY(arg_.table_name_.empty() || arg_.parallel_ <= 0 || + if (OB_UNLIKELY(arg_.table_name_.empty() || arg_.parallel_ <= 0 || arg_.max_error_row_count_ < 0 || arg_.dup_action_ == ObLoadDupActionType::LOAD_INVALID_MODE || arg_.timeout_ <= 0)) { ret = OB_INVALID_ARGUMENT; @@ -64,8 +61,8 @@ int ObTableDirectLoadBeginExecutor::process() ObTableLoadClientTask *client_task = nullptr; if (OB_FAIL(ObTableLoadService::check_tenant())) { LOG_WARN("fail to check tenant", KR(ret)); - } else if (OB_FAIL(resolve_param(param))) { - LOG_WARN("fail to resolve param", KR(ret)); + } else if (OB_FAIL(init_param(param))) { + LOG_WARN("fail to init param", KR(ret)); } else if (OB_FAIL(ObTableLoadClientService::alloc_task(client_task))) { LOG_WARN("fail to alloc client task", KR(ret)); } else if (OB_FAIL(client_task->init(param))) { @@ -80,6 +77,7 @@ int ObTableDirectLoadBeginExecutor::process() ObTableLoadClientStatus client_status = ObTableLoadClientStatus::MAX_STATUS; int client_error_code = OB_SUCCESS; while (OB_SUCC(ret) && ObTableLoadClientStatus::RUNNING != client_status) { + client_task->heart_beat(); // 保持心跳 if (OB_UNLIKELY(THIS_WORKER.is_timeout())) { ret = OB_TIMEOUT; LOG_WARN("worker timeout", KR(ret)); @@ -94,7 +92,8 @@ int ObTableDirectLoadBeginExecutor::process() break; case ObTableLoadClientStatus::ERROR: case ObTableLoadClientStatus::ABORT: - ret = OB_SUCCESS == client_error_code ? OB_CANCELED : client_error_code; + ret = client_error_code; + LOG_WARN("client status error", KR(ret), K(client_status), K(client_error_code)); break; default: ret = OB_ERR_UNEXPECTED; @@ -107,8 +106,8 @@ int ObTableDirectLoadBeginExecutor::process() // fill res if (OB_SUCC(ret)) { - res_.table_id_ = client_task->param_.get_table_id(); - res_.task_id_ = client_task->task_id_; + res_.table_id_ = client_task->get_table_id(); + res_.task_id_ = client_task->get_task_id(); client_task->get_status(res_.status_, res_.error_code_); } if (nullptr != client_task) { @@ -118,64 +117,23 @@ int ObTableDirectLoadBeginExecutor::process() return ret; } -int ObTableDirectLoadBeginExecutor::resolve_param(ObTableLoadClientTaskParam ¶m) +int ObTableDirectLoadBeginExecutor::init_param(ObTableLoadClientTaskParam ¶m) { int ret = OB_SUCCESS; - const uint64_t tenant_id = ctx_.get_tenant_id(); - const uint64_t database_id = ctx_.get_database_id(); - uint64_t table_id = 0; - ObLoadDupActionType dup_action = arg_.dup_action_; - ObDirectLoadMethod::Type method = ObDirectLoadMethod::INVALID_METHOD; - ObDirectLoadInsertMode::Type insert_mode = ObDirectLoadInsertMode::INVALID_INSERT_MODE; param.reset(); - if (OB_FAIL(ObTableLoadSchema::get_table_id(tenant_id, database_id, arg_.table_name_, - table_id))) { - LOG_WARN("fail to get table id", KR(ret), K(tenant_id), K(database_id), K_(arg)); - } else if (arg_.load_method_.empty()) { - method = ObDirectLoadMethod::FULL; - insert_mode = ObDirectLoadInsertMode::NORMAL; - } else { - ObDirectLoadHint::LoadMethod load_method = ObDirectLoadHint::get_load_method_value(arg_.load_method_); - switch (load_method) { - case ObDirectLoadHint::FULL: - method = ObDirectLoadMethod::FULL; - insert_mode = ObDirectLoadInsertMode::NORMAL; - break; - case ObDirectLoadHint::INC: - method = ObDirectLoadMethod::INCREMENTAL; - insert_mode = ObDirectLoadInsertMode::NORMAL; - break; - case ObDirectLoadHint::INC_REPLACE: - if (OB_UNLIKELY(ObLoadDupActionType::LOAD_STOP_ON_DUP != dup_action)) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("replace or ignore for inc_replace load method not supported", KR(ret), - K(arg_.load_method_), K(arg_.dup_action_)); - } else { - dup_action = ObLoadDupActionType::LOAD_REPLACE; //rewrite dup action - method = ObDirectLoadMethod::INCREMENTAL; - insert_mode = ObDirectLoadInsertMode::INC_REPLACE; - } - break; - default: - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid load method", KR(ret), K(arg_.load_method_)); - break; - } - } - if (OB_SUCC(ret)) { - param.set_client_addr(ctx_.get_user_client_addr()); - param.set_tenant_id(tenant_id); - param.set_user_id(ctx_.get_user_id()); - param.set_database_id(database_id); - param.set_table_id(table_id); - param.set_parallel(arg_.parallel_); - param.set_max_error_row_count(arg_.max_error_row_count_); - param.set_dup_action(dup_action); - param.set_timeout_us(arg_.timeout_); - param.set_heartbeat_timeout_us(arg_.heartbeat_timeout_); - param.set_method(method); - param.set_insert_mode(insert_mode); - } + OX(param.set_task_id(ObTableLoadClientService::generate_task_id())); + OX(param.set_client_addr(ctx_.get_user_client_addr())); + OX(param.set_tenant_id(ctx_.get_tenant_id())); + OX(param.set_user_id(ctx_.get_user_id())); + OX(param.set_database_id(ctx_.get_database_id())); + OZ(param.set_table_name(arg_.table_name_)); + OX(param.set_parallel(arg_.parallel_)); + OX(param.set_max_error_row_count(arg_.max_error_row_count_)); + OX(param.set_dup_action(arg_.dup_action_)); + OX(param.set_timeout_us(arg_.timeout_)); + OX(param.set_heartbeat_timeout_us(arg_.heartbeat_timeout_)); + OZ(param.set_load_method(arg_.load_method_)); + OZ(param.set_column_names(arg_.column_names_)); return ret; } @@ -196,9 +154,23 @@ int ObTableDirectLoadCommitExecutor::process() int ret = OB_SUCCESS; LOG_INFO("table direct load commit", K_(arg)); ObTableLoadClientTask *client_task = nullptr; + ObTableLoadClientTaskBrief *client_task_brief = nullptr; ObTableLoadUniqueKey key(arg_.table_id_, arg_.task_id_); if (OB_FAIL(ObTableLoadClientService::get_task(key, client_task))) { - LOG_WARN("fail to get client task", KR(ret), K(key)); + if (OB_UNLIKELY(OB_ENTRY_NOT_EXIST != ret)) { + LOG_WARN("fail to get client task", KR(ret), K(key)); + } else { + // 处理重试场景 + ret = OB_SUCCESS; + if (OB_FAIL(ObTableLoadClientService::get_task_brief(key, client_task_brief))) { + LOG_WARN("fail to get client task brief", KR(ret), K(key)); + } else if (ObTableLoadClientStatus::COMMIT == client_task_brief->client_status_) { + LOG_INFO("client task is commit", KR(ret)); + } else { + ret = client_task_brief->error_code_; + LOG_WARN("client task is failed", KR(ret), KPC(client_task_brief)); + } + } } else if (OB_FAIL(client_task->commit())) { LOG_WARN("fail to commit client task", KR(ret), K(key)); } @@ -264,9 +236,7 @@ int ObTableDirectLoadGetStatusExecutor::process() } else { ret = OB_SUCCESS; if (OB_FAIL(ObTableLoadClientService::get_task_brief(key, client_task_brief))) { - if (OB_UNLIKELY(OB_ENTRY_NOT_EXIST != ret)) { - LOG_WARN("fail to get client task brief", KR(ret), K(key)); - } + LOG_WARN("fail to get client task brief", KR(ret), K(key)); } else { res_.status_ = client_task_brief->client_status_; res_.error_code_ = client_task_brief->error_code_; @@ -374,7 +344,7 @@ int ObTableDirectLoadHeartBeatExecutor::process() if (OB_FAIL(ObTableLoadClientService::get_task(key, client_task))) { LOG_WARN("fail to get client task", KR(ret), K(key)); } else { - client_task->get_exec_ctx()->last_heartbeat_time_ = ObTimeUtil::current_time(); + client_task->heart_beat(); client_task->get_status(res_.status_, res_.error_code_); } if (nullptr != client_task) { diff --git a/src/observer/table_load/client/ob_table_direct_load_rpc_executor.h b/src/observer/table_load/client/ob_table_direct_load_rpc_executor.h index a16f1d226..c8c5b7f22 100644 --- a/src/observer/table_load/client/ob_table_direct_load_rpc_executor.h +++ b/src/observer/table_load/client/ob_table_direct_load_rpc_executor.h @@ -21,8 +21,6 @@ namespace oceanbase namespace observer { class ObTableLoadClientTaskParam; -class ObTableLoadClientTask; -class ObTableLoadTableCtx; template class ObTableDirectLoadRpcExecutor @@ -73,7 +71,7 @@ protected: int process() override; private: - int resolve_param(ObTableLoadClientTaskParam ¶m); + int init_param(ObTableLoadClientTaskParam ¶m); }; // commit diff --git a/src/observer/table_load/client/ob_table_direct_load_rpc_struct.cpp b/src/observer/table_load/client/ob_table_direct_load_rpc_struct.cpp index b49179f48..16cd609b1 100644 --- a/src/observer/table_load/client/ob_table_direct_load_rpc_struct.cpp +++ b/src/observer/table_load/client/ob_table_direct_load_rpc_struct.cpp @@ -29,7 +29,8 @@ OB_SERIALIZE_MEMBER_SIMPLE(ObTableDirectLoadBeginArg, heartbeat_timeout_, force_create_, is_async_, - load_method_); + load_method_, + column_names_); OB_SERIALIZE_MEMBER_SIMPLE(ObTableDirectLoadBeginRes, table_id_, diff --git a/src/observer/table_load/client/ob_table_direct_load_rpc_struct.h b/src/observer/table_load/client/ob_table_direct_load_rpc_struct.h index fb232ce76..9ac4539e5 100644 --- a/src/observer/table_load/client/ob_table_direct_load_rpc_struct.h +++ b/src/observer/table_load/client/ob_table_direct_load_rpc_struct.h @@ -44,7 +44,8 @@ public: K_(heartbeat_timeout), K_(force_create), K_(is_async), - K_(load_method)); + K_(load_method), + K_(column_names)); public: ObString table_name_; int64_t parallel_; @@ -55,6 +56,7 @@ public: bool force_create_; // unused bool is_async_; ObString load_method_; + common::ObSArray column_names_; }; struct ObTableDirectLoadBeginRes diff --git a/src/observer/table_load/ob_table_load_autoinc_nextval.cpp b/src/observer/table_load/ob_table_load_autoinc_nextval.cpp index 2d5754dd7..ef5052bab 100644 --- a/src/observer/table_load/ob_table_load_autoinc_nextval.cpp +++ b/src/observer/table_load/ob_table_load_autoinc_nextval.cpp @@ -78,7 +78,7 @@ int ObTableLoadAutoincNextval::get_input_value(ObStorageDatum &datum, const uint64_t &sql_mode) { int ret = OB_SUCCESS; - if (datum.is_null()) { + if (datum.is_nop() || datum.is_null()) { is_to_generate = true; } else { bool is_zero = false; diff --git a/src/observer/table_load/ob_table_load_client_service.cpp b/src/observer/table_load/ob_table_load_client_service.cpp index 4d2f09dcf..5f06f27fd 100644 --- a/src/observer/table_load/ob_table_load_client_service.cpp +++ b/src/observer/table_load/ob_table_load_client_service.cpp @@ -40,7 +40,14 @@ bool ObTableLoadClientService::ClientTaskBriefEraseIfExpired::operator()( * ObTableLoadClientService */ -ObTableLoadClientService::ObTableLoadClientService() : next_task_id_(1), is_inited_(false) {} +int64_t ObTableLoadClientService::next_task_sequence_ = 0; + +int64_t ObTableLoadClientService::generate_task_id() +{ + return ObTimeUtil::current_time() * 1000 + ATOMIC_FAA(&next_task_sequence_, 1) % 1000; +} + +ObTableLoadClientService::ObTableLoadClientService() : is_inited_(false) {} ObTableLoadClientService::~ObTableLoadClientService() {} @@ -92,7 +99,6 @@ int ObTableLoadClientService::alloc_task(ObTableLoadClientTask *&client_task) ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to new ObTableLoadClientTask", KR(ret)); } else { - new_client_task->task_id_ = service->get_client_service().generate_task_id(); client_task = new_client_task; client_task->inc_ref_count(); } @@ -125,9 +131,7 @@ void ObTableLoadClientService::revert_task(ObTableLoadClientTask *client_task) const int64_t ref_count = client_task->dec_ref_count(); OB_ASSERT(ref_count >= 0); if (0 == ref_count) { - const int64_t task_id = client_task->task_id_; - const uint64_t table_id = client_task->param_.get_table_id(); - LOG_INFO("free client task", K(task_id), K(table_id), KP(client_task)); + LOG_INFO("free client task", KPC(client_task)); free_task(client_task); client_task = nullptr; } @@ -141,8 +145,11 @@ int ObTableLoadClientService::add_task(ObTableLoadClientTask *client_task) if (OB_ISNULL(service = MTL(ObTableLoadService *))) { ret = OB_ERR_SYS; LOG_WARN("null table load service", KR(ret)); + } else if (OB_ISNULL(client_task)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KP(client_task)); } else { - ObTableLoadUniqueKey key(client_task->param_.get_table_id(), client_task->task_id_); + ObTableLoadUniqueKey key(client_task->get_table_id(), client_task->get_task_id()); ret = service->get_client_service().add_client_task(key, client_task); } return ret; @@ -155,8 +162,11 @@ int ObTableLoadClientService::remove_task(ObTableLoadClientTask *client_task) if (OB_ISNULL(service = MTL(ObTableLoadService *))) { ret = OB_ERR_SYS; LOG_WARN("null table load service", KR(ret)); + } else if (OB_ISNULL(client_task)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KP(client_task)); } else { - ObTableLoadUniqueKey key(client_task->param_.get_table_id(), client_task->task_id_); + ObTableLoadUniqueKey key(client_task->get_table_id(), client_task->get_task_id()); ret = service->get_client_service().remove_client_task(key, client_task); } return ret; @@ -236,10 +246,10 @@ int ObTableLoadClientService::remove_client_task(const ObTableLoadUniqueKey &key if (OB_FAIL(client_task_brief_map_.create(key, client_task_brief))) { LOG_WARN("fail to create client task brief", KR(ret), K(key)); } else { - client_task_brief->task_id_ = client_task->task_id_; - client_task_brief->table_id_ = client_task->param_.get_table_id(); + client_task_brief->task_id_ = client_task->get_task_id(); + client_task_brief->table_id_ = client_task->get_table_id(); client_task->get_status(client_task_brief->client_status_, client_task_brief->error_code_); - client_task_brief->result_info_ = client_task->result_info_; + client_task_brief->result_info_ = client_task->get_result_info(); client_task_brief->active_time_ = ObTimeUtil::current_time(); } if (nullptr != client_task_brief) { diff --git a/src/observer/table_load/ob_table_load_client_service.h b/src/observer/table_load/ob_table_load_client_service.h index 65baf99a1..b26a3af6a 100644 --- a/src/observer/table_load/ob_table_load_client_service.h +++ b/src/observer/table_load/ob_table_load_client_service.h @@ -70,11 +70,11 @@ public: return ObTableDirectLoadRpcProxy::dispatch(ctx, request, result); } -private: - OB_INLINE int64_t generate_task_id() { return ATOMIC_FAA(&next_task_id_, 1); } + static int64_t generate_task_id(); private: static const int64_t CLIENT_TASK_RETENTION_PERIOD = 24LL * 60 * 60 * 1000 * 1000; // 1day + static int64_t next_task_sequence_; // key => client_task typedef common::hash::ObHashMap @@ -116,7 +116,6 @@ private: mutable obsys::ObRWLock rwlock_; ClientTaskMap client_task_map_; ClientTaskBriefMap client_task_brief_map_; // thread safety - int64_t next_task_id_; bool is_inited_; }; diff --git a/src/observer/table_load/ob_table_load_client_task.cpp b/src/observer/table_load/ob_table_load_client_task.cpp index acbff1841..b64c80bf7 100644 --- a/src/observer/table_load/ob_table_load_client_task.cpp +++ b/src/observer/table_load/ob_table_load_client_task.cpp @@ -14,7 +14,6 @@ #include "observer/table_load/ob_table_load_client_task.h" #include "observer/ob_server.h" -#include "observer/omt/ob_tenant.h" #include "observer/table_load/ob_table_load_exec_ctx.h" #include "observer/table_load/ob_table_load_schema.h" #include "observer/table_load/ob_table_load_service.h" @@ -39,18 +38,22 @@ using namespace table; */ ObTableLoadClientTaskParam::ObTableLoadClientTaskParam() - : tenant_id_(OB_INVALID_TENANT_ID), + : allocator_("TLD_CTask"), + task_id_(0), + tenant_id_(OB_INVALID_TENANT_ID), user_id_(OB_INVALID_ID), database_id_(OB_INVALID_ID), - table_id_(OB_INVALID_ID), + table_name_(), parallel_(0), max_error_row_count_(0), dup_action_(ObLoadDupActionType::LOAD_INVALID_MODE), timeout_us_(0), heartbeat_timeout_us_(0), - method_(ObDirectLoadMethod::INVALID_METHOD), - insert_mode_(ObDirectLoadInsertMode::INVALID_INSERT_MODE) + load_method_(), + column_names_() { + allocator_.set_tenant_id(MTL_ID()); + column_names_.set_block_allocator(ModulePageAllocator(allocator_)); } ObTableLoadClientTaskParam::~ObTableLoadClientTaskParam() {} @@ -58,17 +61,19 @@ ObTableLoadClientTaskParam::~ObTableLoadClientTaskParam() {} void ObTableLoadClientTaskParam::reset() { client_addr_.reset(); + task_id_ = 0; tenant_id_ = OB_INVALID_TENANT_ID; user_id_ = OB_INVALID_ID; database_id_ = OB_INVALID_ID; - table_id_ = OB_INVALID_ID; + table_name_.reset(); parallel_ = 0; max_error_row_count_ = 0; dup_action_ = ObLoadDupActionType::LOAD_INVALID_MODE; timeout_us_ = 0; heartbeat_timeout_us_ = 0; - method_ = ObDirectLoadMethod::INVALID_METHOD; - insert_mode_ = ObDirectLoadInsertMode::INVALID_INSERT_MODE; + load_method_.reset(); + column_names_.reset(); + allocator_.reset(); } int ObTableLoadClientTaskParam::assign(const ObTableLoadClientTaskParam &other) @@ -77,39 +82,59 @@ int ObTableLoadClientTaskParam::assign(const ObTableLoadClientTaskParam &other) if (this != &other) { reset(); client_addr_ = other.client_addr_; + task_id_ = other.task_id_; tenant_id_ = other.tenant_id_; user_id_ = other.user_id_; database_id_ = other.database_id_; - table_id_ = other.table_id_; parallel_ = other.parallel_; max_error_row_count_ = other.max_error_row_count_; dup_action_ = other.dup_action_; timeout_us_ = other.timeout_us_; heartbeat_timeout_us_ = other.heartbeat_timeout_us_; - method_ = other.method_; - insert_mode_ = other.insert_mode_; + if (OB_FAIL(set_table_name(other.table_name_))) { + LOG_WARN("fail to set table name", KR(ret)); + } else if (OB_FAIL(set_load_method(other.load_method_))) { + LOG_WARN("fail to set load method", KR(ret)); + } else if (OB_FAIL(set_column_names(other.column_names_))) { + LOG_WARN("fail to deep copy column names", KR(ret)); + } } return ret; } bool ObTableLoadClientTaskParam::is_valid() const { - return client_addr_.is_valid() && OB_INVALID_TENANT_ID != tenant_id_ && - OB_INVALID_ID != user_id_ && OB_INVALID_ID != database_id_ && OB_INVALID_ID != table_id_ && - parallel_ > 0 && ObLoadDupActionType::LOAD_INVALID_MODE != dup_action_ && - timeout_us_ > 0 && heartbeat_timeout_us_ > 0 && - ObDirectLoadMethod::is_type_valid(method_) && - ObDirectLoadInsertMode::is_type_valid(insert_mode_) && - (storage::ObDirectLoadMethod::is_full(method_) - ? storage::ObDirectLoadInsertMode::is_valid_for_full_method(insert_mode_) - : true) && - (storage::ObDirectLoadMethod::is_incremental(method_) - ? storage::ObDirectLoadInsertMode::is_valid_for_incremental_method(insert_mode_) - : true) && - (storage::ObDirectLoadInsertMode::INC_REPLACE == insert_mode_ - ? sql::ObLoadDupActionType::LOAD_REPLACE == dup_action_ - : true); + return client_addr_.is_valid() && task_id_ > 0 && OB_INVALID_TENANT_ID != tenant_id_ && + OB_INVALID_ID != user_id_ && OB_INVALID_ID != database_id_ && !table_name_.empty() && + parallel_ > 0 && max_error_row_count_ >= 0 && + ObLoadDupActionType::LOAD_INVALID_MODE != dup_action_ && timeout_us_ > 0 && + heartbeat_timeout_us_ > 0; +} +int ObTableLoadClientTaskParam::set_string(const ObString &src, ObString &dest) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ob_write_string(allocator_, src, dest))) { + LOG_WARN("fail to write string", KR(ret)); + } + return ret; +} + +int ObTableLoadClientTaskParam::set_string_array(const ObIArray &src, + ObIArray &dest) +{ + int ret = OB_SUCCESS; + dest.reset(); + for (int64_t i = 0; OB_SUCC(ret) && i < src.count(); ++i) { + const ObString &src_str = src.at(i); + ObString dest_str; + if (OB_FAIL(ob_write_string(allocator_, src_str, dest_str))) { + LOG_WARN("fail to write string", KR(ret)); + } else if (OB_FAIL(dest.push_back(dest_str))) { + LOG_WARN("fail to push back", KR(ret)); + } + } + return ret; } /** @@ -128,22 +153,40 @@ public: int process() override { int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; ObSQLSessionInfo *origin_session_info = THIS_WORKER.get_session(); - ObSQLSessionInfo *session_info = client_task_->get_session_info(); + ObSQLSessionInfo *session_info = client_task_->session_info_; + ObSchemaGetterGuard &schema_guard = client_task_->schema_guard_; + ObTableLoadParam load_param; + ObArray column_ids; THIS_WORKER.set_session(session_info); if (OB_ISNULL(session_info)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null session info", KR(ret)); } else { - session_info->set_thread_id(GETTID()); + session_info->set_thread_id(get_tid_cache()); session_info->update_last_active_time(); - if (OB_FAIL(session_info->set_session_state(QUERY_ACTIVE))) { - LOG_WARN("fail to set session state", K(ret)); + if (OB_TMP_FAIL(session_info->set_session_state(QUERY_ACTIVE))) { + LOG_WARN("fail to set session state", KR(tmp_ret)); } } + // resolve + if (OB_FAIL( + resolve(client_task_->client_exec_ctx_, client_task_->param_, load_param, column_ids))) { + LOG_WARN("fail to resolve", KR(ret), K(client_task_->param_)); + } + // check support + else if (OB_FAIL(ObTableLoadService::check_support_direct_load(schema_guard, + load_param.table_id_, + load_param.method_, + load_param.insert_mode_, + load_param.load_mode_, + column_ids))) { + LOG_WARN("fail to check support direct load", KR(ret)); + } // begin if (OB_SUCC(ret)) { - if (OB_FAIL(client_task_->init_instance())) { + if (OB_FAIL(client_task_->init_instance(load_param, column_ids))) { LOG_WARN("fail to init instance", KR(ret)); } else if (OB_FAIL(client_task_->set_status_waitting())) { LOG_WARN("fail to set status waitting", KR(ret)); @@ -173,6 +216,168 @@ public: } client_task_->destroy_instance(); THIS_WORKER.set_session(origin_session_info); + if (session_info != nullptr && OB_TMP_FAIL(session_info->set_session_state(SESSION_SLEEP))) { + LOG_WARN("fail to set session state", KR(tmp_ret)); + } + return ret; + } + + static int resolve(ObTableLoadClientExecCtx &client_exec_ctx, + const ObTableLoadClientTaskParam &task_param, ObTableLoadParam &load_param, + ObIArray &column_ids) + { + int ret = OB_SUCCESS; + const uint64_t tenant_id = task_param.get_tenant_id(); + const uint64_t database_id = task_param.get_database_id(); + ObSchemaGetterGuard *schema_guard = nullptr; + const ObTableSchema *table_schema = nullptr; + ObDirectLoadMethod::Type method = ObDirectLoadMethod::INVALID_METHOD; + ObDirectLoadInsertMode::Type insert_mode = ObDirectLoadInsertMode::INVALID_INSERT_MODE; + ObCompressorType compressor_type = INVALID_COMPRESSOR; + bool online_opt_stat_gather = false; + double online_sample_percent = 100.; + if (OB_ISNULL(schema_guard = client_exec_ctx.get_schema_guard())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected schema guard is null", KR(ret)); + } + // resolve table_name_ + else if (OB_FAIL(ObTableLoadSchema::get_table_schema( + *schema_guard, tenant_id, database_id, task_param.get_table_name(), table_schema))) { + LOG_WARN("fail to get table schema", KR(ret), K(tenant_id), K(database_id), + K(task_param.get_table_name())); + } + // resolve column_names_ + else if (OB_FAIL(resolve_columns(table_schema, task_param.get_column_names(), column_ids))) { + LOG_WARN("fail to resolve columns", KR(ret), K(task_param.get_column_names())); + } + // resolve load_method_ + else if (OB_FAIL(resolve_load_method(task_param.get_load_method(), method, insert_mode))) { + LOG_WARN("fail to resolve load method", KR(ret), K(task_param.get_load_method())); + } + // compress type + else if (OB_FAIL(ObDDLUtil::get_temp_store_compress_type( + table_schema->get_compressor_type(), task_param.get_parallel(), compressor_type))) { + LOG_WARN("fail to get tmp store compressor type", KR(ret)); + } + // opt stat gather + else if (OB_FAIL(ObTableLoadSchema::get_tenant_optimizer_gather_stats_on_load( + tenant_id, online_opt_stat_gather))) { + LOG_WARN("fail to get tenant optimizer gather stats on load", KR(ret), K(tenant_id)); + } else if (online_opt_stat_gather && OB_FAIL(ObDbmsStatsUtils::get_sys_online_estimate_percent( + *(client_exec_ctx.exec_ctx_), tenant_id, + table_schema->get_table_id(), online_sample_percent))) { + LOG_WARN("failed to get sys online sample percent", K(ret)); + } + if (OB_SUCC(ret)) { + load_param.tenant_id_ = tenant_id; + load_param.table_id_ = table_schema->get_table_id(); + load_param.parallel_ = task_param.get_parallel(); + load_param.session_count_ = task_param.get_parallel(); + load_param.batch_size_ = 100; + load_param.max_error_row_count_ = task_param.get_max_error_row_count(); + load_param.column_count_ = column_ids.count(); + load_param.need_sort_ = true; + load_param.px_mode_ = false; + load_param.online_opt_stat_gather_ = online_opt_stat_gather; + load_param.dup_action_ = + // rewrite dup action to replace in inc_replace + (method == ObDirectLoadMethod::INCREMENTAL && + insert_mode == ObDirectLoadInsertMode::INC_REPLACE) + ? ObLoadDupActionType::LOAD_REPLACE + : task_param.get_dup_action(); + load_param.method_ = method; + load_param.insert_mode_ = insert_mode; + load_param.load_mode_ = ObDirectLoadMode::TABLE_LOAD; + load_param.compressor_type_ = compressor_type; + load_param.online_sample_percent_ = online_sample_percent; + } + return ret; + } + + static int resolve_columns(const ObTableSchema *table_schema, + const ObIArray &column_names, ObIArray &column_ids) + { + int ret = OB_SUCCESS; + column_ids.reset(); + if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected table schema is null", KR(ret), KP(table_schema)); + } else if (column_names.empty()) { + if (OB_FAIL(ObTableLoadSchema::get_user_column_ids(table_schema, column_ids))) { + LOG_WARN("fail to get user column ids", KR(ret)); + } + } else { + const static uint64_t INVALID_COLUMN_ID = UINT64_MAX; + ObArray user_column_ids; + ObArray user_column_names; + if (OB_FAIL(ObTableLoadSchema::get_user_column_id_and_names(table_schema, + user_column_ids, + user_column_names))) { + LOG_WARN("fail to get user column id and names", KR(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < column_names.count(); ++i) { + const ObString &column_name = column_names.at(i); + if (OB_UNLIKELY(column_name.empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("empty column name is invalid", KR(ret), K(i), K(column_names)); + } else { + int64_t found_column_idx = -1; + for (int64_t j = 0; found_column_idx == -1 && j < user_column_names.count(); ++j) { + const ObString &user_column_name = user_column_names.at(j); + if (column_name.length() != user_column_name.length()) { + } else if (column_name.case_compare(user_column_name) == 0) { + found_column_idx = j; + } + } + if (OB_UNLIKELY(found_column_idx == -1)) { + ret = OB_ERR_BAD_FIELD_ERROR; + LOG_WARN("unknow column", KR(ret), K(column_name), K(user_column_names)); + } else { + const uint64_t user_column_id = user_column_ids.at(found_column_idx); + if (OB_UNLIKELY(user_column_id == INVALID_COLUMN_ID)) { + ret = OB_ERR_FIELD_SPECIFIED_TWICE; + LOG_WARN("column specified twice", KR(ret), K(i), K(column_name), K(column_names)); + } else if (OB_FAIL(column_ids.push_back(user_column_id))) { + LOG_WARN("fail to push back column id", KR(ret)); + } else { + user_column_ids.at(found_column_idx) = INVALID_COLUMN_ID; + } + } + } + } + } + return ret; + } + + static int resolve_load_method(const ObString &load_method_str, ObDirectLoadMethod::Type &method, + ObDirectLoadInsertMode::Type &insert_mode) + { + int ret = OB_SUCCESS; + if (load_method_str.empty()) { + method = ObDirectLoadMethod::FULL; + insert_mode = ObDirectLoadInsertMode::NORMAL; + } else { + const ObDirectLoadHint::LoadMethod load_method = + ObDirectLoadHint::get_load_method_value(load_method_str); + switch (load_method) { + case ObDirectLoadHint::FULL: + method = ObDirectLoadMethod::FULL; + insert_mode = ObDirectLoadInsertMode::NORMAL; + break; + case ObDirectLoadHint::INC: + method = ObDirectLoadMethod::INCREMENTAL; + insert_mode = ObDirectLoadInsertMode::NORMAL; + break; + case ObDirectLoadHint::INC_REPLACE: + method = ObDirectLoadMethod::INCREMENTAL; + insert_mode = ObDirectLoadInsertMode::INC_REPLACE; + break; + default: + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid load method", KR(ret), K(load_method_str)); + break; + } + } return ret; } @@ -195,7 +400,7 @@ public: void callback(int ret_code, ObTableLoadTask *task) override { if (OB_UNLIKELY(OB_SUCCESS != ret_code)) { - client_task_->set_status_abort(ret_code); + client_task_->set_status_error(ret_code); } task->~ObTableLoadTask(); } @@ -209,12 +414,11 @@ private: */ ObTableLoadClientTask::ObTableLoadClientTask() - : task_id_(OB_INVALID_ID), - allocator_("TLD_ClientTask"), - task_scheduler_(nullptr), + : allocator_("TLD_ClientTask"), session_info_(nullptr), plan_ctx_(allocator_), exec_ctx_(allocator_), + task_scheduler_(nullptr), session_count_(0), next_batch_id_(0), client_status_(ObTableLoadClientStatus::MAX_STATUS), @@ -251,41 +455,33 @@ int ObTableLoadClientTask::init(const ObTableLoadClientTaskParam ¶m) } else if (OB_UNLIKELY(!param.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", KR(ret), K(param)); - } else if (OB_FAIL(param_.assign(param))) { - LOG_WARN("fail to assign param", KR(ret)); } else { - const int64_t origin_timeout_ts = THIS_WORKER.get_timeout_ts(); - THIS_WORKER.set_timeout_ts(ObTimeUtil::current_time() + param_.get_timeout_us()); - if (OB_FAIL(init_exec_ctx())) { - LOG_WARN("fail to init client exec ctx", KR(ret)); - } else if (OB_ISNULL(task_scheduler_ = - OB_NEWx(ObTableLoadTaskThreadPoolScheduler, (&allocator_), 1, - param_.get_table_id(), "Client"))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to new ObTableLoadTaskThreadPoolScheduler", KR(ret)); - } else if (OB_FAIL(task_scheduler_->init())) { + if (OB_FAIL(param_.assign(param))) { + LOG_WARN("fail to assign param", KR(ret), K(param)); + } else if (OB_FAIL(init_exec_ctx())) { + LOG_WARN("fail to init exec ctx", KR(ret)); + } else if (OB_FAIL(init_task_scheduler())) { LOG_WARN("fail to init task scheduler", KR(ret)); - } else if (OB_FAIL(task_scheduler_->start())) { - LOG_WARN("fail to start task scheduler", KR(ret)); } else { is_inited_ = true; } - THIS_WORKER.set_timeout_ts(origin_timeout_ts); } return ret; } -int ObTableLoadClientTask::create_session_info(uint64_t tenant_id, uint64_t user_id, - uint64_t database_id, uint64_t table_id, - ObSQLSessionInfo *&session_info, - ObFreeSessionCtx &free_session_ctx) +int ObTableLoadClientTask::create_session_info() { int ret = OB_SUCCESS; - const schema::ObTenantSchema *tenant_info = nullptr; + const uint64_t tenant_id = param_.get_tenant_id(); + const uint64_t user_id = param_.get_user_id(); + const uint64_t database_id = param_.get_database_id(); + const ObTenantSchema *tenant_info = nullptr; const ObUserInfo *user_info = nullptr; const ObDatabaseSchema *database_schema = nullptr; - const ObTableSchema *table_schema = nullptr; - if (OB_FAIL(schema_guard_.get_tenant_info(tenant_id, tenant_info))) { + if (OB_NOT_NULL(session_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected session info not null", KR(ret)); + } else if (OB_FAIL(schema_guard_.get_tenant_info(tenant_id, tenant_info))) { LOG_WARN("get tenant info failed", K(ret)); } else if (OB_ISNULL(tenant_info)) { ret = OB_ERR_UNEXPECTED; @@ -300,43 +496,31 @@ int ObTableLoadClientTask::create_session_info(uint64_t tenant_id, uint64_t user } else if (OB_ISNULL(database_schema)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid database schema", K(ret), K(tenant_id), K(database_id)); - } else if (OB_FAIL(schema_guard_.get_table_schema(tenant_id, table_id, table_schema))) { - LOG_WARN("fail to get table schema", KR(ret)); - } else if (OB_ISNULL(table_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid table schema", K(ret), K(tenant_id), K(table_id)); - } else if (OB_FAIL(ObTableLoadUtils::create_session_info(session_info, free_session_ctx))) { + } else if (OB_FAIL(ObTableLoadUtils::create_session_info(session_info_, free_session_ctx_))) { LOG_WARN("create session id failed", KR(ret)); } else { - ObArenaAllocator allocator("TLD_Tmp"); - allocator.set_tenant_id(MTL_ID()); - ObStringBuffer buffer(&allocator); - buffer.append("DIRECT LOAD: "); - buffer.append(table_schema->get_table_name()); + ObSqlString query_str; ObObj timeout_val; timeout_val.set_int(param_.get_timeout_us()); - OZ(session_info->load_default_sys_variable(false, false)); //加载默认的session参数 - OZ(session_info->load_default_configs_in_pc()); - OX(session_info->init_tenant(tenant_info->get_tenant_name(), tenant_id)); - OX(session_info->set_priv_user_id(user_id)); - OX(session_info->store_query_string(ObString(buffer.length(), buffer.ptr()))); - OX(session_info->set_user(user_info->get_user_name(), user_info->get_host_name_str(), - user_info->get_user_id())); - OX(session_info->set_user_priv_set(OB_PRIV_ALL | OB_PRIV_GRANT)); - OX(session_info->set_default_database(database_schema->get_database_name(), - CS_TYPE_UTF8MB4_GENERAL_CI)); - OX(session_info->set_mysql_cmd(COM_QUERY)); - OX(session_info->set_current_trace_id(ObCurTraceId::get_trace_id())); - OX(session_info->set_client_addr(param_.get_client_addr())); - OX(session_info->set_peer_addr(ObServer::get_instance().get_self())); - OX(session_info->set_query_start_time(ObTimeUtil::current_time())); - OZ(session_info->update_sys_variable(SYS_VAR_OB_QUERY_TIMEOUT, timeout_val)); - } - if (OB_FAIL(ret)) { - if (session_info != nullptr) { - observer::ObTableLoadUtils::free_session_info(session_info, free_session_ctx); - session_info = nullptr; - } + OX(query_str.assign_fmt("DIRECT LOAD: %.*s, task_id:%ld", + static_cast(param_.get_table_name().length()), + param_.get_table_name().ptr(), param_.get_task_id())); + OZ(session_info_->load_default_sys_variable(false /*print_info_log*/, false /*is_sys_tenant*/)); //加载默认的session参数 + OZ(session_info_->load_default_configs_in_pc()); + OX(session_info_->init_tenant(tenant_info->get_tenant_name(), tenant_id)); + OX(session_info_->set_priv_user_id(user_id)); + OX(session_info_->store_query_string(query_str.string())); + OX(session_info_->set_user(user_info->get_user_name(), user_info->get_host_name_str(), + user_info->get_user_id())); + OX(session_info_->set_user_priv_set(OB_PRIV_ALL | OB_PRIV_GRANT)); + OX(session_info_->set_default_database(database_schema->get_database_name(), + CS_TYPE_UTF8MB4_GENERAL_CI)); + OX(session_info_->set_mysql_cmd(COM_QUERY)); + OX(session_info_->set_current_trace_id(ObCurTraceId::get_trace_id())); + OX(session_info_->set_client_addr(param_.get_client_addr())); + OX(session_info_->set_peer_addr(ObServer::get_instance().get_self())); + OX(session_info_->set_query_start_time(ObTimeUtil::current_time())); + OZ(session_info_->update_sys_variable(SYS_VAR_OB_QUERY_TIMEOUT, timeout_val)); } return ret; } @@ -344,13 +528,9 @@ int ObTableLoadClientTask::create_session_info(uint64_t tenant_id, uint64_t user int ObTableLoadClientTask::init_exec_ctx() { int ret = OB_SUCCESS; - if (OB_ISNULL(GCTX.schema_service_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid argument", K(ret), K(GCTX.schema_service_)); - } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(param_.get_tenant_id(), schema_guard_))) { + if (OB_FAIL(ObTableLoadSchema::get_schema_guard(param_.get_tenant_id(), schema_guard_))) { LOG_WARN("get_schema_guard failed", K(ret)); - } else if (OB_FAIL(create_session_info(param_.get_tenant_id(), param_.get_user_id(), - param_.get_database_id(), param_.get_table_id(), session_info_, free_session_ctx_))) { + } else if (OB_FAIL(create_session_info())) { LOG_WARN("fail to create session info", KR(ret)); } else { sql_ctx_.schema_guard_ = &schema_guard_; @@ -359,44 +539,55 @@ int ObTableLoadClientTask::init_exec_ctx() exec_ctx_.set_physical_plan_ctx(&plan_ctx_); exec_ctx_.set_my_session(session_info_); client_exec_ctx_.exec_ctx_ = &exec_ctx_; - client_exec_ctx_.last_heartbeat_time_ = ObTimeUtil::current_time(); - client_exec_ctx_.heartbeat_timeout_us_ = param_.get_heartbeat_timeout_us(); + client_exec_ctx_.init_heart_beat(param_.get_heartbeat_timeout_us()); } return ret; } +int ObTableLoadClientTask::init_task_scheduler() +{ + int ret = OB_SUCCESS; + const int64_t origin_timeout_ts = THIS_WORKER.get_timeout_ts(); + THIS_WORKER.set_timeout_ts(ObTimeUtil::current_time() + param_.get_timeout_us()); + if (OB_ISNULL(task_scheduler_ = OB_NEWx(ObTableLoadTaskThreadPoolScheduler, (&allocator_), 1, + param_.get_task_id(), "Executor"))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObTableLoadTaskThreadPoolScheduler", KR(ret)); + } else if (OB_FAIL(task_scheduler_->init())) { + LOG_WARN("fail to init task scheduler", KR(ret)); + } else if (OB_FAIL(task_scheduler_->start())) { + LOG_WARN("fail to start task scheduler", KR(ret)); + } + THIS_WORKER.set_timeout_ts(origin_timeout_ts); + return ret; +} + int ObTableLoadClientTask::start() { int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ObTableLoadClientTask not init", KR(ret)); + } else if (OB_FAIL(set_status_initializing())) { + LOG_WARN("fail to set status initializing", KR(ret)); } else { - obsys::ObWLockGuard guard(rw_lock_); - if (OB_UNLIKELY(ObTableLoadClientStatus::MAX_STATUS != client_status_)) { - ret = OB_STATE_NOT_MATCH; - LOG_WARN("unexpected status", KR(ret), K(client_status_)); - } else { - client_status_ = ObTableLoadClientStatus::INITIALIZING; - ObTableLoadTask *task = nullptr; - if (OB_ISNULL(task = OB_NEWx(ObTableLoadTask, &allocator_, param_.get_tenant_id()))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to new ObTableLoadTask", KR(ret)); - } else if (OB_FAIL(task->set_processor(this))) { - LOG_WARN("fail to set client task processor", KR(ret)); - } else if (OB_FAIL(task->set_callback(this))) { - LOG_WARN("fail to set common task callback", KR(ret)); - } else if (OB_FAIL(task_scheduler_->add_task(0, task))) { - LOG_WARN("fail to add task", KR(ret)); - } - if (OB_FAIL(ret)) { - client_status_ = ObTableLoadClientStatus::ERROR; - error_code_ = ret; - if (nullptr != task) { - task->~ObTableLoadTask(); - allocator_.free(task); - task = nullptr; - } + ObTableLoadTask *task = nullptr; + if (OB_ISNULL(task = OB_NEWx(ObTableLoadTask, &allocator_, param_.get_tenant_id()))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObTableLoadTask", KR(ret)); + } else if (OB_FAIL(task->set_processor(this))) { + LOG_WARN("fail to set client task processor", KR(ret)); + } else if (OB_FAIL(task->set_callback(this))) { + LOG_WARN("fail to set common task callback", KR(ret)); + } else if (OB_FAIL(task_scheduler_->add_task(0, task))) { + LOG_WARN("fail to add task", KR(ret)); + } + if (OB_FAIL(ret)) { + set_status_error(ret); + if (nullptr != task) { + task->~ObTableLoadTask(); + allocator_.free(task); + task = nullptr; } } } @@ -417,7 +608,6 @@ int ObTableLoadClientTask::write(ObTableLoadObjRowArray &obj_rows) LOG_WARN("unexpected session count", KR(ret), K(session_count_)); } else { const int64_t batch_id = ATOMIC_FAA(&next_batch_id_, 1); - ; const int32_t session_id = batch_id % session_count_ + 1; ObTableLoadSequenceNo start_seq_no(batch_id << ObTableLoadSequenceNo::BATCH_ID_SHIFT); for (int64_t i = 0; OB_SUCC(ret) && i < obj_rows.count(); ++i) { @@ -439,10 +629,15 @@ int ObTableLoadClientTask::commit() if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ObTableLoadClientTask not init", KR(ret)); - } else if (OB_FAIL(check_status(ObTableLoadClientStatus::RUNNING))) { - LOG_WARN("fail to check status", KR(ret)); - } else if (OB_FAIL(set_status_committing())) { - LOG_WARN("fail to set status committing", KR(ret)); + } else { + obsys::ObWLockGuard guard(rw_lock_); + if (ObTableLoadClientStatus::COMMITTING == client_status_ || + ObTableLoadClientStatus::COMMIT == client_status_) { + LOG_INFO("client task already commit", K(client_status_)); + } else { + ret = advance_status_nolock(ObTableLoadClientStatus::RUNNING, + ObTableLoadClientStatus::COMMITTING); + } } return ret; } @@ -460,57 +655,54 @@ void ObTableLoadClientTask::abort() } } } +void ObTableLoadClientTask::heart_beat() { client_exec_ctx_.heart_beat(); } + +int ObTableLoadClientTask::check_status() { return client_exec_ctx_.check_status(); } + +int ObTableLoadClientTask::advance_status_nolock(const ObTableLoadClientStatus expected, + const ObTableLoadClientStatus updated) +{ + int ret = OB_SUCCESS; + if (OB_LIKELY(client_status_ == expected)) { + client_status_ = updated; + LOG_INFO("LOAD DATA client status advance", K(client_status_)); + } else if (ObTableLoadClientStatus::ERROR == client_status_ || + ObTableLoadClientStatus::ABORT == client_status_) { + ret = error_code_; + LOG_WARN("client status error", KR(ret), K(client_status_), K(error_code_)); + } else { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("unexpected status", KR(ret), K(client_status_)); + } + return ret; +} + +int ObTableLoadClientTask::advance_status(const ObTableLoadClientStatus expected, + const ObTableLoadClientStatus updated) +{ + obsys::ObWLockGuard guard(rw_lock_); + + return advance_status_nolock(expected, updated); +} + +int ObTableLoadClientTask::set_status_initializing() +{ + return advance_status(ObTableLoadClientStatus::MAX_STATUS, ObTableLoadClientStatus::INITIALIZING); +} int ObTableLoadClientTask::set_status_waitting() { - int ret = OB_SUCCESS; - obsys::ObWLockGuard guard(rw_lock_); - if (OB_UNLIKELY(ObTableLoadClientStatus::INITIALIZING != client_status_)) { - ret = OB_STATE_NOT_MATCH; - LOG_WARN("unexpected status", KR(ret), K(client_status_)); - } else { - client_status_ = ObTableLoadClientStatus::WAITTING; - } - return ret; + return advance_status(ObTableLoadClientStatus::INITIALIZING, ObTableLoadClientStatus::WAITTING); } int ObTableLoadClientTask::set_status_running() { - int ret = OB_SUCCESS; - obsys::ObWLockGuard guard(rw_lock_); - if (OB_UNLIKELY(ObTableLoadClientStatus::WAITTING != client_status_)) { - ret = OB_STATE_NOT_MATCH; - LOG_WARN("unexpected status", KR(ret), K(client_status_)); - } else { - client_status_ = ObTableLoadClientStatus::RUNNING; - } - return ret; -} - -int ObTableLoadClientTask::set_status_committing() -{ - int ret = OB_SUCCESS; - obsys::ObWLockGuard guard(rw_lock_); - if (OB_UNLIKELY(ObTableLoadClientStatus::RUNNING != client_status_)) { - ret = OB_STATE_NOT_MATCH; - LOG_WARN("unexpected status", KR(ret), K(client_status_)); - } else { - client_status_ = ObTableLoadClientStatus::COMMITTING; - } - return ret; + return advance_status(ObTableLoadClientStatus::WAITTING, ObTableLoadClientStatus::RUNNING); } int ObTableLoadClientTask::set_status_commit() { - int ret = OB_SUCCESS; - obsys::ObWLockGuard guard(rw_lock_); - if (OB_UNLIKELY(ObTableLoadClientStatus::COMMITTING != client_status_)) { - ret = OB_STATE_NOT_MATCH; - LOG_WARN("unexpected status", KR(ret), K(client_status_)); - } else { - client_status_ = ObTableLoadClientStatus::COMMIT; - } - return ret; + return advance_status(ObTableLoadClientStatus::COMMITTING, ObTableLoadClientStatus::COMMIT); } int ObTableLoadClientTask::set_status_error(int error_code) @@ -574,105 +766,22 @@ void ObTableLoadClientTask::get_status(ObTableLoadClientStatus &client_status, error_code = error_code_; } -int ObTableLoadClientTask::get_compressor_type(const uint64_t tenant_id, - const uint64_t table_id, - const int64_t parallel, - ObCompressorType &compressor_type) +int ObTableLoadClientTask::init_instance(ObTableLoadParam &load_param, + const ObIArray &column_ids) { int ret = OB_SUCCESS; - ObCompressorType table_compressor_type = ObCompressorType::NONE_COMPRESSOR; - if (OB_FAIL( - ObTableLoadSchema::get_table_compressor_type(tenant_id, table_id, table_compressor_type))) { - LOG_WARN("fail to get table compressor type", KR(ret)); - } else if (OB_FAIL(ObDDLUtil::get_temp_store_compress_type(table_compressor_type, parallel, - compressor_type))) { - LOG_WARN("fail to get tmp store compressor type", KR(ret)); - } - return ret; -} - - -int ObTableLoadClientTask::init_instance() -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObTableLoadClientTask not init", KR(ret)); + const ObTableLoadTableCtx *tmp_ctx = nullptr; + if (OB_FAIL(instance_.init(load_param, column_ids, &client_exec_ctx_))) { + LOG_WARN("fail to init instance", KR(ret)); + } else if (OB_FAIL(instance_.start_trans(trans_ctx_, ObTableLoadInstance::DEFAULT_SEGMENT_ID, + allocator_))) { + LOG_WARN("fail to start trans", KR(ret)); + } else if (OB_ISNULL(tmp_ctx = instance_.get_table_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get table ctx", KR(ret)); } else { - const uint64_t tenant_id = param_.get_tenant_id(); - const uint64_t table_id = param_.get_table_id(); - const ObDirectLoadMethod::Type method = param_.get_method(); - const ObDirectLoadInsertMode::Type insert_mode = param_.get_insert_mode(); - omt::ObTenant *tenant = nullptr; - ObSchemaGetterGuard schema_guard; - ObArray column_ids; - ObCompressorType compressor_type = INVALID_COMPRESSOR; - bool online_opt_stat_gather = false; - if (OB_FAIL(GCTX.omt_->get_tenant(tenant_id, tenant))) { - LOG_WARN("fail to get tenant handle", KR(ret), K(tenant_id)); - } else if (OB_FAIL(ObTableLoadSchema::get_schema_guard(tenant_id, schema_guard))) { - LOG_WARN("fail to get schema guard", KR(ret), K(tenant_id), K(table_id)); - } else if (OB_FAIL(ObTableLoadService::check_support_direct_load(schema_guard, - table_id, - method, - insert_mode, - ObDirectLoadMode::TABLE_LOAD))) { - LOG_WARN("fail to check support direct load", KR(ret)); - } else if (OB_FAIL(ObTableLoadSchema::get_user_column_ids(schema_guard, - tenant_id, - table_id, - column_ids))) { - LOG_WARN("fail to get user column ids", KR(ret)); - } else if (OB_FAIL(get_compressor_type(tenant_id, table_id, session_count_, compressor_type))) { - LOG_WARN("fail to get compressor type", KR(ret)); - } else if (OB_FAIL(ObTableLoadSchema::get_tenant_optimizer_gather_stats_on_load( - tenant_id, online_opt_stat_gather))) { - LOG_WARN("fail to get tenant optimizer gather stats on load", KR(ret), K(tenant_id)); - } - - ObTableLoadParam load_param; - double online_sample_percent = 100.; - if (OB_SUCC(ret)) { - if (online_opt_stat_gather && - OB_FAIL(ObDbmsStatsUtils::get_sys_online_estimate_percent(exec_ctx_, - tenant_id, - table_id, - online_sample_percent))) { - LOG_WARN("failed to get sys online sample percent", K(ret)); - } else { - load_param.online_sample_percent_ = online_sample_percent; - } - } - - if (OB_SUCC(ret)) { - load_param.tenant_id_ = tenant_id; - load_param.table_id_ = table_id; - load_param.parallel_ = param_.get_parallel(); - load_param.session_count_ = load_param.parallel_; - load_param.batch_size_ = 100; - load_param.max_error_row_count_ = param_.get_max_error_row_count(); - load_param.column_count_ = column_ids.count(); - load_param.need_sort_ = true; - load_param.px_mode_ = false; - load_param.online_opt_stat_gather_ = online_opt_stat_gather; - load_param.dup_action_ = param_.get_dup_action(); - load_param.method_ = method; - load_param.insert_mode_ = insert_mode; - load_param.load_mode_ = ObDirectLoadMode::TABLE_LOAD; - load_param.compressor_type_ = compressor_type; - const ObTableLoadTableCtx *tmp_ctx = nullptr; - if (OB_FAIL(instance_.init(load_param, column_ids, &client_exec_ctx_))) { - LOG_WARN("fail to init instance", KR(ret)); - } else if (OB_FAIL(instance_.start_trans(trans_ctx_, ObTableLoadInstance::DEFAULT_SEGMENT_ID, allocator_))) { - LOG_WARN("fail to start trans", KR(ret)); - } else if (OB_ISNULL(tmp_ctx = instance_.get_table_ctx())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to get table ctx", KR(ret)); - } else { - session_count_ = tmp_ctx->param_.write_session_count_; - tmp_ctx = nullptr; - } - } + session_count_ = tmp_ctx->param_.write_session_count_; + tmp_ctx = nullptr; } return ret; } @@ -680,31 +789,17 @@ int ObTableLoadClientTask::init_instance() int ObTableLoadClientTask::commit_instance() { int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObTableLoadClientTask not init", KR(ret)); + if (OB_FAIL(instance_.commit_trans(trans_ctx_))) { + LOG_WARN("fail to commit trans", KR(ret)); + } else if (OB_FAIL(instance_.commit())) { + LOG_WARN("fail to commit instance", KR(ret)); } else { - if (OB_FAIL(instance_.commit_trans(trans_ctx_))) { - LOG_WARN("fail to commit trans", KR(ret)); - } else if (OB_FAIL(instance_.commit())) { - LOG_WARN("fail to commit instance", KR(ret)); - } else { - result_info_ = instance_.get_result_info(); - } + result_info_ = instance_.get_result_info(); } return ret; } -void ObTableLoadClientTask::destroy_instance() -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObTableLoadClientTask not init", KR(ret)); - } else { - instance_.destroy(); - } -} +void ObTableLoadClientTask::destroy_instance() { instance_.destroy(); } } // namespace observer } // namespace oceanbase diff --git a/src/observer/table_load/ob_table_load_client_task.h b/src/observer/table_load/ob_table_load_client_task.h index cce401e31..e3ad69336 100644 --- a/src/observer/table_load/ob_table_load_client_task.h +++ b/src/observer/table_load/ob_table_load_client_task.h @@ -13,9 +13,9 @@ #pragma once #include "lib/hash/ob_link_hashmap.h" +#include "observer/table_load/ob_table_load_exec_ctx.h" #include "observer/table_load/ob_table_load_instance.h" #include "observer/table_load/ob_table_load_struct.h" -#include "observer/table_load/ob_table_load_exec_ctx.h" #include "share/table/ob_table_load_define.h" #include "share/table/ob_table_load_row_array.h" #include "sql/session/ob_sql_session_mgr.h" @@ -39,51 +39,68 @@ public: int assign(const ObTableLoadClientTaskParam &other); bool is_valid() const; -#define DEFINE_GETTER_AND_SETTER(type, name) \ +#define DEFINE_VAR_GETTER_AND_SETTER(type, name) \ OB_INLINE type get_##name() const { return name##_; } \ OB_INLINE void set_##name(type name) { name##_ = name; } - DEFINE_GETTER_AND_SETTER(ObAddr, client_addr); - DEFINE_GETTER_AND_SETTER(uint64_t, tenant_id); - DEFINE_GETTER_AND_SETTER(uint64_t, user_id); - DEFINE_GETTER_AND_SETTER(uint64_t, database_id); - DEFINE_GETTER_AND_SETTER(uint64_t, table_id); - DEFINE_GETTER_AND_SETTER(int64_t, parallel); - DEFINE_GETTER_AND_SETTER(uint64_t, max_error_row_count); - DEFINE_GETTER_AND_SETTER(sql::ObLoadDupActionType, dup_action); - DEFINE_GETTER_AND_SETTER(uint64_t, timeout_us); - DEFINE_GETTER_AND_SETTER(uint64_t, heartbeat_timeout_us); - DEFINE_GETTER_AND_SETTER(storage::ObDirectLoadMethod::Type, method); - DEFINE_GETTER_AND_SETTER(storage::ObDirectLoadInsertMode::Type, insert_mode); +#define DEFINE_STR_GETTER_AND_SETTER(type, name) \ + OB_INLINE const type &get_##name() const { return name##_; } \ + OB_INLINE int set_##name(const type &name) { return set_string(name, name##_); } -#undef DEFINE_GETTER_AND_SETTER +#define DEFINE_STR_ARRAY_GETTER_AND_SETTER(type, name) \ + OB_INLINE const ObIArray &get_##name() const { return name##_; } \ + OB_INLINE int set_##name(const ObIArray &name) { return set_string_array(name, name##_); } + + DEFINE_VAR_GETTER_AND_SETTER(ObAddr, client_addr); + DEFINE_VAR_GETTER_AND_SETTER(uint64_t, task_id); + DEFINE_VAR_GETTER_AND_SETTER(uint64_t, tenant_id); + DEFINE_VAR_GETTER_AND_SETTER(uint64_t, user_id); + DEFINE_VAR_GETTER_AND_SETTER(uint64_t, database_id); + DEFINE_STR_GETTER_AND_SETTER(ObString, table_name); + DEFINE_VAR_GETTER_AND_SETTER(int64_t, parallel); + DEFINE_VAR_GETTER_AND_SETTER(uint64_t, max_error_row_count); + DEFINE_VAR_GETTER_AND_SETTER(sql::ObLoadDupActionType, dup_action); + DEFINE_VAR_GETTER_AND_SETTER(uint64_t, timeout_us); + DEFINE_VAR_GETTER_AND_SETTER(uint64_t, heartbeat_timeout_us); + DEFINE_STR_GETTER_AND_SETTER(ObString, load_method); + DEFINE_STR_ARRAY_GETTER_AND_SETTER(ObString, column_names); + +#undef DEFINE_VAR_GETTER_AND_SETTER +#undef DEFINE_STR_GETTER_AND_SETTER TO_STRING_KV(K_(client_addr), + K_(task_id), K_(tenant_id), K_(user_id), K_(database_id), - K_(table_id), + K_(table_name), K_(parallel), K_(max_error_row_count), K_(dup_action), K_(timeout_us), K_(heartbeat_timeout_us), - "method", storage::ObDirectLoadMethod::get_type_string(method_), - "insert_mode", storage::ObDirectLoadInsertMode::get_type_string(insert_mode_)); + K_(load_method), + K_(column_names)); private: + int set_string(const ObString &src, ObString &dest); + int set_string_array(const ObIArray &src, ObIArray &dest); + +private: + ObArenaAllocator allocator_; + int64_t task_id_; ObAddr client_addr_; uint64_t tenant_id_; uint64_t user_id_; uint64_t database_id_; - uint64_t table_id_; + ObString table_name_; int64_t parallel_; uint64_t max_error_row_count_; sql::ObLoadDupActionType dup_action_; int64_t timeout_us_; int64_t heartbeat_timeout_us_; - storage::ObDirectLoadMethod::Type method_; - storage::ObDirectLoadInsertMode::Type insert_mode_; + ObString load_method_; + common::ObArray column_names_; }; class ObTableLoadClientTask @@ -99,55 +116,61 @@ public: OB_INLINE int64_t get_ref_count() const { return ATOMIC_LOAD(&ref_count_); } OB_INLINE int64_t inc_ref_count() { return ATOMIC_AAF(&ref_count_, 1); } OB_INLINE int64_t dec_ref_count() { return ATOMIC_SAF(&ref_count_, 1); } - OB_INLINE sql::ObSQLSessionInfo *get_session_info() { return session_info_; } - OB_INLINE ObTableLoadClientExecCtx *get_exec_ctx() { return &client_exec_ctx_; } + OB_INLINE int64_t get_task_id() const { return param_.get_task_id(); } + OB_INLINE uint64_t get_table_id() const { return 0; } + void heart_beat(); + int check_status(); + + int set_status_initializing(); int set_status_waitting(); int set_status_running(); - int set_status_committing(); int set_status_commit(); int set_status_error(int error_code); void set_status_abort(int error_code = OB_CANCELED); table::ObTableLoadClientStatus get_status() const; void get_status(table::ObTableLoadClientStatus &client_status, int &error_code) const; int check_status(table::ObTableLoadClientStatus client_status); - TO_STRING_KV(K_(task_id), K_(param), K_(result_info), KP_(session_info), K_(free_session_ctx), - K_(client_exec_ctx), KP_(task_scheduler), K_(client_status), K_(error_code), + + OB_INLINE const table::ObTableLoadResultInfo &get_result_info() const { return result_info_; } + TO_STRING_KV(K_(param), + KP_(session_info), + K_(free_session_ctx), + K_(client_exec_ctx), + KP_(task_scheduler), + K_(client_status), + K_(error_code), + K_(result_info), K_(ref_count)); private: - int init_task_scheduler(); - int create_session_info(uint64_t tenant_id, uint64_t user_id, uint64_t database_id, - uint64_t table_id, sql::ObSQLSessionInfo *&session_info, - sql::ObFreeSessionCtx &free_session_ctx); + int create_session_info(); int init_exec_ctx(); + int init_task_scheduler(); - int init_instance(); + int advance_status_nolock(const table::ObTableLoadClientStatus expected, + const table::ObTableLoadClientStatus updated); + int advance_status(const table::ObTableLoadClientStatus expected, + const table::ObTableLoadClientStatus updated); + + int init_instance(ObTableLoadParam &load_param, const ObIArray &column_ids); int commit_instance(); void destroy_instance(); - int get_compressor_type(const uint64_t tenant_id, - const uint64_t table_id, - const int64_t parallel, - ObCompressorType &compressor_type); private: class ClientTaskExectueProcessor; class ClientTaskExectueCallback; -public: - uint64_t task_id_; - ObTableLoadClientTaskParam param_; - table::ObTableLoadResultInfo result_info_; - private: ObArenaAllocator allocator_; - ObITableLoadTaskScheduler *task_scheduler_; + ObTableLoadClientTaskParam param_; + share::schema::ObSchemaGetterGuard schema_guard_; sql::ObSQLSessionInfo *session_info_; sql::ObFreeSessionCtx free_session_ctx_; - ObTableLoadClientExecCtx client_exec_ctx_; - ObSchemaGetterGuard schema_guard_; sql::ObSqlCtx sql_ctx_; sql::ObPhysicalPlanCtx plan_ctx_; ObExecContext exec_ctx_; + ObTableLoadClientExecCtx client_exec_ctx_; + ObITableLoadTaskScheduler *task_scheduler_; int64_t session_count_; ObTableLoadInstance instance_; ObTableLoadInstance::TransCtx trans_ctx_; @@ -155,6 +178,7 @@ private: mutable obsys::ObRWLock rw_lock_; table::ObTableLoadClientStatus client_status_; int error_code_; + table::ObTableLoadResultInfo result_info_; int64_t ref_count_ CACHE_ALIGNED; bool is_inited_; }; diff --git a/src/observer/table_load/ob_table_load_coordinator.cpp b/src/observer/table_load/ob_table_load_coordinator.cpp index 816b1febd..b1b16f090 100644 --- a/src/observer/table_load/ob_table_load_coordinator.cpp +++ b/src/observer/table_load/ob_table_load_coordinator.cpp @@ -17,6 +17,7 @@ #include "observer/table_load/control/ob_table_load_control_rpc_proxy.h" #include "observer/table_load/ob_table_load_coordinator_ctx.h" #include "observer/table_load/ob_table_load_coordinator_trans.h" +#include "observer/table_load/ob_table_load_error_row_handler.h" #include "observer/table_load/ob_table_load_redef_table.h" #include "observer/table_load/ob_table_load_service.h" #include "observer/table_load/ob_table_load_stat.h" @@ -1117,7 +1118,7 @@ int ObTableLoadCoordinator::write_sql_stat(ObTableLoadSqlStatistics &sql_statist "TLD_TabStatNode", tenant_id))) { LOG_WARN("fail to create table stats map", KR(ret)); - } else if (OB_FAIL(inc_column_stats.create(ctx_->param_.column_count_, + } else if (OB_FAIL(inc_column_stats.create(ctx_->schema_.store_column_count_, "TLD_ColStatBkt", "TLD_ColStatNode", tenant_id))) { @@ -1659,10 +1660,20 @@ public: int set_objs(const ObTableLoadObjRowArray &obj_rows, const ObIArray &idx_array) { int ret = OB_SUCCESS; + ObTableLoadErrorRowHandler *error_row_handler = ctx_->coordinator_ctx_->error_row_handler_; for (int64_t i = 0; OB_SUCC(ret) && (i < obj_rows.count()); ++i) { const ObTableLoadObjRow &src_obj_row = obj_rows.at(i); ObTableLoadObjRow out_obj_row; - if (OB_FAIL(src_obj_row.project(idx_array, out_obj_row))) { + // 对于客户端导入场景, 需要处理多列或者少列 + if (OB_UNLIKELY(src_obj_row.count_ != ctx_->param_.column_count_)) { + ret = OB_ERR_COULUMN_VALUE_NOT_MATCH; + LOG_WARN("column count doesn't match value count", KR(ret), K(src_obj_row), + K(ctx_->param_.column_count_)); + ObNewRow new_row(src_obj_row.cells_, src_obj_row.count_); + if (OB_FAIL(error_row_handler->handle_error_row(ret, new_row))) { + LOG_WARN("fail to handle error row", KR(ret)); + } + } else if (OB_FAIL(src_obj_row.project(idx_array, out_obj_row))) { LOG_WARN("failed to projecte out_obj_row", KR(ret), K(src_obj_row.count_)); } else if (OB_FAIL(obj_rows_.push_back(out_obj_row))) { LOG_WARN("failed to add row to obj_rows_", KR(ret), K(out_obj_row)); diff --git a/src/observer/table_load/ob_table_load_coordinator_ctx.cpp b/src/observer/table_load/ob_table_load_coordinator_ctx.cpp index f7bb55416..eaaed98a9 100644 --- a/src/observer/table_load/ob_table_load_coordinator_ctx.cpp +++ b/src/observer/table_load/ob_table_load_coordinator_ctx.cpp @@ -353,10 +353,9 @@ int ObTableLoadCoordinatorCtx::init_column_idxs(const ObIArray &column int ret = OB_SUCCESS; idx_array_.reset(); const ObIArray &column_descs = ctx_->schema_.column_descs_; - bool found_column = true; - for (int64_t i = 0; OB_SUCC(ret) && OB_LIKELY(found_column) && i < column_descs.count(); ++i) { + for (int64_t i = 0; OB_SUCC(ret) && i < column_descs.count(); ++i) { const ObColDesc &col_desc = column_descs.at(i); - found_column = (ctx_->schema_.is_heap_table_ && i == 0); // skip hidden pk in heap table + bool found_column = (ctx_->schema_.is_heap_table_ && i == 0); // skip hidden pk in heap table // 在源数据的列数组中找到对应的列 for (int64_t j = 0; OB_SUCC(ret) && OB_LIKELY(!found_column) && j < column_ids.count(); ++j) { const uint64_t column_id = column_ids.at(j); @@ -368,10 +367,14 @@ int ObTableLoadCoordinatorCtx::init_column_idxs(const ObIArray &column } } } - } - if (OB_SUCC(ret) && OB_UNLIKELY(!found_column)) { - ret = OB_SCHEMA_NOT_UPTODATE; - LOG_WARN("column not found", KR(ret), K(idx_array_), K(column_descs), K(column_ids)); + if (OB_SUCC(ret) && !found_column) { + if (OB_UNLIKELY(ctx_->param_.px_mode_)) { + ret = OB_SCHEMA_NOT_UPTODATE; + LOG_WARN("column not found", KR(ret), K(idx_array_), K(column_descs), K(column_ids)); + } else if (OB_FAIL(idx_array_.push_back(-1))) { + LOG_WARN("fail to push back column idx", KR(ret), K(idx_array_), K(i), K(col_desc)); + } + } } return ret; } diff --git a/src/observer/table_load/ob_table_load_exec_ctx.cpp b/src/observer/table_load/ob_table_load_exec_ctx.cpp index 2578d5ad7..33a7c09f8 100644 --- a/src/observer/table_load/ob_table_load_exec_ctx.cpp +++ b/src/observer/table_load/ob_table_load_exec_ctx.cpp @@ -45,6 +45,15 @@ ObSQLSessionInfo *ObTableLoadExecCtx::get_session_info() return session_info; } +ObSchemaGetterGuard *ObTableLoadExecCtx::get_schema_guard() +{ + ObSchemaGetterGuard *schema_guard = nullptr; + if (nullptr != exec_ctx_ && nullptr != exec_ctx_->get_sql_ctx()) { + schema_guard = exec_ctx_->get_sql_ctx()->schema_guard_; + } + return schema_guard; +} + int ObTableLoadExecCtx::check_status() { int ret = OB_SUCCESS; @@ -74,12 +83,24 @@ int ObTableLoadClientExecCtx::check_status() int ret = OB_SUCCESS; if (OB_FAIL(ObTableLoadExecCtx::check_status())) { LOG_WARN("fail to check status", KR(ret)); - } else if (OB_UNLIKELY(ObTimeUtil::current_time() - last_heartbeat_time_ > heartbeat_timeout_us_)) { + } else if (OB_UNLIKELY(last_heartbeat_time_ + heartbeat_timeout_us_ < + ObTimeUtil::current_time())) { ret = OB_TIMEOUT; LOG_WARN("heartbeat is timeout", KR(ret), K(last_heartbeat_time_), K(heartbeat_timeout_us_)); } return ret; } +void ObTableLoadClientExecCtx::init_heart_beat(const int64_t heartbeat_timeout_us) +{ + heartbeat_timeout_us_ = heartbeat_timeout_us; + last_heartbeat_time_ = ObTimeUtil::current_time(); +} + +void ObTableLoadClientExecCtx::heart_beat() +{ + last_heartbeat_time_ = ObTimeUtil::current_time(); +} + } // namespace observer } // namespace oceanbase \ No newline at end of file diff --git a/src/observer/table_load/ob_table_load_exec_ctx.h b/src/observer/table_load/ob_table_load_exec_ctx.h index 1ec33edfa..bfb94e8a7 100644 --- a/src/observer/table_load/ob_table_load_exec_ctx.h +++ b/src/observer/table_load/ob_table_load_exec_ctx.h @@ -17,6 +17,13 @@ namespace oceanbase { +namespace share +{ +namespace schema +{ +class ObSchemaGetterGuard; +} // namespace schema +} // namespace share namespace sql { class ObExecContext; @@ -36,6 +43,7 @@ public: virtual ~ObTableLoadExecCtx() = default; common::ObIAllocator *get_allocator(); sql::ObSQLSessionInfo *get_session_info(); + share::schema::ObSchemaGetterGuard *get_schema_guard(); virtual int check_status(); bool is_valid() const { return nullptr != exec_ctx_; } TO_STRING_KV(KP_(exec_ctx), KP_(tx_desc)); @@ -54,8 +62,10 @@ public: } virtual ~ObTableLoadClientExecCtx() = default; virtual int check_status(); + void init_heart_beat(const int64_t heartbeat_timeout_us); + void heart_beat(); TO_STRING_KV(KP_(exec_ctx), KP_(tx_desc), K_(heartbeat_timeout_us), K_(last_heartbeat_time)); -public: +private: int64_t heartbeat_timeout_us_; int64_t last_heartbeat_time_; }; diff --git a/src/observer/table_load/ob_table_load_instance.cpp b/src/observer/table_load/ob_table_load_instance.cpp index 94c08b4ad..b37a820b2 100644 --- a/src/observer/table_load/ob_table_load_instance.cpp +++ b/src/observer/table_load/ob_table_load_instance.cpp @@ -70,9 +70,10 @@ int ObTableLoadInstance::init(ObTableLoadParam ¶m, if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("ObTableLoadInstance init twice", KR(ret), KP(this)); - } else if (OB_UNLIKELY(!param.is_valid() || !execute_ctx->is_valid())) { + } else if (OB_UNLIKELY(!param.is_valid() || column_ids.empty() || + column_ids.count() != param.column_count_ || !execute_ctx->is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(param), KPC(execute_ctx)); + LOG_WARN("invalid args", KR(ret), K(param), K(column_ids), KPC(execute_ctx)); } else { DISABLE_SQL_MEMLEAK_GUARD; execute_ctx_ = execute_ctx; @@ -92,7 +93,8 @@ int ObTableLoadInstance::init(ObTableLoadParam ¶m, else if (OB_FAIL(ObTableLoadService::check_support_direct_load(param.table_id_, param.method_, param.insert_mode_, - param.load_mode_))) { + param.load_mode_, + column_ids))) { LOG_WARN("fail to check support direct load", KR(ret), K(param)); } // start direct load diff --git a/src/observer/table_load/ob_table_load_mem_compactor.cpp b/src/observer/table_load/ob_table_load_mem_compactor.cpp index d426ab12c..6337a03b7 100644 --- a/src/observer/table_load/ob_table_load_mem_compactor.cpp +++ b/src/observer/table_load/ob_table_load_mem_compactor.cpp @@ -290,7 +290,9 @@ int ObTableLoadMemCompactor::inner_init() mem_ctx_.table_data_desc_ = store_ctx_->table_data_desc_; mem_ctx_.datum_utils_ = &(store_ctx_->ctx_->schema_.datum_utils_); mem_ctx_.need_sort_ = param_->need_sort_; - mem_ctx_.column_count_ = param_->column_count_; + mem_ctx_.column_count_ = (store_ctx_->ctx_->schema_.is_heap_table_ + ? store_ctx_->ctx_->schema_.store_column_count_ - 1 + : store_ctx_->ctx_->schema_.store_column_count_); } mem_ctx_.mem_load_task_count_ = param_->session_count_; diff --git a/src/observer/table_load/ob_table_load_multiple_heap_table_compactor.cpp b/src/observer/table_load/ob_table_load_multiple_heap_table_compactor.cpp index 14f356afa..8d5b8233c 100644 --- a/src/observer/table_load/ob_table_load_multiple_heap_table_compactor.cpp +++ b/src/observer/table_load/ob_table_load_multiple_heap_table_compactor.cpp @@ -331,7 +331,9 @@ int ObTableLoadMultipleHeapTableCompactor::inner_init() mem_ctx_.datum_utils_ = &(store_ctx_->ctx_->schema_.datum_utils_); mem_ctx_.need_sort_ = param_->need_sort_; mem_ctx_.mem_load_task_count_ = param_->session_count_; - mem_ctx_.column_count_ = param_->column_count_; + mem_ctx_.column_count_ = + (store_ctx_->ctx_->schema_.is_heap_table_ ? store_ctx_->ctx_->schema_.store_column_count_ - 1 + : store_ctx_->ctx_->schema_.store_column_count_); mem_ctx_.dml_row_handler_ = store_ctx_->error_row_handler_; mem_ctx_.file_mgr_ = store_ctx_->tmp_file_mgr_; mem_ctx_.dup_action_ = param_->dup_action_; diff --git a/src/observer/table_load/ob_table_load_obj_cast.cpp b/src/observer/table_load/ob_table_load_obj_cast.cpp index b32936dc0..834e9132a 100644 --- a/src/observer/table_load/ob_table_load_obj_cast.cpp +++ b/src/observer/table_load/ob_table_load_obj_cast.cpp @@ -96,18 +96,49 @@ static int pad_obj(ObTableLoadCastObjCtx &cast_obj_ctx, const ObColumnSchemaV2 * } int ObTableLoadObjCaster::cast_obj(ObTableLoadCastObjCtx &cast_obj_ctx, - const ObColumnSchemaV2 *column_schema, const ObObj &src, + const ObColumnSchemaV2 *column_schema, + const ObObj &src, ObObj &dst) { int ret = OB_SUCCESS; const ObObj *convert_src_obj = nullptr; const ObObjType expect_type = column_schema->get_meta_type().get_type(); const ObAccuracy &accuracy = column_schema->get_accuracy(); - if (OB_FAIL(convert_obj(expect_type, src, convert_src_obj))) { - LOG_WARN("fail to convert obj", KR(ret)); + if (src.is_nop_value()) { + // 默认值是表达式 + if (lib::is_mysql_mode() && column_schema->get_cur_default_value().is_ext()) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("column default value is ext", KR(ret), KPC(column_schema)); + } else if (lib::is_oracle_mode() && column_schema->is_default_expr_v2_column()) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("column default value is expr", KR(ret), KPC(column_schema)); + } + // 没有默认值, 且为NOT NULL + // 例外:枚举类型默认为第一个 + else if (column_schema->is_not_null_for_write() && + column_schema->get_cur_default_value().is_null()) { + if (column_schema->get_meta_type().is_enum()) { + const uint64_t ENUM_FIRST_VAL = 1; + dst.set_enum(ENUM_FIRST_VAL); + } else { + ret = OB_ERR_NO_DEFAULT_FOR_FIELD; + LOG_WARN("column can not be null", KR(ret), KPC(column_schema)); + } + } + // mysql模式可以直接用default value + else if (lib::is_mysql_mode()) { + dst = column_schema->get_cur_default_value(); + } + // oracle模式需要转换 + else { + convert_src_obj = &(column_schema->get_cur_default_value()); + } + } else { + if (OB_FAIL(convert_obj(expect_type, src, convert_src_obj))) { + LOG_WARN("fail to convert obj", KR(ret)); + } } - - if (OB_SUCC(ret)) { + if (OB_SUCC(ret) && convert_src_obj != nullptr) { if (column_schema->is_enum_or_set()) { if (OB_FAIL(handle_string_to_enum_set(cast_obj_ctx, column_schema, src, dst))) { LOG_WARN("fail to convert string to enum or set", KR(ret), K(src), K(dst)); @@ -117,18 +148,18 @@ int ObTableLoadObjCaster::cast_obj(ObTableLoadCastObjCtx &cast_obj_ctx, LOG_WARN("fail to do to type", KR(ret)); } } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(pad_obj(cast_obj_ctx, column_schema, dst))) { - LOG_WARN("fail to pad obj", KR(ret)); + if (OB_SUCC(ret)) { + if (OB_FAIL(pad_obj(cast_obj_ctx, column_schema, dst))) { + LOG_WARN("fail to pad obj", KR(ret)); + } } - } - if (OB_SUCC(ret)) { - if (cast_obj_ctx.is_need_check_ && - OB_FAIL(cast_obj_check(cast_obj_ctx, column_schema, dst))) { - LOG_WARN("fail to check cast obj result", KR(ret), K(dst)); + if (OB_SUCC(ret)) { + if (cast_obj_ctx.is_need_check_ && + OB_FAIL(cast_obj_check(cast_obj_ctx, column_schema, dst))) { + LOG_WARN("fail to check cast obj result", KR(ret), K(dst)); + } } } return ret; diff --git a/src/observer/table_load/ob_table_load_partition_calc.cpp b/src/observer/table_load/ob_table_load_partition_calc.cpp index 83573faa5..5ce24bad8 100644 --- a/src/observer/table_load/ob_table_load_partition_calc.cpp +++ b/src/observer/table_load/ob_table_load_partition_calc.cpp @@ -181,11 +181,15 @@ int ObTableLoadPartitionCalc::cast_part_key(common::ObNewRow &part_key, common:: ObObj obj; for (int64_t i = 0; OB_SUCC(ret) && i < part_key_obj_index_.count(); ++i) { const IndexAndType &index_and_type = part_key_obj_index_.at(i); - if (OB_FAIL(ObTableLoadObjCaster::cast_obj(cast_obj_ctx, index_and_type.column_schema_, - part_key.cells_[i], obj))) { + const ObColumnSchemaV2 *column_schema = index_and_type.column_schema_; + ObObj &part_obj = part_key.cells_[i]; + if (OB_FAIL(ObTableLoadObjCaster::cast_obj(cast_obj_ctx, + column_schema, + part_obj, + obj))) { LOG_WARN("fail to cast obj", KR(ret)); } else { - part_key.cells_[i] = obj; + part_obj = obj; } } } diff --git a/src/observer/table_load/ob_table_load_schema.cpp b/src/observer/table_load/ob_table_load_schema.cpp index 675f99b7d..21b3bb57a 100644 --- a/src/observer/table_load/ob_table_load_schema.cpp +++ b/src/observer/table_load/ob_table_load_schema.cpp @@ -40,17 +40,16 @@ int ObTableLoadSchema::get_schema_guard(uint64_t tenant_id, ObSchemaGetterGuard return ret; } -int ObTableLoadSchema::get_table_schema(uint64_t tenant_id, uint64_t database_id, +int ObTableLoadSchema::get_table_schema(ObSchemaGetterGuard &schema_guard, + uint64_t tenant_id, + uint64_t database_id, const ObString &table_name, - ObSchemaGetterGuard &schema_guard, const ObTableSchema *&table_schema) { int ret = OB_SUCCESS; table_schema = nullptr; - if (OB_FAIL(get_schema_guard(tenant_id, schema_guard))) { - LOG_WARN("fail to get schema guard", KR(ret), K(tenant_id)); - } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, database_id, table_name, false, - table_schema))) { + if (OB_FAIL(schema_guard.get_table_schema(tenant_id, database_id, table_name, false /*is_index*/, + table_schema))) { LOG_WARN("fail to get table schema", KR(ret), K(tenant_id), K(database_id), K(table_name)); } else if (OB_ISNULL(table_schema)) { ret = OB_TABLE_NOT_EXIST; @@ -59,20 +58,6 @@ int ObTableLoadSchema::get_table_schema(uint64_t tenant_id, uint64_t database_id return ret; } -int ObTableLoadSchema::get_table_id(uint64_t tenant_id, uint64_t database_id, - const ObString &table_name, uint64_t &table_id) -{ - int ret = OB_SUCCESS; - ObSchemaGetterGuard schema_guard; - const ObTableSchema *table_schema = nullptr; - if (OB_FAIL(get_table_schema(tenant_id, database_id, table_name, schema_guard, table_schema))) { - LOG_WARN("fail to get table schema", KR(ret), K(tenant_id), K(database_id), K(table_name)); - } else { - table_id = table_schema->get_table_id(); - } - return ret; -} - int ObTableLoadSchema::get_table_schema(uint64_t tenant_id, uint64_t table_id, ObSchemaGetterGuard &schema_guard, const ObTableSchema *&table_schema) @@ -112,6 +97,25 @@ int ObTableLoadSchema::get_table_schema(uint64_t tenant_id, uint64_t table_id, return ret; } +int ObTableLoadSchema::get_table_schema(ObSchemaGetterGuard &schema_guard, + uint64_t tenant_id, + uint64_t table_id, + const ObTableSchema *&table_schema) +{ + int ret = OB_SUCCESS; + table_schema = nullptr; + if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || OB_INVALID_ID == table_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(tenant_id), K(table_id)); + } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, table_id, table_schema))) { + LOG_WARN("fail to get table schema", KR(ret), K(tenant_id), K(table_id)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_TABLE_NOT_EXIST; + LOG_WARN("table not exist", KR(ret), K(tenant_id), K(table_id)); + } + return ret; +} + int ObTableLoadSchema::get_user_column_schemas(const ObTableSchema *table_schema, ObIArray &column_schemas) { @@ -155,35 +159,82 @@ int ObTableLoadSchema::get_user_column_schemas(ObSchemaGetterGuard &schema_guard { int ret = OB_SUCCESS; const ObTableSchema *table_schema = nullptr; - if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || OB_INVALID_ID == table_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(tenant_id), K(table_id)); - } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, table_id, table_schema))) { - LOG_WARN("fail to get table schema", KR(ret), K(tenant_id), K(table_id)); - } else if (OB_ISNULL(table_schema)) { - ret = OB_TABLE_NOT_EXIST; - LOG_WARN("table not exist", KR(ret), K(tenant_id), K(table_id)); + if (OB_FAIL(get_table_schema(schema_guard, tenant_id, table_id, table_schema))) { + LOG_WARN("fail to get table schema", KR(ret)); } else { ret = get_user_column_schemas(table_schema, column_schemas); } return ret; } +int ObTableLoadSchema::get_user_column_ids(const ObTableSchema *table_schema, + ObIArray &column_ids) +{ + int ret = OB_SUCCESS; + column_ids.reset(); + ObArray column_schemas; + if (OB_FAIL(get_user_column_schemas(table_schema, column_schemas))) { + LOG_WARN("fail to get user column schemas", KR(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < column_schemas.count(); ++i) { + const ObColumnSchemaV2 *column_schema = column_schemas.at(i); + if (OB_FAIL(column_ids.push_back(column_schema->get_column_id()))) { + LOG_WARN("fail to push back column id", KR(ret)); + } + } + return ret; +} + int ObTableLoadSchema::get_user_column_ids(ObSchemaGetterGuard &schema_guard, uint64_t tenant_id, uint64_t table_id, ObIArray &column_ids) { int ret = OB_SUCCESS; - column_ids.reset(); + const ObTableSchema *table_schema = nullptr; + if (OB_FAIL(get_table_schema(schema_guard, tenant_id, table_id, table_schema))) { + LOG_WARN("fail to get table schema", KR(ret)); + } else { + ret = get_user_column_ids(table_schema, column_ids); + } + return ret; +} + +int ObTableLoadSchema::get_user_column_names(const ObTableSchema *table_schema, + ObIArray &column_names) +{ + int ret = OB_SUCCESS; + column_names.reset(); ObArray column_schemas; - if (OB_FAIL(get_user_column_schemas(schema_guard, tenant_id, table_id, column_schemas))) { + if (OB_FAIL(get_user_column_schemas(table_schema, column_schemas))) { + LOG_WARN("fail to get user column schemas", KR(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < column_schemas.count(); ++i) { + const ObColumnSchemaV2 *column_schema = column_schemas.at(i); + if (OB_FAIL(column_names.push_back(column_schema->get_column_name_str()))) { + LOG_WARN("fail to push back column name", KR(ret)); + } + } + return ret; +} + +int ObTableLoadSchema::get_user_column_id_and_names(const ObTableSchema *table_schema, + ObIArray &column_ids, + ObIArray &column_names) +{ + int ret = OB_SUCCESS; + column_ids.reset(); + column_names.reset(); + ObArray column_schemas; + if (OB_FAIL(get_user_column_schemas(table_schema, column_schemas))) { LOG_WARN("fail to get user column schemas", KR(ret)); } for (int64_t i = 0; OB_SUCC(ret) && i < column_schemas.count(); ++i) { const ObColumnSchemaV2 *column_schema = column_schemas.at(i); if (OB_FAIL(column_ids.push_back(column_schema->get_column_id()))) { LOG_WARN("fail to push back column id", KR(ret)); + } else if (OB_FAIL(column_names.push_back(column_schema->get_column_name_str()))) { + LOG_WARN("fail to push back column name", KR(ret)); } } return ret; @@ -205,6 +256,31 @@ int ObTableLoadSchema::get_user_column_count(ObSchemaGetterGuard &schema_guard, return ret; } +int ObTableLoadSchema::get_column_ids(const ObTableSchema *table_schema, + ObIArray &column_ids, + bool contain_hidden_pk_column) +{ + int ret = OB_SUCCESS; + column_ids.reset(); + if (OB_ISNULL(table_schema)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KP(table_schema)); + } else { + ObArray column_descs; + if (OB_FAIL(table_schema->get_column_ids(column_descs))) { + STORAGE_LOG(WARN, "fail to get column descs", KR(ret), KPC(table_schema)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < column_descs.count(); ++i) { + const ObColDesc &col_desc = column_descs.at(i); + if (ObColumnSchemaV2::is_hidden_pk_column_id(col_desc.col_id_) && !contain_hidden_pk_column) { + } else if (OB_FAIL(column_ids.push_back(col_desc.col_id_))) { + LOG_WARN("failed to push back column id", KR(ret), K(i)); + } + } + } + return ret; +} + int ObTableLoadSchema::get_column_ids(ObSchemaGetterGuard &schema_guard, uint64_t tenant_id, uint64_t table_id, @@ -218,26 +294,13 @@ int ObTableLoadSchema::get_column_ids(ObSchemaGetterGuard &schema_guard, LOG_WARN("invalid args", KR(ret), K(tenant_id), K(table_id)); } else { const ObTableSchema *table_schema = nullptr; - ObArray column_descs; if (OB_FAIL(schema_guard.get_table_schema(tenant_id, table_id, table_schema))) { LOG_WARN("fail to get table schema", KR(ret), K(tenant_id), K(table_id)); } else if (OB_ISNULL(table_schema)) { ret = OB_TABLE_NOT_EXIST; LOG_WARN("table not exist", KR(ret), K(tenant_id), K(table_id)); - } else if (OB_FAIL(table_schema->get_column_ids(column_descs))) { - STORAGE_LOG(WARN, "fail to get column descs", KR(ret), KPC(table_schema)); - } - for (int64_t i = 0; OB_SUCC(ret) && i < column_descs.count(); ++i) { - const ObColDesc &col_desc = column_descs.at(i); - const ObColumnSchemaV2 *col_schema = table_schema->get_column_schema(col_desc.col_id_); - if (OB_ISNULL(col_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null column schema", KR(ret), K(col_desc)); - } else if (ObColumnSchemaV2::is_hidden_pk_column_id(col_schema->get_column_id()) && - !contain_hidden_pk_column) { - } else if (OB_FAIL(column_ids.push_back(col_schema->get_column_id()))) { - LOG_WARN("failed to push back column id", KR(ret), K(i)); - } + } else { + ret = get_column_ids(table_schema, column_ids, contain_hidden_pk_column); } } return ret; @@ -341,6 +404,24 @@ int ObTableLoadSchema::check_has_unused_column(const ObTableSchema *table_schema return ret; } +int ObTableLoadSchema::check_has_roaringbitmap_column(const ObTableSchema *table_schema, bool &bret) +{ + int ret = OB_SUCCESS; + bret = false; + for (ObTableSchema::const_column_iterator iter = table_schema->column_begin(); + OB_SUCC(ret) && iter != table_schema->column_end(); ++iter) { + ObColumnSchemaV2 *column_schema = *iter; + if (OB_ISNULL(column_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("invalid column schema", K(column_schema)); + } else if (column_schema->is_roaringbitmap()) { + bret = true; + break; + } + } + return ret; +} + int ObTableLoadSchema::check_has_lob_column(const ObTableSchema *table_schema, bool &bret) { int ret = OB_SUCCESS; diff --git a/src/observer/table_load/ob_table_load_schema.h b/src/observer/table_load/ob_table_load_schema.h index 86cd7ab10..6a0636079 100644 --- a/src/observer/table_load/ob_table_load_schema.h +++ b/src/observer/table_load/ob_table_load_schema.h @@ -29,29 +29,43 @@ class ObTableLoadSchema { public: static int get_schema_guard(uint64_t tenant_id, share::schema::ObSchemaGetterGuard &schema_guard); - static int get_table_schema(uint64_t tenant_id, uint64_t database_id, + static int get_table_schema(share::schema::ObSchemaGetterGuard &schema_guard, + uint64_t tenant_id, + uint64_t database_id, const common::ObString &table_name, - share::schema::ObSchemaGetterGuard &schema_guard, const share::schema::ObTableSchema *&table_schema); - static int get_table_id(uint64_t tenant_id, uint64_t database_id, - const common::ObString &table_name, uint64_t &table_id); + // 获取最新schema_guard和table_schema static int get_table_schema(uint64_t tenant_id, uint64_t table_id, share::schema::ObSchemaGetterGuard &schema_guard, const share::schema::ObTableSchema *&table_schema); + // 指定schema_guard获取table_schema + static int get_table_schema(share::schema::ObSchemaGetterGuard &schema_guard, + uint64_t tenant_id, uint64_t table_id, + const share::schema::ObTableSchema *&table_schema); static int get_user_column_schemas(const share::schema::ObTableSchema *table_schema, ObIArray &column_schemas); static int get_user_column_schemas(share::schema::ObSchemaGetterGuard &schema_guard, uint64_t tenant_id, uint64_t table_id, ObIArray &column_schemas); + static int get_user_column_ids(const share::schema::ObTableSchema *table_schema, + common::ObIArray &column_ids); static int get_user_column_ids(share::schema::ObSchemaGetterGuard &schema_guard, uint64_t tenant_id, uint64_t table_id, common::ObIArray &column_ids); + static int get_user_column_names(const share::schema::ObTableSchema *table_schema, + common::ObIArray &column_names); + static int get_user_column_id_and_names(const share::schema::ObTableSchema *table_schema, + common::ObIArray &column_ids, + common::ObIArray &column_names); static int get_user_column_count(share::schema::ObSchemaGetterGuard &schema_guard, uint64_t tenant_id, uint64_t table_id, int64_t &column_count); + static int get_column_ids(const share::schema::ObTableSchema *table_schema, + common::ObIArray &column_ids, + bool contain_hidden_pk_column = false); static int get_column_ids(share::schema::ObSchemaGetterGuard &schema_guard, uint64_t tenant_id, uint64_t table_id, @@ -61,6 +75,7 @@ public: static int get_tenant_optimizer_gather_stats_on_load(const uint64_t tenant_id, bool &value); static int check_has_invisible_column(const share::schema::ObTableSchema *table_schema, bool &bret); static int check_has_unused_column(const share::schema::ObTableSchema *table_schema, bool &bret); + static int check_has_roaringbitmap_column(const share::schema::ObTableSchema *table_schema, bool &bret); static int check_has_lob_column(const share::schema::ObTableSchema *table_schema, bool &bret); static int get_table_compressor_type(uint64_t tenant_id, uint64_t table_id, ObCompressorType &compressor_type); diff --git a/src/observer/table_load/ob_table_load_service.cpp b/src/observer/table_load/ob_table_load_service.cpp index e5542401e..73f7b940c 100644 --- a/src/observer/table_load/ob_table_load_service.cpp +++ b/src/observer/table_load/ob_table_load_service.cpp @@ -350,7 +350,7 @@ void ObTableLoadService::ObClientTaskAutoAbortTask::runTimerTask() for (int64_t i = 0; i < client_task_array.count(); ++i) { ObTableLoadClientTask *client_task = client_task_array.at(i); if (OB_UNLIKELY(ObTableLoadClientStatus::ERROR == client_task->get_status() || - client_task->get_exec_ctx()->check_status() != OB_SUCCESS)) { + client_task->check_status() != OB_SUCCESS)) { client_task->abort(); } ObTableLoadClientService::revert_task(client_task); @@ -433,7 +433,8 @@ int ObTableLoadService::check_support_direct_load( const uint64_t table_id, const ObDirectLoadMethod::Type method, const ObDirectLoadInsertMode::Type insert_mode, - const storage::ObDirectLoadMode::Type load_mode) + const ObDirectLoadMode::Type load_mode, + const ObIArray &column_ids) { int ret = OB_SUCCESS; if (OB_UNLIKELY(OB_INVALID_ID == table_id)) { @@ -447,7 +448,7 @@ int ObTableLoadService::check_support_direct_load( ObTableLoadSchema::get_table_schema(tenant_id, table_id, schema_guard, table_schema))) { LOG_WARN("fail to get table schema", KR(ret), K(tenant_id), K(table_id)); } else { - ret = check_support_direct_load(schema_guard, table_schema, method, insert_mode, load_mode); + ret = check_support_direct_load(schema_guard, table_schema, method, insert_mode, load_mode, column_ids); } } return ret; @@ -457,7 +458,8 @@ int ObTableLoadService::check_support_direct_load(ObSchemaGetterGuard &schema_gu uint64_t table_id, const ObDirectLoadMethod::Type method, const ObDirectLoadInsertMode::Type insert_mode, - const storage::ObDirectLoadMode::Type load_mode) + const ObDirectLoadMode::Type load_mode, + const ObIArray &column_ids) { int ret = OB_SUCCESS; if (OB_UNLIKELY(OB_INVALID_ID == table_id)) { @@ -472,7 +474,7 @@ int ObTableLoadService::check_support_direct_load(ObSchemaGetterGuard &schema_gu ret = OB_TABLE_NOT_EXIST; LOG_WARN("table schema is null", KR(ret)); } else { - ret = check_support_direct_load(schema_guard, table_schema, method, insert_mode, load_mode); + ret = check_support_direct_load(schema_guard, table_schema, method, insert_mode, load_mode, column_ids); } } return ret; @@ -484,15 +486,17 @@ int ObTableLoadService::check_support_direct_load(ObSchemaGetterGuard &schema_gu const ObTableSchema *table_schema, const ObDirectLoadMethod::Type method, const ObDirectLoadInsertMode::Type insert_mode, - const storage::ObDirectLoadMode::Type load_mode) + const ObDirectLoadMode::Type load_mode, + const ObIArray &column_ids) { int ret = OB_SUCCESS; if (OB_UNLIKELY(nullptr == table_schema || !ObDirectLoadMethod::is_type_valid(method) || - !ObDirectLoadInsertMode::is_type_valid(insert_mode)) || - !ObDirectLoadMode::is_type_valid(load_mode)) { + !ObDirectLoadInsertMode::is_type_valid(insert_mode) || + !ObDirectLoadMode::is_type_valid(load_mode) || + column_ids.empty())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), KP(table_schema), K(method), K(insert_mode)); + LOG_WARN("invalid args", KR(ret), KP(table_schema), K(method), K(insert_mode), K(column_ids)); } else { const uint64_t tenant_id = MTL_ID(); bool trigger_enabled = false; @@ -501,6 +505,7 @@ int ObTableLoadService::check_support_direct_load(ObSchemaGetterGuard &schema_gu bool has_multivalue_index = false; bool has_invisible_column = false; bool has_unused_column = false; + bool has_roaringbitmap_column = false; // check if it is a user table const char *tmp_prefix = ObDirectLoadMode::is_insert_overwrite(load_mode) ? InsertOverwritePrefix : EmptyPrefix; @@ -574,6 +579,14 @@ int ObTableLoadService::check_support_direct_load(ObSchemaGetterGuard &schema_gu LOG_WARN("direct-load does not support table has unused column", KR(ret)); FORWARD_USER_ERROR_MSG(ret, "%sdirect-load does not support table has unused column", tmp_prefix); } + // check has roaringbitmap column + else if (OB_FAIL(ObTableLoadSchema::check_has_roaringbitmap_column(table_schema, has_roaringbitmap_column))) { + LOG_WARN("fail to check has roaringbitmap column", KR(ret)); + } else if (has_roaringbitmap_column) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("direct-load does not support table has roaringbitmap column", KR(ret)); + FORWARD_USER_ERROR_MSG(ret, "%sdirect-load does not support table has roaringbitmap column", tmp_prefix); + } // check if table has mlog else if (table_schema->has_mlog_table()) { ret = OB_NOT_SUPPORTED; @@ -623,6 +636,56 @@ int ObTableLoadService::check_support_direct_load(ObSchemaGetterGuard &schema_gu } } } + // check default column + if (OB_SUCC(ret)) { + ObArray column_descs; + if (OB_FAIL(table_schema->get_column_ids(column_descs))) { + STORAGE_LOG(WARN, "fail to get column descs", KR(ret), KPC(table_schema)); + } else if (column_ids.count() == (table_schema->is_heap_table() ? column_descs.count() - 1 + : column_descs.count())) { + // non default column + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < column_descs.count(); ++i) { + const ObColDesc &col_desc = column_descs.at(i); + bool found_column = ObColumnSchemaV2::is_hidden_pk_column_id(col_desc.col_id_); + for (int64_t j = 0; !found_column && j < column_ids.count(); ++j) { + if (col_desc.col_id_ == column_ids.at(j)) { + found_column = true; + } + } + if (!found_column) { + const ObColumnSchemaV2 *column_schema = table_schema->get_column_schema(col_desc.col_id_); + if (OB_ISNULL(column_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null column schema", KR(ret), K(col_desc)); + } + // 自增列 + else if (column_schema->is_autoincrement() || column_schema->is_identity_column()) { + } + // 默认值是表达式 + else if (OB_UNLIKELY(lib::is_mysql_mode() && column_schema->get_cur_default_value().is_ext())) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("direct-load does not support column default value is ext", KR(ret), + KPC(column_schema), K(column_schema->get_cur_default_value())); + FORWARD_USER_ERROR_MSG(ret, "direct-load does not support column default value is ext"); + } else if (OB_UNLIKELY(lib::is_oracle_mode() && column_schema->is_default_expr_v2_column())) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("direct-load does not support column default value is expr", KR(ret), + KPC(column_schema), K(column_schema->get_cur_default_value())); + FORWARD_USER_ERROR_MSG(ret, "direct-load does not support column default value is expr"); + } + // 没有默认值, 且为NOT NULL + // 例外:枚举类型默认为第一个 + else if (OB_UNLIKELY(column_schema->is_not_null_for_write() && + column_schema->get_cur_default_value().is_null() && + !column_schema->get_meta_type().is_enum())) { + ret = OB_ERR_NO_DEFAULT_FOR_FIELD; + LOG_WARN("column doesn't have a default value", KR(ret), KPC(column_schema)); + } + } + } + } + } } return ret; } diff --git a/src/observer/table_load/ob_table_load_service.h b/src/observer/table_load/ob_table_load_service.h index 2e2c8655d..aacc14db5 100644 --- a/src/observer/table_load/ob_table_load_service.h +++ b/src/observer/table_load/ob_table_load_service.h @@ -39,18 +39,21 @@ public: static int check_support_direct_load(uint64_t table_id, const storage::ObDirectLoadMethod::Type method, const storage::ObDirectLoadInsertMode::Type insert_mode, - const storage::ObDirectLoadMode::Type load_mode); + const storage::ObDirectLoadMode::Type load_mode, + const common::ObIArray &column_ids); // 业务层指定schema_guard进行检查 static int check_support_direct_load(share::schema::ObSchemaGetterGuard &schema_guard, uint64_t table_id, const storage::ObDirectLoadMethod::Type method, const storage::ObDirectLoadInsertMode::Type insert_mode, - const storage::ObDirectLoadMode::Type load_mode); + const storage::ObDirectLoadMode::Type load_mode, + const common::ObIArray &column_ids); static int check_support_direct_load(share::schema::ObSchemaGetterGuard &schema_guard, const share::schema::ObTableSchema *table_schema, const storage::ObDirectLoadMethod::Type method, const storage::ObDirectLoadInsertMode::Type insert_mode, - const storage::ObDirectLoadMode::Type load_mode); + const storage::ObDirectLoadMode::Type load_mode, + const common::ObIArray &column_ids); static ObTableLoadTableCtx *alloc_ctx(); static void free_ctx(ObTableLoadTableCtx *table_ctx); static int add_ctx(ObTableLoadTableCtx *table_ctx); diff --git a/src/observer/table_load/ob_table_load_store_ctx.cpp b/src/observer/table_load/ob_table_load_store_ctx.cpp index 7ae20591e..0a983b786 100644 --- a/src/observer/table_load/ob_table_load_store_ctx.cpp +++ b/src/observer/table_load/ob_table_load_store_ctx.cpp @@ -103,7 +103,9 @@ int ObTableLoadStoreCtx::init( if (OB_SUCC(ret)) { table_data_desc_.rowkey_column_num_ = (!ctx_->schema_.is_heap_table_ ? ctx_->schema_.rowkey_column_count_ : 0); - table_data_desc_.column_count_ = ctx_->param_.column_count_; + table_data_desc_.column_count_ = + (!ctx_->schema_.is_heap_table_ ? ctx_->schema_.store_column_count_ + : ctx_->schema_.store_column_count_ - 1); table_data_desc_.external_data_block_size_ = ObDirectLoadDataBlock::DEFAULT_DATA_BLOCK_SIZE; table_data_desc_.sstable_index_block_size_ = ObDirectLoadSSTableIndexBlock::DEFAULT_INDEX_BLOCK_SIZE; diff --git a/src/observer/table_load/ob_table_load_table_ctx.cpp b/src/observer/table_load/ob_table_load_table_ctx.cpp index 96a90540f..3e677ea04 100644 --- a/src/observer/table_load/ob_table_load_table_ctx.cpp +++ b/src/observer/table_load/ob_table_load_table_ctx.cpp @@ -69,11 +69,6 @@ int ObTableLoadTableCtx::init(const ObTableLoadParam ¶m, const ObTableLoadDD if (OB_FAIL(schema_.init(param_.tenant_id_, param_.table_id_))) { LOG_WARN("fail to init table load schema", KR(ret), K(param_.tenant_id_), K(param_.table_id_)); - } else if (OB_UNLIKELY(param.column_count_ != (schema_.is_heap_table_ - ? (schema_.store_column_count_ - 1) - : schema_.store_column_count_))) { - ret = OB_SCHEMA_NOT_UPTODATE; - LOG_WARN("unexpected column count", KR(ret), K(param.column_count_), K(schema_.store_column_count_), K(schema_.is_heap_table_)); } else if (OB_FAIL(task_allocator_.init("TLD_TaskPool", param_.tenant_id_))) { LOG_WARN("fail to init allocator", KR(ret)); } else if (OB_FAIL(trans_ctx_allocator_.init("TLD_TCtxPool", param_.tenant_id_))) { diff --git a/src/observer/table_load/ob_table_load_trans_bucket_writer.cpp b/src/observer/table_load/ob_table_load_trans_bucket_writer.cpp index 4960a5873..076b57879 100644 --- a/src/observer/table_load/ob_table_load_trans_bucket_writer.cpp +++ b/src/observer/table_load/ob_table_load_trans_bucket_writer.cpp @@ -67,6 +67,7 @@ ObTableLoadTransBucketWriter::ObTableLoadTransBucketWriter(ObTableLoadTransCtx * param_(trans_ctx_->ctx_->param_), allocator_("TLD_TBWriter"), is_partitioned_(false), + column_count_(0), cast_mode_(CM_NONE), session_ctx_array_(nullptr), ref_count_(0), @@ -98,7 +99,10 @@ int ObTableLoadTransBucketWriter::init() ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null coordinator ctx", KR(ret)); } else { - is_partitioned_ = coordinator_ctx_->ctx_->schema_.is_partitioned_table_; + const ObTableLoadSchema &schema = coordinator_ctx_->ctx_->schema_; + is_partitioned_ = schema.is_partitioned_table_; + column_count_ = + (!schema.is_heap_table_ ? schema.store_column_count_ : schema.store_column_count_ - 1); if (OB_FAIL(ObSQLUtils::get_default_cast_mode(coordinator_ctx_->ctx_->session_info_, cast_mode_))) { LOG_WARN("fail to get_default_cast_mode", KR(ret)); } else if (OB_FAIL(init_session_ctx_array())) { @@ -229,19 +233,27 @@ int ObTableLoadTransBucketWriter::handle_partition_with_autoinc_identity( for (int64_t j = 0; OB_SUCC(ret) && j < row_count; ++j) { ObStorageDatum storage_datum; ObTableLoadObjRow &obj_row = obj_rows.at(j); - out_obj.set_null(); const ObTableLoadPartitionCalc::IndexAndType &index_and_type = coordinator_ctx_->partition_calc_.part_key_obj_index_.at( coordinator_ctx_->partition_calc_.partition_with_autoinc_idx_); const ObColumnSchemaV2 *column_schema = index_and_type.column_schema_; const int64_t obj_index = index_and_type.index_; - if (OB_UNLIKELY(obj_index >= param_.column_count_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid length", KR(ret), K(obj_index), K(param_.column_count_)); - } else if (!obj_row.cells_[obj_index].is_null() && - OB_FAIL(ObTableLoadObjCaster::cast_obj(cast_obj_ctx, index_and_type.column_schema_, - obj_row.cells_[obj_index], out_obj))) { + const ObObj &obj = obj_row.cells_[obj_index]; + if (OB_UNLIKELY(obj_row.count_ != column_count_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected column count not match", KR(ret), K(obj_row), K(column_count_)); + } else if (OB_UNLIKELY(obj_index < 0 || obj_index >= column_count_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected obj index", KR(ret), K(index_and_type), K(column_count_)); + } else if (obj.is_null() || obj.is_nop_value()) { + out_obj = obj; + } else if (OB_FAIL(ObTableLoadObjCaster::cast_obj(cast_obj_ctx, + column_schema, + obj, + out_obj))) { LOG_WARN("fail to cast obj", KR(ret)); + } + if (OB_FAIL(ret)) { } else if (OB_FAIL(storage_datum.from_obj_enhance(out_obj))) { LOG_WARN("fail to from obj enhance", KR(ret), K(out_obj)); } else if (column_schema->is_autoincrement() && @@ -283,19 +295,22 @@ int ObTableLoadTransBucketWriter::handle_identity_column(const ObColumnSchemaV2 ObArenaAllocator &cast_allocator) { int ret = OB_SUCCESS; - if (column_schema->is_always_identity_column()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("direct-load does not support always identity column", KR(ret)); - FORWARD_USER_ERROR_MSG(ret, "direct-load does not support always identity column"); - } else if (column_schema->is_default_identity_column() && datum.is_null()) { - ret = OB_ERR_INVALID_NOT_NULL_CONSTRAINT_ON_IDENTITY_COLUMN; - LOG_WARN("default identity column has null value", KR(ret)); - } else if (column_schema->is_default_on_null_identity_column()) { + // 1. generated always as identity : 不能指定此列导入 + // 2. generated by default as identity : 不指定时自动生成, 不能导入null + // 3. generated by default on null as identity : 不指定或者指定null会自动生成 + if (OB_UNLIKELY(column_schema->is_always_identity_column() && !datum.is_nop())) { + ret = OB_ERR_INSERT_INTO_GENERATED_ALWAYS_IDENTITY_COLUMN; + LOG_USER_ERROR(OB_ERR_INSERT_INTO_GENERATED_ALWAYS_IDENTITY_COLUMN); + } else if (OB_UNLIKELY(column_schema->is_default_identity_column() && datum.is_null())) { + ret = OB_BAD_NULL_ERROR; + LOG_WARN("default identity column cannot insert null", KR(ret)); + } else if (datum.is_nop() || datum.is_null()) { ObSequenceValue seq_value; - if (OB_FAIL(share::ObSequenceCache::get_instance().nextval(coordinator_ctx_->sequence_schema_, - cast_allocator, seq_value))) { + if (OB_FAIL(ObSequenceCache::get_instance().nextval(coordinator_ctx_->sequence_schema_, + cast_allocator, + seq_value))) { LOG_WARN("fail get nextval for seq", KR(ret)); - } else if (datum.is_null()) { + } else { datum.set_number(seq_value.val()); } } @@ -311,9 +326,9 @@ int ObTableLoadTransBucketWriter::write_for_non_partitioned(SessionContext &sess for (int64_t i = 0; OB_SUCC(ret) && i < row_count; ++i) { const ObTableLoadObjRow &row = obj_rows.at(i); bool need_write = false; - if (OB_UNLIKELY(row.count_ != param_.column_count_)) { + if (OB_UNLIKELY(row.count_ != column_count_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected column count not match", KR(ret), K(row.count_), K(param_.column_count_)); + LOG_WARN("unexpected column count not match", KR(ret), K(row), K(column_count_)); } else if (OB_FAIL(load_bucket->add_row(session_ctx.partition_id_.tablet_id_, row, param_.batch_size_, @@ -346,9 +361,9 @@ int ObTableLoadTransBucketWriter::write_for_partitioned(SessionContext &session_ ObNewRow part_key; part_key.count_ = part_key_obj_count; part_key.cells_ = static_cast(allocator.alloc(sizeof(ObObj) * part_key_obj_count)); - if (OB_UNLIKELY(row.count_ != param_.column_count_)) { + if (OB_UNLIKELY(row.count_ != column_count_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected column count not match", KR(ret), K(row.count_), K(param_.column_count_)); + LOG_WARN("unexpected column count not match", KR(ret), K(row), K(column_count_)); } else if (OB_ISNULL(part_key.cells_)) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to alloc memory", KR(ret)); diff --git a/src/observer/table_load/ob_table_load_trans_bucket_writer.h b/src/observer/table_load/ob_table_load_trans_bucket_writer.h index 52fb99021..86123cc14 100644 --- a/src/observer/table_load/ob_table_load_trans_bucket_writer.h +++ b/src/observer/table_load/ob_table_load_trans_bucket_writer.h @@ -70,6 +70,7 @@ private: const ObTableLoadParam ¶m_; common::ObArenaAllocator allocator_; bool is_partitioned_; + int64_t column_count_; common::ObCastMode cast_mode_; struct SessionContext { diff --git a/src/observer/table_load/ob_table_load_trans_store.cpp b/src/observer/table_load/ob_table_load_trans_store.cpp index c378854e3..80ca5f24e 100644 --- a/src/observer/table_load/ob_table_load_trans_store.cpp +++ b/src/observer/table_load/ob_table_load_trans_store.cpp @@ -439,20 +439,7 @@ int ObTableLoadTransStoreWriter::cast_column( (obj.is_null() || obj.is_nop_value()); ObObj out_obj; if (is_null_autoinc) { - out_obj.set_null(); - } else if (obj.is_nop_value()) { - if (column_schema->is_not_null_for_write() && - column_schema->get_cur_default_value().is_null()) { - if (column_schema->get_meta_type().is_enum()) { - const uint64_t ENUM_FIRST_VAL = 1; - out_obj.set_enum(ENUM_FIRST_VAL); - } else { - ret = OB_ERR_NO_DEFAULT_FOR_FIELD; - LOG_WARN("column can not be null", KR(ret), KPC(column_schema)); - } - } else { - out_obj = column_schema->get_cur_default_value(); - } + out_obj = obj; } else if (OB_FAIL(ObTableLoadObjCaster::cast_obj(cast_obj_ctx, column_schema, obj, out_obj))) { LOG_WARN("fail to cast obj and check", KR(ret), K(obj)); } @@ -491,19 +478,22 @@ int ObTableLoadTransStoreWriter::handle_identity_column(const ObColumnSchemaV2 * ObArenaAllocator &cast_allocator) { int ret = OB_SUCCESS; - if (column_schema->is_always_identity_column()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("direct-load does not support always identity column", KR(ret)); - FORWARD_USER_ERROR_MSG(ret, "direct-load does not support always identity column"); - } else if (column_schema->is_default_identity_column() && datum.is_null()) { - ret = OB_ERR_INVALID_NOT_NULL_CONSTRAINT_ON_IDENTITY_COLUMN; - LOG_WARN("default identity column has null value", KR(ret)); - } else if (column_schema->is_default_on_null_identity_column()) { + // 1. generated always as identity : 不能指定此列导入 + // 2. generated by default as identity : 不指定时自动生成, 不能导入null + // 3. generated by default on null as identity : 不指定或者指定null会自动生成 + if (OB_UNLIKELY(column_schema->is_always_identity_column() && !datum.is_nop())) { + ret = OB_ERR_INSERT_INTO_GENERATED_ALWAYS_IDENTITY_COLUMN; + LOG_USER_ERROR(OB_ERR_INSERT_INTO_GENERATED_ALWAYS_IDENTITY_COLUMN); + } else if (OB_UNLIKELY(column_schema->is_default_identity_column() && datum.is_null())) { + ret = OB_BAD_NULL_ERROR; + LOG_WARN("default identity column cannot insert null", KR(ret)); + } else if (datum.is_nop() || datum.is_null()) { ObSequenceValue seq_value; - if (OB_FAIL(share::ObSequenceCache::get_instance().nextval( - trans_ctx_->ctx_->store_ctx_->sequence_schema_, cast_allocator, seq_value))) { + if (OB_FAIL(ObSequenceCache::get_instance().nextval(trans_ctx_->ctx_->store_ctx_->sequence_schema_, + cast_allocator, + seq_value))) { LOG_WARN("fail get nextval for seq", KR(ret)); - } else if (datum.is_null()) { + } else { datum.set_number(seq_value.val()); } } diff --git a/src/share/table/ob_table_load_row.h b/src/share/table/ob_table_load_row.h index ddca73e1d..b5b78a862 100644 --- a/src/share/table/ob_table_load_row.h +++ b/src/share/table/ob_table_load_row.h @@ -152,19 +152,18 @@ template int ObTableLoadRow::project(const ObIArray &idx_projector, ObTableLoadRow &projected_row) const { int ret = OB_SUCCESS; - if (OB_UNLIKELY(idx_projector.count() != count_)) { - ret = OB_ERR_UNEXPECTED; - OB_LOG(WARN, "unexpected count", KR(ret), K(idx_projector), K(count_)); - } else if (OB_FAIL(projected_row.init(count_, allocator_handle_))) { - OB_LOG(WARN, "failed to alloate cells", KR(ret), K(projected_row.count_)); + if (OB_FAIL(projected_row.init(idx_projector.count(), allocator_handle_))) { + OB_LOG(WARN, "failed to alloate cells", KR(ret), K(idx_projector.count())); } else { - for (int64_t j = 0; j < count_; ++j) { - const int64_t idx = idx_projector.at(j); - if (OB_UNLIKELY(idx < 0 || idx >= count_)) { + for (int64_t i = 0; i < idx_projector.count(); ++i) { + const int64_t idx = idx_projector.at(i); + if (idx < 0) { + projected_row.cells_[i].set_nop_value(); + } else if (OB_UNLIKELY(idx >= count_)) { ret = OB_ERR_UNEXPECTED; - OB_LOG(WARN, "unexpected idx", KR(ret), K(j), K(idx), K(idx_projector)); + OB_LOG(WARN, "unexpected idx", KR(ret), K(i), K(idx), K(idx_projector), K(count_)); } else { - projected_row.cells_[j] = cells_[idx]; + projected_row.cells_[i] = cells_[idx]; } } projected_row.seq_no_ = seq_no_; diff --git a/src/sql/engine/cmd/ob_load_data_direct_impl.cpp b/src/sql/engine/cmd/ob_load_data_direct_impl.cpp index 751c56999..2b9195534 100644 --- a/src/sql/engine/cmd/ob_load_data_direct_impl.cpp +++ b/src/sql/engine/cmd/ob_load_data_direct_impl.cpp @@ -2218,7 +2218,8 @@ int ObLoadDataDirectImpl::execute(ObExecContext &ctx, ObLoadDataStmt &load_stmt) execute_param_.table_id_, execute_param_.method_, execute_param_.insert_mode_, - ObDirectLoadMode::LOAD_DATA))) { + ObDirectLoadMode::LOAD_DATA, + execute_param_.column_ids_))) { LOG_WARN("fail to check support direct load", KR(ret)); } else if (OB_FAIL(init_execute_context())) { LOG_WARN("fail to init execute context", KR(ret), K(ctx), K(load_stmt)); @@ -2425,31 +2426,55 @@ int ObLoadDataDirectImpl::init_execute_param() ObSchemaGetterGuard *schema_guard = ctx_->get_sql_ctx()->schema_guard_; int64_t column_count = 0; execute_param_.column_ids_.reset(); - if (is_backup) { + if (is_backup) { // 备份数据导入 if (OB_FAIL(ObTableLoadSchema::get_column_ids(*schema_guard, - execute_param_.tenant_id_, - execute_param_.table_id_, - execute_param_.column_ids_))) { - LOG_WARN("fail to get column ids for backup", KR(ret)); + execute_param_.tenant_id_, + execute_param_.table_id_, + execute_param_.column_ids_))) { + LOG_WARN("fail to get column ids for backup", KR(ret)); } - } else { - if (OB_FAIL(ObTableLoadSchema::get_user_column_count(*schema_guard, - execute_param_.tenant_id_, - execute_param_.table_id_, - column_count))) { - LOG_WARN("fail to get user column count", KR(ret)); - } else if (OB_UNLIKELY(column_count != field_or_var_list.count())) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("not contain all columns is not supported", KR(ret), K(column_count), - K(field_or_var_list)); + } else if (load_stmt_->get_default_table_columns()) { // 默认列导入 + if (OB_FAIL(ObTableLoadSchema::get_user_column_ids(*schema_guard, + execute_param_.tenant_id_, + execute_param_.table_id_, + execute_param_.column_ids_))) { + LOG_WARN("fail to get user column ids", KR(ret)); + } + } else { // 指定列导入 + const static uint64_t INVALID_COLUMN_ID = UINT64_MAX; + const ObTableSchema *table_schema = nullptr; + ObArray user_column_ids; + if (OB_FAIL(ObTableLoadSchema::get_table_schema(*schema_guard, + execute_param_.tenant_id_, + execute_param_.table_id_, + table_schema))) { + LOG_WARN("fail to get table schema", KR(ret), K(execute_param_)); + } else if (OB_FAIL(ObTableLoadSchema::get_user_column_ids(table_schema, user_column_ids))) { + LOG_WARN("fail to get user column ids", KR(ret)); } for (int64_t i = 0; OB_SUCC(ret) && i < field_or_var_list.count(); ++i) { const ObLoadDataStmt::FieldOrVarStruct &field_or_var_struct = field_or_var_list.at(i); if (OB_UNLIKELY(!field_or_var_struct.is_table_column_)) { ret = OB_NOT_SUPPORTED; - LOG_WARN("var is not supported", KR(ret), K(field_or_var_struct), K(i), K(field_or_var_list)); - } else if (OB_FAIL(execute_param_.column_ids_.push_back(field_or_var_struct.column_id_))) { - LOG_WARN("fail to push back column id", KR(ret)); + LOG_WARN("var is not supported", KR(ret), K(field_or_var_struct), K(i), + K(field_or_var_list)); + } else { + const uint64_t column_id = field_or_var_struct.column_id_; + int64_t found_column_idx = -1; + for (int64_t j = 0; found_column_idx == -1 && j < user_column_ids.count(); ++j) { + const uint64_t user_column_id = user_column_ids.at(j); + if (column_id == user_column_id) { + found_column_idx = j; + } + } + if (OB_UNLIKELY(found_column_idx == -1)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unknow column", KR(ret), K(user_column_ids), K(field_or_var_struct)); + } else if (OB_FAIL(execute_param_.column_ids_.push_back(column_id))) { + LOG_WARN("fail to push back column id", KR(ret)); + } else { + user_column_ids.at(found_column_idx) = INVALID_COLUMN_ID; + } } } } diff --git a/src/sql/engine/cmd/ob_table_direct_insert_ctx.cpp b/src/sql/engine/cmd/ob_table_direct_insert_ctx.cpp index a32fcb0c0..0d10ceeb0 100644 --- a/src/sql/engine/cmd/ob_table_direct_insert_ctx.cpp +++ b/src/sql/engine/cmd/ob_table_direct_insert_ctx.cpp @@ -79,8 +79,8 @@ int ObTableDirectInsertCtx::init( LOG_WARN("fail to new ObTableLoadInstance", KR(ret)); } else { load_exec_ctx_->exec_ctx_ = exec_ctx; + const ObTableSchema *table_schema = nullptr; ObArray column_ids; - omt::ObTenant *tenant = nullptr; ObCompressorType compressor_type = ObCompressorType::NONE_COMPRESSOR; ObDirectLoadMethod::Type method = (is_incremental ? ObDirectLoadMethod::INCREMENTAL : ObDirectLoadMethod::FULL); ObDirectLoadInsertMode::Type insert_mode = ObDirectLoadInsertMode::INVALID_INSERT_MODE; @@ -93,23 +93,21 @@ int ObTableDirectInsertCtx::init( } ObDirectLoadMode::Type load_mode = is_insert_overwrite ? ObDirectLoadMode::INSERT_OVERWRITE : ObDirectLoadMode::INSERT_INTO; bool is_heap_table = false; - if (OB_FAIL(GCTX.omt_->get_tenant(MTL_ID(), tenant))) { - LOG_WARN("fail to get tenant handle", KR(ret), K(MTL_ID())); + if (OB_FAIL(ObTableLoadSchema::get_table_schema(*schema_guard, tenant_id, table_id, table_schema))) { + LOG_WARN("fail to get table schema", KR(ret)); + } else if (OB_FAIL(ObDDLUtil::get_temp_store_compress_type(table_schema->get_compressor_type(), + parallel, + compressor_type))) { + LOG_WARN("fail to get tmp store compressor type", KR(ret)); + } else if (OB_FAIL(ObTableLoadSchema::get_column_ids(table_schema, column_ids))) { + LOG_WARN("failed to init store column idxs", KR(ret)); } else if (OB_FAIL(ObTableLoadService::check_support_direct_load(*schema_guard, - table_id, + table_schema, method, insert_mode, - load_mode))) { + load_mode, + column_ids))) { LOG_WARN("fail to check support direct load", KR(ret)); - } else if (OB_FAIL(get_compressor_type(MTL_ID(), table_id, parallel, compressor_type))) { - LOG_WARN("fail to get compressor type", KR(ret)); - } else if (OB_FAIL(ObTableLoadSchema::get_column_ids(*schema_guard, - tenant_id, - table_id, - column_ids))) { - LOG_WARN("failed to init store column idxs", KR(ret)); - } else if(OB_FAIL(get_is_heap_table(*schema_guard, tenant_id, table_id, is_heap_table))) { - LOG_WARN("failed to get is heap table", KR(ret), K(tenant_id), K(table_id)); } else { ObTableLoadParam param; param.tenant_id_ = MTL_ID(); @@ -120,7 +118,7 @@ int ObTableDirectInsertCtx::init( param.column_count_ = column_ids.count(); param.px_mode_ = true; param.online_opt_stat_gather_ = is_online_gather_statistics_; - param.need_sort_ = is_heap_table ? phy_plan.get_direct_load_need_sort() : true; + param.need_sort_ = table_schema->is_heap_table() ? phy_plan.get_direct_load_need_sort() : true; param.max_error_row_count_ = 0; param.dup_action_ = (enable_inc_replace ? sql::ObLoadDupActionType::LOAD_REPLACE : sql::ObLoadDupActionType::LOAD_STOP_ON_DUP); @@ -185,46 +183,5 @@ void ObTableDirectInsertCtx::destroy() is_online_gather_statistics_ = false; } -int ObTableDirectInsertCtx::get_compressor_type(const uint64_t tenant_id, - const uint64_t table_id, - const int64_t parallel, - ObCompressorType &compressor_type) -{ - int ret = OB_SUCCESS; - ObSchemaGetterGuard schema_guard; - const ObTableSchema *table_schema = nullptr; - if (OB_FAIL(ObMultiVersionSchemaService::get_instance().get_tenant_schema_guard(tenant_id, - schema_guard))) { - LOG_WARN("fail to get tenant schema guard", KR(ret), K(tenant_id)); - } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, table_id, table_schema))) { - LOG_WARN("fail to get table schema", KR(ret), K(tenant_id), K(table_id)); - } else if (OB_ISNULL(table_schema)) { - ret = OB_TABLE_NOT_EXIST; - LOG_WARN("table not exist", KR(ret), K(tenant_id), K(table_id)); - } else if (OB_FAIL(ObDDLUtil::get_temp_store_compress_type(table_schema->get_compressor_type(), - parallel, compressor_type))) { - LOG_WARN("fail to get tmp store compressor type", KR(ret)); - } - return ret; -} - -int ObTableDirectInsertCtx::get_is_heap_table( - ObSchemaGetterGuard &schema_guard, - const uint64_t tenant_id, - const uint64_t table_id, - bool &is_heap_table) -{ - int ret = OB_SUCCESS; - const ObTableSchema *table_schema = nullptr; - if (OB_FAIL(schema_guard.get_table_schema(tenant_id, table_id, table_schema))) { - LOG_WARN("fail to get table schema", KR(ret), K(tenant_id), K(table_id)); - } else if (OB_ISNULL(table_schema)) { - ret = OB_TABLE_NOT_EXIST; - LOG_WARN("table schema is null", KR(ret)); - } else { - is_heap_table = table_schema->is_heap_table(); - } - return ret; -} } // namespace sql } // namespace oceanbase diff --git a/src/sql/engine/cmd/ob_table_direct_insert_ctx.h b/src/sql/engine/cmd/ob_table_direct_insert_ctx.h index d1f24912f..269ff3628 100644 --- a/src/sql/engine/cmd/ob_table_direct_insert_ctx.h +++ b/src/sql/engine/cmd/ob_table_direct_insert_ctx.h @@ -70,13 +70,6 @@ public: return online_sample_percent_; } -private: - int get_compressor_type(const uint64_t tenant_id, const uint64_t table_id, const int64_t parallel, - ObCompressorType &compressor_type); - int get_is_heap_table(share::schema::ObSchemaGetterGuard &schema_guard, - const uint64_t tenant_id, - const uint64_t table_id, - bool &is_heap_table); private: observer::ObTableLoadExecCtx *load_exec_ctx_; observer::ObTableLoadInstance *table_load_instance_; diff --git a/tools/deploy/mysql_test/test_suite/direct_load_data/data/misc/specify_column_0.csv b/tools/deploy/mysql_test/test_suite/direct_load_data/data/misc/specify_column_0.csv new file mode 100644 index 000000000..f72941a1e --- /dev/null +++ b/tools/deploy/mysql_test/test_suite/direct_load_data/data/misc/specify_column_0.csv @@ -0,0 +1,3 @@ +3,1,2 +33,11,22 +333,111,222 \ No newline at end of file From 02065bdcae93265d79865f64f737f8a5bba306c3 Mon Sep 17 00:00:00 2001 From: swjtu-wenxiang Date: Wed, 14 Aug 2024 10:14:26 +0000 Subject: [PATCH 046/249] [CP] [to #2024073000103989473]Add top_info column to processlist virtual table, and increase top-level PL SQL display. --- .../virtual_table/ob_show_processlist.cpp | 13 +++++ .../virtual_table/ob_show_processlist.h | 1 + src/pl/ob_pl.cpp | 3 +- .../ob_inner_table_schema.10001_10050.cpp | 15 +++++ .../ob_inner_table_schema.15001_15050.cpp | 15 +++++ .../ob_inner_table_schema.21201_21250.cpp | 4 +- .../ob_inner_table_schema.28101_28150.cpp | 4 +- .../inner_table/ob_inner_table_schema_def.py | 13 +++-- src/sql/session/ob_basic_session_info.cpp | 56 ++++++++++++++++++- src/sql/session/ob_basic_session_info.h | 21 +++++++ .../r/mysql/desc_sys_views_in_mysql.result | 2 + .../r/mysql/desc_sys_views_in_sys.result | 2 + .../mysql/desc_virtual_table_in_mysql.result | 1 + .../r/mysql/desc_virtual_table_in_sys.result | 1 + 14 files changed, 141 insertions(+), 10 deletions(-) diff --git a/src/observer/virtual_table/ob_show_processlist.cpp b/src/observer/virtual_table/ob_show_processlist.cpp index 0d8efdae9..0e66a7be5 100644 --- a/src/observer/virtual_table/ob_show_processlist.cpp +++ b/src/observer/virtual_table/ob_show_processlist.cpp @@ -502,6 +502,19 @@ bool ObShowProcesslist::FillScanner::operator()(sql::ObSQLSessionMgr::Key key, O cur_row_->cells_[cell_idx].set_scale(6); break; } + case TOP_INFO: { + if ((obmysql::COM_QUERY == sess_info->get_mysql_cmd() || + obmysql::COM_STMT_EXECUTE == sess_info->get_mysql_cmd() || + obmysql::COM_STMT_PREPARE == sess_info->get_mysql_cmd() || + obmysql::COM_STMT_PREXECUTE == sess_info->get_mysql_cmd()) && + !sess_info->get_top_query_string().empty()) { + cur_row_->cells_[cell_idx].set_varchar(sess_info->get_top_query_string()); + cur_row_->cells_[cell_idx].set_collation_type(default_collation); + } else { + cur_row_->cells_[cell_idx].set_null(); + } + break; + } default: { ret = OB_ERR_UNEXPECTED; SERVER_LOG(WARN, "invalid column id", K(ret), K(cell_idx), diff --git a/src/observer/virtual_table/ob_show_processlist.h b/src/observer/virtual_table/ob_show_processlist.h index 00179c475..944d0c462 100644 --- a/src/observer/virtual_table/ob_show_processlist.h +++ b/src/observer/virtual_table/ob_show_processlist.h @@ -82,6 +82,7 @@ private: PROXY_USER_NAME, SERVICE_NAME, TOTAL_CPU_TIME, + TOP_INFO }; class FillScanner { diff --git a/src/pl/ob_pl.cpp b/src/pl/ob_pl.cpp index 8ae59e9ff..e1bf0d0c4 100644 --- a/src/pl/ob_pl.cpp +++ b/src/pl/ob_pl.cpp @@ -684,7 +684,7 @@ int ObPLContext::init(ObSQLSessionInfo &session_info, OZ (ob_write_string(allocator != NULL ? *allocator : ctx.get_allocator(), session_info.get_current_query_string(), cur_query_)); - + OZ (session_info.store_top_query_string(cur_query_)); OZ (recursion_ctx_.init(session_info)); OX (session_info.set_pl_stack_ctx(this)); OX (session_info.set_pl_can_retry(true)); @@ -947,6 +947,7 @@ void ObPLContext::destory( LOG_WARN("failed to restore query string", K(ret), K(cur_query_)); ret = OB_SUCCESS == ret ? tmp_ret : ret; } + session_info.reset_top_query_string(); } // 无论如何恢复session上的状态 session_info.set_pl_stack_ctx(NULL); diff --git a/src/share/inner_table/ob_inner_table_schema.10001_10050.cpp b/src/share/inner_table/ob_inner_table_schema.10001_10050.cpp index 5cadb04de..9da14d816 100644 --- a/src/share/inner_table/ob_inner_table_schema.10001_10050.cpp +++ b/src/share/inner_table/ob_inner_table_schema.10001_10050.cpp @@ -2078,6 +2078,21 @@ int ObInnerTableSchema::all_virtual_processlist_schema(ObTableSchema &table_sche false, //is_nullable false); //is_autoincrement } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("top_info", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_COLUMN_VARCHAR_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } if (OB_SUCC(ret)) { table_schema.get_part_option().set_part_num(1); table_schema.set_part_level(PARTITION_LEVEL_ONE); diff --git a/src/share/inner_table/ob_inner_table_schema.15001_15050.cpp b/src/share/inner_table/ob_inner_table_schema.15001_15050.cpp index f0392ec4a..a21b48241 100644 --- a/src/share/inner_table/ob_inner_table_schema.15001_15050.cpp +++ b/src/share/inner_table/ob_inner_table_schema.15001_15050.cpp @@ -9093,6 +9093,21 @@ int ObInnerTableSchema::all_virtual_processlist_ora_schema(ObTableSchema &table_ false, //is_nullable false); //is_autoincrement } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TOP_INFO", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + MAX_COLUMN_VARCHAR_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } if (OB_SUCC(ret)) { table_schema.get_part_option().set_part_num(1); table_schema.set_part_level(PARTITION_LEVEL_ONE); diff --git a/src/share/inner_table/ob_inner_table_schema.21201_21250.cpp b/src/share/inner_table/ob_inner_table_schema.21201_21250.cpp index c0b674e9b..a8aa45ad0 100644 --- a/src/share/inner_table/ob_inner_table_schema.21201_21250.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21201_21250.cpp @@ -1060,7 +1060,7 @@ int ObInnerTableSchema::gv_ob_processlist_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT SVR_IP, SVR_PORT, SQL_PORT, ID, USER, HOST, DB, TENANT, COMMAND, TIME, TOTAL_TIME, STATE, INFO, PROXY_SESSID, MASTER_SESSID, USER_CLIENT_IP, USER_HOST, RETRY_CNT, RETRY_INFO, SQL_ID, TRANS_ID, THREAD_ID, SSL_CIPHER, TRACE_ID, TRANS_STATE, ACTION, MODULE, CLIENT_INFO, LEVEL, SAMPLE_PERCENTAGE, RECORD_POLICY, LB_VID, LB_VIP, LB_VPORT, IN_BYTES, OUT_BYTES, USER_CLIENT_PORT, cast(total_cpu_time as SIGNED) as TOTAL_CPU_TIME, PROXY_USER FROM oceanbase.__all_virtual_processlist )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT SVR_IP, SVR_PORT, SQL_PORT, ID, USER, HOST, DB, TENANT, COMMAND, TIME, TOTAL_TIME, STATE, INFO, PROXY_SESSID, MASTER_SESSID, USER_CLIENT_IP, USER_HOST, RETRY_CNT, RETRY_INFO, SQL_ID, TRANS_ID, THREAD_ID, SSL_CIPHER, TRACE_ID, TRANS_STATE, ACTION, MODULE, CLIENT_INFO, LEVEL, SAMPLE_PERCENTAGE, RECORD_POLICY, LB_VID, LB_VIP, LB_VPORT, IN_BYTES, OUT_BYTES, USER_CLIENT_PORT, cast(total_cpu_time as SIGNED) as TOTAL_CPU_TIME, PROXY_USER, TOP_INFO FROM oceanbase.__all_virtual_processlist )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -1110,7 +1110,7 @@ int ObInnerTableSchema::v_ob_processlist_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT SVR_IP, SVR_PORT, SQL_PORT, ID, USER, HOST, DB, TENANT, COMMAND, TIME, TOTAL_TIME, STATE, INFO, PROXY_SESSID, MASTER_SESSID, USER_CLIENT_IP, USER_HOST, RETRY_CNT, RETRY_INFO, SQL_ID, TRANS_ID, THREAD_ID, SSL_CIPHER, TRACE_ID, TRANS_STATE, ACTION, MODULE, CLIENT_INFO, LEVEL, SAMPLE_PERCENTAGE, RECORD_POLICY, LB_VID, LB_VIP, LB_VPORT, IN_BYTES, OUT_BYTES, USER_CLIENT_PORT, cast(total_cpu_time as SIGNED) as TOTAL_CPU_TIME, PROXY_USER FROM oceanbase.GV$OB_PROCESSLIST WHERE SVR_IP = host_ip() AND SVR_PORT = rpc_port() )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT SVR_IP, SVR_PORT, SQL_PORT, ID, USER, HOST, DB, TENANT, COMMAND, TIME, TOTAL_TIME, STATE, INFO, PROXY_SESSID, MASTER_SESSID, USER_CLIENT_IP, USER_HOST, RETRY_CNT, RETRY_INFO, SQL_ID, TRANS_ID, THREAD_ID, SSL_CIPHER, TRACE_ID, TRANS_STATE, ACTION, MODULE, CLIENT_INFO, LEVEL, SAMPLE_PERCENTAGE, RECORD_POLICY, LB_VID, LB_VIP, LB_VPORT, IN_BYTES, OUT_BYTES, USER_CLIENT_PORT, cast(total_cpu_time as SIGNED) as TOTAL_CPU_TIME, PROXY_USER, TOP_INFO FROM oceanbase.GV$OB_PROCESSLIST WHERE SVR_IP = host_ip() AND SVR_PORT = rpc_port() )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema.28101_28150.cpp b/src/share/inner_table/ob_inner_table_schema.28101_28150.cpp index 8c01b9255..fd2f22643 100644 --- a/src/share/inner_table/ob_inner_table_schema.28101_28150.cpp +++ b/src/share/inner_table/ob_inner_table_schema.28101_28150.cpp @@ -960,7 +960,7 @@ int ObInnerTableSchema::gv_ob_processlist_ora_schema(ObTableSchema &table_schema table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT SVR_IP, SVR_PORT, SQL_PORT, ID, "USER", HOST, DB, TENANT, COMMAND, TIME, TOTAL_TIME, STATE, INFO, PROXY_SESSID, MASTER_SESSID, USER_CLIENT_IP, USER_HOST, RETRY_CNT, RETRY_INFO, SQL_ID, TRANS_ID, THREAD_ID, SSL_CIPHER, TRACE_ID, TRANS_STATE, ACTION, MODULE, CLIENT_INFO, "LEVEL", SAMPLE_PERCENTAGE, RECORD_POLICY, LB_VID, LB_VIP, LB_VPORT, IN_BYTES, OUT_BYTES, USER_CLIENT_PORT, PROXY_USER, CAST(total_cpu_time AS INT) as TOTAL_CPU_TIME FROM SYS.ALL_VIRTUAL_PROCESSLIST )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT SVR_IP, SVR_PORT, SQL_PORT, ID, "USER", HOST, DB, TENANT, COMMAND, TIME, TOTAL_TIME, STATE, INFO, PROXY_SESSID, MASTER_SESSID, USER_CLIENT_IP, USER_HOST, RETRY_CNT, RETRY_INFO, SQL_ID, TRANS_ID, THREAD_ID, SSL_CIPHER, TRACE_ID, TRANS_STATE, ACTION, MODULE, CLIENT_INFO, "LEVEL", SAMPLE_PERCENTAGE, RECORD_POLICY, LB_VID, LB_VIP, LB_VPORT, IN_BYTES, OUT_BYTES, USER_CLIENT_PORT, PROXY_USER, CAST(total_cpu_time AS INT) as TOTAL_CPU_TIME, TOP_INFO FROM SYS.ALL_VIRTUAL_PROCESSLIST )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -1010,7 +1010,7 @@ int ObInnerTableSchema::v_ob_processlist_ora_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT SVR_IP, SVR_PORT, SQL_PORT, ID, "USER", HOST, DB, TENANT, COMMAND, TIME, TOTAL_TIME, STATE, INFO, PROXY_SESSID, MASTER_SESSID, USER_CLIENT_IP, USER_HOST, RETRY_CNT, RETRY_INFO, SQL_ID, TRANS_ID, THREAD_ID, SSL_CIPHER, TRACE_ID, TRANS_STATE, ACTION, MODULE, CLIENT_INFO, "LEVEL", SAMPLE_PERCENTAGE, RECORD_POLICY, LB_VID, LB_VIP, LB_VPORT, IN_BYTES, OUT_BYTES, USER_CLIENT_PORT, PROXY_USER, CAST(total_cpu_time AS INT) as TOTAL_CPU_TIME FROM SYS.GV$OB_PROCESSLIST WHERE SVR_IP = host_ip() AND SVR_PORT = rpc_port() )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT SVR_IP, SVR_PORT, SQL_PORT, ID, "USER", HOST, DB, TENANT, COMMAND, TIME, TOTAL_TIME, STATE, INFO, PROXY_SESSID, MASTER_SESSID, USER_CLIENT_IP, USER_HOST, RETRY_CNT, RETRY_INFO, SQL_ID, TRANS_ID, THREAD_ID, SSL_CIPHER, TRACE_ID, TRANS_STATE, ACTION, MODULE, CLIENT_INFO, "LEVEL", SAMPLE_PERCENTAGE, RECORD_POLICY, LB_VID, LB_VIP, LB_VPORT, IN_BYTES, OUT_BYTES, USER_CLIENT_PORT, PROXY_USER, CAST(total_cpu_time AS INT) as TOTAL_CPU_TIME, TOP_INFO FROM SYS.GV$OB_PROCESSLIST WHERE SVR_IP = host_ip() AND SVR_PORT = rpc_port() )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index 786ba625f..d0599968f 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -7608,6 +7608,7 @@ def_table_schema( ('proxy_user', 'varchar:OB_MAX_USER_NAME_LENGTH_STORE', 'true'), ('service_name', 'varchar:64', 'true'), ('total_cpu_time', 'double', 'false'), + ('top_info', 'varchar:MAX_COLUMN_VARCHAR_LENGTH', 'true'), ], partition_columns = ['svr_ip', 'svr_port'], vtable_route_policy = 'distributed', @@ -24031,7 +24032,8 @@ SELECT OUT_BYTES, USER_CLIENT_PORT, cast(total_cpu_time as SIGNED) as TOTAL_CPU_TIME, - PROXY_USER + PROXY_USER, + TOP_INFO FROM oceanbase.__all_virtual_processlist """.replace("\n", " ") ) @@ -24082,7 +24084,8 @@ def_table_schema( OUT_BYTES, USER_CLIENT_PORT, cast(total_cpu_time as SIGNED) as TOTAL_CPU_TIME, - PROXY_USER + PROXY_USER, + TOP_INFO FROM oceanbase.GV$OB_PROCESSLIST WHERE SVR_IP = host_ip() AND SVR_PORT = rpc_port() """.replace("\n", " ") @@ -60811,7 +60814,8 @@ SELECT OUT_BYTES, USER_CLIENT_PORT, PROXY_USER, - CAST(total_cpu_time AS INT) as TOTAL_CPU_TIME + CAST(total_cpu_time AS INT) as TOTAL_CPU_TIME, + TOP_INFO FROM SYS.ALL_VIRTUAL_PROCESSLIST """.replace("\n", " ") ) @@ -60864,7 +60868,8 @@ def_table_schema( OUT_BYTES, USER_CLIENT_PORT, PROXY_USER, - CAST(total_cpu_time AS INT) as TOTAL_CPU_TIME + CAST(total_cpu_time AS INT) as TOTAL_CPU_TIME, + TOP_INFO FROM SYS.GV$OB_PROCESSLIST WHERE SVR_IP = host_ip() AND SVR_PORT = rpc_port() """.replace("\n", " ") diff --git a/src/sql/session/ob_basic_session_info.cpp b/src/sql/session/ob_basic_session_info.cpp index 319b69ce6..7aab1c45d 100644 --- a/src/sql/session/ob_basic_session_info.cpp +++ b/src/sql/session/ob_basic_session_info.cpp @@ -293,6 +293,11 @@ void ObBasicSessionInfo::destroy() thread_data_.cur_query_ = nullptr; thread_data_.cur_query_buf_len_ = 0; } + if (thread_data_.top_query_ != nullptr) { + ob_free(thread_data_.top_query_); + thread_data_.top_query_ = nullptr; + thread_data_.top_query_buf_len_ = 0; + } total_stmt_tables_.reset(); cur_stmt_tables_.reset(); #ifdef OB_BUILD_ORACLE_PL @@ -320,6 +325,7 @@ void ObBasicSessionInfo::clean_status() set_valid(true); thread_data_.cur_query_start_time_ = 0; thread_data_.cur_query_len_ = 0; + thread_data_.top_query_len_ = 0; thread_data_.last_active_time_ = ObTimeUtility::current_time(); reset_session_changed_info(); } @@ -5918,7 +5924,47 @@ bool ObBasicSessionInfo::get_tx_read_only() const int ObBasicSessionInfo::store_query_string(const ObString &stmt) { LockGuard lock_guard(thread_data_mutex_); - return store_query_string_(stmt); + return store_query_string_(stmt, thread_data_.cur_query_buf_len_, thread_data_.cur_query_, thread_data_.cur_query_len_); +} + +int ObBasicSessionInfo::store_top_query_string(const ObString &stmt) +{ + LockGuard lock_guard(thread_data_mutex_); + return store_query_string_(stmt, thread_data_.top_query_buf_len_, thread_data_.top_query_, thread_data_.top_query_len_); +} + +int ObBasicSessionInfo::store_query_string_(const ObString &stmt, int64_t& buf_len, char *& query, volatile int64_t& query_len) +{ + int ret = OB_SUCCESS; + int64_t truncated_len = std::min(MAX_QUERY_STRING_LEN - 1, + static_cast(stmt.length())); + if (truncated_len < 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid str length", K(ret), K(truncated_len)); + } else if (buf_len - 1 < truncated_len) { + if (query != nullptr) { + ob_free(query); + query = NULL; + buf_len = 0; + } + int64_t len = MAX(MIN_CUR_QUERY_LEN, truncated_len + 1); + char *buf = reinterpret_cast(ob_malloc(len, ObMemAttr(orig_tenant_id_, + ObModIds::OB_SQL_SESSION_QUERY_SQL))); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc memory failed", K(ret)); + } else { + query = buf; + buf_len = len; + } + } + if (OB_SUCC(ret)) { + MEMCPY(query, stmt.ptr(), truncated_len); + //char query[MAX_QUERY_STRING_LEN] 不存在越界风险,且不需要判空 + query[truncated_len] = '\0'; + query_len = truncated_len; + } + return ret; } int ObBasicSessionInfo::store_query_string_(const ObString &stmt) @@ -5966,6 +6012,14 @@ void ObBasicSessionInfo::reset_query_string() thread_data_.cur_query_len_ = 0; } +void ObBasicSessionInfo::reset_top_query_string() +{ + if (thread_data_.top_query_ != nullptr) { + thread_data_.top_query_[0] = '\0'; + thread_data_.top_query_len_ = 0; + } +} + int ObBasicSessionInfo::update_session_timeout() { int ret = OB_SUCCESS; diff --git a/src/sql/session/ob_basic_session_info.h b/src/sql/session/ob_basic_session_info.h index 762c78c7b..26e96d940 100644 --- a/src/sql/session/ob_basic_session_info.h +++ b/src/sql/session/ob_basic_session_info.h @@ -892,7 +892,9 @@ public: obmysql::ObMySQLCmd get_mysql_cmd() const { return thread_data_.mysql_cmd_; } char const *get_mysql_cmd_str() const { return obmysql::get_mysql_cmd_str(thread_data_.mysql_cmd_); } int store_query_string(const common::ObString &stmt); + int store_top_query_string(const common::ObString &stmt); void reset_query_string(); + void reset_top_query_string(); void set_session_sleep(); // for SQL entry point int set_session_active(const ObString &sql, @@ -903,6 +905,7 @@ public: int set_session_active(const ObString &label, obmysql::ObMySQLCmd cmd); const common::ObString get_current_query_string() const; + const common::ObString get_top_query_string() const; uint64_t get_current_statement_id() const { return thread_data_.cur_statement_id_; } int update_session_timeout(); int is_timeout(bool &is_timeout); @@ -1515,6 +1518,7 @@ private: int deep_copy_trace_id_var(const common::ObObj &src_val, common::ObObj *dest_val_ptr); inline int store_query_string_(const ObString &stmt); + inline int store_query_string_(const ObString &stmt, int64_t& buf_len, char *& query, volatile int64_t& query_len); inline int set_session_state_(ObSQLSessionState state); //写入系统变量的默认值, deserialized scene need use base_value as baseline. int init_system_variables(const bool print_info_log, const bool is_sys_tenant, bool is_deserialized = false); @@ -1534,6 +1538,9 @@ protected: cur_query_buf_len_(0), cur_query_(nullptr), cur_query_len_(0), + top_query_buf_len_(0), + top_query_(nullptr), + top_query_len_(0), cur_statement_id_(0), last_active_time_(0), dis_state_(CLIENT_FORCE_DISCONNECT), @@ -1574,7 +1581,11 @@ protected: if (cur_query_ != nullptr) { cur_query_[0] = '\0'; } + if (top_query_ != nullptr) { + top_query_[0] = '\0'; + } cur_query_len_ = 0; + top_query_len_ = 0; cur_statement_id_ = 0; last_active_time_ = 0; dis_state_ = CLIENT_FORCE_DISCONNECT; @@ -1615,6 +1626,9 @@ protected: int64_t cur_query_buf_len_; char *cur_query_; volatile int64_t cur_query_len_; + int64_t top_query_buf_len_; + char *top_query_; + volatile int64_t top_query_len_; uint64_t cur_statement_id_; int64_t last_active_time_; ObDisconnectState dis_state_; @@ -2461,6 +2475,13 @@ inline const common::ObString ObBasicSessionInfo::get_current_query_string() con return str_ret; } +inline const common::ObString ObBasicSessionInfo::get_top_query_string() const +{ + common::ObString str_ret; + str_ret.assign_ptr(const_cast(thread_data_.top_query_), static_cast(thread_data_.top_query_len_)); + return str_ret; +} + inline ObCollationType ObBasicSessionInfo::get_local_collation_connection() const { return static_cast(sys_vars_cache_.get_collation_connection()); diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result index 30434c887..92e1217cc 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result @@ -2659,6 +2659,7 @@ OUT_BYTES bigint(20) NO NULL USER_CLIENT_PORT bigint(20) NO TOTAL_CPU_TIME bigint(21) NO NULL PROXY_USER varchar(128) YES NULL +TOP_INFO varchar(262143) YES NULL select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.GV$OB_PROCESSLIST limit 1); cnt 1 @@ -2703,6 +2704,7 @@ OUT_BYTES bigint(20) NO USER_CLIENT_PORT bigint(20) NO TOTAL_CPU_TIME bigint(21) NO PROXY_USER varchar(128) NO +TOP_INFO varchar(262143) NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_PROCESSLIST limit 1); cnt 1 diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result index 9fe9f06f7..6440d3cb6 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result @@ -3937,6 +3937,7 @@ OUT_BYTES bigint(20) NO NULL USER_CLIENT_PORT bigint(20) NO TOTAL_CPU_TIME bigint(21) NO NULL PROXY_USER varchar(128) YES NULL +TOP_INFO varchar(262143) YES NULL select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.GV$OB_PROCESSLIST limit 1); cnt 1 @@ -3981,6 +3982,7 @@ OUT_BYTES bigint(20) NO USER_CLIENT_PORT bigint(20) NO TOTAL_CPU_TIME bigint(21) NO PROXY_USER varchar(128) NO +TOP_INFO varchar(262143) NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_PROCESSLIST limit 1); cnt 1 diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result index 9ac59d3c2..0bf71a8d8 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result @@ -130,6 +130,7 @@ user_client_port bigint(20) NO 0 proxy_user varchar(128) YES NULL service_name varchar(64) YES NULL total_cpu_time double NO NULL +top_info varchar(262143) YES NULL select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_processlist; IF(count(*) >= 0, 1, 0) 1 diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result index 6f009a4f9..5b631a3c3 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result @@ -131,6 +131,7 @@ user_client_port bigint(20) NO 0 proxy_user varchar(128) YES NULL service_name varchar(64) YES NULL total_cpu_time double NO NULL +top_info varchar(262143) YES NULL select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_processlist; IF(count(*) >= 0, 1, 0) 1 From bd741a9fdc101c29b902f90357e1bc057dd6bfda Mon Sep 17 00:00:00 2001 From: ZenoWang Date: Wed, 14 Aug 2024 10:32:42 +0000 Subject: [PATCH 047/249] quit throttle if ls offline --- src/share/allocator/ob_mds_allocator.cpp | 23 +++++++++++++-- src/share/allocator/ob_mds_allocator.h | 5 ++-- .../ob_shared_memory_allocator_mgr.h | 6 ++-- src/share/allocator/ob_tx_data_allocator.cpp | 29 ++++++++++++++----- src/share/allocator/ob_tx_data_allocator.h | 9 ++++-- src/storage/ls/ob_ls_tx_service.cpp | 2 +- src/storage/ob_storage_table_guard.cpp | 21 ++++++++++---- src/storage/tx/ob_trans_service.cpp | 2 +- src/storage/tx/ob_tx_replay_executor.cpp | 5 +++- 9 files changed, 75 insertions(+), 27 deletions(-) diff --git a/src/share/allocator/ob_mds_allocator.cpp b/src/share/allocator/ob_mds_allocator.cpp index e14ad08d4..a901996c0 100644 --- a/src/share/allocator/ob_mds_allocator.cpp +++ b/src/share/allocator/ob_mds_allocator.cpp @@ -17,6 +17,7 @@ #include "share/throttle/ob_share_throttle_define.h" #include "storage/multi_data_source/runtime_utility/mds_tenant_service.h" #include "storage/tx_storage/ob_tenant_freezer.h" +#include "storage/tx_storage/ob_ls_service.h" using namespace oceanbase::storage::mds; @@ -171,7 +172,8 @@ void ObTenantBufferCtxAllocator::free(void *ptr) MTL(ObTenantMdsService*)->erase_alloc_backtrace(ptr); } -ObMdsThrottleGuard::ObMdsThrottleGuard(const bool for_replay, const int64_t abs_expire_time) : for_replay_(for_replay), abs_expire_time_(abs_expire_time) +ObMdsThrottleGuard::ObMdsThrottleGuard(const share::ObLSID ls_id, const bool for_replay, const int64_t abs_expire_time) + : ls_id_(ls_id), for_replay_(for_replay), abs_expire_time_(abs_expire_time) { throttle_tool_ = &(MTL(ObSharedMemAllocMgr *)->share_resource_throttle_tool()); if (0 == abs_expire_time) { @@ -183,14 +185,29 @@ ObMdsThrottleGuard::ObMdsThrottleGuard(const bool for_replay, const int64_t abs_ ObMdsThrottleGuard::~ObMdsThrottleGuard() { + int ret = OB_SUCCESS; + ObLSHandle ls_handle; ObThrottleInfoGuard share_ti_guard; ObThrottleInfoGuard module_ti_guard; if (OB_ISNULL(throttle_tool_)) { MDS_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "throttle tool is unexpected nullptr", KP(throttle_tool_)); } else if (throttle_tool_->is_throttling(share_ti_guard, module_ti_guard)) { - (void)TxShareMemThrottleUtil::do_throttle( - for_replay_, abs_expire_time_, share::mds_throttled_alloc(), *throttle_tool_, share_ti_guard, module_ti_guard); + + if (OB_FAIL(MTL(ObLSService *)->get_ls(ls_id_, ls_handle, ObLSGetMod::STORAGE_MOD))) { + STORAGE_LOG(WARN, "get ls handle failed", KR(ret), K(ls_id_)); + } else if (OB_ISNULL(ls_handle.get_ls())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(ERROR, "get ls handle failed", KR(ret), K(ls_id_)); + } else { + (void)TxShareMemThrottleUtil::do_throttle(for_replay_, + abs_expire_time_, + share::mds_throttled_alloc(), + *(ls_handle.get_ls()), + *throttle_tool_, + share_ti_guard, + module_ti_guard); + } if (throttle_tool_->still_throttling(share_ti_guard, module_ti_guard)) { (void)throttle_tool_->skip_throttle( diff --git a/src/share/allocator/ob_mds_allocator.h b/src/share/allocator/ob_mds_allocator.h index 31de9324f..1d572dc00 100644 --- a/src/share/allocator/ob_mds_allocator.h +++ b/src/share/allocator/ob_mds_allocator.h @@ -15,9 +15,9 @@ #include "lib/allocator/ob_vslice_alloc.h" #include "share/throttle/ob_share_throttle_define.h" +#include "share/ob_ls_id.h" namespace oceanbase { - namespace share { OB_INLINE int64_t &mds_throttled_alloc() @@ -64,10 +64,11 @@ struct ObTenantBufferCtxAllocator : public ObIAllocator// for now, it is just a class ObMdsThrottleGuard { public: - ObMdsThrottleGuard(const bool for_replay, const int64_t abs_expire_time); + ObMdsThrottleGuard(const share::ObLSID ls_id, const bool for_replay, const int64_t abs_expire_time); ~ObMdsThrottleGuard(); private: + const share::ObLSID ls_id_; bool for_replay_; int64_t abs_expire_time_; share::TxShareThrottleTool *throttle_tool_; diff --git a/src/share/allocator/ob_shared_memory_allocator_mgr.h b/src/share/allocator/ob_shared_memory_allocator_mgr.h index 9cf88bc22..1cdc15dbe 100644 --- a/src/share/allocator/ob_shared_memory_allocator_mgr.h +++ b/src/share/allocator/ob_shared_memory_allocator_mgr.h @@ -19,6 +19,7 @@ #include "share/throttle/ob_share_resource_throttle_tool.h" #include "share/rc/ob_tenant_base.h" #include "storage/tx_storage/ob_tenant_freezer.h" +#include "storage/ls/ob_ls.h" namespace oceanbase { namespace share { @@ -92,6 +93,7 @@ public: static int do_throttle(const bool for_replay, const int64_t abs_expire_time, const int64_t throttle_memory_size, + const ObLS &ls, TxShareThrottleTool &throttle_tool, ObThrottleInfoGuard &share_ti_guard, ObThrottleInfoGuard &module_ti_guard) @@ -112,8 +114,8 @@ public: while (throttle_tool.still_throttling(share_ti_guard, module_ti_guard) && (left_interval > 0)) { int64_t expected_wait_time = 0; - if (for_replay && MTL(ObTenantFreezer *)->exist_ls_throttle_is_skipping()) { - // skip throttle if ls freeze exists + if ((for_replay && MTL(ObTenantFreezer *)->exist_ls_throttle_is_skipping()) || ls.is_offline()) { + // skip throttle if : 1) throttle need skipping; 2) this logstream offline break; } else if ((expected_wait_time = throttle_tool.expected_wait_time(share_ti_guard, module_ti_guard)) <= 0) { diff --git a/src/share/allocator/ob_tx_data_allocator.cpp b/src/share/allocator/ob_tx_data_allocator.cpp index a1f10d051..2b84ff15d 100644 --- a/src/share/allocator/ob_tx_data_allocator.cpp +++ b/src/share/allocator/ob_tx_data_allocator.cpp @@ -16,6 +16,7 @@ #include "share/allocator/ob_shared_memory_allocator_mgr.h" #include "share/rc/ob_tenant_base.h" +#include "storage/tx_storage/ob_ls_service.h" #include "storage/tx/ob_tx_data_define.h" #include "storage/tx_storage/ob_tenant_freezer.h" @@ -109,8 +110,10 @@ void *ObTenantTxDataAllocator::alloc(const bool enable_throttle, const int64_t a return res; } -ObTxDataThrottleGuard::ObTxDataThrottleGuard(const bool for_replay, const int64_t abs_expire_time) - : for_replay_(for_replay), abs_expire_time_(abs_expire_time) +ObTxDataThrottleGuard::ObTxDataThrottleGuard(const ObLSID ls_id, + const bool for_replay, + const int64_t abs_expire_time) + : ls_id_(ls_id), for_replay_(for_replay), abs_expire_time_(abs_expire_time) { throttle_tool_ = &(MTL(ObSharedMemAllocMgr *)->share_resource_throttle_tool()); if (0 == abs_expire_time) { @@ -122,18 +125,28 @@ ObTxDataThrottleGuard::ObTxDataThrottleGuard(const bool for_replay, const int64_ ObTxDataThrottleGuard::~ObTxDataThrottleGuard() { + int ret = OB_SUCCESS; + ObLSHandle ls_handle; ObThrottleInfoGuard share_ti_guard; ObThrottleInfoGuard module_ti_guard; if (OB_ISNULL(throttle_tool_)) { MDS_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "throttle tool is unexpected nullptr", KP(throttle_tool_)); } else if (throttle_tool_->is_throttling(share_ti_guard, module_ti_guard)) { - (void)TxShareMemThrottleUtil::do_throttle(for_replay_, - abs_expire_time_, - share::tx_data_throttled_alloc(), - *throttle_tool_, - share_ti_guard, - module_ti_guard); + if (OB_FAIL(MTL(ObLSService *)->get_ls(ls_id_, ls_handle, ObLSGetMod::STORAGE_MOD))) { + STORAGE_LOG(WARN, "get ls handle failed", KR(ret), K(ls_id_)); + } else if (OB_ISNULL(ls_handle.get_ls())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(ERROR, "get ls handle failed", KR(ret), K(ls_id_)); + } else { + (void)TxShareMemThrottleUtil::do_throttle(for_replay_, + abs_expire_time_, + share::tx_data_throttled_alloc(), + *(ls_handle.get_ls()), + *throttle_tool_, + share_ti_guard, + module_ti_guard); + } if (throttle_tool_->still_throttling(share_ti_guard, module_ti_guard)) { (void)throttle_tool_->skip_throttle( diff --git a/src/share/allocator/ob_tx_data_allocator.h b/src/share/allocator/ob_tx_data_allocator.h index 56c3bf8cb..e32e737c0 100644 --- a/src/share/allocator/ob_tx_data_allocator.h +++ b/src/share/allocator/ob_tx_data_allocator.h @@ -14,12 +14,14 @@ #define OCEANBASE_ALLOCATOR_OB_TX_DATA_ALLOCATOR_H_ #include "lib/allocator/ob_slice_alloc.h" -#include "share/ob_delegate.h" -#include "share/throttle/ob_share_throttle_define.h" #include "lib/allocator/ob_vslice_alloc.h" +#include "share/ob_delegate.h" +#include "share/ob_ls_id.h" +#include "share/throttle/ob_share_throttle_define.h" namespace oceanbase { namespace share { +class ObLSID; OB_INLINE int64_t &tx_data_throttled_alloc() { @@ -67,10 +69,11 @@ private: class ObTxDataThrottleGuard { public: - ObTxDataThrottleGuard(const bool for_replay, const int64_t abs_expire_time); + ObTxDataThrottleGuard(const share::ObLSID ls_id, const bool for_replay, const int64_t abs_expire_time); ~ObTxDataThrottleGuard(); private: + const share::ObLSID ls_id_; bool for_replay_; int64_t abs_expire_time_; share::TxShareThrottleTool *throttle_tool_; diff --git a/src/storage/ls/ob_ls_tx_service.cpp b/src/storage/ls/ob_ls_tx_service.cpp index fd8be1fd9..85085da5e 100644 --- a/src/storage/ls/ob_ls_tx_service.cpp +++ b/src/storage/ls/ob_ls_tx_service.cpp @@ -252,7 +252,7 @@ int ObLSTxService::get_write_store_ctx(ObTxDesc &tx, abs_expire_ts = ObClockGenerator::getClock() + share::ObThrottleUnit::DEFAULT_MAX_THROTTLE_TIME; } - ObTxDataThrottleGuard tx_data_throttle_guard(false /* for_replay */, abs_expire_ts); + ObTxDataThrottleGuard tx_data_throttle_guard(ls_id_, false /* for_replay */, abs_expire_ts); ret = trans_service_->get_write_store_ctx(tx, snapshot, write_flag, store_ctx, spec_seq_no, false); } return ret; diff --git a/src/storage/ob_storage_table_guard.cpp b/src/storage/ob_storage_table_guard.cpp index 1f37afe9f..dd739f63d 100644 --- a/src/storage/ob_storage_table_guard.cpp +++ b/src/storage/ob_storage_table_guard.cpp @@ -74,12 +74,21 @@ void ObStorageTableGuard::throttle_if_needed_() // only do throttle on active memtable if (OB_NOT_NULL(memtable_) && memtable_->is_active_memtable()) { reset(); - (void)TxShareMemThrottleUtil::do_throttle(for_replay_, - store_ctx_.timeout_, - share::memstore_throttled_alloc(), - throttle_tool, - share_ti_guard, - module_ti_guard); + ObLSHandle ls_handle; + ObLS *ls = nullptr; + const ObLSID &ls_id = tablet_->get_tablet_meta().ls_id_; + if (OB_FAIL(MTL(ObLSService *)->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { + STORAGE_LOG(WARN, "get ls handle failed", KR(ret), K(ls_id)); + } else if (OB_ISNULL(ls = ls_handle.get_ls())) { + } else { + (void)TxShareMemThrottleUtil::do_throttle(for_replay_, + store_ctx_.timeout_, + share::memstore_throttled_alloc(), + *ls, + throttle_tool, + share_ti_guard, + module_ti_guard); + } } // if throttle is skipped due to some reasons, advance clock by call skip_throttle() and clean throttle status diff --git a/src/storage/tx/ob_trans_service.cpp b/src/storage/tx/ob_trans_service.cpp index 23f98c772..6608048aa 100644 --- a/src/storage/tx/ob_trans_service.cpp +++ b/src/storage/tx/ob_trans_service.cpp @@ -979,7 +979,7 @@ int ObTransService::register_mds_into_ctx_(ObTxDesc &tx_desc, TRANS_LOG(WARN, "get store ctx failed", KR(ret), K(tx_desc), K(ls_id)); } else { ObPartTransCtx *ctx = store_ctx.mvcc_acc_ctx_.tx_ctx_; - ObMdsThrottleGuard mds_throttle_guard(false/* for_replay */, ctx->get_trans_expired_time()); + ObMdsThrottleGuard mds_throttle_guard(ls_id, false /* for_replay */, ctx->get_trans_expired_time()); if (OB_ISNULL(ctx)) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "unexpected null ptr", KR(ret), K(tx_desc), K(ls_id), K(type)); diff --git a/src/storage/tx/ob_tx_replay_executor.cpp b/src/storage/tx/ob_tx_replay_executor.cpp index 13594d223..bdf79986c 100644 --- a/src/storage/tx/ob_tx_replay_executor.cpp +++ b/src/storage/tx/ob_tx_replay_executor.cpp @@ -301,6 +301,7 @@ int ObTxReplayExecutor::try_get_tx_ctx_() INT64_MAX, /*trans_expired_time_*/ ls_tx_srv_->get_trans_service()); ObTxDataThrottleGuard tx_data_throttle_guard( + ls_id_, true /* for_replay_ */, ObClockGenerator::getClock() + share::ObThrottleUnit::DEFAULT_MAX_THROTTLE_TIME); if (OB_FAIL(ls_tx_srv_->create_tx_ctx(arg, tx_ctx_existed, ctx_))) { @@ -418,6 +419,7 @@ int ObTxReplayExecutor::replay_rollback_to_() ObTxRollbackToLog log; const bool pre_barrier = base_header_.need_pre_replay_barrier(); ObTxDataThrottleGuard tx_data_throttle_guard( + ls_id_, true /* for_replay_ */, ObClockGenerator::getClock() + share::ObThrottleUnit::DEFAULT_MAX_THROTTLE_TIME); if (OB_FAIL(log_block_.deserialize_log_body(log))) { @@ -466,7 +468,8 @@ int ObTxReplayExecutor::replay_multi_source_data_() int ret = OB_SUCCESS; ObTxMultiDataSourceLog log; - ObMdsThrottleGuard mds_throttle_guard(true /* for_replay */, + ObMdsThrottleGuard mds_throttle_guard(ls_id_, + true /* for_replay */, ObClockGenerator::getClock() + share::ObThrottleUnit::DEFAULT_MAX_THROTTLE_TIME); From 24f20ff4100bfc416845bcc9bf810024f64bb165 Mon Sep 17 00:00:00 2001 From: obdev Date: Wed, 14 Aug 2024 10:39:36 +0000 Subject: [PATCH 048/249] [CP] cast target type supports variable representation --- src/pl/ob_pl_resolver.cpp | 46 +++++ src/pl/ob_pl_resolver.h | 11 ++ .../expr/ob_raw_expr_resolver_impl.cpp | 172 ++++++++++++------ 3 files changed, 172 insertions(+), 57 deletions(-) diff --git a/src/pl/ob_pl_resolver.cpp b/src/pl/ob_pl_resolver.cpp index 518343e81..7a6ba5218 100644 --- a/src/pl/ob_pl_resolver.cpp +++ b/src/pl/ob_pl_resolver.cpp @@ -10971,6 +10971,52 @@ int ObPLResolver::resolve_inner_call( return ret; } +int ObPLResolver::resolve_obj_access_node(ParseNode *node, + common::ObIAllocator &allocator, + sql::ObRawExprFactory &expr_factory, + sql::ObSQLSessionInfo &session_info, + share::schema::ObSchemaGetterGuard &schema_guard, + common::ObMySQLProxy *sql_proxy, + pl::ObPLBlockNS *ns, + ObArray &access_idxs) +{ + int ret = OB_SUCCESS; + pl::ObPLPackageGuard dummy_pkg_guard(sql::PACKAGE_RESV_HANDLE); + pl::ObPLResolver pl_resolver(allocator, + session_info, + schema_guard, + NULL == ns ? dummy_pkg_guard + : ns->get_external_ns()->get_resolve_ctx().package_guard_, + NULL == sql_proxy + ? (NULL == ns ? *GCTX.sql_proxy_ + : ns->get_external_ns() + ->get_resolve_ctx().sql_proxy_) + : *sql_proxy, + expr_factory, + NULL == ns ? NULL : ns->get_external_ns()->get_parent_ns(), + false/*not prepare*/, + false/*check mode*/, + NULL == ns ? true : false, + NULL/*param store*/, + NULL); + HEAP_VAR(pl::ObPLFunctionAST, func_ast, allocator) { + ObArray obj_access_idents; + OZ (pl_resolver.init(func_ast)); + OX (pl_resolver.get_current_namespace() = ns!=NULL ? *ns : pl_resolver.get_current_namespace()); + CK (OB_NOT_NULL(node)); + OZ (pl_resolver.resolve_obj_access_idents(*node, obj_access_idents, func_ast)); + for (int64_t i = 0; OB_SUCC(ret) && i < obj_access_idents.count(); ++i) { + OZ (pl_resolver.resolve_access_ident(obj_access_idents.at(i), + pl_resolver.get_current_namespace(), + expr_factory, + &session_info, + access_idxs, + func_ast)); + } + } + return ret; +} + int ObPLResolver::resolve_obj_access_node(const ParseNode &node, ObSQLSessionInfo &session_info, ObRawExprFactory &expr_factory, diff --git a/src/pl/ob_pl_resolver.h b/src/pl/ob_pl_resolver.h index 6e39fe2d2..4e497f035 100644 --- a/src/pl/ob_pl_resolver.h +++ b/src/pl/ob_pl_resolver.h @@ -362,6 +362,16 @@ public: ObIArray &real_exprs, ObPLCompileUnitAST &unit_ast, sql::ObRawExpr *&expr); + + static + int resolve_obj_access_node(ParseNode *node, + common::ObIAllocator &allocator, + sql::ObRawExprFactory &expr_factory, + sql::ObSQLSessionInfo &session_info, + share::schema::ObSchemaGetterGuard &schema_guard, + common::ObMySQLProxy *sql_proxy, + pl::ObPLBlockNS *ns, + ObArray &access_idxs); static int resolve_obj_access_node(const ParseNode &node, ObSQLSessionInfo &session_info, @@ -371,6 +381,7 @@ public: ObIArray &obj_access_idents, ObIArray& access_idxs, ObPLPackageGuard *package_guard); + static int resolve_cparam_list_simple(const ParseNode &node, ObRawExprFactory &expr_factory, diff --git a/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp b/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp index 37cdf42af..0ed38b68e 100644 --- a/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp @@ -326,6 +326,7 @@ int ObRawExprResolverImpl::do_recursive_resolve(const ParseNode *node, } else { ObObj val; ObObjType data_type = static_cast(node->int16_values_[OB_NODE_CAST_TYPE_IDX]); + bool need_set = true; if (ob_is_string_tc(data_type)) { int32_t len = node->int32_values_[1]; if (lib::is_oracle_mode()) { @@ -368,67 +369,122 @@ int ObRawExprResolverImpl::do_recursive_resolve(const ParseNode *node, } } } else if (ob_is_extend(data_type)) { - if (lib::is_mysql_mode()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("extend type in mysql is invalid", K(ret)); - } else { - const ParseNode *name_node = node->children_[0]; - CK (OB_NOT_NULL(ctx_.session_info_), OB_NOT_NULL(ctx_.schema_checker_)); - CK (OB_NOT_NULL(name_node)); - CK (T_SP_TYPE == name_node->type_); - if (OB_SUCC(ret)) { - uint64_t udt_id = OB_INVALID_ID; - uint64_t db_id = ctx_.session_info_->get_database_id(); - ObString udt_name(name_node->children_[1]->str_len_, name_node->children_[1]->str_value_); - - if (NULL != name_node->children_[0]) { - OZ (ctx_.schema_checker_->get_database_id(ctx_.session_info_->get_effective_tenant_id(), - ObString(name_node->children_[0]->str_len_, name_node->children_[0]->str_value_), - db_id)); - } - OZ (ctx_.schema_checker_->get_udt_id(ctx_.session_info_->get_effective_tenant_id(), db_id, OB_INVALID_ID, - ObString(name_node->children_[1]->str_len_, name_node->children_[1]->str_value_), udt_id)); - - if (OB_SUCC(ret) && OB_INVALID_ID == udt_id) { - if(OB_FAIL(ctx_.schema_checker_->get_udt_id(OB_SYS_TENANT_ID, OB_SYS_DATABASE_ID, OB_INVALID_ID, - udt_name, udt_id))) { - LOG_WARN("get udt id from sys fail", K(ret), K(udt_name)); - } - } - - if (OB_SUCC(ret)) { + ParseNode *name_node = node->children_[0]; + CK (lib::is_oracle_mode()); + CK (OB_NOT_NULL(ctx_.session_info_), + OB_NOT_NULL(ctx_.schema_checker_), + OB_NOT_NULL(ctx_.schema_checker_->get_schema_guard())); + CK (OB_NOT_NULL(name_node)); + if (OB_SUCC(ret)) { + ObArray access_idxs; + if (OB_FAIL(pl::ObPLResolver::resolve_obj_access_node(name_node, + ctx_.expr_factory_.get_allocator(), + ctx_.expr_factory_, + *const_cast(ctx_.session_info_), + *(ctx_.schema_checker_->get_schema_guard()), + GCTX.sql_proxy_, + ctx_.secondary_namespace_, + access_idxs))) { + LOG_WARN("failed to resolve sp obj access node", K(ret), K(access_idxs)); + } else if (access_idxs.count() > 0) { + if (pl::ObObjAccessIdx::is_udt_type(access_idxs)) { + uint64_t udt_id = access_idxs.at(access_idxs.count() - 1).var_index_; if (OB_INVALID_ID == udt_id) { - ret = OB_ERR_INVALID_TYPE_FOR_ARGUMENT; - LOG_WARN("Invalid type to cast", K(ObString(name_node->children_[1]->str_len_, name_node->children_[1]->str_value_)), K(ret)); - } else { - c_expr->set_udt_id(udt_id); - } - } - - if (OB_SUCC(ret) && NULL != ctx_.stmt_) { - ObStmt *stmt = ctx_.stmt_; - uint64_t tenant_id = pl::get_tenant_id_by_object_id(udt_id); - const ObUDTTypeInfo *udt_info = NULL; - if (OB_FAIL(ctx_.schema_checker_->get_udt_info(tenant_id, udt_id, udt_info))) { - LOG_WARN("failed to get udt info", K(ret)); - } else if (OB_ISNULL(udt_info)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get null udt info", K(ret)); - } else if (udt_info->get_schema_version() != common::OB_INVALID_VERSION) { - ObSchemaObjVersion udt_schema_version; - udt_schema_version.object_id_ = udt_id; - udt_schema_version.object_type_ = share::schema::DEPENDENCY_TYPE; - udt_schema_version.version_ = udt_info->get_schema_version(); - uint64_t dep_obj_id = ctx_.view_ref_id_; - if (OB_FAIL(stmt->add_global_dependency_table(udt_schema_version))) { - LOG_WARN("failed to add global dependency", K(ret)); - } else if (OB_FAIL(stmt->add_ref_obj_version(dep_obj_id, db_id, - ObObjectType::VIEW, udt_schema_version, - ctx_.expr_factory_.get_allocator()))) { - LOG_WARN("failed to add ref obj version", K(ret)); + LOG_WARN("unexpected udt id from access_idxs", K(ret), K(access_idxs)); + } else { + c_expr->set_udt_id(access_idxs.at(access_idxs.count() - 1).var_index_); + if (OB_NOT_NULL(ctx_.stmt_)) { + ObStmt *stmt = ctx_.stmt_; + uint64_t tenant_id = pl::get_tenant_id_by_object_id(udt_id); + const ObUDTTypeInfo *udt_info = NULL; + if (OB_FAIL(ctx_.schema_checker_->get_udt_info(tenant_id, udt_id, udt_info))) { + LOG_WARN("failed to get udt info", K(ret)); + } else if (OB_ISNULL(udt_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get null udt info", K(ret)); + } else if (udt_info->get_schema_version() != common::OB_INVALID_VERSION) { + ObSchemaObjVersion udt_schema_version( + udt_id, udt_info->get_schema_version(), share::schema::DEPENDENCY_TYPE); + if (OB_FAIL(stmt->add_global_dependency_table(udt_schema_version))) { + LOG_WARN("failed to add global dependency", K(ret)); + } else if (OB_FAIL(stmt->add_ref_obj_version(ctx_.view_ref_id_, + udt_info->get_database_id(), + ObObjectType::VIEW, + udt_schema_version, + ctx_.expr_factory_.get_allocator()))) { + LOG_WARN("failed to add ref obj version", K(ret)); + } + } } } + } else if (OB_NOT_NULL(ctx_.stmt_)) { + ret = OB_ERR_PARAM_INVALID; + LOG_WARN("Invalid type to cast", K(ret), K(access_idxs)); + } else if (pl::ObObjAccessIdx::is_local_variable(access_idxs) || + pl::ObObjAccessIdx::is_table_column(access_idxs) || + pl::ObObjAccessIdx::is_function_return_variable(access_idxs) || + pl::ObObjAccessIdx::is_package_variable(access_idxs) || + pl::ObObjAccessIdx::is_subprogram_variable(access_idxs) || + pl::ObObjAccessIdx::is_type(access_idxs)) { + pl::ObPLDataType final_type; + if (pl::ObObjAccessIdx::is_type(access_idxs)) { + uint64_t udt_id = access_idxs.at(access_idxs.count() - 1).var_index_; + const pl::ObUserDefinedType *user_type = NULL; + CK (udt_id != OB_INVALID_ID); + OZ (ctx_.secondary_namespace_->get_pl_data_type_by_id(udt_id, user_type)); + CK (OB_NOT_NULL(user_type)); + OX (final_type = *user_type); + } else { + final_type = pl::ObObjAccessIdx::get_final_type(access_idxs); + } + if (final_type.is_subtype()) { + CK (OB_NOT_NULL(ctx_.secondary_namespace_)); + OZ (ctx_.secondary_namespace_->get_subtype_actually_basetype(final_type)); + } + if (OB_FAIL(ret)) { + } else if (OB_NOT_NULL(final_type.get_data_type())) { // basic type + val.set_smallint(final_type.get_data_type()->get_meta_type().get_type()); + int16_t type_val[4]; + type_val[0] = final_type.get_data_type()->get_meta_type().get_type(); + if (ObRawType == type_val[0]) { + type_val[1] = BINARY_COLLATION; + type_val[2] = final_type.get_data_type()->get_length() & 0xffff; + type_val[3] = (final_type.get_data_type()->get_length() >> 16) & 0xffff; + } else if (ObCharType == type_val[0] + || ObVarcharType == type_val[0] + || ObNVarchar2Type == type_val[0] + || ObNCharType == type_val[0]) { + type_val[1] = INVALID_COLLATION; + type_val[2] = final_type.get_data_type()->get_length() & 0xffff; + type_val[3] = (final_type.get_data_type()->get_length() >> 16) & 0xffff; + } else if (ObURowIDType == type_val[0]) { + type_val[1] = final_type.get_data_type()->get_length(); + type_val[2] = final_type.get_data_type()->get_precision()& 0xffff; + type_val[3] = final_type.get_data_type()->get_scale()& 0xffff; + } else { + type_val[1] = final_type.get_data_type()->get_collation_type(); + type_val[2] = final_type.get_data_type()->get_precision()& 0xffff; + type_val[3] = final_type.get_data_type()->get_scale()& 0xffff; + } + c_expr->set_accuracy(final_type.get_data_type()->get_accuracy()); + int64_t v = (((unsigned long)type_val[3] << 48) & 0xffff000000000000) + | (((unsigned long)type_val[2] << 32) & 0x0000ffff00000000) + | ((type_val[1] << 16) & 0x00000000ffff0000) + | (type_val[0] & 0x000000000000ffff); + val.set_int(v); + need_set = false; + } else { + ret = OB_NOT_SUPPORTED; + LOG_WARN("cast composite only support udt or basic type", K(ret), K(access_idxs)); + } + } else { + ret = OB_NOT_SUPPORTED; + LOG_WARN("access idxs not include type describe", K(ret), K(access_idxs)); } + } else { + ret = OB_ERR_PARAM_INVALID; + LOG_WARN("not allow null type to cast.", K(ret), K(access_idxs)); } } } else if (ob_is_rowid_tc(data_type)) { @@ -444,7 +500,9 @@ int ObRawExprResolverImpl::do_recursive_resolve(const ParseNode *node, if (OB_SUCC(ret)) { if (is_oracle_mode()) { - val.set_int(node->value_); + if (need_set) { + val.set_int(node->value_); + } } else { ObCollationType coll_type = static_cast(node->int16_values_[OB_NODE_CAST_COLL_IDX]); if (CS_TYPE_INVALID != coll_type) { From 9f549313fa8c56a3b9352293e49555d09331ab6e Mon Sep 17 00:00:00 2001 From: chinaxing Date: Wed, 14 Aug 2024 13:06:48 +0000 Subject: [PATCH 049/249] fix ut: pl.pl_cursor_1_oracle --- src/sql/engine/ob_physical_plan.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sql/engine/ob_physical_plan.cpp b/src/sql/engine/ob_physical_plan.cpp index 95dae8e17..97b4e69e3 100644 --- a/src/sql/engine/ob_physical_plan.cpp +++ b/src/sql/engine/ob_physical_plan.cpp @@ -146,7 +146,7 @@ ObPhysicalPlan::ObPhysicalPlan(MemoryContext &mem_context /* = CURRENT_CONTEXT * can_set_feedback_info_(true), need_switch_to_table_lock_worker_(false), data_complement_gen_doc_id_(false), - dml_table_ids_(), + dml_table_ids_(&allocator_), direct_load_need_sort_(false) { } From 43ff0d87377215f863841cce5395714c56fa26db Mon Sep 17 00:00:00 2001 From: hwx65 <1780011298@qq.com> Date: Wed, 14 Aug 2024 13:12:59 +0000 Subject: [PATCH 050/249] Fixed ObNumber::is_valid_int64 with improved checks for excessive decimal places --- deps/oblib/src/lib/number/ob_number_v2.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/oblib/src/lib/number/ob_number_v2.cpp b/deps/oblib/src/lib/number/ob_number_v2.cpp index 695628bdf..8377b435e 100644 --- a/deps/oblib/src/lib/number/ob_number_v2.cpp +++ b/deps/oblib/src/lib/number/ob_number_v2.cpp @@ -1592,7 +1592,7 @@ int ObNumber::check_range(bool *is_valid_uint64, bool *is_valid_int64, } else { *is_valid = false; //no break } - } else { + } else if (digit != 0) { decimal_parts = digit; break; } From 4ca2928dcece189f9cd24af3101e144e7cfaa657 Mon Sep 17 00:00:00 2001 From: fkuner <784819644@qq.com> Date: Wed, 14 Aug 2024 13:19:02 +0000 Subject: [PATCH 051/249] [OBCDC] adapt insert overwrite about exchanging more than one partition --- .../libobcdc/src/ob_cdc_tablet_to_table_info.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/logservice/libobcdc/src/ob_cdc_tablet_to_table_info.cpp b/src/logservice/libobcdc/src/ob_cdc_tablet_to_table_info.cpp index 86ad3cbd8..46f0cece8 100644 --- a/src/logservice/libobcdc/src/ob_cdc_tablet_to_table_info.cpp +++ b/src/logservice/libobcdc/src/ob_cdc_tablet_to_table_info.cpp @@ -472,7 +472,12 @@ int TabletToTableInfo::exchange_tablet_table_info(const common::ObSArray Date: Wed, 14 Aug 2024 13:24:47 +0000 Subject: [PATCH 052/249] enhance check_kill_gracefully --- src/observer/main.cpp | 27 ++++++++++++++++++--------- src/observer/omt/ob_tenant.cpp | 7 +++++++ 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/observer/main.cpp b/src/observer/main.cpp index 5d9caf0aa..c6dc132cf 100644 --- a/src/observer/main.cpp +++ b/src/observer/main.cpp @@ -398,7 +398,7 @@ static int check_uid_before_start(const char *dir_path) return ret; } -static void print_all_thread(const char* desc) +void print_all_thread(const char* desc, uint64_t tenant_id) { MPRINT("============= [%s] begin to show unstopped thread =============", desc); DIR *dir = opendir("/proc/self/task"); @@ -415,13 +415,22 @@ static void print_all_thread(const char* desc) if (file == NULL) { MPRINT("fail to print thread tid: %s", tid); } else { - char name[256]; - fgets(name, 256, file); - size_t len = strlen(name); - if (len > 0 && name[len - 1] == '\n') { - name[len - 1] = '\0'; + char thread_name[256]; + if (fgets(thread_name, sizeof(thread_name), file) != nullptr) { + size_t len = strlen(thread_name); + if (len > 0 && thread_name[len - 1] == '\n') { + thread_name[len - 1] = '\0'; + } + if (!is_server_tenant(tenant_id)) { + char tenant_id_str[20]; + snprintf(tenant_id_str, sizeof(tenant_id_str), "T%lu_", tenant_id); + if (0 == strncmp(thread_name, tenant_id_str, strlen(tenant_id_str))) { + MPRINT("[CHECK_KILL_GRACEFULLY][T%lu][%s] detect unstopped thread, tid: %s, name: %s", tenant_id, desc, tid, thread_name); + } + } else { + MPRINT("[CHECK_KILL_GRACEFULLY][%s] detect unstopped thread, tid: %s, name: %s", desc, tid, thread_name); + } } - MPRINT("[%s] detect unstopped thread, tid: %s, name: %s", desc, tid, name); fclose(file); } } @@ -598,7 +607,7 @@ int main(int argc, char *argv[]) } else if (OB_FAIL(observer.wait())) { LOG_ERROR("observer wait fail", K(ret)); } - print_all_thread("BEFORE_DESTROY"); + print_all_thread("BEFORE_DESTROY", OB_SERVER_TENANT_ID); observer.destroy(); } curl_global_cleanup(); @@ -606,6 +615,6 @@ int main(int argc, char *argv[]) } LOG_INFO("observer exits", "observer_version", PACKAGE_STRING); - print_all_thread("AFTER_DESTROY"); + print_all_thread("AFTER_DESTROY", OB_SERVER_TENANT_ID); return ret; } diff --git a/src/observer/omt/ob_tenant.cpp b/src/observer/omt/ob_tenant.cpp index a3a653983..85f1aa7cf 100644 --- a/src/observer/omt/ob_tenant.cpp +++ b/src/observer/omt/ob_tenant.cpp @@ -1140,6 +1140,12 @@ int ObTenant::try_wait() return ret; } +void __attribute__((weak)) print_all_thread(const char* desc, uint64_t tenant_id) +{ + UNUSED(desc); + UNUSED(tenant_id); +} + void ObTenant::destroy() { int tmp_ret = OB_SUCCESS; @@ -1154,6 +1160,7 @@ void ObTenant::destroy() } group_map_.destroy_group(); ObTenantSwitchGuard guard(this); + print_all_thread("TENANT_BEFORE_DESTROY", id_); destroy_mtl_module(); // 1.some mtl module(eg: ObDataAccessService) remove tmp file when destroy, // so remove_tenant_file must be after destroy_mtl_module. From 0868a33683989ff5927f90e0e74662a8825c997c Mon Sep 17 00:00:00 2001 From: obdev Date: Thu, 15 Aug 2024 04:10:48 +0000 Subject: [PATCH 053/249] [CP][to #40764052]fixed all_objects status set of package body --- .../ob_inner_table_schema.21151_21200.cpp | 2 +- .../ob_inner_table_schema.21201_21250.cpp | 2 +- .../ob_inner_table_schema.25001_25050.cpp | 6 +-- .../inner_table/ob_inner_table_schema_def.py | 37 ++++++++++++++++++- 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/src/share/inner_table/ob_inner_table_schema.21151_21200.cpp b/src/share/inner_table/ob_inner_table_schema.21151_21200.cpp index 5d3286b26..23e8a030e 100644 --- a/src/share/inner_table/ob_inner_table_schema.21151_21200.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21151_21200.cpp @@ -1910,7 +1910,7 @@ int ObInnerTableSchema::cdb_objects_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(A.TENANT_ID AS SIGNED) AS CON_ID, CAST(B.DATABASE_NAME AS CHAR(128)) AS OWNER, CAST(A.OBJECT_NAME AS CHAR(128)) AS OBJECT_NAME, CAST(A.SUBOBJECT_NAME AS CHAR(128)) AS SUBOBJECT_NAME, CAST(A.OBJECT_ID AS SIGNED) AS OBJECT_ID, CAST(A.DATA_OBJECT_ID AS SIGNED) AS DATA_OBJECT_ID, CAST(A.OBJECT_TYPE AS CHAR(23)) AS OBJECT_TYPE, CAST(A.GMT_CREATE AS DATETIME) AS CREATED, CAST(A.GMT_MODIFIED AS DATETIME) AS LAST_DDL_TIME, CAST(A.GMT_CREATE AS DATETIME) AS TIMESTAMP, CAST(A.STATUS AS CHAR(7)) AS STATUS, CAST(A.TEMPORARY AS CHAR(1)) AS TEMPORARY, CAST(A.`GENERATED` AS CHAR(1)) AS "GENERATED", CAST(A.SECONDARY AS CHAR(1)) AS SECONDARY, CAST(A.NAMESPACE AS SIGNED) AS NAMESPACE, CAST(A.EDITION_NAME AS CHAR(128)) AS EDITION_NAME, CAST(NULL AS CHAR(18)) AS SHARING, CAST(NULL AS CHAR(1)) AS EDITIONABLE, CAST(NULL AS CHAR(1)) AS ORACLE_MAINTAINED, CAST(NULL AS CHAR(1)) AS APPLICATION, CAST(NULL AS CHAR(1)) AS DEFAULT_COLLATION, CAST(NULL AS CHAR(1)) AS DUPLICATED, CAST(NULL AS CHAR(1)) AS SHARDED, CAST(NULL AS CHAR(1)) AS IMPORTED_OBJECT, CAST(NULL AS SIGNED) AS CREATED_APPID, CAST(NULL AS SIGNED) AS CREATED_VSNID, CAST(NULL AS SIGNED) AS MODIFIED_APPID, CAST(NULL AS SIGNED) AS MODIFIED_VSNID FROM ( SELECT A.TENANT_ID, USEC_TO_TIME(B.SCHEMA_VERSION) AS GMT_CREATE, USEC_TO_TIME(A.SCHEMA_VERSION) AS GMT_MODIFIED, A.DATABASE_ID, A.TABLE_NAME AS OBJECT_NAME, NULL AS SUBOBJECT_NAME, CAST(A.TABLE_ID AS SIGNED) AS OBJECT_ID, A.TABLET_ID AS DATA_OBJECT_ID, 'TABLE' AS OBJECT_TYPE, 'VALID' AS STATUS, 'N' AS TEMPORARY, 'N' AS "GENERATED", 'N' AS SECONDARY, 0 AS NAMESPACE, NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_CORE_ALL_TABLE A JOIN OCEANBASE.__ALL_VIRTUAL_CORE_ALL_TABLE B ON A.TENANT_ID = B.TENANT_ID AND B.TABLE_NAME = '__all_core_table' UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,CAST((CASE WHEN DATABASE_ID = 201004 THEN TABLE_NAME WHEN TABLE_TYPE = 5 THEN SUBSTR(TABLE_NAME, 7 + POSITION('_' IN SUBSTR(TABLE_NAME, 7))) ELSE TABLE_NAME END) AS CHAR(128)) AS OBJECT_NAME ,NULL SUBOBJECT_NAME ,TABLE_ID OBJECT_ID ,(CASE WHEN TABLET_ID != 0 THEN TABLET_ID ELSE NULL END) DATA_OBJECT_ID ,CASE WHEN TABLE_TYPE IN (0,3,6,8,9,14) THEN 'TABLE' WHEN TABLE_TYPE IN (2) THEN 'VIRTUAL TABLE' WHEN TABLE_TYPE IN (1,4) THEN 'VIEW' WHEN TABLE_TYPE IN (5) THEN 'INDEX' WHEN TABLE_TYPE IN (7) THEN 'MATERIALIZED VIEW' WHEN TABLE_TYPE IN (15) THEN 'MATERIALIZED VIEW LOG' ELSE NULL END AS OBJECT_TYPE ,CAST(CASE WHEN TABLE_TYPE IN (5,15) THEN CASE WHEN INDEX_STATUS = 2 THEN 'VALID' WHEN INDEX_STATUS = 3 THEN 'CHECKING' WHEN INDEX_STATUS = 4 THEN 'INELEGIBLE' WHEN INDEX_STATUS = 5 THEN 'ERROR' ELSE 'UNUSABLE' END ELSE CASE WHEN OBJECT_STATUS = 1 THEN 'VALID' ELSE 'INVALID' END END AS CHAR(10)) AS STATUS ,CASE WHEN TABLE_TYPE IN (6,8,9) THEN 'Y' ELSE 'N' END AS TEMPORARY ,CASE WHEN TABLE_TYPE IN (0,1) THEN 'Y' ELSE 'N' END AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_TABLE WHERE TABLE_TYPE != 12 AND TABLE_TYPE != 13 AND TABLE_MODE >> 12 & 15 in (0,1) UNION ALL SELECT CST.TENANT_ID ,CST.GMT_CREATE ,CST.GMT_MODIFIED ,DB.DATABASE_ID ,CST.constraint_name AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,TBL.TABLE_ID AS OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'INDEX' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_CONSTRAINT CST, OCEANBASE.__ALL_VIRTUAL_TABLE TBL, OCEANBASE.__ALL_VIRTUAL_DATABASE DB WHERE CST.TENANT_ID = TBL.TENANT_ID AND TBL.TENANT_ID = DB.TENANT_ID AND DB.DATABASE_ID = TBL.DATABASE_ID AND TBL.TABLE_ID = CST.TABLE_ID and CST.CONSTRAINT_TYPE = 1 AND TBL.TABLE_MODE >> 12 & 15 in (0,1) UNION ALL SELECT P.TENANT_ID ,P.GMT_CREATE ,P.GMT_MODIFIED ,T.DATABASE_ID ,CAST((CASE WHEN T.DATABASE_ID = 201004 THEN T.TABLE_NAME WHEN T.TABLE_TYPE = 5 THEN SUBSTR(T.TABLE_NAME, 7 + POSITION('_' IN SUBSTR(T.TABLE_NAME, 7))) ELSE T.TABLE_NAME END) AS CHAR(128)) AS OBJECT_NAME ,P.PART_NAME SUBOBJECT_NAME ,P.PART_ID OBJECT_ID ,CASE WHEN P.TABLET_ID != 0 THEN P.TABLET_ID ELSE NULL END AS DATA_OBJECT_ID ,(CASE WHEN T.TABLE_TYPE = 5 THEN 'INDEX PARTITION' ELSE 'TABLE PARTITION' END) AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY , NULL AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_TABLE T JOIN OCEANBASE.__ALL_VIRTUAL_PART P ON T.TABLE_ID = P.TABLE_ID WHERE T.TENANT_ID = P.TENANT_ID AND T.TABLE_MODE >> 12 & 15 in (0,1) UNION ALL SELECT SUBP.TENANT_ID ,SUBP.GMT_CREATE ,SUBP.GMT_MODIFIED ,T.DATABASE_ID ,CAST((CASE WHEN T.DATABASE_ID = 201004 THEN T.TABLE_NAME WHEN T.TABLE_TYPE = 5 THEN SUBSTR(T.TABLE_NAME, 7 + POSITION('_' IN SUBSTR(T.TABLE_NAME, 7))) ELSE T.TABLE_NAME END) AS CHAR(128)) AS OBJECT_NAME ,SUBP.SUB_PART_NAME SUBOBJECT_NAME ,SUBP.SUB_PART_ID OBJECT_ID ,SUBP.TABLET_ID AS DATA_OBJECT_ID ,(CASE WHEN T.TABLE_TYPE = 5 THEN 'INDEX SUBPARTITION' ELSE 'TABLE SUBPARTITION' END) AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'Y' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_TABLE T, OCEANBASE.__ALL_VIRTUAL_PART P,OCEANBASE.__ALL_VIRTUAL_SUB_PART SUBP WHERE T.TABLE_ID =P.TABLE_ID AND P.TABLE_ID=SUBP.TABLE_ID AND P.PART_ID =SUBP.PART_ID AND T.TENANT_ID = P.TENANT_ID AND P.TENANT_ID = SUBP.TENANT_ID AND T.TABLE_MODE >> 12 & 15 in (0,1) UNION ALL SELECT P.TENANT_ID ,P.GMT_CREATE ,P.GMT_MODIFIED ,P.DATABASE_ID ,P.PACKAGE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,P.PACKAGE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,CASE WHEN TYPE = 1 THEN 'PACKAGE' WHEN TYPE = 2 THEN 'PACKAGE BODY' ELSE NULL END AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM OCEANBASE.__ALL_VIRTUAL_ERROR E WHERE P.TENANT_ID = E.TENANT_ID AND P.PACKAGE_ID = E.OBJ_ID AND (E.OBJ_TYPE = 3 OR E.OBJ_TYPE = 5)) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_PACKAGE P UNION ALL SELECT R.TENANT_ID ,R.GMT_CREATE ,R.GMT_MODIFIED ,R.DATABASE_ID ,R.ROUTINE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,R.ROUTINE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,CASE WHEN ROUTINE_TYPE = 1 THEN 'PROCEDURE' WHEN ROUTINE_TYPE = 2 THEN 'FUNCTION' ELSE NULL END AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM OCEANBASE.__ALL_VIRTUAL_ERROR E WHERE R.TENANT_ID = E.TENANT_ID AND R.ROUTINE_ID = E.OBJ_ID AND (E.OBJ_TYPE = 9 OR E.OBJ_TYPE = 12)) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_ROUTINE R WHERE (ROUTINE_TYPE = 1 OR ROUTINE_TYPE = 2) UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,TYPE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,TYPE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_TYPE UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,OBJECT_TYPE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE BODY' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_OBJECT_TYPE WHERE TYPE = 2 UNION ALL SELECT T.TENANT_ID ,T.GMT_CREATE ,T.GMT_MODIFIED ,T.DATABASE_ID ,T.TRIGGER_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,T.TRIGGER_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TRIGGER' OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM OCEANBASE.__ALL_VIRTUAL_ERROR E WHERE T.TENANT_ID = E.TENANT_ID AND T.TRIGGER_ID = E.OBJ_ID AND (E.OBJ_TYPE = 7)) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_TRIGGER T UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,SEQUENCE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,SEQUENCE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'SEQUENCE' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_SEQUENCE_OBJECT UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,SYNONYM_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,SYNONYM_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'SYNONYM' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_SYNONYM UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,CAST(201006 AS SIGNED) AS DATABASE_ID ,NAMESPACE AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,CONTEXT_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'CONTEXT' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,21 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_TENANT_CONTEXT UNION ALL SELECT TENANT_ID, GMT_CREATE, GMT_MODIFIED, DATABASE_ID, DATABASE_NAME AS OBJECT_NAME, NULL AS SUBOBJECT_NAME, DATABASE_ID AS OBJECT_ID, NULL AS DATA_OBJECT_ID, 'DATABASE' AS OBJECT_TYPE, 'VALID' AS STATUS, 'N' AS TEMPORARY, 'N' AS "GENERATED", 'N' AS SECONDARY, 0 AS NAMESPACE, NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_DATABASE UNION ALL SELECT TENANT_ID, GMT_CREATE, GMT_MODIFIED, CAST(201001 AS SIGNED) AS DATABASE_ID, TABLEGROUP_NAME AS OBJECT_NAME, NULL AS SUBOBJECT_NAME, TABLEGROUP_ID AS OBJECT_ID, NULL AS DATA_OBJECT_ID, 'TABLEGROUP' AS OBJECT_TYPE, 'VALID' AS STATUS, 'N' AS TEMPORARY, 'N' AS "GENERATED", 'N' AS SECONDARY, 0 AS NAMESPACE, NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_TABLEGROUP ) A JOIN OCEANBASE.__ALL_VIRTUAL_DATABASE B ON A.TENANT_ID = B.TENANT_ID AND A.DATABASE_ID = B.DATABASE_ID )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(A.TENANT_ID AS SIGNED) AS CON_ID, CAST(B.DATABASE_NAME AS CHAR(128)) AS OWNER, CAST(A.OBJECT_NAME AS CHAR(128)) AS OBJECT_NAME, CAST(A.SUBOBJECT_NAME AS CHAR(128)) AS SUBOBJECT_NAME, CAST(A.OBJECT_ID AS SIGNED) AS OBJECT_ID, CAST(A.DATA_OBJECT_ID AS SIGNED) AS DATA_OBJECT_ID, CAST(A.OBJECT_TYPE AS CHAR(23)) AS OBJECT_TYPE, CAST(A.GMT_CREATE AS DATETIME) AS CREATED, CAST(A.GMT_MODIFIED AS DATETIME) AS LAST_DDL_TIME, CAST(A.GMT_CREATE AS DATETIME) AS TIMESTAMP, CAST(A.STATUS AS CHAR(7)) AS STATUS, CAST(A.TEMPORARY AS CHAR(1)) AS TEMPORARY, CAST(A.`GENERATED` AS CHAR(1)) AS "GENERATED", CAST(A.SECONDARY AS CHAR(1)) AS SECONDARY, CAST(A.NAMESPACE AS SIGNED) AS NAMESPACE, CAST(A.EDITION_NAME AS CHAR(128)) AS EDITION_NAME, CAST(NULL AS CHAR(18)) AS SHARING, CAST(NULL AS CHAR(1)) AS EDITIONABLE, CAST(NULL AS CHAR(1)) AS ORACLE_MAINTAINED, CAST(NULL AS CHAR(1)) AS APPLICATION, CAST(NULL AS CHAR(1)) AS DEFAULT_COLLATION, CAST(NULL AS CHAR(1)) AS DUPLICATED, CAST(NULL AS CHAR(1)) AS SHARDED, CAST(NULL AS CHAR(1)) AS IMPORTED_OBJECT, CAST(NULL AS SIGNED) AS CREATED_APPID, CAST(NULL AS SIGNED) AS CREATED_VSNID, CAST(NULL AS SIGNED) AS MODIFIED_APPID, CAST(NULL AS SIGNED) AS MODIFIED_VSNID FROM ( SELECT A.TENANT_ID, USEC_TO_TIME(B.SCHEMA_VERSION) AS GMT_CREATE, USEC_TO_TIME(A.SCHEMA_VERSION) AS GMT_MODIFIED, A.DATABASE_ID, A.TABLE_NAME AS OBJECT_NAME, NULL AS SUBOBJECT_NAME, CAST(A.TABLE_ID AS SIGNED) AS OBJECT_ID, A.TABLET_ID AS DATA_OBJECT_ID, 'TABLE' AS OBJECT_TYPE, 'VALID' AS STATUS, 'N' AS TEMPORARY, 'N' AS "GENERATED", 'N' AS SECONDARY, 0 AS NAMESPACE, NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_CORE_ALL_TABLE A JOIN OCEANBASE.__ALL_VIRTUAL_CORE_ALL_TABLE B ON A.TENANT_ID = B.TENANT_ID AND B.TABLE_NAME = '__all_core_table' UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,CAST((CASE WHEN DATABASE_ID = 201004 THEN TABLE_NAME WHEN TABLE_TYPE = 5 THEN SUBSTR(TABLE_NAME, 7 + POSITION('_' IN SUBSTR(TABLE_NAME, 7))) ELSE TABLE_NAME END) AS CHAR(128)) AS OBJECT_NAME ,NULL SUBOBJECT_NAME ,TABLE_ID OBJECT_ID ,(CASE WHEN TABLET_ID != 0 THEN TABLET_ID ELSE NULL END) DATA_OBJECT_ID ,CASE WHEN TABLE_TYPE IN (0,3,6,8,9,14) THEN 'TABLE' WHEN TABLE_TYPE IN (2) THEN 'VIRTUAL TABLE' WHEN TABLE_TYPE IN (1,4) THEN 'VIEW' WHEN TABLE_TYPE IN (5) THEN 'INDEX' WHEN TABLE_TYPE IN (7) THEN 'MATERIALIZED VIEW' WHEN TABLE_TYPE IN (15) THEN 'MATERIALIZED VIEW LOG' ELSE NULL END AS OBJECT_TYPE ,CAST(CASE WHEN TABLE_TYPE IN (5,15) THEN CASE WHEN INDEX_STATUS = 2 THEN 'VALID' WHEN INDEX_STATUS = 3 THEN 'CHECKING' WHEN INDEX_STATUS = 4 THEN 'INELEGIBLE' WHEN INDEX_STATUS = 5 THEN 'ERROR' ELSE 'UNUSABLE' END ELSE CASE WHEN OBJECT_STATUS = 1 THEN 'VALID' ELSE 'INVALID' END END AS CHAR(10)) AS STATUS ,CASE WHEN TABLE_TYPE IN (6,8,9) THEN 'Y' ELSE 'N' END AS TEMPORARY ,CASE WHEN TABLE_TYPE IN (0,1) THEN 'Y' ELSE 'N' END AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_TABLE WHERE TABLE_TYPE != 12 AND TABLE_TYPE != 13 AND TABLE_MODE >> 12 & 15 in (0,1) UNION ALL SELECT CST.TENANT_ID ,CST.GMT_CREATE ,CST.GMT_MODIFIED ,DB.DATABASE_ID ,CST.constraint_name AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,TBL.TABLE_ID AS OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'INDEX' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_CONSTRAINT CST, OCEANBASE.__ALL_VIRTUAL_TABLE TBL, OCEANBASE.__ALL_VIRTUAL_DATABASE DB WHERE CST.TENANT_ID = TBL.TENANT_ID AND TBL.TENANT_ID = DB.TENANT_ID AND DB.DATABASE_ID = TBL.DATABASE_ID AND TBL.TABLE_ID = CST.TABLE_ID and CST.CONSTRAINT_TYPE = 1 AND TBL.TABLE_MODE >> 12 & 15 in (0,1) UNION ALL SELECT P.TENANT_ID ,P.GMT_CREATE ,P.GMT_MODIFIED ,T.DATABASE_ID ,CAST((CASE WHEN T.DATABASE_ID = 201004 THEN T.TABLE_NAME WHEN T.TABLE_TYPE = 5 THEN SUBSTR(T.TABLE_NAME, 7 + POSITION('_' IN SUBSTR(T.TABLE_NAME, 7))) ELSE T.TABLE_NAME END) AS CHAR(128)) AS OBJECT_NAME ,P.PART_NAME SUBOBJECT_NAME ,P.PART_ID OBJECT_ID ,CASE WHEN P.TABLET_ID != 0 THEN P.TABLET_ID ELSE NULL END AS DATA_OBJECT_ID ,(CASE WHEN T.TABLE_TYPE = 5 THEN 'INDEX PARTITION' ELSE 'TABLE PARTITION' END) AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY , NULL AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_TABLE T JOIN OCEANBASE.__ALL_VIRTUAL_PART P ON T.TABLE_ID = P.TABLE_ID WHERE T.TENANT_ID = P.TENANT_ID AND T.TABLE_MODE >> 12 & 15 in (0,1) UNION ALL SELECT SUBP.TENANT_ID ,SUBP.GMT_CREATE ,SUBP.GMT_MODIFIED ,T.DATABASE_ID ,CAST((CASE WHEN T.DATABASE_ID = 201004 THEN T.TABLE_NAME WHEN T.TABLE_TYPE = 5 THEN SUBSTR(T.TABLE_NAME, 7 + POSITION('_' IN SUBSTR(T.TABLE_NAME, 7))) ELSE T.TABLE_NAME END) AS CHAR(128)) AS OBJECT_NAME ,SUBP.SUB_PART_NAME SUBOBJECT_NAME ,SUBP.SUB_PART_ID OBJECT_ID ,SUBP.TABLET_ID AS DATA_OBJECT_ID ,(CASE WHEN T.TABLE_TYPE = 5 THEN 'INDEX SUBPARTITION' ELSE 'TABLE SUBPARTITION' END) AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'Y' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_TABLE T, OCEANBASE.__ALL_VIRTUAL_PART P,OCEANBASE.__ALL_VIRTUAL_SUB_PART SUBP WHERE T.TABLE_ID =P.TABLE_ID AND P.TABLE_ID=SUBP.TABLE_ID AND P.PART_ID =SUBP.PART_ID AND T.TENANT_ID = P.TENANT_ID AND P.TENANT_ID = SUBP.TENANT_ID AND T.TABLE_MODE >> 12 & 15 in (0,1) UNION ALL SELECT P.TENANT_ID ,P.GMT_CREATE ,P.GMT_MODIFIED ,P.DATABASE_ID ,P.PACKAGE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,P.PACKAGE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,CASE WHEN TYPE = 1 THEN 'PACKAGE' WHEN TYPE = 2 THEN 'PACKAGE BODY' ELSE NULL END AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM OCEANBASE.__ALL_VIRTUAL_ERROR E WHERE P.TENANT_ID = E.TENANT_ID AND P.PACKAGE_ID = E.OBJ_ID AND (E.OBJ_TYPE = 3 OR E.OBJ_TYPE = 5)) THEN 'INVALID' WHEN TYPE = 2 AND EXISTS (SELECT OBJ_ID FROM OCEANBASE.__ALL_VIRTUAL_ERROR Eb WHERE OBJ_ID IN (SELECT PACKAGE_ID FROM OCEANBASE.__ALL_VIRTUAL_PACKAGE Pb WHERE Pb.PACKAGE_NAME = P.PACKAGE_NAME AND Pb.DATABASE_ID = P.DATABASE_ID AND Pb.TENANT_ID = P.TENANT_ID AND TYPE = 1) AND Eb.OBJ_TYPE = 3) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_PACKAGE P UNION ALL SELECT R.TENANT_ID ,R.GMT_CREATE ,R.GMT_MODIFIED ,R.DATABASE_ID ,R.ROUTINE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,R.ROUTINE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,CASE WHEN ROUTINE_TYPE = 1 THEN 'PROCEDURE' WHEN ROUTINE_TYPE = 2 THEN 'FUNCTION' ELSE NULL END AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM OCEANBASE.__ALL_VIRTUAL_ERROR E WHERE R.TENANT_ID = E.TENANT_ID AND R.ROUTINE_ID = E.OBJ_ID AND (E.OBJ_TYPE = 9 OR E.OBJ_TYPE = 12)) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_ROUTINE R WHERE (ROUTINE_TYPE = 1 OR ROUTINE_TYPE = 2) UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,TYPE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,TYPE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_TYPE UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,OBJECT_TYPE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE BODY' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_OBJECT_TYPE WHERE TYPE = 2 UNION ALL SELECT T.TENANT_ID ,T.GMT_CREATE ,T.GMT_MODIFIED ,T.DATABASE_ID ,T.TRIGGER_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,T.TRIGGER_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TRIGGER' OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM OCEANBASE.__ALL_VIRTUAL_ERROR E WHERE T.TENANT_ID = E.TENANT_ID AND T.TRIGGER_ID = E.OBJ_ID AND (E.OBJ_TYPE = 7)) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_TRIGGER T UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,SEQUENCE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,SEQUENCE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'SEQUENCE' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_SEQUENCE_OBJECT UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,SYNONYM_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,SYNONYM_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'SYNONYM' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_SYNONYM UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,CAST(201006 AS SIGNED) AS DATABASE_ID ,NAMESPACE AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,CONTEXT_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'CONTEXT' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,21 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_TENANT_CONTEXT UNION ALL SELECT TENANT_ID, GMT_CREATE, GMT_MODIFIED, DATABASE_ID, DATABASE_NAME AS OBJECT_NAME, NULL AS SUBOBJECT_NAME, DATABASE_ID AS OBJECT_ID, NULL AS DATA_OBJECT_ID, 'DATABASE' AS OBJECT_TYPE, 'VALID' AS STATUS, 'N' AS TEMPORARY, 'N' AS "GENERATED", 'N' AS SECONDARY, 0 AS NAMESPACE, NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_DATABASE UNION ALL SELECT TENANT_ID, GMT_CREATE, GMT_MODIFIED, CAST(201001 AS SIGNED) AS DATABASE_ID, TABLEGROUP_NAME AS OBJECT_NAME, NULL AS SUBOBJECT_NAME, TABLEGROUP_ID AS OBJECT_ID, NULL AS DATA_OBJECT_ID, 'TABLEGROUP' AS OBJECT_TYPE, 'VALID' AS STATUS, 'N' AS TEMPORARY, 'N' AS "GENERATED", 'N' AS SECONDARY, 0 AS NAMESPACE, NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_TABLEGROUP ) A JOIN OCEANBASE.__ALL_VIRTUAL_DATABASE B ON A.TENANT_ID = B.TENANT_ID AND A.DATABASE_ID = B.DATABASE_ID )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema.21201_21250.cpp b/src/share/inner_table/ob_inner_table_schema.21201_21250.cpp index a8aa45ad0..bd054eca2 100644 --- a/src/share/inner_table/ob_inner_table_schema.21201_21250.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21201_21250.cpp @@ -260,7 +260,7 @@ int ObInnerTableSchema::dba_objects_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(B.DATABASE_NAME AS CHAR(128)) AS OWNER, CAST(A.OBJECT_NAME AS CHAR(128)) AS OBJECT_NAME, CAST(A.SUBOBJECT_NAME AS CHAR(128)) AS SUBOBJECT_NAME, CAST(A.OBJECT_ID AS SIGNED) AS OBJECT_ID, CAST(A.DATA_OBJECT_ID AS SIGNED) AS DATA_OBJECT_ID, CAST(A.OBJECT_TYPE AS CHAR(23)) AS OBJECT_TYPE, CAST(A.GMT_CREATE AS DATETIME) AS CREATED, CAST(A.GMT_MODIFIED AS DATETIME) AS LAST_DDL_TIME, CAST(A.GMT_CREATE AS DATETIME) AS TIMESTAMP, CAST(A.STATUS AS CHAR(7)) AS STATUS, CAST(A.TEMPORARY AS CHAR(1)) AS TEMPORARY, CAST(A.`GENERATED` AS CHAR(1)) AS "GENERATED", CAST(A.SECONDARY AS CHAR(1)) AS SECONDARY, CAST(A.NAMESPACE AS SIGNED) AS NAMESPACE, CAST(A.EDITION_NAME AS CHAR(128)) AS EDITION_NAME, CAST(NULL AS CHAR(18)) AS SHARING, CAST(NULL AS CHAR(1)) AS EDITIONABLE, CAST(NULL AS CHAR(1)) AS ORACLE_MAINTAINED, CAST(NULL AS CHAR(1)) AS APPLICATION, CAST(NULL AS CHAR(1)) AS DEFAULT_COLLATION, CAST(NULL AS CHAR(1)) AS DUPLICATED, CAST(NULL AS CHAR(1)) AS SHARDED, CAST(NULL AS CHAR(1)) AS IMPORTED_OBJECT, CAST(NULL AS SIGNED) AS CREATED_APPID, CAST(NULL AS SIGNED) AS CREATED_VSNID, CAST(NULL AS SIGNED) AS MODIFIED_APPID, CAST(NULL AS SIGNED) AS MODIFIED_VSNID FROM ( SELECT CAST(0 AS SIGNED) AS TENANT_ID, USEC_TO_TIME(B.SCHEMA_VERSION) AS GMT_CREATE, USEC_TO_TIME(A.SCHEMA_VERSION) AS GMT_MODIFIED, A.DATABASE_ID, A.TABLE_NAME AS OBJECT_NAME, NULL AS SUBOBJECT_NAME, CAST(A.TABLE_ID AS SIGNED) AS OBJECT_ID, A.TABLET_ID AS DATA_OBJECT_ID, 'TABLE' AS OBJECT_TYPE, 'VALID' AS STATUS, 'N' AS TEMPORARY, 'N' AS "GENERATED", 'N' AS SECONDARY, 0 AS NAMESPACE, NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_CORE_ALL_TABLE A JOIN OCEANBASE.__ALL_VIRTUAL_CORE_ALL_TABLE B ON A.TENANT_ID = B.TENANT_ID AND B.TABLE_NAME = '__all_core_table' WHERE A.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,CAST((CASE WHEN DATABASE_ID = 201004 THEN TABLE_NAME WHEN TABLE_TYPE = 5 THEN SUBSTR(TABLE_NAME, 7 + POSITION('_' IN SUBSTR(TABLE_NAME, 7))) ELSE TABLE_NAME END) AS CHAR(128)) AS OBJECT_NAME ,NULL SUBOBJECT_NAME ,CAST(TABLE_ID AS SIGNED) AS OBJECT_ID ,(CASE WHEN TABLET_ID != 0 THEN TABLET_ID ELSE NULL END) DATA_OBJECT_ID ,CASE WHEN TABLE_TYPE IN (0,3,6,8,9,14) THEN 'TABLE' WHEN TABLE_TYPE IN (2) THEN 'VIRTUAL TABLE' WHEN TABLE_TYPE IN (1,4) THEN 'VIEW' WHEN TABLE_TYPE IN (5) THEN 'INDEX' WHEN TABLE_TYPE IN (7) THEN 'MATERIALIZED VIEW' WHEN TABLE_TYPE IN (15) THEN 'MATERIALIZED VIEW LOG' ELSE NULL END AS OBJECT_TYPE ,CAST(CASE WHEN TABLE_TYPE IN (5,15) THEN CASE WHEN INDEX_STATUS = 2 THEN 'VALID' WHEN INDEX_STATUS = 3 THEN 'CHECKING' WHEN INDEX_STATUS = 4 THEN 'INELEGIBLE' WHEN INDEX_STATUS = 5 THEN 'ERROR' ELSE 'UNUSABLE' END ELSE CASE WHEN OBJECT_STATUS = 1 THEN 'VALID' ELSE 'INVALID' END END AS CHAR(10)) AS STATUS ,CASE WHEN TABLE_TYPE IN (6,8,9) THEN 'Y' ELSE 'N' END AS TEMPORARY ,CASE WHEN TABLE_TYPE IN (0,1) THEN 'Y' ELSE 'N' END AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_TABLE WHERE TENANT_ID = 0 AND TABLE_TYPE != 12 AND TABLE_TYPE != 13 AND TABLE_MODE >> 12 & 15 in (0,1) UNION ALL SELECT CST.TENANT_ID ,CST.GMT_CREATE ,CST.GMT_MODIFIED ,DB.DATABASE_ID ,CST.constraint_name AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,CAST(TBL.TABLE_ID AS SIGNED) AS OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'INDEX' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_CONSTRAINT CST, OCEANBASE.__ALL_TABLE TBL, OCEANBASE.__ALL_DATABASE DB WHERE CST.TENANT_ID = 0 AND DB.DATABASE_ID = TBL.DATABASE_ID AND TBL.TABLE_ID = CST.TABLE_ID and CST.CONSTRAINT_TYPE = 1 and TBL.TABLE_MODE >> 12 & 15 in (0,1) UNION ALL SELECT P.TENANT_ID ,P.GMT_CREATE ,P.GMT_MODIFIED ,T.DATABASE_ID ,CAST((CASE WHEN T.DATABASE_ID = 201004 THEN T.TABLE_NAME WHEN T.TABLE_TYPE = 5 THEN SUBSTR(T.TABLE_NAME, 7 + POSITION('_' IN SUBSTR(T.TABLE_NAME, 7))) ELSE T.TABLE_NAME END) AS CHAR(128)) AS OBJECT_NAME ,P.PART_NAME SUBOBJECT_NAME ,P.PART_ID OBJECT_ID ,CASE WHEN P.TABLET_ID != 0 THEN P.TABLET_ID ELSE NULL END AS DATA_OBJECT_ID ,(CASE WHEN T.TABLE_TYPE = 5 THEN 'INDEX PARTITION' ELSE 'TABLE PARTITION' END) AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY , NULL AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_TABLE T JOIN OCEANBASE.__ALL_PART P ON T.TABLE_ID = P.TABLE_ID WHERE T.TENANT_ID = 0 AND T.TENANT_ID = P.TENANT_ID AND T.TABLE_MODE >> 12 & 15 in (0,1) UNION ALL SELECT SUBP.TENANT_ID ,SUBP.GMT_CREATE ,SUBP.GMT_MODIFIED ,T.DATABASE_ID ,CAST((CASE WHEN T.DATABASE_ID = 201004 THEN T.TABLE_NAME WHEN T.TABLE_TYPE = 5 THEN SUBSTR(T.TABLE_NAME, 7 + POSITION('_' IN SUBSTR(T.TABLE_NAME, 7))) ELSE T.TABLE_NAME END) AS CHAR(128)) AS OBJECT_NAME ,SUBP.SUB_PART_NAME SUBOBJECT_NAME ,SUBP.SUB_PART_ID OBJECT_ID ,SUBP.TABLET_ID AS DATA_OBJECT_ID ,(CASE WHEN T.TABLE_TYPE = 5 THEN 'INDEX SUBPARTITION' ELSE 'TABLE SUBPARTITION' END) AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_TABLE T, OCEANBASE.__ALL_PART P,OCEANBASE.__ALL_SUB_PART SUBP WHERE T.TABLE_ID =P.TABLE_ID AND P.TABLE_ID=SUBP.TABLE_ID AND P.PART_ID =SUBP.PART_ID AND T.TENANT_ID = 0 AND T.TENANT_ID = P.TENANT_ID AND P.TENANT_ID = SUBP.TENANT_ID AND T.TABLE_MODE >> 12 & 15 in (0,1) UNION ALL SELECT P.TENANT_ID ,P.GMT_CREATE ,P.GMT_MODIFIED ,P.DATABASE_ID ,P.PACKAGE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,CAST(P.PACKAGE_ID AS SIGNED) AS OBJECT_ID ,NULL AS DATA_OBJECT_ID ,CASE WHEN TYPE = 1 THEN 'PACKAGE' WHEN TYPE = 2 THEN 'PACKAGE BODY' ELSE NULL END AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM OCEANBASE.__ALL_TENANT_ERROR E WHERE P.TENANT_ID = E.TENANT_ID AND P.PACKAGE_ID = E.OBJ_ID AND (E.OBJ_TYPE = 3 OR E.OBJ_TYPE = 5)) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_PACKAGE P WHERE P.TENANT_ID = 0 UNION ALL SELECT R.TENANT_ID ,R.GMT_CREATE ,R.GMT_MODIFIED ,R.DATABASE_ID ,R.ROUTINE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,CAST(R.ROUTINE_ID AS SIGNED) AS OBJECT_ID ,NULL AS DATA_OBJECT_ID ,CASE WHEN ROUTINE_TYPE = 1 THEN 'PROCEDURE' WHEN ROUTINE_TYPE = 2 THEN 'FUNCTION' ELSE NULL END AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM OCEANBASE.__ALL_TENANT_ERROR E WHERE R.TENANT_ID = E.TENANT_ID AND R.ROUTINE_ID = E.OBJ_ID AND (E.OBJ_TYPE = 9 OR E.OBJ_TYPE = 12)) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_ROUTINE R WHERE (ROUTINE_TYPE = 1 OR ROUTINE_TYPE = 2) AND R.TENANT_ID = 0 UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,TYPE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,CAST(TYPE_ID AS SIGNED) AS OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_TYPE WHERE TENANT_ID = 0 UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,CAST(OBJECT_TYPE_ID AS SIGNED) AS OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE BODY' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_TENANT_OBJECT_TYPE WHERE TENANT_ID = 0 AND TYPE = 2 UNION ALL SELECT TENANT_ID ,T.GMT_CREATE ,T.GMT_MODIFIED ,T.DATABASE_ID ,T.TRIGGER_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,CAST(T.TRIGGER_ID AS SIGNED) AS OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TRIGGER' OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM OCEANBASE.__ALL_TENANT_ERROR E WHERE T.TENANT_ID = E.TENANT_ID AND T.TRIGGER_ID = E.OBJ_ID AND (E.OBJ_TYPE = 7)) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_TENANT_TRIGGER T WHERE T.TENANT_ID = 0 UNION ALL SELECT TENANT_ID, GMT_CREATE, GMT_MODIFIED, DATABASE_ID, DATABASE_NAME AS OBJECT_NAME, NULL AS SUBOBJECT_NAME, CAST(DATABASE_ID AS SIGNED) AS OBJECT_ID, NULL AS DATA_OBJECT_ID, 'DATABASE' AS OBJECT_TYPE, 'VALID' AS STATUS, 'N' AS TEMPORARY, 'N' AS "GENERATED", 'N' AS SECONDARY, 0 AS NAMESPACE, NULL AS EDITION_NAME FROM OCEANBASE.__ALL_DATABASE WHERE TENANT_ID = 0 UNION ALL SELECT TENANT_ID, GMT_CREATE, GMT_MODIFIED, CAST(201001 AS SIGNED) AS DATABASE_ID, TABLEGROUP_NAME AS OBJECT_NAME, NULL AS SUBOBJECT_NAME, CAST(TABLEGROUP_ID AS SIGNED) AS OBJECT_ID, NULL AS DATA_OBJECT_ID, 'TABLEGROUP' AS OBJECT_TYPE, 'VALID' AS STATUS, 'N' AS TEMPORARY, 'N' AS "GENERATED", 'N' AS SECONDARY, 0 AS NAMESPACE, NULL AS EDITION_NAME FROM OCEANBASE.__ALL_TABLEGROUP WHERE TENANT_ID = 0 ) A JOIN OCEANBASE.__ALL_DATABASE B ON A.TENANT_ID = B.TENANT_ID AND A.DATABASE_ID = B.DATABASE_ID AND B.TENANT_ID = 0 )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(B.DATABASE_NAME AS CHAR(128)) AS OWNER, CAST(A.OBJECT_NAME AS CHAR(128)) AS OBJECT_NAME, CAST(A.SUBOBJECT_NAME AS CHAR(128)) AS SUBOBJECT_NAME, CAST(A.OBJECT_ID AS SIGNED) AS OBJECT_ID, CAST(A.DATA_OBJECT_ID AS SIGNED) AS DATA_OBJECT_ID, CAST(A.OBJECT_TYPE AS CHAR(23)) AS OBJECT_TYPE, CAST(A.GMT_CREATE AS DATETIME) AS CREATED, CAST(A.GMT_MODIFIED AS DATETIME) AS LAST_DDL_TIME, CAST(A.GMT_CREATE AS DATETIME) AS TIMESTAMP, CAST(A.STATUS AS CHAR(7)) AS STATUS, CAST(A.TEMPORARY AS CHAR(1)) AS TEMPORARY, CAST(A.`GENERATED` AS CHAR(1)) AS "GENERATED", CAST(A.SECONDARY AS CHAR(1)) AS SECONDARY, CAST(A.NAMESPACE AS SIGNED) AS NAMESPACE, CAST(A.EDITION_NAME AS CHAR(128)) AS EDITION_NAME, CAST(NULL AS CHAR(18)) AS SHARING, CAST(NULL AS CHAR(1)) AS EDITIONABLE, CAST(NULL AS CHAR(1)) AS ORACLE_MAINTAINED, CAST(NULL AS CHAR(1)) AS APPLICATION, CAST(NULL AS CHAR(1)) AS DEFAULT_COLLATION, CAST(NULL AS CHAR(1)) AS DUPLICATED, CAST(NULL AS CHAR(1)) AS SHARDED, CAST(NULL AS CHAR(1)) AS IMPORTED_OBJECT, CAST(NULL AS SIGNED) AS CREATED_APPID, CAST(NULL AS SIGNED) AS CREATED_VSNID, CAST(NULL AS SIGNED) AS MODIFIED_APPID, CAST(NULL AS SIGNED) AS MODIFIED_VSNID FROM ( SELECT CAST(0 AS SIGNED) AS TENANT_ID, USEC_TO_TIME(B.SCHEMA_VERSION) AS GMT_CREATE, USEC_TO_TIME(A.SCHEMA_VERSION) AS GMT_MODIFIED, A.DATABASE_ID, A.TABLE_NAME AS OBJECT_NAME, NULL AS SUBOBJECT_NAME, CAST(A.TABLE_ID AS SIGNED) AS OBJECT_ID, A.TABLET_ID AS DATA_OBJECT_ID, 'TABLE' AS OBJECT_TYPE, 'VALID' AS STATUS, 'N' AS TEMPORARY, 'N' AS "GENERATED", 'N' AS SECONDARY, 0 AS NAMESPACE, NULL AS EDITION_NAME FROM OCEANBASE.__ALL_VIRTUAL_CORE_ALL_TABLE A JOIN OCEANBASE.__ALL_VIRTUAL_CORE_ALL_TABLE B ON A.TENANT_ID = B.TENANT_ID AND B.TABLE_NAME = '__all_core_table' WHERE A.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,CAST((CASE WHEN DATABASE_ID = 201004 THEN TABLE_NAME WHEN TABLE_TYPE = 5 THEN SUBSTR(TABLE_NAME, 7 + POSITION('_' IN SUBSTR(TABLE_NAME, 7))) ELSE TABLE_NAME END) AS CHAR(128)) AS OBJECT_NAME ,NULL SUBOBJECT_NAME ,CAST(TABLE_ID AS SIGNED) AS OBJECT_ID ,(CASE WHEN TABLET_ID != 0 THEN TABLET_ID ELSE NULL END) DATA_OBJECT_ID ,CASE WHEN TABLE_TYPE IN (0,3,6,8,9,14) THEN 'TABLE' WHEN TABLE_TYPE IN (2) THEN 'VIRTUAL TABLE' WHEN TABLE_TYPE IN (1,4) THEN 'VIEW' WHEN TABLE_TYPE IN (5) THEN 'INDEX' WHEN TABLE_TYPE IN (7) THEN 'MATERIALIZED VIEW' WHEN TABLE_TYPE IN (15) THEN 'MATERIALIZED VIEW LOG' ELSE NULL END AS OBJECT_TYPE ,CAST(CASE WHEN TABLE_TYPE IN (5,15) THEN CASE WHEN INDEX_STATUS = 2 THEN 'VALID' WHEN INDEX_STATUS = 3 THEN 'CHECKING' WHEN INDEX_STATUS = 4 THEN 'INELEGIBLE' WHEN INDEX_STATUS = 5 THEN 'ERROR' ELSE 'UNUSABLE' END ELSE CASE WHEN OBJECT_STATUS = 1 THEN 'VALID' ELSE 'INVALID' END END AS CHAR(10)) AS STATUS ,CASE WHEN TABLE_TYPE IN (6,8,9) THEN 'Y' ELSE 'N' END AS TEMPORARY ,CASE WHEN TABLE_TYPE IN (0,1) THEN 'Y' ELSE 'N' END AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_TABLE WHERE TENANT_ID = 0 AND TABLE_TYPE != 12 AND TABLE_TYPE != 13 AND TABLE_MODE >> 12 & 15 in (0,1) UNION ALL SELECT CST.TENANT_ID ,CST.GMT_CREATE ,CST.GMT_MODIFIED ,DB.DATABASE_ID ,CST.constraint_name AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,CAST(TBL.TABLE_ID AS SIGNED) AS OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'INDEX' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_CONSTRAINT CST, OCEANBASE.__ALL_TABLE TBL, OCEANBASE.__ALL_DATABASE DB WHERE CST.TENANT_ID = 0 AND DB.DATABASE_ID = TBL.DATABASE_ID AND TBL.TABLE_ID = CST.TABLE_ID and CST.CONSTRAINT_TYPE = 1 and TBL.TABLE_MODE >> 12 & 15 in (0,1) UNION ALL SELECT P.TENANT_ID ,P.GMT_CREATE ,P.GMT_MODIFIED ,T.DATABASE_ID ,CAST((CASE WHEN T.DATABASE_ID = 201004 THEN T.TABLE_NAME WHEN T.TABLE_TYPE = 5 THEN SUBSTR(T.TABLE_NAME, 7 + POSITION('_' IN SUBSTR(T.TABLE_NAME, 7))) ELSE T.TABLE_NAME END) AS CHAR(128)) AS OBJECT_NAME ,P.PART_NAME SUBOBJECT_NAME ,P.PART_ID OBJECT_ID ,CASE WHEN P.TABLET_ID != 0 THEN P.TABLET_ID ELSE NULL END AS DATA_OBJECT_ID ,(CASE WHEN T.TABLE_TYPE = 5 THEN 'INDEX PARTITION' ELSE 'TABLE PARTITION' END) AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY , NULL AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_TABLE T JOIN OCEANBASE.__ALL_PART P ON T.TABLE_ID = P.TABLE_ID WHERE T.TENANT_ID = 0 AND T.TENANT_ID = P.TENANT_ID AND T.TABLE_MODE >> 12 & 15 in (0,1) UNION ALL SELECT SUBP.TENANT_ID ,SUBP.GMT_CREATE ,SUBP.GMT_MODIFIED ,T.DATABASE_ID ,CAST((CASE WHEN T.DATABASE_ID = 201004 THEN T.TABLE_NAME WHEN T.TABLE_TYPE = 5 THEN SUBSTR(T.TABLE_NAME, 7 + POSITION('_' IN SUBSTR(T.TABLE_NAME, 7))) ELSE T.TABLE_NAME END) AS CHAR(128)) AS OBJECT_NAME ,SUBP.SUB_PART_NAME SUBOBJECT_NAME ,SUBP.SUB_PART_ID OBJECT_ID ,SUBP.TABLET_ID AS DATA_OBJECT_ID ,(CASE WHEN T.TABLE_TYPE = 5 THEN 'INDEX SUBPARTITION' ELSE 'TABLE SUBPARTITION' END) AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_TABLE T, OCEANBASE.__ALL_PART P,OCEANBASE.__ALL_SUB_PART SUBP WHERE T.TABLE_ID =P.TABLE_ID AND P.TABLE_ID=SUBP.TABLE_ID AND P.PART_ID =SUBP.PART_ID AND T.TENANT_ID = 0 AND T.TENANT_ID = P.TENANT_ID AND P.TENANT_ID = SUBP.TENANT_ID AND T.TABLE_MODE >> 12 & 15 in (0,1) UNION ALL SELECT P.TENANT_ID ,P.GMT_CREATE ,P.GMT_MODIFIED ,P.DATABASE_ID ,P.PACKAGE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,CAST(P.PACKAGE_ID AS SIGNED) AS OBJECT_ID ,NULL AS DATA_OBJECT_ID ,CASE WHEN TYPE = 1 THEN 'PACKAGE' WHEN TYPE = 2 THEN 'PACKAGE BODY' ELSE NULL END AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM OCEANBASE.__ALL_TENANT_ERROR E WHERE P.TENANT_ID = E.TENANT_ID AND P.PACKAGE_ID = E.OBJ_ID AND (E.OBJ_TYPE = 3 OR E.OBJ_TYPE = 5)) THEN 'INVALID' WHEN TYPE = 2 AND EXISTS (SELECT OBJ_ID FROM OCEANBASE.__ALL_TENANT_ERROR Eb WHERE OBJ_ID IN (SELECT PACKAGE_ID FROM OCEANBASE.__ALL_PACKAGE Pb WHERE Pb.PACKAGE_NAME = P.PACKAGE_NAME AND Pb.DATABASE_ID = P.DATABASE_ID AND Pb.TENANT_ID = P.TENANT_ID AND TYPE = 1) AND Eb.OBJ_TYPE = 3) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_PACKAGE P WHERE P.TENANT_ID = 0 UNION ALL SELECT R.TENANT_ID ,R.GMT_CREATE ,R.GMT_MODIFIED ,R.DATABASE_ID ,R.ROUTINE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,CAST(R.ROUTINE_ID AS SIGNED) AS OBJECT_ID ,NULL AS DATA_OBJECT_ID ,CASE WHEN ROUTINE_TYPE = 1 THEN 'PROCEDURE' WHEN ROUTINE_TYPE = 2 THEN 'FUNCTION' ELSE NULL END AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM OCEANBASE.__ALL_TENANT_ERROR E WHERE R.TENANT_ID = E.TENANT_ID AND R.ROUTINE_ID = E.OBJ_ID AND (E.OBJ_TYPE = 9 OR E.OBJ_TYPE = 12)) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_ROUTINE R WHERE (ROUTINE_TYPE = 1 OR ROUTINE_TYPE = 2) AND R.TENANT_ID = 0 UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,TYPE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,CAST(TYPE_ID AS SIGNED) AS OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_TYPE WHERE TENANT_ID = 0 UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,CAST(OBJECT_TYPE_ID AS SIGNED) AS OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE BODY' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_TENANT_OBJECT_TYPE WHERE TENANT_ID = 0 AND TYPE = 2 UNION ALL SELECT TENANT_ID ,T.GMT_CREATE ,T.GMT_MODIFIED ,T.DATABASE_ID ,T.TRIGGER_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,CAST(T.TRIGGER_ID AS SIGNED) AS OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TRIGGER' OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM OCEANBASE.__ALL_TENANT_ERROR E WHERE T.TENANT_ID = E.TENANT_ID AND T.TRIGGER_ID = E.OBJ_ID AND (E.OBJ_TYPE = 7)) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM OCEANBASE.__ALL_TENANT_TRIGGER T WHERE T.TENANT_ID = 0 UNION ALL SELECT TENANT_ID, GMT_CREATE, GMT_MODIFIED, DATABASE_ID, DATABASE_NAME AS OBJECT_NAME, NULL AS SUBOBJECT_NAME, CAST(DATABASE_ID AS SIGNED) AS OBJECT_ID, NULL AS DATA_OBJECT_ID, 'DATABASE' AS OBJECT_TYPE, 'VALID' AS STATUS, 'N' AS TEMPORARY, 'N' AS "GENERATED", 'N' AS SECONDARY, 0 AS NAMESPACE, NULL AS EDITION_NAME FROM OCEANBASE.__ALL_DATABASE WHERE TENANT_ID = 0 UNION ALL SELECT TENANT_ID, GMT_CREATE, GMT_MODIFIED, CAST(201001 AS SIGNED) AS DATABASE_ID, TABLEGROUP_NAME AS OBJECT_NAME, NULL AS SUBOBJECT_NAME, CAST(TABLEGROUP_ID AS SIGNED) AS OBJECT_ID, NULL AS DATA_OBJECT_ID, 'TABLEGROUP' AS OBJECT_TYPE, 'VALID' AS STATUS, 'N' AS TEMPORARY, 'N' AS "GENERATED", 'N' AS SECONDARY, 0 AS NAMESPACE, NULL AS EDITION_NAME FROM OCEANBASE.__ALL_TABLEGROUP WHERE TENANT_ID = 0 ) A JOIN OCEANBASE.__ALL_DATABASE B ON A.TENANT_ID = B.TENANT_ID AND A.DATABASE_ID = B.DATABASE_ID AND B.TENANT_ID = 0 )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema.25001_25050.cpp b/src/share/inner_table/ob_inner_table_schema.25001_25050.cpp index 694b864fd..053971b8e 100644 --- a/src/share/inner_table/ob_inner_table_schema.25001_25050.cpp +++ b/src/share/inner_table/ob_inner_table_schema.25001_25050.cpp @@ -110,7 +110,7 @@ int ObInnerTableSchema::dba_objects_ora_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(B.DATABASE_NAME AS VARCHAR(128)) AS OWNER, CAST(A.OBJECT_NAME AS VARCHAR(128)) AS OBJECT_NAME, CAST(A.SUBOBJECT_NAME AS VARCHAR2(128)) AS SUBOBJECT_NAME, CAST(A.OBJECT_ID AS NUMBER) AS OBJECT_ID, CAST(A.DATA_OBJECT_ID AS NUMBER) AS DATA_OBJECT_ID, CAST(A.OBJECT_TYPE AS VARCHAR2(23)) AS OBJECT_TYPE, CAST(A.GMT_CREATE AS DATE) AS CREATED, CAST(A.GMT_MODIFIED AS DATE) AS LAST_DDL_TIME, CAST(TO_CHAR(A.GMT_CREATE) AS VARCHAR2(19)) AS TIMESTAMP, CAST(A.STATUS AS VARCHAR2(7)) AS STATUS, CAST(A.TEMPORARY AS VARCHAR2(1)) AS TEMPORARY, CAST(A."GENERATED" AS VARCHAR2(1)) AS "GENERATED", CAST(A.SECONDARY AS VARCHAR2(1)) AS SECONDARY, CAST(A.NAMESPACE AS NUMBER) AS NAMESPACE, CAST(A.EDITION_NAME AS VARCHAR2(128)) AS EDITION_NAME, CAST(NULL AS VARCHAR2(18)) AS SHARING, CAST(NULL AS VARCHAR2(1)) AS EDITIONABLE, CAST(NULL AS VARCHAR2(1)) AS ORACLE_MAINTAINED, CAST(NULL AS VARCHAR2(1)) AS APPLICATION, CAST(NULL AS VARCHAR2(1)) AS DEFAULT_COLLATION, CAST(NULL AS VARCHAR2(1)) AS DUPLICATED, CAST(NULL AS VARCHAR2(1)) AS SHARDED, CAST(NULL AS VARCHAR2(1)) AS IMPORTED_OBJECT, CAST(NULL AS NUMBER) AS CREATED_APPID, CAST(NULL AS NUMBER) AS CREATED_VSNID, CAST(NULL AS NUMBER) AS MODIFIED_APPID, CAST(NULL AS NUMBER) AS MODIFIED_VSNID FROM ( SELECT A.TENANT_ID, (TO_DATE('19700101','YYYYMMDD') + B.SCHEMA_VERSION / 86400 / 1000000 + TO_NUMBER(SUBSTR(TZ_OFFSET(SESSIONTIMEZONE),1,3))/24) AS GMT_CREATE, (TO_DATE('19700101','YYYYMMDD') + A.SCHEMA_VERSION / 86400 / 1000000 + TO_NUMBER(SUBSTR(TZ_OFFSET(SESSIONTIMEZONE),1,3))/24) AS GMT_MODIFIED, A.DATABASE_ID, A.TABLE_NAME AS OBJECT_NAME, NULL AS SUBOBJECT_NAME, CAST(A.TABLE_ID AS NUMBER) AS OBJECT_ID, A.TABLE_ID AS DATA_OBJECT_ID, 'TABLE' AS OBJECT_TYPE, 'VALID' AS STATUS, 'N' AS TEMPORARY, 'N' AS "GENERATED", 'N' AS SECONDARY, 0 AS NAMESPACE, NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_CORE_ALL_TABLE A JOIN SYS.ALL_VIRTUAL_CORE_ALL_TABLE B ON A.TENANT_ID = B.TENANT_ID WHERE B.TABLE_NAME = '__all_core_table' AND A.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,CAST((CASE WHEN DATABASE_ID = 201004 THEN TABLE_NAME WHEN TABLE_TYPE = 5 THEN SUBSTR(TABLE_NAME, 7 + INSTR(SUBSTR(TABLE_NAME, 7), '_')) ELSE TABLE_NAME END) AS VARCHAR2(128)) AS OBJECT_NAME ,NULL SUBOBJECT_NAME ,TABLE_ID OBJECT_ID ,(CASE WHEN TABLET_ID != 0 THEN TABLET_ID ELSE NULL END) DATA_OBJECT_ID ,CASE WHEN TABLE_TYPE IN (0,3,6,8,9,14) THEN 'TABLE' WHEN TABLE_TYPE IN (2) THEN 'VIRTUAL TABLE' WHEN TABLE_TYPE IN (1,4) THEN 'VIEW' WHEN TABLE_TYPE IN (5) THEN 'INDEX' WHEN TABLE_TYPE IN (7) THEN 'MATERIALIZED VIEW' WHEN TABLE_TYPE IN (15) THEN 'MATERIALIZED VIEW LOG' ELSE NULL END AS OBJECT_TYPE ,CAST(CASE WHEN TABLE_TYPE IN (5) THEN CASE WHEN INDEX_STATUS = 2 THEN 'VALID' WHEN INDEX_STATUS = 3 THEN 'CHECKING' WHEN INDEX_STATUS = 4 THEN 'INELEGIBLE' WHEN INDEX_STATUS = 5 THEN 'ERROR' ELSE 'UNUSABLE' END ELSE CASE WHEN OBJECT_STATUS = 1 THEN 'VALID' ELSE 'INVALID' END END AS VARCHAR2(10)) AS STATUS ,CASE WHEN TABLE_TYPE IN (6,8,9) THEN 'Y' ELSE 'N' END AS TEMPORARY ,CASE WHEN TABLE_TYPE IN (0,1) THEN 'Y' ELSE 'N' END AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() AND TABLE_TYPE != 12 AND TABLE_TYPE != 13 AND bitand((TABLE_MODE / 4096), 15) IN (0,1) UNION ALL SELECT CST.TENANT_ID ,CST.GMT_CREATE ,CST.GMT_MODIFIED ,DB.DATABASE_ID ,CST.CONSTRAINT_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,TBL.TABLE_ID AS OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'INDEX' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_CONSTRAINT_REAL_AGENT CST, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT TBL, SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB WHERE CST.TENANT_ID = EFFECTIVE_TENANT_ID() AND DB.DATABASE_ID = TBL.DATABASE_ID AND TBL.TABLE_ID = CST.TABLE_ID and CST.CONSTRAINT_TYPE = 1 AND TBL.TABLE_TYPE != 12 AND TBL.TABLE_TYPE != 13 AND bitand((TBL.TABLE_MODE / 4096), 15) IN (0,1) UNION ALL SELECT P.TENANT_ID ,P.GMT_CREATE ,P.GMT_MODIFIED ,T.DATABASE_ID ,CAST((CASE WHEN T.DATABASE_ID = 201004 THEN T.TABLE_NAME WHEN T.TABLE_TYPE = 5 THEN SUBSTR(T.TABLE_NAME, 7 + INSTR(SUBSTR(T.TABLE_NAME, 7), '_')) ELSE T.TABLE_NAME END) AS VARCHAR2(128)) AS OBJECT_NAME ,P.PART_NAME SUBOBJECT_NAME ,P.PART_ID OBJECT_ID ,CASE WHEN P.TABLET_ID != 0 THEN P.TABLET_ID ELSE NULL END AS DATA_OBJECT_ID ,DECODE (T.TABLE_TYPE, 5, 'INDEX PARTITION', 'TABLE PARTITION') AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY , NULL AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T JOIN SYS.ALL_VIRTUAL_PART_REAL_AGENT P ON T.TABLE_ID = P.TABLE_ID WHERE T.TENANT_ID = EFFECTIVE_TENANT_ID() AND P.TENANT_ID = EFFECTIVE_TENANT_ID() AND T.TABLE_TYPE != 12 AND T.TABLE_TYPE != 13 AND bitand((T.TABLE_MODE / 4096), 15) IN (0,1) UNION ALL SELECT SUBP.TENANT_ID ,SUBP.GMT_CREATE ,SUBP.GMT_MODIFIED ,T.DATABASE_ID ,CAST((CASE WHEN T.DATABASE_ID = 201004 THEN T.TABLE_NAME WHEN T.TABLE_TYPE = 5 THEN SUBSTR(T.TABLE_NAME, 7 + INSTR(SUBSTR(T.TABLE_NAME, 7), '_')) ELSE T.TABLE_NAME END) AS VARCHAR2(128)) AS OBJECT_NAME ,SUBP.SUB_PART_NAME SUBOBJECT_NAME ,SUBP.SUB_PART_ID OBJECT_ID ,SUBP.TABLET_ID AS DATA_OBJECT_ID ,DECODE (T.TABLE_TYPE, 5, 'INDEX SUBPARTITION', 'TABLE SUBPARTITION') AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'Y' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T, SYS.ALL_VIRTUAL_PART_REAL_AGENT P,SYS.ALL_VIRTUAL_SUB_PART_REAL_AGENT SUBP WHERE T.TABLE_ID =P.TABLE_ID AND P.TABLE_ID=SUBP.TABLE_ID AND P.PART_ID =SUBP.PART_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() AND P.TENANT_ID = EFFECTIVE_TENANT_ID() AND SUBP.TENANT_ID = EFFECTIVE_TENANT_ID() AND T.TABLE_TYPE != 12 AND T.TABLE_TYPE != 13 AND bitand((T.TABLE_MODE / 4096), 15) IN (0,1) UNION ALL SELECT EFFECTIVE_TENANT_ID() AS TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,CAST(201006 AS NUMBER) AS DATABASE_ID ,PACKAGE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,PACKAGE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,CASE WHEN TYPE = 1 THEN 'PACKAGE' WHEN TYPE = 2 THEN 'PACKAGE BODY' ELSE NULL END AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_PACKAGE_SYS_AGENT UNION ALL SELECT P.TENANT_ID ,P.GMT_CREATE ,P.GMT_MODIFIED ,P.DATABASE_ID ,P.PACKAGE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,P.PACKAGE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,CASE WHEN TYPE = 1 THEN 'PACKAGE' WHEN TYPE = 2 THEN 'PACKAGE BODY' ELSE NULL END AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE P.TENANT_ID = E.TENANT_ID AND P.PACKAGE_ID = E.OBJ_ID AND (E.OBJ_TYPE = 3 OR E.OBJ_TYPE = 5)) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_PACKAGE_REAL_AGENT P WHERE P.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT R.TENANT_ID ,R.GMT_CREATE ,R.GMT_MODIFIED ,R.DATABASE_ID ,R.ROUTINE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,R.ROUTINE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,CASE WHEN ROUTINE_TYPE = 1 THEN 'PROCEDURE' WHEN ROUTINE_TYPE = 2 THEN 'FUNCTION' ELSE NULL END AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE R.TENANT_ID = E.TENANT_ID AND R.ROUTINE_ID = E.OBJ_ID AND (E.OBJ_TYPE = 9 OR E.OBJ_TYPE = 12)) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_ROUTINE_REAL_AGENT R WHERE (ROUTINE_TYPE = 1 OR ROUTINE_TYPE = 2) AND R.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT EFFECTIVE_TENANT_ID() AS TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,CAST(201006 AS NUMBER) AS DATABASE_ID ,TS.TYPE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,TS.TYPE_ID AS OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE' AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE TS.TENANT_ID = E.TENANT_ID AND TS.TYPE_ID = E.OBJ_ID AND E.OBJ_TYPE = 4) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,TYPE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,TYPE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE' AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE TY.TENANT_ID = E.TENANT_ID AND TY.TYPE_ID = E.OBJ_ID AND E.OBJ_TYPE = 4) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TYPE_REAL_AGENT TY WHERE TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT EFFECTIVE_TENANT_ID() AS TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,CAST(201006 AS NUMBER) AS DATABASE_ID ,OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,OBJECT_TYPE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE BODY' AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE EFFECTIVE_TENANT_ID() = E.TENANT_ID AND TS.OBJECT_TYPE_ID = E.OBJ_ID AND E.OBJ_TYPE = 6) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TENANT_OBJECT_TYPE_SYS_AGENT TS WHERE TYPE = 2 UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,OBJECT_TYPE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE BODY' AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE TY.TENANT_ID = E.TENANT_ID AND TY.OBJECT_TYPE_ID = E.OBJ_ID AND E.OBJ_TYPE = 6) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TENANT_OBJECT_TYPE_REAL_AGENT TY WHERE TENANT_ID = EFFECTIVE_TENANT_ID() and TYPE = 2 UNION ALL SELECT T.TENANT_ID ,T.GMT_CREATE ,T.GMT_MODIFIED ,T.DATABASE_ID ,T.TRIGGER_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,T.TRIGGER_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TRIGGER' OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE T.TENANT_ID = E.TENANT_ID AND T.TRIGGER_ID = E.OBJ_ID AND (E.OBJ_TYPE = 7)) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT T WHERE T.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,SEQUENCE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,SEQUENCE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'SEQUENCE' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_SEQUENCE_OBJECT_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,SYNONYM_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,SYNONYM_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'SYNONYM' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_SYNONYM_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,CAST(201006 AS NUMBER) AS DATABASE_ID ,NAMESPACE AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,CONTEXT_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'CONTEXT' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,21 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_CONTEXT_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID, GMT_CREATE, GMT_MODIFIED, DATABASE_ID, DATABASE_NAME AS OBJECT_NAME, NULL AS SUBOBJECT_NAME, DATABASE_ID AS OBJECT_ID, NULL AS DATA_OBJECT_ID, 'DATABASE' AS OBJECT_TYPE, 'VALID' AS STATUS, 'N' AS TEMPORARY, 'N' AS "GENERATED", 'N' AS SECONDARY, 0 AS NAMESPACE, NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID, GMT_CREATE, GMT_MODIFIED, CAST(201001 AS NUMBER) AS DATABASE_ID, TABLEGROUP_NAME AS OBJECT_NAME, NULL AS SUBOBJECT_NAME, TABLEGROUP_ID AS OBJECT_ID, NULL AS DATA_OBJECT_ID, 'TABLEGROUP' AS OBJECT_TYPE, 'VALID' AS STATUS, 'N' AS TEMPORARY, 'N' AS "GENERATED", 'N' AS SECONDARY, 0 AS NAMESPACE, NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TABLEGROUP_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() ) A JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT B ON A.TENANT_ID = B.TENANT_ID AND A.DATABASE_ID = B.DATABASE_ID AND B.TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(B.DATABASE_NAME AS VARCHAR(128)) AS OWNER, CAST(A.OBJECT_NAME AS VARCHAR(128)) AS OBJECT_NAME, CAST(A.SUBOBJECT_NAME AS VARCHAR2(128)) AS SUBOBJECT_NAME, CAST(A.OBJECT_ID AS NUMBER) AS OBJECT_ID, CAST(A.DATA_OBJECT_ID AS NUMBER) AS DATA_OBJECT_ID, CAST(A.OBJECT_TYPE AS VARCHAR2(23)) AS OBJECT_TYPE, CAST(A.GMT_CREATE AS DATE) AS CREATED, CAST(A.GMT_MODIFIED AS DATE) AS LAST_DDL_TIME, CAST(TO_CHAR(A.GMT_CREATE) AS VARCHAR2(19)) AS TIMESTAMP, CAST(A.STATUS AS VARCHAR2(7)) AS STATUS, CAST(A.TEMPORARY AS VARCHAR2(1)) AS TEMPORARY, CAST(A."GENERATED" AS VARCHAR2(1)) AS "GENERATED", CAST(A.SECONDARY AS VARCHAR2(1)) AS SECONDARY, CAST(A.NAMESPACE AS NUMBER) AS NAMESPACE, CAST(A.EDITION_NAME AS VARCHAR2(128)) AS EDITION_NAME, CAST(NULL AS VARCHAR2(18)) AS SHARING, CAST(NULL AS VARCHAR2(1)) AS EDITIONABLE, CAST(NULL AS VARCHAR2(1)) AS ORACLE_MAINTAINED, CAST(NULL AS VARCHAR2(1)) AS APPLICATION, CAST(NULL AS VARCHAR2(1)) AS DEFAULT_COLLATION, CAST(NULL AS VARCHAR2(1)) AS DUPLICATED, CAST(NULL AS VARCHAR2(1)) AS SHARDED, CAST(NULL AS VARCHAR2(1)) AS IMPORTED_OBJECT, CAST(NULL AS NUMBER) AS CREATED_APPID, CAST(NULL AS NUMBER) AS CREATED_VSNID, CAST(NULL AS NUMBER) AS MODIFIED_APPID, CAST(NULL AS NUMBER) AS MODIFIED_VSNID FROM ( SELECT A.TENANT_ID, (TO_DATE('19700101','YYYYMMDD') + B.SCHEMA_VERSION / 86400 / 1000000 + TO_NUMBER(SUBSTR(TZ_OFFSET(SESSIONTIMEZONE),1,3))/24) AS GMT_CREATE, (TO_DATE('19700101','YYYYMMDD') + A.SCHEMA_VERSION / 86400 / 1000000 + TO_NUMBER(SUBSTR(TZ_OFFSET(SESSIONTIMEZONE),1,3))/24) AS GMT_MODIFIED, A.DATABASE_ID, A.TABLE_NAME AS OBJECT_NAME, NULL AS SUBOBJECT_NAME, CAST(A.TABLE_ID AS NUMBER) AS OBJECT_ID, A.TABLE_ID AS DATA_OBJECT_ID, 'TABLE' AS OBJECT_TYPE, 'VALID' AS STATUS, 'N' AS TEMPORARY, 'N' AS "GENERATED", 'N' AS SECONDARY, 0 AS NAMESPACE, NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_CORE_ALL_TABLE A JOIN SYS.ALL_VIRTUAL_CORE_ALL_TABLE B ON A.TENANT_ID = B.TENANT_ID WHERE B.TABLE_NAME = '__all_core_table' AND A.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,CAST((CASE WHEN DATABASE_ID = 201004 THEN TABLE_NAME WHEN TABLE_TYPE = 5 THEN SUBSTR(TABLE_NAME, 7 + INSTR(SUBSTR(TABLE_NAME, 7), '_')) ELSE TABLE_NAME END) AS VARCHAR2(128)) AS OBJECT_NAME ,NULL SUBOBJECT_NAME ,TABLE_ID OBJECT_ID ,(CASE WHEN TABLET_ID != 0 THEN TABLET_ID ELSE NULL END) DATA_OBJECT_ID ,CASE WHEN TABLE_TYPE IN (0,3,6,8,9,14) THEN 'TABLE' WHEN TABLE_TYPE IN (2) THEN 'VIRTUAL TABLE' WHEN TABLE_TYPE IN (1,4) THEN 'VIEW' WHEN TABLE_TYPE IN (5) THEN 'INDEX' WHEN TABLE_TYPE IN (7) THEN 'MATERIALIZED VIEW' WHEN TABLE_TYPE IN (15) THEN 'MATERIALIZED VIEW LOG' ELSE NULL END AS OBJECT_TYPE ,CAST(CASE WHEN TABLE_TYPE IN (5) THEN CASE WHEN INDEX_STATUS = 2 THEN 'VALID' WHEN INDEX_STATUS = 3 THEN 'CHECKING' WHEN INDEX_STATUS = 4 THEN 'INELEGIBLE' WHEN INDEX_STATUS = 5 THEN 'ERROR' ELSE 'UNUSABLE' END ELSE CASE WHEN OBJECT_STATUS = 1 THEN 'VALID' ELSE 'INVALID' END END AS VARCHAR2(10)) AS STATUS ,CASE WHEN TABLE_TYPE IN (6,8,9) THEN 'Y' ELSE 'N' END AS TEMPORARY ,CASE WHEN TABLE_TYPE IN (0,1) THEN 'Y' ELSE 'N' END AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() AND TABLE_TYPE != 12 AND TABLE_TYPE != 13 AND bitand((TABLE_MODE / 4096), 15) IN (0,1) UNION ALL SELECT CST.TENANT_ID ,CST.GMT_CREATE ,CST.GMT_MODIFIED ,DB.DATABASE_ID ,CST.CONSTRAINT_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,TBL.TABLE_ID AS OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'INDEX' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_CONSTRAINT_REAL_AGENT CST, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT TBL, SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB WHERE CST.TENANT_ID = EFFECTIVE_TENANT_ID() AND DB.DATABASE_ID = TBL.DATABASE_ID AND TBL.TABLE_ID = CST.TABLE_ID and CST.CONSTRAINT_TYPE = 1 AND TBL.TABLE_TYPE != 12 AND TBL.TABLE_TYPE != 13 AND bitand((TBL.TABLE_MODE / 4096), 15) IN (0,1) UNION ALL SELECT P.TENANT_ID ,P.GMT_CREATE ,P.GMT_MODIFIED ,T.DATABASE_ID ,CAST((CASE WHEN T.DATABASE_ID = 201004 THEN T.TABLE_NAME WHEN T.TABLE_TYPE = 5 THEN SUBSTR(T.TABLE_NAME, 7 + INSTR(SUBSTR(T.TABLE_NAME, 7), '_')) ELSE T.TABLE_NAME END) AS VARCHAR2(128)) AS OBJECT_NAME ,P.PART_NAME SUBOBJECT_NAME ,P.PART_ID OBJECT_ID ,CASE WHEN P.TABLET_ID != 0 THEN P.TABLET_ID ELSE NULL END AS DATA_OBJECT_ID ,DECODE (T.TABLE_TYPE, 5, 'INDEX PARTITION', 'TABLE PARTITION') AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY , NULL AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T JOIN SYS.ALL_VIRTUAL_PART_REAL_AGENT P ON T.TABLE_ID = P.TABLE_ID WHERE T.TENANT_ID = EFFECTIVE_TENANT_ID() AND P.TENANT_ID = EFFECTIVE_TENANT_ID() AND T.TABLE_TYPE != 12 AND T.TABLE_TYPE != 13 AND bitand((T.TABLE_MODE / 4096), 15) IN (0,1) UNION ALL SELECT SUBP.TENANT_ID ,SUBP.GMT_CREATE ,SUBP.GMT_MODIFIED ,T.DATABASE_ID ,CAST((CASE WHEN T.DATABASE_ID = 201004 THEN T.TABLE_NAME WHEN T.TABLE_TYPE = 5 THEN SUBSTR(T.TABLE_NAME, 7 + INSTR(SUBSTR(T.TABLE_NAME, 7), '_')) ELSE T.TABLE_NAME END) AS VARCHAR2(128)) AS OBJECT_NAME ,SUBP.SUB_PART_NAME SUBOBJECT_NAME ,SUBP.SUB_PART_ID OBJECT_ID ,SUBP.TABLET_ID AS DATA_OBJECT_ID ,DECODE (T.TABLE_TYPE, 5, 'INDEX SUBPARTITION', 'TABLE SUBPARTITION') AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'Y' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T, SYS.ALL_VIRTUAL_PART_REAL_AGENT P,SYS.ALL_VIRTUAL_SUB_PART_REAL_AGENT SUBP WHERE T.TABLE_ID =P.TABLE_ID AND P.TABLE_ID=SUBP.TABLE_ID AND P.PART_ID =SUBP.PART_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() AND P.TENANT_ID = EFFECTIVE_TENANT_ID() AND SUBP.TENANT_ID = EFFECTIVE_TENANT_ID() AND T.TABLE_TYPE != 12 AND T.TABLE_TYPE != 13 AND bitand((T.TABLE_MODE / 4096), 15) IN (0,1) UNION ALL SELECT EFFECTIVE_TENANT_ID() AS TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,CAST(201006 AS NUMBER) AS DATABASE_ID ,PACKAGE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,PACKAGE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,CASE WHEN TYPE = 1 THEN 'PACKAGE' WHEN TYPE = 2 THEN 'PACKAGE BODY' ELSE NULL END AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_PACKAGE_SYS_AGENT UNION ALL SELECT P.TENANT_ID ,P.GMT_CREATE ,P.GMT_MODIFIED ,P.DATABASE_ID ,P.PACKAGE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,P.PACKAGE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,CASE WHEN TYPE = 1 THEN 'PACKAGE' WHEN TYPE = 2 THEN 'PACKAGE BODY' ELSE NULL END AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE P.TENANT_ID = E.TENANT_ID AND P.PACKAGE_ID = E.OBJ_ID AND (E.OBJ_TYPE = 3 OR E.OBJ_TYPE = 5)) THEN 'INVALID' WHEN TYPE = 2 AND EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT Eb WHERE OBJ_ID IN (SELECT PACKAGE_ID FROM SYS.ALL_VIRTUAL_PACKAGE_REAL_AGENT Pb WHERE Pb.PACKAGE_NAME = P.PACKAGE_NAME AND Pb.DATABASE_ID = P.DATABASE_ID AND Pb.TENANT_ID = P.TENANT_ID AND TYPE = 1) AND Eb.OBJ_TYPE = 3) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_PACKAGE_REAL_AGENT P WHERE P.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT R.TENANT_ID ,R.GMT_CREATE ,R.GMT_MODIFIED ,R.DATABASE_ID ,R.ROUTINE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,R.ROUTINE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,CASE WHEN ROUTINE_TYPE = 1 THEN 'PROCEDURE' WHEN ROUTINE_TYPE = 2 THEN 'FUNCTION' ELSE NULL END AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE R.TENANT_ID = E.TENANT_ID AND R.ROUTINE_ID = E.OBJ_ID AND (E.OBJ_TYPE = 9 OR E.OBJ_TYPE = 12)) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_ROUTINE_REAL_AGENT R WHERE (ROUTINE_TYPE = 1 OR ROUTINE_TYPE = 2) AND R.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT EFFECTIVE_TENANT_ID() AS TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,CAST(201006 AS NUMBER) AS DATABASE_ID ,TS.TYPE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,TS.TYPE_ID AS OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE' AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE TS.TENANT_ID = E.TENANT_ID AND TS.TYPE_ID = E.OBJ_ID AND E.OBJ_TYPE = 4) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,TYPE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,TYPE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE' AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE TY.TENANT_ID = E.TENANT_ID AND TY.TYPE_ID = E.OBJ_ID AND E.OBJ_TYPE = 4) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TYPE_REAL_AGENT TY WHERE TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT EFFECTIVE_TENANT_ID() AS TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,CAST(201006 AS NUMBER) AS DATABASE_ID ,OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,OBJECT_TYPE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE BODY' AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE EFFECTIVE_TENANT_ID() = E.TENANT_ID AND TS.OBJECT_TYPE_ID = E.OBJ_ID AND E.OBJ_TYPE = 6) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TENANT_OBJECT_TYPE_SYS_AGENT TS WHERE TYPE = 2 UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,OBJECT_TYPE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE BODY' AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE TY.TENANT_ID = E.TENANT_ID AND TY.OBJECT_TYPE_ID = E.OBJ_ID AND E.OBJ_TYPE = 6) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TENANT_OBJECT_TYPE_REAL_AGENT TY WHERE TENANT_ID = EFFECTIVE_TENANT_ID() and TYPE = 2 UNION ALL SELECT T.TENANT_ID ,T.GMT_CREATE ,T.GMT_MODIFIED ,T.DATABASE_ID ,T.TRIGGER_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,T.TRIGGER_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TRIGGER' OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE T.TENANT_ID = E.TENANT_ID AND T.TRIGGER_ID = E.OBJ_ID AND (E.OBJ_TYPE = 7)) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT T WHERE T.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,SEQUENCE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,SEQUENCE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'SEQUENCE' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_SEQUENCE_OBJECT_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,SYNONYM_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,SYNONYM_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'SYNONYM' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_SYNONYM_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,CAST(201006 AS NUMBER) AS DATABASE_ID ,NAMESPACE AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,CONTEXT_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'CONTEXT' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,21 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_CONTEXT_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID, GMT_CREATE, GMT_MODIFIED, DATABASE_ID, DATABASE_NAME AS OBJECT_NAME, NULL AS SUBOBJECT_NAME, DATABASE_ID AS OBJECT_ID, NULL AS DATA_OBJECT_ID, 'DATABASE' AS OBJECT_TYPE, 'VALID' AS STATUS, 'N' AS TEMPORARY, 'N' AS "GENERATED", 'N' AS SECONDARY, 0 AS NAMESPACE, NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID, GMT_CREATE, GMT_MODIFIED, CAST(201001 AS NUMBER) AS DATABASE_ID, TABLEGROUP_NAME AS OBJECT_NAME, NULL AS SUBOBJECT_NAME, TABLEGROUP_ID AS OBJECT_ID, NULL AS DATA_OBJECT_ID, 'TABLEGROUP' AS OBJECT_TYPE, 'VALID' AS STATUS, 'N' AS TEMPORARY, 'N' AS "GENERATED", 'N' AS SECONDARY, 0 AS NAMESPACE, NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TABLEGROUP_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() ) A JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT B ON A.TENANT_ID = B.TENANT_ID AND A.DATABASE_ID = B.DATABASE_ID AND B.TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -160,7 +160,7 @@ int ObInnerTableSchema::all_objects_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(B.DATABASE_NAME AS VARCHAR(128)) AS OWNER, CAST(A.OBJECT_NAME AS VARCHAR(128)) AS OBJECT_NAME, CAST(A.SUBOBJECT_NAME AS VARCHAR2(128)) AS SUBOBJECT_NAME, CAST(A.OBJECT_ID AS NUMBER) AS OBJECT_ID, CAST(A.DATA_OBJECT_ID AS NUMBER) AS DATA_OBJECT_ID, CAST(A.OBJECT_TYPE AS VARCHAR2(23)) AS OBJECT_TYPE, CAST(A.GMT_CREATE AS DATE) AS CREATED, CAST(A.GMT_MODIFIED AS DATE) AS LAST_DDL_TIME, CAST(TO_CHAR(A.GMT_CREATE) AS VARCHAR2(19)) AS TIMESTAMP, CAST(A.STATUS AS VARCHAR2(7)) AS STATUS, CAST(A.TEMPORARY AS VARCHAR2(1)) AS TEMPORARY, CAST(A."GENERATED" AS VARCHAR2(1)) AS "GENERATED", CAST(A.SECONDARY AS VARCHAR2(1)) AS SECONDARY, CAST(A.NAMESPACE AS NUMBER) AS NAMESPACE, CAST(A.EDITION_NAME AS VARCHAR2(128)) AS EDITION_NAME, CAST(NULL AS VARCHAR2(18)) AS SHARING, CAST(NULL AS VARCHAR2(1)) AS EDITIONABLE, CAST(NULL AS VARCHAR2(1)) AS ORACLE_MAINTAINED, CAST(NULL AS VARCHAR2(1)) AS APPLICATION, CAST(NULL AS VARCHAR2(1)) AS DEFAULT_COLLATION, CAST(NULL AS VARCHAR2(1)) AS DUPLICATED, CAST(NULL AS VARCHAR2(1)) AS SHARDED, CAST(NULL AS VARCHAR2(1)) AS IMPORTED_OBJECT, CAST(NULL AS NUMBER) AS CREATED_APPID, CAST(NULL AS NUMBER) AS CREATED_VSNID, CAST(NULL AS NUMBER) AS MODIFIED_APPID, CAST(NULL AS NUMBER) AS MODIFIED_VSNID FROM ( SELECT A.TENANT_ID, (TO_DATE('19700101','YYYYMMDD') + B.SCHEMA_VERSION / 86400 / 1000000 + TO_NUMBER(SUBSTR(TZ_OFFSET(SESSIONTIMEZONE),1,3))/24) AS GMT_CREATE, (TO_DATE('19700101','YYYYMMDD') + A.SCHEMA_VERSION / 86400 / 1000000 + TO_NUMBER(SUBSTR(TZ_OFFSET(SESSIONTIMEZONE),1,3))/24) AS GMT_MODIFIED, A.DATABASE_ID, A.TABLE_NAME AS OBJECT_NAME, NULL AS SUBOBJECT_NAME, CAST(A.TABLE_ID AS NUMBER) AS OBJECT_ID, CAST(A.TABLE_ID AS NUMBER) AS PRIV_OBJECT_ID, A.TABLE_ID AS DATA_OBJECT_ID, 'TABLE' AS OBJECT_TYPE, 'VALID' AS STATUS, 'N' AS TEMPORARY, 'N' AS "GENERATED", 'N' AS SECONDARY, 0 AS NAMESPACE, NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_CORE_ALL_TABLE A JOIN SYS.ALL_VIRTUAL_CORE_ALL_TABLE B ON A.TENANT_ID = B.TENANT_ID WHERE B.TABLE_NAME = '__all_core_table' AND A.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,CAST((CASE WHEN DATABASE_ID = 201004 THEN TABLE_NAME WHEN TABLE_TYPE = 5 THEN SUBSTR(TABLE_NAME, 7 + INSTR(SUBSTR(TABLE_NAME, 7), '_')) ELSE TABLE_NAME END) AS VARCHAR2(128)) AS OBJECT_NAME ,NULL SUBOBJECT_NAME ,TABLE_ID OBJECT_ID ,TABLE_ID PRIV_OBJECT_ID ,(CASE WHEN TABLET_ID != 0 THEN TABLET_ID ELSE NULL END) DATA_OBJECT_ID ,CASE WHEN TABLE_TYPE IN (0,3,6,8,9,14) THEN 'TABLE' WHEN TABLE_TYPE IN (2) THEN 'VIRTUAL TABLE' WHEN TABLE_TYPE IN (1,4) THEN 'VIEW' WHEN TABLE_TYPE IN (5) THEN 'INDEX' WHEN TABLE_TYPE IN (7) THEN 'MATERIALIZED VIEW' WHEN TABLE_TYPE IN (15) THEN 'MATERIALIZED VIEW LOG' ELSE NULL END AS OBJECT_TYPE ,CAST(CASE WHEN TABLE_TYPE IN (5,15) THEN CASE WHEN INDEX_STATUS = 2 THEN 'VALID' WHEN INDEX_STATUS = 3 THEN 'CHECKING' WHEN INDEX_STATUS = 4 THEN 'INELEGIBLE' WHEN INDEX_STATUS = 5 THEN 'ERROR' ELSE 'UNUSABLE' END ELSE CASE WHEN OBJECT_STATUS = 1 THEN 'VALID' ELSE 'INVALID' END END AS VARCHAR2(10)) AS STATUS ,CASE WHEN TABLE_TYPE IN (6,8,9) THEN 'Y' ELSE 'N' END AS TEMPORARY ,CASE WHEN TABLE_TYPE IN (0,1) THEN 'Y' ELSE 'N' END AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() AND TABLE_TYPE != 12 AND TABLE_TYPE != 13 and bitand((TABLE_MODE / 4096), 15) IN (0,1) UNION ALL SELECT CST.TENANT_ID ,CST.GMT_CREATE ,CST.GMT_MODIFIED ,DB.DATABASE_ID ,CST.CONSTRAINT_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,TBL.TABLE_ID AS OBJECT_ID ,TBL.TABLE_ID AS PRIV_OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'INDEX' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_CONSTRAINT_REAL_AGENT CST, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT TBL, SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB WHERE CST.TENANT_ID = EFFECTIVE_TENANT_ID() AND DB.DATABASE_ID = TBL.DATABASE_ID AND TBL.TABLE_ID = CST.TABLE_ID and CST.CONSTRAINT_TYPE = 1 AND TBL.TABLE_TYPE != 12 AND TBL.TABLE_TYPE != 13 AND bitand((TBL.TABLE_MODE / 4096), 15) IN (0,1) UNION ALL SELECT P.TENANT_ID ,P.GMT_CREATE ,P.GMT_MODIFIED ,T.DATABASE_ID ,CAST((CASE WHEN T.DATABASE_ID = 201004 THEN T.TABLE_NAME WHEN T.TABLE_TYPE = 5 THEN SUBSTR(T.TABLE_NAME, 7 + INSTR(SUBSTR(T.TABLE_NAME, 7), '_')) ELSE T.TABLE_NAME END) AS VARCHAR2(128)) AS OBJECT_NAME ,P.PART_NAME SUBOBJECT_NAME ,P.PART_ID OBJECT_ID ,P.TABLE_ID PRIV_OBJECT_ID ,CASE WHEN P.TABLET_ID != 0 THEN P.TABLET_ID ELSE NULL END AS DATA_OBJECT_ID ,DECODE (T.TABLE_TYPE, 5, 'INDEX PARTITION', 'TABLE PARTITION') AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY , NULL AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T JOIN SYS.ALL_VIRTUAL_PART_REAL_AGENT P ON T.TABLE_ID = P.TABLE_ID WHERE T.TENANT_ID = EFFECTIVE_TENANT_ID() AND P.TENANT_ID = EFFECTIVE_TENANT_ID() AND T.TABLE_TYPE != 12 AND T.TABLE_TYPE != 13 AND bitand((T.TABLE_MODE / 4096), 15) IN (0,1) UNION ALL SELECT SUBP.TENANT_ID ,SUBP.GMT_CREATE ,SUBP.GMT_MODIFIED ,T.DATABASE_ID ,CAST((CASE WHEN T.DATABASE_ID = 201004 THEN T.TABLE_NAME WHEN T.TABLE_TYPE = 5 THEN SUBSTR(T.TABLE_NAME, 7 + INSTR(SUBSTR(T.TABLE_NAME, 7), '_')) ELSE T.TABLE_NAME END) AS VARCHAR2(128)) AS OBJECT_NAME ,SUBP.SUB_PART_NAME SUBOBJECT_NAME ,SUBP.SUB_PART_ID OBJECT_ID ,SUBP.TABLE_ID PRIV_OBJECT_ID ,SUBP.TABLET_ID AS DATA_OBJECT_ID ,DECODE (T.TABLE_TYPE, 5, 'INDEX SUBPARTITION', 'TABLE SUBPARTITION') AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'Y' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T, SYS.ALL_VIRTUAL_PART_REAL_AGENT P,SYS.ALL_VIRTUAL_SUB_PART_REAL_AGENT SUBP WHERE T.TABLE_ID =P.TABLE_ID AND P.TABLE_ID=SUBP.TABLE_ID AND P.PART_ID =SUBP.PART_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() AND P.TENANT_ID = EFFECTIVE_TENANT_ID() AND SUBP.TENANT_ID = EFFECTIVE_TENANT_ID() AND T.TABLE_TYPE != 12 AND T.TABLE_TYPE != 13 AND bitand((T.TABLE_MODE / 4096), 15) IN (0,1) UNION ALL SELECT EFFECTIVE_TENANT_ID() AS TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,CAST(201006 AS NUMBER) AS DATABASE_ID ,PACKAGE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,PACKAGE_ID OBJECT_ID ,PACKAGE_ID PRIV_OBJECT_ID ,NULL AS DATA_OBJECT_ID ,CASE WHEN TYPE = 1 THEN 'PACKAGE' WHEN TYPE = 2 THEN 'PACKAGE BODY' ELSE NULL END AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_PACKAGE_SYS_AGENT UNION ALL SELECT P.TENANT_ID ,P.GMT_CREATE ,P.GMT_MODIFIED ,P.DATABASE_ID ,P.PACKAGE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,P.PACKAGE_ID OBJECT_ID ,P.PACKAGE_ID PRIV_OBJECT_ID ,NULL AS DATA_OBJECT_ID ,CASE WHEN TYPE = 1 THEN 'PACKAGE' WHEN TYPE = 2 THEN 'PACKAGE BODY' ELSE NULL END AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE P.TENANT_ID = E.TENANT_ID AND P.PACKAGE_ID = E.OBJ_ID AND (E.OBJ_TYPE = 3 OR E.OBJ_TYPE = 5)) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_PACKAGE_REAL_AGENT P WHERE P.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT R.TENANT_ID ,R.GMT_CREATE ,R.GMT_MODIFIED ,R.DATABASE_ID ,R.ROUTINE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,R.ROUTINE_ID OBJECT_ID ,R.ROUTINE_ID PRIV_OBJECT_ID ,NULL AS DATA_OBJECT_ID ,CASE WHEN ROUTINE_TYPE = 1 THEN 'PROCEDURE' WHEN ROUTINE_TYPE = 2 THEN 'FUNCTION' ELSE NULL END AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE R.TENANT_ID = E.TENANT_ID AND R.ROUTINE_ID = E.OBJ_ID AND (E.OBJ_TYPE = 9 OR E.OBJ_TYPE = 12)) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_ROUTINE_REAL_AGENT R WHERE (ROUTINE_TYPE = 1 OR ROUTINE_TYPE = 2) AND R.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT EFFECTIVE_TENANT_ID() AS TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,CAST(201006 AS NUMBER) AS DATABASE_ID ,TS.TYPE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,TS.TYPE_ID AS OBJECT_ID ,TS.TYPE_ID AS PRIV_OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE' AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE TS.TENANT_ID = E.TENANT_ID AND TS.TYPE_ID = E.OBJ_ID AND E.OBJ_TYPE = 4) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,TYPE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,TYPE_ID OBJECT_ID ,TYPE_ID PRIV_OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE' AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE TY.TENANT_ID = E.TENANT_ID AND TY.TYPE_ID = E.OBJ_ID AND E.OBJ_TYPE = 4) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TYPE_REAL_AGENT TY WHERE TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT EFFECTIVE_TENANT_ID() AS TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,CAST(201006 AS NUMBER) AS DATABASE_ID ,OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,OBJECT_TYPE_ID OBJECT_ID ,OBJECT_TYPE_ID PRIV_OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE BODY' AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE EFFECTIVE_TENANT_ID() = E.TENANT_ID AND TS.OBJECT_TYPE_ID = E.OBJ_ID AND E.OBJ_TYPE = 6) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TENANT_OBJECT_TYPE_SYS_AGENT TS WHERE TYPE = 2 UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,OBJECT_TYPE_ID OBJECT_ID ,OBJECT_TYPE_ID PRIV_OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE BODY' AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE TY.TENANT_ID = E.TENANT_ID AND TY.OBJECT_TYPE_ID = E.OBJ_ID AND E.OBJ_TYPE = 6) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TENANT_OBJECT_TYPE_REAL_AGENT TY WHERE TENANT_ID = EFFECTIVE_TENANT_ID() and TYPE = 2 UNION ALL SELECT T.TENANT_ID ,T.GMT_CREATE ,T.GMT_MODIFIED ,T.DATABASE_ID ,T.TRIGGER_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,T.TRIGGER_ID OBJECT_ID ,T.TRIGGER_ID PRIV_OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TRIGGER' OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE T.TENANT_ID = E.TENANT_ID AND T.TRIGGER_ID = E.OBJ_ID AND (E.OBJ_TYPE = 7)) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT T WHERE T.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,SEQUENCE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,SEQUENCE_ID OBJECT_ID ,SEQUENCE_ID PRIV_OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'SEQUENCE' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_SEQUENCE_OBJECT_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,SYNONYM_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,SYNONYM_ID OBJECT_ID ,SYNONYM_ID PRIV_OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'SYNONYM' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_SYNONYM_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() /*UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,CAST(201006 AS NUMBER) AS DATABASE_ID ,NAMESPACE AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,CONTEXT_ID OBJECT_ID ,CONTEXT_ID PRIV_OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'CONTEXT' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,21 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_CONTEXT_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID()*/ UNION ALL SELECT TENANT_ID, GMT_CREATE, GMT_MODIFIED, DATABASE_ID, DATABASE_NAME AS OBJECT_NAME, NULL AS SUBOBJECT_NAME, DATABASE_ID AS OBJECT_ID, CAST(-1 AS NUMBER) AS PRIV_OBJECT_ID, NULL AS DATA_OBJECT_ID, 'DATABASE' AS OBJECT_TYPE, 'VALID' AS STATUS, 'N' AS TEMPORARY, 'N' AS "GENERATED", 'N' AS SECONDARY, 0 AS NAMESPACE, NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() ) A JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT B ON A.TENANT_ID = B.TENANT_ID AND A.DATABASE_ID = B.DATABASE_ID AND B.TENANT_ID = EFFECTIVE_TENANT_ID() AND (A.DATABASE_ID = USERENV('SCHEMAID') OR (A.PRIV_OBJECT_ID != -1 AND USER_CAN_ACCESS_OBJ(DECODE(OBJECT_TYPE, 'TABLE', 1, 'VIEW', 1, 'INDEX', 1, 'MATERIALIZED VIEW', 1, 'MATERIALIZED VIEW LOG', 1, 'TABLE PARTITION', 1, 'TABLE SUBPARTITION', 1, 'INDEX PARTITION', 1, 'INDEX SUBPARTITION', 1, 'SEQUENCE', 2, 'PACKAGE', 3, 'PACKAGE BODY', 3, 'TYPE', 4, 'TYPE BODY', 4, 'TRIGGER', 7, 'FUNCTION', 9, 'PROCEDURE', 12, 'SYNONYM', 13, 1), A.PRIV_OBJECT_ID, A.DATABASE_ID) = 1)) )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(B.DATABASE_NAME AS VARCHAR(128)) AS OWNER, CAST(A.OBJECT_NAME AS VARCHAR(128)) AS OBJECT_NAME, CAST(A.SUBOBJECT_NAME AS VARCHAR2(128)) AS SUBOBJECT_NAME, CAST(A.OBJECT_ID AS NUMBER) AS OBJECT_ID, CAST(A.DATA_OBJECT_ID AS NUMBER) AS DATA_OBJECT_ID, CAST(A.OBJECT_TYPE AS VARCHAR2(23)) AS OBJECT_TYPE, CAST(A.GMT_CREATE AS DATE) AS CREATED, CAST(A.GMT_MODIFIED AS DATE) AS LAST_DDL_TIME, CAST(TO_CHAR(A.GMT_CREATE) AS VARCHAR2(19)) AS TIMESTAMP, CAST(A.STATUS AS VARCHAR2(7)) AS STATUS, CAST(A.TEMPORARY AS VARCHAR2(1)) AS TEMPORARY, CAST(A."GENERATED" AS VARCHAR2(1)) AS "GENERATED", CAST(A.SECONDARY AS VARCHAR2(1)) AS SECONDARY, CAST(A.NAMESPACE AS NUMBER) AS NAMESPACE, CAST(A.EDITION_NAME AS VARCHAR2(128)) AS EDITION_NAME, CAST(NULL AS VARCHAR2(18)) AS SHARING, CAST(NULL AS VARCHAR2(1)) AS EDITIONABLE, CAST(NULL AS VARCHAR2(1)) AS ORACLE_MAINTAINED, CAST(NULL AS VARCHAR2(1)) AS APPLICATION, CAST(NULL AS VARCHAR2(1)) AS DEFAULT_COLLATION, CAST(NULL AS VARCHAR2(1)) AS DUPLICATED, CAST(NULL AS VARCHAR2(1)) AS SHARDED, CAST(NULL AS VARCHAR2(1)) AS IMPORTED_OBJECT, CAST(NULL AS NUMBER) AS CREATED_APPID, CAST(NULL AS NUMBER) AS CREATED_VSNID, CAST(NULL AS NUMBER) AS MODIFIED_APPID, CAST(NULL AS NUMBER) AS MODIFIED_VSNID FROM ( SELECT A.TENANT_ID, (TO_DATE('19700101','YYYYMMDD') + B.SCHEMA_VERSION / 86400 / 1000000 + TO_NUMBER(SUBSTR(TZ_OFFSET(SESSIONTIMEZONE),1,3))/24) AS GMT_CREATE, (TO_DATE('19700101','YYYYMMDD') + A.SCHEMA_VERSION / 86400 / 1000000 + TO_NUMBER(SUBSTR(TZ_OFFSET(SESSIONTIMEZONE),1,3))/24) AS GMT_MODIFIED, A.DATABASE_ID, A.TABLE_NAME AS OBJECT_NAME, NULL AS SUBOBJECT_NAME, CAST(A.TABLE_ID AS NUMBER) AS OBJECT_ID, CAST(A.TABLE_ID AS NUMBER) AS PRIV_OBJECT_ID, A.TABLE_ID AS DATA_OBJECT_ID, 'TABLE' AS OBJECT_TYPE, 'VALID' AS STATUS, 'N' AS TEMPORARY, 'N' AS "GENERATED", 'N' AS SECONDARY, 0 AS NAMESPACE, NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_CORE_ALL_TABLE A JOIN SYS.ALL_VIRTUAL_CORE_ALL_TABLE B ON A.TENANT_ID = B.TENANT_ID WHERE B.TABLE_NAME = '__all_core_table' AND A.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,CAST((CASE WHEN DATABASE_ID = 201004 THEN TABLE_NAME WHEN TABLE_TYPE = 5 THEN SUBSTR(TABLE_NAME, 7 + INSTR(SUBSTR(TABLE_NAME, 7), '_')) ELSE TABLE_NAME END) AS VARCHAR2(128)) AS OBJECT_NAME ,NULL SUBOBJECT_NAME ,TABLE_ID OBJECT_ID ,TABLE_ID PRIV_OBJECT_ID ,(CASE WHEN TABLET_ID != 0 THEN TABLET_ID ELSE NULL END) DATA_OBJECT_ID ,CASE WHEN TABLE_TYPE IN (0,3,6,8,9,14) THEN 'TABLE' WHEN TABLE_TYPE IN (2) THEN 'VIRTUAL TABLE' WHEN TABLE_TYPE IN (1,4) THEN 'VIEW' WHEN TABLE_TYPE IN (5) THEN 'INDEX' WHEN TABLE_TYPE IN (7) THEN 'MATERIALIZED VIEW' WHEN TABLE_TYPE IN (15) THEN 'MATERIALIZED VIEW LOG' ELSE NULL END AS OBJECT_TYPE ,CAST(CASE WHEN TABLE_TYPE IN (5,15) THEN CASE WHEN INDEX_STATUS = 2 THEN 'VALID' WHEN INDEX_STATUS = 3 THEN 'CHECKING' WHEN INDEX_STATUS = 4 THEN 'INELEGIBLE' WHEN INDEX_STATUS = 5 THEN 'ERROR' ELSE 'UNUSABLE' END ELSE CASE WHEN OBJECT_STATUS = 1 THEN 'VALID' ELSE 'INVALID' END END AS VARCHAR2(10)) AS STATUS ,CASE WHEN TABLE_TYPE IN (6,8,9) THEN 'Y' ELSE 'N' END AS TEMPORARY ,CASE WHEN TABLE_TYPE IN (0,1) THEN 'Y' ELSE 'N' END AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() AND TABLE_TYPE != 12 AND TABLE_TYPE != 13 and bitand((TABLE_MODE / 4096), 15) IN (0,1) UNION ALL SELECT CST.TENANT_ID ,CST.GMT_CREATE ,CST.GMT_MODIFIED ,DB.DATABASE_ID ,CST.CONSTRAINT_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,TBL.TABLE_ID AS OBJECT_ID ,TBL.TABLE_ID AS PRIV_OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'INDEX' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_CONSTRAINT_REAL_AGENT CST, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT TBL, SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB WHERE CST.TENANT_ID = EFFECTIVE_TENANT_ID() AND DB.DATABASE_ID = TBL.DATABASE_ID AND TBL.TABLE_ID = CST.TABLE_ID and CST.CONSTRAINT_TYPE = 1 AND TBL.TABLE_TYPE != 12 AND TBL.TABLE_TYPE != 13 AND bitand((TBL.TABLE_MODE / 4096), 15) IN (0,1) UNION ALL SELECT P.TENANT_ID ,P.GMT_CREATE ,P.GMT_MODIFIED ,T.DATABASE_ID ,CAST((CASE WHEN T.DATABASE_ID = 201004 THEN T.TABLE_NAME WHEN T.TABLE_TYPE = 5 THEN SUBSTR(T.TABLE_NAME, 7 + INSTR(SUBSTR(T.TABLE_NAME, 7), '_')) ELSE T.TABLE_NAME END) AS VARCHAR2(128)) AS OBJECT_NAME ,P.PART_NAME SUBOBJECT_NAME ,P.PART_ID OBJECT_ID ,P.TABLE_ID PRIV_OBJECT_ID ,CASE WHEN P.TABLET_ID != 0 THEN P.TABLET_ID ELSE NULL END AS DATA_OBJECT_ID ,DECODE (T.TABLE_TYPE, 5, 'INDEX PARTITION', 'TABLE PARTITION') AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY , NULL AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T JOIN SYS.ALL_VIRTUAL_PART_REAL_AGENT P ON T.TABLE_ID = P.TABLE_ID WHERE T.TENANT_ID = EFFECTIVE_TENANT_ID() AND P.TENANT_ID = EFFECTIVE_TENANT_ID() AND T.TABLE_TYPE != 12 AND T.TABLE_TYPE != 13 AND bitand((T.TABLE_MODE / 4096), 15) IN (0,1) UNION ALL SELECT SUBP.TENANT_ID ,SUBP.GMT_CREATE ,SUBP.GMT_MODIFIED ,T.DATABASE_ID ,CAST((CASE WHEN T.DATABASE_ID = 201004 THEN T.TABLE_NAME WHEN T.TABLE_TYPE = 5 THEN SUBSTR(T.TABLE_NAME, 7 + INSTR(SUBSTR(T.TABLE_NAME, 7), '_')) ELSE T.TABLE_NAME END) AS VARCHAR2(128)) AS OBJECT_NAME ,SUBP.SUB_PART_NAME SUBOBJECT_NAME ,SUBP.SUB_PART_ID OBJECT_ID ,SUBP.TABLE_ID PRIV_OBJECT_ID ,SUBP.TABLET_ID AS DATA_OBJECT_ID ,DECODE (T.TABLE_TYPE, 5, 'INDEX SUBPARTITION', 'TABLE SUBPARTITION') AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'Y' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T, SYS.ALL_VIRTUAL_PART_REAL_AGENT P,SYS.ALL_VIRTUAL_SUB_PART_REAL_AGENT SUBP WHERE T.TABLE_ID =P.TABLE_ID AND P.TABLE_ID=SUBP.TABLE_ID AND P.PART_ID =SUBP.PART_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() AND P.TENANT_ID = EFFECTIVE_TENANT_ID() AND SUBP.TENANT_ID = EFFECTIVE_TENANT_ID() AND T.TABLE_TYPE != 12 AND T.TABLE_TYPE != 13 AND bitand((T.TABLE_MODE / 4096), 15) IN (0,1) UNION ALL SELECT EFFECTIVE_TENANT_ID() AS TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,CAST(201006 AS NUMBER) AS DATABASE_ID ,PACKAGE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,PACKAGE_ID OBJECT_ID ,PACKAGE_ID PRIV_OBJECT_ID ,NULL AS DATA_OBJECT_ID ,CASE WHEN TYPE = 1 THEN 'PACKAGE' WHEN TYPE = 2 THEN 'PACKAGE BODY' ELSE NULL END AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_PACKAGE_SYS_AGENT UNION ALL SELECT P.TENANT_ID ,P.GMT_CREATE ,P.GMT_MODIFIED ,P.DATABASE_ID ,P.PACKAGE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,P.PACKAGE_ID OBJECT_ID ,P.PACKAGE_ID PRIV_OBJECT_ID ,NULL AS DATA_OBJECT_ID ,CASE WHEN TYPE = 1 THEN 'PACKAGE' WHEN TYPE = 2 THEN 'PACKAGE BODY' ELSE NULL END AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE P.TENANT_ID = E.TENANT_ID AND P.PACKAGE_ID = E.OBJ_ID AND (E.OBJ_TYPE = 3 OR E.OBJ_TYPE = 5)) THEN 'INVALID' WHEN TYPE = 2 AND EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT Eb WHERE OBJ_ID IN (SELECT PACKAGE_ID FROM SYS.ALL_VIRTUAL_PACKAGE_REAL_AGENT Pb WHERE Pb.PACKAGE_NAME = P.PACKAGE_NAME AND Pb.DATABASE_ID = P.DATABASE_ID AND Pb.TENANT_ID = P.TENANT_ID AND TYPE = 1) AND Eb.OBJ_TYPE = 3) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_PACKAGE_REAL_AGENT P WHERE P.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT R.TENANT_ID ,R.GMT_CREATE ,R.GMT_MODIFIED ,R.DATABASE_ID ,R.ROUTINE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,R.ROUTINE_ID OBJECT_ID ,R.ROUTINE_ID PRIV_OBJECT_ID ,NULL AS DATA_OBJECT_ID ,CASE WHEN ROUTINE_TYPE = 1 THEN 'PROCEDURE' WHEN ROUTINE_TYPE = 2 THEN 'FUNCTION' ELSE NULL END AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE R.TENANT_ID = E.TENANT_ID AND R.ROUTINE_ID = E.OBJ_ID AND (E.OBJ_TYPE = 9 OR E.OBJ_TYPE = 12)) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_ROUTINE_REAL_AGENT R WHERE (ROUTINE_TYPE = 1 OR ROUTINE_TYPE = 2) AND R.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT EFFECTIVE_TENANT_ID() AS TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,CAST(201006 AS NUMBER) AS DATABASE_ID ,TS.TYPE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,TS.TYPE_ID AS OBJECT_ID ,TS.TYPE_ID AS PRIV_OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE' AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE TS.TENANT_ID = E.TENANT_ID AND TS.TYPE_ID = E.OBJ_ID AND E.OBJ_TYPE = 4) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,TYPE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,TYPE_ID OBJECT_ID ,TYPE_ID PRIV_OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE' AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE TY.TENANT_ID = E.TENANT_ID AND TY.TYPE_ID = E.OBJ_ID AND E.OBJ_TYPE = 4) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TYPE_REAL_AGENT TY WHERE TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT EFFECTIVE_TENANT_ID() AS TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,CAST(201006 AS NUMBER) AS DATABASE_ID ,OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,OBJECT_TYPE_ID OBJECT_ID ,OBJECT_TYPE_ID PRIV_OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE BODY' AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE EFFECTIVE_TENANT_ID() = E.TENANT_ID AND TS.OBJECT_TYPE_ID = E.OBJ_ID AND E.OBJ_TYPE = 6) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TENANT_OBJECT_TYPE_SYS_AGENT TS WHERE TYPE = 2 UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,OBJECT_TYPE_ID OBJECT_ID ,OBJECT_TYPE_ID PRIV_OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE BODY' AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE TY.TENANT_ID = E.TENANT_ID AND TY.OBJECT_TYPE_ID = E.OBJ_ID AND E.OBJ_TYPE = 6) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TENANT_OBJECT_TYPE_REAL_AGENT TY WHERE TENANT_ID = EFFECTIVE_TENANT_ID() and TYPE = 2 UNION ALL SELECT T.TENANT_ID ,T.GMT_CREATE ,T.GMT_MODIFIED ,T.DATABASE_ID ,T.TRIGGER_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,T.TRIGGER_ID OBJECT_ID ,T.TRIGGER_ID PRIV_OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TRIGGER' OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE T.TENANT_ID = E.TENANT_ID AND T.TRIGGER_ID = E.OBJ_ID AND (E.OBJ_TYPE = 7)) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT T WHERE T.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,SEQUENCE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,SEQUENCE_ID OBJECT_ID ,SEQUENCE_ID PRIV_OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'SEQUENCE' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_SEQUENCE_OBJECT_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,SYNONYM_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,SYNONYM_ID OBJECT_ID ,SYNONYM_ID PRIV_OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'SYNONYM' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_SYNONYM_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() /*UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,CAST(201006 AS NUMBER) AS DATABASE_ID ,NAMESPACE AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,CONTEXT_ID OBJECT_ID ,CONTEXT_ID PRIV_OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'CONTEXT' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,21 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_CONTEXT_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID()*/ UNION ALL SELECT TENANT_ID, GMT_CREATE, GMT_MODIFIED, DATABASE_ID, DATABASE_NAME AS OBJECT_NAME, NULL AS SUBOBJECT_NAME, DATABASE_ID AS OBJECT_ID, CAST(-1 AS NUMBER) AS PRIV_OBJECT_ID, NULL AS DATA_OBJECT_ID, 'DATABASE' AS OBJECT_TYPE, 'VALID' AS STATUS, 'N' AS TEMPORARY, 'N' AS "GENERATED", 'N' AS SECONDARY, 0 AS NAMESPACE, NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() ) A JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT B ON A.TENANT_ID = B.TENANT_ID AND A.DATABASE_ID = B.DATABASE_ID AND B.TENANT_ID = EFFECTIVE_TENANT_ID() AND (A.DATABASE_ID = USERENV('SCHEMAID') OR (A.PRIV_OBJECT_ID != -1 AND USER_CAN_ACCESS_OBJ(DECODE(OBJECT_TYPE, 'TABLE', 1, 'VIEW', 1, 'INDEX', 1, 'MATERIALIZED VIEW', 1, 'MATERIALIZED VIEW LOG', 1, 'TABLE PARTITION', 1, 'TABLE SUBPARTITION', 1, 'INDEX PARTITION', 1, 'INDEX SUBPARTITION', 1, 'SEQUENCE', 2, 'PACKAGE', 3, 'PACKAGE BODY', 3, 'TYPE', 4, 'TYPE BODY', 4, 'TRIGGER', 7, 'FUNCTION', 9, 'PROCEDURE', 12, 'SYNONYM', 13, 1), A.PRIV_OBJECT_ID, A.DATABASE_ID) = 1)) )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -210,7 +210,7 @@ int ObInnerTableSchema::user_objects_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(A.OBJECT_NAME AS VARCHAR(128)) AS OBJECT_NAME, CAST(A.SUBOBJECT_NAME AS VARCHAR2(128)) AS SUBOBJECT_NAME, CAST(A.OBJECT_ID AS NUMBER) AS OBJECT_ID, CAST(A.DATA_OBJECT_ID AS NUMBER) AS DATA_OBJECT_ID, CAST(A.OBJECT_TYPE AS VARCHAR2(23)) AS OBJECT_TYPE, CAST(A.GMT_CREATE AS DATE) AS CREATED, CAST(A.GMT_MODIFIED AS DATE) AS LAST_DDL_TIME, CAST(TO_CHAR(A.GMT_CREATE) AS VARCHAR2(19)) AS TIMESTAMP, CAST(A.STATUS AS VARCHAR2(7)) AS STATUS, CAST(A.TEMPORARY AS VARCHAR2(1)) AS TEMPORARY, CAST(A."GENERATED" AS VARCHAR2(1)) AS "GENERATED", CAST(A.SECONDARY AS VARCHAR2(1)) AS SECONDARY, CAST(A.NAMESPACE AS NUMBER) AS NAMESPACE, CAST(A.EDITION_NAME AS VARCHAR2(128)) AS EDITION_NAME, CAST(NULL AS VARCHAR2(18)) AS SHARING, CAST(NULL AS VARCHAR2(1)) AS EDITIONABLE, CAST(NULL AS VARCHAR2(1)) AS ORACLE_MAINTAINED, CAST(NULL AS VARCHAR2(1)) AS APPLICATION, CAST(NULL AS VARCHAR2(1)) AS DEFAULT_COLLATION, CAST(NULL AS VARCHAR2(1)) AS DUPLICATED, CAST(NULL AS VARCHAR2(1)) AS SHARDED, CAST(NULL AS VARCHAR2(1)) AS IMPORTED_OBJECT, CAST(NULL AS NUMBER) AS CREATED_APPID, CAST(NULL AS NUMBER) AS CREATED_VSNID, CAST(NULL AS NUMBER) AS MODIFIED_APPID, CAST(NULL AS NUMBER) AS MODIFIED_VSNID FROM ( SELECT A.TENANT_ID, (TO_DATE('19700101','YYYYMMDD') + B.SCHEMA_VERSION / 86400 / 1000000 + TO_NUMBER(SUBSTR(TZ_OFFSET(SESSIONTIMEZONE),1,3))/24) AS GMT_CREATE, (TO_DATE('19700101','YYYYMMDD') + A.SCHEMA_VERSION / 86400 / 1000000 + TO_NUMBER(SUBSTR(TZ_OFFSET(SESSIONTIMEZONE),1,3))/24) AS GMT_MODIFIED, A.DATABASE_ID, A.TABLE_NAME AS OBJECT_NAME, NULL AS SUBOBJECT_NAME, CAST(A.TABLE_ID AS NUMBER) AS OBJECT_ID, A.TABLE_ID AS DATA_OBJECT_ID, 'TABLE' AS OBJECT_TYPE, 'VALID' AS STATUS, 'N' AS TEMPORARY, 'N' AS "GENERATED", 'N' AS SECONDARY, 0 AS NAMESPACE, NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_CORE_ALL_TABLE A JOIN SYS.ALL_VIRTUAL_CORE_ALL_TABLE B ON A.TENANT_ID = B.TENANT_ID WHERE B.TABLE_NAME = '__all_core_table' AND A.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,CAST((CASE WHEN DATABASE_ID = 201004 THEN TABLE_NAME WHEN TABLE_TYPE = 5 THEN SUBSTR(TABLE_NAME, 7 + INSTR(SUBSTR(TABLE_NAME, 7), '_')) ELSE TABLE_NAME END) AS VARCHAR2(128)) AS OBJECT_NAME ,NULL SUBOBJECT_NAME ,TABLE_ID OBJECT_ID ,(CASE WHEN TABLET_ID != 0 THEN TABLET_ID ELSE NULL END) DATA_OBJECT_ID ,CASE WHEN TABLE_TYPE IN (0,3,6,8,9,14) THEN 'TABLE' WHEN TABLE_TYPE IN (2) THEN 'VIRTUAL TABLE' WHEN TABLE_TYPE IN (1,4) THEN 'VIEW' WHEN TABLE_TYPE IN (5) THEN 'INDEX' WHEN TABLE_TYPE IN (7) THEN 'MATERIALIZED VIEW' WHEN TABLE_TYPE IN (15) THEN 'MATERIALIZED VIEW LOG' ELSE NULL END AS OBJECT_TYPE ,CAST(CASE WHEN TABLE_TYPE IN (5,15) THEN CASE WHEN INDEX_STATUS = 2 THEN 'VALID' WHEN INDEX_STATUS = 3 THEN 'CHECKING' WHEN INDEX_STATUS = 4 THEN 'INELEGIBLE' WHEN INDEX_STATUS = 5 THEN 'ERROR' ELSE 'UNUSABLE' END ELSE CASE WHEN OBJECT_STATUS = 1 THEN 'VALID' ELSE 'INVALID' END END AS VARCHAR2(10)) AS STATUS ,CASE WHEN TABLE_TYPE IN (6,8,9) THEN 'Y' ELSE 'N' END AS TEMPORARY ,CASE WHEN TABLE_TYPE IN (0,1) THEN 'Y' ELSE 'N' END AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() AND TABLE_TYPE != 12 AND TABLE_TYPE != 13 AND bitand((TABLE_MODE / 4096), 15) IN (0,1) UNION ALL SELECT CST.TENANT_ID ,CST.GMT_CREATE ,CST.GMT_MODIFIED ,DB.DATABASE_ID ,CST.CONSTRAINT_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,TBL.TABLE_ID AS OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'INDEX' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_CONSTRAINT_REAL_AGENT CST, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT TBL, SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB WHERE CST.TENANT_ID = EFFECTIVE_TENANT_ID() AND DB.DATABASE_ID = TBL.DATABASE_ID AND TBL.TABLE_ID = CST.TABLE_ID and CST.CONSTRAINT_TYPE = 1 AND TBL.TABLE_TYPE != 12 AND TBL.TABLE_TYPE != 13 AND bitand((TBL.TABLE_MODE / 4096), 15) IN (0,1) UNION ALL SELECT P.TENANT_ID ,P.GMT_CREATE ,P.GMT_MODIFIED ,T.DATABASE_ID ,CAST((CASE WHEN T.DATABASE_ID = 201004 THEN T.TABLE_NAME WHEN T.TABLE_TYPE = 5 THEN SUBSTR(T.TABLE_NAME, 7 + INSTR(SUBSTR(T.TABLE_NAME, 7), '_')) ELSE T.TABLE_NAME END) AS VARCHAR2(128)) AS OBJECT_NAME ,P.PART_NAME SUBOBJECT_NAME ,P.PART_ID OBJECT_ID ,CASE WHEN P.TABLET_ID != 0 THEN P.TABLET_ID ELSE NULL END AS DATA_OBJECT_ID ,DECODE (T.TABLE_TYPE, 5, 'INDEX PARTITION', 'TABLE PARTITION') AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY , NULL AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T JOIN SYS.ALL_VIRTUAL_PART_REAL_AGENT P ON T.TABLE_ID = P.TABLE_ID WHERE T.TENANT_ID = EFFECTIVE_TENANT_ID() AND P.TENANT_ID = EFFECTIVE_TENANT_ID() AND T.TABLE_TYPE != 12 AND T.TABLE_TYPE != 13 AND bitand((T.TABLE_MODE / 4096), 15) IN (0,1) UNION ALL SELECT SUBP.TENANT_ID ,SUBP.GMT_CREATE ,SUBP.GMT_MODIFIED ,T.DATABASE_ID ,CAST((CASE WHEN T.DATABASE_ID = 201004 THEN T.TABLE_NAME WHEN T.TABLE_TYPE = 5 THEN SUBSTR(T.TABLE_NAME, 7 + INSTR(SUBSTR(T.TABLE_NAME, 7), '_')) ELSE T.TABLE_NAME END) AS VARCHAR2(128)) AS OBJECT_NAME ,SUBP.SUB_PART_NAME SUBOBJECT_NAME ,SUBP.SUB_PART_ID OBJECT_ID ,SUBP.TABLET_ID AS DATA_OBJECT_ID ,DECODE (T.TABLE_TYPE, 5, 'INDEX SUBPARTITION', 'TABLE SUBPARTITION') AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'Y' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T, SYS.ALL_VIRTUAL_PART_REAL_AGENT P,SYS.ALL_VIRTUAL_SUB_PART_REAL_AGENT SUBP WHERE T.TABLE_ID =P.TABLE_ID AND P.TABLE_ID=SUBP.TABLE_ID AND P.PART_ID =SUBP.PART_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() AND P.TENANT_ID = EFFECTIVE_TENANT_ID() AND SUBP.TENANT_ID = EFFECTIVE_TENANT_ID() AND T.TABLE_TYPE != 12 AND T.TABLE_TYPE != 13 AND bitand((T.TABLE_MODE / 4096), 15) IN (0,1) UNION ALL SELECT EFFECTIVE_TENANT_ID() AS TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,CAST(201006 AS NUMBER) AS DATABASE_ID ,PACKAGE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,PACKAGE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,CASE WHEN TYPE = 1 THEN 'PACKAGE' WHEN TYPE = 2 THEN 'PACKAGE BODY' ELSE NULL END AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_PACKAGE_SYS_AGENT UNION ALL SELECT P.TENANT_ID ,P.GMT_CREATE ,P.GMT_MODIFIED ,P.DATABASE_ID ,P.PACKAGE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,P.PACKAGE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,CASE WHEN TYPE = 1 THEN 'PACKAGE' WHEN TYPE = 2 THEN 'PACKAGE BODY' ELSE NULL END AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE P.TENANT_ID = E.TENANT_ID AND P.PACKAGE_ID = E.OBJ_ID AND (E.OBJ_TYPE = 3 OR E.OBJ_TYPE = 5)) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_PACKAGE_REAL_AGENT P WHERE P.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT R.TENANT_ID ,R.GMT_CREATE ,R.GMT_MODIFIED ,R.DATABASE_ID ,R.ROUTINE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,R.ROUTINE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,CASE WHEN ROUTINE_TYPE = 1 THEN 'PROCEDURE' WHEN ROUTINE_TYPE = 2 THEN 'FUNCTION' ELSE NULL END AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE R.TENANT_ID = E.TENANT_ID AND R.ROUTINE_ID = E.OBJ_ID AND (E.OBJ_TYPE = 9 OR E.OBJ_TYPE = 12)) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_ROUTINE_REAL_AGENT R WHERE (ROUTINE_TYPE = 1 OR ROUTINE_TYPE = 2) AND R.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT EFFECTIVE_TENANT_ID() AS TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,CAST(201006 AS NUMBER) AS DATABASE_ID ,TS.TYPE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,TS.TYPE_ID AS OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE' AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE TS.TENANT_ID = E.TENANT_ID AND TS.TYPE_ID = E.OBJ_ID AND E.OBJ_TYPE = 4) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,TYPE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,TYPE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE' AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE TY.TENANT_ID = E.TENANT_ID AND TY.TYPE_ID = E.OBJ_ID AND E.OBJ_TYPE = 4) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TYPE_REAL_AGENT TY WHERE TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT EFFECTIVE_TENANT_ID() AS TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,CAST(201006 AS NUMBER) AS DATABASE_ID ,OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,OBJECT_TYPE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE BODY' AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE EFFECTIVE_TENANT_ID() = E.TENANT_ID AND TS.OBJECT_TYPE_ID = E.OBJ_ID AND E.OBJ_TYPE = 6) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TENANT_OBJECT_TYPE_SYS_AGENT TS WHERE TYPE = 2 UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,OBJECT_TYPE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE BODY' AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE TY.TENANT_ID = E.TENANT_ID AND TY.OBJECT_TYPE_ID = E.OBJ_ID AND E.OBJ_TYPE = 6) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TENANT_OBJECT_TYPE_REAL_AGENT TY WHERE TENANT_ID = EFFECTIVE_TENANT_ID() and TYPE = 2 UNION ALL SELECT T.TENANT_ID ,T.GMT_CREATE ,T.GMT_MODIFIED ,T.DATABASE_ID ,T.TRIGGER_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,T.TRIGGER_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TRIGGER' OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE T.TENANT_ID = E.TENANT_ID AND T.TRIGGER_ID = E.OBJ_ID AND (E.OBJ_TYPE = 7)) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT T WHERE T.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,SEQUENCE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,SEQUENCE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'SEQUENCE' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_SEQUENCE_OBJECT_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,SYNONYM_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,SYNONYM_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'SYNONYM' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_SYNONYM_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() /*UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,CAST(201006 AS NUMBER) AS DATABASE_ID ,NAMESPACE AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,CONTEXT_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'CONTEXT' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,21 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_CONTEXT_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID()*/ UNION ALL SELECT TENANT_ID, GMT_CREATE, GMT_MODIFIED, DATABASE_ID, DATABASE_NAME AS OBJECT_NAME, NULL AS SUBOBJECT_NAME, DATABASE_ID AS OBJECT_ID, NULL AS DATA_OBJECT_ID, 'DATABASE' AS OBJECT_TYPE, 'VALID' AS STATUS, 'N' AS TEMPORARY, 'N' AS "GENERATED", 'N' AS SECONDARY, 0 AS NAMESPACE, NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() ) A JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT B ON A.TENANT_ID = B.TENANT_ID AND A.DATABASE_ID = B.DATABASE_ID AND B.TENANT_ID = EFFECTIVE_TENANT_ID() AND A.DATABASE_ID = USERENV('SCHEMAID') )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(A.OBJECT_NAME AS VARCHAR(128)) AS OBJECT_NAME, CAST(A.SUBOBJECT_NAME AS VARCHAR2(128)) AS SUBOBJECT_NAME, CAST(A.OBJECT_ID AS NUMBER) AS OBJECT_ID, CAST(A.DATA_OBJECT_ID AS NUMBER) AS DATA_OBJECT_ID, CAST(A.OBJECT_TYPE AS VARCHAR2(23)) AS OBJECT_TYPE, CAST(A.GMT_CREATE AS DATE) AS CREATED, CAST(A.GMT_MODIFIED AS DATE) AS LAST_DDL_TIME, CAST(TO_CHAR(A.GMT_CREATE) AS VARCHAR2(19)) AS TIMESTAMP, CAST(A.STATUS AS VARCHAR2(7)) AS STATUS, CAST(A.TEMPORARY AS VARCHAR2(1)) AS TEMPORARY, CAST(A."GENERATED" AS VARCHAR2(1)) AS "GENERATED", CAST(A.SECONDARY AS VARCHAR2(1)) AS SECONDARY, CAST(A.NAMESPACE AS NUMBER) AS NAMESPACE, CAST(A.EDITION_NAME AS VARCHAR2(128)) AS EDITION_NAME, CAST(NULL AS VARCHAR2(18)) AS SHARING, CAST(NULL AS VARCHAR2(1)) AS EDITIONABLE, CAST(NULL AS VARCHAR2(1)) AS ORACLE_MAINTAINED, CAST(NULL AS VARCHAR2(1)) AS APPLICATION, CAST(NULL AS VARCHAR2(1)) AS DEFAULT_COLLATION, CAST(NULL AS VARCHAR2(1)) AS DUPLICATED, CAST(NULL AS VARCHAR2(1)) AS SHARDED, CAST(NULL AS VARCHAR2(1)) AS IMPORTED_OBJECT, CAST(NULL AS NUMBER) AS CREATED_APPID, CAST(NULL AS NUMBER) AS CREATED_VSNID, CAST(NULL AS NUMBER) AS MODIFIED_APPID, CAST(NULL AS NUMBER) AS MODIFIED_VSNID FROM ( SELECT A.TENANT_ID, (TO_DATE('19700101','YYYYMMDD') + B.SCHEMA_VERSION / 86400 / 1000000 + TO_NUMBER(SUBSTR(TZ_OFFSET(SESSIONTIMEZONE),1,3))/24) AS GMT_CREATE, (TO_DATE('19700101','YYYYMMDD') + A.SCHEMA_VERSION / 86400 / 1000000 + TO_NUMBER(SUBSTR(TZ_OFFSET(SESSIONTIMEZONE),1,3))/24) AS GMT_MODIFIED, A.DATABASE_ID, A.TABLE_NAME AS OBJECT_NAME, NULL AS SUBOBJECT_NAME, CAST(A.TABLE_ID AS NUMBER) AS OBJECT_ID, A.TABLE_ID AS DATA_OBJECT_ID, 'TABLE' AS OBJECT_TYPE, 'VALID' AS STATUS, 'N' AS TEMPORARY, 'N' AS "GENERATED", 'N' AS SECONDARY, 0 AS NAMESPACE, NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_CORE_ALL_TABLE A JOIN SYS.ALL_VIRTUAL_CORE_ALL_TABLE B ON A.TENANT_ID = B.TENANT_ID WHERE B.TABLE_NAME = '__all_core_table' AND A.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,CAST((CASE WHEN DATABASE_ID = 201004 THEN TABLE_NAME WHEN TABLE_TYPE = 5 THEN SUBSTR(TABLE_NAME, 7 + INSTR(SUBSTR(TABLE_NAME, 7), '_')) ELSE TABLE_NAME END) AS VARCHAR2(128)) AS OBJECT_NAME ,NULL SUBOBJECT_NAME ,TABLE_ID OBJECT_ID ,(CASE WHEN TABLET_ID != 0 THEN TABLET_ID ELSE NULL END) DATA_OBJECT_ID ,CASE WHEN TABLE_TYPE IN (0,3,6,8,9,14) THEN 'TABLE' WHEN TABLE_TYPE IN (2) THEN 'VIRTUAL TABLE' WHEN TABLE_TYPE IN (1,4) THEN 'VIEW' WHEN TABLE_TYPE IN (5) THEN 'INDEX' WHEN TABLE_TYPE IN (7) THEN 'MATERIALIZED VIEW' WHEN TABLE_TYPE IN (15) THEN 'MATERIALIZED VIEW LOG' ELSE NULL END AS OBJECT_TYPE ,CAST(CASE WHEN TABLE_TYPE IN (5,15) THEN CASE WHEN INDEX_STATUS = 2 THEN 'VALID' WHEN INDEX_STATUS = 3 THEN 'CHECKING' WHEN INDEX_STATUS = 4 THEN 'INELEGIBLE' WHEN INDEX_STATUS = 5 THEN 'ERROR' ELSE 'UNUSABLE' END ELSE CASE WHEN OBJECT_STATUS = 1 THEN 'VALID' ELSE 'INVALID' END END AS VARCHAR2(10)) AS STATUS ,CASE WHEN TABLE_TYPE IN (6,8,9) THEN 'Y' ELSE 'N' END AS TEMPORARY ,CASE WHEN TABLE_TYPE IN (0,1) THEN 'Y' ELSE 'N' END AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() AND TABLE_TYPE != 12 AND TABLE_TYPE != 13 AND bitand((TABLE_MODE / 4096), 15) IN (0,1) UNION ALL SELECT CST.TENANT_ID ,CST.GMT_CREATE ,CST.GMT_MODIFIED ,DB.DATABASE_ID ,CST.CONSTRAINT_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,TBL.TABLE_ID AS OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'INDEX' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_CONSTRAINT_REAL_AGENT CST, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT TBL, SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB WHERE CST.TENANT_ID = EFFECTIVE_TENANT_ID() AND DB.DATABASE_ID = TBL.DATABASE_ID AND TBL.TABLE_ID = CST.TABLE_ID and CST.CONSTRAINT_TYPE = 1 AND TBL.TABLE_TYPE != 12 AND TBL.TABLE_TYPE != 13 AND bitand((TBL.TABLE_MODE / 4096), 15) IN (0,1) UNION ALL SELECT P.TENANT_ID ,P.GMT_CREATE ,P.GMT_MODIFIED ,T.DATABASE_ID ,CAST((CASE WHEN T.DATABASE_ID = 201004 THEN T.TABLE_NAME WHEN T.TABLE_TYPE = 5 THEN SUBSTR(T.TABLE_NAME, 7 + INSTR(SUBSTR(T.TABLE_NAME, 7), '_')) ELSE T.TABLE_NAME END) AS VARCHAR2(128)) AS OBJECT_NAME ,P.PART_NAME SUBOBJECT_NAME ,P.PART_ID OBJECT_ID ,CASE WHEN P.TABLET_ID != 0 THEN P.TABLET_ID ELSE NULL END AS DATA_OBJECT_ID ,DECODE (T.TABLE_TYPE, 5, 'INDEX PARTITION', 'TABLE PARTITION') AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY , NULL AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T JOIN SYS.ALL_VIRTUAL_PART_REAL_AGENT P ON T.TABLE_ID = P.TABLE_ID WHERE T.TENANT_ID = EFFECTIVE_TENANT_ID() AND P.TENANT_ID = EFFECTIVE_TENANT_ID() AND T.TABLE_TYPE != 12 AND T.TABLE_TYPE != 13 AND bitand((T.TABLE_MODE / 4096), 15) IN (0,1) UNION ALL SELECT SUBP.TENANT_ID ,SUBP.GMT_CREATE ,SUBP.GMT_MODIFIED ,T.DATABASE_ID ,CAST((CASE WHEN T.DATABASE_ID = 201004 THEN T.TABLE_NAME WHEN T.TABLE_TYPE = 5 THEN SUBSTR(T.TABLE_NAME, 7 + INSTR(SUBSTR(T.TABLE_NAME, 7), '_')) ELSE T.TABLE_NAME END) AS VARCHAR2(128)) AS OBJECT_NAME ,SUBP.SUB_PART_NAME SUBOBJECT_NAME ,SUBP.SUB_PART_ID OBJECT_ID ,SUBP.TABLET_ID AS DATA_OBJECT_ID ,DECODE (T.TABLE_TYPE, 5, 'INDEX SUBPARTITION', 'TABLE SUBPARTITION') AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'Y' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T, SYS.ALL_VIRTUAL_PART_REAL_AGENT P,SYS.ALL_VIRTUAL_SUB_PART_REAL_AGENT SUBP WHERE T.TABLE_ID =P.TABLE_ID AND P.TABLE_ID=SUBP.TABLE_ID AND P.PART_ID =SUBP.PART_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() AND P.TENANT_ID = EFFECTIVE_TENANT_ID() AND SUBP.TENANT_ID = EFFECTIVE_TENANT_ID() AND T.TABLE_TYPE != 12 AND T.TABLE_TYPE != 13 AND bitand((T.TABLE_MODE / 4096), 15) IN (0,1) UNION ALL SELECT EFFECTIVE_TENANT_ID() AS TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,CAST(201006 AS NUMBER) AS DATABASE_ID ,PACKAGE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,PACKAGE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,CASE WHEN TYPE = 1 THEN 'PACKAGE' WHEN TYPE = 2 THEN 'PACKAGE BODY' ELSE NULL END AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_PACKAGE_SYS_AGENT UNION ALL SELECT P.TENANT_ID ,P.GMT_CREATE ,P.GMT_MODIFIED ,P.DATABASE_ID ,P.PACKAGE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,P.PACKAGE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,CASE WHEN TYPE = 1 THEN 'PACKAGE' WHEN TYPE = 2 THEN 'PACKAGE BODY' ELSE NULL END AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE P.TENANT_ID = E.TENANT_ID AND P.PACKAGE_ID = E.OBJ_ID AND (E.OBJ_TYPE = 3 OR E.OBJ_TYPE = 5)) THEN 'INVALID' WHEN TYPE = 2 AND EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT Eb WHERE OBJ_ID IN (SELECT PACKAGE_ID FROM SYS.ALL_VIRTUAL_PACKAGE_REAL_AGENT Pb WHERE Pb.PACKAGE_NAME = P.PACKAGE_NAME AND Pb.DATABASE_ID = P.DATABASE_ID AND Pb.TENANT_ID = P.TENANT_ID AND TYPE = 1) AND Eb.OBJ_TYPE = 3) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_PACKAGE_REAL_AGENT P WHERE P.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT R.TENANT_ID ,R.GMT_CREATE ,R.GMT_MODIFIED ,R.DATABASE_ID ,R.ROUTINE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,R.ROUTINE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,CASE WHEN ROUTINE_TYPE = 1 THEN 'PROCEDURE' WHEN ROUTINE_TYPE = 2 THEN 'FUNCTION' ELSE NULL END AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE R.TENANT_ID = E.TENANT_ID AND R.ROUTINE_ID = E.OBJ_ID AND (E.OBJ_TYPE = 9 OR E.OBJ_TYPE = 12)) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_ROUTINE_REAL_AGENT R WHERE (ROUTINE_TYPE = 1 OR ROUTINE_TYPE = 2) AND R.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT EFFECTIVE_TENANT_ID() AS TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,CAST(201006 AS NUMBER) AS DATABASE_ID ,TS.TYPE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,TS.TYPE_ID AS OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE' AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE TS.TENANT_ID = E.TENANT_ID AND TS.TYPE_ID = E.OBJ_ID AND E.OBJ_TYPE = 4) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,TYPE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,TYPE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE' AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE TY.TENANT_ID = E.TENANT_ID AND TY.TYPE_ID = E.OBJ_ID AND E.OBJ_TYPE = 4) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TYPE_REAL_AGENT TY WHERE TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT EFFECTIVE_TENANT_ID() AS TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,CAST(201006 AS NUMBER) AS DATABASE_ID ,OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,OBJECT_TYPE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE BODY' AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE EFFECTIVE_TENANT_ID() = E.TENANT_ID AND TS.OBJECT_TYPE_ID = E.OBJ_ID AND E.OBJ_TYPE = 6) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TENANT_OBJECT_TYPE_SYS_AGENT TS WHERE TYPE = 2 UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,OBJECT_TYPE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TYPE BODY' AS OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE TY.TENANT_ID = E.TENANT_ID AND TY.OBJECT_TYPE_ID = E.OBJ_ID AND E.OBJ_TYPE = 6) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TENANT_OBJECT_TYPE_REAL_AGENT TY WHERE TENANT_ID = EFFECTIVE_TENANT_ID() and TYPE = 2 UNION ALL SELECT T.TENANT_ID ,T.GMT_CREATE ,T.GMT_MODIFIED ,T.DATABASE_ID ,T.TRIGGER_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,T.TRIGGER_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'TRIGGER' OBJECT_TYPE ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE T.TENANT_ID = E.TENANT_ID AND T.TRIGGER_ID = E.OBJ_ID AND (E.OBJ_TYPE = 7)) THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT T WHERE T.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,SEQUENCE_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,SEQUENCE_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'SEQUENCE' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_SEQUENCE_OBJECT_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,DATABASE_ID ,SYNONYM_NAME AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,SYNONYM_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'SYNONYM' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY , 0 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_SYNONYM_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() /*UNION ALL SELECT TENANT_ID ,GMT_CREATE ,GMT_MODIFIED ,CAST(201006 AS NUMBER) AS DATABASE_ID ,NAMESPACE AS OBJECT_NAME ,NULL AS SUBOBJECT_NAME ,CONTEXT_ID OBJECT_ID ,NULL AS DATA_OBJECT_ID ,'CONTEXT' AS OBJECT_TYPE ,'VALID' AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" ,'N' AS SECONDARY ,21 AS NAMESPACE ,NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_CONTEXT_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID()*/ UNION ALL SELECT TENANT_ID, GMT_CREATE, GMT_MODIFIED, DATABASE_ID, DATABASE_NAME AS OBJECT_NAME, NULL AS SUBOBJECT_NAME, DATABASE_ID AS OBJECT_ID, NULL AS DATA_OBJECT_ID, 'DATABASE' AS OBJECT_TYPE, 'VALID' AS STATUS, 'N' AS TEMPORARY, 'N' AS "GENERATED", 'N' AS SECONDARY, 0 AS NAMESPACE, NULL AS EDITION_NAME FROM SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() ) A JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT B ON A.TENANT_ID = B.TENANT_ID AND A.DATABASE_ID = B.DATABASE_ID AND B.TENANT_ID = EFFECTIVE_TENANT_ID() AND A.DATABASE_ID = USERENV('SCHEMAID') )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index d0599968f..c8d967711 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -20696,6 +20696,13 @@ def_table_schema( (SELECT OBJ_ID FROM OCEANBASE.__ALL_VIRTUAL_ERROR E WHERE P.TENANT_ID = E.TENANT_ID AND P.PACKAGE_ID = E.OBJ_ID AND (E.OBJ_TYPE = 3 OR E.OBJ_TYPE = 5)) THEN 'INVALID' + WHEN TYPE = 2 AND EXISTS + (SELECT OBJ_ID FROM OCEANBASE.__ALL_VIRTUAL_ERROR Eb + WHERE OBJ_ID IN + (SELECT PACKAGE_ID FROM OCEANBASE.__ALL_VIRTUAL_PACKAGE Pb + WHERE Pb.PACKAGE_NAME = P.PACKAGE_NAME AND Pb.DATABASE_ID = P.DATABASE_ID AND Pb.TENANT_ID = P.TENANT_ID AND TYPE = 1) + AND Eb.OBJ_TYPE = 3) + THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" @@ -22720,6 +22727,13 @@ def_table_schema( (SELECT OBJ_ID FROM OCEANBASE.__ALL_TENANT_ERROR E WHERE P.TENANT_ID = E.TENANT_ID AND P.PACKAGE_ID = E.OBJ_ID AND (E.OBJ_TYPE = 3 OR E.OBJ_TYPE = 5)) THEN 'INVALID' + WHEN TYPE = 2 AND EXISTS + (SELECT OBJ_ID FROM OCEANBASE.__ALL_TENANT_ERROR Eb + WHERE OBJ_ID IN + (SELECT PACKAGE_ID FROM OCEANBASE.__ALL_PACKAGE Pb + WHERE Pb.PACKAGE_NAME = P.PACKAGE_NAME AND Pb.DATABASE_ID = P.DATABASE_ID AND Pb.TENANT_ID = P.TENANT_ID AND TYPE = 1) + AND Eb.OBJ_TYPE = 3) + THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" @@ -36228,6 +36242,13 @@ def_table_schema( (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE P.TENANT_ID = E.TENANT_ID AND P.PACKAGE_ID = E.OBJ_ID AND (E.OBJ_TYPE = 3 OR E.OBJ_TYPE = 5)) THEN 'INVALID' + WHEN TYPE = 2 AND EXISTS + (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT Eb + WHERE OBJ_ID IN + (SELECT PACKAGE_ID FROM SYS.ALL_VIRTUAL_PACKAGE_REAL_AGENT Pb + WHERE Pb.PACKAGE_NAME = P.PACKAGE_NAME AND Pb.DATABASE_ID = P.DATABASE_ID AND Pb.TENANT_ID = P.TENANT_ID AND TYPE = 1) + AND Eb.OBJ_TYPE = 3) + THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" @@ -36722,6 +36743,13 @@ def_table_schema( (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE P.TENANT_ID = E.TENANT_ID AND P.PACKAGE_ID = E.OBJ_ID AND (E.OBJ_TYPE = 3 OR E.OBJ_TYPE = 5)) THEN 'INVALID' + WHEN TYPE = 2 AND EXISTS + (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT Eb + WHERE OBJ_ID IN + (SELECT PACKAGE_ID FROM SYS.ALL_VIRTUAL_PACKAGE_REAL_AGENT Pb + WHERE Pb.PACKAGE_NAME = P.PACKAGE_NAME AND Pb.DATABASE_ID = P.DATABASE_ID AND Pb.TENANT_ID = P.TENANT_ID AND TYPE = 1) + AND Eb.OBJ_TYPE = 3) + THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" @@ -37219,7 +37247,14 @@ def_table_schema( ,CASE WHEN EXISTS (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT E WHERE P.TENANT_ID = E.TENANT_ID AND P.PACKAGE_ID = E.OBJ_ID AND (E.OBJ_TYPE = 3 OR E.OBJ_TYPE = 5)) - THEN 'INVALID' + THEN 'INVALID' + WHEN TYPE = 2 AND EXISTS + (SELECT OBJ_ID FROM SYS.ALL_VIRTUAL_TENANT_ERROR_REAL_AGENT Eb + WHERE OBJ_ID IN + (SELECT PACKAGE_ID FROM SYS.ALL_VIRTUAL_PACKAGE_REAL_AGENT Pb + WHERE Pb.PACKAGE_NAME = P.PACKAGE_NAME AND Pb.DATABASE_ID = P.DATABASE_ID AND Pb.TENANT_ID = P.TENANT_ID AND TYPE = 1) + AND Eb.OBJ_TYPE = 3) + THEN 'INVALID' ELSE 'VALID' END AS STATUS ,'N' AS TEMPORARY ,'N' AS "GENERATED" From 9f944176dbe6d780d59cba706836faf9b1897632 Mon Sep 17 00:00:00 2001 From: obdev Date: Thu, 15 Aug 2024 04:50:13 +0000 Subject: [PATCH 054/249] FEATURE: add compat code for tablelock owner id --- .../tablelock/ob_table_lock_common.cpp | 25 +++++++++++++++++-- src/storage/tablelock/ob_table_lock_common.h | 6 +++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/storage/tablelock/ob_table_lock_common.cpp b/src/storage/tablelock/ob_table_lock_common.cpp index ad659429f..f5bdd891c 100644 --- a/src/storage/tablelock/ob_table_lock_common.cpp +++ b/src/storage/tablelock/ob_table_lock_common.cpp @@ -227,11 +227,32 @@ int ObTableLockOwnerID::serialize(char* buf, const int64_t buf_len, int64_t& pos int ObTableLockOwnerID::deserialize(const char* buf, const int64_t data_len, int64_t& pos) { int ret = OB_SUCCESS; + const int64_t origin_pos = pos; + int16_t magic_num = 0; if (OB_ISNULL(buf) || OB_UNLIKELY(data_len <= 0)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", KR(ret), KP(buf), K(data_len)); - } else if (OB_FAIL(serialization::decode_vi64(buf, data_len, pos, &pack_))) { - LOG_WARN("deserialize ID failed", KR(ret), KP(buf), K(data_len), K(pos)); + } else if (OB_FAIL(serialization::decode(buf, data_len, pos, magic_num))) { + LOG_WARN("deserialize magic num failed", KR(ret), KP(buf), K(data_len), K(pos)); + } else { + pos = origin_pos; + // new type + if (magic_num == ObNewTableLockOwnerID::MAGIC_NUM) { + unsigned char type = 0; + int64_t id = 0; + LST_DO_CODE(OB_UNIS_DECODE, + magic_num, + type, + id); + if (OB_SUCC(ret)) { + type_ = type; + id_ = id; + } + } else { + if (OB_FAIL(serialization::decode_vi64(buf, data_len, pos, &pack_))) { + LOG_WARN("deserialize ID failed", KR(ret), KP(buf), K(data_len), K(pos)); + } + } } return ret; } diff --git a/src/storage/tablelock/ob_table_lock_common.h b/src/storage/tablelock/ob_table_lock_common.h index 84d213e9f..193d7fb58 100644 --- a/src/storage/tablelock/ob_table_lock_common.h +++ b/src/storage/tablelock/ob_table_lock_common.h @@ -447,6 +447,12 @@ bool is_lock_owner_type_valid(const ObLockOwnerType &type) return (type < ObLockOwnerType::MAX_OWNER_TYPE); } +class ObNewTableLockOwnerID +{ +public: + static const int16_t MAGIC_NUM = -0xABC; +}; + class ObTableLockOwnerID { public: From 466ba5117198d722ed6d8d67b5b9f597860fbbd8 Mon Sep 17 00:00:00 2001 From: obdev Date: Thu, 15 Aug 2024 05:34:31 +0000 Subject: [PATCH 055/249] fix alloc concurrently not safe. --- src/storage/ddl/ob_ddl_redo_log_writer.cpp | 31 +++++++++++++++------- src/storage/ddl/ob_ddl_redo_log_writer.h | 4 +-- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/storage/ddl/ob_ddl_redo_log_writer.cpp b/src/storage/ddl/ob_ddl_redo_log_writer.cpp index 2cf674f44..2b2ce618c 100644 --- a/src/storage/ddl/ob_ddl_redo_log_writer.cpp +++ b/src/storage/ddl/ob_ddl_redo_log_writer.cpp @@ -340,24 +340,34 @@ void ObDDLCtrlSpeedHandle::ObDDLCtrlSpeedItemHandle::reset() if (nullptr != item_) { if (0 == item_->dec_ref()) { item_->~ObDDLCtrlSpeedItem(); + ObDDLCtrlSpeedHandle::get_instance().get_allocator().free(item_); } item_ = nullptr; } } ObDDLCtrlSpeedHandle::ObDDLCtrlSpeedHandle() - : is_inited_(false), speed_handle_map_(), allocator_(SET_USE_500("DDLClogCtrl")), + : is_inited_(false), speed_handle_map_(), bucket_lock_(), refreshTimerTask_() { } ObDDLCtrlSpeedHandle::~ObDDLCtrlSpeedHandle() { - bucket_lock_.destroy(); + int ret = OB_SUCCESS; if (speed_handle_map_.created()) { + ObArray remove_items; + common::hash::ObHashMap::const_iterator iter = speed_handle_map_.begin(); + for (; iter != speed_handle_map_.end(); ++iter) { + if (OB_FAIL(remove_items.push_back(iter->first))) { + LOG_WARN("push back failed", K(ret), "key", iter->first); + } + } + (void)remove_ctrl_speed_item(remove_items); speed_handle_map_.destroy(); } - allocator_.reset(); + bucket_lock_.destroy(); + allocator_.destroy(); } ObDDLCtrlSpeedHandle &ObDDLCtrlSpeedHandle::get_instance() @@ -371,6 +381,7 @@ int ObDDLCtrlSpeedHandle::init() int ret = OB_SUCCESS; lib::ObMemAttr attr(OB_SERVER_TENANT_ID, "DDLSpeedCtrl"); SET_USE_500(attr); + const int64_t memory_limit = 1024L * 1024L * 1024L * 1L; // 1GB if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("inited twice", K(ret)); @@ -381,13 +392,14 @@ int ObDDLCtrlSpeedHandle::init() LOG_WARN("init bucket lock failed", K(ret)); } else if (OB_FAIL(speed_handle_map_.create(MAP_BUCKET_NUM, attr, attr))) { LOG_WARN("fail to create speed handle map", K(ret)); + } else if (OB_FAIL(allocator_.init(OB_MALLOC_NORMAL_BLOCK_SIZE, + attr.label_, OB_SERVER_TENANT_ID, memory_limit))) { + LOG_WARN("init alloctor failed", K(ret)); + } else if (OB_FAIL(refreshTimerTask_.init(lib::TGDefIDs::ServerGTimer))) { + LOG_WARN("fail to init refreshTimerTask", K(ret)); } else { is_inited_ = true; - if (OB_FAIL(refreshTimerTask_.init(lib::TGDefIDs::ServerGTimer))) { - LOG_WARN("fail to init refreshTimerTask", K(ret)); - } else { - LOG_INFO("succeed to init ObDDLCtrlSpeedHandle", K(ret)); - } + LOG_INFO("succeed to init ObDDLCtrlSpeedHandle", K(ret)); } return ret; } @@ -502,7 +514,7 @@ int ObDDLCtrlSpeedHandle::add_ctrl_speed_item( int ObDDLCtrlSpeedHandle::remove_ctrl_speed_item(const ObIArray &remove_items) { int ret = OB_SUCCESS; - for (int64_t i = 0; OB_SUCC(ret) && i < remove_items.count(); i++) { + for (int64_t i = 0; i < remove_items.count(); i++) { // ignore ret, to free more items. const SpeedHandleKey &speed_handle_key = remove_items.at(i); common::ObBucketHashWLockGuard guard(bucket_lock_, speed_handle_key.hash()); char *buf = nullptr; @@ -526,6 +538,7 @@ int ObDDLCtrlSpeedHandle::remove_ctrl_speed_item(const ObIArray } else { if (0 == speed_handle_item->dec_ref()) { speed_handle_item->~ObDDLCtrlSpeedItem(); + allocator_.free(speed_handle_item); speed_handle_item = nullptr; } } diff --git a/src/storage/ddl/ob_ddl_redo_log_writer.h b/src/storage/ddl/ob_ddl_redo_log_writer.h index 1b171f092..ff20b411a 100644 --- a/src/storage/ddl/ob_ddl_redo_log_writer.h +++ b/src/storage/ddl/ob_ddl_redo_log_writer.h @@ -124,7 +124,7 @@ public: const int64_t task_id, ObDDLNeedStopWriteChecker &checker, int64_t &real_sleep_us); - + ObIAllocator &get_allocator() { return allocator_; } private: struct SpeedHandleKey { public: @@ -203,7 +203,7 @@ private: static const int64_t MAP_BUCKET_NUM = 1024; bool is_inited_; common::hash::ObHashMap speed_handle_map_; - common::ObArenaAllocator allocator_; + common::ObConcurrentFIFOAllocator allocator_; common::ObBucketLock bucket_lock_; RefreshSpeedHandleTask refreshTimerTask_; }; From 49e178c517a422565200814017a4a6c1131dd2e7 Mon Sep 17 00:00:00 2001 From: qingsuijiu <642782632@qq.com> Date: Thu, 15 Aug 2024 05:51:56 +0000 Subject: [PATCH 056/249] Print a log message during update_qc_error_code to indicate which node the new error code originates from. --- src/sql/engine/px/ob_px_coord_op.cpp | 3 ++- src/sql/engine/px/ob_px_scheduler.cpp | 12 ++++++++---- src/sql/engine/px/ob_px_util.h | 5 ++++- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/sql/engine/px/ob_px_coord_op.cpp b/src/sql/engine/px/ob_px_coord_op.cpp index 73e9c7333..c5ce12974 100644 --- a/src/sql/engine/px/ob_px_coord_op.cpp +++ b/src/sql/engine/px/ob_px_coord_op.cpp @@ -845,7 +845,8 @@ int ObPxCoordOp::check_all_sqc(ObIArray &active_dfos, } else if (sqc->is_server_not_alive() || sqc->is_interrupt_by_dm()) { if (sqc->is_interrupt_by_dm()) { ObRpcResultCode err_msg; - ObPxErrorUtil::update_qc_error_code(coord_info_.first_error_code_, OB_RPC_CONNECT_ERROR, err_msg); + ObPxErrorUtil::update_qc_error_code(coord_info_.first_error_code_, + OB_RPC_CONNECT_ERROR, err_msg, sqc->get_exec_addr()); } sqc->set_server_not_alive(false); sqc->set_interrupt_by_dm(false); diff --git a/src/sql/engine/px/ob_px_scheduler.cpp b/src/sql/engine/px/ob_px_scheduler.cpp index a0b49cbf7..b7ec4e0b9 100644 --- a/src/sql/engine/px/ob_px_scheduler.cpp +++ b/src/sql/engine/px/ob_px_scheduler.cpp @@ -174,7 +174,8 @@ int ObPxMsgProc::on_sqc_init_msg(ObExecContext &ctx, const ObPxInitSqcResultMsg } else { if (OB_SUCCESS != pkt.rc_) { ret = pkt.rc_; - ObPxErrorUtil::update_qc_error_code(coord_info_.first_error_code_, pkt.rc_, pkt.err_msg_); + ObPxErrorUtil::update_qc_error_code(coord_info_.first_error_code_, + pkt.rc_, pkt.err_msg_, sqc->get_exec_addr()); LOG_WARN("fail init sqc, please check remote server log for details", "remote_server", sqc->get_exec_addr(), K(pkt), KP(ret)); } else if (pkt.task_count_ <= 0) { @@ -410,7 +411,8 @@ int ObPxMsgProc::process_sqc_finish_msg_once(ObExecContext &ctx, const ObPxFinis * 发送这个finish消息的sqc（包括它的worker）其实已经结束了，需要将它 * 但是因为出错了，后续的调度流程不需要继续了，后面流程会进行错误处理。 */ - ObPxErrorUtil::update_qc_error_code(coord_info_.first_error_code_, pkt.rc_, pkt.err_msg_); + ObPxErrorUtil::update_qc_error_code(coord_info_.first_error_code_, + pkt.rc_, pkt.err_msg_, sqc->get_exec_addr()); if (OB_SUCC(ret)) { if (OB_FAIL(pkt.rc_)) { DAS_CTX(ctx).get_location_router().save_cur_exec_status(pkt.rc_); @@ -618,7 +620,8 @@ int ObPxTerminateMsgProc::on_sqc_init_msg(ObExecContext &ctx, const ObPxInitSqcR if (pkt.rc_ != OB_SUCCESS) { LOG_DEBUG("receive error code from sqc init msg", K(coord_info_.first_error_code_), K(pkt.rc_)); } - ObPxErrorUtil::update_qc_error_code(coord_info_.first_error_code_, pkt.rc_, pkt.err_msg_); + ObPxErrorUtil::update_qc_error_code(coord_info_.first_error_code_, + pkt.rc_, pkt.err_msg_, sqc->get_exec_addr()); } if (OB_SUCC(ret)) { @@ -687,7 +690,8 @@ int ObPxTerminateMsgProc::on_sqc_finish_msg(ObExecContext &ctx, const ObPxFinish if (pkt.rc_ != OB_SUCCESS) { LOG_DEBUG("receive error code from sqc finish msg", K(coord_info_.first_error_code_), K(pkt.rc_)); } - ObPxErrorUtil::update_qc_error_code(coord_info_.first_error_code_, pkt.rc_, pkt.err_msg_); + ObPxErrorUtil::update_qc_error_code(coord_info_.first_error_code_, + pkt.rc_, pkt.err_msg_, sqc->get_exec_addr()); NG_TRACE_EXT(sqc_finish, OB_ID(dfo_id), sqc->get_dfo_id(), diff --git a/src/sql/engine/px/ob_px_util.h b/src/sql/engine/px/ob_px_util.h index 906a094ea..0c6faf08c 100644 --- a/src/sql/engine/px/ob_px_util.h +++ b/src/sql/engine/px/ob_px_util.h @@ -616,7 +616,8 @@ class ObPxErrorUtil public: static inline void update_qc_error_code(int ¤t_error_code, const int new_error_code, - const ObPxUserErrorMsg &from) + const ObPxUserErrorMsg &from, + const common::ObAddr &exec_addr) { int ret = OB_SUCCESS; // **replace** error code & error msg @@ -626,6 +627,8 @@ public: OB_GOT_SIGNAL_ABORTING == current_error_code) && OB_SUCCESS != new_error_code) { current_error_code = new_error_code; + SQL_LOG(WARN, "QC update the error code. Please visit the corresponding address for more details.", + K(new_error_code), K(exec_addr)); FORWARD_USER_ERROR(new_error_code, from.msg_); } } From 6ebe5187be8b95fd13c3453a764e5e3ed80f1bbe Mon Sep 17 00:00:00 2001 From: LoLolobster <949673574@qq.com> Date: Thu, 15 Aug 2024 05:57:44 +0000 Subject: [PATCH 057/249] fix ob_admin dump_backup only displays at most 99 sets/pieces --- .../ob_admin_dump_backup_data_executor.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp b/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp index e54b1c660..07ce8212c 100644 --- a/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp +++ b/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp @@ -2505,7 +2505,7 @@ int ObAdminDumpBackupDataExecutor::dump_tenant_backup_set_infos_(const ObIArray< OB_MAX_TIME_STR_LENGTH, pos))) { STORAGE_LOG(WARN, "fail to convert scn to str", K(ret), K(time_zone_wrap), K(backup_set_desc)); - } else if (OB_FAIL(databuff_printf(buf, OB_MAX_CHAR_LEN, "%ld", i+1))) { + } else if (OB_FAIL(databuff_printf(buf, OB_MAX_INTEGER_DISPLAY_WIDTH, "%ld", i+1))) { STORAGE_LOG(WARN, "fail to printf buf", K(ret), K(i)); } else if (OB_FALSE_IT(backup_set_desc.to_string(min_restore_scn_str_buf, str_buf, OB_MAX_TEXT_LENGTH))) { } else { @@ -2522,9 +2522,9 @@ int ObAdminDumpBackupDataExecutor::dump_backup_ls_meta_infos_file_(const storage PrintHelper::print_dump_title("ls meta infos"); ARRAY_FOREACH_X(ls_meta_infos.ls_meta_packages_, i , cnt, OB_SUCC(ret)) { const storage::ObLSMetaPackage &meta_package = ls_meta_infos.ls_meta_packages_.at(i); - char buf[OB_MAX_CHAR_LEN] = { 0 }; + char buf[OB_MAX_INTEGER_DISPLAY_WIDTH] = { 0 }; char str_buf[OB_MAX_TEXT_LENGTH] = { 0 }; - if (OB_FAIL(databuff_printf(buf, OB_MAX_CHAR_LEN, "%ld", i+1))) { + if (OB_FAIL(databuff_printf(buf, OB_MAX_INTEGER_DISPLAY_WIDTH, "%ld", i+1))) { STORAGE_LOG(WARN, "fail to printf buf", K(ret), K(i)); } else if (OB_FALSE_IT(meta_package.to_string(str_buf, OB_MAX_TEXT_LENGTH))) { } else { @@ -2647,9 +2647,9 @@ int ObAdminDumpBackupDataExecutor::dump_tenant_archive_piece_infos_file_(const s PrintHelper::print_dump_line("path", piece_infos_file.path_.ptr()); ARRAY_FOREACH_X(piece_infos_file.his_frozen_pieces_, i , cnt, OB_SUCC(ret)) { const ObTenantArchivePieceAttr &piece = piece_infos_file.his_frozen_pieces_.at(i); - char buf[OB_MAX_CHAR_LEN] = { 0 }; + char buf[OB_MAX_INTEGER_DISPLAY_WIDTH] = { 0 }; char str_buf[OB_MAX_TEXT_LENGTH] = { 0 }; - if (OB_FAIL(databuff_printf(buf, OB_MAX_CHAR_LEN, "%ld", i+1))) { + if (OB_FAIL(databuff_printf(buf, OB_MAX_INTEGER_DISPLAY_WIDTH, "%ld", i+1))) { STORAGE_LOG(WARN, "fail to printf buf", K(ret), K(i)); } else if (OB_FALSE_IT(piece.to_string(str_buf, OB_MAX_TEXT_LENGTH))) { } else { @@ -2991,9 +2991,9 @@ int ObAdminDumpBackupDataExecutor::dump_tenant_archive_path_() PrintHelper::print_dump_title("tenant archive round infos"); ARRAY_FOREACH_X(rounds, i , cnt, OB_SUCC(ret)) { const ObTenantArchiveRoundAttr &round = rounds.at(i); - char buf[OB_MAX_CHAR_LEN] = { 0 }; + char buf[OB_MAX_INTEGER_DISPLAY_WIDTH] = { 0 }; char str_buf[OB_MAX_TEXT_LENGTH] = { 0 }; - if (OB_FAIL(databuff_printf(buf, OB_MAX_CHAR_LEN, "%ld", i+1))) { + if (OB_FAIL(databuff_printf(buf, OB_MAX_INTEGER_DISPLAY_WIDTH, "%ld", i+1))) { STORAGE_LOG(WARN, "fail to printf buf", K(ret), K(i)); } else if (OB_FALSE_IT(round.to_string(str_buf, OB_MAX_TEXT_LENGTH))) { } else { From 6166e15d3b44fd4d394737ca89b2ef90f32a7cef Mon Sep 17 00:00:00 2001 From: "zhuyangkai@actionsky.com" Date: Thu, 15 Aug 2024 06:41:51 +0000 Subject: [PATCH 058/249] Issue#2078: For LOAD DATA LOCAL, no error is reported when the file name is empty. --- src/sql/resolver/cmd/ob_load_data_resolver.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/sql/resolver/cmd/ob_load_data_resolver.cpp b/src/sql/resolver/cmd/ob_load_data_resolver.cpp index ff10289bb..4e99c836c 100644 --- a/src/sql/resolver/cmd/ob_load_data_resolver.cpp +++ b/src/sql/resolver/cmd/ob_load_data_resolver.cpp @@ -590,8 +590,12 @@ int ObLoadDataResolver::resolve_filename(ObLoadDataStmt *load_stmt, ParseNode *n } else { ObString file_name(file_name_node->str_len_, file_name_node->str_value_); if (OB_UNLIKELY(file_name.empty())) { - ret = OB_FILE_NOT_EXIST; - LOG_WARN("file not exist", K(ret), K(file_name)); + if (ObLoadFileLocation::CLIENT_DISK != load_args.load_file_storage_) { + ret = OB_FILE_NOT_EXIST; + LOG_WARN("file not exist", K(ret), K(file_name)); + } else { + // do nothing + } } else { const char *p = nullptr; ObString sub_file_name; From 0759f42e449b96765c0f90d6303085ce301b4006 Mon Sep 17 00:00:00 2001 From: 0xacc Date: Thu, 15 Aug 2024 07:50:55 +0000 Subject: [PATCH 059/249] [CP] [FEAT MERGE] 424 PL stable features --- .gitignore | 1 + .../src/lib/mysqlclient/ob_isql_connection.h | 1 + .../src/lib/mysqlclient/ob_mysql_proxy.cpp | 1 + .../src/lib/mysqlclient/ob_mysql_proxy.h | 5 +- src/objit/CMakeLists.txt | 1 - src/objit/include/objit/ob_llvm_helper.h | 2 +- src/objit/src/core/jit_context.cpp | 36 +- src/objit/src/core/jit_context.h | 17 +- src/objit/src/core/ob_jit_allocator.cpp | 26 +- src/objit/src/core/ob_orc_jit.cpp | 257 +- src/objit/src/core/ob_orc_jit.h | 180 +- src/objit/src/ob_llvm_helper.cpp | 46 +- src/observer/mysql/ob_query_driver.cpp | 25 +- src/observer/mysql/ob_query_retry_ctrl.cpp | 21 +- src/observer/mysql/obmp_stmt_execute.cpp | 7 +- src/observer/mysql/obmp_stmt_prexecute.cpp | 10 +- src/observer/ob_inner_sql_connection.cpp | 5 + src/observer/ob_inner_sql_connection.h | 1 + src/pl/CMakeLists.txt | 33 +- src/pl/diagnosis/ob_pl_sql_audit_guard.cpp | 119 + src/pl/diagnosis/ob_pl_sql_audit_guard.h | 132 + src/pl/ob_pl.cpp | 7 +- src/pl/ob_pl_code_generator.cpp | 26 +- src/pl/ob_pl_package_manager.cpp | 497 ++- src/pl/ob_pl_package_manager.h | 52 +- src/pl/ob_pl_type.cpp | 13 +- src/pl/sys_package/ob_dbms_upgrade.cpp | 42 +- src/rootserver/ob_upgrade_executor.cpp | 26 +- .../sys_package/__dbms_upgrade_body_mysql.sql | 4 +- .../sys_package/__dbms_upgrade_mysql.sql | 5 +- src/sql/engine/basic/ob_ra_row_store.cpp | 9 + src/sql/ob_spi.cpp | 2779 ++++++----------- src/sql/ob_spi.h | 161 +- src/sql/session/ob_basic_session_info.cpp | 12 + src/sql/session/ob_basic_session_info.h | 14 +- 35 files changed, 2324 insertions(+), 2249 deletions(-) create mode 100644 src/pl/diagnosis/ob_pl_sql_audit_guard.cpp create mode 100644 src/pl/diagnosis/ob_pl_sql_audit_guard.h diff --git a/.gitignore b/.gitignore index 4ef465080..06b531bae 100644 --- a/.gitignore +++ b/.gitignore @@ -171,6 +171,7 @@ src/pl/parser/_gen_parser.error src/pl/parser/_gen_pl_parser.output src/pl/parser/pl_parser_mysql_mode.output src/pl/parser/pl_parser_oracle_mode.output +src/share/inner_table/sys_package/system_package.cpp ############# close_modules ############# close_modules/oracle_pl/pl/parser/*.output diff --git a/deps/oblib/src/lib/mysqlclient/ob_isql_connection.h b/deps/oblib/src/lib/mysqlclient/ob_isql_connection.h index e764c624b..42fc7c92e 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_isql_connection.h +++ b/deps/oblib/src/lib/mysqlclient/ob_isql_connection.h @@ -206,6 +206,7 @@ public: virtual void set_is_load_data_exec(bool v) { UNUSED(v); } virtual void set_force_remote_exec(bool v) { UNUSED(v); } virtual void set_use_external_session(bool v) { UNUSED(v); } + virtual void set_ob_enable_pl_cache(bool v) { UNUSED(v); } virtual int64_t get_cluster_id() const { return common::OB_INVALID_ID; } void set_session_init_status(bool status) { is_inited_ = status;} virtual void set_user_timeout(int64_t user_timeout) { UNUSED(user_timeout); } diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.cpp b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.cpp index 3e83a15ea..3152d5e72 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.cpp +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.cpp @@ -205,6 +205,7 @@ int ObCommonSqlProxy::write(const uint64_t tenant_id, const ObString sql, conn->set_is_load_data_exec(param->is_load_data_exec_); conn->set_use_external_session(param->use_external_session_); conn->set_group_id(param->consumer_group_id_); + conn->set_ob_enable_pl_cache(param->enable_pl_cache_); if (param->is_load_data_exec_) { is_user_sql = true; } diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h index 24ffa4682..ec19ac524 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h @@ -91,8 +91,8 @@ struct ObSessionParam final { public: ObSessionParam() - : sql_mode_(nullptr), tz_info_wrap_(nullptr), ddl_info_(), is_load_data_exec_(false), use_external_session_(false), consumer_group_id_(0), nls_formats_{} - {} + : sql_mode_(nullptr), tz_info_wrap_(nullptr), ddl_info_(), is_load_data_exec_(false), + use_external_session_(false), consumer_group_id_(0), nls_formats_{}, enable_pl_cache_(true) {} ~ObSessionParam() = default; public: int64_t *sql_mode_; @@ -102,6 +102,7 @@ public: bool use_external_session_; // need init remote inner sql conn with sess getting from sess mgr int64_t consumer_group_id_; common::ObString nls_formats_[common::ObNLSFormatEnum::NLS_MAX]; + bool enable_pl_cache_; }; // thread safe sql proxy diff --git a/src/objit/CMakeLists.txt b/src/objit/CMakeLists.txt index d215a7cb8..da695e6c2 100644 --- a/src/objit/CMakeLists.txt +++ b/src/objit/CMakeLists.txt @@ -20,7 +20,6 @@ message(STATUS "This is SOURCE dir " ${PROJECT_SOURCE_DIR}) include_directories(${LLVM_INCLUDE_DIRS}) add_definitions(${LLVM_DEFINITIONS}) -add_definitions(-Wno-deprecated) add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0 -g -O2 -frtti) # Find the libraries that correspond to the LLVM components diff --git a/src/objit/include/objit/ob_llvm_helper.h b/src/objit/include/objit/ob_llvm_helper.h index 7ba69952b..9165b99a2 100644 --- a/src/objit/include/objit/ob_llvm_helper.h +++ b/src/objit/include/objit/ob_llvm_helper.h @@ -365,7 +365,7 @@ public: void dump_module(); void dump_debuginfo(); int verify_module(); - uint64_t get_function_address(const common::ObString &name); + int get_function_address(const common::ObString &name, uint64_t &addr); static void add_symbol(const common::ObString &name, void *value); ObDIRawData get_debug_info() const; diff --git a/src/objit/src/core/jit_context.cpp b/src/objit/src/core/jit_context.cpp index dc61ba5e3..32adf61e0 100644 --- a/src/objit/src/core/jit_context.cpp +++ b/src/objit/src/core/jit_context.cpp @@ -23,22 +23,20 @@ namespace core using namespace llvm; -int JitContext::InitializeModule(ObOrcJit &jit) +int JitContext::InitializeModule(const ObDataLayout &DL) { int ret = OB_SUCCESS; - TheJIT = &jit; - if (nullptr == (TheModule = std::make_unique("PL/SQL", TheJIT->getContext()))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc memory for LLVM Module", K(ret)); - } else if (nullptr == (Builder = std::make_unique>(TheJIT->getContext()))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc memory for LLVM Builder", K(ret)); + if (OB_FAIL(ob_jit_make_unique(TheContext))) { + LOG_WARN("failed to make jit context", K(ret)); + } else if (OB_FAIL(ob_jit_make_unique(TheModule, "PL/SQL", *TheContext))) { + LOG_WARN("failed to make jit module", K(ret)); + } else if (OB_FAIL(ob_jit_make_unique(Builder, *TheContext))) { + LOG_WARN("failed to make ir builder", K(ret)); + } else if (OB_FAIL(ob_jit_make_unique(TheFPM, TheModule.get()))) { + LOG_WARN("failed to make FPM", K(ret)); } else { - TheContext = &TheJIT->getContext(); - TheModule->setDataLayout(TheJIT->getDataLayout()); - Builder = std::make_unique>(*TheContext); - TheFPM = std::make_unique(TheModule.get()); + TheModule->setDataLayout(DL); TheFPM->add(createInstructionCombiningPass()); TheFPM->add(createReassociatePass()); TheFPM->add(createGVNPass()); @@ -49,12 +47,20 @@ int JitContext::InitializeModule(ObOrcJit &jit) return ret; } -void JitContext::compile() +int JitContext::compile(ObOrcJit &jit) { - if (!Compile && nullptr != TheJIT) { - TheJIT->addModule(std::move(TheModule)); + int ret = OB_SUCCESS; + + if (Compile) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("already compiled", K(ret), K(lbt())); + } else if (OB_FAIL(jit.addModule(std::move(TheModule), std::move(TheContext)))) { + LOG_WARN("failed to add module to jit engine", K(ret)); + } else { Compile = true; } + + return ret; } int JitContext::optimize() diff --git a/src/objit/src/core/jit_context.h b/src/objit/src/core/jit_context.h index ac54c61df..3a8be94b0 100644 --- a/src/objit/src/core/jit_context.h +++ b/src/objit/src/core/jit_context.h @@ -31,26 +31,27 @@ struct JitContext { public: explicit JitContext() - : Compile(false), TheContext(nullptr), TheJIT(nullptr) - { - } + : Compile(false), + TheContext(nullptr), + Builder(nullptr), + TheModule(nullptr), + TheFPM(nullptr) + { } - int InitializeModule(ObOrcJit &jit); - void compile(); + int InitializeModule(const ObDataLayout &DL); + int compile(ObOrcJit &jit); int optimize(); ObLLVMContext& get_context() { return *TheContext; } IRBuilder<>& get_builder() { return *Builder; } Module& get_module() { return *TheModule; } - ObOrcJit* get_jit() { return TheJIT; } public: bool Compile; - ObLLVMContext *TheContext; + std::unique_ptr TheContext; std::unique_ptr> Builder; std::unique_ptr TheModule; - ObOrcJit *TheJIT; std::unique_ptr TheFPM; }; diff --git a/src/objit/src/core/ob_jit_allocator.cpp b/src/objit/src/core/ob_jit_allocator.cpp index 9888f8bef..bf6c89987 100644 --- a/src/objit/src/core/ob_jit_allocator.cpp +++ b/src/objit/src/core/ob_jit_allocator.cpp @@ -82,15 +82,8 @@ public: // set memory protection state. static int protect_mapped_memory(const ObJitMemoryBlock &block, int64_t p_flags); - - static bool aarch64_addr_safe(void *addr, int64_t size); }; -bool ObJitMemory::aarch64_addr_safe(void *addr, int64_t size) -{ - return reinterpret_cast(addr) >> 32 == (reinterpret_cast(addr)+size) >> 32; -} - ObJitMemoryBlock ObJitMemory::allocate_mapped_memory(int64_t num_bytes, int64_t p_flags) { @@ -107,25 +100,11 @@ ObJitMemoryBlock ObJitMemory::allocate_mapped_memory(int64_t num_bytes, do { addr = ::mmap(reinterpret_cast(start), page_size*num_pages, p_flags, mm_flags, fd, 0); - if (MAP_FAILED == addr -#if defined(__aarch64__) - || !aarch64_addr_safe(addr, page_size*num_pages) -#endif - ) { + if (MAP_FAILED == addr) { if (REACH_TIME_INTERVAL(10000000)) { //间隔10s打印日志 LOG_ERROR_RET(common::OB_ALLOCATE_MEMORY_FAILED, "allocate jit memory failed", K(addr), K(num_bytes), K(page_size), K(num_pages)); } ::usleep(100000); //100ms -#if defined(__aarch64__) - if (MAP_FAILED != addr) { - if (0 != ::munmap(addr, page_size*num_pages)) { - LOG_WARN_RET(OB_ERR_SYS, "jit block munmap failed", K(addr), K(page_size*num_pages)); - } - start = reinterpret_cast(addr) + UINT32_MAX - page_size*num_pages; //先向上移4G，再移动此次分配的大小，以保证此次分配的地址高16位一致 - LOG_INFO("aarch64 memory allocated not safe, try again", K(addr), K(start), K(page_size), K(num_pages)); - addr = MAP_FAILED; - } -#endif } else { LOG_DEBUG("allocate mapped memory success!", K(addr), K(start), @@ -370,6 +349,7 @@ void ObJitAllocator::reserve(const JitMemType mem_type, int64_t sz, int64_t alig } } +// Returns true if an error occurred, false otherwise. bool ObJitAllocator::finalize() { int ret = OB_SUCCESS; @@ -384,7 +364,7 @@ bool ObJitAllocator::finalize() LOG_WARN("fail to finalize code memory", K(ret)); } - return OB_SUCC(ret); + return OB_FAIL(ret); } void ObJitAllocator::free() { diff --git a/src/objit/src/core/ob_orc_jit.cpp b/src/objit/src/core/ob_orc_jit.cpp index 5d5eb4c80..78416eb1a 100644 --- a/src/objit/src/core/ob_orc_jit.cpp +++ b/src/objit/src/core/ob_orc_jit.cpp @@ -35,7 +35,6 @@ #include "llvm/IR/DataLayout.h" #include "llvm/IR/Mangler.h" #include "llvm/IR/Verifier.h" -#include "llvm/Support/DynamicLibrary.h" #include "llvm/Support/SourceMgr.h" #include "llvm/Support/raw_ostream.h" #include "llvm/AsmParser/Parser.h" @@ -56,94 +55,139 @@ namespace jit namespace core { -#ifndef ORC2 +DenseMap ObJitGlobalSymbolGenerator::symbol_table; + ObOrcJit::ObOrcJit(common::ObIAllocator &Allocator) : DebugBuf(nullptr), DebugLen(0), JITAllocator(), NotifyLoaded(Allocator, DebugBuf, DebugLen, SoObject), - TheContext(), - ObResolver(createLegacyLookupResolver( - ObES, - [this](StringRef Name) { return findMangledSymbol(std::string(Name)); }, - [](Error Err) { cantFail(std::move(Err), "lookupFlags failed"); })), ObTM(EngineBuilder().selectTarget()), ObDL(ObTM->createDataLayout()), - ObObjectLayer(AcknowledgeORCv1Deprecation, - ObES, - [this](ObVModuleKey) { - return ObObjLayerT::Resources{ - std::make_shared(JITAllocator), ObResolver}; }, - NotifyLoaded), - ObCompileLayer(AcknowledgeORCv1Deprecation, ObObjectLayer, SimpleCompiler(*ObTM)) + ObEngineBuilder(), + ObJitEngine() { } -ObVModuleKey ObOrcJit::addModule(std::unique_ptr M) +int ObOrcJit::init() { - auto Key = ObES.allocateVModule(); - cantFail(ObCompileLayer.addModule(Key, std::move(M))); - ObModuleKeys.push_back(Key); - return Key; + int ret = OB_SUCCESS; + + ObEngineBuilder.setObjectLinkingLayerCreator( + [this](ExecutionSession &ES, const Triple &TT) { + auto ObjLinkingLayer = + std::make_unique( + ES, + [&]() { + return std::make_unique(JITAllocator); + }); + +#ifndef NDEBUG + ObjLinkingLayer->registerJITEventListener( + *JITEventListener::createGDBRegistrationListener()); +#endif // NDEBUG + ObjLinkingLayer->registerJITEventListener(NotifyLoaded); + return ObjLinkingLayer; + }); + + auto tm_builder_wrapper = JITTargetMachineBuilder::detectHost(); + + if (!tm_builder_wrapper) { + Error err = tm_builder_wrapper.takeError(); + std::string msg = toString(std::move(err)); + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get target machine", K(msg.c_str())); + } else { + ObEngineBuilder.setJITTargetMachineBuilder(*tm_builder_wrapper); + } + + return ret; } -ObJITSymbol ObOrcJit::lookup(std::string Name) +int ObOrcJit::addModule(std::unique_ptr M, std::unique_ptr TheContext) { - return findMangledSymbol(mangle(Name)); + int ret = OB_SUCCESS; + + if (OB_FAIL(create_jit_engine())) { + LOG_WARN("failed to create jit engine", K(ret)); + } else if (OB_ISNULL(ObJitEngine)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected NULL jit engine", K(ret), K(lbt())); + } else { + Error err = ObJitEngine->addIRModule(ThreadSafeModule{std::move(M), std::move(TheContext)}); + + if (err) { + std::string msg = toString(std::move(err)); + + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to add module to jit engine", + K(ret), K(msg.c_str())); + } + } + + return ret; } -uint64_t ObOrcJit::get_function_address(const std::string Name) +int ObOrcJit::lookup(const std::string &name, ObJITSymbol &symbol) { - return static_cast(cantFail(lookup(Name).getAddress())); + int ret = OB_SUCCESS; + + if (OB_ISNULL(ObJitEngine)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected NULL jit engine", K(ret), K(lbt())); + } else { + auto value = ObJitEngine->lookup(name); + + if (!value) { + Error err = value.takeError(); + + if (err.isA()) { + ret = OB_ENTRY_NOT_EXIST; + } else { + ret = OB_ERR_UNEXPECTED; + } + + std::string msg = toString(std::move(err)); + LOG_WARN("failed to lookup symbol in jit engine", + K(ret), + "name", name.c_str(), + "msg", msg.c_str()); + } else { + symbol = *value; + } + } + + return ret; } -#else -static ExitOnError ExitOnErr; - -ObOrcJit::ObOrcJit(ObIAllocator &Allocator, JITTargetMachineBuilder JTMB, ObDataLayout ObDL) - : DebugBuf(nullptr), - DebugLen(0), - JITAllocator(), - NotifyLoaded(Allocator, DebugBuf, DebugLen), - ObObjectLayer(ObES, - [this]() { return std::make_unique(JITAllocator); }), - ObCompileLayer(ObES, - ObObjectLayer, - std::make_unique(std::move(JTMB))), - ObDL(std::move(ObDL)), - Mangle(ObES, this->ObDL), - Ctx(std::make_unique()), - MainJD(ObES.createBareJITDylib("
")) +int ObOrcJit::get_function_address(const std::string &name, uint64_t &addr) { - /* - MainJD.define(absoluteSymbols({ - { Mangle("eh_personality"), pointerToJITTargetAddress(&ObPLEH::eh_personality) } - })); - */ - MainJD.addGenerator( - cantFail(DynamicLibrarySearchGenerator::GetForCurrentProcess( - ObDL.getGlobalPrefix()))); + int ret = OB_SUCCESS; + + ObJITSymbol sym = nullptr; + + if (OB_FAIL(lookup(name, sym))) { + LOG_WARN("failed to lookup symbol addr", K(name.c_str())); + } else { + auto value = sym.getAddress(); + + if (!value) { + Error err = value.takeError(); + std::string msg = toString(std::move(err)); + + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get symbol address", + K(ret), + "name", name.c_str(), + "msg", msg.c_str()); + } else { + addr = static_cast(*value); + } + } + + return ret; } -Error ObOrcJit::addModule(std::unique_ptr M) -{ - return ObCompileLayer.add(MainJD, ThreadSafeModule(std::move(M), Ctx)); -} - -Expected ObOrcJit::lookup(StringRef Name) -{ - return ObES.lookup({&MainJD}, Mangle(Name.str())); -} - -uint64_t ObOrcJit::get_function_address(const std::string Name) -{ - std::cerr << "get_function_address : " << Name << std::endl; - auto Sym = ExitOnErr(lookup(Name)); - std::cerr << "get_function_address finish : " << Name << std::endl; - return static_cast(Sym.getAddress()); -} -#endif - -void ObNotifyLoaded::operator()( +void ObNotifyLoaded::notifyObjectLoaded( ObVModuleKey Key, const object::ObjectFile &Obj, const RuntimeDyld::LoadedObjectInfo &Info) @@ -169,13 +213,29 @@ void ObNotifyLoaded::operator()( // } } -void ObOrcJit::add_compiled_object(size_t length, const char *ptr) +int ObOrcJit::add_compiled_object(size_t length, const char *ptr) { - ObVModuleKey Key = ObES.allocateVModule(); + int ret = OB_SUCCESS; - cantFail(ObObjectLayer.addObject( - Key, MemoryBuffer::getMemBuffer(StringRef(ptr, length), "", false))); - ObModuleKeys.push_back(Key); + if (OB_FAIL(create_jit_engine())) { + LOG_WARN("failed to create jit engine", K(ret)); + } else if (OB_ISNULL(ObJitEngine)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected NULL jit engine", K(ret), K(lbt())); + } else { + Error err =ObJitEngine->addObjectFile( + MemoryBuffer::getMemBuffer(StringRef(ptr, length), "", false)); + + if (err) { + std::string msg = toString(std::move(err)); + + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to add compile result to jit engine", + K(ret), K(msg.c_str()), K(length), K(ptr)); + } + } + + return ret; } int ObOrcJit::set_optimize_level(ObPLOptLevel level) @@ -187,14 +247,53 @@ int ObOrcJit::set_optimize_level(ObPLOptLevel level) LOG_WARN("unexpected PLSQL_OPTIMIZE_LEVEL", K(ret), K(level), K(lbt())); } - if (OB_SUCC(ret) && OB_ISNULL(ObTM)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected NULL TM", K(ret), K(ObTM.get()), K(lbt())); + if (OB_SUCC(ret) && level == ObPLOptLevel::O0) { + auto &tm_builder = ObEngineBuilder.getJITTargetMachineBuilder(); + if (!tm_builder.hasValue()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected NULL JITTargetMachineBuilder", K(ret), K(lbt())); + } else { + auto &builder = *tm_builder; + builder.setCodeGenOptLevel(CodeGenOpt::Level::None); + builder.getOptions().EnableFastISel = true; + } } - if (OB_SUCC(ret) && level == ObPLOptLevel::O0) { - ObTM->setOptLevel(CodeGenOpt::Level::None); - ObTM->setFastISel(true); + return ret; +} + +int ObOrcJit::create_jit_engine() +{ + int ret = OB_SUCCESS; + + if (OB_NOT_NULL(ObJitEngine)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected NOT NULL jit engine", K(ret), K(lbt())); + } else { + std::unique_ptr symbol_generator = nullptr; + + auto engine_wrapper = ObEngineBuilder.create(); + + if (!engine_wrapper) { + Error err = engine_wrapper.takeError(); + std::string msg = toString(std::move(err)); + + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to build LLVM JIT engine", K(msg.c_str())); + } else { + ObJitEngine = std::move(*engine_wrapper); + } + + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_ISNULL(ObJitEngine)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected NULL jit engine", K(ret)); + } else if (OB_FAIL(ob_jit_make_unique(symbol_generator))) { + LOG_WARN("failed to make ObJitGlobalSymbolGenerator unique_ptr", K(ret)); + } else { + ObJitEngine->getMainJITDylib().addGenerator(std::move(symbol_generator)); + } } return ret; diff --git a/src/objit/src/core/ob_orc_jit.h b/src/objit/src/core/ob_orc_jit.h index 836121958..d5a5f50f9 100644 --- a/src/objit/src/core/ob_orc_jit.h +++ b/src/objit/src/core/ob_orc_jit.h @@ -20,11 +20,11 @@ #include "llvm/ExecutionEngine/ObjectCache.h" #include "llvm/IR/DataLayout.h" #include "llvm/DebugInfo/DWARF/DWARFContext.h" +#include "llvm/ExecutionEngine/Orc/LLJIT.h" #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/ExecutionEngine/JITEventListener.h" #include "llvm/IR/Mangler.h" -#include #include #include "core/ob_jit_memory_manager.h" @@ -40,6 +40,33 @@ enum class ObPLOptLevel : int O2 = 2, O3 = 3 }; +template +static inline int ob_jit_make_unique(std::unique_ptr &ptr, Args&&... args) { + int ret = OB_SUCCESS; + + std::unique_ptr result = nullptr; + + try { + result = std::make_unique(std::forward(args)...); + } catch (const std::bad_alloc &e) { + ret = OB_ALLOCATE_MEMORY_FAILED; + SERVER_LOG(WARN, "failed to allocate memory", K(ret), K(e.what())); + } catch (...) { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "unexpected exception in std::make_unique", K(ret), K(lbt())); + } + + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_ISNULL(result)) { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "unexpected NULL ptr of std::make_unque", K(ret), K(lbt())); + } else { + ptr = std::move(result); + } + + return ret; +} namespace core { using namespace llvm; @@ -51,8 +78,10 @@ typedef ::llvm::TargetMachine ObTargetMachine; typedef ::llvm::DataLayout ObDataLayout; typedef ::llvm::orc::VModuleKey ObVModuleKey; typedef ::llvm::JITSymbol ObJITSymbol; +typedef ::llvm::orc::JITDylib::DefinitionGenerator ObJitDefinitionGenerator; +typedef ::llvm::JITEventListener ObJitEventListener; -class ObNotifyLoaded +class ObNotifyLoaded: public ObJitEventListener { public: explicit ObNotifyLoaded( @@ -60,9 +89,9 @@ public: : Allocator(Allocator), DebugBuf(DebugBuf), DebugLen(DebugLen), SoObject(SoObject) {} virtual ~ObNotifyLoaded() {} - void operator()(ObVModuleKey Key, + void notifyObjectLoaded(ObVModuleKey Key, const object::ObjectFile &Obj, - const RuntimeDyld::LoadedObjectInfo &Info); + const RuntimeDyld::LoadedObjectInfo &Info) override; private: common::ObIAllocator &Allocator; char* &DebugBuf; @@ -70,128 +99,95 @@ private: ObString &SoObject; }; +class ObJitGlobalSymbolGenerator: public ObJitDefinitionGenerator { +public: + Error tryToGenerate(orc::LookupKind K, + orc::JITDylib &JD, + orc::JITDylibLookupFlags JDLookupFlags, + const orc::SymbolLookupSet &LookupSet) override + { + for (const auto &sym : LookupSet) { + auto res = symbol_table.find(*sym.first); + + if (res != symbol_table.end()) { + Error err = JD.define(orc::absoluteSymbols( + {{sym.first, JITEvaluatedSymbol(res->second, {})}})); + + if (err) { + StringRef name = *sym.first; + std::string msg = toString(std::move(err)); + + SERVER_LOG_RET(WARN, OB_ERR_UNEXPECTED, + "failed to define SPI interface symbol", + "name", ObString(name.size(), name.data()), + "msg", msg.c_str(), + K(lbt())); + + return err; + } + } + } + + return Error::success(); + } + + static void add_symbol(StringRef name, void *addr) { + symbol_table[name] = pointerToJITTargetAddress(addr); + } + +private: + static DenseMap symbol_table; +}; -#ifndef ORC2 class ObOrcJit { public: - using ObObjLayerT = llvm::orc::LegacyRTDyldObjectLinkingLayer; - using ObCompileLayerT = llvm::orc::LegacyIRCompileLayer; + using ObLLJITBuilder = llvm::orc::LLJITBuilder; + using ObJitEngineT = llvm::orc::LLJIT; explicit ObOrcJit(common::ObIAllocator &Allocator); virtual ~ObOrcJit() {}; - ObVModuleKey addModule(std::unique_ptr M); - ObJITSymbol lookup(const std::string Name); - uint64_t get_function_address(const std::string Name); + int addModule(std::unique_ptr M, std::unique_ptr TheContext); + int get_function_address(const std::string &name, uint64_t &addr); - ObLLVMContext &getContext() { return TheContext; } const ObDataLayout &getDataLayout() const { return ObDL; } char* get_debug_info_data() { return DebugBuf; } int64_t get_debug_info_size() { return DebugLen; } - void add_compiled_object(size_t length, const char *ptr); + int add_compiled_object(size_t length, const char *ptr); const ObString& get_compiled_object() const { return SoObject; } int set_optimize_level(ObPLOptLevel level); -private: - std::string mangle(const std::string &Name) - { - std::string MangledName; { - raw_string_ostream MangledNameStream(MangledName); - Mangler::getNameWithPrefix(MangledNameStream, Name, ObDL); - } - return MangledName; - } + int init(); - ObJITSymbol findMangledSymbol(const std::string &Name) - { - const bool ExportedSymbolsOnly = true; - for (auto H : make_range(ObModuleKeys.rbegin(), ObModuleKeys.rend())) { - if (auto Sym = ObCompileLayer.findSymbolIn(H, Name, ExportedSymbolsOnly)) { - return Sym; - } - } - if (auto SymAddr = RTDyldMemoryManager::getSymbolAddressInProcess(Name)) { - return ObJITSymbol(SymAddr, JITSymbolFlags::Exported); - } - return nullptr; - } + const ObDataLayout &get_DL() const { return ObDL; } private: + int lookup(const std::string &name, ObJITSymbol &symbol); + + int create_jit_engine(); + + static ObJitGlobalSymbolGenerator symbol_generator; + char *DebugBuf; int64_t DebugLen; ObJitAllocator JITAllocator; ObNotifyLoaded NotifyLoaded; - ObLLVMContext TheContext; - ObExecutionSession ObES; - std::shared_ptr ObResolver; std::unique_ptr ObTM; const ObDataLayout ObDL; - ObObjLayerT ObObjectLayer; - ObCompileLayerT ObCompileLayer; - std::vector ObModuleKeys; ObString SoObject; + + ObLLJITBuilder ObEngineBuilder; + std::unique_ptr ObJitEngine; }; -#else -class ObOrcJit -{ -public: - explicit ObOrcJit( - common::ObIAllocator &Allocator, llvm::orc::JITTargetMachineBuilder JTMB, ObDataLayout ObDL); - virtual ~ObOrcJit() {}; - - Error addModule(std::unique_ptr M); - Expected lookup(StringRef Name); - uint64_t get_function_address(const std::string Name); - - static ObOrcJit* create(ObIAllocator &allocator) - { - auto JTMB = llvm::orc::JITTargetMachineBuilder::detectHost(); - - if (!JTMB) - return nullptr; - - auto ObDL = JTMB->getDefaultDataLayoutForTarget(); - if (!ObDL) - return nullptr; - - return OB_NEWx(ObOrcJit, (&allocator), allocator, std::move(*JTMB), std::move(*ObDL)); - } - - ObLLVMContext &getContext() { return *Ctx.getContext(); } - - const ObDataLayout &getDataLayout() const { return ObDL; } - - char* get_debug_info_data() { return DebugBuf; } - int64_t get_debug_info_size() { return DebugLen; } - -private: - char *DebugBuf; - int64_t DebugLen; - - ObJitAllocator JITAllocator; - ObNotifyLoaded NotifyLoaded; - - llvm::orc::ObExecutionSession ObES; - llvm::orc::RTDyldObjectLinkingLayer ObObjectLayer; - llvm::orc::IRCompileLayer ObCompileLayer; - - llvm::ObDataLayout ObDL; - llvm::orc::MangleAndInterner Mangle; - llvm::orc::ThreadSafeContext Ctx; - - llvm::orc::JITDylib &MainJD; -}; -#endif - } // core } // jit } // oceanbase diff --git a/src/objit/src/ob_llvm_helper.cpp b/src/objit/src/ob_llvm_helper.cpp index 739eeb99c..8b9088e27 100644 --- a/src/objit/src/ob_llvm_helper.cpp +++ b/src/objit/src/ob_llvm_helper.cpp @@ -20,7 +20,6 @@ #include "llvm/IR/GlobalVariable.h" #include "llvm/ExecutionEngine/ExecutionEngine.h" #include "llvm/Support/TargetSelect.h" -#include "llvm/Support/DynamicLibrary.h" #include "share/ob_define.h" #include "objit/ob_llvm_helper.h" @@ -514,18 +513,24 @@ int ObLLVMHelper::init() } else if (nullptr == (jc_ = OB_NEWx(core::JitContext, (&allocator_)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to alloc memory for jit context", K(ret)); -#ifndef ORC2 } else if (nullptr == (jit_ = OB_NEWx(core::ObOrcJit, (&allocator_), allocator_))) { -#else - } else if (nullptr == (jit_ = core::ObOrcJit::create(allocator_))) { -#endif jc_->~JitContext(); allocator_.free(jc_); jc_ = nullptr; ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to alloc memory for jit", K(ret)); - } else if (OB_FAIL(jc_->InitializeModule(*jit_))) { + } else if (OB_FAIL(jit_->init())) { + jit_->~ObOrcJit(); + allocator_.free(jit_); + jit_ = nullptr; + + jc_->~JitContext(); + allocator_.free(jc_); + jc_ = nullptr; + + LOG_WARN("failed to init jit engine", K(ret)); + } else if (OB_FAIL(jc_->InitializeModule(jit_->get_DL()))) { jit_->~ObOrcJit(); allocator_.free(jit_); jit_ = nullptr; @@ -560,9 +565,9 @@ int ObLLVMHelper::initialize() llvm::InitializeNativeTargetAsmPrinter(); llvm::InitializeNativeTargetAsmParser(); - llvm::sys::DynamicLibrary::LoadLibraryPermanently(nullptr); - -#if !defined(__aarch64__) +/*Do not juse use !defined(__aarch64__) here*/ +//#if !defined(__aarch64__) +#if defined(__x86_64__) // initialize LLVM X86 unfold table llvm::lookupUnfoldTable(0); #endif @@ -590,6 +595,7 @@ int ObLLVMHelper::init_llvm() { ObLLVMFunctionType ft; ObLLVMBasicBlock block; ObLLVMValue magic; + uint64_t addr; OZ (helper.get_llvm_type(ObIntType, int64_type)); OZ (arg_types.push_back(int64_type)); @@ -601,7 +607,7 @@ int ObLLVMHelper::init_llvm() { OZ (helper.create_ret(magic)); OZ (helper.compile_module(jit::ObPLOptLevel::O2)); - OX (helper.get_function_address(init_func_name)); + OZ (helper.get_function_address(init_func_name, addr)); return ret; } @@ -620,7 +626,7 @@ int ObLLVMHelper::compile_module(jit::ObPLOptLevel optimization) dump_module(); } OB_LLVM_MALLOC_GUARD(GET_PL_MOD_STRING(pl::OB_PL_JIT)); - jc_->compile(); + jc_->compile(*jit_); } return ret; @@ -670,15 +676,23 @@ int ObLLVMHelper::verify_module() return ret; } -uint64_t ObLLVMHelper::get_function_address(const ObString &name) +int ObLLVMHelper::get_function_address(const ObString &name, uint64_t &addr) { - OB_LLVM_MALLOC_GUARD(GET_PL_MOD_STRING(pl::OB_PL_JIT)); - return jc_->TheJIT->get_function_address(std::string(name.ptr(), name.length())); + int ret = OB_SUCCESS; + + CK (OB_NOT_NULL(jit_)); + + if (OB_SUCC(ret)) { + OB_LLVM_MALLOC_GUARD(GET_PL_MOD_STRING(pl::OB_PL_JIT)); + OZ (jit_->get_function_address(std::string(name.ptr(), name.length()), addr)); + } + + return ret; } void ObLLVMHelper::add_symbol(const ObString &name, void *value) { - llvm::sys::DynamicLibrary::AddSymbol(make_string_ref(name), value); + core::ObJitGlobalSymbolGenerator::add_symbol(make_string_ref(name), value); } ObDIRawData ObLLVMHelper::get_debug_info() const @@ -2248,7 +2262,7 @@ int ObLLVMHelper::add_compiled_object(size_t length, const char *ptr) CK (OB_NOT_NULL(jit_)); CK (OB_NOT_NULL(ptr)); CK (OB_LIKELY(length > 0)); - OX (jit_->add_compiled_object(length, ptr)); + OZ (jit_->add_compiled_object(length, ptr)); return ret; } diff --git a/src/observer/mysql/ob_query_driver.cpp b/src/observer/mysql/ob_query_driver.cpp index 6b02b2174..e6458867d 100644 --- a/src/observer/mysql/ob_query_driver.cpp +++ b/src/observer/mysql/ob_query_driver.cpp @@ -177,21 +177,26 @@ int ObQueryDriver::response_query_result(ObResultSet &result, can_retry = true; bool is_first_row = true; const ObNewRow *result_row = NULL; - bool has_top_limit = result.get_has_top_limit(); bool is_cac_found_rows = result.is_calc_found_rows(); - int64_t limit_count = OB_INVALID_COUNT == fetch_limit ? INT64_MAX : fetch_limit; int64_t row_num = 0; ObSqlCtx *sql_ctx = result.get_exec_context().get_sql_ctx(); - if (!has_top_limit && OB_INVALID_COUNT == fetch_limit) { - limit_count = INT64_MAX; - if (!lib::is_oracle_mode()) { - if (OB_FAIL(session_.get_sql_select_limit(limit_count))) { - LOG_WARN("fail tp get sql select limit", K(ret)); - } - } - } bool is_packed = result.get_physical_plan() ? result.get_physical_plan()->is_packed() : false; MYSQL_PROTOCOL_TYPE protocol_type = is_ps_protocol ? MYSQL_PROTOCOL_TYPE::BINARY : MYSQL_PROTOCOL_TYPE::TEXT; + + int64_t limit_count = INT64_MAX; + if (OB_FAIL(ret)) { + } else if (OB_INVALID_COUNT != fetch_limit) { + limit_count = fetch_limit; + } else if (lib::is_mysql_mode()) { + if (!result.get_has_top_limit() && OB_FAIL(session_.get_sql_select_limit(limit_count))) { + LOG_WARN("failed to get sytem variable sql_select_limit", K(ret)); + } + } else { // lib::is_oracle_mode() + if (OB_FAIL(session_.get_oracle_sql_select_limit(limit_count))) { + LOG_WARN("failed to get sytem variable _oracle_sql_select_limit", K(ret)); + } + } + const common::ColumnsFieldIArray *fields = NULL; if (OB_SUCC(ret)) { fields = result.get_field_columns(); diff --git a/src/observer/mysql/ob_query_retry_ctrl.cpp b/src/observer/mysql/ob_query_retry_ctrl.cpp index 3536765b1..adb30d783 100644 --- a/src/observer/mysql/ob_query_retry_ctrl.cpp +++ b/src/observer/mysql/ob_query_retry_ctrl.cpp @@ -483,11 +483,24 @@ public: { // sql which in pl will local retry first. see ObInnerSQLConnection::process_retry. // sql which not in pl use the same strategy to avoid never getting the lock. - if (v.force_local_retry_ || (v.local_retry_times_ <= 1 && !v.result_.is_pl_stmt(v.result_.get_stmt_type()))) { - v.retry_type_ = RETRY_TYPE_LOCAL; + if (v.is_from_pl_) { + if (v.local_retry_times_ <= 1 || + !v.session_.get_pl_can_retry() || + ObSQLUtils::is_in_autonomous_block(v.session_.get_cur_exec_ctx())) { + v.no_more_test_ = true; + v.retry_type_ = RETRY_TYPE_LOCAL; + } else { + v.no_more_test_ = true; + v.retry_type_ = RETRY_TYPE_NONE; + v.client_ret_ = v.err_; + } } else { - const ObMultiStmtItem &multi_stmr_item = v.ctx_.multi_stmt_item_; - try_packet_retry(v); + if (v.force_local_retry_ || (v.local_retry_times_ <= 1 && !v.result_.is_pl_stmt(v.result_.get_stmt_type()))) { + v.retry_type_ = RETRY_TYPE_LOCAL; + } else { + const ObMultiStmtItem &multi_stmr_item = v.ctx_.multi_stmt_item_; + try_packet_retry(v); + } } } }; diff --git a/src/observer/mysql/obmp_stmt_execute.cpp b/src/observer/mysql/obmp_stmt_execute.cpp index 4c611f71c..82951dd7c 100644 --- a/src/observer/mysql/obmp_stmt_execute.cpp +++ b/src/observer/mysql/obmp_stmt_execute.cpp @@ -1177,7 +1177,12 @@ int ObMPStmtExecute::execute_response(ObSQLSessionInfo &session, if (OB_SUCC(ret)) { ObPLExecCtx pl_ctx(cursor->get_allocator(), &result.get_exec_context(), NULL/*params*/, NULL/*result*/, &ret, NULL/*func*/, true); - if (OB_FAIL(ObSPIService::dbms_dynamic_open(&pl_ctx, *cursor))) { + int64_t orc_max_ret_rows = INT64_MAX; + if (lib::is_oracle_mode() + && OB_FAIL(session.get_oracle_sql_select_limit(orc_max_ret_rows))) { + LOG_WARN("failed to get sytem variable _oracle_sql_select_limit", K(ret)); + } else if (OB_FAIL(ObSPIService::dbms_dynamic_open( + &pl_ctx, *cursor, false, orc_max_ret_rows))) { LOG_WARN("open cursor fail. ", K(ret), K(stmt_id_)); if (!THIS_WORKER.need_retry()) { int cli_ret = OB_SUCCESS; diff --git a/src/observer/mysql/obmp_stmt_prexecute.cpp b/src/observer/mysql/obmp_stmt_prexecute.cpp index ec9acdc3e..7f862fff6 100644 --- a/src/observer/mysql/obmp_stmt_prexecute.cpp +++ b/src/observer/mysql/obmp_stmt_prexecute.cpp @@ -478,12 +478,16 @@ int ObMPStmtPrexecute::execute_response(ObSQLSessionInfo &session, ObPLExecCtx pl_ctx(cursor->get_allocator(), &result.get_exec_context(), NULL/*params*/, NULL/*result*/, &ret, NULL/*func*/, true); get_ctx().cur_sql_ = sql_; - if ( + int64_t orc_max_ret_rows = INT64_MAX; + if (lib::is_oracle_mode() + && OB_FAIL(session.get_oracle_sql_select_limit(orc_max_ret_rows))) { + LOG_WARN("failed to get sytem variable _oracle_sql_select_limit", K(ret)); + } else if ( #ifdef ERRSIM OB_FAIL(common::EventTable::COM_STMT_PREXECUTE_PS_CURSOR_OPEN_ERROR) || #endif - OB_FAIL(ObSPIService::dbms_dynamic_open(&pl_ctx, *cursor)) - ) { + OB_FAIL(ObSPIService::dbms_dynamic_open( + &pl_ctx, *cursor, false, orc_max_ret_rows))) { LOG_WARN("cursor open faild.", K(cursor->get_id())); // select do not support arraybinding if (!THIS_WORKER.need_retry()) { diff --git a/src/observer/ob_inner_sql_connection.cpp b/src/observer/ob_inner_sql_connection.cpp index 939f83c24..35e48fb2d 100644 --- a/src/observer/ob_inner_sql_connection.cpp +++ b/src/observer/ob_inner_sql_connection.cpp @@ -326,6 +326,11 @@ void ObInnerSQLConnection::set_is_load_data_exec(bool v) get_session().set_load_data_exec_session(v); } +void ObInnerSQLConnection::set_ob_enable_pl_cache(bool v) +{ + get_session().set_local_ob_enable_pl_cache(v); +} + int ObInnerSQLConnection::init_session_info( sql::ObSQLSessionInfo *session, const bool is_extern_session, diff --git a/src/observer/ob_inner_sql_connection.h b/src/observer/ob_inner_sql_connection.h index aad53a145..7fcf81f2f 100644 --- a/src/observer/ob_inner_sql_connection.h +++ b/src/observer/ob_inner_sql_connection.h @@ -189,6 +189,7 @@ public: virtual void set_is_load_data_exec(bool v); virtual void set_force_remote_exec(bool v) { force_remote_execute_ = v; } virtual void set_use_external_session(bool v) { use_external_session_ = v; } + virtual void set_ob_enable_pl_cache(bool v) override; bool is_nested_conn(); virtual void set_user_timeout(int64_t timeout) { user_timeout_ = timeout; } virtual int64_t get_user_timeout() const { return user_timeout_; } diff --git a/src/pl/CMakeLists.txt b/src/pl/CMakeLists.txt index 2d3d82441..88701e981 100644 --- a/src/pl/CMakeLists.txt +++ b/src/pl/CMakeLists.txt @@ -41,10 +41,40 @@ ob_set_subtarget(ob_pl pl_cache pl_cache/ob_pl_cache.cpp pl_cache/ob_pl_cache_mgr.cpp pl_cache/ob_pl_cache_object.cpp + diagnosis/ob_pl_sql_audit_guard.cpp ) +set(SYS_PACK_SQL_SOURCE_DIR ${PROJECT_SOURCE_DIR}/src/share/inner_table/sys_package) +set(gen_sys_pack_file ${SYS_PACK_SQL_SOURCE_DIR}/system_package.cpp) +message(STATUS "generating ${gen_sys_pack_file} ...") +file (GLOB embed_sys_pack_files "${SYS_PACK_SQL_SOURCE_DIR}/*.sql") +list (LENGTH embed_sys_pack_files embed_sys_pack_cnt) +file (WRITE ${gen_sys_pack_file} + "// This file is generated by CMake, do not edit it!\n\n" + "#include \n" + "#include \n\n" + "namespace oceanbase {\n" + "namespace pl {\n\n" + "extern const int64_t syspack_source_count = ${embed_sys_pack_cnt};\n" + "extern const std::pair syspack_source_contents[] = {\n") +foreach(embed_file ${embed_sys_pack_files}) + get_filename_component(file_name ${embed_file} NAME) + file(SIZE ${embed_file} embed_file_size) + file(READ ${embed_file} filedata) + file(APPEND ${gen_sys_pack_file} + "{\n" + "\"${file_name}\", // size = ${embed_file_size} bytes\n" + "R\"sys_pack_del\(${filedata}\)sys_pack_del\"\n" + "},\n") +endforeach() +file (APPEND ${gen_sys_pack_file} + "};\n\n" + "} // namespace pl\n" + "} // namespace oceanbase\n") +message(STATUS "generate ${gen_sys_pack_file} done") + ob_set_subtarget(ob_pl sys_package - sys_package/ob_dbms_external_table.cpp + ${gen_sys_pack_file} sys_package/ob_dbms_stats.cpp sys_package/ob_dbms_scheduler_mysql.cpp sys_package/ob_dbms_application.cpp @@ -59,6 +89,7 @@ ob_set_subtarget(ob_pl sys_package sys_package/ob_pl_dbms_resource_manager.cpp sys_package/ob_pl_dbms_trusted_certificate_manager.cpp sys_package/ob_dbms_limit_calculator_mysql.cpp + sys_package/ob_dbms_external_table.cpp ) ob_set_subtarget(ob_pl dblink diff --git a/src/pl/diagnosis/ob_pl_sql_audit_guard.cpp b/src/pl/diagnosis/ob_pl_sql_audit_guard.cpp new file mode 100644 index 000000000..0aa8eab5a --- /dev/null +++ b/src/pl/diagnosis/ob_pl_sql_audit_guard.cpp @@ -0,0 +1,119 @@ + +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX PL + +#include "ob_pl_sql_audit_guard.h" +#include "sql/ob_spi.h" + +namespace oceanbase +{ +namespace pl +{ + +ObPLSqlAuditGuard::ObPLSqlAuditGuard( + sql::ObExecContext &exec_ctx, + ObSQLSessionInfo &session_info, + ObSPIResultSet &spi_result, + ObPLSqlAuditRecord &record, + int &ret, + ObString ps_sql, + observer::ObQueryRetryCtrl &retry_ctrl, + ObPLSPITraceIdGuard &traceid_guard, + stmt::StmtType stmt_type) + : exec_ctx_(exec_ctx), + session_info_(session_info), + spi_result_(spi_result), + record_(record), + ret_(ret), + ps_sql_(ps_sql), + retry_ctrl_(retry_ctrl), + traceid_guard_(traceid_guard), + stmt_type_(stmt_type) +{ + enable_perf_event_ = lib::is_diagnose_info_enabled(); + enable_sql_audit_ = GCONF.enable_sql_audit && session_info_.get_local_ob_enable_sql_audit(); + // enable_sql_stat_ = session_info.is_sqlstat_enabled(); + max_wait_guard_ = new (memory1) ObMaxWaitGuard(enable_perf_event_ ? &max_wait_desc_ : NULL); + total_wait_guard_ = new (memory2) ObTotalWaitGuard(enable_perf_event_ ? &total_wait_desc_ : NULL); + if (enable_perf_event_) { + record_.exec_record_.record_start(); + } + // if (enable_sql_stat_ && OB_NOT_NULL(exec_ctx_.get_sql_ctx())) { + // record_.sqlstat_record_.record_sqlstat_start_value(); + // record_.sqlstat_record_.set_is_in_retry(session_info_.get_is_in_retry()); + // session_info_.sql_sess_record_sql_stat_start_value(record_.sqlstat_record_); + // } + // 监控项统计开始 + record_.time_record_.set_send_timestamp(ObTimeUtility::current_time()); +} + +ObPLSqlAuditGuard::~ObPLSqlAuditGuard() +{ + int &ret = ret_; + record_.time_record_.set_exec_end_timestamp(ObTimeUtility::current_time()); + if (enable_perf_event_) { + record_.exec_record_.record_end(); + } + // if (enable_sql_stat_ && OB_NOT_NULL(exec_ctx_.get_sql_ctx()) && OB_NOT_NULL(spi_result_.get_result_set())) { + // record_.sqlstat_record_.record_sqlstat_end_value(); + // record_.sqlstat_record_.move_to_sqlstat_cache( + // session_info_, exec_ctx_.get_sql_ctx()->cur_sql_, spi_result_.get_result_set()->get_physical_plan()); + // } + max_wait_guard_->~ObMaxWaitGuard(); + total_wait_guard_->~ObTotalWaitGuard(); + + LOG_TRACE("Start PL/Sql Audit Record"/*, KPC(this)*/ ); + + if (OB_NOT_NULL(spi_result_.get_result_set())) { + if (spi_result_.get_result_set()->is_inited()) { + if (ObStmt::is_execute_stmt(stmt_type_)) { + ps_sql_ = session_info_.get_current_query_string(); + } + int64_t try_cnt = session_info_.get_raw_audit_record().try_cnt_; + ObExecRecord record_bak = session_info_.get_raw_audit_record().exec_record_; + session_info_.get_raw_audit_record().try_cnt_ = retry_ctrl_.get_retry_times(); + session_info_.get_raw_audit_record().pl_trace_id_.set(traceid_guard_.origin_trace_id_); + observer::ObInnerSQLConnection::process_record(*(spi_result_.get_result_set()), + spi_result_.get_sql_ctx(), + session_info_, + record_.time_record_, + ret_, + session_info_.get_current_execution_id(), + OB_INVALID_ID, //FIXME@hr351303 + max_wait_desc_, + total_wait_desc_, + record_.exec_record_, + record_.exec_timestamp_, + true, + ps_sql_, + true, + spi_result_.get_exec_params_str_ptr()); + session_info_.get_raw_audit_record().exec_record_ = record_bak; + session_info_.get_raw_audit_record().try_cnt_ = try_cnt; + session_info_.get_raw_audit_record().pl_trace_id_.reset(); + } else { + LOG_DEBUG("result set is not inited, do not process record", K(ret_), K(ps_sql_)); + } + } else { + if (OB_SUCCESS == ret_) { + ret_ = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error, result_set is null", K(ret_), K(ps_sql_)); + } else { + LOG_WARN("result_set is null", K(ret_), K(ps_sql_)); + } + } +} + +} // namespace pl +} // namespace oceanbase diff --git a/src/pl/diagnosis/ob_pl_sql_audit_guard.h b/src/pl/diagnosis/ob_pl_sql_audit_guard.h new file mode 100644 index 000000000..6edfb7db0 --- /dev/null +++ b/src/pl/diagnosis/ob_pl_sql_audit_guard.h @@ -0,0 +1,132 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_SRC_PL_SQL_AUDIT_GUARD_OB_PL_H_ + +#define OCEANBASE_SRC_PL_SQL_AUDIT_GUARD_OB_PL_H_ + +#include "sql/monitor/ob_exec_stat.h" +#include "observer/ob_inner_sql_connection.h" +#include "sql/resolver/ob_stmt_type.h" + +namespace oceanbase +{ +namespace observer +{ +class ObQueryRetryCtrl; +} +namespace sql +{ +struct ObPLSPITraceIdGuard; +} +namespace pl +{ + +class ObPLTimeRecord : public observer::ObITimeRecord +{ +public: + ObPLTimeRecord() + : send_timestamp_(0), + receive_timestamp_(0), + enqueue_timestamp_(0), + run_timestamp_(0), + process_timestamp_(0), + single_process_timestamp_(0), + exec_start_timestamp_(0), + exec_end_timestamp_(0) {} + virtual ~ObPLTimeRecord() {} + + void set_send_timestamp(int64_t send_timestamp) { send_timestamp_ = send_timestamp; } + void set_receive_timestamp(int64_t receive_timestamp) { receive_timestamp_ = receive_timestamp; } + void set_enqueue_timestamp(int64_t enqueue_timestamp) { enqueue_timestamp_ = enqueue_timestamp; } + void set_run_timestamp(int64_t run_timestamp) { run_timestamp_ = run_timestamp; } + void set_process_timestamp(int64_t process_timestamp) { process_timestamp_ = process_timestamp; } + void set_single_process_timestamp(int64_t single_process_timestamp) { single_process_timestamp_ = single_process_timestamp; } + void set_exec_start_timestamp(int64_t exec_start_timestamp) { exec_start_timestamp_ = exec_start_timestamp; } + void set_exec_end_timestamp(int64_t exec_end_timestamp) { exec_end_timestamp_ = exec_end_timestamp; } + + int64_t get_send_timestamp() const { return send_timestamp_; } + int64_t get_receive_timestamp() const { return get_send_timestamp(); } + int64_t get_enqueue_timestamp() const { return get_send_timestamp(); } + int64_t get_run_timestamp() const { return get_send_timestamp(); } + int64_t get_process_timestamp() const { return get_send_timestamp(); } + int64_t get_single_process_timestamp() const { return get_send_timestamp(); } + int64_t get_exec_start_timestamp() const { return get_send_timestamp(); } + int64_t get_exec_end_timestamp() const { return exec_end_timestamp_; } + +public: + int64_t send_timestamp_; + int64_t receive_timestamp_; + int64_t enqueue_timestamp_; + int64_t run_timestamp_; + int64_t process_timestamp_; + int64_t single_process_timestamp_; + int64_t exec_start_timestamp_; + int64_t exec_end_timestamp_; + +}; + +class ObPLSqlAuditRecord +{ +public: + ObPLSqlAuditRecord(sql::ExecType exec_type_) { + exec_timestamp_.exec_type_ = exec_type_; + } +public: + sql::ObExecRecord exec_record_; + // ObExecutingSqlStatRecord sqlstat_record_; + sql::ObExecTimestamp exec_timestamp_; + ObPLTimeRecord time_record_; +}; + +class ObPLSqlAuditGuard +{ +public: + ObPLSqlAuditGuard(sql::ObExecContext &exec_ctx, + sql::ObSQLSessionInfo &session_info, + sql::ObSPIResultSet &spi_result, + ObPLSqlAuditRecord &record, + int &ret, + ObString ps_sql, + observer::ObQueryRetryCtrl &retry_ctrl, + sql::ObPLSPITraceIdGuard &traceid_guard, + sql::stmt::StmtType stmt_type); + + ~ObPLSqlAuditGuard(); + +private: + bool enable_perf_event_; + bool enable_sql_audit_; + bool enable_sql_stat_; + sql::ObExecContext &exec_ctx_; + sql::ObSQLSessionInfo &session_info_; + + ObWaitEventDesc max_wait_desc_; + ObWaitEventStat total_wait_desc_; + ObMaxWaitGuard *max_wait_guard_; + ObTotalWaitGuard *total_wait_guard_; + char memory1[sizeof(ObMaxWaitGuard)]; + char memory2[sizeof(ObTotalWaitGuard)]; + sql::ObSPIResultSet &spi_result_; + ObPLSqlAuditRecord &record_; + + int &ret_; + ObString ps_sql_; + observer::ObQueryRetryCtrl &retry_ctrl_; + sql::ObPLSPITraceIdGuard &traceid_guard_; + sql::stmt::StmtType stmt_type_; +}; + +} +} + +#endif /*OCEANBASE_SRC_PL_SQL_AUDIT_GUARD_OB_PL_H_*/ \ No newline at end of file diff --git a/src/pl/ob_pl.cpp b/src/pl/ob_pl.cpp index e1bf0d0c4..4663e6378 100644 --- a/src/pl/ob_pl.cpp +++ b/src/pl/ob_pl.cpp @@ -4762,8 +4762,13 @@ int ObPL::check_session_alive(const ObBasicSessionInfo &session) { int ObPLFunction::gen_action_from_precompiled(const ObString &name, size_t length, const char *ptr) { int ret = OB_SUCCESS; + + uint64_t addr = 0; + OZ (helper_.add_compiled_object(length, ptr)); - OX (set_action(helper_.get_function_address(name))); + OZ (helper_.get_function_address(name, addr)); + OX (set_action(addr)); + return ret; } diff --git a/src/pl/ob_pl_code_generator.cpp b/src/pl/ob_pl_code_generator.cpp index cb654e5b0..a9866f44e 100644 --- a/src/pl/ob_pl_code_generator.cpp +++ b/src/pl/ob_pl_code_generator.cpp @@ -8094,15 +8094,29 @@ int ObPLCodeGenerator::final_expression(ObPLCompileUnit &pl_func) int ret = OB_SUCCESS; for (int64_t i = 0; OB_SUCC(ret) && i < ast_.get_obj_access_exprs().count(); ++i) { ObRawExpr *expr = ast_.get_obj_access_expr(i); + ObObjAccessRawExpr *obj_access_expr = nullptr; + uint64_t addr = 0; + if (OB_ISNULL(expr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("obj_access_expr is null"); } else if (!expr->is_obj_access_expr()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("not a obj access", K(*expr), K(ret)); + } else if (OB_ISNULL(obj_access_expr = static_cast(expr))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected NULL obj_access_expr", K(ret), K(*expr)); + } else if (OB_FAIL(helper_.get_function_address(obj_access_expr->get_func_name(), addr))) { + if (OB_ENTRY_NOT_EXIST == ret) { + LOG_INFO("failed to find obj_access_expr symbol in JIT engine, will ignore this error", + K(ret), K(obj_access_expr->get_func_name())); + ret = OB_SUCCESS; + obj_access_expr->set_get_attr_func_addr(0); + } else { + LOG_WARN("failed to compile obj_access_expr", K(ret), K(obj_access_expr->get_func_name()), K(addr)); + } } else { - ObObjAccessRawExpr* obj_access_expr = static_cast(expr); - obj_access_expr->set_get_attr_func_addr(helper_.get_function_address(obj_access_expr->get_func_name())); + obj_access_expr->set_get_attr_func_addr(addr); } } @@ -8618,6 +8632,8 @@ int ObPLCodeGenerator::generate_normal(ObPLFunction &pl_func) } if (OB_SUCC(ret)) { + uint64_t addr = 0; + if (OB_FAIL(final_expression(pl_func))) { LOG_WARN("generate obj access expr failed", K(ret)); } else if (OB_FAIL(pl_func.set_variables(get_ast().get_symbol_table()))) { @@ -8626,11 +8642,13 @@ int ObPLCodeGenerator::generate_normal(ObPLFunction &pl_func) LOG_WARN("failed to set ref objects", K(get_ast().get_dependency_table()), K(ret)); } else if (OB_FAIL(pl_func.set_types(get_ast().get_user_type_table()))) { LOG_WARN("failed to set types", K(ret)); + } else if (OB_FAIL(helper_.get_function_address(get_ast().get_name(), addr))) { + LOG_WARN("failed to compile pl routine", K(ret), K(get_ast().get_name()), K(addr)); } else { pl_func.add_members(get_ast().get_flag()); pl_func.set_pipelined(get_ast().get_pipelined()); - pl_func.set_action(helper_.get_function_address(get_ast().get_name())); - pl_func.set_can_cached(get_ast().get_can_cached()); + pl_func.set_action(addr); + pl_func.set_can_cached(get_ast().get_can_cached()); pl_func.set_is_all_sql_stmt(get_ast().get_is_all_sql_stmt()); pl_func.set_has_parallel_affect_factor(get_ast().has_parallel_affect_factor()); } diff --git a/src/pl/ob_pl_package_manager.cpp b/src/pl/ob_pl_package_manager.cpp index b607d4755..7ced6b2c1 100644 --- a/src/pl/ob_pl_package_manager.cpp +++ b/src/pl/ob_pl_package_manager.cpp @@ -39,14 +39,152 @@ using namespace sql; namespace pl { -int ObPLPackageManager::read_package_sql(FILE* file, char* buf, int64_t buf_len, bool &eof) +// Usage: ObCharStream *s -> s.open() -> s.next().is_eos() ... -> s.close() +class ObCharStream +{ +public: + ObCharStream(const char *name) : eos_flag_(false), name_(name) {} + virtual ~ObCharStream() {} + ObCharStream(const ObCharStream &) = delete; + const ObCharStream &operator=(const ObCharStream &) = delete; + + const char *get_name() { return name_; } + virtual int open() = 0; + virtual const ObCharStream &next(char &c) = 0; + virtual bool is_eos() const { return eos_flag_; } + virtual int close() = 0; + + VIRTUAL_TO_STRING_KV(K_(eos_flag), K_(name)); + +protected: + static const char EOS = '\0'; + bool eos_flag_; + +private: + const char *const name_; +}; + +// read package sql from array embeded in the oberver binary file +class ObCStringStream final : public ObCharStream +{ +public: + ObCStringStream(const char *package_name, const char *data) + : ObCharStream(package_name), data_(data), cursor_(nullptr) {} + ~ObCStringStream() {} + + int open() override { + int ret = OB_SUCCESS; + if (OB_ISNULL(data_)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("package sql data is null", K(ret), K(data_)); + } else if (0 == strlen(data_)) { + LOG_INFO("package sql file is empty or not exists", K(ret)); + } else { + cursor_ = data_; + } + return ret; + } + const ObCharStream &next(char &c) override { + if (OB_NOT_NULL(cursor_) && !eos_flag_) { + c = *cursor_++; + eos_flag_ = (c == EOS); + } else { + c = EOS; + } + return *this; + } + int close() override { + int ret = OB_SUCCESS; + if (OB_NOT_NULL(cursor_)) { + if (EOS != *(cursor_ - 1)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("C string stream has not been completely consumed", K(ret), + K(cursor_ - data_), K(data_), K(cursor_)); + } else { + cursor_ = nullptr; + eos_flag_ = false; + } + } + return ret; + } + + INHERIT_TO_STRING_KV("ObCharStream", ObCharStream, + "C string stream bytes comsumed", cursor_ - data_); + +private: + const char *const data_; + const char *cursor_; +}; + +// read package sql from file under admin/ +class ObFileStream final : public ObCharStream +{ +public: + ObFileStream(const char *package_name, const char *file_path) + : ObCharStream(package_name), file_path_(file_path), file_(nullptr) {} + ~ObFileStream() { + // make sure file_ closed + if (file_) { + fclose(file_); + file_ = nullptr; + } + } + + int open() override { + int ret = OB_SUCCESS; + if (OB_ISNULL(file_path_) || 0 != access(file_path_, F_OK)) { + ret = OB_FILE_NOT_EXIST; + LOG_WARN("package sql file not exists", K(ret), K(file_path_)); + } else if (OB_ISNULL(file_ = fopen(file_path_, "rb"))) { + ret = OB_IO_ERROR; + LOG_WARN("package sql file open failed", K(ret), K(file_path_)); + } + return ret; + } + const ObCharStream &next(char &c) override { + int ch; + if (OB_NOT_NULL(file_) && !eos_flag_) { + if (EOF == (ch = fgetc(file_))) { + c = EOS; + eos_flag_ = true; + } else { + c = static_cast(ch); + } + } else { + c = EOS; + } + return *this; + } + int close() override { + int ret = OB_SUCCESS; + if (OB_NOT_NULL(file_)) { + if (0 == feof(file_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("file content has not been completely consumed", K(ret), K(errno)); + } else if (0 != fclose(file_)) { + ret = OB_IO_ERROR; + LOG_WARN("close file failed", K(ret), K(file_path_), K(file_)); + } else { + file_ = nullptr; + eos_flag_ = false; + } + } + return ret; + } + + INHERIT_TO_STRING_KV("ObCharStream", ObCharStream, K_(file_path), + "file stream bytes comsumed", file_ ? ftell(file_) : -1); + +private: + const char *const file_path_; + FILE *file_; +}; + +int ObPLPackageManager::read_package_sql(ObCharStream &stream, char* buf, int64_t buf_len, bool &eos) { int ret = OB_SUCCESS; enum {S_LINE_START, S_NORMAL, S_COMMENT, S_TERMINATE} state = S_LINE_START; - if (OB_ISNULL(file)) { - ret = OB_ERR_NULL_VALUE; - LOG_WARN("package sql file is null", K(ret)); - } else if (OB_ISNULL(buf)) { + if (OB_ISNULL(buf)) { ret = OB_ERR_NULL_VALUE; LOG_WARN("sql buffer is null", K(ret)); } else if (buf_len <= 0) { @@ -56,13 +194,13 @@ int ObPLPackageManager::read_package_sql(FILE* file, char* buf, int64_t buf_len, char *p = buf; char *p_start = p; char *p_end = p + buf_len - 1; - int c; + char c; // clear buffer *p = '\0'; *p_end = '\0'; while (OB_SUCC(ret) && state != S_TERMINATE) { - if (EOF == (c = fgetc(file))) { - ret = OB_IO_ERROR; + if (stream.next(c).is_eos()) { + ret = OB_ITER_END; } else { if (p >= p_end) { ret = OB_INVALID_ARGUMENT; @@ -77,18 +215,17 @@ int ObPLPackageManager::read_package_sql(FILE* file, char* buf, int64_t buf_len, } else if ('#' == c) { state = S_COMMENT; } else if ('-' == c) { - c = fgetc(file); - if ('-' == c) { + if (stream.next(c).is_eos()) { + ret = OB_ITER_END; + } else if ('-' == c) { state = S_COMMENT; - } else if (EOF == c) { - ret = OB_IO_ERROR; } else { *p++ = '-'; - *p++ = static_cast(c); + *p++ = c; state = S_NORMAL; } } else { - *p++ = static_cast(c); + *p++ = c; state = S_NORMAL; } } @@ -107,11 +244,11 @@ int ObPLPackageManager::read_package_sql(FILE* file, char* buf, int64_t buf_len, *(p - 1) = '\0'; state = S_TERMINATE; } else { - *p++ = static_cast(c); + *p++ = c; state = S_NORMAL; } } else { - *p++ = static_cast(c); + *p++ = c; if ('\n' == c) { state = S_LINE_START; } else { @@ -130,19 +267,20 @@ int ObPLPackageManager::read_package_sql(FILE* file, char* buf, int64_t buf_len, } } if (OB_FAIL(ret)) { - if (feof(file)) { - eof = true; + if (stream.is_eos()) { + eos = true; ret = OB_SUCCESS; } else { - LOG_WARN("read package file error", K(errno), K(ret)); + LOG_WARN("read package file error", K(ret), K(stream)); } } } return ret; } -int ObPLPackageManager::read_and_exec_package_sql( - ObMySQLProxy &sql_proxy, const char* package_full_path, ObCompatibilityMode compa_mode) +int ObPLPackageManager::read_and_exec_package_sql(ObMySQLProxy &sql_proxy, + ObCharStream &stream, + ObCompatibilityMode compa_mode) { int ret = OB_SUCCESS; if (!sql_proxy.is_inited() || !sql_proxy.is_active()) { @@ -150,13 +288,9 @@ int ObPLPackageManager::read_and_exec_package_sql( LOG_WARN("sql_proxy not inited or not active", "sql_proxy inited", sql_proxy.is_inited(), "sql_proxy active", sql_proxy.is_active(), K(ret)); } else { - FILE* file = NULL; int64_t affected_rows = 0; - if (access(package_full_path, F_OK) != 0) { - LOG_INFO("package sql file not exists", K(package_full_path), K(ret)); - } else if (OB_ISNULL(file = fopen(package_full_path, "rb"))) { - ret = OB_IO_ERROR; - LOG_WARN("package sql file open failed", K(package_full_path), K(ret)); + if (OB_FAIL(stream.open())) { + LOG_WARN("failed to open package file data stream", K(ret), K(stream)); } else { // system tenant will run with mysql compatibility mode // but we need to create system packages with oralce compatibility @@ -165,35 +299,35 @@ int ObPLPackageManager::read_and_exec_package_sql( bool create_external_table = false; ObSessionParam param; ObSessionParam *param_ptr = nullptr; - char *last_slash = strrchr(const_cast(package_full_path), '/'); - const char *pacakge_filename = (last_slash != NULL) ? last_slash + 1 : package_full_path; int64_t sql_mode = SMO_STRICT_ALL_TABLES | SMO_NO_ZERO_IN_DATE | SMO_NO_AUTO_CREATE_USER; // allow affected_rows > 0 when exec sql in external_table_alert_log.sql - if (strcmp(pacakge_filename, "external_table_alert_log.sql") == 0) { + if (strcmp(stream.get_name(), "external_table_alert_log") == 0) { create_external_table = true; param.sql_mode_ = &sql_mode; param_ptr = ¶m; } + // do not cache the compilation results of system packages into the PL cache when loading system packages. + param.enable_pl_cache_ = false; SMART_VAR(char[OB_MAX_SQL_LENGTH], sql_buf) { while (OB_SUCC(ret) && !eof) { - if (OB_FAIL(read_package_sql(file, sql_buf, OB_MAX_SQL_LENGTH, eof))) { - LOG_WARN("fail to read package sql file", K(ret)); + if (FAILEDx(read_package_sql(stream, sql_buf, OB_MAX_SQL_LENGTH, eof))) { + LOG_WARN("fail to read package sql data", K(ret)); } else if (strlen(sql_buf) != 0 - && OB_FAIL(sql_proxy.write(OB_SYS_TENANT_ID, + && OB_FAIL(sql_proxy.write(OB_SYS_TENANT_ID, sql_buf, affected_rows, static_cast(compa_mode), - param_ptr))) { + ¶m))) { LOG_WARN("fail to exec package sql", K(sql_buf), K(ret)); } else if (affected_rows != 0 && !create_external_table) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("affected_rows expected to be zero", K(affected_rows), K(ret)); + LOG_WARN("affected_rows expected to be zero", K(ret), K(affected_rows), K(stream.get_name())); } else { OZ (ObSPIService::force_refresh_schema(OB_SYS_TENANT_ID)); } + LOG_INFO("package source data consumed", K(ret), K(stream)); } } - fclose(file); if (create_external_table && OB_SUCC(ret)) { uint64_t data_version = 0; common::ObString alter_table_sql("alter external table sys_external_tbs.__all_external_alert_log_info auto_refresh immediate"); @@ -216,29 +350,80 @@ int ObPLPackageManager::read_and_exec_package_sql( return ret; } -int ObPLPackageManager::load_sys_package( - ObMySQLProxy &sql_proxy, const char *package_spec_name, const char *package_body_name, ObCompatibilityMode compa_mode) +// import source file content array declarations, which is defined in system_package.cpp generated by CMake. +extern const int64_t syspack_source_count; +extern const std::pair syspack_source_contents[]; + +int ObPLPackageManager::get_syspack_source_file_content(const char *file_name, const char *&content) { int ret = OB_SUCCESS; - const int64_t begin_time = ObTimeUtility::current_time(); - LOG_INFO("load sys package begin", - "package name", package_spec_name, "package body name", package_body_name); - - if (OB_FAIL(read_and_exec_package_sql(sql_proxy, package_spec_name, compa_mode))) { - LOG_WARN("fail to read and exec package header sql", K(package_spec_name), K(ret)); - } else if (OB_FAIL(read_and_exec_package_sql(sql_proxy, package_body_name, compa_mode))) { - LOG_WARN("fail to read and exec package body sql", K(package_body_name), K(ret)); + OX (content = nullptr); + if (OB_ISNULL(file_name)) { + // return nullptr as `content` + LOG_INFO("file name c string is null", K(file_name)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < syspack_source_count; i++) { + if (0 == ObString(file_name).case_compare(syspack_source_contents[i].first)) { + content = syspack_source_contents[i].second; + break; + } + } + // `content` not found, report error + OV (OB_NOT_NULL(content), OB_ERR_UNEXPECTED, "system source file not found", ret, file_name); } - - const int64_t now = ObTimeUtility::current_time(); - LOG_INFO("load sys package finish", "total_time_used", now - begin_time); return ret; } -static const char* sys_package_dir = "admin"; -static ObSysPackageFile oracle_sys_package_file_table[] = { +int ObPLPackageManager::load_sys_package(ObMySQLProxy &sql_proxy, + const ObSysPackageFile &pack_file_info, + ObCompatibilityMode compa_mode, + bool from_file) +{ + int ret = OB_SUCCESS; + const char *package_name = pack_file_info.package_name; + const char *spec_file = pack_file_info.package_spec_file_name; + const char *body_file = pack_file_info.package_body_file_name; + + const int64_t begin_time = ObTimeUtility::current_time(); + LOG_INFO("load sys package", K(package_name), K(spec_file), K(body_file), K(begin_time)); + + if (from_file) { + const char *sys_package_dir = "admin"; + char spec_file_path[MAX_PATH_SIZE] = {0}; + char body_file_path[MAX_PATH_SIZE] = {0}; + if (OB_SUCC(ret) && OB_NOT_NULL(spec_file)) { + OZ (databuff_printf(spec_file_path, MAX_PATH_SIZE, "%s/%s", sys_package_dir, spec_file)); + ObFileStream spec_stream{package_name, spec_file_path}; + OZ (read_and_exec_package_sql(sql_proxy, spec_stream, compa_mode), spec_stream); + } + if (OB_SUCC(ret) && OB_NOT_NULL(body_file)) { + OZ (databuff_printf(body_file_path, MAX_PATH_SIZE, "%s/%s", sys_package_dir, body_file)); + ObFileStream body_stream{package_name, body_file_path}; + OZ (read_and_exec_package_sql(sql_proxy, body_stream, compa_mode), body_stream); + } + } else { + const char *spec_content = nullptr; + const char *body_content = nullptr; + OZ (get_syspack_source_file_content(spec_file, spec_content)); + if (OB_SUCC(ret) && OB_NOT_NULL(spec_content)) { + ObCStringStream spec_stream{package_name, spec_content}; + OZ (read_and_exec_package_sql(sql_proxy, spec_stream, compa_mode), spec_stream); + } + OZ (get_syspack_source_file_content(body_file, body_content)); + if (OB_SUCC(ret) && OB_NOT_NULL(body_content)) { + ObCStringStream body_stream{package_name, body_content}; + OZ (read_and_exec_package_sql(sql_proxy, body_stream, compa_mode), body_stream); + } + } + + const int64_t now = ObTimeUtility::current_time(); + LOG_INFO("load sys package finish", K(ret), K(package_name), "total_time_used", now - begin_time); + return ret; +} + +static const ObSysPackageFile oracle_syspack_file_list[] = { #ifdef OB_BUILD_ORACLE_PL - {"dbms_standard", "dbms_standard.sql", "dbms_standard_body.sql"}, + {"dbms_standard", "dbms_standard.sql", nullptr}, {"dbms_output", "dbms_output.sql", "dbms_output_body.sql"}, {"dbms_metadata", "dbms_metadata.sql", "dbms_metadata_body.sql"}, {"dbms_spm", "dbms_spm.sql", "dbms_spm_body.sql"}, @@ -251,7 +436,7 @@ static ObSysPackageFile oracle_sys_package_file_table[] = { {"sa_sysdba", "sa_sysdba.sql", "sa_sysdba_body.sql"}, {"sa_user_admin", "sa_user_admin.sql", "sa_user_admin_body.sql"}, {"utl_i18n", "utl_i18n.sql", "utl_i18n_body.sql"}, - {"dbms_crypto", "dbms_crypto.sql","dbms_crypto_body.sql"}, + {"dbms_crypto", "dbms_crypto.sql", "dbms_crypto_body.sql"}, {"dbms_random", "dbms_random.sql", "dbms_random_body.sql"}, {"dbms_debug", "dbms_debug.sql", "dbms_debug_body.sql"}, {"utl_inaddr", "utl_inaddr.sql", "utl_inaddr_body.sql"}, @@ -263,16 +448,15 @@ static ObSysPackageFile oracle_sys_package_file_table[] = { {"dbms_xa", "dbms_xa.sql", "dbms_xa_body.sql"}, {"dbms_resource_manager", "dbms_resource_manager.sql", "dbms_resource_manager_body.sql"}, {"dbms_utility", "dbms_utility.sql", "dbms_utility_body.sql"}, - {"odciconst", "odciconst.sql", "odciconst_body.sql"}, + {"odciconst", "odciconst.sql", nullptr}, {"dbms_stats", "dbms_stats.sql", "dbms_stats_body.sql"}, {"dbms_any", "dbms_any.sql", "dbms_any_body.sql"}, {"xml_type", "xml_type.sql", "xml_type_body.sql"}, - {"dbms_crypto", "dbms_crypto.sql", "dbms_crypto_body.sql"}, {"dbms_ijob", "dbms_ijob.sql", "dbms_ijob_body.sql"}, {"dbms_job", "dbms_job.sql", "dbms_job_body.sql"}, {"dbms_ischeduler", "dbms_ischeduler.sql", "dbms_ischeduler_body.sql"}, {"dbms_scheduler", "dbms_scheduler.sql", "dbms_scheduler_body.sql"}, - {"catodci", "catodci.sql", "catodci_body.sql"}, + {"catodci", "catodci.sql", nullptr}, {"dbms_describe", "dbms_describe.sql", "dbms_describe_body.sql"}, {"utl_file", "utl_file.sql", "utl_file_body.sql"}, {"dbms_plan_cache", "dbms_plancache.sql", "dbms_plancache_body.sql"}, @@ -292,7 +476,7 @@ static ObSysPackageFile oracle_sys_package_file_table[] = { {"dbms_mview", "dbms_mview.sql", "dbms_mview_body.sql"}, {"dbms_mview_stats", "dbms_mview_stats.sql", "dbms_mview_stats_body.sql"}, {"json_array_t", "json_array_type.sql", "json_array_type_body.sql"}, - {"xmlsequence", "xml_sequence_type.sql", "xml_sequence_type_body.sql"}, + {"xmlsequence", "xml_sequence_type.sql", nullptr}, {"utl_recomp", "utl_recomp.sql", "utl_recomp_body.sql"}, {"sdo_geometry", "sdo_geometry.sql", "sdo_geometry_body.sql"}, {"sdo_geom", "sdo_geom.sql", "sdo_geom_body.sql"}, @@ -300,8 +484,7 @@ static ObSysPackageFile oracle_sys_package_file_table[] = { {"dbms_profiler", "dbms_profiler.sql", "dbms_profiler_body.sql"}, #endif }; - -static ObSysPackageFile mysql_sys_package_file_table[] = { +static const ObSysPackageFile mysql_syspack_file_list[] = { {"dbms_stats", "dbms_stats_mysql.sql", "dbms_stats_body_mysql.sql"}, {"dbms_scheduler", "dbms_scheduler_mysql.sql", "dbms_scheduler_mysql_body.sql"}, {"dbms_ischeduler", "dbms_ischeduler_mysql.sql", "dbms_ischeduler_mysql_body.sql"}, @@ -320,119 +503,109 @@ static ObSysPackageFile mysql_sys_package_file_table[] = { {"dbms_trusted_certificate_manager", "dbms_trusted_certificate_manager_mysql.sql", "dbms_trusted_certificate_manager_body_mysql.sql"}, {"dbms_ob_limit_calculator", "dbms_ob_limit_calculator_mysql.sql", "dbms_ob_limit_calculator_body_mysql.sql"}, {"dbms_external_table", "dbms_external_table_mysql.sql", "dbms_external_table_body_mysql.sql"}, - {"external_table_alert_log", "external_table_alert_log.sql", "none"} + {"external_table_alert_log", "external_table_alert_log.sql", nullptr} }; -int ObPLPackageManager::load_sys_package(ObMySQLProxy &sql_proxy, ObString &package_name, ObCompatibilityMode compa_mode) +// for now! we only have one special system package "__DBMS_UPGRADE" +static const ObSysPackageFile oracle_special_syspack_file_list[] = { + {"__dbms_upgrade", "__dbms_upgrade.sql", "__dbms_upgrade_body.sql"}, +}; +static const ObSysPackageFile mysql_special_syspack_file_list[] = { + {"__dbms_upgrade", "__dbms_upgrade_mysql.sql", "__dbms_upgrade_body_mysql.sql"}, +}; + +int ObPLPackageManager::load_sys_package(ObMySQLProxy &sql_proxy, + ObString &package_name, + ObCompatibilityMode compa_mode, + bool from_file) { int ret = OB_SUCCESS; - char package_spec_full_path[MAX_PATH_SIZE] = {}; - char package_body_full_path[MAX_PATH_SIZE] = {}; - bool dir_exists = false; - bool package_exists = false; - if (OB_FAIL(FileDirectoryUtils::is_exists(sys_package_dir, dir_exists))) { - LOG_WARN("check sys package dir whether exist failed", K(ret), K(sys_package_dir)); - } else if (!dir_exists) { - ret = OB_ENTRY_NOT_EXIST; - LOG_WARN("sys package dir not exist", K(ret), K(sys_package_dir)); - } + const ObSysPackageFile *pack_file_info = nullptr; + +#define SEARCH_SYSPACK_FILE_BY_NAME(syspack_file_list) \ + do { \ + if (pack_file_info == nullptr) { \ + int sys_package_count = ARRAYSIZEOF(syspack_file_list); \ + for (int64_t i = 0; OB_SUCC(ret) && i < sys_package_count; ++i) { \ + if (0 == package_name.case_compare(ObString(syspack_file_list[i].package_name))) { \ + pack_file_info = &syspack_file_list[i]; \ + break; \ + } \ + } \ + } \ + } while (0) + if (ObCompatibilityMode::ORACLE_MODE == compa_mode) { - int sys_package_count = ARRAYSIZEOF(oracle_sys_package_file_table); - for (int64_t i = 0; OB_SUCC(ret) && i < sys_package_count; ++i) { - if (0 == package_name.case_compare(ObString(oracle_sys_package_file_table[i].package_name))) { - const char *package_spec_name = oracle_sys_package_file_table[i].package_spec_file_name; - const char *package_body_name = oracle_sys_package_file_table[i].package_body_file_name; - OZ (databuff_printf( - package_spec_full_path, MAX_PATH_SIZE, "%s/%s", sys_package_dir, package_spec_name)); - OZ (databuff_printf( - package_body_full_path, MAX_PATH_SIZE, "%s/%s", sys_package_dir, package_body_name)); - OX (package_exists = true); - break; - } - } + SEARCH_SYSPACK_FILE_BY_NAME(oracle_syspack_file_list); + SEARCH_SYSPACK_FILE_BY_NAME(oracle_special_syspack_file_list); } else if (ObCompatibilityMode::MYSQL_MODE == compa_mode) { - int sys_package_count = ARRAYSIZEOF(mysql_sys_package_file_table); - for (int64_t i = 0; OB_SUCC(ret) && i < sys_package_count; ++i) { - if (0 == package_name.case_compare(ObString(mysql_sys_package_file_table[i].package_name))) { - const char *package_spec_name = mysql_sys_package_file_table[i].package_spec_file_name; - const char *package_body_name = mysql_sys_package_file_table[i].package_body_file_name; - OZ (databuff_printf( - package_spec_full_path, MAX_PATH_SIZE, "%s/%s", sys_package_dir, package_spec_name)); - OZ (databuff_printf( - package_body_full_path, MAX_PATH_SIZE, "%s/%s", sys_package_dir, package_body_name)); - OX (package_exists = true); - break; - } - } + SEARCH_SYSPACK_FILE_BY_NAME(mysql_syspack_file_list); + SEARCH_SYSPACK_FILE_BY_NAME(mysql_special_syspack_file_list); } - if (OB_SUCC(ret) && !package_exists) { +#undef SEARCH_SYSPACK_FILE_BY_NAME + + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(pack_file_info)) { ret = OB_ERR_PACKAGE_DOSE_NOT_EXIST; - LOG_WARN("package not exists", K(ret), K(package_name)); + LOG_WARN("package not exists", K(ret), K(package_name), K(compa_mode)); LOG_USER_ERROR(OB_ERR_PACKAGE_DOSE_NOT_EXIST, "PACKAGE", ObString("oceanbase").length(), ObString("oceanbase").ptr(), package_name.length(), package_name.ptr()); + } else { + OZ (load_sys_package(sql_proxy, *pack_file_info, compa_mode, from_file)); } - OZ (load_sys_package(sql_proxy, package_spec_full_path, package_body_full_path, compa_mode)); return ret; } -int ObPLPackageManager::load_all_common_sys_package(ObMySQLProxy &sql_proxy, - const ObSysPackageFile *package_file, - int sys_package_count, - ObCompatibilityMode compa_mode) +int ObPLPackageManager::load_sys_package_list(ObMySQLProxy &sql_proxy, + const ObSysPackageFile *sys_package_list, + int sys_package_count, + ObCompatibilityMode compa_mode, + bool from_file) { int ret = OB_SUCCESS; - char package_spec_full_path[MAX_PATH_SIZE] = {}; - char package_body_full_path[MAX_PATH_SIZE] = {}; - CK (OB_NOT_NULL(package_file)); - LOG_INFO("load all sys package begin", "sys package total count", sys_package_count); + CK (OB_NOT_NULL(sys_package_list)); + LOG_INFO("load sys package list begin", "sys package total count", sys_package_count); for (int i = 0; OB_SUCC(ret) && i < sys_package_count; ++i) { - const char *package_spec_name = package_file[i].package_spec_file_name; - const char *package_body_name = package_file[i].package_body_file_name; - OZ (databuff_printf( - package_spec_full_path, MAX_PATH_SIZE, "%s/%s", sys_package_dir, package_spec_name)); - OZ (databuff_printf( - package_body_full_path, MAX_PATH_SIZE, "%s/%s", sys_package_dir, package_body_name)); - if (OB_SUCC(ret)) { - LOG_INFO("load sys package begin", K(package_spec_name)); - if (OB_FAIL(load_sys_package(sql_proxy, package_spec_full_path, package_body_full_path, compa_mode))) { - LOG_WARN("load sys package failed", - K(package_spec_full_path), K(package_body_full_path), K(compa_mode), K(ret)); - } else { - LOG_INFO("load sys package success", K(ret), K(package_spec_name)); - } - } + OZ (load_sys_package(sql_proxy, sys_package_list[i], compa_mode, from_file)); + } + if (OB_FAIL(ret)) { + LOG_WARN("load sys package list failed", K(ret), K(compa_mode)); + } else { + LOG_INFO("load sys package list success", K(ret), K(compa_mode)); } return ret; } -int ObPLPackageManager::load_all_common_sys_package(ObMySQLProxy &sql_proxy, ObCompatibilityMode need_compa_mode) -{ +int ObPLPackageManager::load_all_common_sys_package( + ObMySQLProxy &sql_proxy, ObCompatibilityMode compa_mode, bool from_file) { int ret = OB_SUCCESS; - bool exist = false; - if (OB_FAIL(FileDirectoryUtils::is_exists(sys_package_dir, exist))) { - LOG_WARN("check sys package dir whether exist failed", K(sys_package_dir), K(ret)); - } else if (!exist) { - LOG_INFO("sys package dir not exist", K(sys_package_dir)); - } else { - if (need_compa_mode == ObCompatibilityMode::OCEANBASE_MODE) { - OZ (load_all_common_sys_package(sql_proxy, oracle_sys_package_file_table, - ARRAYSIZEOF(oracle_sys_package_file_table), ObCompatibilityMode::ORACLE_MODE)); - OZ (load_all_common_sys_package(sql_proxy, mysql_sys_package_file_table, - ARRAYSIZEOF(mysql_sys_package_file_table), ObCompatibilityMode::MYSQL_MODE)); - } else if (need_compa_mode == ObCompatibilityMode::ORACLE_MODE) { - OZ (load_all_common_sys_package(sql_proxy, oracle_sys_package_file_table, - ARRAYSIZEOF(oracle_sys_package_file_table), ObCompatibilityMode::ORACLE_MODE)); - } else if (need_compa_mode == ObCompatibilityMode::MYSQL_MODE) { - OZ (load_all_common_sys_package(sql_proxy, mysql_sys_package_file_table, - ARRAYSIZEOF(mysql_sys_package_file_table), ObCompatibilityMode::MYSQL_MODE)); - } + if (compa_mode == ObCompatibilityMode::OCEANBASE_MODE) { + OZ (load_sys_package_list(sql_proxy, oracle_syspack_file_list, + ARRAYSIZEOF(oracle_syspack_file_list), + ObCompatibilityMode::ORACLE_MODE, + from_file)); + OZ (load_sys_package_list(sql_proxy, mysql_syspack_file_list, + ARRAYSIZEOF(mysql_syspack_file_list), + ObCompatibilityMode::MYSQL_MODE, + from_file)); + } else if (compa_mode == ObCompatibilityMode::ORACLE_MODE) { + OZ (load_sys_package_list(sql_proxy, oracle_syspack_file_list, + ARRAYSIZEOF(oracle_syspack_file_list), + ObCompatibilityMode::ORACLE_MODE, + from_file)); + } else if (compa_mode == ObCompatibilityMode::MYSQL_MODE) { + OZ (load_sys_package_list(sql_proxy, mysql_syspack_file_list, + ARRAYSIZEOF(mysql_syspack_file_list), + ObCompatibilityMode::MYSQL_MODE, + from_file)); } + if (OB_SUCC(ret)) { - LOG_INFO("load all common sys package success!"); + LOG_INFO("load all common sys package success!", K(ret), K(from_file)); } else { - LOG_INFO("load all common sys package failed!"); + LOG_WARN("load all common sys package failed!", K(ret), K(from_file)); } return ret; } @@ -440,39 +613,23 @@ int ObPLPackageManager::load_all_common_sys_package(ObMySQLProxy &sql_proxy, ObC int ObPLPackageManager::load_all_special_sys_package(ObMySQLProxy &sql_proxy) { int ret = OB_SUCCESS; - bool exist = false; - if (OB_FAIL(FileDirectoryUtils::is_exists(sys_package_dir, exist))) { - LOG_WARN("check sys package dir whether exist failed", K(sys_package_dir), K(ret)); - } else if (!exist) { - LOG_INFO("sys package dir not exist", K(sys_package_dir)); - } else { - // for now! we only have one special system package "__DBMS_UPGRADE" - char package_spec_full_path[MAX_PATH_SIZE] = {}; - char package_body_full_path[MAX_PATH_SIZE] = {}; - #ifdef OB_BUILD_ORACLE_PL - OZ (databuff_printf( - package_spec_full_path, MAX_PATH_SIZE, "%s/%s", sys_package_dir, "__dbms_upgrade.sql")); - OZ (databuff_printf( - package_body_full_path, MAX_PATH_SIZE, "%s/%s", sys_package_dir, "__dbms_upgrade_body.sql")); - OZ (load_sys_package(sql_proxy, package_spec_full_path, package_body_full_path, ObCompatibilityMode::ORACLE_MODE)); + OZ (load_sys_package_list(sql_proxy, oracle_special_syspack_file_list, + ARRAYSIZEOF(oracle_special_syspack_file_list), + ObCompatibilityMode::ORACLE_MODE, + false /* from_file */)); #endif - - memset(package_spec_full_path, 0, sizeof(package_spec_full_path)); - memset(package_body_full_path, 0, sizeof(package_body_full_path)); - OZ (databuff_printf( - package_spec_full_path, MAX_PATH_SIZE, "%s/%s", sys_package_dir, "__dbms_upgrade_mysql.sql")); - OZ (databuff_printf( - package_body_full_path, MAX_PATH_SIZE, "%s/%s", sys_package_dir, "__dbms_upgrade_body_mysql.sql")); - OZ (load_sys_package(sql_proxy, package_spec_full_path, package_body_full_path, ObCompatibilityMode::MYSQL_MODE)); - } + OZ (load_sys_package_list(sql_proxy, mysql_special_syspack_file_list, + ARRAYSIZEOF(mysql_special_syspack_file_list), + ObCompatibilityMode::MYSQL_MODE, + false /* from_file */)); return ret; } int ObPLPackageManager::load_all_sys_package(ObMySQLProxy &sql_proxy) { int ret = OB_SUCCESS; - OZ (load_all_common_sys_package(sql_proxy, ObCompatibilityMode::OCEANBASE_MODE)); + OZ (load_all_common_sys_package(sql_proxy, ObCompatibilityMode::OCEANBASE_MODE, false /* from_file */)); OZ (load_all_special_sys_package(sql_proxy)); if (OB_SUCC(ret)) { LOG_INFO("load all sys package success!", K(ret)); diff --git a/src/pl/ob_pl_package_manager.h b/src/pl/ob_pl_package_manager.h index bfa94513a..6ab4605c6 100644 --- a/src/pl/ob_pl_package_manager.h +++ b/src/pl/ob_pl_package_manager.h @@ -63,11 +63,13 @@ class ObPLCursor; class ObPLCacheCtx; struct ObSysPackageFile { - const char *package_name; - const char *package_spec_file_name; - const char *package_body_file_name; + const char *const package_name; + const char *const package_spec_file_name; + const char *const package_body_file_name; }; +class ObCharStream; + class ObPLPackageManager { public: @@ -139,28 +141,24 @@ public: uint64_t package_id, ObPLPackageState *&package_state, bool for_static_member = false); - static int read_package_sql(FILE *file, char* buf, int64_t buf_len, bool &eof); - static int read_and_exec_package_sql( - common::ObMySQLProxy &sql_proxy, const char* package_full_path, ObCompatibilityMode compa_mode); - static int load_sys_package( - common::ObMySQLProxy &sql_proxy, const char *package_spec_name, const char *package_body_name, ObCompatibilityMode compa_mode); - static int load_sys_package(common::ObMySQLProxy &sql_proxy, common::ObString &package_name, ObCompatibilityMode compa_mode); - static int load_all_common_sys_package(common::ObMySQLProxy &sql_proxy, ObCompatibilityMode compa_mode); + + static int load_sys_package(common::ObMySQLProxy &sql_proxy, + common::ObString &package_name, + ObCompatibilityMode compa_mode, + bool from_file); static int load_all_common_sys_package(common::ObMySQLProxy &sql_proxy, - const ObSysPackageFile *package_file, - int sys_package_count, - ObCompatibilityMode compa_mode); - static int load_all_special_sys_package(common::ObMySQLProxy &sql_proxy); + ObCompatibilityMode compa_mode, + bool from_file); static int load_all_sys_package(common::ObMySQLProxy &sql_proxy); + static int add_package_to_plan_cache(const ObPLResolveCtx &resolve_ctx, ObPLPackage *package); static int get_package_from_plan_cache(const ObPLResolveCtx &resolve_ctx, uint64_t package_id, ObPLPackage *&package); - static int get_package_schema_info( - share::schema::ObSchemaGetterGuard &schema_guard, - uint64_t package_id, - const share::schema::ObPackageInfo *&package_spec_info, - const share::schema::ObPackageInfo *&package_body_info); + static int get_package_schema_info(share::schema::ObSchemaGetterGuard &schema_guard, + uint64_t package_id, + const share::schema::ObPackageInfo *&package_spec_info, + const share::schema::ObPackageInfo *&package_body_info); static int destory_package_state(sql::ObSQLSessionInfo &session_info, uint64_t package_id); int check_version(const ObPLResolveCtx &resolve_ctx, uint64_t package_id, const ObPackageStateVersion &state_version, bool &match); @@ -174,6 +172,22 @@ public: private: DISALLOW_COPY_AND_ASSIGN(ObPLPackageManager); + static int read_package_sql(ObCharStream &stream, char* buf, int64_t buf_len, bool &eos); + static int read_and_exec_package_sql(common::ObMySQLProxy &sql_proxy, + ObCharStream &stream, + ObCompatibilityMode compa_mode); + static int get_syspack_source_file_content(const char *file_name, const char *&content); + static int load_sys_package(ObMySQLProxy &sql_proxy, + const ObSysPackageFile &pack_file_info, + ObCompatibilityMode compa_mode, + bool from_file); + static int load_sys_package_list(common::ObMySQLProxy &sql_proxy, + const ObSysPackageFile *sys_package_list, + int sys_package_count, + ObCompatibilityMode compa_mode, + bool from_file); + static int load_all_special_sys_package(common::ObMySQLProxy &sql_proxy); + int get_cached_package_spec(const ObPLResolveCtx &resolve_ctx, uint64_t package_id, ObPLPackage *&package_spec); diff --git a/src/pl/ob_pl_type.cpp b/src/pl/ob_pl_type.cpp index c3e837cb1..74ac9ae96 100644 --- a/src/pl/ob_pl_type.cpp +++ b/src/pl/ob_pl_type.cpp @@ -2389,10 +2389,13 @@ int ObPLCursorInfo::prepare_spi_result(ObPLExecCtx *ctx, ObSPIResultSet *&spi_re } OX (spi_cursor_ = get_allocator()->alloc(sizeof(ObSPIResultSet))); OV (OB_NOT_NULL(spi_cursor_), OB_ALLOCATE_MEMORY_FAILED); + } else { + CK (OB_NOT_NULL(static_cast(spi_cursor_))); + OX ((static_cast(spi_cursor_))->~ObSPIResultSet()); } OX (spi_result = new (spi_cursor_) ObSPIResultSet()); - OZ (spi_result->init(*ctx->exec_ctx_->get_my_session())); OX (last_stream_cursor_ = true); + OZ (spi_result->init(*ctx->exec_ctx_->get_my_session())); return ret; } @@ -2413,6 +2416,14 @@ int ObPLCursorInfo::prepare_spi_cursor(ObSPICursor *&spi_cursor, : sizeof(ObSPICursor); OX (spi_cursor_ = spi_allocator->alloc(alloc_size)); OV (OB_NOT_NULL(spi_cursor_), OB_ALLOCATE_MEMORY_FAILED); + } else { + if (last_stream_cursor_) { + CK (OB_NOT_NULL(static_cast(spi_cursor_))); + OX (static_cast(spi_cursor_)->~ObSPIResultSet()); + } else { + CK (OB_NOT_NULL(static_cast(spi_cursor_))); + OX (static_cast(spi_cursor_)->~ObSPICursor()); + } } OX (spi_cursor = new (spi_cursor_) ObSPICursor(*spi_allocator, session_info)); OX (last_stream_cursor_ = false); diff --git a/src/pl/sys_package/ob_dbms_upgrade.cpp b/src/pl/sys_package/ob_dbms_upgrade.cpp index ee2305a3b..7a3ac37d2 100644 --- a/src/pl/sys_package/ob_dbms_upgrade.cpp +++ b/src/pl/sys_package/ob_dbms_upgrade.cpp @@ -28,15 +28,33 @@ int ObDBMSUpgrade::upgrade_single( { int ret = OB_SUCCESS; ObString package_name; + bool load_from_file = true; ObCompatibilityMode mode = lib::is_oracle_mode() ? ObCompatibilityMode::ORACLE_MODE : ObCompatibilityMode::MYSQL_MODE; UNUSED(result); CK (OB_NOT_NULL(ctx.get_sql_proxy())); - CK (OB_LIKELY(1 == params.count())); - OV (params.at(0).is_varchar(), OB_INVALID_ARGUMENT); - OZ (params.at(0).get_string(package_name)); - OV (!package_name.empty(), OB_INVALID_ARGUMENT); - OZ (ObPLPackageManager::load_sys_package(*ctx.get_sql_proxy(), package_name, mode)); + // OBServer 4.2.4 has added new parameters on the __DBMS_UPGRADE + // interface to control whether to load the system package source code from + // a file or embeded c string. To maintain compatibility during upgarding, + // it is necessary to distinguish the old and new versions of the interface. + // However, the system package does not have version control, so it depends + // on the number of parameters to judge the old and new versions. + if (OB_FAIL(ret)) { + } else if (1 == params.count()) { + OV (params.at(0).is_varchar(), OB_INVALID_ARGUMENT); + OZ (params.at(0).get_string(package_name)); + OV (!package_name.empty(), OB_INVALID_ARGUMENT); + } else if (2 == params.count()) { + OV (params.at(0).is_varchar(), OB_INVALID_ARGUMENT); + OZ (params.at(0).get_string(package_name)); + OV (!package_name.empty(), OB_INVALID_ARGUMENT); + OV (params.at(1).is_tinyint(), OB_INVALID_ARGUMENT); + OZ (params.at(1).get_bool(load_from_file)); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("__DBMS_UPGRADE.UPGRADE_SINGLE require one or two arguments", K(ret), K(params)); + } + OZ (ObPLPackageManager::load_sys_package(*ctx.get_sql_proxy(), package_name, mode, load_from_file)); return ret; } @@ -44,12 +62,22 @@ int ObDBMSUpgrade::upgrade_all( sql::ObExecContext &ctx, sql::ParamStore ¶ms, common::ObObj &result) { int ret = OB_SUCCESS; + bool load_from_file = true; ObCompatibilityMode mode = lib::is_oracle_mode() ? ObCompatibilityMode::ORACLE_MODE : ObCompatibilityMode::MYSQL_MODE; UNUSED(result); CK (OB_NOT_NULL(ctx.get_sql_proxy())); - CK (OB_LIKELY(0 == params.count())); - OZ (ObPLPackageManager::load_all_common_sys_package(*ctx.get_sql_proxy(), mode)); + if (OB_FAIL(ret)) { + } else if (0 == params.count()) { + // do nothing + } else if (1 == params.count()) { + OV (params.at(0).is_tinyint(), OB_INVALID_ARGUMENT); + OZ (params.at(0).get_bool(load_from_file)); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("__DBMS_UPGRADE.UPGRADE_ALL require none or one arguments", K(ret), K(params)); + } + OZ (ObPLPackageManager::load_all_common_sys_package(*ctx.get_sql_proxy(), mode, load_from_file)); return ret; } diff --git a/src/rootserver/ob_upgrade_executor.cpp b/src/rootserver/ob_upgrade_executor.cpp index 0f6588aaa..5b0b487b7 100644 --- a/src/rootserver/ob_upgrade_executor.cpp +++ b/src/rootserver/ob_upgrade_executor.cpp @@ -915,17 +915,18 @@ int ObUpgradeExecutor::upgrade_mysql_system_package_job_() int64_t timeout = GCONF._ob_ddl_timeout; const char *create_package_sql = "CREATE OR REPLACE PACKAGE __DBMS_UPGRADE \ - PROCEDURE UPGRADE(package_name VARCHAR(1024)); \ - PROCEDURE UPGRADE_ALL(); \ + PROCEDURE UPGRADE(package_name VARCHAR(1024), \ + load_from_file BOOLEAN DEFAULT TRUE); \ + PROCEDURE UPGRADE_ALL(load_from_file BOOLEAN DEFAULT TRUE); \ END;"; const char *create_package_body_sql = "CREATE OR REPLACE PACKAGE BODY __DBMS_UPGRADE \ - PROCEDURE UPGRADE(package_name VARCHAR(1024)); \ + PROCEDURE UPGRADE(package_name VARCHAR(1024), load_from_file BOOLEAN); \ PRAGMA INTERFACE(c, UPGRADE_SINGLE); \ - PROCEDURE UPGRADE_ALL(); \ + PROCEDURE UPGRADE_ALL(load_from_file BOOLEAN); \ PRAGMA INTERFACE(c, UPGRADE_ALL); \ END;"; - const char *upgrade_sql = "CALL __DBMS_UPGRADE.UPGRADE_ALL();"; + const char *upgrade_sql = "CALL __DBMS_UPGRADE.UPGRADE_ALL(FALSE);"; ObTimeoutCtx ctx; int64_t affected_rows = 0; if (OB_FAIL(check_inner_stat_())) { @@ -974,17 +975,18 @@ int ObUpgradeExecutor::upgrade_oracle_system_package_job_() int64_t timeout = GCONF._ob_ddl_timeout; const char *create_package_sql = "CREATE OR REPLACE PACKAGE \"__DBMS_UPGRADE\" IS \ - PROCEDURE UPGRADE(package_name VARCHAR2); \ - PROCEDURE UPGRADE_ALL; \ + PROCEDURE UPGRADE(package_name VARCHAR2, \ + load_from_file BOOLEAN DEFAULT TRUE); \ + PROCEDURE UPGRADE_ALL(load_from_file BOOLEAN DEFAULT TRUE); \ END;"; const char *create_package_body_sql = "CREATE OR REPLACE PACKAGE BODY \"__DBMS_UPGRADE\" IS \ - PROCEDURE UPGRADE(package_name VARCHAR2); \ + PROCEDURE UPGRADE(package_name VARCHAR2, load_from_file BOOLEAN); \ PRAGMA INTERFACE(c, UPGRADE_SINGLE); \ - PROCEDURE UPGRADE_ALL; \ + PROCEDURE UPGRADE_ALL(load_from_file BOOLEAN); \ PRAGMA INTERFACE(c, UPGRADE_ALL); \ END;"; - const char *upgrade_sql = "CALL \"__DBMS_UPGRADE\".UPGRADE_ALL();"; + const char *upgrade_sql = "BEGIN \"__DBMS_UPGRADE\".UPGRADE_ALL(FALSE); END;"; ObTimeoutCtx ctx; int64_t affected_rows = 0; if (OB_FAIL(check_inner_stat_())) { @@ -1017,9 +1019,9 @@ int ObUpgradeExecutor::upgrade_oracle_system_package_job_() OB_SYS_TENANT_ID, upgrade_sql, affected_rows, static_cast(mode)))) { LOG_WARN("fail to execute sql", KR(ret), "sql", upgrade_sql); - } else if (0 != affected_rows) { + } else if (1 != affected_rows) { // default value of oracle anonymous block affected_rows is 1 ret = OB_ERR_UNEXPECTED; - LOG_WARN("affected_rows expected to be zero", KR(ret), K(affected_rows)); + LOG_WARN("affected_rows expected to be 1", KR(ret), K(affected_rows)); } FLOG_INFO("[UPGRADE] finish run upgrade oracle system package job", KR(ret), "cost", ObTimeUtility::current_time() - start_ts); diff --git a/src/share/inner_table/sys_package/__dbms_upgrade_body_mysql.sql b/src/share/inner_table/sys_package/__dbms_upgrade_body_mysql.sql index 2a63d7487..fe1eab6ca 100644 --- a/src/share/inner_table/sys_package/__dbms_upgrade_body_mysql.sql +++ b/src/share/inner_table/sys_package/__dbms_upgrade_body_mysql.sql @@ -3,9 +3,9 @@ CREATE OR REPLACE PACKAGE BODY __DBMS_UPGRADE - PROCEDURE UPGRADE(package_name VARCHAR(1024)); + PROCEDURE UPGRADE(package_name VARCHAR(1024), load_from_file BOOLEAN); PRAGMA INTERFACE(c, UPGRADE_SINGLE); - PROCEDURE UPGRADE_ALL(); + PROCEDURE UPGRADE_ALL(load_from_file BOOLEAN); PRAGMA INTERFACE(c, UPGRADE_ALL); END; // diff --git a/src/share/inner_table/sys_package/__dbms_upgrade_mysql.sql b/src/share/inner_table/sys_package/__dbms_upgrade_mysql.sql index 45b7221bc..6787c0c8e 100644 --- a/src/share/inner_table/sys_package/__dbms_upgrade_mysql.sql +++ b/src/share/inner_table/sys_package/__dbms_upgrade_mysql.sql @@ -3,7 +3,8 @@ CREATE OR REPLACE PACKAGE __DBMS_UPGRADE - PROCEDURE UPGRADE(package_name VARCHAR(1024)); - PROCEDURE UPGRADE_ALL(); + PROCEDURE UPGRADE(package_name VARCHAR(1024), + load_from_file BOOLEAN DEFAULT TRUE); + PROCEDURE UPGRADE_ALL(load_from_file BOOLEAN DEFAULT TRUE); END; // diff --git a/src/sql/engine/basic/ob_ra_row_store.cpp b/src/sql/engine/basic/ob_ra_row_store.cpp index b7e06aa80..0b3469459 100644 --- a/src/sql/engine/basic/ob_ra_row_store.cpp +++ b/src/sql/engine/basic/ob_ra_row_store.cpp @@ -546,6 +546,15 @@ int ObRARowStore::switch_idx_block(bool finish_add /* = false */) if (!is_inited()) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); + } else if (finish_add && NULL == idx_blk_) { + if (OB_FAIL(build_idx_block())) { + LOG_WARN("build index block failed", + K(ret), K(finish_add), KPC(idx_blk_)); + } + } + + if (OB_FAIL(ret)) { + // do nothing } else if (NULL == idx_blk_) { ret = OB_ERR_UNEXPECTED; LOG_WARN("index block should not be null"); diff --git a/src/sql/ob_spi.cpp b/src/sql/ob_spi.cpp index 5520011d8..6008f054b 100644 --- a/src/sql/ob_spi.cpp +++ b/src/sql/ob_spi.cpp @@ -46,6 +46,7 @@ #include "pl/ob_pl_profiler.h" #endif #include "pl/ob_pl_allocator.h" +#include "pl/diagnosis/ob_pl_sql_audit_guard.h" namespace oceanbase { using namespace sqlclient; @@ -1589,50 +1590,6 @@ int ObSPIService::spi_end_trans(ObPLExecCtx *ctx, const char *sql, bool is_rollb return ret; } -class ObSPITimeRecord : public ObITimeRecord -{ -public: - ObSPITimeRecord() - : send_timestamp_(0), - receive_timestamp_(0), - enqueue_timestamp_(0), - run_timestamp_(0), - process_timestamp_(0), - single_process_timestamp_(0), - exec_start_timestamp_(0), - exec_end_timestamp_(0) {} - virtual ~ObSPITimeRecord() {} - - void set_send_timestamp(int64_t send_timestamp) { send_timestamp_ = send_timestamp; } - void set_receive_timestamp(int64_t receive_timestamp) { receive_timestamp_ = receive_timestamp; } - void set_enqueue_timestamp(int64_t enqueue_timestamp) { enqueue_timestamp_ = enqueue_timestamp; } - void set_run_timestamp(int64_t run_timestamp) { run_timestamp_ = run_timestamp; } - void set_process_timestamp(int64_t process_timestamp) { process_timestamp_ = process_timestamp; } - void set_single_process_timestamp(int64_t single_process_timestamp) { single_process_timestamp_ = single_process_timestamp; } - void set_exec_start_timestamp(int64_t exec_start_timestamp) { exec_start_timestamp_ = exec_start_timestamp; } - void set_exec_end_timestamp(int64_t exec_end_timestamp) { exec_end_timestamp_ = exec_end_timestamp; } - - int64_t get_send_timestamp() const { return send_timestamp_; } - int64_t get_receive_timestamp() const { return get_send_timestamp(); } - int64_t get_enqueue_timestamp() const { return get_send_timestamp(); } - int64_t get_run_timestamp() const { return get_send_timestamp(); } - int64_t get_process_timestamp() const { return get_send_timestamp(); } - int64_t get_single_process_timestamp() const { return get_send_timestamp(); } - int64_t get_exec_start_timestamp() const { return get_send_timestamp(); } - int64_t get_exec_end_timestamp() const { return exec_end_timestamp_; } - -public: - int64_t send_timestamp_; - int64_t receive_timestamp_; - int64_t enqueue_timestamp_; - int64_t run_timestamp_; - int64_t process_timestamp_; - int64_t single_process_timestamp_; - int64_t exec_start_timestamp_; - int64_t exec_end_timestamp_; - -}; - ObPLSPITraceIdGuard::ObPLSPITraceIdGuard(const ObString &sql, const ObString &ps_sql, ObSQLSessionInfo &session, @@ -1684,12 +1641,12 @@ ObPLSPITraceIdGuard::~ObPLSPITraceIdGuard() } } -//todo:@hr351303 确认sql 和 ps sql是否可以合一 +// common sql execute interface (static-sql & dynamic-sql & dbms-sql-execute) int ObSPIService::spi_inner_execute(ObPLExecCtx *ctx, const char *sql, const char *ps_sql, int64_t type, - const ObSqlExpression **param_exprs, + void *params, int64_t param_count, const ObSqlExpression **into_exprs, int64_t into_count, @@ -1700,246 +1657,167 @@ int ObSPIService::spi_inner_execute(ObPLExecCtx *ctx, int64_t is_bulk, bool is_forall, bool is_type_record, - bool for_update) + bool for_update, + bool is_dynamic_sql, + ObIArray *using_out_params, + bool is_dbms_sql) { int ret = OB_SUCCESS; - ObWarningBuffer* wb = NULL; - FLTSpanGuard(pl_spi_query); + CK (OB_NOT_NULL(ctx)); CK (OB_NOT_NULL(ctx->allocator_)); CK (OB_NOT_NULL(ctx->exec_ctx_)); CK (OB_NOT_NULL(ctx->exec_ctx_->get_my_session())); CK (OB_NOT_NULL(ctx->exec_ctx_->get_my_session()->get_pl_sqlcode_info())); - CK (OB_NOT_NULL(ctx->func_)); - OX (wb = common::ob_get_tsi_warning_buffer()); - ObExecRecord exec_record; - ObExecTimestamp exec_timestamp; - ObSPITimeRecord time_record; - exec_timestamp.exec_type_ = sql::PLSql; + if (OB_SUCC(ret)) { + HEAP_VAR(ObSPIResultSet, spi_result) { - ObSQLSessionInfo *session = ctx->exec_ctx_->get_my_session(); - ObPLSPITraceIdGuard trace_id_guard(sql, ps_sql, *session, ret); - ObPLSubPLSqlTimeGuard guard(ctx); - if (OB_SUCC(ret) && is_forall && !session->is_enable_batched_multi_statement()) { - /* forall need rollback to for loop */ - ret = OB_BATCHED_MULTI_STMT_ROLLBACK; - LOG_TRACE("cannot batch execute", K(ret), K(sql), K(type)); - } + ObSQLSessionInfo *session = ctx->exec_ctx_->get_my_session(); + stmt::StmtType stmt_type = static_cast(type); + ObString sqlstr(sql); - HEAP_VAR(ObSPIResultSet, spi_result) { - stmt::StmtType stmt_type = stmt::T_NONE; - bool is_diagnostics_stmt = false; - ObString sqlstr(sql); - OZ (spi_result.init(*session)); - OZ (spi_result.start_nested_stmt_if_need(ctx, sqlstr, static_cast(type), for_update)); + if (is_forall && !session->is_enable_batched_multi_statement()) { + /* forall need rollback to for loop */ + ret = OB_BATCHED_MULTI_STMT_ROLLBACK; + LOG_TRACE("cannot batch execute", K(ret), K(sql), K(type)); + } - if (OB_SUCC(ret)) { - int64_t row_count = 0; - ObQueryRetryCtrl retry_ctrl; - int64_t tenant_version = 0; - int64_t sys_version = 0; + OZ (spi_result.init(*session)); + OZ (spi_result.start_nested_stmt_if_need(ctx, sqlstr, stmt_type, for_update)); + OX (spi_result.get_sql_ctx().is_dbms_sql_ = is_dbms_sql); + + if (OB_SUCC(ret)) { + + ObPLSqlAuditRecord audit_record(sql::PLSql); + ObQueryRetryCtrl retry_ctrl; + ObSPIExecEnvGuard env_guard(*session, spi_result); + int save_sqlcode = session->get_pl_sqlcode_info()->get_sqlcode(); + ObString save_sqlmsg = session->get_pl_sqlcode_info()->get_sqlmsg(); - ObSPIOutParams out_params; - int64_t old_query_start_time = session->get_query_start_time(); - HEAP_VAR(ObPLSqlCodeInfo, saved_sqlcode_info) { - session->set_query_start_time(ObTimeUtility::current_time()); - session->init_use_rich_format(); - saved_sqlcode_info = *(ctx->exec_ctx_->get_my_session()->get_pl_sqlcode_info()); - bool is_retry = false; do { - // SQL_AUDIT_START - ObWaitEventDesc max_wait_desc; - ObWaitEventStat total_wait_desc; + + bool can_retry = true; + int64_t row_count = 0; ObArenaAllocator allocator(GET_PL_MOD_STRING(PL_MOD_IDX::OB_PL_STATIC_SQL_EXEC), OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); - const bool enable_perf_event = lib::is_diagnose_info_enabled(); - const bool enable_sql_audit = - GCONF.enable_sql_audit && ctx->exec_ctx_->get_my_session()->get_local_ob_enable_sql_audit(); { - ObMaxWaitGuard max_wait_guard(enable_perf_event ? &max_wait_desc : NULL); - ObTotalWaitGuard total_wait_guard(enable_perf_event ? &total_wait_desc : NULL); - if (enable_perf_event) { - exec_record.record_start(); + ObPLSPITraceIdGuard trace_id_guard(sql, ps_sql, *session, ret); + ObPLSubPLSqlTimeGuard guard(ctx); + ObPLSqlAuditGuard audit_guard( + *(ctx->exec_ctx_), *(session), spi_result, audit_record, ret, (sql != NULL ? sql : ps_sql), retry_ctrl, trace_id_guard, static_cast(type)); + ObSPIRetryCtrlGuard retry_guard(retry_ctrl, spi_result, *session, ret); + + OX (session->get_pl_sqlcode_info()->set_sqlcode(OB_SUCCESS)); + + if (OB_SUCC(ret) && !ObStmt::is_diagnostic_stmt(stmt_type) && lib::is_mysql_mode()) { + ob_reset_tsi_warning_buffer(); } - //监控项统计开始 - time_record.set_send_timestamp(ObTimeUtility::current_time()); - ctx->exec_ctx_->get_my_session()->get_pl_sqlcode_info()->set_sqlcode(OB_SUCCESS); - if (!ObStmt::is_diagnostic_stmt(static_cast(type)) - && lib::is_mysql_mode() - && OB_NOT_NULL(wb)) { - wb->reset(); + + OZ (inner_open(ctx, + allocator, + sql, + ps_sql, + type, + params, + param_count, + into_exprs, + into_count, + spi_result, + spi_result.get_out_params(), + is_forall, + is_dynamic_sql, + is_dbms_sql), K(sql), K(ps_sql)); + OZ (inner_fetch(ctx, + can_retry, + spi_result, + into_exprs, + into_count, + column_types, + type_count, + exprs_not_null_flag, + pl_integer_ranges, + OB_NOT_NULL(using_out_params) && !using_out_params->empty() ? using_out_params : NULL, + row_count, + is_bulk, + is_forall, + is_dynamic_sql, + NULL, + false, + false, + INT64_MAX, + NULL, + 0, + is_type_record), K(ps_sql), K(sql), K(type)); + + if (OB_SUCC(ret) && (is_dynamic_sql || is_dbms_sql) && !is_bulk) { + // if it is bulk into, not allow using out param, so no need deep copy + OZ (dynamic_out_params( + *(ctx->allocator_), spi_result.get_result_set(), params, param_count, is_dbms_sql)); } - row_count = 0; - out_params.reset(); - if (is_retry) { - spi_result.reset_member_for_retry(*session); + if (spi_result.get_out_params().has_out_param()) { + OZ (process_function_out_result(ctx, *spi_result.get_result_set(), spi_result.get_out_params().get_out_params())); } - retry_ctrl.clear_state_before_each_retry(session->get_retry_info_for_update()); - if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(session->get_effective_tenant_id(), spi_result.get_scheme_guard()))) { - LOG_WARN("get schema guard failed", K(ret)); - } else if (OB_FAIL(spi_result.get_scheme_guard().get_schema_version(session->get_effective_tenant_id(), tenant_version))) { - LOG_WARN("fail get schema version", K(ret)); - } else if (OB_FAIL(spi_result.get_scheme_guard().get_schema_version(OB_SYS_TENANT_ID, sys_version))) { - LOG_WARN("fail get sys schema version", K(ret)); - } else { - retry_ctrl.set_tenant_local_schema_version(tenant_version); - retry_ctrl.set_sys_local_schema_version(sys_version); - spi_result.get_sql_ctx().schema_guard_ = &spi_result.get_scheme_guard(); - if (OB_FAIL(inner_open(ctx, - allocator, - sql, - ps_sql, - type, - param_exprs, - param_count, - into_exprs, - into_count, - spi_result, - out_params, - &retry_ctrl, - is_forall))) { - LOG_WARN("failed to open", K(ret), K(sql), K(ps_sql)); - } else if (OB_FAIL(inner_fetch(ctx, - retry_ctrl, - spi_result, - into_exprs, - into_count, - column_types, - type_count, - exprs_not_null_flag, - pl_integer_ranges, - NULL, - row_count, - is_bulk, - is_forall, - false, - NULL, - false, - false, - INT64_MAX, - NULL, - 0, - is_type_record))) { - LOG_WARN("failed to execute inner_fetch for pl/sql", K(ret), K(ps_sql), K(sql), K(type)); - } else if (out_params.has_out_param()) { - OZ (process_function_out_result( - ctx, *spi_result.get_result_set(), out_params.get_out_params()), out_params); - } - if (OB_SUCC(ret)) { // 如果成功记录下stmttype,用于DDL语句的schema刷新 - OX (stmt_type = spi_result.get_result_set()->get_stmt_type()); - OX (is_diagnostics_stmt = ObStmt::is_diagnostic_stmt(spi_result.get_result_set()->get_literal_stmt_type())); - } - // 无论成功或者失败都在这里close result set - // 原因是close函数里面会设置audit的sched info信息, audit会在ObInnerSQLConnection::process_record中处理audit信息 - int close_ret = spi_result.close_result_set(); - if (OB_SUCCESS != close_ret) { - LOG_WARN("close spi result failed", K(ret), K(close_ret)); - } - ret = OB_SUCCESS == ret ? close_ret : ret; - spi_result.destruct_exec_params(*session); + + ObPLSqlCodeInfo *sqlcode_info = session->get_pl_sqlcode_info(); + if (sqlcode_info->get_sqlcode() == OB_SUCCESS) { + sqlcode_info->set_sqlcode(save_sqlcode, save_sqlmsg); } - //if (OB_SUCCESS == ret) { - if (session->get_pl_sqlcode_info()->get_sqlcode() == OB_SUCCESS) { - session->get_pl_sqlcode_info()->set_sqlcode( - saved_sqlcode_info.get_sqlcode(), saved_sqlcode_info.get_sqlmsg()); - } - //} else { - // session->get_pl_sqlcode_info()->set_sqlcode(ret); - //} - //监控项统计结束 - time_record.set_exec_end_timestamp(ObTimeUtility::current_time()); - if (enable_perf_event) { - exec_record.record_end(); + + if (can_retry) { + retry_guard.test(); } } - LOG_DEBUG("start process record", K(ret), K(ps_sql), K(sql), K(type), K(enable_sql_audit)); - // 处理监控统计项 - if (OB_NOT_NULL(spi_result.get_result_set())) { - if (spi_result.get_result_set()->is_inited()) { - ObSQLSessionInfo *session_info = ctx->exec_ctx_->get_my_session(); - ObString record_sql(sql != NULL ? sql : ps_sql); - if (ObStmt::is_execute_stmt(static_cast(type))) { - record_sql = session_info->get_current_query_string(); - } - int64_t try_cnt = session_info->get_raw_audit_record().try_cnt_; - ObExecRecord record_bk = session_info->get_raw_audit_record().exec_record_; - session_info->get_raw_audit_record().try_cnt_ = retry_ctrl.get_retry_times(); - session_info->get_raw_audit_record().pl_trace_id_.set(trace_id_guard.origin_trace_id_); - ObInnerSQLConnection::process_record(*spi_result.get_result_set(), - spi_result.get_sql_ctx(), - *session_info, - time_record, - ret, - session_info->get_current_execution_id(), // sql execute id - OB_INVALID_ID, - max_wait_desc, - total_wait_desc, - exec_record, - exec_timestamp, - true, - record_sql, - true, - spi_result.get_exec_params_str_ptr()); - session_info->get_raw_audit_record().exec_record_ = record_bk; - session_info->get_raw_audit_record().try_cnt_ = try_cnt; - session_info->get_raw_audit_record().pl_trace_id_.reset(); - } else { - LOG_DEBUG("result set is not inited, do not process record", - K(ret), K(ps_sql), K(sql), K(type)); - } - } else { - if (OB_SUCC(ret)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, result_set is null", K(ret), K(ps_sql), K(sql), K(type)); - } else { - LOG_WARN("result_set is null", K(ret), K(ps_sql), K(sql), K(type)); - } + + int close_ret = spi_result.close_result_set(); + if (OB_SUCCESS != close_ret) { + LOG_WARN("close spi result failed", K(ret), K(close_ret)); } - is_retry = true; + ret = OB_SUCCESS == ret ? close_ret : ret; + + spi_result.destruct_exec_params(*session); + } while (RETRY_TYPE_NONE != retry_ctrl.get_retry_type()); //SPI只做LOCAL重试 } - session->get_retry_info_for_update().clear(); - session->set_query_start_time(old_query_start_time); - session->init_use_rich_format(); - } - //自动提交,禁掉PL整体重试 - if (OB_SUCC(ret) - && (ObStmt::is_ddl_stmt(stmt_type, true) - || ObStmt::is_tcl_stmt(stmt_type) - || (lib::is_mysql_mode() && session->get_local_autocommit()))) { - OX (session->set_pl_can_retry(false)); - } - if (OB_SUCC(ret) - && OB_ISNULL(param_exprs)) { - if (ObStmt::is_ddl_stmt(stmt_type, true) || ObStmt::is_savepoint_stmt(stmt_type)) { - if (ObStmt::is_ddl_stmt(stmt_type, true)) { - OZ (force_refresh_schema(session->get_effective_tenant_id()), sql); - } - recreate_implicit_savapoint_if_need(ctx, ret); - } - } - // 记录第一条SQL执行PartitionHit信息, 并对PartitionHit进行Freeze, 防止后续的SQL冲掉 - if (OB_SUCC(ret)) { - if (OB_NOT_NULL(spi_result.get_result_set()->get_physical_plan())) { - session->partition_hit().freeze(); - } - } - spi_result.end_nested_stmt_if_need(ctx, ret); - if (OB_FAIL(ret) && !is_diagnostics_stmt) { - // support `SHOW WARNINGS` in mysql PL - session->set_show_warnings_buf(ret); + if (OB_SUCC(ret) + && (ObStmt::is_ddl_stmt(stmt_type, true) + || ObStmt::is_tcl_stmt(stmt_type) + || (lib::is_mysql_mode() && session->get_local_autocommit()))) { + OX (session->set_pl_can_retry(false)); + } + + if (OB_SUCC(ret) && 0 == param_count) { + if (ObStmt::is_ddl_stmt(stmt_type, true) + || ObStmt::is_tcl_stmt(stmt_type) + || ObStmt::is_savepoint_stmt(stmt_type)) { + if (ObStmt::is_ddl_stmt(stmt_type, true)) { + OZ (force_refresh_schema(session->get_effective_tenant_id()), sql); + } + recreate_implicit_savapoint_if_need(ctx, ret); + } + } + + // 记录第一条SQL执行PartitionHit信息, 并对PartitionHit进行Freeze, 防止后续的SQL冲掉 + if (OB_SUCC(ret)) { + if (OB_NOT_NULL(spi_result.get_result_set()->get_physical_plan())) { + session->partition_hit().freeze(); + } + } + + if (OB_FAIL(ret) && !ObStmt::is_diagnostic_stmt(stmt_type)) { + // support `SHOW WARNINGS` in mysql PL + session->set_show_warnings_buf(ret); + } + + spi_result.end_nested_stmt_if_need(ctx, ret); + + SET_FORALL_BULK_EXCEPTION; + SET_SPI_STATUS; } - SET_FORALL_BULK_EXCEPTION; - SET_SPI_STATUS; } return ret; } -/* - * 本接口是在spi_inner_execute基础上去掉部分代码实现的，因为去掉的代码包括inner_fetch这个关键接口， - * 所以暂时还不方便将两个接口整理重构为一个接口，计划在dbms_sql支持读流程接口和returning语句后再重构。 - */ int ObSPIService::dbms_cursor_execute(ObPLExecCtx *ctx, const ObString ps_sql, stmt::StmtType stmt_type, @@ -1947,213 +1825,25 @@ int ObSPIService::dbms_cursor_execute(ObPLExecCtx *ctx, bool is_dbms_sql) { int ret = OB_SUCCESS; - ObSQLSessionInfo *session = NULL; - OV (OB_NOT_NULL(ctx)); - OV (OB_NOT_NULL(ctx->allocator_)); - OV (OB_NOT_NULL(ctx->exec_ctx_)); - OV (OB_NOT_NULL(ctx->exec_ctx_->get_my_session())); - OX (session = ctx->exec_ctx_->get_my_session()); - OV (OB_NOT_NULL(session->get_pl_sqlcode_info())); - // dbms_sql场景暂时不检查这个 - // OV (OB_NOT_NULL(pl_ctx->func_)); - - ObExecRecord exec_record; - ObExecTimestamp exec_timestamp; - ObSPITimeRecord time_record; - ObString &sql_stmt = cursor.get_sql_stmt(); - ParamStore &exec_params = cursor.get_exec_params(); - exec_timestamp.exec_type_ = sql::PLSql; - - HEAP_VAR(ObSPIResultSet, spi_result) { - OZ (spi_result.init(*session)); - OZ (spi_result.start_nested_stmt_if_need(ctx, sql_stmt, static_cast(stmt_type), cursor.is_for_update())); - OX (spi_result.get_sql_ctx().is_dbms_sql_ = is_dbms_sql); - if (OB_SUCC(ret)) { - int64_t row_count = 0; - ObQueryRetryCtrl retry_ctrl; - int64_t tenant_version = 0; - int64_t sys_version = 0; - ObSPIOutParams out_params; - ObPLSqlCodeInfo saved_sqlcode_info; - uint64_t eff_tenant_id = session->get_effective_tenant_id(); - int64_t old_query_start_time = session->get_query_start_time(); - session->set_query_start_time(ObTimeUtility::current_time()); - bool is_retry = false; - do { - // SQL_AUDIT_START] - ObWaitEventDesc max_wait_desc; - ObWaitEventStat total_wait_desc; - const bool enable_perf_event = lib::is_diagnose_info_enabled(); - const bool enable_sql_audit = - GCONF.enable_sql_audit && session->get_local_ob_enable_sql_audit(); - { - ObMaxWaitGuard max_wait_guard(enable_perf_event ? &max_wait_desc : NULL); - ObTotalWaitGuard total_wait_guard(enable_perf_event ? &total_wait_desc : NULL); - if (enable_perf_event) { - exec_record.record_start(); - } - // 监控项统计开始 - time_record.set_send_timestamp(ObTimeUtility::current_time()); - saved_sqlcode_info = *(session->get_pl_sqlcode_info()); - session->get_pl_sqlcode_info()->set_sqlcode(OB_SUCCESS); - row_count = 0; - out_params.reset(); - if (is_retry) { - spi_result.reset_member_for_retry(*session); - } - retry_ctrl.clear_state_before_each_retry(session->get_retry_info_for_update()); - if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(eff_tenant_id, spi_result.get_scheme_guard()))) { - LOG_WARN("get schema guard failed", K(ret)); - } else if (OB_FAIL(spi_result.get_scheme_guard().get_schema_version(eff_tenant_id, tenant_version))) { - LOG_WARN("fail get schema version", K(ret)); - } else if (OB_FAIL(spi_result.get_scheme_guard().get_schema_version(OB_SYS_TENANT_ID, sys_version))) { - LOG_WARN("fail get sys schema version", K(ret)); - } else { - retry_ctrl.set_tenant_local_schema_version(tenant_version); - retry_ctrl.set_sys_local_schema_version(sys_version); - spi_result.get_sql_ctx().schema_guard_ = &spi_result.get_scheme_guard(); - if (OB_FAIL(inner_open(ctx, - exec_params.count() > 0 ? NULL : sql_stmt.ptr(), - ps_sql, - stmt_type, - exec_params, - spi_result, - out_params))) { - LOG_WARN("failed to open", K(ret), K(ps_sql), K(stmt_type), - K(exec_params), K(sql_stmt), K(sql_stmt.ptr()), K(sql_stmt.length())); - if (spi_result.get_result_set() != NULL) { - int cli_ret = OB_SUCCESS; - retry_ctrl.test_and_save_retry_state( - GCTX, - spi_result.get_sql_ctx(), - *spi_result.get_result_set(), - ret, cli_ret, true, true, true); - LOG_WARN("failed to get_result, check if need retry", - K(ret), K(cli_ret), K(retry_ctrl.need_retry())); - ret = cli_ret; - spi_result.get_sql_ctx().clear(); - ctx->exec_ctx_->get_my_session()->set_session_in_retry(retry_ctrl.need_retry()); - } - } /* else if (OB_FAIL(inner_fetch(ctx, - retry_ctrl, - spi_result.get_mysql_result().get_result(), - NULL, - 0, - NULL, - 0, - NULL, - NULL, - NULL, - row_count))) { - LOG_WARN("failed to get result", K(ret)); - } */ else if (exec_params.count() > 0) { // process out parameters - ObInnerSQLResult *inner_result = NULL; - ObPhysicalPlanCtx *plan_ctx = NULL; - const ParamStore *param_store = NULL; - CK (OB_NOT_NULL(plan_ctx = spi_result.get_result_set()->get_exec_context().get_physical_plan_ctx())); - CK (OB_NOT_NULL(param_store = &(plan_ctx->get_param_store()))); - OV (param_store->count() >= exec_params.count(), OB_ERR_UNEXPECTED, param_store->count()); - for (int64_t i = 0; OB_SUCC(ret) && i < exec_params.count(); ++i) { - OZ (deep_copy_obj(*(ctx->allocator_), param_store->at(i), exec_params.at(i))); - } - } - if (OB_SUCC(ret) && out_params.has_out_param()) { - OZ (process_function_out_result( - ctx, *spi_result.get_result_set(), out_params.get_out_params()), out_params); - } -// TODO: 先注掉，因为目前看参数传进来的stmt_type已经是有效的，不需要再生成。 -// if (OB_SUCC(ret)) { // 如果成功记录下stmttype,用于DDL语句的schema刷新 -// ObInnerSQLResult *inner_result = static_cast(spi_result.get_mysql_result().get_result()); -// CK (OB_NOT_NULL(inner_result)); -// OX (stmt_type = inner_result->result_set().get_stmt_type()); -// } - // 无论成功或者失败都在这里close result set - // 原因是close函数里面会设置audit的sched info信息, audit会在ObInnerSQLConnection::process_record中处理audit信息 - int close_ret = spi_result.close_result_set(); - if (OB_SUCCESS != close_ret) { - LOG_WARN("close spi result failed", K(ret), K(close_ret)); - } - ret = OB_SUCCESS == ret ? close_ret : ret; - } - if (session->get_pl_sqlcode_info()->get_sqlcode() == OB_SUCCESS) { - session->get_pl_sqlcode_info()->set_sqlcode( - saved_sqlcode_info.get_sqlcode(), saved_sqlcode_info.get_sqlmsg()); - } - //} else { - // session->get_pl_sqlcode_info()->set_sqlcode(ret); - //} - //监控项统计结束 - time_record.set_exec_end_timestamp(ObTimeUtility::current_time()); - if (enable_perf_event) { - exec_record.record_end(); - } - } - LOG_DEBUG("start process record", - K(ret), K(sql_stmt), K(stmt_type), K(enable_sql_audit)); - // 处理监控统计项 - if (OB_NOT_NULL(spi_result.get_result_set())) { - if (spi_result.get_result_set()->is_inited()) { - ObInnerSQLConnection::process_record(*spi_result.get_result_set(), - spi_result.get_sql_ctx(), - *session, - time_record, - ret, - session->get_current_execution_id(), - OB_INVALID_ID, //FIXME@hr351303 - max_wait_desc, - total_wait_desc, - exec_record, - exec_timestamp, - true, - ps_sql, - true, - spi_result.get_exec_params_str_ptr()); - } else { - LOG_DEBUG("result set is not inited, do not process record", - K(ret), K(ps_sql), K(sql_stmt), K(stmt_type)); - } - } else { - if (OB_SUCC(ret)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, result_set is null", - K(ret), K(ps_sql), K(sql_stmt), K(stmt_type)); - } else { - LOG_WARN("result_set is null", - K(ret), K(ps_sql), K(sql_stmt), K(stmt_type)); - } - } - is_retry = true; - } while (RETRY_TYPE_NONE != retry_ctrl.get_retry_type()); //SPI只做LOCAL重试 - session->get_retry_info_for_update().clear(); - session->set_query_start_time(old_query_start_time); - } - //自动提交,禁掉PL整体重试 - if (OB_SUCC(ret) - && (ObStmt::is_ddl_stmt(stmt_type, true) - || ObStmt::is_tcl_stmt(stmt_type) - || (lib::is_mysql_mode() && session->get_local_autocommit()))) { - OX (session->set_pl_can_retry(false)); - } - if (OB_SUCC(ret) && exec_params.empty()) { - if (ObStmt::is_ddl_stmt(stmt_type, true) || ObStmt::is_savepoint_stmt(stmt_type)) { - if (ObStmt::is_ddl_stmt(stmt_type, true)) { - OZ (force_refresh_schema(session->get_effective_tenant_id()), sql_stmt); - } - recreate_implicit_savapoint_if_need(ctx, ret); - } - } - // 记录第一条SQL执行PartitionHit信息, 并对PartitionHit进行Freeze, 防止后续的SQL冲掉 - if (OB_SUCC(ret)) { - if (OB_NOT_NULL(spi_result.get_result_set()->get_physical_plan())) { - session->partition_hit().freeze(); - } - } - - spi_result.end_nested_stmt_if_need(ctx, ret); - - SET_FORALL_BULK_EXCEPTION; - SET_SPI_STATUS; - } + OZ (spi_inner_execute(ctx, + cursor.get_sql_stmt().ptr(), + ps_sql.ptr(), + stmt_type, + &cursor.get_exec_params(), + cursor.get_exec_params().count(), + nullptr, /*into_exprs*/ + 0, /*into_count*/ + nullptr, /*column_types*/ + 0,/*type_count*/ + nullptr, /*exprs_not_null_flag*/ + nullptr, /*pl_integer_ranges*/ + false, /*is_bulk*/ + false, /*is_forall*/ + false, /*is_type_record*/ + false, /*for_update*/ + false, /*is_dynamic_sql*/ + nullptr, /*using_out_params*/ + true/*is_dbms_sql*/)); return ret; } @@ -2798,7 +2488,7 @@ int ObSPIService::prepare_dynamic(ObPLExecCtx *ctx, } else { remove_into = true; } - } else if (stmt::T_SELECT == stmt_type) { + } else if (stmt::T_SELECT == stmt_type || pl_prepare_result.result_set_->is_returning()) { /* * 动态语句如果是select into，INTO子句会被忽略掉，INTO子句里面占位符的也不需要绑定实参 * 例如： @@ -2834,8 +2524,7 @@ int ObSPIService::prepare_dynamic(ObPLExecCtx *ctx, } int ObSPIService::dynamic_out_params( - common::ObIAllocator &allocator, - ObResultSet *result_set, common::ObObjParam **params, int64_t param_count) + common::ObIAllocator &allocator, ObResultSet *result_set, void* params, int64_t param_count, bool is_dbms_sql) { int ret = OB_SUCCESS; ObPhysicalPlanCtx *plan_ctx = NULL; @@ -2849,20 +2538,172 @@ int ObSPIService::dynamic_out_params( OV (param_store->count() >= param_count, OB_ERR_UNEXPECTED, param_store->count(), param_count); for (int64_t i = 0; OB_SUCC(ret) && i < param_count; ++i) { - // param_store`memory from result set will released by result close. + // memory of params_store from result set will released by result close. // so we need to deep copy it. + ObObjParam &obj = is_dbms_sql ? reinterpret_cast(params)->at(i) + : *(reinterpret_cast(params)[i]); if (param_store->at(i).is_pl_extend() && param_store->at(i).get_meta().get_extend_type() != PL_CURSOR_TYPE && param_store->at(i).get_meta().get_extend_type() != PL_REF_CURSOR_TYPE) { - OZ (pl::ObUserDefinedType::deep_copy_obj(allocator, param_store->at(i), *params[i], true)); + OZ (pl::ObUserDefinedType::deep_copy_obj(allocator, param_store->at(i), obj, true)); } else { - OZ (deep_copy_obj(allocator, param_store->at(i), *params[i])); + OZ (deep_copy_obj(allocator, param_store->at(i), obj)); } } } return ret; } + +int ObSPIService::check_dynamic_sql_legal(ObPLExecCtx *ctx, + ObSqlString &sql_str, + stmt::StmtType stmt_type, + int64_t into_count, + int64_t inner_into_cnt, + common::ObObjParam **params, + int64_t param_count, + const int64_t *params_mode, + bool is_returning, + bool for_update, + int64_t &exec_param_cnt, + ObIArray &out_using_params) +{ + int ret = OB_SUCCESS; + if (OB_SUCC(ret)) { + if (ObStmt::is_ddl_stmt(stmt_type, false) + && (into_count > 0 || param_count > 0 || is_returning)) { + ret = OB_ERR_DDL_IN_ILLEGAL_CONTEXT; + LOG_WARN("DDL statement is executed in an illegal context", + K(ret), K(into_count), K(param_count), K(is_returning)); + } else if (ObStmt::is_dml_write_stmt(stmt_type) && inner_into_cnt > 0 && 0 == into_count) { + /* + * 处理 + * 仅当dml语句含returning变量，并且外部没有INTO变量时才允许使用USING OUT接收参数 + */ + CK (param_count >= inner_into_cnt); + OX (exec_param_cnt = param_count - inner_into_cnt); + for (int64_t i = exec_param_cnt; OB_SUCC(ret) && i < param_count; ++i) { + ObPLRoutineParamMode pm = static_cast(params_mode[i]); + if (PL_PARAM_INOUT == pm || PL_PARAM_OUT == pm) { + OZ (out_using_params.push_back(params[i])); + } else { + ret = OB_ERR_INOUT_PARAM_PLACEMENT_NOT_PROPERLY; + LOG_WARN("OBE-06536: IN bind variable bound to an OUT position", K(ret)); + } + } + } else if (ObStmt::is_dml_write_stmt(stmt_type) && inner_into_cnt > 0 && into_count > 0 && !is_returning) { + CK (param_count >= inner_into_cnt); + for (int64_t i = param_count - inner_into_cnt; OB_SUCC(ret) && i < param_count; ++i) { + ObPLRoutineParamMode pm = static_cast(params_mode[i]); + if (PL_PARAM_IN == pm) { + ret = OB_ERR_INOUT_PARAM_PLACEMENT_NOT_PROPERLY; + LOG_WARN("OBE-06536: IN bind variable bound to an OUT position", K(ret)); + } + } + } else { /*do nothing*/ } + } + + for (int i = 0; OB_SUCC(ret) && i < param_count; ++i) { + ObPLRoutineParamMode pm = static_cast(params_mode[i]); + if (ObStmt::is_select_stmt(stmt_type) && (PL_PARAM_INOUT == pm || PL_PARAM_OUT == pm)) { + ret = OB_ERR_INOUT_PARAM_PLACEMENT_NOT_PROPERLY; + LOG_WARN("select stmt with using out/inout param mode is not allowed", K(ret)); + } + if (ObStmt::is_dml_write_stmt(stmt_type) + && i < param_count - inner_into_cnt + && PL_PARAM_OUT == pm) { + ret = OB_ERR_INOUT_PARAM_PLACEMENT_NOT_PROPERLY; + LOG_WARN("using out/inout param mode is not allowed", K(ret)); + } else if (ObStmt::is_dml_write_stmt(stmt_type) && + into_count > 0 && !is_returning && + (PL_PARAM_INOUT == pm || PL_PARAM_OUT == pm)) { + ret = OB_ERR_INOUT_PARAM_PLACEMENT_NOT_PROPERLY; + LOG_WARN("using out/inout param mode is not allowed", K(ret)); + } + if (OB_SUCC(ret) && ObStmt::is_dml_stmt(stmt_type)) { + if (PL_PARAM_IN == pm && + NULL != params[i] && + params[i]->is_pl_extend() && + PL_RECORD_TYPE == params[i]->get_meta().get_extend_type()) { + const ObUserDefinedType *user_type = NULL; + ObPLComposite *composite = reinterpret_cast(params[i]->get_ext()); + CK (OB_NOT_NULL(composite)); + OZ (ctx->get_user_type(composite->get_id(), user_type)); + CK (OB_NOT_NULL(user_type)); + if (OB_SUCC(ret) && user_type->is_type_record()) { + ret = OB_ERR_EXPR_SQL_TYPE; + LOG_WARN("expressions have to be of SQL types", K(ret)); + } + } + } + } + + if (OB_SUCC(ret)) { + ObParser parser(*ctx->allocator_, STD_MODE); + ObMPParseStat parse_stat; + ObSEArray queries; + OZ (parser.split_multiple_stmt(sql_str.string(), queries, parse_stat)); + if (OB_SUCC(ret) && queries.count() > 1) { + ret = OB_ERR_CMD_NOT_PROPERLY_ENDED; + LOG_WARN("execute immdeidate only support one stmt", K(ret)); + } + } + return ret; +} + +int ObSPIService::prepare_dynamic_sql_params(ObPLExecCtx *ctx, + ObSPIResultSet &spi_result, + ObIAllocator &allocator, + int64_t exec_param_cnt, + ObObjParam **params, + ParamStore *&exec_params) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(exec_params = reinterpret_cast(allocator.alloc(sizeof(ParamStore))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory for exec params", K(ret)); + } else { + new (exec_params) ParamStore((ObWrapperAllocator(allocator))); + } + for (int64_t i = 0; OB_SUCC(ret) && i < exec_param_cnt; ++i) { + CK (OB_NOT_NULL(params[i])); + if (OB_SUCC(ret)) { + ObObjParam new_param = *params[i]; + if (params[i]->is_pl_extend()) { + if (params[i]->get_meta().get_extend_type() != PL_REF_CURSOR_TYPE) { + new_param.set_int_value(0); + if (!ob_is_xml_pl_type(params[i]->get_type(), params[i]->get_udt_id())) { + OZ (pl::ObUserDefinedType::deep_copy_obj(allocator, *params[i], new_param, true)); + } else { + uint64_t udt_id = params[i]->get_udt_id(); + const ObDataTypeCastParams dtc_params = sql::ObBasicSessionInfo::create_dtc_params(ctx->exec_ctx_->get_my_session()); + ObCastCtx cast_ctx(ctx->allocator_, &dtc_params, CM_NONE, ObCharset::get_system_collation()); + cast_ctx.exec_ctx_ = ctx->exec_ctx_; + uint16_t subschema_id = ObInvalidSqlType; + if (OB_FAIL(cast_ctx.exec_ctx_->get_subschema_id_by_udt_id(udt_id, + subschema_id, + &spi_result.get_scheme_guard()))) { + LOG_WARN("failed to get subschema id by udt_id", K(ret), K(params[i])); + } else if (FALSE_IT(new_param.set_subschema_id(subschema_id))) { + } else if (OB_FAIL(ObObjCaster::to_type(ObUserDefinedSQLType, cast_ctx, *params[i], new_param))) { + LOG_WARN("failed to_type", K(ret), K(new_param)); + } else { // sql udt params need original udt id for plan choose + new_param.set_udt_id(udt_id); + new_param.set_param_meta(); // param meta also changed + } + } + } + } else { + OZ (deep_copy_obj(allocator, *params[i], new_param)); + } + OX (new_param.set_need_to_check_type(true)); + OZ (exec_params->push_back(new_param), K(new_param)); + } + } + OZ (store_params_string(ctx, spi_result, exec_params)); + return ret; +} + int ObSPIService::spi_execute_immediate(ObPLExecCtx *ctx, const int64_t sql_idx, common::ObObjParam **params, @@ -2879,392 +2720,100 @@ int ObSPIService::spi_execute_immediate(ObPLExecCtx *ctx, bool is_type_record) { int ret = OB_SUCCESS; - // HEAP_VAR(char[OB_MAX_SQL_LENGTH], sql_buffer) { - bool need_execute_sql = true; - ObSQLSessionInfo *session = NULL; - ObMySQLProxy *sql_proxy = NULL; - const ObSqlExpression *sql = nullptr; - const ObSqlExpression **into_exprs = nullptr; - ObSqlString sql_str; - ObArenaAllocator allocator(GET_PL_MOD_STRING(PL_MOD_IDX::OB_PL_DYNAMIC_SQL_EXEC), OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); - ObPLSubPLSqlTimeGuard guard(ctx); - HEAP_VAR(ObSPIResultSet, spi_result) { - stmt::StmtType stmt_type = stmt::T_NONE; - ObString ps_sql; - bool for_update = false; - bool hidden_rowid = false; - bool skip_locked = false; - ObQueryRetryCtrl retry_ctrl; - int64_t tenant_version = 0; - int64_t sys_version = 0; - share::schema::ObSchemaGetterGuard schema_guard; - int64_t inner_into_cnt = 0; //动态语句里into子句的变量个数 - ObArray out_using_params; - int64_t exec_param_cnt = param_count; + ObSQLSessionInfo *session = NULL; + ObSqlString sql_str; + const ObSqlExpression *sql = nullptr; + const ObSqlExpression **into_exprs = nullptr; - stmt::StmtType saved_stmt_type = stmt::T_NONE; - CK (OB_NOT_NULL(ctx), ctx->valid()); - CK (OB_NOT_NULL(ctx->func_)); - CK ((OB_NOT_NULL(params) && param_count > 0) || (OB_ISNULL(params) && 0 == param_count)); - CK (OB_NOT_NULL(session = ctx->exec_ctx_->get_my_session())); - CK (OB_NOT_NULL(sql_proxy = ctx->exec_ctx_->get_sql_proxy())); - CK (sql_idx != OB_INVALID_ID); - CK (OB_LIKELY(0 <= sql_idx && sql_idx < ctx->func_->get_expressions().count())); - OX (sql = ctx->func_->get_expressions().at(sql_idx)); - CK (OB_NOT_NULL(sql)); - CK (OB_NOT_NULL(GCTX.sql_engine_)); - OZ(spi_result.init(*session)); - OX(saved_stmt_type = session->get_stmt_type()); - MAKE_EXPR_BUFFER(allocator, into_exprs_idx, into_count, into_exprs); - // Step1: Prepare dynamic SQL! Only prepare once! - OZ (prepare_dynamic(ctx, - sql, - allocator, - is_returning, - param_count, - sql_str, - ps_sql, - stmt_type, - for_update, - hidden_rowid, - inner_into_cnt, - skip_locked)); - if (OB_SUCC(ret)) { - if (ObStmt::is_ddl_stmt(stmt_type, false) - && (into_count > 0 || param_count > 0 || is_returning)) { - ret = OB_ERR_DDL_IN_ILLEGAL_CONTEXT; - LOG_WARN("DDL statement is executed in an illegal context", - K(ret), K(into_count), K(param_count), K(is_returning)); - } else if (ObStmt::is_select_stmt(stmt_type) && !for_update && 0 >= into_count) { - /* If dynamic_sql_statement is a SELECT statement, and you omit both - * into_clause and bulk_collect_into_clause, then - * execute_immediate_statement never executes. - * For example, this statement never increments the sequence: - * EXECUTE IMMEDIATE 'SELECT S.NEXTVAL FROM DUAL' - */ - need_execute_sql = false; - } else if (ObStmt::is_dml_write_stmt(stmt_type) && inner_into_cnt > 0 && 0 == into_count) { - /* - * 处理 - * 仅当dml语句含returning变量，并且外部没有INTO变量时才允许使用USING OUT接收参数 - */ - CK (param_count >= inner_into_cnt); - OX (exec_param_cnt = param_count - inner_into_cnt); - for (int64_t i = exec_param_cnt; OB_SUCC(ret) && i < param_count; ++i) { - ObPLRoutineParamMode pm = static_cast(params_mode[i]); - if (PL_PARAM_INOUT == pm || PL_PARAM_OUT == pm) { - OZ (out_using_params.push_back(params[i])); - } else { - ret = OB_ERR_INOUT_PARAM_PLACEMENT_NOT_PROPERLY; - LOG_WARN("OBE-06536: IN bind variable bound to an OUT position", K(ret)); - } - } - } else if (ObStmt::is_dml_write_stmt(stmt_type) && inner_into_cnt > 0 && into_count > 0 && !is_returning) { - CK (param_count >= inner_into_cnt); - for (int64_t i = param_count - inner_into_cnt; OB_SUCC(ret) && i < param_count; ++i) { - ObPLRoutineParamMode pm = static_cast(params_mode[i]); - if (PL_PARAM_IN == pm) { - ret = OB_ERR_INOUT_PARAM_PLACEMENT_NOT_PROPERLY; - LOG_WARN("OBE-06536: IN bind variable bound to an OUT position", K(ret)); - } - } - } else { /*do nothing*/ } - } - for (int i = 0; OB_SUCC(ret) && i < param_count; ++i) { - ObPLRoutineParamMode pm = static_cast(params_mode[i]); - if (ObStmt::is_select_stmt(stmt_type) && (PL_PARAM_INOUT == pm || PL_PARAM_OUT == pm)) { - ret = OB_ERR_INOUT_PARAM_PLACEMENT_NOT_PROPERLY; - LOG_WARN("select stmt with using out/inout param mode is not allowed", K(ret)); - } - if (ObStmt::is_dml_write_stmt(stmt_type) - && i < param_count - inner_into_cnt - && PL_PARAM_OUT == pm) { - ret = OB_ERR_INOUT_PARAM_PLACEMENT_NOT_PROPERLY; - LOG_WARN("using out/inout param mode is not allowed", K(ret)); - } else if (ObStmt::is_dml_write_stmt(stmt_type) && - into_count > 0 && !is_returning && - (PL_PARAM_INOUT == pm || PL_PARAM_OUT == pm)) { - ret = OB_ERR_INOUT_PARAM_PLACEMENT_NOT_PROPERLY; - LOG_WARN("using out/inout param mode is not allowed", K(ret)); - } - if (OB_SUCC(ret) && ObStmt::is_dml_stmt(stmt_type)) { - if (PL_PARAM_IN == pm && - NULL != params[i] && - params[i]->is_pl_extend() && - PL_RECORD_TYPE == params[i]->get_meta().get_extend_type()) { - const ObUserDefinedType *user_type = NULL; - ObPLComposite *composite = reinterpret_cast(params[i]->get_ext()); - CK (OB_NOT_NULL(composite)); - OZ (ctx->get_user_type(composite->get_id(), user_type)); - CK (OB_NOT_NULL(user_type)); - if (OB_SUCC(ret) && user_type->is_type_record()) { - ret = OB_ERR_EXPR_SQL_TYPE; - LOG_WARN("expressions have to be of SQL types", K(ret)); - } - } - } - } + ObArenaAllocator allocator(GET_PL_MOD_STRING(PL_MOD_IDX::OB_PL_DYNAMIC_SQL_EXEC), OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); - if (OB_SUCC(ret)) { - ObParser parser(*ctx->allocator_, STD_MODE); - ObMPParseStat parse_stat; - ObSEArray queries; - OZ (parser.split_multiple_stmt(sql_str.string(), queries, parse_stat)); - if (OB_SUCC(ret) && queries.count() > 1) { - ret = OB_ERR_CMD_NOT_PROPERLY_ENDED; - LOG_WARN("execute immdeidate only support one stmt", K(ret)); - } - } - OX (session->set_stmt_type(saved_stmt_type)); - OZ (spi_result.start_nested_stmt_if_need(ctx, sql_str.string(), stmt_type, for_update)); + stmt::StmtType stmt_type = stmt::T_NONE; + ObString ps_sql; + bool for_update = false; + bool hidden_rowid = false; + bool skip_locked = false; + int64_t inner_into_cnt = 0; //动态语句里into子句的变量个数 + ObArray out_using_params; + int64_t exec_param_cnt = param_count; - // Step2: execute dynamic SQL now! - ObPLSPITraceIdGuard trace_id_guard(sql_str.string(), ps_sql, *session, ret); - if (OB_FAIL(ret)) { - } else if (need_execute_sql) { - ObExecRecord exec_record; - ObExecTimestamp exec_timestamp; - ObSPITimeRecord time_record; - exec_timestamp.exec_type_ = sql::PLSql; + stmt::StmtType saved_stmt_type = stmt::T_NONE; + CK (OB_NOT_NULL(ctx), ctx->valid()); + CK (OB_NOT_NULL(ctx->func_)); + CK ((OB_NOT_NULL(params) && param_count > 0) || (OB_ISNULL(params) && 0 == param_count)); + CK (OB_NOT_NULL(session = ctx->exec_ctx_->get_my_session())); + CK (sql_idx != OB_INVALID_ID); + CK (OB_LIKELY(0 <= sql_idx && sql_idx < ctx->func_->get_expressions().count())); + OX (sql = ctx->func_->get_expressions().at(sql_idx)); + CK (OB_NOT_NULL(sql)); + CK (OB_NOT_NULL(GCTX.sql_engine_)); + OX (saved_stmt_type = session->get_stmt_type()); - int64_t old_query_start_time = session->get_query_start_time(); - session->set_query_start_time(ObTimeUtility::current_time()); - bool is_retry = false; - do { - ObArenaAllocator allocator(GET_PL_MOD_STRING(PL_MOD_IDX::OB_PL_DYNAMIC_SQL_EXEC), OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); - ParamStore exec_params( (ObWrapperAllocator(allocator)) ); - ObWaitEventDesc max_wait_desc; - ObWaitEventStat total_wait_desc; - const bool enable_perf_event = lib::is_diagnose_info_enabled(); - const bool enable_sql_audit = - GCONF.enable_sql_audit && session->get_local_ob_enable_sql_audit(); - { - ObMaxWaitGuard max_wait_guard(enable_perf_event ? &max_wait_desc : NULL); - ObTotalWaitGuard total_wait_guard(enable_perf_event ? &total_wait_desc : NULL); - if (enable_perf_event) { - exec_record.record_start(); - } - //监控项统计开始 - time_record.set_send_timestamp(ObTimeUtility::current_time()); + MAKE_EXPR_BUFFER(allocator, into_exprs_idx, into_count, into_exprs); - ret = OB_SUCCESS; - if (is_retry) { - spi_result.reset_member_for_retry(*session); - } - retry_ctrl.clear_state_before_each_retry(session->get_retry_info_for_update()); - OZ (GCTX.schema_service_->get_tenant_schema_guard(session->get_effective_tenant_id(), - spi_result.get_scheme_guard())); - OZ (spi_result.get_scheme_guard().get_schema_version(session->get_effective_tenant_id(), tenant_version)); - OZ (spi_result.get_scheme_guard().get_schema_version(OB_SYS_TENANT_ID, sys_version)); + // Step: Prepare dynamic SQL! Only prepare once! + OZ (prepare_dynamic(ctx, + sql, + allocator, + is_returning, + param_count, + sql_str, + ps_sql, + stmt_type, + for_update, + hidden_rowid, + inner_into_cnt, + skip_locked)); - OX (retry_ctrl.set_tenant_local_schema_version(tenant_version)); - OX (retry_ctrl.set_sys_local_schema_version(sys_version)); - OX (spi_result.get_sql_ctx().schema_guard_ = &spi_result.get_scheme_guard()); + OZ (check_dynamic_sql_legal(ctx, + sql_str, + stmt_type, + into_count, + inner_into_cnt, + params, + param_count, + params_mode, + is_returning, + for_update, + exec_param_cnt, + out_using_params)); - bool old_client_return_rowid = session->is_client_return_rowid(); - bool is_inner_session = session->is_inner(); - ObSQLSessionInfo::SessionType old_session_type = session->get_session_type(); - !is_inner_session ? session->set_inner_session() : (void)NULL; - session->set_session_type(ObSQLSessionInfo::USER_SESSION); - if (NULL != ctx->pl_ctx_) { - session->set_client_return_rowid(false); - } - if (OB_SUCC(ret)) { - WITH_CONTEXT(spi_result.get_memory_ctx()) { - if (OB_FAIL(ret)) { - } else if (0 == param_count) { - spi_result.get_result_set()->set_user_sql(true); - OZ (GCTX.sql_engine_->handle_pl_execute( - sql_str.string(), *session, exec_params, *spi_result.get_result_set(), spi_result.get_sql_ctx(), - false /* is_prepare_protocol */, false /* is_dynamic_sql*/)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < exec_param_cnt; ++i) { - CK (OB_NOT_NULL(params[i])); - if (OB_SUCC(ret)) { - ObObjParam new_param = *params[i]; - if (params[i]->is_pl_extend()) { - if (params[i]->get_meta().get_extend_type() != PL_REF_CURSOR_TYPE) { - new_param.set_int_value(0); - if (!ob_is_xml_pl_type(params[i]->get_type(), params[i]->get_udt_id())) { - OZ (pl::ObUserDefinedType::deep_copy_obj(allocator, *params[i], new_param, true)); - } else { - uint64_t udt_id = params[i]->get_udt_id(); - const ObDataTypeCastParams dtc_params = sql::ObBasicSessionInfo::create_dtc_params(ctx->exec_ctx_->get_my_session()); - ObCastCtx cast_ctx(ctx->allocator_, &dtc_params, CM_NONE, ObCharset::get_system_collation()); - cast_ctx.exec_ctx_ = ctx->exec_ctx_; - uint16_t subschema_id = ObInvalidSqlType; - if (OB_FAIL(cast_ctx.exec_ctx_->get_subschema_id_by_udt_id(udt_id, - subschema_id, - &spi_result.get_scheme_guard()))) { - LOG_WARN("failed to get subschema id by udt_id", K(ret), K(params[i])); - } else if (FALSE_IT(new_param.set_subschema_id(subschema_id))) { - } else if (OB_FAIL(ObObjCaster::to_type(ObUserDefinedSQLType, cast_ctx, *params[i], new_param))) { - LOG_WARN("failed to_type", K(ret), K(new_param)); - } else { // sql udt params need original udt id for plan choose - new_param.set_udt_id(udt_id); - new_param.set_param_meta(); // param meta also changed - } - } - } - } else { - OZ (deep_copy_obj(allocator, *params[i], new_param)); - } - OX (new_param.set_need_to_check_type(true)); - OZ (exec_params.push_back(new_param), new_param); - } - } - if (OB_SUCC(ret)) { - char *tmp_ptr = NULL; - int64_t tmp_len = 0; - OZ (ObMPStmtExecute::store_params_value_to_str(spi_result.get_memory_ctx()->get_arena_allocator(), - *ctx->exec_ctx_->get_my_session(), - &exec_params, - tmp_ptr, - tmp_len)); - OX (spi_result.get_exec_params_str_ptr()->assign(tmp_ptr, tmp_len)); - } - LOG_INFO("execute dynamic sql using", K(ps_sql), K(exec_params)); - OZ (GCTX.sql_engine_->handle_pl_execute( - ps_sql, *session, exec_params, *spi_result.get_result_set(), spi_result.get_sql_ctx(), - true /* is_prepare_protocol */, true /* is_dynamic_sql*/)); - } - } - } - if (OB_FAIL(ret) && spi_result.get_result_set() != NULL) { - int cli_ret = OB_SUCCESS; - retry_ctrl.test_and_save_retry_state( - GCTX, - spi_result.get_sql_ctx(), - *spi_result.get_result_set(), - ret, cli_ret, true, true, true); - LOG_WARN("failed to get_result, check if need retry", - K(ret), K(cli_ret), K(retry_ctrl.need_retry())); - ret = cli_ret; - spi_result.get_sql_ctx().clear(); - ctx->exec_ctx_->get_my_session()->set_session_in_retry(retry_ctrl.need_retry()); - } - // todo:@hr351303 确认session标记是否还需要 - !is_inner_session ? session->set_user_session() : (void)NULL; - session->set_session_type(old_session_type); - session->set_client_return_rowid(old_client_return_rowid); - LOG_TRACE("execute dynamic sql", K(ret), K(sql), K(sql_str), K(ps_sql)); + OX (session->set_stmt_type(saved_stmt_type)); - int64_t row_count = 0; - OZ (inner_fetch(ctx, - retry_ctrl, - spi_result, - into_exprs, - into_count, - column_types, - type_count, - exprs_not_null_flag, - pl_integer_ranges, - out_using_params.empty() ? NULL : &out_using_params, - row_count, - is_bulk, - false, - true/*is_dynamic_sql*/, - NULL, - false, - false, - INT64_MAX, - NULL, - 0, - is_type_record)); - - //此处仅需要处理非DML RETURNING返回的USING OUT参数 - // if it is bulk into, not allow using out param, so no need deep copy - if (OB_SUCC(ret) && !is_bulk) { - OZ (dynamic_out_params(*(ctx->allocator_), spi_result.get_result_set(), params, exec_param_cnt)); - } - //监控项统计结束 - time_record.set_exec_end_timestamp(ObTimeUtility::current_time()); - if (enable_perf_event) { - exec_record.record_end(); - } - } - // 处理监控统计项 - if (OB_NOT_NULL(spi_result.get_result_set())) { - if (spi_result.get_result_set()->is_inited()) { - int64_t try_cnt = session->get_raw_audit_record().try_cnt_; - ObExecRecord record_bk = session->get_raw_audit_record().exec_record_; - session->get_raw_audit_record().try_cnt_ = retry_ctrl.get_retry_times(); - session->get_raw_audit_record().pl_trace_id_ = trace_id_guard.origin_trace_id_; - ObInnerSQLConnection::process_record(*spi_result.get_result_set(), - spi_result.get_sql_ctx(), - *session, - time_record, - ret, - session->get_current_execution_id(), // sql execute id - OB_INVALID_ID, // ps stmt id FIXME@hr351303 - max_wait_desc, - total_wait_desc, - exec_record, - exec_timestamp, - true, - 0 == param_count ? sql_str.string() : ps_sql, - true, - spi_result.get_exec_params_str_ptr()); - session->get_raw_audit_record().exec_record_ = record_bk; - session->get_raw_audit_record().try_cnt_ = try_cnt; - session->get_raw_audit_record().pl_trace_id_.reset(); - } else { - LOG_DEBUG("result set is not inited, do not process record", - K(ret), K(ps_sql), K(sql_str), K(stmt_type)); - } - } else { - if (OB_SUCC(ret)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, result_set is null", - K(ret), K(ps_sql), K(sql_str), K(stmt_type)); - } else { - LOG_WARN("result_set is null", - K(ret), K(ps_sql), K(sql_str), K(stmt_type)); - } - } - // 无论成功或者失败都在这里close result set - int close_ret = spi_result.close_result_set(); - if (OB_SUCCESS != close_ret) { - LOG_WARN("close result set failed", K(ret), K(close_ret)); - } - ret = OB_SUCCESS == ret ? close_ret : ret; - spi_result.destruct_exec_params(*session); - is_retry = true; - } while (RETRY_TYPE_NONE != retry_ctrl.get_retry_type()); //SPI只做LOCAL重试 - session->get_retry_info_for_update().clear(); - session->set_query_start_time(old_query_start_time); - } else { - // 不需要执行, 但是要维护隐式游标 - ObPLCursorInfo *implicit_cursor = session->get_pl_implicit_cursor(); - CK (OB_NOT_NULL(implicit_cursor)); - OX (implicit_cursor->set_rowcount(0)); - } - //自动提交,禁掉PL整体重试 - if (OB_SUCC(ret) - && (ObStmt::is_ddl_stmt(stmt_type, true) - || ObStmt::is_tcl_stmt(stmt_type) - || (lib::is_mysql_mode() && session->get_local_autocommit()))) { - OX (session->set_pl_can_retry(false)); - } - if (OB_SUCC(ret) - && (ObStmt::is_ddl_stmt(stmt_type, true) - || ObStmt::is_tcl_stmt(stmt_type) - || ObStmt::is_savepoint_stmt(stmt_type))) { - if (ObStmt::is_ddl_stmt(stmt_type, true)) { - //DDL语句,需要强制刷新schema - OZ (force_refresh_schema(session->get_effective_tenant_id())); - } - recreate_implicit_savapoint_if_need(ctx, ret); - } - - spi_result.end_nested_stmt_if_need(ctx, ret); - - SET_FORALL_BULK_EXCEPTION; - SET_SPI_STATUS; - } - // } + // Step: execute dynamic SQL now! + if (OB_FAIL(ret)) { + } else if (ObStmt::is_select_stmt(stmt_type) && !for_update && into_count <= 0) { + /*! + * If dynamic_sql_statement is a SELECT statement, and you omit both + * into_clause and bulk_collect_into_clause, then + * execute_immediate_statement never executes. + * For example, this statement never increments the sequence: + * EXECUTE IMMEDIATE 'SELECT S.NEXTVAL FROM DUAL' + */ + ObPLCursorInfo *implicit_cursor = session->get_pl_implicit_cursor(); + CK (OB_NOT_NULL(implicit_cursor)); + OX (implicit_cursor->set_rowcount(0)); + } else { + OZ (SMART_CALL(spi_inner_execute(ctx, + sql_str.ptr(), + ps_sql.ptr(), + stmt_type, + params, + exec_param_cnt, + into_exprs, + into_count, + column_types, + type_count, + exprs_not_null_flag, + pl_integer_ranges, + is_bulk, + false, /*is_forall*/ + is_type_record, + for_update, + true, /*is_dynamic_sql*/ + &out_using_params))); + } return ret; } @@ -3692,23 +3241,37 @@ int ObSPIService::spi_dynamic_open(ObPLExecCtx *ctx, int ObSPIService::dbms_dynamic_open(ObPLExecCtx *pl_ctx, ObDbmsCursorInfo &cursor, - bool is_dbms_sql) + bool is_dbms_sql, + int64_t orc_max_ret_rows) { int ret = OB_SUCCESS; - // ObString &sql_stmt = cursor.get_sql_stmt(); - // ParamStore &exec_params = cursor.get_exec_params(); + stmt::StmtType stmt_type = cursor.get_stmt_type(); const ObString ps_sql = cursor.get_ps_sql(); bool for_update = cursor.is_for_update(); bool hidden_rowid = cursor.has_hidden_rowid(); - // int64_t into_cnt = 0; OV (OB_NOT_NULL(pl_ctx->exec_ctx_->get_my_session())); if (ObStmt::is_select_stmt(stmt_type) || cursor.get_into_names().count() > 0) { // NOTICE: DML Returning also use cursor impl. - OZ (dbms_cursor_open(pl_ctx, cursor, ps_sql, stmt_type, for_update, hidden_rowid), cursor); + OZ (dbms_cursor_open( + pl_ctx, cursor, ps_sql, stmt_type, for_update, hidden_rowid, orc_max_ret_rows), + cursor); } else { OZ (dbms_cursor_execute(pl_ctx, ps_sql, stmt_type, cursor, is_dbms_sql), cursor); + if (OB_SUCC(ret) && cursor.get_exec_params().count() > 0) { + ObIAllocator &alloc = cursor.get_dbms_entity()->get_arena_allocator(); + for (int64_t i = 0; i < cursor.get_exec_params().count(); ++i) { + ObObjParam &obj = cursor.get_exec_params().at(i); + if (obj.is_pl_extend() + && obj.get_meta().get_extend_type() != PL_CURSOR_TYPE + && obj.get_meta().get_extend_type() != PL_REF_CURSOR_TYPE) { + OZ (pl::ObUserDefinedType::deep_copy_obj(alloc, obj, obj, true)); + } else { + OZ (deep_copy_obj(alloc, obj, obj)); + } + } + } OX (cursor.set_affected_rows(pl_ctx->exec_ctx_->get_my_session()->get_affected_rows())); } return ret; @@ -3793,6 +3356,233 @@ int ObSPIService::spi_cursor_open_with_param_idx(ObPLExecCtx *ctx, return ret; } +int ObSPIService::streaming_cursor_open(ObPLExecCtx *ctx, + ObPLCursorInfo &cursor, + ObSQLSessionInfo &session_info, + const ObString &sql, + const ObString &ps_sql, + int64_t type, + void *params, + int64_t sql_param_count, + bool is_server_cursor, + bool for_update, + bool has_hidden_rowid, + bool is_dbms_cursor) +{ + int ret = OB_SUCCESS; + + cursor.set_streaming(); + ObSPIResultSet *spi_result = NULL; + ObString ps_sql_copy; + + OZ (cursor.prepare_spi_result(ctx, spi_result)); + CK (OB_NOT_NULL(spi_result)); + CK (OB_NOT_NULL(spi_result->get_memory_ctx())); + OZ (spi_result->start_cursor_stmt(ctx, static_cast(type), for_update)); + + // in a streaming cursor, lifetime of ps_sql may be shorter than the cursor itself, + // so we need to open it with a deep copy. + OZ (ob_write_string(spi_result->get_allocator(), ps_sql, ps_sql_copy)); + + if (OB_SUCC(ret)) { + + ObPLSqlAuditRecord audit_record(sql::PLSql); + ObQueryRetryCtrl retry_ctrl; + ObSPIExecEnvGuard env_guard(session_info, *spi_result); + + do { + { + ObPLSubPLSqlTimeGuard guard(ctx); + ObPLSPITraceIdGuard trace_id_guard(sql, ps_sql, session_info, ret); + ObPLSqlAuditGuard audit_guard( + *(ctx->exec_ctx_), session_info, *spi_result, audit_record, ret, (sql != NULL ? sql : ps_sql), retry_ctrl, trace_id_guard, static_cast(type)); + ObSPIRetryCtrlGuard retry_guard(retry_ctrl, *spi_result, session_info, ret); + + if (OB_FAIL(ret)) { + } else if (is_server_cursor) { + WITH_CONTEXT(cursor.get_cursor_entity()) { + lib::ContextTLOptGuard guard(false); + OZ (inner_open(ctx, + spi_result->get_memory_ctx()->get_arena_allocator(), + sql, + ps_sql_copy, + type, + params, + sql_param_count, + nullptr, + 0, + *spi_result, + spi_result->get_out_params(), + false, /*is_forall*/ + false, /*is_dynamic_sql*/ + is_dbms_cursor /*is_dbms_sql*/)); + } + } else { + OZ (inner_open(ctx, + spi_result->get_memory_ctx()->get_arena_allocator(), + sql, + ps_sql_copy, + type, + params, + sql_param_count, + nullptr, + 0, + *spi_result, + spi_result->get_out_params(), + false, /*is_forall*/ + false, /*is_dynamic_sql*/ + is_dbms_cursor /*is_dbms_sql*/)); + } + CK (OB_NOT_NULL(spi_result->get_result_set()->get_field_columns())); + if (OB_SUCC(ret) && is_dbms_cursor) { + ObDbmsCursorInfo &dbms_cursor = static_cast(cursor); + if (dbms_cursor.get_field_columns().empty()) { + const common::ColumnsFieldArray* field_column = + dynamic_cast + (spi_result->get_result_set()->get_field_columns()); + OX (dbms_cursor.get_field_columns().set_allocator(&dbms_cursor.get_dbms_entity()->get_arena_allocator())); + OZ (dbms_cursor.get_field_columns().assign(*field_column)); + } + } + if (OB_SUCC(ret) && OB_INVALID_ID != cursor.get_id()) { + //如果是客户端游标，设置结果集为二进制模式 + OX (spi_result->get_result_set()->set_ps_protocol()); + } + OX (cursor.open(spi_result)); + OX (for_update ? cursor.set_for_update() : (void)NULL); + OX (for_update ? cursor.set_trans_id(session_info.get_tx_id()) : (void)NULL); + OX (has_hidden_rowid ? cursor.set_hidden_rowid() : (void)NULL); + OZ (setup_cursor_snapshot_verify_(&cursor, spi_result)); + if (!cursor.is_ps_cursor()) { + retry_guard.test(); + } + } + if (OB_FAIL(ret)) { + int close_ret = spi_result->close_result_set(); + if (OB_SUCCESS != close_ret) { + LOG_WARN("close mysql result set failed", K(ret), K(close_ret)); + } + } + } while (RETRY_TYPE_NONE != retry_ctrl.get_retry_type()); + spi_result->end_cursor_stmt(ctx, ret); + cursor.set_last_execute_time(ObTimeUtility::current_time()); + // only care about end_cursor_stmt failed. + // if cursor already open, spi_result will released by cursor close. + // there is not a situation like '!cursor.is_open && spi_result not close'. + // so do not need call spi_result.close_result_set. + if (OB_FAIL(ret) && !cursor.isopen()) { + spi_result->~ObSPIResultSet(); + } + } + return ret; +} + +int ObSPIService::unstreaming_cursor_open(ObPLExecCtx *ctx, + ObPLCursorInfo &cursor, + ObSQLSessionInfo &session_info, + const ObString &sql, + const ObString &ps_sql, + int64_t type, + void *params, + int64_t sql_param_count, + bool is_server_cursor, + bool for_update, + bool has_hidden_rowid, + bool is_dbms_cursor, + int64_t orc_max_ret_rows) +{ + int ret = OB_SUCCESS; + + HEAP_VAR(ObSPIResultSet, spi_result) { + ObSPICursor* spi_cursor = NULL; + OZ (spi_result.init(session_info)); + OZ (spi_result.start_nested_stmt_if_need(ctx, sql, static_cast(type), for_update)); + + if (OB_SUCC(ret)) { + + ObPLSqlAuditRecord audit_record(sql::PLSql); + ObQueryRetryCtrl retry_ctrl; + ObSPIExecEnvGuard env_guard(session_info, spi_result); + + do { + ret = OB_SUCCESS; + uint64_t size = 0; + { + ObPLSubPLSqlTimeGuard guard(ctx); + ObPLSPITraceIdGuard trace_id_guard(sql, ps_sql, session_info, ret); + ObPLSqlAuditGuard audit_guard( + *(ctx->exec_ctx_), session_info, spi_result, audit_record, ret, (sql != NULL ? sql : ps_sql), retry_ctrl, trace_id_guard, static_cast(type)); + ObSPIRetryCtrlGuard retry_guard(retry_ctrl, spi_result, session_info, ret); + + CK (OB_NOT_NULL(spi_result.get_memory_ctx())); + OZ (inner_open(ctx, + spi_result.get_memory_ctx()->get_arena_allocator(), + sql, + ps_sql, + type, + params, + sql_param_count, + NULL, + 0, + spi_result, + spi_result.get_out_params(), + false, /*is_forall*/ + false, /*is_dynamic_sql*/ + is_dbms_cursor /*is_dbms_sql*/)); + OZ (session_info.get_tmp_table_size(size)); + OZ (cursor.prepare_spi_cursor(spi_cursor, + session_info.get_effective_tenant_id(), + size, + (for_update && !is_server_cursor && !is_dbms_cursor), + &session_info), K(size)); + CK (OB_NOT_NULL(spi_result.get_result_set())); + OZ (fill_cursor(*spi_result.get_result_set(), spi_cursor, ObTimeUtility::current_time(), orc_max_ret_rows)); + OZ (spi_cursor->row_store_.finish_add_row()); + if (OB_FAIL(ret)) { + } else if (is_dbms_cursor) { + ObDbmsCursorInfo &dbms_cursor = static_cast(cursor); + OZ (ObDbmsInfo::deep_copy_field_columns( + dbms_cursor.get_dbms_entity()->get_arena_allocator(), + spi_result.get_result_set()->get_field_columns(), + dbms_cursor.get_field_columns())); + } else { + CK (OB_NOT_NULL(cursor.get_allocator())); + OZ (ObDbmsInfo::deep_copy_field_columns( + *cursor.get_allocator(), spi_result.get_result_set()->get_field_columns(), spi_cursor->fields_)); + } + if (OB_SUCC(ret) && OB_NOT_NULL(spi_result.get_result_set()) && OB_NOT_NULL(spi_result.get_result_set()->get_physical_plan())) { + cursor.set_packed(spi_result.get_result_set()->get_physical_plan()->is_packed()); + } + OX (cursor.open(spi_cursor)); + OX (for_update ? cursor.set_for_update() : (void)NULL); + OX (for_update ? cursor.set_trans_id(session_info.get_tx_id()) : (void)NULL); + OX (has_hidden_rowid ? cursor.set_hidden_rowid() : (void)NULL); + if (OB_SUCC(ret) && lib::is_oracle_mode()) { + OZ (setup_cursor_snapshot_verify_(&cursor, &spi_result)); + } + if (!cursor.is_ps_cursor()) { + retry_guard.test(); + } + } + int close_ret = spi_result.close_result_set(); + if (OB_SUCCESS != close_ret) { + LOG_WARN("close mysql result failed", K(ret), K(close_ret)); + } + ret = (OB_SUCCESS == ret ? close_ret : ret); + if (!is_dbms_cursor) { + spi_result.destruct_exec_params(session_info); + } + + } while (RETRY_TYPE_NONE != retry_ctrl.get_retry_type()); + } + if (OB_FAIL(ret) && OB_NOT_NULL(spi_cursor)) { + spi_cursor->~ObSPICursor(); + } + spi_result.end_nested_stmt_if_need(ctx, ret); + } + return ret; +} + int ObSPIService::spi_cursor_open(ObPLExecCtx *ctx, const char *sql, const char *ps_sql, @@ -3824,8 +3614,7 @@ int ObSPIService::spi_cursor_open(ObPLExecCtx *ctx, LOG_WARN("Argument passed in is NULL", K(ctx), K(sql), K(ps_sql), K(type), K(sql_param_exprs), K(sql_param_count), K(ret)); } else if (OB_FAIL(spi_get_cursor_info(ctx, package_id, routine_id, cursor_index, cursor, cursor_var, loc))) { LOG_WARN("failed to get cursor info", K(ret), K(cursor_index)); - } else if (OB_FAIL(cursor_open_check(ctx, package_id, routine_id, - cursor_index, cursor, cursor_var, loc))) { + } else if (OB_FAIL(cursor_open_check(ctx, package_id, routine_id, cursor_index, cursor, cursor_var, loc))) { LOG_WARN("cursor info not init", K(ret), K(cursor)); } else if (cursor->isopen()) { ret = OB_ER_SP_CURSOR_ALREADY_OPEN; @@ -3838,353 +3627,70 @@ int ObSPIService::spi_cursor_open(ObPLExecCtx *ctx, LOG_USER_ERROR(OB_NOT_SUPPORTED, "non-select stmt in cursor"); } else { ParamStore current_params(ObWrapperAllocator(ctx->allocator_)); - ObIAllocator *allocator = cursor->get_allocator(); ObPLSubPLSqlTimeGuard guard(ctx); - if (OB_ISNULL(allocator)) { + if (cursor_param_count > 0 && (NULL == formal_param_idxs || NULL == actual_param_exprs)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("Argument in pl context is NULL", K(allocator), K(ret)); + LOG_WARN("cursor params in not valid", + K(cursor_param_count), K(formal_param_idxs), K(actual_param_exprs), K(ret)); } else { - //首先准备Cursor的实参 - if (cursor_param_count > 0 && (NULL == formal_param_idxs || NULL == actual_param_exprs)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("cursor params in not valid", - K(cursor_param_count), K(formal_param_idxs), K(actual_param_exprs), K(ret)); - } else { - OZ (prepare_cursor_parameters( - ctx, *session_info, package_id, - routine_id, loc, formal_param_idxs, actual_param_exprs, cursor_param_count)); - } + OZ (prepare_cursor_parameters( + ctx, *session_info, package_id, + routine_id, loc, formal_param_idxs, actual_param_exprs, cursor_param_count)); + } - if (OB_SUCC(ret) && DECL_SUBPROG == loc) { - ParamStore *subprog_params = NULL; - OZ (current_params.assign(*ctx->params_)); - OZ (ObPLContext::get_param_store_from_local(*session_info, package_id, routine_id, subprog_params)); - CK (OB_NOT_NULL(subprog_params)); - OZ (ctx->params_->assign(*subprog_params)); + if (OB_SUCC(ret) && DECL_SUBPROG == loc) { + ParamStore *subprog_params = NULL; + OZ (current_params.assign(*ctx->params_)); + OZ (ObPLContext::get_param_store_from_local(*session_info, package_id, routine_id, subprog_params)); + CK (OB_NOT_NULL(subprog_params)); + OZ (ctx->params_->assign(*subprog_params)); + } + + bool is_server_cursor = false; + bool use_stream = false; + if (OB_SUCC(ret)) { + is_server_cursor = OB_INVALID_ID != cursor->get_id() + || (package_id != OB_INVALID_ID && OB_INVALID_ID == routine_id); + if (is_server_cursor) { + OZ (ObPLCursorInfo::prepare_entity(*session_info, cursor->get_cursor_entity())); + OX (cursor->set_spi_cursor(NULL)); } - bool is_server_cursor = false; - bool use_stream = false; - if (OB_SUCC(ret)) { - is_server_cursor = OB_INVALID_ID != cursor->get_id() - || (package_id != OB_INVALID_ID && OB_INVALID_ID == routine_id); - if (is_server_cursor) { - OZ (ObPLCursorInfo::prepare_entity(*session_info, cursor->get_cursor_entity())); - OX (cursor->set_spi_cursor(NULL)); - } - } - OZ (session_info->ps_use_stream_result_set(use_stream)); - if (OB_FAIL(ret)) { - // do nothing - } else if (lib::is_oracle_mode() - && ((is_server_cursor && use_stream) || !is_server_cursor) - && (!for_update || (for_update && skip_locked))) { - cursor->set_streaming(); - ObSPIResultSet *spi_result = NULL; - ObExecRecord exec_record; - ObExecTimestamp exec_timestamp; - ObSPITimeRecord time_record; - exec_timestamp.exec_type_ = sql::PLSql; - if (OB_FAIL(ret)) { - // do nothing - } else if (OB_ISNULL(cursor->get_allocator())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("Argument in pl context is NULL", K(cursor->get_allocator()), K(ret)); - } else { - ObQueryRetryCtrl retry_ctrl; - int64_t tenant_version = 0; - int64_t sys_version = 0; - ObPLSPITraceIdGuard trace_id_guard(sql, ps_sql, *session_info, ret); - do { - // SQL_AUDIT_START - ObWaitEventDesc max_wait_desc; - ObWaitEventStat total_wait_desc; - const bool enable_perf_event = lib::is_diagnose_info_enabled(); - const bool enable_sql_audit = - GCONF.enable_sql_audit && ctx->exec_ctx_->get_my_session()->get_local_ob_enable_sql_audit(); - { - ObMaxWaitGuard max_wait_guard(enable_perf_event ? &max_wait_desc : NULL); - ObTotalWaitGuard total_wait_guard(enable_perf_event ? &total_wait_desc : NULL); - if (enable_perf_event) { - exec_record.record_start(); - } - //监控项统计开始 - time_record.set_send_timestamp(ObTimeUtility::current_time()); - ret = OB_SUCCESS; - // 如果当前cursor已经有spi_result则复用,避免内存占用过多 - retry_ctrl.clear_state_before_each_retry(session_info->get_retry_info_for_update()); - OZ (cursor->prepare_spi_result(ctx, spi_result)); - CK (OB_NOT_NULL(spi_result->get_memory_ctx())); - OZ (spi_result->start_cursor_stmt(ctx, static_cast(type), for_update)); - OZ ((GCTX.schema_service_->get_tenant_schema_guard(session_info->get_effective_tenant_id(), spi_result->get_scheme_guard()))); - OX (spi_result->get_sql_ctx().schema_guard_ = &spi_result->get_scheme_guard()); - OZ (spi_result->get_scheme_guard().get_schema_version(session_info->get_effective_tenant_id(), tenant_version)); - OZ (spi_result->get_scheme_guard().get_schema_version(OB_SYS_TENANT_ID, sys_version)); - OX (retry_ctrl.set_tenant_local_schema_version(tenant_version)); - OX (retry_ctrl.set_sys_local_schema_version(sys_version)); - if (OB_FAIL(ret)) { - // do nothing - } else if (is_server_cursor) { - WITH_CONTEXT(cursor->get_cursor_entity()) { - lib::ContextTLOptGuard guard(false); - OZ (inner_open(ctx, - spi_result->get_memory_ctx()->get_arena_allocator(), + } + OZ (session_info->ps_use_stream_result_set(use_stream)); + + if (OB_FAIL(ret)) { + } else if (lib::is_oracle_mode() // streaming cursor + && ((is_server_cursor && use_stream) || !is_server_cursor) + && (!for_update || (for_update && skip_locked))) { + OZ (streaming_cursor_open(ctx, + *cursor, + *session_info, sql, ps_sql, type, sql_param_exprs, sql_param_count, - NULL, - 0, - *spi_result, - spi_result->get_out_params(), - &retry_ctrl)); - } - } else { - ret = inner_open(ctx, - spi_result->get_memory_ctx()->get_arena_allocator(), - sql, - ps_sql, - type, - sql_param_exprs, - sql_param_count, - NULL, - 0, - *spi_result, - spi_result->get_out_params(), - &retry_ctrl); - } - OX (cursor->open(spi_result)); - CK (OB_NOT_NULL(spi_result->get_result_set())); - if (OB_SUCC(ret) && OB_INVALID_ID != cursor->get_id()) { - //如果是客户端游标，设置结果集为二进制模式 - OX (spi_result->get_result_set()->set_ps_protocol()); - } - OX (for_update ? cursor->set_for_update() : (void)NULL); - OX (for_update ? cursor->set_trans_id(session_info->get_tx_id()) : (void)NULL); - OX (has_hidden_rowid ? cursor->set_hidden_rowid() : (void)NULL); - OZ (setup_cursor_snapshot_verify_(cursor, spi_result)); - bool need_destruct = false; - if (OB_FAIL(ret) && OB_NOT_NULL(spi_result)) { - int tmp_ret = ret; - ret = OB_SUCCESS; - if (OB_NOT_NULL(spi_result->get_result_set())) { - // 此分支所有错误码都被吞掉，最终返回最初的错误码 - int close_ret = spi_result->close_result_set(); - if (OB_SUCCESS != close_ret) { - LOG_WARN("close mysql result set failed", K(ret), K(close_ret)); - } - } - ret = tmp_ret; - need_destruct = true; - LOG_WARN("cursor open result failed.", K(ret)); - } - if (OB_NOT_NULL(spi_result)) { - spi_result->end_cursor_stmt(ctx, ret); - if (!need_destruct && OB_SUCCESS != ret) { - need_destruct = true; - if (OB_NOT_NULL(spi_result->get_result_set())) { - // 此分支所有错误码都被吞掉，最终返回最初的错误码 - int close_ret = spi_result->close_result_set(); - if (OB_SUCCESS != close_ret) { - LOG_WARN("close mysql result set failed", K(ret), K(close_ret)); - } - } - } - } - if (need_destruct) { - spi_result->~ObSPIResultSet(); - } - } - //监控项统计结束 - time_record.set_exec_end_timestamp(ObTimeUtility::current_time()); - if (enable_perf_event) { - exec_record.record_end(); - } - // 处理监控统计项 - if (OB_NOT_NULL(spi_result) && OB_NOT_NULL(spi_result->get_result_set())) { - if (spi_result->get_result_set()->is_inited()) { - ObSQLSessionInfo *session_info = ctx->exec_ctx_->get_my_session(); - int64_t try_cnt = session_info->get_raw_audit_record().try_cnt_; - ObExecRecord record_bk = session_info->get_raw_audit_record().exec_record_; - session_info->get_raw_audit_record().try_cnt_ = retry_ctrl.get_retry_times(); - session_info->get_raw_audit_record().pl_trace_id_.set(trace_id_guard.origin_trace_id_); - ObInnerSQLConnection::process_record(*spi_result->get_result_set(), - spi_result->get_sql_ctx(), - *session_info, - time_record, - ret, - session_info->get_current_execution_id(), // sql execute id - OB_INVALID_ID, - max_wait_desc, - total_wait_desc, - exec_record, - exec_timestamp, - true, - sql != NULL ? sql : ps_sql, - true); - session_info->get_raw_audit_record().exec_record_ = record_bk; - session_info->get_raw_audit_record().try_cnt_ = try_cnt; - session_info->get_raw_audit_record().pl_trace_id_.reset(); - } - } - } while (RETRY_TYPE_NONE != retry_ctrl.get_retry_type()); - } - cursor->set_last_execute_time(ObTimeUtility::current_time()); - } else { //MySQL Cursor/Updated Cursor/Server Cursor(REF_CURSOR, PACKAGE CURSOR) - ObPLSPITraceIdGuard trace_id_guard(sql, ps_sql, *session_info, ret); - HEAP_VAR(ObSPIResultSet, spi_result) { - ObString sqlstr(sql); - OZ (spi_result.init(*session_info)); - OZ (spi_result.start_nested_stmt_if_need(ctx, sqlstr, static_cast(type), for_update)); - int64_t old_query_start_time = session_info->get_query_start_time(); - // query_start_time_ set to 0 in begin_nested_session, here we reset it. - int64_t new_query_start_time = ObTimeUtility::current_time(); - session_info->set_query_start_time(new_query_start_time); - if (OB_SUCC(ret)) { - ObExecRecord exec_record; - ObExecTimestamp exec_timestamp; - ObSPITimeRecord time_record; - exec_timestamp.exec_type_ = sql::PLSql; - ObQueryRetryCtrl retry_ctrl; - int64_t tenant_version = 0; - int64_t sys_version = 0; - bool is_retry = false; - do { - // SQL_AUDIT_START - ObWaitEventDesc max_wait_desc; - ObWaitEventStat total_wait_desc; - const bool enable_perf_event = lib::is_diagnose_info_enabled(); - const bool enable_sql_audit = - GCONF.enable_sql_audit && ctx->exec_ctx_->get_my_session()->get_local_ob_enable_sql_audit(); - { - ObMaxWaitGuard max_wait_guard(enable_perf_event ? &max_wait_desc : NULL); - ObTotalWaitGuard total_wait_guard(enable_perf_event ? &total_wait_desc : NULL); - if (enable_perf_event) { - exec_record.record_start(); - } - //监控项统计开始 - time_record.set_send_timestamp(ObTimeUtility::current_time()); - ret = OB_SUCCESS; - if (is_retry) { - spi_result.get_out_params().reset(); - spi_result.reset_member_for_retry(*session_info); - } - CK (OB_NOT_NULL(spi_result.get_memory_ctx())); - OX (retry_ctrl.clear_state_before_each_retry(session_info->get_retry_info_for_update())); - OZ ((GCTX.schema_service_->get_tenant_schema_guard(session_info->get_effective_tenant_id(), spi_result.get_scheme_guard()))); - OX (spi_result.get_sql_ctx().schema_guard_ = &spi_result.get_scheme_guard()); - OZ (spi_result.get_scheme_guard().get_schema_version(session_info->get_effective_tenant_id(), tenant_version)); - OZ (spi_result.get_scheme_guard().get_schema_version(OB_SYS_TENANT_ID, sys_version)); - OX (retry_ctrl.set_tenant_local_schema_version(tenant_version)); - OX (retry_ctrl.set_sys_local_schema_version(sys_version)); - - OZ (inner_open(ctx, - spi_result.get_memory_ctx()->get_arena_allocator(), - sql, - ps_sql, - type, - sql_param_exprs, - sql_param_count, - NULL, - 0, - spi_result, - spi_result.get_out_params(), - &retry_ctrl)); - if (OB_SUCC(ret)) { - ObSPICursor* spi_cursor = cursor->get_spi_cursor(); - uint64_t size = 0; - OZ (session_info->get_tmp_table_size(size)); - OZ (cursor->prepare_spi_cursor(spi_cursor, - session_info->get_effective_tenant_id(), - size, - for_update && !is_server_cursor, - session_info), K(size)); - CK (OB_NOT_NULL(cursor->get_allocator())); - CK (OB_NOT_NULL(spi_result.get_result_set())); - OZ (fill_cursor(*spi_result.get_result_set(), spi_cursor, new_query_start_time)); - OZ (ObDbmsInfo::deep_copy_field_columns( - *cursor->get_allocator(), - spi_result.get_result_set()->get_field_columns(), - spi_cursor->fields_)); - if (OB_FAIL(ret) && OB_NOT_NULL(spi_result.get_result_set())) { - int cli_ret = OB_SUCCESS; - retry_ctrl.test_and_save_retry_state(GCTX, - spi_result.get_sql_ctx(), - *spi_result.get_result_set(), - ret, - cli_ret, - true, - true, - true); - LOG_WARN("failed to do fill_cursor, check if need retry", K(ret), K(cli_ret), K(retry_ctrl.need_retry()), K(sql), K(ps_sql)); - ret = cli_ret; - spi_result.get_sql_ctx().clear(); - ctx->exec_ctx_->get_my_session()->set_session_in_retry(retry_ctrl.need_retry()); - } - OX (spi_cursor->row_store_.finish_add_row()) - OX (cursor->open(spi_cursor)); - if (OB_FAIL(ret)) { - spi_cursor->~ObSPICursor(); - } - } - OX (for_update ? cursor->set_for_update() : (void)NULL); - OX (for_update ? cursor->set_trans_id(session_info->get_tx_id()) : (void)NULL); - OX (has_hidden_rowid ? cursor->set_hidden_rowid() : (void)NULL); - - CK (OB_NOT_NULL(spi_result.get_result_set())); - if (OB_SUCC(ret) && lib::is_oracle_mode()) { - OZ (setup_cursor_snapshot_verify_(cursor, &spi_result)); - } - } - //监控项统计结束 - time_record.set_exec_end_timestamp(ObTimeUtility::current_time()); - if (OB_SUCC(ret)) { - LOG_DEBUG("start process record", K(ret), K(ps_sql), K(sql), K(type), K(enable_sql_audit)); - if (spi_result.get_result_set()->is_inited()) { - ObSQLSessionInfo *session_info = ctx->exec_ctx_->get_my_session(); - int64_t try_cnt = session_info->get_raw_audit_record().try_cnt_; - ObExecRecord record_bk = session_info->get_raw_audit_record().exec_record_; - session_info->get_raw_audit_record().try_cnt_ = retry_ctrl.get_retry_times(); - session_info->get_raw_audit_record().pl_trace_id_.set(trace_id_guard.origin_trace_id_); - ObInnerSQLConnection::process_record(*spi_result.get_result_set(), - spi_result.get_sql_ctx(), - *session_info, - time_record, - ret, - session_info->get_current_execution_id(), // sql execute id - OB_INVALID_ID, - max_wait_desc, - total_wait_desc, - exec_record, - exec_timestamp, - true, - sql != NULL ? sql : ps_sql, - true); - session_info->get_raw_audit_record().exec_record_ = record_bk; - session_info->get_raw_audit_record().try_cnt_ = try_cnt; - session_info->get_raw_audit_record().pl_trace_id_.reset(); - } - } - - int close_ret = spi_result.close_result_set(); - if (OB_SUCCESS != close_ret) { - LOG_WARN("close mysql result failed", K(ret), K(close_ret)); - } - ret = (OB_SUCCESS == ret ? close_ret : ret); - is_retry = true; - } while (RETRY_TYPE_NONE != retry_ctrl.get_retry_type()); - } - spi_result.destruct_exec_params(*session_info); - spi_result.end_nested_stmt_if_need(ctx, ret); - session_info->set_query_start_time(old_query_start_time); - } - } - if (OB_SUCC(ret)) { - if (DECL_SUBPROG == loc) { - OZ (ctx->params_->assign(current_params)); - } else if (DECL_PKG == loc) { - OZ (spi_update_package_change_info(ctx, package_id, cursor_index)); - } + is_server_cursor, + for_update, + has_hidden_rowid)); + } else { //MySQL Cursor/Updated Cursor/Server Cursor(REF_CURSOR, PACKAGE CURSOR), unstreaming cursor + OZ (unstreaming_cursor_open(ctx, + *cursor, + *session_info, + sql, + ps_sql, + type, + sql_param_exprs, + sql_param_count, + is_server_cursor, + for_update, + has_hidden_rowid)); + } + if (OB_SUCC(ret)) { + if (DECL_SUBPROG == loc) { + OZ (ctx->params_->assign(current_params)); + } else if (DECL_PKG == loc) { + OZ (spi_update_package_change_info(ctx, package_id, cursor_index)); } } } @@ -4200,325 +3706,58 @@ int ObSPIService::dbms_cursor_open(ObPLExecCtx *ctx, const ObString &ps_sql, int64_t stmt_type, bool for_update, - bool hidden_rowid) + bool hidden_rowid, + int64_t orc_max_ret_rows) { int ret = OB_SUCCESS; ObSQLSessionInfo *session = NULL; ParamStore &exec_params = cursor.get_exec_params(); - ObString &sql_stmt = cursor.get_sql_stmt(); - ObString sql_str = - (exec_params.count() > 0 || cursor.get_into_names().count() > 0) ? ObString() : sql_stmt; + ObString sql_str = (exec_params.count() > 0 || cursor.get_into_names().count() > 0) ? ObString() : cursor.get_sql_stmt(); bool use_stream = false; - ObExecRecord exec_record; - ObExecTimestamp exec_timestamp; - ObSPITimeRecord time_record; - exec_timestamp.exec_type_ = cursor.is_ps_cursor() ? sql::PSCursor : sql::DbmsCursor; - ObWaitEventDesc max_wait_desc; - ObWaitEventStat total_wait_desc; - ObArenaAllocator exec_param_alloc; - ObString exec_param_str; - const bool enable_perf_event = lib::is_diagnose_info_enabled(); - const bool enable_sql_audit = GCONF.enable_sql_audit - && ctx->exec_ctx_->get_my_session()->get_local_ob_enable_sql_audit(); ObPLSubPLSqlTimeGuard guard(ctx); + OV (OB_NOT_NULL(ctx) && OB_NOT_NULL(ctx->exec_ctx_) && OB_NOT_NULL(ctx->allocator_) && OB_NOT_NULL(session = ctx->exec_ctx_->get_my_session()), - OB_INVALID_ARGUMENT, ctx, ps_sql, stmt_type); + OB_INVALID_ARGUMENT, KP(ctx), K(ps_sql), K(stmt_type)); OZ (session->ps_use_stream_result_set(use_stream)); - if (enable_perf_event) { - exec_record.record_start(); - } - //监控项统计开始 - time_record.set_send_timestamp(ObTimeUtility::current_time()); if (OB_SUCC(ret) && cursor.isopen()) { - if(OB_FAIL(dbms_cursor_close(*ctx->exec_ctx_, cursor))) { - LOG_WARN("close cursor fail.", K(ret), K(cursor.get_id())); - } else { - //为cursor分配内存空间 - if (OB_FAIL(ObPLCursorInfo::prepare_entity(*session, cursor.get_cursor_entity()))) { - // 此处只能处理cursor.get_cursor_entity(), 不能处理cursor.get_dbms_entity(),否则exec_params等的值的allocator被reset - // 会导致core - LOG_WARN("failed to alloc ref cursor entity", K(ret)); - } else { - cursor.set_spi_cursor(NULL); - } - } - } - - if (OB_SUCC(ret) && exec_params.count() > 0) { - char *tmp_ptr = NULL; - int64_t tmp_len = 0; - OZ (ObMPStmtExecute::store_params_value_to_str(exec_param_alloc, - *session, - &exec_params, - tmp_ptr, - tmp_len)); - OX (exec_param_str.assign(tmp_ptr, tmp_len)); + OZ (dbms_cursor_close(*ctx->exec_ctx_, cursor), K(cursor.get_id())); + OZ (ObPLCursorInfo::prepare_entity(*session, cursor.get_cursor_entity())); + OX (cursor.set_spi_cursor(NULL)); } if (OB_FAIL(ret)) { - // do nothing - } else if (!for_update && use_stream) { - ObSPIResultSet *spi_result = NULL; - if (stmt::T_SELECT != static_cast(stmt_type)) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("only supported select stmt in cursor", K(ret), K(stmt_type)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "non-select stmt in cursor"); - } - OX (cursor.set_streaming()); - OV (OB_NOT_NULL(cursor.get_dbms_entity()), OB_NOT_INIT, sql_stmt, ps_sql, exec_params); - OV (OB_NOT_NULL(cursor.get_cursor_entity()), OB_NOT_INIT, sql_stmt, ps_sql, exec_params); - ObQueryRetryCtrl retry_ctrl; - int64_t tenant_version = 0; - int64_t sys_version = 0; - int64_t retry_cnt = 0; - ObPLSPITraceIdGuard trace_id_guard(sql_stmt, ps_sql, *session, ret); - do { - ret = OB_SUCCESS; - retry_ctrl.clear_state_before_each_retry(session->get_retry_info_for_update()); - OZ (cursor.prepare_spi_result(ctx, spi_result), sql_stmt, ps_sql, exec_params); - OV (OB_NOT_NULL(spi_result), OB_ERR_UNEXPECTED, sql_stmt, ps_sql, exec_params); - OZ (spi_result->start_cursor_stmt(ctx, static_cast(stmt_type)), - sql_stmt, ps_sql, exec_params); - OZ ((GCTX.schema_service_->get_tenant_schema_guard(session->get_effective_tenant_id(), spi_result->get_scheme_guard()))); - OZ (spi_result->get_scheme_guard().get_schema_version(session->get_effective_tenant_id(), tenant_version)); - OZ (spi_result->get_scheme_guard().get_schema_version(OB_SYS_TENANT_ID, sys_version)); - OX (retry_ctrl.set_tenant_local_schema_version(tenant_version)); - OX (retry_ctrl.set_sys_local_schema_version(sys_version)); - OX (spi_result->get_sql_ctx().schema_guard_ = &spi_result->get_scheme_guard()); - if (OB_SUCC(ret)) { - WITH_CONTEXT(cursor.get_cursor_entity()) { - lib::ContextTLOptGuard guard(false); - if (OB_FAIL(inner_open(ctx, sql_str, ps_sql, stmt_type, exec_params, - *spi_result, spi_result->get_out_params()))) { - if (spi_result->get_result_set() != NULL) { - int cli_ret = OB_SUCCESS; - retry_ctrl.test_and_save_retry_state(GCTX, - spi_result->get_sql_ctx(), - *spi_result->get_result_set(), - ret, cli_ret, true, true, true); - LOG_WARN("fail to open, check if need retry", K(ret), K(cli_ret), - K(retry_ctrl.need_retry()), K(sql_str), K(ps_sql), K(exec_params)); - if (!cursor.is_ps_cursor()) { - ret = cli_ret; - spi_result->get_sql_ctx().clear(); - ctx->exec_ctx_->get_my_session()->set_session_in_retry(retry_ctrl.need_retry()); - } - } - } - } - } - OV (OB_NOT_NULL(spi_result->get_result_set()), - OB_ERR_UNEXPECTED, sql_stmt, ps_sql, exec_params); - CK (OB_NOT_NULL(spi_result->get_result_set()->get_field_columns())); - if (OB_SUCC(ret) && cursor.get_field_columns().empty()) { - const common::ColumnsFieldArray* field_column = - dynamic_cast - (spi_result->get_result_set()->get_field_columns()); - OX (cursor.get_field_columns().set_allocator(&cursor.get_dbms_entity()->get_arena_allocator())); - OZ (cursor.get_field_columns().assign(*field_column)); - } - OX (spi_result->get_result_set()->set_ps_protocol()); - if (OB_SUCCESS != ret && OB_NOT_NULL(spi_result)) { - int tmp_ret = ret; - ret = OB_SUCCESS; - if (OB_NOT_NULL(spi_result->get_result_set())) { - // 此分支所有错误码都被吞掉，最终返回最初的错误码 - int close_ret = spi_result->close_result_set(); - if (OB_SUCCESS != close_ret) { - LOG_WARN("close mysql result set failed", K(ret), K(close_ret)); - } - } - ret = tmp_ret; - spi_result->~ObSPIResultSet(); - LOG_WARN("cursor open result failed.", K(ret), K(sql_stmt), K(ps_sql), K(exec_params)); - } - OX (cursor.open(spi_result)); - if (OB_NOT_NULL(spi_result)) { - // no OX - spi_result->end_cursor_stmt(ctx, ret); - } - OZ (setup_cursor_snapshot_verify_(&cursor, spi_result)); - LOG_DEBUG("start process record", K(ret), K(ps_sql), K(sql_str), K(enable_sql_audit)); - //监控项统计结束 - time_record.set_exec_end_timestamp(ObTimeUtility::current_time()); - if (enable_perf_event) { - exec_record.record_end(); - } - - if (OB_NOT_NULL(spi_result) && - OB_NOT_NULL(spi_result->get_result_set()) && - spi_result->get_result_set()->is_inited()) { - ObSQLSessionInfo *session_info = ctx->exec_ctx_->get_my_session(); - int64_t try_cnt = session_info->get_raw_audit_record().try_cnt_; - ObExecRecord record_bk = session_info->get_raw_audit_record().exec_record_; - session_info->get_raw_audit_record().pl_trace_id_.set(trace_id_guard.origin_trace_id_); - // 会在inner_open的时候被改成了 inner ，所以这个地方需要重新设置一下 - exec_timestamp.exec_type_ = cursor.is_ps_cursor() ? sql::PSCursor : sql::DbmsCursor; - ObInnerSQLConnection::process_record(*spi_result->get_result_set(), - spi_result->get_sql_ctx(), - *session_info, - time_record, - ret, - session_info->get_current_execution_id(), - cursor.is_ps_cursor() ? cursor.get_id() : OB_INVALID_ID, - max_wait_desc, - total_wait_desc, - exec_record, - exec_timestamp, - true, - (exec_params.count() > 0 || cursor.is_ps_cursor()) ? ps_sql : sql_str, - true, - &exec_param_str); - session_info->get_raw_audit_record().exec_record_ = record_bk; - session_info->get_raw_audit_record().try_cnt_ = try_cnt; - session_info->get_raw_audit_record().pl_trace_id_.reset(); - } - } while (RETRY_TYPE_NONE != retry_ctrl.get_retry_type() && !cursor.is_ps_cursor()); - } else { - SMART_VAR(ObSPIResultSet, spi_result) { - ObSPICursor *spi_cursor = NULL; - uint64_t size = 0; - OZ (session->get_tmp_table_size(size)); - OZ (spi_result.init(*ctx->exec_ctx_->get_my_session())); - OZ (spi_result.start_nested_stmt_if_need(ctx, sql_stmt, static_cast(stmt_type), for_update), - sql_stmt, ps_sql, exec_params); - - if (OB_SUCC(ret)) { - ObQueryRetryCtrl retry_ctrl; - int64_t tenant_version = 0; - int64_t sys_version = 0; - int64_t retry_cnt = 0; - ObPLSPITraceIdGuard trace_id_guard(sql_stmt, ps_sql, *session, ret); - int64_t old_query_start_time = session->get_query_start_time(); - int64_t new_query_start_time = ObTimeUtility::current_time(); - if (!cursor.is_ps_cursor()) { - session->set_query_start_time(new_query_start_time); - } - do { - ret = OB_SUCCESS; - if (retry_cnt > 0) { - spi_result.get_out_params().reset(); - spi_result.reset_member_for_retry(*session); - } - retry_ctrl.clear_state_before_each_retry(session->get_retry_info_for_update()); - OZ (cursor.prepare_spi_cursor(spi_cursor, - session->get_effective_tenant_id(), - size, - false, - session)); - OZ (GCTX.schema_service_->get_tenant_schema_guard(session->get_effective_tenant_id(), spi_result.get_scheme_guard())); - OZ (spi_result.get_scheme_guard().get_schema_version(session->get_effective_tenant_id(), tenant_version)); - OZ (spi_result.get_scheme_guard().get_schema_version(OB_SYS_TENANT_ID, sys_version)); - OX (retry_ctrl.set_tenant_local_schema_version(tenant_version)); - OX (retry_ctrl.set_sys_local_schema_version(sys_version)); - OX (spi_result.get_sql_ctx().schema_guard_ = &spi_result.get_scheme_guard()); - if (OB_SUCC(ret)) { - OZ (inner_open(ctx, sql_str, ps_sql, stmt_type, exec_params, - spi_result, spi_result.get_out_params()), - sql_stmt, ps_sql, exec_params); - OZ (ObDbmsInfo::deep_copy_field_columns( - cursor.get_dbms_entity()->get_arena_allocator(), - spi_result.get_result_set()->get_field_columns(), - cursor.get_field_columns())); - if (OB_SUCC(ret)) { - if (OB_NOT_NULL(spi_result.get_result_set()) && OB_NOT_NULL(spi_result.get_result_set()->get_physical_plan())) { - ObPhysicalPlan *plan = spi_result.get_result_set()->get_physical_plan(); - cursor.set_packed(plan->is_packed()); - } - } - OZ (fill_cursor(*spi_result.get_result_set(), spi_cursor, new_query_start_time)); - if (OB_FAIL(ret) && OB_NOT_NULL(spi_result.get_result_set())) { - int cli_ret = OB_SUCCESS; - retry_ctrl.test_and_save_retry_state(GCTX, - spi_result.get_sql_ctx(), - *spi_result.get_result_set(), - ret, - cli_ret, - true, - true, - true); - LOG_WARN("failed to fill_cursor, check if need retry", - K(ret), K(cli_ret), K(retry_ctrl.need_retry()), - K(sql_stmt), K(ps_sql), K(exec_params)); - if (!cursor.is_ps_cursor()) { - ret = cli_ret; - spi_result.get_sql_ctx().clear(); - ctx->exec_ctx_->get_my_session()->set_session_in_retry(retry_ctrl.need_retry()); - } - } - } - OX (spi_cursor->row_store_.finish_add_row()); - LOG_DEBUG("start process record", K(ret), K(ps_sql), K(sql_str), K(enable_sql_audit)); - //监控项统计结束 - time_record.set_exec_end_timestamp(ObTimeUtility::current_time()); - if (enable_perf_event) { - exec_record.record_end(); - } - ObResultSet* result_set = spi_result.get_result_set(); - if (OB_NOT_NULL(result_set) && result_set->is_inited()) { - ObSQLSessionInfo *session_info = ctx->exec_ctx_->get_my_session(); - int64_t try_cnt = session_info->get_raw_audit_record().try_cnt_; - ObExecRecord record_bk = session_info->get_raw_audit_record().exec_record_; - session_info->get_raw_audit_record().try_cnt_ = retry_ctrl.get_retry_times(); - session_info->get_raw_audit_record().pl_trace_id_.set(trace_id_guard.origin_trace_id_); - // 会在inner_open的时候被改成了 inner ，所以这个地方需要重新设置一下 - exec_timestamp.exec_type_ = cursor.is_ps_cursor() ? sql::PSCursor : sql::DbmsCursor; - ObInnerSQLConnection::process_record(*result_set, - spi_result.get_sql_ctx(), - *session_info, - time_record, - ret, - session_info->get_current_execution_id(), - cursor.is_ps_cursor() ? cursor.get_id() : OB_INVALID_ID, - max_wait_desc, - total_wait_desc, - exec_record, - exec_timestamp, - true, - (exec_params.count() > 0 || cursor.is_ps_cursor()) ? ps_sql : sql_str, - true, - &exec_param_str); - session_info->get_raw_audit_record().exec_record_ = record_bk; - session_info->get_raw_audit_record().try_cnt_ = retry_cnt; - session_info->get_raw_audit_record().pl_trace_id_.reset(); - } - int close_ret = spi_result.close_result_set(); - if (OB_SUCCESS != close_ret) { - LOG_WARN("close mysql result failed", K(ret), K(close_ret)); - } - ret = OB_SUCCESS == ret ? close_ret : ret; - if (OB_SUCCESS != ret && OB_NOT_NULL(spi_cursor)) { - spi_cursor->~ObSPICursor(); - LOG_WARN("fill cursor failed.", K(ret), K(cursor.get_id()), K(sql_stmt), K(ps_sql), K(session->get_sessid())); - } - - retry_cnt++; - } while (!cursor.is_ps_cursor() && RETRY_TYPE_NONE != retry_ctrl.get_retry_type()); - if (!cursor.is_ps_cursor()) { - session->set_query_start_time(old_query_start_time); - } - } - - if (OB_SUCC(ret)) { - cursor.open(spi_cursor); - if (for_update) { - OX (cursor.set_for_update()); - OX (cursor.set_trans_id(ctx->exec_ctx_->get_my_session()->get_tx_id())); - } - if (hidden_rowid) { - OX (cursor.set_hidden_rowid()); - } - if (OB_SUCC(ret) && lib::is_oracle_mode()) { - OZ (setup_cursor_snapshot_verify_(&cursor, &spi_result)); - } - } - spi_result.end_nested_stmt_if_need(ctx, ret); - } + } else if (!for_update && use_stream) { // streaming branch + OZ (streaming_cursor_open(ctx, + cursor, + *session, + sql_str, + ps_sql, + stmt_type, + &exec_params, + exec_params.count(), + true, /*is_server_cursor*/ + for_update, + hidden_rowid, + true /*is_dbms_cursor*/)); + } else { // unstreaming branch + OZ (unstreaming_cursor_open(ctx, + cursor, + *session, + sql_str, + ps_sql, + stmt_type, + &exec_params, + exec_params.count(), + true, /*is_server_cursor*/ + for_update, + hidden_rowid, + true, /*is_dbms_cursor*/ + orc_max_ret_rows)); } SET_SPI_STATUS; return ret; @@ -4540,22 +3779,14 @@ int ObSPIService::do_cursor_fetch(ObPLExecCtx *ctx, bool is_type_record) { int ret = OB_SUCCESS; + ObSPIResultSet *spi_result = NULL; - ObSQLSessionInfo *session = ctx->exec_ctx_->get_my_session(); - ObExecRecord exec_record; - ObExecTimestamp exec_timestamp; - ObSPITimeRecord time_record; - exec_timestamp.exec_type_ = sql::CursorFetch; - ObWaitEventDesc max_wait_desc; - ObWaitEventStat total_wait_desc; - const bool enable_perf_event = lib::is_diagnose_info_enabled(); - const bool enable_sql_audit = GCONF.enable_sql_audit - && ctx->exec_ctx_->get_my_session()->get_local_ob_enable_sql_audit(); + ObSQLSessionInfo *session = NULL; + CK (OB_NOT_NULL(ctx)); CK (OB_NOT_NULL(ctx->exec_ctx_)); CK (OB_NOT_NULL(session = ctx->exec_ctx_->get_my_session())); CK (OB_NOT_NULL(cursor)); - ObPLSubPLSqlTimeGuard guard(ctx); // check cursor is valid for fetch if (OB_SUCC(ret) @@ -4572,13 +3803,13 @@ int ObSPIService::do_cursor_fetch(ObPLExecCtx *ctx, can't fetch if transaction committed, because snapshot data has been removed */ if (!cursor->get_snapshot().is_valid()) { - ret = OB_ERR_FETCH_OUT_SEQUENCE; - LOG_WARN("snapshot is invalid", K(cursor->get_snapshot()), K(ret)); + ret = OB_ERR_FETCH_OUT_SEQUENCE; + LOG_WARN("snapshot is invalid", K(cursor->get_snapshot()), K(ret)); } else if (cursor->is_for_update()) { if (cursor->get_snapshot().is_committed()) { - ret = OB_ERR_FETCH_OUT_SEQUENCE; - LOG_WARN("transaction has been committed, for update cursor can not fetch", - K(cursor->get_snapshot()), K(ret)); + ret = OB_ERR_FETCH_OUT_SEQUENCE; + LOG_WARN("transaction has been committed, for update cursor can not fetch", + K(cursor->get_snapshot()), K(ret)); } } else if (cursor->is_streaming()) { if(cursor->get_snapshot().is_committed()) { @@ -4588,11 +3819,6 @@ int ObSPIService::do_cursor_fetch(ObPLExecCtx *ctx, } } } - if (enable_perf_event) { - exec_record.record_start(); - } - //监控项统计开始 - time_record.set_send_timestamp(ObTimeUtility::current_time()); if (OB_FAIL(ret)) { } else if (!is_bulk && INT64_MAX != limit) { //limit子句必须和Bulk Collect合用 ret = OB_INVALID_ARGUMENT; @@ -4605,36 +3831,36 @@ int ObSPIService::do_cursor_fetch(ObPLExecCtx *ctx, LOG_USER_ERROR(OB_ER_SP_CURSOR_NOT_OPEN); LOG_WARN("Cursor is not open", K(cursor), K(ret)); } - if (OB_SUCC(ret)) { - if (cursor->is_streaming()) { - CK (OB_NOT_NULL(spi_result = cursor->get_cursor_handler())); - OZ (spi_result->set_cursor_env(*ctx->exec_ctx_->get_my_session())); - OZ (adjust_out_params(ctx, into_exprs, into_count, spi_result->get_out_params())); - } else if (OB_NOT_NULL(cursor->get_spi_cursor()) - && cursor->get_spi_cursor()->row_store_.get_row_cnt() > 0 - && cursor->get_current_row().is_invalid()) { //有数据才需要做，避免重复做 - //仅缓存在ObRowStore的Cursor需要初始化ObNewRow结构 - CK (OB_NOT_NULL(cursor->get_spi_cursor())); - int64_t column_count = cursor->get_spi_cursor()->row_desc_.count(); - ObIAllocator *spi_allocator = NULL; - OX (spi_allocator = NULL == cursor->get_cursor_entity() - ? cursor->get_allocator() - : &cursor->get_cursor_entity()->get_arena_allocator()); - CK (OB_NOT_NULL(spi_allocator)); - if (OB_SUCC(ret)) { - void *ptr = spi_allocator->alloc(column_count * sizeof(ObObj)); - if (OB_ISNULL(ptr)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("alloc memory for row failed", "size", column_count * sizeof(ObObj)); - } else { - cursor->get_current_row().cells_ = new(ptr) common::ObObj[column_count]; - cursor->get_current_row().count_ = column_count; - } + + if (OB_FAIL(ret)) { + } else if (cursor->is_streaming()) { + CK (OB_NOT_NULL(spi_result = cursor->get_cursor_handler())); + OZ (spi_result->set_cursor_env(*ctx->exec_ctx_->get_my_session())); + OZ (adjust_out_params(ctx, into_exprs, into_count, spi_result->get_out_params())); + } else if (OB_NOT_NULL(cursor->get_spi_cursor()) + && cursor->get_spi_cursor()->row_store_.get_row_cnt() > 0 + && cursor->get_current_row().is_invalid()) { //有数据才需要做，避免重复做 + //仅缓存在ObRowStore的Cursor需要初始化ObNewRow结构 + CK (OB_NOT_NULL(cursor->get_spi_cursor())); + int64_t column_count = cursor->get_spi_cursor()->row_desc_.count(); + ObIAllocator *spi_allocator = NULL; + OX (spi_allocator = NULL == cursor->get_cursor_entity() + ? cursor->get_allocator() + : &cursor->get_cursor_entity()->get_arena_allocator()); + CK (OB_NOT_NULL(spi_allocator)); + if (OB_SUCC(ret)) { + void *ptr = spi_allocator->alloc(column_count * sizeof(ObObj)); + if (OB_ISNULL(ptr)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc memory for row failed", "size", column_count * sizeof(ObObj)); + } else { + cursor->get_current_row().cells_ = new(ptr) common::ObObj[column_count]; + cursor->get_current_row().count_ = column_count; } } } + if (OB_SUCC(ret)) { - ObCurTraceId::TraceId pl_trace_id; int64_t row_count = 0; #define GET_RESULT \ @@ -4672,13 +3898,16 @@ int ObSPIService::do_cursor_fetch(ObPLExecCtx *ctx, } else { \ sql = spi_result->get_sql_ctx().cur_sql_; \ } \ - ObPLSPITraceIdGuard trace_id_guard( \ - sql, ps_sql, *session, ret, cursor->get_sql_trace_id()); \ + ObPLSqlAuditRecord audit_record(sql::CursorFetch); \ + ObQueryRetryCtrl retry_ctrl; \ + ObPLSPITraceIdGuard trace_id_guard(sql, ps_sql, *session, ret, cursor->get_sql_trace_id()); \ + ObPLSubPLSqlTimeGuard guard(ctx); \ + ObPLSqlAuditGuard audit_guard( \ + *(ctx->exec_ctx_), *session, *spi_result, audit_record, ret, ps_sql, retry_ctrl, trace_id_guard, spi_result->get_sql_ctx().stmt_type_); \ if (cursor->get_sql_trace_id()->is_invalid() \ && OB_NOT_NULL(ObCurTraceId::get_trace_id())) { \ cursor->get_sql_trace_id()->set(*ObCurTraceId::get_trace_id()); \ } \ - pl_trace_id = trace_id_guard.origin_trace_id_; \ ret = inner_fetch_with_retry(ctx, \ *cursor->get_cursor_handler(), \ into_exprs, \ @@ -4701,11 +3930,8 @@ int ObSPIService::do_cursor_fetch(ObPLExecCtx *ctx, } while(0) if (is_server_cursor) { -// FIXME: (yunxing.cyx) comment in streaming disabled, because report 4002, should be revert after enable streaming -// WITH_CONTEXT(cursor->get_cursor_entity()) { - lib::ContextTLOptGuard guard(false); - GET_RESULT; -// } + lib::ContextTLOptGuard guard(false); + GET_RESULT; } else { GET_RESULT; } @@ -4713,49 +3939,10 @@ int ObSPIService::do_cursor_fetch(ObPLExecCtx *ctx, #undef GET_RESULT if (cursor->is_streaming()) { - ObPLSPITraceIdGuard trace_id_guard(nullptr, nullptr, *session, ret, cursor->get_sql_trace_id()); - if (OB_ISNULL(spi_result)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("spi result must be not null in oracle mode", K(ret), K(spi_result)); - } else if (pl_trace_id.is_invalid() || OB_ISNULL(cursor->get_sql_trace_id()) || cursor->get_sql_trace_id()->is_invalid()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected invalid trace_id", K(ret), K(pl_trace_id), K(trace_id_guard.origin_trace_id_)); - } else { - LOG_DEBUG("start process record", K(ret), K(cursor->get_id()), K(enable_sql_audit)); - //监控项统计结束 - time_record.set_exec_end_timestamp(ObTimeUtility::current_time()); - if (enable_perf_event) { - exec_record.record_end(); - } - ObResultSet* result_set = spi_result->get_result_set(); - if (OB_NOT_NULL(result_set) && result_set->is_inited()) { - ObSQLSessionInfo *session_info = ctx->exec_ctx_->get_my_session(); - int64_t try_cnt = session_info->get_raw_audit_record().try_cnt_; - ObExecRecord record_bk = session_info->get_raw_audit_record().exec_record_; - session_info->get_raw_audit_record().pl_trace_id_.set(pl_trace_id); - ObInnerSQLConnection::process_record(*(result_set), - spi_result->get_sql_ctx(), - *session_info, - time_record, - ret, - session_info->get_current_execution_id(), - cursor->get_id(), // ps stmt id - max_wait_desc, - total_wait_desc, - exec_record, - exec_timestamp, - true, - ObString(), - true, - spi_result->get_exec_params_str_ptr()); - session_info->get_raw_audit_record().exec_record_ = record_bk; - session_info->get_raw_audit_record().try_cnt_ = try_cnt; - session_info->get_raw_audit_record().pl_trace_id_.reset(); - } - spi_result->end_cursor_stmt(ctx, ret); - cursor->set_last_execute_time(ObTimeUtility::current_time()); - } + spi_result->end_cursor_stmt(ctx, ret); + cursor->set_last_execute_time(ObTimeUtility::current_time()); } + // Oracle模式的Cursor会吞掉READ_NOTHING的错误, 为了避免无效日志过多, 只在Mysql模式下打印WARN if (OB_SUCC(ret) || (OB_READ_NOTHING == ret && lib::is_oracle_mode())) { @@ -4770,22 +3957,25 @@ int ObSPIService::do_cursor_fetch(ObPLExecCtx *ctx, return ret; } -int ObSPIService::dbms_cursor_fetch(ObPLExecCtx *ctx, - ObDbmsCursorInfo &cursor, - bool is_server_cursor) +int ObSPIService::dbms_cursor_fetch( + ObPLExecCtx *ctx, ObDbmsCursorInfo &cursor, bool is_server_cursor) { int ret = OB_SUCCESS; - OZ (do_cursor_fetch(ctx, - &cursor, - is_server_cursor,/*ps is server cursor*/ - NULL, - 0, - NULL, - 0, - NULL, - NULL, - false, - INT64_MAX)); + if (OB_FAIL(do_cursor_fetch(ctx, + &cursor, + is_server_cursor,/*ps is server cursor*/ + NULL, + 0, + NULL, + 0, + NULL, + NULL, + false, + INT64_MAX))) { + if (ret != OB_READ_NOTHING) { + LOG_WARN("failed to do cursor fetch", K(ret), K(is_server_cursor), K(cursor)); + } + } return ret; } @@ -4909,10 +4099,7 @@ int ObSPIService::spi_cursor_close(ObPLExecCtx *ctx, return ret; } -// 只 close dbms_cursor 中 cursor 相关内容 -// dbms_cursor 中的 param 信息不做处理 -int ObSPIService::dbms_cursor_close(ObExecContext &exec_ctx, - ObPLCursorInfo &cursor) +int ObSPIService::dbms_cursor_close(ObExecContext &exec_ctx, ObPLCursorInfo &cursor) { int ret = OB_SUCCESS; // dbms cursor与pl cursor不同： @@ -6875,64 +6062,83 @@ int ObSPIService::process_function_out_result(ObPLExecCtx *ctx, return ret; } -int ObSPIService::inner_open(ObPLExecCtx *ctx, - ObIAllocator ¶m_allocator, //用于拷贝执行期参数 - const char *sql, - const char *ps_sql, - int64_t type, - const ObSqlExpression **param_exprs, - int64_t param_count, - const ObSqlExpression **into_exprs, - int64_t into_count, - ObSPIResultSet &spi_result, - ObSPIOutParams &out_params, - observer::ObQueryRetryCtrl *retry_ctrl, - bool is_forall) +int ObSPIService::store_params_string( + ObPLExecCtx *ctx, ObSPIResultSet &spi_result, ParamStore *exec_params) { int ret = OB_SUCCESS; - int64_t query_num = 0; + if (OB_SUCC(ret) + && OB_NOT_NULL(ctx) + && OB_NOT_NULL(ctx->exec_ctx_) + && OB_NOT_NULL(ctx->exec_ctx_->get_my_session())) { + char *tmp_ptr = NULL; + int64_t tmp_len = 0; + OZ (ObMPStmtExecute::store_params_value_to_str(spi_result.get_memory_ctx()->get_arena_allocator(), + *ctx->exec_ctx_->get_my_session(), + exec_params, + tmp_ptr, + tmp_len)); + OX (spi_result.get_exec_params_str_ptr()->assign(tmp_ptr, tmp_len)); + } + return ret; +} + +int ObSPIService::prepare_static_sql_params(ObPLExecCtx *ctx, + ObIAllocator ¶m_allocator, //用于拷贝执行期参数 + const ObString &sql, + const ObString &ps_sql, + int64_t type, + const ObSqlExpression **params, + int64_t param_count, + const ObSqlExpression **into_exprs, + int64_t into_count, + ObSPIResultSet &spi_result, + ObSPIOutParams &out_params, + bool is_forall, + ParamStore *&curr_params) +{ + int ret = OB_SUCCESS; + int64_t array_binding_count = 0; - ParamStore exec_params( (ObWrapperAllocator(param_allocator)) ); - ParamStore *curr_params = &exec_params; + ParamStore *exec_params = NULL; ParamStore *batch_params = NULL; + const ObSqlExpression **param_exprs = reinterpret_cast(params); + + curr_params = NULL; + + if (OB_ISNULL(exec_params = reinterpret_cast(param_allocator.alloc(sizeof(ParamStore))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory for exec params", K(ret)); + } + OX (new (exec_params) ParamStore( (ObWrapperAllocator(param_allocator)) ) ); + OX (curr_params = exec_params); if (NULL == sql) { OZ (construct_exec_params(ctx, param_allocator, param_exprs, param_count, - into_exprs, into_count, exec_params, out_params, is_forall), - K(sql), K(ps_sql), K(type), K(param_count), K(out_params), K(exec_params)); - if (OB_SUCC(ret) - && OB_NOT_NULL(ctx) - && OB_NOT_NULL(ctx->exec_ctx_) - && OB_NOT_NULL(ctx->exec_ctx_->get_my_session())) { - // add exec_param_info for sql_audit - char *tmp_ptr = NULL; - int64_t tmp_len = 0; - OZ (ObMPStmtExecute::store_params_value_to_str(spi_result.get_memory_ctx()->get_arena_allocator(), - *ctx->exec_ctx_->get_my_session(), - &exec_params, - tmp_ptr, - tmp_len)); - OX (spi_result.get_exec_params_str_ptr()->assign(tmp_ptr, tmp_len)); - } + into_exprs, into_count, *exec_params, out_params, is_forall), + K(sql), K(ps_sql), K(type), K(param_count), K(out_params), KPC(exec_params)); + + OZ (store_params_string(ctx, spi_result, exec_params)); } - if (OB_SUCC(ret) && is_forall) { - for (int64_t i = 0; OB_SUCC(ret) && i < exec_params.count(); ++i) { - if (exec_params.at(i).is_ext()) { + if (OB_SUCC(ret) && is_forall && OB_NOT_NULL(exec_params)) { + + for (int64_t i = 0; OB_SUCC(ret) && i < exec_params->count(); ++i) { + if (exec_params->at(i).is_ext()) { pl::ObPLCollection *coll = NULL; - CK (OB_NOT_NULL(coll = reinterpret_cast(exec_params.at(i).get_ext()))); + CK (OB_NOT_NULL(coll = reinterpret_cast(exec_params->at(i).get_ext()))); if (OB_SUCC(ret)) { array_binding_count = coll->get_actual_count(); break; } } } + if (OB_SUCC(ret)) { if (array_binding_count <= 0) { ret = OB_ERR_UNEXPECTED; LOG_WARN("array_binding_count is wrong", K(array_binding_count), K(ret)); } else if (OB_FAIL(ObSQLUtils::transform_pl_ext_type( - exec_params, array_binding_count, param_allocator, batch_params, true))) { + *exec_params, array_binding_count, param_allocator, batch_params, true))) { LOG_WARN("transform failed", K(ret)); } else if (OB_ISNULL(batch_params)) { ret = OB_ERR_UNEXPECTED; @@ -6944,36 +6150,64 @@ int ObSPIService::inner_open(ObPLExecCtx *ctx, } } } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(inner_open(ctx, sql, ps_sql, type, - *curr_params, spi_result, - out_params))) { - if (retry_ctrl != nullptr/*can_retry*/) { - int cli_ret = OB_SUCCESS; - retry_ctrl->test_and_save_retry_state( - GCTX, - spi_result.get_sql_ctx(), - *spi_result.get_result_set(), - ret, cli_ret, true, true, true); - LOG_WARN("failed to get_result, check if need retry", - K(ret), K(cli_ret), K(retry_ctrl->need_retry()), K(sql), K(ps_sql), K(type)); - ret = cli_ret; - spi_result.get_sql_ctx().clear(); - ctx->exec_ctx_->get_my_session()->set_session_in_retry(retry_ctrl->need_retry()); + + return ret; +} + +int ObSPIService::inner_open(ObPLExecCtx *ctx, + ObIAllocator ¶m_allocator, + const ObString &sql, + const ObString &ps_sql, + int64_t type, + void *params, + int64_t param_count, + const ObSqlExpression **into_exprs, + int64_t into_count, + ObSPIResultSet &spi_result, + ObSPIOutParams &out_params, + bool is_forall, + bool is_dynamic_sql, + bool is_dbms_sql) +{ + int ret = OB_SUCCESS; + + ParamStore *curr_params = NULL; + + if (is_dynamic_sql) { + ObObjParam **dynamic_params = reinterpret_cast(params); + OZ (prepare_dynamic_sql_params(ctx, spi_result, param_allocator, param_count, dynamic_params, curr_params)); + } else if (is_dbms_sql) { + OX (curr_params = reinterpret_cast(params)); + OZ (store_params_string(ctx, spi_result, curr_params)); + } else { + const ObSqlExpression **static_params = reinterpret_cast(params); + OZ (prepare_static_sql_params(ctx, + param_allocator, + sql, + ps_sql, + type, + static_params, + param_count, + into_exprs, + into_count, + spi_result, + out_params, + is_forall, + curr_params)); + } + + CK (OB_NOT_NULL(curr_params)); + OZ (inner_open(ctx, sql, ps_sql, type, *curr_params, spi_result, out_params, is_dynamic_sql)); + + // if failed, we need release complex parameter memory in here + if (OB_FAIL(ret) && OB_NOT_NULL(curr_params) && !is_dbms_sql) { + int ret = OB_SUCCESS; // ignore destruct obj error + for (int64_t i = 0; i < curr_params->count(); ++i) { + OZ (ObUserDefinedType::destruct_obj(curr_params->at(i), ctx->exec_ctx_->get_my_session())); + ret = OB_SUCCESS; } } - // if failed, we need release complex parameter memory in here - if (OB_FAIL(ret) - && NULL == sql - && OB_NOT_NULL(ctx) - && OB_NOT_NULL(ctx->exec_ctx_) - && OB_NOT_NULL(ctx->exec_ctx_->get_my_session())) { - int ret = OB_SUCCESS; // ignore destruct obj error - for (int64_t i = 0; OB_SUCC(ret) && i < exec_params.count(); ++i) { - OZ (ObUserDefinedType::destruct_obj(exec_params.at(i), ctx->exec_ctx_->get_my_session())); - } - } return ret; } @@ -6983,12 +6217,13 @@ int ObSPIService::inner_open(ObPLExecCtx *ctx, int64_t type, ParamStore &exec_params, ObSPIResultSet &spi_result, - ObSPIOutParams &out_params) + ObSPIOutParams &out_params, + bool is_dynamic_sql) { int ret = OB_SUCCESS; - // unconditional adjustment OZ (adjust_out_params(*spi_result.get_result_set(), out_params)); + if (OB_ISNULL(ctx) || OB_ISNULL(ctx->exec_ctx_) || (NULL == ctx->allocator_)) { @@ -7016,19 +6251,24 @@ int ObSPIService::inner_open(ObPLExecCtx *ctx, } if (OB_SUCC(ret)) { WITH_CONTEXT(spi_result.get_memory_ctx()) { - if (NULL != sql.ptr()) { + if (exec_params.count() <= 0 && !sql.empty()) { spi_result.get_result_set()->set_user_sql(true); - if (OB_FAIL(GCTX.sql_engine_->handle_pl_execute( - sql, *session, exec_params, *spi_result.get_result_set(), spi_result.get_sql_ctx(), - false /* is_prepare_protocol */, false /* is_dynamic_sql*/))) { - LOG_WARN("query failed", K(ret), K(sql)); - } + OZ (GCTX.sql_engine_->handle_pl_execute(sql, + *session, + exec_params, + *spi_result.get_result_set(), + spi_result.get_sql_ctx(), + false /* is_prepare_protocol */, + false /* is_dynamic_sql*/), K(sql), K(ps_sql), K(exec_params)); } else { spi_result.get_result_set()->set_stmt_type(static_cast(type)); - OZ (GCTX.sql_engine_->handle_pl_execute( - ps_sql, *session, exec_params, *spi_result.get_result_set(), spi_result.get_sql_ctx(), - true /* is_prepare_protocol */, false /* is_dynamic_sql */), - K(ps_sql), K(exec_params)); + OZ (GCTX.sql_engine_->handle_pl_execute(ps_sql, + *session, + exec_params, + *spi_result.get_result_set(), + spi_result.get_sql_ctx(), + true /* is_prepare_protocol */, + is_dynamic_sql /* is_dynamic_sql */), K(sql), K(ps_sql), K(exec_params)); OZ (adjust_out_params(*spi_result.get_result_set(), out_params)); } } @@ -7043,7 +6283,7 @@ int ObSPIService::inner_open(ObPLExecCtx *ctx, } int ObSPIService::inner_fetch(ObPLExecCtx *ctx, - ObQueryRetryCtrl &retry_ctrl, + bool &can_retry, ObSPIResultSet &spi_result, const ObSqlExpression **into_exprs, int64_t into_count, @@ -7072,7 +6312,6 @@ int ObSPIService::inner_fetch(ObPLExecCtx *ctx, CK (OB_NOT_NULL(result_set)); if (OB_SUCC(ret)) { ObNewRow dummy_row; - bool can_retry = true; current_row = NULL != current_row ? current_row : &dummy_row; WITH_CONTEXT(spi_result.get_memory_ctx()) { if (OB_FAIL((get_result(ctx, @@ -7097,22 +6336,11 @@ int ObSPIService::inner_fetch(ObPLExecCtx *ctx, return_types, return_type_count, is_type_record)))) { - if (can_retry) { - int cli_ret = OB_SUCCESS; - retry_ctrl.test_and_save_retry_state( - GCTX, - spi_result.get_sql_ctx(), - *result_set, ret, cli_ret, true, true, true); - if (!for_cursor || (for_cursor && ret != OB_READ_NOTHING)) { - LOG_WARN("failed to get_result, check if need retry", - K(ret), K(cli_ret), K(retry_ctrl.need_retry())); - } - ret = cli_ret; + if (!for_cursor || (for_cursor && ret != OB_READ_NOTHING)) { + LOG_WARN("failed to get_result, check if need retry", K(ret)); } } } - spi_result.get_sql_ctx().clear(); - ctx->exec_ctx_->get_my_session()->set_session_in_retry(retry_ctrl.need_retry()); } return ret; } @@ -7138,16 +6366,11 @@ int ObSPIService::inner_fetch_with_retry(ObPLExecCtx *ctx, { int ret = OB_SUCCESS; ObQueryRetryCtrl retry_ctrl; - int64_t tenant_version = 0; - int64_t sys_version = 0; - share::schema::ObSchemaGetterGuard schema_guard; ObSQLSessionInfo *session = NULL; ObResultSet *result_set = spi_result.get_result_set(); - int64_t query_timeout = 0; CK (OB_NOT_NULL(ctx)); CK (OB_NOT_NULL(ctx->exec_ctx_)); CK (OB_NOT_NULL(session = ctx->exec_ctx_->get_my_session())); - OZ (session->get_query_timeout(query_timeout)); CK (OB_NOT_NULL(result_set)); if (OB_SUCC(ret)) { int64_t time_gap = ObTimeUtility::current_time() - last_exec_time; @@ -7165,46 +6388,38 @@ int ObSPIService::inner_fetch_with_retry(ObPLExecCtx *ctx, if (OB_SUCC(ret)) { do { ret = OB_SUCCESS; - retry_ctrl.clear_state_before_each_retry(session->get_retry_info_for_update()); - if (THIS_WORKER.is_timeout()) { - ret = OB_TIMEOUT; - LOG_WARN("inner fetch with retry already timeout!", - K(ret), - K(old_timeout_ts), K(query_timeout), - K(min_timeout_ts), K(time_gap), - K(THIS_WORKER.get_timeout_ts()), K(old_query_start_time)); - break; + bool can_retry = true; + ObSPIRetryCtrlGuard retry_guard(retry_ctrl, spi_result, *session, ret, true); + if (FAILEDx(inner_fetch(ctx, + can_retry, + spi_result, + into_exprs, + into_count, + column_types, + type_count, + exprs_not_null_flag, + pl_integer_ranges, + NULL, /*out_using_params*/ + row_count, + is_bulk, + false, + false, /*is_dynamic_sql*/ + ¤t_row, + has_hidden_rowid, + for_cursor, + limit, + return_types, + return_type_count, + is_type_record))) { + if (!for_cursor || (for_cursor && ret != OB_READ_NOTHING)) { + LOG_WARN("failed to get_result, check if need retry", K(ret)); + } } - OZ (GCTX.schema_service_->get_tenant_schema_guard( - session->get_effective_tenant_id(), schema_guard)); - OZ (schema_guard.get_schema_version(session->get_effective_tenant_id(), tenant_version)); - OZ (schema_guard.get_schema_version(OB_SYS_TENANT_ID, sys_version)); - OX (retry_ctrl.set_tenant_local_schema_version(tenant_version)); - OX (retry_ctrl.set_sys_local_schema_version(sys_version)); - if (OB_SUCC(ret)) { - ret = inner_fetch(ctx, - retry_ctrl, - spi_result, - into_exprs, - into_count, - column_types, - type_count, - exprs_not_null_flag, - pl_integer_ranges, - NULL, /*out_using_params*/ - row_count, - is_bulk, - false, - false, /*is_dynamic_sql*/ - ¤t_row, - has_hidden_rowid, - for_cursor, - limit, - return_types, - return_type_count, - is_type_record); + if (can_retry) { + retry_guard.test(); } // NOTE: cursor fetch failed can not retry, we only use this to refresh location cache. + // actully, this function only called by cursor, so `for_cursor` just in case. } while (RETRY_TYPE_NONE != retry_ctrl.get_retry_type() && !for_cursor); session->get_retry_info_for_update().clear(); session->set_query_start_time(old_query_start_time); @@ -8669,8 +7884,10 @@ int ObSPIService::store_datum(int64_t ¤t_addr, const ObObj &obj, ObSQLSess return ret; } -int ObSPIService::fill_cursor(ObResultSet &result_set, ObSPICursor *cursor, int64_t new_query_start_time) -{ +int ObSPIService::fill_cursor(ObResultSet &result_set, + ObSPICursor *cursor, + int64_t new_query_start_time, + int64_t orc_max_ret_rows) { int ret = OB_SUCCESS; int64_t old_time_out_ts = THIS_WORKER.get_timeout_ts(); if (OB_ISNULL(cursor) || OB_ISNULL(cursor->allocator_)) { @@ -8700,9 +7917,14 @@ int ObSPIService::fill_cursor(ObResultSet &result_set, ObSPICursor *cursor, int6 K(ret)); } } - while (OB_SUCC(ret)) { + for (int64_t i = 0; OB_SUCC(ret) && i < orc_max_ret_rows; i++) { if (OB_FAIL(result_set.get_next_row(row))) { - //break + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + } else { + LOG_WARN("read result error", K(ret)); + } + break; } else if (OB_ISNULL(row)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get a invalud row", K(ret)); @@ -8725,11 +7947,6 @@ int ObSPIService::fill_cursor(ObResultSet &result_set, ObSPICursor *cursor, int6 } } } - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - } else { - LOG_WARN("read result error", K(ret)); - } } THIS_WORKER.set_timeout_ts(old_time_out_ts); return ret; @@ -9688,5 +8905,71 @@ ObPLSubPLSqlTimeGuard::~ObPLSubPLSqlTimeGuard() } } +ObSPIRetryCtrlGuard::ObSPIRetryCtrlGuard( + ObQueryRetryCtrl &retry_ctrl, ObSPIResultSet &spi_result, ObSQLSessionInfo &session_info, int &ret, bool for_fetch) + : retry_ctrl_(retry_ctrl), spi_result_(spi_result), session_info_(session_info), ret_(ret), init_(false) +{ + int64_t tenant_version = 0; + int64_t sys_version = 0; + uint64_t eff_tenant_id = session_info_.get_effective_tenant_id(); + if (!for_fetch) { + spi_result_.get_out_params().reset(); + spi_result_.reset_member_for_retry(session_info_); + } + retry_ctrl_.clear_state_before_each_retry(session_info_.get_retry_info_for_update()); + if (THIS_WORKER.is_timeout()) { + ret = OB_TIMEOUT; + LOG_WARN("already timeout!", K(ret)); + } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(eff_tenant_id, spi_result_.get_scheme_guard()))) { + LOG_WARN("get schema guard failed", K(ret)); + } else if (OB_FAIL(spi_result_.get_scheme_guard().get_schema_version(eff_tenant_id, tenant_version))) { + LOG_WARN("fail get schema version", K(ret)); + } else if (OB_FAIL(spi_result_.get_scheme_guard().get_schema_version(OB_SYS_TENANT_ID, sys_version))) { + LOG_WARN("fail get sys schema version", K(ret)); + } else { + retry_ctrl_.set_tenant_local_schema_version(tenant_version); + retry_ctrl_.set_sys_local_schema_version(sys_version); + spi_result_.get_sql_ctx().schema_guard_ = &spi_result.get_scheme_guard(); + init_ = true; + } +} + +ObSPIRetryCtrlGuard::~ObSPIRetryCtrlGuard() +{ +} + +void ObSPIRetryCtrlGuard::test() +{ + int &ret = ret_; + if (init_ && OB_FAIL(ret) && spi_result_.get_result_set() != NULL) { + int cli_ret = OB_SUCCESS; + retry_ctrl_.test_and_save_retry_state(GCTX, + spi_result_.get_sql_ctx(), + *(spi_result_.get_result_set()), + ret, + cli_ret, + true, /*force_local_retry in SPI, we can only do local retry*/ + false, /*is_inner_sql*/ + true); /*is_part_of_pl_sql*/ + ret = cli_ret; + spi_result_.get_sql_ctx().clear(); + session_info_.set_session_in_retry(retry_ctrl_.need_retry()); + } +} + +ObSPIExecEnvGuard::ObSPIExecEnvGuard(ObSQLSessionInfo &session_info, ObSPIResultSet &spi_result) + : session_info_(session_info), spi_result_(spi_result) +{ + query_start_time_bk_ = session_info.get_query_start_time(); + session_info.set_query_start_time(ObTimeUtility::current_time()); + +} + +ObSPIExecEnvGuard::~ObSPIExecEnvGuard() +{ + session_info_.get_retry_info_for_update().clear(); + session_info_.set_query_start_time(query_start_time_bk_); +} + } } diff --git a/src/sql/ob_spi.h b/src/sql/ob_spi.h index 530f0e248..1bdd9fb76 100644 --- a/src/sql/ob_spi.h +++ b/src/sql/ob_spi.h @@ -35,6 +35,7 @@ using common::ObPsStmtId; namespace pl { class ObDbmsCursorInfo; +class ObPLSqlCodeInfo; } namespace sql @@ -58,6 +59,40 @@ struct ObPLSPITraceIdGuard int &ret_; }; +class ObSPIRetryCtrlGuard +{ +public: + ObSPIRetryCtrlGuard( + observer::ObQueryRetryCtrl &retry_ctrl, + ObSPIResultSet &spi_result, + ObSQLSessionInfo &session_info, + int &ret, + bool for_fetch = false); + ~ObSPIRetryCtrlGuard(); + + void test(); + +private: + observer::ObQueryRetryCtrl &retry_ctrl_; + ObSPIResultSet &spi_result_; + ObSQLSessionInfo &session_info_; + pl::ObPLSqlCodeInfo save_sqlcode_info_; + int &ret_; + bool init_; +}; + +class ObSPIExecEnvGuard +{ +public: + ObSPIExecEnvGuard(ObSQLSessionInfo &session_info, ObSPIResultSet &spi_result); + ~ObSPIExecEnvGuard(); +private: + ObSQLSessionInfo &session_info_; + ObSPIResultSet &spi_result_; + int64_t query_start_time_bk_; + pl::ObPLSqlCodeInfo *sqlcode_info_bk_; +}; + struct ObSPICursor { ObSPICursor(ObIAllocator &allocator, sql::ObSQLSessionInfo* session_info) : @@ -201,7 +236,7 @@ public: sql::ObSqlCtx &get_sql_ctx() { return sql_ctx_; } ObResultSet *get_result_set() { return result_set_; } ObSPIOutParams &get_out_params() { return out_params_; } - ObIAllocator &get_allocaor() { return allocator_; } + ObIAllocator &get_allocator() { return allocator_; } int destruct_exec_params(ObSQLSessionInfo &session); private: enum EndStmtType @@ -512,6 +547,55 @@ public: bool is_returning = false, bool is_type_record = false); + static int check_dynamic_sql_legal(pl::ObPLExecCtx *ctx, + ObSqlString &sql_str, + stmt::StmtType stmt_type, + int64_t into_count, + int64_t inner_into_count, + common::ObObjParam **params, + int64_t param_count, + const int64_t *params_mode, + bool is_returning, + bool for_update, + int64_t &exec_param_cnt, + common::ObIArray &out_using_params); + + static int prepare_dynamic_sql_params(pl::ObPLExecCtx *ctx, + ObSPIResultSet &spi_result, + common::ObIAllocator &allocator, + int64_t exec_param_cnt, + ObObjParam **params, + ParamStore *&exec_params); + + static int inner_open(pl::ObPLExecCtx *ctx, + ObIAllocator ¶m_allocator, //用于拷贝执行期参数 + const ObString &sql, + const ObString &ps_sql, + int64_t type, + void *params, + int64_t param_count, + const ObSqlExpression **into_exprs, + int64_t into_count, + ObSPIResultSet &spi_result, + ObSPIOutParams &out_params, + bool is_forall, + bool is_dynamic_sql, + bool is_dbms_sql); + + static int prepare_static_sql_params(pl::ObPLExecCtx *ctx, + ObIAllocator ¶m_allocator, + const ObString &sql, + const ObString &ps_sql, + int64_t type, + const ObSqlExpression **params, + int64_t param_count, + const ObSqlExpression **into_exprs, + int64_t into_count, + ObSPIResultSet &spi_result, + ObSPIOutParams &out_params, + bool is_forall, + ParamStore *&curr_params); + static int spi_get_subprogram_cursor_info(pl::ObPLExecCtx *ctx, uint64_t package_id, uint64_t routine_id, @@ -581,7 +665,8 @@ public: const ObString &ps_sql, int64_t stmt_type, bool for_update, - bool has_hidden_rowid); + bool has_hidden_rowid, + int64_t orc_max_ret_rows = INT64_MAX); static int spi_dynamic_open(pl::ObPLExecCtx *ctx, const int64_t sql_idx, const int64_t *sql_param_exprs_idx, @@ -591,7 +676,8 @@ public: int64_t cursor_index); static int dbms_dynamic_open(pl::ObPLExecCtx *ctx, pl::ObDbmsCursorInfo &cursor, - bool is_dbms_sql = false); + bool is_dbms_sql = false, + int64_t orc_max_ret_rows = INT64_MAX); static int dbms_cursor_fetch(pl::ObPLExecCtx *ctx, pl::ObDbmsCursorInfo &cursor, bool is_server_cursor = false); @@ -807,10 +893,14 @@ public: int64_t stmt_type, ParamStore &exec_params, ObSPIResultSet &spi_result, - ObSPIOutParams &out_params); + ObSPIOutParams &out_params, + bool is_dynamic_sql = false); static void adjust_pl_status_for_xa(sql::ObExecContext &ctx, int &result); - static int fill_cursor(ObResultSet &result_set, ObSPICursor *cursor, int64_t new_query_start_time); + static int fill_cursor(ObResultSet &result_set, + ObSPICursor *cursor, + int64_t new_query_start_time, + int64_t orc_max_ret_rows = INT64_MAX); static int spi_opaque_assign_null(int64_t opaque_ptr); @@ -897,7 +987,7 @@ private: const char *sql, const char *ps_sql, int64_t type, - const ObSqlExpression **param_exprs, + void *params, int64_t param_count, const ObSqlExpression **into_exprs, int64_t into_count, @@ -908,7 +998,10 @@ private: int64_t is_bulk, bool is_forall = false, bool is_type_record = false, - bool for_update = false); + bool for_update = false, + bool is_dynamic_sql = false, + ObIArray *using_out_params = nullptr, + bool is_dbms_sql = false); static int dbms_cursor_execute(pl::ObPLExecCtx *ctx, const ObString ps_sql, @@ -940,22 +1033,8 @@ private: ObSPIOutParams &out_params, bool is_forall = false); - static int inner_open(pl::ObPLExecCtx *ctx, - ObIAllocator ¶m_allocator, //用于拷贝执行期参数 - const char* sql, - const char* ps_sql, - int64_t type, - const ObSqlExpression **param_exprs, - int64_t param_count, - const ObSqlExpression **into_exprs, - int64_t into_count, - ObSPIResultSet &spi_result, - ObSPIOutParams &out_params, - observer::ObQueryRetryCtrl *retry_ctrl = nullptr, - bool is_forall = false); - static int inner_fetch(pl::ObPLExecCtx *ctx, - observer::ObQueryRetryCtrl &retry_ctrl, + bool &can_retry, ObSPIResultSet &spi_result, const ObSqlExpression **into_exprs, int64_t into_count, @@ -1136,9 +1215,11 @@ private: static int calc_dynamic_sqlstr( pl::ObPLExecCtx *ctx, const ObSqlExpression *sql, ObSqlString &sqlstr); - static int dynamic_out_params( - common::ObIAllocator &allocator, - ObResultSet *result, common::ObObjParam **params, int64_t param_count); + static int dynamic_out_params(common::ObIAllocator &allocator, + ObResultSet *result, + void *params, + int64_t param_count, + bool is_dbms_sql); static int cursor_close_impl(pl::ObPLExecCtx *ctx, pl::ObPLCursorInfo *cursor, @@ -1177,7 +1258,35 @@ private: const ObSqlExpression **actual_param_exprs, int64_t cursor_param_count); static bool is_sql_type_into_pl(ObObj &dest_addr, ObIArray &obj_array); -private: + + static int streaming_cursor_open(pl::ObPLExecCtx *ctx, + pl::ObPLCursorInfo &cursor, + ObSQLSessionInfo &session_info, + const ObString &sql, + const ObString &ps_sql, + int64_t type, + void *params, + int64_t sql_param_count, + bool is_server_cursor, + bool is_for_update, + bool has_hidden_rowid, + bool is_dbms_cursor = false); + + static int unstreaming_cursor_open(pl::ObPLExecCtx *ctx, + pl::ObPLCursorInfo &cursor, + ObSQLSessionInfo &session_info, + const ObString &sql, + const ObString &ps_sql, + int64_t type, + void *params, + int64_t sql_param_count, + bool is_server_cursor, + bool for_update, + bool has_hidden_rowid, + bool is_dbms_cursor = false, + int64_t orc_max_ret_rows = INT64_MAX); + static int store_params_string(pl::ObPLExecCtx *ctx, ObSPIResultSet &spi_result, ParamStore *exec_params); + static int setup_cursor_snapshot_verify_(pl::ObPLCursorInfo *cursor, ObSPIResultSet *spi_result); }; diff --git a/src/sql/session/ob_basic_session_info.cpp b/src/sql/session/ob_basic_session_info.cpp index 7aab1c45d..00c995d8b 100644 --- a/src/sql/session/ob_basic_session_info.cpp +++ b/src/sql/session/ob_basic_session_info.cpp @@ -2404,6 +2404,12 @@ OB_INLINE int ObBasicSessionInfo::process_session_variable(ObSysVarClassType var } break; } + case SYS_VAR__ORACLE_SQL_SELECT_LIMIT: { + int64_t int_val = 0; + OZ (val.get_int(int_val), val); + OX (sys_vars_cache_.set_oracle_sql_select_limit(int_val)); + break; + } case SYS_VAR_AUTO_INCREMENT_OFFSET: { uint64_t uint_val = 0; OZ (val.get_uint64(uint_val), val); @@ -2969,6 +2975,12 @@ int ObBasicSessionInfo::fill_sys_vars_cache_base_value( OX (sys_vars_cache.set_base_sql_select_limit(int_val)); break; } + case SYS_VAR__ORACLE_SQL_SELECT_LIMIT: { + int64_t int_val = 0; + OZ (val.get_int(int_val), val); + OX (sys_vars_cache.set_base_oracle_sql_select_limit(int_val)); + break; + } case SYS_VAR_AUTO_INCREMENT_OFFSET: { uint64_t uint_val = 0; OZ (val.get_uint64(uint_val), val); diff --git a/src/sql/session/ob_basic_session_info.h b/src/sql/session/ob_basic_session_info.h index 26e96d940..11ef06b2a 100644 --- a/src/sql/session/ob_basic_session_info.h +++ b/src/sql/session/ob_basic_session_info.h @@ -480,6 +480,7 @@ public: uint64_t get_local_auto_increment_increment() const; uint64_t get_local_auto_increment_offset() const; uint64_t get_local_last_insert_id() const; + void set_local_ob_enable_pl_cache(bool v) { sys_vars_cache_.set_ob_enable_pl_cache(v); } bool get_local_ob_enable_pl_cache() const; bool get_local_ob_enable_plan_cache() const; bool get_local_ob_enable_sql_audit() const; @@ -651,6 +652,11 @@ public: sql_select_limit = sys_vars_cache_.get_sql_select_limit(); return common::OB_SUCCESS; } + int get_oracle_sql_select_limit(int64_t &oracle_sql_select_limit) const + { + oracle_sql_select_limit = sys_vars_cache_.get_oracle_sql_select_limit(); + return common::OB_SUCCESS; + } // session保留compatible mode，主要用于传递mode，方便后续进行guard切换，如inner sql connection等 // 其他需要用mode地方请尽量使用线程上的is_oracle|mysql_mode // 同时可以使用check_compatibility_mode来检查线程与session上的mode是否一致 @@ -1667,6 +1673,7 @@ public: sql_throttle_current_priority_(100), ob_last_schema_version_(0), sql_select_limit_(0), + oracle_sql_select_limit_(0), auto_increment_offset_(0), last_insert_id_(0), binlog_row_image_(2), @@ -1729,6 +1736,7 @@ public: sql_throttle_current_priority_ = 100; ob_last_schema_version_ = 0; sql_select_limit_ = 0; + oracle_sql_select_limit_ = 0; auto_increment_offset_ = 0; last_insert_id_ = 0; binlog_row_image_ = 2; @@ -1789,6 +1797,7 @@ public: sql_throttle_current_priority_ == other.sql_throttle_current_priority_ && ob_last_schema_version_ == other.ob_last_schema_version_ && sql_select_limit_ == other.sql_select_limit_ && + oracle_sql_select_limit_ == other.oracle_sql_select_limit_ && auto_increment_offset_ == other.auto_increment_offset_ && last_insert_id_ == other.last_insert_id_ && binlog_row_image_ == other.binlog_row_image_ && @@ -1945,7 +1954,7 @@ public: K(ob_org_cluster_id_), K(ob_query_timeout_), K(ob_trx_timeout_), K(collation_connection_), K(sql_mode_), K(nls_formats_[0]), K(nls_formats_[1]), K(nls_formats_[2]), K(ob_trx_idle_timeout_), K(ob_trx_lock_timeout_), K(nls_collation_), K(nls_nation_collation_), - K_(sql_throttle_current_priority), K_(ob_last_schema_version), K_(sql_select_limit), + K_(sql_throttle_current_priority), K_(ob_last_schema_version), K_(sql_select_limit), K_(oracle_sql_select_limit), K_(optimizer_use_sql_plan_baselines), K_(optimizer_capture_sql_plan_baselines), K_(is_result_accurate), K_(character_set_results), K_(character_set_connection), K_(ob_pl_block_timeout), K_(ob_plsql_ccflags), @@ -1958,6 +1967,7 @@ public: int64_t sql_throttle_current_priority_; int64_t ob_last_schema_version_; int64_t sql_select_limit_; + int64_t oracle_sql_select_limit_; uint64_t auto_increment_offset_; uint64_t last_insert_id_; int64_t binlog_row_image_; @@ -2085,6 +2095,7 @@ private: DEF_SYS_VAR_CACHE_FUNCS(int64_t, sql_throttle_current_priority); DEF_SYS_VAR_CACHE_FUNCS(int64_t, ob_last_schema_version); DEF_SYS_VAR_CACHE_FUNCS(int64_t, sql_select_limit); + DEF_SYS_VAR_CACHE_FUNCS(int64_t, oracle_sql_select_limit); DEF_SYS_VAR_CACHE_FUNCS(uint64_t, auto_increment_offset); DEF_SYS_VAR_CACHE_FUNCS(uint64_t, last_insert_id); DEF_SYS_VAR_CACHE_FUNCS(int64_t, binlog_row_image); @@ -2160,6 +2171,7 @@ private: bool inc_sql_throttle_current_priority_:1; bool inc_ob_last_schema_version_:1; bool inc_sql_select_limit_:1; + bool inc_oracle_sql_select_limit_:1; bool inc_auto_increment_offset_:1; bool inc_last_insert_id_:1; bool inc_binlog_row_image_:1; From c2d549c096967cc1188706063abca699a8180b3d Mon Sep 17 00:00:00 2001 From: YangEfei Date: Thu, 15 Aug 2024 07:56:41 +0000 Subject: [PATCH 060/249] [TABLELOCK] fix bugs about DBMS_LOCK --- .../tablelock/ob_lock_func_executor.cpp | 46 +++++----- src/storage/tablelock/ob_lock_func_executor.h | 4 +- .../ob_lock_inner_connection_util.cpp | 89 +++++-------------- 3 files changed, 48 insertions(+), 91 deletions(-) diff --git a/src/storage/tablelock/ob_lock_func_executor.cpp b/src/storage/tablelock/ob_lock_func_executor.cpp index 4720294e2..490e81466 100644 --- a/src/storage/tablelock/ob_lock_func_executor.cpp +++ b/src/storage/tablelock/ob_lock_func_executor.cpp @@ -209,24 +209,31 @@ int ObLockFuncContext::open_inner_conn_() { int ret = OB_SUCCESS; ObSQLSessionInfo *session = my_exec_ctx_->get_my_session(); + observer::ObInnerSQLConnection *inner_conn = nullptr; if (OB_ISNULL(session) || OB_ISNULL(sql_proxy_ = my_exec_ctx_->get_sql_proxy())) { ret = OB_NOT_INIT; LOG_WARN("session or sql_proxy is NULL", K(ret), KP(session), KP(sql_proxy_)); - } else if (OB_NOT_NULL(inner_conn_ = static_cast(session->get_inner_conn()))) { - // do nothing. - } else if (OB_FAIL(ObInnerConnectionLockUtil::create_inner_conn(session, my_exec_ctx_->get_sql_proxy(), inner_conn_))) { - LOG_WARN("create inner connection failed", K(ret), KPC(session)); - } else if (OB_ISNULL(inner_conn_)) { + } else if (OB_NOT_NULL(inner_conn_) || OB_NOT_NULL(store_inner_conn_)) { ret = OB_ERR_UNEXPECTED; - LOG_ERROR("inner connection is still null", KPC(session)); + LOG_WARN("inner_conn_ or store_inner_conn_ should be null", K(ret), KP(inner_conn_), KP(store_inner_conn_)); + } else if (OB_FAIL(ObInnerConnectionLockUtil::create_inner_conn(session, sql_proxy_, inner_conn))) { + LOG_WARN("create inner connection failed", K(ret), KPC(session)); + } else if (OB_ISNULL(inner_conn)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("inner connection is still null", KPC(session)); } else { /** * session is the only data struct which can pass through multi layer nested sql, * so we put inner conn in session to share it within multi layer nested sql. */ - session->set_inner_conn(inner_conn_); - need_close_conn_ = true; + inner_conn_ = inner_conn; + store_inner_conn_ = static_cast(session->get_inner_conn()); + session->set_inner_conn(inner_conn); + LOG_DEBUG("ObLockFuncContext::open_inner_conn_ successfully", + KP(inner_conn_), + KP(store_inner_conn_), + K(inner_conn_->is_oracle_compat_mode())); } return ret; } @@ -234,19 +241,18 @@ int ObLockFuncContext::open_inner_conn_() int ObLockFuncContext::close_inner_conn_() { int ret = OB_SUCCESS; - if (need_close_conn_) { - ObSQLSessionInfo *session = my_exec_ctx_->get_my_session(); - if (OB_ISNULL(sql_proxy_) || OB_ISNULL(session) || OB_ISNULL(inner_conn_)) { - ret = OB_NOT_INIT; - LOG_WARN("sql_proxy or inner_conn of session is NULL", K(ret), KP(sql_proxy_), KP(session), KP(inner_conn_)); - } else { - OZ (sql_proxy_->close(inner_conn_, true)); - OX (session->set_inner_conn(NULL)); - } - need_close_conn_ = false; + ObSQLSessionInfo *session = my_exec_ctx_->get_my_session(); + + if (OB_ISNULL(sql_proxy_) || OB_ISNULL(session) || OB_ISNULL(inner_conn_)) { + ret = OB_NOT_INIT; + LOG_WARN("sql_proxy or inner_conn of session is NULL", K(ret), KP(sql_proxy_), KP(session), KP(inner_conn_)); + } else { + session->set_inner_conn(store_inner_conn_); // restore inner_conn to session before close the tmp inner_conn + OZ (sql_proxy_->close(inner_conn_, true)); } - sql_proxy_ = NULL; - inner_conn_ = NULL; + sql_proxy_ = nullptr; + inner_conn_ = nullptr; + store_inner_conn_ = nullptr; return ret; } diff --git a/src/storage/tablelock/ob_lock_func_executor.h b/src/storage/tablelock/ob_lock_func_executor.h index b3178323e..9a7843e33 100644 --- a/src/storage/tablelock/ob_lock_func_executor.h +++ b/src/storage/tablelock/ob_lock_func_executor.h @@ -56,7 +56,6 @@ public: { reset_autocommit_ = false; has_inner_dml_write_ = false; - need_close_conn_ = false; have_saved_session_ = false; has_autonomous_tx_ = false; old_worker_timeout_ts_ = 0; @@ -67,6 +66,7 @@ public: database_name_.reset(); sql_proxy_ = nullptr; inner_conn_ = nullptr; + store_inner_conn_ = nullptr; session_info_ = nullptr; my_exec_ctx_ = nullptr; saved_session_.reset(); @@ -97,7 +97,6 @@ private: friend class ObGetLockExecutor; bool reset_autocommit_; bool has_inner_dml_write_; - bool need_close_conn_; bool have_saved_session_; bool has_autonomous_tx_; int64_t old_worker_timeout_ts_; @@ -108,6 +107,7 @@ private: ObSqlString database_name_; common::ObMySQLProxy *sql_proxy_; observer::ObInnerSQLConnection *inner_conn_; + observer::ObInnerSQLConnection *store_inner_conn_; sql::ObSQLSessionInfo *session_info_; sql::ObExecContext *my_exec_ctx_; //my exec context sql::ObBasicSessionInfo::TransSavedValue saved_session_; diff --git a/src/storage/tablelock/ob_lock_inner_connection_util.cpp b/src/storage/tablelock/ob_lock_inner_connection_util.cpp index 0c5946bb0..0601699bc 100644 --- a/src/storage/tablelock/ob_lock_inner_connection_util.cpp +++ b/src/storage/tablelock/ob_lock_inner_connection_util.cpp @@ -462,40 +462,25 @@ int ObInnerConnectionLockUtil::create_inner_conn(sql::ObSQLSessionInfo *session_ observer::ObInnerSQLConnection *&inner_conn) { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - ObObj mysql_mode; - ObObj current_mode; - mysql_mode.set_int(0); - current_mode.set_int(-1); - observer::ObInnerSQLConnectionPool *pool = nullptr; common::sqlclient::ObISQLConnection *conn = nullptr; - lib::CompatModeGuard guard(lib::Worker::CompatMode::MYSQL); if (OB_ISNULL(session_info) || OB_ISNULL(sql_proxy)) { ret = OB_NOT_INIT; LOG_WARN("session or sql_proxy is NULL", KP(session_info), KP(sql_proxy)); - } else if (OB_NOT_NULL(inner_conn = static_cast(session_info->get_inner_conn()))) { - LOG_INFO("session has had inner connection, no need to create again", KPC(session_info)); } else if (OB_ISNULL(pool = static_cast(sql_proxy->get_pool()))) { ret = OB_NOT_INIT; LOG_WARN("connection pool is NULL", K(ret)); - } else if (OB_FAIL(session_info->get_sys_variable(share::SYS_VAR_OB_COMPATIBILITY_MODE, current_mode))) { - LOG_WARN("can not get the compat_mode", KPC(session_info)); - } else if (current_mode != mysql_mode - && OB_FAIL(session_info->update_sys_variable(share::SYS_VAR_OB_COMPATIBILITY_MODE, mysql_mode))) { - LOG_WARN("update session_info to msyql_mode failed", KR(ret), KPC(session_info)); } else if (common::sqlclient::INNER_POOL != pool->get_type()) { LOG_WARN("connection pool type is not inner", K(ret), K(pool->get_type())); - } else if (OB_FAIL(pool->acquire(session_info, conn))) { + // NOTICE: although we can set is_oracle_mode here, internally it will prioritize referencing + // the system variables on the session, so this parameter actually has no effect + } else if (OB_FAIL(pool->acquire(session_info, conn, false /* is_oracle_mode */))) { LOG_WARN("acquire connection from inner sql connection pool failed", KR(ret), KPC(session_info)); } else { inner_conn = static_cast(conn); - } - if (current_mode != mysql_mode && current_mode.get_int() != -1 - && OB_TMP_FAIL(session_info->update_sys_variable(share::SYS_VAR_OB_COMPATIBILITY_MODE, current_mode))) { - ret = OB_SUCCESS == ret ? tmp_ret : ret; - LOG_WARN("failed to update sys variable for compatibility mode", K(current_mode), KPC(session_info)); + // the inner_connection must be mysql_mode, so that we can write inner_table + inner_conn->set_mysql_compat_mode(); } return ret; } @@ -505,71 +490,38 @@ int ObInnerConnectionLockUtil::execute_write_sql(observer::ObInnerSQLConnection int64_t &affected_rows) { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - sql::ObSQLSessionInfo *session = nullptr; - ObObj current_mode; - ObObj mysql_mode; - mysql_mode.set_int(0); - current_mode.set_int(-1); - // If we want to executre dml with mysql_mode, thread worker and session need to meet 2 conditions: - // 1. The thread worker is in mysql_mode, it depends on the connection itself, we cannot modify it; - // 2. The sys_variable of session should be mysql_mode. Otherwise, it will trigger defensive check. + // NOTICE: This will be overwritten by the oracle_made_ field on connection, + // but for safety reason, we still set up a compat_guard here. + lib::CompatModeGuard guard(lib::Worker::CompatMode::MYSQL); + if (OB_ISNULL(conn)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("inner_conn is nullptr", K(sql)); - } else if (OB_ISNULL(session = &conn->get_session())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("session_info is nullptr", K(sql)); - } else if (OB_FAIL(session->get_sys_variable(share::SYS_VAR_OB_COMPATIBILITY_MODE, current_mode))) { - LOG_WARN("can not get the compat_mode", K(sql)); - } else if (current_mode != mysql_mode - && OB_FAIL(session->update_sys_variable(share::SYS_VAR_OB_COMPATIBILITY_MODE, mysql_mode))) { - LOG_WARN("update compat_mode to mysql_mode failed", K(sql)); + LOG_ERROR("inner_conn is nullptr", K(ret), K(sql)); } else if (OB_FAIL(conn->execute_write(MTL_ID(), sql.ptr(), affected_rows))) { - LOG_WARN("execute write for inner_sql failed", K(sql)); - } - if (current_mode != mysql_mode && current_mode.get_int() != -1 - && OB_TMP_FAIL(session->update_sys_variable(share::SYS_VAR_OB_COMPATIBILITY_MODE, current_mode))) { - ret = OB_SUCCESS == ret ? tmp_ret : ret; - LOG_WARN("failed to update sys variable for compatibility mode", K(current_mode), K(sql)); + LOG_WARN("execute write sql failed", K(ret), K(sql)); } + LOG_DEBUG("ObInnerConnectionLockUtil::execute_write_sql", K(ret), KP(conn), K(conn->is_oracle_compat_mode())); return ret; } + int ObInnerConnectionLockUtil::execute_read_sql(observer::ObInnerSQLConnection *conn, const ObSqlString &sql, ObISQLClient::ReadResult &res) { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - sql::ObSQLSessionInfo *session = nullptr; - ObObj current_mode; - ObObj mysql_mode; - mysql_mode.set_int(0); - current_mode.set_int(-1); - // If we want to executre dml with mysql_mode, thread worker and session need to meet 2 conditions: - // 1. The thread worker is in mysql_mode, it depends on the connection itself, we cannot modify it; - // 2. The sys_variable of session should be mysql_mode. Otherwise, it will trigger defensive check. + // NOTICE: This will be overwritten by the oracle_made_ field on connection, + // but for safety reason, we still set up a compat_guard here. + lib::CompatModeGuard guard(lib::Worker::CompatMode::MYSQL); + if (OB_ISNULL(conn)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("inner_conn is nullptr", K(sql)); - } else if (OB_ISNULL(session = &conn->get_session())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("session_info is nullptr", K(sql)); - } else if (OB_FAIL(session->get_sys_variable(share::SYS_VAR_OB_COMPATIBILITY_MODE, current_mode))) { - LOG_WARN("can not get the compat_mode", K(sql)); - } else if (current_mode != mysql_mode - && OB_FAIL(session->update_sys_variable(share::SYS_VAR_OB_COMPATIBILITY_MODE, mysql_mode))) { - LOG_WARN("update compat_mode to mysql_mode failed", K(sql)); + LOG_ERROR("inner_conn is nullptr", K(ret), K(sql)); } else if (OB_FAIL(conn->execute_read(MTL_ID(), sql.ptr(), res))) { - LOG_WARN("execute write for inner_sql failed", K(sql)); - } - if (current_mode != mysql_mode && current_mode.get_int() != -1 - && OB_TMP_FAIL(session->update_sys_variable(share::SYS_VAR_OB_COMPATIBILITY_MODE, current_mode))) { - ret = OB_SUCCESS == ret ? tmp_ret : ret; - LOG_WARN("failed to update sys variable for compatibility mode", K(current_mode), K(sql)); + LOG_WARN("execute read sql failed", K(ret), K(sql)); } + LOG_DEBUG("ObInnerConnectionLockUtil::execute_read_sql", K(ret), KP(conn), K(conn->is_oracle_compat_mode())); return ret; } @@ -967,7 +919,6 @@ int ObInnerConnectionLockUtil::get_org_cluster_id_(ObSQLSessionInfo *session, in } return ret; } - #undef REQUEST_LOCK_4_1 } // tablelock } // transaction From 36140f84c18bb6bf149785d6d68c8ee0b32f7588 Mon Sep 17 00:00:00 2001 From: wu-xingying <729224612@qq.com> Date: Thu, 15 Aug 2024 08:46:37 +0000 Subject: [PATCH 061/249] placehold in master --- deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h | 2 ++ src/share/ob_ddl_common.h | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h index 09dbf33e8..d2e7e8c7d 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h @@ -883,6 +883,8 @@ PCODE_DEF(OB_DETECTOR_LCL_MESSAGE, 0x9F0) PCODE_DEF(OB_DETECTOR_COLLECT_INFO_MESSAGE, 0x9F1) PCODE_DEF(OB_DETECTOR_NOTIFY_PARENT_MESSAGE, 0x9F2) +PCODE_DEF(OB_CHECK_AND_CANCEL_DELETE_LOB_META_ROW_DAG, 0x9F3) + PCODE_DEF(OB_RPC_ASSEMBLE, 0x1000) // Table API (by zhuweng.yzf) diff --git a/src/share/ob_ddl_common.h b/src/share/ob_ddl_common.h index e8a27fa23..4ded62933 100644 --- a/src/share/ob_ddl_common.h +++ b/src/share/ob_ddl_common.h @@ -182,6 +182,8 @@ enum ObDDLTaskStatus { REBUILD_SCHEMA = 35, SWITCH_INDEX_NAME = 36, WRITE_SPLIT_START_LOG = 37, + DROP_AUX_INDEX_TABLE = 38, + DROP_LOB_META_ROW = 39, FAIL = 99, SUCCESS = 100 }; @@ -328,6 +330,12 @@ static const char* ddl_task_status_to_str(const ObDDLTaskStatus &task_status) { case ObDDLTaskStatus::WRITE_SPLIT_START_LOG: str = "WRITE_SPLIT_START_LOG"; break; + case ObDDLTaskStatus::DROP_AUX_INDEX_TABLE: + str = "DROP_AUX_INDEX_TABLE"; + break; + case ObDDLTaskStatus::DROP_LOB_META_ROW: + str = "DROP_LOB_META_ROW"; + break; case ObDDLTaskStatus::FAIL: str = "FAIL"; break; From b873814fa09b6a01d80fef45968598b4bc30905f Mon Sep 17 00:00:00 2001 From: obdev Date: Thu, 15 Aug 2024 09:04:35 +0000 Subject: [PATCH 062/249] fix slave mapping bug --- src/sql/engine/px/ob_dfo_scheduler.cpp | 2 +- src/sql/engine/px/ob_px_util.cpp | 5 +-- src/sql/optimizer/ob_join_order.cpp | 45 +++++++++++++++++--------- src/sql/optimizer/ob_join_order.h | 3 +- 4 files changed, 36 insertions(+), 19 deletions(-) diff --git a/src/sql/engine/px/ob_dfo_scheduler.cpp b/src/sql/engine/px/ob_dfo_scheduler.cpp index 66fa5e927..b8b7f681c 100644 --- a/src/sql/engine/px/ob_dfo_scheduler.cpp +++ b/src/sql/engine/px/ob_dfo_scheduler.cpp @@ -307,7 +307,7 @@ int ObSerialDfoScheduler::init_all_dfo_channel(ObExecContext &ctx) const } else if (parent->is_root_dfo() && !parent->is_thread_inited() && OB_FAIL(ObPXServerAddrUtil::alloc_by_local_distribution(ctx, *parent))) { LOG_WARN("fail to alloc local distribution", K(ret)); - } else if (!parent->is_root_dfo() && + } else if (!parent->is_root_dfo() && !parent->is_thread_inited() && ObPQDistributeMethod::PARTITION_HASH == child->get_dist_method()) { if (OB_FAIL(ObPXServerAddrUtil::alloc_by_reference_child_distribution( coord_info_.pruning_table_location_, diff --git a/src/sql/engine/px/ob_px_util.cpp b/src/sql/engine/px/ob_px_util.cpp index 2d63c832c..0e0f5fe70 100644 --- a/src/sql/engine/px/ob_px_util.cpp +++ b/src/sql/engine/px/ob_px_util.cpp @@ -3826,14 +3826,15 @@ int ObDtlChannelUtil::get_sm_transmit_dtl_channel_set( ObAddr &dst_addr = ch_total_info.receive_exec_server_.exec_addrs_.at(0); bool is_local = true; int64_t chid = 0; - for (int64_t i = 0; i < transmit_task_cnt && OB_SUCC(ret); ++i) { + for (int64_t i = 0; i < receive_task_cnt && OB_SUCC(ret); ++i) { ObDtlChannelInfo ch_info; chid = ch_total_info.start_channel_id_ + receive_task_cnt * task_id + i; ObDtlChannelGroup::make_transmit_channel(ch_total_info.tenant_id_, dst_addr, chid, ch_info, is_local); OZ(ch_set.add_channel_info(ch_info)); } } - LOG_DEBUG("get sm receive dtl channel set", K(sqc_id), K(task_id), K(ch_total_info), K(ch_set)); + LOG_DEBUG("get sm receive dtl channel set", K(sqc_id), K(task_id), K(transmit_task_cnt), + K(receive_task_cnt), K(ch_total_info), K(ch_set)); return ret; } diff --git a/src/sql/optimizer/ob_join_order.cpp b/src/sql/optimizer/ob_join_order.cpp index 357a87542..898b226f7 100644 --- a/src/sql/optimizer/ob_join_order.cpp +++ b/src/sql/optimizer/ob_join_order.cpp @@ -9302,6 +9302,7 @@ int ObJoinOrder::generate_hash_paths(const EqualSets &equal_sets, int ret = OB_SUCCESS; ObSEArray left_best_paths; ObSEArray right_best_paths; + bool can_slave_mapping = false; if (OB_FAIL(find_minimal_cost_path(left_paths, left_best_paths))) { LOG_WARN("failed to find minimal cost path", K(ret)); } else if (OB_FAIL(find_minimal_cost_path(right_paths, right_best_paths))) { @@ -9334,7 +9335,8 @@ int ObJoinOrder::generate_hash_paths(const EqualSets &equal_sets, HASH_JOIN, false, naaj_info.is_naaj_, - dist_method))) { + dist_method, + can_slave_mapping))) { LOG_WARN("failed to get distributed join method", K(ret)); } else { LOG_TRACE("succeed to get distributed hash join method", K(dist_method)); @@ -9346,7 +9348,7 @@ int ObJoinOrder::generate_hash_paths(const EqualSets &equal_sets, right_path, path_info.join_type_, dist_algo, - path_info.force_slave_mapping_, + can_slave_mapping, join_conditions, join_filters, join_quals, @@ -9533,6 +9535,7 @@ int ObJoinOrder::generate_inner_nl_paths(const EqualSets &equal_sets, { int ret = OB_SUCCESS; int64_t dist_method = 0; + bool can_slave_mapping = false; if (OB_UNLIKELY(left_paths.empty()) || OB_ISNULL(left_paths.at(0)) || OB_ISNULL(right_path)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(left_paths.count()), K(right_path), K(ret)); @@ -9546,7 +9549,8 @@ int ObJoinOrder::generate_inner_nl_paths(const EqualSets &equal_sets, NESTED_LOOP_JOIN, true, false, - dist_method))) { + dist_method, + can_slave_mapping))) { LOG_WARN("failed to get distributed join method", K(ret)); } else if (dist_method == 0) { /*do nothing*/ @@ -9562,7 +9566,7 @@ int ObJoinOrder::generate_inner_nl_paths(const EqualSets &equal_sets, right_path, path_info.join_type_, dist_algo, - path_info.force_slave_mapping_, + can_slave_mapping, on_conditions, where_conditions, has_equal_cond, @@ -9593,6 +9597,7 @@ int ObJoinOrder::generate_normal_nl_paths(const EqualSets &equal_sets, Path *left_path = NULL; ObJoinOrder *left_tree = NULL; int64_t dist_method = 0; + bool can_slave_mapping = false; if (OB_UNLIKELY(left_paths.empty()) || OB_ISNULL(left_paths.at(0)) || OB_ISNULL(left_tree = left_paths.at(0)->parent_) || OB_ISNULL(right_path) || OB_ISNULL(right_path->get_sharding())) { @@ -9608,7 +9613,8 @@ int ObJoinOrder::generate_normal_nl_paths(const EqualSets &equal_sets, NESTED_LOOP_JOIN, false, false, - dist_method))) { + dist_method, + can_slave_mapping))) { LOG_WARN("failed to get distributed join method", K(ret)); } else if (dist_method == 0) { /*do nothing*/ @@ -9645,7 +9651,7 @@ int ObJoinOrder::generate_normal_nl_paths(const EqualSets &equal_sets, right_path, path_info.join_type_, dist_algo, - path_info.force_slave_mapping_, + can_slave_mapping, on_conditions, where_conditions, has_equal_cond, @@ -9657,7 +9663,7 @@ int ObJoinOrder::generate_normal_nl_paths(const EqualSets &equal_sets, right_path, path_info.join_type_, dist_algo, - path_info.force_slave_mapping_, + can_slave_mapping, on_conditions, where_conditions, has_equal_cond, @@ -9682,7 +9688,8 @@ int ObJoinOrder::get_distributed_join_method(Path &left_path, const JoinAlgo join_algo, const bool is_push_down, const bool is_naaj, - int64_t &distributed_methods) + int64_t &distributed_methods, + bool &can_slave_mapping) { int ret = OB_SUCCESS; bool is_basic = false; @@ -9699,6 +9706,12 @@ int ObJoinOrder::get_distributed_join_method(Path &left_path, distributed_methods = path_info.distributed_methods_; bool use_shared_hash_join = right_path.parallel_ > ObGlobalHint::DEFAULT_PARALLEL; ObSQLSessionInfo *session = NULL; + int64_t max_path_parallel = max(left_path.parallel_, right_path.parallel_); + can_slave_mapping = + path_info.force_slave_mapping_ && max_path_parallel > ObGlobalHint::DEFAULT_PARALLEL; + if (path_info.force_slave_mapping_ && !can_slave_mapping) { + OPT_TRACE("Disable slave mapping because parallel is 1"); + } if (OB_ISNULL(get_plan()) || OB_ISNULL(left_sharding = left_path.get_sharding()) || OB_ISNULL(session = get_plan()->get_optimizer_context().get_session_info()) || OB_ISNULL(right_sharding = right_path.get_sharding()) || @@ -9910,7 +9923,7 @@ int ObJoinOrder::get_distributed_join_method(Path &left_path, if (OB_SUCC(ret) && ((distributed_methods & DIST_PARTITION_NONE) || (distributed_methods & DIST_HASH_NONE) - || ((distributed_methods & DIST_BROADCAST_NONE) && path_info.force_slave_mapping_))) { + || ((distributed_methods & DIST_BROADCAST_NONE) && can_slave_mapping))) { target_part_keys.reuse(); if (OB_FAIL(right_sharding->get_all_partition_keys(target_part_keys, true))) { LOG_WARN("failed to get partition keys", K(ret)); @@ -9926,7 +9939,7 @@ int ObJoinOrder::get_distributed_join_method(Path &left_path, if (OB_SUCC(ret) && ((distributed_methods & DIST_NONE_PARTITION) || (distributed_methods & DIST_NONE_HASH) - || ((distributed_methods & DIST_NONE_BROADCAST) && path_info.force_slave_mapping_))) { + || ((distributed_methods & DIST_NONE_BROADCAST) && can_slave_mapping))) { target_part_keys.reuse(); if (OB_FAIL(left_sharding->get_all_partition_keys(target_part_keys, true))) { LOG_WARN("failed to get partition keys", K(ret)); @@ -10026,13 +10039,13 @@ int ObJoinOrder::get_distributed_join_method(Path &left_path, } if (OB_SUCC(ret) && (distributed_methods & DIST_BROADCAST_NONE) - && path_info.force_slave_mapping_ && !is_right_match_repart) { + && can_slave_mapping && !is_right_match_repart) { OPT_TRACE("force slave mapping and right path not meet repart, prune broadcast none method"); distributed_methods &= ~DIST_BROADCAST_NONE; } if (OB_SUCC(ret) && (distributed_methods & DIST_NONE_BROADCAST) - && path_info.force_slave_mapping_ && !is_left_match_repart) { + && can_slave_mapping && !is_left_match_repart) { OPT_TRACE("force slave mapping and left path not meet repart, prune none broadcast method"); distributed_methods &= ~DIST_NONE_BROADCAST; } @@ -10196,6 +10209,7 @@ int ObJoinOrder::generate_mj_paths(const EqualSets &equal_sets, bool best_need_sort = false; ObSEArray best_order_items; ObSEArray adjusted_join_conditions; + bool can_slave_mapping = false; if (OB_UNLIKELY(left_paths.empty() || OB_ISNULL(left_paths.at(0))) || OB_UNLIKELY(right_paths.empty()) || OB_ISNULL(right_paths.at(0)) || OB_ISNULL(get_plan())) { ret = OB_ERR_UNEXPECTED; @@ -10210,7 +10224,8 @@ int ObJoinOrder::generate_mj_paths(const EqualSets &equal_sets, MERGE_JOIN, false, false, - dist_method))) { + dist_method, + can_slave_mapping))) { LOG_WARN("failed to get distributed join method", K(ret)); } else if (0 == dist_method) { /*do nothing*/ @@ -10236,7 +10251,7 @@ int ObJoinOrder::generate_mj_paths(const EqualSets &equal_sets, right_join_keys, right_paths, dist_algo, - path_info.force_slave_mapping_, + can_slave_mapping, best_order_items, right_path, best_need_sort, @@ -10248,7 +10263,7 @@ int ObJoinOrder::generate_mj_paths(const EqualSets &equal_sets, right_path, path_info.join_type_, dist_algo, - path_info.force_slave_mapping_, + can_slave_mapping, merge_key->order_directions_, adjusted_join_conditions, other_join_conditions, diff --git a/src/sql/optimizer/ob_join_order.h b/src/sql/optimizer/ob_join_order.h index 2e5a17bf5..4a69e5885 100644 --- a/src/sql/optimizer/ob_join_order.h +++ b/src/sql/optimizer/ob_join_order.h @@ -2026,7 +2026,8 @@ struct NullAwareAntiJoinInfo { const JoinAlgo join_algo, const bool is_push_down, const bool is_naaj, - int64_t &distributed_types); + int64_t &distributed_types, + bool &can_slave_mapping); bool is_partition_wise_valid(const Path &left_path, const Path &right_path); From d358b254370562f8bfb2f8ad90e55205fc4c8222 Mon Sep 17 00:00:00 2001 From: cqliang1995 Date: Thu, 15 Aug 2024 09:28:01 +0000 Subject: [PATCH 063/249] fix serialize bug in of external_properties_. --- src/share/schema/ob_table_schema.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/share/schema/ob_table_schema.cpp b/src/share/schema/ob_table_schema.cpp index 9c864bf91..252524550 100644 --- a/src/share/schema/ob_table_schema.cpp +++ b/src/share/schema/ob_table_schema.cpp @@ -6842,12 +6842,12 @@ OB_DEF_SERIALIZE(ObTableSchema) OB_UNIS_ENCODE(mlog_tid_); OB_UNIS_ENCODE(auto_increment_cache_size_); - OB_UNIS_ENCODE(local_session_vars_); - OB_UNIS_ENCODE(duplicate_read_consistency_); if (OB_SUCC(ret)) { LST_DO_CODE(OB_UNIS_ENCODE, external_properties_); } + OB_UNIS_ENCODE(local_session_vars_); + OB_UNIS_ENCODE(duplicate_read_consistency_); if (OB_SUCC(ret)) { OB_UNIS_ENCODE(index_params_); } @@ -7281,12 +7281,12 @@ OB_DEF_DESERIALIZE(ObTableSchema) OB_UNIS_DECODE(mlog_tid_); OB_UNIS_DECODE(auto_increment_cache_size_); - OB_UNIS_DECODE(local_session_vars_); - OB_UNIS_DECODE(duplicate_read_consistency_); if (OB_SUCC(ret)) { LST_DO_CODE(OB_UNIS_DECODE, external_properties_); } + OB_UNIS_DECODE(local_session_vars_); + OB_UNIS_DECODE(duplicate_read_consistency_); if (OB_SUCC(ret)) { OB_UNIS_DECODE(index_params); @@ -7444,9 +7444,9 @@ OB_DEF_SERIALIZE_SIZE(ObTableSchema) OB_UNIS_ADD_LEN(max_used_column_group_id_); OB_UNIS_ADD_LEN(mlog_tid_); OB_UNIS_ADD_LEN(auto_increment_cache_size_); + OB_UNIS_ADD_LEN(external_properties_); OB_UNIS_ADD_LEN(local_session_vars_); OB_UNIS_ADD_LEN(duplicate_read_consistency_); - OB_UNIS_ADD_LEN(external_properties_); OB_UNIS_ADD_LEN(index_params_); return len; } From c189ffd7226373d32333886bc5867d7b31787bb2 Mon Sep 17 00:00:00 2001 From: JinmaoLi Date: Thu, 15 Aug 2024 10:12:15 +0000 Subject: [PATCH 064/249] fix foreign_primary_join check bug --- .../ob_transform_eliminate_outer_join.cpp | 1 + .../rewrite/ob_transform_join_elimination.cpp | 2 + src/sql/rewrite/ob_transform_utils.cpp | 98 ++++++++++--------- src/sql/rewrite/ob_transform_utils.h | 6 ++ 4 files changed, 63 insertions(+), 44 deletions(-) diff --git a/src/sql/rewrite/ob_transform_eliminate_outer_join.cpp b/src/sql/rewrite/ob_transform_eliminate_outer_join.cpp index 5d513cafc..cf200e81f 100644 --- a/src/sql/rewrite/ob_transform_eliminate_outer_join.cpp +++ b/src/sql/rewrite/ob_transform_eliminate_outer_join.cpp @@ -472,6 +472,7 @@ int ObTransformEliminateOuterJoin::can_be_eliminated_with_foreign_primary_join(O right_col_exprs, ctx_->schema_checker_, ctx_->session_info_, + true, is_foreign_primary_join, is_first_table_parent, foreign_key_info))) { diff --git a/src/sql/rewrite/ob_transform_join_elimination.cpp b/src/sql/rewrite/ob_transform_join_elimination.cpp index 7a5377f0e..bb23928ae 100644 --- a/src/sql/rewrite/ob_transform_join_elimination.cpp +++ b/src/sql/rewrite/ob_transform_join_elimination.cpp @@ -1510,6 +1510,7 @@ int ObTransformJoinElimination::check_transform_validity_foreign_key(const ObDML target_exprs, ctx_->schema_checker_, ctx_->session_info_, + false, is_foreign_primary_join, is_first_table_parent, foreign_key_info))) { @@ -3202,6 +3203,7 @@ int ObTransformJoinElimination::check_transform_validity_foreign_key(const ObDML target_exprs, ctx_->schema_checker_, ctx_->session_info_, + false, is_foreign_primary_join, is_first_table_parent, foreign_key_info))) { diff --git a/src/sql/rewrite/ob_transform_utils.cpp b/src/sql/rewrite/ob_transform_utils.cpp index c09f89bbc..65809be5c 100644 --- a/src/sql/rewrite/ob_transform_utils.cpp +++ b/src/sql/rewrite/ob_transform_utils.cpp @@ -3849,6 +3849,7 @@ int ObTransformUtils::check_foreign_primary_join(const TableItem *first_table, const ObIArray &second_exprs, ObSchemaChecker *schema_checker, ObSQLSessionInfo *session_info, + bool allow_partial_join, bool &is_foreign_primary_join, bool &is_first_table_parent, ObForeignKeyInfo *&foreign_key_info) @@ -3875,13 +3876,13 @@ int ObTransformUtils::check_foreign_primary_join(const TableItem *first_table, if (child_id == first_table->ref_id_ && parent_id == second_table->ref_id_) { // first table is child table is_first_table_parent = false; - if (OB_FAIL(is_all_foreign_key_involved(first_exprs, second_exprs, cur_info, find))) { + if (OB_FAIL(is_all_foreign_key_involved(first_exprs, second_exprs, cur_info, allow_partial_join, find))) { LOG_WARN("failed to check is all foreign key involved", K(ret)); } } else if (child_id == second_table->ref_id_ && parent_id == first_table->ref_id_) { // second table is child table is_first_table_parent = true; - if (OB_FAIL(is_all_foreign_key_involved(second_exprs, first_exprs, cur_info, find))) { + if (OB_FAIL(is_all_foreign_key_involved(second_exprs, first_exprs, cur_info, allow_partial_join, find))) { LOG_WARN("failed to check is all foreign key involved", K(ret)); } } @@ -3900,6 +3901,7 @@ int ObTransformUtils::check_foreign_primary_join(const TableItem *first_table, const ObIArray &second_exprs, ObSchemaChecker *schema_checker, ObSQLSessionInfo *session_info, + bool allow_partial_join, bool &is_foreign_primary_join, bool &is_first_table_parent, ObForeignKeyInfo *&foreign_key_info) @@ -3926,13 +3928,13 @@ int ObTransformUtils::check_foreign_primary_join(const TableItem *first_table, if (child_id == first_table->ref_id_ && parent_id == second_table->ref_id_) { // first table is child table is_first_table_parent = false; - if (OB_FAIL(is_all_foreign_key_involved(first_exprs, second_exprs, cur_info, find))) { + if (OB_FAIL(is_all_foreign_key_involved(first_exprs, second_exprs, cur_info, allow_partial_join, find))) { LOG_WARN("failed to check is all foreign key involved", K(ret)); } } else if (child_id == second_table->ref_id_ && parent_id == first_table->ref_id_) { // second table is child table is_first_table_parent = true; - if (OB_FAIL(is_all_foreign_key_involved(second_exprs, first_exprs, cur_info, find))) { + if (OB_FAIL(is_all_foreign_key_involved(second_exprs, first_exprs, cur_info, allow_partial_join, find))) { LOG_WARN("failed to check is all foreign key involved", K(ret)); } } @@ -3948,43 +3950,47 @@ int ObTransformUtils::check_foreign_primary_join(const TableItem *first_table, int ObTransformUtils::is_all_foreign_key_involved(const ObIArray &child_exprs, const ObIArray &parent_exprs, const ObForeignKeyInfo &info, + bool allow_partial_join, bool &is_all_involved) { int ret = OB_SUCCESS; + bool valid = true; is_all_involved = false; - const int64_t N = info.child_column_ids_.count(); - // generate stmt时会进行去重，不会出现t1.c1 = t2.c1 and t1.c1 = t2.c1的情况 - if (OB_UNLIKELY(child_exprs.count() != parent_exprs.count())) { + ObSEArray matched_idxs; + if (OB_UNLIKELY(child_exprs.count() != parent_exprs.count()) || + OB_UNLIKELY(info.child_column_ids_.count() != info.parent_column_ids_.count())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("child exprs and parent exprs should have equal size", K(ret), K(child_exprs.count()), K(parent_exprs.count())); - } else if (N == child_exprs.count()) { - int64_t match = 0; - for (int64_t i = 0; i < N; ++i) { - bool find = false; - const ObRawExpr *child_expr = child_exprs.at(i); - const ObRawExpr *parent_expr = parent_exprs.at(i); - if (OB_ISNULL(child_expr) || OB_ISNULL(parent_expr)) { + } else { + for (int64_t i = 0; OB_SUCC(ret) && valid && i < child_exprs.count(); i++) { + if (OB_ISNULL(child_exprs.at(i)) || OB_ISNULL(parent_exprs.at(i))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("child expr or parent expr is null", K(ret), K(child_expr), K(parent_expr)); - } else if (OB_UNLIKELY(!child_expr->has_flag(IS_COLUMN) || !parent_expr->has_flag(IS_COLUMN))) { + LOG_WARN("child expr or parent expr is null", K(ret), K(child_exprs.at(i)), K(parent_exprs.at(i))); + } else if (OB_UNLIKELY(!child_exprs.at(i)->is_column_ref_expr() || !parent_exprs.at(i)->is_column_ref_expr())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("not column expr", K(ret)); } else{ const ObColumnRefRawExpr *child_col = static_cast(child_exprs.at(i)); const ObColumnRefRawExpr *parent_col = static_cast(parent_exprs.at(i)); - for (int64_t j = 0; !find && j < N; ++j) { - if(parent_col->get_column_id() == info.parent_column_ids_.at(j) - && child_col->get_column_id() == info.child_column_ids_.at(j)) { - ++match; - find = true; + int64_t parent_idx = -1; + int64_t child_idx = -1; + if (ObOptimizerUtil::find_item(info.parent_column_ids_, parent_col->get_column_id(), &parent_idx) && + ObOptimizerUtil::find_item(info.child_column_ids_, child_col->get_column_id(), &child_idx) && + parent_idx == child_idx) { + if (OB_FAIL(add_var_to_array_no_dup(matched_idxs, child_idx))) { + LOG_WARN("failed to add var to array no dup", K(ret)); } + } else { + // join quals use cols more than foreign keys or miss match with foreign key relationship + // e.g. (t2.c1, t2.c2) => foreign key(t1.c1, t2.c2) + // `t2.c3 = t1.c1`, `t1.c1 = t2.c2` are invalid + valid = false; } } } - if (N == match) { - is_all_involved = true; - } + is_all_involved = valid && (allow_partial_join ? matched_idxs.count() > 0 : + matched_idxs.count() == info.child_column_ids_.count()); } return ret; } @@ -3992,43 +3998,47 @@ int ObTransformUtils::is_all_foreign_key_involved(const ObIArray &child_exprs, const ObIArray &parent_exprs, const ObForeignKeyInfo &info, + bool allow_partial_join, bool &is_all_involved) { int ret = OB_SUCCESS; + bool valid = true; is_all_involved = false; - const int64_t N = info.child_column_ids_.count(); - // generate stmt时会进行去重，不会出现t1.c1 = t2.c1 and t1.c1 = t2.c1的情况 - if (OB_UNLIKELY(child_exprs.count() != parent_exprs.count())) { + ObSEArray matched_idxs; + if (OB_UNLIKELY(child_exprs.count() != parent_exprs.count()) || + OB_UNLIKELY(info.child_column_ids_.count() != info.parent_column_ids_.count())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("child exprs and parent exprs should have equal size", K(ret), K(child_exprs.count()), K(parent_exprs.count())); - } else if (N == child_exprs.count()) { - int64_t match = 0; - for (int64_t i = 0; i < N; ++i) { - bool find = false; - ObRawExpr *child_expr = child_exprs.at(i); - ObRawExpr *parent_expr = parent_exprs.at(i); - if (OB_ISNULL(child_expr) || OB_ISNULL(parent_expr)) { + } else { + for (int64_t i = 0; OB_SUCC(ret) && valid && i < child_exprs.count(); i++) { + if (OB_ISNULL(child_exprs.at(i)) || OB_ISNULL(parent_exprs.at(i))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("child expr or parent expr is null", K(ret), K(child_expr), K(parent_expr)); - } else if (OB_UNLIKELY(!child_expr->has_flag(IS_COLUMN) || !parent_expr->has_flag(IS_COLUMN))) { + LOG_WARN("child expr or parent expr is null", K(ret), K(child_exprs.at(i)), K(parent_exprs.at(i))); + } else if (OB_UNLIKELY(!child_exprs.at(i)->is_column_ref_expr() || !parent_exprs.at(i)->is_column_ref_expr())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("not column expr", K(ret)); } else{ ObColumnRefRawExpr *child_col = static_cast(child_exprs.at(i)); ObColumnRefRawExpr *parent_col = static_cast(parent_exprs.at(i)); - for (int64_t j = 0; !find && j < N; ++j) { - if(parent_col->get_column_id() == info.parent_column_ids_.at(j) - && child_col->get_column_id() == info.child_column_ids_.at(j)) { - ++match; - find = true; + int64_t parent_idx = -1; + int64_t child_idx = -1; + if (ObOptimizerUtil::find_item(info.parent_column_ids_, parent_col->get_column_id(), &parent_idx) && + ObOptimizerUtil::find_item(info.child_column_ids_, child_col->get_column_id(), &child_idx) && + parent_idx == child_idx) { + if (OB_FAIL(add_var_to_array_no_dup(matched_idxs, child_idx))) { + LOG_WARN("failed to add var to array no dup", K(ret)); } + } else { + // join quals use cols more than foreign keys or miss match with foreign key relationship + // e.g. (t2.c1, t2.c2) => foreign key(t1.c1, t2.c2) + // `t2.c3 = t1.c1`, `t1.c1 = t2.c2` are invalid + valid = false; } } } - if (N == match) { - is_all_involved = true; - } + is_all_involved = valid && (allow_partial_join ? matched_idxs.count() > 0 : + matched_idxs.count() == info.child_column_ids_.count()); } return ret; } diff --git a/src/sql/rewrite/ob_transform_utils.h b/src/sql/rewrite/ob_transform_utils.h index 98f5ce7ff..dcc5dc7a0 100644 --- a/src/sql/rewrite/ob_transform_utils.h +++ b/src/sql/rewrite/ob_transform_utils.h @@ -670,6 +670,7 @@ public: * @param second_exprs 第二个表的连接列 * @param is_foreign_primary_join 是否为主外键连接 * @param is_first_table_parent first_table是否为父表 + * @param allow_partial_join 是否允许连接条件只匹配外键关系的非空子集（只用于判定连接无损） */ static int check_foreign_primary_join(const TableItem *first_table, const TableItem * second_table, @@ -677,6 +678,7 @@ public: const ObIArray &second_exprs, ObSchemaChecker *schema_checker, ObSQLSessionInfo *session_info, + bool allow_partial_join, bool &is_foreign_primary_join, bool &is_first_table_parent, share::schema::ObForeignKeyInfo *&foreign_key_info); @@ -686,6 +688,7 @@ public: const ObIArray< ObRawExpr *> &second_exprs, ObSchemaChecker *schema_checker, ObSQLSessionInfo *session_info, + bool allow_partial_join, bool &is_foreign_primary_join, bool &is_first_table_parent, share::schema::ObForeignKeyInfo *&foreign_key_info); @@ -700,14 +703,17 @@ public: * 或child_exprs = [c3, c4] 且 parent_exprs = [c1, c2] * * @param is_all_involved 是否包含了主外键约束中一一对应的所有的键 + * @param allow_partial_join 是否允许只包含主外键约束中的部分键 */ static int is_all_foreign_key_involved(const ObIArray &child_exprs, const ObIArray &parent_exprs, const share::schema::ObForeignKeyInfo &info, + bool allow_partial_join, bool &is_all_involved); static int is_all_foreign_key_involved(const ObIArray< ObRawExpr *> &child_exprs, const ObIArray< ObRawExpr *> &parent_exprs, const share::schema::ObForeignKeyInfo &info, + bool allow_partial_join, bool &is_all_involved); /** From a8970790371724f4115fa1c9925564d9a36ae742 Mon Sep 17 00:00:00 2001 From: obdev Date: Thu, 15 Aug 2024 12:12:57 +0000 Subject: [PATCH 065/249] [FEAT MERGE] shared_nothing_tmp_file Co-authored-by: lalalafeier Co-authored-by: wanyue-wy <345657357@qq.com> Co-authored-by: simonjoylet --- deps/oblib/src/lib/allocator/ob_slice_alloc.h | 2 +- mittest/mtlenv/mock_tenant_module_env.h | 12 +- mittest/mtlenv/storage/CMakeLists.txt | 1 + mittest/mtlenv/storage/test_lob_manager.cpp | 1 - .../mtlenv/storage/tmp_file/CMakeLists.txt | 6 + .../tmp_file/ob_tmp_file_test_helper.h | 510 +++ .../mtlenv/storage/tmp_file/test_tmp_file.cpp | 1354 +++++++ .../tmp_file/test_tmp_file_block_manager.cpp | 598 ++++ .../tmp_file/test_tmp_file_buffer_pool.cpp | 690 ++++ .../tmp_file/test_tmp_file_flush_list.cpp | 697 ++++ .../tmp_file/test_tmp_file_meta_tree.cpp | 2381 +++++++++++++ ...tmp_file_write_buffer_pool_index_cache.cpp | 843 +++++ src/observer/ob_server.cpp | 19 +- src/observer/omt/ob_multi_tenant.cpp | 3 + src/observer/omt/ob_tenant.cpp | 11 - .../backup/ob_backup_table_list_mgr.cpp | 8 +- src/share/io/ob_io_define.cpp | 65 +- src/share/io/ob_io_define.h | 3 +- src/share/ob_thread_define.h | 2 + src/share/rc/ob_tenant_base.h | 4 + src/sql/dtl/ob_dtl_interm_result_manager.cpp | 2 +- src/sql/engine/basic/ob_chunk_datum_store.cpp | 22 +- src/sql/engine/basic/ob_chunk_datum_store.h | 14 +- src/sql/engine/basic/ob_chunk_row_store.cpp | 9 +- src/sql/engine/basic/ob_chunk_row_store.h | 4 +- src/sql/engine/basic/ob_material_op_impl.cpp | 2 +- src/sql/engine/basic/ob_ra_datum_store.cpp | 16 +- src/sql/engine/basic/ob_ra_row_store.cpp | 16 +- src/sql/engine/basic/ob_temp_block_store.cpp | 18 +- src/sql/engine/basic/ob_temp_block_store.h | 12 +- src/storage/CMakeLists.txt | 24 +- src/storage/backup/ob_backup_tmp_file.cpp | 11 +- src/storage/backup/ob_backup_tmp_file.h | 11 +- src/storage/blocksstable/ob_block_manager.cpp | 34 +- .../blocksstable/ob_macro_block_handle.cpp | 16 +- .../blocksstable/ob_macro_block_handle.h | 2 +- src/storage/blocksstable/ob_tmp_file.cpp | 1836 ---------- src/storage/blocksstable/ob_tmp_file.h | 444 --- .../blocksstable/ob_tmp_file_cache.cpp | 1655 --------- src/storage/blocksstable/ob_tmp_file_cache.h | 474 --- .../blocksstable/ob_tmp_file_store.cpp | 2012 ----------- src/storage/blocksstable/ob_tmp_file_store.h | 393 --- .../direct_load/ob_direct_load_tmp_file.cpp | 33 +- .../direct_load/ob_direct_load_tmp_file.h | 8 +- src/storage/ob_disk_usage_reporter.cpp | 35 +- src/storage/ob_parallel_external_sort.h | 48 +- .../tmp_file/ob_shared_nothing_tmp_file.cpp | 2717 ++++++++++++++ .../tmp_file/ob_shared_nothing_tmp_file.h | 364 ++ .../tmp_file/ob_tmp_file_block_manager.cpp | 922 +++++ .../tmp_file/ob_tmp_file_block_manager.h | 221 ++ src/storage/tmp_file/ob_tmp_file_cache.cpp | 674 ++++ src/storage/tmp_file/ob_tmp_file_cache.h | 251 ++ .../tmp_file/ob_tmp_file_eviction_manager.cpp | 218 ++ .../tmp_file/ob_tmp_file_eviction_manager.h | 56 + .../tmp_file/ob_tmp_file_flush_ctx.cpp | 321 ++ src/storage/tmp_file/ob_tmp_file_flush_ctx.h | 305 ++ .../ob_tmp_file_flush_list_iterator.cpp | 776 ++++ .../ob_tmp_file_flush_list_iterator.h | 105 + .../tmp_file/ob_tmp_file_flush_manager.cpp | 969 +++++ .../tmp_file/ob_tmp_file_flush_manager.h | 127 + .../ob_tmp_file_flush_priority_manager.cpp | 354 ++ .../ob_tmp_file_flush_priority_manager.h | 82 + src/storage/tmp_file/ob_tmp_file_global.cpp | 30 + src/storage/tmp_file/ob_tmp_file_global.h | 57 + src/storage/tmp_file/ob_tmp_file_io_ctx.cpp | 497 +++ src/storage/tmp_file/ob_tmp_file_io_ctx.h | 170 + .../tmp_file/ob_tmp_file_io_define.cpp | 236 ++ src/storage/tmp_file/ob_tmp_file_io_define.h | 81 + src/storage/tmp_file/ob_tmp_file_manager.cpp | 530 +++ src/storage/tmp_file/ob_tmp_file_manager.h | 111 + .../tmp_file/ob_tmp_file_meta_tree.cpp | 3139 +++++++++++++++++ src/storage/tmp_file/ob_tmp_file_meta_tree.h | 537 +++ .../ob_tmp_file_page_cache_controller.cpp | 161 + .../ob_tmp_file_page_cache_controller.h | 78 + .../tmp_file/ob_tmp_file_thread_job.cpp | 193 + src/storage/tmp_file/ob_tmp_file_thread_job.h | 111 + .../tmp_file/ob_tmp_file_thread_wrapper.cpp | 1023 ++++++ .../tmp_file/ob_tmp_file_thread_wrapper.h | 158 + .../ob_tmp_file_write_buffer_index_cache.cpp | 783 ++++ .../ob_tmp_file_write_buffer_index_cache.h | 145 + .../ob_tmp_file_write_buffer_pool.cpp | 988 ++++++ .../tmp_file/ob_tmp_file_write_buffer_pool.h | 305 ++ .../engine/basic/test_chunk_datum_store.cpp | 23 +- .../sql/engine/basic/test_chunk_row_store.cpp | 19 +- unittest/sql/engine/test_op_engine.cpp | 2 - .../backup/test_backup_index_merger.cpp | 24 +- .../storage/backup/test_backup_iterator.cpp | 19 +- .../test_backup_macro_block_index_merger.cpp | 20 +- .../storage/backup/test_backup_tmp_file.cpp | 19 +- unittest/storage/backup/test_backup_utils.cpp | 20 +- unittest/storage/blocksstable/CMakeLists.txt | 1 - .../blocksstable/test_block_manager.cpp | 21 +- .../storage/blocksstable/test_tmp_file.cpp | 2378 ------------- .../storage/ddl/test_chunk_compact_store.cpp | 44 +- .../test_direct_load_data_block_writer.cpp | 43 +- .../test_direct_load_index_block_writer.cpp | 42 +- unittest/storage/test_io_manager.cpp | 8 +- .../storage/test_parallel_external_sort.cpp | 45 +- 98 files changed, 25410 insertions(+), 9474 deletions(-) create mode 100644 mittest/mtlenv/storage/tmp_file/CMakeLists.txt create mode 100644 mittest/mtlenv/storage/tmp_file/ob_tmp_file_test_helper.h create mode 100644 mittest/mtlenv/storage/tmp_file/test_tmp_file.cpp create mode 100644 mittest/mtlenv/storage/tmp_file/test_tmp_file_block_manager.cpp create mode 100644 mittest/mtlenv/storage/tmp_file/test_tmp_file_buffer_pool.cpp create mode 100644 mittest/mtlenv/storage/tmp_file/test_tmp_file_flush_list.cpp create mode 100644 mittest/mtlenv/storage/tmp_file/test_tmp_file_meta_tree.cpp create mode 100644 mittest/mtlenv/storage/tmp_file/test_tmp_file_write_buffer_pool_index_cache.cpp delete mode 100644 src/storage/blocksstable/ob_tmp_file.cpp delete mode 100644 src/storage/blocksstable/ob_tmp_file.h delete mode 100644 src/storage/blocksstable/ob_tmp_file_cache.cpp delete mode 100644 src/storage/blocksstable/ob_tmp_file_cache.h delete mode 100644 src/storage/blocksstable/ob_tmp_file_store.cpp delete mode 100644 src/storage/blocksstable/ob_tmp_file_store.h create mode 100644 src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp create mode 100644 src/storage/tmp_file/ob_shared_nothing_tmp_file.h create mode 100644 src/storage/tmp_file/ob_tmp_file_block_manager.cpp create mode 100644 src/storage/tmp_file/ob_tmp_file_block_manager.h create mode 100644 src/storage/tmp_file/ob_tmp_file_cache.cpp create mode 100644 src/storage/tmp_file/ob_tmp_file_cache.h create mode 100644 src/storage/tmp_file/ob_tmp_file_eviction_manager.cpp create mode 100644 src/storage/tmp_file/ob_tmp_file_eviction_manager.h create mode 100644 src/storage/tmp_file/ob_tmp_file_flush_ctx.cpp create mode 100644 src/storage/tmp_file/ob_tmp_file_flush_ctx.h create mode 100644 src/storage/tmp_file/ob_tmp_file_flush_list_iterator.cpp create mode 100644 src/storage/tmp_file/ob_tmp_file_flush_list_iterator.h create mode 100644 src/storage/tmp_file/ob_tmp_file_flush_manager.cpp create mode 100644 src/storage/tmp_file/ob_tmp_file_flush_manager.h create mode 100644 src/storage/tmp_file/ob_tmp_file_flush_priority_manager.cpp create mode 100644 src/storage/tmp_file/ob_tmp_file_flush_priority_manager.h create mode 100644 src/storage/tmp_file/ob_tmp_file_global.cpp create mode 100644 src/storage/tmp_file/ob_tmp_file_global.h create mode 100644 src/storage/tmp_file/ob_tmp_file_io_ctx.cpp create mode 100644 src/storage/tmp_file/ob_tmp_file_io_ctx.h create mode 100644 src/storage/tmp_file/ob_tmp_file_io_define.cpp create mode 100644 src/storage/tmp_file/ob_tmp_file_io_define.h create mode 100644 src/storage/tmp_file/ob_tmp_file_manager.cpp create mode 100644 src/storage/tmp_file/ob_tmp_file_manager.h create mode 100644 src/storage/tmp_file/ob_tmp_file_meta_tree.cpp create mode 100644 src/storage/tmp_file/ob_tmp_file_meta_tree.h create mode 100644 src/storage/tmp_file/ob_tmp_file_page_cache_controller.cpp create mode 100644 src/storage/tmp_file/ob_tmp_file_page_cache_controller.h create mode 100644 src/storage/tmp_file/ob_tmp_file_thread_job.cpp create mode 100644 src/storage/tmp_file/ob_tmp_file_thread_job.h create mode 100644 src/storage/tmp_file/ob_tmp_file_thread_wrapper.cpp create mode 100644 src/storage/tmp_file/ob_tmp_file_thread_wrapper.h create mode 100644 src/storage/tmp_file/ob_tmp_file_write_buffer_index_cache.cpp create mode 100644 src/storage/tmp_file/ob_tmp_file_write_buffer_index_cache.h create mode 100644 src/storage/tmp_file/ob_tmp_file_write_buffer_pool.cpp create mode 100644 src/storage/tmp_file/ob_tmp_file_write_buffer_pool.h delete mode 100644 unittest/storage/blocksstable/test_tmp_file.cpp diff --git a/deps/oblib/src/lib/allocator/ob_slice_alloc.h b/deps/oblib/src/lib/allocator/ob_slice_alloc.h index d9816ca63..ffaf6210c 100644 --- a/deps/oblib/src/lib/allocator/ob_slice_alloc.h +++ b/deps/oblib/src/lib/allocator/ob_slice_alloc.h @@ -293,7 +293,7 @@ private: class ObSliceAlloc { public: - enum { MAX_ARENA_NUM = 32, MAX_REF_NUM = 4096, DEFAULT_BLOCK_SIZE = OB_MALLOC_NORMAL_BLOCK_SIZE }; + enum { MAX_ARENA_NUM = 32, MAX_REF_NUM = 128, DEFAULT_BLOCK_SIZE = OB_MALLOC_NORMAL_BLOCK_SIZE }; typedef ObSimpleSync Sync; typedef ObBlockSlicer Block; typedef ObBlockAllocMgr BlockAlloc; diff --git a/mittest/mtlenv/mock_tenant_module_env.h b/mittest/mtlenv/mock_tenant_module_env.h index 86f696697..6ad735fbd 100644 --- a/mittest/mtlenv/mock_tenant_module_env.h +++ b/mittest/mtlenv/mock_tenant_module_env.h @@ -95,6 +95,7 @@ #include "observer/table/ob_table_session_pool.h" #include "share/index_usage/ob_index_usage_info_mgr.h" #include "storage/tenant_snapshot/ob_tenant_snapshot_service.h" +#include "storage/tmp_file/ob_tmp_file_manager.h" // ObTenantTmpFileManager #include "storage/memtable/ob_lock_wait_mgr.h" namespace oceanbase @@ -658,6 +659,10 @@ int MockTenantModuleEnv::init_before_start_mtl() STORAGE_LOG(WARN, "fail to init env", K(ret)); } else if (OB_FAIL(oceanbase::palf::election::GLOBAL_INIT_ELECTION_MODULE())) { STORAGE_LOG(WARN, "fail to init env", K(ret)); + } else if (OB_FAIL(tmp_file::ObTmpBlockCache::get_instance().init("tmp_block_cache", 1))) { + STORAGE_LOG(WARN, "init tmp block cache failed", KR(ret)); + } else if (OB_FAIL(tmp_file::ObTmpPageCache::get_instance().init("tmp_page_cache", 1))) { + STORAGE_LOG(WARN, "init tmp page cache failed", KR(ret)); } else if (OB_SUCCESS != (ret = bandwidth_throttle_.init(1024 * 1024 * 60))) { STORAGE_LOG(ERROR, "failed to init bandwidth_throttle_", K(ret)); } else if (OB_FAIL(TG_START(lib::TGDefIDs::ServerGTimer))) { @@ -691,6 +696,7 @@ int MockTenantModuleEnv::init() } else { oceanbase::ObClusterVersion::get_instance().update_data_version(DATA_CURRENT_VERSION); MTL_BIND2(ObTenantIOManager::mtl_new, ObTenantIOManager::mtl_init, mtl_start_default, mtl_stop_default, nullptr, ObTenantIOManager::mtl_destroy); + MTL_BIND2(mtl_new_default, tmp_file::ObTenantTmpFileManager::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); MTL_BIND2(mtl_new_default, omt::ObSharedTimer::mtl_init, omt::ObSharedTimer::mtl_start, omt::ObSharedTimer::mtl_stop, omt::ObSharedTimer::mtl_wait, mtl_destroy_default); MTL_BIND2(mtl_new_default, ObTenantSchemaService::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default); MTL_BIND2(mtl_new_default, ObStorageLogger::mtl_init, ObStorageLogger::mtl_start, ObStorageLogger::mtl_stop, ObStorageLogger::mtl_wait, mtl_destroy_default); @@ -753,8 +759,6 @@ int MockTenantModuleEnv::init() STORAGE_LOG(ERROR, "reload memory config failed", K(ret)); } else if (OB_FAIL(start_())) { STORAGE_LOG(ERROR, "mock env start failed", K(ret)); - } else if (OB_FAIL(ObTmpFileManager::get_instance().init())) { - STORAGE_LOG(WARN, "init_tmp_file_manager failed", K(ret)); } else { inited_ = true; } @@ -853,7 +857,6 @@ void MockTenantModuleEnv::destroy() ObKVGlobalCache::get_instance().destroy(); ObServerCheckpointSlogHandler::get_instance().destroy(); SLOGGERMGR.destroy(); - ObTmpFileManager::get_instance().destroy(); OB_SERVER_BLOCK_MGR.stop(); OB_SERVER_BLOCK_MGR.wait(); @@ -867,7 +870,8 @@ void MockTenantModuleEnv::destroy() net_frame_.stop(); net_frame_.wait(); net_frame_.destroy(); - + tmp_file::ObTmpBlockCache::get_instance().destroy(); + tmp_file::ObTmpPageCache::get_instance().destroy(); TG_STOP(lib::TGDefIDs::ServerGTimer); TG_WAIT(lib::TGDefIDs::ServerGTimer); TG_DESTROY(lib::TGDefIDs::ServerGTimer); diff --git a/mittest/mtlenv/storage/CMakeLists.txt b/mittest/mtlenv/storage/CMakeLists.txt index faec8222c..13abc2993 100644 --- a/mittest/mtlenv/storage/CMakeLists.txt +++ b/mittest/mtlenv/storage/CMakeLists.txt @@ -42,5 +42,6 @@ add_subdirectory(checkpoint) add_subdirectory(blocksstable) add_subdirectory(tenant_snapshot) add_subdirectory(tablet_memtable) +add_subdirectory(tmp_file) target_link_libraries(test_memtable PUBLIC mock_tx_ctx mock_tx_log_adapter) diff --git a/mittest/mtlenv/storage/test_lob_manager.cpp b/mittest/mtlenv/storage/test_lob_manager.cpp index 1e33c2f28..d37acf4fa 100644 --- a/mittest/mtlenv/storage/test_lob_manager.cpp +++ b/mittest/mtlenv/storage/test_lob_manager.cpp @@ -23,7 +23,6 @@ #include "lib/random/ob_random.h" #include "storage/blocksstable/ob_data_file_prepare.h" #include "share/ob_simple_mem_limit_getter.h" -#include "storage/blocksstable/ob_tmp_file.h" #include "storage/lob/ob_lob_piece.h" #include "sql/engine/ob_exec_context.h" #include "lib/objectpool/ob_server_object_pool.h" diff --git a/mittest/mtlenv/storage/tmp_file/CMakeLists.txt b/mittest/mtlenv/storage/tmp_file/CMakeLists.txt new file mode 100644 index 000000000..ae96ad2a5 --- /dev/null +++ b/mittest/mtlenv/storage/tmp_file/CMakeLists.txt @@ -0,0 +1,6 @@ +storage_unittest(test_tmp_file) +storage_unittest(test_tmp_file_meta_tree) +storage_unittest(test_tmp_file_write_buffer_pool_index_cache) +storage_unittest(test_tmp_file_buffer_pool) +storage_unittest(test_tmp_file_flush_list) +storage_unittest(test_tmp_file_block_manager) diff --git a/mittest/mtlenv/storage/tmp_file/ob_tmp_file_test_helper.h b/mittest/mtlenv/storage/tmp_file/ob_tmp_file_test_helper.h new file mode 100644 index 000000000..643277a2f --- /dev/null +++ b/mittest/mtlenv/storage/tmp_file/ob_tmp_file_test_helper.h @@ -0,0 +1,510 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OB_TMP_FILE_TEST_HELPER_ +#define OB_TMP_FILE_TEST_HELPER_ +#include +#include +#include +#include +#include +#include +#include +#define USING_LOG_PREFIX STORAGE +#define protected public +#define private public +#include "share/ob_thread_pool.h" +#include "share/rc/ob_tenant_base.h" +#include "storage/tmp_file/ob_tmp_file_io_define.h" +#include "storage/tmp_file/ob_tmp_file_manager.h" + +namespace oceanbase +{ +using namespace common; +using namespace tmp_file; +using namespace share; +/* ------------------------------ Test Helper ------------------------------ */ +void print_hex_data(const char *buffer, int64_t length) +{ + std::cout << std::hex << std::setfill('0'); + for (int64_t i = 0; i < length; ++i) { + std::cout << std::setw(2) << static_cast(static_cast(buffer[i])); + } + std::cout << std::dec << std::endl; +} + +void dump_hex_data(const char *buffer, int length, const std::string &filename) +{ + static SpinRWLock lock_; + SpinWLockGuard guard(lock_); + std::ifstream ifile(filename); + if (ifile) { + } else { + std::ofstream file(filename, std::ios::out | std::ios::binary); + if (file.is_open()) { + for (int i = 0; i < length; ++i) { + if (i != 0 && i % 16 == 0) { + file << std::endl; + } else if (i != 0 && i % 2 == 0) { + file << " "; + } + file << std::hex << std::setw(2) << std::setfill('0') + << (static_cast(buffer[i]) & 0xFF); + } + file.close(); + std::cout << "Data has been written to " << filename << " in hex format." << std::endl; + } else { + std::cerr << "Error opening file " << filename << " for writing." << std::endl; + } + } +} + +bool compare_and_print_hex_data(const char *lhs, const char *rhs, + int64_t buf_length, int64_t print_length, + std::string &filename) +{ + bool is_equal = true; + static SpinRWLock lock_; + SpinWLockGuard guard(lock_); + static int64_t idx = 0; + filename.clear(); + filename = std::to_string(ATOMIC_FAA(&idx, 1)) + "_cmp_and_dump_hex_data.txt"; + std::ofstream file(filename, std::ios::out | std::ios::binary); + if (file.is_open()) { + for (int i = 0; i < buf_length; ++i) { + if (lhs[i] != rhs[i]) { + is_equal = false; + int64_t print_begin = i - print_length / 2 >= 0 ? i - print_length / 2 : 0; + int64_t print_end = print_begin + print_length < buf_length ? print_begin + print_length : buf_length; + file << "First not equal happen at " << i + << ", print length: " << print_end - print_begin + << ", print begin: " << print_begin + << ", print end: " << print_end << std::endl; + file << std::endl << "lhs:" << std::endl; + { + const char *buffer = lhs + print_begin; + int64_t length = print_end - print_begin; + for (int64_t i = 0; i < length; ++i) { + file << std::hex << std::setw(2) << std::setfill('0') << (static_cast(buffer[i]) & 0xFF); + } + } + file << std::endl << "rhs:" << std::endl; + { + const char *buffer = rhs + print_begin; + int64_t length = print_end - print_begin; + for (int64_t i = 0; i < length; ++i) { + file << std::hex << std::setw(2) << std::setfill('0') << (static_cast(buffer[i]) & 0xFF); + } + } + std::cout << "not equal at " << i << std::endl; + break; + } + } + file.close(); + } else { + std::cerr << "Error opening file " << filename << " for writing." << std::endl; + } + return is_equal; +} + +int64_t generate_random_int(const int64_t lower_bound, const int64_t upper_bound) +{ + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution dis(lower_bound, upper_bound); + int64_t random_number = dis(gen); + return random_number; +} + +std::vector generate_random_sequence(const int64_t lower_bound, + const int64_t upper_bound, + const int64_t sequence_sum, + unsigned seed = std::random_device{}()) +{ + std::mt19937 gen(seed); + std::uniform_int_distribution dis(lower_bound, upper_bound); + std::vector random_sequence; + int64_t sum = 0; + while (sum < sequence_sum) { + int64_t rand_num = std::min(sequence_sum - sum, dis(gen)); + random_sequence.push_back(rand_num); + sum += rand_num; + } + return random_sequence; +} + +/* -------------------------- TestTmpFileStress --------------------------- */ +enum TmpFileOp { + WRITE, + READ, + TRUNCATE, + OP_MAX +}; + +class TestTmpFileStress : public share::ObThreadPool +{ +public: + TestTmpFileStress(ObTenantBase *tenant_ctx); + virtual ~TestTmpFileStress(); + int init(const int fd, const TmpFileOp op, const int64_t thread_cnt, + char *buf, const int64_t offset, const int64_t size); + void reset(); + virtual void run1(); + TO_STRING_KV(K_(thread_cnt), K_(fd), K_(op), KP_(buf), K_(offset), K_(size)); +private: + void write_data_(const int64_t write_size); + void truncate_data_(); + void read_data_(const int64_t read_offset, const int64_t read_size); +private: + int64_t thread_cnt_; + int fd_; + TmpFileOp op_; + char *buf_; + int64_t offset_; + int64_t size_; + ObTenantBase *tenant_ctx_; +}; + +TestTmpFileStress::TestTmpFileStress(ObTenantBase *tenant_ctx) + : thread_cnt_(0), fd_(0), + op_(OP_MAX), + buf_(nullptr), offset_(0), + size_(0), + tenant_ctx_(tenant_ctx) +{ +} + +TestTmpFileStress::~TestTmpFileStress() +{ +} + +int TestTmpFileStress::init(const int fd, const TmpFileOp op, + const int64_t thread_cnt, + char *buf, int64_t offset, + const int64_t size) +{ + int ret = OB_SUCCESS; + if (thread_cnt < 0 || OB_ISNULL(buf) || offset < 0 || size <= 0) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", K(ret), K(thread_cnt), KP(buf), K(offset), K(size)); + } else if (TmpFileOp::OP_MAX == op) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", K(ret), K(op)); + } else if ((op == TmpFileOp::WRITE || op == TmpFileOp::TRUNCATE) && 1 != thread_cnt) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", K(ret), K(op), K(thread_cnt)); + } else { + buf_ = buf; + thread_cnt_ = thread_cnt; + fd_ = fd; + op_ = op; + offset_ = offset; + size_ = size; + set_thread_count(static_cast(thread_cnt)); + } + return ret; +} + +void TestTmpFileStress::reset() +{ + thread_cnt_ = 0; + fd_ = 0; + op_ = OP_MAX; + buf_ = nullptr; + offset_ = 0; + size_ = 0; +} + +void TestTmpFileStress::write_data_(const int64_t write_size) +{ + STORAGE_LOG(INFO, "TestTmpFileStress write thread", K(fd_), K(thread_idx_), KP(buf_), K(size_)); + int ret = OB_SUCCESS; + ObArray size_array; + ObTmpFileIOInfo io_info; + ASSERT_EQ(OB_SUCCESS, ret); + io_info.fd_ = fd_; + io_info.io_desc_.set_wait_event(2); + io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; + int64_t already_write = 0; + std::vector turn_write_size = generate_random_sequence(1, write_size / 3, write_size, 3); + for (int i = 0; i < turn_write_size.size(); ++i) { + int64_t this_turn_write_size = turn_write_size[i]; + STORAGE_LOG(INFO, "random write size", K(fd_), K(thread_idx_), KP(buf_), K(size_), K(this_turn_write_size)); + // write data + io_info.buf_ = buf_ + already_write; + if (this_turn_write_size % ObTmpFileGlobal::PAGE_SIZE == 0 && i == 0) { + io_info.size_ = this_turn_write_size - 2 * 1024; + ASSERT_EQ(OB_SUCCESS, MTL(ObTenantTmpFileManager *)->write(io_info)); + + io_info.size_ = 2 * 1024; + io_info.buf_ = buf_ + already_write + this_turn_write_size - 2 * 1024; + ASSERT_EQ(OB_SUCCESS, MTL(ObTenantTmpFileManager *)->write(io_info)); + } else { + io_info.size_ = this_turn_write_size; + ASSERT_EQ(OB_SUCCESS, MTL(ObTenantTmpFileManager *)->write(io_info)); + } + already_write += this_turn_write_size; + } + + ASSERT_EQ(OB_SUCCESS, ret); + STORAGE_LOG(INFO, "TestTmpFileStress write thread finished", K(fd_), K(thread_idx_), KP(buf_), K(size_)); +} + +void TestTmpFileStress::read_data_(const int64_t read_offset, const int64_t read_size) +{ + STORAGE_LOG(INFO, "TestTmpFileStress read thread start", K(fd_), K(thread_idx_), KP(buf_), K(read_offset), K(read_size)); + int ret = OB_SUCCESS; + char *read_buf = new char[read_size]; + ObTmpFileIOInfo io_info; + ObTmpFileIOHandle handle; + io_info.fd_ = fd_; + io_info.size_ = read_size; + io_info.io_desc_.set_wait_event(2); + io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; + io_info.buf_ = read_buf; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); + int cmp = memcmp(handle.get_buffer(), buf_ + read_offset, io_info.size_); + if (cmp != 0 || OB_FAIL(ret)) { + STORAGE_LOG(WARN, "TestTmpFileStress read thread failed", KR(ret), K(fd_), K(cmp), K(thread_idx_), KP(buf_), K(read_offset), K(read_size)); + ob_abort(); + } + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + STORAGE_LOG(INFO, "TestTmpFileStress read thread finished", K(fd_), K(thread_idx_), KP(buf_), K(read_offset), K(read_size)); +} + +void TestTmpFileStress::truncate_data_() +{ + int64_t truncate_offset = offset_ + MIN(size_, MAX(size_ / 10, 8 * 1024)); + STORAGE_LOG(INFO, "TestTmpFileStress truncate thread start", K(fd_), K(thread_idx_), KP(buf_), + K(truncate_offset), K(offset_), K(size_)); + int ret = MTL(ObTenantTmpFileManager *)->truncate(fd_, truncate_offset); + ASSERT_EQ(OB_SUCCESS, ret); + ObTmpFileIOInfo io_info; + io_info.fd_ = fd_; + io_info.io_desc_.set_wait_event(2); + io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; + const int64_t invalid_size = truncate_offset - offset_; + const int64_t valid_size = size_ - invalid_size; + + char *zero_buf = new char[invalid_size]; + MEMSET(zero_buf, 0, invalid_size); + char *read_buf = new char[size_]; + io_info.size_ = size_; + io_info.buf_ = read_buf; + ObTmpFileIOHandle handle; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, offset_, handle); + int cmp = memcmp(handle.get_buffer()+invalid_size, buf_ + truncate_offset, valid_size); + if (cmp != 0 || OB_FAIL(ret)) { + STORAGE_LOG(INFO, "TestTmpFileStress truncate thread failed. " + "fail to compare valid part.", KR(ret), K(cmp), K(fd_), K(thread_idx_), KP(buf_), + K(truncate_offset), K(valid_size), K(invalid_size), K(offset_), K(size_)); + ob_abort(); + } + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(0, cmp); + cmp = memcmp(handle.get_buffer(), zero_buf, invalid_size); + if (cmp != 0) { + STORAGE_LOG(INFO, "TestTmpFileStress truncate thread failed. " + "fail to compare zero part.", KR(ret), K(cmp), K(fd_), K(thread_idx_), KP(buf_), + K(truncate_offset), K(valid_size), K(invalid_size), K(offset_), K(size_)); + } + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + delete[] zero_buf; + + truncate_offset = offset_ + size_; + ret = MTL(ObTenantTmpFileManager *)->truncate(fd_, truncate_offset); + ASSERT_EQ(OB_SUCCESS, ret); + + zero_buf = new char[size_]; + MEMSET(zero_buf, 0, size_); + read_buf = new char[size_]; + io_info.buf_ = read_buf; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, offset_, handle); + cmp = memcmp(handle.get_buffer(), zero_buf, size_); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + delete[] zero_buf; + STORAGE_LOG(INFO, "TestTmpFileStress truncate thread finished", K(fd_), K(thread_idx_), KP(buf_), K(offset_), K(size_)); +} + +void TestTmpFileStress::run1() +{ + ObTenantEnv::set_tenant(tenant_ctx_); + common::ObCurTraceId::TraceId trace_id; + ObCurTraceId::TraceId *cur_trace_id = ObCurTraceId::get_trace_id(); + if (nullptr != cur_trace_id && cur_trace_id->is_valid()) { + trace_id = *cur_trace_id; + LOG_INFO("init TestTmpFileStress with an old trace_id", KPC(cur_trace_id), KPC(this)); + } else { + trace_id.init(GCONF.self_addr_); + LOG_INFO("init TestTmpFileStress with a new trace_id", K(trace_id), KPC(this)); + } + ObTraceIDGuard trace_guard(trace_id); + + if (op_ == TmpFileOp::WRITE) { + write_data_(size_); + } else if (op_ == TmpFileOp::READ) { + int64_t read_offset = offset_ + (size_ / thread_cnt_) * thread_idx_; + int64_t read_size = 0; + if (thread_idx_ == thread_cnt_ - 1) { + read_size = size_ / thread_cnt_ + size_ % thread_cnt_; + } else { + read_size = size_ / thread_cnt_; + } + read_data_(read_offset, read_size); + } else { + truncate_data_(); + } +} + +/* -------------------------- TestMultiTmpFileStress --------------------------- */ +class TestMultiTmpFileStress : public share::ObThreadPool +{ +public: + TestMultiTmpFileStress(ObTenantBase *tenant_ctx); + virtual ~TestMultiTmpFileStress(); + int init(const int64_t file_cnt, const int64_t dir_id, const int64_t thread_cnt, + const int64_t batch_size, const int64_t batch_num); + virtual void run1(); +private: + int64_t file_cnt_; + int64_t dir_id_; + int64_t read_thread_cnt_perf_file_; + int64_t batch_size_; + int64_t batch_num_; + ObTenantBase *tenant_ctx_; +}; + +TestMultiTmpFileStress::TestMultiTmpFileStress(ObTenantBase *tenant_ctx) + : file_cnt_(0), + dir_id_(-1), + read_thread_cnt_perf_file_(0), + batch_size_(0), + batch_num_(0), + tenant_ctx_(tenant_ctx) +{ +} + +TestMultiTmpFileStress::~TestMultiTmpFileStress() +{ +} + +int TestMultiTmpFileStress::init(const int64_t file_cnt, + const int64_t dir_id, + const int64_t thread_cnt, + const int64_t batch_size, + const int64_t batch_num) +{ + int ret = OB_SUCCESS; + if (file_cnt < 0 || thread_cnt < 0) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", K(ret), K(file_cnt), K(thread_cnt)); + } else { + file_cnt_ = file_cnt; + dir_id_ = dir_id; + read_thread_cnt_perf_file_ = thread_cnt; + batch_size_ = batch_size; + batch_num_ = batch_num; + set_thread_count(static_cast(file_cnt)); + } + return ret; +} + +void TestMultiTmpFileStress::run1() +{ + STORAGE_LOG(INFO, "TestMultiTmpFileStress thread run start"); + int ret = OB_SUCCESS; + int64_t fd = 0; + ObTenantEnv::set_tenant(tenant_ctx_); + + ret = MTL(ObTenantTmpFileManager *)->open(fd, dir_id_); + std::cout << "normal case, fd: " << fd << std::endl; + ASSERT_EQ(OB_SUCCESS, ret); + STORAGE_LOG(INFO, "open file success", K(fd)); + ObTmpFileHandle file_handle; + ret = MTL(ObTenantTmpFileManager *)->get_tmp_file(fd, file_handle); + ASSERT_EQ(OB_SUCCESS, ret); + file_handle.get()->page_idx_cache_.max_bucket_array_capacity_ = ObTmpFileWBPIndexCache::INIT_BUCKET_ARRAY_CAPACITY * 2; + file_handle.reset(); + + int64_t file_size = batch_size_ * batch_num_; + char * data_buffer = new char[file_size]; + for (int64_t i = 0; i < file_size;) { + int64_t random_length = generate_random_int(1024, 8 * 1024); + int64_t random_int = generate_random_int(0, 256); + for (int64_t j = 0; j < random_length && i + j < file_size; ++j) { + data_buffer[i + j] = random_int; + } + i += random_length; + } + + + TestTmpFileStress test_truncate(tenant_ctx_); + for (int64_t i = 0; i < batch_num_; ++i) { + if (i > 0) { + // truncate read data in previous round + test_truncate.init(fd, TmpFileOp::TRUNCATE, 1, data_buffer, (i-1) * batch_size_, batch_size_); + ASSERT_EQ(OB_SUCCESS, ret); + STORAGE_LOG(INFO, "test_truncate run start", K(i), K(batch_size_)); + test_truncate.start(); + } + TestTmpFileStress test_write(tenant_ctx_); + ret = test_write.init(fd, TmpFileOp::WRITE, 1, data_buffer + i * batch_size_, 0, batch_size_); + ASSERT_EQ(OB_SUCCESS, ret); + STORAGE_LOG(INFO, "test_write run start"); + test_write.start(); + test_write.wait(); + STORAGE_LOG(INFO, "test_write run end"); + + TestTmpFileStress test_read(tenant_ctx_); + ret = test_read.init(fd, TmpFileOp::READ, read_thread_cnt_perf_file_, data_buffer, i * batch_size_, batch_size_); + ASSERT_EQ(OB_SUCCESS, ret); + + STORAGE_LOG(INFO, "test_read run start", K(i), K(batch_size_)); + test_read.start(); + test_read.wait(); + STORAGE_LOG(INFO, "test_read run end"); + + if (i > 0) { + // wait to truncate read data in last round + test_truncate.wait(); + test_truncate.reset(); + STORAGE_LOG(INFO, "test_truncate run end", K(i)); + } + + STORAGE_LOG(INFO, "TestMultiTmpFileStress thread run a batch end", K(i)); + } + + test_truncate.init(fd, TmpFileOp::TRUNCATE, 1, data_buffer, file_size - batch_size_, batch_size_); + ASSERT_EQ(OB_SUCCESS, ret); + STORAGE_LOG(INFO, "test_truncate run start"); + test_truncate.start(); + test_truncate.wait(); + STORAGE_LOG(INFO, "test_truncate run end"); + + ret = MTL(ObTenantTmpFileManager *)->remove(fd); + ASSERT_EQ(OB_SUCCESS, ret); + + delete[] data_buffer; + STORAGE_LOG(INFO, "TestMultiTmpFileStress thread run end"); +} + +} // namespace oceanbase + +#endif diff --git a/mittest/mtlenv/storage/tmp_file/test_tmp_file.cpp b/mittest/mtlenv/storage/tmp_file/test_tmp_file.cpp new file mode 100644 index 000000000..8e745dc6c --- /dev/null +++ b/mittest/mtlenv/storage/tmp_file/test_tmp_file.cpp @@ -0,0 +1,1354 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "mittest/mtlenv/storage/tmp_file/ob_tmp_file_test_helper.h" +#define USING_LOG_PREFIX STORAGE +#define protected public +#define private public +#include "mittest/mtlenv/mock_tenant_module_env.h" +#include "share/ob_simple_mem_limit_getter.h" +#include "lib/alloc/ob_malloc_allocator.h" +#include "storage/tmp_file/ob_tmp_file_global.h" +#include "storage/tmp_file/ob_tmp_file_meta_tree.h" + +namespace oceanbase +{ +using namespace common; +using namespace blocksstable; +using namespace tmp_file; +using namespace storage; +using namespace share::schema; +/* ------------------------------ Mock Parameter ---------------------------- */ +static const int64_t TENANT_MEMORY = 8L * 1024L * 1024L * 1024L /* 8 GB */; +/********************************* Mock WBP *************************** */ +static const int64_t WBP_BLOCK_SIZE = ObTmpWriteBufferPool::WBP_BLOCK_SIZE; // each wbp block has 253 pages (253 * 8KB == 2024KB) +static const int64_t SMALL_WBP_BLOCK_COUNT = 3; +static const int64_t SMALL_WBP_MEM_LIMIT = SMALL_WBP_BLOCK_COUNT * WBP_BLOCK_SIZE; // the wbp mem size is 5.93MB +static const int64_t BIG_WBP_BLOCK_COUNT = 40; +static const int64_t BIG_WBP_MEM_LIMIT = BIG_WBP_BLOCK_COUNT * WBP_BLOCK_SIZE; // the wbp mem size is 79.06MB +/********************************* Mock WBP Index Cache*************************** */ +// each bucket could indicate a 256KB data in wbp. +// SMALL_WBP_IDX_CACHE_MAX_CAPACITY will indicate 4MB data in wbp +static const int64_t SMALL_WBP_IDX_CACHE_MAX_CAPACITY = ObTmpFileWBPIndexCache::INIT_BUCKET_ARRAY_CAPACITY * 2; +/********************************* Mock Meta Tree *************************** */ +static const int64_t MAX_DATA_ITEM_ARRAY_COUNT = 2; +static const int64_t MAX_PAGE_ITEM_COUNT = 4; // MAX_PAGE_ITEM_COUNT * ObTmpFileGlobal::PAGE_SIZE means + // the max representation range of a meta page (4 * 2MB == 8MB). + // according to the formula of summation for geometric sequence + // (S_n = a_1 * (1-q^n)/(1-q), where a_1 = 8MB, q = 4), + // a two-level meta tree could represent at most 40MB disk data of tmp file + // a three-level meta tree could represent at most 168MB disk data of tmp file + // a four-level meta tree could represent at most 680MB disk data of tmp file + +/* ---------------------------- Unittest Class ----------------------------- */ + +class TestTmpFile : public ::testing::Test +{ +public: + TestTmpFile() = default; + virtual ~TestTmpFile() = default; + virtual void SetUp(); + virtual void TearDown(); + static void SetUpTestCase(); + static void TearDownTestCase(); +}; +static ObSimpleMemLimitGetter getter; +static const int64_t TEST_ROWKEY_COLUMN_CNT = 2; + +// ATTENTION! +// currently, we only initialize modules about tmp file at the beginning of unit test and +// never restart them in the end of test case. +// please make sure that all test cases will not affect the others. +void TestTmpFile::SetUpTestCase() +{ + int ret = OB_SUCCESS; + ASSERT_EQ(OB_SUCCESS, MockTenantModuleEnv::get_instance().init()); + + CHUNK_MGR.set_limit(TENANT_MEMORY); + ObMallocAllocator::get_instance()->set_tenant_limit(MTL_ID(), TENANT_MEMORY); + + MTL(ObTenantTmpFileManager *)->page_cache_controller_.write_buffer_pool_.default_wbp_memory_limit_ = SMALL_WBP_MEM_LIMIT; + ObSharedNothingTmpFileMetaTree::set_max_array_item_cnt(MAX_DATA_ITEM_ARRAY_COUNT); + ObSharedNothingTmpFileMetaTree::set_max_page_item_cnt(MAX_PAGE_ITEM_COUNT); +} + +void TestTmpFile::SetUp() +{ + int ret = OB_SUCCESS; + ASSERT_EQ(true, MockTenantModuleEnv::get_instance().is_inited()); +// if (!MTL(ObTenantTmpFileManager *)->is_inited_) { +// ret = MTL(ObTenantTmpFileManager *)->init(); +// ASSERT_EQ(OB_SUCCESS, ret); +// ret = MTL(ObTenantTmpFileManager *)->start(); +// ASSERT_EQ(OB_SUCCESS, ret); +// MTL(ObTenantTmpFileManager *)->page_cache_controller_.write_buffer_pool_.default_wbp_memory_limit_ = SMALL_WBP_MEM_LIMIT; +// } +} + +void TestTmpFile::TearDownTestCase() +{ + MockTenantModuleEnv::get_instance().destroy(); +} + +void TestTmpFile::TearDown() +{ +// if (MTL(ObTenantTmpFileManager *)->is_inited_) { +// MTL(ObTenantTmpFileManager *)->stop(); +// MTL(ObTenantTmpFileManager *)->wait(); +// MTL(ObTenantTmpFileManager *)->destroy(); +// } +} + +// generate 2MB random data (will not trigger flush and evict logic) +// 1. test write pages and append write tail page +// 2. test write after reading +TEST_F(TestTmpFile, test_unaligned_data_read_write) +{ + int ret = OB_SUCCESS; + const int64_t write_size = 2 * 1024 * 1024; + const int64_t wbp_mem_limit = MTL(ObTenantTmpFileManager *)->page_cache_controller_.write_buffer_pool_.get_memory_limit(); + ASSERT_LT(write_size, wbp_mem_limit); + char * write_buffer = new char[write_size]; + for (int64_t i = 0; i < write_size;) { + int64_t random_length = generate_random_int(1024, 8 * 1024); + int64_t random_int = generate_random_int(0, 256); + for (int64_t j = 0; j < random_length && i + j < write_size; ++j) { + write_buffer[i + j] = random_int; + } + i += random_length; + } + int64_t dir = -1; + int64_t fd = -1; + const int64_t macro_block_size = OB_SERVER_BLOCK_MGR.get_macro_block_size(); + ObTmpFileIOInfo io_info; + ObTmpFileIOHandle handle; + ret = MTL(ObTenantTmpFileManager *)->alloc_dir(dir); + ASSERT_EQ(OB_SUCCESS, ret); + ret = MTL(ObTenantTmpFileManager *)->open(fd, dir); + std::cout << "open temporary file: " << fd << std::endl; + ASSERT_EQ(OB_SUCCESS, ret); + ObTmpFileHandle file_handle; + ret = MTL(ObTenantTmpFileManager *)->get_tmp_file(fd, file_handle); + ASSERT_EQ(OB_SUCCESS, ret); + file_handle.get()->page_idx_cache_.max_bucket_array_capacity_ = SMALL_WBP_IDX_CACHE_MAX_CAPACITY; + file_handle.reset(); + // dump random data + { + std::string r_file_name = std::to_string(fd) + "_raw_write_data.txt"; + dump_hex_data(write_buffer, write_size, r_file_name); + } + + // random write, read, and check + int64_t already_write = 0; + std::vector turn_write_size = generate_random_sequence(1, write_size / 3, write_size, 3); + for (int i = 0; i < turn_write_size.size(); ++i) { + int64_t this_turn_write_size = turn_write_size[i]; + std::cout << "random write and read " << this_turn_write_size << std::endl; + // write data + { + ObTmpFileIOInfo io_info; + io_info.fd_ = fd; + io_info.io_desc_.set_wait_event(2); + io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; + io_info.buf_ = write_buffer + already_write; + if (this_turn_write_size % ObTmpFileGlobal::PAGE_SIZE == 0 && i == 0) { + io_info.size_ = this_turn_write_size - 2 * 1024; + ASSERT_EQ(OB_SUCCESS, MTL(ObTenantTmpFileManager *)->write(io_info)); + + io_info.size_ = 2 * 1024; + io_info.buf_ = write_buffer + already_write + this_turn_write_size - 2 * 1024; + ASSERT_EQ(OB_SUCCESS, MTL(ObTenantTmpFileManager *)->write(io_info)); + } else { + io_info.size_ = this_turn_write_size; + ASSERT_EQ(OB_SUCCESS, MTL(ObTenantTmpFileManager *)->write(io_info)); + } + } + // read data + char * read_check_buffer = new char[this_turn_write_size]; + { + ObTmpFileIOInfo io_info; + ObTmpFileIOHandle handle; + io_info.fd_ = fd; + io_info.size_ = this_turn_write_size; + io_info.io_desc_.set_wait_event(2); + io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; + io_info.buf_ = read_check_buffer; + ASSERT_EQ(OB_SUCCESS, MTL(ObTenantTmpFileManager *)->read(io_info, handle)); + } + // check data + { + std::string compare_file_name = std::to_string(fd) + "_compare_result.txt"; + bool is_equal = compare_and_print_hex_data( + write_buffer + already_write, read_check_buffer, + this_turn_write_size, 200, compare_file_name); + if (!is_equal) { + // dump write data + std::string w_file_name = std::to_string(fd) + "_write_data.txt"; + dump_hex_data(write_buffer + already_write, this_turn_write_size, w_file_name); + // dump read check data + std::string r_file_name = std::to_string(fd) + "_read_data.txt"; + dump_hex_data(read_check_buffer, this_turn_write_size, r_file_name); + // abort + std::cout << "not equal in random data test" + << "\nwrite dumped file: " << w_file_name + << "\nread check dumped file: " << r_file_name + << "\ncompare result file: " << compare_file_name << std::endl; + ob_abort(); + } + } + // update already_write + delete [] read_check_buffer; + already_write += this_turn_write_size; + } + + ret = MTL(ObTenantTmpFileManager *)->remove(fd); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(0, MTL(ObTenantTmpFileManager *)->page_cache_controller_.flush_priority_mgr_.get_file_size()); + ASSERT_EQ(0, MTL(ObTenantTmpFileManager *)->page_cache_controller_.evict_mgr_.get_file_size()); + LOG_INFO("test_unaligned_data_read_write"); +} + +// generate 7MB random data +// this test will trigger flush and evict logic for data pages. +// meta tree will not be evicted in this test. +// 1. test pread +// 1.1 read disk data +// 1.2 read memory data +// 1.3 read both disk and memory data +// 1.4 read OB_ITER_END +// 2. test read +// 2.1 read aligned data +// 2.2 read unaligned data +TEST_F(TestTmpFile, test_read) +{ + int ret = OB_SUCCESS; + const int64_t write_size = 7 * 1024 * 1024; // 7MB + const int64_t wbp_mem_limit = MTL(ObTenantTmpFileManager *)->page_cache_controller_.write_buffer_pool_.get_memory_limit(); + ASSERT_GT(write_size, wbp_mem_limit); + char *write_buf = new char [write_size]; + for (int64_t i = 0; i < write_size;) { + int64_t random_length = generate_random_int(1024, 8 * 1024); + int64_t random_int = generate_random_int(0, 256); + for (int64_t j = 0; j < random_length && i + j < write_size; ++j) { + write_buf[i + j] = random_int; + } + i += random_length; + } + + int64_t dir = -1; + int64_t fd = -1; + ret = MTL(ObTenantTmpFileManager *)->alloc_dir(dir); + ASSERT_EQ(OB_SUCCESS, ret); + ret = MTL(ObTenantTmpFileManager *)->open(fd, dir); + std::cout << "open temporary file: " << fd << std::endl; + ASSERT_EQ(OB_SUCCESS, ret); + ObTmpFileHandle file_handle; + ret = MTL(ObTenantTmpFileManager *)->get_tmp_file(fd, file_handle); + ASSERT_EQ(OB_SUCCESS, ret); + file_handle.get()->page_idx_cache_.max_bucket_array_capacity_ = SMALL_WBP_IDX_CACHE_MAX_CAPACITY; + file_handle.reset(); + + ObTmpFileIOInfo io_info; + io_info.fd_ = fd; + io_info.io_desc_.set_wait_event(2); + io_info.buf_ = write_buf; + io_info.size_ = write_size; + io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; + // Write data + int64_t write_time = ObTimeUtility::current_time(); + ret = MTL(ObTenantTmpFileManager *)->write(io_info); + write_time = ObTimeUtility::current_time() - write_time; + ASSERT_EQ(OB_SUCCESS, ret); + + ret = MTL(ObTenantTmpFileManager *)->get_tmp_file(fd, file_handle); + ASSERT_EQ(OB_SUCCESS, ret); + int64_t wbp_begin_offset = file_handle.get()->cal_wbp_begin_offset(); + ASSERT_GT(wbp_begin_offset, 0); + file_handle.get()->page_idx_cache_.max_bucket_array_capacity_ = SMALL_WBP_IDX_CACHE_MAX_CAPACITY; + file_handle.reset(); + + int64_t read_time = ObTimeUtility::current_time(); + /************** test pread **************/ + // 1. read memory data + char *read_buf = new char [write_size - wbp_begin_offset]; + ObTmpFileIOHandle handle; + io_info.buf_ = read_buf; + io_info.size_ = write_size - wbp_begin_offset; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, wbp_begin_offset, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(io_info.size_, handle.get_done_size()); + int cmp = memcmp(handle.get_buffer(), write_buf + wbp_begin_offset, io_info.size_); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + + // 2. read disk data + read_buf = new char [wbp_begin_offset]; + io_info.buf_ = read_buf; + io_info.size_ = wbp_begin_offset; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, 0, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(io_info.size_, handle.get_done_size()); + cmp = memcmp(handle.get_buffer(), write_buf, io_info.size_); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + + // 3. read both disk and memory data + int64_t read_size = wbp_begin_offset / 2 + 9 * 1024; + int64_t read_offset = wbp_begin_offset / 2 + 1024; + read_buf = new char [read_size]; + io_info.buf_ = read_buf; + io_info.size_ = read_size; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(io_info.size_, handle.get_done_size()); + cmp = memcmp(handle.get_buffer(), write_buf + read_offset, io_info.size_); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + + // 4. read OB_ITER_END + read_buf = new char [200]; + io_info.buf_ = read_buf; + io_info.size_ = 200; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, write_size - 100, handle); + ASSERT_EQ(OB_ITER_END, ret); + ASSERT_EQ(100, handle.get_done_size()); + cmp = memcmp(handle.get_buffer(), write_buf + write_size - 100, 100); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + + /************** test read **************/ + // 1. read aligned data + read_buf = new char [3 * ObTmpFileGlobal::PAGE_SIZE]; + io_info.buf_ = read_buf; + io_info.size_ = 3 * ObTmpFileGlobal::PAGE_SIZE; + ret = MTL(ObTenantTmpFileManager *)->read(io_info, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(io_info.size_, handle.get_done_size()); + cmp = memcmp(handle.get_buffer(), write_buf, io_info.size_); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + // 2. read unaligned data + read_buf = new char [ObTmpFileGlobal::PAGE_SIZE]; + io_info.buf_ = read_buf; + io_info.size_ = 100; + ret = MTL(ObTenantTmpFileManager *)->read(io_info, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(io_info.size_, handle.get_done_size()); + cmp = memcmp(handle.get_buffer(), write_buf + 3 * ObTmpFileGlobal::PAGE_SIZE, io_info.size_); + ASSERT_EQ(0, cmp); + handle.reset(); + + io_info.buf_ = read_buf + 100; + io_info.size_ = ObTmpFileGlobal::PAGE_SIZE - 100; + ret = MTL(ObTenantTmpFileManager *)->read(io_info, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(io_info.size_, handle.get_done_size()); + cmp = memcmp(handle.get_buffer(), write_buf + 3 * ObTmpFileGlobal::PAGE_SIZE + 100, io_info.size_); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + read_time = ObTimeUtility::current_time() - read_time; + + ret = MTL(ObTenantTmpFileManager *)->remove(fd); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(0, MTL(ObTenantTmpFileManager *)->page_cache_controller_.flush_priority_mgr_.get_file_size()); + ASSERT_EQ(0, MTL(ObTenantTmpFileManager *)->page_cache_controller_.evict_mgr_.get_file_size()); + + LOG_INFO("test_read"); + LOG_INFO("io time", K(write_time), K(read_time)); +} + +// generate 8MB random data +// this test will check whether kv_cache caches correct pages in disk +TEST_F(TestTmpFile, test_cached_read) +{ + int ret = OB_SUCCESS; + const int64_t write_size = 8 * 1024 * 1024; // 8MB + const int64_t wbp_mem_limit = MTL(ObTenantTmpFileManager *)->page_cache_controller_.write_buffer_pool_.get_memory_limit(); + ASSERT_GT(write_size, wbp_mem_limit); + char *write_buf = new char [write_size]; + for (int64_t i = 0; i < write_size;) { + int64_t random_length = generate_random_int(1024, 8 * 1024); + int64_t random_int = generate_random_int(0, 256); + for (int64_t j = 0; j < random_length && i + j < write_size; ++j) { + write_buf[i + j] = random_int; + } + i += random_length; + } + + int64_t dir = -1; + int64_t fd = -1; + ret = MTL(ObTenantTmpFileManager *)->alloc_dir(dir); + ASSERT_EQ(OB_SUCCESS, ret); + ret = MTL(ObTenantTmpFileManager *)->open(fd, dir); + std::cout << "open temporary file: " << fd << std::endl; + ASSERT_EQ(OB_SUCCESS, ret); + ObTmpFileHandle file_handle; + ret = MTL(ObTenantTmpFileManager *)->get_tmp_file(fd, file_handle); + ASSERT_EQ(OB_SUCCESS, ret); + file_handle.get()->page_idx_cache_.max_bucket_array_capacity_ = SMALL_WBP_IDX_CACHE_MAX_CAPACITY; + + ObTmpFileIOInfo io_info; + io_info.fd_ = fd; + io_info.io_desc_.set_wait_event(2); + io_info.buf_ = write_buf; + io_info.size_ = write_size; + io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; + + // 1. Write data and wait flushing over + int64_t write_time = ObTimeUtility::current_time(); + ret = MTL(ObTenantTmpFileManager *)->write(io_info); + write_time = ObTimeUtility::current_time() - write_time; + ASSERT_EQ(OB_SUCCESS, ret); + sleep(2); + + int64_t wbp_begin_offset = file_handle.get()->cal_wbp_begin_offset(); + ASSERT_GT(wbp_begin_offset, 0); + ASSERT_EQ(wbp_begin_offset % ObTmpFileGlobal::PAGE_SIZE, 0); + + // 2. check block kv cache + common::ObArray data_items; + ret = file_handle.get()->meta_tree_.search_data_items(0,wbp_begin_offset, data_items); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(false, data_items.empty()); + for (int64_t i = 0; OB_SUCC(ret) && i < data_items.count(); i++) { + const int64_t block_index = data_items[i].block_index_; + ObTmpBlockValueHandle block_value_handle; + ret = ObTmpBlockCache::get_instance().get_block(ObTmpBlockCacheKey(block_index, MTL_ID()), + block_value_handle); + ASSERT_EQ(OB_SUCCESS, ret); + } + + // 3. read data from block kv cache + int64_t read_size = write_size; + int64_t read_offset = 0; + char *read_buf = new char [read_size]; + ObTmpFileIOHandle handle; + io_info.buf_ = read_buf; + io_info.size_ = read_size; + io_info.disable_page_cache_ = true; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(io_info.size_, handle.get_done_size()); + int cmp = memcmp(handle.get_buffer(), write_buf + read_offset, io_info.size_); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + + // 4. clear block kv cache + for (int64_t i = 0; OB_SUCC(ret) && i < data_items.count(); i++) { + const int64_t block_index = data_items[i].block_index_; + ObTmpBlockValueHandle block_value_handle; + ret = ObTmpBlockCache::get_instance().erase(ObTmpBlockCacheKey(block_index, MTL_ID())); + ASSERT_EQ(OB_SUCCESS, ret); + } + + // 5. read disk data and puts them into kv_cache + int64_t read_time = ObTimeUtility::current_time(); + read_size = wbp_begin_offset - ObTmpFileGlobal::PAGE_SIZE; + read_offset = ObTmpFileGlobal::PAGE_SIZE / 2; + read_buf = new char [read_size]; + io_info.buf_ = read_buf; + io_info.size_ = read_size; + io_info.disable_page_cache_ = false; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(io_info.size_, handle.get_done_size()); + cmp = memcmp(handle.get_buffer(), write_buf + read_offset, io_info.size_); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + + // 6. read disk data to check whether kv_cache caches correct pages + read_size = wbp_begin_offset; + read_offset = 0; + read_buf = new char [read_size]; + io_info.buf_ = read_buf; + io_info.size_ = read_size; + io_info.disable_page_cache_ = false; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(io_info.size_, handle.get_done_size()); + cmp = memcmp(handle.get_buffer(), write_buf + read_offset, io_info.size_); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + read_time = ObTimeUtility::current_time() - read_time; + + // 7. check pages in kv_cache + int64_t previous_virtual_page_id = data_items.at(0).virtual_page_id_; + for (int64_t i = 0; i < data_items.count() && OB_SUCC(ret); ++i) { + const ObSharedNothingTmpFileDataItem &data_item = data_items.at(i); + if (i > 0) { + ASSERT_GT(data_item.virtual_page_id_, previous_virtual_page_id); + previous_virtual_page_id = data_item.virtual_page_id_; + } + for (int64_t j = 0; j < data_item.physical_page_num_; j++) { + int64_t physical_page_id = data_item.physical_page_id_ + j; + ObTmpPageCacheKey key(data_item.block_index_, physical_page_id, MTL_ID()); + ObTmpPageValueHandle handle; + ret = ObTmpPageCache::get_instance().get_page(key, handle); + if (OB_FAIL(ret)) { + std::cout << "get cached page failed" << i <<" "<< data_item.block_index_<<" "<< physical_page_id << std::endl; + } + ASSERT_EQ(OB_SUCCESS, ret); + cmp = memcmp(handle.value_->get_buffer(), write_buf + (data_item.virtual_page_id_ + j) * ObTmpFileGlobal::PAGE_SIZE, ObTmpFileGlobal::PAGE_SIZE); + ASSERT_EQ(0, cmp); + } + } + + file_handle.reset(); + ret = MTL(ObTenantTmpFileManager *)->remove(fd); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(0, MTL(ObTenantTmpFileManager *)->page_cache_controller_.flush_priority_mgr_.get_file_size()); + ASSERT_EQ(0, MTL(ObTenantTmpFileManager *)->page_cache_controller_.evict_mgr_.get_file_size()); + + LOG_INFO("test_cached_read"); + LOG_INFO("io time", K(write_time), K(read_time)); +} + +// 1. append write a uncompleted tail page in memory +// 2. append write a uncompleted tail page in disk +TEST_F(TestTmpFile, test_write_tail_page) +{ + int ret = OB_SUCCESS; + const int64_t write_size = 10 * 1024; // 10KB + int64_t already_write_size = 0; + char *write_buf = new char [write_size]; + for (int64_t i = 0; i < write_size;) { + int64_t random_length = generate_random_int(1024, 8 * 1024); + int64_t random_int = generate_random_int(0, 256); + for (int64_t j = 0; j < random_length && i + j < write_size; ++j) { + write_buf[i + j] = random_int; + } + i += random_length; + } + + int64_t dir = -1; + int64_t fd = -1; + ret = MTL(ObTenantTmpFileManager *)->alloc_dir(dir); + ASSERT_EQ(OB_SUCCESS, ret); + ret = MTL(ObTenantTmpFileManager *)->open(fd, dir); + std::cout << "open temporary file: " << fd << std::endl; + ASSERT_EQ(OB_SUCCESS, ret); + ObTmpFileHandle file_handle; + ret = MTL(ObTenantTmpFileManager *)->get_tmp_file(fd, file_handle); + ASSERT_EQ(OB_SUCCESS, ret); + file_handle.get()->page_idx_cache_.max_bucket_array_capacity_ = SMALL_WBP_IDX_CACHE_MAX_CAPACITY; + file_handle.reset(); + + // 1. write 2KB data and check rightness of writing + ObTmpFileIOInfo io_info; + io_info.fd_ = fd; + io_info.io_desc_.set_wait_event(2); + io_info.buf_ = write_buf; + io_info.size_ = 2 * 1024; // 2KB + io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; + ret = MTL(ObTenantTmpFileManager *)->write(io_info); + ASSERT_EQ(OB_SUCCESS, ret); + already_write_size += io_info.size_; + + int64_t read_size = 2 * 1024; // 2KB + int64_t read_offset = 0; + char *read_buf = new char [read_size]; + ObTmpFileIOHandle handle; + io_info.buf_ = read_buf; + io_info.size_ = read_size; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(io_info.size_, handle.get_done_size()); + int cmp = memcmp(handle.get_buffer(), write_buf + read_offset, io_info.size_); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + + // 2. append write 2KB data in memory and check rightness of writing + io_info.buf_ = write_buf + 2 * 1024; // 2KB + io_info.size_ = 2 * 1024; // 2KB + ret = MTL(ObTenantTmpFileManager *)->write(io_info); + ASSERT_EQ(OB_SUCCESS, ret); + already_write_size += io_info.size_; + + read_size = 4 * 1024; // 4KB + read_offset = 0; + read_buf = new char [read_size]; + io_info.buf_ = read_buf; + io_info.size_ = read_size; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(io_info.size_, handle.get_done_size()); + cmp = memcmp(handle.get_buffer(), write_buf + read_offset, io_info.size_); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + + // 3. forcibly evict current page + ObTmpFilePageCacheController &pc_ctrl = MTL(ObTenantTmpFileManager *)->get_page_cache_controller(); + ATOMIC_SET(&pc_ctrl.flush_all_data_, true); + pc_ctrl.flush_tg_.notify_doing_flush(); + sleep(2); + ret = MTL(ObTenantTmpFileManager *)->get_tmp_file(fd, file_handle); + ASSERT_EQ(OB_SUCCESS, ret); + ret = file_handle.get()->page_cache_controller_->invoke_swap_and_wait(write_size); + ASSERT_EQ(OB_SUCCESS, ret); + int64_t wbp_begin_offset = file_handle.get()->cal_wbp_begin_offset(); + ASSERT_EQ(wbp_begin_offset, already_write_size); + + // 4. read disk page and add it into kv_cache + read_offset = 5; + read_size = already_write_size - read_offset; // 4KB - 5B + read_buf = new char [read_size]; + io_info.buf_ = read_buf; + io_info.size_ = read_size; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(io_info.size_, handle.get_done_size()); + cmp = memcmp(handle.get_buffer(), write_buf + read_offset, io_info.size_); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + + // 5. append write 6KB data in memory and check rightness of writing + io_info.buf_ = write_buf + already_write_size; + io_info.size_ = write_size - already_write_size; // 6KB + ret = MTL(ObTenantTmpFileManager *)->write(io_info); + ASSERT_EQ(OB_SUCCESS, ret); + already_write_size += io_info.size_; + + read_size = write_size; + read_offset = 0; + read_buf = new char [read_size]; + io_info.buf_ = read_buf; + io_info.size_ = read_size; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(io_info.size_, handle.get_done_size()); + cmp = memcmp(handle.get_buffer(), write_buf + read_offset, io_info.size_); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + + // 6. forcibly evict all pages and read them from disk to check whether hit old cached page in kv_cache + pc_ctrl.flush_tg_.notify_doing_flush(); + sleep(2); + ret = file_handle.get()->page_cache_controller_->invoke_swap_and_wait(write_size); + ASSERT_EQ(OB_SUCCESS, ret); + wbp_begin_offset = file_handle.get()->cal_wbp_begin_offset(); + ASSERT_EQ(wbp_begin_offset, already_write_size); + ATOMIC_SET(&pc_ctrl.flush_all_data_, false); + + read_offset = 20; + read_size = write_size - read_offset; // 10KB - 20B + read_buf = new char [read_size]; + io_info.buf_ = read_buf; + io_info.size_ = read_size; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(io_info.size_, handle.get_done_size()); + cmp = memcmp(handle.get_buffer(), write_buf + read_offset, io_info.size_); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + + file_handle.reset(); + ret = MTL(ObTenantTmpFileManager *)->remove(fd); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(0, MTL(ObTenantTmpFileManager *)->page_cache_controller_.flush_priority_mgr_.get_file_size()); + ASSERT_EQ(0, MTL(ObTenantTmpFileManager *)->page_cache_controller_.evict_mgr_.get_file_size()); + + LOG_INFO("test_write_tail_page"); +} + +// 1. truncate special cases +// 2. truncate disk data (truncate_offset < wbp begin offset) +// 3. truncate memory data and disk data (wbp begin offset < truncate_offset < file_size_) +// 4. truncate() do nothing (truncate_offset < file's truncate_offset_) +// 5. invalid truncate_offset checking +TEST_F(TestTmpFile, test_tmp_file_truncate) +{ + int ret = OB_SUCCESS; + const int64_t data_size = 30 * 1024 * 1024; // 30MB + const int64_t wbp_mem_limit = MTL(ObTenantTmpFileManager *)->page_cache_controller_.write_buffer_pool_.get_memory_limit(); + ASSERT_GT(data_size, wbp_mem_limit); + char *write_buf = new char [data_size]; + int64_t already_write_size = 0; + for (int64_t i = 0; i < data_size;) { + int64_t random_length = generate_random_int(1024, 8 * 1024); + int64_t random_int = generate_random_int(0, 256); + for (int64_t j = 0; j < random_length && i + j < data_size; ++j) { + write_buf[i + j] = random_int; + } + i += random_length; + } + + int64_t dir = -1; + int64_t fd = -1; + ret = MTL(ObTenantTmpFileManager *)->alloc_dir(dir); + ASSERT_EQ(OB_SUCCESS, ret); + ret = MTL(ObTenantTmpFileManager *)->open(fd, dir); + std::cout << "open temporary file: " << fd << std::endl; + ASSERT_EQ(OB_SUCCESS, ret); + ObTmpFileHandle file_handle; + ret = MTL(ObTenantTmpFileManager *)->get_tmp_file(fd, file_handle); + ASSERT_EQ(OB_SUCCESS, ret); + file_handle.get()->page_idx_cache_.max_bucket_array_capacity_ = SMALL_WBP_IDX_CACHE_MAX_CAPACITY; + + // 1. truncate special cases + // 1.1 truncate a file with several pages + // 1.1.1 write two pages and check rightness of writing + ObTmpFileIOInfo io_info; + io_info.fd_ = fd; + io_info.io_desc_.set_wait_event(2); + io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; + io_info.buf_ = write_buf; + io_info.size_ = 2 * ObTmpFileGlobal::PAGE_SIZE; + ret = MTL(ObTenantTmpFileManager *)->write(io_info); + ASSERT_EQ(OB_SUCCESS, ret); + already_write_size += io_info.size_; + + int64_t read_offset = 0; + int64_t read_size = already_write_size; + char *read_buf = new char [read_size]; + ObTmpFileIOHandle handle; + io_info.buf_ = read_buf; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(read_size, handle.get_done_size()); + int cmp = memcmp(handle.get_buffer(), write_buf + read_offset, read_size); + ASSERT_EQ(0, cmp); + MEMSET(read_buf, 0, read_size); + + // 1.1.2 truncate to the middle offset of the first page + ASSERT_EQ(file_handle.get()->cached_page_nums_, 2); + uint32_t begin_page_id = file_handle.get()->begin_page_id_; + uint32_t end_page_id = file_handle.get()->end_page_id_; + int64_t truncate_offset = ObTmpFileGlobal::PAGE_SIZE / 2; + ret = MTL(ObTenantTmpFileManager *)->truncate(fd, truncate_offset); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(file_handle.get()->begin_page_id_, begin_page_id); + + // read_offset = 0; + // read_size = already_write_size; + io_info.buf_ = read_buf; + io_info.size_ = read_size; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(read_size, handle.get_done_size()); + MEMSET(write_buf, 0, truncate_offset); + cmp = memcmp(handle.get_buffer(), write_buf + read_offset, read_size); + ASSERT_EQ(0, cmp); + handle.reset(); + MEMSET(read_buf, 0, read_size); + + // 1.1.3 truncate the first page + truncate_offset = ObTmpFileGlobal::PAGE_SIZE; + ret = MTL(ObTenantTmpFileManager *)->truncate(fd, truncate_offset); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(file_handle.get()->begin_page_id_, end_page_id); + ASSERT_EQ(file_handle.get()->cached_page_nums_, 1); + + // read_offset = 0; + // read_size = already_write_size; + io_info.buf_ = read_buf; + io_info.size_ = read_size; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(read_size, handle.get_done_size()); + MEMSET(write_buf, 0, truncate_offset); + cmp = memcmp(handle.get_buffer(), write_buf + read_offset, read_size); + ASSERT_EQ(0, cmp); + handle.reset(); + MEMSET(read_buf, 0, read_size); + + // 1.1.4 truncate whole pages + truncate_offset = already_write_size; + ret = MTL(ObTenantTmpFileManager *)->truncate(fd, truncate_offset); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(file_handle.get()->begin_page_id_, ObTmpFileGlobal::INVALID_PAGE_ID); + ASSERT_EQ(file_handle.get()->cached_page_nums_, 0); + + // read_offset = 0; + // read_size = already_write_size; + io_info.buf_ = read_buf; + io_info.size_ = read_size; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(read_size, handle.get_done_size()); + MEMSET(write_buf, 0, truncate_offset); + cmp = memcmp(handle.get_buffer(), write_buf + read_offset, read_size); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + + // 1.2 truncate a offset of a page whose page index is not in index cache (to mock the sparsify case of index cache) + // 1.2.1 write three pages and check rightness of writing + read_offset = already_write_size; + io_info.buf_ = write_buf + already_write_size; + io_info.size_ = 3 * ObTmpFileGlobal::PAGE_SIZE; + ret = MTL(ObTenantTmpFileManager *)->write(io_info); + ASSERT_EQ(OB_SUCCESS, ret); + already_write_size += io_info.size_; + + read_size = io_info.size_; + read_buf = new char [read_size]; + io_info.buf_ = read_buf; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(read_size, handle.get_done_size()); + cmp = memcmp(handle.get_buffer(), write_buf + read_offset, read_size); + ASSERT_EQ(0, cmp); + delete[] read_buf; + + // 1.2.2 pop the first page index of index cache of file + ASSERT_NE(file_handle.get()->page_idx_cache_.page_buckets_, nullptr); + ASSERT_EQ(file_handle.get()->page_idx_cache_.size(), 1); + ObTmpFileWBPIndexCache::ObTmpFilePageIndexBucket *bucket = file_handle.get()->page_idx_cache_.page_buckets_->at(0); + ASSERT_NE(bucket, nullptr); + ASSERT_EQ(bucket->size(), 3); + begin_page_id = file_handle.get()->begin_page_id_; + end_page_id = file_handle.get()->end_page_id_; + ASSERT_EQ(bucket->page_indexes_.at(bucket->left_), begin_page_id); + ASSERT_EQ(bucket->page_indexes_.at(bucket->right_), end_page_id); + ret = bucket->pop_(); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(bucket->size(), 2); + ASSERT_NE(bucket->page_indexes_.at(bucket->left_), begin_page_id); + ASSERT_EQ(bucket->page_indexes_.at(bucket->right_), end_page_id); + + // 1.2.3 truncate the first page + ASSERT_EQ(file_handle.get()->cached_page_nums_, 3); + truncate_offset = read_offset + ObTmpFileGlobal::PAGE_SIZE; + ret = MTL(ObTenantTmpFileManager *)->truncate(fd, truncate_offset); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(file_handle.get()->cached_page_nums_, 2); + + read_size = already_write_size - read_offset; + read_buf = new char [read_size]; + io_info.buf_ = read_buf; + io_info.size_ = read_size; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(read_size, handle.get_done_size()); + MEMSET(write_buf, 0, truncate_offset); + cmp = memcmp(handle.get_buffer(), write_buf + read_offset, read_size); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + + // 2. truncate disk data (truncate_offset < wbp begin offset) + read_offset = already_write_size; + io_info.buf_ = write_buf + already_write_size; + io_info.size_ = data_size - already_write_size; + ret = MTL(ObTenantTmpFileManager *)->write(io_info); + ASSERT_EQ(OB_SUCCESS, ret); + int64_t wbp_begin_offset = file_handle.get()->cal_wbp_begin_offset(); + ASSERT_GT(wbp_begin_offset, 0); + + truncate_offset = wbp_begin_offset/2; + read_size = wbp_begin_offset - read_offset; + ret = MTL(ObTenantTmpFileManager *)->truncate(fd, truncate_offset); + ASSERT_EQ(OB_SUCCESS, ret); + read_buf = new char [read_size]; + io_info.buf_ = read_buf; + io_info.size_ = read_size; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(read_size, handle.get_done_size()); + MEMSET(write_buf, 0, truncate_offset); + cmp = memcmp(handle.get_buffer(), write_buf + read_offset, read_size); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + + // 3. truncate memory data (truncate_offset < file_size_) + // 3.1 truncate_offset is unaligned + read_offset = truncate_offset; + truncate_offset = (wbp_begin_offset + data_size) / 2 - ObTmpFileGlobal::PAGE_SIZE / 2; + read_size = data_size - read_offset; + ret = MTL(ObTenantTmpFileManager *)->truncate(fd, truncate_offset); + ASSERT_EQ(OB_SUCCESS, ret); + read_buf = new char [read_size]; + io_info.buf_ = read_buf; + io_info.size_ = read_size; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(read_size, handle.get_done_size()); + MEMSET(write_buf, 0, truncate_offset); + cmp = memcmp(handle.get_buffer(), write_buf + read_offset, read_size); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + + read_offset = truncate_offset; + truncate_offset = upper_align(truncate_offset, ObTmpFileGlobal::PAGE_SIZE) + ObTmpFileGlobal::PAGE_SIZE; + read_size = data_size - read_offset; + ret = MTL(ObTenantTmpFileManager *)->truncate(fd, truncate_offset); + ASSERT_EQ(OB_SUCCESS, ret); + read_buf = new char [read_size]; + io_info.buf_ = read_buf; + io_info.size_ = read_size; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(read_size, handle.get_done_size()); + MEMSET(write_buf, 0, truncate_offset); + cmp = memcmp(handle.get_buffer(), write_buf + read_offset, read_size); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + + // 3.2 truncate_offset is aligned + ASSERT_EQ(truncate_offset % ObTmpFileGlobal::PAGE_SIZE, 0); + read_offset = truncate_offset; + truncate_offset = truncate_offset + 5 * ObTmpFileGlobal::PAGE_SIZE; + read_size = data_size - read_offset; + ret = MTL(ObTenantTmpFileManager *)->truncate(fd, truncate_offset); + ASSERT_EQ(OB_SUCCESS, ret); + read_buf = new char [read_size]; + io_info.buf_ = read_buf; + io_info.size_ = read_size; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(read_size, handle.get_done_size()); + MEMSET(write_buf, 0, truncate_offset); + cmp = memcmp(handle.get_buffer(), write_buf + read_offset, read_size); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + + read_offset = truncate_offset; + truncate_offset = data_size; + read_size = data_size - read_offset; + ret = MTL(ObTenantTmpFileManager *)->truncate(fd, truncate_offset); + ASSERT_EQ(OB_SUCCESS, ret); + read_buf = new char [read_size]; + io_info.buf_ = read_buf; + io_info.size_ = read_size; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(read_size, handle.get_done_size()); + MEMSET(write_buf, 0, truncate_offset); + cmp = memcmp(handle.get_buffer(), write_buf + read_offset, read_size); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + + // 4. truncate() do nothing (truncate_offset < file's truncate_offset_) + int64_t old_truncate_offset = truncate_offset; + ASSERT_EQ(old_truncate_offset, file_handle.get()->truncated_offset_); + truncate_offset = wbp_begin_offset; + ret = MTL(ObTenantTmpFileManager *)->truncate(fd, truncate_offset); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(old_truncate_offset, file_handle.get()->truncated_offset_); + + // 5. invalid truncate_offset checking + ret = MTL(ObTenantTmpFileManager *)->truncate(fd, -1); + ASSERT_NE(OB_SUCCESS, ret); + ret = MTL(ObTenantTmpFileManager *)->truncate(fd, data_size + 10); + ASSERT_NE(OB_SUCCESS, ret); + + file_handle.reset(); + ret = MTL(ObTenantTmpFileManager *)->remove(fd); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(0, MTL(ObTenantTmpFileManager *)->page_cache_controller_.flush_priority_mgr_.get_file_size()); + ASSERT_EQ(0, MTL(ObTenantTmpFileManager *)->page_cache_controller_.evict_mgr_.get_file_size()); + + LOG_INFO("test_tmp_file_truncate"); +} + +// generate 750MB random data. +// this test will trigger flush and evict logic for both data and meta pages. +void test_big_file(const int64_t write_size, const int64_t wbp_mem_limit, ObTmpFileIOInfo io_info) +{ + int ret = OB_SUCCESS; + ASSERT_GT(write_size, wbp_mem_limit); + MTL(ObTenantTmpFileManager *)->page_cache_controller_.write_buffer_pool_.default_wbp_memory_limit_ = wbp_mem_limit; + const int64_t macro_block_size = OB_SERVER_BLOCK_MGR.get_macro_block_size(); + int cmp = 0; + char *write_buf = (char *)malloc(write_size); + for (int64_t i = 0; i < write_size;) { + int64_t random_length = generate_random_int(1024, 8 * 1024); + int64_t random_int = generate_random_int(0, 256); + for (int64_t j = 0; j < random_length && i + j < write_size; ++j) { + write_buf[i + j] = random_int; + } + i += random_length; + } + + int64_t dir = -1; + int64_t fd = -1; + ret = MTL(ObTenantTmpFileManager *)->alloc_dir(dir); + ASSERT_EQ(OB_SUCCESS, ret); + ret = MTL(ObTenantTmpFileManager *)->open(fd, dir); + std::cout << "open temporary file: " << fd << " tenant_id:"<< MTL_ID() << std::endl; + ASSERT_EQ(OB_SUCCESS, ret); + ObTmpFileHandle file_handle; + ret = MTL(ObTenantTmpFileManager *)->get_tmp_file(fd, file_handle); + ASSERT_EQ(OB_SUCCESS, ret); + file_handle.get()->page_idx_cache_.max_bucket_array_capacity_ = SMALL_WBP_IDX_CACHE_MAX_CAPACITY; + file_handle.reset(); + io_info.fd_ = fd; + io_info.io_desc_.set_wait_event(2); + io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; + + // 1. write 750MB data + io_info.buf_ = write_buf; + io_info.size_ = write_size; + int64_t write_time = ObTimeUtility::current_time(); + ret = MTL(ObTenantTmpFileManager *)->write(io_info); + ASSERT_EQ(OB_SUCCESS, ret); + write_time = ObTimeUtility::current_time() - write_time; + + // 2. read 750MB data + ObTmpFileIOHandle handle; + int64_t read_size = write_size; + char *read_buf = new char [read_size]; + io_info.buf_ = read_buf; + io_info.size_ = read_size; + ret = MTL(ObTenantTmpFileManager *)->read(io_info, handle); + cmp = memcmp(handle.get_buffer(), write_buf, handle.get_done_size()); + ASSERT_EQ(read_size, handle.get_done_size()); + handle.reset(); + ASSERT_EQ(0, cmp); + memset(read_buf, 0, read_size); + + // 3. attempt to read data when reach the end of file + int64_t read_time = ObTimeUtility::current_time(); + io_info.size_ = 10; + ret = MTL(ObTenantTmpFileManager *)->read(io_info, handle); + ASSERT_EQ(OB_ITER_END, ret); + handle.reset(); + + // 4. pread 2MB + int64_t read_offset = 100; + read_size = macro_block_size; + io_info.size_ = read_size; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(read_size, handle.get_done_size()); + cmp = memcmp(handle.get_buffer(), write_buf + read_offset, handle.get_done_size()); + handle.reset(); + ASSERT_EQ(0, cmp); + memset(read_buf + read_offset, 0, read_size); + + // 5. attempt to read data when reach the end of file (after pread) + io_info.size_ = 10; + ret = MTL(ObTenantTmpFileManager *)->read(io_info, handle); + ASSERT_EQ(OB_ITER_END, ret); + handle.reset(); + + // 6. pread data which has been read to use kv_cache + int loop_count = 30; + for (int i = 0; i < loop_count; ++i) { + read_offset = macro_block_size * (40 + i); + read_size = macro_block_size * 2; + io_info.size_ = read_size; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(read_size, handle.get_done_size()); + cmp = memcmp(handle.get_buffer(), write_buf + read_offset, handle.get_done_size()); + handle.reset(); + ASSERT_EQ(0, cmp); + memset(read_buf + read_offset, 0, read_size); + } + read_time = ObTimeUtility::current_time() - read_time; + + free(write_buf); + free(read_buf); + + ret = MTL(ObTenantTmpFileManager *)->remove(fd); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(0, MTL(ObTenantTmpFileManager *)->page_cache_controller_.flush_priority_mgr_.get_file_size()); + ASSERT_EQ(0, MTL(ObTenantTmpFileManager *)->page_cache_controller_.evict_mgr_.get_file_size()); + + STORAGE_LOG(INFO, "test_big_file", K(io_info.disable_page_cache_)); + STORAGE_LOG(INFO, "io time", K(write_time), K(read_time)); +} + +TEST_F(TestTmpFile, test_big_file_with_small_wbp) +{ + const int64_t write_size = 150 * 1024 * 1024; // write 150MB data + const int64_t wbp_mem_limit = SMALL_WBP_MEM_LIMIT; + ObTmpFileIOInfo io_info; + io_info.disable_page_cache_ = true; + test_big_file(write_size, wbp_mem_limit, io_info); +} + +// generate 16MB random data for four files. (total 64MB) +// 1. the first three files write and read 1020KB data (will not trigger flushing) +// 2. the 4th file writes and reads 3MB+1020KB data (will trigger flushing in the processing of writing) +// 3. the first three files write and read 1MB data 3 times (total 3MB) +// 4. each file read and write 12MB+4KB data +TEST_F(TestTmpFile, test_multi_file_single_thread_read_write) +{ + int ret = OB_SUCCESS; + const int64_t buf_size = 64 * 1024 * 1024; // 64MB + const int64_t wbp_mem_limit = MTL(ObTenantTmpFileManager *)->page_cache_controller_.write_buffer_pool_.get_memory_limit(); + ASSERT_GT(buf_size, wbp_mem_limit); + char *random_buf = new char [buf_size]; + for (int64_t i = 0; i < buf_size;) { + int64_t random_length = generate_random_int(1024, 8 * 1024); + int64_t random_int = generate_random_int(0, 256); + for (int64_t j = 0; j < random_length && i + j < buf_size; ++j) { + random_buf[i + j] = random_int; + } + i += random_length; + } + + int64_t dir1 = -1; + int64_t dir2 = -1; + ret = MTL(ObTenantTmpFileManager *)->alloc_dir(dir1); + ASSERT_EQ(OB_SUCCESS, ret); + ret = MTL(ObTenantTmpFileManager *)->alloc_dir(dir2); + ASSERT_EQ(OB_SUCCESS, ret); + const int64_t file_num = 4; + char *write_bufs[file_num] = {nullptr}; + int64_t already_write_sizes[file_num] = {0}; + int64_t fds[file_num] = {-1}; + for (int i = 0; i < file_num; ++i) { + int64_t dir = i % 2 == 0 ? dir1 : dir2; + int64_t fd = -1; + ret = MTL(ObTenantTmpFileManager *)->open(fd, dir); + std::cout << "open temporary file: " << fd << std::endl; + ASSERT_EQ(OB_SUCCESS, ret); + fds[i] = fd; + write_bufs[i] = random_buf + i * buf_size / file_num; + ObTmpFileHandle file_handle; + ret = MTL(ObTenantTmpFileManager *)->get_tmp_file(fd, file_handle); + ASSERT_EQ(OB_SUCCESS, ret); + file_handle.get()->page_idx_cache_.max_bucket_array_capacity_ = SMALL_WBP_IDX_CACHE_MAX_CAPACITY; + file_handle.reset(); + } + ObTmpFileIOInfo io_info; + io_info.io_desc_.set_wait_event(2); + io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; + ObTmpFileIOHandle handle; + int cmp = 0; + + // 1. the first three files write and read 1020KB data (will not trigger flushing) + int64_t write_size = 1020; + io_info.size_ = write_size; + for (int i = 0; OB_SUCC(ret) && i < file_num - 1; i++) { + io_info.fd_ = fds[i]; + io_info.buf_ = write_bufs[i] + already_write_sizes[i]; + ret = MTL(ObTenantTmpFileManager *)->write(io_info); + ASSERT_EQ(OB_SUCCESS, ret); + } + + char *read_buf = new char [write_size]; + io_info.buf_ = read_buf; + io_info.size_ = write_size; + for (int i = 0; OB_SUCC(ret) && i < file_num - 1; i++) { + io_info.fd_ = fds[i]; + ret = MTL(ObTenantTmpFileManager *)->read(io_info, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(io_info.size_, handle.get_done_size()); + cmp = memcmp(handle.get_buffer(), write_bufs[i] + already_write_sizes[i], io_info.size_); + ASSERT_EQ(0, cmp); + handle.reset(); + memset(read_buf, 0, write_size); + } + delete[] read_buf; + + for (int i = 0; OB_SUCC(ret) && i < file_num - 1; i++) { + already_write_sizes[i] += write_size; + } + // 2. the 4th file writes and reads 3MB+1020KB data (will trigger flushing in the processing of writing) + write_size = 1020 + 3 * 1024 * 1024; + io_info.size_ = write_size; + io_info.fd_ = fds[file_num - 1]; + io_info.buf_ = write_bufs[file_num - 1] + already_write_sizes[file_num - 1]; + ret = MTL(ObTenantTmpFileManager *)->write(io_info); + ASSERT_EQ(OB_SUCCESS, ret); + + read_buf = new char [write_size]; + io_info.buf_ = read_buf; + io_info.size_ = write_size; + io_info.fd_ = fds[file_num - 1]; + ret = MTL(ObTenantTmpFileManager *)->read(io_info, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(io_info.size_, handle.get_done_size()); + cmp = memcmp(handle.get_buffer(), write_bufs[file_num - 1] + already_write_sizes[file_num - 1], io_info.size_); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + already_write_sizes[file_num - 1] += write_size; + + // 3. the first three files write and read 1MB data 3 times + write_size = 1024 * 1024; + io_info.size_ = write_size; + const int loop_cnt = 3; + read_buf = new char [write_size]; + for (int cnt = 0; OB_SUCC(ret) && cnt < loop_cnt; cnt++) { + for (int i = 0; OB_SUCC(ret) && i < file_num - 1; i++) { + io_info.fd_ = fds[i]; + io_info.buf_ = write_bufs[i] + already_write_sizes[i]; + ret = MTL(ObTenantTmpFileManager *)->write(io_info); + ASSERT_EQ(OB_SUCCESS, ret); + } + + io_info.buf_ = read_buf; + io_info.size_ = write_size; + for (int i = 0; OB_SUCC(ret) && i < file_num - 1; i++) { + io_info.fd_ = fds[i]; + ret = MTL(ObTenantTmpFileManager *)->read(io_info, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(io_info.size_, handle.get_done_size()); + cmp = memcmp(handle.get_buffer(), write_bufs[i] + already_write_sizes[i], io_info.size_); + ASSERT_EQ(0, cmp); + handle.reset(); + memset(read_buf, 0, write_size); + } + for (int i = 0; OB_SUCC(ret) && i < file_num - 1; i++) { + already_write_sizes[i] += write_size; + } + } + delete[] read_buf; + // 4. each file read and write 12MB+4KB data + write_size = 12 * 1024 * 1024 + 4 * 1024; + io_info.size_ = write_size; + read_buf = new char [write_size]; + for (int i = 0; OB_SUCC(ret) && i < file_num; i++) { + io_info.fd_ = fds[i]; + io_info.buf_ = write_bufs[i] + already_write_sizes[i]; + ret = MTL(ObTenantTmpFileManager *)->write(io_info); + ASSERT_EQ(OB_SUCCESS, ret); + } + + io_info.buf_ = read_buf; + io_info.size_ = write_size; + for (int i = 0; OB_SUCC(ret) && i < file_num; i++) { + io_info.fd_ = fds[i]; + ret = MTL(ObTenantTmpFileManager *)->read(io_info, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(io_info.size_, handle.get_done_size()); + cmp = memcmp(handle.get_buffer(), write_bufs[i] + already_write_sizes[i], io_info.size_); + ASSERT_EQ(0, cmp); + handle.reset(); + memset(read_buf, 0, write_size); + } + delete[] read_buf; + for (int i = 0; OB_SUCC(ret) && i < file_num; i++) { + already_write_sizes[i] += write_size; + } + + for (int i = 0; OB_SUCC(ret) && i < file_num; i++) { + ret = MTL(ObTenantTmpFileManager *)->remove(fds[i]); + ASSERT_EQ(OB_SUCCESS, ret); + } + ASSERT_EQ(0, MTL(ObTenantTmpFileManager *)->page_cache_controller_.flush_priority_mgr_.get_file_size()); + ASSERT_EQ(0, MTL(ObTenantTmpFileManager *)->page_cache_controller_.evict_mgr_.get_file_size()); + + LOG_INFO("test_multi_file_single_thread_read_write"); +} + +TEST_F(TestTmpFile, test_single_file_multi_thread_read_write) +{ + int ret = OB_SUCCESS; + const int64_t read_thread_cnt = 4; + const int64_t file_cnt = 1; + const int64_t batch_size = 64 * 1024 * 1024; // 64MB + const int64_t batch_num = 4; + TestMultiTmpFileStress test(MTL_CTX()); + int64_t dir = -1; + ret = MTL(ObTenantTmpFileManager *)->alloc_dir(dir); + ASSERT_EQ(OB_SUCCESS, ret); + ret = test.init(file_cnt, dir, read_thread_cnt, batch_size, batch_num); + ASSERT_EQ(OB_SUCCESS, ret); + int64_t io_time = ObTimeUtility::current_time(); + test.start(); + test.wait(); + io_time = ObTimeUtility::current_time() - io_time; + + STORAGE_LOG(INFO, "test_single_file_multi_thread_read_write"); + STORAGE_LOG(INFO, "io time", K(io_time)); +} + +TEST_F(TestTmpFile, test_multi_file_multi_thread_read_write) +{ + int ret = OB_SUCCESS; + MTL(ObTenantTmpFileManager *)->page_cache_controller_.write_buffer_pool_.set_max_data_page_usage_ratio_(0.99); + const int64_t read_thread_cnt = 4; + const int64_t file_cnt = 4; + const int64_t batch_size = 16 * 1024 * 1024; // 16MB + const int64_t batch_num = 4; + TestMultiTmpFileStress test(MTL_CTX()); + int64_t dir = -1; + ret = MTL(ObTenantTmpFileManager *)->alloc_dir(dir); + ASSERT_EQ(OB_SUCCESS, ret); + ret = test.init(file_cnt, dir, read_thread_cnt, batch_size, batch_num); + ASSERT_EQ(OB_SUCCESS, ret); + int64_t io_time = ObTimeUtility::current_time(); + test.start(); + test.wait(); + io_time = ObTimeUtility::current_time() - io_time; + MTL(ObTenantTmpFileManager *)->page_cache_controller_.write_buffer_pool_.set_max_data_page_usage_ratio_(0.90); + STORAGE_LOG(INFO, "test_multi_file_multi_thread_read_write"); + STORAGE_LOG(INFO, "io time", K(io_time)); +} + +TEST_F(TestTmpFile, test_more_files_more_threads_read_write) +{ + int ret = OB_SUCCESS; + const int64_t read_thread_cnt = 1; + const int64_t file_cnt = 128; + const int64_t batch_size = 3 * 1024 * 1024; + const int64_t batch_num = 2; // total 128 * 3MB * 2 = 768MB + TestMultiTmpFileStress test(MTL_CTX()); + int64_t dir = -1; + ret = MTL(ObTenantTmpFileManager *)->alloc_dir(dir); + ASSERT_EQ(OB_SUCCESS, ret); + ret = test.init(file_cnt, dir, read_thread_cnt, batch_size, batch_num); + ASSERT_EQ(OB_SUCCESS, ret); + int64_t io_time = ObTimeUtility::current_time(); + test.start(); + test.wait(); + io_time = ObTimeUtility::current_time() - io_time; + + STORAGE_LOG(INFO, "test_more_files_more_threads_read_write"); + STORAGE_LOG(INFO, "io time", K(io_time)); +} + +TEST_F(TestTmpFile, test_big_file) +{ + const int64_t write_size = 750 * 1024 * 1024; // write 750MB data + const int64_t wbp_mem_limit = BIG_WBP_MEM_LIMIT; + ObTmpFileIOInfo io_info; + io_info.disable_page_cache_ = false; + test_big_file(write_size, wbp_mem_limit, io_info); +} + +TEST_F(TestTmpFile, test_big_file_disable_page_cache) +{ + const int64_t write_size = 750 * 1024 * 1024; // write 750MB data + const int64_t wbp_mem_limit = BIG_WBP_MEM_LIMIT; + ObTmpFileIOInfo io_info; + io_info.disable_page_cache_ = true; + test_big_file(write_size, wbp_mem_limit, io_info); +} + +} // namespace oceanbase + +int main(int argc, char **argv) +{ + int ret = 0; + system("rm -f ./test_sn_tmp_file.log*"); + system("rm -rf ./run*"); + OB_LOGGER.set_file_name("test_sn_tmp_file.log", true); + OB_LOGGER.set_log_level("INFO"); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/mittest/mtlenv/storage/tmp_file/test_tmp_file_block_manager.cpp b/mittest/mtlenv/storage/tmp_file/test_tmp_file_block_manager.cpp new file mode 100644 index 000000000..8b94e6ebe --- /dev/null +++ b/mittest/mtlenv/storage/tmp_file/test_tmp_file_block_manager.cpp @@ -0,0 +1,598 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include +#include +#include +#define USING_LOG_PREFIX STORAGE +#define protected public +#define private public +#include "mittest/mtlenv/mock_tenant_module_env.h" +#include "storage/tmp_file/ob_tmp_file_global.h" +#include "storage/tmp_file/ob_tmp_file_block_manager.h" + +namespace oceanbase +{ +using namespace tmp_file; +/* ---------------------------- Unittest Class ----------------------------- */ +class TestTmpFileBlock : public ::testing::Test +{ +public: + TestTmpFileBlock() = default; + virtual ~TestTmpFileBlock() = default; + static void SetUpTestCase(); + static void TearDownTestCase(); +}; + +void TestTmpFileBlock::SetUpTestCase() +{ + ASSERT_EQ(OB_SUCCESS, MockTenantModuleEnv::get_instance().init()); +} + +void TestTmpFileBlock::TearDownTestCase() +{ + MockTenantModuleEnv::get_instance().destroy(); +} + +TEST_F(TestTmpFileBlock, test_block_manager_op) +{ + int ret = OB_SUCCESS; + ObTmpFileBlockManager block_mgr; + static const int64_t META_DEFAULT_LIMIT = 15 * 1024L * 1024L * 1024L; + ret = block_mgr.init(MTL_ID(), META_DEFAULT_LIMIT); + ASSERT_EQ(OB_SUCCESS, ret); + + int64_t begin_page_id = 0; + int64_t page_num = ObTmpFileGlobal::BLOCK_PAGE_NUMS; + int64_t block_index1 = ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX; + int64_t block_index2 = ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX; + int64_t block_index3 = ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX; + int64_t block_index4 = ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX; + int64_t block_index5 = ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX; + int64_t invalid_logic_block_index = ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX; + // 1. create tmp file blocks + ret = block_mgr.create_tmp_file_block(begin_page_id + 10, page_num - 20, block_index1); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(block_index1, ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX); + + ret = block_mgr.create_tmp_file_block(begin_page_id, page_num, block_index2); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(block_index2, ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX); + + ret = block_mgr.create_tmp_file_block(begin_page_id + 10, page_num - 10, block_index3); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(block_index3, ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX); + + ret = block_mgr.create_tmp_file_block(begin_page_id, page_num, block_index4); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(block_index4, ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX); + + ret = block_mgr.create_tmp_file_block(begin_page_id, page_num, block_index5); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(block_index5, ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX); + + ret = block_mgr.create_tmp_file_block(-1, page_num, invalid_logic_block_index); + ASSERT_EQ(OB_INVALID_ARGUMENT, ret); + ret = block_mgr.create_tmp_file_block(begin_page_id, -1, invalid_logic_block_index); + ASSERT_EQ(OB_INVALID_ARGUMENT, ret); + ret = block_mgr.create_tmp_file_block(begin_page_id, page_num+1, invalid_logic_block_index); + ASSERT_EQ(OB_INVALID_ARGUMENT, ret); + ret = block_mgr.create_tmp_file_block(begin_page_id+1, page_num, invalid_logic_block_index); + ASSERT_EQ(OB_INVALID_ARGUMENT, ret); + + // 2. alloc three macro block + ObMacroBlockHandle macro_block_handle1; + ObMacroBlockHandle macro_block_handle2; + ObMacroBlockHandle macro_block_handle3; + ret = OB_SERVER_BLOCK_MGR.alloc_block(macro_block_handle1); + ASSERT_EQ(OB_SUCCESS, ret); + ret = OB_SERVER_BLOCK_MGR.alloc_block(macro_block_handle2); + ASSERT_EQ(OB_SUCCESS, ret); + ret = OB_SERVER_BLOCK_MGR.alloc_block(macro_block_handle3); + ASSERT_EQ(OB_SUCCESS, ret); + MacroBlockId macro_block_id1 = macro_block_handle1.get_macro_id(); + ASSERT_EQ(true, macro_block_id1.is_valid()); + MacroBlockId macro_block_id2 = macro_block_handle2.get_macro_id(); + ASSERT_EQ(true, macro_block_id2.is_valid()); + MacroBlockId macro_block_id3 = macro_block_handle3.get_macro_id(); + ASSERT_EQ(true, macro_block_id3.is_valid()); + + // 3. switch state op + ret = block_mgr.write_back_failed(block_index1); + ASSERT_EQ(OB_OP_NOT_ALLOW, ret); + ret = block_mgr.write_back_succ(block_index1, macro_block_id1); + ASSERT_EQ(OB_OP_NOT_ALLOW, ret); + + ret = block_mgr.write_back_start(block_index2); + ASSERT_EQ(OB_SUCCESS, ret); + ret = block_mgr.write_back_start(block_index3); + ASSERT_EQ(OB_SUCCESS, ret); + ret = block_mgr.write_back_start(block_index4); + ASSERT_EQ(OB_SUCCESS, ret); + ret = block_mgr.write_back_start(block_index5); + ASSERT_EQ(OB_SUCCESS, ret); + + // 4. test releasing all pages of block which is writing back + // 4.1 block exists in block manager before releasing all pages of block + ObTmpFileBlockHandle handle; + ret = block_mgr.get_tmp_file_block_handle(block_index4, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(nullptr, handle.get()); + handle.reset(); + ret = block_mgr.get_tmp_file_block_handle(block_index5, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(nullptr, handle.get()); + handle.reset(); + // 4.2 block exists in block manager after releasing block + ret = block_mgr.release_tmp_file_page(block_index4, begin_page_id, page_num); + ASSERT_EQ(OB_SUCCESS, ret); + ret = block_mgr.release_tmp_file_page(block_index5, begin_page_id, page_num); + ASSERT_EQ(OB_SUCCESS, ret); + + ret = block_mgr.get_tmp_file_block_handle(block_index4, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(nullptr, handle.get()); + handle.reset(); + ret = block_mgr.get_tmp_file_block_handle(block_index5, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(nullptr, handle.get()); + handle.reset(); + // 4.3 block doesn't exist in block manager after writing back over + ret = block_mgr.write_back_succ(block_index4, macro_block_id1); + ASSERT_EQ(OB_SUCCESS, ret); + ret = block_mgr.write_back_failed(block_index5); + ASSERT_EQ(OB_SUCCESS, ret); + ret = block_mgr.get_tmp_file_block_handle(block_index4, handle); + ASSERT_EQ(OB_ENTRY_NOT_EXIST, ret); + ret = block_mgr.get_tmp_file_block_handle(block_index5, handle); + ASSERT_EQ(OB_ENTRY_NOT_EXIST, ret); + + // 5. test releasing pages of block which is writing back over + ret = block_mgr.write_back_succ(block_index2, macro_block_id2); + ASSERT_EQ(OB_SUCCESS, ret); + ret = block_mgr.write_back_succ(block_index2, macro_block_id2); + ASSERT_NE(OB_SUCCESS, ret); + ret = block_mgr.write_back_succ(block_index3, MacroBlockId()); + ASSERT_NE(OB_SUCCESS, ret); + ret = block_mgr.write_back_succ(block_index3, macro_block_id3); + ASSERT_EQ(OB_SUCCESS, ret); + + ret = block_mgr.release_tmp_file_page(block_index1, begin_page_id + 10, page_num - 20); + ASSERT_EQ(OB_SUCCESS, ret); + ret = block_mgr.release_tmp_file_page(block_index2, begin_page_id, page_num); + ASSERT_EQ(OB_SUCCESS, ret); + ret = block_mgr.release_tmp_file_page(block_index3, begin_page_id + 10, 10); + ASSERT_EQ(OB_SUCCESS, ret); + + ret = block_mgr.get_tmp_file_block_handle(block_index1, handle); + ASSERT_EQ(OB_ENTRY_NOT_EXIST, ret); + ret = block_mgr.get_tmp_file_block_handle(block_index2, handle); + ASSERT_EQ(OB_ENTRY_NOT_EXIST, ret); + ret = block_mgr.get_tmp_file_block_handle(block_index3, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(nullptr, handle.get()); + + LOG_INFO("test_block_manager_op"); +} + +TEST_F(TestTmpFileBlock, test_block_manager_stat) +{ + int ret = OB_SUCCESS; + ObTmpFileBlockManager block_mgr; + static const int64_t META_DEFAULT_LIMIT = 15 * 1024L * 1024L * 1024L; + ret = block_mgr.init(MTL_ID(), META_DEFAULT_LIMIT); + ASSERT_EQ(OB_SUCCESS, ret); + + int64_t begin_page_id = 0; + int64_t page_num = ObTmpFileGlobal::BLOCK_PAGE_NUMS; + int64_t block_index1 = ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX; + int64_t block_index2 = ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX; + int64_t block_index3 = ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX; + int64_t block_index4 = ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX; + // 1. create tmp file blocks + ret = block_mgr.create_tmp_file_block(begin_page_id + 10, page_num - 20, block_index1); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(block_index1, ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX); + + ret = block_mgr.create_tmp_file_block(begin_page_id, page_num, block_index2); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(block_index2, ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX); + + ret = block_mgr.create_tmp_file_block(begin_page_id + 10, page_num - 10, block_index3); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(block_index3, ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX); + + ret = block_mgr.create_tmp_file_block(begin_page_id, page_num, block_index4); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(block_index4, ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX); + + // 2. alloc three macro block + ObMacroBlockHandle macro_block_handle1; + ObMacroBlockHandle macro_block_handle2; + ObMacroBlockHandle macro_block_handle3; + ret = OB_SERVER_BLOCK_MGR.alloc_block(macro_block_handle1); + ASSERT_EQ(OB_SUCCESS, ret); + ret = OB_SERVER_BLOCK_MGR.alloc_block(macro_block_handle2); + ASSERT_EQ(OB_SUCCESS, ret); + ret = OB_SERVER_BLOCK_MGR.alloc_block(macro_block_handle3); + ASSERT_EQ(OB_SUCCESS, ret); + MacroBlockId macro_block_id1 = macro_block_handle1.get_macro_id(); + ASSERT_EQ(true, macro_block_id1.is_valid()); + MacroBlockId macro_block_id2 = macro_block_handle2.get_macro_id(); + ASSERT_EQ(true, macro_block_id2.is_valid()); + MacroBlockId macro_block_id3 = macro_block_handle3.get_macro_id(); + ASSERT_EQ(true, macro_block_id3.is_valid()); + + // 3. switch state op + ret = block_mgr.write_back_start(block_index1); + ASSERT_EQ(OB_SUCCESS, ret); + ret = block_mgr.write_back_start(block_index2); + ASSERT_EQ(OB_SUCCESS, ret); + ret = block_mgr.write_back_start(block_index3); + ASSERT_EQ(OB_SUCCESS, ret); + ret = block_mgr.write_back_succ(block_index1, macro_block_id1); + ASSERT_EQ(OB_SUCCESS, ret); + ret = block_mgr.write_back_succ(block_index2, macro_block_id2); + ASSERT_EQ(OB_SUCCESS, ret); + ret = block_mgr.write_back_succ(block_index3, macro_block_id3); + ASSERT_EQ(OB_SUCCESS, ret); + + // 4. test statistic + int64_t macro_block_count = 0; + ret = block_mgr.get_macro_block_count(macro_block_count); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(3, macro_block_count); + + ObArray macro_id_list; + ret = macro_id_list.push_back(macro_block_id1); + ASSERT_EQ(OB_SUCCESS, ret); + ret = macro_id_list.push_back(macro_block_id2); + ASSERT_EQ(OB_SUCCESS, ret); + ret = macro_id_list.push_back(macro_block_id3); + ASSERT_EQ(OB_SUCCESS, ret); + std::sort(macro_id_list.begin(), macro_id_list.end()); + + ObArray macro_id_list_stat; + ret = block_mgr.get_macro_block_list(macro_id_list_stat); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(macro_id_list.count(), macro_id_list_stat.count()); + std::sort(macro_id_list_stat.begin(), macro_id_list_stat.end()); + for (int64_t i = 0; i < macro_id_list.count(); ++i) { + ASSERT_EQ(true, macro_id_list[i] == macro_id_list_stat[i]); + } + + int64_t used_page_num = 0; + macro_block_count = 0; + ret = block_mgr.get_block_usage_stat(used_page_num, macro_block_count); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(3, macro_block_count); + ASSERT_EQ(page_num * 3 - 30, used_page_num); + + LOG_INFO("test_block_manager_stat"); +} + +TEST_F(TestTmpFileBlock, test_block) +{ + int ret = OB_SUCCESS; + ObTmpFileBlock block; + int64_t block_index = 1; + int64_t begin_page_id = 0; + int64_t page_num = 0; + + begin_page_id = -1; + page_num = ObTmpFileGlobal::BLOCK_PAGE_NUMS; + ret = block.init_block(block_index, begin_page_id, page_num); + ASSERT_NE(OB_SUCCESS, ret); + + begin_page_id = 0; + page_num = ObTmpFileGlobal::BLOCK_PAGE_NUMS + 1; + ret = block.init_block(block_index, begin_page_id, page_num); + ASSERT_NE(OB_SUCCESS, ret); + begin_page_id = 0; + + begin_page_id = 0; + page_num = 0; + ret = block.init_block(block_index, begin_page_id, page_num); + ASSERT_NE(OB_SUCCESS, ret); + + begin_page_id = 10; + page_num = ObTmpFileGlobal::BLOCK_PAGE_NUMS; + ret = block.init_block(block_index, begin_page_id, page_num); + ASSERT_NE(OB_SUCCESS, ret); + + begin_page_id = 10; + page_num = ObTmpFileGlobal::BLOCK_PAGE_NUMS-20; + ret = block.init_block(block_index, begin_page_id, page_num); + ASSERT_EQ(OB_SUCCESS, ret); + + ObMacroBlockHandle macro_block_handle1; + ret = OB_SERVER_BLOCK_MGR.alloc_block(macro_block_handle1); + ASSERT_EQ(OB_SUCCESS, ret); + MacroBlockId macro_block_id1 = macro_block_handle1.get_macro_id(); + ASSERT_EQ(true, macro_block_id1.is_valid()); + + ret = block.write_back_failed(); + ASSERT_NE(OB_SUCCESS, ret); + ret = block.write_back_succ(macro_block_id1); + ASSERT_NE(OB_SUCCESS, ret); + + int64_t end_page_id = begin_page_id + page_num - 1; + ret = block.release_pages(0, 15); + ASSERT_NE(OB_SUCCESS, ret); + ret = block.release_pages(end_page_id - 30, 35); + ASSERT_NE(OB_SUCCESS, ret); + ret = block.release_pages(9, 1); + ASSERT_NE(OB_SUCCESS, ret); + ret = block.release_pages(end_page_id + 1, 1); + ASSERT_NE(OB_SUCCESS, ret); + ret = block.release_pages(end_page_id, 1); + ASSERT_EQ(OB_SUCCESS, ret); + + ASSERT_EQ(false, block.on_disk()); + int64_t used_page_num = 0; + ret = block.get_page_usage(used_page_num); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(page_num - 1, used_page_num); + bool can_remove = false; + ret = block.can_remove(can_remove); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(false, can_remove); + + ret = block.write_back_start(); + ASSERT_EQ(OB_SUCCESS, ret); + ret = block.write_back_succ(macro_block_id1); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(true, block.on_disk()); + ret = block.get_page_usage(used_page_num); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(page_num - 1, used_page_num); + ret = block.release_pages(10, used_page_num); + ASSERT_EQ(OB_SUCCESS, ret); + ret = block.can_remove(can_remove); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(true, can_remove); + + LOG_INFO("test_block"); +} + +TEST_F(TestTmpFileBlock, test_block_page_bit_map) +{ + int ret = OB_SUCCESS; + bool value; + ObTmpFileBlockPageBitmap bitmap; + // 1. get/set for one bit of bitmap + ret = bitmap.get_value(7, value); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(false, value); + ret = bitmap.get_value(16, value); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(false, value); + + ret = bitmap.set_bitmap(7, true); + ASSERT_EQ(OB_SUCCESS, ret); + ret = bitmap.set_bitmap(16, true); + ASSERT_EQ(OB_SUCCESS, ret); + + ret = bitmap.get_value(7, value); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(true, value); + ret = bitmap.get_value(16, value); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(true, value); + + ret = bitmap.get_value(bitmap.get_capacity(), value); + ASSERT_NE(OB_SUCCESS, ret); + ret = bitmap.get_value(-1, value); + ASSERT_NE(OB_SUCCESS, ret); + ret = bitmap.set_bitmap(bitmap.get_capacity(), true); + ASSERT_NE(OB_SUCCESS, ret); + ret = bitmap.set_bitmap(-1, true); + ASSERT_NE(OB_SUCCESS, ret); + + // 2. set batch bits of bitmap + int64_t start = 13; + int64_t end = 80; + ret = bitmap.get_value(start, value); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(false, value); + ret = bitmap.get_value(40, value); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(false, value); + ret = bitmap.get_value(end, value); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(false, value); + + ret = bitmap.set_bitmap_batch(start, end - start + 1, true); + ASSERT_EQ(OB_SUCCESS, ret); + for (int i = start; i <= end; i++) { + ret = bitmap.get_value(i, value); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(true, value); + } + ret = bitmap.get_value(start - 1, value); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(false, value); + ret = bitmap.get_value(end + 1, value); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(false, value); + + start = 5; + end = 40; + ret = bitmap.set_bitmap_batch(start , end - start + 1, true); + ASSERT_EQ(OB_SUCCESS, ret); + for (int i = start; i <= end; i++) { + ret = bitmap.get_value(i, value); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(true, value); + } + + start = 60; + end = 100; + ret = bitmap.set_bitmap_batch(start, end - start + 1, false); + ASSERT_EQ(OB_SUCCESS, ret); + for (int i = start; i <= end; i++) { + ret = bitmap.get_value(i, value); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(false, value); + } + + ret = bitmap.set_bitmap_batch(-1, 100, true); + ASSERT_NE(OB_SUCCESS, ret); + ret = bitmap.set_bitmap_batch(0, bitmap.get_capacity() + 1, true); + ASSERT_NE(OB_SUCCESS, ret); + // 3. test is_all_true() and is_all_false() + ret = bitmap.is_all_false(value); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(false, value); + ret = bitmap.is_all_true(value); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(false, value); + + bitmap.reset(); + + ret = bitmap.is_all_false(value); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(true, value); + ret = bitmap.is_all_true(value); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(false, value); + + start = 60; + end = 100; + ret = bitmap.set_bitmap_batch(start, end - start + 1 , true); + ASSERT_EQ(OB_SUCCESS, ret); + ret = bitmap.is_all_true(start, end, value); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(true, value); + ret = bitmap.is_all_true(start - 1, end, value); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(false, value); + ret = bitmap.is_all_true(start, end + 1, value); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(false, value); + + ret = bitmap.is_all_true(-1, end, value); + ASSERT_NE(OB_SUCCESS, ret); + ret = bitmap.is_all_true(100, bitmap.get_capacity(), value); + ASSERT_NE(OB_SUCCESS, ret); + + ret = bitmap.set_bitmap_batch(0, bitmap.get_capacity(), true); + ASSERT_EQ(OB_SUCCESS, ret); + ret = bitmap.is_all_true(value); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(true, value); + ret = bitmap.is_all_false(value); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(false, value); + + start = 60; + end = 100; + ret = bitmap.set_bitmap_batch(start, end - start + 1 , false); + ASSERT_EQ(OB_SUCCESS, ret); + ret = bitmap.is_all_false(start, end, value); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(true, value); + ret = bitmap.is_all_false(start - 1, end, value); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(false, value); + ret = bitmap.is_all_false(start, end + 1, value); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(false, value); + + ret = bitmap.is_all_false(-1, end, value); + ASSERT_NE(OB_SUCCESS, ret); + ret = bitmap.is_all_false(100, bitmap.get_capacity(), value); + ASSERT_NE(OB_SUCCESS, ret); + LOG_INFO("test_block_page_bit_map"); +} + +TEST_F(TestTmpFileBlock, test_block_page_bit_map_iter) +{ + int ret = OB_SUCCESS; + bool value; + ObTmpFileBlockPageBitmap bitmap; + int64_t start_page_id = 0; + int64_t end_page_id = bitmap.get_capacity() - 1; + ObTmpFileBlockPageBitmapIterator iter; + ret = iter.init(nullptr, start_page_id, end_page_id); + ASSERT_NE(OB_SUCCESS, ret); + ret = iter.init(&bitmap, -1, end_page_id); + ASSERT_NE(OB_SUCCESS, ret); + ret = iter.init(&bitmap, start_page_id, bitmap.get_capacity()); + ASSERT_NE(OB_SUCCESS, ret); + ret = iter.init(&bitmap, start_page_id, end_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + + int64_t starts[5] = {0, 10, 84, 163, bitmap.get_capacity() - 1}; + int64_t ends[5] = {0, 40, 101, 230, bitmap.get_capacity() - 1}; + ret = bitmap.set_bitmap(starts[0], true); + ASSERT_EQ(OB_SUCCESS, ret); + ret = bitmap.set_bitmap_batch(starts[1], ends[1] - starts[1] + 1, true); + ASSERT_EQ(OB_SUCCESS, ret); + ret = bitmap.set_bitmap_batch(starts[2], ends[2] - starts[2] + 1, true); + ASSERT_EQ(OB_SUCCESS, ret); + ret = bitmap.set_bitmap_batch(starts[3], ends[3] - starts[3] + 1, true); + ASSERT_EQ(OB_SUCCESS, ret); + ret = bitmap.set_bitmap(starts[4], true); + ASSERT_EQ(OB_SUCCESS, ret); + + ASSERT_EQ(true, iter.has_next()); + + int i = 0; + while (iter.has_next()) { + bool value = false; + ret = iter.next_range(value, start_page_id, end_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + if (value) { + ASSERT_EQ(start_page_id, starts[i]); + ASSERT_EQ(end_page_id, ends[i]); + i++; + } else { + ASSERT_NE(i, 0); + ASSERT_EQ(start_page_id, ends[i-1]+1); + ASSERT_EQ(end_page_id, starts[i]-1); + } + } + + iter.reset(); + ret = iter.init(&bitmap, 5, 180); + ASSERT_EQ(OB_SUCCESS, ret); + ends[0] = 4; + ends[3] = 180; + i = 1; + while (iter.has_next()) { + bool value = false; + ret = iter.next_range(value, start_page_id, end_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + if (value) { + ASSERT_EQ(start_page_id, starts[i]); + ASSERT_EQ(end_page_id, ends[i]); + i++; + } else { + ASSERT_EQ(start_page_id, ends[i-1]+1); + ASSERT_EQ(end_page_id, starts[i]-1); + } + } + LOG_INFO("test_block_page_bit_map_iter"); +} + +} // namespace oceanbase + +int main(int argc, char **argv) +{ + int ret = 0; + system("rm -f ./test_tmp_file_block_manager.log*"); + OB_LOGGER.set_file_name("test_tmp_file_block_manager.log", true); + OB_LOGGER.set_log_level("INFO"); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/mittest/mtlenv/storage/tmp_file/test_tmp_file_buffer_pool.cpp b/mittest/mtlenv/storage/tmp_file/test_tmp_file_buffer_pool.cpp new file mode 100644 index 000000000..98af9fac2 --- /dev/null +++ b/mittest/mtlenv/storage/tmp_file/test_tmp_file_buffer_pool.cpp @@ -0,0 +1,690 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE +#include +#include "mittest/mtlenv/mock_tenant_module_env.h" +#include "storage/tmp_file/ob_tmp_file_write_buffer_pool.h" +#include "lib/random/ob_random.h" + +namespace oceanbase +{ +using namespace common; +using namespace tmp_file; +using namespace storage; + +struct WBPTestHelper +{ +public: + WBPTestHelper(const int64_t fd, ObTmpWriteBufferPool &wbp) + : fd_(fd), + data_size_(0), + data_page_num_(0), + meta_page_num_(0), + data_page_ids_(), + begin_page_id_(ObTmpFileGlobal::INVALID_PAGE_ID), + begin_virtual_page_id_(-1), + end_page_id_(ObTmpFileGlobal::INVALID_PAGE_ID), + wbp_(wbp) {} + int alloc_data_pages(const int64_t num); + // int alloc_meta_pages(const int64_t num); + int free_all_pages(); +public: + int64_t fd_; + int64_t data_size_; + int64_t data_page_num_; + int64_t meta_page_num_; + ObArray data_page_ids_; + uint32_t begin_page_id_; + int64_t begin_virtual_page_id_; + uint32_t end_page_id_; + ObTmpWriteBufferPool &wbp_; +}; + +int WBPTestHelper::alloc_data_pages(const int64_t num) +{ + int ret = OB_SUCCESS; + uint32_t previous_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + int64_t previous_virtual_page_id_ = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + for (int64_t i = 0; OB_SUCC(ret) && i < num; ++i) { + uint32_t new_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + char *buf = nullptr; + int64_t new_virtual_page_id_ = data_size_ / ObTmpFileGlobal::PAGE_SIZE; + if (OB_FAIL(wbp_.alloc_page(fd_, ObTmpFilePageUniqKey(new_virtual_page_id_), new_page_id, buf))) { + LOG_WARN("fail to alloc page", K(fd_), K(previous_page_id)); + } else if (OB_FAIL(data_page_ids_.push_back(new_page_id))) { + LOG_WARN("fail to push back", K(ret)); + } else if (ObTmpFileGlobal::INVALID_PAGE_ID != previous_page_id && + OB_FAIL(wbp_.link_page(fd_, new_page_id, previous_page_id, ObTmpFilePageUniqKey(previous_virtual_page_id_)))) { + LOG_WARN("fail to link page", K(fd_), K(new_page_id), K(previous_page_id), K(previous_virtual_page_id_)); + } else { + previous_virtual_page_id_ = new_virtual_page_id_; + if (ObTmpFileGlobal::INVALID_PAGE_ID == begin_page_id_) { + begin_page_id_ = new_page_id; + begin_virtual_page_id_ = data_size_ / ObTmpFileGlobal::PAGE_SIZE; + } + data_size_ += ObTmpFileGlobal::PAGE_SIZE; + previous_page_id = new_page_id; + } + } + return ret; +} + +int WBPTestHelper::free_all_pages() +{ + int ret = OB_SUCCESS; + uint32_t free_page_id = begin_page_id_; + for (int64_t free_cnt = 0; OB_SUCC(ret) && free_cnt < data_page_num_ + && ObTmpFileGlobal::INVALID_PAGE_ID != free_page_id; ++free_cnt) { + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ret = wbp_.free_page(fd_, free_page_id, ObTmpFilePageUniqKey(begin_virtual_page_id_), next_page_id); + free_page_id = next_page_id; + } + return ret; +} + +struct WBPTestFunctor +{ +public: + WBPTestFunctor(const int64_t fd, const int64_t capacity, const int64_t loop, + ObTmpWriteBufferPool *wbp) + : fd_(fd), wbp_capacity_(capacity), loop_(loop), wbp_(wbp), data_(), + begin_data_page_virtual_id_(-1), end_data_page_virtual_id_(-1) {} + void operator() (); + bool check_wbp_data_success(); + bool check_no_page_belong_self(); + void print_deque(std::deque * dq); +public: + int64_t fd_; + int64_t wbp_capacity_; + int64_t loop_; + ObTmpWriteBufferPool * wbp_; + std::deque data_; + int64_t begin_data_page_virtual_id_; + int64_t end_data_page_virtual_id_; +}; + +/* + * Randomly allocate several pages (0 ~ `wbp_capacity_`), then randomly free + * several pages (0 ~ alloced_page_nums), repeat for `loop_` times, and at the + * end, all pages are automatically returned. + */ +void WBPTestFunctor::operator() () +{ + // alloc and free + for (int i = 0; i < loop_; ++i) { + int ret = OB_SUCCESS; + // random alloc pages + uint32_t new_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + char *new_page_buf = nullptr; + uint32_t prev_page_id = data_.size() == 0 + ? ObTmpFileGlobal::INVALID_PAGE_ID + : data_.back(); + int64_t alloc_page_nums = ObRandom::rand(0, wbp_capacity_); + for (int64_t j = 0; OB_SUCC(ret) && j < alloc_page_nums; ++j) { + int64_t new_page_begin_virtual_id = end_data_page_virtual_id_ < 0 ? 0 : end_data_page_virtual_id_ + 1; + ret = wbp_->alloc_page(fd_, ObTmpFilePageUniqKey(new_page_begin_virtual_id), new_page_id, new_page_buf); + ASSERT_EQ(OB_SUCCESS, ret); + if (prev_page_id != ObTmpFileGlobal::INVALID_PAGE_ID) { + ret = wbp_->link_page(fd_, new_page_id, prev_page_id, ObTmpFilePageUniqKey(end_data_page_virtual_id_)); + ASSERT_EQ(OB_SUCCESS, ret); + } else { + begin_data_page_virtual_id_ = new_page_begin_virtual_id; + } + data_.push_back(new_page_id); + LOG_INFO("alloc page succeed", K(fd_), K(new_page_id), K(prev_page_id)); + end_data_page_virtual_id_ = new_page_begin_virtual_id; + prev_page_id = new_page_id; + } + + if (!check_wbp_data_success()) { + std::cout << "check data fail after alloc, loop: " << i << std::endl; + break; + } + + int64_t free_page_nums = ObRandom::rand(0, data_.size()); + for (int64_t j = 0; OB_SUCC(ret) && j < free_page_nums && data_.size() > 0; ++j) { + uint32_t page_to_free = data_.front(); + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ret = wbp_->free_page(fd_, page_to_free, ObTmpFilePageUniqKey(begin_data_page_virtual_id_), next_page_id); + data_.pop_front(); + ASSERT_EQ(ret, OB_SUCCESS); + begin_data_page_virtual_id_ += 1; + } + + if (!check_wbp_data_success()) { + std::cout << "check data fail after free, loop: " << i << std::endl; + break; + } + } + // free all + { + int ret = OB_SUCCESS; + int64_t total_page_nums = data_.size(); + for (int64_t j = 0; OB_SUCC(ret) && j < total_page_nums; ++j) { + uint32_t page_to_free = data_.front(); + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ret = wbp_->free_page(fd_, page_to_free, ObTmpFilePageUniqKey(begin_data_page_virtual_id_), next_page_id); + ASSERT_EQ(ret, OB_SUCCESS); + data_.pop_front(); + begin_data_page_virtual_id_ += 1; + } + if (data_.size() != 0) { + std::cout << "free all pages error, data size: " << data_.size() << std::endl; + } + if (!check_no_page_belong_self()) { + std::cout << fd_ << " check no page belong self fail" << std::endl; + } + } +} + +bool WBPTestFunctor::check_wbp_data_success() +{ + bool check_res = true; + std::deque wbp_data; + // collect wbp data + int ret = OB_SUCCESS; + uint32_t curr_page_id = data_.size() > 0 ? data_.front() : ObTmpFileGlobal::INVALID_PAGE_ID; + int64_t curr_page_virtual_id = begin_data_page_virtual_id_; + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + char * page_buff = nullptr; + while (OB_SUCC(ret) && curr_page_id != ObTmpFileGlobal::INVALID_PAGE_ID) { + ret = wbp_->read_page(fd_, curr_page_id, ObTmpFilePageUniqKey(curr_page_virtual_id), page_buff, next_page_id); + if (OB_SUCC(ret)) { + wbp_data.push_back(curr_page_id); + curr_page_virtual_id += 1; + } else { + std::cout << "fetch page error, ret: " << ret << std::endl; + } + curr_page_id = next_page_id; + } + // compare data + if (data_.size() != wbp_data.size()) { + std::cout << "check wbp data fail, data size: " << data_.size() + << ", wbp data size: " << wbp_data.size() << std::endl; + check_res = false; + print_deque(&data_); + print_deque(&wbp_data); + } else { + for (int64_t i = 0; i < data_.size(); ++i) { + if (data_.at(i) != wbp_data.at(i)) { + std::cout << "check wbp data fail, not equal happen at: " << i + << ", data: " << data_.at(i) + << ", wbp data: " << wbp_data.at(i) << std::endl; + check_res = false; + print_deque(&data_); + print_deque(&wbp_data); + break; + } + } + } + return check_res; +} + +bool WBPTestFunctor::check_no_page_belong_self() +{ + bool no_page_belong_self = true; + int64_t page_belong_self_nums = 0; + for (int64_t i = 0; i < wbp_capacity_; ++i) { + if (wbp_->fat_[i].fd_ == fd_) { + std::cout << fd_ << " find self page, idx: " << i + << ", fd: " << wbp_->fat_[i].fd_ + << ", next_page_id: " << wbp_->fat_[i].next_page_id_ + << std::endl; + page_belong_self_nums++; + no_page_belong_self = false; + } + } + if (!no_page_belong_self) { + std::cout << fd_ << " occupy " << page_belong_self_nums << " pages" << std::endl; + } + return no_page_belong_self; +} + +void WBPTestFunctor::print_deque(std::deque * dq) +{ + ObArray data; + for (int64_t i = 0; i < dq->size(); ++i) { + data.push_back(dq->at(i)); + } + LOG_INFO("print_deque", K(fd_), K(data)); +} + +class TestBufferPool : public ::testing::Test +{ +public: + virtual void SetUp(); + virtual void TearDown(); + static void SetUpTestCase(); + static void TearDownTestCase(); +}; + +void TestBufferPool::SetUp() +{ + ObTmpFilePageCacheController &pc_ctrl = MTL(ObTenantTmpFileManager *)->get_page_cache_controller(); + ObTmpWriteBufferPool &wbp = pc_ctrl.get_write_buffer_pool(); + wbp.destroy(); + ASSERT_EQ(OB_SUCCESS, wbp.init()); +} + +void TestBufferPool::TearDown() +{ + MTL(ObTenantTmpFileManager *)->get_page_cache_controller().get_write_buffer_pool().destroy(); +} + +void TestBufferPool::SetUpTestCase() +{ + ASSERT_EQ(OB_SUCCESS, MockTenantModuleEnv::get_instance().init()); +} + +void TestBufferPool::TearDownTestCase() +{ + MockTenantModuleEnv::get_instance().destroy(); +} + +TEST_F(TestBufferPool, test_buffer_pool_basic) +{ + ObTmpFilePageCacheController &pc_ctrl = MTL(ObTenantTmpFileManager *)->get_page_cache_controller(); + ObTmpWriteBufferPool &wbp = pc_ctrl.get_write_buffer_pool(); + WBPTestFunctor wbp_test_functor(0, ObTmpWriteBufferPool::BLOCK_PAGE_NUMS, 100, &wbp); + wbp_test_functor(); +} + +TEST_F(TestBufferPool, test_buffer_pool_concurrent) +{ + ObTmpFilePageCacheController &pc_ctrl = MTL(ObTenantTmpFileManager *)->get_page_cache_controller(); + ObTmpWriteBufferPool &wbp = pc_ctrl.get_write_buffer_pool(); + const int64_t MAX_THREAD_NUM = 5; + const int64_t MAX_LOOP_NUM = 100; + std::vector t_vec; + for (int64_t i = 0; i < MAX_THREAD_NUM; ++i) { + WBPTestFunctor functor = WBPTestFunctor(10 + i, ObTmpWriteBufferPool::BLOCK_PAGE_NUMS, MAX_LOOP_NUM, &wbp); + t_vec.push_back(std::thread(functor)); + } + for (int64_t i = 0; i < t_vec.size(); ++i) { + t_vec[i].join(); + } +} + +TEST_F(TestBufferPool, test_entry_state_switch_write_back) +{ + ObTmpWriteBufferPool &wbp = MTL(ObTenantTmpFileManager *)->page_cache_controller_.get_write_buffer_pool(); + int ret = OB_SUCCESS; + int64_t fd = 0; + const int64_t ALLOC_PAGE_NUM = 200; + WBPTestHelper wbp_test(fd, wbp); + ret = wbp_test.alloc_data_pages(ALLOC_PAGE_NUM); + ASSERT_EQ(OB_SUCCESS, ret); + + // dirty + uint32_t cur_page_id = wbp_test.begin_page_id_; + int64_t cur_page_virtual_id = 0; + for (int64_t i = 0; i < ALLOC_PAGE_NUM; ++i) { + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ret = wbp.get_next_page_id(fd, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id), next_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = wbp.notify_dirty(fd, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id)); + ASSERT_EQ(OB_SUCCESS, ret); + cur_page_id = next_page_id; + cur_page_virtual_id += 1; + } + ASSERT_EQ(ALLOC_PAGE_NUM, wbp.dirty_page_num_); + + // write back + cur_page_id = wbp_test.begin_page_id_; + cur_page_virtual_id = 0; + for (int64_t i = 0; i < ALLOC_PAGE_NUM; ++i) { + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ret = wbp.get_next_page_id(fd, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id), next_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = wbp.notify_write_back(fd, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id)); + ASSERT_EQ(OB_SUCCESS, ret); + cur_page_id = next_page_id; + cur_page_virtual_id += 1; + } + ASSERT_EQ(0, wbp.dirty_page_num_); + + // write back fail, page entry return to dirty + cur_page_id = wbp_test.begin_page_id_; + cur_page_virtual_id = 0; + for (int64_t i = 0; i < ALLOC_PAGE_NUM; ++i) { + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ret = wbp.get_next_page_id(fd, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id), next_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = wbp.notify_write_back_fail(fd, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id)); + ASSERT_EQ(OB_SUCCESS, ret); + cur_page_id = next_page_id; + cur_page_virtual_id += 1; + } + printf("after write back fail\n"); + ASSERT_EQ(ALLOC_PAGE_NUM, wbp.dirty_page_num_); + + // write back again + cur_page_id = wbp_test.begin_page_id_; + cur_page_virtual_id = 0; + for (int64_t i = 0; i < ALLOC_PAGE_NUM; ++i) { + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ret = wbp.get_next_page_id(fd, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id), next_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = wbp.notify_write_back(fd, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id)); + ASSERT_EQ(OB_SUCCESS, ret); + cur_page_id = next_page_id; + cur_page_virtual_id += 1; + } + + // write back succ + cur_page_id = wbp_test.begin_page_id_; + cur_page_virtual_id = 0; + for (int64_t i = 0; i < ALLOC_PAGE_NUM; ++i) { + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ret = wbp.get_next_page_id(fd, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id), next_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = wbp.notify_write_back_succ(fd, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id)); + ASSERT_EQ(OB_SUCCESS, ret); + cur_page_id = next_page_id; + cur_page_virtual_id += 1; + } + ASSERT_EQ(0, wbp.dirty_page_num_); + + // write back succ re-entrant + cur_page_id = wbp_test.begin_page_id_; + cur_page_virtual_id = 0; + for (int64_t i = 0; i < ALLOC_PAGE_NUM; ++i) { + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ret = wbp.get_next_page_id(fd, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id), next_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = wbp.notify_write_back_succ(fd, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id)); + ASSERT_EQ(OB_SUCCESS, ret); + cur_page_id = next_page_id; + cur_page_virtual_id += 1; + } + ASSERT_EQ(0, wbp.dirty_page_num_); + + ret = wbp_test.free_all_pages(); + ASSERT_EQ(OB_SUCCESS, ret); +} + +TEST_F(TestBufferPool, test_entry_state_switch_loading) +{ + ObTmpWriteBufferPool &wbp = MTL(ObTenantTmpFileManager *)->page_cache_controller_.get_write_buffer_pool(); + int ret = OB_SUCCESS; + int64_t fd = 0; + const int64_t ALLOC_PAGE_NUM = 200; + WBPTestHelper wbp_test(fd, wbp); + ret = wbp_test.alloc_data_pages(ALLOC_PAGE_NUM); + ASSERT_EQ(OB_SUCCESS, ret); + + // load + uint32_t cur_page_id = wbp_test.begin_page_id_; + int64_t cur_page_virtual_id = 0; + for (int64_t i = 0; i < ALLOC_PAGE_NUM; ++i) { + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ret = wbp.get_next_page_id(fd, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id), next_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = wbp.notify_load(fd, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id)); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_TRUE(wbp.is_loading(fd, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id))); + cur_page_id = next_page_id; + cur_page_virtual_id += 1; + } + + // load fail + cur_page_id = wbp_test.begin_page_id_; + cur_page_virtual_id = 0; + for (int64_t i = 0; i < ALLOC_PAGE_NUM; ++i) { + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ret = wbp.get_next_page_id(fd, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id), next_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = wbp.notify_load_fail(fd, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id)); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_TRUE(wbp.is_exist(fd, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id))); + cur_page_id = next_page_id; + cur_page_virtual_id += 1; + } + + // load again + cur_page_id = wbp_test.begin_page_id_; + cur_page_virtual_id = 0; + for (int64_t i = 0; i < ALLOC_PAGE_NUM; ++i) { + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ret = wbp.get_next_page_id(fd, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id), next_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = wbp.notify_load(fd, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id)); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_TRUE(wbp.is_loading(fd, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id))); + cur_page_id = next_page_id; + cur_page_virtual_id += 1; + } + + // load succ + cur_page_id = wbp_test.begin_page_id_; + cur_page_virtual_id = 0; + for (int64_t i = 0; i < ALLOC_PAGE_NUM; ++i) { + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ret = wbp.get_next_page_id(fd, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id), next_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = wbp.notify_load_succ(fd, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id)); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_TRUE(wbp.is_cached(fd, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id))); + cur_page_id = next_page_id; + cur_page_virtual_id += 1; + } + + ret = wbp_test.free_all_pages(); + ASSERT_EQ(OB_SUCCESS, ret); +} + +TEST_F(TestBufferPool, test_alloc_page_limit) +{ + int ret = OB_SUCCESS; + ObTmpWriteBufferPool &wbp = MTL(ObTenantTmpFileManager *)->page_cache_controller_.get_write_buffer_pool(); + int64_t max_page_num = wbp.get_max_page_num(); + std::cout << "write buffer pool max page num " << max_page_num << std::endl; + LOG_INFO("write buffer pool max page num", K(max_page_num)); + int64_t fd = 0; + int64_t offset = 0; + uint32_t data_head_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + uint32_t cur_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + // 分配 50% 的 data page + const int64_t BATCH_ALLOC_DATA_PAGE_NUM = max_page_num / 2; + int64_t cur_page_virtual_id = 0; + for (int64_t i = 0; i < BATCH_ALLOC_DATA_PAGE_NUM; ++i) { + uint32_t new_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + char *buf = nullptr; + ret = wbp.alloc_page(fd, ObTmpFilePageUniqKey(cur_page_virtual_id), new_page_id, buf); // TODO: 替换成wbp_test + ASSERT_EQ(ret, OB_SUCCESS); + ASSERT_NE(buf, nullptr); + data_head_page_id = ObTmpFileGlobal::INVALID_PAGE_ID == data_head_page_id ? new_page_id : data_head_page_id; + if (ObTmpFileGlobal::INVALID_PAGE_ID != cur_page_id) { + ret = wbp.link_page(fd, new_page_id, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id - 1)); + ASSERT_EQ(ret, OB_SUCCESS); + } + cur_page_virtual_id += 1; + cur_page_id = new_page_id; + } + + // 再分配 50% 的 data page，超过 MAX_DATA_PAGE_USAGE_RATIO(default 0.9) 后 + // 会触发 OB_ALLOCATE_TMP_FILE_PAGE_FAILED，分配页面失败 + for (int64_t i = 0; OB_SUCC(ret) && i < BATCH_ALLOC_DATA_PAGE_NUM; ++i) { + uint32_t new_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + char *buf = nullptr; + ret = wbp.alloc_page(fd, ObTmpFilePageUniqKey(cur_page_virtual_id), new_page_id, buf); + if (ret == OB_ALLOCATE_TMP_FILE_PAGE_FAILED) { + break; + } + ASSERT_EQ(ret, OB_SUCCESS); + if (cur_page_id != ObTmpFileGlobal::INVALID_PAGE_ID) { + ret = wbp.link_page(fd, new_page_id, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id - 1)); + ASSERT_EQ(ret, OB_SUCCESS); + } + cur_page_id = new_page_id; + cur_page_virtual_id += 1; + } + ASSERT_EQ(ret, OB_ALLOCATE_TMP_FILE_PAGE_FAILED); + + // 此时仍可分配少量 meta page(buffer pool最小为2MB，为meta page预留空间最少为25页) + cur_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + int64_t meta_page_num = 0; + for (int64_t i = 0; i < std::max(max_page_num * 0.01, 20.0); ++i) { + uint32_t new_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + char *buf = nullptr; + ObTmpFilePageUniqKey meta_page_offset(1, meta_page_num); + ret = wbp.alloc_page(fd, meta_page_offset, new_page_id, buf); + ASSERT_EQ(ret, OB_SUCCESS); + ASSERT_NE(buf, nullptr); + if (ObTmpFileGlobal::INVALID_PAGE_ID != cur_page_id) { + ret = wbp.link_page(fd, new_page_id, cur_page_id, ObTmpFilePageUniqKey(1, meta_page_num-1)); + ASSERT_EQ(ret, OB_SUCCESS); + } + cur_page_id = new_page_id; + meta_page_num += 1; + } + + // 分配 meta page 到buffer pool上限 + cur_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + for (int64_t i = 0; OB_SUCC(ret) && i < max_page_num; ++i) { + uint32_t new_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + char *buf = nullptr; + ObTmpFilePageUniqKey meta_page_offset(1, meta_page_num); + ret = wbp.alloc_page(fd, meta_page_offset, new_page_id, buf); + if (ret == OB_ALLOCATE_TMP_FILE_PAGE_FAILED) { + break; + } + ASSERT_EQ(ret, OB_SUCCESS); + if (cur_page_id != ObTmpFileGlobal::INVALID_PAGE_ID) { + ret = wbp.link_page(fd, new_page_id, cur_page_id, ObTmpFilePageUniqKey(1, meta_page_num -1)); + ASSERT_EQ(ret, OB_SUCCESS); + } + cur_page_id = new_page_id; + meta_page_num += 1; + } + ASSERT_EQ(ret, OB_ALLOCATE_TMP_FILE_PAGE_FAILED); + ASSERT_EQ(wbp.data_page_cnt_ + wbp.meta_page_cnt_, wbp.used_page_num_); + + // data page释放后，可以继续分配meta page + int64_t cur_meta_page_num = wbp.meta_page_cnt_; + int64_t free_page_id = data_head_page_id; + const int64_t FREE_DATA_PAGE_NUM = max_page_num / 2; + int64_t free_cnt = 0; + cur_page_virtual_id = 0; + for (; free_cnt < FREE_DATA_PAGE_NUM && ObTmpFileGlobal::INVALID_PAGE_ID != free_page_id; ++free_cnt) { + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ret = wbp.free_page(fd, free_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id), next_page_id); + free_page_id = next_page_id; + ASSERT_EQ(ret, OB_SUCCESS); + cur_page_virtual_id += 1; + } + ASSERT_EQ(free_cnt, FREE_DATA_PAGE_NUM); + int64_t alloc_cnt = 0; + for (; OB_SUCC(ret) && alloc_cnt < FREE_DATA_PAGE_NUM; ++alloc_cnt) { + uint32_t new_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + char *buf = nullptr; + ObTmpFilePageUniqKey meta_page_offset(1, meta_page_num); + ret = wbp.alloc_page(fd, meta_page_offset, new_page_id, buf); + ASSERT_EQ(ret, OB_SUCCESS); + if (cur_page_id != ObTmpFileGlobal::INVALID_PAGE_ID) { + ret = wbp.link_page(fd, new_page_id, cur_page_id, ObTmpFilePageUniqKey(1, meta_page_num - 1)); + ASSERT_EQ(ret, OB_SUCCESS); + } + cur_page_id = new_page_id; + meta_page_num += 1; + } + printf("total page num: %ld, data page: %ld, meta page: %ld, capacity: %ld\n", wbp.used_page_num_, wbp.meta_page_cnt_, wbp.data_page_cnt_, wbp.capacity_); + ASSERT_EQ(ret, OB_SUCCESS); + ASSERT_EQ(wbp.meta_page_cnt_, FREE_DATA_PAGE_NUM + cur_meta_page_num); + ASSERT_EQ(wbp.meta_page_cnt_ + wbp.data_page_cnt_, wbp.used_page_num_); +} + +TEST_F(TestBufferPool, test_get_page_id_by_offset) +{ + ObTmpWriteBufferPool &wbp = MTL(ObTenantTmpFileManager *)->page_cache_controller_.get_write_buffer_pool(); + int ret = OB_SUCCESS; + int64_t fd = 0; + const int64_t ALLOC_PAGE_NUM = 400; + WBPTestHelper wbp_test(fd, wbp); + ret = wbp_test.alloc_data_pages(ALLOC_PAGE_NUM); + ASSERT_EQ(OB_SUCCESS, ret); + + uint32_t page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ret = wbp.get_page_id_by_virtual_id(fd, 0, wbp_test.begin_page_id_, page_id); + ASSERT_EQ(ret, OB_SUCCESS); + ASSERT_NE(page_id, ObTmpFileGlobal::INVALID_PAGE_ID); + ASSERT_EQ(page_id, wbp_test.data_page_ids_.at(0)); + + page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ret = wbp.get_page_id_by_virtual_id(fd, 1, wbp_test.begin_page_id_, page_id); + ASSERT_EQ(ret, OB_SUCCESS); + ASSERT_NE(page_id, ObTmpFileGlobal::INVALID_PAGE_ID); + ASSERT_EQ(page_id, wbp_test.data_page_ids_.at(1)); + + page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ret = wbp.get_page_id_by_virtual_id(fd, ALLOC_PAGE_NUM - 1, wbp_test.begin_page_id_, page_id); + ASSERT_EQ(ret, OB_SUCCESS); + ASSERT_NE(page_id, ObTmpFileGlobal::INVALID_PAGE_ID); + ASSERT_EQ(page_id, wbp_test.data_page_ids_.at(ALLOC_PAGE_NUM - 1)); + + // offset out of bound, return INVALID_PAGE_ID + page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ret = wbp.get_page_id_by_virtual_id(fd, ALLOC_PAGE_NUM, wbp_test.begin_page_id_, page_id); + ASSERT_EQ(page_id, ObTmpFileGlobal::INVALID_PAGE_ID); + + ret = wbp_test.free_all_pages(); + ASSERT_EQ(OB_SUCCESS, ret); +} + +TEST_F(TestBufferPool, test_truncate_page) +{ + ObTmpWriteBufferPool &wbp = MTL(ObTenantTmpFileManager *)->page_cache_controller_.get_write_buffer_pool(); + int ret = OB_SUCCESS; + int64_t fd = 0; + const int64_t ALLOC_PAGE_NUM = 200; + WBPTestHelper wbp_test(fd, wbp); + ret = wbp_test.alloc_data_pages(ALLOC_PAGE_NUM); + ASSERT_EQ(OB_SUCCESS, ret); + + ret = wbp.truncate_page(fd, wbp_test.begin_page_id_, ObTmpFilePageUniqKey(0), -1); + ASSERT_EQ(ret, OB_INVALID_ARGUMENT); + + ret = wbp.truncate_page(fd, wbp_test.begin_page_id_, ObTmpFilePageUniqKey(0), 0); + ASSERT_EQ(ret, OB_INVALID_ARGUMENT); + + ret = wbp.truncate_page(2, wbp_test.begin_page_id_, ObTmpFilePageUniqKey(0), 4096); + ASSERT_EQ(ret, OB_STATE_NOT_MATCH); + + const int64_t truncate_size = 4096; + ret = wbp.truncate_page(fd, wbp_test.begin_page_id_, ObTmpFilePageUniqKey(0), truncate_size); + ASSERT_EQ(ret, OB_SUCCESS); + + char null_buf[truncate_size]; + memset(null_buf, 0, sizeof(null_buf)); + char *page_buf = nullptr; + uint32_t unused_next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ret = wbp.read_page(fd, wbp_test.data_page_ids_.at(0), ObTmpFilePageUniqKey(0), page_buf, unused_next_page_id); + ASSERT_EQ(ret, OB_SUCCESS); + int cmp = memcmp(null_buf, page_buf, truncate_size); + ASSERT_EQ(cmp, 0); + + ret = wbp_test.free_all_pages(); + ASSERT_EQ(OB_SUCCESS, ret); +} + +} // namespace oceanbase + +int main(int argc, char **argv) +{ + int ret = 0; + system("rm -f ./test_tmp_file_buffer_pool.log*"); + OB_LOGGER.set_file_name("test_tmp_file_buffer_pool.log", true); + OB_LOGGER.set_log_level("INFO"); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/mittest/mtlenv/storage/tmp_file/test_tmp_file_flush_list.cpp b/mittest/mtlenv/storage/tmp_file/test_tmp_file_flush_list.cpp new file mode 100644 index 000000000..f0f2371ca --- /dev/null +++ b/mittest/mtlenv/storage/tmp_file/test_tmp_file_flush_list.cpp @@ -0,0 +1,697 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE +#include +#include +#include "mittest/mtlenv/mock_tenant_module_env.h" +#include "lib/random/ob_random.h" +#include "storage/tmp_file/ob_tmp_file_global.h" +#include "storage/tmp_file/ob_tmp_file_flush_priority_manager.h" + +namespace oceanbase +{ +using namespace common; +using namespace blocksstable; +using namespace tmp_file; +using namespace storage; +using namespace std; + +typedef ObTmpFileGlobal::FlushCtxState FlushCtxState; + +struct TestDirtyPageRecord +{ + TestDirtyPageRecord() : dir(0), dirty_data_size_(0), non_rightmost_meta_page_num_(0), rightmost_meta_page_num_(0) {} + TestDirtyPageRecord(const int64_t dir, + const int64_t dirty_data_size, + const int64_t non_rightmost_meta_page_num, + const int64_t rightmost_meta_page_num) + : dir(dir), + dirty_data_size_(dirty_data_size), + non_rightmost_meta_page_num_(non_rightmost_meta_page_num), + rightmost_meta_page_num_(rightmost_meta_page_num) {} + int64_t dir; + int64_t dirty_data_size_; + int64_t non_rightmost_meta_page_num_; + int64_t rightmost_meta_page_num_; +}; + +class TestFlushListIterator : public ::testing::Test +{ +public: + TestFlushListIterator() {} + void insert_file_sequence(); + void create_files(const FlushCtxState state, + const int64_t file_num, + ObTmpFileFlushPriorityManager &flush_prio_mgr, + vector &file_handles); + void create_files_with_dir(const FlushCtxState state, + const int64_t dir, + const int64_t file_num, + ObTmpFileFlushPriorityManager &flush_prio_mgr, + vector &file_handles); + void clear_all_files(); +protected: + virtual void TearDown() + { + clear_all_files(); + } + + static void SetUpTestCase() + { + ASSERT_EQ(OB_SUCCESS, MockTenantModuleEnv::get_instance().init()); + } + + static void TearDownTestCase() + { + MockTenantModuleEnv::get_instance().destroy(); + } +private: + char *random_buf_; + const int64_t write_size_ = 4 * 1024 * 1024; // 4MB + unordered_map mock_dirty_record_; // 记录模拟生成的脏页数据代替实际写入文件 +}; + +void get_file_range_by_state(const FlushCtxState state, int64_t &low, int64_t &high) +{ + switch(state) { + case FlushCtxState::FSM_F1: // [2MB, 4MB] + low = 2 * 1024 * 1024; + high = 4 * 1024 * 1024; + break; + case FlushCtxState::FSM_F2: // [1MB, 2MB) + low = 1 * 1024 * 1024; + high = 2 * 1024 * 1024 - 1; + break; + case FlushCtxState::FSM_F3: // [128KB, 1MB) + low = 128 * 1024; + high = 1 * 1024 * 1024 - 1; + break; + case FlushCtxState::FSM_F4: // [8KB, 128KB) + low = 8 * 1024; + high = 128 * 1024 - 1; + break; + case FlushCtxState::FSM_F5: // (0KB, 8KB) + low = 1; + high = 8 * 1024 - 1; + break; + default: + low = 0; + high = 0; + break; + } +} + +void TestFlushListIterator::clear_all_files() +{ + int ret = OB_SUCCESS; +// ObTmpFileFlushPriorityManager &flush_prio_mgr = +// MTL(ObTenantTmpFileManager *)->get_page_cache_controller().get_flush_priority_mgr(); +// for (auto p : mock_dirty_record_) { +// int64_t fd = p.first; +// ObTmpFileHandle file_handle; +// ret = MTL(ObTenantTmpFileManager *)->get_tmp_file(fd, file_handle); +// ASSERT_EQ(OB_SUCCESS, ret); +// ASSERT_NE(nullptr, file_handle.get()); +// ret = flush_prio_mgr.remove_file(*file_handle.get()); +// ASSERT_EQ(OB_SUCCESS, ret); +// } + mock_dirty_record_.clear(); +} + +void TestFlushListIterator::insert_file_sequence() +{ +} + +void TestFlushListIterator::create_files(const FlushCtxState state, const int64_t file_num, + ObTmpFileFlushPriorityManager &flush_prio_mgr, + vector &file_handles) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0 ; i < file_num; ++i) { + int64_t dir = -1; + ret = MTL(ObTenantTmpFileManager *)->alloc_dir(dir); + ASSERT_EQ(OB_SUCCESS, ret); + + int64_t fd = i; + + void *buf = nullptr; + ObSharedNothingTmpFile *tmp_file = nullptr; + buf = MTL(ObTenantTmpFileManager *)->tmp_file_allocator_.alloc(sizeof(ObSharedNothingTmpFile)); + ASSERT_NE(nullptr, buf); + + tmp_file = new (buf) ObSharedNothingTmpFile(); + ret = tmp_file->init(MTL_ID(), fd, dir, + &MTL(ObTenantTmpFileManager *)->tmp_file_block_manager_, + &MTL(ObTenantTmpFileManager *)->callback_allocator_, + &MTL(ObTenantTmpFileManager *)->wbp_index_cache_allocator_, + &MTL(ObTenantTmpFileManager *)->wbp_index_cache_bucket_allocator_, + &MTL(ObTenantTmpFileManager *)->page_cache_controller_); + ASSERT_EQ(OB_SUCCESS, ret); + tmp_file->flush_prio_mgr_ = &flush_prio_mgr; + + // ret = MTL(ObTenantTmpFileManager *)->open(fd, dir); + // ASSERT_EQ(OB_SUCCESS, ret); + + int64_t low = 0, high = 0; + get_file_range_by_state(state, low, high); + + TestDirtyPageRecord dirty_record; + int64_t mock_dirty_data_size = ObRandom::rand(low, high); + + dirty_record.dirty_data_size_ = mock_dirty_data_size; + dirty_record.non_rightmost_meta_page_num_ = 0; + dirty_record.rightmost_meta_page_num_ = 0; + mock_dirty_record_[fd] = dirty_record; + + ObTmpFileHandle file_handle; + file_handle.init(tmp_file); + ObSharedNothingTmpFile &file = *file_handle.get(); + file.file_size_ = mock_dirty_data_size; + file.cached_page_nums_ = + upper_align(mock_dirty_data_size, ObTmpFileGlobal::PAGE_SIZE) / ObTmpFileGlobal::PAGE_SIZE; + file_handles.push_back(file_handle); + } + ASSERT_EQ(file_num, file_handles.size()); +} + +void TestFlushListIterator::create_files_with_dir( + const FlushCtxState state, + const int64_t dir, + const int64_t file_num, + ObTmpFileFlushPriorityManager &flush_prio_mgr, + vector &file_handles) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0 ; i < file_num; ++i) { + int64_t fd = i; + + void *buf = nullptr; + ObSharedNothingTmpFile *tmp_file = nullptr; + buf = MTL(ObTenantTmpFileManager *)->tmp_file_allocator_.alloc(sizeof(ObSharedNothingTmpFile)); + ASSERT_NE(nullptr, buf); + + tmp_file = new (buf) ObSharedNothingTmpFile(); + ret = tmp_file->init(MTL_ID(), fd, dir, + &MTL(ObTenantTmpFileManager *)->tmp_file_block_manager_, + &MTL(ObTenantTmpFileManager *)->callback_allocator_, + &MTL(ObTenantTmpFileManager *)->wbp_index_cache_allocator_, + &MTL(ObTenantTmpFileManager *)->wbp_index_cache_bucket_allocator_, + &MTL(ObTenantTmpFileManager *)->page_cache_controller_); + ASSERT_EQ(OB_SUCCESS, ret); + tmp_file->flush_prio_mgr_ = &flush_prio_mgr; + + int64_t low = 0, high = 0; + get_file_range_by_state(state, low, high); + + TestDirtyPageRecord dirty_record; + int64_t mock_dirty_data_size = ObRandom::rand(low, high); + + dirty_record.dirty_data_size_ = mock_dirty_data_size; + dirty_record.non_rightmost_meta_page_num_ = 0; + dirty_record.rightmost_meta_page_num_ = 0; + mock_dirty_record_[fd] = dirty_record; + + ObTmpFileHandle file_handle; + file_handle.init(tmp_file); + ObSharedNothingTmpFile &file = *file_handle.get(); + file.file_size_ = mock_dirty_data_size; + file.cached_page_nums_ = + upper_align(mock_dirty_data_size, ObTmpFileGlobal::PAGE_SIZE) / ObTmpFileGlobal::PAGE_SIZE; + file_handles.push_back(file_handle); + } + ASSERT_EQ(file_num, file_handles.size()); +} + +TEST_F(TestFlushListIterator, test_iter_order) +{ + int ret = OB_SUCCESS; + ObTmpFileFlushPriorityManager flush_prio_mgr; + ret = flush_prio_mgr.init(); + ASSERT_EQ(OB_SUCCESS, ret); + vector array; + + // dir 1 has 10KB * 4 + array.push_back(TestDirtyPageRecord(1/*dir*/, 10240/*dirty_data_size*/, 0, 0)); + array.push_back(TestDirtyPageRecord(1/*dir*/, 10240/*dirty_data_size*/, 0, 0)); + array.push_back(TestDirtyPageRecord(1/*dir*/, 10240/*dirty_data_size*/, 0, 0)); + array.push_back(TestDirtyPageRecord(1/*dir*/, 10240/*dirty_data_size*/, 0, 0)); + + // dir 2 has (24KB * 4 + 2MB * 4) + array.push_back(TestDirtyPageRecord(2/*dir*/, 24576/*dirty_data_size*/, 0, 0)); + array.push_back(TestDirtyPageRecord(2/*dir*/, 24576/*dirty_data_size*/, 0, 0)); + array.push_back(TestDirtyPageRecord(2/*dir*/, 24576/*dirty_data_size*/, 0, 0)); + array.push_back(TestDirtyPageRecord(2/*dir*/, 24576/*dirty_data_size*/, 0, 0)); + array.push_back(TestDirtyPageRecord(2/*dir*/, 2097152/*dirty_data_size*/, 0, 0)); + array.push_back(TestDirtyPageRecord(2/*dir*/, 2097152/*dirty_data_size*/, 0, 0)); + array.push_back(TestDirtyPageRecord(2/*dir*/, 2097152/*dirty_data_size*/, 0, 0)); + array.push_back(TestDirtyPageRecord(2/*dir*/, 2097152/*dirty_data_size*/, 0, 0)); + + // iterate order: 2MB * 4, 10KB * 4, 4KB * 4 + for (auto &mock_record: array) { + int64_t fd = -1; + int64_t dir = mock_record.dir; + ret = MTL(ObTenantTmpFileManager *)->open(fd, dir); + ASSERT_EQ(OB_SUCCESS, ret); + + mock_dirty_record_[fd] = mock_record; + + ObTmpFileHandle file_handle; + ret = MTL(ObTenantTmpFileManager *)->get_tmp_file(fd, file_handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(file_handle.get(), nullptr); + ObSharedNothingTmpFile &file = *file_handle.get(); + file.file_size_ = mock_record.dirty_data_size_; + file.cached_page_nums_ = upper_align(file.file_size_, ObTmpFileGlobal::PAGE_SIZE) / ObTmpFileGlobal::PAGE_SIZE; + ret = flush_prio_mgr.insert_data_flush_list(file, mock_record.dirty_data_size_); + ASSERT_EQ(OB_SUCCESS, ret); + } + + ObTmpFileFlushListIterator iter; + ret = iter.init(&flush_prio_mgr); + ASSERT_EQ(OB_SUCCESS, ret); + + printf("iterating file...\n"); + bool unused = false; + for (int64_t i = 0; i < 4; i++) { + ObTmpFileHandle file_handle; + ret = iter.next(FlushCtxState::FSM_F1, unused, file_handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(file_handle.get(), nullptr); + ObSharedNothingTmpFile &file = *file_handle.get(); + TestDirtyPageRecord &mock_record = mock_dirty_record_[file.fd_]; + ASSERT_EQ(mock_record.dirty_data_size_, file.file_size_); + printf("fd: %ld, file_size: %ld\n", file.fd_, file.file_size_); + } + + for (int64_t i = 0; i < 4; i++) { + ObTmpFileHandle file_handle; + ret = iter.next(FlushCtxState::FSM_F2, unused, file_handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(file_handle.get(), nullptr); + ObSharedNothingTmpFile &file = *file_handle.get(); + TestDirtyPageRecord &mock_record = mock_dirty_record_[file.fd_]; + ASSERT_EQ(mock_record.dirty_data_size_, file.file_size_); + ASSERT_EQ(24576, file.file_size_); + printf("fd: %ld, file_size: %ld\n", file.fd_, file.file_size_); + } + + for (int64_t i = 0; i < 4; i++) { + ObTmpFileHandle file_handle; + ret = iter.next(FlushCtxState::FSM_F2, unused, file_handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(file_handle.get(), nullptr); + ObSharedNothingTmpFile &file = *file_handle.get(); + TestDirtyPageRecord &mock_record = mock_dirty_record_[file.fd_]; + ASSERT_EQ(mock_record.dirty_data_size_, file.file_size_); + ASSERT_EQ(10240, file.file_size_); + printf("fd: %ld, file_size: %ld\n", file.fd_, file.file_size_); + } + + ObTmpFileHandle file_handle; + ret = iter.next(FlushCtxState::FSM_F3, unused, file_handle); + ASSERT_EQ(OB_ITER_END, ret); +} + +TEST_F(TestFlushListIterator, test_iter_data_basic) +{ + int ret = OB_SUCCESS; + + ObTmpFileFlushPriorityManager flush_prio_mgr; + ret = flush_prio_mgr.init(); + ASSERT_EQ(OB_SUCCESS, ret); + vector file_handles; + int total_file_num = 0; + const int64_t FILE_NUM = 10; + + // 创建文件，根据层级生成模拟的脏页数量 + int64_t total_file_cnt = 0; + for (int64_t t = FlushCtxState::FSM_F1; t < FlushCtxState::FSM_FINISHED; ++t) { + file_handles.clear(); + create_files(FlushCtxState(t), FILE_NUM, flush_prio_mgr, file_handles); + for (int64_t i = 0; i < file_handles.size(); ++i) { + ObTmpFileHandle file_handle = file_handles.at(i); + ASSERT_NE(file_handle.get(), nullptr); + ObSharedNothingTmpFile &tmp_file = *file_handle.get(); + TestDirtyPageRecord &dirty_record = mock_dirty_record_.at(tmp_file.get_fd()); + ret = flush_prio_mgr.insert_data_flush_list(tmp_file, dirty_record.dirty_data_size_); + ASSERT_EQ(OB_SUCCESS, ret); + total_file_cnt += 1; + } + } + + ObTmpFileFlushListIterator iter; + ret = iter.init(&flush_prio_mgr); + ASSERT_EQ(OB_SUCCESS, ret); + + // 遍历所有刷盘层级是否能取出该层级所有文件 + int64_t iter_file_cnt = 0; + for (int64_t t = FlushCtxState::FSM_F1; t < FlushCtxState::FSM_FINISHED; ++t) { + for (int64_t i = 0; OB_SUCC(ret) && i < FILE_NUM * 5; ++i) { + bool is_meta; // unused + ObTmpFileHandle file_handle; + ret = iter.next(FlushCtxState(t), is_meta, file_handle); + if (OB_SUCC(ret)) { + ASSERT_NE(file_handle.get(), nullptr); + iter_file_cnt += 1; + } + } + ASSERT_EQ(ret, OB_ITER_END); + ret = OB_SUCCESS; + } + + ASSERT_EQ(iter_file_cnt, total_file_cnt); +} + +TEST_F(TestFlushListIterator, test_iter_prev_stage) +{ + int ret = OB_SUCCESS; + ObTmpFileFlushPriorityManager flush_prio_mgr; + ret = flush_prio_mgr.init(); + ASSERT_EQ(OB_SUCCESS, ret); + vector file_handles; + int total_file_num = 0; + const int64_t FILE_NUM = 10; + + // 创建文件，根据层级生成模拟的脏页数量 + int64_t total_file_cnt = 0; + for (int64_t t = FlushCtxState::FSM_F1; t < FlushCtxState::FSM_FINISHED; ++t) { + file_handles.clear(); + create_files(FlushCtxState(t), FILE_NUM, flush_prio_mgr, file_handles); + for (int64_t i = 0; i < file_handles.size(); ++i) { + ObTmpFileHandle file_handle = file_handles.at(i); + ASSERT_NE(file_handle.get(), nullptr); + ObSharedNothingTmpFile &tmp_file = *file_handle.get(); + TestDirtyPageRecord &dirty_record = mock_dirty_record_.at(tmp_file.get_fd()); + ret = flush_prio_mgr.insert_data_flush_list(tmp_file, dirty_record.dirty_data_size_); + ASSERT_EQ(OB_SUCCESS, ret); + + total_file_cnt += 1; + } + } + + ObTmpFileFlushListIterator iter; + ret = iter.init(&flush_prio_mgr); + ASSERT_EQ(OB_SUCCESS, ret); + + // 直接开始遍历F3直至OB_ITER_END + bool is_meta; // unused + for (int64_t i = 0; OB_SUCC(ret) && i < FILE_NUM * 10; ++i) { + ObTmpFileHandle file_handle; + ret = iter.next(FlushCtxState::FSM_F3, is_meta, file_handle); + if (OB_SUCC(ret)) { + ASSERT_NE(file_handle.get(), nullptr); + ObSharedNothingTmpFile &tmp_file = *file_handle.get(); + } + } + ASSERT_EQ(OB_ITER_END, ret); + + // 切换到下一层级后，无法再遍历之前的层级 + ObTmpFileHandle file_handle; + ret = iter.next(FlushCtxState::FSM_F1, is_meta, file_handle); + ASSERT_EQ(OB_ERR_UNEXPECTED, ret); +} + +// 某个文件从iterator返回、使用之后重新插回，如果迭代层级没有变更应该能被iterator重新取出 +TEST_F(TestFlushListIterator, test_iter_reinsert_file) +{ + int ret = OB_SUCCESS; + ObTmpFileFlushPriorityManager flush_prio_mgr; + ret = flush_prio_mgr.init(); + ASSERT_EQ(OB_SUCCESS, ret); + vector file_handles; + int total_file_num = 0; + const int64_t FILE_NUM = 10; + + // 创建文件，根据层级生成模拟的脏页数量 + int64_t total_file_cnt = 0; + create_files(FlushCtxState::FSM_F1, FILE_NUM, flush_prio_mgr, file_handles); + for (int64_t i = 0; i < file_handles.size(); ++i) { + ObTmpFileHandle file_handle = file_handles.at(i); + ASSERT_NE(file_handle.get(), nullptr); + ObSharedNothingTmpFile &tmp_file = *file_handle.get(); + TestDirtyPageRecord &dirty_record = mock_dirty_record_.at(tmp_file.get_fd()); + ret = flush_prio_mgr.insert_data_flush_list(tmp_file, dirty_record.dirty_data_size_); + ASSERT_EQ(OB_SUCCESS, ret); + total_file_cnt += 1; + } + + ObTmpFileFlushListIterator iter; + ret = iter.init(&flush_prio_mgr); + ASSERT_EQ(OB_SUCCESS, ret); + + // 通过iterator取出所有文件 + bool is_meta; // unused + int64_t iter_file_cnt = 0; + for (int64_t t = FlushCtxState::FSM_F1; t < FlushCtxState::FSM_FINISHED; ++t) { + for (int64_t i = 0; OB_SUCC(ret) && i < FILE_NUM * 5; ++i) { + ObTmpFileHandle file_handle; + ret = iter.next(FlushCtxState(t), is_meta, file_handle); + if (OB_SUCC(ret)) { + ASSERT_NE(file_handle.get(), nullptr); + iter_file_cnt += 1; + } + } + ASSERT_EQ(ret, OB_ITER_END); + } + ASSERT_EQ(iter_file_cnt, total_file_cnt); + ret = OB_SUCCESS; + + ObTmpFileHandle file_handle = file_handles[0]; + ASSERT_NE(file_handle.get(), nullptr); + ret = flush_prio_mgr.insert_data_flush_list(*file_handle.get(), 2 * 1024 * 1024); + ASSERT_EQ(ret, OB_SUCCESS); + + file_handle.reset(); + ret = iter.next(FlushCtxState::FSM_F1, is_meta, file_handle); + ASSERT_EQ(OB_SUCCESS, ret); + + ret = iter.next(FlushCtxState::FSM_F1, is_meta, file_handle); + ASSERT_EQ(OB_ITER_END, ret); +} + +TEST_F(TestFlushListIterator, test_flush_list_remove) +{ + int ret = OB_SUCCESS; + ObTmpFileFlushPriorityManager flush_prio_mgr; + ret = flush_prio_mgr.init(); + ASSERT_EQ(OB_SUCCESS, ret); + vector file_handles; + const int64_t FILE_NUM = 10; + + // 创建文件，根据层级生成模拟的脏页数量 + int64_t total_file_cnt = 0; + for (int64_t t = FlushCtxState::FSM_F1; t < FlushCtxState::FSM_FINISHED; ++t) { + vector tmp_file_handles; + create_files(FlushCtxState(t), FILE_NUM, flush_prio_mgr, tmp_file_handles); + for (int64_t i = 0; i < tmp_file_handles.size(); ++i) { + ObTmpFileHandle file_handle = tmp_file_handles.at(i); + ASSERT_NE(file_handle.get(), nullptr); + ObSharedNothingTmpFile &tmp_file = *file_handle.get(); + TestDirtyPageRecord &dirty_record = mock_dirty_record_.at(tmp_file.get_fd()); + ret = flush_prio_mgr.insert_data_flush_list(tmp_file, dirty_record.dirty_data_size_); + ASSERT_EQ(OB_SUCCESS, ret); + total_file_cnt += 1; + } + file_handles.insert(file_handles.end(), tmp_file_handles.begin(), tmp_file_handles.end()); + } + + // 从文件链表中删除文件 + const int64_t rand_remove_cnt = ObRandom::rand(1, file_handles.size()); + for (int64_t i = 0; i < rand_remove_cnt; ++i) { + ObTmpFileHandle file_handle = file_handles.at(i); + ASSERT_NE(nullptr, file_handle.get()); + ASSERT_EQ(OB_SUCCESS, ret); + ObSharedNothingTmpFile &file = *file_handle.get(); + ret = flush_prio_mgr.remove_file(false/*is_meta*/, file); + ASSERT_EQ(OB_SUCCESS, ret); + } + + ObTmpFileFlushListIterator iter; + ret = iter.init(&flush_prio_mgr); + ASSERT_EQ(OB_SUCCESS, ret); + + int64_t remain_file_cnt = 0; + for (int64_t t = FlushCtxState::FSM_F1; t < FlushCtxState::FSM_FINISHED; ++t) { + for (int64_t i = 0; OB_SUCC(ret) && i < FILE_NUM * 10; ++i) { + bool is_meta; // unused + ObTmpFileHandle file_handle; + ret = iter.next(FlushCtxState(t), is_meta, file_handle); + if (OB_SUCC(ret)) { + ASSERT_NE(file_handle.get(), nullptr); + ObSharedNothingTmpFile &tmp_file = *file_handle.get(); + int64_t fd = tmp_file.get_fd(); + remain_file_cnt += 1; + } + } + ASSERT_EQ(ret, OB_ITER_END); + ret = OB_SUCCESS; + } + ASSERT_EQ(remain_file_cnt + rand_remove_cnt, total_file_cnt); +} + +TEST_F(TestFlushListIterator, test_flush_list_update) +{ + int ret = OB_SUCCESS; + + ObTmpFileFlushPriorityManager flush_prio_mgr; + ret = flush_prio_mgr.init(); + ASSERT_EQ(OB_SUCCESS, ret); + vector file_handles; + int total_file_num = 0; + const int64_t FILE_NUM = 10; + + // 创建文件，根据层级生成模拟的脏页数量 + int64_t total_file_cnt = 0; + create_files(FlushCtxState::FSM_F1, FILE_NUM, flush_prio_mgr, file_handles); + + // 0.插入文件0～4 + for (int64_t i = 0; i < file_handles.size() / 2; ++i) { + ObTmpFileHandle file_handle = file_handles.at(i); + ASSERT_NE(file_handle.get(), nullptr); + ObSharedNothingTmpFile &tmp_file = *file_handle.get(); + TestDirtyPageRecord &dirty_record = mock_dirty_record_.at(tmp_file.get_fd()); + ret = flush_prio_mgr.insert_data_flush_list(tmp_file, dirty_record.dirty_data_size_); + ASSERT_EQ(OB_SUCCESS, ret); + total_file_cnt += 1; + } + + // 1. 更新不在链表中的文件 + for (int64_t i = file_handles.size() / 2; i < file_handles.size(); ++i) { + ObTmpFileHandle file_handle = file_handles.at(i); + ObSharedNothingTmpFile &tmp_file = *file_handle.get(); + TestDirtyPageRecord &dirty_record = mock_dirty_record_.at(tmp_file.get_fd()); + ret = flush_prio_mgr.update_data_flush_list(tmp_file, dirty_record.dirty_data_size_); + ASSERT_EQ(OB_ERR_UNEXPECTED, ret); + } + + // 2. 对层级没有变动的文件进行更新 + for (int64_t i = 0; i < file_handles.size() / 2; ++i) { + ObTmpFileHandle file_handle = file_handles.at(i); + ObSharedNothingTmpFile &tmp_file = *file_handle.get(); + TestDirtyPageRecord &dirty_record = mock_dirty_record_.at(tmp_file.get_fd()); + ret = flush_prio_mgr.update_data_flush_list(tmp_file, dirty_record.dirty_data_size_); + ASSERT_EQ(OB_SUCCESS, ret); + } + + // 3. 更新到新的层级 + for (int64_t i = 0; i < file_handles.size() / 2; ++i) { + ObTmpFileHandle file_handle = file_handles.at(i); + ObSharedNothingTmpFile &tmp_file = *file_handle.get(); + TestDirtyPageRecord &dirty_record = mock_dirty_record_.at(tmp_file.get_fd()); + dirty_record.dirty_data_size_ = 4096; + ret = flush_prio_mgr.update_data_flush_list(tmp_file, dirty_record.dirty_data_size_); + ASSERT_EQ(OB_SUCCESS, ret); + } + + // 4. 插入文件5～10 + for (int64_t i = file_handles.size() / 2; i < file_handles.size(); ++i) { + ObTmpFileHandle file_handle = file_handles.at(i); + ASSERT_NE(file_handle.get(), nullptr); + ObSharedNothingTmpFile &tmp_file = *file_handle.get(); + TestDirtyPageRecord &dirty_record = mock_dirty_record_.at(tmp_file.get_fd()); + ret = flush_prio_mgr.insert_data_flush_list(tmp_file, dirty_record.dirty_data_size_); + ASSERT_EQ(OB_SUCCESS, ret); + total_file_cnt += 1; + } + + ObTmpFileFlushListIterator iter; + ret = iter.init(&flush_prio_mgr); + ASSERT_EQ(OB_SUCCESS, ret); + + // 5. F1中可以取出5个文件 + bool is_meta; // unused + for (int64_t i = 0; i < file_handles.size() / 2; ++i) { + ObTmpFileHandle file_handle; + ret = iter.next(FlushCtxState::FSM_F1, is_meta, file_handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(file_handle.get(), nullptr); + } + + // 6. F5中可以取出5个更新后的文件 + for (int64_t i = 0; i < file_handles.size() / 2; ++i) { + ObTmpFileHandle file_handle; + ret = iter.next(FlushCtxState::FSM_F3, is_meta, file_handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(file_handle.get(), nullptr); + } +} + +TEST_F(TestFlushListIterator, test_flush_list_reinsert_after_use) +{ + int ret = OB_SUCCESS; + int64_t dir = -1; + ret = MTL(ObTenantTmpFileManager *)->alloc_dir(dir); + ASSERT_EQ(OB_SUCCESS, ret); + + ObTmpFileFlushPriorityManager flush_prio_mgr; + ret = flush_prio_mgr.init(); + ASSERT_EQ(OB_SUCCESS, ret); + ObTmpFilePageCacheController &pc_ctrl = MTL(ObTenantTmpFileManager *)->get_page_cache_controller(); + vector file_handles; + const int64_t FILE_NUM = 10; + FlushCtxState state = FlushCtxState::FSM_F2; + create_files_with_dir(state, dir, FILE_NUM, flush_prio_mgr, file_handles); + ASSERT_EQ(FILE_NUM, file_handles.size()); + + ObTmpFileFlushListIterator iter; + ret = iter.init(&flush_prio_mgr); + ASSERT_EQ(OB_SUCCESS, ret); + + for (int64_t i = 0; i < file_handles.size(); ++i) { + ObTmpFileHandle file_handle = file_handles.at(i); + ASSERT_NE(file_handle.get(), nullptr); + ObSharedNothingTmpFile &tmp_file = *file_handle.get(); + TestDirtyPageRecord &dirty_record = mock_dirty_record_.at(tmp_file.get_fd()); + ret = flush_prio_mgr.insert_data_flush_list(tmp_file, dirty_record.dirty_data_size_); + ASSERT_EQ(OB_SUCCESS, ret); + } + + // 取出迭代器dir中一半的文件 + const int64_t USED_FILE_CNT = file_handles.size() / 2; + bool is_meta = false; // unused + for (int64_t i = 0; i < USED_FILE_CNT; ++i) { + ObTmpFileHandle file_handle; + ret = iter.next(state, is_meta, file_handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(file_handle.get(), nullptr); + printf("use file %ld\n", file_handle.get()->get_fd()); + } + + // 迭代器中剩余文件重新插回flush_prio_mgr + iter.destroy(); + ret = iter.init(&flush_prio_mgr); + ASSERT_EQ(OB_SUCCESS, ret); + + // 重新初始化迭代器，通过迭代器取出剩余文件 + int64_t remain_file_cnt = 0; + while (OB_SUCC(ret)) { + ObTmpFileHandle file_handle; + if (OB_SUCC(iter.next(state, is_meta, file_handle))) { + remain_file_cnt += 1; + } + } + printf("remain_file_cnt:%ld, USED_FILE_CNT:%ld, FILE_NUM:%ld\n", remain_file_cnt, USED_FILE_CNT, FILE_NUM); + ASSERT_EQ(remain_file_cnt + USED_FILE_CNT, FILE_NUM); +} + +} // namespace oceanbase + +int main(int argc, char **argv) +{ + int ret = 0; + system("rm -f ./test_tmp_file_flush_list.log*"); + OB_LOGGER.set_file_name("test_tmp_file_flush_list.log", true); + OB_LOGGER.set_log_level("INFO"); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/mittest/mtlenv/storage/tmp_file/test_tmp_file_meta_tree.cpp b/mittest/mtlenv/storage/tmp_file/test_tmp_file_meta_tree.cpp new file mode 100644 index 000000000..661818c7c --- /dev/null +++ b/mittest/mtlenv/storage/tmp_file/test_tmp_file_meta_tree.cpp @@ -0,0 +1,2381 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include +#include +#include +#include +#define USING_LOG_PREFIX STORAGE +#define protected public +#define private public +#include "mittest/mtlenv/mock_tenant_module_env.h" +#include "share/ob_simple_mem_limit_getter.h" +#include "storage/tmp_file/ob_tmp_file_meta_tree.h" + +namespace oceanbase +{ +using namespace common; +using namespace blocksstable; +using namespace tmp_file; +using namespace storage; +using namespace share::schema; + +/* --------------------------------- Mock META TREE---------------------------------- */ +class ObTmpFileTestMetaTree : public ObSharedNothingTmpFileMetaTree +{ +public: + ObTmpFileTestMetaTree() : release_pages_() {} + ~ObTmpFileTestMetaTree(); + void reset(); +private: + virtual int release_meta_page_(const ObSharedNothingTmpFileMetaItem &page_info, + const int32_t page_index_in_level); + virtual int release_tmp_file_page_(const int64_t block_index, + const int64_t begin_page_id, const int64_t page_num); + virtual int cache_page_for_write_(const uint32_t parent_page_id, + ObSharedNothingTmpFileMetaItem &page_info); +public: + ObArray release_pages_; + //mock rightmost pages in read_cache, page_level is the index + //first means page buffer in read_cache + //second means read count in read_cache + ObArray> read_cache_rightmost_pages_; +}; + +ObTmpFileTestMetaTree::~ObTmpFileTestMetaTree() +{ + reset(); +} + +void ObTmpFileTestMetaTree::reset() +{ + release_pages_.reset(); +} + +int ObTmpFileTestMetaTree::release_meta_page_(const ObSharedNothingTmpFileMetaItem &page_info, + const int32_t page_index_in_level) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(level_page_range_array_.count() <= page_info.page_level_ + || 0 > page_index_in_level)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected level_page_range_array_ or page_info", KR(ret), K(fd_), K(level_page_range_array_), + K(page_info), K(page_index_in_level)); + } else if (is_page_in_write_cache(page_info)) { + const uint32_t start_page_id_in_array = level_page_range_array_.at(page_info.page_level_).start_page_id_; + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + if (OB_FAIL(wbp_->free_page(fd_, page_info.buffer_page_id_, + ObTmpFilePageUniqKey(page_info.page_level_, page_index_in_level), next_page_id))) { + STORAGE_LOG(WARN, "fail to free meta page in write cache", KR(ret), K(fd_), K(page_info), K(page_index_in_level)); + } else if (OB_UNLIKELY(start_page_id_in_array != page_info.buffer_page_id_)) { + //NOTE: pages must be released sequentially (from front to back in array) + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected level_page_range_array_", KR(ret), K(fd_), K(level_page_range_array_), K(page_info)); + } else { + level_page_range_array_.at(page_info.page_level_).start_page_id_ = next_page_id; + if (start_page_id_in_array == level_page_range_array_.at(page_info.page_level_).end_page_id_) { + //next_page_id must be invalid + level_page_range_array_.at(page_info.page_level_).end_page_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + } + if (ObTmpFileGlobal::INVALID_PAGE_ID == level_page_range_array_.at(page_info.page_level_).flushed_end_page_id_) { + level_page_range_array_.at(page_info.page_level_).flushed_page_num_ += 1; + } + if (start_page_id_in_array == level_page_range_array_.at(page_info.page_level_).flushed_end_page_id_) { + level_page_range_array_.at(page_info.page_level_).flushed_end_page_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + } + level_page_range_array_.at(page_info.page_level_).cached_page_num_ -= 1; + level_page_range_array_.at(page_info.page_level_).evicted_page_num_ += 1; + } + } + if (OB_SUCC(ret) && is_page_flushed(page_info)) { + if (OB_FAIL(release_tmp_file_page_(page_info.block_index_, page_info.physical_page_id_, 1))) { + STORAGE_LOG(WARN, "fail to release tmp file page", KR(ret), K(fd_), K(page_info)); + } + } + if (FAILEDx(release_pages_.push_back(page_info.buffer_page_id_))) { + STORAGE_LOG(WARN, "fail to push_back", KR(ret), K(page_info)); + } + return ret; +} + +int ObTmpFileTestMetaTree::release_tmp_file_page_(const int64_t block_index, + const int64_t begin_page_id, const int64_t page_num) +{ + int ret = OB_SUCCESS; + stat_info_.all_type_flush_page_released_cnt_ += page_num; + return ret; +} + +int ObTmpFileTestMetaTree::cache_page_for_write_( + const uint32_t parent_page_id, + ObSharedNothingTmpFileMetaItem &page_info) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!page_info.is_valid() + || page_info.page_level_ >= level_page_range_array_.count() + || (ObTmpFileGlobal::INVALID_PAGE_ID != parent_page_id + && page_info.page_level_ + 1 >= level_page_range_array_.count()))) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), K(page_info), K(level_page_range_array_), K(parent_page_id)); + } else { + if (!is_page_in_write_cache(page_info)) { + uint32_t new_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + char *new_page_buff = NULL; + int32_t level_page_index = level_page_range_array_[page_info.page_level_].evicted_page_num_ - 1; + ObTmpFilePageUniqKey page_key(page_info.page_level_, level_page_index); + if (OB_UNLIKELY(!is_page_flushed(page_info) + || 0 < level_page_range_array_[page_info.page_level_].cached_page_num_ + || 0 > level_page_index + || ObTmpFileGlobal::INVALID_PAGE_ID != level_page_range_array_[page_info.page_level_].end_page_id_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected page_info", KR(ret), K(fd_), K(page_info), K(level_page_range_array_)); + } else if (OB_FAIL(wbp_->alloc_page(fd_, page_key, new_page_id, new_page_buff))) { + STORAGE_LOG(WARN, "fail to alloc meta page", KR(ret), K(fd_), K(page_info), K(level_page_range_array_)); + } else if (OB_ISNULL(new_page_buff)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected null page buff", KR(ret), K(fd_), KP(new_page_buff)); + } else if (OB_FAIL(wbp_->notify_load(fd_, new_page_id, page_key))) { + STORAGE_LOG(WARN, "fail to notify load for meta", KR(ret), K(fd_), K(new_page_id), K(page_key)); + } else { + if (OB_UNLIKELY(read_cache_rightmost_pages_.count() <= page_info.page_level_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected read_cache_rightmost_pages_", KR(ret), K(read_cache_rightmost_pages_.count()), K(page_info)); + } else { + MEMCPY(new_page_buff, read_cache_rightmost_pages_.at(page_info.page_level_).first, ObTmpFileGlobal::PAGE_SIZE); + read_cache_rightmost_pages_.at(page_info.page_level_).second++; + } + if (FAILEDx(check_page_(new_page_buff))) { + STORAGE_LOG(WARN, "the page is invalid or corrupted", KR(ret), K(fd_), KP(new_page_buff)); + } + if (OB_SUCC(ret)) { + //change page state to cached + if (OB_FAIL(wbp_->notify_load_succ(fd_, new_page_id, page_key))) { + STORAGE_LOG(WARN, "fail to notify load succ for meta", KR(ret), K(fd_), K(new_page_id), K(page_key)); + } + } else { + int tmp_ret = OB_SUCCESS; + //change page state to invalid + if (OB_TMP_FAIL(wbp_->notify_load_fail(fd_, new_page_id, page_key))) { + STORAGE_LOG(WARN, "fail to notify load fail for meta", KR(tmp_ret), K(fd_), K(new_page_id), K(page_key)); + } + } + } + if (OB_SUCC(ret)) { + int64_t origin_block_index = page_info.block_index_; + int16_t origin_physical_page_id = page_info.physical_page_id_; + page_info.buffer_page_id_ = new_page_id; + page_info.block_index_ = ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX; + page_info.physical_page_id_ = -1; + if (ObTmpFileGlobal::INVALID_PAGE_ID == parent_page_id) { + root_item_ = page_info; + } else { + char *parent_page_buff = NULL; + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ObSharedNothingTmpFileTreePageHeader page_header; + int32_t parent_level_page_index = level_page_range_array_[page_info.page_level_ + 1].evicted_page_num_ + + level_page_range_array_[page_info.page_level_ + 1].cached_page_num_ - 1; + ObTmpFilePageUniqKey parent_page_offset(page_info.page_level_ + 1, parent_level_page_index); + if (OB_FAIL(wbp_->read_page(fd_, parent_page_id, parent_page_offset, parent_page_buff, next_page_id))) { + STORAGE_LOG(WARN, "fail to read from write cache", KR(ret), K(fd_), K(parent_page_id), K(parent_page_offset)); + } else if (OB_FAIL(read_page_header_(parent_page_buff, page_header))) { + STORAGE_LOG(WARN, "fail to read page header", KR(ret), K(fd_), KP(parent_page_buff)); + } else if (OB_FAIL(rewrite_item_(parent_page_buff, page_header.item_num_ - 1, page_info))) { + STORAGE_LOG(WARN, "fail to rewrite item", KR(ret), K(fd_), K(page_header), K(page_info), KP(parent_page_buff)); + } else if (OB_FAIL(wbp_->notify_dirty(fd_, parent_page_id, parent_page_offset))) { + STORAGE_LOG(WARN, "fail to notify dirty for meta", KR(ret), K(fd_), K(parent_page_id), K(parent_page_offset)); + } + } + if (OB_SUCC(ret)) { + int16_t page_level = page_info.page_level_; + if (OB_FAIL(wbp_->notify_dirty(fd_, new_page_id, page_key))) { + STORAGE_LOG(WARN, "fail to notify dirty", KR(ret), K(fd_), K(new_page_id), K(page_key)); + } else if (OB_FAIL(release_tmp_file_page_(origin_block_index, origin_physical_page_id, 1))) { + STORAGE_LOG(WARN, "fail to release tmp file page", KR(ret), K(fd_), K(origin_block_index), K(origin_physical_page_id)); + } else { + level_page_range_array_[page_level].start_page_id_ = new_page_id; + level_page_range_array_[page_level].end_page_id_ = new_page_id; + level_page_range_array_[page_level].cached_page_num_++; + level_page_range_array_[page_level].evicted_page_num_--; + level_page_range_array_[page_level].flushed_page_num_--; + } + } + } else if (ObTmpFileGlobal::INVALID_PAGE_ID != new_page_id) { //fail + uint32_t unused_next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(wbp_->free_page(fd_, new_page_id, page_key, unused_next_page_id))) { + STORAGE_LOG(WARN, "fail to free meta page", KR(tmp_ret), K(fd_), K(new_page_id), K(page_key)); + } + } + STORAGE_LOG(INFO, "load page to write cache", KR(ret), K(fd_), K(page_info), K(page_key)); + } else { + //still in write cache + if (!is_page_in_write_cache(page_info)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected page_info", KR(ret), K(fd_), K(page_info)); + } + } + } + return ret; +} + +/* ---------------------------- Unittest Class ----------------------------- */ +class TestSNTmpFileMetaTree : public ::testing::Test +{ +public: + TestSNTmpFileMetaTree() = default; + virtual ~TestSNTmpFileMetaTree() = default; + static void SetUpTestCase(); + static void TearDownTestCase(); +public: + void generate_data_items(const int64_t item_num, + const int64_t start_virtual_page_id, + ObArray &data_items); + void generate_wrong_data_items(const int64_t item_num, + const int64_t start_virtual_page_id, + ObArray &data_items); + void test_tree_flush_with_truncate_occurs_before_update_meta( + int64_t truncate_offset, bool insert_after_truncate); +}; + +// static ObSimpleMemLimitGetter getter; + +//TODO: test data_item_array +void TestSNTmpFileMetaTree::SetUpTestCase() +{ + int ret = OB_SUCCESS; + // const int64_t bucket_num = 1024; + // const int64_t max_cache_size = 1024 * 1024 * 1024; + // const int64_t block_size = common::OB_MALLOC_BIG_BLOCK_SIZE; + ASSERT_EQ(OB_SUCCESS, MockTenantModuleEnv::get_instance().init()); + + // ret = getter.add_tenant(1, + // 8L * 1024L * 1024L, 2L * 1024L * 1024L * 1024L); + // ASSERT_EQ(OB_SUCCESS, ret); + // ret = ObKVGlobalCache::get_instance().init(&getter, bucket_num, max_cache_size, block_size); + // if (OB_INIT_TWICE == ret) { + // ret = OB_SUCCESS; + // } else { + // ASSERT_EQ(OB_SUCCESS, ret); + // } +} + +void TestSNTmpFileMetaTree::TearDownTestCase() +{ + MockTenantModuleEnv::get_instance().destroy(); + // ObKVGlobalCache::get_instance().destroy(); +} + +//mock data items +void TestSNTmpFileMetaTree::generate_data_items( + const int64_t item_num, + const int64_t start_virtual_page_id, + ObArray &data_items) +{ + int64_t block_index = 0; + int16_t physical_page_id = 0; + int16_t physical_page_num = 128; //(OB_DEFAULT_MACRO_BLOCK_SIZE / ObTmpFileGlobal::PAGE_SIZE) / 2 + int64_t virtual_page_id = start_virtual_page_id; + ObSharedNothingTmpFileDataItem data_item; + for (int64_t i = block_index; i < item_num; i++) { + data_item.reset(); + data_item.block_index_ = block_index; + data_item.physical_page_id_ = physical_page_id; + data_item.physical_page_num_ = physical_page_num; + data_item.virtual_page_id_ = virtual_page_id; + block_index++; + virtual_page_id += physical_page_num; + ASSERT_EQ(OB_SUCCESS, data_items.push_back(data_item)); + } +} + +void TestSNTmpFileMetaTree::generate_wrong_data_items( + const int64_t item_num, + const int64_t start_virtual_page_id, + ObArray &data_items) +{ + int64_t block_index = 0; + int16_t physical_page_id = 0; + int16_t physical_page_num = 128; //(OB_DEFAULT_MACRO_BLOCK_SIZE / ObTmpFileGlobal::PAGE_SIZE) / 2 + int64_t virtual_page_id = start_virtual_page_id; + ObSharedNothingTmpFileDataItem data_item; + for (int64_t i = block_index; i < item_num; i++) { + data_item.reset(); + data_item.block_index_ = block_index; + data_item.physical_page_id_ = physical_page_id; + data_item.physical_page_num_ = physical_page_num; + data_item.virtual_page_id_ = virtual_page_id > 0 ? virtual_page_id - std::rand() % 50 - 1 : 0; + block_index++; + virtual_page_id += physical_page_num; + ASSERT_EQ(OB_SUCCESS, data_items.push_back(data_item)); + } +} + +TEST_F(TestSNTmpFileMetaTree, test_tree_insert) +{ + STORAGE_LOG(INFO, "=======================test_tree_insert begin======================="); + ObTmpWriteBufferPool wbp; + // wbp.default_wbp_memory_limit_ = ObTmpWriteBufferPool::WBP_BLOCK_SIZE; //253 pages + ASSERT_EQ(OB_SUCCESS, wbp.init()); + common::ObFIFOAllocator callback_allocator; + ASSERT_EQ(OB_SUCCESS, callback_allocator.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileCallback", ObCtxIds::DEFAULT_CTX_ID))); + ObTmpFileTestMetaTree meta_tree_; + ASSERT_EQ(OB_SUCCESS, meta_tree_.init(1, &wbp, &callback_allocator)); + meta_tree_.set_max_array_item_cnt(2); + meta_tree_.set_max_page_item_cnt(3); + const int64_t item_num = 30; + ObArray data_items; + ObArray data_items_1; + ObArray data_items_2; + ObArray data_items_3; + generate_data_items(item_num, 0/*start virtual page id*/, data_items); + ASSERT_EQ(item_num, data_items.count()); + STORAGE_LOG(INFO, "=======================first insert======================="); + //insert 9 items (insert a array) + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + for (int64_t i = 0; i < 9; i++) { + ASSERT_EQ(OB_SUCCESS, data_items_1.push_back(data_items.at(i))); + } + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items_1)); + ASSERT_EQ(2, meta_tree_.level_page_range_array_.count()); + STORAGE_LOG(INFO, "level_page_range_array", K(meta_tree_.level_page_range_array_)); + + //we assume that there are some disk flushing and other operations here. + + STORAGE_LOG(INFO, "=======================second insert======================="); + //insert 18 items (insert one by one) + for (int64_t i = 9; i < 27; i++) { + data_items_2.reset(); + ASSERT_EQ(OB_SUCCESS, data_items_2.push_back(data_items.at(i))); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items_2)); + } + ASSERT_EQ(3, meta_tree_.level_page_range_array_.count()); + STORAGE_LOG(INFO, "level_page_range_array", K(meta_tree_.level_page_range_array_)); + + //we assume that there are some disk flushing and other operations here. + + STORAGE_LOG(INFO, "=======================third insert======================="); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + for (int64_t i = 27; i < item_num; i++) { + ASSERT_EQ(OB_SUCCESS, data_items_3.push_back(data_items.at(i))); + } + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items_3)); + ASSERT_EQ(4, meta_tree_.level_page_range_array_.count()); + STORAGE_LOG(INFO, "level_page_range_array", K(meta_tree_.level_page_range_array_)); + + ASSERT_EQ(OB_SUCCESS, meta_tree_.clear(0, 30 * 128 * ObTmpFileGlobal::PAGE_SIZE)); + meta_tree_.reset(); + STORAGE_LOG(INFO, "=======================test_tree_insert end======================="); +} + +TEST_F(TestSNTmpFileMetaTree, test_tree_insert_fail) +{ + STORAGE_LOG(INFO, "=======================test_tree_insert_fail begin======================="); + ObTmpWriteBufferPool wbp; + wbp.default_wbp_memory_limit_ = ObTmpWriteBufferPool::WBP_BLOCK_SIZE; //253 pages + ASSERT_EQ(OB_SUCCESS, wbp.init()); + common::ObFIFOAllocator callback_allocator; + ASSERT_EQ(OB_SUCCESS, callback_allocator.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileCallback", ObCtxIds::DEFAULT_CTX_ID))); + ObTmpFileTestMetaTree meta_tree_; + ASSERT_EQ(OB_SUCCESS, meta_tree_.init(1, &wbp, &callback_allocator)); + meta_tree_.set_max_array_item_cnt(2); + meta_tree_.set_max_page_item_cnt(3); + const int64_t item_num = 750; + ObArray data_items; + ObArray data_items_1; + generate_data_items(item_num, 0/*start virtual page id*/, data_items); + ASSERT_EQ(item_num, data_items.count()); + STORAGE_LOG(INFO, "=======================first insert======================="); + //insert 750 items (insert a array) + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_ALLOCATE_TMP_FILE_PAGE_FAILED, meta_tree_.insert_items(data_items)); + ASSERT_EQ(0, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(false, meta_tree_.root_item_.is_valid()); + + //we assume that there are some disk flushing and other operations here. + + STORAGE_LOG(INFO, "=======================second insert======================="); + //insert 90 items (insert one by one) + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + for (int64_t i = 0; i < 90; i++) { + ASSERT_EQ(OB_SUCCESS, data_items_1.push_back(data_items.at(i))); + } + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items_1)); + ASSERT_EQ(5, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(30, meta_tree_.level_page_range_array_.at(0).cached_page_num_); + ASSERT_EQ(10, meta_tree_.level_page_range_array_.at(1).cached_page_num_); + ASSERT_EQ(4, meta_tree_.level_page_range_array_.at(2).cached_page_num_); + ASSERT_EQ(2, meta_tree_.level_page_range_array_.at(3).cached_page_num_); + ASSERT_EQ(1, meta_tree_.level_page_range_array_.at(4).cached_page_num_); + + ASSERT_EQ(OB_SUCCESS, meta_tree_.clear(0, 90 * 128 * ObTmpFileGlobal::PAGE_SIZE)); + meta_tree_.reset(); + STORAGE_LOG(INFO, "=======================test_tree_insert_fail end======================="); +} + +TEST_F(TestSNTmpFileMetaTree, test_tree_read) +{ + STORAGE_LOG(INFO, "=======================test_tree_read begin======================="); + ObTmpWriteBufferPool wbp; + wbp.default_wbp_memory_limit_ = ObTmpWriteBufferPool::WBP_BLOCK_SIZE; //253 pages + ASSERT_EQ(OB_SUCCESS, wbp.init()); + common::ObFIFOAllocator callback_allocator; + ASSERT_EQ(OB_SUCCESS, callback_allocator.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileCallback", ObCtxIds::DEFAULT_CTX_ID))); + ObTmpFileTestMetaTree meta_tree_; + ASSERT_EQ(OB_SUCCESS, meta_tree_.init(1, &wbp, &callback_allocator)); + meta_tree_.set_max_array_item_cnt(2); + meta_tree_.set_max_page_item_cnt(3); + const int64_t item_num = 25; + ObArray data_items; + generate_data_items(item_num, 0/*start virtual page id*/, data_items); + ASSERT_EQ(item_num, data_items.count()); + STORAGE_LOG(INFO, "=======================tree insert======================="); + //insert 25 items (insert a array) + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items)); + ASSERT_EQ(3, meta_tree_.level_page_range_array_.count()); + STORAGE_LOG(INFO, "level_page_range_array", K(meta_tree_.level_page_range_array_)); + + //we assume that there are some disk flushing and other operations here. + //After simulating the insert, we can see that the tmp file offset is [0, 25 * 128 * 8K], + // with each data item occupying 128 * 8K. + STORAGE_LOG(INFO, "=======================first tree read======================="); + ObArray get_data_items; + ASSERT_EQ(OB_SUCCESS, meta_tree_.search_data_items(1, 2 * 128 * ObTmpFileGlobal::PAGE_SIZE - 2, get_data_items)); + STORAGE_LOG(INFO, "data_items", K(get_data_items)); + ASSERT_EQ(2, get_data_items.count()); + + STORAGE_LOG(INFO, "=======================second tree read======================="); + get_data_items.reset(); + ASSERT_EQ(OB_SUCCESS, meta_tree_.search_data_items(7 * 128 * ObTmpFileGlobal::PAGE_SIZE, 5 * 128 * ObTmpFileGlobal::PAGE_SIZE, get_data_items)); + STORAGE_LOG(INFO, "data_items", K(get_data_items)); + ASSERT_EQ(5, get_data_items.count()); + + STORAGE_LOG(INFO, "=======================third tree read======================="); + get_data_items.reset(); + ASSERT_EQ(OB_SUCCESS, meta_tree_.search_data_items(0, 4 * 128 * ObTmpFileGlobal::PAGE_SIZE, get_data_items)); + STORAGE_LOG(INFO, "data_items", K(get_data_items)); + ASSERT_EQ(4, get_data_items.count()); + + STORAGE_LOG(INFO, "=======================forth tree read======================="); + get_data_items.reset(); + ASSERT_EQ(OB_SUCCESS, meta_tree_.search_data_items(0, 25 * 128 * ObTmpFileGlobal::PAGE_SIZE, get_data_items)); + STORAGE_LOG(INFO, "data_items", K(get_data_items)); + ASSERT_EQ(25, get_data_items.count()); + + ASSERT_EQ(OB_SUCCESS, meta_tree_.clear(0, 25 * 128 * ObTmpFileGlobal::PAGE_SIZE)); + meta_tree_.reset(); + STORAGE_LOG(INFO, "=======================test_tree_read end======================="); +} + +TEST_F(TestSNTmpFileMetaTree, test_tree_flush) +{ + STORAGE_LOG(INFO, "=======================test_tree_flush begin======================="); + ObTmpWriteBufferPool wbp; + wbp.default_wbp_memory_limit_ = ObTmpWriteBufferPool::WBP_BLOCK_SIZE; //253 pages + ASSERT_EQ(OB_SUCCESS, wbp.init()); + common::ObFIFOAllocator callback_allocator; + ASSERT_EQ(OB_SUCCESS, callback_allocator.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileCallback", ObCtxIds::DEFAULT_CTX_ID))); + ObTmpFileTestMetaTree meta_tree_; + ASSERT_EQ(OB_SUCCESS, meta_tree_.init(1, &wbp, &callback_allocator)); + meta_tree_.set_max_array_item_cnt(2); + meta_tree_.set_max_page_item_cnt(3); + const int64_t item_num = 28; + ObArray data_items; + ObArray data_items_1; + ObArray data_items_2; + generate_data_items(item_num, 0/*start virtual page id*/, data_items); + ASSERT_EQ(item_num, data_items.count()); + STORAGE_LOG(INFO, "=======================first tree insert======================="); + //insert 25 items (insert a array) + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + for (int64_t i = 0; i < 25; i++) { + ASSERT_EQ(OB_SUCCESS, data_items_1.push_back(data_items.at(i))); + } + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items_1)); + ASSERT_EQ(3, meta_tree_.level_page_range_array_.count()); + STORAGE_LOG(INFO, "level_page_range_array", K(meta_tree_.level_page_range_array_)); + + STORAGE_LOG(INFO, "=======================check tree flush pages======================="); + int64_t total_need_flush_page_num = 0; + int64_t total_need_flush_rightmost_page_num = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.get_need_flush_page_num(total_need_flush_page_num, total_need_flush_rightmost_page_num)); + ASSERT_EQ(13, total_need_flush_page_num); + ASSERT_EQ(3, total_need_flush_rightmost_page_num); + + STORAGE_LOG(INFO, "=======================first tree flush======================="); + char *block_buff_1 = new char[OB_DEFAULT_MACRO_BLOCK_SIZE]; + ObTmpFileTreeFlushContext flush_context_1; + ObArray tree_io_array_1; + int64_t write_offset_1 = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.flush_meta_pages_for_block(0/*block_index*/, + ObTmpFileTreeEvictType::FULL, + block_buff_1, + write_offset_1, + flush_context_1, + tree_io_array_1)); + STORAGE_LOG(INFO, "tree_io_array", K(tree_io_array_1)); + ASSERT_EQ(3, tree_io_array_1.count()); + ASSERT_EQ(0 + 13 * ObTmpFileGlobal::PAGE_SIZE, write_offset_1); + total_need_flush_page_num = 0; + total_need_flush_rightmost_page_num = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.get_need_flush_page_num(total_need_flush_page_num, total_need_flush_rightmost_page_num)); + ASSERT_EQ(0, total_need_flush_page_num); + ASSERT_EQ(0, total_need_flush_rightmost_page_num); + + STORAGE_LOG(INFO, "=======================second tree insert======================="); + //insert 3 items (insert a array) + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + for (int64_t i = 25; i < 28; i++) { + ASSERT_EQ(OB_SUCCESS, data_items_2.push_back(data_items.at(i))); + } + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items_2)); + ASSERT_EQ(4, meta_tree_.level_page_range_array_.count()); + STORAGE_LOG(INFO, "level_page_range_array", K(meta_tree_.level_page_range_array_)); + + STORAGE_LOG(INFO, "=======================check tree flush pages======================="); + total_need_flush_page_num = 0; + total_need_flush_rightmost_page_num = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.get_need_flush_page_num(total_need_flush_page_num, total_need_flush_rightmost_page_num)); + ASSERT_EQ(5, total_need_flush_page_num); + ASSERT_EQ(4, total_need_flush_rightmost_page_num); + + STORAGE_LOG(INFO, "=======================second tree flush======================="); + //NOTE: We will not flush the tree again before io returns successfully. + //So here we assume that io is successful and call this function "update_after_flush". + ASSERT_EQ(OB_SUCCESS, meta_tree_.update_after_flush(tree_io_array_1)); + char *block_buff_2 = new char[OB_DEFAULT_MACRO_BLOCK_SIZE]; + ObTmpFileTreeFlushContext flush_context_2; + ObArray tree_io_array_2; + int64_t write_offset_2 = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.flush_meta_pages_for_block(1/*block_index*/, + ObTmpFileTreeEvictType::FULL, + block_buff_2, + write_offset_2, + flush_context_2, + tree_io_array_2)); + STORAGE_LOG(INFO, "tree_io_array", K(tree_io_array_2)); + ASSERT_EQ(4, tree_io_array_2.count()); + //flush 7 dirty pages. + //During the flushing process, + // disk information will be changed to the upper-level pages, so upper-level pages will become dirty pages during this process. + // So you shouldn’t be surprised that the number of dirty pages changed from 5 to 7 + ASSERT_EQ(0 + 7 * ObTmpFileGlobal::PAGE_SIZE, write_offset_2); + ASSERT_EQ(OB_SUCCESS, meta_tree_.update_after_flush(tree_io_array_2)); + for (int64_t i = 0; i < 4; i++) { + ObTmpFileTreeIOInfo& tree_io = tree_io_array_2.at(i); + ASSERT_EQ(tree_io.flush_end_page_id_, meta_tree_.level_page_range_array_.at(tree_io.page_level_).flushed_end_page_id_); + } + total_need_flush_page_num = 0; + total_need_flush_rightmost_page_num = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.get_need_flush_page_num(total_need_flush_page_num, total_need_flush_rightmost_page_num)); + ASSERT_EQ(0, total_need_flush_page_num); + ASSERT_EQ(0, total_need_flush_rightmost_page_num); + + ASSERT_EQ(OB_SUCCESS, meta_tree_.clear(0, 28 * 128 * ObTmpFileGlobal::PAGE_SIZE)); + meta_tree_.reset(); + delete[] block_buff_1; + delete[] block_buff_2; + STORAGE_LOG(INFO, "=======================test_tree_flush end======================="); +} + +TEST_F(TestSNTmpFileMetaTree, test_tree_flush_with_multi_io) +{ + STORAGE_LOG(INFO, "=======================test_tree_flush_with_multi_io begin======================="); + ObTmpWriteBufferPool wbp; + wbp.default_wbp_memory_limit_ = ObTmpWriteBufferPool::WBP_BLOCK_SIZE; //253 pages + ASSERT_EQ(OB_SUCCESS, wbp.init()); + common::ObFIFOAllocator callback_allocator; + ASSERT_EQ(OB_SUCCESS, callback_allocator.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileCallback", ObCtxIds::DEFAULT_CTX_ID))); + ObTmpFileTestMetaTree meta_tree_; + ASSERT_EQ(OB_SUCCESS, meta_tree_.init(1, &wbp, &callback_allocator)); + meta_tree_.set_max_array_item_cnt(2); + meta_tree_.set_max_page_item_cnt(3); + const int64_t item_num = 25; + ObArray data_items; + generate_data_items(item_num, 0/*start virtual page id*/, data_items); + ASSERT_EQ(item_num, data_items.count()); + STORAGE_LOG(INFO, "=======================tree insert======================="); + //insert 25 items (insert a array) + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items)); + ASSERT_EQ(3, meta_tree_.level_page_range_array_.count()); + STORAGE_LOG(INFO, "level_page_range_array", K(meta_tree_.level_page_range_array_)); + + STORAGE_LOG(INFO, "=======================check tree flush pages======================="); + int64_t total_need_flush_page_num = 0; + int64_t total_need_flush_rightmost_page_num = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.get_need_flush_page_num(total_need_flush_page_num, total_need_flush_rightmost_page_num)); + ASSERT_EQ(13, total_need_flush_page_num); + ASSERT_EQ(3, total_need_flush_rightmost_page_num); + + STORAGE_LOG(INFO, "=======================tree flush======================="); + STORAGE_LOG(INFO, "=======================first block======================="); + char *block_buff_1 = new char[OB_DEFAULT_MACRO_BLOCK_SIZE]; + ObTmpFileTreeFlushContext flush_context; + ObArray tree_io_array_1; + int64_t write_offset_1 = OB_DEFAULT_MACRO_BLOCK_SIZE - 3 * ObTmpFileGlobal::PAGE_SIZE; //this block can only accommodate 3 pages + ASSERT_EQ(OB_SUCCESS, meta_tree_.flush_meta_pages_for_block(0/*block_index*/, + ObTmpFileTreeEvictType::FULL, + block_buff_1, + write_offset_1, + flush_context, + tree_io_array_1)); + STORAGE_LOG(INFO, "tree_io_array", K(tree_io_array_1)); + ASSERT_EQ(1, tree_io_array_1.count()); + ASSERT_EQ(OB_DEFAULT_MACRO_BLOCK_SIZE, write_offset_1); + + STORAGE_LOG(INFO, "=======================second block======================="); + char *block_buff_2 = new char[OB_DEFAULT_MACRO_BLOCK_SIZE]; + ObArray tree_io_array_2; + int64_t write_offset_2 = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.flush_meta_pages_for_block(1/*block_index*/, + ObTmpFileTreeEvictType::FULL, + block_buff_2, + write_offset_2, + flush_context, + tree_io_array_2)); + STORAGE_LOG(INFO, "tree_io_array", K(tree_io_array_2)); + ASSERT_EQ(3, tree_io_array_2.count()); + ASSERT_EQ(0 + 10 * ObTmpFileGlobal::PAGE_SIZE, write_offset_2); + + total_need_flush_page_num = 0; + total_need_flush_rightmost_page_num = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.get_need_flush_page_num(total_need_flush_page_num, total_need_flush_rightmost_page_num)); + ASSERT_EQ(0, total_need_flush_page_num); + ASSERT_EQ(0, total_need_flush_rightmost_page_num); + ASSERT_EQ(OB_SUCCESS, meta_tree_.update_after_flush(tree_io_array_1)); + ASSERT_EQ(OB_SUCCESS, meta_tree_.update_after_flush(tree_io_array_2)); + + ASSERT_EQ(OB_SUCCESS, meta_tree_.clear(0, 28 * 128 * ObTmpFileGlobal::PAGE_SIZE)); + meta_tree_.reset(); + delete[] block_buff_1; + delete[] block_buff_2; + STORAGE_LOG(INFO, "=======================test_tree_flush_with_multi_io end======================="); +} + +TEST_F(TestSNTmpFileMetaTree, test_tree_major_flush) +{ + STORAGE_LOG(INFO, "=======================test_tree_major_flush begin======================="); + ObTmpWriteBufferPool wbp; + wbp.default_wbp_memory_limit_ = ObTmpWriteBufferPool::WBP_BLOCK_SIZE; //253 pages + ASSERT_EQ(OB_SUCCESS, wbp.init()); + common::ObFIFOAllocator callback_allocator; + ASSERT_EQ(OB_SUCCESS, callback_allocator.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileCallback", ObCtxIds::DEFAULT_CTX_ID))); + ObTmpFileTestMetaTree meta_tree_; + ASSERT_EQ(OB_SUCCESS, meta_tree_.init(1, &wbp, &callback_allocator)); + meta_tree_.set_max_array_item_cnt(2); + meta_tree_.set_max_page_item_cnt(3); + const int64_t item_num = 25; + ObArray data_items; + generate_data_items(item_num, 0/*start virtual page id*/, data_items); + ASSERT_EQ(item_num, data_items.count()); + STORAGE_LOG(INFO, "=======================tree insert======================="); + //insert 25 items (insert a array) + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items)); + ASSERT_EQ(3, meta_tree_.level_page_range_array_.count()); + + STORAGE_LOG(INFO, "=======================tree flush======================="); + char *block_buff = new char[OB_DEFAULT_MACRO_BLOCK_SIZE]; + ObTmpFileTreeFlushContext flush_context; + ObArray tree_io_array; + int64_t write_offset = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.flush_meta_pages_for_block(0/*block_index*/, + ObTmpFileTreeEvictType::MAJOR, + block_buff, + write_offset, + flush_context, + tree_io_array)); + STORAGE_LOG(INFO, "tree_io_array", K(tree_io_array)); + ASSERT_EQ(2, tree_io_array.count()); + ASSERT_EQ(0 + 10 * ObTmpFileGlobal::PAGE_SIZE, write_offset); //flush 10 pages + int64_t total_need_flush_page_num = 0; + int64_t total_need_flush_rightmost_page_num = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.get_need_flush_page_num(total_need_flush_page_num, total_need_flush_rightmost_page_num)); + ASSERT_EQ(3, total_need_flush_page_num); + ASSERT_EQ(3, total_need_flush_rightmost_page_num); + ASSERT_EQ(OB_SUCCESS, meta_tree_.update_after_flush(tree_io_array)); + + ASSERT_EQ(OB_SUCCESS, meta_tree_.clear(0, 25 * 128 * ObTmpFileGlobal::PAGE_SIZE)); + meta_tree_.reset(); + delete[] block_buff; + STORAGE_LOG(INFO, "=======================test_tree_major_flush end======================="); +} + +TEST_F(TestSNTmpFileMetaTree, test_tree_evict) +{ + STORAGE_LOG(INFO, "=======================test_tree_evict begin======================="); + ObTmpWriteBufferPool wbp; + wbp.default_wbp_memory_limit_ = ObTmpWriteBufferPool::WBP_BLOCK_SIZE; //253 pages + ASSERT_EQ(OB_SUCCESS, wbp.init()); + common::ObFIFOAllocator callback_allocator; + ASSERT_EQ(OB_SUCCESS, callback_allocator.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileCallback", ObCtxIds::DEFAULT_CTX_ID))); + ObTmpFileTestMetaTree meta_tree_; + ASSERT_EQ(OB_SUCCESS, meta_tree_.init(1, &wbp, &callback_allocator)); + meta_tree_.set_max_array_item_cnt(2); + meta_tree_.set_max_page_item_cnt(3); + const int64_t item_num = 25; + ObArray data_items; + generate_data_items(item_num, 0/*start virtual page id*/, data_items); + ASSERT_EQ(item_num, data_items.count()); + STORAGE_LOG(INFO, "=======================tree insert======================="); + //insert 25 items (insert a array) + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items)); + ASSERT_EQ(3, meta_tree_.level_page_range_array_.count()); + + STORAGE_LOG(INFO, "=======================first tree flush======================="); + char *block_buff_1 = new char[OB_DEFAULT_MACRO_BLOCK_SIZE]; + ObTmpFileTreeFlushContext flush_context_1; + ObArray tree_io_array_1; + int64_t write_offset_1 = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.flush_meta_pages_for_block(0/*block_index*/, + ObTmpFileTreeEvictType::MAJOR, + block_buff_1, + write_offset_1, + flush_context_1, + tree_io_array_1)); + ASSERT_EQ(2, tree_io_array_1.count()); + ASSERT_EQ(2, tree_io_array_1.at(1).flush_nums_); + ASSERT_EQ(0 + 10 * ObTmpFileGlobal::PAGE_SIZE, write_offset_1); + ASSERT_EQ(OB_SUCCESS, meta_tree_.update_after_flush(tree_io_array_1)); + + STORAGE_LOG(INFO, "=======================first tree evict======================="); + int64_t total_need_evict_page_num = 0; + int64_t total_need_evict_rightmost_page_num = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.get_need_evict_page_num(total_need_evict_page_num, total_need_evict_rightmost_page_num)); + ASSERT_EQ(total_need_evict_page_num, 10); + ASSERT_EQ(total_need_evict_rightmost_page_num, 0); + int64_t actual_evict_page_num_1 = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.evict_meta_pages(13, + ObTmpFileTreeEvictType::FULL, + actual_evict_page_num_1)); + ASSERT_EQ(actual_evict_page_num_1, 10); + + STORAGE_LOG(INFO, "=======================second tree flush======================="); + char *block_buff_2 = new char[OB_DEFAULT_MACRO_BLOCK_SIZE]; + ObTmpFileTreeFlushContext flush_context_2; + ObArray tree_io_array_2; + int64_t write_offset_2 = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.flush_meta_pages_for_block(1/*block_index*/, + ObTmpFileTreeEvictType::FULL, + block_buff_2, + write_offset_2, + flush_context_2, + tree_io_array_2)); + STORAGE_LOG(INFO, "tree_io_array", K(tree_io_array_2)); + ASSERT_EQ(3, tree_io_array_2.count()); + ASSERT_EQ(0 + 3 * ObTmpFileGlobal::PAGE_SIZE, write_offset_2); + ASSERT_EQ(OB_SUCCESS, meta_tree_.update_after_flush(tree_io_array_2)); + + STORAGE_LOG(INFO, "=======================second tree evict======================="); + int64_t actual_evict_page_num_2 = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.evict_meta_pages(3, + ObTmpFileTreeEvictType::FULL, + actual_evict_page_num_2)); + ASSERT_EQ(actual_evict_page_num_2, 3); + for (int64_t i = 0; i < 3; i++) { + ASSERT_EQ(0, meta_tree_.level_page_range_array_.at(i).cached_page_num_); + ASSERT_EQ(ObTmpFileGlobal::INVALID_PAGE_ID, meta_tree_.level_page_range_array_.at(i).start_page_id_); + ASSERT_EQ(ObTmpFileGlobal::INVALID_PAGE_ID, meta_tree_.level_page_range_array_.at(i).end_page_id_); + ASSERT_EQ(ObTmpFileGlobal::INVALID_PAGE_ID, meta_tree_.level_page_range_array_.at(i).flushed_end_page_id_); + } + + meta_tree_.reset(); + delete[] block_buff_1; + delete[] block_buff_2; + STORAGE_LOG(INFO, "=======================test_tree_evict end======================="); +} + +TEST_F(TestSNTmpFileMetaTree, test_tree_clear) +{ + STORAGE_LOG(INFO, "=======================test_tree_clear begin======================="); + ObTmpWriteBufferPool wbp; + wbp.default_wbp_memory_limit_ = ObTmpWriteBufferPool::WBP_BLOCK_SIZE; //253 pages + ASSERT_EQ(OB_SUCCESS, wbp.init()); + common::ObFIFOAllocator callback_allocator; + ASSERT_EQ(OB_SUCCESS, callback_allocator.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileCallback", ObCtxIds::DEFAULT_CTX_ID))); + ObTmpFileTestMetaTree meta_tree_; + ASSERT_EQ(OB_SUCCESS, meta_tree_.init(1, &wbp, &callback_allocator)); + meta_tree_.set_max_array_item_cnt(2); + meta_tree_.set_max_page_item_cnt(3); + const int64_t item_num = 25; + ObArray data_items; + generate_data_items(item_num, 0/*start virtual page id*/, data_items); + ASSERT_EQ(item_num, data_items.count()); + STORAGE_LOG(INFO, "=======================tree insert======================="); + //insert 25 items (insert a array) + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items)); + ASSERT_EQ(3, meta_tree_.level_page_range_array_.count()); + STORAGE_LOG(INFO, "level_page_range_array", K(meta_tree_.level_page_range_array_)); + + STORAGE_LOG(INFO, "=======================tree clear======================="); + ObSharedNothingTmpFileMetaItem origin_root_item = meta_tree_.root_item_; + ASSERT_EQ(OB_SUCCESS, meta_tree_.clear(0, 25 * 128 * ObTmpFileGlobal::PAGE_SIZE)); + ASSERT_EQ(13, meta_tree_.release_pages_.count()); + ASSERT_EQ(origin_root_item.buffer_page_id_, meta_tree_.release_pages_.at(12)); + + meta_tree_.reset(); + STORAGE_LOG(INFO, "=======================test_tree_clear end======================="); +} + +TEST_F(TestSNTmpFileMetaTree, test_tree_truncate) +{ + STORAGE_LOG(INFO, "=======================test_tree_truncate begin======================="); + ObTmpWriteBufferPool wbp; + wbp.default_wbp_memory_limit_ = ObTmpWriteBufferPool::WBP_BLOCK_SIZE; //253 pages + ASSERT_EQ(OB_SUCCESS, wbp.init()); + common::ObFIFOAllocator callback_allocator; + ASSERT_EQ(OB_SUCCESS, callback_allocator.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileCallback", ObCtxIds::DEFAULT_CTX_ID))); + ObTmpFileTestMetaTree meta_tree_; + ASSERT_EQ(OB_SUCCESS, meta_tree_.init(1, &wbp, &callback_allocator)); + meta_tree_.set_max_array_item_cnt(5); + meta_tree_.set_max_page_item_cnt(5); + const int64_t item_num = 75; + ObArray data_items; + ObArray data_items_1; + ObArray data_items_2; + generate_data_items(item_num, 0/*start virtual page id*/, data_items); + ASSERT_EQ(item_num, data_items.count()); + STORAGE_LOG(INFO, "=======================first tree insert======================="); + //insert 5 items (insert a array) + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + for (int64_t i = 0; i < 5; i++) { + ASSERT_EQ(OB_SUCCESS, data_items_1.push_back(data_items.at(i))); + } + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items_1)); + ASSERT_EQ(0, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(5, meta_tree_.data_item_array_.count()); + STORAGE_LOG(INFO, "data_item_array", K(meta_tree_.data_item_array_)); + + STORAGE_LOG(INFO, "=======================first tree truncate======================="); + ASSERT_EQ(OB_SUCCESS, meta_tree_.truncate(0, 3 * 128 * ObTmpFileGlobal::PAGE_SIZE)); + STORAGE_LOG(INFO, "data_item_array", K(meta_tree_.data_item_array_)); + ASSERT_EQ(2, meta_tree_.data_item_array_.count()); + + STORAGE_LOG(INFO, "=======================second tree insert======================="); + //insert 70 items (insert a array) + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + for (int64_t i = 5; i < 75; i++) { + ASSERT_EQ(OB_SUCCESS, data_items_2.push_back(data_items.at(i))); + } + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items_2)); + ASSERT_EQ(3, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(0, meta_tree_.data_item_array_.count()); + STORAGE_LOG(INFO, "level_page_range_array", K(meta_tree_.level_page_range_array_)); + + STORAGE_LOG(INFO, "=======================second tree truncate======================="); + ASSERT_EQ(OB_SUCCESS, meta_tree_.truncate(3 * 128 * ObTmpFileGlobal::PAGE_SIZE, 26 * 128 * ObTmpFileGlobal::PAGE_SIZE)); + STORAGE_LOG(INFO, "level_page_range_array", K(meta_tree_.level_page_range_array_)); + ASSERT_EQ(4, meta_tree_.release_pages_.count()); + + STORAGE_LOG(INFO, "=======================third tree truncate======================="); + ASSERT_EQ(OB_SUCCESS, meta_tree_.truncate(26 * 128 * ObTmpFileGlobal::PAGE_SIZE, 28 * 128 * ObTmpFileGlobal::PAGE_SIZE)); + STORAGE_LOG(INFO, "level_page_range_array", K(meta_tree_.level_page_range_array_)); + ASSERT_EQ(6, meta_tree_.release_pages_.count()); + + STORAGE_LOG(INFO, "=======================first tree read======================="); + ObArray get_data_items; + ASSERT_EQ(OB_SUCCESS, meta_tree_.search_data_items(28 * 128 * ObTmpFileGlobal::PAGE_SIZE, 20 * 128 * ObTmpFileGlobal::PAGE_SIZE, get_data_items)); + ASSERT_EQ(20, get_data_items.count()); + ASSERT_EQ(28 * 128, get_data_items.at(0).virtual_page_id_); + + STORAGE_LOG(INFO, "=======================forth tree truncate======================="); + ASSERT_EQ(OB_SUCCESS, meta_tree_.truncate(28 * 128 * ObTmpFileGlobal::PAGE_SIZE, 73 * 128 * ObTmpFileGlobal::PAGE_SIZE)); + STORAGE_LOG(INFO, "level_page_range_array", K(meta_tree_.level_page_range_array_)); + ASSERT_EQ(16, meta_tree_.release_pages_.count()); + + STORAGE_LOG(INFO, "=======================fifth tree truncate======================="); + ASSERT_EQ(OB_SUCCESS, meta_tree_.truncate(73 * 128 * ObTmpFileGlobal::PAGE_SIZE, 75 * 128 * ObTmpFileGlobal::PAGE_SIZE)); + STORAGE_LOG(INFO, "level_page_range_array", K(meta_tree_.level_page_range_array_)); + ASSERT_EQ(19, meta_tree_.release_pages_.count()); + + ASSERT_EQ(OB_SUCCESS, meta_tree_.clear(75 * 128 * ObTmpFileGlobal::PAGE_SIZE, 75 * 128 * ObTmpFileGlobal::PAGE_SIZE)); + meta_tree_.reset(); + STORAGE_LOG(INFO, "=======================test_tree_truncate end======================="); +} + +TEST_F(TestSNTmpFileMetaTree, test_tree_truncate_with_unfilled_page) +{ + STORAGE_LOG(INFO, "=======================test_tree_truncate_with_unfilled_page begin======================="); + ObTmpWriteBufferPool wbp; + wbp.default_wbp_memory_limit_ = ObTmpWriteBufferPool::WBP_BLOCK_SIZE; //253 pages + ASSERT_EQ(OB_SUCCESS, wbp.init()); + common::ObFIFOAllocator callback_allocator; + ASSERT_EQ(OB_SUCCESS, callback_allocator.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileCallback", ObCtxIds::DEFAULT_CTX_ID))); + ObTmpFileTestMetaTree meta_tree_; + ASSERT_EQ(OB_SUCCESS, meta_tree_.init(1, &wbp, &callback_allocator)); + meta_tree_.set_max_array_item_cnt(5); + meta_tree_.set_max_page_item_cnt(5); + const int64_t item_num = 75; + ObArray data_items; + ObArray data_items_1; + ObArray data_items_2; + generate_data_items(item_num, 0/*start virtual page id*/, data_items); + ASSERT_EQ(item_num, data_items.count()); + STORAGE_LOG(INFO, "=======================first tree insert======================="); + //insert 25 items (insert a array) + //each data item contains 128 pages, we assume that the last page of the last data item is an unfilled page. + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + for (int64_t i = 0; i < 25; i++) { + ASSERT_EQ(OB_SUCCESS, data_items_1.push_back(data_items.at(i))); + } + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items_1)); + ASSERT_EQ(2, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(0, meta_tree_.data_item_array_.count()); + STORAGE_LOG(INFO, "level_page_range_array", K(meta_tree_.level_page_range_array_)); + + STORAGE_LOG(INFO, "=======================first tree truncate======================="); + ASSERT_EQ(OB_SUCCESS, meta_tree_.truncate(0, (24 * 128 + 1 * 127.5) * ObTmpFileGlobal::PAGE_SIZE)); + ASSERT_EQ(2, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(24 * 128 * ObTmpFileGlobal::PAGE_SIZE, meta_tree_.released_offset_); + ASSERT_EQ(4, meta_tree_.release_pages_.count()); + STORAGE_LOG(INFO, "level_page_range_array", K(meta_tree_.level_page_range_array_)); + + STORAGE_LOG(INFO, "=======================load unfilled page to write cache======================="); + ObSharedNothingTmpFileDataItem last_data_item; + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_write_tail(last_data_item)); + ASSERT_EQ(OB_SUCCESS, meta_tree_.finish_write_tail(last_data_item, true/*release_tail_in_disk*/)); + + //then, we write some data in write buffer + + STORAGE_LOG(INFO, "=======================second tree truncate======================="); + ASSERT_EQ(OB_SUCCESS, meta_tree_.truncate((24 * 128 + 1 * 127.5) * ObTmpFileGlobal::PAGE_SIZE, (24 * 128 + 1 * 127.8) * ObTmpFileGlobal::PAGE_SIZE)); + ASSERT_EQ(0, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ((24 * 128 + 1 *127) * ObTmpFileGlobal::PAGE_SIZE, meta_tree_.released_offset_); + ASSERT_EQ(6, meta_tree_.release_pages_.count()); + + STORAGE_LOG(INFO, "=======================third tree truncate======================="); + ASSERT_EQ(OB_SUCCESS, meta_tree_.truncate((24 * 128 + 1 * 127.8) * ObTmpFileGlobal::PAGE_SIZE, 28 * 128 * ObTmpFileGlobal::PAGE_SIZE)); + ASSERT_EQ(0, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(0, meta_tree_.data_item_array_.count()); + ASSERT_EQ((24 * 128 + 1 *127) * ObTmpFileGlobal::PAGE_SIZE, meta_tree_.released_offset_); + ASSERT_EQ(6, meta_tree_.release_pages_.count()); + + + STORAGE_LOG(INFO, "=======================second tree insert======================="); + //insert 47 items (insert a array) + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + for (int64_t i = 28; i < 75; i++) { + ASSERT_EQ(OB_SUCCESS, data_items_2.push_back(data_items.at(i))); + } + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items_2)); + ASSERT_EQ(3, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(10, meta_tree_.level_page_range_array_[0].evicted_page_num_ + meta_tree_.level_page_range_array_[0].cached_page_num_); + ASSERT_EQ(2, meta_tree_.level_page_range_array_[1].evicted_page_num_ + meta_tree_.level_page_range_array_[1].cached_page_num_); + ASSERT_EQ(1, meta_tree_.level_page_range_array_[2].evicted_page_num_ + meta_tree_.level_page_range_array_[2].cached_page_num_); + + STORAGE_LOG(INFO, "=======================forth tree truncate======================="); + ASSERT_EQ(OB_SUCCESS, meta_tree_.truncate(28 * 128 * ObTmpFileGlobal::PAGE_SIZE, 75 * 128 * ObTmpFileGlobal::PAGE_SIZE)); + STORAGE_LOG(INFO, "level_page_range_array", K(meta_tree_.level_page_range_array_)); + ASSERT_EQ(6 + 13, meta_tree_.release_pages_.count()); + ASSERT_EQ(0, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(false, meta_tree_.root_item_.is_valid()); + ASSERT_EQ(75 * 128 * ObTmpFileGlobal::PAGE_SIZE, meta_tree_.released_offset_); + + ASSERT_EQ(OB_SUCCESS, meta_tree_.clear(75 * 128 * ObTmpFileGlobal::PAGE_SIZE, 80 * 128 * ObTmpFileGlobal::PAGE_SIZE)); + meta_tree_.reset(); + STORAGE_LOG(INFO, "=======================test_tree_truncate_with_unfilled_page end======================="); +} + +TEST_F(TestSNTmpFileMetaTree, test_tree_truncate_with_data_item_remove) +{ + STORAGE_LOG(INFO, "=======================test_tree_truncate_with_data_item_remove begin======================="); + ObTmpWriteBufferPool wbp; + wbp.default_wbp_memory_limit_ = ObTmpWriteBufferPool::WBP_BLOCK_SIZE; //253 pages + ASSERT_EQ(OB_SUCCESS, wbp.init()); + common::ObFIFOAllocator callback_allocator; + ASSERT_EQ(OB_SUCCESS, callback_allocator.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileCallback", ObCtxIds::DEFAULT_CTX_ID))); + ObTmpFileTestMetaTree meta_tree_; + ASSERT_EQ(OB_SUCCESS, meta_tree_.init(1, &wbp, &callback_allocator)); + meta_tree_.set_max_array_item_cnt(5); + meta_tree_.set_max_page_item_cnt(5); + int64_t item_num = 9; + ObArray data_items; + ObArray data_items_1; + ObArray data_items_2; + generate_data_items(item_num, 0/*start virtual page id*/, data_items); + //generate last data item, we assume that the page of the last data item is an unfilled page. + ObSharedNothingTmpFileDataItem last_data_item; + last_data_item.block_index_ = item_num; + last_data_item.physical_page_id_ = 0; + last_data_item.physical_page_num_ = 1; + last_data_item.virtual_page_id_ = item_num * 128; + ASSERT_EQ(OB_SUCCESS, data_items.push_back(last_data_item)); + item_num++; + ASSERT_EQ(item_num, data_items.count()); + STORAGE_LOG(INFO, "=======================first tree insert======================="); + //insert 10 items (insert a array) + // last data item has only one page, and the page is an unfilled page. + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + for (int64_t i = 0; i < 10; i++) { + ASSERT_EQ(OB_SUCCESS, data_items_1.push_back(data_items.at(i))); + } + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items_1)); + ASSERT_EQ(2, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(0, meta_tree_.data_item_array_.count()); + meta_tree_.print_meta_tree_total_info(); + + STORAGE_LOG(INFO, "=======================first tree truncate======================="); + ASSERT_EQ(OB_SUCCESS, meta_tree_.truncate(0, (9 * 128 + 1 * 0.5) * ObTmpFileGlobal::PAGE_SIZE)); + ASSERT_EQ(2, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(9 * 128 * ObTmpFileGlobal::PAGE_SIZE, meta_tree_.released_offset_); + ASSERT_EQ(1, meta_tree_.release_pages_.count()); + STORAGE_LOG(INFO, "level_page_range_array", K(meta_tree_.level_page_range_array_)); + + STORAGE_LOG(INFO, "=======================load unfilled page to write cache======================="); + last_data_item.reset(); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_write_tail(last_data_item)); + ASSERT_EQ(OB_SUCCESS, meta_tree_.finish_write_tail(last_data_item, true/*release_tail_in_disk*/)); + + //then, we write some data in write buffer + + STORAGE_LOG(INFO, "=======================second tree truncate======================="); + ASSERT_EQ(OB_SUCCESS, meta_tree_.truncate((9 * 128 + 1 * 0.5) * ObTmpFileGlobal::PAGE_SIZE, (9 * 128 + 1 * 0.8) * ObTmpFileGlobal::PAGE_SIZE)); + ASSERT_EQ(0, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(9 * 128 * ObTmpFileGlobal::PAGE_SIZE, meta_tree_.released_offset_); + ASSERT_EQ(3, meta_tree_.release_pages_.count()); + ASSERT_EQ(false, meta_tree_.root_item_.is_valid()); + + STORAGE_LOG(INFO, "=======================second tree insert======================="); + item_num = 10; + generate_data_items(item_num, 9 * 128, data_items_2); + //generate last data item, we assume that the page of the last data item is an unfilled page. + last_data_item.reset(); + last_data_item.block_index_ = 30; + last_data_item.physical_page_id_ = 0; + last_data_item.physical_page_num_ = 1; + last_data_item.virtual_page_id_ = (9 + 10) * 128; + ASSERT_EQ(OB_SUCCESS, data_items_2.push_back(last_data_item)); + item_num++; + ASSERT_EQ(item_num, data_items_2.count()); + //insert 11 items (insert a array) + // last data item has only one page, and the page is an unfilled page. + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items_2)); + ASSERT_EQ(2, meta_tree_.level_page_range_array_.count()); + // The third leaf page has only one data item (with an unfilled page) + ASSERT_EQ(3, meta_tree_.level_page_range_array_[0].evicted_page_num_ + meta_tree_.level_page_range_array_[0].cached_page_num_); + ASSERT_EQ(0, meta_tree_.data_item_array_.count()); + meta_tree_.print_meta_tree_total_info(); + + STORAGE_LOG(INFO, "=======================third tree truncate======================="); + ASSERT_EQ(OB_SUCCESS, meta_tree_.truncate((9 * 128 + 1 * 0.8) * ObTmpFileGlobal::PAGE_SIZE, (9 * 128 + 10 * 128 + 1 * 0.5) * ObTmpFileGlobal::PAGE_SIZE)); + ASSERT_EQ(2, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ((9 * 128 + 10 * 128) * ObTmpFileGlobal::PAGE_SIZE, meta_tree_.released_offset_); + ASSERT_EQ(3 + 2, meta_tree_.release_pages_.count()); + meta_tree_.print_meta_tree_total_info(); + + STORAGE_LOG(INFO, "=======================load unfilled page to write cache======================="); + last_data_item.reset(); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_write_tail(last_data_item)); + ASSERT_EQ(OB_SUCCESS, meta_tree_.finish_write_tail(last_data_item, true/*release_tail_in_disk*/)); + + STORAGE_LOG(INFO, "=======================forth tree truncate======================="); + ASSERT_EQ(OB_SUCCESS, meta_tree_.truncate((9 * 128 + 10 * 128 + 1 * 0.5) * ObTmpFileGlobal::PAGE_SIZE, (9 * 128 + 10 * 128 + 1 * 0.6) * ObTmpFileGlobal::PAGE_SIZE)); + ASSERT_EQ(0, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ((9 * 128 + 10 * 128) * ObTmpFileGlobal::PAGE_SIZE, meta_tree_.released_offset_); + ASSERT_EQ(3 + 2 + 2, meta_tree_.release_pages_.count()); + ASSERT_EQ(false, meta_tree_.root_item_.is_valid()); + + ASSERT_EQ(OB_SUCCESS, meta_tree_.clear((9 * 128 + 10 * 128 + 1 * 0.6) * ObTmpFileGlobal::PAGE_SIZE, 30 * 128 * ObTmpFileGlobal::PAGE_SIZE)); + meta_tree_.reset(); + STORAGE_LOG(INFO, "=======================test_tree_truncate_with_data_item_remove end======================="); +} + +TEST_F(TestSNTmpFileMetaTree, test_tree_flush_with_truncate_occurs_between_buf_generation_opers) +{ + STORAGE_LOG(INFO, "====test_tree_flush_with_truncate_occurs_between_buf_generation_opers begin=="); + ObTmpWriteBufferPool wbp; + wbp.default_wbp_memory_limit_ = ObTmpWriteBufferPool::WBP_BLOCK_SIZE; //253 pages + ASSERT_EQ(OB_SUCCESS, wbp.init()); + common::ObFIFOAllocator callback_allocator; + ASSERT_EQ(OB_SUCCESS, callback_allocator.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileCallback", ObCtxIds::DEFAULT_CTX_ID))); + ObTmpFileTestMetaTree meta_tree_; + ASSERT_EQ(OB_SUCCESS, meta_tree_.init(1, &wbp, &callback_allocator)); + meta_tree_.set_max_array_item_cnt(5); + meta_tree_.set_max_page_item_cnt(5); + int64_t item_num = 19; + ObArray data_items; + generate_data_items(item_num, 0/*start virtual page id*/, data_items); + //generate last data item, we assume that the page of the last data item is an unfilled page. + ObSharedNothingTmpFileDataItem last_data_item; + last_data_item.block_index_ = item_num; + last_data_item.physical_page_id_ = 0; + last_data_item.physical_page_num_ = 1; + last_data_item.virtual_page_id_ = item_num * 128; + ASSERT_EQ(OB_SUCCESS, data_items.push_back(last_data_item)); + item_num++; + ASSERT_EQ(item_num, data_items.count()); + STORAGE_LOG(INFO, "=======================first tree insert======================="); + //insert 10 items (insert a array) + // last data item has only one page, and the page is an unfilled page. + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items)); + ASSERT_EQ(2, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(0, meta_tree_.data_item_array_.count()); + STORAGE_LOG(INFO, "level_page_range_array", K(meta_tree_.level_page_range_array_)); + + STORAGE_LOG(INFO, "=======================check tree flush pages======================="); + int64_t total_need_flush_page_num = 0; + int64_t total_need_flush_rightmost_page_num = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.get_need_flush_page_num(total_need_flush_page_num, total_need_flush_rightmost_page_num)); + ASSERT_EQ(5, total_need_flush_page_num); + ASSERT_EQ(2, total_need_flush_rightmost_page_num); + + STORAGE_LOG(INFO, "=======================first round tree flush ============================"); + ObTmpFileTreeFlushContext flush_context_first; + + char *block_buff_1 = new char[OB_DEFAULT_MACRO_BLOCK_SIZE]; + ObArray tree_io_array_1; + int64_t write_offset_1 = OB_DEFAULT_MACRO_BLOCK_SIZE - ObTmpFileGlobal::PAGE_SIZE; + + ASSERT_EQ(OB_SUCCESS, meta_tree_.flush_meta_pages_for_block(0/*block_index*/, + ObTmpFileTreeEvictType::FULL, + block_buff_1, + write_offset_1, + flush_context_first, + tree_io_array_1)); + STORAGE_LOG(INFO, "tree_io_array_1", K(tree_io_array_1)); + + ASSERT_EQ(1, tree_io_array_1.count()); + ASSERT_EQ(OB_DEFAULT_MACRO_BLOCK_SIZE, write_offset_1); + + int64_t truncate_offset = 5 * 128 * ObTmpFileGlobal::PAGE_SIZE; //truncate one meta page + ASSERT_EQ(OB_SUCCESS, meta_tree_.truncate(0, truncate_offset)); + ASSERT_EQ(1, meta_tree_.release_pages_.count()); + + char *block_buff_2 = new char[OB_DEFAULT_MACRO_BLOCK_SIZE]; + int64_t write_offset_2 = OB_DEFAULT_MACRO_BLOCK_SIZE - 4 * ObTmpFileGlobal::PAGE_SIZE; + ObArray tree_io_array_2; + + ASSERT_EQ(OB_SUCCESS, meta_tree_.flush_meta_pages_for_block(1/*block_index*/, + ObTmpFileTreeEvictType::FULL, + block_buff_2, + write_offset_2, + flush_context_first, + tree_io_array_2)); + STORAGE_LOG(INFO, "tree_io_array_2", K(tree_io_array_2)); + + ASSERT_EQ(0, tree_io_array_2.count()); + ASSERT_EQ(true, flush_context_first.is_meta_reach_end_); + + total_need_flush_page_num = 0; + total_need_flush_rightmost_page_num = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.get_need_flush_page_num(total_need_flush_page_num, total_need_flush_rightmost_page_num)); + ASSERT_EQ(4, total_need_flush_page_num); + ASSERT_EQ(2, total_need_flush_rightmost_page_num); + + ASSERT_EQ(OB_SUCCESS, meta_tree_.update_after_flush(tree_io_array_1)); + for (int64_t i = 0; i < tree_io_array_1.count(); i++) { + ObTmpFileTreeIOInfo& tree_io = tree_io_array_1.at(i); + ASSERT_EQ(ObTmpFileGlobal::INVALID_PAGE_ID, meta_tree_.level_page_range_array_.at(tree_io.page_level_).flushed_end_page_id_); + } + + STORAGE_LOG(INFO, "=======================second round tree flush ============================"); + ObTmpFileTreeFlushContext flush_context_second; + + char *block_buff_3 = new char[OB_DEFAULT_MACRO_BLOCK_SIZE]; + int64_t write_offset_3 = OB_DEFAULT_MACRO_BLOCK_SIZE - 4 * ObTmpFileGlobal::PAGE_SIZE; + ObArray tree_io_array_3; + + ASSERT_EQ(OB_SUCCESS, meta_tree_.flush_meta_pages_for_block(2/*block_index*/, + ObTmpFileTreeEvictType::FULL, + block_buff_3, + write_offset_3, + flush_context_second, + tree_io_array_3)); + STORAGE_LOG(INFO, "tree_io_array_3", K(tree_io_array_3)); + + ASSERT_EQ(2, tree_io_array_3.count()); + ASSERT_EQ(true, flush_context_second.is_meta_reach_end_); + + total_need_flush_page_num = 0; + total_need_flush_rightmost_page_num = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.get_need_flush_page_num(total_need_flush_page_num, total_need_flush_rightmost_page_num)); + ASSERT_EQ(0, total_need_flush_page_num); + ASSERT_EQ(0, total_need_flush_rightmost_page_num); + + ASSERT_EQ(OB_SUCCESS, meta_tree_.update_after_flush(tree_io_array_3)); + for (int64_t i = 0; i < tree_io_array_3.count(); i++) { + ObTmpFileTreeIOInfo& tree_io = tree_io_array_3.at(i); + ASSERT_EQ(tree_io.flush_end_page_id_, meta_tree_.level_page_range_array_.at(tree_io.page_level_).flushed_end_page_id_); + } + + ASSERT_EQ(OB_SUCCESS, meta_tree_.clear(truncate_offset, 30 * 128 * ObTmpFileGlobal::PAGE_SIZE)); + delete[] block_buff_1; + delete[] block_buff_2; + delete[] block_buff_3; + meta_tree_.reset(); + STORAGE_LOG(INFO, "====test_tree_flush_with_truncate_occurs_between_buf_generation_opers end=="); +} + +TEST_F(TestSNTmpFileMetaTree, test_tree_flush_with_truncate_occurs_between_buf_generation_opers_2) +{ + STORAGE_LOG(INFO, "==test_tree_flush_with_truncate_occurs_between_buf_generation_opers_2 begin==="); + ObTmpWriteBufferPool wbp; + wbp.default_wbp_memory_limit_ = ObTmpWriteBufferPool::WBP_BLOCK_SIZE; //253 pages + ASSERT_EQ(OB_SUCCESS, wbp.init()); + common::ObFIFOAllocator callback_allocator; + ASSERT_EQ(OB_SUCCESS, callback_allocator.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileCallback", ObCtxIds::DEFAULT_CTX_ID))); + ObTmpFileTestMetaTree meta_tree_; + ASSERT_EQ(OB_SUCCESS, meta_tree_.init(1, &wbp, &callback_allocator)); + meta_tree_.set_max_array_item_cnt(5); + meta_tree_.set_max_page_item_cnt(5); + int64_t item_num = 19; + ObArray data_items; + generate_data_items(item_num, 0/*start virtual page id*/, data_items); + //generate last data item, we assume that the page of the last data item is an unfilled page. + ObSharedNothingTmpFileDataItem last_data_item; + last_data_item.block_index_ = item_num; + last_data_item.physical_page_id_ = 0; + last_data_item.physical_page_num_ = 1; + last_data_item.virtual_page_id_ = item_num * 128; + ASSERT_EQ(OB_SUCCESS, data_items.push_back(last_data_item)); + item_num++; + ASSERT_EQ(item_num, data_items.count()); + STORAGE_LOG(INFO, "=======================first tree insert======================="); + //insert 10 items (insert a array) + // last data item has only one page, and the page is an unfilled page. + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items)); + ASSERT_EQ(2, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(0, meta_tree_.data_item_array_.count()); + STORAGE_LOG(INFO, "level_page_range_array", K(meta_tree_.level_page_range_array_)); + + STORAGE_LOG(INFO, "=======================check tree flush pages======================="); + int64_t total_need_flush_page_num = 0; + int64_t total_need_flush_rightmost_page_num = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.get_need_flush_page_num(total_need_flush_page_num, total_need_flush_rightmost_page_num)); + ASSERT_EQ(5, total_need_flush_page_num); + ASSERT_EQ(2, total_need_flush_rightmost_page_num); + + STORAGE_LOG(INFO, "=======================first tree flush ============================"); + ObTmpFileTreeFlushContext flush_context_first; + + char *block_buff_1 = new char[OB_DEFAULT_MACRO_BLOCK_SIZE]; + ObArray tree_io_array_1; + int64_t write_offset_1 = OB_DEFAULT_MACRO_BLOCK_SIZE - 3 * ObTmpFileGlobal::PAGE_SIZE; + + ASSERT_EQ(OB_SUCCESS, meta_tree_.flush_meta_pages_for_block(0/*block_index*/, + ObTmpFileTreeEvictType::FULL, + block_buff_1, + write_offset_1, + flush_context_first, + tree_io_array_1)); + STORAGE_LOG(INFO, "tree_io_array_1", K(tree_io_array_1)); + + ASSERT_EQ(1, tree_io_array_1.count()); + ASSERT_EQ(OB_DEFAULT_MACRO_BLOCK_SIZE, write_offset_1); + + int64_t truncate_offset = 5 * 1 * 128 * ObTmpFileGlobal::PAGE_SIZE; + ASSERT_EQ(OB_SUCCESS, meta_tree_.truncate(0, truncate_offset)); + + char *block_buff_2 = new char[OB_DEFAULT_MACRO_BLOCK_SIZE]; + int64_t write_offset_2 = OB_DEFAULT_MACRO_BLOCK_SIZE - 2 * ObTmpFileGlobal::PAGE_SIZE; + ObArray tree_io_array_2; + + ASSERT_EQ(OB_SUCCESS, meta_tree_.flush_meta_pages_for_block(1/*block_index*/, + ObTmpFileTreeEvictType::FULL, + block_buff_2, + write_offset_2, + flush_context_first, + tree_io_array_2)); + STORAGE_LOG(INFO, "tree_io_array_2", K(tree_io_array_2)); + ASSERT_EQ(2, tree_io_array_2.count()); + ASSERT_EQ(OB_DEFAULT_MACRO_BLOCK_SIZE, write_offset_2); + ASSERT_EQ(true, flush_context_first.is_meta_reach_end_); + + total_need_flush_page_num = 0; + total_need_flush_rightmost_page_num = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.get_need_flush_page_num(total_need_flush_page_num, total_need_flush_rightmost_page_num)); + ASSERT_EQ(0, total_need_flush_page_num); + ASSERT_EQ(0, total_need_flush_rightmost_page_num); + + ASSERT_EQ(OB_SUCCESS, meta_tree_.update_after_flush(tree_io_array_1)); + ASSERT_EQ(OB_SUCCESS, meta_tree_.update_after_flush(tree_io_array_2)); + + for (int64_t i = 0; i < tree_io_array_2.count(); i++) { + ObTmpFileTreeIOInfo& tree_io = tree_io_array_2.at(i); + ASSERT_EQ(tree_io.flush_end_page_id_, meta_tree_.level_page_range_array_.at(tree_io.page_level_).flushed_end_page_id_); + } + + delete[] block_buff_1; + delete[] block_buff_2; + meta_tree_.reset(); + STORAGE_LOG(INFO, "==test_tree_flush_with_truncate_occurs_between_buf_generation_opers_2 end==="); +} + +TEST_F(TestSNTmpFileMetaTree, test_tree_flush_with_truncate_occurs_between_buf_generation_opers_3) +{ + STORAGE_LOG(INFO, "====test_tree_flush_with_truncate_occurs_between_buf_generation_opers_3 begin=="); + ObTmpWriteBufferPool wbp; + wbp.default_wbp_memory_limit_ = ObTmpWriteBufferPool::WBP_BLOCK_SIZE; //253 pages + ASSERT_EQ(OB_SUCCESS, wbp.init()); + common::ObFIFOAllocator callback_allocator; + ASSERT_EQ(OB_SUCCESS, callback_allocator.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileCallback", ObCtxIds::DEFAULT_CTX_ID))); + ObTmpFileTestMetaTree meta_tree_; + ASSERT_EQ(OB_SUCCESS, meta_tree_.init(1, &wbp, &callback_allocator)); + meta_tree_.set_max_array_item_cnt(5); + meta_tree_.set_max_page_item_cnt(5); + int64_t item_num = 19; + ObArray data_items; + generate_data_items(item_num, 0/*start virtual page id*/, data_items); + //generate last data item, we assume that the page of the last data item is an unfilled page. + ObSharedNothingTmpFileDataItem last_data_item; + last_data_item.block_index_ = item_num; + last_data_item.physical_page_id_ = 0; + last_data_item.physical_page_num_ = 1; + last_data_item.virtual_page_id_ = item_num * 128; + ASSERT_EQ(OB_SUCCESS, data_items.push_back(last_data_item)); + item_num++; + ASSERT_EQ(item_num, data_items.count()); + STORAGE_LOG(INFO, "=======================first tree insert======================="); + //insert 10 items (insert a array) + // last data item has only one page, and the page is an unfilled page. + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items)); + ASSERT_EQ(2, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(0, meta_tree_.data_item_array_.count()); + STORAGE_LOG(INFO, "level_page_range_array", K(meta_tree_.level_page_range_array_)); + + STORAGE_LOG(INFO, "=======================check tree flush pages======================="); + int64_t total_need_flush_page_num = 0; + int64_t total_need_flush_rightmost_page_num = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.get_need_flush_page_num(total_need_flush_page_num, total_need_flush_rightmost_page_num)); + ASSERT_EQ(5, total_need_flush_page_num); + ASSERT_EQ(2, total_need_flush_rightmost_page_num); + + STORAGE_LOG(INFO, "=======================first tree flush ============================"); + ObTmpFileTreeFlushContext flush_context_first; + + char *block_buff_1 = new char[OB_DEFAULT_MACRO_BLOCK_SIZE]; + ObArray tree_io_array_1; + int64_t write_offset_1 = OB_DEFAULT_MACRO_BLOCK_SIZE - ObTmpFileGlobal::PAGE_SIZE; + + ASSERT_EQ(OB_SUCCESS, meta_tree_.flush_meta_pages_for_block(0/*block_index*/, + ObTmpFileTreeEvictType::FULL, + block_buff_1, + write_offset_1, + flush_context_first, + tree_io_array_1)); + STORAGE_LOG(INFO, "tree_io_array_1", K(tree_io_array_1)); + + ASSERT_EQ(1, tree_io_array_1.count()); + ASSERT_EQ(OB_DEFAULT_MACRO_BLOCK_SIZE, write_offset_1); + + int64_t truncate_offset = 5 * 4 * 128 * ObTmpFileGlobal::PAGE_SIZE; + ASSERT_EQ(OB_SUCCESS, meta_tree_.truncate(0, truncate_offset)); + + char *block_buff_2 = new char[OB_DEFAULT_MACRO_BLOCK_SIZE]; + int64_t write_offset_2 = OB_DEFAULT_MACRO_BLOCK_SIZE - 4 * ObTmpFileGlobal::PAGE_SIZE; + ObArray tree_io_array_2; + + ASSERT_EQ(OB_SUCCESS, meta_tree_.flush_meta_pages_for_block(1/*block_index*/, + ObTmpFileTreeEvictType::FULL, + block_buff_2, + write_offset_2, + flush_context_first, + tree_io_array_2)); + STORAGE_LOG(INFO, "tree_io_array_2", K(tree_io_array_2)); + + ASSERT_EQ(0, tree_io_array_2.count()); + ASSERT_EQ(true, flush_context_first.is_meta_reach_end_); + ASSERT_NE(flush_context_first.tree_epoch_, meta_tree_.tree_epoch_); + + total_need_flush_page_num = 0; + total_need_flush_rightmost_page_num = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.get_need_flush_page_num(total_need_flush_page_num, total_need_flush_rightmost_page_num)); + ASSERT_EQ(0, total_need_flush_page_num); + ASSERT_EQ(0, total_need_flush_rightmost_page_num); + + ASSERT_EQ(OB_SUCCESS, meta_tree_.update_after_flush(tree_io_array_1)); + for (int64_t i = 0; i < tree_io_array_1.count(); i++) { + ObTmpFileTreeIOInfo& tree_io = tree_io_array_1.at(i); + ASSERT_NE(tree_io.tree_epoch_, meta_tree_.tree_epoch_); + } + + delete[] block_buff_1; + delete[] block_buff_2; + meta_tree_.reset(); + STORAGE_LOG(INFO, "====test_tree_flush_with_truncate_occurs_between_buf_generation_opers_3 end=="); +} + +void TestSNTmpFileMetaTree::test_tree_flush_with_truncate_occurs_before_update_meta( + int64_t truncate_offset, bool insert_after_truncate) +{ + STORAGE_LOG(INFO, "===============test_tree_flush_with_truncate_occurs_before_update_meta begin ===="); + ObTmpWriteBufferPool wbp; + wbp.default_wbp_memory_limit_ = ObTmpWriteBufferPool::WBP_BLOCK_SIZE; //253 pages + ASSERT_EQ(OB_SUCCESS, wbp.init()); + common::ObFIFOAllocator callback_allocator; + ASSERT_EQ(OB_SUCCESS, callback_allocator.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileCallback", ObCtxIds::DEFAULT_CTX_ID))); + ObTmpFileTestMetaTree meta_tree_; + ASSERT_EQ(OB_SUCCESS, meta_tree_.init(1, &wbp, &callback_allocator)); + meta_tree_.set_max_array_item_cnt(5); + meta_tree_.set_max_page_item_cnt(5); + int64_t item_num = 70; + ObArray data_items; + ObArray data_items_1; + generate_data_items(item_num, 0/*start virtual page id*/, data_items); + //generate last data item, we assume the last data item only contain one page. + ObSharedNothingTmpFileDataItem last_data_item; + last_data_item.block_index_ = item_num; + last_data_item.physical_page_id_ = 0; + last_data_item.physical_page_num_ = 1; + last_data_item.virtual_page_id_ = item_num * 128; + ASSERT_EQ(OB_SUCCESS, data_items.push_back(last_data_item)); + item_num++; + ASSERT_EQ(item_num, data_items.count()); + STORAGE_LOG(INFO, "=======================first tree insert======================="); + //insert 10 items (insert a array) + // last data item has only one page, and the page is an unfilled page. + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items)); + ASSERT_EQ(3, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(0, meta_tree_.data_item_array_.count()); + STORAGE_LOG(INFO, "level_page_range_array", K(meta_tree_.level_page_range_array_)); + + STORAGE_LOG(INFO, "=======================check tree flush pages======================="); + int64_t total_need_flush_page_num = 0; + int64_t total_need_flush_rightmost_page_num = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.get_need_flush_page_num(total_need_flush_page_num, total_need_flush_rightmost_page_num)); + ASSERT_EQ(15 + 3 + 1, total_need_flush_page_num); + ASSERT_EQ(3, total_need_flush_rightmost_page_num); + + STORAGE_LOG(INFO, "======================= tree flush ============================"); + ObTmpFileTreeFlushContext flush_context; + + char *block_buff_1 = new char[OB_DEFAULT_MACRO_BLOCK_SIZE]; + ObArray tree_io_array_1; + int64_t write_offset_1 = OB_DEFAULT_MACRO_BLOCK_SIZE - ObTmpFileGlobal::PAGE_SIZE; + + ASSERT_EQ(OB_SUCCESS, meta_tree_.flush_meta_pages_for_block(0/*block_index*/, + ObTmpFileTreeEvictType::FULL, + block_buff_1, + write_offset_1, + flush_context, + tree_io_array_1)); + STORAGE_LOG(INFO, "tree_io_array_1", K(tree_io_array_1)); + + ASSERT_EQ(1, tree_io_array_1.count()); + ASSERT_EQ(OB_DEFAULT_MACRO_BLOCK_SIZE, write_offset_1); + + char *block_buff_2 = new char[OB_DEFAULT_MACRO_BLOCK_SIZE]; + int64_t write_offset_2 = OB_DEFAULT_MACRO_BLOCK_SIZE - 18 * ObTmpFileGlobal::PAGE_SIZE; + ObArray tree_io_array_2; + + ASSERT_EQ(OB_SUCCESS, meta_tree_.flush_meta_pages_for_block(1/*block_index*/, + ObTmpFileTreeEvictType::FULL, + block_buff_2, + write_offset_2, + flush_context, + tree_io_array_2)); + STORAGE_LOG(INFO, "tree_io_array_2", K(tree_io_array_2)); + + ASSERT_EQ(3, tree_io_array_2.count()); + ASSERT_EQ(true, flush_context.is_meta_reach_end_); + + total_need_flush_page_num = 0; + total_need_flush_rightmost_page_num = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.get_need_flush_page_num(total_need_flush_page_num, total_need_flush_rightmost_page_num)); + ASSERT_EQ(0, total_need_flush_page_num); + ASSERT_EQ(0, total_need_flush_rightmost_page_num); + + ASSERT_EQ(OB_SUCCESS, meta_tree_.truncate(0, truncate_offset)); + + if (insert_after_truncate) { + STORAGE_LOG(INFO, "======================= second tree insert ======================="); + int64_t item_num_2 = 20; + ObArray data_items_1; + //even if truncate_offset is not an integer multiple of PAGE_SIZE, this result is reasonable. + //the start_virtual_page_id we insert into the tree may be smaller than the actual truncate_offset of the tmp file. + int64_t start_virtual_page_id = + MAX(truncate_offset / ObTmpFileGlobal::PAGE_SIZE, last_data_item.virtual_page_id_ + last_data_item.physical_page_num_); + generate_data_items(item_num_2, start_virtual_page_id, data_items_1); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items_1)); + } + + ASSERT_EQ(OB_SUCCESS, meta_tree_.update_after_flush(tree_io_array_1)); + ASSERT_EQ(OB_SUCCESS, meta_tree_.update_after_flush(tree_io_array_2)); + + if (insert_after_truncate) { + ASSERT_EQ(true, meta_tree_.root_item_.is_valid()); + if (truncate_offset >= (last_data_item.virtual_page_id_ + last_data_item.physical_page_num_) * ObTmpFileGlobal::PAGE_SIZE) { + for (int64_t i = 0; i < tree_io_array_2.count(); i++) { + ASSERT_NE(tree_io_array_2.at(i).tree_epoch_, meta_tree_.tree_epoch_); + } + } else { + for (int64_t i = 0; i < tree_io_array_2.count(); i++) { + ObTmpFileTreeIOInfo& tree_io = tree_io_array_2.at(i); + ASSERT_EQ(tree_io.flush_end_page_id_, meta_tree_.level_page_range_array_.at(tree_io.page_level_).flushed_end_page_id_); + } + } + } else { + if (truncate_offset >= (last_data_item.virtual_page_id_ + last_data_item.physical_page_num_) * ObTmpFileGlobal::PAGE_SIZE) { + ASSERT_EQ(false, meta_tree_.root_item_.is_valid()); + } else { + for (int64_t i = 0; i < tree_io_array_2.count(); i++) { + ObTmpFileTreeIOInfo& tree_io = tree_io_array_2.at(i); + ASSERT_EQ(tree_io.flush_end_page_id_, meta_tree_.level_page_range_array_.at(tree_io.page_level_).flushed_end_page_id_); + } + } + } + + ASSERT_EQ(OB_SUCCESS, meta_tree_.clear(truncate_offset, 100 * 128 * ObTmpFileGlobal::PAGE_SIZE)); + delete[] block_buff_1; + delete[] block_buff_2; + meta_tree_.reset(); + STORAGE_LOG(INFO, "=================test_tree_flush_with_truncate_occurs_before_update_meta end======="); +} + +TEST_F(TestSNTmpFileMetaTree, test_tree_flush_with_truncate_to_somewhere_in_the_middle_before_update_meta) +{ + test_tree_flush_with_truncate_occurs_before_update_meta(5 * 128 * ObTmpFileGlobal::PAGE_SIZE, false); +} + +TEST_F(TestSNTmpFileMetaTree, test_tree_flush_with_truncate_to_end_before_update_meta) +{ + test_tree_flush_with_truncate_occurs_before_update_meta(30 * 128 * ObTmpFileGlobal::PAGE_SIZE, false); +} + +TEST_F(TestSNTmpFileMetaTree, test_tree_flush_with_truncate_to_somewhere_in_the_middle_before_update_meta_2) +{ + test_tree_flush_with_truncate_occurs_before_update_meta(8 * 128 * ObTmpFileGlobal::PAGE_SIZE, true); +} + +TEST_F(TestSNTmpFileMetaTree, test_tree_flush_with_truncate_to_end_before_update_meta_2) +{ + test_tree_flush_with_truncate_occurs_before_update_meta(10 * 128 * ObTmpFileGlobal::PAGE_SIZE, true); +} + +//================More detailed tests of meta tree involve more test points================== + +//=========================================insert============================================ +TEST_F(TestSNTmpFileMetaTree, test_array_insert) +{ + STORAGE_LOG(INFO, "=======================test_array_insert begin======================="); + ObTmpWriteBufferPool wbp; + wbp.default_wbp_memory_limit_ = ObTmpWriteBufferPool::WBP_BLOCK_SIZE; //253 pages + ASSERT_EQ(OB_SUCCESS, wbp.init()); + common::ObFIFOAllocator callback_allocator; + ASSERT_EQ(OB_SUCCESS, callback_allocator.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileCallback", ObCtxIds::DEFAULT_CTX_ID))); + ObTmpFileTestMetaTree meta_tree_; + ASSERT_EQ(OB_SUCCESS, meta_tree_.init(1, &wbp, &callback_allocator)); + meta_tree_.set_max_array_item_cnt(100); + meta_tree_.set_max_page_item_cnt(100); + int64_t item_num = 30; + ObArray data_items; + ObArray data_items_1; + generate_data_items(item_num, 0/*start virtual page id*/, data_items); + ASSERT_EQ(item_num, data_items.count()); + STORAGE_LOG(INFO, "=======================first insert======================="); + //insert 30 items (insert a array) + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items)); + ASSERT_EQ(30, meta_tree_.data_item_array_.count()); + + STORAGE_LOG(INFO, "=======================second insert======================="); + item_num = 71; + generate_data_items(item_num, 30 * 128, data_items_1); + ASSERT_EQ(item_num, data_items_1.count()); + //insert 71 items (insert one by one) + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items_1)); + ASSERT_EQ(0, meta_tree_.data_item_array_.count()); + ASSERT_EQ(2, meta_tree_.level_page_range_array_.count()); + + meta_tree_.reset(); + STORAGE_LOG(INFO, "=======================test_array_insert end======================="); +} + +TEST_F(TestSNTmpFileMetaTree, test_tree_prepare_for_insert) +{ + STORAGE_LOG(INFO, "=================test_tree_prepare_for_insert begin==============="); + ObTmpWriteBufferPool wbp; + wbp.default_wbp_memory_limit_ = ObTmpWriteBufferPool::WBP_BLOCK_SIZE; //253 pages + ASSERT_EQ(OB_SUCCESS, wbp.init()); + common::ObFIFOAllocator callback_allocator; + ASSERT_EQ(OB_SUCCESS, callback_allocator.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileCallback", ObCtxIds::DEFAULT_CTX_ID))); + ObTmpFileTestMetaTree meta_tree_; + ASSERT_EQ(OB_SUCCESS, meta_tree_.init(1, &wbp, &callback_allocator)); + meta_tree_.set_max_array_item_cnt(5); + meta_tree_.set_max_page_item_cnt(5); + int64_t item_num = 9; + ObArray data_items; + generate_data_items(item_num, 0/*start virtual page id*/, data_items); + ASSERT_EQ(item_num, data_items.count()); + STORAGE_LOG(INFO, "=======================first tree insert======================="); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items)); + ASSERT_EQ(2, meta_tree_.level_page_range_array_.count()); + STORAGE_LOG(INFO, "level_page_range_array", K(meta_tree_.level_page_range_array_)); + + STORAGE_LOG(INFO, "=======================first tree flush and evict========================="); + ObTmpFileTreeFlushContext flush_context_first; + + char *block_buff_1 = new char[OB_DEFAULT_MACRO_BLOCK_SIZE]; + ObArray tree_io_array_1; + int64_t write_offset_1 = OB_DEFAULT_MACRO_BLOCK_SIZE - 2 * ObTmpFileGlobal::PAGE_SIZE; + + ASSERT_EQ(OB_SUCCESS, meta_tree_.flush_meta_pages_for_block(0/*block_index*/, + ObTmpFileTreeEvictType::FULL, + block_buff_1, + write_offset_1, + flush_context_first, + tree_io_array_1)); + STORAGE_LOG(INFO, "tree_io_array_1", K(tree_io_array_1)); + + ASSERT_EQ(1, tree_io_array_1.count()); + ASSERT_EQ(OB_DEFAULT_MACRO_BLOCK_SIZE, write_offset_1); + char *rightmost_page_buf = block_buff_1 + OB_DEFAULT_MACRO_BLOCK_SIZE - ObTmpFileGlobal::PAGE_SIZE; + meta_tree_.read_cache_rightmost_pages_.reset(); + ASSERT_EQ(OB_SUCCESS, meta_tree_.read_cache_rightmost_pages_.push_back(std::make_pair(rightmost_page_buf, 0))); + ASSERT_EQ(OB_SUCCESS, meta_tree_.update_after_flush(tree_io_array_1)); + int64_t actual_evict_page_num = -1; + ASSERT_EQ(OB_SUCCESS, meta_tree_.evict_meta_pages(3, ObTmpFileTreeEvictType::FULL, actual_evict_page_num)); + ASSERT_EQ(2, actual_evict_page_num); + ASSERT_EQ(ObTmpFileGlobal::INVALID_PAGE_ID, meta_tree_.level_page_range_array_.at(0).end_page_id_); + + STORAGE_LOG(INFO, "=======================second tree insert======================="); + item_num = 6; + ObArray data_items_1; + generate_data_items(item_num, 9 * 128, data_items_1); + ASSERT_EQ(item_num, data_items_1.count()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(1, meta_tree_.read_cache_rightmost_pages_.at(0).second); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items_1)); + ASSERT_EQ(2, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(2, meta_tree_.level_page_range_array_.at(0).cached_page_num_); + ASSERT_EQ(1, meta_tree_.level_page_range_array_.at(0).evicted_page_num_); + STORAGE_LOG(INFO, "level_page_range_array", K(meta_tree_.level_page_range_array_)); + + STORAGE_LOG(INFO, "=======================second tree flush and evict========================="); + char *block_buff_2 = new char[OB_DEFAULT_MACRO_BLOCK_SIZE]; + int64_t write_offset_2 = OB_DEFAULT_MACRO_BLOCK_SIZE - 3 * ObTmpFileGlobal::PAGE_SIZE; + ObArray tree_io_array_2; + ObTmpFileTreeFlushContext flush_context_second; + + ASSERT_EQ(OB_SUCCESS, meta_tree_.flush_meta_pages_for_block(1/*block_index*/, + ObTmpFileTreeEvictType::FULL, + block_buff_2, + write_offset_2, + flush_context_second, + tree_io_array_2)); + STORAGE_LOG(INFO, "tree_io_array_2", K(tree_io_array_2)); + ASSERT_EQ(2, tree_io_array_2.count()); + ASSERT_EQ(true, flush_context_second.is_meta_reach_end_); + + char *rightmost_page_buf_0 = block_buff_2 + OB_DEFAULT_MACRO_BLOCK_SIZE - 2 * ObTmpFileGlobal::PAGE_SIZE; + char *rightmost_page_buf_1 = block_buff_2 + OB_DEFAULT_MACRO_BLOCK_SIZE - 1 * ObTmpFileGlobal::PAGE_SIZE; + meta_tree_.read_cache_rightmost_pages_.reset(); + ASSERT_EQ(OB_SUCCESS, meta_tree_.read_cache_rightmost_pages_.push_back(std::make_pair(rightmost_page_buf_0, 0))); + ASSERT_EQ(OB_SUCCESS, meta_tree_.read_cache_rightmost_pages_.push_back(std::make_pair(rightmost_page_buf_1, 0))); + + ASSERT_EQ(OB_SUCCESS, meta_tree_.update_after_flush(tree_io_array_2)); + actual_evict_page_num = -1; + ASSERT_EQ(OB_SUCCESS, meta_tree_.evict_meta_pages(3, ObTmpFileTreeEvictType::FULL, actual_evict_page_num)); + ASSERT_EQ(3, actual_evict_page_num); + ASSERT_EQ(ObTmpFileGlobal::INVALID_PAGE_ID, meta_tree_.level_page_range_array_.at(1).end_page_id_); + + STORAGE_LOG(INFO, "=======================third tree insert======================="); + item_num = 20; + ObArray data_items_2; + generate_data_items(item_num, 15 * 128, data_items_2); + ASSERT_EQ(item_num, data_items_2.count()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(1, meta_tree_.read_cache_rightmost_pages_.at(0).second); + ASSERT_EQ(1, meta_tree_.read_cache_rightmost_pages_.at(1).second); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items_2)); + ASSERT_EQ(3, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(5, meta_tree_.level_page_range_array_.at(0).cached_page_num_); + ASSERT_EQ(2, meta_tree_.level_page_range_array_.at(0).evicted_page_num_); + + delete[] block_buff_1; + delete[] block_buff_2; + meta_tree_.reset(); + STORAGE_LOG(INFO, "=================test_tree_prepare_for_insert begin==============="); +} + +TEST_F(TestSNTmpFileMetaTree, test_tree_prepare_for_insert_fail) +{ + STORAGE_LOG(INFO, "=======================test_tree_prepare_for_insert_fail begin======================="); + ObTmpWriteBufferPool wbp; + wbp.default_wbp_memory_limit_ = ObTmpWriteBufferPool::WBP_BLOCK_SIZE; //253 pages + ASSERT_EQ(OB_SUCCESS, wbp.init()); + common::ObFIFOAllocator callback_allocator; + ASSERT_EQ(OB_SUCCESS, callback_allocator.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileCallback", ObCtxIds::DEFAULT_CTX_ID))); + ObTmpFileTestMetaTree meta_tree_; + ASSERT_EQ(OB_SUCCESS, meta_tree_.init(1, &wbp, &callback_allocator)); + meta_tree_.set_max_array_item_cnt(2); + meta_tree_.set_max_page_item_cnt(2); + + STORAGE_LOG(INFO, "=======================first insert======================="); + int64_t item_num = 10; + ObArray data_items; + generate_data_items(item_num, 0/*start virtual page id*/, data_items); + ASSERT_EQ(item_num, data_items.count()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items)); + ASSERT_EQ(4, meta_tree_.level_page_range_array_.count()); + + STORAGE_LOG(INFO, "=======================first tree flush and evict========================="); + ObTmpFileTreeFlushContext flush_context_first; + char *block_buff_1 = new char[OB_DEFAULT_MACRO_BLOCK_SIZE]; + ObArray tree_io_array_1; + int64_t write_offset_1 = 0; + + ASSERT_EQ(OB_SUCCESS, meta_tree_.flush_meta_pages_for_block(0/*block_index*/, + ObTmpFileTreeEvictType::FULL, + block_buff_1, + write_offset_1, + flush_context_first, + tree_io_array_1)); + ASSERT_EQ(4, tree_io_array_1.count()); + ASSERT_EQ(true, flush_context_first.is_meta_reach_end_); + + char *rightmost_page_buf_0 = block_buff_1 + 4 * ObTmpFileGlobal::PAGE_SIZE; + char *rightmost_page_buf_1 = block_buff_1 + 7 * ObTmpFileGlobal::PAGE_SIZE; + char *rightmost_page_buf_2 = block_buff_1 + 9 * ObTmpFileGlobal::PAGE_SIZE; + char *rightmost_page_buf_3 = block_buff_1 + 10 * ObTmpFileGlobal::PAGE_SIZE; + meta_tree_.read_cache_rightmost_pages_.reset(); + ASSERT_EQ(OB_SUCCESS, meta_tree_.read_cache_rightmost_pages_.push_back(std::make_pair(rightmost_page_buf_0, 0))); + ASSERT_EQ(OB_SUCCESS, meta_tree_.read_cache_rightmost_pages_.push_back(std::make_pair(rightmost_page_buf_1, 0))); + ASSERT_EQ(OB_SUCCESS, meta_tree_.read_cache_rightmost_pages_.push_back(std::make_pair(rightmost_page_buf_2, 0))); + ASSERT_EQ(OB_SUCCESS, meta_tree_.read_cache_rightmost_pages_.push_back(std::make_pair(rightmost_page_buf_3, 0))); + + ASSERT_EQ(OB_SUCCESS, meta_tree_.update_after_flush(tree_io_array_1)); + int64_t actual_evict_page_num = -1; + ASSERT_EQ(OB_SUCCESS, meta_tree_.evict_meta_pages(12, ObTmpFileTreeEvictType::FULL, actual_evict_page_num)); + ASSERT_EQ(11, actual_evict_page_num); + ASSERT_EQ(ObTmpFileGlobal::INVALID_PAGE_ID, meta_tree_.level_page_range_array_.at(3).end_page_id_); + + STORAGE_LOG(INFO, "=======================build new meta tree======================="); + //We build a new meta tree to take up most of the write cache memory + ObTmpFileTestMetaTree meta_tree_1_; + ASSERT_EQ(OB_SUCCESS, meta_tree_1_.init(2, &wbp, &callback_allocator)); + meta_tree_1_.set_max_array_item_cnt(2); + meta_tree_1_.set_max_page_item_cnt(2); + + STORAGE_LOG(INFO, "=======================new meta tree insert======================="); + item_num = 252; + ObArray data_items_1; + generate_data_items(item_num, 0/*start virtual page id*/, data_items_1); + ASSERT_EQ(item_num, data_items_1.count()); + ASSERT_EQ(OB_SUCCESS, meta_tree_1_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_1_.insert_items(data_items_1)); + ASSERT_EQ(8, meta_tree_1_.level_page_range_array_.count());//252 pages + + STORAGE_LOG(INFO, "=======================second insert======================="); + ASSERT_EQ(OB_ALLOCATE_TMP_FILE_PAGE_FAILED, meta_tree_.prepare_for_insert_items()); + ASSERT_NE(ObTmpFileGlobal::INVALID_PAGE_ID, meta_tree_.level_page_range_array_.at(3).end_page_id_); + ASSERT_EQ(ObTmpFileGlobal::INVALID_PAGE_ID, meta_tree_.level_page_range_array_.at(2).end_page_id_); + ASSERT_EQ(ObTmpFileGlobal::INVALID_PAGE_ID, meta_tree_.level_page_range_array_.at(1).end_page_id_); + ASSERT_EQ(ObTmpFileGlobal::INVALID_PAGE_ID, meta_tree_.level_page_range_array_.at(0).end_page_id_); + + STORAGE_LOG(INFO, "=======================new meta tree clear======================="); + ASSERT_EQ(OB_SUCCESS, meta_tree_1_.clear(0, 252 * 128 * ObTmpFileGlobal::PAGE_SIZE)); + + STORAGE_LOG(INFO, "=======================third insert======================="); + item_num = 10; + ObArray data_items_2; + generate_data_items(item_num, 10 * 128/*start virtual page id*/, data_items_2); + ASSERT_EQ(item_num, data_items_2.count()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items_2)); + ASSERT_EQ(5, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(6, meta_tree_.level_page_range_array_.at(0).cached_page_num_); + ASSERT_EQ(4, meta_tree_.level_page_range_array_.at(0).evicted_page_num_); + + delete[] block_buff_1; + meta_tree_.reset(); + meta_tree_1_.reset(); + STORAGE_LOG(INFO, "=======================test_tree_prepare_for_insert_fail end======================="); +} + +TEST_F(TestSNTmpFileMetaTree, test_tree_insert_fail_after_array_used) +{ + STORAGE_LOG(INFO, "================test_tree_insert_fail_after_array_used begin================"); + ObTmpWriteBufferPool wbp; + wbp.default_wbp_memory_limit_ = ObTmpWriteBufferPool::WBP_BLOCK_SIZE; //253 pages + ASSERT_EQ(OB_SUCCESS, wbp.init()); + common::ObFIFOAllocator callback_allocator; + ASSERT_EQ(OB_SUCCESS, callback_allocator.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileCallback", ObCtxIds::DEFAULT_CTX_ID))); + ObTmpFileTestMetaTree meta_tree_; + ASSERT_EQ(OB_SUCCESS, meta_tree_.init(1, &wbp, &callback_allocator)); + meta_tree_.set_max_array_item_cnt(3); + meta_tree_.set_max_page_item_cnt(3); + STORAGE_LOG(INFO, "=======================first insert======================="); + int64_t item_num = 2; + ObArray data_items; + generate_data_items(item_num, 0/*start virtual page id*/, data_items); + ASSERT_EQ(item_num, data_items.count()); + //insert 750 items (insert a array) + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items)); + ASSERT_EQ(0, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(2, meta_tree_.data_item_array_.count()); + + STORAGE_LOG(INFO, "=======================second insert======================="); + item_num = 10; + ObArray data_items_1; + generate_data_items(item_num, 1 * 128/*start virtual page id*/, data_items_1); + ASSERT_EQ(item_num, data_items_1.count()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_NE(OB_SUCCESS, meta_tree_.insert_items(data_items_1)); + ASSERT_EQ(2, meta_tree_.data_item_array_.count()); + ASSERT_EQ(0, meta_tree_.level_page_range_array_.count()); + + STORAGE_LOG(INFO, "=======================third insert======================="); + item_num = 600; + ObArray data_items_2; + generate_data_items(item_num, 2 * 128/*start virtual page id*/, data_items_2); + ASSERT_EQ(item_num, data_items_2.count()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_ALLOCATE_TMP_FILE_PAGE_FAILED, meta_tree_.insert_items(data_items_2)); + ASSERT_EQ(2, meta_tree_.data_item_array_.count()); + ASSERT_EQ(0, meta_tree_.level_page_range_array_.count()); + + meta_tree_.reset(); + STORAGE_LOG(INFO, "================test_tree_insert_fail_after_array_used begin================"); +} + +TEST_F(TestSNTmpFileMetaTree, test_tree_insert_fail_after_tree_build) +{ + STORAGE_LOG(INFO, "================test_tree_insert_fail_after_tree_build begin================"); + ObTmpWriteBufferPool wbp; + wbp.default_wbp_memory_limit_ = ObTmpWriteBufferPool::WBP_BLOCK_SIZE; //253 pages + ASSERT_EQ(OB_SUCCESS, wbp.init()); + common::ObFIFOAllocator callback_allocator; + ASSERT_EQ(OB_SUCCESS, callback_allocator.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileCallback", ObCtxIds::DEFAULT_CTX_ID))); + ObTmpFileTestMetaTree meta_tree_; + ASSERT_EQ(OB_SUCCESS, meta_tree_.init(1, &wbp, &callback_allocator)); + meta_tree_.set_max_array_item_cnt(2); + meta_tree_.set_max_page_item_cnt(3); + STORAGE_LOG(INFO, "=======================first insert======================="); + int64_t item_num = 361; + ObArray data_items; + generate_data_items(item_num, 0/*start virtual page id*/, data_items); + ASSERT_EQ(item_num, data_items.count()); + //insert 750 items (insert a array) + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items)); + ASSERT_EQ(6, meta_tree_.level_page_range_array_.count()); + + STORAGE_LOG(INFO, "=======================second insert======================="); + item_num = 361; + ObArray data_items_1; + generate_data_items(item_num, 361 * 128/*start virtual page id*/, data_items_1); + ASSERT_EQ(item_num, data_items_1.count()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_ALLOCATE_TMP_FILE_PAGE_FAILED, meta_tree_.insert_items(data_items_1)); + ASSERT_EQ(121, meta_tree_.level_page_range_array_.at(0).cached_page_num_); + ASSERT_EQ(41, meta_tree_.level_page_range_array_.at(1).cached_page_num_); + + STORAGE_LOG(INFO, "=======================third insert======================="); + item_num = 2; + ObArray data_items_2; + generate_data_items(item_num, 361 * 128/*start virtual page id*/ - 10, data_items_2); + ASSERT_EQ(item_num, data_items_2.count()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_NE(OB_SUCCESS, meta_tree_.insert_items(data_items_2)); + data_items_2.reset(); + generate_data_items(item_num, 361 * 128/*start virtual page id*/, data_items_2); + ASSERT_EQ(item_num, data_items_2.count()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items_2)); + ASSERT_EQ(121, meta_tree_.level_page_range_array_.at(0).cached_page_num_); + ASSERT_EQ(41, meta_tree_.level_page_range_array_.at(1).cached_page_num_); + + meta_tree_.reset(); + STORAGE_LOG(INFO, "================test_tree_insert_fail_after_tree_build begin================"); +} + +TEST_F(TestSNTmpFileMetaTree, test_array_read) +{ + STORAGE_LOG(INFO, "=======================test_array_read begin======================="); + ObTmpWriteBufferPool wbp; + wbp.default_wbp_memory_limit_ = ObTmpWriteBufferPool::WBP_BLOCK_SIZE; //253 pages + ASSERT_EQ(OB_SUCCESS, wbp.init()); + common::ObFIFOAllocator callback_allocator; + ASSERT_EQ(OB_SUCCESS, callback_allocator.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileCallback", ObCtxIds::DEFAULT_CTX_ID))); + ObTmpFileTestMetaTree meta_tree_; + ASSERT_EQ(OB_SUCCESS, meta_tree_.init(1, &wbp, &callback_allocator)); + meta_tree_.set_max_array_item_cnt(100); + meta_tree_.set_max_page_item_cnt(100); + STORAGE_LOG(INFO, "=======================first array insert======================="); + int64_t item_num = 50; + ObArray data_items; + generate_data_items(item_num, 0/*start virtual page id*/, data_items); + ASSERT_EQ(item_num, data_items.count()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items)); + ASSERT_EQ(0, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(50, meta_tree_.data_item_array_.count()); + + STORAGE_LOG(INFO, "=======================second array insert======================="); + item_num = 50; + ObArray data_items_1; + generate_data_items(item_num, 50 * 128 - 1/*start virtual page id*/, data_items_1); + ASSERT_EQ(item_num, data_items_1.count()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_NE(OB_SUCCESS, meta_tree_.insert_items(data_items_1)); + data_items_1.reset(); + generate_data_items(item_num, 50 * 128/*start virtual page id*/, data_items_1); + ASSERT_EQ(item_num, data_items_1.count()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items_1)); + ASSERT_EQ(100, meta_tree_.data_item_array_.count()); + + //After simulating the insert, we can see that the tmp file offset is [0, 100 * 128 * 8K], + // with each data item occupying 128 * 8K. + STORAGE_LOG(INFO, "=======================first array read======================="); + ObArray get_data_items; + ASSERT_EQ(OB_SUCCESS, meta_tree_.search_data_items(10, 2 * 128 * ObTmpFileGlobal::PAGE_SIZE - 5, get_data_items)); + STORAGE_LOG(INFO, "data_items", K(get_data_items)); + ASSERT_EQ(3, get_data_items.count()); + + STORAGE_LOG(INFO, "=======================second array read======================="); + get_data_items.reset(); + ASSERT_EQ(OB_SUCCESS, meta_tree_.search_data_items(7 * 128 * ObTmpFileGlobal::PAGE_SIZE, 5 * 128 * ObTmpFileGlobal::PAGE_SIZE, get_data_items)); + ASSERT_EQ(5, get_data_items.count()); + + STORAGE_LOG(INFO, "=======================third array read======================="); + get_data_items.reset(); + ASSERT_EQ(OB_SUCCESS, meta_tree_.search_data_items(0, 100 * 128 * ObTmpFileGlobal::PAGE_SIZE, get_data_items)); + ASSERT_EQ(100, get_data_items.count()); + + STORAGE_LOG(INFO, "=======================forth array read======================="); + get_data_items.reset(); + ASSERT_EQ(OB_SUCCESS, meta_tree_.search_data_items(91 * 128 * ObTmpFileGlobal::PAGE_SIZE + 2, 3 * 128 * ObTmpFileGlobal::PAGE_SIZE, get_data_items)); + STORAGE_LOG(INFO, "data_items", K(get_data_items)); + ASSERT_EQ(4, get_data_items.count()); + ASSERT_EQ(91 * 128, get_data_items.at(0).virtual_page_id_); + + STORAGE_LOG(INFO, "=======================fifth array read======================="); + get_data_items.reset(); + ASSERT_EQ(OB_SUCCESS, meta_tree_.search_data_items(1, 0.5 * ObTmpFileGlobal::PAGE_SIZE, get_data_items)); + STORAGE_LOG(INFO, "data_items", K(get_data_items)); + ASSERT_EQ(1, get_data_items.count()); + ASSERT_EQ(0, get_data_items.at(0).virtual_page_id_); + + STORAGE_LOG(INFO, "=======================fifth array read======================="); + get_data_items.reset(); + ASSERT_EQ(OB_SUCCESS, meta_tree_.search_data_items(1, 0.5 * ObTmpFileGlobal::PAGE_SIZE, get_data_items)); + STORAGE_LOG(INFO, "data_items", K(get_data_items)); + ASSERT_EQ(1, get_data_items.count()); + ASSERT_EQ(0, get_data_items.at(0).virtual_page_id_); + + STORAGE_LOG(INFO, "=======================fifth array read======================="); + get_data_items.reset(); + ASSERT_NE(OB_SUCCESS, meta_tree_.search_data_items(1, 100 * 128 * ObTmpFileGlobal::PAGE_SIZE, get_data_items)); + + meta_tree_.reset(); + STORAGE_LOG(INFO, "=======================test_array_read end======================="); +} + +TEST_F(TestSNTmpFileMetaTree, test_read_fail) +{ + STORAGE_LOG(INFO, "=======================test_read_fail begin======================="); + ObTmpWriteBufferPool wbp; + wbp.default_wbp_memory_limit_ = ObTmpWriteBufferPool::WBP_BLOCK_SIZE; //253 pages + ASSERT_EQ(OB_SUCCESS, wbp.init()); + common::ObFIFOAllocator callback_allocator; + ASSERT_EQ(OB_SUCCESS, callback_allocator.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileCallback", ObCtxIds::DEFAULT_CTX_ID))); + ObTmpFileTestMetaTree meta_tree_; + ASSERT_EQ(OB_SUCCESS, meta_tree_.init(1, &wbp, &callback_allocator)); + meta_tree_.set_max_array_item_cnt(3); + meta_tree_.set_max_page_item_cnt(5); + STORAGE_LOG(INFO, "=======================tree insert======================="); + int64_t item_num = 3; + ObArray data_items; + generate_wrong_data_items(item_num, 0, data_items); + ASSERT_EQ(item_num, data_items.count()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items)); + ASSERT_EQ(3, meta_tree_.data_item_array_.count()); + + STORAGE_LOG(INFO, "=======================first read======================="); + ObArray get_data_items; + ASSERT_NE(OB_SUCCESS, meta_tree_.search_data_items(1 * 128 * ObTmpFileGlobal::PAGE_SIZE, 2 * 128 * ObTmpFileGlobal::PAGE_SIZE, get_data_items)); + + STORAGE_LOG(INFO, "=======================second insert======================="); + ASSERT_EQ(OB_SUCCESS, meta_tree_.truncate(0, 3 * 128 * ObTmpFileGlobal::PAGE_SIZE)); + item_num = 5; + ObArray data_items_1; + generate_wrong_data_items(item_num, 4 * 128, data_items_1); + ASSERT_EQ(item_num, data_items_1.count()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items_1)); + ASSERT_EQ(0, meta_tree_.data_item_array_.count()); + ASSERT_EQ(1, meta_tree_.level_page_range_array_.count()); + + STORAGE_LOG(INFO, "=======================second read======================="); + get_data_items.reset(); + ASSERT_NE(OB_SUCCESS, meta_tree_.search_data_items(4 * 128 * ObTmpFileGlobal::PAGE_SIZE, 2 * 128 * ObTmpFileGlobal::PAGE_SIZE, get_data_items)); + + meta_tree_.reset(); + STORAGE_LOG(INFO, "=======================test_read_fail end======================="); +} + +TEST_F(TestSNTmpFileMetaTree, test_array_read_after_truncate) +{ + STORAGE_LOG(INFO, "=======================test_array_read_after_truncate begin======================="); + ObTmpWriteBufferPool wbp; + wbp.default_wbp_memory_limit_ = ObTmpWriteBufferPool::WBP_BLOCK_SIZE; //253 pages + ASSERT_EQ(OB_SUCCESS, wbp.init()); + common::ObFIFOAllocator callback_allocator; + ASSERT_EQ(OB_SUCCESS, callback_allocator.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileCallback", ObCtxIds::DEFAULT_CTX_ID))); + ObTmpFileTestMetaTree meta_tree_; + ASSERT_EQ(OB_SUCCESS, meta_tree_.init(1, &wbp, &callback_allocator)); + meta_tree_.set_max_array_item_cnt(100); + meta_tree_.set_max_page_item_cnt(100); + STORAGE_LOG(INFO, "=======================array insert======================="); + int64_t item_num = 100; + ObArray data_items; + generate_data_items(item_num, 0/*start virtual page id*/, data_items); + ASSERT_EQ(item_num, data_items.count()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items)); + ASSERT_EQ(0, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(100, meta_tree_.data_item_array_.count()); + //After simulating the insert, we can see that the tmp file offset is [0, 100 * 128 * 8K], + // with each data item occupying 128 * 8K. + STORAGE_LOG(INFO, "=======================array truncate======================="); + ASSERT_EQ(OB_SUCCESS, meta_tree_.truncate(0, 2 * 128 * ObTmpFileGlobal::PAGE_SIZE - 5)); + ASSERT_EQ(99, meta_tree_.data_item_array_.count()); + + ASSERT_EQ(OB_SUCCESS, meta_tree_.truncate(2 * 128 * ObTmpFileGlobal::PAGE_SIZE - 5, 20 * 128 * ObTmpFileGlobal::PAGE_SIZE)); + ASSERT_EQ(80, meta_tree_.data_item_array_.count()); + + STORAGE_LOG(INFO, "=======================array read======================="); + ObArray get_data_items; + get_data_items.reset(); + ASSERT_EQ(OB_SUCCESS, meta_tree_.search_data_items(20 * 128 * ObTmpFileGlobal::PAGE_SIZE, 5 * 128 * ObTmpFileGlobal::PAGE_SIZE, get_data_items)); + ASSERT_EQ(5, get_data_items.count()); + STORAGE_LOG(INFO, "data_items", K(get_data_items)); + + get_data_items.reset(); + ASSERT_NE(OB_SUCCESS, meta_tree_.search_data_items(18 * 128 * ObTmpFileGlobal::PAGE_SIZE, 5 * 128 * ObTmpFileGlobal::PAGE_SIZE, get_data_items)); + + meta_tree_.reset(); + STORAGE_LOG(INFO, "=======================test_array_read_after_truncate end======================="); +} + +TEST_F(TestSNTmpFileMetaTree, test_write_tail) +{ + STORAGE_LOG(INFO, "=======================test_write_tail begin======================="); + ObTmpWriteBufferPool wbp; + wbp.default_wbp_memory_limit_ = ObTmpWriteBufferPool::WBP_BLOCK_SIZE; //253 pages + ASSERT_EQ(OB_SUCCESS, wbp.init()); + common::ObFIFOAllocator callback_allocator; + ASSERT_EQ(OB_SUCCESS, callback_allocator.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileCallback", ObCtxIds::DEFAULT_CTX_ID))); + ObTmpFileTestMetaTree meta_tree_; + ASSERT_EQ(OB_SUCCESS, meta_tree_.init(1, &wbp, &callback_allocator)); + meta_tree_.set_max_array_item_cnt(5); + meta_tree_.set_max_page_item_cnt(5); + STORAGE_LOG(INFO, "=======================first insert======================="); + int64_t item_num = 4; + ObArray data_items; + generate_data_items(item_num, 0/*start virtual page id*/, data_items); + ASSERT_EQ(item_num, data_items.count()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items)); + ASSERT_EQ(0, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(4, meta_tree_.data_item_array_.count()); + //After simulating the insert, we can see that the tmp file offset is [0, 100 * 128 * 8K], + // with each data item occupying 128 * 8K. + // we assume that the last page of the last data item is an unfilled page. + STORAGE_LOG(INFO, "=======================first write tail======================="); + ObSharedNothingTmpFileDataItem last_data_item; + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_write_tail(last_data_item)); + //write tail fail + ASSERT_EQ(OB_SUCCESS, meta_tree_.finish_write_tail(last_data_item, false)); + ASSERT_EQ(128, meta_tree_.data_item_array_.at(meta_tree_.data_item_array_.count() - 1).physical_page_num_); + last_data_item.reset(); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_write_tail(last_data_item)); + ASSERT_EQ(OB_SUCCESS, meta_tree_.finish_write_tail(last_data_item, true)); + ASSERT_EQ(127, meta_tree_.data_item_array_.at(meta_tree_.data_item_array_.count() - 1).physical_page_num_); + ASSERT_EQ(4, meta_tree_.data_item_array_.count()); + + STORAGE_LOG(INFO, "=======================second insert======================="); + data_items.reset(); + last_data_item.reset(); + last_data_item.block_index_ = 30; + last_data_item.physical_page_id_ = 0; + last_data_item.physical_page_num_ = 1; + last_data_item.virtual_page_id_ = 3 * 128 + 127; + ASSERT_EQ(OB_SUCCESS, data_items.push_back(last_data_item)); //unfilled page + ASSERT_EQ(1, data_items.count()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items)); + ASSERT_EQ(5, meta_tree_.data_item_array_.count()); + + STORAGE_LOG(INFO, "=======================second write tail======================="); + last_data_item.reset(); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_write_tail(last_data_item)); + ASSERT_EQ(OB_SUCCESS, meta_tree_.finish_write_tail(last_data_item, true)); + ASSERT_EQ(4, meta_tree_.data_item_array_.count()); + + STORAGE_LOG(INFO, "=======================third insert======================="); + item_num = 2; + ObArray data_items_1; + generate_data_items(item_num, 3 * 128 + 127/*start virtual page id*/, data_items_1); + //we assume that the last page of the last data item is an unfilled page. + ASSERT_EQ(item_num, data_items_1.count()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items_1)); + ASSERT_EQ(2, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(0, meta_tree_.data_item_array_.count()); + + STORAGE_LOG(INFO, "=======================third write tail======================="); + last_data_item.reset(); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_write_tail(last_data_item)); + ASSERT_EQ(OB_SUCCESS, meta_tree_.finish_write_tail(last_data_item, true)); + + STORAGE_LOG(INFO, "=======================first read======================="); + ObArray data_items_2; + ASSERT_EQ(OB_SUCCESS, meta_tree_.search_data_items((3 * 128 + 127) * ObTmpFileGlobal::PAGE_SIZE, 240 * ObTmpFileGlobal::PAGE_SIZE, data_items_2)); + ASSERT_EQ(2, data_items_2.count()); + ASSERT_EQ(127, data_items_2.at(1).physical_page_num_); + + meta_tree_.reset(); + STORAGE_LOG(INFO, "=======================test_write_tail end======================="); +} + +TEST_F(TestSNTmpFileMetaTree, test_page_is_dirty_again_during_flush) +{ + STORAGE_LOG(INFO, "=================test_page_is_dirty_again_during_flush begin==============="); + ObTmpWriteBufferPool wbp; + wbp.default_wbp_memory_limit_ = ObTmpWriteBufferPool::WBP_BLOCK_SIZE; //253 pages + ASSERT_EQ(OB_SUCCESS, wbp.init()); + common::ObFIFOAllocator callback_allocator; + ASSERT_EQ(OB_SUCCESS, callback_allocator.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileCallback", ObCtxIds::DEFAULT_CTX_ID))); + ObTmpFileTestMetaTree meta_tree_; + ASSERT_EQ(OB_SUCCESS, meta_tree_.init(1, &wbp, &callback_allocator)); + meta_tree_.set_max_array_item_cnt(5); + meta_tree_.set_max_page_item_cnt(5); + STORAGE_LOG(INFO, "=======================first tree insert======================="); + int64_t item_num = 10; + ObArray data_items; + generate_data_items(item_num, 0/*start virtual page id*/, data_items); + ASSERT_EQ(item_num, data_items.count()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items)); + ASSERT_EQ(2, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(2, meta_tree_.level_page_range_array_.at(0).cached_page_num_); + + STORAGE_LOG(INFO, "=======================first tree flush========================="); + ObTmpFileTreeFlushContext flush_context_first; + char *block_buff_1 = new char[OB_DEFAULT_MACRO_BLOCK_SIZE]; + ObArray tree_io_array_1; + int64_t write_offset_1 = OB_DEFAULT_MACRO_BLOCK_SIZE - 2 * ObTmpFileGlobal::PAGE_SIZE; + ASSERT_EQ(OB_SUCCESS, meta_tree_.flush_meta_pages_for_block(0/*block_index*/, + ObTmpFileTreeEvictType::FULL, + block_buff_1, + write_offset_1, + flush_context_first, + tree_io_array_1)); + STORAGE_LOG(INFO, "tree_io_array_1", K(tree_io_array_1)); + ASSERT_EQ(1, tree_io_array_1.count()); + ASSERT_EQ(2, tree_io_array_1.at(0).flush_nums_); + ASSERT_EQ(OB_DEFAULT_MACRO_BLOCK_SIZE, write_offset_1); + ASSERT_EQ(false, flush_context_first.is_meta_reach_end_); + + STORAGE_LOG(INFO, "=======================first write tail======================="); + ObSharedNothingTmpFileDataItem last_data_item; + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_write_tail(last_data_item)); + ASSERT_EQ(OB_SUCCESS, meta_tree_.finish_write_tail(last_data_item, true)); + + STORAGE_LOG(INFO, "=======================second tree insert======================="); + item_num = 1; + data_items.reset(); + generate_data_items(item_num, 9 * 128 + 127/*start virtual page id*/, data_items); + ASSERT_EQ(item_num, data_items.count()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items)); + ASSERT_EQ(2, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(3, meta_tree_.level_page_range_array_.at(0).cached_page_num_); + + STORAGE_LOG(INFO, "=======================second tree flush========================="); + char *block_buff_2 = new char[OB_DEFAULT_MACRO_BLOCK_SIZE]; + ObArray tree_io_array_2; + int64_t write_offset_2 = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.flush_meta_pages_for_block(1/*block_index*/, + ObTmpFileTreeEvictType::FULL, + block_buff_2, + write_offset_2, + flush_context_first, + tree_io_array_2)); + ASSERT_EQ(0, tree_io_array_2.count()); + ASSERT_EQ(true, flush_context_first.is_meta_reach_end_); + + STORAGE_LOG(INFO, "=======================third tree flush========================="); + //flush successfully + ASSERT_EQ(OB_SUCCESS, meta_tree_.update_after_flush(tree_io_array_1)); + ASSERT_EQ(tree_io_array_1.at(0).flush_end_page_id_, meta_tree_.level_page_range_array_.at(0).flushed_end_page_id_); + + ObTmpFileTreeFlushContext flush_context_second; + char *block_buff_3 = new char[OB_DEFAULT_MACRO_BLOCK_SIZE]; + ObArray tree_io_array_3; + int64_t write_offset_3 = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.flush_meta_pages_for_block(2/*block_index*/, + ObTmpFileTreeEvictType::FULL, + block_buff_3, + write_offset_3, + flush_context_second, + tree_io_array_3)); + ASSERT_EQ(2, tree_io_array_3.count()); + ASSERT_EQ(2, tree_io_array_3.at(0).flush_nums_); + ASSERT_EQ(1, tree_io_array_3.at(1).flush_nums_); + ASSERT_EQ(true, flush_context_first.is_meta_reach_end_); + + delete[] block_buff_1; + delete[] block_buff_2; + delete[] block_buff_3; + meta_tree_.reset(); + STORAGE_LOG(INFO, "=================test_page_is_dirty_again_during_flush end==============="); +} + +TEST_F(TestSNTmpFileMetaTree, test_insert_items_during_flush) +{ + STORAGE_LOG(INFO, "=================test_insert_items_during_flush begin==============="); + ObTmpWriteBufferPool wbp; + wbp.default_wbp_memory_limit_ = ObTmpWriteBufferPool::WBP_BLOCK_SIZE; //253 pages + ASSERT_EQ(OB_SUCCESS, wbp.init()); + common::ObFIFOAllocator callback_allocator; + ASSERT_EQ(OB_SUCCESS, callback_allocator.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileCallback", ObCtxIds::DEFAULT_CTX_ID))); + ObTmpFileTestMetaTree meta_tree_; + ASSERT_EQ(OB_SUCCESS, meta_tree_.init(1, &wbp, &callback_allocator)); + meta_tree_.set_max_array_item_cnt(5); + meta_tree_.set_max_page_item_cnt(5); + STORAGE_LOG(INFO, "=======================first tree insert======================="); + int64_t item_num = 10; + ObArray data_items; + generate_data_items(item_num, 0/*start virtual page id*/, data_items); + ASSERT_EQ(item_num, data_items.count()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items)); + ASSERT_EQ(2, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(2, meta_tree_.level_page_range_array_.at(0).cached_page_num_); + + STORAGE_LOG(INFO, "=======================first tree flush========================="); + ObTmpFileTreeFlushContext flush_context_first; + char *block_buff_1 = new char[OB_DEFAULT_MACRO_BLOCK_SIZE]; + ObArray tree_io_array_1; + int64_t write_offset_1 = OB_DEFAULT_MACRO_BLOCK_SIZE - 2 * ObTmpFileGlobal::PAGE_SIZE; + ASSERT_EQ(OB_SUCCESS, meta_tree_.flush_meta_pages_for_block(0/*block_index*/, + ObTmpFileTreeEvictType::FULL, + block_buff_1, + write_offset_1, + flush_context_first, + tree_io_array_1)); + ASSERT_EQ(1, tree_io_array_1.count()); + ASSERT_EQ(2, tree_io_array_1.at(0).flush_nums_); + ASSERT_EQ(OB_DEFAULT_MACRO_BLOCK_SIZE, write_offset_1); + ASSERT_EQ(false, flush_context_first.is_meta_reach_end_); + + STORAGE_LOG(INFO, "=======================second tree insert======================="); + item_num = 1; + data_items.reset(); + generate_data_items(item_num, 10 * 128/*start virtual page id*/, data_items); + ASSERT_EQ(item_num, data_items.count()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.prepare_for_insert_items()); + ASSERT_EQ(OB_SUCCESS, meta_tree_.insert_items(data_items)); + ASSERT_EQ(2, meta_tree_.level_page_range_array_.count()); + ASSERT_EQ(3, meta_tree_.level_page_range_array_.at(0).cached_page_num_); + + STORAGE_LOG(INFO, "=======================second tree flush========================="); + char *block_buff_2 = new char[OB_DEFAULT_MACRO_BLOCK_SIZE]; + ObArray tree_io_array_2; + int64_t write_offset_2 = 0; + ASSERT_EQ(OB_SUCCESS, meta_tree_.flush_meta_pages_for_block(1/*block_index*/, + ObTmpFileTreeEvictType::FULL, + block_buff_2, + write_offset_2, + flush_context_first, + tree_io_array_2)); + ASSERT_EQ(2, tree_io_array_2.count()); + ASSERT_EQ(1, tree_io_array_2.at(0).flush_nums_); + ASSERT_EQ(true, flush_context_first.is_meta_reach_end_); + + delete[] block_buff_1; + delete[] block_buff_2; + meta_tree_.reset(); + STORAGE_LOG(INFO, "=================test_insert_items_during_flush end==============="); +} + +} // namespace oceanbase + +int main(int argc, char **argv) +{ + int ret = 0; + system("rm -f ./test_sn_tmp_file_meta_tree.log*"); + OB_LOGGER.set_file_name("test_sn_tmp_file_meta_tree.log", true); + OB_LOGGER.set_log_level("INFO"); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/mittest/mtlenv/storage/tmp_file/test_tmp_file_write_buffer_pool_index_cache.cpp b/mittest/mtlenv/storage/tmp_file/test_tmp_file_write_buffer_pool_index_cache.cpp new file mode 100644 index 000000000..63e19a8e3 --- /dev/null +++ b/mittest/mtlenv/storage/tmp_file/test_tmp_file_write_buffer_pool_index_cache.cpp @@ -0,0 +1,843 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include +#include +#define USING_LOG_PREFIX STORAGE +#define protected public +#define private public +#include "mittest/mtlenv/mock_tenant_module_env.h" +#include "storage/tmp_file/ob_tmp_file_global.h" +#include "storage/tmp_file/ob_tmp_file_write_buffer_index_cache.h" +#include "storage/tmp_file/ob_tmp_file_write_buffer_pool.h" + +namespace oceanbase +{ +using namespace tmp_file; + +static const int64_t INIT_BUCKET_ARRAY_CAPACITY = ObTmpFileWBPIndexCache::INIT_BUCKET_ARRAY_CAPACITY; +static const int64_t MAX_BUCKET_ARRAY_CAPACITY = ObTmpFileWBPIndexCache::MAX_BUCKET_ARRAY_CAPACITY; +static const int64_t BUCKET_CAPACITY = ObTmpFileWBPIndexCache::ObTmpFilePageIndexBucket::BUCKET_CAPACITY; +static const int64_t fd = 1; +static uint64_t tenant_id = OB_SYS_TENANT_ID; +static const int64_t TENANT_MEMORY = 8L * 1024L * 1024L * 1024L /* 8 GB */; +static const double TMP_FILE_WBP_MEM_LIMIT_PROP = 50; //[0, 100] +/* ----------------------------- Mock Class -------------------------------- */ +class MockWBPIndexCache +{ +public: + MockWBPIndexCache(ObTmpWriteBufferPool &wbp) : wbp_(wbp) {} + int push(uint32_t page_index); + int truncate(const int64_t truncate_page_virtual_id); + int compare(const ObTmpFileWBPIndexCache &index_cache); +private: + int sparsify_(const int64_t sparsify_modulus); +public: + ObArray mock_index_cache_; + ObTmpWriteBufferPool &wbp_; +}; + +int MockWBPIndexCache::push(uint32_t page_index) +{ + int ret = OB_SUCCESS; + if (mock_index_cache_.count() >= MAX_BUCKET_ARRAY_CAPACITY * BUCKET_CAPACITY) { + if (OB_FAIL(sparsify_(2))) { + LOG_WARN("failed to sparsify", KR(ret)); + } + } + if (FAILEDx(mock_index_cache_.push_back(page_index))) { + LOG_WARN("failed to push back page index", KR(ret), K(page_index)); + } + return ret; +} + +int MockWBPIndexCache::truncate(const int64_t truncate_page_virtual_id) +{ + int ret = OB_SUCCESS; + ObArray new_mock_index_cache; + if (truncate_page_virtual_id < 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(truncate_page_virtual_id)); + } else { + int64_t i = 0; + for (; OB_SUCC(ret) && i < mock_index_cache_.count(); ++i) { + int64_t virtual_page_id = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + if (OB_FAIL(wbp_.get_page_virtual_id(fd, mock_index_cache_.at(i), virtual_page_id))) { + LOG_WARN("fail to get page virtual id", KR(ret), K(mock_index_cache_.at(i))); + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == virtual_page_id)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid virtual page id", KR(ret), K(virtual_page_id)); + } else if (virtual_page_id >= truncate_page_virtual_id) { + break; + } + } + for (int64_t j = i; j < mock_index_cache_.count(); ++j) { + if (OB_FAIL(new_mock_index_cache.push_back(mock_index_cache_.at(j)))) { + LOG_WARN("fail to push back page index", KR(ret), K(mock_index_cache_.at(j))); + } + } + mock_index_cache_ = new_mock_index_cache; + } + return ret; +} + +int MockWBPIndexCache::sparsify_(const int64_t sparsify_modulus) +{ + int ret = OB_SUCCESS; + ObArray new_mock_index_cache; + if (OB_UNLIKELY(sparsify_modulus < 2 || + mock_index_cache_.count() != MAX_BUCKET_ARRAY_CAPACITY * BUCKET_CAPACITY)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid argument", KR(ret), K(sparsify_modulus), K(mock_index_cache_.count())); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < mock_index_cache_.count(); ++i) { + if (i % sparsify_modulus != 0 && + OB_FAIL(new_mock_index_cache.push_back(mock_index_cache_.at(i)))) { + LOG_WARN("fail to push back page index", KR(ret), K(mock_index_cache_.at(i))); + } + } + mock_index_cache_ = new_mock_index_cache; + } + return ret; +} + +int MockWBPIndexCache::compare(const ObTmpFileWBPIndexCache &index_cache) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(index_cache.is_empty() && mock_index_cache_.empty())) { + // do nothing + } else { + if (OB_UNLIKELY(index_cache.is_empty() || mock_index_cache_.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("one cache is empty", KR(ret), K(index_cache.is_empty()), K(mock_index_cache_.empty())); + } else if (OB_ISNULL(index_cache.page_buckets_) || OB_ISNULL(index_cache.page_buckets_->at(index_cache.left_)) || + OB_ISNULL(index_cache.page_buckets_->at(index_cache.right_))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", KR(ret), KP(index_cache.page_buckets_), K(index_cache.left_), K(index_cache.right_)); + } else if (OB_UNLIKELY(index_cache.page_buckets_->at(index_cache.left_)->is_empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected empty bucket", KR(ret), K(index_cache.left_)); + } else if (OB_UNLIKELY(index_cache.page_buckets_->at(index_cache.right_)->is_empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected empty bucket", KR(ret), K(index_cache.right_)); + } else if (index_cache.size() == 1) { + if (mock_index_cache_.count() != index_cache.page_buckets_->at(index_cache.left_)->size()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected size", KR(ret), K(index_cache.size()), K(mock_index_cache_.count())); + } + } else { + int64_t real_cache_size = index_cache.page_buckets_->at(index_cache.left_)->size() + + index_cache.page_buckets_->at(index_cache.right_)->size() + + (index_cache.get_logic_tail_() - index_cache.left_ - 1) * + BUCKET_CAPACITY; + if (mock_index_cache_.count() != real_cache_size) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected size", KR(ret), K(real_cache_size), K(index_cache.page_buckets_->at(index_cache.left_)->size()), + K(index_cache.page_buckets_->at(index_cache.right_)->size()), K(index_cache.size()), K(mock_index_cache_.count())); + } + } + + if (OB_SUCC(ret)) { + const int64_t logic_end_bkt_pos = index_cache.get_logic_tail_(); + int64_t cur_logic_bkt_pos = index_cache.left_; + int64_t cur_logic_index_pos = index_cache.page_buckets_->at(cur_logic_bkt_pos)->left_; + int64_t i = 0; + for (;OB_SUCC(ret) && i < mock_index_cache_.count() && cur_logic_bkt_pos <= logic_end_bkt_pos; ++i) { + int64_t real_bkt_pos = cur_logic_bkt_pos % index_cache.capacity_; + int64_t real_index_pos = cur_logic_index_pos % BUCKET_CAPACITY; + const ObTmpFileWBPIndexCache::ObTmpFilePageIndexBucket *bkt = index_cache.page_buckets_->at(real_bkt_pos); + uint32_t page_index_in_real_cache = ObTmpFileGlobal::INVALID_PAGE_ID; + uint32_t page_index_in_mock_cache = mock_index_cache_.at(i); + + if (OB_ISNULL(bkt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", KR(ret), KP(bkt)); + } else if (FALSE_IT(page_index_in_real_cache = bkt->page_indexes_.at(real_index_pos))) { + } else if (OB_UNLIKELY(page_index_in_mock_cache != page_index_in_real_cache)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("page index is not same", KR(ret), K(page_index_in_mock_cache), K(page_index_in_real_cache), + K(i), K(cur_logic_bkt_pos), K(cur_logic_index_pos), K(index_cache.left_), K(logic_end_bkt_pos)); + } else if (OB_UNLIKELY(cur_logic_index_pos > bkt->get_logic_tail_())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected index", KR(ret), K(cur_logic_index_pos), K(bkt->get_logic_tail_())); + } else if (cur_logic_index_pos < bkt->get_logic_tail_()) { + cur_logic_index_pos += 1; + } else { // cur_logic_index_pos == bkt->get_logic_tail_() + cur_logic_bkt_pos += 1; + if (cur_logic_bkt_pos <= logic_end_bkt_pos) { + const ObTmpFileWBPIndexCache::ObTmpFilePageIndexBucket *next_bkt = index_cache.page_buckets_->at(cur_logic_bkt_pos % index_cache.capacity_); + if (OB_ISNULL(next_bkt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", KR(ret), K(mock_index_cache_.count()), + K(index_cache.page_buckets_->at(index_cache.left_)->size()), + K(index_cache.page_buckets_->at(index_cache.right_)->size()), + K(index_cache.size()), K(index_cache.capacity_), + K(i), K(cur_logic_bkt_pos), + K(cur_logic_index_pos), K(index_cache.left_), + K(logic_end_bkt_pos), KP(next_bkt)); + } else if (OB_UNLIKELY(next_bkt->is_empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected empty", KR(ret), K(mock_index_cache_.count()), + K(index_cache.page_buckets_->at(index_cache.left_)->size()), + K(index_cache.page_buckets_->at(index_cache.right_)->size()), + K(index_cache.size()), K(index_cache.capacity_), + K(i), K(cur_logic_bkt_pos), K(cur_logic_index_pos), + K(index_cache.left_), K(logic_end_bkt_pos), + KPC(next_bkt)); + } else { + cur_logic_index_pos = next_bkt->left_; + } + } + } + } // end for + } + } + + return ret; +} + +/* ---------------------------- Unittest Class ----------------------------- */ +class TestTmpFileWBPIndexCache : public ::testing::Test +{ +public: + TestTmpFileWBPIndexCache() = default; + virtual ~TestTmpFileWBPIndexCache() = default; + virtual void SetUp(); + virtual void TearDown(); + static void SetUpTestCase(); + static void TearDownTestCase(); + int write_and_push_pages(const uint32_t end_page_id,const int64_t page_num, + MockWBPIndexCache &mock_cache, uint32_t &new_end_page_id); + int truncate_pages(const int64_t truncate_page_num, MockWBPIndexCache &mock_cache); + int truncate_and_free_pages(const uint32_t wbp_begin_page_id, const int64_t truncate_page_num_in_cache, + MockWBPIndexCache &mock_cache, uint32_t &new_begin_page_id); +private: + int write_pages_(const int64_t page_num, const int64_t begin_page_virtual_id, ObArray &page_indexes); + int free_pages_(const uint32_t begin_page_id, const int64_t end_page_virtual_id, uint32_t &new_begin_page_id); + int check_cache_status_after_truncate_(MockWBPIndexCache &mock_cache); +public: + ObTmpWriteBufferPool wbp_; + ObTmpFileWBPIndexCache wbp_index_cache_; + common::ObFIFOAllocator wbp_index_cache_allocator_; + common::ObFIFOAllocator wbp_index_cache_bkt_allocator_; +}; + +void TestTmpFileWBPIndexCache::SetUp() +{ + ASSERT_EQ(OB_SUCCESS, wbp_index_cache_allocator_.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_NORMAL_BLOCK_SIZE, + ObMemAttr(tenant_id, "TmpFileIndCache", + ObCtxIds::DEFAULT_CTX_ID))); + ASSERT_EQ(OB_SUCCESS, wbp_index_cache_bkt_allocator_.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(tenant_id, "TmpFileIndCBkt", + ObCtxIds::DEFAULT_CTX_ID))); + ASSERT_EQ(OB_SUCCESS, wbp_.init()); + ASSERT_EQ(OB_SUCCESS, wbp_index_cache_.init(fd, &wbp_, &wbp_index_cache_allocator_, &wbp_index_cache_bkt_allocator_)); +} + +void TestTmpFileWBPIndexCache::TearDown() +{ + wbp_index_cache_allocator_.reset(); + wbp_index_cache_bkt_allocator_.reset(); + wbp_.destroy(); + wbp_index_cache_.destroy(); +} + +void TestTmpFileWBPIndexCache::SetUpTestCase() +{ + int ret = OB_SUCCESS; + ASSERT_EQ(OB_SUCCESS, MockTenantModuleEnv::get_instance().init()); + + CHUNK_MGR.set_limit(TENANT_MEMORY); + ObMallocAllocator::get_instance()->set_tenant_limit(1, TENANT_MEMORY); + omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID())); + ASSERT_EQ(true, tenant_config.is_valid()); + tenant_config->_temporary_file_io_area_size = TMP_FILE_WBP_MEM_LIMIT_PROP; +} + +void TestTmpFileWBPIndexCache::TearDownTestCase() +{ + MockTenantModuleEnv::get_instance().destroy(); +} + +// end_page_id: the page_id of the last page of wbp +// page_num: the number of pages to be alloced. +// new_end_page_id: page_id of the last page of alloced pages. +// this function will alloc 'page_num' pages and link them after the page of end_page_id, +// then put them into mock_cache and wbp_index_cache_. +int TestTmpFileWBPIndexCache::write_and_push_pages(const uint32_t end_page_id, const int64_t page_num, + MockWBPIndexCache &mock_cache, uint32_t &new_end_page_id) +{ + int ret = OB_SUCCESS; + ObArray page_indexes; + int64_t end_page_virtual_id = 0; + int64_t new_begin_page_virtual_id = 0; + if (OB_UNLIKELY(page_num <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(page_num)); + } else if (end_page_id != ObTmpFileGlobal::INVALID_PAGE_ID) { + if (OB_FAIL(wbp_.get_page_virtual_id(fd, end_page_id, end_page_virtual_id))) { + LOG_WARN("fail to get page virtual id", KR(ret), K(end_page_id), K(end_page_virtual_id)); + } else if (OB_UNLIKELY(end_page_virtual_id < 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid virtual id", KR(ret), K(end_page_virtual_id)); + } else { + new_begin_page_virtual_id = end_page_virtual_id + 1; + } + } + + if (FAILEDx(write_pages_(page_num, new_begin_page_virtual_id, page_indexes))) { + LOG_WARN("fail to write pages", KR(ret), K(page_num)); + } else if (OB_UNLIKELY(page_indexes.count() != page_num)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error", KR(ret), K(page_num), K(page_indexes.count())); + } else if (end_page_id != ObTmpFileGlobal::INVALID_PAGE_ID && + OB_FAIL(wbp_.link_page(fd, page_indexes.at(0), end_page_id, ObTmpFilePageUniqKey(end_page_virtual_id)))) { + LOG_WARN("fail to link page", KR(ret), K(page_indexes.at(0)), K(end_page_id), K(end_page_virtual_id)); + } else if (FALSE_IT(new_end_page_id = page_indexes.at(page_num - 1))) { + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < page_num; ++i) { + if (OB_FAIL(mock_cache.push(page_indexes.at(i)))) { + LOG_WARN("fail to push", KR(ret), K(i), K(page_indexes.at(i))); + } else if (OB_FAIL(wbp_index_cache_.push(page_indexes.at(i)))) { + LOG_WARN("fail to push", KR(ret), K(i), K(page_indexes.at(i))); + } + } + } + return ret; +} + +int TestTmpFileWBPIndexCache::truncate_pages(const int64_t truncate_page_num, MockWBPIndexCache &mock_cache) +{ + int ret = OB_SUCCESS; + ObArray page_indexes; + if (OB_UNLIKELY(truncate_page_num <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(truncate_page_num)); + } else { + int64_t truncate_page_virtual_id = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + uint32_t truncate_page_id = mock_cache.mock_index_cache_.at(truncate_page_num - 1); + if (OB_FAIL(wbp_.get_page_virtual_id(fd, truncate_page_id, truncate_page_virtual_id))) { + LOG_WARN("fail to get page virtual id", KR(ret), K(truncate_page_id)); + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == truncate_page_virtual_id)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid virtual page id", KR(ret), K(truncate_page_virtual_id)); + } else if (FALSE_IT(truncate_page_virtual_id += 1)) { + } else if (OB_FAIL(mock_cache.truncate(truncate_page_virtual_id))) { + LOG_WARN("fail to truncate", KR(ret), K(truncate_page_virtual_id)); + } else if (OB_FAIL(wbp_index_cache_.truncate(truncate_page_virtual_id))) { + LOG_WARN("fail to truncate", KR(ret), K(truncate_page_virtual_id)); + } else if (OB_FAIL(check_cache_status_after_truncate_(mock_cache))) { + LOG_WARN("fail to check cache status after truncate", KR(ret)); + } + } + return ret; +} + +// wbp_begin_page_id: the page_id of the first page of wbp, the pos of it in the wbp list should be +// less than or equal to the first cached index in wbp_index_cache_. +// truncate_page_num_in_cache: the number of page indexes to be truncated in mock_cache, +// we will free pages between [wbp_begin_page_id, begin_page_id_in_cache + truncate_page_num_in_cache] +// new_begin_page_id: page_id of the first page after free pages in wbp. +int TestTmpFileWBPIndexCache::truncate_and_free_pages(const uint32_t wbp_begin_page_id, const int64_t truncate_page_num_in_cache, + MockWBPIndexCache &mock_cache, uint32_t &new_begin_page_id) +{ + int ret = OB_SUCCESS; + ObArray page_indexes; + if (OB_UNLIKELY(truncate_page_num_in_cache <= 0 || ObTmpFileGlobal::INVALID_PAGE_ID == wbp_begin_page_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(truncate_page_num_in_cache), K(wbp_begin_page_id)); + } else { + int64_t truncate_page_virtual_id = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + uint32_t truncate_page_id = mock_cache.mock_index_cache_.at(truncate_page_num_in_cache - 1); + if (OB_FAIL(wbp_.get_page_virtual_id(fd, truncate_page_id, truncate_page_virtual_id))) { + LOG_WARN("fail to get page virtual id", KR(ret), K(truncate_page_id)); + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == truncate_page_virtual_id)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid virtual page id", KR(ret), K(truncate_page_virtual_id)); + } else if (FALSE_IT(truncate_page_virtual_id += 1)) { + } else if (OB_FAIL(mock_cache.truncate(truncate_page_virtual_id))) { + LOG_WARN("fail to truncate", KR(ret), K(truncate_page_virtual_id)); + } else if (OB_FAIL(wbp_index_cache_.truncate(truncate_page_virtual_id))) { + LOG_WARN("fail to truncate", KR(ret), K(truncate_page_virtual_id)); + } else if (OB_FAIL(check_cache_status_after_truncate_(mock_cache))) { + LOG_WARN("fail to check cache status after truncate", KR(ret)); + } else if (OB_FAIL(free_pages_(wbp_begin_page_id, truncate_page_virtual_id, new_begin_page_id))) { + LOG_WARN("fail to free pages", KR(ret), K(wbp_begin_page_id), K(truncate_page_virtual_id)); + } + } + return ret; +} + +int TestTmpFileWBPIndexCache::write_pages_(const int64_t page_num, const int64_t begin_page_virtual_id, ObArray &page_indexes) +{ + int ret = OB_SUCCESS; + page_indexes.reset(); + if (OB_UNLIKELY(page_num <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(page_num)); + } else{ + for (int64_t i = 0; OB_SUCC(ret) && i < page_num; ++i) { + uint32_t new_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + char *unused_buf = nullptr; + int64_t new_page_begin_page_id = begin_page_virtual_id + i; + if (OB_FAIL(wbp_.alloc_page(fd, ObTmpFilePageUniqKey(new_page_begin_page_id), new_page_id, unused_buf))) { + LOG_WARN("fail to alloc data page", KR(ret), K(i)); + } else if (OB_FAIL(wbp_.notify_dirty(fd, new_page_id, ObTmpFilePageUniqKey(new_page_begin_page_id)))) { + LOG_WARN("fail to notify dirty", KR(ret), K(new_page_id), K(new_page_begin_page_id)); + } else if (!page_indexes.empty() && + OB_FAIL(wbp_.link_page(fd, new_page_id, page_indexes.at(page_indexes.count()-1), + ObTmpFilePageUniqKey(new_page_begin_page_id - 1)))) { + LOG_WARN("fail to link page", KR(ret), K(i), K(new_page_id), + K(page_indexes.at(page_indexes.count()-1)), K(new_page_begin_page_id)); + } else if (OB_FAIL(page_indexes.push_back(new_page_id))) { + LOG_WARN("fail to push back", KR(ret), K(i)); + } + } + } + return ret; +} + + +int TestTmpFileWBPIndexCache::free_pages_(const uint32_t begin_page_id, const int64_t end_page_virtual_id, uint32_t &new_begin_page_id) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(end_page_virtual_id <= 0 || begin_page_id == ObTmpFileGlobal::INVALID_PAGE_ID)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(end_page_virtual_id), K(begin_page_id)); + } else { + bool free_over = false; + uint32_t cur_page_id = begin_page_id; + int64_t cnt = 0; + while(!free_over && OB_SUCC(ret) && cur_page_id != ObTmpFileGlobal::INVALID_PAGE_ID) { + int64_t virtual_page_id = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + if (OB_FAIL(wbp_.get_page_virtual_id(fd, cur_page_id, virtual_page_id))) { + LOG_WARN("fail to get page virtual id", KR(ret), K(cur_page_id), K(cnt)); + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == virtual_page_id)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid virtual page id", KR(ret), K(virtual_page_id), K(cnt)); + } else if (virtual_page_id >= end_page_virtual_id) { + free_over = true; + new_begin_page_id = cur_page_id; + } else if (OB_FAIL(wbp_.free_page(fd, cur_page_id, ObTmpFilePageUniqKey(virtual_page_id), next_page_id))) { + LOG_WARN("fail to alloc data page", KR(ret), K(cur_page_id), K(cnt)); + } else { + cur_page_id = next_page_id; + cnt += 1; + } + } + } + return ret; +} + +int TestTmpFileWBPIndexCache::check_cache_status_after_truncate_(MockWBPIndexCache &mock_cache) +{ + int ret = OB_SUCCESS; + int64_t bkt_num = wbp_index_cache_.get_logic_tail_() - wbp_index_cache_.left_ + 1; + int64_t page_num = 0; + if (OB_UNLIKELY(bkt_num != wbp_index_cache_.size())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("bucket num is invalid", KR(ret), K(bkt_num), K(wbp_index_cache_.size())); + } + for (int64_t bkt_arr_logic_idx = wbp_index_cache_.left_; + OB_SUCC(ret) && bkt_arr_logic_idx <= wbp_index_cache_.get_logic_tail_(); + bkt_arr_logic_idx++) { + int64_t bkt_arr_idx = bkt_arr_logic_idx % wbp_index_cache_.capacity_; + int64_t bkt_left_idx = 0; + if (OB_ISNULL(wbp_index_cache_.page_buckets_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("bucket array is null", KR(ret)); + } else if (OB_ISNULL(wbp_index_cache_.page_buckets_->at(bkt_arr_idx))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("bucket is null", KR(ret), K(bkt_arr_idx)); + } else if (OB_UNLIKELY(wbp_index_cache_.page_buckets_->at(bkt_arr_idx)->is_empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("bucket is empty", KR(ret), K(bkt_arr_idx)); + } else if (FALSE_IT(bkt_left_idx = wbp_index_cache_.page_buckets_->at(bkt_arr_idx)->left_)) { + } else if (OB_UNLIKELY(wbp_index_cache_.page_buckets_->at(bkt_arr_idx)->min_page_index_ != + wbp_index_cache_.page_buckets_->at(bkt_arr_idx)->page_indexes_.at(bkt_left_idx))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("min page index is not equal to left page index", KR(ret), + K(wbp_index_cache_.page_buckets_->at(bkt_arr_idx)->min_page_index_), + K(wbp_index_cache_.page_buckets_->at(bkt_arr_idx)->page_indexes_.at(bkt_left_idx))); + } else { + for (int64_t bkt_logic_idx = wbp_index_cache_.page_buckets_->at(bkt_arr_idx)->left_; + OB_SUCC(ret) && bkt_logic_idx <= wbp_index_cache_.page_buckets_->at(bkt_arr_idx)->get_logic_tail_(); + bkt_logic_idx++) { + int64_t bkt_idx = bkt_logic_idx % wbp_index_cache_.page_buckets_->at(bkt_arr_idx)->capacity_; + uint32_t page_idx = wbp_index_cache_.page_buckets_->at(bkt_arr_idx)->page_indexes_.at(bkt_idx); + if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_PAGE_ID == page_idx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("page index is invalid", KR(ret), K(bkt_arr_idx), K(bkt_idx)); + } else { + page_num += 1; + } + } // end for + } + } // end for + if (OB_UNLIKELY(page_num != mock_cache.mock_index_cache_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("page num is invalid", KR(ret), K(page_num), K(mock_cache.mock_index_cache_.count())); + } + return ret; +} + +TEST_F(TestTmpFileWBPIndexCache, test_push_and_pop) +{ + int ret = OB_SUCCESS; + uint32_t begin_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + uint32_t end_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + // 1. push indexes + int64_t page_num = BUCKET_CAPACITY / 2; + MockWBPIndexCache mock_cache(wbp_); + ret = write_and_push_pages(end_page_id, page_num, mock_cache, end_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = mock_cache.compare(wbp_index_cache_); + ASSERT_EQ(OB_SUCCESS, ret); + begin_page_id = mock_cache.mock_index_cache_.at(0); + + // 2. fill a bucket of wbp_index_cache + page_num = BUCKET_CAPACITY - page_num; + ret = write_and_push_pages(end_page_id, page_num, mock_cache, end_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = mock_cache.compare(wbp_index_cache_); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(1, wbp_index_cache_.size_); + ASSERT_EQ(true, wbp_index_cache_.page_buckets_->at(wbp_index_cache_.right_)->is_full()); + + // 3. continue to push some index when bucket is full + page_num = BUCKET_CAPACITY / 4; + ret = write_and_push_pages(end_page_id, page_num, mock_cache, end_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = mock_cache.compare(wbp_index_cache_); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(2, wbp_index_cache_.size_); + ASSERT_EQ(false, wbp_index_cache_.page_buckets_->at(wbp_index_cache_.right_)->is_full()); + + // 4. push more than a bucket number of indexes + page_num = BUCKET_CAPACITY * 2 + BUCKET_CAPACITY / 4; + ret = write_and_push_pages(end_page_id, page_num, mock_cache, end_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = mock_cache.compare(wbp_index_cache_); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(4, wbp_index_cache_.size_); + ASSERT_EQ(false, wbp_index_cache_.page_buckets_->at(wbp_index_cache_.right_)->is_full()); + + // 5. free indexes of wbp_index_cache and mock_cache + page_num = BUCKET_CAPACITY / 4; + ASSERT_LE(page_num, mock_cache.mock_index_cache_.count()); + uint32_t remove_page_id = mock_cache.mock_index_cache_.at(page_num - 1); + ret = truncate_and_free_pages(begin_page_id, page_num, mock_cache, begin_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = mock_cache.compare(wbp_index_cache_); + ASSERT_EQ(OB_SUCCESS, ret); + + // 6. push some indexes after freeing + page_num = 2 * BUCKET_CAPACITY / 4; + ret = write_and_push_pages(end_page_id, page_num, mock_cache, end_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = mock_cache.compare(wbp_index_cache_); + ASSERT_EQ(OB_SUCCESS, ret); + + // 7. free more than a bucket number of indexes + page_num = BUCKET_CAPACITY * 2 + BUCKET_CAPACITY / 2; + ASSERT_LE(page_num, mock_cache.mock_index_cache_.count()); + remove_page_id = mock_cache.mock_index_cache_.at(page_num - 1); + ret = truncate_and_free_pages(begin_page_id, page_num, mock_cache, begin_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = mock_cache.compare(wbp_index_cache_); + ASSERT_EQ(OB_SUCCESS, ret); + + // 8. free all indexes of wbp_index_cache and mock_cache + page_num = mock_cache.mock_index_cache_.count(); + remove_page_id = mock_cache.mock_index_cache_.at(page_num - 1); + ret = truncate_and_free_pages(begin_page_id, page_num, mock_cache, begin_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = mock_cache.compare(wbp_index_cache_); + ASSERT_EQ(OB_SUCCESS, ret); + end_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + + // 9. push indexes into both wbp_index_cache and mock_cache + page_num = BUCKET_CAPACITY / 3; + ret = write_and_push_pages(end_page_id, page_num, mock_cache, end_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = mock_cache.compare(wbp_index_cache_); + ASSERT_EQ(OB_SUCCESS, ret); + + LOG_INFO("test_push_and_pop"); +} + +TEST_F(TestTmpFileWBPIndexCache, test_expand_and_sparsify) +{ + int ret = OB_SUCCESS; + uint32_t begin_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + uint32_t end_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + // 1. push indexes into both wbp_index_cache and mock_cache + int64_t page_num = INIT_BUCKET_ARRAY_CAPACITY * BUCKET_CAPACITY; + MockWBPIndexCache mock_cache(wbp_); + ret = write_and_push_pages(end_page_id, page_num, mock_cache, end_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = mock_cache.compare(wbp_index_cache_); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(true, wbp_index_cache_.is_full()); + ASSERT_EQ(INIT_BUCKET_ARRAY_CAPACITY, wbp_index_cache_.capacity_); + begin_page_id = mock_cache.mock_index_cache_.at(0); + + // 2. push indexes to trigger expand + page_num = 30; + ret = write_and_push_pages(end_page_id, page_num, mock_cache, end_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = mock_cache.compare(wbp_index_cache_); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(false, wbp_index_cache_.is_full()); + ASSERT_EQ(INIT_BUCKET_ARRAY_CAPACITY * 2, wbp_index_cache_.capacity_); + + // 3. push indexes to trigger expand until reach the max capacity + page_num = MAX_BUCKET_ARRAY_CAPACITY * BUCKET_CAPACITY + - mock_cache.mock_index_cache_.count(); + ret = write_and_push_pages(end_page_id, page_num, mock_cache, end_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = mock_cache.compare(wbp_index_cache_); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(true, wbp_index_cache_.is_full()); + ASSERT_EQ(MAX_BUCKET_ARRAY_CAPACITY, wbp_index_cache_.capacity_); + + // 4. push indexes to trigger sparsify + page_num = BUCKET_CAPACITY / 2; + ret = write_and_push_pages(end_page_id, page_num, mock_cache, end_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = mock_cache.compare(wbp_index_cache_); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(false, wbp_index_cache_.is_full()); + ASSERT_EQ(wbp_index_cache_.capacity_ / 2 + 1, wbp_index_cache_.size_); + ASSERT_EQ(page_num, wbp_index_cache_.page_buckets_->at(wbp_index_cache_.right_)->size_); + ASSERT_EQ(MAX_BUCKET_ARRAY_CAPACITY, wbp_index_cache_.capacity_); + + LOG_INFO("test_expand_and_sparsify"); +} + +TEST_F(TestTmpFileWBPIndexCache, test_shrink) +{ + int ret = OB_SUCCESS; + uint32_t begin_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + uint32_t end_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + // 1. push indexes to trigger expand + int64_t page_num = 2 * INIT_BUCKET_ARRAY_CAPACITY * BUCKET_CAPACITY; + MockWBPIndexCache mock_cache(wbp_); + ret = write_and_push_pages(end_page_id, page_num, mock_cache, end_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = mock_cache.compare(wbp_index_cache_); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(2 * INIT_BUCKET_ARRAY_CAPACITY, wbp_index_cache_.size_); + ASSERT_EQ(2 * INIT_BUCKET_ARRAY_CAPACITY, wbp_index_cache_.capacity_); + begin_page_id = mock_cache.mock_index_cache_.at(0); + + // 2. free 7/8 indexes to trigger shrink + page_num = wbp_index_cache_.capacity_ * BUCKET_CAPACITY * 7 / 8; + ASSERT_GE(page_num, mock_cache.mock_index_cache_.count() - mock_cache.mock_index_cache_.count() / ObTmpFileWBPIndexCache::SHRINK_THRESHOLD); + ASSERT_LE(page_num, mock_cache.mock_index_cache_.count()); + uint32_t remove_page_id = mock_cache.mock_index_cache_.at(page_num - 1); + ret = truncate_and_free_pages(begin_page_id, page_num, mock_cache, begin_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = mock_cache.compare(wbp_index_cache_); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(INIT_BUCKET_ARRAY_CAPACITY, wbp_index_cache_.capacity_); + + // 3. push indexes to fill buckets with INIT_BUCKET_ARRAY_CAPACITY + page_num = INIT_BUCKET_ARRAY_CAPACITY * BUCKET_CAPACITY + - mock_cache.mock_index_cache_.count(); + ASSERT_LE(page_num, INIT_BUCKET_ARRAY_CAPACITY * BUCKET_CAPACITY); + ASSERT_GT(page_num, 0); + ret = write_and_push_pages(end_page_id, page_num, mock_cache, end_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = mock_cache.compare(wbp_index_cache_); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(INIT_BUCKET_ARRAY_CAPACITY, wbp_index_cache_.capacity_); + ASSERT_EQ(INIT_BUCKET_ARRAY_CAPACITY, wbp_index_cache_.size_); + ASSERT_EQ(wbp_index_cache_.size_ * BUCKET_CAPACITY, + mock_cache.mock_index_cache_.count()); + + // 4. free 7/8 indexes, but doesn't trigger shrink + page_num = wbp_index_cache_.capacity_ * BUCKET_CAPACITY * 7 / 8; + ASSERT_GE(page_num, mock_cache.mock_index_cache_.count() - mock_cache.mock_index_cache_.count() / ObTmpFileWBPIndexCache::SHRINK_THRESHOLD); + ASSERT_LE(page_num, mock_cache.mock_index_cache_.count()); + remove_page_id = mock_cache.mock_index_cache_.at(page_num - 1); + ret = truncate_and_free_pages(begin_page_id, page_num, mock_cache, begin_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = mock_cache.compare(wbp_index_cache_); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(INIT_BUCKET_ARRAY_CAPACITY, wbp_index_cache_.capacity_); + + LOG_INFO("test_shrink"); +} + +int mock_circle_bucket(ObTmpFileWBPIndexCache::ObTmpFilePageIndexBucket &bucket) +{ + int ret = OB_SUCCESS; + if (bucket.right_ < 0) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid bucket", KR(ret), K(bucket)); + } else if (bucket.left_ < bucket.right_) { + ObArray array; + for (int64_t i = bucket.left_; OB_SUCC(ret) && i <= bucket.right_; ++i) { + if (OB_FAIL(array.push_back(bucket.page_indexes_.at(i)))) { + LOG_WARN("fail to push back", KR(ret), K(i)); + } + } + if (OB_SUCC(ret)) { + int64_t mid = bucket.left_ + (bucket.right_ - bucket.left_) / 2; + for (int64_t i = 0; i < array.size(); ++i) { + int64_t new_pos = (bucket.left_ + i + mid) > bucket.right_ ? + bucket.left_ + i + mid - array.size() : + bucket.left_ + i + mid; + bucket.page_indexes_.at(new_pos) = array.at(i); + } + bucket.left_ = mid; + bucket.right_= mid - 1; + } + } + return ret; +} + +int mock_circle_cache(ObTmpFileWBPIndexCache &wbp_index_cache) +{ + int ret = OB_SUCCESS; + if (wbp_index_cache.right_ < 0) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid bucket", KR(ret), K(wbp_index_cache)); + } else if (wbp_index_cache.left_ < wbp_index_cache.right_) { + ObArray array; + for (int64_t i = wbp_index_cache.left_; OB_SUCC(ret) && i <= wbp_index_cache.right_; ++i) { + if (OB_ISNULL(wbp_index_cache.page_buckets_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr", KR(ret), K(wbp_index_cache)); + } else if (OB_ISNULL(wbp_index_cache.page_buckets_->at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr", KR(ret), K(wbp_index_cache)); + } else if (OB_FAIL(array.push_back(wbp_index_cache.page_buckets_->at(i)))) { + LOG_WARN("fail to push back", KR(ret), K(i)); + } + } + if (OB_SUCC(ret)) { + int64_t mid = wbp_index_cache.left_ + (wbp_index_cache.right_ - wbp_index_cache.left_) / 2; + for (int64_t i = 0; i < array.size(); ++i) { + int64_t new_pos = (wbp_index_cache.left_ + i + mid) > wbp_index_cache.right_ ? + wbp_index_cache.left_ + i + mid - array.size() : + wbp_index_cache.left_ + i + mid; + wbp_index_cache.page_buckets_->at(new_pos) = array.at(i); + } + wbp_index_cache.left_ = mid; + wbp_index_cache.right_= mid - 1; + } + } + + if (OB_SUCC(ret)) { + for (int64_t i = wbp_index_cache.left_; OB_SUCC(ret) && i <= wbp_index_cache.get_logic_tail_(); ++i) { + int64_t pos = i % wbp_index_cache.capacity_; + if (OB_ISNULL(wbp_index_cache.page_buckets_->at(pos))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr", KR(ret), K(wbp_index_cache)); + } else if (OB_FAIL(mock_circle_bucket(*wbp_index_cache.page_buckets_->at(pos)))) { + LOG_WARN("fail to mock circle bucket", KR(ret), K(wbp_index_cache)); + } + } + } + return ret; +} + +TEST_F(TestTmpFileWBPIndexCache, test_search) +{ + int ret = OB_SUCCESS; + uint32_t begin_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + uint32_t end_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + + // 1. push indexes until reach the max capacity + int64_t page_num = MAX_BUCKET_ARRAY_CAPACITY * BUCKET_CAPACITY; + MockWBPIndexCache mock_cache(wbp_); + ret = write_and_push_pages(end_page_id, page_num, mock_cache, end_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = mock_cache.compare(wbp_index_cache_); + ASSERT_EQ(OB_SUCCESS, ret); + begin_page_id = mock_cache.mock_index_cache_.at(0); + ASSERT_NE(begin_page_id, ObTmpFileGlobal::INVALID_PAGE_ID); + ret = mock_circle_cache(wbp_index_cache_); + ASSERT_EQ(OB_SUCCESS, ret); + ret = mock_cache.compare(wbp_index_cache_); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_GT(wbp_index_cache_.left_, wbp_index_cache_.right_); + ASSERT_GT(wbp_index_cache_.page_buckets_->at(wbp_index_cache_.left_)->left_, + wbp_index_cache_.page_buckets_->at(wbp_index_cache_.left_)->right_); + + // 2. push indexes to trigger sparsify + page_num = 20; + ret = write_and_push_pages(end_page_id, page_num, mock_cache, end_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = mock_cache.compare(wbp_index_cache_); + ASSERT_EQ(OB_SUCCESS, ret); + + // 3. truncate indexes to make the first bucket not be full + page_num = 10; + ASSERT_LE(page_num, mock_cache.mock_index_cache_.count()); + ret = truncate_pages(page_num, mock_cache); + ASSERT_EQ(OB_SUCCESS, ret); + ret = mock_cache.compare(wbp_index_cache_); + ASSERT_EQ(OB_SUCCESS, ret); + + // 4. search page_index which exists in wbp_index_cache + int64_t search_page_virtual_id = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + uint32_t search_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + uint32_t remove_page_id = mock_cache.mock_index_cache_.at(mock_cache.mock_index_cache_.count() / 5 + 1); + ret = wbp_.get_page_virtual_id(fd, remove_page_id, search_page_virtual_id); + ASSERT_EQ(OB_SUCCESS, ret); + ret = wbp_index_cache_.binary_search(search_page_virtual_id, search_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(search_page_id, remove_page_id); + + // 5. search page_index which doesn't exist in wbp_index_cache and whose virtual id is not in the range of wbp_index_cache. + // the range of cache is [the page_virtual_id of the first index, INFINITY) + ret = wbp_index_cache_.binary_search(1, search_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(ObTmpFileGlobal::INVALID_PAGE_ID, search_page_id); + + // 6. search page_index which doesn't exist in wbp_index_cache and whose virtual id is in the range of wbp_index_cache. + // the range of cache is [the page_virtual_id of the first index, INFINITY) + int64_t page_virtual_id1 = -1; + int64_t page_virtual_id2 = -1; + ret = wbp_.get_page_virtual_id(fd, mock_cache.mock_index_cache_.at(0), page_virtual_id1); + ASSERT_EQ(OB_SUCCESS, ret); + ret = wbp_.get_page_virtual_id(fd, mock_cache.mock_index_cache_.at(1), page_virtual_id2); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_GT(page_virtual_id2 - page_virtual_id1, 1); + search_page_virtual_id = page_virtual_id1 + 1; + ret = wbp_index_cache_.binary_search(page_virtual_id1, search_page_id); + ASSERT_EQ(OB_SUCCESS, ret); + int64_t res_virtual_id = -1; + ret = wbp_.get_page_virtual_id(fd, search_page_id, res_virtual_id); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(page_virtual_id1, res_virtual_id); + + LOG_INFO("test_search"); +} + +} // namespace oceanbase + +int main(int argc, char **argv) +{ + int ret = 0; + system("rm -f ./test_tmp_file_write_buffer_pool_index_cache.log*"); + OB_LOGGER.set_file_name("test_tmp_file_write_buffer_pool_index_cache.log", true); + OB_LOGGER.set_log_level("INFO"); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/src/observer/ob_server.cpp b/src/observer/ob_server.cpp index 07ce0ff91..ee898e0bc 100644 --- a/src/observer/ob_server.cpp +++ b/src/observer/ob_server.cpp @@ -76,6 +76,7 @@ #include "storage/compaction/ob_sstable_merge_info_mgr.h" #include "storage/tablelock/ob_table_lock_service.h" #include "storage/tx/ob_ts_mgr.h" +#include "storage/tmp_file/ob_tmp_file_cache.h" #include "storage/tx_table/ob_tx_data_cache.h" #include "storage/ob_file_system_router.h" #include "storage/ob_tablet_autoinc_seq_rpc_handler.h" @@ -419,6 +420,10 @@ int ObServer::init(const ObServerOptions &opts, const ObPLogWriterCfg &log_cfg) LOG_ERROR("init storage failed", KR(ret)); } else if (OB_FAIL(init_tx_data_cache())) { LOG_ERROR("init tx data cache failed", KR(ret)); + } else if (OB_FAIL(tmp_file::ObTmpBlockCache::get_instance().init("tmp_block_cache", 1))) { + LOG_ERROR("init tmp block cache failed", KR(ret)); + } else if (OB_FAIL(tmp_file::ObTmpPageCache::get_instance().init("tmp_page_cache", 1))) { + LOG_ERROR("init tmp page cache failed", KR(ret)); } else if (OB_FAIL(init_log_kv_cache())) { LOG_ERROR("init log kv cache failed", KR(ret)); } else if (OB_FAIL(locality_manager_.init(self_addr_, @@ -703,10 +708,6 @@ void ObServer::destroy() disk_usage_report_task_.destroy(); FLOG_INFO("tenant disk usage report task destroyed"); - FLOG_INFO("begin to destroy tmp file manager"); - ObTmpFileManager::get_instance().destroy(); - FLOG_INFO("tmp file manager destroyed"); - FLOG_INFO("begin to destroy disk usage report task"); TG_DESTROY(lib::TGDefIDs::DiskUseReport); FLOG_INFO("disk usage report task destroyed"); @@ -719,6 +720,14 @@ void ObServer::destroy() OB_TX_DATA_KV_CACHE.destroy(); FLOG_INFO("tx data kv cache destroyed"); + FLOG_INFO("begin to destroy tmp block cache"); + tmp_file::ObTmpBlockCache::get_instance().destroy(); + FLOG_INFO("tmp block cache destroyed"); + + FLOG_INFO("begin to destroy tmp page cache"); + tmp_file::ObTmpPageCache::get_instance().destroy(); + FLOG_INFO("tmp page cache destroyed"); + FLOG_INFO("begin to destroy log kv cache"); OB_LOG_KV_CACHE.destroy(); FLOG_INFO("log kv cache destroyed"); @@ -2917,8 +2926,6 @@ int ObServer::init_storage() storage_env_.bf_cache_miss_count_threshold_, storage_env_.storage_meta_cache_priority_))) { LOG_WARN("Fail to init OB_STORE_CACHE, ", KR(ret), K(storage_env_.data_dir_)); - } else if (OB_FAIL(ObTmpFileManager::get_instance().init())) { - LOG_WARN("fail to init temp file manager", KR(ret)); } else if (OB_FAIL(OB_SERVER_BLOCK_MGR.init(THE_IO_DEVICE, storage_env_.default_block_size_))) { LOG_ERROR("init server block mgr fail", KR(ret)); diff --git a/src/observer/omt/ob_multi_tenant.cpp b/src/observer/omt/ob_multi_tenant.cpp index 6353db43e..442faaa4e 100644 --- a/src/observer/omt/ob_multi_tenant.cpp +++ b/src/observer/omt/ob_multi_tenant.cpp @@ -165,6 +165,7 @@ #include "rootserver/mview/ob_mview_maintenance_service.h" #include "share/resource_limit_calculator/ob_resource_limit_calculator.h" #include "storage/checkpoint/ob_checkpoint_diagnose.h" +#include "storage/tmp_file/ob_tmp_file_manager.h" // ObTenantTmpFileManager #include "storage/restore/ob_tenant_restore_info_mgr.h" using namespace oceanbase; @@ -186,6 +187,7 @@ using namespace oceanbase::archive; using namespace oceanbase::observer; using namespace oceanbase::rootserver; using namespace oceanbase::blocksstable; +using namespace oceanbase::tmp_file; #define OB_TENANT_LOCK_BUCKET_NUM 10000L @@ -447,6 +449,7 @@ int ObMultiTenant::init(ObAddr myaddr, if (OB_SUCC(ret) && mtl_bind_flag) { MTL_BIND2(ObTenantIOManager::mtl_new, ObTenantIOManager::mtl_init, mtl_start_default, mtl_stop_default, nullptr, ObTenantIOManager::mtl_destroy); + MTL_BIND2(mtl_new_default, tmp_file::ObTenantTmpFileManager::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); // base mtl MTL_BIND2(mtl_new_default, storage::mds::ObTenantMdsService::mtl_init, storage::mds::ObTenantMdsService::mtl_start, storage::mds::ObTenantMdsService::mtl_stop, storage::mds::ObTenantMdsService::mtl_wait, mtl_destroy_default); diff --git a/src/observer/omt/ob_tenant.cpp b/src/observer/omt/ob_tenant.cpp index 85f1aa7cf..7f2e74bf9 100644 --- a/src/observer/omt/ob_tenant.cpp +++ b/src/observer/omt/ob_tenant.cpp @@ -1162,17 +1162,6 @@ void ObTenant::destroy() ObTenantSwitchGuard guard(this); print_all_thread("TENANT_BEFORE_DESTROY", id_); destroy_mtl_module(); - // 1.some mtl module(eg: ObDataAccessService) remove tmp file when destroy, - // so remove_tenant_file must be after destroy_mtl_module. - // 2.there is tg in ObTmpTenantMemBlockManager, so remove_tenant_file must be before - // ObTenantBase::destroy() in which tg leak is checked. - if (OB_TMP_FAIL(FILE_MANAGER_INSTANCE_V2.remove_tenant_file(id_))) { - if (OB_ENTRY_NOT_EXIST == tmp_ret) { - tmp_ret = OB_SUCCESS; - } else { - LOG_WARN_RET(tmp_ret, "fail to free tmp tenant file store", K(ret), K_(id)); - } - } ObTenantBase::destroy(); if (nullptr != multi_level_queue_) { diff --git a/src/rootserver/backup/ob_backup_table_list_mgr.cpp b/src/rootserver/backup/ob_backup_table_list_mgr.cpp index f425c9c3d..5e3139732 100644 --- a/src/rootserver/backup/ob_backup_table_list_mgr.cpp +++ b/src/rootserver/backup/ob_backup_table_list_mgr.cpp @@ -13,6 +13,7 @@ #define USING_LOG_PREFIX RS #include "ob_backup_table_list_mgr.h" +#include "storage/tmp_file/ob_tmp_file_manager.h" #include "storage/backup/ob_backup_data_store.h" #include "share/backup/ob_backup_io_adapter.h" @@ -354,10 +355,9 @@ int ObBackupTableListMgr::read_from_tmp_file_(const int64_t read_size, const int { int ret = OB_SUCCESS; table_list.reset(); - blocksstable::ObTmpFileIOInfo io_info; - blocksstable::ObTmpFileIOHandle handle; + tmp_file::ObTmpFileIOInfo io_info; + tmp_file::ObTmpFileIOHandle handle; io_info.fd_ = tmp_file_.get_fd(); - io_info.tenant_id_ = tmp_file_.get_tenant_id(); io_info.dir_id_ = tmp_file_.get_dir(); io_info.io_desc_.set_wait_event(2); io_info.size_ = read_size; @@ -370,7 +370,7 @@ int ObBackupTableListMgr::read_from_tmp_file_(const int64_t read_size, const int ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to alloc memory", K(ret), K(read_size)); } else if (FALSE_IT(io_info.buf_ = buf)) { - } else if (OB_FAIL(blocksstable::ObTmpFileManager::get_instance().pread(io_info, offset, handle))) { + } else if (OB_FAIL(MTL(tmp_file::ObTenantTmpFileManager*)->pread(io_info, offset, handle))) { LOG_WARN("failed to pread from tmp file", K(ret), K(io_info), K(offset), K(read_size)); } else { blocksstable::ObBufferReader buffer_reader(buf, read_size); diff --git a/src/share/io/ob_io_define.cpp b/src/share/io/ob_io_define.cpp index f31d49771..5211d2683 100644 --- a/src/share/io/ob_io_define.cpp +++ b/src/share/io/ob_io_define.cpp @@ -616,6 +616,29 @@ void ObIOResult::cancel() } } +int ObIOResult::wait(int64_t wait_ms) +{ + int ret = OB_SUCCESS; + ObThreadCondGuard guard(cond_); + if (OB_FAIL(guard.get_ret())) { + LOG_ERROR("fail to guard result condition", K(ret)); + } else { + int64_t begin_ms = ObTimeUtility::current_time(); + while (OB_SUCC(ret) && !is_finished_ && wait_ms > 0) { + if (OB_FAIL(cond_.wait(wait_ms))) { + LOG_WARN("fail to wait result condition", K(ret), K(wait_ms), K(*this)); + } else if (!is_finished_) { + int64_t duration_ms = ObTimeUtility::current_time() - begin_ms; + wait_ms -= duration_ms; + } + } + if (OB_UNLIKELY(wait_ms <= 0)) { // rarely happen + ret = OB_TIMEOUT; + } + } + return ret; +} + void ObIOResult::inc_ref(const char *msg) { (void)ATOMIC_FAA(&result_ref_cnt_, 1); @@ -1449,13 +1472,19 @@ bool ObIOHandle::is_valid() const return nullptr != result_; } -int ObIOHandle::wait() +int ObIOHandle::wait(const int64_t wait_timeout_ms) { int ret = OB_SUCCESS; if (OB_ISNULL(result_)) { ret = OB_NOT_INIT; LOG_WARN("The IOHandle has not been inited, ", K(ret)); - } else if (!result_->is_finished_) { + } else if (OB_FAIL(result_->ret_code_.io_ret_)) { + LOG_WARN("IO error, ", K(ret), K(*result_)); + } else if (result_->is_finished_) { + // do nothing + } else if (0 == wait_timeout_ms) { + ret = OB_EAGAIN; + } else if (UINT64_MAX == wait_timeout_ms) { const int64_t timeout_ms = ((result_->begin_ts_ > 0 ? result_->begin_ts_ + result_->timeout_us_ : 0) - ObTimeUtility::current_time()) / 1000L; ObWaitEventGuard wait_guard(result_->flag_.get_wait_event(), @@ -1464,24 +1493,10 @@ int ObIOHandle::wait() const int64_t real_wait_timeout = min(OB_IO_MANAGER.get_io_config().data_storage_io_timeout_ms_, timeout_ms); if (real_wait_timeout > 0) { - ObThreadCondGuard guard(result_->cond_); - if (OB_FAIL(guard.get_ret())) { - LOG_ERROR("fail to guard result condition", K(ret)); - } else { - int64_t wait_ms = real_wait_timeout; - int64_t begin_ms = ObTimeUtility::current_time(); - while (OB_SUCC(ret) && !result_->is_finished_ && wait_ms > 0) { - if (OB_FAIL(result_->cond_.wait(wait_ms))) { - LOG_WARN("fail to wait result condition", K(ret), K(wait_ms), K(*result_)); - } else if (!result_->is_finished_) { - int64_t duration_ms = ObTimeUtility::current_time() - begin_ms; - wait_ms = real_wait_timeout - duration_ms; - } - } - if (OB_UNLIKELY(wait_ms <= 0)) { // rarely happen - ret = OB_TIMEOUT; - LOG_WARN("fail to wait result condition due to spurious wakeup", - K(ret), K(wait_ms), K(*result_)); + int64_t wait_ms = real_wait_timeout; + if (OB_FAIL(result_->wait(wait_ms))) { + if (OB_TIMEOUT == ret) { // rarely happen + LOG_WARN("fail to wait result condition due to spurious wakeup", K(ret), K(wait_ms), K(*result_)); } } } else if (result_->is_finished_) { @@ -1489,6 +1504,16 @@ int ObIOHandle::wait() } else { ret = OB_TIMEOUT; } + } else { + int64_t wait_ms = wait_timeout_ms; + if (OB_FAIL(result_->wait(wait_ms))) { + if (OB_TIMEOUT == ret) { + const int64_t real_wait_timeout = min(OB_IO_MANAGER.get_io_config().data_storage_io_timeout_ms_, result_->timeout_us_ / 1000L); + if ((ObTimeUtility::current_time() - result_->begin_ts_) / 1000L < real_wait_timeout) { + ret = OB_EAGAIN; + } + } + } } if (OB_SUCC(ret)) { if (OB_FAIL(result_->ret_code_.io_ret_)) { diff --git a/src/share/io/ob_io_define.h b/src/share/io/ob_io_define.h index c412e12b7..b3652aaa5 100644 --- a/src/share/io/ob_io_define.h +++ b/src/share/io/ob_io_define.h @@ -296,6 +296,7 @@ public: void reset(); void destroy(); void cancel(); + int wait(int64_t wait_ms); void finish(const ObIORetCode &ret_code, ObIORequest *req = nullptr); void calc_io_offset_and_size(int64_t &size, int32_t &offset); ObIOMode get_mode() const; @@ -465,7 +466,7 @@ public: bool is_valid() const; OB_INLINE bool is_finished() const { return nullptr != result_ && result_->is_finished_; } - int wait(); + int wait(const int64_t wait_timeout_ms=UINT64_MAX); const char *get_buffer(); int64_t get_data_size() const; int64_t get_rt() const; diff --git a/src/share/ob_thread_define.h b/src/share/ob_thread_define.h index 6e2036499..c23a374ac 100755 --- a/src/share/ob_thread_define.h +++ b/src/share/ob_thread_define.h @@ -168,4 +168,6 @@ TG_DEF(StartupAccelHandler, StartupAccelHandler, QUEUE_THREAD, 1, observer::ObSt TG_DEF(TenantTTLManager, TTLManager, TIMER) TG_DEF(TenantTabletTTLMgr, TTLTabletMgr, TIMER) TG_DEF(TntSharedTimer, TntSharedTimer, TIMER) + +TG_DEF(TmpFileSwap, TFSwap, THREAD_POOL, 1) #endif diff --git a/src/share/rc/ob_tenant_base.h b/src/share/rc/ob_tenant_base.h index 693eb7d5d..170e06c2d 100755 --- a/src/share/rc/ob_tenant_base.h +++ b/src/share/rc/ob_tenant_base.h @@ -66,6 +66,9 @@ namespace blocksstable { class ObSharedMacroBlockMgr; class ObDecodeResourcePool; } +namespace tmp_file { + class ObTenantTmpFileManager; +} namespace storage { namespace mds { class ObTenantMdsService; @@ -257,6 +260,7 @@ using ObTableScanIteratorObjPool = common::ObServerObjectPoolon_write_io(rdtsc() - begin_io_dump_time); @@ -2128,7 +2126,6 @@ int ObChunkDatumStore::write_file(void *buf, int64_t size) LOG_WARN("open file failed", K(ret)); } else { file_size_ = 0; - io_.tenant_id_ = tenant_id_; io_.io_desc_.set_wait_event(ObWaitEventIds::ROW_STORE_DISK_WRITE); io_.io_timeout_ms_ = timeout_ms; LOG_INFO("open file success", K_(io_.fd), K_(io_.dir_id)); @@ -2157,7 +2154,7 @@ int ObChunkDatumStore::read_file( void *buf, const int64_t size, const int64_t offset, - blocksstable::ObTmpFileIOHandle &handle, + tmp_file::ObTmpFileIOHandle &handle, const int64_t file_size, const int64_t cur_pos, int64_t &tmp_file_size) @@ -2183,7 +2180,7 @@ int ObChunkDatumStore::read_file( CK (cur_pos >= file_size); OX (ret = OB_ITER_END); } else { - blocksstable::ObTmpFileIOInfo tmp_io = io_; + tmp_file::ObTmpFileIOInfo tmp_io = io_; set_io(size, static_cast(buf), tmp_io); tmp_io.io_desc_.set_wait_event(ObWaitEventIds::ROW_STORE_DISK_READ); tmp_io.io_timeout_ms_ = timeout_ms; @@ -2195,10 +2192,10 @@ int ObChunkDatumStore::read_file( if (OB_ITER_END != ret) { LOG_WARN("read form file failed", K(ret), K(tmp_io), K(offset), K(timeout_ms)); } - } else if (handle.get_data_size() != size) { + } else if (handle.get_done_size() != size) { ret = OB_INNER_STAT_ERROR; LOG_WARN("read data less than expected", - K(ret), K(tmp_io), "read_size", handle.get_data_size()); + K(ret), K(tmp_io), "read_size", handle.get_done_size()); } } return ret; @@ -2208,7 +2205,7 @@ int ObChunkDatumStore::aio_read_file( void *buf, const int64_t size, const int64_t offset, - blocksstable::ObTmpFileIOHandle &handle) + tmp_file::ObTmpFileIOHandle &handle) { int ret = OB_SUCCESS; if (!is_inited()) { @@ -2218,7 +2215,7 @@ int ObChunkDatumStore::aio_read_file( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(size), K(offset), KP(buf)); } else if (size > 0) { - blocksstable::ObTmpFileIOInfo tmp_io = io_; + tmp_file::ObTmpFileIOInfo tmp_io = io_; set_io(size, static_cast(buf), tmp_io); tmp_io.io_desc_.set_wait_event(ObWaitEventIds::ROW_STORE_DISK_READ); if (OB_FAIL(get_timeout(tmp_io.io_timeout_ms_))) { @@ -2750,8 +2747,11 @@ int ObChunkDatumStore::Iterator::aio_read(char *buf, const int64_t size) if (!aio_read_handle_.is_valid()) { // first read, wait write finish int64_t timeout_ms = 0; - OZ(store_->get_timeout(timeout_ms)); - OZ(store_->aio_write_handle_.wait()); + if (OB_FAIL(store_->get_timeout(timeout_ms))) { + LOG_WARN("fail to exec store_->get_timeout", K(ret)); + } else if (store_->aio_write_handle_.is_valid() && OB_FAIL(store_->aio_write_handle_.wait())) { + LOG_WARN("fail to exec store_->aio_write_handle_.wait", K(ret)); + } } if (OB_SUCC(ret)) { if (size <= 0 || cur_iter_pos_ >= file_size_) { diff --git a/src/sql/engine/basic/ob_chunk_datum_store.h b/src/sql/engine/basic/ob_chunk_datum_store.h index d3388f68d..9a94b5d45 100644 --- a/src/sql/engine/basic/ob_chunk_datum_store.h +++ b/src/sql/engine/basic/ob_chunk_datum_store.h @@ -22,7 +22,7 @@ #include "common/row/ob_row_iterator.h" #include "share/datum/ob_datum.h" #include "sql/engine/expr/ob_expr.h" -#include "storage/blocksstable/ob_tmp_file.h" +#include "storage/tmp_file/ob_tmp_file_manager.h" #include "sql/engine/basic/ob_sql_mem_callback.h" #include "sql/engine/basic/ob_batch_result_holder.h" @@ -831,7 +831,7 @@ public: // cp from chunk iter ObChunkDatumStore* store_; Block* cur_iter_blk_; - blocksstable::ObTmpFileIOHandle aio_read_handle_; + tmp_file::ObTmpFileIOHandle aio_read_handle_; int64_t cur_nth_blk_; //reading nth blk start from 1 int64_t cur_chunk_n_blocks_; //the number of blocks of current chunk int64_t cur_iter_pos_; //pos in file @@ -1136,16 +1136,16 @@ private: int write_file(void *buf, int64_t size); int read_file( - void *buf, const int64_t size, const int64_t offset, blocksstable::ObTmpFileIOHandle &handle, + void *buf, const int64_t size, const int64_t offset, tmp_file::ObTmpFileIOHandle &handle, const int64_t file_size, const int64_t cur_pos, int64_t &tmp_file_size); int aio_read_file(void *buf, const int64_t size, const int64_t offset, - blocksstable::ObTmpFileIOHandle &handle); + tmp_file::ObTmpFileIOHandle &handle); bool need_dump(int64_t extra_size); BlockBuffer* new_block(); void set_io(int64_t size, char *buf) { io_.size_ = size; io_.buf_ = buf; } - static void set_io(int64_t size, char *buf, blocksstable::ObTmpFileIOInfo &io) { io.size_ = size; io.buf_ = buf; } + static void set_io(int64_t size, char *buf, tmp_file::ObTmpFileIOInfo &io) { io.size_ = size; io.buf_ = buf; } bool find_block_can_hold(const int64_t size, bool &need_shrink); int get_store_row(RowIterator &it, const StoredRow *&sr); inline void callback_alloc(int64_t size) { if (callback_ != nullptr) callback_->alloc(size); } @@ -1175,7 +1175,7 @@ private: int64_t row_cnt_; int64_t col_count_; - blocksstable::ObTmpFileIOHandle aio_write_handle_; + tmp_file::ObTmpFileIOHandle aio_write_handle_; bool enable_dump_; bool has_dumped_; @@ -1184,7 +1184,7 @@ private: ObIOEventObserver *io_event_observer_; //int fd_; - blocksstable::ObTmpFileIOInfo io_; + tmp_file::ObTmpFileIOInfo io_; int64_t file_size_; int64_t n_block_in_file_; diff --git a/src/sql/engine/basic/ob_chunk_row_store.cpp b/src/sql/engine/basic/ob_chunk_row_store.cpp index 1f291c014..633a672e2 100644 --- a/src/sql/engine/basic/ob_chunk_row_store.cpp +++ b/src/sql/engine/basic/ob_chunk_row_store.cpp @@ -861,8 +861,6 @@ int ObChunkRowStore::finish_add_row(bool need_dump) LOG_WARN("finish_add_row dump error", K(ret)); } else if (OB_FAIL(get_timeout(timeout_ms))) { LOG_WARN("get timeout failed", K(ret)); - } else if (OB_FAIL(FILE_MANAGER_INSTANCE_V2.sync(io_.fd_, timeout_ms))) { - LOG_WARN("sync file failed", K(ret), K_(io_.fd), K(timeout_ms)); } } else { LOG_DEBUG("finish_add_row no need to dump", K(ret)); @@ -1627,7 +1625,6 @@ int ObChunkRowStore::write_file(void *buf, int64_t size) LOG_WARN("open file failed", K(ret)); } else { file_size_ = 0; - io_.tenant_id_ = tenant_id_; io_.io_desc_.set_wait_event(ObWaitEventIds::ROW_STORE_DISK_WRITE); io_.io_timeout_ms_ = timeout_ms; LOG_TRACE("open file success", K_(io_.fd), K_(io_.dir_id)); @@ -1674,7 +1671,7 @@ int ObChunkRowStore::read_file(void *buf, const int64_t size, const int64_t offs this->set_io(size, static_cast(buf)); io_.io_desc_.set_wait_event(ObWaitEventIds::ROW_STORE_DISK_READ); io_.io_timeout_ms_ = timeout_ms; - blocksstable::ObTmpFileIOHandle handle; + tmp_file::ObTmpFileIOHandle handle; if (0 == read_size && OB_FAIL(FILE_MANAGER_INSTANCE_V2.get_tmp_file_size(io_.fd_, tmp_file_size))) { LOG_WARN("failed to get tmp file size", K(ret)); @@ -1682,10 +1679,10 @@ int ObChunkRowStore::read_file(void *buf, const int64_t size, const int64_t offs if (OB_ITER_END != ret) { LOG_WARN("read form file failed", K(ret), K(io_), K(offset), K(timeout_ms)); } - } else if (handle.get_data_size() != size) { + } else if (handle.get_done_size() != size) { ret = OB_INNER_STAT_ERROR; LOG_WARN("read data less than expected", - K(ret), K(io_), "read_size", handle.get_data_size()); + K(ret), K(io_), "read_size", handle.get_done_size()); } } return ret; diff --git a/src/sql/engine/basic/ob_chunk_row_store.h b/src/sql/engine/basic/ob_chunk_row_store.h index c96e72878..b82d2ef8f 100644 --- a/src/sql/engine/basic/ob_chunk_row_store.h +++ b/src/sql/engine/basic/ob_chunk_row_store.h @@ -20,7 +20,7 @@ #include "lib/list/ob_dlist.h" #include "common/row/ob_row.h" #include "common/row/ob_row_iterator.h" -#include "storage/blocksstable/ob_tmp_file.h" +#include "storage/tmp_file/ob_tmp_file_manager.h" #include "sql/engine/basic/ob_sql_mem_callback.h" namespace oceanbase @@ -494,7 +494,7 @@ private: int64_t dumped_row_cnt_; //int fd_; - blocksstable::ObTmpFileIOInfo io_; + tmp_file::ObTmpFileIOInfo io_; int64_t file_size_; int64_t n_block_in_file_; diff --git a/src/sql/engine/basic/ob_material_op_impl.cpp b/src/sql/engine/basic/ob_material_op_impl.cpp index 2d64daf67..214101a9b 100644 --- a/src/sql/engine/basic/ob_material_op_impl.cpp +++ b/src/sql/engine/basic/ob_material_op_impl.cpp @@ -53,8 +53,8 @@ void ObMaterialOpImpl::reset() { sql_mem_processor_.unregister_profile(); io_event_observer_ = nullptr; - datum_store_.reset(); datum_store_it_.reset(); + datum_store_.reset(); got_first_row_ = false; inited_ = false; // can not destroy mem_entify here, the memory may hold by %iter_ or %datum_store_ diff --git a/src/sql/engine/basic/ob_ra_datum_store.cpp b/src/sql/engine/basic/ob_ra_datum_store.cpp index b6033435b..5c18edaaa 100644 --- a/src/sql/engine/basic/ob_ra_datum_store.cpp +++ b/src/sql/engine/basic/ob_ra_datum_store.cpp @@ -14,7 +14,7 @@ #include "ob_ra_datum_store.h" #include "lib/container/ob_se_array_iterator.h" -#include "storage/blocksstable/ob_tmp_file.h" +#include "storage/tmp_file/ob_tmp_file_manager.h" #include "lib/utility/ob_tracepoint.h" #include "share/config/ob_server_config.h" #include "sql/engine/basic/ob_chunk_datum_store.h" @@ -1144,11 +1144,10 @@ int ObRADatumStore::write_file(BlockIndex &bi, void *buf, int64_t size) if (NULL != mem_stat_) { mem_stat_->dumped(size); } - blocksstable::ObTmpFileIOInfo io; + tmp_file::ObTmpFileIOInfo io; io.fd_ = fd_; io.buf_ = static_cast(buf); io.size_ = size; - io.tenant_id_ = tenant_id_; io.io_desc_.set_wait_event(ObWaitEventIds::ROW_STORE_DISK_WRITE); io.io_timeout_ms_ = timeout_ms; const uint64_t start = rdtsc(); @@ -1184,22 +1183,21 @@ int ObRADatumStore::read_file(void *buf, const int64_t size, const int64_t offse } if (OB_SUCC(ret) && size > 0) { - blocksstable::ObTmpFileIOInfo io; + tmp_file::ObTmpFileIOInfo io; io.fd_ = fd_; io.dir_id_ = dir_id_; io.buf_ = static_cast(buf); io.size_ = size; - io.tenant_id_ = tenant_id_; io.io_desc_.set_wait_event(ObWaitEventIds::ROW_STORE_DISK_READ); io.io_timeout_ms_ = timeout_ms; const uint64_t start = rdtsc(); - blocksstable::ObTmpFileIOHandle handle; + tmp_file::ObTmpFileIOHandle handle; if (OB_FAIL(FILE_MANAGER_INSTANCE_V2.pread(io, offset, handle))) { LOG_WARN("read form file failed", K(ret), K(io), K(offset), K(timeout_ms)); - } else if (OB_UNLIKELY(handle.get_data_size() != size)) { + } else if (OB_UNLIKELY(handle.get_done_size() != size)) { ret = OB_INNER_STAT_ERROR; LOG_WARN("read data less than expected", - K(ret), K(io), "read_size", handle.get_data_size()); + K(ret), K(io), "read_size", handle.get_done_size()); } if (NULL != io_observer_) { io_observer_->on_read_io(rdtsc() - start); @@ -1323,8 +1321,6 @@ int ObRADatumStore::finish_add_row() LOG_WARN("write last index block to file failed", K(ret), K(ret)); } else if (OB_FAIL(get_timeout(timeout_ms))) { LOG_WARN("get timeout failed", K(ret)); - } else if (OB_FAIL(FILE_MANAGER_INSTANCE_V2.sync(fd_, timeout_ms))) { - LOG_WARN("sync file failed", K(ret), K(fd_), K(timeout_ms)); } else { if (blkbuf_.buf_.is_inited()) { free_blk_mem(blkbuf_.buf_.data(), blkbuf_.buf_.capacity()); diff --git a/src/sql/engine/basic/ob_ra_row_store.cpp b/src/sql/engine/basic/ob_ra_row_store.cpp index 0b3469459..e6abade66 100644 --- a/src/sql/engine/basic/ob_ra_row_store.cpp +++ b/src/sql/engine/basic/ob_ra_row_store.cpp @@ -14,7 +14,7 @@ #include "ob_ra_row_store.h" #include "lib/container/ob_se_array_iterator.h" -#include "storage/blocksstable/ob_tmp_file.h" +#include "storage/tmp_file/ob_tmp_file_manager.h" #include "lib/utility/ob_tracepoint.h" #include "share/config/ob_server_config.h" @@ -962,11 +962,10 @@ int ObRARowStore::write_file(BlockIndex &bi, void *buf, int64_t size) ret = OB_E(EventTable::EN_8) ret; } if (OB_SUCC(ret) && size > 0) { - blocksstable::ObTmpFileIOInfo io; + tmp_file::ObTmpFileIOInfo io; io.fd_ = fd_; io.buf_ = static_cast(buf); io.size_ = size; - io.tenant_id_ = tenant_id_; io.io_desc_.set_wait_event(ObWaitEventIds::ROW_STORE_DISK_WRITE); io.io_timeout_ms_ = timeout_ms; if (OB_FAIL(FILE_MANAGER_INSTANCE_V2.write(io))) { @@ -996,21 +995,20 @@ int ObRARowStore::read_file(void *buf, const int64_t size, const int64_t offset) } if (OB_SUCC(ret) && size > 0) { - blocksstable::ObTmpFileIOInfo io; + tmp_file::ObTmpFileIOInfo io; io.fd_ = fd_; io.dir_id_ = dir_id_; io.buf_ = static_cast(buf); io.size_ = size; - io.tenant_id_ = tenant_id_; io.io_desc_.set_wait_event(ObWaitEventIds::ROW_STORE_DISK_READ); io.io_timeout_ms_ = timeout_ms; - blocksstable::ObTmpFileIOHandle handle; + tmp_file::ObTmpFileIOHandle handle; if (OB_FAIL(FILE_MANAGER_INSTANCE_V2.pread(io, offset, handle))) { LOG_WARN("read form file failed", K(ret), K(io), K(offset), K(timeout_ms)); - } else if (handle.get_data_size() != size) { + } else if (handle.get_done_size() != size) { ret = OB_INNER_STAT_ERROR; LOG_WARN("read data less than expected", - K(ret), K(io), "read_size", handle.get_data_size()); + K(ret), K(io), "read_size", handle.get_done_size()); } } return ret; @@ -1104,8 +1102,6 @@ int ObRARowStore::finish_add_row() LOG_WARN("write last index block to file failed", K(ret), K(ret)); } else if (OB_FAIL(get_timeout(timeout_ms))) { LOG_WARN("get timeout failed", K(ret)); - } else if (OB_FAIL(FILE_MANAGER_INSTANCE_V2.sync(fd_, timeout_ms))) { - LOG_WARN("sync file failed", K(ret), K(fd_), K(timeout_ms)); } else { if (blkbuf_.buf_.is_inited()) { free_blk_mem(blkbuf_.buf_.data(), blkbuf_.buf_.capacity()); diff --git a/src/sql/engine/basic/ob_temp_block_store.cpp b/src/sql/engine/basic/ob_temp_block_store.cpp index ecf87bd82..887890964 100644 --- a/src/sql/engine/basic/ob_temp_block_store.cpp +++ b/src/sql/engine/basic/ob_temp_block_store.cpp @@ -14,7 +14,7 @@ #include "ob_temp_block_store.h" #include "lib/container/ob_se_array_iterator.h" -#include "storage/blocksstable/ob_tmp_file.h" +#include "storage/tmp_file/ob_tmp_file_manager.h" #include "lib/utility/ob_tracepoint.h" #include "share/config/ob_server_config.h" #include "sql/engine/basic/ob_chunk_datum_store.h" @@ -205,8 +205,6 @@ int ObTempBlockStore::finish_add_row(bool need_dump /*true*/) LOG_WARN("get timeout failed", K(ret)); } else if (write_io_handle_.is_valid() && OB_FAIL(write_io_handle_.wait())) { LOG_WARN("fail to wait write", K(ret), K(write_io_handle_)); - } else if (OB_FAIL(FILE_MANAGER_INSTANCE_V2.sync(io_.fd_, timeout_ms))) { - LOG_WARN("sync file failed", K(ret), K(io_.fd_), K(timeout_ms)); } if (OB_LIKELY(nullptr != io_observer_)) { io_observer_->on_write_io(rdtsc() - begin_io_dump_time); @@ -413,7 +411,7 @@ int ObTempBlockStore::decompr_block(BlockReader &reader, const Block *&blk) ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpeteced null pointer", K(ret), KP(blk), KP(reader.buf_.data())); } else { - int64_t comp_size = reader.read_io_handle_.get_data_size() - sizeof(Block); + int64_t comp_size = reader.read_io_handle_.get_done_size() - sizeof(Block); int64_t decomp_size = blk->raw_size_ - sizeof(Block); int64_t actual_uncomp_size = 0; if (OB_FAIL(ensure_reader_buffer(reader, reader.decompr_buf_, blk->raw_size_))) { @@ -1076,7 +1074,6 @@ int ObTempBlockStore::write_file(BlockIndex &bi, void *buf, int64_t size) LOG_WARN("open file failed", K(ret)); } else { file_size_ = 0; - io_.tenant_id_ = tenant_id_; io_.io_desc_.set_wait_event(ObWaitEventIds::ROW_STORE_DISK_WRITE); io_.io_timeout_ms_ = timeout_ms; LOG_INFO("open file success", K_(io_.fd), K_(io_.dir_id), K(get_compressor_type())); @@ -1109,7 +1106,7 @@ int ObTempBlockStore::write_file(BlockIndex &bi, void *buf, int64_t size) } int ObTempBlockStore::read_file(void *buf, const int64_t size, const int64_t offset, - blocksstable::ObTmpFileIOHandle &handle, const bool is_async) + tmp_file::ObTmpFileIOHandle &handle, const bool is_async) { int ret = OB_SUCCESS; int64_t timeout_ms = 0; @@ -1118,12 +1115,12 @@ int ObTempBlockStore::read_file(void *buf, const int64_t size, const int64_t off LOG_WARN("invalid argument", K(size), K(offset), KP(buf)); } else if (OB_FAIL(get_timeout(timeout_ms))) { LOG_WARN("get timeout failed", K(ret)); - } else if (!handle.is_valid() && OB_FAIL(write_io_handle_.wait())) { + } else if (!handle.is_valid() && write_io_handle_.is_valid() && OB_FAIL(write_io_handle_.wait())) { LOG_WARN("fail to wait write", K(ret)); } if (OB_SUCC(ret) && size > 0) { - blocksstable::ObTmpFileIOInfo tmp_read_id = io_; + tmp_file::ObTmpFileIOInfo tmp_read_id = io_; tmp_read_id.buf_ = static_cast(buf); tmp_read_id.size_ = size; tmp_read_id.io_desc_.set_wait_event(ObWaitEventIds::ROW_STORE_DISK_READ); @@ -1136,10 +1133,10 @@ int ObTempBlockStore::read_file(void *buf, const int64_t size, const int64_t off } else { if (OB_FAIL(FILE_MANAGER_INSTANCE_V2.pread(tmp_read_id, offset, handle))) { LOG_WARN("read form file failed", K(ret), K(tmp_read_id), K(offset), K(timeout_ms)); - } else if (OB_UNLIKELY(handle.get_data_size() != size)) { + } else if (OB_UNLIKELY(handle.get_done_size() != size)) { ret = OB_INNER_STAT_ERROR; LOG_WARN("read data less than expected", K(ret), K(tmp_read_id), - "read_size", handle.get_data_size()); + "read_size", handle.get_done_size()); } } if (NULL != io_observer_) { @@ -1421,7 +1418,6 @@ void ObTempBlockStore::BlockReader::reuse() buf_.reset(); decompr_buf_.reset(); } - read_io_handle_.set_last_extent_id(0); } void ObTempBlockStore::BlockReader::reset_cursor(const int64_t file_size, const bool need_release) diff --git a/src/sql/engine/basic/ob_temp_block_store.h b/src/sql/engine/basic/ob_temp_block_store.h index cb1dc99db..0bcdcb5db 100644 --- a/src/sql/engine/basic/ob_temp_block_store.h +++ b/src/sql/engine/basic/ob_temp_block_store.h @@ -20,7 +20,7 @@ #include "sql/engine/basic/ob_sql_mem_callback.h" #include "lib/checksum/ob_crc64.h" #include "sql/engine/basic/chunk_store/ob_chunk_block_compressor.h" -#include "storage/blocksstable/ob_tmp_file.h" +#include "storage/tmp_file/ob_tmp_file_manager.h" namespace oceanbase { @@ -289,7 +289,7 @@ public: inline int64_t get_block_cnt() const { return store_->get_block_cnt(); } void set_iteration_age(IterationAge *age) { age_ = age; } void set_blk_holder(BlockHolder *holder) { blk_holder_ptr_ = holder; } - blocksstable::ObTmpFileIOHandle& get_read_io_handler() { return read_io_handle_; } + tmp_file::ObTmpFileIOHandle& get_read_io_handler() { return read_io_handle_; } inline bool is_async() const { return is_async_; } void reset(); void reuse(); @@ -329,7 +329,7 @@ public: IterationAge inner_age_; // to optimize performance, record the last_extent_id to avoid do binary search every time // calling read. - blocksstable::ObTmpFileIOHandle read_io_handle_; + tmp_file::ObTmpFileIOHandle read_io_handle_; int64_t cur_file_offset_; bool is_async_; int aio_buf_idx_; @@ -507,7 +507,7 @@ private: int ensure_reader_buffer(BlockReader &reader, ShrinkBuffer &buf, const int64_t size); int write_file(BlockIndex &bi, void *buf, int64_t size); int read_file(void *buf, const int64_t size, const int64_t offset, - blocksstable::ObTmpFileIOHandle &handle, const bool is_async); + tmp_file::ObTmpFileIOHandle &handle, const bool is_async); int dump_block_if_need(const int64_t extra_size); bool need_dump(const int64_t extra_size); int write_compressed_block(Block *blk, BlockIndex *bi); @@ -590,8 +590,8 @@ private: ObSqlMemoryCallback *mem_stat_; ObChunkBlockCompressor compressor_; ObIOEventObserver *io_observer_; - blocksstable::ObTmpFileIOHandle write_io_handle_; - blocksstable::ObTmpFileIOInfo io_; + tmp_file::ObTmpFileIOHandle write_io_handle_; + tmp_file::ObTmpFileIOInfo io_; bool last_block_on_disk_; int64_t cur_file_offset_; diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 6d263d2c5..23ac9cb28 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -43,9 +43,6 @@ ob_set_subtarget(ob_storage blocksstable blocksstable/ob_sstable_printer.cpp blocksstable/ob_storage_cache_suite.cpp blocksstable/ob_super_block_buffer_holder.cpp - blocksstable/ob_tmp_file.cpp - blocksstable/ob_tmp_file_cache.cpp - blocksstable/ob_tmp_file_store.cpp blocksstable/ob_datum_row.cpp blocksstable/ob_datum_rowkey.cpp blocksstable/ob_data_store_desc.cpp @@ -142,6 +139,27 @@ ob_set_subtarget(ob_storage blocksstable_index_block blocksstable/index_block/ob_sstable_sec_meta_iterator.cpp ) +ob_set_subtarget(ob_storage tmp_file + tmp_file/ob_shared_nothing_tmp_file.cpp + tmp_file/ob_tmp_file_write_buffer_index_cache.cpp + tmp_file/ob_tmp_file_cache.cpp + tmp_file/ob_tmp_file_io_define.cpp + tmp_file/ob_tmp_file_io_ctx.cpp + tmp_file/ob_tmp_file_global.cpp + tmp_file/ob_tmp_file_block_manager.cpp + tmp_file/ob_tmp_file_manager.cpp + tmp_file/ob_tmp_file_eviction_manager.cpp + tmp_file/ob_tmp_file_flush_priority_manager.cpp + tmp_file/ob_tmp_file_flush_list_iterator.cpp + tmp_file/ob_tmp_file_meta_tree.cpp + tmp_file/ob_tmp_file_flush_manager.cpp + tmp_file/ob_tmp_file_flush_ctx.cpp + tmp_file/ob_tmp_file_page_cache_controller.cpp + tmp_file/ob_tmp_file_thread_wrapper.cpp + tmp_file/ob_tmp_file_thread_job.cpp + tmp_file/ob_tmp_file_write_buffer_pool.cpp +) + ob_set_subtarget(ob_storage slog slog/ob_server_slog_writer.cpp slog/ob_storage_log_batch_header.cpp diff --git a/src/storage/backup/ob_backup_tmp_file.cpp b/src/storage/backup/ob_backup_tmp_file.cpp index 7be81a84d..ab4d73b5d 100644 --- a/src/storage/backup/ob_backup_tmp_file.cpp +++ b/src/storage/backup/ob_backup_tmp_file.cpp @@ -16,6 +16,7 @@ #include "lib/oblog/ob_log_module.h" using namespace oceanbase::blocksstable; +using namespace oceanbase::tmp_file; using namespace oceanbase::storage; namespace oceanbase { @@ -40,9 +41,9 @@ int ObBackupTmpFile::open(const uint64_t tenant_id) if (is_opened_) { ret = OB_INIT_TWICE; LOG_WARN("backup tmp file init twice", K(ret)); - } else if (OB_FAIL(ObTmpFileManager::get_instance().alloc_dir(file_dir_))) { + } else if (OB_FAIL(ObTenantTmpFileManager::get_instance().alloc_dir(file_dir_))) { LOG_WARN("failed to alloc dir", K(ret)); - } else if (OB_FAIL(ObTmpFileManager::get_instance().open(file_fd_, file_dir_))) { + } else if (OB_FAIL(ObTenantTmpFileManager::get_instance().open(file_fd_, file_dir_))) { LOG_WARN("failed to open tmp file", K(ret), K(file_dir_)); } else { tenant_id_ = tenant_id; @@ -63,7 +64,7 @@ int ObBackupTmpFile::write(const char *buf, const int64_t size) LOG_WARN("backup tmp file init twice", K(ret)); } else if (OB_FAIL(get_io_info_(buf, size, timeout_ms, io_info))) { LOG_WARN("failed to get io info", K(ret), K(buf), K(size)); - } else if (OB_FAIL(ObTmpFileManager::get_instance().write(io_info))) { + } else if (OB_FAIL(ObTenantTmpFileManager::get_instance().write(io_info))) { LOG_WARN("failed to write tmp file", K(ret), K(io_info), K(timeout_ms)); } else { file_size_ += size; @@ -78,7 +79,7 @@ int ObBackupTmpFile::close() if (!is_opened_) { ret = OB_NOT_INIT; LOG_WARN("backup tmp file do not init", K(ret)); - } else if (OB_FAIL(ObTmpFileManager::get_instance().remove(file_fd_))) { + } else if (OB_FAIL(ObTenantTmpFileManager::get_instance().remove(file_fd_))) { LOG_WARN("failed to remove tmp file fd", K(ret), K(file_fd_)); } else { is_opened_ = false; @@ -92,7 +93,7 @@ int ObBackupTmpFile::get_io_info_(const char *buf, const int64_t size, const int int ret = OB_SUCCESS; io_info.reset(); io_info.fd_ = file_fd_; - io_info.tenant_id_ = tenant_id_; + io_info.dir_id_ = file_dir_; io_info.io_desc_.set_wait_event(2); io_info.buf_ = const_cast(buf); io_info.size_ = size; diff --git a/src/storage/backup/ob_backup_tmp_file.h b/src/storage/backup/ob_backup_tmp_file.h index 63a335826..f53363b33 100644 --- a/src/storage/backup/ob_backup_tmp_file.h +++ b/src/storage/backup/ob_backup_tmp_file.h @@ -14,7 +14,7 @@ #define STORAGE_LOG_STREAM_BACKUP_TMP_FILE_H_ #include "storage/backup/ob_backup_data_struct.h" -#include "storage/blocksstable/ob_tmp_file.h" +#include "storage/tmp_file/ob_tmp_file_manager.h" #include "storage/blocksstable/ob_data_buffer.h" namespace oceanbase { @@ -49,7 +49,7 @@ public: TO_STRING_KV(K_(is_opened), K_(tenant_id), K_(file_dir), K_(file_fd), K_(file_size)); private: - int get_io_info_(const char *buf, const int64_t size, const int64_t timeout_ms, blocksstable::ObTmpFileIOInfo &io_info); + int get_io_info_(const char *buf, const int64_t size, const int64_t timeout_ms, tmp_file::ObTmpFileIOInfo &io_info); private: bool is_opened_; @@ -139,10 +139,9 @@ int ObBackupIndexBufferNode::get_backup_index(T &backup_index) backup_index.reset(); const int64_t need_read_size = sizeof(T); const int64_t timeout_ms = 5000; - blocksstable::ObTmpFileIOInfo io_info; - blocksstable::ObTmpFileIOHandle handle; + tmp_file::ObTmpFileIOInfo io_info; + tmp_file::ObTmpFileIOHandle handle; io_info.fd_ = tmp_file_.get_fd(); - io_info.tenant_id_ = tmp_file_.get_tenant_id(); io_info.io_desc_.set_wait_event(2); io_info.size_ = std::min(need_read_size, estimate_size_ - read_offset_); io_info.io_timeout_ms_ = timeout_ms; @@ -158,7 +157,7 @@ int ObBackupIndexBufferNode::get_backup_index(T &backup_index) ret = OB_ALLOCATE_MEMORY_FAILED; OB_LOG(WARN, "failed to alloc memory", K(ret), K(need_read_size)); } else if (FALSE_IT(io_info.buf_ = buf)) { - } else if (OB_FAIL(blocksstable::ObTmpFileManager::get_instance().pread(io_info, read_offset_, handle))) { + } else if (OB_FAIL(tmp_file::ObTenantTmpFileManager::get_instance().pread(io_info, read_offset_, handle))) { OB_LOG(WARN, "failed to pread from tmp file", K(ret), K(io_info), K_(read_offset), K(need_read_size)); } else { blocksstable::ObBufferReader buffer_reader(buf, need_read_size); diff --git a/src/storage/blocksstable/ob_block_manager.cpp b/src/storage/blocksstable/ob_block_manager.cpp index 182c19613..c3239d91f 100644 --- a/src/storage/blocksstable/ob_block_manager.cpp +++ b/src/storage/blocksstable/ob_block_manager.cpp @@ -27,7 +27,7 @@ #include "storage/blocksstable/ob_block_manager.h" #include "storage/blocksstable/ob_macro_block_struct.h" #include "storage/blocksstable/ob_sstable_meta.h" -#include "storage/blocksstable/ob_tmp_file_store.h" +#include "storage/tmp_file/ob_tmp_file_manager.h" #include "storage/slog_ckpt/ob_server_checkpoint_slog_handler.h" #include "storage/slog_ckpt/ob_tenant_checkpoint_slog_handler.h" #include "storage/meta_mem/ob_tenant_meta_mem_mgr.h" @@ -41,6 +41,7 @@ using namespace oceanbase::common; using namespace oceanbase::common::hash; using namespace oceanbase::blocksstable; +using namespace oceanbase::tmp_file; using namespace oceanbase::storage; using namespace oceanbase::share; @@ -1493,18 +1494,29 @@ int ObBlockManager::mark_tmp_file_blocks( ObMacroBlockMarkerStatus &tmp_status) { int ret = OB_SUCCESS; - ObArray macro_block_list; + omt::ObMultiTenant *omt = GCTX.omt_; + common::ObSEArray mtl_tenant_ids; - if (OB_FAIL(macro_block_list.reserve(DEFAULT_PENDING_FREE_COUNT))) { - LOG_WARN("fail to reserve macro block list", K(ret)); - } else if (OB_FAIL(OB_TMP_FILE_STORE.get_macro_block_list(macro_block_list))) { - LOG_WARN("fail to get macro block list", K(ret)); - } else if (OB_FAIL(update_mark_info(macro_block_list, macro_id_set, mark_info))){ - LOG_WARN("fail to update mark info", K(ret), K(macro_block_list.count())); - } else { - tmp_status.tmp_file_count_ += macro_block_list.count(); - tmp_status.hold_count_ -= macro_block_list.count(); + omt->get_mtl_tenant_ids(mtl_tenant_ids); + for (int64_t i = 0; OB_SUCC(ret) && i < mtl_tenant_ids.count(); i++) { + const uint64_t tenant_id = mtl_tenant_ids.at(i); + MTL_SWITCH(tenant_id) { + ObArray macro_block_list; + if (OB_FAIL(set_group_id(tenant_id))) { + LOG_WARN("isolate CPU and IOPS failed", K(ret)); + } else if (OB_FAIL(mark_tenant_blocks(mark_info, macro_id_set, tmp_status))) { + LOG_WARN("fail to mark tenant blocks", K(ret), K(tenant_id)); + } else if (OB_FALSE_IT(MTL(ObTenantTmpFileManager*)->get_macro_block_list(macro_block_list))) { + LOG_WARN("fail to get macro block list", K(ret)); + } else if (OB_FAIL(update_mark_info(macro_block_list, macro_id_set, mark_info))){ + LOG_WARN("fail to update mark info", K(ret), K(macro_block_list.count())); + } else { + tmp_status.tmp_file_count_ += macro_block_list.count(); + tmp_status.hold_count_ -= macro_block_list.count(); + } + } } + return ret; } diff --git a/src/storage/blocksstable/ob_macro_block_handle.cpp b/src/storage/blocksstable/ob_macro_block_handle.cpp index 0edeefcc9..9f8e439a5 100644 --- a/src/storage/blocksstable/ob_macro_block_handle.cpp +++ b/src/storage/blocksstable/ob_macro_block_handle.cpp @@ -221,18 +221,20 @@ int ObMacroBlockHandle::async_write(const ObMacroBlockWriteInfo &write_info) return ret; } -int ObMacroBlockHandle::wait() +int ObMacroBlockHandle::wait(const int64_t wait_timeout_ms) { int ret = OB_SUCCESS; if (io_handle_.is_empty()) { // do nothing - } else if (OB_FAIL(io_handle_.wait())) { - LOG_WARN("fail to wait block io, may be retry", K(macro_id_), K(ret)); - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = report_bad_block())) { - LOG_WARN("fail to report bad block", K(tmp_ret), K(ret)); + } else if (OB_FAIL(io_handle_.wait(wait_timeout_ms))) { + if (OB_EAGAIN != ret) { + LOG_WARN("fail to wait block io, may be retry", K(macro_id_), K(ret)); + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = report_bad_block())) { + LOG_WARN("fail to report bad block", K(tmp_ret), K(ret)); + } + io_handle_.reset(); } - io_handle_.reset(); } return ret; } diff --git a/src/storage/blocksstable/ob_macro_block_handle.h b/src/storage/blocksstable/ob_macro_block_handle.h index 777d9d2fa..82d438b81 100644 --- a/src/storage/blocksstable/ob_macro_block_handle.h +++ b/src/storage/blocksstable/ob_macro_block_handle.h @@ -45,7 +45,7 @@ public: int async_read(const ObMacroBlockReadInfo &read_info); int async_write(const ObMacroBlockWriteInfo &write_info); int set_macro_block_id(const MacroBlockId ¯o_block_id); - int wait(); + int wait(const int64_t wait_timeout_ms=UINT64_MAX); TO_STRING_KV(K_(macro_id), K_(io_handle)); private: int report_bad_block() const; diff --git a/src/storage/blocksstable/ob_tmp_file.cpp b/src/storage/blocksstable/ob_tmp_file.cpp deleted file mode 100644 index b7dfd6c59..000000000 --- a/src/storage/blocksstable/ob_tmp_file.cpp +++ /dev/null @@ -1,1836 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#include "ob_tmp_file.h" -#include "ob_tmp_file_cache.h" -#include "observer/ob_server_struct.h" -#include "share/ob_task_define.h" - -namespace oceanbase -{ -using namespace storage; -using namespace share; - -namespace blocksstable -{ - -ObTmpFileIOInfo::ObTmpFileIOInfo() - : fd_(0), dir_id_(0), size_(0), io_timeout_ms_(DEFAULT_IO_WAIT_TIME_MS), - tenant_id_(OB_INVALID_TENANT_ID), buf_(NULL), io_desc_(), - disable_page_cache_(false) -{ -} - -ObTmpFileIOInfo::~ObTmpFileIOInfo() -{ -} - -void ObTmpFileIOInfo::reset() -{ - fd_ = 0; - dir_id_ = 0; - size_ = 0; - io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; - tenant_id_ = OB_INVALID_TENANT_ID; - buf_ = NULL; -} - -bool ObTmpFileIOInfo::is_valid() const -{ - return fd_ >= 0 && dir_id_ >= 0 && size_ > 0 && OB_INVALID_TENANT_ID != tenant_id_ - && NULL != buf_ && io_desc_.is_valid() && io_timeout_ms_ > 0; -} - -ObTmpFileIOHandle::ObTmpFileIOHandle() - : io_handles_(), - page_cache_handles_(), - block_cache_handles_(), - write_block_ids_(), - fd_(OB_INVALID_FD), - dir_id_(OB_INVALID_ID), - tenant_id_(OB_INVALID_TENANT_ID), - buf_(NULL), - size_(0), - is_read_(false), - has_wait_(false), - is_finished_(false), - disable_page_cache_(false), - ret_code_(OB_SUCCESS), - expect_read_size_(0), - last_read_offset_(-1), - io_flag_(), - update_offset_in_file_(false), - last_fd_(OB_INVALID_FD), - last_extent_id_(0) -{ - io_handles_.set_attr(ObMemAttr(MTL_ID(), "TMP_IO_HDL")); - page_cache_handles_.set_attr(ObMemAttr(MTL_ID(), "TMP_PCACHE_HDL")); - block_cache_handles_.set_attr(ObMemAttr(MTL_ID(), "TMP_BCACHE_HDL")); -} - -ObTmpFileIOHandle::~ObTmpFileIOHandle() -{ - reset(); -} - -int ObTmpFileIOHandle::prepare_read( - const int64_t read_size, - const int64_t read_offset, - const common::ObIOFlag io_flag, - char *read_buf, - int64_t fd, - int64_t dir_id, - uint64_t tenant_id, - bool disable_page_cache) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(read_buf)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), KP_(buf)); - } else { - buf_ = read_buf; - size_ = 0; - fd_ = fd; - dir_id_ = dir_id; - tenant_id_ = tenant_id; - is_read_ = true; - has_wait_ = false; - expect_read_size_ = read_size; - last_read_offset_ = read_offset; - io_flag_ = io_flag; - disable_page_cache_ = disable_page_cache; - if (last_fd_ != fd_) { - last_fd_ = fd_; - last_extent_id_ = 0; - } - } - return ret; -} - -int ObTmpFileIOHandle::prepare_write( - char *write_buf, - const int64_t write_size, - int64_t fd, - int64_t dir_id, - uint64_t tenant_id) -{ - int ret = OB_SUCCESS; - const int64_t bkt_cnt = 17; - lib::ObMemAttr bkt_mem_attr(tenant_id, "TmpBlkIDBkt"); - if (OB_ISNULL(write_buf)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), KP_(buf)); - } else if (OB_FAIL(write_block_ids_.create(bkt_cnt, bkt_mem_attr))) { - STORAGE_LOG(WARN, "create write block id set failed", K(ret), K(bkt_cnt)); - } else { - buf_ = write_buf; - size_ = write_size; - fd_ = fd; - dir_id_ = dir_id; - tenant_id_ = tenant_id; - is_read_ = false; - has_wait_ = false; - expect_read_size_ = 0; - last_read_offset_ = -1; - io_flag_.reset(); - } - return ret; -} - -int ObTmpFileIOHandle::wait() -{ - int ret = OB_SUCCESS; - const int64_t timeout_ms = std::max(GCONF._data_storage_io_timeout / 1000, DEFAULT_IO_WAIT_TIME_MS); - if (timeout_ms < 0) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument. timeout must be positive", K(ret), K(timeout_ms)); - } else if (!is_finished_) { - if (is_read_ && OB_FAIL(wait_read_finish(timeout_ms))) { - STORAGE_LOG(WARN, "wait read finish failed", K(ret), K(timeout_ms), K(is_read_)); - } else if (!is_read_ && OB_FAIL(wait_write_finish(timeout_ms))) { - STORAGE_LOG(WARN, "wait write finish failed", K(ret), K(timeout_ms), K(is_read_)); - } - ret_code_ = ret; - is_finished_ = true; - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(ret_code_)) { - STORAGE_LOG(WARN, "tmp file io error", K(ret), KPC(this)); - } - } - - return ret; -} - -int ObTmpFileIOHandle::wait_write_finish(int64_t timeout_ms) -{ - int ret = OB_SUCCESS; - if (write_block_ids_.size() == 0) { - STORAGE_LOG(DEBUG, "write block ids size is 0", K(ret), K(timeout_ms)); - } else { - // iter all blocks, execute wait - common::hash::ObHashSet::const_iterator iter; - int64_t begin_us = ObTimeUtility::fast_current_time(); - int64_t wait_ms = timeout_ms; - for (iter = write_block_ids_.begin(); OB_SUCC(ret) && iter != write_block_ids_.end(); ++iter) { - const int64_t &blk_id = iter->first; - if (OB_FAIL(OB_TMP_FILE_STORE.wait_write_finish(tenant_id_, blk_id, wait_ms))) { - STORAGE_LOG(WARN, "fail to wait write finish", K(ret), K(blk_id), K(timeout_ms)); - } - wait_ms = timeout_ms - (ObTimeUtility::fast_current_time() - begin_us) / 1000; - if (OB_SUCC(ret) && OB_UNLIKELY(wait_ms <= 0)) { - ret = OB_TIMEOUT; - STORAGE_LOG(WARN, "fail to wait tmp file write finish", K(ret), K(wait_ms), K(blk_id), K(timeout_ms)); - } - } - int bret = OB_SUCCESS; - if (OB_UNLIKELY(OB_SUCCESS != (bret = write_block_ids_.destroy()))) { - STORAGE_LOG(WARN, "fail to destroy write block id set", K(bret), K(wait_ms), K(timeout_ms)); - } - } - return ret; -} - -int ObTmpFileIOHandle::wait_read_finish(const int64_t timeout_ms) -{ - int ret = OB_SUCCESS; - ObTmpFileHandle file_handle; - if (OB_UNLIKELY(has_wait_ && is_read_)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(ERROR, "read wait() isn't reentrant interface, shouldn't call again", K(ret)); - } else if (OB_FAIL(do_read_wait(timeout_ms))) { - STORAGE_LOG(WARN, "fail to wait tmp file io", K(ret), K(timeout_ms)); - } else if (is_read_ && !has_wait_) { - if (size_ == expect_read_size_) { - //do nothing - } else if (OB_UNLIKELY(size_ > expect_read_size_)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "read size more than expected size", K(ret), K(timeout_ms)); - } else if (OB_FAIL(FILE_MANAGER_INSTANCE_V2.get_tmp_file_handle(fd_, file_handle))) { - STORAGE_LOG(WARN, "fail to get tmp file handle", K(ret)); - } else { - ObTmpFileIOInfo io_info; - io_info.fd_ = fd_; - io_info.dir_id_ = dir_id_; - io_info.tenant_id_ = tenant_id_; - io_info.size_ = expect_read_size_; - io_info.buf_ = buf_; - io_info.io_desc_ = io_flag_; - io_info.io_timeout_ms_ = timeout_ms; - while (OB_SUCC(ret) && size_ < expect_read_size_) { - if (OB_FAIL(file_handle.get_resource_ptr()->once_aio_read_batch(io_info, - update_offset_in_file_, - last_read_offset_, - *this))) { - STORAGE_LOG(WARN, "fail to read once batch", K(ret), K(timeout_ms), K(io_info), K(*this)); - } else if (OB_FAIL(do_read_wait(timeout_ms))) { - STORAGE_LOG(WARN, "fail to wait tmp file io", K(ret), K(timeout_ms)); - } - } - } - } - - if (OB_SUCC(ret) || OB_ITER_END == ret) { - has_wait_ = true; - expect_read_size_ = 0; - last_read_offset_ = -1; - io_flag_.reset(); - update_offset_in_file_ = false; - } - return ret; -} - -int ObTmpFileIOHandle::do_read_wait(const int64_t timeout_ms) -{ - int ret = OB_SUCCESS; - for (int32_t i = 0; OB_SUCC(ret) && i < block_cache_handles_.count(); i++) { - ObBlockCacheHandle &tmp = block_cache_handles_.at(i); - MEMCPY(tmp.buf_, tmp.block_handle_.value_->get_buffer() + tmp.offset_, tmp.size_); - tmp.block_handle_.reset(); - } - if (0 != block_cache_handles_.count()) { - OB_TMP_FILE_STORE.dec_block_cache_num(tenant_id_, block_cache_handles_.count()); - } - block_cache_handles_.reset(); - - for (int32_t i = 0; OB_SUCC(ret) && i < page_cache_handles_.count(); i++) { - ObPageCacheHandle &tmp = page_cache_handles_.at(i); - MEMCPY(tmp.buf_, tmp.page_handle_.value_->get_buffer() + tmp.offset_, tmp.size_); - tmp.page_handle_.reset(); - } - if (0 != page_cache_handles_.count()) { - OB_TMP_FILE_STORE.dec_page_cache_num(tenant_id_, page_cache_handles_.count()); - } - page_cache_handles_.reset(); - - for (int32_t i = 0; OB_SUCC(ret) && i < io_handles_.count(); i++) { - ObIOReadHandle &tmp = io_handles_.at(i); - if (OB_FAIL(tmp.macro_handle_.wait())) { - STORAGE_LOG(WARN, "fail to wait tmp read io", K(ret)); - } else { - MEMCPY(tmp.buf_, tmp.macro_handle_.get_buffer() + tmp.offset_, tmp.size_); - tmp.macro_handle_.reset(); - } - } - io_handles_.reset(); - return ret; -} - -void ObTmpFileIOHandle::reset() -{ - for (int32_t i = 0; i < io_handles_.count(); i++) { - io_handles_.at(i).macro_handle_.reset(); - } - for (int32_t i = 0; i < block_cache_handles_.count(); i++) { - block_cache_handles_.at(i).block_handle_.reset(); - } - if (0 != block_cache_handles_.count()) { - OB_TMP_FILE_STORE.dec_block_cache_num(tenant_id_, block_cache_handles_.count()); - } - for (int32_t i = 0; i < page_cache_handles_.count(); i++) { - page_cache_handles_.at(i).page_handle_.reset(); - } - if (0 != page_cache_handles_.count()) { - OB_TMP_FILE_STORE.dec_page_cache_num(tenant_id_, page_cache_handles_.count()); - } - io_handles_.reset(); - page_cache_handles_.reset(); - block_cache_handles_.reset(); - write_block_ids_.destroy(); - fd_ = OB_INVALID_FD; - dir_id_ = OB_INVALID_ID; - tenant_id_ = OB_INVALID_TENANT_ID; - buf_ = NULL; - size_ = 0; - is_read_ = false; - has_wait_ = false; - expect_read_size_ = 0; - last_read_offset_ = -1; - io_flag_.reset(); - update_offset_in_file_ = false; - is_finished_ = false; - ret_code_ = OB_SUCCESS; -} - -bool ObTmpFileIOHandle::is_valid() const -{ - return OB_INVALID_FD != fd_ && OB_INVALID_ID != dir_id_ && OB_INVALID_TENANT_ID != tenant_id_ - && NULL != buf_ && size_ >= 0; -} - -int ObTmpFileIOHandle::record_block_id(const int64_t block_id) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(write_block_ids_.set_refactored(block_id, 1))) { - STORAGE_LOG(WARN, "record block id failed", K(ret), K(block_id)); - } - return ret; -} - -void ObTmpFileIOHandle::set_last_extent_id(const int64_t last_extent_id) -{ - last_extent_id_ = last_extent_id; -} - -int64_t ObTmpFileIOHandle::get_last_extent_id() const -{ - return last_extent_id_; -} - -ObTmpFileIOHandle::ObIOReadHandle::ObIOReadHandle() - : macro_handle_(), buf_(NULL), offset_(0), size_(0) -{ -} - -ObTmpFileIOHandle::ObIOReadHandle::ObIOReadHandle(const ObMacroBlockHandle ¯o_handle, - char *buf, const int64_t offset, const int64_t size) - : macro_handle_(macro_handle), buf_(buf), offset_(offset), size_(size) -{ -} - -ObTmpFileIOHandle::ObIOReadHandle::~ObIOReadHandle() -{ -} - -ObTmpFileIOHandle::ObIOReadHandle::ObIOReadHandle(const ObTmpFileIOHandle::ObIOReadHandle &other) - : macro_handle_(), buf_(NULL), offset_(0), size_(0) -{ - *this = other; -} - -ObTmpFileIOHandle::ObIOReadHandle &ObTmpFileIOHandle::ObIOReadHandle::operator=( - const ObTmpFileIOHandle::ObIOReadHandle &other) -{ - if (&other != this) { - macro_handle_ = other.macro_handle_; - offset_ = other.offset_; - buf_ = other.buf_; - size_ = other.size_; - } - return *this; -} -ObTmpFileIOHandle::ObPageCacheHandle::ObPageCacheHandle() - : page_handle_(), buf_(NULL), offset_(0), size_(0) -{ -} - -ObTmpFileIOHandle::ObPageCacheHandle::ObPageCacheHandle(const ObTmpPageValueHandle &page_handle, - char *buf, const int64_t offset, const int64_t size) - : page_handle_(page_handle), buf_(buf), offset_(offset), size_(size) -{ -} - -ObTmpFileIOHandle::ObPageCacheHandle::~ObPageCacheHandle() -{ -} - -ObTmpFileIOHandle::ObPageCacheHandle::ObPageCacheHandle( - const ObTmpFileIOHandle::ObPageCacheHandle &other) - : page_handle_(), buf_(NULL), offset_(0), size_(0) -{ - *this = other; -} - -ObTmpFileIOHandle::ObPageCacheHandle &ObTmpFileIOHandle::ObPageCacheHandle::operator=( - const ObTmpFileIOHandle::ObPageCacheHandle &other) -{ - if (&other != this) { - page_handle_ = other.page_handle_; - offset_ = other.offset_; - buf_ = other.buf_; - size_ = other.size_; - } - return *this; -} - -ObTmpFileIOHandle::ObBlockCacheHandle::ObBlockCacheHandle() - : block_handle_(), buf_(NULL), offset_(0), size_(0) -{ -} - -ObTmpFileIOHandle::ObBlockCacheHandle::ObBlockCacheHandle(const ObTmpBlockValueHandle &block_handle, - char *buf, const int64_t offset, const int64_t size) - : block_handle_(block_handle), buf_(buf), offset_(offset), size_(size) -{ -} - -ObTmpFileIOHandle::ObBlockCacheHandle::~ObBlockCacheHandle() -{ -} - -ObTmpFileIOHandle::ObBlockCacheHandle::ObBlockCacheHandle( - const ObTmpFileIOHandle::ObBlockCacheHandle &other) - : block_handle_(), buf_(NULL), offset_(0), size_(0) -{ - *this = other; -} - -ObTmpFileIOHandle::ObBlockCacheHandle &ObTmpFileIOHandle::ObBlockCacheHandle::operator=( - const ObTmpFileIOHandle::ObBlockCacheHandle &other) -{ - if (&other != this) { - block_handle_ = other.block_handle_; - offset_ = other.offset_; - buf_ = other.buf_; - size_ = other.size_; - } - return *this; -} - -void ObTmpFileExtent::set_global_offset(const int64_t g_offset_start, const int64_t g_offset_end) -{ - g_offset_start_ = g_offset_start; - g_offset_end_ = g_offset_end; -} - -void ObTmpFileExtent::get_global_offset(int64_t &g_offset_start, int64_t &g_offset_end) const -{ - g_offset_start = g_offset_start_; - g_offset_end = g_offset_end_; -} - -ObTmpFileExtent::ObTmpFileExtent(ObTmpFile *file) - : is_alloced_(false), - is_closed_(false), - start_page_id_(-1), - page_nums_(0), - offset_(0), - fd_(file->get_fd()), - g_offset_start_(0), - g_offset_end_(0), - owner_(file), - block_id_(-1), - lock_(common::ObLatchIds::TMP_FILE_EXTENT_LOCK), - is_truncated_(false) -{ -} - -ObTmpFileExtent::~ObTmpFileExtent() -{ -} - -int ObTmpFileExtent::read(const ObTmpFileIOInfo &io_info, const int64_t offset, const int64_t size, - char *buf, ObTmpFileIOHandle &handle) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_alloced_)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "ObTmpFileExtent has not been allocated", K(ret)); - } else if (OB_UNLIKELY(offset < 0 || offset >= get_offset() || size <= 0 - || offset + size > get_offset()) || OB_ISNULL(buf)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(offset), K(get_offset()), K(size), K(buf)); - } else { - ObTmpBlockIOInfo info; - info.buf_ = buf; - info.io_desc_ = io_info.io_desc_; - info.block_id_ = block_id_; - info.offset_ = start_page_id_ * ObTmpMacroBlock::get_default_page_size() + offset; - info.size_ = size; - info.tenant_id_ = io_info.tenant_id_; - info.io_timeout_ms_ = io_info.io_timeout_ms_; - if (OB_FAIL(OB_TMP_FILE_STORE.read(owner_->get_tenant_id(), info, handle))) { - STORAGE_LOG(WARN, "fail to read the extent", K(ret), K(info), K(*this)); - } else { - STORAGE_LOG(DEBUG, "debug tmp file: read extent", K(ret), K(info), K(*this)); - } - } - return ret; -} - -int ObTmpFileExtent::write(const ObTmpFileIOInfo &io_info,int64_t &size, char *&buf) -{ - int ret = OB_SUCCESS; - int write_size = 0; - int64_t remain = 0; - bool is_write = false; - bool need_close = false; - if (OB_UNLIKELY(size <= 0) || OB_ISNULL(buf)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret)); - } else if (OB_UNLIKELY(!is_alloced_)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "ObTmpFileExtent has not been allocated", K(ret)); - } else if (get_offset() == page_nums_ * ObTmpMacroBlock::get_default_page_size()) { - need_close = true; - } else { - SpinWLockGuard guard(lock_); - if (!is_closed()) { - remain = page_nums_ * ObTmpMacroBlock::get_default_page_size() - get_offset(); - write_size = std::min(remain, size); - ObTmpBlockIOInfo info; - info.block_id_ = block_id_; - info.buf_ = buf; - info.io_desc_ = io_info.io_desc_; - info.offset_ = start_page_id_ * ObTmpMacroBlock::get_default_page_size() + get_offset(); - info.size_ = write_size; - info.tenant_id_ = io_info.tenant_id_; - info.io_timeout_ms_ = io_info.io_timeout_ms_; - if (OB_FAIL(OB_TMP_FILE_STORE.write(owner_->get_tenant_id(), info))) { - STORAGE_LOG(WARN, "fail to write the extent", K(ret)); - } else { - ATOMIC_FAA(&offset_, write_size); - g_offset_end_ = get_offset() + g_offset_start_; - buf += write_size; - size -= write_size; - if (remain == write_size) { - need_close = true; - } - STORAGE_LOG(DEBUG, "debug tmp file: write extent", K(ret), K(info), K(*this)); - } - } - } - if (need_close) { - close(); - try_sync_block(); - } - return ret; -} - -void ObTmpFileExtent::reset() -{ - is_alloced_ = false; - fd_ = -1; - g_offset_start_ = 0; - g_offset_end_ = 0; - ATOMIC_SET(&offset_, 0); - owner_ = NULL; - start_page_id_ = -1; - page_nums_ = 0; - block_id_ = -1; - ATOMIC_STORE(&is_closed_, false); - ATOMIC_STORE(&is_truncated_, false); -} - -bool ObTmpFileExtent::is_valid() -{ - return start_page_id_ >= 0 && page_nums_ >= 0 && block_id_ > 0; -} - -bool ObTmpFileExtent::close(bool force) -{ - int ret = OB_SUCCESS; - uint8_t page_start_id = ObTmpFilePageBuddy::MAX_PAGE_NUMS; - uint8_t page_nums = 0; - if (!is_closed_) { - if (close(page_start_id, page_nums, force)) { - if (ObTmpFilePageBuddy::MAX_PAGE_NUMS == page_start_id && 0 == page_nums) { - //nothing to do - } else if (OB_UNLIKELY(page_start_id > ObTmpFilePageBuddy::MAX_PAGE_NUMS - 1 - || page_nums > ObTmpFilePageBuddy::MAX_PAGE_NUMS)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "fail to close the extent", K(ret), K_(block_id), K(page_start_id), - K(page_nums), K_(offset)); - } else if (OB_FAIL(OB_TMP_FILE_STORE.free(owner_->get_tenant_id(), block_id_, page_start_id, - page_nums))) { - STORAGE_LOG(WARN, "fail to free", K(ret)); - } - if (OB_FAIL(ret)) { - unclose(page_nums); - } - } - } - return is_closed(); -} - -bool ObTmpFileExtent::close(uint8_t &free_page_start_id, uint8_t &free_page_nums, bool force) -{ - free_page_start_id = ObTmpFilePageBuddy::MAX_PAGE_NUMS; - free_page_nums = 0; - SpinWLockGuard guard(lock_); - if (!is_closed()) { - if (!force && 0 != page_nums_ && 0 == get_offset()) { - // Nothing to do. This extent is alloced just now, so it cannot be closed. - } else { - if (get_offset() != page_nums_ * ObTmpMacroBlock::get_default_page_size()) { - uint8_t offset_page_id = common::upper_align(get_offset(), ObTmpMacroBlock::get_default_page_size()) - / ObTmpMacroBlock::get_default_page_size(); - free_page_nums = page_nums_ - offset_page_id; - free_page_start_id = start_page_id_ + offset_page_id; - page_nums_ -= free_page_nums; - } - ATOMIC_STORE(&is_closed_, true); - } - } - return is_closed(); -} - -int ObTmpFileExtent::try_sync_block() -{ - int ret = OB_SUCCESS; - ObTmpTenantMemBlockManager::ObIOWaitInfoHandle handle; - ObTmpMacroBlock *blk = NULL; - if (OB_FAIL(OB_TMP_FILE_STORE.get_macro_block(owner_->get_tenant_id(), block_id_, blk))) { - STORAGE_LOG(WARN, "fail to get macro block", K(ret), K(owner_->get_tenant_id()),K(block_id_)); - } else if (OB_ISNULL(blk)) { - ret = OB_ERR_NULL_VALUE; - } else if (0 != blk->get_free_page_nums()) { - STORAGE_LOG(DEBUG, "ob tmp macro block has not been used up", K(ret), K(blk->get_free_page_nums()), K(owner_->get_tenant_id()),K(block_id_)); - } else if (OB_FAIL(OB_TMP_FILE_STORE.wash_block(owner_->get_tenant_id(), block_id_, handle))) { - // try to flush the block to the disk. If fails, do nothing. - STORAGE_LOG(DEBUG, "fail to sync block", K(ret), K(owner_->get_tenant_id()), K(block_id_)); - } else { - STORAGE_LOG(DEBUG, "succeed to sync wash block", K(block_id_)); - } - - return ret; -} - -void ObTmpFileExtent::unclose(const int32_t page_nums) -{ - SpinWLockGuard guard(lock_); - if (page_nums >= 0) { - page_nums_ += page_nums; - } - ATOMIC_STORE(&is_closed_, false); -} - -ObTmpFileMeta::~ObTmpFileMeta() -{ - clear(); -} - -int ObTmpFileMeta::init(const int64_t fd, const int64_t dir_id, common::ObIAllocator *allocator) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(fd < 0 || dir_id < 0) || OB_ISNULL(allocator)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(fd), K(dir_id)); - } else { - fd_ = fd; - dir_id_ = dir_id; - allocator_ = allocator; - } - return ret; -} - -ObTmpFileExtent *ObTmpFileMeta::get_last_extent() -{ - return extents_.count() > 0 ? extents_.at(extents_.count() - 1) : NULL; -} - - -int ObTmpFileMeta::deep_copy(const ObTmpFileMeta &other) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(extents_.assign(other.extents_))) { - STORAGE_LOG(WARN, "fail to assign extents array", K(ret)); - } else { - fd_ = other.fd_; - dir_id_ = other.dir_id_; - allocator_ = other.allocator_; - } - return ret; -} - -int ObTmpFileMeta::clear() -{ - //free extents - int ret = OB_SUCCESS; - ObTmpFileExtent *tmp = NULL; - for (int64_t i = extents_.count() - 1; OB_SUCC(ret) && i >= 0; --i) { - tmp = extents_.at(i); - if (NULL != tmp) { - if (!tmp->is_alloced() || tmp->is_truncated()) { - // nothing to do. - } else if (OB_FAIL(OB_TMP_FILE_STORE.free(tmp->get_owner().get_tenant_id(), tmp))) { - STORAGE_LOG(WARN, "fail to free extents", K(ret)); - } - if (OB_SUCC(ret)) { - tmp->~ObTmpFileExtent(); - allocator_->free(tmp); - extents_.at(i) = NULL; - } - } - } - if (OB_SUCC(ret)) { - extents_.reset(); - allocator_ = NULL; - fd_ = -1; - dir_id_ = -1; - } - return ret; -} - -ObTmpFile::ObTmpFile() - : is_inited_(false), - is_big_(false), - offset_(0), - tenant_id_(-1), - lock_(common::ObLatchIds::TMP_FILE_LOCK), - allocator_(NULL), - file_meta_(), - read_guard_(0), - next_truncated_extent_id_(0) -{ -} - -ObTmpFile::~ObTmpFile() -{ - clear(); -} - -int ObTmpFile::clear() -{ - int ret = OB_SUCCESS; - if (OB_FAIL(file_meta_.clear())) { - STORAGE_LOG(WARN, "fail to clear file meta", K(ret)); - } else { - if (is_inited_) { - is_big_ = false; - tenant_id_ = -1; - offset_ = 0; - allocator_ = NULL; - is_inited_ = false; - read_guard_ = 0; - next_truncated_extent_id_ = 0; - } - } - return ret; -} - -int ObTmpFile::init(const int64_t fd, const int64_t dir_id, common::ObIAllocator &allocator) -{ - int ret = OB_SUCCESS; - if (IS_INIT) { - ret = OB_INIT_TWICE; - STORAGE_LOG(WARN, "ObTmpFile has not been inited", K(ret)); - } else if (fd < 0) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(fd)); - } else if (OB_FAIL(file_meta_.init(fd, dir_id, &allocator))) { - STORAGE_LOG(WARN, "fail to init file meta", K(ret)); - } else { - allocator_ = &allocator; - is_inited_ = true; - } - - if (!is_inited_) { - clear(); - } - return ret; -} - -int64_t ObTmpFile::find_first_extent(const int64_t offset) -{ - common::ObIArray &extents = file_meta_.get_extents(); - int64_t first_extent = 0; - int64_t left = 0; - int64_t right = extents.count() - 1; - ObTmpFileExtent *tmp = nullptr; - while(left < right) { - int64_t mid = (left + right) / 2; - tmp = extents.at(mid); - if (tmp->get_global_start() <= offset && offset < tmp->get_global_end()) { - first_extent = mid; - break; - } else if(tmp->get_global_start() > offset) { - right = mid - 1; - } else if(tmp->get_global_end() <= offset) { - left = mid + 1; - } - } - return first_extent; -} - -int ObTmpFile::aio_read_without_lock(const ObTmpFileIOInfo &io_info, - int64_t &offset, ObTmpFileIOHandle &handle) -{ - int ret = OB_SUCCESS; - ObTmpFileExtent *tmp = nullptr; - if (OB_ISNULL(tmp = file_meta_.get_last_extent())) { - ret = OB_BAD_NULL_ERROR; - STORAGE_LOG(WARN, "fail to read, because the tmp file is empty", K(ret), KP(tmp), K(io_info)); - } else if (OB_FAIL(handle.prepare_read(io_info.size_, - offset, - io_info.io_desc_, - io_info.buf_, - file_meta_.get_fd(), - file_meta_.get_dir_id(), - io_info.tenant_id_, - io_info.disable_page_cache_))) { - STORAGE_LOG(WARN, "fail to prepare read io handle", K(ret), K(io_info), K(offset)); - } else if (OB_UNLIKELY(io_info.size_ > 0 && offset >= tmp->get_global_end())) { - ret = OB_ITER_END; - } else if (OB_FAIL(once_aio_read_batch_without_lock(io_info, offset, handle))) { - STORAGE_LOG(WARN, "fail to read one batch", K(ret), K(offset), K(handle)); - } else { - handle.set_last_read_offset(offset); - } - return ret; -} - -int ObTmpFile::once_aio_read_batch( - const ObTmpFileIOInfo &io_info, - const bool need_update_offset, - int64_t &offset, - ObTmpFileIOHandle &handle) -{ - int ret = OB_SUCCESS; - ObTmpFileExtent *tmp = nullptr; - const int64_t remain_size = io_info.size_ - handle.get_data_size(); - - { - SpinRLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFile has not been initialized", K(ret)); - } else if (OB_UNLIKELY(offset < 0 || remain_size < 0) || OB_ISNULL(io_info.buf_)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(offset), K(remain_size), KP(io_info.buf_)); - } else if (OB_ISNULL(tmp = file_meta_.get_last_extent())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "unexpected error, null tmp file extent", K(ret), KP(tmp), K(io_info)); - } else if (OB_UNLIKELY(remain_size > 0 && offset >= tmp->get_global_end())) { - ret = OB_ITER_END; - } else if (OB_FAIL(once_aio_read_batch_without_lock(io_info, offset, handle))) { - STORAGE_LOG(WARN, "fail to read one batch", K(ret), K(offset), K(handle)); - } else { - handle.set_last_read_offset(offset); - } - } - - if (need_update_offset) { - SpinWLockGuard guard(lock_); - offset_ = offset; - } - return ret; -} - -int ObTmpFile::fill_zero(char *buf, const int64_t size) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(buf) || size < 0) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "buf is null or size is negative", K(ret), K(size), KP(buf)); - } else { - MEMSET(buf, 0, size); - } - return ret; -} - -int ObTmpFile::once_aio_read_batch_without_lock( - const ObTmpFileIOInfo &io_info, - int64_t &offset, - ObTmpFileIOHandle &handle) -{ - int ret = OB_SUCCESS; - int64_t one_batch_read_size = 0; - char *buf = io_info.buf_ + handle.get_data_size(); - int64_t remain_size = io_info.size_ - handle.get_data_size(); - int64_t read_size = 0; - ObTmpFileExtent *tmp = nullptr; - common::ObIArray &extents = file_meta_.get_extents(); - int64_t ith_extent = get_extent_cache(offset, handle); - - while (OB_SUCC(ret) - && ith_extent < extents.count() - && remain_size > 0 - && one_batch_read_size < READ_SIZE_PER_BATCH) { - tmp = extents.at(ith_extent); - if (tmp->get_global_start() <= offset && offset < tmp->get_global_end()) { - if (offset + remain_size > tmp->get_global_end()) { - read_size = tmp->get_global_end() - offset; - } else { - read_size = remain_size; - } - // read from the extent. - if (tmp->is_truncated()) { - if (read_guard_ < tmp->get_global_end()) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "extent is truncated but read_guard not set correctlly", K(ret), K(tmp), K(read_guard_)); - } else if (OB_FAIL(fill_zero(buf, read_size))) { - STORAGE_LOG(WARN, "fail to fill zero data to buf", K(ret)); - } - } else if (offset >= read_guard_) { - if (OB_FAIL(tmp->read(io_info, offset - tmp->get_global_start(), read_size, buf, handle))) { - STORAGE_LOG(WARN, "fail to read the extent", K(ret), K(io_info), K(buf), KP_(io_info.buf)); - } - } else { - if (read_guard_ < offset + read_size) { - const int64_t zero_size = read_guard_ - offset; - const int64_t file_read_size = read_size - zero_size; - if (OB_FAIL(fill_zero(buf, zero_size))) { - STORAGE_LOG(WARN, "fail to read zero from truncated pos", K(ret)); - } else if (OB_FAIL(tmp->read(io_info, read_guard_ - tmp->get_global_start(), file_read_size, buf + zero_size, handle))) { - STORAGE_LOG(WARN, "fail to read the extent", K(ret), K(io_info), K(buf + zero_size), KP_(io_info.buf)); - } - } else { - // read 0; - if (OB_FAIL(fill_zero(buf, read_size))) { - STORAGE_LOG(WARN, "fail to read zero from truncated pos", K(ret), KP(buf), K(read_size)); - } - } - - } - if (OB_SUCC(ret)) { - offset += read_size; - remain_size -= read_size; - buf += read_size; - one_batch_read_size += read_size; - handle.add_data_size(read_size); - } - } - ++ith_extent; - } - - if (OB_SUCC(ret) && OB_LIKELY(ith_extent > 0)) { - handle.set_last_extent_id(ith_extent - 1); - } - return ret; -} - -int64_t ObTmpFile::get_extent_cache(const int64_t offset, const ObTmpFileIOHandle &handle) -{ - common::ObIArray &extents = file_meta_.get_extents(); - int64_t ith_extent = -1; - int64_t last_extent_id = handle.get_last_extent_id(); - if (OB_UNLIKELY(last_extent_id < 0 || last_extent_id >= extents.count() - 1)) { - ith_extent = find_first_extent(offset); - } else if (OB_LIKELY(extents.at(last_extent_id)->get_global_start() <= offset - && offset < extents.at(last_extent_id)->get_global_end())) { - ith_extent = last_extent_id; - } else if (offset == extents.at(last_extent_id)->get_global_end() - && last_extent_id != extents.count() - 1) { - ith_extent = last_extent_id + 1; - } else { - ith_extent = find_first_extent(offset); - } - - return ith_extent; -} - -int ObTmpFile::aio_read(const ObTmpFileIOInfo &io_info, ObTmpFileIOHandle &handle) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFile has not been inited", K(ret)); - } else { - tenant_id_ = io_info.tenant_id_; - SpinWLockGuard guard(lock_); - if (OB_FAIL(aio_read_without_lock(io_info, offset_, handle))) { - if (OB_ITER_END != ret) { - STORAGE_LOG(WARN, "fail to do aio read without lock", K(ret)); - } - } else { - handle.set_update_offset_in_file(); - } - } - return ret; -} -int ObTmpFile::aio_pread(const ObTmpFileIOInfo &io_info, const int64_t offset, - ObTmpFileIOHandle &handle) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFile has not been inited", K(ret)); - } else { - int64_t tmp_offset = offset; - SpinRLockGuard guard(lock_); - if (OB_FAIL(aio_read_without_lock(io_info, tmp_offset, handle))) { - if (OB_ITER_END != ret) { - STORAGE_LOG(WARN, "fail to do aio read without lock", K(ret)); - } - } - } - return ret; -} - -int ObTmpFile::seek(const int64_t offset, const int whence) -{ - int ret = OB_SUCCESS; - SpinWLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFile has not been inited", K(ret)); - } else { - switch (whence) { - case SET_SEEK: - offset_ = offset; - break; - case CUR_SEEK: - offset_ += offset; - break; - default: - ret = OB_NOT_SUPPORTED; - STORAGE_LOG(WARN, "not supported whence", K(ret), K(whence)); - } - } - return ret; -} - -int ObTmpFile::read(const ObTmpFileIOInfo &io_info, ObTmpFileIOHandle &handle) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(aio_read(io_info, handle))) { - if (OB_ITER_END != ret) { - STORAGE_LOG(WARN, "fail to read data using asynchronous io", K(ret), K(io_info)); - } else { - if (OB_FAIL(handle.wait())) { - STORAGE_LOG(WARN, "fail to wait io finish", K(ret), K(io_info)); - } else { - ret = OB_ITER_END; - } - } - } else if (OB_FAIL(handle.wait())) { - if (OB_ITER_END != ret) { - STORAGE_LOG(WARN, "fail to wait io finish", K(ret), K(io_info)); - } - } - return ret; -} - -int ObTmpFile::pread(const ObTmpFileIOInfo &io_info, const int64_t offset, ObTmpFileIOHandle &handle) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFile has not been inited", K(ret)); - } else if (OB_FAIL(aio_pread(io_info, offset, handle))) { - if (OB_ITER_END != ret) { - STORAGE_LOG(WARN, "fail to read data using asynchronous io", K(ret), K(io_info)); - } else { - if (OB_FAIL(handle.wait())) { - STORAGE_LOG(WARN, "fail to wait io finish", K(ret), K(io_info)); - } else { - ret = OB_ITER_END; - } - } - } else if (OB_FAIL(handle.wait())) { - if (OB_ITER_END != ret) { - STORAGE_LOG(WARN, "fail to wait io finish", K(ret), K(io_info)); - } - } - return ret; -} - -int ObTmpFile::aio_write(const ObTmpFileIOInfo &io_info, ObTmpFileIOHandle &handle) -{ - // only support append at present. - int ret = OB_SUCCESS; - DISABLE_SQL_MEMLEAK_GUARD; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFile has not been inited", K(ret)); - } else if (!io_info.is_valid()) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(io_info), K(handle)); - } else if (OB_FAIL(handle.prepare_write(io_info.buf_, - io_info.size_, - file_meta_.get_fd(), - file_meta_.get_dir_id(), - io_info.tenant_id_))) { - STORAGE_LOG(WARN, "fail to prepare write io handle", K(ret)); - } else { - tenant_id_ = io_info.tenant_id_; - int64_t size = io_info.size_; - char *buf = io_info.buf_; - SpinWLockGuard guard(lock_); - ObTmpFileExtent *tmp = file_meta_.get_last_extent(); - void *buff = NULL; - ObTmpFileExtent *extent = NULL; - while (OB_SUCC(ret) && size > 0) { - if (NULL != tmp && tmp->is_alloced() && !tmp->is_closed()) { - if (OB_FAIL(write_file_extent(io_info, tmp, size, buf))) { - STORAGE_LOG(WARN, "fail to write the extent", K(ret)); - } - } else if (NULL == tmp || (tmp->is_alloced() && tmp->is_closed())) { - int64_t alloc_size = 0; - // Pre-Allocation Strategy: - // 1. small file - // a. 0KB < size <= 32KB, alloc_size = 32KB; - // b. 32KB < size <= 64KB, alloc_size = 64KB; - // c. 64KB < size , alloc_size = size; - // 2. big file - // a. 32KB < size <= 64KB, alloc_size = 64KB; - // b. 64KB < size , alloc_size = size; - // - // NOTE: if the size is more than block size, it will be split into - // multiple allocation. - if (size <= big_file_prealloc_size()) { - if (!is_big_ && size <= small_file_prealloc_size()) { - alloc_size = common::upper_align(size, small_file_prealloc_size()); - } else { - alloc_size = common::upper_align(size, big_file_prealloc_size()); - } - } else if (size > ObTmpFileStore::get_block_size()) { - alloc_size = ObTmpFileStore::get_block_size(); - } else { - alloc_size = size; - } - lib::ObMemAttr attr(tenant_id_, "TmpFileExtent"); - if (OB_ISNULL(buff = allocator_->alloc(sizeof(ObTmpFileExtent), attr))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - STORAGE_LOG(WARN, "fail to alloc a buf", K(ret)); - } else if (OB_ISNULL(extent = new (buff) ObTmpFileExtent(this))) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "fail to new a ObTmpFileExtent", K(ret)); - } else if (OB_FAIL(OB_TMP_FILE_STORE.alloc(get_dir_id(), tenant_id_, alloc_size, *extent))) { - STORAGE_LOG(WARN, "fail to allocate extents", K(ret), K_(tenant_id)); - } else { - if (NULL != tmp) { - extent->set_global_offset(tmp->get_global_end(), tmp->get_global_end()); - } - tmp = extent; - if (OB_FAIL(write_file_extent(io_info, tmp, size, buf))) { - STORAGE_LOG(WARN, "fail to write the extent", K(ret)); - } else if (OB_FAIL(file_meta_.push_back_extent(tmp))) { - STORAGE_LOG(WARN, "fail to push the extent", K(ret)); - } else { - extent = NULL; - buff = NULL; - } - } - // If it fails, the unused extent should be returned to the macro block and cleaned up. - if (OB_FAIL(ret)) { - if (OB_NOT_NULL(extent)) { - if (extent->is_valid()) { - OB_TMP_FILE_STORE.free(tenant_id_, extent); - extent->~ObTmpFileExtent(); - } - allocator_->free(extent); - extent = NULL; - } - if (OB_NOT_NULL(tmp)) { - tmp = NULL; - } - } - } else { - // this extent has allocated, no one page, invalid extent(tmp->is_alloced = false). - ret = OB_INVALID_ERROR; - STORAGE_LOG(WARN, "invalid extent", K(ret), KPC(tmp), KPC(this)); - } - if (OB_SUCC(ret)) { - if (OB_ISNULL(tmp)) { - ret = OB_ERR_NULL_VALUE; - STORAGE_LOG(ERROR, "invalid tmp", K(ret)); - } else if (OB_FAIL(handle.record_block_id(tmp->get_block_id()))) { - if (OB_HASH_EXIST == ret) { - ret = OB_SUCCESS; - } else { - STORAGE_LOG(WARN, "set block id failed", K(ret)); - } - } - } - - if (OB_EAGAIN == ret) { - ret = OB_SUCCESS; - } - } - handle.sub_data_size(io_info.size_ - size); - if (OB_SUCC(ret) && !is_big_){ - is_big_ = tmp->get_global_end() >= - SMALL_FILE_MAX_THRESHOLD * ObTmpMacroBlock::get_default_page_size(); - } - } - return ret; -} - -int64_t ObTmpFile::get_dir_id() const -{ - return file_meta_.get_dir_id(); -} - -uint64_t ObTmpFile::get_tenant_id() const -{ - return tenant_id_; -} - -int64_t ObTmpFile::get_fd() const -{ - return file_meta_.get_fd(); -} - -int ObTmpFile::sync(const int64_t timeout_ms) -{ - int ret = OB_SUCCESS; - SpinWLockGuard guard(lock_); - ObTmpFileExtent *tmp = file_meta_.get_last_extent(); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFile has not been inited", K(ret)); - } else if (OB_ISNULL(tmp)) { - ret = OB_BAD_NULL_ERROR; - STORAGE_LOG(WARN, "the file does not have a extent", K(ret), K(timeout_ms)); - } else { - tmp->close(true/*force*/); - // all extents has been closed. - const ObIArray &extents = file_meta_.get_extents(); - common::hash::ObHashSet blk_id_set; - lib::ObMemAttr attr(tenant_id_, "TmpBlkIDSet"); - if (OB_FAIL(blk_id_set.create(min(extents.count(), 1024 * 1024), attr))){ - STORAGE_LOG(WARN, "create block id set failed", K(ret), K(timeout_ms)); - } else { - // get extents block id set. - for (int64_t i=0; OB_SUCC(ret) && i < extents.count(); ++i) { - const ObTmpFileExtent* e = extents.at(i); - const int64_t &blk_id = e->get_block_id(); - if (OB_FAIL(blk_id_set.set_refactored(blk_id))) { - STORAGE_LOG(WARN, "add block id to set failed", K(ret), K(blk_id)); - } - } - // iter all blocks, execute async wash. - common::hash::ObHashSet::const_iterator iter; - common::ObSEArray handles; - handles.set_attr(ObMemAttr(MTL_ID(), "TMP_SYNC_HDL")); - for (iter = blk_id_set.begin(); OB_SUCC(ret) && iter != blk_id_set.end(); ++iter) { - const int64_t &blk_id = iter->first; - ObTmpTenantMemBlockManager::ObIOWaitInfoHandle handle; - if (OB_FAIL(OB_TMP_FILE_STORE.sync_block(tenant_id_, blk_id, handle))) { - // OB_HASH_NOT_EXIST: - // if multiple file sync same block, the block may be not exist in hash map. - // OB_STATE_NOT_MATCH: - // the extents in block may be not all close and shouldn't sync it now. - if (OB_HASH_NOT_EXIST == ret || OB_STATE_NOT_MATCH == ret) { - ret = OB_SUCCESS; - } else { - STORAGE_LOG(WARN, "sync block failed", K(ret), K(blk_id)); - } - } else if (OB_NOT_NULL(handle.get_wait_info()) && OB_FAIL(handles.push_back(handle))) { - STORAGE_LOG(WARN, "push back wait handle to array failed", K(ret), K(blk_id)); - } - } - - int64_t begin_us = ObTimeUtility::fast_current_time(); - int64_t wait_ms = timeout_ms; - for (int64_t i=0; OB_SUCC(ret) && i < handles.count(); ++i) { - const ObTmpTenantMemBlockManager::ObIOWaitInfoHandle handle = handles.at(i); - if (OB_FAIL(handle.get_wait_info()->wait(timeout_ms))) { - STORAGE_LOG(WARN, "add block id to set failed", K(ret), K(timeout_ms)); - } else { - wait_ms = timeout_ms - (ObTimeUtility::fast_current_time() - begin_us) / 1000; - } - - if (OB_SUCC(ret) && OB_UNLIKELY(wait_ms <= 0)) { // rarely happen - ret = OB_TIMEOUT; - STORAGE_LOG(WARN, "fail to wait tmp file sync finish", K(ret), K(wait_ms)); - } - } - } - } - return ret; -} - -int ObTmpFile::deep_copy(char *buf, const int64_t buf_len, ObTmpFile *&value) const -{ - int ret = OB_SUCCESS; - const int64_t deep_copy_size = get_deep_copy_size(); - if (OB_ISNULL(buf) || buf_len < deep_copy_size) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), KP(buf), K(buf_len)); - } else { - ObTmpFile *pvalue = new (buf) ObTmpFile(); - if (OB_FAIL(pvalue->file_meta_.deep_copy(file_meta_))) { - STORAGE_LOG(WARN, "fail to deep copy meta", K(ret)); - } else { - pvalue->is_big_ = is_big_; - pvalue->offset_ = offset_; - pvalue->tenant_id_ = tenant_id_; - pvalue->allocator_ = allocator_; - pvalue->is_inited_ = is_inited_; - value = pvalue; - } - } - return ret; -} - -void ObTmpFile::get_file_size(int64_t &file_size) -{ - ObTmpFileExtent *tmp = file_meta_.get_last_extent(); - file_size = (nullptr == tmp) ? 0 : tmp->get_global_end(); -} - -/* - * to avoid truncating blocks that is using now (e.g., the io request is in io manager but not finish). - * we need to ensure there is no other file operation while calling truncate. - */ -int ObTmpFile::truncate(const int64_t offset) -{ - int ret = OB_SUCCESS; - - SpinWLockGuard guard(lock_); - // release extents - ObTmpFileExtent *tmp = nullptr; - //the extents before read_guard_ is truncated; - int64_t ith_extent = next_truncated_extent_id_; - common::ObIArray &extents = file_meta_.get_extents(); - STORAGE_LOG(INFO, "truncate ", K(offset), K(read_guard_), K(ith_extent)); - - if (OB_ISNULL(tmp = file_meta_.get_last_extent())) { - ret = OB_BAD_NULL_ERROR; - STORAGE_LOG(WARN, "fail to truncate, because the tmp file is empty", K(ret), KP(tmp)); - } else if (offset < 0 || offset > tmp->get_global_end()) { - ret = OB_INDEX_OUT_OF_RANGE; - STORAGE_LOG(WARN, "offset out of range", K(ret), K(tmp), K(offset)); - } - - while (OB_SUCC(ret) && ith_extent >= 0 - && ith_extent < extents.count()) { - tmp = extents.at(ith_extent); - if (tmp->get_global_start() >= offset) { - break; - } else if (!tmp->is_closed()) { - // for extent that is not closed, shouldn't truncate. - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "the truncate extent is not closed", K(ret)); - } else if (tmp->get_global_end() > offset) { - break; - } else { - // release space - if (!tmp->is_truncated()) { - tmp->set_truncated(); - if (OB_FAIL(OB_TMP_FILE_STORE.free(get_tenant_id(), tmp->get_block_id(), - tmp->get_start_page_id(), - tmp->get_page_nums()))) { - STORAGE_LOG(WARN, "fail to release space", K(ret), K(read_guard_), K(tmp)); - } - STORAGE_LOG(TRACE, "release extents", K(ith_extent), K(tmp->get_start_page_id()), K(tmp->get_page_nums())); - } - if (OB_SUCC(ret)) { - // if only part of extent is truncated, we only need to set the read_guard - ith_extent++; - } - } - } - - if (OB_SUCC(ret) && offset > read_guard_) { - read_guard_ = offset; - next_truncated_extent_id_ = ith_extent; - } - return ret; -} - -int ObTmpFile::write_file_extent(const ObTmpFileIOInfo &io_info, ObTmpFileExtent *file_extent, - int64_t &size, char *&buf) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(size <= 0)|| OB_ISNULL(file_extent) || OB_ISNULL(buf)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret)); - } else if (OB_FAIL(file_extent->write(io_info, size, buf))) { - STORAGE_LOG(WARN, "fail to write the extent", K(ret), K(size), KP(buf)); - } - return ret; -} - -int ObTmpFile::write(const ObTmpFileIOInfo &io_info) -{ - int ret = OB_SUCCESS; - ObTmpFileIOHandle handle; - if (OB_FAIL(aio_write(io_info, handle))) { - STORAGE_LOG(WARN, "fail to write using asynchronous io", K(ret), K(io_info)); - } else if (OB_FAIL(handle.wait())) { - STORAGE_LOG(WARN, "fail to wait io finish", K(ret)); - } - return ret; -} - -int64_t ObTmpFile::small_file_prealloc_size() -{ - return SMALL_FILE_MAX_THRESHOLD * ObTmpMacroBlock::get_default_page_size(); -} - -int64_t ObTmpFile::big_file_prealloc_size() -{ - return BIG_FILE_PREALLOC_EXTENT_SIZE * ObTmpMacroBlock::get_default_page_size(); -} - - -ObTmpFileHandle::ObTmpFileHandle() - : ObResourceHandle() -{ -} - -ObTmpFileHandle::~ObTmpFileHandle() -{ - reset(); -} - -void ObTmpFileHandle::reset() -{ - if (NULL != ptr_) { - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = ObTmpFileManager::get_instance().dec_handle_ref(*this))) { - STORAGE_LOG_RET(WARN, tmp_ret, "fail to decrease handle reference count", K(tmp_ret)); - } else { - ptr_ = nullptr; - } - } -} - -ObTmpFileManager &ObTmpFileManager::get_instance() -{ - static ObTmpFileManager instance; - return instance; -} - -int ObTmpFileManager::init() -{ - int ret = OB_SUCCESS; - ObMemAttr attr = SET_USE_500(ObMemAttr(OB_SERVER_TENANT_ID, ObModIds::OB_TMP_FILE_MANAGER)); //TODO: split tmp file map into each tenant. - if (IS_INIT) { - ret = OB_INIT_TWICE; - STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret)); - } else if (OB_FAIL(files_.init(DEFAULT_BUCKET_NUM, attr, *lib::ObMallocAllocator::get_instance()))) { - STORAGE_LOG(WARN, "fail to init map for temporary files", K(ret)); - } else if (OB_FAIL(OB_TMP_FILE_STORE.init())) { - STORAGE_LOG(WARN, "fail to init the block manager for temporary files", K(ret)); - } else { - is_inited_ = true; - } - - if (!is_inited_) { - destroy(); - } - return ret; -} - -int ObTmpFileManager::alloc_dir(int64_t &dir) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret)); - } else if (OB_FAIL(get_next_dir(dir))) { - STORAGE_LOG(WARN, "The directory of ObTmpFileManager has been used up", K(ret)); - } - return ret; -} - -int ObTmpFileManager::get_next_dir(int64_t &next_dir) -{ - int ret = OB_SUCCESS; - next_value(next_dir_, next_dir); - if (INT64_MAX - 1 == next_dir_) { - ret = OB_SIZE_OVERFLOW; - } - return ret; -} - -void ObTmpFileManager::next_value(int64_t ¤t_val, int64_t &next_val) -{ - int64_t old_val = ATOMIC_LOAD(¤t_val); - int64_t new_val = 0; - bool finish = false; - while (!finish) { - new_val = (old_val + 1) % INT64_MAX; - next_val = new_val; - finish = (old_val == (new_val = ATOMIC_VCAS(¤t_val, old_val, new_val))); - old_val = new_val; - } -} - -int ObTmpFileManager::open(int64_t &fd, int64_t &dir) -{ - int ret = OB_SUCCESS; - ObTmpFile file; - common::ObIAllocator *allocator = nullptr; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret)); - } else if (OB_FAIL(OB_TMP_FILE_STORE.get_tenant_extent_allocator(MTL_ID(), allocator))) { - STORAGE_LOG(WARN, "fail to get extent allocator", K(ret)); - } else if (OB_FAIL(get_next_fd(fd))) { - STORAGE_LOG(WARN, "fail to get next fd", K(ret)); - } else if (OB_FAIL(file.init(fd, dir, *allocator))) { - STORAGE_LOG(WARN, "fail to open file", K(ret)); - } else if (OB_FAIL(files_.set(fd, file))) { - STORAGE_LOG(WARN, "fail to set tmp file", K(ret)); - } else { - ObTaskController::get().allow_next_syslog(); - STORAGE_LOG(INFO, "succeed to open a tmp file", K(fd), K(dir), K(common::lbt())); - } - return ret; -} - -int ObTmpFileManager::get_next_fd(int64_t &next_fd) -{ - int ret = OB_SUCCESS; - ObTmpFileHandle handle; - next_value(next_fd_, next_fd); - if (OB_FAIL(files_.get(next_fd, handle))) { - if (OB_ENTRY_NOT_EXIST != ret) { - STORAGE_LOG(WARN, "fail to get from resource map", K(ret), K(next_fd)); - } else { - ret = OB_SUCCESS; - } - } else { - ret = OB_FILE_ALREADY_EXIST; - STORAGE_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "too much file", K(ret)); - } - return ret; -} - -int ObTmpFileManager::aio_read(const ObTmpFileIOInfo &io_info, ObTmpFileIOHandle &handle) -{ - int ret = OB_SUCCESS; - ObTmpFileHandle file_handle; - handle.reset(); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret)); - } else if (!io_info.is_valid()) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(io_info)); - } else if (OB_FAIL(files_.get(io_info.fd_, file_handle))) { - STORAGE_LOG(WARN, "fail to get tmp file handle", K(ret)); - } else if (OB_FAIL(file_handle.get_resource_ptr()->aio_read(io_info, handle))) { - if (common::OB_ITER_END != ret) { - STORAGE_LOG(WARN, "fail to do asynchronous read", K(ret), K(io_info)); - } - } - return ret; -} - -int ObTmpFileManager::aio_pread(const ObTmpFileIOInfo &io_info, const int64_t offset, - ObTmpFileIOHandle &handle) -{ - int ret = OB_SUCCESS; - ObTmpFileHandle file_handle; - handle.reset(); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret)); - } else if (!io_info.is_valid()) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(io_info)); - } else if (OB_FAIL(files_.get(io_info.fd_, file_handle))) { - STORAGE_LOG(WARN, "fail to get tmp file handle", K(ret)); - } else if (OB_FAIL(file_handle.get_resource_ptr()->aio_pread(io_info, offset, handle))) { - if (common::OB_ITER_END != ret) { - STORAGE_LOG(WARN, "fail to do asynchronous pread", K(ret), K(io_info), K(offset)); - } - } - return ret; -} - -int ObTmpFileManager::read(const ObTmpFileIOInfo &io_info, ObTmpFileIOHandle &handle) -{ - int ret = OB_SUCCESS; - ObTmpFileHandle file_handle; - handle.reset(); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret)); - } else if (!io_info.is_valid()) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(io_info)); - } else if (OB_FAIL(files_.get(io_info.fd_, file_handle))) { - STORAGE_LOG(WARN, "fail to get temporary file handle", K(ret), K(io_info)); - } else if (OB_FAIL(file_handle.get_resource_ptr()->read(io_info, handle))) { - if (OB_ITER_END != ret) { - STORAGE_LOG(WARN, "fail to read", K(ret), K(io_info)); - } - } - return ret; -} - -int ObTmpFileManager::pread(const ObTmpFileIOInfo &io_info, const int64_t offset, ObTmpFileIOHandle &handle) -{ - int ret = OB_SUCCESS; - ObTmpFileHandle file_handle; - handle.reset(); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret)); - } else if (!io_info.is_valid()) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(io_info)); - } else if (OB_FAIL(files_.get(io_info.fd_, file_handle))) { - STORAGE_LOG(WARN, "fail to get tmp file handle", K(ret), K(io_info)); - } else if (OB_FAIL(file_handle.get_resource_ptr()->pread(io_info, offset, handle))) { - if (OB_ITER_END != ret) { - STORAGE_LOG(WARN, "fail to pread", K(ret), K(io_info)); - } - } - return ret; -} - -int ObTmpFileManager::aio_write(const ObTmpFileIOInfo &io_info, ObTmpFileIOHandle &handle) -{ - int ret = OB_SUCCESS; - ObTmpFileHandle file_handle; - handle.reset(); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret)); - } else if (!io_info.is_valid()) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret)); - } else if (OB_FAIL(files_.get(io_info.fd_, file_handle))) { - STORAGE_LOG(WARN, "fail to get tmp file handle", K(ret), K(io_info)); - } else if (OB_FAIL(file_handle.get_resource_ptr()->aio_write(io_info, handle))) { - STORAGE_LOG(WARN, "fail to aio_write", K(ret), K(io_info)); - } - return ret; -} - -int ObTmpFileManager::write(const ObTmpFileIOInfo &io_info) -{ - int ret = OB_SUCCESS; - ObTmpFileHandle file_handle; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret)); - } else if (!io_info.is_valid()) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(io_info)); - } else if (OB_FAIL(files_.get(io_info.fd_, file_handle))) { - STORAGE_LOG(WARN, "fail to get temporary file handle", K(ret), K(io_info)); - } else if (OB_FAIL(file_handle.get_resource_ptr()->write(io_info))) { - STORAGE_LOG(WARN, "fail to write", K(ret), K(io_info)); - } - return ret; -} - -int ObTmpFileManager::seek(const int64_t fd, const int64_t offset, const int whence) -{ - int ret = OB_SUCCESS; - ObTmpFileHandle file_handle; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret)); - } else if (OB_FAIL(files_.get(fd, file_handle))) { - STORAGE_LOG(WARN, "fail to get tmp file handle", K(ret), K(fd)); - } else if (OB_FAIL(file_handle.get_resource_ptr()->seek(offset, whence))) { - STORAGE_LOG(WARN, "fail to seek file", K(ret)); - } - return ret; -} - -int ObTmpFileManager::truncate(const int64_t fd, const int64_t offset) -{ - int ret = OB_SUCCESS; - ObTmpFileHandle file_handle; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret)); - } else if (OB_FAIL(files_.get(fd, file_handle))) { - STORAGE_LOG(WARN, "fail to get tmp file handle", K(ret), K(fd)); - } else if (OB_FAIL(file_handle.get_resource_ptr()->truncate(offset))) { - STORAGE_LOG(WARN, "fail to seek file", K(ret)); - } - return ret; -} - -int ObTmpFileManager::get_tmp_file_handle(const int64_t fd, ObTmpFileHandle &handle) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret)); - } else if (OB_FAIL(files_.get(fd, handle))) { - STORAGE_LOG(WARN, "fail to get tmp file handle", K(ret), K(fd)); - } - return ret; -} - -int ObTmpFileManager::remove(const int64_t fd) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret)); - } else{ - if (OB_FAIL(files_.erase(fd))) { - if (common::OB_ENTRY_NOT_EXIST != ret) { - STORAGE_LOG(WARN, "fail to get tmp file handle", K(ret), K(fd)); - } else { - ret = OB_SUCCESS; - STORAGE_LOG(INFO, "this tmp file has been removed", K(fd), K(common::lbt())); - } - } else { - ObTaskController::get().allow_next_syslog(); - STORAGE_LOG(INFO, "succeed to remove a tmp file", K(fd), K(common::lbt())); - } - } - return ret; -} - -int ObTmpFileManager::remove_tenant_file(const uint64_t tenant_id) -{ - int ret = OB_SUCCESS; - common::ObSEArray fd_list; - fd_list.set_attr(ObMemAttr(MTL_ID(), "TMP_FD_LIST")); - RmTenantTmpFileOp rm_tenant_file_op(tenant_id, &fd_list); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret)); - } else if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(tenant_id)); - } else if (OB_FAIL(files_.foreach(rm_tenant_file_op))) { - STORAGE_LOG(WARN, "fail to foreach files_", K(ret), K(tenant_id)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < fd_list.count(); i++) { - if (OB_FAIL(remove(fd_list.at(i)))) { - STORAGE_LOG(WARN, "fail to remove tmp file", K(ret), K(fd_list.at(i)), K(i)); - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(OB_TMP_FILE_STORE.free_tenant_file_store(tenant_id))) { - if (OB_ENTRY_NOT_EXIST == ret) { - ret = OB_SUCCESS; - } else { - STORAGE_LOG(WARN, "fail to free tmp tenant file store", K(ret), K(tenant_id)); - } - } - } - } - return ret; -} - -int ObTmpFileManager::get_all_tenant_id(common::ObIArray &tenant_ids) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret)); - } else if (OB_FAIL(OB_TMP_FILE_STORE.get_all_tenant_id(tenant_ids))) { - STORAGE_LOG(WARN, "fail to get all tenant ids", K(ret)); - } - return ret; -} - -int ObTmpFileManager::sync(const int64_t fd, const int64_t timeout_ms) -{ - int ret = OB_SUCCESS; - ObTmpFileHandle file_handle; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret)); - } else if (OB_FAIL(files_.get(fd, file_handle))) { - STORAGE_LOG(WARN, "fail to get tmp file handle", K(ret), K(fd)); - } else if (OB_FAIL(file_handle.get_resource_ptr()->sync(timeout_ms))) { - STORAGE_LOG(WARN, "fail to close file", K(ret)); - } - return ret; -} - -ObTmpFileManager::ObTmpFileManager() - : is_inited_(false), - next_fd_(-1), - next_dir_(-1), - rm_file_lock_(common::ObLatchIds::TMP_FILE_MGR_LOCK), - files_() -{ -} - -ObTmpFileManager::~ObTmpFileManager() -{ - destroy(); -} - -void ObTmpFileManager::destroy() -{ - files_.destroy(); - OB_TMP_FILE_STORE.destroy(); - next_fd_ = -1; - next_dir_ = -1; - is_inited_ = false; -} - -int ObTmpFileManager::dec_handle_ref(ObTmpFileHandle &handle) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret)); - } else if (OB_FAIL(files_.dec_handle_ref(handle.ptr_))) { - STORAGE_LOG(WARN, "fail to dec handle ref without lock", K(ret)); - } - return ret; -} - -int ObTmpFileManager::get_tmp_file_size(const int64_t fd, int64_t &file_size) -{ - int ret = OB_SUCCESS; - ObTmpFileHandle file_handle; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret)); - } else if (fd < 0) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(fd)); - } else if (OB_FAIL(files_.get(fd, file_handle))) { - STORAGE_LOG(WARN, "fail to get tmp file handle", K(ret), K(fd)); - } else { - file_handle.get_resource_ptr()->get_file_size(file_size); - } - return ret; -} - -} // end namespace blocksstable -} // end namespace oceanbase diff --git a/src/storage/blocksstable/ob_tmp_file.h b/src/storage/blocksstable/ob_tmp_file.h deleted file mode 100644 index da65938f4..000000000 --- a/src/storage/blocksstable/ob_tmp_file.h +++ /dev/null @@ -1,444 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#ifndef OCEANBASE_STORAGE_BLOCKSSTABLE_OB_TMP_FILE_H_ -#define OCEANBASE_STORAGE_BLOCKSSTABLE_OB_TMP_FILE_H_ - -#include "storage/ob_resource_map.h" -#include "lib/container/ob_se_array.h" -#include "storage/ob_resource_map.h" -#include "ob_macro_block_handle.h" -#include "ob_block_manager.h" -#include "ob_tmp_file_store.h" - -namespace oceanbase -{ -namespace blocksstable -{ - -class ObTmpFile; -class ObTmpFileExtent; - -struct ObTmpFileIOInfo final -{ -public: - ObTmpFileIOInfo(); - ~ObTmpFileIOInfo(); - void reset(); - bool is_valid() const; - TO_STRING_KV(K_(fd), K_(dir_id), K_(size), K_(io_timeout_ms), K_(tenant_id), KP_(buf), K_(io_desc)); - int64_t fd_; - int64_t dir_id_; - int64_t size_; - int64_t io_timeout_ms_; - uint64_t tenant_id_; - char *buf_; - common::ObIOFlag io_desc_; - bool disable_page_cache_; -}; - -class ObTmpFileIOHandle final -{ -public: - struct ObIOReadHandle final - { - ObIOReadHandle(); - ObIOReadHandle(const ObMacroBlockHandle ¯o_handle, char *buf, const int64_t offset, - const int64_t size); - ~ObIOReadHandle(); - ObIOReadHandle(const ObIOReadHandle &other); - ObIOReadHandle &operator=(const ObIOReadHandle &other); - TO_STRING_KV(K_(macro_handle), K_(offset), K_(size), KP_(buf)); - ObMacroBlockHandle macro_handle_; - char *buf_; - int64_t offset_; - int64_t size_; - }; - - struct ObPageCacheHandle final - { - ObPageCacheHandle(); - ObPageCacheHandle(const ObTmpPageValueHandle &page_handle, char *buf, const int64_t offset, - const int64_t size); - ~ObPageCacheHandle(); - ObPageCacheHandle(const ObPageCacheHandle &other); - ObPageCacheHandle &operator=(const ObPageCacheHandle &other); - TO_STRING_KV(K_(page_handle), K_(offset), K_(size), KP_(buf)); - ObTmpPageValueHandle page_handle_; - char *buf_; - int64_t offset_; - int64_t size_; - }; - - struct ObBlockCacheHandle final - { - ObBlockCacheHandle(); - ObBlockCacheHandle(const ObTmpBlockValueHandle &block_handle, char *buf, const int64_t offset, - const int64_t size); - ~ObBlockCacheHandle(); - ObBlockCacheHandle(const ObBlockCacheHandle &other); - ObBlockCacheHandle &operator=(const ObBlockCacheHandle &other); - TO_STRING_KV(K_(block_handle), K_(offset), K_(size), KP_(buf)); - ObTmpBlockValueHandle block_handle_; - char *buf_; - int64_t offset_; - int64_t size_; - }; - - ObTmpFileIOHandle(); - ~ObTmpFileIOHandle(); - OB_INLINE char *get_buffer() { return buf_; } - OB_INLINE int64_t get_data_size() { return size_; } - OB_INLINE bool is_disable_page_cache() const { return disable_page_cache_; } - int prepare_read( - const int64_t read_size, - const int64_t read_offset, - const common::ObIOFlag io_flag, - char *read_buf, - int64_t fd, - int64_t dir_id, - uint64_t tenant_id, - const bool disable_page_cache); - int prepare_write( - char *write_buf, - const int64_t write_size, - int64_t fd, - int64_t dir_id, - uint64_t tenant_id); - OB_INLINE void add_data_size(const int64_t size) { size_ += size; } - OB_INLINE void sub_data_size(const int64_t size) { size_ -= size; } - OB_INLINE void set_update_offset_in_file() { update_offset_in_file_ = true; } - OB_INLINE void set_last_read_offset(const int64_t last_read_offset) - { - last_read_offset_ = last_read_offset; - } - int wait(); - void reset(); - bool is_valid() const; - common::ObIArray &get_io_handles() - { - return io_handles_; - } - common::ObIArray &get_page_cache_handles() - { - return page_cache_handles_; - } - common::ObIArray &get_block_cache_handles() - { - return block_cache_handles_; - } - int record_block_id(const int64_t block_it); - - OB_INLINE int64_t get_last_read_offset() const { return last_read_offset_; } - int64_t get_last_extent_id() const; - void set_last_extent_id(const int64_t last_extent_id); - - TO_STRING_KV(KP_(buf), K_(size), K_(is_read), K_(has_wait), K_(expect_read_size), - K_(last_read_offset), K_(io_flag), K_(update_offset_in_file)); - -private: - int wait_write_finish(const int64_t timeout_ms); - int wait_read_finish(const int64_t timeout_ms); - int do_read_wait(const int64_t timeout_ms); - -private: - common::ObSEArray io_handles_; - common::ObSEArray page_cache_handles_; - common::ObSEArray block_cache_handles_; - common::hash::ObHashSet write_block_ids_; - int64_t fd_; - int64_t dir_id_; - uint64_t tenant_id_; - char *buf_; - int64_t size_; //has read or to write size. - bool is_read_; - bool has_wait_; - bool is_finished_; - bool disable_page_cache_; - int ret_code_; - int64_t expect_read_size_; - int64_t last_read_offset_; // only for more than 8MB read. - common::ObIOFlag io_flag_; - bool update_offset_in_file_; - int64_t last_fd_; - int64_t last_extent_id_; - DISALLOW_COPY_AND_ASSIGN(ObTmpFileIOHandle); -}; - -class ObTmpFileExtent final -{ -public: - explicit ObTmpFileExtent(ObTmpFile *file); - ~ObTmpFileExtent(); - int read(const ObTmpFileIOInfo &io_info, const int64_t offset, const int64_t size, - char *buf, ObTmpFileIOHandle &handle); - int write(const ObTmpFileIOInfo &io_info, int64_t &size, char *&buf); - void reset(); - OB_INLINE bool is_closed() const { return ATOMIC_LOAD(&is_closed_); } - OB_INLINE bool is_truncated() const { return ATOMIC_LOAD(&is_truncated_); } - void set_truncated() { ATOMIC_STORE(&is_truncated_, true); } - bool is_valid(); - bool close(bool force = false); - bool close(uint8_t &free_page_start_id, uint8_t &free_page_nums, bool force = false); - void unclose(const int32_t page_nums = -1); - bool is_alloced() const { return is_alloced_; } - OB_INLINE void set_global_offset(const int64_t g_offset_start, const int64_t g_offset_end); - OB_INLINE void get_global_offset(int64_t &g_offset_start, int64_t &g_offset_end) const; - OB_INLINE int64_t get_global_end() const { return g_offset_end_; } - OB_INLINE int64_t get_global_start() const { return g_offset_start_; } - OB_INLINE void alloced() { is_alloced_ = true; } - OB_INLINE void set_start_page_id(const uint8_t start_page_id) { start_page_id_ = start_page_id; } - OB_INLINE uint8_t get_start_page_id() const { return start_page_id_; } - OB_INLINE void set_page_nums(const uint8_t page_nums) { page_nums_ = page_nums; } - OB_INLINE uint8_t get_page_nums() const { return page_nums_; } - OB_INLINE void set_block_id(const int64_t block_id) { block_id_ = block_id; } - OB_INLINE int64_t get_block_id() const { return block_id_; } - OB_INLINE int32_t get_offset() const { return ATOMIC_LOAD(&offset_); } - OB_INLINE ObTmpFile &get_owner() { return *owner_; } - TO_STRING_KV(K_(is_alloced), K_(fd), K_(g_offset_start), K_(g_offset_end), KP_(owner), - K_(start_page_id), K_(page_nums), K_(block_id), K_(offset), K_(is_closed)); - -private: - int try_sync_block(); - -private: - bool is_alloced_; - bool is_closed_; // only if close, this extent cannot be used. - uint8_t start_page_id_; - uint8_t page_nums_; - int32_t offset_; - int64_t fd_; - int64_t g_offset_start_; - int64_t g_offset_end_; - ObTmpFile *owner_; - int64_t block_id_; - common::SpinRWLock lock_; - bool is_truncated_; - DISALLOW_COPY_AND_ASSIGN(ObTmpFileExtent); -}; - -class ObTmpFileMeta final -{ -public: - explicit ObTmpFileMeta() : fd_(-1), dir_id_(-1), allocator_(NULL), extents_() - { - extents_.set_attr(ObMemAttr(MTL_ID(), "TMP_META")); - } - ~ObTmpFileMeta(); - int clear(); - int init(const int64_t fd, const int64_t dir_id, common::ObIAllocator *allocator); - ObTmpFileExtent *get_last_extent(); - common::ObIArray &get_extents() { return extents_; } - int push_back_extent(ObTmpFileExtent *extent) { return extents_.push_back(extent); } - int pop_back_extent(ObTmpFileExtent *&extent) { return extents_.pop_back(extent); } - void pop_back_extent() { extents_.pop_back(); } - int deep_copy(const ObTmpFileMeta &other); - OB_INLINE int64_t get_fd() const { return fd_; } - OB_INLINE int64_t get_dir_id() const { return dir_id_; } - TO_STRING_KV(K_(fd), K_(dir_id), K_(extents)); - -private: - int64_t fd_; - int64_t dir_id_; - common::ObIAllocator *allocator_; - ExtentArray extents_; // b-tree is better - DISALLOW_COPY_AND_ASSIGN(ObTmpFileMeta); -}; - -class ObTmpFile final -{ -public: - enum FileWhence - { - SET_SEEK = 0, - CUR_SEEK, - }; - ObTmpFile(); - ~ObTmpFile(); - int init(const int64_t fd, const int64_t dir_id, common::ObIAllocator &allocator); - int aio_read(const ObTmpFileIOInfo &io_info, ObTmpFileIOHandle &handle); - int aio_pread(const ObTmpFileIOInfo &io_info, const int64_t offset, ObTmpFileIOHandle &handle); - int read(const ObTmpFileIOInfo &io_info, ObTmpFileIOHandle &handle); - int pread(const ObTmpFileIOInfo &io_info, const int64_t offset, ObTmpFileIOHandle &handle); - int aio_write(const ObTmpFileIOInfo &io_info, ObTmpFileIOHandle &handle); - int write(const ObTmpFileIOInfo &io_info); - int seek(const int64_t offset, const int whence); - - // the data before the offset is released - int truncate(const int64_t offset); - int clear(); - int64_t get_dir_id() const; - uint64_t get_tenant_id() const; - int64_t get_fd() const; - int sync(const int64_t timeout_ms); - int deep_copy(char *buf, const int64_t buf_len, ObTmpFile *&value) const; - // only for ObTmpFileIOHandle, once more than READ_SIZE_PER_BATCH read. - int once_aio_read_batch( - const ObTmpFileIOInfo &io_info, - const bool need_update_offset, - int64_t &offset, - ObTmpFileIOHandle &handle); - - void get_file_size(int64_t &file_size); - OB_INLINE int64_t get_deep_copy_size() const { return sizeof(*this); } ; - TO_STRING_KV(K_(file_meta), K_(is_big), K_(tenant_id), K_(is_inited)); - -private: - static int fill_zero(char *buf, const int64_t size); - int write_file_extent(const ObTmpFileIOInfo &io_info, ObTmpFileExtent *file_extent, - int64_t &size, char *&buf); - int aio_read_without_lock( - const ObTmpFileIOInfo &io_info, - int64_t &offset, - ObTmpFileIOHandle &handle); - int once_aio_read_batch_without_lock( - const ObTmpFileIOInfo &io_info, - int64_t &offset, - ObTmpFileIOHandle &handle); - int64_t small_file_prealloc_size(); - int64_t big_file_prealloc_size(); - int64_t find_first_extent(const int64_t offset); - int64_t get_extent_cache(const int64_t offset, const ObTmpFileIOHandle &handle); - -private: - // NOTE: - // 1.The pre-allocated macro should satisfy the following inequality: - // SMALL_FILE_MAX_THRESHOLD < BIG_FILE_PREALLOC_EXTENT_SIZE < block size - static const int64_t SMALL_FILE_MAX_THRESHOLD = 4; - static const int64_t BIG_FILE_PREALLOC_EXTENT_SIZE = 8; - static const int64_t READ_SIZE_PER_BATCH = 8 * 1024 * 1024; // 8MB - - bool is_inited_; - bool is_big_; - int64_t offset_; // read offset - uint64_t tenant_id_; - common::SpinRWLock lock_; - common::ObIAllocator *allocator_; - ObTmpFileMeta file_meta_; - - // content before read_guard_ is truncated, which means the space is released. read before read_guard_ will only return 0; - int64_t read_guard_; - - // to optimize truncated speed, record the last_truncated_extent_id, so that we do not need to binary search the extent id every time we truncated. - int64_t next_truncated_extent_id_; - - DISALLOW_COPY_AND_ASSIGN(ObTmpFile); -}; - -class ObTmpFileHandle final: public storage::ObResourceHandle -{ -public: - ObTmpFileHandle(); - ~ObTmpFileHandle(); - virtual void reset() override; -private: - friend class ObTmpFileManager; - DISALLOW_COPY_AND_ASSIGN(ObTmpFileHandle); -}; - -class ObTmpFileManager final -{ -public: - static ObTmpFileManager &get_instance(); - int init(); - - int alloc_dir(int64_t &dir); - int open(int64_t &fd, int64_t &dir); - // NOTE: - // default order read, if want to read random, should be seek first. - int aio_read(const ObTmpFileIOInfo &io_info, ObTmpFileIOHandle &handle); - int aio_pread(const ObTmpFileIOInfo &io_info, const int64_t offset, ObTmpFileIOHandle &handle); - // NOTE: - // default order read, if want to read random, should be seek first. - int read(const ObTmpFileIOInfo &io_info, ObTmpFileIOHandle &handle); - int pread(const ObTmpFileIOInfo &io_info, const int64_t offset, ObTmpFileIOHandle &handle); - // NOTE: - // only support order write. - int aio_write(const ObTmpFileIOInfo &io_info, ObTmpFileIOHandle &handle); - // NOTE: - // only support order write. - int write(const ObTmpFileIOInfo &io_info); - // only for read: - // 1. whence == SET_SEEK, inner offset = offset; - // 2. whence == CUR_SEEK, inner offset -= offset; - int seek(const int64_t fd, const int64_t offset, const int whence); - // NOTE: - // remove file and all of block in this file, after not used file, should be called in case - // of block leak. - int truncate(const int64_t fd, const int64_t offset); - int remove(const int64_t fd); - int remove_tenant_file(const uint64_t tenant_id); - - int get_all_tenant_id(common::ObIArray &tenant_ids); - - int sync(const int64_t fd, const int64_t timeout_ms); - - void destroy(); - int dec_handle_ref(ObTmpFileHandle &handle); - // Returns the size of the current temporary file - int get_tmp_file_size(const int64_t fd, int64_t &file_size); - -public: - friend class ObTmpFileIOHandle; - -private: - class RmTenantTmpFileOp - { - public: - RmTenantTmpFileOp(const uint64_t tenant_id, common::ObIArray *fd_list) - : tenant_id_(tenant_id), fd_list_(fd_list) - {} - ~RmTenantTmpFileOp() = default; - int operator()(common::hash::HashMapPair &entry) - { - int ret = OB_SUCCESS; - ObTmpFile *tmp_file = entry.second; - if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id_) - || OB_ISNULL(fd_list_) || OB_ISNULL(tmp_file)) { - ret = common::OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret)); - } else if (tmp_file->get_tenant_id() == tenant_id_) { - if (OB_FAIL(fd_list_->push_back(tmp_file->get_fd()))) { - STORAGE_LOG(WARN, "fd_list_ push back failed", K(ret)); - } - } - return ret; - } - private: - const uint64_t tenant_id_; - common::ObIArray *fd_list_; - }; - -private: - ObTmpFileManager(); - ~ObTmpFileManager(); - int get_next_dir(int64_t &next_dir); - int get_next_fd(int64_t &next_fd); - void next_value(int64_t ¤t_val, int64_t &next_val); - int get_tmp_file_handle(const int64_t fd, ObTmpFileHandle &handle); - -private: - static const int64_t DEFAULT_BUCKET_NUM = 10243L; - bool is_inited_; - int64_t next_fd_; - int64_t next_dir_; - common::SpinRWLock rm_file_lock_; - storage::ObResourceMap files_; - - DISALLOW_COPY_AND_ASSIGN(ObTmpFileManager); -}; - -#define FILE_MANAGER_INSTANCE_V2 (::oceanbase::blocksstable::ObTmpFileManager::get_instance()) - -} // end namespace blocksstable -} // end namespace oceanbase -#endif // OCEANBASE_STORAGE_BLOCKSSTABLE_OB_TMP_FILE_H_ diff --git a/src/storage/blocksstable/ob_tmp_file_cache.cpp b/src/storage/blocksstable/ob_tmp_file_cache.cpp deleted file mode 100644 index 3d8c9cd2e..000000000 --- a/src/storage/blocksstable/ob_tmp_file_cache.cpp +++ /dev/null @@ -1,1655 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#include "observer/omt/ob_tenant_config_mgr.h" -#include "lib/stat/ob_diagnose_info.h" -#include "common/ob_smart_var.h" -#include "storage/ob_file_system_router.h" -#include "share/ob_task_define.h" -#include "ob_tmp_file_cache.h" -#include "ob_tmp_file.h" -#include "ob_tmp_file_store.h" -#include "ob_block_manager.h" - -using namespace oceanbase::storage; -using namespace oceanbase::share; - -namespace oceanbase -{ -namespace blocksstable -{ - -ObTmpPageCacheKey::ObTmpPageCacheKey() - : block_id_(-1), page_id_(-1), tenant_id_(0) -{ -} - -ObTmpPageCacheKey::ObTmpPageCacheKey(const int64_t block_id, const int64_t page_id, - const uint64_t tenant_id) - : block_id_(block_id), page_id_(page_id), tenant_id_(tenant_id) -{ -} - -ObTmpPageCacheKey::~ObTmpPageCacheKey() -{ -} - -bool ObTmpPageCacheKey::operator ==(const ObIKVCacheKey &other) const -{ - const ObTmpPageCacheKey &other_key = reinterpret_cast (other); - return block_id_ == other_key.block_id_ - && page_id_ == other_key.page_id_ - && tenant_id_ == other_key.tenant_id_; -} - -uint64_t ObTmpPageCacheKey::get_tenant_id() const -{ - return tenant_id_; -} - -uint64_t ObTmpPageCacheKey::hash() const -{ - return murmurhash(this, sizeof(ObTmpPageCacheKey), 0); -} - -int64_t ObTmpPageCacheKey::size() const -{ - return sizeof(*this); -} - -int ObTmpPageCacheKey::deep_copy(char *buf, const int64_t buf_len, ObIKVCacheKey *&key) const -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(buf) || OB_UNLIKELY(buf_len < size())) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "Invalid argument, ", K(ret)); - } else if (OB_UNLIKELY(!is_valid())) { - ret = OB_INVALID_DATA; - STORAGE_LOG(WARN, "invalid tmp page cache key, ", K(*this), K(ret)); - } else { - key = new (buf) ObTmpPageCacheKey(block_id_, page_id_, tenant_id_); - } - return ret; -} - -bool ObTmpPageCacheKey::is_valid() const -{ - return OB_LIKELY(block_id_ > 0 && page_id_ >= 0 && tenant_id_ > 0 && size() > 0); -} - -ObTmpPageCacheValue::ObTmpPageCacheValue(char *buf) - : buf_(buf), size_(ObTmpMacroBlock::get_default_page_size()) -{ -} - -ObTmpPageCacheValue::~ObTmpPageCacheValue() -{ -} - -int64_t ObTmpPageCacheValue::size() const -{ - return sizeof(*this) + size_; -} - -int ObTmpPageCacheValue::deep_copy(char *buf, const int64_t buf_len, ObIKVCacheValue *&value) const -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(buf) || OB_UNLIKELY(buf_len < size())) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid arguments", K(ret), KP(buf), K(buf_len), - "request_size", size()); - } else if (OB_UNLIKELY(!is_valid())) { - ret = OB_INVALID_DATA; - STORAGE_LOG(WARN, "invalid tmp page cache value", K(ret)); - } else { - ObTmpPageCacheValue *pblk_value = new (buf) ObTmpPageCacheValue(buf + sizeof(*this)); - MEMCPY(buf + sizeof(*this), buf_, size() - sizeof(*this)); - pblk_value->size_ = size_; - value = pblk_value; - } - return ret; -} - -int ObTmpPageCache::inner_read_io(const ObTmpBlockIOInfo &io_info, - ObITmpPageIOCallback *callback, - ObMacroBlockHandle ¯o_block_handle) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(read_io(io_info, callback, macro_block_handle))) { - if (macro_block_handle.get_io_handle().is_empty()) { - // TODO: After the continuous IO has been optimized, this should - // not happen. - if (OB_FAIL(macro_block_handle.wait())) { - STORAGE_LOG(WARN, "fail to wait tmp page io", K(ret), KP(callback)); - } else if (OB_FAIL(read_io(io_info, callback, macro_block_handle))) { - STORAGE_LOG(WARN, "fail to read tmp page from io", K(ret), KP(callback)); - } - } else { - STORAGE_LOG(WARN, "fail to read tmp page from io", K(ret), KP(callback)); - } - } - // Avoid double_free with io_handle - if (OB_FAIL(ret) && OB_NOT_NULL(callback) && OB_NOT_NULL(callback->get_allocator())) { - common::ObIAllocator *allocator = callback->get_allocator(); - callback->~ObITmpPageIOCallback(); - allocator->free(callback); - } - return ret; -} - -int ObTmpPageCache::direct_read(const ObTmpBlockIOInfo &info, - ObMacroBlockHandle &mb_handle, - common::ObIAllocator &allocator) -{ - int ret = OB_SUCCESS; - void *buf = nullptr; - ObTmpDirectReadPageIOCallback *callback = nullptr; - if (OB_ISNULL(buf = allocator.alloc(sizeof(ObTmpDirectReadPageIOCallback)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - STORAGE_LOG(WARN, "allocate callback memory failed", K(ret)); - } else { - // fill the callback - callback = new (buf) ObTmpDirectReadPageIOCallback; - callback->cache_ = this; - callback->offset_ = info.offset_; - callback->allocator_ = &allocator; - if (OB_FAIL(inner_read_io(info, callback, mb_handle))) { - STORAGE_LOG(WARN, "fail to inner read io", K(ret), K(mb_handle)); - } - // There is no need to handle error cases (freeing the memory of the - // callback) because inner_read_io will handle error cases and free the - // memory of the callback. - } - return ret; -} - -int ObTmpPageCache::prefetch( - const ObTmpPageCacheKey &key, - const ObTmpBlockIOInfo &info, - ObMacroBlockHandle &mb_handle, - common::ObIAllocator &allocator) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!key.is_valid() )) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "Invalid arguments", K(ret), K(key)); - } else { - // fill the callback - void *buf = nullptr; - ObTmpPageIOCallback *callback = nullptr; - if (OB_ISNULL(buf = allocator.alloc(sizeof(ObTmpPageIOCallback)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - STORAGE_LOG(WARN, "allocate callback memory failed", K(ret)); - } else { - callback = new (buf) ObTmpPageIOCallback; - callback->cache_ = this; - callback->offset_ = info.offset_; - callback->allocator_ = &allocator; - callback->key_ = key; - if (OB_FAIL(inner_read_io(info, callback, mb_handle))) { - STORAGE_LOG(WARN, "fail to inner read io", K(ret), K(mb_handle)); - } - // There is no need to handle error cases (freeing the memory of the - // callback) because inner_read_io will handle error cases and free the - // memory of the callback. - } - } - return ret; -} - -int ObTmpPageCache::prefetch( - const ObTmpBlockIOInfo &info, - const common::ObIArray &page_io_infos, - ObMacroBlockHandle &mb_handle, - common::ObIAllocator &allocator) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(page_io_infos.count() <= 0)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "Invalid arguments", K(ret), K(page_io_infos.count()), K(info)); - } else { - void *buf = nullptr; - ObTmpMultiPageIOCallback *callback = nullptr; - if (OB_ISNULL(buf = allocator.alloc(sizeof(ObTmpMultiPageIOCallback)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - STORAGE_LOG(WARN, "allocate callback memory failed", K(ret)); - } else { - callback = new (buf) ObTmpMultiPageIOCallback; - callback->cache_ = this; - callback->offset_ = info.offset_; - callback->allocator_ = &allocator; - if (OB_FAIL(callback->page_io_infos_.assign(page_io_infos))) { - STORAGE_LOG(WARN, "fail to assign page io infos", K(ret), K(page_io_infos.count()), K(info)); - if (OB_NOT_NULL(callback)) { // handle ObArray assign fail case and free callback - callback->~ObTmpMultiPageIOCallback(); - allocator.free(callback); - callback = nullptr; - } - } else if (OB_FAIL(inner_read_io(info, callback, mb_handle))) { - STORAGE_LOG(WARN, "fail to inner read io", K(ret), K(mb_handle)); - } - // inner_read_io will handle error cases and free the memory of callback. - } - } - return ret; -} - -int ObTmpPageCache::get_cache_page(const ObTmpPageCacheKey &key, ObTmpPageValueHandle &handle) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!key.is_valid())) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid arguments", K(ret), K(key)); - } else { - const ObTmpPageCacheValue *value = NULL; - if (OB_FAIL(get(key, value, handle.handle_))) { - if (OB_UNLIKELY(OB_ENTRY_NOT_EXIST != ret)) { - STORAGE_LOG(WARN, "fail to get key from page cache", K(ret)); - } else { - EVENT_INC(ObStatEventIds::TMP_PAGE_CACHE_MISS); - } - } else { - if (OB_ISNULL(value)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "unexpected error, the value must not be NULL", K(ret)); - } else { - handle.value_ = const_cast(value); - EVENT_INC(ObStatEventIds::TMP_PAGE_CACHE_HIT); - } - } - } - return ret; -} - -ObTmpPageCache::ObITmpPageIOCallback::ObITmpPageIOCallback() - : cache_(NULL), allocator_(NULL), offset_(0), data_buf_(NULL) -{ - static_assert(sizeof(*this) <= CALLBACK_BUF_SIZE, "IOCallback buf size not enough"); -} - -ObTmpPageCache::ObITmpPageIOCallback::~ObITmpPageIOCallback() -{ - if (NULL != allocator_ && NULL != data_buf_) { - allocator_->free(data_buf_); - data_buf_ = NULL; - } - allocator_ = NULL; -} - -int ObTmpPageCache::ObITmpPageIOCallback::alloc_data_buf(const char *io_data_buffer, const int64_t data_size) -{ - int ret = alloc_and_copy_data(io_data_buffer, data_size, allocator_, data_buf_); - return ret; -} - -int ObTmpPageCache::ObITmpPageIOCallback::process_page( - const ObTmpPageCacheKey &key, const ObTmpPageCacheValue &value) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!key.is_valid() || !value.is_valid())) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid arguments", K(ret), K(key), K(value)); - } else if (OB_FAIL(cache_->put(key, value, true/*overwrite*/))) { - STORAGE_LOG(WARN, "fail to put tmp page into cache", K(ret), K(key), K(value)); - } - return ret; -} - -ObTmpPageCache::ObTmpPageIOCallback::ObTmpPageIOCallback() - : key_() -{ - static_assert(sizeof(*this) <= CALLBACK_BUF_SIZE, "IOCallback buf size not enough"); -} - -ObTmpPageCache::ObTmpPageIOCallback::~ObTmpPageIOCallback() -{ - -} - -int ObTmpPageCache::ObTmpPageIOCallback::inner_process(const char *data_buffer, const int64_t size) -{ - int ret = OB_SUCCESS; - ObTimeGuard time_guard("TmpPage_Callback_Process", 100000); //100ms - if (OB_ISNULL(cache_) || OB_ISNULL(allocator_)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "Invalid tmp page cache callback or allocator", KP_(cache), KP_(allocator), K(ret)); - } else if (OB_UNLIKELY(size <= 0 || data_buffer == nullptr)) { - ret = OB_INVALID_DATA; - STORAGE_LOG(WARN, "invalid data buffer size", K(ret), K(size), KP(data_buffer)); - } else if (OB_FAIL(alloc_data_buf(data_buffer, size))) { - STORAGE_LOG(WARN, "Fail to allocate memory, ", K(ret), K(size)); - } else if (FALSE_IT(time_guard.click("alloc_data_buf"))) { - } else { - ObTmpPageCacheValue value(data_buf_); - if (OB_FAIL(process_page(key_, value))) { - STORAGE_LOG(WARN, "fail to process tmp page cache in callback", K(ret)); - } - time_guard.click("process_page"); - } - if (OB_FAIL(ret) && NULL != allocator_ && NULL != data_buf_) { - allocator_->free(data_buf_); - data_buf_ = NULL; - } - return ret; -} - -int64_t ObTmpPageCache::ObTmpPageIOCallback::size() const -{ - return sizeof(*this); -} - -const char *ObTmpPageCache::ObTmpPageIOCallback::get_data() -{ - return data_buf_; -} - -ObTmpPageCache::ObTmpMultiPageIOCallback::ObTmpMultiPageIOCallback() - : page_io_infos_() -{ - static_assert(sizeof(*this) <= CALLBACK_BUF_SIZE, "IOCallback buf size not enough"); -} -ObTmpPageCache::ObTmpMultiPageIOCallback::~ObTmpMultiPageIOCallback() -{ - page_io_infos_.reset(); - page_io_infos_.~ObIArray(); -} - -int ObTmpPageCache::ObTmpMultiPageIOCallback::inner_process(const char *data_buffer, const int64_t size) -{ - int ret = OB_SUCCESS; - ObTimeGuard time_guard("TmpMultiPage_Callback_Process", 100000); //100ms - if (OB_ISNULL(cache_) || OB_ISNULL(allocator_)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "Invalid tmp page cache callbackor allocator", KP_(cache), KP_(allocator), K(ret)); - } else if (OB_UNLIKELY(size <= 0 || data_buffer == nullptr)) { - ret = OB_INVALID_DATA; - STORAGE_LOG(WARN, "invalid data buffer size", K(ret), K(size), KP(data_buffer)); - } else if (OB_FAIL(alloc_data_buf(data_buffer, size))) { - STORAGE_LOG(WARN, "Fail to allocate memory, ", K(ret), K(size)); - } else if (FALSE_IT(time_guard.click("alloc_data_buf"))) { - } else { - for (int32_t i = 0; OB_SUCC(ret) && i < page_io_infos_.count(); i++) { - int64_t cur_offset = page_io_infos_.at(i).key_.get_page_id() - * ObTmpMacroBlock::get_default_page_size() - offset_; - cur_offset += ObTmpMacroBlock::get_header_padding(); - ObTmpPageCacheValue value(data_buf_ + cur_offset); - if (OB_FAIL(process_page(page_io_infos_.at(i).key_, value))) { - STORAGE_LOG(WARN, "fail to process tmp page cache in callback", K(ret)); - } - } - time_guard.click("process_page"); - page_io_infos_.reset(); - } - if (OB_FAIL(ret) && NULL != allocator_ && NULL != data_buf_) { - allocator_->free(data_buf_); - data_buf_ = NULL; - } - return ret; -} - -int64_t ObTmpPageCache::ObTmpMultiPageIOCallback::size() const -{ - return sizeof(*this); -} - -const char *ObTmpPageCache::ObTmpMultiPageIOCallback::get_data() -{ - return data_buf_; -} - -int64_t ObTmpPageCache::ObTmpDirectReadPageIOCallback::size() const -{ - return sizeof(*this); -} - -const char * ObTmpPageCache::ObTmpDirectReadPageIOCallback::get_data() -{ - return data_buf_; -} - -int ObTmpPageCache::ObTmpDirectReadPageIOCallback::inner_process(const char *data_buffer, const int64_t size) -{ - int ret = OB_SUCCESS; - ObTimeGuard time_guard("ObTmpDirectReadPageIOCallback", 100000); //100ms - if (OB_ISNULL(cache_) || OB_ISNULL(allocator_)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "Invalid tmp page cache callback allocator", KP_(cache), KP_(allocator), K(ret)); - } else if (OB_UNLIKELY(size <= 0 || data_buffer == nullptr)) { - ret = OB_INVALID_DATA; - STORAGE_LOG(WARN, "invalid data buffer size", K(ret), K(size), KP(data_buffer)); - } else if (OB_FAIL(alloc_data_buf(data_buffer, size))) { - STORAGE_LOG(WARN, "Fail to allocate memory, ", K(ret), K(size)); - } else if (FALSE_IT(time_guard.click("alloc_data_buf"))) { - } - if (OB_FAIL(ret) && NULL != allocator_ && NULL != data_buf_) { - allocator_->free(data_buf_); - data_buf_ = NULL; - } - return ret; -} - -int ObTmpPageCache::read_io(const ObTmpBlockIOInfo &io_info, ObITmpPageIOCallback *callback, - ObMacroBlockHandle &handle) -{ - int ret = OB_SUCCESS; - common::ObArenaAllocator allocator(ObModIds::OB_MACRO_FILE); - // fill the read info - ObMacroBlockReadInfo read_info; - read_info.io_desc_ = io_info.io_desc_; - read_info.macro_block_id_ = io_info.macro_block_id_; - read_info.io_timeout_ms_ = io_info.io_timeout_ms_; - read_info.io_callback_ = callback; - read_info.offset_ = io_info.offset_; - read_info.size_ = io_info.size_; - read_info.io_desc_.set_resource_group_id(THIS_WORKER.get_group_id()); - read_info.io_desc_.set_sys_module_id(ObIOModule::TMP_PAGE_CACHE_IO); - if (OB_FAIL(ObBlockManager::async_read_block(read_info, handle))) { - STORAGE_LOG(WARN, "fail to async read block", K(ret), K(read_info), KP(callback)); - } - return ret; -} - -ObTmpPageCache &ObTmpPageCache::get_instance() -{ - static ObTmpPageCache instance; - return instance; -} - -ObTmpPageCache::ObTmpPageCache() -{ -} - -ObTmpPageCache::~ObTmpPageCache() -{ -} - -int ObTmpPageCache::init(const char *cache_name, const int64_t priority) -{ - int ret = OB_SUCCESS; - if (OB_FAIL((common::ObKVCache::init( - cache_name, priority)))) { - STORAGE_LOG(WARN, "Fail to init kv cache, ", K(ret)); - } - return ret; -} - -void ObTmpPageCache::destroy() -{ - common::ObKVCache::destroy(); -} - -int ObTmpPageCache::put_page(const ObTmpPageCacheKey &key, const ObTmpPageCacheValue &value) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!key.is_valid() || !value.is_valid())) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid arguments", K(ret), K(key), K(value)); - } else if (OB_FAIL(put(key, value, true/*overwrite*/))) { - STORAGE_LOG(WARN, "fail to put page to page cache", K(ret), K(key), K(value)); - } - return ret; -} - -int ObTmpPageCache::get_page(const ObTmpPageCacheKey &key, ObTmpPageValueHandle &handle) -{ - int ret = OB_SUCCESS; - const ObTmpPageCacheValue *value = NULL; - if (OB_UNLIKELY(!key.is_valid())) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid arguments", K(ret), K(key)); - } else if (OB_FAIL(get(key, value, handle.handle_))) { - if (OB_UNLIKELY(OB_ENTRY_NOT_EXIST != ret)) { - STORAGE_LOG(WARN, "fail to get key from page cache", K(ret), K(key)); - } else { - EVENT_INC(ObStatEventIds::TMP_PAGE_CACHE_MISS); - } - } else { - if (OB_ISNULL(value)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "unexpected error, the value must not be NULL", K(ret)); - } else { - handle.value_ = const_cast(value); - EVENT_INC(ObStatEventIds::TMP_PAGE_CACHE_HIT); - } - } - return ret; -} - -ObTmpBlockCacheKey::ObTmpBlockCacheKey(const int64_t block_id, const uint64_t tenant_id) - : block_id_(block_id), tenant_id_(tenant_id) -{ -} - -bool ObTmpBlockCacheKey::operator ==(const ObIKVCacheKey &other) const -{ - const ObTmpBlockCacheKey &other_key = reinterpret_cast (other); - return block_id_ == other_key.block_id_ - && tenant_id_ == other_key.tenant_id_; -} - -uint64_t ObTmpBlockCacheKey::get_tenant_id() const -{ - return tenant_id_; -} - -uint64_t ObTmpBlockCacheKey::hash() const -{ - return murmurhash(this, sizeof(ObTmpBlockCacheKey), 0); -} - -int64_t ObTmpBlockCacheKey::size() const -{ - return sizeof(*this); -} - -int ObTmpBlockCacheKey::deep_copy(char *buf, const int64_t buf_len, ObIKVCacheKey *&key) const -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(buf) || OB_UNLIKELY(buf_len < size())) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "Invalid argument, ", K(ret)); - } else if (OB_UNLIKELY(!is_valid())) { - ret = OB_INVALID_DATA; - STORAGE_LOG(WARN, "invalid tmp block cache key, ", K(*this), K(ret)); - } else { - key = new(buf) ObTmpBlockCacheKey(block_id_, tenant_id_); - } - return ret; -} - -ObTmpBlockCacheValue::ObTmpBlockCacheValue(char *buf) - : buf_(buf), size_(ObTmpFileStore::get_block_size()) -{ -} - -int64_t ObTmpBlockCacheValue::size() const -{ - return sizeof(*this) + size_; -} - -int ObTmpBlockCacheValue::deep_copy(char *buf, const int64_t buf_len, ObIKVCacheValue *&value) const -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(buf) || OB_UNLIKELY(buf_len < size())) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid arguments", K(ret), K(ret), KP(buf), K(buf_len), - "request_size", size()); - } else if (OB_UNLIKELY(!is_valid())) { - ret = OB_INVALID_DATA; - STORAGE_LOG(WARN, "invalid tmp block cache value", K(ret)); - } else { - ObTmpBlockCacheValue *pblk_value = new (buf) ObTmpBlockCacheValue(buf + sizeof(*this)); - MEMCPY(buf + sizeof(*this), buf_, size() - sizeof(*this)); - pblk_value->size_ = size_; - value = pblk_value; - } - return ret; -} - -ObTmpBlockCache &ObTmpBlockCache::get_instance() -{ - static ObTmpBlockCache instance; - return instance; -} - -int ObTmpBlockCache::init(const char *cache_name, const int64_t priority) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(cache_name) || OB_UNLIKELY(priority <= 0)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(cache_name), K(priority)); - } else if (OB_FAIL((common::ObKVCache::init(cache_name, priority)))) { - STORAGE_LOG(WARN, "Fail to init kv cache, ", K(ret)); - } - return ret; -} - -int ObTmpBlockCache::get_block(const ObTmpBlockCacheKey &key, ObTmpBlockValueHandle &handle) -{ - int ret = OB_SUCCESS; - const ObTmpBlockCacheValue *value = NULL; - if (OB_UNLIKELY(!key.is_valid())) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid arguments", K(ret), K(key)); - } else if (OB_FAIL(get(key, value, handle.handle_))) { - if (OB_UNLIKELY(OB_ENTRY_NOT_EXIST != ret)) { - STORAGE_LOG(WARN, "fail to get key-value from block cache", K(ret)); - } else { - EVENT_INC(ObStatEventIds::TMP_BLOCK_CACHE_MISS); - } - } else { - if (OB_ISNULL(value)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "unexpected error, the value must not be NULL", K(ret)); - } else { - handle.value_ = const_cast(value); - EVENT_INC(ObStatEventIds::TMP_BLOCK_CACHE_HIT); - } - } - return ret; -} - -int ObTmpBlockCache::alloc_buf(const ObTmpBlockCacheKey &key, ObTmpBlockValueHandle &handle) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(alloc(key.get_tenant_id(), key.size(), - sizeof(ObTmpBlockCacheValue) + ObTmpFileStore::get_block_size(), - handle.kvpair_, handle.handle_, handle.inst_handle_))) { - STORAGE_LOG(WARN, "failed to alloc kvcache buf", K(ret)); - } else if (OB_FAIL(key.deep_copy(reinterpret_cast(handle.kvpair_->key_), - key.size(), handle.kvpair_->key_))) { - STORAGE_LOG(WARN, "failed to deep copy key", K(ret), K(key)); - } else { - char *buf = reinterpret_cast(handle.kvpair_->value_); - handle.value_ = new (buf) ObTmpBlockCacheValue(buf + sizeof(ObTmpBlockCacheValue)); - } - return ret; -} - -int ObTmpBlockCache::put_block(const ObTmpBlockCacheKey &key, ObTmpBlockValueHandle &handle) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!key.is_valid())) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid arguments", K(ret), K(key)); - } else if (OB_FAIL(put_kvpair(handle.inst_handle_, handle.kvpair_, - handle.handle_, true/*overwrite*/))) { - STORAGE_LOG(WARN, "fail to put tmp block to block cache", K(ret)); - } else { - handle.reset(); - } - return ret; -} - -void ObTmpBlockCache::destroy() -{ - common::ObKVCache::destroy(); -} - -ObTmpFileWaitTask::ObTmpFileWaitTask(ObTmpTenantMemBlockManager &mgr) - : mgr_(mgr) -{ -} - -void ObTmpFileWaitTask::runTimerTask() -{ - int ret = OB_SUCCESS; - if (OB_FAIL(mgr_.exec_wait())) { - STORAGE_LOG(WARN, "fail to wait block", K(ret)); - } -} - -ObTmpFileMemTask::ObTmpFileMemTask(ObTmpTenantMemBlockManager &mgr) - : mgr_(mgr) -{ -} - -void ObTmpFileMemTask::runTimerTask() -{ - int ret = OB_SUCCESS; - if (OB_FAIL(mgr_.change_mem())) { - if (OB_EAGAIN != ret){ - STORAGE_LOG(WARN, "fail to wait block", K(ret)); - } - } -} - - -ObTmpTenantMemBlockManager::IOWaitInfo::IOWaitInfo( - ObMacroBlockHandle &block_handle, ObTmpMacroBlock &block, ObIAllocator &allocator) - : block_handle_(&block_handle), block_(block), allocator_(allocator), ref_cnt_(0), ret_code_(OB_SUCCESS) -{ -} - -void ObTmpTenantMemBlockManager::IOWaitInfo::inc_ref() -{ - ATOMIC_INC(&ref_cnt_); -} - -void ObTmpTenantMemBlockManager::IOWaitInfo::dec_ref() -{ - int ret = OB_SUCCESS; - const int64_t tmp_ref = ATOMIC_SAF(&ref_cnt_, 1); - if (tmp_ref < 0) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(ERROR, "bug: ref_cnt < 0", K(ret), K(tmp_ref), K(lbt())); - ob_abort(); - } else if (0 == tmp_ref) { - this->~IOWaitInfo(); - allocator_.free(this); - } -} - -int ObTmpTenantMemBlockManager::IOWaitInfo::wait(int64_t timeout_ms) -{ - int ret = OB_SUCCESS; - if (block_.is_washing()) { - ObThreadCondGuard guard(cond_); - if (OB_FAIL(guard.get_ret())) { - STORAGE_LOG(ERROR, "fail to guard request condition", K(ret), K(block_.get_block_id())); - } else { - int64_t begin_us = ObTimeUtility::fast_current_time(); - int64_t wait_ms = timeout_ms; - while (OB_SUCC(ret) && block_.is_washing() && wait_ms > 0) { - if (OB_FAIL(cond_.wait(wait_ms))) { - STORAGE_LOG(WARN, "fail to wait block write condition", K(ret), K(wait_ms), K(block_.get_block_id())); - } else if (OB_FAIL(ret = ret_code_)) { - STORAGE_LOG(WARN, "fail to wait io info", K(ret), KPC(this)); - } else if (block_.is_washing()) { - wait_ms = timeout_ms - (ObTimeUtility::fast_current_time() - begin_us) / 1000; - } - } - if (OB_SUCC(ret) && OB_UNLIKELY(wait_ms <= 0)) { // rarely happen - ret = OB_TIMEOUT; - STORAGE_LOG(WARN, "fail to wait block io condition due to spurious wakeup", - K(ret), K(wait_ms), K(block_.get_block_id())); - } - } - } - return ret; -} - -int ObTmpTenantMemBlockManager::IOWaitInfo::exec_wait() -{ - int ret = OB_SUCCESS; - ObThreadCondGuard guard(cond_); - if (OB_FAIL(guard.get_ret())) { - STORAGE_LOG(ERROR, "lock io request condition failed", K(ret), K(block_.get_block_id())); - } else if (OB_NOT_NULL(block_handle_) && OB_FAIL(block_handle_->wait())) { - STORAGE_LOG(WARN, "wait handle wait io failed", K(ret), K(block_.get_block_id())); - block_handle_->reset_macro_id(); - } - reset_io(); - return ret; -} - -int ObTmpTenantMemBlockManager::IOWaitInfo::broadcast() -{ - int ret = OB_SUCCESS; - ObThreadCondGuard guard(cond_); - if (OB_FAIL(guard.get_ret())) { - STORAGE_LOG(ERROR, "lock io request condition failed", K(ret), K(block_.get_block_id())); - } else if (OB_FAIL(cond_.broadcast())) { - STORAGE_LOG(WARN, "wait handle wait io failed", K(ret), K(block_.get_block_id())); - } - return ret; -} - -ObTmpTenantMemBlockManager::IOWaitInfo::~IOWaitInfo() -{ - destroy(); -} - -void ObTmpTenantMemBlockManager::IOWaitInfo::destroy() -{ - ret_code_ = OB_SUCCESS; - reset_io(); - if (0 != ATOMIC_LOAD(&ref_cnt_)) { - int ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(ERROR, "unexpected error, ref cnt isn't zero", K(ret), KPC(this)); - } -} - -void ObTmpTenantMemBlockManager::IOWaitInfo::reset_io() -{ - if (OB_NOT_NULL(block_handle_)) { - block_handle_->get_io_handle().reset(); - block_handle_ = nullptr; - } -} - -ObTmpTenantMemBlockManager::ObIOWaitInfoHandle::ObIOWaitInfoHandle() - : wait_info_(nullptr) -{ -} - -ObTmpTenantMemBlockManager::ObIOWaitInfoHandle::ObIOWaitInfoHandle(const ObIOWaitInfoHandle &other) - : wait_info_(nullptr) -{ - *this = other; -} - -ObTmpTenantMemBlockManager::ObIOWaitInfoHandle::~ObIOWaitInfoHandle() -{ - reset(); -} - -void ObTmpTenantMemBlockManager::ObIOWaitInfoHandle::set_wait_info(IOWaitInfo *wait_info) -{ - if (OB_NOT_NULL(wait_info)) { - reset(); - wait_info->inc_ref(); // ref for handle - wait_info_ = wait_info; - } -} - -ObTmpTenantMemBlockManager::ObIOWaitInfoHandle& -ObTmpTenantMemBlockManager::ObIOWaitInfoHandle::operator=(const ObTmpTenantMemBlockManager::ObIOWaitInfoHandle &other) -{ - if (&other != this) { - set_wait_info(other.wait_info_); - } - return *this; -} - -bool ObTmpTenantMemBlockManager::ObIOWaitInfoHandle::is_empty() const -{ - return nullptr == wait_info_; -} - -bool ObTmpTenantMemBlockManager::ObIOWaitInfoHandle::is_valid() const -{ - return nullptr != wait_info_; -} - -void ObTmpTenantMemBlockManager::ObIOWaitInfoHandle::reset() -{ - if (OB_NOT_NULL(wait_info_)) { - wait_info_->dec_ref(); // ref for handle - wait_info_ = nullptr; - } -} - -ObTmpTenantMemBlockManager::ObTmpTenantMemBlockManager(ObTmpTenantFileStore &tenant_store) - : tenant_store_(tenant_store), - wait_info_queue_(), - t_mblk_map_(), - dir_to_blk_map_(), - blk_nums_threshold_(0), - block_cache_(NULL), - allocator_(NULL), - tenant_id_(0), - last_access_tenant_config_ts_(0), - last_tenant_mem_block_num_(1), - is_inited_(false), - tg_id_(OB_INVALID_INDEX), - stopped_(true), - washing_count_(0), - wait_task_(*this), - mem_task_(*this), - io_lock_(), - map_lock_(), - cond_(), - compare_() -{ -} - -ObTmpTenantMemBlockManager::~ObTmpTenantMemBlockManager() -{ -} - -int ObTmpTenantMemBlockManager::init(const uint64_t tenant_id, - common::ObConcurrentFIFOAllocator &allocator, - double blk_nums_threshold) -{ - int ret = OB_SUCCESS; - if (IS_INIT) { - ret = OB_INIT_TWICE; - STORAGE_LOG(WARN, "ObTmpBlockCache has been inited", K(ret)); - } else if (OB_UNLIKELY(blk_nums_threshold <= 0) || OB_UNLIKELY(blk_nums_threshold > 1)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(blk_nums_threshold)); - } else if (OB_FAIL(wait_handles_map_.create(MBLK_HASH_BUCKET_NUM, ObModIds::OB_TMP_BLOCK_MAP, - "WaitHdl", tenant_id))) { - STORAGE_LOG(WARN, "fail to create wait handles map", K(ret)); - } else if (OB_FAIL(t_mblk_map_.create(MBLK_HASH_BUCKET_NUM, ObModIds::OB_TMP_BLOCK_MAP, - "TmpMBlk", tenant_id))) { - STORAGE_LOG(WARN, "Fail to create allocating block map, ", K(ret)); - } else if (OB_FAIL(dir_to_blk_map_.create(MBLK_HASH_BUCKET_NUM, ObModIds::OB_TMP_MAP, - "DirToBlk", tenant_id))) { - STORAGE_LOG(WARN, "Fail to create tmp dir map, ", K(ret)); - } else if (OB_FAIL(map_lock_.init(MBLK_HASH_BUCKET_NUM, ObLatchIds::TMP_FILE_MEM_BLOCK_LOCK, "TmpMemBlkMgr", MTL_ID()))) { - STORAGE_LOG(WARN, "Fail to create tmp dir map, ", K(ret)); - } else if (OB_ISNULL(block_cache_ = &ObTmpBlockCache::get_instance())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "fail to get the block cache", K(ret)); - } else if (OB_FAIL(cond_.init(ObWaitEventIds::IO_CONTROLLER_COND_WAIT))) { - STORAGE_LOG(WARN, "fail to init condition", K(ret)); - } else if (OB_FAIL(TG_CREATE_TENANT(lib::TGDefIDs::COMMON_TIMER_THREAD, tg_id_))) { - STORAGE_LOG(WARN, "TG_CREATE_TENANT failed", KR(ret)); - } else if (OB_FAIL(TG_START(tg_id_))) { - STORAGE_LOG(WARN, "TG_START failed", KR(ret), K_(tg_id)); - } else if (FALSE_IT(stopped_ = false)) { - } else if (OB_FAIL(TG_SCHEDULE(tg_id_, wait_task_, TASK_INTERVAL, true/*repeat*/))) { - STORAGE_LOG(WARN, "TG_SCHEDULE task failed", KR(ret), K_(tg_id), K(TASK_INTERVAL)); - } else if (OB_FAIL(TG_SCHEDULE(tg_id_, mem_task_, MEMORY_TASK_INTERVAL, true/*repeat*/))) { - STORAGE_LOG(WARN, "TG_SCHEDULE task failed", KR(ret), K_(tg_id), K(MEMORY_TASK_INTERVAL)); - } else { - blk_nums_threshold_ = blk_nums_threshold; - tenant_id_ = tenant_id; - allocator_ = &allocator; - is_inited_ = true; - } - if (!is_inited_) { - destroy(); - } - return ret; -} - -int ObTmpTenantMemBlockManager::DestroyBlockMapOp::operator () (oceanbase::common::hash::HashMapPair &entry) -{ - int ret = OB_SUCCESS; - ObTmpMacroBlock *blk = entry.second; - if (OB_ISNULL(blk)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "block is null", K(ret)); - } else if (blk->is_memory()) { - if (OB_FAIL(blk->check_and_set_status( - ObTmpMacroBlock::BlockStatus::MEMORY, ObTmpMacroBlock::BlockStatus::DISKED))) { - if (OB_STATE_NOT_MATCH == ret) { - STORAGE_LOG(DEBUG, "this block is washing", K(ret), K(*blk)); - ret = OB_SUCCESS; - } else { - STORAGE_LOG(WARN, "check and set status failed", K(ret), K(*blk)); - } - } else if (OB_FAIL(blk->give_back_buf_into_cache())) { - STORAGE_LOG(WARN, "fail to put tmp block cache", K(ret), K(blk)); - } else { - tenant_store_.dec_block_cache_num(1); - } - } else { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "block status not correct", K(ret), K(blk)); - } - return ret; -} - -void ObTmpTenantMemBlockManager::destroy() -{ - int ret = OB_SUCCESS; - ObTmpMacroBlock *tmp = NULL; - stopped_ = true; - if (OB_INVALID_INDEX != tg_id_) { - TG_STOP(tg_id_); - TG_WAIT(tg_id_); - TG_DESTROY(tg_id_); - tg_id_ = OB_INVALID_INDEX; - } - const int64_t io_timeout_ms = GCONF._data_storage_io_timeout / 1000L; - ObSpLinkQueue::Link *node = NULL; - while (!wait_info_queue_.is_empty()) { - IOWaitInfo *wait_info = NULL; - if (OB_FAIL(wait_info_queue_.pop(node))) { - STORAGE_LOG(WARN, "pop wait handle failed", K(ret)); - } else if (FALSE_IT(wait_info = static_cast(node))) { - } else if (OB_FAIL(wait_info->exec_wait())) { - // overwrite ret - STORAGE_LOG(WARN, "fail to exec iohandle wait", K(ret), K_(tenant_id)); - } - } - ATOMIC_STORE(&washing_count_, 0); - DestroyBlockMapOp op(tenant_store_); - if (OB_FAIL(t_mblk_map_.foreach_refactored(op))) { - // overwrite ret - STORAGE_LOG(WARN, "destroy mblk map failed", K(ret)); - } - t_mblk_map_.destroy(); - blk_nums_threshold_ = 0; - dir_to_blk_map_.destroy(); - if (NULL != block_cache_) { - block_cache_ = NULL; - } - map_lock_.destroy(); - allocator_ = NULL; - is_inited_ = false; -} - -int ObTmpTenantMemBlockManager::get_block(const ObTmpBlockCacheKey &key, - ObTmpBlockValueHandle &handle) -{ - return block_cache_->get_block(key, handle); -} - -int ObTmpTenantMemBlockManager::alloc_buf(const ObTmpBlockCacheKey &key, - ObTmpBlockValueHandle &handle) -{ - return block_cache_->alloc_buf(key, handle); -} - -int ObTmpTenantMemBlockManager::free_macro_block(const int64_t block_id) -{ - int ret = OB_SUCCESS; - ObTmpMacroBlock *t_mblk = NULL; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpBlockCache has not been inited", K(ret)); - } else if (OB_UNLIKELY(block_id <= 0)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(block_id)); - } else if (OB_FAIL(t_mblk_map_.erase_refactored(block_id))) { - if (OB_HASH_NOT_EXIST != ret) { - STORAGE_LOG(WARN, "fail to erase tmp macro block", K(ret)); - } else { - ret = OB_SUCCESS; - STORAGE_LOG(DEBUG, "macro block has been erased", K(ret)); - } - } - return ret; -} - -int ObTmpTenantMemBlockManager::alloc_extent(const int64_t dir_id, const uint64_t tenant_id, - const int64_t size, ObTmpFileExtent &extent) -{ - int ret = OB_SUCCESS; - const int64_t page_nums = std::ceil(size * 1.0 / ObTmpMacroBlock::get_default_page_size()); - int64_t block_id = -1; - ObTmpMacroBlock *t_mblk = NULL; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpBlockCache has not been inited", K(ret)); - } else if (OB_FAIL(get_block_from_dir_cache(dir_id, tenant_id, page_nums, t_mblk))) { - bool is_found = false; - if (OB_FAIL(get_available_macro_block(dir_id, tenant_id, page_nums, t_mblk, is_found))) { - } else if (!is_found) { - ret = OB_STATE_NOT_MATCH; - STORAGE_LOG(DEBUG, "cannot find available macro block", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_ISNULL(t_mblk)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "unexpected error, t_mblk is nullptr", K(ret), KP(t_mblk)); - } else if (OB_FAIL(t_mblk->alloc(page_nums, extent))){ - STORAGE_LOG(WARN, "fail to alloc tmp extent", K(ret)); - } else if (OB_FAIL(refresh_dir_to_blk_map(dir_id, t_mblk))) { - STORAGE_LOG(WARN, "fail to refresh dir_to_blk_map", K(ret), K(*t_mblk)); - } - } - return ret; -} - -int ObTmpTenantMemBlockManager::get_block_from_dir_cache(const int64_t dir_id, const int64_t tenant_id, - const int64_t page_nums, ObTmpMacroBlock *&t_mblk) -{ - int ret = OB_SUCCESS; - int64_t block_id = -1; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpBlockCache has not been inited", K(ret)); - } else if (OB_FAIL(dir_to_blk_map_.get_refactored(dir_id, block_id))) { - STORAGE_LOG(DEBUG, "fail to get macro block from dir cache", K(ret), K(dir_id), K(block_id), K(dir_to_blk_map_.size())); - } else if (OB_FAIL(t_mblk_map_.get_refactored(block_id, t_mblk))) { - STORAGE_LOG(DEBUG, "the tmp macro block has been washed", K(ret), K(block_id)); - } else if (OB_ISNULL(t_mblk)) { - ret = OB_ERR_NULL_VALUE; - STORAGE_LOG(WARN, "block is null", K(ret)); - } else if (t_mblk->get_max_cont_page_nums() < page_nums - || t_mblk->get_tenant_id() != tenant_id - || !t_mblk->is_memory()) { - ret = OB_STATE_NOT_MATCH; - STORAGE_LOG(DEBUG, "the block is not suitable", K(ret), K(block_id), - "block_page_nums", t_mblk->get_max_cont_page_nums(), K(page_nums), - "block_tenant_id", t_mblk->get_tenant_id(), K(tenant_id), - "block_status", t_mblk->get_block_status()); - } - return ret; -} - -int ObTmpTenantMemBlockManager::GetAvailableBlockMapOp::operator () (oceanbase::common::hash::HashMapPair &entry) -{ - int ret = OB_SUCCESS; - ObTmpMacroBlock *blk = entry.second; - if (!is_found_) { - if (OB_ISNULL(blk)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "unexpected error, blk is nullptr", K(ret), KP(blk)); - } else if (tenant_id_ != blk->get_tenant_id() || dir_id_ != blk->get_dir_id()) { - // do nothing - } else { - if (blk->get_max_cont_page_nums() < page_nums_ - || blk->get_block_status() != ObTmpMacroBlock::BlockStatus::MEMORY) { - // do nothing - } else { - block_ = blk; - is_found_ = true; - } - } - } - return ret; -} - -int ObTmpTenantMemBlockManager::get_available_macro_block(const int64_t dir_id, const uint64_t tenant_id, - const int64_t page_nums, ObTmpMacroBlock *&t_mblk, - bool &is_found) -{ - int ret = OB_SUCCESS; - is_found = false; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpBlockCache has not been inited", K(ret)); - } else { - GetAvailableBlockMapOp op(dir_id, tenant_id, page_nums, t_mblk, is_found); - if (OB_FAIL(t_mblk_map_.foreach_refactored(op))) { - STORAGE_LOG(WARN, "get available macro block failed", K(ret)); - } - } - return ret; -} - -int ObTmpTenantMemBlockManager::check_memory_limit() -{ - int ret = OB_SUCCESS; - const int64_t timeout_ts = THIS_WORKER.get_timeout_ts(); - while (OB_SUCC(ret) && get_tenant_mem_block_num() < t_mblk_map_.size() && !wait_info_queue_.is_empty()) { - ObThreadCondGuard guard(cond_); - if (OB_FAIL(guard.get_ret())) { - STORAGE_LOG(ERROR, "fail to guard request condition", K(ret)); - } else { - int64_t wait_ms = (timeout_ts - ObTimeUtility::current_time()) / 1000; - while (OB_SUCC(ret) - && get_tenant_mem_block_num() < t_mblk_map_.size() - && !wait_info_queue_.is_empty() - && wait_ms > 0) { - if (OB_FAIL(cond_.wait(wait_ms))) { - STORAGE_LOG(WARN, "fail to wait block write condition", K(ret), K(wait_ms)); - } else if (get_tenant_mem_block_num() < t_mblk_map_.size()) { - wait_ms = (timeout_ts - ObTimeUtility::current_time()) / 1000; - } - } - - if (OB_SUCC(ret) && OB_UNLIKELY(wait_ms <= 0)) { - ret = OB_TIMEOUT; - STORAGE_LOG(WARN, "fail to wait block io condition due to spurious wakeup", - K(ret), K(wait_ms), K(timeout_ts), K(ObTimeUtility::current_time())); - } - } - } - return ret; -} -bool ObTmpTenantMemBlockManager::check_block_full() -{ - return get_tenant_mem_block_num() < t_mblk_map_.size(); -} - -ObTmpTenantMemBlockManager::BlockWashScoreCompare::BlockWashScoreCompare() -{ -} - -bool ObTmpTenantMemBlockManager::BlockWashScoreCompare::operator() ( - const ObTmpTenantMemBlockManager::BlockInfo &left, - const ObTmpTenantMemBlockManager::BlockInfo &right) -{ - return left.wash_score_ < right.wash_score_; -} - -int ObTmpTenantMemBlockManager::cleanup() -{ - int ret = OB_SUCCESS; - ObArenaAllocator allocator(common::ObMemAttr(MTL_ID(), "TmpFileRank")); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpBlockCache has not been inited", K(ret)); - } else if (OB_FAIL(check_memory_limit())) { - STORAGE_LOG(WARN, "fail to check memory limit", K(ret), K(t_mblk_map_.size()), K(get_tenant_mem_block_num())); - } else { - const int64_t wash_threshold = get_tenant_mem_block_num() * 0.8; - Heap heap(compare_, &allocator); - ChooseBlocksMapOp op(heap, ObTimeUtility::fast_current_time()); - const int64_t clean_nums = t_mblk_map_.size() - wash_threshold - ATOMIC_LOAD(&washing_count_); - if (clean_nums <= 0) { - STORAGE_LOG(DEBUG, "there is no need to wash blocks", K(ret), K(clean_nums)); - } else if (OB_FAIL(t_mblk_map_.foreach_refactored(op))) { - STORAGE_LOG(WARN, "choose blks failed", K(ret)); - } else { - const int64_t candidate_cnt = heap.count(); - bool wash_success = false; - while (OB_SUCC(ret) && heap.count() > 0 && !wash_success) { - const BlockInfo info = heap.top(); - ObIOWaitInfoHandle handle; - if (OB_FAIL(wash_block(info.block_id_, handle))) { - STORAGE_LOG(WARN, "fail to wash", K(ret), K_(tenant_id), K(info.block_id_)); - } else { - wash_success = handle.is_valid(); - STORAGE_LOG(DEBUG, "succeed to wash block for cleanup", K(info)); - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(heap.pop())) { - STORAGE_LOG(WARN, "pop info from heap failed", K(ret), K_(tenant_id)); - } - } - } - if (OB_SUCC(ret) && !wash_success) { - ret = OB_STATE_NOT_MATCH; - STORAGE_LOG(WARN, "fail to cleanup", K(ret), K(t_mblk_map_.size()), K(candidate_cnt)); - } - } - } - return ret; -} - -int ObTmpTenantMemBlockManager::free_empty_blocks(common::ObIArray &free_blocks) -{ - int ret = OB_SUCCESS; - if (free_blocks.count() > 0) { - for (int64_t i = 0; OB_SUCC(ret) && i < free_blocks.count(); ++i) { - ObTmpMacroBlock* blk = free_blocks.at(i); - if (blk->is_empty()) { - if (OB_FAIL(free_macro_block(blk->get_block_id()))) { - STORAGE_LOG(WARN, "fail to free tmp macro block", K(ret)); - } - } - free_blocks.at(i) = NULL; - } - free_blocks.reset(); - } - return ret; -} - -int ObTmpTenantMemBlockManager::check_and_free_mem_block(ObTmpMacroBlock *&t_mblk) -{ - int ret = OB_SUCCESS; - uint64_t hash_val = 0; - int64_t block_id = t_mblk->get_block_id(); - hash_val = murmurhash(&block_id, sizeof(block_id), hash_val); - ObBucketHashWLockGuard lock_guard(map_lock_, hash_val); - if (OB_FAIL(t_mblk->check_and_set_status( - ObTmpMacroBlock::BlockStatus::MEMORY, ObTmpMacroBlock::BlockStatus::DISKED))) { - if (OB_STATE_NOT_MATCH == ret) { - STORAGE_LOG(DEBUG, "this block is washing", K(ret), K(*t_mblk)); - ret = OB_SUCCESS; - } else { - STORAGE_LOG(WARN, "check and set status failed", K(ret), K(*t_mblk)); - } - } else if (OB_FAIL(t_mblk->give_back_buf_into_cache())) { - STORAGE_LOG(WARN, "fail to put tmp block cache", K(ret), K(t_mblk)); - } else if (OB_FAIL(free_macro_block(t_mblk->get_block_id()))) { - STORAGE_LOG(WARN, "fail to free tmp macro block for block cache", K(ret)); - } else { - tenant_store_.dec_block_cache_num(1); - } - return ret; -} - -int ObTmpTenantMemBlockManager::ChooseBlocksMapOp::operator () (oceanbase::common::hash::HashMapPair &entry) -{ - int ret = OB_SUCCESS; - ObTmpMacroBlock *blk = entry.second; - if (OB_LIKELY(NULL != blk) && OB_LIKELY(blk->is_inited()) && OB_LIKELY(blk->is_memory()) - && OB_LIKELY(0 != blk->get_used_page_nums())) { - BlockInfo info; - info.block_id_ = blk->get_block_id(); - info.wash_score_ = blk->get_wash_score(cur_time_); - if(OB_FAIL(heap_.push(info))) { - STORAGE_LOG(WARN, "insert block to array failed", K(ret)); - } - } - STORAGE_LOG(DEBUG, "choose one block", K(ret), KPC(blk)); - return ret; -} - -int ObTmpTenantMemBlockManager::add_macro_block(ObTmpMacroBlock *&t_mblk) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpBlockCache has not been inited", K(ret)); - } else if (OB_FAIL(t_mblk_map_.set_refactored(t_mblk->get_block_id(), t_mblk))) { - STORAGE_LOG(WARN, "fail to set tmp macro block map", K(ret), K(t_mblk)); - } - return ret; -} - -int ObTmpTenantMemBlockManager::refresh_dir_to_blk_map(const int64_t dir_id, - const ObTmpMacroBlock *t_mblk) -{ - int ret = OB_SUCCESS; - int64_t block_id = 0; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpBlockCache has not been inited", K(ret)); - } else if (OB_FAIL(dir_to_blk_map_.get_refactored(dir_id, block_id))) { - if (OB_HASH_NOT_EXIST == ret) { - if (OB_FAIL(dir_to_blk_map_.set_refactored(dir_id, t_mblk->get_block_id(), 1))) { - STORAGE_LOG(WARN, "fail to set dir_to_blk_map_", K(ret), K(dir_id), K(t_mblk->get_block_id())); - } - } - } else { - ObTmpMacroBlock *dir_mblk = NULL; - if (OB_FAIL(t_mblk_map_.get_refactored(block_id, dir_mblk))) { - if (OB_HASH_NOT_EXIST == ret) { - STORAGE_LOG(DEBUG, "the tmp macro block has been removed or washed", K(ret), K(block_id)); - if (OB_FAIL(dir_to_blk_map_.set_refactored(dir_id, t_mblk->get_block_id(), 1))) { - STORAGE_LOG(WARN, "fail to set dir_to_blk_map_", K(ret), K(dir_id), K(t_mblk->get_block_id())); - } - } else { - STORAGE_LOG(WARN, "fail to get block", K(ret), K(block_id)); - } - } else if (dir_mblk->get_max_cont_page_nums() < t_mblk->get_max_cont_page_nums()) { - if (OB_FAIL(dir_to_blk_map_.set_refactored(dir_id, t_mblk->get_block_id(), 1))) { - STORAGE_LOG(WARN, "fail to set dir_to_blk_map_", K(ret), K(dir_id), K(t_mblk->get_block_id())); - } - } - } - - return ret; -} - -int ObTmpTenantMemBlockManager::get_block_and_set_washing(int64_t block_id, ObTmpMacroBlock *&m_blk) -{ - int ret = OB_SUCCESS; - bool is_sealed = false; - uint64_t hash_val = 0; - hash_val = murmurhash(&block_id, sizeof(block_id), hash_val); - ObBucketHashRLockGuard lock_guard(map_lock_, hash_val); - if (OB_FAIL(t_mblk_map_.get_refactored(block_id, m_blk))) { - STORAGE_LOG(DEBUG, "tenant mem block manager get block failed", K(ret), K(block_id)); - } else if (OB_ISNULL(m_blk)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "the block is null", K(ret), K(*m_blk)); - } else if (OB_UNLIKELY(!m_blk->is_inited())) { - ret = OB_STATE_NOT_MATCH; - STORAGE_LOG(WARN, "the block has not been inited", K(ret), K(*m_blk)); - } else if (OB_UNLIKELY(!m_blk->is_memory())) { - ret = OB_STATE_NOT_MATCH; - STORAGE_LOG(DEBUG, "the block has been disked or washing", K(ret), K(*m_blk)); - } else if (m_blk->is_empty()) { - if (OB_FAIL(refresh_dir_to_blk_map(m_blk->get_dir_id(), m_blk))) { - STORAGE_LOG(WARN, "fail to refresh dir_to_blk_map", K(ret), K(*m_blk)); - } - // refresh ret can be ignored. overwrite the ret. - ret = OB_STATE_NOT_MATCH; - } else if (OB_FAIL(m_blk->seal(is_sealed))) { - STORAGE_LOG(WARN, "fail to seal block", K(ret), K(*m_blk)); - } else if (OB_UNLIKELY(!is_sealed)) { - ret = OB_STATE_NOT_MATCH; - STORAGE_LOG(WARN, "the block has some unclosed extents", K(ret), K(is_sealed), K(*m_blk)); - } else if (OB_UNLIKELY(0 == m_blk->get_used_page_nums())) { - ret = OB_STATE_NOT_MATCH; - STORAGE_LOG(WARN, "the block write has not been finished", K(ret), K(*m_blk)); - } else if (OB_FAIL(m_blk->check_and_set_status( - ObTmpMacroBlock::BlockStatus::MEMORY, ObTmpMacroBlock::BlockStatus::WASHING))) { - if (OB_STATE_NOT_MATCH != ret) { - STORAGE_LOG(WARN, "check and set status failed", K(ret), K(*m_blk)); - } - } - - return ret; -} - -int ObTmpTenantMemBlockManager::wash_block(const int64_t block_id, ObIOWaitInfoHandle &handle) -{ - int ret = OB_SUCCESS; - handle.reset(); - IOWaitInfo *wait_info = NULL; - ObTmpMacroBlock *m_blk = NULL; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "the tenant mem block manager not been inited", K(ret)); - } else if (OB_FAIL(get_block_and_set_washing(block_id, m_blk))) { - if (OB_HASH_NOT_EXIST == ret || OB_STATE_NOT_MATCH == ret) { - ret = OB_SUCCESS; - STORAGE_LOG(DEBUG, "this block may be removed, washed or disked", K(ret), K(block_id)); - } else { - STORAGE_LOG(WARN, "check and set washing failed", K(ret), K(block_id)); - } - } else { - ObTmpBlockIOInfo info; - char *buf = NULL; - SpinWLockGuard io_guard(io_lock_); - ObMacroBlockHandle &mb_handle = m_blk->get_macro_block_handle(); - if (OB_FAIL(m_blk->get_wash_io_info(info))) { - STORAGE_LOG(WARN, "fail to get wash io info", K(ret), K_(tenant_id), K(m_blk)); - } else if (OB_FAIL(write_io(info, mb_handle))) { - STORAGE_LOG(WARN, "fail to write tmp block", K(ret), K_(tenant_id), K(info), K(*m_blk)); - } else if(OB_ISNULL(buf = static_cast(allocator_->alloc(sizeof(IOWaitInfo))))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - STORAGE_LOG(WARN, "fail to alloc io wait info memory", K(ret), K_(tenant_id)); - } else if (FALSE_IT(wait_info = new (buf) IOWaitInfo(mb_handle, *m_blk, *allocator_))) { - } else if (OB_FAIL(wait_info->cond_.init(ObWaitEventIds::IO_CONTROLLER_COND_WAIT))) { - STORAGE_LOG(WARN, "fail to init condition", K(ret), K_(tenant_id)); - } else if (FALSE_IT(handle.set_wait_info(wait_info))) { - } else if (OB_FAIL(wait_handles_map_.set_refactored(m_blk->get_block_id(), handle))) { - STORAGE_LOG(WARN, "fail to set block into write_handles_map", K(ret), "block_id", m_blk->get_block_id()); - } else if (OB_FAIL(wait_info_queue_.push(wait_info))) { - STORAGE_LOG(WARN, "fail to push back into write_handles", K(ret), K(wait_info)); - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(wait_handles_map_.erase_refactored(m_blk->get_block_id()))) { - STORAGE_LOG(WARN, "fail to erase block from wait handles map", K(tmp_ret), K(m_blk->get_block_id())); - } - } else { - ATOMIC_INC(&washing_count_); - } - if (OB_FAIL(ret) && OB_NOT_NULL(m_blk)) { - mb_handle.reset(); - // don't release wait info unless ObIOWaitInfoHandle doesn't hold its ref - if (OB_NOT_NULL(wait_info) && OB_ISNULL(handle.get_wait_info())) { - wait_info->~IOWaitInfo(); - allocator_->free(wait_info); - wait_info = nullptr; - } - handle.reset(); - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(m_blk->check_and_set_status(ObTmpMacroBlock::BlockStatus::WASHING, - ObTmpMacroBlock::BlockStatus::MEMORY))) { - STORAGE_LOG(ERROR, "fail to rollback block status", K(ret), K(tmp_ret)); - } - } - } - return ret; -} - -int ObTmpTenantMemBlockManager::write_io( - const ObTmpBlockIOInfo &io_info, - ObMacroBlockHandle &handle) -{ - int ret = OB_SUCCESS; - const int64_t buf_size = OB_SERVER_BLOCK_MGR.get_macro_block_size(); - const int64_t page_size = ObTmpMacroBlock::get_default_page_size(); - int64_t pos = 0; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFileStore has not been inited", K(ret)); - } else if (OB_FAIL(THE_IO_DEVICE->check_space_full(OB_SERVER_BLOCK_MGR.get_macro_block_size()))) { - STORAGE_LOG(WARN, "fail to check space full", K(ret)); - } else { - ObMacroBlockWriteInfo write_info; - write_info.io_desc_ = io_info.io_desc_; - write_info.buffer_ = io_info.buf_; - write_info.offset_ = ObTmpMacroBlock::get_header_padding(); - write_info.size_ = io_info.size_; - write_info.io_timeout_ms_ = io_info.io_timeout_ms_; - write_info.io_desc_.set_resource_group_id(THIS_WORKER.get_group_id()); - write_info.io_desc_.set_sys_module_id(ObIOModule::TMP_TENANT_MEM_BLOCK_IO); - if (OB_FAIL(ObBlockManager::async_write_block(write_info, handle))) { - STORAGE_LOG(WARN, "Fail to async write block", K(ret), K(write_info), K(handle)); - } else if (OB_FAIL(OB_SERVER_BLOCK_MGR.update_write_time(handle.get_macro_id(), - true/*update_to_max_time*/))) { //just to skip bad block inspect - STORAGE_LOG(WARN, "fail to update macro id write time", K(ret), "macro id", handle.get_macro_id()); - } - } - return ret; -} - -int64_t ObTmpTenantMemBlockManager::get_tenant_mem_block_num() -{ - int64_t tenant_mem_block_num = TENANT_MEM_BLOCK_NUM; - int64_t last_access_ts = ATOMIC_LOAD(&last_access_tenant_config_ts_); - if (last_access_ts > 0 - && common::ObClockGenerator::getClock() - last_access_ts < 10000000) { - tenant_mem_block_num = ATOMIC_LOAD(&last_tenant_mem_block_num_); - } else { - omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id_)); - if (!tenant_config.is_valid()) { - COMMON_LOG(INFO, "failed to get tenant config", K_(tenant_id)); - } else if (0 == tenant_config->_temporary_file_io_area_size) { - tenant_mem_block_num = 1L; - } else { - const int64_t bytes = common::upper_align( - lib::get_tenant_memory_limit(tenant_id_) * tenant_config->_temporary_file_io_area_size / 100, - ObTmpFileStore::get_block_size()); - tenant_mem_block_num = bytes / ObTmpFileStore::get_block_size(); - } - ATOMIC_STORE(&last_tenant_mem_block_num_, tenant_mem_block_num); - ATOMIC_STORE(&last_access_tenant_config_ts_, common::ObClockGenerator::getClock()); - } - return tenant_mem_block_num; -} - -int ObTmpTenantMemBlockManager::exec_wait() -{ - int ret = OB_SUCCESS; - int64_t wait_io_cnt = 0; - int64_t loop_nums = 0; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFileStore has not been inited", K(ret)); - } else if (!stopped_) { - common::ObSpLinkQueue::Link *node = NULL; - SpinWLockGuard io_guard(io_lock_); - const int64_t begin_us = ObTimeUtility::fast_current_time(); - while (OB_SUCC(ret) && (ObTimeUtility::fast_current_time() - begin_us)/1000 < TASK_INTERVAL) { - IOWaitInfo *wait_info = NULL; - if (OB_FAIL(wait_info_queue_.pop(node))) { - if (OB_EAGAIN != ret) { - STORAGE_LOG(WARN, "fail to pop wait info from queue", K(ret)); - } - } else if (FALSE_IT(++loop_nums)) { - } else if (OB_ISNULL(wait_info = static_cast(node))) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(ERROR, "unexpected error, wait info is nullptr", K(ret), KP(node)); - } else if (OB_ISNULL(wait_info->block_handle_)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(ERROR, "unexpected error, macro handle in wait info is nullptr", K(ret), KPC(wait_info)); - } else { - ObTmpMacroBlock &blk = wait_info->get_block(); - const MacroBlockId macro_id = wait_info->block_handle_->get_macro_id(); - const int64_t block_id = blk.get_block_id(); - const int64_t free_page_nums = blk.get_free_page_nums(); - if (OB_FAIL(wait_info->exec_wait())) { - STORAGE_LOG(WARN, "fail to exec io handle wait", K(ret), K_(tenant_id), KPC(wait_info)); - ATOMIC_DEC(&washing_count_); - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(blk.check_and_set_status(ObTmpMacroBlock::WASHING, ObTmpMacroBlock::MEMORY))) { - STORAGE_LOG(ERROR, "fail to rollback block status", K(ret), K(tmp_ret), K(block_id), K(blk)); - } - } else { - STORAGE_LOG(INFO, "start to wash a block", K(block_id), KPC(&blk)); - ObThreadCondGuard cond_guard(cond_); - if (OB_FAIL(cond_guard.get_ret())) { - STORAGE_LOG(WARN, "fail to guard request condition", K(ret)); - } else { - ATOMIC_DEC(&washing_count_); - if (OB_FAIL(blk.give_back_buf_into_cache(true/*set block disked for washed block*/))) { - STORAGE_LOG(WARN, "fail to give back buf into cache", K(ret), K(block_id)); - } else if (OB_FAIL(t_mblk_map_.erase_refactored(block_id))) { - if (OB_HASH_NOT_EXIST != ret) { - STORAGE_LOG(WARN, "fail to erase t_mblk_map", K(ret), K(block_id)); - } else { - ret = OB_SUCCESS; - } - } else { - ++wait_io_cnt; - tenant_store_.dec_block_cache_num(1); - ObTaskController::get().allow_next_syslog(); - STORAGE_LOG(INFO, "succeed to wash a block", K(block_id), K(macro_id), - K(free_page_nums), K(t_mblk_map_.size())); - } - } - } - wait_info->ret_code_ = ret; - int64_t tmp_ret = OB_SUCCESS; - // The broadcast() is executed regardless of success or failure, and the error code is ignored - // so that the next request can be executed. - if (OB_TMP_FAIL(wait_info->broadcast())) { - STORAGE_LOG(ERROR, "signal io request condition failed", K(ret), K(tmp_ret), K(block_id)); - } - // Regardless of success or failure, need to erase wait info handle from map. - if (OB_TMP_FAIL(wait_handles_map_.erase_refactored(block_id))) { - if (OB_HASH_NOT_EXIST != tmp_ret) { - STORAGE_LOG(ERROR, "fail to erase wait handles map", K(ret), K(tmp_ret), K(block_id)); - } - } - } - } - if (OB_EAGAIN == ret) { - ret = OB_SUCCESS; - } - } - if (OB_SUCC(ret)) { - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(cond_.broadcast())) { - STORAGE_LOG(ERROR, "signal wash condition failed", K(ret), K(tmp_ret)); - } - if (loop_nums > 0 || REACH_TIME_INTERVAL(1000 * 1000L)/*1s*/) { - const int64_t washing_count = ATOMIC_LOAD(&washing_count_); - int64_t block_cache_num = -1; - int64_t page_cache_num = -1; - block_cache_num = tenant_store_.get_block_cache_num(); - page_cache_num = tenant_store_.get_page_cache_num(); - ObTaskController::get().allow_next_syslog(); - STORAGE_LOG(INFO, "succeed to do one round of tmp block io", K(ret), K(loop_nums), - K(wait_io_cnt), K(washing_count), K(block_cache_num), K(page_cache_num)); - } - } - return ret; -} - -int ObTmpTenantMemBlockManager::change_mem() -{ - int ret = OB_SUCCESS; - // Here, this memory is used to store temporary file block metadata, which is related to the - // datafile size. So, we set the upper limit of memory to be percentage (default, 70%) of tenant memory to - // avoid excessive tenant memory, and affecting system stability. In theory, the limit - // will be reached only when the tenant's memory is extremely small and the disk is extremely - // large. - tenant_store_.refresh_memory_limit(tenant_id_); - return ret; -} - -int ObTmpTenantMemBlockManager::wait_write_finish(const int64_t block_id, const int64_t timeout_ms) -{ - int ret = OB_SUCCESS; - const int64_t wash_to_flush_wait_interval = 1000; // 1ms - ObIOWaitInfoHandle handle; - ObTmpMacroBlock *blk = nullptr; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpMacroBlock has not been inited", K(ret)); - } else if (timeout_ms < 0) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "Invalid argument, ", K(timeout_ms), K(ret)); - } else if (OB_FAIL(t_mblk_map_.get_refactored(block_id, blk))) { - if (OB_HASH_NOT_EXIST == ret) { - ret = OB_SUCCESS; - } else { - STORAGE_LOG(WARN, "cannot find tmp block", K(ret), K(block_id)); - } - } else { - bool is_found = false; - while (OB_SUCC(ret) && blk->is_washing() && !is_found) { - if (OB_FAIL(wait_handles_map_.get_refactored(block_id, handle))) { - if (OB_HASH_NOT_EXIST == ret) { - ret = OB_SUCCESS; - } else { - STORAGE_LOG(WARN, "cannot find wait hanlde", K(ret), K(block_id)); - } - } else { - is_found = true; - } - } - - if (OB_SUCC(ret) && is_found) { - if (OB_FAIL(handle.get_wait_info()->wait(timeout_ms))) { - STORAGE_LOG(WARN, "wait write io finish failed", K(ret), K(block_id)); - } - } - } - - return ret; -} - - -} // end namespace blocksstable -} // end namespace oceanbase diff --git a/src/storage/blocksstable/ob_tmp_file_cache.h b/src/storage/blocksstable/ob_tmp_file_cache.h deleted file mode 100644 index c66878e4f..000000000 --- a/src/storage/blocksstable/ob_tmp_file_cache.h +++ /dev/null @@ -1,474 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#ifndef OCEANBASE_STORAGE_BLOCKSSTABLE_OB_TMP_FILE_CACHE_H_ -#define OCEANBASE_STORAGE_BLOCKSSTABLE_OB_TMP_FILE_CACHE_H_ - -#include "lib/hash/ob_hashmap.h" -#include "lib/queue/ob_link_queue.h" -#include "share/io/ob_io_manager.h" -#include "share/cache/ob_kv_storecache.h" -#include "storage/ob_i_store.h" - -namespace oceanbase -{ -namespace blocksstable -{ - -struct ObTmpBlockIOInfo; -struct ObTmpFileMacroBlockHeader; -class ObTmpFileIOHandle; -class ObTmpMacroBlock; -class ObTmpFileExtent; -class ObMacroBlockHandle; -class ObTmpTenantFileStore; - -class ObTmpPageCacheKey final : public common::ObIKVCacheKey -{ -public: - ObTmpPageCacheKey(); - ObTmpPageCacheKey(const int64_t block_id, const int64_t page_id, const uint64_t tenant_id); - ~ObTmpPageCacheKey(); - bool operator ==(const ObIKVCacheKey &other) const override; - uint64_t get_tenant_id() const override; - uint64_t hash() const override; - int64_t size() const override; - int deep_copy(char *buf, const int64_t buf_len, ObIKVCacheKey *&key) const override; - bool is_valid() const; - int64_t get_page_id() const { return page_id_; } - TO_STRING_KV(K_(block_id), K_(page_id), K_(tenant_id)); - -private: - int64_t block_id_; - int64_t page_id_; - uint64_t tenant_id_; -}; - -class ObTmpPageCacheValue final : public common::ObIKVCacheValue -{ -public: - explicit ObTmpPageCacheValue(char *buf); - ~ObTmpPageCacheValue(); - int64_t size() const override; - int deep_copy(char *buf, const int64_t buf_len, ObIKVCacheValue *&value) const override; - bool is_valid() const { return NULL != buf_ && size() > 0; } - char *get_buffer() { return buf_; } - void set_buffer(char *buf) { buf_ = buf;} - TO_STRING_KV(KP_(buf), K_(size)); - -private: - char *buf_; - int64_t size_; - DISALLOW_COPY_AND_ASSIGN(ObTmpPageCacheValue); -}; - -struct ObTmpPageValueHandle final -{ -public: - ObTmpPageValueHandle() : value_(NULL), handle_() {} - ~ObTmpPageValueHandle() = default; - void reset() - { - handle_.reset(); - value_ = NULL; - } - TO_STRING_KV(KP_(value), K_(handle)); - ObTmpPageCacheValue *value_; - common::ObKVCacheHandle handle_; -}; - -struct ObTmpPageIOInfo final -{ -public: - ObTmpPageIOInfo() : offset_(0), size_(0), key_() {} - ~ObTmpPageIOInfo() {} - TO_STRING_KV(K_(key), K_(offset), K_(size)); - - int32_t offset_; - int32_t size_; - ObTmpPageCacheKey key_; -}; - -class ObTmpPageCache final : public common::ObKVCache -{ -public: - typedef common::ObKVCache BasePageCache; - static ObTmpPageCache &get_instance(); - int init(const char *cache_name, const int64_t priority); - int direct_read(const ObTmpBlockIOInfo &info, ObMacroBlockHandle &mb_handle, common::ObIAllocator &allocator); - int prefetch( - const ObTmpPageCacheKey &key, - const ObTmpBlockIOInfo &info, - ObMacroBlockHandle &mb_handle, - common::ObIAllocator &allocator); - // multi page prefetch - int prefetch( - const ObTmpBlockIOInfo &info, - const common::ObIArray &page_io_infos, - ObMacroBlockHandle &mb_handle, - common::ObIAllocator &allocator); - int get_cache_page(const ObTmpPageCacheKey &key, ObTmpPageValueHandle &handle); - int get_page(const ObTmpPageCacheKey &key, ObTmpPageValueHandle &handle); - int put_page(const ObTmpPageCacheKey &key, const ObTmpPageCacheValue &value); - void destroy(); -public: - class ObITmpPageIOCallback : public common::ObIOCallback - { - public: - ObITmpPageIOCallback(); - virtual ~ObITmpPageIOCallback(); - virtual int alloc_data_buf(const char *io_data_buffer, const int64_t data_size) override; - protected: - friend class ObTmpPageCache; - virtual int process_page(const ObTmpPageCacheKey &key, const ObTmpPageCacheValue &value); - virtual ObIAllocator *get_allocator() { return allocator_; } - - protected: - BasePageCache *cache_; - common::ObIAllocator *allocator_; - int64_t offset_; // offset in block - char *data_buf_; // actual data buffer - }; - - class ObTmpPageIOCallback final : public ObITmpPageIOCallback - { - public: - ObTmpPageIOCallback(); - ~ObTmpPageIOCallback() override; - int64_t size() const override; - int inner_process(const char *data_buffer, const int64_t size) override; - const char *get_data() override; - TO_STRING_KV("callback_type:", "ObTmpPageIOCallback", KP_(data_buf)); - DISALLOW_COPY_AND_ASSIGN(ObTmpPageIOCallback); - private: - friend class ObTmpPageCache; - ObTmpPageCacheKey key_; - }; - class ObTmpMultiPageIOCallback final : public ObITmpPageIOCallback - { - public: - ObTmpMultiPageIOCallback(); - ~ObTmpMultiPageIOCallback() override; - int64_t size() const override; - int inner_process(const char *data_buffer, const int64_t size) override; - const char *get_data() override; - TO_STRING_KV("callback_type:", "ObTmpMultiPageIOCallback", KP_(data_buf)); - DISALLOW_COPY_AND_ASSIGN(ObTmpMultiPageIOCallback); - private: - friend class ObTmpPageCache; - common::ObArray page_io_infos_; - }; - class ObTmpDirectReadPageIOCallback final : public ObITmpPageIOCallback - { - public: - ObTmpDirectReadPageIOCallback() {} - ~ObTmpDirectReadPageIOCallback() override {} - int64_t size() const override; - int inner_process(const char *data_buffer, const int64_t size) override; - const char *get_data() override; - TO_STRING_KV("callback_type:", "ObTmpDirectReadPageIOCallback", KP_(data_buf)); - DISALLOW_COPY_AND_ASSIGN(ObTmpDirectReadPageIOCallback); - }; -private: - ObTmpPageCache(); - ~ObTmpPageCache(); - int inner_read_io(const ObTmpBlockIOInfo &io_info, - ObITmpPageIOCallback *callback, - ObMacroBlockHandle &handle); - int read_io(const ObTmpBlockIOInfo &io_info, - ObITmpPageIOCallback *callback, - ObMacroBlockHandle &handle); - -private: - DISALLOW_COPY_AND_ASSIGN(ObTmpPageCache); -}; - -class ObTmpBlockCacheKey final : public common::ObIKVCacheKey -{ -public: - ObTmpBlockCacheKey(const int64_t block_id, const uint64_t tenant_id); - ~ObTmpBlockCacheKey() {} - bool operator ==(const ObIKVCacheKey &other) const override; - uint64_t get_tenant_id() const override; - uint64_t hash() const override; - int64_t size() const override; - int deep_copy(char *buf, const int64_t buf_len, ObIKVCacheKey *&key) const override; - int64_t get_block_id() const { return block_id_; } - bool is_valid() const { return block_id_ > 0 && tenant_id_ > 0 && size() > 0; } - TO_STRING_KV(K_(block_id), K_(tenant_id)); - -private: - int64_t block_id_; - uint64_t tenant_id_; - friend class ObTmpBlockCache; - DISALLOW_COPY_AND_ASSIGN(ObTmpBlockCacheKey); -}; - -class ObTmpBlockCacheValue final : public common::ObIKVCacheValue -{ -public: - explicit ObTmpBlockCacheValue(char *buf); - ~ObTmpBlockCacheValue() {} - int64_t size() const override; - int deep_copy(char *buf, const int64_t buf_len, ObIKVCacheValue *&value) const override; - bool is_valid() const { return NULL != buf_ && size() > 0; } - char *get_buffer() { return buf_; } - TO_STRING_KV(K_(size)); - -private: - char *buf_; - int64_t size_; - DISALLOW_COPY_AND_ASSIGN(ObTmpBlockCacheValue); -}; - -struct ObTmpBlockValueHandle final -{ -public: - ObTmpBlockValueHandle() - : value_(NULL), inst_handle_(), kvpair_(NULL), handle_(){} - ~ObTmpBlockValueHandle() = default; - void reset() - { - handle_.reset(); - inst_handle_.reset(); - value_ = NULL; - kvpair_ = NULL; - } - TO_STRING_KV(KP_(value), K_(inst_handle), KP_(kvpair), K_(handle)); - ObTmpBlockCacheValue *value_; - ObKVCacheInstHandle inst_handle_; - ObKVCachePair *kvpair_; - common::ObKVCacheHandle handle_; -}; - -class ObTmpBlockCache final: public common::ObKVCache -{ -public: - static ObTmpBlockCache &get_instance(); - int init(const char *cache_name, const int64_t priority); - int get_block(const ObTmpBlockCacheKey &key, ObTmpBlockValueHandle &handle); - int alloc_buf(const ObTmpBlockCacheKey &key, ObTmpBlockValueHandle &handle); - int put_block(const ObTmpBlockCacheKey &key, ObTmpBlockValueHandle &handle); - void destroy(); - -private: - ObTmpBlockCache() {} - ~ObTmpBlockCache() {} - - DISALLOW_COPY_AND_ASSIGN(ObTmpBlockCache); -}; - -class ObTmpTenantMemBlockManager; - -class ObTmpFileWaitTask : public common::ObTimerTask -{ -public: - explicit ObTmpFileWaitTask(ObTmpTenantMemBlockManager &mgr); - virtual ~ObTmpFileWaitTask() {} - virtual void runTimerTask() override; -private: - ObTmpTenantMemBlockManager &mgr_; -}; - -class ObTmpFileMemTask : public common::ObTimerTask -{ -public: - explicit ObTmpFileMemTask(ObTmpTenantMemBlockManager &mgr); - virtual ~ObTmpFileMemTask() {} - virtual void runTimerTask() override; -private: - ObTmpTenantMemBlockManager &mgr_; -}; - -class ObTmpTenantMemBlockManager final -{ -public: - struct IOWaitInfo :public common::ObSpLinkQueue::Link - { - public: - IOWaitInfo(ObMacroBlockHandle &block_handle, ObTmpMacroBlock &block, ObIAllocator &allocator); - virtual ~IOWaitInfo(); - void inc_ref(); - void dec_ref(); - int wait(const int64_t timout_ms); - int exec_wait(); - void reset_io(); - int broadcast(); - OB_INLINE ObTmpMacroBlock& get_block() { return block_; }; - TO_STRING_KV(K_(block), KPC_(block_handle), K_(ref_cnt), K_(ret_code)); - - private: - void destroy(); - - public: - ObMacroBlockHandle *block_handle_; - ObTmpMacroBlock &block_; - ObThreadCond cond_; - ObIAllocator &allocator_; - volatile int64_t ref_cnt_; - int64_t ret_code_; - private: - DISALLOW_COPY_AND_ASSIGN(IOWaitInfo); - }; - - struct ObIOWaitInfoHandle final - { - public: - ObIOWaitInfoHandle(); - ~ObIOWaitInfoHandle(); - ObIOWaitInfoHandle(const ObIOWaitInfoHandle &other); - ObIOWaitInfoHandle &operator=(const ObIOWaitInfoHandle &other); - void set_wait_info(IOWaitInfo *wait_info); - bool is_empty() const; - bool is_valid() const; - void reset(); - OB_INLINE IOWaitInfo* get_wait_info() const { return wait_info_; }; - TO_STRING_KV(KPC_(wait_info)); - - private: - IOWaitInfo *wait_info_; - }; - - explicit ObTmpTenantMemBlockManager(ObTmpTenantFileStore &tenant_store); - ~ObTmpTenantMemBlockManager(); - int init(const uint64_t tenant_id, - common::ObConcurrentFIFOAllocator &allocator, - double blk_nums_threshold = DEFAULT_MIN_FREE_BLOCK_RATIO); - void destroy(); - int get_block(const ObTmpBlockCacheKey &key, ObTmpBlockValueHandle &handle); - int alloc_buf(const ObTmpBlockCacheKey &key, ObTmpBlockValueHandle &handle); - int alloc_extent(const int64_t dir_id, const uint64_t tenant_id, const int64_t size, ObTmpFileExtent &extent); - int alloc_block_all_pages(ObTmpMacroBlock *t_mblk, ObTmpFileExtent &extent); - int free_macro_block(const int64_t block_id); - int wash_block(const int64_t block_id, ObIOWaitInfoHandle &handle); - int cleanup(); - int add_macro_block(ObTmpMacroBlock *&t_mblk); - int wait_write_finish(const int64_t block_id, const int64_t timeout_ms); - static int64_t get_default_timeout_ms() {return GCONF._data_storage_io_timeout / 1000L;} - int free_empty_blocks(common::ObIArray &free_blocks); - int refresh_dir_to_blk_map(const int64_t dir_id, const ObTmpMacroBlock *t_mblk); - int check_and_free_mem_block(ObTmpMacroBlock *&blk); - bool check_block_full(); - - int exec_wait(); - int change_mem(); - -private: - // 1/256, only one free block each 256 block. - static constexpr double DEFAULT_MIN_FREE_BLOCK_RATIO = 0.00390625; - static const uint64_t DEFAULT_BUCKET_NUM = 1543L; - static const uint64_t MBLK_HASH_BUCKET_NUM = 10243L; - static const int64_t TENANT_MEM_BLOCK_NUM = 64L; - const int64_t TASK_INTERVAL = 10 * 1000; // 10 ms - const int64_t MEMORY_TASK_INTERVAL = 1000 * 1000; // 1 s - typedef common::hash::ObHashMap - TmpMacroBlockMap; - typedef common::hash::ObHashMap Map; - typedef common::hash::ObHashMap WaitHandleMap; - - struct BlockInfo final - { - public: - BlockInfo() :block_id_(0), wash_score_(INT64_MIN) {}; - ~BlockInfo() = default; - - TO_STRING_KV(K_(block_id), K_(wash_score)); - - int64_t block_id_; - double wash_score_; - }; - - class BlockWashScoreCompare final - { - public: - BlockWashScoreCompare(); - ~BlockWashScoreCompare() = default; - bool operator() (const BlockInfo &a, const BlockInfo &b); - int get_error_code() { return OB_SUCCESS; } - }; - - typedef common::ObBinaryHeap Heap; - struct ChooseBlocksMapOp final - { - public: - ChooseBlocksMapOp(Heap &heap, int64_t cur_time) : heap_(heap), cur_time_(cur_time) {} - int operator () (oceanbase::common::hash::HashMapPair &entry); - private: - Heap &heap_; - int64_t cur_time_; - }; - - struct GetAvailableBlockMapOp final - { - public: - GetAvailableBlockMapOp(const int64_t dir_id, const int64_t tenant_id, const int64_t page_nums, - ObTmpMacroBlock *&block, bool &is_found) - : dir_id_(dir_id), tenant_id_(tenant_id), page_nums_(page_nums), block_(block), is_found_(is_found){} - int operator () (oceanbase::common::hash::HashMapPair &entry); - private: - int64_t dir_id_; - int64_t tenant_id_; - int64_t page_nums_; - ObTmpMacroBlock *&block_; - bool &is_found_; - }; - - struct DestroyBlockMapOp final - { - public: - DestroyBlockMapOp(ObTmpTenantFileStore &tenant_store) : tenant_store_(tenant_store) {} - int operator () (oceanbase::common::hash::HashMapPair &entry); - private: - ObTmpTenantFileStore &tenant_store_; // reference to tenant store from ObTmpTenantMemBlockManager - }; - -private: - int get_available_macro_block(const int64_t dir_id, const uint64_t tenant_id, const int64_t page_nums, - ObTmpMacroBlock *&t_mblk, bool &is_found); - int write_io( - const ObTmpBlockIOInfo &io_info, - ObMacroBlockHandle &handle); - int64_t get_tenant_mem_block_num(); - int check_memory_limit(); - int get_block_from_dir_cache(const int64_t dir_id, const int64_t tenant_id, - const int64_t page_nums, ObTmpMacroBlock *&t_mblk); - int get_block_and_set_washing(int64_t block_id, ObTmpMacroBlock *&m_blk); - - ObTmpTenantFileStore &tenant_store_; - ObSpLinkQueue wait_info_queue_; - WaitHandleMap wait_handles_map_; - TmpMacroBlockMap t_mblk_map_; // - Map dir_to_blk_map_; // - double blk_nums_threshold_; // free_page_nums / total_page_nums - ObTmpBlockCache *block_cache_; - common::ObConcurrentFIFOAllocator *allocator_; - uint64_t tenant_id_; - int64_t last_access_tenant_config_ts_; - int64_t last_tenant_mem_block_num_; - bool is_inited_; - int tg_id_; - bool stopped_; - int64_t washing_count_; - ObTmpFileWaitTask wait_task_; - ObTmpFileMemTask mem_task_; - SpinRWLock io_lock_; - common::ObBucketLock map_lock_; - ObThreadCond cond_; - BlockWashScoreCompare compare_; - DISALLOW_COPY_AND_ASSIGN(ObTmpTenantMemBlockManager); -}; - -} // end namespace blocksstable -} // end namespace oceanbase -#endif // OCEANBASE_STORAGE_BLOCKSSTABLE_OB_TMP_FILE_CACHE_H_ - - diff --git a/src/storage/blocksstable/ob_tmp_file_store.cpp b/src/storage/blocksstable/ob_tmp_file_store.cpp deleted file mode 100644 index 518ff7d66..000000000 --- a/src/storage/blocksstable/ob_tmp_file_store.cpp +++ /dev/null @@ -1,2012 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#include "ob_tmp_file_store.h" -#include "ob_tmp_file.h" -#include "share/ob_task_define.h" -#include "observer/omt/ob_tenant_config_mgr.h" - -using namespace oceanbase::share; - -namespace oceanbase -{ -namespace blocksstable -{ - -const int64_t ObTmpMacroBlock::DEFAULT_PAGE_SIZE = 8192L; // 8kb - -ObTmpFilePageBuddy::ObTmpFilePageBuddy() - : is_inited_(false), - max_cont_page_nums_(0), - buf_(NULL), - allocator_(NULL) -{ - MEMSET(free_area_, 0, sizeof(free_area_)); -} - -ObTmpFilePageBuddy::~ObTmpFilePageBuddy() -{ - destroy(); -} - -int ObTmpFilePageBuddy::init(common::ObIAllocator &allocator) -{ - int ret = OB_SUCCESS; - uint8_t start_id = 0; - if (OB_UNLIKELY(is_inited_)) { - ret = OB_INIT_TWICE; - STORAGE_LOG(WARN, "ObTmpFilePageBuddy has not been inited", K(ret)); - } else { - allocator_ = &allocator; - buf_ = reinterpret_cast(allocator_->alloc(sizeof(ObTmpFileArea) * MAX_PAGE_NUMS)); - if (OB_ISNULL(buf_)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - STORAGE_LOG(WARN, "fail to alloc a buf", K(ret)); - } else { - max_cont_page_nums_ = std::pow(2, MAX_ORDER - 1); - /** - * page buddy free_list for a new block: - * -------------- --------- --------- --------- --------- --------- --------- --------- ------- - * |cont_page_nums| 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128 | - * -------------- --------- --------- --------- --------- --------- --------- --------- ------- - * | free_area |[254,254]|[252,253]|[248,251]|[240,247]|[224,239]|[192,223]|[128,191]|[0,127]| - * -------------- --------- --------- --------- --------- --------- --------- --------- ------- - */ - uint8_t nums = max_cont_page_nums_; - for (int32_t i = MIN_ORDER - 1; i >= 0; --i) { - free_area_[i] = NULL; - } - for (int32_t i = MAX_ORDER - 1; i >= MIN_ORDER; --i) { - char *buf = reinterpret_cast(&(buf_[start_id])); - free_area_[i] = new (buf) ObTmpFileArea(start_id, nums); - start_id += nums; - nums /= 2; - } - is_inited_ = true; - } - } - if (!is_inited_) { - destroy(); - } - return ret; -} - -void ObTmpFilePageBuddy::destroy() -{ - if (NULL != buf_) { - allocator_->free(buf_); - buf_ = NULL; - } - allocator_ = NULL; - max_cont_page_nums_ = 0; - is_inited_ = false; -} - -int ObTmpFilePageBuddy::alloc_all_pages() -{ - int ret = OB_SUCCESS; - ObTmpFileArea *tmp = NULL; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFilePageBuddy has not been inited", K(ret)); - } else if (is_empty()) { - for (int32_t i = 0; i < MAX_ORDER; ++i) { - while (NULL != free_area_[i]) { - tmp = free_area_[i]; - free_area_[i] = tmp->next_; - tmp->~ObTmpFileArea(); - } - } - - max_cont_page_nums_ = 0; - } else { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "this tmp block is not empty", K(ret)); - } - return ret; -} - -int ObTmpFilePageBuddy::alloc(const uint8_t page_nums, - uint8_t &start_page_id, - uint8_t &alloced_page_nums) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFilePageBuddy has not been inited", K(ret)); - } else if (OB_UNLIKELY(page_nums <= 0)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(page_nums)); - } else { - int32_t index = std::ceil(std::log(page_nums)/std::log(2)); - bool is_alloced = false; - for (int32_t i = index; i < MAX_ORDER && !is_alloced; ++i) { - if (NULL != free_area_[i]) { - int64_t num = i - index; - ObTmpFileArea *tmp = free_area_[i]; - free_area_[i] = tmp->next_; - tmp->next_ = NULL; - while (num--) { - tmp->page_nums_ /= 2; - char *buf = reinterpret_cast(&(buf_[tmp->start_page_id_ + tmp->page_nums_])); - ObTmpFileArea *area = new (buf) ObTmpFileArea(tmp->start_page_id_ + tmp->page_nums_, - tmp->page_nums_); - area->next_ = free_area_[static_cast(std::log(tmp->page_nums_)/std::log(2))]; - free_area_[static_cast(std::log(tmp->page_nums_)/std::log(2))] = area; - } - start_page_id = tmp->start_page_id_; - alloced_page_nums = std::pow(2, index); - is_alloced = true; - tmp->~ObTmpFileArea(); - } - } - - if (!is_alloced) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "cannot alloc the page", K(ret), K_(max_cont_page_nums), K(page_nums)); - } else { - index = std::ceil(std::log(max_cont_page_nums_)/std::log(2)); - int64_t max = 0; - for (int32_t i = index; i >= 0; --i) { - if (NULL == free_area_[i]) { - // nothing to do. - } else { - max = free_area_[i]->page_nums_; - break; - } - } - max_cont_page_nums_ = max; - } - } - return ret; -} - -void ObTmpFilePageBuddy::free_align(const int32_t start_page_id, const int32_t page_nums, - ObTmpFileArea *&area) -{ - ObTmpFileArea *tmp = NULL; - int64_t nums = page_nums; - int32_t start_id = start_page_id; - while (NULL != (tmp = find_buddy(nums, start_id))) { - // combine free area and buddy area. - if (0 != (start_id % (2 * nums))) { - start_id = tmp->start_page_id_; - std::swap(area, tmp); - } - nums *= 2; - tmp->~ObTmpFileArea(); - tmp = NULL; - } - area->start_page_id_ = start_id; - area->page_nums_ = nums; - area->next_ = free_area_[static_cast(std::log(nums)/std::log(2))]; - free_area_[static_cast(std::log(nums)/std::log(2))] = area; - if (nums > max_cont_page_nums_) { - max_cont_page_nums_ = nums; - } -} - -bool ObTmpFilePageBuddy::is_empty() const -{ - bool is_empty = true; - for (int32_t i = 0; i < MIN_ORDER && is_empty; ++i) { - if (NULL != free_area_[i]) { - is_empty = false; - break; - } - } - for (int32_t i = MIN_ORDER; i < MAX_ORDER && is_empty; ++i) { - if (NULL == free_area_[i]) { - is_empty = false; - break; - } - } - return is_empty; -} - -int64_t ObTmpFilePageBuddy::to_string(char* buf, const int64_t buf_len) const -{ - int64_t pos = 0; - bool first = true; - ObTmpFileArea *area = NULL; - common::databuff_printf(buf, buf_len, pos, "{"); - for (int32_t i = 0; i < MAX_ORDER; ++i) { - area = free_area_[i]; - if (NULL != area) { - common::databuff_print_kv(buf, buf_len, pos, "page_nums", static_cast(std::pow(2, i))); - common::databuff_printf(buf, buf_len, pos, "{"); - while (NULL != area) { - if (first) { - first = false; - } else { - common::databuff_printf(buf, buf_len, pos, ","); - } - common::databuff_printf(buf, buf_len, pos, "{"); - common::databuff_print_kv(buf, buf_len, pos, "start_page_id", area->start_page_id_, - "end_page_id", area->start_page_id_ + area->page_nums_); - common::databuff_printf(buf, buf_len, pos, "}"); - area = area->next_; - } - common::databuff_printf(buf, buf_len, pos, "}"); - } - } - common::databuff_printf(buf, buf_len, pos, "}"); - return pos; -} - -void ObTmpFilePageBuddy::free(const int32_t start_page_id, const int32_t page_nums) -{ - if (OB_UNLIKELY(start_page_id + page_nums >= std::pow(2, MAX_ORDER))) { - STORAGE_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "page id more than max numbers in block", K(start_page_id), K(page_nums)); - ob_abort(); - } else { - int32_t start_id = start_page_id; - int32_t nums = page_nums; - int32_t length = 0; - while (nums > 0) { - /** - * PURPOSE: align free area into power of 2. - * - * The probable value of alloc_start_id: - * page nums start page id - * 128 0 ---------------- 128 ------------------- 256 - * 64 0 ------ 64 ------ 128 ------- 192 ------- 256 - * 32 0 - 32 - 64 - 96 - 128 - 160 - 192 - 224 - 256 - * ... ... - * So, the maximum number of consecutive pages from a start_page_id is the - * gcd(greatest common divisor) between it and 512, except 0. The maximum - * consecutive page nums of 0 is 256. - * - * The layout of free area in alocated area : - * |<---------------alloc_page_nums--------------->| - * <---- |<--free_page_nums-->| - * |==========================|====================| - * alloc_start free_page_id alloc_end - * - * So, free_end always equal to alloc_end. - * - * Based on two observations above, the algorithm is designed as follows: - */ - length = 2; - while(0 == start_id % length && length <= nums) { - length *= 2; - } - length = std::min(length / 2, nums); - - char *buf = reinterpret_cast(&(buf_[start_id])); - ObTmpFileArea *area = new (buf) ObTmpFileArea(start_id, length); - free_align(area->start_page_id_, area->page_nums_, area); - start_id += length; - nums -= length; - } - } -} - -ObTmpFileArea *ObTmpFilePageBuddy::find_buddy(const int32_t page_nums, const int32_t start_page_id) -{ - ObTmpFileArea *tmp = NULL; - if (MAX_PAGE_NUMS < page_nums || page_nums <= 0 || start_page_id < 0 - || start_page_id >= MAX_PAGE_NUMS) { - STORAGE_LOG_RET(WARN, OB_INVALID_ARGUMENT, "invalid argument", K(page_nums), K(start_page_id)); - } else if (MAX_PAGE_NUMS == page_nums) { - // no buddy, so, nothing to do. - } else { - tmp = free_area_[static_cast(std::log(page_nums)/std::log(2))]; - ObTmpFileArea *pre = tmp; - int64_t start_id = 0; - /** - * case 1: case 2: - * |<--page_nums-->|<--page_nums-->| |<--page_nums-->|<--page_nums-->| - * |===============|===============| |===============|===============| - * start_page_id start_id(buddy) start_id(buddy) start_page_id - */ - if (0 == (start_page_id % (2 * page_nums))) { // case 1 - start_id = start_page_id + page_nums; - } else { // case 2 - start_id = start_page_id - page_nums; - } - while (NULL != tmp) { - if (tmp->start_page_id_ == start_id) { - if (pre == tmp) { - free_area_[static_cast(std::log(page_nums)/std::log(2))] = tmp->next_; - } else { - pre->next_ = tmp->next_; - } - tmp->next_ = NULL; - break; - } - pre = tmp; - tmp = tmp->next_; - } - } - return tmp; -} - -ObTmpFileMacroBlockHeader::ObTmpFileMacroBlockHeader() - : version_(TMP_FILE_MACRO_BLOCK_HEADER_VERSION), - magic_(TMP_FILE_MACRO_BLOCK_HEADER_MAGIC), - block_id_(-1), - dir_id_(-1), - tenant_id_(0), - free_page_nums_(-1) -{ -} - -bool ObTmpFileMacroBlockHeader::is_valid() const -{ - return TMP_FILE_MACRO_BLOCK_HEADER_VERSION == version_ - && TMP_FILE_MACRO_BLOCK_HEADER_MAGIC == magic_ - && block_id_ >= 0 - && dir_id_ >= 0 - && tenant_id_ > 0 - && free_page_nums_ >= 0; -} - -int ObTmpFileMacroBlockHeader::serialize(char *buf, const int64_t buf_len, int64_t &pos) const -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(buf) || OB_UNLIKELY(buf_len < 0)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid arguments", K(ret), KP(buf), K(buf_len)); - } else if (OB_UNLIKELY(pos + get_serialize_size() > buf_len)) { - ret = OB_BUF_NOT_ENOUGH; - STORAGE_LOG(WARN, "data buffer is not enough", K(ret), K(pos), K(buf_len), K(*this)); - } else if (OB_UNLIKELY(!is_valid())) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "tmp file macro block header is invalid", K(ret), K(*this)); - } else { - ObTmpFileMacroBlockHeader *header = reinterpret_cast(buf + pos); - header->version_ = version_; - header->magic_ = magic_; - header->block_id_ = block_id_; - header->dir_id_ = dir_id_; - header->tenant_id_ = tenant_id_; - header->free_page_nums_ = free_page_nums_; - pos += header->get_serialize_size(); - } - return ret; -} - -int ObTmpFileMacroBlockHeader::deserialize(const char *buf, const int64_t data_len, int64_t &pos) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(buf) || OB_UNLIKELY(data_len <= 0 || pos < 0)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid arguments", K(ret), KP(buf), K(data_len), K(pos)); - } else if (OB_UNLIKELY(data_len - pos < get_serialize_size())) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "buffer not enough", K(ret), KP(buf), K(data_len), K(pos)); - } else { - const ObTmpFileMacroBlockHeader *ptr = reinterpret_cast(buf + pos); - version_ = ptr->version_; - magic_ = ptr->magic_; - block_id_ = ptr->block_id_; - dir_id_ = ptr->dir_id_; - tenant_id_ = ptr->tenant_id_; - free_page_nums_ = ptr->free_page_nums_; - if (OB_UNLIKELY(!is_valid())) { - ret = OB_DESERIALIZE_ERROR; - STORAGE_LOG(ERROR, "deserialize error", K(ret), K(*this)); - } else { - pos += get_serialize_size(); - } - } - return ret; -} - -void ObTmpFileMacroBlockHeader::reset() -{ - version_ = TMP_FILE_MACRO_BLOCK_HEADER_VERSION; - magic_ = TMP_FILE_MACRO_BLOCK_HEADER_MAGIC; - block_id_ = -1; - dir_id_ = -1; - tenant_id_ = 0; - free_page_nums_ = 0; -} - -ObTmpMacroBlock::ObTmpMacroBlock() - : buffer_(NULL), - handle_(), - using_extents_(), - macro_block_handle_(), - tmp_file_header_(), - io_desc_(), - block_status_(MAX), - is_sealed_(false), - is_inited_(false), - alloc_time_(0), - access_time_(0) -{ - using_extents_.set_attr(ObMemAttr(MTL_ID(), "TMP_US_META")); -} - -ObTmpMacroBlock::~ObTmpMacroBlock() -{ - destroy(); -} - -int ObTmpMacroBlock::init(const int64_t block_id, const int64_t dir_id, const uint64_t tenant_id, - common::ObIAllocator &allocator) -{ - int ret = OB_SUCCESS; - if (IS_INIT) { - ret = OB_INIT_TWICE; - STORAGE_LOG(WARN, "ObTmpMacroBlock has not been inited", K(ret)); - } else if (OB_FAIL(page_buddy_.init(allocator))) { - STORAGE_LOG(WARN, "Fail to init the page buddy", K(ret)); - } else { - tmp_file_header_.block_id_ = block_id; - tmp_file_header_.dir_id_ = dir_id; - tmp_file_header_.tenant_id_ = tenant_id; - tmp_file_header_.free_page_nums_ = ObTmpFilePageBuddy::MAX_PAGE_NUMS; - ATOMIC_STORE(&block_status_, MEMORY); - is_inited_ = true; - alloc_time_ = 0; - ATOMIC_STORE(&access_time_, 0); - } - if (!is_inited_) { - destroy(); - } - return ret; -} - -void ObTmpMacroBlock::destroy() -{ - using_extents_.reset(); - page_buddy_.destroy(); - macro_block_handle_.reset(); - tmp_file_header_.reset(); - buffer_ = NULL; - handle_.reset(); - ATOMIC_STORE(&block_status_, MAX); - is_sealed_ = false; - alloc_time_ = 0; - ATOMIC_STORE(&access_time_, 0); - is_inited_ = false; -} - -int ObTmpMacroBlock::seal(bool &is_sealed) -{ - int ret = OB_SUCCESS; - is_sealed = false; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpMacroBlock has not been inited", K(ret), KPC(this)); - } else { - ObTmpFileExtent *tmp = NULL; - SpinWLockGuard guard(lock_); - bool is_all_closed = using_extents_.count() == 0 ? false : true; - for (int32_t i = 0; OB_SUCC(ret) && i < using_extents_.count() && is_all_closed; ++i) { - tmp = using_extents_.at(i); - if (NULL != tmp && !tmp->is_closed()) { - uint8_t start_id = ObTmpFilePageBuddy::MAX_PAGE_NUMS; - uint8_t page_nums = 0; - if (tmp->close(start_id, page_nums)) { - if (ObTmpFilePageBuddy::MAX_PAGE_NUMS== start_id && 0 == page_nums) { - //nothing to do - } else if (OB_UNLIKELY(start_id > ObTmpFilePageBuddy::MAX_PAGE_NUMS - 1 - || page_nums > ObTmpFilePageBuddy::MAX_PAGE_NUMS)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "fail to close the extent", K(ret), K(start_id), K(page_nums), K(*tmp)); - } else if (OB_FAIL(free(start_id, page_nums))) { - STORAGE_LOG(WARN, "fail to free the extent", K(ret)); - } else { - } - if (OB_FAIL(ret)) { - tmp->unclose(page_nums); - is_all_closed = false; - } - } else { - is_all_closed = false; - } - } - } - if (OB_SUCC(ret) && is_all_closed) { - is_sealed = true; - ATOMIC_SET(&is_sealed_, true); - } - } - return ret; -} - -int ObTmpMacroBlock::is_extents_closed(bool &is_extents_closed) -{ - int ret = OB_SUCCESS; - is_extents_closed = true; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpMacroBlock has not been inited"); - } else { - SpinRLockGuard guard(lock_); - is_extents_closed = using_extents_.count() == 0 ? false : true; - for (int64_t i = 0; i< using_extents_.count(); ++i) { - if (!using_extents_.at(i)->is_closed()) { - STORAGE_LOG(DEBUG, "the tmp macro block's extents is not all closed", K(tmp_file_header_.block_id_)); - is_extents_closed = false; - break; - } - } - } - return ret; -} - -int ObTmpMacroBlock::get_block_cache_handle(ObTmpBlockValueHandle &handle) -{ - int ret = OB_SUCCESS; - ObTmpBlockCacheKey key(tmp_file_header_.block_id_, tmp_file_header_.tenant_id_); - SpinRLockGuard guard(lock_); - if (!is_disked()) { - handle = handle_; - } else if (OB_FAIL(ObTmpBlockCache::get_instance().get_block(key, handle))) { - if (OB_UNLIKELY(OB_ENTRY_NOT_EXIST != ret)) { - STORAGE_LOG(WARN, "fail to get tmp block from cache", K(ret), K(key)); - } else if (REACH_COUNT_INTERVAL(100)) { // print one log per 100 times. - STORAGE_LOG(DEBUG, "block cache miss", K(ret), K(key)); - } - } - return ret; -} - -int ObTmpMacroBlock::get_wash_io_info(ObTmpBlockIOInfo &info) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpMacroBlock has not been inited", K(ret)); - } else { - info.block_id_ = tmp_file_header_.block_id_; - info.offset_ = 0; - info.size_ = ObTmpFileStore::get_block_size(); - info.tenant_id_ = tmp_file_header_.tenant_id_; - info.macro_block_id_ = get_macro_block_id(); - info.buf_ = buffer_; - info.io_desc_ = io_desc_; - info.io_timeout_ms_ = max(GCONF._data_storage_io_timeout / 1000L, DEFAULT_IO_WAIT_TIME_MS); - } - return ret; -} - -int ObTmpMacroBlock::give_back_buf_into_cache(const bool is_wash) -{ - int ret = OB_SUCCESS; - ObTmpBlockCacheKey key(tmp_file_header_.block_id_, tmp_file_header_.tenant_id_); - SpinWLockGuard guard(lock_); - if (OB_FAIL(ObTmpBlockCache::get_instance().put_block(key, handle_))) { - STORAGE_LOG(WARN, "fail to put block into block cache", K(ret), K(key)); - } - if (is_wash) {// set block status disked in lock_ to avoid concurrency issues. - if (OB_FAIL(ret)) { - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(check_and_set_status(BlockStatus::WASHING, BlockStatus::MEMORY))) { - STORAGE_LOG(ERROR, "fail to rollback block status", K(ret), K(tmp_ret), K(key), KPC(this)); - } - } else if (OB_FAIL(check_and_set_status(BlockStatus::WASHING, BlockStatus::DISKED))) { - STORAGE_LOG(WARN, "fail to check and set status", K(ret), K(key), KPC(this)); - } - } - return ret; -} - -int ObTmpMacroBlock::alloc_all_pages(ObTmpFileExtent &extent) -{ - int ret = OB_SUCCESS; - SpinWLockGuard guard(lock_); - const bool sealed = is_sealed(); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpMacroBlock has not been inited", K(ret)); - } else if (OB_UNLIKELY(!is_memory() || sealed)) { - ret = OB_STATE_NOT_MATCH; - STORAGE_LOG(WARN, "the block is not in memory", K(ret), K(ATOMIC_LOAD(&block_status_)), K(sealed)); - } else if (OB_FAIL(page_buddy_.alloc_all_pages())) { - STORAGE_LOG(WARN, "Fail to allocate the tmp extent", K(ret), K_(tmp_file_header), K_(page_buddy)); - } else { - extent.set_block_id(get_block_id()); - extent.set_start_page_id(0); - extent.set_page_nums(ObTmpFilePageBuddy::MAX_PAGE_NUMS); - extent.alloced(); - if (OB_FAIL(using_extents_.push_back(&extent))) { - STORAGE_LOG(WARN, "Fail to push back into using_extexts", K(ret)); - page_buddy_.free(extent.get_start_page_id(), extent.get_page_nums()); - extent.reset(); - } else { - tmp_file_header_.free_page_nums_ -= extent.get_page_nums(); - } - const int64_t cur_time = ObTimeUtility::fast_current_time(); - alloc_time_ = cur_time; - ATOMIC_STORE(&access_time_, alloc_time_ + 1); - } - return ret; -} - -int ObTmpMacroBlock::alloc(const uint8_t page_nums, ObTmpFileExtent &extent) -{ - int ret = OB_SUCCESS; - uint8_t start_page_id = extent.get_start_page_id(); - uint8_t alloced_page_nums = 0; - SpinWLockGuard guard(lock_); - const bool sealed = is_sealed(); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpMacroBlock has not been inited", K(ret)); - } else if (OB_UNLIKELY(!is_memory() || sealed)) { - ret = OB_STATE_NOT_MATCH; - STORAGE_LOG(WARN, "the block is not in memory", K(ret), K(ATOMIC_LOAD(&block_status_)), K(sealed)); - } else if (OB_FAIL(page_buddy_.alloc(page_nums, start_page_id, alloced_page_nums))) { - STORAGE_LOG(WARN, "Fail to allocate the tmp extent", K(ret), K_(tmp_file_header), K_(page_buddy)); - } else { - extent.set_block_id(tmp_file_header_.block_id_); - extent.set_page_nums(alloced_page_nums); - extent.set_start_page_id(start_page_id); - extent.alloced(); - if (OB_FAIL(using_extents_.push_back(&extent))) { - STORAGE_LOG(WARN, "Fail to push back into using_extexts", K(ret)); - page_buddy_.free(extent.get_start_page_id(), extent.get_page_nums()); - extent.reset(); - } else { - tmp_file_header_.free_page_nums_ -= alloced_page_nums; - } - const int64_t cur_time = ObTimeUtility::fast_current_time(); - if (0 == alloc_time_) { - alloc_time_ = cur_time; - } - if (OB_UNLIKELY(0 == ATOMIC_LOAD(&access_time_))) { - ATOMIC_STORE(&access_time_, - alloc_time_ + 60 * 1000000L * int64_t(alloced_page_nums) / get_max_cont_page_nums()); - } else { - ATOMIC_STORE(&access_time_, cur_time); - } - } - return ret; -} - -int ObTmpMacroBlock::free(ObTmpFileExtent &extent) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpMacroBlock has not been inited", K(ret)); - } else { - SpinWLockGuard guard(lock_); - page_buddy_.free(extent.get_start_page_id(), extent.get_page_nums()); - for (int64_t i = using_extents_.count() - 1; i >= 0; --i) { - if (&extent == using_extents_.at(i)) { - using_extents_.remove(i); - break; - } - } - tmp_file_header_.free_page_nums_ += extent.get_page_nums(); - if (tmp_file_header_.free_page_nums_ == ObTmpFilePageBuddy::MAX_PAGE_NUMS) { - alloc_time_ = 0; - ATOMIC_STORE(&access_time_, 0); - } - } - return ret; -} - -int ObTmpMacroBlock::free(const int32_t start_page_id, const int32_t page_nums) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpMacroBlock has not been inited", K(ret)); - } else { - page_buddy_.free(start_page_id, page_nums); - tmp_file_header_.free_page_nums_ += page_nums; - } - return ret; -} - -int64_t ObTmpMacroBlock::get_used_page_nums() const -{ - SpinRLockGuard guard(lock_); - int64_t used_page_nums = 0; - for (int64_t i = using_extents_.count() - 1; i >= 0; --i) { - used_page_nums += std::ceil((1.0 * using_extents_.at(i)->get_offset()) / DEFAULT_PAGE_SIZE); - } - return used_page_nums; -} - -void ObTmpMacroBlock::set_io_desc(const common::ObIOFlag &io_desc) -{ - io_desc_ = io_desc; -} - -int ObTmpMacroBlock::check_and_set_status(const BlockStatus old_block_status, const BlockStatus block_status) -{ - int ret = OB_SUCCESS; - if (old_block_status != ATOMIC_VCAS(&block_status_, old_block_status, block_status)) { - ret = OB_STATE_NOT_MATCH; - } - return ret; -} - -ObTmpTenantMacroBlockManager::ObTmpTenantMacroBlockManager() - : allocator_(), - blocks_(), - is_inited_(false) -{ -} - -ObTmpTenantMacroBlockManager::~ObTmpTenantMacroBlockManager() -{ - destroy(); -} - -int ObTmpTenantMacroBlockManager::init(const uint64_t tenant_id, common::ObIAllocator &allocator) -{ - int ret = OB_SUCCESS; - ObMemAttr attr(tenant_id, ObModIds::OB_TMP_BLOCK_MAP); - if (IS_INIT) { - ret = OB_INIT_TWICE; - STORAGE_LOG(WARN, "ObTmpMacroBlockManager has been inited", K(ret)); - } else if (OB_FAIL(blocks_.create(MBLK_HASH_BUCKET_NUM, attr, attr))) { - STORAGE_LOG(WARN, "Fail to create tmp macro block map, ", K(ret)); - } else { - allocator_ = &allocator; - is_inited_ = true; - } - if (!is_inited_) { - destroy(); - } - return ret; -} - -int ObTmpTenantMacroBlockManager::alloc_macro_block(const int64_t dir_id, const uint64_t tenant_id, - ObTmpMacroBlock *&t_mblk) -{ - int ret = OB_SUCCESS; - void *block_buf = NULL; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpMacroBlockManager has not been inited", K(ret)); - } else if (OB_ISNULL(block_buf = allocator_->alloc(sizeof(ObTmpMacroBlock)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - STORAGE_LOG(WARN, "fail to alloc a buf", K(ret)); - } else if (OB_ISNULL(t_mblk = new (block_buf) ObTmpMacroBlock())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "fail to new a ObTmpMacroBlock", K(ret)); - } else if (OB_FAIL(t_mblk->init(OB_TMP_FILE_STORE.get_next_blk_id(), dir_id, tenant_id, *allocator_))) { - STORAGE_LOG(WARN, "fail to init tmp block", K(ret)); - } else if (OB_FAIL(blocks_.set_refactored(t_mblk->get_block_id(), t_mblk))) { - STORAGE_LOG(WARN, "fail to set tmp macro block map", K(ret), K(t_mblk)); - } - if (OB_FAIL(ret)) { - if (NULL != t_mblk) { - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = blocks_.erase_refactored(t_mblk->get_block_id()))) { - STORAGE_LOG(WARN, "fail to erase from tmp macro block map", K(tmp_ret), K(t_mblk)); - } - t_mblk->~ObTmpMacroBlock(); - allocator_->free(block_buf); - } - } - return ret; -} - -int ObTmpTenantMacroBlockManager::get_macro_block(const int64_t block_id, ObTmpMacroBlock *&t_mblk) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpMacroBlockManager has not been inited", K(ret)); - } else if (OB_UNLIKELY(block_id <= 0)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(block_id)); - } else if (OB_FAIL(blocks_.get_refactored(block_id, t_mblk))) { - STORAGE_LOG(WARN, "fail to get tmp macro block", K(ret), K(block_id)); - } - return ret; -} - -int ObTmpTenantMacroBlockManager::free_macro_block(const int64_t block_id) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpMacroBlockManager has not been inited", K(ret)); - } else if (OB_UNLIKELY(block_id <= 0)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(block_id)); - } else if (OB_FAIL(blocks_.erase_refactored(block_id))) { - STORAGE_LOG(WARN, "fail to erase tmp macro block", K(ret)); - } - return ret; -} - -int ObTmpTenantMacroBlockManager::get_disk_macro_block_count(int64_t &count) const -{ - int ret = OB_SUCCESS; - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpMacroBlockManager has not been inited", K(ret)); - } else { - count = blocks_.size(); - } - - return ret; -} - -int ObTmpTenantMacroBlockManager::get_disk_macro_block_list( - common::ObIArray ¯o_id_list) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpMacroBlockManager has not been inited", K(ret)); - } else { - TmpMacroBlockMap::iterator iter; - ObTmpMacroBlock *tmp = NULL; - for (iter = blocks_.begin(); OB_SUCC(ret) && iter != blocks_.end(); ++iter) { - if (OB_ISNULL(tmp = iter->second)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "fail to iterate tmp macro block map", K(ret)); - } else if (tmp->is_disked() && tmp->get_macro_block_id().is_valid() && - OB_FAIL(macro_id_list.push_back(tmp->get_macro_block_id()))) { - STORAGE_LOG(WARN, "fail to push back macro block id", K(ret), K(tmp->get_macro_block_id())); - } - } - } - - return ret; -} - -void ObTmpTenantMacroBlockManager::print_block_usage() -{ - int64_t disk_count = 0; - int64_t disk_fragment = 0; - int64_t mem_count = 0; - int64_t mem_fragment = 0; - TmpMacroBlockMap::iterator iter; - ObTmpMacroBlock *tmp = NULL; - for (iter = blocks_.begin(); iter != blocks_.end(); ++iter) { - tmp = iter->second; - if (tmp->is_disked()) { - disk_count++; - disk_fragment += tmp->get_free_page_nums(); - } else { - mem_count++; - mem_fragment += tmp->get_free_page_nums(); - } - } - double disk_fragment_ratio = 0; - if (0 != disk_count) { - disk_fragment_ratio = disk_fragment * 1.0 / (disk_count * ObTmpFilePageBuddy::MAX_PAGE_NUMS); - } - double mem_fragment_ratio = 0; - if (0 != mem_count) { - mem_fragment_ratio = mem_fragment * 1.0 / (mem_count * ObTmpFilePageBuddy::MAX_PAGE_NUMS); - } - STORAGE_LOG(INFO, "the block usage for temporary files", - K(disk_count), K(disk_fragment), K(disk_fragment_ratio), - K(mem_count), K(mem_fragment), K(mem_fragment_ratio)); -} - -void ObTmpTenantMacroBlockManager::destroy() -{ - TmpMacroBlockMap::iterator iter; - ObTmpMacroBlock *tmp = NULL; - for (iter = blocks_.begin(); iter != blocks_.end(); ++iter) { - if (OB_NOT_NULL(tmp = iter->second)) { - tmp->~ObTmpMacroBlock(); - allocator_->free(tmp); - } - } - blocks_.destroy(); - allocator_ = NULL; - is_inited_ = false; -} - -ObTmpTenantFileStore::ObTmpTenantFileStore() - : is_inited_(false), - page_cache_num_(0), - block_cache_num_(0), - ref_cnt_(0), - page_cache_(NULL), - lock_(), - allocator_(), - io_allocator_(), - tmp_block_manager_(), - tmp_mem_block_manager_(*this), - last_access_tenant_config_ts_(0), - last_meta_mem_limit_(TOTAL_LIMIT) -{ -} - -ObTmpTenantFileStore::~ObTmpTenantFileStore() -{ - destroy(); -} - -void ObTmpTenantFileStore::inc_ref() -{ - ATOMIC_INC(&ref_cnt_); -} - -int64_t ObTmpTenantFileStore::dec_ref() -{ - int ret = OB_SUCCESS; - const int64_t tmp_ref = ATOMIC_SAF(&ref_cnt_, 1); - if (tmp_ref < 0) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(ERROR, "bug: ref_cnt < 0", K(ret), K(tmp_ref), K(lbt())); - ob_abort(); - } - return tmp_ref; -} - -int ObTmpTenantFileStore::init(const uint64_t tenant_id) -{ - int ret = OB_SUCCESS; - if (IS_INIT) { - ret = OB_INIT_TWICE; - STORAGE_LOG(WARN, "ObTmpTenantFileStore has not been inited", K(ret)); - } else if (OB_FAIL(allocator_.init(BLOCK_SIZE, ObModIds::OB_TMP_BLOCK_MANAGER, tenant_id, get_memory_limit(tenant_id)))) { - STORAGE_LOG(WARN, "fail to init allocator", K(ret)); - } else if (OB_FAIL(io_allocator_.init( - lib::ObMallocAllocator::get_instance(), - OB_MALLOC_MIDDLE_BLOCK_SIZE, - ObMemAttr(tenant_id, ObModIds::OB_TMP_PAGE_CACHE, ObCtxIds::DEFAULT_CTX_ID)))) { - STORAGE_LOG(WARN, "Fail to init io allocator, ", K(ret)); - } else if (OB_ISNULL(page_cache_ = &ObTmpPageCache::get_instance())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "fail to get the page cache", K(ret)); - } else if (OB_FAIL(tmp_block_manager_.init(tenant_id, allocator_))) { - STORAGE_LOG(WARN, "fail to init the block manager for ObTmpFileStore", K(ret)); - } else if (OB_FAIL(tmp_mem_block_manager_.init(tenant_id, allocator_))) { - STORAGE_LOG(WARN, "fail to init memory block manager", K(ret)); - } else { - is_inited_ = true; - } - if (!is_inited_) { - destroy(); - } - return ret; -} - -void ObTmpTenantFileStore::refresh_memory_limit(const uint64_t tenant_id) -{ - const int64_t old_limit = ATOMIC_LOAD(&last_meta_mem_limit_); - const int64_t new_limit = get_memory_limit(tenant_id); - if (old_limit != new_limit) { - allocator_.set_total_limit(new_limit); - ObTaskController::get().allow_next_syslog(); - STORAGE_LOG(INFO, "succeed to refresh temporary file meta memory limit", K(tenant_id), K(old_limit), K(new_limit)); - } -} - -int64_t ObTmpTenantFileStore::get_memory_limit(const uint64_t tenant_id) -{ - const int64_t last_access_ts = ATOMIC_LOAD(&last_access_tenant_config_ts_); - int64_t memory_limit = TOTAL_LIMIT; - if (last_access_ts > 0 && common::ObClockGenerator::getClock() - last_access_ts < REFRESH_CONFIG_INTERVAL) { - memory_limit = ATOMIC_LOAD(&last_meta_mem_limit_); - } else { - omt::ObTenantConfigGuard config(TENANT_CONF(tenant_id)); - const int64_t tenant_mem_limit = lib::get_tenant_memory_limit(tenant_id); - if (!config.is_valid() || 0 == tenant_mem_limit || INT64_MAX == tenant_mem_limit) { - COMMON_LOG(INFO, "failed to get tenant config", K(tenant_id), K(tenant_mem_limit)); - } else { - const int64_t limit_percentage_config = config->_temporary_file_meta_memory_limit_percentage; - const int64_t limit_percentage = 0 == limit_percentage_config ? 70 : limit_percentage_config; - memory_limit = tenant_mem_limit * limit_percentage / 100; - if (OB_UNLIKELY(memory_limit <= 0)) { - STORAGE_LOG(INFO, "memory limit isn't more than 0", K(memory_limit)); - memory_limit = ATOMIC_LOAD(&last_meta_mem_limit_); - } else { - ATOMIC_STORE(&last_meta_mem_limit_, memory_limit); - ATOMIC_STORE(&last_access_tenant_config_ts_, common::ObClockGenerator::getClock()); - } - } - } - return memory_limit; -} - -void ObTmpTenantFileStore::destroy() -{ - tmp_mem_block_manager_.destroy(); - tmp_block_manager_.destroy(); - if (NULL != page_cache_) { - page_cache_ = NULL; - } - allocator_.destroy(); - io_allocator_.reset(); - last_access_tenant_config_ts_ = 0; - last_meta_mem_limit_ = TOTAL_LIMIT; - is_inited_ = false; - STORAGE_LOG(INFO, "cache num when destroy", - K(ATOMIC_LOAD(&page_cache_num_)), K(ATOMIC_LOAD(&block_cache_num_))); - page_cache_num_ = 0; - block_cache_num_ = 0; -} - -int ObTmpTenantFileStore::alloc(const int64_t dir_id, const uint64_t tenant_id, const int64_t alloc_size, - ObTmpFileExtent &extent) -{ - int ret = OB_SUCCESS; - const int64_t block_size = ObTmpFileStore::get_block_size(); - // In buddy allocation, if free space in one block isn't powers of 2, need upper align. - int64_t max_order = std::ceil(std::log(ObTmpFilePageBuddy::MAX_PAGE_NUMS) / std::log(2)); - int64_t origin_max_cont_page_nums = std::pow(2, max_order - 1); - int64_t max_cont_size_per_block = origin_max_cont_page_nums * ObTmpMacroBlock::get_default_page_size(); - ObTmpMacroBlock *t_mblk = NULL; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpTenantFileStore has not been inited", K(ret)); - } else if (alloc_size <= 0 || alloc_size > block_size) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(alloc_size), K(block_size)); - } else { - const int64_t timeout_ts = THIS_WORKER.get_timeout_ts(); - ret = OB_STATE_NOT_MATCH; - while (OB_STATE_NOT_MATCH == ret || OB_SIZE_OVERFLOW == ret) { - if (OB_UNLIKELY(timeout_ts <= ObTimeUtility::current_time())) { - ret = OB_TIMEOUT; - STORAGE_LOG(WARN, "it's timeout", K(ret), K(timeout_ts), K(ObTimeUtility::current_time())); - } else if (OB_FAIL(tmp_mem_block_manager_.cleanup())) { - if (OB_STATE_NOT_MATCH != ret) { - STORAGE_LOG(WARN, "fail to try wash tmp macro block", K(ret), K(dir_id), K(tenant_id)); - } - } - SpinWLockGuard guard(lock_); - if (OB_SUCC(ret)) { - if (alloc_size > max_cont_size_per_block) { - if (OB_FAIL(alloc_macro_block(dir_id, tenant_id, t_mblk))) { - if (OB_SIZE_OVERFLOW != ret) { - STORAGE_LOG(WARN, "cannot allocate a tmp macro block", K(ret), K(dir_id), K(tenant_id)); - } - } else if (OB_ISNULL(t_mblk)) { - ret = OB_ERR_NULL_VALUE; - STORAGE_LOG(WARN, "block alloced is NULL", K(ret), K(dir_id), K(tenant_id)); - } else if (OB_FAIL(t_mblk->alloc_all_pages(extent))) { - STORAGE_LOG(WARN, "Fail to allocate the tmp extent", K(ret), K(t_mblk->get_block_id())); - } else if (alloc_size < block_size) { - const int64_t nums = std::ceil(alloc_size * 1.0 / ObTmpMacroBlock::get_default_page_size()); - if (OB_FAIL(free_extent(t_mblk->get_block_id(), nums, - ObTmpFilePageBuddy::MAX_PAGE_NUMS - nums))) { - STORAGE_LOG(WARN, "fail to free pages", K(ret), K(t_mblk->get_block_id())); - } else { - extent.set_page_nums(nums); - } - } - } else if (OB_FAIL(tmp_mem_block_manager_.alloc_extent(dir_id, tenant_id, alloc_size, extent))) { - if (OB_STATE_NOT_MATCH == ret) { - if (OB_FAIL(alloc_macro_block(dir_id, tenant_id, t_mblk))) { - if (OB_SIZE_OVERFLOW != ret) { - STORAGE_LOG(WARN, "cannot allocate a tmp macro block", K(ret), K(dir_id), K(tenant_id)); - } - } else if (OB_ISNULL(t_mblk)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "unexpected error, t_mblk is nullptr", K(ret), KP(t_mblk)); - } else { - const int64_t page_nums = std::ceil(alloc_size * 1.0 / ObTmpMacroBlock::get_default_page_size()); - if (OB_FAIL(t_mblk->alloc(page_nums, extent))){ - STORAGE_LOG(WARN, "fail to alloc tmp extent", K(ret)); - } else if (OB_FAIL(tmp_mem_block_manager_.refresh_dir_to_blk_map(dir_id, t_mblk))) { - STORAGE_LOG(WARN, "fail to refresh dir_to_blk_map", K(ret), K(*t_mblk)); - } - } - } - } - } - } - } - if (OB_FAIL(ret)) { - STORAGE_LOG(WARN, "fail to alloc tmp extent", K(ret)); - if (OB_ALLOCATE_MEMORY_FAILED == ret) { - STORAGE_LOG(WARN, "alloc memory failed", K(ret), K(ATOMIC_LOAD(&block_cache_num_)), K(ATOMIC_LOAD(&page_cache_num_))); - } - } - return ret; -} - -int ObTmpTenantFileStore::free(ObTmpFileExtent *extent) -{ - int ret = OB_SUCCESS; - SpinWLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpTenantFileStore has not been inited", K(ret)); - } else if (OB_FAIL(free_extent(extent))) { - STORAGE_LOG(WARN, "fail to free the extent", K(ret), K(*extent)); - } - return ret; -} - -int ObTmpTenantFileStore::free(const int64_t block_id, const int32_t start_page_id, - const int32_t page_nums) -{ - int ret = OB_SUCCESS; - SpinWLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpTenantFileStore has not been inited", K(ret)); - } else if (OB_FAIL(free_extent(block_id, start_page_id, page_nums))) { - STORAGE_LOG(WARN, "fail to free the extent", K(ret), K(block_id), K(start_page_id), - K(page_nums)); - } - return ret; -} - -int ObTmpTenantFileStore::free_extent(ObTmpFileExtent *extent) -{ - int ret = OB_SUCCESS; - ObTmpMacroBlock *t_mblk = NULL; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpTenantFileStore has not been inited", K(ret)); - } else if (OB_ISNULL(extent) || OB_UNLIKELY(!extent->is_valid())) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(*extent)); - } else if (OB_FAIL(tmp_block_manager_.get_macro_block(extent->get_block_id(), t_mblk))) { - STORAGE_LOG(WARN, "fail to get tmp macro block", K(ret)); - } else if (OB_ISNULL(t_mblk)) { - ret = OB_ERR_NULL_VALUE; - STORAGE_LOG(WARN, "block is null", K(ret)); - } else if (OB_FAIL(t_mblk->free(*extent))) { - STORAGE_LOG(WARN, "fail to free extent", K(ret)); - } else { - extent->reset(); - if (OB_SUCC(ret) && t_mblk->is_empty()) { - if (OB_FAIL(free_macro_block(t_mblk))) { - STORAGE_LOG(WARN, "fail to free tmp macro block", K(ret)); - } - } - } - return ret; -} - -int ObTmpTenantFileStore::free_extent(const int64_t block_id, const int32_t start_page_id, - const int32_t page_nums) -{ - int ret = OB_SUCCESS; - ObTmpMacroBlock *t_mblk = NULL; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpTenantFileStore has not been inited", K(ret)); - } else if (OB_UNLIKELY(start_page_id < 0 || page_nums < 0 || block_id <= 0)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(block_id), K(start_page_id), K(page_nums)); - } else if (OB_FAIL(tmp_block_manager_.get_macro_block(block_id, t_mblk))) { - STORAGE_LOG(WARN, "fail to get tmp block", K(ret), K(block_id), K(start_page_id), K(page_nums)); - } else if (OB_ISNULL(t_mblk)) { - ret = OB_ERR_NULL_VALUE; - STORAGE_LOG(WARN, "block is null", K(ret)); - } else if (OB_FAIL(t_mblk->free(start_page_id, page_nums))) { - STORAGE_LOG(WARN, "fail to free extent", K(ret)); - } else { - if (OB_SUCC(ret) && t_mblk->is_empty()) { - if (OB_FAIL(free_macro_block(t_mblk))) { - STORAGE_LOG(WARN, "fail to free tmp macro block", K(ret)); - } - } - } - return ret; -} - -int ObTmpTenantFileStore::free_macro_block(ObTmpMacroBlock *&t_mblk) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpTenantFileStore has not been inited", K(ret)); - } else if (OB_ISNULL(t_mblk)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret)); - } else if (OB_FAIL(tmp_block_manager_.free_macro_block(t_mblk->get_block_id()))) { - STORAGE_LOG(WARN, "fail to free tmp macro block for block manager", K(ret)); - } else { - while (OB_SUCC(ret) && !t_mblk->is_disked()) { - if (t_mblk->is_memory() && OB_FAIL(tmp_mem_block_manager_.check_and_free_mem_block(t_mblk))) { - STORAGE_LOG(WARN, "fail to check and free mem block", K(ret), KPC(t_mblk)); - } else if (t_mblk->is_washing() && - OB_FAIL(tmp_mem_block_manager_.wait_write_finish(t_mblk->get_block_id(), - ObTmpTenantMemBlockManager::get_default_timeout_ms()))) { - STORAGE_LOG(WARN, "fail to wait write io finish", K(ret), KPC(t_mblk)); - } - } - if (OB_SUCC(ret)) { - ObTaskController::get().allow_next_syslog(); - STORAGE_LOG(INFO, "finish to free a block", K(ret), KPC(t_mblk)); - t_mblk->~ObTmpMacroBlock(); - allocator_.free(t_mblk); - t_mblk = nullptr; - } - } - return ret; -} - -int ObTmpTenantFileStore::alloc_macro_block(const int64_t dir_id, const uint64_t tenant_id, - ObTmpMacroBlock *&t_mblk) -{ - int ret = OB_SUCCESS; - t_mblk = nullptr; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpMacroBlockManager has not been inited", K(ret)); - } else if (tmp_mem_block_manager_.check_block_full()) { - ret = OB_SIZE_OVERFLOW; - STORAGE_LOG(DEBUG, "mem block is full", K(ret), K(tenant_id), K(dir_id)); - } else if (OB_FAIL(tmp_block_manager_.alloc_macro_block(dir_id, tenant_id, t_mblk))) { - STORAGE_LOG(WARN, "cannot allocate a tmp macro block", K(ret), K(dir_id), K(tenant_id)); - } else if (OB_ISNULL(t_mblk)) { - ret = OB_ERR_NULL_VALUE; - STORAGE_LOG(WARN, "block is null", K(ret)); - } else { - ObTmpBlockCacheKey key(t_mblk->get_block_id(), tenant_id); - if (OB_FAIL(tmp_mem_block_manager_.alloc_buf(key, t_mblk->get_handle()))) { - STORAGE_LOG(WARN, "fail to alloc block cache buf", K(ret)); - } else { - t_mblk->set_buffer(t_mblk->get_handle().value_->get_buffer()); - if (OB_FAIL(tmp_mem_block_manager_.add_macro_block(t_mblk))) { - STORAGE_LOG(WARN, "fail to put meta into block cache", K(ret), K(t_mblk)); - } - inc_block_cache_num(1); - if (OB_FAIL(ret)) { - tmp_mem_block_manager_.free_macro_block(t_mblk->get_block_id()); - t_mblk->give_back_buf_into_cache(); - dec_block_cache_num(1); - } - } - if (OB_FAIL(ret) && OB_NOT_NULL(t_mblk)) { - tmp_block_manager_.free_macro_block(t_mblk->get_block_id()); - t_mblk->~ObTmpMacroBlock(); - allocator_.free(t_mblk); - t_mblk = nullptr; - } - } - - return ret; -} - -int ObTmpTenantFileStore::read(ObTmpBlockIOInfo &io_info, ObTmpFileIOHandle &handle) -{ - int ret = OB_SUCCESS; - ObTmpBlockValueHandle tb_handle; - ObTmpMacroBlock *block = NULL; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpTenantFileStore has not been inited", K(ret)); - } else if (!handle.is_valid()) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(handle)); - } else if (OB_FAIL(tmp_block_manager_.get_macro_block(io_info.block_id_, block))) { - STORAGE_LOG(WARN, "fail to get block from tmp block manager", K(ret), K_(io_info.block_id)); - } else if (OB_ISNULL(block)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "the block is NULL", K(ret), K_(io_info.block_id)); - } else { - if (OB_SUCC(block->get_block_cache_handle(tb_handle))) { - ObTmpFileIOHandle::ObBlockCacheHandle block_handle( - tb_handle, - io_info.buf_, - io_info.offset_, - io_info.size_); - OB_TMP_FILE_STORE.inc_block_cache_num(io_info.tenant_id_, 1); - if (OB_FAIL(handle.get_block_cache_handles().push_back(block_handle))) { - STORAGE_LOG(WARN, "Fail to push back into block_handles", K(ret), K(block_handle)); - } - } else if (OB_SUCC(read_page(block, io_info, handle))) { - //nothing to do. - } else { - STORAGE_LOG(WARN, "fail to read", K(ret)); - } - - if (OB_SUCC(ret)) { - block->set_io_desc(io_info.io_desc_); - } - } - return ret; -} - -int ObTmpTenantFileStore::read_page(ObTmpMacroBlock *block, ObTmpBlockIOInfo &io_info, - ObTmpFileIOHandle &handle) -{ - int ret = OB_SUCCESS; - int64_t page_start_id = io_info.offset_ / ObTmpMacroBlock::get_default_page_size(); - int64_t offset = io_info.offset_ % ObTmpMacroBlock::get_default_page_size(); - int64_t remain_size = io_info.size_; - int64_t size = std::min(ObTmpMacroBlock::get_default_page_size() - offset, remain_size); - int32_t page_nums = 0; - common::ObIArray *page_io_infos = nullptr; - - void *buf = - ob_malloc(sizeof(common::ObSEArray), - ObMemAttr(MTL_ID(), "TmpReadPage")); - if (OB_ISNULL(buf)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - STORAGE_LOG(WARN, "fail to alloc a buf", K(ret)); - } else { - page_io_infos = new (buf) common::ObSEArray(); - do { - ObTmpPageCacheKey key(io_info.block_id_, page_start_id, io_info.tenant_id_); - ObTmpPageValueHandle p_handle; - if (OB_SUCC(page_cache_->get_page(key, p_handle))) { - ObTmpFileIOHandle::ObPageCacheHandle page_handle( - p_handle, - io_info.buf_ + ObTmpMacroBlock::calculate_offset(page_start_id, offset) - io_info.offset_, - offset, - size); - inc_page_cache_num(1); - if (OB_FAIL(handle.get_page_cache_handles().push_back(page_handle))) { - STORAGE_LOG(WARN, "Fail to push back into page_handles", K(ret)); - } - } else if (OB_ENTRY_NOT_EXIST == ret) { - ret = OB_SUCCESS; - // accumulate page io info. - ObTmpPageIOInfo page_io_info; - page_io_info.key_ = key; - page_io_info.offset_ = offset; - page_io_info.size_ = size; - if (OB_FAIL(page_io_infos->push_back(page_io_info))) { - STORAGE_LOG(WARN, "Fail to push back into page_io_infos", K(ret), K(page_io_info)); - } - } else { - STORAGE_LOG(WARN, "fail to get page from page cache", K(ret)); - } - page_nums++; - page_start_id++; - offset = 0; - remain_size -= size; - size = std::min(ObTmpMacroBlock::get_default_page_size(), remain_size); - } while (OB_SUCC(ret) && size > 0); - } - - if (OB_SUCC(ret)) { - if (page_io_infos->count() > DEFAULT_PAGE_IO_MERGE_RATIO * page_nums) { - // merge multi page io into one. - ObMacroBlockHandle mb_handle; - ObTmpBlockIOInfo info(io_info); - const int64_t p_offset = common::lower_align(io_info.offset_, ObTmpMacroBlock::get_default_page_size()); - // just skip header and padding. - info.offset_ = p_offset + ObTmpMacroBlock::get_header_padding(); - info.size_ = page_nums * ObTmpMacroBlock::get_default_page_size(); - info.macro_block_id_ = block->get_macro_block_id(); - if (handle.is_disable_page_cache()) { - if (OB_FAIL(page_cache_->direct_read(info, mb_handle, io_allocator_))) { - STORAGE_LOG(WARN, "fail to direct read multi page", K(ret)); - } - } else { - if (OB_FAIL(page_cache_->prefetch(info, *page_io_infos, mb_handle, io_allocator_))) { - STORAGE_LOG(WARN, "fail to prefetch multi tmp page", K(ret)); - } - } - if (OB_SUCC(ret)) { - ObTmpFileIOHandle::ObIOReadHandle read_handle(mb_handle, io_info.buf_, - io_info.offset_ - p_offset, io_info.size_); - if (OB_FAIL(handle.get_io_handles().push_back(read_handle))) { - STORAGE_LOG(WARN, "Fail to push back into read_handles", K(ret)); - } - } - } else { - // just do io, page by page. - for (int i = 0; OB_SUCC(ret) && i < page_io_infos->count(); i++) { - ObMacroBlockHandle mb_handle; - ObTmpBlockIOInfo info(io_info); - info.offset_ = page_io_infos->at(i).key_.get_page_id() * ObTmpMacroBlock::get_default_page_size(); - // just skip header and padding. - info.offset_ += ObTmpMacroBlock::get_header_padding(); - info.size_ = ObTmpMacroBlock::get_default_page_size(); - info.macro_block_id_ = block->get_macro_block_id(); - if (handle.is_disable_page_cache()) { - if (OB_FAIL(page_cache_->direct_read(info, mb_handle, io_allocator_))) { - STORAGE_LOG(WARN, "fail to direct read tmp page", K(ret)); - } - } else { - if (OB_FAIL(page_cache_->prefetch(page_io_infos->at(i).key_, info, mb_handle, io_allocator_))) { - STORAGE_LOG(WARN, "fail to prefetch tmp page", K(ret)); - } - } - if (OB_SUCC(ret)) { - char *buf = io_info.buf_ + ObTmpMacroBlock::calculate_offset( - page_io_infos->at(i).key_.get_page_id(), page_io_infos->at(i).offset_) - io_info.offset_; - ObTmpFileIOHandle::ObIOReadHandle read_handle(mb_handle, buf, page_io_infos->at(i).offset_, - page_io_infos->at(i).size_); - if (OB_FAIL(handle.get_io_handles().push_back(read_handle))) { - STORAGE_LOG(WARN, "Fail to push back into read_handles", K(ret)); - } - } - } - } - } - if (OB_NOT_NULL(page_io_infos)) { - page_io_infos->destroy(); - ob_free(page_io_infos); - } - return ret; -} - -int ObTmpTenantFileStore::write(const ObTmpBlockIOInfo &io_info) -{ - int ret = OB_SUCCESS; - ObTmpMacroBlock *block = NULL; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpTenantFileStore has not been inited", K(ret)); - } else if (OB_FAIL(tmp_block_manager_.get_macro_block(io_info.block_id_, block))) { - STORAGE_LOG(WARN, "fail to get block from tmp block manager", K(ret), K_(io_info.block_id)); - } else if (OB_ISNULL(block)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "the block is NULL", K(ret), K_(io_info.block_id)); - } else if (block->is_memory()) { - block->set_io_desc(io_info.io_desc_); - MEMCPY(block->get_buffer() + io_info.offset_, io_info.buf_, io_info.size_); - } else { - // The washing and disked block shouldn't write data, Otherwise, it will cause data corrupt. - ret = OB_NOT_SUPPORTED; - STORAGE_LOG(WARN, "block status is not correct", K(ret), K(io_info)); - } - return ret; -} - -int ObTmpTenantFileStore::wash_block(const int64_t block_id, - ObTmpTenantMemBlockManager::ObIOWaitInfoHandle &handle) -{ - int ret = OB_SUCCESS; - SpinWLockGuard guard(lock_); - ObTmpMacroBlock *block = NULL; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpTenantFileStore has not been inited", K(ret), K(block_id)); - } else if (OB_UNLIKELY(block_id <= 0)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(block_id)); - } else if (OB_FAIL(tmp_block_manager_.get_macro_block(block_id, block))) { - STORAGE_LOG(WARN, "fail to get block from tmp block manager", K(ret), K(block_id)); - } else if (OB_ISNULL(block)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "the block is NULL", K(ret), K(block_id)); - } else if (OB_FAIL(tmp_mem_block_manager_.wash_block(block_id, handle))) { - STORAGE_LOG(WARN, "wash block failed", K(ret), K(block_id)); - } - return ret; -} - -int ObTmpTenantFileStore::sync_block(const int64_t block_id, - ObTmpTenantMemBlockManager::ObIOWaitInfoHandle &handle) -{ - int ret = OB_SUCCESS; - ObTmpMacroBlock *blk = NULL; - SpinRLockGuard guard(lock_); - bool is_closed = false; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpTenantFileStore has not been inited", K(ret), K(block_id)); - } else if (OB_UNLIKELY(block_id <= 0)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(block_id)); - } else if (OB_FAIL(tmp_block_manager_.get_macro_block(block_id, blk))) { - STORAGE_LOG(WARN, "fail to get macro block", K(ret), K(block_id)); - } else if (OB_FAIL(blk->is_extents_closed(is_closed))) { - STORAGE_LOG(WARN, "check block closed failed", K(ret), K(block_id)); - } else if (!is_closed) { - //do nothing - } else if (OB_FAIL(tmp_mem_block_manager_.wash_block(block_id, handle))) { - STORAGE_LOG(WARN, "wash block failed", K(ret), K(block_id)); - } else { - STORAGE_LOG(DEBUG, "succeed to sync block", K(block_id)); - } - return ret; -} - -int ObTmpTenantFileStore::wait_write_finish(const int64_t block_id, const int64_t timeout_ms) -{ - int ret = OB_SUCCESS; - ObTmpMacroBlock *blk = NULL; - bool is_closed = false; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpTenantFileStore has not been inited", K(ret), K(block_id)); - } else if (OB_UNLIKELY(block_id <= 0)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(block_id)); - } else if (OB_FAIL(tmp_block_manager_.get_macro_block(block_id, blk))) { - STORAGE_LOG(WARN, "fail to get macro block", K(ret), K(block_id)); - } else if (OB_ISNULL(blk)) { - ret = OB_ERR_NULL_VALUE; - STORAGE_LOG(WARN, "macro block is NULL", K(ret), K(block_id)); - } else if (!blk->is_washing()) { - // block has not been washed, nothing todo. - } else if (blk->is_washing() && - OB_FAIL(tmp_mem_block_manager_.wait_write_finish(block_id, timeout_ms))){ - STORAGE_LOG(WARN, "wait write finish failed", K(ret), K(block_id)); - } - return ret; -} - -int ObTmpTenantFileStore::get_disk_macro_block_count(int64_t &count) const -{ - int ret = OB_SUCCESS; - SpinRLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpTenantFileStore has not been inited", K(ret)); - } else if (OB_FAIL(tmp_block_manager_.get_disk_macro_block_count(count))) { - STORAGE_LOG(WARN, "fail to get disk macro block count from tmp_block_manager_", K(ret)); - } - return ret; -} - -int ObTmpTenantFileStore::get_disk_macro_block_list(common::ObIArray ¯o_id_list) -{ - int ret = OB_SUCCESS; - SpinRLockGuard guard(lock_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpTenantFileStore has not been inited", K(ret)); - } else if (OB_FAIL(tmp_block_manager_.get_disk_macro_block_list(macro_id_list))) { - STORAGE_LOG(WARN, "fail to get disk macro block list from tmp_block_manager_", K(ret)); - } - return ret; -} - -int ObTmpTenantFileStore::get_macro_block(const int64_t block_id, ObTmpMacroBlock *&t_mblk) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpTenantFileStore has not been inited", K(ret)); - } else if (OB_FAIL(tmp_block_manager_.get_macro_block(block_id, t_mblk))) { - STORAGE_LOG(WARN, "ObTmpTenantFileStore get macro block failed", K(ret)); - } - - return ret; -} - -ObTmpTenantFileStoreHandle::ObTmpTenantFileStoreHandle() - : tenant_store_(), allocator_() -{ -} - -ObTmpTenantFileStoreHandle::~ObTmpTenantFileStoreHandle() -{ - reset(); -} -void ObTmpTenantFileStoreHandle::set_tenant_store(ObTmpTenantFileStore *tenant_store, - common::ObConcurrentFIFOAllocator *allocator) -{ - if (OB_NOT_NULL(tenant_store)) { - reset(); - tenant_store->inc_ref(); - tenant_store_ = tenant_store; - allocator_ = allocator; - } -} - -ObTmpTenantFileStoreHandle& -ObTmpTenantFileStoreHandle::operator=(const ObTmpTenantFileStoreHandle &other) -{ - if (&other != this) { - set_tenant_store(other.tenant_store_, other.allocator_); - } - return *this; -} - -bool ObTmpTenantFileStoreHandle::is_empty() const -{ - return NULL == tenant_store_; -} - -bool ObTmpTenantFileStoreHandle::is_valid() const -{ - return NULL != tenant_store_; -} - -void ObTmpTenantFileStoreHandle::reset() -{ - if (OB_NOT_NULL(tenant_store_)) { - int64_t tmp_ref = tenant_store_->dec_ref(); - if (0 == tmp_ref) { - tenant_store_->~ObTmpTenantFileStore(); - allocator_->free(tenant_store_); - } - tenant_store_ = NULL; - } -} - -ObTmpFileStore &ObTmpFileStore::get_instance() -{ - static ObTmpFileStore instance; - return instance; -} - -ObTmpFileStore::ObTmpFileStore() - : next_blk_id_(0), - tenant_file_stores_(), - lock_(), - is_inited_(false), - allocator_() -{ -} - -ObTmpFileStore::~ObTmpFileStore() -{ -} - -int ObTmpFileStore::init() -{ - int ret = OB_SUCCESS; - ObMemAttr attr = SET_USE_500(ObModIds::OB_TMP_FILE_STORE_MAP); - if (IS_INIT) { - ret = OB_INIT_TWICE; - STORAGE_LOG(WARN, "ObTmpFileStore has not been inited", K(ret)); - } else if (OB_FAIL(allocator_.init(TOTAL_LIMIT, HOLD_LIMIT, BLOCK_SIZE))) { - STORAGE_LOG(WARN, "fail to init allocator", K(ret)); - } else if (OB_FAIL(ObTmpPageCache::get_instance().init("tmp_page_cache", - TMP_FILE_PAGE_CACHE_PRIORITY))) { - STORAGE_LOG(WARN, "Fail to init tmp page cache, ", K(ret)); - } else if (OB_FAIL(ObTmpBlockCache::get_instance().init("tmp_block_cache", - TMP_FILE_BLOCK_CACHE_PRIORITY))) { - STORAGE_LOG(WARN, "Fail to init tmp tenant block cache, ", K(ret)); - } else if (OB_FAIL(tenant_file_stores_.create(STORE_HASH_BUCKET_NUM, - attr, attr))) { - STORAGE_LOG(WARN, "Fail to create tmp tenant file store map, ", K(ret)); - } else { - allocator_.set_attr(attr); - is_inited_ = true; - } - if (!is_inited_) { - destroy(); - } - return ret; -} - -int ObTmpFileStore::alloc(const int64_t dir_id, const uint64_t tenant_id, const int64_t size, - ObTmpFileExtent &extent) -{ - int ret = OB_SUCCESS; - DISABLE_SQL_MEMLEAK_GUARD; - ObTmpTenantFileStoreHandle store_handle; - if (OB_FAIL(get_store(tenant_id, store_handle))) { - STORAGE_LOG(WARN, "fail to get tmp tenant file store", K(ret), K(tenant_id)); - } else if (OB_FAIL(store_handle.get_tenant_store()->alloc(dir_id, tenant_id, size, extent))) { - STORAGE_LOG(WARN, "fail to allocate extents", K(ret), K(tenant_id), K(dir_id), K(size), - K(extent)); - } - return ret; -} - -int ObTmpFileStore::read(const uint64_t tenant_id, ObTmpBlockIOInfo &io_info, - ObTmpFileIOHandle &handle) -{ - int ret = OB_SUCCESS; - ObTmpTenantFileStoreHandle store_handle; - if (OB_FAIL(get_store(tenant_id, store_handle))) { - STORAGE_LOG(WARN, "fail to get tmp tenant file store", K(ret), K(tenant_id), K(tenant_id), - K(io_info), K(handle)); - } else if (OB_FAIL(store_handle.get_tenant_store()->read(io_info, handle))) { - STORAGE_LOG(WARN, "fail to read the extent", K(ret), K(tenant_id), K(io_info), K(handle)); - } - return ret; -} - -int ObTmpFileStore::write(const uint64_t tenant_id, const ObTmpBlockIOInfo &io_info) -{ - int ret = OB_SUCCESS; - ObTmpTenantFileStoreHandle store_handle; - if (OB_FAIL(get_store(tenant_id, store_handle))) { - STORAGE_LOG(WARN, "fail to get tmp tenant file store", K(ret), K(tenant_id), K(io_info)); - } else if (OB_FAIL(store_handle.get_tenant_store()->write(io_info))) { - STORAGE_LOG(WARN, "fail to write the extent", K(ret), K(tenant_id), K(io_info)); - } - return ret; -} - -int ObTmpFileStore::wash_block(const uint64_t tenant_id, const int64_t block_id, - ObTmpTenantMemBlockManager::ObIOWaitInfoHandle &handle) -{ - int ret = OB_SUCCESS; - ObTmpTenantFileStoreHandle store_handle; - if (OB_FAIL(get_store(tenant_id, store_handle))) { - STORAGE_LOG(WARN, "fail to get tmp tenant file store", K(ret), K(tenant_id), K(block_id)); - } else if (OB_FAIL(store_handle.get_tenant_store()->wash_block(block_id, handle))) { - STORAGE_LOG(WARN, "fail to write the extent", K(ret), K(tenant_id), K(block_id)); - } - return ret; -} - -int ObTmpFileStore::wait_write_finish(const uint64_t tenant_id, const int64_t block_id, const int64_t timeout_ms) -{ - int ret = OB_SUCCESS; - ObTmpTenantFileStoreHandle store_handle; - if (OB_FAIL(get_store(tenant_id, store_handle))) { - STORAGE_LOG(WARN, "fail to get tmp tenant file store", K(ret), K(tenant_id)); - } else if (OB_FAIL(store_handle.get_tenant_store()->wait_write_finish(block_id, timeout_ms))) { - STORAGE_LOG(WARN, "fail to write the extent", K(ret), K(tenant_id)); - } - return ret; -} - -int ObTmpFileStore::sync_block(const uint64_t tenant_id, const int64_t block_id, - ObTmpTenantMemBlockManager::ObIOWaitInfoHandle &handle) -{ - int ret = OB_SUCCESS; - ObTmpTenantFileStoreHandle store_handle; - if (OB_FAIL(get_store(tenant_id, store_handle))) { - STORAGE_LOG(WARN, "fail to get tmp tenant file store", K(ret), K(tenant_id)); - } else if (OB_FAIL(store_handle.get_tenant_store()->sync_block(block_id, handle))) { - STORAGE_LOG(WARN, "fail to write the extent", K(ret), K(tenant_id)); - } - return ret; -} - -int ObTmpFileStore::inc_block_cache_num(const uint64_t tenant_id, const int64_t num) -{ - int ret = OB_SUCCESS; - ObTmpTenantFileStoreHandle store_handle; - if (OB_FAIL(get_store(tenant_id, store_handle))) { - STORAGE_LOG(WARN, "fail to get tmp tenant file store", K(ret), K(tenant_id)); - } else { - store_handle.get_tenant_store()->inc_block_cache_num(num); - } - return ret; -} - -int ObTmpFileStore::dec_block_cache_num(const uint64_t tenant_id, const int64_t num) -{ - int ret = OB_SUCCESS; - ObTmpTenantFileStoreHandle store_handle; - if (OB_FAIL(get_store(tenant_id, store_handle))) { - STORAGE_LOG(WARN, "fail to get tmp tenant file store", K(ret), K(tenant_id)); - } else { - store_handle.get_tenant_store()->dec_block_cache_num(num); - } - return ret; -} - -int ObTmpFileStore::inc_page_cache_num(const uint64_t tenant_id, const int64_t num) -{ - int ret = OB_SUCCESS; - ObTmpTenantFileStoreHandle store_handle; - if (OB_FAIL(get_store(tenant_id, store_handle))) { - STORAGE_LOG(WARN, "fail to get tmp tenant file store", K(ret), K(tenant_id)); - } else { - store_handle.get_tenant_store()->inc_page_cache_num(num); - } - return ret; -} - -int ObTmpFileStore::dec_page_cache_num(const uint64_t tenant_id, const int64_t num) -{ - int ret = OB_SUCCESS; - ObTmpTenantFileStoreHandle store_handle; - if (OB_FAIL(get_store(tenant_id, store_handle))) { - STORAGE_LOG(WARN, "fail to get tmp tenant file store", K(ret), K(tenant_id)); - } else { - store_handle.get_tenant_store()->dec_page_cache_num(num); - } - return ret; -} - -int ObTmpFileStore::free(const uint64_t tenant_id, ObTmpFileExtent *extent) -{ - int ret = OB_SUCCESS; - ObTmpTenantFileStoreHandle store_handle; - if (OB_FAIL(get_store(tenant_id, store_handle))) { - STORAGE_LOG(WARN, "fail to get tmp tenant file store", K(ret), K(tenant_id), K(*extent)); - } else if (OB_FAIL(store_handle.get_tenant_store()->free(extent))) { - STORAGE_LOG(WARN, "fail to free extents", K(ret), K(tenant_id), K(*extent)); - } - return ret; -} - -int ObTmpFileStore::free(const uint64_t tenant_id, const int64_t block_id, - const int32_t start_page_id, const int32_t page_nums) -{ - int ret = OB_SUCCESS; - ObTmpTenantFileStoreHandle store_handle; - if (OB_FAIL(get_store(tenant_id, store_handle))) { - STORAGE_LOG(WARN, "fail to get tmp tenant file store", K(ret), K(tenant_id)); - } else if (OB_FAIL(store_handle.get_tenant_store()->free(block_id, start_page_id, page_nums))) { - STORAGE_LOG(WARN, "fail to free", K(ret), K(tenant_id), K(block_id), K(start_page_id), - K(page_nums)); - } - return ret; -} - -int ObTmpFileStore::free_tenant_file_store(const uint64_t tenant_id) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(tenant_file_stores_.erase_refactored(tenant_id))) { - if (OB_HASH_NOT_EXIST == ret) { - ret = OB_ENTRY_NOT_EXIST; - } else { - STORAGE_LOG(WARN, "fail to erase tmp tenant file store", K(ret), K(tenant_id)); - } - } - return ret; -} - -int ObTmpFileStore::get_macro_block(const int64_t tenant_id, const int64_t block_id, ObTmpMacroBlock *&t_mblk) -{ - int ret = OB_SUCCESS; - ObTmpTenantFileStoreHandle store_handle; - if (OB_FAIL(get_store(tenant_id, store_handle))) { - STORAGE_LOG(WARN, "fail to get tmp tenant file store", K(ret), K(tenant_id)); - } else if (OB_FAIL(store_handle.get_tenant_store()->get_macro_block(block_id, t_mblk))) { - STORAGE_LOG(WARN, "fail to free", K(ret), K(tenant_id), K(block_id)); - } - return ret; -} - -int ObTmpFileStore::get_macro_block_list(common::ObIArray ¯o_id_list) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFileStore has not been inited", K(ret)); - } else { - macro_id_list.reset(); - TenantFileStoreMap::iterator iter; - ObTmpTenantFileStore *tmp = NULL; - for (iter = tenant_file_stores_.begin(); OB_SUCC(ret) && iter != tenant_file_stores_.end(); - ++iter) { - if (OB_ISNULL(tmp = iter->second.get_tenant_store())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "fail to iterate tmp tenant file store", K(ret)); - } else if (OB_FAIL(tmp->get_disk_macro_block_list(macro_id_list))){ - STORAGE_LOG(WARN, "fail to get list of tenant macro block in disk", K(ret)); - } - } - } - return ret; -} - -int ObTmpFileStore::get_macro_block_list(ObIArray &tmp_block_cnt_pairs) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFileStore has not been inited", K(ret)); - } else { - tmp_block_cnt_pairs.reset(); - TenantFileStoreMap::iterator iter; - ObTmpTenantFileStore *tmp = NULL; - for (iter = tenant_file_stores_.begin(); OB_SUCC(ret) && iter != tenant_file_stores_.end(); - ++iter) { - int64_t macro_id_count = 0; - TenantTmpBlockCntPair pair; - if (OB_ISNULL(tmp = iter->second.get_tenant_store())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "fail to iterate tmp tenant file store", K(ret)); - } else if (OB_FAIL(tmp->get_disk_macro_block_count(macro_id_count))){ - STORAGE_LOG(WARN, "fail to get list of tenant macro block in disk", K(ret)); - } else if (OB_FAIL(pair.init(iter->first, macro_id_count))) { - STORAGE_LOG(WARN, "fail to init tenant tmp block count pair", K(ret), "tenant id", - iter->first, "macro block count", macro_id_count); - } else if (OB_FAIL(tmp_block_cnt_pairs.push_back(pair))) { - STORAGE_LOG(WARN, "fail to push back tmp_block_cnt_pairs", K(ret), K(pair)); - } - } - } - return ret; -} - -int ObTmpFileStore::get_all_tenant_id(common::ObIArray &tenant_ids) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFileStore has not been inited", K(ret)); - } else { - tenant_ids.reset(); - TenantFileStoreMap::iterator iter; - SpinRLockGuard guard(lock_); - for (iter = tenant_file_stores_.begin(); OB_SUCC(ret) && iter != tenant_file_stores_.end(); - ++iter) { - if (OB_ISNULL(iter->second.get_tenant_store())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "fail to iterate tmp tenant file store", K(ret)); - } else if (OB_FAIL(tenant_ids.push_back(iter->first))) { - STORAGE_LOG(WARN, "fail to push back tmp_block_cnt_pairs", K(ret)); - } - } - } - return ret; -} - -int ObTmpFileStore::get_store(const uint64_t tenant_id, ObTmpTenantFileStoreHandle &handle) -{ - int ret = OB_SUCCESS; - DISABLE_SQL_MEMLEAK_GUARD; - void *buf = NULL; - handle.reset(); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFileStore has not been inited", K(ret), K(tenant_id)); - } else { - SpinRLockGuard guard(lock_); - if (OB_FAIL(tenant_file_stores_.get_refactored(tenant_id, handle))) { - if (OB_HASH_NOT_EXIST == ret) { - STORAGE_LOG(DEBUG, "ObTmpFileStore get tenant store failed", K(ret), K(tenant_id)); - } else { - STORAGE_LOG(WARN, "ObTmpFileStore get tenant store failed", K(ret), K(tenant_id)); - } - } - } - - if (OB_UNLIKELY(OB_HASH_NOT_EXIST == ret)) { - SpinWLockGuard guard(lock_); - if (OB_FAIL(tenant_file_stores_.get_refactored(tenant_id, handle))) { - if (OB_HASH_NOT_EXIST == ret) { - ObTmpTenantFileStore *store = NULL; - if (OB_ISNULL(buf = allocator_.alloc(sizeof(ObTmpTenantFileStore)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - STORAGE_LOG(WARN, "fail to alloc a buf", K(ret), K(tenant_id)); - } else if (OB_ISNULL(store = new (buf) ObTmpTenantFileStore())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "fail to new a ObTmpTenantFileStore", K(ret), K(tenant_id)); - } else if (OB_FAIL(store->init(tenant_id))) { - store->~ObTmpTenantFileStore(); - allocator_.free(store); - store = NULL; - STORAGE_LOG(WARN, "fail to init ObTmpTenantFileStore", K(ret), K(tenant_id)); - } else if (FALSE_IT(handle.set_tenant_store(store, &allocator_))) { - } else if (OB_FAIL(tenant_file_stores_.set_refactored(tenant_id, handle))) { - STORAGE_LOG(WARN, "fail to set tenant_file_stores_", K(ret), K(tenant_id)); - } - } else { - STORAGE_LOG(WARN, "fail to get tmp tenant file store", K(ret), K(tenant_id)); - } - } - } - if (OB_SUCC(ret)) { - if (OB_UNLIKELY(!handle.is_valid())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "unexpected error, invalid tenant file store handle", K(ret), K(handle)); - } - } - return ret; -} - -int64_t ObTmpFileStore::get_next_blk_id() -{ - int64_t next_blk_id = -1; - int64_t old_val = ATOMIC_LOAD(&next_blk_id_); - int64_t new_val = 0; - bool finish = false; - while (!finish) { - new_val = (old_val + 1) % INT64_MAX; - next_blk_id = new_val; - finish = (old_val == (new_val = ATOMIC_VCAS(&next_blk_id_, old_val, new_val))); - old_val = new_val; - } - return next_blk_id; -} - -int ObTmpFileStore::get_tenant_extent_allocator(const int64_t tenant_id, common::ObIAllocator *&allocator) -{ - int ret = OB_SUCCESS; - ObTmpTenantFileStoreHandle store_handle; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObTmpFileStore has not been inited", K(ret), K(tenant_id)); - } else if (OB_FAIL(get_store(tenant_id, store_handle))) { - STORAGE_LOG(WARN, "fail to get tmp tenant file store", K(ret), K(tenant_id)); - } else { - allocator = &(store_handle.get_tenant_store()->get_extent_allocator()); - } - return ret; -} - -void ObTmpFileStore::destroy() -{ - ObTmpPageCache::get_instance().destroy(); - tenant_file_stores_.destroy(); - allocator_.destroy(); - ObTmpBlockCache::get_instance().destroy(); - is_inited_ = false; -} - -} // end namespace blocksstable -} // end namespace oceanbase diff --git a/src/storage/blocksstable/ob_tmp_file_store.h b/src/storage/blocksstable/ob_tmp_file_store.h deleted file mode 100644 index c0455158e..000000000 --- a/src/storage/blocksstable/ob_tmp_file_store.h +++ /dev/null @@ -1,393 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#ifndef OCEANBASE_STORAGE_BLOCKSSTABLE_OB_TMP_FILE_STORE_H_ -#define OCEANBASE_STORAGE_BLOCKSSTABLE_OB_TMP_FILE_STORE_H_ - -#include "storage/blocksstable/ob_macro_block_common_header.h" -#include "storage/blocksstable/ob_macro_block_handle.h" -#include "storage/blocksstable/ob_block_manager.h" -#include "storage/blocksstable/ob_tmp_file_cache.h" - -namespace oceanbase -{ -namespace blocksstable -{ - -class ObTmpFile; -class ObTmpFileExtent; -class ObTmpFileIOHandle; -class ObTmpTenantMacroBlockManager; -struct ObTmpBlockValueHandle; -class ObTmpTenantBlockCache; -class ObTmpPageCache; - -typedef common::ObSEArray ExtentArray; - -struct ObTmpFileArea final -{ -public: - ObTmpFileArea(const uint8_t &start_page_id, const uint8_t &page_nums) - : start_page_id_(start_page_id), page_nums_(page_nums), next_(NULL) {} - ~ObTmpFileArea() - { - start_page_id_ = 0; - page_nums_ = 0; - next_ = NULL; - } - TO_STRING_KV(K_(start_page_id), K_(page_nums)); - uint8_t start_page_id_; - uint8_t page_nums_; - ObTmpFileArea *next_; -} __attribute__((packed, __aligned__(1)));; - -class ObTmpFilePageBuddy final -{ -public: - ObTmpFilePageBuddy(); - ~ObTmpFilePageBuddy(); - int init(common::ObIAllocator &allocator); - void destroy(); - int alloc_all_pages(); - int alloc(const uint8_t page_nums, uint8_t &start_page_id, uint8_t &alloced_page_nums); - void free(const int32_t start_page_id, const int32_t page_nums); - OB_INLINE uint8_t get_max_cont_page_nums() const { return max_cont_page_nums_; } - bool is_inited() { return is_inited_; } - bool is_empty() const; - int64_t to_string(char* buf, const int64_t buf_len) const; - - static const uint8_t MAX_PAGE_NUMS = 252; // 2^MAX_ORDER - 2^MIN_ORDER, uint8_t is only for < 256 - -private: - void free_align(const int32_t start_page_id, const int32_t page_nums, ObTmpFileArea *&area); - ObTmpFileArea *find_buddy(const int32_t page_nums, const int32_t start_page_id); - -private: - static const uint8_t MIN_ORDER = 2; - static const uint8_t MAX_ORDER = 8; - bool is_inited_; - uint8_t max_cont_page_nums_; - ObTmpFileArea *buf_; - common::ObIAllocator *allocator_; - ObTmpFileArea *free_area_[ObTmpFilePageBuddy::MAX_ORDER]; - DISALLOW_COPY_AND_ASSIGN(ObTmpFilePageBuddy); -}; - -struct ObTmpFileMacroBlockHeader final -{ -public: - ObTmpFileMacroBlockHeader(); - ~ObTmpFileMacroBlockHeader() = default; - bool is_valid() const; - int serialize(char *buf, const int64_t buf_len, int64_t &pos) const; - int deserialize(const char *buf, const int64_t data_len, int64_t &pos); - static int64_t get_serialize_size() { return sizeof(ObTmpFileMacroBlockHeader); } - void reset(); - - TO_STRING_KV(K_(version), K_(magic), K_(block_id), K_(dir_id), K_(tenant_id), K_(free_page_nums)); -private: - static const int32_t TMP_FILE_MACRO_BLOCK_HEADER_VERSION = 1; - static const int32_t TMP_FILE_MACRO_BLOCK_HEADER_MAGIC = 20720; -public: - int32_t version_; - int32_t magic_; - int64_t block_id_; - int64_t dir_id_; - uint64_t tenant_id_; - int64_t free_page_nums_; -}; - -struct ObTmpBlockIOInfo final -{ -public: - ObTmpBlockIOInfo() - : block_id_(0), offset_(0), size_(0), io_timeout_ms_(DEFAULT_IO_WAIT_TIME_MS), tenant_id_(0), - buf_(NULL), io_desc_(), macro_block_id_() {} - ObTmpBlockIOInfo(const int64_t block_id, const int64_t offset, const int64_t size, - const uint64_t tenant_id, const MacroBlockId macro_block_id, char *buf, - const common::ObIOFlag io_desc) - : block_id_(block_id), offset_(offset), size_(size), tenant_id_(tenant_id), - buf_(buf), io_desc_(io_desc), macro_block_id_(macro_block_id) {} - TO_STRING_KV(K_(block_id), K_(offset), K_(size), K_(io_timeout_ms), K_(tenant_id), K_(macro_block_id), KP_(buf), - K_(io_desc)); - int64_t block_id_; - int64_t offset_; - int64_t size_; - int64_t io_timeout_ms_; - uint64_t tenant_id_; - char *buf_; - common::ObIOFlag io_desc_; - MacroBlockId macro_block_id_; -}; - -class ObTmpMacroBlock final -{ -public: - enum BlockStatus: uint8_t { - MEMORY = 0, - WASHING, - DISKED, - MAX, - }; - ObTmpMacroBlock(); - ~ObTmpMacroBlock(); - int init(const int64_t block_id, const int64_t dir_id, const uint64_t tenant_id, - common::ObIAllocator &allocator); - void destroy(); - int alloc_all_pages(ObTmpFileExtent &extent); - int alloc(const uint8_t page_nums, ObTmpFileExtent &extent); - int free(ObTmpFileExtent &extent); - int free(const int32_t start_page_id, const int32_t page_nums); - OB_INLINE void set_buffer(char *buf) { buffer_ = buf; } - OB_INLINE char *get_buffer() { return buffer_; } - OB_INLINE uint8_t get_max_cont_page_nums() const { return page_buddy_.get_max_cont_page_nums(); } - OB_INLINE uint8_t get_free_page_nums() const { return tmp_file_header_.free_page_nums_; } - int64_t get_used_page_nums() const; - int get_block_cache_handle(ObTmpBlockValueHandle &handle); - int get_wash_io_info(ObTmpBlockIOInfo &info); - void set_io_desc(const common::ObIOFlag &io_desc); - int check_and_set_status(const BlockStatus old_block_status, const BlockStatus new_block_status); - OB_INLINE int get_block_status() const { return ATOMIC_LOAD(&block_status_); } - OB_INLINE bool is_memory() const { return ATOMIC_LOAD(&block_status_) == MEMORY; } - OB_INLINE bool is_disked() const { return ATOMIC_LOAD(&block_status_) == DISKED; } - OB_INLINE bool is_washing() const { return ATOMIC_LOAD(&block_status_) == WASHING; } - OB_INLINE bool is_inited() const { return is_inited_; } - static int64_t get_default_page_size() { return DEFAULT_PAGE_SIZE; } - static int64_t calculate_offset(const int64_t page_start_id, const int64_t offset) - { - return page_start_id * ObTmpMacroBlock::DEFAULT_PAGE_SIZE + offset; - } - static int64_t get_header_padding() - { - return 4 * DEFAULT_PAGE_SIZE; - } - OB_INLINE int64_t get_block_id() const { return tmp_file_header_.block_id_; } - OB_INLINE const ObTmpFileMacroBlockHeader &get_tmp_block_header() const { return tmp_file_header_; } - OB_INLINE uint64_t get_tenant_id() const { return tmp_file_header_.tenant_id_; } - OB_INLINE int64_t get_dir_id() const { return tmp_file_header_.dir_id_; } - OB_INLINE const MacroBlockId& get_macro_block_id() const { return macro_block_handle_.get_macro_id(); } - OB_INLINE ObMacroBlockHandle &get_macro_block_handle() { return macro_block_handle_; } - OB_INLINE int64_t get_alloc_time() const { return alloc_time_; } - OB_INLINE int64_t get_access_time() const { return ATOMIC_LOAD(&access_time_); } - OB_INLINE double get_wash_score(int64_t cur_time) const { - if (get_used_page_nums() == ObTmpFilePageBuddy::MAX_PAGE_NUMS) { - return INT64_MAX; - } - return (double) get_used_page_nums() * (cur_time - get_alloc_time()) / (get_access_time() - get_alloc_time()); - } - common::ObIArray &get_extents() { return using_extents_; } - ObTmpBlockValueHandle &get_handle() { return handle_; } - bool is_empty() const { return page_buddy_.is_empty(); } - int seal(bool &is_sealed); - int is_extents_closed(bool &is_extents_closed); - int give_back_buf_into_cache(const bool is_wash = false); - - TO_STRING_KV(KP_(buffer), K_(page_buddy), K_(handle), K_(macro_block_handle), K_(tmp_file_header), - K_(io_desc), K_(block_status), K_(is_inited), K_(alloc_time), K_(access_time)); -private: - bool is_sealed() const { return ATOMIC_LOAD(&is_sealed_); } -private: - static const int64_t DEFAULT_PAGE_SIZE; - char *buffer_; - ObTmpFilePageBuddy page_buddy_; - ObTmpBlockValueHandle handle_; - ExtentArray using_extents_; - ObMacroBlockHandle macro_block_handle_; - ObTmpFileMacroBlockHeader tmp_file_header_; - common::ObIOFlag io_desc_; - common::SpinRWLock lock_; - BlockStatus block_status_; - bool is_sealed_; - bool is_inited_; - int64_t alloc_time_; - int64_t access_time_; - DISALLOW_COPY_AND_ASSIGN(ObTmpMacroBlock); -}; - -class ObTmpTenantMacroBlockManager final -{ -public: - ObTmpTenantMacroBlockManager(); - ~ObTmpTenantMacroBlockManager(); - int init(const uint64_t tenant_id, common::ObIAllocator &allocator); - void destroy(); - int alloc_macro_block(const int64_t dir_id, const uint64_t tenant_id, ObTmpMacroBlock *&t_mblk); - int free_macro_block(const int64_t block_id); - int get_macro_block(const int64_t block_id, ObTmpMacroBlock *&t_mblk); - int get_disk_macro_block_count(int64_t &count) const; - int get_disk_macro_block_list(common::ObIArray ¯o_id_list); - void print_block_usage(); - -private: - static const uint64_t MBLK_HASH_BUCKET_NUM = 10243L; - typedef common::hash::ObHashMap - TmpMacroBlockMap; - common::ObIAllocator *allocator_; - TmpMacroBlockMap blocks_; // all of block meta. - bool is_inited_; - DISALLOW_COPY_AND_ASSIGN(ObTmpTenantMacroBlockManager); -}; - -class ObTmpTenantFileStore final -{ -public: - ObTmpTenantFileStore(); - ~ObTmpTenantFileStore(); - int init(const uint64_t tenant_id); - void destroy(); - int alloc(const int64_t dir_id, const uint64_t tenant_id, const int64_t alloc_size, - ObTmpFileExtent &extent); - int free(ObTmpFileExtent *extent); - int free(const int64_t block_id, const int32_t start_page_id, const int32_t page_nums); - int read(ObTmpBlockIOInfo &io_info, ObTmpFileIOHandle &handle); - int write(const ObTmpBlockIOInfo &io_info); - int wash_block(const int64_t block_id, ObTmpTenantMemBlockManager::ObIOWaitInfoHandle &handle); - void refresh_memory_limit(const uint64_t tenant_id); - int sync_block(const int64_t block_id, ObTmpTenantMemBlockManager::ObIOWaitInfoHandle &handle); - int wait_write_finish(const int64_t block_id, const int64_t timeout_ms); - int get_disk_macro_block_count(int64_t &count) const; - int get_disk_macro_block_list(common::ObIArray ¯o_id_list); - int get_macro_block(const int64_t block_id, ObTmpMacroBlock *&t_mblk); - // use io_allocator_ to allocate tenant extent memory. - common::ObIAllocator &get_extent_allocator() { return allocator_; } - void print_block_usage() { tmp_block_manager_.print_block_usage(); } - OB_INLINE void inc_page_cache_num(const int64_t num) { - ATOMIC_FAA(&page_cache_num_, num); - }; - OB_INLINE void dec_page_cache_num(const int64_t num) { - ATOMIC_FAS(&page_cache_num_, num); - }; - OB_INLINE void inc_block_cache_num(const int64_t num) { - ATOMIC_FAA(&block_cache_num_, num); - }; - OB_INLINE void dec_block_cache_num(const int64_t num) { - ATOMIC_FAS(&block_cache_num_, num); - }; - OB_INLINE int64_t get_page_cache_num() const { return ATOMIC_LOAD(&page_cache_num_); } - OB_INLINE int64_t get_block_cache_num() const { return ATOMIC_LOAD(&block_cache_num_); } - void inc_ref(); - int64_t dec_ref(); - -private: - int read_page(ObTmpMacroBlock *block, ObTmpBlockIOInfo &io_info, ObTmpFileIOHandle &handle); - int free_extent(ObTmpFileExtent *extent); - int free_extent(const int64_t block_id, const int32_t start_page_id, const int32_t page_nums); - int free_macro_block(ObTmpMacroBlock *&t_mblk); - int alloc_macro_block(const int64_t dir_id, const uint64_t tenant_id, ObTmpMacroBlock *&t_mblk); - int64_t get_memory_limit(const uint64_t tenant_id); - int wait_write_io_finish_if_need(); - -private: - static const uint64_t IO_LIMIT = 4 * 1024L * 1024L * 1024L; - static const uint64_t TOTAL_LIMIT = 15 * 1024L * 1024L * 1024L; - static const uint64_t HOLD_LIMIT = 8 * 1024L * 1024L; - static const uint64_t REFRESH_CONFIG_INTERVAL = 5 * 60 * 1000 * 1000L; // 5min - static const uint64_t BLOCK_SIZE = common::OB_MALLOC_MIDDLE_BLOCK_SIZE; - static constexpr double DEFAULT_PAGE_IO_MERGE_RATIO = 0.5; - - bool is_inited_; - int64_t page_cache_num_; - int64_t block_cache_num_; - volatile int64_t ref_cnt_; - ObTmpPageCache *page_cache_; - common::SpinRWLock lock_; - common::ObConcurrentFIFOAllocator allocator_; - common::ObFIFOAllocator io_allocator_; - ObTmpTenantMacroBlockManager tmp_block_manager_; - ObTmpTenantMemBlockManager tmp_mem_block_manager_; - int64_t last_access_tenant_config_ts_; - int64_t last_meta_mem_limit_; - - DISALLOW_COPY_AND_ASSIGN(ObTmpTenantFileStore); -}; - -struct ObTmpTenantFileStoreHandle final -{ -public: - ObTmpTenantFileStoreHandle(); - ~ObTmpTenantFileStoreHandle(); - ObTmpTenantFileStoreHandle(const ObTmpTenantFileStoreHandle &other); - ObTmpTenantFileStoreHandle &operator=(const ObTmpTenantFileStoreHandle &other); - void set_tenant_store(ObTmpTenantFileStore *store, common::ObConcurrentFIFOAllocator *allocator); - bool is_empty() const; - bool is_valid() const; - void reset(); - OB_INLINE ObTmpTenantFileStore* get_tenant_store() const { return tenant_store_; } - TO_STRING_KV(KP_(tenant_store), KP_(allocator)); -private: - ObTmpTenantFileStore *tenant_store_; - common::ObConcurrentFIFOAllocator *allocator_; -}; - -class ObTmpFileStore final -{ -public: - typedef common::hash::HashMapPair TenantTmpBlockCntPair; - - static ObTmpFileStore &get_instance(); - int init(); - void destroy(); - - int alloc(const int64_t dir_id, const uint64_t tenant_id, const int64_t size, - ObTmpFileExtent &extent); - int read(const uint64_t tenant_id, ObTmpBlockIOInfo &io_info, ObTmpFileIOHandle &handle); - int write(const uint64_t tenant_id, const ObTmpBlockIOInfo &io_info); - int wash_block(const uint64_t tenant_id, const int64_t block_id, - ObTmpTenantMemBlockManager::ObIOWaitInfoHandle &handle); - int sync_block(const uint64_t tenant_id, const int64_t block_id, - ObTmpTenantMemBlockManager::ObIOWaitInfoHandle &handle); - int wait_write_finish(const uint64_t tenant_id, const int64_t block_id, const int64_t timeout_ms); - int free(const uint64_t tenant_id, ObTmpFileExtent *extent); - int free(const uint64_t tenant_id, const int64_t block_id, const int32_t start_page_id, - const int32_t page_nums); - int free_tenant_file_store(const uint64_t tenant_id); - int get_macro_block(const int64_t tenant_id, const int64_t block_id, ObTmpMacroBlock *&t_mblk); - int get_macro_block_list(common::ObIArray ¯o_id_list); - int get_macro_block_list(common::ObIArray &tmp_block_cnt_pairs); - int get_all_tenant_id(common::ObIArray &tenant_ids); - int64_t get_next_blk_id(); - int get_tenant_extent_allocator(const int64_t tenant_id, common::ObIAllocator *&allocator); - - static int64_t get_block_size() - { - return ObTmpFilePageBuddy::MAX_PAGE_NUMS * ObTmpMacroBlock::get_default_page_size(); - } - int inc_page_cache_num(const uint64_t tenant_id, const int64_t num); - int dec_page_cache_num(const uint64_t tenant_id, const int64_t num); - int inc_block_cache_num(const uint64_t tenant_id, const int64_t num); - int dec_block_cache_num(const uint64_t tenant_id, const int64_t num); -private: - ObTmpFileStore(); - ~ObTmpFileStore(); - int get_store(const uint64_t tenant_id, ObTmpTenantFileStoreHandle &handle); - -private: - static const uint64_t STORE_HASH_BUCKET_NUM = 1543L; - static const int64_t TOTAL_LIMIT = 512L * 1024L * 1024L; - static const int64_t HOLD_LIMIT = 8 * 1024L * 1024L; - static const int64_t BLOCK_SIZE = common::OB_MALLOC_NORMAL_BLOCK_SIZE; - static const int TMP_FILE_PAGE_CACHE_PRIORITY = 1; - static const int TMP_FILE_BLOCK_CACHE_PRIORITY = 1; - typedef common::hash::ObHashMap TenantFileStoreMap; - int64_t next_blk_id_; - TenantFileStoreMap tenant_file_stores_; - common::SpinRWLock lock_; - bool is_inited_; - common::ObConcurrentFIFOAllocator allocator_; -}; - -#define OB_TMP_FILE_STORE (::oceanbase::blocksstable::ObTmpFileStore::get_instance()) - -} // end namespace blocksstable -} // end namespace oceanbase -#endif // OCEANBASE_STORAGE_BLOCKSSTABLE_OB_TMP_FILE_STORE_H_ diff --git a/src/storage/direct_load/ob_direct_load_tmp_file.cpp b/src/storage/direct_load/ob_direct_load_tmp_file.cpp index af2cc280b..ec64853cd 100644 --- a/src/storage/direct_load/ob_direct_load_tmp_file.cpp +++ b/src/storage/direct_load/ob_direct_load_tmp_file.cpp @@ -181,9 +181,9 @@ ObDirectLoadTmpFileIOHandle::~ObDirectLoadTmpFileIOHandle() void ObDirectLoadTmpFileIOHandle::reset() { tmp_file_ = nullptr; + file_io_handle_.reset(); file_handle_.reset(); io_info_.reset(); - file_io_handle_.reset(); is_cancel_ = false; } @@ -207,7 +207,6 @@ int ObDirectLoadTmpFileIOHandle::open(const ObDirectLoadTmpFileHandle &file_hand LOG_WARN("fail to assign file handle", KR(ret)); } else { tmp_file_ = tmp_file; - io_info_.tenant_id_ = MTL_ID(); io_info_.dir_id_ = tmp_file_->get_file_id().dir_id_; io_info_.fd_ = tmp_file_->get_file_id().fd_; io_info_.io_desc_.set_resource_group_id(THIS_WORKER.get_group_id()); @@ -424,7 +423,7 @@ int ObDirectLoadTmpFileIOHandle::wait() while (OB_SUCC(ret)) { if (OB_FAIL(check_status())) { LOG_WARN("fail to check status", KR(ret)); - } else if (OB_FAIL(file_io_handle_.wait())) { + } else if (file_io_handle_.is_valid() && OB_FAIL(file_io_handle_.wait())) { LOG_WARN("fail to wait io finish", KR(ret)); if (OB_LIKELY(is_retry_err(ret))) { if (++retry_cnt <= MAX_RETRY_CNT) { @@ -441,34 +440,6 @@ int ObDirectLoadTmpFileIOHandle::wait() return ret; } -int ObDirectLoadTmpFileIOHandle::seek(const ObDirectLoadTmpFileHandle &file_handle, int64_t offset, - int whence) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!file_handle.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(file_handle)); - } else if (OB_FAIL(FILE_MANAGER_INSTANCE_V2.seek(file_handle.get_file()->get_file_id().fd_, - offset, whence))) { - LOG_WARN("fail to seek tmp file", KR(ret), K(file_handle), K(offset), K(whence)); - } - return ret; -} - -int ObDirectLoadTmpFileIOHandle::sync(const ObDirectLoadTmpFileHandle &file_handle, - int64_t timeout_ms) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!file_handle.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(file_handle)); - } else if (OB_FAIL(FILE_MANAGER_INSTANCE_V2.sync(file_handle.get_file()->get_file_id().fd_, - timeout_ms))) { - LOG_WARN("fail to sync tmp file", KR(ret), K(file_handle)); - } - return ret; -} - /** * ObDirectLoadTmpFileManager */ diff --git a/src/storage/direct_load/ob_direct_load_tmp_file.h b/src/storage/direct_load/ob_direct_load_tmp_file.h index 95d203716..b6e180c30 100644 --- a/src/storage/direct_load/ob_direct_load_tmp_file.h +++ b/src/storage/direct_load/ob_direct_load_tmp_file.h @@ -12,7 +12,7 @@ #pragma once #include "observer/table_load/ob_table_load_object_allocator.h" -#include "storage/blocksstable/ob_tmp_file.h" +#include "storage/tmp_file/ob_tmp_file_manager.h" namespace oceanbase { @@ -110,8 +110,6 @@ public: int aio_write(char *buf, int64_t size); int wait(); OB_INLINE void cancel() { is_cancel_ = true; } - static int seek(const ObDirectLoadTmpFileHandle &file_handle, int64_t offset, int whence); - static int sync(const ObDirectLoadTmpFileHandle &file_handle, int64_t timeout_ms); static bool is_retry_err(int ret_code) { return OB_TIMEOUT == ret_code; } TO_STRING_KV(K_(file_handle), K_(io_info)); private: @@ -119,8 +117,8 @@ private: private: ObDirectLoadTmpFileHandle file_handle_; ObDirectLoadTmpFile *tmp_file_; - blocksstable::ObTmpFileIOInfo io_info_; - blocksstable::ObTmpFileIOHandle file_io_handle_; + tmp_file::ObTmpFileIOInfo io_info_; + tmp_file::ObTmpFileIOHandle file_io_handle_; bool is_cancel_; DISABLE_COPY_ASSIGN(ObDirectLoadTmpFileIOHandle); }; diff --git a/src/storage/ob_disk_usage_reporter.cpp b/src/storage/ob_disk_usage_reporter.cpp index 69c4037ee..19dd2d092 100644 --- a/src/storage/ob_disk_usage_reporter.cpp +++ b/src/storage/ob_disk_usage_reporter.cpp @@ -13,7 +13,7 @@ #include "storage/ob_disk_usage_reporter.h" #include "share/ob_disk_usage_table_operator.h" -#include "storage/blocksstable/ob_tmp_file_store.h" +#include "storage/tmp_file/ob_tmp_file_manager.h" #include "observer/omt/ob_multi_tenant.h" #include "share/rc/ob_tenant_base.h" @@ -33,6 +33,7 @@ namespace oceanbase using namespace common; using namespace share; using namespace logservice; +using namespace tmp_file; namespace storage { @@ -399,19 +400,31 @@ int ObDiskUsageReportTask::count_tenant_tmp() int ret = OB_SUCCESS; ObDiskUsageReportKey report_key; int64_t macro_block_cnt = 0; - common::ObArray tenant_block_cnt_pairs; - if (OB_FAIL(OB_TMP_FILE_STORE.get_macro_block_list(tenant_block_cnt_pairs))) { - STORAGE_LOG(WARN, "failed to get tenant tmp macro block list", K(ret)); + common::ObArray tenant_ids; + + if (OB_FAIL(GCTX.omt_->get_mtl_tenant_ids(tenant_ids))) { + STORAGE_LOG(WARN, "fail to get_mtl_tenant_ids", KR(ret)); } else { report_key.file_type_ = ObDiskReportFileType::TENANT_TMP_DATA; - for (int64_t i = 0; OB_SUCC(ret) && i < tenant_block_cnt_pairs.count(); ++i) { - report_key.tenant_id_ = tenant_block_cnt_pairs.at(i).first; - macro_block_cnt = tenant_block_cnt_pairs.at(i).second; - int64_t tenant_tmp_size = macro_block_cnt * common::OB_DEFAULT_MACRO_BLOCK_SIZE; - if (OB_FAIL(result_map_.set_refactored(report_key, std::make_pair(tenant_tmp_size, tenant_tmp_size), 1))) { - STORAGE_LOG(WARN, "failed to set tenant tmp usage into result map", K(ret), K(report_key), K(macro_block_cnt)); + for (int64_t i = 0; OB_SUCC(ret) && i < tenant_ids.size(); i++) { + if (GCTX.omt_->is_available_tenant(tenant_ids.at(i))) { + MTL_SWITCH(tenant_ids.at(i)) { + ObTenantTmpFileManager* tmp_file_manager = MTL(ObTenantTmpFileManager*); + report_key.tenant_id_ = tenant_ids.at(i); + if (OB_ISNULL(tmp_file_manager)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(ERROR, "unexpected null", KR(ret)); + } else if (OB_FAIL(tmp_file_manager->get_macro_block_count(macro_block_cnt))) { + STORAGE_LOG(WARN, "fail to get_macro_block_count", KR(ret)); + } else { + int64_t tenant_tmp_size = macro_block_cnt * common::OB_DEFAULT_MACRO_BLOCK_SIZE; + if (OB_FAIL(result_map_.set_refactored(report_key, std::make_pair(tenant_tmp_size, tenant_tmp_size), 1))) { + STORAGE_LOG(WARN, "failed to set tenant tmp usage into result map", K(ret), K(report_key), K(macro_block_cnt)); + } + } + } // end MTL_SWITCH } - } + } // end for } return ret; } diff --git a/src/storage/ob_parallel_external_sort.h b/src/storage/ob_parallel_external_sort.h index 0779cad5a..f89bcda05 100644 --- a/src/storage/ob_parallel_external_sort.h +++ b/src/storage/ob_parallel_external_sort.h @@ -22,7 +22,7 @@ #include "share/io/ob_io_manager.h" #include "share/scheduler/ob_tenant_dag_scheduler.h" #include "blocksstable/ob_block_sstable_struct.h" -#include "blocksstable/ob_tmp_file.h" +#include "tmp_file/ob_tmp_file_manager.h" #include "share/config/ob_server_config.h" @@ -194,7 +194,7 @@ private: ObMacroBufferWriter macro_buffer_writer_; bool has_sample_item_; T sample_item_; - blocksstable::ObTmpFileIOHandle file_io_handle_; + tmp_file::ObTmpFileIOHandle file_io_handle_; int64_t fd_; int64_t dir_id_; uint64_t tenant_id_; @@ -308,16 +308,13 @@ int ObFragmentWriterV2::flush_buffer() STORAGE_LOG(WARN, "ObFragmentWriterV2 has not been inited", K(ret)); } else if (OB_FAIL(ObExternalSortConstant::get_io_timeout_ms(expire_timestamp_, timeout_ms))) { STORAGE_LOG(WARN, "fail to get io timeout ms", K(ret), K(expire_timestamp_)); - } else if (OB_FAIL(file_io_handle_.wait())) { - STORAGE_LOG(WARN, "fail to wait io finish", K(ret)); } else if (OB_FAIL(macro_buffer_writer_.serialize_header())) { STORAGE_LOG(WARN, "fail to serialize header", K(ret)); } else { - blocksstable::ObTmpFileIOInfo io_info; + tmp_file::ObTmpFileIOInfo io_info; io_info.fd_ = fd_; io_info.dir_id_ = dir_id_; io_info.size_ = buf_size_; - io_info.tenant_id_ = tenant_id_; io_info.buf_ = buf_; io_info.io_desc_.set_wait_event(ObWaitEventIds::DB_FILE_INDEX_BUILD_WRITE); io_info.io_timeout_ms_ = timeout_ms; @@ -343,16 +340,6 @@ int ObFragmentWriterV2::sync() STORAGE_LOG(WARN, "fail to flush buffer", K(ret)); } } - if (OB_SUCC(ret)) { - int64_t timeout_ms = 0; - if (OB_FAIL(file_io_handle_.wait())) { - STORAGE_LOG(WARN, "fail to wait io finish", K(ret)); - } else if (OB_FAIL(ObExternalSortConstant::get_io_timeout_ms(expire_timestamp_, timeout_ms))) { - STORAGE_LOG(WARN, "fail to get io timeout ms", K(ret), K(expire_timestamp_)); - } else if (OB_FAIL(FILE_MANAGER_INSTANCE_V2.sync(fd_, timeout_ms))) { - STORAGE_LOG(WARN, "fail to sync macro file", K(ret)); - } - } } return ret; } @@ -472,9 +459,9 @@ private: int64_t fd_; int64_t dir_id_; T curr_item_; - blocksstable::ObTmpFileIOHandle file_io_handles_[MAX_HANDLE_COUNT]; + tmp_file::ObTmpFileIOHandle file_io_handles_[MAX_HANDLE_COUNT]; int64_t handle_cursor_; - char *buf_; + char *buf_[MAX_HANDLE_COUNT]; uint64_t tenant_id_; bool is_prefetch_end_; int64_t buf_size_; @@ -487,9 +474,12 @@ ObFragmentReaderV2::ObFragmentReaderV2() allocator_(common::ObNewModIds::OB_ASYNC_EXTERNAL_SORTER, common::OB_MALLOC_BIG_BLOCK_SIZE), sample_allocator_(common::ObNewModIds::OB_ASYNC_EXTERNAL_SORTER, OB_MALLOC_NORMAL_BLOCK_SIZE), macro_buffer_reader_(), fd_(-1), dir_id_(-1), curr_item_(), - file_io_handles_(), handle_cursor_(-1), buf_(NULL), tenant_id_(common::OB_INVALID_ID), + file_io_handles_(), handle_cursor_(-1), buf_(), tenant_id_(common::OB_INVALID_ID), is_prefetch_end_(false), buf_size_(0), is_first_prefetch_(true) { + for (int64_t i = 0; i < MAX_HANDLE_COUNT; ++i) { + buf_[i] = nullptr; + } } template @@ -558,23 +548,24 @@ int ObFragmentReaderV2::prefetch() ret = OB_NOT_INIT; STORAGE_LOG(WARN, "ObFragmentReaderV2 has not been inited", K(ret)); } else { - if (nullptr == buf_) { - if (OB_ISNULL(buf_ = static_cast(allocator_.alloc(buf_size_)))) { + int64_t handle_index = handle_cursor_ % MAX_HANDLE_COUNT; + if (nullptr == buf_[handle_index]) { + if (OB_ISNULL(buf_[handle_index] = static_cast(allocator_.alloc(buf_size_)))) { ret = common::OB_ALLOCATE_MEMORY_FAILED; STORAGE_LOG(WARN, "fail to allocate memory", K(ret)); } } if (OB_SUCC(ret)) { - blocksstable::ObTmpFileIOInfo io_info; + tmp_file::ObTmpFileIOInfo io_info; io_info.fd_ = fd_; io_info.dir_id_ = dir_id_; io_info.size_ = buf_size_; - io_info.tenant_id_ = tenant_id_; - io_info.buf_ = buf_; + io_info.buf_ = buf_[handle_index]; + io_info.disable_page_cache_ = true; io_info.io_desc_.set_wait_event(ObWaitEventIds::DB_FILE_INDEX_BUILD_READ); if (OB_FAIL(ObExternalSortConstant::get_io_timeout_ms(expire_timestamp_, io_info.io_timeout_ms_))) { STORAGE_LOG(WARN, "fail to get io timeout ms", K(ret), K(expire_timestamp_), K(io_info.io_timeout_ms_)); - } else if (OB_FAIL(FILE_MANAGER_INSTANCE_V2.aio_read(io_info, file_io_handles_[handle_cursor_ % MAX_HANDLE_COUNT]))) { + } else if (OB_FAIL(FILE_MANAGER_INSTANCE_V2.aio_read(io_info, file_io_handles_[handle_index]))) { if (common::OB_ITER_END != ret) { STORAGE_LOG(WARN, "fail to do aio read from macro file", K(ret), K(fd_), K(io_info)); } else { @@ -676,7 +667,9 @@ void ObFragmentReaderV2::reset() file_io_handles_[i].reset(); } handle_cursor_ = 0; - buf_ = NULL; + for (int64_t i = 0; i < MAX_HANDLE_COUNT; ++i) { + buf_[i] = nullptr; + } tenant_id_ = common::OB_INVALID_ID; is_prefetch_end_ = false; buf_size_ = 0; @@ -688,6 +681,9 @@ int ObFragmentReaderV2::clean_up() { int ret = common::OB_SUCCESS; if (is_inited_) { + for (int64_t i = 0; i < MAX_HANDLE_COUNT; ++i) { + file_io_handles_[i].reset(); + } if (OB_FAIL(FILE_MANAGER_INSTANCE_V2.remove(fd_))) { STORAGE_LOG(WARN, "fail to remove macro file", K(ret)); } diff --git a/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp b/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp new file mode 100644 index 000000000..b0a050810 --- /dev/null +++ b/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp @@ -0,0 +1,2717 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE + +#include "storage/tmp_file/ob_shared_nothing_tmp_file.h" +#include "storage/tmp_file/ob_tmp_file_block_manager.h" +#include "storage/tmp_file/ob_tmp_file_eviction_manager.h" +#include "storage/tmp_file/ob_tmp_file_flush_priority_manager.h" +#include "storage/tmp_file/ob_tmp_file_write_buffer_pool.h" +#include "storage/tmp_file/ob_tmp_file_flush_manager.h" +#include "storage/tmp_file/ob_tmp_file_io_ctx.h" +#include "storage/blocksstable/ob_block_manager.h" +#include "lib/hash/ob_hashmap.h" +#include "lib/random/ob_random.h" +#include "storage/tmp_file/ob_tmp_file_page_cache_controller.h" + +namespace oceanbase +{ +namespace tmp_file +{ +ObTmpFileHandle::ObTmpFileHandle(ObSharedNothingTmpFile *tmp_file) + : ptr_(tmp_file) +{ + if (ptr_ != nullptr) { + ptr_->inc_ref_cnt(); + } +} + +ObTmpFileHandle::ObTmpFileHandle(const ObTmpFileHandle &handle) + : ptr_(nullptr) +{ + operator=(handle); +} + +ObTmpFileHandle & ObTmpFileHandle::operator=(const ObTmpFileHandle &other) +{ + if (other.get() != ptr_) { + reset(); + ptr_ = other.get(); + if (ptr_ != nullptr) { + ptr_->inc_ref_cnt(); + } + } + return *this; +} + +void ObTmpFileHandle::reset() +{ + if (ptr_ != nullptr) { + ptr_->dec_ref_cnt(); + if (ptr_->get_ref_cnt() == 0) { + ptr_->~ObSharedNothingTmpFile(); + } + ptr_ = nullptr; + } +} + +int ObTmpFileHandle::init(ObSharedNothingTmpFile *tmp_file) +{ + int ret = OB_SUCCESS; + + if (is_inited()) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", KR(ret), KP(ptr_)); + } else if (OB_ISNULL(tmp_file)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), KP(tmp_file)); + } else { + ptr_ = tmp_file; + ptr_->inc_ref_cnt(); + } + + return ret; +} + +void ObSharedNothingTmpFile::InnerFlushContext::reset() +{ + meta_finished_continuous_flush_info_num_ = 0; + data_finished_continuous_flush_info_num_ = 0; + meta_flush_infos_.reset(); + data_flush_infos_.reset(); + flush_seq_ = ObTmpFileGlobal::INVALID_FLUSH_SEQUENCE; +} + +int ObSharedNothingTmpFile::InnerFlushContext::update_finished_continuous_flush_info_num(const bool is_meta, const int64_t end_pos) +{ + int ret = OB_SUCCESS; + if (is_meta) { + if (OB_UNLIKELY(end_pos < meta_finished_continuous_flush_info_num_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected flush info num", K(ret), K(end_pos), K(meta_finished_continuous_flush_info_num_)); + } else { + meta_finished_continuous_flush_info_num_ = end_pos; + } + } else { + if (OB_UNLIKELY(end_pos < data_finished_continuous_flush_info_num_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected flush info num", K(ret), K(end_pos), K(data_finished_continuous_flush_info_num_)); + } else { + data_finished_continuous_flush_info_num_ = end_pos; + } + } + return ret; +} + +ObSharedNothingTmpFile::InnerFlushInfo::InnerFlushInfo() + : update_meta_data_done_(false), + flush_data_page_disk_begin_id_(ObTmpFileGlobal::INVALID_PAGE_ID), + flush_data_page_num_(-1), + flush_virtual_page_id_(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID), + file_size_(0), + flush_meta_page_array_() +{ + flush_meta_page_array_.set_attr(ObMemAttr(MTL_ID(), "TFFlushMetaArr")); +} + +void ObSharedNothingTmpFile::InnerFlushInfo::reset() +{ + update_meta_data_done_ = false; + flush_data_page_disk_begin_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + flush_data_page_num_ = -1; + flush_virtual_page_id_ = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + file_size_ = 0; + flush_meta_page_array_.reset(); +} + +int ObSharedNothingTmpFile::InnerFlushInfo::init_by_tmp_file_flush_info(const ObTmpFileFlushInfo& flush_info) +{ + int ret = OB_SUCCESS; + flush_data_page_disk_begin_id_ = flush_info.flush_data_page_disk_begin_id_; + flush_data_page_num_ = flush_info.flush_data_page_num_; + flush_virtual_page_id_ = flush_info.flush_virtual_page_id_; + file_size_ = flush_info.file_size_; + if (!flush_info.flush_meta_page_array_.empty() + && OB_FAIL(flush_meta_page_array_.assign(flush_info.flush_meta_page_array_))) { + LOG_WARN("fail to assign flush_meta_page_array_", KR(ret), K(flush_info)); + } + return ret; +} + +ObSharedNothingTmpFile::ObSharedNothingTmpFile() + : tmp_file_block_manager_(nullptr), + callback_allocator_(nullptr), + page_cache_controller_(nullptr), + flush_prio_mgr_(nullptr), + eviction_mgr_(nullptr), + wbp_(nullptr), + page_idx_cache_(), + is_inited_(false), + is_deleting_(false), + is_in_data_eviction_list_(false), + is_in_meta_eviction_list_(false), + data_page_flush_level_(-1), + meta_page_flush_level_(-1), + tenant_id_(OB_INVALID_TENANT_ID), + dir_id_(ObTmpFileGlobal::INVALID_TMP_FILE_DIR_ID), + fd_(ObTmpFileGlobal::INVALID_TMP_FILE_FD), + ref_cnt_(0), + truncated_offset_(0), + read_offset_(0), + file_size_(0), + flushed_data_page_num_(0), + write_back_data_page_num_(0), + cached_page_nums_(0), + begin_page_id_(ObTmpFileGlobal::INVALID_PAGE_ID), + begin_page_virtual_id_(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID), + flushed_page_id_(ObTmpFileGlobal::INVALID_PAGE_ID), + flushed_page_virtual_id_(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID), + end_page_id_(ObTmpFileGlobal::INVALID_PAGE_ID), + meta_tree_(), + data_flush_node_(*this), + meta_flush_node_(*this), + data_eviction_node_(*this), + meta_eviction_node_(*this), + meta_lock_(common::ObLatchIds::TMP_FILE_LOCK), + last_page_lock_(common::ObLatchIds::TMP_FILE_LOCK), + multi_write_lock_(common::ObLatchIds::TMP_FILE_LOCK), + truncate_lock_(common::ObLatchIds::TMP_FILE_LOCK), + inner_flush_ctx_() +{ +} + +ObSharedNothingTmpFile::~ObSharedNothingTmpFile() +{ + destroy(); +} + +int ObSharedNothingTmpFile::init(const uint64_t tenant_id, const int64_t fd, const int64_t dir_id, + ObTmpFileBlockManager *block_manager, + ObIAllocator *callback_allocator, + ObIAllocator *wbp_index_cache_allocator, + ObIAllocator *wbp_index_cache_bkt_allocator, + ObTmpFilePageCacheController *pc_ctrl) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", KR(ret), KPC(this)); + } else if (!is_valid_tenant_id(tenant_id) || ObTmpFileGlobal::INVALID_TMP_FILE_FD == fd || + ObTmpFileGlobal::INVALID_TMP_FILE_DIR_ID == dir_id || + OB_ISNULL(block_manager) || OB_ISNULL(callback_allocator) || + OB_ISNULL(wbp_index_cache_allocator) || OB_ISNULL(wbp_index_cache_bkt_allocator) || + OB_ISNULL(pc_ctrl)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(fd), K(dir_id), + KP(block_manager), KP(callback_allocator), KP(pc_ctrl), + KP(wbp_index_cache_allocator), KP(wbp_index_cache_bkt_allocator)); + } else if (OB_FAIL(page_idx_cache_.init(fd, &pc_ctrl->get_write_buffer_pool(), + wbp_index_cache_allocator, wbp_index_cache_bkt_allocator))) { + LOG_WARN("fail to init page idx array", KR(ret), K(fd)); + } else if (OB_FAIL(meta_tree_.init(fd, &pc_ctrl->get_write_buffer_pool(), callback_allocator))) { + LOG_WARN("fail to init meta tree", KR(ret), K(fd)); + } else { + is_inited_ = true; + tmp_file_block_manager_ = block_manager; + callback_allocator_ = callback_allocator; + page_cache_controller_ = pc_ctrl; + eviction_mgr_ = &pc_ctrl->get_eviction_manager(); + flush_prio_mgr_ = &pc_ctrl->get_flush_priority_mgr(); + wbp_ = &pc_ctrl->get_write_buffer_pool(); + tenant_id_ = tenant_id; + dir_id_ = dir_id; + fd_ = fd; + } + + LOG_INFO("tmp file init over", KR(ret), K(fd), K(dir_id)); + return ret; +} + +int ObSharedNothingTmpFile::destroy() +{ + int ret = OB_SUCCESS; + int64_t fd_backup = fd_; + + LOG_INFO("tmp file destroy start", KR(ret), K(fd_), KPC(this)); + + if (cached_page_nums_ > 0) { + uint32_t cur_page_id = begin_page_id_; + while (OB_SUCC(ret) && cur_page_id != ObTmpFileGlobal::INVALID_PAGE_ID) { + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == begin_page_virtual_id_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("begin page virtual id is invalid", KR(ret), K(fd_), K(begin_page_virtual_id_)); + } else if (OB_FAIL(wbp_->free_page(fd_, cur_page_id, ObTmpFilePageUniqKey(begin_page_virtual_id_), next_page_id))) { + LOG_WARN("fail to free page", KR(ret), K(fd_), K(cur_page_id), K(begin_page_virtual_id_)); + } else { + cur_page_id = next_page_id; + begin_page_virtual_id_ += 1; + } + } + } + + LOG_INFO("tmp file destroy, free wbp page phase over", KR(ret), K(fd_), KPC(this)); + + if (FAILEDx(meta_tree_.clear(truncated_offset_, file_size_))) { + LOG_WARN("fail to clear", KR(ret), K(fd_), K(truncated_offset_), K(file_size_)); + } + + LOG_INFO("tmp file destroy, meta_tree_ clear phase over", KR(ret), K(fd_), KPC(this)); + + if (OB_SUCC(ret)) { + reset(); + } + + LOG_INFO("tmp file destroy over", KR(ret), "fd", fd_backup); + return ret; +} + +void ObSharedNothingTmpFile::reset() +{ + tmp_file_block_manager_ = nullptr; + callback_allocator_ = nullptr; + page_cache_controller_ = nullptr; + flush_prio_mgr_ = nullptr; + eviction_mgr_ = nullptr; + wbp_ = nullptr; + page_idx_cache_.destroy(); + is_inited_ = false; + is_deleting_ = false; + is_in_data_eviction_list_ = false; + is_in_meta_eviction_list_ = false; + data_page_flush_level_ = -1; + meta_page_flush_level_ = -1; + tenant_id_ = OB_INVALID_TENANT_ID; + dir_id_ = ObTmpFileGlobal::INVALID_TMP_FILE_DIR_ID; + fd_ = ObTmpFileGlobal::INVALID_TMP_FILE_FD; + ref_cnt_ = 0; + truncated_offset_ = 0; + read_offset_ = 0; + file_size_ = 0; + flushed_data_page_num_ = 0; + write_back_data_page_num_ = 0; + cached_page_nums_ = 0; + begin_page_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + begin_page_virtual_id_ = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + flushed_page_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + flushed_page_virtual_id_ = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + end_page_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + meta_tree_.reset(); + data_flush_node_.unlink(); + meta_flush_node_.unlink(); + data_eviction_node_.unlink(); + meta_eviction_node_.unlink(); + inner_flush_ctx_.reset(); +} + +bool ObSharedNothingTmpFile::is_deleting() +{ + common::TCRWLock::RLockGuard guard(meta_lock_); + return is_deleting_; +} + +bool ObSharedNothingTmpFile::can_remove() +{ + common::TCRWLock::RLockGuard guard(meta_lock_); + return is_deleting_ && get_ref_cnt() == 1; +} + +int ObSharedNothingTmpFile::delete_file() +{ + int ret = OB_SUCCESS; + common::TCRWLock::WLockGuard guard(meta_lock_); + if (OB_UNLIKELY(is_deleting_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("the file is deleting", KR(ret), KPC(this)); + } else if (OB_FAIL(eviction_mgr_->remove_file(*this))) { + LOG_WARN("fail to remove file from eviction manager", KR(ret), KPC(this)); + } else if (OB_FAIL(flush_prio_mgr_->remove_file(*this))) { + LOG_WARN("fail to remove file from flush priority manager", KR(ret),KPC(this)); + } else { + // read, write, truncate, flush and evict function will fail when is_deleting_ == true. + is_deleting_ = true; + } + + LOG_INFO("tmp file delete", KR(ret), KPC(this)); + return ret; +} + +int ObSharedNothingTmpFile::aio_pread(ObTmpFileIOCtx &io_ctx) +{ + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObSharedNothingTmpFile has not been inited", KR(ret), K(tenant_id_), KPC(this)); + } else { + common::TCRWLock::RLockGuard guard(meta_lock_); + if (io_ctx.get_read_offset_in_file() < 0) { + io_ctx.set_read_offset_in_file(read_offset_); + } + + LOG_DEBUG("start to inner read tmp file", K(fd_), K(io_ctx.get_read_offset_in_file()), + K(io_ctx.get_todo_size()), K(io_ctx.get_done_size()), KPC(this)); + if (OB_FAIL(ret)) { + } else if (OB_UNLIKELY(!io_ctx.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(fd_), K(io_ctx)); + } else if (OB_UNLIKELY(is_deleting_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("attempt to read a deleting file", KR(ret), K(fd_)); + } else if (OB_UNLIKELY(io_ctx.get_read_offset_in_file() >= file_size_)) { + ret = OB_ITER_END; + LOG_WARN("iter end", KR(ret), K(fd_), K(file_size_), K(io_ctx)); + } else if (io_ctx.get_read_offset_in_file() < truncated_offset_ && + OB_FAIL(inner_read_truncated_part_(io_ctx))) { + LOG_WARN("fail to read truncated part", KR(ret), K(fd_), K(io_ctx)); + } else if (OB_UNLIKELY(io_ctx.get_todo_size() == 0)) { + // do nothing + } else { + // Iterate to read disk data. + int64_t wbp_begin_offset = cal_wbp_begin_offset_(); + if (OB_UNLIKELY(wbp_begin_offset < 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected wbp begin offset", KR(ret), K(fd_), K(wbp_begin_offset), K(io_ctx)); + } else if (io_ctx.get_read_offset_in_file() < wbp_begin_offset) { + const int64_t expected_read_disk_size = MIN(io_ctx.get_todo_size(), + wbp_begin_offset - io_ctx.get_read_offset_in_file()); + + if (OB_UNLIKELY(expected_read_disk_size < 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected read disk size", KR(ret), K(fd_), K(expected_read_disk_size), K(wbp_begin_offset), K(io_ctx)); + } else if (expected_read_disk_size == 0) { + // do nothing + } else if (OB_FAIL(inner_read_from_disk_(expected_read_disk_size, io_ctx))) { + LOG_WARN("fail to read tmp file from disk", KR(ret), K(fd_), K(expected_read_disk_size), + K(wbp_begin_offset), K(io_ctx)); + } else { + LOG_DEBUG("finish disk read", K(fd_), K(io_ctx.get_read_offset_in_file()), + K(io_ctx.get_todo_size()), + K(io_ctx.get_done_size()), + K(wbp_begin_offset), K(expected_read_disk_size)); + } + } + + // Iterate to read memory data (in write buffer pool). + if (OB_SUCC(ret) && io_ctx.get_todo_size() > 0) { + if (OB_UNLIKELY(0 == cached_page_nums_)) { + ret = OB_ITER_END; + LOG_WARN("iter end", KR(ret), K(fd_), K(io_ctx)); + } else if (OB_FAIL(inner_read_from_wbp_(io_ctx))) { + LOG_WARN("fail to read tmp file from wbp", KR(ret), K(fd_), K(io_ctx)); + } else { + LOG_DEBUG("finish wbp read", K(fd_), K(io_ctx.get_read_offset_in_file()), + K(io_ctx.get_todo_size()), + K(io_ctx.get_done_size())); + } + } + } + + LOG_DEBUG("inner read finish once", KR(ret), K(fd_), + K(io_ctx.get_read_offset_in_file()), + K(io_ctx.get_todo_size()), + K(io_ctx.get_done_size()), KPC(this)); + } + return ret; +} + +int ObSharedNothingTmpFile::inner_read_truncated_part_(ObTmpFileIOCtx &io_ctx) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(io_ctx.get_read_offset_in_file() >= truncated_offset_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("read offset should be less than truncated offset", KR(ret), K(fd_), K(io_ctx)); + } else if (OB_UNLIKELY(!io_ctx.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid io_ctx", KR(ret), K(fd_), K(io_ctx)); + } else if (OB_UNLIKELY(io_ctx.get_todo_size() == 0)) { + // do nothing + } else { + int64_t read_size = MIN(truncated_offset_ - io_ctx.get_read_offset_in_file(), + io_ctx.get_todo_size()); + char *read_buf = io_ctx.get_todo_buffer(); + if (OB_UNLIKELY(!io_ctx.check_buf_range_valid(read_buf, read_size))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid buf range", KR(ret), K(fd_), K(read_buf), K(read_size), K(io_ctx)); + } else if (FALSE_IT(MEMSET(read_buf, 0, read_size))) { + } else if (OB_FAIL(io_ctx.update_data_size(read_size))) { + LOG_WARN("fail to update data size", KR(ret), K(fd_), K(read_size)); + } else if (OB_UNLIKELY(io_ctx.get_todo_size() > 0 && + truncated_offset_ == file_size_)) { + ret = OB_ITER_END; + LOG_WARN("iter end", KR(ret), K(fd_), K(file_size_), K(truncated_offset_), K(io_ctx)); + } + } + + return ret; +} + +int ObSharedNothingTmpFile::inner_read_from_disk_(const int64_t expected_read_disk_size, + ObTmpFileIOCtx &io_ctx) +{ + int ret = OB_SUCCESS; + common::ObArray data_items; + if (OB_FAIL(meta_tree_.search_data_items(io_ctx.get_read_offset_in_file(), + expected_read_disk_size, data_items))) { + LOG_WARN("fail to search data items", KR(ret), K(fd_), K(expected_read_disk_size), K(io_ctx)); + } else if (OB_UNLIKELY(data_items.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("no data item found", KR(ret), K(fd_), K(expected_read_disk_size), K(io_ctx)); + } + + // Iterate to read each block. + int64_t remain_read_size = expected_read_disk_size; + int64_t actual_read_size = 0; + for (int64_t i = 0; OB_SUCC(ret) && i < data_items.count() && 0 < remain_read_size; i++) { + const int64_t block_index = data_items[i].block_index_; + const int64_t read_start_virtual_page_id = get_page_virtual_id_from_offset_(io_ctx.get_read_offset_in_file(), + false /*is_open_interval*/); + const int64_t start_page_id_in_data_item = read_start_virtual_page_id - data_items[i].virtual_page_id_; + const int64_t begin_offset_in_block = + (data_items[i].physical_page_id_ + start_page_id_in_data_item) * ObTmpFileGlobal::PAGE_SIZE; + const int64_t end_offset_in_block = + (data_items[i].physical_page_id_ + data_items[i].physical_page_num_) * ObTmpFileGlobal::PAGE_SIZE; + const int64_t begin_read_offset_in_block = begin_offset_in_block + + (0 == i? + get_page_offset_from_file_or_block_offset_(io_ctx.get_read_offset_in_file()) : 0); + const int64_t end_read_offset_in_block = (data_items.count() - 1 == i? + begin_read_offset_in_block + remain_read_size : + end_offset_in_block); + int64_t actual_block_read_size = 0; + + ObTmpBlockValueHandle block_value_handle; + if (OB_SUCC(ObTmpBlockCache::get_instance().get_block(ObTmpBlockCacheKey(block_index, MTL_ID()), + block_value_handle))) { + LOG_DEBUG("hit block cache", K(block_index), K(fd_), K(io_ctx)); + char *read_buf = io_ctx.get_todo_buffer(); + const int64_t read_size = end_read_offset_in_block - begin_read_offset_in_block; + ObTmpFileIOCtx::ObBlockCacheHandle block_cache_handle(block_value_handle, read_buf, + begin_read_offset_in_block, + read_size); + if (OB_FAIL(io_ctx.get_block_cache_handles().push_back(block_cache_handle))) { + LOG_WARN("Fail to push back into block_handles", KR(ret), K(fd_)); + } else if (OB_FAIL(io_ctx.update_data_size(read_size))) { + LOG_WARN("fail to update data size", KR(ret), K(read_size)); + } else { + remain_read_size -= read_size; + actual_read_size += read_size; + LOG_DEBUG("succ to read data from cached block", + KR(ret), K(fd_), K(block_index), K(begin_offset_in_block), K(end_offset_in_block), + K(begin_read_offset_in_block), K(end_read_offset_in_block), + K(remain_read_size), K(read_size), K(actual_read_size), + K(data_items[i]), K(io_ctx)); + } + } else if (OB_ENTRY_NOT_EXIST != ret) { + LOG_WARN("fail to get block", KR(ret), K(fd_), K(block_index)); + } else { // not hit block cache, read page from disk. + ret = OB_SUCCESS; + if (io_ctx.is_disable_page_cache()) { + if (OB_FAIL(inner_seq_read_from_block_(block_index, + begin_read_offset_in_block, + end_read_offset_in_block, + io_ctx, actual_block_read_size))) { + LOG_WARN("fail to seq read from block", + KR(ret), K(fd_), K(block_index), K(begin_offset_in_block), K(end_offset_in_block), + K(begin_read_offset_in_block), K(end_read_offset_in_block), + K(remain_read_size), K(actual_block_read_size), K(actual_read_size), + K(data_items[i]), K(io_ctx)); + } else { + remain_read_size -= actual_block_read_size; + actual_read_size += actual_block_read_size; + LOG_DEBUG("succ to seq read from block", + KR(ret), K(fd_), K(block_index), K(begin_offset_in_block), K(end_offset_in_block), + K(begin_read_offset_in_block), K(end_read_offset_in_block), + K(remain_read_size), K(actual_block_read_size), K(actual_read_size), + K(data_items[i]), K(io_ctx)); + } + } else { + if (OB_FAIL(inner_rand_read_from_block_(block_index, + begin_read_offset_in_block, + end_read_offset_in_block, + io_ctx, actual_block_read_size))) { + LOG_WARN("fail to rand read from block", + KR(ret), K(fd_), K(block_index), K(begin_offset_in_block), K(end_offset_in_block), + K(begin_read_offset_in_block), K(end_read_offset_in_block), + K(remain_read_size), K(actual_block_read_size), K(actual_read_size), + K(data_items[i]), K(io_ctx)); + } else { + remain_read_size -= actual_block_read_size; + actual_read_size += actual_block_read_size; + LOG_DEBUG("succ to rand read from block", + KR(ret), K(fd_), K(block_index), K(begin_offset_in_block), K(end_offset_in_block), + K(begin_read_offset_in_block), K(end_read_offset_in_block), + K(remain_read_size), K(actual_block_read_size), K(actual_read_size), + K(data_items[i]), K(io_ctx)); + } + } + } + } + + return ret; +} + +int ObSharedNothingTmpFile::inner_seq_read_from_block_(const int64_t block_index, + const int64_t begin_read_offset_in_block, + const int64_t end_read_offset_in_block, + ObTmpFileIOCtx &io_ctx, + int64_t &actual_read_size) +{ + int ret = OB_SUCCESS; + const int64_t expected_read_size = end_read_offset_in_block - begin_read_offset_in_block; + ObTmpFileBlockHandle block_handle; + actual_read_size = 0; + + if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX == block_index || + expected_read_size <= 0 || expected_read_size > OB_DEFAULT_MACRO_BLOCK_SIZE)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(fd_), K(block_index), + K(begin_read_offset_in_block), + K(end_read_offset_in_block)); + } else if (OB_FAIL(tmp_file_block_manager_->get_tmp_file_block_handle(block_index, block_handle))) { + LOG_WARN("fail to get tmp file block_handle", KR(ret), K(fd_), K(block_index)); + } else if (OB_ISNULL(block_handle.get()) || OB_UNLIKELY(!block_handle.get()->get_macro_block_id().is_valid())) { + LOG_WARN("fail to get tmp file block_handle", KR(ret), K(fd_), K(block_handle)); + } else { + char *read_buf = io_ctx.get_todo_buffer(); + ObTmpFileIOCtx::ObIOReadHandle io_read_handle(read_buf, + 0 /*offset_in_src_data_buf_*/, + expected_read_size, block_handle); + if (OB_FAIL(io_ctx.get_io_handles().push_back(io_read_handle))) { + LOG_WARN("Fail to push back into io_handles", KR(ret), K(fd_)); + } else if (OB_FAIL(ObTmpPageCache::get_instance().direct_read( + block_handle.get()->get_macro_block_id(), expected_read_size, begin_read_offset_in_block, + io_ctx.get_io_flag(), io_ctx.get_io_timeout_ms(), *callback_allocator_, + io_ctx.get_io_handles().at(io_ctx.get_io_handles().count()-1).handle_))) { + LOG_WARN("fail to cached_read", KR(ret), K(fd_), K(block_handle), K(expected_read_size), + K(begin_read_offset_in_block), K(io_ctx)); + } + } + + // Update read offset and read size. + if (OB_FAIL(ret)) { + } else if (OB_FAIL(io_ctx.update_data_size(expected_read_size))) { + LOG_WARN("fail to update data size", KR(ret), K(expected_read_size)); + } else { + actual_read_size = expected_read_size; + } + + return ret; +} + +int ObSharedNothingTmpFile::inner_rand_read_from_block_(const int64_t block_index, + const int64_t begin_read_offset_in_block, + const int64_t end_read_offset_in_block, + ObTmpFileIOCtx &io_ctx, + int64_t &actual_read_size) +{ + int ret = OB_SUCCESS; + const int64_t begin_page_idx_in_block = get_page_id_in_block_(begin_read_offset_in_block); + const int64_t end_page_idx_in_block = get_page_id_in_block_(end_read_offset_in_block - 1); // -1 for changing open interval to close interval + ObTmpFileBlockPageBitmap bitmap; + ObTmpFileBlockPageBitmapIterator iterator; + ObArray page_value_handles; + + if (OB_FAIL(collect_pages_in_block_(block_index, begin_page_idx_in_block, end_page_idx_in_block, + bitmap, page_value_handles))) { + LOG_WARN("fail to collect pages in block", KR(ret), K(fd_), K(block_index), + K(begin_page_idx_in_block), + K(end_page_idx_in_block)); + } else if (OB_FAIL(iterator.init(&bitmap, begin_page_idx_in_block, end_page_idx_in_block))) { + LOG_WARN("fail to init iterator", KR(ret), K(fd_), K(block_index), + K(begin_page_idx_in_block), K(end_page_idx_in_block)); + } else { + int64_t has_read_cached_page_num = 0; + while (OB_SUCC(ret) && iterator.has_next()) { + bool is_in_cache = false; + int64_t begin_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + int64_t end_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + int64_t begin_read_offset = -1; + int64_t end_read_offset = -1; + if (OB_FAIL(iterator.next_range(is_in_cache, begin_page_id, end_page_id))) { + LOG_WARN("fail to next range", KR(ret), K(fd_)); + } else if (OB_UNLIKELY(begin_page_id > end_page_id || begin_page_id < 0 || end_page_id < 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid range", KR(ret), K(fd_), K(begin_page_id), K(end_page_id)); + } else { + begin_read_offset = begin_page_id == begin_page_idx_in_block ? + begin_read_offset_in_block : + get_page_begin_offset_by_virtual_id_(begin_page_id); + end_read_offset = end_page_id == end_page_idx_in_block ? + end_read_offset_in_block : + get_page_end_offset_by_virtual_id_(end_page_id); + } + + if (OB_FAIL(ret)) { + } else if (is_in_cache) { + if (OB_FAIL(inner_read_continuous_cached_pages_(begin_read_offset, end_read_offset, + page_value_handles, has_read_cached_page_num, + io_ctx))) { + LOG_WARN("fail to inner read continuous cached pages", KR(ret), K(fd_), K(begin_read_offset), + K(end_read_offset), K(io_ctx)); + } else { + has_read_cached_page_num += (end_page_id - begin_page_id + 1); + } + } else { + if (OB_FAIL(inner_read_continuous_uncached_pages_(block_index, begin_read_offset, + end_read_offset, io_ctx))) { + LOG_WARN("fail to inner read continuous uncached pages", KR(ret), K(fd_), K(block_index), + K(begin_read_offset), + K(end_read_offset), + K(io_ctx)); + } + } + + if (OB_SUCC(ret)) { + const int64_t read_size = end_read_offset - begin_read_offset; + actual_read_size += read_size; + } + } + } + + return ret; +} + +int ObSharedNothingTmpFile::collect_pages_in_block_(const int64_t block_index, + const int64_t begin_page_idx_in_block, + const int64_t end_page_idx_in_block, + ObTmpFileBlockPageBitmap &bitmap, + ObIArray &page_value_handles) +{ + int ret = OB_SUCCESS; + bitmap.reset(); + page_value_handles.reset(); + + for (int64_t page_idx_in_block = begin_page_idx_in_block; + OB_SUCC(ret) && page_idx_in_block <= end_page_idx_in_block; + page_idx_in_block++) { + ObTmpPageCacheKey key(block_index, page_idx_in_block, tenant_id_); + ObTmpPageValueHandle p_handle; + if (OB_SUCC(ObTmpPageCache::get_instance().get_page(key, p_handle))) { + if (OB_FAIL(page_value_handles.push_back(p_handle))) { + LOG_WARN("fail to push back", KR(ret), K(fd_), K(key)); + } else if (OB_FAIL(bitmap.set_bitmap(page_idx_in_block, true))) { + LOG_WARN("fail to set bitmap", KR(ret), K(fd_), K(key)); + } + } else if (OB_ENTRY_NOT_EXIST == ret) { + ret = OB_SUCCESS; + if (OB_FAIL(bitmap.set_bitmap(page_idx_in_block, false))) { + LOG_WARN("fail to set bitmap", KR(ret), K(fd_), K(key)); + } + } else { + LOG_WARN("fail to get page from cache", KR(ret), K(fd_), K(key)); + } + } + + return ret; +} + +int ObSharedNothingTmpFile::inner_read_continuous_uncached_pages_(const int64_t block_index, + const int64_t begin_read_offset_in_block, + const int64_t end_read_offset_in_block, + ObTmpFileIOCtx &io_ctx) +{ + int ret = OB_SUCCESS; + ObArray page_keys; + const int64_t begin_page_idx = get_page_id_in_block_(begin_read_offset_in_block); + const int64_t end_page_idx = get_page_id_in_block_(end_read_offset_in_block - 1); // -1 to change open interval to close interval + const int64_t block_read_begin_offset = get_page_begin_offset_by_file_or_block_offset_(begin_read_offset_in_block); + const int64_t block_read_end_offset = get_page_end_offset_by_file_or_block_offset_(end_read_offset_in_block); + const int64_t block_read_size = block_read_end_offset - block_read_begin_offset; // read and cached completed pages from disk + const int64_t usr_read_begin_offset = begin_read_offset_in_block; + const int64_t usr_read_end_offset = end_read_offset_in_block; + // from loaded disk block buf, from "offset_in_block_buf" read "usr_read_size" size data to user's read buf + const int64_t offset_in_block_buf = usr_read_begin_offset - block_read_begin_offset; + const int64_t usr_read_size = block_read_size - + (usr_read_begin_offset - block_read_begin_offset) - + (block_read_end_offset - usr_read_end_offset); + + for (int64_t page_id = begin_page_idx; OB_SUCC(ret) && page_id <= end_page_idx; page_id++) { + ObTmpPageCacheKey key(block_index, page_id, tenant_id_); + if (OB_FAIL(page_keys.push_back(key))) { + LOG_WARN("fail to push back", KR(ret), K(fd_), K(key)); + } + } + + ObTmpFileBlockHandle block_handle; + if (OB_FAIL(ret)) { + } else if (OB_FAIL(tmp_file_block_manager_->get_tmp_file_block_handle(block_index, block_handle))) { + LOG_WARN("fail to get tmp file block_handle", KR(ret), K(fd_), K(block_index)); + } else { + char *user_read_buf = io_ctx.get_todo_buffer(); + ObTmpFileIOCtx::ObIOReadHandle io_read_handle(user_read_buf, + offset_in_block_buf, + usr_read_size, block_handle); + if (OB_FAIL(io_ctx.get_io_handles().push_back(io_read_handle))) { + LOG_WARN("Fail to push back into io_handles", KR(ret), K(fd_)); + } else if (OB_FAIL(ObTmpPageCache::get_instance().cached_read(page_keys, + block_handle.get()->get_macro_block_id(), block_read_begin_offset, + io_ctx.get_io_flag(), io_ctx.get_io_timeout_ms(), *callback_allocator_, + io_ctx.get_io_handles().at(io_ctx.get_io_handles().count()-1).handle_))) { + LOG_WARN("fail to cached_read", KR(ret), K(fd_), K(block_handle), + K(block_read_begin_offset), K(io_ctx)); + } else if (OB_FAIL(io_ctx.update_data_size(usr_read_size))) { + LOG_WARN("fail to update data size", KR(ret), K(fd_), K(usr_read_size)); + } + } + + return ret; +} + +int ObSharedNothingTmpFile::inner_read_continuous_cached_pages_(const int64_t begin_read_offset_in_block, + const int64_t end_read_offset_in_block, + const ObArray &page_value_handles, + const int64_t start_array_idx, + ObTmpFileIOCtx &io_ctx) +{ + int ret = OB_SUCCESS; + const int64_t begin_page_idx = get_page_id_in_block_(begin_read_offset_in_block); + const int64_t end_page_idx = get_page_id_in_block_(end_read_offset_in_block - 1); // -1 for changing open interval to close interval + const int64_t iter_array_cnt = end_page_idx - begin_page_idx + 1; + + if (OB_UNLIKELY(start_array_idx < 0 || start_array_idx + iter_array_cnt > page_value_handles.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid array idx", KR(ret), K(fd_), K(start_array_idx), K(iter_array_cnt), + K(begin_page_idx), K(end_page_idx), K(page_value_handles.count())); + } + + int64_t read_offset = begin_read_offset_in_block; + int64_t cur_array_idx = start_array_idx; + for (int64_t cached_page_idx = begin_page_idx; OB_SUCC(ret) && cached_page_idx <= end_page_idx; cached_page_idx++) { + ObTmpPageValueHandle p_handle = page_value_handles.at(cur_array_idx++); + char *read_buf = io_ctx.get_todo_buffer(); + const int64_t cur_page_end_offset = get_page_end_offset_by_virtual_id_(cached_page_idx); + const int64_t read_size = MIN(cur_page_end_offset, end_read_offset_in_block) - read_offset; + const int64_t read_offset_in_page = get_page_offset_from_file_or_block_offset_(read_offset); + ObTmpFileIOCtx::ObPageCacheHandle page_handle(p_handle, read_buf, + read_offset_in_page, + read_size); + if (OB_FAIL(io_ctx.get_page_cache_handles().push_back(page_handle))) { + LOG_WARN("Fail to push back into page_handles", KR(ret), K(fd_)); + } else if (OB_FAIL(io_ctx.update_data_size(read_size))) { + LOG_WARN("fail to update data size", KR(ret), K(fd_), K(read_size)); + } else { + read_offset += read_size; + } + } // end for + + return ret; +} + +int ObSharedNothingTmpFile::inner_read_from_wbp_(ObTmpFileIOCtx &io_ctx) +{ + int ret = OB_SUCCESS; + const int64_t begin_read_page_virtual_id = get_page_virtual_id_from_offset_(io_ctx.get_read_offset_in_file(), + false /*is_open_interval*/); + uint32_t begin_read_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + int64_t wbp_begin_offset = cal_wbp_begin_offset_(); + + if (OB_UNLIKELY(wbp_begin_offset < 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected wbp begin offset", KR(ret), K(fd_), K(wbp_begin_offset), K(io_ctx)); + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == begin_read_page_virtual_id)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected begin read page virtual id", KR(ret), K(fd_), K(begin_read_page_virtual_id), K(io_ctx)); + } else if (io_ctx.get_read_offset_in_file() < wbp_begin_offset + ObTmpFileGlobal::PAGE_SIZE) { + begin_read_page_id = begin_page_id_; + } else if (OB_FAIL(page_idx_cache_.binary_search(begin_read_page_virtual_id, begin_read_page_id))) { + LOG_WARN("fail to find page index in array", KR(ret), K(fd_), K(io_ctx)); + } else if (ObTmpFileGlobal::INVALID_PAGE_ID == begin_read_page_id && + OB_FAIL(wbp_->get_page_id_by_virtual_id(fd_, begin_read_page_virtual_id, begin_page_id_, begin_read_page_id))) { + LOG_WARN("fail to get page id by virtual id", KR(ret), K(fd_), K(begin_read_page_virtual_id), K(begin_page_id_)); + } else if (OB_UNLIKELY(begin_read_page_id == ObTmpFileGlobal::INVALID_PAGE_ID)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid page index", KR(ret), K(fd_), K(begin_read_page_id)); + } + + uint32_t curr_page_id = begin_read_page_id; + int64_t curr_page_virtual_id = begin_read_page_virtual_id; + while (OB_SUCC(ret) && io_ctx.get_todo_size() > 0) { + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + char *data_page = nullptr; + if (OB_FAIL(wbp_->read_page(fd_, curr_page_id, ObTmpFilePageUniqKey(curr_page_virtual_id), data_page, next_page_id))) { + LOG_WARN("fail to fetch page", KR(ret), K(fd_), K(curr_page_id), K(curr_page_virtual_id), K(io_ctx)); + } else if (OB_ISNULL(data_page)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("data page is null", KR(ret), K(fd_), K(curr_page_id)); + } else { + const int64_t read_offset_in_page = get_page_offset_from_file_or_block_offset_(io_ctx.get_read_offset_in_file()); + const int64_t read_size = MIN3(ObTmpFileGlobal::PAGE_SIZE - read_offset_in_page, + io_ctx.get_todo_size(), + file_size_ - io_ctx.get_read_offset_in_file()); + char *read_buf = io_ctx.get_todo_buffer(); + if (OB_UNLIKELY(!io_ctx.check_buf_range_valid(read_buf, read_size))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid buf range", KR(ret), K(fd_), K(read_buf), K(read_size), K(io_ctx)); + } else if (OB_FAIL(io_ctx.update_data_size(read_size))) { + LOG_WARN("fail to update data size", KR(ret), K(fd_), K(read_size)); + } else { + MEMCPY(read_buf, data_page + read_offset_in_page, read_size); + curr_page_id = next_page_id; + curr_page_virtual_id += 1; + } + } + + if (OB_SUCC(ret) && io_ctx.get_todo_size() > 0 && io_ctx.get_read_offset_in_file() == file_size_) { + ret = OB_ITER_END; + LOG_WARN("iter end", KR(ret), K(fd_), K(io_ctx)); + } + } + + return ret; +} + +// attention: +// in order to avoid blocking reading, flushing and evicting pages, +// write operation only adds write lock for meta_lock when it try to update meta data. +// due to the write operation is appending write, if the meta data doesn't been modified, +// all operation could not write or read the new appending data. +int ObSharedNothingTmpFile::aio_write(ObTmpFileIOCtx &io_ctx) +{ + int ret = OB_SUCCESS; + LOG_DEBUG("aio write start", K(fd_), K(file_size_), K(io_ctx)); + + ObSpinLockGuard guard(multi_write_lock_); + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObSharedNothingTmpFile has not been inited", KR(ret), K(tenant_id_), KPC(this)); + } else if (OB_UNLIKELY(!io_ctx.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(fd_), K(io_ctx)); + } else if (OB_UNLIKELY(is_deleting_)) { + // this check is just a hint. + // although is_deleting_ == false, it might be set as true in the processing of inner_write(). + // we will check is_deleting_ again when try to update meta data in the inner_write() + ret = OB_ERR_UNEXPECTED; + LOG_WARN("attempt to write a deleting file", KR(ret), K(fd_)); + } else { + while (OB_SUCC(ret) && io_ctx.get_todo_size() > 0) { + if (OB_FAIL(inner_write_(io_ctx))) { + if (OB_ALLOCATE_TMP_FILE_PAGE_FAILED == ret) { + ret = OB_SUCCESS; + if (TC_REACH_COUNT_INTERVAL(10)) { + LOG_INFO("alloc mem failed, try to evict pages", K(fd_), K(file_size_), K(io_ctx)); + } + if (OB_FAIL(page_cache_controller_->invoke_swap_and_wait( + MIN(io_ctx.get_todo_size(), ObTmpFileGlobal::TMP_FILE_WRITE_BATCH_PAGE_NUM * ObTmpFileGlobal::PAGE_SIZE), + io_ctx.get_io_timeout_ms()))) { + LOG_WARN("fail to invoke swap and wait", KR(ret), K(io_ctx), K(fd_)); + } + } else { + LOG_WARN("fail to inner write", KR(ret), K(fd_), K(io_ctx), KPC(this)); + } + } + } // end while + } + + if (OB_SUCC(ret)) { + LOG_DEBUG("aio write finish", KR(ret), K(fd_), K(file_size_), K(io_ctx)); + } else { + LOG_DEBUG("aio write failed", KR(ret), K(fd_), K(file_size_), K(io_ctx), KPC(this)); + } + + return ret; +} + +int ObSharedNothingTmpFile::inner_write_(ObTmpFileIOCtx &io_ctx) +{ + int ret = OB_SUCCESS; + if (has_unfinished_page_()) { + if (OB_FAIL(inner_fill_tail_page_(io_ctx))) { + LOG_WARN("fail to fill tail page", KR(ret), K(fd_), K(io_ctx)); + } else if (OB_UNLIKELY(has_unfinished_page_() && io_ctx.get_todo_size() > 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("tmp file page is unaligned", KR(ret), K(fd_)); + } + } + + // each batch at most write TMP_FILE_WRITE_BATCH_PAGE_NUM pages + while (OB_SUCC(ret) && io_ctx.get_todo_size() > 0) { + if (OB_FAIL(inner_write_continuous_pages_(io_ctx))) { + LOG_WARN("fail to write continuous pages", KR(ret), K(fd_), K(io_ctx)); + } + } + + return ret; +} + +int ObSharedNothingTmpFile::inner_fill_tail_page_(ObTmpFileIOCtx &io_ctx) +{ + int ret = OB_SUCCESS; + ObSpinLockGuard last_page_lock_guard(last_page_lock_); + const bool is_in_disk = (0 == cached_page_nums_); + + if (OB_UNLIKELY(!io_ctx.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(fd_), K(io_ctx)); + } else if (is_in_disk) { + if (OB_FAIL(load_disk_tail_page_and_rewrite_(io_ctx))) { + LOG_WARN("fail to load disk tail page and rewrite", KR(ret),K(fd_), K(io_ctx)); + } + } else { + if (OB_FAIL(append_write_memory_tail_page_(io_ctx))) { + LOG_WARN("fail to append write memory tail page", KR(ret), K(fd_), K(io_ctx)); + } + } + + return ret; +} + +int ObSharedNothingTmpFile::load_disk_tail_page_and_rewrite_(ObTmpFileIOCtx &io_ctx) +{ + int ret = OB_SUCCESS; + const int64_t has_written_size = get_page_offset_from_file_or_block_offset_(file_size_); + const int64_t begin_page_virtual_id = get_page_virtual_id_from_offset_(file_size_, + true /*is_open_interval*/); + const int64_t need_write_size = MIN(ObTmpFileGlobal::PAGE_SIZE - has_written_size, io_ctx.get_todo_size()); + char *write_buff = io_ctx.get_todo_buffer(); + uint32_t new_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ObSharedNothingTmpFileDataItem data_item; + bool block_meta_tree_flushing = false; + + if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == begin_page_virtual_id)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("begin page virtual id is invalid", KR(ret), K(fd_), K(begin_page_virtual_id), K(file_size_)); + } else if (OB_UNLIKELY(has_written_size + need_write_size > ObTmpFileGlobal::PAGE_SIZE)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("need write size is invalid", KR(ret), K(fd_), K(has_written_size), K(need_write_size)); + } else if (OB_ISNULL(write_buff)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("write buffer is null", KR(ret), K(fd_), K(write_buff)); + } else if (OB_FAIL(meta_tree_.prepare_for_write_tail(data_item))) { + LOG_WARN("fail to prepare for write tail", KR(ret), K(fd_)); + } else { + block_meta_tree_flushing = true; + } + + if (OB_SUCC(ret)) { + blocksstable::MacroBlockId macro_block_id; + int64_t last_page_begin_offset_in_block = + (data_item.physical_page_id_ + data_item.physical_page_num_ - 1) + * ObTmpFileGlobal::PAGE_SIZE; + char *page_buf = nullptr; + if (OB_FAIL(tmp_file_block_manager_->get_macro_block_id(data_item.block_index_, macro_block_id))) { + LOG_WARN("fail to get macro block id", KR(ret), K(fd_), K(data_item.block_index_)); + } else if (OB_UNLIKELY(!macro_block_id.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("macro block id is invalid", KR(ret), K(fd_), K(data_item.block_index_)); + } else if (OB_FAIL(wbp_->alloc_page(fd_, ObTmpFilePageUniqKey(begin_page_virtual_id), new_page_id, page_buf))) { + LOG_WARN("fail to alloc page", KR(ret), K(fd_), K(begin_page_virtual_id), + K(new_page_id), KP(page_buf)); + } else { + // load last unfilled page from disk + blocksstable::ObMacroBlockHandle mb_handle; + blocksstable::ObMacroBlockReadInfo info; + info.io_desc_ = io_ctx.get_io_flag(); + info.macro_block_id_ = macro_block_id; + info.size_ = has_written_size; + info.offset_ = last_page_begin_offset_in_block; + info.buf_ = page_buf; + info.io_callback_ = nullptr; + info.io_timeout_ms_ = io_ctx.get_io_timeout_ms(); + + if (OB_FAIL(mb_handle.async_read(info))) { + LOG_ERROR("fail to async write block", KR(ret), K(fd_), K(info)); + } else if (OB_FAIL(mb_handle.wait())) { + LOG_WARN("fail to wait", KR(ret), K(fd_), K(info)); + } else if (mb_handle.get_data_size() < has_written_size) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to read expected size", KR(ret), K(fd_), K(info), K(has_written_size)); + } else if (OB_UNLIKELY(!io_ctx.check_buf_range_valid(write_buff, need_write_size))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid buf range", KR(ret), K(fd_), K(write_buff), K(need_write_size), K(io_ctx)); + } else { + // fill last page in memory + MEMCPY(page_buf + has_written_size, write_buff, need_write_size); + } + + if (OB_FAIL(ret)) { + int tmp_ret = OB_SUCCESS; + uint32_t unused_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + if (OB_TMP_FAIL(wbp_->free_page(fd_, new_page_id, ObTmpFilePageUniqKey(begin_page_virtual_id), unused_page_id))) { + LOG_WARN("fail to free page", KR(tmp_ret), K(fd_), K(begin_page_virtual_id), K(new_page_id)); + } + new_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + } else if (OB_FAIL(wbp_->notify_dirty(fd_, new_page_id, ObTmpFilePageUniqKey(begin_page_virtual_id)))) { + LOG_WARN("fail to notify dirty", KR(ret), K(fd_), K(new_page_id)); + } + } + } + + // update meta data + if (OB_SUCC(ret)) { + common::TCRWLock::WLockGuard guard(meta_lock_); + if (OB_UNLIKELY(is_deleting_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("file is deleting", KR(ret), K(fd_)); + } else if (OB_FAIL(meta_tree_.finish_write_tail(data_item, true /*release_tail_in_disk*/))) { + LOG_WARN("fail to finish write tail page", KR(ret), K(fd_)); + } else if (OB_FAIL(io_ctx.update_data_size(need_write_size))) { + LOG_WARN("fail to update data size", KR(ret), K(fd_), K(need_write_size)); + } else if (OB_FAIL(page_idx_cache_.push(new_page_id))) { + LOG_WARN("fail to push back page idx array", KR(ret), K(fd_), K(new_page_id)); + } else { + cached_page_nums_ = 1; + file_size_ += need_write_size; + begin_page_id_ = new_page_id; + begin_page_virtual_id_ = begin_page_virtual_id; + end_page_id_ = new_page_id; + } + + if (FAILEDx(insert_or_update_data_flush_node_())) { + LOG_WARN("fail to insert or update flush data list", KR(ret), K(fd_), KPC(this)); + } else if (OB_FAIL(insert_or_update_meta_flush_node_())) { + LOG_WARN("fail to insert or update flush meta list", KR(ret), K(fd_), KPC(this)); + } + + LOG_DEBUG("load_disk_tail_page_and_rewrite_ end", KR(ret), K(fd_), K(end_page_id_), KPC(this)); + } + + if (OB_FAIL(ret) && block_meta_tree_flushing) { + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(meta_tree_.finish_write_tail(data_item, false /*release_tail_in_disk*/))) { + LOG_WARN("fail to modify items after tail load", KR(tmp_ret), K(fd_)); + } + } + return ret; +} + +// attention: +// last_page_lock_ could promise that there are not truncating task, +// evicting task or flush generator task for the last page. +// however, it might happen that append write for the last page is processing after +// flush generator task has been processed over. +// thus, this function should consider the concurrence problem between append write and callback of flush task. +int ObSharedNothingTmpFile::append_write_memory_tail_page_(ObTmpFileIOCtx &io_ctx) +{ + int ret = OB_SUCCESS; + char *page_buff = nullptr; + const int64_t has_written_size = get_page_offset_from_file_or_block_offset_(file_size_); + const int64_t need_write_size = MIN(ObTmpFileGlobal::PAGE_SIZE - has_written_size, + io_ctx.get_todo_size()); + const int64_t end_page_virtual_id = get_page_virtual_id_from_offset_(file_size_, + true /*is_open_interval*/); + char *write_buff = io_ctx.get_todo_buffer(); + uint32_t unused_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ObSharedNothingTmpFileDataItem rightest_data_item; + + if (OB_UNLIKELY(has_written_size + need_write_size > ObTmpFileGlobal::PAGE_SIZE)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("need write size is invalid", KR(ret), K(fd_), K(has_written_size), K(need_write_size)); + } else if (OB_ISNULL(write_buff)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("write buffer is null", KR(ret), K(fd_), K(write_buff)); + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == end_page_virtual_id)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("end page virtual id is invalid", KR(ret), K(fd_), K(end_page_virtual_id), K(file_size_)); + } else if (OB_FAIL(wbp_->read_page(fd_, end_page_id_, ObTmpFilePageUniqKey(end_page_virtual_id), + page_buff, unused_page_id))) { + LOG_WARN("fail to fetch page", KR(ret), K(fd_), K(end_page_id_), K(end_page_virtual_id)); + } else if (OB_ISNULL(page_buff)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("page buff is null", KR(ret), K(fd_), K(end_page_id_)); + } else if ((wbp_->is_write_back(fd_, end_page_id_, ObTmpFilePageUniqKey(end_page_virtual_id)) || + wbp_->is_cached(fd_, end_page_id_, ObTmpFilePageUniqKey(end_page_virtual_id))) && + OB_FAIL(meta_tree_.prepare_for_write_tail(rightest_data_item))) { + LOG_WARN("fail to prepare for write tail", KR(ret), K(fd_), + K(end_page_id_), K(end_page_virtual_id), K(rightest_data_item)); + } else if (OB_UNLIKELY(!io_ctx.check_buf_range_valid(write_buff, need_write_size))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid buf range", KR(ret), K(fd_), K(write_buff), K(need_write_size), K(io_ctx)); + } else { + MEMCPY(page_buff + has_written_size, write_buff, need_write_size); + } + + // update meta data + if (OB_SUCC(ret)) { + common::TCRWLock::WLockGuard guard(meta_lock_); + const bool is_cached = wbp_->is_cached(fd_, end_page_id_, ObTmpFilePageUniqKey(end_page_virtual_id)); + const bool is_write_back = wbp_->is_write_back(fd_, end_page_id_, ObTmpFilePageUniqKey(end_page_virtual_id)); + if (OB_UNLIKELY(is_deleting_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("file is deleting", KR(ret), K(fd_)); + } else if (is_cached || is_write_back) { + // due to the appending writing for the last page which is flushing or flushed into disk, + // the page carbon in memory and disk will be different. + // thus, we need to rollback the flush status of the last page, + if (OB_FAIL(wbp_->notify_dirty(fd_, end_page_id_, ObTmpFilePageUniqKey(end_page_virtual_id)))) { + LOG_WARN("fail to notify dirty", KR(ret), K(fd_), K(end_page_id_), K(end_page_virtual_id)); + } else if (is_cached) { + // for the last page, if the status of flushed_page_id_ page is not cached, + // we will treat this page as a non-flushed page + flushed_data_page_num_--; + } else if (is_write_back) { + write_back_data_page_num_--; + } + + // modify meta tree + if (OB_FAIL(ret)) { + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(meta_tree_.finish_write_tail(rightest_data_item, false /*release_tail_in_disk*/))) { + LOG_WARN("fail to modify finish write tail page", KR(tmp_ret), K(fd_)); + } + } else { + // add file into flush list if file is not flushing, + // because we produce 1 dirty data page and 1 dirty meta page after writing tail page + if (OB_FAIL(meta_tree_.finish_write_tail(rightest_data_item, true /*release_tail_in_disk*/))) { + LOG_WARN("fail to finish write tail page", KR(ret), K(fd_)); + } else if (OB_FAIL(insert_or_update_data_flush_node_())) { + LOG_WARN("fail to insert or update flush data list", KR(ret), K(fd_), KPC(this)); + } else if (OB_FAIL(insert_or_update_meta_flush_node_())) { + LOG_WARN("fail to insert or update flush meta list", KR(ret), K(fd_), KPC(this)); + } + } + } + + if (FAILEDx(io_ctx.update_data_size(need_write_size))) { + LOG_WARN("fail to update data size", KR(ret), K(fd_), K(need_write_size)); + } else { + file_size_ += need_write_size; + } + + LOG_DEBUG("append_write_memory_tail_page_ end", KR(ret), K(fd_), K(end_page_id_), KPC(this)); + } + + return ret; +} + +int ObSharedNothingTmpFile::inner_write_continuous_pages_(ObTmpFileIOCtx &io_ctx) +{ + int ret = OB_SUCCESS; + bool is_alloc_failed = false; + int64_t write_size = 0; + ObArray page_entry_idxs; + + // write pages + if (OB_UNLIKELY(!io_ctx.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(fd_), K(io_ctx)); + } else if (OB_FAIL(alloc_and_write_pages_(io_ctx, page_entry_idxs, write_size))) { + if (OB_ALLOCATE_TMP_FILE_PAGE_FAILED == ret) { + // this error code will return to caller after modifing meta data of file based on written data pages + ret = OB_SUCCESS; + is_alloc_failed = true; + if (TC_REACH_COUNT_INTERVAL(10)) { + LOG_INFO("fail to alloc memory", K(tenant_id_), K(fd_), K(write_size), + K(page_entry_idxs), K(io_ctx)); + } + } else { + LOG_WARN("fail to batch write pages", KR(ret), K(fd_), K(io_ctx)); + } + } + + // Update meta data. + if (OB_FAIL(ret)) { + } else if (OB_UNLIKELY(page_entry_idxs.empty() || write_size <= 0)) { + if (!is_alloc_failed) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("page entry idxs is empty", KR(ret), K(fd_), K(page_entry_idxs), K(write_size)); + } else { + // do nothing, no need to update meta data + } + } else { + common::TCRWLock::WLockGuard guard(meta_lock_); + const int64_t end_page_virtual_id = cached_page_nums_ == 0 ? + ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID : + get_page_virtual_id_from_offset_(file_size_, true /*is_open_interval*/); + if (OB_UNLIKELY(is_deleting_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("file is deleting", KR(ret), K(fd_)); + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_PAGE_ID != end_page_id_ && + ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == end_page_virtual_id)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("end page virtual id is invalid", KR(ret), K(fd_), K(end_page_virtual_id), K(file_size_)); + } else if (OB_UNLIKELY((ObTmpFileGlobal::INVALID_PAGE_ID == begin_page_id_ && + ObTmpFileGlobal::INVALID_PAGE_ID != end_page_id_) || + (ObTmpFileGlobal::INVALID_PAGE_ID != begin_page_id_ && + ObTmpFileGlobal::INVALID_PAGE_ID == end_page_id_))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("begin or end page id is invalid", KR(ret), K(fd_), + K(begin_page_id_), + K(end_page_id_)); + } else if (OB_FAIL(io_ctx.update_data_size(write_size))) { + LOG_WARN("fail to update data size", KR(ret), K(fd_), K(io_ctx)); + } else { + if (ObTmpFileGlobal::INVALID_PAGE_ID != end_page_id_ && + OB_FAIL(wbp_->link_page(fd_, page_entry_idxs.at(0), end_page_id_, ObTmpFilePageUniqKey(end_page_virtual_id)))) { + LOG_WARN("fail to link page", KR(ret), K(fd_), K(page_entry_idxs.at(0)), + K(end_page_id_), K(end_page_virtual_id)); + } else if (FALSE_IT(end_page_id_ = page_entry_idxs.at(page_entry_idxs.count() - 1))) { + } else { + if (ObTmpFileGlobal::INVALID_PAGE_ID == begin_page_id_) { + begin_page_id_ = page_entry_idxs.at(0); + begin_page_virtual_id_ = get_page_virtual_id_from_offset_(file_size_, false /*is_open_interval*/); + } + file_size_ += write_size; + cached_page_nums_ += page_entry_idxs.count(); + } + + for (int64_t i = 0; i < page_entry_idxs.count() && OB_SUCC(ret); i++) { + if (OB_FAIL(page_idx_cache_.push(page_entry_idxs[i]))) { + LOG_WARN("fail to push page idx array", KR(ret), K(fd_)); + } + } // TODO: we can ignore page_idx_cache_ allocate memory fail to continue writing in the future. + if (FAILEDx(insert_or_update_data_flush_node_())) { + LOG_WARN("fail to insert or update data flush list", KR(ret), K(fd_), KPC(this)); + } + } + + if (OB_FAIL(ret)) { + int tmp_ret = OB_SUCCESS; + uint32_t unused_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + int64_t free_page_virtual_id = file_size_ == 0 ? 0 : end_page_virtual_id + 1; + for (int64_t i = 0; i < page_entry_idxs.count() && OB_LIKELY(OB_SUCCESS == tmp_ret); i++) { + if (OB_TMP_FAIL(tmp_ret = wbp_->free_page(fd_, page_entry_idxs[i], ObTmpFilePageUniqKey(free_page_virtual_id), unused_page_id))) { + LOG_WARN("fail to free page", KR(tmp_ret), K(fd_), K(i), K(free_page_virtual_id), K(page_entry_idxs[i])); + } else { + free_page_virtual_id += 1; + } + } + } + } // end update meta data. + + // reset allocation failure status + ret = is_alloc_failed && OB_SUCC(ret) ? OB_ALLOCATE_TMP_FILE_PAGE_FAILED : ret; + return ret; +} + +int ObSharedNothingTmpFile::alloc_and_write_pages_(const ObTmpFileIOCtx &io_ctx, + ObIArray &alloced_page_id, + int64_t &actual_write_size) +{ + int ret = OB_SUCCESS; + int64_t write_size = 0; + int64_t new_page_virtual_id = get_page_virtual_id_from_offset_(file_size_, false /*is_open_interval*/); + uint32_t previous_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + alloced_page_id.reset(); + actual_write_size = 0; + + if (OB_UNLIKELY(has_unfinished_page_())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("the file has unfinished page", KR(ret), K(fd_)); + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == new_page_virtual_id)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("new page virtual id is invalid", KR(ret), K(fd_), K(file_size_), K(new_page_virtual_id)); + } else if (OB_UNLIKELY(!io_ctx.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(fd_), K(io_ctx)); + } else { + const int64_t expected_write_size = io_ctx.get_todo_size(); + while (OB_SUCC(ret) && actual_write_size < expected_write_size && + alloced_page_id.count() < ObTmpFileGlobal::TMP_FILE_WRITE_BATCH_PAGE_NUM) { + char *page_buf = nullptr; + uint32_t new_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + if (OB_FAIL(wbp_->alloc_page(fd_, ObTmpFilePageUniqKey(new_page_virtual_id), new_page_id, page_buf))) { + LOG_WARN("fail to alloc_page", KR(ret), K(fd_), K(new_page_virtual_id), + K(actual_write_size), K(expected_write_size)); + } else if (OB_ISNULL(page_buf)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("page_buf is null", KR(ret), K(fd_), KP(page_buf)); + } else if (OB_UNLIKELY(new_page_id == ObTmpFileGlobal::INVALID_PAGE_ID)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid page id", KR(ret), K(fd_), K(new_page_id)); + } else { + int64_t write_size = common::min(ObTmpFileGlobal::PAGE_SIZE, + expected_write_size - actual_write_size); + const char *write_buf = io_ctx.get_todo_buffer() + actual_write_size; + + if (OB_UNLIKELY(!io_ctx.check_buf_range_valid(write_buf, write_size))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid buf range", KR(ret), K(fd_), K(write_buf), K(write_size), K(io_ctx)); + } else if (FALSE_IT(MEMCPY(page_buf, write_buf, write_size))) { + } else if (FALSE_IT(actual_write_size += write_size)) { + } else if (OB_FAIL(wbp_->notify_dirty(fd_, new_page_id, ObTmpFilePageUniqKey(new_page_virtual_id)))) { + LOG_WARN("fail to notify dirty", KR(ret), K(fd_), K(new_page_id), K(new_page_virtual_id)); + } else if (previous_page_id != ObTmpFileGlobal::INVALID_PAGE_ID && + OB_FAIL(wbp_->link_page(fd_, new_page_id, previous_page_id, + ObTmpFilePageUniqKey(new_page_virtual_id - 1)))) { + LOG_WARN("fail to link page", KR(ret), K(fd_), K(new_page_id), K(previous_page_id), + K(new_page_virtual_id)); + } else if (OB_FAIL(alloced_page_id.push_back(new_page_id))) { + LOG_WARN("wbp fail to push back page id", KR(ret), K(fd_), K(new_page_id)); + } else { + previous_page_id = new_page_id; + new_page_virtual_id += 1; + } + } + + if (OB_FAIL(ret) && OB_NOT_NULL(page_buf)) { + int tmp_ret = OB_SUCCESS; + uint32_t tmp_pid = ObTmpFileGlobal::INVALID_PAGE_ID; // unused + if (OB_TMP_FAIL(wbp_->free_page(fd_, new_page_id, ObTmpFilePageUniqKey(new_page_virtual_id), tmp_pid))) { + LOG_WARN("fail to free page", KR(tmp_ret), K(fd_), K(new_page_id)); + } + } + } // end while + } + LOG_DEBUG("alloc_and_write", KR(ret), K(fd_), K(io_ctx), K(file_size_), + K(end_page_id_), K(actual_write_size), K(alloced_page_id)); + return ret; +} + +// Attention!!!! +// in order to prevent concurrency problems of eviction list +// from the operation from eviction manager and flush manager, +// before eviction manager calls this function, +// it removes the file's node from its' eviction list, +// but still keeps the file's `is_in_data_eviction_list_` be true. +// when this function run over, +// if `remain_flushed_page_num` > 0, this function will reinsert node into the list of eviction manager again; +// otherwise, this function will set `is_in_data_eviction_list_` be false. +int ObSharedNothingTmpFile::evict_data_pages(const int64_t expected_evict_page_num, + int64_t &actual_evict_page_num, + int64_t &remain_flushed_page_num) +{ + int ret = OB_SUCCESS; + bool lock_last_page = false; + common::TCRWLock::WLockGuard meta_lock_guard(meta_lock_); + const int64_t end_page_virtual_id = get_page_virtual_id_from_offset_(file_size_, true /*is_open_interval*/); + actual_evict_page_num = 0; + remain_flushed_page_num = 0; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObSharedNothingTmpFile has not been inited", KR(ret), K(tenant_id_), K(fd_), KPC(this)); + } else if (OB_UNLIKELY(is_deleting_)) { + // actual_evict_page_num = 0; + // remain_flushed_page_num = 0; + is_in_data_eviction_list_ = false; + LOG_INFO("try to evict data pages when file is deleting", K(fd_), K(tenant_id_), KPC(this)); + } else if (OB_UNLIKELY(expected_evict_page_num <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(fd_), K(expected_evict_page_num)); + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == end_page_virtual_id)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid end page virtual id", KR(ret), K(fd_), K(end_page_virtual_id), K(file_size_), KPC(this)); + } else if (OB_UNLIKELY(0 == flushed_data_page_num_)) { + is_in_data_eviction_list_ = false; + LOG_INFO("tmp file has no flushed data pages need to be evicted", KR(ret), K(fd_), K(flushed_data_page_num_), KPC(this)); + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_PAGE_ID == flushed_page_id_ || + ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == flushed_page_virtual_id_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid flush status", KR(ret), K(fd_), K(flushed_page_id_), K(flushed_page_virtual_id_), KPC(this)); + } else if (OB_UNLIKELY(!is_in_data_eviction_list_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN( "the file is not in data eviction list", K(fd_), K(is_in_data_eviction_list_), KPC(this)); + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == begin_page_virtual_id_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid begin page virtual id", KR(ret), K(fd_), K(begin_page_virtual_id_), KPC(this)); + } else { + bool need_to_evict_last_page = false; + if (flushed_data_page_num_ == cached_page_nums_ && expected_evict_page_num >= flushed_data_page_num_ && + wbp_->is_cached(fd_, end_page_id_, ObTmpFilePageUniqKey(end_page_virtual_id))) { + need_to_evict_last_page = true; + // last page should be evicted, try to lock the last page. + if (OB_FAIL(last_page_lock_.trylock())) { + // fail to get the lock of last data page, it will not be evicted. + ret = OB_SUCCESS; + } else { + lock_last_page = true; + } + } + + int64_t remain_evict_page_num = 0; + if (lock_last_page) { + remain_evict_page_num = flushed_data_page_num_; + } else if (need_to_evict_last_page) { // need_to_evict_last_page && !lock_last_page + remain_evict_page_num = flushed_data_page_num_ - 1; + } else { + remain_evict_page_num = MIN(flushed_data_page_num_, expected_evict_page_num); + } + + const int64_t evict_end_virtual_id = begin_page_virtual_id_ + remain_evict_page_num; + if (OB_FAIL(ret)) { + } else if (OB_FAIL(page_idx_cache_.truncate(evict_end_virtual_id))) { + LOG_WARN("fail to truncate page idx cache", KR(ret), K(fd_), K(evict_end_virtual_id), KPC(this)); + } + + // evict data pages + const bool evict_last_page = lock_last_page; + while (OB_SUCC(ret) && remain_evict_page_num > 0) { + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + uint32_t first_cached_page_idx = ObTmpFileGlobal::INVALID_PAGE_ID; + + if (OB_UNLIKELY(!wbp_->is_cached(fd_, begin_page_id_, ObTmpFilePageUniqKey(begin_page_virtual_id_)))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("the page is not cached", KR(ret), K(fd_), K(begin_page_id_), K(begin_page_virtual_id_), KPC(this)); + } else if (OB_FAIL(wbp_->free_page(fd_, begin_page_id_, ObTmpFilePageUniqKey(begin_page_virtual_id_), next_page_id))) { + LOG_WARN("fail to free page", KR(ret), K(fd_), K(begin_page_id_), K(begin_page_virtual_id_), K(next_page_id), KPC(this)); + } else { + if (begin_page_id_ == flushed_page_id_) { + flushed_page_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + flushed_page_virtual_id_ = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + } + begin_page_id_ = next_page_id; + if (ObTmpFileGlobal::INVALID_PAGE_ID == begin_page_id_) { + end_page_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + begin_page_virtual_id_ = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + } else { + begin_page_virtual_id_ += 1; + } + actual_evict_page_num++; + remain_evict_page_num--; + flushed_data_page_num_--; + cached_page_nums_--; + } + } // end while + + if (OB_FAIL(ret)) { + } else if (flushed_data_page_num_ > 0) { + if (OB_FAIL(eviction_mgr_->add_file(false/*is_meta*/, *this))) { + LOG_WARN("fail to add file to eviction mgr", KR(ret), KPC(this)); + } + } else { + is_in_data_eviction_list_ = false; + LOG_DEBUG("all data pages of file have been evicted", KPC(this)); + } + + if (OB_SUCC(ret)) { + remain_flushed_page_num = flushed_data_page_num_; + } + + if (lock_last_page) { + last_page_lock_.unlock(); + } + + LOG_DEBUG("evict data page of tmp file over", KR(ret), K(fd_), K(is_deleting_), + K(expected_evict_page_num), K(actual_evict_page_num), + K(remain_flushed_page_num), + K(begin_page_id_), K(begin_page_virtual_id_), + K(flushed_page_id_), K(flushed_page_virtual_id_), KPC(this)); + } + + return ret; +} + +// Attention!!!! +// in order to prevent concurrency problems of eviction list +// from the operation from eviction manager and flush manager, +// before eviction manager calls this function, +// it removes the file's node from its' eviction list, +// but still keeps the file's `is_in_meta_eviction_list_` be true. +// when this function run over, +// if `remain_flushed_page_num` > 0, this function will reinsert node into the list of eviction manager again; +// otherwise, this function will set `is_in_meta_eviction_list_` be false. +int ObSharedNothingTmpFile::evict_meta_pages(const int64_t expected_evict_page_num, + int64_t &actual_evict_page_num) +{ + int ret = OB_SUCCESS; + // TODO: wanyue.wy, wuyuefei.wyf + // if meta_tree_ could protect the consistency by itself, this lock could be removed + common::TCRWLock::WLockGuard meta_lock_guard(meta_lock_); + actual_evict_page_num = 0; + int64_t total_need_evict_page_num = 0; + int64_t total_need_evict_rightmost_page_num = 0; + int64_t remain_need_evict_page_num = 0; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObSharedNothingTmpFile has not been inited", KR(ret), K(tenant_id_), K(fd_), KPC(this)); + } else if (OB_UNLIKELY(is_deleting_)) { + // actual_evict_page_num = 0; + is_in_meta_eviction_list_ = false; + LOG_INFO("try to evict data pages when file is deleting", K(fd_), K(tenant_id_), KPC(this)); + } else if (OB_UNLIKELY(expected_evict_page_num <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(fd_), K(expected_evict_page_num)); + } else if (OB_UNLIKELY(!is_in_meta_eviction_list_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN( "the file is not in meta eviction list", K(fd_), K(is_in_meta_eviction_list_), KPC(this)); + } else if (OB_FAIL(meta_tree_.get_need_evict_page_num(total_need_evict_page_num, total_need_evict_rightmost_page_num))) { + LOG_WARN( "fail to get need evict page num", KR(ret), K(fd_), KPC(this)); + } else if (OB_UNLIKELY(total_need_evict_page_num <= 0)) { + is_in_meta_eviction_list_ = false; + LOG_INFO("meta tree has no flushed pages need to be evicted", K(fd_), K(total_need_evict_page_num), KPC(this)); + } else if (OB_FAIL(meta_tree_.evict_meta_pages(expected_evict_page_num, + ObTmpFileTreeEvictType::FULL, actual_evict_page_num))) { + LOG_WARN("fail to evict meta pages", KR(ret), K(fd_), K(expected_evict_page_num), KPC(this)); + } else { + remain_need_evict_page_num = total_need_evict_page_num - actual_evict_page_num; + if (remain_need_evict_page_num > 0) { + if (OB_FAIL(eviction_mgr_->add_file(true/*is_meta*/, *this))) { + LOG_WARN("fail to add file to eviction mgr", KR(ret), K(fd_), KPC(this)); + } + } else { + is_in_meta_eviction_list_ = false; + LOG_DEBUG("all meta pages are evicted", KPC(this)); + } + LOG_DEBUG("evict meta page of tmp file over", KR(ret), K(fd_), K(is_deleting_), + K(expected_evict_page_num), K(actual_evict_page_num), K(remain_need_evict_page_num), KPC(this)); + } + + return ret; +} + +// Attention!! +// 1. currently truncate() only gc the memory resource for data page. it doesn't free disk space. +// thus, it will not modify meta tree and not affect flushing mechanism +// 2. if truncate_offset is not the begin or end offset of page, +// we will fill zero from begin_offset to truncate_offset for truncated page in wbp. +// 3. truncate_offset is a open interval number, which means the offset before than it need to be truncated +int ObSharedNothingTmpFile::truncate(const int64_t truncate_offset) +{ + int ret = OB_SUCCESS; + SpinWLockGuard truncate_lock_guard(truncate_lock_); + ObSpinLockGuard last_page_lock_guard(last_page_lock_); + common::TCRWLock::WLockGuard guard(meta_lock_); + int64_t wbp_begin_offset = cal_wbp_begin_offset_(); + LOG_INFO("start to truncate a temporary file", KR(ret), K(fd_), K(truncate_offset), K(wbp_begin_offset), KPC(this)); + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObSharedNothingTmpFile has not been inited", KR(ret), K(tenant_id_), K(fd_), KPC(this)); + } else if (OB_UNLIKELY(is_deleting_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("attempt to truncate a deleting file", KR(ret), K(tenant_id_), K(fd_), KPC(this)); + } else if (OB_UNLIKELY(wbp_begin_offset < 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected wbp begin offset", KR(ret), K(fd_), K(wbp_begin_offset), K(truncate_offset), KPC(this)); + } else if (OB_UNLIKELY(truncate_offset <= 0 || truncate_offset > file_size_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid truncate_offset", KR(ret), K(fd_), K(truncate_offset), K(file_size_), KPC(this)); + } else if (OB_UNLIKELY(truncate_offset <= truncated_offset_)) { + // do nothing + } else { + // release disk space between truncated_offset_ and truncate_offset + if (OB_FAIL(meta_tree_.truncate(truncated_offset_, truncate_offset))) { + LOG_WARN("fail to truncate meta tree", KR(ret), K(fd_), K(truncated_offset_), K(truncate_offset), KPC(this)); + } + + if (FAILEDx(inner_truncate_(truncate_offset, wbp_begin_offset))) { + LOG_WARN("fail to truncate data page", KR(ret), K(fd_), K(truncate_offset), KPC(this)); + } else { + truncated_offset_ = truncate_offset; + } + } + + LOG_INFO("truncate a temporary file over", KR(ret), K(truncate_offset), K(wbp_begin_offset), KPC(this)); + return ret; +} + +int ObSharedNothingTmpFile::inner_truncate_(const int64_t truncate_offset, const int64_t wbp_begin_offset) +{ + int ret = OB_SUCCESS; + uint32_t truncate_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + + if (OB_UNLIKELY(wbp_begin_offset < 0 || truncate_offset <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid offset", KR(ret), K(fd_), K(wbp_begin_offset), K(truncate_offset)); + } else if (truncate_offset > wbp_begin_offset) { + const int64_t truncate_page_virtual_id = get_page_virtual_id_from_offset_(truncate_offset, + true /*is_open_interval*/); + if (OB_FAIL(page_idx_cache_.binary_search(truncate_page_virtual_id, truncate_page_id))) { + LOG_WARN("fail to find page index in array", KR(ret), K(fd_), K(truncate_page_virtual_id)); + } else if (ObTmpFileGlobal::INVALID_PAGE_ID != truncate_page_id) { + // the page index of truncate_offset is in the range of cached page index. + // truncate all page indexes whose offset is smaller than truncate_offset. + if (OB_FAIL(page_idx_cache_.truncate(truncate_page_virtual_id+1))) { + LOG_WARN("fail to truncate page idx cache", KR(ret), K(fd_), K(truncate_page_virtual_id)); + } + } else { // ObTmpFileGlobal::INVALID_PAGE_ID == truncate_page_id + // the page index of truncate_offset is smaller than the smallest cached page index. + // we need to find truncate_page_id by iterating wbp_. + if (OB_FAIL(wbp_->get_page_id_by_virtual_id(fd_, truncate_page_virtual_id, begin_page_id_, truncate_page_id))) { + LOG_WARN("fail to get page id by virtual id", KR(ret), K(fd_), K(truncate_offset), K(begin_page_id_)); + } + } + + if (OB_SUCC(ret)) { + // truncate complete pages except the last page + while(OB_SUCC(ret) && begin_page_id_ != truncate_page_id) { + if (OB_FAIL(truncate_the_first_wbp_page_())) { + LOG_WARN("fail to truncate first wbp page", KR(ret), K(fd_), + K(begin_page_id_), K(truncate_page_id), K(end_page_id_)); + } + } + + // truncate the last page + if (OB_SUCC(ret)) { + const int64_t truncate_offset_in_page = get_page_offset_from_file_or_block_offset_(truncate_offset); + if (OB_UNLIKELY(truncate_page_virtual_id != begin_page_virtual_id_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected begin page virtual id", KR(ret), K(fd_), K(truncate_page_virtual_id), K(begin_page_virtual_id_)); + } else if (0 == truncate_offset_in_page) { + if (OB_FAIL(truncate_the_first_wbp_page_())) { + LOG_WARN("fail to truncate first wbp page", KR(ret), K(fd_), + K(begin_page_id_), K(truncate_offset_in_page), K(end_page_id_)); + } + } else { // truncate_offset_in_page is in the middle of page, fill zero before the truncate_offset of page + if (OB_FAIL(wbp_->truncate_page(fd_, truncate_page_id, + ObTmpFilePageUniqKey(truncate_page_virtual_id), + truncate_offset_in_page))) { + LOG_WARN("fail to truncate page", KR(ret), K(fd_), K(truncate_page_id), K(truncate_offset_in_page)); + } else { + LOG_INFO("truncate part of page", KR(ret), K(truncate_page_id), K(truncate_offset), + K(truncate_offset_in_page), K(wbp_begin_offset), KPC(this)); + } + } + } + } + } + return ret; +} + +int ObSharedNothingTmpFile::truncate_the_first_wbp_page_() +{ + int ret = OB_SUCCESS; + bool is_flushed_page = false; + bool is_write_back_page = false; + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + + if (ObTmpFileGlobal::INVALID_PAGE_ID == begin_page_id_ || + ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == begin_page_virtual_id_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("begin_page_id_ is already INVALID", KR(ret), K(fd_), K(begin_page_id_), + K(begin_page_virtual_id_)); + } else if (wbp_->is_cached(fd_, begin_page_id_, ObTmpFilePageUniqKey(begin_page_virtual_id_))) { + // [begin_page_id_, flushed_page_id_] has been flushed + is_flushed_page = true; + } else if (wbp_->is_write_back(fd_, begin_page_id_, ObTmpFilePageUniqKey(begin_page_virtual_id_))) { + is_write_back_page = true; + } + + if (FAILEDx(wbp_->free_page(fd_, begin_page_id_, ObTmpFilePageUniqKey(begin_page_virtual_id_), next_page_id))) { + LOG_WARN("fail to free page", KR(ret), K(fd_), K(begin_page_id_), K(begin_page_virtual_id_)); + } else { + if (is_flushed_page) { + if (flushed_data_page_num_ <= 0) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("flushed_data_page_num_ is unexpected", KR(ret), KPC(this)); + } else { + flushed_data_page_num_--; + if (0 == flushed_data_page_num_) { + if (OB_UNLIKELY(flushed_page_id_ != begin_page_id_)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("flushed_page_id_ or flushed_data_page_num_ is unexpected", KR(ret), KPC(this)); + } else { + flushed_page_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + flushed_page_virtual_id_ = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + LOG_INFO("all flushed page has been truncated", KPC(this)); + } + } + } + } else if (is_write_back_page) { + write_back_data_page_num_--; + } + + if (OB_SUCC(ret)) { + cached_page_nums_--; + begin_page_id_ = next_page_id; + if (ObTmpFileGlobal::INVALID_PAGE_ID == begin_page_id_) { + end_page_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + begin_page_virtual_id_ = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + } else { + begin_page_virtual_id_ += 1; + } + } + } + return ret; +} + +void ObSharedNothingTmpFile::update_read_offset(int64_t read_offset) +{ + common::TCRWLock::WLockGuard guard(meta_lock_); + if (read_offset > read_offset_) { + read_offset_ = read_offset; + } +} + +void ObSharedNothingTmpFile::get_dirty_meta_page_num_with_lock(int64_t &non_rightmost_dirty_page_num, int64_t &rightmost_dirty_page_num) +{ + common::TCRWLock::RLockGuard guard(meta_lock_); + get_dirty_meta_page_num(non_rightmost_dirty_page_num, rightmost_dirty_page_num); +} + +void ObSharedNothingTmpFile::get_dirty_meta_page_num(int64_t &non_rightmost_dirty_page_num, int64_t &rightmost_dirty_page_num) const +{ + int ret = OB_SUCCESS; + int64_t total_need_flush_page_num = 0; + if (OB_FAIL(meta_tree_.get_need_flush_page_num(total_need_flush_page_num, rightmost_dirty_page_num))) { + non_rightmost_dirty_page_num = 1; + rightmost_dirty_page_num = 1; + LOG_WARN("fail to get need flush page num", KR(ret), KPC(this)); + } else { + non_rightmost_dirty_page_num = total_need_flush_page_num - rightmost_dirty_page_num; + } +} + +int64_t ObSharedNothingTmpFile::get_dirty_data_page_size_with_lock() +{ + common::TCRWLock::RLockGuard guard(meta_lock_); + return get_dirty_data_page_size(); +} + +int64_t ObSharedNothingTmpFile::get_dirty_data_page_size() const +{ + int ret = OB_SUCCESS; + + int64_t dirty_size = 0; + // cached_page_nums == flushed_data_page_num + dirty_data_page_num + if (0 == cached_page_nums_ || flushed_data_page_num_ == cached_page_nums_) { + dirty_size = 0; + } else { + if (0 == file_size_ % ObTmpFileGlobal::PAGE_SIZE) { + dirty_size = + (cached_page_nums_ - flushed_data_page_num_ - write_back_data_page_num_) * ObTmpFileGlobal::PAGE_SIZE; + } else { + dirty_size = + (cached_page_nums_ - flushed_data_page_num_ - write_back_data_page_num_ - 1) * ObTmpFileGlobal::PAGE_SIZE + + file_size_ % ObTmpFileGlobal::PAGE_SIZE; + } + } + + if (dirty_size < 0) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("dirty_size is unexpected", KR(ret), K(dirty_size), KPC(this)); + dirty_size = 1; + } + return dirty_size; +} + +int64_t ObSharedNothingTmpFile::get_file_size() +{ + common::TCRWLock::RLockGuard guard(meta_lock_); + return file_size_; +} + +int64_t ObSharedNothingTmpFile::cal_wbp_begin_offset() +{ + common::TCRWLock::RLockGuard guard(meta_lock_); + return cal_wbp_begin_offset_(); +} + +int64_t ObSharedNothingTmpFile::cal_wbp_begin_offset_() const +{ + int ret = OB_SUCCESS; + int64_t res = -1; + if (0 == cached_page_nums_) { + res = file_size_; + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == begin_page_virtual_id_ || + begin_page_virtual_id_ * ObTmpFileGlobal::PAGE_SIZE != + get_page_end_offset_by_file_or_block_offset_(file_size_) - + cached_page_nums_ * ObTmpFileGlobal::PAGE_SIZE)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("begin_page_offset_in_file_ is unexpected", KR(ret), + K(begin_page_virtual_id_), K(file_size_), K(cached_page_nums_), KPC(this)); + } else { + res = begin_page_virtual_id_ * ObTmpFileGlobal::PAGE_SIZE; + } + + return res; +} + +int ObSharedNothingTmpFile::remove_flush_node(const bool is_meta) +{ + int ret = OB_SUCCESS; + common::TCRWLock::WLockGuard guard(meta_lock_); + if (OB_FAIL(flush_prio_mgr_->remove_file(is_meta, *this))) { + LOG_WARN("fail to remove flush node", KR(ret), K(is_meta), KPC(this)); + } + return ret; +} + +int ObSharedNothingTmpFile::reinsert_flush_node(const bool is_meta) +{ + int ret = OB_SUCCESS; + common::TCRWLock::WLockGuard guard(meta_lock_); + if (OB_FAIL(reinsert_flush_node_(is_meta))) { + LOG_WARN("fail to reinsert flush node", KR(ret), K(is_meta), KPC(this)); + } + + return ret; +} + +int ObSharedNothingTmpFile::reinsert_flush_node_(const bool is_meta) +{ + int ret = OB_SUCCESS; + ObTmpFileNode &flush_node = is_meta ? meta_flush_node_ : data_flush_node_; + + if (OB_UNLIKELY(nullptr != flush_node.get_next())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("flush node should not have next", KR(ret), K(fd_), K(is_meta)); + } else if (OB_UNLIKELY(is_deleting_)) { + // do nothing + } else if (is_meta) { + int64_t non_rightmost_dirty_page_num = 0; + int64_t rightmost_dirty_page_num = 0; + get_dirty_meta_page_num(non_rightmost_dirty_page_num, rightmost_dirty_page_num); + if (0 == non_rightmost_dirty_page_num && 0 == rightmost_dirty_page_num) { + // no need to reinsert + meta_page_flush_level_ = -1; + } else if (OB_FAIL(flush_prio_mgr_->insert_meta_flush_list(*this, non_rightmost_dirty_page_num, + rightmost_dirty_page_num))) { + LOG_WARN("fail to insert meta flush list", KR(ret), K(fd_), + K(non_rightmost_dirty_page_num), K(rightmost_dirty_page_num)); + } + } else { + int64_t dirty_page_size = get_dirty_data_page_size(); + if (0 == dirty_page_size) { + // no need to reinsert + data_page_flush_level_ = -1; + } else if (OB_FAIL(flush_prio_mgr_->insert_data_flush_list(*this, dirty_page_size))) { + LOG_WARN("fail to insert data flush list", KR(ret), K(fd_), K(dirty_page_size)); + } + } + + return ret; +} + +bool ObSharedNothingTmpFile::is_in_meta_flush_list() +{ + common::TCRWLock::RLockGuard guard(meta_lock_); + return OB_NOT_NULL(meta_flush_node_.get_next()); +} + +bool ObSharedNothingTmpFile::is_flushing() +{ + common::TCRWLock::RLockGuard guard(meta_lock_); + return inner_flush_ctx_.is_flushing(); +} + +// ATTENTION! need to be protected by meta_lock_ +int ObSharedNothingTmpFile::insert_or_update_data_flush_node_() +{ + int ret = OB_SUCCESS; + if (!inner_flush_ctx_.is_flushing()) { + int64_t dirty_data_page_size = get_dirty_data_page_size(); + if (data_page_flush_level_ >= 0 && OB_ISNULL(data_flush_node_.get_next())) { + // flush task mgr is processing this file. + // it will add this file to according flush list in the end + } else if (0 == dirty_data_page_size) { + // do nothing + } else if (data_page_flush_level_ < 0) { + if (OB_FAIL(flush_prio_mgr_->insert_data_flush_list(*this, dirty_data_page_size))) { + LOG_WARN("fail to get list idx", KR(ret), K(dirty_data_page_size), KPC(this)); + } + } else if (OB_FAIL(flush_prio_mgr_->update_data_flush_list(*this, dirty_data_page_size))) { + LOG_WARN("fail to update flush list", KR(ret), K(dirty_data_page_size), KPC(this)); + } + } + return ret; +} + +// ATTENTION! call this need to be protected it in meta_lock_ +int ObSharedNothingTmpFile::insert_or_update_meta_flush_node_() +{ + int ret = OB_SUCCESS; + if (!inner_flush_ctx_.is_flushing()) { + int64_t non_rightmost_dirty_page_num = 0; + int64_t rightmost_dirty_page_num = 0; + get_dirty_meta_page_num(non_rightmost_dirty_page_num, rightmost_dirty_page_num); + if (meta_page_flush_level_ >= 0 && OB_ISNULL(meta_flush_node_.get_next())) { + // flush task mgr is processing this file. + // it will add this file to according flush list in the end + } else if (0 == non_rightmost_dirty_page_num + rightmost_dirty_page_num) { + // do nothing + } else if (meta_page_flush_level_ < 0) { + if (OB_FAIL(flush_prio_mgr_->insert_meta_flush_list(*this, non_rightmost_dirty_page_num, + rightmost_dirty_page_num))) { + LOG_WARN("fail to get list idx", KR(ret), + K(non_rightmost_dirty_page_num), K(rightmost_dirty_page_num), KPC(this)); + } + } else if (OB_FAIL(flush_prio_mgr_->update_meta_flush_list(*this, non_rightmost_dirty_page_num, + rightmost_dirty_page_num))) { + LOG_WARN("fail to update flush list", KR(ret), + K(non_rightmost_dirty_page_num), K(rightmost_dirty_page_num), KPC(this)); + } + } + return ret; +} + +int ObSharedNothingTmpFile::cal_end_position_( + ObArray &flush_infos, + const int64_t start_pos, + int64_t &end_pos, + int64_t &flushed_data_page_num) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(start_pos >= flush_infos.count() || start_pos < 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid flushed info num", + KR(ret), K(start_pos), K(flush_infos.count()), K(inner_flush_ctx_), KPC(this)); + } else if (OB_UNLIKELY(flush_infos[start_pos].has_data() && flush_infos[start_pos].has_meta())) { + // we require that each flush info only represents one type of data or meta + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid flush info", KR(ret), K(start_pos), K(flush_infos[start_pos]), KPC(this)); + } else if (flush_infos[start_pos].update_meta_data_done_) { + for (end_pos = start_pos; OB_SUCC(ret) && end_pos < flush_infos.count(); end_pos++) { + if (OB_UNLIKELY(flush_infos[end_pos].has_data() && flush_infos[end_pos].has_meta()) || + OB_UNLIKELY(!flush_infos[end_pos].has_data() && !flush_infos[end_pos].has_meta())) { + ret = OB_ERR_UNEXPECTED; // flush_info contains one and only one type of pages + LOG_WARN("invalid flush info", KR(ret), K(flush_infos[end_pos]), KPC(this)); + } else if (flush_infos[end_pos].update_meta_data_done_) { + if (flush_infos[end_pos].has_data()) { + flushed_data_page_num += flush_infos[end_pos].flush_data_page_num_; + } + } else { + break; + } + } // end for + } + return ret; +} + +int ObSharedNothingTmpFile::update_meta_after_flush(const int64_t info_idx, const bool is_meta, bool &reset_ctx) +{ + int ret = OB_SUCCESS; + reset_ctx = false; + common::TCRWLock::WLockGuard guard(meta_lock_); + int64_t start_pos = is_meta ? inner_flush_ctx_.meta_finished_continuous_flush_info_num_ + : inner_flush_ctx_.data_finished_continuous_flush_info_num_; + int64_t end_pos = start_pos; + ObArray &flush_infos_ = is_meta ? inner_flush_ctx_.meta_flush_infos_ + : inner_flush_ctx_.data_flush_infos_; + int64_t flushed_data_page_num = 0; + if (OB_UNLIKELY(info_idx < 0 || info_idx >= flush_infos_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid idx", KR(ret), K(info_idx), K(is_meta), KPC(this)); + } else if (FALSE_IT(flush_infos_[info_idx].update_meta_data_done_ = true)) { + } else if (OB_FAIL(cal_end_position_(flush_infos_, start_pos, end_pos, flushed_data_page_num))) { + LOG_WARN("fail to cal end position for update after flush", KR(ret), K(info_idx), KPC(this)); + } else if (start_pos < end_pos) { // have new continuous finished flush infos + if (is_meta && OB_FAIL(update_meta_tree_after_flush_(start_pos, end_pos))) { + // meta tree may be updated before file meta infos if data_flush_info IO hang or + // data_flush_info could not insert data items + LOG_WARN("fail to update meta tree", KR(ret), K(start_pos), K(end_pos), KPC(this)); + } else if (!is_meta && OB_FAIL(update_file_meta_after_flush_(start_pos, end_pos, flushed_data_page_num))) { + LOG_WARN("fail to update file meta", KR(ret), K(start_pos), K(end_pos), K(flushed_data_page_num), KPC(this)); + } else { + reset_ctx = true; + } + LOG_DEBUG("update meta based a set of flush info over", KR(ret), K(info_idx), K(start_pos), + K(end_pos), K(inner_flush_ctx_), KPC(this)); + } + + if (OB_FAIL(inner_flush_ctx_.update_finished_continuous_flush_info_num(is_meta, end_pos))) { + LOG_WARN("fail to update finished continuous flush info num", KR(ret), K(start_pos), K(end_pos), KPC(this)); + } else if (inner_flush_ctx_.is_all_finished()) { + if (OB_ISNULL(data_flush_node_.get_next()) && OB_FAIL(reinsert_flush_node_(false /*is_meta*/))) { + LOG_ERROR("fail to reinsert data flush node", KR(ret), K(is_meta), K(inner_flush_ctx_), KPC(this)); + } else if (OB_ISNULL(meta_flush_node_.get_next()) && OB_FAIL(reinsert_flush_node_(true /*is_meta*/))) { + LOG_ERROR("fail to reinsert meta flush node", KR(ret), K(is_meta), K(inner_flush_ctx_), KPC(this)); + } else { + inner_flush_ctx_.reset(); + } + } + + LOG_DEBUG("update_meta_after_flush finish", KR(ret), K(info_idx), K(is_meta), K(inner_flush_ctx_), KPC(this)); + return ret; +} + +// 1. skip truncated flush infos +// 2. abort updating file meta for tail data page if it is appending written after generating flushing task +int ObSharedNothingTmpFile::remove_useless_page_in_data_flush_infos_(const int64_t start_pos, + const int64_t end_pos, + const int64_t flushed_data_page_num, + int64_t &new_start_pos, + int64_t &new_flushed_data_page_num) +{ + int ret = OB_SUCCESS; + ObArray &flush_infos_ = inner_flush_ctx_.data_flush_infos_; + new_start_pos = start_pos; + new_flushed_data_page_num = flushed_data_page_num; + + if (OB_UNLIKELY(start_pos >= end_pos || end_pos < 1 || flushed_data_page_num <= 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid param", KR(ret), K(fd_), K(start_pos), K(end_pos), K(flushed_data_page_num)); + } else if (start_pos >= flush_infos_.count() || end_pos > flush_infos_.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid param", KR(ret), K(fd_), K(start_pos), K(end_pos), K(flush_infos_), KPC(this)); + } else { + // skip truncated flush infos + for (int64_t i = start_pos; OB_SUCC(ret) && i < end_pos; i++) { + int64_t flushed_start_offset = get_page_begin_offset_by_virtual_id_(flush_infos_[start_pos].flush_virtual_page_id_); + int64_t flushed_end_offset = flushed_start_offset + + flush_infos_[i].flush_data_page_num_ * ObTmpFileGlobal::PAGE_SIZE; + if (truncated_offset_ <= flushed_start_offset) { + // the following flushing pages are not affected by truncate_offset + break; + } else if (truncated_offset_ < flushed_end_offset) { + // although a page is truncated partly, it will also be flushed whole page + int64_t flush_info_flushed_size = flushed_end_offset - truncated_offset_; + int64_t flush_info_data_page_num = + common::upper_align(flush_info_flushed_size, ObTmpFileGlobal::PAGE_SIZE) / ObTmpFileGlobal::PAGE_SIZE; + uint32_t flush_info_virtual_page_id = + common::lower_align(truncated_offset_, ObTmpFileGlobal::PAGE_SIZE) / ObTmpFileGlobal::PAGE_SIZE; + new_flushed_data_page_num -= (flush_infos_[i].flush_data_page_num_ - flush_info_data_page_num); + + LOG_INFO("due to truncating, modify flush info", KR(ret), K(fd_), + K(start_pos), K(end_pos), K(flushed_data_page_num), K(i), + K(new_start_pos), K(new_flushed_data_page_num), + K(truncated_offset_), K(flushed_start_offset), K(flushed_end_offset), K(flush_infos_[i]), + K(flush_info_data_page_num), K(flush_info_virtual_page_id)); + flush_infos_[i].flush_data_page_num_ = flush_info_data_page_num; + flush_infos_[i].flush_virtual_page_id_ = flush_info_virtual_page_id; + break; + } else { + // all pages of this flush_info have been truncated, abort it + new_start_pos++; + new_flushed_data_page_num -= flush_infos_[i].flush_data_page_num_; + LOG_INFO("due to truncating, abort flush info", KR(ret), K(fd_), + K(start_pos), K(end_pos), K(flushed_data_page_num), K(i), + K(new_start_pos), K(new_flushed_data_page_num), + K(truncated_offset_), K(flushed_start_offset), K(flushed_end_offset), K(flush_infos_[i])); + } + } // end for + + // abort updating file meta for tail data page + if (OB_SUCC(ret) && new_flushed_data_page_num > 0 && new_start_pos < end_pos) { + if (flush_infos_[end_pos - 1].file_size_ > 0 && flush_infos_[end_pos - 1].file_size_ != file_size_) { + // the last page has been written after flush task has been generated. + // we will discard the flushed last page when update meta data + // (append write has correct the meta data of the last page, here only need to ignore it) + + int64_t discard_page_virtual_id = flush_infos_[end_pos - 1].flush_virtual_page_id_ + + flush_infos_[end_pos - 1].flush_data_page_num_ - 1; + uint32_t discard_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + + if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == discard_page_virtual_id)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("discard page virtual id is invalid", KR(ret), K(fd_), + K(discard_page_virtual_id), K(flush_infos_[end_pos - 1])); + } else if (OB_FAIL(get_physical_page_id_in_wbp(discard_page_virtual_id, discard_page_id))) { + LOG_WARN("fail to get physical page id in wbp", KR(ret), K(fd_), K(discard_page_virtual_id)); + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_PAGE_ID == discard_page_id)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("discard page id is invalid", KR(ret), K(fd_), K(discard_page_id)); + } else if (OB_UNLIKELY(!wbp_->is_dirty(fd_, discard_page_id, ObTmpFilePageUniqKey(discard_page_virtual_id)))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("discard page is not dirty", KR(ret), K(fd_), K(discard_page_id), K(discard_page_virtual_id)); + } else { + new_flushed_data_page_num -= 1; + flush_infos_[end_pos - 1].flush_data_page_num_ -= 1; + } + } + } + } + return ret; +} + +int ObSharedNothingTmpFile::update_file_meta_after_flush_(const int64_t start_pos, + const int64_t end_pos, + const int64_t flushed_data_page_num) +{ + int ret = OB_SUCCESS; + ObArray &flush_infos_ = inner_flush_ctx_.data_flush_infos_; + LOG_DEBUG("update_file_meta_after_flush start", + KR(ret), K(start_pos), K(end_pos), K(flushed_data_page_num), KPC(this)); + int64_t new_start_pos = start_pos; + int64_t new_flushed_data_page_num = flushed_data_page_num; + + if (OB_UNLIKELY(start_pos >= end_pos || end_pos < 1 || flushed_data_page_num <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid param", KR(ret), K(fd_), K(end_pos), K(flushed_data_page_num)); + } else if (start_pos >= flush_infos_.count() || end_pos > flush_infos_.count()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid param", KR(ret), K(start_pos), K(end_pos), K(flush_infos_), KPC(this)); + } else if (OB_FAIL(remove_useless_page_in_data_flush_infos_(start_pos, end_pos, flushed_data_page_num, + new_start_pos, new_flushed_data_page_num))) { + LOG_WARN("fail to remove useless page in flush infos", KR(ret), K(fd_), K(start_pos), K(end_pos), + K(flushed_data_page_num), K(end_pos)); + } else if (0 == new_flushed_data_page_num) { + // do nothing + } else if (OB_UNLIKELY(new_flushed_data_page_num < 0 || new_start_pos >= end_pos)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid param", KR(ret), K(fd_), K(start_pos), K(end_pos), K(flushed_data_page_num), + K(new_start_pos), K(new_flushed_data_page_num)); + } else { // exist multiple continuous pages have been flushed over + uint32_t last_flushed_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + uint32_t cur_flush_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + int64_t cur_flush_page_virtual_id = flush_infos_[new_start_pos].flush_virtual_page_id_; + if (OB_FAIL(get_physical_page_id_in_wbp(flush_infos_[new_start_pos].flush_virtual_page_id_, cur_flush_page_id))) { + LOG_WARN("fail to get physical page id in wbp", + KR(ret),K(fd_), K(start_pos), K(end_pos), K(flushed_data_page_num), + K(new_start_pos), K(new_flushed_data_page_num), + K(flush_infos_[new_start_pos])); + } else { + int64_t write_back_succ_data_page_num = 0; + + // update each flush info + for (int64_t i = new_start_pos; OB_SUCC(ret) && i < end_pos; ++i) { + const InnerFlushInfo& flush_info = flush_infos_[i]; + const int64_t cur_flushed_data_page_num = flush_info.flush_data_page_num_; + + if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_PAGE_ID == cur_flush_page_id)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid next flush page id", KR(ret), K(fd_), K(cur_flush_page_id), + K(begin_page_id_), K(flushed_page_id_), K(end_page_id_)); + } else { + int64_t cur_page_virtual_id_in_flush_info = cur_flush_page_virtual_id; + // update each page of flush info + for (int64_t j = 0; OB_SUCC(ret) && j < cur_flushed_data_page_num; ++j) { + if (OB_FAIL(wbp_->notify_write_back_succ(fd_, cur_flush_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id_in_flush_info)))) { + LOG_WARN("fail to mark page as clean", KR(ret), K(fd_), + K(flushed_page_id_), K(cur_page_virtual_id_in_flush_info), K(cur_flush_page_id), + K(flushed_data_page_num), K(new_flushed_data_page_num)); + } else if (FALSE_IT(last_flushed_page_id = cur_flush_page_id)) { + } else if (OB_FAIL(wbp_->get_next_page_id(fd_, cur_flush_page_id, + ObTmpFilePageUniqKey(cur_page_virtual_id_in_flush_info), cur_flush_page_id))) { + LOG_WARN("fail to get next page id", KR(ret), K(fd_), K(cur_flush_page_id), + K(cur_page_virtual_id_in_flush_info)); + } else { + cur_page_virtual_id_in_flush_info += 1; + } + } // end for + if (OB_SUCC(ret)) { + write_back_succ_data_page_num += cur_flushed_data_page_num; + cur_flush_page_virtual_id += cur_flushed_data_page_num; + } + } + } // end for + + // update file meta + if (OB_SUCC(ret)) { + if (write_back_succ_data_page_num != new_flushed_data_page_num) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("write_back_succ_data_page_num not correct", + KR(ret), K(fd_), K(write_back_succ_data_page_num), + K(flushed_data_page_num), K(new_flushed_data_page_num), KPC(this)); + } else { + flushed_page_id_ = last_flushed_page_id; + flushed_page_virtual_id_ = cur_flush_page_virtual_id - 1; + flushed_data_page_num_ += new_flushed_data_page_num; + write_back_data_page_num_ -= new_flushed_data_page_num; + if (write_back_data_page_num_ < 0) { + int tmp_ret = OB_ERR_UNEXPECTED; + LOG_WARN("write_back_data_page_num_ is unexpected", KR(tmp_ret), K(fd_), + K(write_back_data_page_num_), K(flushed_data_page_num), K(new_flushed_data_page_num), KPC(this)); + write_back_data_page_num_ = 0; + } + + if (!is_deleting_ && !is_in_data_eviction_list_ && OB_ISNULL(data_eviction_node_.get_next())) { + if (OB_FAIL(eviction_mgr_->add_file(false/*is_meta*/, *this))) { + LOG_WARN("fail to insert into eviction list", KR(ret), K(fd_)); + } else { + is_in_data_eviction_list_ = true; + } + } + } + } // update file meta over + } + } // handle continuous success flush infos over + + LOG_DEBUG("update_file_meta_after_flush finish", KR(ret), K(fd_), + K(start_pos), K(end_pos), K(flushed_data_page_num), + K(new_start_pos), K(new_flushed_data_page_num), KPC(this)); + return ret; +} + +int ObSharedNothingTmpFile::update_meta_tree_after_flush_(const int64_t start_pos, const int64_t end_pos) +{ + int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + + ObArray &flush_infos_ = inner_flush_ctx_.meta_flush_infos_; + for (int64_t i = start_pos; i < end_pos; i++) { + // ATTENTION! need to alloc memory inside, caller must retry update meta data if alloc fail + if (OB_FAIL(meta_tree_.update_after_flush(flush_infos_[i].flush_meta_page_array_))) { + LOG_ERROR("fail to update meta items", KR(ret), K(fd_), K(flush_infos_[i]), KPC(this)); + } else { + LOG_INFO("succ to update meta items", KR(ret), K(fd_), K(flush_infos_[i]), KPC(this)); + } + } + + if (OB_SUCC(ret)) { + if (!is_deleting_ && !is_in_meta_eviction_list_ && OB_ISNULL(meta_eviction_node_.get_next())) { + int64_t total_need_evict_page_num = 1; + int64_t total_need_evict_rightmost_page_num = 1; + if (OB_TMP_FAIL(meta_tree_.get_need_evict_page_num(total_need_evict_page_num, + total_need_evict_rightmost_page_num))) { + LOG_ERROR("fail to get_need_evict_page_num", KR(tmp_ret), + K(total_need_evict_page_num), K(total_need_evict_rightmost_page_num), KPC(this)); + } + + if (total_need_evict_page_num > 0) { + if (OB_FAIL(eviction_mgr_->add_file(true/*is_meta*/, *this))) { + LOG_WARN("fail to insert into eviction list", KR(ret), K(fd_)); + } else { + is_in_meta_eviction_list_ = true; + } + } + } + } + return ret; +} + +int ObSharedNothingTmpFile::generate_data_flush_info( + ObTmpFileFlushTask &flush_task, + ObTmpFileFlushInfo &info, + ObTmpFileDataFlushContext &data_flush_context, + const int64_t flush_sequence, + const bool need_flush_tail) +{ + int ret = OB_SUCCESS; + info.reset(); + + if (!truncate_lock_.try_rdlock()) { + ret = OB_ITER_END; + LOG_WARN("fail to get truncate lock", KR(ret), K(fd_), KPC(this)); + } else { + common::TCRWLock::RLockGuard guard(meta_lock_); + uint32_t copy_begin_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + int64_t copy_begin_page_virtual_id = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + + uint32_t copy_end_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + if (OB_FAIL(cal_next_flush_page_id_from_flush_ctx_or_file_(data_flush_context, + copy_begin_page_id, + copy_begin_page_virtual_id))) { + LOG_WARN("fail to calculate next_flush_page_id", KR(ret), + K(flush_task), K(info), K(data_flush_context), KPC(this)); + } else if (ObTmpFileGlobal::INVALID_PAGE_ID == copy_begin_page_id) { + ret = OB_ITER_END; + LOG_DEBUG("no more data to flush", KR(ret), K(fd_), KPC(this)); + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == copy_begin_page_virtual_id)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("next_flush_page_virtual_id is invalid", KR(ret), K(fd_), K(copy_begin_page_id), + K(copy_begin_page_virtual_id), K(flush_task), K(info), K(data_flush_context), + K(flush_sequence), K(need_flush_tail), KPC(this)); + } else if (OB_FAIL(get_flush_end_page_id_(copy_end_page_id, need_flush_tail))) { + LOG_WARN("fail to get_flush_end_page_id_", KR(ret), + K(flush_task), K(info), K(data_flush_context), KPC(this)); + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_FLUSH_SEQUENCE != inner_flush_ctx_.flush_seq_ + && flush_sequence != inner_flush_ctx_.flush_seq_ + && flush_sequence != flush_task.get_flush_seq())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("flush sequence not match", + KR(ret), K(inner_flush_ctx_.flush_seq_), K(flush_task), KPC(this)); + } else if (OB_FAIL(copy_flush_data_from_wbp_(flush_task, info, data_flush_context, + copy_begin_page_id, copy_begin_page_virtual_id, + copy_end_page_id, + flush_sequence, need_flush_tail))) { + LOG_WARN("fail to copy flush data from wbp", KR(ret), + K(flush_task), K(info), K(data_flush_context), KPC(this)); + } + if (OB_FAIL(ret)) { + truncate_lock_.unlock(); + } + LOG_DEBUG("generate flush data info end", KR(ret), K(fd_), + K(data_flush_context), K(info), K(flush_task), KPC(this)); + } + return ret; +} + +int ObSharedNothingTmpFile::copy_flush_data_from_wbp_( + ObTmpFileFlushTask &flush_task, + ObTmpFileFlushInfo &info, + ObTmpFileDataFlushContext &data_flush_context, + const uint32_t copy_begin_page_id, + const int64_t copy_begin_page_virtual_id, + const uint32_t copy_end_page_id, + const int64_t flush_sequence, + const bool need_flush_tail) +{ + int ret = OB_SUCCESS; + char *buf = flush_task.get_data_buf(); + int64_t write_offset = flush_task.get_total_page_num() * ObTmpFileGlobal::PAGE_SIZE; + + bool has_last_page_lock = false; + int64_t next_disk_page_id = flush_task.get_next_free_page_id(); + int64_t flushing_page_num = 0; + int64_t origin_info_num = inner_flush_ctx_.data_flush_infos_.size(); + int64_t cur_page_file_offset = -1; + uint32_t cur_flushed_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + uint32_t cur_page_id = copy_begin_page_id; + int64_t cur_page_virtual_id = copy_begin_page_virtual_id; + + if (OB_ISNULL(buf) || OB_UNLIKELY(OB_SERVER_BLOCK_MGR.get_macro_block_size() <= write_offset)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid buf or write_offset", KR(ret), KP(buf), K(write_offset), K(flush_task), KPC(this)); + } else if (OB_FAIL(inner_flush_ctx_.data_flush_infos_.push_back(InnerFlushInfo()))) { + LOG_WARN("fail to push back empty flush info", KR(ret), K(fd_), K(info), K(flush_task), KPC(this)); + } + while (OB_SUCC(ret) && cur_page_id != copy_end_page_id && write_offset < OB_SERVER_BLOCK_MGR.get_macro_block_size()) { + if (need_flush_tail && cur_page_id == end_page_id_ && file_size_ % ObTmpFileGlobal::PAGE_SIZE != 0) { + if (OB_SUCC(last_page_lock_.trylock())) { + has_last_page_lock = true; + } else { + LOG_WARN("fail to get last page lock", KR(ret), K(fd_)); + ret = OB_SUCCESS; // ignore error to continue flushing the copied data + break; + } + } + char *page_buf = nullptr; + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + bool original_state_is_dirty = wbp_->is_dirty(fd_, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id)); + if (OB_FAIL(wbp_->read_page(fd_, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id), page_buf, next_page_id))) { + LOG_WARN("fail to read page", KR(ret), K(fd_), K(cur_page_id)); + } else if (OB_FAIL(wbp_->notify_write_back(fd_, cur_page_id, ObTmpFilePageUniqKey(cur_page_virtual_id)))) { + LOG_WARN("fail to notify write back", KR(ret), K(fd_), K(cur_page_id)); + } else if (OB_UNLIKELY(!flush_task.check_buf_range_valid(buf, ObTmpFileGlobal::PAGE_SIZE))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid buffer range", KR(ret), K(fd_), K(write_offset), KP(buf)); + } else { + // ObTmpPageCacheKey cache_key(flush_task.get_block_index(), + // write_offset / ObTmpFileGlobal::PAGE_SIZE, tenant_id_); + // ObTmpPageCacheValue cache_value(page_buf); + // ObTmpPageCache::get_instance().try_put_page_to_cache(cache_key, cache_value); + + MEMCPY(buf + write_offset, page_buf, ObTmpFileGlobal::PAGE_SIZE); + write_offset += ObTmpFileGlobal::PAGE_SIZE; + flushing_page_num += 1; + cur_flushed_page_id = cur_page_id; + cur_page_id = next_page_id; + cur_page_virtual_id += 1; + if (original_state_is_dirty) { + write_back_data_page_num_++; + } + } + } + + if (OB_SUCC(ret) && 0 == flushing_page_num) { + ret = OB_ITER_END; + } + if (OB_SUCC(ret) && ObTmpFileGlobal::INVALID_PAGE_ID == cur_flushed_page_id) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("cur_flushed_page_id is unexpected", KR(ret), K(cur_flushed_page_id), KPC(this)); + } + + if (OB_SUCC(ret)) { + int64_t flush_info_idx = inner_flush_ctx_.data_flush_infos_.size() - 1; + // set flush_info in flush_task + info.flush_data_page_disk_begin_id_ = next_disk_page_id; + info.flush_data_page_num_ = flushing_page_num; + info.batch_flush_idx_ = flush_info_idx; + info.flush_virtual_page_id_ = copy_begin_page_virtual_id; + info.has_last_page_lock_ = has_last_page_lock; + // record file_size to check if the last page is appended while flushing + if (has_last_page_lock) { + info.file_size_ = file_size_; + } + + info.fd_ = fd_; + // set flush_info in file inner_flush_ctx + if (OB_FAIL(info.file_handle_.init(this))) { + LOG_WARN("fail to init tmp file handle", KR(ret), K(fd_), K(flush_task), KPC(this)); + } else if (OB_FAIL(inner_flush_ctx_.data_flush_infos_.at(flush_info_idx).init_by_tmp_file_flush_info(info))) { + LOG_WARN("fail to init_by_tmp_file_flush_info", KR(ret), K(fd_), K(flush_task), KPC(this)); + } + } + + if (OB_SUCC(ret)) { + // maintain flushed_page_id recorded in flush_mgr + data_flush_context.set_flushed_page_id(cur_flushed_page_id); + data_flush_context.set_flushed_page_virtual_id(cur_page_virtual_id - 1); + data_flush_context.set_next_flush_page_id(cur_page_id); + data_flush_context.set_next_flush_page_virtual_id(cur_page_virtual_id); + data_flush_context.set_is_valid(true); + if (has_last_page_lock) { + data_flush_context.set_has_flushed_last_partially_written_page(true); + } + flush_task.set_data_length(write_offset); + + inner_flush_ctx_.flush_seq_ = flush_sequence; + } else { + LOG_WARN("fail to generate data flush info", KR(ret), K(fd_), K(need_flush_tail), + K(flush_sequence), K(data_flush_context), K(info), K(flush_task), KPC(this)); + if (inner_flush_ctx_.data_flush_infos_.size() == origin_info_num + 1) { + inner_flush_ctx_.data_flush_infos_.pop_back(); + } + if (has_last_page_lock) { + last_page_lock_.unlock(); + } + } + return ret; +} + +int ObSharedNothingTmpFile::generate_meta_flush_info( + ObTmpFileFlushTask &flush_task, + ObTmpFileFlushInfo &info, + ObTmpFileTreeFlushContext &meta_flush_context, + const int64_t flush_sequence, + const bool need_flush_tail) +{ + int ret = OB_SUCCESS; + info.reset(); + + common::TCRWLock::RLockGuard guard(meta_lock_); + ObArray &flush_infos_ = inner_flush_ctx_.meta_flush_infos_; + int64_t origin_info_num = flush_infos_.size(); + + const int64_t block_index = flush_task.get_block_index(); + char *buf = flush_task.get_data_buf(); + int64_t write_offset = flush_task.get_total_page_num() * ObTmpFileGlobal::PAGE_SIZE; + + ObTmpFileTreeEvictType flush_type = need_flush_tail ? + ObTmpFileTreeEvictType::FULL : ObTmpFileTreeEvictType::MAJOR; + if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_FLUSH_SEQUENCE != inner_flush_ctx_.flush_seq_ + && flush_sequence != inner_flush_ctx_.flush_seq_ + && flush_sequence != flush_task.get_flush_seq())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("flush sequence not match", KR(ret), K(flush_sequence), K(inner_flush_ctx_.flush_seq_), + K(flush_task), KPC(this)); + } else if (OB_ISNULL(buf) || OB_UNLIKELY(OB_SERVER_BLOCK_MGR.get_macro_block_size() <= write_offset)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid buf or write_offset", KR(ret), KP(buf), K(write_offset), K(flush_task), KPC(this)); + } else if (OB_FAIL(flush_infos_.push_back(InnerFlushInfo()))) { + LOG_WARN("fail to push back empty flush info", KR(ret), K(fd_), K(info), K(flush_task), KPC(this)); + } else if (OB_FAIL(meta_tree_.flush_meta_pages_for_block(block_index, flush_type, buf, write_offset, + meta_flush_context, info.flush_meta_page_array_))) { + LOG_WARN("fail to flush meta pages for block", KR(ret), K(fd_), K(flush_task), K(meta_flush_context), KPC(this)); + } else if (0 == info.flush_meta_page_array_.count()) { + ret = OB_ITER_END; + } + + if (OB_SUCC(ret)) { + int64_t flush_info_idx = flush_infos_.size() - 1; + info.batch_flush_idx_ = flush_info_idx; + + info.fd_ = fd_; + // set flush_info in flush_task + if (OB_FAIL(info.file_handle_.init(this))) { + LOG_WARN("fail to init tmp file handle", KR(ret), K(fd_), K(flush_task), KPC(this)); + } else if (OB_FAIL(inner_flush_ctx_.meta_flush_infos_.at(flush_info_idx).init_by_tmp_file_flush_info(info))) { + LOG_WARN("fail to init_by_tmp_file_flush_info", KR(ret), K(fd_), K(flush_task), KPC(this)); + } + } + + if (OB_SUCC(ret)) { + flush_task.set_data_length(write_offset); + inner_flush_ctx_.flush_seq_ = flush_sequence; + } else { + LOG_WARN("fail to generate meta flush info", KR(ret), K(fd_), K(flush_task), + K(meta_flush_context), K(flush_sequence), K(need_flush_tail)); + if (flush_infos_.size() == origin_info_num + 1) { + flush_infos_.pop_back(); + } + } + + LOG_INFO("generate flush meta info end", KR(ret), K(fd_), K(need_flush_tail), + K(inner_flush_ctx_), K(meta_flush_context), K(info), K(flush_task), KPC(this)); + return ret; +} + +int ObSharedNothingTmpFile::insert_meta_tree_item(const ObTmpFileFlushInfo &info, int64_t block_index) +{ + int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + + common::TCRWLock::WLockGuard guard(meta_lock_); + if (info.has_data()) { + ObSharedNothingTmpFileDataItem data_item; + data_item.block_index_ = block_index; + data_item.physical_page_id_ = info.flush_data_page_disk_begin_id_; + data_item.physical_page_num_ = info.flush_data_page_num_; + data_item.virtual_page_id_ = info.flush_virtual_page_id_; + ObArray data_items; + + if (OB_FAIL(meta_tree_.prepare_for_insert_items())) { + LOG_WARN("fail to prepare for insert items", KR(ret), K(info), K(block_index), KPC(this)); + } else if (OB_FAIL(data_items.push_back(data_item))) { + LOG_WARN("fail to push back data item", KR(ret), K(info), K(block_index), KPC(this)); + } else if (OB_FAIL(meta_tree_.insert_items(data_items))) { + LOG_WARN("fail to insert data items", KR(ret), K(info), K(block_index), KPC(this)); + } + + if (OB_SUCC(ret) && info.has_last_page_lock_) { + last_page_lock_.unlock(); + } + + if (OB_SUCC(ret)) { + truncate_lock_.unlock(); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("flush info does not contain data info", KR(ret), K(info), KPC(this)); + } + + if (!is_deleting_ && !is_in_meta_eviction_list_ && OB_ISNULL(meta_eviction_node_.get_next())) { + int64_t total_need_evict_page_num = 1; + int64_t total_need_evict_rightmost_page_num = 1; + if (OB_TMP_FAIL(meta_tree_.get_need_evict_page_num(total_need_evict_page_num, + total_need_evict_rightmost_page_num))) { + LOG_ERROR("fail to get_need_evict_page_num", KR(tmp_ret), + K(total_need_evict_page_num), K(total_need_evict_rightmost_page_num), KPC(this)); + } + + if (total_need_evict_page_num > 0) { + if (OB_TMP_FAIL(eviction_mgr_->add_file(true/*is_meta*/, *this))) { + LOG_WARN("fail to insert into eviction list", KR(ret), K(fd_), KPC(this)); + } else { + is_in_meta_eviction_list_ = true; + } + } + } + + // reinsert meta flush node during flushing to allow meta pages to be flushed if + // insert_meta_tree_item need tp allocate new meta pages + if (!is_deleting_ && OB_ISNULL(meta_flush_node_.get_next())) { + if (OB_TMP_FAIL(reinsert_flush_node_(true/*is_meta*/))) { + LOG_WARN("fail to reinsert flush node", KR(ret), K(fd_), K(info), K(block_index), KPC(this)); + } + } + + LOG_DEBUG("insert_meta_tree_item end", KR(ret), KPC(this)); + return ret; +} + +int ObSharedNothingTmpFile::cal_next_flush_page_id_from_flush_ctx_or_file_( + const ObTmpFileDataFlushContext &data_flush_context, + uint32_t &next_flush_page_id, + int64_t &next_flush_page_virtual_id) +{ + int ret = OB_SUCCESS; + if (data_flush_context.is_valid()) { + // use next_flush_page_id from data_flush_ctx + if (data_flush_context.has_flushed_last_partially_written_page()) { + next_flush_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + next_flush_page_virtual_id = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + } else { + next_flush_page_id = data_flush_context.get_next_flush_page_id(); + next_flush_page_virtual_id = data_flush_context.get_next_flush_page_virtual_id(); + int64_t truncate_page_virtual_id = get_page_virtual_id_from_offset_(truncated_offset_, + false /*is_open_interval*/); + if (ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID != truncate_page_virtual_id && + truncate_page_virtual_id > data_flush_context.get_flushed_page_virtual_id()) { + if (OB_FAIL(get_next_flush_page_id_(next_flush_page_id, next_flush_page_virtual_id))) { + LOG_ERROR("origin next flush page has been truncated, fail to get_next_flush_page_id_", + KR(ret), K(data_flush_context), KPC(this)); + } else { + LOG_INFO("origin next flush page has been truncated", + KR(ret), K(next_flush_page_id), K(next_flush_page_virtual_id), K(data_flush_context), KPC(this)); + } + } + + if (OB_SUCC(ret) && ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID != truncate_page_virtual_id && + truncate_page_virtual_id <= data_flush_context.get_flushed_page_virtual_id()) { + uint32_t last_flushed_page_id = data_flush_context.get_flushed_page_id(); + int64_t last_flushed_page_virtual_id = data_flush_context.get_flushed_page_virtual_id(); + if (ObTmpFileGlobal::INVALID_PAGE_ID != last_flushed_page_id) { + if (!wbp_->is_write_back(fd_, last_flushed_page_id, ObTmpFilePageUniqKey(last_flushed_page_virtual_id))) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("last flushed page state not match", + KR(ret), K(last_flushed_page_id), K(data_flush_context), KPC(this)); + } + } + } + } + } else { + // cal next_flush_page_id from file meta when doing flush for the first time + if (OB_FAIL(get_next_flush_page_id_(next_flush_page_id, next_flush_page_virtual_id))) { + LOG_WARN("fail to get_next_flush_page_id_", KR(ret), K(fd_), K(data_flush_context)); + } + } + + if (OB_SUCC(ret) && ObTmpFileGlobal::INVALID_PAGE_ID != next_flush_page_id) { + if (!wbp_->is_dirty(fd_, next_flush_page_id, ObTmpFilePageUniqKey(next_flush_page_virtual_id)) && + !wbp_->is_write_back(fd_, next_flush_page_id, ObTmpFilePageUniqKey(next_flush_page_virtual_id))) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("next_flush_page_id state not validate", + KR(ret), K(next_flush_page_id), K(data_flush_context), KPC(this)); + } + } + + return ret; +} + +// output next page id after flushed_page_id_ in normal case，or output flushed_page_id for write operation appending last page +int ObSharedNothingTmpFile::get_next_flush_page_id_(uint32_t& next_flush_page_id, int64_t& next_flush_page_virtual_id) const +{ + int ret = OB_SUCCESS; + next_flush_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + next_flush_page_virtual_id = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + if (ObTmpFileGlobal::INVALID_PAGE_ID == flushed_page_id_) { + // all pages are dirty + if (OB_UNLIKELY((ObTmpFileGlobal::INVALID_PAGE_ID == begin_page_id_ && + ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID != begin_page_virtual_id_) || + (ObTmpFileGlobal::INVALID_PAGE_ID != begin_page_id_ && + ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == begin_page_virtual_id_) )) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("begin_page_id_ and begin_page_virtual_id_ not match", KR(ret), K(begin_page_id_), K(begin_page_virtual_id_), KPC(this)); + } else { + next_flush_page_id = begin_page_id_; + next_flush_page_virtual_id = begin_page_virtual_id_; + } + } else if (wbp_->is_dirty(fd_, flushed_page_id_, ObTmpFilePageUniqKey(flushed_page_virtual_id_))) { + // start from flushed_page_id_ if flushed_page_id_ is dirty caused by appending write + if (OB_UNLIKELY((ObTmpFileGlobal::INVALID_PAGE_ID == flushed_page_id_ && + ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID != flushed_page_virtual_id_) || + (ObTmpFileGlobal::INVALID_PAGE_ID != flushed_page_id_ && + ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == flushed_page_virtual_id_) )) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("flushed_page_id_ and flushed_page_virtual_id_ not match", + KR(ret), K(flushed_page_id_), K(flushed_page_virtual_id_), KPC(this)); + } else { + next_flush_page_id = flushed_page_id_; + next_flush_page_virtual_id = flushed_page_virtual_id_; + } + } else if (OB_FAIL(wbp_->get_next_page_id(fd_, flushed_page_id_, ObTmpFilePageUniqKey(flushed_page_virtual_id_), next_flush_page_id))){ + // start from the next page, could return INVALID_PAGE_ID if flushed_page_id_ == end_page_id_ + LOG_WARN("fail to get next page id", KR(ret), K(fd_), K(begin_page_id_), K(flushed_page_id_), K(end_page_id_)); + } else if (ObTmpFileGlobal::INVALID_PAGE_ID == next_flush_page_id) { + next_flush_page_virtual_id = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + } else { + next_flush_page_virtual_id = flushed_page_virtual_id_ + 1; + } + + LOG_DEBUG("get_next_flush_page_id_", KR(ret), K(fd_), K(begin_page_id_), K(flushed_page_id_), KPC(this)); + return ret; +} + +int ObSharedNothingTmpFile::get_physical_page_id_in_wbp(const int64_t virtual_page_id, uint32_t& page_id) const +{ + int ret = OB_SUCCESS; + int64_t end_page_virtual_id = cached_page_nums_ == 0 ? + ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID : + get_page_virtual_id_from_offset_(file_size_, true /*is_open_interval*/); + page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + + if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == virtual_page_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(fd_), K(virtual_page_id)); + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == begin_page_virtual_id_ || + ObTmpFileGlobal::INVALID_PAGE_ID == begin_page_id_ || + ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == end_page_virtual_id || + ObTmpFileGlobal::INVALID_PAGE_ID == end_page_id_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("no pages exist in wbp", KR(ret), K(begin_page_id_), K(begin_page_virtual_id_), + K(end_page_id_), K(end_page_virtual_id), KPC(this)); + } else if (OB_UNLIKELY(virtual_page_id < begin_page_virtual_id_ || virtual_page_id > end_page_virtual_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("the page doesn't exist in wbp", KR(ret), K(virtual_page_id), K(end_page_virtual_id), K(begin_page_virtual_id_), KPC(this)); + } else if (virtual_page_id == begin_page_virtual_id_) { + page_id = begin_page_id_; + } else if (virtual_page_id < flushed_page_virtual_id_) { + if (OB_FAIL(wbp_->get_page_id_by_virtual_id(fd_, virtual_page_id, begin_page_id_, page_id))) { + LOG_WARN("fail to get page id by virtual id", KR(ret), K(virtual_page_id), K(begin_page_id_), KPC(this)); + } + } else if (virtual_page_id == flushed_page_virtual_id_) { + page_id = flushed_page_id_; + } else if (virtual_page_id == end_page_virtual_id) { + page_id = end_page_id_; + } else { // virtual_page_id < end_page_virtual_id + if (OB_FAIL(wbp_->get_page_id_by_virtual_id(fd_, virtual_page_id, flushed_page_id_, page_id))) { + LOG_WARN("fail to get page id by virtual id", KR(ret), K(virtual_page_id), K(flushed_page_id_), KPC(this)); + } + } + + return ret; +} + +// output the last page to be flushed +int ObSharedNothingTmpFile::get_flush_end_page_id_(uint32_t& flush_end_page_id, const bool need_flush_tail) const +{ + int ret = OB_SUCCESS; + const int64_t INVALID_PAGE_ID = ObTmpFileGlobal::INVALID_PAGE_ID; + flush_end_page_id = INVALID_PAGE_ID; + if (file_size_ % ObTmpFileGlobal::PAGE_SIZE == 0) { + flush_end_page_id = INVALID_PAGE_ID; + } else { // determined whether to flush last page based on flag if last page is not full + flush_end_page_id = need_flush_tail ? INVALID_PAGE_ID : end_page_id_; + } + + LOG_DEBUG("get_flush_end_page_id_", K(fd_), K(need_flush_tail), K(flush_end_page_id), KPC(this)); + return ret; +} + +} // end namespace tmp_file +} // end namespace oceanbase diff --git a/src/storage/tmp_file/ob_shared_nothing_tmp_file.h b/src/storage/tmp_file/ob_shared_nothing_tmp_file.h new file mode 100644 index 000000000..03d05e1b9 --- /dev/null +++ b/src/storage/tmp_file/ob_shared_nothing_tmp_file.h @@ -0,0 +1,364 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_SHARE_NOTHING_TMP_FILE_H_ +#define OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_SHARE_NOTHING_TMP_FILE_H_ + +#include "storage/tmp_file/ob_tmp_file_cache.h" +#include "storage/tmp_file/ob_tmp_file_write_buffer_index_cache.h" +#include "storage/tmp_file/ob_tmp_file_global.h" +#include "storage/blocksstable/ob_macro_block_id.h" +#include "lib/lock/ob_spin_rwlock.h" +#include "lib/lock/ob_spin_lock.h" +#include "lib/lock/ob_tc_rwlock.h" +#include "storage/tmp_file/ob_tmp_file_meta_tree.h" + +namespace oceanbase +{ +namespace tmp_file +{ +class ObTmpFileIOCtx; +class ObTenantTmpFileManager; +class ObTmpWriteBufferPool; +class ObTmpFileBlockPageBitmap; +class ObTmpFileBlockManager; +class ObTmpFileEvictionManager; +class ObTmpFileFlushPriorityManager; +class ObTmpFileFlushInfo; +class ObTmpFileFlushTask; +class ObTmpFilePageCacheController; +class ObTmpFileFlushManager; +class ObTmpFileTreeFlushContext; +class ObTmpFileDataFlushContext; + +class ObSharedNothingTmpFile final +{ +public: + struct ObTmpFileNode : public ObDLinkBase + { + ObTmpFileNode(ObSharedNothingTmpFile &file) : file_(file) {} + ObSharedNothingTmpFile &file_; + }; + + struct InnerFlushInfo + { + public: + InnerFlushInfo(); + ~InnerFlushInfo() { reset(); } + void reset(); + bool has_data() const { return flush_data_page_num_ > 0; } + bool has_meta() const { return !flush_meta_page_array_.empty(); } + int init_by_tmp_file_flush_info(const ObTmpFileFlushInfo& flush_info); + TO_STRING_KV(K(flush_data_page_disk_begin_id_), K(flush_data_page_num_), K(flush_virtual_page_id_), K(file_size_), + K(flush_meta_page_array_)); + public: + bool update_meta_data_done_; + // information for updating data + int64_t flush_data_page_disk_begin_id_; // record begin page id in the macro block，for updating meta tree item + int64_t flush_data_page_num_; + int64_t flush_virtual_page_id_; // record virtual_page_id while copying data, pass to meta tree while inserting items + int64_t file_size_; // if file_size > 0, it means the last page is in flushing + // information for updating meta tree + ObArray flush_meta_page_array_; + }; + + struct InnerFlushContext + { + InnerFlushContext() + : flush_seq_(ObTmpFileGlobal::INVALID_FLUSH_SEQUENCE), + data_finished_continuous_flush_info_num_(0), + meta_finished_continuous_flush_info_num_(0), + data_flush_infos_(), + meta_flush_infos_() + { + data_flush_infos_.set_attr(ObMemAttr(MTL_ID(), "TmpFileFInfo")); + meta_flush_infos_.set_attr(ObMemAttr(MTL_ID(), "TmpFileFInfo")); + } + void reset(); + int update_finished_continuous_flush_info_num(const bool is_meta, const int64_t end_pos); + bool is_all_finished() const + { + return data_flush_infos_.size() == data_finished_continuous_flush_info_num_ && + meta_flush_infos_.size() == meta_finished_continuous_flush_info_num_; + } + bool is_flushing() const + { + return !data_flush_infos_.empty() || !meta_flush_infos_.empty(); + } + TO_STRING_KV(K(flush_seq_), K(data_flush_infos_.size()), K(meta_flush_infos_.size()), + K(data_finished_continuous_flush_info_num_), + K(meta_finished_continuous_flush_info_num_)); + int64_t flush_seq_; + int64_t data_finished_continuous_flush_info_num_; + int64_t meta_finished_continuous_flush_info_num_; + ObArray data_flush_infos_; + ObArray meta_flush_infos_; + }; +public: + ObSharedNothingTmpFile(); + ~ObSharedNothingTmpFile(); + int init(const uint64_t tenant_id, const int64_t fd, const int64_t dir_id, + ObTmpFileBlockManager *block_manager, + ObIAllocator *callback_allocator, + ObIAllocator *wbp_index_cache_allocator, + ObIAllocator *wbp_index_cache_bkt_allocator, + ObTmpFilePageCacheController *page_cache_controller); + int destroy(); + void reset(); + bool can_remove(); + bool is_deleting(); + int delete_file(); + +// XXX Currently, K(tmp_file) is used to print the ObSharedNothingTmpFile structure without holding +// the file lock. Before adding the print field, make sure it is thread-safe. + TO_STRING_KV(K(is_inited_), K(is_deleting_), + K(tenant_id_), K(dir_id_), K(fd_), + K(ref_cnt_), K(truncated_offset_), K(read_offset_), + K(file_size_), K(flushed_data_page_num_), K(write_back_data_page_num_), + K(cached_page_nums_), + K(begin_page_id_), K(begin_page_virtual_id_), + K(flushed_page_id_), K(flushed_page_virtual_id_), + K(end_page_id_), + K(data_page_flush_level_), K(meta_page_flush_level_), + KP(data_flush_node_.get_next()), + KP(meta_flush_node_.get_next()), + K(is_in_data_eviction_list_), K(is_in_meta_eviction_list_), + KP(data_eviction_node_.get_next()), + KP(meta_eviction_node_.get_next())); +// XXX Currently, K(tmp_file) is used to print the ObSharedNothingTmpFile structure without holding +// the file lock. Before adding the print field, make sure it is thread-safe. + +public: + int aio_pread(ObTmpFileIOCtx &io_ctx); + int aio_write(ObTmpFileIOCtx &io_ctx); + + int evict_data_pages(const int64_t expected_evict_page_num, + int64_t &actual_evict_page_num, + int64_t &remain_flushed_page_num); + int evict_meta_pages(const int64_t expected_evict_page_num, + int64_t &actual_evict_page_num); + // truncate offset is open interval + int truncate(const int64_t truncate_offset); + +public: + int update_meta_after_flush(const int64_t info_idx, const bool is_meta, bool &reset_ctx); + int generate_data_flush_info(ObTmpFileFlushTask &flush_task, + ObTmpFileFlushInfo &info, + ObTmpFileDataFlushContext &data_flush_context, + const int64_t flush_sequence, + const bool need_flush_tail); + int generate_meta_flush_info(ObTmpFileFlushTask &flush_task, + ObTmpFileFlushInfo &info, + ObTmpFileTreeFlushContext &meta_flush_context, + const int64_t flush_sequence, + const bool need_flush_tail); + int insert_meta_tree_item(const ObTmpFileFlushInfo &info, int64_t block_index); +public: + int remove_flush_node(const bool is_meta); + int reinsert_flush_node(const bool is_meta); + int insert_or_update_data_flush_node_(); + int insert_or_update_meta_flush_node_(); + bool is_in_meta_flush_list(); + bool is_flushing(); + OB_INLINE ObTmpFileNode &get_data_flush_node() { return data_flush_node_; } + OB_INLINE ObTmpFileNode &get_meta_flush_node() { return meta_flush_node_; } + OB_INLINE ObTmpFileNode &get_data_eviction_node() { return data_eviction_node_; } + OB_INLINE ObTmpFileNode &get_meta_eviction_node() { return meta_eviction_node_; } + OB_INLINE const ObTmpFileNode &get_data_eviction_node() const { return data_eviction_node_; } + OB_INLINE const ObTmpFileNode &get_meta_eviction_node() const { return meta_eviction_node_; } + + OB_INLINE int64_t get_fd() const { return fd_; } + OB_INLINE int64_t get_dir_id() const { return dir_id_; } + OB_INLINE void inc_ref_cnt() { ATOMIC_INC(&ref_cnt_); } + OB_INLINE void dec_ref_cnt() { ATOMIC_AAF(&ref_cnt_, -1); } + OB_INLINE int64_t get_ref_cnt() const { return ATOMIC_LOAD(&ref_cnt_); } + void update_read_offset(int64_t read_offset); + int64_t get_file_size(); + + OB_INLINE void set_data_page_flush_level(int64_t data_page_flush_level) + { + data_page_flush_level_ = data_page_flush_level; + } + OB_INLINE int64_t get_data_page_flush_level() const { return data_page_flush_level_; } + OB_INLINE void set_meta_page_flush_level(int64_t meta_page_flush_level) + { + meta_page_flush_level_ = meta_page_flush_level; + } + OB_INLINE int64_t get_meta_page_flush_level() const { return meta_page_flush_level_; } + int64_t get_dirty_data_page_size() const; + int64_t get_dirty_data_page_size_with_lock(); + void get_dirty_meta_page_num(int64_t &non_rightmost_dirty_page_num, int64_t &rightmost_dirty_page_num) const; + void get_dirty_meta_page_num_with_lock(int64_t &non_rightmost_dirty_page_num, int64_t &rightmost_dirty_page_num); + int64_t cal_wbp_begin_offset(); + +private: + int inner_read_truncated_part_(ObTmpFileIOCtx &io_ctx); + int inner_read_from_wbp_(ObTmpFileIOCtx &io_ctx); + int inner_read_from_disk_(const int64_t expected_read_disk_size, ObTmpFileIOCtx &io_ctx); + int inner_seq_read_from_block_(const int64_t block_index, + const int64_t begin_read_offset_in_block, const int64_t end_read_offset_in_block, + ObTmpFileIOCtx &io_ctx, int64_t &actual_read_size); + int inner_rand_read_from_block_(const int64_t block_index, + const int64_t begin_read_offset_in_block, const int64_t end_read_offset_in_block, + ObTmpFileIOCtx &io_ctx, int64_t &actual_read_size); + int collect_pages_in_block_(const int64_t block_index, + const int64_t begin_page_idx_in_block, + const int64_t end_page_idx_in_block, + ObTmpFileBlockPageBitmap &bitmap, + ObIArray &page_value_handles); + int inner_read_continuous_cached_pages_(const int64_t begin_read_offset_in_block, + const int64_t end_read_offset_in_block, + const ObArray &page_value_handles, + const int64_t start_array_idx, + ObTmpFileIOCtx &io_ctx); + int inner_read_continuous_uncached_pages_(const int64_t block_index, + const int64_t begin_read_offset_in_block, + const int64_t end_read_offset_in_block, + ObTmpFileIOCtx &io_ctx); + int inner_truncate_(const int64_t truncate_offset, const int64_t wbp_begin_offset); +private: + int inner_write_(ObTmpFileIOCtx &io_ctx); + int inner_fill_tail_page_(ObTmpFileIOCtx &io_ctx); + int load_disk_tail_page_and_rewrite_(ObTmpFileIOCtx &io_ctx); + int append_write_memory_tail_page_(ObTmpFileIOCtx &io_ctx); + int inner_write_continuous_pages_(ObTmpFileIOCtx &io_ctx); + int alloc_and_write_pages_(const ObTmpFileIOCtx &io_ctx, + ObIArray &alloced_page_id, + int64_t &actual_write_size); + int truncate_the_first_wbp_page_(); + + int copy_flush_data_from_wbp_(ObTmpFileFlushTask &flush_task, ObTmpFileFlushInfo &info, + ObTmpFileDataFlushContext &data_flush_context, + const uint32_t copy_begin_page_id, const int64_t copy_begin_page_virtual_id, + const uint32_t copy_end_page_id, const int64_t flush_sequence, const bool need_flush_tail); + int cal_end_position_(ObArray &flush_infos, + const int64_t start_pos, + int64_t &end_pos, + int64_t &flushed_data_page_num); + int cal_next_flush_page_id_from_flush_ctx_or_file_(const ObTmpFileDataFlushContext &data_flush_context, + uint32_t &next_flush_page_id, + int64_t &next_flush_page_virtual_id); + int get_next_flush_page_id_(uint32_t& next_flush_page_id, int64_t& next_flush_page_virtual_id) const; + int get_flush_end_page_id_(uint32_t& flush_end_page_id, const bool need_flush_tail) const; + int get_physical_page_id_in_wbp(const int64_t virtual_page_id, uint32_t& page_id) const ; + int remove_useless_page_in_data_flush_infos_(const int64_t start_pos, + const int64_t end_pos, + const int64_t flushed_data_page_num, + int64_t &new_start_pos, + int64_t &new_flushed_data_page_num); + int update_file_meta_after_flush_(const int64_t start_pos, + const int64_t end_pos, + const int64_t flushed_data_page_num); + int update_meta_tree_after_flush_(const int64_t start_pos, const int64_t end_pos); +private: + int reinsert_flush_node_(const bool is_meta); + OB_INLINE bool has_unfinished_page_() const { return file_size_ % ObTmpFileGlobal::PAGE_SIZE != 0; } + + int64_t cal_wbp_begin_offset_() const; + OB_INLINE int64_t get_page_begin_offset_by_file_or_block_offset_(const int64_t offset) const + { + return common::lower_align(offset, ObTmpFileGlobal::PAGE_SIZE); + } + OB_INLINE int64_t get_page_end_offset_by_file_or_block_offset_(const int64_t offset) const + { + return common::upper_align(offset, ObTmpFileGlobal::PAGE_SIZE); + } + OB_INLINE int64_t get_page_begin_offset_by_virtual_id_(const int64_t virtual_page_id) const + { + return virtual_page_id * ObTmpFileGlobal::PAGE_SIZE; + } + OB_INLINE int64_t get_page_end_offset_by_virtual_id_(const int64_t virtual_page_id) const + { + return (virtual_page_id + 1) * ObTmpFileGlobal::PAGE_SIZE; + } + OB_INLINE int64_t get_page_virtual_id_from_offset_(const int64_t page_offset_in_file, const bool is_open_interval) const + { + return is_open_interval ? + common::upper_align(page_offset_in_file, ObTmpFileGlobal::PAGE_SIZE) / ObTmpFileGlobal::PAGE_SIZE - 1 : + page_offset_in_file / ObTmpFileGlobal::PAGE_SIZE; + } + OB_INLINE int64_t get_page_id_in_block_(const int64_t page_offset_in_block) const + { + return page_offset_in_block / ObTmpFileGlobal::PAGE_SIZE; + } + OB_INLINE int64_t get_page_offset_from_file_or_block_offset_(const int64_t offset) const + { + return offset % ObTmpFileGlobal::PAGE_SIZE; + } + +private: + ObTmpFileBlockManager *tmp_file_block_manager_; + ObIAllocator *callback_allocator_; + ObTmpFilePageCacheController *page_cache_controller_; + ObTmpFileFlushPriorityManager *flush_prio_mgr_; + ObTmpFileEvictionManager *eviction_mgr_; + ObTmpWriteBufferPool *wbp_; + ObTmpFileWBPIndexCache page_idx_cache_; + bool is_inited_; + bool is_deleting_; + bool is_in_data_eviction_list_; + bool is_in_meta_eviction_list_; + int64_t data_page_flush_level_; + int64_t meta_page_flush_level_; + uint64_t tenant_id_; + int64_t dir_id_; + int64_t fd_; + int64_t ref_cnt_; + int64_t truncated_offset_; // read data befor truncated_offset will be set as 0 + int64_t read_offset_; // read offset is on the entire file + int64_t file_size_; // has written size of this file + int64_t flushed_data_page_num_; // equal to the page num between [begin_page_id_, flushed_page_id_] in wbp + int64_t write_back_data_page_num_; + int64_t cached_page_nums_; // page nums in write buffer pool + uint32_t begin_page_id_; // the first page index in write buffer pool + int64_t begin_page_virtual_id_; + uint32_t flushed_page_id_; // the last flushed page index in write buffer pool. + // specifically, if flushed_page_id_ == end_page_id_, + // it means the last page has been flushed. However, + // the last page might be appending written after flushing. + // thus, in this case, whether "flushed_page_id_" page in wbp is flushed + // needs to be checked with page status (with some get functions). + int64_t flushed_page_virtual_id_; + uint32_t end_page_id_; // the last page index in write buffer pool + ObSharedNothingTmpFileMetaTree meta_tree_; + ObTmpFileNode data_flush_node_; + ObTmpFileNode meta_flush_node_; + ObTmpFileNode data_eviction_node_; + ObTmpFileNode meta_eviction_node_; + common::TCRWLock meta_lock_; // handle conflicts between writing and reading meta tree and meta data of file + ObSpinLock last_page_lock_; // handle conflicts between writing and evicting for last page + ObSpinLock multi_write_lock_; // handle conflicts between multiple writes + SpinRWLock truncate_lock_; // handle conflicts between truncate and flushing + InnerFlushContext inner_flush_ctx_; // file-level flush context +}; + +class ObTmpFileHandle final +{ +public: + ObTmpFileHandle() : ptr_(nullptr) {} + ObTmpFileHandle(ObSharedNothingTmpFile *tmp_file); + ObTmpFileHandle(const ObTmpFileHandle &handle); + ObTmpFileHandle & operator=(const ObTmpFileHandle &other); + ~ObTmpFileHandle() { reset(); } + OB_INLINE ObSharedNothingTmpFile * get() const {return ptr_; } + bool is_inited() { return nullptr != ptr_; } + void reset(); + int init(ObSharedNothingTmpFile *tmp_file); + TO_STRING_KV(KP(ptr_)); +private: + ObSharedNothingTmpFile *ptr_; +}; + +} // end namespace tmp_file +} // end namespace oceanbase + +#endif // OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_SHARE_NOTHING_TMP_FILE_H_ diff --git a/src/storage/tmp_file/ob_tmp_file_block_manager.cpp b/src/storage/tmp_file/ob_tmp_file_block_manager.cpp new file mode 100644 index 000000000..7fe4d4ada --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_block_manager.cpp @@ -0,0 +1,922 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE + +#include "storage/tmp_file/ob_tmp_file_block_manager.h" +namespace oceanbase +{ +using namespace storage; +using namespace share; + +namespace tmp_file +{ +//-----------------------ObTmpFileBlockPageBitmapIterator----------------------- +int ObTmpFileBlockPageBitmapIterator::init(const ObTmpFileBlockPageBitmap *bitmap, + const int64_t start_idx, int64_t end_idx) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", KR(ret)); + } else if (OB_ISNULL(bitmap) || + OB_UNLIKELY(start_idx < 0 || end_idx < 0 || start_idx > end_idx || + end_idx >= ObTmpFileBlockPageBitmap::get_capacity())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), KPC(bitmap), K(start_idx), K(end_idx)); + } else { + is_inited_ = true; + bitmap_ = bitmap; + cur_idx_ = start_idx; + end_idx_ = end_idx; + } + return ret; +} + +void ObTmpFileBlockPageBitmapIterator::reset() +{ + bitmap_ = nullptr; + is_inited_ = false; + cur_idx_ = OB_INVALID_INDEX; + end_idx_ = OB_INVALID_INDEX; +} + +int ObTmpFileBlockPageBitmapIterator::next_range(bool &value, int64_t &start_page_id, int64_t &end_page_id) +{ + int ret = OB_SUCCESS; + start_page_id = OB_INVALID_INDEX; + end_page_id = OB_INVALID_INDEX; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (has_next()) { + start_page_id = cur_idx_; + if (OB_FAIL(bitmap_->get_value(cur_idx_++, value))) { + LOG_WARN("fail to get value", KR(ret), K(cur_idx_)); + } else { + while(cur_idx_ <= end_idx_ && OB_SUCC(ret)) { + bool cur_value = false; + if (OB_FAIL(bitmap_->get_value(cur_idx_, cur_value))) { + LOG_WARN("fail to get value", KR(ret), K(cur_idx_)); + } else if (cur_value != value) { + break; + } else { + cur_idx_ += 1; + } + } + end_page_id = cur_idx_ - 1; + } + } else { + ret = OB_ITER_END; + LOG_WARN("iter end", KR(ret)); + } + return ret; +} + +//---------------------------ObTmpFileBlockPageBitmap--------------------------- +int ObTmpFileBlockPageBitmap::get_value(const int64_t offset, bool &value) const +{ + int ret = OB_SUCCESS; + const int64_t capacity = PAGE_CAPACITY; + + if (OB_UNLIKELY(capacity <= offset || offset < 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid argument", KR(ret), K(capacity), K(offset)); + } else { + value = (bitmap_[offset / 8] & (1 << (offset % 8))) != 0; + } + return ret; +} + +int ObTmpFileBlockPageBitmap::set_bitmap(const int64_t offset, const bool value) +{ + int ret = OB_SUCCESS; + const int64_t capacity = PAGE_CAPACITY; + if (OB_UNLIKELY(capacity <= offset || offset < 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid argument", KR(ret), K(capacity), K(offset)); + } else if (value) { + bitmap_[offset / 8] |= (1 <<(offset % 8)); + } else { + bitmap_[offset / 8] &= ~(1 <<(offset % 8)); + } + return ret; +} + +int ObTmpFileBlockPageBitmap::set_bitmap_batch(const int64_t offset, const int64_t count, const bool value) +{ + int ret = OB_SUCCESS; + const int64_t capacity = PAGE_CAPACITY; + uint8_t start_byte_pos = offset / 8; + uint8_t end_byte_pos = (offset + count - 1) / 8; + uint8_t start_bit_pos = offset % 8; + uint8_t end_bit_pos = (offset + count - 1) % 8; + if (OB_UNLIKELY(capacity < (offset + count) || count <= 0 || offset < 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid argument", KR(ret), K(capacity), K(offset), K(count)); + } else if (start_byte_pos == end_byte_pos) { + // only one byte + uint8_t mask = ((1 << (end_bit_pos + 1)) - 1) & ~((1 << start_bit_pos) - 1); + if (value) { + bitmap_[start_byte_pos] |= mask; + } else { + bitmap_[start_byte_pos] &= ~mask; + } + } else { + uint8_t start_mask = ~((1 << start_bit_pos) - 1); + uint8_t end_mask = (1 << (end_bit_pos + 1)) - 1; + if (value) { + bitmap_[start_byte_pos] |= start_mask; + bitmap_[end_byte_pos] |= end_mask; + } else { + bitmap_[start_byte_pos] &= ~start_mask; + bitmap_[end_byte_pos] &= ~end_mask; + } + for (uint8_t i = start_byte_pos + 1; i < end_byte_pos; ++i) { + bitmap_[i] = value ? (1 << 8) - 1 : 0; + } + } + + return ret; +} + +int ObTmpFileBlockPageBitmap::is_all_true(const int64_t start, const int64_t end, bool &is_all_true) const +{ + int ret = OB_SUCCESS; + const int64_t capacity = PAGE_CAPACITY; + const uint8_t start_byte_pos = start / 8; + const uint8_t end_byte_pos = end / 8; + const uint8_t start_bit_pos = start % 8; + const uint8_t end_bit_pos = end % 8; + is_all_true = false; + + if (OB_UNLIKELY(start > end || start < 0 || end >= capacity)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid argument", KR(ret), K(capacity), K(start), K(end)); + } else if (start_byte_pos == end_byte_pos) { + // only one byte + uint8_t mask = ((1 << (end_bit_pos + 1)) - 1) & ~((1 << start_bit_pos) - 1); + is_all_true = (bitmap_[start_byte_pos] & mask) == mask; + } else { + uint8_t start_mask = ~((1 << start_bit_pos) - 1); + uint8_t end_mask = (1 << (end_bit_pos + 1)) - 1; + + if ((bitmap_[start_byte_pos] & start_mask) != start_mask || + (bitmap_[end_byte_pos] & end_mask) != end_mask) { + is_all_true = false; + } else { + is_all_true = true; + for (int64_t i = start_byte_pos + 1; is_all_true && i < end_byte_pos; ++i) { + if (bitmap_[i] != 0XFF) { + is_all_true = false; + } + } + } + } + + return ret; +} + +int ObTmpFileBlockPageBitmap::is_all_false(const int64_t start, const int64_t end, bool &is_all_false) const +{ + int ret = OB_SUCCESS; + const int64_t capacity = PAGE_CAPACITY; + const uint8_t start_byte_pos = start / 8; + const uint8_t end_byte_pos = end / 8; + const uint8_t start_bit_pos = start % 8; + const uint8_t end_bit_pos = end % 8; + is_all_false = false; + + if (OB_UNLIKELY(start > end || start < 0 || end >= capacity)) { + is_all_false = false; + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid argument", KR(ret), K(capacity), K(start), K(end)); + } else if (start_byte_pos == end_byte_pos) { + // only one byte + uint8_t mask = ((1 << (end_bit_pos + 1)) - 1) & ~((1 << start_bit_pos) - 1); + is_all_false = (bitmap_[start_byte_pos] & mask) == 0; + } else { + uint8_t start_mask = ~((1 << start_bit_pos) - 1); + uint8_t end_mask = (1 << (end_bit_pos + 1)) - 1; + + if ((bitmap_[start_byte_pos] & start_mask) != 0 || + (bitmap_[end_byte_pos] & end_mask) != 0) { + is_all_false = false; + } else { + is_all_false = true; + for (int64_t i = start_byte_pos + 1; is_all_false && i < end_byte_pos; ++i) { + if (bitmap_[i] != 0) { + is_all_false = false; + } + } + } + } + + return ret; +} + +int ObTmpFileBlockPageBitmap::is_all_true(bool &b_ret) const +{ + return is_all_true(0, PAGE_CAPACITY - 1, b_ret); +} + +int ObTmpFileBlockPageBitmap::is_all_false(bool &b_ret) const +{ + return is_all_false(0, PAGE_CAPACITY - 1, b_ret); +} + +int64_t ObTmpFileBlockPageBitmap::to_string(char* buf, const int64_t buf_len) const +{ + int64_t pos = 0; + J_OBJ_START(); + for (int i = 0; i < PAGE_BYTE_CAPACITY / sizeof(uint64_t); ++i) { + int64_t offset = i * sizeof(uint64_t); + if (i > 0) { + J_COMMA(); + } + char bitmap_id[11]; + snprintf(bitmap_id, sizeof(bitmap_id), "bitmap_[%d]", i); + J_KV(bitmap_id, *(reinterpret_cast(bitmap_ + offset))); + } + J_OBJ_END(); + return pos; +} +//-----------------------ObTmpFileBlock----------------------- +ObTmpFileBlock::~ObTmpFileBlock() +{ + reset(); +} + +int ObTmpFileBlock::reset() +{ + int ret = OB_SUCCESS; + if (is_valid() && BlockState::ON_DISK == block_state_ && OB_FAIL(OB_SERVER_BLOCK_MGR.dec_ref(macro_block_id_))) { + LOG_ERROR("failed to dec macro block ref cnt", KR(ret), KPC(this)); + } else { + block_state_ = BlockState::INVALID; + page_bitmap_.reset(); + block_index_ = ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX; + macro_block_id_.reset(); + ref_cnt_ = 0; + } + return ret; +} + +int ObTmpFileBlock::init_block(const int64_t block_index, + const int64_t begin_page_id, const int64_t page_num) +{ + int ret = OB_SUCCESS; + SpinWLockGuard guard(lock_); + bool use_released_pages = false; + + if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX == block_index)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid block_index", KR(ret), K(block_index), K(begin_page_id), K(page_num)); + } else if (OB_UNLIKELY(page_num <= 0 || begin_page_id < 0 || + begin_page_id + page_num > ObTmpFileGlobal::BLOCK_PAGE_NUMS)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(block_index), K(begin_page_id), K(page_num)); + } else if (OB_UNLIKELY(block_index_ != ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tmp file block has been inited", KR(ret), K(block_index_)); + } else if (block_state_ != BlockState::INVALID) { + ret = OB_OP_NOT_ALLOW; + LOG_WARN("attempt to reinit a block not in invalid state", KR(ret), KPC(this)); + } else if (OB_FAIL(page_bitmap_.is_all_false(begin_page_id, begin_page_id + page_num - 1, use_released_pages))) { + LOG_WARN("fail to check whether the pages are all false", KR(ret), K(begin_page_id), K(page_num), K(page_bitmap_)); + } else if (OB_UNLIKELY(!use_released_pages)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("attempt to use a allocated page", KR(ret), K(block_index), K(begin_page_id), K(page_num), + K(page_bitmap_), K(use_released_pages)); + } else if (OB_FAIL(page_bitmap_.set_bitmap_batch(begin_page_id, page_num, true))) { + LOG_WARN("fail to set bitmap", KR(ret), K(begin_page_id), K(page_num), K(page_bitmap_)); + } else { + block_index_ = block_index; + block_state_ = BlockState::IN_MEMORY; + LOG_DEBUG("init block successfully", K(block_index), K(begin_page_id), K(page_num), KPC(this)); + } + return ret; +} + +int ObTmpFileBlock::write_back_start() +{ + int ret = OB_SUCCESS; + SpinWLockGuard guard(lock_); + if (block_state_ != BlockState::IN_MEMORY && block_state_ != BlockState::WRITE_BACK) { + ret = OB_OP_NOT_ALLOW; + LOG_WARN("write back a block not in in_memory state and not in write back state", + KR(ret), KPC(this)); + } else { + block_state_ = BlockState::WRITE_BACK; + LOG_DEBUG("switch tmp file block to write back state", KPC(this)); + } + return ret; +} + +int ObTmpFileBlock::write_back_failed() +{ + int ret = OB_SUCCESS; + SpinWLockGuard guard(lock_); + if (block_state_ != BlockState::WRITE_BACK) { + ret = OB_OP_NOT_ALLOW; + LOG_WARN("notify a block write_back_failed, but the block not in write back state", + KR(ret), KPC(this)); + } else { + block_state_ = BlockState::IN_MEMORY; + LOG_DEBUG("switch tmp file block to in memory state", KPC(this)); + } + return ret; +} + +int ObTmpFileBlock::write_back_succ(blocksstable::MacroBlockId macro_block_id) +{ + int ret = OB_SUCCESS; + SpinWLockGuard guard(lock_); + if (OB_UNLIKELY(!macro_block_id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(macro_block_id)); + } else if (block_state_ != BlockState::WRITE_BACK) { + ret = OB_OP_NOT_ALLOW; + LOG_WARN("attempt to set macro_block_id for a block not in write_back state", KR(ret), KPC(this)); + } else if (OB_FAIL(OB_SERVER_BLOCK_MGR.inc_ref(macro_block_id))) { + LOG_ERROR("failed to dec macro block ref cnt", KR(ret), K(macro_block_id), KPC(this)); + } else { + macro_block_id_ = macro_block_id; + block_state_ = BlockState::ON_DISK; + LOG_DEBUG("switch tmp file block to on disk state", KPC(this)); + } + return ret; +} + +int ObTmpFileBlock::release_pages(const int64_t begin_page_id, const int64_t page_num) +{ + int ret = OB_SUCCESS; + SpinWLockGuard guard(lock_); + bool release_allocated_page = false; + if (OB_UNLIKELY(!is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tmp file block is invalid", KR(ret), KPC(this)); + } else if (OB_UNLIKELY(page_num <= 0 || begin_page_id < 0 || + begin_page_id + page_num > ObTmpFileGlobal::BLOCK_PAGE_NUMS)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(block_index_), K(begin_page_id), K(page_num)); + } else if (OB_FAIL(page_bitmap_.is_all_true(begin_page_id, begin_page_id + page_num - 1, release_allocated_page))) { + LOG_WARN("fail to check whether the pages are all true", KR(ret), K(block_index_), K(macro_block_id_), + K(begin_page_id), K(page_num), K(page_bitmap_)); + } else if (OB_UNLIKELY(!release_allocated_page)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("attempt to release a released page", KR(ret), K(block_index_), K(macro_block_id_), + K(begin_page_id), K(page_num), + K(page_bitmap_), K(release_allocated_page)); + } else if (OB_FAIL(page_bitmap_.set_bitmap_batch(begin_page_id, page_num, false))) { + LOG_WARN("fail to set bitmap", KR(ret), K(begin_page_id), K(page_num), K(page_bitmap_)); + } + return ret; +} + +int ObTmpFileBlock::get_page_usage(int64_t &page_num) const +{ + int ret = OB_SUCCESS; + SpinRLockGuard guard(lock_); + page_num = 0; + if (OB_UNLIKELY(!is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tmp file block is invalid", KR(ret), KPC(this)); + } else { + for (int i = 0; OB_SUCC(ret) && i < ObTmpFileBlockPageBitmap::get_capacity(); ++i) { + bool value = false; + if (OB_FAIL(page_bitmap_.get_value(i, value))) { + LOG_WARN("fail to get value from bitmap", KR(ret), K(i), KPC(this)); + } else if (value) { + page_num++; + } + } // end for + } + return ret; +} + +int ObTmpFileBlock::inc_ref_cnt() +{ + int ret = OB_SUCCESS; + SpinWLockGuard guard(lock_); + + if (OB_UNLIKELY(!is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tmp file block is invalid", KR(ret), KPC(this)); + } else { + ref_cnt_++; + } + + return ret; +} + +int ObTmpFileBlock::dec_ref_cnt(int64_t &ref_cnt) +{ + int ret = OB_SUCCESS; + SpinWLockGuard guard(lock_); + + if (OB_UNLIKELY(!is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tmp file block is invalid", KR(ret), KPC(this)); + } else if (OB_UNLIKELY(ref_cnt_ <= 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid ref cnt", KR(ret), KPC(this)); + } else { + ref_cnt_ -= 1; + ref_cnt = ref_cnt_; + } + + return ret; +} + +bool ObTmpFileBlock::on_disk() const +{ + return BlockState::ON_DISK == block_state_; +} + +int ObTmpFileBlock::can_remove(bool &can_remove) const +{ + int ret = OB_SUCCESS; + SpinRLockGuard guard(lock_); + + can_remove = false; + bool is_all_page_released = false; + if (OB_FAIL(page_bitmap_.is_all_false(is_all_page_released))) { + LOG_WARN("fail to check whether the pages are all false", KR(ret), KPC(this)); + } else { + can_remove = is_all_page_released && BlockState::WRITE_BACK != block_state_; + } + return ret; +} + +blocksstable::MacroBlockId ObTmpFileBlock::get_macro_block_id() const +{ + SpinRLockGuard guard(lock_); + return macro_block_id_; +} + +int64_t ObTmpFileBlock::get_block_index() const +{ + SpinRLockGuard guard(lock_); + return block_index_; +} +//-----------------------ObTmpFileBlockHandle----------------------- + +ObTmpFileBlockHandle::ObTmpFileBlockHandle(ObTmpFileBlock *block, ObTmpFileBlockManager *tmp_file_blk_mgr) + : ptr_(nullptr), tmp_file_blk_mgr_(nullptr) +{ + if (nullptr != block && nullptr != tmp_file_blk_mgr) { + ptr_ = block; + tmp_file_blk_mgr_ = tmp_file_blk_mgr; + ptr_->inc_ref_cnt(); + } +} + +ObTmpFileBlockHandle::ObTmpFileBlockHandle(const ObTmpFileBlockHandle &handle) + : ptr_(nullptr), tmp_file_blk_mgr_(nullptr) +{ + operator=(handle); +} + +ObTmpFileBlockHandle & ObTmpFileBlockHandle::operator=(const ObTmpFileBlockHandle &other) +{ + if (other.ptr_ != ptr_) { + reset(); + ptr_ = other.ptr_; + tmp_file_blk_mgr_ = other.tmp_file_blk_mgr_; + if (is_inited()) { + ptr_->inc_ref_cnt(); + } + } + return *this; +} + +void ObTmpFileBlockHandle::reset() +{ + int ret = OB_SUCCESS; + if (is_inited()) { + int64_t cur_ref_cnt = -1; + int64_t block_index = ptr_->get_block_index(); + if (OB_FAIL(ptr_->dec_ref_cnt(cur_ref_cnt))) { + LOG_ERROR("fail to dec block ref cnt", KR(ret), KPC(ptr_)); + } else if (0 == cur_ref_cnt) { + ptr_->reset(); + tmp_file_blk_mgr_->get_block_allocator().free(ptr_); + } + ptr_ = nullptr; + } +} + +int ObTmpFileBlockHandle::init(ObTmpFileBlock *block, ObTmpFileBlockManager *tmp_file_blk_mgr) +{ + int ret = OB_SUCCESS; + + if (is_inited()) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", KR(ret), KP(ptr_)); + } else if (OB_ISNULL(block) || OB_ISNULL(tmp_file_blk_mgr)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), KP(block), KP(tmp_file_blk_mgr)); + } else if (OB_FAIL(block->inc_ref_cnt())) { + LOG_WARN("fail to inc block ref cnt", KR(ret), KPC(block)); + } else { + ptr_ = block; + tmp_file_blk_mgr_ = tmp_file_blk_mgr; + } + + return ret; +} + +//-----------------------ObTmpFileBlockManager----------------------- +ObTmpFileBlockManager::ObTmpFileBlockManager() : + is_inited_(false), + tenant_id_(OB_INVALID_TENANT_ID), + used_page_num_(0), + physical_block_num_(0), + block_index_generator_(0), + block_map_(), + block_allocator_(), + stat_lock_(), + map_lock_() + {} + +ObTmpFileBlockManager::~ObTmpFileBlockManager() +{ + destroy(); +} + +void ObTmpFileBlockManager::destroy() +{ + is_inited_ = false; + tenant_id_ = OB_INVALID_TENANT_ID; + used_page_num_ = 0; + physical_block_num_ = 0; + block_index_generator_ = 0; + block_map_.destroy(); + block_allocator_.reset(); +} + +int ObTmpFileBlockManager::init(const uint64_t tenant_id, const int64_t block_mem_limit) +{ + int ret = OB_SUCCESS; + + if (OB_UNLIKELY(is_inited_)) { + ret = OB_INIT_TWICE; + LOG_WARN("ObTmpFileBlockManager init twice", KR(ret), K(is_inited_)); + } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", KR(ret), K(tenant_id)); + } else if (OB_FAIL(block_map_.init("TmpFileBlkMgr", tenant_id))) { + LOG_WARN("fail to init tenant temporary file block manager", KR(ret), K(tenant_id)); + } else if (OB_FAIL(block_allocator_.init(common::OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObModIds::OB_TMP_BLOCK_MANAGER, tenant_id_, + block_mem_limit))) { + LOG_WARN("fail to init temporary file block allocator", KR(ret), K(tenant_id_), K(block_mem_limit)); + } else { + block_allocator_.set_attr(ObMemAttr(tenant_id, "TmpFileBlk")); + tenant_id_ = tenant_id; + is_inited_ = true; + } + + return ret; +} + +int ObTmpFileBlockManager::create_tmp_file_block(const int64_t begin_page_id, const int64_t page_num, + int64_t &block_index) +{ + int ret = OB_SUCCESS; + ObTmpFileBlockHandle handle; + ObTmpFileBlock* blk = nullptr; + const uint64_t blk_size = sizeof(ObTmpFileBlock); + void *buf = nullptr; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTmpFileBlockManager has not been inited", KR(ret), K(tenant_id_)); + } else if (OB_ISNULL(buf = block_allocator_.alloc(blk_size, lib::ObMemAttr(tenant_id_, "TmpFileBlk")))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate memory for tmp file block", KR(ret), K(blk_size)); + } else if (FALSE_IT(blk = new (buf) ObTmpFileBlock())) { + } else if (FALSE_IT(block_index = ATOMIC_AAF(&block_index_generator_, 1))) { + } else if (OB_FAIL(blk->init_block(block_index, begin_page_id, page_num))) { + LOG_WARN("fail to init tmp file block", KR(ret), K(block_index)); + } else if (OB_FAIL(handle.init(blk, this))) { + LOG_WARN("fail to init tmp file block handle", KR(ret), K(block_index)); + } else { + SharedLockGuard guard(map_lock_); + if (OB_FAIL(block_map_.insert(ObTmpFileBlockKey(block_index), handle))) { + LOG_WARN("fail to insert tmp file block into map", KR(ret), K(block_index)); + } + LOG_DEBUG("create tmp file block succ", KR(ret), K(block_index)); + } + + if (OB_FAIL(ret) && OB_NOT_NULL(blk)) { + blk->~ObTmpFileBlock(); + block_allocator_.free(blk); + blk = nullptr; + } + + return ret; +} + +int ObTmpFileBlockManager::write_back_start(const int64_t block_index) +{ + int ret = OB_SUCCESS; + ObTmpFileBlockHandle handle; + ObTmpFileBlock *blk = nullptr; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTmpFileBlockManager has not been inited", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(block_map_.get(ObTmpFileBlockKey(block_index), handle))) { + LOG_WARN("fail to get tmp file block", KR(ret), K(block_index)); + } else if (OB_ISNULL(blk = handle.get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tmp file block is null", KR(ret), K(block_index)); + } else if (OB_FAIL(blk->write_back_start())) { + LOG_WARN("fail to write back start", KR(ret), K(block_index)); + } + + return ret; +} + +int ObTmpFileBlockManager::write_back_failed(const int64_t block_index) +{ + int ret = OB_SUCCESS; + ObTmpFileBlockHandle handle; + ObTmpFileBlock *blk = nullptr; + bool can_remove = false; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTmpFileBlockManager has not been inited", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(block_map_.get(ObTmpFileBlockKey(block_index), handle))) { + LOG_WARN("fail to get tmp file block", KR(ret), K(block_index)); + } else if (OB_ISNULL(blk = handle.get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tmp file block is null", KR(ret), K(block_index)); + } else if (OB_FAIL(blk->write_back_failed())) { + LOG_WARN("fail to notify block write_back_failed", KR(ret), K(handle)); + } + + if (FAILEDx(handle.get()->can_remove(can_remove))) { + LOG_WARN("check block can remove failed", KR(ret), K(handle)); + } else if (can_remove) { + if (OB_FAIL(remove_tmp_file_block_(block_index))) { + LOG_ERROR("fail to remove tmp file block", KR(ret), K(handle)); + } + } + return ret; +} + +int ObTmpFileBlockManager::write_back_succ(const int64_t block_index, const blocksstable::MacroBlockId macro_block_id) +{ + int ret = OB_SUCCESS; + ObTmpFileBlockHandle handle; + ObTmpFileBlock *blk = nullptr; + bool can_remove = false; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTmpFileBlockManager has not been inited", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(block_map_.get(ObTmpFileBlockKey(block_index), handle))) { + LOG_WARN("fail to get tmp file block", KR(ret), K(block_index)); + } else if (OB_ISNULL(blk = handle.get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tmp file block is null", KR(ret), K(block_index)); + } else if (OB_FAIL(blk->write_back_succ(macro_block_id))) { + LOG_WARN("fail to notify write_back_succ", KR(ret), K(block_index), K(macro_block_id), K(handle)); + } else { + SpinWLockGuard guard(stat_lock_); + int64_t used_page_num = 0; + if (OB_FAIL(blk->get_page_usage(used_page_num))) { + LOG_WARN("fail to get page usage", KR(ret), K(handle)); + } else { + used_page_num_ += used_page_num; + physical_block_num_ += 1; + } + } + + if (FAILEDx(handle.get()->can_remove(can_remove))) { + LOG_WARN("check block can remove failed", KR(ret), K(handle)); + } else if (can_remove) { + if (OB_FAIL(remove_tmp_file_block_(block_index))) { + LOG_ERROR("fail to remove tmp file block", KR(ret), K(handle)); + } + } + return ret; +} + +// if a block is released all pages in this function, +// when ObTmpFileBlockHandle destructed, the handle will remove itself from block_map +int ObTmpFileBlockManager::release_tmp_file_page(const int64_t block_index, + const int64_t begin_page_id, const int64_t page_num) +{ + int ret = OB_SUCCESS; + ObTmpFileBlockHandle handle; + bool can_remove = false; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTmpFileBlockManager has not been inited", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(block_map_.get(ObTmpFileBlockKey(block_index), handle))) { + LOG_WARN("fail to get tmp file block", KR(ret), K(block_index)); + } else if (OB_ISNULL(handle.get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error, block should not be null", KR(ret), K(block_index)); + } else if (OB_FAIL(handle.get()->release_pages(begin_page_id, page_num))) { + LOG_WARN("fail to release pages", KR(ret), K(begin_page_id), K(page_num), K(handle)); + } else if (handle.get()->on_disk()) { + SpinWLockGuard guard(stat_lock_); + used_page_num_ -= page_num; + } + + LOG_DEBUG("release_tmp_file_page", KR(ret), K(block_index), K(begin_page_id), K(page_num), K(handle)); + + if (FAILEDx(handle.get()->can_remove(can_remove))) { + LOG_WARN("check block can remove failed", KR(ret), K(handle)); + } else if (can_remove) { + if (OB_FAIL(remove_tmp_file_block_(block_index))) { + LOG_ERROR("fail to remove tmp file block", KR(ret), K(handle)); + } + } + + return ret; +} + +// only called by ObTmpFileBlockHandle::reset() +int ObTmpFileBlockManager::remove_tmp_file_block_(const int64_t block_index) +{ + int ret = OB_SUCCESS; + ObTmpFileBlock* blk = nullptr; + ObTmpFileBlockHandle handle; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTmpFileBlockManager has not been inited", KR(ret), K(tenant_id_)); + } else { + SharedLockGuard guard(map_lock_); + if (OB_FAIL(block_map_.erase(ObTmpFileBlockKey(block_index), handle))) { + if (ret != OB_ENTRY_NOT_EXIST) { + LOG_WARN("fail to erase tmp file block", KR(ret), K(block_index)); + } else { + LOG_DEBUG("erase tmp file block succ", KR(ret), K(block_index)); + ret = OB_SUCCESS; + } + } else if (OB_UNLIKELY(nullptr == (blk = handle.get()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", KR(ret), K(block_index)); + } else { + LOG_DEBUG("erase tmp file block from map succ", KR(ret), K(handle)); + } + } + + if (OB_SUCC(ret) && OB_NOT_NULL(blk) && blk->on_disk()) { + SpinWLockGuard guard(stat_lock_); + int64_t used_page_num = 0; + if (OB_FAIL(blk->get_page_usage(used_page_num))) { + LOG_WARN("fail to get page usage", KR(ret), K(block_index), K(handle)); + } else { + used_page_num_ -= used_page_num; + physical_block_num_ -= 1; + } + } + + return ret; +} + +int ObTmpFileBlockManager::get_block_usage_stat(int64_t &used_page_num, int64_t ¯o_block_count) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTmpFileBlockManager has not been inited", KR(ret), K(tenant_id_)); + } else { + SpinRLockGuard guard(stat_lock_); + used_page_num = used_page_num_; + macro_block_count = physical_block_num_; + } + + return ret; +} + +void ObTmpFileBlockManager::print_block_usage() +{ + int ret = OB_SUCCESS; + int64_t used_page_num = 0; + int64_t block_num = 0; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTenantTmpFileManager has not been inited", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(get_block_usage_stat(used_page_num, block_num))) { + LOG_WARN("fail to get block usage stat", KR(ret)); + } else if (OB_UNLIKELY(0 == block_num)) { + LOG_INFO("temporary file module use no blocks"); + } else { + int64_t occupied_page_num = block_num * ObTmpFileGlobal::BLOCK_PAGE_NUMS; + double disk_fragment_ratio = static_cast(used_page_num) / static_cast(occupied_page_num); + LOG_INFO("the block usage for temporary files", + K(used_page_num), K(occupied_page_num), K(disk_fragment_ratio)); + } +} + +int ObTmpFileBlockManager::get_macro_block_count(int64_t ¯o_block_count) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTmpFileBlockManager has not been inited", KR(ret), K(tenant_id_)); + } else { + SpinRLockGuard guard(stat_lock_); + macro_block_count = physical_block_num_; + } + return ret; +} + +int ObTmpFileBlockManager::get_macro_block_list(common::ObIArray ¯o_id_list) +{ + int ret = OB_SUCCESS; + ExclusiveLockGuard guard(map_lock_); + CollectMacroBlockIdFunctor func(macro_id_list); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTmpFileBlockManager has not been inited", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(block_map_.for_each(func))) { + LOG_WARN("fail to collect macro block ids", KR(ret)); + } + + return ret; +} + +bool ObTmpFileBlockManager::CollectMacroBlockIdFunctor::operator()(const ObTmpFileBlockKey &block_index, const ObTmpFileBlockHandle &handle) +{ + int ret = OB_SUCCESS; + + if (OB_ISNULL(handle.get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("handle should not be null", KR(ret), K(block_index)); + } else if (!handle.get()->is_valid()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("handle should not be invalid", KR(ret), K(block_index), K(handle)); + } else if (!handle.get()->on_disk()) { + // do nothing + } else if (OB_FAIL(macro_id_list_.push_back(handle.get()->get_macro_block_id()))) { + LOG_WARN("failed to push back", KR(ret), K(block_index), K(handle)); + } + return OB_SUCCESS == ret; +} + +int ObTmpFileBlockManager::get_macro_block_id(const int64_t block_index, blocksstable::MacroBlockId ¯o_block_id) +{ + int ret = OB_SUCCESS; + ObTmpFileBlockHandle handle; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTmpFileBlockManager has not been inited", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(block_map_.get(ObTmpFileBlockKey(block_index), handle))) { + LOG_WARN("fail to get tmp file block", KR(ret), K(block_index)); + } else if (OB_ISNULL(handle.get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error, block should not be null", KR(ret), K(block_index)); + } else if (OB_UNLIKELY(!handle.get()->is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error, block should not be invalid", KR(ret), K(block_index), K(handle)); + } else { + macro_block_id = handle.get()->get_macro_block_id(); + } + + return ret; +} + +int ObTmpFileBlockManager::get_tmp_file_block_handle(const int64_t block_index, ObTmpFileBlockHandle &handle) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTmpFileBlockManager has not been inited", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(block_map_.get(ObTmpFileBlockKey(block_index), handle))) { + LOG_WARN("fail to get tmp file block", KR(ret), K(block_index)); + } else if (OB_ISNULL(handle.get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error, block should not be null", KR(ret), K(block_index)); + } else if (OB_UNLIKELY(!handle.get()->is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error, block should not be invalid", KR(ret), K(block_index), K(handle)); + } + + return ret; +} + +} // end namespace tmp_file +} // end namespace oceanbase diff --git a/src/storage/tmp_file/ob_tmp_file_block_manager.h b/src/storage/tmp_file/ob_tmp_file_block_manager.h new file mode 100644 index 000000000..593ec8f9a --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_block_manager.h @@ -0,0 +1,221 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_BLOCK_MANAGER_H_ +#define OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_BLOCK_MANAGER_H_ + +#include "storage/blocksstable/ob_macro_block_id.h" +#include "storage/blocksstable/ob_block_manager.h" +#include "storage/tmp_file/ob_tmp_file_global.h" +#include "lib/hash/ob_linear_hash_map.h" +#include "lib/lock/ob_spin_rwlock.h" +#include "lib/allocator/page_arena.h" + +namespace oceanbase +{ +namespace tmp_file +{ +class ObTmpFileBlockPageBitmap; +class ObTmpFileBlockManager; + +struct ObTmpFileBlockKey final +{ + explicit ObTmpFileBlockKey(const int64_t block_index) : block_index_(block_index) {} + OB_INLINE int hash(uint64_t &hash_val) const + { + hash_val = murmurhash(&block_index_, sizeof(int64_t), 0); + return OB_SUCCESS; + } + OB_INLINE bool operator==(const ObTmpFileBlockKey &other) const + { + return block_index_ == other.block_index_; + } + TO_STRING_KV(K(block_index_)); + int64_t block_index_; +}; + +// search every set of continuous existence or non-existence pages in the range of [start_idx, end_idx_] +class ObTmpFileBlockPageBitmapIterator +{ +public: + ObTmpFileBlockPageBitmapIterator() : + bitmap_(nullptr), is_inited_(false), cur_idx_(OB_INVALID_INDEX), end_idx_(OB_INVALID_INDEX) {} + ~ObTmpFileBlockPageBitmapIterator() { reset(); } + int init(const ObTmpFileBlockPageBitmap *bitmap, const int64_t start_idx, int64_t end_idx); + void reset(); + int next_range(bool &value, int64_t &start_page_id, int64_t &end_page_id); + OB_INLINE bool has_next() const { return is_inited_ && cur_idx_ <= end_idx_; } +private: + const ObTmpFileBlockPageBitmap *bitmap_; + bool is_inited_; + int64_t cur_idx_; + int64_t end_idx_; +}; + +class ObTmpFileBlockPageBitmap +{ +public: + ObTmpFileBlockPageBitmap() { reset(); } + ~ObTmpFileBlockPageBitmap() { reset(); } + OB_INLINE void reset() { MEMSET(bitmap_, 0, PAGE_CAPACITY / 8); } + int get_value(const int64_t offset, bool &value) const; + int set_bitmap(const int64_t offset, const bool value); + int set_bitmap_batch(const int64_t offset, const int64_t count, const bool value); + // [start, end] + int is_all_true(const int64_t start, const int64_t end, bool &is_all_true) const; + int is_all_false(const int64_t start, const int64_t end, bool &is_all_false) const; + int is_all_true(bool &b_ret) const; + int is_all_false(bool &b_ret) const; + OB_INLINE static int64_t get_capacity() { return PAGE_CAPACITY; } + int64_t to_string(char* buf, const int64_t buf_len) const; +private: + static constexpr int64_t PAGE_CAPACITY = ObTmpFileGlobal::BLOCK_PAGE_NUMS; + // upper_align(PAGE_CAPACITY/8, 8) + static constexpr int64_t PAGE_BYTE_CAPACITY = (PAGE_CAPACITY / 8 + 7) & ~7; + +private: + uint8_t bitmap_[PAGE_BYTE_CAPACITY]; +}; + +// ObTmpFileBlock records the usage of a macro block. +// each macro block could be referenced by one or more tmp files of a tenant. +class ObTmpFileBlock final +{ + enum BlockState { + INVALID = -1, + IN_MEMORY = 0, + WRITE_BACK = 1, + ON_DISK = 2, + }; +public: + ObTmpFileBlock(): block_state_(BlockState::INVALID), lock_(common::ObLatchIds::TMP_FILE_LOCK), + page_bitmap_(), block_index_(ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX), + macro_block_id_(), ref_cnt_(0) {} + ~ObTmpFileBlock(); + + OB_INLINE bool is_valid() const { + bool b_ret = false; + + if (block_state_ < BlockState::INVALID || block_state_ > BlockState::ON_DISK) { + b_ret = false; + } else { + b_ret = block_index_ != ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX; + if (BlockState::ON_DISK == block_state_) { + b_ret = b_ret && macro_block_id_.is_valid(); + } + } + return b_ret; + } + + int reset(); + int init_block(const int64_t block_index, const int64_t begin_page_id, const int64_t page_num); + int write_back_start(); + int write_back_failed(); + int write_back_succ(const blocksstable::MacroBlockId macro_block_id); + int release_pages(const int64_t begin_page_id, const int64_t page_num); + int get_page_usage(int64_t &page_num) const; + int inc_ref_cnt(); + int dec_ref_cnt(int64_t &ref_cnt); + bool on_disk() const; + int can_remove(bool &can_remove) const; + blocksstable::MacroBlockId get_macro_block_id() const; + int64_t get_block_index() const; + + TO_STRING_KV(K(block_index_), K(macro_block_id_), + K(block_state_), K(page_bitmap_), + K(ref_cnt_)); + +private: + BlockState block_state_; + common::SpinRWLock lock_; + ObTmpFileBlockPageBitmap page_bitmap_; // records the page usage of this block + int64_t block_index_; + blocksstable::MacroBlockId macro_block_id_; + int64_t ref_cnt_; + DISALLOW_COPY_AND_ASSIGN(ObTmpFileBlock); +}; + +class ObTmpFileBlockHandle final +{ +public: + ObTmpFileBlockHandle() : ptr_(nullptr) {} + ObTmpFileBlockHandle(ObTmpFileBlock *block, ObTmpFileBlockManager *tmp_file_blk_mgr); + ObTmpFileBlockHandle(const ObTmpFileBlockHandle &handle); + ObTmpFileBlockHandle & operator=(const ObTmpFileBlockHandle &other); + ~ObTmpFileBlockHandle() { reset(); } + OB_INLINE ObTmpFileBlock * get() const {return ptr_; } + bool is_inited() { return nullptr != ptr_ && nullptr != tmp_file_blk_mgr_; } + void reset(); + int init(ObTmpFileBlock *block, ObTmpFileBlockManager *tmp_file_blk_mgr); + TO_STRING_KV(KPC(ptr_)); +private: + ObTmpFileBlock *ptr_; + ObTmpFileBlockManager *tmp_file_blk_mgr_; +}; + +class ObTmpFileBlockManager final +{ +public: + ObTmpFileBlockManager(); + ~ObTmpFileBlockManager(); + int init(const uint64_t tenant_id, const int64_t block_mem_limit); + void destroy(); + int create_tmp_file_block(const int64_t begin_page_id, const int64_t page_num, + int64_t &block_index); + + int write_back_start(const int64_t block_index); + int write_back_failed(const int64_t block_index); + int write_back_succ(const int64_t block_index, const blocksstable::MacroBlockId macro_block_id); + int release_tmp_file_page(const int64_t block_index, + const int64_t begin_page_id, const int64_t page_num); + int get_macro_block_list(common::ObIArray ¯o_id_list); + int get_macro_block_count(int64_t ¯o_block_count); + int get_tmp_file_block_handle(const int64_t block_index, ObTmpFileBlockHandle &handle); + int get_macro_block_id(const int64_t block_index, blocksstable::MacroBlockId ¯o_block_id); + int get_block_usage_stat(int64_t &used_page_num, int64_t ¯o_block_count); + void print_block_usage(); + OB_INLINE common::ObConcurrentFIFOAllocator &get_block_allocator() { return block_allocator_; } + TO_STRING_KV(K(is_inited_), K(tenant_id_), K(used_page_num_), + K(physical_block_num_), K(block_index_generator_)); +private: + int remove_tmp_file_block_(const int64_t block_index); +private: + typedef common::ObLinearHashMap ObTmpFileBlockMap; + typedef SpinWLockGuard ExclusiveLockGuard; + typedef SpinRLockGuard SharedLockGuard; +private: + class CollectMacroBlockIdFunctor final + { + public: + CollectMacroBlockIdFunctor(common::ObIArray ¯o_id_list) + : macro_id_list_(macro_id_list) {} + bool operator()(const ObTmpFileBlockKey &block_index, const ObTmpFileBlockHandle &block); + + private: + common::ObIArray ¯o_id_list_; + }; +private: + bool is_inited_; + uint64_t tenant_id_; + uint64_t used_page_num_; + uint64_t physical_block_num_; + uint64_t block_index_generator_; + ObTmpFileBlockMap block_map_; + common::ObConcurrentFIFOAllocator block_allocator_; + common::SpinRWLock stat_lock_; // to protect the consistency of used_page_num_ and physical_block_num_ + common::SpinRWLock map_lock_; // to protect the for_each operation of block_map_ + DISALLOW_COPY_AND_ASSIGN(ObTmpFileBlockManager); +}; + +} // end namespace tmp_file +} // end namespace oceanbase +#endif // OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_BLOCK_MANAGER_H_ diff --git a/src/storage/tmp_file/ob_tmp_file_cache.cpp b/src/storage/tmp_file/ob_tmp_file_cache.cpp new file mode 100644 index 000000000..3e4d08d52 --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_cache.cpp @@ -0,0 +1,674 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "observer/omt/ob_tenant_config_mgr.h" +#include "lib/stat/ob_diagnose_info.h" +#include "common/ob_smart_var.h" +#include "storage/ob_file_system_router.h" +#include "share/ob_task_define.h" +#include "ob_tmp_file_cache.h" +#include "storage/blocksstable/ob_block_manager.h" +#include "storage/tmp_file/ob_tmp_file_global.h" +#include "storage/tmp_file/ob_tmp_file_manager.h" + +using namespace oceanbase::storage; +using namespace oceanbase::share; + +namespace oceanbase +{ +namespace tmp_file +{ +/* -------------------------- ObTmpBlockCacheKey --------------------------- */ +ObTmpBlockCacheKey::ObTmpBlockCacheKey() + : block_id_(-1), tenant_id_(OB_INVALID_TENANT_ID) +{ +} + +ObTmpBlockCacheKey::ObTmpBlockCacheKey(const int64_t block_id, const uint64_t tenant_id) + : block_id_(block_id), tenant_id_(tenant_id) +{ +} + +ObTmpBlockCacheKey::~ObTmpBlockCacheKey() +{ +} + +bool ObTmpBlockCacheKey::operator ==(const ObIKVCacheKey &other) const +{ + const ObTmpBlockCacheKey &other_key = reinterpret_cast (other); + return block_id_ == other_key.block_id_ && tenant_id_ == other_key.tenant_id_; +} + +uint64_t ObTmpBlockCacheKey::get_tenant_id() const +{ + return tenant_id_; +} + +uint64_t ObTmpBlockCacheKey::hash() const +{ + return murmurhash(this, sizeof(ObTmpBlockCacheKey), 0); +} + +int64_t ObTmpBlockCacheKey::size() const +{ + return sizeof(*this); +} + +int ObTmpBlockCacheKey::deep_copy(char *buf, const int64_t buf_len, ObIKVCacheKey *&key) const +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(buf) || OB_UNLIKELY(buf_len < size())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Invalid argument, ", KR(ret)); + } else if (OB_UNLIKELY(!is_valid())) { + ret = OB_INVALID_DATA; + STORAGE_LOG(WARN, "invalid tmp block cache key, ", KPC(this), KR(ret)); + } else { + key = new (buf) ObTmpBlockCacheKey(block_id_, tenant_id_); + } + return ret; +} + +bool ObTmpBlockCacheKey::is_valid() const +{ + return OB_LIKELY(block_id_ >= 0 && tenant_id_ > 0 && size() > 0); +} + +/* -------------------------- ObTmpBlockCacheValue --------------------------- */ +ObTmpBlockCacheValue::ObTmpBlockCacheValue(char *buf) + : buf_(buf), size_(OB_DEFAULT_MACRO_BLOCK_SIZE) +{ +} + +ObTmpBlockCacheValue::~ObTmpBlockCacheValue() +{ +} + +int64_t ObTmpBlockCacheValue::size() const +{ + return sizeof(*this) + size_; +} + +int ObTmpBlockCacheValue::deep_copy(char *buf, const int64_t buf_len, ObIKVCacheValue *&value) const +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(buf) || OB_UNLIKELY(buf_len < size())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid arguments", KR(ret), KP(buf), K(buf_len), + "request_size", size()); + } else if (OB_UNLIKELY(!is_valid())) { + ret = OB_INVALID_DATA; + STORAGE_LOG(WARN, "invalid tmp block cache value", KR(ret)); + } else { + ObTmpBlockCacheValue *blk_cache_value = new (buf) ObTmpBlockCacheValue(buf + sizeof(*this)); + MEMCPY(buf + sizeof(*this), buf_, size() - sizeof(*this)); + blk_cache_value->size_ = size_; + value = blk_cache_value; + } + return ret; +} + +/* -------------------------- ObTmpBlockCache --------------------------- */ + +ObTmpBlockCache &ObTmpBlockCache::get_instance() +{ + static ObTmpBlockCache instance; + return instance; +} + +int ObTmpBlockCache::init(const char *cache_name, const int64_t priority) +{ + int ret = OB_SUCCESS; + if (OB_FAIL((common::ObKVCache::init( + cache_name, priority)))) { + STORAGE_LOG(WARN, "Fail to init kv cache, ", KR(ret)); + } + return ret; +} + +void ObTmpBlockCache::destroy() +{ + common::ObKVCache::destroy(); +} + +int ObTmpBlockCache::get_block(const ObTmpBlockCacheKey &key, ObTmpBlockValueHandle &handle) +{ + int ret = OB_SUCCESS; + const ObTmpBlockCacheValue *value = NULL; + if (OB_UNLIKELY(!key.is_valid())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid arguments", KR(ret), K(key)); + } else if (OB_FAIL(get(key, value, handle.handle_))) { + if (OB_UNLIKELY(OB_ENTRY_NOT_EXIST != ret)) { + STORAGE_LOG(WARN, "fail to get key from block cache", KR(ret), K(key)); + } else { + EVENT_INC(ObStatEventIds::TMP_BLOCK_CACHE_MISS); + } + } else { + if (OB_ISNULL(value)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected error, the value must not be NULL", KR(ret)); + } else { + handle.value_ = const_cast(value); + EVENT_INC(ObStatEventIds::TMP_BLOCK_CACHE_HIT); + } + } + return ret; +} + +int ObTmpBlockCache::put_block(ObKVCacheInstHandle &inst_handle, + ObKVCachePair *&kvpair, + ObTmpBlockValueHandle &block_handle) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!inst_handle.is_valid() || nullptr == kvpair || !block_handle.is_valid())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(inst_handle), KP(kvpair), K(block_handle)); + } else if (OB_FAIL(put_kvpair(inst_handle, kvpair, block_handle.handle_, false/*overwrite*/))) { + STORAGE_LOG(WARN, "fail to put tmp block to block cache", KR(ret)); + } + + return ret; +} + +int ObTmpBlockCache::prealloc_block(const ObTmpBlockCacheKey &key, ObKVCacheInstHandle &inst_handle, + ObKVCachePair *&kvpair, + ObTmpBlockValueHandle &block_handle) +{ + int ret = OB_SUCCESS; + inst_handle.reset(); + kvpair = nullptr; + block_handle.reset(); + if (OB_UNLIKELY(!key.is_valid())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(key)); + } else if (OB_FAIL(alloc(key.get_tenant_id(), key.size(), + sizeof(ObTmpBlockCacheValue) + OB_DEFAULT_MACRO_BLOCK_SIZE, + kvpair, block_handle.handle_, inst_handle))) { + STORAGE_LOG(WARN, "failed to alloc kvcache buf", KR(ret), K(key)); + } else if (OB_FAIL(key.deep_copy(reinterpret_cast(kvpair->key_), + key.size(), kvpair->key_))) { + STORAGE_LOG(WARN, "failed to deep copy key", KR(ret), K(key)); + } else { + char *buf = reinterpret_cast(kvpair->value_); + block_handle.value_ = new (buf) ObTmpBlockCacheValue(buf + sizeof(ObTmpBlockCacheValue)); + } + + if (OB_FAIL(ret)) { + block_handle.reset(); + inst_handle.reset(); + kvpair = NULL; + } + return ret; +} + +/* -------------------------- ObTmpPageCacheKey --------------------------- */ +ObTmpPageCacheKey::ObTmpPageCacheKey() + : block_id_(-1), page_id_(-1), tenant_id_(OB_INVALID_TENANT_ID) +{ +} + +ObTmpPageCacheKey::ObTmpPageCacheKey(const int64_t block_id, const int64_t page_id, + const uint64_t tenant_id) + : block_id_(block_id), page_id_(page_id), tenant_id_(tenant_id) +{ +} + +ObTmpPageCacheKey::~ObTmpPageCacheKey() +{ +} + +bool ObTmpPageCacheKey::operator ==(const ObIKVCacheKey &other) const +{ + const ObTmpPageCacheKey &other_key = reinterpret_cast (other); + return block_id_ == other_key.block_id_ + && page_id_ == other_key.page_id_ + && tenant_id_ == other_key.tenant_id_; +} + +uint64_t ObTmpPageCacheKey::get_tenant_id() const +{ + return tenant_id_; +} + +uint64_t ObTmpPageCacheKey::hash() const +{ + return murmurhash(this, sizeof(ObTmpPageCacheKey), 0); +} + +int64_t ObTmpPageCacheKey::size() const +{ + return sizeof(*this); +} + +int ObTmpPageCacheKey::deep_copy(char *buf, const int64_t buf_len, ObIKVCacheKey *&key) const +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(buf) || OB_UNLIKELY(buf_len < size())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Invalid argument, ", KR(ret)); + } else if (OB_UNLIKELY(!is_valid())) { + ret = OB_INVALID_DATA; + STORAGE_LOG(WARN, "invalid tmp page cache key, ", KPC(this), KR(ret)); + } else { + key = new (buf) ObTmpPageCacheKey(block_id_, page_id_, tenant_id_); + } + return ret; +} + +bool ObTmpPageCacheKey::is_valid() const +{ + return OB_LIKELY(block_id_ >= 0 && page_id_ >= 0 && tenant_id_ > 0 && size() > 0); +} + +/* -------------------------- ObTmpPageCacheValue --------------------------- */ +ObTmpPageCacheValue::ObTmpPageCacheValue(char *buf) + : buf_(buf), size_(ObTmpFileGlobal::PAGE_SIZE) +{ +} + +ObTmpPageCacheValue::~ObTmpPageCacheValue() +{ +} + +int64_t ObTmpPageCacheValue::size() const +{ + return sizeof(*this) + size_; +} + +int ObTmpPageCacheValue::deep_copy(char *buf, const int64_t buf_len, ObIKVCacheValue *&value) const +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(buf) || OB_UNLIKELY(buf_len < size())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid arguments", KR(ret), KP(buf), K(buf_len), + "request_size", size()); + } else if (OB_UNLIKELY(!is_valid())) { + ret = OB_INVALID_DATA; + STORAGE_LOG(WARN, "invalid tmp page cache value", KR(ret)); + } else { + ObTmpPageCacheValue *page_cache_value = new (buf) ObTmpPageCacheValue(buf + sizeof(*this)); + MEMCPY(buf + sizeof(*this), buf_, size() - sizeof(*this)); + page_cache_value->size_ = size_; + value = page_cache_value; + } + return ret; +} + +/* -------------------------- ObTmpPageCache --------------------------- */ + +ObTmpPageCache &ObTmpPageCache::get_instance() +{ + static ObTmpPageCache instance; + return instance; +} + +int ObTmpPageCache::init(const char *cache_name, const int64_t priority) +{ + int ret = OB_SUCCESS; + if (OB_FAIL((common::ObKVCache::init( + cache_name, priority)))) { + STORAGE_LOG(WARN, "Fail to init kv cache, ", KR(ret)); + } + return ret; +} + +void ObTmpPageCache::destroy() +{ + common::ObKVCache::destroy(); +} + +int ObTmpPageCache::get_page(const ObTmpPageCacheKey &key, ObTmpPageValueHandle &handle) +{ + int ret = OB_SUCCESS; + const ObTmpPageCacheValue *value = NULL; + if (OB_UNLIKELY(!key.is_valid())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid arguments", KR(ret), K(key)); + } else if (OB_FAIL(get(key, value, handle.handle_))) { + if (OB_UNLIKELY(OB_ENTRY_NOT_EXIST != ret)) { + STORAGE_LOG(WARN, "fail to get key from page cache", KR(ret), K(key)); + } else { + EVENT_INC(ObStatEventIds::TMP_PAGE_CACHE_MISS); + } + } else { + if (OB_ISNULL(value)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected error, the value must not be NULL", KR(ret)); + } else { + handle.value_ = const_cast(value); + EVENT_INC(ObStatEventIds::TMP_PAGE_CACHE_HIT); + } + } + return ret; +} + +void ObTmpPageCache::try_put_page_to_cache(const ObTmpPageCacheKey &key, + const ObTmpPageCacheValue &value) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!key.is_valid() || !value.is_valid())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid arguments", KR(ret), K(key), K(value)); + } else if (OB_FAIL(put(key, value, true/*overwrite*/))) { + STORAGE_LOG(WARN, "fail to put tmp page into cache", KR(ret), K(key), K(value)); + } +} + +int ObTmpPageCache::load_page(const ObTmpPageCacheKey &key, + ObIAllocator *callback_allocator, + ObTmpPageValueHandle &p_handle) +{ + int ret = OB_SUCCESS; + ObKVCacheInstHandle inst_handle; + ObKVCachePair *kvpair = NULL; + p_handle.reset(); + if (OB_UNLIKELY(!key.is_valid())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(key)); + } else if (OB_ISNULL(callback_allocator)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "callback_allocator is unexpected nullptr", KR(ret), K(key)); + } else if (OB_FAIL(alloc(key.get_tenant_id(), key.size(), + sizeof(ObTmpPageCacheValue) + ObTmpFileGlobal::PAGE_SIZE, + kvpair, p_handle.handle_, inst_handle))) { + STORAGE_LOG(WARN, "failed to alloc kvcache buf", KR(ret), K(key)); + } else if (OB_FAIL(key.deep_copy(reinterpret_cast(kvpair->key_), + key.size(), kvpair->key_))) { + STORAGE_LOG(WARN, "failed to deep copy key", KR(ret), K(key)); + } else { + char *buf = reinterpret_cast(kvpair->value_); + p_handle.value_ = new (buf) ObTmpPageCacheValue(buf + sizeof(ObTmpPageCacheValue)); + } + if (OB_SUCC(ret)) { + ObTmpFileBlockManager &block_manager = MTL(ObTenantTmpFileManager*)->get_tmp_file_block_manager(); + blocksstable::ObMacroBlockHandle mb_handle; + blocksstable::MacroBlockId macro_block_id; + //TODO: io_desc and io_timeout_ms value settings + common::ObIOFlag io_desc; + io_desc.set_wait_event(ObWaitEventIds::DB_FILE_DATA_READ); + int64_t io_timeout_ms = 10 * 1000; // 10s + if (OB_FAIL(block_manager.get_macro_block_id(key.get_block_id(), macro_block_id))) { + STORAGE_LOG(WARN, "failed to get macro block id", KR(ret), K(key)); + } else if (OB_FAIL(direct_read(macro_block_id, ObTmpFileGlobal::PAGE_SIZE, + key.get_page_id() * ObTmpFileGlobal::PAGE_SIZE, + io_desc, io_timeout_ms, *callback_allocator, mb_handle))) { + STORAGE_LOG(WARN, "failed to alloc kvcache buf", KR(ret), K(key)); + } else if (OB_FAIL(mb_handle.wait())) { + STORAGE_LOG(WARN, "fail to do handle read wait", KR(ret), K(key)); + } else { + MEMCPY(p_handle.value_->get_buffer(), mb_handle.get_buffer(), ObTmpFileGlobal::PAGE_SIZE); + } + } + if (FAILEDx(put_kvpair(inst_handle, kvpair, p_handle.handle_, false/*overwrite*/))) { + if (OB_ENTRY_EXIST == ret) { + ret = OB_SUCCESS; + } else { + STORAGE_LOG(WARN, "fail to put tmp page to page cache", KR(ret), K(key)); + } + } + if (OB_FAIL(ret)) { + p_handle.reset(); + inst_handle.reset(); + kvpair = NULL; + } + return ret; +} + +// only read pages from disk +int ObTmpPageCache::direct_read(const blocksstable::MacroBlockId macro_block_id, + const int64_t read_size, + const int64_t begin_offset_in_block, + const common::ObIOFlag io_desc, + const int64_t io_timeout_ms, + common::ObIAllocator &callback_allocator, + blocksstable::ObMacroBlockHandle &mb_handle) +{ + int ret = OB_SUCCESS; + void *buf = nullptr; + ObTmpDirectReadPageIOCallback *callback = nullptr; + if (OB_UNLIKELY(!macro_block_id.is_valid() || read_size <= 0 || begin_offset_in_block < 0 || + (begin_offset_in_block + read_size > OB_DEFAULT_MACRO_BLOCK_SIZE) || + !io_desc.is_valid() || io_timeout_ms < 0)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Invalid arguments", KR(ret), K(macro_block_id), K(read_size), + K(begin_offset_in_block), K(io_desc), K(io_timeout_ms)); + } else if (OB_ISNULL(buf = callback_allocator.alloc(sizeof(ObTmpDirectReadPageIOCallback)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + STORAGE_LOG(WARN, "allocate callback memory failed", KR(ret)); + } else { + callback = new (buf) ObTmpDirectReadPageIOCallback; + callback->cache_ = this; + callback->allocator_ = &callback_allocator; + if (OB_FAIL(inner_read_io_(macro_block_id, read_size, begin_offset_in_block, + io_desc, io_timeout_ms, callback, mb_handle))) { + STORAGE_LOG(WARN, "fail to inner read io", KR(ret), K(macro_block_id), K(read_size), + K(begin_offset_in_block), K(io_desc), K(io_timeout_ms)); + } + // There is no need to handle error cases (freeing the memory of the + // callback) because inner_read_io_ will handle error cases and free the + // memory of the callback. + } + return ret; +} + +// read pages from disk and put them into kv_cache +int ObTmpPageCache::cached_read(const common::ObIArray &page_keys, + const blocksstable::MacroBlockId macro_block_id, + const int64_t begin_offset_in_block, + const common::ObIOFlag io_desc, + const int64_t io_timeout_ms, + common::ObIAllocator &callback_allocator, + blocksstable::ObMacroBlockHandle &mb_handle) +{ + int ret = OB_SUCCESS; + void *buf = nullptr; + ObTmpCachedReadPageIOCallback *callback = nullptr; + const int64_t read_size = page_keys.count() * ObTmpFileGlobal::PAGE_SIZE; + if (OB_UNLIKELY(page_keys.count() <= 0)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Invalid arguments", KR(ret), K(page_keys.count())); + } else if (OB_UNLIKELY(!macro_block_id.is_valid() || begin_offset_in_block < 0 || + !io_desc.is_valid() || io_timeout_ms < 0)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Invalid arguments", KR(ret), K(macro_block_id), K(begin_offset_in_block), + K(io_desc), K(io_timeout_ms)); + } else if (OB_UNLIKELY(begin_offset_in_block % ObTmpFileGlobal::PAGE_SIZE != 0)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "cached_read should read completed pages", KR(ret), K(begin_offset_in_block)); + } else if (OB_UNLIKELY(begin_offset_in_block + read_size > OB_DEFAULT_MACRO_BLOCK_SIZE)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid range", KR(ret), K(read_size), K(begin_offset_in_block)); + } else if (OB_ISNULL(buf = callback_allocator.alloc(sizeof(ObTmpCachedReadPageIOCallback)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + STORAGE_LOG(WARN, "allocate callback memory failed", KR(ret), K(macro_block_id)); + } else { + callback = new (buf) ObTmpCachedReadPageIOCallback; + callback->cache_ = this; + callback->allocator_ = &callback_allocator; + if (OB_FAIL(callback->page_keys_.assign(page_keys))) { + STORAGE_LOG(WARN, "fail to assign page keys", KR(ret), K(page_keys.count())); + } else if (OB_FAIL(inner_read_io_(macro_block_id, read_size, begin_offset_in_block, + io_desc, io_timeout_ms, callback, mb_handle))) { + STORAGE_LOG(WARN, "fail to inner read io", KR(ret), K(macro_block_id), K(read_size), + K(begin_offset_in_block), K(io_desc), K(io_timeout_ms)); + } + // There is no need to handle error cases (freeing the memory of the + // callback) because inner_read_io_ will handle error cases and free the + // memory of the callback. + } + return ret; +} + +// TODO: wanyue.wy +// refactor logic of callback. +// currently, the buffer of callback is freed in different places. +// if async_read_() is failed, it will be released in here; +// otherwise, it will be released after ObTmpFileIOCtx::do_read_wait_() has been called. +// (callback is bound with ObIOResult of io_handle, do_read_wait_() will call reset() of io_handle to +// destroy callback and its data buf) +// we need to refactor it by removing callback allocator and directly copy read io buf to user buf +// rather than using a callback buf +int ObTmpPageCache::inner_read_io_(const blocksstable::MacroBlockId macro_block_id, + const int64_t read_size, + const int64_t begin_offset_in_block, + const common::ObIOFlag io_desc, + const int64_t io_timeout_ms, + ObITmpPageIOCallback *callback, + blocksstable::ObMacroBlockHandle &handle) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(async_read_(macro_block_id, read_size, begin_offset_in_block, + io_desc, io_timeout_ms, callback, handle))) { + STORAGE_LOG(WARN, "fail to read tmp page from io", KR(ret), K(macro_block_id), K(read_size), + K(begin_offset_in_block), K(io_desc), K(io_timeout_ms), KP(callback)); + } + + // if read successful, callback will be freed after user calls ObTmpFileIOHandle::wait() + // for copying data from callback's buf to user's buf + // thus, here just free memory of the failed cases + if (OB_FAIL(ret) && OB_NOT_NULL(callback) && OB_NOT_NULL(callback->get_allocator())) { + common::ObIAllocator *allocator = callback->get_allocator(); + callback->~ObITmpPageIOCallback(); + allocator->free(callback); + } + + return ret; +} + +int ObTmpPageCache::async_read_(const blocksstable::MacroBlockId macro_block_id, + const int64_t read_size, + const int64_t begin_offset_in_block, + const common::ObIOFlag io_desc, + const int64_t io_timeout_ms, + ObITmpPageIOCallback *callback, + blocksstable::ObMacroBlockHandle &handle) +{ + int ret = OB_SUCCESS; + blocksstable::ObMacroBlockReadInfo read_info; + read_info.macro_block_id_ = macro_block_id; + read_info.size_ = read_size; + read_info.offset_ = begin_offset_in_block; + read_info.io_desc_ = io_desc; + read_info.io_timeout_ms_ = io_timeout_ms; + read_info.io_callback_ = callback; + read_info.io_desc_.set_resource_group_id(THIS_WORKER.get_group_id()); + read_info.io_desc_.set_sys_module_id(ObIOModule::TMP_TENANT_MEM_BLOCK_IO); + + if (OB_FAIL(handle.async_read(read_info))) { + STORAGE_LOG(WARN, "fail to async read block", KR(ret), K(read_info)); + } + return ret; +} + +ObTmpPageCache::ObITmpPageIOCallback::ObITmpPageIOCallback() + : cache_(NULL), allocator_(NULL), data_buf_(NULL) +{ + static_assert(sizeof(*this) <= CALLBACK_BUF_SIZE, "IOCallback buf size not enough"); +} + +ObTmpPageCache::ObITmpPageIOCallback::~ObITmpPageIOCallback() +{ + if (NULL != allocator_ && NULL != data_buf_) { + allocator_->free(data_buf_); + data_buf_ = NULL; + } + allocator_ = NULL; +} + +int ObTmpPageCache::ObITmpPageIOCallback::alloc_data_buf(const char *io_data_buffer, const int64_t data_size) +{ + int ret = alloc_and_copy_data(io_data_buffer, data_size, allocator_, data_buf_); + return ret; +} + +int ObTmpPageCache::ObITmpPageIOCallback::process_page( + const ObTmpPageCacheKey &key, const ObTmpPageCacheValue &value) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!key.is_valid() || !value.is_valid())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid arguments", KR(ret), K(key), K(value)); + } else if (OB_FAIL(cache_->put(key, value, true/*overwrite*/))) { + STORAGE_LOG(WARN, "fail to put tmp page into cache", KR(ret), K(key), K(value)); + } + return ret; +} + +ObTmpPageCache::ObTmpCachedReadPageIOCallback::ObTmpCachedReadPageIOCallback() + : page_keys_() +{ + static_assert(sizeof(*this) <= CALLBACK_BUF_SIZE, "IOCallback buf size not enough"); + page_keys_.set_attr(ObMemAttr(MTL_ID(), "TFCacheRead")); +} + +ObTmpPageCache::ObTmpCachedReadPageIOCallback::~ObTmpCachedReadPageIOCallback() +{ + page_keys_.reset(); +} + +int ObTmpPageCache::ObTmpCachedReadPageIOCallback::inner_process(const char *data_buffer, const int64_t size) +{ + int ret = OB_SUCCESS; + ObTimeGuard time_guard("TmpCachedReadPage_Callback_Process", 100000); //100ms + if (OB_ISNULL(cache_) || OB_ISNULL(allocator_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "Invalid tmp page cache callback allocator", KR(ret), KP(cache_), KP(allocator_)); + } else if (OB_UNLIKELY(size <= 0 || data_buffer == nullptr)) { + ret = OB_INVALID_DATA; + STORAGE_LOG(WARN, "invalid data buffer size", KR(ret), K(size), KP(data_buffer)); + } else if (OB_UNLIKELY(page_keys_.count() * ObTmpFileGlobal::PAGE_SIZE != size)) { + ret = OB_INVALID_DATA; + STORAGE_LOG(WARN, "invalid data buffer size", KR(ret), K(size), K(page_keys_.count())); + } else if (OB_FAIL(alloc_data_buf(data_buffer, size))) { + STORAGE_LOG(WARN, "Fail to allocate memory, ", KR(ret), K(size)); + } else if (FALSE_IT(time_guard.click("alloc_data_buf"))) { + } else { + for (int32_t i = 0; OB_SUCC(ret) && i < page_keys_.count(); i++) { + ObTmpPageCacheValue value(nullptr); + value.set_buffer(data_buf_ + i * ObTmpFileGlobal::PAGE_SIZE); + if (OB_FAIL(process_page(page_keys_.at(i), value))) { + STORAGE_LOG(WARN, "fail to process tmp page cache in callback", KR(ret)); + } + } + time_guard.click("process_page"); + } + if (OB_FAIL(ret) && NULL != allocator_ && NULL != data_buf_) { + allocator_->free(data_buf_); + data_buf_ = NULL; + } + return ret; +} + +int ObTmpPageCache::ObTmpDirectReadPageIOCallback::inner_process(const char *data_buffer, const int64_t size) +{ + int ret = OB_SUCCESS; + ObTimeGuard time_guard("ObTmpDirectReadPageIOCallback", 100000); //100ms + if (OB_ISNULL(cache_) || OB_ISNULL(allocator_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "Invalid tmp page cache callback allocator", KR(ret), KP(cache_), KP(allocator_)); + } else if (OB_UNLIKELY(size <= 0 || data_buffer == nullptr)) { + ret = OB_INVALID_DATA; + STORAGE_LOG(WARN, "invalid data buffer size", KR(ret), K(size), KP(data_buffer)); + } else if (OB_FAIL(alloc_data_buf(data_buffer, size))) { + STORAGE_LOG(WARN, "Fail to allocate memory, ", KR(ret), K(size)); + } else if (FALSE_IT(time_guard.click("alloc_data_buf"))) { + } + if (OB_FAIL(ret) && NULL != allocator_ && NULL != data_buf_) { + allocator_->free(data_buf_); + data_buf_ = NULL; + } + return ret; +} + +} // end namespace tmp_file +} // end namespace oceanbase diff --git a/src/storage/tmp_file/ob_tmp_file_cache.h b/src/storage/tmp_file/ob_tmp_file_cache.h new file mode 100644 index 000000000..bece9b6ab --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_cache.h @@ -0,0 +1,251 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_CACHE_H_ +#define OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_CACHE_H_ + +#include "lib/hash/ob_hashmap.h" +#include "lib/queue/ob_link_queue.h" +#include "share/io/ob_io_manager.h" +#include "share/cache/ob_kv_storecache.h" +#include "storage/ob_i_store.h" + +namespace oceanbase +{ +namespace tmp_file +{ +class ObTmpBlockCacheKey final : public common::ObIKVCacheKey +{ +public: + ObTmpBlockCacheKey(); + ObTmpBlockCacheKey(const int64_t block_id, const uint64_t tenant_id); + ~ObTmpBlockCacheKey(); + bool operator ==(const ObIKVCacheKey &other) const override; + uint64_t get_tenant_id() const override; + uint64_t hash() const override; + int64_t size() const override; + int deep_copy(char *buf, const int64_t buf_len, ObIKVCacheKey *&key) const override; + bool is_valid() const; + int64_t get_block_id() const { return block_id_; } + TO_STRING_KV(K(block_id_), K(tenant_id_)); + +private: + int64_t block_id_; + uint64_t tenant_id_; +}; + +class ObTmpBlockCacheValue final : public common::ObIKVCacheValue +{ +public: + explicit ObTmpBlockCacheValue(char *buf); + ~ObTmpBlockCacheValue(); + int64_t size() const override; + int deep_copy(char *buf, const int64_t buf_len, ObIKVCacheValue *&value) const override; + bool is_valid() const { return NULL != buf_ && size() > 0; } + char *get_buffer() { return buf_; } + void set_buffer(char *buf) { buf_ = buf;} + TO_STRING_KV(KP(buf_), K(size_)); + +private: + char *buf_; + int64_t size_; + DISALLOW_COPY_AND_ASSIGN(ObTmpBlockCacheValue); +}; + +struct ObTmpBlockValueHandle final +{ +public: + ObTmpBlockValueHandle() : value_(NULL), handle_() {} + ~ObTmpBlockValueHandle() = default; + bool is_valid() const { return NULL != value_ && handle_.is_valid(); } + void reset() + { + handle_.reset(); + value_ = NULL; + } + TO_STRING_KV(KP(value_), K(handle_)); + ObTmpBlockCacheValue *value_; + common::ObKVCacheHandle handle_; +}; + +class ObTmpBlockCache final : public common::ObKVCache +{ +public: + typedef common::ObKVCache BasePageCache; + static ObTmpBlockCache &get_instance(); + int init(const char *cache_name, const int64_t priority); + void destroy(); + int get_block(const ObTmpBlockCacheKey &key, ObTmpBlockValueHandle &handle); + int put_block(ObKVCacheInstHandle &inst_handle, + ObKVCachePair *&kvpair, + ObTmpBlockValueHandle &block_handle); + int prealloc_block(const ObTmpBlockCacheKey &key, + ObKVCacheInstHandle &inst_handle, + ObKVCachePair *&kvpair, + ObTmpBlockValueHandle &block_handle); +private: + ObTmpBlockCache() {} + ~ObTmpBlockCache() {} + +private: + DISALLOW_COPY_AND_ASSIGN(ObTmpBlockCache); +}; + +class ObTmpPageCacheKey final : public common::ObIKVCacheKey +{ +public: + ObTmpPageCacheKey(); + ObTmpPageCacheKey(const int64_t block_id, const int64_t page_id, const uint64_t tenant_id); + ~ObTmpPageCacheKey(); + bool operator ==(const ObIKVCacheKey &other) const override; + uint64_t get_tenant_id() const override; + uint64_t hash() const override; + int64_t size() const override; + int deep_copy(char *buf, const int64_t buf_len, ObIKVCacheKey *&key) const override; + bool is_valid() const; + int64_t get_page_id() const { return page_id_; } + int64_t get_block_id() const { return block_id_; } + TO_STRING_KV(K(block_id_), K(page_id_), K(tenant_id_)); + +private: + int64_t block_id_; + int64_t page_id_; + uint64_t tenant_id_; +}; + +class ObTmpPageCacheValue final : public common::ObIKVCacheValue +{ +public: + explicit ObTmpPageCacheValue(char *buf); + ~ObTmpPageCacheValue(); + int64_t size() const override; + int deep_copy(char *buf, const int64_t buf_len, ObIKVCacheValue *&value) const override; + bool is_valid() const { return NULL != buf_ && size() > 0; } + char *get_buffer() { return buf_; } + void set_buffer(char *buf) { buf_ = buf;} + TO_STRING_KV(KP(buf_), K(size_)); + +private: + char *buf_; + int64_t size_; + DISALLOW_COPY_AND_ASSIGN(ObTmpPageCacheValue); +}; + +struct ObTmpPageValueHandle final +{ +public: + ObTmpPageValueHandle() : value_(NULL), handle_() {} + ~ObTmpPageValueHandle() = default; + void reset() + { + handle_.reset(); + value_ = NULL; + } + TO_STRING_KV(KP(value_), K(handle_)); + ObTmpPageCacheValue *value_; + common::ObKVCacheHandle handle_; +}; + +class ObTmpPageCache final : public common::ObKVCache +{ +public: + typedef common::ObKVCache BasePageCache; + static ObTmpPageCache &get_instance(); + int init(const char *cache_name, const int64_t priority); + int direct_read(const blocksstable::MacroBlockId macro_block_id, + const int64_t read_size, + const int64_t begin_offset_in_block, + const common::ObIOFlag io_desc, + const int64_t io_timeout_ms, + common::ObIAllocator &callback_allocator, + blocksstable::ObMacroBlockHandle &mb_handle); + // multi page cached_read + int cached_read(const common::ObIArray &page_keys, + const blocksstable::MacroBlockId macro_block_id, + const int64_t begin_offset_in_block, + const common::ObIOFlag io_desc, + const int64_t io_timeout_ms, + common::ObIAllocator &callback_allocator, + blocksstable::ObMacroBlockHandle &mb_handle); + int get_page(const ObTmpPageCacheKey &key, ObTmpPageValueHandle &handle); + int load_page(const ObTmpPageCacheKey &key, + ObIAllocator *callback_allocator, + ObTmpPageValueHandle &p_handle); + void try_put_page_to_cache(const ObTmpPageCacheKey &key, const ObTmpPageCacheValue &value); + void destroy(); +public: + class ObITmpPageIOCallback : public common::ObIOCallback + { + public: + ObITmpPageIOCallback(); + virtual ~ObITmpPageIOCallback(); + virtual int alloc_data_buf(const char *io_data_buffer, const int64_t data_size) override; + const char *get_data() override { return data_buf_; } + protected: + friend class ObTmpPageCache; + virtual int process_page(const ObTmpPageCacheKey &key, const ObTmpPageCacheValue &value); + virtual ObIAllocator *get_allocator() { return allocator_; } + + protected: + BasePageCache *cache_; + common::ObIAllocator *allocator_; + char *data_buf_; // actual data buffer + }; + + class ObTmpCachedReadPageIOCallback final : public ObITmpPageIOCallback + { + public: + ObTmpCachedReadPageIOCallback(); + ~ObTmpCachedReadPageIOCallback(); + int64_t size() const override { return sizeof(*this); } + int inner_process(const char *data_buffer, const int64_t size) override; + TO_STRING_KV("callback_type:", "ObTmpCachedReadPageIOCallback", KP(data_buf_)); + DISALLOW_COPY_AND_ASSIGN(ObTmpCachedReadPageIOCallback); + private: + friend class ObTmpPageCache; + common::ObArray page_keys_; + }; + class ObTmpDirectReadPageIOCallback final : public ObITmpPageIOCallback + { + public: + ObTmpDirectReadPageIOCallback() {} + ~ObTmpDirectReadPageIOCallback() {} + int64_t size() const override { return sizeof(*this); } + int inner_process(const char *data_buffer, const int64_t size) override; + TO_STRING_KV("callback_type:", "ObTmpDirectReadPageIOCallback", KP(data_buf_)); + DISALLOW_COPY_AND_ASSIGN(ObTmpDirectReadPageIOCallback); + }; +private: + ObTmpPageCache() {} + ~ObTmpPageCache() {} + int inner_read_io_(const blocksstable::MacroBlockId macro_block_id, + const int64_t read_size, + const int64_t offset_in_block, + const common::ObIOFlag io_desc, + const int64_t io_timeout_ms, + ObITmpPageIOCallback *callback, + blocksstable::ObMacroBlockHandle &handle); + int async_read_(const blocksstable::MacroBlockId macro_block_id, + const int64_t read_size, + const int64_t offset_in_block, + const common::ObIOFlag io_desc, + const int64_t io_timeout_ms, + ObITmpPageIOCallback *callback, + blocksstable::ObMacroBlockHandle &handle); + +private: + DISALLOW_COPY_AND_ASSIGN(ObTmpPageCache); +}; + +} // end namespace tmp_file +} // end namespace oceanbase +#endif // OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_CACHE_H_ diff --git a/src/storage/tmp_file/ob_tmp_file_eviction_manager.cpp b/src/storage/tmp_file/ob_tmp_file_eviction_manager.cpp new file mode 100644 index 000000000..7e43812e7 --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_eviction_manager.cpp @@ -0,0 +1,218 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE + +#include "storage/tmp_file/ob_tmp_file_eviction_manager.h" + +namespace oceanbase +{ +using namespace storage; +using namespace share; + +namespace tmp_file +{ +void ObTmpFileEvictionManager::destroy() +{ + { + ObSpinLockGuard guard(meta_list_lock_); + file_meta_eviction_list_.reset(); + } + { + ObSpinLockGuard guard(data_list_lock_); + file_data_eviction_list_.reset(); + } +} + +int64_t ObTmpFileEvictionManager::get_file_size() +{ + int64_t file_size = 0; + { + ObSpinLockGuard guard(meta_list_lock_); + file_size += file_meta_eviction_list_.get_size(); + } + { + ObSpinLockGuard guard(data_list_lock_); + file_size += file_data_eviction_list_.get_size(); + } + return file_size; +} + +int ObTmpFileEvictionManager::add_file(const bool is_meta, ObSharedNothingTmpFile &file) +{ + int ret = OB_SUCCESS; + ObSpinLock &lock = is_meta ? meta_list_lock_ : data_list_lock_; + TmpFileEvictionList &eviction_list = is_meta ? file_meta_eviction_list_ : file_data_eviction_list_; + ObSharedNothingTmpFile::ObTmpFileNode &eviction_node = is_meta ? file.get_meta_eviction_node() : file.get_data_eviction_node(); + + ObSpinLockGuard guard(lock); + file.inc_ref_cnt(); + if (OB_UNLIKELY(!eviction_list.add_last(&eviction_node))) { + file.dec_ref_cnt(); + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to add node", KR(ret)); + } + + return ret; +} + +int ObTmpFileEvictionManager::remove_file(ObSharedNothingTmpFile &file) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(remove_file(true, file))) { + LOG_WARN("fail to remove file from meta list", KR(ret), K(file)); + } else if (OB_FAIL(remove_file(false, file))) { + LOG_WARN("fail to remove file from data list", KR(ret), K(file)); + } + return ret; +} + +int ObTmpFileEvictionManager::remove_file(const bool is_meta, ObSharedNothingTmpFile &file) +{ + int ret = OB_SUCCESS; + ObSpinLock &lock = is_meta ? meta_list_lock_ : data_list_lock_; + TmpFileEvictionList &eviction_list = is_meta ? file_meta_eviction_list_ : file_data_eviction_list_; + ObSharedNothingTmpFile::ObTmpFileNode &eviction_node = is_meta ? file.get_meta_eviction_node() : file.get_data_eviction_node(); + ObSpinLockGuard guard(lock); + if (OB_NOT_NULL(eviction_node.get_next())) { + if (OB_UNLIKELY(!eviction_list.remove(&eviction_node))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to remove node", KR(ret), K(file)); + } else { + file.dec_ref_cnt(); + } + } + + return ret; +} + +int ObTmpFileEvictionManager::evict(const int64_t expected_evict_page_num, int64_t &actual_evict_page_num) +{ + int ret = OB_SUCCESS; + int64_t remain_evict_page_num = expected_evict_page_num; + actual_evict_page_num = 0; + + int64_t actual_evict_data_page_num = 0; + if (OB_FAIL(evict_file_from_list_(false/*is_meta*/, remain_evict_page_num, actual_evict_data_page_num))) { + LOG_WARN("fail to evict file from list", KR(ret), K(remain_evict_page_num), K(actual_evict_data_page_num)); + } else { + remain_evict_page_num -= actual_evict_data_page_num; + actual_evict_page_num += actual_evict_data_page_num; + } + LOG_DEBUG("evict data pages over", KR(ret), K(expected_evict_page_num), K(remain_evict_page_num), + K(actual_evict_page_num), K(actual_evict_data_page_num)); + + int64_t actual_evict_meta_page_num = 0; + if (FAILEDx(evict_file_from_list_(true/*is_meta*/, remain_evict_page_num, actual_evict_meta_page_num))) { + LOG_WARN("fail to evict file from list", KR(ret), K(remain_evict_page_num), K(actual_evict_meta_page_num)); + } else { + remain_evict_page_num -= actual_evict_meta_page_num; + actual_evict_page_num += actual_evict_meta_page_num; + } + LOG_DEBUG("evict meta pages over", KR(ret), K(expected_evict_page_num), K(remain_evict_page_num), + K(actual_evict_page_num), K(actual_evict_meta_page_num)); + + return ret; +} + +int ObTmpFileEvictionManager::evict_file_from_list_(const bool &is_meta, + const int64_t expected_evict_page_num, + int64_t &actual_evict_page_num) +{ + int ret = OB_SUCCESS; + bool is_empty_list = false; + int64_t remain_evict_page_num = expected_evict_page_num; + actual_evict_page_num = 0; + + // attention: + // in order to avoid repeated inserting file node into list by flush manager, + // even thought evict manager has popped file from list, + // we also keep the file's `is_in_meta_eviction_list_` or `is_in_data_eviction_list_` be true. + // in evict_page() of file, if `remain_flushed_file_page_num` > 0, it will reinsert file node + // into eviction list; otherwise, it will set `is_in_meta_eviction_list_` or `is_in_data_eviction_list_` be false + + int64_t list_cnt = + is_meta ? file_meta_eviction_list_.get_size() : file_data_eviction_list_.get_size(); + + while(OB_SUCC(ret) && remain_evict_page_num > 0 && !is_empty_list && list_cnt-- > 0) { + ObTmpFileHandle file_handle; + int64_t actual_evict_file_page_num = 0; + int64_t remain_flushed_file_page_num = 0; + if (OB_FAIL(pop_file_from_list_(is_meta, file_handle))) { + if (OB_EMPTY_RESULT == ret) { + ret = OB_SUCCESS; + is_empty_list = true; + LOG_DEBUG("no tmp file in list", K(is_meta), K(expected_evict_page_num), + K(remain_evict_page_num), K(actual_evict_page_num)); + } else { + LOG_WARN("fail to pop file from list", KR(ret), K(is_meta)); + } + } else if (OB_ISNULL(file_handle.get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("file handle is invalid", KR(ret), K(file_handle)); + } else if (is_meta) { + if (OB_FAIL(file_handle.get()->evict_meta_pages(remain_evict_page_num, + actual_evict_file_page_num))) { + LOG_WARN("fail to evict meta pages", KR(ret), K(file_handle), K(remain_evict_page_num), + K(actual_evict_file_page_num)); + } + } else { + if (OB_FAIL(file_handle.get()->evict_data_pages(remain_evict_page_num, + actual_evict_file_page_num, + remain_flushed_file_page_num))) { + LOG_WARN("fail to evict data pages", KR(ret), K(file_handle), K(remain_evict_page_num), + K(actual_evict_page_num), K(remain_flushed_file_page_num)); + } else if (OB_UNLIKELY(remain_evict_page_num > actual_evict_file_page_num && remain_flushed_file_page_num > 1)) { + // we allow to not evict the last data page + ret = OB_ERR_UNEXPECTED; + LOG_WARN("evict_data_pages unexpected finishes before expected pages are eliminated", + KR(ret), K(is_meta), K(file_handle), K(remain_evict_page_num), + K(actual_evict_page_num), K(actual_evict_file_page_num), + K(remain_flushed_file_page_num)); + } + } + + if (OB_SUCC(ret)) { + remain_evict_page_num -= actual_evict_file_page_num; + actual_evict_page_num += actual_evict_file_page_num; + } + } + + return ret; +} + +int ObTmpFileEvictionManager::pop_file_from_list_(const bool &is_meta, ObTmpFileHandle &file_handle) +{ + int ret = OB_SUCCESS; + file_handle.reset(); + ObSharedNothingTmpFile *file = nullptr; + ObSpinLock &lock = is_meta ? meta_list_lock_ : data_list_lock_; + TmpFileEvictionList &eviction_list = is_meta ? file_meta_eviction_list_ : file_data_eviction_list_; + + ObSpinLockGuard guard(lock); + if (eviction_list.is_empty()) { + ret = OB_EMPTY_RESULT; + LOG_DEBUG("eviction_list is empty", K(is_meta)); + } else if (OB_ISNULL(file = &eviction_list.remove_first()->file_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("file is null", KR(ret)); + } else if (OB_FAIL(file_handle.init(file))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to init file", KR(ret), KP(file)); + } else { + file->dec_ref_cnt(); + } + return ret; +} + +} // end namespace tmp_file +} // end namespace oceanbase diff --git a/src/storage/tmp_file/ob_tmp_file_eviction_manager.h b/src/storage/tmp_file/ob_tmp_file_eviction_manager.h new file mode 100644 index 000000000..6b72de249 --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_eviction_manager.h @@ -0,0 +1,56 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_ELIMINATION_MANAGER_H_ +#define OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_ELIMINATION_MANAGER_H_ + +#include "storage/blocksstable/ob_macro_block_id.h" +#include "storage/tmp_file/ob_shared_nothing_tmp_file.h" +#include "lib/allocator/ob_fifo_allocator.h" +#include "lib/list/ob_dlist.h" +#include "lib/lock/ob_spin_lock.h" + +namespace oceanbase +{ +namespace tmp_file +{ +class ObTmpFileEvictionManager +{ +public: + typedef ObSharedNothingTmpFile::ObTmpFileNode TmpFileNode; + typedef common::ObDList TmpFileEvictionList; + +public: + ObTmpFileEvictionManager() : data_list_lock_(), file_data_eviction_list_(), + meta_list_lock_(), file_meta_eviction_list_() {} + ~ObTmpFileEvictionManager() { destroy(); } + void destroy(); + int64_t get_file_size(); + int add_file(const bool is_meta, ObSharedNothingTmpFile &file); + int remove_file(ObSharedNothingTmpFile &file); + int remove_file(const bool is_meta, ObSharedNothingTmpFile &file); + int evict(const int64_t expected_evict_page_num, int64_t &actual_evict_page_num); + +private: + int evict_file_from_list_(const bool &is_meta, const int64_t expected_evict_page_num, int64_t &actual_evict_page_num); + int pop_file_from_list_(const bool &is_meta, ObTmpFileHandle &file_handle); +private: + ObSpinLock data_list_lock_; + TmpFileEvictionList file_data_eviction_list_; + ObSpinLock meta_list_lock_; + TmpFileEvictionList file_meta_eviction_list_; +}; + +} // end namespace tmp_file +} // end namespace oceanbase + +#endif // OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_ELIMINATION_MANAGER_H_ diff --git a/src/storage/tmp_file/ob_tmp_file_flush_ctx.cpp b/src/storage/tmp_file/ob_tmp_file_flush_ctx.cpp new file mode 100644 index 000000000..b2c759e3c --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_flush_ctx.cpp @@ -0,0 +1,321 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE + +#include "share/ob_errno.h" // KR +#include "storage/tmp_file/ob_tmp_file_flush_ctx.h" + +namespace oceanbase +{ +namespace tmp_file +{ + +ObTmpFileFlushInfo::ObTmpFileFlushInfo() + : fd_(ObTmpFileGlobal::INVALID_TMP_FILE_FD), + batch_flush_idx_(0), + has_last_page_lock_(false), + insert_meta_tree_done_(false), + update_meta_data_done_(false), + file_handle_(), + flush_data_page_disk_begin_id_(ObTmpFileGlobal::INVALID_PAGE_ID), + flush_data_page_num_(-1), + flush_virtual_page_id_(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID), + file_size_(0), + flush_meta_page_array_() +{ + flush_meta_page_array_.set_attr(ObMemAttr(MTL_ID(), "TFFlushMetaArr")); +} + +void ObTmpFileFlushInfo::reset() +{ + fd_ = ObTmpFileGlobal::INVALID_TMP_FILE_FD; + batch_flush_idx_ = 0; + has_last_page_lock_ = false; + insert_meta_tree_done_ = false; + update_meta_data_done_ = false; + file_handle_.reset(); + flush_data_page_disk_begin_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + flush_data_page_num_ = -1; + flush_virtual_page_id_ = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + file_size_ = 0; + flush_meta_page_array_.reset(); +} + +// -------------- ObTmpFileBatchFlushContext --------------- // + +int ObTmpFileBatchFlushContext::init() +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObTmpFileBatchFlushContext init twice", KR(ret)); + } else if (OB_FAIL(file_ctx_hash_.create(256, ObMemAttr(MTL_ID(), "TFileFLCtx")))) { + LOG_WARN("failed to create hash map", K(ret)); + } else { + flush_failed_array_.set_attr(ObMemAttr(MTL_ID(), "TFFlushFailArr")); + state_ = FlushCtxState::FSM_F1; + is_inited_ = true; + } + return ret; +} + +int ObTmpFileBatchFlushContext::prepare_flush_ctx( + const int64_t expect_flush_size, + ObTmpFileFlushPriorityManager *prio_mgr, + ObTmpFileFlushMonitor *flush_monitor) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(expect_flush_size <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(expect_flush_size)); + } else if (OB_ISNULL(prio_mgr) || OB_ISNULL(flush_monitor)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), KP(prio_mgr), KP(flush_monitor)); + } else if (OB_FAIL(iter_.init(prio_mgr))) { + LOG_WARN("failed to init iterator", KR(ret), K(*this)); + } else if (OB_FAIL(flush_failed_array_.reserve(MAX_COPY_FAIL_COUNT))) { + LOG_WARN("fail to reserve flush filed array", KR(ret), K(*this)); + } else { + expect_flush_size_ = expect_flush_size; + flush_monitor_ptr_ = flush_monitor; + } + return ret; +} + +int ObTmpFileBatchFlushContext::clear_flush_ctx(ObTmpFileFlushPriorityManager &prio_mgr) +{ + int ret = OB_SUCCESS; + // insert the remaining files in the iterator back into the flush priority manager + if (OB_FAIL(iter_.reset())) { + LOG_ERROR("failed to reset flush iterator", KR(ret), KPC(this)); + } + + // insert the files that failed to copy data back into the flush queue. after this step, + // call files taken out by the iterator for this round + // (excluding files with no dirty pages) will appear in the flush priority manager. + for (int64_t i = 0; OB_SUCC(ret) && i < flush_failed_array_.count(); i++) { + const ObTmpFileFlushFailRecord &record = flush_failed_array_.at(i); + const ObTmpFileHandle &file_handle = record.file_handle_; + if (OB_ISNULL(file_handle.get())) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("file handle is nullptr", KR(ret)); + } else { + ObSharedNothingTmpFile &file = *file_handle.get(); + file.reinsert_flush_node(record.is_meta_); + } + } + + if (OB_FAIL(ret)) { + } else if (flush_seq_ctx_.send_io_succ_cnt_ == flush_seq_ctx_.create_flush_task_cnt_) { + LOG_DEBUG("reset flush_seq_ctx_", KPC(this)); + flush_seq_ctx_.send_io_succ_cnt_ = 0; + flush_seq_ctx_.create_flush_task_cnt_ = 0; + flush_seq_ctx_.flush_sequence_ += 1; + // remove the files that have already been successfully flushed(recorded in file_ctx_hash_) + // from flush priority manager, ensuring that these files do not trigger flushing + // before the completion of this round of flush IO. + RemoveFileOp remove_op(prio_mgr); + if (OB_FAIL(file_ctx_hash_.foreach_refactored(remove_op))) { + LOG_ERROR("fail to erase file ctx from hash", KR(ret), KPC(this)); + } else { + file_ctx_hash_.clear(); + } + } else if (flush_seq_ctx_.send_io_succ_cnt_ < flush_seq_ctx_.create_flush_task_cnt_) { + // likely to occur when write buffer pool is full and need to fast flush meta page + LOG_WARN("flush_seq_ctx_ could not increase flush sequence", KPC(this)); + } else if (OB_UNLIKELY(flush_seq_ctx_.send_io_succ_cnt_ > flush_seq_ctx_.create_flush_task_cnt_)) { + LOG_ERROR("unexpected flush_seq_ctx_", KPC(this)); + } + + fail_too_many_ = false; + expect_flush_size_ = 0; + actual_flush_size_ = 0; + iter_.destroy(); + flush_failed_array_.reset(); + state_ = FlushCtxState::FSM_F1; + return ret; +} + +int ObTmpFileBatchFlushContext::RemoveFileOp::operator () (hash::HashMapPair &kv) +{ + int ret = OB_SUCCESS; + ObTmpFileHandle &file_handle = kv.second.file_handle_; + if (OB_ISNULL(file_handle.get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("file handle is nullptr", KR(ret)); + } else { + // remove all flush nodes for this file to prevent repeated flushing when flushing only data page or only meta page, + // this avoids triggering a data page flush while waiting for I/O on meta pages, which could lead to new data items + // being inserted to the flushing meta page, causing the memory meta page to become inconsistent with the page on disk. + ObSharedNothingTmpFile &file = *file_handle.get(); + if (file.is_flushing()) { + if (OB_FAIL(file.remove_flush_node(false/*is_meta*/))) { + LOG_ERROR("fail to remove file data node from flush priority mgr", KR(ret), K(file)); + } else if (OB_FAIL(file.remove_flush_node(true/*is_meta*/))) { + LOG_ERROR("fail to remove file meta node from flush priority mgr", KR(ret), K(file)); + } else { + LOG_DEBUG("succ to remove flush node from flush priority mgr", K(file)); + } + } + } + return ret; +} + +void ObTmpFileBatchFlushContext::destroy() +{ + is_inited_ = false; + fail_too_many_ = false; + expect_flush_size_ = 0; + actual_flush_size_ = 0; + flush_monitor_ptr_ = nullptr; + state_ = FlushCtxState::FSM_F1; + iter_.destroy(); + flush_failed_array_.reset(); + file_ctx_hash_.destroy(); +} + +void ObTmpFileBatchFlushContext::update_ctx_by_flush_task(const ObTmpFileFlushTask &flush_task) +{ + actual_flush_size_ += flush_task.get_data_length(); + + if (ObTmpFileFlushTask::TFFT_WAIT == flush_task.get_state() || flush_task.get_data_length() == 0) { + // we will free task if its data_length == 0, therefore we count it as succ here + flush_seq_ctx_.send_io_succ_cnt_ += 1; + } +} + +void ObTmpFileBatchFlushContext::record_flush_stage() +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(flush_monitor_ptr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("flush monitor is null", KR(ret)); + } else { + flush_monitor_ptr_->record_flush_stage(state_); + } +} + +void ObTmpFileBatchFlushContext::record_flush_task(const int64_t data_length) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(flush_monitor_ptr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("flush monitor is null", KR(ret)); + } else { + flush_monitor_ptr_->record_flush_task(data_length); + } +} + +// -------------- ObTmpFileFlushTask --------------- // + +ObTmpFileFlushTask::ObTmpFileFlushTask(ObIAllocator &task_allocator) + : inst_handle_(), + kvpair_(nullptr), + block_handle_(), + ret_code_(OB_SUCCESS), + data_length_(0), + block_index_(-1), + flush_seq_(-1), + create_ts_(-1), + is_io_finished_(false), + fast_flush_tree_page_(false), + task_state_(ObTmpFileFlushTaskState::TFFT_INITED), + tmp_file_block_handle_(), + handle_(), + flush_infos_(), + task_allocator_(task_allocator) +{ + flush_infos_.set_attr(ObMemAttr(MTL_ID(), "TFFlushInfos")); +} + +void ObTmpFileFlushTask::destroy() +{ + block_handle_.reset(); + inst_handle_.reset(); + kvpair_ = nullptr; + ret_code_ = OB_SUCCESS; + data_length_ = 0; + block_index_ = -1; + flush_seq_ = -1; + create_ts_ = -1; + is_io_finished_ = false; + fast_flush_tree_page_ = false; + task_state_ = ObTmpFileFlushTaskState::TFFT_INITED; + flush_infos_.reset(); +} + +int ObTmpFileFlushTask::prealloc_block_buf() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("prealloc block buf twice", KR(ret), KPC(this)); + } else if (OB_FAIL(ObTmpBlockCache::get_instance().prealloc_block( + ObTmpBlockCacheKey(block_index_, MTL_ID()), inst_handle_, kvpair_, block_handle_))) { + LOG_WARN("fail to prealloc block", KR(ret), K(block_index_), K(MTL_ID())); + } + return ret; +} + +int ObTmpFileFlushTask::write_one_block() +{ + int ret = OB_SUCCESS; + handle_.reset(); + + blocksstable::ObMacroBlockWriteInfo write_info; + write_info.io_desc_.set_wait_event(2); // TODO: 检查是否需要用临时文件自己的event + write_info.io_desc_.set_resource_group_id(THIS_WORKER.get_group_id()); + write_info.io_desc_.set_sys_module_id(ObIOModule::TMP_TENANT_MEM_BLOCK_IO); + write_info.buffer_ = get_data_buf(); + write_info.size_ = OB_SERVER_BLOCK_MGR.get_macro_block_size(); + write_info.offset_ = 0; + + if (FAILEDx(blocksstable::ObBlockManager::async_write_block(write_info, handle_))) { + LOG_ERROR("fail to async write block", KR(ret), K(write_info)); + } else if (OB_FAIL(OB_SERVER_BLOCK_MGR.update_write_time(handle_.get_macro_id(), + true/*update_to_max_time)*/))){ // update to max time to skip bad block inspect + LOG_WARN("failed to update write time", KR(ret), K(handle_)); + } + return ret; +} + +// async wait, return OB_EAGAIN immediately if IO is not finished +int ObTmpFileFlushTask::wait_macro_block_handle() +{ + int ret = OB_SUCCESS; + int64_t wait_timeout_ms = 0; // timeout == 0 for async wait + if (OB_FAIL(handle_.wait(wait_timeout_ms))) { + if (OB_EAGAIN == ret) { + // do nothing + } else { + atomic_set_ret_code(ret); + atomic_set_io_finished(true); + LOG_WARN("fail to wait macro block handle", KR(ret), KPC(this)); + ret = OB_SUCCESS; + } + } else { + atomic_set_ret_code(OB_SUCCESS); + atomic_set_io_finished(true); + LOG_DEBUG("macro block handle io finished", KR(ret), KPC(this)); + } + return ret; +} + +int64_t ObTmpFileFlushTask::get_total_page_num() const +{ + const int64_t PAGE_SIZE = ObTmpFileGlobal::PAGE_SIZE; + return upper_align(data_length_, PAGE_SIZE) / PAGE_SIZE; +} + +} // end namespace tmp_file +} // end namespace oceanbase diff --git a/src/storage/tmp_file/ob_tmp_file_flush_ctx.h b/src/storage/tmp_file/ob_tmp_file_flush_ctx.h new file mode 100644 index 000000000..ad7a8c8ca --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_flush_ctx.h @@ -0,0 +1,305 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_FLUSH_CTX_H_ +#define OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_FLUSH_CTX_H_ + +#include "lib/queue/ob_link_queue.h" +#include "lib/utility/ob_print_utils.h" +#include "storage/tmp_file/ob_tmp_file_block_manager.h" +#include "storage/tmp_file/ob_shared_nothing_tmp_file.h" +#include "storage/tmp_file/ob_tmp_file_global.h" +#include "storage/tmp_file/ob_tmp_file_flush_list_iterator.h" +#include "storage/tmp_file/ob_tmp_file_meta_tree.h" +#include "storage/tmp_file/ob_tmp_file_thread_job.h" + +namespace oceanbase +{ +namespace tmp_file +{ +class ObTmpFileFlushTG; + +struct ObTmpFileDataFlushContext +{ +public: + ObTmpFileDataFlushContext () + : is_valid_(false), + has_flushed_last_partially_written_page_(false), + flushed_page_id_(ObTmpFileGlobal::INVALID_PAGE_ID), + flushed_page_virtual_id_(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID), + next_flush_page_id_(ObTmpFileGlobal::INVALID_PAGE_ID), + next_flush_page_virtual_id_(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID) {} + + ~ObTmpFileDataFlushContext () { reset(); } + void reset() { + is_valid_ = false; + has_flushed_last_partially_written_page_ = false; + flushed_page_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + flushed_page_virtual_id_ = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + next_flush_page_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + next_flush_page_virtual_id_ = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + } + OB_INLINE bool is_valid() const { return is_valid_; } + OB_INLINE void set_is_valid(const bool is_valid) { is_valid_ = is_valid; } + OB_INLINE bool has_flushed_last_partially_written_page() const { + return has_flushed_last_partially_written_page_; + } + OB_INLINE void set_has_flushed_last_partially_written_page(bool has_flushed_last_partially_written_page) { + has_flushed_last_partially_written_page_ = has_flushed_last_partially_written_page; + } + OB_INLINE uint32_t get_flushed_page_id() const { return flushed_page_id_; } + OB_INLINE void set_flushed_page_id(const uint32_t flushed_page_id) { flushed_page_id_ = flushed_page_id; } + OB_INLINE int64_t get_flushed_page_virtual_id() const { + return flushed_page_virtual_id_; + } + OB_INLINE void set_flushed_page_virtual_id(int64_t virtual_page_id) { + flushed_page_virtual_id_ = virtual_page_id; + } + OB_INLINE uint32_t get_next_flush_page_id() const { return next_flush_page_id_; } + OB_INLINE void set_next_flush_page_id(const uint32_t next_flush_page_id) { + next_flush_page_id_ = next_flush_page_id; + } + OB_INLINE void set_next_flush_page_virtual_id(const int64_t virtual_page_id) { + next_flush_page_virtual_id_ = virtual_page_id; + } + OB_INLINE int64_t get_next_flush_page_virtual_id() const { return next_flush_page_virtual_id_; } + TO_STRING_KV(K(is_valid_), K(has_flushed_last_partially_written_page_), K(flushed_page_id_), + K(flushed_page_virtual_id_), + K(next_flush_page_id_), K(next_flush_page_virtual_id_)); +public: + bool is_valid_; + bool has_flushed_last_partially_written_page_; + uint32_t flushed_page_id_; + int64_t flushed_page_virtual_id_; + uint32_t next_flush_page_id_; + int64_t next_flush_page_virtual_id_; +}; + +struct ObTmpFileSingleFlushContext +{ +public: + ObTmpFileSingleFlushContext() : file_handle_(), data_ctx_(), meta_ctx_() {} + ObTmpFileSingleFlushContext(ObTmpFileHandle file_handle) : file_handle_(file_handle), data_ctx_(), meta_ctx_() {} + TO_STRING_KV(K(file_handle_), K(data_ctx_), K(meta_ctx_)); +public: + ObTmpFileHandle file_handle_; + ObTmpFileDataFlushContext data_ctx_; + ObTmpFileTreeFlushContext meta_ctx_; +}; + +// The ObTmpFileFlushManager maintains multiple file flushing contexts during a round of flushing +class ObTmpFileBatchFlushContext +{ +public: + ObTmpFileBatchFlushContext() + : is_inited_(), + fail_too_many_(false), + expect_flush_size_(0), + actual_flush_size_(0), + flush_seq_ctx_(), + flush_monitor_ptr_(nullptr), + state_(ObTmpFileGlobal::FlushCtxState::FSM_F1), + iter_(), + file_ctx_hash_(), + flush_failed_array_() + { + } + ~ObTmpFileBatchFlushContext() { destroy(); } + int init(); + int prepare_flush_ctx(const int64_t expect_flush_size, + ObTmpFileFlushPriorityManager *prio_mgr, + ObTmpFileFlushMonitor *flush_monitor); + int clear_flush_ctx(ObTmpFileFlushPriorityManager &prio_mgr); + void destroy(); + void record_flush_stage(); + void record_flush_task(const int64_t data_length); + void update_ctx_by_flush_task(const ObTmpFileFlushTask &flush_task); +public: + static const int64_t MAX_COPY_FAIL_COUNT = 512; + typedef hash::ObHashMap ObTmpFileFlushCtxHash; + struct ObTmpFileFlushFailRecord + { + public: + ObTmpFileFlushFailRecord() : is_meta_(false), file_handle_() {} + ObTmpFileFlushFailRecord(bool is_meta, ObTmpFileHandle file_handle) : is_meta_(is_meta), file_handle_(file_handle) {} + TO_STRING_KV(K(is_meta_), K(file_handle_)); + public: + bool is_meta_; + ObTmpFileHandle file_handle_; + }; + struct FlushSequenceContext + { + FlushSequenceContext() : create_flush_task_cnt_(0), send_io_succ_cnt_(0), flush_sequence_(0) {} + int64_t create_flush_task_cnt_; // created flush task number in one round + int64_t send_io_succ_cnt_; + int64_t flush_sequence_; // increate flush seq only when all task in this round send io succ + TO_STRING_KV(K(create_flush_task_cnt_), K(send_io_succ_cnt_), K(flush_sequence_)); + }; + struct RemoveFileOp + { + public: + RemoveFileOp(ObTmpFileFlushPriorityManager &flush_priority_mgr) : flush_priority_mgr_(flush_priority_mgr) {} + int operator () (hash::HashMapPair &kv); + private: + ObTmpFileFlushPriorityManager &flush_priority_mgr_; + }; + bool can_clear_flush_ctx() const + { + return flush_seq_ctx_.send_io_succ_cnt_ == flush_seq_ctx_.create_flush_task_cnt_ && + flush_seq_ctx_.send_io_succ_cnt_ > 0; + } + OB_INLINE void set_fail_too_many(const bool fail_too_many) { fail_too_many_ = fail_too_many; } + OB_INLINE bool is_fail_too_many() { return fail_too_many_; } + OB_INLINE ObTmpFileFlushListIterator &get_flush_list_iterator() { return iter_; } + OB_INLINE void set_expect_flush_size(const int64_t flush_size) { expect_flush_size_ = flush_size; } + OB_INLINE int64_t get_expect_flush_size() { return expect_flush_size_; } + OB_INLINE void add_actual_flush_size(const int64_t flush_size) { actual_flush_size_ += flush_size; } + OB_INLINE int64_t get_actual_flush_size() { return actual_flush_size_; } + OB_INLINE bool is_inited() { return is_inited_; } + OB_INLINE void set_state(const FlushCtxState state) { state_ = state; } + OB_INLINE FlushCtxState get_state() { return state_; } + OB_INLINE ObTmpFileFlushCtxHash &get_file_ctx_hash() { return file_ctx_hash_; } + OB_INLINE ObArray &get_flush_failed_array() { return flush_failed_array_; } + OB_INLINE ObTmpFileFlushMonitor *get_flush_monitor() { return flush_monitor_ptr_; } + OB_INLINE void inc_create_flush_task_cnt() { ++flush_seq_ctx_.create_flush_task_cnt_; } + OB_INLINE int64_t get_flush_sequence() { return flush_seq_ctx_.flush_sequence_; } + TO_STRING_KV(K(is_inited_), K(fail_too_many_), K(expect_flush_size_), K(actual_flush_size_), K(flush_seq_ctx_), K(state_), K(iter_)); +private: + bool is_inited_; + bool fail_too_many_; // indicate wether the number of file copy failures exceeds MAX_COPY_FAIL_COUNT in one round. + int64_t expect_flush_size_; + int64_t actual_flush_size_; + FlushSequenceContext flush_seq_ctx_; + ObTmpFileFlushMonitor *flush_monitor_ptr_; + FlushCtxState state_; + ObTmpFileFlushListIterator iter_; + ObTmpFileFlushCtxHash file_ctx_hash_; // maintain the data/meta flushing offset for each file in one round + ObArray flush_failed_array_; // record files that failed to copy data/meta pages, re-insert them into flush priority mgr + // after the iteration ends to prevent these files from appearing multiple times in flush iterator. +}; + +// represent up to 2MB flush information of a file, +// if a macro block contains multiple tmp files, it will include multiple flush infos +struct ObTmpFileFlushInfo +{ +public: + ObTmpFileFlushInfo(); + ~ObTmpFileFlushInfo() { reset(); } + void reset(); + bool has_data() const { return flush_data_page_num_ > 0; } + bool has_meta() const { return !flush_meta_page_array_.empty(); } + TO_STRING_KV(K(fd_), K(batch_flush_idx_), K(has_last_page_lock_), K(insert_meta_tree_done_), K(update_meta_data_done_), + K(flush_data_page_disk_begin_id_), K(flush_data_page_num_), K(flush_virtual_page_id_), K(file_size_), + K(flush_meta_page_array_), KP(file_handle_.get())); +public: + int64_t fd_; + int64_t batch_flush_idx_; // during one round of flushing, multiple FlushInfo may be generated. + // records the sequence number to which this info belongs (starting from 0). + bool has_last_page_lock_; // indicate the last page is in flushing and holds last_page_lock_ in file + bool insert_meta_tree_done_; // indicate the insertion of the corresponding data item into the meta tree is completed + bool update_meta_data_done_; // indicate the file metadata or metadata tree update is completed + ObTmpFileHandle file_handle_; + + // information for updating data + int64_t flush_data_page_disk_begin_id_; // record begin page id in the macro block, for updating meta tree item + int64_t flush_data_page_num_; + int64_t flush_virtual_page_id_; // record virtual_page_id while copying data, pass to meta tree while inserting items + int64_t file_size_; // if file_size > 0, it means the last page is in flushing + // information for updating meta tree + ObArray flush_meta_page_array_; +}; + +// Each ObTmpFileFlushTask corresponds to a flushing macro block, which can be exclusively used by one file +// or shared by multiple files based on observer config. The task internally maintains the state machine, +// and the ObTmpFileFlushManager continuously advances each task to TFFT_FINISH terminal state. +// Each task that are higher than TFFT_FILL_BLOCK_BUF will be retried if errors occurred. +struct ObTmpFileFlushTask : public common::ObSpLinkQueue::Link +{ +public: + ObTmpFileFlushTask(ObIAllocator &task_allocator); + ~ObTmpFileFlushTask() { destroy(); } + enum ObTmpFileFlushTaskState + { + TFFT_INITED = 0, + TFFT_ALLOC_BLOCK_BUF = 1, + TFFT_CREATE_BLOCK_INDEX = 2, + TFFT_FILL_BLOCK_BUF = 3, + TFFT_INSERT_META_TREE = 4, + TFFT_ASYNC_WRITE = 5, + TFFT_WAIT = 6, + TFFT_FINISH = 7, + }; +public: + void destroy(); + int prealloc_block_buf(); + int write_one_block(); + int wait_macro_block_handle(); + int64_t get_total_page_num() const; + OB_INLINE ObKVCacheInstHandle& get_inst_handle() { return inst_handle_; } + OB_INLINE ObKVCachePair*& get_kvpair() { return kvpair_; } + OB_INLINE ObTmpBlockValueHandle& get_block_handle() { return block_handle_; } + OB_INLINE bool is_valid() const { return OB_NOT_NULL(get_data_buf()); } + OB_INLINE bool is_full() const { return data_length_ == OB_SERVER_BLOCK_MGR.get_macro_block_size(); } + OB_INLINE char *get_data_buf() const { return block_handle_.value_ == nullptr ? nullptr : block_handle_.value_->get_buffer(); } + OB_INLINE void atomic_set_ret_code(int ret_code) { ATOMIC_SET(&ret_code_, ret_code); } + OB_INLINE int atomic_get_ret_code() const { return ATOMIC_LOAD(&ret_code_); } + OB_INLINE void set_data_length(const int64_t len) { data_length_ = len; } + OB_INLINE int64_t get_data_length() const { return data_length_; } + OB_INLINE void set_block_index(const int64_t block_index) { block_index_ = block_index; } + OB_INLINE int64_t get_block_index() const { return block_index_; } + OB_INLINE void set_flush_seq(const int64_t flush_seq) { flush_seq_ = flush_seq; } + OB_INLINE int64_t get_flush_seq() const { return flush_seq_; } + OB_INLINE void set_create_ts(const int64_t create_ts) { create_ts_ = create_ts; } + OB_INLINE int64_t get_create_ts() const { return create_ts_; } + OB_INLINE void atomic_set_io_finished(const bool is_finished) { ATOMIC_SET(&is_io_finished_, is_finished); } + OB_INLINE bool atomic_get_io_finished() const { return ATOMIC_LOAD(&is_io_finished_); } + OB_INLINE void set_is_fast_flush_tree(const bool is_fast_flush_tree) { fast_flush_tree_page_ = is_fast_flush_tree; } + OB_INLINE bool get_is_fast_flush_tree() const { return fast_flush_tree_page_; } + OB_INLINE void set_state(const ObTmpFileFlushTaskState state) { task_state_ = state; } + OB_INLINE ObTmpFileFlushTaskState get_state() const { return task_state_; } + OB_INLINE void set_tmp_file_block_handle(const ObTmpFileBlockHandle &tfb_handle) { tmp_file_block_handle_ = tfb_handle; } + OB_INLINE ObTmpFileBlockHandle &get_tmp_file_block_handle() { return tmp_file_block_handle_; } + OB_INLINE void set_macro_block_handle(const blocksstable::ObMacroBlockHandle &handle) { handle_ = handle; } + OB_INLINE blocksstable::ObMacroBlockHandle &get_macro_block_handle() { return handle_; } + OB_INLINE ObArray &get_flush_infos() { return flush_infos_; } + OB_INLINE int64_t get_next_free_page_id() { return get_total_page_num(); } + OB_INLINE bool check_buf_range_valid(const char* buffer, const int64_t length) const + { + return buffer != nullptr && get_data_buf() != nullptr && + buffer >= get_data_buf() && buffer + length <= get_data_buf() + OB_SERVER_BLOCK_MGR.get_macro_block_size(); + } + TO_STRING_KV(KP(this), KP(kvpair_), K(ret_code_), K(data_length_), + K(block_index_), K(flush_seq_), K(create_ts_), K(is_io_finished_), + K(fast_flush_tree_page_), K(task_state_), K(tmp_file_block_handle_), K(flush_infos_)); +private: + ObKVCacheInstHandle inst_handle_; + ObKVCachePair *kvpair_; + ObTmpBlockValueHandle block_handle_; + int ret_code_; + int64_t data_length_; // data length (including padding to make length upper align to page size) + int64_t block_index_; // tmp file block logical index in ObTmpFileBlockManager + int64_t flush_seq_; // flush sequence, for verification purpose + int64_t create_ts_; + bool is_io_finished_; + bool fast_flush_tree_page_; // indicate the task requires fast flush tree pages + ObTmpFileFlushTaskState task_state_; + ObTmpFileBlockHandle tmp_file_block_handle_;// hold a reference to the corresponding tmp file block to prevent it from being released + blocksstable::ObMacroBlockHandle handle_; + ObArray flush_infos_; // multi file flush into one block if size > 0 + ObIAllocator &task_allocator_; // ref to ObTmpFilePageCacheController::task_allocator_, used to free data_buf_ +}; + +} // end namespace tmp_file +} // end namespace oceanbase +#endif // OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_FLUSH_CTX_H_ diff --git a/src/storage/tmp_file/ob_tmp_file_flush_list_iterator.cpp b/src/storage/tmp_file/ob_tmp_file_flush_list_iterator.cpp new file mode 100644 index 000000000..7ca0b8eb1 --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_flush_list_iterator.cpp @@ -0,0 +1,776 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE + +#include "lib/utility/ob_sort.h" +#include "storage/tmp_file/ob_tmp_file_flush_list_iterator.h" + +namespace oceanbase +{ +namespace tmp_file +{ +ObTmpFileFlushListIterator::ObTmpFileFlushListIterator() : + is_inited_(false), files_(), dirs_(), cur_caching_list_is_meta_(false), + cur_caching_list_idx_(ObTmpFileFlushPriorityManager::FileList::L1), + cur_iter_dir_idx_(-1), cur_iter_file_idx_(-1), + cached_file_num_(0), cached_dir_num_(0) +{} + +ObTmpFileFlushListIterator::~ObTmpFileFlushListIterator() +{ + destroy(); +} + +int ObTmpFileFlushListIterator::init(ObTmpFileFlushPriorityManager *prio_mgr) +{ + int ret = OB_SUCCESS; + if (is_inited_) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", KR(ret)); + } else if (OB_ISNULL(prio_mgr)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), KP(prio_mgr)); + } else if (FALSE_IT(files_.set_attr(ObMemAttr(MTL_ID(), "TFFlushIterFile")))) { + } else if (OB_FAIL(files_.prepare_allocate(MAX_CACHE_NUM))) { + LOG_WARN("fail to prepare allocate", KR(ret)); + } else if (FALSE_IT(dirs_.set_attr(ObMemAttr(MTL_ID(), "TFFlushIterDir")))) { + } else if (OB_FAIL(dirs_.prepare_allocate(MAX_CACHE_NUM))) { + LOG_WARN("fail to prepare allocate", KR(ret)); + } else { + is_inited_ = true; + prio_mgr_ = prio_mgr; + } + return ret; +} + +int ObTmpFileFlushListIterator::clear() +{ + int ret = OB_SUCCESS; + + // reinsert unused cached file into flush list + FlushCtxState cur_stage = cal_current_flush_stage_(); + if (cur_stage < FlushCtxState::FSM_F1 || cur_stage >= FlushCtxState::FSM_FINISHED) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("unexpected flush stage", KR(ret), K(cur_stage)); + } else if (0 == cached_file_num_) { + // no need to reinsert files, do nothing + } else if (FlushCtxState::FSM_F1 == cur_stage) { + if (OB_FAIL(reinsert_files_into_flush_list_(cur_iter_file_idx_, cached_file_num_ - 1))){ + LOG_ERROR("fail to reinsert files into flush list", KR(ret), K(cur_iter_file_idx_), K(cached_file_num_)); + } + } else { + for (int64_t i = cur_iter_dir_idx_; OB_SUCC(ret) && (i >= 0 && i < cached_dir_num_); i++) { + if (OB_UNLIKELY(!dirs_[i].is_inited_)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("uninitialized dir is unexpected", KR(ret), K(dirs_[i])); + } else { + int64_t start_file_idx = dirs_[i].start_file_idx_; + int64_t end_file_idx = dirs_[i].end_file_idx_; + if (i == cur_iter_dir_idx_) { + start_file_idx = cur_iter_file_idx_; + if (start_file_idx < dirs_[i].start_file_idx_ || start_file_idx > dirs_[i].end_file_idx_) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("start file idx not in cur dir range", KR(ret), K(start_file_idx), K(dirs_[i])); + } + } + if (FAILEDx(reinsert_files_into_flush_list_(start_file_idx, end_file_idx))) { + LOG_ERROR("fail to reinsert files into flush list", + KR(ret), K(i), K(dirs_[i]), K(start_file_idx), K(end_file_idx)); + } + } + } + } + + if (OB_SUCC(ret)) { + cur_caching_list_is_meta_ = false; + cur_caching_list_idx_ = ObTmpFileFlushPriorityManager::FileList::L1; + cur_iter_dir_idx_ = -1; + cur_iter_file_idx_ = -1; + cached_file_num_ = 0; + cached_dir_num_ = 0; + for (int64_t i = 0; i < files_.count(); ++i) { + files_[i].reset(); + } + for (int64_t i = 0; i < dirs_.count(); ++i) { + dirs_[i].reset(); + } + } + + return ret; +} + +int ObTmpFileFlushListIterator::reset() +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + if (OB_FAIL(clear())) { + LOG_WARN("fail to clear", KR(ret)); + } + is_inited_ = false; + } + return ret; +} + +void ObTmpFileFlushListIterator::destroy() +{ + reset(); +} + +int ObTmpFileFlushListIterator::reinsert_files_into_flush_list_(const int64_t start_file_idx, + const int64_t end_file_idx) +{ + int ret = OB_SUCCESS; + for (int64_t i = start_file_idx; OB_SUCC(ret) && (i >= 0 && i <= end_file_idx); i++) { + if (OB_UNLIKELY(i < 0 || i >= cached_file_num_ || cached_file_num_ > files_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid file idx", KR(ret), K(i), K(cached_file_num_), K(files_)); + } else if (OB_UNLIKELY(!files_[i].is_inited_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("uninitialized file is unexpected", KR(ret), K(i), K(files_[i])); + } else { + ObTmpFileHandle &file_handle = files_[i].file_handle_; + if (OB_ISNULL(file_handle.get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", KR(ret)); + } else if (files_[i].is_meta_ && file_handle.get()->is_in_meta_flush_list()) { + // do nothing, because meta flush node may be re-inserted after + // tmp file insert meta tree item; do not handle data flush node here + // because data node will not be re-inserted during flushing procedure + } else if (OB_FAIL(file_handle.get()->reinsert_flush_node(files_[i].is_meta_))) { + LOG_WARN("fail to reinsert flush node", KR(ret), K(files_[i])); + } + } + } + return ret; +} + +int ObTmpFileFlushListIterator::next(const FlushCtxState iter_stage, bool &is_meta, ObTmpFileHandle &file_handle) +{ + int ret = OB_SUCCESS; + FlushCtxState cur_stage = FlushCtxState::FSM_FINISHED; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (OB_UNLIKELY(dirs_.count() != MAX_CACHE_NUM || + cached_dir_num_ > MAX_CACHE_NUM || + cached_dir_num_ < 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected status", KR(ret), K(files_.count()), K(cached_dir_num_)); + } else if (OB_UNLIKELY(files_.count() != MAX_CACHE_NUM || + cached_file_num_ > MAX_CACHE_NUM || + cached_file_num_ < 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected status", KR(ret), K(files_.count()), K(cached_file_num_)); + } else if (OB_UNLIKELY(FlushCtxState::FSM_FINISHED <= iter_stage || iter_stage < FlushCtxState::FSM_F1)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid iter_stage", KR(ret), K(iter_stage)); + } else if (FALSE_IT(cur_stage = cal_current_flush_stage_())) { + } else if (OB_UNLIKELY(FlushCtxState::FSM_FINISHED <= cur_stage || FlushCtxState::FSM_F1 > cur_stage)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected status", KR(ret), K(cur_stage)); + } else if (OB_UNLIKELY(cur_stage > iter_stage)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid iter_stage", KR(ret), K(iter_stage), K(cur_stage)); + } else if (cur_stage < iter_stage || + (FlushCtxState::FSM_F1 == cur_stage && cur_iter_file_idx_ == cached_file_num_) || + (FlushCtxState::FSM_F1 < cur_stage && cur_iter_dir_idx_ == cached_dir_num_)) { + // code will run here when: + // 1. expected iterating flush stage is over than current flush stage; + // 2. all cached file has been iterated + if (OB_FAIL(clear())) { + LOG_WARN("fail to clear", KR(ret)); + } + } + + if (OB_FAIL(ret)) { + } else if (0 == cached_file_num_ && OB_FAIL(cache_files_(iter_stage))) { + if (OB_ITER_END == ret) { + LOG_DEBUG("fail to cache files", KR(ret)); + } else { + LOG_WARN("fail to cache files", KR(ret)); + } + } else if (OB_FAIL(check_cur_idx_status_())) { + LOG_WARN("fail to check cur idx status", KR(ret)); + } else { + is_meta = files_[cur_iter_file_idx_].is_meta_; + file_handle = files_[cur_iter_file_idx_].file_handle_; + if (ObTmpFileFlushPriorityManager::FileList::L1 == cur_caching_list_idx_) { + if (OB_FAIL(advance_big_file_idx_())) { + LOG_WARN("fail to advance big file idx", KR(ret)); + } + } else if (OB_FAIL(advance_small_file_idx_())) { + LOG_WARN("fail to advance small file idx", KR(ret)); + } + } + LOG_DEBUG("try to get next file", KR(ret), K(iter_stage), K(cur_stage), K(is_meta), + K(cur_iter_file_idx_), K(cached_file_num_), K(file_handle)); + return ret; +} + +int ObTmpFileFlushListIterator::cache_files_(const FlushCtxState iter_stage) +{ + int ret = OB_SUCCESS; + ObTmpFileFlushPriorityManager::FileList end_list_idx = ObTmpFileFlushPriorityManager::FileList::MAX; + const int64_t target_cache_file_num = iter_stage == FlushCtxState::FSM_F1 ? + BIG_FILE_CACHE_NUM : MAX_CACHE_NUM; + ObArray file_handles; + + if (OB_UNLIKELY(0 != cached_dir_num_ || 0 != cached_file_num_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid cached num", KR(ret), K(cached_dir_num_), K(cached_file_num_)); + } else if (OB_FAIL(init_caching_list_with_flush_stage_(iter_stage))) { + LOG_WARN("fail to init caching list", KR(ret), K(iter_stage)); + } else if (OB_FAIL(acquire_final_list_of_flush_stage_(iter_stage, end_list_idx))) { + LOG_WARN("fail to acquire final list of flush stage", KR(ret), K(iter_stage)); + } else if (OB_FAIL(file_handles.prepare_allocate_and_keep_count(target_cache_file_num))) { + LOG_WARN("fail to prepare allocate", KR(ret), K(target_cache_file_num)); + } else { // pop enough files from priority manager for caching + int64_t remain_cache_file_num = target_cache_file_num; + bool cache_over = false; + while (OB_SUCC(ret) && !cache_over && remain_cache_file_num > 0) { + int64_t actual_cache_file_num = 0; + if (OB_FAIL(prio_mgr_->popN_from_file_list(cur_caching_list_is_meta_, cur_caching_list_idx_, + remain_cache_file_num, actual_cache_file_num, + file_handles))) { + LOG_WARN("fail to pop N from file list", KR(ret), K(cur_caching_list_idx_)); + } else if (FALSE_IT(remain_cache_file_num -= actual_cache_file_num)) { + } else if (OB_UNLIKELY(remain_cache_file_num < 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected cache file num", KR(ret), K(target_cache_file_num), + K(remain_cache_file_num), K(actual_cache_file_num)); + } else if (0 == remain_cache_file_num) { // cache enough files + cache_over = true; + } else { // remain_cache_file_num > 0 + if (cur_caching_list_idx_ == end_list_idx) { // has reached the last list of this flush stage + if (file_handles.empty()) { + ret = OB_ITER_END; + LOG_DEBUG("iter end in current flush stage", KR(ret), K(iter_stage), + K(cur_caching_list_idx_), K(remain_cache_file_num)); + } else { // cache files successful, but the num is not enough + cache_over = true; + } + } else if (OB_FAIL(advance_caching_list_idx_())) { // pop files of the next list of this flush stage + LOG_WARN("fail to advance caching list idx", KR(ret)); + } + } + } // end while + } + + if (OB_SUCC(ret)) { + if (ObTmpFileFlushPriorityManager::FileList::L1 == cur_caching_list_idx_) { + if (OB_FAIL(cache_big_files_(file_handles))) { + LOG_WARN("fail to cache big files", KR(ret)); + } + } else if (OB_FAIL(cache_small_files_(file_handles))) { + LOG_WARN("fail to cache big files", KR(ret)); + } + } + + if (OB_FAIL(ret)) { + int tmp_ret = OB_SUCCESS; + for (int64_t i = 0; OB_LIKELY(OB_SUCCESS == tmp_ret) && i < file_handles.count(); ++i) { + if (OB_ISNULL(file_handles[i].get())) { + // could not happen, just skip + } else if (OB_TMP_FAIL(file_handles[i].get()->reinsert_flush_node(cur_caching_list_is_meta_))) { + LOG_WARN("fail to reinsert flush node", KR(tmp_ret), K(i), K(file_handles[i])); + } + } + } + return ret; +} + +int ObTmpFileFlushListIterator::cache_big_files_(const ObArray &file_handles) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(build_file_wrappers_(file_handles))) { + LOG_WARN("fail to build file wrappers", KR(ret)); + } else { + cur_iter_file_idx_ = 0; + cur_iter_dir_idx_ = -1; // no need to aggregate files into dir + cached_dir_num_ = 0; + } + return ret; +} + +int ObTmpFileFlushListIterator::cache_small_files_(const ObArray &file_handles) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(build_file_wrappers_(file_handles))) { + LOG_WARN("fail to build file wrappers", KR(ret)); + } else if (OB_FAIL(build_dir_wrappers_())) { + LOG_WARN("fail to build dir wrappers", KR(ret)); + } else { + cur_iter_file_idx_ = dirs_[0].start_file_idx_; + cur_iter_dir_idx_ = 0; + } + return ret; +} + +int ObTmpFileFlushListIterator::build_file_wrappers_(const ObArray &file_handles) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(file_handles.empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(file_handles.empty())); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < file_handles.count(); ++i) { + if (i >= files_.count()) { + ret = OB_ERROR_OUT_OF_RANGE; + LOG_ERROR("index is out of range", KR(ret), K(i), K(files_)); + } else if (OB_FAIL(files_[i].init(cur_caching_list_is_meta_, file_handles[i]))) { + LOG_WARN("fail to init tmp file handle wrapper", KR(ret), K(i), K(file_handles[i]), K(files_[i])); + + for (int64_t j = 0; j < i; ++j) { + files_[j].reset(); + } + } + } + + if (OB_SUCC(ret)) { + lib::ob_sort(files_.begin(), files_.begin() + file_handles.count()); + cached_file_num_ = file_handles.count(); + } + } + return ret; +} + +int ObTmpFileFlushListIterator::build_dir_wrappers_() +{ + int ret = OB_SUCCESS; + int cached_dir_num = 0; + + if (OB_UNLIKELY(cached_file_num_ <= 0 || cached_file_num_ > files_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid cached file num", KR(ret), K(cached_file_num_)); + } else { // we assume that files_ has been initialized and sorted + int64_t dir_id = ObTmpFileGlobal::INVALID_TMP_FILE_DIR_ID; + int64_t start_file_idx = -1; + int64_t end_file_idx = -1; + int64_t page_num = 0; + for (int64_t i = 0; OB_SUCC(ret) && i < cached_file_num_; ++i) { + ObSharedNothingTmpFile *file = nullptr; + int64_t file_dirty_page_num = 0; + if (OB_UNLIKELY(!files_[i].is_inited_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tmp file wrapper is not inited", KR(ret), K(files_[i])); + } else if (OB_ISNULL(file = files_[i].file_handle_.get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("file ptr is null", K(ret), KP(file)); + } else if (OB_FAIL(get_flushing_file_dirty_page_num_(*file, file_dirty_page_num))) { + LOG_WARN("fail to get flushing file dirty page num", KR(ret)); + } else if (file->get_dir_id() != dir_id) { + if (0 != i) { + end_file_idx = i - 1; + if (cached_dir_num >= dirs_.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("cached_dir_num is equal or bigger than dirs_.count", + KR(ret), K(cached_dir_num), K(dirs_)); + } else if (OB_FAIL(dirs_[cached_dir_num].init(cur_caching_list_is_meta_, page_num, + start_file_idx, end_file_idx))) { + LOG_WARN("fail to init tmp file dir wrapper", KR(ret), K(i), K(dirs_[cached_dir_num]), K(page_num), + K(start_file_idx), K(end_file_idx)); + } else { + cached_dir_num++; + } + } + + if (OB_SUCC(ret)) { + dir_id = file->get_dir_id(); + start_file_idx = i; + page_num = file_dirty_page_num; + } + } else { + page_num += file_dirty_page_num; + } + + if (OB_SUCC(ret) && cached_file_num_ - 1 == i) { + end_file_idx = i; + if (cached_dir_num >= dirs_.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("cached_dir_num is equal or bigger than dirs_.count", + KR(ret), K(cached_dir_num), K(dirs_)); + } else if (OB_FAIL(dirs_[cached_dir_num].init(cur_caching_list_is_meta_, page_num, + start_file_idx, end_file_idx))) { + LOG_WARN("fail to init tmp file dir wrapper", KR(ret), K(page_num), + K(start_file_idx), K(end_file_idx)); + } else { + cached_dir_num++; + } + } + } // end for + + } + + if (OB_FAIL(ret)) { + for (int64_t i = 0; i < cached_dir_num; i++) { + dirs_[i].reset(); + } + for (int64_t i = 0; i < cached_file_num_; i++) { + files_[i].reset(); + cached_file_num_ = 0; + } + } else { + lib::ob_sort(dirs_.begin(), dirs_.begin() + cached_dir_num); + cached_dir_num_ = cached_dir_num; + } + + return ret; +} + +int ObTmpFileFlushListIterator::get_flushing_file_dirty_page_num_(const ObSharedNothingTmpFile &file, int64_t &page_num) +{ + int ret = OB_SUCCESS; + FlushCtxState cur_stage = cal_current_flush_stage_(); + if (OB_UNLIKELY(cur_stage < FlushCtxState::FSM_F1 || cur_stage > FlushCtxState::FSM_F5)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected flush stage", KR(ret), K(cur_stage)); + } else if (cur_stage <= FlushCtxState::FSM_F3) { + ObSharedNothingTmpFile &mutable_file_ref = const_cast(file); + int64_t dirty_page_size = mutable_file_ref.get_dirty_data_page_size_with_lock(); + page_num = upper_align(dirty_page_size, ObTmpFileGlobal::PAGE_SIZE) / ObTmpFileGlobal::PAGE_SIZE; + } else { + int64_t non_rightmost_dirty_page_num = 0; + int64_t rightmost_dirty_page_num = 0; + ObSharedNothingTmpFile &mutable_file_ref = const_cast(file); + mutable_file_ref.get_dirty_meta_page_num_with_lock(non_rightmost_dirty_page_num, rightmost_dirty_page_num); + if (cur_stage == FlushCtxState::FSM_F4) { + page_num = non_rightmost_dirty_page_num; + } else if (cur_stage == FlushCtxState::FSM_F5) { + page_num = rightmost_dirty_page_num; + } + } + return ret; +} + +int ObTmpFileFlushListIterator::advance_caching_list_idx_() +{ + int ret = OB_SUCCESS; + if (!cur_caching_list_is_meta_ && ObTmpFileFlushPriorityManager::FileList::L5 == cur_caching_list_idx_) { + cur_caching_list_is_meta_ = true; + cur_caching_list_idx_ = ObTmpFileFlushPriorityManager::FileList::L1; + } else { + switch(cur_caching_list_idx_) { + case ObTmpFileFlushPriorityManager::FileList::L1: + cur_caching_list_idx_ = ObTmpFileFlushPriorityManager::FileList::L2; + break; + case ObTmpFileFlushPriorityManager::FileList::L2: + cur_caching_list_idx_ = ObTmpFileFlushPriorityManager::FileList::L3; + break; + case ObTmpFileFlushPriorityManager::FileList::L3: + cur_caching_list_idx_ = ObTmpFileFlushPriorityManager::FileList::L4; + break; + case ObTmpFileFlushPriorityManager::FileList::L4: + cur_caching_list_idx_ = ObTmpFileFlushPriorityManager::FileList::L5; + break; + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected status", KR(ret), K(cur_caching_list_idx_)); + break; + } + } + + return ret; +} + +int ObTmpFileFlushListIterator::check_cur_idx_status_() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(cur_iter_file_idx_ < 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected status", KR(ret), K(cur_iter_file_idx_)); + } else if (ObTmpFileFlushPriorityManager::FileList::L1 == cur_caching_list_idx_) { + // the file in L1 list will not be flushed with an aggregating dir. + // thus, it is no need to check dir + if (OB_UNLIKELY(cur_iter_file_idx_ >= cached_file_num_ || cached_file_num_ > files_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected status", KR(ret), K(cur_iter_file_idx_), K(cached_file_num_)); + } + } else if (OB_UNLIKELY(cur_iter_dir_idx_ < 0 || cur_iter_dir_idx_ >= cached_dir_num_ || + cached_dir_num_ > dirs_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected status", KR(ret), K(cur_iter_dir_idx_), K(cached_dir_num_), K(dirs_)); + } else if (OB_UNLIKELY(!dirs_[cur_iter_dir_idx_].is_inited_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected status", KR(ret), K(cur_iter_dir_idx_), K(dirs_[cur_iter_dir_idx_])); + } else if (OB_UNLIKELY(cur_iter_file_idx_ < dirs_[cur_iter_dir_idx_].start_file_idx_ || + cur_iter_file_idx_ > dirs_[cur_iter_dir_idx_].end_file_idx_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected status", KR(ret), K(cur_iter_dir_idx_), K(cur_iter_file_idx_), + K(dirs_[cur_iter_dir_idx_])); + } + + if (OB_SUCC(ret) && OB_UNLIKELY(!files_[cur_iter_file_idx_].is_inited_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected status", KR(ret), K(cur_iter_file_idx_), K(files_[cur_iter_file_idx_])); + } + + return ret; +} + +int ObTmpFileFlushListIterator::advance_big_file_idx_() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(ObTmpFileFlushPriorityManager::FileList::L1 != cur_caching_list_idx_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid caching list idx", KR(ret), K(cur_caching_list_idx_)); + } else if (OB_UNLIKELY(cur_iter_file_idx_ < 0 || cur_iter_file_idx_ >= cached_file_num_ || + cached_file_num_ > files_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid file idx", KR(ret), K(cur_iter_file_idx_), K(cached_file_num_), K(files_)); + } else { + files_[cur_iter_file_idx_].reset(); + cur_iter_file_idx_++; + } + return ret; +} + +int ObTmpFileFlushListIterator::advance_small_file_idx_() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(cur_caching_list_idx_ <= ObTmpFileFlushPriorityManager::FileList::L1 || + cur_caching_list_idx_ > ObTmpFileFlushPriorityManager::FileList::L5)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid caching list idx", KR(ret), K(cur_caching_list_idx_)); + } else if (OB_UNLIKELY(cur_iter_dir_idx_ < 0 || cur_iter_dir_idx_ >= cached_dir_num_ || + cached_dir_num_ > dirs_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid dir idx", KR(ret), K(cur_iter_dir_idx_), K(cached_dir_num_), K(dirs_)); + } else if (OB_UNLIKELY(cur_iter_file_idx_ < 0 || cur_iter_file_idx_ >= cached_file_num_ || + cached_file_num_ > files_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected status", KR(ret), K(cur_iter_file_idx_), K(cached_file_num_)); + } else if (OB_UNLIKELY(cur_iter_file_idx_ < dirs_[cur_iter_dir_idx_].start_file_idx_ || + cur_iter_file_idx_ > dirs_[cur_iter_dir_idx_].end_file_idx_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid file idx", KR(ret), K(cur_iter_file_idx_), K(dirs_[cur_iter_dir_idx_])); + } else { + files_[cur_iter_file_idx_].reset(); + cur_iter_file_idx_++; + if (cur_iter_file_idx_ > dirs_[cur_iter_dir_idx_].end_file_idx_) { + if (OB_FAIL(advance_dir_idx_())) { + LOG_WARN("fail to advance dir idx", KR(ret)); + } else if (cur_iter_dir_idx_ < cached_dir_num_) { + if (OB_UNLIKELY(!dirs_[cur_iter_dir_idx_].is_inited_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected status", KR(ret), K(cur_iter_dir_idx_), K(dirs_[cur_iter_dir_idx_])); + } else { + cur_iter_file_idx_ = dirs_[cur_iter_dir_idx_].start_file_idx_; + } + } else { + // iter end + } + } + } + return ret; +} + +int ObTmpFileFlushListIterator::advance_dir_idx_() +{ + int ret = OB_SUCCESS; + + if (OB_UNLIKELY(ObTmpFileFlushPriorityManager::FileList::L1 == cur_caching_list_idx_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid caching list idx", KR(ret), K(cur_caching_list_idx_)); + } else if (OB_UNLIKELY(cur_iter_dir_idx_ < 0 || cur_iter_dir_idx_ >= cached_dir_num_ || + cached_dir_num_ > dirs_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid dir idx", KR(ret), K(cur_iter_dir_idx_), K(cached_dir_num_), K(dirs_)); + } else { + dirs_[cur_iter_dir_idx_].reset(); + cur_iter_dir_idx_++; + } + + return ret; +} + +FlushCtxState ObTmpFileFlushListIterator::cal_current_flush_stage_() +{ + FlushCtxState stage = FlushCtxState::FSM_FINISHED; + if (OB_UNLIKELY(cur_caching_list_idx_ < ObTmpFileFlushPriorityManager::FileList::L1 || + cur_caching_list_idx_ > ObTmpFileFlushPriorityManager::FileList::L5)) { + // stage = FlushCtxState::FSM_FINISHED; + } else if (!cur_caching_list_is_meta_) { + if (cur_caching_list_idx_ == ObTmpFileFlushPriorityManager::FileList::L1) { + stage = FlushCtxState::FSM_F1; + } else if (cur_caching_list_idx_ <= ObTmpFileFlushPriorityManager::FileList::L4) { + stage = FlushCtxState::FSM_F2; + } else if (cur_caching_list_idx_ == ObTmpFileFlushPriorityManager::FileList::L5) { + stage = FlushCtxState::FSM_F3; + } + } else { + if (cur_caching_list_idx_ <= ObTmpFileFlushPriorityManager::FileList::L4) { + stage = FlushCtxState::FSM_F4; + } else if (cur_caching_list_idx_ == ObTmpFileFlushPriorityManager::FileList::L5) { + stage = FlushCtxState::FSM_F5; + } + } + + return stage; +} + +int ObTmpFileFlushListIterator::init_caching_list_with_flush_stage_(const FlushCtxState iter_stage) +{ + int ret = OB_SUCCESS; + if (cal_current_flush_stage_() == iter_stage) { + // no need to change caching list + } else if (iter_stage == FlushCtxState::FSM_F1) { + cur_caching_list_is_meta_ = false; + cur_caching_list_idx_ = ObTmpFileFlushPriorityManager::FileList::L1; + } else if (iter_stage == FlushCtxState::FSM_F2) { + cur_caching_list_is_meta_ = false; + cur_caching_list_idx_ = ObTmpFileFlushPriorityManager::FileList::L2; + } else if (iter_stage == FlushCtxState::FSM_F3) { + cur_caching_list_is_meta_ = false; + cur_caching_list_idx_ = ObTmpFileFlushPriorityManager::FileList::L5; + } else if (iter_stage == FlushCtxState::FSM_F4) { + cur_caching_list_is_meta_ = true; + cur_caching_list_idx_ = ObTmpFileFlushPriorityManager::FileList::L1; + } else if (iter_stage == FlushCtxState::FSM_F5) { + cur_caching_list_is_meta_ = true; + cur_caching_list_idx_ = ObTmpFileFlushPriorityManager::FileList::L5; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid stage", KR(ret), K(iter_stage)); + } + + return ret; +} + +int ObTmpFileFlushListIterator::acquire_final_list_of_flush_stage_(const FlushCtxState iter_stage, + ObTmpFileFlushPriorityManager::FileList &list_idx) +{ + int ret = OB_SUCCESS; + switch(iter_stage) { + case FlushCtxState::FSM_F1: + list_idx = ObTmpFileFlushPriorityManager::FileList::L1; + break; + case FlushCtxState::FSM_F2: + list_idx = ObTmpFileFlushPriorityManager::FileList::L4; + break; + case FlushCtxState::FSM_F3: + list_idx = ObTmpFileFlushPriorityManager::FileList::L5; + break; + case FlushCtxState::FSM_F4: + list_idx = ObTmpFileFlushPriorityManager::FileList::L4; + break; + case FlushCtxState::FSM_F5: + list_idx = ObTmpFileFlushPriorityManager::FileList::L5; + break; + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid stage", KR(ret), K(iter_stage)); + } + return ret; +} + +int ObTmpFileFlushListIterator::ObFlushingTmpFileWrapper::init(const bool is_meta, const ObTmpFileHandle &file_handle) +{ + int ret = OB_SUCCESS; + if (is_inited_) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", KR(ret)); + } else if (OB_ISNULL(file_handle.get())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), KP(file_handle.get())); + } else { + is_meta_ = is_meta; + file_handle_ = file_handle; + is_inited_ = true; + } + return ret; +} + +void ObTmpFileFlushListIterator::ObFlushingTmpFileWrapper::reset() +{ + if (IS_INIT) { + is_inited_ = false; + is_meta_ = false; + file_handle_.reset(); + } +} + +bool ObTmpFileFlushListIterator::ObFlushingTmpFileWrapper::operator <(const ObFlushingTmpFileWrapper &other) +{ + int ret = OB_SUCCESS; + bool b_ret = false; + if (OB_UNLIKELY(!other.is_inited_ || !is_inited_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected status", KR(ret), K(other), K(*this)); + } else if (OB_ISNULL(other.file_handle_.get()) || OB_ISNULL(file_handle_.get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("attempt to compare file handle with nullptr", KR(ret), K(file_handle_), K(other.file_handle_)); + } else if (!is_meta_ && other.is_meta_) { + b_ret = true; + } else if (is_meta_ && !other.is_meta_) { + b_ret = false; + } else if (file_handle_.get()->get_dir_id() < other.file_handle_.get()->get_dir_id()) { + b_ret = true; + } else if (file_handle_.get()->get_dir_id() > other.file_handle_.get()->get_dir_id()) { + b_ret = false; + } else if (is_meta_) { + b_ret = file_handle_.get()->get_meta_page_flush_level() < other.file_handle_.get()->get_meta_page_flush_level(); + } else { + b_ret = file_handle_.get()->get_data_page_flush_level() < other.file_handle_.get()->get_data_page_flush_level(); + } + return b_ret; +} + +int ObTmpFileFlushListIterator::ObFlushingTmpFileDirWrapper::init(const bool is_meta, const int64_t page_num, + const int64_t start_file_idx, + const int64_t end_file_idx) +{ + int ret = OB_SUCCESS; + if (is_inited_) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", KR(ret)); + } else if (OB_UNLIKELY(page_num < 0 || start_file_idx < 0 || start_file_idx > end_file_idx)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(page_num), K(start_file_idx), K(end_file_idx)); + } else { + is_inited_ = true; + is_meta_ = is_meta; + page_num_ = page_num; + start_file_idx_ = start_file_idx; + end_file_idx_ = end_file_idx; + } + return ret; +} + +void ObTmpFileFlushListIterator::ObFlushingTmpFileDirWrapper::reset() +{ + if (IS_INIT) { + is_meta_ = false; + page_num_ = 0; + start_file_idx_ = -1; + end_file_idx_ = -1; + is_inited_ = false; + } +} + +bool ObTmpFileFlushListIterator::ObFlushingTmpFileDirWrapper::operator <(const ObFlushingTmpFileDirWrapper &other) +{ + int ret = OB_SUCCESS; + bool b_ret = false; + if (OB_UNLIKELY(!other.is_inited_ || !is_inited_)) { + LOG_WARN("unexpected status", K(other), K(*this)); + } else if (!is_meta_ && other.is_meta_) { + b_ret = true; + } else if (is_meta_ && !other.is_meta_) { + b_ret = false; + } else { + b_ret = page_num_ > other.page_num_; + } + return b_ret; +} +} // end namespace tmp_file +} // end namespace oceanbase diff --git a/src/storage/tmp_file/ob_tmp_file_flush_list_iterator.h b/src/storage/tmp_file/ob_tmp_file_flush_list_iterator.h new file mode 100644 index 000000000..28b017d9d --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_flush_list_iterator.h @@ -0,0 +1,105 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_FLUSH_LIST_ITERATOR_H_ +#define OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_FLUSH_LIST_ITERATOR_H_ + +#include "storage/tmp_file/ob_tmp_file_flush_priority_manager.h" +#include "storage/tmp_file/ob_tmp_file_global.h" + +namespace oceanbase +{ +namespace tmp_file +{ + +typedef ObTmpFileGlobal::FlushCtxState FlushCtxState; + +class ObTmpFileFlushListIterator +{ +public: + ObTmpFileFlushListIterator(); + ~ObTmpFileFlushListIterator(); + + int init(ObTmpFileFlushPriorityManager *prio_mgr); + int clear(); + int reset(); + void destroy(); + int next(const FlushCtxState iter_stage, bool &is_meta, ObTmpFileHandle &file_handle); + + TO_STRING_KV(K(is_inited_), K(cur_caching_list_idx_), K(cur_caching_list_is_meta_), + K(cur_iter_dir_idx_), K(cur_iter_file_idx_), K(cached_file_num_), K(cached_dir_num_)) +private: + int reinsert_files_into_flush_list_(const int64_t start_file_idx, const int64_t end_file_idx); + FlushCtxState cal_current_flush_stage_(); + int init_caching_list_with_flush_stage_(const FlushCtxState iter_stage); + int acquire_final_list_of_flush_stage_(const FlushCtxState iter_stage, + ObTmpFileFlushPriorityManager::FileList &list_idx); + int cache_files_(const FlushCtxState iter_stage); + int build_file_wrappers_(const ObArray &file_handles); + int build_dir_wrappers_(); + int cache_big_files_(const ObArray &file_handles); + int cache_small_files_(const ObArray &file_handles); + int get_flushing_file_dirty_page_num_(const ObSharedNothingTmpFile &file, int64_t &page_num); + int check_cur_idx_status_(); + int advance_big_file_idx_(); + int advance_small_file_idx_(); + int advance_dir_idx_(); + int advance_caching_list_idx_(); +private: + struct ObFlushingTmpFileWrapper + { + ObFlushingTmpFileWrapper() : is_inited_(false), is_meta_(false), file_handle_() {} + ~ObFlushingTmpFileWrapper() { reset(); }; + int init(const bool is_meta, const ObTmpFileHandle &file_handle); + void reset(); + bool operator <(const ObFlushingTmpFileWrapper &other); + TO_STRING_KV(K(is_inited_), K(is_meta_), K(file_handle_)); + + bool is_inited_; + bool is_meta_; + ObTmpFileHandle file_handle_; + }; + + struct ObFlushingTmpFileDirWrapper + { + ObFlushingTmpFileDirWrapper() : is_inited_(false), is_meta_(false), page_num_(0), + start_file_idx_(-1), end_file_idx_(-1) {} + ~ObFlushingTmpFileDirWrapper() { reset(); }; + int init(const bool is_meta, const int64_t page_num, const int64_t start_file_idx, const int64_t end_file_idx); + void reset(); + bool operator <(const ObFlushingTmpFileDirWrapper &other); + TO_STRING_KV(K(is_inited_), K(is_meta_), K(page_num_), K(start_file_idx_), K(end_file_idx_)); + + bool is_inited_; + bool is_meta_; + int64_t page_num_; + int64_t start_file_idx_; + int64_t end_file_idx_; + }; +private: + static constexpr int64_t MAX_CACHE_NUM = 256; + static constexpr int64_t BIG_FILE_CACHE_NUM = 8; + bool is_inited_; + ObTmpFileFlushPriorityManager *prio_mgr_; + ObArray files_; + ObArray dirs_; + bool cur_caching_list_is_meta_; + ObTmpFileFlushPriorityManager::FileList cur_caching_list_idx_; + int64_t cur_iter_dir_idx_; + int64_t cur_iter_file_idx_; + int64_t cached_file_num_; + int64_t cached_dir_num_; +}; + +} // end namespace tmp_file +} // end namespace oceanbase +#endif // OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_FLUSH_LIST_ITERATOR_H_ diff --git a/src/storage/tmp_file/ob_tmp_file_flush_manager.cpp b/src/storage/tmp_file/ob_tmp_file_flush_manager.cpp new file mode 100644 index 000000000..d98eecf5f --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_flush_manager.cpp @@ -0,0 +1,969 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE + +#include "storage/tmp_file/ob_tmp_file_flush_manager.h" +#include "storage/tmp_file/ob_tmp_file_manager.h" +#include "storage/blocksstable/ob_block_manager.h" +#include "storage/tmp_file/ob_tmp_file_flush_list_iterator.h" + +namespace oceanbase +{ +namespace tmp_file +{ + +ObTmpFileFlushManager::ObTmpFileFlushManager(ObTmpFilePageCacheController &pc_ctrl) + : is_inited_(false), + flush_ctx_(), + pc_ctrl_(pc_ctrl), + tmp_file_block_mgr_(pc_ctrl.get_tmp_file_block_manager()), + task_allocator_(pc_ctrl.get_task_allocator()), + write_buffer_pool_(pc_ctrl.get_write_buffer_pool()), + evict_mgr_(pc_ctrl.get_eviction_manager()), + flush_priority_mgr_(pc_ctrl.get_flush_priority_mgr()) +{ +} + +int ObTmpFileFlushManager::init() +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + STORAGE_LOG(WARN, "ObTmpFileFlushManager inited twice", KR(ret)); + } else if (OB_FAIL(flush_ctx_.init())) { + STORAGE_LOG(WARN, "fail to init flush ctx", KR(ret)); + } else { + is_inited_ = true; + } + return ret; +} + +void ObTmpFileFlushManager::destroy() +{ + is_inited_ = false; + flush_ctx_.destroy(); +} + +int ObTmpFileFlushManager::alloc_flush_task(ObTmpFileFlushTask *&flush_task) +{ + int ret = OB_SUCCESS; + flush_task = nullptr; + + const int64_t BLOCK_SIZE = OB_SERVER_BLOCK_MGR.get_macro_block_size(); + void *task_buf = nullptr; + if (OB_ISNULL(task_buf = task_allocator_.alloc(sizeof(ObTmpFileFlushTask)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + STORAGE_LOG(WARN, "fail to allocate memory for flush callback", KR(ret)); + } else { + flush_task = new (task_buf) ObTmpFileFlushTask(task_allocator_); + } + return ret; +} + +int ObTmpFileFlushManager::free_flush_task(ObTmpFileFlushTask *flush_task) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(flush_task)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "flush task ptr is null", KR(ret)); + } else { + LOG_DEBUG("free flush task", KPC(flush_task)); + flush_task->~ObTmpFileFlushTask(); + task_allocator_.free(flush_task); + } + return ret; +} + +int ObTmpFileFlushManager::notify_write_back_failed(ObTmpFileFlushTask *flush_task) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(flush_task)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "flush task ptr is null", KR(ret)); + } else if (OB_FAIL(tmp_file_block_mgr_.write_back_failed(flush_task->get_block_index()))) { + STORAGE_LOG(ERROR, "fail to notify tmp file block write back failed", KR(ret), KPC(flush_task)); + } + return ret; +} + +// release whole block +int ObTmpFileFlushManager::free_tmp_file_block(ObTmpFileFlushTask &flush_task) +{ + int ret = OB_SUCCESS; + int64_t block_index = flush_task.get_block_index(); + if (ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX == block_index) { + // do nothing + } else if (OB_FAIL(tmp_file_block_mgr_.release_tmp_file_page(block_index, + 0/*begin_page_id*/, + ObTmpFileGlobal::BLOCK_PAGE_NUMS))) { + STORAGE_LOG(WARN, "fail to remove tmp file block", KR(ret), K(block_index), K(flush_task)); + } + return ret; +} + +void ObTmpFileFlushManager::init_flush_level_() +{ + int64_t dirty_page_percentage = write_buffer_pool_.get_dirty_page_percentage(); + if (pc_ctrl_.is_flush_all_data()) { // only for unittest + flush_ctx_.set_state(FlushCtxState::FSM_F1); + } else { + if (FLUSH_WATERMARK_F1 <= dirty_page_percentage) { + flush_ctx_.set_state(FlushCtxState::FSM_F1); + } else if (FLUSH_WATERMARK_F2 <= dirty_page_percentage) { + flush_ctx_.set_state(FlushCtxState::FSM_F2); + } else if (FLUSH_WATERMARK_F3 <= dirty_page_percentage) { + flush_ctx_.set_state(FlushCtxState::FSM_F3); + } else if (FLUSH_WATERMARK_F4 <= dirty_page_percentage) { + flush_ctx_.set_state(FlushCtxState::FSM_F4); + } else if (FLUSH_WATERMARK_F5 <= dirty_page_percentage) { + flush_ctx_.set_state(FlushCtxState::FSM_F5); + } else { + flush_ctx_.set_state(FlushCtxState::FSM_FINISHED); + } + } +} + +void ObTmpFileFlushManager::advance_flush_level_(const int ret_code) +{ + int64_t dirty_page_percentage = write_buffer_pool_.get_dirty_page_percentage(); + + if (pc_ctrl_.is_flush_all_data()) { // only for unittest + if (OB_SUCCESS == ret_code) { + // continue to flush to OB_ITER_END, do nothing + } else if (OB_ITER_END == ret_code) { + inner_advance_flush_level_without_checking_watermark_(); + } else { + flush_ctx_.set_state(FlushCtxState::FSM_FINISHED); + } + } else { + if (OB_SUCCESS == ret_code) { + // if lower than low_watermark or reach expect_flush_size, terminate flush process + if (get_low_watermark_(flush_ctx_.get_state()) >= dirty_page_percentage + || flush_ctx_.get_actual_flush_size() >= flush_ctx_.get_expect_flush_size()) { + flush_ctx_.set_state(FlushCtxState::FSM_FINISHED); + } + } else if (OB_ITER_END == ret_code) { + inner_advance_flush_level_(); + } else { + flush_ctx_.set_state(FlushCtxState::FSM_FINISHED); + } + } + + LOG_DEBUG("advance_flush_level_", K(flush_ctx_)); +} + +int64_t ObTmpFileFlushManager::get_low_watermark_(FlushCtxState state) +{ + int ret = OB_SUCCESS; + int64_t low_watermark = 100; + switch(state) { + case FlushCtxState::FSM_F1: + low_watermark = FLUSH_LOW_WATERMARK_F1; + break; + case FlushCtxState::FSM_F2: + low_watermark = FLUSH_LOW_WATERMARK_F2; + break; + case FlushCtxState::FSM_F3: + low_watermark = FLUSH_LOW_WATERMARK_F3; + break; + case FlushCtxState::FSM_F4: + low_watermark = FLUSH_LOW_WATERMARK_F4; + break; + case FlushCtxState::FSM_F5: + low_watermark = FLUSH_LOW_WATERMARK_F5; + break; + case FlushCtxState::FSM_FINISHED: + break; + default: + STORAGE_LOG(WARN, "unexpected flush state", K(state), K(flush_ctx_)); + break; + } + return low_watermark; +} + +void ObTmpFileFlushManager::inner_advance_flush_level_() +{ + if (flush_ctx_.get_actual_flush_size() >= flush_ctx_.get_expect_flush_size()) { + flush_ctx_.set_state(FlushCtxState::FSM_FINISHED); + } + + int64_t dirty_page_percentage = write_buffer_pool_.get_dirty_page_percentage(); + if (FLUSH_WATERMARK_F2 <= dirty_page_percentage && flush_ctx_.get_state() < FlushCtxState::FSM_F2) { + flush_ctx_.set_state(FlushCtxState::FSM_F2); + } else if (FLUSH_WATERMARK_F3 <= dirty_page_percentage && flush_ctx_.get_state() < FlushCtxState::FSM_F3) { + flush_ctx_.set_state(FlushCtxState::FSM_F3); + } else if (FLUSH_WATERMARK_F4 <= dirty_page_percentage && flush_ctx_.get_state() < FlushCtxState::FSM_F4) { + flush_ctx_.set_state(FlushCtxState::FSM_F4); + } else if (FLUSH_WATERMARK_F5 <= dirty_page_percentage && flush_ctx_.get_state() < FlushCtxState::FSM_F5) { + flush_ctx_.set_state(FlushCtxState::FSM_F5); + } else { + flush_ctx_.set_state(FlushCtxState::FSM_FINISHED); + } +} + +// only for unittest +void ObTmpFileFlushManager::inner_advance_flush_level_without_checking_watermark_() +{ + switch(flush_ctx_.get_state()) { + case FlushCtxState::FSM_F1: + flush_ctx_.set_state(FlushCtxState::FSM_F2); + break; + case FlushCtxState::FSM_F2: + flush_ctx_.set_state(FlushCtxState::FSM_F3); + break; + case FlushCtxState::FSM_F3: + flush_ctx_.set_state(FlushCtxState::FSM_F4); + break; + case FlushCtxState::FSM_F4: + flush_ctx_.set_state(FlushCtxState::FSM_F5); + break; + case FlushCtxState::FSM_F5: + flush_ctx_.set_state(FlushCtxState::FSM_FINISHED); + break; + default: + break; + } +} + +// Generate a set of flush tasks based on the flushing level to flush dirty pages, +// and attempt to advance the task status to TFFT_WAIT (waiting for asynchronous IO). +// When OB_ALLOCATE_TMP_FILE_PAGE_FAILED error occurs during flushing, +// a special flush task is generated to flush meta pages and terminate the current round of flushing. +// Setting fast_flush_meta to true by the caller also triggers this process. +int ObTmpFileFlushManager::flush(ObSpLinkQueue &flushing_queue, + ObTmpFileFlushMonitor &flush_monitor, + const int64_t expect_flush_size, + const bool is_flush_meta_tree) +{ + int ret = OB_SUCCESS; + bool fast_flush_meta = is_flush_meta_tree; + + init_flush_level_(); + + if (FlushCtxState::FSM_FINISHED == flush_ctx_.get_state() && !fast_flush_meta) { + ret = OB_SUCCESS; + } else if (OB_FAIL(flush_ctx_.prepare_flush_ctx(expect_flush_size, &flush_priority_mgr_, &flush_monitor))) { + STORAGE_LOG(WARN, "fail to prepare flush iterator", KR(ret), K(flush_ctx_)); + } else { + while (OB_SUCC(ret) && !flush_ctx_.is_fail_too_many() + && (FlushCtxState::FSM_FINISHED != flush_ctx_.get_state() || fast_flush_meta)) { + ObTmpFileFlushTask *flush_task = nullptr; + if (OB_FAIL(handle_alloc_flush_task_(fast_flush_meta, flush_task))) { + STORAGE_LOG(WARN, "fail to alloc flush task", KR(ret), K(flush_ctx_)); + } else { + flush_ctx_.inc_create_flush_task_cnt(); + flushing_queue.push(flush_task); + STORAGE_LOG(DEBUG, "create new flush task", K(fast_flush_meta), KPC(flush_task), K(flush_ctx_)); + + FlushState state = ObTmpFileFlushTask::TFFT_INITED; + FlushState next_state = state; + do { + next_state = state = flush_task->get_state(); + if (OB_FAIL(drive_flush_task_prepare_(*flush_task, state, next_state))) { + STORAGE_LOG(WARN, "fail to drive flush task prepare", KR(ret), K(flush_ctx_)); + } else if (flush_task->get_state() >= next_state) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected flush state after drive task succ", KR(ret), K(flush_ctx_), K(state)); + } else if (OB_FAIL(advance_status_(*flush_task, next_state))) { + STORAGE_LOG(WARN, "fail to advance status", + KR(ret), K(flush_ctx_), K(flush_task->get_state()), K(state), K(next_state)); + } + } while (OB_SUCC(ret) && FlushState::TFFT_WAIT != next_state); + + STORAGE_LOG(DEBUG, "drive flush task finished", KR(ret), K(fast_flush_meta), KPC(flush_task), K(flush_ctx_)); + flush_ctx_.update_ctx_by_flush_task(*flush_task); + if (ObTmpFileFlushTask::TFFT_FILL_BLOCK_BUF < flush_task->get_state()) { + flush_ctx_.record_flush_task(flush_task->get_data_length()); // maintain statistics + } + if (flush_task->get_is_fast_flush_tree()) { + if (OB_FAIL(ret)) { + STORAGE_LOG(ERROR, "fail to execute fast_flush_tree_page flush task to TFFT_WAIT", KR(ret), KPC(flush_task)); + } + break; // generate only one fast_flush_tree_page_ task to avoid excessive flushing of the meta + } + if (OB_ALLOCATE_TMP_FILE_PAGE_FAILED == ret){ + if (flush_task->get_state() == FlushState::TFFT_INSERT_META_TREE) { + STORAGE_LOG(WARN, "fail to insert meta tree, generating fast_flush_meta task", KR(ret)); + fast_flush_meta = true; // set this flag generate fast_flush_tree_page_ task in the next loop + ret = OB_SUCCESS; + } else { + STORAGE_LOG(ERROR, "flush task is not in TFFT_INSERT_META_TREE state", KPC(flush_task)); + } + } + } + } + + if (!flushing_queue.is_empty()) { + STORAGE_LOG(DEBUG, "ObTmpFileFlushManager flush finish", KR(ret), K(fast_flush_meta), K(flush_ctx_)); + } + if (OB_FAIL(ret) && !flushing_queue.is_empty()) { + ret = OB_SUCCESS; // ignore error if generate at least 1 task + } + flush_ctx_.clear_flush_ctx(flush_priority_mgr_); + } + return ret; +} + +// skip flush level, copy meta tree pages directly +int ObTmpFileFlushManager::fast_fill_block_buf_with_meta_(ObTmpFileFlushTask &flush_task) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(inner_fill_block_buf_(flush_task, FlushCtxState::FSM_F4, true/*is_meta*/, false/*flush_tail*/))) { + LOG_WARN("fail to fast generate flush meta task in flush policy f4", KR(ret), K(flush_task)); + } + + if (flush_task.get_data_length() < FAST_FLUSH_TREE_PAGE_NUM * ObTmpFileGlobal::PAGE_SIZE) { // ignore error code and try F5 + if (OB_FAIL(inner_fill_block_buf_(flush_task, FlushCtxState::FSM_F5, true/*is_meta*/, true/*flush_tail*/))) { + LOG_WARN("fail to fast generate flush meta task in flush policy f5", KR(ret), K(flush_task)); + } + } + + if (OB_FAIL(ret)) { + if (OB_ITER_END == ret && flush_task.get_flush_infos().size() != 0) { + // ignore OB_ITER_END error code to continue to next stage + LOG_INFO("fast_fill_block_buf_with_meta_ iterator reach end, ignore error code to continue", + KR(ret), K(flush_task), K(flush_ctx_)); + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to fast fill block buf with meta", KR(ret), K(flush_task), K(flush_ctx_)); + } + } + return ret; +} + +int ObTmpFileFlushManager::fill_block_buf_(ObTmpFileFlushTask &flush_task) +{ + int ret = OB_SUCCESS; + switch(flush_ctx_.get_state()) { + case FlushCtxState::FSM_F1: + if (!flush_task.is_full() && FlushCtxState::FSM_FINISHED != flush_ctx_.get_state() + && OB_FAIL(inner_fill_block_buf_(flush_task, flush_ctx_.get_state(), + false/*is_meta*/, false/*flush_tail*/))) { + if (OB_ITER_END != ret) { + STORAGE_LOG(WARN, "fail to generate flush data task in flush policy f1", KR(ret), K(flush_task)); + } + } + advance_flush_level_(ret); + // go through + case FlushCtxState::FSM_F2: + if (!flush_task.is_full() && FlushCtxState::FSM_FINISHED != flush_ctx_.get_state() + && OB_FAIL(inner_fill_block_buf_(flush_task, flush_ctx_.get_state(), + false/*is_meta*/, false/*flush_tail*/))) { + if (OB_ITER_END != ret) { + STORAGE_LOG(WARN, "fail to generate flush data task in flush policy f2", KR(ret), K(flush_task)); + } + } + advance_flush_level_(ret); + // go through + case FlushCtxState::FSM_F3: + if (!flush_task.is_full() && FlushCtxState::FSM_FINISHED != flush_ctx_.get_state() + && OB_FAIL(inner_fill_block_buf_(flush_task, flush_ctx_.get_state(), + false/*is_meta*/, true/*flush_tail*/))) { + if (OB_ITER_END != ret) { + STORAGE_LOG(WARN, "fail to generate flush data task in flush policy f3", KR(ret), K(flush_task)); + } + } + advance_flush_level_(ret); + // break here to forbid flush data pages and meta pages in the same flush task + // to prevent 1 flush task contains all of meta page that can be flushed, + // but is stuck in TFFT_INSERT_META_TREE state and new task has no meta pages to flush + break; + case FlushCtxState::FSM_F4: + if (!flush_task.is_full() && FlushCtxState::FSM_FINISHED != flush_ctx_.get_state() + && OB_FAIL(inner_fill_block_buf_(flush_task, flush_ctx_.get_state(), + true/*is_meta*/, false/*flush_tail*/))) { + if (OB_ITER_END != ret) { + STORAGE_LOG(WARN, "fail to generate flush meta task in flush policy f4", KR(ret), K(flush_task)); + } + } + advance_flush_level_(ret); + // go through + case FlushCtxState::FSM_F5: + if (!flush_task.is_full() && FlushCtxState::FSM_FINISHED != flush_ctx_.get_state() + && OB_FAIL(inner_fill_block_buf_(flush_task, flush_ctx_.get_state(), + true/*is_meta*/, true/*flush_tail*/))) { + if (OB_ITER_END != ret) { + STORAGE_LOG(WARN, "fail to generate flush meta task in flush policy f5", KR(ret), K(flush_task)); + } + } + advance_flush_level_(ret); + // go through + case FlushCtxState::FSM_FINISHED: + // do nothing + break; + default: + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(ERROR, "unexpected flush state", KR(ret), K(flush_ctx_)); + break; + } + if (OB_ITER_END == ret && flush_task.get_flush_infos().size() != 0) { + // ignore OB_ITER_END error code to continue TFFT_INSERT_META_TREE state + STORAGE_LOG(DEBUG, "fill_block_buf_ iterator reach end, ignore error code to continue", + KR(ret), K(flush_task), K(flush_ctx_)); + ret = OB_SUCCESS; + } + return ret; +} + +int ObTmpFileFlushManager::inner_fill_block_buf_( + ObTmpFileFlushTask &flush_task, + const FlushCtxState flush_stage, + const bool is_meta, + const bool flush_tail) +{ + int ret = OB_SUCCESS; + const int64_t BLOCK_SIZE = OB_SERVER_BLOCK_MGR.get_macro_block_size(); + bool fail_too_many = false; + + ObArray &flush_failed_array = flush_ctx_.get_flush_failed_array(); + ObTmpFileFlushListIterator &iter = flush_ctx_.get_flush_list_iterator(); + bool tmp_is_meta = false; + ObTmpFileHandle file_handle; + while (OB_SUCC(ret) && !fail_too_many && !flush_task.is_full()) { + if (OB_FAIL(iter.next(flush_stage, tmp_is_meta, file_handle))) { + if (OB_ITER_END != ret) { + STORAGE_LOG(WARN, "fail to get file from iterator", + KR(ret), K(flush_stage), K(is_meta), K(flush_tail), K(flush_ctx_)); + } + } else if (OB_ISNULL(file_handle.get())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "file handle is nullptr", KR(ret)); + } else { + ObSharedNothingTmpFile &file = *file_handle.get(); + if (file.is_deleting()) { + STORAGE_LOG(INFO, "file is deleted while generating flush task", K(file)); + } else { + STORAGE_LOG(DEBUG, "try to copy data from file", K(file.get_fd()), K(is_meta), K(flush_tail), K(file)); + int64_t last_idx = -1; + bool copy_flush_info_fail = false; + ObArray &flush_infos = flush_task.get_flush_infos(); + const int64_t origin_info_cnt = flush_infos.count(); + ObTmpFileSingleFlushContext file_flush_ctx(file_handle); + // push back first to prevent array resizing or + // hash map allocate node failure AFTER copying the file data + if (FAILEDx(get_or_create_file_in_ctx_(file.get_fd(), file_flush_ctx))) { + STORAGE_LOG(WARN, "fail to get or create file in file flush ctx", KR(ret), K(file)); + } else if (OB_FAIL(flush_infos.push_back(ObTmpFileFlushInfo()))) { + STORAGE_LOG(WARN, "fail to insert flush info", KR(ret), K(file), K(flush_task)); + } else if (FALSE_IT(last_idx = flush_infos.size() - 1)) { + } else if (!is_meta && OB_FAIL(file.generate_data_flush_info(flush_task, flush_infos.at(last_idx), + file_flush_ctx.data_ctx_, flush_ctx_.get_flush_sequence(), flush_tail))) { + STORAGE_LOG(WARN, "fail to generate flush data info", KR(ret), K(flush_task), + K(flush_stage), K(is_meta), K(flush_tail), K(file)); + copy_flush_info_fail = true; + } else if (is_meta && OB_FAIL(file.generate_meta_flush_info(flush_task, flush_infos.at(last_idx), + file_flush_ctx.meta_ctx_, flush_ctx_.get_flush_sequence(), flush_tail))) { + STORAGE_LOG(WARN, "fail to generate flush meta info", KR(ret), K(flush_task), + K(flush_stage), K(is_meta), K(flush_tail), K(file)); + copy_flush_info_fail = true; + } else { + if (!(flush_tail || (is_meta && file_flush_ctx.meta_ctx_.is_meta_reach_end_))) { + file.reinsert_flush_node(is_meta); + } + UpdateFlushCtx update_op(file_flush_ctx); + if (OB_FAIL(flush_ctx_.get_file_ctx_hash().set_or_update(file.get_fd(), file_flush_ctx, update_op))) { + // if update fails, the copy offset will be incorrect when the file is flushed a second time in the same round + STORAGE_LOG(ERROR, "fail to set flush ctx after copying data", KR(ret), K(file)); + copy_flush_info_fail = true; + } + flush_ctx_.record_flush_stage(); + } + + if (OB_FAIL(ret)) { + int tmp_ret = OB_SUCCESS; + ObTmpFileBatchFlushContext::ObTmpFileFlushFailRecord record(is_meta, ObTmpFileHandle(&file)); + if (OB_TMP_FAIL(flush_failed_array.push_back(record))) { + // array is pre-allocated to MAX_COPY_FAIL_COUNT, + // file could not be flush afterwards if push_back failed + LOG_ERROR("fail to push back flush failed array", KR(tmp_ret), K(file.get_fd())); + } else if (!flush_task.get_is_fast_flush_tree() + && flush_failed_array.size() >= ObTmpFileBatchFlushContext::MAX_COPY_FAIL_COUNT - 8) { + fail_too_many = true; + LOG_WARN("inner_fill_block_buf_ fail too many times", K(file.get_fd()), K(flush_failed_array.size())); + } else if (flush_task.get_is_fast_flush_tree() + && flush_failed_array.size() >= ObTmpFileBatchFlushContext::MAX_COPY_FAIL_COUNT) { + fail_too_many = true; + LOG_WARN("inner_fill_block_buf_ fail too many times", K(file.get_fd()), K(flush_failed_array.size())); + flush_ctx_.set_fail_too_many(true); // set this flag to exit flush + } + + if (flush_infos.size() > origin_info_cnt) { + flush_infos.pop_back(); + } + if (!file_flush_ctx.data_ctx_.is_valid() && !file_flush_ctx.meta_ctx_.is_valid()) { + // clear pre-created files to ensure that file_ctx_hash_ only records files that were able to send IO + if (OB_TMP_FAIL(flush_ctx_.get_file_ctx_hash().erase_refactored(file.get_fd()))) { + if (OB_HASH_NOT_EXIST != tmp_ret) { + STORAGE_LOG(ERROR, "fail to erase file ctx from hash", KR(tmp_ret), K(file.get_fd())); + } + } + } + + if (!copy_flush_info_fail) { + STORAGE_LOG(WARN, "inner fill block buffer fail, try next file", KR(ret), K(file.get_fd())); + ret = OB_SUCCESS; // ignore error code if fail before copying data + } + } + } + } + file_handle.reset(); + } // end while + + return ret; +} + +void ObTmpFileFlushManager::UpdateFlushCtx::operator() (hash::HashMapPair &pair) +{ + int64_t fd = pair.first; + ObTmpFileDataFlushContext &data_ctx = pair.second.data_ctx_; + ObTmpFileTreeFlushContext &meta_ctx = pair.second.meta_ctx_; + if (input_data_ctx_.is_valid()) { + data_ctx = input_data_ctx_; + } + if (input_meta_ctx_.is_valid()) { + meta_ctx = input_meta_ctx_; + } + STORAGE_LOG(DEBUG, "UpdateFlushCtx after fill block data", K(fd), K(data_ctx), K(meta_ctx)); +} + +int ObTmpFileFlushManager::get_or_create_file_in_ctx_(const int64_t fd, ObTmpFileSingleFlushContext &file_flush_ctx) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(flush_ctx_.get_file_ctx_hash().get_refactored(fd, file_flush_ctx))) { + if (OB_HASH_NOT_EXIST == ret) { + ret = OB_SUCCESS; + if (OB_FAIL(flush_ctx_.get_file_ctx_hash().set_refactored(fd, file_flush_ctx))) { + STORAGE_LOG(WARN, "fail to insert file flush context", KR(ret), K(fd)); + } + } else { + STORAGE_LOG(WARN, "fail to get file flush context", KR(ret), K(fd)); + } + } + return ret; +} + +// iterate all file to insert data items into its meta tree +int ObTmpFileFlushManager::insert_items_into_meta_tree_(ObTmpFileFlushTask &flush_task, + const int64_t logic_block_index) +{ + int ret = OB_SUCCESS; + ObArray &flush_infos = flush_task.get_flush_infos(); + for (int64_t i = 0; OB_SUCC(ret) && i < flush_infos.count(); ++i) { + ObTmpFileFlushInfo &flush_info = flush_infos.at(i); + ObSharedNothingTmpFile *file = flush_info.file_handle_.get(); + if (OB_ISNULL(file)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "file is nullptr", KR(ret), K(flush_info)); + } else { + if (flush_info.has_data() && !flush_info.insert_meta_tree_done_) { + if (OB_FAIL(file->insert_meta_tree_item(flush_info, logic_block_index))) { + STORAGE_LOG(WARN, "fail to insert meta tree item", KR(ret), K(flush_info), K(logic_block_index), KP(&flush_task)); + // flushing data pages may generate new meta pages. ff there is not enough space allocated for the meta pages, + // it will result in the failure of the current data flushing. therefore, we need to evict some pages to free up space + if (OB_ALLOCATE_TMP_FILE_PAGE_FAILED == ret) { + if (OB_FAIL(evict_pages_and_retry_insert_(flush_task, flush_info, logic_block_index))) { + STORAGE_LOG(WARN, "fail to evict pages and retry insert meta tree item", + KR(ret), K(flush_info), K(logic_block_index)); + } + } + } else { + flush_info.insert_meta_tree_done_ = true; + } + } + } + } + return ret; +} + +// If eviction or insertion fails, retry the operation on the next round +int ObTmpFileFlushManager::evict_pages_and_retry_insert_(ObTmpFileFlushTask &flush_task, + ObTmpFileFlushInfo &flush_info, + const int64_t logic_block_index) +{ + int ret = OB_SUCCESS; + int64_t expect_evict_page = FAST_FLUSH_TREE_PAGE_NUM; + int64_t actual_evict_page = 0; + ObSharedNothingTmpFile *file = flush_info.file_handle_.get(); + if (OB_ISNULL(file)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "file ptr is null", KR(ret), K(flush_task), K(logic_block_index)); + } else if (OB_FAIL(evict_mgr_.evict(expect_evict_page, actual_evict_page))) { + STORAGE_LOG(WARN, "fail to evict meta pages while flushing data pages", + KR(ret), K(flush_task), K(logic_block_index)); + } else if (OB_FAIL(file->insert_meta_tree_item(flush_info, logic_block_index))) { + STORAGE_LOG(WARN, "fail to insert meta tree item again, retry later", + KR(ret), K(flush_info), K(logic_block_index)); + } else { + flush_info.insert_meta_tree_done_ = true; + } + return ret; +} + +int ObTmpFileFlushManager::drive_flush_task_prepare_(ObTmpFileFlushTask &flush_task, + const FlushState state, + FlushState &next_state) +{ + int ret = OB_SUCCESS; + next_state = state; + switch (state) { + case FlushState::TFFT_CREATE_BLOCK_INDEX: + if (OB_FAIL(handle_create_block_index_(flush_task, next_state))) { + STORAGE_LOG(WARN, "fail to handle flush task create block index", KR(ret), K(flush_task)); + } + break; + case FlushState::TFFT_FILL_BLOCK_BUF: + if (OB_FAIL(handle_fill_block_buf_(flush_task, next_state))) { + STORAGE_LOG(WARN, "fail to handle flush task fill block", KR(ret), K(flush_task)); + } + break; + case FlushState::TFFT_INSERT_META_TREE: + if (OB_FAIL(handle_insert_meta_tree_(flush_task, next_state))) { + STORAGE_LOG(WARN, "fail to handle flush task insert meta tree", KR(ret), K(flush_task)); + } + break; + case FlushState::TFFT_ASYNC_WRITE: + if (OB_FAIL(handle_async_write_(flush_task, next_state))) { + STORAGE_LOG(WARN, "fail to handle flush task async write", KR(ret), K(flush_task)); + } + break; + default: + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected state in drive_flush_task_prepare_", K(state)); + break; + } + return ret; +} + +int ObTmpFileFlushManager::drive_flush_task_retry_( + ObTmpFileFlushTask &flush_task, + const FlushState state, + FlushState &next_state) +{ + int ret = OB_SUCCESS; + next_state = state; + switch (state) { + case FlushState::TFFT_INSERT_META_TREE: + if (OB_FAIL(handle_insert_meta_tree_(flush_task, next_state))) { + STORAGE_LOG(WARN, "fail to handle flush task insert meta tree", KR(ret), K(flush_task)); + } + break; + case FlushState::TFFT_ASYNC_WRITE: + if (OB_FAIL(handle_async_write_(flush_task, next_state))) { + STORAGE_LOG(WARN, "fail to handle flush task async write", KR(ret), K(flush_task)); + } + break; + default: + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected state in drive_flush_task_retry_", K(state), K(flush_task)); + break; + } + return ret; +} + +int ObTmpFileFlushManager::drive_flush_task_wait_to_finish_(ObTmpFileFlushTask &flush_task, FlushState &next_state) +{ + int ret = OB_SUCCESS; + ObTmpFileFlushTask::ObTmpFileFlushTaskState state = flush_task.get_state(); + switch (state) { + case FlushState::TFFT_WAIT: + if (OB_FAIL(handle_wait_(flush_task, next_state))) { + STORAGE_LOG(WARN, "fail to handle wait", KR(ret), K(flush_task)); + } + break; + default: + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected state in drive_flush_task_wait_to_finish_", K(state)); + break; + } + return ret; +} + +int ObTmpFileFlushManager::retry(ObTmpFileFlushTask &flush_task) +{ + int ret = OB_SUCCESS; + + FlushState state = flush_task.get_state(); + FlushState next_state = state; + do { + next_state = state = flush_task.get_state(); + if (OB_FAIL(drive_flush_task_retry_(flush_task, state, next_state))) { + STORAGE_LOG(WARN, "fail to drive flush state machine", KR(ret), K(flush_task)); + } else if (flush_task.get_state() >= next_state) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected flush state after drive task succ", KR(ret), K(state), K(flush_task)); + } else if (OB_FAIL(advance_status_(flush_task, next_state))) { + STORAGE_LOG(WARN, "fail to advance status", KR(ret), K(state), K(next_state), K(flush_task)); + } + } while (OB_SUCC(ret) && FlushState::TFFT_WAIT != next_state); + + flush_ctx_.update_ctx_by_flush_task(flush_task); + if (flush_ctx_.can_clear_flush_ctx()) { + flush_ctx_.clear_flush_ctx(flush_priority_mgr_); + } + return ret; +} + +int ObTmpFileFlushManager::io_finished(ObTmpFileFlushTask &flush_task) +{ + int ret = OB_SUCCESS; + FlushState next_state = FlushState::TFFT_INITED; + if (OB_FAIL(drive_flush_task_wait_to_finish_(flush_task, next_state))) { + STORAGE_LOG(WARN, "fail to drive flush state machine to FINISHED", KR(ret), K(flush_task)); + } else if (flush_task.get_state() < next_state && OB_FAIL(advance_status_(flush_task, next_state))) { + // if the task encounters an IO error, its status will silently revert to TFFT_ASYNC_WRITE; do not verify status here. + STORAGE_LOG(WARN, "fail to advance status", KR(ret), K(flush_task.get_state()), K(next_state)); + } + return ret; +} + +int ObTmpFileFlushManager::update_file_meta_after_flush(ObTmpFileFlushTask &flush_task) +{ + int ret = OB_SUCCESS; + if (FlushState::TFFT_FINISH != flush_task.get_state()) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected flush state after drive task succ", KR(ret), K(flush_task)); + } else if (OB_FAIL(handle_finish_(flush_task))) { + STORAGE_LOG(WARN, "fail to update file meta after flush", KR(ret), K(flush_task)); + } + return ret; +} + +int ObTmpFileFlushManager::advance_status_(ObTmpFileFlushTask &flush_task, const FlushState &state) +{ + int ret = OB_SUCCESS; + if (flush_task.get_state() >= state) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "unexpected state in advance_status_", K(state), K(flush_task)); + } else { + flush_task.set_state(state); + STORAGE_LOG(DEBUG, "advance flush task status succ", K(state), K(flush_task)); + } + return ret; +} + +int ObTmpFileFlushManager::handle_alloc_flush_task_(const bool fast_flush_meta, ObTmpFileFlushTask *&flush_task) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(alloc_flush_task(flush_task))) { + STORAGE_LOG(WARN, "fail to alloc flush callback", KR(ret)); + } else if (OB_ISNULL(flush_task)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "flush cb is null", KR(ret)); + } else { + flush_task->set_state(FlushState::TFFT_CREATE_BLOCK_INDEX); + flush_task->set_create_ts(ObTimeUtil::current_time()); + flush_task->set_flush_seq(flush_ctx_.get_flush_sequence()); + flush_task->set_is_fast_flush_tree(fast_flush_meta); + } + return ret; +} + +int ObTmpFileFlushManager::handle_create_block_index_(ObTmpFileFlushTask &flush_task, FlushState &next_state) +{ + int ret = OB_SUCCESS; + int64_t block_index = ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX; + ObTmpFileBlockHandle tmp_file_block_handle; + // occupy the whole tmp file block because we don't know how many pages we will use before TFFT_FILL_BLOCK_BUF, + // and we still create block first because copy meta tree pages needs to provide tmp file block's block_index, + // unused pages will be released after TFFT_FILL_BLOCK_BUF succ + if (OB_FAIL(tmp_file_block_mgr_.create_tmp_file_block(0/*begin_page_id*/, + ObTmpFileGlobal::BLOCK_PAGE_NUMS, + block_index))) { + STORAGE_LOG(WARN, "fail to create tmp file block", KR(ret), K(flush_task)); + } else if (OB_FAIL(tmp_file_block_mgr_.get_tmp_file_block_handle(block_index, tmp_file_block_handle))) { + // keep a tmp file block handle in flush task to prevent rollback operations + // caused by "append writes last not full page during the flush process" release tmp file block. + // for example, a tmp file block will be unexpected released if a flush task + // only contains "not full pages" and all there pages are rollback by append writes + STORAGE_LOG(WARN, "fail to get tmp file block handle", KR(ret), K(block_index), K(flush_task)); + } else { + flush_task.set_tmp_file_block_handle(tmp_file_block_handle); + flush_task.set_block_index(block_index); + next_state = FlushState::TFFT_FILL_BLOCK_BUF; + } + return ret; +} + +int ObTmpFileFlushManager::handle_fill_block_buf_(ObTmpFileFlushTask &flush_task, FlushState &next_state) +{ + int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + if (OB_FAIL(flush_task.prealloc_block_buf())) { + STORAGE_LOG(WARN, "fail to prealloc block buf", KR(ret), K(flush_task)); + } else if (flush_task.get_is_fast_flush_tree()) { // skip flush level, copy meta tree pages directly + if (OB_FAIL(fast_fill_block_buf_with_meta_(flush_task))) { + STORAGE_LOG(WARN, "fail to fill block buffer with meta", KR(ret), K(flush_task)); + } + } else { + if (OB_FAIL(fill_block_buf_(flush_task))) { + if (OB_ITER_END != ret) { + STORAGE_LOG(WARN, "fail to fill block buf", KR(ret), K(flush_task)); + } + } else if (0 == flush_task.get_data_length()) { + ret = OB_ITER_END; + } + } + + if (OB_FAIL(ret)){ + STORAGE_LOG(WARN, "fail to fill block buf, skip release page", KR(ret)); + } else if (OB_UNLIKELY(flush_task.get_flush_infos().empty())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(ERROR, "flush infos is empty", KR(ret), K(flush_task)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < flush_task.get_flush_infos().count(); ++i) { + if (OB_UNLIKELY(flush_task.get_flush_infos().at(i).has_data() && + flush_task.get_flush_infos().at(i).has_meta())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(ERROR, "flush info has both data and meta", KR(ret), K(flush_task)); + } else if (OB_UNLIKELY((flush_task.get_flush_infos().at(0).has_data() != + flush_task.get_flush_infos().at(i).has_data()) || + (flush_task.get_flush_infos().at(0).has_meta() != + flush_task.get_flush_infos().at(i).has_meta()))) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(ERROR, "flush infos mixed storage meta and data pages", KR(ret), K(flush_task)); + } + } + } + + if (OB_SUCC(ret)) { + const bool is_whole_data_page = flush_task.get_flush_infos().at(0).has_data(); + if (is_whole_data_page && + OB_TMP_FAIL(ObTmpBlockCache::get_instance().put_block(flush_task.get_inst_handle(), + flush_task.get_kvpair(), + flush_task.get_block_handle()))) { + STORAGE_LOG(WARN, "fail to put block into block cache", KR(tmp_ret), K(flush_task)); + } + + int64_t used_page_num = flush_task.get_total_page_num(); + int64_t unused_page_id = used_page_num; + int64_t unused_page_num = ObTmpFileGlobal::BLOCK_PAGE_NUMS - used_page_num; + int64_t block_index = flush_task.get_block_index(); + bool need_release_page = unused_page_num > 0; + + if (OB_FAIL(tmp_file_block_mgr_.write_back_start(block_index))) { + STORAGE_LOG(ERROR, "fail to notify tmp file block write back start", KR(ret), K(block_index)); + } else if (need_release_page && OB_FAIL(tmp_file_block_mgr_.release_tmp_file_page( + block_index, unused_page_id, unused_page_num))) { + STORAGE_LOG(ERROR, "fail to release tmp file page", + KR(ret), K(unused_page_id), K(unused_page_num), K(used_page_num), K(block_index), K(flush_task)); + } else { + next_state = FlushState::TFFT_INSERT_META_TREE; + } + } + + return ret; +} + +int ObTmpFileFlushManager::handle_insert_meta_tree_(ObTmpFileFlushTask &flush_task, FlushState &next_state) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(insert_items_into_meta_tree_(flush_task, flush_task.get_block_index()))) { + STORAGE_LOG(WARN, "fail to insert meta tree", KR(ret), K(flush_task)); + } else { + next_state = FlushState::TFFT_ASYNC_WRITE; + } + return ret; +} + +int ObTmpFileFlushManager::handle_async_write_(ObTmpFileFlushTask &flush_task, FlushState &next_state) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(flush_task.write_one_block())) { + STORAGE_LOG(WARN, "fail to async write blocks", KR(ret), K(flush_task)); + } else { + next_state = FlushState::TFFT_WAIT; + } + return ret; +} + +int ObTmpFileFlushManager::handle_wait_(ObTmpFileFlushTask &flush_task, FlushState &next_state) +{ + int ret = OB_SUCCESS; + int task_ret_code = flush_task.atomic_get_ret_code(); + if (OB_SUCCESS != task_ret_code) { + // rollback the status to TFFT_ASYNC_WRITE if IO failed, and re-send the I/O in the retry process. + STORAGE_LOG(INFO, "flush_task io fail, retry it later", KR(task_ret_code), K(flush_task)); + flush_task.set_state(FlushState::TFFT_ASYNC_WRITE); + } else if (OB_FAIL(tmp_file_block_mgr_.write_back_succ(flush_task.get_block_index(), + flush_task.get_macro_block_handle().get_macro_id()))) { + STORAGE_LOG(WARN, "fail to notify tmp file block write back succ", KR(ret), K(flush_task)); + } else { + next_state = FlushState::TFFT_FINISH; + } + return ret; +} + +// Update file meta after flush task IO complete, ensures reentrancy +int ObTmpFileFlushManager::handle_finish_(ObTmpFileFlushTask &flush_task) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(update_meta_data_after_flush_for_files_(flush_task))) { + STORAGE_LOG(WARN, "fail to update meta data after flush for files", KR(ret), K(flush_task)); + } else { + STORAGE_LOG(DEBUG, "flush task finish successfully", K(flush_task)); + } + + return ret; +} + +void ObTmpFileFlushManager::ResetFlushCtxOp::operator() (hash::HashMapPair &pair) +{ + if (is_meta_) { + pair.second.meta_ctx_.reset(); + } else { + pair.second.data_ctx_.reset(); + } +} + +int ObTmpFileFlushManager::update_meta_data_after_flush_for_files_(ObTmpFileFlushTask &flush_task) +{ + int ret = OB_SUCCESS; + ObArray &flush_infos = flush_task.get_flush_infos(); + for (int64_t i = 0; OB_SUCC(ret) && i < flush_infos.size(); ++i) { + ObTmpFileFlushInfo &flush_info = flush_infos.at(i); + ObSharedNothingTmpFile *file = flush_info.file_handle_.get(); + bool is_meta = false; + bool reset_ctx = false; + if (!flush_info.update_meta_data_done_) { + if (OB_ISNULL(file)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "tmp file ptr is null", KR(ret), K(i), K(flush_task)); + } else if ((OB_UNLIKELY(flush_info.has_data() && flush_info.has_meta()) || + OB_UNLIKELY(!flush_info.has_data() && !flush_info.has_meta()))) { + ret = OB_ERR_UNEXPECTED; // expect one flush_info only contains one type of pages + STORAGE_LOG(ERROR, "flush info is not valid", KR(ret), K(flush_info)); + } else if (FALSE_IT(is_meta = flush_info.has_meta())) { + } else if (OB_FAIL(file->update_meta_after_flush(flush_info.batch_flush_idx_, is_meta, reset_ctx))){ + STORAGE_LOG(WARN, "fail to update meta data", KR(ret), K(is_meta), K(flush_info)); + } else { + int tmp_ret = OB_SUCCESS; + // reset data/meta flush ctx after file update meta complete to prevent + // flush use stale flushed_page_id to copy data after the page is evicted. + // use empty ctx here because we have inserted ctx for every file when flushing begins. + ResetFlushCtxOp reset_op(is_meta); + ObTmpFileSingleFlushContext empty_ctx; + if (flush_task.get_flush_seq() == flush_ctx_.get_flush_sequence() && + OB_TMP_FAIL(flush_ctx_.get_file_ctx_hash().set_or_update(file->get_fd(), empty_ctx, reset_op))) { + STORAGE_LOG(WARN, "fail to clean file ctx from hash", KR(tmp_ret), K(file)); + } + flush_info.update_meta_data_done_ = true; + } + } + } + return ret; +} + +} // end namespace tmp_file +} // end namespace oceanbase diff --git a/src/storage/tmp_file/ob_tmp_file_flush_manager.h b/src/storage/tmp_file/ob_tmp_file_flush_manager.h new file mode 100644 index 000000000..87e1383eb --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_flush_manager.h @@ -0,0 +1,127 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_TASK_MANAGER_H_ +#define OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_TASK_MANAGER_H_ + +#include "storage/tmp_file/ob_tmp_file_eviction_manager.h" +#include "storage/tmp_file/ob_tmp_file_global.h" +#include "storage/tmp_file/ob_tmp_file_flush_ctx.h" +#include "storage/tmp_file/ob_tmp_file_flush_priority_manager.h" +#include "storage/tmp_file/ob_tmp_file_thread_job.h" + +namespace oceanbase +{ +namespace tmp_file +{ + +// flush manager generates flush tasks according to dirty page watermark +class ObTmpFileFlushManager +{ +public: + static const int32_t FLUSH_WATERMARK_F1 = 50; // percentage + static const int32_t FLUSH_WATERMARK_F2 = 70; + static const int32_t FLUSH_WATERMARK_F3 = 75; + static const int32_t FLUSH_WATERMARK_F4 = 80; + static const int32_t FLUSH_WATERMARK_F5 = 90; + static const int32_t FLUSH_LOW_WATERMARK_F1 = 25; + static const int32_t FLUSH_LOW_WATERMARK_F2 = 60; + static const int32_t FLUSH_LOW_WATERMARK_F3 = 65; + static const int32_t FLUSH_LOW_WATERMARK_F4 = 70; + static const int32_t FLUSH_LOW_WATERMARK_F5 = 80; + static const int64_t FAST_FLUSH_TREE_PAGE_NUM = 32; + struct UpdateFlushCtx + { + public: + UpdateFlushCtx(ObTmpFileSingleFlushContext &file_ctx) + : input_data_ctx_(file_ctx.data_ctx_), + input_meta_ctx_(file_ctx.meta_ctx_) {} + void operator() (hash::HashMapPair &pair); + private: + ObTmpFileDataFlushContext &input_data_ctx_; + ObTmpFileTreeFlushContext &input_meta_ctx_; + }; + struct ResetFlushCtxOp + { + public: + public: + ResetFlushCtxOp(const bool is_meta) : is_meta_(is_meta) {} + void operator() (hash::HashMapPair &pair); + private: + bool is_meta_; + }; +public: + typedef ObTmpFileGlobal::FlushCtxState FlushCtxState; + typedef common::ObDList ObTmpFileFlushList; + typedef ObTmpFileFlushTask::ObTmpFileFlushTaskState FlushState; + ObTmpFileFlushManager(ObTmpFilePageCacheController &pc_ctrl); + ~ObTmpFileFlushManager() {} + int init(); + void destroy(); +public: + int free_tmp_file_block(ObTmpFileFlushTask &flush_task); + int alloc_flush_task(ObTmpFileFlushTask *&flush_task); + int free_flush_task(ObTmpFileFlushTask *flush_task); + int notify_write_back_failed(ObTmpFileFlushTask *flush_task); + int flush(ObSpLinkQueue &flushing_queue, + ObTmpFileFlushMonitor &flush_monitor, + const int64_t expect_flush_size, + const bool is_flush_meta_tree); + int retry(ObTmpFileFlushTask &flush_task); + int io_finished(ObTmpFileFlushTask &flush_task); + int update_file_meta_after_flush(ObTmpFileFlushTask &flush_task); +private: + int fill_block_buf_(ObTmpFileFlushTask &flush_task); + int fast_fill_block_buf_with_meta_(ObTmpFileFlushTask &flush_task); + int inner_fill_block_buf_(ObTmpFileFlushTask &flush_task, + const FlushCtxState flush_stage, + const bool is_meta, + const bool flush_tail); + int insert_items_into_meta_tree_(ObTmpFileFlushTask &flush_task, + const int64_t logic_block_index); + void init_flush_level_(); + void advance_flush_level_(const int ret_code); + void inner_advance_flush_level_(); + void inner_advance_flush_level_without_checking_watermark_(); + int64_t get_low_watermark_(FlushCtxState state); + int advance_status_(ObTmpFileFlushTask &flush_task, const FlushState &state); + int drive_flush_task_prepare_(ObTmpFileFlushTask &flush_task, const FlushState state, FlushState &next_state); + int drive_flush_task_retry_(ObTmpFileFlushTask &flush_task, const FlushState state, FlushState &next_state); + int drive_flush_task_wait_to_finish_(ObTmpFileFlushTask &flush_task, FlushState &next_state); + int handle_alloc_flush_task_(const bool fast_flush_meta, ObTmpFileFlushTask *&flush_task); + int handle_create_block_index_(ObTmpFileFlushTask &flush_task, FlushState &next_state); + int handle_fill_block_buf_(ObTmpFileFlushTask &flush_task, FlushState &next_state); + int handle_insert_meta_tree_(ObTmpFileFlushTask &flush_task, FlushState &next_state); + int handle_async_write_(ObTmpFileFlushTask &flush_task, FlushState &next_state); + int handle_wait_(ObTmpFileFlushTask &flush_task, FlushState &next_state); + int handle_finish_(ObTmpFileFlushTask &flush_task); +private: + int update_meta_data_after_flush_for_files_(ObTmpFileFlushTask &flush_task); + int get_or_create_file_in_ctx_(const int64_t fd, ObTmpFileSingleFlushContext &file_flush_ctx); + int evict_pages_and_retry_insert_(ObTmpFileFlushTask &flush_task, + ObTmpFileFlushInfo &flush_info, + const int64_t logic_block_index); + DISALLOW_COPY_AND_ASSIGN(ObTmpFileFlushManager); +private: + bool is_inited_; + ObTmpFileBatchFlushContext flush_ctx_; + ObTmpFilePageCacheController &pc_ctrl_; + ObTmpFileBlockManager &tmp_file_block_mgr_; + ObIAllocator &task_allocator_; // ref to ObTmpFilePageCacheController::task_allocator_ + ObTmpWriteBufferPool &write_buffer_pool_; + ObTmpFileEvictionManager &evict_mgr_; + ObTmpFileFlushPriorityManager &flush_priority_mgr_; +}; + +} // end namespace tmp_file +} // end namespace oceanbase +#endif // OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_TASK_MANAGER_H_ diff --git a/src/storage/tmp_file/ob_tmp_file_flush_priority_manager.cpp b/src/storage/tmp_file/ob_tmp_file_flush_priority_manager.cpp new file mode 100644 index 000000000..5c4567138 --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_flush_priority_manager.cpp @@ -0,0 +1,354 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE + +#include "storage/tmp_file/ob_tmp_file_flush_priority_manager.h" + +namespace oceanbase +{ +namespace tmp_file +{ +//----------------------- ObTmpFileFlushPriorityManager -------------------// +int ObTmpFileFlushPriorityManager::init() +{ + int ret = OB_SUCCESS; + STATIC_ASSERT(ARRAYSIZEOF(data_flush_lists_) == (int64_t)FileList::MAX, + "data_flush_lists_ size mismatch enum FileList count"); + STATIC_ASSERT(ARRAYSIZEOF(data_list_locks_) == (int64_t)FileList::MAX, + "data_list_locks_ size mismatch enum FileList count"); + STATIC_ASSERT(ARRAYSIZEOF(meta_flush_lists_) == (int64_t)FileList::MAX, + "meta_flush_lists_ size mismatch enum FileList count"); + STATIC_ASSERT(ARRAYSIZEOF(meta_list_locks_) == (int64_t)FileList::MAX, + "meta_list_locks_ size mismatch enum FileList count"); + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObTmpFileFlushPriorityManager inited twice", KR(ret)); + } else { + is_inited_ = true; + LOG_INFO("ObTmpFileFlushPriorityManager init succ", K(is_inited_)); + } + return ret; +} + +void ObTmpFileFlushPriorityManager::destroy() +{ + is_inited_ = false; + for (int64_t i = 0; i < FileList::MAX; i++) { + ObSpinLockGuard guard(data_list_locks_[i]); + data_flush_lists_[i].reset(); + } + for (int64_t i = 0; i < FileList::MAX; i++) { + ObSpinLockGuard guard(meta_list_locks_[i]); + meta_flush_lists_[i].reset(); + } +} + +int64_t ObTmpFileFlushPriorityManager::get_file_size() +{ + int64_t size = 0; + is_inited_ = false; + for (int64_t i = 0; i < FileList::MAX; i++) { + ObSpinLockGuard guard(data_list_locks_[i]); + size += data_flush_lists_[i].get_size(); + } + for (int64_t i = 0; i < FileList::MAX; i++) { + ObSpinLockGuard guard(meta_list_locks_[i]); + size += meta_flush_lists_[i].get_size(); + } + return size; +} + +// attention: +// call this function with protection of ObSharedNothingTmpFile's meta_lock +int ObTmpFileFlushPriorityManager::insert_data_flush_list(ObSharedNothingTmpFile &file, const int64_t dirty_page_size) +{ + int ret = OB_SUCCESS; + FileList flush_idx = FileList::MAX; + + if (OB_FAIL(get_data_list_idx_(dirty_page_size, flush_idx))) { + LOG_WARN("fail to get data list idx", KR(ret), K(dirty_page_size)); + } else if (OB_FAIL(insert_flush_list_(false/*is_meta*/, file, flush_idx))) { + LOG_WARN("fail to insert data flush list", KR(ret), K(file), K(dirty_page_size)); + } else { + LOG_DEBUG("insert_data_flush_list succ", K(file), K(dirty_page_size)); + } + + return ret; +} + +// attention: +// call this function with protection of ObSharedNothingTmpFile's meta_lock +int ObTmpFileFlushPriorityManager::insert_meta_flush_list(ObSharedNothingTmpFile &file, + const int64_t non_rightmost_dirty_page_num, + const int64_t rightmost_dirty_page_num) +{ + int ret = OB_SUCCESS; + FileList flush_idx = FileList::MAX; + + if (OB_FAIL(get_meta_list_idx_(non_rightmost_dirty_page_num, rightmost_dirty_page_num, flush_idx))) { + LOG_WARN("fail to get meta list idx", KR(ret), K(non_rightmost_dirty_page_num), K(rightmost_dirty_page_num)); + } else if (OB_FAIL(insert_flush_list_(true/*is_meta*/, file, flush_idx))) { + LOG_WARN("fail to insert meta flush list", KR(ret), K(file), K(flush_idx), + K(non_rightmost_dirty_page_num), K(rightmost_dirty_page_num)); + } else { + LOG_DEBUG("insert_meta_flush_list succ", K(file), K(non_rightmost_dirty_page_num), K(rightmost_dirty_page_num)); + } + + return ret; +} + +int ObTmpFileFlushPriorityManager::insert_flush_list_(const bool is_meta, ObSharedNothingTmpFile &file, + const FileList flush_idx) +{ + int ret = OB_SUCCESS; + ObSharedNothingTmpFile::ObTmpFileNode &flush_node = is_meta ? file.get_meta_flush_node() : file.get_data_flush_node(); + ObSpinLock* locks = is_meta ? meta_list_locks_ : data_list_locks_; + ObTmpFileFlushList *flush_lists = is_meta ? meta_flush_lists_ : data_flush_lists_; + + if (OB_UNLIKELY(flush_node.get_next() != nullptr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("file already in flush list", KR(ret), K(flush_node)); + } else if (flush_idx < FileList::L1 || flush_idx >= FileList::MAX){ + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid flush list idx", KR(ret), K(flush_idx)); + } else { + ObSpinLockGuard guard(locks[flush_idx]); + file.inc_ref_cnt(); + if (OB_UNLIKELY(!flush_lists[flush_idx].add_last(&flush_node))) { + file.dec_ref_cnt(); + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to add node to list", KR(ret), K(flush_idx), K(flush_node)); + } else if (is_meta) { + file.set_meta_page_flush_level(flush_idx); + } else { + file.set_data_page_flush_level(flush_idx); + } + } + + return ret; +} + +// attention: +// call this function with protection of ObSharedNothingTmpFile's meta_lock +int ObTmpFileFlushPriorityManager::update_data_flush_list(ObSharedNothingTmpFile &file, const int64_t dirty_page_size) +{ + int ret = OB_SUCCESS; + FileList new_flush_idx = FileList::MAX; + + if (OB_FAIL(get_data_list_idx_(dirty_page_size, new_flush_idx))) { + LOG_WARN("fail to get data list idx", KR(ret), K(dirty_page_size)); + } else if (OB_FAIL(update_flush_list_(false/*is_meta*/, file, new_flush_idx))) { + LOG_WARN("fail to update data flush list", KR(ret), K(file), K(dirty_page_size)); + } else { + LOG_DEBUG("update_data_flush_list succ", K(file), K(dirty_page_size)); + } + return ret; +} + +// attention: +// call this function with protection of ObSharedNothingTmpFile's meta_lock +int ObTmpFileFlushPriorityManager::update_meta_flush_list(ObSharedNothingTmpFile &file, + const int64_t non_rightmost_dirty_page_num, + const int64_t rightmost_dirty_page_num) +{ + int ret = OB_SUCCESS; + FileList new_flush_idx = FileList::MAX; + + if (OB_FAIL(get_meta_list_idx_(non_rightmost_dirty_page_num, rightmost_dirty_page_num, new_flush_idx))) { + LOG_WARN("fail to get meta list idx", KR(ret), K(non_rightmost_dirty_page_num), K(rightmost_dirty_page_num)); + } else if (OB_FAIL(update_flush_list_(true/*is_meta*/, file, new_flush_idx))) { + LOG_WARN("fail to update meta flush list", KR(ret), K(file), K(new_flush_idx), + K(non_rightmost_dirty_page_num), K(rightmost_dirty_page_num)); + } else { + LOG_DEBUG("update_meta_flush_list succ", K(file), K(non_rightmost_dirty_page_num), K(rightmost_dirty_page_num)); + } + return ret; +} + +int ObTmpFileFlushPriorityManager::update_flush_list_(const bool is_meta, ObSharedNothingTmpFile &file, + const FileList new_flush_idx) +{ + int ret = OB_SUCCESS; + ObSharedNothingTmpFile::ObTmpFileNode &flush_node = is_meta ? file.get_meta_flush_node() : file.get_data_flush_node(); + ObSpinLock* locks = is_meta ? meta_list_locks_ : data_list_locks_; + ObTmpFileFlushList *flush_lists = is_meta ? meta_flush_lists_ : data_flush_lists_; + int64_t cur_flush_idx = is_meta ? file.get_meta_page_flush_level() : file.get_data_page_flush_level(); + + if (cur_flush_idx < FileList::L1 || cur_flush_idx >= FileList::MAX){ + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid flush list idx", KR(ret), K(new_flush_idx)); + } else if (new_flush_idx == cur_flush_idx) { + // no need to update + } else { // need to move file into a new flush list + bool is_in_flushing = false; + { + ObSpinLockGuard guard(locks[cur_flush_idx]); + if (OB_ISNULL(flush_node.get_next())) { + // before we lock the list, flush task mgr has popped the node from list and is operating it, do nothing + is_in_flushing = true; + } else if (OB_UNLIKELY(!flush_lists[cur_flush_idx].remove(&flush_node))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to remove node from old list", KR(ret), K(cur_flush_idx)); + } + } + + if (OB_FAIL(ret)) { + } else if (is_in_flushing) { + // the node will be added into flush list again by flush task mgr + // do nothing + } else { + ObSpinLockGuard guard(locks[new_flush_idx]); + if (OB_UNLIKELY(!flush_lists[new_flush_idx].add_last(&flush_node))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to add node to new list", KR(ret), K(new_flush_idx)); + } else { + if (is_meta) { + file.set_meta_page_flush_level(new_flush_idx); + } else { + file.set_data_page_flush_level(new_flush_idx); + } + } + } + } + return ret; +} + +int ObTmpFileFlushPriorityManager::remove_file(ObSharedNothingTmpFile &file) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(remove_file(true /*is_meta*/, file))) { + LOG_WARN("fail to remove file from meta flush list", KR(ret)); + } else if (OB_FAIL(remove_file(false /*is_meta*/, file))) { + LOG_WARN("fail to remove file from data flush list", KR(ret)); + } + return ret; +} + +// attention: +// call this function with protection of ObSharedNothingTmpFile's meta_lock +int ObTmpFileFlushPriorityManager::remove_file(const bool is_meta, ObSharedNothingTmpFile &file) +{ + int ret = OB_SUCCESS; + int64_t flush_idx = is_meta ? file.get_meta_page_flush_level() : file.get_data_page_flush_level(); + ObSharedNothingTmpFile::ObTmpFileNode &flush_node = is_meta ? file.get_meta_flush_node() : file.get_data_flush_node(); + if (FileList::INVALID == flush_idx) { + // file doesn't exist in the flushing list + // do nothing + } else if (flush_idx < FileList::L1 || flush_idx >= FileList::MAX){ + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid flush list idx", KR(ret), K(flush_idx)); + } else { + ObSpinLock* locks = is_meta ? meta_list_locks_ : data_list_locks_; + ObTmpFileFlushList *flush_lists = is_meta ? meta_flush_lists_ : data_flush_lists_; + ObSpinLockGuard guard(locks[flush_idx]); + if (OB_ISNULL(flush_node.get_next())) { + // node has not been inserted, do nothing + } else if (OB_ISNULL(flush_lists[flush_idx].remove(&flush_node))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to remove node from list", KR(ret), K(flush_idx)); + } else { + file.dec_ref_cnt(); + if (is_meta) { + file.set_meta_page_flush_level(-1); + } else { + file.set_data_page_flush_level(-1); + } + LOG_DEBUG("remove file succ", K(file), K(is_meta)); + } + } + return ret; +} + +int ObTmpFileFlushPriorityManager::popN_from_file_list(const bool is_meta, const int64_t list_idx, + const int64_t expected_count, int64_t &actual_count, + ObArray &file_handles) +{ + int ret = OB_SUCCESS; + ObSpinLock* locks = is_meta ? meta_list_locks_ : data_list_locks_; + ObTmpFileFlushList *flush_lists = is_meta ? meta_flush_lists_ : data_flush_lists_; + actual_count = 0; + + if (OB_UNLIKELY(list_idx < FileList::L1 || list_idx >= FileList::MAX)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(list_idx)); + } else { + ObSpinLockGuard guard(locks[list_idx]); + while (OB_SUCC(ret) && !flush_lists[list_idx].is_empty() && actual_count < expected_count) { + ObSharedNothingTmpFile *file = nullptr; + if (OB_ISNULL(file = &flush_lists[list_idx].remove_first()->file_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("file is null", KR(ret)); + } else if (OB_FAIL(file_handles.push_back(file))) { + LOG_WARN("fail to push back", KR(ret), KP(file)); + int tmp_ret = OB_SUCCESS; + ObSharedNothingTmpFile::ObTmpFileNode &node = is_meta ? file->get_meta_flush_node() : file->get_data_flush_node(); + if (OB_UNLIKELY(!flush_lists[list_idx].add_last(&node))) { + tmp_ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to add node to list", KR(tmp_ret), K(list_idx), KP(file)); + } + } else { + LOG_DEBUG("pop file succ", KPC(file), K(is_meta)); + file->dec_ref_cnt(); // ref_cnt of flush list + actual_count++; + } + } // end while + } + + return ret; +} + +int ObTmpFileFlushPriorityManager::get_meta_list_idx_(const int64_t non_rightmost_dirty_page_num, + const int64_t rightmost_dirty_page_num, FileList &idx) +{ + int ret = OB_SUCCESS; + idx = FileList::MAX; + if (non_rightmost_dirty_page_num + rightmost_dirty_page_num <= 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(non_rightmost_dirty_page_num), K(rightmost_dirty_page_num)); + } else if (non_rightmost_dirty_page_num == 0) { // all dirty pages is the rightmost page + idx = FileList::L5; + } else if (non_rightmost_dirty_page_num < 16) { // (0KB, 128KB) + idx = FileList::L4; + } else if (non_rightmost_dirty_page_num < 128) { // [128KB, 1MB) + idx = FileList::L3; + } else if (non_rightmost_dirty_page_num < 256) { // [1MB, 2MB) + idx = FileList::L2; + } else { // [2MB, INFINITE) + idx = FileList::L1; + } + return ret; +} + +int ObTmpFileFlushPriorityManager::get_data_list_idx_(const int64_t dirty_page_size, FileList &idx) +{ + int ret = OB_SUCCESS; + idx = FileList::MAX; + if (dirty_page_size <= 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(dirty_page_size)); + } else if (dirty_page_size < (1 << 13)) { // page_size 8KB, (0, 8KB) + idx = FileList::L5; + } else if (dirty_page_size < (1 << 17)) { // [8KB, 128KB) + idx = FileList::L4; + } else if (dirty_page_size < (1 << 20)) { // [128KB, 1MB) + idx = FileList::L3; + } else if (dirty_page_size < (1 << 21)) { // [1MB, 2MB) + idx = FileList::L2; + } else { // [2MB, INFINITE) + idx = FileList::L1; + } + return ret; +} + +} // end namespace tmp_file +} // end namespace oceanbase diff --git a/src/storage/tmp_file/ob_tmp_file_flush_priority_manager.h b/src/storage/tmp_file/ob_tmp_file_flush_priority_manager.h new file mode 100644 index 000000000..47f9e3600 --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_flush_priority_manager.h @@ -0,0 +1,82 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_FLUSH_PRIORITY_MANAGER_H_ +#define OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_FLUSH_PRIORITY_MANAGER_H_ + +#include "storage/tmp_file/ob_tmp_file_global.h" +#include "storage/tmp_file/ob_shared_nothing_tmp_file.h" + +namespace oceanbase +{ +namespace tmp_file +{ +class ObTmpFileFlushPriorityManager +{ +public: + ObTmpFileFlushPriorityManager() + : is_inited_(false), + data_flush_lists_(), + data_list_locks_(), + meta_flush_lists_(), + meta_list_locks_() + {} + ~ObTmpFileFlushPriorityManager() {} + int init(); + void destroy(); + +private: + enum FileList { + INVALID = -1, + L1 = 0, // [2MB, INFINITE) + L2, // [1MB, 2MB) + L3, // [128KB, 1MB) + L4, // data_list: [8KB, 128KB); meta_list: (0KB, 128KB) + L5, // data_list: (0, 8KB); meta_list: 0KB + MAX + }; + typedef common::ObDList ObTmpFileFlushList; + friend class ObTmpFileFlushListIterator; + +public: + int insert_data_flush_list(ObSharedNothingTmpFile &file, const int64_t dirty_page_size); + int insert_meta_flush_list(ObSharedNothingTmpFile &file, const int64_t non_rightmost_dirty_page_num, + const int64_t rightmost_dirty_page_num); + int update_data_flush_list(ObSharedNothingTmpFile &file, const int64_t dirty_page_size); + int update_meta_flush_list(ObSharedNothingTmpFile &file, const int64_t non_rightmost_dirty_page_num, + const int64_t rightmost_dirty_page_num); + int remove_file(ObSharedNothingTmpFile &file); + int remove_file(const bool is_meta, ObSharedNothingTmpFile &file); + int popN_from_file_list(const bool is_meta, const int64_t list_idx, + const int64_t expected_count, int64_t &actual_count, + ObArray &file_handles); + int64_t get_file_size(); +private: + int get_meta_list_idx_(const int64_t non_rightmost_dirty_page_num, + const int64_t rightmost_dirty_page_num, FileList &idx); + int get_data_list_idx_(const int64_t dirty_page_size, FileList &idx); + int insert_flush_list_(const bool is_meta, ObSharedNothingTmpFile &file, + const FileList flush_idx); + int update_flush_list_(const bool is_meta, ObSharedNothingTmpFile &file, + const FileList new_flush_idx); + +private: + bool is_inited_; + ObTmpFileFlushList data_flush_lists_[FileList::MAX]; + ObSpinLock data_list_locks_[FileList::MAX]; + ObTmpFileFlushList meta_flush_lists_[FileList::MAX]; + ObSpinLock meta_list_locks_[FileList::MAX]; +}; + +} // end namespace tmp_file +} // end namespace oceanbase +#endif // OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_FLUSH_PRIORITY_MANAGER_H_ diff --git a/src/storage/tmp_file/ob_tmp_file_global.cpp b/src/storage/tmp_file/ob_tmp_file_global.cpp new file mode 100644 index 000000000..f2427f94a --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_global.cpp @@ -0,0 +1,30 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE + +#include "storage/tmp_file/ob_tmp_file_global.h" + +namespace oceanbase +{ +namespace tmp_file +{ +const int64_t ObTmpFileGlobal::INVALID_TMP_FILE_FD = -1; +const int64_t ObTmpFileGlobal::INVALID_TMP_FILE_DIR_ID = -1; +const int64_t ObTmpFileGlobal::TMP_FILE_READ_BATCH_SIZE = 8 * 1024 * 1024; // 8MB +const int64_t ObTmpFileGlobal::TMP_FILE_WRITE_BATCH_PAGE_NUM = 16; +const int64_t ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX = -1; +const uint32_t ObTmpFileGlobal::INVALID_PAGE_ID = UINT32_MAX; +const int64_t ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID = -1; + +} // end namespace tmp_file +} // end namespace oceanbase diff --git a/src/storage/tmp_file/ob_tmp_file_global.h b/src/storage/tmp_file/ob_tmp_file_global.h new file mode 100644 index 000000000..f0dcb69d9 --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_global.h @@ -0,0 +1,57 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_GLOBAL_H_ +#define OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_GLOBAL_H_ +#include "deps/oblib/src/lib/ob_define.h" + +namespace oceanbase +{ +namespace tmp_file +{ +struct ObTmpFileGlobal final +{ + // TMP_FILE + static const int64_t INVALID_TMP_FILE_FD; + static const int64_t INVALID_TMP_FILE_DIR_ID; + + static const int64_t PAGE_SIZE = 8 * 1024; // 8KB + static const int64_t BLOCK_PAGE_NUMS = + OB_DEFAULT_MACRO_BLOCK_SIZE / PAGE_SIZE; // 256 pages per macro block + + static const int64_t TMP_FILE_READ_BATCH_SIZE; + static const int64_t TMP_FILE_WRITE_BATCH_PAGE_NUM; + + // TMP_FILE_BLOCK + static const int64_t INVALID_TMP_FILE_BLOCK_INDEX; + + // TMP_FILE_WRITE_BUFFER + static const uint32_t INVALID_PAGE_ID; + static const int64_t INVALID_VIRTUAL_PAGE_ID; + + // TMP_FILE_FLUSH_STAGE + enum FlushCtxState + { + FSM_F1 = 0, // flush data list L1 + FSM_F2 = 1, // flush data list L2 & L3 & L4 + FSM_F3 = 2, // flush data list L5 + FSM_F4 = 3, // flush meta list non-rightmost pages + FSM_F5 = 4, // flush meta list rightmost pages + FSM_FINISHED = 5 + }; + static const int64_t INVALID_FLUSH_SEQUENCE = -1; +}; + + +} // end namespace tmp_file +} // end namespace oceanbase +#endif // OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_GLOBAL_H_ diff --git a/src/storage/tmp_file/ob_tmp_file_io_ctx.cpp b/src/storage/tmp_file/ob_tmp_file_io_ctx.cpp new file mode 100644 index 000000000..19ed94c5b --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_io_ctx.cpp @@ -0,0 +1,497 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE + +#include "storage/tmp_file/ob_tmp_file_io_ctx.h" + +namespace oceanbase +{ +using namespace storage; +using namespace share; + +namespace tmp_file +{ +ObTmpFileIOCtx::ObTmpFileIOCtx(): + is_inited_(false), + is_read_(false), + fd_(ObTmpFileGlobal::INVALID_TMP_FILE_FD), + dir_id_(ObTmpFileGlobal::INVALID_TMP_FILE_DIR_ID), + buf_(nullptr), + buf_size_(-1), + done_size_(-1), + todo_size_(-1), + read_offset_in_file_(-1), + disable_page_cache_(false), + io_flag_(), + io_timeout_ms_(DEFAULT_IO_WAIT_TIME_MS), + io_handles_(), + page_cache_handles_() +{ + io_handles_.set_attr(ObMemAttr(MTL_ID(), "TMP_IO_HDL")); + page_cache_handles_.set_attr(ObMemAttr(MTL_ID(), "TMP_PCACHE_HDL")); +} + +ObTmpFileIOCtx::~ObTmpFileIOCtx() +{ + reset(); +} + +int ObTmpFileIOCtx::init(const int64_t fd, const int64_t dir_id, + const bool is_read, + const common::ObIOFlag io_flag, + const int64_t io_timeout_ms, + const bool disable_page_cache) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(is_inited_)) { + ret = OB_INIT_TWICE; + LOG_ERROR("ObTmpFileIOCtx init twice", KR(ret)); + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_TMP_FILE_FD == fd || + ObTmpFileGlobal::INVALID_TMP_FILE_DIR_ID == dir_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(fd), K(dir_id)); + } else if (OB_UNLIKELY(!io_flag.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(io_flag)); + } else if (OB_UNLIKELY(io_timeout_ms < 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(io_timeout_ms)); + } else { + fd_ = fd; + dir_id_ = dir_id; + is_read_ = is_read; + io_flag_ = io_flag; + io_timeout_ms_ = io_timeout_ms; + disable_page_cache_ = disable_page_cache; + is_inited_ = true; + } + return ret; +} + +void ObTmpFileIOCtx::reuse() +{ + buf_ = nullptr; + buf_size_ = -1; + done_size_ = -1; + todo_size_ = -1; + read_offset_in_file_ = -1; + for (int32_t i = 0; i < io_handles_.count(); i++) { + io_handles_.at(i).handle_.reset(); + } + for (int32_t i = 0; i < page_cache_handles_.count(); i++) { + page_cache_handles_.at(i).page_handle_.reset(); + } + io_handles_.reset(); + page_cache_handles_.reset(); +} + +void ObTmpFileIOCtx::reset() +{ + reuse(); + is_inited_ = false; + is_read_ = false; + fd_ = ObTmpFileGlobal::INVALID_TMP_FILE_FD; + dir_id_ = ObTmpFileGlobal::INVALID_TMP_FILE_DIR_ID; + disable_page_cache_ = false; + io_flag_.reset(); + io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; +} + +bool ObTmpFileIOCtx::is_valid() const +{ + return is_inited_ && + fd_ != ObTmpFileGlobal::INVALID_TMP_FILE_FD && + dir_id_ != ObTmpFileGlobal::INVALID_TMP_FILE_DIR_ID && + nullptr != buf_ && buf_size_ > 0 && done_size_ >= 0 && todo_size_ >= 0 && + (is_read_ ? read_offset_in_file_ >= 0 : true) && + io_timeout_ms_ >= 0 && io_flag_.is_valid(); +} + +int ObTmpFileIOCtx::prepare_read(char *read_buf, const int64_t read_size) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", KR(ret)); + } else if (OB_ISNULL(read_buf)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), KP(read_buf)); + } else if (OB_UNLIKELY(read_size <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(read_size)); + } else { + is_read_ = true; + buf_ = read_buf; + buf_size_ = read_size; + done_size_ = 0; + todo_size_ = read_size; + read_offset_in_file_ = -1; + } + return ret; +} + +int ObTmpFileIOCtx::prepare_read(char *read_buf, const int64_t read_size, const int64_t read_offset) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(read_offset < 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(read_offset)); + } else if (OB_FAIL(prepare_read(read_buf, read_size))) { + LOG_WARN("failed to prepare read", KR(ret), KP(read_buf), K(read_size)); + } else { + read_offset_in_file_ = read_offset; + } + return ret; +} + +int ObTmpFileIOCtx::prepare_write(char *write_buf, const int64_t write_size) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", KR(ret)); + } else if (OB_ISNULL(write_buf)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), KP(write_buf)); + } else if (OB_UNLIKELY(write_size <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(write_size)); + } else { + is_read_ = false; + buf_ = write_buf; + buf_size_ = write_size; + done_size_ = 0; + todo_size_ = write_size; + read_offset_in_file_ = -1; + } + return ret; +} + +int ObTmpFileIOCtx::update_data_size(const int64_t size) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", KR(ret)); + } else if (OB_UNLIKELY(!is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid ctx", KR(ret), KPC(this)); + } else if (OB_UNLIKELY(size > todo_size_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(size), K(todo_size_)); + } else if (OB_UNLIKELY(is_read_ && read_offset_in_file_ < 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("read offset is invalid", KR(ret), K(read_offset_in_file_)); + } else { + if (is_read_) { + read_offset_in_file_ += size; + } + done_size_ += size; + todo_size_ -= size; + } + return ret; +} + +int ObTmpFileIOCtx::wait() +{ + int ret = OB_SUCCESS; + + const int64_t timeout_ms = std::max(GCONF._data_storage_io_timeout / 1000, io_timeout_ms_); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", KR(ret)); + } else if (timeout_ms < 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("fail to wait, invalid argument, timeout must be positive", KR(ret), K(timeout_ms)); + } else if (OB_UNLIKELY(!is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid ctx", KR(ret), KPC(this)); + } else if (!is_read_) { + // due to tmp file always writes data in buffer, + // there are no asynchronous io tasks need to wait + // do nothing + } else if (OB_FAIL(wait_read_finish_(timeout_ms))) { + STORAGE_LOG(WARN, "wait read finish failed", KR(ret), K(timeout_ms), K(is_read_)); + } + + return ret; +} + +int ObTmpFileIOCtx::wait_read_finish_(const int64_t timeout_ms) +{ + int ret = OB_SUCCESS; + + if (OB_UNLIKELY(!is_read_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("the handle is prepared for writing, not allowed to wait read finish", KR(ret), KPC(this)); + } else if (OB_UNLIKELY(buf_size_ != done_size_ + todo_size_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("done_size_ + todo_size_ is not equal to buf size", KR(ret), KPC(this)); + } else if (OB_FAIL(do_read_wait_())) { + LOG_WARN("fail to wait tmp file io", KR(ret), K(fd_), K(timeout_ms)); + } + + return ret; +} + +int ObTmpFileIOCtx::do_read_wait_() +{ + int ret = OB_SUCCESS; + + for (int64_t i = 0; OB_SUCC(ret) && i < page_cache_handles_.count(); ++i) { + ObPageCacheHandle &page_cache_handle = page_cache_handles_.at(i); + if (OB_UNLIKELY(!page_cache_handle.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("page cache handle is not valid", KR(ret), K(page_cache_handle)); + } else { + const char * page_buf = page_cache_handle.page_handle_.value_->get_buffer(); + const int64_t offset_in_page = page_cache_handle.offset_in_src_data_buf_; + const int64_t read_size = page_cache_handle.read_size_; + char * read_buf = page_cache_handle.dest_user_read_buf_; + if (OB_UNLIKELY(!check_buf_range_valid(read_buf, read_size))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid range", KR(ret), K(read_buf), K(read_size), K(buf_size_)); + } else if (OB_UNLIKELY(offset_in_page + read_size > ObTmpFileGlobal::PAGE_SIZE)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("read size is over than page range", KR(ret), KPC(this), K(offset_in_page), K(read_size)); + } else { + MEMCPY(read_buf, page_buf + offset_in_page, read_size); + page_cache_handle.page_handle_.reset(); + } + } + } + if (OB_SUCC(ret)) { + page_cache_handles_.reset(); + } + + for (int64_t i = 0; OB_SUCC(ret) && i < block_cache_handles_.count(); ++i) { + ObBlockCacheHandle &block_cache_handle = block_cache_handles_.at(i); + if (OB_UNLIKELY(!block_cache_handle.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("block cache handle is not valid", KR(ret), K(block_cache_handle)); + } else { + const char * block_buf = block_cache_handle.block_handle_.value_->get_buffer(); + const int64_t offset_in_block = block_cache_handle.offset_in_src_data_buf_; + const int64_t read_size = block_cache_handle.read_size_; + char * read_buf = block_cache_handle.dest_user_read_buf_; + if (OB_UNLIKELY(!check_buf_range_valid(read_buf, read_size))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid range", KR(ret), K(read_buf), K(read_size), K(buf_size_)); + } else if (OB_UNLIKELY(offset_in_block + read_size > OB_DEFAULT_MACRO_BLOCK_SIZE)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("read size is over than macro block range", KR(ret), KPC(this), K(offset_in_block), K(read_size)); + } else { + MEMCPY(read_buf, block_buf + offset_in_block, read_size); + block_cache_handle.block_handle_.reset(); + } + } + } + if (OB_SUCC(ret)) { + block_cache_handles_.reset(); + } + + // Wait read io finish. + for (int64_t i = 0; OB_SUCC(ret) && i < io_handles_.count(); ++i) { + ObIOReadHandle &io_handle = io_handles_.at(i); + if (OB_UNLIKELY(!io_handle.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("io handle is not valid", KR(ret), K(io_handle)); + } else if (OB_FAIL(io_handle.handle_.wait())) { + LOG_WARN("fail to do object handle read wait", KR(ret)); + } else { + const char * data_buf = io_handle.handle_.get_buffer(); + const int64_t offset = io_handle.offset_in_src_data_buf_; + const int64_t size = io_handle.read_size_; + char * read_buf = io_handle.dest_user_read_buf_; + if (OB_UNLIKELY(!check_buf_range_valid(read_buf, size))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid range", KR(ret), K(read_buf), K(size), K(buf_size_)); + } else { + MEMCPY(read_buf, data_buf + offset, size); + io_handle.handle_.reset(); + } + } + } + + if (OB_SUCC(ret)) { + io_handles_.reset(); + } + + return ret; +} + +ObTmpFileIOCtx::ObIReadHandle::ObIReadHandle() + : dest_user_read_buf_(nullptr), offset_in_src_data_buf_(0), read_size_(0) +{ +} + +ObTmpFileIOCtx::ObIReadHandle::ObIReadHandle( + char *dest_user_read_buf, const int64_t offset_in_src_data_buf, const int64_t read_size) + : dest_user_read_buf_(dest_user_read_buf), offset_in_src_data_buf_(offset_in_src_data_buf), read_size_(read_size) +{ +} + +ObTmpFileIOCtx::ObIReadHandle::~ObIReadHandle() +{ +} + +ObTmpFileIOCtx::ObIReadHandle::ObIReadHandle(const ObTmpFileIOCtx::ObIReadHandle &other) +{ + *this = other; +} + +ObTmpFileIOCtx::ObIReadHandle &ObTmpFileIOCtx::ObIReadHandle::operator=( + const ObTmpFileIOCtx::ObIReadHandle &other) +{ + if (&other != this) { + offset_in_src_data_buf_ = other.offset_in_src_data_buf_; + dest_user_read_buf_ = other.dest_user_read_buf_; + read_size_ = other.read_size_; + } + return *this; +} + +ObTmpFileIOCtx::ObIOReadHandle::ObIOReadHandle() + : ObIReadHandle(), handle_() +{ +} + +ObTmpFileIOCtx::ObIOReadHandle::ObIOReadHandle( + char *dest_user_read_buf, const int64_t offset_in_src_data_buf, const int64_t read_size, + ObTmpFileBlockHandle block_handle) + : ObIReadHandle(dest_user_read_buf, offset_in_src_data_buf, read_size), handle_(), block_handle_(block_handle) +{ +} + +ObTmpFileIOCtx::ObIOReadHandle::~ObIOReadHandle() +{ +} + +ObTmpFileIOCtx::ObIOReadHandle::ObIOReadHandle(const ObTmpFileIOCtx::ObIOReadHandle &other) +{ + *this = other; +} + +ObTmpFileIOCtx::ObIOReadHandle &ObTmpFileIOCtx::ObIOReadHandle::operator=( + const ObTmpFileIOCtx::ObIOReadHandle &other) +{ + if (&other != this) { + handle_ = other.handle_; + block_handle_ = other.block_handle_; + ObIReadHandle::operator=(other); + } + return *this; +} + +bool ObTmpFileIOCtx::ObIOReadHandle::is_valid() +{ + bool bret = false; + if (OB_NOT_NULL(dest_user_read_buf_) && offset_in_src_data_buf_ >= 0 && + read_size_ >= 0 && + read_size_ <= OB_DEFAULT_MACRO_BLOCK_SIZE && + handle_.is_valid() && block_handle_.is_inited()) { + bret = true; + } + return bret; +} + +ObTmpFileIOCtx::ObBlockCacheHandle::ObBlockCacheHandle() + : ObIReadHandle(), block_handle_() +{ +} + +ObTmpFileIOCtx::ObBlockCacheHandle::ObBlockCacheHandle(const ObTmpBlockValueHandle &block_handle, + char *dest_user_read_buf, const int64_t offset_in_src_data_buf, const int64_t read_size) + : ObIReadHandle(dest_user_read_buf, offset_in_src_data_buf, read_size), block_handle_(block_handle) +{ +} + +ObTmpFileIOCtx::ObBlockCacheHandle::~ObBlockCacheHandle() +{ +} + +ObTmpFileIOCtx::ObBlockCacheHandle::ObBlockCacheHandle( + const ObTmpFileIOCtx::ObBlockCacheHandle &other) +{ + *this = other; +} + +ObTmpFileIOCtx::ObBlockCacheHandle &ObTmpFileIOCtx::ObBlockCacheHandle::operator=( + const ObTmpFileIOCtx::ObBlockCacheHandle &other) +{ + if (&other != this) { + block_handle_ = other.block_handle_; + ObIReadHandle::operator=(other); + } + return *this; +} + +bool ObTmpFileIOCtx::ObBlockCacheHandle::is_valid() +{ + bool bret = false; + if (OB_NOT_NULL(dest_user_read_buf_) && offset_in_src_data_buf_ >= 0 && + read_size_ >= 0 && + read_size_ <= OB_DEFAULT_MACRO_BLOCK_SIZE && + OB_NOT_NULL(block_handle_.value_) && + block_handle_.handle_.is_valid()) { + bret = true; + } + return bret; +} + +ObTmpFileIOCtx::ObPageCacheHandle::ObPageCacheHandle() + : ObIReadHandle(), page_handle_() +{ +} + +ObTmpFileIOCtx::ObPageCacheHandle::ObPageCacheHandle(const ObTmpPageValueHandle &page_handle, + char *dest_user_read_buf, const int64_t offset_in_src_data_buf, const int64_t read_size) + : ObIReadHandle(dest_user_read_buf, offset_in_src_data_buf, read_size), page_handle_(page_handle) +{ +} + +ObTmpFileIOCtx::ObPageCacheHandle::~ObPageCacheHandle() +{ +} + +ObTmpFileIOCtx::ObPageCacheHandle::ObPageCacheHandle( + const ObTmpFileIOCtx::ObPageCacheHandle &other) +{ + *this = other; +} + +ObTmpFileIOCtx::ObPageCacheHandle &ObTmpFileIOCtx::ObPageCacheHandle::operator=( + const ObTmpFileIOCtx::ObPageCacheHandle &other) +{ + if (&other != this) { + page_handle_ = other.page_handle_; + ObIReadHandle::operator=(other); + } + return *this; +} + +bool ObTmpFileIOCtx::ObPageCacheHandle::is_valid() +{ + bool bret = false; + if (OB_NOT_NULL(dest_user_read_buf_) && offset_in_src_data_buf_ >= 0 && + read_size_ >= 0 && + read_size_ <= ObTmpFileGlobal::PAGE_SIZE && + OB_NOT_NULL(page_handle_.value_) && + page_handle_.handle_.is_valid()) { + bret = true; + } + return bret; +} + +} // end namespace tmp_file +} // end namespace oceanbase diff --git a/src/storage/tmp_file/ob_tmp_file_io_ctx.h b/src/storage/tmp_file/ob_tmp_file_io_ctx.h new file mode 100644 index 000000000..bd985c8f1 --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_io_ctx.h @@ -0,0 +1,170 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_IO_CTX_H_ +#define OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_IO_CTX_H_ + +#include "storage/blocksstable/ob_macro_block_handle.h" +#include "storage/tmp_file/ob_tmp_file_cache.h" +#include "storage/tmp_file/ob_tmp_file_block_manager.h" + +namespace oceanbase +{ +namespace tmp_file +{ +class ObTmpFileIOCtx +{ +public: + ObTmpFileIOCtx(); + ~ObTmpFileIOCtx(); + + int init(const int64_t fd, const int64_t dir_id, + const bool is_read, + const common::ObIOFlag io_flag, + const int64_t io_timeout_ms, + const bool disable_page_cache); + void reuse(); + void reset(); + bool is_valid() const; + int prepare_read(char *read_buf, const int64_t read_size); + int prepare_read(char *read_buf, const int64_t read_size, const int64_t read_offset); + int prepare_write(char *write_buf, const int64_t write_size); + int update_data_size(const int64_t size); + int wait(); + +public: + OB_INLINE bool check_buf_range_valid(const char* buffer, const int64_t length) const + { + return buffer != nullptr && buffer >= buf_ && buffer + length <= buf_ + buf_size_; + } + OB_INLINE int64_t get_fd() const { return fd_; } + OB_INLINE int64_t get_dir_id() const { return dir_id_; } + OB_INLINE bool is_read() const { return is_read_; } + OB_INLINE char *get_buffer() { return buf_; } + OB_INLINE char *get_buffer() const { return buf_; } + OB_INLINE char *get_todo_buffer() { return buf_ + done_size_; } + OB_INLINE char *get_todo_buffer() const { return buf_ + done_size_; } + OB_INLINE int64_t get_done_size() const { return done_size_; } + OB_INLINE int64_t get_todo_size() const { return todo_size_; } + OB_INLINE int64_t get_read_offset_in_file() const { return read_offset_in_file_; } + OB_INLINE void set_read_offset_in_file(const int64_t offset) { read_offset_in_file_ = offset; } + OB_INLINE bool is_disable_page_cache() const { return disable_page_cache_; } + OB_INLINE common::ObIOFlag get_io_flag() const { return io_flag_; } + OB_INLINE int64_t get_io_timeout_ms() const { return io_timeout_ms_; } + + TO_STRING_KV(K(is_inited_), K(is_read_), + K(fd_), K(dir_id_), KP(buf_), + K(buf_size_), K(done_size_), K(todo_size_), + K(read_offset_in_file_), + K(disable_page_cache_), + K(io_flag_), K(io_timeout_ms_)); + +public: + struct ObIReadHandle + { + ObIReadHandle(); + ObIReadHandle(char *dest_user_read_buf, + const int64_t offset_in_src_data_buf, + const int64_t read_size); + ~ObIReadHandle(); + ObIReadHandle(const ObIReadHandle &other); + ObIReadHandle &operator=(const ObIReadHandle &other); + virtual bool is_valid() = 0; + TO_STRING_KV(KP(dest_user_read_buf_), K(offset_in_src_data_buf_), K(read_size_)); + + char *dest_user_read_buf_; // user buf + int64_t offset_in_src_data_buf_; + int64_t read_size_; + }; + + struct ObIOReadHandle final : public ObIReadHandle + { + ObIOReadHandle(); + ObIOReadHandle(char *dest_user_read_buf, + const int64_t offset_in_src_data_buf, const int64_t read_size, + ObTmpFileBlockHandle block_handle); + ~ObIOReadHandle(); + ObIOReadHandle(const ObIOReadHandle &other); + ObIOReadHandle &operator=(const ObIOReadHandle &other); + virtual bool is_valid() override; + INHERIT_TO_STRING_KV("ObIReadHandle", ObIReadHandle, K(handle_), K(block_handle_)); + blocksstable::ObMacroBlockHandle handle_; + ObTmpFileBlockHandle block_handle_; + }; + + struct ObBlockCacheHandle final : public ObIReadHandle + { + ObBlockCacheHandle(); + ObBlockCacheHandle(const ObTmpBlockValueHandle &block_handle_, char *dest_user_read_buf, + const int64_t offset_in_src_data_buf, + const int64_t read_size); + ~ObBlockCacheHandle(); + ObBlockCacheHandle(const ObBlockCacheHandle &other); + ObBlockCacheHandle &operator=(const ObBlockCacheHandle &other); + virtual bool is_valid() override; + INHERIT_TO_STRING_KV("ObIReadHandle", ObIReadHandle, K(block_handle_)); + ObTmpBlockValueHandle block_handle_; + }; + + struct ObPageCacheHandle final : public ObIReadHandle + { + ObPageCacheHandle(); + ObPageCacheHandle(const ObTmpPageValueHandle &page_handle, char *dest_user_read_buf, + const int64_t offset_in_src_data_buf, + const int64_t read_size); + ~ObPageCacheHandle(); + ObPageCacheHandle(const ObPageCacheHandle &other); + ObPageCacheHandle &operator=(const ObPageCacheHandle &other); + virtual bool is_valid() override; + INHERIT_TO_STRING_KV("ObIReadHandle", ObIReadHandle, K(page_handle_)); + ObTmpPageValueHandle page_handle_; + }; + + common::ObIArray &get_io_handles() + { + return io_handles_; + } + common::ObIArray &get_page_cache_handles() + { + return page_cache_handles_; + } + common::ObIArray &get_block_cache_handles() + { + return block_cache_handles_; + } + +private: + int wait_read_finish_(const int64_t timeout_ms); + int do_read_wait_(); + +private: + bool is_inited_; + bool is_read_; + int64_t fd_; + int64_t dir_id_; + char *buf_; + int64_t buf_size_; + int64_t done_size_; + int64_t todo_size_; + int64_t read_offset_in_file_; + bool disable_page_cache_; + common::ObIOFlag io_flag_; + int64_t io_timeout_ms_; + common::ObSEArray io_handles_; + common::ObSEArray page_cache_handles_; + common::ObSEArray block_cache_handles_; + DISALLOW_COPY_AND_ASSIGN(ObTmpFileIOCtx); +}; + +} // end namespace tmp_file +} // end namespace oceanbase +#endif // OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_IO_CTX_H_ diff --git a/src/storage/tmp_file/ob_tmp_file_io_define.cpp b/src/storage/tmp_file/ob_tmp_file_io_define.cpp new file mode 100644 index 000000000..de904c1b9 --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_io_define.cpp @@ -0,0 +1,236 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE + +#include "storage/tmp_file/ob_tmp_file_io_define.h" + +namespace oceanbase +{ +using namespace storage; +using namespace share; + +namespace tmp_file +{ +/* -------------------------- ObTmpFileIOInfo --------------------------- */ + +ObTmpFileIOInfo::ObTmpFileIOInfo() + : fd_(0), dir_id_(0), buf_(nullptr), size_(0), + disable_page_cache_(false), + io_desc_(), io_timeout_ms_(DEFAULT_IO_WAIT_TIME_MS) +{} + +ObTmpFileIOInfo::~ObTmpFileIOInfo() +{ + reset(); +} + +void ObTmpFileIOInfo::reset() +{ + fd_ = ObTmpFileGlobal::INVALID_TMP_FILE_FD; + dir_id_ = ObTmpFileGlobal::INVALID_TMP_FILE_DIR_ID; + size_ = 0; + io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; + buf_ = nullptr; + io_desc_.reset(); + disable_page_cache_ = false; +} + +bool ObTmpFileIOInfo::is_valid() const +{ + return fd_ != ObTmpFileGlobal::INVALID_TMP_FILE_FD && + dir_id_ != ObTmpFileGlobal::INVALID_TMP_FILE_DIR_ID && + size_ > 0 && + nullptr != buf_ && io_desc_.is_valid() && io_timeout_ms_ >= 0; +} + +/* -------------------------- ObTmpFileIOHandle --------------------------- */ + +ObTmpFileIOHandle::ObTmpFileIOHandle() + : is_inited_(false), + tmp_file_handle_(), + ctx_(), + buf_(nullptr), + update_offset_in_file_(false), + buf_size_(-1), + done_size_(-1), + read_offset_in_file_(-1) +{ +} + +ObTmpFileIOHandle::~ObTmpFileIOHandle() +{ + reset(); +} + +void ObTmpFileIOHandle::reset() +{ + is_inited_ = false; + ctx_.reset(); + tmp_file_handle_.reset(); + buf_ = nullptr; + update_offset_in_file_ = false; + buf_size_ = -1; + done_size_ = -1; + read_offset_in_file_ = -1; +} + +int ObTmpFileIOHandle::init_write(const ObTmpFileIOInfo &io_info, ObTmpFileHandle &tmp_file_handle) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObTmpFileIOHandle has been inited twice", KR(ret), KPC(this)); + } else if (OB_UNLIKELY(!io_info.is_valid()) || OB_ISNULL(tmp_file_handle.get())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(io_info)); + } else if (OB_FAIL(ctx_.init(io_info.fd_, io_info.dir_id_, false /*is_read*/, io_info.io_desc_, + io_info.io_timeout_ms_, io_info.disable_page_cache_))) { + LOG_WARN("failed to init io handle context", KR(ret), K(io_info)); + } else if (OB_FAIL(ctx_.prepare_write(io_info.buf_, io_info.size_))) { + LOG_WARN("fail to prepare write context", KR(ret), KPC(this)); + } else { + is_inited_ = true; + tmp_file_handle_ = tmp_file_handle; + buf_ = io_info.buf_; + buf_size_ = io_info.size_; + done_size_ = 0; + } + + return ret; +} + +int ObTmpFileIOHandle::init_pread(const ObTmpFileIOInfo &io_info, const int64_t read_offset, ObTmpFileHandle &tmp_file_handle) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObTmpFileIOHandle has been inited twice", KR(ret), KPC(this)); + } else if (OB_UNLIKELY(!io_info.is_valid()) || OB_ISNULL(tmp_file_handle.get())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(io_info)); + } else if (OB_UNLIKELY(read_offset < 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(read_offset)); + } else if (OB_FAIL(ctx_.init(io_info.fd_, io_info.dir_id_, true /*is_read*/, io_info.io_desc_, + io_info.io_timeout_ms_, io_info.disable_page_cache_))) { + LOG_WARN("failed to init io handle context", KR(ret), K(io_info)); + } else if (OB_FAIL(ctx_.prepare_read(io_info.buf_, MIN(io_info.size_, ObTmpFileGlobal::TMP_FILE_READ_BATCH_SIZE), read_offset))) { + LOG_WARN("fail to prepare read context", KR(ret), KPC(this), K(read_offset)); + } else { + is_inited_ = true; + tmp_file_handle_ = tmp_file_handle; + buf_ = io_info.buf_; + buf_size_ = io_info.size_; + done_size_ = 0; + read_offset_in_file_ = read_offset; + } + + return ret; +} + +int ObTmpFileIOHandle::init_read(const ObTmpFileIOInfo &io_info, ObTmpFileHandle &tmp_file_handle) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObTmpFileIOHandle has been inited twice", KR(ret), KPC(this)); + } else if (OB_UNLIKELY(!io_info.is_valid()) || OB_ISNULL(tmp_file_handle.get())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(io_info)); + } else if (OB_FAIL(ctx_.init(io_info.fd_, io_info.dir_id_, true /*is_read*/, io_info.io_desc_, + io_info.io_timeout_ms_, io_info.disable_page_cache_))) { + LOG_WARN("failed to init io handle context", KR(ret), K(io_info)); + } else if (OB_FAIL(ctx_.prepare_read(io_info.buf_, MIN(io_info.size_, ObTmpFileGlobal::TMP_FILE_READ_BATCH_SIZE)))) { + LOG_WARN("fail to prepare read context", KR(ret), KPC(this)); + } else { + is_inited_ = true; + tmp_file_handle_ = tmp_file_handle; + buf_ = io_info.buf_; + buf_size_ = io_info.size_; + done_size_ = 0; + read_offset_in_file_ = -1; + update_offset_in_file_ = true; + } + + return ret; +} + +bool ObTmpFileIOHandle::is_valid() const +{ + return is_inited_ && + nullptr != buf_ && + done_size_ >= 0 && buf_size_ > 0 && + buf_size_ >= done_size_; +} + +int ObTmpFileIOHandle::wait() +{ + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", KR(ret)); + } else if (OB_UNLIKELY(!is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid handle", KR(ret), KPC(this)); + } else if (is_finished() || !ctx_.is_read()) { + // do nothing + } else if (OB_FAIL(ctx_.wait())) { + LOG_WARN("fail to wait tmp file io", KR(ret), K(ctx_), KPC(this)); + } else if (OB_FAIL(handle_finished_ctx_(ctx_))) { + LOG_WARN("fail to handle finished ctx", KR(ret), KPC(this)); + } else if (OB_UNLIKELY(done_size_ > buf_size_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("done size is larger than total todo size", KR(ret), KPC(this)); + } else { + while (OB_SUCC(ret) && !is_finished()) { + if (OB_FAIL(ctx_.prepare_read(buf_ + done_size_, + MIN(buf_size_ - done_size_, + ObTmpFileGlobal::TMP_FILE_READ_BATCH_SIZE), + read_offset_in_file_))) { + LOG_WARN("fail to generate read ctx", KR(ret), KPC(this)); + } else if (OB_FAIL(tmp_file_handle_.get()->aio_pread(ctx_))) { + LOG_WARN("fail to continue read once batch", KR(ret), K(ctx_)); + } else if (OB_FAIL(ctx_.wait())) { + LOG_WARN("fail to wait tmp file io", KR(ret), K(ctx_)); + } else if (OB_FAIL(handle_finished_ctx_(ctx_))) { + LOG_WARN("fail to handle finished ctx", KR(ret), KPC(this)); + } + } // end while + } + + if (update_offset_in_file_ && (OB_SUCC(ret) || OB_ITER_END == ret)) { + tmp_file_handle_.get()->update_read_offset(read_offset_in_file_); + } + return ret; +} + +int ObTmpFileIOHandle::handle_finished_ctx_(ObTmpFileIOCtx &ctx) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!ctx.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(ctx)); + } else { + if (ctx_.is_read()) { + read_offset_in_file_ = ctx.get_read_offset_in_file(); + } + done_size_ += ctx.get_done_size(); + ctx.reuse(); + } + + return ret; +} + +} // end namespace tmp_file +} // end namespace oceanbase diff --git a/src/storage/tmp_file/ob_tmp_file_io_define.h b/src/storage/tmp_file/ob_tmp_file_io_define.h new file mode 100644 index 000000000..e17daecac --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_io_define.h @@ -0,0 +1,81 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_IO_DEFINE_H_ +#define OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_IO_DEFINE_H_ + +#include "storage/tmp_file/ob_shared_nothing_tmp_file.h" +#include "storage/tmp_file/ob_tmp_file_io_ctx.h" + +namespace oceanbase +{ +namespace tmp_file +{ +struct ObTmpFileIOInfo final +{ + ObTmpFileIOInfo(); + ~ObTmpFileIOInfo(); + void reset(); + bool is_valid() const; + TO_STRING_KV(K(fd_), K(dir_id_), KP(buf_), K(size_), K(disable_page_cache_), + K(io_timeout_ms_), K(io_desc_)); + + int64_t fd_; + int64_t dir_id_; + char *buf_; + int64_t size_; + bool disable_page_cache_; + common::ObIOFlag io_desc_; + int64_t io_timeout_ms_; +}; + +class ObTmpFileIOHandle final +{ +public: + ObTmpFileIOHandle(); + ~ObTmpFileIOHandle(); + int init_write(const ObTmpFileIOInfo &io_info, ObTmpFileHandle &tmp_file_handle); + int init_read(const ObTmpFileIOInfo &io_info, ObTmpFileHandle &tmp_file_handle); + int init_pread(const ObTmpFileIOInfo &io_info, const int64_t read_offset, ObTmpFileHandle &tmp_file_handle); + int wait(); + void reset(); + bool is_valid() const; + + TO_STRING_KV(K(is_inited_), K(tmp_file_handle_), K(ctx_), + KP(buf_), K(update_offset_in_file_), + K(buf_size_), K(done_size_), + K(read_offset_in_file_)); +public: + OB_INLINE char *get_buffer() { return buf_; } + OB_INLINE int64_t get_done_size() const { return done_size_; } + OB_INLINE int64_t get_buf_size() const { return buf_size_; } + OB_INLINE ObTmpFileIOCtx &get_io_ctx() { return ctx_; } + OB_INLINE bool is_finished() const { return done_size_ == buf_size_; } +private: + int handle_finished_ctx_(ObTmpFileIOCtx &ctx); + +private: + bool is_inited_; + ObTmpFileHandle tmp_file_handle_; + ObTmpFileIOCtx ctx_; + char *buf_; + bool update_offset_in_file_; + int64_t buf_size_; // excepted total read or write size + int64_t done_size_; // has finished read or write size + int64_t read_offset_in_file_; // records the beginning read offset for current read ctx + + DISALLOW_COPY_AND_ASSIGN(ObTmpFileIOHandle); +}; + +} // end namespace tmp_file +} // end namespace oceanbase +#endif // OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_IO_DEFINE_H_ diff --git a/src/storage/tmp_file/ob_tmp_file_manager.cpp b/src/storage/tmp_file/ob_tmp_file_manager.cpp new file mode 100644 index 000000000..69f4109a9 --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_manager.cpp @@ -0,0 +1,530 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE + +#include "storage/tmp_file/ob_tmp_file_manager.h" +#include "storage/tmp_file/ob_tmp_file_cache.h" + +namespace oceanbase +{ +namespace tmp_file +{ +ObTenantTmpFileManager::ObTenantTmpFileManager() + : is_inited_(false), + tenant_id_(OB_INVALID_TENANT_ID), + last_access_tenant_config_ts_(-1), + last_meta_mem_limit_(META_DEFAULT_LIMIT), + tmp_file_allocator_(), + callback_allocator_(), + wbp_index_cache_allocator_(), + wbp_index_cache_bucket_allocator_(), + files_(), + tmp_file_block_manager_(), + page_cache_controller_(tmp_file_block_manager_), + current_fd_(ObTmpFileGlobal::INVALID_TMP_FILE_FD), + current_dir_id_(ObTmpFileGlobal::INVALID_TMP_FILE_DIR_ID) +{ +} + +ObTenantTmpFileManager::~ObTenantTmpFileManager() +{ + destroy(); +} + +int ObTenantTmpFileManager::mtl_init(ObTenantTmpFileManager *&manager) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(manager)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("fail to mtl init tmp file manager, null pointer argument", KR(ret), KP(manager)); + } else if (OB_FAIL(manager->init())) { + LOG_WARN("fail to init ObTenantTmpFileManager", KR(ret)); + } + return ret; +} + +ObTenantTmpFileManager &ObTenantTmpFileManager::get_instance() +{ + int ret = OB_SUCCESS; + ObTenantTmpFileManager *tmp_file_manager = MTL(tmp_file::ObTenantTmpFileManager *); + if (OB_ISNULL(tmp_file_manager)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("ObTenantTmpFileManager is null, please check whether MTL module is normal", KR(ret)); + ob_abort(); + } + + return *tmp_file_manager; +} + +int ObTenantTmpFileManager::init() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(is_inited_)) { + ret = OB_INIT_TWICE; + LOG_WARN("ObTenantTmpFileManager init twice", KR(ret), K(is_inited_)); + } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id_ = MTL_ID()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid tenant id", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(files_.init("TmpFileMap", tenant_id_))) { + LOG_WARN("fail to init tmp files map", KR(ret)); + } else if (FALSE_IT(refresh_meta_memory_limit())) { + } else if (OB_FAIL(tmp_file_block_manager_.init(tenant_id_, last_meta_mem_limit_))) { + LOG_WARN("fail to init tenant tmp file block manager", KR(ret)); + } else if (OB_FAIL(tmp_file_allocator_.init(common::OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObModIds::OB_TMP_FILE_MANAGER, tenant_id_, + last_meta_mem_limit_))) { + LOG_WARN("fail to init tmp file allocator", KR(ret), K(tenant_id_), K(last_meta_mem_limit_)); + } else if (OB_FAIL(callback_allocator_.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(tenant_id_, "TmpFileCallback", ObCtxIds::DEFAULT_CTX_ID)))) { + LOG_WARN("fail to init callback allocator", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(wbp_index_cache_allocator_.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_NORMAL_BLOCK_SIZE, + ObMemAttr(tenant_id_, "TmpFileIndCache", + ObCtxIds::DEFAULT_CTX_ID)))) { + LOG_WARN("fail to init wbp index cache allocator", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(wbp_index_cache_bucket_allocator_.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(tenant_id_, "TmpFileIndCBkt", + ObCtxIds::DEFAULT_CTX_ID)))) { + LOG_WARN("fail to init wbp index cache bucket allocator", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(page_cache_controller_.init(*this))) { + LOG_WARN("fail to init page cache controller", KR(ret)); + } else { + is_inited_ = true; + LOG_INFO("ObTenantTmpFileManager init successful", K(tenant_id_), KP(this)); + } + return ret; +} + +int ObTenantTmpFileManager::start() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTenantTmpFileManager has not been inited", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(page_cache_controller_.start())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to start page cache controller background threads", KR(ret)); + } else { + LOG_INFO("ObTenantTmpFileManager start successful", K(tenant_id_), KP(this)); + } + return ret; +} + +void ObTenantTmpFileManager::stop() +{ + page_cache_controller_.stop(); + LOG_INFO("ObTenantTmpFileManager stop successful", K(tenant_id_), KP(this)); +} + +void ObTenantTmpFileManager::wait() +{ + page_cache_controller_.wait(); + LOG_INFO("ObTenantTmpFileManager wait successful", K(tenant_id_), KP(this)); +} + +void ObTenantTmpFileManager::destroy() +{ + last_access_tenant_config_ts_ = -1; + last_meta_mem_limit_ = META_DEFAULT_LIMIT; + page_cache_controller_.destroy(); + files_.destroy(); + tmp_file_block_manager_.destroy(); + tmp_file_allocator_.reset(); + callback_allocator_.reset(); + wbp_index_cache_allocator_.reset(); + wbp_index_cache_bucket_allocator_.reset(); + is_inited_ = false; + current_fd_ = ObTmpFileGlobal::INVALID_TMP_FILE_FD; + current_dir_id_ = ObTmpFileGlobal::INVALID_TMP_FILE_DIR_ID; + + LOG_INFO("ObTenantTmpFileManager destroy", K(tenant_id_), KP(this)); +} + +int ObTenantTmpFileManager::alloc_dir(int64_t &dir_id) +{ + int ret = OB_SUCCESS; + dir_id = ObTmpFileGlobal::INVALID_TMP_FILE_DIR_ID; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTenantTmpFileManager has not been inited", KR(ret), K(tenant_id_)); + } else { + dir_id = ATOMIC_AAF(¤t_dir_id_, 1); + } + + LOG_DEBUG("alloc dir over", KR(ret), K(dir_id), K(lbt())); + return ret; +} + +int ObTenantTmpFileManager::open(int64_t &fd, const int64_t &dir_id) +{ + int ret = OB_SUCCESS; + fd = ObTmpFileGlobal::INVALID_TMP_FILE_FD; + void *buf = nullptr; + ObSharedNothingTmpFile *tmp_file = nullptr; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTenantTmpFileManager has not been inited", KR(ret), K(tenant_id_)); + } else if (OB_ISNULL(buf = tmp_file_allocator_.alloc(sizeof(ObSharedNothingTmpFile), + lib::ObMemAttr(tenant_id_, "SNTmpFile")))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate memory for tmp file", + KR(ret), K(tenant_id_), K(sizeof(ObSharedNothingTmpFile))); + } else if (FALSE_IT(tmp_file = new (buf) ObSharedNothingTmpFile())) { + } else if (FALSE_IT(fd = ATOMIC_AAF(¤t_fd_, 1))) { + } else if (OB_FAIL(tmp_file->init(tenant_id_, fd, dir_id, + &tmp_file_block_manager_, &callback_allocator_, + &wbp_index_cache_allocator_, &wbp_index_cache_bucket_allocator_, + &page_cache_controller_))) { + LOG_WARN("fail to init tmp file", KR(ret), K(fd), K(dir_id)); + } else if (OB_FAIL(files_.insert(ObTmpFileKey(fd), tmp_file))) { + LOG_WARN("fail to set refactored to tmp file map", KR(ret), K(fd), KP(tmp_file)); + } + + if (OB_FAIL(ret) && OB_NOT_NULL(tmp_file)) { + tmp_file->~ObSharedNothingTmpFile(); + tmp_file_allocator_.free(tmp_file); + tmp_file = nullptr; + } + + LOG_INFO("open a tmp file over", KR(ret), K(fd), K(dir_id), KP(tmp_file), K(lbt())); + return ret; +} + +int ObTenantTmpFileManager::remove(const int64_t fd) +{ + int ret = OB_SUCCESS; + ObTmpFileHandle tmp_file_handle; + int64_t start_remove_ts = ObTimeUtility::current_time(); + LOG_INFO("remove a tmp file start", KR(ret), K(start_remove_ts), K(fd), K(lbt())); + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTenantTmpFileManager has not been inited", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(files_.erase(ObTmpFileKey(fd), tmp_file_handle))) { + if (OB_ENTRY_NOT_EXIST == ret) { + ret = OB_SUCCESS; + LOG_INFO("erase non-exist tmp file", K(fd), K(lbt())); + } else { + LOG_WARN("fail to erase tmp file", KR(ret), K(fd), K(lbt())); + } + } else if (OB_ISNULL(tmp_file_handle.get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", KR(ret), KP(tmp_file_handle.get()), K(fd)); + } else if (OB_FAIL(tmp_file_handle.get()->delete_file())) { + LOG_WARN("fail to delete tmp file", KR(ret), K(fd), K(lbt())); + } else { + ObSharedNothingTmpFile *tmp_file = tmp_file_handle.get(); + + int64_t LOG_WARN_TIMEOUT = 10 * 1000 * 1000; + int64_t LOG_ERROR_TIMEOUT = 60 * 1000 * 1000; + while(!tmp_file->can_remove()) + { + if (start_remove_ts + LOG_ERROR_TIMEOUT < ObTimeUtility::current_time()) { + LOG_ERROR("wait thread release reference too long", + K(start_remove_ts), KP(tmp_file), KPC(tmp_file), K(lbt())); + sleep(10); // 10s + } else if (start_remove_ts + LOG_WARN_TIMEOUT < ObTimeUtility::current_time()) { + LOG_WARN("wait thread release reference too long", + K(start_remove_ts), KP(tmp_file), KPC(tmp_file), K(lbt())); + sleep(2); // 2s + } else { + usleep(100 * 1000); // 100ms + } + } + tmp_file_handle.reset(); + tmp_file_allocator_.free(tmp_file); + } + + LOG_INFO("remove a tmp file over", KR(ret), K(start_remove_ts), K(fd), K(lbt())); + return ret; +} + +void ObTenantTmpFileManager::refresh_meta_memory_limit() +{ + int ret = OB_SUCCESS; + const int64_t last_access_ts = ATOMIC_LOAD(&last_access_tenant_config_ts_); + int64_t mem_limit = META_DEFAULT_LIMIT; + + if (last_access_ts < 0 || common::ObClockGenerator::getClock() - last_access_ts > REFRESH_CONFIG_INTERVAL) { + omt::ObTenantConfigGuard config(TENANT_CONF(tenant_id_)); + const int64_t tenant_mem_limit = lib::get_tenant_memory_limit(tenant_id_); + if (!config.is_valid() || 0 == tenant_mem_limit || INT64_MAX == tenant_mem_limit) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get tenant config or tenant memory", KR(ret), K(tenant_id_), K(tenant_mem_limit)); + } else { + const int64_t last_memory_limit = ATOMIC_LOAD(&last_meta_mem_limit_); + const int64_t limit_percentage_config = config->_temporary_file_meta_memory_limit_percentage; + const int64_t limit_percentage = 0 == limit_percentage_config ? 70 : limit_percentage_config; + mem_limit = tenant_mem_limit * limit_percentage / 100; + if (OB_UNLIKELY(mem_limit <= 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("memory limit isn't more than 0", KR(ret), K(mem_limit)); + } else if (FALSE_IT(ATOMIC_STORE(&last_access_tenant_config_ts_, common::ObClockGenerator::getClock()))) { + } else if (mem_limit != last_memory_limit) { + ATOMIC_STORE(&last_meta_mem_limit_, mem_limit); + tmp_file_allocator_.set_total_limit(mem_limit); + tmp_file_block_manager_.get_block_allocator().set_total_limit(mem_limit); + share::ObTaskController::get().allow_next_syslog(); + LOG_INFO("succeed to refresh tmp file meta memory limit", + K(tenant_id_), K(last_memory_limit), K(mem_limit), KP(this)); + } + } + } +} + +int ObTenantTmpFileManager::aio_read(const ObTmpFileIOInfo &io_info, ObTmpFileIOHandle &io_handle) +{ + int ret = OB_SUCCESS; + ObTmpFileHandle tmp_file_handle; + io_handle.reset(); + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTenantTmpFileManager has not been inited", KR(ret), K(tenant_id_)); + } else if (!io_info.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("fail to aio read, invalid argument", KR(ret), K(io_info)); + } else if (OB_FAIL(get_tmp_file(io_info.fd_, tmp_file_handle))) { + LOG_WARN("fail to get tmp file io handle", KR(ret), K(io_info)); + } else if (OB_FAIL(io_handle.init_read(io_info, tmp_file_handle))) { + LOG_WARN("fail to init io handle", KR(ret), K(io_info), K(tmp_file_handle)); + } else if (OB_FAIL(tmp_file_handle.get()->aio_pread(io_handle.get_io_ctx()))) { + LOG_WARN("fail to aio pread", KR(ret), K(io_info)); + } + + LOG_DEBUG("aio_read a tmp file over", KR(ret), K(io_info), K(io_handle), KPC(tmp_file_handle.get())); + return ret; +} + +int ObTenantTmpFileManager::aio_pread(const ObTmpFileIOInfo &io_info, + const int64_t offset, + ObTmpFileIOHandle &io_handle) +{ + int ret = OB_SUCCESS; + ObTmpFileHandle tmp_file_handle; + io_handle.reset(); + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTenantTmpFileManager has not been inited", KR(ret), K(tenant_id_)); + } else if (!io_info.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("fail to aio read, invalid argument", KR(ret), K(io_info)); + } else if (OB_FAIL(get_tmp_file(io_info.fd_, tmp_file_handle))) { + LOG_WARN("fail to get tmp file io handle", KR(ret), K(io_info)); + } else if (OB_FAIL(io_handle.init_pread(io_info, offset, tmp_file_handle))) { + LOG_WARN("fail to init io handle", KR(ret), K(io_info), K(tmp_file_handle)); + } else if (OB_FAIL(tmp_file_handle.get()->aio_pread(io_handle.get_io_ctx()))) { + LOG_WARN("fail to aio pread", KR(ret), K(io_info)); + } + + LOG_DEBUG("aio_pread a tmp file over", KR(ret), K(io_info), K(offset), K(io_handle), KPC(tmp_file_handle.get())); + return ret; +} + +int ObTenantTmpFileManager::read(const ObTmpFileIOInfo &io_info, ObTmpFileIOHandle &io_handle) +{ + int ret = OB_SUCCESS; + ObTmpFileHandle tmp_file_handle; + io_handle.reset(); + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTenantTmpFileManager has not been inited", KR(ret), K(tenant_id_)); + } else if (!io_info.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("fail to aio read, invalid argument", KR(ret), K(io_info)); + } else if (OB_FAIL(get_tmp_file(io_info.fd_, tmp_file_handle))) { + LOG_WARN("fail to get tmp file io handle", KR(ret), K(io_info)); + } else if (OB_FAIL(io_handle.init_read(io_info, tmp_file_handle))) { + LOG_WARN("fail to init io handle", KR(ret), K(io_info), K(tmp_file_handle)); + } else if (OB_FAIL(tmp_file_handle.get()->aio_pread(io_handle.get_io_ctx()))) { + LOG_WARN("fail to aio pread", KR(ret), K(io_info)); + } + + if (OB_SUCC(ret) || OB_ITER_END == ret) { + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(io_handle.wait())) { + LOG_WARN("fail to wait", KR(tmp_ret), K(io_info)); + } + ret = OB_SUCCESS == ret ? tmp_ret : ret; + } + + LOG_DEBUG("read a tmp file over", KR(ret), K(io_info), K(io_handle), KPC(tmp_file_handle.get())); + return ret; +} + +int ObTenantTmpFileManager::pread(const ObTmpFileIOInfo &io_info, const int64_t offset, ObTmpFileIOHandle &io_handle) +{ + int ret = OB_SUCCESS; + ObTmpFileHandle tmp_file_handle; + io_handle.reset(); + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTenantTmpFileManager has not been inited", KR(ret), K(tenant_id_)); + } else if (!io_info.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("fail to aio read, invalid argument", KR(ret), K(io_info)); + } else if (OB_FAIL(get_tmp_file(io_info.fd_, tmp_file_handle))) { + LOG_WARN("fail to get tmp file io handle", KR(ret), K(io_info)); + } else if (OB_FAIL(io_handle.init_pread(io_info, offset, tmp_file_handle))) { + LOG_WARN("fail to init io handle", KR(ret), K(io_info), K(tmp_file_handle)); + } else if (OB_FAIL(tmp_file_handle.get()->aio_pread(io_handle.get_io_ctx()))) { + LOG_WARN("fail to aio pread", KR(ret), K(io_info)); + } + + if (OB_SUCC(ret) || OB_ITER_END == ret) { + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(io_handle.wait())) { + LOG_WARN("fail to wait", KR(tmp_ret), K(io_info)); + } + ret = OB_SUCCESS == ret ? tmp_ret : ret; + } + + LOG_DEBUG("pread a tmp file over", KR(ret), K(io_info), K(offset), K(io_handle), KPC(tmp_file_handle.get())); + return ret; +} + +int ObTenantTmpFileManager::aio_write(const ObTmpFileIOInfo &io_info, ObTmpFileIOHandle &io_handle) +{ + int ret = OB_SUCCESS; + ObTmpFileHandle tmp_file_handle; + io_handle.reset(); + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTenantTmpFileManager has not been inited", KR(ret), K(tenant_id_)); + } else if (!io_info.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(io_info)); + } else if (OB_FAIL(get_tmp_file(io_info.fd_, tmp_file_handle))) { + LOG_WARN("fail to get tmp file io handle", KR(ret), K(io_info)); + } else if (OB_FAIL(io_handle.init_write(io_info, tmp_file_handle))) { + LOG_WARN("fail to init io handle", KR(ret), K(io_info), K(tmp_file_handle)); + } else if (OB_FAIL(tmp_file_handle.get()->aio_write(io_handle.get_io_ctx()))) { + LOG_WARN("fail to aio write", KR(ret), K(io_info)); + } + + LOG_DEBUG("aio_write a tmp file over", KR(ret), K(io_info), K(io_handle), KPC(tmp_file_handle.get())); + return ret; +} + +// tmp file is always buffer writing, there are no io tasks need to be waited +int ObTenantTmpFileManager::write(const ObTmpFileIOInfo &io_info) +{ + int ret = OB_SUCCESS; + ObTmpFileIOHandle io_handle; + + if (OB_FAIL(aio_write(io_info, io_handle))) { + LOG_WARN("fail to aio write", KR(ret), K(io_info)); + } + + LOG_DEBUG("write a tmp file over", KR(ret), K(io_info), K(io_handle)); + return ret; +} + +// attention: +// currently truncate() only release memory, but not release disk space. +// we will support to release disk space in future. +int ObTenantTmpFileManager::truncate(const int64_t fd, const int64_t offset) +{ + int ret = OB_SUCCESS; + ObTmpFileHandle tmp_file_handle; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTenantTmpFileManager has not been inited", KR(ret), K(tenant_id_)); + } else if (OB_UNLIKELY(offset < 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(offset)); + } else if (OB_FAIL(get_tmp_file(fd, tmp_file_handle))) { + LOG_WARN("fail to get tmp file handle", KR(ret), K(fd)); + } else if (OB_FAIL(tmp_file_handle.get()->truncate(offset))) { + LOG_WARN("fail to truncate", KR(ret), K(fd), K(offset)); + } else { + LOG_INFO("truncate a tmp file over", KR(ret), K(fd), K(offset)); + } + return ret; +} + +// Get tmp file and increase its refcnt +int ObTenantTmpFileManager::get_tmp_file(const int64_t fd, ObTmpFileHandle &file_handle) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(files_.get(ObTmpFileKey(fd), file_handle))) { + if (OB_HASH_NOT_EXIST == ret) { + LOG_WARN("tmp file does not exist", KR(ret), K(fd)); + } else { + LOG_WARN("fail to get tmp file", KR(ret), K(fd)); + } + } else if (OB_ISNULL(file_handle.get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get invalid tmp file pointer", KR(ret), K(fd), KP(file_handle.get())); + } + + return ret; +} + +int ObTenantTmpFileManager::get_tmp_file_size(const int64_t fd, int64_t &size) +{ + int ret = OB_SUCCESS; + ObTmpFileHandle tmp_file_handle; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTenantTmpFileManager has not been inited", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(get_tmp_file(fd, tmp_file_handle))) { + LOG_WARN("fail to get tmp file handle", KR(ret), K(fd)); + } else { + size = tmp_file_handle.get()->get_file_size(); + } + + LOG_DEBUG("get tmp file size", KR(ret), K(fd), K(size)); + return ret; +} + +int ObTenantTmpFileManager::get_macro_block_list(common::ObIArray ¯o_id_list) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTenantTmpFileManager has not been inited", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(tmp_file_block_manager_.get_macro_block_list(macro_id_list))) { + LOG_WARN("fail to get macro block id list", KR(ret)); + } + + LOG_INFO("get tmp file macro block list", KR(ret), K(macro_id_list.count())); + return ret; +} + +int ObTenantTmpFileManager::get_macro_block_count(int64_t ¯o_block_count) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTenantTmpFileManager has not been inited", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(tmp_file_block_manager_.get_macro_block_count(macro_block_count))) { + LOG_WARN("fail to get macro block id count", KR(ret)); + } + + LOG_INFO("get tmp file macro block count", KR(ret), K(macro_block_count)); + return ret; +} + +} // end namespace tmp_file +} // end namespace oceanbase diff --git a/src/storage/tmp_file/ob_tmp_file_manager.h b/src/storage/tmp_file/ob_tmp_file_manager.h new file mode 100644 index 000000000..2d9353288 --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_manager.h @@ -0,0 +1,111 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_MANAGER_H_ +#define OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_MANAGER_H_ + +#include "lib/hash/ob_linear_hash_map.h" +#include "lib/container/ob_array.h" +#include "lib/lock/ob_spin_rwlock.h" +#include "storage/blocksstable/ob_macro_block_id.h" +#include "storage/tmp_file/ob_shared_nothing_tmp_file.h" +#include "storage/tmp_file/ob_tmp_file_io_define.h" +#include "storage/tmp_file/ob_tmp_file_block_manager.h" +#include "storage/tmp_file/ob_tmp_file_eviction_manager.h" +#include "storage/tmp_file/ob_tmp_file_page_cache_controller.h" + +namespace oceanbase +{ +namespace tmp_file +{ +struct ObTmpFileKey final +{ + explicit ObTmpFileKey(const int64_t fd) : fd_(fd) {} + OB_INLINE int hash(uint64_t &hash_val) const + { + hash_val = murmurhash(&fd_, sizeof(int64_t), 0); + return OB_SUCCESS; + } + OB_INLINE bool operator==(const ObTmpFileKey &other) const { return fd_ == other.fd_; } + TO_STRING_KV(K(fd_)); + int64_t fd_; +}; + +class ObTenantTmpFileManager final +{ +public: + typedef common::ObLinearHashMap TmpFileMap; +public: + ObTenantTmpFileManager(); + ~ObTenantTmpFileManager(); + static int mtl_init(ObTenantTmpFileManager *&manager); + static ObTenantTmpFileManager &get_instance(); + int init(); + int start(); + void stop(); + void wait(); + void destroy(); + + int alloc_dir(int64_t &dir_id); + int open(int64_t &fd, const int64_t &dir_id); + int remove(const int64_t fd); + + void refresh_meta_memory_limit(); + +public: + int aio_read(const ObTmpFileIOInfo &io_info, ObTmpFileIOHandle &io_handle); + int aio_pread(const ObTmpFileIOInfo &io_info, const int64_t offset, ObTmpFileIOHandle &io_handle); + int read(const ObTmpFileIOInfo &io_info, ObTmpFileIOHandle &io_handle); + int pread(const ObTmpFileIOInfo &io_info, const int64_t offset, ObTmpFileIOHandle &io_handle); + // NOTE: + // only support append write. + int aio_write(const ObTmpFileIOInfo &io_info, ObTmpFileIOHandle &io_handle); + // NOTE: + // only support append write. + int write(const ObTmpFileIOInfo &io_info); + int truncate(const int64_t fd, const int64_t offset); + +public: + int get_tmp_file(const int64_t fd, ObTmpFileHandle &file_handle); + int get_tmp_file_size(const int64_t fd, int64_t &size); + int get_macro_block_list(common::ObIArray ¯o_id_list); + int get_macro_block_count(int64_t ¯o_block_count); + OB_INLINE ObIAllocator * get_callback_allocator() { return &callback_allocator_; } + OB_INLINE ObTmpFileBlockManager &get_tmp_file_block_manager() { return tmp_file_block_manager_; } + OB_INLINE ObTmpFilePageCacheController &get_page_cache_controller() { return page_cache_controller_; } + +private: + static const int64_t REFRESH_CONFIG_INTERVAL = 5 * 60 * 1000 * 1000L; // 5min + static const int64_t META_DEFAULT_LIMIT = 15 * 1024L * 1024L * 1024L; + +private: + bool is_inited_; + uint64_t tenant_id_; + int64_t last_access_tenant_config_ts_; + int64_t last_meta_mem_limit_; + common::ObConcurrentFIFOAllocator tmp_file_allocator_; + common::ObFIFOAllocator callback_allocator_; + common::ObFIFOAllocator wbp_index_cache_allocator_; + common::ObFIFOAllocator wbp_index_cache_bucket_allocator_; + TmpFileMap files_; + ObTmpFileBlockManager tmp_file_block_manager_; + ObTmpFilePageCacheController page_cache_controller_; + + int64_t current_fd_; + int64_t current_dir_id_; +}; + +#define FILE_MANAGER_INSTANCE_V2 (::oceanbase::tmp_file::ObTenantTmpFileManager::get_instance()) +} // end namespace tmp_file +} // end namespace oceanbase + +#endif // OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_MANAGER_H_ diff --git a/src/storage/tmp_file/ob_tmp_file_meta_tree.cpp b/src/storage/tmp_file/ob_tmp_file_meta_tree.cpp new file mode 100644 index 000000000..b49a59868 --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_meta_tree.cpp @@ -0,0 +1,3139 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "ob_tmp_file_meta_tree.h" +#include "storage/tmp_file/ob_tmp_file_cache.h" +#include "storage/tmp_file/ob_tmp_file_manager.h" + +namespace oceanbase +{ +using namespace storage; +using namespace share; + +namespace tmp_file +{ + +const uint16_t ObSharedNothingTmpFileMetaTree::PAGE_MAGIC_NUM = 0xa12f; +const int16_t ObSharedNothingTmpFileMetaTree::PAGE_HEADER_SIZE = 16; //16B +int16_t ObSharedNothingTmpFileMetaTree::MAX_DATA_ITEM_ARRAY_COUNT = 16; + +//only for unittest +int16_t ObSharedNothingTmpFileMetaTree::MAX_PAGE_ITEM_COUNT = INT16_MAX; + +ObSharedNothingTmpFileMetaTree::ObSharedNothingTmpFileMetaTree() : + is_inited_(false), + fd_(-1), + tree_epoch_(0), + root_item_(), + data_item_array_(), + level_page_range_array_(), + is_writing_(false), + lock_(common::ObLatchIds::TMP_FILE_LOCK), + last_truncate_leaf_info_(), + released_offset_(0), + stat_info_() +{ + STATIC_ASSERT(sizeof(struct ObSharedNothingTmpFileMetaItem) == 24, "size of tree meta item is mismatch"); + STATIC_ASSERT(sizeof(struct ObSharedNothingTmpFileDataItem) == 24, "size of tree data item is mismatch"); + STATIC_ASSERT(sizeof(struct ObSharedNothingTmpFileTreePageHeader) == 16, "size of tree page header is mismatch"); +} + +ObSharedNothingTmpFileMetaTree::~ObSharedNothingTmpFileMetaTree() +{ + reset(); +} + +void ObSharedNothingTmpFileMetaTree::reset() +{ + is_inited_ = false; + fd_ = -1; + wbp_ = NULL; + tree_epoch_ = 0, + root_item_.reset(); + data_item_array_.reset(); + level_page_range_array_.reset(); + is_writing_ = false; + last_truncate_leaf_info_.reset(); + released_offset_ = 0; + stat_info_.reset(); +} + +int ObSharedNothingTmpFileMetaTree::init(const int64_t fd, + ObTmpWriteBufferPool *wbp, + ObIAllocator *callback_allocator) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(wbp)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), KP(wbp)); + } else { + data_item_array_.set_attr(ObMemAttr(MTL_ID(), "TFDataItemArr")); + level_page_range_array_.set_attr(ObMemAttr(MTL_ID(), "TFTreeLevelArr")); + fd_ = fd; + wbp_ = wbp; + callback_allocator_ = callback_allocator; + is_inited_ = true; + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::prepare_for_insert_items() +{ + int ret = OB_SUCCESS; + SpinWLockGuard guard(lock_); + if (!root_item_.is_valid()) { + //do nothing + } else { + ObSharedNothingTmpFileMetaItem page_info; + if (OB_FAIL(get_rightmost_leaf_page_for_write_(page_info))) { + STORAGE_LOG(WARN, "fail to get rightmost leaf page for write", KR(ret), KPC(this)); + } + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::get_rightmost_leaf_page_for_write_( + ObSharedNothingTmpFileMetaItem &page_info) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(is_writing_ + || level_page_range_array_.empty())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected is_writing_ or level_array_", KR(ret), K(fd_), K(is_writing_), K(level_page_range_array_)); + } else { + is_writing_ = true; //need to be protected by lock_ + ObSharedNothingTmpFileMetaItem meta_item = root_item_; + uint32_t parent_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ObSharedNothingTmpFileMetaItem next_level_meta_item; + while (OB_SUCC(ret) + && 0 < meta_item.page_level_) { + next_level_meta_item.reset(); + if (OB_FAIL(get_last_item_of_internal_page_(parent_page_id, meta_item, next_level_meta_item))) { + STORAGE_LOG(WARN, "fail to get last item of internal page", KR(ret), K(fd_), K(parent_page_id), K(meta_item)); + } else { + parent_page_id = meta_item.buffer_page_id_; + meta_item = next_level_meta_item; + } + } + if (OB_SUCC(ret)) { + if (OB_UNLIKELY(0 != meta_item.page_level_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected page_level_", KR(ret), K(meta_item)); + } else if (OB_FAIL(cache_page_for_write_(parent_page_id, meta_item))) { + STORAGE_LOG(WARN, "fail to cache page for write", KR(ret), K(fd_), K(parent_page_id), K(meta_item)); + } else { + page_info = meta_item; + } + } + + //there is no need to remove the successfully loaded rightmost page from the write cache + if (OB_FAIL(ret)) { + is_writing_ = false; + } + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::get_last_item_of_internal_page_( + const uint32_t parent_page_id, + ObSharedNothingTmpFileMetaItem &page_info, + ObSharedNothingTmpFileMetaItem &last_meta_item) +{ + int ret = OB_SUCCESS; + ObSharedNothingTmpFileTreePageHeader page_header; + if (OB_FAIL(cache_page_for_write_(parent_page_id, page_info))) { + STORAGE_LOG(WARN, "fail to cache page for write", KR(ret), K(fd_), K(parent_page_id), K(page_info)); + } else { + char* page_buff = NULL; + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + int32_t level_page_index = level_page_range_array_[page_info.page_level_].evicted_page_num_ + + level_page_range_array_[page_info.page_level_].cached_page_num_ - 1; + ObTmpFilePageUniqKey page_key(page_info.page_level_, level_page_index); + if (OB_FAIL(wbp_->read_page(fd_, page_info.buffer_page_id_, page_key, page_buff, next_page_id))) { + STORAGE_LOG(WARN, "fail to read from write cache", KR(ret), K(fd_), K(page_info), K(page_key), K(level_page_range_array_)); + } else if (OB_FAIL(read_page_header_(page_buff, page_header))) { + STORAGE_LOG(WARN, "fail to read page header", KR(ret), K(fd_), KP(page_buff)); + } else if (OB_UNLIKELY(0 >= page_header.item_num_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected item_num", KR(ret), K(page_header)); + } else if (OB_FAIL(read_item_(page_buff, page_header.item_num_ - 1, last_meta_item))) { + STORAGE_LOG(WARN, "fail to read item", KR(ret), K(fd_), KP(page_buff), K(page_header)); + } + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::insert_items( + const ObIArray &data_items) +{ + int ret = OB_SUCCESS; + SpinWLockGuard guard(lock_); + if (OB_UNLIKELY(data_items.empty())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(data_items)); + } else if (OB_UNLIKELY(!data_items.at(0).is_valid() || + released_offset_ > data_items.at(0).virtual_page_id_ * ObTmpFileGlobal::PAGE_SIZE)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(data_items.at(0)), KPC(this)); + } else { + int64_t write_count = 0; + const int64_t data_item_count = data_items.count(); + ObSEArray level_origin_page_write_counts; + ObSEArray, 2> level_new_pages; + if (!root_item_.is_valid() && OB_FAIL(try_to_insert_items_to_array_(data_items, level_new_pages))) { + STORAGE_LOG(WARN, "fail to try to insert items to array", KR(ret), K(data_items), KPC(this)); + } else if (root_item_.is_valid()) { + //we set is_writing_ in previous step, + // so, we need not to worry about end pages of each level will be evicted. + if (OB_UNLIKELY(!is_writing_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected is_writing_", KR(ret), KPC(this)); + } else if (OB_FAIL(try_to_fill_rightmost_leaf_page_(data_items, level_new_pages, write_count, + level_origin_page_write_counts))) { + STORAGE_LOG(WARN, "fail to try to fill rightmost leaf page", KR(ret), K(data_items), K(level_new_pages), KPC(this)); + } else if (write_count < data_item_count) { + ObSEArray meta_items; + ObSharedNothingTmpFileMetaItem meta_item; + if (level_page_range_array_.count() <= 1) { + //there is only a root page + if (OB_FAIL(meta_items.push_back(root_item_))) { + STORAGE_LOG(WARN, "fail to push back", KR(ret), K(fd_), K(root_item_)); + } + } + //alloc new leaf pages + while (OB_SUCC(ret) + && write_count < data_item_count) { + meta_item.reset(); + if (OB_FAIL(add_new_page_and_fill_items_at_leaf_(data_items, write_count, meta_item, level_new_pages))) { + STORAGE_LOG(WARN, "fail to add new page and fill items at leaf", KR(ret), + K(data_items), K(write_count), K(level_new_pages), KPC(this)); + } else if (OB_FAIL(meta_items.push_back(meta_item))) { + STORAGE_LOG(WARN, "fail to push back", KR(ret), K(fd_), K(meta_item)); + } + } + //cascade to modify the internal pages + if (FAILEDx(cascade_modification_at_internal_(meta_items, + level_origin_page_write_counts, level_new_pages))) { + STORAGE_LOG(WARN, "fail to cascade modification at internal", KR(ret), K(meta_items), + K(level_origin_page_write_counts), K(level_new_pages), KPC(this)); + } + } + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(finish_insert_(ret, level_origin_page_write_counts, level_new_pages))) { + STORAGE_LOG(WARN, "fail to finish insert", KR(tmp_ret), KR(ret), + K(level_origin_page_write_counts), K(level_new_pages), KPC(this)); + } + if (OB_SUCC(ret)) { + ret = tmp_ret; + } + if (OB_SUCC(ret)) { + is_writing_ = false; + } + } + } + if (OB_SUCC(ret)) { + ARRAY_FOREACH_N(data_items, i, cnt) { + stat_info_.all_type_page_flush_cnt_ += data_items.at(i).physical_page_num_; + } + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::try_to_insert_items_to_array_( + const ObIArray &data_items, + ObIArray> &level_new_pages) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(data_items.empty() + || !level_page_range_array_.empty() + || MAX_DATA_ITEM_ARRAY_COUNT > MAX_PAGE_ITEM_COUNT + || is_writing_)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), K(data_items), K(level_page_range_array_), + K(MAX_DATA_ITEM_ARRAY_COUNT), K(MAX_PAGE_ITEM_COUNT), K(is_writing_)); + } else if (!data_item_array_.empty()) { + const ObSharedNothingTmpFileDataItem &last_item = data_item_array_.at(data_item_array_.count() - 1); + if (OB_UNLIKELY(data_items.at(0).virtual_page_id_ != last_item.virtual_page_id_ + last_item.physical_page_num_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected data_items or data_item_array_", KR(ret), K(fd_), K(data_items), K(last_item)); + } + } + if (OB_SUCC(ret)) { + bool need_build_tree = false; + int16_t push_cnt = 0; + if (data_item_array_.count() + data_items.count() <= MAX_DATA_ITEM_ARRAY_COUNT) { + ARRAY_FOREACH_N(data_items, i, cnt) { + if (OB_UNLIKELY(!data_items.at(i).is_valid())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), K(data_items.at(i))); + } else if (OB_FAIL(data_item_array_.push_back(data_items.at(i)))) { + STORAGE_LOG(WARN, "fail to push back", KR(ret), K(fd_), K(data_items.at(i))); + if (OB_ALLOCATE_MEMORY_FAILED == ret) { + need_build_tree = true; + } + } else { + push_cnt++; + } + } + if (OB_FAIL(ret)) { + for (int16_t i = 0; i < push_cnt; i++) { + data_item_array_.pop_back(); + } + } + } else { + need_build_tree = true; + } + if (true == need_build_tree) { + ret = OB_SUCCESS; + uint32_t new_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + char *new_page_buff = NULL; + int16_t count = 0; + ObTmpFilePageUniqKey leaf_page_offset(0/*tree level*/, 0/*level page index*/); + if (OB_FAIL(wbp_->alloc_page(fd_, leaf_page_offset, new_page_id, new_page_buff))) { + STORAGE_LOG(WARN, "fail to alloc page from write cache", KR(ret), K(fd_), K(leaf_page_offset)); + } else if (OB_FAIL(init_page_header_(new_page_buff, 0 /*level*/))) { + STORAGE_LOG(WARN, "fail to init page header", KR(ret), K(fd_), KP(new_page_buff)); + } else if (!data_item_array_.empty() && + OB_FAIL(write_items_(new_page_buff, data_item_array_, 0/*begin_index*/, count))) { + STORAGE_LOG(WARN, "fail to write items", KR(ret), K(fd_), K(data_item_array_), KP(new_page_buff)); + } else if (OB_FAIL(wbp_->notify_dirty(fd_, new_page_id, leaf_page_offset))) { + STORAGE_LOG(WARN, "fail to notify dirty for meta", KR(ret), K(fd_), K(new_page_id), K(leaf_page_offset)); + } else { + is_writing_ = true; //must be protected by lock + ObSEArray new_pages; + if (OB_FAIL(new_pages.push_back(new_page_id))) { + STORAGE_LOG(WARN, "fail to push back", KR(ret), K(fd_), K(new_page_id)); + } else if (OB_FAIL(level_new_pages.push_back(new_pages))) { + STORAGE_LOG(WARN, "fail to push back", KR(ret), K(fd_), K(new_pages)); + } else { + root_item_.page_level_ = 0; + root_item_.buffer_page_id_ = new_page_id; + root_item_.virtual_page_id_ = data_item_array_.empty() ? + data_items.at(0).virtual_page_id_ : data_item_array_.at(0).virtual_page_id_; + //print page content + ObSharedNothingTmpFileTreePageHeader tmp_page_header; + ObSharedNothingTmpFileDataItem first_item; + ObSharedNothingTmpFileDataItem last_item; + read_page_simple_content_(new_page_buff, tmp_page_header, first_item, last_item); + STORAGE_LOG(INFO, "dump tree page", KR(ret), K(fd_), K(new_page_id), K(leaf_page_offset), + KP(new_page_buff), K(tmp_page_header), K(first_item), K(last_item)); + } + } + } + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::try_to_fill_rightmost_leaf_page_( + const ObIArray &data_items, + const ObIArray> &level_new_pages, + int64_t &write_count, + ObIArray &level_origin_page_write_counts) +{ + int ret = OB_SUCCESS; + write_count = 0; + if (OB_UNLIKELY(data_items.empty() || !data_items.at(0).is_valid() + || (level_new_pages.empty() && level_page_range_array_.empty()))) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), K(data_items), K(level_new_pages), K(level_page_range_array_)); + } else { + const int16_t MAX_PAGE_DATA_ITEM_NUM = + MIN(MAX_PAGE_ITEM_COUNT, (ObTmpFileGlobal::PAGE_SIZE - PAGE_HEADER_SIZE) / sizeof(data_items.at(0))); + uint32_t page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + int32_t level_page_index = -1; + int16_t count = 0; + ObSharedNothingTmpFileTreePageHeader page_header; + char *leaf_page_buff = NULL; + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + if (!level_new_pages.empty()) { + page_id = level_new_pages.at(0/*level*/).at(0); + level_page_index = 0; + } else { + page_id = level_page_range_array_.at(0/*level*/).end_page_id_; + level_page_index = level_page_range_array_.at(0).evicted_page_num_ + level_page_range_array_.at(0).cached_page_num_ - 1; + } + ObTmpFilePageUniqKey leaf_page_offset(0, level_page_index); + if (OB_FAIL(wbp_->read_page(fd_, page_id, leaf_page_offset, leaf_page_buff, next_page_id))) { + STORAGE_LOG(WARN, "fail to read from write cache", KR(ret), K(fd_), K(page_id), K(leaf_page_offset), + K(level_new_pages), K(level_page_range_array_)); + } else if (OB_FAIL(read_page_header_(leaf_page_buff, page_header))) { + STORAGE_LOG(WARN, "fail to read header", KR(ret), K(fd_), KP(leaf_page_buff)); + } else if (0 == page_header.item_num_) { + //maybe the item have been cleared (corresponds to an unfilled data page) + //or maybe this is a newly allocated meta page + //so "0 == item_num" + //but we do not need to rewrite page info (change virtual_page_id) + STORAGE_LOG(INFO, "item_num is 0", KR(ret), K(fd_), KP(leaf_page_buff), K(page_id)); + } else { + //check last data item + ObSharedNothingTmpFileDataItem origin_last_item; + if (OB_FAIL(read_item_(leaf_page_buff, page_header.item_num_ - 1, origin_last_item))) { + STORAGE_LOG(WARN, "fail to read item", KR(ret), K(fd_), KP(leaf_page_buff), K(page_header)); + } else if (OB_UNLIKELY(data_items.at(0).virtual_page_id_ != origin_last_item.virtual_page_id_ + origin_last_item.physical_page_num_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected data_items or origin_last_item", KR(ret), K(fd_), K(data_items), K(origin_last_item)); + } + } + if (OB_SUCC(ret) && page_header.item_num_ < MAX_PAGE_DATA_ITEM_NUM) { + if (OB_FAIL(write_items_(leaf_page_buff, data_items, 0, count))) { + STORAGE_LOG(WARN, "fail to write items", KR(ret), K(fd_), KP(leaf_page_buff), K(data_items)); + } else if (OB_FAIL(wbp_->notify_dirty(fd_, page_id, leaf_page_offset))) { + STORAGE_LOG(WARN, "fail to notify dirty for meta", KR(ret), K(fd_), K(page_id), K(leaf_page_offset)); + } else { + write_count += count; + } + if (OB_SUCC(ret)) { + //print page content + ObSharedNothingTmpFileTreePageHeader tmp_page_header; + ObSharedNothingTmpFileDataItem first_item; + ObSharedNothingTmpFileDataItem last_item; + read_page_simple_content_(leaf_page_buff, tmp_page_header, first_item, last_item); + STORAGE_LOG(DEBUG, "dump tree page", KR(ret), K(fd_), K(page_id), K(leaf_page_offset), + KP(leaf_page_buff), K(tmp_page_header), K(first_item), K(last_item)); + } + } + if (OB_SUCC(ret) && !level_page_range_array_.empty() + && OB_FAIL(level_origin_page_write_counts.push_back(count))) { + STORAGE_LOG(WARN, "fail to push back", KR(ret), K(fd_), K(count)); + } + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::add_new_page_and_fill_items_at_leaf_( + const ObIArray &data_items, + int64_t &write_count, + ObSharedNothingTmpFileMetaItem &meta_item, + ObIArray> &level_new_pages) +{ + int ret = OB_SUCCESS; + meta_item.reset(); + uint32_t new_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + char *new_page_buff = NULL; + int16_t count = 0; + int32_t level_page_index = 0; + if (!level_page_range_array_.empty()) { + level_page_index += (level_page_range_array_.at(0).evicted_page_num_ + + level_page_range_array_.at(0).cached_page_num_); + } + if (!level_new_pages.empty()) { + level_page_index += level_new_pages.at(0).count(); + } + ObTmpFilePageUniqKey leaf_page_offset(0, level_page_index); + if (OB_UNLIKELY(!level_new_pages.empty() && 1 != level_new_pages.count())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), K(level_new_pages)); + } else if (OB_UNLIKELY(data_items.count() <= write_count || !data_items.at(write_count).is_valid())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), K(data_items), K(write_count)); + } else if (OB_FAIL(wbp_->alloc_page(fd_, leaf_page_offset, new_page_id, new_page_buff))) { + STORAGE_LOG(WARN, "fail to alloc page from write cache", KR(ret), K(fd_), K(leaf_page_offset), + K(level_new_pages), K(level_page_range_array_)); + } else if (OB_FAIL(init_page_header_(new_page_buff, 0 /*level*/))) { + STORAGE_LOG(WARN, "fail to init page header", KR(ret), K(fd_), KP(new_page_buff)); + } else if (OB_FAIL(write_items_(new_page_buff, data_items, write_count/*begin_index*/, count))) { + STORAGE_LOG(WARN, "fail to write items", KR(ret), K(fd_), KP(new_page_buff), K(data_items), K(write_count)); + } else if (OB_FAIL(wbp_->notify_dirty(fd_, new_page_id, leaf_page_offset))) { + STORAGE_LOG(WARN, "fail to notify dirty for meta", KR(ret), K(fd_), K(new_page_id), K(leaf_page_offset)); + } else { + meta_item.page_level_ = 0; + meta_item.buffer_page_id_ = new_page_id; + meta_item.virtual_page_id_ = data_items.at(write_count).virtual_page_id_; + write_count += count; + if (level_new_pages.empty()) { + ObSEArray new_pages; + if (OB_FAIL(level_new_pages.push_back(new_pages))) { + STORAGE_LOG(WARN, "fail to push back", KR(ret), K(fd_), K(new_pages)); + } + } + if (FAILEDx(level_new_pages.at(0).push_back(new_page_id))) { + STORAGE_LOG(WARN, "fail to push back", KR(ret), K(fd_), K(new_page_id)); + } + } + if (OB_SUCC(ret)) { + //print page content + ObSharedNothingTmpFileTreePageHeader tmp_page_header; + ObSharedNothingTmpFileDataItem first_item; + ObSharedNothingTmpFileDataItem last_item; + read_page_simple_content_(new_page_buff, tmp_page_header, first_item, last_item); + STORAGE_LOG(INFO, "dump tree page", KR(ret), K(fd_), K(new_page_id), K(leaf_page_offset), + KP(new_page_buff), K(tmp_page_header), K(first_item), K(last_item)); + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::cascade_modification_at_internal_( + const ObIArray &new_leaf_page_infos, + ObIArray &level_origin_page_write_counts, + ObIArray> &level_new_pages) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(new_leaf_page_infos.empty() + || level_new_pages.empty())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), K(new_leaf_page_infos), K(level_new_pages)); + } else { + ObSEArray prev_meta_items; + ObSEArray meta_items; + ObSharedNothingTmpFileMetaItem meta_item; + const int16_t origin_level_count = level_page_range_array_.count(); + int16_t cur_level = 1; + //TODO: time limitation in while loop + while (OB_SUCC(ret)) { + int64_t write_count = 0; + prev_meta_items.reset(); + if (cur_level > origin_level_count && meta_items.count() == 1) { + //we need to change root_item_ + break; + } else if (cur_level == 1 && OB_FAIL(prev_meta_items.assign(new_leaf_page_infos))) { + STORAGE_LOG(WARN, "fail to assign", KR(ret), K(fd_), K(new_leaf_page_infos)); + } else if (cur_level > 1 && OB_FAIL(prev_meta_items.assign(meta_items))) { + STORAGE_LOG(WARN, "fail to assign", KR(ret), K(fd_), K(meta_items)); + } else if (cur_level < origin_level_count && + OB_FAIL(try_to_fill_rightmost_internal_page_(prev_meta_items, cur_level, write_count, + level_origin_page_write_counts))) { + STORAGE_LOG(WARN, "fail to try to fill rightmost internal page", KR(ret), K(fd_), K(prev_meta_items), + K(cur_level), K(level_origin_page_write_counts)); + } else { + int64_t meta_item_count = prev_meta_items.count(); + if (write_count < meta_item_count) { + meta_items.reset(); + if (origin_level_count - 1 == cur_level) { + if (OB_FAIL(meta_items.push_back(root_item_))) { + STORAGE_LOG(WARN, "fail to push back", KR(ret), K(fd_), K(root_item_)); + } + } + while (OB_SUCC(ret) && write_count < meta_item_count) { + meta_item.reset(); + if (OB_FAIL(add_new_page_and_fill_items_at_internal_(prev_meta_items, cur_level, write_count, + meta_item, level_new_pages))) { + STORAGE_LOG(WARN, "fail to add new page and fill items at internal", KR(ret), K(fd_), + K(prev_meta_items), K(cur_level), K(write_count)); + } else if (OB_FAIL(meta_items.push_back(meta_item))) { + STORAGE_LOG(WARN, "fail to push_back", KR(ret), K(fd_), K(meta_item)); + } + } + } else { + //cur level does not alloc new pages + break; + } + } + cur_level++; + } + if (OB_SUCC(ret)) { + if (OB_UNLIKELY(origin_level_count == level_new_pages.count())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected level_new_pages", KR(ret), K(fd_), K(level_new_pages), K(origin_level_count)); + } else if (origin_level_count < level_new_pages.count()) { + if (OB_UNLIKELY(1 != meta_items.count() + || 1 != level_new_pages.at(level_new_pages.count() - 1).count())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected meta_items or level_new_pages", KR(ret), K(fd_), K(meta_items), + K(level_new_pages), K(origin_level_count)); + } else { + root_item_ = meta_items.at(0); + } + } + } + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::try_to_fill_rightmost_internal_page_( + const ObIArray &meta_items, + const int16_t page_level, + int64_t &write_count, + ObIArray &level_origin_page_write_counts) +{ + int ret = OB_SUCCESS; + write_count = 0; + //only a writing thread, array count will not change + if (OB_UNLIKELY(meta_items.empty() + || page_level < 1 + || page_level >= level_page_range_array_.count())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), K(meta_items), K(page_level), K(level_page_range_array_)); + } else { + const int16_t MAX_PAGE_META_ITEM_NUM = + MIN(MAX_PAGE_ITEM_COUNT, (ObTmpFileGlobal::PAGE_SIZE - PAGE_HEADER_SIZE) / sizeof(meta_items.at(0))); + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + char *internal_page_buff = NULL; + ObSharedNothingTmpFileTreePageHeader page_header; + int16_t count = 0; + uint32_t page_id = level_page_range_array_[page_level].end_page_id_; + int32_t level_page_index = level_page_range_array_[page_level].evicted_page_num_ + + level_page_range_array_[page_level].cached_page_num_ - 1; + ObTmpFilePageUniqKey internal_page_offset(page_level, level_page_index); + //only a writing thread. + if (OB_FAIL(wbp_->read_page(fd_, page_id, internal_page_offset, internal_page_buff, next_page_id))) { + STORAGE_LOG(WARN, "fail to read from write cache", KR(ret), K(fd_), K(page_id), K(internal_page_offset)); + } else if (OB_FAIL(read_page_header_(internal_page_buff, page_header))) { + STORAGE_LOG(WARN, "fail to read page header", KR(ret), K(fd_), KP(internal_page_buff)); + } else if (page_header.item_num_ < MAX_PAGE_META_ITEM_NUM) { + if (page_header.item_num_ <= 0) { + //the rightmost page in internal level must has items + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected item_num", KR(ret), K(fd_), K(page_header)); + } else if (OB_FAIL(write_items_(internal_page_buff, meta_items, 0, count))) { + STORAGE_LOG(WARN, "fail to write items", KR(ret), K(fd_), KP(internal_page_buff), K(meta_items)); + } else if (OB_FAIL(wbp_->notify_dirty(fd_, page_id, internal_page_offset))) { + STORAGE_LOG(WARN, "fail to notify dirty for meta", KR(ret), K(fd_), K(page_id), K(internal_page_offset)); + } else { + write_count += count; + //print page content + ObSharedNothingTmpFileTreePageHeader tmp_page_header; + ObSharedNothingTmpFileMetaItem first_item; + ObSharedNothingTmpFileMetaItem last_item; + read_page_simple_content_(internal_page_buff, tmp_page_header, first_item, last_item); + STORAGE_LOG(INFO, "dump tree page", KR(ret), K(fd_), K(page_id), K(internal_page_offset), + KP(internal_page_buff), K(tmp_page_header), K(first_item), K(last_item)); + } + } + if (FAILEDx(level_origin_page_write_counts.push_back(count))) { + STORAGE_LOG(WARN, "fail to push back", KR(ret), K(fd_), K(count)); + } + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::add_new_page_and_fill_items_at_internal_( + const ObIArray &meta_items, + const int16_t page_level, + int64_t &write_count, + ObSharedNothingTmpFileMetaItem &meta_item, + ObIArray> &level_new_pages) +{ + int ret = OB_SUCCESS; + uint32_t new_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + char *new_page_buff = NULL; + int16_t count = 0; + int32_t level_page_index = 0; + if (page_level < level_page_range_array_.count()) { + level_page_index += (level_page_range_array_.at(page_level).evicted_page_num_ + + level_page_range_array_.at(page_level).cached_page_num_); + } + if (page_level < level_new_pages.count()) { + level_page_index += level_new_pages.at(page_level).count(); + } + ObTmpFilePageUniqKey internal_page_offset(page_level, level_page_index); + if (OB_UNLIKELY(meta_items.count() <= write_count || !meta_items.at(write_count).is_valid())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), K(meta_items), K(write_count)); + } else if (OB_FAIL(wbp_->alloc_page(fd_, internal_page_offset, new_page_id, new_page_buff))) { + STORAGE_LOG(WARN, "fail to alloc page from write cache", KR(ret), K(fd_), K(internal_page_offset)); + } else if (OB_FAIL(init_page_header_(new_page_buff, page_level))) { + STORAGE_LOG(WARN, "fail to init page header", KR(ret), K(fd_), KP(new_page_buff)); + } else if (OB_FAIL(write_items_(new_page_buff, meta_items, write_count/*begin_index*/, count))) { + STORAGE_LOG(WARN, "fail to write items", KR(ret), K(fd_), KP(new_page_buff), K(meta_items), K(write_count)); + } else if (OB_FAIL(wbp_->notify_dirty(fd_, new_page_id, internal_page_offset))) { + STORAGE_LOG(WARN, "fail to notify dirty for meta", KR(ret), K(fd_), K(new_page_id), K(internal_page_offset)); + } else { + meta_item.page_level_ = page_level; + meta_item.buffer_page_id_ = new_page_id; + meta_item.virtual_page_id_ = meta_items.at(write_count).virtual_page_id_; + write_count += count; + if (OB_UNLIKELY(level_new_pages.count() != page_level + && level_new_pages.count() != page_level + 1)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected level_new_pages", KR(ret), K(fd_), K(page_level), K(level_new_pages)); + } else if (level_new_pages.count() == page_level) { + ObSEArray new_pages; + if (OB_FAIL(level_new_pages.push_back(new_pages))) { + STORAGE_LOG(WARN, "fail to push back", KR(ret), K(fd_), K(new_pages)); + } + } + if (FAILEDx(level_new_pages.at(page_level).push_back(new_page_id))) { + STORAGE_LOG(WARN, "fail to push back", KR(ret), K(fd_), K(new_page_id)); + } + } + if (OB_SUCC(ret)) { + //print page content + ObSharedNothingTmpFileTreePageHeader tmp_page_header; + ObSharedNothingTmpFileMetaItem first_item; + ObSharedNothingTmpFileMetaItem last_item; + read_page_simple_content_(new_page_buff, tmp_page_header, first_item, last_item); + STORAGE_LOG(INFO, "dump tree page", KR(ret), K(fd_), K(new_page_id), K(internal_page_offset), + KP(new_page_buff), K(tmp_page_header), K(first_item), K(last_item)); + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::finish_insert_( + const int return_ret, + ObIArray &level_origin_page_write_counts, + ObIArray> &level_new_pages) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(level_origin_page_write_counts.count() > level_page_range_array_.count())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected new record level page infos", KR(ret), K(fd_), + K(level_origin_page_write_counts), K(level_page_range_array_)); + } else if (OB_SUCCESS == return_ret) { + ARRAY_FOREACH_N(level_new_pages, level, level_cnt) { + ARRAY_FOREACH_N(level_new_pages.at(level), i, new_page_cnt) { + const uint32_t new_page_id = level_new_pages.at(level).at(i); + if (0 == i && level_page_range_array_.count() <= level) { + if (OB_FAIL(level_page_range_array_.push_back( + LevelPageRangeInfo(new_page_id, ObTmpFileGlobal::INVALID_PAGE_ID, new_page_id, 1, 0, 0)))) { + STORAGE_LOG(WARN, "fail to push back", KR(ret), K(fd_), K(new_page_id)); + } + } else if (OB_UNLIKELY(level_page_range_array_.count() <= level)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected level_page_range_array_", KR(ret), K(fd_), K(level_page_range_array_), K(level)); + } else { + uint32_t end_page_id_in_array = level_page_range_array_[level].end_page_id_; + int32_t level_prev_page_index = level_page_range_array_[level].evicted_page_num_ + + level_page_range_array_[level].cached_page_num_ - 1; + if (ObTmpFileGlobal::INVALID_PAGE_ID != end_page_id_in_array + && OB_FAIL(wbp_->link_page(fd_, new_page_id, end_page_id_in_array, + ObTmpFilePageUniqKey(level, level_prev_page_index)))) { + STORAGE_LOG(WARN, "fail to link page in write cache", KR(ret), K(fd_), K(level_page_range_array_), + K(new_page_id), K(level)); + } else { + if (ObTmpFileGlobal::INVALID_PAGE_ID == end_page_id_in_array) { + level_page_range_array_[level].start_page_id_ = new_page_id; + } + level_page_range_array_[level].end_page_id_ = new_page_id; + level_page_range_array_[level].cached_page_num_++; + } + } + } + } + if (OB_SUCC(ret) && !data_item_array_.empty()) { + data_item_array_.reset(); + } + } else { //fail + //rollback + ARRAY_FOREACH_N(level_origin_page_write_counts, i, cnt) { + const int16_t remove_cnt = level_origin_page_write_counts.at(i); + if (0 != remove_cnt) { + char *page_buff = NULL; + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + uint32_t end_page_id = level_page_range_array_[i].end_page_id_; + int32_t level_page_index = level_page_range_array_[i].evicted_page_num_ + + level_page_range_array_[i].cached_page_num_ - 1; + ObTmpFilePageUniqKey page_key(i, level_page_index); + if (OB_FAIL(wbp_->read_page(fd_, end_page_id, page_key, page_buff, next_page_id))) { + STORAGE_LOG(WARN, "fail to read from write cache", KR(ret), K(fd_), K(end_page_id), K(page_key)); + } else if (OB_FAIL(remove_page_item_from_tail_(page_buff, remove_cnt))) { + STORAGE_LOG(WARN, "fail to remove page item from tail", KR(ret), K(fd_), KP(page_buff), K(remove_cnt)); + } + } + } + ARRAY_FOREACH_N(level_new_pages, level, level_cnt) { + int32_t level_page_index = 0; + if (level_page_range_array_.count() > level) { + level_page_index += (level_page_range_array_[level].evicted_page_num_ + + level_page_range_array_[level].cached_page_num_); + } + ARRAY_FOREACH_N(level_new_pages.at(level), i, new_page_cnt) { + uint32_t unused_next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + const uint32_t new_page_id = level_new_pages.at(level).at(i); + ObTmpFilePageUniqKey page_key(level, level_page_index + i); + if (OB_FAIL(wbp_->free_page(fd_, new_page_id, page_key, unused_next_page_id))) { + STORAGE_LOG(WARN, "fail to free meta page in write cache", KR(ret), K(fd_), K(new_page_id), K(page_key)); + } + } + } + if (OB_SUCC(ret)) { + if (level_page_range_array_.empty()) { + root_item_.reset(); + } + is_writing_ = false; + } + STORAGE_LOG(INFO, "fail to insert, finish rollback to before", KR(ret), K(return_ret), KPC(this)); + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::search_data_items( + const int64_t start_offset, + const int64_t read_size, + ObIArray &data_items) +{ + int ret = OB_SUCCESS; + data_items.reset(); + const int16_t FULL_PAGE_META_ITEM_NUM = + MIN(MAX_PAGE_ITEM_COUNT, (ObTmpFileGlobal::PAGE_SIZE - PAGE_HEADER_SIZE) / sizeof(ObSharedNothingTmpFileMetaItem)); + const int64_t end_offset = start_offset + read_size; + int64_t offset = start_offset; + SpinRLockGuard guard(lock_); + if (OB_UNLIKELY(start_offset < released_offset_)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(start_offset), KPC(this)); + } else if (!root_item_.is_valid()) { + //read from data_item_array_ + if (OB_FAIL(search_data_items_from_array_(end_offset, offset, data_items))) { + STORAGE_LOG(WARN, "fail to get data items from array", KR(ret), K(end_offset), K(offset), KPC(this)); + } + } else { + const int64_t target_virtual_page_id = start_offset / ObTmpFileGlobal::PAGE_SIZE; + ObSharedNothingTmpFileMetaItem meta_item = root_item_; + ObSharedNothingTmpFileMetaItem next_level_meta_item; + //read from meta tree + //root page index must be 0 + int32_t level_page_index = 0; + //we use a array to simulate a stack + //TODO: 初始值的设置 + ObSEArray search_path; + while (OB_SUCC(ret) + && 0 < meta_item.page_level_) { + next_level_meta_item.reset(); + int16_t item_index = -1; + ObSharedNothingTmpFileTreePageHeader page_header; + char *page_buff = NULL; + ObTmpPageValueHandle p_handle; + if (OB_FAIL(get_page_(meta_item, level_page_index, page_buff, p_handle))) { + STORAGE_LOG(WARN, "fail to get page", KR(ret), K(meta_item), K(level_page_index), KPC(this)); + } else if (OB_FAIL(read_item_(page_buff, target_virtual_page_id, item_index, next_level_meta_item))) { + STORAGE_LOG(WARN, "fail to read item", KR(ret), KP(page_buff), + K(target_virtual_page_id), K(meta_item), K(level_page_index), KPC(this)); + } else if (OB_FAIL(search_path.push_back(BacktraceNode(meta_item /*page info*/, + level_page_index, /*page index in level*/ + item_index /*item index on the page*/)))) { + STORAGE_LOG(WARN, "fail to push back", KR(ret), K(fd_), K(meta_item), K(level_page_index), K(item_index)); + } else { + meta_item = next_level_meta_item; + level_page_index = level_page_index * FULL_PAGE_META_ITEM_NUM + item_index; + } + p_handle.reset(); + } + if (OB_SUCC(ret)) { + if (0 == meta_item.page_level_) { + if (OB_FAIL(get_items_of_leaf_page_(meta_item, level_page_index, end_offset, + true /*need read from specified item index*/, offset, data_items))) { + STORAGE_LOG(WARN, "fail to get items of leaf page", KR(ret), K(meta_item), + K(level_page_index), K(end_offset), K(offset), KPC(this)); + } else if (offset < end_offset + && OB_FAIL(backtrace_search_data_items_(end_offset, offset, search_path, data_items))) { + STORAGE_LOG(WARN, "fail to backtrace search data items", KR(ret), + K(end_offset), K(offset), K(search_path), KPC(this)); + } + } else { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected page type", KR(ret), K(meta_item), KPC(this)); + } + } + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::search_data_items_from_array_( + const int64_t end_offset, + int64_t &cur_offset, + common::ObIArray &data_items) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(cur_offset < 0 + || cur_offset >= end_offset + || data_item_array_.empty() + || data_item_array_.count() > MAX_DATA_ITEM_ARRAY_COUNT)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), K(cur_offset), K(end_offset), K(data_item_array_)); + } else { + const int64_t target_virtual_page_id = cur_offset / ObTmpFileGlobal::PAGE_SIZE; + int16_t index = -1; + ARRAY_FOREACH_N(data_item_array_, i, cnt) { + const ObSharedNothingTmpFileDataItem &data_item = data_item_array_.at(i); + if (OB_UNLIKELY(!data_item.is_valid())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected data_item", KR(ret), K(fd_), K(i), K(data_item)); + } else if (data_item.virtual_page_id_ <= target_virtual_page_id) { + index = i; + } else { + break; + } + } + if (OB_UNLIKELY(0 > index)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected index", KR(ret), K(fd_), K(index), K(cur_offset), K(data_item_array_)); + } else { + for (int16_t i = index; OB_SUCC(ret) && i < data_item_array_.count() && cur_offset < end_offset; i++) { + const ObSharedNothingTmpFileDataItem &data_item = data_item_array_.at(i); + if (OB_UNLIKELY(i > index && cur_offset != data_item.virtual_page_id_ * ObTmpFileGlobal::PAGE_SIZE)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected virtual_page_id", KR(ret), K(fd_), K(cur_offset), K(i), K(index), K(data_item)); + } else if (OB_FAIL(data_items.push_back(data_item))) { + STORAGE_LOG(WARN, "fail to push back", KR(ret), K(fd_), K(data_item)); + } else { + cur_offset = (data_item.virtual_page_id_ + data_item.physical_page_num_) * ObTmpFileGlobal::PAGE_SIZE; + } + } + } + if (OB_SUCC(ret) && cur_offset < end_offset) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected offset", KR(ret), K(fd_), K(cur_offset), K(end_offset)); + } + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::get_items_of_leaf_page_( + const ObSharedNothingTmpFileMetaItem &page_info, + const int32_t level_page_index, + const int64_t end_offset, + const bool need_find_index, + int64_t &cur_offset, + common::ObIArray &data_items) +{ + int ret = OB_SUCCESS; + char *page_buff = NULL; + int16_t item_index = -1; + int64_t tmp_offset = -1; + const int64_t target_virtual_page_id = cur_offset / ObTmpFileGlobal::PAGE_SIZE; + ObSharedNothingTmpFileDataItem data_item; + ObSharedNothingTmpFileTreePageHeader page_header; + ObTmpPageValueHandle p_handle; + if (OB_FAIL(get_page_(page_info, level_page_index, page_buff, p_handle))) { + STORAGE_LOG(WARN, "fail to get page", KR(ret), K(fd_), K(page_info), K(level_page_index)); + } else if (OB_FAIL(read_page_header_(page_buff, page_header))) { + STORAGE_LOG(WARN, "fail to read page header", KR(ret), K(fd_), KP(page_buff)); + } else { + if (!need_find_index) { + if (OB_UNLIKELY(0 != cur_offset % ObTmpFileGlobal::PAGE_SIZE)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected cur_offset", KR(ret), K(fd_), K(cur_offset)); + } else { + //read from beginning of the page + item_index = 0; + } + } else { + //get the specified item_index based on target_virtual_page_id + if (OB_FAIL(read_item_(page_buff, target_virtual_page_id, item_index, data_item))) { + STORAGE_LOG(WARN, "fail to read item", KR(ret), K(fd_), KP(page_buff), K(target_virtual_page_id)); + } else if (FALSE_IT(tmp_offset = (data_item.virtual_page_id_ + data_item.physical_page_num_) * ObTmpFileGlobal::PAGE_SIZE)) { + } else if (cur_offset >= tmp_offset) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected offset", KR(ret), K(fd_), K(cur_offset), K(tmp_offset), K(data_item)); + } else if (OB_FAIL(data_items.push_back(data_item))) { + STORAGE_LOG(WARN, "fail to push back", KR(ret), K(fd_), K(data_item)); + } else { + cur_offset = tmp_offset; + item_index++; + } + } + if (OB_SUCC(ret)) { + while (OB_SUCC(ret) + && cur_offset < end_offset + && item_index < page_header.item_num_) { + data_item.reset(); + if (OB_FAIL(read_item_(page_buff, item_index, data_item))) { + STORAGE_LOG(WARN, "fail to read item", KR(ret), K(fd_), KP(page_buff), K(item_index), K(page_info), K(level_page_index)); + } else if (OB_UNLIKELY(cur_offset != data_item.virtual_page_id_ * ObTmpFileGlobal::PAGE_SIZE)) { + ret = OB_ERR_UNEXPECTED; + //print page content + //NOTE: control print frequence + ObSharedNothingTmpFileTreePageHeader tmp_page_header; + ObArray items; + read_page_content_(page_buff, tmp_page_header, items); + STORAGE_LOG(WARN, "unexpected virtual_page_id, dump tree page", KR(ret), K(fd_), KP(page_buff), K(cur_offset), + K(item_index), K(data_item), K(target_virtual_page_id), K(tmp_page_header), K(items)); + } else if (OB_FAIL(data_items.push_back(data_item))) { + STORAGE_LOG(WARN, "fail to push back", KR(ret), K(fd_), K(data_item)); + } else { + cur_offset = (data_item.virtual_page_id_ + data_item.physical_page_num_) * ObTmpFileGlobal::PAGE_SIZE; + item_index++; + } + } + } + if (OB_FAIL(ret)) { + //print page content + ObSharedNothingTmpFileTreePageHeader tmp_page_header; + ObSharedNothingTmpFileDataItem first_item; + ObSharedNothingTmpFileDataItem last_item; + read_page_simple_content_(page_buff, page_header, first_item, last_item); + STORAGE_LOG(WARN, "dump tree page", KR(ret), K(fd_), K(page_info), K(level_page_index), + KP(page_buff), K(tmp_page_header), K(first_item), K(last_item)); + } + } + p_handle.reset(); + return ret; +} + +int ObSharedNothingTmpFileMetaTree::backtrace_search_data_items_( + const int64_t end_offset, + int64_t &offset, + ObIArray &search_path, + ObIArray &data_items) +{ + int ret = OB_SUCCESS; + //TODO: check last node in search_path must be level_1 page info + if (OB_UNLIKELY(search_path.empty() + || data_items.empty() + || offset >= end_offset)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), K(search_path), K(data_items), + K(end_offset), K(offset)); + } else { + const int16_t FULL_PAGE_META_ITEM_NUM = + MIN(MAX_PAGE_ITEM_COUNT, (ObTmpFileGlobal::PAGE_SIZE - PAGE_HEADER_SIZE) / sizeof(ObSharedNothingTmpFileMetaItem)); + ObSEArray meta_items; + while (OB_SUCC(ret) + && !search_path.empty() + && offset < end_offset) { + meta_items.reset(); + bool reach_last_item = false; + int64_t last_node_index = search_path.count() - 1; + //meta_item points to a page + const ObSharedNothingTmpFileMetaItem &page_info = search_path.at(last_node_index).page_meta_info_; + //the page index in its level + const int32_t level_page_index = search_path.at(last_node_index).level_page_index_; + //current pos need to be processed on this page. + int16_t cur_item_index = search_path.at(last_node_index).prev_item_index_ + 1; + if (OB_FAIL(get_items_of_internal_page_(page_info, level_page_index, cur_item_index, + end_offset, reach_last_item, meta_items))) { + STORAGE_LOG(WARN, "fail to get items of internal page", KR(ret), K(fd_), K(page_info), + K(level_page_index), K(cur_item_index), K(end_offset)); + } else if (1 == page_info.page_level_) { + //process level_0 + int64_t i = 0; + const int64_t cnt = meta_items.count(); + const int32_t start_leaf_level_page_index = level_page_index * FULL_PAGE_META_ITEM_NUM + cur_item_index; + for (; OB_SUCC(ret) && i < cnt && offset < end_offset; i++) { + const ObSharedNothingTmpFileMetaItem &leaf_page_info = meta_items.at(i); + bool need_lock = ((i + 1 == cnt) && reach_last_item); + if (OB_FAIL(get_items_of_leaf_page_(leaf_page_info, start_leaf_level_page_index + i, end_offset, + false /*need read from specified item index*/, offset, data_items))) { + STORAGE_LOG(WARN, "fail to get items of leaf page", KR(ret), K(fd_), K(leaf_page_info), + K(start_leaf_level_page_index + i), K(end_offset), K(need_lock), K(offset)); + } + } + if (OB_SUCC(ret)) { + if (OB_UNLIKELY(cnt != i || (!reach_last_item && offset < end_offset))) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected meta_items", KR(ret), K(fd_), K(meta_items), K(i), + K(reach_last_item), K(offset), K(end_offset)); + } else { + search_path.pop_back(); + } + } + } else if (page_info.page_level_ > 1) { + if (meta_items.count() == 1) { + const int32_t child_level_page_index = level_page_index * FULL_PAGE_META_ITEM_NUM + cur_item_index; + search_path.at(last_node_index).prev_item_index_ = cur_item_index; + if (OB_FAIL(search_path.push_back(BacktraceNode(meta_items.at(0), child_level_page_index, -1)))) { + STORAGE_LOG(WARN, "fail to push back", KR(ret), K(fd_), K(meta_items.at(0)), K(child_level_page_index)); + } + } else if (meta_items.empty()) { + search_path.pop_back(); + } else { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected meta_items", KR(ret), K(fd_), K(meta_items)); + } + } else { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected page info", KR(ret), K(fd_), K(page_info)); + } + } + if (OB_SUCC(ret) && end_offset > offset) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected offset", KR(ret), K(fd_), K(end_offset), K(offset)); + } + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::get_items_of_internal_page_( + const ObSharedNothingTmpFileMetaItem &page_info, + const int32_t level_page_index, + const int16_t item_index, + const int64_t end_offset, + bool &reach_last_item, + ObIArray &meta_items) +{ + int ret = OB_SUCCESS; + reach_last_item = false; + meta_items.reset(); + int16_t cur_item_index = item_index; + ObSharedNothingTmpFileTreePageHeader page_header; + ObSharedNothingTmpFileMetaItem meta_item; + char *page_buff = NULL; + ObTmpPageValueHandle p_handle; + if (OB_FAIL(get_page_(page_info, level_page_index, page_buff, p_handle))) { + STORAGE_LOG(WARN, "fail to get page", KR(ret), K(fd_), K(page_info), K(level_page_index)); + } else if (OB_FAIL(read_page_header_(page_buff, page_header))) { + STORAGE_LOG(WARN, "fail to read page header", KR(ret), K(fd_), KP(page_buff)); + } else if (OB_UNLIKELY(0 >= page_header.item_num_ + || item_index > page_header.item_num_ + || item_index < 0)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected page_header", KR(ret), K(fd_), K(page_header), K(item_index)); + } else { + int64_t end_virtual_page_id = upper_align(end_offset, ObTmpFileGlobal::PAGE_SIZE) / ObTmpFileGlobal::PAGE_SIZE; + while (OB_SUCC(ret) + && cur_item_index < page_header.item_num_) { + meta_item.reset(); + if (OB_FAIL(read_item_(page_buff, cur_item_index, meta_item))) { + STORAGE_LOG(WARN, "fail to read item", KR(ret), K(fd_), KP(page_buff), K(cur_item_index)); + } else if (meta_item.virtual_page_id_ >= end_virtual_page_id) { + break; + } else if (OB_FAIL(meta_items.push_back(meta_item))) { + STORAGE_LOG(WARN, "fail to push back", KR(ret), K(fd_), K(meta_item)); + } else if (FALSE_IT(cur_item_index++)) { + } else if (1 != page_info.page_level_) { + break; + } + } + } + p_handle.reset(); + if (OB_SUCC(ret) && cur_item_index == page_header.item_num_) { + reach_last_item = true; + } + return ret; +} + +//TODO: how to check whether block_index is valid +int ObSharedNothingTmpFileMetaTree::flush_meta_pages_for_block( + const int64_t block_index, + const ObTmpFileTreeEvictType flush_type, + char *block_buff, + int64_t &write_offset, + ObTmpFileTreeFlushContext &flush_context, + ObIArray &tree_io_array) +{ + int ret = OB_SUCCESS; + SpinWLockGuard guard(lock_); + if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX == block_index + || ObTmpFileTreeEvictType::INVALID == flush_type + || NULL == block_buff + || 0 != write_offset % ObTmpFileGlobal::PAGE_SIZE + || level_page_range_array_.count() >= INT16_MAX)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(block_index), K(flush_type), + KP(block_buff), K(write_offset), KPC(this)); + } else if (!root_item_.is_valid()) { + if (!level_page_range_array_.empty()) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected level_page_range_array_", KR(ret), KPC(this)); + } else { + flush_context.is_meta_reach_end_ = true; + } + } else { + int16_t level = 0; + bool need_use_context = false; + bool need_flush = true; + const int16_t max_level = level_page_range_array_.count() - 1; + ObTmpFileTreeIOInfo tree_io_info; + if (flush_context.is_valid()) { + if (flush_context.tree_epoch_ != tree_epoch_) { + STORAGE_LOG(INFO, "the tree_epoch_ in flush context is not equal to current tree_epoch_", + K(fd_), K(flush_context), K(tree_epoch_)); + need_flush = false; + flush_context.is_meta_reach_end_ = true; + } else { + //flush context means: in this round of flushing, + // the meta tree has already flushed some pages in previous blocks, + // so we need "flush context" to avoid repeated flushing. + level = flush_context.last_flush_level_; + need_use_context = true; + if (last_truncate_leaf_info_.is_valid()) { + if (OB_UNLIKELY(max_level < level)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected item_index_arr", KR(ret), K(level), KPC(this)); + } else if (flush_context.last_flush_page_index_in_level_ < level_page_range_array_.at(level).evicted_page_num_) { + need_flush = false; + flush_context.is_meta_reach_end_ = true; + } + } + } + } + if (OB_SUCC(ret) && need_flush) { + while (OB_SUCC(ret) + && level <= max_level + && write_offset < OB_DEFAULT_MACRO_BLOCK_SIZE) + { + uint32_t flush_start_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + int32_t level_page_index = -1; + if (need_use_context) { + level_page_index = flush_context.last_flush_page_index_in_level_; + ObTmpFilePageUniqKey page_key(level, level_page_index); + if (wbp_->is_dirty(fd_, flush_context.last_flush_page_id_, page_key)) { + //do nothing, we just skip this level. + } else if (OB_FAIL(wbp_->get_next_page_id(fd_, flush_context.last_flush_page_id_, + page_key, flush_start_page_id))) { + STORAGE_LOG(WARN, "fail to get next meta page id", KR(ret), K(fd_), K(flush_context), K(page_key)); + } else { + level_page_index++; + } + //only used the first time + need_use_context = false; + } else { + uint32_t flushed_page_id = level_page_range_array_[level].flushed_end_page_id_; + level_page_index = level_page_range_array_[level].flushed_page_num_; + if (ObTmpFileGlobal::INVALID_PAGE_ID == flushed_page_id) { + flush_start_page_id = level_page_range_array_[level].start_page_id_; + } else { + level_page_index--; + ObTmpFilePageUniqKey page_key(level, level_page_index); + if (wbp_->is_dirty(fd_, flushed_page_id, page_key)) { + //flushed page -> write cache + //we only write rightmost page, so pages before flushed_end_page_id_ can not be dirty. + flush_start_page_id = flushed_page_id; + } else if (OB_FAIL(wbp_->get_next_page_id(fd_, flushed_page_id, page_key, flush_start_page_id))) { + STORAGE_LOG(WARN, "fail to get next meta page id", KR(ret), K(fd_), K(flushed_page_id), K(page_key)); + } else { + level_page_index++; + } + } + } + if (OB_SUCC(ret) && ObTmpFileGlobal::INVALID_PAGE_ID != flush_start_page_id) { + tree_io_info.reset(); + tree_io_info.tree_epoch_ = tree_epoch_; + tree_io_info.block_index_ = block_index; + if (0 == level && OB_FAIL(flush_leaf_pages_(flush_start_page_id, level_page_index, flush_type, + block_buff, write_offset, tree_io_info))) { + STORAGE_LOG(WARN, "fail to flush leaf pages", KR(ret), K(flush_start_page_id), + K(level_page_index), K(flush_type), KP(block_buff), K(write_offset), KPC(this)); + } else if (0 < level && OB_FAIL(flush_internal_pages_(flush_start_page_id, level, level_page_index, + flush_type, block_buff, write_offset, tree_io_info))) { + STORAGE_LOG(WARN, "fail to flush internal pages", KR(ret), K(flush_start_page_id), + K(level), K(level_page_index), K(flush_type), KP(block_buff), K(write_offset), KPC(this)); + } else if (0 == tree_io_info.flush_nums_) { + //do nothing + STORAGE_LOG(INFO, "no meta page flush in this level", KR(ret), K(fd_), K(level), K(level_page_range_array_)); + } else if (OB_UNLIKELY(!tree_io_info.is_valid())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected tree_io_info", KR(ret), K(tree_io_info), KPC(this)); + //set the page flush information on the parent node, so that we can flush parent pages in advance. + } else if (OB_FAIL(modify_meta_items_at_parent_level_(tree_io_info, level_page_index))) { + STORAGE_LOG(WARN, "fail to modify meta items at parent level", KR(ret), + K(tree_io_info), K(level_page_index), KPC(this)); + } else if (OB_FAIL(tree_io_array.push_back(tree_io_info))) { + STORAGE_LOG(WARN, "fail to push back", KR(ret), K(fd_), K(tree_io_info)); + } else { + flush_context.tree_epoch_ = tree_epoch_; + flush_context.last_flush_level_ = level; + flush_context.last_flush_page_id_ = tree_io_info.flush_end_page_id_; + flush_context.last_flush_page_index_in_level_ = level_page_index + tree_io_info.flush_nums_ - 1; + stat_info_.meta_page_flushing_cnt_ += tree_io_info.flush_nums_; + stat_info_.all_type_page_flush_cnt_ += tree_io_info.flush_nums_; + } + } + if (OB_SUCC(ret) && max_level == level) { + flush_context.is_meta_reach_end_ = true; + } + level++; + } + } + } + STORAGE_LOG(INFO, "finish flush meta pages for block", KR(ret), K(fd_), K(tree_io_array)); + return ret; +} + +int ObSharedNothingTmpFileMetaTree::flush_leaf_pages_( + const uint32_t flush_start_page_id, + const int32_t start_page_index_in_level, + const ObTmpFileTreeEvictType flush_type, + char *block_buff, + int64_t &write_offset, + ObTmpFileTreeIOInfo &tree_io_info) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_PAGE_ID == flush_start_page_id + || 0 > start_page_index_in_level + || 0 != write_offset % ObTmpFileGlobal::PAGE_SIZE + || level_page_range_array_.empty())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), K(flush_start_page_id), K(start_page_index_in_level), + K(write_offset), K(level_page_range_array_)); + } else { + //TODO: start page id in block is from 0 + tree_io_info.physical_start_page_id_ = write_offset / ObTmpFileGlobal::PAGE_SIZE; + tree_io_info.page_level_ = 0; + uint32_t cur_page_id = flush_start_page_id; + const uint32_t end_page_id = level_page_range_array_[0].end_page_id_; + int32_t page_index_in_level = start_page_index_in_level; + while (OB_SUCC(ret) + && ObTmpFileGlobal::INVALID_PAGE_ID != cur_page_id + && write_offset + ObTmpFileGlobal::PAGE_SIZE <= OB_DEFAULT_MACRO_BLOCK_SIZE + && (ObTmpFileTreeEvictType::FULL == flush_type || end_page_id != cur_page_id)) { + char *page_buff = NULL; + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ObTmpFilePageUniqKey page_key(0, page_index_in_level); + if (OB_UNLIKELY(!wbp_->is_dirty(fd_, cur_page_id, page_key))) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected page state", KR(ret), K(fd_), K(cur_page_id)); + } else if (OB_FAIL(wbp_->read_page(fd_, cur_page_id, page_key, page_buff, next_page_id))) { + STORAGE_LOG(WARN, "fail to read from write cache", KR(ret), K(fd_), K(cur_page_id), K(page_key)); + //change page state to write back + } else if (OB_FAIL(wbp_->notify_write_back(fd_, cur_page_id, page_key))) { + STORAGE_LOG(WARN, "fail to notify write back for meta", KR(ret), K(fd_), K(cur_page_id), K(page_key)); + } else { + ObTmpPageCacheKey cache_key(tree_io_info.block_index_, + write_offset / ObTmpFileGlobal::PAGE_SIZE, MTL_ID()); + ObTmpPageCacheValue cache_value(page_buff); + MEMCPY(block_buff + write_offset, page_buff, ObTmpFileGlobal::PAGE_SIZE); + if (OB_FAIL(calc_and_set_page_checksum_(block_buff + write_offset))) { + STORAGE_LOG(WARN, "fail to calc and set page checksum", KR(ret), K(fd_), KP(block_buff + write_offset)); + } else { + ObTmpPageCacheValue cache_value(block_buff + write_offset); + ObTmpPageCache::get_instance().try_put_page_to_cache(cache_key, cache_value); + write_offset += ObTmpFileGlobal::PAGE_SIZE; + if (flush_start_page_id == cur_page_id) { + tree_io_info.flush_start_page_id_ = cur_page_id; + tree_io_info.flush_start_level_page_index_ = page_index_in_level; + } + tree_io_info.flush_end_page_id_ = cur_page_id; + tree_io_info.flush_nums_++; + cur_page_id = next_page_id; + page_index_in_level++; + } + } + } + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::calc_and_set_page_checksum_(char* page_buff) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(page_buff)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), KP(page_buff)); + } else { + ObSharedNothingTmpFileTreePageHeader page_header = *((ObSharedNothingTmpFileTreePageHeader *)(page_buff)); + page_header.checksum_ = ob_crc64(page_buff + PAGE_HEADER_SIZE, ObTmpFileGlobal::PAGE_SIZE - PAGE_HEADER_SIZE); + MEMCPY(page_buff, &page_header, PAGE_HEADER_SIZE); + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::flush_internal_pages_( + const uint32_t flush_start_page_id, + const int16_t level, + const int32_t start_page_index_in_level, + const ObTmpFileTreeEvictType flush_type, + char *block_buff, + int64_t &write_offset, + ObTmpFileTreeIOInfo &tree_io_info) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_PAGE_ID == flush_start_page_id + || level >= level_page_range_array_.count() + || level < 1 + || 0 > start_page_index_in_level + || ObTmpFileTreeEvictType::INVALID == flush_type + || NULL == block_buff + || 0 != write_offset % ObTmpFileGlobal::PAGE_SIZE)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), K(flush_start_page_id), K(level), K(start_page_index_in_level), + K(flush_type), KP(block_buff), K(level_page_range_array_), K(write_offset)); + } else { + const int16_t FULL_PAGE_META_ITEM_NUM = + MIN(MAX_PAGE_ITEM_COUNT, (ObTmpFileGlobal::PAGE_SIZE - PAGE_HEADER_SIZE) / sizeof(ObSharedNothingTmpFileMetaItem)); + //TODO: start page id in block is from 0 + tree_io_info.physical_start_page_id_ = write_offset / ObTmpFileGlobal::PAGE_SIZE; + tree_io_info.page_level_ = level; + uint32_t cur_page_id = flush_start_page_id; + const uint32_t end_page_id = level_page_range_array_[level].end_page_id_; + ObSharedNothingTmpFileMetaItem last_item; + int32_t page_index_in_level = start_page_index_in_level; + while (OB_SUCC(ret) + && ObTmpFileGlobal::INVALID_PAGE_ID != cur_page_id + && write_offset + ObTmpFileGlobal::PAGE_SIZE <= OB_DEFAULT_MACRO_BLOCK_SIZE + && (ObTmpFileTreeEvictType::FULL == flush_type || end_page_id != cur_page_id)) { + char * page_buff = NULL; + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ObTmpFilePageUniqKey page_key(level, page_index_in_level); + if (OB_UNLIKELY(!wbp_->is_dirty(fd_, cur_page_id, page_key))) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected page state", KR(ret), K(fd_), K(cur_page_id)); + } else if (OB_FAIL(wbp_->read_page(fd_, cur_page_id, page_key, page_buff, next_page_id))) { + STORAGE_LOG(WARN, "fail to read from write cache", KR(ret), K(fd_), K(cur_page_id), K(page_key)); + } else { + //check whether the page is satisfied for flush + last_item.reset(); + ObSharedNothingTmpFileTreePageHeader page_header; + int32_t rightmost_child_page_index = -1; + if (OB_FAIL(read_page_header_(page_buff, page_header))) { + STORAGE_LOG(WARN, "fail to read page header", KR(ret), K(fd_), KP(page_buff)); + } else if (page_header.item_num_ <= 0) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected item_num", KR(ret), K(fd_), K(page_header)); + } else if (OB_FAIL(read_item_(page_buff, page_header.item_num_ - 1/*item_index*/, last_item))) { + STORAGE_LOG(WARN, "fail to read item", KR(ret), K(fd_), KP(page_buff), K(page_header)); + } else if (FALSE_IT(rightmost_child_page_index = page_index_in_level * FULL_PAGE_META_ITEM_NUM + page_header.item_num_ - 1)) { + } else if (!is_page_flushed(last_item) + || (is_page_in_write_cache(last_item) + && wbp_->is_dirty(fd_, last_item.buffer_page_id_, ObTmpFilePageUniqKey(level - 1, rightmost_child_page_index)))) { + //we can be sure that the following pages in this level will not satisfy + break; + //change page state to write back + } else if (OB_FAIL(wbp_->notify_write_back(fd_, cur_page_id, page_key))) { + STORAGE_LOG(WARN, "fail to notify write back for meta", KR(ret), K(fd_), K(cur_page_id), K(page_key)); + } else { + ObTmpPageCacheKey cache_key(tree_io_info.block_index_, + write_offset / ObTmpFileGlobal::PAGE_SIZE, MTL_ID()); + MEMCPY(block_buff + write_offset, page_buff, ObTmpFileGlobal::PAGE_SIZE); + if (OB_FAIL(modify_child_pages_location(block_buff + write_offset))) { + STORAGE_LOG(WARN, "fail to modify child pages location", KR(ret), K(fd_), KP(block_buff + write_offset)); + } else if (OB_FAIL(calc_and_set_page_checksum_(block_buff + write_offset))) { + STORAGE_LOG(WARN, "fail to calc and set page checksum", KR(ret), K(fd_), KP(block_buff + write_offset)); + } else { + ObTmpPageCacheValue cache_value(block_buff + write_offset); + ObTmpPageCache::get_instance().try_put_page_to_cache(cache_key, cache_value); + write_offset += ObTmpFileGlobal::PAGE_SIZE; + if (flush_start_page_id == cur_page_id) { + tree_io_info.flush_start_page_id_ = cur_page_id; + tree_io_info.flush_start_level_page_index_ = page_index_in_level; + } + tree_io_info.flush_end_page_id_ = cur_page_id; + tree_io_info.flush_nums_++; + cur_page_id = next_page_id; + page_index_in_level++; + } + } + } + } + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::modify_child_pages_location( + char *page_buff) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(page_buff)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), KP(page_buff)); + } else { + ObSharedNothingTmpFileTreePageHeader page_header; + if (OB_FAIL(read_page_header_(page_buff, page_header))) { + STORAGE_LOG(WARN, "fail to read page header", KR(ret), K(fd_), KP(page_buff)); + } else { + int16_t item_index = 0; + ObSharedNothingTmpFileMetaItem meta_item; + while (OB_SUCC(ret) + && item_index < page_header.item_num_) { + meta_item.reset(); + if (OB_FAIL(read_item_(page_buff, item_index, meta_item))) { + STORAGE_LOG(WARN, "fail to read item", KR(ret), K(fd_), KP(page_buff), K(item_index)); + } else if (ObTmpFileGlobal::INVALID_PAGE_ID == meta_item.buffer_page_id_) { + //do nothing + } else { + meta_item.buffer_page_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + if (OB_FAIL(rewrite_item_(page_buff, item_index, meta_item))) { + STORAGE_LOG(WARN, "fail to rewrite item", KR(ret), K(fd_), KP(page_buff), K(item_index)); + } + } + item_index++; + } + } + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::modify_meta_items_at_parent_level_( + const ObTmpFileTreeIOInfo &tree_io, + const int32_t start_page_index_in_level) +{ + int ret = OB_SUCCESS; + const uint32_t cur_level = tree_io.page_level_ + 1; + if (OB_UNLIKELY(cur_level > level_page_range_array_.count() + || !tree_io.is_valid())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), K(tree_io), K(level_page_range_array_)); + } else if (cur_level == level_page_range_array_.count()) { + //this must be the root page being flushed + if (OB_UNLIKELY(1 != tree_io.flush_nums_ + || tree_io.flush_start_page_id_ != tree_io.flush_end_page_id_ + || tree_io.flush_start_page_id_ != root_item_.buffer_page_id_ + || 0 != start_page_index_in_level)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected tree_io", KR(ret), K(fd_), K(tree_io), K(root_item_), K(start_page_index_in_level)); + //NOTE: It doesn't matter if we release this page early, because under wlock. + } else if (is_page_flushed(root_item_) + && OB_FAIL(release_tmp_file_page_(root_item_.block_index_, root_item_.physical_page_id_, 1))) { + STORAGE_LOG(WARN, "fail to release tmp file page", KR(ret), K(fd_), K(root_item_)); + } else { + root_item_.block_index_ = tree_io.block_index_; + root_item_.physical_page_id_ = tree_io.physical_start_page_id_; + } + } else { + const int16_t FULL_PAGE_META_ITEM_NUM = + MIN(MAX_PAGE_ITEM_COUNT, (ObTmpFileGlobal::PAGE_SIZE - PAGE_HEADER_SIZE) / sizeof(ObSharedNothingTmpFileMetaItem)); + int32_t child_level_page_index = start_page_index_in_level; + uint32_t physical_page_id = tree_io.physical_start_page_id_; + ObSharedNothingTmpFileMetaItem meta_item; + bool is_end = false; + bool has_find = false; + uint32_t last_flushed_page_id = level_page_range_array_.at(cur_level).flushed_end_page_id_; + //the child page of last meta item on last_flushed_page may be flushed again. + uint32_t cur_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + int32_t cur_level_page_index = level_page_range_array_.at(cur_level).flushed_page_num_; + if (ObTmpFileGlobal::INVALID_PAGE_ID == last_flushed_page_id) { + cur_page_id = level_page_range_array_.at(cur_level).start_page_id_; + } else { + cur_page_id = last_flushed_page_id; + cur_level_page_index--; + } + while (OB_SUCC(ret) + && ObTmpFileGlobal::INVALID_PAGE_ID != cur_page_id + && !is_end) { + int16_t item_index = 0; + char *page_buff = NULL; + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ObSharedNothingTmpFileTreePageHeader page_header; + ObTmpFilePageUniqKey page_key(cur_level, cur_level_page_index); + //this upper layer page must be in write cache + if (OB_FAIL(wbp_->read_page(fd_, cur_page_id, page_key, page_buff, next_page_id))) { + STORAGE_LOG(WARN, "fail to read from write cache", KR(ret), K(fd_), K(cur_page_id), K(page_key)); + } else if (OB_FAIL(read_page_header_(page_buff, page_header))) { + STORAGE_LOG(WARN, "fail to read page header", KR(ret), K(fd_), KP(page_buff)); + } else if (OB_UNLIKELY(0 >= page_header.item_num_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected page_header", KR(ret), K(fd_), K(page_header)); + } else { + while (OB_SUCC(ret) + && item_index < page_header.item_num_ + && !is_end) { + meta_item.reset(); + if (OB_FAIL(read_item_(page_buff, item_index, meta_item))) { + STORAGE_LOG(WARN, "fail to read item", KR(ret), K(fd_), KP(page_buff), K(item_index)); + } else { + //TODO: flush_start_page_id_ -> buffer_start_page_id_ + if (has_find || cur_level_page_index * FULL_PAGE_META_ITEM_NUM + item_index == child_level_page_index) { + if (OB_UNLIKELY((!has_find && tree_io.flush_start_page_id_ != meta_item.buffer_page_id_) + || cur_level_page_index * FULL_PAGE_META_ITEM_NUM + item_index != child_level_page_index)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected meta_item", KR(ret), K(fd_), K(tree_io), K(meta_item), + K(child_level_page_index), K(cur_page_id), K(cur_level_page_index), K(item_index)); + } else { + has_find = true; + if (is_page_flushed(meta_item) + && OB_FAIL(release_tmp_file_page_(meta_item.block_index_, meta_item.physical_page_id_, 1))) { + STORAGE_LOG(WARN, "fail to release tmp file page", KR(ret), K(fd_), K(meta_item), K(item_index), K(page_header)); + } else { + meta_item.block_index_ = tree_io.block_index_; + meta_item.physical_page_id_ = physical_page_id; + physical_page_id++; + child_level_page_index++; + if (OB_FAIL(rewrite_item_(page_buff, item_index, meta_item))) { + STORAGE_LOG(WARN, "fail to rewrite item", KR(ret), K(fd_), KP(page_buff), K(item_index), K(meta_item)); + } else if (tree_io.flush_end_page_id_ == meta_item.buffer_page_id_) { + is_end = true; + } + } + } + } + item_index++; + } + } + } + if (OB_SUCC(ret) && has_find && OB_FAIL(wbp_->notify_dirty(fd_, cur_page_id, page_key))) { + STORAGE_LOG(WARN, "fail to notify dirty for meta", KR(ret), K(fd_), K(cur_page_id), K(page_key)); + } + cur_page_id = next_page_id; + cur_level_page_index++; + } + if (OB_SUCC(ret)) { + if (OB_UNLIKELY(!is_end + || physical_page_id - tree_io.physical_start_page_id_ != tree_io.flush_nums_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected is_end or flush_nums_", KR(ret), K(fd_), + K(is_end), K(physical_page_id), K(tree_io)); + } + } + } + return ret; +} + +//TODO: 或许可以传进来ObIArray* > &tree_io_arrays二维数组 +// check tree io seq +int ObSharedNothingTmpFileMetaTree::update_after_flush( + const common::ObIArray &tree_io_array) +{ + int ret = OB_SUCCESS; + SpinWLockGuard guard(lock_); + if (OB_UNLIKELY(tree_io_array.empty())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), K(tree_io_array), K(root_item_)); + } else { + ARRAY_FOREACH_N(tree_io_array, i, cnt) { + bool tree_io_is_empty = false; + const ObTmpFileTreeIOInfo &tree_io = tree_io_array.at(i); + const int16_t origin_tree_flush_num = tree_io.flush_nums_; + bool end_page_flush_again = false; + if (OB_UNLIKELY(!tree_io.is_valid())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(tree_io), KPC(this)); + } else if (tree_io.tree_epoch_ != tree_epoch_) { + STORAGE_LOG(INFO, "the tree_epoch_ in tree_io is not equal to current tree_epoch_", + K(fd_), K(tree_io), K(tree_epoch_)); + } else if (tree_io.page_level_ >= level_page_range_array_.count()) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(tree_io), KPC(this)); + } else { + const uint32_t flushed_end_page_id_in_array = level_page_range_array_[tree_io.page_level_].flushed_end_page_id_; + const uint32_t start_page_id_in_array = level_page_range_array_[tree_io.page_level_].start_page_id_; + uint32_t next_page_id_in_array = ObTmpFileGlobal::INVALID_PAGE_ID; + int32_t level_page_index = level_page_range_array_[tree_io.page_level_].flushed_page_num_; + if (ObTmpFileGlobal::INVALID_PAGE_ID != flushed_end_page_id_in_array) { + level_page_index--; + ObTmpFilePageUniqKey page_key(tree_io.page_level_, level_page_index); + if (OB_FAIL(wbp_->get_next_page_id(fd_, flushed_end_page_id_in_array, page_key, next_page_id_in_array))) { + STORAGE_LOG(WARN, "fail to get next meta page id", KR(ret), K(fd_), K(flushed_end_page_id_in_array), K(page_key)); + } else if (OB_UNLIKELY(flushed_end_page_id_in_array != tree_io.flush_start_page_id_ + && next_page_id_in_array != tree_io.flush_start_page_id_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected tree_io", KR(ret), K(tree_io), + K(flushed_end_page_id_in_array), K(next_page_id_in_array), KPC(this)); + } else if (flushed_end_page_id_in_array == tree_io.flush_start_page_id_) { + end_page_flush_again = true; + } else { + level_page_index++; + } + } else { + if (level_page_index > tree_io.flush_start_level_page_index_) { + int64_t truncated_num = level_page_index - tree_io.flush_start_level_page_index_; + if (truncated_num >= tree_io.flush_nums_) { + tree_io_is_empty = true; + } else { + ObTmpFileTreeIOInfo& tree_io_mutable_ref = const_cast(tree_io); + tree_io_mutable_ref.flush_start_page_id_ = start_page_id_in_array; + tree_io_mutable_ref.flush_nums_ -= truncated_num; + } + } else if (OB_UNLIKELY(start_page_id_in_array != tree_io.flush_start_page_id_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected tree_io", KR(ret), K(tree_io), KPC(this)); + } + } + if (OB_SUCC(ret) && !tree_io_is_empty) { + uint32_t cur_page_id = tree_io.flush_start_page_id_; + int16_t num = 0; + //change page state from writeback to cached + while (OB_SUCC(ret) && ObTmpFileGlobal::INVALID_PAGE_ID != cur_page_id) { + ObTmpFilePageUniqKey page_key(tree_io.page_level_, level_page_index); + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + if (cur_page_id == tree_io.flush_end_page_id_ && wbp_->is_dirty(fd_, cur_page_id, page_key)) { + //do nothing + STORAGE_LOG(INFO, "page is dirty again, do not change page status", KR(ret), K(fd_), K(cur_page_id)); + } else if (OB_FAIL(wbp_->notify_write_back_succ(fd_, cur_page_id, page_key))) { + STORAGE_LOG(WARN, "fail to notify write back succ for meta", KR(ret), K(fd_), K(cur_page_id), K(page_key)); + } + if (OB_SUCC(ret)) { + num++; + if (OB_UNLIKELY(num > tree_io.flush_nums_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected num", KR(ret), K(tree_io), K(num), K(cur_page_id), KPC(this)); + } else if (cur_page_id == tree_io.flush_end_page_id_) { + break; + } else if (OB_FAIL(wbp_->get_next_page_id(fd_, cur_page_id, page_key, next_page_id))) { + STORAGE_LOG(WARN, "fail to get next meta page id", KR(ret), K(fd_), K(cur_page_id), K(tree_io), K(page_key)); + } else { + cur_page_id = next_page_id; + level_page_index++; + } + } + } + if (OB_SUCC(ret)) { + if (OB_UNLIKELY(tree_io.flush_nums_ != num + || cur_page_id != tree_io.flush_end_page_id_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected page level or flush num", KR(ret), K(tree_io), K(num), K(cur_page_id), KPC(this)); + } else { + level_page_range_array_[tree_io.page_level_].flushed_end_page_id_ = tree_io.flush_end_page_id_; + if (end_page_flush_again) { + num--; + } + level_page_range_array_[tree_io.page_level_].flushed_page_num_ += num; + } + } + } + } + if (OB_SUCC(ret)) { + stat_info_.meta_page_flushing_cnt_ -= origin_tree_flush_num; + } + } + } + STORAGE_LOG(INFO, "finish update after flush", KR(ret), K(fd_), K(level_page_range_array_)); + return ret; +} + +int ObSharedNothingTmpFileMetaTree::prepare_for_write_tail( + ObSharedNothingTmpFileDataItem &last_data_item) +{ + int ret = OB_SUCCESS; + SpinWLockGuard guard(lock_); + if (!root_item_.is_valid()) { + if (OB_UNLIKELY(data_item_array_.empty())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected data_item_array_ count", KR(ret), K(fd_), K(data_item_array_)); + } else { + last_data_item = data_item_array_.at(data_item_array_.count() - 1); + if (OB_UNLIKELY(0 >= last_data_item.physical_page_num_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected physical_page_num", KR(ret), K(last_data_item), KPC(this)); + } + } + } else { + ObSharedNothingTmpFileTreePageHeader page_header; + ObSharedNothingTmpFileMetaItem page_info; + if (OB_UNLIKELY(level_page_range_array_.empty())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected level_page_range_array_", KR(ret), KPC(this)); + } else if (OB_FAIL(get_rightmost_leaf_page_for_write_(page_info))) { + STORAGE_LOG(WARN, "fail to get rightmost leaf page for write", KR(ret), KPC(this)); + } else { + //we don't need to worry about the rightmost leaf page being evicted, + // because we set is_writing_ = true. + char *leaf_page_buff = NULL; + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + int32_t leaf_level_page_index = level_page_range_array_.at(0).evicted_page_num_ + + level_page_range_array_.at(0).cached_page_num_ - 1; + if (OB_FAIL(wbp_->read_page(fd_, page_info.buffer_page_id_, ObTmpFilePageUniqKey(0, leaf_level_page_index), + leaf_page_buff, next_page_id))) { + STORAGE_LOG(WARN, "fail to read from write cache", KR(ret), K(fd_), K(page_info), K(leaf_level_page_index)); + } else if (OB_FAIL(read_page_header_(leaf_page_buff, page_header))) { + STORAGE_LOG(WARN, "fail to read page header", KR(ret), KP(leaf_page_buff), KPC(this)); + } else if (OB_UNLIKELY(0 >= page_header.item_num_)) { + //There is no concurrent writing + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected item_num", KR(ret), KP(leaf_page_buff), K(page_header), KPC(this)); + } else if (OB_FAIL(read_item_(leaf_page_buff, page_header.item_num_ - 1, last_data_item))) { + STORAGE_LOG(WARN, "fail to read item", KR(ret), KP(leaf_page_buff), K(page_header), KPC(this)); + } else if (OB_UNLIKELY(0 >= last_data_item.physical_page_num_)) { + ret = OB_ERR_UNEXPECTED; + //print page content + //NOTE: control print frequence + ObSharedNothingTmpFileTreePageHeader tmp_page_header; + ObArray items; + read_page_content_(leaf_page_buff, tmp_page_header, items); + STORAGE_LOG(WARN, "unexpected physical_page_num, dump tree page", KR(ret), K(last_data_item), KP(leaf_page_buff), + K(page_info), K(leaf_level_page_index), K(tmp_page_header), K(items), KPC(this)); + } + } + } + return ret; +} + +//After the tail is written +int ObSharedNothingTmpFileMetaTree::finish_write_tail( + const ObSharedNothingTmpFileDataItem &last_data_item, + const bool release_tail_in_disk) +{ + int ret = OB_SUCCESS; + SpinWLockGuard guard(lock_); + if (!root_item_.is_valid()) { + if (OB_UNLIKELY(data_item_array_.empty() + || last_data_item != data_item_array_[data_item_array_.count() - 1])) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected data_item_array_ or last_data_item", KR(ret), K(last_data_item), KPC(this)); + } else if (release_tail_in_disk) { + if (0 == --data_item_array_[data_item_array_.count() - 1].physical_page_num_) { + data_item_array_.pop_back(); + } + } + } else { + if (OB_UNLIKELY(level_page_range_array_.empty() + || !is_writing_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected level_array_ or is_writing_", KR(ret), KPC(this)); + } else if (release_tail_in_disk) { + char *leaf_page_buff = NULL; + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + uint32_t page_id = level_page_range_array_[0].end_page_id_; + int32_t level_page_index = level_page_range_array_.at(0).evicted_page_num_ + + level_page_range_array_.at(0).cached_page_num_ - 1; + ObSharedNothingTmpFileTreePageHeader page_header; + ObSharedNothingTmpFileDataItem data_item; + ObTmpFilePageUniqKey page_key(0, level_page_index); + if (OB_FAIL(wbp_->read_page(fd_, page_id, page_key, leaf_page_buff, next_page_id))) { + STORAGE_LOG(WARN, "fail to read from write cache", KR(ret), K(fd_), K(page_id), K(page_key)); + } else if (OB_FAIL(read_page_header_(leaf_page_buff, page_header))) { + STORAGE_LOG(WARN, "fail to read page header", KR(ret), KP(leaf_page_buff), KPC(this)); + } else if (OB_FAIL(read_item_(leaf_page_buff, page_header.item_num_ - 1, data_item))) { + STORAGE_LOG(WARN, "fail to read item", KR(ret), KP(leaf_page_buff), K(page_header), KPC(this)); + } else if (OB_UNLIKELY(last_data_item != data_item)) { + ret = OB_ERR_UNEXPECTED; + //print page content + //NOTE: control print frequence + ObSharedNothingTmpFileTreePageHeader tmp_page_header; + ObArray items; + read_page_content_(leaf_page_buff, tmp_page_header, items); + STORAGE_LOG(WARN, "unexpected virtual_page_id, dump tree page", KR(ret), KP(leaf_page_buff), K(page_id), K(page_key), + K(last_data_item), K(data_item), K(tmp_page_header), K(items), KPC(this)); + } else if (FALSE_IT(data_item.physical_page_num_--)) { + } else if (0 == data_item.physical_page_num_) { + if (OB_FAIL(remove_page_item_from_tail_(leaf_page_buff, 1/*remove_num*/))) { + STORAGE_LOG(WARN, "unexpected item_num", KR(ret), KP(leaf_page_buff), KPC(this)); + } + } else if (OB_FAIL(rewrite_item_(leaf_page_buff, page_header.item_num_ - 1, data_item))) { + STORAGE_LOG(WARN, "fail to rewrite item", KR(ret), KP(leaf_page_buff), + K(page_header), K(data_item), KPC(this)); + } + if (FAILEDx(wbp_->notify_dirty(fd_, page_id, page_key))) { + STORAGE_LOG(WARN, "fail to notify dirty for meta", KR(ret), K(fd_), K(page_id), K(page_key)); + } + } + if (OB_SUCC(ret)) { + is_writing_ = false; + } + } + if (OB_SUCC(ret) && release_tail_in_disk) { + if (OB_FAIL(release_tmp_file_page_(last_data_item.block_index_, + last_data_item.physical_page_id_ + last_data_item.physical_page_num_ - 1, 1))) { + STORAGE_LOG(WARN, "fail to release tmp file page", KR(ret), K(fd_), K(last_data_item)); + } + } + STORAGE_LOG(INFO, "finish write tail", KR(ret), K(fd_), K(release_tail_in_disk)); + return ret; +} + +int ObSharedNothingTmpFileMetaTree::evict_meta_pages( + const int64_t expected_page_num, + const ObTmpFileTreeEvictType flush_type, + int64_t &actual_evict_page_num) +{ + int ret = OB_SUCCESS; + actual_evict_page_num = 0; + if (OB_UNLIKELY(0 >= expected_page_num + || ObTmpFileTreeEvictType::INVALID == flush_type)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), K(expected_page_num), K(flush_type)); + } else { + SpinWLockGuard guard(lock_); + if (!root_item_.is_valid()) { + //do nothing + } else { + ObSEArray evict_pages; + ARRAY_FOREACH_X(level_page_range_array_, level, level_cnt, + OB_SUCC(ret) && actual_evict_page_num < expected_page_num) { + evict_pages.reset(); + uint32_t next_page_id = level_page_range_array_[level].start_page_id_; + const uint32_t end_evict_page = level_page_range_array_[level].flushed_end_page_id_; + const uint32_t end_page_id = level_page_range_array_[level].end_page_id_; + const int32_t start_level_page_index = level_page_range_array_.at(level).evicted_page_num_; + uint32_t cur_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + int32_t level_page_index = start_level_page_index; + if (ObTmpFileGlobal::INVALID_PAGE_ID == end_evict_page) { + //no pages need to be evicted at this level + continue; + } + while (OB_SUCC(ret) + && ObTmpFileGlobal::INVALID_PAGE_ID != next_page_id + && actual_evict_page_num < expected_page_num + && (!is_writing_ || end_page_id != next_page_id) //not evict end page if is writing + && (ObTmpFileTreeEvictType::FULL == flush_type || end_page_id != next_page_id)) { + ObTmpFilePageUniqKey page_key(level, level_page_index); + if (OB_UNLIKELY(end_evict_page != next_page_id && !wbp_->is_cached(fd_, next_page_id, page_key))) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected next_page_id", KR(ret), K(next_page_id), KPC(this)); + } else if (end_evict_page == next_page_id && !wbp_->is_cached(fd_, next_page_id, page_key)) { + break; + } else if (level > 0) { + //check whether the page is satisfied for evict + char * page_buff = NULL; + ObSharedNothingTmpFileMetaItem last_item; + uint32_t unused_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + if (OB_FAIL(wbp_->read_page(fd_, next_page_id, page_key, page_buff, unused_page_id))) { + STORAGE_LOG(WARN, "fail to read from write cache", KR(ret), K(fd_), K(next_page_id), K(page_key)); + } else { + last_item.reset(); + ObSharedNothingTmpFileTreePageHeader page_header; + if (OB_FAIL(read_page_header_(page_buff, page_header))) { + STORAGE_LOG(WARN, "fail to read page header", KR(ret), K(fd_), KP(page_buff)); + } else if (page_header.item_num_ <= 0) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected item_num", KR(ret), K(page_header), KPC(this)); + } else if (OB_FAIL(read_item_(page_buff, page_header.item_num_ - 1/*item_index*/, last_item))) { + STORAGE_LOG(WARN, "fail to read item", KR(ret), KP(page_buff), K(page_header), KPC(this)); + } else if (is_page_in_write_cache(last_item)) { + if (OB_UNLIKELY(end_evict_page != next_page_id)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected next_page", KR(ret), K(next_page_id), K(last_item), KPC(this)); + } else { + break; + } + } + } + } + if (OB_SUCC(ret)) { + cur_page_id = next_page_id; + next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + if (OB_FAIL(evict_pages.push_back(cur_page_id))) { + STORAGE_LOG(WARN, "fail to push back", KR(ret), K(fd_), K(cur_page_id)); + } else if (OB_FAIL(wbp_->get_next_page_id(fd_, cur_page_id, page_key, next_page_id))) { + STORAGE_LOG(WARN, "fail to get next meta page id", KR(ret), K(fd_), K(cur_page_id), K(page_key)); + } else { + actual_evict_page_num++; + level_page_index++; + if (end_evict_page == cur_page_id) { + break; + } + } + } + } + if (OB_SUCC(ret) && !evict_pages.empty()) { + if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_PAGE_ID == cur_page_id)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected cur_page_id", KR(ret), K(cur_page_id), K(level), K(level_page_index), KPC(this)); + } else if (OB_FAIL(modify_meta_items_during_evict_(evict_pages, level + 1, start_level_page_index))) { + STORAGE_LOG(WARN, "fail to modify meta items during evict", KR(ret), K(evict_pages), + K(level), K(start_level_page_index), KPC(this)); + } else { + uint32_t unused_next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ARRAY_FOREACH_N(evict_pages, i, cnt) { + //change page state from flushed/cached to evicted/invalid + //TODO: check whether page state is cached/clean. + if (OB_FAIL(wbp_->free_page(fd_, evict_pages.at(i), + ObTmpFilePageUniqKey(level, start_level_page_index + i), unused_next_page_id))) { + STORAGE_LOG(WARN, "fail to free meta page in write cache", KR(ret), K(fd_), K(evict_pages.at(i)), + K(level), K(start_level_page_index + i)); + } + } + } + if (OB_SUCC(ret)) { + if (end_page_id == cur_page_id) { + //all pages in this level are evicted + level_page_range_array_[level].start_page_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + level_page_range_array_[level].end_page_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + level_page_range_array_[level].flushed_end_page_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_PAGE_ID == next_page_id)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected next_page_id", KR(ret), K(next_page_id), K(cur_page_id), KPC(this)); + } else if (end_evict_page == cur_page_id) { + //flushed_end_page is not equal to end_page, + // and pages are evicted to flushed_end_page(including flushed_end_page) + level_page_range_array_[level].start_page_id_ = next_page_id; + level_page_range_array_[level].flushed_end_page_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + } else { + //pages are not evicted to flushed_end_page + level_page_range_array_[level].start_page_id_ = next_page_id; + } + if (OB_SUCC(ret)) { + level_page_range_array_[level].cached_page_num_ -= evict_pages.count(); + level_page_range_array_[level].evicted_page_num_ += evict_pages.count(); + } + } + } + } + } + } + STORAGE_LOG(INFO, "finish evict meta pages", KR(ret), K(fd_), K(level_page_range_array_)); + return ret; +} + +int ObSharedNothingTmpFileMetaTree::modify_meta_items_during_evict_( + const ObIArray &evict_pages, + const int16_t level, + const int32_t start_page_index_in_level) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(evict_pages.empty() + || level > level_page_range_array_.count())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected evict_pages", KR(ret), K(fd_), K(evict_pages), + K(level), K(level_page_range_array_)); + } else { + char *page_buff = NULL; + const int64_t level_count = level_page_range_array_.count(); + const int64_t evict_page_count = evict_pages.count(); + int64_t array_index = 0; + bool has_find = false; + if (level < level_count) { + const int16_t FULL_PAGE_META_ITEM_NUM = + MIN(MAX_PAGE_ITEM_COUNT, (ObTmpFileGlobal::PAGE_SIZE - PAGE_HEADER_SIZE) / sizeof(ObSharedNothingTmpFileMetaItem)); + uint32_t cur_page_id = level_page_range_array_[level].start_page_id_; + const uint32_t end_page_id = level_page_range_array_[level].end_page_id_; + int32_t level_page_index = level_page_range_array_[level].evicted_page_num_; + int32_t evict_page_index_in_level = start_page_index_in_level; + ObSharedNothingTmpFileMetaItem meta_item; + if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_PAGE_ID == cur_page_id + || ObTmpFileGlobal::INVALID_PAGE_ID == end_page_id + || 0 > evict_page_index_in_level)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected page_id", KR(ret), K(fd_), K(level_page_range_array_), + K(level), K(evict_page_index_in_level)); + } else { + while(OB_SUCC(ret) + && ObTmpFileGlobal::INVALID_PAGE_ID != cur_page_id + && array_index < evict_page_count) { + ObSharedNothingTmpFileTreePageHeader page_header; + char *page_buff = NULL; + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + if (OB_FAIL(wbp_->read_page(fd_, cur_page_id, ObTmpFilePageUniqKey(level, level_page_index), page_buff, next_page_id))) { + STORAGE_LOG(WARN, "fail to read from write cache", KR(ret), K(fd_), K(cur_page_id), K(level), K(level_page_index)); + } else if (OB_FAIL(read_page_header_(page_buff, page_header))) { + STORAGE_LOG(WARN, "fail to read page header", KR(ret), K(fd_), KP(page_buff)); + } else { + int16_t item_index = 0; + while (OB_SUCC(ret) + && item_index < page_header.item_num_ + && array_index < evict_page_count) { + meta_item.reset(); + if (OB_FAIL(read_item_(page_buff, item_index, meta_item))) { + STORAGE_LOG(WARN, "fail to read item", KR(ret), K(fd_), KP(page_buff), K(item_index)); + } else { + if (has_find || level_page_index * FULL_PAGE_META_ITEM_NUM + item_index == evict_page_index_in_level) { + if (OB_UNLIKELY(meta_item.buffer_page_id_ != evict_pages.at(array_index) + || level_page_index * FULL_PAGE_META_ITEM_NUM + item_index != evict_page_index_in_level + || !is_page_in_write_cache(meta_item) + || !is_page_flushed(meta_item))) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected meta_item", KR(ret), K(fd_), K(evict_pages), K(array_index), K(meta_item), + K(evict_page_index_in_level), K(cur_page_id), K(level_page_index), K(item_index)); + } else { + has_find = true; + meta_item.buffer_page_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + array_index++; + evict_page_index_in_level++; + if (OB_FAIL(rewrite_item_(page_buff, item_index, meta_item))) { + STORAGE_LOG(WARN, "fail to rewrite item", KR(ret), K(fd_), KP(page_buff), K(item_index)); + } + } + } + } + item_index++; + } + } + //TODO: Is it necessary to mark it as dirty? Maybe not. + cur_page_id = next_page_id; + level_page_index++; + } + if (OB_SUCC(ret) && evict_page_count != array_index) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected array_index", KR(ret), K(fd_), K(array_index), K(evict_pages), K(level_page_range_array_)); + } + } + } else { //level == level_count + if (OB_UNLIKELY(!is_page_in_write_cache(root_item_) + || !is_page_flushed(root_item_) + || 1 != evict_page_count + || evict_pages.at(0) != root_item_.buffer_page_id_ + || 0 != start_page_index_in_level)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected root_item_ or evict_pages", KR(ret), K(fd_), K(root_item_), + K(evict_pages), K(start_page_index_in_level)); + } else { + root_item_.buffer_page_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + } + } + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::clear( + const int64_t last_truncate_offset, + const int64_t total_file_size) +{ + int ret = OB_SUCCESS; + SpinWLockGuard guard(lock_); + int64_t end_truncate_offset = upper_align(total_file_size, ObTmpFileGlobal::PAGE_SIZE); + + if (OB_UNLIKELY(last_truncate_offset < released_offset_ + || last_truncate_offset > total_file_size + || is_writing_ + || 0 < stat_info_.meta_page_flushing_cnt_)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(last_truncate_offset), K(total_file_size), KPC(this)); + } else if (OB_FAIL(truncate_(end_truncate_offset))) { + STORAGE_LOG(WARN, "fail to truncate_", KR(ret), K(last_truncate_offset), K(end_truncate_offset), K(total_file_size), KPC(this)); + } else if (OB_UNLIKELY(root_item_.is_valid() + || !level_page_range_array_.empty() + || !data_item_array_.empty())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(ERROR, "unexpected root_item_ or array_", KR(ret), KPC(this)); + } else if (OB_UNLIKELY(stat_info_.all_type_page_flush_cnt_ != stat_info_.all_type_flush_page_released_cnt_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(ERROR, "unexpected stat_info_", KR(ret), KPC(this)); + } + return ret; +} + +//clear leaf page or internal page in meta tree +int ObSharedNothingTmpFileMetaTree::release_meta_page_( + const ObSharedNothingTmpFileMetaItem &page_info, + const int32_t page_index_in_level) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(level_page_range_array_.count() <= page_info.page_level_ + || 0 > page_index_in_level)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected level_page_range_array_ or page_info", KR(ret), K(fd_), K(level_page_range_array_), + K(page_info), K(page_index_in_level)); + } else if (is_page_in_write_cache(page_info)) { + const uint32_t start_page_id_in_array = level_page_range_array_.at(page_info.page_level_).start_page_id_; + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + if (OB_FAIL(wbp_->free_page(fd_, page_info.buffer_page_id_, + ObTmpFilePageUniqKey(page_info.page_level_, page_index_in_level), next_page_id))) { + STORAGE_LOG(WARN, "fail to free meta page in write cache", KR(ret), K(fd_), K(page_info), K(page_index_in_level)); + } else if (OB_UNLIKELY(start_page_id_in_array != page_info.buffer_page_id_)) { + //NOTE: pages must be released sequentially (from front to back in array) + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected level_page_range_array_", KR(ret), K(fd_), K(level_page_range_array_), K(page_info)); + } else { + level_page_range_array_.at(page_info.page_level_).start_page_id_ = next_page_id; + if (start_page_id_in_array == level_page_range_array_.at(page_info.page_level_).end_page_id_) { + //next_page_id must be invalid + level_page_range_array_.at(page_info.page_level_).end_page_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + } + if (ObTmpFileGlobal::INVALID_PAGE_ID == level_page_range_array_.at(page_info.page_level_).flushed_end_page_id_) { + level_page_range_array_.at(page_info.page_level_).flushed_page_num_ += 1; + } + if (start_page_id_in_array == level_page_range_array_.at(page_info.page_level_).flushed_end_page_id_) { + level_page_range_array_.at(page_info.page_level_).flushed_end_page_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + } + level_page_range_array_.at(page_info.page_level_).cached_page_num_ -= 1; + level_page_range_array_.at(page_info.page_level_).evicted_page_num_ += 1; + } + } + if (OB_SUCC(ret) && is_page_flushed(page_info)) { + if (OB_FAIL(release_tmp_file_page_(page_info.block_index_, page_info.physical_page_id_, 1))) { + STORAGE_LOG(WARN, "fail to release tmp file page", KR(ret), K(fd_), K(page_info)); + } + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::release_tmp_file_page_( + const int64_t block_index, const int64_t begin_page_id, const int64_t page_num) +{ + int ret = OB_SUCCESS; + + // XXX 最终需要消除对 MTL 的依赖； + ObTmpFileBlockManager &block_manager = MTL(ObTenantTmpFileManager*)->get_tmp_file_block_manager(); + if (OB_FAIL(block_manager.release_tmp_file_page(block_index, begin_page_id, page_num))) { + STORAGE_LOG(WARN, "fail to release tmp file page", + KR(ret), K(fd_), K(block_index), K(begin_page_id), K(page_num)); + } else { + stat_info_.all_type_flush_page_released_cnt_ += page_num; + } + + return ret; +} + +int ObSharedNothingTmpFileMetaTree::truncate( + const int64_t last_truncate_offset, + const int64_t end_truncate_offset) +{ + int ret = OB_SUCCESS; + SpinWLockGuard guard(lock_); + if (OB_UNLIKELY(last_truncate_offset < released_offset_ + || 0 != released_offset_ % ObTmpFileGlobal::PAGE_SIZE + || last_truncate_offset > end_truncate_offset + || is_writing_)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(last_truncate_offset), K(end_truncate_offset), KPC(this)); + } else if (OB_FAIL(truncate_(end_truncate_offset))) { + STORAGE_LOG(WARN, "fail to truncate_", KR(ret), K(last_truncate_offset), K(end_truncate_offset), KPC(this)); + } else if (!root_item_.is_valid()) { + //NOTE: end_truncate_offset >= max offset in tree. + // released_offset_ = end_truncate_offset; + } + STORAGE_LOG(INFO, "finish truncate array or meta tree", KR(ret), K(fd_), K(released_offset_), K(last_truncate_offset), K(end_truncate_offset)); + return ret; +} + +int ObSharedNothingTmpFileMetaTree::truncate_( + const int64_t end_truncate_offset) +{ + int ret = OB_SUCCESS; + if (!root_item_.is_valid()) { + if (OB_FAIL(truncate_array_(end_truncate_offset))) { + STORAGE_LOG(WARN, "fail to truncate array", KR(ret), K(fd_), K(end_truncate_offset)); + } + } else { + //TODO: 可以将FULL_PAGE_META_ITEM_NUM作为类的成员吗？ + const int16_t FULL_PAGE_META_ITEM_NUM = + MIN(MAX_PAGE_ITEM_COUNT, (ObTmpFileGlobal::PAGE_SIZE - PAGE_HEADER_SIZE) / sizeof(ObSharedNothingTmpFileMetaItem)); + ObArray> item_index_arr; + if (OB_FAIL(calculate_truncate_index_path_(item_index_arr))) { + STORAGE_LOG(WARN, "fail to calculate truncate index path", KR(ret), K(fd_), K(item_index_arr)); + } else if (OB_UNLIKELY(item_index_arr.empty())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected item_index_arr", KR(ret), K(fd_), K(item_index_arr)); + } else { + ObSharedNothingTmpFileMetaItem meta_item = root_item_; + ObSharedNothingTmpFileMetaItem next_level_meta_item; + ObSEArray truncate_path; + while (OB_SUCC(ret) + && 0 < meta_item.page_level_) { + next_level_meta_item.reset(); + int16_t item_index = -1; + int32_t level_page_index = -1; + ObSharedNothingTmpFileTreePageHeader page_header; + char *page_buff = NULL; + ObTmpPageValueHandle p_handle; + if (OB_UNLIKELY(item_index_arr.count() <= meta_item.page_level_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected item_index_arr", KR(ret), K(fd_), K(item_index_arr), K(meta_item)); + } else if (FALSE_IT(level_page_index = item_index_arr.at(meta_item.page_level_).first)) { + } else if (FALSE_IT(item_index = item_index_arr.at(meta_item.page_level_).second)) { + } else if (OB_FAIL(get_page_(meta_item, level_page_index, page_buff, p_handle))) { + STORAGE_LOG(WARN, "fail to get page", KR(ret), K(fd_), K(meta_item)); + } else if (OB_FAIL(read_item_(page_buff, item_index, next_level_meta_item))) { + STORAGE_LOG(WARN, "fail to read item", KR(ret), K(fd_), KP(page_buff), K(item_index)); + } else if (OB_FAIL(truncate_path.push_back(BacktraceNode(meta_item /*page info*/, + level_page_index, /*page index in its level*/ + item_index /*item index on the page*/)))) { + STORAGE_LOG(WARN, "fail to push back", KR(ret), K(fd_), K(meta_item)); + } else { + meta_item = next_level_meta_item; + } + p_handle.reset(); + } + if (OB_SUCC(ret)) { + bool release_last_item = false; + if (0 == meta_item.page_level_) { + if (OB_FAIL(release_items_of_leaf_page_(meta_item, item_index_arr.at(0).first, end_truncate_offset, + item_index_arr.at(0).second, release_last_item))) { + STORAGE_LOG(WARN, "fail to release items of leaf page", KR(ret), K(fd_), K(meta_item), + K(end_truncate_offset), K(item_index_arr)); + } else if (release_last_item) { + if (OB_FAIL(release_meta_page_(meta_item, item_index_arr.at(0).first))) { + STORAGE_LOG(WARN, "fail to release meta page", KR(ret), K(fd_), K(meta_item), K(item_index_arr.at(0).first)); + //even release_offset == end_offset,We won't end here, because the upper-level pages may need to be released + } else if (OB_FAIL(backtrace_truncate_tree_(end_truncate_offset, truncate_path))) { + STORAGE_LOG(WARN, "fail to backtrace truncate tree", KR(ret), K(fd_), K(end_truncate_offset), K(truncate_path)); + } + } + } else { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected page type", KR(ret), K(fd_), K(meta_item)); + } + } + } + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::truncate_array_( + const int64_t end_offset) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(end_offset < 0 + || data_item_array_.count() > MAX_DATA_ITEM_ARRAY_COUNT)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), K(end_offset), K(data_item_array_)); + } else if (data_item_array_.empty()) { + STORAGE_LOG(INFO, "no data to truncate", KR(ret), K(fd_), K(data_item_array_), K(root_item_)); + } else { + const ObSharedNothingTmpFileDataItem &last_item = data_item_array_.at(data_item_array_.count() - 1); + if (OB_UNLIKELY(!last_item.is_valid())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected last item", KR(ret), K(fd_), K(last_item)); + } else if (end_offset >= (last_item.virtual_page_id_ + last_item.physical_page_num_) * ObTmpFileGlobal::PAGE_SIZE) { + //clear all items + ARRAY_FOREACH_N(data_item_array_, i, cnt) { + const ObSharedNothingTmpFileDataItem &item = data_item_array_.at(i); + if (OB_FAIL(release_tmp_file_page_(item.block_index_, + item.physical_page_id_, item.physical_page_num_))) { + STORAGE_LOG(WARN, "fail to release tmp file page", KR(ret), K(fd_), K(item), K(i), K(cnt)); + } else if (i + 1 == cnt) { + released_offset_ = (item.virtual_page_id_ + item.physical_page_num_) * ObTmpFileGlobal::PAGE_SIZE; + } + } + if (OB_SUCC(ret)) { + data_item_array_.reset(); + } + } else { + //clear some items + const int64_t target_virtual_page_id = end_offset / ObTmpFileGlobal::PAGE_SIZE; + int16_t index = 0; + ARRAY_FOREACH_N(data_item_array_, i, cnt) { + const ObSharedNothingTmpFileDataItem &data_item = data_item_array_.at(i); + if (OB_UNLIKELY(!data_item.is_valid())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected data item", KR(ret), K(fd_), K(data_item)); + } else if (data_item.virtual_page_id_ <= target_virtual_page_id) { + index = i; + } else { + break; + } + } + if (OB_SUCC(ret)) { + if (0 == index) { + //do nothing + } else { + //release items before "index" + int16_t array_cnt = data_item_array_.count(); + for (int16_t i = 0; OB_SUCC(ret) && i < index; i++) { + ObSharedNothingTmpFileDataItem &data_item = data_item_array_.at(i); + if (OB_UNLIKELY(!data_item.is_valid())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected data item", KR(ret), K(fd_), K(data_item)); + } else if (OB_FAIL(release_tmp_file_page_(data_item.block_index_, + data_item.physical_page_id_, + data_item.physical_page_num_))) { + STORAGE_LOG(WARN, "fail to release tmp file page", KR(ret), K(fd_), K(data_item), K(i), K(index)); + } else if (i + 1 == index) { + released_offset_ = (data_item.virtual_page_id_ + data_item.physical_page_num_) * ObTmpFileGlobal::PAGE_SIZE; + } else { + data_item.reset(); + } + } + if (OB_SUCC(ret)) { + //move items + for (int16_t i = index; i < array_cnt; i++) { + data_item_array_.at(i - index) = data_item_array_.at(i); + } + for (int16_t i = 0; i < index; i++) { + data_item_array_.pop_back(); + } + } + } + } + } + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::calculate_truncate_index_path_( + ObIArray> &item_index_arr) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!root_item_.is_valid())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected root_item_", KR(ret), K(fd_), K(root_item_)); + } else { + const int16_t FULL_PAGE_META_ITEM_NUM = + MIN(MAX_PAGE_ITEM_COUNT, (ObTmpFileGlobal::PAGE_SIZE - PAGE_HEADER_SIZE) / sizeof(ObSharedNothingTmpFileMetaItem)); + int32_t child_level_page_index = -1; + ARRAY_FOREACH_N(level_page_range_array_, i, cnt) { + int16_t item_index = -1; + if (!last_truncate_leaf_info_.is_valid()) { + child_level_page_index = 0; + item_index = 0; + } else { + int32_t level_page_index = -1; + int32_t level_page_num = level_page_range_array_.at(i).evicted_page_num_ + level_page_range_array_.at(i).cached_page_num_; + if (0 == i) { + level_page_index = last_truncate_leaf_info_.page_index_in_leaf_level_; + item_index = last_truncate_leaf_info_.item_index_in_page_ + 1; + if (last_truncate_leaf_info_.release_to_end_in_page_) { + level_page_index++; + item_index = 0; + } + } else { //internal level + item_index = child_level_page_index % FULL_PAGE_META_ITEM_NUM; + level_page_index = child_level_page_index / FULL_PAGE_META_ITEM_NUM; + } + if (level_page_index >= level_page_num) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "no data to truncate but root_item_ is valid", KR(ret), K(fd_), + K(i), K(level_page_index), K(level_page_num), K(level_page_range_array_)); + } else { + child_level_page_index = level_page_index; + } + } + if (FAILEDx(item_index_arr.push_back(std::make_pair(child_level_page_index, item_index)))) { + STORAGE_LOG(WARN, "fail to push back", KR(ret), K(fd_), K(child_level_page_index), K(item_index)); + } + } + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::release_items_of_leaf_page_( + const ObSharedNothingTmpFileMetaItem &page_info, + const int32_t level_page_index, + const int64_t end_offset, + const int16_t begin_release_index, + bool &release_last_item) +{ + int ret = OB_SUCCESS; + release_last_item = false; + char *page_buff = NULL; + int16_t item_index = -1; + int16_t end_release_index = -1; + int64_t tmp_release_offset = released_offset_; + ObSharedNothingTmpFileDataItem data_item; + ObSharedNothingTmpFileTreePageHeader page_header; + ObTmpPageValueHandle p_handle; + if (OB_UNLIKELY(0 != tmp_release_offset % ObTmpFileGlobal::PAGE_SIZE + || 0 > begin_release_index + || PAGE_HEADER_SIZE + (begin_release_index + 1) * sizeof(ObSharedNothingTmpFileDataItem) > ObTmpFileGlobal::PAGE_SIZE + || begin_release_index >= MAX_PAGE_ITEM_COUNT)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected release_offset or begin_release_index", KR(ret), K(fd_), K(tmp_release_offset), K(begin_release_index)); + } else if (OB_FAIL(get_page_(page_info, level_page_index, page_buff, p_handle))) { + STORAGE_LOG(WARN, "fail to get page", KR(ret), K(fd_), K(page_info), K(level_page_index)); + } else if (OB_FAIL(read_page_header_(page_buff, page_header))) { + STORAGE_LOG(WARN, "fail to read page header", KR(ret), K(fd_), KP(page_buff)); + } else if (0 == page_header.item_num_ || begin_release_index == page_header.item_num_) { + //maybe the item have been cleared (corresponds to an unfilled data page) + release_last_item = true; + } else { + //get end_release_index + int64_t target_virtual_page_id = end_offset / ObTmpFileGlobal::PAGE_SIZE; + item_index = -1; + data_item.reset(); + if (OB_FAIL(read_item_(page_buff, target_virtual_page_id, item_index, data_item))) { + STORAGE_LOG(WARN, "fail to read item", KR(ret), K(fd_), KP(page_buff), K(target_virtual_page_id)); + } else if (OB_UNLIKELY(!data_item.is_valid())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected data item", KR(ret), K(fd_), K(data_item)); + } else if (target_virtual_page_id >= data_item.virtual_page_id_ + data_item.physical_page_num_) { + if (OB_UNLIKELY(item_index + 1 != page_header.item_num_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected item_index", KR(ret), K(fd_), K(item_index)); + } else { + end_release_index = item_index; + release_last_item = true; + } + } else { + end_release_index = item_index - 1; + } + if (OB_SUCC(ret)) { + item_index = begin_release_index; + while (OB_SUCC(ret) + && item_index <= end_release_index) { + data_item.reset(); + if (OB_FAIL(read_item_(page_buff, item_index, data_item))) { + STORAGE_LOG(WARN, "fail to read item", KR(ret), K(fd_), KP(page_buff), K(item_index)); + } else if (OB_UNLIKELY(!data_item.is_valid())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected data item", KR(ret), K(fd_), K(data_item)); + } else if (OB_FAIL(release_tmp_file_page_(data_item.block_index_, + data_item.physical_page_id_, + data_item.physical_page_num_))) { + STORAGE_LOG(WARN, "fail to release tmp file page", KR(ret), K(fd_), K(data_item), + K(begin_release_index), K(item_index), K(end_release_index)); + } else { + item_index++; + } + } + } + if (OB_SUCC(ret) && begin_release_index <= end_release_index) { + tmp_release_offset = (data_item.virtual_page_id_ + data_item.physical_page_num_) * ObTmpFileGlobal::PAGE_SIZE; + if (OB_UNLIKELY(tmp_release_offset > end_offset + || tmp_release_offset <= released_offset_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected release_offset", KR(ret), K(fd_), K(tmp_release_offset), K(end_offset), K(released_offset_)); + } else { + released_offset_ = tmp_release_offset; + last_truncate_leaf_info_.page_index_in_leaf_level_ = level_page_index; + last_truncate_leaf_info_.item_index_in_page_ = end_release_index; + last_truncate_leaf_info_.release_to_end_in_page_ = release_last_item; + } + } + if (OB_FAIL(ret)) { + //print page content + //NOTE: control print frequence + ObSharedNothingTmpFileTreePageHeader tmp_page_header; + ObArray items; + read_page_content_(page_buff, tmp_page_header, items); + STORAGE_LOG(WARN, "fail to release leaf page items, dump tree page", KR(ret), K(fd_), KP(page_buff), K(page_info), + K(level_page_index), K(begin_release_index), K(end_release_index), K(end_offset), K(tmp_page_header), K(items)); + } + p_handle.reset(); + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::backtrace_truncate_tree_( + const int64_t end_offset, + ObIArray &search_path) +{ + int ret = OB_SUCCESS; + const int16_t FULL_PAGE_META_ITEM_NUM = + MIN(MAX_PAGE_ITEM_COUNT, (ObTmpFileGlobal::PAGE_SIZE - PAGE_HEADER_SIZE) / sizeof(ObSharedNothingTmpFileMetaItem)); + ObSEArray meta_items; + bool need_finish = false; + while (OB_SUCC(ret) + && !search_path.empty() + && !need_finish) { + meta_items.reset(); + bool reach_last_item = false; + int64_t last_node_index = search_path.count() - 1; + //meta_item points to a page + const ObSharedNothingTmpFileMetaItem &page_info = search_path.at(last_node_index).page_meta_info_; + //the page index in its level + const int32_t level_page_index = search_path.at(last_node_index).level_page_index_; + //current pos need to be processed on this page. + int16_t cur_item_index = search_path.at(last_node_index).prev_item_index_ + 1; + if (OB_FAIL(get_items_of_internal_page_(page_info, level_page_index, cur_item_index, + end_offset, reach_last_item, meta_items))) { + STORAGE_LOG(WARN, "fail to get items of internal page", KR(ret), K(fd_), K(page_info), + K(level_page_index), K(cur_item_index), K(end_offset)); + } else if (1 == page_info.page_level_) { + int16_t release_page_cnt = 0; + const int32_t start_leaf_level_page_index = level_page_index * FULL_PAGE_META_ITEM_NUM + cur_item_index; + ARRAY_FOREACH_X(meta_items, i, cnt, OB_SUCC(ret) && !need_finish) { + bool release_last_item = false; + if (OB_FAIL(release_items_of_leaf_page_(meta_items.at(i), start_leaf_level_page_index + i, end_offset, + 0 /*release from start of the page*/, release_last_item))) { + STORAGE_LOG(WARN, "fail to release items of leaf page", KR(ret), K(fd_), K(meta_items.at(i)), + K(start_leaf_level_page_index + i), K(end_offset)); + } else if (release_last_item) { + if (OB_FAIL(release_meta_page_(meta_items.at(i), start_leaf_level_page_index + i))) { + STORAGE_LOG(WARN, "fail to release meta page", KR(ret), K(fd_), K(meta_items.at(i)), + K(start_leaf_level_page_index + i)); + } else { + release_page_cnt++; + } + } else { + need_finish = true; + } + } + if (OB_SUCC(ret)) { + if (!reach_last_item) { + need_finish = true; + } else if (reach_last_item && (meta_items.empty() || release_page_cnt == meta_items.count())) { + if (OB_FAIL(release_meta_page_(page_info, level_page_index))) { + STORAGE_LOG(WARN, "fail to release meta page", KR(ret), K(fd_), K(page_info), K(level_page_index)); + } else { + search_path.pop_back(); + } + } + } + } else if (1 < page_info.page_level_) { + if (meta_items.count() == 1) { + const int32_t child_level_page_index = level_page_index * FULL_PAGE_META_ITEM_NUM + cur_item_index; + search_path.at(last_node_index).prev_item_index_ = cur_item_index; + if (OB_FAIL(search_path.push_back(BacktraceNode(meta_items.at(0), child_level_page_index, -1)))) { + STORAGE_LOG(WARN, "fail to push back", KR(ret), K(fd_), K(meta_items.at(0)), K(child_level_page_index)); + } + } else if (meta_items.empty()) { + if (reach_last_item) { + if (OB_FAIL(release_meta_page_(page_info, level_page_index))) { + STORAGE_LOG(WARN, "fail to release meta page", KR(ret), K(fd_), K(page_info), K(level_page_index)); + } else { + search_path.pop_back(); + } + } else { + need_finish = true; + } + } else { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected meta_items", KR(ret), K(fd_), K(meta_items)); + } + } else { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected page info", KR(ret), K(fd_), K(page_info)); + } + } + if (OB_SUCC(ret) && !need_finish) { + //all pages are released + if (OB_FAIL(check_tree_is_empty_())) { + STORAGE_LOG(WARN, "unexpected, tree is not empty", KR(ret), K(fd_)); + } else if (OB_UNLIKELY(stat_info_.all_type_page_flush_cnt_ != stat_info_.all_type_flush_page_released_cnt_)) { + //TODO: do not throw errors in the future + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected stat_info_", KR(ret), K(fd_), K(stat_info_)); + } else { + ++tree_epoch_; + root_item_.reset(); + level_page_range_array_.reset(); + last_truncate_leaf_info_.reset(); + STORAGE_LOG(INFO, "meta tree pages are all truncated", KR(ret), K(fd_), K(tree_epoch_)); + } + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::check_tree_is_empty_() +{ + int ret = OB_SUCCESS; + ARRAY_FOREACH_N(level_page_range_array_, i, cnt) { + if (OB_UNLIKELY(0 != level_page_range_array_.at(i).cached_page_num_ + || ObTmpFileGlobal::INVALID_PAGE_ID != level_page_range_array_.at(i).start_page_id_ + || ObTmpFileGlobal::INVALID_PAGE_ID != level_page_range_array_.at(i).end_page_id_ + || ObTmpFileGlobal::INVALID_PAGE_ID != level_page_range_array_.at(i).flushed_end_page_id_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected level_page_range_array_", KR(ret), K(fd_), K(level_page_range_array_), K(i)); + } + } + return ret; +} + +//total_need_flush_rightmost_page_num will not consider "is_writing_ = true" or "ObTmpFileTreeEvictType" +int ObSharedNothingTmpFileMetaTree::get_need_flush_page_num( + int64_t &total_need_flush_page_num, + int64_t &total_need_flush_rightmost_page_num) const +{ + int ret = OB_SUCCESS; + total_need_flush_page_num = 0; + total_need_flush_rightmost_page_num = 0; + SpinRLockGuard guard(lock_); + ARRAY_FOREACH_N(level_page_range_array_, level, level_cnt) { + const uint32_t start_page_id = level_page_range_array_[level].start_page_id_; + const uint32_t flushed_end_page_id = level_page_range_array_[level].flushed_end_page_id_; + const uint32_t end_page_id = level_page_range_array_[level].end_page_id_; + if (ObTmpFileGlobal::INVALID_PAGE_ID != start_page_id) { + if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_PAGE_ID == end_page_id)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected end_page_id", KR(ret), K(fd_), K(level_page_range_array_), K(level)); + } else { + uint32_t cur_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + int32_t level_page_index = level_page_range_array_[level].flushed_page_num_; + if (ObTmpFileGlobal::INVALID_PAGE_ID == flushed_end_page_id) { + cur_page_id = start_page_id; + } else { + cur_page_id = flushed_end_page_id; + level_page_index--; + } + while (OB_SUCC(ret) + && ObTmpFileGlobal::INVALID_PAGE_ID != cur_page_id) { + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ObTmpFilePageUniqKey page_key(level, level_page_index); + if (wbp_->is_dirty(fd_, cur_page_id, page_key)) { + total_need_flush_page_num++; + if (end_page_id == cur_page_id) { + total_need_flush_rightmost_page_num++; + break; + } + } + if (OB_FAIL(wbp_->get_next_page_id(fd_, cur_page_id, page_key, next_page_id))) { + STORAGE_LOG(WARN, "fail to get next meta page id", KR(ret), K(fd_), K(cur_page_id), K(page_key)); + } else { + cur_page_id = next_page_id; + level_page_index++; + } + } + if (OB_SUCC(ret) && end_page_id != cur_page_id && ObTmpFileGlobal::INVALID_PAGE_ID != cur_page_id) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected cur_page_id", KR(ret), K(fd_), K(cur_page_id), + K(level_page_range_array_), K(level)); + } + } + } + } + return ret; +} + +//total_need_evict_rightmost_page_num will not consider "is_writing_ = true" or "ObTmpFileTreeEvictType" +int ObSharedNothingTmpFileMetaTree::get_need_evict_page_num( + int64_t &total_need_evict_page_num, + int64_t &total_need_evict_rightmost_page_num) const +{ + int ret = OB_SUCCESS; + total_need_evict_page_num = 0; + total_need_evict_rightmost_page_num = 0; + SpinRLockGuard guard(lock_); + ARRAY_FOREACH_N(level_page_range_array_, level, level_cnt) { + const uint32_t start_page_id = level_page_range_array_[level].start_page_id_; + const uint32_t flushed_end_page_id = level_page_range_array_[level].flushed_end_page_id_; + const uint32_t end_page_id = level_page_range_array_[level].end_page_id_; + if (ObTmpFileGlobal::INVALID_PAGE_ID != flushed_end_page_id) { + if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_PAGE_ID == start_page_id + || ObTmpFileGlobal::INVALID_PAGE_ID == end_page_id)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected start_page_id or end_page_id", KR(ret), K(fd_), K(level_page_range_array_), K(level)); + } else { + uint32_t cur_page_id = start_page_id; + int32_t level_page_index = level_page_range_array_[level].evicted_page_num_; + while (OB_SUCC(ret) + && ObTmpFileGlobal::INVALID_PAGE_ID != cur_page_id) { + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ObTmpFilePageUniqKey page_key(level, level_page_index); + if (OB_UNLIKELY(flushed_end_page_id != cur_page_id && !wbp_->is_cached(fd_, cur_page_id, page_key))) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected cur_page_id", KR(ret), K(fd_), K(cur_page_id), K(flushed_end_page_id)); + } else if (!wbp_->is_cached(fd_, cur_page_id, page_key)) { + break; + } else if (FALSE_IT(total_need_evict_page_num++)) { + } else if (end_page_id == cur_page_id && FALSE_IT(total_need_evict_rightmost_page_num++)) { + } else if (flushed_end_page_id == cur_page_id) { + break; + } else if (OB_FAIL(wbp_->get_next_page_id(fd_, cur_page_id, page_key, next_page_id))) { + STORAGE_LOG(WARN, "fail to get next meta page id", KR(ret), K(fd_), K(cur_page_id), K(page_key)); + } else { + cur_page_id = next_page_id; + level_page_index++; + } + } + if (OB_SUCC(ret) && flushed_end_page_id != cur_page_id) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected cur_page_id", KR(ret), K(fd_), K(cur_page_id), + K(level_page_range_array_), K(level)); + } + } + } + } + return ret; +} + +//NOTE: 这个页被取出来时，并没有加锁，到时候可能需要为write_cache的读写单独弄一把锁 +int ObSharedNothingTmpFileMetaTree::get_page_( + const ObSharedNothingTmpFileMetaItem &page_info, + const int32_t level_page_index, + char *&page_buff, + ObTmpPageValueHandle &p_handle) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!page_info.is_valid())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), K(page_info)); + } else { + if (!is_page_in_write_cache(page_info)) { + ObTmpPageCacheKey key(page_info.block_index_, page_info.physical_page_id_, MTL_ID()); + if (OB_SUCC(ObTmpPageCache::get_instance().get_page(key, p_handle))) { + page_buff = p_handle.value_->get_buffer(); + } else if (OB_ENTRY_NOT_EXIST != ret) { + STORAGE_LOG(WARN, "fail to read from read_cache", KR(ret), K(fd_), K(key)); + } else { + ret = OB_SUCCESS; + if (OB_FAIL(ObTmpPageCache::get_instance().load_page(key, callback_allocator_, p_handle))) { + STORAGE_LOG(WARN, "fail to load page from disk", KR(ret), K(fd_), K(key)); + } else { + page_buff = p_handle.value_->get_buffer(); + } + } + if (FAILEDx(check_page_(page_buff))) { + STORAGE_LOG(WARN, "the page is invalid or corrupted", KR(ret), K(fd_), KP(page_buff)); + } + } else { + //still in write cache + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ObTmpFilePageUniqKey page_key(page_info.page_level_, level_page_index); + if (OB_FAIL(wbp_->read_page(fd_, page_info.buffer_page_id_, page_key, page_buff, next_page_id))) { + STORAGE_LOG(WARN, "fail to read from write cache", KR(ret), K(fd_), K(page_info), K(page_key)); + } + } + if (OB_SUCC(ret) && OB_ISNULL(page_buff)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected page_buff", KR(ret), K(fd_), KP(page_buff)); + } + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::check_page_(const char* const page_buff) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(page_buff)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), KP(page_buff)); + } else { + ObSharedNothingTmpFileTreePageHeader page_header = *((ObSharedNothingTmpFileTreePageHeader *)(page_buff)); + const uint64_t checksum = ob_crc64(page_buff + PAGE_HEADER_SIZE, ObTmpFileGlobal::PAGE_SIZE - PAGE_HEADER_SIZE); + if (OB_UNLIKELY(PAGE_MAGIC_NUM != page_header.magic_number_ + || checksum != page_header.checksum_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(ERROR, "checksum or magic_number is not equal, page is invalid", KR(ret), K(fd_), + KP(page_buff), K(page_header), K(checksum), K(PAGE_MAGIC_NUM)); + } + } + return ret; +} + +//get page and put into write cache (if not exists) +//put page_id into level_page_range_array_ +int ObSharedNothingTmpFileMetaTree::cache_page_for_write_( + const uint32_t parent_page_id, + ObSharedNothingTmpFileMetaItem &page_info) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!page_info.is_valid() + || page_info.page_level_ >= level_page_range_array_.count() + || (ObTmpFileGlobal::INVALID_PAGE_ID != parent_page_id + && page_info.page_level_ + 1 >= level_page_range_array_.count()))) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), K(page_info), K(level_page_range_array_), K(parent_page_id)); + } else { + if (!is_page_in_write_cache(page_info)) { + uint32_t new_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + char *new_page_buff = NULL; + int32_t level_page_index = level_page_range_array_[page_info.page_level_].evicted_page_num_ - 1; + ObTmpFilePageUniqKey page_key(page_info.page_level_, level_page_index); + if (OB_UNLIKELY(!is_page_flushed(page_info) + || 0 < level_page_range_array_[page_info.page_level_].cached_page_num_ + || 0 > level_page_index + || ObTmpFileGlobal::INVALID_PAGE_ID != level_page_range_array_[page_info.page_level_].end_page_id_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected page_info", KR(ret), K(fd_), K(page_info), K(level_page_range_array_)); + } else if (OB_FAIL(wbp_->alloc_page(fd_, page_key, new_page_id, new_page_buff))) { + STORAGE_LOG(WARN, "fail to alloc meta page", KR(ret), K(fd_), K(page_info), K(level_page_range_array_)); + } else if (OB_ISNULL(new_page_buff)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected null page buff", KR(ret), K(fd_), KP(new_page_buff)); + } else if (OB_FAIL(wbp_->notify_load(fd_, new_page_id, page_key))) { + STORAGE_LOG(WARN, "fail to notify load for meta", KR(ret), K(fd_), K(new_page_id), K(page_key)); + } else { + ObTmpPageValueHandle p_handle; + ObTmpPageCacheKey key(page_info.block_index_, page_info.physical_page_id_, MTL_ID()); + if (OB_SUCC(ObTmpPageCache::get_instance().get_page(key, p_handle))) { + MEMCPY(new_page_buff, p_handle.value_->get_buffer(), ObTmpFileGlobal::PAGE_SIZE); + } else if (OB_ENTRY_NOT_EXIST != ret) { + STORAGE_LOG(WARN, "fail to read from read_cache", KR(ret), K(fd_), K(key)); + } else { + ret = OB_SUCCESS; + p_handle.reset(); + if (OB_FAIL(ObTmpPageCache::get_instance().load_page(key, callback_allocator_, p_handle))) { + STORAGE_LOG(WARN, "fail to load page from disk", KR(ret), K(fd_), K(key)); + } else { + MEMCPY(new_page_buff, p_handle.value_->get_buffer(), ObTmpFileGlobal::PAGE_SIZE); + } + } + if (FAILEDx(check_page_(new_page_buff))) { + STORAGE_LOG(WARN, "the page is invalid or corrupted", KR(ret), K(fd_), KP(new_page_buff)); + } + if (OB_SUCC(ret)) { + //change page state to cached + if (OB_FAIL(wbp_->notify_load_succ(fd_, new_page_id, page_key))) { + STORAGE_LOG(WARN, "fail to notify load succ for meta", KR(ret), K(fd_), K(new_page_id), K(page_key)); + } + } else { + int tmp_ret = OB_SUCCESS; + //change page state to invalid + if (OB_TMP_FAIL(wbp_->notify_load_fail(fd_, new_page_id, page_key))) { + STORAGE_LOG(WARN, "fail to notify load fail for meta", KR(tmp_ret), K(fd_), K(new_page_id), K(page_key)); + } + } + p_handle.reset(); + } + if (OB_SUCC(ret)) { + int64_t origin_block_index = page_info.block_index_; + int16_t origin_physical_page_id = page_info.physical_page_id_; + page_info.buffer_page_id_ = new_page_id; + page_info.block_index_ = ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX; + page_info.physical_page_id_ = -1; + if (ObTmpFileGlobal::INVALID_PAGE_ID == parent_page_id) { + root_item_ = page_info; + } else { + char *parent_page_buff = NULL; + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ObSharedNothingTmpFileTreePageHeader page_header; + int32_t parent_level_page_index = level_page_range_array_[page_info.page_level_ + 1].evicted_page_num_ + + level_page_range_array_[page_info.page_level_ + 1].cached_page_num_ - 1; + ObTmpFilePageUniqKey parent_page_offset(page_info.page_level_ + 1, parent_level_page_index); + if (OB_FAIL(wbp_->read_page(fd_, parent_page_id, parent_page_offset, parent_page_buff, next_page_id))) { + STORAGE_LOG(WARN, "fail to read from write cache", KR(ret), K(fd_), K(parent_page_id), K(parent_page_offset)); + } else if (OB_FAIL(read_page_header_(parent_page_buff, page_header))) { + STORAGE_LOG(WARN, "fail to read page header", KR(ret), K(fd_), KP(parent_page_buff)); + } else if (OB_FAIL(rewrite_item_(parent_page_buff, page_header.item_num_ - 1, page_info))) { + STORAGE_LOG(WARN, "fail to rewrite item", KR(ret), K(fd_), K(page_header), K(page_info), KP(parent_page_buff)); + } else if (OB_FAIL(wbp_->notify_dirty(fd_, parent_page_id, parent_page_offset))) { + STORAGE_LOG(WARN, "fail to notify dirty for meta", KR(ret), K(fd_), K(parent_page_id), K(parent_page_offset)); + } + } + if (OB_SUCC(ret)) { + int16_t page_level = page_info.page_level_; + if (OB_FAIL(wbp_->notify_dirty(fd_, new_page_id, page_key))) { + STORAGE_LOG(WARN, "fail to notify dirty", KR(ret), K(fd_), K(new_page_id), K(page_key)); + } else if (OB_FAIL(release_tmp_file_page_(origin_block_index, origin_physical_page_id, 1))) { + STORAGE_LOG(WARN, "fail to release tmp file page", KR(ret), K(fd_), K(origin_block_index), K(origin_physical_page_id)); + } else { + level_page_range_array_[page_level].start_page_id_ = new_page_id; + level_page_range_array_[page_level].end_page_id_ = new_page_id; + level_page_range_array_[page_level].cached_page_num_++; + level_page_range_array_[page_level].evicted_page_num_--; + level_page_range_array_[page_level].flushed_page_num_--; + } + } + } else if (ObTmpFileGlobal::INVALID_PAGE_ID != new_page_id) { //fail + uint32_t unused_next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(wbp_->free_page(fd_, new_page_id, page_key, unused_next_page_id))) { + STORAGE_LOG(WARN, "fail to free meta page", KR(tmp_ret), K(fd_), K(new_page_id), K(page_key)); + } + } + STORAGE_LOG(INFO, "load page to write cache", KR(ret), K(fd_), K(page_info), K(page_key)); + } else { + //still in write cache + if (!is_page_in_write_cache(page_info)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected page_info", KR(ret), K(fd_), K(page_info)); + } + } + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::init_page_header_( + char* page_buff, + const int16_t page_level) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(page_buff)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), KP(page_buff)); + } else { + ObSharedNothingTmpFileTreePageHeader page_header; + page_header.item_num_ = 0; + page_header.page_level_ = page_level; + page_header.magic_number_ = PAGE_MAGIC_NUM; + MEMCPY(page_buff, &page_header, PAGE_HEADER_SIZE); + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::read_page_header_( + const char* page_buff, + ObSharedNothingTmpFileTreePageHeader &page_header) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(NULL == page_buff)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), KP(page_buff)); + } else { + page_header = *((ObSharedNothingTmpFileTreePageHeader *)(page_buff)); + } + return ret; +} + +int ObSharedNothingTmpFileMetaTree::remove_page_item_from_tail_( + char* page_buff, + const int16_t remove_item_num) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(NULL == page_buff + || 0 > remove_item_num)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), KP(page_buff), K(remove_item_num)); + } else { + ObSharedNothingTmpFileTreePageHeader page_header = *((ObSharedNothingTmpFileTreePageHeader *)(page_buff)); + page_header.item_num_ -= remove_item_num; + if (OB_UNLIKELY(page_header.item_num_ < 0)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected item_num", KR(ret), K(fd_), K(page_header)); + } else { + MEMCPY(page_buff, &page_header, PAGE_HEADER_SIZE); + } + } + return ret; +} + +template +int ObSharedNothingTmpFileMetaTree::read_item_( + const char* page_buff, + const int64_t target_virtual_page_id, + int16_t &item_index, + ItemType &item) +{ + int ret = OB_SUCCESS; + item_index = -1; + item.reset(); + ObSharedNothingTmpFileTreePageHeader page_header = *((ObSharedNothingTmpFileTreePageHeader *)(page_buff)); + int16_t item_num = page_header.item_num_; + if (OB_UNLIKELY(NULL == page_buff + || target_virtual_page_id < 0)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), KP(page_buff), K(target_virtual_page_id)); + } else if (OB_UNLIKELY(PAGE_HEADER_SIZE + item_num * sizeof(item) > ObTmpFileGlobal::PAGE_SIZE + || item_num > MAX_PAGE_ITEM_COUNT)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected item_num", KR(ret), K(fd_), K(page_header)); + } else { + int16_t left = 0; + int16_t right = item_num - 1; + const char *items_buff = page_buff + PAGE_HEADER_SIZE; + //find the last index less than or equal to the target_virtual_page_id + while (left <= right) { + int16_t mid = (left + right) / 2; + ItemType mid_item = *((ItemType *)(items_buff + mid * sizeof(item))); + if (OB_UNLIKELY(!mid_item.is_valid())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected mid item", KR(ret), K(fd_), K(mid_item)); + } else if (mid_item.virtual_page_id_ <= target_virtual_page_id) { + left = mid + 1; + } else { + right = mid - 1; + } + } + if (OB_UNLIKELY(right < 0)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected dichotomy result", KR(ret), K(fd_), K(right)); + } else { + item_index = right; + item = *((ItemType *)(items_buff + item_index * sizeof(item))); + if (OB_UNLIKELY(!item.is_valid())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected item", KR(ret), K(fd_), K(item)); + } else if (item.virtual_page_id_ > target_virtual_page_id) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected virtual_page_id", KR(ret), K(fd_), K(item), K(target_virtual_page_id)); + } + } + } + if (OB_FAIL(ret)) { + //print page content + ObSharedNothingTmpFileTreePageHeader tmp_page_header; + ItemType first_item; + ItemType last_item; + read_page_simple_content_(page_buff, tmp_page_header, first_item, last_item); + STORAGE_LOG(WARN, "fail to read item, dump tree page", KR(ret), K(fd_), KP(page_buff), K(target_virtual_page_id), + K(tmp_page_header), K(first_item), K(last_item)); + } + return ret; +} + +// XXX 增加 page 物理范围校验； +template +int ObSharedNothingTmpFileMetaTree::read_item_( + const char* page_buff, + const int16_t item_index, + ItemType &item) +{ + int ret = OB_SUCCESS; + item.reset(); + ObSharedNothingTmpFileTreePageHeader page_header = *((ObSharedNothingTmpFileTreePageHeader *)(page_buff)); + if (OB_UNLIKELY(NULL == page_buff + || item_index < 0 + || item_index >= page_header.item_num_ + || PAGE_HEADER_SIZE + page_header.item_num_ * sizeof(item) > ObTmpFileGlobal::PAGE_SIZE + || PAGE_HEADER_SIZE + (item_index + 1) * sizeof(item) > ObTmpFileGlobal::PAGE_SIZE)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), KP(page_buff), K(item_index), K(page_header)); + } else { + const char *items_buff = page_buff + PAGE_HEADER_SIZE; + item = *((ItemType *)(items_buff + item_index * sizeof(item))); + } + return ret; +} + +template +int ObSharedNothingTmpFileMetaTree::rewrite_item_( + char* page_buff, + const int16_t item_index, + const ItemType &item) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(NULL == page_buff + || item_index < 0 + || PAGE_HEADER_SIZE + (item_index + 1) * sizeof(item) > ObTmpFileGlobal::PAGE_SIZE + || !item.is_valid())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), KP(page_buff), K(item_index), K(item)); + } else { + char *items_buff = page_buff + PAGE_HEADER_SIZE; + MEMCPY(items_buff + item_index * sizeof(item), &item, sizeof(item)); + } + return ret; +} + +template +int ObSharedNothingTmpFileMetaTree::write_items_( + char* page_buff, + const ObIArray &items, + const int64_t begin_index, + int16_t &count) +{ + int ret = OB_SUCCESS; + count = 0; + if (OB_UNLIKELY(NULL == page_buff + || items.empty() + || begin_index < 0 + || begin_index >= items.count())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), KP(page_buff), K(items), K(begin_index)); + } else { + ObSharedNothingTmpFileTreePageHeader page_header = *((ObSharedNothingTmpFileTreePageHeader *)(page_buff)); + char *items_buff = page_buff + PAGE_HEADER_SIZE; + int16_t item_index = page_header.item_num_; + for (int64_t i = begin_index; i < items.count() + && PAGE_HEADER_SIZE + (item_index + 1) * sizeof(items.at(i)) <= ObTmpFileGlobal::PAGE_SIZE && item_index < MAX_PAGE_ITEM_COUNT; i++) { + MEMCPY(items_buff + item_index * sizeof(items.at(i)), &items.at(i), sizeof(items.at(i))); + item_index++; + count++; + } + page_header.item_num_ = item_index; + MEMCPY(page_buff, &page_header, PAGE_HEADER_SIZE); + } + return ret; +} + +void ObSharedNothingTmpFileMetaTree::print_meta_tree_overview_info() +{ + SpinRLockGuard guard(lock_); + STORAGE_LOG(INFO, "dump meta tree", KPC(this)); +} + +void ObSharedNothingTmpFileMetaTree::print_meta_tree_total_info() +{ + int ret = OB_SUCCESS; + ObArray data_items; + ObArray meta_items; + SpinRLockGuard guard(lock_); + STORAGE_LOG(INFO, "dump meta tree", KPC(this)); + ARRAY_FOREACH_N(level_page_range_array_, level, level_cnt) { + const uint32_t start_page_id = level_page_range_array_[level].start_page_id_; + const uint32_t end_page_id = level_page_range_array_[level].end_page_id_; + if (ObTmpFileGlobal::INVALID_PAGE_ID != start_page_id) { + uint32_t cur_page_id = start_page_id; + int32_t level_page_index = level_page_range_array_[level].evicted_page_num_; + while (OB_SUCC(ret) + && ObTmpFileGlobal::INVALID_PAGE_ID != cur_page_id) { + char *page_buff = NULL; + uint32_t next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + ObTmpFilePageUniqKey page_key(level, level_page_index); + if (OB_FAIL(wbp_->read_page(fd_, cur_page_id, page_key, page_buff, next_page_id))) { + STORAGE_LOG(WARN, "fail to read from write cache", KR(ret), K(fd_), K(cur_page_id), K(page_key)); + } else { + ObSharedNothingTmpFileTreePageHeader tmp_page_header; + if (0 == level) { + data_items.reset(); + read_page_content_(page_buff, tmp_page_header, data_items); + STORAGE_LOG(INFO, "dump cached leaf page", KR(ret), K(fd_), KP(page_buff), K(cur_page_id), K(page_key), K(tmp_page_header), K(data_items)); + } else { + meta_items.reset(); + read_page_content_(page_buff, tmp_page_header, meta_items); + STORAGE_LOG(INFO, "dump cached internal page", KR(ret), K(fd_), KP(page_buff), K(cur_page_id), K(page_key), K(tmp_page_header), K(meta_items)); + } + } + cur_page_id = next_page_id; + level_page_index++; + } + } + } +} + +template +void ObSharedNothingTmpFileMetaTree::read_page_content_( + const char *page_buff, + ObSharedNothingTmpFileTreePageHeader &page_header, + ObIArray &items) +{ + int ret = OB_SUCCESS; + items.reset(); + if (OB_ISNULL(page_buff)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), KP(page_buff)); + } else { + const char *items_buff = page_buff + PAGE_HEADER_SIZE; + int16_t index = 0; + page_header = *((ObSharedNothingTmpFileTreePageHeader *)(page_buff)); + while (OB_SUCC(ret) + && index < page_header.item_num_ + && PAGE_HEADER_SIZE + (index + 1) * sizeof(ItemType) <= ObTmpFileGlobal::PAGE_SIZE) { + ItemType item = *((ItemType *)(items_buff + index * sizeof(ItemType))); + if (OB_FAIL(items.push_back(item))) { + STORAGE_LOG(WARN, "fail to push back", KR(ret), K(fd_), K(item)); + } else { + index++; + } + } + } +} + +template +void ObSharedNothingTmpFileMetaTree::read_page_simple_content_( + const char *page_buff, + ObSharedNothingTmpFileTreePageHeader &page_header, + ItemType &first_item, + ItemType &last_item) +{ + int ret = OB_SUCCESS; + first_item.reset(); + last_item.reset(); + if (OB_ISNULL(page_buff)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", KR(ret), K(fd_), KP(page_buff)); + } else { + const char *items_buff = page_buff + PAGE_HEADER_SIZE; + page_header = *((ObSharedNothingTmpFileTreePageHeader *)(page_buff)); + if (OB_UNLIKELY(0 > page_header.item_num_ + || PAGE_HEADER_SIZE + page_header.item_num_ * sizeof(ItemType) > ObTmpFileGlobal::PAGE_SIZE)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected item_num_", KR(ret), K(fd_), KP(page_buff), K(page_header)); + } else if (0 == page_header.item_num_) { + } else { + first_item = *((ItemType *)(items_buff)); + last_item = *((ItemType *)(items_buff + (page_header.item_num_ - 1) * sizeof(ItemType))); + } + } +} + +} +} diff --git a/src/storage/tmp_file/ob_tmp_file_meta_tree.h b/src/storage/tmp_file/ob_tmp_file_meta_tree.h new file mode 100644 index 000000000..4abc7edad --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_meta_tree.h @@ -0,0 +1,537 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_STORAGE_BLOCKSSTABLE_OB_TMP_FILE_META_TREE_H_ +#define OCEANBASE_STORAGE_BLOCKSSTABLE_OB_TMP_FILE_META_TREE_H_ + +#include "deps/oblib/src/lib/container/ob_se_array.h" +#include "storage/tmp_file/ob_tmp_file_write_buffer_pool.h" +#include "storage/tmp_file/ob_tmp_file_global.h" + +namespace oceanbase +{ +namespace tmp_file +{ + +class ObTmpPageValueHandle; + +struct ObSharedNothingTmpFileMetaItem +{ +public: + ObSharedNothingTmpFileMetaItem() : + page_level_(-1), + physical_page_id_(-1), + buffer_page_id_(ObTmpFileGlobal::INVALID_PAGE_ID), + block_index_(ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX), + virtual_page_id_(-1) {} + bool is_valid() const + { + return page_level_ >= 0 + && virtual_page_id_ >= 0; + } + void reset() + { + page_level_ = -1; + physical_page_id_ = -1; + buffer_page_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + block_index_ = ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX; + virtual_page_id_ = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + } + int16_t page_level_; // child page level + int16_t physical_page_id_; // child page 在宏块内的page id + uint32_t buffer_page_id_; // child page 在写缓存中的page id + int64_t block_index_; // child page 对应的宏块block index + int64_t virtual_page_id_; // child page 对应的文件逻辑页号最小值（即索引） + TO_STRING_KV(K(page_level_), K(buffer_page_id_), K(physical_page_id_), + K(block_index_), K(virtual_page_id_)); +}; // 24B + +struct ObSharedNothingTmpFileDataItem +{ + ObSharedNothingTmpFileDataItem() : + block_index_(ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX), + physical_page_id_(-1), + physical_page_num_(0), + virtual_page_id_(-1) {} + bool operator!=(const ObSharedNothingTmpFileDataItem &other) const + { + return other.block_index_ != block_index_ + || other.physical_page_id_ != physical_page_id_ + || other.physical_page_num_ != physical_page_num_ + || other.virtual_page_id_ != virtual_page_id_; + } + bool is_valid() const + { + return ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX != block_index_ + && physical_page_id_ >= 0 + && physical_page_num_ > 0 //data item must be cleaned up if physical_page_num_ = 0 + && virtual_page_id_ >= 0; + } + void reset() + { + block_index_ = ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX; + physical_page_id_ = -1; + physical_page_num_ = 0; + virtual_page_id_ = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + } + int64_t block_index_; // 刷盘data extent所在的宏块block index + int16_t physical_page_id_; // 刷盘data extent在宏块内的起始page id + int16_t physical_page_num_; // 刷盘data extent在宏块内的连续page数量 + int64_t virtual_page_id_; // 刷盘data extent在整个文件中逻辑页号（偏移位置) + TO_STRING_KV(K(block_index_), K(physical_page_id_), + K(physical_page_num_), K(virtual_page_id_)); +}; // 24B + +struct ObSharedNothingTmpFileTreePageHeader +{ + ObSharedNothingTmpFileTreePageHeader() : + page_level_(-1), + item_num_(0), + magic_number_(0), + checksum_(0) {} + int16_t page_level_; + int16_t item_num_; + uint32_t magic_number_; + uint64_t checksum_; + TO_STRING_KV(K(page_level_), K(item_num_), K(magic_number_), K(checksum_)); +}; + +struct ObTmpFileTreeIOInfo +{ + ObTmpFileTreeIOInfo() :tree_epoch_(-1), block_index_(ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX), + flush_start_page_id_(ObTmpFileGlobal::INVALID_PAGE_ID), + flush_start_level_page_index_(-1), + flush_end_page_id_(ObTmpFileGlobal::INVALID_PAGE_ID), + physical_start_page_id_(-1), + flush_nums_(0), + page_level_(-1) {} + void reset() + { + tree_epoch_ = -1; + block_index_ = ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX; + flush_start_page_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + flush_start_level_page_index_ = -1; + flush_end_page_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + physical_start_page_id_ = -1; + flush_nums_ = 0; + page_level_ = -1; + } + bool is_valid() const + { + return 0 <= tree_epoch_ + && ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX != block_index_ + && ObTmpFileGlobal::INVALID_PAGE_ID != flush_start_page_id_ + && 0 <= flush_start_level_page_index_ + && ObTmpFileGlobal::INVALID_PAGE_ID != flush_end_page_id_ + && physical_start_page_id_ >= 0 + && flush_nums_ > 0 + && page_level_ >= 0; + } + int64_t tree_epoch_; + int64_t block_index_; //刷脏页所在的block index + uint32_t flush_start_page_id_; //刷脏起始页在写缓存中的page_id + int32_t flush_start_level_page_index_; //刷脏起始页在其所在层的 page_index + uint32_t flush_end_page_id_; //刷脏最后一个页在写缓存中的page_id + int16_t physical_start_page_id_; //这批页在block中的start page id（物理偏移） + int16_t flush_nums_; //刷脏的页数 + int16_t page_level_; //刷脏的页所属的元数据树层次 + TO_STRING_KV(K(tree_epoch_), K(block_index_), K(flush_start_page_id_), + K(flush_start_level_page_index_), K(flush_end_page_id_), + K(physical_start_page_id_), K(flush_nums_), K(page_level_)); +}; + +struct ObTmpFileTreeFlushContext +{ + ObTmpFileTreeFlushContext() : tree_epoch_(-1), + is_meta_reach_end_(false), + last_flush_level_(-1), + last_flush_page_id_(ObTmpFileGlobal::INVALID_PAGE_ID), + last_flush_page_index_in_level_(-1) {} + void reset() + { + tree_epoch_ = -1; + is_meta_reach_end_ = false; + last_flush_level_ = -1; + last_flush_page_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + last_flush_page_index_in_level_ = -1; + } + bool is_valid() const + { + return tree_epoch_ >= 0 + && last_flush_level_ >= 0 + && ObTmpFileGlobal::INVALID_PAGE_ID != last_flush_page_id_ + && last_flush_page_index_in_level_ >= 0; + } + int64_t tree_epoch_; + bool is_meta_reach_end_; + int16_t last_flush_level_; + uint32_t last_flush_page_id_; + int32_t last_flush_page_index_in_level_; + TO_STRING_KV(K(tree_epoch_), K(is_meta_reach_end_), K(last_flush_level_), K(last_flush_page_id_), + K(last_flush_page_index_in_level_)); +}; + +enum ObTmpFileTreeEvictType : int16_t +{ + INVALID = -1, + FULL = 0, //flush all pages + MAJOR, //flush all pages except rightmost page of each level +}; + +class ObSharedNothingTmpFileMetaTree +{ +public: + struct LevelPageRangeInfo + { + LevelPageRangeInfo() : start_page_id_(ObTmpFileGlobal::INVALID_PAGE_ID), + flushed_end_page_id_(ObTmpFileGlobal::INVALID_PAGE_ID), + end_page_id_(ObTmpFileGlobal::INVALID_PAGE_ID), + cached_page_num_(0), + flushed_page_num_(0), + evicted_page_num_(0) {} + LevelPageRangeInfo(const uint32_t start_page_id, + const uint32_t flushed_page_id, + const uint32_t end_page_id, + const int32_t cached_page_num, + const int32_t flushed_page_num, + const int32_t evicted_page_num) : + start_page_id_(start_page_id), + flushed_end_page_id_(flushed_page_id), + end_page_id_(end_page_id), + cached_page_num_(cached_page_num), + flushed_page_num_(flushed_page_num), + evicted_page_num_(evicted_page_num) {} + uint32_t start_page_id_; + uint32_t flushed_end_page_id_; + uint32_t end_page_id_; + //Rules: + // cached_page_num_ + evicted_page_num_ = level total page num + // flushed_page_num_ >= evicted_page_num_ + // flushed_pages must contail evicted_pages, but maybe some cached_pages. + // cached_pages -> [start_page_id_, end_page_id_] + // flushed_pages -> [start_page_id_, flushed_end_page_id_] or none(depnds on whether flushed_end_page_id_ is valid or not) + // They do not contain duplicate pages, even if the page is cached/flushed/evicted multiple times. + int32_t cached_page_num_; + int32_t flushed_page_num_; + int32_t evicted_page_num_; + TO_STRING_KV(K(start_page_id_), K(flushed_end_page_id_), K(end_page_id_), + K(cached_page_num_), K(flushed_page_num_), K(evicted_page_num_)); + }; + + struct BacktraceNode + { + BacktraceNode() : page_meta_info_(), + level_page_index_(-1), + prev_item_index_(-1) {} + BacktraceNode(const ObSharedNothingTmpFileMetaItem &item, + const int32_t level_page_index, + const int16_t prev_item_index) + : page_meta_info_(item), + level_page_index_(level_page_index), + prev_item_index_(prev_item_index) {} + bool is_valid() const + { + return page_meta_info_.is_valid() + && 0 <= level_page_index_; + } + //corresponding to a page (represent meta info of the page) + ObSharedNothingTmpFileMetaItem page_meta_info_; + //the page index in its level + int32_t level_page_index_; + //prev processed item index on the page + int16_t prev_item_index_; + TO_STRING_KV(K(page_meta_info_), K(level_page_index_), K(prev_item_index_)); + }; + + struct LastTruncateLeafInfo + { + LastTruncateLeafInfo() : page_index_in_leaf_level_(-1), + item_index_in_page_(-1), + release_to_end_in_page_(false) {} + bool is_valid() const + { + return 0 <= page_index_in_leaf_level_ + && 0 <= item_index_in_page_; + } + void reset() + { + page_index_in_leaf_level_ = -1; + item_index_in_page_ = -1; + release_to_end_in_page_ = false; + } + int32_t page_index_in_leaf_level_; + int16_t item_index_in_page_; + bool release_to_end_in_page_; + TO_STRING_KV(K(page_index_in_leaf_level_), K(item_index_in_page_), K(release_to_end_in_page_)); + }; + + struct StatInfo + { + StatInfo() : meta_page_flushing_cnt_(0), + all_type_page_flush_cnt_(0), + all_type_flush_page_released_cnt_(0) {} + void reset() + { + meta_page_flushing_cnt_ = 0; + all_type_page_flush_cnt_ = 0; + all_type_flush_page_released_cnt_ = 0; + } + int64_t meta_page_flushing_cnt_; + //can contain the same page if page is flushed again + // contain total pages(meta and data) + int64_t all_type_page_flush_cnt_; + int64_t all_type_flush_page_released_cnt_; + TO_STRING_KV(K(meta_page_flushing_cnt_), K(all_type_page_flush_cnt_), K(all_type_flush_page_released_cnt_)); + }; + +public: + ObSharedNothingTmpFileMetaTree(); + ~ObSharedNothingTmpFileMetaTree(); + void reset(); +public: + int init(const int64_t fd, ObTmpWriteBufferPool *wbp, ObIAllocator *callback_allocator); + + //append write: We always write the rightmost page of the leaf layer + //It happens after a tmp file write request: + // data pages of a write request is persisted on disk + int prepare_for_insert_items(); + int insert_items(const common::ObIArray &data_items); + + //It happens when there is a tmp file read request + int search_data_items(const int64_t offset, + const int64_t read_size, + common::ObIArray &data_items); + + //It happens when there is a meta tree pages flush request: + // tmp file data and meta pages are aggregated to flush + int flush_meta_pages_for_block(const int64_t block_index, + const ObTmpFileTreeEvictType flush_type, + char *block_buff, + int64_t &write_offset, + ObTmpFileTreeFlushContext &flush_context, + common::ObIArray &meta_io_array); + + //It happens after meta tree pages flush: + // meta pages is persisted on disk successfully + int update_after_flush(const common::ObIArray &meta_io_array); + + //It happens when there is a tmp file write request: + // a tmp file page that is not full need to be continued writing. + //last_data_item is the page info of the unfilled data page. + int prepare_for_write_tail(ObSharedNothingTmpFileDataItem &last_data_item); + // After the tail is written (the tail page exists in disk) + int finish_write_tail(const ObSharedNothingTmpFileDataItem &last_data_item, + const bool release_tail_in_disk); + + //It happens when there is a tmp file eviction + int evict_meta_pages(const int64_t expected_page_num, + const ObTmpFileTreeEvictType flush_type, + int64_t &actual_evict_page_num); + + //it happens when there is a tmp file clearing + int clear(const int64_t last_truncate_offset, const int64_t total_file_size); + + //it happens when there is a need to truncate a tmp file + int truncate(const int64_t last_truncate_offset, const int64_t end_truncate_offset); + + //get the num of tree pages that need to be flushed + int get_need_flush_page_num(int64_t &total_need_flush_page_num, + int64_t &total_need_flush_rightmost_page_num) const; + //get the num of tree pages that need to be evicted + int get_need_evict_page_num(int64_t &total_need_evict_page_num, + int64_t &total_need_evict_rightmost_page_num) const; + //NOTE: the following function is called externally. + // It needs to be called internally without holding lock_ (avoid deadlock). + void print_meta_tree_overview_info(); + //NOTE: need control print frequency. + void print_meta_tree_total_info(); + +private: + int modify_meta_items_at_parent_level_(const ObTmpFileTreeIOInfo &meta_io, + const int32_t start_page_index_in_level); + int truncate_array_(const int64_t end_offset); + int search_data_items_from_array_(const int64_t end_offset, + int64_t &cur_offset, + common::ObIArray &data_items); + int get_items_of_internal_page_(const ObSharedNothingTmpFileMetaItem &page_info, + const int32_t level_page_index, + const int16_t cur_item_index, + const int64_t end_offset, + bool &is_last_item, + ObIArray &meta_items); + int backtrace_truncate_tree_(const int64_t end_offset, + ObIArray &search_path); + int backtrace_search_data_items_(const int64_t end_offset, + int64_t &offset, + ObIArray &search_path, + ObIArray &data_items); + int finish_insert_(const int return_ret, + ObIArray &level_origin_page_write_counts, + ObIArray> &level_new_pages); + int try_to_insert_items_to_array_(const ObIArray &data_items, + ObIArray> &level_new_pages); + int release_items_of_leaf_page_(const ObSharedNothingTmpFileMetaItem &page_info, + const int32_t level_page_index, + const int64_t end_offset, + const int16_t begin_release_index, + bool &release_last_item); + int get_items_of_leaf_page_(const ObSharedNothingTmpFileMetaItem &page_info, + const int32_t level_page_index, + const int64_t end_offset, + const bool need_find_index, + int64_t &cur_offset, + common::ObIArray &data_items); + virtual int release_meta_page_(const ObSharedNothingTmpFileMetaItem &page_info, + const int32_t page_index_in_level); + virtual int release_tmp_file_page_(const int64_t block_index, + const int64_t begin_page_id, const int64_t page_num); + int get_rightmost_leaf_page_for_write_(ObSharedNothingTmpFileMetaItem &page_info); + int get_last_item_of_internal_page_(const uint32_t parent_page_id, + ObSharedNothingTmpFileMetaItem &page_info, + ObSharedNothingTmpFileMetaItem &last_meta_item); + int try_to_fill_rightmost_internal_page_(const ObIArray &meta_items, + const int16_t page_level, + int64_t &write_count, + ObIArray &level_origin_page_write_counts); + int try_to_fill_rightmost_leaf_page_(const ObIArray &data_items, + const ObIArray> &level_new_pages, + int64_t &write_count, + ObIArray &level_origin_page_write_counts); + int add_new_page_and_fill_items_at_leaf_(const ObIArray &data_items, + int64_t &write_count, + ObSharedNothingTmpFileMetaItem &meta_item, + ObIArray> &level_new_pages); + int add_new_page_and_fill_items_at_internal_(const ObIArray &meta_items, + const int16_t page_level, + int64_t &write_count, + ObSharedNothingTmpFileMetaItem &meta_item, + ObIArray> &level_new_pages); + int cascade_modification_at_internal_(const ObIArray &new_leaf_page_infos, + ObIArray &level_origin_page_write_counts, + ObIArray> &level_new_pages); + int modify_child_pages_location(char *page_buff); + int modify_meta_items_during_evict_(const ObIArray &evict_pages, + const int16_t level, + const int32_t start_page_index_in_level); + int flush_leaf_pages_(const uint32_t flush_start_page_id, + const int32_t start_page_index_in_level, + const ObTmpFileTreeEvictType flush_type, + char *block_buff, + int64_t &write_offset, + ObTmpFileTreeIOInfo &meta_io_info); + int calc_and_set_page_checksum_(char* page_buff); + int flush_internal_pages_(const uint32_t flush_start_page_id, + const int16_t level, + const int32_t start_page_index_in_level, + const ObTmpFileTreeEvictType flush_type, + char *block_buff, + int64_t &write_offset, + ObTmpFileTreeIOInfo &meta_io_info); + int get_page_(const ObSharedNothingTmpFileMetaItem &page_info, + const int32_t level_page_index, + char *&page_buff, + ObTmpPageValueHandle &p_handle); + int check_page_(const char* const page_buff); + virtual int cache_page_for_write_(const uint32_t parent_page_id, + ObSharedNothingTmpFileMetaItem &page_info); + int init_page_header_(char* page_buff, + const int16_t page_level); + int read_page_header_(const char* page_buff, + ObSharedNothingTmpFileTreePageHeader &page_header); + int remove_page_item_from_tail_(char* page_buff, + const int16_t remove_item_num); + int truncate_(const int64_t end_offset); + int calculate_truncate_index_path_(ObIArray> &item_index_arr); + template + int read_item_(const char* page_buff, + const int64_t target_virtual_page_id, + int16_t &item_index, + ItemType &item); + template + int read_item_(const char* page_buff, + const int16_t item_index, + ItemType &item); + template + int rewrite_item_(char* page_buff, + const int16_t item_index, + const ItemType &item) __attribute__((noinline)); + template + int write_items_(char* page_buff, + const ObIArray &items, + const int64_t begin_index, + int16_t &write_count); + int check_tree_is_empty_(); + template + void read_page_content_(const char *page_buff, + ObSharedNothingTmpFileTreePageHeader &page_header, + ObIArray &data_items); + template + void read_page_simple_content_(const char *page_buff, + ObSharedNothingTmpFileTreePageHeader &page_header, + ItemType &first_item, + ItemType &last_item); + inline bool is_page_in_write_cache(const ObSharedNothingTmpFileMetaItem &meta_item) + { + return ObTmpFileGlobal::INVALID_PAGE_ID != meta_item.buffer_page_id_; + } + inline bool is_page_flushed(const ObSharedNothingTmpFileMetaItem &meta_item) + { + return ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX != meta_item.block_index_ + && 0 <= meta_item.physical_page_id_; + } + +private: + //only for unittest + static void set_max_array_item_cnt(const int16_t cnt) + { + MAX_DATA_ITEM_ARRAY_COUNT = cnt; + } + static void set_max_page_item_cnt(const int16_t cnt) + { + MAX_PAGE_ITEM_COUNT = cnt; + } +private: + static const uint16_t PAGE_MAGIC_NUM; + static const int16_t PAGE_HEADER_SIZE; + static int16_t MAX_DATA_ITEM_ARRAY_COUNT; + //only for unittest + static int16_t MAX_PAGE_ITEM_COUNT; + +private: + bool is_inited_; + int64_t fd_; + ObTmpWriteBufferPool *wbp_; + ObIAllocator *callback_allocator_; + int64_t tree_epoch_; + ObSharedNothingTmpFileMetaItem root_item_; + //When the tmp file writes less data, we can use an array instead of a tree to store metadata + common::ObSEArray data_item_array_; + //level page ranges in write cache + common::ObSEArray level_page_range_array_; + //Only writing the rightmost pages + //Used to prevent the rightmost pages from being evicted + bool is_writing_; + common::SpinRWLock lock_; + LastTruncateLeafInfo last_truncate_leaf_info_; + int64_t released_offset_; +public: + StatInfo stat_info_; + //NOTE: without protection of lock_, we do not recommend using K(meta_tree_) externally. + TO_STRING_KV(K(fd_), K(tree_epoch_), K(root_item_), + K(data_item_array_), K(level_page_range_array_), K(is_writing_), + K(last_truncate_leaf_info_), K(released_offset_), K(stat_info_)); +}; + + +} // end namespace tmp_file +} // end namespace oceanbase +#endif // OCEANBASE_STORAGE_BLOCKSSTABLE_OB_TMP_FILE_META_TREE_H_ \ No newline at end of file diff --git a/src/storage/tmp_file/ob_tmp_file_page_cache_controller.cpp b/src/storage/tmp_file/ob_tmp_file_page_cache_controller.cpp new file mode 100644 index 000000000..802c1633e --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_page_cache_controller.cpp @@ -0,0 +1,161 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE + +#include "storage/tmp_file/ob_tmp_file_page_cache_controller.h" + +namespace oceanbase +{ +namespace tmp_file +{ + +int ObTmpFilePageCacheController::init(ObTenantTmpFileManager &file_mgr) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + STORAGE_LOG(WARN, "ObTmpFilePageCacheController init twice"); + } else if (OB_FAIL(task_allocator_.init(lib::ObMallocAllocator::get_instance(), + OB_MALLOC_MIDDLE_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileCtl", ObCtxIds::DEFAULT_CTX_ID)))) { + STORAGE_LOG(WARN, "fail to init task allocator", KR(ret)); + } else if (OB_FAIL(flush_mgr_.init())) { + STORAGE_LOG(WARN, "fail to init flush task mgr", KR(ret)); + } else if (OB_FAIL(flush_priority_mgr_.init())) { + STORAGE_LOG(WARN, "fail to init flush priority mgr", KR(ret)); + } else if (OB_FAIL(write_buffer_pool_.init())) { + STORAGE_LOG(WARN, "fail to init write buffer pool", KR(ret)); + } else if (OB_FAIL(flush_tg_.init())) { + STORAGE_LOG(WARN, "fail to init flush thread", KR(ret)); + } else if (OB_FAIL(swap_tg_.init(file_mgr))) { + STORAGE_LOG(WARN, "fail to init swap thread", KR(ret)); + } else { + flush_all_data_ = false; + is_inited_ = true; + } + return ret; +} + +int ObTmpFilePageCacheController::start() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + STORAGE_LOG(WARN, "tmp file page cache controller is not inited"); + } else if (OB_FAIL(swap_tg_.start())) { + STORAGE_LOG(WARN, "fail to start swap thread", KR(ret)); + } + return ret; +} + +void ObTmpFilePageCacheController::stop() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + STORAGE_LOG(WARN, "tmp file page cache controller is not inited"); + } else { + // stop background threads should follow the order 'swap' -> 'flush' because 'swap' holds ref to 'flush' + swap_tg_.stop(); + } +} + +void ObTmpFilePageCacheController::wait() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + STORAGE_LOG(WARN, "tmp file page cache controller is not inited"); + } else { + swap_tg_.wait(); + } +} + +void ObTmpFilePageCacheController::destroy() +{ + swap_tg_.destroy(); + flush_tg_.destroy(); + task_allocator_.reset(); + write_buffer_pool_.destroy(); + flush_mgr_.destroy(); + evict_mgr_.destroy(); + flush_priority_mgr_.destroy(); + flush_all_data_ = false; + is_inited_ = false; +} + +int ObTmpFilePageCacheController::swap_job_enqueue_(ObTmpFileSwapJob *swap_job) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(swap_job)){ + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "swap job is null", KR(ret)); + } else if (!swap_job->is_valid()) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "swap job is not valid", KR(ret), KPC(swap_job)); + } else if (OB_FAIL(swap_tg_.swap_job_enqueue(swap_job))) { + STORAGE_LOG(WARN, "fail to enqueue swap job", KR(ret), KP(swap_job)); + } + return ret; +} + +int ObTmpFilePageCacheController::free_swap_job_(ObTmpFileSwapJob *swap_job) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(swap_job)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "swap job is null", KR(ret)); + } else if (swap_job->is_inited() && !swap_job->is_finished()) { + ret = OB_EAGAIN; + STORAGE_LOG(ERROR, "swap job is not finished", KR(ret), KPC(swap_job)); + } else { + swap_job->~ObTmpFileSwapJob(); + task_allocator_.free(swap_job); + } + return ret; +} + +int ObTmpFilePageCacheController::invoke_swap_and_wait(int64_t expect_swap_size, int64_t timeout_ms) +{ + int ret = OB_SUCCESS; + + int64_t mem_limit = write_buffer_pool_.get_memory_limit(); + expect_swap_size = min(expect_swap_size, static_cast(0.2 * mem_limit)); + + void *task_buf = nullptr; + ObTmpFileSwapJob *swap_job = nullptr; + if (OB_ISNULL(task_buf = task_allocator_.alloc(sizeof(ObTmpFileSwapJob)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + STORAGE_LOG(WARN, "fail to allocate memory for swap job", KR(ret)); + } else if (FALSE_IT(swap_job = new (task_buf) ObTmpFileSwapJob())) { + } else if (OB_FAIL(swap_job->init(expect_swap_size, timeout_ms))) { + STORAGE_LOG(WARN, "fail to init sync swap job", KR(ret), KPC(swap_job)); + } else if (OB_FAIL(swap_job_enqueue_(swap_job))) { + STORAGE_LOG(WARN, "fail to enqueue swap job", KR(ret), KPC(swap_job)); + } else { + swap_tg_.notify_doing_swap(); + if (OB_FAIL(swap_job->wait_swap_complete())) { + STORAGE_LOG(WARN, "fail to wait for swap job complete timeout", KR(ret)); + } + } + + if (OB_NOT_NULL(swap_job)) { + // reset swap job to set is_finished to false in case of failure to push into queue: + // otherwise job is not finished, but it will not be executed, so it will never become finished. + swap_job->reset(); + if (OB_FAIL(free_swap_job_(swap_job))) { + STORAGE_LOG(ERROR, "fail to free swap job", KR(ret)); + } + } + return ret; +} + +} // end namespace tmp_file +} // end namespace oceanbase diff --git a/src/storage/tmp_file/ob_tmp_file_page_cache_controller.h b/src/storage/tmp_file/ob_tmp_file_page_cache_controller.h new file mode 100644 index 000000000..93b10f108 --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_page_cache_controller.h @@ -0,0 +1,78 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_PAGE_CACHE_CONTROLLER_H_ +#define OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_PAGE_CACHE_CONTROLLER_H_ + +#include "storage/tmp_file/ob_tmp_file_thread_wrapper.h" +#include "storage/tmp_file/ob_tmp_file_flush_manager.h" + +namespace oceanbase +{ +namespace tmp_file +{ + +class ObTmpFilePageCacheController +{ +public: + ObTmpFilePageCacheController(ObTmpFileBlockManager &tmp_file_block_manager) + : is_inited_(false), + flush_all_data_(false), + tmp_file_block_manager_(tmp_file_block_manager), + task_allocator_(), + write_buffer_pool_(), + flush_priority_mgr_(), + evict_mgr_(), + flush_mgr_(*this), + flush_tg_(write_buffer_pool_, flush_mgr_, task_allocator_, tmp_file_block_manager_), + swap_tg_(write_buffer_pool_, evict_mgr_, flush_tg_) + { + } + ~ObTmpFilePageCacheController() {} +public: + static const int64_t FLUSH_FAST_INTERVAL = 5; // 5ms + static const int64_t FLUSH_INTERVAL = 1000; // 1s + static const int64_t SWAP_FAST_INTERVAL = 5; // 5ms + static const int64_t SWAP_INTERVAL = 1000; // 2s + int init(ObTenantTmpFileManager &file_mgr); + int start(); + void stop(); + void wait(); + void destroy(); + ObIAllocator &get_task_allocator() { return task_allocator_; } + ObTmpWriteBufferPool &get_write_buffer_pool() { return write_buffer_pool_; } + ObTmpFileFlushManager &get_flush_task_mgr() { return flush_mgr_; } + ObTmpFileEvictionManager &get_eviction_manager() { return evict_mgr_; } + ObTmpFileFlushPriorityManager &get_flush_priority_mgr() { return flush_priority_mgr_; } + ObTmpFileBlockManager &get_tmp_file_block_manager() { return tmp_file_block_manager_; } + OB_INLINE bool is_flush_all_data() { return ATOMIC_LOAD(&flush_all_data_); } + int invoke_swap_and_wait(int64_t expect_swap_size, int64_t timeout_ms = ObTmpFileSwapJob::DEFAULT_TIMEOUT_MS); +private: + int swap_job_enqueue_(ObTmpFileSwapJob *swap_job); + int free_swap_job_(ObTmpFileSwapJob *swap_job); + DISALLOW_COPY_AND_ASSIGN(ObTmpFilePageCacheController); +private: + bool is_inited_; + bool flush_all_data_; // unit test only + ObTmpFileBlockManager &tmp_file_block_manager_; // ref to ObTmpFileBlockManager + ObFIFOAllocator task_allocator_; // used by flush_mgr_ to allocate flush tasks + ObTmpWriteBufferPool write_buffer_pool_; + ObTmpFileFlushPriorityManager flush_priority_mgr_; + ObTmpFileEvictionManager evict_mgr_; // maintain evict lists and evict pages from write buffer pool + ObTmpFileFlushManager flush_mgr_; // maintain flush lists and generate flush tasks + ObTmpFileFlushTG flush_tg_; // flush thread + ObTmpFileSwapTG swap_tg_; // swap thread +}; + +} // end namespace tmp_file +} // end namespace oceanbase +#endif // OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_PAGE_CACHE_CONTROLLER_H_ diff --git a/src/storage/tmp_file/ob_tmp_file_thread_job.cpp b/src/storage/tmp_file/ob_tmp_file_thread_job.cpp new file mode 100644 index 000000000..7c7d3c291 --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_thread_job.cpp @@ -0,0 +1,193 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE + +#include "share/ob_errno.h" // KR +#include "storage/tmp_file/ob_tmp_file_thread_job.h" +#include "lib/utility/utility.h" + +namespace oceanbase +{ +namespace tmp_file +{ + +int ObTmpFileSwapJob::init(int64_t expect_swap_size, uint32_t timeout_ms) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + STORAGE_LOG(WARN, "ObTmpFileSwapJob init twice", KR(ret)); + } else if (timeout_ms <= 0) { + ret = OB_INVALID_ARGUMENT; + } else if (OB_FAIL(swap_cond_.init(ObWaitEventIds::NULL_EVENT))) { + STORAGE_LOG(WARN, "ObTmpFileSwapJob init cond failed", KR(ret)); + } else { + is_inited_ = true; + is_finished_ = false; + expect_swap_size_ = expect_swap_size; + timeout_ms_ = timeout_ms; + create_ts_ = ObTimeUtility::current_time(); + abs_timeout_ts_ = ObTimeUtility::current_time() + timeout_ms_ * 1000; + } + return ret; +} + +void ObTmpFileSwapJob::reset() +{ + is_inited_ = false; + is_finished_ = false; + expect_swap_size_ = 0; + timeout_ms_ = DEFAULT_TIMEOUT_MS; + create_ts_ = 0; + abs_timeout_ts_ = 0; + swap_cond_.destroy(); +} + +// waits for swap job to finish, loop inside until is_finished_ is true +int ObTmpFileSwapJob::wait_swap_complete() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + STORAGE_LOG(WARN, "ObTmpFileSwapJob not init", KR(ret)); + } else { + bool is_finished = false; + ObThreadCondGuard guard(swap_cond_); + while (false == (is_finished = ATOMIC_LOAD(&is_finished_))) { + if (OB_FAIL(swap_cond_.wait())) { + if (OB_TIMEOUT == ret) { + STORAGE_LOG(WARN, "fail to wait swap job complete", KR(ret), K(is_finished), K(timeout_ms_)); + ret = OB_SUCCESS; + } else { + STORAGE_LOG(ERROR, "fail to wait swap job thread cond", KR(ret), K(is_finished), KPC(this)); + } + } + } + } + return ret; +} + +// set swap job is_finished, wake up threads that invoke swap job +int ObTmpFileSwapJob::signal_swap_complete() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + STORAGE_LOG(WARN, "ObTmpFileSwapJob not init", KR(ret)); + } else { + ObThreadCondGuard guard(swap_cond_); + ATOMIC_SET(&is_finished_, true); + if (OB_FAIL(swap_cond_.signal())) { + STORAGE_LOG(WARN, "ObTmpFileSwapJob signal swap complete failed", KR(ret)); + } + } + return ret; +} + +void ObTmpFileFlushMonitor::reset() +{ + flush_task_cnt_ = 0; + total_flush_data_length_ = 0; + max_flush_data_length_ = -1; + min_flush_data_length_ = INT64_MAX; + f1_cnt_ = 0; + f2_cnt_ = 0; + f3_cnt_ = 0; + f4_cnt_ = 0; + f5_cnt_ = 0; +} + +void ObTmpFileFlushMonitor::print_statistics() +{ + int64_t flush_task_cnt = flush_task_cnt_; + int64_t avg_flush_data_len = total_flush_data_length_ / max(flush_task_cnt, 1); + int64_t max_flush_data_len = max_flush_data_length_; + int64_t min_flush_data_len = min_flush_data_length_ == INT64_MAX ? -1 : min_flush_data_length_; + int64_t f1_cnt = f1_cnt_; + int64_t f2_cnt = f2_cnt_; + int64_t f3_cnt = f3_cnt_; + int64_t f4_cnt = f4_cnt_; + int64_t f5_cnt = f5_cnt_; + STORAGE_LOG(INFO, "tmp file flush statistics", K(flush_task_cnt), K(avg_flush_data_len), + K(max_flush_data_len), K(min_flush_data_len), K(f1_cnt), K(f2_cnt), K(f3_cnt), K(f4_cnt), K(f5_cnt)); + reset(); +} + +void ObTmpFileFlushMonitor::record_flush_stage(const ObTmpFileGlobal::FlushCtxState flush_stage) +{ + switch(flush_stage) { + case ObTmpFileGlobal::FSM_F1: + f1_cnt_++; + break; + case ObTmpFileGlobal::FSM_F2: + f2_cnt_++; + break; + case ObTmpFileGlobal::FSM_F3: + f3_cnt_++; + break; + case ObTmpFileGlobal::FSM_F4: + f4_cnt_++; + break; + case ObTmpFileGlobal::FSM_F5: + f5_cnt_++; + break; + default: + break; + } +} + +void ObTmpFileFlushMonitor::record_flush_task(const int64_t data_length) +{ + flush_task_cnt_ += 1; + total_flush_data_length_ += data_length; + if (data_length > 0 && data_length > max_flush_data_length_) { + max_flush_data_length_ = data_length; + } + if (data_length > 0 && data_length < min_flush_data_length_) { + min_flush_data_length_ = data_length; + } +} + +void ObTmpFileSwapMonitor::reset() +{ + swap_task_cnt_ = 0; + swap_total_response_time_= 0; + swap_max_response_time_ = -1; + swap_min_response_time_ = INT64_MAX; +} + +void ObTmpFileSwapMonitor::print_statistics() +{ + int64_t swap_task_cnt = swap_task_cnt_; + int64_t avg_swap_response_time = swap_total_response_time_ / max(swap_task_cnt, 1); + int64_t max_swap_response_time = swap_max_response_time_; + int64_t min_swap_response_time = swap_min_response_time_ == INT64_MAX ? -1 : swap_min_response_time_; + STORAGE_LOG(INFO, "tmp file swap statistics", K(swap_task_cnt), + K(avg_swap_response_time), K(max_swap_response_time), K(min_swap_response_time)); + reset(); +} + +void ObTmpFileSwapMonitor::record_swap_response_time(const int64_t response_time) +{ + swap_task_cnt_ += 1; + swap_total_response_time_ += response_time; + if (response_time > swap_max_response_time_) { + swap_max_response_time_ = response_time; + } + if (response_time < swap_min_response_time_) { + swap_min_response_time_ = response_time; + } +} + +} // end namespace tmp_file +} // end namespace oceanbase diff --git a/src/storage/tmp_file/ob_tmp_file_thread_job.h b/src/storage/tmp_file/ob_tmp_file_thread_job.h new file mode 100644 index 000000000..24a92914b --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_thread_job.h @@ -0,0 +1,111 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_THREAD_JOB_H_ +#define OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_THREAD_JOB_H_ + +#include "lib/lock/ob_thread_cond.h" +#include "lib/queue/ob_link_queue.h" +#include "lib/utility/ob_print_utils.h" +#include "storage/tmp_file/ob_tmp_file_global.h" + +namespace oceanbase +{ +namespace tmp_file +{ + +class ObTmpFileSwapJob : public ObSpLinkQueue::Link +{ +public: + static const uint32_t DEFAULT_TIMEOUT_MS = 10 * 1000; + ObTmpFileSwapJob() + : is_inited_(false), + is_finished_(false), + timeout_ms_(DEFAULT_TIMEOUT_MS), + create_ts_(0), + abs_timeout_ts_(0), + expect_swap_size_(0), + swap_cond_() {} + ~ObTmpFileSwapJob() { reset(); } + int init(int64_t expect_swap_size, uint32_t timeout_ms = DEFAULT_TIMEOUT_MS); + void reset(); + int wait_swap_complete(); + int signal_swap_complete(); + OB_INLINE int64_t get_create_ts() const { return create_ts_; } + OB_INLINE int64_t get_abs_timeout_ts() const { return abs_timeout_ts_; } + OB_INLINE int64_t get_expect_swap_size() const { return expect_swap_size_; } + OB_INLINE bool is_valid() { return ATOMIC_LOAD(&is_inited_) && swap_cond_.is_inited(); } + OB_INLINE bool is_finished() const { return ATOMIC_LOAD(&is_finished_); } + OB_INLINE bool is_inited() const { return ATOMIC_LOAD(&is_inited_); } + TO_STRING_KV(KP(this), K(is_inited_), K(is_finished_), K(create_ts_), K(timeout_ms_), K(abs_timeout_ts_), K(expect_swap_size_)); +private: + bool is_inited_; + bool is_finished_; + uint32_t timeout_ms_; + int64_t create_ts_; + int64_t abs_timeout_ts_; + int64_t expect_swap_size_; // in bytes + ObThreadCond swap_cond_; +}; + +// record statistics for flush tasks in flushTG thread +class ObTmpFileFlushMonitor +{ +public: + ObTmpFileFlushMonitor() + : flush_task_cnt_(0), + total_flush_data_length_(0), + max_flush_data_length_(-1), + min_flush_data_length_(INT64_MAX), + f1_cnt_(0), + f2_cnt_(0), + f3_cnt_(0), + f4_cnt_(0), + f5_cnt_(0) {} + void reset(); + void print_statistics(); + void record_flush_stage(const ObTmpFileGlobal::FlushCtxState flush_stage); + void record_flush_task(const int64_t data_length); +private: + int64_t flush_task_cnt_; + int64_t total_flush_data_length_; + int64_t max_flush_data_length_; + int64_t min_flush_data_length_; + int64_t f1_cnt_; + int64_t f2_cnt_; + int64_t f3_cnt_; + int64_t f4_cnt_; + int64_t f5_cnt_; +}; + +// record statistics for swap tasks in swapTG thread +class ObTmpFileSwapMonitor +{ +public: + ObTmpFileSwapMonitor() + : swap_task_cnt_(0), + swap_total_response_time_(0), + swap_max_response_time_(-1), + swap_min_response_time_(INT64_MAX) {} + void reset(); + void print_statistics(); + void record_swap_response_time(const int64_t response_time); +private: + int64_t swap_task_cnt_; + int64_t swap_total_response_time_; + int64_t swap_max_response_time_; + int64_t swap_min_response_time_; +}; + +} // end namespace tmp_file +} // end namespace oceanbase +#endif // OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_THREAD_JOB_H_ diff --git a/src/storage/tmp_file/ob_tmp_file_thread_wrapper.cpp b/src/storage/tmp_file/ob_tmp_file_thread_wrapper.cpp new file mode 100644 index 000000000..d08c739e9 --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_thread_wrapper.cpp @@ -0,0 +1,1023 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE + +#include "storage/tmp_file/ob_tmp_file_thread_wrapper.h" +#include "share/ob_thread_mgr.h" +#include "storage/blocksstable/ob_block_manager.h" +#include "storage/tmp_file/ob_tmp_file_page_cache_controller.h" +#include "storage/tmp_file/ob_tmp_file_manager.h" + +namespace oceanbase +{ +namespace tmp_file +{ + +ObTmpFileFlushTG::ObTmpFileFlushTG( + ObTmpWriteBufferPool &wbp, + ObTmpFileFlushManager &flush_mgr, + ObIAllocator &allocator, + ObTmpFileBlockManager &tmp_file_block_mgr) + : is_inited_(false), + mode_(RUNNING_MODE::INVALID), + last_flush_timestamp_(0), + flush_io_finished_round_(0), + flushing_block_num_(0), + is_fast_flush_meta_(false), + fast_flush_meta_task_cnt_(0), + wait_list_size_(0), + retry_list_size_(0), + finished_list_size_(0), + wait_list_(), + retry_list_(), + finished_list_(), + flush_monitor_(), + flush_mgr_(flush_mgr), + wbp_(wbp), + tmp_file_block_mgr_(tmp_file_block_mgr), + normal_loop_cnt_(0), + normal_idle_loop_cnt_(0), + fast_loop_cnt_(0), + fast_idle_loop_cnt_(0) +{ +} + +int ObTmpFileFlushTG::init() +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + STORAGE_LOG(WARN, "ObTmpFileSwapTG init twice"); + } else { + is_inited_ = true; + mode_ = RUNNING_MODE::NORMAL; + last_flush_timestamp_ = 0; + flush_io_finished_round_ = 0; + flushing_block_num_ = 0; + + fast_flush_meta_task_cnt_ = 0; + wait_list_size_ = 0; + retry_list_size_ = 0; + finished_list_size_ = 0; + + normal_loop_cnt_ = 0; + normal_idle_loop_cnt_ = 0; + fast_loop_cnt_ = 0; + fast_idle_loop_cnt_ = 0; + } + return ret; +} + +void ObTmpFileFlushTG::destroy() +{ + if (IS_INIT) { + clean_up_lists(); + mode_ = RUNNING_MODE::INVALID; + last_flush_timestamp_ = 0; + flush_io_finished_round_ = 0; + flushing_block_num_ = 0; + + is_fast_flush_meta_ = false; + fast_flush_meta_task_cnt_ = 0; + wait_list_size_ = 0; + retry_list_size_ = 0; + finished_list_size_ = 0; + + normal_loop_cnt_ = 0; + normal_idle_loop_cnt_ = 0; + fast_loop_cnt_ = 0; + fast_idle_loop_cnt_ = 0; + + is_inited_ = false; + } +} + +void ObTmpFileFlushTG::clean_up_lists() +{ + int ret = OB_SUCCESS; + while (!retry_list_.is_empty()) { + ObTmpFileFlushTask *flush_task = nullptr; + pop_retry_list_(flush_task); + if (OB_ISNULL(flush_task)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "flush task is null", KR(ret)); + } else { + STORAGE_LOG(INFO, "free flush task in retry_list_", KPC(flush_task)); + flush_mgr_.free_flush_task(flush_task); + } + } + + while (!finished_list_.is_empty()) { + ObTmpFileFlushTask *flush_task = nullptr; + pop_finished_list_(flush_task); + if (OB_ISNULL(flush_task)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "flush task is null", KR(ret)); + } else { + STORAGE_LOG(INFO, "free flush task in finished_list_", KPC(flush_task)); + flush_mgr_.free_flush_task(flush_task); + } + } + + while (!wait_list_.is_empty()) { // clean up tasks after IO complete + STORAGE_LOG(WARN, "wait_list_ is not empty after flush thread stop", K(wait_list_size_)); + for (int64_t cnt = 0; cnt < wait_list_size_ && !wait_list_.is_empty(); ++cnt) { + ObTmpFileFlushTask *flush_task = nullptr; + pop_wait_list_(flush_task); + if (OB_ISNULL(flush_task)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "flush task is null", K(cnt), K(wait_list_size_)); + } else if (OB_FAIL(flush_task->wait_macro_block_handle())) { + if (OB_EAGAIN == ret) { + push_wait_list_(flush_task); + ret = OB_SUCCESS; + } else { + STORAGE_LOG(ERROR, "unexpected error in waiting flush task finished", KR(ret), KPC(this)); + } + } else if (!flush_task->atomic_get_io_finished()) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(ERROR, "unexpected flush task state", KR(ret), KPC(flush_task)); + } else if (OB_FAIL(flush_mgr_.notify_write_back_failed(flush_task))) { + STORAGE_LOG(ERROR, "fail to notify_write_back_failed", KR(ret), KPC(flush_task)); + } else { + STORAGE_LOG(DEBUG, "free flush task in wait_list_", KPC(flush_task)); + flush_mgr_.free_flush_task(flush_task); + } + } + usleep(10 * 1000); // 10ms + } +} + +void ObTmpFileFlushTG::set_running_mode(const RUNNING_MODE mode) +{ + mode_ = mode; +} + +void ObTmpFileFlushTG::notify_doing_flush() +{ + last_flush_timestamp_ = 0; +} + +void ObTmpFileFlushTG::signal_io_finish() +{ + ++flush_io_finished_round_; +} + +int64_t ObTmpFileFlushTG::get_flush_io_finished_round() +{ + return flush_io_finished_round_; +} + +int64_t ObTmpFileFlushTG::cal_idle_time() +{ + int64_t idle_time = 0; + int64_t dirty_page_percentage = wbp_.get_dirty_page_percentage(); + if (!wait_list_.is_empty() || !retry_list_.is_empty() || !finished_list_.is_empty() + || ObTmpFileFlushManager::FLUSH_WATERMARK_F1 <= dirty_page_percentage) { + idle_time = ObTmpFilePageCacheController::FLUSH_FAST_INTERVAL; + } else if (RUNNING_MODE::FAST == mode_) { + int64_t flushing_block_num = ATOMIC_LOAD(&flushing_block_num_); + if (flushing_block_num >= get_flushing_block_num_threshold_()) { + idle_time = ObTmpFilePageCacheController::FLUSH_FAST_INTERVAL; + } else { + idle_time = 0; + } + } else { + idle_time = ObTmpFilePageCacheController::FLUSH_INTERVAL; + } + return idle_time; +} + +int ObTmpFileFlushTG::try_work() +{ + int ret = OB_SUCCESS; + + int64_t cur_time = ObTimeUtility::current_monotonic_time(); + if (0 == last_flush_timestamp_ || cur_time - last_flush_timestamp_ >= cal_idle_time() * 1000) { + if (OB_FAIL(do_work_())) { + STORAGE_LOG(WARN, "fail do flush", KR(ret), KPC(this)); + } + last_flush_timestamp_ = ObTimeUtility::current_monotonic_time(); + } + + return ret; +} + +int ObTmpFileFlushTG::do_work_() +{ + int ret = OB_SUCCESS; + + if (is_fast_flush_meta_) { + check_flush_task_io_finished_(); + } else { + if (RUNNING_MODE::FAST == mode_) { + flush_fast_(); + } else if (RUNNING_MODE::NORMAL == mode_) { + flush_normal_(); + } else { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unknown running mode", KR(ret), K(mode_)); + } + } + + if (TC_REACH_TIME_INTERVAL(1 * 1000 * 1000)) { + tmp_file_block_mgr_.print_block_usage(); + flush_monitor_.print_statistics(); + wbp_.print_statistics(); + STORAGE_LOG(INFO, "ObTmpFileFlushTG information", KPC(this)); + normal_loop_cnt_ = 0; + normal_idle_loop_cnt_ = 0; + fast_loop_cnt_ = 0; + fast_idle_loop_cnt_ = 0; + } + return ret; +} + +// 1. check wait_list_ for IO complete task; 2. retry old task if exists; 3. build new task if needed +void ObTmpFileFlushTG::flush_fast_() +{ + int ret = OB_SUCCESS; + int64_t BLOCK_SIZE = OB_SERVER_BLOCK_MGR.get_macro_block_size(); + int64_t flush_size = min(get_fast_flush_size_(), get_flushing_block_num_threshold_() * BLOCK_SIZE); + if (OB_FAIL(check_flush_task_io_finished_())) { + STORAGE_LOG(WARN, "fail to check flush task io finished", KR(ret)); + } + if (OB_FAIL(retry_task_())) { + STORAGE_LOG(WARN, "fail to retry task", KR(ret)); + } + if (flush_size > 0) { + if (OB_FAIL(wash_(flush_size, RUNNING_MODE::FAST))) { + STORAGE_LOG(WARN, "fail to flush fast", KR(ret), KPC(this), K(flush_size)); + } + } else { + STORAGE_LOG(DEBUG, "current expect flush size is 0, skip flush", K(flush_size), K(this)); + } +} + +void ObTmpFileFlushTG::flush_normal_() +{ + int ret = OB_SUCCESS; + int64_t BLOCK_SIZE = OB_SERVER_BLOCK_MGR.get_macro_block_size(); + int64_t normal_flush_size = max(0, (MAX_FLUSHING_BLOCK_NUM - ATOMIC_LOAD(&flushing_block_num_)) * BLOCK_SIZE); + if (OB_FAIL(check_flush_task_io_finished_())) { + STORAGE_LOG(WARN, "fail to check flush task io finished", KR(ret)); + } + if (OB_FAIL(retry_task_())) { + STORAGE_LOG(WARN, "fail to retry task", KR(ret)); + } + if (normal_flush_size > 0) { + if (OB_FAIL(wash_(normal_flush_size, RUNNING_MODE::NORMAL))) { + STORAGE_LOG(WARN, "fail to flush normal", KR(ret), KPC(this)); + } + } else { + STORAGE_LOG(DEBUG, "current expect flush size is 0, skip flush", K(normal_flush_size), K(this)); + } +} + +int ObTmpFileFlushTG::handle_generated_flush_tasks_(ObSpLinkQueue &flushing_list, int64_t &task_num) +{ + int ret = OB_SUCCESS; + task_num = 0; + while (!flushing_list.is_empty()) { // ignore error code to handle all flush tasks + task_num += 1; + ObSpLinkQueue::Link *link = nullptr; + flushing_list.pop(link); + if (OB_ISNULL(link)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "flush task is null", KR(ret)); + } else { + ObTmpFileFlushTask *flush_task = static_cast(link); + FlushState state = flush_task->get_state(); + bool need_release_resource = false; + STORAGE_LOG(DEBUG, "check flush task state", KPC(flush_task)); + if (FlushState::TFFT_WAIT == state) { + ATOMIC_INC(&flushing_block_num_); + push_wait_list_(flush_task); + if (flush_task->get_is_fast_flush_tree()) { + // after setting this flag, the thread will only process tasks that are already in wait_list_ and finished_list_, + // and will not process tasks in retry_list_ or generate new flush tasks + is_fast_flush_meta_ = true; + fast_flush_meta_task_cnt_ += 1; + } + } else if (FlushState::TFFT_FILL_BLOCK_BUF > state) { + need_release_resource = true; + } else if (FlushState::TFFT_FILL_BLOCK_BUF == state && 0 == flush_task->get_data_length()) { + need_release_resource = true; + } else if (FlushState::TFFT_FILL_BLOCK_BUF == state && 0 < flush_task->get_data_length()) { + // ATTENTION! after the data has been copied, it should transition to the next state. + // staying in the TFFT_FILL_BLOCK_BUF is an abnormal behavior and the file should be discarded, + // do not free task here to avoid further corruption. + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("fail to switch state after data copied from tmp file", KR(ret), KPC(flush_task)); + } else if (FlushState::TFFT_FILL_BLOCK_BUF < state) { + push_retry_list_(flush_task); + ATOMIC_INC(&flushing_block_num_); + STORAGE_LOG(DEBUG, "push flush task to retry list", KPC(flush_task)); + } + + if (need_release_resource) { + if (ObTmpFileGlobal::INVALID_TMP_FILE_BLOCK_INDEX != flush_task->get_block_index()) { + flush_mgr_.free_tmp_file_block(*flush_task); + } + flush_mgr_.free_flush_task(flush_task); + } + } + } // end while + return ret; +} + +int ObTmpFileFlushTG::wash_(const int64_t expect_flush_size, const RUNNING_MODE mode) +{ + int ret = OB_SUCCESS; + int64_t flushing_task_cnt = 0; + ObSpLinkQueue flushing_list; + if (OB_FAIL(flush_mgr_.flush(flushing_list, flush_monitor_, expect_flush_size, is_fast_flush_meta_))) { + STORAGE_LOG(WARN, "flush mgr fail to do flush", KR(ret), KPC(this)); + } else if (OB_FAIL(handle_generated_flush_tasks_(flushing_list, flushing_task_cnt))) { + STORAGE_LOG(WARN, "fail to handle generated flush tasks", KR(ret), K(flushing_task_cnt), KPC(this)); + } + + bool idle_loop = flushing_task_cnt == 0; + if (idle_loop && wbp_.get_dirty_page_percentage() < ObTmpFileFlushManager::FLUSH_WATERMARK_F5) { + signal_io_finish(); + } + + if (RUNNING_MODE::NORMAL == mode) { + ++normal_loop_cnt_; + if (idle_loop) { + ++normal_idle_loop_cnt_; + } + } else if (RUNNING_MODE::FAST == mode) { + ++fast_loop_cnt_; + if (idle_loop) { + ++fast_idle_loop_cnt_; + } + } + return ret; +} + +int ObTmpFileFlushTG::retry_task_() +{ + int ret = OB_SUCCESS; + for (int64_t cnt = retry_list_size_; cnt > 0 && !retry_list_.is_empty(); --cnt) { + ObTmpFileFlushTask *flush_task = nullptr; + pop_retry_list_(flush_task); + if (OB_ISNULL(flush_task)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "flush task is nullptr", KR(ret)); + } else { + STORAGE_LOG(DEBUG, "retry flush task", KPC(flush_task)); + if (OB_FAIL(flush_mgr_.retry(*flush_task))) { + STORAGE_LOG(WARN, "fail to retry flush task", KR(ret), KPC(flush_task)); + } + // push task into wait_list_/retry_list_ according to task state, ignore error code + FlushState state = flush_task->get_state(); + if (FlushState::TFFT_WAIT == state) { + push_wait_list_(flush_task); + } else if (FlushState::TFFT_FILL_BLOCK_BUF < state) { + push_retry_list_(flush_task); + if (FlushState::TFFT_INSERT_META_TREE == state && OB_ALLOCATE_TMP_FILE_PAGE_FAILED == ret) { + STORAGE_LOG(WARN, "fail to retry insert meta item in TFFT_INSERT_META_TREE", KPC(flush_task)); + is_fast_flush_meta_ = true; + if (OB_FAIL(special_flush_meta_tree_page_())) { + STORAGE_LOG(WARN, "fail to flush meta tree page", KR(ret)); + } + break; + } + } + } + } + return ret; +} + +int ObTmpFileFlushTG::special_flush_meta_tree_page_() +{ + int ret = OB_SUCCESS; + ObSpLinkQueue flushing_list; + int64_t flushing_task_cnt = 0; + int64_t expect_flush_size = OB_SERVER_BLOCK_MGR.get_macro_block_size(); + if (OB_FAIL(flush_mgr_.flush(flushing_list, flush_monitor_, expect_flush_size, true/*is_flush_meta_tree*/))) { + STORAGE_LOG(ERROR, "flush mgr fail to do fast flush meta tree page", KR(ret), KPC(this)); + } else if (OB_FAIL(handle_generated_flush_tasks_(flushing_list, flushing_task_cnt))) { + STORAGE_LOG(WARN, "fail to handle fast flush meta tasks", KR(ret), K(flushing_task_cnt), KPC(this)); + } + return ret; +} + +int ObTmpFileFlushTG::check_flush_task_io_finished_() +{ + int ret = OB_SUCCESS; + for (int64_t cnt = ATOMIC_LOAD(&wait_list_size_); cnt > 0 && !wait_list_.is_empty(); --cnt) { + ObTmpFileFlushTask *flush_task = nullptr; + pop_wait_list_(flush_task); + if (OB_ISNULL(flush_task)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "flush task is nullptr", KR(ret)); + } else if (OB_FAIL(flush_task->wait_macro_block_handle())) { + if (OB_EAGAIN == ret) { + push_wait_list_(flush_task); // IO is not completed, continue waiting + ret = OB_SUCCESS; + } else { + STORAGE_LOG(ERROR, "unexpected error in waiting flush task finished", KR(ret), KPC(this)); + } + } else if (!flush_task->atomic_get_io_finished()) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(ERROR, "unexpected flush task state", KR(ret), KPC(flush_task)); + } else if (OB_FAIL(flush_mgr_.io_finished(*flush_task))) { + STORAGE_LOG(WARN, "fail to handle flush task finished", KR(ret), KPC(flush_task)); + } else { + if (FlushState::TFFT_FINISH == flush_task->get_state()) { + push_finished_list_(flush_task); + STORAGE_LOG(DEBUG, "flush task push to finished list", KPC(flush_task)); + } else if (FlushState::TFFT_ASYNC_WRITE == flush_task->get_state()) { + push_retry_list_(flush_task); + STORAGE_LOG(DEBUG, "flush task push to retry list", KPC(flush_task)); + } else { + STORAGE_LOG(ERROR, "unexpected flush task state", KR(ret), KPC(flush_task)); + } + } + } + + for (int64_t cnt = ATOMIC_LOAD(&finished_list_size_); cnt > 0 && !finished_list_.is_empty(); --cnt) { + ObTmpFileFlushTask *flush_task = nullptr; + pop_finished_list_(flush_task); + if (OB_ISNULL(flush_task)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "flush task is nullptr", KR(ret)); + } else { + bool is_flush_tree_page = flush_task->get_is_fast_flush_tree(); + STORAGE_LOG(DEBUG, "flush task io complete", K(flushing_block_num_), KPC(flush_task)); + // if the update fails, it will be retried during the next wakeup + if (OB_FAIL(flush_mgr_.update_file_meta_after_flush(*flush_task))) { + STORAGE_LOG(WARN, "fail to drive flush state machine", KR(ret), KPC(flush_task)); + push_finished_list_(flush_task); + } else { + flush_mgr_.free_flush_task(flush_task); + ATOMIC_DEC(&flushing_block_num_); + fast_flush_meta_task_cnt_ -= is_flush_tree_page ? 1 : 0; + if (fast_flush_meta_task_cnt_ == 0) { + // reset is_fast_flush_meta_ flag to resume retry task and flush + is_fast_flush_meta_ = false; + } else if (OB_UNLIKELY(fast_flush_meta_task_cnt_ < 0)) { + STORAGE_LOG(ERROR, "fast_flush_meta_task_cnt_ is negative", KPC(this)); + } + signal_io_finish(); + } + } + } + return ret; +} + +int ObTmpFileFlushTG::push_wait_list_(ObTmpFileFlushTask *flush_task) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(flush_task)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "flush task is nullptr", KR(ret)); + } else if (OB_FAIL(wait_list_.push(flush_task))) { + STORAGE_LOG(WARN, "fail push flush task into wait_list_", KR(ret), KP(flush_task)); + } else { + ATOMIC_INC(&wait_list_size_); + } + return ret; +} + +int ObTmpFileFlushTG::pop_wait_list_(ObTmpFileFlushTask *&flush_task) +{ + int ret = OB_SUCCESS; + ObSpLinkQueue::Link *link = nullptr; + if (OB_FAIL(wait_list_.pop(link))) { + STORAGE_LOG(DEBUG, "fail to pop flush task from wait_list_", KR(ret)); + } else if (OB_ISNULL(link)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "flush task ptr in wait list is null", KR(ret)); + } else { + ATOMIC_DEC(&wait_list_size_); + flush_task = static_cast(link); + } + return ret; +} + +int ObTmpFileFlushTG::push_retry_list_(ObTmpFileFlushTask *flush_task) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(flush_task)){ + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "flush task is nullptr", KR(ret)); + } else if (OB_FAIL(retry_list_.push(flush_task))) { + STORAGE_LOG(WARN, "fail push flush task into retry_list_", KR(ret), KP(flush_task)); + } else { + ATOMIC_INC(&retry_list_size_); + } + return ret; +} + +int ObTmpFileFlushTG::pop_retry_list_(ObTmpFileFlushTask *&flush_task) +{ + int ret = OB_SUCCESS; + ObSpLinkQueue::Link *link = nullptr; + if (OB_FAIL(retry_list_.pop(link))) { + STORAGE_LOG(DEBUG, "fail to pop flush task from retry_list_", KR(ret)); + } else if (OB_ISNULL(link)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "flush task ptr is null", KR(ret)); + } else { + ATOMIC_DEC(&retry_list_size_); + flush_task = static_cast(link); + } + return ret; +} + +int ObTmpFileFlushTG::push_finished_list_(ObTmpFileFlushTask *flush_task) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(flush_task)){ + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "flush task is nullptr", KR(ret)); + } else if (OB_FAIL(finished_list_.push(flush_task))) { + STORAGE_LOG(WARN, "fail push flush task into finished_list_", KR(ret), KP(flush_task)); + } else { + ATOMIC_INC(&finished_list_size_); + } + return ret; +} + +int ObTmpFileFlushTG::pop_finished_list_(ObTmpFileFlushTask *&flush_task) +{ + int ret = OB_SUCCESS; + ObSpLinkQueue::Link *link = nullptr; + if (OB_FAIL(finished_list_.pop(link))) { + STORAGE_LOG(DEBUG, "fail to pop flush task from finished_list_", KR(ret)); + } else if (OB_ISNULL(link)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "flush task ptr is null", KR(ret)); + } else { + ATOMIC_DEC(&finished_list_size_); + flush_task = static_cast(link); + } + return ret; +} + +// fast mode flush size is max(2MB, min(5% * tmp_file_memory，30MB)) +int ObTmpFileFlushTG::get_fast_flush_size_() +{ + // TODO: move to page cache controller + const int64_t BLOCK_SIZE = OB_SERVER_BLOCK_MGR.get_macro_block_size(); + int64_t wbp_mem_limit = wbp_.get_memory_limit(); + int64_t flush_size = max(BLOCK_SIZE, min(MAX_FLUSHING_BLOCK_NUM * BLOCK_SIZE, upper_align(0.05 * wbp_mem_limit, BLOCK_SIZE))); + return flush_size; +} + +// flushing threshold is MIN(20MB, (20% * tmp_file_memory)) +int ObTmpFileFlushTG::get_flushing_block_num_threshold_() +{ + const int64_t BLOCK_SIZE = OB_SERVER_BLOCK_MGR.get_macro_block_size(); + int64_t wbp_mem_limit = wbp_.get_memory_limit(); + int64_t flush_threshold = max(BLOCK_SIZE, min(MAX_FLUSHING_BLOCK_NUM, static_cast(0.2 * wbp_mem_limit / BLOCK_SIZE))); + return flush_threshold; +} + +// --------------- swap ----------------// + +ObTmpFileSwapTG::ObTmpFileSwapTG(ObTmpWriteBufferPool &wbp, + ObTmpFileEvictionManager &elimination_mgr, + ObTmpFileFlushTG &flush_tg) + : is_inited_(false), + tg_id_(-1), + idle_cond_(), + last_swap_timestamp_(0), + swap_job_num_(0), + swap_job_list_(), + working_list_size_(0), + working_list_(), + swap_monitor_(), + flush_tg_ref_(flush_tg), + flush_io_finished_round_(0), + wbp_(wbp), + evict_mgr_(elimination_mgr), + file_mgr_(nullptr) +{ +} + +int ObTmpFileSwapTG::init(ObTenantTmpFileManager &file_mgr) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + STORAGE_LOG(WARN, "ObTmpFileSwapTG init twice"); + } else if (OB_FAIL(idle_cond_.init(ObWaitEventIds::NULL_EVENT))) { + STORAGE_LOG(WARN, "failed to init condition variable", KR(ret)); + } else if (OB_FAIL(TG_CREATE_TENANT(lib::TGDefIDs::TmpFileSwap, tg_id_))) { + STORAGE_LOG(WARN, "fail to create swap thread", KR(ret)); + } else if (OB_FAIL(TG_SET_RUNNABLE(tg_id_, *this))) { + STORAGE_LOG(WARN, "fail to set swap tg runnable", KR(ret)); + } else { + is_inited_ = true; + last_swap_timestamp_ = 0; + swap_job_num_ = 0; + working_list_size_ = 0; + flush_io_finished_round_ = 0; + file_mgr_ = &file_mgr; + } + return ret; +} + +int ObTmpFileSwapTG::start() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + STORAGE_LOG(WARN, "ObTmpFileSwapTG thread is not inited", KR(ret)); + } else if (OB_FAIL(TG_START(tg_id_))) { + STORAGE_LOG(WARN, "fail to start tmp file ObTmpFileSwapTG thread", KR(ret)); + } + return ret; +} + +void ObTmpFileSwapTG::stop() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + STORAGE_LOG(WARN, "ObTmpFileSwapTG thread is not inited", KR(ret)); + } else { + TG_STOP(tg_id_); + ObThreadCondGuard guard(idle_cond_); + idle_cond_.signal(); + } +} + +void ObTmpFileSwapTG::wait() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + STORAGE_LOG(WARN, "ObTmpFileSwapTG thread is not inited", KR(ret)); + } else { + TG_WAIT(tg_id_); + } +} + +void ObTmpFileSwapTG::destroy() +{ + int ret = OB_SUCCESS; + if (-1 != tg_id_) { + TG_DESTROY(tg_id_); + tg_id_ = -1; + } + + clean_up_lists_(); + last_swap_timestamp_ = 0; + swap_job_num_ = 0; + working_list_size_ = 0; + flush_io_finished_round_ = 0; + idle_cond_.destroy(); + file_mgr_ = nullptr; + is_inited_ = false; +} + +int ObTmpFileSwapTG::swap_job_enqueue(ObTmpFileSwapJob *swap_job) +{ + int ret = OB_SUCCESS; + if (has_set_stop()) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "swap thread has been stopped", KR(ret), K(tg_id_), KP(swap_job)); + } else if (OB_FAIL(swap_job_list_.push(swap_job))) { + STORAGE_LOG(WARN, "fail push swap job", KR(ret), K(tg_id_), KP(swap_job)); + } else { + ATOMIC_INC(&swap_job_num_); + } + return ret; +} + +int ObTmpFileSwapTG::swap_job_dequeue(ObTmpFileSwapJob *&swap_job) +{ + int ret = OB_SUCCESS; + ObSpLinkQueue::Link *link = nullptr; + if (OB_FAIL(swap_job_list_.pop(link))) { + STORAGE_LOG(DEBUG, "fail to pop swap job", KR(ret), K(tg_id_)); + } else if (OB_ISNULL(link)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "fail to get swap job, ptr is nullptr", KR(ret), K(tg_id_)); + } else { + ATOMIC_DEC(&swap_job_num_); + swap_job = static_cast(link); + } + return ret; +} + +void ObTmpFileSwapTG::notify_doing_swap() +{ + ObThreadCondGuard guard(idle_cond_); + idle_cond_.signal(); +} + +int64_t ObTmpFileSwapTG::cal_idle_time() +{ + int64_t swap_idle_time = ObTmpFilePageCacheController::SWAP_INTERVAL; + if (ATOMIC_LOAD(&swap_job_num_) != 0 || ATOMIC_LOAD(&working_list_size_) != 0) { + if (flush_io_finished_round_ < flush_tg_ref_.get_flush_io_finished_round()) { + swap_idle_time = 0; + } else { + swap_idle_time = ObTmpFilePageCacheController::SWAP_FAST_INTERVAL; + } + } + return swap_idle_time; +} + +void ObTmpFileSwapTG::run1() +{ + int ret = OB_SUCCESS; + lib::set_thread_name("TFSwap"); + while (!has_set_stop()) { + if (OB_FAIL(try_work())) { + STORAGE_LOG(WARN, "fail to try swap work", KR(ret)); + } + + if (OB_FAIL(flush_tg_ref_.try_work())) { + // overwrite ret + STORAGE_LOG(WARN, "fail to try flush work", KR(ret)); + } + + ObThreadCondGuard guard(idle_cond_); + int64_t swap_idle_time = cal_idle_time(); + int64_t flush_idle_time = flush_tg_ref_.cal_idle_time(); + int64_t idle_time = min(swap_idle_time, flush_idle_time); + if (!has_set_stop() && idle_time != 0) { + idle_cond_.wait(idle_time); + } + + if (OB_NOT_NULL(file_mgr_)) { + file_mgr_->refresh_meta_memory_limit(); + } + } + + if (has_set_stop()) { + flush_tg_ref_.clean_up_lists(); + clean_up_lists_(); + } +} + +void ObTmpFileSwapTG::clean_up_lists_() +{ + int ret = OB_SUCCESS; + while (!swap_job_list_.is_empty()) { + ObTmpFileSwapJob *swap_job = nullptr; + if (OB_FAIL(swap_job_dequeue(swap_job))) { + STORAGE_LOG(WARN, "fail dequeue swap job or swap job is nullptr", KR(ret), KP(swap_job)); + } else if (OB_FAIL(swap_job->signal_swap_complete())){ + STORAGE_LOG(WARN, "fail to signal swap complete", KR(ret)); + } + } + + ret = OB_SUCCESS; + while (!working_list_.is_empty()) { + ObTmpFileSwapJob *swap_job = nullptr; + if (OB_FAIL(pop_working_job_(swap_job))) { + STORAGE_LOG(WARN, "fail to pop working job or ptr is null", KR(ret), KP(swap_job)); + } else if (OB_FAIL(swap_job->signal_swap_complete())){ + STORAGE_LOG(WARN, "fail to signal swap complete", KR(ret)); + } + } +} + +int ObTmpFileSwapTG::try_work() +{ + int ret = OB_SUCCESS; + + int64_t cur_time = ObTimeUtility::current_monotonic_time(); + if (0 == last_swap_timestamp_ || cur_time - last_swap_timestamp_ >= cal_idle_time() * 1000) { + if (OB_FAIL(do_work_())) { + STORAGE_LOG(WARN, "fail do swap", KR(ret), KPC(this)); + } + last_swap_timestamp_ = ObTimeUtility::current_monotonic_time(); + } + + return ret; +} + +int ObTmpFileSwapTG::do_work_() +{ + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + STORAGE_LOG(WARN, "ObTmpFileSwapTG not init", KR(ret)); + } + + if (OB_SUCC(ret)) { + if (ATOMIC_LOAD(&swap_job_num_) == 0 && ATOMIC_LOAD(&working_list_size_) == 0) { + if (OB_FAIL(swap_normal_())){ + STORAGE_LOG(WARN, "fail to do normal swap", KR(ret)); + } + } else { + if (OB_FAIL(swap_fast_())){ + STORAGE_LOG(WARN, "fail to do fast swap", KR(ret)); + } + if (OB_FAIL(swap_normal_())) { + STORAGE_LOG(WARN, "fail to do normal swap", KR(ret)); + } + } + if (ATOMIC_LOAD(&swap_job_num_) == 0 && ATOMIC_LOAD(&working_list_size_) == 0) { + flush_tg_ref_.set_running_mode(ObTmpFileFlushTG::RUNNING_MODE::NORMAL); + } + } + if (TC_REACH_TIME_INTERVAL(1 * 1000 * 1000)) { + swap_monitor_.print_statistics(); + } + return ret; +} + +// runs in normal mode when there is no swap job. +// since memory is not tight at this point, we try to evict pages +// but not guarantee eviction occurs if clean pages are not enough +int ObTmpFileSwapTG::swap_normal_() +{ + int ret = OB_SUCCESS; + + int64_t swap_size = wbp_.get_swap_size(); + int64_t actual_swap_page_cnt = 0; + if (swap_size > 0) { // do swap + int64_t swap_page_cnt = swap_size / PAGE_SIZE; + if (OB_FAIL(evict_mgr_.evict(swap_page_cnt, actual_swap_page_cnt))) { + STORAGE_LOG(WARN, "fail to swap out pages", KR(ret), K(swap_size)); + } + } + return ret; +} + +// attempt to evict pages and wake up caller threads as soon as possible, +// this may trigger the flush thread to flush a small number of pages (FAST mode). +// caller threads will be awakened if swap job timeout +int ObTmpFileSwapTG::swap_fast_() +{ + int ret = OB_SUCCESS; + while (OB_SUCC(ret) && !swap_job_list_.is_empty()) { + ObTmpFileSwapJob *swap_job = nullptr; + if (OB_FAIL(swap_job_dequeue(swap_job))) { + STORAGE_LOG(WARN, "fail to get swap job", KR(ret)); + } else if (OB_FAIL(push_working_job_(swap_job))) { + STORAGE_LOG(WARN, "fail to push working job", KR(ret), KP(swap_job)); + } + } + + while (OB_SUCC(ret) && !working_list_.is_empty()) { + int64_t expect_swap_page_cnt = 0; + int64_t actual_swap_page_cnt = 0; + // calculate expect swap pages number for a batch of jobs + if (OB_FAIL(calculate_swap_page_num_(PROCCESS_JOB_NUM_PER_BATCH, expect_swap_page_cnt))) { + STORAGE_LOG(WARN, "fail to calculate swap page num", KR(ret)); + } else if (OB_UNLIKELY(expect_swap_page_cnt <= 0)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "cur expect swap page cnt is invalid", KR(ret), K(expect_swap_page_cnt)); + } else if (OB_FAIL(evict_mgr_.evict(expect_swap_page_cnt, actual_swap_page_cnt))) { + STORAGE_LOG(WARN, "fail to swap out pages", KR(ret), K(expect_swap_page_cnt), K(actual_swap_page_cnt)); + } + + int64_t wakeup_job_cnt = 0; + wakeup_satisfied_jobs_(wakeup_job_cnt); + wakeup_timeout_jobs_(); + + // do flush if could not evict enough pages + if (OB_SUCC(ret) && !working_list_.is_empty() && wakeup_job_cnt < PROCCESS_JOB_NUM_PER_BATCH) { + flush_io_finished_round_ = flush_tg_ref_.get_flush_io_finished_round(); + flush_tg_ref_.set_running_mode(ObTmpFileFlushTG::RUNNING_MODE::FAST); + flush_tg_ref_.notify_doing_flush(); + break; + } + } // end while + + return ret; +} + +int ObTmpFileSwapTG::calculate_swap_page_num_(const int64_t batch_size, int64_t &expect_swap_cnt) +{ + int ret = OB_SUCCESS; + ObSpLinkQueue cur_working_list; + for (int64_t i = 0; OB_SUCC(ret) && i < batch_size && !working_list_.is_empty(); ++i) { + ObTmpFileSwapJob *swap_job = nullptr; + if (OB_FAIL(pop_working_job_(swap_job))) { + STORAGE_LOG(WARN, "fail to pop working job or ptr is null", KR(ret), KP(swap_job)); + } else { + expect_swap_cnt += upper_align(swap_job->get_expect_swap_size(), PAGE_SIZE) / PAGE_SIZE; + cur_working_list.push_front(swap_job); + } + } + while (!cur_working_list.is_empty()) { + ObSpLinkQueue::Link *link = nullptr; + cur_working_list.pop(link); + ObTmpFileSwapJob *swap_job = static_cast(link); + if (OB_ISNULL(swap_job)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "swap job is invalid", KR(ret)); + } else if (OB_FAIL(push_working_job_front_(swap_job))) { + STORAGE_LOG(WARN, "fail to push swap job", KR(ret), KPC(swap_job)); + } + } + return ret; +} + +int ObTmpFileSwapTG::wakeup_satisfied_jobs_(int64_t& wakeup_job_cnt) +{ + int ret = OB_SUCCESS; + wakeup_job_cnt = 0; + int64_t wbp_free_page_cnt = wbp_.get_max_data_page_num() - wbp_.get_data_page_num(); + while (OB_SUCC(ret) && wbp_free_page_cnt > 0 && !working_list_.is_empty()) { + ObTmpFileSwapJob *swap_job = nullptr; + if (OB_FAIL(pop_working_job_(swap_job))) { + STORAGE_LOG(WARN, "fail to pop working job or ptr is null", KR(ret), KP(swap_job)); + } else { + // wake up threads even if free page number < swap job's expect swap size + int64_t single_job_swap_page_cnt = upper_align(swap_job->get_expect_swap_size(), PAGE_SIZE) / PAGE_SIZE; + wbp_free_page_cnt -= min(wbp_free_page_cnt, single_job_swap_page_cnt); + int64_t response_time = ObTimeUtility::current_time() - swap_job->get_create_ts(); + swap_monitor_.record_swap_response_time(response_time); + if (OB_FAIL(swap_job->signal_swap_complete())) { + STORAGE_LOG(WARN, "fail to signal swap complete", KR(ret), KPC(swap_job)); + } else { + ++wakeup_job_cnt; + } + } + } + return ret; +} + +int ObTmpFileSwapTG::wakeup_timeout_jobs_() +{ + int ret = OB_SUCCESS; + for (int64_t i = working_list_size_; OB_SUCC(ret) && i > 0 && !working_list_.is_empty(); --i) { + ObTmpFileSwapJob *swap_job = nullptr; + if (OB_FAIL(pop_working_job_(swap_job))) { + STORAGE_LOG(WARN, "fail to pop working job or ptr is null", KR(ret), KP(swap_job)); + } else if (swap_job->get_abs_timeout_ts() <= ObTimeUtility::current_time()) { + // timeout, wake it up + int64_t response_time = ObTimeUtility::current_time() - swap_job->get_create_ts(); + swap_monitor_.record_swap_response_time(response_time); + if (OB_FAIL(swap_job->signal_swap_complete())) { + STORAGE_LOG(WARN, "fail to signal swap complete", KR(ret), KP(swap_job)); + } + } else { + if (OB_FAIL(push_working_job_(swap_job))) { + STORAGE_LOG(WARN, "fail to push swap job", KR(ret), K(tg_id_), KPC(swap_job)); + } + } + } + return ret; +} + +int ObTmpFileSwapTG::push_working_job_(ObTmpFileSwapJob *swap_job) +{ + int ret = OB_SUCCESS; + if (has_set_stop()) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "swap thread has been stopped", KR(ret), K(tg_id_), KP(swap_job)); + } else if (OB_FAIL(working_list_.push(swap_job))) { + STORAGE_LOG(WARN, "fail push swap job", KR(ret), K(tg_id_), KPC(swap_job)); + } else { + ATOMIC_INC(&working_list_size_); + } + return ret; +} + +int ObTmpFileSwapTG::pop_working_job_(ObTmpFileSwapJob *&swap_job) +{ + int ret = OB_SUCCESS; + ObSpLinkQueue::Link *link = nullptr; + if (OB_FAIL(working_list_.pop(link))) { + STORAGE_LOG(DEBUG, "fail to pop swap job", KR(ret), K(tg_id_)); + } else if (OB_ISNULL(link)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "fail to get swap job, ptr is nullptr", KR(ret), K(tg_id_)); + } else { + ATOMIC_DEC(&working_list_size_); + swap_job = static_cast(link); + } + return ret; +} + +int ObTmpFileSwapTG::push_working_job_front_(ObTmpFileSwapJob *swap_job) +{ + int ret = OB_SUCCESS; + if (has_set_stop()) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "swap thread has been stopped", KR(ret), K(tg_id_), KP(swap_job)); + } else if (OB_FAIL(working_list_.push_front(swap_job))) { + STORAGE_LOG(WARN, "fail push swap job", KR(ret), K(tg_id_), KPC(swap_job)); + } else { + ATOMIC_INC(&working_list_size_); + } + return ret; +} + +} // end namespace tmp_file +} // end namespace oceanbase diff --git a/src/storage/tmp_file/ob_tmp_file_thread_wrapper.h b/src/storage/tmp_file/ob_tmp_file_thread_wrapper.h new file mode 100644 index 000000000..79fe3fbae --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_thread_wrapper.h @@ -0,0 +1,158 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_THREAD_WRAPPER_H_ +#define OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_THREAD_WRAPPER_H_ + +#include "storage/tmp_file/ob_tmp_file_thread_job.h" +#include "storage/tmp_file/ob_tmp_file_write_buffer_pool.h" +#include "storage/tmp_file/ob_tmp_file_eviction_manager.h" +#include "storage/tmp_file/ob_tmp_file_global.h" +#include "storage/tmp_file/ob_tmp_file_flush_manager.h" +#include "lib/thread/thread_mgr_interface.h" + +namespace oceanbase +{ +namespace tmp_file +{ + +class ObTmpFileFlushManager; + +// When originally designed, ObTmpFileFlushTG was an independent thread. in order to reduce the +// number of threads, the thread of ObTmpFileFlushTG was drived ObTmpFileSwapTG. +class ObTmpFileFlushTG +{ +public: + typedef ObTmpFileFlushTask::ObTmpFileFlushTaskState FlushState; + static const int64_t MAX_FLUSHING_BLOCK_NUM = 50; + enum RUNNING_MODE { + INVALID = 0, + NORMAL = 1, + FAST = 2 + }; +public: + ObTmpFileFlushTG(ObTmpWriteBufferPool &wbp, + ObTmpFileFlushManager &flush_mgr, + ObIAllocator &allocator, + ObTmpFileBlockManager &tmp_file_block_mgr); + int init(); + void destroy(); + + int try_work(); + void set_running_mode(const RUNNING_MODE mode); + void notify_doing_flush(); + void signal_io_finish(); + int64_t get_flush_io_finished_round(); + int64_t cal_idle_time(); + void clean_up_lists(); + TO_STRING_KV(K(is_inited_), K(flushing_block_num_), K(is_fast_flush_meta_), K(fast_flush_meta_task_cnt_), K(mode_), + K(wait_list_size_), K(retry_list_size_), K(finished_list_size_), + K(normal_loop_cnt_), K(normal_idle_loop_cnt_), K(fast_loop_cnt_), K(fast_idle_loop_cnt_)); +private: + int do_work_(); + int handle_generated_flush_tasks_(ObSpLinkQueue &flushing_list, int64_t &task_num); + int wash_(const int64_t expect_flush_size, const RUNNING_MODE mode); + int check_flush_task_io_finished_(); + int retry_task_(); + int special_flush_meta_tree_page_(); + void flush_fast_(); + void flush_normal_(); + int get_fast_flush_size_(); + int get_flushing_block_num_threshold_(); + int push_wait_list_(ObTmpFileFlushTask *flush_task); + int pop_wait_list_(ObTmpFileFlushTask *&flush_task); + int push_retry_list_(ObTmpFileFlushTask *flush_task); + int pop_retry_list_(ObTmpFileFlushTask *&flush_task); + int push_finished_list_(ObTmpFileFlushTask *flush_task); + int pop_finished_list_(ObTmpFileFlushTask *&flush_task); +private: + bool is_inited_; + RUNNING_MODE mode_; + int64_t last_flush_timestamp_; + int64_t flush_io_finished_round_; + + int64_t flushing_block_num_; // maintain it when ObTmpFileFlushTask is created and freed + bool is_fast_flush_meta_; // indicate thread is fast flushing meta page, no new flush tasks will be added and no tasks will be retried + int64_t fast_flush_meta_task_cnt_; + int64_t wait_list_size_; + int64_t retry_list_size_; + int64_t finished_list_size_; + ObSpLinkQueue wait_list_; // list for tasks that are waiting for IO finished + ObSpLinkQueue retry_list_; // list for tasks that need to be retried + ObSpLinkQueue finished_list_; // list for tasks which have finished IO and need to update file's meta + ObTmpFileFlushMonitor flush_monitor_; + ObTmpFileFlushManager &flush_mgr_; + ObTmpWriteBufferPool &wbp_; + ObTmpFileBlockManager &tmp_file_block_mgr_; + + int64_t normal_loop_cnt_; + int64_t normal_idle_loop_cnt_; + int64_t fast_loop_cnt_; + int64_t fast_idle_loop_cnt_; +}; + +class ObTmpFileSwapTG : public lib::TGRunnable +{ +public: + ObTmpFileSwapTG(ObTmpWriteBufferPool &wbp, + ObTmpFileEvictionManager &elimination_mgr, + ObTmpFileFlushTG &flush_tg); + int init(ObTenantTmpFileManager &file_mgr); + int start(); + void stop(); + void wait(); + void destroy(); + void run1() override; + int64_t cal_idle_time(); + int try_work(); + int swap_job_enqueue(ObTmpFileSwapJob *swap_job); + int swap_job_dequeue(ObTmpFileSwapJob *&swap_job); + // wake up swap thread to do work + void notify_doing_swap(); + TO_STRING_KV(K(is_inited_), K(tg_id_), K(swap_job_num_), K(has_set_stop())); +private: + static const int64_t PAGE_SIZE = ObTmpFileGlobal::PAGE_SIZE; + static const int64_t PROCCESS_JOB_NUM_PER_BATCH = 128; + void clean_up_lists_(); + int do_work_(); + int swap_normal_(); + int swap_fast_(); + int push_working_job_(ObTmpFileSwapJob *swap_job); + int push_working_job_front_(ObTmpFileSwapJob *swap_job); + int pop_working_job_(ObTmpFileSwapJob *&swap_job); + int calculate_swap_page_num_(const int64_t batch_size, int64_t &expect_swap_cnt); + int wakeup_satisfied_jobs_(int64_t& wakeup_job_cnt); + int wakeup_timeout_jobs_(); +private: + bool is_inited_; + int tg_id_; + ObThreadCond idle_cond_; + int64_t last_swap_timestamp_; + + int64_t swap_job_num_; + ObSpLinkQueue swap_job_list_; // list for swap jobs to be processed + int64_t working_list_size_; + ObSpLinkQueue working_list_; // list for swap jobs that are being processed + + ObTmpFileSwapMonitor swap_monitor_; + + ObTmpFileFlushTG &flush_tg_ref_; + int64_t flush_io_finished_round_; + + ObTmpWriteBufferPool &wbp_; + ObTmpFileEvictionManager &evict_mgr_; + ObTenantTmpFileManager *file_mgr_; +}; + +} // end namespace tmp_file +} // end namespace oceanbase +#endif // OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_THREAD_WRAPPER_H_ diff --git a/src/storage/tmp_file/ob_tmp_file_write_buffer_index_cache.cpp b/src/storage/tmp_file/ob_tmp_file_write_buffer_index_cache.cpp new file mode 100644 index 000000000..091f83b43 --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_write_buffer_index_cache.cpp @@ -0,0 +1,783 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE + +#include "storage/tmp_file/ob_tmp_file_write_buffer_index_cache.h" +#include "storage/tmp_file/ob_tmp_file_write_buffer_pool.h" +#include "storage/tmp_file/ob_tmp_file_global.h" +#include "share/ob_errno.h" +#include "share/rc/ob_tenant_base.h" + +namespace oceanbase +{ +namespace tmp_file +{ +ObTmpFileWBPIndexCache::ObTmpFileWBPIndexCache() : + ObTmpFileCircleArray(), bucket_array_allocator_(), bucket_allocator_(), + page_buckets_(nullptr) , + fd_(ObTmpFileGlobal::INVALID_TMP_FILE_FD), wbp_(nullptr), + max_bucket_array_capacity_(MAX_BUCKET_ARRAY_CAPACITY) {} + +ObTmpFileWBPIndexCache::~ObTmpFileWBPIndexCache() +{ + destroy(); +} + +int ObTmpFileWBPIndexCache::init(const int64_t fd, ObTmpWriteBufferPool* wbp, + ObIAllocator *wbp_index_cache_allocator, + ObIAllocator *wbp_index_cache_bkt_allocator) +{ + int ret = OB_SUCCESS; + void *buf = nullptr; + uint64_t tenant_id = MTL_ID(); + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", KR(ret)); + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_TMP_FILE_FD == fd)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tmp file fd", KR(ret), K(fd)); + } else if (OB_ISNULL(wbp) || OB_ISNULL(wbp_index_cache_allocator) || + OB_ISNULL(wbp_index_cache_bkt_allocator)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), KP(wbp), KP(wbp_index_cache_allocator), + KP(wbp_index_cache_bkt_allocator)); + } else if (OB_ISNULL(buf = wbp_index_cache_allocator->alloc(sizeof(ObArray), + lib::ObMemAttr(tenant_id, "TmpFileIdxCache")))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate memory for temporary file page index bucket", + KR(ret), K(tenant_id), K(sizeof(ObArray))); + } else if (FALSE_IT(page_buckets_ = new (buf) ObArray())) { + } else if (FALSE_IT(page_buckets_->set_attr(lib::ObMemAttr(tenant_id, "TmpFileIdxCache")))) { + } else if (OB_FAIL(page_buckets_->prepare_allocate(INIT_BUCKET_ARRAY_CAPACITY, nullptr))) { + page_buckets_->destroy(); + wbp_index_cache_allocator->free(buf); + page_buckets_ = nullptr; + LOG_WARN("fail to prepare allocate array", KR(ret)); + } else { + is_inited_ = true; + fd_ = fd; + wbp_ = wbp; + bucket_allocator_ = wbp_index_cache_bkt_allocator; + bucket_array_allocator_ = wbp_index_cache_allocator; + left_ = 0; + right_ = -1; + size_ = 0; + capacity_ = INIT_BUCKET_ARRAY_CAPACITY; + } + return ret; +} + +void ObTmpFileWBPIndexCache::reset() +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + left_ = 0; + right_ = -1; + size_ = 0; + capacity_ = 0; + if (OB_ISNULL(page_buckets_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("page buckets is null", KR(ret), K(fd_)); + } else if (OB_ISNULL(bucket_allocator_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("bucket allocator is null", KR(ret), K(fd_)); + } else { + for (int64_t i = 0; i < page_buckets_->count(); ++i) { + if (OB_NOT_NULL(page_buckets_->at(i))) { + page_buckets_->at(i)->destroy(); + bucket_allocator_->free(page_buckets_->at(i)); + page_buckets_->at(i) = nullptr; + } + } + page_buckets_->reset(); + } + } +} + +void ObTmpFileWBPIndexCache::destroy() +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + reset(); + is_inited_ = false; + fd_ = ObTmpFileGlobal::INVALID_TMP_FILE_FD; + wbp_ = nullptr; + if (OB_ISNULL(page_buckets_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("page buckets is null", KR(ret), K(fd_)); + } else if (OB_ISNULL(bucket_array_allocator_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("bucket array allocator is null", KR(ret), K(fd_)); + } else { + page_buckets_->destroy(); + bucket_array_allocator_->free(page_buckets_); + page_buckets_ = nullptr; + bucket_allocator_ = nullptr; + bucket_array_allocator_ = nullptr; + } + } +} + +int ObTmpFileWBPIndexCache::push(const uint32_t page_index) +{ + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (0 == capacity_ || is_full()) { + if (max_bucket_array_capacity_ <= capacity_) { + if (OB_FAIL(sparsify_())) { + LOG_WARN("fail to sparsify", KR(ret), KPC(this)); + } + } else if (OB_FAIL(expand_())) { + LOG_WARN("fail to expand array", KR(ret), KPC(this)); + } + } + + if (OB_SUCC(ret)) { + if (is_empty() || (OB_NOT_NULL(page_buckets_->at(right_)) && page_buckets_->at(right_)->is_full())) { + // alloc a new bucket + inc_pos_(right_); + uint64_t tenant_id = MTL_ID(); + void *buf = nullptr; + ObTmpFilePageIndexBucket* bucket = nullptr; + if (OB_ISNULL(buf = bucket_allocator_->alloc(sizeof(ObTmpFilePageIndexBucket), + lib::ObMemAttr(tenant_id, "TmpFileIdxBkt")))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate memory for temporary file page index bucket", + KR(ret), K(fd_), K(tenant_id), K(sizeof(ObTmpFilePageIndexBucket)), KPC(this)); + } else if (FALSE_IT(bucket = new (buf) ObTmpFilePageIndexBucket())) { + } else if (OB_FAIL(bucket->init(fd_, wbp_))) { + LOG_WARN("fail to init temporary file page index bucket", KR(ret), K(fd_), KPC(this)); + } else if (OB_FAIL(bucket->push(page_index))) { + LOG_WARN("fail to push page_index", KR(ret), K(fd_), K(page_index), KPC(this)); + } else { + page_buckets_->at(right_) = bucket; + } + + if (OB_FAIL(ret)) { + if (OB_NOT_NULL(bucket)) { + bucket->destroy(); + bucket_allocator_->free(bucket); + page_buckets_->at(right_) = nullptr; + } + dec_pos_(right_); + } else { + size_ += 1; + } + } else if (OB_ISNULL(page_buckets_->at(right_))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", KR(ret), K(fd_), K(left_), K(right_), K(size_), K(capacity_), KPC(this)); + } else if (OB_FAIL(page_buckets_->at(right_)->push(page_index))) { // bucket is not full + LOG_WARN("fail to push page index", KR(ret), K(fd_), K(page_index), KPC(this)); + } + } + + return ret; +} + +int ObTmpFileWBPIndexCache::truncate(const int64_t truncate_page_virtual_id) +{ + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (OB_UNLIKELY(is_empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("array is empty", KR(ret), K(fd_), K(size_)); + } else { + const int64_t logic_begin_pos = left_; + const int64_t logic_end_pos = get_logic_tail_(); + bool truncate_over = false; + for (int64_t i = logic_begin_pos; OB_SUCC(ret) && !truncate_over && i <= logic_end_pos; i++) { + int64_t bkt_min_page_virtual_id = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + int64_t bkt_min_page_index = ObTmpFileGlobal::INVALID_PAGE_ID; + ObTmpFilePageIndexBucket *&cur_bucket = page_buckets_->at(i % capacity_); + if (OB_ISNULL(cur_bucket)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", KR(ret), K(fd_), K(i), K(capacity_), KP(cur_bucket), KPC(this)); + } else if (OB_UNLIKELY((bkt_min_page_index = cur_bucket->get_min_page_index()) == + ObTmpFileGlobal::INVALID_PAGE_ID)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected invalid page index", KR(ret), K(fd_), KPC(cur_bucket), KPC(this)); + } else if (OB_FAIL(wbp_->get_page_virtual_id(fd_, bkt_min_page_index, bkt_min_page_virtual_id))) { + LOG_WARN("fail to get page virtual id in file", KR(ret), K(fd_), K(bkt_min_page_index), KPC(this)); + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == bkt_min_page_virtual_id)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected invalid page virtual id", KR(ret), K(fd_), K(bkt_min_page_virtual_id), KPC(this)); + } else if (i != logic_begin_pos) { + // truncate previous bucket + if (truncate_page_virtual_id < bkt_min_page_virtual_id) { + truncate_over = true; + ObTmpFilePageIndexBucket *&previous_bucket = page_buckets_->at(get_previous_pos(i % capacity_)); + if (OB_ISNULL(previous_bucket)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", KR(ret), K(fd_), K(i), K(capacity_),KP(previous_bucket), + K(truncate_page_virtual_id), K(logic_begin_pos), K(logic_end_pos), KPC(this)); + } else if (OB_FAIL(previous_bucket->truncate(truncate_page_virtual_id))) { + LOG_WARN("fail to truncate bucket", KR(ret), K(fd_), K(truncate_page_virtual_id), KPC(previous_bucket), KPC(this)); + } else if (OB_UNLIKELY(previous_bucket->is_empty())) { + // when truncate_page_virtual_id is smaller than min_page_virtual_id of cur bucket, + // there must exist at least one page in previous bucket whose virtual page id is larger than + // or equal to truncate_page_virtual_id (which means this page index doesn't need to be truncated). + // however, truncate all page indexes of previous bucket will not cause some problems. + // thus, we just allow code continue to run + previous_bucket->destroy(); + bucket_allocator_->free(previous_bucket); + previous_bucket = nullptr; + inc_pos_(left_); + size_ -= 1; + } + } else { // truncate_page_virtual_id >= bkt_min_page_virtual_id + ObTmpFilePageIndexBucket *&previous_bucket = page_buckets_->at(get_previous_pos(i % capacity_)); + if (OB_ISNULL(previous_bucket)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", KR(ret), K(fd_), K(i), K(capacity_),KP(previous_bucket), + K(truncate_page_virtual_id), K(logic_begin_pos), K(logic_end_pos), KPC(this)); + } else { + previous_bucket->destroy(); + bucket_allocator_->free(previous_bucket); + previous_bucket = nullptr; + inc_pos_(left_); + size_ -= 1; + } + } + } + if (i == logic_end_pos && truncate_page_virtual_id > bkt_min_page_virtual_id) { + if (FAILEDx(cur_bucket->truncate(truncate_page_virtual_id))) { + LOG_WARN("fail to truncate bucket", KR(ret), K(fd_), K(truncate_page_virtual_id), KPC(cur_bucket), KPC(this)); + } else if (cur_bucket->is_empty()) { + cur_bucket->destroy(); + bucket_allocator_->free(cur_bucket); + cur_bucket = nullptr; + inc_pos_(left_); + size_ -= 1; + } + } + } // end for + } + + if (OB_FAIL(ret)) { + } else if (is_empty()) { + reset(); + } else { + shrink_(); + } + return ret; +} + +// 1. if 'target_page_virtual_id' is smaller than virtual page id of first page in cache, +// page_index will be INVALID_PAGE_ID. +// in this case, caller should directly search write buffer pool with the beginning page index of file +// 2. if 'target_page_virtual_id' is in the range of cached pages but the page index of it doesn't exist in cache, +// the cache will find a page whose virtual id is smaller than and closest to 'target_page_virtual_id' +// and then iterate the page index of wbp from the closest page index to find the according page_index +int ObTmpFileWBPIndexCache::binary_search(const int64_t target_page_virtual_id, uint32_t &page_index) +{ + int ret = OB_SUCCESS; + page_index = ObTmpFileGlobal::INVALID_PAGE_ID; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (OB_UNLIKELY(is_empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("array is empty", KR(ret), K(fd_), K(size_)); + } else { + int64_t left_pos = left_; + int64_t right_pos = get_logic_tail_(); + ObTmpFilePageIndexBucket *target_bucket = nullptr; + bool find = false; + while (left_pos <= right_pos && !find && OB_SUCC(ret)) { + // find the bucket whose min_page_virtual_id is small than and closest to target_page_virtual_id + const int64_t logic_mid = left_pos + (right_pos - left_pos) / 2; + const int64_t mid = logic_mid % capacity_; + ObTmpFilePageIndexBucket *mid_bucket = page_buckets_->at(mid); + int64_t min_page_virtual_id = -1; + if (OB_ISNULL(mid_bucket)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", KR(ret), K(fd_), K(logic_mid), K(mid), KP(mid_bucket), K(size_), K(capacity_), KPC(this)); + } else if (OB_FAIL(wbp_->get_page_virtual_id(fd_, mid_bucket->get_min_page_index(), min_page_virtual_id))) { + LOG_WARN("fail to get page virtual id in file", KR(ret), K(fd_), K(mid_bucket->get_min_page_index()), KPC(this)); + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == min_page_virtual_id)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected invalid page virtual id", KR(ret), K(fd_), K(min_page_virtual_id), KPC(this)); + } else if (min_page_virtual_id <= target_page_virtual_id) { + target_bucket = mid_bucket; + if (min_page_virtual_id == target_page_virtual_id) { + find = true; + page_index = mid_bucket->get_min_page_index(); + } else { + left_pos = logic_mid + 1; + } + } else { + right_pos = logic_mid - 1; + } + } // end while + + if (OB_FAIL(ret)) { + } else if (find) { + // do nothing + } else if (OB_ISNULL(target_bucket)) { + // page_index = ObTmpFileGlobal::INVALID_PAGE_ID; + LOG_DEBUG("the target page_index might be removed from cache", K(fd_), K(target_page_virtual_id), KPC(this)); + } else if (OB_FAIL(target_bucket->binary_search(target_page_virtual_id, page_index))) { + LOG_WARN("fail to binary search page index", KR(ret), K(fd_), K(target_page_virtual_id), KPC(this)); + } + } + return ret; +} + +int ObTmpFileWBPIndexCache::expand_() +{ + int ret = OB_SUCCESS; + LOG_DEBUG("start to expand tmp file wbp index cache", KPC(this)); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret), K(fd_)); + } else if (OB_UNLIKELY(max_bucket_array_capacity_ == capacity_)) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("cannot expand array any more", KR(ret), K(fd_), K(capacity_)); + } else if (OB_UNLIKELY(capacity_ != 0 && !is_full())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expand array when it is not full", KR(ret), K(fd_), K(size_), K(capacity_)); + } else { + int64_t new_capacity = 0 == capacity_ ? + INIT_BUCKET_ARRAY_CAPACITY: + MIN(capacity_ * 2, max_bucket_array_capacity_); + if (OB_FAIL(page_buckets_->prepare_allocate(new_capacity, nullptr))) { + LOG_WARN("fail to prepare allocate array", KR(ret), K(fd_), K(new_capacity)); + } else if (!is_empty() && right_ < left_) { + if (OB_UNLIKELY(right_ + capacity_ >= new_capacity)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid capacity", KR(ret), K(fd_), K(right_), K(capacity_), K(new_capacity)); + } else { + for (int64_t i = 0; i <= right_; ++i) { + ObTmpFilePageIndexBucket*& cur_bucket = page_buckets_->at(i); + ObTmpFilePageIndexBucket*& new_bucket = page_buckets_->at(capacity_ + i); + new_bucket = cur_bucket; + page_buckets_->at(i) = nullptr; + } + right_ += capacity_; + } + } + + if (OB_SUCC(ret)) { + capacity_ = new_capacity; + } + } + LOG_DEBUG("expand tmp file wbp index cache over", KR(ret), KPC(this)); + return ret; +} + +void ObTmpFileWBPIndexCache::shrink_() +{ + int ret = OB_SUCCESS; + LOG_DEBUG("start to shrink tmp file wbp index cache", KPC(this)); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret), K(fd_)); + } else if (OB_UNLIKELY(get_logic_tail_() - left_ + 1 != size_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("cache pos is wrong", KR(ret), K(fd_), K(left_), K(right_), K(size_), K(capacity_)); + } else if (size_ > capacity_ / SHRINK_THRESHOLD || + INIT_BUCKET_ARRAY_CAPACITY == capacity_) { + // no need to shrink + } else { + int64_t new_capacity = capacity_ / 2; + void *buf = nullptr; + uint64_t tenant_id = MTL_ID(); + ObArray *new_buckets = nullptr; + + if (OB_UNLIKELY(size_ > new_capacity)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid new_capacity", KR(ret), K(fd_), K(left_), K(right_), K(size_), K(capacity_), K(new_capacity)); + } else if (OB_ISNULL(buf = bucket_array_allocator_->alloc(sizeof(ObArray), + lib::ObMemAttr(tenant_id, "TmpFileIdxCache")))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate memory for temporary file page index bucket", + KR(ret), K(fd_), K(tenant_id), K(sizeof(ObArray))); + } else if (FALSE_IT(new_buckets = new (buf) ObArray())) { + } else if (FALSE_IT(new_buckets->set_attr(lib::ObMemAttr(tenant_id, "TmpFileIdxCache")))) { + } else if (OB_FAIL(new_buckets->prepare_allocate(new_capacity, nullptr))) { + new_buckets->destroy(); + bucket_array_allocator_->free(buf); + new_buckets = nullptr; + LOG_WARN("fail to prepare allocate array", KR(ret), K(fd_)); + } else { + for (int64_t i = left_; i <= get_logic_tail_() && OB_SUCC(ret); i++) { + ObTmpFilePageIndexBucket *bucket = page_buckets_->at(i % capacity_); + new_buckets->at(i - left_) = bucket; + } // end for + page_buckets_->destroy(); + bucket_array_allocator_->free(page_buckets_); + left_ = 0; + right_ = size_ - 1; + capacity_ = new_capacity; + page_buckets_ = new_buckets; + LOG_DEBUG("successfully shrink tmp file page index cache", K(fd_), K(size_), K(capacity_), KPC(this)); + } + } + LOG_DEBUG("shrink tmp file wbp index cache over", KR(ret), KPC(this)); +} + +int ObTmpFileWBPIndexCache::sparsify_() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(max_bucket_array_capacity_ != capacity_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected capacity", KR(ret), K(fd_), K(capacity_)); + } else if (OB_UNLIKELY(capacity_ % 2 != 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("capacity should be even", KR(ret), K(fd_), K(capacity_)); + } else if (OB_UNLIKELY(!is_full())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("cache is not full when try to eliminate half pages", KR(ret), K(fd_), K(size_), K(capacity_)); + } else { + int64_t cur_bucket_pos = left_; + for (int64_t i = left_; i <= get_logic_tail_() && OB_SUCC(ret); i++) { + if (OB_ISNULL(page_buckets_->at(i % capacity_))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", KR(ret), K(fd_), K(cur_bucket_pos), K(i), KP(page_buckets_->at(i % capacity_))); + } else if (OB_FAIL(page_buckets_->at(i % capacity_)->shrink_half())) { + LOG_WARN("fail to shrink half", KR(ret), K(fd_), K(cur_bucket_pos), K(i), KP(page_buckets_->at(i % capacity_))); + } else if ((i - left_) % 2 == 0) { + page_buckets_->at(cur_bucket_pos % capacity_) = page_buckets_->at(i % capacity_); + } else { + if (OB_FAIL(page_buckets_->at(cur_bucket_pos % capacity_)->merge(*page_buckets_->at(i % capacity_)))) { + LOG_WARN("fail to merge two buckets", KR(ret), K(fd_), K(cur_bucket_pos), K(i), + KPC(page_buckets_->at(cur_bucket_pos % capacity_)), KP(page_buckets_->at(i % capacity_))); + } else { + page_buckets_->at(i % capacity_)->destroy(); + bucket_allocator_->free(page_buckets_->at(i % capacity_)); + cur_bucket_pos++; + } + } + + if (OB_SUCC(ret) && i > (get_logic_tail_() - left_) / 2 + left_) { + page_buckets_->at(i % capacity_) = nullptr; + } + } + + if (OB_SUCC(ret)) { + right_ = (cur_bucket_pos - 1) % capacity_; + size_ /= 2; + } + } + LOG_INFO("sparsify tmp file wbp index cache over", KR(ret), KPC(this)); + return ret; +} + +ObTmpFileWBPIndexCache::ObTmpFilePageIndexBucket::ObTmpFilePageIndexBucket() : + page_indexes_(), fd_(ObTmpFileGlobal::INVALID_TMP_FILE_FD), + wbp_(nullptr), min_page_index_(ObTmpFileGlobal::INVALID_PAGE_ID) {} + +ObTmpFileWBPIndexCache::ObTmpFilePageIndexBucket::~ObTmpFilePageIndexBucket() +{ + destroy(); +} + +int ObTmpFileWBPIndexCache::ObTmpFilePageIndexBucket::init(int64_t fd, ObTmpWriteBufferPool* wbp) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", KR(ret)); + } else if (OB_UNLIKELY(fd == ObTmpFileGlobal::INVALID_TMP_FILE_FD) || OB_ISNULL(wbp)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(fd), KP(wbp)); + } else if (FALSE_IT(page_indexes_.set_attr(ObMemAttr(MTL_ID(), "TmpFileIdxBkt")))) { + } else if (OB_FAIL(page_indexes_.prepare_allocate(BUCKET_CAPACITY, ObTmpFileGlobal::INVALID_PAGE_ID))) { + LOG_WARN("fail to prepare allocate array", KR(ret)); + } else { + is_inited_ = true; + fd_ = fd; + wbp_ = wbp; + min_page_index_ = ObTmpFileGlobal::INVALID_PAGE_ID; + left_ = 0; + right_ = -1; + size_ = 0; + capacity_ = BUCKET_CAPACITY; + } + return ret; +} + +void ObTmpFileWBPIndexCache::ObTmpFilePageIndexBucket::destroy() +{ + is_inited_ = false; + fd_ = ObTmpFileGlobal::INVALID_TMP_FILE_FD; + wbp_ = nullptr; + left_ = 0; + right_ = -1; + size_ = 0; + capacity_ = 0; + min_page_index_ = ObTmpFileGlobal::INVALID_PAGE_ID; + page_indexes_.reset(); +} + +int ObTmpFileWBPIndexCache::ObTmpFilePageIndexBucket::push(const uint32_t page_index) +{ + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret), KPC(this)); + } else if (OB_UNLIKELY(is_full())) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("bucket is full", KR(ret), K(size_), K(capacity_), KPC(this)); + } else { + if (OB_UNLIKELY(is_empty())) { + min_page_index_ = page_index; + } + inc_pos_(right_); + page_indexes_[right_] = page_index; + size_ += 1; + } + + return ret; +} + +int ObTmpFileWBPIndexCache::ObTmpFilePageIndexBucket::pop_() +{ + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret), K(fd_)); + } else if (OB_UNLIKELY(is_empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("pop a empty array", KR(ret), K(size_), K(fd_)); + } else { + page_indexes_[left_] = ObTmpFileGlobal::INVALID_PAGE_ID; + inc_pos_(left_); + size_ -= 1; + + if (is_empty()) { + destroy(); + } else { + min_page_index_ = page_indexes_[left_]; + } + } + return ret; +} + +int ObTmpFileWBPIndexCache::ObTmpFilePageIndexBucket::truncate(const int64_t truncate_page_virtual_id) +{ + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (OB_UNLIKELY(is_empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("attempt binary search in an empty array", KR(ret), K(size_)); + } else { + const int64_t logic_begin_pos = left_; + const int64_t logic_end_pos = get_logic_tail_(); + bool truncate_over = false; + for (int64_t i = logic_begin_pos; OB_SUCC(ret) && !truncate_over && i <= logic_end_pos; i++) { + int64_t page_virtual_id = -1; + if (OB_UNLIKELY(page_indexes_[i % capacity_] == ObTmpFileGlobal::INVALID_PAGE_ID)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected invalid page index", KR(ret), K(page_indexes_[i % capacity_]), KPC(this)); + } else if (OB_FAIL(wbp_->get_page_virtual_id(fd_, page_indexes_[i % capacity_], page_virtual_id))) { + LOG_WARN("fail to get page virtual id in file", KR(ret), K(page_indexes_[i % capacity_]), KPC(this)); + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == page_virtual_id)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected invalid page virtual id", KR(ret), K(page_virtual_id), KPC(this)); + } else if (page_virtual_id < truncate_page_virtual_id) { + if (OB_FAIL(pop_())) { + LOG_WARN("fail to pop", KR(ret), KPC(this)); + } + } else { + truncate_over = true; + } + } // end for + } + return ret; +} + +int ObTmpFileWBPIndexCache::ObTmpFilePageIndexBucket::binary_search( + const int64_t target_page_virtual_id, uint32_t &page_index) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (OB_UNLIKELY(is_empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("attempt binary search in an empty array", KR(ret), K(size_)); + } else { + bool find = false; + int64_t left_pos = left_; + int64_t right_pos = get_logic_tail_(); + uint32_t target_page_index = ObTmpFileGlobal::INVALID_PAGE_ID; + int64_t cur_page_virtual_id = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + while (left_pos <= right_pos && !find && OB_SUCC(ret)) { + const int64_t logic_mid = left_pos + (right_pos - left_pos) / 2; + const int64_t mid = logic_mid % capacity_; + const uint32_t mid_page = page_indexes_[mid]; + if (OB_FAIL(wbp_->get_page_virtual_id(fd_, mid_page, cur_page_virtual_id))) { + LOG_WARN("fail to get page virtual id in file", KR(ret), K(logic_mid), K(mid), K(mid_page), KPC(this)); + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == cur_page_virtual_id)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected invalid page virtual id", KR(ret), K(cur_page_virtual_id), KPC(this)); + } else if (cur_page_virtual_id <= target_page_virtual_id) { + target_page_index = mid_page; + if (cur_page_virtual_id == target_page_virtual_id) { + find = true; + } else { + left_pos = logic_mid + 1; + } + } else { + right_pos = logic_mid - 1; + } + } // end while + + if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_PAGE_ID == target_page_index)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("cannot find the target page index", KR(ret), K(page_indexes_[left_]), + K(target_page_index), + K(target_page_virtual_id), KPC(this)); + } else if (find) { + page_index = target_page_index; + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_PAGE_ID == target_page_index)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("target page index doesn't exist in bucket", KR(ret), K(left_), K(page_indexes_[left_]), + K(target_page_index), + K(target_page_virtual_id), KPC(this)); + } else { + int64_t cur_page_index = target_page_index; + while (OB_SUCC(ret) && !find) { + uint32_t next_page_index = ObTmpFileGlobal::INVALID_PAGE_ID; + int64_t page_virtual_id = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + if (OB_FAIL(wbp_->get_page_virtual_id(fd_, cur_page_index, cur_page_virtual_id))) { + LOG_WARN("fail to get virtual page id", KR(ret), K(fd_), K(cur_page_index), KPC(this)); + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == cur_page_virtual_id)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected invalid page virtual id", KR(ret), K(cur_page_virtual_id), KPC(this)); + } else if (OB_FAIL(wbp_->get_next_page_id(fd_, cur_page_index, ObTmpFilePageUniqKey(cur_page_virtual_id), + next_page_index))) { + LOG_WARN("fail to get next page id", KR(ret), K(fd_), K(cur_page_index), K(cur_page_virtual_id), KPC(this)); + } else if (ObTmpFileGlobal::INVALID_PAGE_ID == next_page_index) { + ret = OB_ENTRY_NOT_EXIST; + LOG_WARN("attempt to find a non-existent page in write buffer pool", KR(ret), + K(target_page_virtual_id), KPC(this)); + } else if (OB_FAIL(wbp_->get_page_virtual_id(fd_, next_page_index, page_virtual_id))) { + LOG_WARN("fail to get virtual page id", KR(ret), K(fd_), K(next_page_index), KPC(this)); + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == page_virtual_id)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected invalid page virtual id", KR(ret), K(page_virtual_id), KPC(this)); + } else if (page_virtual_id == target_page_virtual_id) { + page_index = next_page_index; + find = true; + } else if (page_virtual_id < target_page_virtual_id) { + cur_page_index = next_page_index; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("lose target page in wbp", KR(ret), K(next_page_index), K(page_virtual_id), + K(target_page_virtual_id), KPC(this)); + } + } // end while + } + } + return ret; +} + +int ObTmpFileWBPIndexCache::ObTmpFilePageIndexBucket::shrink_half() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(capacity_ % 2 != 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("capacity_ should be even", KR(ret), K(capacity_), KPC(this)); + } else if (size_ <= capacity_ / 2) { + // no need to shrink, do nothing + } else { + const int64_t eliminate_num = size_ - capacity_ / 2; + const int64_t eliminate_modulus = size_ / eliminate_num; + int64_t remain_eliminate_num = eliminate_num; + int64_t logic_pos = left_; + for (int64_t i = left_; i <= get_logic_tail_(); i++) { + if ((i - left_) % eliminate_modulus != 0 || 0 == remain_eliminate_num) { + page_indexes_[logic_pos % capacity_] = page_indexes_[i % capacity_]; + logic_pos++; + } else { // eliminate current page index + remain_eliminate_num--; + } + } + + for (int64_t i = logic_pos; i <= get_logic_tail_(); i++) { + page_indexes_[i % capacity_] = ObTmpFileGlobal::INVALID_PAGE_ID; + } + + right_ = (logic_pos - 1) % capacity_; + size_ -= eliminate_num; + + if (!is_empty()) { + min_page_index_ = page_indexes_[left_]; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("tmp file page index bucket is unexpected empty", KR(ret), KPC(this)); + } + } + return ret; +} + +int ObTmpFileWBPIndexCache::ObTmpFilePageIndexBucket::merge(ObTmpFilePageIndexBucket& other) +{ + int ret = OB_SUCCESS; + + if (OB_UNLIKELY(other.is_empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("attempt to merge an empty array", KR(ret), K(other.size_), KPC(this)); + } else if (OB_UNLIKELY(other.size_ + size_ > capacity_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("attempt to merge array with too many elements", KR(ret), K(size_), K(other.size_), K(capacity_), KPC(this)); + } else if (OB_UNLIKELY(!is_empty())) { + int64_t bkt_min_page_virtual_id = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + int64_t other_bkt_min_page_virtual_id = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + if (OB_FAIL(wbp_->get_page_virtual_id(fd_, min_page_index_, bkt_min_page_virtual_id))) { + LOG_WARN("fail to get page virtual id", KR(ret), K(bkt_min_page_virtual_id), KPC(this)); + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == bkt_min_page_virtual_id)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected invalid page virtual id", KR(ret), K(bkt_min_page_virtual_id), KPC(this)); + } else if (OB_FAIL(wbp_->get_page_virtual_id(fd_, other.min_page_index_, other_bkt_min_page_virtual_id))) { + LOG_WARN("fail to get page virtual id", KR(ret), K(other_bkt_min_page_virtual_id), KPC(this)); + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == other_bkt_min_page_virtual_id)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected invalid page virtual id", KR(ret), K(other_bkt_min_page_virtual_id), KPC(this)); + } else if (OB_UNLIKELY(bkt_min_page_virtual_id > other_bkt_min_page_virtual_id)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("attempt to merge a bucket whose min_page_virtual_id is less that itself", KR(ret), + K(min_page_index_), K(other_bkt_min_page_virtual_id), KPC(this)); + } + } + + if (OB_SUCC(ret)) { + for (int64_t i = other.left_; OB_SUCC(ret) && i <= other.get_logic_tail_(); i++) { + uint32_t page_index = other.page_indexes_[i % other.capacity_]; + if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_PAGE_ID == page_index)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid page index", KR(ret), K(page_index), K(i), KPC(this)); + } else if (OB_FAIL(push(page_index))) { + LOG_WARN("fail to push a page index", KR(ret), K(page_index), KPC(this)); + } + } + } + return ret; +} + +} // end namespace tmp_file +} // end namespace oceanbase diff --git a/src/storage/tmp_file/ob_tmp_file_write_buffer_index_cache.h b/src/storage/tmp_file/ob_tmp_file_write_buffer_index_cache.h new file mode 100644 index 000000000..3650ebff5 --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_write_buffer_index_cache.h @@ -0,0 +1,145 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_WRITE_BUFFER_INDEX_CACHE_H_ +#define OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_WRITE_BUFFER_INDEX_CACHE_H_ + +#include "lib/container/ob_array.h" +#include "lib/allocator/ob_fifo_allocator.h" + +namespace oceanbase +{ +namespace tmp_file +{ +class ObTmpWriteBufferPool; + +class ObTmpFileCircleArray +{ +public: + ObTmpFileCircleArray() : is_inited_(false), left_(0), right_(-1), size_(0), capacity_(0) {} + virtual ~ObTmpFileCircleArray() {}; + virtual int push(const uint32_t page_index) = 0; + virtual int truncate(const int64_t truncate_page_virtual_id) = 0; + virtual int binary_search(const int64_t target_page_virtual_id, uint32_t &page_index) = 0; + +public: + OB_INLINE int64_t size() const { return size_; } + OB_INLINE bool is_empty() const { return size_ == 0; } + virtual OB_INLINE bool is_full() const { return size_ == capacity_; } + TO_STRING_KV(K(is_inited_), K(left_), K(right_), K(size_), K(capacity_)); + +protected: + OB_INLINE void inc_pos_(int64_t &pos) { pos = pos == capacity_ - 1 ? 0 : pos + 1; } + OB_INLINE void dec_pos_(int64_t &pos) { pos = pos == 0 ? capacity_ - 1 : pos - 1; } + OB_INLINE int64_t get_previous_pos(const int64_t pos) const { return pos == 0 ? capacity_ - 1 : pos - 1; } + OB_INLINE int64_t get_logic_tail_() const { return get_logic_pos_(right_); } + OB_INLINE int64_t get_logic_pos_(const int64_t pos) const + { + return pos >= left_ ? pos : capacity_ + pos; + } + +protected: + bool is_inited_; + int64_t left_; // close interval + int64_t right_; // close interval + int64_t size_; // record valid number of array + int64_t capacity_; // record the capacity of array +}; + +// Attention: +// ObTmpFileWBPIndexCache caches page indexes of a tmp file in write buffer pool. +// When ObTmpFileWBPIndexCache reaches the limitation of capacity, +// it will eliminates half page indexes of buckets for caching new page index. +// thus, the range of cached indexes might be not continuous. +class ObTmpFileWBPIndexCache : public ObTmpFileCircleArray +{ +public: + ObTmpFileWBPIndexCache(); + ~ObTmpFileWBPIndexCache(); + int init(const int64_t fd, ObTmpWriteBufferPool* wb, + ObIAllocator *wbp_index_cache_allocator, + ObIAllocator *wbp_index_cache_bkt_allocator); + void destroy(); + void reset(); + virtual int push(const uint32_t page_index) override; + // truncate page virtual id is open interval + virtual int truncate(const int64_t truncate_page_virtual_id) override; + // truncate page virtual id is close interval + virtual int binary_search(const int64_t target_page_virtual_id, uint32_t &page_index) override; + virtual OB_INLINE bool is_full() const override + { + return ObTmpFileCircleArray::is_full() && + right_ >= 0 && right_ < capacity_ && + OB_NOT_NULL(page_buckets_) && + OB_NOT_NULL(page_buckets_->at(right_)) && + page_buckets_->at(right_)->is_full(); + } + INHERIT_TO_STRING_KV("ObTmpFileCircleArray", ObTmpFileCircleArray, + K(fd_), KP(wbp_), KP(page_buckets_), + KP(bucket_array_allocator_), KP(bucket_allocator_)); + +private: + class ObTmpFilePageIndexBucket : public ObTmpFileCircleArray + { + public: + ObTmpFilePageIndexBucket(); + ~ObTmpFilePageIndexBucket(); + int init(int64_t fd, ObTmpWriteBufferPool* wbp); + void destroy(); + virtual int push(const uint32_t page_index) override; + // truncate offset is open interval + virtual int truncate(const int64_t truncate_page_virtual_id) override; + // target offset is close interval + virtual int binary_search(const int64_t target_page_virtual_id, uint32_t &page_index) override; + int shrink_half(); + int merge(ObTmpFilePageIndexBucket& other); + OB_INLINE int64_t get_min_page_index() const { return min_page_index_; } + INHERIT_TO_STRING_KV("ObTmpFileCircleArray", ObTmpFileCircleArray, + K(fd_), KP(wbp_), K(min_page_index_)); + private: + int pop_(); + private: + // due to each page index in bucket directs to a 8KB page in wbp, + // a bucket could indicate a 256KB data and hold on 128B memory + static const int64_t BUCKET_CAPACITY = 1 << 5; + + private: + common::ObArray page_indexes_; + int64_t fd_; + ObTmpWriteBufferPool* wbp_; + int64_t min_page_index_; + }; + +private: + // due to each bucket could indicate a 256KB data in wbp, + // the cache could indicate a 256MB data and hold on (128 + 8)*2^10 = 136KB memory at most + static const int64_t MAX_BUCKET_ARRAY_CAPACITY = 1 << 10; + static const int64_t INIT_BUCKET_ARRAY_CAPACITY = 1 << 3; // indicates 2MB data in wbp at most + static const int64_t SHRINK_THRESHOLD = 4; // attention: this value must be larger than 2 + +private: + int expand_(); + void shrink_(); + int sparsify_(); + +private: + ObIAllocator *bucket_array_allocator_; + ObIAllocator *bucket_allocator_; + common::ObArray *page_buckets_; + int64_t fd_; + ObTmpWriteBufferPool* wbp_; + int64_t max_bucket_array_capacity_; // only allowed to modify this var in unit test!!! +}; + +} // end namespace tmp_file +} // end namespace oceanbase +#endif // OCEANBASE_STORAGE_BLOCKSSTABLE_TMP_FILE_OB_TMP_FILE_WRITE_BUFFER_INDEX_CACHE_H_ diff --git a/src/storage/tmp_file/ob_tmp_file_write_buffer_pool.cpp b/src/storage/tmp_file/ob_tmp_file_write_buffer_pool.cpp new file mode 100644 index 000000000..13190e11f --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_write_buffer_pool.cpp @@ -0,0 +1,988 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE + +#include "share/config/ob_server_config.h" +#include "storage/tmp_file/ob_tmp_file_write_buffer_pool.h" +#include "observer/omt/ob_tenant_config_mgr.h" +#include "storage/blocksstable/ob_block_manager.h" + +namespace oceanbase +{ +namespace tmp_file +{ + +int ObPageEntry::switch_state(const int64_t op) +{ + int ret = OB_SUCCESS; + static const int64_t N = State::N; + static const int64_t INV = State::INVALID; + static const int64_t INITED = State::INITED; + static const int64_t LOADING = State::LOADING; + static const int64_t CACHED = State::CACHED; + static const int64_t DIRTY = State::DIRTY; + static const int64_t WRITE_BACK = State::WRITE_BACK; + static const int64_t MAX = State::MAX; + + static const int64_t STATE_MAP[State::MAX][Ops::MAX] = { + // ALLOC, LOAD, LOAD_FAIL, LOAD_SUCC, DELETE, WRITE, WRITE_BACK, WRITE_BACK_FAILED, WRITE_BACK_SUCC + {INITED, N, N, N, INV, N, N, N, N}, //INVALID + {N, LOADING, N, N, INV, DIRTY, N, N, N}, //INITED + {N, N, INITED, CACHED, N, N, N, N, N}, //LOADING + {N, N, N, N, INV, DIRTY, N, N, CACHED}, //CACHED + {N, N, N, N, INV, DIRTY, WRITE_BACK, N, N}, //DIRTY + {N, N, N, N, INV, DIRTY, WRITE_BACK, DIRTY, CACHED} //WRITE_BACK + }; + + if (OB_UNLIKELY(!Ops::is_valid(op))) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("invalid operation", KR(ret), K(op)); + } else if (OB_UNLIKELY(!State::is_valid(state_))) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("invalid state", KR(ret), K(state_)); + } else { + const int64_t n_stat = STATE_MAP[state_][op]; + if (OB_UNLIKELY(!State::is_valid(n_stat))) { + ret = OB_STATE_NOT_MATCH; + LOG_ERROR("invalid state transition", KR(ret), K(state_), K(op), K(n_stat)); + } else { + state_ = n_stat; + } + } + return ret; +} + +double ObTmpWriteBufferPool::MAX_DATA_PAGE_USAGE_RATIO = 0.9; + +ObTmpWriteBufferPool::ObTmpWriteBufferPool() + : fat_(), + lock_(), + allocator_(), + is_inited_(false), + capacity_(0), + dirty_page_num_(0), + used_page_num_(0), + first_free_page_id_(ObTmpFileGlobal::INVALID_PAGE_ID), + wbp_memory_limit_(-1), + default_wbp_memory_limit_(-1), + last_access_tenant_config_ts_(-1), + meta_page_cnt_(0), + data_page_cnt_(0), + dirty_meta_page_cnt_(0), + dirty_data_page_cnt_(0), + write_back_data_cnt_(0), + write_back_meta_cnt_(0) +{ +} + +ObTmpWriteBufferPool::~ObTmpWriteBufferPool() +{ + destroy(); +} + +int ObTmpWriteBufferPool::init() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(is_inited_)) { + ret = OB_INIT_TWICE; + LOG_WARN("fail to init wbp, init twice", KR(ret), K(is_inited_)); + } else if (OB_FAIL(allocator_.init( + lib::ObMallocAllocator::get_instance(), OB_MALLOC_BIG_BLOCK_SIZE, + ObMemAttr(MTL_ID(), "TmpFileWBP", ObCtxIds::DEFAULT_CTX_ID)))) { + LOG_WARN("wbp fail to init fifo allocator", KR(ret)); + } else if (FALSE_IT(fat_.set_attr(ObMemAttr(MTL_ID(), "TmpFileWBP")))) { + } else if (OB_FAIL(expand_())) { + LOG_WARN("wbp fail to expand capacity", KR(ret)); + } else { + is_inited_ = true; + } + return ret; +} + +void ObTmpWriteBufferPool::destroy() +{ + reduce_(); + capacity_ = 0; + dirty_page_num_ = 0; + used_page_num_ = 0; + first_free_page_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + last_access_tenant_config_ts_ = -1; + data_page_cnt_ = 0; + meta_page_cnt_ = 0; + dirty_meta_page_cnt_ = 0; + dirty_data_page_cnt_ = 0; + write_back_data_cnt_ = 0; + write_back_meta_cnt_ = 0; + wbp_memory_limit_ = -1; + default_wbp_memory_limit_ = -1; + fat_.destroy(); + allocator_.reset(); + is_inited_ = false; +} + +// limit data pages to use a maximum of 90% of the total space in the write buffer pool; +// considering that the total amount of meta pages for a single file accounts for less than 1% of data pages, +// there is no limit set for meta pages when allocating pages. +int ObTmpWriteBufferPool::inner_alloc_page_(const int64_t fd, + const ObTmpFilePageUniqKey page_key, + uint32_t &new_page_id, + char *&new_page_buf) +{ + int ret = OB_SUCCESS; + uint32_t curr_first_free_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + uint32_t next_first_free_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + + common::TCRWLock::RLockGuard guard(lock_); + // limit page allocation depending on page type + if (has_free_page_(page_key.type_)) { + // fetch a page from the free list through CAS operation + bool cas_succeed = false; + do { + curr_first_free_page_id = ATOMIC_LOAD(&first_free_page_id_); + if (!is_valid_page_id_(curr_first_free_page_id)) { + ret = OB_SEARCH_NOT_FOUND; + break; + } + next_first_free_page_id = fat_[curr_first_free_page_id].next_page_id_; + cas_succeed = ATOMIC_BCAS(&first_free_page_id_, curr_first_free_page_id, next_first_free_page_id); + } while (OB_SUCC(ret) && !cas_succeed); + + if (OB_SUCC(ret) && is_valid_page_id_(curr_first_free_page_id)) { + fat_[curr_first_free_page_id].fd_ = fd; + fat_[curr_first_free_page_id].next_page_id_ = ObTmpFileGlobal::INVALID_PAGE_ID; + fat_[curr_first_free_page_id].page_key_ = page_key; + fat_[curr_first_free_page_id].switch_state(ObPageEntry::Ops::ALLOC); + new_page_id = curr_first_free_page_id; + new_page_buf = fat_[new_page_id].buf_; + ATOMIC_INC(&used_page_num_); + } + } + + if (ObTmpFileGlobal::INVALID_PAGE_ID == new_page_id) { + ret = OB_SEARCH_NOT_FOUND; + } + + return ret; +} + +int ObTmpWriteBufferPool::alloc_page_(const int64_t fd, + const ObTmpFilePageUniqKey page_key, + uint32_t &new_page_id, + char *&new_page_buf) +{ + int ret = OB_SUCCESS; + + new_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + new_page_buf = nullptr; + + // validate input argument. + if (ObTmpFileGlobal::INVALID_TMP_FILE_FD == fd) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("wbp fail to alloc_page, invalid fd", KR(ret), K(fd)); + } + + int64_t memory_limit = 0; + int64_t current_capacity = -1; + // continuously trying to allocate page and expand pool size, until capacity reach maximum memory limit. + while (OB_SUCC(ret) && ObTmpFileGlobal::INVALID_PAGE_ID == new_page_id && current_capacity < memory_limit) { + if (OB_FAIL(inner_alloc_page_(fd, page_key, new_page_id, new_page_buf))) { + if (OB_SEARCH_NOT_FOUND != ret) { + LOG_WARN("wbp fail to inner alloc page", KR(ret), K(fd), K(page_key), K(new_page_id), KP(new_page_buf)); + } else { // no free pages, try to expand pool size + ret = OB_SUCCESS; + memory_limit = get_memory_limit(); + current_capacity = ATOMIC_LOAD(&capacity_); + if (current_capacity < memory_limit && OB_FAIL(expand_())) { + LOG_WARN("wbp fail to expand", KR(ret), K(fd), K(ATOMIC_LOAD(&capacity_))); + } + } + } + } + + if (OB_SUCC(ret) && ObTmpFileGlobal::INVALID_PAGE_ID == new_page_id) { + ret = OB_ALLOCATE_TMP_FILE_PAGE_FAILED; // reaches maximum memory limit, can not allocate page + } + + return ret; +} + +int ObTmpWriteBufferPool::alloc_page(const int64_t fd, + const ObTmpFilePageUniqKey page_key, + uint32_t &new_page_id, + char *&new_page_buf) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_TMP_FILE_FD == fd || !page_key.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(fd), K(page_key)); + } else if (OB_FAIL(alloc_page_(fd, page_key, new_page_id, new_page_buf))) { + LOG_WARN("wbp fail to alloc page", KR(ret), K(fd), K(page_key)); + } else if (page_key.type_ == PageEntryType::META) { + ATOMIC_INC(&meta_page_cnt_); + LOG_INFO("alloc meta page", KR(ret), K(new_page_id), K(fd)); + } else { + ATOMIC_INC(&data_page_cnt_); + LOG_DEBUG("alloc data page", KR(ret), K(new_page_id), K(fd)); + } + return ret; +} + +int ObTmpWriteBufferPool::get_next_page_id( + const int64_t fd, + const uint32_t page_id, + const ObTmpFilePageUniqKey page_key, + uint32_t &next_page_id) +{ + int ret = OB_SUCCESS; + common::TCRWLock::RLockGuard guard(lock_); + if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_TMP_FILE_FD == fd || + !is_valid_page_id_(page_id) || !page_key.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(fd), K(page_id), K(page_key)); + } else if (OB_UNLIKELY(fd != fat_[page_id].fd_)) { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("fd not match", KR(ret), K(fd), K(page_id), K(fat_[page_id])); + } else if (OB_UNLIKELY(page_key != fat_[page_id].page_key_)) { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("page key not match", KR(ret), K(page_key), K(page_id), K(fat_[page_id])); + } else { + next_page_id = ATOMIC_LOAD(&fat_[page_id].next_page_id_); + } + return ret; +} + +int ObTmpWriteBufferPool::read_page( + const int64_t fd, + const uint32_t page_id, + const ObTmpFilePageUniqKey page_key, + char *&buf, + uint32_t &next_page_id) +{ + int ret = OB_SUCCESS; + buf = nullptr; + next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + common::TCRWLock::RLockGuard guard(lock_); + + if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_TMP_FILE_FD == fd || + !is_valid_page_id_(page_id) || + !page_key.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("wbp fail to read page, invalid page id", KR(ret), K(page_id), K(fd), K(page_key)); + } else if (OB_UNLIKELY(fd != fat_[page_id].fd_ || page_key != fat_[page_id].page_key_)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("wbp fail to fetch page, PageEntry fd or offset not equal", KR(ret), K(page_id), K(fd), + K(page_key), K(fat_[page_id])); + } else { + buf = fat_[page_id].buf_; + next_page_id = fat_[page_id].next_page_id_; + } + return ret; +} + +int ObTmpWriteBufferPool::get_page_id_by_virtual_id(const int64_t fd, + const int64_t virtual_page_id, + const uint32_t begin_page_id, + uint32_t &page_id) +{ + int ret = OB_SUCCESS; + page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + + common::TCRWLock::RLockGuard guard(lock_); + if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_TMP_FILE_FD == fd || + ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == virtual_page_id || + !is_valid_page_id_(begin_page_id) || + fd != fat_[begin_page_id].fd_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(fd), K(virtual_page_id), K(begin_page_id)); + } else if (virtual_page_id < fat_[begin_page_id].page_key_.virtual_page_id_) { + ret = OB_SEARCH_NOT_FOUND; + LOG_WARN("virtual_page_id is smaller than that of page of begin_page_id", + KR(ret), K(virtual_page_id), K(begin_page_id), + K(fat_[begin_page_id].page_key_.virtual_page_id_)); + } else { + uint32_t cur_page_id = begin_page_id; + while (cur_page_id != ObTmpFileGlobal::INVALID_PAGE_ID) { // iter to the end of this file + if (fat_[cur_page_id].page_key_.virtual_page_id_ >= virtual_page_id) { + if (fat_[cur_page_id].page_key_.virtual_page_id_ >= virtual_page_id) { + page_id = cur_page_id; + } + break; + } else { + cur_page_id = fat_[cur_page_id].next_page_id_; + } + } + if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_PAGE_ID == page_id)) { + ret = OB_ITER_END; + LOG_WARN("wbp fail to find page by given offset", KR(ret), K(virtual_page_id), K(begin_page_id), K(fat_[begin_page_id])); + } + } + return ret; +} + +int ObTmpWriteBufferPool::get_page_virtual_id(const int64_t fd, const uint32_t page_id, int64_t &virtual_page_id) +{ + int ret = OB_SUCCESS; + common::TCRWLock::RLockGuard guard(lock_); + if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_TMP_FILE_FD == fd || !is_valid_page_id_(page_id))) { + ret = OB_SEARCH_NOT_FOUND; + LOG_WARN("wbp fail to get page offset in file, invalid page id", KR(ret), K(fd), K(page_id)); + } else if (fd != fat_[page_id].fd_) { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("wbp fail to get page offset in file, fd not match", KR(ret), K(fd), K(page_id), K(fat_[page_id])); + } else { + virtual_page_id = fat_[page_id].page_key_.virtual_page_id_; + } + return ret; +} + +int ObTmpWriteBufferPool::truncate_page(const int64_t fd, const uint32_t page_id, + const ObTmpFilePageUniqKey page_key, + const int64_t truncate_size) +{ + int ret = OB_SUCCESS; + common::TCRWLock::RLockGuard guard(lock_); + if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_TMP_FILE_FD == fd || + !is_valid_page_id_(page_id) || !page_key.is_valid() || + truncate_size > ObTmpFileGlobal::PAGE_SIZE || truncate_size <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(fd), K(page_id), K(page_key), K(truncate_size)); + } else if (fd != fat_[page_id].fd_) { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("wbp fail to truncate page, fd not match", KR(ret), K(fd), K(page_id), K(fat_[page_id])); + } else if (page_key != fat_[page_id].page_key_) { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("wbp fail to truncate page, page_key not match", KR(ret), K(page_key), K(page_id), K(fat_[page_id])); + } else { + MEMSET(fat_[page_id].buf_, 0, truncate_size); + } + return ret; +} + +int ObTmpWriteBufferPool::link_page( + const int64_t fd, + const uint32_t page_id, + const uint32_t prev_page_id, + const ObTmpFilePageUniqKey prev_page_key) +{ + int ret = OB_SUCCESS; + common::TCRWLock::RLockGuard guard(lock_); + if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_TMP_FILE_FD == fd || + !is_valid_page_id_(page_id) || !is_valid_page_id_(prev_page_id) || + !prev_page_key.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(page_id), K(prev_page_id), K(prev_page_key)); + } else if (OB_UNLIKELY(fat_[page_id].fd_ != fd || fat_[prev_page_id].fd_ != fd || + fat_[prev_page_id].next_page_id_ != ObTmpFileGlobal::INVALID_PAGE_ID || + prev_page_key != fat_[prev_page_id].page_key_)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("fail to link page, unexpected page id or offset", KR(ret), K(fd), + K(page_id), K(fat_[page_id]), K(prev_page_key), K(prev_page_id), + K(fat_[prev_page_id])); + } else if (prev_page_key.type_ == PageEntryType::META) { + //just for meta page check + ObTmpFilePageUniqKey page_key(prev_page_key.tree_level_, prev_page_key.level_page_index_ + 1); + if (OB_UNLIKELY(prev_page_key != fat_[prev_page_id].page_key_ + || page_key != fat_[page_id].page_key_)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("fail to link meta page, unexpected page offset", KR(ret), K(fd), K(prev_page_key), K(page_key), + K(fat_[prev_page_id]), K(fat_[page_id])); + } + } + + if (OB_SUCC(ret)) { + fat_[prev_page_id].next_page_id_ = page_id; + if (prev_page_key.type_ == PageEntryType::META) { + LOG_INFO("link meta page", KR(ret), K(fd), K(page_id), K(prev_page_id)); + } else { + LOG_DEBUG("link data page", KR(ret), K(fd), K(page_id), K(prev_page_id)); + } + } + return ret; +} + +int ObTmpWriteBufferPool::free_page( + const int64_t fd, + const uint32_t page_id, + const ObTmpFilePageUniqKey page_key, + uint32_t &next_page_id) +{ + int ret = OB_SUCCESS; + next_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + common::TCRWLock::RLockGuard guard(lock_); + + if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_TMP_FILE_FD == fd || + !is_valid_page_id_(page_id) || + !page_key.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(fd), K(page_id), K(page_key)); + } else if (OB_UNLIKELY(fd != fat_[page_id].fd_ + || page_key != fat_[page_id].page_key_)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("wbp fail to free page, fd or offset not equal", KR(ret), K(page_id), K(fd), + K(page_key), K(fat_[page_id])); + } else { + // reset PageEntry, add it to free list head and update buffer pool statistics + next_page_id = ATOMIC_LOAD(&(fat_[page_id].next_page_id_)); + PageEntryType page_type = fat_[page_id].page_key_.type_; + if (PageEntryType::DATA == page_type) { + LOG_DEBUG("free data page", KR(ret), K(page_id), K(fd), K(fat_[page_id])); + } else { + LOG_INFO("free meta page", KR(ret), K(page_id), K(fd), K(fat_[page_id])); + } + ATOMIC_SET(&(fat_[page_id].fd_), -1); + ATOMIC_SET(&(fat_[page_id].next_page_id_), ObTmpFileGlobal::INVALID_PAGE_ID); + if (ObPageEntry::State::DIRTY == ATOMIC_LOAD(&fat_[page_id].state_)) { + ATOMIC_DEC(&dirty_page_num_); + if (PageEntryType::DATA == page_type) { + ATOMIC_DEC(&dirty_data_page_cnt_); + } else if (PageEntryType::META == page_type) { + ATOMIC_DEC(&dirty_meta_page_cnt_); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("invalid tmp file page type", KR(ret), K(page_id), K(fd), K(page_type)); + } + } + if (ObPageEntry::State::WRITE_BACK == ATOMIC_LOAD(&fat_[page_id].state_)) { + if (PageEntryType::DATA == page_type) { + ATOMIC_DEC(&write_back_data_cnt_); + } else if (PageEntryType::META == page_type) { + ATOMIC_DEC(&write_back_meta_cnt_); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("invalid tmp file page type", KR(ret), K(page_id), K(fd), K(page_type)); + } + } + fat_[page_id].switch_state(ObPageEntry::Ops::DELETE); + fat_[page_id].page_key_.reset(); + + bool cas_succeed = false; + do { + uint32_t first_free_page_id_before = ATOMIC_LOAD(&first_free_page_id_); + ATOMIC_SET(&(fat_[page_id].next_page_id_), first_free_page_id_before); + cas_succeed = ATOMIC_BCAS(&first_free_page_id_, first_free_page_id_before, page_id); + } while (false == cas_succeed); + + ATOMIC_DEC(&used_page_num_); + + if (PageEntryType::DATA == page_type) { + ATOMIC_DEC(&data_page_cnt_); + } else if (PageEntryType::META == page_type) { + ATOMIC_DEC(&meta_page_cnt_); + } else { + LOG_ERROR("invalid tmp file page type", KR(ret), K(page_id), K(fd), K(page_type)); + } + } + return ret; +} + +// allocate a block size of WBP_BLOCK_SIZE each iteration +// therefore max capacity may slightly exceed memory_limit +int ObTmpWriteBufferPool::expand_() +{ + int ret = OB_SUCCESS; + + // expand the buffer pool to twice the current size, with a minimum of WBP_BLOCK_SIZE + const int64_t memory_limit = get_memory_limit(); + int64_t current_capacity = ATOMIC_LOAD(&capacity_); + const int64_t expect_capacity = std::min( + memory_limit, std::max(current_capacity * 2, int64_t(WBP_BLOCK_SIZE))); + + // continuously allocate 2MB blocks and add them into the buffer pool. + while (OB_SUCC(ret) && current_capacity < expect_capacity) { + common::TCRWLock::WLockGuard guard(lock_); + current_capacity = ATOMIC_LOAD(&capacity_); + if (current_capacity < expect_capacity) { + char * new_expand_buf = nullptr; + // allocate a chunk of WBP_BLOCK_SIZE each time + if (OB_ISNULL(new_expand_buf = static_cast(allocator_.alloc(WBP_BLOCK_SIZE)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("wbp fail to allocate new expand buffer", KR(ret)); + } else { + uint32_t new_page_id = fat_.count(); + for (uint32_t count = 0; OB_SUCC(ret) && count < BLOCK_PAGE_NUMS; ++new_page_id, ++count) { + if (OB_FAIL(fat_.push_back(ObPageEntry(-1, + ObTmpFileGlobal::INVALID_PAGE_ID, + new_expand_buf + count * ObTmpFileGlobal::PAGE_SIZE)))) { + LOG_WARN("wbp fail to push back page into fat", KR(ret), K(count), K(new_page_id)); + } else { + fat_[new_page_id].next_page_id_ = ATOMIC_LOAD(&first_free_page_id_); + ATOMIC_SET(&first_free_page_id_, new_page_id); + ATOMIC_FAA(&capacity_, ObTmpFileGlobal::PAGE_SIZE); + } + } + current_capacity += WBP_BLOCK_SIZE; + } + } else { + // maybe another thread has finish allocation, do nothing. + } + } + + LOG_INFO("wbp expand", K(expect_capacity), K(memory_limit), K(ATOMIC_LOAD(&capacity_))); + + return ret; +} + +int ObTmpWriteBufferPool::reduce_() +{ + int ret = OB_SUCCESS; + // TODO(wendongbo): write buffer pool shrinking is currently not supported, use it as destroy() now + common::TCRWLock::WLockGuard guard(lock_); + for (int64_t i = 0; i < fat_.count(); i += BLOCK_PAGE_NUMS) { + char * buf = fat_.at(i).buf_; + if (OB_ISNULL(buf)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("wbp get unexpected page buffer", KR(ret), K(i), KP(buf)); + } else { + allocator_.free(buf); + } + } + return ret; +} + +int64_t ObTmpWriteBufferPool::get_memory_limit() +{ + int64_t memory_limit = 0; + int64_t last_access_ts = ATOMIC_LOAD(&last_access_tenant_config_ts_); + if (default_wbp_memory_limit_ > 0) { + memory_limit = default_wbp_memory_limit_; + } else if (last_access_ts > 0 && common::ObClockGenerator::getClock() - last_access_ts < 10000000) { // 10s + memory_limit = ATOMIC_LOAD(&wbp_memory_limit_); + } else { + omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID())); + if (!tenant_config.is_valid()) { + static const int64_t DEFAULT_MEMORY_LIMIT = 64 * 2 * 1024 * 1024; // 128MB + memory_limit = wbp_memory_limit_ == 0 ? DEFAULT_MEMORY_LIMIT : wbp_memory_limit_; + LOG_INFO("failed to get tenant config", K(MTL_ID()), K(memory_limit), K(wbp_memory_limit_)); + } else if (0 == tenant_config->_temporary_file_io_area_size) { + memory_limit = 0; + } else { + memory_limit = common::upper_align( + lib::get_tenant_memory_limit(MTL_ID()) * tenant_config->_temporary_file_io_area_size / 100, + WBP_BLOCK_SIZE); + } + ATOMIC_STORE(&wbp_memory_limit_, memory_limit); + ATOMIC_STORE(&last_access_tenant_config_ts_, common::ObClockGenerator::getClock()); + } + return memory_limit; +} + +// expect swap min(10% * page cache memory, 20MB) each time +// if clean data size smaller than this min_swap_size return 0 +int64_t ObTmpWriteBufferPool::get_swap_size() +{ + const int64_t HIGH_WATERMARK_PECENTAGE = 55; + const int64_t LOW_WATERMARK_PECENTAGE = 30; + + int64_t memory_limit = get_memory_limit(); + int64_t used_page_num = ATOMIC_LOAD(&used_page_num_); + + int64_t high_watermark_bytes = (double)HIGH_WATERMARK_PECENTAGE / 100 * memory_limit; + int64_t low_watermark_bytes = (double)LOW_WATERMARK_PECENTAGE / 100 * memory_limit; + int64_t used_bytes = used_page_num * ObTmpFileGlobal::PAGE_SIZE; + + const int64_t MACRO_BLOCK_SIZE = OB_SERVER_BLOCK_MGR.get_macro_block_size(); + + int64_t swap_size = 0; + if (used_bytes > high_watermark_bytes) { + int64_t expected_swap_size = used_bytes - low_watermark_bytes; + int64_t dirty_data_bytes = ATOMIC_LOAD(&dirty_page_num_) * ObTmpFileGlobal::PAGE_SIZE; + swap_size = min(used_bytes - dirty_data_bytes, expected_swap_size); + } + + return swap_size; +} + +bool ObTmpWriteBufferPool::is_exist(const int64_t fd, const uint32_t page_id, + const ObTmpFilePageUniqKey page_key) +{ + common::TCRWLock::RLockGuard guard(lock_); + bool exist = false; + if (OB_LIKELY(is_valid_page_id_(page_id) + && fd == fat_[page_id].fd_ + && page_key.is_valid() + && page_key == fat_[page_id].page_key_)) { + exist = ObPageEntry::State::INVALID < fat_[page_id].state_; + } else { + int ret = OB_ERR_UNEXPECTED; // TODO: too many warn logs + LOG_WARN("wbp get unexpected page entry", KR(ret), K(fd), K(page_id), K(page_key), K(fat_[page_id])); + } + return exist; +} + +bool ObTmpWriteBufferPool::is_inited(const int64_t fd, const uint32_t page_id, + const ObTmpFilePageUniqKey page_key) +{ + common::TCRWLock::RLockGuard guard(lock_); + bool inited = false; + if (OB_LIKELY(is_valid_page_id_(page_id) + && fd == fat_[page_id].fd_ + && page_key.is_valid() + && page_key == fat_[page_id].page_key_)) { + inited = ObPageEntry::State::INITED == fat_[page_id].state_; + } else { + int ret = OB_ERR_UNEXPECTED; + LOG_ERROR("wbp get unexpected page entry", KR(ret), K(fd), K(page_id), K(page_key), K(fat_[page_id])); + } + return inited; +} + +bool ObTmpWriteBufferPool::is_loading(const int64_t fd, const uint32_t page_id, + const ObTmpFilePageUniqKey page_key) +{ + common::TCRWLock::RLockGuard guard(lock_); + bool loading = false; + if (OB_LIKELY(is_valid_page_id_(page_id) + && fd == fat_[page_id].fd_ + && page_key.is_valid() + && page_key == fat_[page_id].page_key_)) { + loading = ObPageEntry::State::LOADING == fat_[page_id].state_; + } else { + int ret = OB_ERR_UNEXPECTED; + LOG_ERROR("wbp get unexpected page entry", KR(ret), K(fd), K(page_id), K(page_key), K(fat_[page_id])); + } + return loading; +} + +bool ObTmpWriteBufferPool::is_cached( + const int64_t fd, + const uint32_t page_id, + const ObTmpFilePageUniqKey page_key) +{ + common::TCRWLock::RLockGuard guard(lock_); + bool cached = false; + if (OB_LIKELY(is_valid_page_id_(page_id) + && fd == fat_[page_id].fd_ + && page_key.is_valid() + && page_key == fat_[page_id].page_key_)) { + cached = ObPageEntry::State::CACHED == fat_[page_id].state_; + } else { + int ret = OB_ERR_UNEXPECTED; + LOG_ERROR("wbp get unexpected page entry", KR(ret), K(fd), K(page_id), K(page_key), K(fat_[page_id])); + } + return cached; +} + +bool ObTmpWriteBufferPool::is_write_back(const int64_t fd, const uint32_t page_id, + const ObTmpFilePageUniqKey page_key) +{ + common::TCRWLock::RLockGuard guard(lock_); + bool write_back = false; + if (OB_LIKELY(is_valid_page_id_(page_id) + && fd == fat_[page_id].fd_ + && page_key.is_valid() + && page_key == fat_[page_id].page_key_)) { + write_back = ObPageEntry::State::WRITE_BACK == fat_[page_id].state_; + } else { + int ret = OB_ERR_UNEXPECTED; + LOG_ERROR("wbp get unexpected page entry", KR(ret), K(fd), K(page_id), K(page_key), K(fat_[page_id])); + } + return write_back; +} + +bool ObTmpWriteBufferPool::is_dirty( + const int64_t fd, + const uint32_t page_id, + const ObTmpFilePageUniqKey page_key) +{ + common::TCRWLock::RLockGuard guard(lock_); + bool dirty = false; + if (OB_LIKELY(is_valid_page_id_(page_id) + && fd == fat_[page_id].fd_ + && page_key.is_valid() + && page_key == fat_[page_id].page_key_)) { + dirty = ObPageEntry::State::DIRTY == fat_[page_id].state_; + } else { + int ret = OB_ERR_UNEXPECTED; + LOG_ERROR("wbp get unexpected page entry", KR(ret), K(fd), K(page_id), K(page_key), K(fat_[page_id])); + } + return dirty; +} + +// 允许 INITED/CACHED/DIRTY/WRITE_BACK 状态页面切换为 DIRTY 状态 +int ObTmpWriteBufferPool::notify_dirty( + const int64_t fd, + const uint32_t page_id, + const ObTmpFilePageUniqKey page_key) +{ + int ret = OB_SUCCESS; + common::TCRWLock::RLockGuard guard(lock_); + bool is_already_dirty = false; + bool is_write_back = false; + if (OB_UNLIKELY(!is_valid_page_id_(page_id) + || INVALID_FD == fd + || fd != fat_[page_id].fd_ + || !page_key.is_valid() + || page_key != fat_[page_id].page_key_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(fd), K(page_id), K(page_key), K(fat_[page_id])); + } else if (FALSE_IT(is_already_dirty = (ObPageEntry::State::DIRTY == fat_[page_id].state_))) { + } else if (FALSE_IT(is_write_back = (ObPageEntry::State::WRITE_BACK == fat_[page_id].state_))) { + } else if (OB_FAIL(fat_[page_id].switch_state(ObPageEntry::Ops::WRITE))) { + LOG_WARN("fail to switch state to DIRTY", KR(ret), K(fd), K(page_id), K(fat_[page_id])); + } else { + if (!is_already_dirty) { + ATOMIC_INC(&dirty_page_num_); + if (PageEntryType::DATA == page_key.type_) { + ATOMIC_INC(&dirty_data_page_cnt_); + } else if (PageEntryType::META == page_key.type_) { + ATOMIC_INC(&dirty_meta_page_cnt_); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("invalid page type", K(page_key)); + } + } + if (is_write_back) { + if (PageEntryType::DATA == page_key.type_) { + ATOMIC_DEC(&write_back_data_cnt_); + } else if (PageEntryType::META == page_key.type_) { + ATOMIC_DEC(&write_back_meta_cnt_); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("invalid page type", K(page_key)); + } + } + } + return ret; +} + +int ObTmpWriteBufferPool::notify_load( + const int64_t fd, + const uint32_t page_id, + const ObTmpFilePageUniqKey page_key) +{ + int ret = OB_SUCCESS; + common::TCRWLock::RLockGuard guard(lock_); + if (OB_UNLIKELY(!is_valid_page_id_(page_id) + || INVALID_FD == fd + || fd != fat_[page_id].fd_ + || !page_key.is_valid() + || page_key != fat_[page_id].page_key_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(fd), K(page_id), K(page_key), K(fat_[page_id])); + } else if (OB_FAIL(fat_[page_id].switch_state(ObPageEntry::Ops::LOAD))) { + LOG_WARN("fail to switch state from INITED to LOADING", KR(ret), K(fd), K(page_id), K(fat_[page_id])); + } + return ret; +} + +int ObTmpWriteBufferPool::notify_load_succ(const int64_t fd, const uint32_t page_id, + const ObTmpFilePageUniqKey page_key) +{ + int ret = OB_SUCCESS; + common::TCRWLock::RLockGuard guard(lock_); + if (OB_UNLIKELY(!is_valid_page_id_(page_id) + || INVALID_FD == fd + || fd != fat_[page_id].fd_ + || !page_key.is_valid() + || page_key != fat_[page_id].page_key_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(fd), K(page_id), K(page_key), K(fat_[page_id])); + } else if (OB_FAIL(fat_[page_id].switch_state(ObPageEntry::Ops::LOAD_SUCC))) { + LOG_WARN("fail to switch state from LOADING to CACHED", KR(ret), K(fd), K(page_id), K(fat_[page_id])); + } + return ret; +} + +int ObTmpWriteBufferPool::notify_load_fail( + const int64_t fd, + const uint32_t page_id, + const ObTmpFilePageUniqKey page_key) +{ + int ret = OB_SUCCESS; + common::TCRWLock::RLockGuard guard(lock_); + if (OB_UNLIKELY(!is_valid_page_id_(page_id) + || INVALID_FD == fd + || fd != fat_[page_id].fd_ + || !page_key.is_valid() + || page_key != fat_[page_id].page_key_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(fd), K(page_id), K(page_key), K(fat_[page_id])); + } else if (OB_FAIL(fat_[page_id].switch_state(ObPageEntry::Ops::LOAD_FAIL))) { + LOG_WARN("fail to switch state from LOADING to INITED", KR(ret), K(fd), K(page_id), K(fat_[page_id])); + } + return ret; +} + +int ObTmpWriteBufferPool::notify_write_back( + const int64_t fd, + const uint32_t page_id, + const ObTmpFilePageUniqKey page_key) +{ + int ret = OB_SUCCESS; + bool is_dirty = false; + common::TCRWLock::RLockGuard guard(lock_); + if (OB_UNLIKELY(!is_valid_page_id_(page_id) + || INVALID_FD == fd + || fd != fat_[page_id].fd_ + || !page_key.is_valid() + || page_key != fat_[page_id].page_key_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(fd), K(page_id), K(page_key), K(fat_[page_id])); + } else if (FALSE_IT(is_dirty = (ObPageEntry::State::DIRTY == fat_[page_id].state_))) { + } else if (OB_FAIL(fat_[page_id].switch_state(ObPageEntry::Ops::WRITE_BACK))) { + LOG_WARN("fail to switch state from DIRTY to WRITE_BACK", KR(ret), K(fd), K(page_id), K(fat_[page_id])); + } else if (is_dirty) { + ATOMIC_DEC(&dirty_page_num_); + if (PageEntryType::DATA == fat_[page_id].page_key_.type_) { + ATOMIC_DEC(&dirty_data_page_cnt_); + ATOMIC_INC(&write_back_data_cnt_); + LOG_DEBUG("notify data write back", K(fd), K(page_id), K(fat_[page_id])); + } else if (PageEntryType::META == fat_[page_id].page_key_.type_) { + ATOMIC_DEC(&dirty_meta_page_cnt_); + ATOMIC_INC(&write_back_meta_cnt_); + LOG_INFO("notify meta write back", K(fd), K(page_id), K(fat_[page_id])); + } + } + return ret; +} + +int ObTmpWriteBufferPool::notify_write_back_succ( + const int64_t fd, + const uint32_t page_id, + const ObTmpFilePageUniqKey page_key) +{ + int ret = OB_SUCCESS; + common::TCRWLock::RLockGuard guard(lock_); + if (OB_UNLIKELY(!is_valid_page_id_(page_id) + || INVALID_FD == fd + || fd != fat_[page_id].fd_ + || !page_key.is_valid() + || page_key != fat_[page_id].page_key_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(fd), K(page_id), K(page_key), K(fat_[page_id])); + } else if (OB_FAIL(fat_[page_id].switch_state(ObPageEntry::Ops::WRITE_BACK_SUCC))) { + LOG_WARN("fail to switch state from WRITE_BACK to CACHED", KR(ret), K(fd), K(page_id), K(fat_[page_id])); + } else { + if (PageEntryType::DATA == fat_[page_id].page_key_.type_) { + ATOMIC_DEC(&write_back_data_cnt_); + LOG_DEBUG("notify data write back succ", K(fd), K(page_id), K(fat_[page_id])); + } else if (PageEntryType::META == fat_[page_id].page_key_.type_) { + ATOMIC_DEC(&write_back_meta_cnt_); + LOG_INFO("notify meta write back succ", K(fd), K(page_id), K(fat_[page_id])); + } + } + return ret; +} + +int ObTmpWriteBufferPool::notify_write_back_fail(int64_t fd, uint32_t page_id, + const ObTmpFilePageUniqKey page_key) +{ + int ret = OB_SUCCESS; + common::TCRWLock::RLockGuard guard(lock_); + if (OB_UNLIKELY(!is_valid_page_id_(page_id) + || INVALID_FD == fd + || fd != fat_[page_id].fd_ + || !page_key.is_valid() + || page_key != fat_[page_id].page_key_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(fd), K(page_id), K(page_key), K(fat_[page_id])); + } else if (OB_FAIL(fat_[page_id].switch_state(ObPageEntry::Ops::WRITE_BACK_FAILED))) { + LOG_WARN("fail to switch state from WRITE_BACK to DIRTY", K(fd), K(page_id), K(fat_[page_id])); + } else { + ATOMIC_INC(&dirty_page_num_); + if (PageEntryType::DATA == fat_[page_id].page_key_.type_) { + ATOMIC_INC(&dirty_data_page_cnt_); + ATOMIC_DEC(&write_back_data_cnt_); + LOG_INFO("notify data write back fail", K(fd), K(page_id), K(fat_[page_id])); + } else if (PageEntryType::META == fat_[page_id].page_key_.type_) { + ATOMIC_INC(&dirty_meta_page_cnt_); + ATOMIC_DEC(&write_back_meta_cnt_); + LOG_INFO("notify meta write back fail", K(fd), K(page_id), K(fat_[page_id])); + } + } + return ret; +} + +// return write buffer pool maximum page number, which is determined by tenant memory and config +int64_t ObTmpWriteBufferPool::get_max_page_num() +{ + int64_t mem_limit = get_memory_limit(); + return mem_limit / ObTmpFileGlobal::PAGE_SIZE; +} + +// return dirty page percentage +int64_t ObTmpWriteBufferPool::get_dirty_page_percentage() +{ + int64_t max_page_num = get_max_page_num(); + int64_t dirty_page_num = ATOMIC_LOAD(&dirty_page_num_); + return max_page_num == 0 ? 0 : dirty_page_num * 100 / max_page_num; +} + +int64_t ObTmpWriteBufferPool::get_dirty_page_num() +{ + return ATOMIC_LOAD(&dirty_page_num_); +} + +int64_t ObTmpWriteBufferPool::get_dirty_meta_page_num() +{ + return ATOMIC_LOAD(&dirty_meta_page_cnt_); +} + +int64_t ObTmpWriteBufferPool::get_dirty_data_page_num() +{ + return ATOMIC_LOAD(&dirty_data_page_cnt_); +} + +int64_t ObTmpWriteBufferPool::get_data_page_num() +{ + return ATOMIC_LOAD(&data_page_cnt_); +} + +int64_t ObTmpWriteBufferPool::get_max_data_page_num() +{ + return get_max_page_num() * MAX_DATA_PAGE_USAGE_RATIO; +} + +int64_t ObTmpWriteBufferPool::get_meta_page_num() +{ + return ATOMIC_LOAD(&meta_page_cnt_); +} + +bool ObTmpWriteBufferPool::has_free_page_(PageEntryType type) +{ + int ret = OB_SUCCESS; + bool b_ret = true; + if (PageEntryType::DATA == type) { + b_ret = get_data_page_num() < get_max_data_page_num(); + } else if (PageEntryType::META == type) { + b_ret = true; // no limit for meta page + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected page type", KR(ret), K(type)); + } + return b_ret; +} + +void ObTmpWriteBufferPool::print_statistics() +{ + int64_t dirty_page_percentage = get_dirty_page_percentage(); + int64_t max_page_num = get_max_page_num(); + int64_t meta_page_num = get_meta_page_num(); + int64_t data_page_num = get_data_page_num(); + int64_t dirty_page_num = get_dirty_page_num(); + int64_t dirty_meta_page_num = get_dirty_meta_page_num(); + int64_t dirty_data_page_num = get_dirty_data_page_num(); + int64_t write_back_data_num = ATOMIC_LOAD(&write_back_data_cnt_); + int64_t write_back_meta_num = ATOMIC_LOAD(&write_back_meta_cnt_); + int64_t data_page_watermark = data_page_num * 100 / max(max_page_num, 1); + int64_t meta_page_watermark = meta_page_num * 100 / max(max_page_num, 1); + int64_t total_write_back_num = write_back_data_num + write_back_meta_num; + LOG_INFO("tmp file write buffer pool statistics", + K(dirty_page_percentage), K(max_page_num), K(dirty_page_num), K(total_write_back_num), + K(meta_page_num), K(dirty_meta_page_num), K(write_back_meta_num), + K(data_page_num), K(dirty_data_page_num), K(write_back_data_num), + K(data_page_watermark), K(meta_page_watermark)); +} + +} // end namespace tmp_file +} // end namespace oceanbase diff --git a/src/storage/tmp_file/ob_tmp_file_write_buffer_pool.h b/src/storage/tmp_file/ob_tmp_file_write_buffer_pool.h new file mode 100644 index 000000000..58a741c10 --- /dev/null +++ b/src/storage/tmp_file/ob_tmp_file_write_buffer_pool.h @@ -0,0 +1,305 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_STORAGE_BLOCKSSTABLE_OB_TMP_WRITE_BUFFER_POOL_H_ +#define OCEANBASE_STORAGE_BLOCKSSTABLE_OB_TMP_WRITE_BUFFER_POOL_H_ + +#include "lib/lock/ob_spin_rwlock.h" +#include "lib/container/ob_array.h" +#include "lib/queue/ob_link.h" +#include "lib/queue/ob_link_queue.h" +#include "lib/allocator/ob_fifo_allocator.h" +#include "share/io/ob_io_define.h" +#include "storage/blocksstable/ob_macro_block_id.h" +#include "storage/tmp_file/ob_tmp_file_global.h" + +namespace oceanbase +{ +namespace tmp_file +{ + +class ObTmpWriteBufferPool; +enum class PageEntryType +{ + INVALID = -1, + DATA = 0, + META = 1 +}; + +struct ObTmpFilePageUniqKey +{ +public: + ObTmpFilePageUniqKey() : type_(PageEntryType::INVALID), virtual_page_id_(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID) {} + explicit ObTmpFilePageUniqKey(const int64_t virtual_page_id) : + type_(PageEntryType::DATA), virtual_page_id_(virtual_page_id) {} + explicit ObTmpFilePageUniqKey(const int64_t tree_level, const int64_t level_page_index) : + type_(PageEntryType::META), tree_level_(tree_level), + level_page_index_ (level_page_index) {} + + void reset() + { + virtual_page_id_ = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + type_ = PageEntryType::INVALID; + } + + OB_INLINE bool is_valid() const + { + return type_ != PageEntryType::INVALID && + type_ == PageEntryType::META ? + 0 <= tree_level_ && 0 <= level_page_index_ : + ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID != virtual_page_id_; + } + bool operator==(const ObTmpFilePageUniqKey other) const + { + return type_ == other.type_ && virtual_page_id_ == other.virtual_page_id_; + } + bool operator!=(const ObTmpFilePageUniqKey other) const + { + return type_ != other.type_ || virtual_page_id_ != other.virtual_page_id_; + } + +public: + PageEntryType type_; + union { + int64_t virtual_page_id_; // page_offset / page_size + struct { //The specific value for the tree pages + int64_t tree_level_:16; + int64_t level_page_index_:48; + }; + }; + TO_STRING_KV(K(type_), K(virtual_page_id_), K(tree_level_), K(level_page_index_)); +}; + +struct ObPageEntry final +{ + friend class ObTmpWriteBufferPool; +public: + ObPageEntry() + : buf_(nullptr), + fd_(-1), + state_(State::INVALID), + next_page_id_(ObTmpFileGlobal::INVALID_PAGE_ID), + page_key_() {} + ObPageEntry(const int64_t fd, const uint32_t next_page_id, char *buf) + : buf_(buf), + fd_(fd), + state_(State::INVALID), + next_page_id_(next_page_id), + page_key_() {} + + int switch_state(const int64_t op); + + TO_STRING_KV(K(fd_), K(page_key_), K(next_page_id_), K(state_), KP(buf_)); +public: + struct State + { + public: + static const int32_t N = -1; // illegal state + static const int32_t INVALID = 0; // page entry is INVALID after page is freed or before allocating + static const int32_t INITED = 1; // page entry is INITED after allocating + static const int32_t LOADING = 2; // page entry is LOADING after sending async io to read page from disk + static const int32_t CACHED = 3; // page entry is CACHED when page is clean + static const int32_t DIRTY = 4; // page entry is DIRTY after page is written + static const int32_t WRITE_BACK = 5; // page entry is WRITE_BACK when sending async io to write page to disk + static const int32_t MAX = 6; + public: + static bool is_valid(const int32_t state){ + return state > N && state < MAX; + } + }; + + struct Ops + { + public: + static const int64_t INVALID = -1; + static const int64_t ALLOC = 0; + static const int64_t LOAD = 1; + static const int64_t LOAD_FAIL = 2; + static const int64_t LOAD_SUCC = 3; + static const int64_t DELETE = 4; + static const int64_t WRITE = 5; + static const int64_t WRITE_BACK = 6; + static const int64_t WRITE_BACK_FAILED = 7; + static const int64_t WRITE_BACK_SUCC = 8; + static const int64_t MAX = 9; + public: + static bool is_valid(const int64_t op){ + return op > INVALID && op < MAX; + } + }; +public: + char *buf_; + int64_t fd_; + int32_t state_; + uint32_t next_page_id_; + ObTmpFilePageUniqKey page_key_; +}; + +// preallocate a set of pages for the tmp file to write data. the pages are divided into data and meta types. +// data type pages can use up to 90% of the entire buffer pool space, while meta type pages have no upper limit. +// we build ObTmpWriteBufferPool upon this assumption: caller ensure only 1 writer operating a page entry at a time, +// and write operation must be exclusive with other r/w operations, therefore we have no need to limit the concurrency +// for every single page entry here. +class ObTmpWriteBufferPool final +{ +public: + // block size: 2MB - 24KB (header), use block size smaller than 2MB to avoid redundant AObject header + static const int64_t WBP_BLOCK_SIZE = 2 * 1024 * 1024 - 24 * 1024; + static const int64_t BLOCK_PAGE_NUMS = WBP_BLOCK_SIZE / ObTmpFileGlobal::PAGE_SIZE; // 253 pages per block (24KB for header) + static const int64_t INITIAL_POOL_SIZE = WBP_BLOCK_SIZE; + static const int64_t INITIAL_PAGE_NUMS = INITIAL_POOL_SIZE / ObTmpFileGlobal::PAGE_SIZE; + +public: + ObTmpWriteBufferPool(); + ~ObTmpWriteBufferPool(); + int init(); + void destroy(); + +public: + // 1. according to the type_ of page_key, allocate a meta page or data page and set its state to INITED + // 2. return OB_ALLOCATE_TMP_FILE_PAGE_FAILED if data page number exceeds limits + // 3. always allow to alloc a meta page + int alloc_page(const int64_t fd, + const ObTmpFilePageUniqKey page_key, + uint32_t &new_page_id, + char *&buf); + + // read the content of a page and keep its original page state + int read_page(const int64_t fd, const uint32_t page_id, const ObTmpFilePageUniqKey page_key, + char *&buf, uint32_t &next_page_id); + + // set prev_page_id.next_page_id to point to 'page_id' without changing the page status + int link_page(const int64_t fd, const uint32_t page_id, const uint32_t prev_page_id, + const ObTmpFilePageUniqKey prev_page_key); + + // free given pages with INITED/CACHED/DIRTY state, + // return OB_STATE_NOT_MATCH if try to delete pages with other states + int free_page(const int64_t fd, const uint32_t page_id, + const ObTmpFilePageUniqKey page_key, uint32_t &next_page_id); + + /** + * truncate a page from beginning to 'truncate_size' [0, truncate_size - 1], set data to 0; + * truncate_page will not change page state. if page is already flushed to disk, truncate_page + * only truncate data in buffer pool and will not mark page as dirty + */ + int truncate_page(const int64_t fd, const uint32_t page_id, + const ObTmpFilePageUniqKey page_key, + const int64_t truncate_size); +public: + /** + * given page_id, output next_page_id if existed; + * return OB_ITER_END for no more pages, others for error + */ + int get_next_page_id(const int64_t fd, + const uint32_t page_id, + const ObTmpFilePageUniqKey page_key, + uint32_t &next_page_id); + + /** + * iter from the given page_id to end of the page list of a tmp file, + * to find a page which has a same virtual_page_id + * @param[in] virtual_page_id: the page we want to read(it equal to page_offset / page_size) + * @param[in] begin_page_id: the first cached page id of the tmp file + * @param[out] page_id: the target page + */ + int get_page_id_by_virtual_id(const int64_t fd, + const int64_t virtual_page_id, + const uint32_t begin_page_id, + uint32_t &page_id); + + /** + * get page_virtual_id of the given page_id, + * return OB_SEARCH_NOT_FOUND for page is INVALID, + */ + int get_page_virtual_id(const int64_t fd, const uint32_t page_id, int64_t &virtual_page_id); + +public: + int64_t get_swap_size(); + int64_t get_memory_limit(); + bool is_exist(const int64_t fd, const uint32_t page_id, const ObTmpFilePageUniqKey page_key); + bool is_inited(const int64_t fd, const uint32_t page_id, const ObTmpFilePageUniqKey page_key); + bool is_loading(const int64_t fd, const uint32_t page_id, const ObTmpFilePageUniqKey page_key); + bool is_cached(const int64_t fd, const uint32_t page_id, const ObTmpFilePageUniqKey page_key); + bool is_dirty(const int64_t fd, const uint32_t page_id, const ObTmpFilePageUniqKey page_key); + bool is_write_back(const int64_t fd, const uint32_t page_id, const ObTmpFilePageUniqKey page_key); + int notify_dirty(const int64_t fd, const uint32_t page_id, const ObTmpFilePageUniqKey page_key); + int notify_load(const int64_t fd, const uint32_t page_id, const ObTmpFilePageUniqKey page_key); + int notify_load_succ(const int64_t fd, const uint32_t page_id, const ObTmpFilePageUniqKey page_key); + int notify_load_fail(const int64_t fd, const uint32_t page_id, const ObTmpFilePageUniqKey page_key); + int notify_write_back(const int64_t fd, const uint32_t page_id, const ObTmpFilePageUniqKey page_key); + int notify_write_back_succ(const int64_t fd, const uint32_t page_id, const ObTmpFilePageUniqKey page_key); + int notify_write_back_fail(int64_t fd, uint32_t page_id, const ObTmpFilePageUniqKey page_key); + int64_t get_max_page_num(); + int64_t get_data_page_num(); + int64_t get_dirty_page_num(); + int64_t get_dirty_meta_page_num(); + int64_t get_dirty_data_page_num(); + int64_t get_dirty_page_percentage(); + int64_t get_max_data_page_num(); + int64_t get_meta_page_num(); + void print_statistics(); +private: + static double MAX_DATA_PAGE_USAGE_RATIO; // control data pages ratio, can be preempted by meta pages + // only for unittest + OB_INLINE void set_max_data_page_usage_ratio_(const double ratio) + { + MAX_DATA_PAGE_USAGE_RATIO = ratio; + } + int read_page_(const int64_t fd, + const uint32_t page_id, + const ObTmpFilePageUniqKey page_key, + char *&buf, + uint32_t &next_page_id); + int alloc_page_(const int64_t fd, + const ObTmpFilePageUniqKey page_key, + uint32_t &new_page_id, + char *&buf); + int inner_alloc_page_(const int64_t fd, + const ObTmpFilePageUniqKey page_key, + uint32_t &new_page_id, + char *&buf); + // check if the specified PageEntryType has available space + bool has_free_page_(PageEntryType type); + // ATTENTION! access fat_, needs to be protected by r-lock + OB_INLINE bool is_valid_page_id_(const uint32_t page_id) const + { + return page_id != ObTmpFileGlobal::INVALID_PAGE_ID && page_id >= 0 && + page_id < fat_.count() && OB_NOT_NULL(fat_[page_id].buf_); + } + int expand_(); + int reduce_(); + DISALLOW_COPY_AND_ASSIGN(ObTmpWriteBufferPool); +private: + common::ObArray fat_; // file allocation table + common::TCRWLock lock_; // holds w-lock when expanding and shrinking fat_, holds r-lock when reading fat_ + common::ObFIFOAllocator allocator_; + bool is_inited_; + int64_t capacity_; // in bytes + int64_t dirty_page_num_; + int64_t used_page_num_; + uint32_t first_free_page_id_; + int64_t wbp_memory_limit_; // in bytes + int64_t default_wbp_memory_limit_; // if this var is valid, the wbp memory limit will always be it. + // currently, this var is only modified in ut. + int64_t last_access_tenant_config_ts_; + int64_t meta_page_cnt_; + int64_t data_page_cnt_; + int64_t dirty_meta_page_cnt_; + int64_t dirty_data_page_cnt_; + int64_t write_back_data_cnt_; + int64_t write_back_meta_cnt_; +}; + +} // end namespace tmp_file +} // end namespace oceanbase + +#endif diff --git a/unittest/sql/engine/basic/test_chunk_datum_store.cpp b/unittest/sql/engine/basic/test_chunk_datum_store.cpp index 4a686fcf5..b4e06b760 100644 --- a/unittest/sql/engine/basic/test_chunk_datum_store.cpp +++ b/unittest/sql/engine/basic/test_chunk_datum_store.cpp @@ -13,10 +13,10 @@ #define USING_LOG_PREFIX SQL #include +#include "mtlenv/mock_tenant_module_env.h" #include "lib/alloc/ob_malloc_allocator.h" #include "lib/allocator/ob_malloc.h" #include "storage/blocksstable/ob_data_file_prepare.h" -#include "storage/blocksstable/ob_tmp_file.h" #include "sql/engine/basic/ob_chunk_datum_store.h" #include "sql/engine/basic/ob_ra_row_store.h" #include "common/row/ob_row_store.h" @@ -163,8 +163,8 @@ public: int ret = OB_SUCCESS; ASSERT_EQ(OB_SUCCESS, init_tenant_mgr()); blocksstable::TestDataFilePrepare::SetUp(); - ret = blocksstable::ObTmpFileManager::get_instance().init(); - ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpBlockCache::get_instance().init("tmp_block_cache", 1)); + ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpPageCache::get_instance().init("tmp_page_cache", 1)); if (!is_server_tenant(tenant_id_)) { static ObTenantBase tenant_ctx(tenant_id_); ObTenantEnv::set_tenant(&tenant_ctx); @@ -173,15 +173,23 @@ public: EXPECT_EQ(OB_SUCCESS, ObTenantIOManager::mtl_init(io_service)); EXPECT_EQ(OB_SUCCESS, io_service->start()); tenant_ctx.set(io_service); + + tmp_file::ObTenantTmpFileManager *tf_mgr = nullptr; + EXPECT_EQ(OB_SUCCESS, mtl_new_default(tf_mgr)); + EXPECT_EQ(OB_SUCCESS, tmp_file::ObTenantTmpFileManager::mtl_init(tf_mgr)); + tf_mgr->page_cache_controller_.write_buffer_pool_.default_wbp_memory_limit_ = 40*1024*1024; + EXPECT_EQ(OB_SUCCESS, tf_mgr->start()); + tenant_ctx.set(tf_mgr); + ObTenantEnv::set_tenant(&tenant_ctx); } cell_cnt_ = COLS; init_exprs(); - plan_.set_batch_size(batch_size_); - plan_ctx_.set_phy_plan(&plan_); - eval_ctx_.set_max_batch_size(batch_size_); + plan_.set_batch_size(batch_size_); + plan_ctx_.set_phy_plan(&plan_); + eval_ctx_.set_max_batch_size(batch_size_); exec_ctx_.set_physical_plan_ctx(&plan_ctx_); skip_ = (ObBitVector *)alloc_.alloc(ObBitVector::memory_size(batch_size_)); @@ -207,7 +215,8 @@ public: rs_.reset(); rs_.~ObChunkDatumStore(); - blocksstable::ObTmpFileManager::get_instance().destroy(); + tmp_file::ObTmpBlockCache::get_instance().destroy(); + tmp_file::ObTmpPageCache::get_instance().destroy(); blocksstable::TestDataFilePrepare::TearDown(); LOG_INFO("TearDown finished", K_(rs)); } diff --git a/unittest/sql/engine/basic/test_chunk_row_store.cpp b/unittest/sql/engine/basic/test_chunk_row_store.cpp index f3f82aabf..f63dce500 100644 --- a/unittest/sql/engine/basic/test_chunk_row_store.cpp +++ b/unittest/sql/engine/basic/test_chunk_row_store.cpp @@ -13,10 +13,10 @@ #define USING_LOG_PREFIX SQL #include +#include "mtlenv/mock_tenant_module_env.h" #include "lib/alloc/ob_malloc_allocator.h" #include "lib/allocator/ob_malloc.h" #include "storage/blocksstable/ob_data_file_prepare.h" -#include "storage/blocksstable/ob_tmp_file.h" #include "sql/engine/basic/ob_chunk_row_store.h" #include "sql/engine/basic/ob_ra_row_store.h" #include "common/row/ob_row_store.h" @@ -24,7 +24,6 @@ #include "sql/ob_sql_init.h" #include "share/ob_simple_mem_limit_getter.h" - namespace oceanbase { namespace sql @@ -69,16 +68,25 @@ public: int ret = OB_SUCCESS; ASSERT_EQ(OB_SUCCESS, init_tenant_mgr()); blocksstable::TestDataFilePrepare::SetUp(); - ret = blocksstable::ObTmpFileManager::get_instance().init(); - ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpBlockCache::get_instance().init("tmp_block_cache", 1)); + ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpPageCache::get_instance().init("tmp_page_cache", 1)); if (!is_server_tenant(tenant_id_)) { static ObTenantBase tenant_ctx(tenant_id_); ObTenantEnv::set_tenant(&tenant_ctx); + ObTenantIOManager *io_service = nullptr; EXPECT_EQ(OB_SUCCESS, ObTenantIOManager::mtl_new(io_service)); EXPECT_EQ(OB_SUCCESS, ObTenantIOManager::mtl_init(io_service)); EXPECT_EQ(OB_SUCCESS, io_service->start()); tenant_ctx.set(io_service); + + tmp_file::ObTenantTmpFileManager *tf_mgr = nullptr; + EXPECT_EQ(OB_SUCCESS, mtl_new_default(tf_mgr)); + EXPECT_EQ(OB_SUCCESS, tmp_file::ObTenantTmpFileManager::mtl_init(tf_mgr)); + tf_mgr->page_cache_controller_.write_buffer_pool_.default_wbp_memory_limit_ = 40*1024*1024; + EXPECT_EQ(OB_SUCCESS, tf_mgr->start()); + tenant_ctx.set(tf_mgr); + ObTenantEnv::set_tenant(&tenant_ctx); } @@ -107,7 +115,8 @@ public: rs_.reset(); rs_.~ObChunkRowStore(); - blocksstable::ObTmpFileManager::get_instance().destroy(); + tmp_file::ObTmpBlockCache::get_instance().destroy(); + tmp_file::ObTmpPageCache::get_instance().destroy(); blocksstable::TestDataFilePrepare::TearDown(); LOG_INFO("TearDown finished", K_(rs)); } diff --git a/unittest/sql/engine/test_op_engine.cpp b/unittest/sql/engine/test_op_engine.cpp index 5edb34ef6..692ac6ce9 100644 --- a/unittest/sql/engine/test_op_engine.cpp +++ b/unittest/sql/engine/test_op_engine.cpp @@ -79,7 +79,6 @@ void TestOpEngine::destory() ObIOManager::get_instance().destroy(); ObKVGlobalCache::get_instance().destroy(); ObClusterVersion::get_instance().destroy(); - oceanbase::blocksstable::ObTmpFileManager::get_instance().destroy(); // THE_IO_DEVICE->destroy(); } @@ -179,7 +178,6 @@ int TestOpEngine::prepare_io(const string & test_data_name_suffix) LOG_WARN("fail to init OB_STORE_CACHE, ", K(ret)); } else { } - FILE_MANAGER_INSTANCE_V2.init(); return ret; } diff --git a/unittest/storage/backup/test_backup_index_merger.cpp b/unittest/storage/backup/test_backup_index_merger.cpp index 9a9a495af..c35292c1b 100644 --- a/unittest/storage/backup/test_backup_index_merger.cpp +++ b/unittest/storage/backup/test_backup_index_merger.cpp @@ -28,6 +28,7 @@ #include "test_backup.h" #include "test_backup_include.h" #include "storage/blocksstable/ob_logic_macro_id.h" +#include "mtlenv/mock_tenant_module_env.h" using namespace testing; using namespace oceanbase; @@ -317,12 +318,11 @@ void TestBackupIndexMerger::SetUp() } // set observer memory limit CHUNK_MGR.set_limit(8L * 1024L * 1024L * 1024L); - ret = ObTmpFileManager::get_instance().init(); - if (OB_INIT_TWICE == ret) { - ret = OB_SUCCESS; - } else { - ASSERT_EQ(OB_SUCCESS, ret); - } + + ASSERT_EQ(OB_SUCCESS, common::ObClockGenerator::init()); + ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpBlockCache::get_instance().init("tmp_block_cache", 1)); + ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpPageCache::get_instance().init("tmp_page_cache", 1)); + static ObTenantBase tenant_ctx(OB_SYS_TENANT_ID); ObTenantEnv::set_tenant(&tenant_ctx); ObTenantIOManager *io_service = nullptr; @@ -330,14 +330,24 @@ void TestBackupIndexMerger::SetUp() EXPECT_EQ(OB_SUCCESS, ObTenantIOManager::mtl_init(io_service)); EXPECT_EQ(OB_SUCCESS, io_service->start()); tenant_ctx.set(io_service); + + tmp_file::ObTenantTmpFileManager *tf_mgr = nullptr; + EXPECT_EQ(OB_SUCCESS, mtl_new_default(tf_mgr)); + EXPECT_EQ(OB_SUCCESS, tmp_file::ObTenantTmpFileManager::mtl_init(tf_mgr)); + tf_mgr->page_cache_controller_.write_buffer_pool_.default_wbp_memory_limit_ = 40*1024*1024; + EXPECT_EQ(OB_SUCCESS, tf_mgr->start()); + tenant_ctx.set(tf_mgr); + ObTenantEnv::set_tenant(&tenant_ctx); inner_init_(); } void TestBackupIndexMerger::TearDown() { - ObTmpFileManager::get_instance().destroy(); + tmp_file::ObTmpBlockCache::get_instance().destroy(); + tmp_file::ObTmpPageCache::get_instance().destroy(); ObKVGlobalCache::get_instance().destroy(); + common::ObClockGenerator::destroy(); } void TestBackupIndexMerger::inner_init_() diff --git a/unittest/storage/backup/test_backup_iterator.cpp b/unittest/storage/backup/test_backup_iterator.cpp index 2138134c6..e46bb75fb 100644 --- a/unittest/storage/backup/test_backup_iterator.cpp +++ b/unittest/storage/backup/test_backup_iterator.cpp @@ -24,6 +24,7 @@ #include "test_backup_include.h" #include "share/backup/ob_backup_io_adapter.h" #include "lib/string/ob_string.h" +#include "mtlenv/mock_tenant_module_env.h" using namespace oceanbase; using namespace oceanbase::common; @@ -111,7 +112,11 @@ void TestBackupIndexIterator::SetUp() } // set observer memory limit CHUNK_MGR.set_limit(8L * 1024L * 1024L * 1024L); - ret = ObTmpFileManager::get_instance().init(); + + ASSERT_EQ(OB_SUCCESS, common::ObClockGenerator::init()); + ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpBlockCache::get_instance().init("tmp_block_cache", 1)); + ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpPageCache::get_instance().init("tmp_page_cache", 1)); + if (OB_INIT_TWICE == ret) { ret = OB_SUCCESS; } else { @@ -124,6 +129,14 @@ void TestBackupIndexIterator::SetUp() EXPECT_EQ(OB_SUCCESS, ObTenantIOManager::mtl_init(io_service)); EXPECT_EQ(OB_SUCCESS, io_service->start()); tenant_ctx.set(io_service); + + tmp_file::ObTenantTmpFileManager *tf_mgr = nullptr; + EXPECT_EQ(OB_SUCCESS, mtl_new_default(tf_mgr)); + EXPECT_EQ(OB_SUCCESS, tmp_file::ObTenantTmpFileManager::mtl_init(tf_mgr)); + tf_mgr->page_cache_controller_.write_buffer_pool_.default_wbp_memory_limit_ = 40*1024*1024; + EXPECT_EQ(OB_SUCCESS, tf_mgr->start()); + tenant_ctx.set(tf_mgr); + ObTenantEnv::set_tenant(&tenant_ctx); inner_init_(); ASSERT_EQ(OB_SUCCESS, bandwidth_throttle_.init(1024 * 1024 * 60)); @@ -131,8 +144,10 @@ void TestBackupIndexIterator::SetUp() void TestBackupIndexIterator::TearDown() { - ObTmpFileManager::get_instance().destroy(); + tmp_file::ObTmpBlockCache::get_instance().destroy(); + tmp_file::ObTmpPageCache::get_instance().destroy(); ObKVGlobalCache::get_instance().destroy(); + common::ObClockGenerator::destroy(); } void TestBackupIndexIterator::inner_init_() diff --git a/unittest/storage/backup/test_backup_macro_block_index_merger.cpp b/unittest/storage/backup/test_backup_macro_block_index_merger.cpp index 15eee1306..dd338f9fe 100644 --- a/unittest/storage/backup/test_backup_macro_block_index_merger.cpp +++ b/unittest/storage/backup/test_backup_macro_block_index_merger.cpp @@ -15,6 +15,7 @@ #define private public #define protected public +#include "observer/omt/ob_tenant_mtl_helper.h" #include "storage/backup/ob_backup_data_struct.h" #include "storage/backup/ob_backup_index_merger.h" #include "storage/backup/ob_backup_index_store.h" @@ -190,12 +191,8 @@ void TestBackupMacroIndexMerger::SetUp() } // set observer memory limit CHUNK_MGR.set_limit(8L * 1024L * 1024L * 1024L); - ret = ObTmpFileManager::get_instance().init(); - if (OB_INIT_TWICE == ret) { - ret = OB_SUCCESS; - } else { - ASSERT_EQ(OB_SUCCESS, ret); - } + ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpPageCache::get_instance().init("tmp_page_cache", 1)); + static ObTenantBase tenant_ctx(OB_SYS_TENANT_ID); ObTenantEnv::set_tenant(&tenant_ctx); ObTenantIOManager *io_service = nullptr; @@ -203,14 +200,23 @@ void TestBackupMacroIndexMerger::SetUp() EXPECT_EQ(OB_SUCCESS, ObTenantIOManager::mtl_init(io_service)); EXPECT_EQ(OB_SUCCESS, io_service->start()); tenant_ctx.set(io_service); + + tmp_file::ObTenantTmpFileManager *tf_mgr = nullptr; + EXPECT_EQ(OB_SUCCESS, mtl_new_default(tf_mgr)); + EXPECT_EQ(OB_SUCCESS, tmp_file::ObTenantTmpFileManager::mtl_init(tf_mgr)); + tf_mgr->page_cache_controller_.write_buffer_pool_.default_wbp_memory_limit_ = 40*1024*1024; + EXPECT_EQ(OB_SUCCESS, tf_mgr->start()); + tenant_ctx.set(tf_mgr); + ObTenantEnv::set_tenant(&tenant_ctx); inner_init_(); } void TestBackupMacroIndexMerger::TearDown() { - ObTmpFileManager::get_instance().destroy(); + tmp_file::ObTmpPageCache::get_instance().destroy(); ObKVGlobalCache::get_instance().destroy(); + common::ObClockGenerator::destroy(); } void TestBackupMacroIndexMerger::clean_env_() diff --git a/unittest/storage/backup/test_backup_tmp_file.cpp b/unittest/storage/backup/test_backup_tmp_file.cpp index ed0a83c26..60be7510a 100644 --- a/unittest/storage/backup/test_backup_tmp_file.cpp +++ b/unittest/storage/backup/test_backup_tmp_file.cpp @@ -20,6 +20,7 @@ #include "storage/backup/ob_backup_data_struct.h" #include "storage/blocksstable/ob_data_file_prepare.h" #include "test_backup.h" +#include "mtlenv/mock_tenant_module_env.h" using namespace oceanbase; using namespace oceanbase::common; @@ -66,8 +67,10 @@ void TestBackupTmpFile::SetUp() } // set observer memory limit CHUNK_MGR.set_limit(8L * 1024L * 1024L * 1024L); - ret = ObTmpFileManager::get_instance().init(); - ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(OB_SUCCESS, common::ObClockGenerator::init()); + ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpBlockCache::get_instance().init("tmp_block_cache", 1)); + ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpPageCache::get_instance().init("tmp_page_cache", 1)); + static ObTenantBase tenant_ctx(OB_SYS_TENANT_ID); ObTenantEnv::set_tenant(&tenant_ctx); ObTenantIOManager *io_service = nullptr; @@ -75,14 +78,24 @@ void TestBackupTmpFile::SetUp() EXPECT_EQ(OB_SUCCESS, ObTenantIOManager::mtl_init(io_service)); EXPECT_EQ(OB_SUCCESS, io_service->start()); tenant_ctx.set(io_service); + + tmp_file::ObTenantTmpFileManager *tf_mgr = nullptr; + EXPECT_EQ(OB_SUCCESS, mtl_new_default(tf_mgr)); + EXPECT_EQ(OB_SUCCESS, tmp_file::ObTenantTmpFileManager::mtl_init(tf_mgr)); + tf_mgr->page_cache_controller_.write_buffer_pool_.default_wbp_memory_limit_ = 40*1024*1024; + EXPECT_EQ(OB_SUCCESS, tf_mgr->start()); + tenant_ctx.set(tf_mgr); + ObTenantEnv::set_tenant(&tenant_ctx); } void TestBackupTmpFile::TearDown() { - ObTmpFileManager::get_instance().destroy(); + tmp_file::ObTmpBlockCache::get_instance().destroy(); + tmp_file::ObTmpPageCache::get_instance().destroy(); ObKVGlobalCache::get_instance().destroy(); TestDataFilePrepare::TearDown(); + common::ObClockGenerator::destroy(); } static void make_meta_index(backup::ObBackupMetaIndex &meta_index) diff --git a/unittest/storage/backup/test_backup_utils.cpp b/unittest/storage/backup/test_backup_utils.cpp index 55c54fe51..a3fc638a0 100644 --- a/unittest/storage/backup/test_backup_utils.cpp +++ b/unittest/storage/backup/test_backup_utils.cpp @@ -23,6 +23,7 @@ #include "storage/blocksstable/ob_logic_macro_id.h" #include "storage/ob_parallel_external_sort.h" #include "storage/blocksstable/ob_data_file_prepare.h" +#include "mtlenv/mock_tenant_module_env.h" #include "storage/backup/ob_backup_ctx.h" using namespace oceanbase; @@ -261,7 +262,10 @@ void TestBackupExternalSort::SetUp() { TestDataFilePrepare::SetUp(); EXPECT_EQ(OB_SUCCESS, init_tenant_mgr()); - EXPECT_EQ(OB_SUCCESS, ObTmpFileManager::get_instance().init()); + ASSERT_EQ(OB_SUCCESS, common::ObClockGenerator::init()); + ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpBlockCache::get_instance().init("tmp_block_cache", 1)); + ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpPageCache::get_instance().init("tmp_page_cache", 1)); + static ObTenantBase tenant_ctx(OB_SYS_TENANT_ID); ObTenantEnv::set_tenant(&tenant_ctx); ObTenantIOManager *io_service = nullptr; @@ -269,14 +273,26 @@ void TestBackupExternalSort::SetUp() EXPECT_EQ(OB_SUCCESS, ObTenantIOManager::mtl_init(io_service)); EXPECT_EQ(OB_SUCCESS, io_service->start()); tenant_ctx.set(io_service); + + tmp_file::ObTenantTmpFileManager *tf_mgr = nullptr; + EXPECT_EQ(OB_SUCCESS, mtl_new_default(tf_mgr)); + EXPECT_EQ(OB_SUCCESS, tmp_file::ObTenantTmpFileManager::mtl_init(tf_mgr)); + tf_mgr->page_cache_controller_.write_buffer_pool_.default_wbp_memory_limit_ = 40*1024*1024; + EXPECT_EQ(OB_SUCCESS, tf_mgr->start()); + tenant_ctx.set(tf_mgr); + ObTenantEnv::set_tenant(&tenant_ctx); } void TestBackupExternalSort::TearDown() { allocator_.reuse(); - ObTmpFileManager::get_instance().destroy(); + + tmp_file::ObTmpBlockCache::get_instance().destroy(); + tmp_file::ObTmpPageCache::get_instance().destroy(); TestDataFilePrepare::TearDown(); + common::ObClockGenerator::destroy(); + destroy_tenant_mgr(); } diff --git a/unittest/storage/blocksstable/CMakeLists.txt b/unittest/storage/blocksstable/CMakeLists.txt index 1f900df20..1496d57cf 100644 --- a/unittest/storage/blocksstable/CMakeLists.txt +++ b/unittest/storage/blocksstable/CMakeLists.txt @@ -4,7 +4,6 @@ storage_unittest(test_block_sstable_struct) storage_unittest(test_data_buffer) storage_unittest(test_index_block_aggregator) #storage_unittest(test_storage_cache_suite) -storage_unittest(test_tmp_file) #storage_unittest(test_sstable_sec_meta_iterator) storage_unittest(test_sstable_meta) #storage_unittest(test_inspect_bad_block) diff --git a/unittest/storage/blocksstable/test_block_manager.cpp b/unittest/storage/blocksstable/test_block_manager.cpp index 6654a136d..1b4745fe4 100644 --- a/unittest/storage/blocksstable/test_block_manager.cpp +++ b/unittest/storage/blocksstable/test_block_manager.cpp @@ -21,10 +21,10 @@ #define private public #include "storage/blocksstable/ob_data_file_prepare.h" -#include "storage/blocksstable/ob_tmp_file.h" #include "share/ob_simple_mem_limit_getter.h" #include "observer/omt/ob_worker_processor.h" #include "observer/ob_srv_network_frame.h" +#include "mtlenv/mock_tenant_module_env.h" namespace oceanbase { @@ -138,9 +138,18 @@ TEST_F(TestBlockManager, test_mark_and_sweep) } else { ASSERT_EQ(common::OB_SUCCESS, ret); } - ret = FILE_MANAGER_INSTANCE_V2.init(); - ASSERT_EQ(common::OB_SUCCESS, ret); - ASSERT_EQ(common::OB_SUCCESS, ret); + + ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpBlockCache::get_instance().init("tmp_block_cache", 1)); + ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpPageCache::get_instance().init("tmp_page_cache", 1)); + static ObTenantBase tenant_ctx(OB_SYS_TENANT_ID); + ObTenantEnv::set_tenant(&tenant_ctx); + tmp_file::ObTenantTmpFileManager *tf_mgr = nullptr; + EXPECT_EQ(OB_SUCCESS, mtl_new_default(tf_mgr)); + EXPECT_EQ(OB_SUCCESS, tmp_file::ObTenantTmpFileManager::mtl_init(tf_mgr)); + tf_mgr->page_cache_controller_.write_buffer_pool_.default_wbp_memory_limit_ = 40*1024*1024; + EXPECT_EQ(OB_SUCCESS, tf_mgr->start()); + tenant_ctx.set(tf_mgr); + ObTenantEnv::set_tenant(&tenant_ctx); ASSERT_EQ(0, OB_SERVER_BLOCK_MGR.block_map_.count()); @@ -188,8 +197,10 @@ TEST_F(TestBlockManager, test_mark_and_sweep) macro_handle.reset(); - FILE_MANAGER_INSTANCE_V2.destroy(); + tmp_file::ObTmpBlockCache::get_instance().destroy(); + tmp_file::ObTmpPageCache::get_instance().destroy(); ObKVGlobalCache::get_instance().destroy(); + common::ObClockGenerator::destroy(); } TEST_F(TestBlockManager, test_ref_cnt_wash_and_load) diff --git a/unittest/storage/blocksstable/test_tmp_file.cpp b/unittest/storage/blocksstable/test_tmp_file.cpp deleted file mode 100644 index 890cd8279..000000000 --- a/unittest/storage/blocksstable/test_tmp_file.cpp +++ /dev/null @@ -1,2378 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#include -#include -#include -#define protected public -#define private public -#include "lib/allocator/ob_fifo_allocator.h" -#include "storage/blocksstable/ob_tmp_file.h" -#include "storage/blocksstable/ob_tmp_file_store.h" -#include "storage/blocksstable/ob_tmp_file_cache.h" -#include "ob_row_generate.h" -#include "ob_data_file_prepare.h" -#include "share/ob_simple_mem_limit_getter.h" - -namespace oceanbase -{ -using namespace common; -using namespace blocksstable; -using namespace storage; -using namespace share::schema; -static ObSimpleMemLimitGetter getter; - -namespace unittest -{ - -static const int64_t TEST_COLUMN_CNT = ObExtendType - 1; -static const int64_t TEST_ROWKEY_COLUMN_CNT = 2; - -struct BufHeader -{ -public: - BufHeader() - : data_size_(0), start_row_(0) - {} - virtual ~BufHeader() {} - int serialize(char *buf, const int64_t buf_len, int64_t &pos); - int deserialize(const char *buf, const int64_t data_len, int64_t &pos); - int64_t get_serialize_size() const; - int64_t data_size_; - int64_t start_row_; -}; - -int BufHeader::serialize(char *buf, const int64_t buf_len, int64_t &pos) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(serialization::encode_i64(buf, buf_len, pos, data_size_))) { - STORAGE_LOG(WARN, "fail to serialize data size", K(ret)); - } else if (OB_FAIL(serialization::encode_i64(buf, buf_len, pos, start_row_))) { - STORAGE_LOG(WARN, "fail to serialize start row", K(ret)); - } - return ret; -} - -int BufHeader::deserialize(const char *buf, const int64_t data_len, int64_t &pos) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(serialization::decode_i64(buf, data_len, pos, &data_size_))) { - STORAGE_LOG(WARN, "fail to decode data size", K(ret)); - } else if (OB_FAIL(serialization::decode_i64(buf, data_len, pos, &start_row_))) { - STORAGE_LOG(WARN, "fail to decode start row", K(ret)); - } - return ret; -} - -int64_t BufHeader::get_serialize_size() const -{ - int64_t size = 0; - size += serialization::encoded_length_i64(1L); - size += serialization::encoded_length_i64(1L); - return size; -} - -class TestTmpFileStress : public share::ObThreadPool -{ -public: - TestTmpFileStress(); - TestTmpFileStress(ObTenantBase *tenant_ctx); - virtual ~TestTmpFileStress(); - int init(const int fd, const bool is_write, const int64_t thread_cnt, ObTableSchema *table_schema, - const bool is_plain_data, const bool is_big_file, bool is_truncate = false); - virtual void run1(); -private: - void prepare_data(char *buf, const int64_t macro_block_size); - void prepare_plain_data(const int64_t buf_size, char *buf, ObIArray &size_array); - void prepare_one_buffer(const int64_t macro_block_size, const int64_t start_index, char *buf, int64_t &end_index); - void check_data(const char *buf, const int64_t buf_len); - void check_plain_data(const char *read_buf, const char *right_buf, const int64_t buf_len); - void write_data(const int64_t macro_block_size); - void write_plain_data(char *&buf, const int64_t macro_block_size); - void read_data(const int64_t macro_block_size); - void read_and_truncate(const int64_t macro_block_size); - void read_plain_data(const char *buf, const int64_t macro_block_size); -private: - static const int64_t BUF_COUNT = 16; - int64_t thread_cnt_; - int64_t size_; - int fd_; - bool is_write_; - bool is_big_file_; - ObTableSchema *table_schema_; - bool is_plain_; - bool is_truncate_; - ObTenantBase *tenant_ctx_; -}; - -TestTmpFileStress::TestTmpFileStress() - : thread_cnt_(0), size_(OB_SERVER_BLOCK_MGR.get_macro_block_size()), fd_(0), - is_write_(false), is_big_file_(false), table_schema_(NULL), is_plain_(false), - is_truncate_(false) -{ -} - -TestTmpFileStress::TestTmpFileStress(ObTenantBase *tenant_ctx) - : thread_cnt_(0), size_(OB_SERVER_BLOCK_MGR.get_macro_block_size()), fd_(0), - is_write_(false), is_big_file_(false), table_schema_(NULL), is_plain_(false), - is_truncate_(false), tenant_ctx_(tenant_ctx) -{ -} - -TestTmpFileStress::~TestTmpFileStress() -{ -} - -int TestTmpFileStress::init(const int fd, const bool is_write, - const int64_t thread_cnt, ObTableSchema *table_schema, - const bool is_plain, const bool is_big_file, const bool is_truncate) -{ - int ret = OB_SUCCESS; - if (thread_cnt < 0) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(thread_cnt)); - } else { - thread_cnt_ = thread_cnt; - fd_ = fd; - is_write_ = is_write; - table_schema_ = table_schema; - is_plain_ = is_plain; - is_big_file_ = is_big_file; - if (!is_big_file_) { - size_ = 16L * 1024L; - } - is_truncate_ = is_truncate; - set_thread_count(static_cast(thread_cnt)); - } - return ret; -} - -void TestTmpFileStress::prepare_one_buffer(const int64_t macro_block_size, const int64_t start_index, char *buf, int64_t &end_index) -{ - int ret = OB_SUCCESS; - ObStoreRow row; - BufHeader header; - ObArenaAllocator allocator; - ObRowGenerate row_generate; - int64_t buf_pos = header.get_serialize_size(); - int64_t header_pos = 0; - ObObj cells[TEST_COLUMN_CNT]; - row.row_val_.cells_ = cells; - row.row_val_.count_ = TEST_COLUMN_CNT; - header.start_row_ = start_index; - const int64_t buf_capacity = macro_block_size; - ASSERT_EQ(OB_SUCCESS, row_generate.init(*table_schema_, &allocator)); - for (int64_t i = start_index; OB_SUCC(ret) && buf_pos < buf_capacity; ++i) { - ret = row_generate.get_next_row(i, row); - ASSERT_EQ(OB_SUCCESS, ret); - if (buf_pos + row.get_serialize_size() <= buf_capacity) { - ASSERT_EQ(OB_SUCCESS, row.serialize(buf, buf_capacity, buf_pos)); - } else { - end_index = i; - break; - } - } - header.data_size_ = buf_pos; - ASSERT_EQ(OB_SUCCESS, header.serialize(buf, buf_capacity, header_pos)); -} - -void TestTmpFileStress::prepare_data(char *buf, const int64_t macro_block_size) -{ - const int64_t macro_block_buffer_count = BUF_COUNT; - int64_t buf_pos = 0; - int64_t start_index = 0; - for (int64_t i = 0; i < macro_block_buffer_count; ++i) { - int64_t end_index = 0; - prepare_one_buffer(macro_block_size, start_index, buf + buf_pos, end_index); - buf_pos += macro_block_size; - start_index = end_index; - } -} - -void TestTmpFileStress::prepare_plain_data(const int64_t buf_capacity, char *buf, - ObIArray &size_array) -{ - ObRandom random; - int64_t left_size = buf_capacity; - int8_t data = 0; - const int64_t macro_block_size = OB_SERVER_BLOCK_MGR.get_macro_block_size(); - while (left_size > 0) { - if (left_size < macro_block_size) { - memset(buf, data, left_size); - ASSERT_EQ(OB_SUCCESS, size_array.push_back(left_size)); - left_size = 0; - } else { - const int64_t rand_data = random.get(0, macro_block_size); - memset(buf, data, rand_data); - left_size -= rand_data; - buf += rand_data; - ASSERT_EQ(OB_SUCCESS, size_array.push_back(rand_data)); - } - ++data; - } -} - -void TestTmpFileStress::check_plain_data(const char *read_buf, const char *right_buf, const int64_t buf_len) -{ - int cmp = memcmp(read_buf, right_buf, buf_len); - ASSERT_EQ(0, cmp); -} - -void TestTmpFileStress::check_data(const char *buf, const int64_t buf_len) -{ - int ret = OB_SUCCESS; - int64_t header_pos = 0; - int64_t data_pos = 0; - const char *data = NULL; - BufHeader header; - ObArenaAllocator allocator; - ObRowGenerate row_generate; - ASSERT_EQ(OB_SUCCESS, row_generate.init(*table_schema_, &allocator)); - const int64_t serialize_size = header.get_serialize_size(); - ObStoreRow lhs_row; - ObStoreRow rhs_row; - ObObj lhs_cells[TEST_COLUMN_CNT]; - ObObj rhs_cells[TEST_COLUMN_CNT]; - lhs_row.row_val_.cells_ = lhs_cells; - lhs_row.row_val_.count_ = TEST_COLUMN_CNT; - rhs_row.row_val_.cells_ = rhs_cells; - rhs_row.row_val_.count_ = TEST_COLUMN_CNT; - ret = header.deserialize(buf, buf_len, header_pos); - const int64_t data_len = header.data_size_; - int64_t i = header.start_row_; - ASSERT_EQ(OB_SUCCESS, ret); - data = buf + header_pos; - while (data_pos < data_len - serialize_size) { - ret = lhs_row.deserialize(data, data_len, data_pos); - ASSERT_EQ(OB_SUCCESS, ret); - ret = row_generate.get_next_row(i, rhs_row); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_TRUE(lhs_row.row_val_ == rhs_row.row_val_); - ++i; - } -} - -void TestTmpFileStress::write_data(const int64_t macro_block_size) -{ - int ret = OB_SUCCESS; - ObArenaAllocator allocator; - ObRowGenerate row_generate; - ObTmpFileIOInfo io_info; - row_generate.reset(); - ret = row_generate.init(*table_schema_, &allocator); - ASSERT_EQ(OB_SUCCESS, ret); - io_info.fd_ = fd_; - io_info.size_ = macro_block_size; - io_info.tenant_id_ = 1; - io_info.io_desc_.set_wait_event(2); - io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; - char *buf = new char[BUF_COUNT * macro_block_size]; - const int64_t timeout_ms = 5000; - prepare_data(buf, macro_block_size); - for (int64_t i = 0; i < BUF_COUNT; ++i) { - io_info.buf_ = buf + i * macro_block_size; - check_data(io_info.buf_, macro_block_size); - ret = ObTmpFileManager::get_instance().write(io_info); - ASSERT_EQ(OB_SUCCESS, ret); - } -} - -void TestTmpFileStress::write_plain_data(char *&buf, const int64_t macro_block_size) -{ - int ret = OB_SUCCESS; - ObArray size_array; - ObTmpFileIOInfo io_info; - ASSERT_EQ(OB_SUCCESS, ret); - io_info.fd_ = fd_; - io_info.tenant_id_ = 1; - io_info.io_desc_.set_wait_event(2); - io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; - buf = new char[BUF_COUNT * macro_block_size]; - const int64_t timeout_ms = 5000; - int64_t sum_size = 0; - prepare_plain_data(BUF_COUNT * macro_block_size, buf, size_array); - for (int64_t i = 0; i < size_array.count(); ++i) { - io_info.buf_ = buf + sum_size; - io_info.size_ = size_array.at(i); - ret = ObTmpFileManager::get_instance().write(io_info); - ASSERT_EQ(OB_SUCCESS, ret); - sum_size += size_array.at(i); - } - ret = ObTmpFileManager::get_instance().sync(fd_, timeout_ms); - ASSERT_EQ(OB_SUCCESS, ret); -} - -void TestTmpFileStress::read_data(const int64_t macro_block_size) -{ - int ret = OB_SUCCESS; - ObTmpFileIOInfo io_info; - ObTmpFileIOHandle handle; - io_info.fd_ = fd_; - io_info.size_ = macro_block_size; - io_info.tenant_id_ = 1; - io_info.io_desc_.set_wait_event(2); - io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; - char *buf = new char[macro_block_size]; - for (int64_t i = 0; i < BUF_COUNT; ++i) { - io_info.buf_ = buf; - ret = ObTmpFileManager::get_instance().read(io_info, handle); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(macro_block_size, handle.get_data_size()); - check_data(handle.get_buffer(), handle.get_data_size()); - ASSERT_EQ(OB_SUCCESS, ret); - } - handle.reset(); -} - -void TestTmpFileStress::read_and_truncate(const int64_t macro_block_size) -{ - int ret = OB_SUCCESS; - const int64_t timeout_ms = 100000; - ObTmpFileIOInfo io_info; - ObTmpFileIOHandle handle; - io_info.fd_ = fd_; - io_info.size_ = macro_block_size; - io_info.tenant_id_ = 1; - io_info.io_desc_.set_wait_event(2); - int64_t trunc_offset = 0; - char *buf = new char[macro_block_size]; - char *zero_buf = new char[macro_block_size]; - memset(zero_buf, 0, macro_block_size); - for (int64_t i = 0; i < BUF_COUNT; ++i) { - io_info.buf_ = buf; - ret = ObTmpFileManager::get_instance().read(io_info, handle); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(macro_block_size, handle.get_data_size()); - check_data(handle.get_buffer(), handle.get_data_size()); - ASSERT_EQ(OB_SUCCESS, ret); - // truncate data - // truncate trunc_offset + macro_block_size; - ret = ObTmpFileManager::get_instance().truncate(io_info.fd_, trunc_offset + macro_block_size); - ASSERT_EQ(OB_SUCCESS, ret); - ret = ObTmpFileManager::get_instance().pread(io_info, trunc_offset, handle); - ASSERT_EQ(OB_SUCCESS, ret); - check_plain_data(zero_buf, buf, macro_block_size); - trunc_offset += macro_block_size; - } - // check tuncated(0) won't reset the read_guard; - if (ret == OB_SUCCESS) { - io_info.buf_ = buf; - ret = ObTmpFileManager::get_instance().pread(io_info, 0, handle); - ASSERT_EQ(OB_SUCCESS, ret); - check_plain_data(zero_buf, buf, macro_block_size); - ret = ObTmpFileManager::get_instance().truncate(io_info.fd_, 0); - ASSERT_EQ(OB_SUCCESS, ret); - ret = ObTmpFileManager::get_instance().pread(io_info, 0, handle); - ASSERT_EQ(OB_SUCCESS, ret); - check_plain_data(zero_buf, buf, macro_block_size); - } - handle.reset(); -} - -void TestTmpFileStress::read_plain_data(const char *read_buf, const int64_t macro_block_size) -{ - int ret = OB_SUCCESS; - ObTmpFileIOInfo io_info; - ObTmpFileIOHandle handle; - io_info.fd_ = fd_; - io_info.size_ = macro_block_size; - io_info.tenant_id_ = 1; - io_info.io_desc_.set_wait_event(2); - io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; - char *buf = new char[BUF_COUNT * macro_block_size]; - int64_t offset = 0; - for (int64_t i = 0; i < BUF_COUNT; ++i) { - io_info.buf_ = buf + i * macro_block_size; - offset = i * macro_block_size; - ret = ObTmpFileManager::get_instance().pread(io_info, offset, handle); - ASSERT_EQ(OB_SUCCESS, ret); - } - offset += macro_block_size; - ret = ObTmpFileManager::get_instance().pread(io_info, offset, handle); - ASSERT_EQ(OB_ITER_END, ret); - check_plain_data(read_buf, buf, BUF_COUNT * macro_block_size); - handle.reset(); -} - -void TestTmpFileStress::run1() -{ - ObTenantEnv::set_tenant(tenant_ctx_); - if (is_plain_) { - char *buf = NULL; - write_plain_data(buf, size_); - read_plain_data(buf, size_); - } else { - if (is_write_) { - write_data(size_); - } else { - read_data(size_); - } - } -} - -class TestMultiTmpFileStress : public share::ObThreadPool -{ -public: - TestMultiTmpFileStress(); - TestMultiTmpFileStress(ObTenantBase *tenant_ctx); - virtual ~TestMultiTmpFileStress(); - int init(const int64_t file_cnt, const int64_t dir_id, const int64_t thread_cnt, - ObTableSchema *table_schema, const bool is_plain_data, const bool is_big_file, - const bool is_truncate = false); - virtual void run1(); -private: - void run_plain_case(); - void run_normal_case(); -private: - int64_t file_cnt_; - int64_t dir_id_; - int64_t thread_cnt_perf_file_; - ObTableSchema *table_schema_; - bool is_big_file_; - bool is_plain_data_; - bool is_truncate_; - ObTenantBase *tenant_ctx_; -}; - -TestMultiTmpFileStress::TestMultiTmpFileStress() - : file_cnt_(0), - dir_id_(-1), - thread_cnt_perf_file_(0), - table_schema_(NULL), - is_big_file_(false), - is_plain_data_(false), - is_truncate_(false) -{ -} -TestMultiTmpFileStress::TestMultiTmpFileStress(ObTenantBase *tenant_ctx) - : file_cnt_(0), - dir_id_(-1), - thread_cnt_perf_file_(0), - table_schema_(NULL), - is_big_file_(false), - is_plain_data_(false), - is_truncate_(false), - tenant_ctx_(tenant_ctx) -{ -} - -TestMultiTmpFileStress::~TestMultiTmpFileStress() -{ -} - -int TestMultiTmpFileStress::init(const int64_t file_cnt, - const int64_t dir_id, - const int64_t thread_cnt, - ObTableSchema *table_schema, - const bool is_plain_data, - const bool is_big_file, - const bool is_truncate) -{ - int ret = OB_SUCCESS; - if (file_cnt < 0 || thread_cnt < 0 || NULL == table_schema) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(file_cnt), K(thread_cnt), - KP(table_schema)); - } else { - file_cnt_ = file_cnt; - dir_id_ = dir_id; - thread_cnt_perf_file_ = thread_cnt; - table_schema_ = table_schema; - is_big_file_ = is_big_file; - is_plain_data_ = is_plain_data; - is_truncate_ = is_truncate; - set_thread_count(static_cast(file_cnt)); - } - return ret; -} - -void TestMultiTmpFileStress::run_plain_case() -{ - int ret = OB_SUCCESS; - int64_t fd = 0; - TestTmpFileStress test(tenant_ctx_); - ret = ObTmpFileManager::get_instance().open(fd, dir_id_); - ASSERT_EQ(OB_SUCCESS, ret); - ret = test.init(fd, true, thread_cnt_perf_file_, table_schema_, is_plain_data_, is_big_file_); - ASSERT_EQ(OB_SUCCESS, ret); - test.start(); - test.wait(); - ret = ObTmpFileManager::get_instance().remove(fd); - ASSERT_EQ(OB_SUCCESS, ret); -} - -void TestMultiTmpFileStress::run_normal_case() -{ - int ret = OB_SUCCESS; - int64_t fd = 0; - const int64_t timeout_ms = 5000; - TestTmpFileStress test_write(tenant_ctx_); - TestTmpFileStress test_read(tenant_ctx_); - ret = ObTmpFileManager::get_instance().open(fd, dir_id_); - ASSERT_EQ(OB_SUCCESS, ret); - STORAGE_LOG(INFO, "open file success", K(fd)); - ret = test_write.init(fd, true, thread_cnt_perf_file_, table_schema_, is_plain_data_, is_big_file_, is_truncate_); - ASSERT_EQ(OB_SUCCESS, ret); - ret = test_read.init(fd, false, thread_cnt_perf_file_, table_schema_, is_plain_data_, is_big_file_, is_truncate_); - ASSERT_EQ(OB_SUCCESS, ret); - test_write.start(); - test_write.wait(); - ret = ObTmpFileManager::get_instance().sync(fd, timeout_ms); - ASSERT_EQ(OB_SUCCESS, ret); - test_read.start(); - test_read.wait(); - ret = ObTmpFileManager::get_instance().remove(fd); - ASSERT_EQ(OB_SUCCESS, ret); -} - -void TestMultiTmpFileStress::run1() -{ - ObTenantEnv::set_tenant(tenant_ctx_); - if (is_plain_data_) { - run_plain_case(); - } else { - run_normal_case(); - } -} - -class TestTmpFile : public TestDataFilePrepare -{ -public: - TestTmpFile(); - virtual ~TestTmpFile(); - virtual void SetUp(); - virtual void TearDown(); -protected: - ObTableSchema table_schema_; -private: - void prepare_schema(); -}; - -TestTmpFile::TestTmpFile() - : TestDataFilePrepare(&getter, "TestTmpFile", 2 * 1024 * 1024, 2048) -{ -} - -TestTmpFile::~TestTmpFile() -{ -} - -void TestTmpFile::prepare_schema() -{ - ObColumnSchemaV2 column; - int64_t table_id = 3001; - int64_t micro_block_size = 16 * 1024; - //init table schema - table_schema_.reset(); - ASSERT_EQ(OB_SUCCESS, table_schema_.set_table_name("test_macro_file")); - table_schema_.set_tenant_id(1); - table_schema_.set_tablegroup_id(1); - table_schema_.set_database_id(1); - table_schema_.set_table_id(table_id); - table_schema_.set_rowkey_column_num(TEST_ROWKEY_COLUMN_CNT); - table_schema_.set_max_used_column_id(TEST_COLUMN_CNT); - table_schema_.set_block_size(micro_block_size); - table_schema_.set_compress_func_name("none"); - //init column - char name[OB_MAX_FILE_NAME_LENGTH]; - memset(name, 0, sizeof(name)); - for(int64_t i = 0; i < TEST_COLUMN_CNT; ++i) { - ObObjType obj_type = static_cast(i + 1); - column.reset(); - column.set_table_id(table_id); - column.set_column_id(i + OB_APP_MIN_COLUMN_ID); - sprintf(name, "test%020ld", i); - ASSERT_EQ(OB_SUCCESS, column.set_column_name(name)); - column.set_data_type(obj_type); - column.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); - column.set_data_length(1); - if(obj_type == common::ObIntType){ - column.set_rowkey_position(1); - } else if(obj_type == common::ObVarcharType) { - column.set_rowkey_position(2); - } else { - column.set_rowkey_position(0); - } - ASSERT_EQ(OB_SUCCESS, table_schema_.add_column(column)); - } - ObTmpFileManager::get_instance().destroy(); -} - -void TestTmpFile::SetUp() -{ - int ret = OB_SUCCESS; - const int64_t bucket_num = 1024; - const int64_t max_cache_size = 1024 * 1024 * 1024; - const int64_t block_size = common::OB_MALLOC_BIG_BLOCK_SIZE; - TestDataFilePrepare::SetUp(); - prepare_schema(); - - ret = getter.add_tenant(1, - 8L * 1024L * 1024L, 2L * 1024L * 1024L * 1024L); - ASSERT_EQ(OB_SUCCESS, ret); - ret = ObKVGlobalCache::get_instance().init(&getter, bucket_num, max_cache_size, block_size); - if (OB_INIT_TWICE == ret) { - ret = OB_SUCCESS; - } else { - ASSERT_EQ(OB_SUCCESS, ret); - } - // set observer memory limit - CHUNK_MGR.set_limit(8L * 1024L * 1024L * 1024L); - ret = ObTmpFileManager::get_instance().init(); - ASSERT_EQ(OB_SUCCESS, ret); - static ObTenantBase tenant_ctx(OB_SYS_TENANT_ID); - ObTenantEnv::set_tenant(&tenant_ctx); - ObTenantIOManager *io_service = nullptr; - EXPECT_EQ(OB_SUCCESS, ObTenantIOManager::mtl_new(io_service)); - EXPECT_EQ(OB_SUCCESS, ObTenantIOManager::mtl_init(io_service)); - EXPECT_EQ(OB_SUCCESS, io_service->start()); - tenant_ctx.set(io_service); - ObTenantEnv::set_tenant(&tenant_ctx); - - ObMallocAllocator::get_instance()->set_tenant_limit(1, 8L * 1024L * 1024L * 1024L /* 8 GB */); -} - -void TestTmpFile::TearDown() -{ - table_schema_.reset(); - ObTmpFileManager::get_instance().destroy(); - ObKVGlobalCache::get_instance().destroy(); - ObTmpFileStore::get_instance().destroy(); - TestDataFilePrepare::TearDown(); -} - -TEST_F(TestTmpFile, test_big_file) -{ - int ret = OB_SUCCESS; - int64_t dir = -1; - int64_t fd = -1; - const int64_t macro_block_size = OB_SERVER_BLOCK_MGR.get_macro_block_size(); - ObTmpFileIOInfo io_info; - ObTmpFileIOHandle handle; - ret = ObTmpFileManager::get_instance().alloc_dir(dir); - ASSERT_EQ(OB_SUCCESS, ret); - ret = ObTmpFileManager::get_instance().open(fd, dir); - ASSERT_EQ(OB_SUCCESS, ret); - int64_t write_size = macro_block_size * 512; - char *write_buf = (char *)malloc(write_size); - for (int64_t i = 0; i < write_size; ++i) { - write_buf[i] = static_cast(i % 256); - } - char *read_buf = (char *)malloc(write_size); - io_info.fd_ = fd; - io_info.tenant_id_ = 1; - io_info.io_desc_.set_wait_event(2); - io_info.buf_ = write_buf; - io_info.size_ = write_size; - io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; - int64_t write_time = ObTimeUtility::current_time(); - ret = ObTmpFileManager::get_instance().write(io_info); - ASSERT_EQ(OB_SUCCESS, ret); - write_time = ObTimeUtility::current_time() - write_time; - io_info.buf_ = read_buf; - - // Flush all held block caches to ensure that subsequent read processes will go through I/O. - ObKVGlobalCache::get_instance().erase_cache(1, "tmp_block_cache"); - - io_info.size_ = write_size; - ret = ObTmpFileManager::get_instance().aio_read(io_info, handle); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_TRUE(handle.size_ < handle.expect_read_size_); - ASSERT_EQ(OB_SUCCESS, handle.wait()); - ASSERT_EQ(write_size, handle.get_data_size()); - int cmp = memcmp(handle.get_buffer(), write_buf, handle.get_data_size()); - ASSERT_EQ(0, cmp); - - io_info.size_ = macro_block_size; - ret = ObTmpFileManager::get_instance().pread(io_info, 100, handle); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(macro_block_size, handle.get_data_size()); - cmp = memcmp(handle.get_buffer(), write_buf + 100, handle.get_data_size()); - ASSERT_EQ(0, cmp); - - io_info.size_ = write_size; - int64_t read_time = ObTimeUtility::current_time(); - ret = ObTmpFileManager::get_instance().pread(io_info, 0, handle); - read_time = ObTimeUtility::current_time() - read_time; - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(write_size, handle.get_data_size()); - cmp = memcmp(handle.get_buffer(), write_buf, write_size); - ASSERT_EQ(0, cmp); - - io_info.size_ = 200; - ret = ObTmpFileManager::get_instance().pread(io_info, 200, handle); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(200, handle.get_data_size()); - cmp = memcmp(handle.get_buffer(), write_buf + 200, 200); - ASSERT_EQ(0, cmp); - - free(write_buf); - free(read_buf); - - STORAGE_LOG(INFO, "test_big_file"); - STORAGE_LOG(INFO, "io time", K(write_time), K(read_time)); - ObTmpTenantFileStoreHandle store_handle; - OB_TMP_FILE_STORE.get_store(1, store_handle); - store_handle.get_tenant_store()->print_block_usage(); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(500); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500); - - ObTmpFileManager::get_instance().remove(fd); -} - -TEST_F(TestTmpFile, test_big_file_disable_page_cache) -{ - int ret = OB_SUCCESS; - int64_t dir = -1; - int64_t fd = -1; - const int64_t macro_block_size = OB_SERVER_BLOCK_MGR.get_macro_block_size(); - ObTmpFileIOInfo io_info; - ObTmpFileIOHandle handle; - ret = ObTmpFileManager::get_instance().alloc_dir(dir); - ASSERT_EQ(OB_SUCCESS, ret); - ret = ObTmpFileManager::get_instance().open(fd, dir); - ASSERT_EQ(OB_SUCCESS, ret); - int64_t write_size = macro_block_size * 512; - char *write_buf = (char *)malloc(write_size); - for (int64_t i = 0; i < write_size; ++i) { - write_buf[i] = static_cast(i % 256); - } - char *read_buf = (char *)malloc(write_size); - io_info.fd_ = fd; - io_info.tenant_id_ = 1; - io_info.io_desc_.set_wait_event(2); - io_info.buf_ = write_buf; - io_info.size_ = write_size; - io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; - io_info.disable_page_cache_ = true; - int64_t write_time = ObTimeUtility::current_time(); - ret = ObTmpFileManager::get_instance().write(io_info); - ASSERT_EQ(OB_SUCCESS, ret); - write_time = ObTimeUtility::current_time() - write_time; - io_info.buf_ = read_buf; - - // Flush all held block caches to ensure that subsequent read processes will go through I/O. - ObKVGlobalCache::get_instance().erase_cache(1, "tmp_block_cache"); - - io_info.size_ = write_size; - ret = ObTmpFileManager::get_instance().aio_read(io_info, handle); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_TRUE(handle.size_ < handle.expect_read_size_); - ASSERT_EQ(OB_SUCCESS, handle.wait()); - ASSERT_EQ(write_size, handle.get_data_size()); - int cmp = memcmp(handle.get_buffer(), write_buf, handle.get_data_size()); - ASSERT_EQ(0, cmp); - - io_info.size_ = macro_block_size; - ret = ObTmpFileManager::get_instance().pread(io_info, 100, handle); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(macro_block_size, handle.get_data_size()); - cmp = memcmp(handle.get_buffer(), write_buf + 100, handle.get_data_size()); - ASSERT_EQ(0, cmp); - - io_info.size_ = write_size; - int64_t read_time = ObTimeUtility::current_time(); - ret = ObTmpFileManager::get_instance().pread(io_info, 0, handle); - read_time = ObTimeUtility::current_time() - read_time; - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(write_size, handle.get_data_size()); - cmp = memcmp(handle.get_buffer(), write_buf, write_size); - ASSERT_EQ(0, cmp); - - io_info.size_ = 200; - ret = ObTmpFileManager::get_instance().pread(io_info, 200, handle); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(200, handle.get_data_size()); - cmp = memcmp(handle.get_buffer(), write_buf + 200, 200); - ASSERT_EQ(0, cmp); - - free(write_buf); - free(read_buf); - - STORAGE_LOG(INFO, "test_big_file_disable_page_cache"); - STORAGE_LOG(INFO, "io time", K(write_time), K(read_time)); - ObTmpTenantFileStoreHandle store_handle; - OB_TMP_FILE_STORE.get_store(1, store_handle); - store_handle.get_tenant_store()->print_block_usage(); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(500); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500); - - ObTmpFileManager::get_instance().remove(fd); -} - -TEST_F(TestTmpFile, test_multi_small_file_single_thread_read_write) -{ - int ret = OB_SUCCESS; - const int64_t thread_cnt = 1; - const int64_t file_cnt = 4; - const bool is_plain_data = false; - const bool is_big_file = false; - TestMultiTmpFileStress test(MTL_CTX()); - int64_t dir = -1; - ret = ObTmpFileManager::get_instance().alloc_dir(dir); - ASSERT_EQ(OB_SUCCESS, ret); - ret = test.init(file_cnt, dir, thread_cnt, &table_schema_, is_plain_data, is_big_file); - ASSERT_EQ(OB_SUCCESS, ret); - int64_t io_time = ObTimeUtility::current_time(); - test.start(); - test.wait(); - io_time = ObTimeUtility::current_time() - io_time; - - STORAGE_LOG(INFO, "test_multi_small_file_single_thread_read_write"); - STORAGE_LOG(INFO, "io time", K(io_time)); - ObTmpTenantFileStoreHandle store_handle; - OB_TMP_FILE_STORE.get_store(1, store_handle); - store_handle.get_tenant_store()->print_block_usage(); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(500); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500); -} - -TEST_F(TestTmpFile, test_multi_small_file_multi_thread_read_write ) -{ - int ret = OB_SUCCESS; - const int64_t thread_cnt = 4; - const int64_t file_cnt = 4; - const bool is_plain_data = false; - const bool is_big_file = false; - TestMultiTmpFileStress test(MTL_CTX()); - int64_t dir = -1; - ret = ObTmpFileManager::get_instance().alloc_dir(dir); - ASSERT_EQ(OB_SUCCESS, ret); - ret = test.init(file_cnt, dir, thread_cnt, &table_schema_, is_plain_data, is_big_file); - ASSERT_EQ(OB_SUCCESS, ret); - int64_t io_time = ObTimeUtility::current_time(); - test.start(); - test.wait(); - io_time = ObTimeUtility::current_time() - io_time; - - STORAGE_LOG(INFO, "test_multi_small_file_multi_thread_read_write"); - STORAGE_LOG(INFO, "io time", K(io_time)); - ObTmpTenantFileStoreHandle store_handle; - OB_TMP_FILE_STORE.get_store(1, store_handle); - store_handle.get_tenant_store()->print_block_usage(); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(500); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500); -} - -TEST_F(TestTmpFile, test_inner_read_offset_and_seek) -{ - int ret = OB_SUCCESS; - int64_t dir = -1; - int64_t fd = -1; - const int64_t macro_block_size = OB_SERVER_BLOCK_MGR.get_macro_block_size(); - ObTmpFileIOInfo io_info; - ObTmpFileIOHandle handle; - ret = ObTmpFileManager::get_instance().alloc_dir(dir); - ASSERT_EQ(OB_SUCCESS, ret); - ret = ObTmpFileManager::get_instance().open(fd, dir); - ASSERT_EQ(OB_SUCCESS, ret); - char *write_buf = new char [macro_block_size + 256]; - for (int i = 0; i < macro_block_size + 256; ++i) { - write_buf[i] = static_cast(i % 256); - } - char *read_buf = new char [macro_block_size + 256]; - io_info.fd_ = fd; - io_info.tenant_id_ = 1; - io_info.io_desc_.set_wait_event(2); - io_info.buf_ = write_buf; - io_info.size_ = macro_block_size + 256; - io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; - int64_t write_time = ObTimeUtility::current_time(); - ret = ObTmpFileManager::get_instance().write(io_info); - write_time = ObTimeUtility::current_time() - write_time; - ASSERT_EQ(OB_SUCCESS, ret); - io_info.buf_ = read_buf; - - - int64_t read_time = ObTimeUtility::current_time(); - ret = ObTmpFileManager::get_instance().read(io_info, handle); - read_time = ObTimeUtility::current_time() - read_time; - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(macro_block_size + 256, handle.get_data_size()); - int cmp = memcmp(handle.get_buffer(), write_buf, macro_block_size + 256); - ASSERT_EQ(0, cmp); - - - io_info.size_ = 200; - ret = ObTmpFileManager::get_instance().read(io_info, handle); - ASSERT_EQ(OB_ITER_END, ret); - - - ret = ObTmpFileManager::get_instance().seek(fd, 0, ObTmpFile::SET_SEEK); - ASSERT_EQ(OB_SUCCESS, ret); - - io_info.size_ = 201; - ret = ObTmpFileManager::get_instance().read(io_info, handle); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(201, handle.get_data_size()); - cmp = memcmp(handle.get_buffer(), write_buf, 201); - ASSERT_EQ(0, cmp); - - - ret = ObTmpFileManager::get_instance().seek(fd, 199, ObTmpFile::CUR_SEEK); - ASSERT_EQ(OB_SUCCESS, ret); - - io_info.size_ = 199; - ret = ObTmpFileManager::get_instance().read(io_info, handle); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(199, handle.get_data_size()); - cmp = memcmp(handle.get_buffer(), write_buf + 400, 199); - ASSERT_EQ(0, cmp); - - - STORAGE_LOG(INFO, "test_inner_read_offset_and_seek"); - STORAGE_LOG(INFO, "io time", K(write_time), K(read_time)); - ObTmpTenantFileStoreHandle store_handle; - OB_TMP_FILE_STORE.get_store(1, store_handle); - store_handle.get_tenant_store()->print_block_usage(); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(500); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500); - - ObTmpFileManager::get_instance().remove(fd); -} - -TEST_F(TestTmpFile, test_single_file_single_thread_read_write) -{ - int ret = OB_SUCCESS; - const int64_t thread_cnt = 1; - const int64_t file_cnt = 1; - const bool is_plain_data = false; - const bool is_big_file = true; - TestMultiTmpFileStress test(MTL_CTX()); - int64_t dir = -1; - ret = ObTmpFileManager::get_instance().alloc_dir(dir); - ASSERT_EQ(OB_SUCCESS, ret); - ret = test.init(file_cnt, dir, thread_cnt, &table_schema_, is_plain_data, is_big_file); - ASSERT_EQ(OB_SUCCESS, ret); - int64_t io_time = ObTimeUtility::current_time(); - test.start(); - test.wait(); - io_time = ObTimeUtility::current_time() - io_time; - - - STORAGE_LOG(INFO, "test_single_file_single_thread_read_write"); - STORAGE_LOG(INFO, "io time", K(io_time)); - ObTmpTenantFileStoreHandle store_handle; - OB_TMP_FILE_STORE.get_store(1, store_handle); - store_handle.get_tenant_store()->print_block_usage(); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(500); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500); -} - -TEST_F(TestTmpFile, test_aio_read_and_write) -{ - int ret = OB_SUCCESS; - int64_t dir = -1; - int64_t fd = -1; - const int64_t macro_block_size = OB_SERVER_BLOCK_MGR.get_macro_block_size(); - ObTmpFileIOInfo io_info; - ObTmpFileIOHandle handle; - ret = ObTmpFileManager::get_instance().alloc_dir(dir); - ASSERT_EQ(OB_SUCCESS, ret); - ret = ObTmpFileManager::get_instance().open(fd, dir); - ASSERT_EQ(OB_SUCCESS, ret); - char *write_buf = new char [macro_block_size + 256]; - for (int i = 0; i < macro_block_size + 256; ++i) { - write_buf[i] = static_cast(i % 256); - } - char *read_buf = new char [macro_block_size + 256]; - - - io_info.fd_ = fd; - io_info.tenant_id_ = 1; - io_info.io_desc_.set_wait_event(2); - io_info.buf_ = write_buf; - io_info.size_ = macro_block_size + 256; - io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; - int64_t write_time = ObTimeUtility::current_time(); - ret = ObTmpFileManager::get_instance().aio_write(io_info, handle); - ASSERT_EQ(OB_SUCCESS, ret); - ret = handle.wait(); - write_time = ObTimeUtility::current_time() - write_time; - ASSERT_EQ(OB_SUCCESS, ret); - handle.reset(); - - io_info.buf_ = read_buf; - - int64_t read_time = ObTimeUtility::current_time(); - ret = ObTmpFileManager::get_instance().read(io_info, handle); - read_time = ObTimeUtility::current_time() - read_time; - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(macro_block_size + 256, handle.get_data_size()); - int cmp = memcmp(handle.get_buffer(), write_buf, macro_block_size + 256); - ASSERT_EQ(0, cmp); - - - ret = ObTmpFileManager::get_instance().seek(fd, 100, ObTmpFile::SET_SEEK); - ASSERT_EQ(OB_SUCCESS, ret); - - - io_info.size_ = macro_block_size; - ret = ObTmpFileManager::get_instance().aio_read(io_info, handle); - ASSERT_EQ(OB_SUCCESS, ret); - ret = handle.wait(); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(macro_block_size, handle.get_data_size()); - cmp = memcmp(handle.get_buffer(), write_buf + 100, macro_block_size); - ASSERT_EQ(0, cmp); - handle.reset(); - - - io_info.size_ = macro_block_size; - ret = ObTmpFileManager::get_instance().aio_pread(io_info, 0, handle); - ASSERT_EQ(OB_SUCCESS, ret); - ret = handle.wait(); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(macro_block_size, handle.get_data_size()); - cmp = memcmp(handle.get_buffer(), write_buf, macro_block_size); - ASSERT_EQ(0, cmp); - handle.reset(); - - - STORAGE_LOG(INFO, "test_aio_read_and_write"); - STORAGE_LOG(INFO, "io time", K(write_time), K(read_time)); - ObTmpTenantFileStoreHandle store_handle; - OB_TMP_FILE_STORE.get_store(1, store_handle); - store_handle.get_tenant_store()->print_block_usage(); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(500); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500); - - ObTmpFileManager::get_instance().remove(fd); -} - -TEST_F(TestTmpFile, test_100_small_files) -{ - int ret = OB_SUCCESS; - int64_t dir = 0; - int64_t fd = 0; - int count = 100; - const int64_t timeout_ms = 5000; - TestTmpFileStress test_write(MTL_CTX()); - TestTmpFileStress test_read(MTL_CTX()); - ret = ObTmpFileManager::get_instance().alloc_dir(dir); - ASSERT_EQ(OB_SUCCESS, ret); - while (count--) { - ret = ObTmpFileManager::get_instance().open(fd, dir); - ASSERT_EQ(OB_SUCCESS, ret); - STORAGE_LOG(INFO, "open file success", K(fd)); - ret = test_write.init(fd, true, 1, &table_schema_, false, false); - ASSERT_EQ(OB_SUCCESS, ret); - ret = test_read.init(fd, false, 1, &table_schema_, false, false); - ASSERT_EQ(OB_SUCCESS, ret); - test_write.start(); - test_write.wait(); - ret = ObTmpFileManager::get_instance().sync(fd, timeout_ms); - ASSERT_EQ(OB_SUCCESS, ret); - test_read.start(); - test_read.wait(); - } - - STORAGE_LOG(INFO, "test_1000_small_files"); - ObTmpTenantFileStoreHandle store_handle; - OB_TMP_FILE_STORE.get_store(1, store_handle); - store_handle.get_tenant_store()->print_block_usage(); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(500); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500); - count = 100; - while (count--) { - ret = ObTmpFileManager::get_instance().remove(count); - ASSERT_EQ(OB_SUCCESS, ret); - } -} - -TEST_F(TestTmpFile, test_single_file_multi_thread_read_write) -{ - int ret = OB_SUCCESS; - const int64_t thread_cnt = 4; - const int64_t file_cnt = 1; - const bool is_plain_data = false; - const bool is_big_file = true; - TestMultiTmpFileStress test(MTL_CTX()); - int64_t dir = -1; - ret = ObTmpFileManager::get_instance().alloc_dir(dir); - ASSERT_EQ(OB_SUCCESS, ret); - ret = test.init(file_cnt, dir, thread_cnt, &table_schema_, is_plain_data, is_big_file); - ASSERT_EQ(OB_SUCCESS, ret); - int64_t io_time = ObTimeUtility::current_time(); - test.start(); - test.wait(); - io_time = ObTimeUtility::current_time() - io_time; - - - STORAGE_LOG(INFO, "test_single_file_multi_thread_read_write"); - STORAGE_LOG(INFO, "io time", K(io_time)); - ObTmpTenantFileStoreHandle store_handle; - OB_TMP_FILE_STORE.get_store(1, store_handle); - store_handle.get_tenant_store()->print_block_usage(); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(500); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500); -} - -TEST_F(TestTmpFile, test_multi_file_single_thread_read_write) -{ - int ret = OB_SUCCESS; - const int64_t thread_cnt = 1; - const int64_t file_cnt = 4; - const bool is_plain_data = false; - const bool is_big_file = true; - TestMultiTmpFileStress test(MTL_CTX()); - int64_t dir = -1; - ret = ObTmpFileManager::get_instance().alloc_dir(dir); - ASSERT_EQ(OB_SUCCESS, ret); - ret = test.init(file_cnt, dir, thread_cnt, &table_schema_, is_plain_data, is_big_file); - ASSERT_EQ(OB_SUCCESS, ret); - int64_t io_time = ObTimeUtility::current_time(); - test.start(); - test.wait(); - io_time = ObTimeUtility::current_time() - io_time; - - - STORAGE_LOG(INFO, "test_multi_file_single_thread_read_write"); - STORAGE_LOG(INFO, "io time", K(io_time)); - ObTmpTenantFileStoreHandle store_handle; - OB_TMP_FILE_STORE.get_store(1, store_handle); - store_handle.get_tenant_store()->print_block_usage(); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(500); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500); -} - -TEST_F(TestTmpFile, test_multi_file_multi_thread_read_write) -{ - int ret = OB_SUCCESS; - const int64_t thread_cnt = 4; - const int64_t file_cnt = 4; - const bool is_plain_data = false; - const bool is_big_file = true; - TestMultiTmpFileStress test(MTL_CTX()); - int64_t dir = -1; - ret = ObTmpFileManager::get_instance().alloc_dir(dir); - ASSERT_EQ(OB_SUCCESS, ret); - ret = test.init(file_cnt, dir, thread_cnt, &table_schema_, is_plain_data, is_big_file); - ASSERT_EQ(OB_SUCCESS, ret); - int64_t io_time = ObTimeUtility::current_time(); - test.start(); - test.wait(); - io_time = ObTimeUtility::current_time() - io_time; - - - STORAGE_LOG(INFO, "test_multi_file_multi_thread_read_write"); - STORAGE_LOG(INFO, "io time", K(io_time)); - ObTmpTenantFileStoreHandle store_handle; - OB_TMP_FILE_STORE.get_store(1, store_handle); - store_handle.get_tenant_store()->print_block_usage(); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(500); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500); -} - -TEST_F(TestTmpFile, test_write_not_macro_size) -{ - int ret = OB_SUCCESS; - const int64_t thread_cnt = 1; - const int64_t file_cnt = 1; - const bool is_plain_data = true; - const bool is_big_file = true; - TestMultiTmpFileStress test(MTL_CTX()); - int64_t dir = -1; - ret = ObTmpFileManager::get_instance().alloc_dir(dir); - ASSERT_EQ(OB_SUCCESS, ret); - ret = test.init(file_cnt, dir, thread_cnt, &table_schema_, is_plain_data, is_big_file); - ASSERT_EQ(OB_SUCCESS, ret); - int64_t io_time = ObTimeUtility::current_time(); - test.start(); - test.wait(); - io_time = ObTimeUtility::current_time() - io_time; - - - STORAGE_LOG(INFO, "test_write_not_macro_size"); - STORAGE_LOG(INFO, "io time", K(io_time)); - ObTmpTenantFileStoreHandle store_handle; - OB_TMP_FILE_STORE.get_store(1, store_handle); - store_handle.get_tenant_store()->print_block_usage(); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(500); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500); -} - -TEST_F(TestTmpFile, test_write_less_than_macro_block_size) -{ - int ret = OB_SUCCESS; - int64_t dir = -1; - int64_t fd = -1; - ObTmpFileIOInfo io_info; - ObTmpFileIOHandle handle; - ret = ObTmpFileManager::get_instance().alloc_dir(dir); - ASSERT_EQ(OB_SUCCESS, ret); - ret = ObTmpFileManager::get_instance().open(fd, dir); - ASSERT_EQ(OB_SUCCESS, ret); - char *write_buf = new char [256]; - for (int i = 0; i < 256; ++i) { - write_buf[i] = static_cast(i); - } - char *read_buf = new char [256]; - io_info.fd_ = fd; - io_info.tenant_id_ = 1; - io_info.io_desc_.set_wait_event(2); - io_info.buf_ = write_buf; - io_info.size_ = 256; - io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; - int64_t write_time = ObTimeUtility::current_time(); - ret = ObTmpFileManager::get_instance().write(io_info); - write_time = ObTimeUtility::current_time() - write_time; - ASSERT_EQ(OB_SUCCESS, ret); - io_info.buf_ = read_buf; - - - int64_t read_time = ObTimeUtility::current_time(); - ret = ObTmpFileManager::get_instance().pread(io_info, 0, handle); - read_time = ObTimeUtility::current_time() - read_time; - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(256, handle.get_data_size()); - int cmp = memcmp(handle.get_buffer(), write_buf, 256); - ASSERT_EQ(0, cmp); - - - io_info.size_ = 255; - ret = ObTmpFileManager::get_instance().pread(io_info, 0, handle); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(255, handle.get_data_size()); - cmp = memcmp(handle.get_buffer(), write_buf, 255); - ASSERT_EQ(0, cmp); - - - ret = ObTmpFileManager::get_instance().pread(io_info, 20, handle); - ASSERT_EQ(OB_ITER_END, ret); - ASSERT_EQ(256 - 20, handle.get_data_size()); - cmp = memcmp(handle.get_buffer(), write_buf + 20, 256 - 20); - ASSERT_EQ(0, cmp); - - - io_info.size_ = 20; - ret = ObTmpFileManager::get_instance().pread(io_info, 40, handle); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(20, handle.get_data_size()); - cmp = memcmp(handle.get_buffer(), write_buf + 40, handle.get_data_size()); - ASSERT_EQ(0, cmp); - - - io_info.size_ = 100; - ret = ObTmpFileManager::get_instance().pread(io_info, 156, handle); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(100, handle.get_data_size()); - cmp = memcmp(handle.get_buffer(), write_buf + 156, handle.get_data_size()); - ASSERT_EQ(0, cmp); - - - ret = ObTmpFileManager::get_instance().pread(io_info, 256, handle); - ASSERT_EQ(OB_ITER_END, ret); - - - STORAGE_LOG(INFO, "test_write_less_than_macro_block_size"); - STORAGE_LOG(INFO, "io time", K(write_time), K(read_time)); - ObTmpTenantFileStoreHandle store_handle; - OB_TMP_FILE_STORE.get_store(1, store_handle); - store_handle.get_tenant_store()->print_block_usage(); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(500); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500); - - ObTmpFileManager::get_instance().remove(fd); -} - -TEST_F(TestTmpFile, test_write_more_than_one_macro_block) -{ - int ret = OB_SUCCESS; - int64_t dir = -1; - int64_t fd = -1; - const int64_t macro_block_size = OB_SERVER_BLOCK_MGR.get_macro_block_size(); - ObTmpFileIOInfo io_info; - ObTmpFileIOHandle handle; - ret = ObTmpFileManager::get_instance().alloc_dir(dir); - ASSERT_EQ(OB_SUCCESS, ret); - ret = ObTmpFileManager::get_instance().open(fd, dir); - ASSERT_EQ(OB_SUCCESS, ret); - char *write_buf = new char [macro_block_size + 256]; - for (int i = 0; i < macro_block_size + 256; ++i) { - write_buf[i] = static_cast(i % 256); - } - char *read_buf = new char [macro_block_size + 256]; - io_info.fd_ = fd; - io_info.tenant_id_ = 1; - io_info.io_desc_.set_wait_event(2); - io_info.buf_ = write_buf; - io_info.size_ = macro_block_size + 256; - io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; - int64_t write_time = ObTimeUtility::current_time(); - ret = ObTmpFileManager::get_instance().write(io_info); - write_time = ObTimeUtility::current_time() - write_time; - ASSERT_EQ(OB_SUCCESS, ret); - io_info.buf_ = read_buf; - - int64_t read_time = ObTimeUtility::current_time(); - ret = ObTmpFileManager::get_instance().pread(io_info, 0, handle); - read_time = ObTimeUtility::current_time() - read_time; - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(macro_block_size + 256, handle.get_data_size()); - int cmp = memcmp(handle.get_buffer(), write_buf, macro_block_size + 256); - ASSERT_EQ(0, cmp); - - io_info.size_ = 200; - ret = ObTmpFileManager::get_instance().pread(io_info, 200, handle); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(200, handle.get_data_size()); - cmp = memcmp(handle.get_buffer(), write_buf + 200, 200); - ASSERT_EQ(0, cmp); - - io_info.size_ = macro_block_size; - ret = ObTmpFileManager::get_instance().pread(io_info, 200, handle); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(macro_block_size, handle.get_data_size()); - cmp = memcmp(handle.get_buffer(), write_buf + 200, macro_block_size); - ASSERT_EQ(0, cmp); - - io_info.size_ = macro_block_size; - ret = ObTmpFileManager::get_instance().pread(io_info, 400, handle); - ASSERT_EQ(OB_ITER_END, ret); - ASSERT_EQ(macro_block_size + 256 - 400, handle.get_data_size()); - cmp = memcmp(handle.get_buffer(), write_buf + 400, macro_block_size + 256 - 400); - ASSERT_EQ(0, cmp); - - io_info.size_ = 100; - ret = ObTmpFileManager::get_instance().pread(io_info, macro_block_size, handle); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(100, handle.get_data_size()); - cmp = memcmp(handle.get_buffer(), write_buf + macro_block_size, handle.get_data_size()); - ASSERT_EQ(0, cmp); - - io_info.size_ = 100; - ret = ObTmpFileManager::get_instance().pread(io_info, macro_block_size + 10, handle); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(100, handle.get_data_size()); - cmp = memcmp(handle.get_buffer(), write_buf + macro_block_size + 10, handle.get_data_size()); - ASSERT_EQ(0, cmp); - - - io_info.size_ = 200; - ret = ObTmpFileManager::get_instance().pread(io_info, macro_block_size + 100, handle); - ASSERT_EQ(OB_ITER_END, ret); - ASSERT_EQ(156, handle.get_data_size()); - cmp = memcmp(handle.get_buffer(), write_buf + macro_block_size + 100, handle.get_data_size()); - ASSERT_EQ(0, cmp); - - ret = ObTmpFileManager::get_instance().pread(io_info, macro_block_size + 256, handle); - ASSERT_EQ(OB_ITER_END, ret); - - - STORAGE_LOG(INFO, "test_write_more_than_one_macro_block"); - STORAGE_LOG(INFO, "io time", K(write_time), K(read_time)); - ObTmpTenantFileStoreHandle store_handle; - OB_TMP_FILE_STORE.get_store(1, store_handle); - store_handle.get_tenant_store()->print_block_usage(); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(500); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500); - - ObTmpFileManager::get_instance().remove(fd); -} - -TEST_F(TestTmpFile, test_single_dir_two_file) -{ - int ret = OB_SUCCESS; - int64_t dir = -1; - int64_t fd_1 = -1; - int64_t fd_2 = -1; - const int64_t macro_block_size = 64 * 1024; - ObTmpFileIOInfo io_info1; - ObTmpFileIOInfo io_info2; - ObTmpFileIOHandle handle1; - ObTmpFileIOHandle handle2; - ret = ObTmpFileManager::get_instance().alloc_dir(dir); - ASSERT_EQ(OB_SUCCESS, ret); - - char *write_buf = new char [macro_block_size + 256]; - for (int i = 0; i < macro_block_size + 256; ++i) { - write_buf[i] = static_cast(i % 256); - } - char *read_buf = new char [macro_block_size + 256]; - - ret = ObTmpFileManager::get_instance().open(fd_1, dir); - ASSERT_EQ(OB_SUCCESS, ret); - io_info1.fd_ = fd_1; - io_info1.tenant_id_ = 1; - io_info1.io_desc_.set_wait_event(2); - io_info1.buf_ = write_buf; - io_info1.size_ = macro_block_size + 256; - io_info1.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; - - - ret = ObTmpFileManager::get_instance().open(fd_2, dir); - ASSERT_EQ(OB_SUCCESS, ret); - io_info2.fd_ = fd_2; - io_info2.tenant_id_ = 1; - io_info2.io_desc_.set_wait_event(2); - io_info2.buf_ = write_buf; - io_info2.size_ = macro_block_size + 256; - io_info2.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; - - const int64_t timeout_ms = 5000; - int64_t write_time = ObTimeUtility::current_time(); - ret = ObTmpFileManager::get_instance().write(io_info1); - write_time = ObTimeUtility::current_time() - write_time; - ASSERT_EQ(OB_SUCCESS, ret); - - write_time = ObTimeUtility::current_time(); - ret = ObTmpFileManager::get_instance().write(io_info2); - write_time = ObTimeUtility::current_time() - write_time; - ASSERT_EQ(OB_SUCCESS, ret); - - - io_info1.buf_ = read_buf; - int64_t read_time = ObTimeUtility::current_time(); - ret = ObTmpFileManager::get_instance().pread(io_info1, 0, handle1); - read_time = ObTimeUtility::current_time() - read_time; - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(macro_block_size + 256, handle1.get_data_size()); - int cmp = memcmp(handle1.get_buffer(), write_buf, macro_block_size + 256); - ASSERT_EQ(0, cmp); - - - io_info2.buf_ = read_buf; - read_time = ObTimeUtility::current_time(); - ret = ObTmpFileManager::get_instance().pread(io_info2, 0, handle2); - read_time = ObTimeUtility::current_time() - read_time; - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(macro_block_size + 256, handle2.get_data_size()); - cmp = memcmp(handle2.get_buffer(), write_buf, macro_block_size + 256); - ASSERT_EQ(0, cmp); - - ObTmpTenantFileStoreHandle store_handle; - OB_TMP_FILE_STORE.get_store(1, store_handle); - store_handle.get_tenant_store()->print_block_usage(); - ObTmpFileManager::get_instance().remove(fd_1); - ObTmpFileManager::get_instance().remove(fd_2); - OB_TMP_FILE_STORE.get_store(1, store_handle); - store_handle.get_tenant_store()->print_block_usage(); -} - -/*TEST_F(TestTmpFile, test_iter_end) -{ - int old_ret = OB_SUCCESS; - int new_ret = OB_SUCCESS; - int64_t new_dir = -1; - int64_t new_fd = -1; - int old_fd = -1; - const int64_t macro_block_size = OB_SERVER_BLOCK_MGR.get_macro_block_size(); - ObTmpFileIOInfo new_io_info; - ObTmpFileIOHandle new_handle; - ObMacroFileIOInfo old_io_info; - ObMacroFileIOHandle old_handle; - new_ret = ObTmpFileManager::get_instance().alloc_dir(new_dir); - ASSERT_EQ(OB_SUCCESS, new_ret); - new_ret = ObTmpFileManager::get_instance().open(new_fd, new_dir); - ASSERT_EQ(OB_SUCCESS, new_ret); - old_ret = ObMacroFileManager::get_instance().open(old_fd); - ASSERT_EQ(OB_SUCCESS, old_ret); - char *write_buf = new char [macro_block_size + 256]; - for (int i = 0; i < macro_block_size + 256; ++i) { - write_buf[i] = static_cast(i % 256); - } - char *read_buf = new char [macro_block_size + 256]; - - new_io_info.fd_ = new_fd; - new_io_info.tenant_id_ = 1; - new_io_info.io_desc_.set_wait_event(2); - new_io_info.buf_ = write_buf; - new_io_info.size_ = macro_block_size + 256; - - old_io_info.fd_ = old_fd; - old_io_info.tenant_id_ = 1; - old_io_info.io_desc_.set_wait_event(2); - old_io_info.buf_ = write_buf; - old_io_info.size_ = macro_block_size + 256; - - const int64_t timeout_ms = 5000; - int64_t write_time = ObTimeUtility::current_time(); - new_ret = ObTmpFileManager::get_instance().write(new_io_info, timeout_ms); - write_time = ObTimeUtility::current_time() - write_time; - old_ret = ObMacroFileManager::get_instance().write(old_io_info, timeout_ms); - ASSERT_EQ(OB_SUCCESS, new_ret); - ASSERT_EQ(old_ret, new_ret); - - new_io_info.buf_ = read_buf; - old_io_info.buf_ = read_buf; - - - int64_t read_time = ObTimeUtility::current_time(); - new_ret = ObTmpFileManager::get_instance().pread(new_io_info, 0, timeout_ms, new_handle); - read_time = ObTimeUtility::current_time() - read_time; - old_ret = ObMacroFileManager::get_instance().pread(old_io_info, 0,timeout_ms, old_handle); - ASSERT_EQ(OB_SUCCESS, new_ret); - ASSERT_EQ(old_ret, new_ret); - - ASSERT_EQ(macro_block_size + 256, new_handle.get_data_size()); - int cmp = memcmp(new_handle.get_buffer(), write_buf, macro_block_size + 256); - ASSERT_EQ(0, cmp); - new_ret = ObTmpFileManager::get_instance().pread(new_io_info, macro_block_size + 256, timeout_ms, - new_handle); - old_ret = ObMacroFileManager::get_instance().pread(old_io_info, macro_block_size + 256, - timeout_ms, old_handle); - ASSERT_EQ(OB_ITER_END, new_ret); - ASSERT_EQ(OB_ITER_END, old_ret); - ASSERT_EQ(old_ret, new_ret); -}*/ - -TEST_F(TestTmpFile, test_single_dir_multi_file) -{ - int ret = OB_SUCCESS; - const int64_t thread_cnt = 1; - const int64_t file_cnt = 4; - const bool is_plain_data = false; - const bool is_big_file = false; - TestMultiTmpFileStress test(MTL_CTX()); - int64_t dir = -1; - ret = ObTmpFileManager::get_instance().alloc_dir(dir); - ASSERT_EQ(OB_SUCCESS, ret); - ret = test.init(file_cnt, dir, thread_cnt, &table_schema_, is_plain_data, is_big_file); - ASSERT_EQ(OB_SUCCESS, ret); - int64_t io_time = ObTimeUtility::current_time(); - test.start(); - test.wait(); - io_time = ObTimeUtility::current_time() - io_time; - - - STORAGE_LOG(INFO, "test_single_dir_multi_file"); - STORAGE_LOG(INFO, "io time", K(io_time)); - ObTmpTenantFileStoreHandle store_handle; - OB_TMP_FILE_STORE.get_store(1, store_handle); - store_handle.get_tenant_store()->print_block_usage(); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(500); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500); -} - -TEST_F(TestTmpFile, test_drop_tenant_file) -{ - int ret = OB_SUCCESS; - const int64_t thread_cnt = 4; - const int64_t file_cnt = 4; - const bool is_plain_data = false; - const bool is_big_file = true; - TestMultiTmpFileStress test(MTL_CTX()); - int64_t dir = -1; - ret = ObTmpFileManager::get_instance().alloc_dir(dir); - ASSERT_EQ(OB_SUCCESS, ret); - ret = test.init(file_cnt, dir, thread_cnt, &table_schema_, is_plain_data, is_big_file); - ASSERT_EQ(OB_SUCCESS, ret); - test.start(); - test.wait(); - ASSERT_EQ(0, ObTmpFileManager::get_instance().files_.map_.size()); - ASSERT_EQ(1, ObTmpFileStore::get_instance().tenant_file_stores_.size()); - - - ret = ObTmpFileManager::get_instance().remove_tenant_file(1); - ASSERT_EQ(OB_SUCCESS, ret); - - ASSERT_EQ(0, ObTmpFileManager::get_instance().files_.map_.size()); - ASSERT_EQ(0, ObTmpFileStore::get_instance().tenant_file_stores_.size()); - - int64_t fd = 0; - int count = 100; - const int64_t timeout_ms = 5000; - TestTmpFileStress test_write(MTL_CTX()); - TestTmpFileStress test_read(MTL_CTX()); - ret = ObTmpFileManager::get_instance().alloc_dir(dir); - ASSERT_EQ(OB_SUCCESS, ret); - while (count--) { - ret = ObTmpFileManager::get_instance().open(fd, dir); - ASSERT_EQ(OB_SUCCESS, ret); - STORAGE_LOG(INFO, "open file success", K(fd)); - ret = test_write.init(fd, true, 1, &table_schema_, false, false); - ASSERT_EQ(OB_SUCCESS, ret); - ret = test_read.init(fd, false, 1, &table_schema_, false, false); - ASSERT_EQ(OB_SUCCESS, ret); - test_write.start(); - test_write.wait(); - ret = ObTmpFileManager::get_instance().sync(fd, timeout_ms); - ASSERT_EQ(OB_SUCCESS, ret); - test_read.start(); - test_read.wait(); - } - - ASSERT_EQ(100, ObTmpFileManager::get_instance().files_.map_.size()); - ASSERT_EQ(1, ObTmpFileStore::get_instance().tenant_file_stores_.size()); - - - ret = ObTmpFileManager::get_instance().remove_tenant_file(1); - ASSERT_EQ(OB_SUCCESS, ret); - - ASSERT_EQ(0, ObTmpFileManager::get_instance().files_.map_.size()); - ASSERT_EQ(0, ObTmpFileStore::get_instance().tenant_file_stores_.size()); -} - -TEST_F(TestTmpFile, test_handle_double_wait) -{ - int ret = OB_SUCCESS; - int64_t dir = -1; - int64_t fd = -1; - ObTmpFileIOInfo io_info; - ObTmpFileIOHandle handle; - ret = ObTmpFileManager::get_instance().alloc_dir(dir); - ASSERT_EQ(OB_SUCCESS, ret); - ret = ObTmpFileManager::get_instance().open(fd, dir); - ASSERT_EQ(OB_SUCCESS, ret); - char *write_buf = new char [256]; - for (int i = 0; i < 256; ++i) { - write_buf[i] = static_cast(i); - } - char *read_buf = new char [256]; - io_info.fd_ = fd; - io_info.tenant_id_ = 1; - io_info.io_desc_.set_wait_event(2); - io_info.buf_ = write_buf; - io_info.size_ = 256; - io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; - int64_t write_time = ObTimeUtility::current_time(); - ret = ObTmpFileManager::get_instance().write(io_info); - write_time = ObTimeUtility::current_time() - write_time; - ASSERT_EQ(OB_SUCCESS, ret); - io_info.buf_ = read_buf; - - - int64_t read_time = ObTimeUtility::current_time(); - ret = ObTmpFileManager::get_instance().pread(io_info, 0, handle); - read_time = ObTimeUtility::current_time() - read_time; - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(256, handle.get_data_size()); - int cmp = memcmp(handle.get_buffer(), write_buf, 256); - ASSERT_EQ(0, cmp); - - ASSERT_EQ(OB_SUCCESS, handle.wait()); - - STORAGE_LOG(INFO, "test_handle_double_wait"); - STORAGE_LOG(INFO, "io time", K(write_time), K(read_time)); - ObTmpTenantFileStoreHandle store_handle; - OB_TMP_FILE_STORE.get_store(1, store_handle); - store_handle.get_tenant_store()->print_block_usage(); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(500); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500); - - ObTmpFileManager::get_instance().remove(fd); -} - -TEST_F(TestTmpFile, test_sql_workload) -{ - int ret = OB_SUCCESS; - int64_t dir = -1; - int64_t fd = -1; - const int64_t macro_block_size = OB_SERVER_BLOCK_MGR.get_macro_block_size(); - ObTmpFileIOInfo io_info; - ObTmpFileIOHandle handle; - ret = ObTmpFileManager::get_instance().alloc_dir(dir); - ASSERT_EQ(OB_SUCCESS, ret); - ret = ObTmpFileManager::get_instance().open(fd, dir); - ASSERT_EQ(OB_SUCCESS, ret); - const int64_t blk_cnt = 16; - int64_t write_size = macro_block_size * blk_cnt; - char *write_buf = (char *)malloc(write_size); - for (int64_t i = 0; i < write_size; ++i) { - write_buf[i] = static_cast(i % 256); - } - char *read_buf = (char *)malloc(write_size); - - - io_info.fd_ = fd; - io_info.tenant_id_ = 1; - io_info.io_desc_.set_wait_event(2); - io_info.buf_ = write_buf; - io_info.size_ = write_size; - io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; - int64_t write_time = ObTimeUtility::current_time(); - - const int cnt = 1; - const int64_t sql_read_size = 64 * 1024; - const int64_t sql_cnt = write_size / sql_read_size; - - for (int i = 0; i < cnt; i++) { - for (int64_t j = 0; j < sql_cnt; j++) { - io_info.size_ = sql_read_size; - io_info.buf_ = write_buf + j * sql_read_size; - ret = ObTmpFileManager::get_instance().write(io_info); - ASSERT_EQ(OB_SUCCESS, ret); - } - } - write_time = ObTimeUtility::current_time() - write_time; - - - io_info.buf_ = read_buf; - - io_info.size_ = macro_block_size; - ret = ObTmpFileManager::get_instance().pread(io_info, 100, handle); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(macro_block_size, handle.get_data_size()); - int cmp = memcmp(handle.get_buffer(), write_buf + 100, handle.get_data_size()); - ASSERT_EQ(0, cmp); - - - io_info.size_ = write_size; - int64_t read_time = ObTimeUtility::current_time(); - - ret = ObTmpFileManager::get_instance().seek(fd, 0, ObTmpFile::SET_SEEK); - ASSERT_EQ(OB_SUCCESS, ret); - - for (int i = 0; i < cnt; i++) { - for (int64_t j = 0; j < sql_cnt; j++) { - io_info.size_ = sql_read_size; - io_info.buf_ = read_buf + j * sql_read_size; - ret = ObTmpFileManager::get_instance().read(io_info, handle); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(sql_read_size, handle.get_data_size()); - cmp = memcmp(handle.get_buffer(), write_buf + j * sql_read_size, sql_read_size); - ASSERT_EQ(0, cmp); - } - } - read_time = ObTimeUtility::current_time() - read_time; - - io_info.size_ = 200; - ret = ObTmpFileManager::get_instance().pread(io_info, 200, handle); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(200, handle.get_data_size()); - cmp = memcmp(handle.get_buffer(), write_buf + 200, 200); - ASSERT_EQ(0, cmp); - - free(write_buf); - free(read_buf); - - - STORAGE_LOG(INFO, "test_sql_workload"); - STORAGE_LOG(INFO, "io time", K((write_size * cnt) / (1024*1024*1024)), K(write_time), K(read_time)); - ObTmpTenantFileStoreHandle store_handle; - OB_TMP_FILE_STORE.get_store(1, store_handle); - store_handle.get_tenant_store()->print_block_usage(); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(500); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500); - - ObTmpFileManager::get_instance().remove(fd); -} - -TEST_F(TestTmpFile, test_page_buddy) -{ - int ret = OB_SUCCESS; - ObArenaAllocator allocator; - ObTmpFilePageBuddy page_buddy_1; - - ret = page_buddy_1.init(allocator); - ASSERT_EQ(OB_SUCCESS, ret); - - uint8_t page_nums = 64; - uint8_t alloced_page_nums = 64; - uint8_t start_page_id = 255; - ASSERT_EQ(true, page_buddy_1.is_empty()); - ret = page_buddy_1.alloc(page_nums, start_page_id, alloced_page_nums); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(false, page_buddy_1.is_empty()); - - uint8_t start_page_id_2 = 255; - ret = page_buddy_1.alloc(page_nums, start_page_id_2, alloced_page_nums); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(false, page_buddy_1.is_empty()); - - page_buddy_1.free(start_page_id + 63, page_nums -63); - page_buddy_1.free(start_page_id_2 + 1, page_nums - 1); - page_nums = 63; - page_buddy_1.free(start_page_id, page_nums); - page_nums = 1; - page_buddy_1.free(start_page_id_2, page_nums); - STORAGE_LOG(INFO, "page buddy", K(page_buddy_1)); - ASSERT_EQ(true, page_buddy_1.is_empty()); - - ObTmpFilePageBuddy page_buddy_2; - ret = page_buddy_2.init(allocator); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(true, page_buddy_2.is_empty()); - start_page_id = 0; - ret = page_buddy_2.alloc_all_pages(); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(false, page_buddy_2.is_empty()); - - int32_t free_nums = 252 - 129; - page_buddy_2.free(start_page_id + 129, free_nums); - free_nums = 127; - page_buddy_2.free(start_page_id + 2, free_nums); - free_nums = 2; - page_buddy_2.free(start_page_id, free_nums); - STORAGE_LOG(INFO, "page buddy", K(page_buddy_2)); - ASSERT_EQ(true, page_buddy_2.is_empty()); - - for (int32_t i = 1; i < 129; i++) { - ObTmpFilePageBuddy page_buddy_3; - int32_t page_num_2 = i; - ret = page_buddy_3.init(allocator); - ASSERT_EQ(OB_SUCCESS, ret); - ret = page_buddy_3.alloc(page_num_2, start_page_id, alloced_page_nums); - ASSERT_EQ(OB_SUCCESS, ret); - page_buddy_3.free(start_page_id, alloced_page_nums); - STORAGE_LOG(INFO, "page buddy", K(page_buddy_3)); - ASSERT_EQ(true, page_buddy_3.is_empty()); - STORAGE_LOG(INFO, "page buddy", K(page_buddy_3)); - } - - ObTmpFilePageBuddy page_buddy_4; - ret = page_buddy_4.init(allocator); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(true, page_buddy_4.is_empty()); - - page_nums = 2; - alloced_page_nums = -1; - start_page_id = -1; - ASSERT_EQ(true, page_buddy_4.is_empty()); - ret = page_buddy_4.alloc(page_nums, start_page_id, alloced_page_nums); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(alloced_page_nums, page_nums); - ASSERT_EQ(false, page_buddy_4.is_empty()); -} - -TEST_F(TestTmpFile, test_page_io_info_unrelease) -{ - int ret = OB_SUCCESS; - - ObTmpTenantFileStoreHandle store_handle; - OB_TMP_FILE_STORE.get_store(1, store_handle); - common::ObFIFOAllocator *fifo_allocator = &(store_handle.get_tenant_store()->io_allocator_); - - // case 1: construct callback - { - int64_t begin_used = fifo_allocator->used(); - { - ObTmpPageCache::ObTmpMultiPageIOCallback callback; - callback.allocator_ = fifo_allocator; - callback.cache_ = &(ObTmpPageCache::get_instance()); - callback.page_io_infos_.assign(common::ObSEArray()); - - int64_t final_used = fifo_allocator->used(); - ASSERT_EQ(final_used, begin_used); - } - ASSERT_EQ(begin_used, fifo_allocator->used()); - } - - // case 2: never call alloc_data_buf - { - int64_t begin_used = fifo_allocator->used(); - { - ObTmpPageCache::ObTmpMultiPageIOCallback callback; - callback.allocator_ = fifo_allocator; - callback.cache_ = &(ObTmpPageCache::get_instance()); - callback.page_io_infos_.assign(common::ObSEArray()); - - int64_t tmp_buf_size = 4096; - char *tmp_buf = static_cast(fifo_allocator->alloc(tmp_buf_size)); - ASSERT_EQ(callback.alloc_data_buf(tmp_buf, tmp_buf_size), OB_SUCCESS); - fifo_allocator->free(tmp_buf); - int64_t after_alloc_io_buf_used = fifo_allocator->used(); - ASSERT_EQ(after_alloc_io_buf_used, begin_used + tmp_buf_size); - - int64_t after_process_pos = fifo_allocator->used(); - ASSERT_EQ(after_process_pos, begin_used + tmp_buf_size); - ASSERT_EQ(after_alloc_io_buf_used, after_process_pos); - } - ASSERT_EQ(begin_used, fifo_allocator->used()); - } - - // case 3: call inner_process - { - int64_t begin_used = fifo_allocator->used(); - { - ObTmpPageCache::ObTmpMultiPageIOCallback callback; - callback.allocator_ = fifo_allocator; - callback.cache_ = &(ObTmpPageCache::get_instance()); - callback.page_io_infos_.assign(common::ObSEArray()); - - int64_t tmp_buf_size = 4096; - char *tmp_buf = static_cast(fifo_allocator->alloc(tmp_buf_size)); - ASSERT_EQ(callback.inner_process(tmp_buf, tmp_buf_size), OB_SUCCESS); - int64_t after_process_used = fifo_allocator->used(); - ASSERT_EQ(after_process_used, begin_used + tmp_buf_size * 2); - fifo_allocator->free(tmp_buf); - int64_t final_used = fifo_allocator->used(); - ASSERT_EQ(final_used, begin_used + tmp_buf_size); - } - ASSERT_EQ(begin_used, fifo_allocator->used()); - } -} - -TEST_F(TestTmpFile, test_tmp_file_sync) -{ - int ret = OB_SUCCESS; - int64_t dir = -1; - int64_t fd = -1; - ObTmpFileIOInfo io_info; - ObTmpFileIOHandle handle; - ret = ObTmpFileManager::get_instance().alloc_dir(dir); - ASSERT_EQ(OB_SUCCESS, ret); - ret = ObTmpFileManager::get_instance().open(fd, dir); - ASSERT_EQ(OB_SUCCESS, ret); - int64_t write_size = 16*1024; - char *write_buf = (char *)malloc(write_size); - for (int64_t i = 0; i < write_size; ++i) { - write_buf[i] = static_cast(i % 256); - } - io_info.fd_ = fd; - io_info.tenant_id_ = 1; - io_info.io_desc_.set_resource_group_id(THIS_WORKER.get_group_id()); - io_info.io_desc_.set_wait_event(2); - io_info.buf_ = write_buf; - io_info.size_ = write_size; - io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; - int64_t write_time = ObTimeUtility::current_time(); - ret = ObTmpFileManager::get_instance().write(io_info); - write_time = ObTimeUtility::current_time() - write_time; - ASSERT_EQ(OB_SUCCESS, ret); - free(write_buf); - - STORAGE_LOG(INFO, "test_tmp_file_sync"); - STORAGE_LOG(INFO, "io time", K(write_time)); - ObTmpTenantFileStoreHandle store_handle; - OB_TMP_FILE_STORE.get_store(1, store_handle); - ASSERT_EQ(1, store_handle.get_tenant_store()->tmp_mem_block_manager_.t_mblk_map_.size()); - ObTmpFileManager::get_instance().sync(fd, 5000); - ASSERT_EQ(0, store_handle.get_tenant_store()->tmp_mem_block_manager_.t_mblk_map_.size()); - - store_handle.get_tenant_store()->print_block_usage(); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(500); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500); - - ObTmpFileManager::get_instance().remove(fd); -} - -TEST_F(TestTmpFile, test_tmp_file_sync_same_block) -{ - int ret = OB_SUCCESS; - int64_t dir = -1; - int64_t fd1, fd2 = -1; - const int64_t timeout_ms = 5000; - ObTmpFileIOHandle handle; - ObTmpFileIOInfo io_info; - io_info.tenant_id_ = 1; - io_info.io_desc_.set_resource_group_id(THIS_WORKER.get_group_id()); - io_info.io_desc_.set_wait_event(2); - int64_t write_size = 16 *1024; - char *write_buf = (char *)malloc(write_size); - for (int64_t i = 0; i < write_size; ++i) { - write_buf[i] = static_cast(i % 256); - } - io_info.buf_ = write_buf; - io_info.size_ = write_size; - io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; - - ret = ObTmpFileManager::get_instance().alloc_dir(dir); - ASSERT_EQ(OB_SUCCESS, ret); - - ret = ObTmpFileManager::get_instance().open(fd1, dir); - ASSERT_EQ(OB_SUCCESS, ret); - io_info.fd_ = fd1; - int64_t write_time = ObTimeUtility::current_time(); - ret = ObTmpFileManager::get_instance().write(io_info); - write_time = ObTimeUtility::current_time() - write_time; - ASSERT_EQ(OB_SUCCESS, ret); - - ret = ObTmpFileManager::get_instance().open(fd2, dir); - ASSERT_EQ(OB_SUCCESS, ret); - io_info.fd_ = fd2; - write_time = ObTimeUtility::current_time(); - ret = ObTmpFileManager::get_instance().write(io_info); - write_time = ObTimeUtility::current_time() - write_time; - ASSERT_EQ(OB_SUCCESS, ret); - - free(write_buf); - - STORAGE_LOG(INFO, "test_tmp_file_sync_same_block"); - STORAGE_LOG(INFO, "io time", K(write_time)); - ObTmpTenantFileStoreHandle store_handle; - OB_TMP_FILE_STORE.get_store(1, store_handle); - ASSERT_EQ(1, store_handle.get_tenant_store()->tmp_mem_block_manager_.t_mblk_map_.size()); - ObTmpFileManager::get_instance().sync(fd1, 5000); - ASSERT_EQ(1, store_handle.get_tenant_store()->tmp_mem_block_manager_.t_mblk_map_.size()); - ObTmpFileManager::get_instance().sync(fd2, 5000); - ASSERT_EQ(0, store_handle.get_tenant_store()->tmp_mem_block_manager_.t_mblk_map_.size()); - - store_handle.get_tenant_store()->print_block_usage(); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(500); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500); - - ObTmpFileManager::get_instance().remove(fd1); - ObTmpFileManager::get_instance().remove(fd2); -} - -TEST_F(TestTmpFile, test_tmp_file_wash) -{ - int ret = OB_SUCCESS; - const int64_t timeout_ms = 5000; - int count = 64 * 0.8; - int64_t dir = -1; - int64_t fd = -1; - ObTmpFileIOHandle handle; - ObTmpFileIOInfo io_info, io_info_2; - io_info.tenant_id_ = 1; - io_info.io_desc_.set_resource_group_id(THIS_WORKER.get_group_id()); - io_info.io_desc_.set_wait_event(2); - int64_t write_size = 1024 *1024; - char *write_buf = (char *)malloc(write_size); - for (int64_t i = 0; i < write_size; ++i) { - write_buf[i] = static_cast(i % 256); - } - io_info.buf_ = write_buf; - io_info.size_ = write_size; - io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; - - io_info_2 = io_info; - - int64_t write_size_2 = 2016 *1024; - char *write_buf_2 = (char *)malloc(write_size_2); - for (int64_t i = 0; i < write_size_2; ++i) { - write_buf_2[i] = static_cast(i % 256); - } - io_info_2.buf_ = write_buf_2; - io_info_2.size_ = write_size_2; - io_info_2.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; - - STORAGE_LOG(INFO, "test_tmp_file_wash"); - ObTmpTenantFileStoreHandle store_handle; - OB_TMP_FILE_STORE.get_store(1, store_handle); - - for (int64_t i=0; itmp_mem_block_manager_.t_mblk_map_.begin(); - iter != store_handle.get_tenant_store()->tmp_mem_block_manager_.t_mblk_map_.end(); ++iter) { - int64_t alloc_time = iter->second->get_alloc_time(); - if (alloc_time < oldest_time) { - oldest_id = iter->first; - oldest_time = alloc_time; - } - if (alloc_time > newest_time) { - newest_id = iter->first; - newest_time = alloc_time; - } - } - ObTmpMacroBlock* wash_block; - ret = store_handle.get_tenant_store()->tmp_mem_block_manager_.t_mblk_map_.get_refactored(newest_id, wash_block); - ASSERT_EQ(OB_SUCCESS, ret); - wash_block->alloc_time_ = wash_block->alloc_time_ - 60 * 1000000L; - - ObArray free_blocks; - // 1 macro block has been disked immediately because its memory has been exhausted. - ASSERT_EQ(count-1, store_handle.get_tenant_store()->tmp_mem_block_manager_.t_mblk_map_.size()); - - for (int64_t i=0; i< 3; i++) { - ret = ObTmpFileManager::get_instance().alloc_dir(dir); - ASSERT_EQ(OB_SUCCESS, ret); - ret = ObTmpFileManager::get_instance().open(fd, dir); - ASSERT_EQ(OB_SUCCESS, ret); - io_info.fd_ = fd; - ret = ObTmpFileManager::get_instance().write(io_info); - ASSERT_EQ(OB_SUCCESS, ret); - } - - store_handle.get_tenant_store()->tmp_mem_block_manager_.cleanup(); - - std::chrono::milliseconds(50); - ret = store_handle.get_tenant_store()->tmp_mem_block_manager_.wait_write_finish(oldest_id, ObTmpTenantMemBlockManager::get_default_timeout_ms()); - ASSERT_EQ(OB_SUCCESS, ret); - ret = store_handle.get_tenant_store()->tmp_mem_block_manager_.wait_write_finish(newest_id, ObTmpTenantMemBlockManager::get_default_timeout_ms()); - ASSERT_EQ(OB_SUCCESS, ret); - - ret = store_handle.get_tenant_store()->tmp_mem_block_manager_.t_mblk_map_.get_refactored(oldest_id, wash_block); - ASSERT_NE(OB_SUCCESS, ret); - ret = store_handle.get_tenant_store()->tmp_mem_block_manager_.t_mblk_map_.get_refactored(newest_id, wash_block); - ASSERT_NE(OB_SUCCESS, ret); - ASSERT_EQ(count, store_handle.get_tenant_store()->tmp_mem_block_manager_.t_mblk_map_.size()); - - free(write_buf); - free(write_buf_2); - - store_handle.get_tenant_store()->print_block_usage(); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(500); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500); - - count = 64 * 0.8 + 3; - while (count--) { - ret = ObTmpFileManager::get_instance().remove(count); - ASSERT_EQ(OB_SUCCESS, ret); - } -} - -// test truncate, simple thread and multi thread -TEST_F(TestTmpFile, test_tmp_file_truncate) -{ - int ret = OB_SUCCESS; - int64_t dir = -1; - int64_t fd = -1; - const int64_t timeout_ms = 5000; - ObTmpFileIOHandle handle; - ObTmpFileIOInfo io_info; - const int64_t macro_block_size = OB_SERVER_BLOCK_MGR.get_macro_block_size(); - - ret = ObTmpFileManager::get_instance().alloc_dir(dir); - ASSERT_EQ(OB_SUCCESS, ret); - - ret = ObTmpFileManager::get_instance().open(fd, dir); - ASSERT_EQ(OB_SUCCESS, ret); - char *write_buf = new char [macro_block_size + 256]; - for (int i = 0; i < macro_block_size + 256; ++i) { - write_buf[i] = static_cast(i % 256); - } - char *read_buf = new char [macro_block_size + 256]; - io_info.fd_ = fd; - io_info.tenant_id_ = 1; - io_info.io_desc_.set_wait_event(2); - io_info.buf_ = write_buf; - io_info.size_ = macro_block_size + 256; - io_info.io_desc_.set_resource_group_id(THIS_WORKER.get_group_id()); - - int64_t write_time = ObTimeUtility::current_time(); - ret = ObTmpFileManager::get_instance().write(io_info); - write_time = ObTimeUtility::current_time() - write_time; - ASSERT_EQ(OB_SUCCESS, ret); - io_info.buf_ = read_buf; - - int64_t read_time = ObTimeUtility::current_time(); - ret = ObTmpFileManager::get_instance().read(io_info, handle); - read_time = ObTimeUtility::current_time() - read_time; - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(macro_block_size + 256, handle.get_data_size()); - int cmp = memcmp(handle.get_buffer(), write_buf, macro_block_size + 256); - ASSERT_EQ(0, cmp); - - - ret = ObTmpFileManager::get_instance().seek(fd, 0, ObTmpFile::SET_SEEK); - ASSERT_EQ(OB_SUCCESS, ret); - io_info.size_ = 200; - ret = ObTmpFileManager::get_instance().read(io_info, handle); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(200, handle.get_data_size()); - cmp = memcmp(handle.get_buffer(), write_buf, 200); - ASSERT_EQ(0, cmp); - - io_info.size_ = 200; - ret = ObTmpFileManager::get_instance().seek(fd, 0, ObTmpFile::SET_SEEK); - ASSERT_EQ(OB_SUCCESS, ret); - ret = ObTmpFileManager::get_instance().truncate(fd, 100); - ASSERT_EQ(OB_SUCCESS, ret); - ret = ObTmpFileManager::get_instance().read(io_info, handle); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(200, handle.get_data_size()); - MEMSET(write_buf, 0, 100); - cmp = memcmp(handle.get_buffer(), write_buf, 200); - ASSERT_EQ(0, cmp); - - io_info.size_ = 200; - ret = ObTmpFileManager::get_instance().truncate(fd, 300); - ASSERT_EQ(OB_SUCCESS, ret); - ret = ObTmpFileManager::get_instance().read(io_info, handle); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(200, handle.get_data_size()); - MEMSET(write_buf + 100, 0, 200); - cmp = memcmp(handle.get_buffer(), write_buf + 200, 200); - ASSERT_EQ(0, cmp); - - free(write_buf); - free(read_buf); - - STORAGE_LOG(INFO, "test_tmp_file_truncate"); - STORAGE_LOG(INFO, "io time", K(write_time), K(read_time)); - ObTmpTenantFileStoreHandle store_handle; - OB_TMP_FILE_STORE.get_store(1, store_handle); - store_handle.get_tenant_store()->print_block_usage(); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(500); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500); - ObTmpFileManager::get_instance().remove(fd); -} - -TEST_F(TestTmpFile, test_multi_thread_truncate) -{ - int ret = OB_SUCCESS; - const int64_t thread_cnt = 4; - const int64_t file_cnt = 1; - const bool is_plain_data = false; - const bool is_big_file = true; - const bool is_truncate = true; - TestMultiTmpFileStress test(MTL_CTX()); - int64_t dir = -1; - ret = ObTmpFileManager::get_instance().alloc_dir(dir); - ASSERT_EQ(OB_SUCCESS, ret); - ret = test.init(file_cnt, dir, thread_cnt, &table_schema_, is_plain_data, is_big_file, is_truncate); - ASSERT_EQ(OB_SUCCESS, ret); - int64_t io_time = ObTimeUtility::current_time(); - test.start(); - test.wait(); - io_time = ObTimeUtility::current_time() - io_time; - - - STORAGE_LOG(INFO, "test_multi_thread_truncate"); - STORAGE_LOG(INFO, "io time", K(io_time)); - ObTmpTenantFileStoreHandle store_handle; - OB_TMP_FILE_STORE.get_store(1, store_handle); - store_handle.get_tenant_store()->print_block_usage(); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(500); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500); -} - -TEST_F(TestTmpFile, test_truncate_free_block) { - int ret = OB_SUCCESS; - int count = 32; - int64_t dir = -1; - int64_t fd = -1; - ObTmpFileIOHandle handle; - ObTmpFileIOInfo io_info; - io_info.tenant_id_ = 1; - io_info.io_desc_.set_resource_group_id(THIS_WORKER.get_group_id()); - io_info.io_desc_.set_wait_event(2); - //int64_t write_size = OB_SERVER_BLOCK_MGR.get_macro_block_size(); - int64_t write_size = 1024 * 1024; - - char *write_buf = (char *)malloc(write_size); - for (int64_t i = 0; i < write_size; ++i) { - write_buf[i] = static_cast(i % 256); - } - ret = ObTmpFileManager::get_instance().alloc_dir(dir); - ASSERT_EQ(OB_SUCCESS, ret); - ret = ObTmpFileManager::get_instance().open(fd, dir); - ASSERT_EQ(OB_SUCCESS, ret); - io_info.fd_ = fd; - io_info.buf_ = write_buf; - io_info.size_ = write_size; - - for (int64_t i = 0; i < count; i++) { - ret = ObTmpFileManager::get_instance().write(io_info); - ASSERT_EQ(OB_SUCCESS, ret); - } - - STORAGE_LOG(INFO, "test_truncate_free_block"); - ObTmpTenantFileStoreHandle store_handle; - OB_TMP_FILE_STORE.get_store(1, store_handle); - ASSERT_EQ(count, store_handle.get_tenant_store()->tmp_mem_block_manager_.t_mblk_map_.size()); - - for (int64_t i = 0; i < count; i++) { - ret = ObTmpFileManager::get_instance().truncate(fd, (i + 1) * write_size); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(count - i - 1 , store_handle.get_tenant_store()->tmp_mem_block_manager_.t_mblk_map_.size()); - } - - ret = ObTmpFileManager::get_instance().truncate(fd, 0); - ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(0 , store_handle.get_tenant_store()->tmp_mem_block_manager_.t_mblk_map_.size()); - int64_t read_size = write_size; - char *read_buf = (char *)malloc(read_size); - memset(write_buf, 0, write_size); - io_info.buf_ = read_buf; - ret = ObTmpFileManager::get_instance().read(io_info, handle); - int cmp = memcmp(read_buf, write_buf, read_size); - ASSERT_EQ(0, cmp); - free(write_buf); - free(read_buf); - - store_handle.get_tenant_store()->print_block_usage(); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(1); - ObMallocAllocator::get_instance()->print_tenant_memory_usage(500); - ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500); - - ObTmpFileManager::get_instance().remove(fd); - -} - - -} // end namespace unittest -} // end namespace oceanbase - -void sig_49_handler(int signo) -{ - // do nothing. -} - -int main(int argc, char **argv) -{ - struct sigaction sa; - sa.sa_handler = sig_49_handler; - sa.sa_flags = 0; - sigemptyset(&sa.sa_mask); - - // catch 49 signal and do nothing. - if (sigaction(49, &sa, NULL) == -1) { - perror("sigaction"); - return 1; - } - - system("rm -f test_tmp_file.log*"); - oceanbase::common::ObLogger::get_logger().set_log_level("INFO"); - OB_LOGGER.set_file_name("test_tmp_file.log", true, true); - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/unittest/storage/ddl/test_chunk_compact_store.cpp b/unittest/storage/ddl/test_chunk_compact_store.cpp index c809c9289..164f4bf22 100644 --- a/unittest/storage/ddl/test_chunk_compact_store.cpp +++ b/unittest/storage/ddl/test_chunk_compact_store.cpp @@ -16,6 +16,7 @@ #include #define private public +#include "share/ob_tenant_mgr.h" #include "storage/meta_mem/ob_tenant_meta_mem_mgr.h" #include "storage/blocksstable/ob_row_generate.h" #include "storage/blocksstable/ob_data_file_prepare.h" @@ -23,7 +24,7 @@ #include "unittest/storage/blocksstable/ob_data_file_prepare.h" #include "src/sql/engine/basic/chunk_store/ob_compact_store.h" #include "src/sql/engine/basic/ob_temp_block_store.h" - +#include "mtlenv/mock_tenant_module_env.h" #undef private namespace oceanbase @@ -162,6 +163,25 @@ public: void SetUp(); void TearDown(); + int init_tenant_mgr() + { + int ret = OB_SUCCESS; + ObAddr self; + obrpc::ObSrvRpcProxy rpc_proxy; + obrpc::ObCommonRpcProxy rs_rpc_proxy; + share::ObRsMgr rs_mgr; + self.set_ip_addr("127.0.0.1", 8086); + rpc::frame::ObReqTransport req_transport(NULL, NULL); + const int64_t ulmt = 128LL << 30; + const int64_t llmt = 128LL << 30; + ret = getter.add_tenant(OB_SYS_TENANT_ID, ulmt, llmt); + EXPECT_EQ(OB_SUCCESS, ret); + ret = getter.add_tenant(OB_SERVER_TENANT_ID, ulmt, llmt); + EXPECT_EQ(OB_SUCCESS, ret); + lib::set_memory_limit(128LL << 32); + return ret; + } + protected: ObStoredRowGenerate row_generate_; ObArenaAllocator allocator_; @@ -184,8 +204,12 @@ void TestCompactChunk::SetUp() } // set observer memory limit CHUNK_MGR.set_limit(8L * 1024L * 1024L * 1024L); - ret = ObTmpFileManager::get_instance().init(); - ASSERT_EQ(OB_SUCCESS, ret); + + EXPECT_EQ(OB_SUCCESS, init_tenant_mgr()); + ASSERT_EQ(OB_SUCCESS, common::ObClockGenerator::init()); + ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpBlockCache::get_instance().init("tmp_block_cache", 1)); + ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpPageCache::get_instance().init("tmp_page_cache", 1)); + static ObTenantBase tenant_ctx(OB_SYS_TENANT_ID); ObTenantEnv::set_tenant(&tenant_ctx); ObTenantIOManager *io_service = nullptr; @@ -193,17 +217,27 @@ void TestCompactChunk::SetUp() EXPECT_EQ(OB_SUCCESS, ObTenantIOManager::mtl_init(io_service)); EXPECT_EQ(OB_SUCCESS, io_service->start()); tenant_ctx.set(io_service); + + tmp_file::ObTenantTmpFileManager *tf_mgr = nullptr; + EXPECT_EQ(OB_SUCCESS, mtl_new_default(tf_mgr)); + EXPECT_EQ(OB_SUCCESS, tmp_file::ObTenantTmpFileManager::mtl_init(tf_mgr)); + tf_mgr->page_cache_controller_.write_buffer_pool_.default_wbp_memory_limit_ = 40*1024*1024; + EXPECT_EQ(OB_SUCCESS, tf_mgr->start()); + tenant_ctx.set(tf_mgr); + ObTenantEnv::set_tenant(&tenant_ctx); } void TestCompactChunk::TearDown() { - ObTmpFileManager::get_instance().destroy(); ObKVGlobalCache::get_instance().destroy(); - ObTmpFileStore::get_instance().destroy(); allocator_.reuse(); row_generate_.allocator_.reuse(); TestDataFilePrepare::TearDown(); + + tmp_file::ObTmpBlockCache::get_instance().destroy(); + tmp_file::ObTmpPageCache::get_instance().destroy(); + common::ObClockGenerator::destroy(); } TEST_F(TestCompactChunk, test_read_writer_compact) diff --git a/unittest/storage/direct_load/test_direct_load_data_block_writer.cpp b/unittest/storage/direct_load/test_direct_load_data_block_writer.cpp index 4780a7595..a8b62475f 100644 --- a/unittest/storage/direct_load/test_direct_load_data_block_writer.cpp +++ b/unittest/storage/direct_load/test_direct_load_data_block_writer.cpp @@ -15,15 +15,16 @@ #define protected public #include #include +#include "share/ob_tenant_mgr.h" #include "../unittest/storage/blocksstable/ob_data_file_prepare.h" #include "../unittest/storage/blocksstable/ob_row_generate.h" #include "observer/table_load/ob_table_load_partition_location.h" #include "share/ob_simple_mem_limit_getter.h" #include "share/table/ob_table_load_define.h" -#include "storage/blocksstable/ob_tmp_file.h" #include "storage/direct_load/ob_direct_load_sstable_scanner.h" #include "storage/direct_load/ob_direct_load_sstable_compactor.h" #include "storage/ob_i_store.h" +#include "mtlenv/mock_tenant_module_env.h" namespace oceanbase { @@ -50,11 +51,29 @@ public: static const int64_t SNAPSHOT_VERSION = 2; public: - TestDataBlockWriter() : TestDataFilePrepare(&getter, "TestDataBlockWriter", 8 * 1024 * 1024, 2048){}; + TestDataBlockWriter() : TestDataFilePrepare(&getter, "TestDataBlockWriter", 2 * 1024 * 1024, 2048){}; virtual void SetUp(); virtual void TearDown(); void check_row(const ObDatumRow *next_row, const ObDatumRow *curr_row); void test_alloc(char *&ptr, const int64_t size); + int init_tenant_mgr() + { + int ret = OB_SUCCESS; + ObAddr self; + obrpc::ObSrvRpcProxy rpc_proxy; + obrpc::ObCommonRpcProxy rs_rpc_proxy; + share::ObRsMgr rs_mgr; + self.set_ip_addr("127.0.0.1", 8086); + rpc::frame::ObReqTransport req_transport(NULL, NULL); + const int64_t ulmt = 128LL << 30; + const int64_t llmt = 128LL << 30; + ret = getter.add_tenant(OB_SYS_TENANT_ID, ulmt, llmt); + EXPECT_EQ(OB_SUCCESS, ret); + ret = getter.add_tenant(OB_SERVER_TENANT_ID, ulmt, llmt); + EXPECT_EQ(OB_SUCCESS, ret); + lib::set_memory_limit(128LL << 32); + return ret; + } private: void prepare_schema(); @@ -124,7 +143,6 @@ void TestDataBlockWriter::prepare_schema() column.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_GENERAL_CI); ASSERT_EQ(OB_SUCCESS, table_schema_.add_column(column)); } - ObTmpFileManager::get_instance().destroy(); } void TestDataBlockWriter::SetUp() @@ -165,8 +183,11 @@ void TestDataBlockWriter::SetUp() } // set observer memory limit CHUNK_MGR.set_limit(8L * 1024L * 1024L * 1024L); - ret = ObTmpFileManager::get_instance().init(); - ASSERT_EQ(OB_SUCCESS, ret); + EXPECT_EQ(OB_SUCCESS, init_tenant_mgr()); + ASSERT_EQ(OB_SUCCESS, common::ObClockGenerator::init()); + ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpBlockCache::get_instance().init("tmp_block_cache", 1)); + ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpPageCache::get_instance().init("tmp_page_cache", 1)); + static ObTenantBase tenant_ctx(OB_SYS_TENANT_ID); ObTenantEnv::set_tenant(&tenant_ctx); @@ -175,13 +196,23 @@ void TestDataBlockWriter::SetUp() EXPECT_EQ(OB_SUCCESS, ObTenantIOManager::mtl_init(io_service)); EXPECT_EQ(OB_SUCCESS, io_service->start()); tenant_ctx.set(io_service); + + tmp_file::ObTenantTmpFileManager *tf_mgr = nullptr; + EXPECT_EQ(OB_SUCCESS, mtl_new_default(tf_mgr)); + EXPECT_EQ(OB_SUCCESS, tmp_file::ObTenantTmpFileManager::mtl_init(tf_mgr)); + tf_mgr->page_cache_controller_.write_buffer_pool_.default_wbp_memory_limit_ = 40*1024*1024; + EXPECT_EQ(OB_SUCCESS, tf_mgr->start()); + tenant_ctx.set(tf_mgr); + ObTenantEnv::set_tenant(&tenant_ctx); } void TestDataBlockWriter::TearDown() { file_mgr_->~ObDirectLoadTmpFileManager(); - ObTmpFileManager::get_instance().destroy(); + tmp_file::ObTmpBlockCache::get_instance().destroy(); + tmp_file::ObTmpPageCache::get_instance().destroy(); + common::ObClockGenerator::destroy(); ObKVGlobalCache::get_instance().destroy(); TestDataFilePrepare::TearDown(); } diff --git a/unittest/storage/direct_load/test_direct_load_index_block_writer.cpp b/unittest/storage/direct_load/test_direct_load_index_block_writer.cpp index 5291cee34..d255ee513 100644 --- a/unittest/storage/direct_load/test_direct_load_index_block_writer.cpp +++ b/unittest/storage/direct_load/test_direct_load_index_block_writer.cpp @@ -15,14 +15,15 @@ #define protected public #include #include +#include "share/ob_tenant_mgr.h" #include "../unittest/storage/blocksstable/ob_data_file_prepare.h" #include "../unittest/storage/blocksstable/ob_row_generate.h" #include "observer/table_load/ob_table_load_partition_location.h" #include "share/ob_simple_mem_limit_getter.h" -#include "storage/blocksstable/ob_tmp_file.h" #include "storage/direct_load/ob_direct_load_tmp_file.h" #include "storage/direct_load/ob_direct_load_sstable_scanner.h" #include "storage/ob_i_store.h" +#include "mtlenv/mock_tenant_module_env.h" namespace oceanbase { using namespace common; @@ -53,6 +54,25 @@ public: static void TearDownTestCase() {} void test_alloc(char *&ptr, const int64_t size); + int init_tenant_mgr() + { + int ret = OB_SUCCESS; + ObAddr self; + obrpc::ObSrvRpcProxy rpc_proxy; + obrpc::ObCommonRpcProxy rs_rpc_proxy; + share::ObRsMgr rs_mgr; + self.set_ip_addr("127.0.0.1", 8086); + rpc::frame::ObReqTransport req_transport(NULL, NULL); + const int64_t ulmt = 128LL << 30; + const int64_t llmt = 128LL << 30; + ret = getter.add_tenant(OB_SYS_TENANT_ID, ulmt, llmt); + EXPECT_EQ(OB_SUCCESS, ret); + ret = getter.add_tenant(OB_SERVER_TENANT_ID, ulmt, llmt); + EXPECT_EQ(OB_SUCCESS, ret); + lib::set_memory_limit(128LL << 32); + return ret; + } + private: void prepare_schema(); @@ -106,7 +126,6 @@ void TestIndexBlockWriter::prepare_schema() column.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_GENERAL_CI); ASSERT_EQ(OB_SUCCESS, table_schema_.add_column(column)); } - ObTmpFileManager::get_instance().destroy(); } void TestIndexBlockWriter::SetUp() @@ -134,8 +153,11 @@ void TestIndexBlockWriter::SetUp() } // set observer memory limit CHUNK_MGR.set_limit(8L * 1024L * 1024L * 1024L); - ret = ObTmpFileManager::get_instance().init(); - ASSERT_EQ(OB_SUCCESS, ret); + EXPECT_EQ(OB_SUCCESS, init_tenant_mgr()); + ASSERT_EQ(OB_SUCCESS, common::ObClockGenerator::init()); + ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpBlockCache::get_instance().init("tmp_block_cache", 1)); + ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpPageCache::get_instance().init("tmp_page_cache", 1)); + static ObTenantBase tenant_ctx(OB_SYS_TENANT_ID); ObTenantEnv::set_tenant(&tenant_ctx); ObTenantIOManager *io_service = nullptr; @@ -143,6 +165,14 @@ void TestIndexBlockWriter::SetUp() EXPECT_EQ(OB_SUCCESS, ObTenantIOManager::mtl_init(io_service)); EXPECT_EQ(OB_SUCCESS, io_service->start()); tenant_ctx.set(io_service); + + tmp_file::ObTenantTmpFileManager *tf_mgr = nullptr; + EXPECT_EQ(OB_SUCCESS, mtl_new_default(tf_mgr)); + EXPECT_EQ(OB_SUCCESS, tmp_file::ObTenantTmpFileManager::mtl_init(tf_mgr)); + tf_mgr->page_cache_controller_.write_buffer_pool_.default_wbp_memory_limit_ = 40*1024*1024; + EXPECT_EQ(OB_SUCCESS, tf_mgr->start()); + tenant_ctx.set(tf_mgr); + ObTenantEnv::set_tenant(&tenant_ctx); } @@ -150,9 +180,11 @@ void TestIndexBlockWriter::TearDown() { file_mgr_->~ObDirectLoadTmpFileManager(); table_schema_.reset(); - ObTmpFileManager::get_instance().destroy(); ObKVGlobalCache::get_instance().destroy(); TestDataFilePrepare::TearDown(); + tmp_file::ObTmpBlockCache::get_instance().destroy(); + tmp_file::ObTmpPageCache::get_instance().destroy(); + common::ObClockGenerator::destroy(); } TEST_F(TestIndexBlockWriter, test_write_and_read) diff --git a/unittest/storage/test_io_manager.cpp b/unittest/storage/test_io_manager.cpp index cf70dac6d..a54c7a72a 100644 --- a/unittest/storage/test_io_manager.cpp +++ b/unittest/storage/test_io_manager.cpp @@ -27,8 +27,7 @@ #include "lib/file/file_directory_utils.h" #include "common/ob_clock_generator.h" #include "storage/blocksstable/ob_micro_block_cache.h" -#include "storage/blocksstable/ob_tmp_file_cache.h" -#include "storage/blocksstable/ob_tmp_file_store.h" +#include "storage/tmp_file/ob_tmp_file_cache.h" #include "storage/meta_mem/ob_storage_meta_cache.h" #define ASSERT_SUCC(ret) ASSERT_EQ((ret), ::oceanbase::common::OB_SUCCESS) @@ -38,6 +37,7 @@ using namespace oceanbase::lib; using namespace oceanbase::common; using namespace oceanbase::share; using namespace oceanbase::blocksstable; +using namespace oceanbase::tmp_file; #define TEST_ROOT_DIR "./" #define TEST_DATA_DIR TEST_ROOT_DIR "/data_dir" @@ -615,8 +615,8 @@ TEST_F(TestIOStruct, Test_Size) int64_t size1 = sizeof(ObAsyncSingleMicroBlockIOCallback); int64_t size2 = sizeof(ObMultiDataBlockIOCallback); int64_t size3 = sizeof(ObSyncSingleMicroBLockIOCallback); - int64_t size4 = sizeof(ObTmpPageCache::ObTmpPageIOCallback); - int64_t size5 = sizeof(ObTmpPageCache::ObTmpMultiPageIOCallback); + int64_t size4 = sizeof(ObTmpPageCache::ObTmpCachedReadPageIOCallback); + int64_t size5 = sizeof(ObTmpPageCache::ObTmpDirectReadPageIOCallback); int64_t size6 = sizeof(oceanbase::ObStorageMetaCache::ObStorageMetaIOCallback); int64_t max_callback_size = std::max({size1, size2, size3, size4, size5, size6}); int64_t size_request = sizeof(ObIORequest); diff --git a/unittest/storage/test_parallel_external_sort.cpp b/unittest/storage/test_parallel_external_sort.cpp index 160d04172..e5e06cdd7 100644 --- a/unittest/storage/test_parallel_external_sort.cpp +++ b/unittest/storage/test_parallel_external_sort.cpp @@ -24,6 +24,7 @@ #include "share/ob_srv_rpc_proxy.h" #include "./blocksstable/ob_data_file_prepare.h" #include "share/ob_simple_mem_limit_getter.h" +#include "mtlenv/mock_tenant_module_env.h" namespace oceanbase { @@ -186,7 +187,9 @@ void TestParallelExternalSort::SetUp() { TestDataFilePrepare::SetUp(); ASSERT_EQ(OB_SUCCESS, init_tenant_mgr()); - ASSERT_EQ(OB_SUCCESS, ObTmpFileManager::get_instance().init()); + ASSERT_EQ(OB_SUCCESS, common::ObClockGenerator::init()); + ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpBlockCache::get_instance().init("tmp_block_cache", 1)); + ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpPageCache::get_instance().init("tmp_page_cache", 1)); static ObTenantBase tenant_ctx(OB_SYS_TENANT_ID); ObTenantEnv::set_tenant(&tenant_ctx); ObTenantIOManager *io_service = nullptr; @@ -194,14 +197,24 @@ void TestParallelExternalSort::SetUp() EXPECT_EQ(OB_SUCCESS, ObTenantIOManager::mtl_init(io_service)); EXPECT_EQ(OB_SUCCESS, io_service->start()); tenant_ctx.set(io_service); + + tmp_file::ObTenantTmpFileManager *tf_mgr = nullptr; + EXPECT_EQ(OB_SUCCESS, mtl_new_default(tf_mgr)); + EXPECT_EQ(OB_SUCCESS, tmp_file::ObTenantTmpFileManager::mtl_init(tf_mgr)); + tf_mgr->page_cache_controller_.write_buffer_pool_.default_wbp_memory_limit_ = 40*1024*1024; + EXPECT_EQ(OB_SUCCESS, tf_mgr->start()); + tenant_ctx.set(tf_mgr); + ObTenantEnv::set_tenant(&tenant_ctx); } void TestParallelExternalSort::TearDown() { allocator_.reuse(); - ObTmpFileManager::get_instance().destroy(); + tmp_file::ObTmpBlockCache::get_instance().destroy(); + tmp_file::ObTmpPageCache::get_instance().destroy(); TestDataFilePrepare::TearDown(); + common::ObClockGenerator::destroy(); destroy_tenant_mgr(); } @@ -978,6 +991,34 @@ TEST_F(TestParallelExternalSort, test_get_before_sort) ASSERT_EQ(OB_ITER_END, ret); } +TEST_F(TestParallelExternalSort, test_sort_then_get) +{ + int ret = OB_SUCCESS; + const int64_t file_buf_size = MACRO_BLOCK_SIZE; + const int64_t buf_mem_limit = 8 * 1024 * 1024L; + typedef ObExternalSort ExternalSort; + ExternalSort external_sort; + ObVectortotal_items; + TestItemCompare compare(ret); + const int64_t expire_timestamp = 0; + ret = external_sort.init(buf_mem_limit, file_buf_size, expire_timestamp, OB_SYS_TENANT_ID, &compare); + ASSERT_EQ(OB_SUCCESS, ret); + ret = generate_items(81920, false, total_items); + ASSERT_EQ(OB_SUCCESS, ret); + for (int64_t i = 0; OB_SUCC(ret) && i < total_items.size(); ++i) { + ret = external_sort.add_item(*total_items.at(i)); + ASSERT_EQ(OB_SUCCESS, ret); + } + ASSERT_EQ(OB_SUCCESS, external_sort.do_sort(true)); + const TestItem *item = NULL; + for (int64_t i = 0; OB_SUCC(ret) && i < total_items.size(); ++i) { + ret = external_sort.get_next_item(item); + ASSERT_EQ(OB_SUCCESS, ret); + } + ret = external_sort.get_next_item(item); + ASSERT_EQ(OB_ITER_END, ret); +} + } // end namespace common } // end namespace oceanbase From d73ae43cb630a5fc12ba041a15cba1dd1ae87c87 Mon Sep 17 00:00:00 2001 From: Hooper9973 Date: Thu, 15 Aug 2024 12:18:45 +0000 Subject: [PATCH 066/249] fix merge_index_tree 4016 --- src/storage/blocksstable/index_block/ob_index_block_builder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/blocksstable/index_block/ob_index_block_builder.cpp b/src/storage/blocksstable/index_block/ob_index_block_builder.cpp index b264b95bd..cbd599af3 100755 --- a/src/storage/blocksstable/index_block/ob_index_block_builder.cpp +++ b/src/storage/blocksstable/index_block/ob_index_block_builder.cpp @@ -772,7 +772,7 @@ int ObSSTableIndexBuilder::merge_index_tree(ObSSTableMergeRes &res) case REBUILD_BACKUP_TASK: case REBUILD_BACKUP_DDL_TASK: for (int64_t i = 0; i < roots_.count(); ++i) { - if (roots_[i]->meta_block_info_.in_disk()) { + if (roots_[i]->index_tree_info_.in_disk()) { all_in_mem = false; break; } From f1a98e84b17b6b5e2b3314d43bf3d18de1578dd9 Mon Sep 17 00:00:00 2001 From: wxhwang Date: Thu, 15 Aug 2024 12:43:12 +0000 Subject: [PATCH 067/249] fix restore root key failed --- .../restore/ob_restore_scheduler.cpp | 30 ++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/src/rootserver/restore/ob_restore_scheduler.cpp b/src/rootserver/restore/ob_restore_scheduler.cpp index 05d125469..988112a39 100644 --- a/src/rootserver/restore/ob_restore_scheduler.cpp +++ b/src/rootserver/restore/ob_restore_scheduler.cpp @@ -419,16 +419,21 @@ int ObRestoreScheduler::restore_pre(const ObPhysicalRestoreJob &job_info) LOG_WARN("fail to restore root key", K(ret)); } else if (OB_FAIL(restore_keystore(job_info))) { LOG_WARN("fail to restore keystore", K(ret), K(job_info)); - } else { - if (OB_FAIL(fill_restore_statistics(job_info))) { - LOG_WARN("fail to fill restore statistics", K(ret), K(job_info)); - } + } else if (OB_FAIL(fill_restore_statistics(job_info))) { + LOG_WARN("fail to fill restore statistics", K(ret), K(job_info)); + } + + if (OB_IO_ERROR == ret || OB_SUCC(ret)) { int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = try_update_job_status(*sql_proxy_, ret, job_info))) { + if (OB_TMP_FAIL(try_update_job_status(*sql_proxy_, ret, job_info))) { LOG_WARN("fail to update job status", K(ret), K(tmp_ret), K(job_info)); } + + ret = COVER_SUCC(tmp_ret); } + LOG_INFO("[RESTORE] restore pre", K(ret), K(job_info)); + return ret; } @@ -516,12 +521,23 @@ int ObRestoreScheduler::convert_tde_parameters( return ret; } +ERRSIM_POINT_DEF(EN_RESTORE_ROOT_KEY_FAILED); int ObRestoreScheduler::restore_root_key(const share::ObPhysicalRestoreJob &job_info) { int ret = OB_SUCCESS; + +#ifdef ERRSIM + ret = EN_RESTORE_ROOT_KEY_FAILED ? : OB_SUCCESS; + if (OB_FAIL(ret)) { + LOG_WARN("fake EN_RESTORE_ROOT_KEY_FAILED", K(ret)); + } +#endif + #ifdef OB_BUILD_TDE_SECURITY - int64_t idx = job_info.get_multi_restore_path_list().get_backup_set_path_list().count() - 1; - if (idx < 0) { + int64_t idx = 0; + if (OB_FAIL(ret)) { + } else if (FALSE_IT(idx = job_info.get_multi_restore_path_list().get_backup_set_path_list().count() - 1)) { + } else if (idx < 0) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid job info", K(ret), K(idx), K(job_info)); } else if (OB_ISNULL(srv_rpc_proxy_) || OB_ISNULL(sql_proxy_)) { From 52452b6e2d189f09e5566c3b3141fb67d05794a3 Mon Sep 17 00:00:00 2001 From: suz-yang Date: Thu, 15 Aug 2024 12:49:13 +0000 Subject: [PATCH 068/249] fix direct load identity column --- .../ob_table_load_trans_bucket_writer.cpp | 103 +++++++++++------- .../ob_table_load_trans_bucket_writer.h | 10 +- .../table_load/ob_table_load_trans_store.cpp | 84 +++++++++----- .../table_load/ob_table_load_trans_store.h | 5 +- 4 files changed, 134 insertions(+), 68 deletions(-) diff --git a/src/observer/table_load/ob_table_load_trans_bucket_writer.cpp b/src/observer/table_load/ob_table_load_trans_bucket_writer.cpp index 076b57879..88ace5245 100644 --- a/src/observer/table_load/ob_table_load_trans_bucket_writer.cpp +++ b/src/observer/table_load/ob_table_load_trans_bucket_writer.cpp @@ -26,6 +26,7 @@ #include "share/ob_autoincrement_service.h" #include "share/sequence/ob_sequence_cache.h" #include "sql/ob_sql_utils.h" +#include "lib/number/ob_number_v2.h" namespace oceanbase { @@ -37,6 +38,7 @@ using namespace common::hash; using namespace share::schema; using namespace sql; using namespace table; +using namespace common::number; ObTableLoadTransBucketWriter::SessionContext::SessionContext() : session_id_(0), allocator_("TLD_TB_SessCtx"), last_receive_sequence_no_(0) @@ -229,89 +231,116 @@ int ObTableLoadTransBucketWriter::handle_partition_with_autoinc_identity( ObCharset::get_system_collation()); ObTableLoadCastObjCtx cast_obj_ctx(param_, &(coordinator_ctx_->partition_calc_.time_cvrt_), &cast_ctx, true); + ObObj tmp_obj; ObObj out_obj; for (int64_t j = 0; OB_SUCC(ret) && j < row_count; ++j) { - ObStorageDatum storage_datum; ObTableLoadObjRow &obj_row = obj_rows.at(j); const ObTableLoadPartitionCalc::IndexAndType &index_and_type = coordinator_ctx_->partition_calc_.part_key_obj_index_.at( coordinator_ctx_->partition_calc_.partition_with_autoinc_idx_); const ObColumnSchemaV2 *column_schema = index_and_type.column_schema_; const int64_t obj_index = index_and_type.index_; - const ObObj &obj = obj_row.cells_[obj_index]; + ObObj &obj = obj_row.cells_[obj_index]; if (OB_UNLIKELY(obj_row.count_ != column_count_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected column count not match", KR(ret), K(obj_row), K(column_count_)); } else if (OB_UNLIKELY(obj_index < 0 || obj_index >= column_count_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected obj index", KR(ret), K(index_and_type), K(column_count_)); - } else if (obj.is_null() || obj.is_nop_value()) { - out_obj = obj; - } else if (OB_FAIL(ObTableLoadObjCaster::cast_obj(cast_obj_ctx, - column_schema, - obj, - out_obj))) { - LOG_WARN("fail to cast obj", KR(ret)); + } else if (column_schema->is_autoincrement()) { + if (obj.is_null() || obj.is_nop_value()) { + tmp_obj = obj; + } else if (OB_FAIL(ObTableLoadObjCaster::cast_obj(cast_obj_ctx, + column_schema, + obj, + tmp_obj))) { + LOG_WARN("fail to cast obj", KR(ret), K(obj), KPC(column_schema)); + } + if (OB_SUCC(ret)) { + if (OB_FAIL(handle_autoinc_column(column_schema, + tmp_obj, + out_obj, + session_id, + sql_mode))) { + LOG_WARN("fail to handle autoinc column", KR(ret), K(tmp_obj)); + } + } + } else if (column_schema->is_identity_column()) { + // 生成的seq_value是number, 可能需要转换成decimal int + if (OB_FAIL(handle_identity_column(column_schema, obj, tmp_obj, autoinc_allocator))) { + LOG_WARN("fail to handle identity column", KR(ret), K(obj)); + } else if (OB_FAIL(ObTableLoadObjCaster::cast_obj(cast_obj_ctx, + column_schema, + tmp_obj, + out_obj))) { + LOG_WARN("fail to cast obj", KR(ret), K(tmp_obj), KPC(column_schema)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected column not autoinc or identity", KR(ret), KPC(column_schema)); } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(storage_datum.from_obj_enhance(out_obj))) { - LOG_WARN("fail to from obj enhance", KR(ret), K(out_obj)); - } else if (column_schema->is_autoincrement() && - OB_FAIL(handle_autoinc_column(storage_datum, - column_schema->get_meta_type().get_type_class(), - session_id, sql_mode))) { - LOG_WARN("fail to handle autoinc column", KR(ret), K(storage_datum)); - } else if (column_schema->is_identity_column() && - OB_FAIL(handle_identity_column(column_schema, storage_datum, - autoinc_allocator))) { - LOG_WARN("fail to handle identity column", KR(ret), K(storage_datum)); - } else if (OB_FAIL(storage_datum.to_obj_enhance(obj_row.cells_[obj_index], - column_schema->get_meta_type()))) { - LOG_WARN("fail to obj enhance", KR(ret), K(obj_row.cells_[obj_index])); - } else if (OB_FAIL(ob_write_obj(*(obj_row.get_allocator_handler()), obj_row.cells_[obj_index], - obj_row.cells_[obj_index]))) { - LOG_WARN("fail to deep copy obj", KR(ret), K(obj_row.cells_[obj_index])); + if (OB_SUCC(ret)) { + if (OB_FAIL(ob_write_obj(*(obj_row.get_allocator_handler()), out_obj, obj))) { + LOG_WARN("fail to deep copy obj", KR(ret), K(tmp_obj)); + } } } return ret; } -int ObTableLoadTransBucketWriter::handle_autoinc_column(ObStorageDatum &datum, - const ObObjTypeClass &tc, +int ObTableLoadTransBucketWriter::handle_autoinc_column(const ObColumnSchemaV2 *column_schema, + const ObObj &obj, + ObObj &out_obj, int32_t session_id, const uint64_t &sql_mode) { int ret = OB_SUCCESS; - if (OB_FAIL(ObTableLoadAutoincNextval::eval_nextval( - &(coordinator_ctx_->session_ctx_array_[session_id - 1].autoinc_param_), datum, tc, - sql_mode))) { + const ObObjTypeClass &tc = column_schema->get_meta_type().get_type_class(); + ObStorageDatum datum; + if (OB_FAIL(datum.from_obj_enhance(obj))) { + LOG_WARN("fail to from obj enhance", KR(ret), K(obj)); + } else if (OB_FAIL(ObTableLoadAutoincNextval::eval_nextval( + &(coordinator_ctx_->session_ctx_array_[session_id - 1].autoinc_param_), datum, tc, + sql_mode))) { LOG_WARN("fail to get auto increment next value", KR(ret)); + } else if (OB_FAIL(datum.to_obj_enhance(out_obj, column_schema->get_meta_type()))) { + LOG_WARN("fail to obj enhance", KR(ret), K(datum)); } return ret; } int ObTableLoadTransBucketWriter::handle_identity_column(const ObColumnSchemaV2 *column_schema, - ObStorageDatum &datum, + const ObObj &obj, + ObObj &out_obj, ObArenaAllocator &cast_allocator) { int ret = OB_SUCCESS; // 1. generated always as identity : 不能指定此列导入 // 2. generated by default as identity : 不指定时自动生成, 不能导入null // 3. generated by default on null as identity : 不指定或者指定null会自动生成 - if (OB_UNLIKELY(column_schema->is_always_identity_column() && !datum.is_nop())) { + if (OB_UNLIKELY(column_schema->is_always_identity_column() && !obj.is_nop_value())) { ret = OB_ERR_INSERT_INTO_GENERATED_ALWAYS_IDENTITY_COLUMN; LOG_USER_ERROR(OB_ERR_INSERT_INTO_GENERATED_ALWAYS_IDENTITY_COLUMN); - } else if (OB_UNLIKELY(column_schema->is_default_identity_column() && datum.is_null())) { + } else if (OB_UNLIKELY(column_schema->is_default_identity_column() && obj.is_null())) { ret = OB_BAD_NULL_ERROR; LOG_WARN("default identity column cannot insert null", KR(ret)); - } else if (datum.is_nop() || datum.is_null()) { + } else { + // 不论用户有没有指定自增列的值, 都取一个seq_value, 行为与insert into保持一致 + // 取seq_value的性能受表的参数cache影响 ObSequenceValue seq_value; if (OB_FAIL(ObSequenceCache::get_instance().nextval(coordinator_ctx_->sequence_schema_, cast_allocator, seq_value))) { LOG_WARN("fail get nextval for seq", KR(ret)); + } else if (obj.is_nop_value() || obj.is_null()) { + ObNumber number; + if (OB_FAIL(number.from(seq_value.val(), cast_allocator))) { + LOG_WARN("fail deep copy value", KR(ret), K(seq_value)); + } else { + out_obj.set_number(number); + } } else { - datum.set_number(seq_value.val()); + out_obj = obj; } } return ret; diff --git a/src/observer/table_load/ob_table_load_trans_bucket_writer.h b/src/observer/table_load/ob_table_load_trans_bucket_writer.h index 86123cc14..7700878b1 100644 --- a/src/observer/table_load/ob_table_load_trans_bucket_writer.h +++ b/src/observer/table_load/ob_table_load_trans_bucket_writer.h @@ -50,10 +50,14 @@ private: int handle_partition_with_autoinc_identity(SessionContext &session_ctx, table::ObTableLoadObjRowArray &obj_rows, const uint64_t &sql_mode, int32_t session_id); - int handle_autoinc_column(blocksstable::ObStorageDatum &datum, const ObObjTypeClass &tc, - int32_t session_id, const uint64_t &sql_mode); + int handle_autoinc_column(const share::schema::ObColumnSchemaV2 *column_schema, + const common::ObObj &obj, + common::ObObj &out_obj, + int32_t session_id, + const uint64_t &sql_mode); int handle_identity_column(const share::schema::ObColumnSchemaV2 *column_schema, - blocksstable::ObStorageDatum &datum, + const common::ObObj &obj, + common::ObObj &out_obj, common::ObArenaAllocator &cast_allocator); // 非分区表 int write_for_non_partitioned(SessionContext &session_ctx, diff --git a/src/observer/table_load/ob_table_load_trans_store.cpp b/src/observer/table_load/ob_table_load_trans_store.cpp index 80ca5f24e..d41fb2ff8 100644 --- a/src/observer/table_load/ob_table_load_trans_store.cpp +++ b/src/observer/table_load/ob_table_load_trans_store.cpp @@ -434,38 +434,60 @@ int ObTableLoadTransStoreWriter::cast_column( int ret = OB_SUCCESS; ObCastCtx cast_ctx(&cast_allocator, &cast_params, cast_mode_, column_schema->get_collation_type()); ObTableLoadCastObjCtx cast_obj_ctx(param_, &time_cvrt_, &cast_ctx, true); - const bool is_null_autoinc = - (column_schema->is_autoincrement() || column_schema->is_identity_column()) && - (obj.is_null() || obj.is_nop_value()); ObObj out_obj; - if (is_null_autoinc) { - out_obj = obj; - } else if (OB_FAIL(ObTableLoadObjCaster::cast_obj(cast_obj_ctx, column_schema, obj, out_obj))) { - LOG_WARN("fail to cast obj and check", KR(ret), K(obj)); - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(datum.from_obj_enhance(out_obj))) { - LOG_WARN("fail to from obj enhance", KR(ret), K(out_obj)); - } else if (column_schema->is_autoincrement()) { - if (OB_FAIL(handle_autoinc_column( - column_schema, datum, column_schema->get_meta_type().get_type_class(), session_id))) { - LOG_WARN("fail to handle autoinc column", KR(ret), K(datum)); + if (column_schema->is_autoincrement()) { + if (obj.is_null() || obj.is_nop_value()) { + out_obj = obj; + } else if (OB_FAIL(ObTableLoadObjCaster::cast_obj(cast_obj_ctx, + column_schema, + obj, + out_obj))) { + LOG_WARN("fail to cast obj", KR(ret), K(obj), KPC(column_schema)); } - } else if (column_schema->is_identity_column() && !column_schema->is_tbl_part_key_column()) { - // if identity column is part key column, the value is determined before partition calculation - if (OB_FAIL(handle_identity_column(column_schema, datum, cast_allocator))) { - LOG_WARN("fail to handle identity column", KR(ret), K(datum)); + if (OB_SUCC(ret)) { + if (OB_FAIL(handle_autoinc_column(column_schema, out_obj, datum, session_id))) { + LOG_WARN("fail to handle autoinc column", KR(ret), K(out_obj)); + } + } + } else if (column_schema->is_identity_column()) { + if (column_schema->is_tbl_part_key_column()) { + // 自增列是分区键, 在分区计算的时候就已经确定值了 + out_obj = obj; + } else { + // 生成的seq_value是number, 可能需要转换成decimal int + ObObj tmp_obj; + if (OB_FAIL(handle_identity_column(column_schema, obj, tmp_obj, cast_allocator))) { + LOG_WARN("fail to handle identity column", KR(ret), K(obj)); + } else if (OB_FAIL(ObTableLoadObjCaster::cast_obj(cast_obj_ctx, column_schema, tmp_obj, out_obj))) { + LOG_WARN("fail to cast obj and check", KR(ret), K(tmp_obj)); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(datum.from_obj_enhance(out_obj))) { + LOG_WARN("fail to from obj enhance", KR(ret), K(out_obj)); + } + } + } else { + // 普通列 + if (OB_FAIL(ObTableLoadObjCaster::cast_obj(cast_obj_ctx, column_schema, obj, out_obj))) { + LOG_WARN("fail to cast obj and check", KR(ret), K(obj)); + } else if (OB_FAIL(datum.from_obj_enhance(out_obj))) { + LOG_WARN("fail to from obj enhance", KR(ret), K(out_obj)); } } return ret; } int ObTableLoadTransStoreWriter::handle_autoinc_column(const ObColumnSchemaV2 *column_schema, + const ObObj &obj, ObStorageDatum &datum, - const ObObjTypeClass &tc, int32_t session_id) + int32_t session_id) { int ret = OB_SUCCESS; - if (OB_FAIL(ObTableLoadAutoincNextval::eval_nextval( + const ObObjTypeClass &tc = column_schema->get_meta_type().get_type_class(); + if (OB_FAIL(datum.from_obj_enhance(obj))) { + LOG_WARN("fail to from obj enhance", KR(ret), K(obj)); + } else if (OB_FAIL(ObTableLoadAutoincNextval::eval_nextval( &(store_ctx_->session_ctx_array_[session_id - 1].autoinc_param_), datum, tc, store_ctx_->ctx_->session_info_->get_sql_mode()))) { LOG_WARN("fail to get auto increment next value", KR(ret)); @@ -474,27 +496,37 @@ int ObTableLoadTransStoreWriter::handle_autoinc_column(const ObColumnSchemaV2 *c } int ObTableLoadTransStoreWriter::handle_identity_column(const ObColumnSchemaV2 *column_schema, - ObStorageDatum &datum, + const ObObj &obj, + ObObj &out_obj, ObArenaAllocator &cast_allocator) { int ret = OB_SUCCESS; // 1. generated always as identity : 不能指定此列导入 // 2. generated by default as identity : 不指定时自动生成, 不能导入null // 3. generated by default on null as identity : 不指定或者指定null会自动生成 - if (OB_UNLIKELY(column_schema->is_always_identity_column() && !datum.is_nop())) { + if (OB_UNLIKELY(column_schema->is_always_identity_column() && !obj.is_nop_value())) { ret = OB_ERR_INSERT_INTO_GENERATED_ALWAYS_IDENTITY_COLUMN; LOG_USER_ERROR(OB_ERR_INSERT_INTO_GENERATED_ALWAYS_IDENTITY_COLUMN); - } else if (OB_UNLIKELY(column_schema->is_default_identity_column() && datum.is_null())) { + } else if (OB_UNLIKELY(column_schema->is_default_identity_column() && obj.is_null())) { ret = OB_BAD_NULL_ERROR; LOG_WARN("default identity column cannot insert null", KR(ret)); - } else if (datum.is_nop() || datum.is_null()) { + } else { + // 不论用户有没有指定自增列的值, 都取一个seq_value, 行为与insert into保持一致 + // 取seq_value的性能受表的参数cache影响 ObSequenceValue seq_value; if (OB_FAIL(ObSequenceCache::get_instance().nextval(trans_ctx_->ctx_->store_ctx_->sequence_schema_, cast_allocator, seq_value))) { LOG_WARN("fail get nextval for seq", KR(ret)); + } else if (obj.is_nop_value() || obj.is_null()) { + ObNumber number; + if (OB_FAIL(number.from(seq_value.val(), cast_allocator))) { + LOG_WARN("fail deep copy value", KR(ret), K(seq_value)); + } else { + out_obj.set_number(number); + } } else { - datum.set_number(seq_value.val()); + out_obj = obj; } } return ret; diff --git a/src/observer/table_load/ob_table_load_trans_store.h b/src/observer/table_load/ob_table_load_trans_store.h index f6f469e7e..338df857b 100644 --- a/src/observer/table_load/ob_table_load_trans_store.h +++ b/src/observer/table_load/ob_table_load_trans_store.h @@ -95,11 +95,12 @@ private: blocksstable::ObStorageDatum &datum, int32_t session_id); int handle_autoinc_column(const share::schema::ObColumnSchemaV2 *column_schema, + const common::ObObj &obj, blocksstable::ObStorageDatum &datum, - const ObObjTypeClass &tc, int32_t session_id); int handle_identity_column(const share::schema::ObColumnSchemaV2 *column_schema, - blocksstable::ObStorageDatum &datum, + const common::ObObj &obj, + common::ObObj &out_obj, common::ObArenaAllocator &cast_allocator); int write_row_to_table_store(storage::ObDirectLoadTableStore &table_store, const common::ObTabletID &tablet_id, From db50ec5142313eb9d33f2604270d0e05201a2131 Mon Sep 17 00:00:00 2001 From: bit-dance <2634358021@qq.com> Date: Thu, 15 Aug 2024 12:55:24 +0000 Subject: [PATCH 069/249] Remove pl cache when remove pl obj. --- src/observer/ob_rpc_processor_simple.cpp | 2 + src/pl/pl_cache/ob_pl_cache_mgr.cpp | 61 +++++++++++++++++++ src/pl/pl_cache/ob_pl_cache_mgr.h | 31 ++++++++++ src/rootserver/ob_ddl_operator.cpp | 53 +++++++++++----- .../engine/cmd/ob_alter_system_executor.cpp | 2 + src/sql/plan_cache/ob_plan_cache.cpp | 2 + 6 files changed, 136 insertions(+), 15 deletions(-) diff --git a/src/observer/ob_rpc_processor_simple.cpp b/src/observer/ob_rpc_processor_simple.cpp index ab56b6d19..d28d9f79c 100644 --- a/src/observer/ob_rpc_processor_simple.cpp +++ b/src/observer/ob_rpc_processor_simple.cpp @@ -1342,6 +1342,8 @@ int ObFlushCacheP::process() for (uint64_t i=0; iflush_pl_cache_single_cache_obj(arg_.db_ids_.at(i), arg_.schema_id_); + } else if (OB_ISNULL(arg_.sql_id_)) { + ret = plan_cache->flush_pl_cache_single_cache_obj(arg_.db_ids_.at(i), arg_.schema_id_); } else { ret = plan_cache->flush_pl_cache_single_cache_obj(arg_.db_ids_.at(i), arg_.sql_id_); } diff --git a/src/pl/pl_cache/ob_pl_cache_mgr.cpp b/src/pl/pl_cache/ob_pl_cache_mgr.cpp index 32b571d47..026117775 100644 --- a/src/pl/pl_cache/ob_pl_cache_mgr.cpp +++ b/src/pl/pl_cache/ob_pl_cache_mgr.cpp @@ -188,6 +188,66 @@ int ObPLCacheMgr::add_pl_cache(ObPlanCache *lib_cache, ObILibCacheObject *pl_obj return ret; } +int ObPLCacheMgr::flush_pl_cache_by_sql( + uint64_t key_id, + uint64_t db_id, + uint64_t tenant_id, + share::schema::ObMultiVersionSchemaService & schema_service) +{ + int ret = OB_SUCCESS; + ObSchemaGetterGuard tenant_schema_guard; + const ObSimpleTenantSchema *tenant = NULL; + + ObString db_name; + ObString tenant_name; + //get tenant name + if (OB_FAIL(schema_service.get_tenant_schema_guard(tenant_id, tenant_schema_guard))) { + LOG_WARN("failed to get tenant schema guard", KR(ret), K(tenant_id)); + } else if (OB_FAIL(tenant_schema_guard.get_tenant_info(tenant_id, tenant))) { + LOG_WARN("failed get tenant info", K(ret)); + } else if (OB_ISNULL(tenant)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant is null", K(ret)); + } else { + tenant_name = tenant->get_tenant_name_str(); + } + //get db name + const ObSimpleDatabaseSchema *database_schema = NULL; + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(tenant_schema_guard.get_database_schema(tenant_id, db_id, database_schema))) { + LOG_WARN("failed get db schema", K(ret)); + } else if (OB_ISNULL(database_schema)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant is null", K(ret)); + } else { + db_name = database_schema->get_database_name(); + } + + share::ObTenantRole tenant_role; + ObMySQLProxy *sql_proxy = nullptr; + // system tenant execute global flush + const uint64_t exec_tenant_id = ObSchemaUtils::get_exec_tenant_id(OB_SYS_TENANT_ID); + ObSqlString sql; + int64_t affected_rows = 0; + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_ISNULL(sql_proxy = GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected sql proxy", K(ret)); + } else if (OB_FAIL(sql.assign_fmt("alter system flush pl cache schema_id = %lu databases = \"%.*s\" TENANT = \"%.*s\" global", key_id, + db_name.length(), db_name.ptr(), tenant_name.length(), tenant_name.ptr()))) { + LOG_WARN("alter system flush pl cache failed.", K(ret), K(key_id)); + } else { + if (OB_FAIL(sql_proxy->write(exec_tenant_id, sql.ptr(), affected_rows))) { + LOG_WARN("execute query failed", K(ret), K(sql)); + } else { + // do nothing + LOG_INFO("succ to flush pl cache", K(key_id), K(tenant_id), K(affected_rows)); + } + } + return ret; +} // delete all pl cache obj int ObPLCacheMgr::cache_evict_all_pl(ObPlanCache *lib_cache) @@ -229,6 +289,7 @@ int ObPLCacheMgr::cache_evict_pl_cache_single(ObPlanCache *lib_cache, uint64_t d } template int ObPLCacheMgr::cache_evict_pl_cache_single(ObPlanCache *lib_cache, uint64_t db_id, uint64_t &schema_id); +template int ObPLCacheMgr::cache_evict_pl_cache_single(ObPlanCache *lib_cache, uint64_t db_id, uint64_t &schema_id); template int ObPLCacheMgr::cache_evict_pl_cache_single(ObPlanCache *lib_cache, uint64_t db_id, common::ObString &sql_id); } diff --git a/src/pl/pl_cache/ob_pl_cache_mgr.h b/src/pl/pl_cache/ob_pl_cache_mgr.h index 59d4f886b..6179bba7a 100644 --- a/src/pl/pl_cache/ob_pl_cache_mgr.h +++ b/src/pl/pl_cache/ob_pl_cache_mgr.h @@ -87,6 +87,32 @@ struct ObGetPLKVEntryBySchemaIdOp : public ObKVEntryTraverseOp uint64_t schema_id_; }; +struct ObGetPLKVEntryByDbIdOp : public ObKVEntryTraverseOp +{ + explicit ObGetPLKVEntryByDbIdOp(uint64_t db_id, + uint64_t schema_id, + LCKeyValueArray *key_val_list, + const CacheRefHandleID ref_handle) + : ObKVEntryTraverseOp(key_val_list, ref_handle), + db_id_(db_id) + { + } + virtual int check_entry_match(LibCacheKVEntry &entry, bool &is_match) + { + int ret = OB_SUCCESS; + is_match = false; + ObPLObjectKey *key = static_cast(entry.first); + if (db_id_ != common::OB_INVALID_ID && db_id_ != key->db_id_) { + // skip entry that has non-matched db_id + } else { + is_match = true; + } + return ret; + } + + uint64_t db_id_; +}; + struct ObGetPLKVEntryBySQLIDOp : public ObKVEntryTraverseOp { explicit ObGetPLKVEntryBySQLIDOp(uint64_t db_id, @@ -145,6 +171,11 @@ public: static int cache_evict_all_pl(ObPlanCache *lib_cache); template static int cache_evict_pl_cache_single(ObPlanCache *lib_cache, uint64_t db_id, EvictAttr &attr); + static int flush_pl_cache_by_sql( + uint64_t key_id, + uint64_t db_id, + uint64_t tenant_id, + share::schema::ObMultiVersionSchemaService & schema_service); private: static int add_pl_object(ObPlanCache *lib_cache, diff --git a/src/rootserver/ob_ddl_operator.cpp b/src/rootserver/ob_ddl_operator.cpp index 0aefbd329..d56cc0569 100644 --- a/src/rootserver/ob_ddl_operator.cpp +++ b/src/rootserver/ob_ddl_operator.cpp @@ -81,6 +81,7 @@ #include "share/schema/ob_mview_refresh_stats_params.h" #include "storage/mview/ob_mview_sched_job_utils.h" #include "pl/ob_pl_persistent.h" +#include "pl/pl_cache/ob_pl_cache_mgr.h" namespace oceanbase { @@ -699,7 +700,6 @@ int ObDDLOperator::drop_database(const ObDatabaseSchema &db_schema, const uint64_t tenant_id = db_schema.get_tenant_id(); const uint64_t database_id = db_schema.get_database_id(); int64_t new_schema_version = OB_INVALID_VERSION; - if (OB_ISNULL(schema_service_impl)) { ret = OB_ERR_SYS; LOG_ERROR("schama service_impl and schema manage must not null", @@ -842,7 +842,7 @@ int ObDDLOperator::drop_database(const ObDatabaseSchema &db_schema, LOG_WARN("failed to get schema guard", KR(ret), K(tenant_id)); } else if (OB_FAIL(schema_guard.get_simple_package_schemas_in_database(tenant_id, database_id, package_schemas))) { LOG_WARN("get packages in database failed", K(tenant_id), KT(database_id), K(ret)); - } else { + } else { ObArray audits; common::ObSqlString public_sql_string; for (int64_t i = 0; OB_SUCC(ret) && i < package_schemas.count(); ++i) { @@ -998,27 +998,31 @@ int ObDDLOperator::drop_database(const ObDatabaseSchema &db_schema, LOG_WARN("drop routine failed", "routine_id", udt_info->get_type_id(), K(ret)); } if (OB_SUCC(ret)) { + uint64_t udt_db_id = udt_info->get_database_id(); if (udt_info->is_object_spec_ddl() && OB_INVALID_ID != ObUDTObjectType::mask_object_id(udt_info->get_object_spec_id(tenant_id))) { - OZ (pl::ObRoutinePersistentInfo::delete_dll_from_disk(trans, tenant_id, - ObUDTObjectType::mask_object_id(udt_info->get_object_spec_id(tenant_id)), - udt_info->get_database_id())); + uint64_t udt_spec_id = ObUDTObjectType::mask_object_id(udt_info->get_object_spec_id(tenant_id)); + OZ (pl::ObRoutinePersistentInfo::delete_dll_from_disk(trans, tenant_id, udt_spec_id, udt_db_id)); if (udt_info->has_type_body() && OB_INVALID_ID != ObUDTObjectType::mask_object_id(udt_info->get_object_body_id(tenant_id))) { - OZ (pl::ObRoutinePersistentInfo::delete_dll_from_disk(trans, tenant_id, - ObUDTObjectType::mask_object_id(udt_info->get_object_body_id(tenant_id)), - udt_info->get_database_id())); + uint64_t udt_body_id = ObUDTObjectType::mask_object_id(udt_info->get_object_body_id(tenant_id)); + OZ (pl::ObRoutinePersistentInfo::delete_dll_from_disk(trans, tenant_id, udt_body_id, udt_db_id)); } } else if (udt_info->is_object_body_ddl() && OB_INVALID_ID != ObUDTObjectType::mask_object_id(udt_info->get_object_body_id(tenant_id))) { - OZ (pl::ObRoutinePersistentInfo::delete_dll_from_disk(trans, tenant_id, - ObUDTObjectType::mask_object_id(udt_info->get_object_body_id(tenant_id)), - udt_info->get_database_id())); + uint64_t udt_body_id = ObUDTObjectType::mask_object_id(udt_info->get_object_body_id(tenant_id)); + OZ (pl::ObRoutinePersistentInfo::delete_dll_from_disk(trans, tenant_id, udt_body_id, udt_db_id)); + if (OB_INVALID_ID != ObUDTObjectType::mask_object_id(udt_info->get_object_spec_id(tenant_id))) { + uint64_t udt_spec_id = ObUDTObjectType::mask_object_id(udt_info->get_object_spec_id(tenant_id)); + OZ (pl::ObRoutinePersistentInfo::delete_dll_from_disk(trans, tenant_id, udt_spec_id, udt_db_id)); + } } } } } } + // flush pl cache + OZ (pl::ObPLCacheMgr::flush_pl_cache_by_sql(OB_INVALID_ID, database_id, tenant_id, schema_service_)); // delete sequences in database if (OB_SUCC(ret)) { @@ -9113,6 +9117,9 @@ int ObDDLOperator::drop_routine(const ObRoutineInfo &routine_info, routine_info, new_schema_version, trans, ddl_stmt_str))) { LOG_WARN("drop routine info failed", K(routine_info), K(ret)); } + uint64_t rt_id = routine_info.get_routine_id(); + uint64_t db_id = routine_info.get_database_id(); + OZ (pl::ObPLCacheMgr::flush_pl_cache_by_sql(rt_id, db_id, tenant_id, schema_service_)); OZ (pl::ObRoutinePersistentInfo::delete_dll_from_disk(trans, routine_info.get_tenant_id(), routine_info.get_routine_id(), routine_info.get_database_id())); OZ (ObDependencyInfo::delete_schema_object_dependency(trans, routine_info.get_tenant_id(), @@ -9480,6 +9487,8 @@ int ObDDLOperator::drop_package(const ObPackageInfo &package_info, package_info.get_package_id(), new_schema_version, package_info.get_object_type())); + OZ (pl::ObPLCacheMgr::flush_pl_cache_by_sql(package_info.get_package_id(), package_info.get_database_id(), tenant_id, + schema_service_)); OZ (pl::ObRoutinePersistentInfo::delete_dll_from_disk(trans, tenant_id, package_info.get_package_id(), package_info.get_database_id())); if (OB_NOT_NULL(package_body_info)) { @@ -9487,11 +9496,15 @@ int ObDDLOperator::drop_package(const ObPackageInfo &package_info, package_body_info->get_package_id(), new_schema_version, package_body_info->get_object_type())); + OZ (pl::ObPLCacheMgr::flush_pl_cache_by_sql(package_body_info->get_package_id(), package_body_info->get_database_id(), + tenant_id, schema_service_)); OZ (pl::ObRoutinePersistentInfo::delete_dll_from_disk(trans, tenant_id, package_body_info->get_package_id(), package_body_info->get_database_id())); } } } else { + OZ (pl::ObPLCacheMgr::flush_pl_cache_by_sql(package_info.get_package_id(), package_info.get_database_id(), tenant_id, + schema_service_)); OZ (pl::ObRoutinePersistentInfo::delete_dll_from_disk(trans, tenant_id, package_info.get_package_id(), package_info.get_database_id())); } @@ -9675,9 +9688,13 @@ int ObDDLOperator::drop_trigger(const ObTriggerInfo &trigger_info, trigger_info.get_trigger_id(), new_schema_version, trigger_info.get_object_type())); + uint64_t spec_trig_id = share::schema::ObTriggerInfo::get_trigger_spec_package_id(trigger_info.get_trigger_id()); + uint64_t body_trig_id = share::schema::ObTriggerInfo::get_trigger_body_package_id(trigger_info.get_trigger_id()); + OZ (pl::ObPLCacheMgr::flush_pl_cache_by_sql(spec_trig_id, trigger_info.get_database_id(), tenant_id, schema_service_)); OZ (pl::ObRoutinePersistentInfo::delete_dll_from_disk(trans, tenant_id, share::schema::ObTriggerInfo::get_trigger_spec_package_id(trigger_info.get_trigger_id()), trigger_info.get_database_id())); + OZ (pl::ObPLCacheMgr::flush_pl_cache_by_sql(body_trig_id, trigger_info.get_database_id(), tenant_id, schema_service_)); OZ (pl::ObRoutinePersistentInfo::delete_dll_from_disk(trans, tenant_id, share::schema::ObTriggerInfo::get_trigger_body_package_id(trigger_info.get_trigger_id()), trigger_info.get_database_id())); @@ -10198,21 +10215,27 @@ int ObDDLOperator::drop_udt(const ObUDTTypeInfo &udt_info, LOG_WARN("drop udt info failed", K(udt_info), K(ret)); } if (OB_SUCC(ret)) { + uint64_t spec_udt_id = ObUDTObjectType::mask_object_id(udt_info.get_object_spec_id(tenant_id)); if (udt_info.is_object_spec_ddl() && - OB_INVALID_ID != ObUDTObjectType::mask_object_id(udt_info.get_object_spec_id(tenant_id))) { + OB_INVALID_ID != spec_udt_id) { + OZ (pl::ObPLCacheMgr::flush_pl_cache_by_sql(spec_udt_id, udt_info.get_database_id(), tenant_id, schema_service_)); OZ (pl::ObRoutinePersistentInfo::delete_dll_from_disk(trans, tenant_id, - ObUDTObjectType::mask_object_id(udt_info.get_object_spec_id(tenant_id)), + spec_udt_id, udt_info.get_database_id())); if (udt_info.has_type_body() && OB_INVALID_ID != ObUDTObjectType::mask_object_id(udt_info.get_object_body_id(tenant_id))) { + uint64_t body_udt_id = ObUDTObjectType::mask_object_id(udt_info.get_object_body_id(tenant_id)); + OZ (pl::ObPLCacheMgr::flush_pl_cache_by_sql(body_udt_id, udt_info.get_database_id(), tenant_id, schema_service_)); OZ (pl::ObRoutinePersistentInfo::delete_dll_from_disk(trans, tenant_id, - ObUDTObjectType::mask_object_id(udt_info.get_object_body_id(tenant_id)), + body_udt_id, udt_info.get_database_id())); } } else if (udt_info.is_object_body_ddl() && OB_INVALID_ID != ObUDTObjectType::mask_object_id(udt_info.get_object_body_id(tenant_id))) { + uint64_t body_udt_id = ObUDTObjectType::mask_object_id(udt_info.get_object_body_id(tenant_id)); + OZ (pl::ObPLCacheMgr::flush_pl_cache_by_sql(body_udt_id, udt_info.get_database_id(), tenant_id, schema_service_)); OZ (pl::ObRoutinePersistentInfo::delete_dll_from_disk(trans, tenant_id, - ObUDTObjectType::mask_object_id(udt_info.get_object_body_id(tenant_id)), + body_udt_id, udt_info.get_database_id())); } } diff --git a/src/sql/engine/cmd/ob_alter_system_executor.cpp b/src/sql/engine/cmd/ob_alter_system_executor.cpp index 63b99f498..0f27eea70 100644 --- a/src/sql/engine/cmd/ob_alter_system_executor.cpp +++ b/src/sql/engine/cmd/ob_alter_system_executor.cpp @@ -405,6 +405,8 @@ int ObFlushCacheExecutor::execute(ObExecContext &ctx, ObFlushCacheStmt &stmt) for(int64_t j = 0; j < db_num; j++) { // ignore ret if (is_evict_by_schema_id) { ret = plan_cache->flush_pl_cache_single_cache_obj(stmt.flush_cache_arg_.db_ids_.at(j), stmt.flush_cache_arg_.schema_id_); + } else if(OB_ISNULL(sql_id)){ + ret = plan_cache->flush_pl_cache_single_cache_obj(stmt.flush_cache_arg_.db_ids_.at(j), stmt.flush_cache_arg_.schema_id_); } else { ret = plan_cache->flush_pl_cache_single_cache_obj(stmt.flush_cache_arg_.db_ids_.at(j), sql_id); } diff --git a/src/sql/plan_cache/ob_plan_cache.cpp b/src/sql/plan_cache/ob_plan_cache.cpp index 379a12fe0..713c66220 100644 --- a/src/sql/plan_cache/ob_plan_cache.cpp +++ b/src/sql/plan_cache/ob_plan_cache.cpp @@ -1343,6 +1343,7 @@ int ObPlanCache::foreach_cache_evict(CallBack &cb) template int ObPlanCache::foreach_cache_evict(pl::ObGetPLKVEntryOp &); template int ObPlanCache::foreach_cache_evict(pl::ObGetPLKVEntryBySchemaIdOp &); template int ObPlanCache::foreach_cache_evict(pl::ObGetPLKVEntryBySQLIDOp &); +template int ObPlanCache::foreach_cache_evict(pl::ObGetPLKVEntryByDbIdOp &); // Remove all cache object in the lib cache int ObPlanCache::cache_evict_all_obj() @@ -2632,6 +2633,7 @@ int ObPlanCache::flush_pl_cache_single_cache_obj(uint64_t db_id, EvictAttr &attr } template int ObPlanCache::flush_pl_cache_single_cache_obj(uint64_t db_id, uint64_t &schema_id); +template int ObPlanCache::flush_pl_cache_single_cache_obj(uint64_t db_id, uint64_t &schema_id); template int ObPlanCache::flush_pl_cache_single_cache_obj(uint64_t db_id, common::ObString &sql_id); int ObPlanCache::flush_pl_cache() From c184a29fb11fdcae84b5b277e4075926e6d0ae13 Mon Sep 17 00:00:00 2001 From: ZenoWang Date: Thu, 15 Aug 2024 13:01:49 +0000 Subject: [PATCH 070/249] use async tablet freeze when need fast freeze --- src/storage/compaction/ob_tenant_tablet_scheduler.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/storage/compaction/ob_tenant_tablet_scheduler.cpp b/src/storage/compaction/ob_tenant_tablet_scheduler.cpp index 669db69dc..1a34973f4 100644 --- a/src/storage/compaction/ob_tenant_tablet_scheduler.cpp +++ b/src/storage/compaction/ob_tenant_tablet_scheduler.cpp @@ -1501,7 +1501,8 @@ int ObTenantTabletScheduler::schedule_ls_minor_merge( } } // end of while - const bool is_sync = true; + // ATTENTION! : do not use sync freeze because cyclic dependencies exist + const bool is_sync = false; start_time_us = ObClockGenerator::getClock(); if (need_fast_freeze_tablets.empty()) { // empty array. do not need freeze From 1320abda9282ca928cd3134252a41cff63945861 Mon Sep 17 00:00:00 2001 From: zhangzhenyuyu Date: Thu, 15 Aug 2024 13:08:33 +0000 Subject: [PATCH 071/249] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E8=BF=81=E7=A7=BB?= =?UTF-8?q?=E5=A4=8D=E5=88=B6=E5=BA=95=E5=B1=82RPC=20process?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/observer/ob_rpc_processor_simple.cpp | 47 +--------------------- src/observer/ob_service.cpp | 50 +++++++++++++++++++++++- src/observer/ob_service.h | 1 + src/share/ob_rpc_struct.cpp | 24 ++++++------ src/share/ob_rpc_struct.h | 34 ++++++++-------- 5 files changed, 80 insertions(+), 76 deletions(-) diff --git a/src/observer/ob_rpc_processor_simple.cpp b/src/observer/ob_rpc_processor_simple.cpp index d28d9f79c..61e9e97c6 100644 --- a/src/observer/ob_rpc_processor_simple.cpp +++ b/src/observer/ob_rpc_processor_simple.cpp @@ -169,52 +169,7 @@ int ObRpcLSCancelReplicaP::process() int ObRpcLSMigrateReplicaP::process() { - int ret = OB_SUCCESS; - uint64_t tenant_id = arg_.tenant_id_; - ObLSService *ls_service = nullptr; - bool is_exist = false; - ObMigrationOpArg migration_op_arg; - - if (tenant_id != MTL_ID()) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("ObRpcLSMigrateReplicaP::proces tenant not match", K(tenant_id), K(ret)); - } - ObCurTraceId::set(arg_.task_id_); - if (OB_SUCC(ret)) { - SERVER_EVENT_ADD("storage_ha", "schedule_ls_migration start", "tenant_id", arg_.tenant_id_, "ls_id", arg_.ls_id_.id(), - "data_src", arg_.data_source_.get_server(), "dest", arg_.dst_.get_server()); - - ls_service = MTL(ObLSService*); - if (OB_ISNULL(ls_service)) { - ret = OB_ERR_UNEXPECTED; - COMMON_LOG(ERROR, "mtl ObLSService should not be null", K(ret)); - } else if (OB_FAIL(ls_service->check_ls_exist(arg_.ls_id_, is_exist))) { - COMMON_LOG(WARN, "failed to check ls exist", K(ret), K(arg_)); - } else if (is_exist) { - ret = OB_LS_EXIST; - COMMON_LOG(WARN, "can not migrate ls which local ls is exist", K(ret), K(arg_), K(is_exist)); - } else { - migration_op_arg.cluster_id_ = GCONF.cluster_id; - migration_op_arg.data_src_ = arg_.force_data_source_; - migration_op_arg.dst_ = arg_.dst_; - migration_op_arg.ls_id_ = arg_.ls_id_; - //TODO(muwei.ym) need check priority in 4.2 RC3 - migration_op_arg.priority_ = ObMigrationOpPriority::PRIO_HIGH; - migration_op_arg.paxos_replica_number_ = arg_.paxos_replica_number_; - migration_op_arg.src_ = arg_.src_; - migration_op_arg.type_ = ObMigrationOpType::MIGRATE_LS_OP; - - if (OB_FAIL(ls_service->create_ls_for_ha(arg_.task_id_, migration_op_arg))) { - COMMON_LOG(WARN, "failed to create ls for ha", K(ret), K(arg_), K(migration_op_arg)); - } - } - } - - if (OB_FAIL(ret)) { - SERVER_EVENT_ADD("storage_ha", "schedule_ls_migration failed", "ls_id", arg_.ls_id_.id(), "result", ret); - } - - return ret; + return observer::ObService::do_migrate_ls_replica(arg_); } int ObRpcLSAddReplicaP::process() diff --git a/src/observer/ob_service.cpp b/src/observer/ob_service.cpp index 59951c906..7c4076df5 100644 --- a/src/observer/ob_service.cpp +++ b/src/observer/ob_service.cpp @@ -1708,6 +1708,51 @@ int ObService::get_partition_count(obrpc::ObGetPartitionCountResult &result) return ret; } +int ObService::do_migrate_ls_replica(const obrpc::ObLSMigrateReplicaArg &arg) +{ + int ret = OB_SUCCESS; + uint64_t tenant_id = arg.tenant_id_; + ObLSService *ls_service = nullptr; + bool is_exist = false; + ObMigrationOpArg migration_op_arg; + if (tenant_id != MTL_ID()) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("ObRpcLSMigrateReplicaP::process tenant not match", KR(ret), K(tenant_id)); + } + ObCurTraceId::set(arg.task_id_); + if (OB_SUCC(ret)) { + SERVER_EVENT_ADD("storage_ha", "schedule_ls_migration start", "tenant_id", arg.tenant_id_, "ls_id", arg.ls_id_.id(), + "data_src", arg.force_data_source_.get_server(), "dest", arg.dst_.get_server()); + ls_service = MTL(ObLSService*); + if (OB_ISNULL(ls_service)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("mtl ObLSService should not be null", KR(ret)); + } else if (OB_FAIL(ls_service->check_ls_exist(arg.ls_id_, is_exist))) { + LOG_WARN("failed to check ls exist", KR(ret), K(arg)); + } else if (is_exist) { + ret = OB_LS_EXIST; + LOG_WARN("can not migrate ls which local ls is exist", KR(ret), K(arg), K(is_exist)); + } else { + migration_op_arg.cluster_id_ = GCONF.cluster_id; + migration_op_arg.data_src_ = arg.force_data_source_; + migration_op_arg.dst_ = arg.dst_; + migration_op_arg.ls_id_ = arg.ls_id_; + //TODO(muwei.ym) need check priority + migration_op_arg.priority_ = ObMigrationOpPriority::PRIO_HIGH; + migration_op_arg.paxos_replica_number_ = arg.paxos_replica_number_; + migration_op_arg.src_ = arg.src_; + migration_op_arg.type_ = ObMigrationOpType::MIGRATE_LS_OP; + if (OB_FAIL(ls_service->create_ls_for_ha(arg.task_id_, migration_op_arg))) { + LOG_WARN("failed to create ls for ha", KR(ret), K(arg), K(migration_op_arg)); + } + } + } + if (OB_FAIL(ret)) { + SERVER_EVENT_ADD("storage_ha", "schedule_ls_migration failed", "ls_id", arg.ls_id_.id(), "result", ret); + } + return ret; +} + int ObService::do_add_ls_replica(const obrpc::ObLSAddReplicaArg &arg) { int ret = OB_SUCCESS; @@ -1722,7 +1767,7 @@ int ObService::do_add_ls_replica(const obrpc::ObLSAddReplicaArg &arg) ObCurTraceId::set(arg.task_id_); if (OB_SUCC(ret)) { SERVER_EVENT_ADD("storage_ha", "schedule_ls_add start", "tenant_id", arg.tenant_id_, "ls_id", arg.ls_id_.id(), - "data_src", arg.data_source_.get_server(), "dest", arg.dst_.get_server()); + "data_src", arg.force_data_source_.get_server(), "dest", arg.dst_.get_server()); ls_service = MTL(ObLSService*); if (OB_ISNULL(ls_service)) { ret = OB_ERR_UNEXPECTED; @@ -1737,9 +1782,10 @@ int ObService::do_add_ls_replica(const obrpc::ObLSAddReplicaArg &arg) migration_op_arg.data_src_ = arg.force_data_source_; migration_op_arg.dst_ = arg.dst_; migration_op_arg.ls_id_ = arg.ls_id_; - //TODO(muwei.ym) need check priority in 4.2 RC3 + //TODO(muwei.ym) need check priority migration_op_arg.priority_ = ObMigrationOpPriority::PRIO_HIGH; migration_op_arg.paxos_replica_number_ = arg.new_paxos_replica_number_; + // for add tasks, the src_ field is useless, but must be valid migration_op_arg.src_ = arg.dst_; migration_op_arg.type_ = ObMigrationOpType::ADD_LS_OP; if (OB_FAIL(ls_service->create_ls_for_ha(arg.task_id_, migration_op_arg))) { diff --git a/src/observer/ob_service.h b/src/observer/ob_service.h index 62b0f0d46..a4ed78244 100644 --- a/src/observer/ob_service.h +++ b/src/observer/ob_service.h @@ -203,6 +203,7 @@ public: static int do_remove_ls_paxos_replica(const obrpc::ObLSDropPaxosReplicaArg &arg); static int do_remove_ls_nonpaxos_replica(const obrpc::ObLSDropNonPaxosReplicaArg &arg); static int do_add_ls_replica(const obrpc::ObLSAddReplicaArg &arg); + static int do_migrate_ls_replica(const obrpc::ObLSMigrateReplicaArg &arg); // ObRpcIsEmptyServerP @RS bootstrap int is_empty_server(const obrpc::ObCheckServerEmptyArg &arg, obrpc::Bool &is_empty); // ObRpcCheckDeploymentModeP diff --git a/src/share/ob_rpc_struct.cpp b/src/share/ob_rpc_struct.cpp index 7687b87e5..b53da0777 100644 --- a/src/share/ob_rpc_struct.cpp +++ b/src/share/ob_rpc_struct.cpp @@ -4281,10 +4281,10 @@ OB_SERIALIZE_MEMBER(ObLSMigrateReplicaArg, ls_id_, src_, dst_, - data_source_, + discarded_data_source_, // FARM COMPAT WHITELIST paxos_replica_number_, skip_change_member_list_, - force_use_data_source_, + discarded_force_use_data_source_, // FARM COMPAT WHITELIST force_data_source_, prioritize_same_zone_src_); @@ -4297,10 +4297,10 @@ int ObLSMigrateReplicaArg::assign( ls_id_ = that.ls_id_; src_ = that.src_; dst_ = that.dst_; - data_source_ = that.data_source_; + discarded_data_source_ = that.discarded_data_source_; paxos_replica_number_ = that.paxos_replica_number_; skip_change_member_list_ = that.skip_change_member_list_; - force_use_data_source_ = that.force_use_data_source_; + discarded_force_use_data_source_ = that.discarded_force_use_data_source_; force_data_source_ = that.force_data_source_; prioritize_same_zone_src_ = that.prioritize_same_zone_src_; return ret; @@ -4312,7 +4312,7 @@ int ObLSMigrateReplicaArg::init( const share::ObLSID &ls_id, const common::ObReplicaMember &src, const common::ObReplicaMember &dst, - const common::ObReplicaMember &data_source, + const common::ObReplicaMember &discarded_data_source, const int64_t paxos_replica_number, const bool skip_change_member_list, const common::ObReplicaMember &force_data_source) @@ -4323,7 +4323,7 @@ int ObLSMigrateReplicaArg::init( ls_id_ = ls_id; src_ = src; dst_ = dst; - data_source_ = data_source; + discarded_data_source_ = discarded_data_source; paxos_replica_number_ = paxos_replica_number; skip_change_member_list_ = skip_change_member_list; force_data_source_ = force_data_source; @@ -4335,11 +4335,11 @@ OB_SERIALIZE_MEMBER(ObLSAddReplicaArg, tenant_id_, ls_id_, dst_, - data_source_, + discarded_data_source_, // FARM COMPAT WHITELIST orig_paxos_replica_number_, new_paxos_replica_number_, skip_change_member_list_, - force_use_data_source_, + discarded_force_use_data_source_, // FARM COMPAT WHITELIST force_data_source_); int ObLSAddReplicaArg::assign( @@ -4350,11 +4350,11 @@ int ObLSAddReplicaArg::assign( tenant_id_ = that.tenant_id_; ls_id_ = that.ls_id_; dst_ = that.dst_; - data_source_ = that.data_source_; + discarded_data_source_ = that.discarded_data_source_; orig_paxos_replica_number_ = that.orig_paxos_replica_number_; new_paxos_replica_number_ = that.new_paxos_replica_number_; skip_change_member_list_ = that.skip_change_member_list_; - force_use_data_source_ = that.force_use_data_source_; + discarded_force_use_data_source_ = that.discarded_force_use_data_source_; force_data_source_ = that.force_data_source_; return ret; } @@ -4364,7 +4364,7 @@ int ObLSAddReplicaArg::init( const uint64_t tenant_id, const share::ObLSID &ls_id, const common::ObReplicaMember &dst, - const common::ObReplicaMember &data_source, + const common::ObReplicaMember &discarded_data_source, const int64_t orig_paxos_replica_number, const int64_t new_paxos_replica_number, const bool skip_change_member_list, @@ -4375,7 +4375,7 @@ int ObLSAddReplicaArg::init( tenant_id_ = tenant_id; ls_id_ = ls_id; dst_ = dst; - data_source_ = data_source; + discarded_data_source_ = discarded_data_source; orig_paxos_replica_number_ = orig_paxos_replica_number; new_paxos_replica_number_ = new_paxos_replica_number; skip_change_member_list_ = skip_change_member_list; diff --git a/src/share/ob_rpc_struct.h b/src/share/ob_rpc_struct.h index d3d919dd8..94b70c546 100644 --- a/src/share/ob_rpc_struct.h +++ b/src/share/ob_rpc_struct.h @@ -4492,10 +4492,10 @@ public: ls_id_(), src_(), dst_(), - data_source_(), + discarded_data_source_(), paxos_replica_number_(0), skip_change_member_list_(), - force_use_data_source_(false), + discarded_force_use_data_source_(false), force_data_source_(), prioritize_same_zone_src_(false) {} public: @@ -4507,7 +4507,7 @@ public: const share::ObLSID &ls_id, const common::ObReplicaMember &src, const common::ObReplicaMember &dst, - const common::ObReplicaMember &data_source, + const common::ObReplicaMember &discarded_data_source, const int64_t paxos_replica_number, const bool skip_change_member_list, const common::ObReplicaMember &force_data_source); @@ -4517,10 +4517,10 @@ public: K_(ls_id), K_(src), K_(dst), - K_(data_source), + K_(discarded_data_source), K_(paxos_replica_number), K_(skip_change_member_list), - K_(force_use_data_source), + K_(discarded_force_use_data_source), K_(force_data_source), K_(prioritize_same_zone_src)); @@ -4530,7 +4530,7 @@ public: && ls_id_.is_valid() && src_.is_valid() && dst_.is_valid() - && data_source_.is_valid() + && discarded_data_source_.is_valid() && paxos_replica_number_ > 0; } public: @@ -4539,11 +4539,12 @@ public: share::ObLSID ls_id_; common::ObReplicaMember src_; common::ObReplicaMember dst_; - common::ObReplicaMember data_source_; + // deprecated field, to ensure compatibility, it must be valid + common::ObReplicaMember discarded_data_source_; int64_t paxos_replica_number_; bool skip_change_member_list_; - bool force_use_data_source_; // deprecated field, in order to fix the upgrade compatibility issue from 430rc2 to master + bool discarded_force_use_data_source_; common::ObReplicaMember force_data_source_; bool prioritize_same_zone_src_; }; @@ -4558,11 +4559,11 @@ public: tenant_id_(OB_INVALID_ID), ls_id_(), dst_(), - data_source_(), + discarded_data_source_(), orig_paxos_replica_number_(0), new_paxos_replica_number_(0), skip_change_member_list_(false), - force_use_data_source_(false), + discarded_force_use_data_source_(false), force_data_source_() {} public: int assign(const ObLSAddReplicaArg &that); @@ -4572,7 +4573,7 @@ public: const uint64_t tenant_id, const share::ObLSID &ls_id, const common::ObReplicaMember &dst, - const common::ObReplicaMember &data_source, + const common::ObReplicaMember &discarded_data_source, const int64_t orig_paxos_replica_number, const int64_t new_paxos_replica_number, const bool skip_change_member_list, @@ -4582,11 +4583,11 @@ public: K_(tenant_id), K_(ls_id), K_(dst), - K_(data_source), + K_(discarded_data_source), K_(orig_paxos_replica_number), K_(new_paxos_replica_number), K_(skip_change_member_list), - K_(force_use_data_source), + K_(discarded_force_use_data_source), K_(force_data_source)); bool is_valid() const { @@ -4594,7 +4595,7 @@ public: && common::OB_INVALID_ID != tenant_id_ && ls_id_.is_valid() && dst_.is_valid() - && data_source_.is_valid() + && discarded_data_source_.is_valid() && orig_paxos_replica_number_ > 0 && new_paxos_replica_number_ > 0; } @@ -4603,12 +4604,13 @@ public: uint64_t tenant_id_; share::ObLSID ls_id_; common::ObReplicaMember dst_; - common::ObReplicaMember data_source_; + // deprecated field, to ensure compatibility, it must be valid + common::ObReplicaMember discarded_data_source_; int64_t orig_paxos_replica_number_; int64_t new_paxos_replica_number_; bool skip_change_member_list_; - bool force_use_data_source_; // deprecated field, in order to fix the upgrade compatibility issue from 430rc2 to master + bool discarded_force_use_data_source_; common::ObReplicaMember force_data_source_; }; From f2b2472f5e26311f16440b68dfa3240b44549f1b Mon Sep 17 00:00:00 2001 From: xianyu-w <707512433@qq.com> Date: Thu, 15 Aug 2024 13:15:09 +0000 Subject: [PATCH 072/249] [FEAT MERGE] ENHANCE CARDINALITY ESTIMATION AND OPTIMIZER STATS MANAGEMENT Co-authored-by: ChangerR Co-authored-by: wangt1xiuyi <13547954130@163.com> --- .../ob_dbms_sched_table_operator.cpp | 3 +- src/pl/ob_pl_interface_pragma.h | 1 + src/pl/sys_package/ob_dbms_stats.cpp | 615 ++++++- src/pl/sys_package/ob_dbms_stats.h | 31 +- src/rootserver/ob_ddl_operator.cpp | 8 +- .../ob_inner_table_schema.21351_21400.cpp | 4 +- .../ob_inner_table_schema.25201_25250.cpp | 2 +- .../ob_inner_table_schema.28151_28200.cpp | 2 +- .../inner_table/ob_inner_table_schema_def.py | 19 +- .../sys_package/dbms_stats_body_mysql.sql | 7 +- .../sys_package/dbms_stats_mysql.sql | 6 +- src/share/ob_upgrade_utils.cpp | 40 + src/share/ob_upgrade_utils.h | 2 +- src/share/stat/ob_basic_stats_estimator.cpp | 73 + src/share/stat/ob_basic_stats_estimator.h | 7 + src/share/stat/ob_dbms_stats_executor.cpp | 165 +- src/share/stat/ob_dbms_stats_executor.h | 7 + src/share/stat/ob_dbms_stats_gather.cpp | 37 + src/share/stat/ob_dbms_stats_gather.h | 3 + .../stat/ob_dbms_stats_history_manager.cpp | 76 +- .../stat/ob_dbms_stats_history_manager.h | 12 +- .../stat/ob_dbms_stats_maintenance_window.cpp | 285 ++- .../stat/ob_dbms_stats_maintenance_window.h | 20 +- src/share/stat/ob_dbms_stats_preferences.cpp | 513 +++--- src/share/stat/ob_dbms_stats_preferences.h | 91 +- src/share/stat/ob_dbms_stats_utils.cpp | 139 +- src/share/stat/ob_dbms_stats_utils.h | 8 + src/share/stat/ob_hybrid_hist_estimator.cpp | 36 +- src/share/stat/ob_hybrid_hist_estimator.h | 2 + .../stat/ob_incremental_stat_estimator.cpp | 17 +- src/share/stat/ob_opt_stat_gather_stat.cpp | 7 +- src/share/stat/ob_opt_stat_gather_stat.h | 5 +- src/share/stat/ob_opt_stat_manager.cpp | 1 + .../stat/ob_opt_stat_monitor_manager.cpp | 810 ++++++++- src/share/stat/ob_opt_stat_monitor_manager.h | 67 + src/share/stat/ob_opt_stat_sql_service.cpp | 3 + src/share/stat/ob_opt_table_stat.h | 14 +- src/share/stat/ob_stat_define.cpp | 27 +- src/share/stat/ob_stat_define.h | 95 +- src/share/stat/ob_stat_item.cpp | 4 +- src/share/stat/ob_stat_item.h | 10 +- src/sql/engine/cmd/ob_analyze_executor.cpp | 2 + src/sql/ob_optimizer_trace_impl.h | 22 + .../optimizer/ob_access_path_estimation.cpp | 159 +- src/sql/optimizer/ob_access_path_estimation.h | 7 +- src/sql/optimizer/ob_dynamic_sampling.cpp | 5 +- src/sql/optimizer/ob_join_order.cpp | 647 ++++++- src/sql/optimizer/ob_join_order.h | 74 +- src/sql/optimizer/ob_log_count.cpp | 6 +- src/sql/optimizer/ob_log_distinct.cpp | 16 +- src/sql/optimizer/ob_log_distinct.h | 2 +- src/sql/optimizer/ob_log_join.cpp | 17 +- src/sql/optimizer/ob_log_join.h | 1 + src/sql/optimizer/ob_log_plan.cpp | 116 +- src/sql/optimizer/ob_log_plan.h | 14 +- src/sql/optimizer/ob_log_set.cpp | 6 + src/sql/optimizer/ob_log_set.h | 1 + src/sql/optimizer/ob_log_sort.cpp | 1 + src/sql/optimizer/ob_log_subplan_filter.cpp | 11 +- src/sql/optimizer/ob_log_subplan_filter.h | 1 + src/sql/optimizer/ob_log_table_scan.cpp | 8 +- .../optimizer/ob_log_temp_table_access.cpp | 1 + .../ob_log_temp_table_transformation.cpp | 10 + .../ob_log_temp_table_transformation.h | 1 + src/sql/optimizer/ob_logical_operator.cpp | 39 + src/sql/optimizer/ob_logical_operator.h | 6 + src/sql/optimizer/ob_opt_default_stat.h | 4 +- src/sql/optimizer/ob_opt_est_cost.cpp | 76 +- src/sql/optimizer/ob_opt_est_utils.cpp | 143 +- src/sql/optimizer/ob_opt_est_utils.h | 9 +- src/sql/optimizer/ob_opt_selectivity.cpp | 1590 +++++++++++++++-- src/sql/optimizer/ob_opt_selectivity.h | 350 +++- src/sql/optimizer/ob_optimizer.cpp | 35 + src/sql/optimizer/ob_optimizer.h | 1 + src/sql/optimizer/ob_optimizer_context.h | 14 +- src/sql/optimizer/ob_optimizer_util.cpp | 12 + src/sql/optimizer/ob_optimizer_util.h | 3 + src/sql/optimizer/ob_sel_estimator.cpp | 1261 ++++++++++--- src/sql/optimizer/ob_sel_estimator.h | 280 +-- src/sql/optimizer/ob_select_log_plan.cpp | 11 +- src/sql/optimizer/ob_select_log_plan.h | 8 +- .../resolver/ddl/ob_analyze_stmt_resolver.cpp | 2 + src/sql/resolver/dml/ob_dml_stmt.cpp | 20 + src/sql/resolver/dml/ob_dml_stmt.h | 1 + src/sql/resolver/dml/ob_hint.cpp | 40 + src/sql/resolver/dml/ob_hint.h | 2 + src/sql/resolver/expr/ob_raw_expr_util.cpp | 4 +- src/sql/resolver/expr/ob_raw_expr_util.h | 4 +- .../rewrite/ob_transform_groupby_pullup.cpp | 2 +- tools/deploy/mysql_test/r/mysql/view_2.result | 4 +- .../r/mysql/basic_cs_encoding.result | 4 +- .../test_suite/datatype/r/mysql/div.result | 2 +- .../test_suite/executor/r/mysql/basic.result | 8 +- .../r/mysql/geometry_filter_mysql.result | 150 +- .../r/mysql/geometry_index2_mysql.result | 12 +- .../geometry_partition_table_mysql.result | 8 +- .../mysql/spatial_relation_join_mysql.result | 12 +- .../join/r/mysql/anti_semi_join.result | 12 +- .../test_suite/join/r/mysql/join_merge.result | 54 +- ...d_loop_join_right_null_joinon_where.result | 4 +- .../test_suite/px/r/mysql/agg.result | 12 +- ...for_producer_consumer_schedule_mode.result | 16 +- .../r/mysql/skyline_basic_mysql.result | 68 +- .../r/mysql/skyline_complicate_mysql.result | 41 +- .../r/mysql/hash_distinct.result | 6 +- .../r/mysql/subplan_filter.result | 98 +- .../static_engine/r/mysql/table_scan.result | 6 +- .../subquery/r/mysql/spf_bug13044302.result | 4 +- 108 files changed, 7295 insertions(+), 1545 deletions(-) diff --git a/src/observer/dbms_scheduler/ob_dbms_sched_table_operator.cpp b/src/observer/dbms_scheduler/ob_dbms_sched_table_operator.cpp index 52bf2ffc1..d540e0ddd 100644 --- a/src/observer/dbms_scheduler/ob_dbms_sched_table_operator.cpp +++ b/src/observer/dbms_scheduler/ob_dbms_sched_table_operator.cpp @@ -320,7 +320,8 @@ int ObDBMSSchedTableOperator::update_for_end(ObDBMSSchedJobInfo &job_info, int e OZ (_build_job_drop_dml(now, job_info, sql1)); } else { OX (job_info.failures_ = (err == 0) ? 0 : (job_info.failures_ + 1)); - OX (job_info.flag_ = job_info.failures_ > 15 ? (job_info.flag_ | 0x1) : (job_info.flag_ & 0xfffffffffffffffE)); + // + OX (job_info.flag_ = (job_info.failures_ > 15 && !ObDbmsStatsMaintenanceWindow::is_stats_job(job_info.get_job_name())) ? (job_info.flag_ | 0x1) : (job_info.flag_ & 0xfffffffffffffffE)); OX (job_info.total_ += (job_info.this_date_ > 0 ? now - job_info.this_date_ : 0)); if (OB_SUCC(ret) && ((job_info.flag_ & 0x1) != 0)) { // when if failures > 16 then set broken state. diff --git a/src/pl/ob_pl_interface_pragma.h b/src/pl/ob_pl_interface_pragma.h index a7c8675e1..e17c96c19 100644 --- a/src/pl/ob_pl_interface_pragma.h +++ b/src/pl/ob_pl_interface_pragma.h @@ -351,6 +351,7 @@ INTERFACE_DEF(INTERFACE_DBMS_STATS_GATHER_SYSTEM_STATS, "GATHER_SYSTEM_STATS", (ObDbmsStats::gather_system_stats)) INTERFACE_DEF(INTERFACE_DBMS_STATS_DELETE_SYSTEM_STATS, "DELETE_SYSTEM_STATS", (ObDbmsStats::delete_system_stats)) INTERFACE_DEF(INTERFACE_DBMS_STATS_SET_SYSTEM_STATS, "SET_SYSTEM_STATS", (ObDbmsStats::set_system_stats)) + INTERFACE_DEF(INTERFACE_DBMS_STATS_ASYNC_GATHER_STATS_JOB_PROC, "ASYNC_GATHER_STATS_JOB_PROC", (ObDbmsStats::async_gather_stats_job_proc)) //end of dbms_stat #ifdef OB_BUILD_ORACLE_PL diff --git a/src/pl/sys_package/ob_dbms_stats.cpp b/src/pl/sys_package/ob_dbms_stats.cpp index 7abc5a1b6..8c88848e7 100644 --- a/src/pl/sys_package/ob_dbms_stats.cpp +++ b/src/pl/sys_package/ob_dbms_stats.cpp @@ -33,6 +33,7 @@ #include "sql/engine/expr/ob_expr_uuid.h" #include "sql/privilege_check/ob_ora_priv_check.h" #include "sql/ob_result_set.h" +#include "share/stat/ob_dbms_stats_maintenance_window.h" namespace oceanbase { @@ -59,8 +60,8 @@ namespace pl { * 12. no_invalidate BOOLEAN DEFAULT to_no_invalidate_type(get_param('NO_INVALIDATE')), * 13. stattype VARCHAR2 DEFAULT 'DATA', * 14. force BOOLEAN DEFAULT false, - * 15. context DBMS_STATS.CONTEXT DEFAULT NULL, - * 16. options VARCHAR2 DEFAULT 'GATHER' + * 15. hist_est_percent NUMBER DEFAULT AUTO_SAMPLE_SIZE + * 16. hist_block_sample BOOLEAN DEFAULT FALSE, * @param result * @return */ @@ -79,9 +80,11 @@ int ObDbmsStats::gather_table_stats(ObExecContext &ctx, ParamStore ¶ms, ObOb LOG_WARN("failed to check tenant is restore", K(ret)); } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { LOG_WARN("failed to implicit commit before gather stats", K(ret)); - } else if (OB_ISNULL(ctx.get_my_session()) || OB_ISNULL(ctx.get_task_executor_ctx())) { + } else if (OB_ISNULL(ctx.get_my_session())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(ctx.get_my_session()), K(ctx.get_task_executor_ctx())); + LOG_WARN("get unexpected error", K(ret), K(ctx.get_my_session())); + } else if (OB_FAIL(ObDbmsStatsUtils::cancel_async_gather_stats(ctx))) { + LOG_WARN("failed to cancel async gather stats", K(ret)); } else if (OB_FAIL(init_gather_task_info(ctx, ObOptStatGatherType::MANUAL_GATHER, start_time, task_cnt, task_info))) { LOG_WARN("failed to init gather task info", K(ret)); } else { @@ -106,6 +109,8 @@ int ObDbmsStats::gather_table_stats(ObExecContext &ctx, ParamStore ¶ms, ObOb params.at(8), params.at(12), params.at(14), + params.count() > 15 ? ¶ms.at(15) : NULL, + params.count() > 16 ? ¶ms.at(16) : NULL, stat_param))) { LOG_WARN("failed to parse stat optitions", K(ret)); } else if (OB_FAIL(running_monitor.add_table_info(stat_param))) { @@ -181,13 +186,15 @@ int ObDbmsStats::gather_schema_stats(ObExecContext &ctx, ParamStore ¶ms, ObO LOG_WARN("failed to check tenant is restore", K(ret)); } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { LOG_WARN("failed to implicit commit before gather stats", K(ret)); - } else if (OB_ISNULL(ctx.get_my_session()) || OB_ISNULL(ctx.get_task_executor_ctx())) { + } else if (OB_ISNULL(ctx.get_my_session())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(ctx.get_my_session()), K(ctx.get_task_executor_ctx())); + LOG_WARN("get unexpected error", K(ret), K(ctx.get_my_session())); } else if (ctx.get_my_session()->get_is_in_retry()) { ret = OB_ERR_DBMS_STATS_PL; LOG_WARN("retry gather schema stats is not allowed", K(ret)); LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL,"retry gather schema stats is not allowed"); + } else if (OB_FAIL(ObDbmsStatsUtils::cancel_async_gather_stats(ctx))) { + LOG_WARN("failed to cancel async gather stats", K(ret)); } else if (OB_FAIL(ObOptStatMonitorManager::flush_database_monitoring_info(ctx, false, true))) { LOG_WARN("failed to do flush database monitoring info", K(ret)); } else if (OB_FAIL(get_all_table_ids_in_database(ctx, params.at(0), global_param, table_ids))) { @@ -228,6 +235,8 @@ int ObDbmsStats::gather_schema_stats(ObExecContext &ctx, ParamStore ¶ms, ObO params.at(6), params.at(10), params.at(12), + NULL/*hist_est_percent*/, + NULL/*hist_block_sample*/, stat_param))) { LOG_WARN("failed to parse stat optitions", K(ret)); } else if (OB_FAIL(running_monitor.add_table_info(stat_param))) { @@ -342,6 +351,8 @@ int ObDbmsStats::gather_index_stats(ObExecContext &ctx, ParamStore ¶ms, ObOb empty_cascade, params.at(9), params.at(10), + NULL/*hist_est_percent*/, + NULL/*hist_block_sample*/, ind_stat_param))) { LOG_WARN("failed to parse stat optitions", K(ret)); } else if (ObDbmsStatsUtils::is_virtual_index_table(ind_stat_param.table_id_)) {//not gather virtual table index. @@ -2851,6 +2862,7 @@ int ObDbmsStats::get_prefs(sql::ObExecContext &ctx, ObTableStatParam param; param.allocator_ = &ctx.get_allocator(); ObStatPrefs *stat_pref = NULL; + uint64_t tenant_id = ctx.get_my_session()->get_effective_tenant_id(); if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); } else if (!params.at(0).is_null() && OB_FAIL(params.at(0).get_string(opt_name))) { @@ -2869,7 +2881,9 @@ int ObDbmsStats::get_prefs(sql::ObExecContext &ctx, ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret), K(stat_pref)); } else if (FALSE_IT(stat_pref->set_is_global_prefs(true))) { - } else if (OB_FAIL(ObDbmsStatsPreferences::get_prefs(ctx, param, opt_name, result))) { + } else if (OB_FAIL(ObDbmsStatsPreferences::get_prefs(ctx.get_sql_proxy(), ctx.get_allocator(), + tenant_id, param.table_id_, + opt_name, result))) { LOG_WARN("failed to get prefs", K(ret)); } else {/*do nothing*/} return ret; @@ -3016,6 +3030,7 @@ int ObDbmsStats::set_table_prefs(sql::ObExecContext &ctx, ObSEArray table_ids; ObStatPrefs *stat_pref = NULL; bool use_size_auto = false; + bool is_async_gather = false; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { @@ -3048,7 +3063,7 @@ int ObDbmsStats::set_table_prefs(sql::ObExecContext &ctx, } else if (OB_FAIL(stat_pref->dump_pref_name_and_value(opt_name, opt_value))) { LOG_WARN("failed to dump pref name and value"); } else if (0 == opt_name.case_compare("METHOD_OPT") && - OB_FAIL(parse_method_opt(ctx, param.allocator_, param.column_params_, opt_value, use_size_auto))) { + OB_FAIL(parse_method_opt(ctx, param.allocator_, param.column_params_, opt_value, is_async_gather, use_size_auto))) { LOG_WARN("failed to parse method opt", K(ret)); } else if (OB_FAIL(ObDbmsStatsPreferences::set_prefs(ctx, table_ids, opt_name, opt_value))) { LOG_WARN("failed to set prefs", K(ret)); @@ -3188,6 +3203,87 @@ int ObDbmsStats::cancel_gather_stats(sql::ObExecContext &ctx, return ret; } +/** + * @brief ObDbmsStats::async_gather_stats_job_proc + * @param ctx + * @param params + * 0. duration NUMBER + * @param result + * @return int + */ +int ObDbmsStats::async_gather_stats_job_proc(sql::ObExecContext &ctx, + sql::ParamStore ¶ms, + common::ObObj &result) +{ + int ret = OB_SUCCESS; + UNUSED(result); + const int64_t start_time = ObTimeUtility::current_time(); + ObOptStatTaskInfo task_info; + number::ObNumber num_duration; + int64_t duration_time = -1; + int64_t succeed_cnt = 0; + bool no_async_gather = (OB_E(EventTable::EN_LEADER_STORAGE_ESTIMATION) OB_SUCCESS) != OB_SUCCESS; + ObSQLSessionInfo *session = ctx.get_my_session(); + uint64_t tenant_id = session->get_effective_tenant_id(); + ObSQLSessionInfo::LockGuard query_lock_guard(session->get_query_lock()); + uint64_t data_version = 0; + if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, data_version))) { + LOG_WARN("fail to get tenant data version", KR(ret), K(tenant_id), K(data_version)); + } else if (data_version < MOCK_DATA_VERSION_4_2_4_0 || + (data_version >= DATA_VERSION_4_3_0_0 && data_version < DATA_VERSION_4_3_3_0)) { + //do nothing + } else if (OB_FAIL(check_statistic_table_writeable(ctx))) { + ret = OB_SUCCESS; + LOG_INFO("async gather stats abort because of statistic table is unwriteable"); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); + } else if (!session->is_user_session() && no_async_gather) { + //do nothing + LOG_INFO("async gather stats abort because of the trace point and not user seesion", K(session->is_user_session()), K(no_async_gather)); + } else if (is_virtual_tenant_id(tenant_id)) { + // do nothing + } else if (GCONF.in_upgrade_mode()) { + //in upgrade, don't async gather table stats + } else if (lib::is_oracle_mode() && !params.empty() && !params.at(0).is_null() && + OB_FAIL(params.at(0).get_number(num_duration))) { + LOG_WARN("failed to get duration", K(ret), K(params.at(0))); + } else if (lib::is_oracle_mode() && !params.empty() && !params.at(0).is_null() && + OB_FAIL(num_duration.extract_valid_int64_with_trunc(duration_time))) { + LOG_WARN("extract_valid_int64_with_trunc failed", K(ret), K(num_duration)); + } else if (lib::is_mysql_mode() && !params.empty() && !params.at(0).is_null() && + OB_FAIL(params.at(0).get_int(duration_time))) { + LOG_WARN("failed to get duration", K(ret), K(params.at(0))); + } else { + bool is_can_async_gather = true; + if (duration_time > 0) { + THIS_WORKER.set_timeout_ts(duration_time + ObTimeUtility::current_time()); + } + if (OB_FAIL(init_gather_task_info(ctx, ObOptStatGatherType::AYSNC_GATHER, start_time, 0, task_info))) { + LOG_WARN("failed to init gather task info", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::check_can_async_gather_stats(ctx))) { + LOG_WARN("failed to check can async gather stats", K(ret)); + is_can_async_gather = (ret != OB_ERR_DBMS_STATS_PL); + } else if (OB_FAIL(ObOptStatMonitorManager::flush_database_monitoring_info(ctx))) { + LOG_WARN("failed to flush database monitoring info", K(ret)); + } else if (OB_FAIL(async_gather_table_stats(ctx, duration_time, succeed_cnt, task_info))) { + LOG_WARN("failed to gather table stats", K(ret)); + } else {/*do nothing*/} + const int64_t exe_time = ObTimeUtility::current_time() - start_time; + LOG_INFO("have been async gathered stats job", + "the total used time:", exe_time, + "the max duration time:", duration_time, + "the toatal gather table cnt:", task_info.task_table_count_, + "the succeed to gather table cnt:", succeed_cnt, + "the failed to gather table cnt:", task_info.failed_count_, K(ret)); + //reset the error code, the reason is that the total gather time is reach the duration time. + ret = ret == OB_TIMEOUT ? OB_SUCCESS : ret; + task_info.task_end_time_ = ObTimeUtility::current_time(); + task_info.ret_code_ = is_can_async_gather ? ret : OB_ERR_QUERY_INTERRUPTED; + update_optimizer_gather_stat_info(&task_info, NULL); + } + return ret; +} + int ObDbmsStats::update_stat_cache(const uint64_t rpc_tenant_id, const ObTableStatParam ¶m, ObOptStatRunningMonitor *running_monitor/*default null*/) @@ -3219,66 +3315,78 @@ int ObDbmsStats::update_stat_cache(const uint64_t rpc_tenant_id, } } if (OB_SUCC(ret)) { - LOG_TRACE("update stat cache", K(stat_arg)); - bool evict_plan_failed = false; - int64_t timeout = -1; - ObSEArray all_server_arr; - bool has_read_only_zone = false; // UNUSED; - if (OB_ISNULL(GCTX.srv_rpc_proxy_) || OB_ISNULL(GCTX.locality_manager_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("rpc_proxy or session is null", K(ret), K(GCTX.srv_rpc_proxy_), K(GCTX.locality_manager_)); - } else if (OB_FAIL(GCTX.locality_manager_->get_server_locality_array(all_server_arr, - has_read_only_zone))) { - LOG_WARN("fail to get server locality", K(ret)); - } else { - ObSEArray failed_server_arr; - for (int64_t i = 0; OB_SUCC(ret) && i < all_server_arr.count(); i++) { - if (!all_server_arr.at(i).is_active() - || ObServerStatus::OB_SERVER_ACTIVE != all_server_arr.at(i).get_server_status() - || 0 == all_server_arr.at(i).get_start_service_time() - || 0 != all_server_arr.at(i).get_server_stop_time()) { - //server may not serving - } else if (0 >= (timeout = THIS_WORKER.get_timeout_remain())) { - ret = OB_TIMEOUT; - LOG_WARN("query timeout is reached", K(ret), K(timeout)); - } else if (OB_FAIL(GCTX.srv_rpc_proxy_->to(all_server_arr.at(i).get_addr()) - .timeout(timeout) - .by(rpc_tenant_id) - .update_local_stat_cache(stat_arg))) { - LOG_WARN("failed to update local stat cache caused by unknow error", - K(ret), K(all_server_arr.at(i).get_addr()), K(stat_arg)); - if (OB_FAIL(failed_server_arr.push_back(all_server_arr.at(i)))) { - LOG_WARN("failed to push back", K(ret)); - } + if (OB_FAIL(update_stat_cache(rpc_tenant_id, stat_arg, running_monitor))) { + LOG_WARN("failed to update stat cache", K(ret)); + } + } + return ret; +} + +int ObDbmsStats::update_stat_cache(const uint64_t tenant_id, + obrpc::ObUpdateStatCacheArg &stat_arg, + ObOptStatRunningMonitor *running_monitor/*default null*/) +{ + int ret = OB_SUCCESS; + LOG_TRACE("update stat cache", K(stat_arg)); + bool evict_plan_failed = false; + int64_t timeout = -1; + ObSEArray all_server_arr; + bool has_read_only_zone = false; // UNUSED; + if (OB_ISNULL(GCTX.srv_rpc_proxy_) || OB_ISNULL(GCTX.locality_manager_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("rpc_proxy or session is null", K(ret), K(GCTX.srv_rpc_proxy_), K(GCTX.locality_manager_)); + } else if (OB_FAIL(GCTX.locality_manager_->get_server_locality_array(all_server_arr, + has_read_only_zone))) { + LOG_WARN("fail to get server locality", K(ret)); + } else { + ObSEArray failed_server_arr; + for (int64_t i = 0; OB_SUCC(ret) && i < all_server_arr.count(); i++) { + timeout = std::min(MAX_OPT_STATS_PROCESS_RPC_TIMEOUT, THIS_WORKER.get_timeout_remain()); + if (!all_server_arr.at(i).is_active() + || ObServerStatus::OB_SERVER_ACTIVE != all_server_arr.at(i).get_server_status() + || 0 == all_server_arr.at(i).get_start_service_time() + || 0 != all_server_arr.at(i).get_server_stop_time()) { + //server may not serving + } else if (0 >=(timeout)) { + ret = OB_TIMEOUT; + LOG_WARN("query timeout is reached", K(ret), K(timeout)); + } else if (OB_FAIL(GCTX.srv_rpc_proxy_->to(all_server_arr.at(i).get_addr()) + .timeout(timeout) + .by(tenant_id) + .update_local_stat_cache(stat_arg))) { + LOG_WARN("failed to update local stat cache caused by unknow error", + K(ret), K(all_server_arr.at(i).get_addr()), K(stat_arg)); + if (OB_FAIL(failed_server_arr.push_back(all_server_arr.at(i)))) { + LOG_WARN("failed to push back", K(ret)); } } - LOG_TRACE("update stat cache", K(param), K(stat_arg), K(failed_server_arr), K(all_server_arr)); - if (OB_SUCC(ret) && !failed_server_arr.empty() && running_monitor != NULL) { - ObSqlString tmp_str; - char *buf = NULL; - if (failed_server_arr.count() * (common::MAX_IP_ADDR_LENGTH + 1) <= common::MAX_VALUE_LENGTH) { - for (int64_t i = 0; OB_SUCC(ret) && i < failed_server_arr.count(); ++i) { - char svr_buf[common::MAX_IP_ADDR_LENGTH] = {0}; - failed_server_arr.at(i).get_addr().to_string(svr_buf, common::MAX_IP_ADDR_LENGTH); - if (OB_FAIL(tmp_str.append_fmt("%s%s", svr_buf, i == 0 ? "" : ","))) { - LOG_WARN("failed to append fmt", K(ret)); - } + } + LOG_TRACE("update stat cache", K(stat_arg), K(failed_server_arr), K(all_server_arr)); + if (OB_SUCC(ret) && !failed_server_arr.empty() && running_monitor != NULL) { + ObSqlString tmp_str; + char *buf = NULL; + if (failed_server_arr.count() * (common::MAX_IP_ADDR_LENGTH + 1) <= common::MAX_VALUE_LENGTH) { + for (int64_t i = 0; OB_SUCC(ret) && i < failed_server_arr.count(); ++i) { + char svr_buf[common::MAX_IP_ADDR_LENGTH] = {0}; + failed_server_arr.at(i).get_addr().to_string(svr_buf, common::MAX_IP_ADDR_LENGTH); + if (OB_FAIL(tmp_str.append_fmt("%s%s", svr_buf, i == 0 ? "" : ","))) { + LOG_WARN("failed to append fmt", K(ret)); } - } else if (OB_FAIL(tmp_str.append_fmt("more than %ld servers refresh stat cache failed", - failed_server_arr.count()))) { - LOG_WARN("failed to append fmt", K(ret)); - } - if (OB_FAIL(ret)) { - //do nothing - } else if (OB_ISNULL(buf = static_cast(running_monitor->allocator_.alloc(tmp_str.length())))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("memory is not enough", K(ret), K(tmp_str)); - } else { - MEMCPY(buf, tmp_str.ptr(), tmp_str.length()); - ObString tmp_failed_list(tmp_str.length(), buf); - ObOptStatGatherStatList::instance().update_gather_stat_refresh_failed_list(tmp_failed_list, - running_monitor->opt_stat_gather_stat_); } + } else if (OB_FAIL(tmp_str.append_fmt("more than %ld servers refresh stat cache failed", + failed_server_arr.count()))) { + LOG_WARN("failed to append fmt", K(ret)); + } + if (OB_FAIL(ret)) { + //do nothing + } else if (OB_ISNULL(buf = static_cast(running_monitor->allocator_.alloc(tmp_str.length())))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("memory is not enough", K(ret), K(tmp_str)); + } else { + MEMCPY(buf, tmp_str.ptr(), tmp_str.length()); + ObString tmp_failed_list(tmp_str.length(), buf); + ObOptStatGatherStatList::instance().update_gather_stat_refresh_failed_list(tmp_failed_list, + running_monitor->opt_stat_gather_stat_); } } } @@ -4019,6 +4127,8 @@ int ObDbmsStats::parse_gather_stat_options(ObExecContext &ctx, const ObObjParam &cascade, const ObObjParam &no_invalidate, const ObObjParam &force, + const ObObjParam *hist_est_percent, + const ObObjParam *hist_block_sample, ObTableStatParam ¶m) { int ret = OB_SUCCESS; @@ -4122,6 +4232,41 @@ int ObDbmsStats::parse_gather_stat_options(ObExecContext &ctx, } } + if (OB_SUCC(ret)) { + if (hist_est_percent != NULL) { + double percent = 0.0; + number::ObNumber num_hist_est_percent; + if (hist_est_percent->is_null()) { + param.hist_sample_info_.set_percent(100.0); + } else if (OB_FAIL(hist_est_percent->get_number(num_hist_est_percent))) { + LOG_WARN("failed to get number", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::cast_number_to_double(num_hist_est_percent, percent))) { + LOG_WARN("failed to cast number to double" , K(ret)); + } else if (percent == 0.0) { + stat_options |= StatOptionFlags::OPT_HIST_EST_PERCENT; + } else if (OB_UNLIKELY(percent < 0.000001 || percent > 100.0)) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("Illegal sample percent: must be in the range[0.000001,100]", K(ret)); + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL, "Illegal sample percent: must be in the range[0.000001,100]"); + } else { + param.hist_sample_info_.set_percent(percent); + } + } + } + + if (OB_SUCC(ret)) { + if (hist_block_sample != NULL) { + bool is_block_sample = false; + if (hist_block_sample->is_null()) { + stat_options |= StatOptionFlags::OPT_HIST_BLOCK_SAMPLE; + } else if (OB_FAIL(hist_block_sample->get_bool(is_block_sample))) { + LOG_WARN("failed to get block sample", K(ret)); + } else { + param.hist_sample_info_.set_is_block_sample(is_block_sample); + } + } + } + if (OB_SUCC(ret)) { if (stat_options > 0 && OB_FAIL(get_default_stat_options(ctx, stat_options, param))) { LOG_WARN("failed to get default stat options", K(ret)); @@ -4133,7 +4278,6 @@ int ObDbmsStats::parse_gather_stat_options(ObExecContext &ctx, } int ObDbmsStats::use_default_gather_stat_options(ObExecContext &ctx, - const StatTable &stat_table, ObTableStatParam ¶m) { int ret = OB_SUCCESS; @@ -4243,6 +4387,38 @@ int ObDbmsStats::get_default_stat_options(ObExecContext &ctx, LOG_WARN("failed to push back", K(ret)); } } + if (OB_SUCC(ret) && param.is_async_gather_ && stat_options & StatOptionFlags::OPT_ASYNC_GATHER_SAMPLE_SIZE) { + ObAsyncGatherSampleSizePrefs *tmp_pref = NULL; + if (OB_FAIL(new_stat_prefs(*param.allocator_, ctx.get_my_session(), ObString(), tmp_pref))) { + LOG_WARN("failed to new stat prefs", K(ret)); + } else if (OB_FAIL(stat_prefs.push_back(tmp_pref))) { + LOG_WARN("failed to push back", K(ret)); + } + } + if (OB_SUCC(ret) && param.is_async_gather_ && stat_options & StatOptionFlags::OPT_ASYNC_GATHER_FULL_TABLE_SIZE) { + ObAsyncGatherFullTableSizePrefs *tmp_pref = NULL; + if (OB_FAIL(new_stat_prefs(*param.allocator_, ctx.get_my_session(), ObString(), tmp_pref))) { + LOG_WARN("failed to new stat prefs", K(ret)); + } else if (OB_FAIL(stat_prefs.push_back(tmp_pref))) { + LOG_WARN("failed to push back", K(ret)); + } + } + if (OB_SUCC(ret) && stat_options & StatOptionFlags::OPT_HIST_EST_PERCENT) { + ObHistEstPercentPrefs *tmp_pref = NULL; + if (OB_FAIL(new_stat_prefs(*param.allocator_, ctx.get_my_session(), ObString(), tmp_pref))) { + LOG_WARN("failed to new stat prefs", K(ret)); + } else if (OB_FAIL(stat_prefs.push_back(tmp_pref))) { + LOG_WARN("failed to push back", K(ret)); + } + } + if (OB_SUCC(ret) && stat_options & StatOptionFlags::OPT_HIST_BLOCK_SAMPLE) { + ObHistBlockSamplePrefs *tmp_pref = NULL; + if (OB_FAIL(new_stat_prefs(*param.allocator_, ctx.get_my_session(), ObString(), tmp_pref))) { + LOG_WARN("failed to new stat prefs", K(ret)); + } else if (OB_FAIL(stat_prefs.push_back(tmp_pref))) { + LOG_WARN("failed to push back", K(ret)); + } + } if (OB_SUCC(ret)) { if (OB_FAIL(ObDbmsStatsPreferences::get_sys_default_stat_options(ctx, stat_prefs, param))) { LOG_WARN("failed to get sys default stat options", K(ret)); @@ -4276,6 +4452,7 @@ int ObDbmsStats::parse_granularity_and_method_opt(ObExecContext &ctx, if (OB_FAIL(ObDbmsStats::parse_method_opt(ctx, param.allocator_, param.column_params_, param.method_opt_, + param.is_async_gather_, use_size_auto))) { LOG_WARN("failed to parse method opt", K(ret)); } @@ -4454,6 +4631,7 @@ int ObDbmsStats::parse_method_opt(sql::ObExecContext &ctx, ObIAllocator *allocator, ObIArray &column_params, const ObString &method_opt, + const bool is_async_gather, bool &use_size_auto) { int ret = OB_SUCCESS; @@ -4487,7 +4665,7 @@ int ObDbmsStats::parse_method_opt(sql::ObExecContext &ctx, ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret), K(child_node)); } else if (T_FOR_ALL == child_node->type_) { - if (OB_FAIL(parser_for_all_clause(child_node, column_params, use_size_auto))) { + if (OB_FAIL(parser_for_all_clause(child_node, column_params, is_async_gather, use_size_auto))) { LOG_WARN("failed to parser for all clause", K(ret)); } else {/*do nothing*/} } else if (T_FOR_COLUMNS == child_node->type_) { @@ -4505,6 +4683,7 @@ int ObDbmsStats::parse_method_opt(sql::ObExecContext &ctx, int ObDbmsStats::parser_for_all_clause(const ParseNode *for_all_node, ObIArray &column_params, + const bool is_async_gather, bool &use_size_auto) { int ret = OB_SUCCESS; @@ -4534,6 +4713,9 @@ int ObDbmsStats::parser_for_all_clause(const ParseNode *for_all_node, LOG_WARN("failed to parse size clause", K(ret)); } else { use_size_auto = size_conf.is_auto(); + if (is_async_gather && size_conf.is_auto()) {//async gather don't gather histogram default + size_conf.set_manual(1); + } } } for (int64_t i = 0; OB_SUCC(ret) && i < column_params.count(); ++i) { @@ -5448,6 +5630,8 @@ int ObDbmsStats::gather_database_stats_job_proc(sql::ObExecContext &ctx, //do nothing LOG_INFO("auto gather stat abort because of the trace point and not user seesion", K(ctx.get_my_session()->is_user_session()), K(no_auto_gather)); + } else if (OB_FAIL(ObDbmsStatsUtils::cancel_async_gather_stats(ctx))) { + LOG_WARN("failed to cancel async gather stats", K(ret)); } else if (lib::is_oracle_mode() && !params.empty() && !params.at(0).is_null() && OB_FAIL(params.at(0).get_number(num_duration))) { LOG_WARN("failed to get duration", K(ret), K(params.at(0))); @@ -5567,18 +5751,26 @@ int ObDbmsStats::do_gather_table_stats(sql::ObExecContext &ctx, // 1. user table // 2. valid sys table // 3. virtual table + if (OB_LIKELY(task_info.task_table_count_ > 0)) { + -- task_info.task_table_count_; + } } else if (OB_FAIL(schema_guard->get_table_schema(tenant_id, table_id, table_schema))) { LOG_WARN("failed to get table schema", K(ret)); } else if (OB_ISNULL(table_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret)); + //table may be droped during auto table statistic gathering, caller should ignore this err code + ret = OB_TABLE_NOT_EXIST; + if (OB_LIKELY(task_info.task_table_count_ > 0)) { + -- task_info.task_table_count_; + } } else if (is_recyclebin_database_id(table_schema->get_database_id()) || (lib::is_oracle_mode() && is_oceanbase_sys_database_id(table_schema->get_database_id()))) { - //do nothing + if (OB_LIKELY(task_info.task_table_count_ > 0)) { + -- task_info.task_table_count_; + } } else { StatTable stat_table(table_schema->get_database_id(), table_id); double stale_percent_threshold = OPT_DEFAULT_STALE_PERCENT; - if (OB_FAIL(get_table_stale_percent_threshold(ctx, + if (OB_FAIL(get_table_stale_percent_threshold(ctx.get_sql_proxy(), tenant_id, table_schema->get_table_id(), stale_percent_threshold))) { @@ -5662,7 +5854,7 @@ int ObDbmsStats::get_common_table_stale_percent(sql::ObExecContext &ctx, uint64_t table_id = share::is_oracle_mapping_real_virtual_table(table_schema.get_table_id()) ? share::get_real_table_mappings_tid(table_schema.get_table_id()) : table_schema.get_table_id(); - const int64_t part_id = PARTITION_LEVEL_ZERO == table_schema.get_part_level() ? table_id : -1; + const int64_t part_id = PARTITION_LEVEL_ZERO == table_schema.get_part_level() ? table_schema.get_table_id() : -1; int64_t inc_modified_count = 0; int64_t row_cnt = 0; if (OB_UNLIKELY(table_schema.is_user_table() && -1 == part_id)) { @@ -5755,6 +5947,8 @@ int ObDbmsStats::gather_table_stats_with_default_param(ObExecContext &ctx, ObTableStatParam stat_param; stat_param.allocator_ = &tmp_alloc; stat_param.db_id_ = stat_table.database_id_; + stat_param.is_async_gather_ = stat_table.is_async_gather_; + stat_param.async_partition_ids_ = &stat_table.async_partition_ids_; bool is_all_fast_gather = false; ObSEArray no_gather_index_ids; ObOptStatGatherStat gather_stat(task_info); @@ -5769,9 +5963,13 @@ int ObDbmsStats::gather_table_stats_with_default_param(ObExecContext &ctx, LOG_WARN("failed to get valid duration time", K(ret)); } else if (OB_FAIL(parse_table_part_info(ctx, stat_table, stat_param, true))) { LOG_WARN("failed to parse owner", K(ret)); - } else if (OB_FAIL(use_default_gather_stat_options(ctx, stat_table, stat_param))) { + } else if (OB_FAIL(use_default_gather_stat_options(ctx, stat_param))) { LOG_WARN("failed to use default gather stat optitions", K(ret)); - } else if (OB_FAIL(adjust_auto_gather_stat_option(stat_table.partition_stat_infos_, stat_param))) { + } else if (!stat_table.is_async_gather_ && + OB_FAIL(adjust_auto_gather_stat_option(stat_table.partition_stat_infos_, stat_param))) { + LOG_WARN("failed to use default gather stat optitions", K(ret)); + } else if (stat_table.is_async_gather_ && + OB_FAIL(adjust_async_gather_stat_option(ctx, stat_table.async_partition_ids_, stat_param))) { LOG_WARN("failed to use default gather stat optitions", K(ret)); } else if (!stat_param.need_gather_stats()) { //do nothing @@ -5933,6 +6131,48 @@ int ObDbmsStats::get_new_stat_pref(ObExecContext &ctx, } else { stat_pref = tmp_pref; } + } else if (0 == opt_name.case_compare("ASYNC_GATHER_STALE_RATIO")) { + ObAsyncGatherStaleRatioPrefs *tmp_pref = NULL; + if (OB_FAIL(new_stat_prefs(allocator, ctx.get_my_session(), opt_value, tmp_pref))) { + LOG_WARN("failed to new stat prefs", K(ret)); + } else { + stat_pref = tmp_pref; + } + } else if (0 == opt_name.case_compare("ASYNC_GATHER_SAMPLE_SIZE")) { + ObAsyncGatherSampleSizePrefs *tmp_pref = NULL; + if (OB_FAIL(new_stat_prefs(allocator, ctx.get_my_session(), opt_value, tmp_pref))) { + LOG_WARN("failed to new stat prefs", K(ret)); + } else { + stat_pref = tmp_pref; + } + } else if (0 == opt_name.case_compare("ASYNC_GATHER_FULL_TABLE_SIZE")) { + ObAsyncGatherFullTableSizePrefs *tmp_pref = NULL; + if (OB_FAIL(new_stat_prefs(allocator, ctx.get_my_session(), opt_value, tmp_pref))) { + LOG_WARN("failed to new stat prefs", K(ret)); + } else { + stat_pref = tmp_pref; + } + } else if (0 == opt_name.case_compare("ASYNC_STALE_MAX_TABLE_SIZE")) { + ObAsyncStaleMaxTableSizePrefs *tmp_pref = NULL; + if (OB_FAIL(new_stat_prefs(allocator, ctx.get_my_session(), opt_value, tmp_pref))) { + LOG_WARN("failed to new stat prefs", K(ret)); + } else { + stat_pref = tmp_pref; + } + } else if (0 == opt_name.case_compare("HIST_EST_PERCENT")) { + ObHistEstPercentPrefs *tmp_pref = NULL; + if (OB_FAIL(new_stat_prefs(allocator, ctx.get_my_session(), opt_value, tmp_pref))) { + LOG_WARN("failed to new stat prefs", K(ret)); + } else { + stat_pref = tmp_pref; + } + } else if (0 == opt_name.case_compare("HIST_BLOCK_SAMPLE")) { + ObHistBlockSamplePrefs *tmp_pref = NULL; + if (OB_FAIL(new_stat_prefs(allocator, ctx.get_my_session(), opt_value, tmp_pref))) { + LOG_WARN("failed to new stat prefs", K(ret)); + } else { + stat_pref = tmp_pref; + } } else if (0 == opt_name.case_compare("ONLINE_ESTIMATE_PERCENT")) { ObOnlineEstimatePercentPrefs *tmp_pref = NULL; if (OB_FAIL(new_stat_prefs(allocator, ctx.get_my_session(), opt_value, tmp_pref))) { @@ -5944,15 +6184,17 @@ int ObDbmsStats::get_new_stat_pref(ObExecContext &ctx, ret = OB_ERR_DBMS_STATS_PL; LOG_WARN("Invalid input values for pname", K(ret), K(opt_name)); LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL, "Invalid input values for pname, Only Support CASCADE |"\ - "DEGREE | ESTIMATE_PERCENT | GRANULARITY | INCREMENTAL |"\ - "INCREMENTAL_LEVEL | METHOD_OPT | NO_INVALIDATE | OPTIONS |"\ - "STALE_PERCENT | ESTIMATE_BLOCK | BLOCK_SAMPLE |"\ - "APPROXIMATE_NDV(global prefs unique) | ONLINE_ESTIMATE_PERCENT prefs"); + "DEGREE | ESTIMATE_PERCENT | GRANULARITY | INCREMENTAL |"\ + "INCREMENTAL_LEVEL | METHOD_OPT | NO_INVALIDATE | OPTIONS |"\ + "STALE_PERCENT | ESTIMATE_BLOCK | ASYNC_GATHER_STALE_RATIO |"\ + "ASYNC_GATHER_SAMPLE_SIZE | ASYNC_GATHER_FULL_TABLE_SIZE |"\ + "ASYNC_STALE_MAX_TABLE_SIZE | HIST_EST_PERCENT | HIST_BLOCK_SAMPLE |"\ + "APPROXIMATE_NDV(global prefs unique) | ONLINE_ESTIMATE_PERCENT prefs"); } return ret; } -int ObDbmsStats::get_table_stale_percent_threshold(sql::ObExecContext &ctx, +int ObDbmsStats::get_table_stale_percent_threshold(ObMySQLProxy *mysql_proxy, const uint64_t tenant_id, const uint64_t table_id, double &stale_percent_threshold) @@ -5962,14 +6204,12 @@ int ObDbmsStats::get_table_stale_percent_threshold(sql::ObExecContext &ctx, ObTableStatParam param; ObString opt_name("STALE_PERCENT"); ObArenaAllocator tmp_alloc("OptStatPrefs", OB_MALLOC_NORMAL_BLOCK_SIZE, tenant_id); - param.tenant_id_ = tenant_id; - param.table_id_ = table_id; - param.allocator_ = &tmp_alloc; - if (OB_FAIL(ObDbmsStatsPreferences::get_prefs(ctx, param, opt_name, result))) { + if (OB_FAIL(ObDbmsStatsPreferences::get_prefs(mysql_proxy, tmp_alloc, + tenant_id, table_id, + opt_name, result))) { LOG_WARN("failed to get prefs", K(ret)); } else if (!result.is_null()) { - ObArenaAllocator calc_buf(ObModIds::OB_SQL_PARSER); - ObCastCtx cast_ctx(&calc_buf, NULL, CM_NONE, ObCharset::get_system_collation()); + ObCastCtx cast_ctx(&tmp_alloc, NULL, CM_NONE, ObCharset::get_system_collation()); ObObj dest_obj; if (OB_FAIL(ObObjCaster::to_type(ObDoubleType, cast_ctx, result, dest_obj))) { LOG_WARN("failed to cast number to double type", K(ret)); @@ -6208,9 +6448,7 @@ int ObDbmsStats::resovle_granularity(ObGranularityType granu_type, ObTableStatParam ¶m) { int ret = OB_SUCCESS; - bool is_specify_sample = param.sample_info_.is_sample_ && - param.sample_info_.sample_value_ >= 0.000001 && - param.sample_info_.sample_value_ < 100.0; + bool is_specify_sample = param.is_specify_sample(); if (ObGranularityType::GRANULARITY_AUTO == granu_type) { param.global_stat_param_.set_gather_stat(param.part_name_.empty() && !is_specify_sample); param.part_stat_param_.set_gather_stat(param.part_name_.empty() && @@ -6870,5 +7108,208 @@ void ObDbmsStats::update_optimizer_gather_stat_info(const ObOptStatTaskInfo *tas THIS_WORKER.set_timeout_ts(origin_timeout); } +int ObDbmsStats::async_gather_table_stats(sql::ObExecContext &ctx, + const int64_t duration_time, + int64_t &succeed_cnt, + ObOptStatTaskInfo &task_info) +{ + int ret = OB_SUCCESS; + ObSQLSessionInfo *session = ctx.get_my_session(); + uint64_t tenant_id = OB_INVALID_ID; + if (OB_ISNULL(session)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(session)); + } else if (OB_FALSE_IT(tenant_id = session->get_effective_tenant_id())) { + } else if (is_virtual_tenant_id(tenant_id)) { + // do nothing + } else if (GCONF.in_upgrade_mode()) { + //in upgrade, don't async gather table stats + } else { + int64_t slice_cnt = 1000; // maximum tables we can async gather stats at each iteration + ObSEArray async_stat_tables; + do { + async_stat_tables.reuse(); + if (OB_FAIL(THIS_WORKER.check_status())) { + LOG_WARN("check status failed", KR(ret)); + } else if (OB_FAIL(ObBasicStatsEstimator::get_async_gather_stats_tables(ctx, tenant_id, slice_cnt, async_stat_tables))) { + LOG_WARN("failed to get async gather stats tables", K(ret)); + } else { + task_info.task_table_count_ += async_stat_tables.count(); + for (int64_t i = 0; OB_SUCC(ret) && i < async_stat_tables.count(); ++i) { + if (OB_FAIL(refresh_tenant_schema_guard(ctx, tenant_id))) { + LOG_WARN("refresh tenant schema guard failed", K(ret)); + } else if (OB_FAIL(do_async_gather_table_stats(ctx, tenant_id, async_stat_tables.at(i), + duration_time, succeed_cnt, task_info))) { + LOG_WARN("failed to do async gather table stats", K(ret)); + } + } + } + } while (OB_SUCC(ret) && async_stat_tables.count() == slice_cnt); + } + return ret; +} + +int ObDbmsStats::do_async_gather_table_stats(sql::ObExecContext &ctx, + const uint64_t tenant_id, + const AsyncStatTable &async_table, + const int64_t duration_time, + int64_t &succeed_cnt, + ObOptStatTaskInfo &task_info) +{ + int ret = OB_SUCCESS; + bool is_valid = false; + const ObTableSchema *table_schema = NULL; + share::schema::ObSchemaGetterGuard *schema_guard = ctx.get_virtual_table_ctx().schema_guard_; + if (OB_ISNULL(schema_guard)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(schema_guard)); + } else if (OB_FAIL(ObDbmsStatsUtils::check_is_stat_table(*schema_guard, tenant_id, + async_table.table_id_, is_valid))) { + LOG_WARN("failed to check sy table validity", K(ret)); + } else if (!is_valid) { + // only gather statistics for following tables: + // 1. user table + // 2. valid sys table + // 3. virtual table + if (OB_LIKELY(task_info.task_table_count_ > 0)) { + -- task_info.task_table_count_; + } + } else if (OB_FAIL(schema_guard->get_table_schema(tenant_id, async_table.table_id_, table_schema))) { + LOG_WARN("failed to get table schema", K(ret)); + } else if (OB_ISNULL(table_schema)) { + //table may be droped during auto table statistic gathering, caller should ignore this err code + ret = OB_TABLE_NOT_EXIST; + if (OB_LIKELY(task_info.task_table_count_ > 0)) { + -- task_info.task_table_count_; + } + } else if (is_recyclebin_database_id(table_schema->get_database_id()) || + (lib::is_oracle_mode() && is_oceanbase_sys_database_id(table_schema->get_database_id()))) { + if (OB_LIKELY(task_info.task_table_count_ > 0)) { + -- task_info.task_table_count_; + } + } else { + //begin async gather table stats + StatTable stat_table(table_schema->get_database_id(), async_table.table_id_, true/*is_async_gather*/); + if (OB_FAIL(append(stat_table.async_partition_ids_, async_table.partition_ids_))) { + LOG_WARN("failed to append", K(ret)); + } else if (OB_FAIL(gather_table_stats_with_default_param(ctx, duration_time, stat_table, task_info))) { + LOG_WARN("failed to gather table stats with default param", K(ret)); + } + if (OB_FAIL(ret)) { + if (OB_ERR_QUERY_INTERRUPTED == ret) { + LOG_WARN("query interrupted", K(ret)); + } else if (OB_TABLE_NOT_EXIST == ret || OB_TIMEOUT == ret) { + ++task_info.failed_count_; + // do nothing + ret = OB_SUCCESS; + } else { + ++task_info.failed_count_; + LOG_WARN("failed to gather table stats with some unknown reason", K(ret)); + ret = OB_SUCCESS; + } + } else { + ++succeed_cnt; + } + } + return ret; +} + +int ObDbmsStats::adjust_async_gather_stat_option(ObExecContext &ctx, + const ObIArray &async_partition_ids, + ObTableStatParam ¶m) +{ + int ret = OB_SUCCESS; + ObSEArray approx_first_part_ids; + //If the value of async_full_table_size_ is 0, it means that the table no need to async gather stats. + if (param.async_full_table_size_ == 0) { + param.subpart_stat_param_.reset_gather_stat(); + param.part_stat_param_.reset_gather_stat(); + param.global_stat_param_.reset_gather_stat(); + } + if (param.subpart_stat_param_.need_modify_) { + ObSEArray new_subpart_infos; + for (int64_t i = 0; OB_SUCC(ret) && i < param.subpart_infos_.count(); ++i) { + bool found_it = false; + int64_t first_part_id = 0; + for (int64_t j = 0; OB_SUCC(ret) && !found_it && j < async_partition_ids.count(); ++j) { + if (async_partition_ids.at(j) == param.subpart_infos_.at(i).part_id_) { + if (OB_FAIL(new_subpart_infos.push_back(param.subpart_infos_.at(i)))) { + LOG_WARN("failed to push back", K(ret)); + } else { + found_it = true; + first_part_id = param.subpart_infos_.at(i).first_part_id_; + } + } + } + if (OB_SUCC(ret)) { + if (found_it) {//check first partition id need approx regather + bool has_it = false; + for (int64_t j = 0; !has_it && j < approx_first_part_ids.count(); ++j) { + has_it = (first_part_id == approx_first_part_ids.at(j)); + } + if (!has_it) { + if (OB_FAIL(add_var_to_array_no_dup(approx_first_part_ids, first_part_id))) { + LOG_WARN("failed to add var to array no dup", K(ret)); + } + } + } else if (OB_FAIL(param.no_regather_partition_ids_.push_back(param.subpart_infos_.at(i).part_id_))) { + LOG_WARN("failed to push back", K(ret)); + } + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(param.subpart_infos_.assign(new_subpart_infos))) { + LOG_WARN("failed to assign", K(ret)); + } else { + param.subpart_stat_param_.need_modify_ = !new_subpart_infos.empty(); + } + } + } + if (OB_SUCC(ret) && param.part_stat_param_.need_modify_) { + ObSEArray new_part_infos; + for (int64_t i = 0; OB_SUCC(ret) && i < param.part_infos_.count(); ++i) { + bool gather_part = false; + bool approx_found_it = false; + for (int64_t j = 0; OB_SUCC(ret) && !approx_found_it && j < approx_first_part_ids.count(); ++j) { + if (approx_first_part_ids.at(j) == param.part_infos_.at(i).part_id_) { + approx_found_it = true; + if (param.part_stat_param_.can_use_approx_ && + param.subpart_stat_param_.need_modify_ && + param.part_level_ == share::schema::ObPartitionLevel::PARTITION_LEVEL_TWO) { + if (OB_FAIL(param.approx_part_infos_.push_back(param.part_infos_.at(i)))) { + LOG_WARN("failed to push back", K(ret)); + } else { + gather_part = true; + } + } else {/*do nothing*/} + } + } + for (int64_t j = 0; OB_SUCC(ret) && !gather_part && j < async_partition_ids.count(); ++j) { + if (async_partition_ids.at(j) == param.part_infos_.at(i).part_id_) { + gather_part = true; + if (OB_FAIL(new_part_infos.push_back(param.part_infos_.at(i)))) { + LOG_WARN("failed to push back", K(ret)); + } else {/*do nothing*/} + } + } + if (OB_SUCC(ret) && !gather_part) { + if (OB_FAIL(param.no_regather_partition_ids_.push_back(param.part_infos_.at(i).part_id_))) { + LOG_WARN("failed to push back", K(ret)); + } + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(param.part_infos_.assign(new_part_infos))) { + LOG_WARN("failed to assign", K(ret)); + } else { + param.part_stat_param_.can_use_approx_ = !param.approx_part_infos_.empty(); + param.part_stat_param_.need_modify_ = !new_part_infos.empty() || !param.approx_part_infos_.empty(); + } + } + } + LOG_TRACE("succeed to adjust auto gather stat option", K(async_partition_ids), K(param)); + return ret; +} + } } diff --git a/src/pl/sys_package/ob_dbms_stats.h b/src/pl/sys_package/ob_dbms_stats.h index f5b3ab9bb..c81fd0369 100644 --- a/src/pl/sys_package/ob_dbms_stats.h +++ b/src/pl/sys_package/ob_dbms_stats.h @@ -50,6 +50,7 @@ struct MethodOptSizeConf inline bool is_repeat() const { return mode_ == 0 && val_ == 1; } inline bool is_skewonly() const { return mode_ == 0 && val_ == 2; } inline bool is_manual() const {return mode_ == 1; } + inline void set_manual(int32_t bucket_size) { mode_ = 1; val_ = bucket_size; } int32_t mode_; int32_t val_; @@ -246,14 +247,20 @@ public: sql::ParamStore ¶ms, common::ObObj &result); + static int async_gather_stats_job_proc(sql::ObExecContext &ctx, + sql::ParamStore ¶ms, + common::ObObj &result); + static int parse_method_opt(sql::ObExecContext &ctx, ObIAllocator *allocator, ObIArray &column_params, const ObString &method_opt, + const bool is_async_gather, bool &use_size_auto); static int parser_for_all_clause(const ParseNode *for_all_node, ObIArray &column_params, + const bool is_async_gather, bool &use_size_auto); static int parser_for_columns_clause(const ParseNode *for_col_node, @@ -340,10 +347,11 @@ public: const ObObjParam &cascade, const ObObjParam &no_invalidate, const ObObjParam &force, + const ObObjParam *hist_est_percent, + const ObObjParam *hist_block_sample, ObTableStatParam ¶m); static int use_default_gather_stat_options(ObExecContext &ctx, - const StatTable &stat_table, ObTableStatParam ¶m); static int get_default_stat_options(ObExecContext &ctx, @@ -396,6 +404,10 @@ public: const ObTableStatParam ¶m, ObOptStatRunningMonitor *running_monitor = NULL); + static int update_stat_cache(const uint64_t tenant_id, + obrpc::ObUpdateStatCacheArg &stat_arg, + ObOptStatRunningMonitor *running_monitor = NULL); + static int parse_set_table_stat_options(ObExecContext &ctx, const ObObjParam &stattab, const ObObjParam &statid, @@ -499,7 +511,7 @@ public: int64_t task_table_count, ObOptStatTaskInfo &task_info); - static int get_table_stale_percent_threshold(sql::ObExecContext &ctx, + static int get_table_stale_percent_threshold(ObMySQLProxy *mysql_proxy, const uint64_t tenant_id, const uint64_t table_id, double &stale_percent_threshold); @@ -626,6 +638,21 @@ private: static int check_system_stat_table_ready(int64_t tenant_id); + static int async_gather_table_stats(sql::ObExecContext &ctx, + const int64_t duration_time, + int64_t &succeed_cnt, + ObOptStatTaskInfo &task_info); + + static int do_async_gather_table_stats(sql::ObExecContext &ctx, + const uint64_t tenant_id, + const AsyncStatTable &async_table, + const int64_t duration_time, + int64_t &succeed_cnt, + ObOptStatTaskInfo &task_info); + + static int adjust_async_gather_stat_option(ObExecContext &ctx, + const ObIArray &async_partition_ids, + ObTableStatParam ¶m); }; } diff --git a/src/rootserver/ob_ddl_operator.cpp b/src/rootserver/ob_ddl_operator.cpp index d56cc0569..de500493b 100644 --- a/src/rootserver/ob_ddl_operator.cpp +++ b/src/rootserver/ob_ddl_operator.cpp @@ -5703,6 +5703,10 @@ int ObDDLOperator::init_tenant_env( LOG_WARN("insert default tablegroup failed", K(tenant_id), K(ret)); } else if (OB_FAIL(init_tenant_databases(tenant_schema, sys_variable, trans))) { LOG_WARN("insert default databases failed,", K(tenant_id), K(ret)); + } else if (OB_FAIL(init_tenant_optimizer_stats_info(sys_variable, tenant_id, trans))) { + LOG_WARN("failed to init tenant optimizer stats info", K(tenant_id), K(ret)); + } else if (OB_FAIL(init_tenant_spm_configure(tenant_id, trans))) { + LOG_WARN("failed to init tenant spm configure", K(tenant_id), K(ret)); } else if (OB_FAIL(init_tenant_profile(tenant_id, sys_variable, trans))) { LOG_WARN("fail to init tenant profile", K(tenant_id), K(ret)); } else if (OB_FAIL(init_tenant_users(tenant_schema, sys_variable, trans))) { @@ -5895,10 +5899,6 @@ int ObDDLOperator::init_tenant_databases(const ObTenantSchema &tenant_schema, OB_PUBLIC_SCHEMA_ID, "public schema", trans, is_oracle_mode))) { RS_LOG(WARN, "insert public schema failed", K(tenant_id), K(ret)); - } else if (OB_FAIL(init_tenant_optimizer_stats_info(sys_variable, tenant_id, trans))) { - RS_LOG(WARN, "init tenant tenant optimizer control table", K(tenant_id), K(ret)); - } else if (OB_FAIL(init_tenant_spm_configure(tenant_id, trans))) { - RS_LOG(WARN, "init tenant spm configure failed", K(tenant_id), K(ret)); } else { if (!is_oracle_mode) { if (OB_FAIL(init_tenant_database(tenant_schema, mysql_schema, diff --git a/src/share/inner_table/ob_inner_table_schema.21351_21400.cpp b/src/share/inner_table/ob_inner_table_schema.21351_21400.cpp index 8dc49828d..cc47533d5 100644 --- a/src/share/inner_table/ob_inner_table_schema.21351_21400.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21351_21400.cpp @@ -1160,7 +1160,7 @@ int ObInnerTableSchema::gv_ob_opt_stat_gather_monitor_schema(ObTableSchema &tabl table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT CAST(TENANT_ID AS SIGNED) AS TENANT_ID, CAST(SVR_IP AS CHAR(46)) AS SVR_IP, CAST(SVR_PORT AS SIGNED) AS SVR_PORT, CAST(SESSION_ID AS SIGNED) AS SESSION_ID, CAST(TRACE_ID AS CHAR(64)) AS TRACE_ID, CAST(TASK_ID AS CHAR(36)) AS TASK_ID, CAST((CASE WHEN TYPE = 0 THEN 'MANUAL GATHER' ELSE (CASE WHEN TYPE = 1 THEN 'AUTO GATHER' ELSE 'UNDEFINED GATHER' END) END) AS CHAR(16)) AS TYPE, CAST(TASK_START_TIME AS DATETIME(6)) AS TASK_START_TIME, CAST(TASK_DURATION_TIME AS SIGNED) AS TASK_DURATION_TIME, CAST(TASK_TABLE_COUNT AS SIGNED) AS TASK_TABLE_COUNT, CAST(COMPLETED_TABLE_COUNT AS SIGNED) AS COMPLETED_TABLE_COUNT, CAST(RUNNING_TABLE_OWNER AS CHAR(128)) AS RUNNING_TABLE_OWNER, CAST(RUNNING_TABLE_NAME AS CHAR(256)) AS RUNNING_TABLE_NAME, CAST(RUNNING_TABLE_DURATION_TIME AS SIGNED) AS RUNNING_TABLE_DURATION_TIME, CAST(SPARE2 AS CHAR(256)) AS RUNNING_TABLE_PROGRESS FROM oceanbase.__all_virtual_opt_stat_gather_monitor )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT CAST(TENANT_ID AS SIGNED) AS TENANT_ID, CAST(SVR_IP AS CHAR(46)) AS SVR_IP, CAST(SVR_PORT AS SIGNED) AS SVR_PORT, CAST(SESSION_ID AS SIGNED) AS SESSION_ID, CAST(TRACE_ID AS CHAR(64)) AS TRACE_ID, CAST(TASK_ID AS CHAR(36)) AS TASK_ID, CAST((CASE WHEN TYPE = 0 THEN 'MANUAL GATHER' ELSE (CASE WHEN TYPE = 1 THEN 'AUTO GATHER' ELSE (CASE WHEN TYPE = 2 THEN 'ASYNC GATHER' ELSE 'UNDEFINED GATHER' END) END) END) AS CHAR(16)) AS TYPE, CAST(TASK_START_TIME AS DATETIME(6)) AS TASK_START_TIME, CAST(TASK_DURATION_TIME AS SIGNED) AS TASK_DURATION_TIME, CAST(TASK_TABLE_COUNT AS SIGNED) AS TASK_TABLE_COUNT, CAST(COMPLETED_TABLE_COUNT AS SIGNED) AS COMPLETED_TABLE_COUNT, CAST(RUNNING_TABLE_OWNER AS CHAR(128)) AS RUNNING_TABLE_OWNER, CAST(RUNNING_TABLE_NAME AS CHAR(256)) AS RUNNING_TABLE_NAME, CAST(RUNNING_TABLE_DURATION_TIME AS SIGNED) AS RUNNING_TABLE_DURATION_TIME, CAST(SPARE2 AS CHAR(256)) AS RUNNING_TABLE_PROGRESS FROM oceanbase.__all_virtual_opt_stat_gather_monitor )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -1260,7 +1260,7 @@ int ObInnerTableSchema::dba_ob_task_opt_stat_gather_history_schema(ObTableSchema table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(TENANT_ID AS SIGNED) AS TENANT_ID, CAST(TASK_ID AS CHAR(36)) AS TASK_ID, CAST((CASE WHEN type = 0 THEN 'MANUAL GATHER' ELSE ( CASE WHEN type = 1 THEN 'AUTO GATHER' ELSE ( CASE WHEN type IS NULL THEN NULL ELSE 'UNDEFINED GATHER' END )END ) END ) AS CHAR(16)) AS TYPE, CAST((CASE WHEN RET_CODE = 0 THEN 'SUCCESS' ELSE (CASE WHEN RET_CODE IS NULL THEN NULL ELSE (CASE WHEN RET_CODE = -5065 THEN 'CANCELED' ELSE 'FAILED' END) END) END) AS CHAR(8)) AS STATUS, CAST(TABLE_COUNT AS SIGNED) AS TABLE_COUNT, CAST(FAILED_COUNT AS SIGNED) AS FAILED_COUNT, CAST(START_TIME AS DATETIME(6)) AS START_TIME, CAST(END_TIME AS DATETIME(6)) AS END_TIME FROM oceanbase.__all_virtual_task_opt_stat_gather_history WHERE TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(TENANT_ID AS SIGNED) AS TENANT_ID, CAST(TASK_ID AS CHAR(36)) AS TASK_ID, CAST((CASE WHEN type = 0 THEN 'MANUAL GATHER' ELSE (CASE WHEN type = 1 THEN 'AUTO GATHER' ELSE (CASE WHEN type = 2 THEN 'ASYNC GATHER' ELSE (CASE WHEN type IS NULL THEN NULL ELSE 'UNDEFINED GATHER' END )END ) END ) END) AS CHAR(16)) AS TYPE, CAST((CASE WHEN RET_CODE = 0 THEN 'SUCCESS' ELSE (CASE WHEN RET_CODE IS NULL THEN NULL ELSE (CASE WHEN RET_CODE = -5065 THEN 'CANCELED' ELSE 'FAILED' END) END) END) AS CHAR(8)) AS STATUS, CAST(TABLE_COUNT AS SIGNED) AS TABLE_COUNT, CAST(FAILED_COUNT AS SIGNED) AS FAILED_COUNT, CAST(START_TIME AS DATETIME(6)) AS START_TIME, CAST(END_TIME AS DATETIME(6)) AS END_TIME FROM oceanbase.__all_virtual_task_opt_stat_gather_history WHERE TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema.25201_25250.cpp b/src/share/inner_table/ob_inner_table_schema.25201_25250.cpp index e08bda956..7c2c19dd0 100644 --- a/src/share/inner_table/ob_inner_table_schema.25201_25250.cpp +++ b/src/share/inner_table/ob_inner_table_schema.25201_25250.cpp @@ -1310,7 +1310,7 @@ int ObInnerTableSchema::dba_ob_task_opt_stat_gather_history_ora_schema(ObTableSc table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(TENANT_ID AS NUMBER) AS TENANT_ID, CAST(TASK_ID AS VARCHAR2(36)) AS TASK_ID, CAST((CASE WHEN type = 0 THEN 'MANUAL GATHER' ELSE ( CASE WHEN type = 1 THEN 'AUTO GATHER' ELSE ( CASE WHEN type IS NULL THEN NULL ELSE 'UNDEFINED GATHER' END )END ) END ) AS VARCHAR2(16)) AS TYPE, CAST((CASE WHEN RET_CODE = 0 THEN 'SUCCESS' ELSE (CASE WHEN RET_CODE IS NULL THEN NULL ELSE (CASE WHEN RET_CODE = -5065 THEN 'CANCELED' ELSE 'FAILED' END) END) END) AS VARCHAR2(8)) AS STATUS, CAST(TABLE_COUNT AS NUMBER) AS TASK_TABLE_COUNT, CAST(FAILED_COUNT AS NUMBER) AS FAILED_COUNT, CAST(START_TIME AS TIMESTAMP(6)) AS TASK_START_TIME, CAST(END_TIME AS TIMESTAMP(6)) AS TASK_END_TIME FROM SYS.ALL_VIRTUAL_TASK_OPT_STAT_GATHER_HISTORY WHERE TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(TENANT_ID AS NUMBER) AS TENANT_ID, CAST(TASK_ID AS VARCHAR2(36)) AS TASK_ID, CAST((CASE WHEN type = 0 THEN 'MANUAL GATHER' ELSE (CASE WHEN type = 1 THEN 'AUTO GATHER' ELSE (CASE WHEN type = 2 THEN 'ASYNC GATHER' ELSE (CASE WHEN type IS NULL THEN NULL ELSE 'UNDEFINED GATHER' END )END ) END ) END) AS VARCHAR2(16)) AS TYPE, CAST((CASE WHEN RET_CODE = 0 THEN 'SUCCESS' ELSE (CASE WHEN RET_CODE IS NULL THEN NULL ELSE (CASE WHEN RET_CODE = -5065 THEN 'CANCELED' ELSE 'FAILED' END) END) END) AS VARCHAR2(8)) AS STATUS, CAST(TABLE_COUNT AS NUMBER) AS TASK_TABLE_COUNT, CAST(FAILED_COUNT AS NUMBER) AS FAILED_COUNT, CAST(START_TIME AS TIMESTAMP(6)) AS TASK_START_TIME, CAST(END_TIME AS TIMESTAMP(6)) AS TASK_END_TIME FROM SYS.ALL_VIRTUAL_TASK_OPT_STAT_GATHER_HISTORY WHERE TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema.28151_28200.cpp b/src/share/inner_table/ob_inner_table_schema.28151_28200.cpp index 686830ab3..6d7fd6948 100644 --- a/src/share/inner_table/ob_inner_table_schema.28151_28200.cpp +++ b/src/share/inner_table/ob_inner_table_schema.28151_28200.cpp @@ -910,7 +910,7 @@ int ObInnerTableSchema::gv_ob_opt_stat_gather_monitor_ora_schema(ObTableSchema & table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT CAST(TENANT_ID AS NUMBER) AS TENANT_ID, CAST(SVR_IP AS VARCHAR2(46)) AS SVR_IP, CAST(SVR_PORT AS NUMBER) AS SVR_PORT, CAST(SESSION_ID AS NUMBER) AS SESSION_ID, CAST(TRACE_ID AS VARCHAR2(64)) AS TRACE_ID, CAST(TASK_ID AS VARCHAR(36)) AS TASK_ID, CAST(DECODE(TYPE, 0, 'MANUAL GATHER', 1, 'AUTO GATHER', 'UNDEFINED GATHER') AS VARCHAR2(16)) AS TYPE, CAST(TASK_START_TIME AS TIMESTAMP(6)) AS TASK_START_TIME, CAST(TASK_DURATION_TIME AS NUMBER) AS TASK_DURATION_TIME, CAST(TASK_TABLE_COUNT AS NUMBER) AS TASK_TABLE_COUNT, CAST(COMPLETED_TABLE_COUNT AS NUMBER) AS COMPLETED_TABLE_COUNT, CAST(RUNNING_TABLE_OWNER AS VARCHAR2(128)) AS RUNNING_TABLE_OWNER, CAST(RUNNING_TABLE_NAME AS VARCHAR2(256)) AS RUNNING_TABLE_NAME, CAST(RUNNING_TABLE_DURATION_TIME AS NUMBER) AS RUNNING_TABLE_DURATION_TIME, CAST(SPARE2 AS VARCHAR2(256)) AS RUNNING_TABLE_PROGRESS FROM SYS.ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT CAST(TENANT_ID AS NUMBER) AS TENANT_ID, CAST(SVR_IP AS VARCHAR2(46)) AS SVR_IP, CAST(SVR_PORT AS NUMBER) AS SVR_PORT, CAST(SESSION_ID AS NUMBER) AS SESSION_ID, CAST(TRACE_ID AS VARCHAR2(64)) AS TRACE_ID, CAST(TASK_ID AS VARCHAR(36)) AS TASK_ID, CAST(DECODE(TYPE, 0, 'MANUAL GATHER', 1, 'AUTO GATHER', 2, 'ASYNC GATHER', 'UNDEFINED GATHER') AS VARCHAR2(16)) AS TYPE, CAST(TASK_START_TIME AS TIMESTAMP(6)) AS TASK_START_TIME, CAST(TASK_DURATION_TIME AS NUMBER) AS TASK_DURATION_TIME, CAST(TASK_TABLE_COUNT AS NUMBER) AS TASK_TABLE_COUNT, CAST(COMPLETED_TABLE_COUNT AS NUMBER) AS COMPLETED_TABLE_COUNT, CAST(RUNNING_TABLE_OWNER AS VARCHAR2(128)) AS RUNNING_TABLE_OWNER, CAST(RUNNING_TABLE_NAME AS VARCHAR2(256)) AS RUNNING_TABLE_NAME, CAST(RUNNING_TABLE_DURATION_TIME AS NUMBER) AS RUNNING_TABLE_DURATION_TIME, CAST(SPARE2 AS VARCHAR2(256)) AS RUNNING_TABLE_PROGRESS FROM SYS.ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index c8d967711..2c6ff5023 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -30663,7 +30663,8 @@ def_table_schema( CAST(TRACE_ID AS CHAR(64)) AS TRACE_ID, CAST(TASK_ID AS CHAR(36)) AS TASK_ID, CAST((CASE WHEN TYPE = 0 THEN 'MANUAL GATHER' ELSE - (CASE WHEN TYPE = 1 THEN 'AUTO GATHER' ELSE 'UNDEFINED GATHER' END) END) AS CHAR(16)) AS TYPE, + (CASE WHEN TYPE = 1 THEN 'AUTO GATHER' ELSE + (CASE WHEN TYPE = 2 THEN 'ASYNC GATHER' ELSE 'UNDEFINED GATHER' END) END) END) AS CHAR(16)) AS TYPE, CAST(TASK_START_TIME AS DATETIME(6)) AS TASK_START_TIME, CAST(TASK_DURATION_TIME AS SIGNED) AS TASK_DURATION_TIME, CAST(TASK_TABLE_COUNT AS SIGNED) AS TASK_TABLE_COUNT, @@ -30718,9 +30719,10 @@ def_table_schema( CAST(TENANT_ID AS SIGNED) AS TENANT_ID, CAST(TASK_ID AS CHAR(36)) AS TASK_ID, CAST((CASE WHEN type = 0 THEN 'MANUAL GATHER' - ELSE ( CASE WHEN type = 1 THEN 'AUTO GATHER' - ELSE ( CASE WHEN type IS NULL THEN NULL - ELSE 'UNDEFINED GATHER' END )END ) END ) AS CHAR(16)) AS TYPE, + ELSE (CASE WHEN type = 1 THEN 'AUTO GATHER' + ELSE (CASE WHEN type = 2 THEN 'ASYNC GATHER' + ELSE (CASE WHEN type IS NULL THEN NULL + ELSE 'UNDEFINED GATHER' END )END ) END ) END) AS CHAR(16)) AS TYPE, CAST((CASE WHEN RET_CODE = 0 THEN 'SUCCESS' ELSE (CASE WHEN RET_CODE IS NULL THEN NULL ELSE (CASE WHEN RET_CODE = -5065 THEN 'CANCELED' ELSE 'FAILED' END) END) END) AS CHAR(8)) AS STATUS, @@ -53607,9 +53609,10 @@ def_table_schema( CAST(TENANT_ID AS NUMBER) AS TENANT_ID, CAST(TASK_ID AS VARCHAR2(36)) AS TASK_ID, CAST((CASE WHEN type = 0 THEN 'MANUAL GATHER' - ELSE ( CASE WHEN type = 1 THEN 'AUTO GATHER' - ELSE ( CASE WHEN type IS NULL THEN NULL - ELSE 'UNDEFINED GATHER' END )END ) END ) AS VARCHAR2(16)) AS TYPE, + ELSE (CASE WHEN type = 1 THEN 'AUTO GATHER' + ELSE (CASE WHEN type = 2 THEN 'ASYNC GATHER' + ELSE (CASE WHEN type IS NULL THEN NULL + ELSE 'UNDEFINED GATHER' END )END ) END ) END) AS VARCHAR2(16)) AS TYPE, CAST((CASE WHEN RET_CODE = 0 THEN 'SUCCESS' ELSE (CASE WHEN RET_CODE IS NULL THEN NULL ELSE (CASE WHEN RET_CODE = -5065 THEN 'CANCELED' ELSE 'FAILED' END) END) END) AS VARCHAR2(8)) AS STATUS, @@ -62868,7 +62871,7 @@ def_table_schema( CAST(SESSION_ID AS NUMBER) AS SESSION_ID, CAST(TRACE_ID AS VARCHAR2(64)) AS TRACE_ID, CAST(TASK_ID AS VARCHAR(36)) AS TASK_ID, - CAST(DECODE(TYPE, 0, 'MANUAL GATHER', 1, 'AUTO GATHER', 'UNDEFINED GATHER') AS VARCHAR2(16)) AS TYPE, + CAST(DECODE(TYPE, 0, 'MANUAL GATHER', 1, 'AUTO GATHER', 2, 'ASYNC GATHER', 'UNDEFINED GATHER') AS VARCHAR2(16)) AS TYPE, CAST(TASK_START_TIME AS TIMESTAMP(6)) AS TASK_START_TIME, CAST(TASK_DURATION_TIME AS NUMBER) AS TASK_DURATION_TIME, CAST(TASK_TABLE_COUNT AS NUMBER) AS TASK_TABLE_COUNT, diff --git a/src/share/inner_table/sys_package/dbms_stats_body_mysql.sql b/src/share/inner_table/sys_package/dbms_stats_body_mysql.sql index 20b965f22..c63d78c6e 100644 --- a/src/share/inner_table/sys_package/dbms_stats_body_mysql.sql +++ b/src/share/inner_table/sys_package/dbms_stats_body_mysql.sql @@ -17,7 +17,9 @@ CREATE OR REPLACE PACKAGE BODY dbms_stats statown VARCHAR(65535) DEFAULT NULL, no_invalidate BOOLEAN DEFAULT FALSE, stattype VARCHAR(65535) DEFAULT 'DATA', - force BOOLEAN DEFAULT FALSE + force BOOLEAN DEFAULT FALSE, + hist_est_percent DECIMAL DEFAULT AUTO_SAMPLE_SIZE, + hist_block_sample BOOLEAN DEFAULT NULL ); PRAGMA INTERFACE(C, GATHER_TABLE_STATS); @@ -450,6 +452,7 @@ CREATE OR REPLACE PACKAGE BODY dbms_stats taskid VARCHAR(65535) ); PRAGMA INTERFACE(C, CANCEL_GATHER_STATS); + PROCEDURE GATHER_SYSTEM_STATS(); PRAGMA INTERFACE(C, GATHER_SYSTEM_STATS); @@ -462,4 +465,6 @@ CREATE OR REPLACE PACKAGE BODY dbms_stats ); PRAGMA INTERFACE(C, SET_SYSTEM_STATS); + PROCEDURE async_gather_stats_job_proc (duration BIGINT DEFAULT NULL); + PRAGMA INTERFACE(C, ASYNC_GATHER_STATS_JOB_PROC); END dbms_stats; diff --git a/src/share/inner_table/sys_package/dbms_stats_mysql.sql b/src/share/inner_table/sys_package/dbms_stats_mysql.sql index 953476218..67908c912 100644 --- a/src/share/inner_table/sys_package/dbms_stats_mysql.sql +++ b/src/share/inner_table/sys_package/dbms_stats_mysql.sql @@ -23,7 +23,9 @@ create or replace PACKAGE dbms_stats AUTHID CURRENT_USER statown VARCHAR(65535) DEFAULT NULL, no_invalidate BOOLEAN DEFAULT FALSE, stattype VARCHAR(65535) DEFAULT 'DATA', - force BOOLEAN DEFAULT FALSE + force BOOLEAN DEFAULT FALSE, + hist_est_percent DECIMAL DEFAULT AUTO_SAMPLE_SIZE, + hist_block_sample BOOLEAN DEFAULT NULL ); PROCEDURE gather_schema_stats ( @@ -405,6 +407,7 @@ create or replace PACKAGE dbms_stats AUTHID CURRENT_USER PROCEDURE cancel_gather_stats ( taskid VARCHAR(65535) ); + PROCEDURE GATHER_SYSTEM_STATS(); PROCEDURE DELETE_SYSTEM_STATS(); @@ -414,4 +417,5 @@ create or replace PACKAGE dbms_stats AUTHID CURRENT_USER pvalue DECIMAL(20, 10) ); + PROCEDURE async_gather_stats_job_proc (duration BIGINT DEFAULT NULL); END dbms_stats; diff --git a/src/share/ob_upgrade_utils.cpp b/src/share/ob_upgrade_utils.cpp index f7607390c..d264065f1 100755 --- a/src/share/ob_upgrade_utils.cpp +++ b/src/share/ob_upgrade_utils.cpp @@ -26,6 +26,8 @@ #include "share/ob_tenant_info_proxy.h"//update max ls id #include "ob_upgrade_utils.h" #include "share/config/ob_config_helper.h" +#include "share/stat/ob_dbms_stats_maintenance_window.h" +#include "share/stat/ob_dbms_stats_preferences.h" namespace oceanbase { @@ -1460,6 +1462,8 @@ int ObUpgradeFor4330Processor::post_upgrade() LOG_WARN("fail to check inner stat", KR(ret)); } else if (OB_FAIL(post_upgrade_for_external_table_flag())) { LOG_WARN("fail to alter log external table flag", KR(ret)); + } else if (OB_FAIL(post_upgrade_for_optimizer_stats())) { + LOG_WARN("fail to upgrade optimizer stats", KR(ret)); } return ret; } @@ -1477,6 +1481,42 @@ int ObUpgradeFor4330Processor::post_upgrade_for_external_table_flag() } return ret; } + +int ObUpgradeFor4330Processor::post_upgrade_for_optimizer_stats() +{ + int ret = OB_SUCCESS; + ObSqlString extra_stats_perfs_sql; + ObSqlString add_async_stats_job_sql; + int64_t affected_rows = 0; + bool is_primary_tenant = false; + if (sql_proxy_ == NULL) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql_proxy is null", K(ret), K(tenant_id_)); + } else if (OB_FAIL(ObAllTenantInfoProxy::is_primary_tenant(sql_proxy_, tenant_id_, is_primary_tenant))) { + LOG_WARN("check is standby tenant failed", K(ret), K(tenant_id_)); + } else if (!is_primary_tenant) { + LOG_INFO("tenant isn't primary standby, no refer to gather stats, skip", K(tenant_id_)); + } else if (OB_FAIL(ObDbmsStatsPreferences::get_extra_stats_perfs_for_upgrade(extra_stats_perfs_sql))) { + LOG_WARN("failed to get extra stats perfs for upgrade", K(ret)); + } else if (OB_FAIL(sql_proxy_->write(tenant_id_, extra_stats_perfs_sql.ptr(), affected_rows))) { + LOG_WARN("failed to write", K(ret)); + } else if (OB_FAIL(ObDbmsStatsMaintenanceWindow::get_async_gather_stats_job_for_upgrade(sql_proxy_, + tenant_id_, + add_async_stats_job_sql))) { + LOG_WARN("failed to get async gather stats job for upgrade", K(ret)); + } else if (OB_UNLIKELY(add_async_stats_job_sql.empty())) { + LOG_INFO("failed to add async stats job in upgrade, perhaps the join already exists, need check after the upgrade."); + } else if (OB_FAIL(sql_proxy_->write(tenant_id_, add_async_stats_job_sql.ptr(), affected_rows))) { + LOG_WARN("failed to write", K(ret)); + } + if (OB_FAIL(ret)) { + LOG_WARN("[UPGRADE] post upgrade for optimizer stats failed", KR(ret), K_(tenant_id)); + } else { + LOG_INFO("[UPGRADE] post upgrade for optimizer stats succeed", K_(tenant_id)); + } + return ret; +} + /* =========== 4330 upgrade processor end ============= */ /* =========== special upgrade processor end ============= */ diff --git a/src/share/ob_upgrade_utils.h b/src/share/ob_upgrade_utils.h index 510e7cdee..2a2fcd52a 100755 --- a/src/share/ob_upgrade_utils.h +++ b/src/share/ob_upgrade_utils.h @@ -259,7 +259,6 @@ private: int post_upgrade_for_spm(); int post_upgrade_for_online_estimate_percent(); }; - DEF_SIMPLE_UPGRARD_PROCESSER(4, 3, 2, 1) class ObUpgradeFor4330Processor : public ObBaseUpgradeProcessor @@ -271,6 +270,7 @@ public: virtual int post_upgrade() override; private: int post_upgrade_for_external_table_flag(); + int post_upgrade_for_optimizer_stats(); }; /* =========== special upgrade processor end ============= */ diff --git a/src/share/stat/ob_basic_stats_estimator.cpp b/src/share/stat/ob_basic_stats_estimator.cpp index 4e561c8c4..ea13427a6 100644 --- a/src/share/stat/ob_basic_stats_estimator.cpp +++ b/src/share/stat/ob_basic_stats_estimator.cpp @@ -1407,5 +1407,78 @@ int ObBasicStatsEstimator::get_gather_table_type_list(ObSqlString &gather_table_ return ret; } + + +int ObBasicStatsEstimator::get_async_gather_stats_tables(ObExecContext &ctx, + const int64_t tenant_id, + const int64_t max_table_cnt, + ObIArray &stat_tables) +{ + int ret = OB_SUCCESS; + ObSqlString select_sql; + if (OB_FAIL(select_sql.append_fmt("SELECT table_id, "\ + " partition_id "\ + " FROM %s"\ + " WHERE table_id IN (SELECT DISTINCT table_id "\ + " FROM %s "\ + " WHERE tenant_id = %lu "\ + " AND stale_stats = 1 "\ + " AND stattype_locked = 0 limit %lu) "\ + " AND tenant_id = %lu "\ + " AND stale_stats = 1 "\ + " AND stattype_locked = 0 order by 1, 2", + share::OB_ALL_TABLE_STAT_TNAME, + share::OB_ALL_TABLE_STAT_TNAME, + share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id), + max_table_cnt, + share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id)))) { + LOG_WARN("failed to append fmt", K(ret)); + } else { + ObCommonSqlProxy *sql_proxy = ctx.get_sql_proxy(); + SMART_VAR(ObMySQLProxy::MySQLResult, proxy_result) { + sqlclient::ObMySQLResult *client_result = NULL; + ObSQLClientRetryWeak sql_client_retry_weak(sql_proxy); + if (OB_FAIL(sql_client_retry_weak.read(proxy_result, tenant_id, select_sql.ptr()))) { + LOG_WARN("failed to execute sql", K(ret), K(select_sql)); + } else if (OB_ISNULL(client_result = proxy_result.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to execute sql", K(ret)); + } else { + while (OB_SUCC(ret) && OB_SUCC(client_result->next())) { + int64_t idx_col1 = 0; + int64_t idx_col2 = 1; + ObObj obj; + int64_t table_id = 0; + int64_t partition_id = 0; + if (OB_FAIL(client_result->get_obj(idx_col1, obj))) { + LOG_WARN("failed to get object", K(ret)); + } else if (OB_FAIL(obj.get_int(table_id))) { + LOG_WARN("failed to get int", K(ret), K(obj)); + } else if (OB_FAIL(client_result->get_obj(idx_col2, obj))) { + LOG_WARN("failed to get object", K(ret)); + } else if (OB_FAIL(obj.get_int(partition_id))) { + LOG_WARN("failed to get int", K(ret), K(obj)); + } else if ((stat_tables.empty() || table_id != (stat_tables.at(stat_tables.count() - 1).table_id_)) && + OB_FAIL(stat_tables.push_back(AsyncStatTable(table_id)))) { + LOG_WARN("failed to push back", K(ret)); + } else if (OB_FAIL(stat_tables.at(stat_tables.count() - 1).partition_ids_.push_back(partition_id))) { + LOG_WARN("failed to push back", K(ret)); + } + } + ret = OB_ITER_END == ret ? OB_SUCCESS : ret; + } + int tmp_ret = OB_SUCCESS; + if (NULL != client_result) { + if (OB_SUCCESS != (tmp_ret = client_result->close())) { + LOG_WARN("close result set failed", K(ret), K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + } + } + LOG_TRACE("succeed to get async gather stats tables", K(ret), K(stat_tables)); + } + return ret; +} + } // end of common } // end of oceanbase diff --git a/src/share/stat/ob_basic_stats_estimator.h b/src/share/stat/ob_basic_stats_estimator.h index 5727d10ba..e8b71f4a9 100644 --- a/src/share/stat/ob_basic_stats_estimator.h +++ b/src/share/stat/ob_basic_stats_estimator.h @@ -142,6 +142,13 @@ public: const int64_t slice_cnt, ObIArray &table_ids); + static int get_async_gather_stats_tables(ObExecContext &ctx, + const int64_t tenant_id, + const int64_t max_table_cnt, + ObIArray &stat_tables); + + static int check_async_gather_need_sample(ObExecContext &ctx, ObTableStatParam ¶m); + int estimate(const ObOptStatGatherParam ¶m, ObIArray &dst_opt_stats); diff --git a/src/share/stat/ob_dbms_stats_executor.cpp b/src/share/stat/ob_dbms_stats_executor.cpp index 2cf03b09e..a761a27f8 100644 --- a/src/share/stat/ob_dbms_stats_executor.cpp +++ b/src/share/stat/ob_dbms_stats_executor.cpp @@ -273,7 +273,8 @@ int ObDbmsStatsExecutor::no_split_gather_stats(ObExecContext &ctx, /** @brief ObDbmsStatsExecutor::prepare_gather_stats used to prepare gather table stats, including: * 1.estimate block count; - * 2.get the maximum num of partitions and columns for each stat gather. + * 2.adjust async gather param base on the estimate rowcnt info. + * 3.get the maximum num of partitions and columns for each stat gather. */ int ObDbmsStatsExecutor::prepare_gather_stats(ObExecContext &ctx, ObMySQLTransaction &trans, @@ -294,6 +295,11 @@ int ObDbmsStatsExecutor::prepare_gather_stats(ObExecContext &ctx, gather_helper.use_column_store_, gather_helper.use_split_part_))) { LOG_WARN("failed to estimate block count", K(ret)); + } else if (!gather_helper.use_split_part_ && + OB_FAIL(adjsut_async_gather_param(partition_id_block_map, + const_cast(param), + gather_helper.use_split_part_))) { + LOG_WARN("failed to adjsut async gather param", K(ret)); } else if (OB_FAIL(check_need_split_gather(param, gather_helper))) { LOG_WARN("failed to check need split gather", K(ret)); } else { @@ -1801,6 +1807,163 @@ int ObDbmsStatsExecutor::set_system_stats(ObExecContext &ctx, const ObSetSystemS return ret; } +int ObDbmsStatsExecutor::adjsut_async_gather_param(const PartitionIdBlockMap &partition_id_block_map, + ObTableStatParam ¶m, + bool &need_split_part) +{ + int ret = OB_SUCCESS; + need_split_part = false; + if (param.is_async_gather_) { + LOG_TRACE("begin to adjsut async gather param", K(param)); + if (param.part_level_ == share::schema::ObPartitionLevel::PARTITION_LEVEL_ZERO) { + //do nohting + } else { + BlockNumStat *block_num_stat = NULL; + if (OB_FAIL(partition_id_block_map.get_refactored(param.global_part_id_, block_num_stat))) { + if (OB_LIKELY(OB_HASH_NOT_EXIST == ret)) { + ret = OB_SUCCESS; + } else { + LOG_WARN("failed to get refactored", K(ret)); + } + } else if (OB_ISNULL(block_num_stat)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(block_num_stat)); + } else { + int64_t total_row_cnt = block_num_stat->sstable_row_cnt_ + block_num_stat->memtable_row_cnt_; + if (total_row_cnt < param.async_full_table_size_) { + //do nothing + } else if (param.part_level_ == share::schema::ObPartitionLevel::PARTITION_LEVEL_ONE || + param.part_level_ == share::schema::ObPartitionLevel::PARTITION_LEVEL_TWO) { + bool can_derive = true; + int64_t max_part_scan_row_cnt = DEFAULT_ASYNC_MAX_SCAN_ROWCOUNT / 2; + if (param.part_level_ == share::schema::ObPartitionLevel::PARTITION_LEVEL_TWO && + param.subpart_stat_param_.need_modify_) { + int64_t gather_scan_row_cnt = 0; + int64_t i = 0; + ObSEArray no_derive_part_ids; + for (; OB_SUCC(ret) && i < param.subpart_infos_.count() && gather_scan_row_cnt < max_part_scan_row_cnt; ++i) { + if (OB_FAIL(partition_id_block_map.get_refactored(param.subpart_infos_.at(i).part_id_, block_num_stat))) { + if (OB_LIKELY(OB_HASH_NOT_EXIST == ret)) { + ret = OB_SUCCESS; + } else { + LOG_WARN("failed to get refactored", K(ret)); + } + } else if (OB_ISNULL(block_num_stat)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(block_num_stat)); + } else { + int64_t row_cnt = block_num_stat->sstable_row_cnt_ + block_num_stat->memtable_row_cnt_; + gather_scan_row_cnt += row_cnt; + if (row_cnt > param.async_full_table_size_) { + need_split_part = true; + if (OB_FAIL(add_var_to_array_no_dup(no_derive_part_ids, + param.subpart_infos_.at(i).first_part_id_))) { + LOG_WARN("failed to push back", K(ret)); + } + } + } + } + if (OB_SUCC(ret) && (i < param.subpart_infos_.count() || !no_derive_part_ids.empty())) { + while (OB_SUCC(ret) && i < param.subpart_infos_.count()) { + int64_t idx = param.subpart_infos_.count() - 1; + if (OB_FAIL(param.no_regather_partition_ids_.push_back(param.subpart_infos_.at(idx).part_id_))) { + LOG_WARN("failed to push back", K(ret)); + } else { + param.subpart_infos_.pop_back(); + } + } + if (OB_SUCC(ret)) { + ObSEArray new_approx_part_infos; + for (int64_t j = 0; OB_SUCC(ret) && j < param.approx_part_infos_.count(); ++j) { + for (int64_t k = 0; can_derive && k < no_derive_part_ids.count(); ++k) { + if (no_derive_part_ids.at(k) == param.approx_part_infos_.at(j).part_id_) { + can_derive = false; + } + } + bool found_it = false; + for (int64_t k = 0; can_derive && !found_it && k < param.subpart_infos_.count(); ++k) { + found_it = param.subpart_infos_.at(k).first_part_id_ == param.approx_part_infos_.at(j).part_id_; + } + if (!found_it || !can_derive) { + if (!can_derive && is_async_gather_partition_id(param.approx_part_infos_.at(j).part_id_, param.async_partition_ids_)) { + if (OB_FAIL(param.part_infos_.push_back(param.approx_part_infos_.at(j)))) { + LOG_WARN("failed to push back", K(ret)); + } + } else if (OB_FAIL(param.no_regather_partition_ids_.push_back(param.approx_part_infos_.at(j).part_id_))) { + LOG_WARN("failed to push back", K(ret)); + } + } else if (OB_FAIL(new_approx_part_infos.push_back(param.approx_part_infos_.at(j)))) { + LOG_WARN("failed to push back", K(ret)); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(param.approx_part_infos_.assign(new_approx_part_infos))) { + LOG_WARN("failed to assign", K(ret)); + } + } + } + } + } + if (OB_SUCC(ret) && param.part_stat_param_.need_modify_) { + int64_t gather_scan_row_cnt = 0; + int64_t i = 0; + for (; OB_SUCC(ret) && i < param.part_infos_.count() && gather_scan_row_cnt < max_part_scan_row_cnt; ++i) { + if (OB_FAIL(partition_id_block_map.get_refactored(param.part_infos_.at(i).part_id_, block_num_stat))) { + if (OB_LIKELY(OB_HASH_NOT_EXIST == ret)) { + ret = OB_SUCCESS; + } else { + LOG_WARN("failed to get refactored", K(ret)); + } + } else if (OB_ISNULL(block_num_stat)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(block_num_stat)); + } else { + int64_t row_cnt = block_num_stat->sstable_row_cnt_ + block_num_stat->memtable_row_cnt_; + gather_scan_row_cnt += row_cnt; + if (row_cnt > param.async_full_table_size_) { + need_split_part = true; + can_derive = false; + } + } + } + if (OB_SUCC(ret) && i < param.part_infos_.count()) { + while (OB_SUCC(ret) && i < param.part_infos_.count()) { + int64_t idx = param.part_infos_.count() - 1; + if (OB_FAIL(param.no_regather_partition_ids_.push_back(param.part_infos_.at(idx).part_id_))) { + LOG_WARN("failed to push back", K(ret)); + } else { + param.part_infos_.pop_back(); + } + } + } + } + if (OB_SUCC(ret) && param.global_stat_param_.need_modify_ && !can_derive) { + if (is_async_gather_partition_id(param.global_part_id_, param.async_partition_ids_)) { + param.global_stat_param_.gather_approx_ = false; + } else { + param.global_stat_param_.reset_gather_stat(); + } + } + } + } + } + LOG_TRACE("end to adjsut async gather param", K(param)); + } + return ret; +} + +bool ObDbmsStatsExecutor::is_async_gather_partition_id(const int64_t partition_id, + const ObIArray *async_partition_ids) +{ + bool is_found = false; + if (async_partition_ids != NULL) { + for (int64_t i = 0; !is_found && i < async_partition_ids->count(); ++i) { + is_found = partition_id == async_partition_ids->at(i); + } + } + return is_found; +} + } // namespace common } // namespace oceanbase diff --git a/src/share/stat/ob_dbms_stats_executor.h b/src/share/stat/ob_dbms_stats_executor.h index b74b72f3f..0eb7f2431 100644 --- a/src/share/stat/ob_dbms_stats_executor.h +++ b/src/share/stat/ob_dbms_stats_executor.h @@ -202,6 +202,13 @@ private: char *&svr_ip, int32_t &svr_port); + static int adjsut_async_gather_param(const PartitionIdBlockMap &partition_id_block_map, + ObTableStatParam ¶m, + bool &need_split_part); + + static bool is_async_gather_partition_id(const int64_t partition_id, + const ObIArray *async_partition_ids); + }; diff --git a/src/share/stat/ob_dbms_stats_gather.cpp b/src/share/stat/ob_dbms_stats_gather.cpp index 5b4c25660..bbef2a5b7 100644 --- a/src/share/stat/ob_dbms_stats_gather.cpp +++ b/src/share/stat/ob_dbms_stats_gather.cpp @@ -42,6 +42,8 @@ int ObDbmsStatsGather::gather_stats(ObExecContext &ctx, LOG_WARN("get unexpected error", K(ret), K(param.allocator_)); } else if (OB_FAIL(init_opt_stats(*param.allocator_, param, opt_stats))) { LOG_WARN("failed to init opt stats", K(ret)); + } else if (OB_FAIL(refine_sample_block_for_async_gather(opt_stats, const_cast(param)))) { + LOG_WARN("failed to refine sample block for async gather", K(ret)); } else if (!opt_stats.empty()) { //1.firstly esimate basic stat ObBasicStatsEstimator basic_est(ctx, *param.allocator_); @@ -250,5 +252,40 @@ int ObDbmsStatsGather::gather_index_stats(ObExecContext &ctx, return ret; } +int ObDbmsStatsGather::refine_sample_block_for_async_gather(const ObIArray &opt_stats, + ObOptStatGatherParam ¶m) +{ + int ret = OB_SUCCESS; + if (param.is_async_gather_ && !param.sample_info_.is_specify_sample()) { + int64_t sstable_row_cnt = 0; + int64_t memtable_row_cnt = 0; + for (int64_t i = 0; OB_SUCC(ret) && i < opt_stats.count(); ++i) { + if (OB_ISNULL(opt_stats.at(i).table_stat_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(opt_stats.at(i).table_stat_)); + } else { + sstable_row_cnt += opt_stats.at(i).table_stat_->get_sstable_row_count(); + memtable_row_cnt += opt_stats.at(i).table_stat_->get_memtable_row_count(); + } + } + if (OB_UNLIKELY(opt_stats.count() > 1 && + sstable_row_cnt + memtable_row_cnt > DEFAULT_ASYNC_MAX_SCAN_ROWCOUNT)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(sstable_row_cnt), K(memtable_row_cnt), K(opt_stats), K(param)); + } else if (opt_stats.count() == 1) { + if (param.async_full_table_size_ < sstable_row_cnt + memtable_row_cnt) { + double sample_ratio = 100.0; + sample_ratio = 1.0 * param.async_gather_sample_size_ / (sstable_row_cnt + memtable_row_cnt) * 100.0; + if (sample_ratio > 0.0 && sample_ratio < 100.0) { + param.sample_info_.set_percent(sample_ratio); + param.sample_info_.set_is_block_sample(true); + } + LOG_INFO("decide async gather stats need sample", K(param), K(opt_stats)); + } + } + } + return ret; +} + } // namespace common } // namespace oceanbase diff --git a/src/share/stat/ob_dbms_stats_gather.h b/src/share/stat/ob_dbms_stats_gather.h index 22eca959c..9ee73747e 100644 --- a/src/share/stat/ob_dbms_stats_gather.h +++ b/src/share/stat/ob_dbms_stats_gather.h @@ -48,6 +48,9 @@ private: static int classfy_column_histogram(const ObOptStatGatherParam ¶m, ObOptStat &opt_stat); + static int refine_sample_block_for_async_gather(const ObIArray &opt_stats, + ObOptStatGatherParam ¶m); + }; } // end of sql diff --git a/src/share/stat/ob_dbms_stats_history_manager.cpp b/src/share/stat/ob_dbms_stats_history_manager.cpp index cb32d0f4c..f2367e04f 100644 --- a/src/share/stat/ob_dbms_stats_history_manager.cpp +++ b/src/share/stat/ob_dbms_stats_history_manager.cpp @@ -221,12 +221,12 @@ int ObDbmsStatsHistoryManager::backup_table_stats(ObExecContext &ctx, int ret = OB_SUCCESS; ObSEArray no_stat_part_ids; ObSEArray have_stat_part_ids; - bool is_specify_partition_gather = param.is_specify_partition_gather(); + bool is_specify_partition = param.is_specify_partition(); if (part_ids.empty()) { } else if (OB_FAIL(calssify_table_stat_part_ids(ctx, param.tenant_id_, param.table_id_, - is_specify_partition_gather, + is_specify_partition, part_ids, no_stat_part_ids, have_stat_part_ids))) { @@ -234,7 +234,7 @@ int ObDbmsStatsHistoryManager::backup_table_stats(ObExecContext &ctx, } else if (OB_FAIL(backup_having_table_part_stats(trans, param.tenant_id_, param.table_id_, - (is_specify_partition_gather || have_stat_part_ids.count() != part_ids.count()), + (is_specify_partition || have_stat_part_ids.count() != part_ids.count()), have_stat_part_ids, saving_time))) { LOG_WARN("failed to backup having table part stats", K(ret)); @@ -247,7 +247,7 @@ int ObDbmsStatsHistoryManager::backup_table_stats(ObExecContext &ctx, int ObDbmsStatsHistoryManager::calssify_table_stat_part_ids(ObExecContext &ctx, const uint64_t tenant_id, const uint64_t table_id, - const bool is_specify_partition_gather, + const bool is_specify_partition, const ObIArray &partition_ids, ObIArray &no_stat_part_ids, ObIArray &have_stat_part_ids) @@ -261,17 +261,17 @@ int ObDbmsStatsHistoryManager::calssify_table_stat_part_ids(ObExecContext &ctx, if (OB_ISNULL(mysql_proxy) || OB_ISNULL(session) || OB_UNLIKELY(partition_ids.empty())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(mysql_proxy), K(session), K(partition_ids)); - } else if (is_specify_partition_gather && + } else if (is_specify_partition && OB_FAIL(gen_partition_list(partition_ids, partition_list))) { LOG_WARN("failed to gen partition list", K(ret)); - } else if (is_specify_partition_gather && + } else if (is_specify_partition && OB_FAIL(extra_where_str.append_fmt(" and partition_id in %s", partition_list.ptr()))) { LOG_WARN("failed to append fmt", K(ret)); } else if (OB_FAIL(raw_sql.append_fmt(CHECK_TABLE_STAT, share::OB_ALL_TABLE_STAT_TNAME, share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id), share::schema::ObSchemaUtils::get_extract_schema_id(tenant_id, table_id), - is_specify_partition_gather ? extra_where_str.ptr() : " "))) { + is_specify_partition ? extra_where_str.ptr() : " "))) { LOG_WARN("failed to append fmt", K(ret)); } else { SMART_VAR(ObMySQLProxy::MySQLResult, proxy_result) { @@ -338,7 +338,7 @@ int ObDbmsStatsHistoryManager::calssify_table_stat_part_ids(ObExecContext &ctx, int ObDbmsStatsHistoryManager::backup_having_table_part_stats(ObMySQLTransaction &trans, const uint64_t tenant_id, const uint64_t table_id, - const bool is_specify_partition_gather, + const bool is_specify_partition, const ObIArray &partition_ids, const int64_t saving_time) { @@ -349,10 +349,10 @@ int ObDbmsStatsHistoryManager::backup_having_table_part_stats(ObMySQLTransaction ObSqlString select_sql; int64_t affected_rows = 0; if (partition_ids.empty()) { - } else if (is_specify_partition_gather && + } else if (is_specify_partition && OB_FAIL(gen_partition_list(partition_ids, partition_list))) { LOG_WARN("failed to gen partition list", K(ret)); - } else if (is_specify_partition_gather && + } else if (is_specify_partition && OB_FAIL(extra_where_str.append_fmt(" and partition_id in %s", partition_list.ptr()))) { LOG_WARN("failed to append fmt", K(ret)); } else if (OB_FAIL(select_sql.append_fmt(SELECT_TABLE_STAT, @@ -360,7 +360,7 @@ int ObDbmsStatsHistoryManager::backup_having_table_part_stats(ObMySQLTransaction share::OB_ALL_TABLE_STAT_TNAME, share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id), share::schema::ObSchemaUtils::get_extract_schema_id(tenant_id, table_id), - is_specify_partition_gather ? extra_where_str.ptr() : " "))) { + is_specify_partition ? extra_where_str.ptr() : " "))) { LOG_WARN("failed to append fmt", K(ret)); } else if (OB_FAIL(raw_sql.append_fmt(INSERT_TABLE_STAT_HISTORY, share::OB_ALL_TABLE_STAT_HISTORY_TNAME, @@ -432,8 +432,8 @@ int ObDbmsStatsHistoryManager::backup_column_stats(ObExecContext &ctx, int ret = OB_SUCCESS; hash::ObHashMap having_stat_part_col_map; int64_t map_size = part_ids.count() * column_ids.count(); - bool is_specify_partition_gather = param.is_specify_partition_gather(); - bool is_specify_column_gather = param.is_specify_column_gather(); + bool is_specify_partition = param.is_specify_partition(); + bool is_specify_column = param.is_specify_column(); if (part_ids.empty() || column_ids.empty()) { } else if (OB_FAIL(having_stat_part_col_map.create(map_size, "PartColHashMap", @@ -443,14 +443,14 @@ int ObDbmsStatsHistoryManager::backup_column_stats(ObExecContext &ctx, } else if (OB_FAIL(generate_having_stat_part_col_map(ctx, param.tenant_id_, param.table_id_, - is_specify_partition_gather, - is_specify_column_gather, + is_specify_partition, + is_specify_column, part_ids, column_ids, having_stat_part_col_map))) { LOG_WARN("failed to calssify table stat part ids", K(ret)); } else if (OB_FAIL(backup_having_column_stats(trans, param.tenant_id_, param.table_id_, - is_specify_partition_gather || is_specify_column_gather, + is_specify_partition || is_specify_column, part_ids, column_ids, having_stat_part_col_map, saving_time))) { @@ -461,8 +461,8 @@ int ObDbmsStatsHistoryManager::backup_column_stats(ObExecContext &ctx, saving_time))) { LOG_WARN("failed to backup column part stats", K(ret)); } else if (OB_FAIL(backup_histogram_stats(trans, param.tenant_id_, param.table_id_, - is_specify_partition_gather, - is_specify_column_gather, + is_specify_partition, + is_specify_column, part_ids, column_ids, having_stat_part_col_map, saving_time))) { @@ -474,8 +474,8 @@ int ObDbmsStatsHistoryManager::backup_column_stats(ObExecContext &ctx, int ObDbmsStatsHistoryManager::generate_having_stat_part_col_map(ObExecContext &ctx, const uint64_t tenant_id, const uint64_t table_id, - const bool is_specify_partition_gather, - const bool is_specify_column_gather, + const bool is_specify_partition, + const bool is_specify_column, const ObIArray &partition_ids, const ObIArray &column_ids, hash::ObHashMap &have_stat_part_col_map) @@ -493,28 +493,28 @@ int ObDbmsStatsHistoryManager::generate_having_stat_part_col_map(ObExecContext & OB_UNLIKELY(partition_ids.empty() || column_ids.empty())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(mysql_proxy), K(session), K(partition_ids), K(column_ids)); - } else if (is_specify_partition_gather && + } else if (is_specify_partition && OB_FAIL(gen_partition_list(partition_ids, partition_list))) { LOG_WARN("failed to gen partition list", K(ret)); - } else if (is_specify_partition_gather && + } else if (is_specify_partition && OB_FAIL(extra_partition_str.append_fmt(" and partition_id in %s", partition_list.ptr()))) { LOG_WARN("failed to append fmt", K(ret)); - } else if (is_specify_column_gather && + } else if (is_specify_column && OB_FAIL(gen_column_list(column_ids, column_list))) { LOG_WARN("failed to gen column list", K(ret)); - } else if (is_specify_column_gather && + } else if (is_specify_column && OB_FAIL(extra_column_str.append_fmt(" and column_id in %s", column_list.ptr()))) { LOG_WARN("failed to append fmt", K(ret)); - } else if ((is_specify_partition_gather || is_specify_column_gather) && + } else if ((is_specify_partition || is_specify_column) && OB_FAIL(extra_where_str.append_fmt("%s%s", - is_specify_partition_gather ? extra_partition_str.ptr() : " ", - is_specify_column_gather ? extra_column_str.ptr() : " "))) { + is_specify_partition ? extra_partition_str.ptr() : " ", + is_specify_column ? extra_column_str.ptr() : " "))) { LOG_WARN("failed to append fmt", K(ret)); } else if (OB_FAIL(raw_sql.append_fmt(CHECK_COLUMN_STAT, share::OB_ALL_COLUMN_STAT_TNAME, share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id), share::schema::ObSchemaUtils::get_extract_schema_id(tenant_id, table_id), - (is_specify_partition_gather || is_specify_column_gather) ? extra_where_str.ptr() : " "))) { + (is_specify_partition || is_specify_column) ? extra_where_str.ptr() : " "))) { LOG_WARN("failed to append fmt", K(ret)); } else { SMART_VAR(ObMySQLProxy::MySQLResult, proxy_result) { @@ -810,8 +810,8 @@ int ObDbmsStatsHistoryManager::backup_no_column_stats(ObMySQLTransaction &trans, int ObDbmsStatsHistoryManager::backup_histogram_stats(ObMySQLTransaction &trans, const uint64_t tenant_id, const uint64_t table_id, - const bool is_specify_partition_gather, - const bool is_specify_column_gather, + const bool is_specify_partition, + const bool is_specify_column, const ObIArray &partition_ids, const ObIArray &column_ids, hash::ObHashMap &having_stat_part_col_map, @@ -827,25 +827,25 @@ int ObDbmsStatsHistoryManager::backup_histogram_stats(ObMySQLTransaction &trans, ObSqlString partition_list; ObSqlString column_list; int64_t affected_rows = 0; - if (is_specify_partition_gather && OB_FAIL(gen_partition_list(partition_ids, partition_list))) { + if (is_specify_partition && OB_FAIL(gen_partition_list(partition_ids, partition_list))) { LOG_WARN("failed to gen partition list", K(ret)); - } else if (is_specify_partition_gather && + } else if (is_specify_partition && OB_FAIL(extra_partition_str.append_fmt(" and partition_id in %s", partition_list.ptr()))) { LOG_WARN("failed to append fmt", K(ret)); - } else if (is_specify_column_gather && OB_FAIL(gen_column_list(column_ids, column_list))) { + } else if (is_specify_column && OB_FAIL(gen_column_list(column_ids, column_list))) { LOG_WARN("failed to gen column list", K(ret)); - } else if (is_specify_column_gather && + } else if (is_specify_column && OB_FAIL(extra_column_str.append_fmt(" and column_id in %s", column_list.ptr()))) { LOG_WARN("failed to append fmt", K(ret)); - } else if ((is_specify_partition_gather || is_specify_column_gather) && + } else if ((is_specify_partition || is_specify_column) && OB_FAIL(extra_where_str.append_fmt("%s%s", - is_specify_partition_gather ? extra_partition_str.ptr() : " ", - is_specify_column_gather ? extra_column_str.ptr() : " "))) { + is_specify_partition ? extra_partition_str.ptr() : " ", + is_specify_column ? extra_column_str.ptr() : " "))) { LOG_WARN("failed to append fmt", K(ret)); } else if (OB_FAIL(where_str.append_fmt(" tenant_id = %lu and table_id = %lu %s", share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id), share::schema::ObSchemaUtils::get_extract_schema_id(tenant_id, table_id), - (is_specify_partition_gather || is_specify_column_gather) ? extra_where_str.ptr() : " "))) { + (is_specify_partition || is_specify_column) ? extra_where_str.ptr() : " "))) { LOG_WARN("failed to append fmt", K(ret)); } else if (OB_FAIL(raw_sql.append_fmt(INSERT_HISTOGRAM_STAT_HISTORY, share::OB_ALL_HISTOGRAM_STAT_HISTORY_TNAME, diff --git a/src/share/stat/ob_dbms_stats_history_manager.h b/src/share/stat/ob_dbms_stats_history_manager.h index 8777cbffa..b9facad41 100644 --- a/src/share/stat/ob_dbms_stats_history_manager.h +++ b/src/share/stat/ob_dbms_stats_history_manager.h @@ -119,7 +119,7 @@ private: static int calssify_table_stat_part_ids(ObExecContext &ctx, const uint64_t tenant_id, const uint64_t table_id, - const bool is_specify_partition_gather, + const bool is_specify_partition, const ObIArray &partition_ids, ObIArray &no_stat_part_ids, ObIArray &have_stat_part_ids); @@ -127,7 +127,7 @@ private: static int backup_having_table_part_stats(ObMySQLTransaction &trans, const uint64_t tenant_id, const uint64_t table_id, - const bool is_specify_partition_gather, + const bool is_specify_partition, const ObIArray &partition_ids, const int64_t saving_time); @@ -147,8 +147,8 @@ private: static int generate_having_stat_part_col_map(ObExecContext &ctx, const uint64_t tenant_id, const uint64_t table_id, - const bool is_specify_partition_gather, - const bool is_specify_column_gather, + const bool is_specify_partition, + const bool is_specify_column, const ObIArray &partition_ids, const ObIArray &column_ids, hash::ObHashMap &have_stat_part_col_map); @@ -173,8 +173,8 @@ private: static int backup_histogram_stats(ObMySQLTransaction &trans, const uint64_t tenant_id, const uint64_t table_id, - const bool is_specify_partition_gather, - const bool is_specify_column_gather, + const bool is_specify_partition, + const bool is_specify_column, const ObIArray &partition_ids, const ObIArray &column_ids, hash::ObHashMap &having_stat_part_col_map, diff --git a/src/share/stat/ob_dbms_stats_maintenance_window.cpp b/src/share/stat/ob_dbms_stats_maintenance_window.cpp index d9cc26af2..3d9d8b412 100644 --- a/src/share/stat/ob_dbms_stats_maintenance_window.cpp +++ b/src/share/stat/ob_dbms_stats_maintenance_window.cpp @@ -20,6 +20,7 @@ #include "sql/session/ob_basic_session_info.h" #include "observer/omt/ob_tenant_timezone_mgr.h" #include "lib/timezone/ob_timezone_info.h" +#include "observer/dbms_scheduler/ob_dbms_sched_table_operator.h" #define ALL_TENANT_SCHEDULER_JOB_COLUMN_NAME "tenant_id, " \ "job_name, " \ @@ -64,6 +65,8 @@ const char *windows_name[DAY_OF_WEEK] = {"MONDAY_WINDOW", "SATURDAY_WINDOW", "SUNDAY_WINDOW"}; const char *opt_stats_history_manager = "OPT_STATS_HISTORY_MANAGER"; +const char *async_gather_stats_job_proc = "ASYNC_GATHER_STATS_JOB_PROC"; +const int64_t OPT_STATS_HISTORY_MANAGER_JOB_ID = 8; int ObDbmsStatsMaintenanceWindow::get_stats_maintenance_window_jobs_sql(const ObSysVariableSchema &sys_variable, const uint64_t tenant_id, @@ -156,6 +159,27 @@ int ObDbmsStatsMaintenanceWindow::get_stats_maintenance_window_jobs_sql(const Ob } } + //set async gather stats job + if (OB_FAIL(ret)) { + } else if (OB_FAIL(get_async_gather_stats_job_sql(is_oracle_mode, tenant_id, + job_id++, exec_env, tmp_sql))) { + LOG_WARN("failed to get async gather stats job sql", K(ret)); + } else if (OB_FAIL(raw_sql.append_fmt(", (%s)", tmp_sql.ptr()))) { + LOG_WARN("failed to append sql", K(ret)); + } else { + ++ expected_affected_rows; + tmp_sql.reset(); + if (OB_FAIL(get_async_gather_stats_job_sql(is_oracle_mode, tenant_id, + 0, exec_env, tmp_sql))) { + LOG_WARN("failed to get async gather stats job sql", K(ret)); + } else if (OB_FAIL(raw_sql.append_fmt(", (%s)", tmp_sql.ptr()))) { + LOG_WARN("failed to append sql", K(ret)); + } else { + ++ expected_affected_rows; + tmp_sql.reset(); + } + } + //set dummy guard job if (OB_FAIL(ret)) { } else if (OB_FAIL(get_dummy_guard_job_sql(tenant_id, job_id, tmp_sql))) { @@ -264,6 +288,51 @@ int ObDbmsStatsMaintenanceWindow::get_stats_history_manager_job_sql(const bool i return ret; } +int ObDbmsStatsMaintenanceWindow::get_async_gather_stats_job_sql(const bool is_oracle_mode, + const uint64_t tenant_id, + const int64_t job_id, + const ObString &exec_env, + ObSqlString &raw_sql) +{ + int ret = OB_SUCCESS; + int64_t interval_ts = DEFAULT_ASYNC_GATHER_STATS_INTERVAL_USEC; + int64_t end_date = 64060560000000000;//4000-01-01 00:00:00.000000 + int64_t current = ObTimeUtility::current_time() + DEFAULT_ASYNC_GATHER_STATS_INTERVAL_USEC; + share::ObDMLSqlSplicer dml; + OZ (dml.add_pk_column("tenant_id", share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id))); + OZ (dml.add_column("job_name", ObHexEscapeSqlStr(ObString(async_gather_stats_job_proc)))); + OZ (dml.add_pk_column("job", job_id)); + OZ (dml.add_column("lowner", is_oracle_mode ? ObHexEscapeSqlStr("SYS") : ObHexEscapeSqlStr("root@%"))); + OZ (dml.add_column("powner", is_oracle_mode ? ObHexEscapeSqlStr("SYS") : ObHexEscapeSqlStr("root@%"))); + OZ (dml.add_column("cowner", is_oracle_mode ? ObHexEscapeSqlStr("SYS") : ObHexEscapeSqlStr("oceanbase"))); + OZ (dml.add_time_column("next_date", current)); + OZ (dml.add_column("total", 0)); + OZ (dml.add_column("`interval#`", ObHexEscapeSqlStr(ObString("FREQ=MINUTELY; INTERVAL=15")))); + OZ (dml.add_column("flag", 0)); + OZ (dml.add_column("what", ObHexEscapeSqlStr("DBMS_STATS.ASYNC_GATHER_STATS_JOB_PROC(600000000)"))); + OZ (dml.add_column("nlsenv", ObHexEscapeSqlStr(ObString("")))); + OZ (dml.add_column("field1", ObHexEscapeSqlStr(ObString("")))); + OZ (dml.add_column("exec_env", ObHexEscapeSqlStr(exec_env))); + OZ (dml.add_column("job_style", ObHexEscapeSqlStr(ObString("REGULER")))); + OZ (dml.add_column("program_name", ObHexEscapeSqlStr(ObString("")))); + OZ (dml.add_column("job_type", ObHexEscapeSqlStr(ObString("STORED_PROCEDURE")))); + OZ (dml.add_column("job_action", ObHexEscapeSqlStr("DBMS_STATS.ASYNC_GATHER_STATS_JOB_PROC(600000000)"))); + OZ (dml.add_column("number_of_argument", 0)); + OZ (dml.add_raw_time_column("start_date", current)); + OZ (dml.add_column("repeat_interval", ObHexEscapeSqlStr(ObString("FREQ=MINUTELY; INTERVAL=15")))); + OZ (dml.add_raw_time_column("end_date", end_date)); + OZ (dml.add_column("job_class", ObHexEscapeSqlStr(ObString("DEFAULT_JOB_CLASS")))); + OZ (dml.add_column("enabled", true)); + OZ (dml.add_column("auto_drop", false)); + OZ (dml.add_column("comments", ObHexEscapeSqlStr(ObString("used to async gather stats")))); + OZ (dml.add_column("credential_name", ObHexEscapeSqlStr(ObString("")))); + OZ (dml.add_column("destination_name", ObHexEscapeSqlStr(ObString("")))); + OZ (dml.add_column("interval_ts", interval_ts)); + OZ (dml.add_column("max_run_duration", DEFAULT_ASYNC_GATHER_STATS_DURATION_SEC)); + OZ (dml.splice_values(raw_sql)); + return ret; +} + //this dummy guard job is used to make sure the job id is monotonically increaseing int ObDbmsStatsMaintenanceWindow::get_dummy_guard_job_sql(const uint64_t tenant_id, const int64_t job_id, @@ -372,28 +441,30 @@ int ObDbmsStatsMaintenanceWindow::is_stats_maintenance_window_attr(const sql::Ob } else if (is_stats_job(job_name)) { //now we just support modify job_action、start_date if (0 == attr_name.case_compare("job_action")) { - if (0 == job_name.case_compare(opt_stats_history_manager)) { - const char *job_action_name = "DBMS_STATS.PURGE_STATS("; - if (!val_name.empty() && 0 == strncasecmp(val_name.ptr(), job_action_name, strlen(job_action_name))) { - if (OB_FAIL(dml.add_column("job_action", ObHexEscapeSqlStr(val_name)))) { - LOG_WARN("failed to add column", K(ret)); - } else if (OB_FAIL(dml.add_column("what", ObHexEscapeSqlStr(val_name)))) { - LOG_WARN("failed to add column", K(ret)); - } else { - is_window_attr = true; - } - } else {/*do nothing*/} + const char *history_stats_job = "DBMS_STATS.PURGE_STATS("; + const char *async_gather_stats_job = "DBMS_STATS.ASYNC_GATHER_STATS_JOB_PROC("; + const char *maintenance_window_job = "DBMS_STATS.GATHER_DATABASE_STATS_JOB_PROC("; + if ((0 == job_name.case_compare(opt_stats_history_manager) && + !val_name.empty() && + 0 == strncasecmp(val_name.ptr(), history_stats_job, strlen(history_stats_job))) || + (0 == job_name.case_compare(async_gather_stats_job_proc) && + !val_name.empty() && + 0 == strncasecmp(val_name.ptr(), async_gather_stats_job, strlen(async_gather_stats_job))) || + (0 != job_name.case_compare(opt_stats_history_manager) && + 0 != job_name.case_compare(async_gather_stats_job_proc) && + !val_name.empty() && + 0 == strncasecmp(val_name.ptr(), maintenance_window_job, strlen(maintenance_window_job)))) { + if (OB_FAIL(dml.add_column("job_action", ObHexEscapeSqlStr(val_name)))) { + LOG_WARN("failed to add column", K(ret)); + } else if (OB_FAIL(dml.add_column("what", ObHexEscapeSqlStr(val_name)))) { + LOG_WARN("failed to add column", K(ret)); + } else { + is_window_attr = true; + } } else { - const char *job_action_name = "DBMS_STATS.GATHER_DATABASE_STATS_JOB_PROC("; - if (!val_name.empty() && 0 == strncasecmp(val_name.ptr(), job_action_name, strlen(job_action_name))) { - if (OB_FAIL(dml.add_column("job_action", ObHexEscapeSqlStr(val_name)))) { - LOG_WARN("failed to add column", K(ret)); - } else if (OB_FAIL(dml.add_column("what", ObHexEscapeSqlStr(val_name)))) { - LOG_WARN("failed to add column", K(ret)); - } else { - is_window_attr = true; - } - } else {/*do nothing*/} + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("the hour of interval must be between 0 and 24", K(ret)); + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL, "the hour of interval must be between 0 and 24"); } } else if (0 == attr_name.case_compare("next_date")) { ObObj time_obj; @@ -477,7 +548,8 @@ bool ObDbmsStatsMaintenanceWindow::is_stats_job(const ObString &job_name) } } if (!is_true) { - is_true = (0 == job_name.case_compare(opt_stats_history_manager)); + is_true = (0 == job_name.case_compare(opt_stats_history_manager) || + 0 == job_name.case_compare(async_gather_stats_job_proc)); } return is_true; } @@ -550,7 +622,8 @@ int ObDbmsStatsMaintenanceWindow::check_date_validate(const ObString &job_name, LOG_WARN("get unexpected error", K(ret), K(specify_time)); } else if (current_time > specify_time) { is_valid = false; - } else if (0 == job_name.case_compare(opt_stats_history_manager)) { + } else if (0 == job_name.case_compare(opt_stats_history_manager) || + 0 == job_name.case_compare(async_gather_stats_job_proc)) { is_valid = true; } else if (OB_FAIL(ObTimeConverter::usec_to_ob_time(specify_time, ob_time))) { LOG_WARN("failed to usec to ob time", K(ret), K(specify_time)); @@ -567,6 +640,172 @@ int ObDbmsStatsMaintenanceWindow::check_date_validate(const ObString &job_name, return ret; } +int ObDbmsStatsMaintenanceWindow::get_async_gather_stats_job_for_upgrade(common::ObMySQLProxy *sql_proxy, + const uint64_t tenant_id, + ObSqlString &sql) +{ + int ret = OB_SUCCESS; + lib::Worker::CompatMode compat_mode = lib::Worker::CompatMode::INVALID; + int64_t job_id = 0; + ObString exec_env; + ObSqlString values_list; + sql.reset(); + bool is_join_exists = false; + //bug: + ObArenaAllocator allocator("AsyncStatsJob"); + if (OB_FAIL(check_async_gather_job_exists(sql_proxy, tenant_id, is_join_exists))) { + LOG_WARN("failed to check async gather job exists", K(ret)); + } else if (is_join_exists) { + //do nothing + } else if (OB_FAIL(get_async_gather_stats_job_id_and_exec_env(sql_proxy, allocator, tenant_id, job_id, exec_env))) { + LOG_WARN("failed to get async gather stats job id and exec env", K(ret)); + } else if (OB_UNLIKELY(job_id > dbms_scheduler::ObDBMSSchedTableOperator::JOB_ID_OFFSET || + exec_env.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(job_id), K(exec_env)); + } else if (OB_FAIL(ObCompatModeGetter::get_tenant_mode(tenant_id, compat_mode))) { + LOG_WARN("failed to get tenant compat mode", KR(ret), K(tenant_id)); + } else if (OB_FAIL(get_async_gather_stats_job_sql(lib::Worker::CompatMode::ORACLE == compat_mode, + tenant_id, job_id, exec_env, values_list))) { + LOG_WARN("failed to get async gather stats job sql", K(ret)); + } else if (OB_FAIL(sql.append_fmt("REPLACE INTO %s( "ALL_TENANT_SCHEDULER_JOB_COLUMN_NAME") VALUES (%s)", + share::OB_ALL_TENANT_SCHEDULER_JOB_TNAME, + values_list.ptr()))) { + LOG_WARN("failed to append fmt", K(ret)); + } else { + values_list.reset(); + if (OB_FAIL(get_async_gather_stats_job_sql(lib::Worker::CompatMode::ORACLE == compat_mode, + tenant_id, 0, exec_env, values_list))) { + LOG_WARN("failed to get async gather stats job sql", K(ret)); + } else if (OB_FAIL(sql.append_fmt(", (%s);", values_list.ptr()))) { + LOG_WARN("failed to append fmt", K(ret)); + } + } + return ret; +} + +int ObDbmsStatsMaintenanceWindow::get_async_gather_stats_job_id_and_exec_env(common::ObMySQLProxy *sql_proxy, + ObIAllocator &allocator, + const uint64_t tenant_id, + int64_t &job_id, + ObString &exec_env) +{ + int ret = OB_SUCCESS; + ObSqlString select_sql; + if (OB_FAIL(select_sql.append_fmt("SELECT tt.job, t.exec_env FROM"\ + " %s t, (SELECT max(job) + 1 AS job FROM %s"\ + " WHERE tenant_id = %ld and job <= %ld AND job > 0) tt"\ + " WHERE t.tenant_id = %ld and t.job_name = '%s' AND t.job = %ld;", + share::OB_ALL_TENANT_SCHEDULER_JOB_TNAME, + share::OB_ALL_TENANT_SCHEDULER_JOB_TNAME, + share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id), + dbms_scheduler::ObDBMSSchedTableOperator::JOB_ID_OFFSET, + share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id), + opt_stats_history_manager, + OPT_STATS_HISTORY_MANAGER_JOB_ID))) { + LOG_WARN("failed to append fmt", K(ret)); + } else { + SMART_VAR(ObMySQLProxy::MySQLResult, proxy_result) { + sqlclient::ObMySQLResult *client_result = NULL; + ObSQLClientRetryWeak sql_client_retry_weak(sql_proxy); + if (OB_FAIL(sql_client_retry_weak.read(proxy_result, tenant_id, select_sql.ptr()))) { + LOG_WARN("failed to execute sql", K(ret), K(select_sql)); + } else if (OB_ISNULL(client_result = proxy_result.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to execute sql", K(ret)); + } else { + int64_t get_rows = 0; + //expected only get one row. + while (OB_SUCC(ret) && OB_SUCC(client_result->next())) { + int64_t fisrt_col = 0; + int64_t second_col = 1; + ObObj obj; + ObString tmp_exec_env; + if (get_rows > 0) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error, expected only one row", K(ret)); + } else if (OB_FAIL(client_result->get_obj(fisrt_col, obj))) { + LOG_WARN("failed to get object", K(ret)); + } else if (OB_FAIL(obj.get_int(job_id))) { + LOG_WARN("failed to get int", K(ret), K(obj)); + } else if (OB_FAIL(client_result->get_obj(second_col, obj))) { + LOG_WARN("failed to get object", K(ret)); + } else if (OB_FAIL(obj.get_varchar(tmp_exec_env))) { + LOG_WARN("failed to get int", K(ret), K(obj)); + } else if (OB_FAIL(ob_write_string(allocator, tmp_exec_env, exec_env))) { + LOG_WARN("failed to ob write string", K(ret)); + } else { + ++ get_rows; + } + } + ret = OB_ITER_END == ret ? OB_SUCCESS : ret; + } + int tmp_ret = OB_SUCCESS; + if (NULL != client_result) { + if (OB_SUCCESS != (tmp_ret = client_result->close())) { + LOG_WARN("close result set failed", K(ret), K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + } + } + LOG_INFO("succeed to get async gather stats job id and exec env", K(ret), K(select_sql), K(job_id), K(exec_env)); + } + return ret; +} + +int ObDbmsStatsMaintenanceWindow::check_async_gather_job_exists(common::ObMySQLProxy *sql_proxy, + const uint64_t tenant_id, + bool &is_join_exists) +{ + int ret = OB_SUCCESS; + is_join_exists = false; + ObSqlString select_sql; + int64_t row_count = 0; + if (OB_FAIL(select_sql.append_fmt("SELECT count(*) FROM %s WHERE tenant_id = %ld and job_name = '%s';", + share::OB_ALL_TENANT_SCHEDULER_JOB_TNAME, + share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id), + async_gather_stats_job_proc))) { + LOG_WARN("failed to append fmt", K(ret)); + } else { + SMART_VAR(ObMySQLProxy::MySQLResult, proxy_result) { + sqlclient::ObMySQLResult *client_result = NULL; + ObSQLClientRetryWeak sql_client_retry_weak(sql_proxy); + if (OB_FAIL(sql_client_retry_weak.read(proxy_result, tenant_id, select_sql.ptr()))) { + LOG_WARN("failed to execute sql", K(ret), K(select_sql)); + } else if (OB_ISNULL(client_result = proxy_result.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to execute sql", K(ret)); + } else { + //expected only get one row. + while (OB_SUCC(ret) && OB_SUCC(client_result->next())) { + int64_t idx = 0; + ObObj obj; + if (OB_FAIL(client_result->get_obj(idx, obj))) { + LOG_WARN("failed to get object", K(ret)); + } else if (OB_FAIL(obj.get_int(row_count))) { + LOG_WARN("failed to get int", K(ret), K(obj)); + } else if (OB_UNLIKELY(row_count != 2 && row_count != 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(row_count)); + } else { + is_join_exists = row_count > 0; + } + } + ret = OB_ITER_END == ret ? OB_SUCCESS : ret; + } + int tmp_ret = OB_SUCCESS; + if (NULL != client_result) { + if (OB_SUCCESS != (tmp_ret = client_result->close())) { + LOG_WARN("close result set failed", K(ret), K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + } + } + LOG_INFO("succeed to check async gather job exists", K(ret), K(select_sql), K(is_join_exists), K(row_count)); + } + return ret; +} + } // namespace common } // namespace oceanbase diff --git a/src/share/stat/ob_dbms_stats_maintenance_window.h b/src/share/stat/ob_dbms_stats_maintenance_window.h index ccca65545..62f34eaa7 100644 --- a/src/share/stat/ob_dbms_stats_maintenance_window.h +++ b/src/share/stat/ob_dbms_stats_maintenance_window.h @@ -31,8 +31,9 @@ #define DEFAULT_NON_WORKING_DAY_START_HOHR 6 #define DEFAULT_NON_WORKING_DAY_DURATION_SEC (20 * 60 * 60) #define DEFAULT_NON_WORKING_DAY_DURATION_USEC (20 * 60 * 60 * 1000000LL) -#define DEFAULT_DML_STATS_INTERVAL_USEC (15*60*1000000LL) #define DEFAULT_HISTORY_MANAGER_DURATION_SEC (12 * 60 * 60) +#define DEFAULT_ASYNC_GATHER_STATS_DURATION_SEC (10 * 60) +#define DEFAULT_ASYNC_GATHER_STATS_INTERVAL_USEC (15 * 60 * 1000000LL) namespace oceanbase { @@ -57,6 +58,10 @@ public: static bool is_stats_job(const ObString &job_name); + static int get_async_gather_stats_job_for_upgrade(common::ObMySQLProxy *sql_proxy, + const uint64_t tenant_id, + ObSqlString &sql); + private: static int get_window_job_info(const int64_t current_time, const int64_t nth_window, @@ -92,6 +97,19 @@ private: const int64_t specify_time, const int64_t current_time, bool &is_valid); + static int get_async_gather_stats_job_sql(const bool is_oracle_mode, + const uint64_t tenant_id, + const int64_t job_id, + const ObString &exec_env, + ObSqlString &raw_sql); + static int get_async_gather_stats_job_id_and_exec_env(common::ObMySQLProxy *sql_proxy, + ObIAllocator &allocator, + const uint64_t tenant_id, + int64_t &job_id, + ObString &exec_env); + static int check_async_gather_job_exists(common::ObMySQLProxy *sql_proxy, + const uint64_t tenant_id, + bool &is_join_exists); }; diff --git a/src/share/stat/ob_dbms_stats_preferences.cpp b/src/share/stat/ob_dbms_stats_preferences.cpp index fb5c1b3ae..abbb5cab7 100644 --- a/src/share/stat/ob_dbms_stats_preferences.cpp +++ b/src/share/stat/ob_dbms_stats_preferences.cpp @@ -66,27 +66,28 @@ int ObDbmsStatsPreferences::reset_global_pref_defaults(ObExecContext &ctx) return ret; } -int ObDbmsStatsPreferences::get_prefs(ObExecContext &ctx, - const ObTableStatParam ¶m, +int ObDbmsStatsPreferences::get_prefs(ObMySQLProxy *mysql_proxy, + ObIAllocator &allocator, + const uint64_t tenant_id, + const uint64_t table_id, const ObString &opt_name, ObObj &result) { int ret = OB_SUCCESS; ObSqlString get_user_sql; ObSqlString get_global_sql; - bool is_user_prefs = (param.table_id_ != OB_INVALID_ID); + bool is_user_prefs = (table_id != OB_INVALID_ID); if (OB_FAIL(get_global_sql.append_fmt(FETCH_GLOBAL_PREFS, share::OB_ALL_OPTSTAT_GLOBAL_PREFS_TNAME, opt_name.length(), opt_name.ptr()))) { LOG_WARN("failed to append fmt", K(ret), K(get_global_sql)); } else if (is_user_prefs) { - uint64_t tenant_id = param.tenant_id_; uint64_t exec_tenant_id = share::schema::ObSchemaUtils::get_exec_tenant_id(tenant_id); if (OB_FAIL(get_user_sql.append_fmt(FETCH_USER_PREFS, share::OB_ALL_OPTSTAT_USER_PREFS_TNAME, share::schema::ObSchemaUtils::get_extract_tenant_id(exec_tenant_id, tenant_id), - share::schema::ObSchemaUtils::get_extract_schema_id(exec_tenant_id, param.table_id_), + share::schema::ObSchemaUtils::get_extract_schema_id(exec_tenant_id, table_id), opt_name.length(), opt_name.ptr()))) { LOG_WARN("failed to append fmt", K(ret), K(get_user_sql)); @@ -94,11 +95,11 @@ int ObDbmsStatsPreferences::get_prefs(ObExecContext &ctx, } else {/*do nothing*/} if (OB_SUCC(ret)) { bool got_result = false; - if (is_user_prefs && OB_FAIL(do_get_prefs(ctx, param.allocator_, get_user_sql, got_result, result))) { + if (is_user_prefs && OB_FAIL(do_get_prefs(mysql_proxy, allocator, tenant_id, get_user_sql, got_result, result))) { LOG_WARN("failed to do get prefs", K(ret)); } else if (got_result) { /*do nothing*/ - } else if OB_FAIL(do_get_prefs(ctx, param.allocator_, get_global_sql, got_result, result)) { + } else if OB_FAIL(do_get_prefs(mysql_proxy, allocator, tenant_id, get_global_sql, got_result, result)) { LOG_WARN("failed to do get prefs", K(ret)); } else if (got_result) { /*do nothing*/ @@ -216,23 +217,19 @@ int ObDbmsStatsPreferences::delete_user_prefs(ObExecContext &ctx, return ret; } -int ObDbmsStatsPreferences::do_get_prefs(ObExecContext &ctx, - ObIAllocator *allocator, +int ObDbmsStatsPreferences::do_get_prefs(ObMySQLProxy *mysql_proxy, + ObIAllocator &allocator, + const uint64_t tenant_id, const ObSqlString &raw_sql, bool &get_result, ObObj &result) { int ret = OB_SUCCESS; get_result = false; - ObSQLSessionInfo *session = ctx.get_my_session(); - ObMySQLProxy *mysql_proxy = ctx.get_sql_proxy(); - if (OB_ISNULL(mysql_proxy) || OB_ISNULL(session) || - OB_ISNULL(allocator) || OB_UNLIKELY(raw_sql.empty())) { + if (OB_ISNULL(mysql_proxy) || OB_UNLIKELY(raw_sql.empty())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(mysql_proxy), K(session), - K(allocator), K(raw_sql.empty())); + LOG_WARN("get unexpected error", K(ret), K(mysql_proxy), K(raw_sql.empty())); } else { - uint64_t tenant_id = session->get_effective_tenant_id(); SMART_VAR(ObMySQLProxy::MySQLResult, proxy_result) { sqlclient::ObMySQLResult *client_result = NULL; ObSQLClientRetryWeak sql_client_retry_weak(mysql_proxy); @@ -251,7 +248,7 @@ int ObDbmsStatsPreferences::do_get_prefs(ObExecContext &ctx, LOG_WARN("get unexpected error", K(ret), K(result), K(raw_sql)); } else if (OB_FAIL(client_result->get_obj(idx, tmp))) { LOG_WARN("failed to get object", K(ret)); - } else if (OB_FAIL(ob_write_obj(*allocator, tmp, result))) { + } else if (OB_FAIL(ob_write_obj(allocator, tmp, result))) { LOG_WARN("failed to write object", K(ret)); } else { is_first = false; @@ -366,23 +363,26 @@ int ObDbmsStatsPreferences::gen_init_global_prefs_sql(ObSqlString &raw_sql, ++ total_rows; } } - if (OB_SUCC(ret)) {//init cascade - ObCascadePrefs prefs; - if (OB_ISNULL(prefs.get_stat_pref_name()) || OB_ISNULL(prefs.get_stat_pref_default_value())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(prefs.get_stat_pref_name()), - K(prefs.get_stat_pref_default_value())); - } else if (OB_FAIL(value_str.append_fmt("('%s', %s, %s, '%s'), ", - prefs.get_stat_pref_name(), - null_str, - time_str, - prefs.get_stat_pref_default_value()))) { - LOG_WARN("failed to append", K(ret)); - } else { - ++ total_rows; - } +#define init_perfs_value(perfs_type, is_last_value) \ + if (OB_SUCC(ret)) { \ + perfs_type prefs; \ + if (OB_ISNULL(prefs.get_stat_pref_name()) || OB_ISNULL(prefs.get_stat_pref_default_value())) { \ + ret = OB_ERR_UNEXPECTED; \ + LOG_WARN("get unexpected error", K(ret), K(prefs.get_stat_pref_name()), \ + K(prefs.get_stat_pref_default_value())); \ + } else if (OB_FAIL(value_str.append_fmt("('%s', %s, %s, '%s')%s ", \ + prefs.get_stat_pref_name(), \ + null_str, \ + time_str, \ + prefs.get_stat_pref_default_value(), \ + is_last_value ? ";" : ","))) { \ + LOG_WARN("failed to append", K(ret)); \ + } else { \ + ++ total_rows; \ + } \ } - if (OB_SUCC(ret)) {//init degree + init_perfs_value(ObCascadePrefs, false/*last value*/);//init cascade + if (OB_SUCC(ret)) { ObDegreePrefs prefs; if (OB_ISNULL(prefs.get_stat_pref_name()) || OB_NOT_NULL(prefs.get_stat_pref_default_value())) { ret = OB_ERR_UNEXPECTED; @@ -398,198 +398,24 @@ int ObDbmsStatsPreferences::gen_init_global_prefs_sql(ObSqlString &raw_sql, ++ total_rows; } } - if (OB_SUCC(ret)) {//init esimate_percent - ObEstimatePercentPrefs prefs; - if (OB_ISNULL(prefs.get_stat_pref_name()) || OB_ISNULL(prefs.get_stat_pref_default_value())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(prefs.get_stat_pref_name()), - K(prefs.get_stat_pref_default_value())); - } else if (OB_FAIL(value_str.append_fmt("('%s', %s, %s, '%s'), ", - prefs.get_stat_pref_name(), - null_str, - time_str, - prefs.get_stat_pref_default_value()))) { - LOG_WARN("failed to append", K(ret)); - } else { - ++ total_rows; - } - } - if (OB_SUCC(ret)) {//init incremental - ObIncrementalPrefs prefs; - if (OB_ISNULL(prefs.get_stat_pref_name()) || OB_ISNULL(prefs.get_stat_pref_default_value())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(prefs.get_stat_pref_name()), - K(prefs.get_stat_pref_default_value())); - } else if (OB_FAIL(value_str.append_fmt("('%s', %s, %s, '%s'), ", - prefs.get_stat_pref_name(), - null_str, - time_str, - prefs.get_stat_pref_default_value()))) { - LOG_WARN("failed to append", K(ret)); - } else { - ++ total_rows; - } - } - if (OB_SUCC(ret)) {//init incremental_level - ObIncrementalLevelPrefs prefs; - if (OB_ISNULL(prefs.get_stat_pref_name()) || OB_ISNULL(prefs.get_stat_pref_default_value())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(prefs.get_stat_pref_name()), - K(prefs.get_stat_pref_default_value())); - } else if (OB_FAIL(value_str.append_fmt("('%s', %s, %s, '%s'), ", - prefs.get_stat_pref_name(), - null_str, - time_str, - prefs.get_stat_pref_default_value()))) { - LOG_WARN("failed to append", K(ret)); - } else { - ++ total_rows; - } - } - if (OB_SUCC(ret)) {//init granularity - ObGranularityPrefs prefs; - if (OB_ISNULL(prefs.get_stat_pref_name()) || OB_ISNULL(prefs.get_stat_pref_default_value())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(prefs.get_stat_pref_name()), - K(prefs.get_stat_pref_default_value())); - } else if (OB_FAIL(value_str.append_fmt("('%s', %s, %s, '%s'), ", - prefs.get_stat_pref_name(), - null_str, - time_str, - prefs.get_stat_pref_default_value()))) { - LOG_WARN("failed to append", K(ret)); - } else { - ++ total_rows; - } - } - if (OB_SUCC(ret)) {//init method_opt - ObMethodOptPrefs prefs; - if (OB_ISNULL(prefs.get_stat_pref_name()) || OB_ISNULL(prefs.get_stat_pref_default_value())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(prefs.get_stat_pref_name()), - K(prefs.get_stat_pref_default_value())); - } else if (OB_FAIL(value_str.append_fmt("('%s', %s, %s, '%s'), ", - prefs.get_stat_pref_name(), - null_str, - time_str, - prefs.get_stat_pref_default_value()))) { - LOG_WARN("failed to append", K(ret)); - } else { - ++ total_rows; - } - } - if (OB_SUCC(ret)) {//init no_invalidate - ObNoInvalidatePrefs prefs; - if (OB_ISNULL(prefs.get_stat_pref_name()) || OB_ISNULL(prefs.get_stat_pref_default_value())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(prefs.get_stat_pref_name()), - K(prefs.get_stat_pref_default_value())); - } else if (OB_FAIL(value_str.append_fmt("('%s', %s, %s, '%s'), ", - prefs.get_stat_pref_name(), - null_str, - time_str, - prefs.get_stat_pref_default_value()))) { - LOG_WARN("failed to append", K(ret)); - } else { - ++ total_rows; - } - } - if (OB_SUCC(ret)) {//init options - ObOptionsPrefs prefs; - if (OB_ISNULL(prefs.get_stat_pref_name()) || OB_ISNULL(prefs.get_stat_pref_default_value())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(prefs.get_stat_pref_name()), - K(prefs.get_stat_pref_default_value())); - } else if (OB_FAIL(value_str.append_fmt("('%s', %s, %s, '%s'), ", - prefs.get_stat_pref_name(), - null_str, - time_str, - prefs.get_stat_pref_default_value()))) { - LOG_WARN("failed to append", K(ret)); - } else { - ++ total_rows; - } - } - if (OB_SUCC(ret)) {//init stale_percent - ObStalePercentPrefs prefs; - if (OB_ISNULL(prefs.get_stat_pref_name()) || OB_ISNULL(prefs.get_stat_pref_default_value())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(prefs.get_stat_pref_name()), - K(prefs.get_stat_pref_default_value())); - } else if (OB_FAIL(value_str.append_fmt("('%s', %s, %s, '%s'), ", - prefs.get_stat_pref_name(), - null_str, - time_str, - prefs.get_stat_pref_default_value()))) { - LOG_WARN("failed to append", K(ret)); - } else { - ++ total_rows; - } - } - if (OB_SUCC(ret)) {//init approximate_ndv - ObApproximateNdvPrefs prefs; - if (OB_ISNULL(prefs.get_stat_pref_name()) || OB_ISNULL(prefs.get_stat_pref_default_value())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(prefs.get_stat_pref_name()), - K(prefs.get_stat_pref_default_value())); - } else if (OB_FAIL(value_str.append_fmt("('%s', %s, %s, '%s'),", - prefs.get_stat_pref_name(), - null_str, - time_str, - prefs.get_stat_pref_default_value()))) { - LOG_WARN("failed to append", K(ret)); - } else { - ++ total_rows; - } - } - if (OB_SUCC(ret)) {//init estimate_block - ObEstimateBlockPrefs prefs; - if (OB_ISNULL(prefs.get_stat_pref_name()) || OB_ISNULL(prefs.get_stat_pref_default_value())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(prefs.get_stat_pref_name()), - K(prefs.get_stat_pref_default_value())); - } else if (OB_FAIL(value_str.append_fmt("('%s', %s, %s, '%s'),", - prefs.get_stat_pref_name(), - null_str, - time_str, - prefs.get_stat_pref_default_value()))) { - LOG_WARN("failed to append", K(ret)); - } else { - ++ total_rows; - } - } - if (OB_SUCC(ret)) {//init block_sample - ObBlockSamplePrefs prefs; - if (OB_ISNULL(prefs.get_stat_pref_name()) || OB_ISNULL(prefs.get_stat_pref_default_value())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(prefs.get_stat_pref_name()), - K(prefs.get_stat_pref_default_value())); - } else if (OB_FAIL(value_str.append_fmt("('%s', %s, %s, '%s'),", - prefs.get_stat_pref_name(), - null_str, - time_str, - prefs.get_stat_pref_default_value()))) { - LOG_WARN("failed to append", K(ret)); - } else { - ++ total_rows; - } - } - if (OB_SUCC(ret)) { - ObOnlineEstimatePercentPrefs prefs; - if (OB_ISNULL(prefs.get_stat_pref_name()) || OB_ISNULL(prefs.get_stat_pref_default_value())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(prefs.get_stat_pref_name()), - K(prefs.get_stat_pref_default_value())); - } else if (OB_FAIL(value_str.append_fmt("('%s', %s, %s, '%s');", - prefs.get_stat_pref_name(), - null_str, - time_str, - prefs.get_stat_pref_default_value()))) { - LOG_WARN("failed to append", K(ret)); - } else { - ++ total_rows; - } - } + init_perfs_value(ObEstimatePercentPrefs, false/*last value*/);//init esimate_percent + init_perfs_value(ObIncrementalPrefs, false/*last value*/);//init incremental + init_perfs_value(ObIncrementalLevelPrefs, false/*last value*/);//init incremental_level + init_perfs_value(ObGranularityPrefs, false/*last value*/);//init granularity + init_perfs_value(ObMethodOptPrefs, false/*last value*/);//init method_opt + init_perfs_value(ObNoInvalidatePrefs, false/*last value*/);//init no_invalidate + init_perfs_value(ObOptionsPrefs, false/*last value*/);//init options + init_perfs_value(ObStalePercentPrefs, false/*last value*/);//init stale_percent + init_perfs_value(ObApproximateNdvPrefs, false/*last value*/);//init approximate_ndv + init_perfs_value(ObEstimateBlockPrefs, false/*last value*/);//init estimate_block + init_perfs_value(ObBlockSamplePrefs, false/*last value*/);//init block_sample + init_perfs_value(ObOnlineEstimatePercentPrefs, false/*last value*/); + init_perfs_value(ObAsyncGatherStaleRatioPrefs, false/*last value*/);//init async gather stale ratio + init_perfs_value(ObAsyncGatherSampleSizePrefs, false/*last value*/);//init async gather sample size + init_perfs_value(ObAsyncGatherFullTableSizePrefs, false/*last value*/);//init async gather full table size + init_perfs_value(ObAsyncStaleMaxTableSizePrefs, false/*last value*/);//init async stale max table size + init_perfs_value(ObHistEstPercentPrefs, false/*last value*/);//init hist_est_percent + init_perfs_value(ObHistBlockSamplePrefs, true/*last value*/);//init hist_block_sample if (OB_SUCC(ret)) { if (OB_FAIL(raw_sql.append_fmt(INIT_GLOBAL_PREFS, share::OB_ALL_OPTSTAT_GLOBAL_PREFS_TNAME, @@ -1122,6 +948,171 @@ int ObBlockSamplePrefs::check_pref_value_validity(ObTableStatParam *param/*defau return ret; } +int ObAsyncGatherStaleRatioPrefs::check_pref_value_validity(ObTableStatParam *param/*default null*/) +{ + int ret = OB_SUCCESS; + if (!pvalue_.empty()) { + ObObj src_obj; + ObObj dest_obj; + src_obj.set_string(ObVarcharType, pvalue_); + ObArenaAllocator calc_buf("StaleRatio"); + ObCastCtx cast_ctx(&calc_buf, NULL, CM_NONE, ObCharset::get_system_collation()); + double dst_val = 0.0; + if (OB_FAIL(ObObjCaster::to_type(ObNumberType, cast_ctx, src_obj, dest_obj))) { + LOG_WARN("failed to type", K(ret), K(src_obj)); + } else if (OB_FAIL(ObDbmsStatsUtils::cast_number_to_double(dest_obj.get_number(), dst_val))) { + LOG_WARN("failed to cast number to double", K(ret), K(src_obj)); + } else if (dst_val < MINIMUM_OF_ASYNC_GATHER_STALE_RATIO) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("Illegal async gather stale ratio", K(ret), K(dst_val)); + } else if (param != NULL) { + //not implement + } else {/*do nothing*/} + if (OB_FAIL(ret)) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("Illegal async gather stale ratio", K(ret), K(pvalue_)); + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL, "Illegal async gather stale ration, the minimum of stale ratio is not less than 2"); + } + } + return ret; +} + +int ObAsyncGatherSampleSizePrefs::check_pref_value_validity(ObTableStatParam *param/*default null*/) +{ + int ret = OB_SUCCESS; + if (!pvalue_.empty()) { + ObObj src_obj; + ObObj dest_obj; + src_obj.set_string(ObVarcharType, pvalue_); + ObArenaAllocator calc_buf("SampleSize"); + ObCastCtx cast_ctx(&calc_buf, NULL, CM_NONE, ObCharset::get_system_collation()); + int64_t sample_size = 0; + if (OB_FAIL(ObObjCaster::to_type(ObNumberType, cast_ctx, src_obj, dest_obj))) { + LOG_WARN("failed to type", K(ret), K(src_obj)); + } else if (OB_FAIL(dest_obj.get_number().extract_valid_int64_with_trunc(sample_size))) { + LOG_WARN("failed to extract valid int64 with trunc", K(ret), K(src_obj)); + } else if (sample_size < MAGIC_SAMPLE_SIZE) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("Illegal async gather sample size", K(ret), K(sample_size)); + } else if (param != NULL) { + param->async_gather_sample_size_ = sample_size; + } else {/*do nothing*/} + if (OB_FAIL(ret)) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("Illegal async gather sample size", K(ret), K(pvalue_)); + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL, "Illegal async gather sample size, the minimum number of rows is not less than 5500."); + } + } + return ret; +} + +int ObAsyncGatherFullTableSizePrefs::check_pref_value_validity(ObTableStatParam *param/*default null*/) +{ + int ret = OB_SUCCESS; + if (!pvalue_.empty()) { + ObObj src_obj; + ObObj dest_obj; + src_obj.set_string(ObVarcharType, pvalue_); + ObArenaAllocator calc_buf("FullTableSize"); + ObCastCtx cast_ctx(&calc_buf, NULL, CM_NONE, ObCharset::get_system_collation()); + int64_t table_size = 0; + if (OB_FAIL(ObObjCaster::to_type(ObNumberType, cast_ctx, src_obj, dest_obj))) { + LOG_WARN("failed to type", K(ret), K(src_obj)); + } else if (OB_FAIL(dest_obj.get_number().extract_valid_int64_with_trunc(table_size))) { + LOG_WARN("failed to extract valid int64 with trunc", K(ret), K(src_obj)); + } else if (table_size < DEFAULT_ASYNC_MIN_TABLE_SIZE && table_size != 0) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("Illegal async gather gather full table size", K(ret), K(table_size)); + } else if (param != NULL) { + param->async_full_table_size_ = table_size; + } else {/*do nothing*/} + if (OB_FAIL(ret)) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("Illegal async gather gather full table size", K(ret), K(pvalue_)); + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL, "Illegal async gather gather full table size, the minimum number of rows is not less than 10000."); + } + } + return ret; +} + +int ObAsyncStaleMaxTableSizePrefs::check_pref_value_validity(ObTableStatParam *param/*default null*/) +{ + int ret = OB_SUCCESS; + if (!pvalue_.empty()) { + ObObj src_obj; + ObObj dest_obj; + src_obj.set_string(ObVarcharType, pvalue_); + ObArenaAllocator calc_buf("StaleMaxTabSize"); + ObCastCtx cast_ctx(&calc_buf, NULL, CM_NONE, ObCharset::get_system_collation()); + int64_t table_size = 0; + if (OB_FAIL(ObObjCaster::to_type(ObNumberType, cast_ctx, src_obj, dest_obj))) { + LOG_WARN("failed to type", K(ret), K(src_obj)); + } else if (OB_FAIL(dest_obj.get_number().extract_valid_int64_with_trunc(table_size))) { + LOG_WARN("failed to extract valid int64 with trunc", K(ret), K(src_obj)); + } else if (table_size < DEFAULT_ASYNC_MIN_TABLE_SIZE && table_size != 0) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("Illegal async stale max table size", K(ret), K(table_size)); + } else {/*do nothing*/} + if (OB_FAIL(ret)) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("Illegal async stale max table size", K(ret), K(pvalue_)); + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL, "Illegal async stale max table size, the minimum number of rows is not less than 10000."); + } + } + return ret; +} + +int ObHistEstPercentPrefs::check_pref_value_validity(ObTableStatParam *param/*default null*/) +{ + int ret = OB_SUCCESS; + if (!pvalue_.empty()) { + if (0 == pvalue_.case_compare("DBMS_STATS.AUTO_SAMPLE_SIZE")) { + /*do nothing*/ + } else { + ObObj src_obj; + ObObj dest_obj; + src_obj.set_string(ObVarcharType, pvalue_); + ObArenaAllocator calc_buf("HistEstPercent"); + ObCastCtx cast_ctx(&calc_buf, NULL, CM_NONE, ObCharset::get_system_collation()); + double dst_val = 0.0; + if (OB_FAIL(ObObjCaster::to_type(ObNumberType, cast_ctx, src_obj, dest_obj))) { + LOG_WARN("failed to type", K(ret), K(src_obj)); + } else if (OB_FAIL(ObDbmsStatsUtils::cast_number_to_double(dest_obj.get_number(), dst_val))) { + LOG_WARN("failed to cast number to double", K(ret), K(src_obj)); + } else if (dst_val < 0.000001 || dst_val > 100.0) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("Illegal value for hist est percent", K(ret), K(dst_val)); + } else if (param != NULL) { + param->hist_sample_info_.set_percent(dst_val); + } else {/*do nothing*/} + if (OB_FAIL(ret)) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL, "Illegal sample percent: must be in the range [0.000001,100]"); + } + } + } + return ret; +} + +int ObHistBlockSamplePrefs::check_pref_value_validity(ObTableStatParam *param/*default null*/) +{ + int ret = OB_SUCCESS; + if (pvalue_.empty() || 0 == pvalue_.case_compare("FALSE")) { + if (param != NULL) { + param->hist_sample_info_.set_is_block_sample(false); + } + } else if (0 == pvalue_.case_compare("TRUE")) { + if (param != NULL) { + param->hist_sample_info_.set_is_block_sample(true); + } + } else { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("Illegal value for BLOCK_SAMPLE", K(ret), K(pvalue_)); + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL,"Illegal value for BLOCK_SAMPLE: must be {TRUE, FALSE}"); + } + return ret; +} + #define ISSPACE(c) ((c) == ' ' || (c) == '\n' || (c) == '\r' || (c) == '\t' || (c) == '\f' || (c) == '\v') //compatible oracle, global prefs/schema prefs just only can set "for all columns...." @@ -1186,6 +1177,78 @@ int ObOnlineEstimatePercentPrefs::check_pref_value_validity(ObTableStatParam *pa return ret; } +int ObDbmsStatsPreferences::get_extra_stats_perfs_for_upgrade(ObSqlString &raw_sql) +{ + int ret = OB_SUCCESS; + const char *null_str = "NULL"; + const char *time_str = "CURRENT_TIMESTAMP"; + ObSqlString value_str; + if (OB_SUCC(ret)) {//init async gather stale ratio + ObAsyncGatherStaleRatioPrefs prefs; + if (OB_ISNULL(prefs.get_stat_pref_name()) || OB_ISNULL(prefs.get_stat_pref_default_value())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(prefs.get_stat_pref_name()), + K(prefs.get_stat_pref_default_value())); + } else if (OB_FAIL(value_str.append_fmt("('%s', %s, %s, '%s'),", + prefs.get_stat_pref_name(), + null_str, + time_str, + prefs.get_stat_pref_default_value()))) { + LOG_WARN("failed to append", K(ret)); + } + } + if (OB_SUCC(ret)) {//init async gather sample size + ObAsyncGatherSampleSizePrefs prefs; + if (OB_ISNULL(prefs.get_stat_pref_name()) || OB_ISNULL(prefs.get_stat_pref_default_value())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(prefs.get_stat_pref_name()), + K(prefs.get_stat_pref_default_value())); + } else if (OB_FAIL(value_str.append_fmt("('%s', %s, %s, '%s'),", + prefs.get_stat_pref_name(), + null_str, + time_str, + prefs.get_stat_pref_default_value()))) { + LOG_WARN("failed to append", K(ret)); + } + } + if (OB_SUCC(ret)) {//init async gather full table size + ObAsyncGatherFullTableSizePrefs prefs; + if (OB_ISNULL(prefs.get_stat_pref_name()) || OB_ISNULL(prefs.get_stat_pref_default_value())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(prefs.get_stat_pref_name()), + K(prefs.get_stat_pref_default_value())); + } else if (OB_FAIL(value_str.append_fmt("('%s', %s, %s, '%s'), ", + prefs.get_stat_pref_name(), + null_str, + time_str, + prefs.get_stat_pref_default_value()))) { + LOG_WARN("failed to append", K(ret)); + } + } + if (OB_SUCC(ret)) {//init async stale max table size + ObAsyncStaleMaxTableSizePrefs prefs; + if (OB_ISNULL(prefs.get_stat_pref_name()) || OB_ISNULL(prefs.get_stat_pref_default_value())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(prefs.get_stat_pref_name()), + K(prefs.get_stat_pref_default_value())); + } else if (OB_FAIL(value_str.append_fmt("('%s', %s, %s, '%s');", + prefs.get_stat_pref_name(), + null_str, + time_str, + prefs.get_stat_pref_default_value()))) { + LOG_WARN("failed to append", K(ret)); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(raw_sql.append_fmt(UPGRADE_GLOBAL_PREFS, + share::OB_ALL_OPTSTAT_GLOBAL_PREFS_TNAME, + value_str.ptr()))) { + LOG_WARN("failed to append fmt", K(ret)); + } + } + return ret; +} + int ObDbmsStatsPreferences::get_online_estimate_percent_for_upgrade(ObSqlString &raw_sql) { int ret = OB_SUCCESS; diff --git a/src/share/stat/ob_dbms_stats_preferences.h b/src/share/stat/ob_dbms_stats_preferences.h index 140fef096..4dd94d338 100644 --- a/src/share/stat/ob_dbms_stats_preferences.h +++ b/src/share/stat/ob_dbms_stats_preferences.h @@ -241,6 +241,84 @@ class ObOnlineEstimatePercentPrefs : public ObStatPrefs const char* get_stat_pref_for_update() const { return "100"; } }; +class ObAsyncGatherStaleRatioPrefs : public ObStatPrefs +{ + public: + ObAsyncGatherStaleRatioPrefs() : ObStatPrefs() {} + ObAsyncGatherStaleRatioPrefs(ObIAllocator *alloc, + ObSQLSessionInfo *session_info, + const ObString &pvalue) : + ObStatPrefs(alloc, session_info, pvalue) {} + virtual int check_pref_value_validity(ObTableStatParam *param = NULL) override; + virtual const char* get_stat_pref_name() const { return "ASYNC_GATHER_STALE_RATIO"; } + virtual const char* get_stat_pref_default_value() const { return "10"; } +}; + +class ObAsyncGatherSampleSizePrefs : public ObStatPrefs +{ + public: + ObAsyncGatherSampleSizePrefs() : ObStatPrefs() {} + ObAsyncGatherSampleSizePrefs(ObIAllocator *alloc, + ObSQLSessionInfo *session_info, + const ObString &pvalue) : + ObStatPrefs(alloc, session_info, pvalue) {} + virtual int check_pref_value_validity(ObTableStatParam *param = NULL) override; + virtual const char* get_stat_pref_name() const { return "ASYNC_GATHER_SAMPLE_SIZE"; } + virtual const char* get_stat_pref_default_value() const { return "1000000"; } +}; + +class ObAsyncGatherFullTableSizePrefs : public ObStatPrefs +{ + public: + ObAsyncGatherFullTableSizePrefs() : ObStatPrefs() {} + ObAsyncGatherFullTableSizePrefs(ObIAllocator *alloc, + ObSQLSessionInfo *session_info, + const ObString &pvalue) : + ObStatPrefs(alloc, session_info, pvalue) {} + virtual int check_pref_value_validity(ObTableStatParam *param = NULL) override; + virtual const char* get_stat_pref_name() const { return "ASYNC_GATHER_FULL_TABLE_SIZE"; } + virtual const char* get_stat_pref_default_value() const { return "10000000"; } +}; + +class ObAsyncStaleMaxTableSizePrefs : public ObStatPrefs +{ + public: + ObAsyncStaleMaxTableSizePrefs() : ObStatPrefs() {} + ObAsyncStaleMaxTableSizePrefs(ObIAllocator *alloc, + ObSQLSessionInfo *session_info, + const ObString &pvalue) : + ObStatPrefs(alloc, session_info, pvalue) {} + virtual int check_pref_value_validity(ObTableStatParam *param = NULL) override; + virtual const char* get_stat_pref_name() const { return "ASYNC_STALE_MAX_TABLE_SIZE"; } + virtual const char* get_stat_pref_default_value() const { return "100000000"; } +}; + +class ObHistEstPercentPrefs : public ObStatPrefs +{ + public: + ObHistEstPercentPrefs() : ObStatPrefs() {} + ObHistEstPercentPrefs(ObIAllocator *alloc, + ObSQLSessionInfo *session_info, + const ObString &pvalue) : + ObStatPrefs(alloc, session_info, pvalue) {} + virtual int check_pref_value_validity(ObTableStatParam *param = NULL) override; + virtual const char* get_stat_pref_name() const { return "HIST_EST_PERCENT"; } + virtual const char* get_stat_pref_default_value() const { return "DBMS_STATS.AUTO_SAMPLE_SIZE";} +}; + +class ObHistBlockSamplePrefs : public ObStatPrefs +{ + public: + ObHistBlockSamplePrefs() : ObStatPrefs() {} + ObHistBlockSamplePrefs(ObIAllocator *alloc, + ObSQLSessionInfo *session_info, + const ObString &pvalue) : + ObStatPrefs(alloc, session_info, pvalue) {} + virtual int check_pref_value_validity(ObTableStatParam *param = NULL) override; + virtual const char* get_stat_pref_name() const { return "HIST_BLOCK_SAMPLE"; } + virtual const char* get_stat_pref_default_value() const { return "FALSE"; } +}; + template static int new_stat_prefs(ObIAllocator &allocator, ObSQLSessionInfo *session_info, const ObString &opt_value, T *&src) @@ -263,8 +341,10 @@ public: static int reset_global_pref_defaults(ObExecContext &ctx); - static int get_prefs(ObExecContext &ctx, - const ObTableStatParam ¶m, + static int get_prefs(ObMySQLProxy *mysql_proxy, + ObIAllocator &allocator, + const uint64_t tenant_id, + const uint64_t table_id, const ObString &opt_name, ObObj &result); @@ -287,9 +367,12 @@ public: static int get_online_estimate_percent_for_upgrade(ObSqlString &sql); + static int get_extra_stats_perfs_for_upgrade(ObSqlString &sql); + private: - static int do_get_prefs(ObExecContext &ctx, - ObIAllocator *allocator, + static int do_get_prefs(ObMySQLProxy *mysql_proxy, + ObIAllocator &allocator, + const uint64_t tenant_id, const ObSqlString &raw_sql, bool &get_result, ObObj &result); diff --git a/src/share/stat/ob_dbms_stats_utils.cpp b/src/share/stat/ob_dbms_stats_utils.cpp index 068b6ca61..38003395f 100644 --- a/src/share/stat/ob_dbms_stats_utils.cpp +++ b/src/share/stat/ob_dbms_stats_utils.cpp @@ -26,6 +26,8 @@ #include "sql/ob_result_set.h" #include "sql/optimizer/ob_opt_selectivity.h" #include "share/stat/ob_dbms_stats_preferences.h" +#include "observer/ob_sql_client_decorator.h" +#include "share/stat/ob_dbms_stats_executor.h" #ifdef OB_BUILD_ORACLE_PL #include "pl/sys_package/ob_json_pl_utils.h" @@ -171,7 +173,7 @@ int ObDbmsStatsUtils::check_is_stat_table(share::schema::ObSchemaGetterGuard &sc const int64_t table_id, bool &is_valid) { - bool ret = OB_SUCCESS; + int ret = OB_SUCCESS; is_valid = false; const ObTableSchema *table_schema = NULL; if (is_sys_table(table_id)) {//check sys table @@ -1103,6 +1105,13 @@ int ObDbmsStatsUtils::prepare_gather_stat_param(const ObTableStatParam ¶m, gather_param.global_part_id_ = param.global_part_id_; gather_param.gather_vectorize_ = gather_vectorize; gather_param.use_column_store_ = use_column_store; + gather_param.is_async_gather_ = param.is_async_gather_; + gather_param.async_gather_sample_size_ = param.async_gather_sample_size_; + gather_param.async_full_table_size_ = param.async_full_table_size_; + gather_param.hist_sample_info_.is_sample_ = param.hist_sample_info_.is_sample_; + gather_param.hist_sample_info_.is_block_sample_ = param.hist_sample_info_.is_block_sample_; + gather_param.hist_sample_info_.sample_type_ = param.hist_sample_info_.sample_type_; + gather_param.hist_sample_info_.sample_value_ = param.hist_sample_info_.sample_value_; return gather_param.column_group_params_.assign(param.column_group_params_); } @@ -1293,7 +1302,7 @@ int ObDbmsStatsUtils::implicit_commit_before_gather_stats(sql::ObExecContext &ct LOG_WARN("failed to get_optimizer_features_enable_version", K(ret)); } else if (optimizer_features_enable_version < COMPAT_VERSION_4_2_4 || (optimizer_features_enable_version >= COMPAT_VERSION_4_3_0 && - optimizer_features_enable_version < COMPAT_VERSION_4_3_2)) { + optimizer_features_enable_version < COMPAT_VERSION_4_3_3)) { //do nothing } else if (OB_FAIL(ObResultSet::implicit_commit_before_cmd_execute(*ctx.get_my_session(), ctx, stmt::T_ANALYZE))) { LOG_WARN("failed to implicit commit before cmd execute", K(ret)); @@ -1405,5 +1414,131 @@ int ObDbmsStatsUtils::get_sys_online_estimate_percent(sql::ObExecContext &ctx, return ret; } +int ObDbmsStatsUtils::check_can_async_gather_stats(sql::ObExecContext &ctx) +{ + int ret = OB_SUCCESS; + ObSqlString raw_sql; + if (OB_ISNULL(ctx.get_my_session())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(ctx.get_my_session())); + } else if (OB_FAIL(raw_sql.append_fmt("SELECT 1 FROM dual WHERE EXISTS(SELECT 1 FROM %s WHERE tenant_id = %lu);", + share::OB_ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR_TNAME, + ctx.get_my_session()->get_effective_tenant_id()))) { + LOG_WARN("failed to append fmt", K(ret)); + } else { + uint64_t tenant_id = ctx.get_my_session()->get_effective_tenant_id(); + SMART_VAR(ObMySQLProxy::MySQLResult, proxy_result) { + sqlclient::ObMySQLResult *client_result = NULL; + ObSQLClientRetryWeak sql_client_retry_weak(ctx.get_sql_proxy()); + if (OB_FAIL(sql_client_retry_weak.read(proxy_result, tenant_id, raw_sql.ptr()))) { + LOG_WARN("failed to execute sql", K(ret), K(raw_sql)); + } else if (OB_ISNULL(client_result = proxy_result.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to execute sql", K(ret)); + } else if (OB_FAIL(client_result->next())) { + if (OB_ITER_END != ret) { + LOG_WARN("failed to get next", K(ret)); + } else { + ret = OB_SUCCESS; + } + } else { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("async stats gathering needs to wait for other stats gathering tasks to finish", K(ret)); + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL,"async stats gathering needs to wait for other stats gathering tasks to finish"); + } + int tmp_ret = OB_SUCCESS; + if (NULL != client_result) { + if (OB_SUCCESS != (tmp_ret = client_result->close())) { + LOG_WARN("close result set failed", K(ret), K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + } + } + } + return ret; +} + +int ObDbmsStatsUtils::cancel_async_gather_stats(sql::ObExecContext &ctx) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(ctx.get_my_session())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(ctx.get_my_session())); + } else { + ObSEArray task_ids; + uint64_t tenant_id = ctx.get_my_session()->get_effective_tenant_id(); + ObArenaAllocator allocator("CancelAsyGather", OB_MALLOC_NORMAL_BLOCK_SIZE, tenant_id); + if (OB_FAIL(fetch_need_cancel_async_gather_stats_task(allocator, ctx, task_ids))) { + LOG_WARN("failed to fetch need cancel async gather stats task", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < task_ids.count(); ++i) { + if (OB_FAIL(ObDbmsStatsExecutor::cancel_gather_stats(ctx, task_ids.at(i)))) { + if (ret != OB_ERR_DBMS_STATS_PL) { + LOG_WARN("failed to cancel gather stats", K(ret)); + } else { + ret = OB_SUCCESS; + } + } + } + } + } + return ret; +} + +int ObDbmsStatsUtils::fetch_need_cancel_async_gather_stats_task(ObIAllocator &allocator, + sql::ObExecContext &ctx, + ObIArray &task_ids) +{ + int ret = OB_SUCCESS; + ObSqlString raw_sql; + if (OB_ISNULL(ctx.get_my_session())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(ctx.get_my_session())); + } else if (OB_FAIL(raw_sql.append_fmt("SELECT task_id FROM %s WHERE tenant_id = %lu and type = %d;", + share::OB_ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR_TNAME, + ctx.get_my_session()->get_effective_tenant_id(), + ObOptStatGatherType::AYSNC_GATHER))) { + LOG_WARN("failed to append fmt", K(ret)); + } else { + uint64_t tenant_id = ctx.get_my_session()->get_effective_tenant_id(); + SMART_VAR(ObMySQLProxy::MySQLResult, proxy_result) { + sqlclient::ObMySQLResult *client_result = NULL; + ObSQLClientRetryWeak sql_client_retry_weak(ctx.get_sql_proxy()); + if (OB_FAIL(sql_client_retry_weak.read(proxy_result, tenant_id, raw_sql.ptr()))) { + LOG_WARN("failed to execute sql", K(ret), K(raw_sql)); + } else if (OB_ISNULL(client_result = proxy_result.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to execute sql", K(ret)); + } else { + while (OB_SUCC(ret) && OB_SUCC(client_result->next())) { + int64_t idx = 0; + ObObj obj; + ObString str; + ObString tmp_str; + if (OB_FAIL(client_result->get_obj(idx, obj))) { + LOG_WARN("failed to get object", K(ret)); + } else if (OB_FAIL(obj.get_string(str))) { + LOG_WARN("failed to get string", K(ret)); + } else if (OB_FAIL(ob_write_string(allocator, str, tmp_str))) { + LOG_WARN("failed to get int", K(ret), K(obj)); + } else if (OB_FAIL(task_ids.push_back(tmp_str))) { + LOG_WARN("failed to push back", K(ret)); + } + } + ret = OB_ITER_END == ret ? OB_SUCCESS : ret; + } + int tmp_ret = OB_SUCCESS; + if (NULL != client_result) { + if (OB_SUCCESS != (tmp_ret = client_result->close())) { + LOG_WARN("close result set failed", K(ret), K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + } + } + LOG_TRACE("failed to fetch need cancel async gather stats task", K(task_ids)); + } + return ret; +} + } } diff --git a/src/share/stat/ob_dbms_stats_utils.h b/src/share/stat/ob_dbms_stats_utils.h index 7f966536e..8ab24a738 100644 --- a/src/share/stat/ob_dbms_stats_utils.h +++ b/src/share/stat/ob_dbms_stats_utils.h @@ -195,6 +195,10 @@ public: const uint64_t tenant_id, const uint64_t table_id, double &percent); + static int check_can_async_gather_stats(sql::ObExecContext &ctx); + + static int cancel_async_gather_stats(sql::ObExecContext &ctx); + private: static int batch_write(share::schema::ObSchemaGetterGuard *schema_guard, const uint64_t tenant_id, @@ -206,6 +210,10 @@ private: const bool is_online_stat = false, const ObObjPrintParams &print_params = ObObjPrintParams()); + static int fetch_need_cancel_async_gather_stats_task(ObIAllocator &allocator, + sql::ObExecContext &ctx, + ObIArray &task_ids); + }; } diff --git a/src/share/stat/ob_hybrid_hist_estimator.cpp b/src/share/stat/ob_hybrid_hist_estimator.cpp index e2e81ee9d..8d17a91db 100644 --- a/src/share/stat/ob_hybrid_hist_estimator.cpp +++ b/src/share/stat/ob_hybrid_hist_estimator.cpp @@ -53,8 +53,10 @@ int ObHybridHistEstimator::estimate(const ObOptStatGatherParam ¶m, ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(hybrid_col_params.count()), K(hybrid_col_stats.count())); } else if (OB_FAIL(compute_estimate_percent(opt_stat.table_stat_->get_row_count(), + opt_stat.table_stat_->get_micro_block_num(), + opt_stat.table_stat_->get_sstable_row_count() >= opt_stat.table_stat_->get_memtable_row_count(), max_num_buckets, - param.sample_info_, + param.hist_sample_info_, need_sample, est_percent, is_block_sample))) { @@ -328,6 +330,8 @@ int ObHybridHistEstimator::try_build_hybrid_hist(const ObColumnStatParam ¶m, * ii: if total_row_count > MAGIC_SAMPLE_SIZE then choosing MAGIC_SAMPLE_SIZE to sample; */ int ObHybridHistEstimator::compute_estimate_percent(int64_t total_row_count, + int64_t micro_block_num, + bool sstable_rows_more, int64_t max_num_bkts, const ObAnalyzeSampleInfo &sample_info, bool &need_sample, @@ -335,11 +339,11 @@ int ObHybridHistEstimator::compute_estimate_percent(int64_t total_row_count, bool &is_block_sample) { int ret = OB_SUCCESS; + is_block_sample = sample_info.is_block_sample_; if (0 == total_row_count) { need_sample = false; } else if (sample_info.is_sample_) { need_sample = true; - is_block_sample = sample_info.is_block_sample_; if (sample_info.sample_type_ == SampleType::RowSample) { if (sample_info.sample_value_ < total_row_count) { est_percent = (sample_info.sample_value_ * 100.0) / total_row_count; @@ -354,32 +358,22 @@ int ObHybridHistEstimator::compute_estimate_percent(int64_t total_row_count, } if (OB_SUCC(ret) && need_sample) { if (total_row_count * est_percent / 100 >= MAGIC_MIN_SAMPLE_SIZE) { - const int64_t MAGIC_MAX_SPECIFY_SAMPLE_SIZE = 1000000; - is_block_sample = !is_block_sample ? total_row_count >= MAX_AUTO_GATHER_FULL_TABLE_ROWS : is_block_sample; - int64_t max_allowed_multiple = max_num_bkts <= ObColumnStatParam::DEFAULT_HISTOGRAM_BUCKET_NUM ? 1 : - max_num_bkts / ObColumnStatParam::DEFAULT_HISTOGRAM_BUCKET_NUM; - int64_t max_specify_sample_size = MAGIC_MAX_SPECIFY_SAMPLE_SIZE * max_allowed_multiple; - if (total_row_count * est_percent / 100 >= max_specify_sample_size) { - est_percent = max_specify_sample_size * 100.0 / total_row_count; - } + //do nothing } else if (total_row_count <= MAGIC_SAMPLE_SIZE) { need_sample = false; est_percent = 0.0; is_block_sample = false; } else { - is_block_sample = total_row_count >= MAX_AUTO_GATHER_FULL_TABLE_ROWS; est_percent = (MAGIC_SAMPLE_SIZE * 100.0) / total_row_count; } } - } else if (total_row_count >= MAX_AUTO_GATHER_FULL_TABLE_ROWS) { - need_sample = true; - is_block_sample = true; - const int64_t MAGIC_MAX_SAMPLE_SIZE = 100000; - est_percent = MAGIC_MAX_SAMPLE_SIZE * 100.0 / total_row_count; } else if (total_row_count >= MAGIC_MAX_AUTO_SAMPLE_SIZE) { + if (micro_block_num > MAXIMUM_BLOCK_CNT_OF_ROW_SAMPLE_GATHER_HYBRID_HIST || + total_row_count > MAXIMUM_ROWS_OF_ROW_SAMPLE_GATHER_HYBRID_HIST) { + is_block_sample = true; + } if (max_num_bkts <= ObColumnStatParam::DEFAULT_HISTOGRAM_BUCKET_NUM) { need_sample = true; - is_block_sample = false; est_percent = (MAGIC_SAMPLE_SIZE * 100.0) / total_row_count; } else { int64_t num_bound_bkts = static_cast(std::round(total_row_count * MAGIC_SAMPLE_CUT_RATIO)); @@ -389,7 +383,6 @@ int ObHybridHistEstimator::compute_estimate_percent(int64_t total_row_count, int64_t sample_size = MAGIC_SAMPLE_SIZE + MAGIC_BASE_SAMPLE_SIZE + (max_num_bkts - ObColumnStatParam::DEFAULT_HISTOGRAM_BUCKET_NUM) * MAGIC_MIN_SAMPLE_SIZE * 0.01; need_sample = true; - is_block_sample = false; est_percent = (sample_size * 100.0) / total_row_count; } } @@ -397,6 +390,11 @@ int ObHybridHistEstimator::compute_estimate_percent(int64_t total_row_count, need_sample = false; } if (OB_SUCC(ret)) { + //refine est_percent avoid sampling block cnt is too small. + if (is_block_sample && sstable_rows_more && + (est_percent / 100.0 * micro_block_num) < MINIMUM_BLOCK_CNT_OF_BLOCK_SAMPLE_HYBRID_HIST) { + est_percent = MINIMUM_BLOCK_CNT_OF_BLOCK_SAMPLE_HYBRID_HIST * 100.0 / micro_block_num; + } // refine est_percent est_percent = std::max(0.000001, est_percent); if (est_percent >= 100) { @@ -404,7 +402,7 @@ int ObHybridHistEstimator::compute_estimate_percent(int64_t total_row_count, } } - LOG_TRACE("Succeed to compute estimate percent", K(ret), K(total_row_count), K(max_num_bkts), + LOG_TRACE("Succeed to compute estimate percent", K(ret), K(total_row_count), K(max_num_bkts), K(micro_block_num), K(need_sample), K(est_percent), K(is_block_sample)); return ret; } diff --git a/src/share/stat/ob_hybrid_hist_estimator.h b/src/share/stat/ob_hybrid_hist_estimator.h index 3c2fd7008..e67ac64c0 100644 --- a/src/share/stat/ob_hybrid_hist_estimator.h +++ b/src/share/stat/ob_hybrid_hist_estimator.h @@ -109,6 +109,8 @@ private: bool &is_done); int compute_estimate_percent(int64_t total_row_count, + int64_t micro_block_num, + bool sstable_rows_more, int64_t max_num_buckets, const ObAnalyzeSampleInfo &sample_info, bool &need_sample, diff --git a/src/share/stat/ob_incremental_stat_estimator.cpp b/src/share/stat/ob_incremental_stat_estimator.cpp index 54939a9a1..a7b25baa1 100644 --- a/src/share/stat/ob_incremental_stat_estimator.cpp +++ b/src/share/stat/ob_incremental_stat_estimator.cpp @@ -603,7 +603,6 @@ int ObIncrementalStatEstimator::derive_global_col_stat(ObExecContext &ctx, ObGlobalNdvEval ndv_eval; ObGlobalAvglenEval avglen_eval; ObSEArray all_part_histograms; - bool can_try_derive_hist = need_derive_hist; int64_t max_bucket_num = param.column_params_.at(i).bucket_num_; for (int64_t j = 0; OB_SUCC(ret) && j < part_cnt; ++j) { ObOptColumnStat *opt_col_stat = NULL; @@ -630,7 +629,6 @@ int ObIncrementalStatEstimator::derive_global_col_stat(ObExecContext &ctx, OB_FAIL(all_part_histograms.push_back(opt_col_stat->get_histogram()))) { LOG_WARN("failed to push back histogram", K(ret)); } else { - can_try_derive_hist &= (opt_col_stat->get_num_distinct() == 0 || opt_col_stat->get_histogram().is_valid()); null_eval.add(opt_col_stat->get_num_null()); if (opt_col_stat->get_num_distinct() != 0) { min_eval.add(opt_col_stat->get_min_value()); @@ -667,7 +665,7 @@ int ObIncrementalStatEstimator::derive_global_col_stat(ObExecContext &ctx, } else if (OB_FAIL(col_stats.push_back(col_stat))) { LOG_WARN("failed to push back", K(ret)); } else if (need_derive_hist) { - if (can_try_derive_hist && col_stat->get_num_distinct() > 0) { + if (col_stat->get_num_distinct() > 0 && !all_part_histograms.empty()) { if (OB_FAIL(derive_global_histogram(all_part_histograms, alloc, max_bucket_num, @@ -678,15 +676,16 @@ int ObIncrementalStatEstimator::derive_global_col_stat(ObExecContext &ctx, need_gather_hist))) { LOG_WARN("failed to derive global histogram from part histogram", K(ret)); } - } else if (max_bucket_num > 1 && + } else if (approx_level == PARTITION_LEVEL && + max_bucket_num > 1 && param.column_params_.at(i).need_basic_stat() && col_stat->get_num_distinct() > 0) { need_gather_hist = true; - int64_t max_disuse_cnt = std::ceil(col_stat->get_num_not_null() * 1.0 / max_bucket_num); //After testing, the error of using hyperloglog to estimate ndv is within %5. + //In order to improve the gathering efficiency, it is required that ndv cannot be less than the number of buckets const double MAX_LLC_NDV_ERR_RATE = !param.need_approx_ndv_ ? 0.0 : 0.05; const int64_t fault_tolerance_cnt = std::ceil(col_stat->get_num_distinct() * MAX_LLC_NDV_ERR_RATE); - if (col_stat->get_num_distinct() >= max_bucket_num + max_disuse_cnt + fault_tolerance_cnt) { + if (col_stat->get_num_distinct() >= max_bucket_num + fault_tolerance_cnt) { //directly gather hybrid histogram col_stat->get_histogram().set_type(ObHistType::HYBIRD); } else { @@ -694,7 +693,7 @@ int ObIncrementalStatEstimator::derive_global_col_stat(ObExecContext &ctx, col_stat->get_histogram().set_type(ObHistType::TOP_FREQUENCY); } } - LOG_TRACE("succeed to derive global col stat", K(*col_stat)); + LOG_TRACE("succeed to derive global col stat", K(*col_stat), K(need_gather_hist)); } } } @@ -754,8 +753,8 @@ int ObIncrementalStatEstimator::derive_global_histogram(ObIArray &a LOG_WARN("failed to allocate memory", K(ret), K(ptr)); } else { ObTopKFrequencyHistograms *top_k_fre_hist = new (ptr) ObTopKFrequencyHistograms(); - top_k_fre_hist->set_window_size(1000); - top_k_fre_hist->set_item_size(256); + top_k_fre_hist->set_window_size(ObStatTopKHist::get_window_size(max_bucket_num)); + top_k_fre_hist->set_item_size(max_bucket_num); top_k_fre_hist->set_is_topk_hist_need_des_row(true); top_k_fre_hist->set_max_disuse_cnt(std::ceil(not_null_count * 1.0 / max_bucket_num)); for (int64_t i = 0; OB_SUCC(ret) && i < all_part_histograms.count(); ++i) { diff --git a/src/share/stat/ob_opt_stat_gather_stat.cpp b/src/share/stat/ob_opt_stat_gather_stat.cpp index 641031aa4..800bd5cf3 100644 --- a/src/share/stat/ob_opt_stat_gather_stat.cpp +++ b/src/share/stat/ob_opt_stat_gather_stat.cpp @@ -192,7 +192,7 @@ int ObOptStatGatherStat::deep_copy(common::ObIAllocator &allocator, ObOptStatGat return ret; } //----------------------------------------------------- -int ObOptStatRunningMonitor::add_table_info(common::ObTableStatParam &table_param, +int ObOptStatRunningMonitor::add_table_info(const common::ObTableStatParam &table_param, double stale_percent) { int ret = OB_SUCCESS; @@ -211,7 +211,7 @@ int ObOptStatRunningMonitor::add_table_info(common::ObTableStatParam &table_para opt_stat_gather_stat_.set_table_id(table_param.table_id_); ObSqlString properties_sql_str; char *buf = NULL; - if (OB_FAIL(properties_sql_str.append_fmt("GRANULARITY:%.*s;METHOD_OPT:%.*s;DEGREE:%ld;ESTIMATE_PERCENT:%lf;BLOCK_SAMPLE:%d;STALE_PERCENT:%lf;", + if (OB_FAIL(properties_sql_str.append_fmt("GRANULARITY:%.*s;METHOD_OPT:%.*s;DEGREE:%ld;ESTIMATE_PERCENT:%lf;BLOCK_SAMPLE:%d;STALE_PERCENT:%lf;HIST_EST_PERCENT:%lf", table_param.granularity_.length(), table_param.granularity_.ptr(), table_param.method_opt_.length(), @@ -219,7 +219,8 @@ int ObOptStatRunningMonitor::add_table_info(common::ObTableStatParam &table_para table_param.degree_, table_param.sample_info_.is_sample_ ? table_param.sample_info_.sample_value_ : 100.0, table_param.sample_info_.is_block_sample_, - stale_percent))) { + stale_percent, + table_param.hist_sample_info_.is_sample_ ? table_param.hist_sample_info_.sample_value_ : 100.0))) { LOG_WARN("failed to append fmt", K(ret)); } else if (OB_ISNULL(buf = static_cast(allocator_.alloc(properties_sql_str.length())))) { ret = OB_ALLOCATE_MEMORY_FAILED; diff --git a/src/share/stat/ob_opt_stat_gather_stat.h b/src/share/stat/ob_opt_stat_gather_stat.h index 3bfe8ef69..3a47b4473 100644 --- a/src/share/stat/ob_opt_stat_gather_stat.h +++ b/src/share/stat/ob_opt_stat_gather_stat.h @@ -26,7 +26,8 @@ namespace common enum ObOptStatGatherType { INVALID_GATHER_TYPE = -1, MANUAL_GATHER, - AUTO_GATHER + AUTO_GATHER, + AYSNC_GATHER }; enum ObOptStatRunningPhase { @@ -213,7 +214,7 @@ struct ObOptStatRunningMonitor void init(int64_t current_time, int64_t current_memory_used, ObOptStatGatherStat &opt_stat_gather_stat); - int add_table_info(common::ObTableStatParam &table_param, + int add_table_info(const common::ObTableStatParam &table_param, double stale_percent = -1.0); int add_monitor_info(ObOptStatRunningPhase current_phase, double extra_progress_ratio = 0); double get_monitor_extra_progress_ratio(ObOptStatRunningPhase current_phase, diff --git a/src/share/stat/ob_opt_stat_manager.cpp b/src/share/stat/ob_opt_stat_manager.cpp index 2e42a468d..8afbe48eb 100644 --- a/src/share/stat/ob_opt_stat_manager.cpp +++ b/src/share/stat/ob_opt_stat_manager.cpp @@ -637,6 +637,7 @@ int ObOptStatManager::get_table_stat(const uint64_t tenant_id, opt_stat.get_micro_block_num() * scale_ratio); stat.set_last_analyzed(opt_stat.get_last_analyzed()); stat.set_stat_locked(opt_stat.is_locked()); + stat.set_stale_stats(opt_stat.is_stat_expired()); } return ret; } diff --git a/src/share/stat/ob_opt_stat_monitor_manager.cpp b/src/share/stat/ob_opt_stat_monitor_manager.cpp index e2648ddc2..466aafd35 100644 --- a/src/share/stat/ob_opt_stat_monitor_manager.cpp +++ b/src/share/stat/ob_opt_stat_monitor_manager.cpp @@ -26,6 +26,7 @@ #include "storage/ob_locality_manager.h" #include "lib/rc/ob_rc.h" #include "observer/ob_server.h" +#include "pl/sys_package/ob_dbms_stats.h" namespace oceanbase { @@ -73,6 +74,30 @@ namespace common "updates = updates + values(updates)," \ "deletes = deletes + values(deletes);" +#define INSERT_STALE_TABLE_STAT_SQL "INSERT /*+QUERY_TIMEOUT(60000000)*/INTO %s(tenant_id," \ + "table_id," \ + "partition_id," \ + "index_type," \ + "object_type," \ + "last_analyzed," \ + "sstable_row_cnt," \ + "sstable_avg_row_len," \ + "macro_blk_cnt," \ + "micro_blk_cnt," \ + "memtable_row_cnt," \ + "memtable_avg_row_len," \ + "row_cnt," \ + "avg_row_len," \ + "global_stats," \ + "user_stats," \ + "stattype_locked," \ + "stale_stats) VALUES %s" \ + "ON DUPLICATE KEY UPDATE " \ + "stale_stats = if(last_analyzed > 0, stale_stats, values(stale_stats))" + +#define STALE_TABLE_STAT_MOCK_VALUE_PATTERN "(%lu, %lu, %ld, 0, 0, 0, -1, -1, 0, 0, -1, -1, 0, 0, 0, 0, 0, 1)" + + void ObOptStatMonitorFlushAllTask::runTimerTask() { int ret = OB_SUCCESS; @@ -81,6 +106,7 @@ void ObOptStatMonitorFlushAllTask::runTimerTask() share::schema::ObMultiVersionSchemaService &schema_service = share::schema::ObMultiVersionSchemaService::get_instance(); share::schema::ObSchemaGetterGuard schema_guard; bool in_restore = false; + THIS_WORKER.set_timeout_ts(FLUSH_INTERVAL / 2 + ObTimeUtility::current_time()); if (OB_FAIL(schema_service.get_tenant_schema_guard(optstat_monitor_mgr_->tenant_id_, schema_guard))) { LOG_WARN("failed to get schema guard", K(ret)); } else if (OB_FAIL(schema_guard.check_tenant_is_restore(optstat_monitor_mgr_->tenant_id_, in_restore))) { @@ -103,6 +129,7 @@ void ObOptStatMonitorCheckTask::runTimerTask() share::schema::ObMultiVersionSchemaService &schema_service = share::schema::ObMultiVersionSchemaService::get_instance(); share::schema::ObSchemaGetterGuard schema_guard; bool in_restore = false; + THIS_WORKER.set_timeout_ts(CHECK_INTERVAL + ObTimeUtility::current_time()); if (OB_FAIL(schema_service.get_tenant_schema_guard(optstat_monitor_mgr_->tenant_id_, schema_guard))) { LOG_WARN("failed to get schema guard", K(ret)); } else if (OB_FAIL(schema_guard.check_tenant_is_restore(optstat_monitor_mgr_->tenant_id_, in_restore))) { @@ -349,6 +376,7 @@ int ObOptStatMonitorManager::update_dml_stat_info() } else if (!dml_stats.empty()) { ObSqlString value_sql; int count = 0; + uint64_t tenant_id = dml_stats.at(0).tenant_id_; for (int64_t i = 0; OB_SUCC(ret) && i < dml_stats.count(); ++i) { if (OB_FAIL(get_dml_stat_sql(dml_stats.at(i), 0 != count, value_sql))) { LOG_WARN("failed to get dml stat sql", K(ret)); @@ -367,8 +395,18 @@ int ObOptStatMonitorManager::update_dml_stat_info() } } if (OB_SUCC(ret)) { + bool no_check = (OB_E(EventTable::EN_LEADER_STORAGE_ESTIMATION) OB_SUCCESS) != OB_SUCCESS; + uint64_t data_version = 0; if (OB_FAIL(clean_useless_dml_stat_info())) { LOG_WARN("failed to clean useless dml stat info", K(ret)); + } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, data_version))) { + LOG_WARN("fail to get tenant data version", KR(ret), K(tenant_id), K(data_version)); + } else if (data_version < MOCK_DATA_VERSION_4_2_4_0 || + (data_version >= DATA_VERSION_4_3_0_0 && data_version < DATA_VERSION_4_3_3_0) || + no_check) { + //do nothing + } else if (OB_FAIL(check_opt_stats_expired(dml_stats))) { + LOG_WARN("failed to check opt stats expired", K(ret)); } else {/*do nohting*/} } } @@ -699,12 +737,15 @@ int ObOptStatMonitorManager::update_dml_stat_info(const ObIArray { int ret = OB_SUCCESS; ObSqlString value_sql; - int count = 0; + int64_t count = 0; LOG_TRACE("begin to update dml stat info from direct load", K(dml_stats)); + ObSEArray tmp_dml_stats; for (int64_t i = 0; OB_SUCC(ret) && i < dml_stats.count(); ++i) { if (OB_ISNULL(dml_stats.at(i))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpcted error", K(ret), K(dml_stats.at(i))); + } else if (OB_FAIL(tmp_dml_stats.push_back(*dml_stats.at(i)))) { + LOG_WARN("failed to push back", K(ret)); } else { if (OB_FAIL(get_dml_stat_sql(*dml_stats.at(i), 0 != count, value_sql))) { LOG_WARN("failed to get dml stat sql", K(ret)); @@ -723,6 +764,19 @@ int ObOptStatMonitorManager::update_dml_stat_info(const ObIArray LOG_WARN("failed to exec insert sql", K(ret)); } } + if (OB_SUCC(ret) && !tmp_dml_stats.empty()) { + bool no_check = (OB_E(EventTable::EN_LEADER_STORAGE_ESTIMATION) OB_SUCCESS) != OB_SUCCESS; + uint64_t data_version = 0; + if (OB_FAIL(GET_MIN_DATA_VERSION(tmp_dml_stats.at(0).tenant_id_, data_version))) { + LOG_WARN("fail to get tenant data version", KR(ret), K(tmp_dml_stats.at(0).tenant_id_), K(data_version)); + } else if (data_version < MOCK_DATA_VERSION_4_2_4_0 || + (data_version >= DATA_VERSION_4_3_0_0 && data_version < DATA_VERSION_4_3_3_0) || + no_check) { + //do nothing + } else if (OB_FAIL(check_opt_stats_expired(tmp_dml_stats, true/*is_from_direct_load*/))) { + LOG_WARN("failed to check opt stats expired", K(ret)); + } else {/*do nohting*/} + } return ret; } @@ -816,5 +870,759 @@ int ObOptStatMonitorManager::get_dml_stats(ObIArray &dml_stats) return ret; } +int ObOptStatMonitorManager::check_opt_stats_expired(ObIArray &dml_stats, + bool is_from_direct_load/*default false*/) +{ + int ret = OB_SUCCESS; + if (!dml_stats.empty()) { + ObSEArray stale_infos; + int64_t begin_ts = ObTimeUtility::current_time(); + int64_t global_async_stale_max_table_size = DEFAULT_ASYNC_STALE_MAX_TABLE_SIZE; + if (OB_FAIL(get_async_stale_max_table_size(dml_stats.at(0).tenant_id_, + OB_INVALID_ID, + global_async_stale_max_table_size))) { + LOG_WARN("failed to get async stale max table size", K(ret)); + } else if (OB_UNLIKELY(global_async_stale_max_table_size <= 0)) { + LOG_INFO("skip to check opt stats expired", K(global_async_stale_max_table_size)); + } else if (OB_FAIL(get_opt_stats_expired_table_info(dml_stats, stale_infos, is_from_direct_load))) { + LOG_WARN("failed to get opt stats expired table info", K(ret)); + } else { + const int64_t MIN_ASYNC_GATHER_TABLE_ROW_CNT = 500; + for (int64_t i = 0; OB_SUCC(ret) && i < stale_infos.count(); ++i) { + if (stale_infos.at(i).inserts_ <= MIN_ASYNC_GATHER_TABLE_ROW_CNT) { + //do nothing + } else if (OB_FAIL(mark_the_opt_stat_expired(stale_infos.at(i)))) { + LOG_WARN("failed to mark the opt stat expired", K(ret)); + } + } + } + if (ObTimeUtility::current_time() - begin_ts > ObOptStatMonitorCheckTask::CHECK_INTERVAL) { + LOG_INFO("check opt stats expired cost too much time", K(begin_ts), K(ObTimeUtility::current_time() - begin_ts), K(dml_stats)); + } + } + return ret; } + +int ObOptStatMonitorManager::get_opt_stats_expired_table_info(ObIArray &dml_stats, + ObIArray &stale_infos, + bool is_from_direct_load) +{ + int ret = OB_SUCCESS; + int64_t begin_idx = 0; + while (OB_SUCC(ret) && begin_idx < dml_stats.count()) { + ObSqlString where_list; + int64_t end_idx = std::min(begin_idx + MAX_PROCESS_BATCH_TABLET_CNT, dml_stats.count()); + uint64_t tenant_id = dml_stats.at(begin_idx).tenant_id_; + if (OB_FAIL(gen_tablet_list(dml_stats, begin_idx, end_idx, is_from_direct_load, where_list))) { + LOG_WARN("failed to gen tablet list", K(ret)); + } else if (where_list.empty()) { + //do nothing + } else if (OB_FAIL(do_get_opt_stats_expired_table_info(tenant_id, where_list, stale_infos))) { + LOG_WARN("failed to do get opt stats expired table info", K(ret)); + } + begin_idx = end_idx; + } + return ret; } + +int ObOptStatMonitorManager::gen_tablet_list(const ObIArray &dml_stats, + const int64_t begin_idx, + const int64_t end_idx, + const bool is_from_direct_load, + ObSqlString &tablet_list) +{ + int ret = OB_SUCCESS; + tablet_list.reset(); + int64_t begin_ts = ObTimeUtility::current_time(); + ObSchemaGetterGuard schema_guard; + if (OB_UNLIKELY(begin_idx < 0 || end_idx < 0 || + begin_idx >= end_idx || end_idx > dml_stats.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(begin_idx), K(end_idx), K(dml_stats)); + } else if (OB_ISNULL(GCTX.schema_service_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(GCTX.schema_service_)); + } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(dml_stats.at(begin_idx).tenant_id_, schema_guard))) { + LOG_WARN("get tenant schema guard failed", K(ret)); + } else { + bool is_first = true; + for (int64_t i = begin_idx; OB_SUCC(ret) && i < end_idx; ++i) { + bool is_valid = is_from_direct_load && !is_inner_table(dml_stats.at(i).table_id_); + if (!is_valid && OB_FAIL(ObDbmsStatsUtils::check_is_stat_table(schema_guard, + dml_stats.at(i).tenant_id_, + dml_stats.at(i).table_id_, + is_valid))) { + LOG_WARN("failed to check is stat table", K(ret)); + } else if (is_valid) { + uint64_t ext_tenant_id = share::schema::ObSchemaUtils::get_extract_tenant_id(dml_stats.at(i).tenant_id_, dml_stats.at(i).tenant_id_); + uint64_t pure_table_id = share::schema::ObSchemaUtils::get_extract_schema_id(dml_stats.at(i).tenant_id_, dml_stats.at(i).table_id_); + if (OB_FAIL(tablet_list.append_fmt("%s(%lu, %lu, %ld)", is_first ? "(" : " ,", + ext_tenant_id, pure_table_id, + dml_stats.at(i).tablet_id_))) { + LOG_WARN("failed to append sql", K(ret)); + } else { + is_first = false; + } + } + } + if (OB_SUCC(ret) && !is_first) { + if (OB_FAIL(tablet_list.append(")"))) { + LOG_WARN("failed to append", K(ret)); + } + } + } + return ret; +} + +int ObOptStatMonitorManager::do_get_opt_stats_expired_table_info(const int64_t tenant_id, + const ObSqlString &where_str, + ObIArray &stale_infos) +{ + int ret = OB_SUCCESS; + ObSqlString select_sql; + if (OB_FAIL(select_sql.append_fmt("SELECT m.table_id, m.tablet_id, m.inserts "\ + "FROM %s m " \ + "LEFT JOIN %s up " \ + "ON m.table_id = up.table_id "\ + "AND up.pname = 'ASYNC_GATHER_STALE_RATIO' "\ + "JOIN %s gp "\ + "ON gp.sname = 'ASYNC_GATHER_STALE_RATIO' "\ + "WHERE (CASE WHEN m.last_inserts = 0 THEN 1 + cast(coalesce(up.valchar, gp.spare4) as signed) "\ + "ELSE m.inserts * 1.0 / m.last_inserts END) > cast(coalesce(up.valchar, gp.spare4) as signed) "\ + "AND (m.tenant_id, m.table_id, m.tablet_id) in %s", + share::OB_ALL_MONITOR_MODIFIED_TNAME, + share::OB_ALL_OPTSTAT_USER_PREFS_TNAME, + share::OB_ALL_OPTSTAT_GLOBAL_PREFS_TNAME, + where_str.ptr()))) { + LOG_WARN("failed to append fmt", K(ret)); + } else { + SMART_VAR(ObMySQLProxy::MySQLResult, proxy_result) { + sqlclient::ObMySQLResult *client_result = NULL; + ObSQLClientRetryWeak sql_client_retry_weak(mysql_proxy_); + if (OB_FAIL(sql_client_retry_weak.read(proxy_result, tenant_id, select_sql.ptr()))) { + LOG_WARN("failed to execute sql", K(ret), K(select_sql)); + } else if (OB_ISNULL(client_result = proxy_result.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to execute sql", K(ret)); + } else { + while (OB_SUCC(ret) && OB_SUCC(client_result->next())) { + int64_t idx1 = 0; + int64_t idx2 = 1; + int64_t idx3 = 2; + ObObj obj1; + ObObj obj2; + ObObj obj3; + int64_t table_id = -1; + int64_t tablet_id = 0; + int64_t inserts = 0; + if (OB_FAIL(client_result->get_obj(idx1, obj1)) || + OB_FAIL(client_result->get_obj(idx2, obj2)) || + OB_FAIL(client_result->get_obj(idx3, obj3))) { + LOG_WARN("failed to get object", K(ret)); + } else if (OB_FAIL(obj1.get_int(table_id)) || + OB_FAIL(obj2.get_int(tablet_id)) || + OB_FAIL(obj3.get_int(inserts))) { + LOG_WARN("failed to get int", K(ret), K(obj1), K(obj2), K(inserts)); + } else { + bool is_found = false; + for (int64_t i = 0; !is_found && OB_SUCC(ret) && i < stale_infos.count(); ++i) { + if (table_id == stale_infos.at(i).table_id_) { + is_found = true; + if (OB_FAIL(stale_infos.at(i).tablet_ids_.push_back(tablet_id))) { + LOG_WARN("failed to push back", K(ret)); + } else { + stale_infos.at(i).inserts_ += inserts; + } + } + } + if (OB_SUCC(ret) && !is_found) { + OptStatExpiredTableInfo stale_info; + stale_info.tenant_id_ = tenant_id; + stale_info.table_id_ = table_id; + stale_info.inserts_ = inserts; + if (OB_FAIL(stale_info.tablet_ids_.push_back(tablet_id))) { + LOG_WARN("failed to push back", K(ret)); + } else if (OB_FAIL(stale_infos.push_back(stale_info))) { + LOG_WARN("failed to push back", K(ret)); + } + } + } + } + ret = OB_ITER_END == ret ? OB_SUCCESS : ret; + } + int tmp_ret = OB_SUCCESS; + if (NULL != client_result) { + if (OB_SUCCESS != (tmp_ret = client_result->close())) { + LOG_WARN("close result set failed", K(ret), K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + } + } + LOG_TRACE("do get opt stats expired table info", K(select_sql), K(stale_infos)); + } + return ret; +} + +int ObOptStatMonitorManager::mark_the_opt_stat_expired(const OptStatExpiredTableInfo &expired_table_info) +{ + int ret = OB_SUCCESS; + ObSEArray part_infos; + ObSEArray subpart_infos; + ObSEArray partition_ids; + ObSEArray expired_partition_ids; + share::schema::ObPartitionLevel part_level = share::schema::ObPartitionLevel::PARTITION_LEVEL_MAX; + ObSEArray table_stats; + ObSEArray expired_table_stats; + ObSEArray no_table_stats; + ObArenaAllocator allocator("OptStatMonitor", OB_MALLOC_NORMAL_BLOCK_SIZE, expired_table_info.tenant_id_); + int64_t begin_ts = ObTimeUtility::current_time(); + int64_t async_stale_max_table_size = DEFAULT_ASYNC_STALE_MAX_TABLE_SIZE; + if (OB_FAIL(get_expired_table_part_info(allocator, expired_table_info, part_level, part_infos, subpart_infos))) { + LOG_WARN("failed to get expired table part info", K(ret)); + } else if (part_level == share::schema::ObPartitionLevel::PARTITION_LEVEL_MAX) { + //do nothing + } else if (OB_FAIL(get_need_check_opt_stat_partition_ids(expired_table_info, + part_infos, + subpart_infos, + partition_ids))) { + LOG_WARN("failed to get need check opt stat partition ids", K(ret)); + } else if (OB_FAIL(ObOptStatManager::get_instance().get_table_stat(expired_table_info.tenant_id_, + expired_table_info.table_id_, + partition_ids, + table_stats))) { + LOG_WARN("failed to get table stat", K(ret)); + } else if (OB_FAIL(get_async_stale_max_table_size(expired_table_info.tenant_id_, + expired_table_info.table_id_, + async_stale_max_table_size))) { + LOG_WARN("failed to get async stale max table size", K(ret)); + } else if (OB_UNLIKELY(async_stale_max_table_size <= 0)) { + LOG_INFO("skip to mark the opt stat expired", K(async_stale_max_table_size)); + } else if (OB_FAIL(get_need_mark_opt_stats_expired(table_stats, + expired_table_info, + async_stale_max_table_size, + begin_ts, + part_level, + part_infos, + subpart_infos, + expired_table_stats, + no_table_stats))) { + LOG_WARN("failed to get need mark opt stats expired", K(ret)); + } else if (OB_FAIL(do_mark_the_opt_stat_expired(expired_table_info.tenant_id_, + expired_table_stats, + expired_partition_ids))) { + LOG_WARN("failed to do mark the opt stat expired", K(ret)); + } else if (OB_FAIL(do_mark_the_opt_stat_missing(expired_table_info.tenant_id_, + no_table_stats))) { + LOG_WARN("failed to do mark the opt stat missing", K(ret)); + } else { + obrpc::ObUpdateStatCacheArg stat_arg; + stat_arg.tenant_id_ = expired_table_info.tenant_id_; + stat_arg.table_id_ = expired_table_info.table_id_; + if (OB_FAIL(append(stat_arg.partition_ids_, expired_partition_ids))) { + LOG_WARN("failed to append", K(ret)); + } else if (OB_FAIL(pl::ObDbmsStats::update_stat_cache(expired_table_info.tenant_id_, stat_arg))) { + LOG_WARN("failed to update stat cache", K(ret)); + } + } + return ret; +} + +int ObOptStatMonitorManager::get_expired_table_part_info(ObIAllocator &allocator, + const OptStatExpiredTableInfo &expired_table_info, + share::schema::ObPartitionLevel &part_level, + ObIArray &part_infos, + ObIArray &subpart_infos) +{ + int ret = OB_SUCCESS; + ObSchemaGetterGuard schema_guard; + const ObTableSchema *table_schema = NULL; + part_level = share::schema::ObPartitionLevel::PARTITION_LEVEL_MAX; + part_infos.reset(); + subpart_infos.reset(); + if (OB_ISNULL(GCTX.schema_service_) || OB_UNLIKELY(!expired_table_info.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(expired_table_info), K(GCTX.schema_service_)); + } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(expired_table_info.tenant_id_, schema_guard))) { + LOG_WARN("get tenant schema guard failed", K(ret)); + } else if (OB_FAIL(schema_guard.get_table_schema(expired_table_info.tenant_id_, + expired_table_info.table_id_, + table_schema))) { + LOG_WARN("get table schema failed", K(ret), K(expired_table_info.tenant_id_), K(expired_table_info.table_id_)); + } else if (OB_ISNULL(table_schema)) {//maybe table isn't exists. + //do nothing + } else if (OB_FAIL(pl::ObDbmsStats::get_table_part_infos(table_schema, allocator, part_infos, subpart_infos))) { + LOG_WARN("failed to get table part infos", K(ret)); + } else { + part_level = table_schema->get_part_level(); + } + return ret; +} + +int ObOptStatMonitorManager::get_need_check_opt_stat_partition_ids(const OptStatExpiredTableInfo &expired_table_info, + ObIArray &part_infos, + ObIArray &subpart_infos, + ObIArray &partition_ids) +{ + int ret = OB_SUCCESS; + //non partition table + if (part_infos.empty() && subpart_infos.empty()) { + if (OB_FAIL(partition_ids.push_back(expired_table_info.table_id_))) { + LOG_WARN("failed to push back", K(ret)); + } + //partition table, global stat partition id is -1 + } else if (OB_FAIL(partition_ids.push_back(-1))) { + LOG_WARN("failed to push back", K(ret)); + } else if (!part_infos.empty() && subpart_infos.empty()) {//part table + if (expired_table_info.tablet_ids_.count() == part_infos.count()) { + for (int64_t i = 0; OB_SUCC(ret) && i < part_infos.count(); ++i) { + if (OB_FAIL(partition_ids.push_back(part_infos.at(i).part_id_))) { + LOG_WARN("failed to push back", K(ret)); + } + } + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < expired_table_info.tablet_ids_.count(); ++i) { + bool found_it = false; + for (int64_t j = 0; OB_SUCC(ret) && !found_it && j < part_infos.count(); ++j) { + if (expired_table_info.tablet_ids_.at(i) == static_cast(part_infos.at(j).tablet_id_.id())) { + if (OB_FAIL(partition_ids.push_back(part_infos.at(j).part_id_))) { + LOG_WARN("failed to push back", K(ret)); + } else { + found_it = true; + } + } + } + } + } + } else if (!part_infos.empty() && !subpart_infos.empty()) {//subpart table + hash::ObHashMap partition_ids_map; + if (expired_table_info.tablet_ids_.count() == subpart_infos.count()) { + for (int64_t i = 0; OB_SUCC(ret) && i < part_infos.count(); ++i) { + if (OB_FAIL(partition_ids.push_back(part_infos.at(i).part_id_))) { + LOG_WARN("failed to push back", K(ret)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < subpart_infos.count(); ++i) { + if (OB_FAIL(partition_ids.push_back(subpart_infos.at(i).part_id_))) { + LOG_WARN("failed to push back", K(ret)); + } + } + } else if (OB_FAIL(partition_ids_map.create(part_infos.count(), "PartIdsMap", "PartIdsMapNode", expired_table_info.tenant_id_))) { + LOG_WARN("fail to create hash map", K(ret), K(expired_table_info.tablet_ids_.count())); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < expired_table_info.tablet_ids_.count(); ++i) { + bool found_it = false; + for (int64_t j = 0; OB_SUCC(ret) && !found_it && j < subpart_infos.count(); ++j) { + if (expired_table_info.tablet_ids_.at(i) == static_cast(subpart_infos.at(j).tablet_id_.id())) { + bool tmp_var = false; + if (OB_FAIL(partition_ids.push_back(subpart_infos.at(j).part_id_))) { + LOG_WARN("failed to push back", K(ret)); + } else { + found_it = true; + if (OB_FAIL(partition_ids_map.get_refactored(subpart_infos.at(j).first_part_id_, tmp_var))) { + if (OB_HASH_NOT_EXIST == ret) { + ret = OB_SUCCESS; + if (OB_FAIL(partition_ids.push_back(subpart_infos.at(j).first_part_id_))) { + LOG_WARN("failed to push back", K(ret)); + } else if (OB_FAIL(partition_ids_map.set_refactored(subpart_infos.at(j).first_part_id_, true))) { + LOG_WARN("failed to set refactored", K(ret)); + } else {/*do nothing*/} + } else { + LOG_WARN("failed to get refactored", K(ret)); + } + } + } + } + } + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret)); + } + LOG_INFO("get need check opt stat partition ids", K(expired_table_info), K(part_infos), + K(subpart_infos), K(partition_ids)); + return ret; +} + +int ObOptStatMonitorManager::get_need_mark_opt_stats_expired(const ObIArray &table_stats, + const OptStatExpiredTableInfo &expired_table_info, + const int64_t async_stale_max_table_size, + const int64_t begin_ts, + const share::schema::ObPartitionLevel &part_level, + const ObIArray &part_infos, + const ObIArray &subpart_infos, + ObIArray &expired_table_stats, + ObIArray &no_table_stats) +{ + int ret = OB_SUCCESS; + bool have_table_stats = false; + for (int64_t i = 0; OB_SUCC(ret) && i < table_stats.count(); ++i) { + bool is_stat_expired = false; + have_table_stats |= table_stats.at(i).get_last_analyzed() > 0; + if (table_stats.at(i).get_last_analyzed() <= 0) { + if (!table_stats.at(i).is_stat_expired()) { + if (OB_FAIL(no_table_stats.push_back(table_stats.at(i)))) { + LOG_WARN("failed to push back", K(ret)); + } + } + } else if (table_stats.at(i).is_stat_expired() || + table_stats.at(i).get_last_analyzed() >= begin_ts || + table_stats.at(i).get_row_count() > async_stale_max_table_size) { + //do nothing + } else if (part_level == share::schema::ObPartitionLevel::PARTITION_LEVEL_ZERO) { + is_stat_expired = true; + } else if (part_level == share::schema::ObPartitionLevel::PARTITION_LEVEL_ONE) { + if (table_stats.count() == part_infos.count() + 1 || + table_stats.at(i).get_object_type() == StatLevel::PARTITION_LEVEL) { + is_stat_expired = true; + } else if (table_stats.at(i).get_object_type() == StatLevel::TABLE_LEVEL) { + ObSEArray tablet_ids; + if (OB_FAIL(check_table_stat_expired_by_dml_info(expired_table_info.tenant_id_, + expired_table_info.table_id_, + tablet_ids, + is_stat_expired))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpcted error", K(ret)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(table_stats.at(i))); + } + } else if (part_level == share::schema::ObPartitionLevel::PARTITION_LEVEL_TWO) { + if (table_stats.count() == part_infos.count() + subpart_infos.count() + 1 || + table_stats.at(i).get_object_type() == StatLevel::SUBPARTITION_LEVEL) { + is_stat_expired = true; + } else if (table_stats.at(i).get_object_type() == StatLevel::PARTITION_LEVEL) { + ObSEArray tablet_ids; + bool is_all_subpart_expired = true; + for (int64_t j = 0; OB_SUCC(ret) && j < subpart_infos.count(); ++j) { + if (table_stats.at(i).get_partition_id() == subpart_infos.at(j).first_part_id_) { + if (OB_FAIL(tablet_ids.push_back(subpart_infos.at(j).tablet_id_.id()))) { + LOG_WARN("failed to push back", K(ret)); + } else if (is_all_subpart_expired) { + bool found_it = false; + for (int64_t k = 0; !found_it && k < table_stats.count(); ++k) { + if (table_stats.at(i).get_partition_id() == subpart_infos.at(j).part_id_) { + found_it = true; + is_all_subpart_expired &= table_stats.at(i).get_last_analyzed() > 0; + } + } + is_all_subpart_expired &= found_it; + } + } + } + if (OB_SUCC(ret)) { + if (is_all_subpart_expired) { + is_stat_expired = true; + } else if (OB_FAIL(check_table_stat_expired_by_dml_info(expired_table_info.tenant_id_, + expired_table_info.table_id_, + tablet_ids, + is_stat_expired))) { + LOG_WARN("failed to check table stat expired by dml info", K(ret)); + } + } + } else if (table_stats.at(i).get_object_type() == StatLevel::TABLE_LEVEL) { + ObSEArray tablet_ids; + if (OB_FAIL(check_table_stat_expired_by_dml_info(expired_table_info.tenant_id_, + expired_table_info.table_id_, + tablet_ids, + is_stat_expired))) { + LOG_WARN("failed to check table stat expired by dml info", K(ret)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(table_stats.at(i))); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(table_stats.at(i))); + } + if (OB_SUCC(ret) && is_stat_expired) { + if (OB_FAIL(expired_table_stats.push_back(table_stats.at(i)))) { + LOG_WARN("failed to push back", K(ret)); + } + } + if (OB_SUCC(ret) && have_table_stats) { + no_table_stats.reset(); + } + } + LOG_INFO("get need mark opt stats expired", K(expired_table_stats), K(no_table_stats)); + return ret; +} + +int ObOptStatMonitorManager::check_table_stat_expired_by_dml_info(const uint64_t tenant_id, + const uint64_t table_id, + const ObIArray &tablet_ids, + bool &is_stat_expired) +{ + int ret = OB_SUCCESS; + ObSqlString tablet_list; + is_stat_expired = false; + if (OB_FAIL(gen_tablet_list(tablet_ids, tablet_list))) { + LOG_WARN("failed to gen tablet list", K(ret)); + } else { + uint64_t ext_tenant_id = share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id); + uint64_t pure_table_id = share::schema::ObSchemaUtils::get_extract_schema_id(tenant_id, table_id); + ObSqlString select_sql; + if (OB_FAIL(select_sql.append_fmt("SELECT 1 "\ + "FROM (SELECT table_id,"\ + "sum(inserts-deletes) AS row_cnt,"\ + "sum(inserts+updates+deletes) AS total_modified_cnt,"\ + "sum(last_inserts+last_updates+last_deletes) AS last_modified_cnt "\ + "from %s "\ + "WHERE tenant_id = %lu "\ + "AND table_id = %lu %s%s "\ + "GROUP BY table_id) m "\ + "LEFT JOIN %s up "\ + "ON m.table_id = up.table_id "\ + "AND up.pname = 'ASYNC_GATHER_STALE_RATIO' "\ + "JOIN %s gp "\ + "ON gp.sname = 'ASYNC_GATHER_STALE_RATIO' "\ + "WHERE (CASE WHEN last_modified_cnt = 0 THEN 1 + cast(coalesce(up.valchar, gp.spare4) as signed) "\ + "ELSE total_modified_cnt * 1.0 / last_modified_cnt END) > cast(COALESCE(up.valchar, gp.spare4) AS signed) "\ + "AND row_cnt > 0;", + share::OB_ALL_MONITOR_MODIFIED_TNAME, + ext_tenant_id, + pure_table_id, + tablet_list.empty() ? " " : " AND tablet_id in ", + tablet_list.empty() ? " " : tablet_list.ptr(), + share::OB_ALL_OPTSTAT_USER_PREFS_TNAME, + share::OB_ALL_OPTSTAT_GLOBAL_PREFS_TNAME + ))) { + LOG_WARN("failed to append fmt", K(ret)); + } else { + LOG_TRACE("check table stat expired by dml info", K(select_sql)); + SMART_VAR(ObMySQLProxy::MySQLResult, proxy_result) { + sqlclient::ObMySQLResult *client_result = NULL; + ObSQLClientRetryWeak sql_client_retry_weak(mysql_proxy_); + if (OB_FAIL(sql_client_retry_weak.read(proxy_result, tenant_id, select_sql.ptr()))) { + LOG_WARN("failed to execute sql", K(ret), K(select_sql)); + } else if (OB_ISNULL(client_result = proxy_result.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to execute sql", K(ret)); + } else { + while (OB_SUCC(ret) && !is_stat_expired && OB_SUCC(client_result->next())) { + is_stat_expired = true; + } + ret = OB_ITER_END == ret ? OB_SUCCESS : ret; + } + int tmp_ret = OB_SUCCESS; + if (NULL != client_result) { + if (OB_SUCCESS != (tmp_ret = client_result->close())) { + LOG_WARN("close result set failed", K(ret), K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + } + } + } + LOG_TRACE("check_table_stat_expired_by_dml_info end", K(is_stat_expired)); + } + return ret; +} + +int ObOptStatMonitorManager::gen_tablet_list(const ObIArray &tablet_ids, + ObSqlString &tablet_list) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); ++i) { + char prefix = (i == 0 ? '(' : ' '); + char suffix = (i == tablet_ids.count() - 1 ? ')' : ','); + if (OB_FAIL(tablet_list.append_fmt("%c%lu%c", prefix, tablet_ids.at(i), suffix))) { + LOG_WARN("failed to append sql", K(ret)); + } else {/*do nothing*/} + } + return ret; +} + +int ObOptStatMonitorManager::do_mark_the_opt_stat_missing(const uint64_t tenant_id, + const ObIArray &no_table_stats) +{ + int ret = OB_SUCCESS; + if (!no_table_stats.empty()) { + int64_t begin_idx = 0; + ObMySQLTransaction trans; + if (OB_ISNULL(mysql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(mysql_proxy_)); + } else if (OB_FAIL(trans.start(mysql_proxy_, tenant_id))) { + LOG_WARN("fail to start transaction", K(ret)); + } else { + while (OB_SUCC(ret) && begin_idx < no_table_stats.count()) { + ObSqlString insert_sql; + ObSqlString values_list; + int64_t affected_rows = 0; + int64_t end_idx = std::min(begin_idx + MAX_PROCESS_BATCH_TABLET_CNT, no_table_stats.count()); + if (OB_FAIL(gen_values_list(tenant_id, no_table_stats, begin_idx, end_idx, values_list))) { + LOG_WARN("failed to gen values list", K(ret)); + } else if (OB_UNLIKELY(values_list.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(values_list)); + } else if (OB_FAIL(insert_sql.append_fmt(INSERT_STALE_TABLE_STAT_SQL, + share::OB_ALL_TABLE_STAT_TNAME, + values_list.ptr()))) { + } else if (OB_FAIL(trans.write(tenant_id, insert_sql.ptr(), affected_rows))) { + LOG_WARN("fail to exec sql", K(insert_sql), K(ret)); + } else { + begin_idx = end_idx; + LOG_INFO("Succeed to do mark the opt stat expired", K(insert_sql), K(no_table_stats), K(affected_rows)); + } + } + //end gather trans + if (OB_SUCC(ret)) { + if (OB_FAIL(trans.end(true))) { + LOG_WARN("fail to commit transaction", K(ret)); + } + } else { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(false))) { + LOG_WARN("fail to roll back transaction", K(tmp_ret)); + } + } + } + } + return ret; +} + +int ObOptStatMonitorManager::do_mark_the_opt_stat_expired(const uint64_t tenant_id, + const ObIArray &expired_table_stats, + ObIArray &expired_partition_ids) +{ + int ret = OB_SUCCESS; + int64_t begin_idx = 0; + if (OB_ISNULL(mysql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(mysql_proxy_)); + } + while (OB_SUCC(ret) && begin_idx < expired_table_stats.count()) { + ObSqlString update_sql; + ObSqlString same_part_analyzed_list; + ObSqlString diff_part_analyzed_list; + int64_t affected_rows = 0; + int64_t end_idx = std::min(begin_idx + MAX_PROCESS_BATCH_TABLET_CNT, expired_table_stats.count()); + uint64_t ext_tenant_id = share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id); + uint64_t pure_table_id = share::schema::ObSchemaUtils::get_extract_schema_id(tenant_id, expired_table_stats.at(begin_idx).get_table_id()); + if (OB_FAIL(gen_part_analyzed_list(expired_table_stats, begin_idx, end_idx, + same_part_analyzed_list, + diff_part_analyzed_list, + expired_partition_ids))) { + LOG_WARN("failed to gen part analyzed list", K(ret)); + } else if (OB_FAIL(update_sql.append_fmt("update /*+QUERY_TIMEOUT(60000000)*/%s set stale_stats = 1 where tenant_id = %lu and table_id = %lu and %s", + share::OB_ALL_TABLE_STAT_TNAME, + ext_tenant_id, + pure_table_id, + !same_part_analyzed_list.empty() ? same_part_analyzed_list.ptr() : diff_part_analyzed_list.ptr()))) { + } else if (OB_FAIL(mysql_proxy_->write(tenant_id, update_sql.ptr(), affected_rows))) { + LOG_WARN("fail to exec sql", K(update_sql), K(ret)); + } else { + begin_idx = end_idx; + LOG_INFO("Succeed to do mark the opt stat expired", K(update_sql), K(expired_table_stats), K(affected_rows)); + } + } + return ret; +} + +int ObOptStatMonitorManager::gen_part_analyzed_list(const ObIArray &expired_table_stats, + const int64_t begin_idx, + const int64_t end_idx, + ObSqlString &same_part_analyzed_list, + ObSqlString &diff_part_analyzed_list, + ObIArray &expired_partition_ids) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(begin_idx < 0 || end_idx < 0 || + begin_idx >= end_idx || end_idx > expired_table_stats.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(begin_idx), K(end_idx), K(expired_table_stats)); + } else { + int64_t last_analyzed = -1; + for (int64_t i = begin_idx; OB_SUCC(ret) && i < end_idx; ++i) { + char suffix = (i == end_idx - 1 ? ')' : ','); + if (OB_FAIL(expired_partition_ids.push_back(expired_table_stats.at(i).get_partition_id()))) { + LOG_WARN("failed to push back", K(ret)); + } else if (OB_FAIL(diff_part_analyzed_list.append_fmt("%s(%ld,usec_to_time(%ld))%c", i == begin_idx ? "(partition_id, last_analyzed) in (" : " ", + expired_table_stats.at(i).get_partition_id(), + expired_table_stats.at(i).get_last_analyzed(), + suffix))) { + LOG_WARN("failed to append sql", K(ret)); + } else if (i == begin_idx || last_analyzed == expired_table_stats.at(i).get_last_analyzed()) { + last_analyzed = expired_table_stats.at(i).get_last_analyzed(); + if (OB_FAIL(same_part_analyzed_list.append_fmt("%s%ld%c", i == begin_idx ? "partition_id in (" : " ", + expired_table_stats.at(i).get_partition_id(), + suffix))) { + LOG_WARN("failed to append sql", K(ret)); + } else if (i == end_idx - 1) { + if (OB_FAIL(same_part_analyzed_list.append_fmt(" AND last_analyzed = usec_to_time(%ld)", last_analyzed))) { + LOG_WARN("failed to append sql", K(ret)); + } else { + diff_part_analyzed_list.reset(); + } + } + } else { + last_analyzed = -1; + same_part_analyzed_list.reset(); + } + } + } + return ret; +} + +int ObOptStatMonitorManager::gen_values_list(const uint64_t tenant_id, + const ObIArray &no_table_stats, + const int64_t begin_idx, + const int64_t end_idx, + ObSqlString &values_list) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(begin_idx < 0 || end_idx < 0 || + begin_idx >= end_idx || end_idx > no_table_stats.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(begin_idx), K(end_idx), K(no_table_stats)); + } else { + for (int64_t i = begin_idx; OB_SUCC(ret) && i < end_idx; ++i) { + ObSqlString value; + uint64_t ext_tenant_id = share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id); + uint64_t pure_table_id = share::schema::ObSchemaUtils::get_extract_schema_id(tenant_id, no_table_stats.at(i).get_table_id()); + if (OB_FAIL(value.append_fmt(STALE_TABLE_STAT_MOCK_VALUE_PATTERN, + ext_tenant_id, + pure_table_id, + no_table_stats.at(i).get_partition_id()))) { + LOG_WARN("failed to append fmt", K(ret)); + } else if (OB_FAIL(values_list.append_fmt("%s%s", i == begin_idx ? " " : ", ", value.ptr()))) { + LOG_WARN("failed to push back", K(ret)); + } + } + } + return ret; +} + +int ObOptStatMonitorManager::get_async_stale_max_table_size(const uint64_t tenant_id, + const uint64_t table_id, + int64_t &async_stale_max_table_size) +{ + int ret = OB_SUCCESS; + ObArenaAllocator tmp_alloc("OptStatPrefs", OB_MALLOC_NORMAL_BLOCK_SIZE, tenant_id); + ObAsyncStaleMaxTableSizePrefs prefs; + ObString opt_name(prefs.get_stat_pref_name()); + ObObj result; + ObObj dest_obj; + ObCastCtx cast_ctx(&tmp_alloc, NULL, CM_NONE, ObCharset::get_system_collation()); + async_stale_max_table_size = DEFAULT_ASYNC_MAX_SCAN_ROWCOUNT; + if (OB_FAIL(ObDbmsStatsPreferences::get_prefs(mysql_proxy_, tmp_alloc, tenant_id, + table_id, opt_name, result))) { + LOG_WARN("failed to get prefs", K(ret)); + } else if (result.is_null()) { + //do nothing + } else if (OB_FAIL(ObObjCaster::to_type(ObNumberType, cast_ctx, result, dest_obj))) { + LOG_WARN("failed to type", K(ret), K(result)); + } else if (OB_FAIL(dest_obj.get_number().extract_valid_int64_with_trunc(async_stale_max_table_size))) { + LOG_WARN("failed to extract valid int64 with trunc", K(ret), K(result)); + } else if (async_stale_max_table_size < 0) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("Illegal async stale max table size", K(ret), K(async_stale_max_table_size)); + } + LOG_TRACE("get_async_stale_max_table_size", K(async_stale_max_table_size), K(result)); + return ret; +} + +} +} \ No newline at end of file diff --git a/src/share/stat/ob_opt_stat_monitor_manager.h b/src/share/stat/ob_opt_stat_monitor_manager.h index df1e48ac0..7a21c1ef7 100644 --- a/src/share/stat/ob_opt_stat_monitor_manager.h +++ b/src/share/stat/ob_opt_stat_monitor_manager.h @@ -65,6 +65,17 @@ public: ObOptStatMonitorManager *optstat_monitor_mgr_; }; +struct OptStatExpiredTableInfo +{ + OptStatExpiredTableInfo() : tenant_id_(0), table_id_(0), tablet_ids_(), inserts_(0) {} + bool is_valid() const { return tenant_id_ > 0 && table_id_ > 0 && !tablet_ids_.empty(); } + uint64_t tenant_id_; + uint64_t table_id_; + ObSEArray tablet_ids_; + uint64_t inserts_; + TO_STRING_KV(K(tenant_id_), K(table_id_), K(tablet_ids_), K(inserts_)); +}; + class ObOptStatMonitorManager { friend class ObOptStatMonitorFlushAllTask; @@ -142,11 +153,67 @@ public: ObOptStatMonitorFlushAllTask &get_flush_all_task() { return flush_all_task_; } ObOptStatMonitorCheckTask &get_check_task() { return check_task_; } int init(uint64_t tenant_id); + int check_opt_stats_expired(ObIArray &dml_stats, bool is_from_direct_load = false); + int get_opt_stats_expired_table_info(ObIArray &dml_stats, + ObIArray &stale_infos, + bool is_from_direct_load); + int gen_tablet_list(const ObIArray &dml_stats, + const int64_t begin_idx, + const int64_t end_idx, + const bool is_from_direct_load, + ObSqlString &tablet_list); + int do_get_opt_stats_expired_table_info(const int64_t tenant_id, + const ObSqlString &where_str, + ObIArray &stale_infos); + int mark_the_opt_stat_expired(const OptStatExpiredTableInfo &expired_table_info); + int get_expired_table_part_info(ObIAllocator &allocator, + const OptStatExpiredTableInfo &expired_table_info, + share::schema::ObPartitionLevel &part_level, + ObIArray &part_infos, + ObIArray &subpart_infos); + int get_need_check_opt_stat_partition_ids(const OptStatExpiredTableInfo &expired_table_info, + ObIArray &part_infos, + ObIArray &subpart_infos, + ObIArray &partition_ids); + int check_table_stat_expired_by_dml_info(const uint64_t tenant_id, + const uint64_t table_id, + const ObIArray &tablet_ids, + bool &is_stat_expired); + int get_need_mark_opt_stats_expired(const ObIArray &table_stats, + const OptStatExpiredTableInfo &expired_table_info, + const int64_t async_stale_max_table_size, + const int64_t begin_ts, + const share::schema::ObPartitionLevel &part_level, + const ObIArray &part_infos, + const ObIArray &subpart_infos, + ObIArray &expired_table_stats, + ObIArray &no_table_stats); + int gen_tablet_list(const ObIArray &tablet_ids, ObSqlString &tablet_list); + int do_mark_the_opt_stat_expired(const uint64_t tenant_id, + const ObIArray &expired_table_stats, + ObIArray &expired_partition_ids); + int do_mark_the_opt_stat_missing(const uint64_t tenant_id, + const ObIArray &no_table_stats); + int gen_part_analyzed_list(const ObIArray &expired_table_stats, + const int64_t begin_idx, + const int64_t end_idx, + ObSqlString &same_part_analyzed_list, + ObSqlString &diff_part_analyzed_list, + ObIArray &expired_partition_ids); + int gen_values_list(const uint64_t tenant_id, + const ObIArray &no_table_stats, + const int64_t begin_idx, + const int64_t end_idx, + ObSqlString &values_list); + int get_async_stale_max_table_size(const uint64_t tenant_id, + const uint64_t table_id, + int64_t &async_stale_max_table_size); private: DISALLOW_COPY_AND_ASSIGN(ObOptStatMonitorManager); const static int64_t UPDATE_OPT_STAT_BATCH_CNT = 200; const static int64_t info_count = 8; + const static int64_t MAX_PROCESS_BATCH_TABLET_CNT = 1000; bool inited_; uint64_t tenant_id_; int tg_id_; diff --git a/src/share/stat/ob_opt_stat_sql_service.cpp b/src/share/stat/ob_opt_stat_sql_service.cpp index cb649d754..5ae9e3f8f 100644 --- a/src/share/stat/ob_opt_stat_sql_service.cpp +++ b/src/share/stat/ob_opt_stat_sql_service.cpp @@ -350,6 +350,7 @@ int ObOptStatSqlService::fetch_table_stat(const uint64_t tenant_id, "macro_blk_cnt as macro_block_num, " "micro_blk_cnt as micro_block_num, " "stattype_locked as stattype_locked," + "stale_stats as stale_stats," "last_analyzed," "spare1 as sample_size FROM %s ", share::OB_ALL_TABLE_STAT_TNAME))) { LOG_WARN("fail to append SQL stmt string.", K(sql), K(ret)); @@ -411,6 +412,7 @@ int ObOptStatSqlService::batch_fetch_table_stats(sqlclient::ObISQLConnection *co "macro_blk_cnt as macro_block_num, " "micro_blk_cnt as micro_block_num, " "stattype_locked as stattype_locked," + "stale_stats as stale_stats," "last_analyzed," "spare1 as sample_size FROM %s", share::OB_ALL_TABLE_STAT_TNAME))) { LOG_WARN("fail to append SQL stmt string.", K(sql), K(ret)); @@ -1183,6 +1185,7 @@ int ObOptStatSqlService::fill_table_stat(common::sqlclient::ObMySQLResult &resul EXTRACT_INT_FIELD_TO_CLASS_MYSQL(result, macro_block_num, stat, int64_t); EXTRACT_INT_FIELD_TO_CLASS_MYSQL(result, micro_block_num, stat, int64_t); EXTRACT_INT_FIELD_TO_CLASS_MYSQL(result, stattype_locked, stat, int64_t); + EXTRACT_INT_FIELD_TO_CLASS_MYSQL(result, stale_stats, stat, int64_t); if (OB_SUCCESS != (ret = result.get_timestamp("last_analyzed", NULL, int_value))) { LOG_WARN("fail to get column in row. ", "column_name", "last_analyzed", K(ret)); } else { diff --git a/src/share/stat/ob_opt_table_stat.h b/src/share/stat/ob_opt_table_stat.h index 2210e21b2..a3e3fec7c 100644 --- a/src/share/stat/ob_opt_table_stat.h +++ b/src/share/stat/ob_opt_table_stat.h @@ -137,7 +137,8 @@ public: modified_count_(0), sample_size_(0), tablet_id_(ObTabletID::INVALID_TABLET_ID), - stat_expired_time_(-1) {} + stat_expired_time_(-1), + stale_stats_(0) {} ObOptTableStat(uint64_t table_id, int64_t partition_id, int64_t object_type, @@ -169,7 +170,8 @@ public: modified_count_(0), sample_size_(0), tablet_id_(ObTabletID::INVALID_TABLET_ID), - stat_expired_time_(-1) {} + stat_expired_time_(-1), + stale_stats_(false) {} virtual ~ObOptTableStat() {} @@ -237,6 +239,9 @@ public: int64_t get_stat_expired_time() const { return stat_expired_time_; } void set_stat_expired_time(int64_t expired_time) { stat_expired_time_ = expired_time; } + bool is_stat_expired() const { return stale_stats_; } + void set_stale_stats(int64_t stale_stats) { stale_stats_ = stale_stats > 0; } + bool is_locked() const { return stattype_locked_ > 0; } void add_row_count(int64_t rc) { row_count_ += rc; } @@ -318,6 +323,7 @@ public: sample_size_ = 0; tablet_id_ = ObTabletID::INVALID_TABLET_ID; stat_expired_time_ = -1; + stale_stats_ = false; } TO_STRING_KV(K(table_id_), @@ -338,7 +344,8 @@ public: K(modified_count_), K(sample_size_), K(tablet_id_), - K(stat_expired_time_)); + K(stat_expired_time_), + K(stale_stats_)); private: uint64_t table_id_; @@ -362,6 +369,7 @@ private: int64_t sample_size_; uint64_t tablet_id_;//now only use estimate table rowcnt by meta table. int64_t stat_expired_time_;//mark the stat in cache is arrived expired time, if arrived at expired time need reload, -1 meanings no expire forever. + bool stale_stats_;//mark the stat is expired or not. }; } diff --git a/src/share/stat/ob_stat_define.cpp b/src/share/stat/ob_stat_define.cpp index 02b77bc50..342fb7972 100644 --- a/src/share/stat/ob_stat_define.cpp +++ b/src/share/stat/ob_stat_define.cpp @@ -181,6 +181,14 @@ int ObTableStatParam::assign(const ObTableStatParam &other) is_temp_table_ = other.is_temp_table_; allocator_ = other.allocator_; ref_table_type_ = other.ref_table_type_; + is_async_gather_ = other.is_async_gather_; + async_gather_sample_size_ = other.async_gather_sample_size_; + async_full_table_size_ = other.async_full_table_size_; + async_partition_ids_ = other.async_partition_ids_; + hist_sample_info_.is_sample_ = other.hist_sample_info_.is_sample_; + hist_sample_info_.is_block_sample_ = other.hist_sample_info_.is_block_sample_; + hist_sample_info_.sample_type_ = other.hist_sample_info_.sample_type_; + hist_sample_info_.sample_value_ = other.hist_sample_info_.sample_value_; if (OB_FAIL(part_infos_.assign(other.part_infos_))) { LOG_WARN("failed to assign", K(ret)); } else if (OB_FAIL(subpart_infos_.assign(other.subpart_infos_))) { @@ -226,6 +234,10 @@ int ObTableStatParam::assign_common_property(const ObTableStatParam &other) duration_time_ = other.duration_time_; allocator_ = other.allocator_; online_sample_percent_ = other.online_sample_percent_; + hist_sample_info_.is_sample_ = other.hist_sample_info_.is_sample_; + hist_sample_info_.is_block_sample_ = other.hist_sample_info_.is_block_sample_; + hist_sample_info_.sample_type_ = other.hist_sample_info_.sample_type_; + hist_sample_info_.sample_value_ = other.hist_sample_info_.sample_value_; return ret; } @@ -254,6 +266,10 @@ int ObOptStatGatherParam::assign(const ObOptStatGatherParam &other) global_part_id_ = other.global_part_id_; gather_vectorize_ = other.gather_vectorize_; sepcify_scn_ = other.sepcify_scn_; + hist_sample_info_.is_sample_ = other.hist_sample_info_.is_sample_; + hist_sample_info_.is_block_sample_ = other.hist_sample_info_.is_block_sample_; + hist_sample_info_.sample_type_ = other.hist_sample_info_.sample_type_; + hist_sample_info_.sample_value_ = other.hist_sample_info_.sample_value_; if (OB_FAIL(partition_infos_.assign(other.partition_infos_))) { LOG_WARN("failed to assign", K(ret)); } else if (OB_FAIL(column_params_.assign(other.column_params_))) { @@ -262,7 +278,7 @@ int ObOptStatGatherParam::assign(const ObOptStatGatherParam &other) return ret; } -bool ObTableStatParam::is_specify_partition_gather() const +bool ObTableStatParam::is_specify_partition() const { bool is_specify = false; if (part_level_ == share::schema::PARTITION_LEVEL_ZERO) { @@ -276,7 +292,7 @@ bool ObTableStatParam::is_specify_partition_gather() const return is_specify; } -bool ObTableStatParam::is_specify_column_gather() const +bool ObTableStatParam::is_specify_column() const { bool is_specify = false; for (int64_t i = 0; !is_specify && i < column_params_.count(); ++i) { @@ -307,6 +323,13 @@ int64_t ObOptStatGatherParam::get_need_gather_column() const return valid_column; } +int AsyncStatTable::assign(const AsyncStatTable &other) +{ + int ret = OB_SUCCESS; + table_id_ = other.table_id_; + return partition_ids_.assign(other.partition_ids_); +} + OB_SERIALIZE_MEMBER(ObOptDmlStat, tenant_id_, table_id_, diff --git a/src/share/stat/ob_stat_define.h b/src/share/stat/ob_stat_define.h index 23ef64d04..dab90f143 100644 --- a/src/share/stat/ob_stat_define.h +++ b/src/share/stat/ob_stat_define.h @@ -51,7 +51,12 @@ enum StatOptionFlags OPT_FORCE = 1 << 11, OPT_APPROXIMATE_NDV = 1 << 12, OPT_ESTIMATE_BLOCK = 1 << 13, - OPT_STAT_OPTION_ALL = (1 << 14) -1 + OPT_ASYNC_GATHER_STALE_RATION = 1 << 14, + OPT_ASYNC_GATHER_SAMPLE_SIZE = 1 << 15, + OPT_ASYNC_GATHER_FULL_TABLE_SIZE = 1 << 16, + OPT_HIST_EST_PERCENT = 1 << 17, + OPT_HIST_BLOCK_SAMPLE = 1 << 18, + OPT_STAT_OPTION_ALL = (1 << 19) -1 }; const static double OPT_DEFAULT_STALE_PERCENT = 0.1; const static int64_t OPT_DEFAULT_STATS_RETENTION = 31; @@ -66,6 +71,15 @@ const static int64_t MAX_NUM_OF_WRITE_STATS = 2000; const static int64_t DEFAULT_STAT_GATHER_VECTOR_BATCH_SIZE = 256; const static int64_t MIN_GATHER_WORK_ARANA_SIZE = 10 * 1024L * 1024L; //10M const int64_t MAX_OPT_STATS_PROCESS_RPC_TIMEOUT = 300000000;//one optimizer stats processing rpc time should not exceed 300 seconds +const static int64_t DEFAULT_ASYNC_SAMPLE_SIZE = 1000000; +const static int64_t DEFAULT_ASYNC_FULL_TABLE_SIZE = 10000000; +const static int64_t DEFAULT_ASYNC_MIN_TABLE_SIZE = 10000; +const static int64_t DEFAULT_ASYNC_STALE_MAX_TABLE_SIZE = 100000000; +const static int64_t DEFAULT_ASYNC_MAX_SCAN_ROWCOUNT = 100000000; +const static int64_t MINIMUM_OF_ASYNC_GATHER_STALE_RATIO = 2; +const int64_t MAXIMUM_BLOCK_CNT_OF_ROW_SAMPLE_GATHER_HYBRID_HIST = 100000; +const int64_t MAXIMUM_ROWS_OF_ROW_SAMPLE_GATHER_HYBRID_HIST = 10000000; +const int64_t MINIMUM_BLOCK_CNT_OF_BLOCK_SAMPLE_HYBRID_HIST = 16; enum StatLevel { @@ -156,7 +170,7 @@ struct BlockNumStat K(cg_macro_cnt_arr_), K(cg_micro_cnt_arr_), K(sstable_row_cnt_), - K(memtable_row_cnt_)) + K(memtable_row_cnt_)); }; //TODO@jiangxiu.wt: improve the expression of PartInfo, use the map is better. @@ -193,28 +207,55 @@ struct StatTable database_id_(OB_INVALID_ID), table_id_(OB_INVALID_ID), stale_percent_(0.0), - partition_stat_infos_() + partition_stat_infos_(), + is_async_gather_(false), + async_partition_ids_() { partition_stat_infos_.set_attr(lib::ObMemAttr(MTL_ID(), "StatTable")); + async_partition_ids_.set_attr(lib::ObMemAttr(MTL_ID(), "StatTable")); } - StatTable(uint64_t database_id, uint64_t table_id) : + StatTable(uint64_t database_id, uint64_t table_id, bool is_async_gather = false) : database_id_(database_id), table_id_(table_id), stale_percent_(0.0), - partition_stat_infos_() + partition_stat_infos_(), + is_async_gather_(is_async_gather), + async_partition_ids_() { partition_stat_infos_.set_attr(lib::ObMemAttr(MTL_ID(), "StatTable")); + async_partition_ids_.set_attr(lib::ObMemAttr(MTL_ID(), "StatTable")); } bool operator<(const StatTable &other) const; int assign(const StatTable &other); TO_STRING_KV(K_(database_id), K_(table_id), K_(stale_percent), - K_(partition_stat_infos)); + K_(partition_stat_infos), + K_(is_async_gather), + K_(async_partition_ids)); uint64_t database_id_; uint64_t table_id_; double stale_percent_; ObArray partition_stat_infos_; + bool is_async_gather_; + ObArray async_partition_ids_; +}; + +struct AsyncStatTable +{ + AsyncStatTable() : + table_id_(OB_INVALID_ID), + partition_ids_() + {} + AsyncStatTable(int64_t table_id) : + table_id_(table_id), + partition_ids_() + {} + int assign(const AsyncStatTable &other); + TO_STRING_KV(K_(table_id), + K_(partition_ids)); + uint64_t table_id_; + ObArray partition_ids_; }; enum ObStatTableType { @@ -346,6 +387,7 @@ struct ObAnalyzeSampleInfo void set_percent(double percent); void set_rows(double row_num); void set_is_block_sample(bool is_block_sample) { is_block_sample_ = is_block_sample; } + bool is_specify_sample() const { return is_sample_ && sample_value_ >= 0.000001 && sample_value_ < 100.0; } TO_STRING_KV(K_(is_sample), K_(is_block_sample), @@ -462,7 +504,12 @@ struct ObTableStatParam { allocator_(NULL), ref_table_type_(share::schema::ObTableType::MAX_TABLE_TYPE), column_group_params_(), - online_sample_percent_(1.) + online_sample_percent_(1.), + is_async_gather_(false), + async_gather_sample_size_(DEFAULT_ASYNC_SAMPLE_SIZE), + async_full_table_size_(DEFAULT_ASYNC_FULL_TABLE_SIZE), + async_partition_ids_(NULL), + hist_sample_info_() {} int assign(const ObTableStatParam &other); @@ -478,9 +525,9 @@ struct ObTableStatParam { return global_data_part_id_ != INVALID_GLOBAL_PART_ID; } - bool is_specify_partition_gather() const; + bool is_specify_partition() const; - bool is_specify_column_gather() const; + bool is_specify_column() const; int64_t get_need_gather_column() const; @@ -488,6 +535,8 @@ struct ObTableStatParam { part_stat_param_.need_modify_ || subpart_stat_param_.need_modify_; } + bool is_specify_sample() const { return sample_info_.is_specify_sample(); } + uint64_t tenant_id_; ObString db_name_; @@ -543,6 +592,11 @@ struct ObTableStatParam { share::schema::ObTableType ref_table_type_; ObArray column_group_params_; double online_sample_percent_; + bool is_async_gather_; + int64_t async_gather_sample_size_; + int64_t async_full_table_size_; + const ObIArray *async_partition_ids_; + ObAnalyzeSampleInfo hist_sample_info_; TO_STRING_KV(K(tenant_id_), K(db_name_), @@ -587,7 +641,12 @@ struct ObTableStatParam { K(is_temp_table_), K(ref_table_type_), K(column_group_params_), - K(online_sample_percent_)); + K(online_sample_percent_), + K(is_async_gather_), + K(async_gather_sample_size_), + K(async_full_table_size_), + KPC(async_partition_ids_), + K(hist_sample_info_)); }; struct ObOptStatGatherParam { @@ -615,7 +674,11 @@ struct ObOptStatGatherParam { gather_vectorize_(DEFAULT_STAT_GATHER_VECTOR_BATCH_SIZE), sepcify_scn_(0), use_column_store_(false), - is_specify_partition_(false) + is_specify_partition_(false), + is_async_gather_(false), + async_gather_sample_size_(DEFAULT_ASYNC_SAMPLE_SIZE), + async_full_table_size_(DEFAULT_ASYNC_FULL_TABLE_SIZE), + hist_sample_info_() {} int assign(const ObOptStatGatherParam &other); int64_t get_need_gather_column() const; @@ -643,6 +706,10 @@ struct ObOptStatGatherParam { uint64_t sepcify_scn_; bool use_column_store_; bool is_specify_partition_; + int64_t is_async_gather_; + int64_t async_gather_sample_size_; + int64_t async_full_table_size_; + ObAnalyzeSampleInfo hist_sample_info_; TO_STRING_KV(K(tenant_id_), K(db_name_), @@ -665,7 +732,11 @@ struct ObOptStatGatherParam { K(gather_vectorize_), K(sepcify_scn_), K(use_column_store_), - K(is_specify_partition_)); + K(is_specify_partition_), + K(is_async_gather_), + K(async_gather_sample_size_), + K(async_full_table_size_), + K(hist_sample_info_)); }; struct ObOptStat diff --git a/src/share/stat/ob_stat_item.cpp b/src/share/stat/ob_stat_item.cpp index f58e58e8a..94d95caf4 100644 --- a/src/share/stat/ob_stat_item.cpp +++ b/src/share/stat/ob_stat_item.cpp @@ -234,8 +234,6 @@ bool ObStatTopKHist::is_needed() const int ObStatTopKHist::gen_expr(char *buf, const int64_t buf_len, int64_t &pos) { int ret = OB_SUCCESS; - const int64_t MIN_BUCKET_SIZE = 256; - const int64_t MAX_BUCKET_SIZE = 2048; if (OB_ISNULL(col_param_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("column param is null", K(ret), K(col_param_)); @@ -247,7 +245,7 @@ int ObStatTopKHist::gen_expr(char *buf, const int64_t buf_len, int64_t &pos) ret = OB_ERR_INVALID_SIZE_SPECIFIED; LOG_WARN("get invalid argument, expected value in the range[1, 2048]", K(ret), K(bkt_num)); } - double err_rate = 1.0 / (1000 * (bkt_num / MIN_BUCKET_SIZE)); + double err_rate = 1.0 / get_window_size(bkt_num); if (OB_SUCC(ret)) { if (OB_FAIL(databuff_printf(buf, buf_len, pos, lib::is_oracle_mode() ? " TOP_K_FRE_HIST(%lf, \"%.*s\", %ld, %ld)" : diff --git a/src/share/stat/ob_stat_item.h b/src/share/stat/ob_stat_item.h index 17398afe1..b6179ccd9 100644 --- a/src/share/stat/ob_stat_item.h +++ b/src/share/stat/ob_stat_item.h @@ -223,6 +223,8 @@ public: class ObStatTopKHist : public ObStatColItem { + const static int64_t MIN_BUCKET_SIZE = 256; + const static int64_t MAX_BUCKET_SIZE = 2048; public: ObStatTopKHist() : ObStatColItem(), tab_stat_(NULL), max_disuse_cnt_(0) {} ObStatTopKHist(const ObColumnStatParam *param, @@ -259,6 +261,8 @@ public: virtual bool is_needed() const override; virtual int gen_expr(char *buf, const int64_t buf_len, int64_t &pos) override; virtual int decode(ObObj &obj, ObIAllocator &allocator) override; + static int64_t get_window_size(int64_t bucket_num) { + return 1000 * (bucket_num < MIN_BUCKET_SIZE ? 1 : bucket_num / MIN_BUCKET_SIZE); } protected: ObOptTableStat *tab_stat_; int64_t max_disuse_cnt_; @@ -301,7 +305,7 @@ public: ObGlobalTableStat() : row_count_(0), row_size_(0), data_size_(0), macro_block_count_(0), micro_block_count_(0), part_cnt_(0), last_analyzed_(0), - cg_macro_cnt_arr_(), cg_micro_cnt_arr_(), stat_locked_(false), + cg_macro_cnt_arr_(), cg_micro_cnt_arr_(), stat_locked_(false), stale_stats_(false), sstable_row_cnt_(0), memtable_row_cnt_(0) {} @@ -321,6 +325,8 @@ public: void set_last_analyzed(int64_t last_analyzed) { last_analyzed_ = last_analyzed; } void set_stat_locked(bool locked) { stat_locked_ = locked; } bool get_stat_locked() const { return stat_locked_; } + void set_stale_stats(bool stale_stats) { stale_stats_ = stale_stats; } + bool get_stale_stats() const { return stale_stats_; } int64_t get_sstable_row_cnt() const { return sstable_row_cnt_; } int64_t get_memtable_row_cnt() const { return memtable_row_cnt_; } @@ -335,6 +341,7 @@ public: K(cg_macro_cnt_arr_), K(cg_micro_cnt_arr_), K(stat_locked_), + K(stale_stats_), K(sstable_row_cnt_), K(memtable_row_cnt_)); @@ -349,6 +356,7 @@ private: ObArray cg_macro_cnt_arr_; ObArray cg_micro_cnt_arr_; bool stat_locked_; + bool stale_stats_; int64_t sstable_row_cnt_; int64_t memtable_row_cnt_; }; diff --git a/src/sql/engine/cmd/ob_analyze_executor.cpp b/src/sql/engine/cmd/ob_analyze_executor.cpp index 842abf31c..3786086e0 100644 --- a/src/sql/engine/cmd/ob_analyze_executor.cpp +++ b/src/sql/engine/cmd/ob_analyze_executor.cpp @@ -60,6 +60,8 @@ int ObAnalyzeExecutor::execute(ObExecContext &ctx, ObAnalyzeStmt &stmt) LOG_USER_ERROR(OB_NOT_SUPPORTED, "analyze table during restore or standby cluster"); } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { LOG_WARN("failed to implicit commit before gather stats", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::cancel_async_gather_stats(ctx))) { + LOG_WARN("failed to cancel async gather stats", K(ret)); } else if (OB_FAIL(stmt.fill_table_stat_params(ctx, params))) { LOG_WARN("failed to fill table stat param", K(ret)); } else { diff --git a/src/sql/ob_optimizer_trace_impl.h b/src/sql/ob_optimizer_trace_impl.h index 8599392b0..ebdce9213 100644 --- a/src/sql/ob_optimizer_trace_impl.h +++ b/src/sql/ob_optimizer_trace_impl.h @@ -363,6 +363,11 @@ public: typename std::enable_if, T>::value, int>::type append(const T& value); + //for ObIArrayWrap + template + typename std::enable_if, T>::value, int>::type + append(const T& value); + //for ObIArray template typename std::enable_if, T>::value, int>::type @@ -505,6 +510,23 @@ ObOptimizerTraceImpl::append(const T& value) return ret; } +//for ObIArray +template +typename std::enable_if, T>::value, int>::type +ObOptimizerTraceImpl::append(const T& value) +{ + int ret = OB_SUCCESS; + append("["); + for (int i = 0; OB_SUCC(ret) && i < value.count(); ++i) { + if (i > 0) { + append(", "); + } + ret = append(value.at(i)); + } + append("]"); + return ret; +} + //for ObIArray template typename std::enable_if, T>::value, int>::type diff --git a/src/sql/optimizer/ob_access_path_estimation.cpp b/src/sql/optimizer/ob_access_path_estimation.cpp index 1c713ac91..59362114c 100644 --- a/src/sql/optimizer/ob_access_path_estimation.cpp +++ b/src/sql/optimizer/ob_access_path_estimation.cpp @@ -76,7 +76,7 @@ int ObAccessPathEstimation::inner_estimate_rowcount(ObOptimizerContext &ctx, valid_methods & hint_specify_methods ? valid_methods & hint_specify_methods : valid_methods, method))) { LOG_WARN("failed to choose one est method", K(ret), K(valid_methods)); - } else if (OB_FAIL(do_estimate_rowcount(ctx, paths, is_inner_path, filter_exprs, valid_methods, method))) { + } else if (OB_FAIL(do_estimate_rowcount(ctx, paths, filter_exprs, valid_methods, method))) { LOG_WARN("failed to do estimate rowcount", K(ret), K(method), K(valid_methods)); } for (int64_t i = 0; OB_SUCC(ret) && i < paths.count(); i ++) { @@ -92,14 +92,13 @@ int ObAccessPathEstimation::inner_estimate_rowcount(ObOptimizerContext &ctx, int ObAccessPathEstimation::do_estimate_rowcount(ObOptimizerContext &ctx, common::ObIArray &paths, - const bool is_inner_path, const ObIArray &filter_exprs, ObBaseTableEstMethod &valid_methods, ObBaseTableEstMethod &method) { int ret = OB_SUCCESS; bool is_success = true; - LOG_TRACE("Try to do estimate rowcount", K(method), K(is_inner_path)); + LOG_TRACE("Try to do estimate rowcount", K(method)); if (OB_UNLIKELY(EST_INVALID == method) || OB_UNLIKELY((method & EST_DS_FULL) && (method & EST_DS_BASIC)) || @@ -111,7 +110,7 @@ int ObAccessPathEstimation::do_estimate_rowcount(ObOptimizerContext &ctx, if (OB_SUCC(ret) && (method & (EST_DS_BASIC | EST_DS_FULL))) { bool only_ds_basic_stat = (method & EST_DS_BASIC); if (OB_FAIL(process_dynamic_sampling_estimation( - ctx, paths, is_inner_path, filter_exprs, only_ds_basic_stat, is_success))) { + ctx, paths, filter_exprs, only_ds_basic_stat, is_success))) { LOG_WARN("failed to process statistics estimation", K(ret)); } else if (!is_success) { valid_methods &= ~EST_DS_BASIC; @@ -311,6 +310,11 @@ int ObAccessPathEstimation::choose_best_est_method(ObOptimizerContext &ctx, bool can_use_ds = valid_methods & (EST_DS_FULL | EST_DS_BASIC); bool can_use_storage = valid_methods & EST_STORAGE; + if (OB_ISNULL(ctx.get_session_info())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null param", K(ret)); + } + // check is simple scene bool is_table_get = false; for (int64_t i = 0; OB_SUCC(ret) && !is_table_get && i < paths.count(); ++i) { @@ -342,8 +346,7 @@ int ObAccessPathEstimation::choose_best_est_method(ObOptimizerContext &ctx, // check is complex scene if (OB_SUCC(ret) && !is_simple_scene && !is_complex_scene && (valid_methods | EST_DS_FULL)) { - ObArenaAllocator tmp_alloc("ObOptSel"); - ObSelEstimatorFactory factory(tmp_alloc); + ObSelEstimatorFactory factory(ctx.get_session_info()->get_effective_tenant_id()); const OptSelectivityCtx* sel_ctx = NULL; if (OB_UNLIKELY(paths.empty()) || OB_ISNULL(paths.at(0)) || @@ -364,6 +367,20 @@ int ObAccessPathEstimation::choose_best_est_method(ObOptimizerContext &ctx, } } + //check opt stats is expired + if (OB_SUCC(ret) && !is_simple_scene && !is_complex_scene && can_use_ds) { + const ObLogPlan* log_plan = NULL; + const OptTableMeta *table_meta = NULL; + if (!paths.empty() && paths.at(0)->parent_ != NULL && + (log_plan = paths.at(0)->parent_->get_plan()) != NULL && + (table_meta = log_plan->get_basic_table_metas().get_table_meta_by_table_id(paths.at(0)->table_id_)) != NULL && + table_meta->is_opt_stat_expired() && + !table_meta->is_stat_locked()) { + is_simple_scene = false; + is_complex_scene = true; + } + } + if (OB_FAIL(ret)) { } else if (is_simple_scene) { method = choose_one_est_method(valid_methods, simple_est_priority, priority_cnt); @@ -1676,13 +1693,12 @@ int ObAccessPathEstimation::estimate_full_table_rowcount_by_meta_table(ObOptimiz int ObAccessPathEstimation::process_dynamic_sampling_estimation(ObOptimizerContext &ctx, ObIArray &paths, - const bool is_inner_path, const ObIArray &filter_exprs, bool only_ds_basic_stat, bool &is_success) { int ret = OB_SUCCESS; - LOG_TRACE("begin process dynamic sampling estimation", K(paths), K(is_inner_path)); + LOG_TRACE("begin process dynamic sampling estimation", K(paths)); ObDSTableParam ds_table_param; ObSEArray ds_result_items; is_success = true; @@ -1704,47 +1720,52 @@ int ObAccessPathEstimation::process_dynamic_sampling_estimation(ObOptimizerConte LOG_WARN("failed to get ds table param", K(ret), K(ds_table_param)); } else if (!ds_table_param.is_valid()) { is_success = false; - } else if (OB_FAIL(add_ds_result_items(paths, filter_exprs, specify_ds, - ds_result_items, only_ds_basic_stat))) { - LOG_WARN("failed to init ds result items", K(ret)); } else { - OPT_TRACE_TITLE("BEGIN DYNAMIC SAMPLE ESTIMATION"); - ObArenaAllocator allocator("ObOpTableDS", OB_MALLOC_NORMAL_BLOCK_SIZE, ctx.get_session_info()->get_effective_tenant_id()); - ObDynamicSampling dynamic_sampling(ctx, allocator); - int64_t start_time = ObTimeUtility::current_time(); - bool throw_ds_error = false; - if (OB_FAIL(dynamic_sampling.estimate_table_rowcount(ds_table_param, ds_result_items, throw_ds_error))) { - if (!throw_ds_error && !is_retry_ret(ret)) { - LOG_WARN("failed to estimate table rowcount caused by some reason, please check!!!", K(ret), - K(start_time), K(ObTimeUtility::current_time() - start_time), K(ds_table_param), - K(ctx.get_session_info()->get_current_query_string())); - if (OB_FAIL(ObDynamicSamplingUtils::add_failed_ds_table_list(table_meta->get_ref_table_id(), - table_meta->get_all_used_parts(), - ctx.get_failed_ds_tab_list()))) { - LOG_WARN("failed to add failed ds table list", K(ret)); + bool only_ds_filter = (table_meta->use_opt_stat() && !table_meta->is_opt_stat_expired()) || table_meta->is_stat_locked(); + if (OB_FAIL(add_ds_result_items(paths, filter_exprs, specify_ds, + ds_result_items, + only_ds_basic_stat, + only_ds_filter))) { + LOG_WARN("failed to init ds result items", K(ret)); + } else if (!ds_result_items.empty()) { + OPT_TRACE_TITLE("BEGIN DYNAMIC SAMPLE ESTIMATION"); + ObArenaAllocator allocator("ObOpTableDS", OB_MALLOC_NORMAL_BLOCK_SIZE, ctx.get_session_info()->get_effective_tenant_id()); + ObDynamicSampling dynamic_sampling(ctx, allocator); + int64_t start_time = ObTimeUtility::current_time(); + bool throw_ds_error = false; + if (OB_FAIL(dynamic_sampling.estimate_table_rowcount(ds_table_param, ds_result_items, throw_ds_error))) { + if (!throw_ds_error && !is_retry_ret(ret)) { + LOG_WARN("failed to estimate table rowcount caused by some reason, please check!!!", K(ret), + K(start_time), K(ObTimeUtility::current_time() - start_time), K(ds_table_param), + K(ctx.get_session_info()->get_current_query_string())); + if (OB_FAIL(ObDynamicSamplingUtils::add_failed_ds_table_list(table_meta->get_ref_table_id(), + table_meta->get_all_used_parts(), + ctx.get_failed_ds_tab_list()))) { + LOG_WARN("failed to add failed ds table list", K(ret)); + } else { + is_success = false; + } } else { - is_success = false; + LOG_WARN("failed to dynamic sampling", K(ret), K(start_time), K(ds_table_param)); } - } else { - LOG_WARN("failed to dynamic sampling", K(ret), K(start_time), K(ds_table_param)); + } else if (OB_FAIL(update_table_stat_info_by_dynamic_sampling(paths.at(0), + ds_table_param.ds_level_, + ds_result_items, + only_ds_filter, + no_ds_data))) { + LOG_WARN("failed to update table stat info by dynamic sampling", K(ret)); + } else if (only_ds_basic_stat || no_ds_data) { + if (OB_FAIL(process_statistics_estimation(paths))) { + LOG_WARN("failed to process statistics estimation", K(ret)); + } + } else if (OB_FAIL(estimate_path_rowcount_by_dynamic_sampling(ds_table_param.table_id_, paths, ds_result_items))) { + LOG_WARN("failed to estimate path rowcount by dynamic sampling", K(ret)); } - } else if (OB_FAIL(update_table_stat_info_by_dynamic_sampling(paths.at(0), - ds_table_param.ds_level_, - ds_result_items, - no_ds_data))) { - LOG_WARN("failed to update table stat info by dynamic sampling", K(ret)); - } else if (only_ds_basic_stat || no_ds_data) { - if (OB_FAIL(process_statistics_estimation(paths))) { - LOG_WARN("failed to process statistics estimation", K(ret)); - } - } else if (OB_FAIL(estimate_path_rowcount_by_dynamic_sampling(ds_table_param.table_id_, paths, - is_inner_path, ds_result_items))) { - LOG_WARN("failed to estimate path rowcount by dynamic sampling", K(ret)); - LOG_TRACE("finish dynamic sampling", K(only_ds_basic_stat), K(no_ds_data), K(is_success)); + LOG_TRACE("finish dynamic sampling", K(only_ds_basic_stat), K(only_ds_filter), K(no_ds_data), K(is_success)); + OPT_TRACE("end to process table dynamic sampling estimation"); + OPT_TRACE("dynamic sampling estimation result:"); + OPT_TRACE(ds_result_items); } - OPT_TRACE("end to process table dynamic sampling estimation"); - OPT_TRACE("dynamic sampling estimation result:"); - OPT_TRACE(ds_result_items); } return ret; } @@ -1754,7 +1775,8 @@ int ObAccessPathEstimation::add_ds_result_items(ObIArray &paths, const ObIArray &filter_exprs, const bool specify_ds, ObIArray &ds_result_items, - bool only_ds_basic_stat) + bool only_ds_basic_stat, + bool only_ds_filter) { int ret = OB_SUCCESS; bool all_path_is_get = false; @@ -1776,7 +1798,8 @@ int ObAccessPathEstimation::add_ds_result_items(ObIArray &paths, } else { //1.init ds basic stat item ObDSResultItem basic_item(ObDSResultItemType::OB_DS_BASIC_STAT, paths.at(0)->ref_table_id_); - if (OB_FAIL(get_need_dynamic_sampling_columns(paths.at(0)->parent_->get_plan(), + if (!only_ds_filter && + OB_FAIL(get_need_dynamic_sampling_columns(paths.at(0)->parent_->get_plan(), paths.at(0)->table_id_, filter_exprs, true, false, basic_item.exprs_))) { @@ -1820,7 +1843,7 @@ int ObAccessPathEstimation::add_ds_result_items(ObIArray &paths, } } LOG_TRACE("succeed to add_ds result items", K(paths), K(all_path_is_get), K(filter_exprs), - K(ds_result_items), K(only_ds_basic_stat)); + K(ds_result_items), K(only_ds_basic_stat), K(only_ds_filter)); return ret; } @@ -1887,6 +1910,7 @@ int ObAccessPathEstimation::get_need_dynamic_sampling_columns(const ObLogPlan* l int ObAccessPathEstimation::update_table_stat_info_by_dynamic_sampling(AccessPath *path, int64_t ds_level, ObIArray &ds_result_items, + bool only_ds_filter, bool &no_ds_data) { int ret = OB_SUCCESS; @@ -1909,21 +1933,25 @@ int ObAccessPathEstimation::update_table_stat_info_by_dynamic_sampling(AccessPat ObCostTableScanInfo &est_cost_info = path->est_cost_info_; OptTableMetas &table_metas = path->parent_->get_plan()->get_basic_table_metas(); OptTableMeta *table_meta = table_metas.get_table_meta_by_table_id(path->table_id_); - bool no_add_micro_block = (OB_E(EventTable::EN_LEADER_STORAGE_ESTIMATION) OB_SUCCESS) != OB_SUCCESS; - if (!no_add_micro_block) { - est_cost_info.table_meta_info_->micro_block_count_ = item->stat_handle_.stat_->get_micro_block_num(); - } - est_cost_info.table_meta_info_->table_row_count_ = row_count; if (OB_ISNULL(table_meta) || OB_UNLIKELY(OB_INVALID_ID == table_meta->get_ref_table_id())) { //do nothing - } else if (OB_FAIL(update_column_metas_by_ds_col_stat(row_count, - item->stat_handle_.stat_->get_ds_col_stats(), - table_meta->get_column_metas()))) { - LOG_WARN("failed to fill ds col stat", K(ret)); } else { - table_meta->set_rows(row_count); - table_meta->set_use_ds_stat(); table_meta->set_ds_level(ds_level); + if (!only_ds_filter) { + bool no_add_micro_block = (OB_E(EventTable::EN_LEADER_STORAGE_ESTIMATION) OB_SUCCESS) != OB_SUCCESS; + if (!no_add_micro_block) { + est_cost_info.table_meta_info_->micro_block_count_ = item->stat_handle_.stat_->get_micro_block_num(); + } + est_cost_info.table_meta_info_->table_row_count_ = row_count; + if (OB_FAIL(update_column_metas_by_ds_col_stat(row_count, + item->stat_handle_.stat_->get_ds_col_stats(), + table_meta->get_column_metas()))) { + LOG_WARN("failed to fill ds col stat", K(ret)); + } else { + table_meta->set_rows(row_count); + table_meta->set_use_ds_stat(); + } + } } } return ret; @@ -1950,7 +1978,6 @@ int ObAccessPathEstimation::update_table_stat_info_by_default(AccessPath *path) int ObAccessPathEstimation::estimate_path_rowcount_by_dynamic_sampling(const uint64_t table_id, ObIArray &paths, - const bool is_inner_path, ObIArray &ds_result_items) { int ret = OB_SUCCESS; @@ -2010,20 +2037,6 @@ int ObAccessPathEstimation::estimate_path_rowcount_by_dynamic_sampling(const uin index_back_row_count = index_back_row_count != 0 ? index_back_row_count : logical_row_count; physical_row_count = logical_row_count; } - if (is_inner_path) { - if (OB_FAIL(ObOptSelectivity::calculate_selectivity(*est_cost_info.table_metas_, - *est_cost_info.sel_ctx_, - est_cost_info.pushdown_prefix_filters_, - est_cost_info.pushdown_prefix_filter_sel_, - paths.at(i)->parent_->get_plan()->get_predicate_selectivities()))) { - LOG_WARN("failed to calculate selectivity", K(est_cost_info.pushdown_prefix_filters_), K(ret)); - } else { - logical_row_count = logical_row_count * est_cost_info.pushdown_prefix_filter_sel_; - index_back_row_count = index_back_row_count * est_cost_info.pushdown_prefix_filter_sel_; - physical_row_count = logical_row_count; - output_rowcnt = output_rowcnt * est_cost_info.pushdown_prefix_filter_sel_; - } - } if (OB_SUCC(ret)) { // block sampling double block_sample_ratio = est_cost_info.sample_info_.is_block_sample() ? diff --git a/src/sql/optimizer/ob_access_path_estimation.h b/src/sql/optimizer/ob_access_path_estimation.h index 6fe3c9ecc..2489c4231 100644 --- a/src/sql/optimizer/ob_access_path_estimation.h +++ b/src/sql/optimizer/ob_access_path_estimation.h @@ -98,7 +98,6 @@ private: static int do_estimate_rowcount(ObOptimizerContext &ctx, common::ObIArray &paths, - const bool is_inner_path, const ObIArray &filter_exprs, ObBaseTableEstMethod &valid_methods, ObBaseTableEstMethod &method); @@ -137,7 +136,6 @@ private: static int process_dynamic_sampling_estimation(ObOptimizerContext &ctx, ObIArray &paths, - const bool is_inner_path, const ObIArray &filter_exprs, bool only_ds_basic_stat, bool &is_success); @@ -237,17 +235,18 @@ private: const ObIArray &filter_exprs, const bool specify_ds, ObIArray &ds_result_items, - bool only_ds_basic_stat); + bool only_ds_basic_stat, + bool only_ds_filter); static int update_table_stat_info_by_dynamic_sampling(AccessPath *path, int64_t ds_level, ObIArray &ds_result_items, + bool only_ds_filter, bool &no_ds_data); static int update_table_stat_info_by_default(AccessPath *path); static int estimate_path_rowcount_by_dynamic_sampling(const uint64_t table_id, ObIArray &paths, - const bool is_inner_path, ObIArray &ds_result_items); static int classify_paths(common::ObIArray &paths, common::ObIArray &normal_paths, diff --git a/src/sql/optimizer/ob_dynamic_sampling.cpp b/src/sql/optimizer/ob_dynamic_sampling.cpp index 2b155691a..6bc28f57a 100644 --- a/src/sql/optimizer/ob_dynamic_sampling.cpp +++ b/src/sql/optimizer/ob_dynamic_sampling.cpp @@ -403,7 +403,7 @@ int ObDynamicSampling::get_table_dml_info(const uint64_t tenant_id, cur_modified_dml_cnt, false))) { LOG_WARN("failed to estimate modified count", K(ret)); - } else if (OB_FAIL(pl::ObDbmsStats::get_table_stale_percent_threshold(*ctx_->get_exec_ctx(), + } else if (OB_FAIL(pl::ObDbmsStats::get_table_stale_percent_threshold(ctx_->get_exec_ctx()->get_sql_proxy(), tenant_id, table_id, stale_percent_threshold))) { @@ -772,7 +772,6 @@ int ObDynamicSampling::calc_table_sample_block_ratio(const ObDSTableParam ¶m { int ret = OB_SUCCESS; int64_t sample_micro_cnt = param.sample_block_cnt_; - const int64_t MAX_FULL_SCAN_ROW_COUNT = 100000; int64_t macro_threshold = 200; if (param.is_virtual_table_) { sample_block_ratio_ = 100.0; @@ -783,6 +782,8 @@ int ObDynamicSampling::calc_table_sample_block_ratio(const ObDSTableParam ¶m LOG_WARN("get unexpected error", K(ret), K(param)); } else if (OB_FAIL(estimate_table_block_count_and_row_count(param))) { LOG_WARN("failed to estimate table block count and row count", K(ret)); + } else if (sstable_row_count_ + memtable_row_count_ <= MAGIC_MAX_AUTO_SAMPLE_SIZE) { + sample_block_ratio_ = 100.0; } else { int64_t max_allowed_multiple = sample_micro_cnt > OB_DS_BASIC_SAMPLE_MICRO_CNT ? sample_micro_cnt / OB_DS_BASIC_SAMPLE_MICRO_CNT : 1; if (micro_block_num_ > OB_DS_MAX_BASIC_SAMPLE_MICRO_CNT * max_allowed_multiple && diff --git a/src/sql/optimizer/ob_join_order.cpp b/src/sql/optimizer/ob_join_order.cpp index 898b226f7..f06e5ab72 100644 --- a/src/sql/optimizer/ob_join_order.cpp +++ b/src/sql/optimizer/ob_join_order.cpp @@ -2964,7 +2964,6 @@ int ObJoinOrder::estimate_rowcount_for_access_path(ObIArray &all_pa int ret = OB_SUCCESS; bool is_use_ds = false; method = EST_INVALID; - get_plan()->get_selectivity_ctx().set_dependency_type(FilterDependencyType::INDEPENDENT); if (OB_FAIL(ObAccessPathEstimation::estimate_rowcount(OPT_CTX, all_paths, is_inner_path, filter_exprs, @@ -4378,6 +4377,7 @@ int ObJoinOrder::estimate_size_and_width_for_subquery(uint64_t table_id, table_id_, output_row_size_))) { LOG_WARN("estimate width of row failed", K(table_id_), K(ret)); + } else if (FALSE_IT(get_plan()->get_selectivity_ctx().init_op_ctx(NULL, root->get_card()))) { } else if (OB_FAIL(ObOptSelectivity::calculate_selectivity(get_plan()->get_basic_table_metas(), get_plan()->get_selectivity_ctx(), get_restrict_infos(), @@ -7073,7 +7073,7 @@ int JoinPath::do_re_estimate_cost(EstimateCostInfo &info, double &card, double & LOG_WARN("failed to re estimate cost", K(ret)); } else if (OB_FAIL(try_set_batch_nlj_for_right_access_path(false))) { LOG_WARN("failed to try set batch nlj for right access path", K(ret)); - } else if (OB_FAIL(re_estimate_rows(left_output_rows, right_output_rows, card))) { + } else if (OB_FAIL(re_estimate_rows(info.join_filter_infos_, left_output_rows, right_output_rows, card))) { LOG_WARN("failed to re estimate rows", K(ret)); } else if (NESTED_LOOP_JOIN == join_algo_) { if (OB_FAIL(cost_nest_loop_join(info.need_parallel_, @@ -7194,21 +7194,29 @@ int JoinPath::get_re_estimate_param(EstimateCostInfo ¶m, return ret; } -int JoinPath::re_estimate_rows(double left_output_rows, double right_output_rows, double &row_count) +int JoinPath::re_estimate_rows(ObIArray &pushdown_join_filter_infos, + double left_output_rows, + double right_output_rows, + double &row_count) { int ret = OB_SUCCESS; double selectivity = 1.0; ObLogPlan *plan = NULL; ObJoinOrder *left_tree = NULL; ObJoinOrder *right_tree = NULL; + const ObDMLStmt *stmt = NULL; if (OB_ISNULL(left_path_) || OB_ISNULL(right_path_) || OB_ISNULL(parent_) || OB_ISNULL(plan = parent_->get_plan()) || OB_ISNULL(left_tree = left_path_->parent_) || - OB_ISNULL(right_tree = right_path_->parent_)) { + OB_ISNULL(right_tree = right_path_->parent_) || + OB_ISNULL(stmt = plan->get_stmt()) || + OB_ISNULL(plan->get_optimizer_context().get_query_ctx())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(left_path_), K(right_path_), K(plan), K(ret)); - } else if (HASH_JOIN == join_algo_ && - !join_filter_infos_.empty()) { + } else if (!plan->get_optimizer_context().get_query_ctx()-> + check_opt_compat_version(COMPAT_VERSION_4_2_4, COMPAT_VERSION_4_3_0, COMPAT_VERSION_4_3_3) ? + HASH_JOIN == join_algo_ && !join_filter_infos_.empty() : + HASH_JOIN == join_algo_ && pushdown_join_filter_infos.empty()) { row_count = get_path_output_rows(); } else if (right_path_->is_inner_path()) { if (left_tree->get_output_rows() > 0) { @@ -7257,6 +7265,12 @@ int JoinPath::re_estimate_rows(double left_output_rows, double right_output_rows selectivity, equal_sets))) { LOG_WARN("failed to calc join output rows", K(ret)); + } else { + for (int64_t i = 0; i < join_filter_infos_.count(); i ++) { + if (join_filter_infos_.at(i).join_filter_selectivity_ > OB_DOUBLE_EPSINON) { + row_count /= join_filter_infos_.at(i).join_filter_selectivity_; + } + } } } return ret; @@ -7846,9 +7860,32 @@ int ObJoinOrder::generate_base_paths() ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected base path type", K(get_type()), K(ret)); } + if (FAILEDx(init_ambient_card())) { + LOG_WARN("failed to init ambient cardinality", K(ret)); + } return ret; } +int ObJoinOrder::init_ambient_card() +{ + int ret = OB_SUCCESS; + const ObDMLStmt* stmt = NULL; + int64_t idx = -1; + if (OB_ISNULL(get_plan()) || OB_ISNULL(stmt = get_plan()->get_stmt()) || + FALSE_IT(idx = get_plan()->get_stmt()->get_table_bit_index(table_id_)) || + OB_UNLIKELY(idx < 0) || OB_UNLIKELY(idx > stmt->get_table_size())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected param", K(get_plan()), K(table_id_), K(idx), KPC(stmt)); + } else if (OB_FAIL(ambient_card_.prepare_allocate(stmt->get_table_size() + 1))) { + LOG_WARN("failed to allocate", K(ret)); + } else { + for (int64_t i = 0; i < ambient_card_.count(); i ++) { + ambient_card_.at(i) = -1; + } + ambient_card_.at(idx) = output_rows_; + } + return ret; +} int ObJoinOrder::generate_json_table_paths() { @@ -8147,9 +8184,7 @@ int ObJoinOrder::estimate_size_and_width_for_fake_cte(uint64_t table_id, ObSelec } else if (OB_ISNULL(nonrecursive_root)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret), K(nonrecursive_root)); - } else if (FALSE_IT(nonrecursive_plan->get_selectivity_ctx().init_op_ctx( - &nonrecursive_root->get_output_equal_sets(), nonrecursive_root->get_card()))) { - // do nothing + } else if (FALSE_IT(nonrecursive_plan->get_selectivity_ctx().init_op_ctx(nonrecursive_root))) { } else if (OB_FAIL(get_plan()->get_basic_table_metas().add_generate_table_meta_info( get_plan()->get_stmt(), static_cast(nonrecursive_plan->get_stmt()), @@ -8158,6 +8193,7 @@ int ObJoinOrder::estimate_size_and_width_for_fake_cte(uint64_t table_id, ObSelec nonrecursive_plan->get_selectivity_ctx(), nonrecursive_root->get_card()))) { LOG_WARN("failed to add generate table meta info", K(ret)); + } else if (FALSE_IT(get_plan()->get_selectivity_ctx().init_op_ctx(NULL, nonrecursive_root->get_card()))) { } else if (OB_FAIL(ObOptEstCost::estimate_width_for_table(get_plan()->get_basic_table_metas(), get_plan()->get_selectivity_ctx(), stmt->get_column_items(), @@ -8532,6 +8568,9 @@ int ObJoinOrder::generate_normal_subquery_paths() LOG_WARN("failed to push down filter into subquery", K(ret)); } else if (OB_FAIL(append(helper.filters_, candi_nonpushdown_quals))) { LOG_WARN("failed to append", K(ret)); + } else if (OB_FAIL(ObOptimizerUtil::get_onetime_exprs(helper.pushdown_filters_, + helper.exec_params_))) { + LOG_WARN("failed to get onetime exprs", K(ret)); } else if (OB_FAIL(generate_subquery_paths(helper))) { LOG_WARN("failed to generate subquery path", K(ret)); } @@ -8547,6 +8586,7 @@ int ObJoinOrder::generate_subquery_paths(PathHelper &helper) const ObDMLStmt *parent_stmt = NULL; const ObDMLStmt *child_stmt = NULL; ObLogicalOperator *best_child_plan = NULL; + ObSEArray pushdown_onetimes; if (OB_ISNULL(get_plan()) || OB_ISNULL(parent_stmt = get_plan()->get_stmt()) || OB_ISNULL(child_stmt = static_cast(helper.child_stmt_))) { ret = OB_ERR_UNEXPECTED; @@ -8557,6 +8597,10 @@ int ObJoinOrder::generate_subquery_paths(PathHelper &helper) LOG_WARN("failed to create plan", K(ret)); } else if (OB_FAIL(log_plan->add_pushdown_filters(helper.pushdown_filters_))) { LOG_WARN("failed to add pushdown filters", K(ret)); + } else if (OB_FAIL(log_plan->add_exec_params_meta(helper.exec_params_, + get_plan()->get_basic_table_metas(), + get_plan()->get_selectivity_ctx()))) { + LOG_WARN("failed to prepare opt exec param meta", K(ret)); } else { log_plan->set_is_subplan_scan(true); log_plan->set_nonrecursive_plan_for_fake_cte(get_plan()->get_nonrecursive_plan_for_fake_cte()); @@ -8754,6 +8798,15 @@ int ObJoinOrder::init_join_order(const ObJoinOrder *left_tree, } else if (OB_FAIL(get_output_tables().add_members(right_tree->get_output_tables()))) { LOG_WARN("fail to add left tree's output tables", K(ret)); } + + if (FAILEDx(merge_ambient_card(left_tree->get_ambient_card(), + right_tree->get_ambient_card(), + ambient_card_))) { + LOG_WARN("failed to merge rowcnts", K(ret)); + } else { + set_output_rows(-1.0); + } + //设置join info if (OB_SUCC(ret)) { JoinInfo* temp_join_info = NULL; @@ -9099,14 +9152,14 @@ int ObJoinOrder::inner_generate_join_paths(const ObJoinOrder &left_tree, &right_tree.get_tables(), left_tree.get_output_rows(), right_tree.get_output_rows()); - if (OB_FAIL(ObOptSelectivity::calculate_selectivity( + if (OB_FAIL(ObOptSelectivity::calculate_join_selectivity( get_plan()->get_update_table_metas(), get_plan()->get_selectivity_ctx(), hash_join_conditions, equal_cond_sel, get_plan()->get_predicate_selectivities()))) { LOG_WARN("failed to calculate selectivity", K(ret), K(hash_join_conditions)); - } else if (OB_FAIL(ObOptSelectivity::calculate_selectivity( + } else if (OB_FAIL(ObOptSelectivity::calculate_join_selectivity( get_plan()->get_update_table_metas(), get_plan()->get_selectivity_ctx(), hash_join_filters, @@ -9215,14 +9268,14 @@ int ObJoinOrder::inner_generate_join_paths(const ObJoinOrder &left_tree, &right_tree.get_tables(), left_tree.get_output_rows(), right_tree.get_output_rows()); - if (OB_FAIL(ObOptSelectivity::calculate_selectivity( + if (OB_FAIL(ObOptSelectivity::calculate_join_selectivity( get_plan()->get_update_table_metas(), get_plan()->get_selectivity_ctx(), merge_join_conditions, equal_cond_sel, get_plan()->get_predicate_selectivities()))) { LOG_WARN("failed to calculate selectivity", K(ret), K(merge_join_conditions)); - } else if (OB_FAIL(ObOptSelectivity::calculate_selectivity( + } else if (OB_FAIL(ObOptSelectivity::calculate_join_selectivity( get_plan()->get_update_table_metas(), get_plan()->get_selectivity_ctx(), merge_join_filters, @@ -9282,6 +9335,7 @@ int ObJoinOrder::inner_generate_join_paths(const ObJoinOrder &left_tree, } } } + get_plan()->get_selectivity_ctx().clear(); return ret; } @@ -10734,16 +10788,33 @@ int ObJoinOrder::find_possible_join_filter_tables(const Path &left_path, int ObJoinOrder::fill_join_filter_info(JoinFilterInfo &join_filter_info) { int ret = OB_SUCCESS; - if (OB_ISNULL(get_plan())) { + uint64_t opt_version = 0; + if (OB_ISNULL(get_plan()) || OB_ISNULL(OPT_CTX.get_query_ctx())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret), K(get_plan())); + } else if (FALSE_IT(get_plan()->get_selectivity_ctx().clear())) { } else if (OB_FAIL(ObOptSelectivity::calculate_distinct(get_plan()->get_update_table_metas(), get_plan()->get_selectivity_ctx(), join_filter_info.rexprs_, join_filter_info.row_count_, - join_filter_info.right_distinct_card_, - false))) { + join_filter_info.right_distinct_card_))) { LOG_WARN("failed to calc distinct", K(ret)); + } else if (!OPT_CTX.get_query_ctx()->check_opt_compat_version(COMPAT_VERSION_4_2_4, COMPAT_VERSION_4_3_0, + COMPAT_VERSION_4_3_3)) { + // do nothing + } else if (OB_FAIL(ObOptSelectivity::is_columns_contain_pkey(get_plan()->get_basic_table_metas(), + join_filter_info.rexprs_, + join_filter_info.is_right_contain_pk_, + join_filter_info.is_right_union_pk_))) { + LOG_WARN("failed to check is columns contain pkey", K(ret)); + } else if (join_filter_info.is_right_contain_pk_ && join_filter_info.is_right_union_pk_) { + const OptTableMeta *table_meta = get_plan()->get_update_table_metas().get_table_meta_by_table_id(join_filter_info.table_id_); + if (OB_NOT_NULL(table_meta)) { + join_filter_info.right_distinct_card_ = std::max(1.0, table_meta->get_rows()); + } + if (OB_NOT_NULL(table_meta = get_plan()->get_basic_table_metas().get_table_meta_by_table_id(join_filter_info.table_id_))) { + join_filter_info.right_origin_rows_ = std::max(1.0, table_meta->get_rows()); + } } return ret; @@ -10999,7 +11070,8 @@ int ObJoinOrder::check_normal_join_filter_valid(const Path& left_path, bool cur_dfo_has_shuffle_bf = false; if (OB_ISNULL(plan) || OB_ISNULL(left_tree=left_path.parent_) || - OB_ISNULL(stmt = plan->get_stmt())) { + OB_ISNULL(stmt = plan->get_stmt()) || + OB_ISNULL(OPT_CTX.get_query_ctx())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null plan", K(ret)); } @@ -11015,14 +11087,22 @@ int ObJoinOrder::check_normal_join_filter_valid(const Path& left_path, } else { double rate = 1 - join_filter_sel; double threshold = 0.6; + double misjudgment_rate = (static_cast(GCONF._bloom_filter_ratio) / 100.0); if (info.in_current_dfo_) { threshold = 0.9; } info.join_filter_selectivity_ = join_filter_sel; + if (OPT_CTX.get_query_ctx()->check_opt_compat_version(COMPAT_VERSION_4_2_4, COMPAT_VERSION_4_3_0, + COMPAT_VERSION_4_3_3) && + 0 <= misjudgment_rate && misjudgment_rate <= 1.0) { + info.join_filter_selectivity_ += (1 - join_filter_sel) * misjudgment_rate; + } info.can_use_join_filter_ = rate >= threshold || NULL != info.force_filter_; OPT_TRACE("join filter info:"); OPT_TRACE("in current dfo:", info.in_current_dfo_); - OPT_TRACE("filter selectivity:", info.join_filter_selectivity_); + OPT_TRACE("right distinct card:", info.right_distinct_card_); + OPT_TRACE("theoretical filter selectivity:", join_filter_sel); + OPT_TRACE("actual filter selectivity:", info.join_filter_selectivity_); OPT_TRACE("force use join filter:", NULL != info.force_filter_); OPT_TRACE("use join filter:", info.can_use_join_filter_); LOG_TRACE("succeed to check normal join filter", K(info)); @@ -11040,29 +11120,94 @@ int ObJoinOrder::calc_join_filter_selectivity(const Path& left_path, double left_distinct_card = 1.0; double right_distinct_card = 1.0; join_filter_selectivity = 1.0; - if (OB_ISNULL(plan)) { + bool is_pk_join_fk = false; + bool est_enhance_enable = OPT_CTX.get_query_ctx()->check_opt_compat_version(COMPAT_VERSION_4_2_4, COMPAT_VERSION_4_3_0, + COMPAT_VERSION_4_3_3); + if (OB_ISNULL(plan) || OB_ISNULL(left_path.parent_) || + OB_ISNULL(OPT_CTX.get_query_ctx())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null plan", K(ret)); + } else if (FALSE_IT(get_plan()->get_selectivity_ctx().init_op_ctx(&left_path.parent_->get_output_equal_sets(), + left_path.get_path_output_rows(), + &left_path.parent_->get_ambient_card()))) { + } else if (est_enhance_enable && + OB_FAIL(calc_join_filter_sel_for_pk_join_fk(left_path, info, join_filter_selectivity, is_pk_join_fk))) { + LOG_WARN("failed to calc pk join fk join filter sel", K(ret)); + } else if (is_pk_join_fk) { + // do nothing } else if (OB_FAIL(ObOptSelectivity::calculate_distinct(plan->get_update_table_metas(), plan->get_selectivity_ctx(), info.lexprs_, left_path.get_path_output_rows(), left_distinct_card, - false))) { + est_enhance_enable))) { LOG_WARN("failed to calc distinct", K(ret)); } else { join_filter_selectivity = left_distinct_card / info.right_distinct_card_; + } + if (OB_SUCC(ret)) { if (join_filter_selectivity < 0) { join_filter_selectivity = 0; } else if (join_filter_selectivity > 0.9) { join_filter_selectivity = 0.9; } - LOG_TRACE("succeed to calc join filter selectivity", K(join_filter_selectivity), + LOG_TRACE("succeed to calc join filter selectivity", K(is_pk_join_fk), K(join_filter_selectivity), K(left_distinct_card), K(info.right_distinct_card_)); } return ret; } +int ObJoinOrder::calc_join_filter_sel_for_pk_join_fk(const Path& left_path, + JoinFilterInfo& info, + double &join_filter_selectivity, + bool &is_valid) +{ + int ret = OB_SUCCESS; + is_valid = false; + ObLogPlan *plan = get_plan(); + bool left_contain_pk = false; + bool is_left_union_pk = false; + uint64_t left_table_id = OB_INVALID_ID; + double pk_origin_rows = 1.0; + double left_ndv = 1.0; + if (OB_ISNULL(plan) || OB_ISNULL(left_path.parent_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null plan", K(ret)); + } else if (OB_FAIL(ObOptSelectivity::is_columns_contain_pkey(plan->get_update_table_metas(), + info.lexprs_, + left_contain_pk, + is_left_union_pk, + &left_table_id))) { + LOG_WARN("failed to check is columns contain pkey", K(ret)); + } else if (!info.is_right_contain_pk_ && left_contain_pk && is_left_union_pk) { + pk_origin_rows = plan->get_basic_table_metas().get_rows(left_table_id); + if (pk_origin_rows > OB_DOUBLE_EPSINON) { + is_valid = true; + if (OB_FAIL(plan->get_selectivity_ctx().get_ambient_card(left_table_id, left_ndv))) { + LOG_WARN("failed to get ambient card", K(ret)); + } else { + join_filter_selectivity = left_ndv / std::min(pk_origin_rows, info.right_distinct_card_); + } + } + } else if (info.is_right_union_pk_ && info.is_right_contain_pk_ && !left_contain_pk && OB_INVALID_ID != left_table_id) { + pk_origin_rows = info.right_origin_rows_; + is_valid = true; + double fk_origin_rows = plan->get_basic_table_metas().get_rows(left_table_id); + if (OB_FAIL(ObOptSelectivity::calculate_distinct(plan->get_update_table_metas(), + plan->get_selectivity_ctx(), + info.lexprs_, + left_path.get_path_output_rows(), + left_ndv))) { + LOG_WARN("failed to calculate distinct", K(ret), K(left_table_id), K(info)); + } else { + double fk_ndv = ObOptSelectivity::scale_distinct(left_path.get_path_output_rows(), fk_origin_rows, pk_origin_rows); + left_ndv = std::min(left_ndv, fk_ndv); + join_filter_selectivity = left_ndv / info.right_distinct_card_; + } + } + return ret; +} + int ObJoinOrder::find_shuffle_join_filter(const Path& path, bool &find) { int ret = OB_SUCCESS; @@ -12804,6 +12949,7 @@ int ObJoinOrder::init_est_sel_info_for_access_path(const uint64_t table_id, bool use_global = false; ObSEArray global_part_ids; double scale_ratio = 1.0; + bool stale_stats = false; if (OPT_CTX.use_default_stat()) { // do nothing } else if (OB_ISNULL(OPT_CTX.get_opt_stat_manager())) { @@ -12853,6 +12999,7 @@ int ObJoinOrder::init_est_sel_info_for_access_path(const uint64_t table_id, } else { last_analyzed = stat.get_last_analyzed(); is_stat_locked = stat.get_stat_locked(); + stale_stats = stat.get_stale_stats(); table_meta_info_.table_row_count_ = stat.get_row_count(); table_meta_info_.part_size_ = !use_global ? static_cast(stat.get_avg_data_size()) : static_cast(stat.get_avg_data_size() * all_used_part_id.count()) @@ -12862,7 +13009,7 @@ int ObJoinOrder::init_est_sel_info_for_access_path(const uint64_t table_id, table_meta_info_.has_opt_stat_ = has_opt_stat; LOG_TRACE("total rowcount, use statistics", K(table_meta_info_.table_row_count_), K(table_meta_info_.average_row_size_), K(table_meta_info_.micro_block_count_), - K(table_meta_info_.part_size_)); + K(table_meta_info_.part_size_), K(has_opt_stat), K(is_stat_locked), K(stale_stats)); } } @@ -12914,7 +13061,8 @@ int ObJoinOrder::init_est_sel_info_for_access_path(const uint64_t table_id, last_analyzed, is_stat_locked, table_partition_info_, - &table_meta_info_))) { + &table_meta_info_, + stale_stats))) { LOG_WARN("failed to add base table meta info", K(ret)); } } @@ -13110,7 +13258,7 @@ int ObJoinOrder::init_est_sel_info_for_subquery(const uint64_t table_id, ret = OB_INVALID_ARGUMENT; LOG_WARN("Invalid argument", K(ret), K(get_plan()), K(root), K(child_plan), K(child_stmt)); } else { - child_plan->get_selectivity_ctx().init_op_ctx(&root->get_output_equal_sets(), root->get_card()); + child_plan->get_selectivity_ctx().init_op_ctx(root); if (OB_FAIL(get_plan()->get_basic_table_metas().add_generate_table_meta_info( get_plan()->get_stmt(), static_cast(child_stmt), @@ -13221,6 +13369,417 @@ int ObJoinOrder::check_and_remove_is_null_qual(ObLogPlan *plan, return ret; } +int ObJoinOrder::merge_ambient_card(const ObIArray &left_ambient_card, + const ObIArray &right_ambient_card, + ObIArray &cur_ambient_card) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(left_ambient_card.count() != right_ambient_card.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected param", K(left_ambient_card), K(right_ambient_card)); + } else if (OB_FAIL(cur_ambient_card.prepare_allocate(left_ambient_card.count()))) { + LOG_WARN("failed to allocate", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < cur_ambient_card.count(); i ++) { + double left_rowcnt = left_ambient_card.at(i); + double right_rowcnt = right_ambient_card.at(i); + if (OB_UNLIKELY(left_rowcnt >= 0 && right_rowcnt >= 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected ambient card", K(left_ambient_card), K(right_ambient_card)); + } else if (left_rowcnt >= 0) { + cur_ambient_card.at(i) = left_rowcnt; + } else if (right_rowcnt >= 0) { + cur_ambient_card.at(i) = right_rowcnt; + } else { + cur_ambient_card.at(i) = -1; + } + } + return ret; +} + +int ObJoinOrder::scale_ambient_card(const double origin_rows, + const double new_rows, + const ObIArray &origin_ambient_card, + ObIArray &ambient_card) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ambient_card.assign(origin_ambient_card))) { + LOG_WARN("failed to assign", K(ret)); + } else if (new_rows < origin_rows) { + for (int64_t i = 0; i < ambient_card.count(); i ++) { + if (ambient_card.at(i) >= 0) { + ambient_card.at(i) = ObOptSelectivity::scale_distinct(new_rows, origin_rows, ambient_card.at(i)); + } + } + } + return ret; +} + +int ObJoinOrder::revise_cardinality(const ObJoinOrder *left_tree, + const ObJoinOrder *right_tree, + const JoinInfo &join_info) +{ + int ret = OB_SUCCESS; + double sel = 1.0; + EqualSets equal_sets; + ObSEArray cur_join_ambient_card; + if (OB_ISNULL(left_tree) || OB_ISNULL(right_tree) || + OB_ISNULL(get_plan()) || OB_ISNULL(OPT_CTX.get_query_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(left_tree), K(right_tree), K(get_plan()), K(ret)); + } else if (!OPT_CTX.get_query_ctx()->check_opt_compat_version(COMPAT_VERSION_4_2_4, COMPAT_VERSION_4_3_0, + COMPAT_VERSION_4_3_3)) { + // do nothing + } else if (OB_FAIL(append(equal_sets, left_tree->get_output_equal_sets())) || + OB_FAIL(append(equal_sets, right_tree->get_output_equal_sets()))) { + LOG_WARN("failed to append equal sets", K(ret)); + } else if (OB_FAIL(merge_ambient_card(left_tree->get_ambient_card(), right_tree->get_ambient_card(), cur_join_ambient_card))) { + LOG_WARN("failed to merge rowcnts", K(ret)); + } else if (OB_FAIL(calc_join_ambient_card(get_plan(), + *left_tree, + *right_tree, + output_rows_, + join_info, equal_sets, + cur_join_ambient_card))) { + LOG_WARN("failed to scale base table rowcnts", K(ret)); + } else if (OB_UNLIKELY(cur_join_ambient_card.count() != ambient_card_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected ambient card", K(left_tree->get_ambient_card()), + K(right_tree->get_ambient_card()), K(cur_join_ambient_card), K(ambient_card_)); + } else { + get_plan()->get_selectivity_ctx().clear(); + for (int64_t i = 0; i < ambient_card_.count(); i ++) { + ambient_card_.at(i) = std::min(ambient_card_.at(i), cur_join_ambient_card.at(i)); + } + OPT_TRACE("left output rows :", left_tree->get_output_rows(), " ambient cardinality :", left_tree->get_ambient_card()); + OPT_TRACE("right output rows :", right_tree->get_output_rows(), " ambient cardinality :", right_tree->get_ambient_card()); + OPT_TRACE("output rows of", left_tree, "join", right_tree, ":", get_output_rows(), " ambient cardinality :", cur_join_ambient_card); + OPT_TRACE("Revised ambient cardinality :", ambient_card_); + LOG_DEBUG("estimate join ambient card", K(table_set_), K(left_tree->get_tables()), K(right_tree->get_tables()), K(cur_join_ambient_card)); + } + return ret; +} + + +int ObJoinOrder::calc_join_ambient_card(ObLogPlan *plan, + const ObJoinOrder &left_tree, + const ObJoinOrder &right_tree, + const double join_output_rows, + const JoinInfo &join_info, + EqualSets &equal_sets, + ObIArray &ambient_card) +{ + int ret = OB_SUCCESS; + const ObJoinType join_type = join_info.join_type_; + JoinInfo tmp_join_info; + double left_ambient_card_sel = 1.0; + double right_ambient_card_sel = 1.0; + double where_sel_for_oj = 1.0; + double tmp_rows = 0.0; + ObSEArray join_conditions; + ObSEArray ambient_card_sels; + const ObRelIds &left_ids = left_tree.get_tables(); + const ObRelIds &right_ids = right_tree.get_tables(); + double left_output_rows = left_tree.get_output_rows(); + double right_output_rows = right_tree.get_output_rows(); + if (OB_ISNULL(plan)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (OB_FAIL(append(join_conditions, join_info.on_conditions_)) || + OB_FAIL(append(join_conditions, join_info.where_conditions_))) { + LOG_WARN("failed to append", K(ret)); + } else if (OB_FAIL(ambient_card_sels.prepare_allocate(ambient_card.count()))) { + LOG_WARN("failed to prepare allocate", K(ret)); + } else { + plan->get_selectivity_ctx().set_assumption_type(join_type); + } + for (int64_t i = 0; i < ambient_card_sels.count(); i ++) { + ambient_card_sels.at(i) = 1.0; + } + + // calculate selectivity for left ambient cardinality + if (OB_SUCC(ret)) { + tmp_join_info.join_type_ = IS_ANTI_JOIN(join_type) ? LEFT_ANTI_JOIN : LEFT_SEMI_JOIN; + tmp_join_info.where_conditions_.reuse(); + if (CONNECT_BY_JOIN == join_type) { + // todo + } else if (IS_RIGHT_SEMI_ANTI_JOIN(join_type)) { + left_ambient_card_sel = 0.0; + for (int64_t i = 0; i < ambient_card_sels.count(); i ++) { + if (left_ids.has_member(i)) { + ambient_card_sels.at(i) = 0.0; + } + } + } else if (LEFT_OUTER_JOIN == join_type || FULL_OUTER_JOIN == join_type) { + left_ambient_card_sel = 1.0; + for (int64_t i = 0; i < ambient_card_sels.count(); i ++) { + if (left_ids.has_member(i)) { + ambient_card_sels.at(i) = 1.0; + } + } + } else if (RIGHT_OUTER_JOIN == join_type && OB_FAIL(append(tmp_join_info.where_conditions_, join_info.on_conditions_))) { + LOG_WARN("failed to append on conditions", K(ret)); + } else if (!IS_OUTER_JOIN(join_type) && OB_FAIL(append(tmp_join_info.where_conditions_, join_info.where_conditions_))) { + LOG_WARN("failed to append", K(ret)); + } else if (OB_FAIL(calc_join_output_rows(plan, + left_tree.get_tables(), + right_tree.get_tables(), + left_output_rows, + right_output_rows, + tmp_join_info, + tmp_rows, + left_ambient_card_sel, + equal_sets))) { + LOG_WARN("failed to calc join output rows", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < ambient_card_sels.count(); i ++) { + if (!left_ids.has_member(i) || !join_info.table_set_.has_member(i)) { + // do nothing + } else if (OB_FAIL(calc_table_ambient_card(plan, + i, + left_tree, + right_tree, + left_output_rows, + right_output_rows, + tmp_join_info, + equal_sets, + ambient_card, + ambient_card_sels.at(i), + join_type))) { + LOG_WARN("failed to calc table ambient card", K(ret)); + } + } + } + } + + // calculate selectivity for right ambient cardinality + if (OB_SUCC(ret)) { + tmp_join_info.join_type_ = IS_ANTI_JOIN(join_type) ? RIGHT_ANTI_JOIN : RIGHT_SEMI_JOIN; + tmp_join_info.where_conditions_.reuse(); + if (CONNECT_BY_JOIN == join_type) { + // todo + } else if (IS_LEFT_SEMI_ANTI_JOIN(join_type)) { + right_ambient_card_sel = 0.0; + for (int64_t i = 0; i < ambient_card_sels.count(); i ++) { + if (right_ids.has_member(i)) { + ambient_card_sels.at(i) = 0.0; + } + } + } else if (RIGHT_OUTER_JOIN == join_type || FULL_OUTER_JOIN == join_type) { + right_ambient_card_sel = 1.0; + for (int64_t i = 0; i < ambient_card_sels.count(); i ++) { + if (right_ids.has_member(i)) { + ambient_card_sels.at(i) = 1.0; + } + } + } else if (LEFT_OUTER_JOIN == join_type && OB_FAIL(append(tmp_join_info.where_conditions_, join_info.on_conditions_))) { + LOG_WARN("failed to assign on conditions", K(ret)); + } else if (!IS_OUTER_JOIN(join_type) && OB_FAIL(append(tmp_join_info.where_conditions_, join_info.where_conditions_))) { + LOG_WARN("failed to append", K(ret)); + } else if (OB_FAIL(calc_join_output_rows(plan, + left_tree.get_tables(), + right_tree.get_tables(), + left_output_rows, + right_output_rows, + tmp_join_info, + tmp_rows, + right_ambient_card_sel, + equal_sets))) { + LOG_WARN("failed to calc join output rows", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < ambient_card_sels.count(); i ++) { + if (!right_ids.has_member(i) || !join_info.table_set_.has_member(i)) { + // do nothing + } else if (OB_FAIL(calc_table_ambient_card(plan, + i, + left_tree, + right_tree, + left_output_rows, + right_output_rows, + tmp_join_info, + equal_sets, + ambient_card, + ambient_card_sels.at(i), + join_type))) { + LOG_WARN("failed to calc table ambient card", K(ret)); + } + } + } + } + + if (OB_SUCC(ret) && IS_OUTER_JOIN(join_type) && !join_info.where_conditions_.empty()) { + plan->get_selectivity_ctx().init_join_ctx(join_type, + &left_tree.get_tables(), + &right_tree.get_tables(), + left_output_rows, + right_output_rows, + &equal_sets); + if (OB_FAIL(ObOptSelectivity::calculate_join_selectivity( + plan->get_update_table_metas(), + plan->get_selectivity_ctx(), + join_info.where_conditions_, where_sel_for_oj, + plan->get_predicate_selectivities()))) { + LOG_WARN("failed to calc filter selectivities", K(join_info.where_conditions_), K(ret)); + } + } + + OPT_TRACE("outer join filter selectivity :", where_sel_for_oj); + OPT_TRACE("selectivity of the left side :", left_ambient_card_sel); + OPT_TRACE("selectivity of the right side :", right_ambient_card_sel); + OPT_TRACE("selectivity of each table :", ambient_card_sels); + LOG_TRACE("succeed to calc selectivity of all ambient cardinality", K(ret), K(left_ids), K(right_ids), + K(ambient_card), K(left_ambient_card_sel), K(right_ambient_card_sel), K(ambient_card_sels), K(where_sel_for_oj)); + + /** + * For (t1, t2) left join (t3, t4) on t1.c1 = t3.c1 and t2.c1 = t4.c1 and t1.c2 + t2.c2 < t3.c2 where t1.c3 <=> t3.c3 + * step 1 : table t1 is filtered by the direct join condition, `t1.c1 = t3.c1` + * step 2 : table t1 is filtered by the indirect join condition, `t2.c1 = t4.c1 and t1.c2 + t2.c2 < t3.c2` + * step 3 : table t1 is fiterred by the where condition, `t1.c3 <=> t3.c3` + */ + for (int64_t i = 0; OB_SUCC(ret) && i < ambient_card.count(); i ++) { + double step1_rows = 0; + double step2_rows = 0; + if (left_ids.has_member(i)) { + ambient_card.at(i) *= ambient_card_sels.at(i); + step1_rows = left_output_rows * ambient_card_sels.at(i); + step2_rows = left_output_rows * left_ambient_card_sel; + } else if (right_ids.has_member(i)) { + ambient_card.at(i) *= ambient_card_sels.at(i); + step1_rows = right_output_rows * ambient_card_sels.at(i); + step2_rows = right_output_rows * right_ambient_card_sel; + } + if (ambient_card.at(i) >= 0) { + if (step2_rows < step1_rows) { + ambient_card.at(i) = ObOptSelectivity::scale_distinct(step2_rows, step1_rows, ambient_card.at(i)); + } + if (std::fabs(where_sel_for_oj) <= OB_DOUBLE_EPSINON) { + ambient_card.at(i) = 0; + } else { + ambient_card.at(i) = ObOptSelectivity::scale_distinct(join_output_rows, join_output_rows / where_sel_for_oj, ambient_card.at(i)); + } + ambient_card.at(i) = std::min(join_output_rows, ambient_card.at(i)); + } + } + if (OB_SUCC(ret)) { + plan->get_selectivity_ctx().set_assumption_type(UNKNOWN_JOIN); + } + return ret; +} + +/** + * (t1 join t2 on 1 = 1) join t3 on t1.c1 = t3.c1 and t2.c2 = t3.c2 + * In this case, the ambient cardinality selectivity of (t1, t2) is invalid for single table t1 or t2. + * It is too small while the ambient cardinality of t1 and t2 might be lossless. + * So, we calculate the ambient cardinality for each table. +*/ +int ObJoinOrder::calc_table_ambient_card(ObLogPlan *plan, + uint64_t table_index, + const ObJoinOrder &left_tree, + const ObJoinOrder &right_tree, + double input_rows, + double right_rows, + const JoinInfo &join_info, + EqualSets &equal_sets, + const ObIArray &ambient_card, + double &ambient_card_sel, + const ObJoinType assumption_type) +{ + int ret = OB_SUCCESS; + JoinInfo table_join_info; + table_join_info.join_type_ = join_info.join_type_; + ObRelIds table_id; + ObRelIds exclusion_ids; + double tmp_rows = 1.0; + ambient_card_sel = 1.0; + const ObRelIds &left_ids = left_tree.get_tables(); + const ObRelIds &right_ids = right_tree.get_tables(); + bool in_left = left_ids.has_member(table_index); + bool in_right = right_ids.has_member(table_index); + if (OB_ISNULL(plan) || + OB_UNLIKELY(!IS_SEMI_ANTI_JOIN(join_info.join_type_)) || + OB_UNLIKELY(!in_left && !in_right)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unepxected param", K(plan), K(join_info), K(left_ids), K(right_ids), K(table_index)); + } else if (OB_FAIL(table_id.add_member(table_index))) { + LOG_WARN("failed to add member", K(ret)); + } else if (OB_FAIL(exclusion_ids.except(in_left ? left_ids : right_ids, table_id))) { + LOG_WARN("failed to except", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < join_info.where_conditions_.count(); i ++) { + bool is_direct_condition = false; + if (OB_FAIL(check_direct_join_condition(join_info.where_conditions_.at(i), + equal_sets, + table_id, + exclusion_ids, + is_direct_condition))) { + LOG_WARN("failed to check join condition", K(ret), K(left_ids), K(right_ids)); + } else if (!is_direct_condition) { + // do nothing + } else if (OB_FAIL(table_join_info.where_conditions_.push_back(join_info.where_conditions_.at(i)))) { + LOG_WARN("failed to push back expr", K(ret)); + } + } + if (OB_FAIL(ret)) { + } else if (table_join_info.where_conditions_.empty()) { + ambient_card_sel = 1.0; + } else if (OB_FAIL(calc_join_output_rows(plan, + left_tree.get_tables(), + right_tree.get_tables(), + input_rows, + right_rows, + table_join_info, + tmp_rows, + ambient_card_sel, + equal_sets))) { + LOG_WARN("failed to calc join output rows", K(ret)); + } + return ret; +} + +/** + * For `(t1 join t2 on t1.c1 = t2.c1) join t3 on t1.c1 = t3.c1 and t1.c2 = t3.c2`, + * `t1.c1 = t3.c1` is a direct join condition for `t1`, `t2` and `t3`, + * `t1.c2 = t3.c2` is a direct join condition only for `t1` and `t3`. +*/ +int ObJoinOrder::check_direct_join_condition(ObRawExpr *expr, + const EqualSets &equal_sets, + const ObRelIds &table_id, + const ObRelIds &exclusion_ids, + bool &is_valid) +{ + int ret = OB_SUCCESS; + ObSEArray col_exprs; + is_valid = false; + if (OB_ISNULL(expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected join condition", K(ret), K(expr), K(table_id)); + } else if (expr->get_relation_ids().is_superset(table_id) && + !expr->get_relation_ids().overlap(exclusion_ids)) { + is_valid = true; + } else if (OB_FAIL(ObRawExprUtils::extract_column_exprs(expr, exclusion_ids, col_exprs))) { + LOG_WARN("failed to extract column exprs", K(ret)); + } else if (col_exprs.count() == 1) { + int64_t eq_set_idx = OB_INVALID_ID; + if (OB_FAIL(ObOptimizerUtil::find_expr_in_equal_sets(equal_sets, + col_exprs.at(0), + eq_set_idx))) { + LOG_WARN("failed to find expr", K(ret)); + } else if (eq_set_idx != OB_INVALID_ID) { + const EqualSet& equal_set = *equal_sets.at(eq_set_idx); + for (int64_t j = 0; OB_SUCC(ret) && !is_valid && j < equal_set.count(); j++) { + ObRawExpr *equal_expr = equal_set.at(j); + if (OB_ISNULL(equal_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (equal_expr->get_relation_ids().equal(table_id)) { + is_valid = true; + } + } + } + } + return ret; +} + int ObJoinOrder::calc_join_output_rows(ObLogPlan *plan, const ObRelIds &left_ids, const ObRelIds &right_ids, @@ -13229,7 +13788,7 @@ int ObJoinOrder::calc_join_output_rows(ObLogPlan *plan, const JoinInfo &join_info, double &new_rows, double &selectivity, - EqualSets &equal_sets) + const EqualSets &equal_sets) { int ret = OB_SUCCESS; const ObJoinType join_type = join_info.join_type_; @@ -13243,11 +13802,11 @@ int ObJoinOrder::calc_join_output_rows(ObLogPlan *plan, right_output_rows, &equal_sets))) { } else if (INNER_JOIN == join_type) { - if (OB_FAIL(ObOptSelectivity::calculate_selectivity(plan->get_update_table_metas(), - plan->get_selectivity_ctx(), - join_info.where_conditions_, - selectivity, - plan->get_predicate_selectivities()))) { + if (OB_FAIL(ObOptSelectivity::calculate_join_selectivity(plan->get_update_table_metas(), + plan->get_selectivity_ctx(), + join_info.where_conditions_, + selectivity, + plan->get_predicate_selectivities()))) { LOG_WARN("Failed to calc filter selectivities", K(ret)); } else { new_rows = left_output_rows * right_output_rows * selectivity; @@ -13267,7 +13826,7 @@ int ObJoinOrder::calc_join_output_rows(ObLogPlan *plan, left_has_is_null, right_has_is_null))) { LOG_WARN("failed to check and remove is null qual", K(ret)); - } else if (OB_FAIL(ObOptSelectivity::calculate_selectivity( + } else if (OB_FAIL(ObOptSelectivity::calculate_join_selectivity( plan->get_update_table_metas(), plan->get_selectivity_ctx(), normal_quals, oj_qual_sel, @@ -13326,7 +13885,7 @@ int ObJoinOrder::calc_join_output_rows(ObLogPlan *plan, // selectivity. So refine selectivity as output_row / (left_row * right_row) selectivity = new_rows / (left_output_rows * right_output_rows); } - } else if (OB_FAIL(ObOptSelectivity::calculate_selectivity( + } else if (OB_FAIL(ObOptSelectivity::calculate_join_selectivity( plan->get_update_table_metas(), plan->get_selectivity_ctx(), join_info.on_conditions_, oj_filter_sel, @@ -13356,11 +13915,11 @@ int ObJoinOrder::calc_join_output_rows(ObLogPlan *plan, } } else if (IS_SEMI_ANTI_JOIN(join_type)) { // semi/anti join is treated as table filter, use origin table metas - if (OB_FAIL(ObOptSelectivity::calculate_selectivity(plan->get_update_table_metas(), - plan->get_selectivity_ctx(), - join_info.where_conditions_, - selectivity, - plan->get_predicate_selectivities()))) { + if (OB_FAIL(ObOptSelectivity::calculate_join_selectivity(plan->get_update_table_metas(), + plan->get_selectivity_ctx(), + join_info.where_conditions_, + selectivity, + plan->get_predicate_selectivities()))) { LOG_WARN("Failed to calc filter selectivities", K(ret)); } else { double outer_rows = IS_LEFT_SEMI_ANTI_JOIN(join_type)? @@ -13371,7 +13930,8 @@ int ObJoinOrder::calc_join_output_rows(ObLogPlan *plan, //如果有anti join的笛卡尔积，要么左表全输出、要么不输出任何行， //取决于右表是否有输出，但是，我们不应该直接估行为0， //一个简单的策略是，如果右表估行为0，那么应该输出左表的行数，而不是0 - new_rows = outer_rows - outer_rows * selectivity; + selectivity = 1 - selectivity; + new_rows = outer_rows * selectivity; if (LEFT_ANTI_JOIN == join_type && std::fabs(right_output_rows) < OB_DOUBLE_EPSINON) { new_rows = left_output_rows; @@ -13384,14 +13944,14 @@ int ObJoinOrder::calc_join_output_rows(ObLogPlan *plan, } else if (CONNECT_BY_JOIN == join_type) { double join_qual_sel = 1.0; double join_filter_sel = 1.0; - if (OB_FAIL(ObOptSelectivity::calculate_selectivity( + if (OB_FAIL(ObOptSelectivity::calculate_join_selectivity( plan->get_update_table_metas(), plan->get_selectivity_ctx(), join_info.where_conditions_, join_qual_sel, plan->get_predicate_selectivities()))) { LOG_WARN("failed to calc filter selectivities", K(join_info.where_conditions_), K(ret)); - } else if (OB_FAIL(ObOptSelectivity::calculate_selectivity( + } else if (OB_FAIL(ObOptSelectivity::calculate_join_selectivity( plan->get_update_table_metas(), plan->get_selectivity_ctx(), join_info.on_conditions_, @@ -13410,7 +13970,7 @@ int ObJoinOrder::calc_join_output_rows(ObLogPlan *plan, selectivity = connect_by_selectivity; } } - plan->get_selectivity_ctx().clear_equal_sets(); + plan->get_selectivity_ctx().clear(); LOG_TRACE("estimate join size and width", K(left_output_rows), K(right_output_rows), K(selectivity), K(new_rows)); return ret; @@ -14271,6 +14831,11 @@ int ObJoinOrder::generate_inner_subquery_paths(const ObDMLStmt &parent_stmt, LOG_WARN("failed to rename pushdown filter", K(ret)); } else if (OB_FAIL(append(helper.filters_, candi_nonpushdown_quals))) { LOG_WARN("failed to append", K(ret)); + } else if (OB_FAIL(ObOptimizerUtil::get_onetime_exprs(helper.pushdown_filters_, + helper.exec_params_))) { + LOG_WARN("failed to get onetime exprs", K(ret)); + } else if (OB_FAIL(append(helper.exec_params_, nl_params))) { + LOG_WARN("failed to append", K(ret)); } else if (OB_FAIL(generate_subquery_paths(helper))) { LOG_WARN("failed to generate subquery path", K(ret)); } else if (OB_FAIL(check_and_fill_inner_path_info(helper, diff --git a/src/sql/optimizer/ob_join_order.h b/src/sql/optimizer/ob_join_order.h index 4a69e5885..846e2c27e 100644 --- a/src/sql/optimizer/ob_join_order.h +++ b/src/sql/optimizer/ob_join_order.h @@ -172,7 +172,10 @@ namespace sql pushdown_filter_table_(), in_current_dfo_(true), skip_subpart_(false), - use_column_store_(false) {} + use_column_store_(false), + is_right_contain_pk_(false), + is_right_union_pk_(false), + right_origin_rows_(1.0) {} TO_STRING_KV( K_(lexprs), @@ -191,7 +194,10 @@ namespace sql K_(force_part_filter), K_(in_current_dfo), K_(skip_subpart), - K_(use_column_store) + K_(use_column_store), + K_(is_right_contain_pk), + K_(is_right_union_pk), + K_(right_origin_rows) ); common::ObSEArray lexprs_; @@ -215,6 +221,10 @@ namespace sql // If the table is a 1-level partition, this value is false. bool skip_subpart_; bool use_column_store_; + bool is_right_contain_pk_; + bool is_right_union_pk_; + double right_origin_rows_; + }; struct EstimateCostInfo { @@ -428,6 +438,8 @@ struct EstimateCostInfo { } int compute_path_property_from_log_op(); int set_parallel_and_server_info_for_match_all(); + ObIArray &get_ambient_card() { return ambient_card_; } + const ObIArray &get_ambient_card() const { return ambient_card_; } TO_STRING_KV(K_(is_local_order), K_(ordering), K_(interesting_order_info), @@ -442,7 +454,8 @@ struct EstimateCostInfo { K_(phy_plan_type), K_(location_type), K_(is_pipelined_path), - K_(is_nl_style_pipelined_path)); + K_(is_nl_style_pipelined_path), + K_(ambient_card)); public: /** * 表示当前join order最终的父join order节点 @@ -483,6 +496,7 @@ struct EstimateCostInfo { common::ObSEArray server_list_; bool is_pipelined_path_; bool is_nl_style_pipelined_path_; + common::ObSEArray ambient_card_; private: DISALLOW_COPY_AND_ASSIGN(Path); @@ -730,7 +744,10 @@ struct EstimateCostInfo { EstimateCostInfo &right_param, bool re_est_for_op); int try_set_batch_nlj_for_right_access_path(bool enable); - int re_estimate_rows(double left_output_rows, double right_output_rows, double &row_count); + int re_estimate_rows(ObIArray &pushdown_join_filter_infos, + double left_output_rows, + double right_output_rows, + double &row_count); int cost_nest_loop_join(int64_t join_parallel, double left_output_rows, double left_cost, @@ -1214,6 +1231,8 @@ struct NullAwareAntiJoinInfo { ObSEArray expr_constraints_; ObBaseTableEstMethod est_method_; + // include nl params and onetime params + ObSEArray exec_params_; }; struct DeducedExprInfo { @@ -1698,6 +1717,8 @@ struct NullAwareAntiJoinInfo { */ int init_base_join_order(const TableItem *table_item); + int init_ambient_card(); + int generate_base_paths(); int generate_normal_base_table_paths(); @@ -1984,6 +2005,11 @@ struct NullAwareAntiJoinInfo { JoinFilterInfo& info, double &join_filter_selectivity); + int calc_join_filter_sel_for_pk_join_fk(const Path& left_path, + JoinFilterInfo& info, + double &join_filter_selectivity, + bool &is_valid); + int find_shuffle_join_filter(const Path& path, bool &find); int check_partition_join_filter_valid(const DistAlgo join_dist_algo, @@ -2120,6 +2146,9 @@ struct NullAwareAntiJoinInfo { InnerPathInfos &get_inner_path_infos() { return inner_path_infos_; } const InnerPathInfos &get_inner_path_infos() const { return inner_path_infos_; } + ObIArray &get_ambient_card() { return ambient_card_; } + const ObIArray &get_ambient_card() const { return ambient_card_; } + int64_t get_name(char *buf, const int64_t buf_len) { int64_t pos = 0; @@ -2359,7 +2388,35 @@ struct NullAwareAntiJoinInfo { const JoinInfo &join_info, double &new_rows, double &selectivity, - EqualSets &equal_sets); + const EqualSets &equal_sets); + static int merge_ambient_card(const ObIArray &left_ambient_card, + const ObIArray &right_ambient_card, + ObIArray &cur_ambient_card); + static int scale_ambient_card(const double origin_rows, + const double new_rows, + const ObIArray &origin_ambient_card, + ObIArray &ambient_card); + static int calc_join_ambient_card(ObLogPlan *plan, + const ObJoinOrder &left_tree, + const ObJoinOrder &right_tree, + const double join_output_rows, + const JoinInfo &join_info, + EqualSets &equal_sets, + ObIArray &ambient_card); + static int calc_table_ambient_card(ObLogPlan *plan, + uint64_t table_index, + const ObJoinOrder &left_ids, + const ObJoinOrder &right_ids, + double input_rows, + double right_rows, + const JoinInfo &join_info, + EqualSets &equal_sets, + const ObIArray &ambient_card, + double &new_ambient_card, + const ObJoinType assumption_type); + int revise_cardinality(const ObJoinOrder *left_tree, + const ObJoinOrder *right_tree, + const JoinInfo &join_info); inline void set_cnt_rownum(const bool cnt_rownum) { cnt_rownum_ = cnt_rownum; } inline bool get_cnt_rownum() const { return cnt_rownum_; } inline void increase_total_path_num() { total_path_num_ ++; } @@ -2377,6 +2434,12 @@ struct NullAwareAntiJoinInfo { bool &left_has_is_null, bool &right_has_is_null); + static int check_direct_join_condition(ObRawExpr *expr, + const EqualSets &equal_sets, + const ObRelIds &table_id, + const ObRelIds &exclusion_ids, + bool &is_valid); + int get_cached_inner_paths(const ObIArray &join_conditions, ObJoinOrder &left_tree, ObJoinOrder &right_tree, @@ -2557,6 +2620,7 @@ struct NullAwareAntiJoinInfo { common::ObSEArray deduced_exprs_info_; bool cnt_rownum_; uint64_t total_path_num_; + common::ObSEArray ambient_card_; private: DISALLOW_COPY_AND_ASSIGN(ObJoinOrder); }; diff --git a/src/sql/optimizer/ob_log_count.cpp b/src/sql/optimizer/ob_log_count.cpp index a37d8c229..74336d77f 100644 --- a/src/sql/optimizer/ob_log_count.cpp +++ b/src/sql/optimizer/ob_log_count.cpp @@ -36,8 +36,7 @@ int ObLogCount::est_cost() if (OB_ISNULL(get_plan()) || OB_ISNULL(child = get_child(ObLogicalOperator::first_child))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(get_plan()), K(child), K(ret)); - } else if (OB_FALSE_IT(get_plan()->get_selectivity_ctx().init_op_ctx( - &child->get_output_equal_sets(), child->get_card()))) { + } else if (OB_FALSE_IT(get_plan()->get_selectivity_ctx().init_op_ctx(child))) { } else if (OB_FAIL(ObOptSelectivity::calculate_selectivity(get_plan()->get_update_table_metas(), get_plan()->get_selectivity_ctx(), get_filter_exprs(), @@ -82,8 +81,7 @@ int ObLogCount::do_re_est_cost(EstimateCostInfo ¶m, double &card, double &op OB_ISNULL(child = get_child(ObLogicalOperator::first_child))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(get_plan()), K(child), K(ret)); - } else if (OB_FALSE_IT(get_plan()->get_selectivity_ctx().init_op_ctx( - &child->get_output_equal_sets(), child->get_card()))) { + } else if (OB_FALSE_IT(get_plan()->get_selectivity_ctx().init_op_ctx(child))) { } else if (OB_FAIL(ObOptSelectivity::calculate_selectivity(get_plan()->get_basic_table_metas(), get_plan()->get_selectivity_ctx(), get_filter_exprs(), diff --git a/src/sql/optimizer/ob_log_distinct.cpp b/src/sql/optimizer/ob_log_distinct.cpp index 9910816d1..fbb2c9310 100644 --- a/src/sql/optimizer/ob_log_distinct.cpp +++ b/src/sql/optimizer/ob_log_distinct.cpp @@ -130,18 +130,19 @@ int ObLogDistinct::est_cost() int ret = OB_SUCCESS; double distinct_cost = 0.0; ObLogicalOperator *child = NULL; + double child_ndv = total_ndv_; if (OB_ISNULL(child = get_child(ObLogicalOperator::first_child))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(child), K(ret)); - } else if (OB_UNLIKELY(total_ndv_ < 0)) { + } else if (OB_UNLIKELY(child_ndv < 0)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected total ndv", K(total_ndv_), K(ret)); - } else if (OB_FAIL(inner_est_cost(get_parallel(), child->get_card(), total_ndv_, distinct_cost))) { + LOG_WARN("get unexpected total ndv", K(child_ndv), K(ret)); + } else if (OB_FAIL(inner_est_cost(get_parallel(), child->get_card(), child_ndv, distinct_cost))) { LOG_WARN("failed to est distinct cost", K(ret)); } else { set_op_cost(distinct_cost); set_cost(child->get_cost() + distinct_cost); - set_card(total_ndv_); + set_card(child_ndv); } return ret; } @@ -200,7 +201,7 @@ int ObLogDistinct::do_re_est_cost(EstimateCostInfo ¶m, double &card, double return ret; } -int ObLogDistinct::inner_est_cost(const int64_t parallel, double child_card, double child_ndv, double &op_cost) +int ObLogDistinct::inner_est_cost(const int64_t parallel, double child_card, double &child_ndv, double &op_cost) { int ret = OB_SUCCESS; double per_dop_card = 0.0; @@ -239,6 +240,11 @@ int ObLogDistinct::inner_est_cost(const int64_t parallel, double child_card, dou distinct_exprs_, opt_ctx); } + + if (opt_ctx.get_query_ctx()->check_opt_compat_version(COMPAT_VERSION_4_2_4, COMPAT_VERSION_4_3_0, + COMPAT_VERSION_4_3_3)) { + child_ndv = std::min(child_card, per_dop_ndv * parallel); + } } return ret; } diff --git a/src/sql/optimizer/ob_log_distinct.h b/src/sql/optimizer/ob_log_distinct.h index 1760e77ee..eed1ccaf5 100644 --- a/src/sql/optimizer/ob_log_distinct.h +++ b/src/sql/optimizer/ob_log_distinct.h @@ -54,7 +54,7 @@ public: virtual int est_cost() override; virtual int est_width() override; virtual int do_re_est_cost(EstimateCostInfo ¶m, double &card, double &op_cost, double &cost) override; - int inner_est_cost(const int64_t parallel, double child_card, double child_ndv, double &op_cost); + int inner_est_cost(const int64_t parallel, double child_card, double &child_ndv, double &op_cost); virtual bool is_block_op() const override { return false; } virtual int compute_fd_item_set() override; virtual int allocate_granule_post(AllocGIContext &ctx) override; diff --git a/src/sql/optimizer/ob_log_join.cpp b/src/sql/optimizer/ob_log_join.cpp index befd9a3bb..c07996916 100644 --- a/src/sql/optimizer/ob_log_join.cpp +++ b/src/sql/optimizer/ob_log_join.cpp @@ -394,6 +394,16 @@ int ObLogJoin::inner_replace_op_exprs(ObRawExprReplacer &replacer) return ret; } +int ObLogJoin::est_ambient_card() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ambient_card_.assign(join_path_->parent_->get_ambient_card()))) { + LOG_WARN("failed to assign ambient cards", K(ret)); + } + // do nothing + return ret; +} + int ObLogJoin::do_re_est_cost(EstimateCostInfo ¶m, double &card, double &op_cost, double &cost) { int ret = OB_SUCCESS; @@ -430,9 +440,10 @@ int ObLogJoin::do_re_est_cost(EstimateCostInfo ¶m, double &card, double &op_ LOG_WARN("failed to re estimate cost", K(ret)); } else if (OB_FAIL(join_path_->try_set_batch_nlj_for_right_access_path(false))) { LOG_WARN("failed to try set batch nlj for right access path", K(ret)); - } else if (OB_FAIL(join_path_->re_estimate_rows(left_output_rows, - right_output_rows, - card))) { + } else if (OB_FAIL(join_path_->re_estimate_rows(param.join_filter_infos_, + left_output_rows, + right_output_rows, + card))) { LOG_WARN("failed to re estimate rows", K(ret)); } else if (NESTED_LOOP_JOIN == join_algo_) { if (OB_FAIL(join_path_->cost_nest_loop_join(parallel, diff --git a/src/sql/optimizer/ob_log_join.h b/src/sql/optimizer/ob_log_join.h index 65e156fbf..b7730d0a3 100644 --- a/src/sql/optimizer/ob_log_join.h +++ b/src/sql/optimizer/ob_log_join.h @@ -119,6 +119,7 @@ namespace sql const int64_t buf_len, int64_t &pos); virtual int do_re_est_cost(EstimateCostInfo ¶m, double &card, double &op_cost, double &cost) override; + virtual int est_ambient_card() override; /* * IN right_child_sharding_info the join's right child sharding info * IN right_keys the right join equal condition diff --git a/src/sql/optimizer/ob_log_plan.cpp b/src/sql/optimizer/ob_log_plan.cpp index cd17f38b9..fe9fa6972 100644 --- a/src/sql/optimizer/ob_log_plan.cpp +++ b/src/sql/optimizer/ob_log_plan.cpp @@ -2072,6 +2072,10 @@ int ObLogPlan::inner_generate_join_order(ObIArray &join_rels, OPT_TRACE_TITLE("Now", left_tree, "join", right_tree, join_info); if (OB_FAIL(ret)) { //do nothing + } else if (OB_FAIL(join_tree->revise_cardinality(left_tree, + right_tree, + join_info))) { + LOG_WARN("failed to revise ambient card", K(ret)); } else if (OB_FAIL(join_tree->generate_join_paths(*left_tree, *right_tree, join_info, @@ -2638,8 +2642,16 @@ int ObLogPlan::generate_subplan_for_query_ref(ObQueryRefRawExpr *query_ref, LOG_WARN("failed to create plan", K(ret), K(opt_ctx.get_query_ctx()->get_sql_stmt())); } else if (FALSE_IT(logical_plan->set_nonrecursive_plan_for_fake_cte(get_nonrecursive_plan_for_fake_cte()))) { // never reach + } else if (OB_FAIL(logical_plan->add_exec_params_meta(query_ref->get_exec_params(), + get_basic_table_metas(), + get_selectivity_ctx()))) { + LOG_WARN("failed to prepare exec param meta", K(ret)); } else if (OB_FAIL(SMART_CALL(static_cast(logical_plan)->generate_raw_plan()))) { LOG_WARN("failed to optimize sub-select", K(ret)); + } else if (OB_FAIL(add_query_ref_meta(query_ref, + logical_plan->get_update_table_metas(), + logical_plan->get_selectivity_ctx()))) { + LOG_WARN("failed to add expr meta", K(ret)); } else { SubPlanInfo *info = static_cast(get_allocator().alloc(sizeof(SubPlanInfo))); bool has_ref_assign_user_var = false; @@ -2677,6 +2689,68 @@ int ObLogPlan::generate_subplan_for_query_ref(ObQueryRefRawExpr *query_ref, return ret; } +int ObLogPlan::add_exec_params_meta(ObIArray &exec_params, + const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < exec_params.count(); i ++) { + ObExecParamRawExpr *exec_param = exec_params.at(i); + double avg_len = 0; + OptDynamicExprMeta dynamic_expr_meta; + if (OB_ISNULL(exec_param)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null param", K(ret), KPC(exec_param)); + } else if (OB_FAIL(ObOptSelectivity::calculate_expr_avg_len(table_metas, + ctx, + exec_param, + avg_len))) { + LOG_WARN("failed to calc expr avg len", K(ret), KPC(exec_param)); + } else { + dynamic_expr_meta.set_expr(exec_param); + dynamic_expr_meta.set_avg_len(avg_len); + } + if (FAILEDx(get_basic_table_metas().add_dynamic_expr_meta(dynamic_expr_meta))) { + LOG_WARN("failed to add expr meta", K(ret)); + } else if (OB_FAIL(get_update_table_metas().add_dynamic_expr_meta(dynamic_expr_meta))) { + LOG_WARN("failed to add expr meta", K(ret)); + } + } + return ret; +} + +int ObLogPlan::add_query_ref_meta(ObQueryRefRawExpr *expr, + const OptTableMetas &child_table_metas, + const OptSelectivityCtx &child_ctx) +{ + int ret = OB_SUCCESS; + ObRawExpr *ref_expr = NULL; + double avg_len = 0; + OptDynamicExprMeta dynamic_expr_meta; + ObSelectStmt *stmt = NULL; + if (OB_ISNULL(expr) || OB_ISNULL(stmt = expr->get_ref_stmt())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null param", K(ret), KPC(expr)); + } else if (!expr->is_scalar()) { + // do nothing + } else if (OB_UNLIKELY(stmt->get_select_item_size() != 1)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected query ref", K(ret), KPC(stmt)); + } else if (OB_FAIL(ObOptSelectivity::calculate_expr_avg_len( + child_table_metas, child_ctx, stmt->get_select_item(0).expr_, avg_len))) { + LOG_WARN("failed to calc expr avg len", K(ret), KPC(expr)); + } else { + dynamic_expr_meta.set_expr(expr); + dynamic_expr_meta.set_avg_len(avg_len); + if (OB_FAIL(get_basic_table_metas().add_dynamic_expr_meta(dynamic_expr_meta))) { + LOG_WARN("failed to add expr meta", K(ret)); + } else if (OB_FAIL(get_update_table_metas().add_dynamic_expr_meta(dynamic_expr_meta))) { + LOG_WARN("failed to add expr meta", K(ret)); + } + } + return ret; +} + //在已有sub_plan_infos中查找expr对应的subplan int ObLogPlan::get_subplan(const ObRawExpr *expr, SubPlanInfo *&info) { @@ -5556,7 +5630,7 @@ int ObLogPlan::init_groupby_helper(const ObIArray &group_exprs, } if (OB_SUCC(ret)) { - get_selectivity_ctx().init_op_ctx(&best_plan->get_output_equal_sets(), best_plan->get_card()); + get_selectivity_ctx().init_op_ctx(best_plan); if (group_rollup_exprs.empty()) { groupby_helper.group_ndv_ = 1.0; } else if (OB_FAIL(ObOptSelectivity::calculate_distinct(get_update_table_metas(), @@ -5582,7 +5656,7 @@ int ObLogPlan::calculate_group_distinct_ndv(const ObIArray &groupby_ ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret)); } else { - get_selectivity_ctx().init_op_ctx(&best_plan->get_output_equal_sets(), best_plan->get_card()); + get_selectivity_ctx().init_op_ctx(best_plan); } for (int64_t i = 0; OB_SUCC(ret) && i < groupby_helper.distinct_aggr_batch_.count(); ++i) { ObSEArray group_distinct_exprs; @@ -5647,7 +5721,7 @@ int ObLogPlan::init_distinct_helper(const ObIArray &distinct_exprs, } if (OB_SUCC(ret)) { - get_selectivity_ctx().init_op_ctx(&best_plan->get_output_equal_sets(), best_plan->get_card()); + get_selectivity_ctx().init_op_ctx(best_plan); if (distinct_exprs.empty()) { distinct_helper.group_ndv_ = 1.0; } else if (get_stmt()->is_set_stmt()) { @@ -8131,7 +8205,7 @@ int ObLogPlan::generate_subplan_filter_info(const ObIArray &subquer SubPlanInfo *info = NULL; if (OB_FAIL(get_subplan(candi_query_refs.at(i), info))) { LOG_WARN("failed to get subplan", K(ret)); - } else if (NULL != info && !for_on_condition) { + } else if (NULL != info && !for_on_condition && info->allocated_) { // do nothing } else if (OB_FAIL(append(exec_params, candi_query_refs.at(i)->get_exec_params()))) { LOG_WARN("failed to append exec params", K(ret)); @@ -8144,6 +8218,7 @@ int ObLogPlan::generate_subplan_filter_info(const ObIArray &subquer LOG_WARN("failed to push back query ref expr", K(ret)); } else { ++ idx; + info->allocated_ = true; for_cursor_expr = for_cursor_expr || candi_query_refs.at(i)->is_cursor(); if (info->init_plan_) { if (ObOptimizerUtil::find_item(onetime_query_refs, candi_query_refs.at(i))) { @@ -8799,14 +8874,14 @@ int ObLogPlan::candi_allocate_filter(const ObIArray &filter_exprs) LOG_WARN("get unexpected null", K(ret)); } else if (OB_FAIL(best_plan->get_input_equal_sets(equal_sets))) { LOG_WARN("failed to get input equal sets", K(ret)); - } else if (OB_FALSE_IT(get_selectivity_ctx().init_op_ctx(&equal_sets, best_plan->get_card()))) { + } else if (OB_FALSE_IT(get_selectivity_ctx().init_op_ctx(best_plan))) { } else if (OB_FAIL(ObOptSelectivity::calculate_selectivity(get_update_table_metas(), get_selectivity_ctx(), filter_exprs, sel, get_predicate_selectivities()))) { LOG_WARN("failed to calc selectivity", K(ret)); - } else if (OB_FALSE_IT(get_selectivity_ctx().init_op_ctx(NULL, -1.0))) { + } else if (OB_FALSE_IT(get_selectivity_ctx().clear())) { } else { for (int64_t i = 0; OB_SUCC(ret) && i < candidates_.candidate_plans_.count(); i++) { ObLogicalOperator *top = NULL; @@ -9056,6 +9131,7 @@ int ObLogPlan::init_onetime_subquery_info() bool dummy_shared = false; ObRawExpr *expr = exprs.at(i); ObSEArray onetime_list; + ObSEArray queryref_list; if (OB_ISNULL(expr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("expr is null", K(ret)); @@ -9069,6 +9145,24 @@ int ObLogPlan::init_onetime_subquery_info() // do nothing } else if (OB_FAIL(create_onetime_param(expr, onetime_list))) { LOG_WARN("failed to create onetime param expr", K(ret)); + } else if (OB_FAIL(ObTransformUtils::extract_query_ref_expr(onetime_list, + queryref_list, + false))) { + LOG_WARN("failed to extract query ref exprs", K(ret)); + } + for (int64_t j = 0; OB_SUCC(ret) && j < queryref_list.count(); ++j) { + SubPlanInfo *info = NULL; + ObQueryRefRawExpr *onetime_queryref_expr = queryref_list.at(j); + if (OB_ISNULL(onetime_queryref_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected onetime expr", K(ret), KPC(onetime_queryref_expr)); + } else if (OB_FAIL(get_subplan(onetime_queryref_expr, info))) { + LOG_WARN("failed to get subplan", K(ret)); + } else if (NULL != info) { + // do nothing + } else if (OB_FAIL(generate_subplan_for_query_ref(onetime_queryref_expr, info))) { + LOG_WARN("failed to generate subplan for query ref", K(ret)); + } } } } @@ -13391,12 +13485,12 @@ int ObLogPlan::fill_join_filter_info(JoinFilterInfo &join_filter_info) if (OB_ISNULL(stmt = get_stmt())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret), K(get_stmt())); + } else if (FALSE_IT(get_selectivity_ctx().clear())) { } else if (OB_FAIL(ObOptSelectivity::calculate_distinct(get_update_table_metas(), - get_selectivity_ctx(), - join_filter_info.rexprs_, - join_filter_info.row_count_, - join_filter_info.right_distinct_card_, - false))) { + get_selectivity_ctx(), + join_filter_info.rexprs_, + join_filter_info.row_count_, + join_filter_info.right_distinct_card_))) { LOG_WARN("failed to calc distinct", K(ret)); } else if (join_filter_info.table_id_ == join_filter_info.filter_table_id_) { /* do nothing */ diff --git a/src/sql/optimizer/ob_log_plan.h b/src/sql/optimizer/ob_log_plan.h index b6f84085f..a645e52f1 100644 --- a/src/sql/optimizer/ob_log_plan.h +++ b/src/sql/optimizer/ob_log_plan.h @@ -102,16 +102,17 @@ struct TableDependInfo { struct SubPlanInfo { - SubPlanInfo() : init_expr_(NULL), subplan_(NULL), init_plan_(false) {} + SubPlanInfo() : init_expr_(NULL), subplan_(NULL), init_plan_(false), allocated_(false) {} SubPlanInfo(ObQueryRefRawExpr *expr, ObLogPlan *plan, bool init_) - : init_expr_(expr), subplan_(plan), init_plan_(init_) {} + : init_expr_(expr), subplan_(plan), init_plan_(init_), allocated_(false) {} virtual ~SubPlanInfo() {} void set_subplan(ObLogPlan *plan) { subplan_ = plan; } ObQueryRefRawExpr *init_expr_; ObLogPlan *subplan_; bool init_plan_; - TO_STRING_KV(K_(init_expr), K_(subplan), K_(init_plan)); + bool allocated_; + TO_STRING_KV(K_(init_expr), K_(subplan), K_(init_plan), K_(allocated)); }; struct ObDistinctAggrBatch @@ -426,6 +427,13 @@ public: const ObInsertStmt *get_insert_stmt() const { return insert_stmt_; } void set_nonrecursive_plan_for_fake_cte(ObSelectLogPlan *plan) { nonrecursive_plan_for_fake_cte_ = plan; } ObSelectLogPlan *get_nonrecursive_plan_for_fake_cte() { return nonrecursive_plan_for_fake_cte_; } + + int add_exec_params_meta(ObIArray &exec_params, + const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx); + int add_query_ref_meta(ObQueryRefRawExpr *expr, + const OptTableMetas &child_table_metas, + const OptSelectivityCtx &child_ctx); public: struct All_Candidate_Plans diff --git a/src/sql/optimizer/ob_log_set.cpp b/src/sql/optimizer/ob_log_set.cpp index a93f6cc88..3592a5453 100644 --- a/src/sql/optimizer/ob_log_set.cpp +++ b/src/sql/optimizer/ob_log_set.cpp @@ -492,6 +492,12 @@ int ObLogSet::get_re_est_cost_infos(const EstimateCostInfo ¶m, return ret; } +int ObLogSet::est_ambient_card() +{ + // do nothing + return OB_SUCCESS; +} + int ObLogSet::do_re_est_cost(EstimateCostInfo ¶m, double &card, double &op_cost, double &cost) { int ret = OB_SUCCESS; diff --git a/src/sql/optimizer/ob_log_set.h b/src/sql/optimizer/ob_log_set.h index 813d4541e..2ccca9a68 100644 --- a/src/sql/optimizer/ob_log_set.h +++ b/src/sql/optimizer/ob_log_set.h @@ -77,6 +77,7 @@ public: virtual int est_cost() override; virtual int est_width() override; virtual int do_re_est_cost(EstimateCostInfo ¶m, double &card, double &op_cost, double &cost) override; + virtual int est_ambient_card() override; int get_re_est_cost_infos(const EstimateCostInfo ¶m, ObIArray &cost_infos, double &child_cost, diff --git a/src/sql/optimizer/ob_log_sort.cpp b/src/sql/optimizer/ob_log_sort.cpp index 9a10893e6..49cb83bd6 100644 --- a/src/sql/optimizer/ob_log_sort.cpp +++ b/src/sql/optimizer/ob_log_sort.cpp @@ -462,6 +462,7 @@ int ObLogSort::inner_est_cost(const int64_t parallel, double child_card, double if (NULL != topn_expr_) { double_topn_count = static_cast(topn_count); } + get_plan()->get_selectivity_ctx().init_op_ctx(child); double child_card_per_dop = child_card / parallel; if (double_topn_count > child_card_per_dop) { double_topn_count = child_card_per_dop; diff --git a/src/sql/optimizer/ob_log_subplan_filter.cpp b/src/sql/optimizer/ob_log_subplan_filter.cpp index 476326187..b62c58abe 100644 --- a/src/sql/optimizer/ob_log_subplan_filter.cpp +++ b/src/sql/optimizer/ob_log_subplan_filter.cpp @@ -191,6 +191,15 @@ int ObLogSubPlanFilter::get_plan_item_info(PlanText &plan_text, return ret; } +int ObLogSubPlanFilter::est_ambient_card() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(inner_est_ambient_card_by_child(ObLogicalOperator::first_child))) { + LOG_WARN("failed to est ambient cards by first child", K(ret), K(get_type())); + } + return ret; +} + int ObLogSubPlanFilter::est_cost() { int ret = OB_SUCCESS; @@ -224,7 +233,7 @@ int ObLogSubPlanFilter::do_re_est_cost(EstimateCostInfo ¶m, double &card, do LOG_WARN("unexpected params", K(ret), K(get_plan()), K(child), K(param.need_parallel_)); } else if (param.need_row_count_ < 0 || param.need_row_count_ >= child->get_card()) { param.need_row_count_ = -1; - } else if (OB_FALSE_IT(get_plan()->get_selectivity_ctx().init_op_ctx(&child->get_output_equal_sets(), child->get_card()))) { + } else if (OB_FALSE_IT(get_plan()->get_selectivity_ctx().init_op_ctx(child))) { } else if (OB_FAIL(ObOptSelectivity::calculate_selectivity(get_plan()->get_basic_table_metas(), get_plan()->get_selectivity_ctx(), get_filter_exprs(), diff --git a/src/sql/optimizer/ob_log_subplan_filter.h b/src/sql/optimizer/ob_log_subplan_filter.h index 984fe1035..89a08340d 100644 --- a/src/sql/optimizer/ob_log_subplan_filter.h +++ b/src/sql/optimizer/ob_log_subplan_filter.h @@ -39,6 +39,7 @@ public: virtual int do_re_est_cost(EstimateCostInfo ¶m, double &card, double &op_cost, double &cost) override; // re est children cost and gather cost infos int get_re_est_cost_infos(const EstimateCostInfo ¶m, ObIArray &cost_infos); + virtual int est_ambient_card() override; inline int add_subquery_exprs(const ObIArray &query_exprs) { diff --git a/src/sql/optimizer/ob_log_table_scan.cpp b/src/sql/optimizer/ob_log_table_scan.cpp index dcf98d6a7..bb935d2a5 100644 --- a/src/sql/optimizer/ob_log_table_scan.cpp +++ b/src/sql/optimizer/ob_log_table_scan.cpp @@ -1352,7 +1352,13 @@ int ObLogTableScan::get_plan_item_info(PlanText &plan_text, } else if (OB_ISNULL(table_meta = plan->get_basic_table_metas().get_table_meta_by_table_id(table_id_))) { //do nothing - } else if (OB_FAIL(BUF_PRINTF("stats version:%ld", table_meta->get_version()))) { + } else if (OB_FAIL(BUF_PRINTF("stats info:[version=%ld", table_meta->get_version()))) { + LOG_WARN("BUF_PRINTF fails", K(ret)); + } else if (OB_FAIL(BUF_PRINTF(", is_locked=%d", table_meta->is_stat_locked()))) { + LOG_WARN("BUF_PRINTF fails", K(ret)); + } else if (OB_FAIL(BUF_PRINTF(", is_expired=%d", table_meta->is_opt_stat_expired()))) { + LOG_WARN("BUF_PRINTF fails", K(ret)); + } else if (OB_FAIL(BUF_PRINTF("]"))) { LOG_WARN("BUF_PRINTF fails", K(ret)); } else if (OB_FAIL(BUF_PRINTF(NEW_LINE))) { LOG_WARN("BUF_PRINTF fails", K(ret)); diff --git a/src/sql/optimizer/ob_log_temp_table_access.cpp b/src/sql/optimizer/ob_log_temp_table_access.cpp index c10286ac4..6817e390d 100644 --- a/src/sql/optimizer/ob_log_temp_table_access.cpp +++ b/src/sql/optimizer/ob_log_temp_table_access.cpp @@ -98,6 +98,7 @@ int ObLogTempTableAccess::do_re_est_cost(EstimateCostInfo ¶m, double &card, cost = get_cost(); double selectivity = 1.0; const int64_t parallel = param.need_parallel_; + get_plan()->get_selectivity_ctx().init_op_ctx(NULL, -1); if (OB_ISNULL(get_plan())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(get_plan()),K(ret)); diff --git a/src/sql/optimizer/ob_log_temp_table_transformation.cpp b/src/sql/optimizer/ob_log_temp_table_transformation.cpp index 76fd7708c..084ff2b7b 100644 --- a/src/sql/optimizer/ob_log_temp_table_transformation.cpp +++ b/src/sql/optimizer/ob_log_temp_table_transformation.cpp @@ -124,6 +124,16 @@ int ObLogTempTableTransformation::compute_op_parallel_and_server_info() return ret; } +int ObLogTempTableTransformation::est_ambient_card() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(inner_est_ambient_card_by_child(get_num_of_child() - 1))) { + LOG_WARN("failed to est ambient cards by last child", K(ret), K(get_type())); + } + return ret; +} + + int ObLogTempTableTransformation::do_re_est_cost(EstimateCostInfo ¶m, double &card, double &op_cost, double &cost) { int ret = OB_SUCCESS; diff --git a/src/sql/optimizer/ob_log_temp_table_transformation.h b/src/sql/optimizer/ob_log_temp_table_transformation.h index 03eab9f45..4d4d9418f 100644 --- a/src/sql/optimizer/ob_log_temp_table_transformation.h +++ b/src/sql/optimizer/ob_log_temp_table_transformation.h @@ -34,6 +34,7 @@ public: virtual bool is_block_input(const int64_t child_idx) const override { return child_idx != get_num_of_child() - 1; } virtual int compute_op_parallel_and_server_info() override; virtual int do_re_est_cost(EstimateCostInfo ¶m, double &card, double &op_cost, double &cost) override; + virtual int est_ambient_card() override; int get_temp_table_exprs(ObIArray &set_exprs) const; int allocate_startup_expr_post() override; virtual int get_card_without_filter(double &card) override; diff --git a/src/sql/optimizer/ob_logical_operator.cpp b/src/sql/optimizer/ob_logical_operator.cpp index 755d46a0d..23f35096d 100644 --- a/src/sql/optimizer/ob_logical_operator.cpp +++ b/src/sql/optimizer/ob_logical_operator.cpp @@ -1067,6 +1067,8 @@ int ObLogicalOperator::compute_property(Path *path) set_server_cnt(path->server_cnt_); if (OB_FAIL(server_list_.assign(path->server_list_))) { LOG_WARN("failed to assign path's server list to op", K(ret)); + } else if (OB_FAIL(ambient_card_.assign(path->parent_->get_ambient_card()))) { + LOG_WARN("failed to assign ambient cards", K(ret)); } else if (OB_FAIL(check_property_valid())) { LOG_WARN("failed to check property valid", K(ret), KPC(path)); } else { @@ -1273,6 +1275,8 @@ int ObLogicalOperator::compute_property() LOG_WARN("failed to compute width", K(ret)); } else if (OB_FAIL(est_cost())) { LOG_WARN("failed to estimate cost", K(ret)); + } else if (OB_FAIL(est_ambient_card())) { + LOG_WARN("failed to est ambient card"); } else if (OB_FAIL(check_property_valid())) { LOG_WARN("failed to check property valid", K(ret)); } else { @@ -1299,6 +1303,41 @@ int ObLogicalOperator::compute_property() return ret; } +int ObLogicalOperator::est_ambient_card() +{ + int ret = OB_SUCCESS; + if (1 == get_num_of_child()) { + if (OB_FAIL(inner_est_ambient_card_by_child(ObLogicalOperator::first_child))) { + LOG_WARN("failed to est ambient cards by first child", K(ret), K(get_type())); + } + } else if (0 == get_num_of_child()) { + // do nothing + // ambient cardinality of the leaf node is inited by the path + } else { + // ret = OB_ERR_UNEXPECTED; + LOG_WARN("multi child op called default est_ambient_card function", K(ret), K(get_type())); + } + return ret; +} + +int ObLogicalOperator::inner_est_ambient_card_by_child(int64_t child_idx) +{ + int ret = OB_SUCCESS; + ObLogicalOperator *child = NULL; + if (OB_UNLIKELY(child_idx >= get_num_of_child()) || + OB_ISNULL(child = get_child(child_idx))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(child), K(ret)); + } else if (OB_FAIL(ambient_card_.assign(child->get_ambient_card()))) { + LOG_WARN("failed to assign", K(ret)); + } else { + for (int64_t i = 0; i < ambient_card_.count(); i ++) { + ambient_card_.at(i) = ObOptSelectivity::scale_distinct(get_card(), child->get_card(), ambient_card_.at(i)); + } + } + return ret; +} + int ObLogicalOperator::check_property_valid() const { int ret = OB_SUCCESS; diff --git a/src/sql/optimizer/ob_logical_operator.h b/src/sql/optimizer/ob_logical_operator.h index 09060b52d..4fd1dcb44 100644 --- a/src/sql/optimizer/ob_logical_operator.h +++ b/src/sql/optimizer/ob_logical_operator.h @@ -1323,6 +1323,9 @@ public: int re_est_cost(EstimateCostInfo ¶m, double &card, double &cost); virtual int do_re_est_cost(EstimateCostInfo ¶m, double &card, double &op_cost, double &cost); + virtual int est_ambient_card(); + int inner_est_ambient_card_by_child(int64_t child_idx); + /** * @brief compute_property * convert property fields from a path into a logical operator @@ -1719,6 +1722,8 @@ public: virtual int close_px_resource_analyze(CLOSE_PX_RESOURCE_ANALYZE_DECLARE_ARG); int find_max_px_resource_child(OPEN_PX_RESOURCE_ANALYZE_DECLARE_ARG, int64_t start_idx); + inline ObIArray &get_ambient_card() { return ambient_card_; } + public: ObSEArray child_; ObSEArray equal_param_constraints_; @@ -1800,6 +1805,7 @@ protected: const EqualSets *output_equal_sets_; const ObFdItemSet *fd_item_set_; const ObRelIds *table_set_; + common::ObSEArray ambient_card_; uint64_t id_; // operator 0-based depth-first id uint64_t branch_id_; diff --git a/src/sql/optimizer/ob_opt_default_stat.h b/src/sql/optimizer/ob_opt_default_stat.h index 9cc301d8a..821440bad 100644 --- a/src/sql/optimizer/ob_opt_default_stat.h +++ b/src/sql/optimizer/ob_opt_default_stat.h @@ -73,8 +73,8 @@ const double DEFAULT_SEL = 0.5; const double DEFAULT_AGG_RANGE = 0.05; // [aggr(expr) = const]的默认选择率，参考oracle const double DEFAULT_AGG_EQ = 0.01; -// clob/blob like "xxx" 的默认选择率 -const double DEFAULT_CLOB_LIKE_SEL = 0.05; +// like 的默认选择率，参考oracle +const double DEFAULT_LIKE_SEL = 0.05; const double DEFAULT_ANTI_JOIN_SEL = 0.01; // 范围谓词越界部分选择率，参考 SQLserver const double DEFAULT_OUT_OF_BOUNDS_SEL = 0.3; diff --git a/src/sql/optimizer/ob_opt_est_cost.cpp b/src/sql/optimizer/ob_opt_est_cost.cpp index 09878a45c..4ac434940 100644 --- a/src/sql/optimizer/ob_opt_est_cost.cpp +++ b/src/sql/optimizer/ob_opt_est_cost.cpp @@ -677,39 +677,53 @@ int ObOptEstCost::calculate_filter_selectivity(ObCostTableScanInfo &est_cost_inf ObIArray &all_predicate_sel) { int ret = OB_SUCCESS; - if (OB_ISNULL(est_cost_info.table_metas_) || OB_ISNULL(est_cost_info.sel_ctx_)) { + ObSEArray apply_filters; + double total_sel = 1.0; + if (OB_ISNULL(est_cost_info.table_metas_) || OB_ISNULL(est_cost_info.sel_ctx_) || + OB_ISNULL(est_cost_info.table_meta_info_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("null point error", K(est_cost_info.table_metas_), K(est_cost_info.sel_ctx_), K(ret)); - } else if (OB_FAIL(ObOptSelectivity::calculate_selectivity(*est_cost_info.table_metas_, - *est_cost_info.sel_ctx_, - est_cost_info.prefix_filters_, - est_cost_info.prefix_filter_sel_, - all_predicate_sel))) { - LOG_WARN("failed to calculate selectivity", K(est_cost_info.postfix_filters_), K(ret)); - } else if (OB_FAIL(ObOptSelectivity::calculate_selectivity(*est_cost_info.table_metas_, - *est_cost_info.sel_ctx_, - est_cost_info.pushdown_prefix_filters_, - est_cost_info.pushdown_prefix_filter_sel_, - all_predicate_sel))) { - LOG_WARN("failed to calculate selectivity", K(est_cost_info.pushdown_prefix_filters_), K(ret)); - } else if (OB_FAIL(ObOptSelectivity::calculate_selectivity(*est_cost_info.table_metas_, - *est_cost_info.sel_ctx_, - est_cost_info.ss_postfix_range_filters_, - est_cost_info.ss_postfix_range_filters_sel_, - all_predicate_sel))) { - LOG_WARN("failed to calculate selectivity", K(est_cost_info.ss_postfix_range_filters_), K(ret)); - } else if (OB_FAIL(ObOptSelectivity::calculate_selectivity(*est_cost_info.table_metas_, - *est_cost_info.sel_ctx_, - est_cost_info.postfix_filters_, - est_cost_info.postfix_filter_sel_, - all_predicate_sel))) { - LOG_WARN("failed to calculate selectivity", K(est_cost_info.postfix_filters_), K(ret)); - } else if (OB_FAIL(ObOptSelectivity::calculate_selectivity(*est_cost_info.table_metas_, - *est_cost_info.sel_ctx_, - est_cost_info.table_filters_, - est_cost_info.table_filter_sel_, - all_predicate_sel))) { - LOG_WARN("failed to calculate selectivity", K(est_cost_info.table_filters_), K(ret)); + } else if (FALSE_IT(est_cost_info.sel_ctx_->init_op_ctx(NULL, est_cost_info.table_meta_info_->table_row_count_))) { + } else if (OB_FAIL(ObOptSelectivity::calculate_conditional_selectivity(*est_cost_info.table_metas_, + *est_cost_info.sel_ctx_, + apply_filters, + est_cost_info.prefix_filters_, + total_sel, + est_cost_info.prefix_filter_sel_, + all_predicate_sel))) { + LOG_WARN("failed to calculate prefix filter sel", K(est_cost_info.prefix_filters_)); + } else if (OB_FAIL(ObOptSelectivity::calculate_conditional_selectivity(*est_cost_info.table_metas_, + *est_cost_info.sel_ctx_, + apply_filters, + est_cost_info.pushdown_prefix_filters_, + total_sel, + est_cost_info.pushdown_prefix_filter_sel_, + all_predicate_sel))) { + LOG_WARN("failed to calculate prefix filter sel", K(est_cost_info.pushdown_prefix_filters_)); + } else if (OB_FAIL(ObOptSelectivity::calculate_conditional_selectivity(*est_cost_info.table_metas_, + *est_cost_info.sel_ctx_, + apply_filters, + est_cost_info.ss_postfix_range_filters_, + total_sel, + est_cost_info.ss_postfix_range_filters_sel_, + all_predicate_sel))) { + LOG_WARN("failed to calculate prefix filter sel", K(est_cost_info.ss_postfix_range_filters_)); + } else if (OB_FAIL(ObOptSelectivity::calculate_conditional_selectivity(*est_cost_info.table_metas_, + *est_cost_info.sel_ctx_, + apply_filters, + est_cost_info.postfix_filters_, + total_sel, + est_cost_info.postfix_filter_sel_, + all_predicate_sel))) { + LOG_WARN("failed to calculate prefix filter sel", K(est_cost_info.postfix_filters_)); + } else if (OB_FAIL(ObOptSelectivity::calculate_conditional_selectivity(*est_cost_info.table_metas_, + *est_cost_info.sel_ctx_, + apply_filters, + est_cost_info.table_filters_, + total_sel, + est_cost_info.table_filter_sel_, + all_predicate_sel))) { + LOG_WARN("failed to calculate prefix filter sel", K(est_cost_info.table_filters_)); } else { LOG_TRACE("table filter info", K(est_cost_info.ref_table_id_), K(est_cost_info.index_id_), K(est_cost_info.prefix_filters_), K(est_cost_info.pushdown_prefix_filters_), diff --git a/src/sql/optimizer/ob_opt_est_utils.cpp b/src/sql/optimizer/ob_opt_est_utils.cpp index b32c27e1c..0e2124521 100644 --- a/src/sql/optimizer/ob_opt_est_utils.cpp +++ b/src/sql/optimizer/ob_opt_est_utils.cpp @@ -75,21 +75,34 @@ int ObOptEstUtils::extract_column_exprs_with_op_check( } -int ObOptEstUtils::is_range_expr(const ObRawExpr *qual, bool &is_simple_filter, const int64_t level) +int ObOptEstUtils::is_range_expr(const ObRawExpr *qual, bool &is_simple_filter) { int ret = OB_SUCCESS; - if (0 == level) { - is_simple_filter = true; - } + is_simple_filter = true; if (OB_ISNULL(qual)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("qual is null", K(ret)); - } else if (IS_RANGE_CMP_OP(qual->get_expr_type()) && qual->has_flag(IS_RANGE_COND)) { - // c1 > 1 , 1 < c1 do nothing + } else if (IS_RANGE_CMP_OP(qual->get_expr_type()) || + T_OP_BTW == qual->get_expr_type() || + T_OP_NOT_BTW == qual->get_expr_type()) { + // c1 > 1 , 1 < c1, c1 (not) between 1 and '2' + const ObRawExpr *var = NULL; + const ObRawExpr *const_expr1 = NULL; + const ObRawExpr *const_expr2 = NULL; + ObItemType dummy = T_INVALID; + if (OB_FAIL(extract_var_op_const(qual, var, const_expr1, const_expr2, dummy, is_simple_filter))) { + LOG_WARN("failed to extract var", K(ret)); + } else if (!is_simple_filter) { + // do nothing + } else if (OB_FAIL(ObOptimizerUtil::get_expr_without_lossless_cast(var, var))) { + LOG_WARN("failed to get expr without lossless cast", K(ret)); + } else if (!var->is_column_ref_expr()) { + is_simple_filter = false; + } } else if (T_OP_AND == qual->get_expr_type() || T_OP_OR == qual->get_expr_type()) { const ObOpRawExpr *op_expr = static_cast(qual); for (int idx = 0 ; idx < op_expr->get_param_count() && is_simple_filter && OB_SUCC(ret); ++idx) { - if (OB_FAIL(is_range_expr(op_expr->get_param_expr(idx), is_simple_filter, level + 1))) { + if (OB_FAIL(is_range_expr(op_expr->get_param_expr(idx), is_simple_filter))) { LOG_WARN("failed to judge if expr is range", K(ret)); } } @@ -99,6 +112,70 @@ int ObOptEstUtils::is_range_expr(const ObRawExpr *qual, bool &is_simple_filter, return ret; } +int ObOptEstUtils::extract_var_op_const(const ObRawExpr *qual, + const ObRawExpr *&var_expr, + const ObRawExpr *&const_expr1, + const ObRawExpr *&const_expr2, + ObItemType &type, + bool &is_valid) +{ + int ret = OB_SUCCESS; + type = T_INVALID; + is_valid = false; + var_expr = NULL; + const_expr1 = NULL; + const_expr2 = NULL; + if (OB_ISNULL(qual)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (FALSE_IT(type = qual->get_expr_type())) { + } else if (IS_RANGE_CMP_OP(type) || T_OP_EQ == type || T_OP_NSEQ == type || T_OP_NE == type) { + if (OB_UNLIKELY(qual->get_param_count() != 2) || + OB_ISNULL(qual->get_param_expr(0)) || + OB_ISNULL(qual->get_param_expr(1))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected param", KPC(qual)); + } else if (!qual->get_param_expr(0)->is_const_expr() && + qual->get_param_expr(1)->is_const_expr()) { + var_expr = qual->get_param_expr(0); + const_expr1 = qual->get_param_expr(1); + is_valid = true; + } else if (!qual->get_param_expr(1)->is_const_expr() && + qual->get_param_expr(0)->is_const_expr()) { + var_expr = qual->get_param_expr(1); + const_expr1 = qual->get_param_expr(0); + is_valid = true; + type = get_opposite_compare_type(type); + } + } else if (T_OP_IS == type || T_OP_IS_NOT == type) { + if (OB_UNLIKELY(qual->get_param_count() != 2) || + OB_ISNULL(qual->get_param_expr(0)) || + OB_ISNULL(qual->get_param_expr(1))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected param", KPC(qual)); + } else if (!qual->get_param_expr(0)->is_const_expr() && + qual->get_param_expr(1)->is_const_expr()) { + var_expr = qual->get_param_expr(0); + const_expr1 = qual->get_param_expr(1); + is_valid = true; + } + } else if (T_OP_BTW == type || T_OP_NOT_BTW == type) { + if (OB_UNLIKELY(3 != qual->get_param_count()) || OB_ISNULL(qual->get_param_expr(0)) || + OB_ISNULL(qual->get_param_expr(1)) || OB_ISNULL(qual->get_param_expr(2))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected param", K(ret), KPC(qual)); + } else if (!qual->get_param_expr(0)->is_const_expr() && + qual->get_param_expr(1)->is_const_expr() && + qual->get_param_expr(2)->is_const_expr()) { + var_expr = qual->get_param_expr(0); + const_expr1 = qual->get_param_expr(1); + const_expr2 = qual->get_param_expr(2); + is_valid = true; + } + } + return ret; +} + int ObOptEstUtils::extract_simple_cond_filters(ObRawExpr &qual, bool &can_be_extracted, ObIArray &column_exprs_array) @@ -205,7 +282,7 @@ int ObOptEstUtils::if_expr_start_with_patten_sign(const ParamStore *params, is_start_with = false; all_is_percent_sign = false; bool get_value = false; - bool empty_escape = false; + bool valid_escape = true; char escape; ObObj value; ObObj esp_value; @@ -218,32 +295,48 @@ int ObOptEstUtils::if_expr_start_with_patten_sign(const ParamStore *params, } else if (!get_value || !esp_value.is_string_type()) { // do nothing } else { - if (esp_value.get_char().length() > 0) { - escape = esp_value.get_char()[0]; - } else { - empty_escape = true; + size_t escape_length = ObCharset::strlen_char(esp_expr->get_collation_type(), + esp_value.get_string().ptr(), + esp_value.get_string().length()); + int32_t escape_wc = 0; + if (1 != escape_length) { + valid_escape = false; + } else if (OB_FAIL(ObCharset::mb_wc(esp_expr->get_collation_type(), esp_value.get_string(), escape_wc))) { + ret = OB_SUCCESS; + valid_escape = false; } if (OB_FAIL(get_expr_value(params, *expr, exec_ctx, allocator, get_value, value))) { LOG_WARN("Failed to get expr value", K(ret)); } else if (get_value && value.is_string_type() && value.get_string().length() > 0) { // 1. patten not start with `escape sign` // 2. patten start with `%` or `_` && `%` or `_` is not `escape sign` - char start_c = value.get_string()[0]; - if (empty_escape) { - is_start_with = ('%' == start_c || '_' == start_c); - } else { - is_start_with = (escape != start_c && ('%' == start_c || '_' == start_c)); + ObStringScanner scanner(value.get_string(), expr->get_collation_type()); + ObString encoding; + int32_t wc = 0; + ObString first_c; + bool is_first_char = true; + all_is_percent_sign = true; + while (OB_SUCC(ret) + && scanner.next_character(encoding, wc, ret) + && all_is_percent_sign) { + if (is_first_char) { + bool is_wild = (static_cast('%') == wc || static_cast('_') == wc); + if (!valid_escape) { + is_start_with = is_wild; + } else { + is_start_with = (escape_wc != wc && is_wild); + } + is_first_char = false; + } + if (static_cast('%') != wc) { + all_is_percent_sign = false; + } } - } else { /* do nothing */ } - } - if (OB_SUCC(ret) && is_start_with) { - all_is_percent_sign = true; - const ObString &expr_str = value.get_string(); - for (int64_t i = 0; all_is_percent_sign && i < expr_str.length(); i++) { - if (expr_str[i] != '%') { + if (OB_FAIL(ret)) { + ret = OB_SUCCESS; all_is_percent_sign = false; } - } + } else { /* do nothing */ } } return ret; } diff --git a/src/sql/optimizer/ob_opt_est_utils.h b/src/sql/optimizer/ob_opt_est_utils.h index 7af297313..e74ee2c6f 100644 --- a/src/sql/optimizer/ob_opt_est_utils.h +++ b/src/sql/optimizer/ob_opt_est_utils.h @@ -49,7 +49,14 @@ public: //such as, c1 > 1 and c1 < 2, c1 between 1 and 1000, c1 > 100 or c1 < 10000 //@param in qual //@param out is_range - static int is_range_expr(const ObRawExpr *qual, bool &is_simple_filter, const int64_t level = 0); + static int is_range_expr(const ObRawExpr *qual, bool &is_simple_filter); + + static int extract_var_op_const(const ObRawExpr *qual, + const ObRawExpr *&var_expr, + const ObRawExpr *&const_expr1, + const ObRawExpr *&const_expr2, + ObItemType &type, + bool &is_valid); //extract column exprs with simple operator check. //level must be initialized with 0(default value) diff --git a/src/sql/optimizer/ob_opt_selectivity.cpp b/src/sql/optimizer/ob_opt_selectivity.cpp index d50f72b26..bdf5a9f46 100644 --- a/src/sql/optimizer/ob_opt_selectivity.cpp +++ b/src/sql/optimizer/ob_opt_selectivity.cpp @@ -40,6 +40,98 @@ namespace sql { inline double revise_ndv(double ndv) { return ndv < 1.0 ? 1.0 : ndv; } +void OptSelectivityCtx::init_op_ctx(ObLogicalOperator *child_op) +{ + if (OB_NOT_NULL(child_op)) { + init_op_ctx(&child_op->get_output_equal_sets(), + child_op->get_card(), + &child_op->get_ambient_card()); + } else { + init_op_ctx(NULL, -1, NULL); + } +} + +int OptSelectivityCtx::get_ambient_card(const uint64_t table_id, double &table_ambient_card) const +{ + int ret = OB_SUCCESS; + table_ambient_card = -1.0; + if (OB_ISNULL(get_stmt())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_NOT_NULL(get_ambient_card())) { + uint64_t table_index = get_stmt()->get_table_bit_index(table_id); + if (OB_UNLIKELY(table_index < 1) || + OB_UNLIKELY(table_index >= get_ambient_card()->count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected table index", K(table_index), K(table_id), KPC(get_stmt())); + } else { + table_ambient_card = get_ambient_card()->at(table_index); + } + } + return ret; +} + +ObEstCorrelationModel &ObIndependentModel::get_model() +{ + static ObIndependentModel model; + return model; +} + +ObEstCorrelationModel &ObPartialCorrelationModel::get_model() +{ + static ObPartialCorrelationModel model; + return model; +} + +ObEstCorrelationModel &ObFullCorrelationModel::get_model() +{ + static ObFullCorrelationModel model; + return model; +} + +ObEstCorrelationModel &ObEstCorrelationModel::get_correlation_model(ObEstCorrelationType type) +{ + switch (type) { + case ObEstCorrelationType::INDEPENDENT: return ObIndependentModel::get_model(); + case ObEstCorrelationType::PARTIAL: return ObPartialCorrelationModel::get_model(); + case ObEstCorrelationType::FULL: return ObFullCorrelationModel::get_model(); + default: break; + } + return ObPartialCorrelationModel::get_model(); +} + +double ObIndependentModel::combine_filters_selectivity(ObIArray &selectivities) const +{ + double combine_selectivity = 1.0; + for (int64_t i = 0; i < selectivities.count(); i ++) { + combine_selectivity *= selectivities.at(i); + } + return combine_selectivity; +} + +double ObPartialCorrelationModel::combine_filters_selectivity(ObIArray &selectivities) const +{ + double selectivity = 1.0; + if (!selectivities.empty()) { + double exp = 1.0; + lib::ob_sort(&selectivities.at(0), &selectivities.at(0) + selectivities.count()); + for (int64_t i = 0; i < selectivities.count(); i ++) { + selectivity *= std::pow(selectivities.at(i), 1 / exp); + exp *= 2; + } + } + return selectivity; +} + +double ObFullCorrelationModel::combine_filters_selectivity(ObIArray &selectivities) const +{ + double combine_selectivity = 1.0; + for (int64_t i = 0; i < selectivities.count(); i ++) { + combine_selectivity = std::min(combine_selectivity, selectivities.at(i)); + } + return combine_selectivity; +} + int OptColumnMeta::assign(const OptColumnMeta &other) { int ret = OB_SUCCESS; @@ -87,6 +179,7 @@ int OptTableMeta::assign(const OptTableMeta &other) table_partition_info_ = other.table_partition_info_; base_meta_info_ = other.base_meta_info_; real_rows_ = other.real_rows_; + stale_stats_ = other.stale_stats_; if (OB_FAIL(all_used_parts_.assign(other.all_used_parts_))) { LOG_WARN("failed to assign all used parts", K(ret)); @@ -206,6 +299,12 @@ int OptTableMeta::init_column_meta(const OptSelectivityCtx &ctx, } if (OB_SUCC(ret)) { + if (rows_ < col_meta.get_ndv()) { + col_meta.set_ndv(rows_); + } + if (rows_ < col_meta.get_num_null()) { + col_meta.set_num_null(rows_); + } col_meta.set_column_id(column_id); col_meta.set_avg_len(stat.avglen_val_); col_meta.set_cg_macro_blk_cnt(stat.cg_macro_blk_cnt_); @@ -270,14 +369,6 @@ const OptColumnMeta* OptTableMeta::get_column_meta(const uint64_t column_id) con return column_meta; } -void OptTableMeta::set_ndv_for_all_column(double ndv) -{ - for (int64_t i = 0; i < column_metas_.count(); ++i) { - column_metas_.at(i).set_ndv(ndv); - } - return; -} - int OptTableMetas::copy_table_meta_info(const OptTableMeta &src_meta, OptTableMeta *&dst_meta) { int ret = OB_SUCCESS; @@ -318,7 +409,8 @@ int OptTableMetas::add_base_table_meta_info(OptSelectivityCtx &ctx, int64_t last_analyzed, bool is_stat_locked, const ObTablePartitionInfo *table_partition_info, - const ObTableMetaInfo *base_meta_info) + const ObTableMetaInfo *base_meta_info, + bool stale_stats) { int ret = OB_SUCCESS; ObSqlSchemaGuard *schema_guard = ctx.get_sql_schema_guard(); @@ -337,6 +429,7 @@ int OptTableMetas::add_base_table_meta_info(OptSelectivityCtx &ctx, } else { table_meta->set_version(last_analyzed); table_meta->set_stat_locked(is_stat_locked); + table_meta->set_stale_stats(stale_stats); LOG_TRACE("add base table meta info success", K(*table_meta)); } return ret; @@ -499,12 +592,17 @@ int OptTableMetas::add_generate_table_meta_info(const ObDMLStmt *parent_stmt, ObObj minobj; maxobj.set_max_value(); minobj.set_min_value(); - if (select_expr->is_column_ref_expr() && - OB_FAIL(ObOptSelectivity::get_column_min_max(child_table_metas, child_ctx, *select_expr, minobj, maxobj))) { - LOG_WARN("failed to get column min max", K(ret)); - } else { + avg_len = 0; + if (OB_FAIL(ObOptSelectivity::calculate_expr_avg_len(child_table_metas, child_ctx, select_expr, avg_len))) { + LOG_WARN("failed to get avg len", K(ret)); + } else if (OB_FAIL(ObOptSelectivity::calc_expr_min_max(child_table_metas, child_ctx, select_expr, minobj, maxobj))) { + LOG_WARN("failed to calc expr min max", KPC(select_expr)); + } + + if (OB_SUCC(ret)) { column_meta->set_min_value(minobj); column_meta->set_max_value(maxobj); + column_meta->set_avg_len(avg_len); } } } @@ -666,6 +764,90 @@ const OptColumnMeta* OptTableMetas::get_column_meta_by_table_id(const uint64_t t return column_meta; } +const OptDynamicExprMeta* OptTableMetas::get_dynamic_expr_meta(const ObRawExpr *expr) const +{ + const OptDynamicExprMeta* dynamic_expr_meta = NULL; + for (int64_t i = 0; NULL == dynamic_expr_meta && i < dynamic_expr_metas_.count(); ++i) { + if (expr == dynamic_expr_metas_.at(i).get_expr()) { + dynamic_expr_meta = &dynamic_expr_metas_.at(i); + } + } + return dynamic_expr_meta; +} + +double OptTableMetas::get_rows(const uint64_t table_id) const +{ + const OptTableMeta *table_meta = get_table_meta_by_table_id(table_id); + double rows = 1.0; + if (OB_NOT_NULL(table_meta)) { + rows = table_meta->get_rows(); + } + return rows; +} + +int ObOptSelectivity::calculate_conditional_selectivity(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + common::ObIArray &total_filters, + common::ObIArray &append_filters, + double &total_sel, + double &conditional_sel, + ObIArray &all_predicate_sel) +{ + int ret = OB_SUCCESS; + double new_sel = 1.0; + if (OB_FAIL(append(total_filters, append_filters))) { + LOG_WARN("failed to append filters", K(ret)); + } else if (total_sel > OB_DOUBLE_EPSINON && !ctx.get_correlation_model().is_independent()) { + if (OB_FAIL(calculate_selectivity(table_metas, + ctx, + total_filters, + new_sel, + all_predicate_sel))) { + LOG_WARN("failed to calculate selectivity", K(total_filters), K(ret)); + } else { + conditional_sel = new_sel / total_sel; + total_sel = new_sel; + } + } else if (OB_FAIL(calculate_selectivity(table_metas, + ctx, + append_filters, + conditional_sel, + all_predicate_sel))) { + LOG_WARN("failed to calculate selectivity", K(append_filters), K(ret)); + } else { + total_sel *= conditional_sel; + } + return ret; +} + +int ObOptSelectivity::calculate_selectivity(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + ObIArray &sel_estimators, + double &selectivity) +{ + int ret = OB_SUCCESS; + selectivity = 1.0; + ObSEArray selectivities; + ObSEArray dummy; + for (int64_t i = 0; OB_SUCC(ret) && i < sel_estimators.count(); ++i) { + ObSelEstimator *estimator = sel_estimators.at(i); + double tmp_selectivity = 0.0; + if (OB_ISNULL(sel_estimators.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("estimator is null", K(ret), K(sel_estimators)); + } else if (OB_FAIL(estimator->get_sel(table_metas, ctx, tmp_selectivity, dummy))) { + LOG_WARN("failed to get sel", K(ret), KPC(estimator)); + } else if (OB_FAIL(selectivities.push_back(revise_between_0_1(tmp_selectivity)))) { + LOG_WARN("failed to push back", K(ret)); + } + } + if (OB_SUCC(ret)) { + selectivity = ctx.get_correlation_model().combine_filters_selectivity(selectivities); + } + LOG_DEBUG("calculate predicates selectivity", K(selectivity), K(selectivities), K(sel_estimators)); + return ret; +} + int ObOptSelectivity::calculate_selectivity(const OptTableMetas &table_metas, const OptSelectivityCtx &ctx, const ObIArray &predicates, @@ -676,8 +858,13 @@ int ObOptSelectivity::calculate_selectivity(const OptTableMetas &table_metas, selectivity = 1.0; ObSEArray sel_estimators; ObSEArray selectivities; - ObArenaAllocator tmp_alloc("ObOptSel"); - ObSelEstimatorFactory factory(tmp_alloc); + ObSelEstimatorFactory factory; + if (OB_ISNULL(ctx.get_session_info())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else { + factory.get_allocator().set_tenant_id(ctx.get_session_info()->get_effective_tenant_id()); + } for (int64_t i = 0; OB_SUCC(ret) && i < predicates.count(); ++i) { const ObRawExpr *qual = predicates.at(i); ObSelEstimator *estimator = NULL; @@ -713,7 +900,7 @@ int ObOptSelectivity::calculate_selectivity(const OptTableMetas &table_metas, LOG_WARN("failed to get sel", K(ret), KPC(estimator)); } else { selectivities.at(i) = revise_between_0_1(tmp_selectivity); - if (ObSelEstType::RANGE == estimator->get_type()) { + if (ObSelEstType::COLUMN_RANGE == estimator->get_type()) { ObRangeSelEstimator *range_estimator = static_cast(estimator); if (OB_FAIL(add_var_to_array_no_dup(all_predicate_sel, ObExprSelPair(range_estimator->get_column_expr(), tmp_selectivity, true)))) { @@ -722,7 +909,63 @@ int ObOptSelectivity::calculate_selectivity(const OptTableMetas &table_metas, } } } - selectivity = ObOptSelectivity::get_filters_selectivity(selectivities, ctx.get_dependency_type()); + if (OB_SUCC(ret)) { + selectivity = ctx.get_correlation_model().combine_filters_selectivity(selectivities); + LOG_DEBUG("calculate predicates selectivity", K(selectivity), K(selectivities), K(sel_estimators)); + } + return ret; +} + +int ObOptSelectivity::calculate_join_selectivity(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + const ObIArray &predicates, + double &selectivity, + ObIArray &all_predicate_sel) +{ + int ret = OB_SUCCESS; + selectivity = 1.0; + ObSEArray sel_estimators; + ObSelEstimatorFactory factory; + if (OB_ISNULL(ctx.get_session_info())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else { + factory.get_allocator().set_tenant_id(ctx.get_session_info()->get_effective_tenant_id()); + } + for (int64_t i = 0; OB_SUCC(ret) && i < predicates.count(); ++i) { + const ObRawExpr *qual = predicates.at(i); + ObSelEstimator *estimator = NULL; + double single_sel = false; + if (OB_FAIL(factory.create_estimator(ctx, qual, estimator))) { + LOG_WARN("failed to create estimator", KPC(qual)); + } else if (OB_FAIL(ObSelEstimator::append_estimators(sel_estimators, estimator))) { + LOG_WARN("failed to append estimators", KPC(qual)); + } else if (ObOptimizerUtil::find_item(all_predicate_sel, ObExprSelPair(qual, 0))) { + // do nothing + } else if (OB_FAIL(estimator->get_sel(table_metas, ctx, single_sel, all_predicate_sel))) { + LOG_WARN("failed to calculate one qual selectivity", KPC(estimator), K(qual), K(ret)); + } else if (FALSE_IT(single_sel = revise_between_0_1(single_sel))) { + // never reach + } else if (OB_FAIL(add_var_to_array_no_dup(all_predicate_sel, ObExprSelPair(qual, single_sel)))) { + LOG_WARN("fail ed to add selectivity to plan", K(ret), K(qual), K(selectivity)); + } else { + // We remember each predicate's selectivity in the plan so that we can reorder them + // in the vector of filters according to their selectivity. + LOG_PRINT_EXPR(TRACE, "calculate one qual selectivity", *qual, K(single_sel)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < sel_estimators.count(); ++i) { + ObSelEstimator *estimator = sel_estimators.at(i); + double tmp_selectivity = 0.0; + if (OB_ISNULL(sel_estimators.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("estimator is null", K(ret), K(sel_estimators)); + } else if (OB_FAIL(estimator->get_sel(table_metas, ctx, tmp_selectivity, all_predicate_sel))) { + LOG_WARN("failed to get sel", K(ret), KPC(estimator)); + } else { + selectivity *= revise_between_0_1(tmp_selectivity); + } + } return ret; } @@ -975,8 +1218,7 @@ int ObOptSelectivity::calculate_qual_selectivity(const OptTableMetas &table_meta ObIArray &all_predicate_sel) { int ret = OB_SUCCESS; - ObArenaAllocator tmp_alloc("ObOptSel"); - ObSelEstimatorFactory factory(tmp_alloc); + ObSelEstimatorFactory factory; ObSelEstimator *estimator = NULL; if (OB_FAIL(factory.create_estimator(ctx, &qual, estimator))) { LOG_WARN("failed to create estimator", K(qual)); @@ -1019,7 +1261,10 @@ int ObOptSelectivity::update_table_meta_info(const OptTableMetas &base_table_met table_meta->clear_base_table_info(); if (filtered_rows >= origin_rows) { // only update table rows - } else if (OB_FAIL(classify_quals(ctx, quals, all_predicate_sel, column_sel_infos))) { + } else if (OB_FAIL(!ctx.check_opt_compat_version(COMPAT_VERSION_4_2_4, COMPAT_VERSION_4_3_0, + COMPAT_VERSION_4_3_3) ? + classify_quals_deprecated(ctx, quals, all_predicate_sel, column_sel_infos) : + classify_quals(base_table_metas, ctx, quals, all_predicate_sel, column_sel_infos))) { LOG_WARN("failed to classify quals", K(ret)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < table_meta->get_column_metas().count(); ++i) { @@ -1039,32 +1284,19 @@ int ObOptSelectivity::update_table_meta_info(const OptTableMetas &base_table_met * 使用第一步得到的ndv和rows作为column的原始ndv和rows，再基于所有过滤谓词过滤后的行数， * 使用缩放公式缩放列的ndv。 */ + double origin_ndv = column_meta.get_ndv(); double step1_ndv = column_meta.get_ndv(); double step2_ndv = column_meta.get_ndv(); double step1_row = origin_rows; double null_num = column_meta.get_num_null(); double hist_scale = -1; - // step 1 - if (OB_NOT_NULL(sel_info)) { - step1_row *= sel_info->selectivity_; - hist_scale = sel_info->selectivity_; - if (sel_info->equal_count_ > 0) { - step1_ndv = sel_info->equal_count_; - } else if (sel_info->has_range_exprs_) { - step1_ndv *= sel_info->range_selectivity_; - } else { - step1_ndv = scale_distinct(step1_row, origin_rows, column_meta.get_ndv()); - } - } - // step 2 - if (filtered_rows < step1_row) { - step2_ndv = scale_distinct(filtered_rows, step1_row, step1_ndv); - } else { - step2_ndv = step1_ndv; + if (null_num > origin_rows - origin_ndv) { + null_num = std::max(origin_rows - origin_ndv, 0.0); } + double nns = origin_rows <= OB_DOUBLE_EPSINON ? 1.0 : 1 - revise_between_0_1(null_num / origin_rows); + bool null_reject = false; // update null number if (null_num > 0) { - bool null_reject = false; const ObColumnRefRawExpr *column_expr = log_plan->get_column_expr_by_id( table_meta->get_table_id(), column_meta.get_column_id()); if (OB_ISNULL(column_expr)) { @@ -1080,6 +1312,52 @@ int ObOptSelectivity::update_table_meta_info(const OptTableMetas &base_table_met null_num = null_num * filtered_rows / origin_rows; } } + // step 1 + if (OB_NOT_NULL(sel_info)) { + if (!ctx.check_opt_compat_version(COMPAT_VERSION_4_2_4, COMPAT_VERSION_4_3_0, + COMPAT_VERSION_4_3_3)) { + step1_row *= sel_info->selectivity_; + hist_scale = sel_info->selectivity_; + if (sel_info->equal_count_ > 0) { + step1_ndv = sel_info->equal_count_; + } else if (sel_info->has_range_exprs_) { + step1_ndv *= sel_info->range_selectivity_; + } else { + step1_ndv = scale_distinct(step1_row, origin_rows, column_meta.get_ndv()); + } + } else { + double step1_sel = sel_info->selectivity_; + double direct_ndv_sel = 1.0; + if (origin_rows * step1_sel < filtered_rows && + origin_rows > OB_DOUBLE_EPSINON) { + step1_sel = filtered_rows / origin_rows; + } + if (null_reject) { + if (step1_sel <= nns && nns > OB_DOUBLE_EPSINON) { + direct_ndv_sel = step1_sel / nns; + } else { + direct_ndv_sel = 1.0; + } + } else { + // complex quals, the selectivity might be default + // do not handle null + direct_ndv_sel = step1_sel; + } + step1_row *= step1_sel; + hist_scale = step1_sel; + if (sel_info->equal_count_ > 0) { + step1_ndv = sel_info->equal_count_; + } else { + step1_ndv *= direct_ndv_sel; + } + } + } + // step 2 + if (filtered_rows < step1_row) { + step2_ndv = scale_distinct(filtered_rows, step1_row, step1_ndv); + } else { + step2_ndv = step1_ndv; + } // set new column meta if (OB_SUCC(ret)) { column_meta.set_ndv(revise_ndv(step2_ndv)); @@ -1885,12 +2163,12 @@ int ObOptSelectivity::get_column_hist_scale(const OptTableMetas &table_metas, } int ObOptSelectivity::get_column_basic_info(const OptTableMetas &table_metas, - const OptSelectivityCtx &ctx, - const ObRawExpr &expr, - double *ndv_ptr, - double *num_null_ptr, - double *avg_len_ptr, - double *row_count_ptr) + const OptSelectivityCtx &ctx, + const ObRawExpr &expr, + double *ndv_ptr, + double *num_null_ptr, + double *avg_len_ptr, + double *row_count_ptr) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!expr.is_column_ref_expr())) { @@ -1903,6 +2181,8 @@ int ObOptSelectivity::get_column_basic_info(const OptTableMetas &table_metas, double num_null = 0; double avg_len = 0; double row_count = 0; + double cur_rowcnt = ctx.get_current_rows(); + if (OB_FAIL(get_column_basic_from_meta(table_metas, column_expr, need_default, @@ -1917,17 +2197,36 @@ int ObOptSelectivity::get_column_basic_info(const OptTableMetas &table_metas, if (num_null > row_count - ndv) { num_null = row_count - ndv > 0 ? row_count - ndv : 0; } - if (ctx.get_current_rows() > 0.0 && ctx.get_current_rows() < row_count) { - ndv = scale_distinct(ctx.get_current_rows(), row_count, ndv); + if (NULL != ctx.get_ambient_card() && + !ctx.get_ambient_card()->empty() && + ctx.check_opt_compat_version(COMPAT_VERSION_4_2_4, COMPAT_VERSION_4_3_0, + COMPAT_VERSION_4_3_3)) { + ObSEArray table_idx; + if (OB_FAIL(expr.get_relation_ids().to_array(table_idx))) { + LOG_WARN("failed to get table idx", K(ret), K(expr)); + } else if (OB_UNLIKELY(table_idx.count() != 1) || + OB_UNLIKELY(table_idx.at(0) < 1) || + OB_UNLIKELY(table_idx.at(0) >= ctx.get_ambient_card()->count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr relation ids", K(expr), K(table_idx), K(ctx)); + } else { + cur_rowcnt = ctx.get_ambient_card()->at(table_idx.at(0)); + } } - LOG_TRACE("show column basic info", K(row_count), K(ctx.get_current_rows()), K(num_null), K(avg_len), K(ndv)); + if (OB_SUCC(ret) && cur_rowcnt > 0.0 && cur_rowcnt < row_count) { + ndv = scale_distinct(cur_rowcnt, row_count, ndv); + } + + LOG_TRACE("show column basic info", K(row_count), K(cur_rowcnt), K(num_null), K(avg_len), K(ndv)); // set return - assign_value(row_count, row_count_ptr); - assign_value(ndv, ndv_ptr); - assign_value(num_null, num_null_ptr); - assign_value(avg_len, avg_len_ptr); + if (OB_SUCC(ret)) { + assign_value(row_count, row_count_ptr); + assign_value(ndv, ndv_ptr); + assign_value(num_null, num_null_ptr); + assign_value(avg_len, avg_len_ptr); + } } } return ret; @@ -2038,26 +2337,8 @@ int ObOptSelectivity::get_compare_value(const OptSelectivityCtx &ctx, can_cmp = false; } else if (expr_value.get_type() != col->get_result_type().get_type() || expr_value.get_collation_type() != col->get_result_type().get_collation_type()) { - const ObDataTypeCastParams dtc_params = - ObBasicSessionInfo::create_dtc_params(ctx.get_session_info()); - ObObj dest_value; - ObCastCtx cast_ctx(&ctx.get_allocator(), - &dtc_params, - CM_NONE, - col->get_result_type().get_collation_type()); - ObAccuracy res_acc; - if (col->get_result_type().is_decimal_int()) { - res_acc = col->get_result_type().get_accuracy(); - cast_ctx.res_accuracy_ = &res_acc; - } - if (OB_FAIL(ObObjCaster::to_type(col->get_result_type().get_type(), - col->get_result_type().get_collation_type(), - cast_ctx, - expr_value, - dest_value))) { - LOG_WARN("failed to cast value", K(ret)); - } else { - expr_value = dest_value; + if (OB_FAIL(convert_obj_to_expr_type(ctx, col, CM_NONE, expr_value))) { + LOG_WARN("failed to convert obj", K(ret), K(expr_value)); } } return ret; @@ -2431,6 +2712,142 @@ int ObOptSelectivity::get_simple_mutex_column(const ObRawExpr *qual, const ObRaw return ret; } +int ObOptSelectivity::calculate_table_ambient_cardinality(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + const ObRelIds &rel_ids, + const double cur_rows, + double &table_ambient_card) +{ + int ret = OB_SUCCESS; + if (NULL != ctx.get_ambient_card() && + !ctx.get_ambient_card()->empty()) { + table_ambient_card = 1.0; + for (int64_t i = 0; i < ctx.get_ambient_card()->count(); i ++) { + if (rel_ids.has_member(i)) { + table_ambient_card *= std::max(1.0, ctx.get_ambient_card()->at(i)); + } + } + } else { + ObSEArray table_ids; + const ObDMLStmt *stmt = ctx.get_stmt(); + table_ambient_card = 1.0; + if (OB_ISNULL(stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("null stmt", K(ret)); + } else if (OB_FAIL(stmt->relids_to_table_ids(rel_ids, table_ids))) { + LOG_WARN("faile to get table ids", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < table_ids.count(); i ++) { + table_ambient_card *= std::max(1.0, table_metas.get_rows(table_ids.at(i))); + } + } + if (cur_rows > 0) { + table_ambient_card = std::min(cur_rows, table_ambient_card); + } + return ret; +} + +int ObOptSelectivity::calculate_distinct_in_single_table(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + const ObRelIds &rel_id, + const common::ObIArray& exprs, + const double cur_rows, + double &rows) +{ + int ret = OB_SUCCESS; + rows = 1.0; + ObSEArray special_exprs; + ObSEArray expr_ndv; + ObSEArray filtered_exprs; + double ambient_card = -1.0; + //classify expr and get ndv + if (OB_UNLIKELY(rel_id.num_members() != 1)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected relation id", K(ret)); + } else if (OB_FAIL(calculate_table_ambient_cardinality(table_metas, ctx, rel_id, cur_rows, ambient_card))) { + LOG_WARN("failed to calculate ambient card", K(ret)); + } else if (OB_FAIL(filter_column_by_equal_set(table_metas, ctx, exprs, filtered_exprs))) { + LOG_WARN("failed filter column by equal set", K(ret)); + } else if (OB_FAIL(calculate_expr_ndv(filtered_exprs, expr_ndv, table_metas, ctx, ambient_card))) { + LOG_WARN("fail to calculate expr ndv", K(ret)); + } else if (!ctx.check_opt_compat_version(COMPAT_VERSION_4_2_4, COMPAT_VERSION_4_3_0, + COMPAT_VERSION_4_3_3)) { + for (int64_t i = 0; OB_SUCC(ret) && i < expr_ndv.count(); ++i) { + if (0 == i) { + rows *= expr_ndv.at(i); + } else { + rows *= expr_ndv.at(i) / std::sqrt(2); + } + } + } else { + rows = combine_ndvs(ambient_card, expr_ndv); + } + LOG_TRACE("succeed to calculate distinct in single table", K(rel_id), K(ambient_card), K(rows), K(expr_ndv), K(exprs)); + + return ret; +} + +int ObOptSelectivity::remove_dummy_distinct_exprs(ObIArray &helpers, + ObIArray &exprs) +{ + int ret = OB_SUCCESS; + ObSEArray new_exprs; + for (int64_t i = 0; OB_SUCC(ret) && i < exprs.count(); i ++) { + ObRawExpr *expr = exprs.at(i); + bool is_dummy = false; + if (OB_ISNULL(expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(exprs)); + } else if (expr->has_flag(CNT_WINDOW_FUNC) || + expr->is_column_ref_expr()) { + // do nothing + } else if (OB_FAIL(check_expr_in_distinct_helper(expr, helpers, is_dummy))) { + LOG_WARN("failed to check expr", K(ret)); + } + if (OB_SUCC(ret) && !is_dummy) { + if (OB_FAIL(new_exprs.push_back(expr))) { + LOG_WARN("failed to push back"); + } + } + } + if (OB_SUCC(ret) && new_exprs.count() != exprs.count()) { + LOG_DEBUG("remove dummy distinct exprs", K(exprs), K(new_exprs)); + if (OB_FAIL(exprs.assign(new_exprs))) { + LOG_WARN("failed to assign exprs", K(ret)); + } + } + return ret; +} + +int ObOptSelectivity::check_expr_in_distinct_helper(const ObRawExpr *expr, + const ObIArray &helpers, + bool &is_dummy_expr) +{ + int ret = OB_SUCCESS; + ObSEArray column_exprs; + bool found = false; + is_dummy_expr = true; + if (OB_FAIL(ObRawExprUtils::extract_column_exprs(expr, column_exprs))) { + LOG_WARN("failed to extract column exprs", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && is_dummy_expr && i < column_exprs.count(); i ++) { + ObRawExpr *col_expr = column_exprs.at(i); + if (OB_ISNULL(col_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(column_exprs)); + } + for (int64_t j = 0; OB_SUCC(ret) && !found && j < helpers.count(); j ++) { + if (col_expr->get_relation_ids() == helpers.at(j).rel_id_) { + found = ObOptimizerUtil::find_item(helpers.at(j).exprs_, col_expr); + } + } + if (!found) { + is_dummy_expr = false; + } + } + return ret; +} + int ObOptSelectivity::calculate_distinct(const OptTableMetas &table_metas, const OptSelectivityCtx &ctx, const ObIArray& exprs, @@ -2440,93 +2857,180 @@ int ObOptSelectivity::calculate_distinct(const OptTableMetas &table_metas, { int ret = OB_SUCCESS; rows = 1; - ObSEArray column_exprs; - ObSEArray special_exprs; - ObSEArray expr_ndv; - ObSEArray filtered_exprs; - //classify expr and get ndv - if (OB_FAIL(classify_exprs(exprs, column_exprs, special_exprs, table_metas, ctx))) { + ObSEArray single_ndvs; + ObSEArray helpers; + ObSEArray special_exprs; + /** + * 1. 将 exprs 根据基表分组，sepcial exprs 中保存不在基表计算的表达式，例如 window function 等 + * 2. 基表内计算 NDV 后（每张表的 NDV 最大值受基表行数限制），再根据当前行数计算联合 NDV + */ + if (OB_FAIL(classify_exprs(ctx, exprs, helpers, special_exprs))) { LOG_WARN("failed to classify_exprs", K(ret)); - } else if (OB_FAIL(filter_column_by_equal_set(table_metas, ctx, column_exprs, filtered_exprs))) { - LOG_WARN("failed filter column by equal set", K(ret)); - } else if (OB_FAIL(calculate_expr_ndv(filtered_exprs, expr_ndv, table_metas, ctx, origin_rows))) { + } else if (OB_FAIL(remove_dummy_distinct_exprs(helpers, special_exprs))) { + LOG_WARN("failed to remove dummy exprs", K(ret)); + } else if (OB_FAIL(calculate_expr_ndv(special_exprs, single_ndvs, table_metas, ctx, origin_rows))) { LOG_WARN("fail to calculate expr ndv", K(ret)); - } else if (OB_FAIL(calculate_expr_ndv(special_exprs, expr_ndv, table_metas, ctx, origin_rows))) { - LOG_WARN("fail to calculate special expr ndv", K(ret)); } - //calculate rows - for (int64_t i = 0; OB_SUCC(ret) && i < expr_ndv.count(); ++i) { - if (0 == i) { - rows *= expr_ndv.at(i); - } else { - rows *= expr_ndv.at(i) / std::sqrt(2); + for (int64_t i = 0; OB_SUCC(ret) && i < helpers.count(); i ++) { + OptDistinctHelper &helper = helpers.at(i); + double single_table_ndv = 1.0; + if (OB_FAIL(remove_dummy_distinct_exprs(helpers, helper.exprs_))) { + LOG_WARN("failed to remove dummy exprs", K(ret)); + } else if (OB_FAIL(calculate_distinct_in_single_table( + table_metas, ctx, helper.rel_id_, helper.exprs_, need_refine ? origin_rows : -1, single_table_ndv))) { + LOG_WARN("failed to calculate distinct in single table", K(helper.exprs_)); + } else if (OB_FAIL(single_ndvs.push_back(single_table_ndv))) { + LOG_WARN("failed to push back", K(ret)); } } - //refine - if (OB_SUCC(ret) && need_refine) { - rows = std::min(rows, origin_rows); - LOG_TRACE("succeed to calculate distinct", K(origin_rows), K(rows), K(exprs)); + if (!ctx.check_opt_compat_version(COMPAT_VERSION_4_2_4, COMPAT_VERSION_4_3_0, + COMPAT_VERSION_4_3_3)) { + for (int64_t i = 0; OB_SUCC(ret) && i < single_ndvs.count(); ++i) { + if (0 == i) { + rows *= single_ndvs.at(i); + } else { + rows *= single_ndvs.at(i) / std::sqrt(2); + } + } + //refine + if (OB_SUCC(ret) && need_refine && origin_rows >= 0.0) { + rows = std::min(rows, origin_rows); + } + } else { + rows = combine_ndvs(need_refine ? origin_rows : -1, single_ndvs); + } + LOG_TRACE("succeed to calculate distinct", K(ctx), K(origin_rows), K(rows), K(single_ndvs), K(exprs)); + return ret; +} + +int ObOptSelectivity::calculate_distinct(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + const ObRawExpr& expr, + const double origin_rows, + double &rows, + const bool need_refine) +{ + int ret = OB_SUCCESS; + ObSEArray expr_array; + if (OB_FAIL(expr_array.push_back(const_cast(&expr)))) { + LOG_WARN("failed to push back expr", K(ret)); + } else if (OB_FAIL(calculate_distinct(table_metas, ctx, expr_array, origin_rows, rows, need_refine))) { + LOG_WARN("failed to calculate distinct", K(expr)); } return ret; } -int ObOptSelectivity::classify_exprs(const ObIArray& exprs, - ObIArray& column_exprs, - ObIArray& special_exprs, - const OptTableMetas &table_metas, - const OptSelectivityCtx &ctx) +double ObOptSelectivity::combine_two_ndvs(double ambient_card, double ndv1, double ndv2) +{ + ndv1 = std::max(1.0, ndv1); + ndv2 = std::max(1.0, ndv2); + double max_ndv = std::max(ndv1, ndv2); + double min_ndv = std::min(ndv1, ndv2); + double combine_ndv = 1.0; + if (ambient_card >= 0.0) { + // due to the precision, we need to refine the result + combine_ndv = ndv1 * ndv2 * (1 - pow(1 - 1 / min_ndv , ambient_card / max_ndv)); + combine_ndv = std::max(combine_ndv, max_ndv); + combine_ndv = std::min(ambient_card, combine_ndv); + } else { + combine_ndv = ndv1 * ndv2 / std::sqrt(2); + combine_ndv = std::max(combine_ndv, max_ndv); + } + return combine_ndv; +} + +double ObOptSelectivity::combine_ndvs(double ambient_card, ObIArray &ndvs) +{ + double ndv = 1.0; + if (!ndvs.empty()) { + lib::ob_sort(&ndvs.at(0), &ndvs.at(0) + ndvs.count()); + ndv = ndvs.at(0); + for (int64_t i = 1; i < ndvs.count(); i ++) { + ndv = combine_two_ndvs(ambient_card, ndv, ndvs.at(i)); + } + } + if (ambient_card >= 0.0) { + ndv = std::min(ambient_card, ndv); + } + return ndv; +} + +int ObOptSelectivity::classify_exprs(const OptSelectivityCtx &ctx, + const ObIArray& exprs, + ObIArray &helpers, + ObIArray& special_exprs) { int ret = OB_SUCCESS; for (int64_t i = 0; OB_SUCC(ret) && i < exprs.count(); ++i) { - ObRawExpr *child_expr = NULL; - if (OB_ISNULL(child_expr = exprs.at(i))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get null expr", K(ret), K(i)); - } else if (OB_FAIL(classify_exprs(child_expr, column_exprs, special_exprs, table_metas, ctx))) { + if (OB_FAIL(classify_exprs(ctx, exprs.at(i), helpers, special_exprs))) { LOG_WARN("failed to classify_exprs", K(ret)); } } return ret; } -int ObOptSelectivity::classify_exprs(ObRawExpr* expr, - ObIArray& column_exprs, - ObIArray& special_exprs, - const OptTableMetas &table_metas, - const OptSelectivityCtx &ctx) +int ObOptSelectivity::classify_exprs(const OptSelectivityCtx &ctx, + ObRawExpr *expr, + ObIArray &helpers, + ObIArray& special_exprs) { int ret = OB_SUCCESS; + bool is_special = false; if (OB_ISNULL(expr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null pointer", K(expr), K(ret)); - } else if (is_special_expr(*expr)) { - if (OB_FAIL(add_var_to_array_no_dup(special_exprs, expr))) { - LOG_WARN("fail to add expr to array", K(ret)); + } else if (OB_FAIL(check_is_special_distinct_expr(ctx, expr, is_special))) { + LOG_WARN("failed to check expr", K(ret)); + } else if (is_special) { + if (expr->has_flag(CNT_WINDOW_FUNC) || + expr->get_relation_ids().num_members() != 1) { + if (OB_FAIL(add_var_to_array_no_dup(special_exprs, expr))) { + LOG_WARN("failed to push back", K(ret)); + } + } else { + if (OB_FAIL(add_expr_to_distinct_helper(helpers, expr->get_relation_ids(), expr))) { + LOG_WARN("failed to add expr to helper", K(ret)); + } } } else if (expr->is_column_ref_expr()) { - if (OB_FAIL(add_var_to_array_no_dup(column_exprs, expr))) { - LOG_WARN("fail to add expr to array", K(ret)); + if (OB_FAIL(add_expr_to_distinct_helper(helpers, expr->get_relation_ids(), expr))) { + LOG_WARN("failed to add expr to helper", K(ret)); } } else { for (int64_t i = 0; OB_SUCC(ret) && i < expr->get_param_count(); ++i) { - ObRawExpr *child_expr = NULL; - if (OB_ISNULL(child_expr = expr->get_param_expr(i))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get null expr", K(ret), K(i)); - } else if (OB_FAIL(classify_exprs(child_expr, column_exprs, special_exprs, table_metas, ctx))) { + if (OB_FAIL(SMART_CALL(classify_exprs(ctx, expr->get_param_expr(i), helpers, special_exprs)))) { LOG_WARN("failed to classify_exprs", K(ret)); } } } + LOG_DEBUG("succeed to classify distinct exprs", K(helpers), K(special_exprs)); return ret; } -bool ObOptSelectivity::is_special_expr(const ObRawExpr &expr) { - bool is_special = false; - if (expr.is_win_func_expr()) { - is_special = true; +int ObOptSelectivity::add_expr_to_distinct_helper(ObIArray &helpers, + const ObRelIds &rel_id, + ObRawExpr *expr) +{ + int ret = OB_SUCCESS; + bool found = false; + OptDistinctHelper *helper = NULL; + for (int64_t idx = 0; NULL == helper && idx < helpers.count(); idx ++) { + if (helpers.at(idx).rel_id_.equal(rel_id)) { + helper = &helpers.at(idx); + } } - return is_special; + if (NULL == helper) { + if (OB_ISNULL(helper = helpers.alloc_place_holder())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate", K(ret)); + } else if (OB_FAIL(helper->rel_id_.add_members(rel_id))) { + LOG_WARN("failed to add member", K(ret)); + } + } + if (OB_SUCC(ret) && OB_FAIL(add_var_to_array_no_dup(helper->exprs_, expr))) { + LOG_WARN("failed to push back", K(ret)); + } + return ret; } int ObOptSelectivity::calculate_expr_ndv(const ObIArray& exprs, @@ -2559,6 +3063,56 @@ int ObOptSelectivity::calculate_expr_ndv(const ObIArray& exprs, return ret; } +int ObOptSelectivity::check_is_special_distinct_expr(const OptSelectivityCtx &ctx, + const ObRawExpr *expr, + bool &is_special) +{ + int ret = OB_SUCCESS; + is_special = false; + if (OB_ISNULL(expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null pointer", K(expr), K(ret)); + } else if (expr->is_win_func_expr()) { + is_special = true; + } else if (!ctx.check_opt_compat_version(COMPAT_VERSION_4_2_4, COMPAT_VERSION_4_3_0, + COMPAT_VERSION_4_3_3)) { + is_special = false; + } else if (expr->is_const_expr()) { + is_special = false; + } else if (T_OP_MOD == expr->get_expr_type()) { + is_special = expr->get_param_count() == 2 && + OB_NOT_NULL(expr->get_param_expr(1)) && + expr->get_param_expr(1)->is_static_scalar_const_expr(); + } else if (T_FUN_SYS_SUBSTR == expr->get_expr_type() || T_FUN_SYS_SUBSTRB == expr->get_expr_type()) { + if (expr->get_param_count() == 2) { + is_special = OB_NOT_NULL(expr->get_param_expr(1)) && expr->get_param_expr(1)->is_static_scalar_const_expr(); + } else if (expr->get_param_count() == 3) { + is_special = OB_NOT_NULL(expr->get_param_expr(2)) && expr->get_param_expr(2)->is_static_scalar_const_expr(); + } + } else if (is_dense_time_expr_type(expr->get_expr_type()) || + T_FUN_SYS_MONTH_NAME == expr->get_expr_type() || + T_FUN_SYS_DAY_NAME == expr->get_expr_type()) { + is_special = expr->get_param_count() == 1; + } else if (T_FUN_SYS_EXTRACT == expr->get_expr_type()) { + is_special = expr->get_param_count() == 2 && + OB_NOT_NULL(expr->get_param_expr(0)) && + expr->get_param_expr(0)->is_static_scalar_const_expr(); + } else if (T_FUN_SYS_CAST == expr->get_expr_type()) { + const ObRawExpr *param_expr = NULL; + bool is_monotonic = false; + if (OB_UNLIKELY(expr->get_param_count() < 2) || + OB_ISNULL(param_expr = expr->get_param_expr(0))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr", KPC(expr)); + } else if (expr->get_data_type() != ObDateType) { + is_special = false; + } else if (OB_FAIL(ObObjCaster::is_cast_monotonic(param_expr->get_data_type(), expr->get_data_type(), is_special))) { + LOG_WARN("check cast monotonic error", KPC(expr), K(ret)); + } + } + return ret; +} + int ObOptSelectivity::calculate_special_ndv(const OptTableMetas &table_metas, const ObRawExpr* expr, const OptSelectivityCtx &ctx, @@ -2566,10 +3120,129 @@ int ObOptSelectivity::calculate_special_ndv(const OptTableMetas &table_metas, const double origin_rows) { int ret = OB_SUCCESS; - if (OB_ISNULL(expr)) { + const ObRawExpr *param_expr = NULL; + special_ndv = std::max(origin_rows, 1.0); + bool is_special = false; + bool need_refine_by_param_expr = true; + if (OB_FAIL(check_is_special_distinct_expr(ctx, expr, is_special))) { + LOG_WARN("failed to check expr", K(ret)); + } else if (OB_UNLIKELY(!is_special)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr", KPC(expr), K(ret)); + } else if (expr->is_win_func_expr()) { + if (OB_FAIL(calculate_winfunc_ndv(table_metas, expr, ctx, special_ndv, origin_rows))) { + LOG_WARN("failed to calculate windown function ndv", K(ret)); + } + need_refine_by_param_expr = false; + } else if (T_OP_MOD == expr->get_expr_type()) { + param_expr = expr->get_param_expr(0); + const ObRawExpr *const_expr = expr->get_param_expr(1); + bool valid = false; + if (OB_FAIL(calc_const_numeric_value(ctx, const_expr, special_ndv, valid))) { + LOG_WARN("failed to calc const value", K(ret)); + } else { + special_ndv = std::abs(special_ndv); + } + } else if (T_FUN_SYS_SUBSTR == expr->get_expr_type() || T_FUN_SYS_SUBSTRB == expr->get_expr_type()) { + double substr_len = 0.0; + double dummy = 0.0; + need_refine_by_param_expr = false; // substr ndv will not be greater than its param + if (OB_FAIL(calculate_expr_avg_len(table_metas, ctx, expr, substr_len))) { + LOG_WARN("failed to calc expr length", K(ret)); + } else if (OB_FAIL(calculate_substrb_info(table_metas, + ctx, + expr->get_param_expr(0), + substr_len - ObOptEstCostModel::DEFAULT_FIXED_OBJ_WIDTH, + origin_rows, + special_ndv, + dummy))) { + LOG_WARN("failed to calculate substr ndv", K(ret)); + } + } else if (is_dense_time_expr_type(expr->get_expr_type()) || + (T_FUN_SYS_CAST == expr->get_expr_type() && expr->get_data_type() == ObDateType) || + T_FUN_SYS_EXTRACT == expr->get_expr_type()) { + if (T_FUN_SYS_EXTRACT == expr->get_expr_type()) { + param_expr = expr->get_param_expr(1); + } else { + param_expr = expr->get_param_expr(0); + } + ObObj min_value; + ObObj max_value; + bool use_default = false; + double min_scalar = 0.0; + double max_scalar = 0.0; + if (OB_FAIL(calc_expr_min_max(table_metas, + ctx, + expr, + min_value, + max_value))) { + LOG_WARN("failed to calculate expr min max", K(ret)); + } else if (min_value.is_min_value() || max_value.is_max_value() || + !(min_value.is_integer_type() || min_value.is_number() || min_value.is_date()) || + !(max_value.is_integer_type() || max_value.is_number() || max_value.is_date())) { + use_default = true; + } else if (OB_FAIL(ObOptEstObjToScalar::convert_obj_to_double(&min_value, min_scalar)) || + OB_FAIL(ObOptEstObjToScalar::convert_obj_to_double(&max_value, max_scalar))) { + LOG_WARN("failed to convert obj to double", K(ret), K(min_value), K(max_value)); + } else { + special_ndv = max_scalar - min_scalar + 1; + } + if (OB_SUCC(ret) && !use_default) { + if (T_FUN_SYS_YEARWEEK_OF_DATE == expr->get_expr_type()) { + special_ndv *= 54.0 / 100.0; + } else if (T_FUN_SYS_EXTRACT == expr->get_expr_type()) { + ObObj result; + bool got_result = false; + if (OB_FAIL(ObSQLUtils::calc_const_or_calculable_expr(ctx.get_opt_ctx().get_exec_ctx(), + expr->get_param_expr(0), + result, + got_result, + ctx.get_allocator()))) { + LOG_WARN("fail to calc_const_or_calculable_expr", K(ret)); + } else if (!got_result || result.is_null() || !result.is_int()) { + // do nothing + } else if (DATE_UNIT_YEAR_MONTH == result.get_int()) { + special_ndv *= 12.0 / 100.0; + } + } + } + } else if (T_FUN_SYS_MONTH_NAME == expr->get_expr_type()) { + special_ndv = 12; + param_expr = expr->get_param_expr(0); + } else if (T_FUN_SYS_DAY_NAME == expr->get_expr_type()) { + special_ndv = 7; + param_expr = expr->get_param_expr(0); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected special expr", KPC(expr)); + } + if (OB_SUCC(ret) && need_refine_by_param_expr && NULL != param_expr) { + double ndv_upper_bound = 1.0; + if (OB_FAIL(SMART_CALL(calculate_distinct(table_metas, + ctx, + *param_expr, + origin_rows, + ndv_upper_bound)))) { + LOG_WARN("failed to calculate distinct", K(ret), KPC(param_expr)); + } else { + special_ndv = std::min(special_ndv, ndv_upper_bound); + } + } + special_ndv = revise_ndv(special_ndv); + return ret; +} + +int ObOptSelectivity::calculate_winfunc_ndv(const OptTableMetas &table_metas, + const ObRawExpr* expr, + const OptSelectivityCtx &ctx, + double &special_ndv, + const double origin_rows) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(expr) || OB_UNLIKELY(!expr->is_win_func_expr())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null pointer", K(expr), K(ret)); - } else if (expr->is_win_func_expr()) { + } else { double part_order_ndv = 1.0; double order_ndv = 1.0; double part_ndv = 1.0; @@ -2578,6 +3251,10 @@ int ObOptSelectivity::calculate_special_ndv(const OptTableMetas &table_metas, ObSEArray part_order_exprs; const ObWinFunRawExpr *win_expr = reinterpret_cast(expr); const ObIArray &order_items = win_expr->get_order_items(); + if (OB_UNLIKELY(origin_rows < 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("win function ndv depends on current rows", K(ret), K(origin_rows)); + } for (int64_t i = 0; OB_SUCC(ret) && i < order_items.count(); ++i) { const OrderItem &order_item = order_items.at(i); ObRawExpr *order_expr = order_item.expr_; @@ -2597,11 +3274,11 @@ int ObOptSelectivity::calculate_special_ndv(const OptTableMetas &table_metas, LOG_WARN("fail to assign exprs", K(ret)); } else if (OB_FAIL(append(part_order_exprs, order_exprs))) { LOG_WARN("failed to append exprs", K(ret)); - } else if (OB_FAIL(SMART_CALL(calculate_distinct(table_metas, ctx, part_order_exprs, origin_rows, part_order_ndv, false)))) { - LOG_WARN("failed to calculate_distinct", K(ret)); - } else if (OB_FAIL(SMART_CALL(calculate_distinct(table_metas, ctx, order_exprs, origin_rows, order_ndv, false)))) { + } else if (OB_FAIL(SMART_CALL(calculate_distinct(table_metas, ctx, part_order_exprs, origin_rows, part_order_ndv)))) { LOG_WARN("failed to calculate_distinct", K(ret)); - } else if (OB_FAIL(SMART_CALL(calculate_distinct(table_metas, ctx, part_exprs, origin_rows, part_ndv, false)))) { + } else if (OB_FAIL(SMART_CALL(calculate_distinct(table_metas, ctx, order_exprs, origin_rows, order_ndv)))) { + LOG_WARN("failed to calculate_distinct", K(ret)); + } else if (OB_FAIL(SMART_CALL(calculate_distinct(table_metas, ctx, part_exprs, origin_rows, part_ndv)))) { LOG_WARN("failed to calculate_distinct", K(ret)); } @@ -2620,13 +3297,12 @@ int ObOptSelectivity::calculate_special_ndv(const OptTableMetas &table_metas, ObRawExpr* const_expr = NULL; ObObj result,out_ptr; bool got_result = false; - const ParamStore *params = ctx.get_params(); if (OB_FAIL(param_exprs.assign(win_expr->get_func_params()))) { LOG_WARN("fail to assign exprs", K(ret)); } else if (param_exprs.count() == 0|| OB_ISNULL(const_expr = param_exprs.at(0))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected error", K(param_exprs.count()), K(const_expr), K(ret)); - } else if (ObOptEstUtils::is_calculable_expr(*const_expr, params->count())) { + } else if (const_expr->is_static_const_expr()) { if (OB_FAIL(ObSQLUtils::calc_const_or_calculable_expr(ctx.get_opt_ctx().get_exec_ctx(), const_expr, result, @@ -2653,7 +3329,7 @@ int ObOptSelectivity::calculate_special_ndv(const OptTableMetas &table_metas, LOG_WARN("unexpected null pointer", K(aggr_expr), K(ret)); } else if (OB_FAIL(param_exprs.assign(aggr_expr->get_real_param_exprs()))) { LOG_WARN("fail to assign exprs", K(ret)); - } else if (OB_FAIL(SMART_CALL(calculate_distinct(table_metas, ctx, param_exprs, origin_rows, param_ndv, false)))) { + } else if (OB_FAIL(SMART_CALL(calculate_distinct(table_metas, ctx, param_exprs, origin_rows, param_ndv)))) { LOG_WARN("failed to calculate_distinct", K(ret)); } else { special_ndv = std::min(part_order_ndv, param_ndv); @@ -2665,7 +3341,7 @@ int ObOptSelectivity::calculate_special_ndv(const OptTableMetas &table_metas, double param_ndv = 1.0; if (OB_FAIL(param_exprs.assign(win_expr->get_func_params()))) { LOG_WARN("fail to assign exprs", K(ret)); - } else if (OB_FAIL(SMART_CALL(calculate_distinct(table_metas, ctx, param_exprs, origin_rows, param_ndv, false)))) { + } else if (OB_FAIL(SMART_CALL(calculate_distinct(table_metas, ctx, param_exprs, origin_rows, param_ndv)))) { LOG_WARN("failed to calculate_distinct", K(ret)); } else { special_ndv = std::min(part_order_ndv, param_ndv); @@ -2676,7 +3352,7 @@ int ObOptSelectivity::calculate_special_ndv(const OptTableMetas &table_metas, double param_ndv = 1.0; if (OB_FAIL(param_exprs.assign(win_expr->get_func_params()))) { LOG_WARN("fail to assign exprs", K(ret)); - } else if (OB_FAIL(SMART_CALL(calculate_distinct(table_metas, ctx, param_exprs, origin_rows, param_ndv, false)))) { + } else if (OB_FAIL(SMART_CALL(calculate_distinct(table_metas, ctx, param_exprs, origin_rows, param_ndv)))) { LOG_WARN("failed to calculate_distinct", K(ret)); } else { special_ndv = param_ndv; @@ -2684,7 +3360,7 @@ int ObOptSelectivity::calculate_special_ndv(const OptTableMetas &table_metas, } else { special_ndv = part_order_ndv; } - LOG_TRACE("calculate win expr ndv", K(win_expr->get_func_type()), K(part_exprs.count()), K(order_exprs.count())); + LOG_TRACE("calculate window function ndv", KPC(win_expr), K(special_ndv)); } special_ndv = revise_ndv(special_ndv); return ret; @@ -2717,7 +3393,7 @@ int ObOptSelectivity::filter_column_by_equal_set(const OptTableMetas &table_meta LOG_WARN("failed to find the expr with min ndv", K(ret)); } else if (!find && FALSE_IT(filtered_expr = column_exprs.at(i))) { // never reach - } else if (OB_FAIL(filtered_exprs.push_back(filtered_expr))) { + } else if (OB_FAIL(add_var_to_array_no_dup(filtered_exprs, filtered_expr))) { LOG_WARN("failed to push back expr", K(ret)); } } @@ -2768,9 +3444,11 @@ int ObOptSelectivity::get_min_ndv_by_equal_set(const OptTableMetas &table_metas, ObBitSet<> col_added; find = false; const EqualSets *eq_sets = ctx.get_equal_sets(); - if (OB_ISNULL(eq_sets)) { + if (OB_ISNULL(eq_sets) || OB_ISNULL(col_expr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret)); + } else if (!col_expr->is_column_ref_expr()) { + // do nothing } else { for (int64_t i = 0; OB_SUCC(ret) && !find && i < eq_sets->count(); i++) { const ObRawExprSet *equal_set = eq_sets->at(i); @@ -2825,17 +3503,20 @@ int ObOptSelectivity::get_min_ndv_by_equal_set(const OptTableMetas &table_metas, } int ObOptSelectivity::is_columns_contain_pkey(const OptTableMetas &table_metas, - const ObIArray &col_exprs, + const ObIArray &exprs, bool &is_pkey, - bool &is_union_pkey) + bool &is_union_pkey, + uint64_t *table_id_ptr) { int ret = OB_SUCCESS; ObSEArray col_ids; - uint64_t table_id; - if (OB_FAIL(extract_column_ids(col_exprs, col_ids, table_id))) { + uint64_t table_id = OB_INVALID_INDEX; + if (OB_FAIL(extract_column_ids(exprs, col_ids, table_id))) { LOG_WARN("failed to extract column ids", K(ret)); } else if (OB_FAIL(is_columns_contain_pkey(table_metas, col_ids, table_id, is_pkey, is_union_pkey))) { LOG_WARN("failed to check is columns contain pkey", K(ret)); + } else { + assign_value(table_id, table_id_ptr); } return ret; } @@ -2878,25 +3559,28 @@ int ObOptSelectivity::extract_column_ids(const ObIArray &col_exprs, int ret = OB_SUCCESS; ObColumnRefRawExpr *column_expr = NULL; table_id = OB_INVALID_INDEX; - for (int64_t i = 0; OB_SUCC(ret) && i < col_exprs.count(); ++i) { + bool from_same_table = true; + for (int64_t i = 0; OB_SUCC(ret) && from_same_table && i < col_exprs.count(); ++i) { ObRawExpr *cur_expr = col_exprs.at(i); - if (OB_ISNULL(cur_expr) || OB_UNLIKELY(!cur_expr->is_column_ref_expr())) { + if (OB_ISNULL(cur_expr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected expr", K(ret)); + } else if (!cur_expr->is_column_ref_expr()) { + // do nothing } else if (FALSE_IT(column_expr = static_cast(cur_expr))) { } else if (OB_FAIL(col_ids.push_back(column_expr->get_column_id()))) { LOG_WARN("failed to push back column id", K(ret)); - } else if (0 == i) { + } else if (OB_INVALID_INDEX == table_id) { table_id = column_expr->get_table_id(); - } else if (OB_UNLIKELY(table_id != column_expr->get_table_id())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("columns not belong to same table", K(ret), K(*column_expr), K(table_id)); + } else if (table_id != column_expr->get_table_id()) { + from_same_table = false; + table_id = OB_INVALID_INDEX; } } return ret; } -int ObOptSelectivity::classify_quals(const OptSelectivityCtx &ctx, +int ObOptSelectivity::classify_quals_deprecated(const OptSelectivityCtx &ctx, const ObIArray &quals, ObIArray &all_predicate_sel, ObIArray &column_sel_infos) @@ -2906,9 +3590,14 @@ int ObOptSelectivity::classify_quals(const OptSelectivityCtx &ctx, ObSEArray column_exprs; OptSelInfo *sel_info = NULL; double tmp_selectivity = 1.0; - ObArenaAllocator tmp_alloc("ObOptSel"); - ObSelEstimatorFactory factory(tmp_alloc); + ObSelEstimatorFactory factory; ObSEArray range_estimators; + if (OB_ISNULL(ctx.get_session_info())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else { + factory.get_allocator().set_tenant_id(ctx.get_session_info()->get_effective_tenant_id()); + } for (int64_t i = 0; OB_SUCC(ret) && i < quals.count(); ++i) { column_exprs.reset(); uint64_t column_id = OB_INVALID_ID; @@ -2974,7 +3663,7 @@ int ObOptSelectivity::classify_quals(const OptSelectivityCtx &ctx, ObObj obj_min; ObObj obj_max; if (OB_ISNULL(range_estimator = static_cast(range_estimators.at(i))) || - OB_UNLIKELY(ObSelEstType::RANGE != range_estimator->get_type()) || + OB_UNLIKELY(ObSelEstType::COLUMN_RANGE != range_estimator->get_type()) || OB_ISNULL(column_expr = range_estimator->get_column_expr())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected expr", K(ret)); @@ -2995,6 +3684,92 @@ int ObOptSelectivity::classify_quals(const OptSelectivityCtx &ctx, return ret; } +int ObOptSelectivity::classify_quals(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + const ObIArray &quals, + ObIArray &all_predicate_sel, + ObIArray &column_sel_infos) +{ + int ret = OB_SUCCESS; + ObRawExpr *qual = NULL; + ObSEArray column_exprs; + OptSelInfo *sel_info = NULL; + double tmp_selectivity = 1.0; + ObSelEstimatorFactory factory; + ObSEArray estimators; + ObObj obj_min; + ObObj obj_max; + if (OB_ISNULL(ctx.get_session_info())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else { + factory.get_allocator().set_tenant_id(ctx.get_session_info()->get_effective_tenant_id()); + } + for (int64_t i = 0; OB_SUCC(ret) && i < quals.count(); ++i) { + column_exprs.reset(); + uint64_t column_id = OB_INVALID_ID; + ObColumnRefRawExpr *column_expr = NULL; + ObSelEstimator *range_estimator = NULL; + uint64_t temp_equal_count = 0; + sel_info = NULL; + if (OB_ISNULL(qual = quals.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected expr", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::extract_column_exprs(qual, column_exprs))) { + LOG_WARN("failed to extract column exprs", K(ret)); + } else if (1 == column_exprs.count()) { + column_expr = static_cast(column_exprs.at(0)); + column_id = column_expr->get_column_id(); + if (OB_FAIL(get_opt_sel_info(column_sel_infos, column_expr->get_column_id(), sel_info))) { + LOG_WARN("failed to get opt sel info", K(ret)); + } else if (OB_FAIL(sel_info->quals_.push_back(qual))) { + LOG_WARN("failed to push back", K(ret)); + } else if (OB_FAIL(extract_equal_count(*qual, temp_equal_count))) { + LOG_WARN("failed to extract equal count", K(ret)); + } else if (0 == sel_info->equal_count_) { + sel_info->equal_count_ = temp_equal_count; + } else if (temp_equal_count > 0) { + sel_info->equal_count_ = std::min(sel_info->equal_count_, temp_equal_count); + } + } else { + // use OB_INVALID_ID represent qual contain more than one column + } + } + + for (int64_t i = 0; OB_SUCC(ret) && i < column_sel_infos.count(); ++i) { + estimators.reuse(); + OptSelInfo &sel_info = column_sel_infos.at(i); + ObRangeSelEstimator *range_estimator = NULL; + obj_min.set_min_value(); + obj_max.set_max_value(); + if (OB_FAIL(factory.create_estimators(ctx, sel_info.quals_, estimators))) { + LOG_WARN("failed to create estimators", K(ret)); + } else if (OB_FAIL(calculate_selectivity(table_metas, ctx, estimators, sel_info.selectivity_))) { + LOG_WARN("failed to calc sel", K(ret)); + } + for (int64_t j = 0; OB_SUCC(ret) && NULL == range_estimator && j < estimators.count(); j ++) { + if (OB_ISNULL(estimators.at(j))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(sel_info), K(estimators)); + } else if (ObSelEstType::COLUMN_RANGE == estimators.at(j)->get_type()) { + range_estimator = static_cast(estimators.at(j)); + if (OB_FAIL(ObOptSelectivity::get_column_range_min_max( + ctx, range_estimator->get_column_expr(), range_estimator->get_range_exprs(), obj_min, obj_max))) { + LOG_WARN("failed to get min max", K(ret)); + } else { + if (!obj_min.is_null()) { + sel_info.min_ = obj_min; + } + if (!obj_max.is_null()) { + sel_info.max_ = obj_max; + } + } + } + } + } + return ret; +} + int ObOptSelectivity::get_opt_sel_info(ObIArray &column_sel_infos, const uint64_t column_id, OptSelInfo *&sel_info) @@ -3108,6 +3883,7 @@ double ObOptSelectivity::scale_distinct(double selected_rows, if (ndv > OB_DOUBLE_EPSINON && rows > OB_DOUBLE_EPSINON) { new_ndv = ndv * (1 - std::pow(1 - selected_rows / rows, rows / ndv)); } + new_ndv = std::min(new_ndv, selected_rows); } new_ndv = revise_ndv(new_ndv); return new_ndv; @@ -3262,33 +4038,6 @@ int ObOptSelectivity::get_join_pred_rows(const ObHistogram &left_hist, // return ret; // } -double ObOptSelectivity::get_filters_selectivity(ObIArray &selectivities, FilterDependencyType type) -{ - double selectivity = 0.0; - if (FilterDependencyType::INDEPENDENT == type) { - selectivity = 1.0; - for (int64_t i = 0; i < selectivities.count(); i ++) { - selectivity *= selectivities.at(i); - } - } else if (FilterDependencyType::MUTEX_OR == type) { - selectivity = 0.0; - for (int64_t i = 0; i < selectivities.count(); i ++) { - selectivity += selectivities.at(i); - } - } else if (FilterDependencyType::EXPONENTIAL_BACKOFF == type) { - selectivity = 1.0; - if (!selectivities.empty()) { - double exp = 1.0; - lib::ob_sort(&selectivities.at(0), &selectivities.at(0) + selectivities.count()); - for (int64_t i = 0; i < selectivities.count(); i ++) { - selectivity *= std::pow(selectivities.at(i), exp); - exp /= 2; - } - } - } - selectivity = revise_between_0_1(selectivity); - return selectivity; -} int ObOptSelectivity::remove_ignorable_func_for_est_sel(const ObRawExpr *&expr) { @@ -3365,5 +4114,492 @@ double ObOptSelectivity::get_set_stmt_output_count(double count1, double count2, return output_count; } +int ObOptSelectivity::calculate_expr_avg_len(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + const ObRawExpr *expr, + double &avg_len) +{ + int ret = OB_SUCCESS; + // default + avg_len = ObOptEstCost::get_estimate_width_from_type(expr->get_result_type()); + const OptDynamicExprMeta *dynamic_expr_meta = NULL; + if (OB_ISNULL(expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null expr", K(ret)); + } else if (expr->is_column_ref_expr()) { + if (OB_FAIL(get_column_avg_len(table_metas, ctx, expr, avg_len))) { + LOG_WARN("failed to get avg len", K(ret)); + } + } else if (expr->is_static_scalar_const_expr()) { + ObObj value; + bool get_value = false; + if (OB_FAIL(ObSQLUtils::calc_const_or_calculable_expr(ctx.get_opt_ctx().get_exec_ctx(), + expr, + value, + get_value, + ctx.get_allocator()))) { + LOG_WARN("Failed to get const or calculable expr value", K(ret)); + } else if (get_value) { + avg_len = ObOptEstCostModel::DEFAULT_FIXED_OBJ_WIDTH + value.get_deep_copy_size(); + } + } else if (T_OP_CNN == expr->get_expr_type()) { + avg_len = ObOptEstCostModel::DEFAULT_FIXED_OBJ_WIDTH; + for (int64_t i = 0; OB_SUCC(ret) && i < expr->get_param_count(); i ++) { + double child_len = 0; + if (OB_FAIL(SMART_CALL(calculate_expr_avg_len(table_metas, ctx, expr->get_param_expr(i), child_len)))) { + LOG_WARN("failed to calc child avg len", K(ret), KPC(expr)); + } else { + avg_len += child_len - ObOptEstCostModel::DEFAULT_FIXED_OBJ_WIDTH; + } + } + } else if (expr->is_sys_func_expr()) { + if (T_FUN_SYS_REPLACE == expr->get_expr_type() || + (T_FUN_SYS_CAST == expr->get_expr_type() && CM_IS_IMPLICIT_CAST(expr->get_extra()) )) { + if (OB_UNLIKELY(expr->get_param_count() < 1)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr param", KPC(expr)); + } else if (OB_FAIL(SMART_CALL(calculate_expr_avg_len(table_metas, ctx, expr->get_param_expr(0), avg_len)))) { + LOG_WARN("failed to calc child avg len", K(ret), KPC(expr)); + } + } else if (T_FUN_SYS_SUBSTR == expr->get_expr_type() || T_FUN_SYS_SUBSTRB == expr->get_expr_type()) { + double pos = 1; + double sub_len = -1; + double child_len = 0; + ObObj value; + bool get_value = false; + if (OB_UNLIKELY(expr->get_param_count() < 2) || + OB_ISNULL(expr->get_param_expr(1)) || + (expr->get_param_count() == 3 && OB_ISNULL(expr->get_param_expr(2)))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr param", KPC(expr)); + } else if (OB_FAIL(SMART_CALL(calculate_expr_avg_len(table_metas, ctx, expr->get_param_expr(0), child_len)))) { + LOG_WARN("failed to calc child avg len", K(ret), KPC(expr)); + } else if (expr->get_param_expr(1)->is_static_scalar_const_expr()) { + if (OB_FAIL(ObSQLUtils::calc_const_or_calculable_expr(ctx.get_opt_ctx().get_exec_ctx(), + expr->get_param_expr(1), + value, + get_value, + ctx.get_allocator()))) { + LOG_WARN("Failed to get const or calculable expr value", K(ret)); + } else if (!get_value) { + // do nothing + } else if (OB_FAIL(ObOptEstObjToScalar::convert_obj_to_double(&value, pos))) { + LOG_WARN("failed to convert obj to double", K(ret)); + } + } + if (OB_FAIL(ret)) { + } else if (3 == expr->get_param_count() && expr->get_param_expr(2)->is_static_scalar_const_expr()) { + if (OB_FAIL(ObSQLUtils::calc_const_or_calculable_expr(ctx.get_opt_ctx().get_exec_ctx(), + expr->get_param_expr(2), + value, + get_value, + ctx.get_allocator()))) { + LOG_WARN("Failed to get const or calculable expr value", K(ret)); + } else if (!get_value) { + // do nothing + } else if (OB_FAIL(ObOptEstObjToScalar::convert_obj_to_double(&value, sub_len))) { + LOG_WARN("failed to convert obj to double", K(ret)); + } + } + avg_len = child_len; + if (OB_SUCC(ret)) { + avg_len -= ObOptEstCostModel::DEFAULT_FIXED_OBJ_WIDTH; + if (pos <= 0) { + avg_len = std::min(avg_len, -pos); + } else if (pos >= 1) { + avg_len -= pos - 1; + } + if (sub_len >= 0) { + avg_len = std::min(avg_len, sub_len); + } + avg_len += ObOptEstCostModel::DEFAULT_FIXED_OBJ_WIDTH; + } + } + } else if (OB_NOT_NULL(dynamic_expr_meta = table_metas.get_dynamic_expr_meta(expr))) { + avg_len = dynamic_expr_meta->get_avg_len(); + } else if (expr->is_exec_param_expr()) { + if (OB_FAIL(SMART_CALL(calculate_expr_avg_len( + table_metas, ctx, static_cast(expr)->get_ref_expr(), avg_len)))) { + LOG_WARN("failed to calc ref avg len", K(ret), KPC(expr)); + } + } else { /*do nothing*/ } + return ret; +} + +int ObOptSelectivity::get_column_avg_len(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + const ObRawExpr *expr, + double &avg_len) +{ + int ret = OB_SUCCESS; + const ObColumnRefRawExpr *column_expr = static_cast(expr); + if (OB_ISNULL(expr) || OB_UNLIKELY(!expr->is_column_ref_expr()) || + OB_ISNULL(ctx.get_opt_stat_manager()) || + OB_ISNULL(ctx.get_session_info())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null param", K(ret), KPC(expr), K(ctx.get_opt_stat_manager())); + } else { + uint64_t table_id = column_expr->get_table_id(); + uint64_t column_id = column_expr->get_column_id(); + uint64_t ref_table_id; + ObSEArray part_ids; + ObSEArray global_part_ids; + const OptTableMeta *table_meta = table_metas.get_table_meta_by_table_id(table_id); + ObGlobalColumnStat stat; + TableItem *table = NULL; + avg_len = ObOptEstCost::get_estimate_width_from_type(expr->get_result_type()); + if (OB_NOT_NULL(table_meta)) { + const OptColumnMeta *column_meta = table_meta->get_column_meta(column_id); + if (OB_NOT_NULL(column_meta)) { + avg_len = column_meta->get_avg_len(); + } + } + } + return ret; +} + +int ObOptSelectivity::calculate_substrb_info(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + const ObRawExpr *str_expr, + const double substrb_len, + const double cur_rows, + double &substr_ndv, + double &nns) +{ + int ret = OB_SUCCESS; + double expr_len = 0.0; + double expr_ndv = 0.0; + nns = 1.0; + substr_ndv = 1.0; + if (OB_ISNULL(str_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (substrb_len <= 0) { + substr_ndv = 1.0; + } else if (OB_FAIL(calculate_expr_avg_len(table_metas, + ctx, + str_expr, + expr_len))) { + LOG_WARN("failed to get expr length", K(ret), KPC(str_expr)); + } else if (OB_FAIL(calculate_distinct(table_metas, + ctx, + *str_expr, + cur_rows, + expr_ndv))) { + LOG_WARN("failed to calculate distinct", K(ret)); + } else if (OB_FAIL(calculate_expr_nns(table_metas, ctx, str_expr, nns))) { + LOG_WARN("failed to calculate nns", KPC(str_expr)); + } else { + expr_len -= ObOptEstCostModel::DEFAULT_FIXED_OBJ_WIDTH; + if (nns > OB_DOUBLE_EPSINON) { + expr_len /= nns; + } + if (expr_len < OB_DOUBLE_EPSINON || expr_len <= substrb_len) { + substr_ndv = expr_ndv; + } else { + substr_ndv = std::pow(expr_ndv, substrb_len / expr_len); + } + } + LOG_TRACE("succeed to calculate substrb ndv", K(substr_ndv), K(expr_ndv), K(substrb_len), K(expr_len)); + return ret; +} + +int ObOptSelectivity::calculate_expr_nns(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + const ObRawExpr *expr, + double &nns) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (expr->is_column_ref_expr()) { + if (OB_FAIL(get_column_ndv_and_nns(table_metas, ctx, *expr, NULL, &nns))) { + LOG_WARN("failed to get column ndv and nns", K(ret)); + } + } else { + // todo: wuyuming.wym null propagate expr + nns = 1.0; + } + return ret; +} + +int ObOptSelectivity::calc_expr_min_max(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + const ObRawExpr *expr, + ObObj &min_value, + ObObj &max_value) +{ + int ret = OB_SUCCESS; + min_value.set_min_value(); + max_value.set_max_value(); + if (OB_ISNULL(expr) || OB_ISNULL(ctx.get_opt_ctx().get_exec_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (expr->get_result_type().is_ext()) { + // do nothing + } else if (expr->is_column_ref_expr()) { + if (OB_FAIL(ObOptSelectivity::get_column_min_max(table_metas, ctx, *expr, + min_value, max_value))) { + LOG_WARN("failed to get min max", K(ret)); + } + } else if (expr->is_static_scalar_const_expr()) { + ObObj result; + bool got_result = false; + if (OB_FAIL(ObSQLUtils::calc_const_or_calculable_expr(ctx.get_opt_ctx().get_exec_ctx(), + expr, + result, + got_result, + ctx.get_allocator()))) { + LOG_WARN("fail to calc_const_or_calculable_expr", K(ret)); + } else if (!got_result || result.is_null()) { + // do nothing + } else { + min_value = result; + max_value = result; + } + } else if (expr->get_param_count() < 1) { + // do nothing + } else if (T_FUN_SYS_CAST == expr->get_expr_type()) { + bool is_monotonic = false; + const ObRawExpr *param_expr = expr->get_param_expr(0); + if (OB_FAIL(ObObjCaster::is_cast_monotonic(param_expr->get_data_type(), expr->get_data_type(), is_monotonic))) { + LOG_WARN("check cast monotonic error", KPC(expr), K(ret)); + } else if (!is_monotonic) { + // do nothing + } else if (OB_FAIL(SMART_CALL(calc_expr_min_max(table_metas, ctx, param_expr, + min_value, max_value)))) { + LOG_WARN("failed to calc date min max", KPC(expr)); + } else if (min_value.is_min_value() || max_value.is_min_value() || + min_value.is_max_value() || max_value.is_max_value() || + min_value.is_null() || max_value.is_null()) { + // do nothing + } else if (OB_FAIL(convert_obj_to_expr_type(ctx, expr, expr->get_extra(), min_value))) { + ret = OB_SUCCESS; + min_value.set_min_value(); + } else if (OB_FAIL(convert_obj_to_expr_type(ctx, expr, expr->get_extra(), max_value))) { + ret = OB_SUCCESS; + max_value.set_max_value(); + } + } else if (T_FUN_SYS_DATE == expr->get_expr_type()) { + if (OB_FAIL(SMART_CALL(calc_expr_min_max(table_metas, ctx, expr->get_param_expr(0), + min_value, max_value)))) { + LOG_WARN("failed to calc date min max", KPC(expr)); + } + } else if (is_dense_time_expr_type(expr->get_expr_type()) || + T_FUN_SYS_EXTRACT == expr->get_expr_type()) { + const ObRawExpr *param_expr = NULL; + ObItemType est_type = expr->get_expr_type(); + int64_t extract_type = DATE_UNIT_MAX; + if (T_FUN_SYS_EXTRACT == expr->get_expr_type()) { + bool valid = false; + ObObj result; + param_expr = expr->get_param_expr(1); + if (OB_FAIL(ObSQLUtils::calc_const_or_calculable_expr(ctx.get_opt_ctx().get_exec_ctx(), + expr->get_param_expr(0), + result, + valid, + ctx.get_allocator()))) { + LOG_WARN("fail to calc_const_or_calculable_expr", K(ret)); + } else if (valid && result.is_int()) { + extract_type = result.get_int(); + switch (extract_type) { + case DATE_UNIT_DAY: est_type = T_FUN_SYS_DAY; break; + case DATE_UNIT_WEEK: est_type = T_FUN_SYS_WEEK; break; + case DATE_UNIT_MONTH: est_type = T_FUN_SYS_MONTH; break; + case DATE_UNIT_QUARTER: est_type = T_FUN_SYS_QUARTER; break; + case DATE_UNIT_SECOND: est_type = T_FUN_SYS_SECOND; break; + case DATE_UNIT_MINUTE: est_type = T_FUN_SYS_MINUTE; break; + case DATE_UNIT_HOUR: est_type = T_FUN_SYS_HOUR; break; + case DATE_UNIT_YEAR: est_type = T_FUN_SYS_YEAR; break; + // we only estimate the min/max value by year, so yearmonth is similar with yearweek + case DATE_UNIT_YEAR_MONTH: est_type = T_FUN_SYS_YEARWEEK_OF_DATE; break; + default: break; + } + } + } else { + param_expr = expr->get_param_expr(0); + } + if (OB_SUCC(ret)) { + bool use_default = false; + int64_t min_int_value = 0; + int64_t max_int_value = 0; + switch (est_type) { + case T_FUN_SYS_MONTH: min_int_value = 1; max_int_value = 12; break; + case T_FUN_SYS_DAY_OF_MONTH: + case T_FUN_SYS_DAY: min_int_value = 1; max_int_value = 31; break; + case T_FUN_SYS_DAY_OF_YEAR: min_int_value = 1; max_int_value = 366; break; + case T_FUN_SYS_WEEK_OF_YEAR: + case T_FUN_SYS_WEEK: min_int_value = 0; max_int_value = 53; break; + case T_FUN_SYS_WEEKDAY_OF_DATE: min_int_value = 0; max_int_value = 6; break; + case T_FUN_SYS_DAY_OF_WEEK: min_int_value = 1; max_int_value = 7; break; + case T_FUN_SYS_QUARTER: min_int_value = 1; max_int_value = 4; break; + case T_FUN_SYS_HOUR: min_int_value = 0; max_int_value = 23; break; + case T_FUN_SYS_MINUTE: min_int_value = 0; max_int_value = 59; break; + case T_FUN_SYS_SECOND: min_int_value = 0; max_int_value = 59; break; + case T_FUN_SYS_YEAR: + case T_FUN_SYS_YEARWEEK_OF_DATE: { + if (OB_FAIL(calc_year_min_max(table_metas, + ctx, + param_expr, + min_int_value, + max_int_value, + use_default))) { + LOG_WARN("failed to calculate expr min max", K(ret)); + } else if (!use_default && T_FUN_SYS_YEARWEEK_OF_DATE == est_type) { + // approximately + min_int_value = min_int_value * 100; + max_int_value = max_int_value * 100 + 100; + } + break; + } + default: use_default = true; break; + } + if (OB_SUCC(ret) && !use_default) { + min_value.set_int(min_int_value); + max_value.set_int(max_int_value); + if (OB_FAIL(convert_obj_to_expr_type(ctx, expr, CM_NONE, min_value))) { + LOG_WARN("failed to convert obj", K(ret), K(min_value)); + } else if (OB_FAIL(convert_obj_to_expr_type(ctx, expr, CM_NONE, min_value))) { + LOG_WARN("failed to convert obj", K(ret), K(max_value)); + } + } + } + } + if (OB_SUCC(ret)) { + int cmp_result = 0; + bool can_cmp = min_value.can_compare(max_value); + if (min_value.is_min_value() || max_value.is_max_value()) { + // do nothing + } else if (can_cmp && + OB_FAIL(min_value.compare(max_value, cmp_result))) { + LOG_WARN("failed to compare", K(ret)); + } else if (!can_cmp || 1 == cmp_result || + min_value.is_null() || max_value.is_null()) { + min_value.set_min_value(); + max_value.set_max_value(); + } + } + return ret; +} + +int ObOptSelectivity::calc_year_min_max(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + const ObRawExpr *expr, + int64_t &min_year, + int64_t &max_year, + bool &use_default) +{ + int ret = OB_SUCCESS; + ObObj min_value; + ObObj max_value; + ObObj min_year_obj; + ObObj max_year_obj; + ObTime min_time; + ObTime max_time; + use_default = false; + ObDateSqlMode date_sql_mode; + if (OB_ISNULL(expr) || OB_ISNULL(ctx.get_session_info()) || OB_ISNULL(ctx.get_opt_ctx().get_exec_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (ObDateTimeTC != expr->get_type_class() && + ObDateTC != expr->get_type_class() && + ObOTimestampTC != expr->get_type_class()) { + use_default = true; + } else if (OB_FAIL(SMART_CALL(calc_expr_min_max(table_metas, + ctx, + expr, + min_value, + max_value)) )) { + LOG_WARN("failed to calculate expr min max", K(ret)); + } else if (min_value.is_min_value() || max_value.is_max_value()) { + use_default = true; + } else if (FALSE_IT(date_sql_mode.init(ctx.get_session_info()->get_sql_mode()))) { + } else if (OB_FAIL(ob_obj_to_ob_time_with_date(min_value, + get_timezone_info(ctx.get_session_info()), + min_time, + get_cur_time(ctx.get_opt_ctx().get_exec_ctx()->get_physical_plan_ctx()), + date_sql_mode))) { + ret = OB_SUCCESS; + use_default = true; + } else if (OB_FAIL(ob_obj_to_ob_time_with_date(max_value, + get_timezone_info(ctx.get_session_info()), + max_time, + get_cur_time(ctx.get_opt_ctx().get_exec_ctx()->get_physical_plan_ctx()), + date_sql_mode))) { + ret = OB_SUCCESS; + use_default = true; + } else { + min_year = min_time.parts_[DT_YEAR]; + max_year = max_time.parts_[DT_YEAR]; + } + return ret; +} + +int ObOptSelectivity::calc_const_numeric_value(const OptSelectivityCtx &ctx, + const ObRawExpr *expr, + double &value, + bool &succ) +{ + int ret = OB_SUCCESS; + ObObj result; + bool got_result = false; + if (OB_ISNULL(expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (!expr->is_static_scalar_const_expr()) { + succ = false; + } else if (OB_FAIL(ObSQLUtils::calc_const_or_calculable_expr(ctx.get_opt_ctx().get_exec_ctx(), + expr, + result, + got_result, + ctx.get_allocator()))) { + LOG_WARN("fail to calc_const_or_calculable_expr", K(ret)); + } else if (!got_result || result.is_null() || !ob_is_numeric_type(result.get_type())) { + succ = false; + } else if (OB_FAIL(ObOptEstObjToScalar::convert_obj_to_double(&result, value))) { + LOG_WARN("Failed to convert obj using old method", K(ret)); + } else { + succ = true; + } + return ret; +} + +int ObOptSelectivity::convert_obj_to_expr_type(const OptSelectivityCtx &ctx, + const ObRawExpr *expr, + ObCastMode cast_mode, + ObObj &obj) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(expr) || OB_ISNULL(ctx.get_opt_ctx().get_exec_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", KPC(expr)); + } else { + ObObj tmp; + const ObDataTypeCastParams dtc_params = + ObBasicSessionInfo::create_dtc_params(ctx.get_session_info()); + ObCastCtx cast_ctx(&ctx.get_allocator(), + &dtc_params, + get_cur_time(ctx.get_opt_ctx().get_exec_ctx()->get_physical_plan_ctx()), + cast_mode, + expr->get_result_type().get_collation_type()); + ObAccuracy res_acc; + if (expr->get_result_type().is_decimal_int()) { + res_acc = expr->get_result_type().get_accuracy(); + cast_ctx.res_accuracy_ = &res_acc; + } + if (OB_FAIL(ObObjCaster::to_type(expr->get_result_type().get_type(), + expr->get_result_type().get_collation_type(), + cast_ctx, + obj, + tmp))) { + LOG_WARN("failed to cast value", K(ret)); + } else { + obj = tmp; + } + } + return ret; +} + }//end of namespace sql }//end of namespace oceanbase diff --git a/src/sql/optimizer/ob_opt_selectivity.h b/src/sql/optimizer/ob_opt_selectivity.h index a6b818810..d88dd99b4 100644 --- a/src/sql/optimizer/ob_opt_selectivity.h +++ b/src/sql/optimizer/ob_opt_selectivity.h @@ -51,11 +51,72 @@ struct ColumnItem; struct RangeExprs; struct ObExprSelPair; -enum class FilterDependencyType +class ObEstCorrelationModel { - INDEPENDENT, - MUTEX_OR, - EXPONENTIAL_BACKOFF, +public: + static ObEstCorrelationModel &get_correlation_model(ObEstCorrelationType type); + + virtual double combine_filters_selectivity(ObIArray &selectivities) const = 0; + + virtual bool is_independent() const = 0; + +protected: + ObEstCorrelationModel() {} + virtual ~ObEstCorrelationModel() = default; + +private: + DISALLOW_COPY_AND_ASSIGN(ObEstCorrelationModel); +}; + +class ObIndependentModel : public ObEstCorrelationModel +{ +public: + static ObEstCorrelationModel& get_model(); + + virtual double combine_filters_selectivity(ObIArray &selectivities) const override; + + virtual bool is_independent() const override { return true; }; + +protected: + ObIndependentModel() {} + virtual ~ObIndependentModel() = default; + +private: + DISALLOW_COPY_AND_ASSIGN(ObIndependentModel); +}; + +class ObPartialCorrelationModel : public ObEstCorrelationModel +{ +public: + static ObEstCorrelationModel& get_model(); + + virtual double combine_filters_selectivity(ObIArray &selectivities) const override; + + virtual bool is_independent() const { return false; } + +protected: + ObPartialCorrelationModel() {} + virtual ~ObPartialCorrelationModel() = default; + +private: + DISALLOW_COPY_AND_ASSIGN(ObPartialCorrelationModel); +}; + +class ObFullCorrelationModel : public ObEstCorrelationModel +{ +public: + static ObEstCorrelationModel& get_model(); + + virtual double combine_filters_selectivity(ObIArray &selectivities) const override; + + virtual bool is_independent() const { return false; } + +protected: + ObFullCorrelationModel() {} + virtual ~ObFullCorrelationModel() = default; + +private: + DISALLOW_COPY_AND_ASSIGN(ObFullCorrelationModel); }; class OptSelectivityCtx @@ -72,7 +133,8 @@ class OptSelectivityCtx row_count_1_(-1.0), row_count_2_(-1.0), current_rows_(-1.0), - dependency_type_(FilterDependencyType::INDEPENDENT) + ambient_card_(NULL), + assumption_type_(UNKNOWN_JOIN) { } ObOptimizerContext &get_opt_ctx() const { return const_cast(opt_ctx_); } @@ -108,11 +170,16 @@ class OptSelectivityCtx const ObRelIds *get_right_rel_ids() const { return right_rel_ids_; } double get_row_count_1() const { return row_count_1_; } double get_row_count_2() const { return row_count_2_; } + double get_left_row_count() const { return row_count_1_; } + double get_right_row_count() const { return row_count_2_; } double get_current_rows() const { return current_rows_; } void set_current_rows(const double current_rows) { current_rows_ = current_rows; } - FilterDependencyType get_dependency_type() const { return dependency_type_; } - void set_dependency_type(FilterDependencyType type) { dependency_type_ = type; } + + const ObEstCorrelationModel &get_correlation_model() const + { + return ObEstCorrelationModel::get_correlation_model(opt_ctx_.get_correlation_type()); + } uint64_t get_compat_version() const { return OB_ISNULL(opt_ctx_.get_query_ctx()) ? 0 : @@ -126,20 +193,29 @@ class OptSelectivityCtx get_opt_ctx().get_query_ctx()->check_opt_compat_version(args...); } + void set_ambient_card(const ObIArray *ambient_card) { ambient_card_ = ambient_card; } + const ObIArray *get_ambient_card() const { return ambient_card_; } + int get_ambient_card(const uint64_t table_id, double &table_ambient_card) const; + int get_ambient_card(const ObRelIds &rel_ids, double &table_ambient_card) const; + void set_assumption_type(ObJoinType type) { assumption_type_ = type; } + ObJoinType get_assumption_type() const { + return UNKNOWN_JOIN == assumption_type_ ? join_type_ : assumption_type_; + } + void init_op_ctx(const EqualSets *equal_sets, const double current_rows, - FilterDependencyType dependency_type = FilterDependencyType::INDEPENDENT) + const ObIArray *ambient_card = NULL) { + join_type_ = UNKNOWN_JOIN; + left_rel_ids_ = NULL; + right_rel_ids_ = NULL; equal_sets_ = equal_sets; current_rows_ = current_rows; - dependency_type_ = dependency_type; - } - void init_row_count(const double row_count1, const double row_count2) - { - row_count_1_ = row_count1; - row_count_2_ = row_count2; - dependency_type_ = FilterDependencyType::INDEPENDENT; + ambient_card_ = ambient_card; } + // child should be in the same query block + void init_op_ctx(ObLogicalOperator *child); + void init_join_ctx(const ObJoinType join_type, const ObRelIds *left_rel_ids, const ObRelIds *right_rel_ids, const double rc1, const double rc2, const EqualSets *equal_sets = NULL) @@ -151,13 +227,31 @@ class OptSelectivityCtx row_count_2_ = rc2; current_rows_ = -1.0; equal_sets_ = equal_sets; - dependency_type_ = FilterDependencyType::INDEPENDENT; + ambient_card_ = NULL; } - void clear_equal_sets() { equal_sets_ = NULL; } + void clear() + { + join_type_ = UNKNOWN_JOIN; + left_rel_ids_ = NULL; + right_rel_ids_ = NULL; + equal_sets_ = NULL; + current_rows_ = -1; + ambient_card_ = NULL; + } + + void init_row_count(const double row_count1, const double row_count2) + { + join_type_ = UNKNOWN_JOIN; + left_rel_ids_ = NULL; + right_rel_ids_ = NULL; + row_count_1_ = row_count1; + row_count_2_ = row_count2; + ambient_card_ = NULL; + } TO_STRING_KV(KP_(stmt), KP_(equal_sets), K_(join_type), KP_(left_rel_ids), KP_(right_rel_ids), - K_(row_count_1), K_(row_count_2), K_(current_rows), K_(dependency_type)); + K_(row_count_1), K_(row_count_2), K_(current_rows), KPC_(ambient_card)); private: ObOptimizerContext &opt_ctx_; @@ -178,7 +272,13 @@ class OptSelectivityCtx double row_count_1_; double row_count_2_; double current_rows_; - FilterDependencyType dependency_type_; + const ObIArray *ambient_card_; + + /** + * The join type which determines the estimation assumption. + * Used to calculate the selectivity of ambient card. + */ + ObJoinType assumption_type_; }; class OptColumnMeta @@ -265,6 +365,30 @@ enum OptTableStatType { DS_TABLE_STAT //dynamic sampling table stat }; +class OptDynamicExprMeta +{ +public: + OptDynamicExprMeta(): avg_len_(0) {} + void set_expr(const ObRawExpr *expr) { expr_ = expr; } + const ObRawExpr *get_expr() const { return expr_; } + void set_avg_len(double avg_len) { avg_len_ = avg_len; } + double get_avg_len() const { return avg_len_; } + + int assign(const OptDynamicExprMeta &other) + { + int ret = OB_SUCCESS; + expr_ = other.expr_; + avg_len_ = other.avg_len_; + return ret; + } + + TO_STRING_KV(KP_(expr), KPC_(expr), K_(avg_len)); +private: + const ObRawExpr *expr_; + double avg_len_; + DISALLOW_COPY_AND_ASSIGN(OptDynamicExprMeta); +}; + class OptTableMeta { public: @@ -286,7 +410,8 @@ public: distinct_rows_(0.0), table_partition_info_(NULL), base_meta_info_(NULL), - real_rows_(-1.0) + real_rows_(-1.0), + stale_stats_(false) {} int assign(const OptTableMeta &other); @@ -348,7 +473,8 @@ public: void set_stat_locked(bool locked) { stat_locked_ = locked; } double get_distinct_rows() const { return distinct_rows_; } void set_distinct_rows(double rows) { distinct_rows_ = rows; } - void set_ndv_for_all_column(double ndv); + bool is_opt_stat_expired() const { return stale_stats_; } + void set_stale_stats(bool stale_stats) { stale_stats_ = stale_stats; } share::schema::ObTableType get_table_type() const { return table_type_; } @@ -389,6 +515,8 @@ private: const ObTablePartitionInfo *table_partition_info_; const ObTableMetaInfo *base_meta_info_; double real_rows_; + //mark stat is expired + bool stale_stats_; }; struct OptSelectivityDSParam { @@ -423,7 +551,8 @@ public: int64_t last_analyzed, bool is_stat_locked, const ObTablePartitionInfo *table_partition_info, - const ObTableMetaInfo *base_meta_info); + const ObTableMetaInfo *base_meta_info, + bool stale_stats); int add_set_child_stmt_meta_info(const ObSelectStmt *parent_stmt, const ObSelectStmt *child_stmt, @@ -453,15 +582,22 @@ public: double &ndv); common::ObIArray& get_table_metas() { return table_metas_; } + const common::ObIArray& get_table_metas() const { return table_metas_; } const OptTableMeta* get_table_meta_by_table_id(const uint64_t table_id) const; OptTableMeta* get_table_meta_by_table_id(const uint64_t table_id); const OptColumnMeta* get_column_meta_by_table_id(const uint64_t table_id, const uint64_t column_id) const; + const OptDynamicExprMeta* get_dynamic_expr_meta(const ObRawExpr *expr) const; + int add_dynamic_expr_meta(const OptDynamicExprMeta &dynamic_expr_meta) { + return dynamic_expr_metas_.push_back(dynamic_expr_meta); + } + const ObIArray &get_dynamic_expr_metas() const { return dynamic_expr_metas_; } - int get_rows(const uint64_t table_id, double &rows); - TO_STRING_KV(K_(table_metas)); + double get_rows(const uint64_t table_id) const; + TO_STRING_KV(K_(table_metas), K_(dynamic_expr_metas)); private: common::ObSEArray table_metas_; + common::ObSEArray dynamic_expr_metas_; }; struct OptSelInfo @@ -487,13 +623,30 @@ struct OptSelInfo bool has_range_exprs_; ObObj min_; ObObj max_; + ObSEArray quals_; }; class ObSelEstimator; +struct OptDistinctHelper +{ +public: + OptDistinctHelper() {} + + TO_STRING_KV(K_(rel_id), K_(exprs)); + + ObRelIds rel_id_; + ObSEArray exprs_; +}; + class ObOptSelectivity { public: + static int calculate_selectivity(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + ObIArray &sel_estimators, + double &selectivity); + // @brief 计算一组条件的选择率，条件之间是and关系，基于独立性假设 static int calculate_selectivity(const OptTableMetas &table_metas, const OptSelectivityCtx &ctx, @@ -501,6 +654,20 @@ public: double &selectivity, common::ObIArray &all_predicate_sel); + static int calculate_conditional_selectivity(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + common::ObIArray &total_filters, + common::ObIArray &append_filters, + double &total_sel, + double &conditional_sel, + ObIArray &all_predicate_sel); + + static int calculate_join_selectivity(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + const common::ObIArray &quals, + double &selectivity, + common::ObIArray &all_predicate_sel); + static int calculate_qual_selectivity(const OptTableMetas &table_metas, const OptSelectivityCtx &ctx, const ObRawExpr &qual, @@ -528,6 +695,26 @@ public: const ObNewRange &range, double &selectivity); + static int calculate_table_ambient_cardinality(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + const ObRelIds &rel_id, + const double cur_rows, + double &table_ambient_card); + + static int calculate_distinct_in_single_table(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + const ObRelIds &rel_ids, + const common::ObIArray& exprs, + const double cur_rows, + double &rows); + + static int remove_dummy_distinct_exprs(ObIArray &helpers, + ObIArray &exprs); + + static int check_expr_in_distinct_helper(const ObRawExpr *expr, + const ObIArray &helpers, + bool &is_dummy_expr); + // @brief 计算一组变量的distinct static int calculate_distinct(const OptTableMetas &table_metas, const OptSelectivityCtx &ctx, @@ -536,6 +723,17 @@ public: double &rows, const bool need_refine = true); + static int calculate_distinct(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + const ObRawExpr& expr, + const double origin_rows, + double &rows, + const bool need_refine = true); + + static double combine_two_ndvs(double ambient_card, double ndv1, double ndv2); + + static double combine_ndvs(double ambient_card, ObIArray &ndvs); + // ndv 按照行数进行缩放. static double scale_distinct(double selected_rows, double rows, double ndv); @@ -642,8 +840,8 @@ public: const ObRawExpr &expr, double *ndv_ptr, double *num_null_ptr, - double *row_count_ptr, - double *avg_len_ptr); + double *avg_len_ptr, + double *row_count_ptr); static int get_column_hist_scale(const OptTableMetas &table_metas, const OptSelectivityCtx &ctx, @@ -751,9 +949,10 @@ public: * 检查一组expr是否包含所在表的主键 */ static int is_columns_contain_pkey(const OptTableMetas &table_metas, - const ObIArray &col_exprs, + const ObIArray &exprs, bool &is_pkey, - bool &is_union_pkey); + bool &is_union_pkey, + uint64_t *table_id_ptr = NULL); static int is_columns_contain_pkey(const OptTableMetas &table_metas, const ObIArray &col_ids, @@ -768,7 +967,13 @@ public: ObIArray &col_ids, uint64_t &table_id); - static int classify_quals(const OptSelectivityCtx &ctx, + static int classify_quals_deprecated(const OptSelectivityCtx &ctx, + const ObIArray &quals, + ObIArray &all_predicate_sel, + ObIArray &column_sel_infos); + + static int classify_quals(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, const ObIArray &quals, ObIArray &all_predicate_sel, ObIArray &column_sel_infos); @@ -825,7 +1030,6 @@ public: // const ObIArray &predicates, // ObOptDSJoinParam &ds_join_param); - static double get_filters_selectivity(ObIArray &selectivities, FilterDependencyType type); static int get_column_min_max(ObRawExpr *expr, OptSelInfo &sel_info); @@ -834,27 +1038,91 @@ public: const OptSelectivityCtx &ctx, double &special_ndv, const double origin_rows); + static int calculate_winfunc_ndv(const OptTableMetas &table_meta, + const ObRawExpr* expr, + const OptSelectivityCtx &ctx, + double &special_ndv, + const double origin_rows); static int calculate_expr_ndv(const ObIArray& exprs, ObIArray& expr_ndv, const OptTableMetas &table_metas, const OptSelectivityCtx &ctx, const double origin_rows); - static bool is_special_expr(const ObRawExpr &expr); - static int classify_exprs(const ObIArray& exprs, - ObIArray& column_exprs, - ObIArray& special_exprs, - const OptTableMetas &table_metas, - const OptSelectivityCtx &ctx); - static int classify_exprs(ObRawExpr* expr, - ObIArray& column_exprs, - ObIArray& special_exprs, - const OptTableMetas &table_metas, - const OptSelectivityCtx &ctx); + static int check_is_special_distinct_expr(const OptSelectivityCtx &ctx, + const ObRawExpr *expr, + bool &is_special); + static int classify_exprs(const OptSelectivityCtx &ctx, + const ObIArray& exprs, + ObIArray &helpers, + ObIArray& special_exprs); + static int classify_exprs(const OptSelectivityCtx &ctx, + ObRawExpr *expr, + ObIArray &helpers, + ObIArray& special_exprs); + static int add_expr_to_distinct_helper(ObIArray &helpers, + const ObRelIds &rel_id, + ObRawExpr *expr); static int remove_ignorable_func_for_est_sel(const ObRawExpr *&expr); static int remove_ignorable_func_for_est_sel(ObRawExpr *&expr); static double get_set_stmt_output_count(double count1, double count2, ObSelectStmt::SetOperator set_type); + static int calculate_expr_avg_len(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + const ObRawExpr *expr, + double &avg_len); + static int get_column_avg_len(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + const ObRawExpr *expr, + double &avg_len); + static int calculate_substrb_info(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + const ObRawExpr *str_expr, + const double substrb_len, + const double cur_rows, + double &ndv, + double &nns); + static int calculate_expr_nns(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + const ObRawExpr *expr, + double &nns); + static int calc_expr_min_max(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + const ObRawExpr *expr, + ObObj &min_value, + ObObj &max_value); + static int calc_year_min_max(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + const ObRawExpr *expr, + int64_t &min_year, + int64_t &max_year, + bool &use_default); + static int calc_const_numeric_value(const OptSelectivityCtx &ctx, + const ObRawExpr *expr, + double &value, + bool &succ); + static int convert_obj_to_expr_type(const OptSelectivityCtx &ctx, + const ObRawExpr *expr, + ObCastMode cast_mode, + ObObj &obj); + static bool is_dense_time_expr_type(ObItemType type) + { + return T_FUN_SYS_YEAR == type || + T_FUN_SYS_DAY == type || + T_FUN_SYS_DAY_OF_MONTH == type || + T_FUN_SYS_MONTH == type || + T_FUN_SYS_DAY_OF_YEAR == type || + T_FUN_SYS_WEEK_OF_YEAR == type || + T_FUN_SYS_WEEKDAY_OF_DATE == type || + T_FUN_SYS_YEARWEEK_OF_DATE == type || + T_FUN_SYS_DAY_OF_WEEK == type || + T_FUN_SYS_WEEK == type || + T_FUN_SYS_QUARTER == type || + T_FUN_SYS_HOUR == type || + T_FUN_SYS_MINUTE == type || + T_FUN_SYS_SECOND == type; + } + private: DISALLOW_COPY_AND_ASSIGN(ObOptSelectivity); }; diff --git a/src/sql/optimizer/ob_optimizer.cpp b/src/sql/optimizer/ob_optimizer.cpp index 93ba34c37..89dea29cc 100644 --- a/src/sql/optimizer/ob_optimizer.cpp +++ b/src/sql/optimizer/ob_optimizer.cpp @@ -543,6 +543,8 @@ int ObOptimizer::init_env_info(ObDMLStmt &stmt) LOG_WARN("fail to check enable pdml", K(ret)); } else if (OB_FAIL(init_parallel_policy(stmt, *session_info))) { // call after check pdml enabled LOG_WARN("fail to check enable pdml", K(ret)); + } else if (OB_FAIL(init_correlation_model(stmt, *session_info))) { + LOG_WARN("failed to init correlation model", K(ret)); } return ret; } @@ -692,6 +694,39 @@ int ObOptimizer::init_parallel_policy(ObDMLStmt &stmt, const ObSQLSessionInfo &s return ret; } +int ObOptimizer::init_correlation_model(ObDMLStmt &stmt, const ObSQLSessionInfo &session) +{ + int ret = OB_SUCCESS; + ObEstCorrelationModel* correlation_model = NULL; + int64_t type = 0; + bool has_hint = false; + if (OB_ISNULL(ctx_.get_query_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null ctx", K(ret)); + } else if (!ctx_.get_query_ctx()->check_opt_compat_version(COMPAT_VERSION_4_2_4, COMPAT_VERSION_4_3_0, + COMPAT_VERSION_4_3_3)) { + type = static_cast(ObEstCorrelationType::INDEPENDENT); + } else if (OB_FAIL(ctx_.get_global_hint().opt_params_.has_opt_param(ObOptParamHint::CORRELATION_FOR_CARDINALITY_ESTIMATION, has_hint))) { + LOG_WARN("failed to check whether has hint param", K(ret)); + } else if (has_hint) { + if (OB_FAIL(ctx_.get_global_hint().opt_params_.get_enum_opt_param(ObOptParamHint::CORRELATION_FOR_CARDINALITY_ESTIMATION, type))) { + LOG_WARN("failed to get bool hint param", K(ret)); + } + } else if (OB_FAIL(session.get_sys_variable(share::SYS_VAR_CARDINALITY_ESTIMATION_MODEL, type))) { + LOG_WARN("failed to get sys variable", K(ret)); + } + if (OB_SUCC(ret)) { + if (OB_UNLIKELY(type < 0) || + OB_UNLIKELY(type >= static_cast(ObEstCorrelationType::MAX))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected correlation type", K(type)); + } else { + ctx_.set_correlation_type(static_cast(type)); + } + } + return ret; +} + int ObOptimizer::set_auto_dop_params(const ObSQLSessionInfo &session) { int ret = OB_SUCCESS; diff --git a/src/sql/optimizer/ob_optimizer.h b/src/sql/optimizer/ob_optimizer.h index 078a513df..24bdcfe94 100644 --- a/src/sql/optimizer/ob_optimizer.h +++ b/src/sql/optimizer/ob_optimizer.h @@ -223,6 +223,7 @@ namespace sql int check_force_default_stat(); int init_system_stat(); int calc_link_stmt_count(const ObDMLStmt &stmt, int64_t &count); + int init_correlation_model(ObDMLStmt &stmt, const ObSQLSessionInfo &session); private: ObOptimizerContext &ctx_; diff --git a/src/sql/optimizer/ob_optimizer_context.h b/src/sql/optimizer/ob_optimizer_context.h index c7ee0c02e..8e0ed049a 100644 --- a/src/sql/optimizer/ob_optimizer_context.h +++ b/src/sql/optimizer/ob_optimizer_context.h @@ -41,6 +41,14 @@ namespace sql class ObRawExprFactory; class ObLogPlanFactory; +enum class ObEstCorrelationType +{ + INDEPENDENT, + PARTIAL, + FULL, + MAX +}; + typedef common::ObArray ObPlanNotes; //table location local index id related info //tablet_loc_id and ref_table_id_ are used to uniquely determine @@ -235,7 +243,8 @@ ObOptimizerContext(ObSQLSessionInfo *session_info, system_stat_(), storage_estimation_enabled_(false), das_keep_order_enabled_(true), - generate_random_plan_(false) + generate_random_plan_(false), + correlation_type_(ObEstCorrelationType::MAX) { } inline common::ObOptStatManager *get_opt_stat_manager() { return opt_stat_manager_; } inline void set_opt_stat_manager(common::ObOptStatManager *sm) { opt_stat_manager_ = sm; } @@ -610,6 +619,8 @@ ObOptimizerContext(ObSQLSessionInfo *session_info, inline bool generate_random_plan() const { return generate_random_plan_; } inline void set_generate_random_plan(bool rand_plan) { generate_random_plan_ = rand_plan; } + inline void set_correlation_type(ObEstCorrelationType type) { correlation_type_ = type; } + inline ObEstCorrelationType get_correlation_type() const { return correlation_type_; } private: ObSQLSessionInfo *session_info_; ObExecContext *exec_ctx_; @@ -696,6 +707,7 @@ private: bool das_keep_order_enabled_; bool generate_random_plan_; + ObEstCorrelationType correlation_type_; }; } } diff --git a/src/sql/optimizer/ob_optimizer_util.cpp b/src/sql/optimizer/ob_optimizer_util.cpp index f90dc325f..05b60f3da 100644 --- a/src/sql/optimizer/ob_optimizer_util.cpp +++ b/src/sql/optimizer/ob_optimizer_util.cpp @@ -3205,6 +3205,18 @@ int ObOptimizerUtil::get_onetime_exprs(ObRawExpr* expr, return ret; } +int ObOptimizerUtil::get_onetime_exprs(ObIArray &exprs, + ObIArray &onetime_exprs) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < exprs.count(); i++) { + if (OB_FAIL(get_onetime_exprs(exprs.at(i), onetime_exprs))) { + LOG_WARN("failed to get onetime exprs", K(ret)); + } + } + return ret; +} + int ObOptimizerUtil::get_query_ref_exprs(ObIArray &exprs, ObIArray &subqueries, ObIArray &nested_subqueries) diff --git a/src/sql/optimizer/ob_optimizer_util.h b/src/sql/optimizer/ob_optimizer_util.h index 9a35e431e..67b595aac 100644 --- a/src/sql/optimizer/ob_optimizer_util.h +++ b/src/sql/optimizer/ob_optimizer_util.h @@ -611,6 +611,9 @@ public: static int get_onetime_exprs(ObRawExpr* expr, ObIArray &onetime_exprs); + static int get_onetime_exprs(ObIArray &exprs, + ObIArray &onetime_exprs); + static int get_query_ref_exprs(ObIArray &subquery_exprs, ObIArray &subqueries, ObIArray &nested_subqueries); diff --git a/src/sql/optimizer/ob_sel_estimator.cpp b/src/sql/optimizer/ob_sel_estimator.cpp index 293fbbb37..e41fd2fe8 100644 --- a/src/sql/optimizer/ob_sel_estimator.cpp +++ b/src/sql/optimizer/ob_sel_estimator.cpp @@ -38,6 +38,189 @@ namespace sql { inline double revise_ndv(double ndv) { return ndv < 1.0 ? 1.0 : ndv; } +void SimpleRange::set_whole_range() +{ + start_.set_min_value(); + end_.set_max_value(); + inclusive_start_ = false; + inclusive_end_ = false; +} + +void SimpleRange::set_false_range() +{ + start_.set_max_value(); + end_.set_min_value(); + inclusive_start_ = false; + inclusive_end_ = false; +} + +int SimpleRange::compare_with_end(const SimpleRange &r) const +{ + int cmp = 0; + if (end_.is_max_value()) { + if (!r.end_.is_max_value()) { + cmp = 1; + } + } else if (r.end_.is_max_value()) { + cmp = -1; + } else { + cmp = end_.compare(r.end_); + if (0 == cmp) { + if (inclusive_end_ && !r.inclusive_end_) { + cmp = 1; + } else if (!inclusive_end_ && r.inclusive_end_) { + cmp = -1; + } + } + } + return cmp; +} + +int SimpleRange::compare_with_start(const SimpleRange &r) const +{ + int cmp = 0; + if (start_.is_min_value()) { + if (!r.start_.is_min_value()) { + cmp = -1; + } + } else if (r.start_.is_min_value()) { + cmp = 1; + } else { + cmp = start_.compare(r.start_); + if (0 == cmp) { + if (inclusive_start_ && !r.inclusive_start_) { + cmp = -1; + } else if (!inclusive_start_ && r.inclusive_start_) { + cmp = 1; + } + } + } + return cmp; +} + +bool SimpleRange::intersect(const SimpleRange &r) +{ + bool bret = false; + if (start_.can_compare(r.start_) && end_.can_compare(r.end_)) { + bret = true; + int cmp_start = compare_with_start(r); + if (cmp_start == -1) { + start_ = r.start_; + inclusive_start_ = r.inclusive_start_; + } + int cmp_end = compare_with_end(r); + if (cmp_end == 1) { + end_ = r.end_; + inclusive_end_ = r.inclusive_end_; + } + } + return bret; +} + +void SimpleRange::set_bound(ObItemType item_type, ObObj bound) +{ + if (bound.is_null()) { + if (T_OP_IS == item_type || T_OP_NSEQ == item_type) { + start_.set_null(); + end_.set_null(); + inclusive_start_ = true; + inclusive_end_ = true; + } else if (T_OP_IS_NOT == item_type) { + set_whole_range(); + } else { + set_false_range(); + } + } else if (T_OP_LE == item_type) { + end_ = bound; + inclusive_end_ = true; + } else if (T_OP_LT == item_type) { + end_ = bound; + inclusive_end_ = false; + } else if (T_OP_GE == item_type) { + start_ = bound; + inclusive_start_ = true; + } else if (T_OP_GT == item_type) { + start_ = bound; + inclusive_start_ = false; + } else if (T_OP_EQ == item_type || T_OP_NSEQ == item_type) { + start_ = bound; + end_ = bound; + inclusive_start_ = true; + inclusive_end_ = true; + } +} + +void SimpleRange::set_bound(ObItemType item_type, double bound) +{ + ObObj obj; + obj.set_double(bound); + set_bound(item_type, obj); +} + +bool SimpleRange::is_valid_range() +{ + bool bret = false; + if (!start_.can_compare(end_)) { + bret = false; + } else if (start_.is_null() && end_.is_null()) { + bret = true; + } else if (start_.is_max_value() || end_.is_min_value() || + start_.is_null() || end_.is_null()) { + bret = false; + } else if (start_.is_min_value() || end_.is_max_value()) { + bret = true; + } else { + int cmp = start_.compare(end_); + if (-1 == cmp) { + bret = true; + } else if (1 == cmp) { + bret = false; + } else if (0 == cmp) { + if (inclusive_start_ && inclusive_end_) { + bret = true; + } else { + bret = false; + } + } + } + return bret; +} + +bool SimpleRange::is_superset(const SimpleRange &r) const +{ + bool bret = false; + if (start_.can_compare(r.start_) && end_.can_compare(r.end_)) { + int cmp1 = compare_with_start(r); + int cmp2 = compare_with_end(r); + bret = cmp1 <= 0 && cmp2 >= 0; + } + return bret; +} + +void SimpleRange::multiply_double(double coff) +{ + if (start_.is_double()) { + start_.set_double(start_.get_double() * coff); + } + if (end_.is_double()) { + end_.set_double(end_.get_double() * coff); + } + if (coff < 0) { + std::swap(start_, end_); + std::swap(inclusive_start_, inclusive_end_); + if (start_.is_min_value()) { + start_.set_max_value(); + } else if (start_.is_max_value()) { + start_.set_min_value(); + } + if (end_.is_min_value()) { + end_.set_max_value(); + } else if (end_.is_max_value()) { + end_.set_min_value(); + } + } +} + int ObSelEstimator::append_estimators(ObIArray &sel_estimators, ObSelEstimator *new_estimator) { int ret = OB_SUCCESS; @@ -281,46 +464,26 @@ int ObInSelEstimator::get_in_sel(const OptTableMetas &table_metas, return ret; } -int ObIsSelEstimator::get_is_sel(const OptTableMetas &table_metas, - const OptSelectivityCtx &ctx, - const ObRawExpr &qual, - double &selectivity) +int ObIsSelEstimator::get_sel(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + double &selectivity, + ObIArray &all_predicate_sel) { int ret = OB_SUCCESS; selectivity = DEFAULT_SEL; - const ParamStore *params = ctx.get_params(); - const ObDMLStmt *stmt = ctx.get_stmt(); - const ObRawExpr *left_expr = qual.get_param_expr(0); - const ObRawExpr *right_expr = qual.get_param_expr(1); - ObObj result; - bool got_result = false; - if (OB_ISNULL(params) || OB_ISNULL(stmt) || OB_ISNULL(left_expr) || OB_ISNULL(right_expr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpect null", K(ret), K(params), K(stmt), K(left_expr), K(right_expr)); - } else if (OB_UNLIKELY(!ObOptEstUtils::is_calculable_expr(*right_expr, params->count()))) { - // do nothing - } else if (OB_FAIL(ObSQLUtils::calc_const_or_calculable_expr(ctx.get_opt_ctx().get_exec_ctx(), - right_expr, - result, - got_result, - ctx.get_allocator()))) { - LOG_WARN("failed to calculate const or calculable expr", K(ret)); - } else if (!got_result) { - // do nothing - } else if (OB_FAIL(ObOptSelectivity::remove_ignorable_func_for_est_sel(left_expr))) { - LOG_WARN("failed to remove ignorable func", KPC(left_expr)); - } else if (left_expr->is_column_ref_expr()) { - if (OB_FAIL(ObOptSelectivity::check_column_in_current_level_stmt(stmt, *left_expr))) { - LOG_WARN("Failed to check column whether is in current stmt", K(ret)); - } else if (OB_LIKELY(result.is_null())) { - if (OB_FAIL(ObOptSelectivity::get_column_basic_sel(table_metas, ctx, *left_expr, NULL, &selectivity))) { + if (can_calc_sel_) { + if (OB_ISNULL(expr_) || OB_ISNULL(left_expr_) || OB_UNLIKELY(!left_expr_->is_column_ref_expr())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpect error", K(ret), KPC(expr_), KPC(left_expr_)); + } else if (OB_LIKELY(right_const_obj_.is_null())) { + if (OB_FAIL(ObOptSelectivity::get_column_basic_sel(table_metas, ctx, *left_expr_, NULL, &selectivity))) { LOG_WARN("Failed to get var distinct sel", K(ret)); } - } else if (result.is_tinyint() && - !ob_is_string_or_lob_type(left_expr->get_data_type())) { + } else if (right_const_obj_.is_tinyint() && + !ob_is_string_or_lob_type(left_expr_->get_data_type())) { double distinct_sel = 0.0; double null_sel = 0.0; - if (OB_FAIL(ObOptSelectivity::get_column_basic_sel(table_metas, ctx, *left_expr, &distinct_sel, &null_sel))) { + if (OB_FAIL(ObOptSelectivity::get_column_basic_sel(table_metas, ctx, *left_expr_, &distinct_sel, &null_sel))) { LOG_WARN("Failed to get var distinct sel", K(ret)); } else { //distinct_num < 2. That is distinct_num only 1,(As double and statistics not completely accurate, @@ -331,116 +494,51 @@ int ObIsSelEstimator::get_is_sel(const OptTableMetas &table_metas, //But we don't kown whether distinct value is 0. So gess the selectivity: (1 - null_sel)/2.0 distinct_sel = (1- null_sel) / 2.0;//don't kow the value, just get half. } - selectivity = (result.is_true()) ? (1 - distinct_sel - null_sel) : distinct_sel; + selectivity = (right_const_obj_.is_true()) ? (1 - distinct_sel - null_sel) : distinct_sel; } - } else { }//default sel - } else { - //TODO func(cnt_column) + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), KPC(left_expr_), K(right_const_obj_)); + } } - if (T_OP_IS_NOT == qual.get_expr_type()) { + if (OB_SUCC(ret) && T_OP_IS_NOT == expr_->get_expr_type()) { selectivity = 1.0 - selectivity; } return ret; } -int ObCmpSelEstimator::get_range_cmp_sel(const OptTableMetas &table_metas, - const OptSelectivityCtx &ctx, - const ObRawExpr &qual, - double &selectivity) +int ObCmpSelEstimator::get_sel(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + double &selectivity, + ObIArray &all_predicate_sel) { int ret = OB_SUCCESS; selectivity = DEFAULT_INEQ_SEL; - const ObRawExpr *left_expr = qual.get_param_expr(0); - const ObRawExpr *right_expr = qual.get_param_expr(1); - if (OB_ISNULL(left_expr) || OB_ISNULL(right_expr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get null expr", K(ret), K(left_expr), K(right_expr)); - } else if (OB_FAIL(ObOptimizerUtil::get_expr_without_lossless_cast(left_expr, left_expr)) || - OB_FAIL(ObOptimizerUtil::get_expr_without_lossless_cast(right_expr, right_expr))) { - LOG_WARN("failed to get expr without lossless cast", K(ret)); - } else if ((left_expr->is_column_ref_expr() && right_expr->is_const_expr()) || - (left_expr->is_const_expr() && right_expr->is_column_ref_expr())) { - const ObRawExpr *col_expr = left_expr->is_column_ref_expr() ? left_expr : right_expr; - if (OB_FAIL(ObOptSelectivity::get_column_range_sel(table_metas, ctx, - static_cast(*col_expr), - qual, true, selectivity))) { - LOG_WARN("Failed to get column range sel", K(qual), K(ret)); - } - } else if (T_OP_ROW == left_expr->get_expr_type() && T_OP_ROW == right_expr->get_expr_type()) { - //only deal (col1, xx, xx) CMP (const, xx, xx) - if (left_expr->get_param_count() == 1 && OB_NOT_NULL(left_expr->get_param_expr(0)) && - T_OP_ROW == left_expr->get_param_expr(0)->get_expr_type()) { - left_expr = left_expr->get_param_expr(0); - } - if (right_expr->get_param_count() == 1 && OB_NOT_NULL(right_expr->get_param_expr(0)) && - T_OP_ROW == right_expr->get_param_expr(0)->get_expr_type()) { - right_expr = right_expr->get_param_expr(0); - } - if (left_expr->get_param_count() != right_expr->get_param_count()) { + if (can_calc_sel_) { + if (OB_ISNULL(expr_) || OB_ISNULL(col_expr_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("param count should be equal", - K(left_expr->get_param_count()), K(right_expr->get_param_count())); - } else if (left_expr->get_param_count() <= 1) { - // do nothing - } else if (OB_ISNULL(left_expr = left_expr->get_param_expr(0)) || - OB_ISNULL(right_expr = right_expr->get_param_expr(0))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(left_expr), K(right_expr)); - } else if ((left_expr->is_column_ref_expr() && right_expr->is_const_expr()) || - (left_expr->is_const_expr() && right_expr->is_column_ref_expr())) { - const ObRawExpr *col_expr = (left_expr->is_column_ref_expr()) ? (left_expr) : (right_expr); - if (OB_FAIL(ObOptSelectivity::get_column_range_sel(table_metas, ctx, - static_cast(*col_expr), - qual, true, selectivity))) { - LOG_WARN("failed to get column range sel", K(ret)); - } - } else { /* no dothing */ } + LOG_WARN("get null expr", K(ret), KPC(col_expr_), KPC(expr_)); + } else if (OB_FAIL(ObOptSelectivity::get_column_range_sel(table_metas, ctx, *col_expr_, *expr_, true, selectivity))) { + LOG_WARN("Failed to get column range sel", KPC(expr_), KPC(col_expr_), K(ret)); + } else {/*do nothing*/} } return ret; } -int ObBtwSelEstimator::get_btw_sel(const OptTableMetas &table_metas, - const OptSelectivityCtx &ctx, - const ObRawExpr &qual, - double &selectivity) +int ObBtwSelEstimator::get_sel(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + double &selectivity, + ObIArray &all_predicate_sel) { int ret = OB_SUCCESS; selectivity = DEFAULT_SEL; - const ObRawExpr *cmp_expr = NULL; - const ObRawExpr *l_expr = NULL; - const ObRawExpr *r_expr = NULL; - const ObRawExpr *col_expr = NULL; - const ParamStore *params = ctx.get_params(); - if (3 != qual.get_param_count()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("between expr should have 3 param", K(ret), K(qual)); - } else if (OB_ISNULL(params) || - OB_ISNULL(cmp_expr = qual.get_param_expr(0)) || - OB_ISNULL(l_expr = qual.get_param_expr(1)) || - OB_ISNULL(r_expr = qual.get_param_expr(2))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get null params", K(ret), K(params), K(cmp_expr), K(l_expr), K(r_expr)); - } else if (OB_FAIL(ObOptimizerUtil::get_expr_without_lossless_cast(cmp_expr, cmp_expr))) { - LOG_WARN("failed to get expr without lossless cast", K(ret)); - } else if (cmp_expr->is_column_ref_expr() && - ObOptEstUtils::is_calculable_expr(*l_expr, params->count()) && - ObOptEstUtils::is_calculable_expr(*r_expr, params->count())) { - col_expr = cmp_expr; - } else if (ObOptEstUtils::is_calculable_expr(*cmp_expr, params->count()) && - l_expr->is_column_ref_expr() && - ObOptEstUtils::is_calculable_expr(*r_expr, params->count())) { - col_expr = l_expr; - } else if (ObOptEstUtils::is_calculable_expr(*cmp_expr, params->count()) && - ObOptEstUtils::is_calculable_expr(*l_expr, params->count()) && - r_expr->is_column_ref_expr()) { - col_expr = r_expr; - } - if (NULL != col_expr) { - if (OB_FAIL(ObOptSelectivity::get_column_range_sel(table_metas, ctx, - static_cast(*col_expr), - qual, true, selectivity))) { - LOG_WARN("failed to get column range sel", K(ret)); + if (can_calc_sel_) { + if (OB_ISNULL(expr_) || OB_ISNULL(col_expr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get null expr", K(ret), KPC(col_expr_), KPC(expr_)); + } else if (OB_FAIL(ObOptSelectivity::get_column_range_sel(table_metas, ctx, *col_expr_, *expr_, true, selectivity))) { + LOG_WARN("failed to get column range sel", K(ret), KPC(expr_), KPC(col_expr_)); } } return ret; @@ -509,6 +607,7 @@ int ObEqualSelEstimator::get_ne_sel(const OptTableMetas &table_metas, LOG_WARN("get unexpected expr", KPC(l_row), KPC(r_row), K(ret)); } else { int64_t num = l_row->get_param_count(); + ObSEArray selectivities; for (int64_t i = 0; OB_SUCC(ret) && i < num; ++i) { if (OB_ISNULL(l_param = l_row->get_param_expr(i)) || OB_ISNULL(r_param = r_row->get_param_expr(i))) { @@ -517,10 +616,11 @@ int ObEqualSelEstimator::get_ne_sel(const OptTableMetas &table_metas, } else if (OB_FAIL(SMART_CALL(get_ne_sel(table_metas, ctx, *l_param, *r_param, tmp_selectivity)))) { LOG_WARN("failed to get equal selectivity", K(ret)); - } else { - selectivity += tmp_selectivity - selectivity * tmp_selectivity; + } else if (OB_FAIL(selectivities.push_back(1 - tmp_selectivity))) { + LOG_WARN("failed to push back", K(ret)); } } + selectivity = 1 - ctx.get_correlation_model().combine_filters_selectivity(selectivities); } } else if (l_expr.has_flag(CNT_COLUMN) && r_expr.has_flag(CNT_COLUMN)) { if (OB_FAIL(get_cntcol_op_cntcol_sel(table_metas, ctx, l_expr, r_expr, T_OP_NE, selectivity))) { @@ -648,6 +748,7 @@ int ObEqualSelEstimator::get_equal_sel(const OptTableMetas &table_metas, LOG_WARN("get unexpected expr", KPC(l_row), KPC(l_row), K(ret)); } else { int64_t num = l_row->get_param_count(); + ObSEArray selectivities; for (int64_t i = 0; OB_SUCC(ret) && i < num; ++i) { if (OB_ISNULL(l_expr = l_row->get_param_expr(i)) || OB_ISNULL(r_expr = r_row->get_param_expr(i))) { @@ -656,10 +757,11 @@ int ObEqualSelEstimator::get_equal_sel(const OptTableMetas &table_metas, } else if (OB_FAIL(SMART_CALL(get_equal_sel(table_metas, ctx, *l_expr, *r_expr, null_safe, tmp_selectivity)))) { LOG_WARN("failed to get equal selectivity", K(ret)); - } else { - selectivity *= tmp_selectivity; + } else if (OB_FAIL(selectivities.push_back(tmp_selectivity))) { + LOG_WARN("failed to push back"); } } + selectivity = ctx.get_correlation_model().combine_filters_selectivity(selectivities); } } else if ((left_expr.has_flag(CNT_COLUMN) && !right_expr.has_flag(CNT_COLUMN)) || (!left_expr.has_flag(CNT_COLUMN) && right_expr.has_flag(CNT_COLUMN))) { @@ -793,6 +895,13 @@ int ObEqualSelEstimator::get_simple_equal_sel(const OptTableMetas &table_metas, return ret; } +/** + * For the equal predicate 'a = b', we calculate the NDV of (a, b), + * and use the maximum number of tuples that might satisfy the equality as the result of predicate filtering. + * Therefore, the selectivity should be 'min(ndv(a), ndv(b)) / ndv(a, b)'. + * In the case of a join, the left and right sides of the equality are always independent, + * so the selectivity can be simplified as '1 / max(ndv(a), ndv(b))'. +*/ int ObEqualSelEstimator::get_cntcol_op_cntcol_sel(const OptTableMetas &table_metas, const OptSelectivityCtx &ctx, const ObRawExpr &input_left_expr, @@ -814,6 +923,52 @@ int ObEqualSelEstimator::get_cntcol_op_cntcol_sel(const OptTableMetas &table_met } else if (OB_ISNULL(left_expr) || OB_ISNULL(right_expr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret), K(left_expr), K(right_expr)); + } else if (left_expr->get_relation_ids().equal(right_expr->get_relation_ids()) && + ctx.check_opt_compat_version(COMPAT_VERSION_4_2_4, COMPAT_VERSION_4_3_0, + COMPAT_VERSION_4_3_3)) { + double combine_ndv = 1; + if (left_expr->is_column_ref_expr() && right_expr->is_column_ref_expr()) { + const ObColumnRefRawExpr* left_col = static_cast(left_expr); + const ObColumnRefRawExpr* right_col = static_cast(right_expr); + if (OB_FAIL(ObOptSelectivity::get_column_ndv_and_nns(table_metas, ctx, *left_expr, &left_ndv, &left_nns))) { + LOG_WARN("failed to get column basic sel", K(ret)); + } else if (OB_FAIL(ObOptSelectivity::get_column_ndv_and_nns(table_metas, ctx, *right_expr, &right_ndv, &right_nns))) { + LOG_WARN("failed to get column basic sel", K(ret)); + } else if (left_col->get_column_id() == right_col->get_column_id()) { + // same table same column + if (T_OP_NSEQ == op_type) { + selectivity = 1.0; + } else if (T_OP_EQ == op_type) { + selectivity = left_nns; + } else if (T_OP_NE == op_type) { + selectivity = 0.0; + } + } else { + combine_ndv = ObOptSelectivity::combine_two_ndvs(ctx.get_current_rows(), left_ndv, right_ndv); + combine_ndv = std::max(1.0, combine_ndv); + selectivity = std::min(left_ndv, right_ndv) / combine_ndv; + if (T_OP_NSEQ == op_type) { + selectivity += (1 - left_nns) * (1 - right_nns); + } else if (T_OP_EQ == op_type) { + // do nothing + } else if (T_OP_NE == op_type) { + selectivity = std::max(1 - selectivity, 1 / combine_ndv / 2.0); + } + } + } else { + if (OB_FAIL(ObOptSelectivity::calculate_distinct(table_metas, ctx, *left_expr, ctx.get_current_rows(), left_ndv))) { + LOG_WARN("Failed to calculate distinct", K(ret)); + } else if (OB_FAIL(ObOptSelectivity::calculate_distinct(table_metas, ctx, *right_expr, ctx.get_current_rows(), right_ndv))) { + LOG_WARN("Failed to calculate distinct", K(ret)); + } else { + combine_ndv = ObOptSelectivity::combine_two_ndvs(ctx.get_current_rows(), left_ndv, right_ndv); + combine_ndv = std::max(1.0, combine_ndv); + selectivity = std::min(left_ndv, right_ndv) / combine_ndv; + if (T_OP_NE == op_type) { + selectivity = std::max(1 - selectivity, 1 / combine_ndv / 2.0); + } + } + } } else if (left_expr->is_column_ref_expr() && right_expr->is_column_ref_expr()) { const ObColumnRefRawExpr* left_col = NULL; const ObColumnRefRawExpr* right_col = NULL; @@ -1532,11 +1687,11 @@ int ObLikeSelEstimator::create_estimator(ObSelEstimatorFactory &factory, like_estimator->match_all_str_))) { LOG_WARN("failed to check if expr start with percent sign", K(ret)); } else if (like_estimator->match_all_str_) { - like_estimator->can_calc_sel_ = true; + like_estimator->can_calc_sel_by_prefix_ = true; } else if (is_lob_storage(like_estimator->variable_->get_data_type())) { // do nothing } else if (!is_start_with) { - like_estimator->can_calc_sel_ = true; + like_estimator->can_calc_sel_by_prefix_ = true; } } } @@ -1598,14 +1753,14 @@ int ObLikeSelEstimator::get_sel(const OptTableMetas &table_metas, if (OB_ISNULL(expr_) || OB_ISNULL(variable_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null expr", KPC(this)); - } else if (match_all_str_ && can_calc_sel_) { + } else if (match_all_str_ && can_calc_sel_by_prefix_) { double nns = 0.0; if (OB_FAIL(ObOptSelectivity::get_column_ndv_and_nns(table_metas, ctx, *variable_, NULL, &nns))) { LOG_WARN("failed to get nns"); } else { selectivity = nns; } - } else if (can_calc_sel_) { + } else if (can_calc_sel_by_prefix_) { if (OB_UNLIKELY(!variable_->is_column_ref_expr())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected expr", KPC(variable_)); @@ -1616,15 +1771,134 @@ int ObLikeSelEstimator::get_sel(const OptTableMetas &table_metas, } } else if (is_lob_storage(variable_->get_data_type())) { // no statistics for lob type, use default selectivity - selectivity = DEFAULT_CLOB_LIKE_SEL; + selectivity = DEFAULT_LIKE_SEL; + } else if (!ctx.check_opt_compat_version(COMPAT_VERSION_4_2_4, COMPAT_VERSION_4_3_0, + COMPAT_VERSION_4_3_3)) { + selectivity = DEFAULT_INEQ_SEL; + } else if (OB_FAIL(calculate_like_sel_by_substr(table_metas, + ctx, + selectivity))) { + LOG_WARN("failed to calculate like sel", K(ret)); + } + return ret; +} + +int ObLikeSelEstimator::get_wildcard_length(const OptSelectivityCtx &ctx, double &wildcard_length) +{ + int ret = OB_SUCCESS; + ObObj pattern_value; + bool got_result = false; + wildcard_length = 1.0; // default guess value + if (OB_ISNULL(pattern_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (!pattern_->is_static_const_expr()) { + ObString percent_str = ObCharsetUtils::get_const_str(pattern_->get_collation_type(), '%'); + wildcard_length = percent_str.length(); + } else if (OB_FAIL(ObSQLUtils::calc_const_or_calculable_expr(ctx.get_opt_ctx().get_exec_ctx(), + pattern_, + pattern_value, + got_result, + ctx.get_allocator()))) { + LOG_WARN("failed to calc const or calculable expr", K(ret)); + } else if (!got_result || !pattern_value.is_string_type() || pattern_value.is_null()) { + // do nothing } else { - //try find the calc sel from dynamic sampling - int64_t idx = -1; - if (ObOptimizerUtil::find_item(all_predicate_sel, ObExprSelPair(&qual, 0), &idx)) { - selectivity = all_predicate_sel.at(idx).sel_; - } else { - selectivity = DEFAULT_INEQ_SEL; + const ObString &expr_str = pattern_value.get_string(); + ObStringScanner scanner(expr_str, pattern_->get_collation_type()); + ObString percent_str = ObCharsetUtils::get_const_str(pattern_->get_collation_type(), '%'); + ObString underline_str = ObCharsetUtils::get_const_str(pattern_->get_collation_type(), '_'); + ObString encoding; + int32_t wc = 0; + wildcard_length = 0.0; + while (OB_SUCC(ret) + && scanner.next_character(encoding, wc, ret)) { + if (0 == percent_str.compare(encoding)) { + wildcard_length += percent_str.length(); + } + if (0 == underline_str.compare(encoding)) { + wildcard_length += underline_str.length(); + } } + if (OB_FAIL(ret)) { + ret = OB_SUCCESS; + wildcard_length = percent_str.length(); + } + } + return ret; +} + +/** + * try estimate the like selectivity by substr + * e.g. + * `c1 like '%abc'` <=> `substr(c1, -3) = 'abc'` + * Assumption: + * 1. All strings in the variable and pattern have the same length. + * 2. The positions of non-wildcard characters are fixed. + * 3. If the pattern is not a constant, then it contains exactly one wildcard. + * 4. The pattern will not be null +*/ +int ObLikeSelEstimator::calculate_like_sel_by_substr(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + double &selectivity) +{ + int ret = OB_SUCCESS; + // default strategy, not reliable + double variable_len = 0; + double pattern_len = 0; + double substr_ndv = 1.0; + double pattern_ndv = 1.0; + double substr_nns = 1.0; + double pattern_nns = 1.0; // assume that the pattern is not null + double wildcard_length = 1.0; + selectivity = DEFAULT_LIKE_SEL; + if (OB_ISNULL(variable_) || OB_ISNULL(pattern_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (!variable_->get_result_type().is_string_type() || + !pattern_->get_result_type().is_string_type() || + !variable_->is_column_ref_expr()) { + // The length is not reliable, use default selectivity + } else if (OB_FAIL(get_wildcard_length(ctx, wildcard_length))) { + LOG_WARN("failed to get wildcard count", K(ret)); + } else if (OB_FAIL(ObOptSelectivity::calculate_expr_avg_len(table_metas, ctx, pattern_, pattern_len))) { + LOG_WARN("failed to calc expr len", KPC(pattern_), K(ret)); + } else if (pattern_len <= ObOptEstCostModel::DEFAULT_FIXED_OBJ_WIDTH + wildcard_length) { + // do nothing + } else if (FALSE_IT(pattern_len -= ObOptEstCostModel::DEFAULT_FIXED_OBJ_WIDTH)) { + } else if (OB_FAIL(ObOptSelectivity::calculate_substrb_info( + table_metas, ctx, variable_, pattern_len - wildcard_length, ctx.get_current_rows(), substr_ndv, substr_nns))) { + LOG_WARN("failed to calculate substrb ndv", KPC_(variable)); + } else if (OB_FAIL(ObOptSelectivity::calculate_distinct(table_metas, ctx, *pattern_, ctx.get_current_rows(), pattern_ndv))) { + LOG_WARN("failed to calcualte distinct", KPC_(pattern)); + } else { + if (NULL == ctx.get_left_rel_ids() || NULL == ctx.get_right_rel_ids()) { + double combine_ndv = ObOptSelectivity::combine_two_ndvs(ctx.get_current_rows(), substr_ndv, pattern_ndv); + selectivity = std::min(substr_ndv, pattern_ndv) / std::max(1.0, combine_ndv); + selectivity *= substr_nns * pattern_nns; + } else { + double left_ndv = substr_ndv; + double right_ndv = pattern_ndv; + double left_nns = substr_nns; + double right_nns = pattern_nns; + if (variable_->get_relation_ids().overlap(*ctx.get_right_rel_ids()) || + pattern_->get_relation_ids().overlap(*ctx.get_left_rel_ids())) { + std::swap(left_ndv, right_ndv); + std::swap(left_nns, right_nns); + } + if (IS_LEFT_SEMI_ANTI_JOIN(ctx.get_join_type())) { + selectivity = (std::min(left_ndv, right_ndv) / left_ndv) * left_nns; + } else if (IS_RIGHT_SEMI_ANTI_JOIN(ctx.get_join_type())) { + selectivity = (std::min(left_ndv, right_ndv) / right_ndv) * right_nns; + } else { + selectivity = left_nns * right_nns / std::max(left_ndv, right_ndv); + } + if (OB_SUCC(ret) && selectivity >= 1.0 && IS_ANTI_JOIN(ctx.get_join_type())) { + selectivity = 1 - DEFAULT_ANTI_JOIN_SEL; + } + } + LOG_WARN("succeed to calculate like selectivity by substr", + K(selectivity), K(substr_ndv), K(substr_nns), K(pattern_ndv), K(pattern_nns), K(wildcard_length)); } return ret; } @@ -1691,7 +1965,7 @@ int ObBoolOpSelEstimator::get_sel(const OptTableMetas &table_metas, const ObRawExpr &qual = *expr_; if (OB_ISNULL(expr_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null expr", KPC(this)); + LOG_WARN("unexpected null param", KPC(this), K(ctx)); } else if (T_OP_NOT == qual.get_expr_type() || T_FUN_SYS_LNNVL == qual.get_expr_type() || T_OP_BOOL == qual.get_expr_type()) { @@ -1752,17 +2026,21 @@ int ObBoolOpSelEstimator::get_sel(const OptTableMetas &table_metas, if (OB_FAIL(ObOptSelectivity::check_mutex_or(qual, is_mutex))) { LOG_WARN("failed to check mutex or", K(ret)); } else if (is_mutex) { - selectivity = ObOptSelectivity::get_filters_selectivity(selectivities, FilterDependencyType::MUTEX_OR); + selectivity = 0.0; + for (int64_t i = 0; i < selectivities.count(); i ++) { + selectivity += selectivities.at(i); + } + selectivity = ObOptSelectivity::revise_between_0_1(selectivity); } else { // sel(p1 or p2 or p3) = sel(!(!p1 and !p2 and !p3)) for (int64_t i = 0; i < selectivities.count(); i ++) { selectivities.at(i) = 1 - selectivities.at(i); } - selectivity = ObOptSelectivity::get_filters_selectivity(selectivities, ctx.get_dependency_type()); + selectivity = ctx.get_correlation_model().combine_filters_selectivity(selectivities); selectivity = 1- selectivity; } } else { - selectivity = ObOptSelectivity::get_filters_selectivity(selectivities, ctx.get_dependency_type()); + selectivity = ctx.get_correlation_model().combine_filters_selectivity(selectivities); } } else { ret = OB_ERR_UNEXPECTED; @@ -1859,6 +2137,7 @@ int ObSimpleJoinSelEstimator::create_estimator(ObSelEstimatorFactory &factory, } else { simple_join_estimator->left_rel_ids_ = left_rel_ids; simple_join_estimator->right_rel_ids_ = right_rel_ids; + simple_join_estimator->join_rel_ids_ = &expr.get_relation_ids(); estimator = simple_join_estimator; } return ret; @@ -1905,11 +2184,14 @@ int ObSimpleJoinSelEstimator::merge(const ObSelEstimator &other, bool &is_succes if (get_type() == other.get_type()) { const ObSimpleJoinSelEstimator &est_other = static_cast(other); if (OB_ISNULL(left_rel_ids_) || OB_ISNULL(right_rel_ids_) || - OB_ISNULL(est_other.left_rel_ids_) || OB_ISNULL(est_other.right_rel_ids_)) { + OB_ISNULL(est_other.left_rel_ids_) || OB_ISNULL(est_other.right_rel_ids_) || + OB_ISNULL(join_rel_ids_) || OB_ISNULL(est_other.join_rel_ids_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected NULL", KPC(this), K(est_other)); } else if (*left_rel_ids_ == *est_other.left_rel_ids_ && - *right_rel_ids_ == *est_other.right_rel_ids_) { + *right_rel_ids_ == *est_other.right_rel_ids_ && + *join_rel_ids_ == *est_other.join_rel_ids_ + ) { is_success = true; if (OB_FAIL(append(join_conditions_, est_other.join_conditions_))) { LOG_WARN("failed to append", K(ret)); @@ -1966,21 +2248,8 @@ int ObSimpleJoinSelEstimator::get_multi_equal_sel(const OptTableMetas &table_met LOG_WARN("failed get unexpected null", K(ret), K(ctx)); } else if (OB_FAIL(is_valid_multi_join(quals, is_valid))) { LOG_WARN("failed to check is valid multi join", K(ret)); - } else if (!is_valid) { - // multi join condition related to more than two table. Calculate selectivity for each join - // condition independently. - for (int64_t i = 0; OB_SUCC(ret) && i < quals.count(); ++i) { - ObRawExpr *cur_expr = quals.at(i); - double tmp_sel = 1.0; - if (OB_ISNULL(cur_expr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret)); - } else if (OB_FAIL(ObEqualSelEstimator::get_equal_sel(table_metas, ctx, *cur_expr, tmp_sel))) { - LOG_WARN("failed to get equal selectivity", K(ret)); - } else { - selectivity *= tmp_sel; - } - } + } else if (OB_UNLIKELY(!is_valid)) { + ret = OB_ERR_UNEXPECTED; } else if (OB_FAIL(extract_join_exprs(quals, *ctx.get_left_rel_ids(), *ctx.get_right_rel_ids(), left_exprs, right_exprs, null_safes))) { LOG_WARN("failed to extract join exprs", K(ret)); @@ -2073,7 +2342,7 @@ int ObSimpleJoinSelEstimator::get_cntcols_eq_cntcols_sel(const OptTableMetas &ta const ObIArray &left_exprs, const ObIArray &right_exprs, const ObIArray &null_safes, - double &selectivity) + double &selectivity) { int ret = OB_SUCCESS; selectivity = DEFAULT_EQ_SEL; @@ -2161,6 +2430,18 @@ int ObSimpleJoinSelEstimator::get_cntcols_eq_cntcols_sel(const OptTableMetas &ta * ## NULL safe * a) semi: non NULL safe selectivity + `nullsafe(i) && left_not_null_sel(i) < 1.0 ? null_sel(i) * selectivity(j) [where j != i]: 0` */ + if (IS_SEMI_ANTI_JOIN(ctx.get_assumption_type())) { + // do nothing + } else if (left_contain_pk == right_contain_pk) { + // 两侧都不是主键或都是主键, 不做修正 + } else if (refine_right_ndv) { + // 一侧有主键时, 认为是主外键连接, 外键上最大的ndv为即为主键的原始ndv + right_ndv = std::min(right_ndv, left_origin_rows); + } else if (refine_left_ndv) { + left_ndv = std::min(left_ndv, right_origin_rows); + } else { + // do nothing + } if (IS_LEFT_SEMI_ANTI_JOIN(ctx.get_join_type())) { selectivity = std::min(left_ndv, right_ndv) / left_ndv; for (int64_t i = 0; i < left_not_null_sels.count(); ++i) { @@ -2291,7 +2572,7 @@ int ObInequalJoinSelEstimator::extract_column_offset(const OptSelectivityCtx &ct } else { is_valid = false; } - } else if (expr->is_static_const_expr()) { + } else if (expr->is_static_scalar_const_expr()) { ObObj const_value; ObObj scalar_value; bool got_result = false; @@ -2344,7 +2625,7 @@ int ObInequalJoinSelEstimator::create_estimator(ObSelEstimatorFactory &factory, LOG_WARN("failed to create estimator ", K(ret)); } else { ineq_join_estimator->term_ = term; - ineq_join_estimator->set_bound(expr.get_expr_type(), -offset); + ineq_join_estimator->range_.set_bound(expr.get_expr_type(), -offset); } } else if (T_OP_BTW == expr.get_expr_type()) { Term term1; @@ -2374,8 +2655,8 @@ int ObInequalJoinSelEstimator::create_estimator(ObSelEstimatorFactory &factory, LOG_WARN("failed to create estimator ", K(ret)); } else { ineq_join_estimator->term_ = term1; - ineq_join_estimator->set_bound(T_OP_GE, -offset1); - ineq_join_estimator->set_bound(T_OP_LE, -offset2); + ineq_join_estimator->range_.set_bound(T_OP_GE, -offset1); + ineq_join_estimator->range_.set_bound(T_OP_LE, -offset2); } } estimator = ineq_join_estimator; @@ -2405,57 +2686,6 @@ void ObInequalJoinSelEstimator::cmp_term(const ObInequalJoinSelEstimator::Term & } } -void ObInequalJoinSelEstimator::set_bound(ObItemType item_type, double bound) -{ - if (T_OP_LE == item_type) { - has_upper_bound_ = true; - upper_bound_ = bound; - include_upper_bound_ = true; - } else if (T_OP_LT == item_type) { - has_upper_bound_ = true; - upper_bound_ = bound; - include_upper_bound_ = false; - } else if (T_OP_GE == item_type) { - has_lower_bound_ = true; - lower_bound_ = bound; - include_lower_bound_ = true; - } else if (T_OP_GT == item_type) { - has_lower_bound_ = true; - lower_bound_ = bound; - include_lower_bound_ = false; - } -} - -void ObInequalJoinSelEstimator::reverse() -{ - term_.coefficient1_ = -term_.coefficient1_; - term_.coefficient2_ = -term_.coefficient2_; - std::swap(has_lower_bound_, has_upper_bound_); - std::swap(include_lower_bound_, include_upper_bound_); - std::swap(lower_bound_, upper_bound_); - lower_bound_ = -lower_bound_; - upper_bound_ = -upper_bound_; -} - -void ObInequalJoinSelEstimator::update_lower_bound(double bound, bool include) -{ - if (!has_lower_bound_ || - is_higher_lower_bound(bound, include, lower_bound_, include_lower_bound_)) { - include_lower_bound_ = include; - lower_bound_ = bound; - } - has_lower_bound_ = true; -} - -void ObInequalJoinSelEstimator::update_upper_bound(double bound, bool include) { - if (!has_upper_bound_ || - is_higher_upper_bound(upper_bound_, include_upper_bound_, bound, include)) { - include_upper_bound_ = include; - upper_bound_ = bound; - } - has_upper_bound_= true; -} - int ObInequalJoinSelEstimator::merge(const ObSelEstimator &other_estmator, bool &is_success) { int ret = OB_SUCCESS; @@ -2464,16 +2694,13 @@ int ObInequalJoinSelEstimator::merge(const ObSelEstimator &other_estmator, bool const ObInequalJoinSelEstimator &other = static_cast(other_estmator); bool need_reverse = false; cmp_term(term_, other.term_, is_success, need_reverse); - if (is_success){ + if (is_success) { if (need_reverse) { - reverse(); - } - if (other.has_lower_bound_) { - update_lower_bound(other.lower_bound_, other.include_lower_bound_); - } - if (other.has_upper_bound_) { - update_upper_bound(other.upper_bound_, other.include_upper_bound_); + term_.coefficient1_ = -term_.coefficient1_; + term_.coefficient2_ = -term_.coefficient2_; + range_.multiply_double(-1.0); } + range_.intersect(other.range_); } } return ret; @@ -2591,14 +2818,16 @@ int ObInequalJoinSelEstimator::get_sel(const OptTableMetas &table_metas, selectivity = 1.0; double nns1, nns2, ndv1, ndv2; double min1, min2, max1, max2; - double lower_bound = lower_bound_; - double upper_bound = upper_bound_; - bool is_eq = include_lower_bound_ && include_upper_bound_ && - upper_bound - lower_bound <= OB_DOUBLE_EPSINON && - lower_bound - upper_bound <= OB_DOUBLE_EPSINON; + double lower_bound = range_.start_.get_double(); + bool has_lower_bound = !range_.start_.is_min_value(); + double upper_bound = range_.end_.get_double(); + bool has_upper_bound = !range_.end_.is_max_value(); + bool is_valid = range_.is_valid_range(); + bool is_eq = range_.inclusive_start_ && range_.inclusive_end_ && + !range_.start_.is_min_value() && !range_.end_.is_max_value() && + fabs(range_.end_.get_double() - range_.start_.get_double()) <= OB_DOUBLE_EPSINON; if (OB_ISNULL(term_.col1_) || OB_ISNULL(term_.col2_) || - OB_UNLIKELY(!has_lower_bound_ && !has_upper_bound_) || OB_UNLIKELY(fabs(term_.coefficient1_) != 1.0) || OB_UNLIKELY(fabs(term_.coefficient2_) != 1.0)) { ret = OB_ERR_UNEXPECTED; @@ -2607,8 +2836,7 @@ int ObInequalJoinSelEstimator::get_sel(const OptTableMetas &table_metas, LOG_WARN("failed to get nns"); } else if (OB_FAIL(ObOptSelectivity::get_column_ndv_and_nns(table_metas, ctx, *term_.col2_, &ndv2, &nns2))) { LOG_WARN("failed to get nns"); - } else if (has_lower_bound_ && has_upper_bound_ && - lower_bound >= upper_bound && !is_eq) { + } else if (!range_.is_valid_range()) { // always false // e.g. 1 < c1 + c2 < 0 selectivity = 0.0; @@ -2616,18 +2844,9 @@ int ObInequalJoinSelEstimator::get_sel(const OptTableMetas &table_metas, term_.col1_->get_column_id() == term_.col2_->get_column_id()) { // same column if (fabs(term_.coefficient1_ + term_.coefficient2_) <= OB_DOUBLE_EPSINON) { - if (has_lower_bound_ && - is_higher_lower_bound(lower_bound, include_lower_bound_, 0, true)) { - // e.g. : c1 - c1 > 1 - selectivity = 0.0; - } else if (has_upper_bound_ && - is_higher_upper_bound(0, true, upper_bound, include_upper_bound_)) { - // e.g. : c1 - c1 < - 1 - selectivity = 0.0; - } else { - // e.g. : c1 - c1 < 1 - selectivity = nns1; - } + // e.g. : c1 - c1 < 1 + // c1 - c1 > 1 + selectivity = get_sel_for_point(0.0) * nns1; } else { // TODO : c1 + c1 < 1 selectivity = DEFAULT_INEQ_JOIN_SEL; @@ -2692,7 +2911,7 @@ int ObInequalJoinSelEstimator::get_sel(const OptTableMetas &table_metas, } else if (fabs(max1 - min1) <= OB_DOUBLE_EPSINON && fabs(max2 - min2) <= OB_DOUBLE_EPSINON) { // Both c1 and c2 have only one value // e.g. c1 in [1,1] and c2 in [2,2] - selectivity = get_sel_for_point(min1, min2); + selectivity = get_sel_for_point(min1 + min2); } else if (is_eq) { // lower bound is the same as the upper bound // e.g : 1 <= c1 + c2 <= 1; @@ -2700,29 +2919,29 @@ int ObInequalJoinSelEstimator::get_sel(const OptTableMetas &table_metas, } else if (is_semi) { // calculate selectivity for semi join // e.g. : 0 <= c1 + c2 < 1 - double sel1 = has_lower_bound_ ? ObInequalJoinSelEstimator::get_any_gt_sel(min1, max1, min2, max2, lower_bound) : 1.0; - double sel2 = has_upper_bound_ ? ObInequalJoinSelEstimator::get_all_gt_sel(min1, max1, min2, max2, upper_bound) : 0.0; + double sel1 = has_lower_bound ? ObInequalJoinSelEstimator::get_any_gt_sel(min1, max1, min2, max2, lower_bound) : 1.0; + double sel2 = has_upper_bound ? ObInequalJoinSelEstimator::get_all_gt_sel(min1, max1, min2, max2, upper_bound) : 0.0; // the sel of `any c2 satisfy 'a < c1 + c2 < b'` = // the sel of `any c2 satisfy 'c1 + c2 > a'` minus the sel of `all c2 satisfy 'c1 + c2 > b'` selectivity = sel1 - sel2; - if (include_lower_bound_ && ndv1 > 1) { + if (range_.inclusive_start_ && ndv1 > 1) { selectivity += 1 / ndv1; } - if (include_upper_bound_ && ndv1 > 1) { + if (range_.inclusive_end_ && ndv1 > 1) { selectivity += 1 / ndv1; } } else { // calculate selectivity for inner join // e.g. : 0 <= c1 + c2 < 1 - double sel1 = has_lower_bound_ ? ObInequalJoinSelEstimator::get_gt_sel(min1, max1, min2, max2, lower_bound) : 1.0; - double sel2 = has_upper_bound_ ? ObInequalJoinSelEstimator::get_gt_sel(min1, max1, min2, max2, upper_bound) : 0.0; + double sel1 = has_lower_bound ? ObInequalJoinSelEstimator::get_gt_sel(min1, max1, min2, max2, lower_bound) : 1.0; + double sel2 = has_upper_bound ? ObInequalJoinSelEstimator::get_gt_sel(min1, max1, min2, max2, upper_bound) : 0.0; // the sel of 'a < c1 + c2 < b' = // the sel of 'c1 + c2 > a' minus the sel of 'c1 + c2 > b' selectivity = sel1 - sel2; - if (include_lower_bound_) { + if (range_.inclusive_start_) { selectivity += ObInequalJoinSelEstimator::get_equal_sel(min1, max1, ndv1, min2, max2, ndv2, lower_bound, is_semi); } - if (include_upper_bound_) { + if (range_.inclusive_end_) { selectivity += ObInequalJoinSelEstimator::get_equal_sel(min1, max1, ndv1, min2, max2, ndv2, upper_bound, is_semi); } } @@ -2740,17 +2959,11 @@ int ObInequalJoinSelEstimator::get_sel(const OptTableMetas &table_metas, return ret; } -double ObInequalJoinSelEstimator::get_sel_for_point(double point1, double point2) +double ObInequalJoinSelEstimator::get_sel_for_point(double point) { - bool within_interval = true; - double sum = point1 + point2; - if (has_lower_bound_) { - within_interval &= include_lower_bound_ ? sum >= lower_bound_ : sum > lower_bound_; - } - if (has_upper_bound_) { - within_interval &= include_upper_bound_ ? sum <= upper_bound_ : sum < upper_bound_; - } - return within_interval ? 1.0 : 0.0; + SimpleRange point_range; + point_range.set_bound(T_OP_EQ, point); + return range_.is_superset(point_range) ? 1.0 : 0.0; } int ObSelEstimatorFactory::create_estimator(const OptSelectivityCtx &ctx, @@ -2775,6 +2988,7 @@ int ObSelEstimatorFactory::create_estimator(const OptSelectivityCtx &ctx, ObBoolOpSelEstimator::create_estimator, ObInSelEstimator::create_estimator, ObIsSelEstimator::create_estimator, + ObUniformRangeSelEstimator::create_estimator, ObCmpSelEstimator::create_estimator, ObBtwSelEstimator::create_estimator, ObDefaultSelEstimator::create_estimator, @@ -2799,5 +3013,526 @@ int ObSelEstimatorFactory::create_estimator(const OptSelectivityCtx &ctx, return ret; } +int ObSelEstimatorFactory::create_estimators(const OptSelectivityCtx &ctx, + ObIArray &exprs, + ObIArray &estimators) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < exprs.count(); i ++) { + ObSelEstimator *estimator = NULL; + if (OB_FAIL(create_estimator(ctx, exprs.at(i), estimator))) { + LOG_WARN("failed to create estimator", K(ret)); + } else if (OB_FAIL(ObSelEstimator::append_estimators(estimators, estimator))) { + LOG_WARN("failed to append estimators", K(ret)); + } + } + return ret; +} + +int ObEqualSelEstimator::create_estimator(ObSelEstimatorFactory &factory, + const OptSelectivityCtx &ctx, + const ObRawExpr &expr, + ObSelEstimator *&estimator) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(create_simple_estimator(factory, ctx, expr, estimator))) { + LOG_WARN("failed to create simple estimator", K(ret)); + } else if (OB_ISNULL(estimator)) { + //do nothing + } else if (OB_UNLIKELY(expr.get_param_count() != 2) || + OB_ISNULL(expr.get_param_expr(0)) || + OB_ISNULL(expr.get_param_expr(1))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get null expr", K(ret), K(expr)); + } else if (OB_FAIL(check_can_calc_sel(*expr.get_param_expr(0), + *expr.get_param_expr(1), + static_cast(estimator)->can_calc_sel_))) { + LOG_WARN("failed to check can calc sel", K(ret)); + } else {/*do nothing*/} + return ret; +} + +int ObEqualSelEstimator::check_can_calc_sel(const ObRawExpr &l_expr, + const ObRawExpr &r_expr, + bool &can_calc_sel) +{ + int ret = OB_SUCCESS; + can_calc_sel = true; + if (T_OP_ROW == l_expr.get_expr_type() && T_OP_ROW == r_expr.get_expr_type()) { + //row compare row + const ObRawExpr *l_param = NULL; + const ObRawExpr *r_param = NULL; + const ObRawExpr *l_row = &l_expr; + const ObRawExpr *r_row = &r_expr; + if (l_expr.get_param_count() == 1 && OB_NOT_NULL(l_expr.get_param_expr(0)) && + T_OP_ROW == l_expr.get_param_expr(0)->get_expr_type()) { + l_row = l_expr.get_param_expr(0); + } + if (r_expr.get_param_count() == 1 && OB_NOT_NULL(r_expr.get_param_expr(0)) && + T_OP_ROW == r_expr.get_param_expr(0)->get_expr_type()) { + r_row = r_expr.get_param_expr(0); + } + if (OB_UNLIKELY(l_row->get_param_count() != r_row->get_param_count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected expr", KPC(l_row), KPC(r_row), K(ret)); + } else { + int64_t num = l_row->get_param_count(); + for (int64_t i = 0; OB_SUCC(ret) && can_calc_sel && i < num; ++i) { + if (OB_ISNULL(l_param = l_row->get_param_expr(i)) || + OB_ISNULL(r_param = r_row->get_param_expr(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get null expr", K(ret), K(l_row), K(r_row), K(i)); + } else if (OB_FAIL(SMART_CALL(check_can_calc_sel(*l_param, *r_param, can_calc_sel)))) { + LOG_WARN("failed to check can calc sel", K(ret)); + } + } + } + } else if ((l_expr.has_flag(CNT_COLUMN) && !r_expr.has_flag(CNT_COLUMN)) || + (!l_expr.has_flag(CNT_COLUMN) && r_expr.has_flag(CNT_COLUMN))) { + //column compare const + const ObRawExpr *cnt_col_expr = l_expr.has_flag(CNT_COLUMN) ? &l_expr : &r_expr; + ObSEArray column_exprs; + bool only_monotonic_op = true; + if (OB_FAIL(ObOptSelectivity::remove_ignorable_func_for_est_sel(cnt_col_expr))) { + LOG_WARN("failed to remove ignorable function", K(ret)); + } else if (cnt_col_expr->is_column_ref_expr()) { + //do nothing + } else if (OB_FAIL(ObOptEstUtils::extract_column_exprs_with_op_check(cnt_col_expr, + column_exprs, + only_monotonic_op))) { + LOG_WARN("failed to extract column exprs with op check", K(ret)); + } else if (!only_monotonic_op || column_exprs.count() > 1) { + can_calc_sel= false; + } else {/*do nothing*/} + } else if (l_expr.has_flag(CNT_COLUMN) && r_expr.has_flag(CNT_COLUMN)) { + //column compare column + const ObRawExpr* left_expr = &l_expr; + const ObRawExpr* right_expr = &r_expr; + if (left_expr->get_relation_ids() != right_expr->get_relation_ids()) { + //do noting, not same table, dynamic sampling not support join. + } else if (OB_FAIL(ObOptSelectivity::remove_ignorable_func_for_est_sel(left_expr)) || + OB_FAIL(ObOptSelectivity::remove_ignorable_func_for_est_sel(right_expr))) { + LOG_WARN("failed to remove ignorable function", K(ret)); + } else if (OB_ISNULL(left_expr) || OB_ISNULL(right_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(left_expr), K(right_expr)); + } else if (left_expr->is_column_ref_expr() && right_expr->is_column_ref_expr()) { + //do nothing + } else {// func(col) = func(col) or col = func(col) + can_calc_sel = false; + } + } else { + //const compare const + //do nothing + } + return ret; +} + +int ObIsSelEstimator::create_estimator(ObSelEstimatorFactory &factory, + const OptSelectivityCtx &ctx, + const ObRawExpr &expr, + ObSelEstimator *&estimator) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(create_simple_estimator(factory, ctx, expr, estimator))) { + LOG_WARN("failed to create simple estimator", K(ret)); + } else if (OB_ISNULL(estimator)) { + //do nothing + } else if (OB_UNLIKELY(expr.get_param_count() != 2)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(expr)); + } else { + const ParamStore *params = ctx.get_params(); + const ObDMLStmt *stmt = ctx.get_stmt(); + const ObRawExpr *left_expr = expr.get_param_expr(0); + const ObRawExpr *right_expr = expr.get_param_expr(1); + bool got_result = false; + if (OB_ISNULL(params) || OB_ISNULL(stmt) || OB_ISNULL(left_expr) || OB_ISNULL(right_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpect null", K(ret), K(params), K(stmt), K(left_expr), K(right_expr)); + } else if (OB_UNLIKELY(!ObOptEstUtils::is_calculable_expr(*right_expr, params->count()))) { + //do nothing + } else if (OB_FAIL(ObSQLUtils::calc_const_or_calculable_expr(ctx.get_opt_ctx().get_exec_ctx(), + right_expr, + static_cast(estimator)->right_const_obj_, + got_result, + ctx.get_allocator()))) { + LOG_WARN("failed to calculate const or calculable expr", K(ret)); + } else if (!got_result) { + // do nothing + } else if (OB_FAIL(ObOptSelectivity::remove_ignorable_func_for_est_sel(left_expr))) { + LOG_WARN("failed to remove ignorable func", KPC(left_expr)); + } else if (left_expr->is_column_ref_expr()) { + if (OB_FAIL(ObOptSelectivity::check_column_in_current_level_stmt(stmt, *left_expr))) { + LOG_WARN("Failed to check column whether is in current stmt", K(ret)); + } else if (static_cast(estimator)->right_const_obj_.is_null() || + (static_cast(estimator)->right_const_obj_.is_tinyint() && + !ob_is_string_or_lob_type(left_expr->get_data_type()))) { + static_cast(estimator)->can_calc_sel_ = true; + static_cast(estimator)->left_expr_ = left_expr; + } + } + } + return ret; +} + +int ObCmpSelEstimator::create_estimator(ObSelEstimatorFactory &factory, + const OptSelectivityCtx &ctx, + const ObRawExpr &expr, + ObSelEstimator *&estimator) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(create_simple_estimator(factory, ctx, expr, estimator))) { + LOG_WARN("failed to create simple estimator", K(ret)); + } else if (OB_ISNULL(estimator)) { + //do nothing + } else if (OB_UNLIKELY(expr.get_param_count() != 2)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(expr)); + } else { + const ObRawExpr *left_expr = expr.get_param_expr(0); + const ObRawExpr *right_expr = expr.get_param_expr(1); + if (OB_ISNULL(left_expr) || OB_ISNULL(right_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get null expr", K(ret), K(left_expr), K(right_expr)); + } else if (OB_FAIL(ObOptimizerUtil::get_expr_without_lossless_cast(left_expr, left_expr)) || + OB_FAIL(ObOptimizerUtil::get_expr_without_lossless_cast(right_expr, right_expr))) { + LOG_WARN("failed to get expr without lossless cast", K(ret)); + } else if ((left_expr->is_column_ref_expr() && right_expr->is_const_expr()) || + (left_expr->is_const_expr() && right_expr->is_column_ref_expr())) { + static_cast(estimator)->can_calc_sel_ = true; + static_cast(estimator)->col_expr_ = left_expr->is_column_ref_expr() ? static_cast(left_expr) : + static_cast(right_expr); + } else if (T_OP_ROW == left_expr->get_expr_type() && T_OP_ROW == right_expr->get_expr_type()) { + //only deal (col1, xx, xx) CMP (const, xx, xx) + if (left_expr->get_param_count() == 1 && OB_NOT_NULL(left_expr->get_param_expr(0)) && + T_OP_ROW == left_expr->get_param_expr(0)->get_expr_type()) { + left_expr = left_expr->get_param_expr(0); + } + if (right_expr->get_param_count() == 1 && OB_NOT_NULL(right_expr->get_param_expr(0)) && + T_OP_ROW == right_expr->get_param_expr(0)->get_expr_type()) { + right_expr = right_expr->get_param_expr(0); + } + if (left_expr->get_param_count() != right_expr->get_param_count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("param count should be equal", + K(left_expr->get_param_count()), K(right_expr->get_param_count())); + } else if (left_expr->get_param_count() <= 1) { + // do nothing + } else if (OB_ISNULL(left_expr = left_expr->get_param_expr(0)) || + OB_ISNULL(right_expr = right_expr->get_param_expr(0))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(left_expr), K(right_expr)); + } else if ((left_expr->is_column_ref_expr() && right_expr->is_const_expr()) || + (left_expr->is_const_expr() && right_expr->is_column_ref_expr())) { + static_cast(estimator)->can_calc_sel_ = true; + static_cast(estimator)->col_expr_ = left_expr->is_column_ref_expr() ? static_cast(left_expr) : + static_cast(right_expr); + } else { /* no dothing */ } + } + } + return ret; +} + +int ObBtwSelEstimator::create_estimator(ObSelEstimatorFactory &factory, + const OptSelectivityCtx &ctx, + const ObRawExpr &expr, + ObSelEstimator *&estimator) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(create_simple_estimator(factory, ctx, expr, estimator))) { + LOG_WARN("failed to create simple estimator", K(ret)); + } else if (OB_ISNULL(estimator)) { + //do nothing + } else { + const ObRawExpr *cmp_expr = NULL; + const ObRawExpr *l_expr = NULL; + const ObRawExpr *r_expr = NULL; + const ObRawExpr *col_expr = NULL; + const ParamStore *params = ctx.get_params(); + if (3 != expr.get_param_count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("between expr should have 3 param", K(ret), K(expr)); + } else if (OB_ISNULL(params) || + OB_ISNULL(cmp_expr = expr.get_param_expr(0)) || + OB_ISNULL(l_expr = expr.get_param_expr(1)) || + OB_ISNULL(r_expr = expr.get_param_expr(2))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get null params", K(ret), K(params), K(cmp_expr), K(l_expr), K(r_expr)); + } else if (OB_FAIL(ObOptimizerUtil::get_expr_without_lossless_cast(cmp_expr, cmp_expr))) { + LOG_WARN("failed to get expr without lossless cast", K(ret)); + } else if (cmp_expr->is_column_ref_expr() && + ObOptEstUtils::is_calculable_expr(*l_expr, params->count()) && + ObOptEstUtils::is_calculable_expr(*r_expr, params->count())) { + static_cast(estimator)->can_calc_sel_ = true; + static_cast(estimator)->col_expr_ = static_cast(cmp_expr); + } else if (ObOptEstUtils::is_calculable_expr(*cmp_expr, params->count()) && + l_expr->is_column_ref_expr() && + ObOptEstUtils::is_calculable_expr(*r_expr, params->count())) { + static_cast(estimator)->can_calc_sel_ = true; + static_cast(estimator)->col_expr_ = static_cast(l_expr); + } else if (ObOptEstUtils::is_calculable_expr(*cmp_expr, params->count()) && + ObOptEstUtils::is_calculable_expr(*l_expr, params->count()) && + r_expr->is_column_ref_expr()) { + static_cast(estimator)->can_calc_sel_ = true; + static_cast(estimator)->col_expr_ = static_cast(r_expr); + } + } + return ret; +} + +int ObNormalRangeSelEstimator::get_expr_range(const OptSelectivityCtx &ctx, + const ObRawExpr &qual, + const ObRawExpr *&expr, + SimpleRange &range, + bool &is_not_op, + bool &is_valid) +{ + int ret = OB_SUCCESS; + is_valid = false; + is_not_op = false; + expr = NULL; + const ObRawExpr *const_expr1 = NULL; + const ObRawExpr *const_expr2 = NULL; + ObObj const_value1; + ObObj const_value2; + bool got_result = false; + range.set_whole_range(); + ObItemType type = qual.get_expr_type(); + if (OB_FAIL(ObOptEstUtils::extract_var_op_const(&qual, + expr, + const_expr1, + const_expr2, + type, + is_valid))) { + LOG_WARN("failed to extract var and const", K(ret), K(qual)); + } else if (!is_valid) { + // do nothing + } else if (NULL == const_expr1 || !const_expr1->is_static_scalar_const_expr()) { + is_valid = false; + } else if (OB_FAIL(ObSQLUtils::calc_const_or_calculable_expr(ctx.get_opt_ctx().get_exec_ctx(), + const_expr1, + const_value1, + got_result, + ctx.get_allocator()))) { + LOG_WARN("failed to calc const value", K(expr), K(ret)); + } else if (!got_result) { + is_valid = false; + } else if (NULL == const_expr2 || !const_expr2->is_static_scalar_const_expr()) { + // do nothing + } else if (OB_FAIL(ObSQLUtils::calc_const_or_calculable_expr(ctx.get_opt_ctx().get_exec_ctx(), + const_expr2, + const_value2, + got_result, + ctx.get_allocator()))) { + LOG_WARN("failed to calc const value", K(expr), K(ret)); + } else if (!got_result) { + is_valid = false; + } + if (OB_SUCC(ret) && is_valid) { + if (IS_RANGE_CMP_OP(type) || T_OP_EQ == type || T_OP_NSEQ == type) { + range.set_bound(type, const_value1); + } else if (T_OP_NE == type) { + range.set_bound(T_OP_EQ, const_value1); + is_not_op = true; + } else if (T_OP_IS_NOT == type || T_OP_IS == type) { + if (const_value1.is_null()) { + range.set_bound(type, const_value1); + } else { + is_valid = false; + } + } else if (T_OP_BTW == type || T_OP_NOT_BTW == type) { + range.set_bound(T_OP_GE, const_value1); + range.set_bound(T_OP_LE, const_value2); + is_not_op = (T_OP_NOT_BTW == type); + } + } + return ret; +} + +int ObNormalRangeSelEstimator::merge(const ObSelEstimator &other_estmator, bool &is_success) +{ + int ret = OB_SUCCESS; + is_success = false; + if (get_type() == other_estmator.get_type() && !is_not_op_) { + const ObNormalRangeSelEstimator &other = static_cast(other_estmator); + if (!other.is_not_op_ && expr_ == other.expr_ && range_.intersect(other.range_)) { + is_success = true; + } + } + return ret; +} + +int ObUniformRangeSelEstimator::create_estimator(ObSelEstimatorFactory &factory, + const OptSelectivityCtx &ctx, + const ObRawExpr &expr, + ObSelEstimator *&estimator) +{ + int ret = OB_SUCCESS; + estimator = NULL; + ObUniformRangeSelEstimator *range_estimator = NULL; + bool is_valid = false; + const ObRawExpr *param_expr = NULL; + SimpleRange range; + bool is_not_op = false; + if (!ctx.check_opt_compat_version(COMPAT_VERSION_4_2_4, COMPAT_VERSION_4_3_0, + COMPAT_VERSION_4_3_3)) { + // do nothing + } else if (OB_FAIL(get_expr_range(ctx, expr, param_expr, range, is_not_op, is_valid))) { + LOG_WARN("failed to get the range form", K(ret), K(expr)); + } else if (!is_valid) { + // do nothing + } else if (OB_FAIL(factory.create_estimator_inner(range_estimator))) { + LOG_WARN("failed to create estimator", K(ret)); + } else { + estimator = range_estimator; + range_estimator->expr_ = param_expr; + range_estimator->range_ = range; + range_estimator->is_not_op_ = is_not_op; + } + return ret; +} + +int ObUniformRangeSelEstimator::get_sel(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + double &selectivity, + ObIArray &all_predicate_sel) +{ + int ret = OB_SUCCESS; + selectivity = DEFAULT_INEQ_SEL; + ObObj expr_min; + ObObj expr_max; + ObObj min_scalar; + ObObj max_scalar; + ObObj start_scalar; + ObObj end_scalar; + expr_min.set_min_value(); + expr_max.set_max_value(); + double ndv = 1.0; + double not_null_sel = 1.0; // todo + bool dummy = false; + bool discrete = (expr_->get_type_class() != ObFloatTC) && (expr_->get_type_class() != ObDoubleTC); + ObBorderFlag border_flag; + if (range_.inclusive_start_){ + border_flag.set_inclusive_start(); + } + if (range_.inclusive_end_) { + border_flag.set_inclusive_end(); + } + if (!range_.is_valid_range()) { + selectivity = 0.0; + } else if (OB_FAIL(ObOptSelectivity::calc_expr_min_max(table_metas, ctx, expr_, + expr_min, expr_max))) { + LOG_WARN("failed to get min max", K(ret)); + } else if (expr_min.is_min_value() || expr_min.is_max_value() || expr_min.is_null() || + expr_max.is_min_value() || expr_max.is_max_value() || expr_max.is_null()) { + // do nothing + } else if (OB_UNLIKELY(!expr_min.can_compare(expr_max)) || + OB_UNLIKELY(!expr_min.can_compare(range_.start_)) || + OB_UNLIKELY(!expr_min.can_compare(range_.end_))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("obj type is not consistent", K(expr_min), K(expr_max), KPC(this)); + } else if (OB_FAIL(ObOptEstObjToScalar::convert_objs_to_scalars(&expr_min, &expr_max, + &range_.start_, &range_.end_, + &min_scalar, &max_scalar, + &start_scalar, &end_scalar))) { + LOG_WARN("failed to convert obj to scalars", K(ret)); + } else if (OB_FAIL(ObOptSelectivity::calculate_distinct(table_metas, ctx, *expr_, ctx.get_current_rows(), ndv))) { + LOG_WARN("failed to calculate distinct", K(ret)); + } else if (OB_FAIL(ObOptSelectivity::do_calc_range_selectivity(min_scalar.get_double(), + max_scalar.get_double(), + start_scalar, + end_scalar, + ndv, + discrete, + border_flag, + dummy, + selectivity))) { + LOG_WARN("failed to do calc range selectivity", K(ret)); + } else if (!is_not_op_ && + OB_FAIL(refine_out_of_bounds_sel(table_metas, + ctx, + expr_min, + expr_max, + min_scalar.get_double(), + max_scalar.get_double(), + start_scalar.get_double(), + end_scalar.get_double(), + selectivity))) { + LOG_WARN("failed to refine out of bounds sel", K(ret)); + } else { + if (is_not_op_) { + selectivity = 1 - selectivity; + } + selectivity = std::max(selectivity, 1.0 / ndv); + selectivity *= not_null_sel; + } + LOG_DEBUG("succeed to calculate uniform range sel", + K(selectivity), K(discrete), K(expr_min), K(expr_max), K(range_), K(not_null_sel), K(ndv), KPC(expr_)); + return ret; +} + +int ObUniformRangeSelEstimator::refine_out_of_bounds_sel(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + const ObObj &min_val, + const ObObj &max_val, + const double min_scalar, + const double max_scalar, + const double start_scalar, + const double end_scalar, + double &selectivity) +{ + int ret = OB_SUCCESS; + ObSEArray column_exprs; + const OptTableMeta *table_meta = NULL; + double increase_rows_ratio = 0.0; + bool is_half = range_.start_.is_min_value() || range_.end_.is_max_value(); + double out_of_bounds_sel = 0.0; + bool need_calc = true; + if (OB_ISNULL(expr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", KPC(this)); + } else if (expr_->get_relation_ids().num_members() != 1 || + min_val.is_min_value() || max_val.is_min_value() || + min_val.is_max_value() || max_val.is_max_value() || + (max_scalar - min_scalar < OB_DOUBLE_EPSINON) || + fabs(selectivity - 1.0) < OB_DOUBLE_EPSINON || + !range_.is_valid_range()) { + need_calc = false; + } else if (is_half) { + need_calc = true; + } else if (start_scalar >= min_scalar && end_scalar <= max_scalar) { + need_calc = false; + } else if (start_scalar <= min_scalar && end_scalar >= max_scalar) { + need_calc = false; + selectivity = 1.0; + } else if (start_scalar < min_scalar) { + need_calc = true; + out_of_bounds_sel = (std::min(min_scalar, end_scalar) - start_scalar) / (max_scalar - min_scalar); + } else if (end_scalar > max_scalar) { + need_calc = true; + out_of_bounds_sel = (end_scalar - std::max(max_scalar, start_scalar)) / (max_scalar - min_scalar); + } + if (OB_FAIL(ret) || !need_calc) { + } else if (OB_FAIL(ObRawExprUtils::extract_column_exprs(expr_, column_exprs))) { + LOG_WARN("extract_column_exprs error in clause_selectivity", K(ret)); + } else if (OB_UNLIKELY(column_exprs.count() < 1) || OB_ISNULL(column_exprs.at(0))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected column count", KPC(expr_), K(column_exprs)); + } else if (FALSE_IT(table_meta = table_metas.get_table_meta_by_table_id( + static_cast(column_exprs.at(0))->get_table_id()))) { + } else if (NULL == table_meta) { + // do nothing + } else if (OB_FAIL(table_meta->get_increase_rows_ratio(ctx.get_opt_ctx(), increase_rows_ratio))) { + LOG_WARN("failed to get extra rows", K(ret)); + } else if (is_half) { + selectivity = std::max(selectivity, DEFAULT_OUT_OF_BOUNDS_SEL * increase_rows_ratio); + } else { + selectivity += std::min(out_of_bounds_sel, increase_rows_ratio); + } + selectivity = ObOptSelectivity::revise_between_0_1(selectivity); + return ret; +} + }//end of namespace sql }//end of namespace oceanbase diff --git a/src/sql/optimizer/ob_sel_estimator.h b/src/sql/optimizer/ob_sel_estimator.h index b90ba5839..b394b5618 100644 --- a/src/sql/optimizer/ob_sel_estimator.h +++ b/src/sql/optimizer/ob_sel_estimator.h @@ -20,6 +20,44 @@ namespace oceanbase namespace sql { +struct SimpleRange +{ +public: + SimpleRange() + { + set_whole_range(); + } + + void set_whole_range(); + + void set_false_range(); + + int compare_with_end(const SimpleRange &r) const; + + int compare_with_start(const SimpleRange &r) const; + + bool intersect(const SimpleRange &r); + + void set_bound(ObItemType item_type, ObObj bound); + + void set_bound(ObItemType item_type, double bound); + + bool is_valid_range(); + + bool is_superset(const SimpleRange &r) const; + + void multiply_double(double coff); + + TO_STRING_KV(K_(start), K_(end), + K_(inclusive_start), + K_(inclusive_end)); + + ObObj start_; + ObObj end_; + bool inclusive_start_; + bool inclusive_end_; +}; + enum class ObSelEstType { INVALID = 0, @@ -34,9 +72,10 @@ enum class ObSelEstType EQUAL, LIKE, BOOL_OP, - RANGE, + COLUMN_RANGE, SIMPLE_JOIN, INEQUAL_JOIN, + UNIFORM_RANGE, }; class ObSelEstimatorFactory; @@ -73,16 +112,21 @@ private: class ObSelEstimatorFactory { public: - explicit ObSelEstimatorFactory(common::ObIAllocator &alloc) - : allocator_(alloc), - estimator_store_(alloc) + explicit ObSelEstimatorFactory() + : allocator_("ObOptSel"), + estimator_store_(allocator_) + {} + + explicit ObSelEstimatorFactory(int64_t tenant_id) + : allocator_("ObOptSel", OB_MALLOC_NORMAL_BLOCK_SIZE, tenant_id), + estimator_store_(allocator_) {} ~ObSelEstimatorFactory() { destory(); } - inline common::ObIAllocator &get_allocator() { return allocator_; } + inline common::ObArenaAllocator &get_allocator() { return allocator_; } inline void destory() { DLIST_FOREACH_NORET(node, estimator_store_.get_obj_list()) { @@ -96,6 +140,9 @@ public: int create_estimator(const OptSelectivityCtx &ctx, const ObRawExpr *expr, ObSelEstimator *&new_estimator); + int create_estimators(const OptSelectivityCtx &ctx, + ObIArray &exprs, + ObIArray &estimators); template inline int create_estimator_inner(EstimatorType *&new_estimator) @@ -121,7 +168,7 @@ public: const ObRawExpr &, ObSelEstimator *&); private: - common::ObIAllocator &allocator_; + common::ObArenaAllocator allocator_; common::ObObjStore estimator_store_; private: DISALLOW_COPY_AND_ASSIGN(ObSelEstimatorFactory); @@ -421,40 +468,31 @@ private: class ObIsSelEstimator : public ObIndependentSelEstimator { public: - ObIsSelEstimator() : ObIndependentSelEstimator(ObSelEstType::IS) {} + ObIsSelEstimator() : + ObIndependentSelEstimator(ObSelEstType::IS), + can_calc_sel_(false), + left_expr_(NULL), + right_const_obj_() + {} virtual ~ObIsSelEstimator() = default; static int create_estimator(ObSelEstimatorFactory &factory, const OptSelectivityCtx &ctx, const ObRawExpr &expr, - ObSelEstimator *&estimator) - { - return create_simple_estimator(factory, ctx, expr, estimator); - } - virtual bool tend_to_use_ds() override { return false; } + ObSelEstimator *&estimator); + + virtual bool tend_to_use_ds() override { return !can_calc_sel_; } virtual int get_sel(const OptTableMetas &table_metas, const OptSelectivityCtx &ctx, double &selectivity, - ObIArray &all_predicate_sel) override - { - int ret = OB_SUCCESS; - if (OB_ISNULL(expr_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", KPC(this)); - } else { - ret = get_is_sel(table_metas, ctx, *expr_, selectivity); - } - return ret; - } + ObIArray &all_predicate_sel) override; inline static bool check_expr_valid(const ObRawExpr &expr) { return T_OP_IS == expr.get_expr_type() || T_OP_IS_NOT == expr.get_expr_type(); } private: - static int get_is_sel(const OptTableMetas &table_metas, - const OptSelectivityCtx &ctx, - const ObRawExpr &qual, - double &selectivity); -private: + bool can_calc_sel_; + const ObRawExpr *left_expr_; + common::ObObj right_const_obj_; DISABLE_COPY_ASSIGN(ObIsSelEstimator); }; @@ -463,40 +501,28 @@ private: class ObBtwSelEstimator : public ObIndependentSelEstimator { public: - ObBtwSelEstimator() : ObIndependentSelEstimator(ObSelEstType::BTW) {} + ObBtwSelEstimator() : + ObIndependentSelEstimator(ObSelEstType::BTW), + can_calc_sel_(false), + col_expr_(NULL) + {} virtual ~ObBtwSelEstimator() = default; static int create_estimator(ObSelEstimatorFactory &factory, const OptSelectivityCtx &ctx, const ObRawExpr &expr, - ObSelEstimator *&estimator) - { - return create_simple_estimator(factory, ctx, expr, estimator); - } - virtual bool tend_to_use_ds() override { return false; } + ObSelEstimator *&estimator); + virtual bool tend_to_use_ds() override { return !can_calc_sel_; } virtual int get_sel(const OptTableMetas &table_metas, const OptSelectivityCtx &ctx, double &selectivity, - ObIArray &all_predicate_sel) override - { - int ret = OB_SUCCESS; - if (OB_ISNULL(expr_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", KPC(this)); - } else { - ret = get_btw_sel(table_metas, ctx, *expr_, selectivity); - } - return ret; - } + ObIArray &all_predicate_sel) override; inline static bool check_expr_valid(const ObRawExpr &expr) { return T_OP_BTW == expr.get_expr_type() || T_OP_NOT_BTW == expr.get_expr_type(); } private: - static int get_btw_sel(const OptTableMetas &table_metas, - const OptSelectivityCtx &ctx, - const ObRawExpr &qual, - double &selectivity); -private: + bool can_calc_sel_; + const ObColumnRefRawExpr *col_expr_; DISABLE_COPY_ASSIGN(ObBtwSelEstimator); }; @@ -506,40 +532,28 @@ private: class ObCmpSelEstimator : public ObIndependentSelEstimator { public: - ObCmpSelEstimator() : ObIndependentSelEstimator(ObSelEstType::CMP) {} + ObCmpSelEstimator() : + ObIndependentSelEstimator(ObSelEstType::CMP), + can_calc_sel_(false), + col_expr_(NULL) + {} virtual ~ObCmpSelEstimator() = default; static int create_estimator(ObSelEstimatorFactory &factory, const OptSelectivityCtx &ctx, const ObRawExpr &expr, - ObSelEstimator *&estimator) - { - return create_simple_estimator(factory, ctx, expr, estimator); - } - virtual bool tend_to_use_ds() override { return false; } + ObSelEstimator *&estimator); + virtual bool tend_to_use_ds() override { return !can_calc_sel_; } virtual int get_sel(const OptTableMetas &table_metas, const OptSelectivityCtx &ctx, double &selectivity, - ObIArray &all_predicate_sel) override - { - int ret = OB_SUCCESS; - if (OB_ISNULL(expr_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", KPC(this)); - } else { - ret = get_range_cmp_sel(table_metas, ctx, *expr_, selectivity); - } - return ret; - } + ObIArray &all_predicate_sel) override; inline static bool check_expr_valid(const ObRawExpr &expr) { return IS_RANGE_CMP_OP(expr.get_expr_type()); } private: - static int get_range_cmp_sel(const OptTableMetas &table_metas, - const OptSelectivityCtx &ctx, - const ObRawExpr &qual, - double &selectivity); -private: + bool can_calc_sel_; + const ObColumnRefRawExpr *col_expr_; DISABLE_COPY_ASSIGN(ObCmpSelEstimator); }; @@ -549,17 +563,16 @@ class ObEqualSelEstimator : public ObIndependentSelEstimator { public: ObEqualSelEstimator() : - ObIndependentSelEstimator(ObSelEstType::EQUAL) {} + ObIndependentSelEstimator(ObSelEstType::EQUAL), + can_calc_sel_(false) + {} virtual ~ObEqualSelEstimator() = default; static int create_estimator(ObSelEstimatorFactory &factory, const OptSelectivityCtx &ctx, const ObRawExpr &expr, - ObSelEstimator *&estimator) - { - return create_simple_estimator(factory, ctx, expr, estimator); - } - virtual bool tend_to_use_ds() override { return false; } + ObSelEstimator *&estimator); + virtual bool tend_to_use_ds() override { return !can_calc_sel_; } virtual int get_sel(const OptTableMetas &table_metas, const OptSelectivityCtx &ctx, double &selectivity, @@ -625,7 +638,12 @@ private: const ObRawExpr &input_right_expr, ObItemType op_type, double &selectivity); + + static int check_can_calc_sel(const ObRawExpr &l_expr, + const ObRawExpr &r_expr, + bool &can_calc_sel); private: + bool can_calc_sel_; DISABLE_COPY_ASSIGN(ObEqualSelEstimator); }; @@ -643,7 +661,7 @@ public: variable_(NULL), pattern_(NULL), escape_(NULL), - can_calc_sel_(false), + can_calc_sel_by_prefix_(false), match_all_str_(false) {} virtual ~ObLikeSelEstimator() = default; @@ -651,18 +669,21 @@ public: const OptSelectivityCtx &ctx, const ObRawExpr &expr, ObSelEstimator *&estimator); - virtual bool tend_to_use_ds() override { return !can_calc_sel_; } + virtual bool tend_to_use_ds() override { return !can_calc_sel_by_prefix_; } virtual int get_sel(const OptTableMetas &table_metas, const OptSelectivityCtx &ctx, double &selectivity, ObIArray &all_predicate_sel) override; static int can_calc_like_sel(const OptSelectivityCtx &ctx, const ObRawExpr &expr, bool &can_calc_sel); - + int get_wildcard_length(const OptSelectivityCtx &ctx, double &wildcard_length); + int calculate_like_sel_by_substr(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + double &selectivity); private: const ObRawExpr *variable_; const ObRawExpr *pattern_; const ObRawExpr *escape_; - bool can_calc_sel_; + bool can_calc_sel_by_prefix_; bool match_all_str_; private: DISABLE_COPY_ASSIGN(ObLikeSelEstimator); @@ -687,6 +708,7 @@ public: const OptSelectivityCtx &ctx, double &selectivity, ObIArray &all_predicate_sel) override; + VIRTUAL_TO_STRING_KV(K_(type), K_(child_estimators)); private: common::ObSEArray child_estimators_; @@ -701,7 +723,7 @@ private: class ObRangeSelEstimator : public ObSelEstimator { public: - ObRangeSelEstimator() : ObSelEstimator(ObSelEstType::RANGE), column_expr_(NULL) {} + ObRangeSelEstimator() : ObSelEstimator(ObSelEstType::COLUMN_RANGE), column_expr_(NULL) {} virtual ~ObRangeSelEstimator() = default; static int create_estimator(ObSelEstimatorFactory &factory, @@ -792,6 +814,7 @@ private: const ObRelIds *left_rel_ids_; const ObRelIds *right_rel_ids_; + const ObRelIds *join_rel_ids_; common::ObSEArray join_conditions_; private: @@ -820,13 +843,7 @@ public: public: ObInequalJoinSelEstimator() : - ObSelEstimator(ObSelEstType::INEQUAL_JOIN), - has_lower_bound_(false), - has_upper_bound_(false), - include_lower_bound_(false), - include_upper_bound_(false), - lower_bound_(0), - upper_bound_(0) {} + ObSelEstimator(ObSelEstType::INEQUAL_JOIN) {} virtual ~ObInequalJoinSelEstimator() = default; static int create_estimator(ObSelEstimatorFactory &factory, @@ -843,10 +860,7 @@ public: virtual bool tend_to_use_ds() override { return false; } - VIRTUAL_TO_STRING_KV(K_(type), K_(term), - K_(has_lower_bound), K_(has_upper_bound), - K_(include_lower_bound), K_(include_upper_bound), - K_(lower_bound), K_(upper_bound)); + VIRTUAL_TO_STRING_KV(K_(type), K_(term), K_(range)); private: @@ -863,14 +877,6 @@ private: Term &term, double &offset); - static bool is_higher_lower_bound(double bound1, bool include1, double bound2, bool include2) - { - return bound1 > bound2 || (bound1 == bound2 && !include1 && include2); - } - static bool is_higher_upper_bound(double bound1, bool include1, double bound2, bool include2) - { - return bound1 > bound2 || (bound1 == bound2 && include1 && !include2); - } // c1 in [min1, max1], c2 in [min2, max2] // calc the sel of `c1 + c2 > offset`; static double get_gt_sel(double min1, @@ -902,25 +908,71 @@ private: double offset, bool is_semi); - double get_sel_for_point(double point1, double point2); - - void reverse(); - void update_lower_bound(double bound, bool include); - void update_upper_bound(double bound, bool include); - void set_bound(ObItemType item_type, double bound); + double get_sel_for_point(double point); Term term_; - bool has_lower_bound_; - bool has_upper_bound_; - bool include_lower_bound_; - bool include_upper_bound_; - double lower_bound_; - double upper_bound_; + SimpleRange range_; private: DISALLOW_COPY_AND_ASSIGN(ObInequalJoinSelEstimator); }; +class ObNormalRangeSelEstimator : public ObSelEstimator +{ +public: + ObNormalRangeSelEstimator(ObSelEstType type) : + ObSelEstimator(type), + expr_(NULL), + is_not_op_(false) {} + virtual ~ObNormalRangeSelEstimator() = default; + static int get_expr_range(const OptSelectivityCtx &ctx, + const ObRawExpr &qual, + const ObRawExpr *&expr, + SimpleRange &range, + bool &is_not_op, + bool &is_valid); + + virtual int merge(const ObSelEstimator &other, bool &is_success) override; + virtual bool is_independent() const override { return is_not_op_; } + void set_is_not_op(bool is_not) { is_not_op_ = is_not; } + + VIRTUAL_TO_STRING_KV(K_(type), KPC_(expr), K_(range), K_(is_not_op)); +protected: + const ObRawExpr *expr_; + SimpleRange range_; + bool is_not_op_; // not between +}; + +class ObUniformRangeSelEstimator : public ObNormalRangeSelEstimator +{ +public: + ObUniformRangeSelEstimator() : + ObNormalRangeSelEstimator(ObSelEstType::UNIFORM_RANGE) {} + virtual ~ObUniformRangeSelEstimator() = default; + + static int create_estimator(ObSelEstimatorFactory &factory, + const OptSelectivityCtx &ctx, + const ObRawExpr &expr, + ObSelEstimator *&estimator); + + virtual int get_sel(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + double &selectivity, + ObIArray &all_predicate_sel) override; + + int refine_out_of_bounds_sel(const OptTableMetas &table_metas, + const OptSelectivityCtx &ctx, + const ObObj &min_value, + const ObObj &max_value, + const double min_scalar, + const double max_scalar, + const double start_scalar, + const double end_scalar, + double &selectivity); + + virtual bool tend_to_use_ds() override { return true; } +}; + } } diff --git a/src/sql/optimizer/ob_select_log_plan.cpp b/src/sql/optimizer/ob_select_log_plan.cpp index 1b33c59c8..186ac23bc 100644 --- a/src/sql/optimizer/ob_select_log_plan.cpp +++ b/src/sql/optimizer/ob_select_log_plan.cpp @@ -3491,8 +3491,7 @@ int ObSelectLogPlan::get_minimal_cost_set_plan(const int64_t in_parallel, bool is_local_order = right_child->get_is_local_order() && !is_fully_partition_wise; ObPQDistributeMethod::Type dist_method = ObOptimizerUtil::get_right_dist_method (*right_child->get_sharding(), set_dist_algo); - right_plan->get_selectivity_ctx().init_op_ctx( - &right_child->get_output_equal_sets(), right_child->get_card()); + right_plan->get_selectivity_ctx().init_op_ctx(right_child); info.reset(); // is single, may allocate exchange above, set need_parallel_ as 1 and compute exchange cost in cost_sort_and_exchange info.need_parallel_ = right_child->is_single() ? ObGlobalHint::DEFAULT_PARALLEL : in_parallel; @@ -4845,7 +4844,8 @@ int ObSelectLogPlan::candi_allocate_window_function_with_hint(const ObIArrayget_output_const_exprs(), orig_top->get_card(), orig_top->get_is_at_most_one_row(), - qualify_filters); + qualify_filters, + orig_top->get_ambient_card()); while (OB_SUCC(ret) && !candi_plans.empty() && !remaining_exprs.empty()) { tmp_plans.reuse(); if (OB_FAIL(init_win_func_helper_with_hint(candi_plans, @@ -5100,7 +5100,8 @@ int ObSelectLogPlan::candi_allocate_window_function(const ObIArrayget_output_const_exprs(), orig_top->get_card(), orig_top->get_is_at_most_one_row(), - qualify_filters); + qualify_filters, + orig_top->get_ambient_card()); for (int64_t i = 0; OB_SUCC(ret) && i < candi_plans.count(); ++i) { OPT_TRACE("generate window function for plan:", candi_plans.at(i)); if (OB_FAIL(generate_window_functions_plan(win_func_helper, @@ -5847,6 +5848,7 @@ int ObSelectLogPlan::calc_ndvs_and_pby_oby_prefix(const ObIArray sort_key_exprs; sort_key_ndvs.reuse(); pby_oby_prefixes.reuse(); + get_selectivity_ctx().init_op_ctx(&win_func_helper.equal_sets_, card, &win_func_helper.ambient_card_); // Get NDV for each sort_keys prefix first. for (int64_t i = 0; i < sort_keys.count() && OB_SUCC(ret); i++) { @@ -7872,6 +7874,7 @@ int ObSelectLogPlan::init_selectivity_metas_for_set(ObSelectLogPlan *sub_plan, } else if (OB_ISNULL(best_plan)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret)); + } else if (FALSE_IT(sub_plan->get_selectivity_ctx().init_op_ctx(best_plan))) { } else if (OB_FAIL(get_basic_table_metas().add_set_child_stmt_meta_info( get_stmt(), sub_stmt, child_offset, sub_plan->get_update_table_metas(), diff --git a/src/sql/optimizer/ob_select_log_plan.h b/src/sql/optimizer/ob_select_log_plan.h index eb000756d..0ae361ee9 100644 --- a/src/sql/optimizer/ob_select_log_plan.h +++ b/src/sql/optimizer/ob_select_log_plan.h @@ -485,7 +485,8 @@ private: const ObIArray &const_exprs, const double card, const bool is_at_most_one_row, - const ObIArray &qualify_filters) + const ObIArray &qualify_filters, + const ObIArray &ambient_card) : all_win_func_exprs_(all_win_func_exprs), win_dist_hint_(win_dist_hint), explicit_hint_(explicit_hint), @@ -507,7 +508,8 @@ private: enable_topn_(false), topn_const_(NULL), is_fetch_with_ties_(false), - origin_sort_card_(0.0) + origin_sort_card_(0.0), + ambient_card_(ambient_card) { } virtual ~WinFuncOpHelper() {} @@ -545,6 +547,7 @@ private: ObRawExpr* topn_const_; bool is_fetch_with_ties_; double origin_sort_card_; + const ObIArray &ambient_card_; TO_STRING_KV(K_(win_dist_method), K_(win_op_idx), @@ -557,6 +560,7 @@ private: K_(ordered_win_func_exprs), K_(win_dist_hint), K_(explicit_hint), + K_(ambient_card), K_(enable_topn), K_(topn_const), K_(is_fetch_with_ties), diff --git a/src/sql/resolver/ddl/ob_analyze_stmt_resolver.cpp b/src/sql/resolver/ddl/ob_analyze_stmt_resolver.cpp index 20bfea94d..966623f13 100644 --- a/src/sql/resolver/ddl/ob_analyze_stmt_resolver.cpp +++ b/src/sql/resolver/ddl/ob_analyze_stmt_resolver.cpp @@ -462,6 +462,7 @@ int ObAnalyzeStmtResolver::resolve_for_clause_element(const ParseNode *for_claus int ret = OB_SUCCESS; ObSEArray all_for_col; ObAnalyzeTableInfo &table_info = analyze_stmt.get_tables().at(0); + bool is_async_gather = false; if (OB_ISNULL(for_clause_node)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("null parse node", K(ret)); @@ -474,6 +475,7 @@ int ObAnalyzeStmtResolver::resolve_for_clause_element(const ParseNode *for_claus bool use_size_auto = false; if (OB_FAIL(pl::ObDbmsStats::parser_for_all_clause(for_clause_node, table_info.get_column_params(), + is_async_gather, use_size_auto))) { LOG_WARN("failed to resolve for all clause", K(ret)); } else { diff --git a/src/sql/resolver/dml/ob_dml_stmt.cpp b/src/sql/resolver/dml/ob_dml_stmt.cpp index a1669c360..b2b8ff088 100644 --- a/src/sql/resolver/dml/ob_dml_stmt.cpp +++ b/src/sql/resolver/dml/ob_dml_stmt.cpp @@ -3019,6 +3019,26 @@ int ObDMLStmt::relids_to_table_ids(const ObSqlBitSet<> &table_set, return ret; } +int ObDMLStmt::relids_to_table_ids(const ObRelIds &table_set, + ObIArray &table_ids) const +{ + int ret = OB_SUCCESS; + TableItem *table = NULL; + int64_t idx = OB_INVALID_INDEX; + for (int64_t i = 0; OB_SUCC(ret) && i < table_items_.count(); ++i) { + if (OB_ISNULL(table = table_items_.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table item is null", K(ret)); + } else if (OB_UNLIKELY((idx = get_table_bit_index(table->table_id_)) == OB_INVALID_INDEX)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get table item invalid idx", K(idx), K(table->table_id_)); + } else if (table_set.has_member(idx)) { + ret = table_ids.push_back(table->table_id_); + } + } + return ret; +} + int ObDMLStmt::relids_to_table_items(const ObRelIds &table_set, ObIArray &tables) const { diff --git a/src/sql/resolver/dml/ob_dml_stmt.h b/src/sql/resolver/dml/ob_dml_stmt.h index 71a82785e..9c52250e9 100644 --- a/src/sql/resolver/dml/ob_dml_stmt.h +++ b/src/sql/resolver/dml/ob_dml_stmt.h @@ -959,6 +959,7 @@ public: int relids_to_table_items(const ObRelIds &table_set, ObIArray &tables) const; int relids_to_table_items(const ObSqlBitSet<> &table_set, ObIArray &tables) const; + int relids_to_table_ids(const ObRelIds &table_set, ObIArray &table_ids) const; int relids_to_table_ids(const ObSqlBitSet<> &table_set, ObIArray &table_ids) const; int get_table_rel_ids(const TableItem &target, ObSqlBitSet<> &table_set) const; int get_table_rel_ids(const ObIArray &table_ids, ObSqlBitSet<> &table_set) const; diff --git a/src/sql/resolver/dml/ob_hint.cpp b/src/sql/resolver/dml/ob_hint.cpp index d4a395e5f..b838c6051 100644 --- a/src/sql/resolver/dml/ob_hint.cpp +++ b/src/sql/resolver/dml/ob_hint.cpp @@ -876,6 +876,16 @@ bool ObOptParamHint::is_param_val_valid(const OptParamType param_type, const ObO is_valid = val.is_int() && (0 <= val.get_int() && val.get_int() <= 4); break; } + case CORRELATION_FOR_CARDINALITY_ESTIMATION: { + if (val.is_int()) { + is_valid = 0 <= val.get_int() && val.get_int() < static_cast(ObEstCorrelationType::MAX); + } else if (val.is_varchar()) { + int64_t type = OB_INVALID_ID; + ObSysVarCardinalityEstimationModel sv; + is_valid = (OB_SUCCESS == sv.find_type(val.get_varchar(), type)); + } + break; + } default: LOG_TRACE("invalid opt param val", K(param_type), K(val)); break; @@ -960,6 +970,36 @@ int ObOptParamHint::get_integer_opt_param(const OptParamType param_type, int64_t return ret; } +int ObOptParamHint::get_enum_opt_param(const OptParamType param_type, int64_t &val) const +{ + int ret = OB_SUCCESS; + ObObj obj; + if (OB_FAIL(get_opt_param(param_type, obj))) { + LOG_WARN("fail to get rowsets_enabled opt_param", K(ret)); + } else if (obj.is_nop_value()) { + // do nothing + } else if (obj.is_int()) { + val = obj.get_int(); + } else if (obj.is_varchar()) { + switch (param_type) { + case CORRELATION_FOR_CARDINALITY_ESTIMATION: { + ObSysVarCardinalityEstimationModel sv; + if (OB_FAIL(sv.find_type(obj.get_varchar(), val))) { + LOG_WARN("param obj is invalid", K(ret), K(obj)); + } + break; + } + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("enum param is invalid", K(ret), K(obj)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("param obj is invalid", K(ret), K(obj)); + } + return ret; +} + int ObOptParamHint::has_opt_param(const OptParamType param_type, bool &has_hint) const { int ret = OB_SUCCESS; diff --git a/src/sql/resolver/dml/ob_hint.h b/src/sql/resolver/dml/ob_hint.h index 135c0df63..40afd02ca 100644 --- a/src/sql/resolver/dml/ob_hint.h +++ b/src/sql/resolver/dml/ob_hint.h @@ -160,6 +160,7 @@ struct ObOptParamHint DEF(SPILL_COMPRESSION_CODEC,) \ DEF(INLIST_REWRITE_THRESHOLD,) \ DEF(PUSHDOWN_STORAGE_LEVEL,) \ + DEF(CORRELATION_FOR_CARDINALITY_ESTIMATION,) \ DECLARE_ENUM(OptParamType, opt_param, OPT_PARAM_TYPE_DEF, static); @@ -175,6 +176,7 @@ struct ObOptParamHint // if the corresponding opt_param is specified, the `val` will be overwritten int get_bool_opt_param(const OptParamType param_type, bool &val) const; int get_integer_opt_param(const OptParamType param_type, int64_t &val) const; + int get_enum_opt_param(const OptParamType param_type, int64_t &val) const; int has_opt_param(const OptParamType param_type, bool &has_hint) const; bool empty() const { return param_types_.empty(); } int check_and_get_bool_opt_param(const OptParamType param_type, bool &has_opt_param, bool &val) const; diff --git a/src/sql/resolver/expr/ob_raw_expr_util.cpp b/src/sql/resolver/expr/ob_raw_expr_util.cpp index 25b1ec17b..bf94aa0a4 100644 --- a/src/sql/resolver/expr/ob_raw_expr_util.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_util.cpp @@ -3681,7 +3681,7 @@ int ObRawExprUtils::extract_contain_exprs(ObRawExpr *raw_expr, } int ObRawExprUtils::extract_column_exprs(ObIArray &exprs, - ObRelIds &rel_ids, + const ObRelIds &rel_ids, ObIArray &column_exprs) { int ret = OB_SUCCESS; @@ -3697,7 +3697,7 @@ int ObRawExprUtils::extract_column_exprs(ObIArray &exprs, } int ObRawExprUtils::extract_column_exprs(ObRawExpr* expr, - ObRelIds &rel_ids, + const ObRelIds &rel_ids, ObIArray &column_exprs) { int ret = OB_SUCCESS; diff --git a/src/sql/resolver/expr/ob_raw_expr_util.h b/src/sql/resolver/expr/ob_raw_expr_util.h index 1b126bff5..cc62bfc11 100644 --- a/src/sql/resolver/expr/ob_raw_expr_util.h +++ b/src/sql/resolver/expr/ob_raw_expr_util.h @@ -431,10 +431,10 @@ public: static int extract_column_exprs(const ObRawExpr *expr, ObIArray &column_exprs); static int extract_column_exprs(ObRawExpr* expr, - ObRelIds &rel_ids, + const ObRelIds &rel_ids, ObIArray &column_exprs); static int extract_column_exprs(ObIArray &exprs, - ObRelIds &rel_ids, + const ObRelIds &rel_ids, ObIArray &column_exprs); static int extract_contain_exprs(ObRawExpr *raw_expr, const common::ObIArray &src_exprs, diff --git a/src/sql/rewrite/ob_transform_groupby_pullup.cpp b/src/sql/rewrite/ob_transform_groupby_pullup.cpp index 698ec1af6..caebe79b4 100644 --- a/src/sql/rewrite/ob_transform_groupby_pullup.cpp +++ b/src/sql/rewrite/ob_transform_groupby_pullup.cpp @@ -1193,7 +1193,7 @@ int ObTransformGroupByPullup::calc_group_exprs_ndv(const ObIArray &g LOG_WARN("unexpect null logical operator", K(ret)); } else { card = child_op->get_card(); - plan->get_selectivity_ctx().init_op_ctx(&child_op->get_output_equal_sets(), card); + plan->get_selectivity_ctx().init_op_ctx(child_op); if (group_exprs.empty()) { group_ndv = 1.0; } else if (OB_FAIL(ObOptSelectivity::calculate_distinct(plan->get_update_table_metas(), diff --git a/tools/deploy/mysql_test/r/mysql/view_2.result b/tools/deploy/mysql_test/r/mysql/view_2.result index 145f1ce05..f8040d647 100644 --- a/tools/deploy/mysql_test/r/mysql/view_2.result +++ b/tools/deploy/mysql_test/r/mysql/view_2.result @@ -481,7 +481,7 @@ Optimization Info: table_dop:1 dop_method:Table DOP avaiable_index_name:[t11] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:1 estimation method:[DYNAMIC SAMPLING FULL] t21: @@ -493,7 +493,7 @@ Optimization Info: table_dop:1 dop_method:Table DOP avaiable_index_name:[t21] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:1 estimation method:[DYNAMIC SAMPLING FULL] Plan Type: diff --git a/tools/deploy/mysql_test/test_suite/column_store_encoding/r/mysql/basic_cs_encoding.result b/tools/deploy/mysql_test/test_suite/column_store_encoding/r/mysql/basic_cs_encoding.result index 0b120b7ed..7e0e7c806 100644 --- a/tools/deploy/mysql_test/test_suite/column_store_encoding/r/mysql/basic_cs_encoding.result +++ b/tools/deploy/mysql_test/test_suite/column_store_encoding/r/mysql/basic_cs_encoding.result @@ -42,8 +42,8 @@ Query Plan ================================================================ |ID|OPERATOR |NAME|EST.ROWS|EST.TIME(us)| ---------------------------------------------------------------- -|0 |SORT | |6 |35 | -|1 |└─NESTED-LOOP JOIN | |6 |35 | +|0 |SORT | |5 |35 | +|1 |└─NESTED-LOOP JOIN | |5 |35 | |2 | ├─COLUMN TABLE FULL SCAN |t4 |2 |3 | |3 | └─DISTRIBUTED TABLE RANGE SCAN|t3 |3 |16 | ================================================================ diff --git a/tools/deploy/mysql_test/test_suite/datatype/r/mysql/div.result b/tools/deploy/mysql_test/test_suite/datatype/r/mysql/div.result index 88eabcee6..183cbc1c5 100644 --- a/tools/deploy/mysql_test/test_suite/datatype/r/mysql/div.result +++ b/tools/deploy/mysql_test/test_suite/datatype/r/mysql/div.result @@ -198557,7 +198557,7 @@ Query Plan |1 |└─PX COORDINATOR | |1 |6 | |2 | └─EXCHANGE OUT DISTR |:EX10000 |1 |6 | |3 | └─MERGE GROUP BY | |1 |6 | -|4 | └─NESTED-LOOP ANTI JOIN | |14 |5 | +|4 | └─NESTED-LOOP ANTI JOIN | |14 |6 | |5 | ├─PX PARTITION ITERATOR| |14 |5 | |6 | │ └─TABLE FULL SCAN |table1000_key_pk_parts_2|14 |5 | |7 | └─MATERIAL | |1 |1 | diff --git a/tools/deploy/mysql_test/test_suite/executor/r/mysql/basic.result b/tools/deploy/mysql_test/test_suite/executor/r/mysql/basic.result index 5fe19c057..be99f608b 100644 --- a/tools/deploy/mysql_test/test_suite/executor/r/mysql/basic.result +++ b/tools/deploy/mysql_test/test_suite/executor/r/mysql/basic.result @@ -351,8 +351,8 @@ Query Plan ========================================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------------------------------- -|0 |PX COORDINATOR MERGE SORT | |1 |41 | -|1 |└─EXCHANGE OUT DISTR |:EX10001 |1 |41 | +|0 |PX COORDINATOR MERGE SORT | |1 |42 | +|1 |└─EXCHANGE OUT DISTR |:EX10001 |1 |42 | |2 | └─MERGE GROUP BY | |1 |41 | |3 | └─EXCHANGE IN MERGE SORT DISTR | |20 |39 | |4 | └─EXCHANGE OUT DISTR (HASH) |:EX10000 |20 |31 | @@ -462,7 +462,7 @@ Query Plan |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------------------------------- |0 |PX COORDINATOR MERGE SORT | |1 |44 | -|1 |└─EXCHANGE OUT DISTR |:EX10001 |1 |43 | +|1 |└─EXCHANGE OUT DISTR |:EX10001 |1 |44 | |2 | └─MERGE GROUP BY | |1 |43 | |3 | └─EXCHANGE IN MERGE SORT DISTR | |20 |41 | |4 | └─EXCHANGE OUT DISTR (HASH) |:EX10000 |20 |33 | @@ -571,7 +571,7 @@ Query Plan ============================================================================ |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ---------------------------------------------------------------------------- -|0 |PX COORDINATOR MERGE SORT | |1 |60 | +|0 |PX COORDINATOR MERGE SORT | |1 |61 | |1 |└─EXCHANGE OUT DISTR |:EX10001 |1 |60 | |2 | └─MERGE GROUP BY | |1 |60 | |3 | └─EXCHANGE IN MERGE SORT DISTR | |20 |58 | diff --git a/tools/deploy/mysql_test/test_suite/geometry/r/mysql/geometry_filter_mysql.result b/tools/deploy/mysql_test/test_suite/geometry/r/mysql/geometry_filter_mysql.result index 6480e4d33..2df7cdf8a 100644 --- a/tools/deploy/mysql_test/test_suite/geometry/r/mysql/geometry_filter_mysql.result +++ b/tools/deploy/mysql_test/test_suite/geometry/r/mysql/geometry_filter_mysql.result @@ -133,7 +133,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -195,7 +195,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -257,7 +257,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -269,7 +269,7 @@ Query Plan ================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------- -|0 |TABLE FULL SCAN|t(idx)|2 |879 | +|0 |TABLE FULL SCAN|t(idx)|4 |879 | ================================================= Outputs & filters: ------------------------------------- @@ -315,12 +315,12 @@ Optimization Info: physical_range_rows:26 logical_range_rows:26 index_back_rows:6 - output_rows:1 + output_rows:3 table_dop:1 dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -382,7 +382,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -445,7 +445,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -457,7 +457,7 @@ Query Plan ================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------- -|0 |TABLE FULL SCAN|t(idx)|2 |799 | +|0 |TABLE FULL SCAN|t(idx)|3 |799 | ================================================= Outputs & filters: ------------------------------------- @@ -503,12 +503,12 @@ Optimization Info: physical_range_rows:24 logical_range_rows:24 index_back_rows:6 - output_rows:1 + output_rows:3 table_dop:1 dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -570,7 +570,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -691,7 +691,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -703,7 +703,7 @@ Query Plan ================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------- -|0 |TABLE FULL SCAN|t(idx)|1 |157 | +|0 |TABLE FULL SCAN|t(idx)|1 |158 | ================================================= Outputs & filters: ------------------------------------- @@ -753,7 +753,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -1024,7 +1024,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[geom, geo_table2] pruned_index_name:[geo_table2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -1094,7 +1094,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[geom, geo_table2] pruned_index_name:[geo_table2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -1180,7 +1180,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[geom, geo_table] pruned_index_name:[geo_table] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -1348,7 +1348,7 @@ Query Plan ================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------- -|0 |TABLE FULL SCAN|t(idx)|2 |999 | +|0 |TABLE FULL SCAN|t(idx)|4 |999 | ================================================= Outputs & filters: ------------------------------------- @@ -1393,12 +1393,12 @@ Optimization Info: physical_range_rows:31 logical_range_rows:31 index_back_rows:7 - output_rows:1 + output_rows:3 table_dop:1 dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -1410,7 +1410,7 @@ Query Plan ================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------- -|0 |TABLE FULL SCAN|t(idx)|2 |999 | +|0 |TABLE FULL SCAN|t(idx)|4 |999 | ================================================= Outputs & filters: ------------------------------------- @@ -1455,12 +1455,12 @@ Optimization Info: physical_range_rows:31 logical_range_rows:31 index_back_rows:7 - output_rows:1 + output_rows:3 table_dop:1 dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -1472,7 +1472,7 @@ Query Plan ================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------- -|0 |TABLE FULL SCAN|t(idx)|2 |999 | +|0 |TABLE FULL SCAN|t(idx)|4 |999 | ================================================= Outputs & filters: ------------------------------------- @@ -1517,12 +1517,12 @@ Optimization Info: physical_range_rows:31 logical_range_rows:31 index_back_rows:7 - output_rows:1 + output_rows:3 table_dop:1 dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -1534,7 +1534,7 @@ Query Plan ================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------- -|0 |TABLE FULL SCAN|t(idx)|3 |1128 | +|0 |TABLE FULL SCAN|t(idx)|5 |1128 | ================================================= Outputs & filters: ------------------------------------- @@ -1581,12 +1581,12 @@ Optimization Info: physical_range_rows:35 logical_range_rows:35 index_back_rows:8 - output_rows:2 + output_rows:4 table_dop:1 dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -1598,7 +1598,7 @@ Query Plan ================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------- -|0 |TABLE FULL SCAN|t(idx)|2 |999 | +|0 |TABLE FULL SCAN|t(idx)|4 |999 | ================================================= Outputs & filters: ------------------------------------- @@ -1643,12 +1643,12 @@ Optimization Info: physical_range_rows:31 logical_range_rows:31 index_back_rows:7 - output_rows:1 + output_rows:3 table_dop:1 dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -1660,7 +1660,7 @@ Query Plan ================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------- -|0 |TABLE FULL SCAN|t(idx)|2 |1031 | +|0 |TABLE FULL SCAN|t(idx)|4 |1032 | ================================================= Outputs & filters: ------------------------------------- @@ -1706,12 +1706,12 @@ Optimization Info: physical_range_rows:32 logical_range_rows:32 index_back_rows:8 - output_rows:2 + output_rows:4 table_dop:1 dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -1774,7 +1774,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -1786,7 +1786,7 @@ Query Plan ================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------- -|0 |TABLE FULL SCAN|t(idx)|2 |999 | +|0 |TABLE FULL SCAN|t(idx)|4 |999 | ================================================= Outputs & filters: ------------------------------------- @@ -1831,12 +1831,12 @@ Optimization Info: physical_range_rows:31 logical_range_rows:31 index_back_rows:7 - output_rows:1 + output_rows:3 table_dop:1 dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -2010,7 +2010,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -2072,7 +2072,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -2134,7 +2134,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -2146,7 +2146,7 @@ Query Plan ================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------- -|0 |TABLE FULL SCAN|t(idx)|4 |1526 | +|0 |TABLE FULL SCAN|t(idx)|7 |1526 | ================================================= Outputs & filters: ------------------------------------- @@ -2192,12 +2192,12 @@ Optimization Info: physical_range_rows:49 logical_range_rows:49 index_back_rows:12 - output_rows:3 + output_rows:6 table_dop:1 dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -2259,7 +2259,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -2322,7 +2322,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -2334,7 +2334,7 @@ Query Plan ================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------- -|0 |TABLE FULL SCAN|t(idx)|3 |1387 | +|0 |TABLE FULL SCAN|t(idx)|6 |1388 | ================================================= Outputs & filters: ------------------------------------- @@ -2380,12 +2380,12 @@ Optimization Info: physical_range_rows:45 logical_range_rows:45 index_back_rows:11 - output_rows:2 + output_rows:5 table_dop:1 dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -2447,7 +2447,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -2459,7 +2459,7 @@ Query Plan ================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------- -|0 |TABLE FULL SCAN|t(idx)|2 |999 | +|0 |TABLE FULL SCAN|t(idx)|4 |999 | ================================================= Outputs & filters: ------------------------------------- @@ -2504,12 +2504,12 @@ Optimization Info: physical_range_rows:31 logical_range_rows:31 index_back_rows:7 - output_rows:1 + output_rows:3 table_dop:1 dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -2521,7 +2521,7 @@ Query Plan ================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------- -|0 |TABLE FULL SCAN|t(idx)|2 |999 | +|0 |TABLE FULL SCAN|t(idx)|4 |999 | ================================================= Outputs & filters: ------------------------------------- @@ -2566,12 +2566,12 @@ Optimization Info: physical_range_rows:31 logical_range_rows:31 index_back_rows:7 - output_rows:1 + output_rows:3 table_dop:1 dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -2583,7 +2583,7 @@ Query Plan ================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------- -|0 |TABLE FULL SCAN|t(idx)|2 |999 | +|0 |TABLE FULL SCAN|t(idx)|4 |999 | ================================================= Outputs & filters: ------------------------------------- @@ -2628,12 +2628,12 @@ Optimization Info: physical_range_rows:31 logical_range_rows:31 index_back_rows:7 - output_rows:1 + output_rows:3 table_dop:1 dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -2645,7 +2645,7 @@ Query Plan ================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------- -|0 |TABLE FULL SCAN|t(idx)|3 |1128 | +|0 |TABLE FULL SCAN|t(idx)|5 |1128 | ================================================= Outputs & filters: ------------------------------------- @@ -2692,12 +2692,12 @@ Optimization Info: physical_range_rows:35 logical_range_rows:35 index_back_rows:8 - output_rows:2 + output_rows:4 table_dop:1 dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -2709,7 +2709,7 @@ Query Plan ================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------- -|0 |TABLE FULL SCAN|t(idx)|2 |999 | +|0 |TABLE FULL SCAN|t(idx)|4 |999 | ================================================= Outputs & filters: ------------------------------------- @@ -2754,12 +2754,12 @@ Optimization Info: physical_range_rows:31 logical_range_rows:31 index_back_rows:7 - output_rows:1 + output_rows:3 table_dop:1 dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -2771,7 +2771,7 @@ Query Plan ================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------- -|0 |TABLE FULL SCAN|t(idx)|2 |1031 | +|0 |TABLE FULL SCAN|t(idx)|4 |1032 | ================================================= Outputs & filters: ------------------------------------- @@ -2817,12 +2817,12 @@ Optimization Info: physical_range_rows:32 logical_range_rows:32 index_back_rows:8 - output_rows:2 + output_rows:4 table_dop:1 dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -2885,7 +2885,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -2897,7 +2897,7 @@ Query Plan ================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------- -|0 |TABLE FULL SCAN|t(idx)|2 |999 | +|0 |TABLE FULL SCAN|t(idx)|4 |999 | ================================================= Outputs & filters: ------------------------------------- @@ -2942,12 +2942,12 @@ Optimization Info: physical_range_rows:31 logical_range_rows:31 index_back_rows:7 - output_rows:1 + output_rows:3 table_dop:1 dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -3031,7 +3031,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[idx, t] pruned_index_name:[t] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: diff --git a/tools/deploy/mysql_test/test_suite/geometry/r/mysql/geometry_index2_mysql.result b/tools/deploy/mysql_test/test_suite/geometry/r/mysql/geometry_index2_mysql.result index 088d643c6..535014443 100644 --- a/tools/deploy/mysql_test/test_suite/geometry/r/mysql/geometry_index2_mysql.result +++ b/tools/deploy/mysql_test/test_suite/geometry/r/mysql/geometry_index2_mysql.result @@ -247,7 +247,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[idx, tt2] pruned_index_name:[idx] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -296,7 +296,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[idx, tt2] pruned_index_name:[idx] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -438,7 +438,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[idx, tt2] unstable_index_name:[tt2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -577,7 +577,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[idx, tt2] pruned_index_name:[tt2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -716,7 +716,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[idx, tt2] unstable_index_name:[tt2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -855,7 +855,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[idx, tt2] pruned_index_name:[tt2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: diff --git a/tools/deploy/mysql_test/test_suite/geometry/r/mysql/geometry_partition_table_mysql.result b/tools/deploy/mysql_test/test_suite/geometry/r/mysql/geometry_partition_table_mysql.result index 47c1a2405..98b28e096 100644 --- a/tools/deploy/mysql_test/test_suite/geometry/r/mysql/geometry_partition_table_mysql.result +++ b/tools/deploy/mysql_test/test_suite/geometry/r/mysql/geometry_partition_table_mysql.result @@ -61,10 +61,10 @@ Query Plan ============================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------------------- -|0 |PX COORDINATOR | |1 |543 | -|1 |└─EXCHANGE OUT DISTR |:EX10000|1 |542 | -|2 | └─PX PARTITION ITERATOR| |1 |541 | -|3 | └─TABLE FULL SCAN |t1(idx) |1 |541 | +|0 |PX COORDINATOR | |2 |544 | +|1 |└─EXCHANGE OUT DISTR |:EX10000|2 |543 | +|2 | └─PX PARTITION ITERATOR| |2 |541 | +|3 | └─TABLE FULL SCAN |t1(idx) |2 |541 | ============================================================= Outputs & filters: ------------------------------------- diff --git a/tools/deploy/mysql_test/test_suite/geometry/r/mysql/spatial_relation_join_mysql.result b/tools/deploy/mysql_test/test_suite/geometry/r/mysql/spatial_relation_join_mysql.result index 06ce1c9ae..bc29ba4c5 100644 --- a/tools/deploy/mysql_test/test_suite/geometry/r/mysql/spatial_relation_join_mysql.result +++ b/tools/deploy/mysql_test/test_suite/geometry/r/mysql/spatial_relation_join_mysql.result @@ -89,7 +89,7 @@ Optimization Info: table_dop:1 dop_method:Table DOP avaiable_index_name:[tgnoindex2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:1 estimation method:[DYNAMIC SAMPLING FULL] tgnoindex1: @@ -101,7 +101,7 @@ Optimization Info: table_dop:1 dop_method:Table DOP avaiable_index_name:[tgnoindex1] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:1 estimation method:[DYNAMIC SAMPLING FULL] Plan Type: @@ -196,7 +196,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[gidx1, tgeom1] pruned_index_name:[gidx1] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:1 estimation method:[DYNAMIC SAMPLING FULL] tgeom2: @@ -208,7 +208,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[gidx2, tgeom2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:1 estimation method:[DYNAMIC SAMPLING BASIC] Plan Type: @@ -312,7 +312,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[pgidx1, ptgeom1] pruned_index_name:[pgidx1] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:1 estimation method:[DYNAMIC SAMPLING FULL] ptgeom2: @@ -324,7 +324,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[pgidx2, ptgeom2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:1 estimation method:[DYNAMIC SAMPLING BASIC] Plan Type: diff --git a/tools/deploy/mysql_test/test_suite/join/r/mysql/anti_semi_join.result b/tools/deploy/mysql_test/test_suite/join/r/mysql/anti_semi_join.result index 0a9e0afe8..944f13961 100644 --- a/tools/deploy/mysql_test/test_suite/join/r/mysql/anti_semi_join.result +++ b/tools/deploy/mysql_test/test_suite/join/r/mysql/anti_semi_join.result @@ -954,7 +954,7 @@ Query Plan ======================================================== |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| -------------------------------------------------------- -|0 |NESTED-LOOP ANTI JOIN | |66 |55 | +|0 |NESTED-LOOP ANTI JOIN | |66 |56 | |1 |├─TABLE FULL SCAN |xy_x_t|74 |6 | |2 |└─MATERIAL | |16 |6 | |3 | └─TABLE FULL SCAN |xy_y_t|16 |3 | @@ -2352,7 +2352,7 @@ Query Plan ======================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------------- -|0 |NESTED-LOOP ANTI JOIN | |1 |14 | +|0 |NESTED-LOOP ANTI JOIN | |1 |13 | |1 |├─TABLE FULL SCAN |xy_t2|8 |3 | |2 |└─MATERIAL | |116 |9 | |3 | └─TABLE FULL SCAN |xy_t1|116 |6 | @@ -2751,8 +2751,8 @@ Query Plan ==================================================== |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ---------------------------------------------------- -|0 |SCALAR GROUP BY | |1 |582 | -|1 |└─SUBPLAN FILTER | |29 |581 | +|0 |SCALAR GROUP BY | |1 |583 | +|1 |└─SUBPLAN FILTER | |42 |581 | |2 | ├─TABLE FULL SCAN|xy_t1|116 |8 | |3 | ├─TABLE FULL SCAN|xy_t3|1 |3 | |4 | └─TABLE FULL SCAN|xy_t2|1 |3 | @@ -2829,7 +2829,7 @@ Query Plan |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------------------------------- |0 |SCALAR GROUP BY | |1 |4168 | -|1 |└─SUBPLAN FILTER | |29 |4167 | +|1 |└─SUBPLAN FILTER | |42 |4167 | |2 | ├─TABLE FULL SCAN |xy_t1 |116 |8 | |3 | ├─DISTRIBUTED TABLE RANGE SCAN|xy_t3(idx_c2)|1 |18 | |4 | └─DISTRIBUTED TABLE RANGE SCAN|xy_t2(idx_c2)|1 |18 | @@ -4318,7 +4318,7 @@ Query Plan ======================================================== |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| -------------------------------------------------------- -|0 |NESTED-LOOP ANTI JOIN | |5 |96 | +|0 |NESTED-LOOP ANTI JOIN | |5 |97 | |1 |├─TABLE FULL SCAN |xy_t2|5 |3 | |2 |└─DISTRIBUTED TABLE GET|xy_t1|1 |18 | ======================================================== diff --git a/tools/deploy/mysql_test/test_suite/join/r/mysql/join_merge.result b/tools/deploy/mysql_test/test_suite/join/r/mysql/join_merge.result index f52789986..038324228 100644 --- a/tools/deploy/mysql_test/test_suite/join/r/mysql/join_merge.result +++ b/tools/deploy/mysql_test/test_suite/join/r/mysql/join_merge.result @@ -19,7 +19,7 @@ Query Plan =================================================== |ID|OPERATOR |NAME|EST.ROWS|EST.TIME(us)| --------------------------------------------------- -|0 |MERGE JOIN | |5 |7 | +|0 |MERGE JOIN | |6 |7 | |1 |├─SORT | |5 |3 | |2 |│ └─TABLE FULL SCAN|bb |5 |3 | |3 |└─SORT | |6 |4 | @@ -52,7 +52,7 @@ Query Plan =================================================== |ID|OPERATOR |NAME|EST.ROWS|EST.TIME(us)| --------------------------------------------------- -|0 |MERGE JOIN | |5 |7 | +|0 |MERGE JOIN | |6 |7 | |1 |├─SORT | |5 |3 | |2 |│ └─TABLE FULL SCAN|bb |5 |3 | |3 |└─SORT | |6 |4 | @@ -85,7 +85,7 @@ Query Plan =================================================== |ID|OPERATOR |NAME|EST.ROWS|EST.TIME(us)| --------------------------------------------------- -|0 |MERGE JOIN | |5 |7 | +|0 |MERGE JOIN | |6 |7 | |1 |├─SORT | |5 |3 | |2 |│ └─TABLE FULL SCAN|bb |5 |3 | |3 |└─SORT | |6 |4 | @@ -118,7 +118,7 @@ Query Plan =================================================== |ID|OPERATOR |NAME|EST.ROWS|EST.TIME(us)| --------------------------------------------------- -|0 |MERGE JOIN | |5 |7 | +|0 |MERGE JOIN | |6 |7 | |1 |├─SORT | |5 |3 | |2 |│ └─TABLE FULL SCAN|bb |5 |3 | |3 |└─SORT | |6 |4 | @@ -155,7 +155,7 @@ Query Plan =================================================== |ID|OPERATOR |NAME|EST.ROWS|EST.TIME(us)| --------------------------------------------------- -|0 |MERGE JOIN | |5 |8 | +|0 |MERGE JOIN | |6 |8 | |1 |├─SORT | |5 |4 | |2 |│ └─TABLE FULL SCAN|bb |5 |3 | |3 |└─SORT | |6 |4 | @@ -188,7 +188,7 @@ Query Plan =================================================== |ID|OPERATOR |NAME|EST.ROWS|EST.TIME(us)| --------------------------------------------------- -|0 |MERGE JOIN | |5 |8 | +|0 |MERGE JOIN | |6 |8 | |1 |├─SORT | |5 |4 | |2 |│ └─TABLE FULL SCAN|bb |5 |3 | |3 |└─SORT | |6 |4 | @@ -221,7 +221,7 @@ Query Plan =================================================== |ID|OPERATOR |NAME|EST.ROWS|EST.TIME(us)| --------------------------------------------------- -|0 |MERGE JOIN | |5 |8 | +|0 |MERGE JOIN | |6 |8 | |1 |├─SORT | |5 |4 | |2 |│ └─TABLE FULL SCAN|bb |5 |3 | |3 |└─SORT | |6 |4 | @@ -254,7 +254,7 @@ Query Plan =================================================== |ID|OPERATOR |NAME|EST.ROWS|EST.TIME(us)| --------------------------------------------------- -|0 |MERGE JOIN | |5 |8 | +|0 |MERGE JOIN | |6 |8 | |1 |├─SORT | |5 |4 | |2 |│ └─TABLE FULL SCAN|bb |5 |3 | |3 |└─SORT | |6 |4 | @@ -289,7 +289,7 @@ Query Plan |ID|OPERATOR |NAME|EST.ROWS|EST.TIME(us)| ----------------------------------------------------- |0 |MERGE JOIN | |2 |12 | -|1 |├─MERGE JOIN | |5 |8 | +|1 |├─MERGE JOIN | |6 |8 | |2 |│ ├─SORT | |6 |4 | |3 |│ │ └─TABLE FULL SCAN|aa |6 |3 | |4 |│ └─SORT | |5 |4 | @@ -333,7 +333,7 @@ Query Plan |ID|OPERATOR |NAME|EST.ROWS|EST.TIME(us)| ----------------------------------------------------- |0 |MERGE JOIN | |2 |12 | -|1 |├─MERGE JOIN | |5 |8 | +|1 |├─MERGE JOIN | |6 |8 | |2 |│ ├─SORT | |6 |4 | |3 |│ │ └─TABLE FULL SCAN|aa |6 |3 | |4 |│ └─SORT | |5 |4 | @@ -476,7 +476,7 @@ Query Plan ================================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ----------------------------------------------------------------- -|0 |MERGE JOIN | |5 |35 | +|0 |MERGE JOIN | |6 |35 | |1 |├─PX COORDINATOR MERGE SORT | |5 |17 | |2 |│ └─EXCHANGE OUT DISTR |:EX10000|5 |15 | |3 |│ └─SORT | |5 |12 | @@ -527,7 +527,7 @@ Query Plan ================================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ----------------------------------------------------------------- -|0 |MERGE JOIN | |5 |35 | +|0 |MERGE JOIN | |6 |35 | |1 |├─PX COORDINATOR MERGE SORT | |5 |17 | |2 |│ └─EXCHANGE OUT DISTR |:EX10000|5 |15 | |3 |│ └─SORT | |5 |12 | @@ -578,7 +578,7 @@ Query Plan ================================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ----------------------------------------------------------------- -|0 |MERGE JOIN | |5 |35 | +|0 |MERGE JOIN | |6 |35 | |1 |├─PX COORDINATOR MERGE SORT | |5 |17 | |2 |│ └─EXCHANGE OUT DISTR |:EX10000|5 |15 | |3 |│ └─SORT | |5 |12 | @@ -629,7 +629,7 @@ Query Plan ================================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ----------------------------------------------------------------- -|0 |MERGE JOIN | |5 |35 | +|0 |MERGE JOIN | |6 |35 | |1 |├─PX COORDINATOR MERGE SORT | |5 |17 | |2 |│ └─EXCHANGE OUT DISTR |:EX10000|5 |15 | |3 |│ └─SORT | |5 |12 | @@ -684,7 +684,7 @@ Query Plan ================================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ----------------------------------------------------------------- -|0 |MERGE JOIN | |5 |35 | +|0 |MERGE JOIN | |6 |35 | |1 |├─PX COORDINATOR MERGE SORT | |5 |17 | |2 |│ └─EXCHANGE OUT DISTR |:EX10000|5 |15 | |3 |│ └─SORT | |5 |12 | @@ -735,7 +735,7 @@ Query Plan ================================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ----------------------------------------------------------------- -|0 |MERGE JOIN | |5 |35 | +|0 |MERGE JOIN | |6 |35 | |1 |├─PX COORDINATOR MERGE SORT | |5 |17 | |2 |│ └─EXCHANGE OUT DISTR |:EX10000|5 |15 | |3 |│ └─SORT | |5 |12 | @@ -786,7 +786,7 @@ Query Plan ================================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ----------------------------------------------------------------- -|0 |MERGE JOIN | |5 |35 | +|0 |MERGE JOIN | |6 |35 | |1 |├─PX COORDINATOR MERGE SORT | |5 |17 | |2 |│ └─EXCHANGE OUT DISTR |:EX10000|5 |15 | |3 |│ └─SORT | |5 |12 | @@ -837,7 +837,7 @@ Query Plan ================================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ----------------------------------------------------------------- -|0 |MERGE JOIN | |5 |35 | +|0 |MERGE JOIN | |6 |35 | |1 |├─PX COORDINATOR MERGE SORT | |5 |17 | |2 |│ └─EXCHANGE OUT DISTR |:EX10000|5 |15 | |3 |│ └─SORT | |5 |12 | @@ -889,8 +889,8 @@ Query Plan =================================================================== |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------------------------- -|0 |MERGE JOIN | |2 |50 | -|1 |├─MERGE JOIN | |5 |35 | +|0 |MERGE JOIN | |2 |51 | +|1 |├─MERGE JOIN | |6 |35 | |2 |│ ├─PX COORDINATOR MERGE SORT | |6 |18 | |3 |│ │ └─EXCHANGE OUT DISTR |:EX10000|6 |16 | |4 |│ │ └─SORT | |6 |12 | @@ -960,8 +960,8 @@ Query Plan =================================================================== |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------------------------- -|0 |MERGE JOIN | |2 |50 | -|1 |├─MERGE JOIN | |5 |35 | +|0 |MERGE JOIN | |2 |51 | +|1 |├─MERGE JOIN | |6 |35 | |2 |│ ├─PX COORDINATOR MERGE SORT | |6 |18 | |3 |│ │ └─EXCHANGE OUT DISTR |:EX10000|6 |16 | |4 |│ │ └─SORT | |6 |12 | @@ -1659,7 +1659,7 @@ Query Plan |ID|OPERATOR |NAME|EST.ROWS|EST.TIME(us)| --------------------------------------------------------- |0 |SORT | |13 |9 | -|1 |└─NESTED-LOOP OUTER JOIN | |13 |5 | +|1 |└─NESTED-LOOP OUTER JOIN | |13 |4 | |2 | ├─TABLE FULL SCAN |t2 |13 |3 | |3 | └─MATERIAL | |5 |4 | |4 | └─TABLE FULL SCAN |t7 |5 |3 | @@ -1700,7 +1700,7 @@ Query Plan ==================================================== |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ---------------------------------------------------- -|0 |MERGE JOIN | |7 |6 | +|0 |MERGE JOIN | |8 |6 | |1 |├─TABLE FULL SCAN|t8(idx)|6 |3 | |2 |└─TABLE FULL SCAN|t9(idx)|7 |3 | ==================================================== @@ -1761,7 +1761,7 @@ Query Plan ==================================================== |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ---------------------------------------------------- -|0 |MERGE OUTER JOIN | |7 |6 | +|0 |MERGE OUTER JOIN | |8 |6 | |1 |├─TABLE FULL SCAN|t8(idx)|6 |3 | |2 |└─TABLE FULL SCAN|t9(idx)|7 |3 | ==================================================== @@ -1827,7 +1827,7 @@ Query Plan ========================================================== |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ---------------------------------------------------------- -|0 |MERGE RIGHT OUTER JOIN | |7 |6 | +|0 |MERGE RIGHT OUTER JOIN | |8 |6 | |1 |├─TABLE FULL SCAN |t8(idx)|6 |3 | |2 |└─TABLE FULL SCAN |t9(idx)|7 |3 | ========================================================== @@ -1895,7 +1895,7 @@ Query Plan ========================================================== |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ---------------------------------------------------------- -|0 |MERGE RIGHT OUTER JOIN | |7 |6 | +|0 |MERGE RIGHT OUTER JOIN | |8 |6 | |1 |├─TABLE FULL SCAN |t8(idx)|6 |3 | |2 |└─TABLE FULL SCAN |t9(idx)|7 |3 | ========================================================== diff --git a/tools/deploy/mysql_test/test_suite/join/r/mysql/nested_loop_join_right_null_joinon_where.result b/tools/deploy/mysql_test/test_suite/join/r/mysql/nested_loop_join_right_null_joinon_where.result index d2e6e9ef8..2d3c1c522 100644 --- a/tools/deploy/mysql_test/test_suite/join/r/mysql/nested_loop_join_right_null_joinon_where.result +++ b/tools/deploy/mysql_test/test_suite/join/r/mysql/nested_loop_join_right_null_joinon_where.result @@ -52,7 +52,7 @@ Query Plan ================================================== |ID|OPERATOR |NAME|EST.ROWS|EST.TIME(us)| -------------------------------------------------- -|0 |MERGE JOIN | |2 |5 | +|0 |MERGE JOIN | |1 |5 | |1 |├─TABLE RANGE SCAN|t1 |1 |3 | |2 |└─TABLE RANGE SCAN|t2 |2 |3 | ================================================== @@ -82,7 +82,7 @@ Query Plan ================================================== |ID|OPERATOR |NAME|EST.ROWS|EST.TIME(us)| -------------------------------------------------- -|0 |MERGE JOIN | |2 |5 | +|0 |MERGE JOIN | |1 |5 | |1 |├─TABLE RANGE SCAN|a |1 |3 | |2 |└─TABLE RANGE SCAN|b |2 |3 | ================================================== diff --git a/tools/deploy/mysql_test/test_suite/px/r/mysql/agg.result b/tools/deploy/mysql_test/test_suite/px/r/mysql/agg.result index c1ead3a1f..22d904b07 100644 --- a/tools/deploy/mysql_test/test_suite/px/r/mysql/agg.result +++ b/tools/deploy/mysql_test/test_suite/px/r/mysql/agg.result @@ -110,12 +110,12 @@ Query Plan ===================================================================== |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| --------------------------------------------------------------------- -|0 |PX COORDINATOR | |9 |66 | -|1 |└─EXCHANGE OUT DISTR |:EX10001|9 |57 | -|2 | └─HASH GROUP BY | |9 |48 | -|3 | └─EXCHANGE IN DISTR | |16 |45 | -|4 | └─EXCHANGE OUT DISTR (HASH)|:EX10000|16 |37 | -|5 | └─HASH GROUP BY | |16 |20 | +|0 |PX COORDINATOR | |12 |79 | +|1 |└─EXCHANGE OUT DISTR |:EX10001|12 |68 | +|2 | └─HASH GROUP BY | |12 |55 | +|3 | └─EXCHANGE IN DISTR | |19 |51 | +|4 | └─EXCHANGE OUT DISTR (HASH)|:EX10000|19 |42 | +|5 | └─HASH GROUP BY | |19 |21 | |6 | └─PX PARTITION ITERATOR| |28 |16 | |7 | └─MERGE JOIN | |28 |16 | |8 | ├─TABLE FULL SCAN |score |28 |8 | diff --git a/tools/deploy/mysql_test/test_suite/px/r/mysql/alloc_material_for_producer_consumer_schedule_mode.result b/tools/deploy/mysql_test/test_suite/px/r/mysql/alloc_material_for_producer_consumer_schedule_mode.result index 4daa7ec65..0bd6dc828 100644 --- a/tools/deploy/mysql_test/test_suite/px/r/mysql/alloc_material_for_producer_consumer_schedule_mode.result +++ b/tools/deploy/mysql_test/test_suite/px/r/mysql/alloc_material_for_producer_consumer_schedule_mode.result @@ -8,7 +8,7 @@ Query Plan ==================================================================================== |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------------------------------------------ -|0 |TEMP TABLE TRANSFORMATION | |1 |4 | +|0 |TEMP TABLE TRANSFORMATION | |1 |6 | |1 |├─PX COORDINATOR | |0 |3 | |2 |│ └─EXCHANGE OUT DISTR |:EX10001 |0 |3 | |3 |│ └─TEMP TABLE INSERT |TEMP1 |0 |3 | @@ -18,10 +18,10 @@ Query Plan |7 |│ └─HASH GROUP BY | |1 |2 | |8 |│ └─PX BLOCK ITERATOR | |1 |2 | |9 |│ └─TABLE FULL SCAN |t1 |1 |2 | -|10|└─PX COORDINATOR | |1 |2 | -|11| └─EXCHANGE OUT DISTR |:EX20002 |1 |2 | -|12| └─SHARED HASH JOIN | |1 |1 | -|13| ├─EXCHANGE IN DISTR | |1 |1 | +|10|└─PX COORDINATOR | |1 |3 | +|11| └─EXCHANGE OUT DISTR |:EX20002 |1 |3 | +|12| └─SHARED HASH JOIN | |1 |2 | +|13| ├─EXCHANGE IN DISTR | |1 |2 | |14| │ └─EXCHANGE OUT DISTR (BC2HOST) |:EX20001 |1 |1 | |15| │ └─SHARED HASH JOIN | |1 |1 | |16| │ ├─EXCHANGE IN DISTR | |1 |1 | @@ -73,11 +73,11 @@ Query Plan ===================================================================================== |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------------------------------------------- -|0 |PX COORDINATOR | |1 |5 | +|0 |PX COORDINATOR | |1 |6 | |1 |└─EXCHANGE OUT DISTR |:EX10003|1 |5 | |2 | └─HASH GROUP BY | |1 |5 | |3 | └─EXCHANGE IN DISTR | |1 |5 | -|4 | └─EXCHANGE OUT DISTR (HASH) |:EX10002|1 |4 | +|4 | └─EXCHANGE OUT DISTR (HASH) |:EX10002|1 |5 | |5 | └─HASH GROUP BY | |1 |4 | |6 | └─SUBPLAN SCAN |VIEW1 |1 |4 | |7 | └─MERGE GROUP BY | |1 |4 | @@ -140,7 +140,7 @@ Query Plan =========================================================================== |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| --------------------------------------------------------------------------- -|0 |PX COORDINATOR | |1 |5 | +|0 |PX COORDINATOR | |1 |6 | |1 |└─EXCHANGE OUT DISTR |:EX10002|1 |5 | |2 | └─MERGE GROUP BY | |1 |4 | |3 | └─PARTITION SORT | |1 |4 | diff --git a/tools/deploy/mysql_test/test_suite/skyline/r/mysql/skyline_basic_mysql.result b/tools/deploy/mysql_test/test_suite/skyline/r/mysql/skyline_basic_mysql.result index 0e7fe1fff..b5f983095 100644 --- a/tools/deploy/mysql_test/test_suite/skyline/r/mysql/skyline_basic_mysql.result +++ b/tools/deploy/mysql_test/test_suite/skyline/r/mysql/skyline_basic_mysql.result @@ -941,7 +941,7 @@ Query Plan ============================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------------------- -|0 |MERGE UNION DISTINCT| |1 |5 | +|0 |MERGE UNION DISTINCT| |2 |5 | |1 |├─TABLE FULL SCAN |t1(idx_a_b_c)|1 |3 | |2 |└─TABLE FULL SCAN |t2(idx_x_y_z)|1 |3 | ============================================================= @@ -961,7 +961,7 @@ Query Plan ============================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------------------- -|0 |MERGE UNION DISTINCT| |1 |5 | +|0 |MERGE UNION DISTINCT| |2 |5 | |1 |├─TABLE FULL SCAN |t1(idx_a_b_c)|1 |3 | |2 |└─TABLE FULL SCAN |t2(idx_x_y_z)|1 |3 | ============================================================= @@ -981,7 +981,7 @@ Query Plan ============================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------------------- -|0 |MERGE UNION DISTINCT| |1 |5 | +|0 |MERGE UNION DISTINCT| |2 |5 | |1 |├─TABLE RANGE SCAN |t1(idx_b_c_a)|1 |3 | |2 |└─TABLE RANGE SCAN |t2(idx_x_y_z)|1 |3 | ============================================================= @@ -1003,7 +1003,7 @@ Query Plan ============================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------------------- -|0 |MERGE UNION DISTINCT| |1 |5 | +|0 |MERGE UNION DISTINCT| |2 |5 | |1 |├─TABLE RANGE SCAN |t1(idx_b_c_a)|1 |3 | |2 |└─TABLE RANGE SCAN |t2(idx_x_y_z)|1 |3 | ============================================================= @@ -1466,7 +1466,7 @@ Query Plan ================================================================ |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ---------------------------------------------------------------- -|0 |MERGE UNION DISTINCT | |1 |7 | +|0 |MERGE UNION DISTINCT | |2 |7 | |1 |├─TABLE FULL SCAN |t2(idx_x_y_z)|1 |3 | |2 |└─SORT | |1 |5 | |3 | └─MERGE JOIN | |1 |5 | @@ -3080,7 +3080,7 @@ Query Plan ================================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ----------------------------------------------------------------- -|0 |MERGE UNION DISTINCT| |1 |5 | +|0 |MERGE UNION DISTINCT| |2 |5 | |1 |├─TABLE RANGE SCAN |t6(idx_b_e_d_c_a)|1 |3 | |2 |└─TABLE FULL SCAN |tmp(idx_c1_c2_c3)|1 |3 | ================================================================= @@ -3101,7 +3101,7 @@ Query Plan ================================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ----------------------------------------------------------------- -|0 |MERGE UNION DISTINCT| |1 |5 | +|0 |MERGE UNION DISTINCT| |2 |5 | |1 |├─TABLE FULL SCAN |t6(idx_b_e_d_c_a)|1 |3 | |2 |└─TABLE FULL SCAN |tmp(idx_c1_c2_c3)|1 |3 | ================================================================= @@ -3121,7 +3121,7 @@ Query Plan ================================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ----------------------------------------------------------------- -|0 |MERGE UNION DISTINCT| |1 |10 | +|0 |MERGE UNION DISTINCT| |2 |10 | |1 |├─TABLE FULL SCAN |tmp(idx_c1_c2_c3)|1 |3 | |2 |└─SORT | |1 |8 | |3 | └─TABLE FULL SCAN |t6(idx_b_c) |1 |8 | @@ -3144,7 +3144,7 @@ Query Plan ================================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ----------------------------------------------------------------- -|0 |MERGE UNION DISTINCT| |1 |10 | +|0 |MERGE UNION DISTINCT| |2 |10 | |1 |├─TABLE FULL SCAN |tmp(idx_c1_c2_c3)|1 |3 | |2 |└─SORT | |1 |8 | |3 | └─TABLE FULL SCAN |t6(idx_b_a_c) |1 |8 | @@ -3167,7 +3167,7 @@ Query Plan ================================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ----------------------------------------------------------------- -|0 |MERGE UNION DISTINCT| |1 |5 | +|0 |MERGE UNION DISTINCT| |2 |5 | |1 |├─TABLE FULL SCAN |t6(idx_b_e_d_c_a)|1 |3 | |2 |└─TABLE FULL SCAN |tmp(idx_c1_c2_c3)|1 |3 | ================================================================= @@ -3762,7 +3762,7 @@ Optimization Info: avaiable_index_name:[t10i1, t10i2, t10i3, t10] pruned_index_name:[t10i3] unstable_index_name:[t10] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -3812,7 +3812,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[t10i1, t10i2, t10i3, t10] unstable_index_name:[t10] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -3861,7 +3861,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[t10i1, t10i2, t10i3, t10] pruned_index_name:[t10i3] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -3910,7 +3910,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[t10i1, t10i2, t10i3, t10] pruned_index_name:[t10i3] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -3958,7 +3958,7 @@ Optimization Info: table_dop:1 dop_method:Table DOP avaiable_index_name:[t10i1, t10i2, t10i3, t10] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -4009,7 +4009,7 @@ Optimization Info: avaiable_index_name:[t10i1, t10i2, t10i3, t10] pruned_index_name:[t10i1] unstable_index_name:[t10] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -4061,7 +4061,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[t10i1, t10i2, t10i3, t10] pruned_index_name:[t10i1, t10i3] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -4113,7 +4113,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[t10i1, t10i2, t10i3, t10] pruned_index_name:[t10i2, t10i3] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -4165,7 +4165,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[t10i1, t10i2, t10i3, t10] pruned_index_name:[t10i1, t10i3] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -4217,7 +4217,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[t10i1, t10i2, t10i3, t10] pruned_index_name:[t10i1, t10i2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -4266,7 +4266,7 @@ Optimization Info: table_dop:1 dop_method:Table DOP avaiable_index_name:[t11i1, t11] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -4315,7 +4315,7 @@ Optimization Info: table_dop:1 dop_method:Table DOP avaiable_index_name:[t11i1, t11] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -4365,7 +4365,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[t11i1, t11] pruned_index_name:[t11i1] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -4415,7 +4415,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[t11i1, t11] pruned_index_name:[t11i1] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -4467,7 +4467,7 @@ Optimization Info: table_dop:1 dop_method:Table DOP avaiable_index_name:[t11i1, t11] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -4518,7 +4518,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[t11i1, t11] pruned_index_name:[t11i1] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -4569,7 +4569,7 @@ Optimization Info: table_dop:1 dop_method:Table DOP avaiable_index_name:[t11i1, t11] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -4618,7 +4618,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[t11i1, t11] pruned_index_name:[t11] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -4668,7 +4668,7 @@ Optimization Info: table_dop:1 dop_method:Table DOP avaiable_index_name:[t11i1, t11] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -4723,7 +4723,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[t11i1, t11] pruned_index_name:[t11i1] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -4772,7 +4772,7 @@ Optimization Info: table_dop:1 dop_method:Table DOP avaiable_index_name:[t12i1, t12i2, t12] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -4821,7 +4821,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[t12i1, t12i2, t12] pruned_index_name:[t12i1, t12i2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -4870,7 +4870,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[t12i1, t12i2, t12] pruned_index_name:[t12i1] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -4919,7 +4919,7 @@ Optimization Info: dop_method:Table DOP avaiable_index_name:[t12i1, t12i2, t12] pruned_index_name:[t12i1, t12i2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: diff --git a/tools/deploy/mysql_test/test_suite/skyline/r/mysql/skyline_complicate_mysql.result b/tools/deploy/mysql_test/test_suite/skyline/r/mysql/skyline_complicate_mysql.result index c3d5f81a3..e8f75dcca 100644 --- a/tools/deploy/mysql_test/test_suite/skyline/r/mysql/skyline_complicate_mysql.result +++ b/tools/deploy/mysql_test/test_suite/skyline/r/mysql/skyline_complicate_mysql.result @@ -725,33 +725,30 @@ Outputs & filters: range_key([other.c1]), range(MIN ; MAX)always true explain select max(v1), tenant_id, v6 from skyline_int join other on v3 = c1 group by v5, v4, v3 order by v3, v4, v5; Query Plan -================================================================================ -|ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| --------------------------------------------------------------------------------- -|0 |SORT | |1 |5 | -|1 |└─HASH GROUP BY | |1 |5 | -|2 | └─MERGE JOIN | |1 |5 | -|3 | ├─TABLE FULL SCAN|other |1 |3 | -|4 | └─TABLE FULL SCAN|skyline_int(idx_v3_v4_v5_v6_v2)|1 |3 | -================================================================================ +============================================================================== +|ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| +------------------------------------------------------------------------------ +|0 |MERGE GROUP BY | |1 |5 | +|1 |└─MERGE JOIN | |1 |5 | +|2 | ├─TABLE FULL SCAN|skyline_int(idx_v3_v4_v5_v6_v2)|1 |3 | +|3 | └─TABLE FULL SCAN|other |1 |3 | +============================================================================== Outputs & filters: ------------------------------------- 0 - output([T_FUN_MAX(skyline_int.v1)], [skyline_int.tenant_id], [skyline_int.v6]), filter(nil), rowset=16 - sort_keys([skyline_int.v3, ASC], [skyline_int.v4, ASC], [skyline_int.v5, ASC]) - 1 - output([skyline_int.v3], [skyline_int.v4], [skyline_int.v5], [T_FUN_MAX(skyline_int.v1)], [skyline_int.tenant_id], [skyline_int.v6]), filter(nil), rowset=16 - group([skyline_int.v5], [skyline_int.v4], [skyline_int.v3]), agg_func([T_FUN_MAX(skyline_int.v1)]) - 2 - output([skyline_int.v3], [skyline_int.v4], [skyline_int.v5], [skyline_int.tenant_id], [skyline_int.v6], [skyline_int.v1]), filter(nil), rowset=16 + group([skyline_int.v3], [skyline_int.v4], [skyline_int.v5]), agg_func([T_FUN_MAX(skyline_int.v1)]) + 1 - output([skyline_int.v3], [skyline_int.v4], [skyline_int.v5], [skyline_int.tenant_id], [skyline_int.v6], [skyline_int.v1]), filter(nil), rowset=16 equal_conds([skyline_int.v3 = other.c1]), other_conds(nil) merge_directions([ASC]) - 3 - output([other.c1]), filter(nil), rowset=16 - access([other.c1]), partitions(p0) - is_index_back=false, is_global_index=false, - range_key([other.c1]), range(MIN ; MAX)always true - 4 - output([skyline_int.v1], [skyline_int.tenant_id], [skyline_int.v3], [skyline_int.v6], [skyline_int.v5], [skyline_int.v4]), filter(nil), rowset=16 + 2 - output([skyline_int.v1], [skyline_int.tenant_id], [skyline_int.v3], [skyline_int.v6], [skyline_int.v5], [skyline_int.v4]), filter(nil), rowset=16 access([skyline_int.v1], [skyline_int.tenant_id], [skyline_int.v3], [skyline_int.v6], [skyline_int.v5], [skyline_int.v4]), partitions(p0) is_index_back=false, is_global_index=false, range_key([skyline_int.v3], [skyline_int.v4], [skyline_int.v5], [skyline_int.v6], [skyline_int.v2], [skyline_int.v1], [skyline_int.tenant_id]), range(MIN, MIN,MIN,MIN,MIN,MIN,MIN ; MAX,MAX,MAX,MAX,MAX,MAX,MAX)always true + 3 - output([other.c1]), filter(nil), rowset=16 + access([other.c1]), partitions(p0) + is_index_back=false, is_global_index=false, + range_key([other.c1]), range(MIN ; MAX)always true explain select distinct(v3) from skyline_int join other on v3 = c1 order by v3, v4, v5; Query Plan =========================================================================== @@ -928,7 +925,7 @@ Query Plan ============================================================================ |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ---------------------------------------------------------------------------- -|0 |MERGE UNION DISTINCT| |1 |5 | +|0 |MERGE UNION DISTINCT| |2 |5 | |1 |├─TABLE FULL SCAN |skyline_int(idx_v3_v4_v5_v2)|1 |3 | |2 |└─TABLE FULL SCAN |other |1 |3 | ============================================================================ @@ -971,7 +968,7 @@ Query Plan =============================================================================== |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------------------------------------- -|0 |MERGE UNION DISTINCT| |1 |5 | +|0 |MERGE UNION DISTINCT| |2 |5 | |1 |├─TABLE FULL SCAN |skyline_int(idx_v4_v5_v6_v2_v3)|1 |3 | |2 |└─TABLE FULL SCAN |skyline_int(idx_v2_v3_v4) |1 |3 | =============================================================================== @@ -1350,7 +1347,7 @@ Query Plan =============================================================================== |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------------------------------------- -|0 |MERGE UNION DISTINCT| |1 |7 | +|0 |MERGE UNION DISTINCT| |2 |7 | |1 |├─MERGE JOIN | |1 |5 | |2 |│ ├─TABLE FULL SCAN |skyline_int(idx_v4_v5_v6_v2_v3)|1 |3 | |3 |│ └─TABLE FULL SCAN |other |1 |3 | @@ -1414,7 +1411,7 @@ Query Plan ===================================================== |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ----------------------------------------------------- -|0 |MERGE UNION DISTINCT| |1 |5 | +|0 |MERGE UNION DISTINCT| |2 |5 | |1 |├─TABLE FULL SCAN |other|1 |3 | |2 |└─TABLE FULL SCAN |other|1 |3 | ===================================================== diff --git a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/hash_distinct.result b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/hash_distinct.result index 426375887..b7a5a0823 100644 --- a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/hash_distinct.result +++ b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/hash_distinct.result @@ -5964,7 +5964,8 @@ Outputs & filters: ------------------------------------- 0 - output([T_FUN_COUNT(*)]), filter(nil), rowset=256 group(nil), agg_func([T_FUN_COUNT(*)]) - 1 - output(nil), filter([result.t2c0 = 1489403758], [concat(result.t1c0, '') = 'q6h]zjLt)|[?S*C'], [concat(result.t0c0, '') IS NULL], [result.t0c1 = 398204275]), rowset=256 + 1 - output(nil), filter([result.t0c1 = 398204275], [result.t2c0 = 1489403758], [concat(result.t1c0, '') = 'q6h]zjLt)|[?S*C'], [concat(result.t0c0, '') + IS NULL]), rowset=256 access([result.t0c0], [result.t0c1], [result.t2c0], [result.t1c0]) 2 - output([t0.c0], [t0.c1], [t2.c0], [t1.c0]), filter(nil), rowset=256 limit(2147483647), offset(0) @@ -6020,7 +6021,8 @@ Outputs & filters: ------------------------------------- 0 - output([T_FUN_COUNT(*)]), filter(nil), rowset=256 group(nil), agg_func([T_FUN_COUNT(*)]) - 1 - output(nil), filter([result.t2c0 = 1489403758], [concat(result.t1c0, '') = 'q6h]zjLt)|[?S*C'], [concat(result.t0c0, '') IS NULL], [result.t0c1 = 398204275]), rowset=256 + 1 - output(nil), filter([result.t0c1 = 398204275], [result.t2c0 = 1489403758], [concat(result.t1c0, '') = 'q6h]zjLt)|[?S*C'], [concat(result.t0c0, '') + IS NULL]), rowset=256 access([result.t0c0], [result.t0c1], [result.t2c0], [result.t1c0]) 2 - output([t0.c0], [t0.c1], [t2.c0], [t1.c0]), filter(nil), rowset=256 limit(2147483647), offset(0) diff --git a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/subplan_filter.result b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/subplan_filter.result index f494696f0..746a15a97 100644 --- a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/subplan_filter.result +++ b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/subplan_filter.result @@ -1251,8 +1251,8 @@ Query Plan ============================================================================== |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------------------------------------ -|0 |SORT | |1 |6064 | -|1 |└─SUBPLAN FILTER | |1 |6064 | +|0 |SORT | |2 |6064 | +|1 |└─SUBPLAN FILTER | |2 |6064 | |2 | ├─PX COORDINATOR | |5 |9 | |3 | │ └─EXCHANGE OUT DISTR |:EX10000|5 |8 | |4 | │ └─PX BLOCK ITERATOR | |5 |6 | @@ -1775,7 +1775,7 @@ Optimization Info: table_dop:1 dop_method:Table DOP avaiable_index_name:[t1] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:1 estimation method:[DYNAMIC SAMPLING FULL] t2: @@ -1787,7 +1787,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -1879,7 +1879,7 @@ Optimization Info: table_dop:1 dop_method:Table DOP avaiable_index_name:[t1] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:1 estimation method:[DYNAMIC SAMPLING FULL] t2: @@ -1887,11 +1887,11 @@ Optimization Info: physical_range_rows:7 logical_range_rows:7 index_back_rows:0 - output_rows:2 + output_rows:4 table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -1983,7 +1983,7 @@ Optimization Info: table_dop:1 dop_method:Table DOP avaiable_index_name:[t1] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:1 estimation method:[DYNAMIC SAMPLING FULL] t2: @@ -1995,7 +1995,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -2095,7 +2095,7 @@ Optimization Info: table_dop:1 dop_method:Table DOP avaiable_index_name:[t1] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:1 estimation method:[DYNAMIC SAMPLING FULL] t2: @@ -2107,7 +2107,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] t2: @@ -2119,7 +2119,7 @@ Optimization Info: table_dop:1 dop_method:Table DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -2219,7 +2219,7 @@ Optimization Info: table_dop:1 dop_method:Table DOP avaiable_index_name:[t1] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:1 estimation method:[DYNAMIC SAMPLING FULL] t2: @@ -2231,7 +2231,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] t2: @@ -2243,7 +2243,7 @@ Optimization Info: table_dop:1 dop_method:Table DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -2343,7 +2343,7 @@ Optimization Info: table_dop:1 dop_method:Table DOP avaiable_index_name:[t1] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:1 estimation method:[DYNAMIC SAMPLING FULL] t2: @@ -2355,7 +2355,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] t2: @@ -2367,7 +2367,7 @@ Optimization Info: table_dop:1 dop_method:Table DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -2467,7 +2467,7 @@ Optimization Info: table_dop:1 dop_method:Table DOP avaiable_index_name:[t1] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:1 estimation method:[DYNAMIC SAMPLING FULL] t2: @@ -2479,7 +2479,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] t2: @@ -2491,7 +2491,7 @@ Optimization Info: table_dop:1 dop_method:Table DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -2600,7 +2600,7 @@ Optimization Info: table_dop:1 dop_method:Table DOP avaiable_index_name:[t1] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:1 estimation method:[DYNAMIC SAMPLING FULL] t2: @@ -2612,7 +2612,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] t2: @@ -2624,7 +2624,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] t2: @@ -2636,7 +2636,7 @@ Optimization Info: table_dop:1 dop_method:Table DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -2787,7 +2787,7 @@ Optimization Info: table_dop:1 dop_method:Table DOP avaiable_index_name:[t1] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:1 estimation method:[DYNAMIC SAMPLING FULL] t2: @@ -2799,7 +2799,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] t2: @@ -2811,7 +2811,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] t2: @@ -2823,7 +2823,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] t2: @@ -2835,7 +2835,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] t2: @@ -2847,7 +2847,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] t2: @@ -2859,7 +2859,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] t2: @@ -2871,7 +2871,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -3022,7 +3022,7 @@ Optimization Info: table_dop:1 dop_method:Table DOP avaiable_index_name:[t1] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:1 estimation method:[DYNAMIC SAMPLING FULL] t2: @@ -3034,7 +3034,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] t2: @@ -3046,7 +3046,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] t2: @@ -3058,7 +3058,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] t2: @@ -3070,7 +3070,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] t2: @@ -3082,7 +3082,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] t2: @@ -3094,7 +3094,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] t2: @@ -3106,7 +3106,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: @@ -3257,7 +3257,7 @@ Optimization Info: table_dop:1 dop_method:Table DOP avaiable_index_name:[t1] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:1 estimation method:[DYNAMIC SAMPLING FULL] t2: @@ -3269,7 +3269,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] t2: @@ -3281,7 +3281,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] t2: @@ -3293,7 +3293,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] t2: @@ -3305,7 +3305,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] t2: @@ -3317,7 +3317,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] t2: @@ -3329,7 +3329,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] t2: @@ -3341,7 +3341,7 @@ Optimization Info: table_dop:1 dop_method:DAS DOP avaiable_index_name:[t2] - stats version:0 + stats info:[version=0, is_locked=0, is_expired=0] dynamic sampling level:0 estimation method:[DEFAULT, STORAGE] Plan Type: diff --git a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/table_scan.result b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/table_scan.result index a53d1fa0a..59f56fd18 100644 --- a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/table_scan.result +++ b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/table_scan.result @@ -183,7 +183,7 @@ Query Plan ================================================== |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| -------------------------------------------------- -|0 |TABLE RANGE SCAN|t1(i1)|1 |7 | +|0 |TABLE RANGE SCAN|t1(i1)|1 |8 | ================================================== Outputs & filters: ------------------------------------- @@ -251,7 +251,7 @@ Query Plan =================================================== |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| --------------------------------------------------- -|0 |TABLE RANGE SCAN|t1(idx)|1 |7 | +|0 |TABLE RANGE SCAN|t1(idx)|2 |10 | =================================================== Outputs & filters: ------------------------------------- @@ -275,7 +275,7 @@ Query Plan =================================================== |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| --------------------------------------------------- -|0 |TABLE RANGE SCAN|t1(idx)|1 |19 | +|0 |TABLE RANGE SCAN|t1(idx)|2 |20 | =================================================== Outputs & filters: ------------------------------------- diff --git a/tools/deploy/mysql_test/test_suite/subquery/r/mysql/spf_bug13044302.result b/tools/deploy/mysql_test/test_suite/subquery/r/mysql/spf_bug13044302.result index e01834e79..eedf962cd 100644 --- a/tools/deploy/mysql_test/test_suite/subquery/r/mysql/spf_bug13044302.result +++ b/tools/deploy/mysql_test/test_suite/subquery/r/mysql/spf_bug13044302.result @@ -151,8 +151,8 @@ Query Plan ============================================================================= |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ----------------------------------------------------------------------------- -|0 |PX COORDINATOR | |2 |41 | -|1 |└─EXCHANGE OUT DISTR |:EX10003|2 |40 | +|0 |PX COORDINATOR | |2 |43 | +|1 |└─EXCHANGE OUT DISTR |:EX10003|2 |41 | |2 | └─HASH UNION DISTINCT | |2 |38 | |3 | ├─HASH JOIN | |1 |18 | |4 | │ ├─PX PARTITION ITERATOR | |1 |11 | From 6b78bf44207aceef4b3c5a7d002ec01a24378bef Mon Sep 17 00:00:00 2001 From: chimyue Date: Thu, 15 Aug 2024 13:56:22 +0000 Subject: [PATCH 073/249] fix generate group by plan bug when simplify group exprs exists in rollup --- src/sql/optimizer/ob_select_log_plan.cpp | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/sql/optimizer/ob_select_log_plan.cpp b/src/sql/optimizer/ob_select_log_plan.cpp index 186ac23bc..a8c5cc54a 100644 --- a/src/sql/optimizer/ob_select_log_plan.cpp +++ b/src/sql/optimizer/ob_select_log_plan.cpp @@ -198,17 +198,18 @@ int ObSelectLogPlan::get_groupby_rollup_exprs(const ObLogicalOperator *top, reduce_exprs, group_by_exprs))) { LOG_WARN("failed to simplify group exprs", K(ret)); - } else if (OB_FAIL(ObOptimizerUtil::find_stmt_expr_direction(*stmt, - group_by_exprs, - top->get_output_equal_sets(), - group_directions))) { } else if (OB_FAIL(rollup_exprs.assign(stmt->get_rollup_exprs()))) { LOG_WARN("failed to assign to rollup exprs.", K(ret)); } else if (rollup_exprs.count() > 0) { bool has_rollup_dir = stmt->has_rollup_dir(); + ObSEArray tmp_exprs; if (OB_UNLIKELY(has_rollup_dir && (stmt->get_rollup_dir_size() != rollup_exprs.count()))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to check rollup exprs and directions count.", K (ret)); + } else if (OB_FAIL(ObOptimizerUtil::intersect_exprs(rollup_exprs, reduce_exprs, tmp_exprs))) { + LOG_WARN("failed to get intersect exprs", K (ret)); + } else if (OB_FAIL(append_array_no_dup(group_by_exprs, tmp_exprs))) { + LOG_WARN("failed to append array no dup", K (ret)); } else {/* do nothing. */} for (int64_t i = 0; OB_SUCC(ret) && i < rollup_exprs.count(); i++) { ObOrderDirection dir = has_rollup_dir ? stmt->get_rollup_dirs().at(i) : default_asc_direction(); @@ -218,7 +219,14 @@ int ObSelectLogPlan::get_groupby_rollup_exprs(const ObLogicalOperator *top, } } // do nothing } - if (OB_SUCC(ret)) { + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(ObOptimizerUtil::find_stmt_expr_direction(*stmt, + group_by_exprs, + top->get_output_equal_sets(), + group_directions))) { + LOG_WARN("failed to find stmt expr direction", K (ret)); + } else { LOG_TRACE("succeed to get group by exprs and rollup exprs", K(reduce_exprs), K(group_by_exprs), K(rollup_exprs)); } } From 14be3cb8fb6ae7ad775c28a77b4547aba540b0d6 Mon Sep 17 00:00:00 2001 From: dimstars Date: Thu, 15 Aug 2024 14:18:29 +0000 Subject: [PATCH 074/249] revert old DBA log for script --- src/logservice/palf/palf_env_impl.cpp | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/logservice/palf/palf_env_impl.cpp b/src/logservice/palf/palf_env_impl.cpp index 2721aea94..05f5e2ea8 100644 --- a/src/logservice/palf/palf_env_impl.cpp +++ b/src/logservice/palf/palf_env_impl.cpp @@ -771,7 +771,21 @@ int PalfEnvImpl::try_recycle_blocks() const int64_t log_disk_usage_limit_size = disk_opts_for_stopping_writing.log_disk_usage_limit_size_; const int64_t log_disk_warn_percent = disk_opts_for_stopping_writing.log_disk_utilization_threshold_; const int64_t log_disk_limit_percent = disk_opts_for_stopping_writing.log_disk_utilization_limit_threshold_; - LOG_DBA_ERROR_V2(OB_LOG_DISK_SPACE_ALMOST_FULL, tmp_ret, "msg", "log disk space is almost full", + LOG_DBA_ERROR(OB_LOG_OUTOF_DISK_SPACE, "msg", "log disk space is almost full", "ret", tmp_ret, + "total_size(MB)", log_disk_usage_limit_size/MB, + "used_size(MB)", total_used_size_byte/MB, + "used_percent(%)", (total_used_size_byte*100) / (log_disk_usage_limit_size+1), + "warn_size(MB)", (log_disk_usage_limit_size*log_disk_warn_percent)/100/MB, + "warn_percent(%)", log_disk_warn_percent, + "limit_size(MB)", (log_disk_usage_limit_size*log_disk_limit_percent)/100/MB, + "limit_percent(%)", log_disk_limit_percent, + "total_unrecyclable_size_byte(MB)", total_unrecyclable_size_byte/MB, + "maximum_used_size(MB)", maximum_used_size/MB, + "maximum_log_stream", palf_id, + "oldest_log_stream", oldest_palf_id, + "oldest_scn", oldest_scn, + "in_shrinking", in_shrinking); + LOG_DBA_ERROR_(OB_LOG_DISK_SPACE_ALMOST_FULL, tmp_ret, "log disk space is almost full", ", total_size(MB)=", log_disk_usage_limit_size/MB, ", used_size(MB)=", total_used_size_byte/MB, ", used_percent(%)=", (total_used_size_byte*100) / (log_disk_usage_limit_size+1), @@ -1345,7 +1359,15 @@ void PalfEnvImpl::period_calc_disk_usage() constexpr int64_t INTERVAL = 1*1000*1000; if (palf_reach_time_interval(INTERVAL, disk_not_enough_print_interval_in_loop_thread_)) { int tmp_ret = OB_LOG_OUTOF_DISK_SPACE; - LOG_DBA_ERROR_V2(OB_LOG_DISK_SPACE_ALMOST_FULL, tmp_ret, "msg", "log disk space is almost full", + LOG_DBA_ERROR(OB_LOG_OUTOF_DISK_SPACE, "msg", "log disk space is almost full", "ret", tmp_ret, + "total_size(MB)", log_disk_usage_limit_size/MB, + "used_size(MB)", used_size_byte/MB, + "used_percent(%)", (used_size_byte*100) / (log_disk_usage_limit_size + 1), + "warn_size(MB)", warn_siz/MB, + "warn_percent(%)", log_disk_warn_percent, + "limit_size(MB)", usable_disk_limit_size_to_stop_writing/MB, + "limit_percent(%)", log_disk_limit_percent); + LOG_DBA_ERROR_(OB_LOG_DISK_SPACE_ALMOST_FULL, tmp_ret, "log disk space is almost full", ", total_size(MB)=", log_disk_usage_limit_size/MB, ", used_size(MB)=", used_size_byte/MB, ", used_percent(%)=", (used_size_byte*100) / (log_disk_usage_limit_size + 1), From df78b19c9a5637580c7427818bb34e5f860e776d Mon Sep 17 00:00:00 2001 From: hanr881 <1741282579@qq.com> Date: Thu, 15 Aug 2024 14:24:14 +0000 Subject: [PATCH 075/249] [CP] to issue<55739667>:fix truncate table cause pl cache mismatch after alter table add/drop column --- src/pl/pl_cache/ob_pl_cache.cpp | 101 +++++++++++++++++++------------- 1 file changed, 60 insertions(+), 41 deletions(-) diff --git a/src/pl/pl_cache/ob_pl_cache.cpp b/src/pl/pl_cache/ob_pl_cache.cpp index 50c033798..d2b246726 100644 --- a/src/pl/pl_cache/ob_pl_cache.cpp +++ b/src/pl/pl_cache/ob_pl_cache.cpp @@ -76,46 +76,61 @@ int PCVPlSchemaObj::deep_copy_column_infos(const ObTableSchema *schema) ret = OB_INVALID_ARGUMENT; LOG_WARN("unexpected null argument", K(ret), K(schema), K(inner_alloc_)); } else { - void *obj_buf = nullptr; - ObPLTableColumnInfo *column_info = nullptr; - column_cnt_ = schema->get_column_count(); - column_infos_.set_allocator(inner_alloc_); - if (OB_FAIL(column_infos_.init(column_cnt_))) { - LOG_WARN("failed to init column_infos", K(ret)); - } else { - ObTableSchema::const_column_iterator cs_iter = schema->column_begin(); - ObTableSchema::const_column_iterator cs_iter_end = schema->column_end(); - for (; OB_SUCC(ret) && cs_iter != cs_iter_end; cs_iter++) { - const ObColumnSchemaV2 &column_schema = **cs_iter; - if (nullptr == (obj_buf = inner_alloc_->alloc(sizeof(ObPLTableColumnInfo)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to allocate memory", K(ret)); - } else if (FALSE_IT(column_info = new(obj_buf)ObPLTableColumnInfo(inner_alloc_))) { - // do nothing - } else { - column_info->column_id_ = column_schema.get_column_id(); - column_info->meta_type_ = column_schema.get_meta_type(); - column_info->charset_type_ = column_schema.get_charset_type(); - column_info->accuracy_ = column_schema.get_accuracy(); - OZ (column_info->deep_copy_type_info(column_schema.get_extended_type_info())); - - if (OB_SUCC(ret)) { - char *name_buf = NULL; - const ObString &column_name = column_schema.get_column_name_str(); - if (OB_ISNULL(name_buf = - static_cast(inner_alloc_->alloc(column_name.length() + 1)))) { + ObTableSchema::const_column_iterator cs_iter = schema->column_begin(); + ObTableSchema::const_column_iterator cs_iter_end = schema->column_end(); + int64_t real_column_cnt = 0; + for (; OB_SUCC(ret) && cs_iter != cs_iter_end; cs_iter++) { + const ObColumnSchemaV2 &column_schema = **cs_iter; + if (!column_schema.is_hidden()) { + real_column_cnt++; + } + } + if (OB_SUCC(ret)) { + column_cnt_ = real_column_cnt; + column_infos_.set_allocator(inner_alloc_); + if (OB_FAIL(column_infos_.init(column_cnt_))) { + LOG_WARN("failed to init column_infos", K(ret)); + } else { + void *obj_buf = nullptr; + ObPLTableColumnInfo *column_info = nullptr; + cs_iter = schema->column_begin(); + cs_iter_end = schema->column_end(); + for (; OB_SUCC(ret) && cs_iter != cs_iter_end; cs_iter++) { + const ObColumnSchemaV2 &column_schema = **cs_iter; + if (column_schema.is_hidden()) { + // do nothing + } else { + if (nullptr == (obj_buf = inner_alloc_->alloc(sizeof(ObPLTableColumnInfo)))) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc column name buf", K(ret), K(column_name)); + LOG_WARN("failed to allocate memory", K(ret)); + } else if (FALSE_IT(column_info = new(obj_buf)ObPLTableColumnInfo(inner_alloc_))) { + // do nothing } else { - MEMCPY(name_buf, column_name.ptr(), column_name.length()); - ObString deep_copy_name(column_name.length(), name_buf); - column_info->column_name_ = deep_copy_name; - OZ (column_infos_.push_back(column_info)); + column_info->column_id_ = column_schema.get_column_id(); + column_info->meta_type_ = column_schema.get_meta_type(); + column_info->charset_type_ = column_schema.get_charset_type(); + column_info->accuracy_ = column_schema.get_accuracy(); + OZ (column_info->deep_copy_type_info(column_schema.get_extended_type_info())); + + if (OB_SUCC(ret)) { + char *name_buf = NULL; + const ObString &column_name = column_schema.get_column_name_str(); + if (OB_ISNULL(name_buf = + static_cast(inner_alloc_->alloc(column_name.length() + 1)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc column name buf", K(ret), K(column_name)); + } else { + MEMCPY(name_buf, column_name.ptr(), column_name.length()); + ObString deep_copy_name(column_name.length(), name_buf); + column_info->column_name_ = deep_copy_name; + OZ (column_infos_.push_back(column_info)); + } + } } } } + CK (column_cnt_ == column_infos_.count()); } - CK (column_cnt_ == column_infos_.count()); } } @@ -400,13 +415,17 @@ int ObPLObjectValue::obtain_new_column_infos(share::schema::ObSchemaGetterGuard ObTableSchema::const_column_iterator cs_iter_end = table_schema->column_end(); for (; OB_SUCC(ret) && cs_iter != cs_iter_end; cs_iter++) { const ObColumnSchemaV2 &column_schema = **cs_iter; - column_info.column_id_ = column_schema.get_column_id(); - column_info.meta_type_ = column_schema.get_meta_type(); - column_info.charset_type_ = column_schema.get_charset_type(); - column_info.accuracy_ = column_schema.get_accuracy(); - OZ (column_info.type_info_.assign(column_schema.get_extended_type_info())); - OX (column_info.column_name_ = column_schema.get_column_name_str()); - OZ (column_infos.push_back(column_info)); + if (column_schema.is_hidden()) { + // do nothing + } else { + column_info.column_id_ = column_schema.get_column_id(); + column_info.meta_type_ = column_schema.get_meta_type(); + column_info.charset_type_ = column_schema.get_charset_type(); + column_info.accuracy_ = column_schema.get_accuracy(); + OZ (column_info.type_info_.assign(column_schema.get_extended_type_info())); + OX (column_info.column_name_ = column_schema.get_column_name_str()); + OZ (column_infos.push_back(column_info)); + } } } From 93e793722fc7f961ea9e83ba6c3fd02e992dab6d Mon Sep 17 00:00:00 2001 From: JiahuaChen Date: Fri, 16 Aug 2024 03:27:43 +0000 Subject: [PATCH 076/249] Support empty shell table do mds mvs upgrade --- .../mtlenv/storage/test_ls_tablet_service.cpp | 52 ++++++++++ src/storage/tablet/ob_tablet.cpp | 98 ++++++++++++++++--- src/storage/tablet/ob_tablet.h | 9 ++ 3 files changed, 147 insertions(+), 12 deletions(-) diff --git a/mittest/mtlenv/storage/test_ls_tablet_service.cpp b/mittest/mtlenv/storage/test_ls_tablet_service.cpp index 17ef66884..6bafbe2fc 100644 --- a/mittest/mtlenv/storage/test_ls_tablet_service.cpp +++ b/mittest/mtlenv/storage/test_ls_tablet_service.cpp @@ -1087,6 +1087,58 @@ TEST_F(TestLSTabletService, update_tablet_ddl_commit_scn) ASSERT_EQ(OB_SUCCESS, ret); } + +TEST_F(TestLSTabletService, test_empty_shell_mds_compat) +{ + // create an empty shell tablet + int ret = OB_SUCCESS; + ObTabletID tablet_id(10000009); + share::schema::ObTableSchema schema; + TestSchemaUtils::prepare_data_schema(schema); + ObLSHandle ls_handle; + ObLSService *ls_svr = MTL(ObLSService*); + ret = ls_svr->get_ls(ls_id_, ls_handle, ObLSGetMod::STORAGE_MOD); + ASSERT_EQ(OB_SUCCESS, ret); + ret = TestTabletHelper::create_tablet(ls_handle, tablet_id, schema, allocator_, ObTabletStatus::Status::DELETED); + ASSERT_EQ(OB_SUCCESS, ret); + ret = ls_tablet_service_->update_tablet_to_empty_shell(tablet_id); + ASSERT_EQ(OB_SUCCESS, ret); + ObTabletHandle tablet_handle; + ret = ls_tablet_service_->get_tablet(tablet_id, tablet_handle, 0, ObMDSGetTabletMode::READ_WITHOUT_CHECK); + ASSERT_EQ(OB_SUCCESS, ret); + ObTablet &empty_shell_tablet = *tablet_handle.get_obj(); + ASSERT_TRUE(empty_shell_tablet.is_empty_shell()); + ASSERT_TRUE(nullptr == empty_shell_tablet.mds_data_); + ASSERT_TRUE(ObTabletStatus::Status::DELETED == empty_shell_tablet.tablet_meta_.last_persisted_committed_tablet_status_.tablet_status_); + + ObArenaAllocator compat_allocator; + ObTableHandleV2 empty_mds_sstable_hdl; + ObTablet upgrade_tablet; + ret = upgrade_tablet.init_for_compat(compat_allocator, empty_shell_tablet, empty_mds_sstable_hdl); + // mds data is null + ASSERT_EQ(OB_ERR_UNEXPECTED, ret); + + // mock an old tablet + empty_shell_tablet.version_ = ObTablet::VERSION_V3; + empty_shell_tablet.mds_data_ = OB_NEWx(ObTabletMdsData, &compat_allocator); + ASSERT_TRUE(nullptr != empty_shell_tablet.mds_data_); + empty_shell_tablet.mds_data_->tablet_status_cache_.assign(empty_shell_tablet.tablet_meta_.last_persisted_committed_tablet_status_); + empty_shell_tablet.tablet_meta_.last_persisted_committed_tablet_status_.on_init(); + ASSERT_TRUE(ObTabletStatus::Status::DELETED == empty_shell_tablet.mds_data_->tablet_status_cache_.tablet_status_); + + // compat to new format + upgrade_tablet.assign_pointer_handle(empty_shell_tablet.get_pointer_handle()); + upgrade_tablet.log_handler_ = empty_shell_tablet.log_handler_; + ret = upgrade_tablet.init_for_compat(compat_allocator, empty_shell_tablet, empty_mds_sstable_hdl); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_TRUE(ObTablet::VERSION_V4 == upgrade_tablet.version_); + ASSERT_TRUE(nullptr == upgrade_tablet.mds_data_); + ASSERT_TRUE(ObTabletStatus::Status::DELETED == upgrade_tablet.tablet_meta_.last_persisted_committed_tablet_status_.tablet_status_); + + ret = ls_tablet_service_->do_remove_tablet(ls_id_, tablet_id); + ASSERT_EQ(OB_SUCCESS, ret); +} + } // end storage } // end oceanbase diff --git a/src/storage/tablet/ob_tablet.cpp b/src/storage/tablet/ob_tablet.cpp index 953f65927..165e68034 100644 --- a/src/storage/tablet/ob_tablet.cpp +++ b/src/storage/tablet/ob_tablet.cpp @@ -1352,6 +1352,91 @@ int ObTablet::init_for_compat( common::ObArenaAllocator &allocator, const ObTablet &old_tablet, ObTableHandleV2 &mds_mini_sstable) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(is_inited_)) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", K(ret), K(is_inited_)); + } else if (OB_UNLIKELY(!pointer_hdl_.is_valid()) || OB_ISNULL(log_handler_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tablet pointer handle is invalid", K(ret), K_(pointer_hdl), K_(log_handler)); + } else if (OB_UNLIKELY(old_tablet.is_ls_inner_tablet() && mds_mini_sstable.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ls inner tablet should NOT have valid mds sstable", K(ret), + "ls_id", old_tablet.get_ls_id(), "tablet_id", old_tablet.get_tablet_id(), K(mds_mini_sstable)); + } else if (old_tablet.is_empty_shell()) { + if (OB_FAIL(inner_init_compat_empty_shell(allocator, old_tablet, mds_mini_sstable))) { + LOG_WARN("fail to compat empty shell tablet", K(ret), + "ls_id", old_tablet.get_ls_id(), "tablet_id", old_tablet.get_tablet_id(), K(mds_mini_sstable)); + } + } else { + if (OB_FAIL(inner_init_compat_normal_tablet(allocator, old_tablet, mds_mini_sstable))) { + LOG_WARN("fail to compat normal tablet", K(ret), + "ls_id", old_tablet.get_ls_id(), "tablet_id", old_tablet.get_tablet_id(), K(mds_mini_sstable)); + } + } + + if (OB_UNLIKELY(!is_inited_)) { + reset(); + } + return ret; +} + +int ObTablet::inner_init_compat_empty_shell( + common::ObArenaAllocator &allocator, + const ObTablet &old_tablet, + ObTableHandleV2 &mds_mini_sstable) +{ + int ret = OB_SUCCESS; + + if (OB_UNLIKELY(mds_mini_sstable.is_valid())) { + ret = OB_ERR_UNEXPECTED;; + LOG_WARN("empth shell should not have valid mds sstable", K(ret), K(mds_mini_sstable)); + } else if (OB_UNLIKELY(old_tablet.get_tablet_meta().has_next_tablet_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("old tablet should not have next tablet", K(ret), K(old_tablet.get_tablet_meta())); + } else if (OB_ISNULL(old_tablet.mds_data_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("old tablet has null mds data", K(ret), K(old_tablet.get_tablet_meta())); + } else { + const ObTabletCreateDeleteMdsUserData &user_data = old_tablet.mds_data_->tablet_status_cache_; + if (ObTabletStatus::DELETED != user_data.tablet_status_ + && ObTabletStatus::TRANSFER_OUT_DELETED != user_data.tablet_status_) { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("old tablet status is not deleted", K(ret), K(old_tablet.get_tablet_meta())); + } else if (OB_FAIL(tablet_meta_.assign(old_tablet.tablet_meta_))) { + LOG_WARN("failed to copy tablet meta", K(ret), K(old_tablet.tablet_meta_)); + } else if (OB_FAIL(tablet_meta_.last_persisted_committed_tablet_status_.assign(user_data))) { + LOG_WARN("fail to init last_persisted_committed_tablet_status", K(ret), K(user_data)); + } else if (OB_FAIL(wait_release_memtables_())) { + LOG_ERROR("fail to release memtables", K(ret), K(old_tablet)); + } else if (OB_FAIL(mark_mds_table_switched_to_empty_shell_())) {// to avoid calculate it's rec_scn + LOG_WARN("fail to mark mds table switched to empty shell", K(ret), K(old_tablet)); + } else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, table_store_addr_.ptr_))) { + LOG_WARN("fail to allocate and new rowkey read info", K(ret)); + } else if (OB_FAIL(table_store_addr_.ptr_->init(allocator, *this))) { + LOG_WARN("fail to init table store", K(ret)); + } else if (OB_FAIL(try_update_start_scn())) { + LOG_WARN("failed to update start scn", K(ret), K(table_store_addr_)); + } else { + tablet_meta_.extra_medium_info_.reset(); + table_store_addr_.addr_.set_none_addr(); + storage_schema_addr_.addr_.set_none_addr(); + macro_info_addr_.addr_.set_none_addr(); + tablet_meta_.clog_checkpoint_scn_ = user_data.delete_commit_scn_ > tablet_meta_.clog_checkpoint_scn_ ? + user_data.delete_commit_scn_ : tablet_meta_.clog_checkpoint_scn_; + tablet_meta_.mds_checkpoint_scn_ = user_data.delete_commit_scn_; + is_inited_ = true; + LOG_INFO("succeeded to init empty shell tablet for compat", K(ret), K(old_tablet), KPC(this)); + } + } + return ret; +} + +int ObTablet::inner_init_compat_normal_tablet( + common::ObArenaAllocator &allocator, + const ObTablet &old_tablet, + ObTableHandleV2 &mds_mini_sstable) { TIMEGUARD_INIT(STORAGE, 10_ms); int ret = OB_SUCCESS; @@ -1364,18 +1449,7 @@ int ObTablet::init_for_compat( int64_t ddl_kv_count = 0; ObLinkedMacroBlockItemWriter linked_writer; - if (OB_UNLIKELY(is_inited_)) { - ret = OB_INIT_TWICE; - LOG_WARN("init twice", K(ret), K(is_inited_)); - } else if (OB_UNLIKELY(!pointer_hdl_.is_valid()) - || OB_ISNULL(log_handler_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet pointer handle is invalid", K(ret), K_(pointer_hdl), K_(log_handler)); - } else if (OB_UNLIKELY(old_tablet.is_ls_inner_tablet() && mds_mini_sstable.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls inner tablet should NOT have valid mds sstable", K(ret), - "ls_id", old_tablet.get_ls_id(), "tablet_id", old_tablet.get_tablet_id(), K(mds_mini_sstable)); - } else if (CLICK_FAIL(old_tablet.fetch_table_store(old_table_store_wrapper))) { + if (CLICK_FAIL(old_tablet.fetch_table_store(old_table_store_wrapper))) { LOG_WARN("failed to fetch old table store", K(ret), K(old_tablet)); } else if (CLICK_FAIL(old_table_store_wrapper.get_member(old_table_store))) { LOG_WARN("failed to get old table store", K(ret)); diff --git a/src/storage/tablet/ob_tablet.h b/src/storage/tablet/ob_tablet.h index 74e7c4c62..1e66ba0c1 100644 --- a/src/storage/tablet/ob_tablet.h +++ b/src/storage/tablet/ob_tablet.h @@ -597,6 +597,15 @@ private: int inner_release_memtables(const share::SCN scn); int calc_sstable_occupy_size(int64_t &occupy_size, int64_t &pure_backup_sstable_occupy_size); inline void set_space_usage_(const ObTabletSpaceUsage &space_usage) { tablet_meta_.set_space_usage_(space_usage); } + + int inner_init_compat_normal_tablet( + common::ObArenaAllocator &allocator, + const ObTablet &old_tablet, + ObTableHandleV2 &mds_mini_sstable); + int inner_init_compat_empty_shell( + common::ObArenaAllocator &allocator, + const ObTablet &old_tablet, + ObTableHandleV2 &mds_mini_sstable); private: static bool ignore_ret(const int ret); int inner_check_valid(const bool ignore_ha_status = false) const; From 1744e546fd7a1730e263c29dcd1a060f78c3571f Mon Sep 17 00:00:00 2001 From: ZenoWang Date: Fri, 16 Aug 2024 03:48:49 +0000 Subject: [PATCH 077/249] print undo status list when print tx data --- src/storage/tx/ob_tx_data_define.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/storage/tx/ob_tx_data_define.cpp b/src/storage/tx/ob_tx_data_define.cpp index 0b1c8ff67..7995dc0d2 100644 --- a/src/storage/tx/ob_tx_data_define.cpp +++ b/src/storage/tx/ob_tx_data_define.cpp @@ -756,6 +756,9 @@ DEF_TO_STRING(ObTxData) K_(start_scn), K_(end_scn), K_(op_guard)); + if (op_guard_.is_valid()) { + J_KV("UndoStatusList", op_guard_->get_undo_status_list()); + } J_OBJ_END(); return pos; } From bed514fc54e01e3ccb995c25d91feb5a18459fe5 Mon Sep 17 00:00:00 2001 From: obdev Date: Fri, 16 Aug 2024 04:47:38 +0000 Subject: [PATCH 078/249] [CP] [to #024072000103866409]Fix bug, avoid OB_ALLOCATE_MEMORY_FAILED be overwritten by OB_ERR_UNEXPECTED --- src/pl/ob_pl_user_type.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pl/ob_pl_user_type.cpp b/src/pl/ob_pl_user_type.cpp index 3e7293f96..8bdc871e1 100644 --- a/src/pl/ob_pl_user_type.cpp +++ b/src/pl/ob_pl_user_type.cpp @@ -1382,7 +1382,7 @@ int ObRecordType::newx(common::ObIAllocator &allocator, const ObPLINS *ns, int64 ObObj *member = NULL; int64_t init_size = ObRecordType::get_init_size(get_member_count()); OX (record = reinterpret_cast(allocator.alloc(init_size))); - CK (OB_NOT_NULL(record)); + OV (OB_NOT_NULL(record), OB_ALLOCATE_MEMORY_FAILED) OX (new (record)ObPLRecord(user_type_id_, get_member_count())); OX (ptr = reinterpret_cast(record)); for (int64_t i = 0; OB_SUCC(ret) && i < get_member_count(); ++i) { From db8843beddfa0a0b91f5abe06eed10041cf4663c Mon Sep 17 00:00:00 2001 From: obdev Date: Fri, 16 Aug 2024 05:19:08 +0000 Subject: [PATCH 079/249] [CP] [to #2024072000103866275] fix bugs, modify ALL_PROCEDURES view_definition and ALL_TRIGGER view_definitions --- .../ob_inner_table_schema.25051_25100.cpp | 6 ++-- .../ob_inner_table_schema.25151_25200.cpp | 6 ++-- .../inner_table/ob_inner_table_schema_def.py | 36 +++++++++---------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/share/inner_table/ob_inner_table_schema.25051_25100.cpp b/src/share/inner_table/ob_inner_table_schema.25051_25100.cpp index deb8bc4d8..e2256a670 100644 --- a/src/share/inner_table/ob_inner_table_schema.25051_25100.cpp +++ b/src/share/inner_table/ob_inner_table_schema.25051_25100.cpp @@ -210,7 +210,7 @@ int ObInnerTableSchema::dba_procedures_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT D.DATABASE_NAME AS OWNER, CASE R.ROUTINE_TYPE WHEN 1 THEN R.ROUTINE_NAME WHEN 2 THEN R.ROUTINE_NAME WHEN 3 THEN P.PACKAGE_NAME WHEN 4 THEN T.TYPE_NAME END AS OBJECT_NAME, CASE R.ROUTINE_TYPE WHEN 1 THEN NULL WHEN 2 THEN NULL WHEN 3 THEN R.ROUTINE_NAME WHEN 4 THEN R.ROUTINE_NAME END AS PROCEDURE_NAME, CASE R.ROUTINE_TYPE WHEN 1 THEN R.ROUTINE_ID WHEN 2 THEN R.ROUTINE_ID WHEN 3 THEN P.PACKAGE_ID WHEN 4 THEN T.TYPE_ID END AS OBJECT_ID, CASE R.SUBPROGRAM_ID WHEN 0 THEN 1 ELSE R.SUBPROGRAM_ID END AS SUBPROGRAM_ID, CASE R.OVERLOAD WHEN 0 THEN NULL ELSE R.OVERLOAD END AS OVERLOAD, CASE R.ROUTINE_TYPE WHEN 1 THEN 'PROCEDURE' WHEN 2 THEN 'FUNCTION' WHEN 3 THEN 'PACKAGE' WHEN 4 THEN 'TYPE' END AS OBJECT_TYPE, CAST(DECODE(BITAND(R.FLAG, 16384), 16484, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, CAST(DECODE(BITAND(R.FLAG, 128), 128, 'YES', 'NO') AS VARCHAR2(3)) AS PIPELINED, D1.DATABASE_NAME AS IMPLTYPEOWNER, T1.TYPE_NAME AS IMPLTYPENAME, CAST(DECODE(BITAND(R.FLAG, 8), 8, 'YES', 'NO') AS VARCHAR2(3)) AS PARALLEL, CAST('NO' AS VARCHAR2(3)) AS INTERFACE, CAST(DECODE(BITAND(R.FLAG, 4), 4, 'YES', 'NO') AS VARCHAR2(3)) AS DETERMINISTIC, CAST(DECODE(BITAND(R.FLAG, 16), 16, 'INVOKER', 'DEFINER') AS VARCHAR2(12)) AS AUTHID, R.TENANT_ID AS ORIGIN_CON_ID FROM (SELECT * FROM SYS.ALL_VIRTUAL_ROUTINE_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID())R LEFT JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D ON R.DATABASE_ID = D.DATABASE_ID AND D.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_PACKAGE_REAL_AGENT P ON R.PACKAGE_ID = P.PACKAGE_ID AND P.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T ON R.PACKAGE_ID = T.TYPE_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T1 ON R.TYPE_ID = T1.TYPE_ID AND T1.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D1 ON T1.DATABASE_ID = D1.DATABASE_ID AND T1.TENANT_ID = EFFECTIVE_TENANT_ID() WHERE D.IN_RECYCLEBIN = 0 UNION ALL SELECT CAST('SYS' AS VARCHAR2(30)) AS OWNER, CASE RS.ROUTINE_TYPE WHEN 1 THEN RS.ROUTINE_NAME WHEN 2 THEN RS.ROUTINE_NAME WHEN 3 THEN PS.PACKAGE_NAME WHEN 4 THEN TS.TYPE_NAME END AS OBJECT_NAME, CASE RS.ROUTINE_TYPE WHEN 1 THEN NULL WHEN 2 THEN NULL WHEN 3 THEN RS.ROUTINE_NAME WHEN 4 THEN RS.ROUTINE_NAME END AS PROCEDURE_NAME, CASE RS.ROUTINE_TYPE WHEN 1 THEN RS.ROUTINE_ID WHEN 2 THEN RS.ROUTINE_ID WHEN 3 THEN PS.PACKAGE_ID WHEN 4 THEN TS.TYPE_ID END AS OBJECT_ID, CASE RS.SUBPROGRAM_ID WHEN 0 THEN 1 ELSE RS.SUBPROGRAM_ID END AS SUBPROGRAM_ID, CASE RS.OVERLOAD WHEN 0 THEN NULL ELSE RS.OVERLOAD END AS OVERLOAD, CASE RS.ROUTINE_TYPE WHEN 1 THEN 'PROCEDURE' WHEN 2 THEN 'FUNCTION' WHEN 3 THEN 'PACKAGE' WHEN 4 THEN 'TYPE' END AS OBJECT_TYPE, CAST(DECODE(BITAND(RS.FLAG, 16384), 16484, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, CAST(DECODE(BITAND(RS.FLAG, 128), 128, 'YES', 'NO') AS VARCHAR2(3)) AS PIPELINED, CAST(CASE WHEN TS1.TYPE_NAME IS NULL THEN NULL ELSE 'SYS' END AS VARCHAR2(30)) AS IMPLTYPEOWNER, TS1.TYPE_NAME AS IMPLTYPENAME, CAST(DECODE(BITAND(RS.FLAG, 8), 8, 'YES', 'NO') AS VARCHAR2(3)) AS PARALLEL, CAST('NO' AS VARCHAR2(3)) AS INTERFACE, CAST(DECODE(BITAND(RS.FLAG, 4), 4, 'YES', 'NO') AS VARCHAR2(3)) AS DETERMINISTIC, CAST(DECODE(BITAND(RS.FLAG, 16), 16, 'INVOKER', 'DEFINER') AS VARCHAR2(12)) AS AUTHID, RS.TENANT_ID AS ORIGIN_CON_ID FROM SYS.ALL_VIRTUAL_ROUTINE_SYS_AGENT RS LEFT JOIN SYS.ALL_VIRTUAL_PACKAGE_SYS_AGENT PS ON RS.PACKAGE_ID = PS.PACKAGE_ID LEFT JOIN SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS ON RS.PACKAGE_ID = TS.TYPE_ID LEFT JOIN SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS1 ON RS.TYPE_ID = TS1.TYPE_ID WHERE RS.ROUTINE_TYPE != 1 AND RS.ROUTINE_TYPE != 2 -- sys tenant only have sys package and type. )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT D.DATABASE_NAME AS OWNER, CASE R.ROUTINE_TYPE WHEN 1 THEN R.ROUTINE_NAME WHEN 2 THEN R.ROUTINE_NAME WHEN 3 THEN P.PACKAGE_NAME WHEN 4 THEN T.TYPE_NAME END AS OBJECT_NAME, CASE R.ROUTINE_TYPE WHEN 1 THEN NULL WHEN 2 THEN NULL WHEN 3 THEN R.ROUTINE_NAME WHEN 4 THEN R.ROUTINE_NAME END AS PROCEDURE_NAME, CASE R.ROUTINE_TYPE WHEN 1 THEN R.ROUTINE_ID WHEN 2 THEN R.ROUTINE_ID WHEN 3 THEN P.PACKAGE_ID WHEN 4 THEN T.TYPE_ID END AS OBJECT_ID, CASE R.SUBPROGRAM_ID WHEN 0 THEN 1 ELSE R.SUBPROGRAM_ID END AS SUBPROGRAM_ID, CASE R.OVERLOAD WHEN 0 THEN NULL ELSE R.OVERLOAD END AS OVERLOAD, CASE R.ROUTINE_TYPE WHEN 1 THEN 'PROCEDURE' WHEN 2 THEN 'FUNCTION' WHEN 3 THEN 'PACKAGE' WHEN 4 THEN 'TYPE' END AS OBJECT_TYPE, CAST(DECODE(BITAND(R.FLAG, 16384), 16384, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, CAST(DECODE(BITAND(R.FLAG, 128), 128, 'YES', 'NO') AS VARCHAR2(3)) AS PIPELINED, D1.DATABASE_NAME AS IMPLTYPEOWNER, T1.TYPE_NAME AS IMPLTYPENAME, CAST(DECODE(BITAND(R.FLAG, 8), 8, 'YES', 'NO') AS VARCHAR2(3)) AS PARALLEL, CAST('NO' AS VARCHAR2(3)) AS INTERFACE, CAST(DECODE(BITAND(R.FLAG, 4), 4, 'YES', 'NO') AS VARCHAR2(3)) AS DETERMINISTIC, CAST(DECODE(BITAND(R.FLAG, 16), 16, 'INVOKER', 'DEFINER') AS VARCHAR2(12)) AS AUTHID, R.TENANT_ID AS ORIGIN_CON_ID FROM (SELECT * FROM SYS.ALL_VIRTUAL_ROUTINE_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID())R LEFT JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D ON R.DATABASE_ID = D.DATABASE_ID AND D.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_PACKAGE_REAL_AGENT P ON R.PACKAGE_ID = P.PACKAGE_ID AND P.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T ON R.PACKAGE_ID = T.TYPE_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T1 ON R.TYPE_ID = T1.TYPE_ID AND T1.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D1 ON T1.DATABASE_ID = D1.DATABASE_ID AND T1.TENANT_ID = EFFECTIVE_TENANT_ID() WHERE D.IN_RECYCLEBIN = 0 UNION ALL SELECT CAST('SYS' AS VARCHAR2(30)) AS OWNER, CASE RS.ROUTINE_TYPE WHEN 1 THEN RS.ROUTINE_NAME WHEN 2 THEN RS.ROUTINE_NAME WHEN 3 THEN PS.PACKAGE_NAME WHEN 4 THEN TS.TYPE_NAME END AS OBJECT_NAME, CASE RS.ROUTINE_TYPE WHEN 1 THEN NULL WHEN 2 THEN NULL WHEN 3 THEN RS.ROUTINE_NAME WHEN 4 THEN RS.ROUTINE_NAME END AS PROCEDURE_NAME, CASE RS.ROUTINE_TYPE WHEN 1 THEN RS.ROUTINE_ID WHEN 2 THEN RS.ROUTINE_ID WHEN 3 THEN PS.PACKAGE_ID WHEN 4 THEN TS.TYPE_ID END AS OBJECT_ID, CASE RS.SUBPROGRAM_ID WHEN 0 THEN 1 ELSE RS.SUBPROGRAM_ID END AS SUBPROGRAM_ID, CASE RS.OVERLOAD WHEN 0 THEN NULL ELSE RS.OVERLOAD END AS OVERLOAD, CASE RS.ROUTINE_TYPE WHEN 1 THEN 'PROCEDURE' WHEN 2 THEN 'FUNCTION' WHEN 3 THEN 'PACKAGE' WHEN 4 THEN 'TYPE' END AS OBJECT_TYPE, CAST(DECODE(BITAND(RS.FLAG, 16384), 16484, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, CAST(DECODE(BITAND(RS.FLAG, 128), 128, 'YES', 'NO') AS VARCHAR2(3)) AS PIPELINED, CAST(CASE WHEN TS1.TYPE_NAME IS NULL THEN NULL ELSE 'SYS' END AS VARCHAR2(30)) AS IMPLTYPEOWNER, TS1.TYPE_NAME AS IMPLTYPENAME, CAST(DECODE(BITAND(RS.FLAG, 8), 8, 'YES', 'NO') AS VARCHAR2(3)) AS PARALLEL, CAST('NO' AS VARCHAR2(3)) AS INTERFACE, CAST(DECODE(BITAND(RS.FLAG, 4), 4, 'YES', 'NO') AS VARCHAR2(3)) AS DETERMINISTIC, CAST(DECODE(BITAND(RS.FLAG, 16), 16, 'INVOKER', 'DEFINER') AS VARCHAR2(12)) AS AUTHID, RS.TENANT_ID AS ORIGIN_CON_ID FROM SYS.ALL_VIRTUAL_ROUTINE_SYS_AGENT RS LEFT JOIN SYS.ALL_VIRTUAL_PACKAGE_SYS_AGENT PS ON RS.PACKAGE_ID = PS.PACKAGE_ID LEFT JOIN SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS ON RS.PACKAGE_ID = TS.TYPE_ID LEFT JOIN SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS1 ON RS.TYPE_ID = TS1.TYPE_ID WHERE RS.ROUTINE_TYPE != 1 AND RS.ROUTINE_TYPE != 2 -- sys tenant only have sys package and type. )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -360,7 +360,7 @@ int ObInnerTableSchema::all_procedures_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT D.DATABASE_NAME AS OWNER, CASE R.ROUTINE_TYPE WHEN 1 THEN R.ROUTINE_NAME WHEN 2 THEN R.ROUTINE_NAME WHEN 3 THEN P.PACKAGE_NAME WHEN 4 THEN T.TYPE_NAME END AS OBJECT_NAME, CASE R.ROUTINE_TYPE WHEN 1 THEN NULL WHEN 2 THEN NULL WHEN 3 THEN R.ROUTINE_NAME WHEN 4 THEN R.ROUTINE_NAME END AS PROCEDURE_NAME, CASE R.ROUTINE_TYPE WHEN 1 THEN R.ROUTINE_ID WHEN 2 THEN R.ROUTINE_ID WHEN 3 THEN P.PACKAGE_ID WHEN 4 THEN T.TYPE_ID END AS OBJECT_ID, CASE R.SUBPROGRAM_ID WHEN 0 THEN 1 ELSE R.SUBPROGRAM_ID END AS SUBPROGRAM_ID, CASE R.OVERLOAD WHEN 0 THEN NULL ELSE R.OVERLOAD END AS OVERLOAD, CASE R.ROUTINE_TYPE WHEN 1 THEN 'PROCEDURE' WHEN 2 THEN 'FUNCTION' WHEN 3 THEN 'PACKAGE' WHEN 4 THEN 'TYPE' END AS OBJECT_TYPE, CAST(DECODE(BITAND(R.FLAG, 16384), 16484, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, CAST(DECODE(BITAND(R.FLAG, 128), 128, 'YES', 'NO') AS VARCHAR2(3)) AS PIPELINED, D1.DATABASE_NAME AS IMPLTYPEOWNER, T1.TYPE_NAME AS IMPLTYPENAME, CAST(DECODE(BITAND(R.FLAG, 8), 8, 'YES', 'NO') AS VARCHAR2(3)) AS PARALLEL, CAST('NO' AS VARCHAR2(3)) AS INTERFACE, CAST(DECODE(BITAND(R.FLAG, 4), 4, 'YES', 'NO') AS VARCHAR2(3)) AS DETERMINISTIC, CAST(DECODE(BITAND(R.FLAG, 16), 16, 'INVOKER', 'DEFINER') AS VARCHAR2(12)) AS AUTHID, R.TENANT_ID AS ORIGIN_CON_ID FROM (SELECT * FROM SYS.ALL_VIRTUAL_ROUTINE_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID()) R LEFT JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D ON R.DATABASE_ID = D.DATABASE_ID AND D.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_PACKAGE_REAL_AGENT P ON R.PACKAGE_ID = P.PACKAGE_ID AND P.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T ON R.PACKAGE_ID = T.TYPE_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T1 ON R.TYPE_ID = T1.TYPE_ID AND T1.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D1 ON T1.DATABASE_ID = D1.DATABASE_ID AND T1.TENANT_ID = EFFECTIVE_TENANT_ID() WHERE (R.DATABASE_ID = USERENV('SCHEMAID') OR USER_CAN_ACCESS_OBJ(12, R.ROUTINE_ID, R.DATABASE_ID) = 1) AND D.IN_RECYCLEBIN = 0 UNION ALL SELECT CAST('SYS' AS VARCHAR2(30)) AS OWNER, CASE RS.ROUTINE_TYPE WHEN 1 THEN RS.ROUTINE_NAME WHEN 2 THEN RS.ROUTINE_NAME WHEN 3 THEN PS.PACKAGE_NAME WHEN 4 THEN TS.TYPE_NAME END AS OBJECT_NAME, CASE RS.ROUTINE_TYPE WHEN 1 THEN NULL WHEN 2 THEN NULL WHEN 3 THEN RS.ROUTINE_NAME WHEN 4 THEN RS.ROUTINE_NAME END AS PROCEDURE_NAME, CASE RS.ROUTINE_TYPE WHEN 1 THEN RS.ROUTINE_ID WHEN 2 THEN RS.ROUTINE_ID WHEN 3 THEN PS.PACKAGE_ID WHEN 4 THEN TS.TYPE_ID END AS OBJECT_ID, CASE RS.SUBPROGRAM_ID WHEN 0 THEN 1 ELSE RS.SUBPROGRAM_ID END AS SUBPROGRAM_ID, CASE RS.OVERLOAD WHEN 0 THEN NULL ELSE RS.OVERLOAD END AS OVERLOAD, CASE RS.ROUTINE_TYPE WHEN 1 THEN 'PROCEDURE' WHEN 2 THEN 'FUNCTION' WHEN 3 THEN 'PACKAGE' WHEN 4 THEN 'TYPE' END AS OBJECT_TYPE, CAST(DECODE(BITAND(RS.FLAG, 16384), 16484, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, CAST(DECODE(BITAND(RS.FLAG, 128), 128, 'YES', 'NO') AS VARCHAR2(3)) AS PIPELINED, CAST(CASE WHEN TS1.TYPE_NAME IS NULL THEN NULL ELSE 'SYS' END AS VARCHAR2(30)) AS IMPLTYPEOWNER, TS1.TYPE_NAME AS IMPLTYPENAME, CAST(DECODE(BITAND(RS.FLAG, 8), 8, 'YES', 'NO') AS VARCHAR2(3)) AS PARALLEL, CAST('NO' AS VARCHAR2(3)) AS INTERFACE, CAST(DECODE(BITAND(RS.FLAG, 4), 4, 'YES', 'NO') AS VARCHAR2(3)) AS DETERMINISTIC, CAST(DECODE(BITAND(RS.FLAG, 16), 16, 'INVOKER', 'DEFINER') AS VARCHAR2(12)) AS AUTHID, RS.TENANT_ID AS ORIGIN_CON_ID FROM SYS.ALL_VIRTUAL_ROUTINE_SYS_AGENT RS LEFT JOIN SYS.ALL_VIRTUAL_PACKAGE_SYS_AGENT PS ON RS.PACKAGE_ID = PS.PACKAGE_ID LEFT JOIN SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS ON RS.PACKAGE_ID = TS.TYPE_ID LEFT JOIN SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS1 ON RS.TYPE_ID = TS1.TYPE_ID WHERE RS.ROUTINE_TYPE != 1 AND RS.ROUTINE_TYPE != 2 -- sys tenant only have sys package and type. )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT D.DATABASE_NAME AS OWNER, CASE R.ROUTINE_TYPE WHEN 1 THEN R.ROUTINE_NAME WHEN 2 THEN R.ROUTINE_NAME WHEN 3 THEN P.PACKAGE_NAME WHEN 4 THEN T.TYPE_NAME END AS OBJECT_NAME, CASE R.ROUTINE_TYPE WHEN 1 THEN NULL WHEN 2 THEN NULL WHEN 3 THEN R.ROUTINE_NAME WHEN 4 THEN R.ROUTINE_NAME END AS PROCEDURE_NAME, CASE R.ROUTINE_TYPE WHEN 1 THEN R.ROUTINE_ID WHEN 2 THEN R.ROUTINE_ID WHEN 3 THEN P.PACKAGE_ID WHEN 4 THEN T.TYPE_ID END AS OBJECT_ID, CASE R.SUBPROGRAM_ID WHEN 0 THEN 1 ELSE R.SUBPROGRAM_ID END AS SUBPROGRAM_ID, CASE R.OVERLOAD WHEN 0 THEN NULL ELSE R.OVERLOAD END AS OVERLOAD, CASE R.ROUTINE_TYPE WHEN 1 THEN 'PROCEDURE' WHEN 2 THEN 'FUNCTION' WHEN 3 THEN 'PACKAGE' WHEN 4 THEN 'TYPE' END AS OBJECT_TYPE, CAST(DECODE(BITAND(R.FLAG, 16384), 16384, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, CAST(DECODE(BITAND(R.FLAG, 128), 128, 'YES', 'NO') AS VARCHAR2(3)) AS PIPELINED, D1.DATABASE_NAME AS IMPLTYPEOWNER, T1.TYPE_NAME AS IMPLTYPENAME, CAST(DECODE(BITAND(R.FLAG, 8), 8, 'YES', 'NO') AS VARCHAR2(3)) AS PARALLEL, CAST('NO' AS VARCHAR2(3)) AS INTERFACE, CAST(DECODE(BITAND(R.FLAG, 4), 4, 'YES', 'NO') AS VARCHAR2(3)) AS DETERMINISTIC, CAST(DECODE(BITAND(R.FLAG, 16), 16, 'INVOKER', 'DEFINER') AS VARCHAR2(12)) AS AUTHID, R.TENANT_ID AS ORIGIN_CON_ID FROM (SELECT * FROM SYS.ALL_VIRTUAL_ROUTINE_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID()) R LEFT JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D ON R.DATABASE_ID = D.DATABASE_ID AND D.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_PACKAGE_REAL_AGENT P ON R.PACKAGE_ID = P.PACKAGE_ID AND P.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T ON R.PACKAGE_ID = T.TYPE_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T1 ON R.TYPE_ID = T1.TYPE_ID AND T1.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D1 ON T1.DATABASE_ID = D1.DATABASE_ID AND T1.TENANT_ID = EFFECTIVE_TENANT_ID() WHERE (R.DATABASE_ID = USERENV('SCHEMAID') OR USER_CAN_ACCESS_OBJ(12, R.ROUTINE_ID, R.DATABASE_ID) = 1) AND D.IN_RECYCLEBIN = 0 UNION ALL SELECT CAST('SYS' AS VARCHAR2(30)) AS OWNER, CASE RS.ROUTINE_TYPE WHEN 1 THEN RS.ROUTINE_NAME WHEN 2 THEN RS.ROUTINE_NAME WHEN 3 THEN PS.PACKAGE_NAME WHEN 4 THEN TS.TYPE_NAME END AS OBJECT_NAME, CASE RS.ROUTINE_TYPE WHEN 1 THEN NULL WHEN 2 THEN NULL WHEN 3 THEN RS.ROUTINE_NAME WHEN 4 THEN RS.ROUTINE_NAME END AS PROCEDURE_NAME, CASE RS.ROUTINE_TYPE WHEN 1 THEN RS.ROUTINE_ID WHEN 2 THEN RS.ROUTINE_ID WHEN 3 THEN PS.PACKAGE_ID WHEN 4 THEN TS.TYPE_ID END AS OBJECT_ID, CASE RS.SUBPROGRAM_ID WHEN 0 THEN 1 ELSE RS.SUBPROGRAM_ID END AS SUBPROGRAM_ID, CASE RS.OVERLOAD WHEN 0 THEN NULL ELSE RS.OVERLOAD END AS OVERLOAD, CASE RS.ROUTINE_TYPE WHEN 1 THEN 'PROCEDURE' WHEN 2 THEN 'FUNCTION' WHEN 3 THEN 'PACKAGE' WHEN 4 THEN 'TYPE' END AS OBJECT_TYPE, CAST(DECODE(BITAND(RS.FLAG, 16384), 16484, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, CAST(DECODE(BITAND(RS.FLAG, 128), 128, 'YES', 'NO') AS VARCHAR2(3)) AS PIPELINED, CAST(CASE WHEN TS1.TYPE_NAME IS NULL THEN NULL ELSE 'SYS' END AS VARCHAR2(30)) AS IMPLTYPEOWNER, TS1.TYPE_NAME AS IMPLTYPENAME, CAST(DECODE(BITAND(RS.FLAG, 8), 8, 'YES', 'NO') AS VARCHAR2(3)) AS PARALLEL, CAST('NO' AS VARCHAR2(3)) AS INTERFACE, CAST(DECODE(BITAND(RS.FLAG, 4), 4, 'YES', 'NO') AS VARCHAR2(3)) AS DETERMINISTIC, CAST(DECODE(BITAND(RS.FLAG, 16), 16, 'INVOKER', 'DEFINER') AS VARCHAR2(12)) AS AUTHID, RS.TENANT_ID AS ORIGIN_CON_ID FROM SYS.ALL_VIRTUAL_ROUTINE_SYS_AGENT RS LEFT JOIN SYS.ALL_VIRTUAL_PACKAGE_SYS_AGENT PS ON RS.PACKAGE_ID = PS.PACKAGE_ID LEFT JOIN SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS ON RS.PACKAGE_ID = TS.TYPE_ID LEFT JOIN SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS1 ON RS.TYPE_ID = TS1.TYPE_ID WHERE RS.ROUTINE_TYPE != 1 AND RS.ROUTINE_TYPE != 2 -- sys tenant only have sys package and type. )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -510,7 +510,7 @@ int ObInnerTableSchema::user_procedures_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CASE R.ROUTINE_TYPE WHEN 1 THEN R.ROUTINE_NAME WHEN 2 THEN R.ROUTINE_NAME WHEN 3 THEN P.PACKAGE_NAME WHEN 4 THEN T.TYPE_NAME END AS OBJECT_NAME, CASE R.ROUTINE_TYPE WHEN 1 THEN NULL WHEN 2 THEN NULL WHEN 3 THEN R.ROUTINE_NAME WHEN 4 THEN R.ROUTINE_NAME END AS PROCEDURE_NAME, CASE R.ROUTINE_TYPE WHEN 1 THEN R.ROUTINE_ID WHEN 2 THEN R.ROUTINE_ID WHEN 3 THEN P.PACKAGE_ID WHEN 4 THEN T.TYPE_ID END AS OBJECT_ID, CASE R.SUBPROGRAM_ID WHEN 0 THEN 1 ELSE R.SUBPROGRAM_ID END AS SUBPROGRAM_ID, CASE R.OVERLOAD WHEN 0 THEN NULL ELSE R.OVERLOAD END AS OVERLOAD, CASE R.ROUTINE_TYPE WHEN 1 THEN 'PROCEDURE' WHEN 2 THEN 'FUNCTION' WHEN 3 THEN 'PACKAGE' WHEN 4 THEN 'TYPE' END AS OBJECT_TYPE, CAST(DECODE(BITAND(R.FLAG, 16384), 16484, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, CAST(DECODE(BITAND(R.FLAG, 128), 128, 'YES', 'NO') AS VARCHAR2(3)) AS PIPELINED, D1.DATABASE_NAME AS IMPLTYPEOWNER, T1.TYPE_NAME AS IMPLTYPENAME, CAST(DECODE(BITAND(R.FLAG, 8), 8, 'YES', 'NO') AS VARCHAR2(3)) AS PARALLEL, CAST('NO' AS VARCHAR2(3)) AS INTERFACE, CAST(DECODE(BITAND(R.FLAG, 4), 4, 'YES', 'NO') AS VARCHAR2(3)) AS DETERMINISTIC, CAST(DECODE(BITAND(R.FLAG, 16), 16, 'INVOKER', 'DEFINER') AS VARCHAR2(12)) AS AUTHID, R.TENANT_ID AS ORIGIN_CON_ID FROM (SELECT * FROM SYS.ALL_VIRTUAL_ROUTINE_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID()) R LEFT JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D ON R.DATABASE_ID = D.DATABASE_ID AND D.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_PACKAGE_REAL_AGENT P ON R.PACKAGE_ID = P.PACKAGE_ID AND P.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T ON R.PACKAGE_ID = T.TYPE_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T1 ON R.TYPE_ID = T1.TYPE_ID AND T1.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D1 ON T1.DATABASE_ID = D1.DATABASE_ID AND T1.TENANT_ID = EFFECTIVE_TENANT_ID() WHERE D.IN_RECYCLEBIN = 0 AND R.DATABASE_ID = USERENV('SCHEMAID') )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CASE R.ROUTINE_TYPE WHEN 1 THEN R.ROUTINE_NAME WHEN 2 THEN R.ROUTINE_NAME WHEN 3 THEN P.PACKAGE_NAME WHEN 4 THEN T.TYPE_NAME END AS OBJECT_NAME, CASE R.ROUTINE_TYPE WHEN 1 THEN NULL WHEN 2 THEN NULL WHEN 3 THEN R.ROUTINE_NAME WHEN 4 THEN R.ROUTINE_NAME END AS PROCEDURE_NAME, CASE R.ROUTINE_TYPE WHEN 1 THEN R.ROUTINE_ID WHEN 2 THEN R.ROUTINE_ID WHEN 3 THEN P.PACKAGE_ID WHEN 4 THEN T.TYPE_ID END AS OBJECT_ID, CASE R.SUBPROGRAM_ID WHEN 0 THEN 1 ELSE R.SUBPROGRAM_ID END AS SUBPROGRAM_ID, CASE R.OVERLOAD WHEN 0 THEN NULL ELSE R.OVERLOAD END AS OVERLOAD, CASE R.ROUTINE_TYPE WHEN 1 THEN 'PROCEDURE' WHEN 2 THEN 'FUNCTION' WHEN 3 THEN 'PACKAGE' WHEN 4 THEN 'TYPE' END AS OBJECT_TYPE, CAST(DECODE(BITAND(R.FLAG, 16384), 16384, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, CAST(DECODE(BITAND(R.FLAG, 128), 128, 'YES', 'NO') AS VARCHAR2(3)) AS PIPELINED, D1.DATABASE_NAME AS IMPLTYPEOWNER, T1.TYPE_NAME AS IMPLTYPENAME, CAST(DECODE(BITAND(R.FLAG, 8), 8, 'YES', 'NO') AS VARCHAR2(3)) AS PARALLEL, CAST('NO' AS VARCHAR2(3)) AS INTERFACE, CAST(DECODE(BITAND(R.FLAG, 4), 4, 'YES', 'NO') AS VARCHAR2(3)) AS DETERMINISTIC, CAST(DECODE(BITAND(R.FLAG, 16), 16, 'INVOKER', 'DEFINER') AS VARCHAR2(12)) AS AUTHID, R.TENANT_ID AS ORIGIN_CON_ID FROM (SELECT * FROM SYS.ALL_VIRTUAL_ROUTINE_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID()) R LEFT JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D ON R.DATABASE_ID = D.DATABASE_ID AND D.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_PACKAGE_REAL_AGENT P ON R.PACKAGE_ID = P.PACKAGE_ID AND P.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T ON R.PACKAGE_ID = T.TYPE_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T1 ON R.TYPE_ID = T1.TYPE_ID AND T1.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D1 ON T1.DATABASE_ID = D1.DATABASE_ID AND T1.TENANT_ID = EFFECTIVE_TENANT_ID() WHERE D.IN_RECYCLEBIN = 0 AND R.DATABASE_ID = USERENV('SCHEMAID') )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema.25151_25200.cpp b/src/share/inner_table/ob_inner_table_schema.25151_25200.cpp index d1878c847..7e67bc32c 100644 --- a/src/share/inner_table/ob_inner_table_schema.25151_25200.cpp +++ b/src/share/inner_table/ob_inner_table_schema.25151_25200.cpp @@ -110,7 +110,7 @@ int ObInnerTableSchema::all_triggers_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT DB1.DATABASE_NAME AS OWNER, TRG.TRIGGER_NAME AS TRIGGER_NAME, CAST((case when TRG.TRIGGER_TYPE=1 then DECODE(BITAND(TRG.TIMING_POINTS, 30), 2, 'BEFORE STATEMENT', 4, 'BEFORE EACH ROW', 8, 'AFTER EACH ROW', 16, 'AFTER STATEMENT') when TRG.TRIGGER_TYPE=2 then 'COMPOUND' when TRG.TRIGGER_TYPE=3 then 'INSTEAD OF' END) AS VARCHAR2(16)) AS TRIGGER_TYPE, CAST(DECODE(TRG.TRIGGER_EVENTS, 1, 'INSERT', 2, 'UPDATE', 4, 'DELETE', 1 + 2, 'INSERT OR UPDATE', 1 + 4, 'INSERT OR DELETE', 2 + 4, 'UPDATE OR DELETE', 1 + 2 + 4, 'INSERT OR UPDATE OR DELETE') AS VARCHAR2(246)) AS TRIGGERING_EVENT, DB2.DATABASE_NAME AS TABLE_OWNER, CAST(DECODE(TRG.BASE_OBJECT_TYPE, 5, 'TABLE', 34, 'VIEW') AS VARCHAR2(18)) AS BASE_OBJECT_TYPE, TBL.TABLE_NAME AS TABLE_NAME, CAST(NULL AS VARCHAR2(4000)) AS COLUMN_NAME, CAST(CONCAT('REFERENCING', CONCAT(CONCAT(' NEW AS ', REF_NEW_NAME), CONCAT(' OLD AS ', REF_OLD_NAME))) AS VARCHAR2(422)) AS REFERENCING_NAMES, WHEN_CONDITION AS WHEN_CLAUSE, CAST(decode(BITAND(TRG.trigger_flags, 1), 1, 'ENABLED', 'DISABLED') AS VARCHAR2(8)) AS STATUS, TRIGGER_BODY AS DESCRIPTION, CAST('PL/SQL' AS VARCHAR2(11)) AS ACTION_TYPE, TRIGGER_BODY AS TRIGGER_BODY, CAST('NO' AS VARCHAR2(7)) AS CROSSEDITION, CAST('NO' AS VARCHAR2(3)) AS BEFORE_STATEMENT, CAST('NO' AS VARCHAR2(3)) AS BEFORE_ROW, CAST('NO' AS VARCHAR2(3)) AS AFTER_ROW, CAST('NO' AS VARCHAR2(3)) AS AFTER_STATEMENT, CAST('NO' AS VARCHAR2(3)) AS INSTEAD_OF_ROW, CAST('YES' AS VARCHAR2(3)) AS FIRE_ONCE, CAST('NO' AS VARCHAR2(3)) AS APPLY_SERVER_ONLY FROM SYS.ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT TRG INNER JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB1 ON TRG.DATABASE_ID = DB1.DATABASE_ID AND TRG.TENANT_ID = EFFECTIVE_TENANT_ID() AND DB1.TENANT_ID = EFFECTIVE_TENANT_ID() AND (TRG.DATABASE_ID = USERENV('SCHEMAID') OR USER_CAN_ACCESS_OBJ(1, abs(nvl(TRG.BASE_OBJECT_ID,0)), TRG.DATABASE_ID) = 1) LEFT JOIN SYS.ALL_VIRTUAL_TABLE_REAL_AGENT TBL ON TRG.BASE_OBJECT_ID = TBL.TABLE_ID AND TBL.TENANT_ID = EFFECTIVE_TENANT_ID() AND bitand((TBL.TABLE_MODE / 4096), 15) IN (0,1) INNER JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB2 ON TBL.DATABASE_ID = DB2.DATABASE_ID AND DB2.TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT DB1.DATABASE_NAME AS OWNER, TRG.TRIGGER_NAME AS TRIGGER_NAME, CAST((case when TRG.TRIGGER_TYPE=1 then DECODE(BITAND(TRG.TIMING_POINTS, 30), 2, 'BEFORE STATEMENT', 4, 'BEFORE EACH ROW', 8, 'AFTER EACH ROW', 16, 'AFTER STATEMENT') when TRG.TRIGGER_TYPE=2 then 'COMPOUND' when TRG.TRIGGER_TYPE=3 then 'INSTEAD OF' END) AS VARCHAR2(16)) AS TRIGGER_TYPE, CAST(DECODE(TRG.TRIGGER_EVENTS, 1, 'INSERT', 2, 'UPDATE', 4, 'DELETE', 1 + 2, 'INSERT OR UPDATE', 1 + 4, 'INSERT OR DELETE', 2 + 4, 'UPDATE OR DELETE', 1 + 2 + 4, 'INSERT OR UPDATE OR DELETE') AS VARCHAR2(246)) AS TRIGGERING_EVENT, DB2.DATABASE_NAME AS TABLE_OWNER, CAST(DECODE(TRG.BASE_OBJECT_TYPE, 5, 'TABLE', 34, 'VIEW') AS VARCHAR2(18)) AS BASE_OBJECT_TYPE, TBL.TABLE_NAME AS TABLE_NAME, CAST(NULL AS VARCHAR2(4000)) AS COLUMN_NAME, CAST(CONCAT('REFERENCING', CONCAT(CONCAT(' NEW AS ', REF_NEW_NAME), CONCAT(' OLD AS ', REF_OLD_NAME))) AS VARCHAR2(422)) AS REFERENCING_NAMES, WHEN_CONDITION AS WHEN_CLAUSE, CAST(decode(BITAND(TRG.trigger_flags, 1), 1, 'ENABLED', 'DISABLED') AS VARCHAR2(8)) AS STATUS, TRIGGER_BODY AS DESCRIPTION, CAST('PL/SQL' AS VARCHAR2(11)) AS ACTION_TYPE, TRIGGER_BODY AS TRIGGER_BODY, CAST('NO' AS VARCHAR2(7)) AS CROSSEDITION, CAST(DECODE(BITAND(TRG.TIMING_POINTS, 2), 2, 'YES', 'NO') AS VARCHAR2(3)) AS BEFORE_STATEMENT, CAST(DECODE(BITAND(TRG.TIMING_POINTS, 4), 4, 'YES', 'NO') AS VARCHAR2(3)) AS BEFORE_ROW, CAST(DECODE(BITAND(TRG.TIMING_POINTS, 8), 8, 'YES', 'NO') AS VARCHAR2(3)) AS AFTER_ROW, CAST(DECODE(BITAND(TRG.TIMING_POINTS, 16), 16, 'YES', 'NO') AS VARCHAR2(3)) AS AFTER_STATEMENT, CAST(DECODE(BITAND(TRG.TIMING_POINTS, 32), 32, 'YES', 'NO') AS VARCHAR2(3)) AS INSTEAD_OF_ROW, CAST('YES' AS VARCHAR2(3)) AS FIRE_ONCE, CAST('NO' AS VARCHAR2(3)) AS APPLY_SERVER_ONLY FROM SYS.ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT TRG INNER JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB1 ON TRG.DATABASE_ID = DB1.DATABASE_ID AND TRG.TENANT_ID = EFFECTIVE_TENANT_ID() AND DB1.TENANT_ID = EFFECTIVE_TENANT_ID() AND (TRG.DATABASE_ID = USERENV('SCHEMAID') OR USER_CAN_ACCESS_OBJ(1, abs(nvl(TRG.BASE_OBJECT_ID,0)), TRG.DATABASE_ID) = 1) LEFT JOIN SYS.ALL_VIRTUAL_TABLE_REAL_AGENT TBL ON TRG.BASE_OBJECT_ID = TBL.TABLE_ID AND TBL.TENANT_ID = EFFECTIVE_TENANT_ID() AND bitand((TBL.TABLE_MODE / 4096), 15) IN (0,1) INNER JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB2 ON TBL.DATABASE_ID = DB2.DATABASE_ID AND DB2.TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -160,7 +160,7 @@ int ObInnerTableSchema::dba_triggers_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT DB1.DATABASE_NAME AS OWNER, TRG.TRIGGER_NAME AS TRIGGER_NAME, CAST((case when TRG.TRIGGER_TYPE=1 then DECODE(BITAND(TRG.TIMING_POINTS, 30), 2, 'BEFORE STATEMENT', 4, 'BEFORE EACH ROW', 8, 'AFTER EACH ROW', 16, 'AFTER STATEMENT') when TRG.TRIGGER_TYPE=2 then 'COMPOUND' when TRG.TRIGGER_TYPE=3 then 'INSTEAD OF' END) AS VARCHAR2(16)) AS TRIGGER_TYPE, CAST(DECODE(TRG.TRIGGER_EVENTS, 1, 'INSERT', 2, 'UPDATE', 4, 'DELETE', 1 + 2, 'INSERT OR UPDATE', 1 + 4, 'INSERT OR DELETE', 2 + 4, 'UPDATE OR DELETE', 1 + 2 + 4, 'INSERT OR UPDATE OR DELETE') AS VARCHAR2(246)) AS TRIGGERING_EVENT, DB2.DATABASE_NAME AS TABLE_OWNER, CAST(DECODE(TRG.BASE_OBJECT_TYPE, 5, 'TABLE') AS VARCHAR2(18)) AS BASE_OBJECT_TYPE, TBL.TABLE_NAME AS TABLE_NAME, CAST(NULL AS VARCHAR2(4000)) AS COLUMN_NAME, CAST(CONCAT('REFERENCING', CONCAT(CONCAT(' NEW AS ', REF_NEW_NAME), CONCAT(' OLD AS ', REF_OLD_NAME))) AS VARCHAR2(422)) AS REFERENCING_NAMES, WHEN_CONDITION AS WHEN_CLAUSE, CAST(decode(BITAND(TRG.trigger_flags, 1), 1, 'ENABLED', 'DISABLED') AS VARCHAR2(8)) AS STATUS, TRIGGER_BODY AS DESCRIPTION, CAST('PL/SQL' AS VARCHAR2(11)) AS ACTION_TYPE, TRIGGER_BODY AS TRIGGER_BODY, CAST('NO' AS VARCHAR2(7)) AS CROSSEDITION, CAST('NO' AS VARCHAR2(3)) AS BEFORE_STATEMENT, CAST('NO' AS VARCHAR2(3)) AS BEFORE_ROW, CAST('NO' AS VARCHAR2(3)) AS AFTER_ROW, CAST('NO' AS VARCHAR2(3)) AS AFTER_STATEMENT, CAST('NO' AS VARCHAR2(3)) AS INSTEAD_OF_ROW, CAST('YES' AS VARCHAR2(3)) AS FIRE_ONCE, CAST('NO' AS VARCHAR2(3)) AS APPLY_SERVER_ONLY FROM SYS.ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT TRG INNER JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB1 ON TRG.DATABASE_ID = DB1.DATABASE_ID AND TRG.TENANT_ID = EFFECTIVE_TENANT_ID() AND DB1.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TABLE_REAL_AGENT TBL ON TRG.BASE_OBJECT_ID = TBL.TABLE_ID AND TBL.TENANT_ID = EFFECTIVE_TENANT_ID() AND bitand((TBL.TABLE_MODE / 4096), 15) IN (0,1) INNER JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB2 ON TBL.DATABASE_ID = DB2.DATABASE_ID AND DB2.TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT DB1.DATABASE_NAME AS OWNER, TRG.TRIGGER_NAME AS TRIGGER_NAME, CAST((case when TRG.TRIGGER_TYPE=1 then DECODE(BITAND(TRG.TIMING_POINTS, 30), 2, 'BEFORE STATEMENT', 4, 'BEFORE EACH ROW', 8, 'AFTER EACH ROW', 16, 'AFTER STATEMENT') when TRG.TRIGGER_TYPE=2 then 'COMPOUND' when TRG.TRIGGER_TYPE=3 then 'INSTEAD OF' END) AS VARCHAR2(16)) AS TRIGGER_TYPE, CAST(DECODE(TRG.TRIGGER_EVENTS, 1, 'INSERT', 2, 'UPDATE', 4, 'DELETE', 1 + 2, 'INSERT OR UPDATE', 1 + 4, 'INSERT OR DELETE', 2 + 4, 'UPDATE OR DELETE', 1 + 2 + 4, 'INSERT OR UPDATE OR DELETE') AS VARCHAR2(246)) AS TRIGGERING_EVENT, DB2.DATABASE_NAME AS TABLE_OWNER, CAST(DECODE(TRG.BASE_OBJECT_TYPE, 5, 'TABLE') AS VARCHAR2(18)) AS BASE_OBJECT_TYPE, TBL.TABLE_NAME AS TABLE_NAME, CAST(NULL AS VARCHAR2(4000)) AS COLUMN_NAME, CAST(CONCAT('REFERENCING', CONCAT(CONCAT(' NEW AS ', REF_NEW_NAME), CONCAT(' OLD AS ', REF_OLD_NAME))) AS VARCHAR2(422)) AS REFERENCING_NAMES, WHEN_CONDITION AS WHEN_CLAUSE, CAST(decode(BITAND(TRG.trigger_flags, 1), 1, 'ENABLED', 'DISABLED') AS VARCHAR2(8)) AS STATUS, TRIGGER_BODY AS DESCRIPTION, CAST('PL/SQL' AS VARCHAR2(11)) AS ACTION_TYPE, TRIGGER_BODY AS TRIGGER_BODY, CAST('NO' AS VARCHAR2(7)) AS CROSSEDITION, CAST(DECODE(BITAND(TRG.TIMING_POINTS, 2), 2, 'YES', 'NO') AS VARCHAR2(3)) AS BEFORE_STATEMENT, CAST(DECODE(BITAND(TRG.TIMING_POINTS, 4), 4, 'YES', 'NO') AS VARCHAR2(3)) AS BEFORE_ROW, CAST(DECODE(BITAND(TRG.TIMING_POINTS, 8), 8, 'YES', 'NO') AS VARCHAR2(3)) AS AFTER_ROW, CAST(DECODE(BITAND(TRG.TIMING_POINTS, 16), 16, 'YES', 'NO') AS VARCHAR2(3)) AS AFTER_STATEMENT, CAST(DECODE(BITAND(TRG.TIMING_POINTS, 32), 32, 'YES', 'NO') AS VARCHAR2(3)) AS INSTEAD_OF_ROW, CAST('YES' AS VARCHAR2(3)) AS FIRE_ONCE, CAST('NO' AS VARCHAR2(3)) AS APPLY_SERVER_ONLY FROM SYS.ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT TRG INNER JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB1 ON TRG.DATABASE_ID = DB1.DATABASE_ID AND TRG.TENANT_ID = EFFECTIVE_TENANT_ID() AND DB1.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TABLE_REAL_AGENT TBL ON TRG.BASE_OBJECT_ID = TBL.TABLE_ID AND TBL.TENANT_ID = EFFECTIVE_TENANT_ID() AND bitand((TBL.TABLE_MODE / 4096), 15) IN (0,1) INNER JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB2 ON TBL.DATABASE_ID = DB2.DATABASE_ID AND DB2.TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -210,7 +210,7 @@ int ObInnerTableSchema::user_triggers_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT TRG.TRIGGER_NAME AS TRIGGER_NAME, CAST((case when TRG.TRIGGER_TYPE=1 then DECODE(BITAND(TRG.TIMING_POINTS, 30), 2, 'BEFORE STATEMENT', 4, 'BEFORE EACH ROW', 8, 'AFTER EACH ROW', 16, 'AFTER STATEMENT') when TRG.TRIGGER_TYPE=2 then 'COMPOUND' when TRG.TRIGGER_TYPE=3 then 'INSTEAD OF' END) AS VARCHAR2(16)) AS TRIGGER_TYPE, CAST(DECODE(TRG.TRIGGER_EVENTS, 1, 'INSERT', 2, 'UPDATE', 4, 'DELETE', 1 + 2, 'INSERT OR UPDATE', 1 + 4, 'INSERT OR DELETE', 2 + 4, 'UPDATE OR DELETE', 1 + 2 + 4, 'INSERT OR UPDATE OR DELETE') AS VARCHAR2(246)) AS TRIGGERING_EVENT, DB2.DATABASE_NAME AS TABLE_OWNER, CAST(DECODE(TRG.BASE_OBJECT_TYPE, 5, 'TABLE', 34, 'VIEW') AS VARCHAR2(18)) AS BASE_OBJECT_TYPE, TBL.TABLE_NAME AS TABLE_NAME, CAST(NULL AS VARCHAR2(4000)) AS COLUMN_NAME, CAST(CONCAT('REFERENCING', CONCAT(CONCAT(' NEW AS ', REF_NEW_NAME), CONCAT(' OLD AS ', REF_OLD_NAME))) AS VARCHAR2(422)) AS REFERENCING_NAMES, WHEN_CONDITION AS WHEN_CLAUSE, CAST(decode(BITAND(TRG.trigger_flags, 1), 1, 'ENABLED', 'DISABLED') AS VARCHAR2(8)) AS STATUS, TRIGGER_BODY AS DESCRIPTION, CAST('PL/SQL' AS VARCHAR2(11)) AS ACTION_TYPE, TRIGGER_BODY AS TRIGGER_BODY, CAST('NO' AS VARCHAR2(7)) AS CROSSEDITION, CAST('NO' AS VARCHAR2(3)) AS BEFORE_STATEMENT, CAST('NO' AS VARCHAR2(3)) AS BEFORE_ROW, CAST('NO' AS VARCHAR2(3)) AS AFTER_ROW, CAST('NO' AS VARCHAR2(3)) AS AFTER_STATEMENT, CAST('NO' AS VARCHAR2(3)) AS INSTEAD_OF_ROW, CAST('YES' AS VARCHAR2(3)) AS FIRE_ONCE, CAST('NO' AS VARCHAR2(3)) AS APPLY_SERVER_ONLY FROM (SELECT * FROM SYS.ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID())TRG LEFT JOIN SYS.ALL_VIRTUAL_TABLE_REAL_AGENT TBL ON TRG.BASE_OBJECT_ID = TBL.TABLE_ID AND TBL.TENANT_ID = EFFECTIVE_TENANT_ID() AND bitand((TBL.TABLE_MODE / 4096), 15) IN (0,1) INNER JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB2 ON TBL.DATABASE_ID = DB2.DATABASE_ID AND DB2.TENANT_ID = EFFECTIVE_TENANT_ID() WHERE TRG.DATABASE_ID = USERENV('SCHEMAID') )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT TRG.TRIGGER_NAME AS TRIGGER_NAME, CAST((case when TRG.TRIGGER_TYPE=1 then DECODE(BITAND(TRG.TIMING_POINTS, 30), 2, 'BEFORE STATEMENT', 4, 'BEFORE EACH ROW', 8, 'AFTER EACH ROW', 16, 'AFTER STATEMENT') when TRG.TRIGGER_TYPE=2 then 'COMPOUND' when TRG.TRIGGER_TYPE=3 then 'INSTEAD OF' END) AS VARCHAR2(16)) AS TRIGGER_TYPE, CAST(DECODE(TRG.TRIGGER_EVENTS, 1, 'INSERT', 2, 'UPDATE', 4, 'DELETE', 1 + 2, 'INSERT OR UPDATE', 1 + 4, 'INSERT OR DELETE', 2 + 4, 'UPDATE OR DELETE', 1 + 2 + 4, 'INSERT OR UPDATE OR DELETE') AS VARCHAR2(246)) AS TRIGGERING_EVENT, DB2.DATABASE_NAME AS TABLE_OWNER, CAST(DECODE(TRG.BASE_OBJECT_TYPE, 5, 'TABLE', 34, 'VIEW') AS VARCHAR2(18)) AS BASE_OBJECT_TYPE, TBL.TABLE_NAME AS TABLE_NAME, CAST(NULL AS VARCHAR2(4000)) AS COLUMN_NAME, CAST(CONCAT('REFERENCING', CONCAT(CONCAT(' NEW AS ', REF_NEW_NAME), CONCAT(' OLD AS ', REF_OLD_NAME))) AS VARCHAR2(422)) AS REFERENCING_NAMES, WHEN_CONDITION AS WHEN_CLAUSE, CAST(decode(BITAND(TRG.trigger_flags, 1), 1, 'ENABLED', 'DISABLED') AS VARCHAR2(8)) AS STATUS, TRIGGER_BODY AS DESCRIPTION, CAST('PL/SQL' AS VARCHAR2(11)) AS ACTION_TYPE, TRIGGER_BODY AS TRIGGER_BODY, CAST('NO' AS VARCHAR2(7)) AS CROSSEDITION, CAST(DECODE(BITAND(TRG.TIMING_POINTS, 2), 2, 'YES', 'NO') AS VARCHAR2(3)) AS BEFORE_STATEMENT, CAST(DECODE(BITAND(TRG.TIMING_POINTS, 4), 4, 'YES', 'NO') AS VARCHAR2(3)) AS BEFORE_ROW, CAST(DECODE(BITAND(TRG.TIMING_POINTS, 8), 8, 'YES', 'NO') AS VARCHAR2(3)) AS AFTER_ROW, CAST(DECODE(BITAND(TRG.TIMING_POINTS, 16), 16, 'YES', 'NO') AS VARCHAR2(3)) AS AFTER_STATEMENT, CAST(DECODE(BITAND(TRG.TIMING_POINTS, 32), 32, 'YES', 'NO') AS VARCHAR2(3)) AS INSTEAD_OF_ROW, CAST('YES' AS VARCHAR2(3)) AS FIRE_ONCE, CAST('NO' AS VARCHAR2(3)) AS APPLY_SERVER_ONLY FROM (SELECT * FROM SYS.ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID())TRG LEFT JOIN SYS.ALL_VIRTUAL_TABLE_REAL_AGENT TBL ON TRG.BASE_OBJECT_ID = TBL.TABLE_ID AND TBL.TENANT_ID = EFFECTIVE_TENANT_ID() AND bitand((TBL.TABLE_MODE / 4096), 15) IN (0,1) INNER JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB2 ON TBL.DATABASE_ID = DB2.DATABASE_ID AND DB2.TENANT_ID = EFFECTIVE_TENANT_ID() WHERE TRG.DATABASE_ID = USERENV('SCHEMAID') )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index 2c6ff5023..d8c9925b0 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -42581,7 +42581,7 @@ def_table_schema( WHEN 2 THEN 'FUNCTION' WHEN 3 THEN 'PACKAGE' WHEN 4 THEN 'TYPE' END AS OBJECT_TYPE, - CAST(DECODE(BITAND(R.FLAG, 16384), 16484, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, + CAST(DECODE(BITAND(R.FLAG, 16384), 16384, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, CAST(DECODE(BITAND(R.FLAG, 128), 128, 'YES', 'NO') AS VARCHAR2(3)) AS PIPELINED, D1.DATABASE_NAME AS IMPLTYPEOWNER, T1.TYPE_NAME AS IMPLTYPENAME, @@ -42890,7 +42890,7 @@ def_table_schema( WHEN 2 THEN 'FUNCTION' WHEN 3 THEN 'PACKAGE' WHEN 4 THEN 'TYPE' END AS OBJECT_TYPE, - CAST(DECODE(BITAND(R.FLAG, 16384), 16484, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, + CAST(DECODE(BITAND(R.FLAG, 16384), 16384, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, CAST(DECODE(BITAND(R.FLAG, 128), 128, 'YES', 'NO') AS VARCHAR2(3)) AS PIPELINED, D1.DATABASE_NAME AS IMPLTYPEOWNER, T1.TYPE_NAME AS IMPLTYPENAME, @@ -43211,7 +43211,7 @@ def_table_schema( WHEN 2 THEN 'FUNCTION' WHEN 3 THEN 'PACKAGE' WHEN 4 THEN 'TYPE' END AS OBJECT_TYPE, - CAST(DECODE(BITAND(R.FLAG, 16384), 16484, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, + CAST(DECODE(BITAND(R.FLAG, 16384), 16384, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, CAST(DECODE(BITAND(R.FLAG, 128), 128, 'YES', 'NO') AS VARCHAR2(3)) AS PIPELINED, D1.DATABASE_NAME AS IMPLTYPEOWNER, T1.TYPE_NAME AS IMPLTYPENAME, @@ -49894,11 +49894,11 @@ SELECT DB1.DATABASE_NAME AS OWNER, CAST('PL/SQL' AS VARCHAR2(11)) AS ACTION_TYPE, TRIGGER_BODY AS TRIGGER_BODY, CAST('NO' AS VARCHAR2(7)) AS CROSSEDITION, - CAST('NO' AS VARCHAR2(3)) AS BEFORE_STATEMENT, - CAST('NO' AS VARCHAR2(3)) AS BEFORE_ROW, - CAST('NO' AS VARCHAR2(3)) AS AFTER_ROW, - CAST('NO' AS VARCHAR2(3)) AS AFTER_STATEMENT, - CAST('NO' AS VARCHAR2(3)) AS INSTEAD_OF_ROW, + CAST(DECODE(BITAND(TRG.TIMING_POINTS, 2), 2, 'YES', 'NO') AS VARCHAR2(3)) AS BEFORE_STATEMENT, + CAST(DECODE(BITAND(TRG.TIMING_POINTS, 4), 4, 'YES', 'NO') AS VARCHAR2(3)) AS BEFORE_ROW, + CAST(DECODE(BITAND(TRG.TIMING_POINTS, 8), 8, 'YES', 'NO') AS VARCHAR2(3)) AS AFTER_ROW, + CAST(DECODE(BITAND(TRG.TIMING_POINTS, 16), 16, 'YES', 'NO') AS VARCHAR2(3)) AS AFTER_STATEMENT, + CAST(DECODE(BITAND(TRG.TIMING_POINTS, 32), 32, 'YES', 'NO') AS VARCHAR2(3)) AS INSTEAD_OF_ROW, CAST('YES' AS VARCHAR2(3)) AS FIRE_ONCE, CAST('NO' AS VARCHAR2(3)) AS APPLY_SERVER_ONLY FROM SYS.ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT TRG @@ -49965,11 +49965,11 @@ SELECT DB1.DATABASE_NAME AS OWNER, CAST('PL/SQL' AS VARCHAR2(11)) AS ACTION_TYPE, TRIGGER_BODY AS TRIGGER_BODY, CAST('NO' AS VARCHAR2(7)) AS CROSSEDITION, - CAST('NO' AS VARCHAR2(3)) AS BEFORE_STATEMENT, - CAST('NO' AS VARCHAR2(3)) AS BEFORE_ROW, - CAST('NO' AS VARCHAR2(3)) AS AFTER_ROW, - CAST('NO' AS VARCHAR2(3)) AS AFTER_STATEMENT, - CAST('NO' AS VARCHAR2(3)) AS INSTEAD_OF_ROW, + CAST(DECODE(BITAND(TRG.TIMING_POINTS, 2), 2, 'YES', 'NO') AS VARCHAR2(3)) AS BEFORE_STATEMENT, + CAST(DECODE(BITAND(TRG.TIMING_POINTS, 4), 4, 'YES', 'NO') AS VARCHAR2(3)) AS BEFORE_ROW, + CAST(DECODE(BITAND(TRG.TIMING_POINTS, 8), 8, 'YES', 'NO') AS VARCHAR2(3)) AS AFTER_ROW, + CAST(DECODE(BITAND(TRG.TIMING_POINTS, 16), 16, 'YES', 'NO') AS VARCHAR2(3)) AS AFTER_STATEMENT, + CAST(DECODE(BITAND(TRG.TIMING_POINTS, 32), 32, 'YES', 'NO') AS VARCHAR2(3)) AS INSTEAD_OF_ROW, CAST('YES' AS VARCHAR2(3)) AS FIRE_ONCE, CAST('NO' AS VARCHAR2(3)) AS APPLY_SERVER_ONLY FROM SYS.ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT TRG @@ -50034,11 +50034,11 @@ SELECT TRG.TRIGGER_NAME AS TRIGGER_NAME, CAST('PL/SQL' AS VARCHAR2(11)) AS ACTION_TYPE, TRIGGER_BODY AS TRIGGER_BODY, CAST('NO' AS VARCHAR2(7)) AS CROSSEDITION, - CAST('NO' AS VARCHAR2(3)) AS BEFORE_STATEMENT, - CAST('NO' AS VARCHAR2(3)) AS BEFORE_ROW, - CAST('NO' AS VARCHAR2(3)) AS AFTER_ROW, - CAST('NO' AS VARCHAR2(3)) AS AFTER_STATEMENT, - CAST('NO' AS VARCHAR2(3)) AS INSTEAD_OF_ROW, + CAST(DECODE(BITAND(TRG.TIMING_POINTS, 2), 2, 'YES', 'NO') AS VARCHAR2(3)) AS BEFORE_STATEMENT, + CAST(DECODE(BITAND(TRG.TIMING_POINTS, 4), 4, 'YES', 'NO') AS VARCHAR2(3)) AS BEFORE_ROW, + CAST(DECODE(BITAND(TRG.TIMING_POINTS, 8), 8, 'YES', 'NO') AS VARCHAR2(3)) AS AFTER_ROW, + CAST(DECODE(BITAND(TRG.TIMING_POINTS, 16), 16, 'YES', 'NO') AS VARCHAR2(3)) AS AFTER_STATEMENT, + CAST(DECODE(BITAND(TRG.TIMING_POINTS, 32), 32, 'YES', 'NO') AS VARCHAR2(3)) AS INSTEAD_OF_ROW, CAST('YES' AS VARCHAR2(3)) AS FIRE_ONCE, CAST('NO' AS VARCHAR2(3)) AS APPLY_SERVER_ONLY FROM (SELECT * FROM SYS.ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT From 290ec920af64f838e02be80ddfe8d2cf4d72d1ea Mon Sep 17 00:00:00 2001 From: obdev Date: Fri, 16 Aug 2024 05:54:46 +0000 Subject: [PATCH 080/249] fix unhandled error for null checking of user info --- src/sql/privilege_check/ob_privilege_check.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sql/privilege_check/ob_privilege_check.cpp b/src/sql/privilege_check/ob_privilege_check.cpp index 3158640fa..c0596a096 100644 --- a/src/sql/privilege_check/ob_privilege_check.cpp +++ b/src/sql/privilege_check/ob_privilege_check.cpp @@ -2021,7 +2021,7 @@ int get_revoke_stmt_need_privs( const ObUserInfo *user_info = NULL; OZ(schema_guard.get_user_info(session_priv.tenant_id_, stmt->get_users().at(i), user_info)); CK (user_info != NULL); - need_add = (0 != (user_info->get_priv_set() & OB_PRIV_SUPER)); + OX(need_add = (0 != (user_info->get_priv_set() & OB_PRIV_SUPER))); } if (OB_FAIL(ret)) { } else if (need_add) { //mysql8.0 if exists dynamic privs, then need SYSTEM_USER dynamic privilge to revoke all, now use SUPER to do so. From f1845ce8e94044d174dc6995c27991ffb61f50ad Mon Sep 17 00:00:00 2001 From: obdev Date: Fri, 16 Aug 2024 06:47:32 +0000 Subject: [PATCH 081/249] bugfix:ObSharedNothingTmpFile::get_physical_page_id_in_wbp reports error when flushed_page_virtual_id_ is INVALID --- src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp b/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp index b0a050810..35145bc9a 100644 --- a/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp +++ b/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp @@ -1992,7 +1992,7 @@ int ObSharedNothingTmpFile::update_meta_after_flush(const int64_t info_idx, cons K(end_pos), K(inner_flush_ctx_), KPC(this)); } - if (OB_FAIL(inner_flush_ctx_.update_finished_continuous_flush_info_num(is_meta, end_pos))) { + if (FAILEDx(inner_flush_ctx_.update_finished_continuous_flush_info_num(is_meta, end_pos))) { LOG_WARN("fail to update finished continuous flush info num", KR(ret), K(start_pos), K(end_pos), KPC(this)); } else if (inner_flush_ctx_.is_all_finished()) { if (OB_ISNULL(data_flush_node_.get_next()) && OB_FAIL(reinsert_flush_node_(false /*is_meta*/))) { @@ -2680,7 +2680,8 @@ int ObSharedNothingTmpFile::get_physical_page_id_in_wbp(const int64_t virtual_pa LOG_WARN("the page doesn't exist in wbp", KR(ret), K(virtual_page_id), K(end_page_virtual_id), K(begin_page_virtual_id_), KPC(this)); } else if (virtual_page_id == begin_page_virtual_id_) { page_id = begin_page_id_; - } else if (virtual_page_id < flushed_page_virtual_id_) { + } else if (ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == flushed_page_virtual_id_ || + virtual_page_id < flushed_page_virtual_id_) { if (OB_FAIL(wbp_->get_page_id_by_virtual_id(fd_, virtual_page_id, begin_page_id_, page_id))) { LOG_WARN("fail to get page id by virtual id", KR(ret), K(virtual_page_id), K(begin_page_id_), KPC(this)); } From 315a2dd5ecd13521ff6ac6ee42e130e7eb15ef35 Mon Sep 17 00:00:00 2001 From: LoLolobster <949673574@qq.com> Date: Fri, 16 Aug 2024 07:17:56 +0000 Subject: [PATCH 082/249] fix log archive checkpoint issuing LIST and TAGGING requests too frequently when delete mode is tagging --- .../oblib/src/lib/restore/ob_storage_info.cpp | 13 ++++- deps/oblib/src/lib/restore/ob_storage_info.h | 4 +- .../backup/ob_archive_scheduler_service.cpp | 39 ++++++++++--- .../backup/ob_archive_scheduler_service.h | 8 +-- .../backup/ob_tenant_archive_scheduler.cpp | 9 +-- .../backup/ob_archive_checkpoint_mgr.cpp | 57 ++++++++++++++----- src/share/backup/ob_archive_checkpoint_mgr.h | 11 ++-- src/share/backup/ob_archive_store.cpp | 25 +++++++- src/share/backup/ob_archive_store.h | 3 +- src/storage/backup/ob_backup_task.cpp | 3 +- .../backup/test_archive_checkpoint_mgr.cpp | 8 +-- 11 files changed, 132 insertions(+), 48 deletions(-) diff --git a/deps/oblib/src/lib/restore/ob_storage_info.cpp b/deps/oblib/src/lib/restore/ob_storage_info.cpp index 591c55412..412e75391 100644 --- a/deps/oblib/src/lib/restore/ob_storage_info.cpp +++ b/deps/oblib/src/lib/restore/ob_storage_info.cpp @@ -35,7 +35,8 @@ const char *get_storage_checksum_type_str(const ObStorageChecksumType &type) //***********************ObObjectStorageInfo*************************** ObObjectStorageInfo::ObObjectStorageInfo() - : device_type_(ObStorageType::OB_STORAGE_MAX_TYPE), + : delete_mode_(ObIStorageUtil::DELETE), + device_type_(ObStorageType::OB_STORAGE_MAX_TYPE), checksum_type_(ObStorageChecksumType::OB_MD5_ALGO) { endpoint_[0] = '\0'; @@ -51,6 +52,7 @@ ObObjectStorageInfo::~ObObjectStorageInfo() void ObObjectStorageInfo::reset() { + delete_mode_ = ObIStorageUtil::DELETE; device_type_ = ObStorageType::OB_STORAGE_MAX_TYPE; checksum_type_ = ObStorageChecksumType::OB_MD5_ALGO; endpoint_[0] = '\0'; @@ -230,8 +232,8 @@ int ObObjectStorageInfo::parse_storage_info_(const char *storage_info, bool &has } return ret; } - -int ObObjectStorageInfo::check_delete_mode_(const char *delete_mode) const +//TODO(shifagndan): define delete mode as enum +int ObObjectStorageInfo::check_delete_mode_(const char *delete_mode) { int ret = OB_SUCCESS; if (OB_ISNULL(delete_mode)) { @@ -240,6 +242,10 @@ int ObObjectStorageInfo::check_delete_mode_(const char *delete_mode) const } else if (0 != strcmp(delete_mode, "delete") && 0 != strcmp(delete_mode, "tagging")) { ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "delete mode is invalid", K(ret), K(delete_mode)); + } else if (0 == strcmp(delete_mode, "delete")) { + delete_mode_ = ObIStorageUtil::DELETE; + } else { + delete_mode_ = ObIStorageUtil::TAGGING; } return ret; } @@ -324,6 +330,7 @@ int ObObjectStorageInfo::set_storage_info_field_(const char *info, char *field, int ObObjectStorageInfo::assign(const ObObjectStorageInfo &storage_info) { int ret = OB_SUCCESS; + delete_mode_ = storage_info.delete_mode_; device_type_ = storage_info.device_type_; checksum_type_ = storage_info.checksum_type_; MEMCPY(endpoint_, storage_info.endpoint_, sizeof(endpoint_)); diff --git a/deps/oblib/src/lib/restore/ob_storage_info.h b/deps/oblib/src/lib/restore/ob_storage_info.h index 5a851b33a..8daee6299 100644 --- a/deps/oblib/src/lib/restore/ob_storage_info.h +++ b/deps/oblib/src/lib/restore/ob_storage_info.h @@ -76,6 +76,7 @@ public: ObStorageChecksumType get_checksum_type() const; const char *get_checksum_type_str() const; virtual int get_storage_info_str(char *storage_info, const int64_t info_len) const; + int get_delete_mode() const { return delete_mode_; } virtual bool is_valid() const; virtual void reset(); @@ -88,11 +89,12 @@ public: protected: virtual int get_access_key_(char *key_buf, const int64_t key_buf_len) const; virtual int parse_storage_info_(const char *storage_info, bool &has_appid); - int check_delete_mode_(const char *delete_mode) const; + int check_delete_mode_(const char *delete_mode); int set_checksum_type_(const char *checksum_type_str); int set_storage_info_field_(const char *info, char *field, const int64_t length); public: + int delete_mode_; // TODO: Rename device_type_ to storage_protocol_type_ for better clarity // Prefix in the storage_info string, such as 's3://', indicates the protocol used to access the target // Currently, both OBS and GCS are accessed via the s3 protocol, diff --git a/src/rootserver/backup/ob_archive_scheduler_service.cpp b/src/rootserver/backup/ob_archive_scheduler_service.cpp index d71094012..9d0aa38ce 100644 --- a/src/rootserver/backup/ob_archive_scheduler_service.cpp +++ b/src/rootserver/backup/ob_archive_scheduler_service.cpp @@ -71,7 +71,7 @@ void ObArchiveSchedulerService::run2() { int tmp_ret = OB_SUCCESS; int64_t round = 0; - share::ObLogArchiveStatus::STATUS last_log_archive_status = ObLogArchiveStatus::INVALID; + ObArchiveRoundState round_state; FLOG_INFO("ObArchiveSchedulerService run start"); if (IS_NOT_INIT) { @@ -80,18 +80,19 @@ void ObArchiveSchedulerService::run2() } else { while (true) { ++round; + round_state.set_invalid(); ObCurTraceId::init(GCONF.self_addr_); FLOG_INFO("start do ObArchiveSchedulerService round", K(round)); if (has_set_stop()) { tmp_ret = OB_IN_STOP_STATE; LOG_WARN_RET(tmp_ret, "exit for stop state", K(tmp_ret)); break; - } else if (OB_SUCCESS != (tmp_ret = process_())) { + } else if (OB_SUCCESS != (tmp_ret = process_(round_state))) { LOG_WARN_RET(tmp_ret, "failed to do process", K(tmp_ret)); } - int64_t checkpoint_interval = 1 * 1000 * 1000L; - set_checkpoint_interval_(checkpoint_interval); + int64_t checkpoint_interval = 120 * 1000 * 1000L; + set_checkpoint_interval_(checkpoint_interval, round_state); idle(); } } @@ -259,7 +260,7 @@ int ObArchiveSchedulerService::stop_tenant_archive_(const uint64_t tenant_id) return ret; } -int ObArchiveSchedulerService::process_() +int ObArchiveSchedulerService::process_(ObArchiveRoundState &round_state) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -271,6 +272,7 @@ int ObArchiveSchedulerService::process_() const bool lock = false; bool no_dest = false; bool no_round = false; + round_state.set_invalid(); ObArchiveHandler tenant_scheduler; const uint64_t tenant_id = tenant_id_; @@ -307,6 +309,8 @@ int ObArchiveSchedulerService::process_() no_round = true; ret = OB_SUCCESS; } + } else { + round_state = round.state_; } if (OB_FAIL(ret)) { @@ -315,17 +319,23 @@ int ObArchiveSchedulerService::process_() if (no_round || round.state_.is_stop() || round.state_.is_stopping()) { } else if (OB_FAIL(tenant_scheduler.disable_archive(dest_no))) { LOG_WARN("failed to disable archive", K(ret), K(tenant_id), K(dest_no), K(dest_state)); + } else { + round_state.set_stopping(); } } else if (dest_state.is_defer()) { if (no_round || round.state_.is_stop() || round.state_.is_suspend() || round.state_.is_suspending()) { } else if (OB_FAIL(tenant_scheduler.defer_archive(dest_no))) { LOG_WARN("failed to defer archive", K(ret), K(tenant_id), K(dest_no), K(dest_state)); + } else { + round_state.set_suspending(); } } else { // dest is enable if (no_round || round.state_.is_suspend() || round.state_.is_stop()) { if (OB_FAIL(tenant_scheduler.enable_archive(dest_no))) { LOG_WARN("failed to enable archive", K(ret), K(tenant_id), K(dest_no), K(dest_state)); + } else { + round_state.set_beginning(); } } } @@ -371,22 +381,33 @@ int ObArchiveSchedulerService::get_all_tenant_ids_(common::ObIArray &t return ret; } -void ObArchiveSchedulerService::set_checkpoint_interval_(const int64_t interval_us) +void ObArchiveSchedulerService::set_checkpoint_interval_(const int64_t interval_us, const share::ObArchiveRoundState &round_state) { - const int64_t max_idle_us = interval_us / 2 - RESERVED_FETCH_US; + int64_t max_idle_us = interval_us / 2 - RESERVED_FETCH_US; int64_t idle_time_us = 0; + omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id_)); + const int64_t lag_target = tenant_config.is_valid() ? tenant_config->archive_lag_target : 0L; + if (interval_us <= 0) { idle_time_us = MAX_IDLE_INTERVAL_US; } else { if (max_idle_us <= MIN_IDLE_INTERVAL_US) { - idle_time_us = MIN_IDLE_INTERVAL_US; + idle_time_us = max(MIN_IDLE_INTERVAL_US, lag_target / 2); } else if (max_idle_us > MAX_IDLE_INTERVAL_US) { - idle_time_us = MAX_IDLE_INTERVAL_US; + idle_time_us = min(MAX_IDLE_INTERVAL_US, max(lag_target / 2, MIN_IDLE_INTERVAL_US)); } else { idle_time_us = max_idle_us; } } + if (idle_time_us > FAST_IDLE_INTERVAL_US + && (round_state.is_prepare() + || round_state.is_beginning() + || round_state.is_suspending() + || round_state.is_stopping())) { + idle_time_us = FAST_IDLE_INTERVAL_US; + } + if (idle_time_us != get_idle_time()) { FLOG_INFO("change idle_time_us", "idle_time_ts_", get_idle_time(), K(idle_time_us)); set_idle_time(idle_time_us); diff --git a/src/rootserver/backup/ob_archive_scheduler_service.h b/src/rootserver/backup/ob_archive_scheduler_service.h index 7d55883f4..9e01bc23b 100644 --- a/src/rootserver/backup/ob_archive_scheduler_service.h +++ b/src/rootserver/backup/ob_archive_scheduler_service.h @@ -17,6 +17,7 @@ #include "lib/mysqlclient/ob_isql_client.h" #include "lib/container/ob_iarray.h" #include "share/backup/ob_backup_struct.h" +#include "share/backup/ob_archive_struct.h" namespace oceanbase { @@ -41,8 +42,7 @@ public: const int64_t RESERVED_FETCH_US = 10 * 1000 * 1000; // 10s, used for fetch observer log archive status const int64_t MIN_IDLE_INTERVAL_US = 2 * 1000 * 1000; // 2s const int64_t FAST_IDLE_INTERVAL_US = 10 * 1000 * 1000; // 10s, used during BEGINNING or STOPPING - //const int64_t MAX_IDLE_INTERVAL_US = 60 * 1000 * 1000; // 60s - const int64_t MAX_IDLE_INTERVAL_US = 10 * 1000 * 1000; // 60s + const int64_t MAX_IDLE_INTERVAL_US = 60 * 1000 * 1000; // 60s DEFINE_MTL_FUNC(ObArchiveSchedulerService); int init(); void run2() override; @@ -64,7 +64,7 @@ public: int stop_archive(const uint64_t tenant_id, const common::ObIArray &archive_tenant_ids); private: - int process_(); + int process_(share::ObArchiveRoundState &round_state); int start_tenant_archive_(const uint64_t tenant_id); // Return the first error that failed to start archive if force_start is true. Otherwise, // ignore all error. @@ -75,7 +75,7 @@ private: int stop_tenant_archive_(const uint64_t tenant_id); int get_all_tenant_ids_(common::ObIArray &tenantid_array); - void set_checkpoint_interval_(const int64_t interval_us); + void set_checkpoint_interval_(const int64_t interval_us, const share::ObArchiveRoundState &round_state); int open_tenant_archive_mode_(const common::ObIArray &tenant_ids_array); int open_tenant_archive_mode_(const uint64_t tenant_id); int close_tenant_archive_mode_(const common::ObIArray &tenant_ids_array); diff --git a/src/rootserver/backup/ob_tenant_archive_scheduler.cpp b/src/rootserver/backup/ob_tenant_archive_scheduler.cpp index 2a58b0bf8..138d204f9 100644 --- a/src/rootserver/backup/ob_tenant_archive_scheduler.cpp +++ b/src/rootserver/backup/ob_tenant_archive_scheduler.cpp @@ -129,7 +129,7 @@ static int record_piece_extend_info( return ret; } -static int record_piece_checkpoint(const ObTenantArchivePieceAttr &piece_info, const ObArchiveStore &store) +static int record_piece_checkpoint(const ObTenantArchiveRoundAttr &old_round_info, const ObTenantArchivePieceAttr &piece_info, const ObArchiveStore &store) { int ret = OB_SUCCESS; if (!(piece_info.status_.is_active() @@ -147,7 +147,7 @@ static int record_piece_checkpoint(const ObTenantArchivePieceAttr &piece_info, c checkpoint_desc.checkpoint_scn_ = piece_info.checkpoint_scn_; checkpoint_desc.max_scn_ = piece_info.max_scn_; checkpoint_desc.end_scn_ = piece_info.end_scn_; - if (OB_FAIL(store.write_piece_checkpoint(piece_info.key_.dest_id_, piece_info.key_.round_id_, piece_info.key_.piece_id_, 0, checkpoint_desc))) { + if (OB_FAIL(store.write_piece_checkpoint(piece_info.key_.dest_id_, piece_info.key_.round_id_, piece_info.key_.piece_id_, 0, old_round_info.checkpoint_scn_, checkpoint_desc))) { LOG_WARN("failed to write piece checkpoint info file", K(ret), K(piece_info), K(checkpoint_desc)); } } @@ -233,6 +233,7 @@ static int record_piece_inner_placeholder(const ObTenantArchivePieceAttr &piece_ static int record_single_piece_info(const ObTenantArchivePieceAttr &piece_info, const ObArchiveStore &store) { int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; bool is_exist = false; ObSinglePieceDesc single_piece_desc; if (!piece_info.status_.is_frozen()) { @@ -244,7 +245,7 @@ static int record_single_piece_info(const ObTenantArchivePieceAttr &piece_info, } else if (OB_FAIL(store.write_single_piece(piece_info.key_.dest_id_, piece_info.key_.round_id_, piece_info.key_.piece_id_, single_piece_desc))) { LOG_WARN("failed to write single piece info file", K(ret), K(piece_info), K(single_piece_desc)); } - + return ret; } @@ -340,7 +341,7 @@ static int piece_generated_cb( LOG_WARN("failed to record piece start", K(ret), K(old_round_info), K(piece)); } else if (OB_FAIL(record_piece_extend_info(*sql_proxy, old_round_info, result, piece.piece_info_, store))) { LOG_WARN("failed to record piece extend info", K(ret)); - } else if (OB_FAIL(record_piece_checkpoint(piece_info, store))) { + } else if (OB_FAIL(record_piece_checkpoint(old_round_info, piece_info, store))) { LOG_WARN("failed to record piece checkpoint", K(ret), K(old_round_info), K(piece)); } else if (OB_FAIL(record_piece_info(piece, store))) { LOG_WARN("failed to record piece info", K(ret), K(old_round_info), K(piece)); diff --git a/src/share/backup/ob_archive_checkpoint_mgr.cpp b/src/share/backup/ob_archive_checkpoint_mgr.cpp index 0b243371d..1f2ea17b0 100644 --- a/src/share/backup/ob_archive_checkpoint_mgr.cpp +++ b/src/share/backup/ob_archive_checkpoint_mgr.cpp @@ -66,6 +66,7 @@ int ObDelHisCheckpointFileOp::func(const dirent *entry) ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "invalid list entry, d_name is null", K(ret)); } else { + handled_file_num_++; uint64_t checkpoint_scn = 0; ObBackupPath full_path = path_; ObBackupIoAdapter io_util; @@ -135,13 +136,16 @@ int ObArchiveCheckpointMgr::check_is_tagging_(const ObBackupStorageInfo *storage { int ret = OB_SUCCESS; is_tagging = false; - if (OB_STORAGE_OSS == storage_info_ -> device_type_) { - //TODO(zhixing.yh) Adapt the analytic interface in storage_info + if (OB_ISNULL(storage_info)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("storage info is nullptr", K(ret)); + } else if (ObIStorageUtil::TAGGING == storage_info->get_delete_mode()) { + is_tagging = true; } return ret; } -int ObArchiveCheckpointMgr::write(const uint64_t checkpoint_scn) const +int ObArchiveCheckpointMgr::write(const uint64_t old_checkpoint_scn, const uint64_t checkpoint_scn) const { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -149,24 +153,23 @@ int ObArchiveCheckpointMgr::write(const uint64_t checkpoint_scn) const ObBackupPath full_path = path_; //checkpoint scn file path ObBackupPath dir_path = path_; //checkpoint dir file path ObBackupIoAdapter io_util; - bool is_tagging = false; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("Archive checkpoint mgr not init", K(ret)); } else if (checkpoint_scn <= 0) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument!", K(ret), K(checkpoint_scn)); - } else if (OB_FAIL(check_is_tagging_(storage_info_, is_tagging))) { - LOG_WARN("failed to judge delete mode", K(ret)); + } else if (old_checkpoint_scn >= checkpoint_scn) { //do nothing } else if (OB_FAIL(full_path.join_checkpoint_info_file(file_name_, checkpoint_scn, type_))) { LOG_WARN("failed to get piece checkpoint file path", K(ret), K(checkpoint_scn), KP(file_name_), K(full_path), K(type_)); } else if (OB_FAIL(write_checkpoint_file_(full_path))) { LOG_WARN("failed to write checkpoint file", K(ret), K(full_path)); - } - //if the delete mode is not 'tagging', need to list files for deleting smaller checkpoint scn files - if (OB_SUCC(ret) && !is_tagging && OB_TMP_FAIL(del_history_files_(dir_path, checkpoint_scn))) { - LOG_WARN("failed to delete files", K(ret), K(dir_path), K(checkpoint_scn), K(tmp_ret)); + // delete only the last ckpt file: + // 1. reduce listing requests + // 2. guarantee all expired files can be eventually removed + } else if (OB_TMP_FAIL(del_last_ckpt_file_(dir_path, old_checkpoint_scn))) { + LOG_WARN("failed to delete last ckpt file", K(ret), K(dir_path), K(old_checkpoint_scn), K(tmp_ret)); } return ret; @@ -202,16 +205,40 @@ int ObArchiveCheckpointMgr::get_max_checkpoint_scn_( return ret; } -int ObArchiveCheckpointMgr::del_history_files_( +int ObArchiveCheckpointMgr::del_last_ckpt_file_( const ObBackupPath &dir_path, + const uint64_t old_checkpoint_scn) const +{ + int ret = OB_SUCCESS; + ObBackupIoAdapter io_util; + ObBackupPath file_path = dir_path; + + if (dir_path.is_empty()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("dir path is empty", K(ret), K(dir_path)); + } else if (0 == old_checkpoint_scn) { // 0 is reserved + } else if (OB_FAIL(file_path.join_checkpoint_info_file(file_name_, old_checkpoint_scn, type_))) { + LOG_WARN("fail to join checkpoint info file", K(ret), K(old_checkpoint_scn)); + } else if (OB_FAIL(io_util.del_file(file_path.get_obstr(), storage_info_))) { + LOG_WARN("fail to delete file", K(ret), K(file_path)); + } + return ret; +} + +int ObArchiveCheckpointMgr::del_history_files( const uint64_t write_checkpoint_scn) const { int ret = OB_SUCCESS; ObBackupIoAdapter io_util; - ObDelHisCheckpointFileOp del_his_file_op(write_checkpoint_scn, dir_path, file_name_, type_, storage_info_); - if (OB_FAIL(io_util.list_files(dir_path.get_ptr(), storage_info_, del_his_file_op))) { - LOG_WARN("failed to del history checkpoint file", - K(ret), K(dir_path), K(write_checkpoint_scn), K(path_), KP(file_name_), K(type_)); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObArchiveCheckpointMgr not inited", K(ret)); + } else { + ObDelHisCheckpointFileOp del_his_file_op(write_checkpoint_scn, path_, file_name_, type_, storage_info_); + if (OB_FAIL(io_util.list_files(path_.get_ptr(), storage_info_, del_his_file_op))) { + LOG_WARN("failed to del history checkpoint file", + K(ret), K(path_), K(write_checkpoint_scn), K(path_), KP(file_name_), K(type_)); + } } return ret; } diff --git a/src/share/backup/ob_archive_checkpoint_mgr.h b/src/share/backup/ob_archive_checkpoint_mgr.h index f0bc9eb57..ed63d4340 100644 --- a/src/share/backup/ob_archive_checkpoint_mgr.h +++ b/src/share/backup/ob_archive_checkpoint_mgr.h @@ -55,17 +55,19 @@ public: path_(path), file_name_(file_name), type_(type), - storage_info_(storage_info) {} + storage_info_(storage_info), + handled_file_num_(0) {} virtual ~ObDelHisCheckpointFileOp() {} bool is_valid() const; int func(const dirent *entry) ; - + int64_t get_handed_file_num() { return handled_file_num_; }; private: uint64_t checkpoint_scn_; ObBackupPath path_; const char *file_name_; ObBackupFileSuffix type_; const share::ObBackupStorageInfo *storage_info_; + int64_t handled_file_num_; DISALLOW_COPY_AND_ASSIGN(ObDelHisCheckpointFileOp); }; @@ -87,11 +89,12 @@ public: const ObBackupStorageInfo *storage_info); void reset(); bool is_valid() const; - int write(const uint64_t checkpoint_scn) const; + int write(const uint64_t old_checkpoint_scn, const uint64_t checkpoint_scn) const; int read(uint64_t &checkpoint_scn) const; + int del_history_files(const uint64_t write_checkpoint_scn) const; private: int get_max_checkpoint_scn_(const ObBackupPath &path, uint64_t &max_checkpoint_scn) const; - int del_history_files_(const ObBackupPath &dir_path, const uint64_t write_checkpoint_scn) const; + int del_last_ckpt_file_(const ObBackupPath &dir_path, const uint64_t write_checkpoint_scn) const; int write_checkpoint_file_(const ObBackupPath &path) const; int check_is_tagging_(const ObBackupStorageInfo *storage_info, bool &is_tagging) const; diff --git a/src/share/backup/ob_archive_store.cpp b/src/share/backup/ob_archive_store.cpp index bcadc210b..3087f4a93 100644 --- a/src/share/backup/ob_archive_store.cpp +++ b/src/share/backup/ob_archive_store.cpp @@ -892,7 +892,8 @@ int ObArchiveStore::read_piece_checkpoint(const int64_t dest_id, const int64_t r return ret; } -int ObArchiveStore::write_piece_checkpoint(const int64_t dest_id, const int64_t round_id, const int64_t piece_id, const int64_t file_id, const ObPieceCheckpointDesc &desc) const +int ObArchiveStore::write_piece_checkpoint(const int64_t dest_id, const int64_t round_id, const int64_t piece_id, + const int64_t file_id, const share::SCN &old_checkpoint_scn, const ObPieceCheckpointDesc &desc) const { int ret = OB_SUCCESS; ObBackupPath full_path; @@ -919,13 +920,33 @@ int ObArchiveStore::write_piece_checkpoint(const int64_t dest_id, const int64_t LOG_WARN("failed to get piece checkpoint dir path", K(ret), K(dest), K(dest_id), K(round_id), K(piece_id)); } else if (OB_FAIL(mgr.init(dir_path, OB_STR_CHECKPOINT_FILE_NAME, ObBackupFileSuffix::ARCHIVE, get_storage_info()))) { LOG_WARN("failed to init ObArchiveCheckPointMgr", K(ret), K(dir_path)); - } else if (OB_FAIL(mgr.write(desc.checkpoint_scn_.get_val_for_inner_table_field()))) { + } else if (OB_FAIL(mgr.write(old_checkpoint_scn.get_val_for_inner_table_field(), + desc.checkpoint_scn_.get_val_for_inner_table_field()))) { LOG_WARN("failed to write checkpoint info", K(ret), K(desc)); } } return ret; } +int ObArchiveStore::delete_piece_his_checkpoint(const int64_t dest_id, const int64_t round_id, const int64_t piece_id, const int64_t file_id, const uint64_t checkpoint_scn) const +{ + int ret = OB_SUCCESS; + ObBackupPath dir_path; + const ObBackupDest &dest = get_backup_dest(); + ObArchiveCheckpointMgr mgr; + if (!is_init()) { + ret = OB_NOT_INIT; + LOG_WARN("ObArchiveStore not init", K(ret)); + } else if (OB_FAIL(ObArchivePathUtil::get_piece_checkpoint_dir_path(dest, dest_id, round_id, piece_id, dir_path))) { + LOG_WARN("failed to get piece checkpoint dir path", K(ret), K(dest), K(dest_id), K(round_id), K(piece_id)); + } else if (OB_FAIL(mgr.init(dir_path, OB_STR_CHECKPOINT_FILE_NAME, ObBackupFileSuffix::ARCHIVE, get_storage_info()))) { + LOG_WARN("failed to init ObArchiveCheckPointMgr", K(ret), K(dir_path)); + } else if (OB_FAIL(mgr.del_history_files(checkpoint_scn))) { + LOG_WARN("fail to delete all checkpoint files", K(ret)); + } + return ret; +} + int ObArchiveStore::read_piece_checkpoint(ObPieceCheckpointDesc &desc) const { int ret = OB_SUCCESS; diff --git a/src/share/backup/ob_archive_store.h b/src/share/backup/ob_archive_store.h index 67bd82d23..152d32f9d 100644 --- a/src/share/backup/ob_archive_store.h +++ b/src/share/backup/ob_archive_store.h @@ -378,7 +378,8 @@ public: // oss://archive/d[dest_id]r[round_id]p[piece_id]/checkpoint/checkpoint_info.[file_id].obarc int is_piece_checkpoint_file_exist(const int64_t dest_id, const int64_t round_id, const int64_t piece_id, const int64_t file_id, bool &is_exist) const; int read_piece_checkpoint(const int64_t dest_id, const int64_t round_id, const int64_t piece_id, const int64_t file_id, ObPieceCheckpointDesc &desc) const; - int write_piece_checkpoint(const int64_t dest_id, const int64_t round_id, const int64_t piece_id, const int64_t file_id, const ObPieceCheckpointDesc &desc) const; + int write_piece_checkpoint(const int64_t dest_id, const int64_t round_id, const int64_t piece_id, const int64_t file_id, const share::SCN &old_checkpoint_scn, const ObPieceCheckpointDesc &desc) const; + int delete_piece_his_checkpoint(const int64_t dest_id, const int64_t round_id, const int64_t piece_id, const int64_t file_id, const uint64_t checkpoint_scn) const; // oss://[user_specified_path]/checkpoint/checkpoint_info.0.obarc int read_piece_checkpoint(ObPieceCheckpointDesc &desc) const; // oss://archive/d[dest_id]r[round_id]p[piece_id]/piece_d[dest_id]r[round_id]p[piece_id]_20220601T120000_20220602T120000.obarc diff --git a/src/storage/backup/ob_backup_task.cpp b/src/storage/backup/ob_backup_task.cpp index d392765b2..6cbd40144 100644 --- a/src/storage/backup/ob_backup_task.cpp +++ b/src/storage/backup/ob_backup_task.cpp @@ -6227,13 +6227,14 @@ int ObLSBackupComplementLogTask::copy_checkpoint_info(const ObTenantArchivePiece ObPieceCheckpointDesc desc; bool is_exist = false; const int64_t file_id = 0; + const share::SCN old_ckpt_scn = SCN::min_scn(); //set 0, but will not delete if (OB_FAIL(src_store.is_piece_checkpoint_file_exist(src_dest_id, round_id, piece_id, file_id, is_exist))) { LOG_WARN("failed to check is piece checkpoint file file exist", K(ret), K(src_dest_id), K(round_id), K(piece_id)); } else if (!is_exist) { // do nothing } else if (OB_FAIL(src_store.read_piece_checkpoint(src_dest_id, round_id, piece_id, file_id, desc))) { LOG_WARN("failed to read piece checkpoint", K(ret), K(piece_attr)); - } else if (OB_FAIL(dest_store.write_piece_checkpoint(dest_dest_id, round_id, piece_id, file_id, desc))) { + } else if (OB_FAIL(dest_store.write_piece_checkpoint(dest_dest_id, round_id, piece_id, file_id, old_ckpt_scn, desc))) { LOG_WARN("failed to write piece checkpoint", K(ret), K(piece_attr)); } return ret; diff --git a/unittest/share/backup/test_archive_checkpoint_mgr.cpp b/unittest/share/backup/test_archive_checkpoint_mgr.cpp index 689c802d5..4d54c3f90 100644 --- a/unittest/share/backup/test_archive_checkpoint_mgr.cpp +++ b/unittest/share/backup/test_archive_checkpoint_mgr.cpp @@ -245,17 +245,17 @@ int TestArchiveCheckpointMgr::test_write_and_read_checkpoint(const ObStorageType uint64_t checkpoint = 0; if (OB_FAIL(mgr.init(root_path, OB_STR_CHECKPOINT_FILE_NAME, ObBackupFileSuffix::ARCHIVE, &storage_info_))) { LOG_WARN("failed to init checkpoint mgr", K(ret), K(root_path)); - } else if (OB_FAIL(mgr.write(10))) { + } else if (OB_FAIL(mgr.write(10, 9))) { LOG_WARN("failed to write files", K(ret), K(root_path)); - } else if (OB_FAIL(mgr.write(9))) { + } else if (OB_FAIL(mgr.write(9, 8))) { LOG_WARN("failed to write files", K(ret), K(root_path)); } else if (OB_FAIL(mgr.read(checkpoint))) { LOG_WARN("failed to read files", K(ret)); } else if (checkpoint != 10) { ret = OB_ERROR; - } else if (OB_FAIL(mgr.write(100))) { + } else if (OB_FAIL(mgr.write(100, 99))) { LOG_WARN("failed to write files", K(ret), K(root_path)); - } else if (OB_FAIL(mgr.write(50))) { + } else if (OB_FAIL(mgr.write(50, 49))) { LOG_WARN("failed to write files", K(ret), K(root_path)); } else if (OB_FAIL(mgr.read(checkpoint))) { LOG_WARN("failed to read files", K(ret)); From 0386a03a83f24c4e678c1fa6a0a6dfcc5e743df1 Mon Sep 17 00:00:00 2001 From: hnwyllmm Date: Fri, 16 Aug 2024 07:48:12 +0000 Subject: [PATCH 083/249] fix memory leak in load data --- src/sql/engine/cmd/ob_load_data_direct_impl.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/sql/engine/cmd/ob_load_data_direct_impl.cpp b/src/sql/engine/cmd/ob_load_data_direct_impl.cpp index 2b9195534..361033e7b 100644 --- a/src/sql/engine/cmd/ob_load_data_direct_impl.cpp +++ b/src/sql/engine/cmd/ob_load_data_direct_impl.cpp @@ -905,8 +905,7 @@ int ObLoadDataDirectImpl::SimpleDataSplitUtils::split(const DataAccessParam &dat } if (OB_NOT_NULL(file_reader)) { - file_reader->~ObFileReader(); - allocator.free(file_reader); + ObFileReader::destroy(file_reader); } } return ret; From 8cf3e4cf495806522d8ec7df643ea92dcf0eb56a Mon Sep 17 00:00:00 2001 From: hezuojiao Date: Fri, 16 Aug 2024 08:41:55 +0000 Subject: [PATCH 084/249] Fix decimal int precision exceeds the maximum value --- src/sql/code_generator/ob_static_engine_expr_cg.cpp | 2 +- src/sql/resolver/ddl/ob_ddl_resolver.cpp | 2 +- src/sql/resolver/expr/ob_raw_expr_deduce_type.cpp | 7 +++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/sql/code_generator/ob_static_engine_expr_cg.cpp b/src/sql/code_generator/ob_static_engine_expr_cg.cpp index 45c129e2d..157325a85 100644 --- a/src/sql/code_generator/ob_static_engine_expr_cg.cpp +++ b/src/sql/code_generator/ob_static_engine_expr_cg.cpp @@ -308,7 +308,7 @@ int ObStaticEngineExprCG::cg_expr_basic(const ObIArray &raw_exprs) if (ob_is_decimal_int(rt_expr->datum_meta_.type_)) { const int16_t precision = rt_expr->datum_meta_.precision_; const int16_t scale = rt_expr->datum_meta_.scale_; - if (precision < 0 || precision > MAX_PRECISION_DECIMAL_INT_512 + if (precision < 0 || precision > OB_MAX_DECIMAL_POSSIBLE_PRECISION || scale < 0 || scale > precision) { ret = OB_ERR_UNEXPECTED; LOG_WARN("Unexpected ps meta for decimal int type", K(ret), K(precision), K(scale)); diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.cpp b/src/sql/resolver/ddl/ob_ddl_resolver.cpp index 366b7514f..e1c633e22 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.cpp +++ b/src/sql/resolver/ddl/ob_ddl_resolver.cpp @@ -2072,7 +2072,7 @@ int ObDDLResolver::resolve_table_option(const ParseNode *option_node, const bool ret = OB_NOT_SUPPORTED; LOG_WARN("alter table auto_increment_cache_size is not supported in data version less than 4.2.3", K(ret), K(tenant_data_version)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter table auto_increment_cache_size is not supported in data version less than 4.2.3"); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter table auto_increment_cache_size in data version less than 4.2.3"); } else if (OB_ISNULL(option_node->children_[0])) { ret = OB_ERR_UNEXPECTED; SQL_RESV_LOG(WARN, "option_node child is null", K(option_node->children_[0]), K(ret)); diff --git a/src/sql/resolver/expr/ob_raw_expr_deduce_type.cpp b/src/sql/resolver/expr/ob_raw_expr_deduce_type.cpp index 9ed9658cb..c9147c56c 100644 --- a/src/sql/resolver/expr/ob_raw_expr_deduce_type.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_deduce_type.cpp @@ -583,6 +583,13 @@ int ObRawExprDeduceType::calc_result_type(ObNonTerminalRawExpr &expr, LOG_WARN("fail to calc result type with const arguments", K(ret)); } } + if (OB_SUCC(ret)) { + // refine result type precision and scale here + if (lib::is_mysql_mode() && result_type.is_decimal_int()) { + result_type.set_precision(MIN(result_type.get_precision(), + OB_MAX_DECIMAL_POSSIBLE_PRECISION)); + } + } if (OB_FAIL(ret) && my_session_->is_varparams_sql_prepare()) { // the ps prepare stage does not do type deduction, and directly gives a default type. result_type.set_null(); From 7a4c62f2c492b67816611578099f22d66f05bd71 Mon Sep 17 00:00:00 2001 From: obdev Date: Fri, 16 Aug 2024 08:47:49 +0000 Subject: [PATCH 085/249] [CP] [to #2024072900103970893]fix bugs, mysql.proc ignore is_in_recyclebin routine --- .../virtual_table/ob_mysql_proc_table.cpp | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/observer/virtual_table/ob_mysql_proc_table.cpp b/src/observer/virtual_table/ob_mysql_proc_table.cpp index c1aff4e28..d63c11877 100644 --- a/src/observer/virtual_table/ob_mysql_proc_table.cpp +++ b/src/observer/virtual_table/ob_mysql_proc_table.cpp @@ -65,6 +65,7 @@ int ObMySQLProcTable::inner_get_next_row(common::ObNewRow *&row) const ObRoutineInfo *routine_info = NULL; sql::ObExecEnv exec_env; for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < routine_array.count(); ++row_idx) { + const ObDatabaseSchema *db_schema = NULL; exec_env.reset(); if (OB_ISNULL(routine_info = routine_array.at(row_idx))) { ret = OB_ERR_UNEXPECTED; @@ -73,6 +74,15 @@ int ObMySQLProcTable::inner_get_next_row(common::ObNewRow *&row) || ROUTINE_UDT_TYPE == routine_info->get_routine_type()) { // mysql compatible view, ignore oracle system package/udt routine continue; + } else if (OB_FAIL(schema_guard_->get_database_schema(tenant_id_, + routine_info->get_database_id(), db_schema))) { + SERVER_LOG(WARN, "Failed to get database schema", K_(tenant_id), K(routine_info->get_database_id()), K(ret)); + } else if (OB_ISNULL(db_schema)) { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "Database schema should not be NULL", K(ret)); + } else if (db_schema->is_in_recyclebin()) { + // ignore is_in_recyclebin routine + continue; } else if (OB_FAIL(exec_env.init(routine_info->get_exec_env()))) { SERVER_LOG(ERROR, "fail to load exec env", K(ret)); } else { @@ -117,18 +127,8 @@ int ObMySQLProcTable::inner_get_next_row(common::ObNewRow *&row) const uint64_t col_id = output_column_ids_.at(col_idx); switch (col_id) { case (DB): { - const ObDatabaseSchema *db_schema = NULL; - if (OB_FAIL(schema_guard_->get_database_schema(tenant_id_, - routine_info->get_database_id(), db_schema))) { - SERVER_LOG(WARN, "Failed to get database schema", K_(tenant_id), - K(routine_info->get_database_id()), K(ret)); - } else if (OB_ISNULL(db_schema)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "Database schema should not be NULL", K(ret)); - } else { - cells[col_idx].set_varchar(db_schema->get_database_name()); - cells[col_idx].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - } + cells[col_idx].set_varchar(db_schema->get_database_name()); + cells[col_idx].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); break; } case (DEFINER): { From 579021c456d5f8295afec8dc95e5436f1da5d423 Mon Sep 17 00:00:00 2001 From: SanmuWangZJU Date: Fri, 16 Aug 2024 09:05:55 +0000 Subject: [PATCH 086/249] [CP] [DATA_DICT] Fix report incompleted dict record --- .../data_dictionary/ob_data_dict_service.cpp | 38 ++++++++++++++++--- .../data_dictionary/ob_data_dict_storager.cpp | 7 ++-- .../data_dictionary/ob_data_dict_utils.cpp | 8 ++-- .../data_dictionary/ob_data_dict_utils.h | 2 +- src/share/ob_debug_sync_point.h | 1 + 5 files changed, 41 insertions(+), 15 deletions(-) diff --git a/src/logservice/data_dictionary/ob_data_dict_service.cpp b/src/logservice/data_dictionary/ob_data_dict_service.cpp index 01d7d3ae0..7e393cabc 100644 --- a/src/logservice/data_dictionary/ob_data_dict_service.cpp +++ b/src/logservice/data_dictionary/ob_data_dict_service.cpp @@ -264,6 +264,8 @@ int ObDataDictService::do_dump_data_dict_() share::SCN snapshot_scn; palf::LSN start_lsn; palf::LSN end_lsn; + int64_t start_proposal_id = 0; + int64_t end_proposal_id = 0; bool is_cluster_status_normal = false; bool is_data_dict_dump_success = false; bool is_any_log_callback_fail = false; @@ -293,17 +295,18 @@ int ObDataDictService::do_dump_data_dict_() } else if (OB_ISNULL(log_handler = ls->get_log_handler())) { ret = OB_ERR_UNEXPECTED; DDLOG(WARN, "invalid log_handler_ get from OBLS", KR(ret), K_(tenant_id)); - } else if (check_ls_leader(log_handler, is_leader)) { - DDLOG(WARN, "check_is_sys_ls_leader failed", KR(ret)); + } else if (OB_FAIL(check_ls_leader(log_handler, is_leader, start_proposal_id))) { + DDLOG(WARN, "check_is_sys_ls_leader failed", KR(ret), K(start_proposal_id)); } else if (! is_leader) { - DDLOG(DEBUG, "won't do_dump_data_dict_ cause not ls_leader", KR(ret), K(is_leader)); + ret = OB_STATE_NOT_MATCH; + DDLOG(WARN, "won't do_dump_data_dict_ cause not ls_leader", KR(ret), K(is_leader), K(start_proposal_id)); // do nothing if not ls leader. } else if (OB_FAIL(get_snapshot_scn_(snapshot_scn))) { DDLOG(WARN, "get_snapshot_scn failed", KR(ret), K(snapshot_scn)); } else if (OB_FAIL(storage_.prepare(snapshot_scn, log_handler))) { DDLOG(WARN, "storage prepare for data_dict_dump failed", KR(ret), K(snapshot_scn)); } else if (OB_FAIL(generate_dict_and_dump_(snapshot_scn))) { - DDLOG(WARN, "generate_dict_and_dump_", KR(ret), K_(tenant_id), K(snapshot_scn)); + DDLOG(WARN, "generate_dict_and_dump_", KR(ret), K_(tenant_id), K(snapshot_scn), K(start_proposal_id)); } else { is_data_dict_dump_success = true; } @@ -319,7 +322,17 @@ int ObDataDictService::do_dump_data_dict_() K(snapshot_scn), K(start_lsn), K(end_lsn), K_(stop_flag), K_(is_inited)); } ret = tmp_ret; - } else if (is_data_dict_dump_success && ! is_any_log_callback_fail) { + } else if (OB_UNLIKELY(! is_data_dict_dump_success || is_any_log_callback_fail)) { + ret = OB_STATE_NOT_MATCH; + DDLOG(INFO, "won't report data_dict persist info cause data_dict dump failed or log_callback failed", + KR(ret), K(is_data_dict_dump_success), K(is_any_log_callback_fail)); + } else if (OB_FAIL(check_ls_leader(log_handler, is_leader, end_proposal_id))) { + DDLOG(WARN, "check_is_sys_ls_leader failed", KR(ret), K(start_proposal_id), K(end_proposal_id)); + } else if (OB_UNLIKELY(! is_leader || start_proposal_id != end_proposal_id)) { + ret = OB_STATE_NOT_MATCH; + DDLOG(INFO, "won't report data_dict persist info cause currently not ls_leader or not the same election term", + KR(ret), K(is_leader), K(start_proposal_id), K(end_proposal_id)); + } else { // only report when dict dump success and all log_callback success. const int64_t half_dump_interval = ATOMIC_LOAD(&dump_interval_) / 2; const int64_t report_timeout = DEFAULT_REPORT_TIMEOUT > half_dump_interval ? half_dump_interval : DEFAULT_REPORT_TIMEOUT; @@ -607,13 +620,15 @@ int ObDataDictService::handle_table_metas_( int64_t &filter_table_count) { int ret = OB_SUCCESS; + const int64_t total_table_count = table_ids.count(); lib::ObMemAttr mem_attr(tenant_id_, "ObDatDictTbMeta"); ObArenaAllocator tb_meta_allocator(mem_attr); static const int64_t batch_table_meta_size = 200; filter_table_count = 0; + int64_t dump_succ_tb_cnt = 0; schema::ObSchemaGetterGuard schema_guard; // will reset while getting schem_guard - for (int i = 0; OB_SUCC(ret) && ! stop_flag_ && i < table_ids.count(); i++) { + for (int i = 0; OB_SUCC(ret) && i < total_table_count; i++) { const ObTableSchema *table_schema = NULL; uint64_t table_id = OB_INVALID_ID; // NOTICE: get schema_guard for each table_meta in case of too much memory usage in schema_service. @@ -649,10 +664,21 @@ int ObDataDictService::handle_table_metas_( } else if (OB_FAIL(storage_.handle_dict_meta(table_meta, header))) { DDLOG(WARN, "handle dict_table_meta failed", KR(ret), K(table_meta), K(header), KPC(table_schema)); } else { + dump_succ_tb_cnt ++; DDLOG(DEBUG, "handle dict_table_meta succ", KR(ret), K(table_meta), K(header), KPC(table_schema)); + if (i == 0) { + DEBUG_SYNC(BEFORE_DATA_DICT_DUMP_FINISH); + } + } + + if (OB_SUCC(ret) && stop_flag_) { + ret = OB_STATE_NOT_MATCH; + DDLOG(WARN, "data_dict service marked stop_flag, may already switch to follower", KR(ret), K_(stop_flag)); } } + DDLOG(INFO, "handle_table_metas_ done", KR(ret), K(total_table_count), K(dump_succ_tb_cnt), K(filter_table_count), K_(stop_flag)); + return ret; } diff --git a/src/logservice/data_dictionary/ob_data_dict_storager.cpp b/src/logservice/data_dictionary/ob_data_dict_storager.cpp index ac587a22e..4b9ae7918 100644 --- a/src/logservice/data_dictionary/ob_data_dict_storager.cpp +++ b/src/logservice/data_dictionary/ob_data_dict_storager.cpp @@ -556,6 +556,7 @@ int ObDataDictStorage::submit_to_palf_() const bool allow_compression = true; palf::LSN lsn; SCN submit_scn; + int64_t proposal_id = 0; if (OB_ISNULL(palf_buf_) || OB_ISNULL(log_handler_) @@ -568,11 +569,11 @@ int ObDataDictStorage::submit_to_palf_() DDLOG(WARN, "log_handler_ is not valid", KR(ret)); } else if (OB_UNLIKELY(palf_pos_ == 0)) { DDLOG(INFO, "empty palf_buf, do nothing", K_(palf_buf_len), K_(palf_pos)); - } else if (OB_FAIL(check_ls_leader(log_handler_, is_leader))) { - DDLOG(WARN, "check_ls_leader failed", KR(ret), K(is_leader)); + } else if (OB_FAIL(check_ls_leader(log_handler_, is_leader, proposal_id))) { + DDLOG(WARN, "check_ls_leader failed", KR(ret), K(is_leader), K(proposal_id)); } else if (OB_UNLIKELY(! is_leader)) { ret = OB_STATE_NOT_MATCH; - DDLOG(INFO, "do-nothing on non-leader logstream.", KR(ret), K(is_leader)); + DDLOG(INFO, "do-nothing on non-leader logstream.", KR(ret), K(is_leader), K(proposal_id)); } else if (OB_FAIL(alloc_palf_cb_(callback))) { DDLOG(WARN, "alloc_palf_cb_ failed", KR(ret)); } else if (OB_FAIL(log_handler_->append( diff --git a/src/logservice/data_dictionary/ob_data_dict_utils.cpp b/src/logservice/data_dictionary/ob_data_dict_utils.cpp index 91b3b44b3..3a1ae8d2e 100644 --- a/src/logservice/data_dictionary/ob_data_dict_utils.cpp +++ b/src/logservice/data_dictionary/ob_data_dict_utils.cpp @@ -128,21 +128,19 @@ int deep_copy_str_array( return ret; } -int check_ls_leader(logservice::ObLogHandler *log_handler, bool &is_leader) +int check_ls_leader(logservice::ObLogHandler *log_handler, bool &is_leader, int64_t &proposal_id) { int ret = OB_SUCCESS; common::ObRole role = common::ObRole::INVALID_ROLE; - int64_t proposal_id = 0; + proposal_id = 0; if (OB_ISNULL(log_handler)) { ret = OB_ERR_UNEXPECTED; DDLOG(WARN, "log_handler_ is invalid", KR(ret)); } else if (OB_FAIL(log_handler->get_role(role, proposal_id))) { DDLOG(WARN, "get ls role fail", K(ret), K(proposal_id)); - } else if (common::ObRole::LEADER == role) { - is_leader = true; } else { - is_leader = false; + is_leader = is_strong_leader(role); } return ret; diff --git a/src/logservice/data_dictionary/ob_data_dict_utils.h b/src/logservice/data_dictionary/ob_data_dict_utils.h index f8a8e2da6..ae199b9f2 100644 --- a/src/logservice/data_dictionary/ob_data_dict_utils.h +++ b/src/logservice/data_dictionary/ob_data_dict_utils.h @@ -124,7 +124,7 @@ OB_INLINE const char *extract_str(const ObString &str) return str.empty() ? "" : str.ptr(); } -int check_ls_leader(logservice::ObLogHandler *handler, bool &is_leader); +int check_ls_leader(logservice::ObLogHandler *handler, bool &is_leader, int64_t &proposal_id); } // namespace datadict } // namespace oceanbase diff --git a/src/share/ob_debug_sync_point.h b/src/share/ob_debug_sync_point.h index 52b1e7eeb..17736d12b 100755 --- a/src/share/ob_debug_sync_point.h +++ b/src/share/ob_debug_sync_point.h @@ -631,6 +631,7 @@ class ObString; ACT(BEFORE_RESTORE_CREATE_TABLETS_SSTABLE,)\ ACT(BEFORE_CLOSE_BACKUP_INDEX_BUILDER,)\ ACT(BEFROE_UPDATE_DATA_VERSION,)\ + ACT(BEFORE_DATA_DICT_DUMP_FINISH,)\ ACT(MAX_DEBUG_SYNC_POINT,) DECLARE_ENUM(ObDebugSyncPoint, debug_sync_point, OB_DEBUG_SYNC_POINT_DEF); From b14d24478f053d029761bfa2414e56dcefc33e19 Mon Sep 17 00:00:00 2001 From: obdev Date: Fri, 16 Aug 2024 09:12:00 +0000 Subject: [PATCH 087/249] Fix row lock checker cannot trigger bloomfilter construction --- src/storage/access/ob_sstable_row_scanner.cpp | 7 +++++++ .../ob_micro_block_row_lock_checker.cpp | 13 +++++++++++++ .../blocksstable/ob_micro_block_row_lock_checker.h | 1 + 3 files changed, 21 insertions(+) diff --git a/src/storage/access/ob_sstable_row_scanner.cpp b/src/storage/access/ob_sstable_row_scanner.cpp index 164cd8ffd..afc350fc3 100644 --- a/src/storage/access/ob_sstable_row_scanner.cpp +++ b/src/storage/access/ob_sstable_row_scanner.cpp @@ -14,7 +14,9 @@ #include "ob_sstable_row_scanner.h" #include "ob_sstable_index_filter.h" #include "ob_block_row_store.h" +#include "storage/access/ob_store_row_iterator.h" #include "storage/blocksstable/ob_datum_row.h" +#include "storage/blocksstable/ob_micro_block_row_lock_checker.h" #include "storage/column_store/ob_co_sstable_rows_filter.h" #include "lib/statistic_event/ob_stat_event.h" #include "lib/stat/ob_diagnose_info.h" @@ -376,6 +378,11 @@ int ObSSTableRowScanner::fetch_row(ObSSTableReadHandle &read_handl LOG_WARN("Fail to get next row", K(ret)); } else if (prefetcher_.cur_micro_data_fetch_idx_ >= read_handle.micro_end_idx_) { ret = OB_ITER_END; + if (ObStoreRowIterator::IteratorRowLockAndDuplicationCheck == type_ || + ObStoreRowIterator::IteratorRowLockCheck == type_) { + ObMicroBlockRowLockChecker *checker = static_cast(micro_scanner_); + checker->inc_empty_read(read_handle.get_rowkey().get_datum_cnt()); + } LOG_DEBUG("[INDEX BLOCK] Open data block handle iter end", K(ret), K(prefetcher_.cur_micro_data_fetch_idx_), K(read_handle)); } else if (FALSE_IT(prefetcher_.inc_cur_micro_data_fetch_idx())) { diff --git a/src/storage/blocksstable/ob_micro_block_row_lock_checker.cpp b/src/storage/blocksstable/ob_micro_block_row_lock_checker.cpp index 3d52a04cd..2114cf198 100644 --- a/src/storage/blocksstable/ob_micro_block_row_lock_checker.cpp +++ b/src/storage/blocksstable/ob_micro_block_row_lock_checker.cpp @@ -36,6 +36,19 @@ ObMicroBlockRowLockChecker::~ObMicroBlockRowLockChecker() { } +void ObMicroBlockRowLockChecker::inc_empty_read(int64_t empty_read_prefix) +{ + if (OB_NOT_NULL(context_) && OB_NOT_NULL(sstable_) && + !context_->query_flag_.is_index_back() && context_->query_flag_.is_use_bloomfilter_cache() && + !sstable_->is_small_sstable()) { + (void) OB_STORE_CACHE.get_bf_cache().inc_empty_read( + MTL_ID(), + param_->table_id_, + macro_id_, + empty_read_prefix); + } +} + int ObMicroBlockRowLockChecker::inner_get_next_row( bool &row_lock_checked, int64_t ¤t, diff --git a/src/storage/blocksstable/ob_micro_block_row_lock_checker.h b/src/storage/blocksstable/ob_micro_block_row_lock_checker.h index e9d9372d9..a5bda2b08 100644 --- a/src/storage/blocksstable/ob_micro_block_row_lock_checker.h +++ b/src/storage/blocksstable/ob_micro_block_row_lock_checker.h @@ -40,6 +40,7 @@ public: { check_exist_ = check_eixst; } + void inc_empty_read(int64_t empty_read_prefix); protected: virtual int inner_get_next_row( bool &row_lock_checked, From 6b0f3f5f42e147ed35de659a372649917fbfc812 Mon Sep 17 00:00:00 2001 From: hanr881 <1741282579@qq.com> Date: Fri, 16 Aug 2024 09:17:59 +0000 Subject: [PATCH 088/249] [CP] to issue<2024081300104109793>:won't catch -4023 in pl --- src/pl/ob_pl_exception_handling.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pl/ob_pl_exception_handling.cpp b/src/pl/ob_pl_exception_handling.cpp index fc6092b4f..1629ca585 100644 --- a/src/pl/ob_pl_exception_handling.cpp +++ b/src/pl/ob_pl_exception_handling.cpp @@ -516,7 +516,8 @@ bool ObPLEH::is_internal_error(int error_code) || OB_TRANS_SQL_SEQUENCE_ILLEGAL == error_code || OB_ERR_SESSION_INTERRUPTED == error_code || OB_ERR_QUERY_INTERRUPTED == error_code - || OB_TIMEOUT == error_code; + || OB_TIMEOUT == error_code + || OB_EAGAIN == error_code; } ObPLConditionType ObPLEH::eh_classify_exception(const char *sql_state) From ca630e2f98b7359b1eed96bb25f10e65dd26b50d Mon Sep 17 00:00:00 2001 From: fengdeyiji <546976189@qq.com> Date: Fri, 16 Aug 2024 09:35:56 +0000 Subject: [PATCH 089/249] [MDS] ignore state rollback operation in MDS --- src/storage/multi_data_source/mds_ctx.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/storage/multi_data_source/mds_ctx.h b/src/storage/multi_data_source/mds_ctx.h index 55a9e28b1..add374871 100644 --- a/src/storage/multi_data_source/mds_ctx.h +++ b/src/storage/multi_data_source/mds_ctx.h @@ -74,8 +74,7 @@ private: int64_t try_times = 0; do { MdsWLockGuard lg(lock_); - if (state_ == TwoPhaseCommitState::ON_PREPARE && new_state == TwoPhaseCommitState::BEFORE_PREPARE) {// due to force majeure - // do nothing, just accept it + if (state_ >= new_state) { operate_all_nodes_succeed = true; } else { operate_all_nodes_succeed = op(); From a30e1d99788b4094f2a538895b1c2e888afad029 Mon Sep 17 00:00:00 2001 From: obdev Date: Fri, 16 Aug 2024 09:54:21 +0000 Subject: [PATCH 090/249] [CP] fix: add check for dbms_scheduler interval argument --- src/observer/dbms_scheduler/ob_dbms_sched_job_master.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/observer/dbms_scheduler/ob_dbms_sched_job_master.cpp b/src/observer/dbms_scheduler/ob_dbms_sched_job_master.cpp index c0a9ef0ff..23d050a1e 100644 --- a/src/observer/dbms_scheduler/ob_dbms_sched_job_master.cpp +++ b/src/observer/dbms_scheduler/ob_dbms_sched_job_master.cpp @@ -252,6 +252,7 @@ int ObDBMSSchedJobMaster::stop() int64_t ObDBMSSchedJobMaster::calc_next_date(ObDBMSSchedJobInfo &job_info) { + int64_t ret = 0; int64_t next_date = 0; const int64_t now = ObTimeUtility::current_time(); if (job_info.is_date_expression_job_class() @@ -266,6 +267,9 @@ int64_t ObDBMSSchedJobMaster::calc_next_date(ObDBMSSchedJobInfo &job_info) } else { next_date = next_date_ts; } + } else if (job_info.get_interval_ts() < 0) { + next_date = 64060560000000000; + LOG_WARN("job interval is not valid, so regard as once job", K(job_info.get_interval_ts()), K(job_info)); } else if (job_info.get_interval_ts() == 0) { next_date = 64060560000000000; } else { From 51ffa2d8aee0841e0a95c9e1192fdf553e5d1484 Mon Sep 17 00:00:00 2001 From: chinaxing Date: Mon, 19 Aug 2024 06:00:19 +0000 Subject: [PATCH 091/249] [trans-route] support reset connection protocol --- src/observer/mysql/obmp_reset_connection.cpp | 17 ++++++++++++++--- src/sql/session/ob_sql_session_info.cpp | 1 + src/sql/session/ob_sql_session_info.h | 1 + 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/observer/mysql/obmp_reset_connection.cpp b/src/observer/mysql/obmp_reset_connection.cpp index 46536ecb2..73678ed39 100644 --- a/src/observer/mysql/obmp_reset_connection.cpp +++ b/src/observer/mysql/obmp_reset_connection.cpp @@ -45,6 +45,7 @@ int ObMPResetConnection::process() { int ret = OB_SUCCESS; bool need_disconnect = false; + bool need_response_error = false; ObSQLSessionInfo *session = NULL; ObSMConnection *conn = NULL; ObSchemaGetterGuard schema_guard; @@ -67,11 +68,16 @@ int ObMPResetConnection::process() ObDiagnoseSessionInfo *di = ObDiagnoseSessionInfo::get_local_diagnose_info(); ObMaxWaitGuard max_wait_guard(NULL, di); ObTotalWaitGuard total_wait_guard(NULL, di); + const ObMySQLRawPacket &pkt = reinterpret_cast(req_->get_packet()); session->update_last_active_time(); session->set_query_start_time(ObTimeUtility::current_time()); LOG_DEBUG("begin reset connection. ", K(session->get_sessid()), K(session->get_effective_tenant_id())); tenant_id = session->get_effective_tenant_id(); - if (OB_FAIL(gctx_.schema_service_->get_tenant_schema_guard(tenant_id, schema_guard))) { + session->set_txn_free_route(pkt.txn_free_route()); + if (OB_FAIL(process_extra_info(*session, pkt, need_response_error))) { + LOG_WARN("fail get process extra info", K(ret)); + } else if (FALSE_IT(session->post_sync_session_info())) { + } else if (OB_FAIL(gctx_.schema_service_->get_tenant_schema_guard(tenant_id, schema_guard))) { OB_LOG(WARN,"fail get schema guard", K(ret)); } else if (OB_FAIL(schema_guard.get_sys_variable_schema(tenant_id, sys_variable_schema))) { LOG_WARN("get sys variable schema failed", K(ret)); @@ -120,8 +126,13 @@ int ObMPResetConnection::process() // 1. Rolls back any active transactions and resets autocommit mode. if (OB_SUCC(ret)) { - if (OB_FAIL(ObSqlTransControl::kill_tx(session, OB_TRANS_IDLE_TIMEOUT))) { - OB_LOG(WARN, "fail to rollback trans for change user", K(ret), K(need_disconnect)); + // for XA trans, can not rollback it directly, use kill_tx to abort it + if (session->associated_xa()) { + if (OB_FAIL(ObSqlTransControl::kill_tx(session, OB_TRANS_ROLLBACKED))) { + OB_LOG(WARN, "fail to kill xa trans for reset connection", K(ret)); + } + } else if (OB_FAIL(ObSqlTransControl::rollback_trans(session, need_disconnect))) { + OB_LOG(WARN, "fail to rollback trans for reset connection", K(ret), K(need_disconnect)); } } diff --git a/src/sql/session/ob_sql_session_info.cpp b/src/sql/session/ob_sql_session_info.cpp index df0916591..3289708c1 100644 --- a/src/sql/session/ob_sql_session_info.cpp +++ b/src/sql/session/ob_sql_session_info.cpp @@ -3321,6 +3321,7 @@ int ObSQLSessionInfo::update_sess_sync_info(const SessionSyncInfoType sess_sync_ LOG_WARN("invalid session sync info encoder", K(ret), K(sess_sync_info_type)); } else if (OB_FAIL(sess_encoders_[sess_sync_info_type]->deserialize(*this, buf, length, pos))) { LOG_WARN("failed to deserialize sess sync info", K(ret), K(sess_sync_info_type), KPHEX(buf, length), K(length), K(pos)); + } else if (FALSE_IT(sess_encoders_[sess_sync_info_type]->is_changed_ = false)) { } else { // do nothing LOG_DEBUG("get app info", K(client_app_info_.module_name_), K(client_app_info_.action_name_), K(client_app_info_.client_info_)); diff --git a/src/sql/session/ob_sql_session_info.h b/src/sql/session/ob_sql_session_info.h index 99af0323e..2ec293897 100644 --- a/src/sql/session/ob_sql_session_info.h +++ b/src/sql/session/ob_sql_session_info.h @@ -1180,6 +1180,7 @@ public: uint64_t seq_id); void reuse_all_sequence_value() { + sequence_currval_encoder_.is_changed_ = !sequence_currval_map_.empty() || !dblink_sequence_id_map_.empty(); sequence_currval_map_.reuse(); dblink_sequence_id_map_.reuse(); } From 0b6dbb39f21d3ff2cb7848b3cf1d6802f8928e42 Mon Sep 17 00:00:00 2001 From: Tsunaou <895254752@qq.com> Date: Mon, 19 Aug 2024 06:12:54 +0000 Subject: [PATCH 092/249] fix row col switch co merge hung if finish dag add into dag net failed. --- src/storage/column_store/ob_co_merge_ctx.h | 5 +++ src/storage/column_store/ob_co_merge_dag.cpp | 39 ++++++++++++++++---- src/storage/column_store/ob_co_merge_dag.h | 1 + 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/src/storage/column_store/ob_co_merge_ctx.h b/src/storage/column_store/ob_co_merge_ctx.h index 579772598..cd8ccd0bf 100644 --- a/src/storage/column_store/ob_co_merge_ctx.h +++ b/src/storage/column_store/ob_co_merge_ctx.h @@ -70,6 +70,11 @@ struct ObCOTabletMergeCtx : public ObBasicTabletMergeCtx exe_stat_.finish_cg_count_ += cg_cnt; exe_stat_.period_finish_cg_count_ += cg_cnt; } + OB_INLINE void set_batch_finish_for_row_store(const int64_t cg_cnt) + { + exe_stat_.finish_cg_count_ = cg_cnt; + exe_stat_.period_finish_cg_count_ = cg_cnt; + } OB_INLINE void one_batch_fail() { ++exe_stat_.error_count_; diff --git a/src/storage/column_store/ob_co_merge_dag.cpp b/src/storage/column_store/ob_co_merge_dag.cpp index 0db2340e7..db435c348 100644 --- a/src/storage/column_store/ob_co_merge_dag.cpp +++ b/src/storage/column_store/ob_co_merge_dag.cpp @@ -34,6 +34,8 @@ using namespace storage; namespace compaction { +ERRSIM_POINT_DEF(EN_COMPACTION_ADD_CO_MREGE_FINISH_DAG_INTO_DAG_NET_FAILED); + ObCOMergeDagParam::ObCOMergeDagParam() : ObTabletMergeDagParam(), start_cg_idx_(0), @@ -1177,6 +1179,25 @@ int ObCOMergeDagNet::choose_merge_batch_size(const int64_t column_group_cnt) return ret; } +int ObCOMergeDagNet::init_cg_schedule_status_for_row_store() +{ + int ret = OB_SUCCESS; + const int64_t max_cg_idx = co_merge_ctx_->get_schema()->get_column_group_count(); + int32_t start_cg_idx = OB_INVALID_INDEX; + + if (!co_merge_ctx_->is_build_row_store()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid co merge type", KPC_(co_merge_ctx)); + } else if (OB_FAIL(co_merge_ctx_->get_schema()->get_base_rowkey_column_group_index(start_cg_idx))) { + LOG_WARN("failed to get all cg or rowkey cg in cg schemas", K(ret), KPC_(co_merge_ctx)); + } else { + MARK_CG_SCHEDULE_STATUS(0, max_cg_idx, ObCOTabletMergeCtx::CG_SCHE_STATUS_FINISHED); + co_merge_ctx_->cg_schedule_status_array_[start_cg_idx] = ObCOTabletMergeCtx::CG_SCHE_STATUS_IDLE; + co_merge_ctx_->set_batch_finish_for_row_store(max_cg_idx - 1); + } + return ret; +} + int ObCOMergeDagNet::inner_schedule_finish_dag(ObIDag *parent_dag) { int ret = OB_SUCCESS; @@ -1246,8 +1267,14 @@ int ObCOMergeDagNet::inner_create_and_schedule_dags(ObIDag *parent_dag) */ if (OB_NOT_NULL(parent_dag) && (co_merge_ctx_->is_build_row_store() || max_cg_idx < DELAY_SCHEDULE_FINISH_DAG_CG_CNT)) { +#ifdef ERRSIM + if (EN_COMPACTION_ADD_CO_MREGE_FINISH_DAG_INTO_DAG_NET_FAILED) { + ret = EN_COMPACTION_ADD_CO_MREGE_FINISH_DAG_INTO_DAG_NET_FAILED; + LOG_INFO("ERRSIM EN_COMPACTION_ADD_CO_MREGE_FINISH_DAG_INTO_DAG_NET_FAILED", K(ret), KPC(parent_dag)); + } +#endif // add into dag_scheduler after parent-child relation generated - if (OB_FAIL(create_dag(0, 0, finish_dag_, parent_dag/*parent*/, false/*add_scheduler_flag*/))) { + if (FAILEDx(create_dag(0, 0, finish_dag_, parent_dag/*parent*/, false/*add_scheduler_flag*/))) { LOG_WARN("failed to create finish dag", K(ret), K_(finish_dag)); } } @@ -1299,13 +1326,9 @@ int ObCOMergeDagNet::inner_create_row_store_dag( if (OB_FAIL(co_merge_ctx_->get_schema()->get_base_rowkey_column_group_index(start_cg_idx))) { LOG_WARN("failed to get all cg or rowkey cg in cg schemas", K(ret)); - } else if (OB_NOT_NULL(parent_dag)) { - // in schedule state, firstly into inner_create_and_schedule_dags - MARK_CG_SCHEDULE_STATUS(0, max_cg_idx, ObCOTabletMergeCtx::CG_SCHE_STATUS_FINISHED); - co_merge_ctx_->cg_schedule_status_array_[start_cg_idx] = ObCOTabletMergeCtx::CG_SCHE_STATUS_IDLE; - co_merge_ctx_->one_batch_finish(max_cg_idx - 1); - } - if (OB_FAIL(ret) || !ObCOTabletMergeCtx::is_cg_could_schedule(co_merge_ctx_->cg_schedule_status_array_[start_cg_idx])) { + } else if (!ObCOTabletMergeCtx::is_cg_could_schedule(co_merge_ctx_->cg_schedule_status_array_[start_cg_idx])) { + } else if (OB_FAIL(init_cg_schedule_status_for_row_store())) { + LOG_WARN("failed to init cg schedule status", K(ret)); } else if (OB_FAIL(inner_create_exe_dags(start_cg_idx, start_cg_idx + 1, max_cg_idx, allowed_schedule_dag_count_place_holder, dag, exe_dag_array))) { LOG_WARN("failed to create or add cg dag", K(ret), K(start_cg_idx)); diff --git a/src/storage/column_store/ob_co_merge_dag.h b/src/storage/column_store/ob_co_merge_dag.h index 6c43294b6..8feb2c2a8 100644 --- a/src/storage/column_store/ob_co_merge_dag.h +++ b/src/storage/column_store/ob_co_merge_dag.h @@ -322,6 +322,7 @@ private: ObCOMergeBatchExeDag *&dag, common::ObIArray &exe_dag_array); int choose_merge_batch_size(const int64_t column_group_cnt); + int init_cg_schedule_status_for_row_store(); int inner_schedule_finish_dag(ObIDag *parent_dag = nullptr); void try_update_merge_batch_size(const int64_t column_group_cnt); int inner_create_and_schedule_dags(ObIDag *parent_dag = nullptr); From bdb3bec64279c68b2b71963a6a3bc560e7c7db79 Mon Sep 17 00:00:00 2001 From: liucc1997 <1192520566@qq.com> Date: Mon, 19 Aug 2024 06:32:13 +0000 Subject: [PATCH 093/249] [CP] save rpc decode error code in rpc async_cb --- deps/oblib/src/rpc/frame/ob_req_transport.cpp | 1 + deps/oblib/src/rpc/frame/ob_req_transport.h | 5 ++++- deps/oblib/src/rpc/obrpc/ob_poc_rpc_proxy.cpp | 14 ++++++++------ deps/oblib/src/rpc/obrpc/ob_rpc_endec.cpp | 14 ++++++-------- deps/oblib/src/rpc/obrpc/ob_rpc_endec.h | 2 +- src/sql/das/ob_das_rpc_processor.cpp | 4 +++- src/sql/engine/px/ob_px_sqc_async_proxy.cpp | 4 +++- 7 files changed, 26 insertions(+), 18 deletions(-) diff --git a/deps/oblib/src/rpc/frame/ob_req_transport.cpp b/deps/oblib/src/rpc/frame/ob_req_transport.cpp index 9f0cdbc21..1dc71a856 100644 --- a/deps/oblib/src/rpc/frame/ob_req_transport.cpp +++ b/deps/oblib/src/rpc/frame/ob_req_transport.cpp @@ -97,6 +97,7 @@ int async_cb(easy_request_t *r) ret = OB_LIBEASY_ERROR; } else if (OB_FAIL(cb->decode(r->ipacket))) { + cb->set_error(ret); cb->on_invalid(); LOG_WARN("decode failed", K(ret), K(pcode)); } else if (OB_PACKET_CLUSTER_ID_NOT_MATCH == cb->get_rcode()) { diff --git a/deps/oblib/src/rpc/frame/ob_req_transport.h b/deps/oblib/src/rpc/frame/ob_req_transport.h index 5646c90b2..020f91a88 100644 --- a/deps/oblib/src/rpc/frame/ob_req_transport.h +++ b/deps/oblib/src/rpc/frame/ob_req_transport.h @@ -110,7 +110,10 @@ public: virtual bool get_cloned() = 0; // invoke when get a valid packet on protocol level, but can't decode it. - virtual void on_invalid() { RPC_FRAME_LOG_RET(ERROR, common::OB_INVALID_ARGUMENT, "invalid packet"); } + virtual void on_invalid() { + int ret = err_; + RPC_FRAME_LOG(ERROR, "rpc response decode failed, tenant oom or deserialization failed", K_(pcode), K_(tenant_id), K_(dst)); + } // invoke when can't get a valid or completed packet. virtual void on_timeout() { RPC_FRAME_LOG(DEBUG, "packet timeout"); } virtual int on_error(int err); diff --git a/deps/oblib/src/rpc/obrpc/ob_poc_rpc_proxy.cpp b/deps/oblib/src/rpc/obrpc/ob_poc_rpc_proxy.cpp index f863b9c9a..9bcfbf8d8 100644 --- a/deps/oblib/src/rpc/obrpc/ob_poc_rpc_proxy.cpp +++ b/deps/oblib/src/rpc/obrpc/ob_poc_rpc_proxy.cpp @@ -128,7 +128,7 @@ int ObAsyncRespCallback::handle_resp(int io_err, const char* buf, int64_t sz) int64_t after_decode_time = 0; int64_t after_process_time = 0; ObRpcPacketCode pcode = OB_INVALID_RPC_CODE; - ObRpcPacket* ret_pkt = NULL; + ObRpcPacket ret_pkt; if (buf != NULL && sz > easy_head_size) { EVENT_INC(RPC_PACKET_IN); EVENT_ADD(RPC_PACKET_IN_BYTES, sz); @@ -151,22 +151,24 @@ int ObAsyncRespCallback::handle_resp(int io_err, const char* buf, int64_t sz) } } else if (NULL == buf) { ucb_->on_timeout(); - } else if (OB_FAIL(rpc_decode_ob_packet(pool_, buf, sz, ret_pkt))) { + } else if (OB_FAIL(rpc_decode_ob_packet(buf, sz, ret_pkt))) { + ucb_->set_error(ret); ucb_->on_invalid(); RPC_LOG(WARN, "rpc_decode_ob_packet fail", K(ret)); - } else if (OB_FALSE_IT(ObCurTraceId::set(ret_pkt->get_trace_id()))) { + } else if (OB_FALSE_IT(ObCurTraceId::set(ret_pkt.get_trace_id()))) { } #ifdef ERRSIM - else if (OB_FALSE_IT(THIS_WORKER.set_module_type(ret_pkt->get_module_type()))) { + else if (OB_FALSE_IT(THIS_WORKER.set_module_type(ret_pkt.get_module_type()))) { } #endif - else if (OB_FAIL(ucb_->decode(ret_pkt))) { + else if (OB_FAIL(ucb_->decode(&ret_pkt))) { + ucb_->set_error(ret); ucb_->on_invalid(); RPC_LOG(WARN, "ucb.decode fail", K(ret)); } else { after_decode_time = ObTimeUtility::current_time(); int tmp_ret = OB_SUCCESS; - pcode = ret_pkt->get_pcode(); + pcode = ret_pkt.get_pcode(); if (OB_SUCCESS != (tmp_ret = ucb_->process())) { RPC_LOG(WARN, "ucb.process fail", K(tmp_ret)); } diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_endec.cpp b/deps/oblib/src/rpc/obrpc/ob_rpc_endec.cpp index c40a008df..6229893c2 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_endec.cpp +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_endec.cpp @@ -71,16 +71,14 @@ int init_packet(ObRpcProxy& proxy, ObRpcPacket& pkt, ObRpcPacketCode pcode, cons return proxy.init_pkt(&pkt, pcode, opts, unneed_response); } -int rpc_decode_ob_packet(ObRpcMemPool& pool, const char* buf, int64_t sz, ObRpcPacket*& ret_pkt) +int rpc_decode_ob_packet(const char* buf, int64_t sz, ObRpcPacket& pkt) { int ret = common::OB_SUCCESS; - ObRpcPacket* pkt = (ObRpcPacket*)pool.alloc(sizeof(ObRpcPacket)); - if (NULL == pkt) { - ret = common::OB_ALLOCATE_MEMORY_FAILED; - } else { - new(pkt)ObRpcPacket(); - if (OB_SUCC(pkt->decode(buf, sz))) { - ret_pkt = pkt; + if (OB_SUCC(pkt.decode(buf, sz))) { + const int64_t fly_ts = ObTimeUtility::current_time() - pkt.get_timestamp(); + if (pkt.get_timestamp() > 0 && fly_ts > oceanbase::common::OB_MAX_PACKET_FLY_TS && TC_REACH_TIME_INTERVAL(100 * 1000)) { + RPC_LOG_RET(WARN, common::OB_ERR_TOO_MUCH_TIME, "PNIO packet wait too much time between response and client_cb", "pcode", pkt.get_pcode(), + "fly_ts", fly_ts, "send_timestamp", pkt.get_timestamp(), K(sz)); } } return ret; diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_endec.h b/deps/oblib/src/rpc/obrpc/ob_rpc_endec.h index 18617b1ff..decdb76fc 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_endec.h +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_endec.h @@ -175,7 +175,7 @@ int rpc_decode_resp(const char* resp_buf, int64_t resp_sz, T& result, ObRpcPacke return ret; } -int rpc_decode_ob_packet(ObRpcMemPool& pool, const char* buf, int64_t sz, ObRpcPacket*& ret_pkt); +int rpc_decode_ob_packet(const char* buf, int64_t sz, ObRpcPacket& ret_pkt); int rpc_encode_ob_packet(ObRpcMemPool& pool, ObRpcPacket* pkt, char*& buf, int64_t& sz, int64_t reserve_buf_size); }; // end namespace obrpc diff --git a/src/sql/das/ob_das_rpc_processor.cpp b/src/sql/das/ob_das_rpc_processor.cpp index a0242f886..172f4ec58 100644 --- a/src/sql/das/ob_das_rpc_processor.cpp +++ b/src/sql/das/ob_das_rpc_processor.cpp @@ -242,8 +242,10 @@ void ObRpcDasAsyncAccessCallBack::on_invalid() { int ret = OB_SUCCESS; // a valid packet on protocol level, but can't decode it. + ret = get_error() == OB_ALLOCATE_MEMORY_FAILED ? + OB_ALLOCATE_MEMORY_FAILED : OB_INVALID_ERROR; LOG_WARN("das async task invalid", K(get_task_ops())); - result_.set_err_code(OB_INVALID_ERROR); + result_.set_err_code(ret); result_.get_op_results().reuse(); context_->get_das_ref().inc_concurrency_limit_with_signal(); } diff --git a/src/sql/engine/px/ob_px_sqc_async_proxy.cpp b/src/sql/engine/px/ob_px_sqc_async_proxy.cpp index 47428a16c..126d3bd3b 100644 --- a/src/sql/engine/px/ob_px_sqc_async_proxy.cpp +++ b/src/sql/engine/px/ob_px_sqc_async_proxy.cpp @@ -190,7 +190,9 @@ int ObPxSqcAsyncProxy::wait_all() { } else if (!callback.is_visited() && callback.is_invalid()) { // rpc解析pack失败，callback调用on_invalid方法，不需要重试 return_cb_count_++; - ret = OB_RPC_PACKET_INVALID; + ret = callback.get_error() == OB_ALLOCATE_MEMORY_FAILED ? + OB_ALLOCATE_MEMORY_FAILED : OB_RPC_PACKET_INVALID; + LOG_WARN("callback invalid", K(ret), K(callback.get_error())); callback.set_visited(true); } else if (!callback.is_visited() && callback.is_processed()) { return_cb_count_++; From 4dbf5fad236abe5fc7d5c42b0741b36dee7ce046 Mon Sep 17 00:00:00 2001 From: wanyue-wy <345657357@qq.com> Date: Mon, 19 Aug 2024 06:39:07 +0000 Subject: [PATCH 094/249] fix truncate tmp file write buffer index cache bug --- .../tmp_file/ob_tmp_file_test_helper.h | 27 ++-- .../mtlenv/storage/tmp_file/test_tmp_file.cpp | 120 +++++++++++++++--- .../tmp_file/ob_shared_nothing_tmp_file.cpp | 14 +- src/storage/tmp_file/ob_tmp_file_io_ctx.cpp | 6 +- src/storage/tmp_file/ob_tmp_file_io_ctx.h | 6 +- .../tmp_file/ob_tmp_file_io_define.cpp | 9 +- src/storage/tmp_file/ob_tmp_file_io_define.h | 3 +- .../ob_tmp_file_write_buffer_index_cache.cpp | 5 +- 8 files changed, 151 insertions(+), 39 deletions(-) diff --git a/mittest/mtlenv/storage/tmp_file/ob_tmp_file_test_helper.h b/mittest/mtlenv/storage/tmp_file/ob_tmp_file_test_helper.h index 643277a2f..9e746c75a 100644 --- a/mittest/mtlenv/storage/tmp_file/ob_tmp_file_test_helper.h +++ b/mittest/mtlenv/storage/tmp_file/ob_tmp_file_test_helper.h @@ -156,7 +156,7 @@ public: TestTmpFileStress(ObTenantBase *tenant_ctx); virtual ~TestTmpFileStress(); int init(const int fd, const TmpFileOp op, const int64_t thread_cnt, - char *buf, const int64_t offset, const int64_t size); + char *buf, const int64_t offset, const int64_t size, const bool disable_block_cache); void reset(); virtual void run1(); TO_STRING_KV(K_(thread_cnt), K_(fd), K_(op), KP_(buf), K_(offset), K_(size)); @@ -171,6 +171,7 @@ private: char *buf_; int64_t offset_; int64_t size_; + bool disable_block_cache_; ObTenantBase *tenant_ctx_; }; @@ -179,6 +180,7 @@ TestTmpFileStress::TestTmpFileStress(ObTenantBase *tenant_ctx) op_(OP_MAX), buf_(nullptr), offset_(0), size_(0), + disable_block_cache_(false), tenant_ctx_(tenant_ctx) { } @@ -190,7 +192,8 @@ TestTmpFileStress::~TestTmpFileStress() int TestTmpFileStress::init(const int fd, const TmpFileOp op, const int64_t thread_cnt, char *buf, int64_t offset, - const int64_t size) + const int64_t size, + const bool disable_block_cache) { int ret = OB_SUCCESS; if (thread_cnt < 0 || OB_ISNULL(buf) || offset < 0 || size <= 0) { @@ -209,6 +212,7 @@ int TestTmpFileStress::init(const int fd, const TmpFileOp op, op_ = op; offset_ = offset; size_ = size; + disable_block_cache_ = disable_block_cache; set_thread_count(static_cast(thread_cnt)); } return ret; @@ -222,6 +226,7 @@ void TestTmpFileStress::reset() buf_ = nullptr; offset_ = 0; size_ = 0; + disable_block_cache_ = false; } void TestTmpFileStress::write_data_(const int64_t write_size) @@ -271,6 +276,7 @@ void TestTmpFileStress::read_data_(const int64_t read_offset, const int64_t read io_info.io_desc_.set_wait_event(2); io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; io_info.buf_ = read_buf; + io_info.disable_block_cache_ = disable_block_cache_; ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); int cmp = memcmp(handle.get_buffer(), buf_ + read_offset, io_info.size_); if (cmp != 0 || OB_FAIL(ret)) { @@ -295,6 +301,7 @@ void TestTmpFileStress::truncate_data_() io_info.fd_ = fd_; io_info.io_desc_.set_wait_event(2); io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; + io_info.disable_block_cache_ = disable_block_cache_; const int64_t invalid_size = truncate_offset - offset_; const int64_t valid_size = size_ - invalid_size; @@ -380,7 +387,7 @@ public: TestMultiTmpFileStress(ObTenantBase *tenant_ctx); virtual ~TestMultiTmpFileStress(); int init(const int64_t file_cnt, const int64_t dir_id, const int64_t thread_cnt, - const int64_t batch_size, const int64_t batch_num); + const int64_t batch_size, const int64_t batch_num, const bool disable_block_cache); virtual void run1(); private: int64_t file_cnt_; @@ -388,6 +395,7 @@ private: int64_t read_thread_cnt_perf_file_; int64_t batch_size_; int64_t batch_num_; + bool disable_block_cache_; ObTenantBase *tenant_ctx_; }; @@ -397,6 +405,7 @@ TestMultiTmpFileStress::TestMultiTmpFileStress(ObTenantBase *tenant_ctx) read_thread_cnt_perf_file_(0), batch_size_(0), batch_num_(0), + disable_block_cache_(true), tenant_ctx_(tenant_ctx) { } @@ -409,7 +418,8 @@ int TestMultiTmpFileStress::init(const int64_t file_cnt, const int64_t dir_id, const int64_t thread_cnt, const int64_t batch_size, - const int64_t batch_num) + const int64_t batch_num, + const bool disable_block_cache) { int ret = OB_SUCCESS; if (file_cnt < 0 || thread_cnt < 0) { @@ -421,6 +431,7 @@ int TestMultiTmpFileStress::init(const int64_t file_cnt, read_thread_cnt_perf_file_ = thread_cnt; batch_size_ = batch_size; batch_num_ = batch_num; + disable_block_cache_ = disable_block_cache; set_thread_count(static_cast(file_cnt)); } return ret; @@ -459,13 +470,13 @@ void TestMultiTmpFileStress::run1() for (int64_t i = 0; i < batch_num_; ++i) { if (i > 0) { // truncate read data in previous round - test_truncate.init(fd, TmpFileOp::TRUNCATE, 1, data_buffer, (i-1) * batch_size_, batch_size_); + test_truncate.init(fd, TmpFileOp::TRUNCATE, 1, data_buffer, (i-1) * batch_size_, batch_size_, disable_block_cache_); ASSERT_EQ(OB_SUCCESS, ret); STORAGE_LOG(INFO, "test_truncate run start", K(i), K(batch_size_)); test_truncate.start(); } TestTmpFileStress test_write(tenant_ctx_); - ret = test_write.init(fd, TmpFileOp::WRITE, 1, data_buffer + i * batch_size_, 0, batch_size_); + ret = test_write.init(fd, TmpFileOp::WRITE, 1, data_buffer + i * batch_size_, 0, batch_size_, disable_block_cache_); ASSERT_EQ(OB_SUCCESS, ret); STORAGE_LOG(INFO, "test_write run start"); test_write.start(); @@ -473,7 +484,7 @@ void TestMultiTmpFileStress::run1() STORAGE_LOG(INFO, "test_write run end"); TestTmpFileStress test_read(tenant_ctx_); - ret = test_read.init(fd, TmpFileOp::READ, read_thread_cnt_perf_file_, data_buffer, i * batch_size_, batch_size_); + ret = test_read.init(fd, TmpFileOp::READ, read_thread_cnt_perf_file_, data_buffer, i * batch_size_, batch_size_, disable_block_cache_); ASSERT_EQ(OB_SUCCESS, ret); STORAGE_LOG(INFO, "test_read run start", K(i), K(batch_size_)); @@ -491,7 +502,7 @@ void TestMultiTmpFileStress::run1() STORAGE_LOG(INFO, "TestMultiTmpFileStress thread run a batch end", K(i)); } - test_truncate.init(fd, TmpFileOp::TRUNCATE, 1, data_buffer, file_size - batch_size_, batch_size_); + test_truncate.init(fd, TmpFileOp::TRUNCATE, 1, data_buffer, file_size - batch_size_, batch_size_, disable_block_cache_); ASSERT_EQ(OB_SUCCESS, ret); STORAGE_LOG(INFO, "test_truncate run start"); test_truncate.start(); diff --git a/mittest/mtlenv/storage/tmp_file/test_tmp_file.cpp b/mittest/mtlenv/storage/tmp_file/test_tmp_file.cpp index 8e745dc6c..e9eb74acc 100644 --- a/mittest/mtlenv/storage/tmp_file/test_tmp_file.cpp +++ b/mittest/mtlenv/storage/tmp_file/test_tmp_file.cpp @@ -84,7 +84,18 @@ void TestTmpFile::SetUpTestCase() void TestTmpFile::SetUp() { int ret = OB_SUCCESS; + + const int64_t bucket_num = 1024L; + const int64_t max_cache_size = 1024L * 1024L * 512; + const int64_t block_size = common::OB_MALLOC_BIG_BLOCK_SIZE; + ASSERT_EQ(true, MockTenantModuleEnv::get_instance().is_inited()); + if (!ObKVGlobalCache::get_instance().inited_) { + ASSERT_EQ(OB_SUCCESS, ObKVGlobalCache::get_instance().init(&getter, + bucket_num, + max_cache_size, + block_size)); + } // if (!MTL(ObTenantTmpFileManager *)->is_inited_) { // ret = MTL(ObTenantTmpFileManager *)->init(); // ASSERT_EQ(OB_SUCCESS, ret); @@ -101,6 +112,7 @@ void TestTmpFile::TearDownTestCase() void TestTmpFile::TearDown() { + ObKVGlobalCache::get_instance().destroy(); // if (MTL(ObTenantTmpFileManager *)->is_inited_) { // MTL(ObTenantTmpFileManager *)->stop(); // MTL(ObTenantTmpFileManager *)->wait(); @@ -295,6 +307,19 @@ TEST_F(TestTmpFile, test_read) read_buf = new char [wbp_begin_offset]; io_info.buf_ = read_buf; io_info.size_ = wbp_begin_offset; + io_info.disable_block_cache_ = true; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, 0, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(io_info.size_, handle.get_done_size()); + cmp = memcmp(handle.get_buffer(), write_buf, io_info.size_); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + + read_buf = new char [wbp_begin_offset]; + io_info.buf_ = read_buf; + io_info.size_ = wbp_begin_offset; + io_info.disable_block_cache_ = false; ret = MTL(ObTenantTmpFileManager *)->pread(io_info, 0, handle); ASSERT_EQ(OB_SUCCESS, ret); ASSERT_EQ(io_info.size_, handle.get_done_size()); @@ -309,6 +334,21 @@ TEST_F(TestTmpFile, test_read) read_buf = new char [read_size]; io_info.buf_ = read_buf; io_info.size_ = read_size; + io_info.disable_block_cache_ = true; + ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(io_info.size_, handle.get_done_size()); + cmp = memcmp(handle.get_buffer(), write_buf + read_offset, io_info.size_); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + + read_size = wbp_begin_offset / 2 + 9 * 1024; + read_offset = wbp_begin_offset / 2 + 1024; + read_buf = new char [read_size]; + io_info.buf_ = read_buf; + io_info.size_ = read_size; + io_info.disable_block_cache_ = false; ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); ASSERT_EQ(OB_SUCCESS, ret); ASSERT_EQ(io_info.size_, handle.get_done_size()); @@ -321,6 +361,7 @@ TEST_F(TestTmpFile, test_read) read_buf = new char [200]; io_info.buf_ = read_buf; io_info.size_ = 200; + io_info.disable_block_cache_ = true; ret = MTL(ObTenantTmpFileManager *)->pread(io_info, write_size - 100, handle); ASSERT_EQ(OB_ITER_END, ret); ASSERT_EQ(100, handle.get_done_size()); @@ -441,6 +482,7 @@ TEST_F(TestTmpFile, test_cached_read) io_info.buf_ = read_buf; io_info.size_ = read_size; io_info.disable_page_cache_ = true; + io_info.disable_block_cache_ = false; ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); ASSERT_EQ(OB_SUCCESS, ret); ASSERT_EQ(io_info.size_, handle.get_done_size()); @@ -449,15 +491,7 @@ TEST_F(TestTmpFile, test_cached_read) handle.reset(); delete[] read_buf; - // 4. clear block kv cache - for (int64_t i = 0; OB_SUCC(ret) && i < data_items.count(); i++) { - const int64_t block_index = data_items[i].block_index_; - ObTmpBlockValueHandle block_value_handle; - ret = ObTmpBlockCache::get_instance().erase(ObTmpBlockCacheKey(block_index, MTL_ID())); - ASSERT_EQ(OB_SUCCESS, ret); - } - - // 5. read disk data and puts them into kv_cache + // 4. read disk data and puts them into kv_cache int64_t read_time = ObTimeUtility::current_time(); read_size = wbp_begin_offset - ObTmpFileGlobal::PAGE_SIZE; read_offset = ObTmpFileGlobal::PAGE_SIZE / 2; @@ -465,6 +499,7 @@ TEST_F(TestTmpFile, test_cached_read) io_info.buf_ = read_buf; io_info.size_ = read_size; io_info.disable_page_cache_ = false; + io_info.disable_block_cache_ = true; ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); ASSERT_EQ(OB_SUCCESS, ret); ASSERT_EQ(io_info.size_, handle.get_done_size()); @@ -480,6 +515,7 @@ TEST_F(TestTmpFile, test_cached_read) io_info.buf_ = read_buf; io_info.size_ = read_size; io_info.disable_page_cache_ = false; + io_info.disable_block_cache_ = true; ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); ASSERT_EQ(OB_SUCCESS, ret); ASSERT_EQ(io_info.size_, handle.get_done_size()); @@ -504,6 +540,7 @@ TEST_F(TestTmpFile, test_cached_read) ret = ObTmpPageCache::get_instance().get_page(key, handle); if (OB_FAIL(ret)) { std::cout << "get cached page failed" << i <<" "<< data_item.block_index_<<" "<< physical_page_id << std::endl; + ob_abort(); } ASSERT_EQ(OB_SUCCESS, ret); cmp = memcmp(handle.value_->get_buffer(), write_buf + (data_item.virtual_page_id_ + j) * ObTmpFileGlobal::PAGE_SIZE, ObTmpFileGlobal::PAGE_SIZE); @@ -558,6 +595,7 @@ TEST_F(TestTmpFile, test_write_tail_page) io_info.buf_ = write_buf; io_info.size_ = 2 * 1024; // 2KB io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; + io_info.disable_block_cache_ = true; ret = MTL(ObTenantTmpFileManager *)->write(io_info); ASSERT_EQ(OB_SUCCESS, ret); already_write_size += io_info.size_; @@ -725,6 +763,7 @@ TEST_F(TestTmpFile, test_tmp_file_truncate) char *read_buf = new char [read_size]; ObTmpFileIOHandle handle; io_info.buf_ = read_buf; + io_info.disable_block_cache_ = true; ret = MTL(ObTenantTmpFileManager *)->pread(io_info, read_offset, handle); ASSERT_EQ(OB_SUCCESS, ret); ASSERT_EQ(read_size, handle.get_done_size()); @@ -1074,7 +1113,7 @@ void test_big_file(const int64_t write_size, const int64_t wbp_mem_limit, ObTmpF ASSERT_EQ(0, MTL(ObTenantTmpFileManager *)->page_cache_controller_.flush_priority_mgr_.get_file_size()); ASSERT_EQ(0, MTL(ObTenantTmpFileManager *)->page_cache_controller_.evict_mgr_.get_file_size()); - STORAGE_LOG(INFO, "test_big_file", K(io_info.disable_page_cache_)); + STORAGE_LOG(INFO, "test_big_file", K(io_info.disable_page_cache_), K(io_info.disable_block_cache_)); STORAGE_LOG(INFO, "io time", K(write_time), K(read_time)); } @@ -1084,6 +1123,7 @@ TEST_F(TestTmpFile, test_big_file_with_small_wbp) const int64_t wbp_mem_limit = SMALL_WBP_MEM_LIMIT; ObTmpFileIOInfo io_info; io_info.disable_page_cache_ = true; + io_info.disable_block_cache_ = true; test_big_file(write_size, wbp_mem_limit, io_info); } @@ -1092,7 +1132,7 @@ TEST_F(TestTmpFile, test_big_file_with_small_wbp) // 2. the 4th file writes and reads 3MB+1020KB data (will trigger flushing in the processing of writing) // 3. the first three files write and read 1MB data 3 times (total 3MB) // 4. each file read and write 12MB+4KB data -TEST_F(TestTmpFile, test_multi_file_single_thread_read_write) +void test_multi_file_single_thread_read_write(bool disable_block_cache) { int ret = OB_SUCCESS; const int64_t buf_size = 64 * 1024 * 1024; // 64MB @@ -1135,6 +1175,7 @@ TEST_F(TestTmpFile, test_multi_file_single_thread_read_write) ObTmpFileIOInfo io_info; io_info.io_desc_.set_wait_event(2); io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; + io_info.disable_block_cache_ = disable_block_cache; ObTmpFileIOHandle handle; int cmp = 0; @@ -1252,7 +1293,17 @@ TEST_F(TestTmpFile, test_multi_file_single_thread_read_write) ASSERT_EQ(0, MTL(ObTenantTmpFileManager *)->page_cache_controller_.flush_priority_mgr_.get_file_size()); ASSERT_EQ(0, MTL(ObTenantTmpFileManager *)->page_cache_controller_.evict_mgr_.get_file_size()); - LOG_INFO("test_multi_file_single_thread_read_write"); + LOG_INFO("test_multi_file_single_thread_read_write", K(disable_block_cache)); +} + +TEST_F(TestTmpFile, test_multi_file_single_thread_read_write) +{ + test_multi_file_single_thread_read_write(false); +} + +TEST_F(TestTmpFile, test_multi_file_single_thread_read_write_with_disable_block_cache) +{ + test_multi_file_single_thread_read_write(true); } TEST_F(TestTmpFile, test_single_file_multi_thread_read_write) @@ -1262,11 +1313,12 @@ TEST_F(TestTmpFile, test_single_file_multi_thread_read_write) const int64_t file_cnt = 1; const int64_t batch_size = 64 * 1024 * 1024; // 64MB const int64_t batch_num = 4; + const bool disable_block_cache = true; TestMultiTmpFileStress test(MTL_CTX()); int64_t dir = -1; ret = MTL(ObTenantTmpFileManager *)->alloc_dir(dir); ASSERT_EQ(OB_SUCCESS, ret); - ret = test.init(file_cnt, dir, read_thread_cnt, batch_size, batch_num); + ret = test.init(file_cnt, dir, read_thread_cnt, batch_size, batch_num, disable_block_cache); ASSERT_EQ(OB_SUCCESS, ret); int64_t io_time = ObTimeUtility::current_time(); test.start(); @@ -1285,11 +1337,36 @@ TEST_F(TestTmpFile, test_multi_file_multi_thread_read_write) const int64_t file_cnt = 4; const int64_t batch_size = 16 * 1024 * 1024; // 16MB const int64_t batch_num = 4; + const bool disable_block_cache = true; TestMultiTmpFileStress test(MTL_CTX()); int64_t dir = -1; ret = MTL(ObTenantTmpFileManager *)->alloc_dir(dir); ASSERT_EQ(OB_SUCCESS, ret); - ret = test.init(file_cnt, dir, read_thread_cnt, batch_size, batch_num); + ret = test.init(file_cnt, dir, read_thread_cnt, batch_size, batch_num, disable_block_cache); + ASSERT_EQ(OB_SUCCESS, ret); + int64_t io_time = ObTimeUtility::current_time(); + test.start(); + test.wait(); + io_time = ObTimeUtility::current_time() - io_time; + MTL(ObTenantTmpFileManager *)->page_cache_controller_.write_buffer_pool_.set_max_data_page_usage_ratio_(0.90); + STORAGE_LOG(INFO, "test_multi_file_multi_thread_read_write"); + STORAGE_LOG(INFO, "io time", K(io_time)); +} + +TEST_F(TestTmpFile, test_multi_file_multi_thread_read_write_with_block_cache) +{ + int ret = OB_SUCCESS; + MTL(ObTenantTmpFileManager *)->page_cache_controller_.write_buffer_pool_.set_max_data_page_usage_ratio_(0.99); + const int64_t read_thread_cnt = 4; + const int64_t file_cnt = 4; + const int64_t batch_size = 16 * 1024 * 1024; // 16MB + const int64_t batch_num = 4; + const bool disable_block_cache = false; + TestMultiTmpFileStress test(MTL_CTX()); + int64_t dir = -1; + ret = MTL(ObTenantTmpFileManager *)->alloc_dir(dir); + ASSERT_EQ(OB_SUCCESS, ret); + ret = test.init(file_cnt, dir, read_thread_cnt, batch_size, batch_num, disable_block_cache); ASSERT_EQ(OB_SUCCESS, ret); int64_t io_time = ObTimeUtility::current_time(); test.start(); @@ -1307,11 +1384,12 @@ TEST_F(TestTmpFile, test_more_files_more_threads_read_write) const int64_t file_cnt = 128; const int64_t batch_size = 3 * 1024 * 1024; const int64_t batch_num = 2; // total 128 * 3MB * 2 = 768MB + const bool disable_block_cache = true; TestMultiTmpFileStress test(MTL_CTX()); int64_t dir = -1; ret = MTL(ObTenantTmpFileManager *)->alloc_dir(dir); ASSERT_EQ(OB_SUCCESS, ret); - ret = test.init(file_cnt, dir, read_thread_cnt, batch_size, batch_num); + ret = test.init(file_cnt, dir, read_thread_cnt, batch_size, batch_num, disable_block_cache); ASSERT_EQ(OB_SUCCESS, ret); int64_t io_time = ObTimeUtility::current_time(); test.start(); @@ -1328,6 +1406,17 @@ TEST_F(TestTmpFile, test_big_file) const int64_t wbp_mem_limit = BIG_WBP_MEM_LIMIT; ObTmpFileIOInfo io_info; io_info.disable_page_cache_ = false; + io_info.disable_block_cache_ = false; + test_big_file(write_size, wbp_mem_limit, io_info); +} + +TEST_F(TestTmpFile, test_big_file_disable_block_cache) +{ + const int64_t write_size = 750 * 1024 * 1024; // write 750MB data + const int64_t wbp_mem_limit = BIG_WBP_MEM_LIMIT; + ObTmpFileIOInfo io_info; + io_info.disable_page_cache_ = false; + io_info.disable_block_cache_ = true; test_big_file(write_size, wbp_mem_limit, io_info); } @@ -1337,6 +1426,7 @@ TEST_F(TestTmpFile, test_big_file_disable_page_cache) const int64_t wbp_mem_limit = BIG_WBP_MEM_LIMIT; ObTmpFileIOInfo io_info; io_info.disable_page_cache_ = true; + io_info.disable_block_cache_ = true; test_big_file(write_size, wbp_mem_limit, io_info); } diff --git a/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp b/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp index 35145bc9a..76edc77ee 100644 --- a/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp +++ b/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp @@ -490,7 +490,8 @@ int ObSharedNothingTmpFile::inner_read_from_disk_(const int64_t expected_read_di int64_t actual_block_read_size = 0; ObTmpBlockValueHandle block_value_handle; - if (OB_SUCC(ObTmpBlockCache::get_instance().get_block(ObTmpBlockCacheKey(block_index, MTL_ID()), + if (!io_ctx.is_disable_block_cache() && + OB_SUCC(ObTmpBlockCache::get_instance().get_block(ObTmpBlockCacheKey(block_index, MTL_ID()), block_value_handle))) { LOG_DEBUG("hit block cache", K(block_index), K(fd_), K(io_ctx)); char *read_buf = io_ctx.get_todo_buffer(); @@ -511,7 +512,7 @@ int ObSharedNothingTmpFile::inner_read_from_disk_(const int64_t expected_read_di K(remain_read_size), K(read_size), K(actual_read_size), K(data_items[i]), K(io_ctx)); } - } else if (OB_ENTRY_NOT_EXIST != ret) { + } else if (OB_ENTRY_NOT_EXIST != ret && OB_SUCCESS != ret) { LOG_WARN("fail to get block", KR(ret), K(fd_), K(block_index)); } else { // not hit block cache, read page from disk. ret = OB_SUCCESS; @@ -1609,13 +1610,17 @@ int ObSharedNothingTmpFile::inner_truncate_(const int64_t truncate_offset, const } else if (truncate_offset > wbp_begin_offset) { const int64_t truncate_page_virtual_id = get_page_virtual_id_from_offset_(truncate_offset, true /*is_open_interval*/); + const int64_t truncate_offset_in_page = get_page_offset_from_file_or_block_offset_(truncate_offset); if (OB_FAIL(page_idx_cache_.binary_search(truncate_page_virtual_id, truncate_page_id))) { LOG_WARN("fail to find page index in array", KR(ret), K(fd_), K(truncate_page_virtual_id)); } else if (ObTmpFileGlobal::INVALID_PAGE_ID != truncate_page_id) { // the page index of truncate_offset is in the range of cached page index. // truncate all page indexes whose offset is smaller than truncate_offset. - if (OB_FAIL(page_idx_cache_.truncate(truncate_page_virtual_id+1))) { - LOG_WARN("fail to truncate page idx cache", KR(ret), K(fd_), K(truncate_page_virtual_id)); + const int64_t truncate_page_virtual_id_in_cache = truncate_offset_in_page == 0 ? + truncate_page_virtual_id + 1 : + truncate_page_virtual_id; + if (OB_FAIL(page_idx_cache_.truncate(truncate_page_virtual_id_in_cache))) { + LOG_WARN("fail to truncate page idx cache", KR(ret), K(fd_), K(truncate_page_virtual_id), K(truncate_page_virtual_id_in_cache)); } } else { // ObTmpFileGlobal::INVALID_PAGE_ID == truncate_page_id // the page index of truncate_offset is smaller than the smallest cached page index. @@ -1636,7 +1641,6 @@ int ObSharedNothingTmpFile::inner_truncate_(const int64_t truncate_offset, const // truncate the last page if (OB_SUCC(ret)) { - const int64_t truncate_offset_in_page = get_page_offset_from_file_or_block_offset_(truncate_offset); if (OB_UNLIKELY(truncate_page_virtual_id != begin_page_virtual_id_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected begin page virtual id", KR(ret), K(fd_), K(truncate_page_virtual_id), K(begin_page_virtual_id_)); diff --git a/src/storage/tmp_file/ob_tmp_file_io_ctx.cpp b/src/storage/tmp_file/ob_tmp_file_io_ctx.cpp index 19ed94c5b..bc846f7e7 100644 --- a/src/storage/tmp_file/ob_tmp_file_io_ctx.cpp +++ b/src/storage/tmp_file/ob_tmp_file_io_ctx.cpp @@ -32,6 +32,7 @@ ObTmpFileIOCtx::ObTmpFileIOCtx(): todo_size_(-1), read_offset_in_file_(-1), disable_page_cache_(false), + disable_block_cache_(false), io_flag_(), io_timeout_ms_(DEFAULT_IO_WAIT_TIME_MS), io_handles_(), @@ -50,7 +51,8 @@ int ObTmpFileIOCtx::init(const int64_t fd, const int64_t dir_id, const bool is_read, const common::ObIOFlag io_flag, const int64_t io_timeout_ms, - const bool disable_page_cache) + const bool disable_page_cache, + const bool disable_block_cache) { int ret = OB_SUCCESS; if (OB_UNLIKELY(is_inited_)) { @@ -73,6 +75,7 @@ int ObTmpFileIOCtx::init(const int64_t fd, const int64_t dir_id, io_flag_ = io_flag; io_timeout_ms_ = io_timeout_ms; disable_page_cache_ = disable_page_cache; + disable_block_cache_ = disable_block_cache; is_inited_ = true; } return ret; @@ -103,6 +106,7 @@ void ObTmpFileIOCtx::reset() fd_ = ObTmpFileGlobal::INVALID_TMP_FILE_FD; dir_id_ = ObTmpFileGlobal::INVALID_TMP_FILE_DIR_ID; disable_page_cache_ = false; + disable_block_cache_ = false; io_flag_.reset(); io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; } diff --git a/src/storage/tmp_file/ob_tmp_file_io_ctx.h b/src/storage/tmp_file/ob_tmp_file_io_ctx.h index bd985c8f1..6ea971511 100644 --- a/src/storage/tmp_file/ob_tmp_file_io_ctx.h +++ b/src/storage/tmp_file/ob_tmp_file_io_ctx.h @@ -31,7 +31,8 @@ public: const bool is_read, const common::ObIOFlag io_flag, const int64_t io_timeout_ms, - const bool disable_page_cache); + const bool disable_page_cache, + const bool disable_block_cache); void reuse(); void reset(); bool is_valid() const; @@ -58,6 +59,7 @@ public: OB_INLINE int64_t get_read_offset_in_file() const { return read_offset_in_file_; } OB_INLINE void set_read_offset_in_file(const int64_t offset) { read_offset_in_file_ = offset; } OB_INLINE bool is_disable_page_cache() const { return disable_page_cache_; } + OB_INLINE bool is_disable_block_cache() const { return disable_block_cache_; } OB_INLINE common::ObIOFlag get_io_flag() const { return io_flag_; } OB_INLINE int64_t get_io_timeout_ms() const { return io_timeout_ms_; } @@ -66,6 +68,7 @@ public: K(buf_size_), K(done_size_), K(todo_size_), K(read_offset_in_file_), K(disable_page_cache_), + K(disable_block_cache_), K(io_flag_), K(io_timeout_ms_)); public: @@ -157,6 +160,7 @@ private: int64_t todo_size_; int64_t read_offset_in_file_; bool disable_page_cache_; + bool disable_block_cache_; // only used in ut, to control whether read data from block cache common::ObIOFlag io_flag_; int64_t io_timeout_ms_; common::ObSEArray io_handles_; diff --git a/src/storage/tmp_file/ob_tmp_file_io_define.cpp b/src/storage/tmp_file/ob_tmp_file_io_define.cpp index de904c1b9..6aab99c6e 100644 --- a/src/storage/tmp_file/ob_tmp_file_io_define.cpp +++ b/src/storage/tmp_file/ob_tmp_file_io_define.cpp @@ -25,7 +25,7 @@ namespace tmp_file ObTmpFileIOInfo::ObTmpFileIOInfo() : fd_(0), dir_id_(0), buf_(nullptr), size_(0), - disable_page_cache_(false), + disable_page_cache_(false), disable_block_cache_(false), io_desc_(), io_timeout_ms_(DEFAULT_IO_WAIT_TIME_MS) {} @@ -43,6 +43,7 @@ void ObTmpFileIOInfo::reset() buf_ = nullptr; io_desc_.reset(); disable_page_cache_ = false; + disable_block_cache_ = false; } bool ObTmpFileIOInfo::is_valid() const @@ -94,7 +95,7 @@ int ObTmpFileIOHandle::init_write(const ObTmpFileIOInfo &io_info, ObTmpFileHandl ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(io_info)); } else if (OB_FAIL(ctx_.init(io_info.fd_, io_info.dir_id_, false /*is_read*/, io_info.io_desc_, - io_info.io_timeout_ms_, io_info.disable_page_cache_))) { + io_info.io_timeout_ms_, io_info.disable_page_cache_, io_info.disable_block_cache_))) { LOG_WARN("failed to init io handle context", KR(ret), K(io_info)); } else if (OB_FAIL(ctx_.prepare_write(io_info.buf_, io_info.size_))) { LOG_WARN("fail to prepare write context", KR(ret), KPC(this)); @@ -122,7 +123,7 @@ int ObTmpFileIOHandle::init_pread(const ObTmpFileIOInfo &io_info, const int64_t ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(read_offset)); } else if (OB_FAIL(ctx_.init(io_info.fd_, io_info.dir_id_, true /*is_read*/, io_info.io_desc_, - io_info.io_timeout_ms_, io_info.disable_page_cache_))) { + io_info.io_timeout_ms_, io_info.disable_page_cache_, io_info.disable_block_cache_))) { LOG_WARN("failed to init io handle context", KR(ret), K(io_info)); } else if (OB_FAIL(ctx_.prepare_read(io_info.buf_, MIN(io_info.size_, ObTmpFileGlobal::TMP_FILE_READ_BATCH_SIZE), read_offset))) { LOG_WARN("fail to prepare read context", KR(ret), KPC(this), K(read_offset)); @@ -148,7 +149,7 @@ int ObTmpFileIOHandle::init_read(const ObTmpFileIOInfo &io_info, ObTmpFileHandle ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(io_info)); } else if (OB_FAIL(ctx_.init(io_info.fd_, io_info.dir_id_, true /*is_read*/, io_info.io_desc_, - io_info.io_timeout_ms_, io_info.disable_page_cache_))) { + io_info.io_timeout_ms_, io_info.disable_page_cache_, io_info.disable_block_cache_))) { LOG_WARN("failed to init io handle context", KR(ret), K(io_info)); } else if (OB_FAIL(ctx_.prepare_read(io_info.buf_, MIN(io_info.size_, ObTmpFileGlobal::TMP_FILE_READ_BATCH_SIZE)))) { LOG_WARN("fail to prepare read context", KR(ret), KPC(this)); diff --git a/src/storage/tmp_file/ob_tmp_file_io_define.h b/src/storage/tmp_file/ob_tmp_file_io_define.h index e17daecac..4ab16df5b 100644 --- a/src/storage/tmp_file/ob_tmp_file_io_define.h +++ b/src/storage/tmp_file/ob_tmp_file_io_define.h @@ -27,13 +27,14 @@ struct ObTmpFileIOInfo final void reset(); bool is_valid() const; TO_STRING_KV(K(fd_), K(dir_id_), KP(buf_), K(size_), K(disable_page_cache_), - K(io_timeout_ms_), K(io_desc_)); + K(disable_block_cache_), K(io_timeout_ms_), K(io_desc_)); int64_t fd_; int64_t dir_id_; char *buf_; int64_t size_; bool disable_page_cache_; + bool disable_block_cache_; // only used in ut, to control whether read data from block cache common::ObIOFlag io_desc_; int64_t io_timeout_ms_; }; diff --git a/src/storage/tmp_file/ob_tmp_file_write_buffer_index_cache.cpp b/src/storage/tmp_file/ob_tmp_file_write_buffer_index_cache.cpp index 091f83b43..680e5e580 100644 --- a/src/storage/tmp_file/ob_tmp_file_write_buffer_index_cache.cpp +++ b/src/storage/tmp_file/ob_tmp_file_write_buffer_index_cache.cpp @@ -292,10 +292,7 @@ int ObTmpFileWBPIndexCache::binary_search(const int64_t target_page_virtual_id, if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); - } else if (OB_UNLIKELY(is_empty())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("array is empty", KR(ret), K(fd_), K(size_)); - } else { + } else if (OB_LIKELY(!is_empty())) { int64_t left_pos = left_; int64_t right_pos = get_logic_tail_(); ObTmpFilePageIndexBucket *target_bucket = nullptr; From 445002cafa0a14a1d148b70fefe17096e235f4bf Mon Sep 17 00:00:00 2001 From: swjtu-wenxiang Date: Mon, 19 Aug 2024 06:45:07 +0000 Subject: [PATCH 095/249] [CP] [FEAT MERGE]424 PL dev features Co-authored-by: seuwebber Co-authored-by: hanr881 <1741282579@qq.com> --- .../src/lib/mysqlclient/ob_isql_connection.h | 10 +- .../lib/mysqlclient/ob_mysql_connection.cpp | 83 +- .../src/lib/mysqlclient/ob_mysql_connection.h | 28 +- .../mysqlclient/ob_mysql_prepared_param.cpp | 11 +- .../lib/mysqlclient/ob_mysql_prepared_param.h | 3 +- .../mysqlclient/ob_mysql_prepared_result.cpp | 7 +- .../mysqlclient/ob_mysql_prepared_result.h | 4 +- .../ob_mysql_prepared_statement.cpp | 971 +++++++++++++++--- .../mysqlclient/ob_mysql_prepared_statement.h | 102 +- .../src/lib/mysqlclient/ob_mysql_proxy.cpp | 17 +- .../src/lib/mysqlclient/ob_mysql_proxy.h | 11 +- .../lib/mysqlclient/ob_mysql_statement.cpp | 2 +- .../src/lib/mysqlclient/ob_mysql_statement.h | 2 +- src/observer/ob_inner_sql_connection.cpp | 5 +- src/observer/ob_inner_sql_connection.h | 3 +- src/pl/dblink/ob_pl_dblink_guard.cpp | 85 +- src/pl/dblink/ob_pl_dblink_guard.h | 3 + src/pl/ob_pl.cpp | 95 +- src/pl/ob_pl.h | 39 +- src/pl/ob_pl_code_generator.cpp | 52 + src/pl/ob_pl_code_generator.h | 2 + src/pl/ob_pl_compile.cpp | 6 +- src/pl/ob_pl_exception_handling.cpp | 68 +- src/pl/ob_pl_exception_handling.h | 2 + src/pl/ob_pl_package_manager.cpp | 65 +- src/pl/ob_pl_package_manager.h | 4 + src/pl/ob_pl_resolver.cpp | 331 +++--- src/pl/ob_pl_resolver.h | 7 +- src/pl/ob_pl_type.cpp | 3 +- src/pl/ob_pl_type.h | 1 + src/pl/ob_pl_user_type.cpp | 14 +- src/pl/ob_pl_user_type.h | 2 + .../ob_inner_table_schema.21551_21600.cpp | 50 + src/share/inner_table/ob_inner_table_schema.h | 10 +- .../ob_inner_table_schema_constants.h | 2 + .../inner_table/ob_inner_table_schema_def.py | 16 +- src/share/inner_table/table_id_to_name | 1 + src/share/schema/ob_routine_info.h | 2 +- src/sql/ob_spi.cpp | 33 +- src/sql/ob_spi.h | 2 +- src/sql/printer/ob_raw_expr_printer.cpp | 13 +- .../privilege_check/ob_privilege_check.cpp | 5 +- .../expr/ob_raw_expr_resolver_impl.cpp | 2 +- .../r/mysql/information_schema.result | 6 + .../r/mysql/desc_sys_views_in_mysql.result | 9 + .../r/mysql/desc_sys_views_in_sys.result | 9 + .../r/mysql/inner_table_overall.result | 1 + .../inner_table/r/mysql/views.result | 1 + 48 files changed, 1699 insertions(+), 501 deletions(-) diff --git a/deps/oblib/src/lib/mysqlclient/ob_isql_connection.h b/deps/oblib/src/lib/mysqlclient/ob_isql_connection.h index 42fc7c92e..7d27a40b4 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_isql_connection.h +++ b/deps/oblib/src/lib/mysqlclient/ob_isql_connection.h @@ -135,8 +135,9 @@ public: const share::schema::ObRoutineInfo &routine_info, const common::ObIArray &udts, const ObTimeZoneInfo *tz_info, - ObObj *result) = 0; - virtual int prepare(const char *sql) { + ObObj *result, + bool is_sql) = 0; + virtual int prepare(const char *sql, int64_t param_count, ObIAllocator *allocator = NULL) { UNUSED(sql); return OB_NOT_SUPPORTED; } @@ -144,9 +145,10 @@ public: void *param, int64_t param_size, int32_t datatype, - int32_t &indicator) + int32_t &indicator, + bool is_out_param) { - UNUSEDx(position, param, param_size, datatype); + UNUSEDx(position, param, param_size, datatype, indicator, is_out_param); return OB_NOT_SUPPORTED; } virtual int bind_array_type_by_pos(uint64_t position, diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_connection.cpp b/deps/oblib/src/lib/mysqlclient/ob_mysql_connection.cpp index 0171f0885..68364b822 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_connection.cpp +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_connection.cpp @@ -21,6 +21,7 @@ #include "lib/string/ob_sql_string.h" #include "lib/mysqlclient/ob_mysql_read_context.h" #include "lib/mysqlclient/ob_dblink_error_trans.h" +#include "share/schema/ob_routine_info.h" namespace oceanbase { @@ -95,7 +96,7 @@ void ObMySQLConnection::reset() int ObMySQLConnection::prepare_statement(ObMySQLPreparedStatement &stmt, const char *sql) { int ret = OB_SUCCESS; - if (OB_FAIL(stmt.init(*this, sql))) { + if (OB_FAIL(stmt.init(*this, sql, 0))) { LOG_WARN("fail to init prepared statement", K(ret)); } return ret; @@ -518,19 +519,25 @@ int ObMySQLConnection::execute_proc(const uint64_t tenant_id, const share::schema::ObRoutineInfo &routine_info, const common::ObIArray &udts, const ObTimeZoneInfo *tz_info, - ObObj *result) + ObObj *result, + bool is_sql) { int ret = OB_SUCCESS; if (OB_UNLIKELY(closed_)) { ret = OB_NOT_INIT; LOG_WARN("connection not established. call connect first", K(ret)); } else { - ObMySQLProcStatement stmt; - if (OB_FAIL(create_statement(stmt, tenant_id, sql.ptr()))) { + int64_t real_param_cnt = routine_info.is_function() ? 1 : 0; + for (int64_t i = 0; i < params.count(); i++) { + if (!params.at(i).is_pl_mock_default_param()) { + real_param_cnt++; + } + } + if (OB_FAIL(prepare_proc_stmt(sql.ptr(), real_param_cnt, &allocator))) { LOG_WARN("create statement failed", K(sql), K(ret)); - } else if (OB_FAIL(stmt.execute_proc(allocator, params, routine_info, tz_info))) { + } else if (OB_FAIL(proc_stmt_.execute_proc(allocator, params, routine_info, tz_info, result, is_sql))) { LOG_WARN("statement execute update failed", K(sql), K(ret)); - } else if (OB_FAIL(stmt.close())) { + } else if (OB_FAIL(proc_stmt_.close())) { LOG_WARN("fail to close stmt", K(ret)); } } @@ -692,6 +699,70 @@ int ObMySQLConnection::connect_dblink(const bool use_ssl, int64_t sql_request_le return ret; } +int ObMySQLConnection::prepare(const char *sql, int64_t param_count, ObIAllocator *allocator) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(prepare_proc_stmt(sql, param_count, allocator))) { + LOG_WARN("prepare proc stmt failed", K(ret), KCSTRING(sql)); + } + return ret; +} + +int ObMySQLConnection::prepare_proc_stmt(const char *sql, int64_t param_count, ObIAllocator *allocator) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(allocator)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("allocator is NULL", K(ret)); + } else if (FALSE_IT(proc_stmt_.set_allocator(allocator))) { + } else if (OB_UNLIKELY(closed_)) { + ret = OB_NOT_INIT; + LOG_WARN("connection not established. call connect first", K(ret)); + } else if (OB_FAIL(create_statement(proc_stmt_, OB_INVALID_TENANT_ID, sql, param_count))) { + LOG_WARN("create statement failed", K(ret), KCSTRING(sql)); + } + return ret; +} + +int ObMySQLConnection::bind_basic_type_by_pos(uint64_t position, + void *param, + int64_t param_size, + int32_t datatype, + int32_t &indicator, + bool is_out_param) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(proc_stmt_.bind_basic_type_by_pos(position - 1, param, param_size, datatype, indicator, is_out_param))) { + LOG_WARN("bind basic type failed", K(ret), K(position), K(param_size), K(datatype)); + } + return ret; +} + +int ObMySQLConnection::bind_array_type_by_pos(uint64_t position, + void *array, + int32_t *indicators, + int64_t ele_size, + int32_t ele_datatype, + uint64_t array_size, + uint32_t *out_valid_array_size) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(proc_stmt_.bind_array_type_by_pos(position - 1, array, indicators, ele_size, + ele_datatype, array_size, out_valid_array_size))) { + LOG_WARN("bind array type failed", K(ret), K(position), K(ele_size), K(ele_datatype), K(array_size)); + } + return ret; +} + +int ObMySQLConnection::execute_proc() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(proc_stmt_.execute_proc())) { + LOG_WARN("failed to execute proc", K(ret)); + } + return ret; +} + } // end namespace sqlclient } // end namespace common } // end namespace oceanbase diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_connection.h b/deps/oblib/src/lib/mysqlclient/ob_mysql_connection.h index f8734b615..571f77975 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_connection.h +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_connection.h @@ -19,6 +19,7 @@ #include "lib/container/ob_se_array.h" #include "lib/allocator/ob_malloc.h" #include "lib/net/ob_addr.h" +#include "lib/mysqlclient/ob_mysql_prepared_statement.h" namespace oceanbase { namespace common @@ -56,7 +57,7 @@ public: virtual bool is_closed() const; // use user provided the statement template - int create_statement(T &stmt, const uint64_t tenant_id, const char *sql); + int create_statement(T &stmt, const uint64_t tenant_id, const char *sql, int64_t param_count = 0); int prepare_statement(ObMySQLPreparedStatement &stmt, const char *sql); int escape(const char *from, const int64_t from_size, char *to, const int64_t to_size, int64_t &out_size); @@ -91,7 +92,8 @@ public: const share::schema::ObRoutineInfo &routine_info, const common::ObIArray &udts, const ObTimeZoneInfo *tz_info, - ObObj *result) override; + ObObj *result, + bool is_sql) override; virtual int start_transaction(const uint64_t &tenant_id, bool with_snap_shot = false) override; virtual int rollback() override; virtual int commit() override; @@ -121,7 +123,22 @@ public: // dblink. virtual int connect_dblink(const bool use_ssl, int64_t sql_request_level); - + int prepare(const char *sql, int64_t param_count, ObIAllocator *allocator); + int prepare_proc_stmt(const char *sql, int64_t param_count, ObIAllocator *allocator); + int bind_basic_type_by_pos(uint64_t position, + void *param, + int64_t param_size, + int32_t datatype, + int32_t &indicator, + bool is_out_param); + int bind_array_type_by_pos(uint64_t position, + void *array, + int32_t *indicators, + int64_t ele_size, + int32_t ele_datatype, + uint64_t array_size, + uint32_t *out_valid_array_size); + int execute_proc(); private: int switch_tenant(const uint64_t tenant_id); int reset_read_consistency(); @@ -145,6 +162,7 @@ private: const char *db_name_; uint64_t tenant_id_; int64_t read_consistency_; + ObMySQLProcStatement proc_stmt_; DISALLOW_COPY_AND_ASSIGN(ObMySQLConnection); }; inline bool ObMySQLConnection::is_busy() const @@ -181,14 +199,14 @@ inline int64_t ObMySQLConnection::connection_version() const } template -int ObMySQLConnection::create_statement(T &stmt, const uint64_t tenant_id, const char *sql) +int ObMySQLConnection::create_statement(T &stmt, const uint64_t tenant_id, const char *sql, int64_t param_count) { int ret = OB_SUCCESS; if (OB_FAIL(switch_tenant(tenant_id))) { _OB_LOG(WARN, "switch tenant failed, tenant_id=%ld, ret=%d", tenant_id, ret); } else if (OB_FAIL(reset_read_consistency())) { _OB_LOG(WARN, "fail to set read consistency, ret=%d", ret); - } else if (OB_FAIL(stmt.init(*this, sql))) { + } else if (OB_FAIL(stmt.init(*this, sql, param_count))) { _OB_LOG(WARN, "fail to init statement, ret=%d", ret); } return ret; diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_param.cpp b/deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_param.cpp index 6746b005c..6f2c101a0 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_param.cpp +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_param.cpp @@ -49,10 +49,11 @@ int ObMySQLPreparedParam::init() ret = OB_ERR_SQL_CLIENT; } else if (0 == param_count_) { // insert or replace that do not produce result sets - } else if (OB_ISNULL(bind_ = reinterpret_cast(alloc_.alloc(sizeof(MYSQL_BIND) * param_count_)))) { + } else if (OB_ISNULL(bind_ = reinterpret_cast(alloc_->alloc(sizeof(MYSQL_BIND) * param_count_)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_ERROR("out of memory, alloc mem for mysql_bind error", K(ret)); } else { + MEMSET(bind_, 0, sizeof(MYSQL_BIND) * param_count_); LOG_DEBUG("statement field", K(param_count_)); } return ret; @@ -75,8 +76,10 @@ int ObMySQLPreparedParam::bind_param() void ObMySQLPreparedParam::close() { if (OB_LIKELY(NULL != bind_)) { - alloc_.free(bind_); + alloc_->free(bind_); bind_ = NULL; + param_count_ = 0; + alloc_ = NULL; } } @@ -93,6 +96,10 @@ int ObMySQLPreparedParam::bind_param(ObBindParam ¶m) bind_[param.col_idx_].length = ¶m.length_; bind_[param.col_idx_].is_null = ¶m.is_null_; bind_[param.col_idx_].is_unsigned = param.is_unsigned_; + bind_[param.col_idx_].long_data_used = 0; +#ifdef OB_BUILD_ORACLE_PL + bind_[param.col_idx_].mysql = stmt_.get_conn_handler(); +#endif } else { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid index", K(param), K(param_count_)); diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_param.h b/deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_param.h index 43e8919f6..3a9f11175 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_param.h +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_param.h @@ -29,6 +29,7 @@ class ObBindParam; class ObMySQLPreparedStatement; class ObMySQLPreparedParam { +friend ObMySQLPreparedStatement; public: explicit ObMySQLPreparedParam(ObMySQLPreparedStatement &stmt); ~ObMySQLPreparedParam(); @@ -40,7 +41,7 @@ public: private: ObMySQLPreparedStatement &stmt_; - common::ObIAllocator &alloc_; + common::ObIAllocator *alloc_; int64_t param_count_; MYSQL_BIND *bind_; }; diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_result.cpp b/deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_result.cpp index 727a34dfb..0a8e904eb 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_result.cpp +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_result.cpp @@ -50,11 +50,12 @@ int ObMySQLPreparedResult::init() ret = OB_ERR_SQL_CLIENT; } else if (result_column_count_ == 0) { // insert or replace that do not produce result sets - } else if (OB_ISNULL(bind_ = reinterpret_cast(alloc_.alloc(sizeof(MYSQL_BIND) * + } else if (OB_ISNULL(bind_ = reinterpret_cast(alloc_->alloc(sizeof(MYSQL_BIND) * result_column_count_)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_ERROR("out of memory, alloc mem for mysql bind error", K(ret)); } else { + MEMSET(bind_, 0, sizeof(MYSQL_BIND) * result_column_count_); LOG_TRACE("statemen field count = ", K(result_column_count_)); } return ret; @@ -77,8 +78,9 @@ int ObMySQLPreparedResult::bind_result_param() void ObMySQLPreparedResult::close() { result_column_count_ = 0; - alloc_.free(bind_); + alloc_->free(bind_); bind_ = NULL; + alloc_ = NULL; } int ObMySQLPreparedResult::next() @@ -120,6 +122,7 @@ int ObMySQLPreparedResult::bind_result(ObBindParam ¶m) bind_[param.col_idx_].length = ¶m.length_; bind_[param.col_idx_].is_null = ¶m.is_null_; bind_[param.col_idx_].is_unsigned = param.is_unsigned_; + bind_[param.col_idx_].error = &bind_[param.col_idx_].error_value; } else { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid index", K(param), K(result_column_count_)); diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_result.h b/deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_result.h index 83cf1aeee..f5c7f0e37 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_result.h +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_result.h @@ -29,6 +29,7 @@ class ObBindParam; class ObMySQLPreparedStatement; class ObMySQLPreparedResult { +friend ObMySQLPreparedStatement; public: explicit ObMySQLPreparedResult(ObMySQLPreparedStatement &stmt); ~ObMySQLPreparedResult(); @@ -50,9 +51,10 @@ public: int64_t get_result_column_count() const { return result_column_count_; } int bind_result(ObBindParam ¶m); + MYSQL_BIND *&get_bind() { return bind_; } private: ObMySQLPreparedStatement &stmt_; - common::ObIAllocator &alloc_; + common::ObIAllocator *alloc_; int64_t result_column_count_; MYSQL_BIND *bind_; }; diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_statement.cpp b/deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_statement.cpp index dd9241b36..50cdb5161 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_statement.cpp +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_statement.cpp @@ -19,6 +19,7 @@ #include "lib/mysqlclient/ob_mysql_prepared_result.h" #include "lib/mysqlclient/ob_mysql_prepared_statement.h" #include "share/schema/ob_routine_info.h" +#include "lib/mysqlclient/ob_dblink_error_trans.h" namespace oceanbase { @@ -90,7 +91,6 @@ int ObBindParamEncode::encode_null(ENCODE_FUNC_ARG_DECL) UNUSEDx(is_output_param, tz_info, allocator); int ret = OB_SUCCESS; const ObObjType obj_type = param.get_type(); - enum_field_types buffer_type = static_cast(ob_type_to_mysql_type[obj_type]); bind_param.col_idx_ = col_idx; bind_param.buffer_type_ = buffer_type; bind_param.is_null_ = 1; @@ -102,7 +102,6 @@ int ObBindParamEncode::encode_int(ENCODE_FUNC_ARG_DECL) UNUSEDx(is_output_param, tz_info, allocator); int ret = OB_SUCCESS; const ObObjType obj_type = param.get_type(); - enum_field_types buffer_type = static_cast(ob_type_to_mysql_type[obj_type]); bind_param.col_idx_ = col_idx; bind_param.buffer_type_ = buffer_type; bind_param.buffer_ = &(param.v_.int64_); @@ -114,7 +113,7 @@ int ObBindParamEncode::encode_uint(ENCODE_FUNC_ARG_DECL) { UNUSEDx(is_output_param, tz_info, allocator); int ret = OB_SUCCESS; - if (OB_FAIL(encode_int(col_idx, is_output_param, tz_info, param, bind_param, allocator))) { + if (OB_FAIL(encode_int(col_idx, is_output_param, tz_info, param, bind_param, allocator, buffer_type))) { LOG_WARN("fail to encode", K(ret)); } else { bind_param.is_unsigned_ = 1; @@ -127,7 +126,6 @@ int ObBindParamEncode::encode_float(ENCODE_FUNC_ARG_DECL) UNUSEDx(is_output_param, tz_info, allocator); int ret = OB_SUCCESS; const ObObjType obj_type = param.get_type(); - enum_field_types buffer_type = static_cast(ob_type_to_mysql_type[obj_type]); bind_param.col_idx_ = col_idx; bind_param.buffer_type_ = buffer_type; bind_param.buffer_ = &(param.v_.float_); @@ -139,7 +137,7 @@ int ObBindParamEncode::encode_ufloat(ENCODE_FUNC_ARG_DECL) { UNUSEDx(is_output_param, tz_info, allocator); int ret = OB_SUCCESS; - if (OB_FAIL(encode_float(col_idx, is_output_param, tz_info, param, bind_param, allocator))) { + if (OB_FAIL(encode_float(col_idx, is_output_param, tz_info, param, bind_param, allocator, buffer_type))) { LOG_WARN("fail to encode", K(ret)); } else { bind_param.is_unsigned_ = 1; @@ -152,7 +150,6 @@ int ObBindParamEncode::encode_double(ENCODE_FUNC_ARG_DECL) UNUSEDx(is_output_param, tz_info, allocator); int ret = OB_SUCCESS; const ObObjType obj_type = param.get_type(); - enum_field_types buffer_type = static_cast(ob_type_to_mysql_type[obj_type]); bind_param.col_idx_ = col_idx; bind_param.buffer_type_ = buffer_type; bind_param.buffer_ = &(param.v_.double_); @@ -164,7 +161,7 @@ int ObBindParamEncode::encode_udouble(ENCODE_FUNC_ARG_DECL) { UNUSEDx(is_output_param, tz_info, allocator); int ret = OB_SUCCESS; - if (OB_FAIL(encode_double(col_idx, is_output_param, tz_info, param, bind_param, allocator))) { + if (OB_FAIL(encode_double(col_idx, is_output_param, tz_info, param, bind_param, allocator, buffer_type))) { LOG_WARN("fail to encode", K(ret)); } else { bind_param.is_unsigned_ = 1; @@ -181,7 +178,6 @@ int ObBindParamEncode::encode_number(ENCODE_FUNC_ARG_DECL) number::ObNumber num; const int64_t buf_len = OB_CAST_TO_VARCHAR_MAX_LENGTH; const ObObjType obj_type = param.get_type(); - enum_field_types buffer_type = static_cast(ob_type_to_mysql_type[obj_type]); bind_param.col_idx_ = col_idx; bind_param.buffer_type_ = buffer_type; if (OB_ISNULL(buf = reinterpret_cast(allocator.alloc(buf_len)))) { @@ -203,7 +199,7 @@ int ObBindParamEncode::encode_unumber(ENCODE_FUNC_ARG_DECL) { UNUSEDx(is_output_param, tz_info); int ret = OB_SUCCESS; - if (OB_FAIL(encode_number(col_idx, is_output_param, tz_info, param, bind_param, allocator))) { + if (OB_FAIL(encode_number(col_idx, is_output_param, tz_info, param, bind_param, allocator, buffer_type))) { LOG_WARN("fail to encode", K(ret)); } else { bind_param.is_unsigned_ = 1; @@ -218,23 +214,26 @@ int ObBindParamEncode::encode_datetime(ENCODE_FUNC_ARG_DECL) ObTime ob_time; MYSQL_TIME *tm = nullptr; const ObObjType obj_type = param.get_type(); - enum_field_types buffer_type = static_cast(ob_type_to_mysql_type[obj_type]); bind_param.col_idx_ = col_idx; bind_param.buffer_type_ = buffer_type; + const ObTimeZoneInfo *tmp_tz = &tz_info; + if (obj_type == ObObjType::ObDateTimeType) { + tmp_tz = NULL; + } if (OB_ISNULL(tm = reinterpret_cast(allocator.alloc(sizeof(MYSQL_TIME))))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to alloc memory", K(ret)); - } else if (OB_FAIL(ObTimeConverter::datetime_to_ob_time(param.get_datetime(), &tz_info, ob_time))) { + } else if (OB_FAIL(ObTimeConverter::datetime_to_ob_time(param.get_datetime(), tmp_tz, ob_time))) { LOG_WARN("convert usec ", K(ret)); } else { MEMSET(tm, 0, sizeof(MYSQL_TIME)); tm->year = ob_time.parts_[DT_YEAR]; tm->month = ob_time.parts_[DT_MON]; tm->day = ob_time.parts_[DT_MDAY]; - tm->hour= ob_time.parts_[DT_HOUR]; - tm->minute= ob_time.parts_[DT_MIN]; - tm->second= ob_time.parts_[DT_SEC]; - tm->second_part= ob_time.parts_[DT_USEC]; + tm->hour = ob_time.parts_[DT_HOUR]; + tm->minute = ob_time.parts_[DT_MIN]; + tm->second = ob_time.parts_[DT_SEC]; + tm->second_part = ob_time.parts_[DT_USEC]; tm->neg = DT_MODE_NEG & ob_time.mode_; bind_param.buffer_ = tm; bind_param.buffer_len_ = sizeof(MYSQL_TIME); @@ -242,6 +241,71 @@ int ObBindParamEncode::encode_datetime(ENCODE_FUNC_ARG_DECL) return ret; } +int ObBindParamEncode::encode_timestamp(ENCODE_FUNC_ARG_DECL) +{ + int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL + ret = OB_NOT_SUPPORTED; +#else + const ObObjType obj_type = param.get_type(); + const bool is_tz = (ObTimestampTZType == obj_type); + ORACLE_TIME *ora_time = NULL; + int64_t len = sizeof(ORACLE_TIME); + bind_param.col_idx_ = col_idx; + bind_param.buffer_type_ = buffer_type; + const ObOTimestampData &ot_data = param.get_otimestamp_value(); + ObTime ob_time(DT_TYPE_ORACLE_TIMESTAMP); + if (OB_FAIL(ObTimeConverter::otimestamp_to_ob_time(obj_type, ot_data, &tz_info, + ob_time, false))) { + LOG_WARN("failed to convert timestamp to ob time", K(ret)); + } else if (!ObTimeConverter::valid_oracle_year(ob_time)) { + ret = OB_ERR_DATETIME_INTERVAL_INTERNAL_ERROR; + LOG_WARN("invalid oracle timestamp", K(ret), K(ob_time)); + } else if (OB_ISNULL(ora_time = reinterpret_cast(allocator.alloc(len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc memory", K(ret)); + } else { + MEMSET(ora_time, 0, sizeof(ORACLE_TIME)); + const int32_t unsigned_year = ob_time.parts_[DT_YEAR] >= 0 ? ob_time.parts_[DT_YEAR] : (0 - ob_time.parts_[DT_YEAR]); + int32_t century = static_cast(unsigned_year / YEARS_PER_CENTURY * (ob_time.parts_[DT_YEAR] >= 0 ? 1 : -1)); + int32_t decade = static_cast(unsigned_year % YEARS_PER_CENTURY); + if (0 == century) { + decade *= static_cast(ob_time.parts_[DT_YEAR] >= 0 ? 1 : -1); + } + ora_time->century = century; + ora_time->year = decade; + ora_time->month = ob_time.parts_[DT_MON]; + ora_time->day = ob_time.parts_[DT_MDAY]; + ora_time->hour = ob_time.parts_[DT_HOUR]; + ora_time->minute = ob_time.parts_[DT_MIN]; + ora_time->second = ob_time.parts_[DT_SEC]; + ora_time->second_part = ob_time.parts_[DT_USEC]; + ora_time->scale = param.get_scale(); + if (is_tz) { + if (!ob_time.get_tz_name_str().empty()) { + ora_time->tz_name = ob_time.get_tz_name_str().ptr(); + } + if (!ob_time.get_tzd_abbr_str().empty()) { + ora_time->tz_abbr = ob_time.get_tzd_abbr_str().ptr(); + } + const int32_t unsigned_offset = (ob_time.parts_[DT_OFFSET_MIN] >= 0 ? ob_time.parts_[DT_OFFSET_MIN] : (0 - ob_time.parts_[DT_OFFSET_MIN])); + int32_t offset_hour = static_cast(unsigned_offset / MINS_PER_HOUR * (ob_time.parts_[DT_OFFSET_MIN] >= 0 ? 1 : -1)); + int32_t offset_minute = static_cast(unsigned_offset % MINS_PER_HOUR); + if (0 == offset_hour) { + offset_minute *= static_cast(ob_time.parts_[DT_OFFSET_MIN] >= 0 ? 1 : -1); + } + ora_time->offset_hour = offset_hour; + ora_time->offset_minute = offset_minute; + } + + bind_param.buffer_ = ora_time; + bind_param.buffer_len_ = len; + bind_param.length_ = len; + } +#endif + return ret; +} + int ObBindParamEncode::encode_date(ENCODE_FUNC_ARG_DECL) { UNUSEDx(is_output_param); @@ -249,7 +313,6 @@ int ObBindParamEncode::encode_date(ENCODE_FUNC_ARG_DECL) ObTime ob_time; MYSQL_TIME *tm = nullptr; const ObObjType obj_type = param.get_type(); - enum_field_types buffer_type = static_cast(ob_type_to_mysql_type[obj_type]); bind_param.col_idx_ = col_idx; bind_param.buffer_type_ = buffer_type; if (OB_ISNULL(tm = reinterpret_cast(allocator.alloc(sizeof(MYSQL_TIME))))) { @@ -276,7 +339,6 @@ int ObBindParamEncode::encode_time(ENCODE_FUNC_ARG_DECL) ObTime ob_time; MYSQL_TIME *tm = nullptr; const ObObjType obj_type = param.get_type(); - enum_field_types buffer_type = static_cast(ob_type_to_mysql_type[obj_type]); bind_param.col_idx_ = col_idx; bind_param.buffer_type_ = buffer_type; if (OB_ISNULL(tm = reinterpret_cast(allocator.alloc(sizeof(MYSQL_TIME))))) { @@ -304,7 +366,6 @@ int ObBindParamEncode::encode_year(ENCODE_FUNC_ARG_DECL) int ret = OB_SUCCESS; int64_t *year = nullptr; const ObObjType obj_type = param.get_type(); - enum_field_types buffer_type = static_cast(ob_type_to_mysql_type[obj_type]); bind_param.col_idx_ = col_idx; bind_param.buffer_type_ = buffer_type; if (OB_ISNULL(year = reinterpret_cast(allocator.alloc(sizeof(int64_t))))) { @@ -324,28 +385,33 @@ int ObBindParamEncode::encode_year(ENCODE_FUNC_ARG_DECL) int ObBindParamEncode::encode_string(ENCODE_FUNC_ARG_DECL) { int ret = OB_SUCCESS; - ObLength length = param.get_length(); ObString val = param.get_string(); const ObObjType obj_type = param.get_type(); - enum_field_types buffer_type = static_cast(ob_type_to_mysql_type[obj_type]); bind_param.col_idx_ = col_idx; bind_param.buffer_type_ = buffer_type; - if (is_output_param) { - char *buf = nullptr; - const int64_t buf_len = length * 3; - if (OB_ISNULL(buf = reinterpret_cast(allocator.alloc(buf_len)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to alloc memory", K(ret), K(buf_len)); - } else { - MEMCPY(buf, val.ptr(), val.length()); - bind_param.buffer_ = buf; - bind_param.buffer_len_ = buf_len; - bind_param.length_ = val.length(); - } + bind_param.buffer_ = val.ptr(); + bind_param.buffer_len_ = val.length(); + bind_param.length_ = val.length(); + return ret; +} + +int ObBindParamEncode::encode_number_float(ENCODE_FUNC_ARG_DECL) +{ + int ret = OB_SUCCESS; + int64_t pos = 0; + void *buf = NULL; + int64_t buf_len = 41; + if (OB_ISNULL(buf = allocator.alloc(buf_len))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc buf", K(ret)); + } else if (OB_FAIL(param.get_number().format_v1((char *)buf, buf_len, pos, param.get_scale()))) { + LOG_WARN("fail to format float", K(ret)); } else { - bind_param.buffer_ = val.ptr(); - bind_param.buffer_len_ = val.length(); - bind_param.length_ = val.length(); + bind_param.col_idx_ = col_idx; + bind_param.buffer_type_ = buffer_type; + bind_param.buffer_ = buf; + bind_param.buffer_len_ = buf_len; + bind_param.length_ = pos; } return ret; } @@ -398,13 +464,13 @@ const ObBindParamEncode::EncodeFunc ObBindParamEncode::encode_map_[ObMaxType + 1 ObBindParamEncode::encode_not_supported, // ObSetType ObBindParamEncode::encode_not_supported, // ObEnumInnerType ObBindParamEncode::encode_not_supported, // ObSetInnerType - ObBindParamEncode::encode_not_supported, // ObTimestampTZType - ObBindParamEncode::encode_not_supported, // ObTimestampLTZType - ObBindParamEncode::encode_not_supported, // ObTimestampNanoType + ObBindParamEncode::encode_timestamp, // ObTimestampTZType + ObBindParamEncode::encode_timestamp, // ObTimestampLTZType + ObBindParamEncode::encode_timestamp, // ObTimestampNanoType ObBindParamEncode::encode_not_supported, // ObRawType ObBindParamEncode::encode_not_supported, // ObIntervalYMType ObBindParamEncode::encode_not_supported, // ObIntervalDSType - ObBindParamEncode::encode_not_supported, // ObNumberFloatType + ObBindParamEncode::encode_number_float, // ObNumberFloatType ObBindParamEncode::encode_not_supported, // ObNVarchar2Type ObBindParamEncode::encode_not_supported, // ObNCharType ObBindParamEncode::encode_not_supported, // ObURowIDType @@ -611,6 +677,71 @@ int ObBindParamDecode::decode_time(DECODE_FUNC_ARG_DECL) return ret; } +int ObBindParamDecode::decode_timestamp(DECODE_FUNC_ARG_DECL) +{ + int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL + ret = OB_NOT_SUPPORTED; +#else + const bool is_tz = (MYSQL_TYPE_OB_TIMESTAMP_WITH_TIME_ZONE == field_type); + ObOTimestampData ot_data; + ot_data.reset(); + ORACLE_TIME *ora_time = reinterpret_cast(bind_param.buffer_); + ObTime ob_time(DT_TYPE_ORACLE_TIMESTAMP); + if (OB_ISNULL(ora_time)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("oracle time is NULL", K(ret)); + } else { + int32_t century = ora_time->century; + int32_t year = ora_time->year; + ob_time.parts_[DT_YEAR] = (int32_t)(century > 0 ? (century * YEARS_PER_CENTURY + year) + : (0 - (0 - century * YEARS_PER_CENTURY + year))); + ob_time.parts_[DT_MON] = ora_time->month; + ob_time.parts_[DT_MDAY] = ora_time->day; + ob_time.parts_[DT_HOUR] = ora_time->hour; + ob_time.parts_[DT_MIN] = ora_time->minute; + ob_time.parts_[DT_SEC] = ora_time->second; + ob_time.parts_[DT_USEC] = ora_time->second_part; + ob_time.parts_[DT_DATE] = ObTimeConverter::ob_time_to_date(ob_time); + ObTimeConvertCtx ctx(&tz_info, ObTimeConverter::COMPAT_OLD_NLS_TIMESTAMP_TZ_FORMAT, true); + if (is_tz) { + ob_time.mode_ |= DT_TYPE_TIMEZONE; + if (OB_NOT_NULL(ora_time->tz_name)) { + if (OB_FAIL(ob_time.set_tz_name(ObString(ora_time->tz_name)))) { + LOG_WARN("fail to set_tz_name", K(ret), K(ObString(ora_time->tz_name))); + } else if (OB_NOT_NULL(ora_time->tz_abbr) + && OB_FAIL(ob_time.set_tzd_abbr(ObString(ora_time->tz_abbr)))) { + LOG_WARN("fail to set_tzd_abbr", K(ret), K(ObString(ora_time->tz_abbr))); + } else if (OB_FAIL(ObTimeConverter::str_to_tz_offset(ctx, ob_time))) { + LOG_WARN("fail to convert string to tz_offset", K(ret)); + } + } else { + int32_t offset_hour = ora_time->offset_hour; + int32_t offset_min = ora_time->offset_minute; + ob_time.parts_[DT_OFFSET_MIN] = static_cast(offset_hour >= 0 ? (offset_hour * MINS_PER_HOUR + offset_min) : (0 - (0 - offset_hour * MINS_PER_HOUR + offset_min))); + ob_time.is_tz_name_valid_ = true; + } + } + if (OB_SUCC(ret)) { + ObObjType obj_type = ObTimestampNanoType; + if (field_type == MYSQL_TYPE_OB_TIMESTAMP_WITH_TIME_ZONE) { + obj_type = ObTimestampTZType; + } else if (field_type == MYSQL_TYPE_OB_TIMESTAMP_WITH_LOCAL_TIME_ZONE) { + obj_type = ObTimestampLTZType; + } + if (OB_FAIL(ObTimeConverter::ob_time_to_utc(obj_type, ctx, ob_time))) { + LOG_WARN("failed to convert ob_time to utc", K(ret)); + } else if (OB_FAIL(ObTimeConverter::ob_time_to_otimestamp(ob_time, ot_data))) { + LOG_WARN("failed to ob_time to otimestamp", K(ret)); + } else { + param.set_otimestamp_value(param.get_type(), ot_data); + } + } + } +#endif + return ret; +} + int ObBindParamDecode::decode_year(DECODE_FUNC_ARG_DECL) { UNUSEDx(field_type, tz_info, allocator); @@ -634,14 +765,19 @@ int ObBindParamDecode::decode_string(DECODE_FUNC_ARG_DECL) break; case MYSQL_TYPE_STRING: param.set_char(dst); + break; case MYSQL_TYPE_TINY_BLOB: param.set_lob_value(ObTinyTextType, dst.ptr(), dst.length()); + break; case MYSQL_TYPE_BLOB: param.set_lob_value(ObTextType, dst.ptr(), dst.length()); + break; case MYSQL_TYPE_MEDIUM_BLOB: param.set_lob_value(ObMediumTextType, dst.ptr(), dst.length()); + break; case MYSQL_TYPE_LONG_BLOB: param.set_lob_value(ObLongTextType, dst.ptr(), dst.length()); + break; default: ret = OB_ERR_UNEXPECTED; LOG_WARN("unknown type", K(ret), K(field_type)); @@ -651,6 +787,19 @@ int ObBindParamDecode::decode_string(DECODE_FUNC_ARG_DECL) return ret; } +int ObBindParamDecode::decode_number_float(DECODE_FUNC_ARG_DECL) +{ + int ret = OB_SUCCESS; + ObString value(bind_param.length_, (char *)bind_param.buffer_); + number::ObNumber nb; + if (OB_FAIL(nb.from(value.ptr(), value.length(), allocator))) { + LOG_WARN("fail to decode number float", K(ret), K(value)); + } else { + param.set_number_float(nb); + } + return ret; +} + int ObBindParamDecode::decode_not_supported(DECODE_FUNC_ARG_DECL) { UNUSEDx(tz_info, bind_param, param, allocator); @@ -698,13 +847,13 @@ const ObBindParamDecode::DecodeFunc ObBindParamDecode::decode_map_[ObMaxType + 1 ObBindParamDecode::decode_not_supported, // ObSetType ObBindParamDecode::decode_not_supported, // ObEnumInnerType ObBindParamDecode::decode_not_supported, // ObSetInnerType - ObBindParamDecode::decode_not_supported, // ObTimestampTZType - ObBindParamDecode::decode_not_supported, // ObTimestampLTZType - ObBindParamDecode::decode_not_supported, // ObTimestampNanoType + ObBindParamDecode::decode_timestamp, // ObTimestampTZType + ObBindParamDecode::decode_timestamp, // ObTimestampLTZType + ObBindParamDecode::decode_timestamp, // ObTimestampNanoType ObBindParamDecode::decode_not_supported, // ObRawType ObBindParamDecode::decode_not_supported, // ObIntervalYMType ObBindParamDecode::decode_not_supported, // ObIntervalDSType - ObBindParamDecode::decode_not_supported, // ObNumberFloatType + ObBindParamDecode::decode_number_float, // ObNumberFloatType ObBindParamDecode::decode_not_supported, // ObNVarchar2Type ObBindParamDecode::decode_not_supported, // ObNCharType ObBindParamDecode::decode_not_supported, // ObURowIDType @@ -714,9 +863,8 @@ const ObBindParamDecode::DecodeFunc ObBindParamDecode::decode_map_[ObMaxType + 1 ObBindParamDecode::decode_not_supported // ObMaxType }; -int ObBindParam::assign(const ObBindParam &other) +void ObBindParam::assign(const ObBindParam &other) { - int ret = OB_SUCCESS; col_idx_ = other.col_idx_; buffer_type_ = other.buffer_type_; buffer_ = other.buffer_; @@ -724,7 +872,10 @@ int ObBindParam::assign(const ObBindParam &other) length_ = other.length_; is_unsigned_ = other.is_unsigned_; is_null_ = other.is_null_; - return ret; + array_buffer_ = other.array_buffer_; + ele_size_ = other.ele_size_; + max_array_size_ = other.max_array_size_; + out_valid_array_size_ = other.out_valid_array_size_; } ObMySQLPreparedStatement::ObMySQLPreparedStatement() : @@ -745,9 +896,16 @@ ObMySQLPreparedStatement::~ObMySQLPreparedStatement() { } -ObIAllocator &ObMySQLPreparedStatement::get_allocator() +ObIAllocator *ObMySQLPreparedStatement::get_allocator() { - return *alloc_; + return alloc_; +} + +void ObMySQLPreparedStatement::set_allocator(ObIAllocator *alloc) +{ + alloc_ = alloc; + result_.alloc_ = alloc; + param_.alloc_ = alloc; } MYSQL_STMT *ObMySQLPreparedStatement::get_stmt_handler() @@ -937,7 +1095,7 @@ int ObMySQLPreparedStatement::get_ob_type(ObObjType &ob_type, obmysql::EMySQLFie return ret; } -int ObMySQLPreparedStatement::init(ObMySQLConnection &conn, const char *sql) +int ObMySQLPreparedStatement::init(ObMySQLConnection &conn, const char *sql, int64_t param_count) { int ret = OB_SUCCESS; conn_ = &conn; @@ -979,39 +1137,31 @@ int ObMySQLPreparedStatement::close() result_column_count_ = 0; bind_params_ = NULL; result_params_ = NULL; + param_.close(); + result_.close(); return ret; } -int ObMySQLPreparedStatement::bind_param(const ObBindParam &bind_param) +int ObMySQLPreparedStatement::bind_param(ObBindParam &bind_param) { int ret = OB_SUCCESS; if (bind_param.col_idx_ >= stmt_param_count_) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid index", K(ret), K(bind_param), K(stmt_param_count_)); - } else { - ObBindParam ¶m = bind_params_[bind_param.col_idx_]; - if (OB_FAIL(param.assign(bind_param))) { - LOG_WARN("fail to assing bind param", K(ret)); - } else if (OB_FAIL(param_.bind_param(param))) { - LOG_WARN("fail to bind param", K(ret), K(bind_param)); - } + } else if (OB_FAIL(param_.bind_param(bind_param))) { + LOG_WARN("fail to bind param", K(ret), K(bind_param)); } return ret; } -int ObMySQLPreparedStatement::bind_result(const ObBindParam &bind_param) +int ObMySQLPreparedStatement::bind_result(ObBindParam &bind_param) { int ret = OB_SUCCESS; if (bind_param.col_idx_ >= result_column_count_) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid index", K(ret), K(bind_param), K(result_column_count_)); - } else { - ObBindParam ¶m = result_params_[bind_param.col_idx_]; - if (OB_FAIL(param.assign(bind_param))) { - LOG_WARN("fail to assing bind param", K(ret)); - } else if (OB_FAIL(result_.bind_result(param))) { - LOG_WARN("fail to bind param", K(ret), K(bind_param)); - } + } else if (OB_FAIL(result_.bind_result(bind_param))) { + LOG_WARN("fail to bind param", K(ret), K(bind_param)); } return ret; } @@ -1147,75 +1297,253 @@ ObMySQLPreparedResult *ObMySQLPreparedStatement::execute_query() } int ObMySQLProcStatement::bind_param(const int64_t col_idx, + const int64_t param_idx, const bool is_output_param, const ObTimeZoneInfo *tz_info, - ObObjParam ¶m, + ObObj ¶m, const share::schema::ObRoutineInfo &routine_info, ObIAllocator &allocator) { int ret = OB_SUCCESS; ObBindParam *bind_param = nullptr; const ObObjType obj_type = param.get_type(); - if (OB_FAIL(get_bind_param_by_idx(col_idx, bind_param))) { - LOG_WARN("fail to get bind param by idx", K(ret), K(col_idx)); - } else if (OB_ISNULL(bind_param)) { + const share::schema::ObRoutineParam *routine_param = NULL; + if (param_idx >= routine_info.get_routine_params().count()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("col_idx invalid", K(ret), K(param_idx)); + } else if (FALSE_IT(routine_param = routine_info.get_routine_params().at(param_idx))) { + } else if (OB_ISNULL(routine_param)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to get bind param by idx", K(ret), K(col_idx), K(stmt_param_count_)); - } else if (OB_FAIL(ObBindParamEncode::encode_map_[obj_type](col_idx, - is_output_param, - *tz_info, - param, - *bind_param, - allocator))) { - LOG_WARN("fail to encode param", K(ret)); - } else if (OB_FAIL(param_.bind_param(*bind_param))) { - LOG_WARN("failed to bind param", K(ret), KPC(bind_param)); + LOG_WARN("routine_param is NULL", K(ret), K(col_idx)); + } else { + enum_field_types buffer_type = MAX_NO_FIELD_TYPES; + if (param.is_null()) { + buffer_type = static_cast(ob_type_to_mysql_type[routine_param->get_param_type().get_obj_type()]); + } else { + buffer_type = static_cast(ob_type_to_mysql_type[param.get_type()]); + } + if (OB_FAIL(get_bind_param_by_idx(col_idx, bind_param))) { + LOG_WARN("fail to get bind param by idx", K(ret), K(col_idx)); + } else if (OB_ISNULL(bind_param)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get bind param by idx", K(ret), K(col_idx), K(stmt_param_count_)); + } else if (OB_FAIL(ObBindParamEncode::encode_map_[obj_type](col_idx, + is_output_param, + *tz_info, + param, + *bind_param, + allocator, + buffer_type))) { + LOG_WARN("fail to encode param", K(ret)); + } else if (OB_FAIL(param_.bind_param(*bind_param))) { + LOG_WARN("failed to bind param", K(ret), KPC(bind_param)); + } } return ret; } -int ObMySQLProcStatement::bind_proc_param(ObIAllocator &allocator, - ParamStore ¶ms, - const share::schema::ObRoutineInfo &routine_info, - common::ObIArray &basic_out_param, - const ObTimeZoneInfo *tz_info) +int ObMySQLProcStatement::bind_basic_type_by_pos(uint64_t position, + void *param_buffer, + int64_t param_size, + int32_t datatype, + int32_t &indicator, + bool is_out_param) { int ret = OB_SUCCESS; - for (int64_t i = 0; OB_SUCC(ret) && i < params.count(); i++) { - ObObjParam ¶m = params.at(i); - const share::schema::ObRoutineParam *r_param = routine_info.get_routine_params().at(i); - bool is_output = false; - if (OB_ISNULL(r_param)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("param is null", K(ret), K(i)); - } else { - is_output = r_param->is_out_sp_param() || r_param->is_inout_sp_param(); - if (is_output && OB_FAIL(basic_out_param.push_back(i))) { - LOG_WARN("push back failed", K(ret), K(i)); - } else if (OB_FAIL(bind_param(i, is_output, tz_info, param, routine_info, allocator))) { - LOG_WARN("failed to bind param", K(ret)); + ObBindParam *bind_param = NULL; + if (OB_FAIL(get_bind_param_by_idx(position, bind_param))) { + LOG_WARN("fail to get bind param by idx", K(ret), K(position)); + } else if (OB_ISNULL(bind_param)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get bind param by idx", K(ret), K(position), K(stmt_param_count_)); + } else { + enum_field_types buffer_type = static_cast(ob_type_to_mysql_type[(datatype)]); + bind_param->col_idx_ = position; + bind_param->buffer_type_ = buffer_type; + bind_param->buffer_ = param_buffer; + bind_param->buffer_len_ = param_size; + bind_param->is_null_ = 0; + bind_param->length_ = param_size; + if (OB_FAIL(in_out_map_.push_back(is_out_param))) { + LOG_WARN("failed to push back", K(ret)); + } else if (OB_FAIL(param_.bind_param(*bind_param))) { + LOG_WARN("failed tp bind param", K(ret), KPC(bind_param)); + } else if (stmt_param_count_ == position + 1) { + if (OB_FAIL(param_.bind_param())) { + LOG_WARN("failed to bind param", K(ret), + "info", mysql_stmt_error(stmt_), "info", mysql_error(conn_->get_handler())); } } } return ret; } -int ObMySQLProcStatement::convert_proc_output_param_result(const ObTimeZoneInfo &tz_info, - const ObBindParam &bind_param, - ObObjParam ¶m, - const share::schema::ObRoutineInfo &routine_info, - ObIAllocator &allocator) +int ObMySQLProcStatement::bind_array_type_by_pos(uint64_t position, + void *array, + int32_t *indicators, + int64_t ele_size, + int32_t ele_datatype, + uint64_t array_size, + uint32_t *out_valid_array_size) { int ret = OB_SUCCESS; - ObObjType obj_type = ObNullType; - if (OB_FAIL(get_ob_type(obj_type, static_cast(bind_param.buffer_type_)))) { - LOG_WARN("fail to get ob type", K(ret), K(bind_param)); - } else if (OB_FAIL(ObBindParamDecode::decode_map_[obj_type](bind_param.buffer_type_, - tz_info, - bind_param, - param, - allocator))) { - LOG_WARN("failed to decode param", K(ret)); +#ifndef OB_BUILD_ORACLE_PL + ret = OB_NOT_SUPPORTED; +#else + ObBindParam *bind_param = NULL; + if (OB_FAIL(get_bind_param_by_idx(position, bind_param))) { + LOG_WARN("fail to get bind param by idx", K(ret), K(position)); + } else if (OB_ISNULL(bind_param)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get bind param by idx", K(ret), K(position), K(stmt_param_count_)); + } else { + enum_field_types ele_type = static_cast(ob_type_to_mysql_type[(ele_datatype)]); + MYSQL_COMPLEX_BIND_PLARRAY *my_array = NULL; + // input param out_valid_array_size is 0 by default + int64_t array_elem_size = convert_type_to_complex(ele_type); + // libobclient has a bug, even if the length of the input param array is 0, + // still need to apply for a memory to prevent observer core. + int64_t buf_len = sizeof(MYSQL_COMPLEX_BIND_PLARRAY) + array_elem_size; + char *buf = NULL; + if (OB_ISNULL(buf = (char *)alloc_->alloc(buf_len))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc memory failed", K(ret)); + } else { + MEMSET(buf, 0, buf_len); + bind_param->col_idx_ = position; + bind_param->buffer_type_ = MYSQL_TYPE_OBJECT; + bind_param->buffer_ = buf; + bind_param->buffer_len_ = buf_len; + bind_param->is_null_ = 0; + bind_param->array_buffer_ = array; + bind_param->ele_size_ = ele_size; + bind_param->max_array_size_ = array_size; + bind_param->out_valid_array_size_ = out_valid_array_size; + + my_array = (MYSQL_COMPLEX_BIND_PLARRAY *)buf; + my_array->buffer_type = MYSQL_TYPE_PLARRAY; + my_array->buffer = buf + sizeof(MYSQL_COMPLEX_BIND_PLARRAY); + my_array->is_null = 0; + my_array->type_name = NULL; + my_array->owner_name = NULL; + my_array->elem_type = ele_type; + my_array->length = 0; + my_array->maxrarr_len = array_size; + if (OB_FAIL(in_out_map_.push_back(true))) { + LOG_WARN("failed to push back", K(ret)); + } else if (OB_FAIL(param_.bind_param(*bind_param))) { + LOG_WARN("failed tp bind param", K(ret), KPC(bind_param)); + } else if (stmt_param_count_ == position + 1 + && OB_FAIL(param_.bind_param())) { + LOG_WARN("failed to bind param", K(ret), "info", mysql_stmt_error(stmt_)); + } + } + } +#endif + return ret; +} + +int ObMySQLProcStatement::bind_proc_param(ObIAllocator &allocator, + ParamStore ¶ms, + const share::schema::ObRoutineInfo &routine_info, + common::ObIArray> &basic_out_param, + const ObTimeZoneInfo *tz_info, + ObObj *result, + bool is_sql) +{ + int ret = OB_SUCCESS; + int64_t start_idx = routine_info.get_param_start_idx(); + if (routine_info.is_function() && !is_sql) { + if (OB_ISNULL(result)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result is NULL", K(ret)); + } else if (OB_FAIL(basic_out_param.push_back(std::make_pair(0, -1)))) { + LOG_WARN("push back failed", K(ret)); + } else if (OB_FAIL(bind_param(0, 0, true, tz_info, *result, routine_info, allocator))) { + LOG_WARN("failed to bind param", K(ret)); + } + } + int64_t start_pos = (routine_info.is_function() ? (is_sql ? 0 : 1) : 0); + int64_t skip_cnt = 0; + for (int64_t i = 0; OB_SUCC(ret) && i < params.count(); i++) { + ObObjParam ¶m = params.at(i); + const share::schema::ObRoutineParam *r_param = routine_info.get_routine_params().at(i + start_idx); + bool is_output = false; + if (OB_ISNULL(r_param)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("param is null", K(ret), K(i)); + } else if (param.is_pl_mock_default_param()) { + skip_cnt++; + } else { + is_output = r_param->is_out_sp_param() || r_param->is_inout_sp_param(); + if (is_output && OB_FAIL(basic_out_param.push_back(std::make_pair(i + start_pos - skip_cnt, i)))) { + LOG_WARN("push back failed", K(ret), K(i)); + } else if (OB_FAIL(bind_param(i + start_pos - skip_cnt, i + (routine_info.is_function() ? 1 : 0), + is_output, tz_info, param, routine_info, allocator))) { + LOG_WARN("failed to bind param", K(ret)); + } + } + } + if (routine_info.is_function() && is_sql) { + if (OB_ISNULL(result)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result is NULL", K(ret)); + } else if (OB_FAIL(basic_out_param.push_back(std::make_pair(params.count() - skip_cnt, -1)))) { + LOG_WARN("push back failed", K(ret)); + } else if (OB_FAIL(bind_param(params.count() - skip_cnt, 0, true, tz_info, *result, routine_info, allocator))) { + LOG_WARN("failed to bind param", K(ret)); + } + } + return ret; +} + +int ObMySQLProcStatement::convert_proc_output_param_result(int64_t out_param_idx, + const ObTimeZoneInfo &tz_info, + const ObBindParam &bind_param, + ObObj *param, + const share::schema::ObRoutineInfo &routine_info, + ObIAllocator &allocator, + bool is_return_value) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(param)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("param is NULL", K(ret)); + } else { + if (bind_param.is_null_) { + param->set_null(); + } else { + ObObjType obj_type = ObNullType; + const share::schema::ObRoutineParam *routine_param = routine_info.get_routine_params().at(out_param_idx); + if (OB_ISNULL(routine_param)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("routine_param is NULL", K(ret), K(out_param_idx)); + } else { + const ObDataType &data_type = routine_param->get_param_type(); + if (param->is_null()) { + param->set_meta_type(data_type.get_meta_type()); + if (!is_return_value) { + ObObjParam *obj_param = static_cast(param); + if (OB_ISNULL(obj_param)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("obj_param is NULL", K(ret)); + } else { + obj_param->set_param_meta(); + obj_param->set_accuracy(data_type.get_accuracy()); + } + } + } + if (FAILEDx(get_ob_type(obj_type, static_cast(bind_param.buffer_type_)))) { + LOG_WARN("fail to get ob type", K(ret), K(bind_param)); + } else if (OB_FAIL(ObBindParamDecode::decode_map_[obj_type](bind_param.buffer_type_, + tz_info, + bind_param, + *param, + allocator))) { + LOG_WARN("failed to decode param", K(ret)); + } + } + } } return ret; } @@ -1223,49 +1551,428 @@ int ObMySQLProcStatement::convert_proc_output_param_result(const ObTimeZoneInfo int ObMySQLProcStatement::process_proc_output_params(ObIAllocator &allocator, ParamStore ¶ms, const share::schema::ObRoutineInfo &routine_info, - common::ObIArray &basic_out_param, - const ObTimeZoneInfo *tz_info) + common::ObIArray> &basic_out_param, + const ObTimeZoneInfo *tz_info, + ObObj *result, + bool is_sql) { int ret = OB_SUCCESS; - const int64_t params_count = params.count(); - for (int64_t i = 0; OB_SUCC(ret) && i < basic_out_param.count(); i++) { - const int64_t col_idx = basic_out_param.at(i); - ObBindParam *bind_param = nullptr; - if (col_idx >= params_count || col_idx > stmt_param_count_) { + if (basic_out_param.count() > 0) { + const int64_t params_count = params.count(); + if (OB_FAIL(result_.init())) { + LOG_WARN("failed to init result_", K(ret)); + } else if (FALSE_IT(result_column_count_ = result_.get_result_column_count())) { + } else if (result_column_count_ != basic_out_param.count()) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("idx out of range", K(ret), K(col_idx), K(params_count)); - } else if (OB_FAIL(get_bind_param_by_idx(col_idx, bind_param))) { - LOG_WARN("fail to get bind param by idx", K(ret), K(col_idx), K_(stmt_param_count)); - } else if (OB_FAIL(convert_proc_output_param_result(*tz_info, *bind_param, params.at(col_idx), routine_info, allocator))) { - LOG_WARN("fail to convert proc output param result", K(ret)); + LOG_WARN("only support basic type out param", K(ret), K(result_column_count_), K(basic_out_param.count())); + } else if (result_column_count_ > 0 + && OB_FAIL(alloc_bind_params(result_column_count_, result_params_))) { + LOG_WARN("failed to alloc bind params", K(ret), K(result_column_count_)); + } else if (result_column_count_ > 0) { + ObBindParam *in_param = NULL; + ObBindParam *out_param = NULL; + MYSQL_BIND *mysql_bind = result_.get_bind(); + int64_t out_idx = 0; + for (int64_t i = 0; OB_SUCC(ret) && i < basic_out_param.count(); i++) { + if (OB_FAIL(get_bind_param_by_idx(basic_out_param.at(i).first, in_param))) { + LOG_WARN("failed to get param", K(ret), K(i)); + } else if (OB_ISNULL(in_param)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("in_param is NULL", K(ret), K(i)); + } else if (i >= result_column_count_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("out_idx is error", K(ret), K(out_idx), K(result_column_count_)); + } else if (OB_FAIL(get_bind_result_param_by_idx(i, out_param))) { + LOG_WARN("fail to get bind result param by idx", K(ret), K(out_idx)); + } else if (OB_ISNULL(out_param) || OB_ISNULL(mysql_bind)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("out_param is NULL", K(ret), K(out_idx), K(mysql_bind)); + } else { + out_param->assign(*in_param); + mysql_bind[i].buffer_type = out_param->buffer_type_; + mysql_bind[i].buffer = out_param->buffer_; + mysql_bind[i].buffer_length = out_param->buffer_len_; + mysql_bind[i].length = &out_param->length_; + mysql_bind[i].error = &mysql_bind[i].error_value; + mysql_bind[i].is_null = &out_param->is_null_; + if (OB_ISNULL(mysql_bind[i].buffer)) { + void *tmp_buf = NULL; + int64_t tmp_buf_len = 0; + if (MYSQL_TYPE_DATETIME == out_param->buffer_type_) { + tmp_buf_len = sizeof(MYSQL_TIME); + } else if (MYSQL_TYPE_FLOAT == out_param->buffer_type_) { + tmp_buf_len = sizeof(float); + } else if (MYSQL_TYPE_DOUBLE == out_param->buffer_type_) { + tmp_buf_len = sizeof(double); + } else if (MYSQL_TYPE_LONG == out_param->buffer_type_) { + tmp_buf_len = sizeof(int64_t); + } + if (tmp_buf_len > 0) { + if (OB_ISNULL(tmp_buf = allocator.alloc(tmp_buf_len))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory", K(ret)); + } else { + mysql_bind[i].buffer = tmp_buf; + mysql_bind[i].buffer_length = tmp_buf_len; + out_param->buffer_ = tmp_buf; + out_param->buffer_len_ =tmp_buf_len; + } + } + } + } + } + if (OB_SUCC(ret)) { + int tmp_ret = 0; + if (0 != (tmp_ret = mysql_stmt_bind_result(stmt_, mysql_bind))) { + ret = -mysql_stmt_errno(stmt_); + LOG_WARN("failed to bind out param", K(ret), "info", mysql_stmt_error(stmt_)); + } else { + tmp_ret = mysql_stmt_fetch(stmt_); + if (MYSQL_DATA_TRUNCATED == tmp_ret) { + if (OB_FAIL(handle_data_truncated(allocator))) { + LOG_WARN("failed to handler data", K(ret)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < basic_out_param.count(); i++) { + const int64_t col_idx = basic_out_param.at(i).first; + ObBindParam *bind_param = nullptr; + if (OB_FAIL(get_bind_result_param_by_idx(i, bind_param))) { + LOG_WARN("fail to get bind param by idx", K(ret), K(col_idx), K_(stmt_param_count)); + } else if (OB_ISNULL(bind_param)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("bind_param is NULL", K(ret)); + } else { + ObObj *obj = NULL; + bool is_return_value = false; + if (routine_info.is_function() + && ((is_sql && i == result_column_count_ - 1) + ||(!is_sql && (col_idx == 0)))) { + obj = result; + is_return_value = true; + } else { + obj = ¶ms.at(basic_out_param.at(i).second); + } + if (OB_ISNULL(obj)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("obj is NULL", K(ret)); + } else if (OB_FAIL(convert_proc_output_param_result(col_idx, *tz_info, *bind_param, obj, + routine_info, allocator, is_return_value))) { + LOG_WARN("fail to convert proc output param result", K(ret)); + } + } + } + } + } } - LOG_INFO("end process param", K(i), KPC(bind_param), K(ret)); } return ret; } +int ObMySQLProcStatement::init(ObMySQLConnection &conn, + const char *sql, + int64_t param_count) +{ + int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL + ret = OB_NOT_SUPPORTED; +#else + conn_ = &conn; + stmt_param_count_ = param_count; + in_out_map_.reset(); + if (OB_ISNULL(proc_ = sql)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid sql", KCSTRING(sql), K(ret)); + } else if (OB_ISNULL(conn_) || OB_ISNULL(conn_->get_handler())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("conn_ is NULL", K(ret)); + } else if (FALSE_IT(conn_->get_handler()->field_count = 0)) { + // After executing the previous statement, libObClient did not set the value of `mysql->filed_count` to 0, + // which may cause coredump. Set it manually here. + } else if (OB_ISNULL(stmt_ = mysql_stmt_init(conn_->get_handler()))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to init stmt", K(ret)); + } else if (0 != mysql_stmt_prepare_v2(stmt_, sql, STRLEN(sql), &stmt_param_count_)) { + ret = -mysql_errno(conn_->get_handler()); + LOG_WARN("fail to prepare stmt", "info", mysql_error(conn_->get_handler()), K(ret)); + } else if (OB_FAIL(param_.init())) { + LOG_WARN("fail to init prepared result", K(ret)); + } else if (stmt_param_count_ > 0 && OB_FAIL(alloc_bind_params(stmt_param_count_, bind_params_))) { + LOG_WARN("fail to alloc stmt bind params", K(ret)); + } +#endif + return ret; +} + int ObMySQLProcStatement::execute_proc(ObIAllocator &allocator, ParamStore ¶ms, const share::schema::ObRoutineInfo &routine_info, - const ObTimeZoneInfo *tz_info) + const ObTimeZoneInfo *tz_info, + ObObj *result, + bool is_sql) { int ret = OB_SUCCESS; - common::ObSEArray basic_out_param; + // pair.first is: out param position in @this.bind_params_ + // pair.second is: out param position in @params, if the routine is a function, the values is -1 + common::ObSEArray, 8> basic_out_param; + void* execute_extend_arg = NULL; if (OB_ISNULL(tz_info)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("tz info is null", K(ret)); - } else if (OB_FAIL(bind_proc_param(allocator, params, routine_info, basic_out_param, tz_info))) { + } else if (OB_FAIL(bind_proc_param(allocator, params, routine_info, basic_out_param, tz_info, result, is_sql))) { LOG_WARN("failed to bind proc param", K(ret)); } else if (OB_FAIL(param_.bind_param())) { LOG_WARN("failed to bind prepared input param", "info", mysql_error(conn_->get_handler()), K(ret)); - } else if (0 != mysql_stmt_execute(stmt_)) { - ret = -mysql_stmt_errno(stmt_); - LOG_WARN("fail to execute stmt", "info", mysql_stmt_error(stmt_), "info", mysql_error(conn_->get_handler()), K(ret)); - } else if (OB_FAIL(process_proc_output_params(allocator, params, routine_info, basic_out_param, tz_info))) { + } else if (OB_FAIL(execute_stmt_v2_interface())) { + LOG_WARN("failed to execute PL", K(ret)); + } else if (OB_FAIL(process_proc_output_params(allocator, params, routine_info, basic_out_param, + tz_info, result, is_sql))) { LOG_WARN("fail to process proc output params", K(ret)); } return ret; -}; +} + +int ObMySQLProcStatement::execute_proc() +{ + int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL + ret = OB_NOT_SUPPORTED; +#else + void* execute_extend_arg = NULL; + MYSQL_RES *res = NULL; + uint64_t ps_stmt_checksum = ob_crc64(proc_, STRLEN(proc_)); + LOG_INFO("execute v2 info", K(ret), K(ps_stmt_checksum), K(STRLEN(proc_)), K(proc_)); + if (OB_FAIL(execute_stmt_v2_interface())) { + LOG_WARN("failed to execute PL", K(ret)); + } else if (OB_ISNULL(res = mysql_stmt_result_metadata(stmt_))) { + ret = -mysql_stmt_errno(stmt_); + char errmsg[256] = {0}; + const char *srcmsg = mysql_stmt_error(stmt_); + MEMCPY(errmsg, srcmsg, MIN(255, STRLEN(srcmsg))); + TRANSLATE_CLIENT_ERR(ret, errmsg); + LOG_WARN("failed to execute proc", K(ret), "info", mysql_stmt_error(stmt_)); + } else if (OB_FAIL(result_.init())) { + LOG_WARN("fail to init prepared result", K(ret)); + } else if (FALSE_IT(result_column_count_ = res->field_count)) { + } else if (result_column_count_ > 0 && OB_FAIL(alloc_bind_params(result_column_count_, result_params_))) { + LOG_WARN("fail to alloc result bind params", K(ret), K(result_column_count_)); + } else if (result_column_count_ > 0) { + ObBindParam *in_param = NULL; + ObBindParam *out_param = NULL; + MYSQL_BIND *mysql_bind = result_.get_bind(); + int64_t out_idx = 0; + for (int64_t i = 0; OB_SUCC(ret) && i < stmt_param_count_; i++) { + if (in_out_map_.at(i)) { + if (OB_FAIL(get_bind_param_by_idx(i, in_param))) { + LOG_WARN("fail to get bind param by idx", K(ret), K(i)); + } else if (OB_ISNULL(in_param)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("in_param is NULL", K(ret), K(i), K(stmt_param_count_)); + } else if (out_idx >= result_column_count_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("out_idx is error", K(ret), K(out_idx), K(result_column_count_), K(stmt_param_count_)); + } else if (OB_FAIL(get_bind_result_param_by_idx(out_idx, out_param))) { + LOG_WARN("fail to get bind result param by idx", K(ret), K(out_idx)); + } else if (OB_ISNULL(out_param) || OB_ISNULL(mysql_bind)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("out_param is NULL", K(ret), K(out_idx), K(stmt_param_count_), K(mysql_bind)); + } else { + out_param->assign(*in_param); + mysql_bind[out_idx].buffer_type = in_param->buffer_type_; + mysql_bind[out_idx].buffer = in_param->buffer_; + mysql_bind[out_idx].buffer_length = in_param->buffer_len_; + mysql_bind[out_idx].length = &in_param->length_; + mysql_bind[out_idx].error = &mysql_bind[out_idx].error_value; + mysql_bind[out_idx].is_null = &in_param->is_null_; + out_idx++; + } + } + } + if (OB_SUCC(ret)) { + int tmp_ret = 0; + if (0 != (tmp_ret = mysql_stmt_bind_result(stmt_, mysql_bind))) { + ret = -mysql_stmt_errno(stmt_); + LOG_WARN("failed to bind out param", K(ret), "info", mysql_stmt_error(stmt_)); + } else if (0 != (tmp_ret = mysql_stmt_fetch(stmt_))) { + if (MYSQL_DATA_TRUNCATED != tmp_ret) { + ret = -mysql_stmt_errno(stmt_); + LOG_WARN("failed to fetch", K(ret), K(tmp_ret), + "info", mysql_stmt_error(stmt_), "info", mysql_error(conn_->get_handler())); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < result_column_count_; i++) { + MYSQL_BIND &res_bind = mysql_bind[i]; + void *res_buffer = NULL; + if (OB_ISNULL(res_buffer = alloc_->alloc(*mysql_bind[i].length))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc memory failed", K(ret)); + } else { + mysql_bind[i].buffer = res_buffer; + mysql_bind[i].buffer_length = *mysql_bind[i].length; + if (0 != mysql_stmt_fetch_column(stmt_, &mysql_bind[i], i, 0)) { + ret = -mysql_stmt_errno(stmt_); + LOG_WARN("failed to fetch column", K(ret), "info", mysql_stmt_error(stmt_)); + } + } + } + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < result_column_count_; i++) { + if (mysql_bind[i].buffer_type == MYSQL_TYPE_OBJECT) { + MYSQL_COMPLEX_BIND_ARRAY *pl_array = (MYSQL_COMPLEX_BIND_ARRAY *)mysql_bind[i].buffer; + MYSQL_COMPLEX_BIND_HEADER *header = NULL; + char *copy_buff = (char *)result_params_[i].array_buffer_; + if (OB_ISNULL(pl_array)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("pl_array is NULL", K(ret)); + } else if (OB_ISNULL(header = (MYSQL_COMPLEX_BIND_HEADER *)pl_array->buffer)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("header is NULL", K(ret)); + } else if (OB_ISNULL(copy_buff) || OB_ISNULL(result_params_[i].out_valid_array_size_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("argument is NULL", K(ret), K(result_params_[i].out_valid_array_size_)); + } else { + *result_params_[i].out_valid_array_size_ = (int32_t)pl_array->length; + } + for (int64_t pos = 0; OB_SUCC(ret) && pos < pl_array->length; pos++) { + if (header->buffer_type == MYSQL_TYPE_LONG) { + MYSQL_COMPLEX_BIND_BASIC *bb = ((MYSQL_COMPLEX_BIND_BASIC *)pl_array->buffer) + pos; + if (OB_ISNULL(bb) || OB_ISNULL(bb->buffer)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("argument is NULL", K(ret), K(bb), K(bb->buffer)); + } else { + *(int *)copy_buff = *(int *)bb->buffer; + } + } else if (header->buffer_type == MYSQL_TYPE_VARCHAR) { + MYSQL_COMPLEX_BIND_STRING *bs = ((MYSQL_COMPLEX_BIND_STRING *)pl_array->buffer) + pos; + memcpy(copy_buff, (char *)bs->buffer, bs->length); + } else { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not support type", K(ret), K(header->buffer_type)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "some type "); + } + if (OB_SUCC(ret)) { + copy_buff += result_params_[i].ele_size_; + } + } + } + } + } + } + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = close())) { + LOG_WARN("close proc stmt failed", K(ret)); + } + ret = (ret == OB_SUCCESS) ? tmp_ret : ret; +#endif + return ret; +} + +int ObMySQLProcStatement::close() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(close_mysql_stmt())) { + LOG_WARN("close mysql stmt failed", K(ret)); + } + free_resouce(); + return ret; +} + +void ObMySQLProcStatement::free_resouce() +{ + if (NULL != bind_params_) { + alloc_->free(bind_params_); + bind_params_ = NULL; + stmt_param_count_ = 0; + } + if (NULL != result_params_) { + alloc_->free(result_params_); + result_params_ = NULL; + result_column_count_ = 0; + } + param_.close(); + result_.close(); + in_out_map_.reset(); + proc_ = NULL; +} + +int ObMySQLProcStatement::close_mysql_stmt() +{ + int ret = OB_SUCCESS; + if (NULL != stmt_) { + if (0 != mysql_stmt_close(stmt_)) { + ret = -mysql_errno(conn_->get_handler()); + LOG_WARN("fail to close stmt", "info", mysql_error(conn_->get_handler()), K(ret)); + } + } + return ret; +} + +int ObMySQLProcStatement::execute_stmt_v2_interface() +{ + int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL + ret = OB_NOT_SUPPORTED; +#else + void* execute_extend_arg = NULL; + if (OB_ISNULL(stmt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("stmt_ is NULL", K(ret)); + } else if (OB_ISNULL(proc_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("proc_ is NULL", K(ret)); + } else if (0 != mysql_stmt_execute_v2(stmt_, proc_, STRLEN(proc_), 1, 0, execute_extend_arg)) { + ret = -mysql_stmt_errno(stmt_); + char errmsg[256] = {0}; + const char *srcmsg = mysql_stmt_error(stmt_); + MEMCPY(errmsg, srcmsg, MIN(255, STRLEN(srcmsg))); + TRANSLATE_CLIENT_ERR(ret, errmsg); + LOG_WARN("failed to execute proc", "info", mysql_stmt_error(stmt_), K(ret)); + } +#endif + return ret; +} + +int ObMySQLProcStatement::handle_data_truncated(ObIAllocator &allocator) +{ + int ret = OB_SUCCESS; + MYSQL_BIND *mysql_bind = result_.get_bind(); + if (OB_ISNULL(mysql_bind)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mysql_bind is NULL", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < result_column_count_; i++) { + MYSQL_BIND &res_bind = mysql_bind[i]; + if (*res_bind.is_null) { + result_params_[i].is_null_ = 1; + } else { + if (res_bind.buffer_length < *res_bind.length) { + void *res_buffer = NULL; + if (OB_ISNULL(res_buffer = allocator.alloc(*res_bind.length))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc memory failed", K(ret)); + } else { + res_bind.buffer = res_buffer; + res_bind.buffer_length = *res_bind.length; + } + } + if (OB_SUCC(ret)) { + if (0 != mysql_stmt_fetch_column(stmt_, &res_bind, i, 0)) { + ret = -mysql_stmt_errno(stmt_); + LOG_WARN("failed to fetch column", K(ret), "info", mysql_stmt_error(stmt_)); + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to fetch column", K(ret), K(res_bind.buffer_type)); + } else { + result_params_[i].buffer_type_ = res_bind.buffer_type; + result_params_[i].buffer_ = res_bind.buffer; + result_params_[i].buffer_len_ = res_bind.buffer_length; + result_params_[i].length_ = *res_bind.length; + result_params_[i].is_unsigned_ = res_bind.is_unsigned; + result_params_[i].is_null_ = *res_bind.is_null; + } + } + } + } + return ret; +} } // end namespace sqlcient } // end namespace common diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_statement.h b/deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_statement.h index 679a56f42..66f34bb2a 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_statement.h +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_statement.h @@ -29,30 +29,37 @@ namespace sqlclient struct ObBindParam { ObBindParam() : col_idx_(-1), buffer_type_(enum_field_types::MAX_NO_FIELD_TYPES), buffer_(nullptr), - buffer_len_(0), length_(0), is_unsigned_(0), is_null_(0) + buffer_len_(0), length_(0), is_unsigned_(0), is_null_(0), array_buffer_(nullptr), + ele_size_(0), max_array_size_(0), out_valid_array_size_(nullptr) { } ObBindParam(int64_t col_idx, enum_field_types buffer_type, void *buffer, int64_t buffer_len, unsigned long length) : col_idx_(col_idx), buffer_type_(buffer_type), buffer_(buffer), buffer_len_(buffer_len), - length_(length), is_unsigned_(0), is_null_(0) + length_(length), is_unsigned_(0), is_null_(0), array_buffer_(nullptr), + ele_size_(0), max_array_size_(0), out_valid_array_size_(nullptr) { } ObBindParam(int64_t col_idx, enum_field_types buffer_type, void *buffer, int64_t buffer_len, unsigned long length, my_bool is_unsigned, my_bool is_null) : col_idx_(col_idx), buffer_type_(buffer_type), buffer_(buffer), buffer_len_(buffer_len), - length_(length), is_unsigned_(is_unsigned), is_null_(is_null) + length_(length), is_unsigned_(is_unsigned), is_null_(is_null), array_buffer_(nullptr), + ele_size_(0), max_array_size_(0), out_valid_array_size_(nullptr) { } - int assign(const ObBindParam &other); + void assign(const ObBindParam &other); TO_STRING_KV(K_(col_idx), K_(buffer_type), K_(buffer), K_(buffer_len), K_(length), K_(is_unsigned), - K_(is_null)); + K_(is_null), + K_(array_buffer), + K_(ele_size), + K_(max_array_size), + K_(out_valid_array_size)); public: int64_t col_idx_; @@ -62,6 +69,10 @@ public: unsigned long length_; my_bool is_unsigned_; my_bool is_null_; + void *array_buffer_; + int64_t ele_size_; + int64_t max_array_size_; + uint32_t *out_valid_array_size_; }; class ObBindParamEncode @@ -70,9 +81,10 @@ public: #define ENCODE_FUNC_ARG_DECL const int64_t col_idx, \ const bool is_output_param, \ const ObTimeZoneInfo &tz_info, \ - ObObjParam ¶m, \ + ObObj ¶m, \ ObBindParam &bind_param, \ - ObIAllocator &allocator + ObIAllocator &allocator, \ + enum_field_types buffer_type #define DEF_ENCODE_FUNC(name) \ static int encode_##name(ENCODE_FUNC_ARG_DECL); using EncodeFunc = int (*)(ENCODE_FUNC_ARG_DECL); @@ -87,10 +99,12 @@ public: DEF_ENCODE_FUNC(number); DEF_ENCODE_FUNC(unumber); DEF_ENCODE_FUNC(datetime); + DEF_ENCODE_FUNC(timestamp); DEF_ENCODE_FUNC(date); DEF_ENCODE_FUNC(time); DEF_ENCODE_FUNC(year); DEF_ENCODE_FUNC(string); + DEF_ENCODE_FUNC(number_float); static int encode_not_supported(ENCODE_FUNC_ARG_DECL); public: @@ -103,7 +117,7 @@ public: #define DECODE_FUNC_ARG_DECL const enum_field_types field_type, \ const ObTimeZoneInfo &tz_info, \ const ObBindParam &bind_param, \ - ObObjParam ¶m, \ + ObObj ¶m, \ ObIAllocator &allocator #define DEF_DECODE_FUNC(name) \ static int decode_##name(DECODE_FUNC_ARG_DECL); @@ -119,9 +133,11 @@ public: DEF_DECODE_FUNC(number); DEF_DECODE_FUNC(unumber); DEF_DECODE_FUNC(datetime); + DEF_DECODE_FUNC(timestamp); DEF_DECODE_FUNC(time); DEF_DECODE_FUNC(year); DEF_DECODE_FUNC(string); + DEF_DECODE_FUNC(number_float); static int decode_not_supported(DECODE_FUNC_ARG_DECL); public: @@ -133,14 +149,15 @@ class ObMySQLPreparedStatement public: ObMySQLPreparedStatement(); virtual ~ObMySQLPreparedStatement(); - ObIAllocator &get_allocator(); + ObIAllocator *get_allocator(); + void set_allocator(ObIAllocator *alloc); ObMySQLConnection *get_connection(); MYSQL_STMT *get_stmt_handler(); MYSQL *get_conn_handler(); - int close(); - int init(ObMySQLConnection &conn, const char *sql); - int bind_param(const ObBindParam ¶m); - int bind_result(const ObBindParam ¶m); + virtual int close(); + virtual int init(ObMySQLConnection &conn, const char *sql, int64_t param_count); + int bind_param(ObBindParam ¶m); + int bind_result(ObBindParam ¶m); int bind_param_int(const int64_t col_idx, int64_t *out_buf); int bind_param_varchar(const int64_t col_idx, char *out_buf, unsigned long res_len); @@ -188,39 +205,78 @@ protected: class ObMySQLProcStatement : public ObMySQLPreparedStatement { public: - ObMySQLProcStatement() + ObMySQLProcStatement() : ObMySQLPreparedStatement() { + in_out_map_.reset(); + proc_ = NULL; } ~ObMySQLProcStatement() { + in_out_map_.reset(); + proc_ = NULL; } + virtual int init(ObMySQLConnection &conn, const char *sql, int64_t param_count); + virtual int close(); + virtual void free_resouce(); + virtual int close_mysql_stmt(); int execute_proc(ObIAllocator &allocator, ParamStore ¶ms, const share::schema::ObRoutineInfo &routine_info, - const ObTimeZoneInfo *tz_info); + const ObTimeZoneInfo *tz_info, + ObObj *result, + bool is_sql); + int execute_proc(); + int bind_basic_type_by_pos(uint64_t position, + void *param_buffer, + int64_t param_size, + int32_t datatype, + int32_t &indicator, + bool is_out_param); + int bind_array_type_by_pos(uint64_t position, + void *array, + int32_t *indicators, + int64_t ele_size, + int32_t ele_datatype, + uint64_t array_size, + uint32_t *out_valid_array_size); + inline void set_proc(const char *sql) { proc_ = sql; } + private: int bind_proc_param(ObIAllocator &allocator, ParamStore ¶ms, const share::schema::ObRoutineInfo &routine_info, - common::ObIArray &basic_out_param, - const ObTimeZoneInfo *tz_info); + common::ObIArray> &basic_out_param, + const ObTimeZoneInfo *tz_info, + ObObj *result, + bool is_sql); int bind_param(const int64_t col_idx, + const int64_t param_idx, const bool is_output_param, const ObTimeZoneInfo *tz_info, - ObObjParam &obj, + ObObj &obj, const share::schema::ObRoutineInfo &routine_info, ObIAllocator &allocator); int process_proc_output_params(ObIAllocator &allocator, ParamStore ¶ms, const share::schema::ObRoutineInfo &routine_info, - common::ObIArray &basic_out_param, - const ObTimeZoneInfo *tz_info); - int convert_proc_output_param_result(const ObTimeZoneInfo &tz_info, + common::ObIArray> &basic_out_param, + const ObTimeZoneInfo *tz_info, + ObObj *result, + bool is_sql); + int convert_proc_output_param_result(int64_t out_param_idx, + const ObTimeZoneInfo &tz_info, const ObBindParam &bind_param, - ObObjParam ¶m, + ObObj *param, const share::schema::ObRoutineInfo &routine_info, - ObIAllocator &allocator); + ObIAllocator &allocator, + bool is_return_value); + int execute_stmt_v2_interface(); + int handle_data_truncated(ObIAllocator &allocator); +private: + common::ObSEArray in_out_map_; + const char * proc_; + }; } //namespace sqlclient } diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.cpp b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.cpp index 3152d5e72..5134bc434 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.cpp +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.cpp @@ -644,13 +644,16 @@ int ObDbLinkProxy::dblink_execute_proc(ObISQLConnection *dblink_conn) } -int ObDbLinkProxy::dblink_prepare(sqlclient::ObISQLConnection *dblink_conn, const char *sql) +int ObDbLinkProxy::dblink_prepare(sqlclient::ObISQLConnection *dblink_conn, + const char *sql, + int64_t param_count, + ObIAllocator *allocator) { int ret = OB_SUCCESS; if (OB_ISNULL(dblink_conn) || OB_ISNULL(sql)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("null ptr", K(ret), KP(dblink_conn), KP(sql)); - } else if (OB_FAIL(dblink_conn->prepare(sql))) { + } else if (OB_FAIL(dblink_conn->prepare(sql, param_count, allocator))) { LOG_WARN("prepare to dblink failed", K(ret), K(ObString(sql))); } return ret; @@ -661,13 +664,14 @@ int ObDbLinkProxy::dblink_bind_basic_type_by_pos(sqlclient::ObISQLConnection *db void *param, int64_t param_size, int32_t datatype, - int32_t &indicator) + int32_t &indicator, + bool is_out_param) { int ret = OB_SUCCESS; if (OB_ISNULL(dblink_conn)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("null ptr", K(ret), KP(dblink_conn)); - } else if (OB_FAIL(dblink_conn->bind_basic_type_by_pos(position, param, param_size, datatype, indicator))) { + } else if (OB_FAIL(dblink_conn->bind_basic_type_by_pos(position, param, param_size, datatype, indicator, is_out_param))) { LOG_WARN("bind_basic_type_by_pos to dblink failed", K(ret)); } else { LOG_DEBUG("succ to bind_basic_type_by_pos dblink", K(ret)); @@ -741,14 +745,15 @@ int ObDbLinkProxy::dblink_execute_proc(const uint64_t tenant_id, const share::schema::ObRoutineInfo &routine_info, const common::ObIArray &udts, const ObTimeZoneInfo *tz_info, - ObObj *result) + ObObj *result, + bool is_sql) { int ret = OB_SUCCESS; if (OB_ISNULL(dblink_conn) || sql.empty()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("null ptr", K(ret), KP(dblink_conn), K(sql)); } else if (OB_FAIL(dblink_conn->execute_proc(tenant_id, allocator, params, sql, - routine_info, udts, tz_info, result))) { + routine_info, udts, tz_info, result, is_sql))) { LOG_WARN("call procedure to dblink failed", K(ret), K(dblink_conn), K(sql)); } else { LOG_DEBUG("succ to call procedure by dblink", K(sql)); diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h index ec19ac524..c8962c387 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h @@ -213,14 +213,19 @@ public: const share::schema::ObRoutineInfo &routine_info, const common::ObIArray &udts, const ObTimeZoneInfo *tz_info, - ObObj *result); - int dblink_prepare(sqlclient::ObISQLConnection *dblink_conn, const char *sql); + ObObj *result, + bool is_sql); + int dblink_prepare(sqlclient::ObISQLConnection *dblink_conn, + const char *sql, + int64_t param_count, + ObIAllocator *allocator = NULL); int dblink_bind_basic_type_by_pos(sqlclient::ObISQLConnection *dblink_conn, uint64_t position, void *param, int64_t param_size, int32_t datatype, - int32_t &indicator); + int32_t &indicator, + bool is_out_param); int dblink_bind_array_type_by_pos(sqlclient::ObISQLConnection *dblink_conn, uint64_t position, void *array, diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_statement.cpp b/deps/oblib/src/lib/mysqlclient/ob_mysql_statement.cpp index 18d993443..d19476adb 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_statement.cpp +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_statement.cpp @@ -54,7 +54,7 @@ ObMySQLConnection *ObMySQLStatement::get_connection() return conn_; } -int ObMySQLStatement::init(ObMySQLConnection &conn, const char *sql) +int ObMySQLStatement::init(ObMySQLConnection &conn, const char *sql, int64_t param_count) { int ret = OB_SUCCESS; conn_ = &conn; diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_statement.h b/deps/oblib/src/lib/mysqlclient/ob_mysql_statement.h index 9a40d9e79..85963517f 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_statement.h +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_statement.h @@ -31,7 +31,7 @@ public: ObMySQLConnection *get_connection(); MYSQL *get_stmt_handler(); MYSQL *get_conn_handler(); - int init(ObMySQLConnection &conn, const char *sql); + int init(ObMySQLConnection &conn, const char *sql, int64_t param_count = 0); /* * close statement diff --git a/src/observer/ob_inner_sql_connection.cpp b/src/observer/ob_inner_sql_connection.cpp index 35e48fb2d..6d868d5cb 100644 --- a/src/observer/ob_inner_sql_connection.cpp +++ b/src/observer/ob_inner_sql_connection.cpp @@ -1446,9 +1446,10 @@ int ObInnerSQLConnection::execute_proc(const uint64_t tenant_id, const share::schema::ObRoutineInfo &routine_info, const common::ObIArray &udts, const ObTimeZoneInfo *tz_info, - ObObj *result) + ObObj *result, + bool is_sql) { - UNUSEDx(tenant_id, allocator, params, sql, routine_info, udts, tz_info, result); + UNUSEDx(tenant_id, allocator, params, sql, routine_info, udts, tz_info, result, is_sql); int ret = OB_SUCCESS; return ret; } diff --git a/src/observer/ob_inner_sql_connection.h b/src/observer/ob_inner_sql_connection.h index 7fcf81f2f..028b83dcb 100644 --- a/src/observer/ob_inner_sql_connection.h +++ b/src/observer/ob_inner_sql_connection.h @@ -169,7 +169,8 @@ public: const share::schema::ObRoutineInfo &routine_info, const common::ObIArray &udts, const ObTimeZoneInfo *tz_info, - ObObj *result) override; + ObObj *result, + bool is_sql) override; virtual int start_transaction(const uint64_t &tenant_id, bool with_snap_shot = false) override; virtual sqlclient::ObCommonServerConnectionPool *get_common_server_pool() override; virtual int rollback() override; diff --git a/src/pl/dblink/ob_pl_dblink_guard.cpp b/src/pl/dblink/ob_pl_dblink_guard.cpp index b9b44f24c..67423a771 100644 --- a/src/pl/dblink/ob_pl_dblink_guard.cpp +++ b/src/pl/dblink/ob_pl_dblink_guard.cpp @@ -59,6 +59,7 @@ int ObPLDbLinkGuard::get_routine_infos_with_synonym(sql::ObSQLSessionInfo &sessi OZ (ObPLDblinkUtil::init_dblink(dblink_proxy, dblink_conn, session_info, schema_guard, dblink_name, link_type, false)); CK (OB_NOT_NULL(dblink_proxy)); CK (OB_NOT_NULL(dblink_conn)); + OZ (check_remote_version(*dblink_proxy, *dblink_conn)); OZ (ObPLDblinkUtil::print_full_name(alloc_, full_name, part1, part2, part3)); OZ (dblink_name_resolve(dblink_proxy, dblink_conn, @@ -195,6 +196,7 @@ int ObPLDbLinkGuard::get_dblink_type_with_synonym(sql::ObSQLSessionInfo &session const ObDbLinkSchema *dblink_schema = NULL; OZ (schema_guard.get_dblink_schema(MTL_ID(), dblink_name, dblink_schema), dblink_name); OV (OB_NOT_NULL(dblink_schema), OB_ERR_UNEXPECTED, dblink_name); + OZ (check_remote_version(*dblink_proxy, *dblink_conn)); OZ (ObPLDblinkUtil::print_full_name(alloc_, full_name, part1, part2, part3)); OZ (dblink_name_resolve(dblink_proxy, dblink_conn, @@ -343,10 +345,10 @@ int ObPLDbLinkGuard::dblink_name_resolve(common::ObDbLinkProxy *dblink_proxy, " :part1_type, " " object_number); " "end; "; - if (OB_ISNULL(dblink_proxy)) { + if (OB_ISNULL(dblink_proxy) || OB_ISNULL(dblink_conn) || OB_ISNULL(dblink_schema)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("dblink_proxy is NULL", K(ret)); - } else if (OB_FAIL(dblink_proxy->dblink_prepare(dblink_conn, call_proc))) { + LOG_WARN("param is NULL", K(ret), K(dblink_proxy), K(dblink_conn), K(dblink_schema)); + } else if (OB_FAIL(dblink_proxy->dblink_prepare(dblink_conn, call_proc, 7, &alloctor))) { LOG_WARN("prepare sql failed", K(ret), K(ObString(call_proc))); } if (OB_SUCC(ret)) { @@ -363,27 +365,29 @@ int ObPLDbLinkGuard::dblink_name_resolve(common::ObDbLinkProxy *dblink_proxy, memset(part1, 0, ident_size); memset(part2, 0, ident_size); memset(dblink, 0, ident_size); - int32_t oci_sql_str = static_cast(OciDataType::OCI_SQLT_STR); - int32_t oci_sql_int = static_cast(OciDataType::OCI_SQLT_INT); -#define BIND_BASIC_BY_POS(param_pos, param, param_size, param_type) \ + bool is_oracle = (DblinkDriverProto::DBLINK_DRV_OCI + == static_cast(dblink_schema->get_driver_proto())); +#define BIND_BASIC_BY_POS(param_pos, param, param_size, param_type, is_out_param) \ if (FAILEDx(dblink_proxy->dblink_bind_basic_type_by_pos(dblink_conn, \ param_pos, \ param, \ param_size, \ param_type, \ - indicator))) { \ + indicator, \ + is_out_param))) { \ LOG_WARN("bind param failed", K(ret), K(param_pos), K(param_size), K(param_type)); \ } - BIND_BASIC_BY_POS(1, full_name_copy.ptr(), static_cast(full_name_copy.length() + 1), oci_sql_str); - BIND_BASIC_BY_POS(2, &context, static_cast(sizeof(int)), oci_sql_int); - BIND_BASIC_BY_POS(3, schema1, ident_size, oci_sql_str); - BIND_BASIC_BY_POS(4, part1, ident_size, oci_sql_str); - BIND_BASIC_BY_POS(5, part2, ident_size, oci_sql_str); - BIND_BASIC_BY_POS(6, dblink, ident_size, oci_sql_str); - BIND_BASIC_BY_POS(7, &part1_type, static_cast(sizeof(int)), oci_sql_int); + BIND_BASIC_BY_POS(1, full_name_copy.ptr(), static_cast(full_name_copy.length() + (is_oracle ? 1 : 0)), ObObjType::ObVarcharType, false); + BIND_BASIC_BY_POS(2, &context, static_cast(sizeof(int)), ObObjType::ObInt32Type, false); + BIND_BASIC_BY_POS(3, schema1, ident_size, ObObjType::ObVarcharType, true); + BIND_BASIC_BY_POS(4, part1, ident_size, ObObjType::ObVarcharType, true); + BIND_BASIC_BY_POS(5, part2, ident_size, ObObjType::ObVarcharType, true); + BIND_BASIC_BY_POS(6, dblink, ident_size, ObObjType::ObVarcharType, true); + BIND_BASIC_BY_POS(7, &part1_type, static_cast(sizeof(int)), ObObjType::ObInt32Type, true); if (FAILEDx(dblink_proxy->dblink_execute_proc(dblink_conn))) { const DblinkDriverProto link_type = static_cast(dblink_schema->get_driver_proto()); - if (OB_ERR_ILL_OBJ_FLAG == ret) { + if (OB_ERR_ILL_OBJ_FLAG == ret + || OB_ERR_MISSING_IDENTIFIER == ret) { ret = OB_ERR_KEY_COLUMN_DOES_NOT_EXITS; LOG_WARN("invalid identifier", K(ret), K(full_name_copy)); LOG_USER_ERROR(OB_ERR_KEY_COLUMN_DOES_NOT_EXITS, full_name_copy.length(), full_name_copy.ptr()); @@ -593,6 +597,57 @@ int ObPLDbLinkGuard::get_dblink_info(const uint64_t dblink_id, } return ret; } + +int ObPLDbLinkGuard::check_remote_version(common::ObDbLinkProxy &dblink_proxy, + common::sqlclient::ObISQLConnection &dblink_conn) +{ + int ret = OB_SUCCESS; + if (DblinkDriverProto::DBLINK_DRV_OB == dblink_conn.get_dblink_driver_proto()) { + int part1 = 0; + int part2 = 0; + int part3 = 0; + int32_t ind = 0; + int64_t size = static_cast(sizeof(int)); + const char *anonymous_block = "declare " + " version_str varchar2(100); " + " begin " + " select OB_VERSION() into version_str from dual; " + " :1 := TO_NUMBER(REGEXP_SUBSTR(version_str, '[^.]+', 1, 1)); " + " :2 := TO_NUMBER(REGEXP_SUBSTR(version_str, '[^.]+', 1, 2)); " + " :3 := TO_NUMBER(REGEXP_SUBSTR(version_str, '[^.]+', 1, 3)); " + " end; "; + OZ (dblink_proxy.dblink_prepare(&dblink_conn, anonymous_block, 3, &alloc_)); + OZ (dblink_proxy.dblink_bind_basic_type_by_pos(&dblink_conn, 1, &part1, size, ObObjType::ObInt32Type, ind, true)); + OZ (dblink_proxy.dblink_bind_basic_type_by_pos(&dblink_conn, 2, &part2, size, ObObjType::ObInt32Type, ind, true)); + OZ (dblink_proxy.dblink_bind_basic_type_by_pos(&dblink_conn, 3, &part3, size, ObObjType::ObInt32Type, ind, true)); + OZ (dblink_proxy.dblink_execute_proc(&dblink_conn)); + if (OB_SUCC(ret)) { + bool not_support = false; + if (part1 < 4) { + not_support = true; + } else if (part1 == 4) { + if (part2 < 2) { + not_support = true; + } else if (part2 == 2) { + if (part3 < 4) { + not_support = true; + } + } else if (part2 == 3) { + if (part3 < 3) { + not_support = true; + } + } + } + if (not_support) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not support dblink", K(ret), K(part1), K(part2), K(part3)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, + "oceanbase PL dblink oceanbase PL oracle mode, remote database version less then 4.2.4.0"); + } + } + } + return ret; +} #endif } diff --git a/src/pl/dblink/ob_pl_dblink_guard.h b/src/pl/dblink/ob_pl_dblink_guard.h index cbd5fd843..cbfa10b0e 100644 --- a/src/pl/dblink/ob_pl_dblink_guard.h +++ b/src/pl/dblink/ob_pl_dblink_guard.h @@ -148,6 +148,9 @@ private: const common::ObString &udt_name, const pl::ObUserDefinedType *&udt); + int check_remote_version(common::ObDbLinkProxy &dblink_proxy, + common::sqlclient::ObISQLConnection &dblink_conn); + private: uint64_t next_link_object_id_; common::ObArenaAllocator &alloc_; diff --git a/src/pl/ob_pl.cpp b/src/pl/ob_pl.cpp index 4663e6378..eba9af041 100644 --- a/src/pl/ob_pl.cpp +++ b/src/pl/ob_pl.cpp @@ -48,6 +48,7 @@ #include "pl/debug/ob_pl_debugger_manager.h" #include "pl/opaque/ob_pl_json_type.h" #include "pl/ob_pl_profiler.h" +#include "pl/ob_pl_call_stack_trace.h" #endif #include "pl/pl_cache/ob_pl_cache_mgr.h" #include "sql/engine/dml/ob_trigger_handler.h" @@ -125,6 +126,8 @@ int ObPL::init(common::ObMySQLProxy &sql_proxy) (void*)(sql::ObSPIService::spi_clear_diagnostic_area)); jit::ObLLVMHelper::add_symbol(ObString("spi_end_trans"), (void*)(sql::ObSPIService::spi_end_trans)); + jit::ObLLVMHelper::add_symbol(ObString("spi_update_location"), + (void*)(sql::ObSPIService::spi_update_location)); jit::ObLLVMHelper::add_symbol(ObString("spi_set_pl_exception_code"), (void*)(sql::ObSPIService::spi_set_pl_exception_code)); jit::ObLLVMHelper::add_symbol(ObString("spi_get_pl_exception_code"), @@ -1442,41 +1445,25 @@ int ObPLContext::set_subprogram_var_from_local( } #ifdef OB_BUILD_ORACLE_PL -int ObPLContext::get_exact_error_msg(ObIArray &error_trace, - ObIArray &call_stack, - common::ObSqlString &err_msg) +ObPLCallStackTrace* ObPLContext::get_call_stack_trace() { int ret = OB_SUCCESS; - err_msg.reuse(); - if (0 == call_stack.count() || 0 == error_trace.count()) { - LOG_INFO("error message is incomplete. ", K(call_stack.count()), K(error_trace.count())); - } else { - int64_t i = call_stack.count() - 1; - for (; OB_SUCC(ret) && i>=0 && OB_NOT_NULL(call_stack.at(i)); i--) { - int64_t line = call_stack.at(i)->get_line_number(); - int64_t col = call_stack.at(i)->get_column_number(); - for (int64_t j = 0 ; j < error_trace.count(); j++) { - if (OB_NOT_NULL(error_trace.at(j)) - && call_stack.at(i)->handler == error_trace.at(j)->handler) { - line = error_trace.at(j)->get_line_number(); - col = error_trace.at(j)->get_column_number(); - break; - } - } - if (OB_FAIL(err_msg.append_fmt("\nat "))) { - LOG_WARN("fail to get call stack name.", K(ret)); - } else if (OB_FAIL(err_msg.append_fmt("%.*s , line : %ld, col : %ld", - call_stack.at(i)->object_name.length(), - call_stack.at(i)->object_name.ptr(), line, col))) { - LOG_WARN("fail to get call stack name.", K(ret), - K(call_stack.at(i)->object_name), K(line), K(col)); - } else { - LOG_DEBUG("exact error msg: ", K(call_stack.at(i)->object_name), - K(line), K(col)); - } + if (OB_ISNULL(call_stack_trace_)) { + ObIAllocator *allocator = nullptr; + ObPLCallStackTrace *call_stack_trace = nullptr; + CK (get_exec_stack().count() > 0); + CK (OB_NOT_NULL(get_exec_stack().at(0))); + CK (OB_NOT_NULL(allocator = get_exec_stack().at(0)->get_allocator())); + OX (call_stack_trace = reinterpret_cast(allocator->alloc(sizeof(ObPLCallStackTrace)))); + if (OB_SUCC(ret) && OB_ISNULL(call_stack_trace)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory for call stack trace", K(ret)); + } else { + new (call_stack_trace) ObPLCallStackTrace(); + call_stack_trace_ = call_stack_trace; } } - return ret; + return call_stack_trace_; } #endif @@ -3671,6 +3658,16 @@ int ObPLExecState::init(const ParamStore *params, bool is_anonymous) // init params may use exec stack, need append to pl context first CK (OB_NOT_NULL(top_context_ = ctx_.exec_ctx_->get_my_session()->get_pl_context())); + if (OB_SUCC(ret) + && !is_called_from_sql_ + && !is_anonymous + && !ObTriggerInfo::is_trigger_package_id(func_.get_package_id()) + && top_context_->get_exec_stack().count() > 0) { + ObPLExecState *parent = top_context_->get_exec_stack().at(top_context_->get_exec_stack().count() - 1); + CK (OB_NOT_NULL(parent)); + OX (parent->set_loc(this->get_loc())); + OX (this->set_loc(0)); + } OZ (top_context_->get_exec_stack().push_back(this)); OZ (init_params(params, is_anonymous)); @@ -3943,7 +3940,9 @@ int ObPL::simple_execute(ObPLExecCtx *ctx, int64_t argc, int64_t *argv) ObIArray &sql_infos = func->get_sql_infos(); for (int64_t i = 0; OB_SUCC(ret) && i < sql_infos.count(); ++i) { ObPLSqlInfo &sql_info = sql_infos.at(i); - if (sql_info.params_.empty()) { + OZ (ObSPIService::spi_update_location(ctx, sql_info.loc_)); + if (OB_FAIL(ret)) { + } else if (sql_info.params_.empty()) { OZ (ObSPIService::spi_query( ctx, sql_infos.at(i).sql_.ptr(), sql_infos.at(i).stmt_type_, sql_info.into_.get_data(), sql_info.into_.count(), @@ -3969,7 +3968,9 @@ int ObPL::simple_execute(ObPLExecCtx *ctx, int64_t argc, int64_t *argv) } #ifdef OB_BUILD_ORACLE_PL - ObPLEH::eh_adjust_call_stack(func, ctx->pl_ctx_, sql_infos.at(i).loc_, ret); + if (OB_NOT_NULL(ctx->pl_ctx_)) { + ObPLEH::eh_adjust_call_stack(*ctx->pl_ctx_, sql_info.loc_, ret); + } #endif } @@ -4202,29 +4203,19 @@ int ObPLExecState::execute() } } - if (OB_SUCC(ret)) { - ObSQLSessionInfo *session_info = ctx_.exec_ctx_->get_my_session(); #ifdef OB_BUILD_ORACLE_PL - int64_t call_cnt = top_context_->get_call_stack().count(); - int64_t err_cnt = top_context_->get_error_trace().count(); - if (OB_SUCC(ret) && err_cnt > 0) { - DbmsUtilityHelper::BtInfo* top_error = top_context_->get_error_trace().at(err_cnt - 1); - if (func_.get_action() == top_error->handler) { - OX (top_context_->get_error_trace().pop_back()); + if (OB_FAIL(ret)&& top_call_ && OB_NOT_NULL(ctx_.exec_ctx_->get_my_session()) && OB_NOT_NULL(top_context_)) { + ObSQLSessionInfo *session_info = ctx_.exec_ctx_->get_my_session(); + if (OB_ISNULL(top_context_->get_call_stack_trace())) { + LOG_WARN("call stack trace is null when pl exception found.", K(ret)); + } else { + int tmp_ret = top_context_->get_call_stack_trace()->format_exception_stack(session_info->get_pl_exact_err_msg(), true); + if (tmp_ret != OB_SUCCESS) { + LOG_WARN("failed to format exception stack for session info", K(ret), K(tmp_ret)); } } - if (OB_SUCC(ret) && call_cnt > 0) { - DbmsUtilityHelper::BtInfo* top_call = top_context_->get_call_stack().at(call_cnt - 1); - if (func_.get_action() == top_call->handler) { - OX (top_context_->get_call_stack().pop_back()); - } - } - } else if (!inner_call_) { - ObPLContext::get_exact_error_msg(top_context_->get_error_trace(), - top_context_->get_call_stack(), - ctx_.exec_ctx_->get_my_session()->get_pl_exact_err_msg()); -#endif } +#endif } return ret; } diff --git a/src/pl/ob_pl.h b/src/pl/ob_pl.h index d5fc73549..053798839 100644 --- a/src/pl/ob_pl.h +++ b/src/pl/ob_pl.h @@ -35,6 +35,9 @@ #include "sql/plan_cache/ob_cache_object_factory.h" #include "pl/pl_cache/ob_pl_cache.h" #include "pl/pl_cache/ob_pl_cache_object.h" +#ifdef OB_BUILD_ORACLE_PL +#include "pl/ob_pl_call_stack_trace.h" +#endif namespace test { @@ -715,7 +718,6 @@ public: top_call_(top_call), need_reset_physical_plan_(false), top_context_(NULL), - current_line_(OB_INVALID_INDEX), loc_(loc), is_called_from_sql_(is_called_from_sql), dwarf_helper_(NULL), @@ -759,9 +761,8 @@ public: inline bool is_top_call() const { return top_call_; } inline uint64_t get_loc() const { return loc_; } inline void set_loc(uint64_t loc) { loc_ = loc; } - inline uint64_t get_line_number() { return static_cast(get_loc() >> 32 & 0xffffffff); } - inline uint64_t get_current_line() { return get_line_number() + 1; } + inline uint64_t get_current_line() { return (get_loc() >> 32) + 1; } inline void set_current_line(int64_t current_line) { @@ -821,8 +822,6 @@ private: bool need_reset_physical_plan_; ObPLContext *top_context_; - - int64_t current_line_; uint64_t loc_; // combine of line and column number bool is_called_from_sql_; jit::ObDWARFHelper *dwarf_helper_; // for decode dwarf debuginfo @@ -832,11 +831,16 @@ private: ObPLProfilerTimeStack *profiler_time_stack_; }; +class ObPLCallStackTrace; class ObPLContext { friend class LinkPLStackGuard; public: - ObPLContext() { reset(); } + ObPLContext() +#ifdef OB_BUILD_ORACLE_PL + : call_stack_trace_(nullptr) +#endif + { reset(); } virtual ~ObPLContext() { reset(); } void reset() { @@ -861,7 +865,10 @@ public: old_in_definer_ = false; has_output_arguments_ = false; #ifdef OB_BUILD_ORACLE_PL - call_trace_.reset(); + if (call_stack_trace_ != nullptr) { + call_stack_trace_->~ObPLCallStackTrace(); + } + call_stack_trace_ = nullptr; #endif old_worker_timeout_ts_ = 0; old_phy_plan_timeout_ts_ = 0; @@ -945,12 +952,7 @@ public: void reset_role_id_array(int &ret); ObIArray &get_exec_stack() { return exec_stack_; } -#ifdef OB_BUILD_ORACLE_PL - ObIArray &get_error_trace() { return call_trace_.error_trace; } - ObIArray &get_call_stack() { return call_trace_.call_stack; } - void set_call_trace_error_code(int errcode) { call_trace_.err_code = errcode; } - int get_call_trace_error_code() const { return call_trace_.err_code; } -#endif + ObPLExecState *get_current_state() { return exec_stack_.empty() ? NULL : exec_stack_.at(exec_stack_.count() - 1); @@ -963,11 +965,6 @@ public: { return NULL == get_current_state() ? NULL : &get_current_state()->get_exec_ctx(); } -#ifdef OB_BUILD_ORACLE_PL - static int get_exact_error_msg(ObIArray &error_trace, - ObIArray &call_stack, - common::ObSqlString &err_msg); -#endif bool has_output_arguments() { return has_output_arguments_; } void set_has_output_arguments(bool has_output_arguments) { @@ -990,6 +987,10 @@ public: sql::ObExecContext *get_my_exec_ctx() { return my_exec_ctx_; } ObCurTraceId::TraceId get_trace_id() const { return trace_id_; } +#ifdef OB_BUILD_ORACLE_PL + ObPLCallStackTrace *get_call_stack_trace(); +#endif + private: ObPLContext* get_stack_pl_ctx(); void set_parent_stack_ctx(pl::ObPLContext *parent_stack_ctx) @@ -1040,7 +1041,7 @@ private: common::ObSEArray exec_stack_; #ifdef OB_BUILD_ORACLE_PL - DbmsUtilityHelper::BackTrace call_trace_; + ObPLCallStackTrace *call_stack_trace_; #endif ObPLContext *parent_stack_ctx_; ObPLContext *top_stack_ctx_; diff --git a/src/pl/ob_pl_code_generator.cpp b/src/pl/ob_pl_code_generator.cpp index a9866f44e..ba14fd272 100644 --- a/src/pl/ob_pl_code_generator.cpp +++ b/src/pl/ob_pl_code_generator.cpp @@ -359,6 +359,7 @@ int ObPLCodeGenerateVisitor::visit(const ObPLDeclareVarStmt &s) OZ (generator_.extract_obobj_ptr_from_objparam(into_obj, dest_datum)); OZ (var->get_type().generate_copy(generator_, *s.get_namespace(), allocator, src_datum, dest_datum, + s.get_location(), s.get_block()->in_notfound(), s.get_block()->in_warning(), OB_INVALID_ID)); @@ -563,6 +564,7 @@ int ObPLCodeGenerateVisitor::visit(const ObPLAssignStmt &s) allocator, src_datum, dest_datum, + s.get_location(), s.get_block()->in_notfound(), s.get_block()->in_warning(), package_id)); @@ -578,6 +580,7 @@ int ObPLCodeGenerateVisitor::visit(const ObPLAssignStmt &s) allocator, src_datum, var_addr, + s.get_location(), s.get_block()->in_notfound(), s.get_block()->in_warning(), package_id)); @@ -589,6 +592,7 @@ int ObPLCodeGenerateVisitor::visit(const ObPLAssignStmt &s) allocator, src_datum, dest_datum, + s.get_location(), s.get_block()->in_notfound(), s.get_block()->in_warning(), package_id)); @@ -647,6 +651,7 @@ int ObPLCodeGenerateVisitor::visit(const ObPLAssignStmt &s) allocator, src_datum, dest_datum, + s.get_location(), s.get_block()->in_notfound(), s.get_block()->in_warning(), package_id)); @@ -717,6 +722,7 @@ int ObPLCodeGenerateVisitor::visit(const ObPLAssignStmt &s) allocator, src_datum, dest_datum, + s.get_location(), s.get_block()->in_notfound(), s.get_block()->in_warning(), package_id)); @@ -1600,6 +1606,7 @@ int ObPLCodeGenerateVisitor::visit(const ObPLReturnStmt &s) allocator, src_datum, dest_datum, + s.get_location(), s.get_block()->in_notfound(), s.get_block()->in_warning(), OB_INVALID_ID)); @@ -1707,6 +1714,7 @@ int ObPLCodeGenerateVisitor::visit(const ObPLExecuteStmt &s) OZ (generator_.get_helper().set_insert_point(generator_.get_current())); OZ (generator_.set_debug_location(s)); OZ (generator_.generate_goto_label(s)); + OZ (generator_.generate_update_location(s)); OZ (generator_.generate_spi_pl_profiler_before_record(s)); OZ (generator_.get_helper().get_llvm_type(ObIntType, int_type)); @@ -1808,6 +1816,7 @@ int ObPLCodeGenerateVisitor::visit(const ObPLExecuteStmt &s) allocator, src_datum, dest_datum, + s.get_location(), s.get_block()->in_notfound(), s.get_block()->in_warning(), OB_INVALID_ID)); @@ -2321,6 +2330,7 @@ int ObPLCodeGenerateVisitor::visit(const ObPLDeclareHandlerStmt &s) ObLLVMValue result; ObLLVMValue p_old_sqlcode, is_need_pop_warning_buf; ObLLVMType int_type; + ObLLVMValue level; OZ (generator_.set_current(case_branch)); #ifndef NDEBUG OZ (generator_.get_helper().get_int64(1111+i, int_value)); @@ -2350,10 +2360,14 @@ int ObPLCodeGenerateVisitor::visit(const ObPLDeclareHandlerStmt &s) // 设置当前ExceptionCode到SQLCODE OX (args.reset()); + // OZ (generator_.get_helper().get_llvm_type(ObIntType, int_type)); + // OZ (generator_.get_helper().create_alloca(ObString("level"), int_type, p_level)); + OZ (generator_.get_helper().get_int32(s.get_level(), level)); OZ (generator_.get_helper().get_int8(false, is_need_pop_warning_buf)); OZ (args.push_back(generator_.get_vars().at(generator_.CTX_IDX))); OZ (args.push_back(code)); OZ (args.push_back(is_need_pop_warning_buf)); + OZ (args.push_back(level)); OZ (generator_.get_helper().create_call(ObString("spi_set_pl_exception_code"), generator_.get_spi_service().spi_set_pl_exception_code_, args, result)); OZ (generator_.check_success(result, s.get_stmt_id(), s.get_block()->in_notfound(), s.get_block()->in_warning())); @@ -2405,6 +2419,7 @@ int ObPLCodeGenerateVisitor::visit(const ObPLDeclareHandlerStmt &s) OZ (args.push_back(generator_.get_vars().at(generator_.CTX_IDX))); OZ (args.push_back(old_code)); OZ (args.push_back(is_need_pop_warning_buf)); + OZ (args.push_back(level)); OZ (generator_.get_helper().create_call(ObString("spi_set_pl_exception_code"), generator_.get_spi_service().spi_set_pl_exception_code_, args, result)); OZ (generator_.check_success(result, s.get_stmt_id(), s.get_block()->in_notfound(), s.get_block()->in_warning())); } @@ -2777,6 +2792,7 @@ int ObPLCodeGenerateVisitor::visit(const ObPLCallStmt &s) allocator, src_datum, dest_datum, + s.get_location(), s.get_block()->in_notfound(), s.get_block()->in_warning(), OB_INVALID_ID)); @@ -2893,6 +2909,7 @@ int ObPLCodeGenerateVisitor::visit(const ObPLOpenStmt &s) OZ (generator_.generate_goto_label(s)); OZ (generator_.generate_spi_pl_profiler_before_record(s)); CK (OB_NOT_NULL(s.get_cursor())); + OZ (generator_.generate_update_location(s)); OZ (generator_.generate_open(static_cast(s), s.get_cursor()->get_value(), s.get_cursor()->get_package_id(), @@ -2906,6 +2923,7 @@ int ObPLCodeGenerateVisitor::visit(const ObPLOpenForStmt &s) { int ret = OB_SUCCESS; OZ (generator_.generate_goto_label(s)); + OZ (generator_.generate_update_location(s)); OZ (generator_.generate_spi_pl_profiler_before_record(s)); OZ (generator_.generate_open_for(s)); OZ (generator_.generate_spi_pl_profiler_after_record(s)); @@ -2916,6 +2934,7 @@ int ObPLCodeGenerateVisitor::visit(const ObPLFetchStmt &s) { int ret = OB_SUCCESS; ObLLVMValue ret_err; + OZ (generator_.generate_update_location(s)); OZ (generator_.generate_goto_label(s)); OZ (generator_.generate_spi_pl_profiler_before_record(s)); @@ -3947,6 +3966,14 @@ int ObPLCodeGenerator::init_spi_service() OZ (helper_.create_function(ObString("spi_end_trans"), ft, spi_service_.spi_end_trans_)); } + if (OB_SUCC(ret)) { + arg_types.reset(); + OZ (arg_types.push_back(pl_exec_context_pointer_type)); + OZ (arg_types.push_back(int64_type)); + OZ (ObLLVMFunctionType::get(int32_type, arg_types, ft)); + OZ (helper_.create_function(ObString("spi_update_location"), ft, spi_service_.spi_update_location_)); + } + if (OB_SUCC(ret)) { arg_types.reset(); if (OB_FAIL(arg_types.push_back(pl_exec_context_pointer_type))) { //函数第一个参数必须是基础环境信息隐藏参数 @@ -4258,6 +4285,8 @@ int ObPLCodeGenerator::init_spi_service() LOG_WARN("push_back error", K(ret)); } else if (OB_FAIL(arg_types.push_back(bool_type))) { LOG_WARN("push_back error", K(ret)); + } else if (OB_FAIL(arg_types.push_back(int32_type))) { + LOG_WARN("push_back error", K(ret)); } else if (OB_FAIL(ObLLVMFunctionType::get(int32_type, arg_types, ft))) { LOG_WARN("failed to get function type", K(ret)); } else if (OB_FAIL(helper_.create_function(ObString("spi_set_pl_exception_code"), ft, spi_service_.spi_set_pl_exception_code_))) { @@ -5639,6 +5668,25 @@ int ObPLCodeGenerator::generate_bound_and_check(const ObPLForLoopStmt &s, return ret; } +int ObPLCodeGenerator::generate_update_location(const ObPLStmt &s) +{ + int ret = OB_SUCCESS; + if (NULL == get_current().get_v()) { + //控制流已断，后面的语句不再处理 + } else { + ObSEArray args; + ObLLVMValue location; + ObLLVMValue ret_err; + OZ (args.push_back(get_vars().at(CTX_IDX))); + OZ (get_helper().get_int64(s.get_location(), location)); + OZ (args.push_back(location)); + OZ (get_helper().create_call(ObString("spi_update_location"), get_spi_service().spi_update_location_, args, ret_err)); + OZ (check_success( + ret_err, s.get_stmt_id(), s.get_block()->in_notfound(), s.get_block()->in_warning())); + } + return ret; +} + int ObPLCodeGenerator::generate_sql(const ObPLSqlStmt &s, ObLLVMValue &ret_err) { int ret = OB_SUCCESS; @@ -5647,6 +5695,7 @@ int ObPLCodeGenerator::generate_sql(const ObPLSqlStmt &s, ObLLVMValue &ret_err) } else { OZ (get_helper().set_insert_point(get_current())); OZ (set_debug_location(s)); + OZ (generate_update_location(s)); } if (OB_FAIL(ret)) { } else if (stmt::T_END_TRANS == s.get_stmt_type()) { @@ -8315,6 +8364,7 @@ int ObPLCodeGenerator::generate_out_param( allocator, src_datum, dest_datum, + s.get_location(), s.get_block()->in_notfound(), s.get_block()->in_warning(), OB_INVALID_ID)); @@ -8385,6 +8435,7 @@ int ObPLCodeGenerator::generate_out_param( allocator, src_datum, dest_datum, + s.get_location(), s.get_block()->in_notfound(), s.get_block()->in_warning(), package_id)); @@ -8396,6 +8447,7 @@ int ObPLCodeGenerator::generate_out_param( allocator, src_datum, dest_datum, + s.get_location(), s.get_block()->in_notfound(), s.get_block()->in_warning(), package_id)); diff --git a/src/pl/ob_pl_code_generator.h b/src/pl/ob_pl_code_generator.h index a2f2bd9c9..baf435ea0 100644 --- a/src/pl/ob_pl_code_generator.h +++ b/src/pl/ob_pl_code_generator.h @@ -82,6 +82,7 @@ public: jit::ObLLVMFunction spi_construct_collection_; jit::ObLLVMFunction spi_clear_diagnostic_area_; jit::ObLLVMFunction spi_end_trans_; + jit::ObLLVMFunction spi_update_location_; jit::ObLLVMFunction spi_set_pl_exception_code_; jit::ObLLVMFunction spi_get_pl_exception_code_; jit::ObLLVMFunction spi_check_early_exit_; @@ -351,6 +352,7 @@ public: int generate_update_package_changed_info( const ObPLStmt &s, uint64_t package_id, uint64_t var_idx); int restart_cg_when_goto_dest(const ObPLStmt &stmt); + int generate_update_location(const ObPLStmt &s); public: inline jit::ObLLVMHelper &get_helper() { return helper_; } inline jit::ObLLVMBasicBlock &get_entry() { return entry_; } diff --git a/src/pl/ob_pl_compile.cpp b/src/pl/ob_pl_compile.cpp index 112ba1d7c..1645fd734 100644 --- a/src/pl/ob_pl_compile.cpp +++ b/src/pl/ob_pl_compile.cpp @@ -147,7 +147,11 @@ int ObPLCompiler::init_anonymous_ast( } else { data_type.reset(); data_type.set_accuracy(params->at(i).get_accuracy()); - data_type.set_meta_type(params->at(i).get_meta()); + if (params->at(i).is_null()) { + data_type.set_meta_type(params->at(i).get_param_meta()); + } else { + data_type.set_meta_type(params->at(i).get_meta()); + } pl_type.reset(); int64_t int_value = 0; // 参数化整型常量按照会按照numbger来生成param diff --git a/src/pl/ob_pl_exception_handling.cpp b/src/pl/ob_pl_exception_handling.cpp index 1629ca585..620ebcd39 100644 --- a/src/pl/ob_pl_exception_handling.cpp +++ b/src/pl/ob_pl_exception_handling.cpp @@ -12,6 +12,9 @@ #define USING_LOG_PREFIX PL #include "ob_pl_exception_handling.h" +#ifdef OB_BUILD_ORACLE_PL +#include "pl/ob_pl_call_stack_trace.h" +#endif #include "share/ob_errno.h" #include "lib/utility/ob_macro_utils.h" #include "lib/oblog/ob_log_module.h" @@ -121,62 +124,15 @@ ObPLException::ObPLException(int64_t error_code) } #ifdef OB_BUILD_ORACLE_PL -int ObPLEH::eh_adjust_call_stack( - ObPLFunction *pl_func, ObPLContext *pl_ctx, uint64_t loc, int error_code) +int ObPLEH::eh_adjust_call_stack(ObPLContext &pl_ctx, uint64_t location, int err_code) { int ret = OB_SUCCESS; - - if (error_code != OB_SUCCESS - && OB_ALLOCATE_MEMORY_FAILED != error_code - && OB_TENANT_OUT_OF_MEM != error_code - && OB_EXCEED_MEM_LIMIT != error_code - && OB_NOT_NULL(pl_ctx) - && lib::is_oracle_mode()) { - - ObIAllocator *allocator = NULL; - CK (OB_NOT_NULL(pl_ctx->get_exec_stack().at(0))); - CK (OB_NOT_NULL(pl_ctx->get_exec_stack().at(0)->get_exec_ctx().status_)); - OX (*(pl_ctx->get_exec_stack().at(0)->get_exec_ctx().status_) = error_code); - OX (allocator = pl_ctx->get_exec_stack().at(0)->get_exec_ctx().allocator_); - CK (OB_NOT_NULL(allocator)); - if (0 == pl_ctx->get_call_stack().count()) { - OZ (DbmsUtilityHelper::get_backtrace(*allocator, *pl_ctx, pl_ctx->get_call_stack())); - } - if (OB_SUCC(ret) && OB_NOT_NULL(pl_func)) { - bool is_exist = false; - DbmsUtilityHelper::BtInfo *tbi = NULL; - DbmsUtilityHelper::BtInfo *bi = NULL; - // check if same error code exist, this is possible - for (int64_t i = 0; - OB_SUCC(ret) && !is_exist && i < pl_ctx->get_error_trace().count(); ++i) { - tbi = pl_ctx->get_error_trace().at(i); - CK (OB_NOT_NULL(tbi)); - OX (is_exist = (tbi->error_code == error_code)); - } - if (OB_SUCC(ret) && OB_NOT_NULL(allocator)) { - bi = static_cast( - allocator->alloc(sizeof(DbmsUtilityHelper::BtInfo))); - } - if (OB_SUCC(ret) && !is_exist && OB_NOT_NULL(bi)) { - bi->init(); - bi->error_type = DbmsUtilityHelper::BTErrorType::ERROR_INFO; // 0: stack info, 1 : error info - bi->loc = loc + (1LL << 32); - bi->handler = pl_func->get_action(); - bi->error_code = error_code; - ObSqlString exception_msg; - OZ (pl_ctx->get_error_trace().push_back(bi)); - for (int64_t i = 0; OB_SUCC(ret) && i < pl_ctx->get_error_trace().count(); ++i) { - if (pl_ctx->get_error_trace().at(i)->handler == bi->handler) { - pl_ctx->get_error_trace().at(i)->loc = loc + (1LL << 32); - } - } - OZ (pl_ctx->get_exact_error_msg(pl_ctx->get_error_trace(), - pl_ctx->get_call_stack(), - exception_msg)); - LOG_WARN("got exception! ", KR(error_code), K(exception_msg)); - } - } - pl_ctx->set_call_trace_error_code(ret); + int64_t stack_depth = 0; + if (lib::is_oracle_mode() && OB_NOT_NULL(pl_ctx.get_call_stack_trace())) { + CK ((stack_depth = pl_ctx.get_exec_stack().count()) > 0); + CK (OB_NOT_NULL(pl_ctx.get_exec_stack().at(stack_depth - 1))); + OX (pl_ctx.get_exec_stack().at(stack_depth - 1)->set_loc(location)); + OZ (pl_ctx.get_call_stack_trace()->format_exception_stack(err_code, pl_ctx.get_exec_stack())); } return ret; } @@ -235,8 +191,8 @@ ObUnwindException *ObPLEH::eh_create_exception(int64_t pl_context, tl_eptr = unwind; #ifdef OB_BUILD_ORACLE_PL - OZ (eh_adjust_call_stack( - reinterpret_cast(pl_function), pl_ctx, loc, value->error_code_)); + CK (OB_NOT_NULL(pl_ctx)); + OZ (eh_adjust_call_stack(*pl_ctx, loc, value->error_code_)); #endif } diff --git a/src/pl/ob_pl_exception_handling.h b/src/pl/ob_pl_exception_handling.h index 2180646c0..1bd4aed25 100644 --- a/src/pl/ob_pl_exception_handling.h +++ b/src/pl/ob_pl_exception_handling.h @@ -65,6 +65,8 @@ public: static int eh_convert_exception(bool oracle_mode, int oberr, ObPLConditionType *type, int64_t *error_code, const char **sql_state, int64_t *str_len); static ObPLConditionType eh_classify_exception(const char *sql_state); + static int eh_adjust_call_stack(ObPLContext &pl_ctx, uint64_t location, int err_code); + #ifdef OB_BUILD_ORACLE_PL static int eh_adjust_call_stack( ObPLFunction *pl_func, ObPLContext *pl_ctx, uint64_t loc, int error_code); diff --git a/src/pl/ob_pl_package_manager.cpp b/src/pl/ob_pl_package_manager.cpp index 7ced6b2c1..ee8e7683e 100644 --- a/src/pl/ob_pl_package_manager.cpp +++ b/src/pl/ob_pl_package_manager.cpp @@ -810,6 +810,68 @@ int ObPLPackageManager::get_package_expr(const ObPLResolveCtx &resolve_ctx, return ret; } +int ObPLPackageManager::get_package_expr(const ObPLResolveCtx &resolve_ctx, + ObRawExprFactory &expr_factory, + uint64_t package_id, + int64_t expr_idx, + ObRawExpr *&expr) +{ + int ret = OB_SUCCESS; + const ObPackageInfo *package_spec_info = NULL; + const ObPackageInfo *package_body_info = NULL; + CK (package_id != OB_INVALID_ID); + CK (expr_idx != OB_INVALID_ID); + OZ (get_package_schema_info(resolve_ctx.schema_guard_, package_id, package_spec_info, package_body_info)); + CK (OB_NOT_NULL(package_spec_info)); + CK (package_spec_info->get_package_id() == package_id); + if (OB_SUCC(ret)) { + ObPLCompiler compiler(resolve_ctx.allocator_, + resolve_ctx.session_info_, + resolve_ctx.schema_guard_, + resolve_ctx.package_guard_, + resolve_ctx.sql_proxy_); + const uint64_t tenant_id = package_spec_info->get_tenant_id(); + uint64_t db_id = package_spec_info->get_database_id(); + uint64_t package_spec_id = package_spec_info->get_package_id(); + ObPLBlockNS *null_parent_ns = NULL; + const ObDatabaseSchema *db_schema = NULL; + uint64_t effective_tenant_id = resolve_ctx.session_info_.get_effective_tenant_id(); + HEAP_VAR(ObPLPackageAST, package_spec_ast, resolve_ctx.allocator_) { + ObString source; + if (package_spec_info->is_for_trigger()) { + OZ (ObTriggerInfo::gen_package_source(package_spec_info->get_tenant_id(), + package_spec_info->get_package_id(), + source, + PACKAGE_TYPE, + resolve_ctx.schema_guard_, + resolve_ctx.allocator_)); + } else { + source = package_spec_info->get_source(); + } + OZ (ObSQLUtils::convert_sql_text_from_schema_for_resolve( + resolve_ctx.allocator_, resolve_ctx.session_info_.get_dtc_params(), source)); + OZ (resolve_ctx.schema_guard_.get_database_schema(tenant_id, db_id, db_schema)); + OZ (package_spec_ast.init(db_schema->get_database_name_str(), + package_spec_info->get_package_name(), + PL_PACKAGE_SPEC, + package_spec_info->get_database_id(), + package_spec_id, + package_spec_info->get_schema_version(), + NULL)); + { + ObPLCompilerEnvGuard guard( + *package_spec_info, resolve_ctx.session_info_, resolve_ctx.schema_guard_, ret); + OZ (compiler.analyze_package(source, null_parent_ns, + package_spec_ast, package_spec_info->is_for_trigger())); + } + CK (expr_idx >= 0 && package_spec_ast.get_exprs().count() > expr_idx); + CK (OB_NOT_NULL(package_spec_ast.get_expr(expr_idx))); + OZ (ObPLExprCopier::copy_expr(expr_factory, package_spec_ast.get_expr(expr_idx), expr)); + } + } + return ret; +} + int ObPLPackageManager::get_package_cursor(const ObPLResolveCtx &resolve_ctx, uint64_t package_id, int64_t cursor_id, @@ -1225,7 +1287,8 @@ int ObPLPackageManager::load_package_body(const ObPLResolveCtx &resolve_ctx, // destoried by map's destructor ObCacheObjGuard* cacheobj_guard = NULL; void* buf = NULL; - if (OB_ISNULL(buf = resolve_ctx.package_guard_.alloc_.alloc(sizeof(ObCacheObjGuard)))) { + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(buf = resolve_ctx.package_guard_.alloc_.alloc(sizeof(ObCacheObjGuard)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to allocate memory.", K(ret)); } else if (FALSE_IT(cacheobj_guard = new (buf)ObCacheObjGuard(PACKAGE_BODY_HANDLE))) { diff --git a/src/pl/ob_pl_package_manager.h b/src/pl/ob_pl_package_manager.h index 6ab4605c6..2cdb54f43 100644 --- a/src/pl/ob_pl_package_manager.h +++ b/src/pl/ob_pl_package_manager.h @@ -33,6 +33,8 @@ class ObSQLSessionInfo; class ObSqlExpression; class ObBasicSessionInfo; class ObSessionVariable; +class ObRawExprFactory; +class ObRawExpr; } namespace share @@ -93,6 +95,8 @@ public: int get_package_expr(const ObPLResolveCtx &resolve_ctx, uint64_t package_id, int64_t expr_idx, sql::ObSqlExpression *&expr); + int get_package_expr(const ObPLResolveCtx &resolve_ctx, sql::ObRawExprFactory &expr_factory, + uint64_t package_id, int64_t expr_idx, sql::ObRawExpr *&expr); int get_package_cursor(const ObPLResolveCtx &resolve_ctx, uint64_t package_id, diff --git a/src/pl/ob_pl_resolver.cpp b/src/pl/ob_pl_resolver.cpp index 7a6ba5218..7366dd993 100644 --- a/src/pl/ob_pl_resolver.cpp +++ b/src/pl/ob_pl_resolver.cpp @@ -4063,12 +4063,13 @@ bool ObPLResolver::is_question_mark_value(ObRawExpr *into_expr, ObPLBlockNS *ns) return ret; } -int ObPLResolver::set_question_mark_type(ObRawExpr *into_expr, ObPLBlockNS *ns, const ObPLDataType *type) +int ObPLResolver::set_question_mark_type(ObRawExpr *into_expr, ObPLBlockNS *ns, const ObPLDataType *type, bool need_check) { int ret = OB_SUCCESS; ObConstRawExpr *const_expr = NULL; const ObPLVar *var = NULL; ObExprResType res_type; + bool need_set = true; CK (OB_NOT_NULL(into_expr)); CK (OB_NOT_NULL(type)); CK (OB_NOT_NULL(ns)); @@ -4077,19 +4078,51 @@ int ObPLResolver::set_question_mark_type(ObRawExpr *into_expr, ObPLBlockNS *ns, CK (OB_NOT_NULL(ns->get_symbol_table())); CK (OB_NOT_NULL(var = ns->get_symbol_table()->get_symbol(const_expr->get_value().get_unknown()))); CK (var->get_name().prefix_match(ANONYMOUS_ARG)); - OX ((const_cast(var))->set_type(*type)); - OX ((const_cast(var))->set_readonly(false)); - if (OB_FAIL(ret)) { - } else if (type->is_obj_type()) { - CK (OB_NOT_NULL(type->get_data_type())); - OX (res_type.set_meta(type->get_data_type()->get_meta_type())); - OX (res_type.set_accuracy(type->get_data_type()->get_accuracy())); - } else { - OX (res_type.set_ext()); - OX (res_type.set_extend_type(type->get_type())); - OX (res_type.set_udt_id(type->get_user_type_id())); + if (OB_SUCC(ret) && need_check) { + const ObPLDataType *left = type; + const ObPLDataType *right = &((const_cast(var))->get_type()); + if (left->is_obj_type() && + right->is_obj_type() && + NULL != right->get_data_type() && + !right->get_data_type()->get_meta_type().is_ext()) { + if (right->get_data_type()->get_meta_type().is_null()) { + need_set = true; + } else { + CK (OB_NOT_NULL(left->get_data_type())); + OX (need_set = !cast_supported(left->get_data_type()->get_obj_type(), + left->get_data_type()->get_collation_type(), + right->get_data_type()->get_obj_type(), + right->get_data_type()->get_collation_type())); + } + } else if ((!left->is_obj_type() || + (left->get_data_type() != NULL && left->get_data_type()->get_meta_type().is_ext())) + && + (!right->is_obj_type() || + (right->get_data_type() != NULL && right->get_data_type()->get_meta_type().is_ext()))) { + uint64_t left_udt_id = (NULL == left->get_data_type()) ? left->get_user_type_id() + : left->get_data_type()->get_udt_id(); + uint64_t right_udt_id = (NULL == right->get_data_type()) ? right->get_user_type_id() + : right->get_data_type()->get_udt_id(); + if (left_udt_id == right_udt_id) { + need_set = false; + } + } + } + if (OB_SUCC(ret) && need_set) { + OX ((const_cast(var))->set_type(*type)); + OX ((const_cast(var))->set_readonly(false)); + if (OB_FAIL(ret)) { + } else if (type->is_obj_type()) { + CK (OB_NOT_NULL(type->get_data_type())); + OX (res_type.set_meta(type->get_data_type()->get_meta_type())); + OX (res_type.set_accuracy(type->get_data_type()->get_accuracy())); + } else { + OX (res_type.set_ext()); + OX (res_type.set_extend_type(type->get_type())); + OX (res_type.set_udt_id(type->get_user_type_id())); + } + OX (into_expr->set_result_type(res_type)); } - OX (into_expr->set_result_type(res_type)); return ret; } @@ -7557,6 +7590,26 @@ int ObPLResolver::resolve_cparams(ObIArray &exprs, if (OB_SUCC(ret) && (PL_PARAM_INOUT == param_mode || PL_PARAM_OUT == param_mode)) { OZ (resolve_inout_param(params.at(i), param_mode, out_idx), K(i), K(params), K(exprs)); + if (OB_SUCC(ret) && is_question_mark_value(params.at(i), &(current_block_->get_namespace()))) { + ObPLDataType data_type; + if (param_info->is_schema_routine_param()) { + const ObRoutineParam* iparam = static_cast(param_info); + OZ (pl::ObPLDataType::transform_from_iparam(iparam, + resolve_ctx_.schema_guard_, + resolve_ctx_.session_info_, + resolve_ctx_.allocator_, + resolve_ctx_.sql_proxy_, + data_type)); + const ObUserDefinedType *user_type = NULL; + if (OB_SUCC(ret) && OB_INVALID_ID != data_type.get_user_type_id()) { + OZ (current_block_->get_namespace().get_pl_data_type_by_id(data_type.get_user_type_id(), user_type)); + } + } else { + const ObPLRoutineParam* iparam = static_cast(param_info); + OX (data_type = iparam->get_type()); + } + OZ (set_question_mark_type(params.at(i), &(current_block_->get_namespace()), &data_type, true)); + } } if (OB_SUCC(ret) && OB_LIKELY(OB_INVALID_INDEX != expr_idx.at(i))) { @@ -9924,10 +9977,9 @@ int ObPLResolver::set_udf_expr_line_number(ObRawExpr *expr, uint64_t line_number if (expr->is_udf_expr()) { ObUDFRawExpr *udf_expr = static_cast(expr); udf_expr->set_loc(line_number); - } else { - for (int64_t i = 0; i < expr->get_children_count(); ++i) { - OZ (SMART_CALL(set_udf_expr_line_number(expr->get_param_expr(i), line_number))); - } + } + for (int64_t i = 0; OB_SUCC(ret) && i < expr->get_children_count(); ++i) { + OZ (SMART_CALL(set_udf_expr_line_number(expr->get_param_expr(i), line_number))); } } return ret; @@ -10960,10 +11012,8 @@ int ObPLResolver::resolve_inner_call( } } } else { - ret = OB_NOT_SUPPORTED; - LOG_WARN("not support inner call function type", K(ret), K(access_idxs.at(idx_cnt - 1))); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "inner call function type"); - LOG_USER_ERROR(OB_NOT_SUPPORTED, const_cast (func.get_name().ptr())); + ret = OB_ERR_UNDEFINED; + LOG_WARN("object is not a procedure or is undefined", K(ret), K(access_idxs)); } } } @@ -11521,7 +11571,8 @@ int ObPLResolver::resolve_dblink_udf(sql::ObQualifiedName &q_name, OX (dblink_name = dblink_schema->get_dblink_name()); } OZ (schema_checker.init(resolve_ctx_.schema_guard_, resolve_ctx_.session_info_.get_sessid())); - OZ (ObRawExprUtils::resolve_udf_common_info(db_name, + OZ (ObRawExprUtils::resolve_udf_common_info(sch_routine_info->is_dblink_routine() ? + sch_routine_info->get_dblink_db_name() : db_name, sch_routine_info->is_dblink_routine() ? sch_routine_info->get_dblink_pkg_name() : pkg_name, routine_id, @@ -11614,8 +11665,8 @@ int ObPLResolver::resolve_construct(const ObQualifiedName &q_name, int ret = OB_SUCCESS; if (user_type.is_nested_table_type() || user_type.is_varray_type()) { OZ (resolve_collection_construct(q_name, udf_info, &user_type, expr)); - } else if (user_type.is_object_type() && user_type.is_udt_type()) { - OZ (resolve_object_construct(q_name, udf_info, &user_type, expr)); + } else if (user_type.is_record_type() || user_type.is_opaque_type()) { + OZ (resolve_object_construct(q_name, udf_info, user_type, expr)); } else { ret = OB_NOT_SUPPORTED; LOG_WARN("only allow collection construct and user define record construct", K(ret), K(user_type)); @@ -11624,51 +11675,21 @@ int ObPLResolver::resolve_construct(const ObQualifiedName &q_name, return ret; } -int ObPLResolver::resolve_construct(const ObQualifiedName &q_name, - const ObUDFInfo &udf_info, - ObRawExpr *&expr) -{ - int ret = OB_SUCCESS; - const ObUserDefinedType *user_type = NULL; - CK (OB_NOT_NULL(current_block_)); - ObString package_name - = udf_info.is_udf_udt_cons() - ? ObString("") - : udf_info.udf_package_ != udf_info.udf_name_ ? udf_info.udf_package_ : ObString(""); - OZ (current_block_->get_namespace().get_pl_data_type_by_name( - resolve_ctx_, udf_info.udf_database_, package_name, - udf_info.udf_name_, user_type)); - CK (OB_NOT_NULL(user_type)); - // reset the error code, we have to try udf. - if (OB_FAIL(ret)) { - LOG_WARN("failed to get type", K(ret), K(package_name), K(udf_info), K(q_name)); - ret = OB_ERR_SP_UNDECLARED_TYPE; - } - if (OB_SUCC(ret)) { - if (user_type->is_nested_table_type() - || user_type->is_varray_type()) { - OZ (resolve_collection_construct(q_name, udf_info, user_type, expr)); - } else if (user_type->is_object_type() && user_type->is_udt_type()) { - OZ (resolve_object_construct(q_name, udf_info, user_type, expr)); - } else { - ret = OB_NOT_SUPPORTED; - LOG_WARN("only allow collection construct and user define record construct", K(ret), KPC(user_type)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "Constructs other than collection constructs and user-defined record constructs"); - } - } - return ret; -} - int ObPLResolver::resolve_object_construct(const sql::ObQualifiedName &q_name, const sql::ObUDFInfo &udf_info, - const ObUserDefinedType *user_type, + const ObUserDefinedType &user_type, ObRawExpr *&expr) { int ret = OB_SUCCESS; - uint64_t type_id = OB_INVALID_ID; - OX (type_id = user_type->get_user_type_id()); - if (OB_SUCC(ret)) { - ObUDFInfo &uinfo = const_cast(udf_info); + ObUDFInfo &uinfo = const_cast(udf_info); + bool try_udf_constructor = OB_NOT_NULL(uinfo.ref_expr_) && OB_INVALID_ID == uinfo.ref_expr_->get_udf_id() && user_type.is_object_type(); + bool use_buildin_constructor = !try_udf_constructor ? true : false; + + // try object or opaque type user define constructor first. + if (OB_NOT_NULL(uinfo.ref_expr_) && uinfo.ref_expr_->get_udf_id() != OB_INVALID_ID) { + CK (OB_NOT_NULL(expr = uinfo.ref_expr_)); + OX (use_buildin_constructor = false); + } else if (try_udf_constructor) { if (uinfo.udf_package_.empty()) { // object type name should same as constructor name uinfo.udf_package_ = uinfo.udf_name_; @@ -11680,34 +11701,23 @@ int ObPLResolver::resolve_object_construct(const sql::ObQualifiedName &q_name, ret = OB_ERR_UNEXPECTED; LOG_WARN("type name is not same as constructor name", K(uinfo), K(ret)); } - } else { - // do nothing } - if (OB_SUCC(ret) && OB_NOT_NULL(uinfo.ref_expr_) && uinfo.ref_expr_->get_udf_id() == OB_INVALID_ID) { + if (OB_SUCC(ret)) { SMART_VAR(ObPLFunctionAST, dummy_ast, resolve_ctx_.allocator_) { ObSEArray access_idxs; OZ (resolve_udf_info(uinfo, access_idxs, dummy_ast)); } } - // - // check is use default constructor - // for example: object(a number, constructor object(a varchar)); - // begin v object := object(4); resolve_udf will pick the user defined cons, - // actually, here we need to use the default construtor, that is: reolsve record construct - // on the other side: object(a number, constructor(a number) constructor object(a varchar)); - // resolve_udf will pick the right one, we dont need to resolve record_construct. - - bool use_buildin_default_constructor = false; - if (OB_SUCC(ret) - && !uinfo.is_udt_overload_default_cons() - && !user_type->is_opaque_type()) { // opaque type has no member, do not check - const ObRecordType *object_type = NULL; - CK (OB_NOT_NULL(object_type = dynamic_cast(user_type))); - - // must have same attribute and param, exclude self param - if (udf_info.ref_expr_->get_param_exprs().count() - 1 == object_type->get_member_count()) { - use_buildin_default_constructor = true; + // check if can use default buildin constructor, only object type should check this. + if (OB_SUCC(ret) && !uinfo.is_udt_overload_default_cons() && !user_type.is_opaque_type()) { + const ObRecordType *object_type = static_cast(&user_type); + CK (OB_NOT_NULL(object_type)); + // object type buildin constructor has no default parameter value, + // so only allow same parameter count with record member count. + // also, after resolve resolve_udf, first argument must be self parameter, here, ignore first argument + if (OB_SUCC(ret) && (udf_info.ref_expr_->get_param_exprs().count() - 1) == object_type->get_member_count()) { + use_buildin_constructor = true; for (int64_t i = 1; OB_SUCC(ret) && i < udf_info.ref_expr_->get_param_exprs().count(); ++i) { const ObRawExpr *param_expr = udf_info.ref_expr_->get_param_exprs().at(i); if (OB_ISNULL(param_expr)) { @@ -11721,7 +11731,7 @@ int ObPLResolver::resolve_object_construct(const sql::ObQualifiedName &q_name, && (param_res_type.get_type() == pl_type->get_meta_type()->get_type())) { // do nothing } else { - use_buildin_default_constructor = false; + use_buildin_constructor = false; break; } } @@ -11729,19 +11739,22 @@ int ObPLResolver::resolve_object_construct(const sql::ObQualifiedName &q_name, } } OX (expr = udf_info.ref_expr_); - // if cant find user define construtor, try default construct - if ((OB_SUCCESS == ret && use_buildin_default_constructor) - || OB_ERR_CALL_WRONG_ARG == ret - || OB_ERR_SP_WRONG_ARG_NUM == ret - || OB_ERR_FUNCTION_UNKNOWN == ret - || OB_ERR_SP_UNDECLARED_VAR == ret - || OB_ERR_INVALID_TYPE_FOR_OP == ret - || OB_ERR_PACKAGE_DOSE_NOT_EXIST == ret - || OB_ERR_SP_DOES_NOT_EXIST == ret) { - ret = OB_SUCCESS; - pl_reset_warning_buffer(); - OZ (resolve_record_construct(q_name, udf_info, user_type, expr)); - } + } + + // try buildin constructor, for record type and object type only. + if (user_type.is_record_type() + && OB_NOT_NULL(udf_info.ref_expr_) + && ((OB_SUCCESS == ret && use_buildin_constructor) + || OB_ERR_CALL_WRONG_ARG == ret + || OB_ERR_SP_WRONG_ARG_NUM == ret + || OB_ERR_FUNCTION_UNKNOWN == ret + || OB_ERR_SP_UNDECLARED_VAR == ret + || OB_ERR_INVALID_TYPE_FOR_OP == ret + || OB_ERR_PACKAGE_DOSE_NOT_EXIST == ret + || OB_ERR_SP_DOES_NOT_EXIST == ret)) { + ret = OB_SUCCESS; + pl_reset_warning_buffer(); + OZ (resolve_record_construct(q_name, udf_info, &user_type, expr)); } return ret; } @@ -11756,27 +11769,18 @@ int ObPLResolver::resolve_record_construct(const ObQualifiedName &q_name, ObObjectConstructRawExpr *object_expr = NULL; ObExprResType res_type; int64_t rowsize = 0; - const ObUDTTypeInfo *udt_info = NULL; - uint64_t tenant_id = OB_INVALID_ID; - bool is_udt_type = false; + int64_t param_pos = udf_info.is_udf_udt_cons() && udf_info.is_contain_self_param_ ? 1 : 0; // ignore self param + int64_t total_assign_cnt = 0; CK (OB_NOT_NULL(user_type)); - OX (is_udt_type = user_type->is_udt_type()); CK (OB_NOT_NULL(udf_info.ref_expr_)); CK (OB_NOT_NULL(object_type = static_cast(user_type))); - if (OB_SUCC(ret) && udf_info.param_names_.count() > 0) { // 构造函数暂时不允许使用=>赋值 - ret = OB_ERR_CALL_WRONG_ARG; - LOG_WARN("PLS-00306: wrong number or types of arguments in call procedure", K(ret)); - } + OZ (expr_factory_.create_raw_expr(T_FUN_PL_OBJECT_CONSTRUCT, object_expr)); + CK (OB_NOT_NULL(object_expr)); if (OB_SUCC(ret)) { int64_t param_cnt = udf_info.ref_expr_->get_param_exprs().count(); - int64_t member_cnt = object_type->is_opaque_type() ? 0 : object_type->get_member_count(); - bool is_opaque_cons_and_no_self_param - = object_type->is_opaque_type() && (param_cnt - 2) == member_cnt && udf_info.is_udf_udt_cons(); - - if (OB_SUCC(ret) - && ((!udf_info.is_udf_udt_cons() && param_cnt != member_cnt) - || (udf_info.is_udf_udt_cons() && param_cnt - 1 != member_cnt)) - && !is_opaque_cons_and_no_self_param) { + int64_t member_cnt = object_type->get_member_count(); + if ((udf_info.is_udf_udt_cons() && (param_cnt - 1) != member_cnt) + || (!udf_info.is_udf_udt_cons() && param_cnt > member_cnt)) { ret = OB_ERR_CALL_WRONG_ARG; LOG_WARN("PLS-00306: wrong number or types of arguments in call", K(ret), @@ -11785,8 +11789,63 @@ int ObPLResolver::resolve_record_construct(const ObQualifiedName &q_name, K(object_type->get_member_count())); } } - OZ (expr_factory_.create_raw_expr(T_FUN_PL_OBJECT_CONSTRUCT, object_expr)); - CK (OB_NOT_NULL(object_expr)); + for (; OB_SUCC(ret) && param_pos < udf_info.ref_expr_->get_param_exprs().count(); ++param_pos) { + OZ (object_expr->add_param_expr(udf_info.ref_expr_->get_param_exprs().at(param_pos))); + } + ObConstRawExpr *null_expr = NULL; + OZ (expr_factory_.create_raw_expr(T_NULL, null_expr)); + CK (OB_NOT_NULL(null_expr)); + CK (udf_info.param_names_.count() == udf_info.param_exprs_.count()); + for (; OB_SUCC(ret) && param_pos < object_type->get_member_count(); ++param_pos) { + bool found = false; + const ObString *member_name = object_type->get_record_member_name(param_pos); + const ObRecordMember *member = object_type->get_record_member(param_pos); + int64_t i = 0; + CK (OB_NOT_NULL(member_name)); + CK (OB_NOT_NULL(member)); + for (; OB_SUCC(ret) && i < udf_info.param_names_.count(); ++i) { + if (ObCharset::case_compat_mode_equal(*member_name, udf_info.param_names_.at(i))) { + found = true; + total_assign_cnt++; + break; + } + } + if (OB_FAIL(ret)) { + } else if (found) { + OZ (object_expr->add_param_expr(udf_info.param_exprs_.at(i))); + } else if (member->get_default() != OB_INVALID_INDEX) { + if (OB_NOT_NULL(member->get_default_expr())) { + OZ (object_expr->add_param_expr(member->get_default_expr())); + } else { + ObPLPackageManager &package_manager = + resolve_ctx_.session_info_.get_pl_engine()->get_package_manager(); + ObRawExpr *expr = NULL; + OZ (package_manager.get_package_expr(resolve_ctx_, + expr_factory_, + extract_package_id(user_type->get_user_type_id()), + member->get_default(), + expr)); + OZ (object_expr->add_param_expr(expr)); + } + } else if (!object_type->is_udt_type()) { + OZ (object_expr->add_param_expr(null_expr)); + } else { + ret = OB_ERR_CALL_WRONG_ARG; + LOG_WARN("PLS-00306: wrong number or types of arguments in call", + K(ret), + K(q_name), + K(udf_info.ref_expr_->get_param_exprs().count()), + K(object_type->get_member_count())); + } + } + if (OB_SUCC(ret) && total_assign_cnt != udf_info.param_names_.count()) { + ret = OB_ERR_CALL_WRONG_ARG; + LOG_WARN("PLS-00306: wrong number or types of arguments in call", + K(ret), + K(q_name), + K(udf_info.ref_expr_->get_param_exprs().count()), + K(object_type->get_member_count())); + } OZ (user_type->get_size(pl::PL_TYPE_ROW_SIZE, rowsize)); OX (object_expr->set_rowsize(rowsize)); OX (res_type.set_type(ObExtendType)); @@ -11794,7 +11853,9 @@ int ObPLResolver::resolve_record_construct(const ObQualifiedName &q_name, OX (res_type.set_udt_id(user_type->get_user_type_id())); OX (object_expr->set_udt_id(user_type->get_user_type_id())); OX (object_expr->set_result_type(res_type)); - if (is_udt_type) { + if (OB_SUCC(ret) && user_type->is_udt_type()) { + const ObUDTTypeInfo *udt_info = NULL; + uint64_t tenant_id = OB_INVALID_ID; OX (tenant_id = get_tenant_id_by_object_id(user_type->get_user_type_id())); OZ (resolve_ctx_.schema_guard_.get_udt_info( tenant_id, user_type->get_user_type_id(), udt_info)); @@ -11821,10 +11882,6 @@ int ObPLResolver::resolve_record_construct(const ObQualifiedName &q_name, } OZ (object_expr->set_access_names(q_name.access_idents_)); OX (object_expr->set_func_name(object_type->get_name())); - int64_t i = udf_info.is_udf_udt_cons() && udf_info.is_contain_self_param_ ? 1 : 0; // ignore the self param - for (; OB_SUCC(ret) && i < udf_info.ref_expr_->get_param_exprs().count(); ++i) { - OZ (object_expr->add_param_expr(udf_info.ref_expr_->get_param_exprs().at(i))); - } OX (expr = object_expr); return ret; } @@ -12406,18 +12463,14 @@ int ObPLResolver::resolve_udf_info( #ifdef OB_BUILD_ORACLE_PL if (OB_SUCC(ret) && package_routine_info->is_udt_cons()) { - bool is_overloaded = false; const ObUDTTypeInfo *udt_info = NULL; const uint64_t tenant_id = package_routine_info->get_tenant_id(); OZ (resolve_ctx_.schema_guard_.get_udt_info( tenant_id, package_routine_info->get_pkg_id(), udt_info)); CK (OB_NOT_NULL(udt_info)); - OZ (ObPLUDTObjectManager::check_overload_default_cons(package_routine_info, - udt_info, - is_overloaded)); - if (is_overloaded) { - OX (udf_info.set_is_udt_overload_default_cons()); - } + OZ (ObPLUDTObjectManager::check_overload_default_cons(udf_info, + package_routine_info, + udt_info)); } #endif @@ -12510,20 +12563,15 @@ int ObPLResolver::resolve_udf_info( OX (schema_version = udt_info->get_schema_version()); // to check is this overload the default constructor - if (OB_SUCC(ret) && schema_routine_info->is_udt_cons()) { - bool is_overloaded = false; - OZ (ObPLUDTObjectManager::check_overload_default_cons(schema_routine_info, - udt_info, - is_overloaded)); - if (is_overloaded) { - OX (udf_info.set_is_udt_overload_default_cons()); - } - } + OZ (ObPLUDTObjectManager::check_overload_default_cons(udf_info, + schema_routine_info, + udt_info)); } #endif } GET_DBLINK_NAME(schema_routine_info); - OZ (ObRawExprUtils::resolve_udf_common_info(db_name, + OZ (ObRawExprUtils::resolve_udf_common_info(schema_routine_info->is_dblink_routine() ? + schema_routine_info->get_dblink_db_name() : db_name, schema_routine_info->is_dblink_routine() ? schema_routine_info->get_dblink_pkg_name() : package_name, routine_id, @@ -13628,7 +13676,7 @@ int ObPLResolver::resolve_local_var(const ObString &var_name, } if (OB_SUCC(ret) && for_write) { const ObPLVar *var = NULL; - const ObPLSymbolTable *symbol_table = NULL; + const ObPLSymbolTable *symbol_table = ns.get_symbol_table(); CK (OB_NOT_NULL(symbol_table)) CK (OB_NOT_NULL(var = symbol_table->get_symbol(var_index))); if (OB_SUCC(ret) && var->is_readonly()) { @@ -14329,11 +14377,10 @@ int ObPLResolver::resolve_construct(ObObjAccessIdent &access_ident, ObRawExpr* expr = NULL; const ObUserDefinedType *user_type = NULL; ObObjAccessIdx access_idx; - if (!access_ident.is_pl_udf()) { + if ((!access_ident.is_pl_udf() && !access_ident.has_brackets_) || OB_ISNULL(access_ident.udf_info_.ref_expr_)) { ret = OB_ERR_UNDEFINED; LOG_WARN("object is not a procedure or is undefined", K(ret), K(access_ident)); } - OV (access_ident.is_pl_udf(), OB_ERR_UNEXPECTED, K(access_ident)); OZ (ns.get_pl_data_type_by_id(user_type_id, user_type)); CK (OB_NOT_NULL(user_type)); if (OB_FAIL(ret)) { diff --git a/src/pl/ob_pl_resolver.h b/src/pl/ob_pl_resolver.h index 4e497f035..75467cb96 100644 --- a/src/pl/ob_pl_resolver.h +++ b/src/pl/ob_pl_resolver.h @@ -342,16 +342,13 @@ public: const ObUDFInfo &udf_info, const ObUserDefinedType &user_type, ObRawExpr *&expr); - int resolve_construct(const sql::ObQualifiedName &q_name, - const sql::ObUDFInfo &udf_info, - ObRawExpr *&expr); int resolve_record_construct(const sql::ObQualifiedName &q_name, const sql::ObUDFInfo &udf_info, const ObUserDefinedType *user_type, ObRawExpr *&expr); int resolve_object_construct(const sql::ObQualifiedName &q_name, const sql::ObUDFInfo &udf_info, - const ObUserDefinedType *user_type, + const ObUserDefinedType &user_type, ObRawExpr *&expr); int resolve_collection_construct(const sql::ObQualifiedName &q_name, const sql::ObUDFInfo &udf_info, @@ -468,7 +465,7 @@ public: const ObPLDataType &ret_type); static int build_pl_integer_type(ObPLIntegerType type, ObPLDataType &data_type); static bool is_question_mark_value(ObRawExpr *into_expr, ObPLBlockNS *ns); - static int set_question_mark_type(ObRawExpr *into_expr, ObPLBlockNS *ns, const ObPLDataType *type); + static int set_question_mark_type(ObRawExpr *into_expr, ObPLBlockNS *ns, const ObPLDataType *type, bool need_check = false); static int build_obj_access_func_name(const ObIArray &access_idxs, diff --git a/src/pl/ob_pl_type.cpp b/src/pl/ob_pl_type.cpp index 74ac9ae96..ca7383b7d 100644 --- a/src/pl/ob_pl_type.cpp +++ b/src/pl/ob_pl_type.cpp @@ -678,6 +678,7 @@ int ObPLDataType::generate_copy(ObPLCodeGenerator &generator, jit::ObLLVMValue &allocator, jit::ObLLVMValue &src, jit::ObLLVMValue &dest, + uint64_t location, bool in_notfound, bool in_warning, uint64_t package_id) const @@ -709,7 +710,7 @@ int ObPLDataType::generate_copy(ObPLCodeGenerator &generator, jit::ObLLVMValue ret_err; if (OB_FAIL(generator.get_helper().create_call(ObString("spi_copy_datum"), generator.get_spi_service().spi_copy_datum_, args, ret_err))) { LOG_WARN("failed to create call", K(ret)); - } else if (OB_FAIL(generator.check_success(ret_err, OB_INVALID_ID, in_notfound, in_warning))) { + } else if (OB_FAIL(generator.check_success(ret_err, location, in_notfound, in_warning))) { LOG_WARN("failed to check success", K(ret)); } else { /*do nothing*/ } } diff --git a/src/pl/ob_pl_type.h b/src/pl/ob_pl_type.h index 1b46e5bb2..f08afe2b3 100644 --- a/src/pl/ob_pl_type.h +++ b/src/pl/ob_pl_type.h @@ -506,6 +506,7 @@ public: jit::ObLLVMValue &allocator, jit::ObLLVMValue &src, jit::ObLLVMValue &dest, + uint64_t location, bool in_notfound, bool in_warning, uint64_t package_id = OB_INVALID_ID) const; diff --git a/src/pl/ob_pl_user_type.cpp b/src/pl/ob_pl_user_type.cpp index 8bdc871e1..a3ede8d75 100644 --- a/src/pl/ob_pl_user_type.cpp +++ b/src/pl/ob_pl_user_type.cpp @@ -68,7 +68,7 @@ int ObUserDefinedType::generate_default_value( int ObUserDefinedType::generate_copy( ObPLCodeGenerator &generator, const ObPLBlockNS &ns, jit::ObLLVMValue &allocator, jit::ObLLVMValue &src, jit::ObLLVMValue &dest, - bool in_notfound, bool in_warning, uint64_t package_id) const + uint64_t location, bool in_notfound, bool in_warning, uint64_t package_id) const { UNUSEDx(generator, ns, allocator, src, dest, in_notfound, in_warning, package_id); LOG_WARN_RET(OB_NOT_SUPPORTED, "Call virtual func of ObUserDefinedType! May forgot implement in SubClass", K(this)); @@ -842,13 +842,14 @@ int ObUserDefinedSubType::generate_copy(ObPLCodeGenerator &generator, jit::ObLLVMValue &allocator, jit::ObLLVMValue &src, jit::ObLLVMValue &dest, + uint64_t location, bool in_notfound, bool in_warning, uint64_t package_id) const { int ret = OB_SUCCESS; OZ (SMART_CALL(base_type_.generate_copy( - generator, ns, allocator, src, dest, in_notfound, in_warning, package_id))); + generator, ns, allocator, src, dest, location, in_notfound, in_warning, package_id))); return ret; } @@ -1507,6 +1508,7 @@ int ObRecordType::generate_default_value(ObPLCodeGenerator &generator, allocator, src_datum, ptr_elem, + stmt->get_location(), stmt->get_block()->in_notfound(), stmt->get_block()->in_warning(), OB_INVALID_ID)); @@ -2941,6 +2943,14 @@ int ObCollectionType::deserialize(ObSchemaGetterGuard &schema_guard, LOG_WARN("allocate memory failed", K(ret), KPC(this), KPC(table), K(element_init_size), K(count), K(max_count)); } + if (OB_SUCC(ret)) { + // initialize all ObObj + ObObj *obj = reinterpret_cast(table_data); + CK (OB_NOT_NULL(obj)); + for (int64_t i = 0; OB_SUCC(ret) && i < max_count; i++) { + obj[i].reset(); + } + } if (OB_SUCC(ret) && element_type_.is_record_type()) { int table_data_pos_tmp = table_data_pos; for (int i = 0; OB_SUCC(ret) && i < count; ++i) { diff --git a/src/pl/ob_pl_user_type.h b/src/pl/ob_pl_user_type.h index 4ab0adb90..1adc3161e 100644 --- a/src/pl/ob_pl_user_type.h +++ b/src/pl/ob_pl_user_type.h @@ -81,6 +81,7 @@ public: jit::ObLLVMValue &allocator, jit::ObLLVMValue &src, jit::ObLLVMValue &dest, + uint64_t location, bool in_notfound, bool in_warning, uint64_t package_id = OB_INVALID_ID) const; @@ -205,6 +206,7 @@ public: jit::ObLLVMValue &allocator, jit::ObLLVMValue &src, jit::ObLLVMValue &dest, + uint64_t location, bool in_notfound, bool in_warning, uint64_t package_id = OB_INVALID_ID) const; diff --git a/src/share/inner_table/ob_inner_table_schema.21551_21600.cpp b/src/share/inner_table/ob_inner_table_schema.21551_21600.cpp index 64cbb72d6..410bf9af7 100644 --- a/src/share/inner_table/ob_inner_table_schema.21551_21600.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21551_21600.cpp @@ -1225,6 +1225,56 @@ int ObInnerTableSchema::v_ob_nic_info_schema(ObTableSchema &table_schema) return ret; } +int ObInnerTableSchema::func_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_MYSQL_SCHEMA_ID); + table_schema.set_table_id(OB_FUNC_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_FUNC_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT name, ret, dl, type FROM oceanbase.__all_func )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::gv_ob_nic_info_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.h b/src/share/inner_table/ob_inner_table_schema.h index 16492e107..039d421b8 100644 --- a/src/share/inner_table/ob_inner_table_schema.h +++ b/src/share/inner_table/ob_inner_table_schema.h @@ -1802,6 +1802,7 @@ public: static int innodb_metrics_schema(share::schema::ObTableSchema &table_schema); static int events_schema(share::schema::ObTableSchema &table_schema); static int v_ob_nic_info_schema(share::schema::ObTableSchema &table_schema); + static int func_schema(share::schema::ObTableSchema &table_schema); static int gv_ob_nic_info_schema(share::schema::ObTableSchema &table_schema); static int dba_scheduler_job_run_details_schema(share::schema::ObTableSchema &table_schema); static int cdb_scheduler_job_run_details_schema(share::schema::ObTableSchema &table_schema); @@ -4667,6 +4668,7 @@ const schema_create_func sys_view_schema_creators [] = { ObInnerTableSchema::innodb_metrics_schema, ObInnerTableSchema::events_schema, ObInnerTableSchema::v_ob_nic_info_schema, + ObInnerTableSchema::func_schema, ObInnerTableSchema::gv_ob_nic_info_schema, ObInnerTableSchema::dba_scheduler_job_run_details_schema, ObInnerTableSchema::cdb_scheduler_job_run_details_schema, @@ -6382,6 +6384,7 @@ const uint64_t tenant_space_tables [] = { OB_INNODB_METRICS_TID, OB_EVENTS_TID, OB_V_OB_NIC_INFO_TID, + OB_FUNC_TID, OB_GV_OB_NIC_INFO_TID, OB_DBA_SCHEDULER_JOB_RUN_DETAILS_TID, OB_INNODB_SYS_FIELDS_TID, @@ -9013,6 +9016,7 @@ const char* const tenant_space_table_names [] = { OB_INNODB_METRICS_TNAME, OB_EVENTS_TNAME, OB_V_OB_NIC_INFO_TNAME, + OB_FUNC_TNAME, OB_GV_OB_NIC_INFO_TNAME, OB_DBA_SCHEDULER_JOB_RUN_DETAILS_TNAME, OB_INNODB_SYS_FIELDS_TNAME, @@ -13166,10 +13170,10 @@ static inline int get_sys_table_lob_aux_schema(const uint64_t tid, const int64_t OB_CORE_TABLE_COUNT = 4; const int64_t OB_SYS_TABLE_COUNT = 297; const int64_t OB_VIRTUAL_TABLE_COUNT = 825; -const int64_t OB_SYS_VIEW_COUNT = 915; -const int64_t OB_SYS_TENANT_TABLE_COUNT = 2042; +const int64_t OB_SYS_VIEW_COUNT = 916; +const int64_t OB_SYS_TENANT_TABLE_COUNT = 2043; const int64_t OB_CORE_SCHEMA_VERSION = 1; -const int64_t OB_BOOTSTRAP_SCHEMA_VERSION = 2045; +const int64_t OB_BOOTSTRAP_SCHEMA_VERSION = 2046; } // end namespace share } // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema_constants.h b/src/share/inner_table/ob_inner_table_schema_constants.h index 06953678d..26653165e 100644 --- a/src/share/inner_table/ob_inner_table_schema_constants.h +++ b/src/share/inner_table/ob_inner_table_schema_constants.h @@ -1502,6 +1502,7 @@ const uint64_t OB_INNODB_TEMP_TABLE_INFO_TID = 21578; // "INNODB_TEMP_TABLE_INFO const uint64_t OB_INNODB_METRICS_TID = 21579; // "INNODB_METRICS" const uint64_t OB_EVENTS_TID = 21580; // "EVENTS" const uint64_t OB_V_OB_NIC_INFO_TID = 21581; // "V$OB_NIC_INFO" +const uint64_t OB_FUNC_TID = 21585; // "func" const uint64_t OB_GV_OB_NIC_INFO_TID = 21586; // "GV$OB_NIC_INFO" const uint64_t OB_DBA_SCHEDULER_JOB_RUN_DETAILS_TID = 21589; // "DBA_SCHEDULER_JOB_RUN_DETAILS" const uint64_t OB_CDB_SCHEDULER_JOB_RUN_DETAILS_TID = 21590; // "CDB_SCHEDULER_JOB_RUN_DETAILS" @@ -4254,6 +4255,7 @@ const char *const OB_INNODB_TEMP_TABLE_INFO_TNAME = "INNODB_TEMP_TABLE_INFO"; const char *const OB_INNODB_METRICS_TNAME = "INNODB_METRICS"; const char *const OB_EVENTS_TNAME = "EVENTS"; const char *const OB_V_OB_NIC_INFO_TNAME = "V$OB_NIC_INFO"; +const char *const OB_FUNC_TNAME = "func"; const char *const OB_GV_OB_NIC_INFO_TNAME = "GV$OB_NIC_INFO"; const char *const OB_DBA_SCHEDULER_JOB_RUN_DETAILS_TNAME = "DBA_SCHEDULER_JOB_RUN_DETAILS"; const char *const OB_CDB_SCHEDULER_JOB_RUN_DETAILS_TNAME = "CDB_SCHEDULER_JOB_RUN_DETAILS"; diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index d8c9925b0..ae499a131 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -35672,7 +35672,21 @@ def_table_schema( # 21582: ROLE_TABLE_GRANTS # 21583: ROLE_COLUMN_GRANTS # 21584: ROLE_ROUTINE_GRANTS -# 21585: func +def_table_schema( + owner = 'wangbai.wx', + database_id = 'OB_MYSQL_SCHEMA_ID', + table_name = 'func', + table_id = '21585', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT name, ret, dl, type + FROM oceanbase.__all_func +""".replace("\n", " ") +) def_table_schema( owner = 'gengfu.zpc', diff --git a/src/share/inner_table/table_id_to_name b/src/share/inner_table/table_id_to_name index 01fc57da9..ebfc2fc85 100644 --- a/src/share/inner_table/table_id_to_name +++ b/src/share/inner_table/table_id_to_name @@ -2179,6 +2179,7 @@ # 21579: INNODB_METRICS # 21580: EVENTS # 21581: V$OB_NIC_INFO +# 21585: func # 21586: GV$OB_NIC_INFO # 21589: DBA_SCHEDULER_JOB_RUN_DETAILS # 21590: CDB_SCHEDULER_JOB_RUN_DETAILS diff --git a/src/share/schema/ob_routine_info.h b/src/share/schema/ob_routine_info.h index b4c6f9c0f..94e348ea2 100644 --- a/src/share/schema/ob_routine_info.h +++ b/src/share/schema/ob_routine_info.h @@ -173,7 +173,7 @@ public: virtual const common::ObString &get_routine_name() const = 0; virtual uint64_t get_dblink_id() const { return OB_INVALID_ID; } virtual uint64_t get_routine_id() const = 0; - + virtual bool is_function() const = 0; TO_STRING_EMPTY(); }; diff --git a/src/sql/ob_spi.cpp b/src/sql/ob_spi.cpp index 6008f054b..98d37f91e 100644 --- a/src/sql/ob_spi.cpp +++ b/src/sql/ob_spi.cpp @@ -44,6 +44,7 @@ #ifdef OB_BUILD_ORACLE_PL #include "pl/dblink/ob_pl_dblink_util.h" #include "pl/ob_pl_profiler.h" +#include "pl/ob_pl_call_stack_trace.h" #endif #include "pl/ob_pl_allocator.h" #include "pl/diagnosis/ob_pl_sql_audit_guard.h" @@ -4113,13 +4114,16 @@ int ObSPIService::dbms_cursor_close(ObExecContext &exec_ctx, ObPLCursorInfo &cur return ret; } -int ObSPIService::spi_set_pl_exception_code(pl::ObPLExecCtx *ctx, int64_t code, bool is_pop_warning_buf) +int ObSPIService::spi_set_pl_exception_code( + pl::ObPLExecCtx *ctx, int64_t code, bool is_pop_warning_buf, int level) { int ret = OB_SUCCESS; + ObPLContext *pl_ctx = NULL; ObPLSqlCodeInfo *sqlcode_info = NULL; CK (OB_NOT_NULL(ctx)); CK (OB_NOT_NULL(ctx->exec_ctx_)); CK (OB_NOT_NULL(ctx->exec_ctx_->get_my_session())); + CK (OB_NOT_NULL(pl_ctx = ctx->exec_ctx_->get_my_session()->get_pl_context())); CK (OB_NOT_NULL(sqlcode_info = ctx->exec_ctx_->get_my_session()->get_pl_sqlcode_info())); if (OB_SUCC(ret) && code != sqlcode_info->get_sqlcode()) { if (lib::is_oracle_mode()) { @@ -4137,6 +4141,16 @@ int ObSPIService::spi_set_pl_exception_code(pl::ObPLExecCtx *ctx, int64_t code, OX (sqlcode_info->get_stack_warning_buf().pop_back()); OX (ctx->exec_ctx_->get_my_session()->set_show_warnings_buf(OB_SUCCESS)); } + +#ifdef OB_BUILD_ORACLE_PL + if (OB_FAIL(ret) || lib::is_mysql_mode() || OB_ISNULL(pl_ctx->get_call_stack_trace())) { + } else if (!is_pop_warning_buf) { + OZ (pl_ctx->get_call_stack_trace()->format_error_trace(pl_ctx->get_exec_stack(), level)); + } else { + OZ (pl_ctx->get_call_stack_trace()->format_error_trace(pl_ctx->get_exec_stack().count(), level)); + } +#endif + return ret; } @@ -7303,7 +7317,14 @@ int ObSPIService::convert_obj(ObPLExecCtx *ctx, cast_ctx.cur_time_ = cur_time.get_timestamp(); ObExprResType result_type; OX (result_type.set_meta(result_types[i].get_meta_type())); - OX (result_type.set_accuracy(result_types[i].get_accuracy())); + if (result_types[i].get_meta_type().is_number() + && result_types[i].get_precision() == -1 + && result_types[i].get_scale() == -1 + && current_type.at(i).get_meta_type().is_number()) { + OX (result_type.set_accuracy(current_type.at(i).get_accuracy())); + } else { + OX (result_type.set_accuracy(result_types[i].get_accuracy())); + } if (OB_SUCC(ret) && (result_type.is_blob() || result_type.is_blob_locator() || obj.is_blob() || obj.is_blob_locator()) && lib::is_oracle_mode()) { cast_ctx.cast_mode_ |= CM_ENABLE_BLOB_CAST; @@ -8506,7 +8527,6 @@ int ObSPIService::spi_update_location(pl::ObPLExecCtx *ctx, uint64_t location) ObIArray &stack = ctx->pl_ctx_->get_exec_stack(); ObPLExecState *state = stack.at(stack.count() - 1); state->set_loc(location); - int64_t line = location >> 32 & 0xffffffff; } return ret; } @@ -8560,10 +8580,7 @@ int ObSPIService::spi_execute_dblink(ObExecContext &exec_ctx, OX (tenant_id = session->get_effective_tenant_id()); OZ (ObPLDblinkUtil::init_dblink(dblink_proxy, dblink_conn, routine_info->get_dblink_id(), *session, link_type, true)); CK (OB_NOT_NULL(dblink_conn)); - if (DBLINK_DRV_OB == link_type) { - OZ (ObPLDblinkUtil::print_dblink_call_stmt(allocator, *session, call_stmt, params, routine_info)); - OZ (dblink_proxy->dblink_write(dblink_conn, affected_rows, call_stmt.ptr()), call_stmt); - } else { + if (OB_SUCC(ret)) { const int64_t out_param_cnt = routine_info->get_out_param_count(); int64_t out_param_idx[out_param_cnt]; for (int64_t i = 0; i < out_param_cnt; i++) { @@ -8599,7 +8616,7 @@ int ObSPIService::spi_execute_dblink(ObExecContext &exec_ctx, OZ (ObTMService::tm_rm_start(exec_ctx, link_type, dblink_conn, tx_id)); OZ (dblink_proxy->dblink_execute_proc(OB_INVALID_TENANT_ID, dblink_conn, allocator, exec_params, call_stmt, *routine_info, udts, - session->get_timezone_info(), &tmp_result), call_stmt); + session->get_timezone_info(), &tmp_result, is_print_sql), call_stmt); OZ (spi_after_execute_dblink(session, routine_info, allocator, params, exec_params, result, tmp_result)); for (int64_t i = 0; i < exec_params.count(); i++) { if (exec_params.at(i).is_pl_extend()) { diff --git a/src/sql/ob_spi.h b/src/sql/ob_spi.h index 1bdd9fb76..795c631b9 100644 --- a/src/sql/ob_spi.h +++ b/src/sql/ob_spi.h @@ -811,7 +811,7 @@ public: ObIAllocator *allocator, ObObj *result); - static int spi_set_pl_exception_code(pl::ObPLExecCtx *ctx, int64_t code, bool is_pop_warning_buf); + static int spi_set_pl_exception_code(pl::ObPLExecCtx *ctx, int64_t code, bool is_pop_warning_buf, int level); static int spi_get_pl_exception_code(pl::ObPLExecCtx *ctx, int64_t *code); diff --git a/src/sql/printer/ob_raw_expr_printer.cpp b/src/sql/printer/ob_raw_expr_printer.cpp index bbba7b4bb..d45999219 100644 --- a/src/sql/printer/ob_raw_expr_printer.cpp +++ b/src/sql/printer/ob_raw_expr_printer.cpp @@ -3463,7 +3463,12 @@ int ObRawExprPrinter::print(ObUDFRawExpr *expr) LOG_WARN("stmt_ is NULL of buf_ is NULL or pos_ is NULL or expr is NULL", K(ret)); } else { if (!print_params_.for_dblink_) { - if (!expr->get_database_name().empty()) { + if (expr->is_dblink_sys_func()) { + if (!expr->get_database_name().empty()) { + PRINT_IDENT_WITH_QUOT(expr->get_database_name()); + DATA_PRINTF("."); + } + } else if (!expr->get_database_name().empty()) { if (expr->get_database_name().case_compare("oceanbase") != 0) { PRINT_IDENT_WITH_QUOT(expr->get_database_name()); DATA_PRINTF("."); @@ -3516,6 +3521,12 @@ do { \ } #undef PRINT_IMPLICIT_DATABASE_NAME + } else { + if (!expr->get_dblink_name().empty() + && !expr->get_database_name().empty()) { + PRINT_IDENT_WITH_QUOT(expr->get_database_name()); + DATA_PRINTF("."); + } } if (!expr->get_package_name().empty() && diff --git a/src/sql/privilege_check/ob_privilege_check.cpp b/src/sql/privilege_check/ob_privilege_check.cpp index c0596a096..62d7fe6f3 100644 --- a/src/sql/privilege_check/ob_privilege_check.cpp +++ b/src/sql/privilege_check/ob_privilege_check.cpp @@ -710,6 +710,7 @@ int add_udf_expr_priv( { int ret = OB_SUCCESS; bool is_sys_udf = false; + bool is_dblink_udf = false; ObOraNeedPriv need_priv; ObString db_name; ObPackedObjPriv packed_privs = 0; @@ -717,6 +718,8 @@ int add_udf_expr_priv( if (OB_ISNULL(expr) || OB_ISNULL(udf_expr = static_cast(expr))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", KPC(expr), K(ret)); + } else if (!udf_expr->get_dblink_name().empty()) { + is_dblink_udf = true; } else if (0 == udf_expr->get_database_name().case_compare(OB_SYS_DATABASE_NAME)) { is_sys_udf = true; } else if (common::OB_INVALID_ID != udf_expr->get_type_id()) { @@ -732,7 +735,7 @@ int add_udf_expr_priv( ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid udf expr", KPC(udf_expr), K(ret)); } - if (OB_SUCC(ret) && !is_sys_udf) { + if (OB_SUCC(ret) && !is_sys_udf && !is_dblink_udf) { // todo: check sys udf privilege after grant privs to public role need_priv.grantee_id_ = user_id; need_priv.obj_level_ = OBJ_LEVEL_FOR_TAB_PRIV; diff --git a/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp b/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp index 0ed38b68e..aa27f57d1 100644 --- a/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp @@ -7085,7 +7085,7 @@ int ObRawExprResolverImpl::resolve_udf_node(const ParseNode *node, ObUDFInfo &ud } } } else if (has_assign_expr) { - ret = OB_ERR_UNEXPECTED; + ret = OB_ERR_POSITIONAL_FOLLOW_NAME; LOG_WARN("can not get parameter after assign", K(ret)); } else if (OB_FAIL(SMART_CALL(recursive_resolve(param_node, param_expr)))) { LOG_WARN("fail to recursive resolve udf parameters", K(ret), K(param_node)); diff --git a/tools/deploy/mysql_test/r/mysql/information_schema.result b/tools/deploy/mysql_test/r/mysql/information_schema.result index 3d35af953..41bfa5e16 100644 --- a/tools/deploy/mysql_test/r/mysql/information_schema.result +++ b/tools/deploy/mysql_test/r/mysql/information_schema.result @@ -248,6 +248,7 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | mysql | columns_priv | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | mysql | db | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | mysql | default_roles | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | func | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | mysql | help_category | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | mysql | help_keyword | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | mysql | help_relation | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | @@ -1673,6 +1674,7 @@ select * from information_schema.views where table_schema in ('oceanbase', 'mysq +---------------+--------------+---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ | def | mysql | columns_priv | SELECT cast(b.host as char(255)) as Host, cast(a.database_name as char(128)) as Db, cast(b.user_name as char(128)) as User, cast(a.table_name as char(128)) as Table_name, cast(a.column_name as char(128)) as Column_name, substr(concat(case when (a.all_priv & 1) > 0 then ',Select' else '' end, case when (a.all_priv & 2) > 0 then ',Insert' else '' end, case when (a.all_priv & 4) > 0 then ',Update' else '' end, case when (a.all_priv & 8) > 0 then ',References' else '' end), 2) as Column_priv, cast(a.gmt_modified as datetime) as Timestamp FROM oceanbase.__all_column_privilege a, oceanbase.__all_user b WHERE a.tenant_id = 0 and a.tenant_id = b.tenant_id AND a.user_id = b.user_id | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | | def | mysql | default_roles | SELECT cast(to_user.host AS char(255)) HOST, cast(to_user.user_name AS char(128)) USER, cast(from_user.host AS char(255)) DEFAULT_ROLE_HOST, cast(from_user.user_name AS char(128)) DEFAULT_ROLE_USER FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id AND role_map.disable_flag = 0; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | func | SELECT name, ret, dl, type FROM oceanbase.__all_func | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | | def | mysql | procs_priv | SELECT cast(b.host as char(60)) as Host, cast(a.database_name as char(64)) as Db, cast(b.user_name as char(32)) as User, cast(a.routine_name as char(64)) as Routine_name, case when a.routine_type = 1 then 'PROCEDURE' else 'FUNCTION' end as Routine_type, cast(c.priv_user as char(93)) as Grantor, substr(concat(case when (a.all_priv & 1) > 0 then ',Execute' else '' end, case when (a.all_priv & 2) > 0 then ',Alter Routine' else '' end, case when (a.all_priv & 4) > 0 then ',Grant' else '' end), 2) as Proc_priv, cast(a.gmt_modified as date) as Timestamp FROM oceanbase.__all_routine_privilege a, oceanbase.__all_user b, oceanbase.__all_routine c, oceanbase.__all_database d WHERE a.tenant_id = b.tenant_id AND a.user_id = b.user_id AND a.tenant_id = d.tenant_id and a.database_name = d.database_name AND a.tenant_id = c.tenant_id AND a.routine_name = c.routine_name AND a.routine_type = c.routine_type AND c.database_id = d.database_id AND c.package_id = -1; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | | def | mysql | role_edges | SELECT cast(from_user.host AS char(255)) FROM_HOST, cast(from_user.user_name AS char(128)) FROM_USER, cast(to_user.host AS char(255)) TO_HOST, cast(to_user.user_name AS char(128)) TO_USER, cast(CASE role_map.admin_option WHEN 1 THEN 'Y' ELSE 'N' END AS char(1)) WITH_ADMIN_OPTION FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | | def | mysql | time_zone | SELECT time_zone_id as Time_zone_id, use_leap_seconds as Use_leap_seconds FROM oceanbase.__all_tenant_time_zone | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | @@ -1756,6 +1758,7 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | mysql | columns_priv | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | mysql | db | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | mysql | default_roles | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | func | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | mysql | help_category | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | mysql | help_keyword | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | mysql | help_relation | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | @@ -2471,6 +2474,7 @@ select * from information_schema.views where table_schema in ('oceanbase', 'mysq +---------------+--------------+---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ | def | mysql | columns_priv | SELECT cast(b.host as char(255)) as Host, cast(a.database_name as char(128)) as Db, cast(b.user_name as char(128)) as User, cast(a.table_name as char(128)) as Table_name, cast(a.column_name as char(128)) as Column_name, substr(concat(case when (a.all_priv & 1) > 0 then ',Select' else '' end, case when (a.all_priv & 2) > 0 then ',Insert' else '' end, case when (a.all_priv & 4) > 0 then ',Update' else '' end, case when (a.all_priv & 8) > 0 then ',References' else '' end), 2) as Column_priv, cast(a.gmt_modified as datetime) as Timestamp FROM oceanbase.__all_column_privilege a, oceanbase.__all_user b WHERE a.tenant_id = 0 and a.tenant_id = b.tenant_id AND a.user_id = b.user_id | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | | def | mysql | default_roles | SELECT cast(to_user.host AS char(255)) HOST, cast(to_user.user_name AS char(128)) USER, cast(from_user.host AS char(255)) DEFAULT_ROLE_HOST, cast(from_user.user_name AS char(128)) DEFAULT_ROLE_USER FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id AND role_map.disable_flag = 0; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | func | SELECT name, ret, dl, type FROM oceanbase.__all_func | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | | def | mysql | procs_priv | SELECT cast(b.host as char(60)) as Host, cast(a.database_name as char(64)) as Db, cast(b.user_name as char(32)) as User, cast(a.routine_name as char(64)) as Routine_name, case when a.routine_type = 1 then 'PROCEDURE' else 'FUNCTION' end as Routine_type, cast(c.priv_user as char(93)) as Grantor, substr(concat(case when (a.all_priv & 1) > 0 then ',Execute' else '' end, case when (a.all_priv & 2) > 0 then ',Alter Routine' else '' end, case when (a.all_priv & 4) > 0 then ',Grant' else '' end), 2) as Proc_priv, cast(a.gmt_modified as date) as Timestamp FROM oceanbase.__all_routine_privilege a, oceanbase.__all_user b, oceanbase.__all_routine c, oceanbase.__all_database d WHERE a.tenant_id = b.tenant_id AND a.user_id = b.user_id AND a.tenant_id = d.tenant_id and a.database_name = d.database_name AND a.tenant_id = c.tenant_id AND a.routine_name = c.routine_name AND a.routine_type = c.routine_type AND c.database_id = d.database_id AND c.package_id = -1; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | | def | mysql | role_edges | SELECT cast(from_user.host AS char(255)) FROM_HOST, cast(from_user.user_name AS char(128)) FROM_USER, cast(to_user.host AS char(255)) TO_HOST, cast(to_user.user_name AS char(128)) TO_USER, cast(CASE role_map.admin_option WHEN 1 THEN 'Y' ELSE 'N' END AS char(1)) WITH_ADMIN_OPTION FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | | def | mysql | time_zone | SELECT time_zone_id as Time_zone_id, use_leap_seconds as Use_leap_seconds FROM oceanbase.__all_tenant_time_zone | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | @@ -3525,6 +3529,7 @@ select * from information_schema.views where table_schema in ('oceanbase', 'mysq +---------------+--------------+---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ | def | mysql | columns_priv | SELECT cast(b.host as char(255)) as Host, cast(a.database_name as char(128)) as Db, cast(b.user_name as char(128)) as User, cast(a.table_name as char(128)) as Table_name, cast(a.column_name as char(128)) as Column_name, substr(concat(case when (a.all_priv & 1) > 0 then ',Select' else '' end, case when (a.all_priv & 2) > 0 then ',Insert' else '' end, case when (a.all_priv & 4) > 0 then ',Update' else '' end, case when (a.all_priv & 8) > 0 then ',References' else '' end), 2) as Column_priv, cast(a.gmt_modified as datetime) as Timestamp FROM oceanbase.__all_column_privilege a, oceanbase.__all_user b WHERE a.tenant_id = 0 and a.tenant_id = b.tenant_id AND a.user_id = b.user_id | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | | def | mysql | default_roles | SELECT cast(to_user.host AS char(255)) HOST, cast(to_user.user_name AS char(128)) USER, cast(from_user.host AS char(255)) DEFAULT_ROLE_HOST, cast(from_user.user_name AS char(128)) DEFAULT_ROLE_USER FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id AND role_map.disable_flag = 0; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | func | SELECT name, ret, dl, type FROM oceanbase.__all_func | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | | def | mysql | procs_priv | SELECT cast(b.host as char(60)) as Host, cast(a.database_name as char(64)) as Db, cast(b.user_name as char(32)) as User, cast(a.routine_name as char(64)) as Routine_name, case when a.routine_type = 1 then 'PROCEDURE' else 'FUNCTION' end as Routine_type, cast(c.priv_user as char(93)) as Grantor, substr(concat(case when (a.all_priv & 1) > 0 then ',Execute' else '' end, case when (a.all_priv & 2) > 0 then ',Alter Routine' else '' end, case when (a.all_priv & 4) > 0 then ',Grant' else '' end), 2) as Proc_priv, cast(a.gmt_modified as date) as Timestamp FROM oceanbase.__all_routine_privilege a, oceanbase.__all_user b, oceanbase.__all_routine c, oceanbase.__all_database d WHERE a.tenant_id = b.tenant_id AND a.user_id = b.user_id AND a.tenant_id = d.tenant_id and a.database_name = d.database_name AND a.tenant_id = c.tenant_id AND a.routine_name = c.routine_name AND a.routine_type = c.routine_type AND c.database_id = d.database_id AND c.package_id = -1; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | | def | mysql | role_edges | SELECT cast(from_user.host AS char(255)) FROM_HOST, cast(from_user.user_name AS char(128)) FROM_USER, cast(to_user.host AS char(255)) TO_HOST, cast(to_user.user_name AS char(128)) TO_USER, cast(CASE role_map.admin_option WHEN 1 THEN 'Y' ELSE 'N' END AS char(1)) WITH_ADMIN_OPTION FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | | def | mysql | time_zone | SELECT time_zone_id as Time_zone_id, use_leap_seconds as Use_leap_seconds FROM oceanbase.__all_tenant_time_zone | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | @@ -4261,6 +4266,7 @@ select * from information_schema.views where table_schema in ('oceanbase', 'mysq +---------------+--------------+---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ | def | mysql | columns_priv | SELECT cast(b.host as char(255)) as Host, cast(a.database_name as char(128)) as Db, cast(b.user_name as char(128)) as User, cast(a.table_name as char(128)) as Table_name, cast(a.column_name as char(128)) as Column_name, substr(concat(case when (a.all_priv & 1) > 0 then ',Select' else '' end, case when (a.all_priv & 2) > 0 then ',Insert' else '' end, case when (a.all_priv & 4) > 0 then ',Update' else '' end, case when (a.all_priv & 8) > 0 then ',References' else '' end), 2) as Column_priv, cast(a.gmt_modified as datetime) as Timestamp FROM oceanbase.__all_column_privilege a, oceanbase.__all_user b WHERE a.tenant_id = 0 and a.tenant_id = b.tenant_id AND a.user_id = b.user_id | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | | def | mysql | default_roles | SELECT cast(to_user.host AS char(255)) HOST, cast(to_user.user_name AS char(128)) USER, cast(from_user.host AS char(255)) DEFAULT_ROLE_HOST, cast(from_user.user_name AS char(128)) DEFAULT_ROLE_USER FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id AND role_map.disable_flag = 0; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | func | SELECT name, ret, dl, type FROM oceanbase.__all_func | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | | def | mysql | procs_priv | SELECT cast(b.host as char(60)) as Host, cast(a.database_name as char(64)) as Db, cast(b.user_name as char(32)) as User, cast(a.routine_name as char(64)) as Routine_name, case when a.routine_type = 1 then 'PROCEDURE' else 'FUNCTION' end as Routine_type, cast(c.priv_user as char(93)) as Grantor, substr(concat(case when (a.all_priv & 1) > 0 then ',Execute' else '' end, case when (a.all_priv & 2) > 0 then ',Alter Routine' else '' end, case when (a.all_priv & 4) > 0 then ',Grant' else '' end), 2) as Proc_priv, cast(a.gmt_modified as date) as Timestamp FROM oceanbase.__all_routine_privilege a, oceanbase.__all_user b, oceanbase.__all_routine c, oceanbase.__all_database d WHERE a.tenant_id = b.tenant_id AND a.user_id = b.user_id AND a.tenant_id = d.tenant_id and a.database_name = d.database_name AND a.tenant_id = c.tenant_id AND a.routine_name = c.routine_name AND a.routine_type = c.routine_type AND c.database_id = d.database_id AND c.package_id = -1; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | | def | mysql | role_edges | SELECT cast(from_user.host AS char(255)) FROM_HOST, cast(from_user.user_name AS char(128)) FROM_USER, cast(to_user.host AS char(255)) TO_HOST, cast(to_user.user_name AS char(128)) TO_USER, cast(CASE role_map.admin_option WHEN 1 THEN 'Y' ELSE 'N' END AS char(1)) WITH_ADMIN_OPTION FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | | def | mysql | time_zone | SELECT time_zone_id as Time_zone_id, use_leap_seconds as Use_leap_seconds FROM oceanbase.__all_tenant_time_zone | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result index 92e1217cc..b8d7f829d 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result @@ -6650,6 +6650,15 @@ SPEED_MBPS bigint(20) NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_NIC_INFO limit 1); cnt 1 +desc mysql.func; +Field Type Null Key Default Extra +name varchar(64) NO NULL +ret bigint(20) NO NULL +dl varchar(128) NO NULL +type bigint(20) NO NULL +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from mysql.func limit 1); +cnt +1 desc oceanbase.GV$OB_NIC_INFO; Field Type Null Key Default Extra SVR_IP varchar(46) NO NULL diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result index 6440d3cb6..4eb37c6a4 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result @@ -9446,6 +9446,15 @@ SPEED_MBPS bigint(20) NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_NIC_INFO limit 1); cnt 1 +desc mysql.func; +Field Type Null Key Default Extra +name varchar(64) NO NULL +ret bigint(20) NO NULL +dl varchar(128) NO NULL +type bigint(20) NO NULL +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from mysql.func limit 1); +cnt +1 desc oceanbase.GV$OB_NIC_INFO; Field Type Null Key Default Extra SVR_IP varchar(46) NO NULL diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result index f7b93ef3d..be1f568b1 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result @@ -1199,6 +1199,7 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr 21579 INNODB_METRICS 1 201002 1 21580 EVENTS 1 201002 1 21581 V$OB_NIC_INFO 1 201001 1 +21585 func 1 201003 1 21586 GV$OB_NIC_INFO 1 201001 1 21589 DBA_SCHEDULER_JOB_RUN_DETAILS 1 201001 1 21590 CDB_SCHEDULER_JOB_RUN_DETAILS 1 201001 1 diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/views.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/views.result index 5a4364fee..08a24468d 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/views.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/views.result @@ -7,6 +7,7 @@ def mysql time_zone_name SELECT name as Name, time_zone_id as Ti def mysql time_zone SELECT time_zone_id as Time_zone_id, use_leap_seconds as Use_leap_seconds FROM oceanbase.__all_tenant_time_zone NONE NO NONE NONE utf8mb4 utf8mb4_general_ci def mysql role_edges SELECT cast(from_user.host AS char(255)) FROM_HOST, cast(from_user.user_name AS char(128)) FROM_USER, cast(to_user.host AS char(255)) TO_HOST, cast(to_user.user_name AS char(128)) TO_USER, cast(CASE role_map.admin_option WHEN 1 THEN 'Y' ELSE 'N' END AS char(1)) WITH_ADMIN_OPTION FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id; NONE NO NONE NONE utf8mb4 utf8mb4_general_ci def mysql procs_priv SELECT cast(b.host as char(60)) as Host, cast(a.database_name as char(64)) as Db, cast(b.user_name as char(32)) as User, cast(a.routine_name as char(64)) as Routine_name, case when a.routine_type = 1 then 'PROCEDURE' else 'FUNCTION' end as Routine_type, cast(c.priv_user as char(93)) as Grantor, substr(concat(case when (a.all_priv & 1) > 0 then ',Execute' else '' end, case when (a.all_priv & 2) > 0 then ',Alter Routine' else '' end, case when (a.all_priv & 4) > 0 then ',Grant' else '' end), 2) as Proc_priv, cast(a.gmt_modified as date) as Timestamp FROM oceanbase.__all_routine_privilege a, oceanbase.__all_user b, oceanbase.__all_routine c, oceanbase.__all_database d WHERE a.tenant_id = b.tenant_id AND a.user_id = b.user_id AND a.tenant_id = d.tenant_id and a.database_name = d.database_name AND a.tenant_id = c.tenant_id AND a.routine_name = c.routine_name AND a.routine_type = c.routine_type AND c.database_id = d.database_id AND c.package_id = -1; NONE NO NONE NONE utf8mb4 utf8mb4_general_ci +def mysql func SELECT name, ret, dl, type FROM oceanbase.__all_func NONE NO NONE NONE utf8mb4 utf8mb4_general_ci def mysql default_roles SELECT cast(to_user.host AS char(255)) HOST, cast(to_user.user_name AS char(128)) USER, cast(from_user.host AS char(255)) DEFAULT_ROLE_HOST, cast(from_user.user_name AS char(128)) DEFAULT_ROLE_USER FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id AND role_map.disable_flag = 0; NONE NO NONE NONE utf8mb4 utf8mb4_general_ci def mysql columns_priv SELECT cast(b.host as char(255)) as Host, cast(a.database_name as char(128)) as Db, cast(b.user_name as char(128)) as User, cast(a.table_name as char(128)) as Table_name, cast(a.column_name as char(128)) as Column_name, substr(concat(case when (a.all_priv & 1) > 0 then ',Select' else '' end, case when (a.all_priv & 2) > 0 then ',Insert' else '' end, case when (a.all_priv & 4) > 0 then ',Update' else '' end, case when (a.all_priv & 8) > 0 then ',References' else '' end), 2) as Column_priv, cast(a.gmt_modified as datetime) as Timestamp FROM oceanbase.__all_column_privilege a, oceanbase.__all_user b WHERE a.tenant_id = 0 and a.tenant_id = b.tenant_id AND a.user_id = b.user_id NONE NO NONE NONE utf8mb4 utf8mb4_general_ci show create view views; From 3ede8a0f8caf237a1295970887d2c6305c7f8879 Mon Sep 17 00:00:00 2001 From: shadowao Date: Mon, 19 Aug 2024 06:51:20 +0000 Subject: [PATCH 096/249] [CP] fix incorrect tablet_id of lob aux table schame in partition table --- src/rootserver/ob_lob_meta_builder.cpp | 4 ++++ src/rootserver/ob_lob_piece_builder.cpp | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/rootserver/ob_lob_meta_builder.cpp b/src/rootserver/ob_lob_meta_builder.cpp index ebb0523ab..9d9ffb0b0 100644 --- a/src/rootserver/ob_lob_meta_builder.cpp +++ b/src/rootserver/ob_lob_meta_builder.cpp @@ -123,6 +123,10 @@ int ObLobMetaBuilder::set_basic_infos( aux_lob_meta_schema.set_table_type(AUX_LOB_META); aux_lob_meta_schema.set_data_table_id(data_schema.get_table_id()); + // reset tablet id to zero + // real tablet id will be generated by generate_tablet_id according to partition info + aux_lob_meta_schema.set_tablet_id(0); + // priority same with data table schema aux_lob_meta_schema.set_tenant_id(data_schema.get_tenant_id()); aux_lob_meta_schema.set_database_id(data_schema.get_database_id()); diff --git a/src/rootserver/ob_lob_piece_builder.cpp b/src/rootserver/ob_lob_piece_builder.cpp index be95ee2e7..a12e34a4c 100644 --- a/src/rootserver/ob_lob_piece_builder.cpp +++ b/src/rootserver/ob_lob_piece_builder.cpp @@ -122,6 +122,10 @@ int ObLobPieceBuilder::set_basic_infos( int ret = OB_SUCCESS; aux_lob_piece_schema.set_data_table_id(data_schema.get_table_id()); + // reset tablet id to zero + // real tablet id will be generated by generate_tablet_id according to partition info + aux_lob_piece_schema.set_tablet_id(0); + // priority same with data table schema aux_lob_piece_schema.set_tenant_id(data_schema.get_tenant_id()); aux_lob_piece_schema.set_database_id(data_schema.get_database_id()); From bf865be77b29dbbe4a9490f1973af553d6567627 Mon Sep 17 00:00:00 2001 From: obdev Date: Mon, 19 Aug 2024 06:57:25 +0000 Subject: [PATCH 097/249] [CP] [to #2024071200103691754]fix bugs, all_types methods, local_methods, local_attributes columns bugs --- .../ob_inner_table_schema.12001_12050.cpp | 2 +- .../ob_inner_table_schema.25001_25050.cpp | 6 ++--- .../inner_table/ob_inner_table_schema_def.py | 22 +++++++++---------- src/share/schema/ob_udt_info.h | 2 +- src/sql/engine/expr/ob_expr_sql_udt_utils.cpp | 2 +- .../mysql/desc_virtual_table_in_mysql.result | 2 +- .../r/mysql/desc_virtual_table_in_sys.result | 2 +- 7 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/share/inner_table/ob_inner_table_schema.12001_12050.cpp b/src/share/inner_table/ob_inner_table_schema.12001_12050.cpp index 28ef082c8..cc0a65202 100644 --- a/src/share/inner_table/ob_inner_table_schema.12001_12050.cpp +++ b/src/share/inner_table/ob_inner_table_schema.12001_12050.cpp @@ -2210,7 +2210,7 @@ int ObInnerTableSchema::proc_schema(ObTableSchema &table_schema) 0, //part_key_pos ObVarcharType, //column_type CS_TYPE_INVALID, //column_collation_type - 32, //column_length + OB_MAX_VARCHAR_LENGTH, //column_length -1, //column_precision -1, //column_scale false, //is_nullable diff --git a/src/share/inner_table/ob_inner_table_schema.25001_25050.cpp b/src/share/inner_table/ob_inner_table_schema.25001_25050.cpp index 053971b8e..c320977e3 100644 --- a/src/share/inner_table/ob_inner_table_schema.25001_25050.cpp +++ b/src/share/inner_table/ob_inner_table_schema.25001_25050.cpp @@ -2210,7 +2210,7 @@ int ObInnerTableSchema::dba_types_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT D.DATABASE_NAME AS OWNER, T.TYPE_NAME AS TYPE_NAME, T.TYPE_ID AS TYPE_OID, CAST( CASE T.TYPECODE WHEN 1 THEN 'COLLECTION' WHEN 2 THEN 'OBJECT' END AS VARCHAR2(10)) AS TYPECODE, T.ATTRIBUTES AS ATTRIBUTES, T.METHODS AS METHODS, CAST('NO' AS CHAR(2)) AS PREDEFINED, CAST('NO' AS CHAR(2)) AS INCOMPLETE, CAST('YES' AS CHAR(3)) AS FINAL, CAST('YES' AS CHAR(3)) AS INSTANTIABLE, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_OWNER, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_NAME, T.LOCAL_ATTRS AS LOCAL_ATTRIBUTES, T.LOCAL_METHODS AS LOCAL_METHODS, T.TYPE_ID AS TYPEID FROM SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D ON D.DATABASE_ID = T.DATABASE_ID AND T.TENANT_ID = SYS_CONTEXT('USERENV', 'CON_ID') AND T.TENANT_ID = EFFECTIVE_TENANT_ID() AND D.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT CAST('SYS' AS VARCHAR2(30)) AS OWNER, TS.TYPE_NAME AS TYPE_NAME, TS.TYPE_ID AS TYPE_OID, CAST( CASE TS.TYPECODE WHEN 1 THEN 'COLLECTION' WHEN 2 THEN 'OBJECT' END AS VARCHAR2(10)) AS TYPECODE, TS.ATTRIBUTES AS ATTRIBUTES, TS.METHODS AS METHODS, CAST('NO' AS CHAR(2)) AS PREDEFINED, CAST('NO' AS CHAR(2)) AS INCOMPLETE, CAST('YES' AS CHAR(3)) AS FINAL, CAST('YES' AS CHAR(3)) AS INSTANTIABLE, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_OWNER, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_NAME, TS.LOCAL_ATTRS AS LOCAL_ATTRIBUTES, TS.LOCAL_METHODS AS LOCAL_METHODS, TS.TYPE_ID AS TYPEID FROM SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT D.DATABASE_NAME AS OWNER, T.TYPE_NAME AS TYPE_NAME, T.TYPE_ID AS TYPE_OID, CAST( CASE T.TYPECODE WHEN 1 THEN 'COLLECTION' WHEN 2 THEN 'OBJECT' END AS VARCHAR2(10)) AS TYPECODE, T.ATTRIBUTES AS ATTRIBUTES, T.METHODS AS METHODS, CAST('NO' AS CHAR(2)) AS PREDEFINED, CAST('NO' AS CHAR(2)) AS INCOMPLETE, CAST('YES' AS CHAR(3)) AS FINAL, CAST('YES' AS CHAR(3)) AS INSTANTIABLE, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_OWNER, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_NAME, CAST(NULL AS NUMBER(38)) AS LOCAL_ATTRIBUTES, CAST(NULL AS NUMBER(38)) AS LOCAL_METHODS, T.TYPE_ID AS TYPEID FROM SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D ON D.DATABASE_ID = T.DATABASE_ID AND T.TENANT_ID = SYS_CONTEXT('USERENV', 'CON_ID') AND T.TENANT_ID = EFFECTIVE_TENANT_ID() AND D.TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT CAST('SYS' AS VARCHAR2(30)) AS OWNER, TS.TYPE_NAME AS TYPE_NAME, TS.TYPE_ID AS TYPE_OID, CAST( CASE TS.TYPECODE WHEN 1 THEN 'COLLECTION' WHEN 2 THEN 'OBJECT' END AS VARCHAR2(10)) AS TYPECODE, TS.ATTRIBUTES AS ATTRIBUTES, TS.METHODS AS METHODS, CAST('NO' AS CHAR(2)) AS PREDEFINED, CAST('NO' AS CHAR(2)) AS INCOMPLETE, CAST('YES' AS CHAR(3)) AS FINAL, CAST('YES' AS CHAR(3)) AS INSTANTIABLE, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_OWNER, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_NAME, CAST(NULL AS NUMBER(38)) AS LOCAL_ATTRIBUTES, CAST(NULL AS NUMBER(38)) AS LOCAL_METHODS, TS.TYPE_ID AS TYPEID FROM SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -2260,7 +2260,7 @@ int ObInnerTableSchema::all_types_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT D.DATABASE_NAME AS OWNER, T.TYPE_NAME AS TYPE_NAME, T.TYPE_ID AS TYPE_OID, CAST( CASE T.TYPECODE WHEN 1 THEN 'COLLECTION' WHEN 2 THEN 'OBJECT' END AS VARCHAR2(10)) AS TYPECODE, T.ATTRIBUTES AS ATTRIBUTES, T.METHODS AS METHODS, CAST('NO' AS CHAR(2)) AS PREDEFINED, CAST('NO' AS CHAR(2)) AS INCOMPLETE, CAST('YES' AS CHAR(3)) AS FINAL, CAST('YES' AS CHAR(3)) AS INSTANTIABLE, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_OWNER, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_NAME, T.LOCAL_ATTRS AS LOCAL_ATTRIBUTES, T.LOCAL_METHODS AS LOCAL_METHODS, T.TYPE_ID AS TYPEID FROM SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D ON D.DATABASE_ID = T.DATABASE_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() AND D.TENANT_ID = EFFECTIVE_TENANT_ID() AND (T.DATABASE_ID = USERENV('SCHEMAID') or USER_CAN_ACCESS_OBJ(4, T.TYPE_ID, T.DATABASE_ID) = 1) AND T.TENANT_ID = SYS_CONTEXT('USERENV', 'CON_ID') UNION ALL SELECT CAST('SYS' AS VARCHAR2(30)) AS OWNER, TS.TYPE_NAME AS TYPE_NAME, TS.TYPE_ID AS TYPE_OID, CAST( CASE TS.TYPECODE WHEN 1 THEN 'COLLECTION' WHEN 2 THEN 'OBJECT' END AS VARCHAR2(10)) AS TYPECODE, TS.ATTRIBUTES AS ATTRIBUTES, TS.METHODS AS METHODS, CAST('NO' AS CHAR(2)) AS PREDEFINED, CAST('NO' AS CHAR(2)) AS INCOMPLETE, CAST('YES' AS CHAR(3)) AS FINAL, CAST('YES' AS CHAR(3)) AS INSTANTIABLE, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_OWNER, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_NAME, TS.LOCAL_ATTRS AS LOCAL_ATTRIBUTES, TS.LOCAL_METHODS AS LOCAL_METHODS, TS.TYPE_ID AS TYPEID FROM SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT D.DATABASE_NAME AS OWNER, T.TYPE_NAME AS TYPE_NAME, T.TYPE_ID AS TYPE_OID, CAST( CASE T.TYPECODE WHEN 1 THEN 'COLLECTION' WHEN 2 THEN 'OBJECT' END AS VARCHAR2(10)) AS TYPECODE, T.ATTRIBUTES AS ATTRIBUTES, T.METHODS AS METHODS, CAST('NO' AS CHAR(2)) AS PREDEFINED, CAST('NO' AS CHAR(2)) AS INCOMPLETE, CAST('YES' AS CHAR(3)) AS FINAL, CAST('YES' AS CHAR(3)) AS INSTANTIABLE, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_OWNER, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_NAME, CAST(NULL AS NUMBER(38)) AS LOCAL_ATTRIBUTES, CAST(NULL AS NUMBER(38)) AS LOCAL_METHODS, T.TYPE_ID AS TYPEID FROM SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D ON D.DATABASE_ID = T.DATABASE_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() AND D.TENANT_ID = EFFECTIVE_TENANT_ID() AND (T.DATABASE_ID = USERENV('SCHEMAID') or USER_CAN_ACCESS_OBJ(4, T.TYPE_ID, T.DATABASE_ID) = 1) AND T.TENANT_ID = SYS_CONTEXT('USERENV', 'CON_ID') UNION ALL SELECT CAST('SYS' AS VARCHAR2(30)) AS OWNER, TS.TYPE_NAME AS TYPE_NAME, TS.TYPE_ID AS TYPE_OID, CAST( CASE TS.TYPECODE WHEN 1 THEN 'COLLECTION' WHEN 2 THEN 'OBJECT' END AS VARCHAR2(10)) AS TYPECODE, TS.ATTRIBUTES AS ATTRIBUTES, TS.METHODS AS METHODS, CAST('NO' AS CHAR(2)) AS PREDEFINED, CAST('NO' AS CHAR(2)) AS INCOMPLETE, CAST('YES' AS CHAR(3)) AS FINAL, CAST('YES' AS CHAR(3)) AS INSTANTIABLE, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_OWNER, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_NAME, CAST(NULL AS NUMBER(38)) AS LOCAL_ATTRIBUTES, CAST(NULL AS NUMBER(38)) AS LOCAL_METHODS, TS.TYPE_ID AS TYPEID FROM SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -2310,7 +2310,7 @@ int ObInnerTableSchema::user_types_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT T.TYPE_NAME AS TYPE_NAME, T.TYPE_ID AS TYPE_OID, CAST( CASE T.TYPECODE WHEN 1 THEN 'COLLECTION' WHEN 2 THEN 'OBJECT' END AS VARCHAR2(10)) AS TYPECODE, T.ATTRIBUTES AS ATTRIBUTES, T.METHODS AS METHODS, CAST('NO' AS CHAR(2)) AS PREDEFINED, CAST('NO' AS CHAR(2)) AS INCOMPLETE, CAST('YES' AS CHAR(3)) AS FINAL, CAST('YES' AS CHAR(3)) AS INSTANTIABLE, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_OWNER, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_NAME, T.LOCAL_ATTRS AS LOCAL_ATTRIBUTES, T.LOCAL_METHODS AS LOCAL_METHODS, T.TYPE_ID AS TYPEID FROM SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D ON D.DATABASE_ID = T.DATABASE_ID AND T.TENANT_ID = SYS_CONTEXT('USERENV', 'CON_ID') AND D.DATABASE_ID = USERENV('SCHEMAID') AND T.TENANT_ID = EFFECTIVE_TENANT_ID() AND D.TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT T.TYPE_NAME AS TYPE_NAME, T.TYPE_ID AS TYPE_OID, CAST( CASE T.TYPECODE WHEN 1 THEN 'COLLECTION' WHEN 2 THEN 'OBJECT' END AS VARCHAR2(10)) AS TYPECODE, T.ATTRIBUTES AS ATTRIBUTES, T.METHODS AS METHODS, CAST('NO' AS CHAR(2)) AS PREDEFINED, CAST('NO' AS CHAR(2)) AS INCOMPLETE, CAST('YES' AS CHAR(3)) AS FINAL, CAST('YES' AS CHAR(3)) AS INSTANTIABLE, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_OWNER, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_NAME, CAST(NULL AS NUMBER(38)) AS LOCAL_ATTRIBUTES, CAST(NULL AS NUMBER(38)) AS LOCAL_METHODS, T.TYPE_ID AS TYPEID FROM SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D ON D.DATABASE_ID = T.DATABASE_ID AND T.TENANT_ID = SYS_CONTEXT('USERENV', 'CON_ID') AND D.DATABASE_ID = USERENV('SCHEMAID') AND T.TENANT_ID = EFFECTIVE_TENANT_ID() AND D.TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index ae499a131..6117549df 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -10315,7 +10315,7 @@ def_table_schema( ('definer', 'varchar:77', 'false', ''), ('created', 'timestamp'), ('modified', 'timestamp',), - ('sql_mode', 'varchar:32', 'false', ''), + ('sql_mode', 'varchar:OB_MAX_VARCHAR_LENGTH', 'false', ''), ('comment', 'varchar:OB_MAX_VARCHAR_LENGTH', 'false', ''), ('character_set_client', 'varchar:MAX_CHARSET_LENGTH'), ('collation_connection', 'varchar:MAX_CHARSET_LENGTH'), @@ -41499,8 +41499,8 @@ def_table_schema( CAST('YES' AS CHAR(3)) AS INSTANTIABLE, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_OWNER, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_NAME, - T.LOCAL_ATTRS AS LOCAL_ATTRIBUTES, - T.LOCAL_METHODS AS LOCAL_METHODS, + CAST(NULL AS NUMBER(38)) AS LOCAL_ATTRIBUTES, + CAST(NULL AS NUMBER(38)) AS LOCAL_METHODS, T.TYPE_ID AS TYPEID FROM SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D @@ -41525,8 +41525,8 @@ def_table_schema( CAST('YES' AS CHAR(3)) AS INSTANTIABLE, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_OWNER, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_NAME, - TS.LOCAL_ATTRS AS LOCAL_ATTRIBUTES, - TS.LOCAL_METHODS AS LOCAL_METHODS, + CAST(NULL AS NUMBER(38)) AS LOCAL_ATTRIBUTES, + CAST(NULL AS NUMBER(38)) AS LOCAL_METHODS, TS.TYPE_ID AS TYPEID FROM SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS @@ -41560,8 +41560,8 @@ def_table_schema( CAST('YES' AS CHAR(3)) AS INSTANTIABLE, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_OWNER, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_NAME, - T.LOCAL_ATTRS AS LOCAL_ATTRIBUTES, - T.LOCAL_METHODS AS LOCAL_METHODS, + CAST(NULL AS NUMBER(38)) AS LOCAL_ATTRIBUTES, + CAST(NULL AS NUMBER(38)) AS LOCAL_METHODS, T.TYPE_ID AS TYPEID FROM SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D @@ -41588,8 +41588,8 @@ def_table_schema( CAST('YES' AS CHAR(3)) AS INSTANTIABLE, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_OWNER, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_NAME, - TS.LOCAL_ATTRS AS LOCAL_ATTRIBUTES, - TS.LOCAL_METHODS AS LOCAL_METHODS, + CAST(NULL AS NUMBER(38)) AS LOCAL_ATTRIBUTES, + CAST(NULL AS NUMBER(38)) AS LOCAL_METHODS, TS.TYPE_ID AS TYPEID FROM SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS @@ -41622,8 +41622,8 @@ def_table_schema( CAST('YES' AS CHAR(3)) AS INSTANTIABLE, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_OWNER, CAST(NULL AS VARCHAR2(30)) AS SUPERTYPE_NAME, - T.LOCAL_ATTRS AS LOCAL_ATTRIBUTES, - T.LOCAL_METHODS AS LOCAL_METHODS, + CAST(NULL AS NUMBER(38)) AS LOCAL_ATTRIBUTES, + CAST(NULL AS NUMBER(38)) AS LOCAL_METHODS, T.TYPE_ID AS TYPEID FROM SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D diff --git a/src/share/schema/ob_udt_info.h b/src/share/schema/ob_udt_info.h index b181b0e9b..876f2126c 100644 --- a/src/share/schema/ob_udt_info.h +++ b/src/share/schema/ob_udt_info.h @@ -430,7 +430,7 @@ public: OB_INLINE void set_type_id(int64_t type_id) { type_id_ = type_id; } OB_INLINE void set_typecode(int64_t typecode) { typecode_ = typecode; } OB_INLINE void set_properties(int64_t properties) { properties_ = properties; } - OB_INLINE void set_attributes(int64_t attributes) { UNUSED(attributes);/*attributes_ = attributes;*/ } + OB_INLINE void set_attributes(int64_t attributes) { attributes_ = attributes; } OB_INLINE void set_methods(int64_t methods) { methods_ = methods; } OB_INLINE void set_hiddenmethods(int64_t hiddenmethods) { hiddenmethods_ = hiddenmethods; } OB_INLINE void set_supertypes(int64_t supertypes) { supertypes_ = supertypes; } diff --git a/src/sql/engine/expr/ob_expr_sql_udt_utils.cpp b/src/sql/engine/expr/ob_expr_sql_udt_utils.cpp index 662d33879..5dff52a1d 100644 --- a/src/sql/engine/expr/ob_expr_sql_udt_utils.cpp +++ b/src/sql/engine/expr/ob_expr_sql_udt_utils.cpp @@ -1504,7 +1504,7 @@ int ObSqlUdtMetaUtils::generate_udt_meta_from_schema(ObSchemaGetterGuard *schema udt_meta.udt_id_ = udt_id; if (root_udt_info->is_object_type()) { pl_type = static_cast(pl::PL_RECORD_TYPE); - child_attrs_cnt = root_udt_info->get_local_attrs(); + child_attrs_cnt = root_udt_info->get_attributes(); } else if (root_udt_info->is_varray()) { pl_type = static_cast(pl::PL_VARRAY_TYPE); if (OB_NOT_NULL(root_udt_info->get_coll_info())) { diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result index 0bf71a8d8..dce795d9e 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result @@ -1637,7 +1637,7 @@ body varchar(1048576) NO definer varchar(77) NO created timestamp(6) NO NULL modified timestamp(6) NO NULL -sql_mode varchar(32) NO +sql_mode varchar(1048576) NO comment varchar(1048576) NO character_set_client varchar(128) NO NULL collation_connection varchar(128) NO NULL diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result index 5b631a3c3..b65613a55 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result @@ -2134,7 +2134,7 @@ body varchar(1048576) NO definer varchar(77) NO created timestamp(6) NO NULL modified timestamp(6) NO NULL -sql_mode varchar(32) NO +sql_mode varchar(1048576) NO comment varchar(1048576) NO character_set_client varchar(128) NO NULL collation_connection varchar(128) NO NULL From 93a5cadec5f472b995868d382976cd92a6cf3a6d Mon Sep 17 00:00:00 2001 From: obdev Date: Mon, 19 Aug 2024 07:04:09 +0000 Subject: [PATCH 098/249] occupy __all_virtual_function_io_stat in master --- src/share/inner_table/ob_inner_table_schema_def.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index 6117549df..a5291f631 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -14660,6 +14660,7 @@ def_table_schema(**gen_iterate_virtual_table_def( # 12501: __all_virtual_wr_sql_plan # 12502: __all_virtual_wr_res_mgr_sysstat # 12503: __all_virtual_kv_redis_table +# 12504: __all_virtual_function_io_stat # 余留位置（此行之前占位） # 本区域占位建议：采用真实表名进行占位 ################################################################################ @@ -15166,7 +15167,7 @@ def_table_schema(**no_direct_access(gen_oracle_mapping_real_virtual_table_def('1 # 15481: __all_virtual_wr_sql_plan # 15482: __all_virtual_res_mgr_sysstat # 15483: __all_virtual_wr_res_mgr_sysstat - +# 15484: __all_virtual_function_io_stat # 余留位置（此行之前占位） # 本区域定义的Oracle表名比较复杂，一般都采用gen_xxx_table_def()方式定义，占位建议采用基表表名占位 # - 示例：def_table_schema(**no_direct_access(gen_oracle_mapping_virtual_table_def('15009', all_def_keywords['__all_virtual_sql_audit']))) @@ -35951,6 +35952,8 @@ def_table_schema( # 21617: CDB_OB_SPM_EVO_RESULT # 21618: DBA_OB_KV_REDIS_TABLE # 21619: CDB_OB_KV_REDIS_TABLE +# 21620: GV$OB_FUNCTION_IO_STAT +# 21621: V$OB_FUNCTION_IO_STAT # 余留位置（此行之前占位） # 本区域占位建议：采用真实视图名进行占位 ################################################################################ @@ -63986,7 +63989,8 @@ left join # 28259: DBA_WR_SQL_PLAN # 28260: DBA_WR_RES_MGR_SYSSTAT # 28261: DBA_OB_SPM_EVO_RESULT - +# 28262: GV$OB_FUNCTION_IO_STAT +# 28263: V$OB_FUNCTION_IO_STAT # 余留位置（此行之前占位） # 本区域占位建议：采用真实视图名进行占位 ################################################################################ From 04e46b934ab151edaa6eaaa831676fa765811a64 Mon Sep 17 00:00:00 2001 From: wxhwang Date: Mon, 19 Aug 2024 07:10:38 +0000 Subject: [PATCH 099/249] forbid restore from backup prior to 4.3.3 --- src/share/ob_upgrade_utils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/share/ob_upgrade_utils.cpp b/src/share/ob_upgrade_utils.cpp index d264065f1..5daa37d98 100755 --- a/src/share/ob_upgrade_utils.cpp +++ b/src/share/ob_upgrade_utils.cpp @@ -118,7 +118,7 @@ bool ObUpgradeChecker::check_data_version_exist( // For now, just consider the valid upgrade path for 4.x . bool ObUpgradeChecker::check_data_version_valid_for_backup(const uint64_t data_version) { - return DATA_VERSION_4_3_0_0 <= data_version; + return DATA_VERSION_4_3_3_0 <= data_version; } //FIXME:(yanmu.ztl) cluster version should be discrete. From bba362774faebc75f97cd5ca875279b94701f12c Mon Sep 17 00:00:00 2001 From: zhjc1124 Date: Mon, 19 Aug 2024 07:16:46 +0000 Subject: [PATCH 100/249] fix direct connection from SLB --- src/observer/ob_srv_deliver.cpp | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/src/observer/ob_srv_deliver.cpp b/src/observer/ob_srv_deliver.cpp index 7740043e9..520bf09a8 100644 --- a/src/observer/ob_srv_deliver.cpp +++ b/src/observer/ob_srv_deliver.cpp @@ -188,24 +188,20 @@ int get_user_tenant(ObRequest &req, char *user_name_buf, char *tenant_name_buf) char *endpoint_tenant_mapping_buf = nullptr; obmysql::OMPKHandshakeResponse hsr = static_cast(req.get_packet()); - if (OB_FAIL(hsr.decode())) { - LOG_WARN("decode hsr fail", K(ret)); + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(hsr.decode())) { + // ignore error and handle in ObMPConnect + LOG_WARN("decode hsr fail", K(tmp_ret)); + } else if (OB_TMP_FAIL(extract_user_tenant(hsr.get_username(), user_name, tenant_name))) { // ignore error and handle in ObMPConnect - ret = OB_SUCCESS; - } else if (OB_FAIL(extract_user_tenant(hsr.get_username(), user_name, tenant_name))) { LOG_WARN("parse user@tenant fail", K(ret), "str", hsr.get_username()); - // ignore error and handle in ObMPConnect - ret = OB_SUCCESS; - } else if (OB_FAIL(ObVTOAUtility::get_virtual_addr(fd, is_slb, vid, vaddr))) { - LOG_WARN("failed to get virtual addr", K(ret), K(fd)); + } else if (!tenant_name.empty()) { + // use this tenant_name } else { - if (!is_slb) { - // not from LB, do nothing - } else if (!tenant_name.empty()) { - ret = OB_ERR_UNEXPECTED; - LOG_DBA_WARN_V2(OB_SERVER_TENANT_NAME_NOT_EMPTY, OB_INVALID_CONFIG, - "The connections from the load balancer cannot have tenant names."); - } else { + if (OB_TMP_FAIL(ObVTOAUtility::get_virtual_addr(fd, is_slb, vid, vaddr))) { + LOG_WARN("failed to get virtual addr", K(tmp_ret), K(fd)); + } + if (is_slb) { const int64_t endpoint_tenant_mapping_buf_len = STRLEN(GCONF._endpoint_tenant_mapping.str()); endpoint_tenant_mapping_buf = static_cast(common::ob_malloc(sizeof(char) * (endpoint_tenant_mapping_buf_len + 1), "EndpointTenant")); From 284849d6d5f5f841a2becf14ce5054243fa539f4 Mon Sep 17 00:00:00 2001 From: yishenglanlingzui <395329313@qq.com> Date: Mon, 19 Aug 2024 08:09:24 +0000 Subject: [PATCH 101/249] master place_holder for pdml check_affected_row --- src/sql/engine/ob_exec_feedback_info.cpp | 3 ++- src/sql/engine/ob_exec_feedback_info.h | 5 +++-- src/sql/engine/ob_physical_plan_ctx.cpp | 6 +++++- src/sql/engine/ob_physical_plan_ctx.h | 1 + 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/sql/engine/ob_exec_feedback_info.cpp b/src/sql/engine/ob_exec_feedback_info.cpp index 71e2b4a46..28a55ce7c 100644 --- a/src/sql/engine/ob_exec_feedback_info.cpp +++ b/src/sql/engine/ob_exec_feedback_info.cpp @@ -25,7 +25,8 @@ OB_SERIALIZE_MEMBER(ObExecFeedbackNode, op_last_row_time_, db_time_, block_time_, - worker_count_); + worker_count_, + pdml_op_write_rows_); OB_SERIALIZE_MEMBER(ObExecFeedbackInfo, nodes_, diff --git a/src/sql/engine/ob_exec_feedback_info.h b/src/sql/engine/ob_exec_feedback_info.h index 66e91c03a..98bbbfeb3 100644 --- a/src/sql/engine/ob_exec_feedback_info.h +++ b/src/sql/engine/ob_exec_feedback_info.h @@ -29,10 +29,10 @@ struct ObExecFeedbackNode final public: ObExecFeedbackNode(int64_t op_id) : op_id_(op_id), output_row_count_(0), op_open_time_(INT64_MAX), op_close_time_(0), op_first_row_time_(INT64_MAX), - op_last_row_time_(0), db_time_(0), block_time_(0), worker_count_(0) {} + op_last_row_time_(0), db_time_(0), block_time_(0), worker_count_(0), pdml_op_write_rows_(0) {} ObExecFeedbackNode() : op_id_(OB_INVALID_ID), output_row_count_(0), op_open_time_(INT64_MAX), op_close_time_(0), op_first_row_time_(INT64_MAX), - op_last_row_time_(0), db_time_(0), block_time_(0), worker_count_(0) {} + op_last_row_time_(0), db_time_(0), block_time_(0), worker_count_(0), pdml_op_write_rows_(0) {} ~ObExecFeedbackNode() {} TO_STRING_KV(K_(op_id), K_(output_row_count), K_(op_open_time), K_(op_close_time), K_(op_first_row_time), K_(op_last_row_time), @@ -47,6 +47,7 @@ public: int64_t db_time_; // rdtsc cpu cycles spend on this op, include cpu instructions & io int64_t block_time_; // rdtsc cpu cycles wait for network, io etc int64_t worker_count_; + int64_t pdml_op_write_rows_; }; class ObExecFeedbackInfo final diff --git a/src/sql/engine/ob_physical_plan_ctx.cpp b/src/sql/engine/ob_physical_plan_ctx.cpp index 1bcbe5f3d..034a113e0 100644 --- a/src/sql/engine/ob_physical_plan_ctx.cpp +++ b/src/sql/engine/ob_physical_plan_ctx.cpp @@ -126,7 +126,8 @@ ObPhysicalPlanCtx::ObPhysicalPlanCtx(common::ObIAllocator &allocator) main_xa_trans_branch_(false), total_memstore_read_row_count_(0), total_ssstore_read_row_count_(0), - is_direct_insert_plan_(false) + is_direct_insert_plan_(false), + check_pdml_affected_rows_(false) { } @@ -780,6 +781,7 @@ OB_DEF_SERIALIZE(ObPhysicalPlanCtx) OB_UNIS_ENCODE(mview_ids_); OB_UNIS_ENCODE(last_refresh_scns_); OB_UNIS_ENCODE(is_direct_insert_plan_); + OB_UNIS_ENCODE(check_pdml_affected_rows_); return ret; } @@ -880,6 +882,7 @@ OB_DEF_SERIALIZE_SIZE(ObPhysicalPlanCtx) OB_UNIS_ADD_LEN(mview_ids_); OB_UNIS_ADD_LEN(last_refresh_scns_); OB_UNIS_ADD_LEN(is_direct_insert_plan_); + OB_UNIS_ADD_LEN(check_pdml_affected_rows_); return len; } @@ -1006,6 +1009,7 @@ OB_DEF_DESERIALIZE(ObPhysicalPlanCtx) OB_UNIS_DECODE(mview_ids_); OB_UNIS_DECODE(last_refresh_scns_); OB_UNIS_DECODE(is_direct_insert_plan_); + OB_UNIS_DECODE(check_pdml_affected_rows_); return ret; } diff --git a/src/sql/engine/ob_physical_plan_ctx.h b/src/sql/engine/ob_physical_plan_ctx.h index 38efe75e4..4e68c1807 100644 --- a/src/sql/engine/ob_physical_plan_ctx.h +++ b/src/sql/engine/ob_physical_plan_ctx.h @@ -689,6 +689,7 @@ private: int64_t total_memstore_read_row_count_; int64_t total_ssstore_read_row_count_; bool is_direct_insert_plan_; // for direct load: insert into/overwrite select + bool check_pdml_affected_rows_; // now only worked for pdml checking affected_rows }; } From 654605f4409e60df8edd2fe08da375da95b7a3b0 Mon Sep 17 00:00:00 2001 From: liucc1997 <1192520566@qq.com> Date: Mon, 19 Aug 2024 08:33:07 +0000 Subject: [PATCH 102/249] [CP] fix get_next_cond unlock failed --- deps/oblib/src/rpc/obrpc/ob_rpc_session_handler.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_session_handler.cpp b/deps/oblib/src/rpc/obrpc/ob_rpc_session_handler.cpp index aed8f8745..b03eccfe3 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_session_handler.cpp +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_session_handler.cpp @@ -241,9 +241,9 @@ int ObRpcSessionHandler::wait_for_next_request(int64_t sessid, // when waiting for OB_REMOTE_EXECUTE/OB_REMOTE_SYNC_EXECUTE/OB_INNER_SQL_SYNC_TRANSMIT request more than 30s, // try to send reverse keepalive request. if (current_time_us >= keepalive_timeout_us && reverse_keepalive_arg.is_valid()) { - get_next_cond_(wait_object.thid_).unlock(); + get_next_cond_(thid).unlock(); ret = stream_rpc_reverse_probe(reverse_keepalive_arg); - get_next_cond_(wait_object.thid_).lock(); + get_next_cond_(thid).lock(); if (OB_FAIL(ret)) { LOG_WARN("stream rpc sender has been aborted, unneed to wait", K(sessid), K(timeout), K(reverse_keepalive_arg)); break; @@ -276,7 +276,7 @@ int ObRpcSessionHandler::wait_for_next_request(int64_t sessid, K(hash_ret), K(sessid), K(req)); } - get_next_cond_(wait_object.thid_).unlock(); + get_next_cond_(thid).unlock(); ATOMIC_DEC(&waiting_thread_count_); } else { //do nothing From a4b79b6ac7ff78c8a38dd02b2ac8a5e4882fb84b Mon Sep 17 00:00:00 2001 From: wjhh2008 Date: Mon, 19 Aug 2024 08:39:30 +0000 Subject: [PATCH 103/249] [CP] fix csv external table bug --- src/sql/das/ob_das_scan_op.cpp | 1 + src/sql/engine/cmd/ob_load_data_parser.h | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/sql/das/ob_das_scan_op.cpp b/src/sql/das/ob_das_scan_op.cpp index 1d7ba10e6..b65b18238 100644 --- a/src/sql/das/ob_das_scan_op.cpp +++ b/src/sql/das/ob_das_scan_op.cpp @@ -313,6 +313,7 @@ int ObDASScanOp::init_scan_param() } } scan_param_.external_file_format_.csv_format_.file_column_nums_ = static_cast(max_idx); + scan_param_.external_file_format_.csv_format_.ignore_extra_fields_ = true; } } LOG_DEBUG("init scan param", K(ret), K(scan_param_)); diff --git a/src/sql/engine/cmd/ob_load_data_parser.h b/src/sql/engine/cmd/ob_load_data_parser.h index fd43ec831..14870e89d 100644 --- a/src/sql/engine/cmd/ob_load_data_parser.h +++ b/src/sql/engine/cmd/ob_load_data_parser.h @@ -40,6 +40,7 @@ struct ObCSVGeneralFormat { cs_type_(common::CHARSET_INVALID), skip_header_lines_(0), skip_blank_lines_(false), + ignore_extra_fields_(false), trim_space_(false), null_if_(), empty_field_as_null_(false), @@ -65,6 +66,7 @@ struct ObCSVGeneralFormat { common::ObCharsetType cs_type_; // charset type of format strings int64_t skip_header_lines_; bool skip_blank_lines_; + bool ignore_extra_fields_; bool trim_space_; common::ObArrayWrap null_if_; bool empty_field_as_null_; @@ -442,7 +444,8 @@ int ObCSVGeneralParser::scan_proto(const char *&str, } if (OB_LIKELY(find_new_line) || is_end_file) { if (!format_.skip_blank_lines_ || field_idx > 0) { - if (field_idx != format_.file_column_nums_) { + if (field_idx < format_.file_column_nums_ + || (field_idx > format_.file_column_nums_ && !format_.ignore_extra_fields_)) { ret = handle_irregular_line(field_idx, line_no, errors); } if (OB_SUCC(ret)) { From 4fdf9e4944aa59aa019b05e6ff901cfa335c2635 Mon Sep 17 00:00:00 2001 From: bit-dance <2634358021@qq.com> Date: Mon, 19 Aug 2024 08:45:20 +0000 Subject: [PATCH 104/249] [to #2024081600104152821]fix declare variable with default value core in mysql mode --- src/pl/ob_pl_resolver.cpp | 3 ++- src/pl/parser/pl_parser_mysql_mode.y | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/pl/ob_pl_resolver.cpp b/src/pl/ob_pl_resolver.cpp index 7366dd993..f0eae06a5 100644 --- a/src/pl/ob_pl_resolver.cpp +++ b/src/pl/ob_pl_resolver.cpp @@ -3873,7 +3873,8 @@ int ObPLResolver::resolve_declare_var_comm(const ObStmtNodeTree *parse_tree, } } - if (T_CHAR == default_node->children_[0]->type_ + if (OB_SUCC(ret) + && T_CHAR == default_node->children_[0]->type_ && (T_CHAR == type_node->type_ || T_NCHAR == type_node->type_) && default_node->children_[0]->str_len_ == 0) { default_node->children_[0]->str_len_++; diff --git a/src/pl/parser/pl_parser_mysql_mode.y b/src/pl/parser/pl_parser_mysql_mode.y index 3a2b7f9f3..5ccd95be9 100644 --- a/src/pl/parser/pl_parser_mysql_mode.y +++ b/src/pl/parser/pl_parser_mysql_mode.y @@ -1844,6 +1844,9 @@ opt_sp_decl_default: /*Empty*/ { $$ = NULL; } | DEFAULT default_expr { + if (NULL == $2) { + YYERROR; + } malloc_non_terminal_node($$, parse_ctx->mem_pool_, T_SP_DECL_DEFAULT, 1, $2); } ; From f4c874d121274a526695810834362078abd6e578 Mon Sep 17 00:00:00 2001 From: sdc Date: Mon, 19 Aug 2024 09:24:43 +0000 Subject: [PATCH 105/249] [CP] fix physical plan worker map memory leak --- src/sql/engine/ob_physical_plan.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sql/engine/ob_physical_plan.cpp b/src/sql/engine/ob_physical_plan.cpp index 97b4e69e3..c603966b0 100644 --- a/src/sql/engine/ob_physical_plan.cpp +++ b/src/sql/engine/ob_physical_plan.cpp @@ -1191,7 +1191,7 @@ int ObPhysicalPlan::set_minimal_worker_map(const common::hash::ObHashMap &c) { int ret = OB_SUCCESS; - ObMemAttr attr(MTL_ID(), "WorkerMap"); + ObMemAttr attr(tenant_id_, "WorkerMap"); if (worker_map.created()) { worker_map.clear(); } else if (OB_FAIL(worker_map.create(common::hash::cal_next_prime(100), attr, attr))){ From 0925c8d1fc342ad9f402e4a408e33f9c472d2595 Mon Sep 17 00:00:00 2001 From: AA-tuliwei-BB <1123152962@qq.com> Date: Mon, 19 Aug 2024 11:24:50 +0000 Subject: [PATCH 106/249] fix update set subquery coalesce bug --- src/sql/rewrite/ob_transform_subquery_coalesce.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/sql/rewrite/ob_transform_subquery_coalesce.cpp b/src/sql/rewrite/ob_transform_subquery_coalesce.cpp index 9a926a275..4d1625946 100644 --- a/src/sql/rewrite/ob_transform_subquery_coalesce.cpp +++ b/src/sql/rewrite/ob_transform_subquery_coalesce.cpp @@ -1711,6 +1711,9 @@ int ObTransformSubqueryCoalesce::get_subquery_assign_exprs(ObIArray } else if (alias_exprs.count() > 1 || query_ref_exprs.count() > 1) { //disable subquery coalescing in this scenes is_valid = false; + } else if ((query_ref_exprs.count() == 1 && query_ref_exprs.at(0)->get_ref_count() > 1) || + (alias_exprs.count() == 1 && alias_exprs.at(0)->get_ref_count() > 1)) { + is_valid = false; } for (int64_t j = 0; OB_SUCC(ret) && is_valid && j < query_ref_exprs.count(); ++j) { ObQueryRefRawExpr *query_ref_expr = query_ref_exprs.at(j); From 203cab32b670c0fb4f2ac976b94221b5fdc3a3f4 Mon Sep 17 00:00:00 2001 From: obdev Date: Mon, 19 Aug 2024 11:30:40 +0000 Subject: [PATCH 107/249] Minor, placeholder the serialized symbol for select_into op --- src/sql/engine/basic/ob_select_into_op.cpp | 2 +- src/sql/engine/basic/ob_select_into_op.h | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/sql/engine/basic/ob_select_into_op.cpp b/src/sql/engine/basic/ob_select_into_op.cpp index b16d1d091..efb13d6c0 100644 --- a/src/sql/engine/basic/ob_select_into_op.cpp +++ b/src/sql/engine/basic/ob_select_into_op.cpp @@ -31,7 +31,7 @@ OB_SERIALIZE_MEMBER((ObSelectIntoSpec, ObOpSpec), into_type_, user_vars_, outfil field_str_, // FARM COMPAT WHITELIST FOR filed_str_: renamed line_str_, closed_cht_, is_optional_, select_exprs_, is_single_, max_file_size_, escaped_cht_, cs_type_, parallel_, file_partition_expr_, buffer_size_, is_overwrite_, - external_properties_, external_partition_); + external_properties_, external_partition_, per_row_group_size_, compression_algorithm_); int ObSelectIntoOp::inner_open() diff --git a/src/sql/engine/basic/ob_select_into_op.h b/src/sql/engine/basic/ob_select_into_op.h index f6e9a4bff..47c107e81 100644 --- a/src/sql/engine/basic/ob_select_into_op.h +++ b/src/sql/engine/basic/ob_select_into_op.h @@ -13,6 +13,11 @@ #ifndef SRC_SQL_ENGINE_BASIC_OB_SELECT_INTO_OP_H_ #define SRC_SQL_ENGINE_BASIC_OB_SELECT_INTO_OP_H_ +#include +#include +#include +#include + #include "sql/engine/ob_operator.h" #include "lib/file/ob_file.h" #include "common/storage/ob_io_device.h" @@ -73,6 +78,8 @@ public: external_properties_(alloc), external_partition_(alloc) { + compression_algorithm_ = parquet::Compression::UNCOMPRESSED; + per_row_group_size_ = DEFAULT_MAX_FILE_SIZE; cs_type_ = ObCharset::get_system_collation(); } @@ -95,6 +102,8 @@ public: bool is_overwrite_; ObExternalFileFormat::StringData external_properties_; ObExternalFileFormat::StringData external_partition_; + int64_t per_row_group_size_; + parquet::Compression::type compression_algorithm_; static const int64_t DEFAULT_MAX_FILE_SIZE = 256LL * 1024 * 1024; static const int64_t DEFAULT_BUFFER_SIZE = 1LL * 1024 * 1024; }; From a99910d1be67d60476acf24a4acc6971c85c50d7 Mon Sep 17 00:00:00 2001 From: sdc Date: Mon, 19 Aug 2024 11:49:20 +0000 Subject: [PATCH 108/249] fix calc slice index report 4201 when first level partition is fixed --- src/sql/executor/ob_slice_calc.cpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/sql/executor/ob_slice_calc.cpp b/src/sql/executor/ob_slice_calc.cpp index 5a8a28f8a..cdc5c66ea 100644 --- a/src/sql/executor/ob_slice_calc.cpp +++ b/src/sql/executor/ob_slice_calc.cpp @@ -249,7 +249,12 @@ int ObRepartSliceIdxCalc::get_sub_part_id_by_one_level_first_ch_map( int ret = OB_SUCCESS; if (part2tablet_id_map_.size() > 0) { if (OB_FAIL(part2tablet_id_map_.get_refactored(part_id, tablet_id))) { - LOG_WARN("fail to get tablet id", K(ret)); + if (OB_HASH_NOT_EXIST == ret) { + tablet_id = ObExprCalcPartitionId::NONE_PARTITION_ID; + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to get tablet id", K(ret), K(part2tablet_id_map_.size())); + } } } else { ret = OB_ERR_UNEXPECTED; @@ -270,11 +275,9 @@ int ObRepartSliceIdxCalc::get_tablet_id(ObEvalCtx &eval_ctx, int64_t &tab LOG_WARN("fail to calc part id", K(ret), K(*calc_part_id_expr_)); } else if (ObExprCalcPartitionId::NONE_PARTITION_ID == (tablet_id = tablet_id_datum->get_int())) { - // Usually, calc_part_id_expr_ returns tablet_id and set tablet_id_datum = 0 - // if no partition match(refer to ObExprCalcPartitionBase::calc_partition_id). - // In this condition, it will not go to this branch because NONE_PARTITION_ID equals to -1. - // But when repart_type_ equals to OB_REPARTITION_ONE_SIDE_ONE_LEVEL_FIRST(means value of sub part key is fixed), - // calc_part_id_expr_ returns partition_id of first part and set tablet_id_datum = -1. + // When repart_type_ equals to OB_REPARTITION_ONE_SIDE_ONE_LEVEL_FIRST(means value of sub part key is fixed), + // calc_part_id_expr_ returns partition_id of first level partition and + // set tablet_id_datum = NONE_PARTITION_ID if no first level partition match. } else if (OB_REPARTITION_ONE_SIDE_ONE_LEVEL_FIRST == repart_type_) { int64_t part_id = tablet_id; if (OB_FAIL(get_sub_part_id_by_one_level_first_ch_map(part_id, tablet_id))) { @@ -352,7 +355,7 @@ int ObRepartSliceIdxCalc::get_tablet_ids(ObEvalCtx &eval_ctx, ObBitVector } else if (OB_REPARTITION_ONE_SIDE_ONE_LEVEL_FIRST == repart_type_) { int64_t part_id = ObSliceIdxCalc::tablet_ids_[i]; if (OB_FAIL(get_sub_part_id_by_one_level_first_ch_map(part_id, ObSliceIdxCalc::tablet_ids_[i]))) { - LOG_WARN("fail to get part id by ch map", K(ret)); + LOG_WARN("fail to get part id by ch map", K(ret), K(part_id)); } } if (OB_SUCC(ret)) { From 951f3da82ffd64f212ff61315bb00b56f113bece Mon Sep 17 00:00:00 2001 From: hanr881 <1741282579@qq.com> Date: Mon, 19 Aug 2024 11:56:01 +0000 Subject: [PATCH 109/249] [CP] to issue<2024080700104067845>:fix pl cache hit issue when pl contain sql with synonym --- src/sql/resolver/expr/ob_raw_expr.h | 5 +++ src/sql/resolver/ob_resolver_utils.cpp | 46 ++++++++++++++++---------- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/src/sql/resolver/expr/ob_raw_expr.h b/src/sql/resolver/expr/ob_raw_expr.h index 0f29e2ea4..8efe637f5 100644 --- a/src/sql/resolver/expr/ob_raw_expr.h +++ b/src/sql/resolver/expr/ob_raw_expr.h @@ -4131,6 +4131,11 @@ public: inline void set_pkg_body_udf(bool v) { is_pkg_body_udf_ = v; } inline bool is_pkg_body_udf() const { return is_pkg_body_udf_; } + inline bool is_standalone_udf() const + { + return common::OB_INVALID_ID == pkg_id_ && common::OB_INVALID_ID == type_id_; + } + VIRTUAL_TO_STRING_KV_CHECK_STACK_OVERFLOW(N_ITEM_TYPE, type_, N_RESULT_TYPE, result_type_, N_EXPR_INFO, info_, diff --git a/src/sql/resolver/ob_resolver_utils.cpp b/src/sql/resolver/ob_resolver_utils.cpp index 03085e54b..5e7226130 100644 --- a/src/sql/resolver/ob_resolver_utils.cpp +++ b/src/sql/resolver/ob_resolver_utils.cpp @@ -265,9 +265,10 @@ int ObResolverUtils::collect_schema_version(share::schema::ObSchemaGetterGuard & CK (OB_NOT_NULL(udf_expr)); if (OB_SUCC(ret) && udf_expr->need_add_dependency()) { OZ (schema_guard.get_database_id(session_info->get_effective_tenant_id(), - udf_expr->get_database_name().empty() ? session_info->get_database_name() : udf_expr->get_database_name(), - database_id)); - if (OB_SUCC(ret)) { + (udf_expr->get_database_name().empty() || (0 == udf_expr->get_database_name().case_compare(OB_SYS_DATABASE_NAME))) + ? session_info->get_database_name() : udf_expr->get_database_name(), + database_id)); + if (OB_SUCC(ret) && udf_expr->is_standalone_udf()) { bool exist = false; uint64_t object_db_id = OB_INVALID_ID; ObSchemaChecker schema_checker; @@ -281,21 +282,30 @@ int ObResolverUtils::collect_schema_version(share::schema::ObSchemaGetterGuard & udf_expr->get_func_name(), object_db_id, object_name, exist)); if (OB_SUCC(ret) && exist) { - for (int64_t i = 0; OB_SUCC(ret) && i < synonym_checker.get_synonym_ids().count(); ++i) { - int64_t schema_version = OB_INVALID_VERSION; - uint64_t obj_id = synonym_checker.get_synonym_ids().at(i); - uint64_t dep_db_id = synonym_checker.get_database_ids().at(i); - ObSchemaObjVersion syn_version; - OZ (schema_guard.get_schema_version(SYNONYM_SCHEMA, - session_info->get_effective_tenant_id(), - obj_id, - schema_version)); - OX (syn_version.object_id_ = obj_id); - OX (syn_version.version_ = schema_version); - OX (syn_version.object_type_ = DEPENDENCY_SYNONYM); - OZ (dependency_objects.push_back(syn_version)); - if (OB_NOT_NULL(dep_db_array)) { - OZ (dep_db_array->push_back(dep_db_id)); + bool exist_non_syn_object= false; + bool is_private_syn = false; + OZ (schema_checker.check_exist_same_name_object_with_synonym(session_info->get_effective_tenant_id(), + database_id, + udf_expr->get_func_name(), + exist_non_syn_object, + is_private_syn)); + if (OB_SUCC(ret) && (!exist_non_syn_object || is_private_syn)) { + for (int64_t i = 0; OB_SUCC(ret) && i < synonym_checker.get_synonym_ids().count(); ++i) { + int64_t schema_version = OB_INVALID_VERSION; + uint64_t obj_id = synonym_checker.get_synonym_ids().at(i); + uint64_t dep_db_id = synonym_checker.get_database_ids().at(i); + ObSchemaObjVersion syn_version; + OZ (schema_guard.get_schema_version(SYNONYM_SCHEMA, + session_info->get_effective_tenant_id(), + obj_id, + schema_version)); + OX (syn_version.object_id_ = obj_id); + OX (syn_version.version_ = schema_version); + OX (syn_version.object_type_ = DEPENDENCY_SYNONYM); + OZ (dependency_objects.push_back(syn_version)); + if (OB_NOT_NULL(dep_db_array)) { + OZ (dep_db_array->push_back(dep_db_id)); + } } } } From f3d74d99657c26165fc67ad7b3e1392b7a162d38 Mon Sep 17 00:00:00 2001 From: HaHaJeff Date: Mon, 19 Aug 2024 12:17:44 +0000 Subject: [PATCH 110/249] rpc place holder --- deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h index d2e7e8c7d..7ed1aaa1f 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h @@ -1042,6 +1042,10 @@ PCODE_DEF(OB_LOG_ACQUIRE_REBUILD_INFO, 0x1525) // for ob_admin PCODE_DEF(OB_LOG_FORCE_SET_TENANT_LOG_DISK, 0x1526) PCODE_DEF(OB_LOG_SYNC_BASE_LSN_REQ, 0x1527) +#ifdef OB_BUILD_SHARED_STORAGE +PCODE_DEF(OB_LOG_GET_UPLOADED_PROGRESS, 0x1528) +PCODE_DEF(OB_LOG_FLUSH_UNTIL, 0x1529) +#endif // 1531-1550 for obesi // PCODE_DEF(OB_ESI_IS_EXIST, 0x1531) From 74184e04d58c19fa92ec64e54c6f0b86d874c45e Mon Sep 17 00:00:00 2001 From: wanyue-wy <345657357@qq.com> Date: Mon, 19 Aug 2024 12:53:18 +0000 Subject: [PATCH 111/249] fix reset bug of tmp file ctx --- .../mtlenv/storage/tmp_file/test_tmp_file.cpp | 85 +++++++++++++++++++ src/storage/tmp_file/ob_tmp_file_io_ctx.cpp | 10 ++- src/storage/tmp_file/ob_tmp_file_manager.cpp | 20 ++++- 3 files changed, 108 insertions(+), 7 deletions(-) diff --git a/mittest/mtlenv/storage/tmp_file/test_tmp_file.cpp b/mittest/mtlenv/storage/tmp_file/test_tmp_file.cpp index e9eb74acc..fb9137d0f 100644 --- a/mittest/mtlenv/storage/tmp_file/test_tmp_file.cpp +++ b/mittest/mtlenv/storage/tmp_file/test_tmp_file.cpp @@ -1430,6 +1430,91 @@ TEST_F(TestTmpFile, test_big_file_disable_page_cache) test_big_file(write_size, wbp_mem_limit, io_info); } +TEST_F(TestTmpFile, test_aio_pread) +{ + int ret = OB_SUCCESS; + const int64_t write_size = 10 * 1024 * 1024; // 10MB + char *write_buf = new char [write_size]; + for (int64_t i = 0; i < write_size;) { + int64_t random_length = generate_random_int(1024, 8 * 1024); + int64_t random_int = generate_random_int(0, 256); + for (int64_t j = 0; j < random_length && i + j < write_size; ++j) { + write_buf[i + j] = random_int; + } + i += random_length; + } + + int64_t dir = -1; + int64_t fd = -1; + ret = MTL(ObTenantTmpFileManager *)->alloc_dir(dir); + ASSERT_EQ(OB_SUCCESS, ret); + ret = MTL(ObTenantTmpFileManager *)->open(fd, dir); + std::cout << "open temporary file: " << fd << std::endl; + ASSERT_EQ(OB_SUCCESS, ret); + ObTmpFileHandle file_handle; + ret = MTL(ObTenantTmpFileManager *)->get_tmp_file(fd, file_handle); + ASSERT_EQ(OB_SUCCESS, ret); + file_handle.get()->page_idx_cache_.max_bucket_array_capacity_ = SMALL_WBP_IDX_CACHE_MAX_CAPACITY; + + ObTmpFileIOInfo io_info; + io_info.fd_ = fd; + io_info.io_desc_.set_wait_event(2); + io_info.buf_ = write_buf; + io_info.size_ = write_size; + io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; + + // 1. Write data + int64_t write_time = ObTimeUtility::current_time(); + ret = MTL(ObTenantTmpFileManager *)->write(io_info); + write_time = ObTimeUtility::current_time() - write_time; + ASSERT_EQ(OB_SUCCESS, ret); + + // 2. check aio_pread + int64_t read_size = 9 * 1024 * 1024; // 9MB + int64_t read_offset = 0; + char *read_buf = new char [read_size]; + ObTmpFileIOHandle handle; + io_info.buf_ = read_buf; + io_info.size_ = read_size; + ret = MTL(ObTenantTmpFileManager *)->aio_pread(io_info, read_offset, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(0, handle.get_done_size()); + ret = handle.wait(); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(io_info.size_, handle.get_done_size()); + int cmp = memcmp(handle.get_buffer(), write_buf + read_offset, io_info.size_); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + + // 3. execute two aio_pread, but io_handle doesn't not call wait() + read_size = 5 * 1024 * 1024; // 5MB + read_offset = 0; + read_buf = new char [read_size]; + io_info.buf_ = read_buf; + io_info.size_ = read_size; + ret = MTL(ObTenantTmpFileManager *)->aio_pread(io_info, read_offset, handle); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(0, handle.get_done_size()); + + int read_offset2 = read_offset + read_size; + ret = MTL(ObTenantTmpFileManager *)->aio_pread(io_info, read_offset2, handle); + ASSERT_NE(OB_SUCCESS, ret); + + ret = handle.wait(); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(io_info.size_, handle.get_done_size()); + cmp = memcmp(handle.get_buffer(), write_buf + read_offset, io_info.size_); + ASSERT_EQ(0, cmp); + handle.reset(); + delete[] read_buf; + + file_handle.reset(); + ret = MTL(ObTenantTmpFileManager *)->remove(fd); + ASSERT_EQ(OB_SUCCESS, ret); + + LOG_INFO("test_cached_read"); +} } // namespace oceanbase int main(int argc, char **argv) diff --git a/src/storage/tmp_file/ob_tmp_file_io_ctx.cpp b/src/storage/tmp_file/ob_tmp_file_io_ctx.cpp index bc846f7e7..0930b1dd3 100644 --- a/src/storage/tmp_file/ob_tmp_file_io_ctx.cpp +++ b/src/storage/tmp_file/ob_tmp_file_io_ctx.cpp @@ -94,8 +94,12 @@ void ObTmpFileIOCtx::reuse() for (int32_t i = 0; i < page_cache_handles_.count(); i++) { page_cache_handles_.at(i).page_handle_.reset(); } + for (int32_t i = 0; i < block_cache_handles_.count(); i++) { + block_cache_handles_.at(i).block_handle_.reset(); + } io_handles_.reset(); page_cache_handles_.reset(); + block_cache_handles_.reset(); } void ObTmpFileIOCtx::reset() @@ -264,7 +268,7 @@ int ObTmpFileIOCtx::do_read_wait_() char * read_buf = page_cache_handle.dest_user_read_buf_; if (OB_UNLIKELY(!check_buf_range_valid(read_buf, read_size))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid range", KR(ret), K(read_buf), K(read_size), K(buf_size_)); + LOG_WARN("invalid range", KR(ret), KP(read_buf), KP(buf_), K(read_size), K(buf_size_)); } else if (OB_UNLIKELY(offset_in_page + read_size > ObTmpFileGlobal::PAGE_SIZE)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("read size is over than page range", KR(ret), KPC(this), K(offset_in_page), K(read_size)); @@ -290,7 +294,7 @@ int ObTmpFileIOCtx::do_read_wait_() char * read_buf = block_cache_handle.dest_user_read_buf_; if (OB_UNLIKELY(!check_buf_range_valid(read_buf, read_size))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid range", KR(ret), K(read_buf), K(read_size), K(buf_size_)); + LOG_WARN("invalid range", KR(ret), KP(read_buf), KP(buf_), K(read_size), K(buf_size_)); } else if (OB_UNLIKELY(offset_in_block + read_size > OB_DEFAULT_MACRO_BLOCK_SIZE)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("read size is over than macro block range", KR(ret), KPC(this), K(offset_in_block), K(read_size)); @@ -319,7 +323,7 @@ int ObTmpFileIOCtx::do_read_wait_() char * read_buf = io_handle.dest_user_read_buf_; if (OB_UNLIKELY(!check_buf_range_valid(read_buf, size))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid range", KR(ret), K(read_buf), K(size), K(buf_size_)); + LOG_WARN("invalid range", KR(ret), KP(read_buf), KP(buf_), K(size), K(buf_size_)); } else { MEMCPY(read_buf, data_buf + offset, size); io_handle.handle_.reset(); diff --git a/src/storage/tmp_file/ob_tmp_file_manager.cpp b/src/storage/tmp_file/ob_tmp_file_manager.cpp index 69f4109a9..790dbfd14 100644 --- a/src/storage/tmp_file/ob_tmp_file_manager.cpp +++ b/src/storage/tmp_file/ob_tmp_file_manager.cpp @@ -290,7 +290,6 @@ int ObTenantTmpFileManager::aio_read(const ObTmpFileIOInfo &io_info, ObTmpFileIO { int ret = OB_SUCCESS; ObTmpFileHandle tmp_file_handle; - io_handle.reset(); if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -298,6 +297,10 @@ int ObTenantTmpFileManager::aio_read(const ObTmpFileIOInfo &io_info, ObTmpFileIO } else if (!io_info.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("fail to aio read, invalid argument", KR(ret), K(io_info)); + } else if (OB_UNLIKELY(io_handle.is_valid() && !io_handle.is_finished())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tmp file io handle has remain data need to be waited", KR(ret), K(io_info), K(io_handle)); + } else if (FALSE_IT(io_handle.reset())) { } else if (OB_FAIL(get_tmp_file(io_info.fd_, tmp_file_handle))) { LOG_WARN("fail to get tmp file io handle", KR(ret), K(io_info)); } else if (OB_FAIL(io_handle.init_read(io_info, tmp_file_handle))) { @@ -316,7 +319,6 @@ int ObTenantTmpFileManager::aio_pread(const ObTmpFileIOInfo &io_info, { int ret = OB_SUCCESS; ObTmpFileHandle tmp_file_handle; - io_handle.reset(); if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -324,6 +326,10 @@ int ObTenantTmpFileManager::aio_pread(const ObTmpFileIOInfo &io_info, } else if (!io_info.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("fail to aio read, invalid argument", KR(ret), K(io_info)); + } else if (OB_UNLIKELY(io_handle.is_valid() && !io_handle.is_finished())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tmp file io handle has remain data need to be waited", KR(ret), K(io_info), K(io_handle)); + } else if (FALSE_IT(io_handle.reset())) { } else if (OB_FAIL(get_tmp_file(io_info.fd_, tmp_file_handle))) { LOG_WARN("fail to get tmp file io handle", KR(ret), K(io_info)); } else if (OB_FAIL(io_handle.init_pread(io_info, offset, tmp_file_handle))) { @@ -340,7 +346,6 @@ int ObTenantTmpFileManager::read(const ObTmpFileIOInfo &io_info, ObTmpFileIOHand { int ret = OB_SUCCESS; ObTmpFileHandle tmp_file_handle; - io_handle.reset(); if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -348,6 +353,10 @@ int ObTenantTmpFileManager::read(const ObTmpFileIOInfo &io_info, ObTmpFileIOHand } else if (!io_info.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("fail to aio read, invalid argument", KR(ret), K(io_info)); + } else if (OB_UNLIKELY(io_handle.is_valid() && !io_handle.is_finished())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tmp file io handle has remain data need to be waited", KR(ret), K(io_info), K(io_handle)); + } else if (FALSE_IT(io_handle.reset())) { } else if (OB_FAIL(get_tmp_file(io_info.fd_, tmp_file_handle))) { LOG_WARN("fail to get tmp file io handle", KR(ret), K(io_info)); } else if (OB_FAIL(io_handle.init_read(io_info, tmp_file_handle))) { @@ -372,7 +381,6 @@ int ObTenantTmpFileManager::pread(const ObTmpFileIOInfo &io_info, const int64_t { int ret = OB_SUCCESS; ObTmpFileHandle tmp_file_handle; - io_handle.reset(); if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -380,6 +388,10 @@ int ObTenantTmpFileManager::pread(const ObTmpFileIOInfo &io_info, const int64_t } else if (!io_info.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("fail to aio read, invalid argument", KR(ret), K(io_info)); + } else if (OB_UNLIKELY(io_handle.is_valid() && !io_handle.is_finished())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tmp file io handle has remain data need to be waited", KR(ret), K(io_info), K(io_handle)); + } else if (FALSE_IT(io_handle.reset())) { } else if (OB_FAIL(get_tmp_file(io_info.fd_, tmp_file_handle))) { LOG_WARN("fail to get tmp file io handle", KR(ret), K(io_info)); } else if (OB_FAIL(io_handle.init_pread(io_info, offset, tmp_file_handle))) { From 19be5ac56241ffb5db224d3f617b8c2ef04af7fe Mon Sep 17 00:00:00 2001 From: 0xacc Date: Mon, 19 Aug 2024 14:38:29 +0000 Subject: [PATCH 112/249] [CP] [to #2024071900103852700] fix: always call DEBUG_STOP after DEBUG_START succeeded --- src/pl/ob_pl.cpp | 69 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 65 insertions(+), 4 deletions(-) diff --git a/src/pl/ob_pl.cpp b/src/pl/ob_pl.cpp index eba9af041..865730be5 100644 --- a/src/pl/ob_pl.cpp +++ b/src/pl/ob_pl.cpp @@ -69,6 +69,12 @@ extern int sys_pkg_need_priv_check(uint64_t pkg_id, ObSchemaGetterGuard *schema_ } namespace pl { + + +#ifdef ERRSIM +ERRSIM_POINT_DEF(OBPLCONTEXT_INIT); +#endif // ERRSIM + int ObPL::init(common::ObMySQLProxy &sql_proxy) { int ret = OB_SUCCESS; @@ -608,6 +614,11 @@ int ObPLContext::init(ObSQLSessionInfo &session_info, const bool is_dblink) { int ret = OB_SUCCESS; + + // to mark what session status we need to do to rollback if init failed + bool need_remove_top_stack = false; + bool need_debug_stop = false; + int64_t pl_block_timeout = 0; int64_t query_start_time = session_info.get_query_start_time(); if (!is_dblink) { @@ -648,7 +659,6 @@ int ObPLContext::init(ObSQLSessionInfo &session_info, } } if (lib::is_oracle_mode()) { - OZ (ObPLContext::debug_start(&session_info)); if (OB_SUCC(ret) && !in_nested_sql_ctrl()) { /*! * 如果已经开始了STMT, 说明在嵌套语句中, 此时不需要设置SAVEPOINT, @@ -690,6 +700,7 @@ int ObPLContext::init(ObSQLSessionInfo &session_info, OZ (session_info.store_top_query_string(cur_query_)); OZ (recursion_ctx_.init(session_info)); OX (session_info.set_pl_stack_ctx(this)); + OX (need_remove_top_stack = true); OX (session_info.set_pl_can_retry(true)); } else if (is_function_or_trigger && lib::is_mysql_mode()) { //mysql模式, 内层function或者trigger不需要创建隐式savepoint, 只需要重置ac @@ -723,6 +734,14 @@ int ObPLContext::init(ObSQLSessionInfo &session_info, if (OB_SUCC(ret) && is_function_or_trigger && lib::is_mysql_mode()) { last_insert_id_ = session_info.get_local_last_insert_id(); } +#ifdef ERRSIM + OX (ret = OBPLCONTEXT_INIT); +#endif // ERRSIM + OZ (ObPLContext::debug_start(&session_info)); + OX (need_debug_stop = true); +#ifdef ERRSIM + OX (ret = OBPLCONTEXT_INIT); +#endif // ERRSIM if (OB_SUCC(ret) && is_autonomous_) { has_inner_dml_write_ = session_info.has_exec_inner_dml(); session_info.set_has_exec_inner_dml(false); @@ -732,10 +751,52 @@ int ObPLContext::init(ObSQLSessionInfo &session_info, OZ (session_info.begin_autonomous_session(saved_session_)); OX (saved_has_implicit_savepoint_ = session_info.has_pl_implicit_savepoint()); OX (session_info.clear_pl_implicit_savepoint()); - OZ (ObSqlTransControl::explicit_start_trans(ctx, false)); - (void) register_after_begin_autonomous_session_for_deadlock_(session_info, last_trans_id); + + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(ObSqlTransControl::explicit_start_trans(ctx, false))) { + LOG_WARN("failed to ObSqlTransControl::explicit_start_trans", K(ret)); + int tmp_ret = session_info.end_autonomous_session(saved_session_); + if (OB_SUCCESS != tmp_ret) { + LOG_WARN("failed to end_autonomous_session after explicit_start_trans failed, will ignore this error", + K(tmp_ret), K(ret)); + } + } else { + (void) register_after_begin_autonomous_session_for_deadlock_(session_info, last_trans_id); + } } - OX (session_info_ = &session_info); + + // add any code may fail before this line. + // if it should fail, the side-effect must be cleared by its own, or after this line. + + if (OB_SUCC(ret)) { + // this marks init succeed + session_info_ = &session_info; + } else { // failed to init, do some clean up work + // restore autocommit + if (reset_autocommit_) { + session_info.set_autocommit(true); + } + + // restore timeout + if (old_worker_timeout_ts_ != 0) { + THIS_WORKER.set_timeout_ts(old_worker_timeout_ts_); + if (OB_NOT_NULL(ctx.get_physical_plan_ctx())) { + ctx.get_physical_plan_ctx()->set_timeout_timestamp(old_phy_plan_timeout_ts_); + } + } + + // remove top stack + if (need_remove_top_stack) { + session_info.set_pl_stack_ctx(nullptr); + } + + // stop debugger + if (need_debug_stop) { + IGNORE_RETURN ObPLContext::debug_stop(&session_info); + } + } + return ret; } From b776f6378ff4c6054440eab73dd1e69c7d2f9649 Mon Sep 17 00:00:00 2001 From: ZenoWang Date: Mon, 19 Aug 2024 14:44:57 +0000 Subject: [PATCH 113/249] Do sync freeze if ObDataCheckpoint::flush() is called by ALTER SYSTEM MINOR FREEZE --- src/storage/checkpoint/ob_data_checkpoint.cpp | 52 +++++++++---------- src/storage/ls/ob_freezer.cpp | 8 +-- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/src/storage/checkpoint/ob_data_checkpoint.cpp b/src/storage/checkpoint/ob_data_checkpoint.cpp index b5ae087e1..77410132b 100644 --- a/src/storage/checkpoint/ob_data_checkpoint.cpp +++ b/src/storage/checkpoint/ob_data_checkpoint.cpp @@ -259,7 +259,13 @@ SCN ObDataCheckpoint::get_active_rec_scn() int ObDataCheckpoint::flush(SCN recycle_scn, int64_t trace_id, bool need_freeze) { int ret = OB_SUCCESS; - if (need_freeze) { + if (is_tenant_freeze()) { + const bool is_sync = true; + const bool abs_timeout_ts = ObClockGenerator::getClock() + 10LL * 1000LL * 1000LL; // retry at most 10 seconds + if (OB_FAIL(ls_->logstream_freeze(trace_id, is_sync, abs_timeout_ts))) { + STORAGE_LOG(WARN, "minor freeze failed", K(ret), K(ls_->get_ls_id())); + } + } else if (need_freeze) { SCN active_rec_scn = get_active_rec_scn(); if (active_rec_scn > recycle_scn) { STORAGE_LOG(INFO, @@ -273,7 +279,6 @@ int ObDataCheckpoint::flush(SCN recycle_scn, int64_t trace_id, bool need_freeze) } else if (OB_FAIL(traversal_flush_())) { STORAGE_LOG(WARN, "traversal_flush failed", K(ret), K(ls_->get_ls_id())); } - return ret; } @@ -894,33 +899,28 @@ int ObDataCheckpoint::freeze_base_on_needs_(const int64_t trace_id, share::SCN recycle_scn) { int ret = OB_SUCCESS; - if (get_rec_scn() <= recycle_scn) { - if (is_tenant_freeze() || !is_flushing()) { - int64_t wait_flush_num = - new_create_list_.checkpoint_list_.get_size() - + active_list_.checkpoint_list_.get_size(); - bool logstream_freeze = true; - ObSArray need_flush_tablets; - if (wait_flush_num > MAX_FREEZE_CHECKPOINT_NUM) { - if (OB_FAIL(get_need_flush_tablets_(recycle_scn, need_flush_tablets))) { - // do nothing - } else { - int need_flush_num = need_flush_tablets.count(); - logstream_freeze = - need_flush_num * 100 / wait_flush_num > TABLET_FREEZE_PERCENT; - } + if (get_rec_scn() <= recycle_scn && !is_flushing()) { + int64_t wait_flush_num = new_create_list_.checkpoint_list_.get_size() + active_list_.checkpoint_list_.get_size(); + bool logstream_freeze = true; + ObSArray need_flush_tablets; + if (wait_flush_num > MAX_FREEZE_CHECKPOINT_NUM) { + if (OB_FAIL(get_need_flush_tablets_(recycle_scn, need_flush_tablets))) { + // do nothing + } else { + int need_flush_num = need_flush_tablets.count(); + logstream_freeze = need_flush_num * 100 / wait_flush_num > TABLET_FREEZE_PERCENT; } + } - const bool is_sync = false; - const bool abs_timeout_ts = 0; // async freeze do not need - if (OB_FAIL(ret)) { - } else if (logstream_freeze) { - if (OB_FAIL(ls_->logstream_freeze(trace_id, is_sync, abs_timeout_ts))) { - STORAGE_LOG(WARN, "minor freeze failed", K(ret), K(ls_->get_ls_id())); - } - } else if (OB_FAIL(ls_->tablet_freeze(trace_id, need_flush_tablets, is_sync))) { - STORAGE_LOG(WARN, "batch tablet freeze failed", K(ret), K(ls_->get_ls_id()), K(need_flush_tablets)); + const bool is_sync = false; + const bool abs_timeout_ts = 0; // async freeze do not need + if (OB_FAIL(ret)) { + } else if (logstream_freeze) { + if (OB_FAIL(ls_->logstream_freeze(trace_id, is_sync, abs_timeout_ts))) { + STORAGE_LOG(WARN, "minor freeze failed", K(ret), K(ls_->get_ls_id())); } + } else if (OB_FAIL(ls_->tablet_freeze(trace_id, need_flush_tablets, is_sync))) { + STORAGE_LOG(WARN, "batch tablet freeze failed", K(ret), K(ls_->get_ls_id()), K(need_flush_tablets)); } } return ret; diff --git a/src/storage/ls/ob_freezer.cpp b/src/storage/ls/ob_freezer.cpp index 77e77657b..cabdeca78 100644 --- a/src/storage/ls/ob_freezer.cpp +++ b/src/storage/ls/ob_freezer.cpp @@ -737,7 +737,7 @@ void ObFreezer::submit_an_async_freeze_task(const int64_t trace_id, const bool i } } - if (!submit_succ) { + if (!submit_succ && REACH_TIME_INTERVAL(1LL * 1000LL * 1000LL /* 1 second */)) { TRANS_LOG(INFO, "async freeze task already exists, skip submitting one task", K(ls_id), @@ -820,15 +820,15 @@ void ObFreezer::async_tablet_freeze_consumer(const int64_t trace_id) void ObFreezer::try_freeze_tx_data_() { int ret = OB_SUCCESS; - const int64_t MAX_RETRY_DURATION = 10LL * 1000LL * 1000LL; // 10 seconds + const int64_t MAX_RETRY_DURATION = 5LL * 1000LL * 1000LL; // 5 seconds int64_t retry_times = 0; int64_t start_freeze_ts = ObClockGenerator::getClock(); do { if (OB_FAIL(ls_->get_tx_table()->self_freeze_task())) { if (OB_EAGAIN == ret) { - // sleep and retry + // sleep 100ms and retry retry_times++; - usleep(100); + usleep(100LL * 1000LL); } else { STORAGE_LOG(WARN, "freeze tx data table failed", KR(ret), K(get_ls_id())); } From d2435523c0fd8992e356c3a8a38063e5962e84bb Mon Sep 17 00:00:00 2001 From: shadowao Date: Mon, 19 Aug 2024 15:02:58 +0000 Subject: [PATCH 114/249] opt lob_id alloc and add write length check in direct load --- src/storage/ddl/ob_direct_load_struct.cpp | 26 +++++++++--------- src/storage/ddl/ob_direct_load_struct.h | 2 +- src/storage/lob/ob_lob_manager.cpp | 17 +++--------- src/storage/lob/ob_lob_meta.cpp | 32 +++++++++++++++++++---- src/storage/lob/ob_lob_meta.h | 2 ++ src/storage/lob/ob_lob_util.cpp | 6 +++-- src/storage/lob/ob_lob_util.h | 13 ++++++--- 7 files changed, 58 insertions(+), 40 deletions(-) diff --git a/src/storage/ddl/ob_direct_load_struct.cpp b/src/storage/ddl/ob_direct_load_struct.cpp index b0ca08102..5dedd1e88 100644 --- a/src/storage/ddl/ob_direct_load_struct.cpp +++ b/src/storage/ddl/ob_direct_load_struct.cpp @@ -853,7 +853,6 @@ int ObDirectLoadSliceWriter::prepare_iters( const int64_t trans_version, const ObObjType &obj_type, const ObCollationType &cs_type, - const ObLobId &lob_id, const transaction::ObTransID trans_id, const int64_t seq_no, const int64_t timeout_ts, @@ -861,6 +860,7 @@ int ObDirectLoadSliceWriter::prepare_iters( const uint64_t src_tenant_id, const ObDirectLoadType direct_load_type, transaction::ObTxDesc* tx_desc, + share::ObTabletCacheInterval &pk_interval, ObLobMetaRowIterator *&row_iter) { int ret = OB_SUCCESS; @@ -896,11 +896,11 @@ int ObDirectLoadSliceWriter::prepare_iters( if (is_incremental_direct_load(direct_load_type) && OB_ISNULL(tx_desc)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("tx_desc should not be null if is incremental_direct_load", K(ret), K(direct_load_type), - K(ls_id), K(tablet_id), K(lob_id), K(trans_version), K(seq_no), K(obj_type), K(cs_type), K(trans_id)); + K(ls_id), K(tablet_id), K(trans_version), K(seq_no), K(obj_type), K(cs_type), K(trans_id)); } else if (OB_FAIL(ObInsertLobColumnHelper::insert_lob_column( - allocator, tx_desc, ls_id, tablet_id, lob_id, obj_type, cs_type, lob_storage_param, datum, - timeout_ts, true/*has_lob_header*/, src_tenant_id, *meta_write_iter_))) { - LOG_WARN("fail to insert_lob_col", K(ret), K(ls_id), K(tablet_id), K(lob_id), K(src_tenant_id)); + allocator, tx_desc, pk_interval, ls_id, tablet_id/* tablet_id of main table */, tablet_direct_load_mgr_->get_tablet_id()/*tablet id of lob meta table*/, + obj_type, cs_type, lob_storage_param, datum, timeout_ts, true/*has_lob_header*/, src_tenant_id, *meta_write_iter_))) { + LOG_WARN("fail to insert_lob_col", K(ret), K(ls_id), K(tablet_id), K(src_tenant_id)); } else if (OB_FAIL(row_iterator_->init(meta_write_iter_, trans_id, trans_version, seq_no, direct_load_type))) { LOG_WARN("fail to lob meta row iterator", K(ret), K(trans_id), K(trans_version), K(seq_no), K(direct_load_type)); @@ -985,17 +985,12 @@ int ObDirectLoadSliceWriter::fill_lob_into_macro_block( int64_t idx = lob_column_idxs.at(i); ObStorageDatum &datum = datum_row.storage_datums_[idx]; if (!datum.is_nop() && !datum.is_null()) { - uint64_t pk_seq = OB_INVALID_ID; - if (OB_FAIL(pk_interval.next_value(pk_seq))) { - LOG_WARN("fail to get next lob_id", K(ret), K(pk_seq)); - } else { - ObLobId lob_id; - lob_id.lob_id_ = pk_seq; - lob_id.tablet_id_ = tablet_direct_load_mgr_->get_tablet_id().id(); // lob meta tablet id. 与ObDirectLoadInsertTabletContext::tablet_id_in_lob_id_的取值保持一致 + { ObLobMetaRowIterator *row_iter = nullptr; if (OB_FAIL(prepare_iters(allocator, iter_allocator, datum, info.ls_id_, - info.data_tablet_id_, info.trans_version_, col_types.at(i).get_type(), col_types.at(i).get_collation_type(), lob_id, - info.trans_id_, info.seq_no_, timeout_ts, lob_inrow_threshold, info.src_tenant_id_, info.direct_load_type_, info.tx_desc_, row_iter))) { + info.data_tablet_id_, info.trans_version_, col_types.at(i).get_type(), col_types.at(i).get_collation_type(), + info.trans_id_, info.seq_no_, timeout_ts, lob_inrow_threshold, info.src_tenant_id_, info.direct_load_type_, + info.tx_desc_, pk_interval, row_iter))) { LOG_WARN("fail to prepare iters", K(ret), KP(row_iter), K(datum)); } else { while (OB_SUCC(ret)) { @@ -1028,6 +1023,9 @@ int ObDirectLoadSliceWriter::fill_lob_into_macro_block( LOG_DEBUG("sstable insert op append row", K(unused_affected_rows), KPC(cur_row)); } } + if (OB_SUCC(ret) && OB_NOT_NULL(meta_write_iter_) && OB_FAIL(meta_write_iter_->check_write_length())) { + LOG_WARN("check_write_length fail", K(ret), KPC(meta_write_iter_)); + } if (OB_SUCC(ret)) { if (OB_NOT_NULL(meta_write_iter_)) { meta_write_iter_->reuse(); diff --git a/src/storage/ddl/ob_direct_load_struct.h b/src/storage/ddl/ob_direct_load_struct.h index 212a923ad..85f80a0be 100644 --- a/src/storage/ddl/ob_direct_load_struct.h +++ b/src/storage/ddl/ob_direct_load_struct.h @@ -591,7 +591,6 @@ private: const int64_t trans_version, const ObObjType &obj_type, const ObCollationType &cs_type, - const ObLobId &lob_id, const transaction::ObTransID trans_id, const int64_t seq_no, const int64_t timeout_ts, @@ -599,6 +598,7 @@ private: const uint64_t src_tenant_id, const ObDirectLoadType direct_load_type, transaction::ObTxDesc* tx_desc, + share::ObTabletCacheInterval &pk_interval, ObLobMetaRowIterator *&row_iter); int mock_chunk_store(const int64_t row_cnt); private: diff --git a/src/storage/lob/ob_lob_manager.cpp b/src/storage/lob/ob_lob_manager.cpp index 811055426..9f4420482 100644 --- a/src/storage/lob/ob_lob_manager.cpp +++ b/src/storage/lob/ob_lob_manager.cpp @@ -1576,17 +1576,13 @@ int ObLobManager::check_need_out_row( } else { // init lob data and alloc lob id(when not init) ObLobData *new_lob_data = new(new_lob_common->buffer_)ObLobData(); - if (param.spec_lob_id_.is_valid()) { - new_lob_data->id_ = param.spec_lob_id_; - } else if (OB_FAIL(lob_ctx_.lob_meta_mngr_->fetch_lob_id(param, new_lob_data->id_.lob_id_))) { + if (OB_FAIL(lob_ctx_.lob_meta_mngr_->fetch_lob_id(param, new_lob_data->id_.lob_id_))) { LOG_WARN("get lob id failed.", K(ret), K(param)); } else if (! param.lob_meta_tablet_id_.is_valid()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("lob_meta_tablet_id is invalid", K(ret), K(param)); } else { new_lob_data->id_.tablet_id_ = param.lob_meta_tablet_id_.id(); - } - if (OB_SUCC(ret)) { transform_lob_id(new_lob_data->id_.lob_id_, new_lob_data->id_.lob_id_); new_lob_common->is_init_ = true; } @@ -2049,6 +2045,7 @@ int ObLobManager::prepare_lob_common(ObLobAccessParam& param, bool &alloc_inside alloc_inside = false; if (OB_ISNULL(param.lob_common_)) { // alloc new lob_data + // here just fake a lob locator void *tbuf = param.allocator_->alloc(LOB_OUTROW_FULL_SIZE); if (OB_ISNULL(tbuf)) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -2057,10 +2054,6 @@ int ObLobManager::prepare_lob_common(ObLobAccessParam& param, bool &alloc_inside // init full out row param.lob_common_ = new(tbuf)ObLobCommon(); param.lob_data_ = new(param.lob_common_->buffer_)ObLobData(); - param.lob_data_->id_.tablet_id_ = param.tablet_id_.id(); - if (param.spec_lob_id_.is_valid()) { - param.lob_data_->id_ = param.spec_lob_id_; - } ObLobDataOutRowCtx *outrow_ctx = new(param.lob_data_->buffer_)ObLobDataOutRowCtx(); outrow_ctx->chunk_size_ = param.get_schema_chunk_size() / ObLobDataOutRowCtx::OUTROW_LOB_CHUNK_SIZE_UNIT; // init char len @@ -2236,17 +2229,13 @@ int ObLobManager::prepare_for_write( } else { // init lob data and alloc lob id(when not init) ObLobData *new_lob_data = new(new_lob_common->buffer_)ObLobData(); - if (param.spec_lob_id_.is_valid()) { - new_lob_data->id_ = param.spec_lob_id_; - } else if (OB_FAIL(lob_ctx_.lob_meta_mngr_->fetch_lob_id(param, new_lob_data->id_.lob_id_))) { + if (OB_FAIL(lob_ctx_.lob_meta_mngr_->fetch_lob_id(param, new_lob_data->id_.lob_id_))) { LOG_WARN("get lob id failed.", K(ret), K(param)); } else if (! param.lob_meta_tablet_id_.is_valid()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("lob_meta_tablet_id is invalid", K(ret), K(param)); } else { new_lob_data->id_.tablet_id_ = param.lob_meta_tablet_id_.id(); - } - if (OB_SUCC(ret)) { transform_lob_id(new_lob_data->id_.lob_id_, new_lob_data->id_.lob_id_); new_lob_common->is_init_ = true; } diff --git a/src/storage/lob/ob_lob_meta.cpp b/src/storage/lob/ob_lob_meta.cpp index f0d35001b..93382f6ba 100644 --- a/src/storage/lob/ob_lob_meta.cpp +++ b/src/storage/lob/ob_lob_meta.cpp @@ -634,9 +634,9 @@ int ObLobMetaWriteIter::open(ObLobAccessParam ¶m, read_param_ = read_param; iter_ = iter; lob_common_ = param.lob_common_; - if (OB_ISNULL(iter) || OB_ISNULL(read_param)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("null query iter", K(ret)); + if (OB_ISNULL(iter) || OB_ISNULL(read_param) || OB_ISNULL(lob_common_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", K(ret), KP(iter), KP(read_param), KP(lob_common_)); } else if (OB_FAIL(open(param, iter, read_buf, 0/*padding_size*/, post_data, remain_buf, seq_id_st, seq_id_end, nullptr))) { LOG_WARN("open fail", K(ret), K(param), KP(read_param)); } @@ -942,6 +942,26 @@ int ObLobMetaWriteIter::update_disk_lob_locator(ObLobMetaWriteResult &result) return ret; } +int ObLobMetaWriteIter::check_write_length() +{ + int ret = OB_SUCCESS; + if (is_end_) { //means inrow , so skip + } else if (OB_ISNULL(lob_common_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("lob_common is null", K(ret)); + } else if (OB_ISNULL(read_param_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("read_param is null", K(ret)); + } else { + ObLobData *lob_data = reinterpret_cast(lob_common_->buffer_); + ObLobDataOutRowCtx *lob_outrow_ctx = reinterpret_cast(lob_data->buffer_); + if (read_param_->byte_size_ != lob_data->byte_size_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("write size not match", K(ret), "write_size", lob_data->byte_size_, "data_size", read_param_->byte_size_, KPC(read_param_)); + } + } + return ret; +} int ObLobMetaManager::write(ObLobAccessParam& param, ObLobMetaInfo& in_row) { @@ -1041,8 +1061,10 @@ int ObLobMetaManager::update(ObLobAccessParam& param, ObLobMetaInfo& old_row, Ob int ObLobMetaManager::fetch_lob_id(ObLobAccessParam& param, uint64_t &lob_id) { int ret = OB_SUCCESS; - if (param.spec_lob_id_.is_valid()) { - lob_id = param.spec_lob_id_.lob_id_; + if (nullptr != param.lob_id_geneator_) { + if (OB_FAIL(param.lob_id_geneator_->next_value(lob_id))) { + LOG_WARN("fail to get next lob_id", K(ret), KPC(param.lob_id_geneator_)); + } } else if (OB_FAIL(persistent_lob_adapter_.fetch_lob_id(param, lob_id))) { LOG_WARN("fetch lob id failed.", K(ret), K(param)); } diff --git a/src/storage/lob/ob_lob_meta.h b/src/storage/lob/ob_lob_meta.h index 1dff7a1fc..3d840d07d 100644 --- a/src/storage/lob/ob_lob_meta.h +++ b/src/storage/lob/ob_lob_meta.h @@ -179,6 +179,8 @@ public: int close(); void set_end() { is_end_ = true; } void reuse(); + // only used by ddl when write done + int check_write_length(); TO_STRING_KV(K_(seq_id), K_(offset), K_(lob_id), K_(piece_id), K_(coll_type), K_(piece_block_size), K_(scan_iter), K_(padding_size), K_(seq_id_end), K_(last_info), K_(is_store_char_len)); diff --git a/src/storage/lob/ob_lob_util.cpp b/src/storage/lob/ob_lob_util.cpp index 005a72b79..f1c6769df 100644 --- a/src/storage/lob/ob_lob_util.cpp +++ b/src/storage/lob/ob_lob_util.cpp @@ -294,9 +294,10 @@ int ObInsertLobColumnHelper::insert_lob_column(ObIAllocator &allocator, int ObInsertLobColumnHelper::insert_lob_column(ObIAllocator &allocator, transaction::ObTxDesc *tx_desc, + share::ObTabletCacheInterval &lob_id_geneator, const share::ObLSID ls_id, const common::ObTabletID tablet_id, - const ObLobId &lob_id, + const common::ObTabletID lob_meta_tablet_id, const ObObjType &obj_type, const ObCollationType collation_type, const ObLobStorageParam &lob_storage_param, @@ -351,7 +352,8 @@ int ObInsertLobColumnHelper::insert_lob_column(ObIAllocator &allocator, lob_param.timeout_ = timeout_ts; lob_param.scan_backward_ = false; lob_param.offset_ = 0; - lob_param.spec_lob_id_ = lob_id; + lob_param.lob_meta_tablet_id_ = lob_meta_tablet_id; + lob_param.lob_id_geneator_ = &lob_id_geneator; lob_param.inrow_threshold_ = lob_storage_param.inrow_threshold_; lob_param.src_tenant_id_ = src_tenant_id; if (!src.is_valid()) { diff --git a/src/storage/lob/ob_lob_util.h b/src/storage/lob/ob_lob_util.h index cebe53864..d5cbf090f 100644 --- a/src/storage/lob/ob_lob_util.h +++ b/src/storage/lob/ob_lob_util.h @@ -29,6 +29,10 @@ namespace oceanbase { +namespace share { +class ObTabletCacheInterval; +} + namespace storage { @@ -72,7 +76,7 @@ public: parent_seq_no_(), seq_no_st_(), used_seq_cnt_(0), total_seq_cnt_(0), checksum_(0), update_len_(0), op_type_(ObLobDataOutRowCtx::OpType::SQL), is_fill_zero_(false), from_rpc_(false), inrow_read_nocopy_(false), inrow_threshold_(OB_DEFAULT_LOB_INROW_THRESHOLD), schema_chunk_size_(OB_DEFAULT_LOB_CHUNK_SIZE), - access_ctx_(nullptr), is_store_char_len_(true), spec_lob_id_(), remote_query_ctx_(nullptr) + access_ctx_(nullptr), is_store_char_len_(true), lob_id_geneator_(nullptr), remote_query_ctx_(nullptr) {} ~ObLobAccessParam(); @@ -110,7 +114,7 @@ public: K_(coll_type), K_(scan_backward), K_(offset), K_(len), K_(parent_seq_no), K_(seq_no_st), K_(used_seq_cnt), K_(total_seq_cnt), K_(checksum), K_(update_len), K_(op_type), K_(is_fill_zero), K_(from_rpc), K_(snapshot), K_(tx_id), K_(read_latest), K_(inrow_read_nocopy), K_(schema_chunk_size), K_(inrow_threshold), K_(is_store_char_len), KP_(remote_query_ctx), - KP_(access_ctx), K_(spec_lob_id)); + KP_(access_ctx), KPC_(lob_id_geneator)); private: ObIAllocator *tmp_allocator_; @@ -167,7 +171,7 @@ public: ObObj ext_info_log_; ObLobAccessCtx *access_ctx_; bool is_store_char_len_; - ObLobId spec_lob_id_; + share::ObTabletCacheInterval *lob_id_geneator_; // remote query ctx void *remote_query_ctx_; }; @@ -297,9 +301,10 @@ public: // should call iter.close outter static int insert_lob_column(ObIAllocator &allocator, transaction::ObTxDesc *tx_desc, + share::ObTabletCacheInterval &lob_id_geneator, const share::ObLSID ls_id, const common::ObTabletID tablet_id, - const ObLobId &lob_id, + const common::ObTabletID lob_meta_tablet_id, const ObObjType &obj_type, const ObCollationType collation_type, const ObLobStorageParam &lob_storage_param, From 00933cc4f44a575e6eac13e8a242e2b74e3c1ad0 Mon Sep 17 00:00:00 2001 From: yishenglanlingzui <395329313@qq.com> Date: Mon, 19 Aug 2024 15:09:31 +0000 Subject: [PATCH 115/249] fix bug update shadow_pk is use minimal_writing --- src/sql/code_generator/ob_dml_cg_service.cpp | 61 +++++++------------- src/sql/code_generator/ob_dml_cg_service.h | 8 +-- 2 files changed, 26 insertions(+), 43 deletions(-) diff --git a/src/sql/code_generator/ob_dml_cg_service.cpp b/src/sql/code_generator/ob_dml_cg_service.cpp index 4f4504284..0c0a05b17 100644 --- a/src/sql/code_generator/ob_dml_cg_service.cpp +++ b/src/sql/code_generator/ob_dml_cg_service.cpp @@ -1265,25 +1265,16 @@ int ObDmlCgService::append_heap_table_part_key_dependcy_column(const ObTableSche int ObDmlCgService::check_unique_key_is_updated(ObSchemaGetterGuard *schema_guard, const ObTableSchema *table_schema, - const IndexDMLInfo &index_dml_info, + const ObIArray &upd_cids, bool &is_updated) { int ret = OB_SUCCESS; is_updated = false; - const ObAssignments &assignments = index_dml_info.assignments_; - for (int64_t i = 0; OB_SUCC(ret) && !is_updated && i < assignments.count(); i++) { - // We cannot use col_expr->is_unique_key_column_ here, which may be false for a unique index column. - ObColumnRefRawExpr *col_expr = assignments.at(i).column_expr_; - bool is_unique_col = false; - if (OB_ISNULL(col_expr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret)); - } else if (OB_FAIL(table_schema->is_real_unique_index_column(*schema_guard, - col_expr->get_column_id(), - is_unique_col))) { + for (int64_t i = 0; OB_SUCC(ret) && !is_updated && i < upd_cids.count(); i++) { + if (OB_FAIL(table_schema->is_real_unique_index_column(*schema_guard, + upd_cids.at(i), + is_updated))) { LOG_WARN("is_unique_key_column failed", K(ret)); - } else if (is_unique_col) { - is_updated = true; } } return ret; @@ -1381,31 +1372,21 @@ int ObDmlCgService::append_udt_hidden_column_id(const ObTableSchema *table_schem int ObDmlCgService::check_has_upd_rowkey(ObLogicalOperator &op, const ObTableSchema *table_schema, - const IndexDMLInfo &index_dml_info, + const ObIArray &upd_cids, bool &upd_rowkey) { int ret = OB_SUCCESS; upd_rowkey = false; ObSEArray pk_ids; - const ObDMLStmt *stmt = op.get_stmt(); - const ObAssignments &assignment = index_dml_info.assignments_; - if (OB_ISNULL(stmt)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected nullptr", K(ret), K(op)); - } else if (OB_FAIL(table_schema->get_rowkey_info().get_column_ids(pk_ids))) { + if (OB_FAIL(table_schema->get_rowkey_info().get_column_ids(pk_ids))) { LOG_WARN("failed to get rowkey column ids", K(ret)); } - for (int64_t i = 0; !upd_rowkey && OB_SUCC(ret) && i < assignment.count(); ++i) { - ColumnItem *column_item = nullptr; - const ObColumnRefRawExpr *column_expr = assignment.at(i).column_expr_; - if (OB_ISNULL(column_expr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get null column expr", K(ret)); - } else if (OB_ISNULL(column_item = stmt->get_column_item_by_id(column_expr->get_table_id(), - column_expr->get_column_id()))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get null column item", K(ret), KPC(column_expr)); - } else if (has_exist_in_array(pk_ids, column_item->base_cid_)) { + for (int64_t i = 0; !upd_rowkey && OB_SUCC(ret) && i < pk_ids.count(); ++i) { + uint64_t real_column_id = pk_ids.at(i); + if (is_shadow_column(pk_ids.at(i))) { + real_column_id = pk_ids.at(i) - OB_MIN_SHADOW_COLUMN_ID; + } + if (has_exist_in_array(upd_cids, real_column_id)) { upd_rowkey = true; } } @@ -1497,7 +1478,7 @@ int ObDmlCgService::is_table_has_unique_key(ObSchemaGetterGuard *schema_guard, int ObDmlCgService::check_upd_need_all_columns(ObLogDelUpd &op, ObSchemaGetterGuard *schema_guard, const ObTableSchema *table_schema, - const IndexDMLInfo &index_dml_info, + const ObIArray &upd_cids, bool is_primary_index, bool &need_all_columns) { @@ -1524,7 +1505,7 @@ int ObDmlCgService::check_upd_need_all_columns(ObLogDelUpd &op, LOG_TRACE("update materialized view log, need all columns", K(table_schema->is_mlog_table())); } else if (!is_primary_index) { // index_table if update PK, also need record all_columns - if (OB_FAIL(check_has_upd_rowkey(op, table_schema, index_dml_info, is_update_pk))) { + if (OB_FAIL(check_has_upd_rowkey(op, table_schema, upd_cids, is_update_pk))) { LOG_WARN("fail to check has update UK", K(ret)); } else if (is_update_pk) { need_all_columns = true; @@ -1533,8 +1514,8 @@ int ObDmlCgService::check_upd_need_all_columns(ObLogDelUpd &op, } else if (OB_FAIL(is_table_has_unique_key(schema_guard, table_schema, has_uk))) { LOG_WARN("fail to check table has UK", K(ret)); } else if (has_uk && - OB_FAIL(check_unique_key_is_updated(schema_guard, table_schema, index_dml_info, is_uk_updated))) { - LOG_WARN("fail to check unique key is updated", K(ret), K(index_dml_info)); + OB_FAIL(check_unique_key_is_updated(schema_guard, table_schema, upd_cids, is_uk_updated))) { + LOG_WARN("fail to check unique key is updated", K(ret), K(upd_cids)); } else if (is_uk_updated) { // need all columns need_all_columns = true; @@ -1544,7 +1525,7 @@ int ObDmlCgService::check_upd_need_all_columns(ObLogDelUpd &op, } else if (need_all_columns) { // need all columns LOG_TRACE("is heap table and don't has not_null UK, need all columns", K(need_all_columns)); - } else if (OB_FAIL(check_has_upd_rowkey(op, table_schema, index_dml_info, is_update_pk))) { + } else if (OB_FAIL(check_has_upd_rowkey(op, table_schema, upd_cids, is_update_pk))) { LOG_TRACE("update primary_table primary key, need all columns", K(need_all_columns)); } else if (is_update_pk) { // rowkey is changed, need all columns @@ -1610,6 +1591,7 @@ int ObDmlCgService::generate_minimal_upd_old_row_cid(ObLogDelUpd &op, ObTableID index_tid, ObDASUpdCtDef &das_upd_ctdef, const IndexDMLInfo &index_dml_info, + const ObIArray &upd_cids, bool is_primary_index, bool &need_all_columns, ObIArray &minimal_column_ids) @@ -1633,7 +1615,7 @@ int ObDmlCgService::generate_minimal_upd_old_row_cid(ObLogDelUpd &op, } else if (OB_FAIL(check_upd_need_all_columns(op, schema_guard, table_schema, - index_dml_info, + upd_cids, is_primary_index, need_all_columns))) { LOG_WARN("fail to check whether update need all columns", K(ret), K(index_tid)); @@ -1650,7 +1632,7 @@ int ObDmlCgService::generate_minimal_upd_old_row_cid(ObLogDelUpd &op, minimal_column_ids))) { LOG_WARN("fail to append update old_row column_id", K(ret), K(index_tid)); } - + LOG_TRACE("print need_all_columns", K(ret), K(need_all_columns), K(minimal_column_ids)); return ret; } @@ -2160,6 +2142,7 @@ int ObDmlCgService::generate_das_upd_ctdef(ObLogDelUpd &op, index_tid, das_upd_ctdef, index_dml_info, + das_upd_ctdef.updated_column_ids_, is_primary_table, need_all_columns, minimal_column_ids))) { diff --git a/src/sql/code_generator/ob_dml_cg_service.h b/src/sql/code_generator/ob_dml_cg_service.h index b3c7bfe66..151dd3e4d 100644 --- a/src/sql/code_generator/ob_dml_cg_service.h +++ b/src/sql/code_generator/ob_dml_cg_service.h @@ -125,11 +125,11 @@ private: ObTableID index_tid, ObDASUpdCtDef &das_upd_ctdef, const IndexDMLInfo &index_dml_info, + const ObIArray &upd_cids, bool is_primary_index, bool &need_all_columns, ObIArray &minimal_column_ids); - int append_upd_old_row_cid(ObLogicalOperator &op, ObSchemaGetterGuard *schema_guard, const ObTableSchema *table_schema, @@ -141,7 +141,7 @@ private: int check_upd_need_all_columns(ObLogDelUpd &op, ObSchemaGetterGuard *schema_guard, const ObTableSchema *table_schema, - const IndexDMLInfo &index_dml_info, + const ObIArray &upd_cids, bool is_primary_index, bool &need_all_columns); @@ -160,7 +160,7 @@ private: int check_has_upd_rowkey(ObLogicalOperator &op, const ObTableSchema *table_schema, - const IndexDMLInfo &index_dml_info, + const ObIArray &upd_cids, bool &upd_rowkey); int append_udt_hidden_column_id(const ObTableSchema *table_schema, @@ -170,7 +170,7 @@ private: int check_unique_key_is_updated(ObSchemaGetterGuard *schema_guard, const ObTableSchema *table_schema, - const IndexDMLInfo &index_dml_info, + const ObIArray &upd_cids, bool &is_updated); int append_time_type_column_id(const ObTableSchema *table_schema, From 863a8a27ea9f9e5fd7a1ad18e2470e43d2e98737 Mon Sep 17 00:00:00 2001 From: yishenglanlingzui <395329313@qq.com> Date: Mon, 19 Aug 2024 15:15:31 +0000 Subject: [PATCH 116/249] fix bug arraybinding with number type, can't support batc optimization --- src/sql/ob_sql.h | 6 ++++-- src/sql/ob_sql_utils.cpp | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/sql/ob_sql.h b/src/sql/ob_sql.h index 53d561d43..487d5652f 100644 --- a/src/sql/ob_sql.h +++ b/src/sql/ob_sql.h @@ -494,12 +494,14 @@ private: ObSchemaGetterGuard &schema_guard, ObSqlTraits &sql_traits); int get_reconstructed_batch_stmt(ObPlanCacheCtx &pc_ctx, ObString& stmt_sql); - static int add_param_to_param_store(const ObObjParam ¶m, - ParamStore ¶m_store); int check_need_switch_thread(ObSqlCtx &ctx, const ObStmt *stmt, bool &need_switch); void rollback_implicit_trans_when_fail(ObResultSet &result, int &ret); typedef hash::ObHashMap PlanCacheMap; friend class ::test::TestOptimizerUtils; + +public: + static int add_param_to_param_store(const ObObjParam ¶m, + ParamStore ¶m_store); private: bool inited_; // BEGIN 全局单例依赖接口 diff --git a/src/sql/ob_sql_utils.cpp b/src/sql/ob_sql_utils.cpp index bb2bf462d..0066f3cd7 100644 --- a/src/sql/ob_sql_utils.cpp +++ b/src/sql/ob_sql_utils.cpp @@ -5265,6 +5265,7 @@ int ObSQLUtils::get_one_group_params(int64_t &actual_pos, ParamStore &src, Param ObObj *data = NULL; if (OB_UNLIKELY(!obj.is_ext())) { OZ (obj_params.push_back(obj)); + OZ (ObSql::add_param_to_param_store(obj, obj_params)); } else { CK (OB_NOT_NULL(coll = reinterpret_cast(obj.get_ext()))); CK (coll->get_count() > actual_pos); @@ -5278,7 +5279,7 @@ int ObSQLUtils::get_one_group_params(int64_t &actual_pos, ParamStore &src, Param ++actual_pos; } } - OX (obj_params.push_back(*(data + actual_pos))); + OZ (ObSql::add_param_to_param_store(*(data + actual_pos), obj_params)); } } } @@ -5898,4 +5899,4 @@ bool ObSQLUtils::is_data_version_ge_423_or_431(uint64_t data_version) bool ObSQLUtils::is_data_version_ge_423_or_432(uint64_t data_version) { return ((MOCK_DATA_VERSION_4_2_3_0 <= data_version && data_version < DATA_VERSION_4_3_0_0) || data_version >= DATA_VERSION_4_3_2_0); -} \ No newline at end of file +} From 4691b8173725c1e7b324707c9171daf19c99009a Mon Sep 17 00:00:00 2001 From: cqliang1995 Date: Mon, 19 Aug 2024 15:21:26 +0000 Subject: [PATCH 117/249] [CP] fix dblink sequece error in transaction --- src/sql/ob_sql_trans_control.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/sql/ob_sql_trans_control.cpp b/src/sql/ob_sql_trans_control.cpp index 3cbe4e06d..b4e9701eb 100644 --- a/src/sql/ob_sql_trans_control.cpp +++ b/src/sql/ob_sql_trans_control.cpp @@ -684,6 +684,14 @@ int ObSqlTransControl::dblink_xa_prepare(ObExecContext &exec_ctx) } else if (OB_FAIL(session->get_dblink_context().get_dblink_conn(dblink_id, dblink_conn))) { LOG_WARN("failed to get dblink connection from session", K(dblink_id), K(sessid), K(ret)); } else if (OB_NOT_NULL(dblink_conn)) { + if (OB_FAIL(ObTMService::tm_rm_start(exec_ctx, // some conn get from session may not in xa trasaction, use tm_rm_start make sure it in xa + static_cast(dblink_schema->get_driver_proto()), + dblink_conn, + session->get_dblink_context().get_tx_id()))) { + LOG_WARN("failed to tm_rm_start", K(ret), K(dblink_id), K(dblink_conn), K(sessid)); + } else { + LOG_TRACE("link succ to prepare xa connection", KP(plan_ctx), K(ret), K(dblink_id)); + } ObDblinkCtxInSession::revert_dblink_conn(dblink_conn); // release rlock locked by get_dblink_conn } else { common::sqlclient::dblink_param_ctx dblink_param_ctx; From 90540377f3f9721478da45dcdda3b35c6665158e Mon Sep 17 00:00:00 2001 From: zhjc1124 Date: Mon, 19 Aug 2024 15:27:23 +0000 Subject: [PATCH 118/249] check if req is null before free --- src/share/io/ob_io_manager.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/share/io/ob_io_manager.cpp b/src/share/io/ob_io_manager.cpp index a181a67c7..6b4c4bfa5 100644 --- a/src/share/io/ob_io_manager.cpp +++ b/src/share/io/ob_io_manager.cpp @@ -952,7 +952,9 @@ int ObTenantIOManager::detect_aio(const ObIOInfo &info, ObIOHandle &handle) LOG_INFO("submit_detect_request cost too much time", K(ret), K(time_guard), K(req)); } if (OB_FAIL(ret)) { - req->free(); + if (OB_NOT_NULL(req)) { + req->free(); + } handle.reset(); } return ret; From 0914824b4036228d7619321602afe3cf72105c0a Mon Sep 17 00:00:00 2001 From: YangEfei Date: Mon, 19 Aug 2024 18:47:36 +0000 Subject: [PATCH 119/249] [TABLELOCK] change system_variable on session when execute inner_sql to avoid defense errors --- .../tablelock/ob_lock_func_executor.cpp | 17 ++- .../ob_lock_inner_connection_util.cpp | 100 ++++++++++++++++-- .../tablelock/ob_lock_inner_connection_util.h | 6 ++ 3 files changed, 112 insertions(+), 11 deletions(-) diff --git a/src/storage/tablelock/ob_lock_func_executor.cpp b/src/storage/tablelock/ob_lock_func_executor.cpp index 490e81466..95822a95f 100644 --- a/src/storage/tablelock/ob_lock_func_executor.cpp +++ b/src/storage/tablelock/ob_lock_func_executor.cpp @@ -217,6 +217,8 @@ int ObLockFuncContext::open_inner_conn_() } else if (OB_NOT_NULL(inner_conn_) || OB_NOT_NULL(store_inner_conn_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("inner_conn_ or store_inner_conn_ should be null", K(ret), KP(inner_conn_), KP(store_inner_conn_)); + } else if (FALSE_IT(store_inner_conn_ = static_cast(session->get_inner_conn()))) { + } else if (FALSE_IT(session->set_inner_conn(nullptr))) { } else if (OB_FAIL(ObInnerConnectionLockUtil::create_inner_conn(session, sql_proxy_, inner_conn))) { LOG_WARN("create inner connection failed", K(ret), KPC(session)); } else if (OB_ISNULL(inner_conn)) { @@ -228,7 +230,6 @@ int ObLockFuncContext::open_inner_conn_() * so we put inner conn in session to share it within multi layer nested sql. */ inner_conn_ = inner_conn; - store_inner_conn_ = static_cast(session->get_inner_conn()); session->set_inner_conn(inner_conn); LOG_DEBUG("ObLockFuncContext::open_inner_conn_ successfully", KP(inner_conn_), @@ -243,13 +244,23 @@ int ObLockFuncContext::close_inner_conn_() int ret = OB_SUCCESS; ObSQLSessionInfo *session = my_exec_ctx_->get_my_session(); - if (OB_ISNULL(sql_proxy_) || OB_ISNULL(session) || OB_ISNULL(inner_conn_)) { + if (OB_ISNULL(sql_proxy_) || OB_ISNULL(inner_conn_)) { ret = OB_NOT_INIT; LOG_WARN("sql_proxy or inner_conn of session is NULL", K(ret), KP(sql_proxy_), KP(session), KP(inner_conn_)); } else { - session->set_inner_conn(store_inner_conn_); // restore inner_conn to session before close the tmp inner_conn OZ (sql_proxy_->close(inner_conn_, true)); } + if (OB_ISNULL(session)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("session is NULL", K(ret), KP(session)); + } else if (OB_NOT_NULL(inner_conn_) || OB_NOT_NULL(store_inner_conn_)) { + // 1. if inner_conn_ is not null, means that we have created inner_conn successfully before, so we must have already + // set store_inner_conn_ successfully, just restore it to the session. + // 2. if inner_conn_ is null, it's uncertain whether store_inner_conn_ has been set before. If store_inner_conn_ + // is not null, it must have been set. Otherwise, the inner_conn on the session may be null, or it may have existed + // with an error code before store_inner_conn_ being set. At this case, we do not set inner_conn on the session. + session->set_inner_conn(store_inner_conn_); + } sql_proxy_ = nullptr; inner_conn_ = nullptr; store_inner_conn_ = nullptr; diff --git a/src/storage/tablelock/ob_lock_inner_connection_util.cpp b/src/storage/tablelock/ob_lock_inner_connection_util.cpp index 0601699bc..c38f1460d 100644 --- a/src/storage/tablelock/ob_lock_inner_connection_util.cpp +++ b/src/storage/tablelock/ob_lock_inner_connection_util.cpp @@ -462,26 +462,51 @@ int ObInnerConnectionLockUtil::create_inner_conn(sql::ObSQLSessionInfo *session_ observer::ObInnerSQLConnection *&inner_conn) { int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + ObObj mysql_mode; + ObObj current_mode; + mysql_mode.set_int(0); + current_mode.set_int(-1); + observer::ObInnerSQLConnectionPool *pool = nullptr; common::sqlclient::ObISQLConnection *conn = nullptr; + lib::CompatModeGuard guard(lib::Worker::CompatMode::MYSQL); if (OB_ISNULL(session_info) || OB_ISNULL(sql_proxy)) { ret = OB_NOT_INIT; LOG_WARN("session or sql_proxy is NULL", KP(session_info), KP(sql_proxy)); } else if (OB_ISNULL(pool = static_cast(sql_proxy->get_pool()))) { ret = OB_NOT_INIT; LOG_WARN("connection pool is NULL", K(ret)); + } else if (OB_FAIL(session_info->get_sys_variable(share::SYS_VAR_OB_COMPATIBILITY_MODE, current_mode))) { + LOG_WARN("can not get the compat_mode", KPC(session_info)); + } else if (current_mode != mysql_mode + && OB_FAIL(session_info->update_sys_variable(share::SYS_VAR_OB_COMPATIBILITY_MODE, mysql_mode))) { + LOG_WARN("update session_info to msyql_mode failed", KR(ret), KPC(session_info)); } else if (common::sqlclient::INNER_POOL != pool->get_type()) { LOG_WARN("connection pool type is not inner", K(ret), K(pool->get_type())); // NOTICE: although we can set is_oracle_mode here, internally it will prioritize referencing // the system variables on the session, so this parameter actually has no effect } else if (OB_FAIL(pool->acquire(session_info, conn, false /* is_oracle_mode */))) { LOG_WARN("acquire connection from inner sql connection pool failed", KR(ret), KPC(session_info)); + } else if (OB_ISNULL(conn)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("acquire new connection but it's null", KR(ret), KPC(session_info)); } else { inner_conn = static_cast(conn); - // the inner_connection must be mysql_mode, so that we can write inner_table - inner_conn->set_mysql_compat_mode(); + // we must use mysql_mode connection to write inner_table + if (OB_UNLIKELY(inner_conn->is_oracle_compat_mode())) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("create an oracle mode inner connection", K(ret)); + } } + + if (current_mode != mysql_mode && current_mode.get_int() != -1 && OB_NOT_NULL(session_info) + && OB_TMP_FAIL(session_info->update_sys_variable(share::SYS_VAR_OB_COMPATIBILITY_MODE, current_mode))) { + ret = OB_SUCCESS == ret ? tmp_ret : ret; + LOG_WARN("failed to update sys variable for compatibility mode", K(current_mode), KPC(session_info)); + } + return ret; } @@ -490,6 +515,9 @@ int ObInnerConnectionLockUtil::execute_write_sql(observer::ObInnerSQLConnection int64_t &affected_rows) { int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + bool need_reset_sess_mode = false; + bool need_reset_conn_mode = false; // NOTICE: This will be overwritten by the oracle_made_ field on connection, // but for safety reason, we still set up a compat_guard here. @@ -498,10 +526,17 @@ int ObInnerConnectionLockUtil::execute_write_sql(observer::ObInnerSQLConnection if (OB_ISNULL(conn)) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("inner_conn is nullptr", K(ret), K(sql)); - } else if (OB_FAIL(conn->execute_write(MTL_ID(), sql.ptr(), affected_rows))) { - LOG_WARN("execute write sql failed", K(ret), K(sql)); + } else { + if (OB_FAIL(set_to_mysql_compat_mode_(conn, need_reset_sess_mode, need_reset_conn_mode))) { + LOG_WARN("set to mysql compat_mode failed", K(ret), K(sql)); + } else if (OB_FAIL(conn->execute_write(MTL_ID(), sql.ptr(), affected_rows))) { + LOG_WARN("execute write sql failed", K(ret), K(sql)); + } + if (OB_TMP_FAIL(reset_compat_mode_(conn, need_reset_sess_mode, need_reset_conn_mode))) { + LOG_WARN("reset compat_mode failed", K(ret), K(tmp_ret), K(sql)); + ret = COVER_SUCC(tmp_ret); + } } - LOG_DEBUG("ObInnerConnectionLockUtil::execute_write_sql", K(ret), KP(conn), K(conn->is_oracle_compat_mode())); return ret; } @@ -510,6 +545,9 @@ int ObInnerConnectionLockUtil::execute_read_sql(observer::ObInnerSQLConnection * ObISQLClient::ReadResult &res) { int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + bool need_reset_sess_mode = false; + bool need_reset_conn_mode = false; // NOTICE: This will be overwritten by the oracle_made_ field on connection, // but for safety reason, we still set up a compat_guard here. @@ -518,10 +556,17 @@ int ObInnerConnectionLockUtil::execute_read_sql(observer::ObInnerSQLConnection * if (OB_ISNULL(conn)) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("inner_conn is nullptr", K(ret), K(sql)); - } else if (OB_FAIL(conn->execute_read(MTL_ID(), sql.ptr(), res))) { - LOG_WARN("execute read sql failed", K(ret), K(sql)); + } else { + if (OB_FAIL(set_to_mysql_compat_mode_(conn, need_reset_sess_mode, need_reset_conn_mode))) { + LOG_WARN("set to mysql compat_mode failed", K(ret), K(sql)); + } else if (OB_FAIL(conn->execute_read(MTL_ID(), sql.ptr(), res))) { + LOG_WARN("execute read sql failed", K(ret), K(sql)); + } + if (OB_TMP_FAIL(reset_compat_mode_(conn, need_reset_sess_mode, need_reset_conn_mode))) { + LOG_WARN("reset compat_mode failed", K(ret), K(tmp_ret), K(sql)); + ret = COVER_SUCC(tmp_ret); + } } - LOG_DEBUG("ObInnerConnectionLockUtil::execute_read_sql", K(ret), KP(conn), K(conn->is_oracle_compat_mode())); return ret; } @@ -919,6 +964,45 @@ int ObInnerConnectionLockUtil::get_org_cluster_id_(ObSQLSessionInfo *session, in } return ret; } + +int ObInnerConnectionLockUtil::set_to_mysql_compat_mode_(observer::ObInnerSQLConnection *conn, bool &need_reset_sess_mode, bool &need_reset_conn_mode) +{ + int ret = OB_SUCCESS; + ObObj current_mode; + ObObj mysql_mode; + current_mode.set_int(-1); + mysql_mode.set_int(0); + need_reset_sess_mode = false; + need_reset_conn_mode = false; + + if (OB_FAIL(conn->get_session().get_sys_variable(share::SYS_VAR_OB_COMPATIBILITY_MODE, current_mode))) { + LOG_WARN("can not get the compat_mode", ); + } else if (FALSE_IT(need_reset_sess_mode = (current_mode != mysql_mode))) { + } else if (need_reset_sess_mode + && OB_FAIL(conn->get_session().update_sys_variable(share::SYS_VAR_OB_COMPATIBILITY_MODE, mysql_mode))) { + LOG_WARN("update compat_mode to mysql_mode failed"); + } else if (conn->is_oracle_compat_mode()) { + need_reset_conn_mode = true; + conn->set_mysql_compat_mode(); + } + return ret; +} + +int ObInnerConnectionLockUtil::reset_compat_mode_(observer::ObInnerSQLConnection *conn, const bool need_reset_sess_mode, const bool need_reset_conn_mode) +{ + int ret = OB_SUCCESS; + ObObj oracle_mode; + oracle_mode.set_int(1); + + if (need_reset_sess_mode + && OB_FAIL(conn->get_session().update_sys_variable(share::SYS_VAR_OB_COMPATIBILITY_MODE, oracle_mode))) { + LOG_WARN("failed to update sys variable for compatibility mode", K(ret)); + } + if (need_reset_conn_mode) { + conn->set_oracle_compat_mode(); + } + return ret; +} #undef REQUEST_LOCK_4_1 } // tablelock } // transaction diff --git a/src/storage/tablelock/ob_lock_inner_connection_util.h b/src/storage/tablelock/ob_lock_inner_connection_util.h index 9671626e2..6e7a09bd0 100644 --- a/src/storage/tablelock/ob_lock_inner_connection_util.h +++ b/src/storage/tablelock/ob_lock_inner_connection_util.h @@ -176,6 +176,12 @@ private: const obrpc::ObInnerSQLTransmitArg::InnerSQLOperationType operation_type, observer::ObInnerSQLConnection *conn); static int get_org_cluster_id_(sql::ObSQLSessionInfo *session, int64_t &org_cluster_id); + static int set_to_mysql_compat_mode_(observer::ObInnerSQLConnection *conn, + bool &need_reset_sess_mode, + bool &need_reset_conn_mode); + static int reset_compat_mode_(observer::ObInnerSQLConnection *conn, + const bool need_reset_sess_mode, + const bool need_reset_conn_mode); }; } // tablelock From ef59375e8afbc60cc653871d6683c735d975b42b Mon Sep 17 00:00:00 2001 From: obdev Date: Mon, 19 Aug 2024 18:53:31 +0000 Subject: [PATCH 120/249] BUGFIX: fix tablelock compat problem --- src/storage/tablelock/ob_table_lock_common.cpp | 2 +- src/storage/tablelock/ob_table_lock_common.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/storage/tablelock/ob_table_lock_common.cpp b/src/storage/tablelock/ob_table_lock_common.cpp index f5bdd891c..a3f529cf3 100644 --- a/src/storage/tablelock/ob_table_lock_common.cpp +++ b/src/storage/tablelock/ob_table_lock_common.cpp @@ -228,7 +228,7 @@ int ObTableLockOwnerID::deserialize(const char* buf, const int64_t data_len, int { int ret = OB_SUCCESS; const int64_t origin_pos = pos; - int16_t magic_num = 0; + int64_t magic_num = 0; if (OB_ISNULL(buf) || OB_UNLIKELY(data_len <= 0)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", KR(ret), KP(buf), K(data_len)); diff --git a/src/storage/tablelock/ob_table_lock_common.h b/src/storage/tablelock/ob_table_lock_common.h index 193d7fb58..ca20e404c 100644 --- a/src/storage/tablelock/ob_table_lock_common.h +++ b/src/storage/tablelock/ob_table_lock_common.h @@ -450,7 +450,7 @@ bool is_lock_owner_type_valid(const ObLockOwnerType &type) class ObNewTableLockOwnerID { public: - static const int16_t MAGIC_NUM = -0xABC; + static const int64_t MAGIC_NUM = -0xABC; }; class ObTableLockOwnerID From 2f7388913b4b610859d85946432f963288f38f66 Mon Sep 17 00:00:00 2001 From: obdev Date: Mon, 19 Aug 2024 18:59:16 +0000 Subject: [PATCH 121/249] [CP] [to #2024071200103692093]fix bugs, all/user/dba_procedures add trigger results --- .../ob_inner_table_schema.25051_25100.cpp | 6 +- .../inner_table/ob_inner_table_schema_def.py | 85 ++++++++++++++++++- 2 files changed, 84 insertions(+), 7 deletions(-) diff --git a/src/share/inner_table/ob_inner_table_schema.25051_25100.cpp b/src/share/inner_table/ob_inner_table_schema.25051_25100.cpp index e2256a670..ef0f29f89 100644 --- a/src/share/inner_table/ob_inner_table_schema.25051_25100.cpp +++ b/src/share/inner_table/ob_inner_table_schema.25051_25100.cpp @@ -210,7 +210,7 @@ int ObInnerTableSchema::dba_procedures_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT D.DATABASE_NAME AS OWNER, CASE R.ROUTINE_TYPE WHEN 1 THEN R.ROUTINE_NAME WHEN 2 THEN R.ROUTINE_NAME WHEN 3 THEN P.PACKAGE_NAME WHEN 4 THEN T.TYPE_NAME END AS OBJECT_NAME, CASE R.ROUTINE_TYPE WHEN 1 THEN NULL WHEN 2 THEN NULL WHEN 3 THEN R.ROUTINE_NAME WHEN 4 THEN R.ROUTINE_NAME END AS PROCEDURE_NAME, CASE R.ROUTINE_TYPE WHEN 1 THEN R.ROUTINE_ID WHEN 2 THEN R.ROUTINE_ID WHEN 3 THEN P.PACKAGE_ID WHEN 4 THEN T.TYPE_ID END AS OBJECT_ID, CASE R.SUBPROGRAM_ID WHEN 0 THEN 1 ELSE R.SUBPROGRAM_ID END AS SUBPROGRAM_ID, CASE R.OVERLOAD WHEN 0 THEN NULL ELSE R.OVERLOAD END AS OVERLOAD, CASE R.ROUTINE_TYPE WHEN 1 THEN 'PROCEDURE' WHEN 2 THEN 'FUNCTION' WHEN 3 THEN 'PACKAGE' WHEN 4 THEN 'TYPE' END AS OBJECT_TYPE, CAST(DECODE(BITAND(R.FLAG, 16384), 16384, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, CAST(DECODE(BITAND(R.FLAG, 128), 128, 'YES', 'NO') AS VARCHAR2(3)) AS PIPELINED, D1.DATABASE_NAME AS IMPLTYPEOWNER, T1.TYPE_NAME AS IMPLTYPENAME, CAST(DECODE(BITAND(R.FLAG, 8), 8, 'YES', 'NO') AS VARCHAR2(3)) AS PARALLEL, CAST('NO' AS VARCHAR2(3)) AS INTERFACE, CAST(DECODE(BITAND(R.FLAG, 4), 4, 'YES', 'NO') AS VARCHAR2(3)) AS DETERMINISTIC, CAST(DECODE(BITAND(R.FLAG, 16), 16, 'INVOKER', 'DEFINER') AS VARCHAR2(12)) AS AUTHID, R.TENANT_ID AS ORIGIN_CON_ID FROM (SELECT * FROM SYS.ALL_VIRTUAL_ROUTINE_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID())R LEFT JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D ON R.DATABASE_ID = D.DATABASE_ID AND D.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_PACKAGE_REAL_AGENT P ON R.PACKAGE_ID = P.PACKAGE_ID AND P.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T ON R.PACKAGE_ID = T.TYPE_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T1 ON R.TYPE_ID = T1.TYPE_ID AND T1.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D1 ON T1.DATABASE_ID = D1.DATABASE_ID AND T1.TENANT_ID = EFFECTIVE_TENANT_ID() WHERE D.IN_RECYCLEBIN = 0 UNION ALL SELECT CAST('SYS' AS VARCHAR2(30)) AS OWNER, CASE RS.ROUTINE_TYPE WHEN 1 THEN RS.ROUTINE_NAME WHEN 2 THEN RS.ROUTINE_NAME WHEN 3 THEN PS.PACKAGE_NAME WHEN 4 THEN TS.TYPE_NAME END AS OBJECT_NAME, CASE RS.ROUTINE_TYPE WHEN 1 THEN NULL WHEN 2 THEN NULL WHEN 3 THEN RS.ROUTINE_NAME WHEN 4 THEN RS.ROUTINE_NAME END AS PROCEDURE_NAME, CASE RS.ROUTINE_TYPE WHEN 1 THEN RS.ROUTINE_ID WHEN 2 THEN RS.ROUTINE_ID WHEN 3 THEN PS.PACKAGE_ID WHEN 4 THEN TS.TYPE_ID END AS OBJECT_ID, CASE RS.SUBPROGRAM_ID WHEN 0 THEN 1 ELSE RS.SUBPROGRAM_ID END AS SUBPROGRAM_ID, CASE RS.OVERLOAD WHEN 0 THEN NULL ELSE RS.OVERLOAD END AS OVERLOAD, CASE RS.ROUTINE_TYPE WHEN 1 THEN 'PROCEDURE' WHEN 2 THEN 'FUNCTION' WHEN 3 THEN 'PACKAGE' WHEN 4 THEN 'TYPE' END AS OBJECT_TYPE, CAST(DECODE(BITAND(RS.FLAG, 16384), 16484, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, CAST(DECODE(BITAND(RS.FLAG, 128), 128, 'YES', 'NO') AS VARCHAR2(3)) AS PIPELINED, CAST(CASE WHEN TS1.TYPE_NAME IS NULL THEN NULL ELSE 'SYS' END AS VARCHAR2(30)) AS IMPLTYPEOWNER, TS1.TYPE_NAME AS IMPLTYPENAME, CAST(DECODE(BITAND(RS.FLAG, 8), 8, 'YES', 'NO') AS VARCHAR2(3)) AS PARALLEL, CAST('NO' AS VARCHAR2(3)) AS INTERFACE, CAST(DECODE(BITAND(RS.FLAG, 4), 4, 'YES', 'NO') AS VARCHAR2(3)) AS DETERMINISTIC, CAST(DECODE(BITAND(RS.FLAG, 16), 16, 'INVOKER', 'DEFINER') AS VARCHAR2(12)) AS AUTHID, RS.TENANT_ID AS ORIGIN_CON_ID FROM SYS.ALL_VIRTUAL_ROUTINE_SYS_AGENT RS LEFT JOIN SYS.ALL_VIRTUAL_PACKAGE_SYS_AGENT PS ON RS.PACKAGE_ID = PS.PACKAGE_ID LEFT JOIN SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS ON RS.PACKAGE_ID = TS.TYPE_ID LEFT JOIN SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS1 ON RS.TYPE_ID = TS1.TYPE_ID WHERE RS.ROUTINE_TYPE != 1 AND RS.ROUTINE_TYPE != 2 -- sys tenant only have sys package and type. )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT D.DATABASE_NAME AS OWNER, CASE R.ROUTINE_TYPE WHEN 1 THEN R.ROUTINE_NAME WHEN 2 THEN R.ROUTINE_NAME WHEN 3 THEN P.PACKAGE_NAME WHEN 4 THEN T.TYPE_NAME END AS OBJECT_NAME, CASE R.ROUTINE_TYPE WHEN 1 THEN NULL WHEN 2 THEN NULL WHEN 3 THEN R.ROUTINE_NAME WHEN 4 THEN R.ROUTINE_NAME END AS PROCEDURE_NAME, CASE R.ROUTINE_TYPE WHEN 1 THEN R.ROUTINE_ID WHEN 2 THEN R.ROUTINE_ID WHEN 3 THEN P.PACKAGE_ID WHEN 4 THEN T.TYPE_ID END AS OBJECT_ID, CASE R.SUBPROGRAM_ID WHEN 0 THEN 1 ELSE R.SUBPROGRAM_ID END AS SUBPROGRAM_ID, CASE R.OVERLOAD WHEN 0 THEN NULL ELSE R.OVERLOAD END AS OVERLOAD, CASE R.ROUTINE_TYPE WHEN 1 THEN 'PROCEDURE' WHEN 2 THEN 'FUNCTION' WHEN 3 THEN 'PACKAGE' WHEN 4 THEN 'TYPE' END AS OBJECT_TYPE, CAST(DECODE(BITAND(R.FLAG, 16384), 16384, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, CAST(DECODE(BITAND(R.FLAG, 128), 128, 'YES', 'NO') AS VARCHAR2(3)) AS PIPELINED, D1.DATABASE_NAME AS IMPLTYPEOWNER, T1.TYPE_NAME AS IMPLTYPENAME, CAST(DECODE(BITAND(R.FLAG, 8), 8, 'YES', 'NO') AS VARCHAR2(3)) AS PARALLEL, CAST('NO' AS VARCHAR2(3)) AS INTERFACE, CAST(DECODE(BITAND(R.FLAG, 4), 4, 'YES', 'NO') AS VARCHAR2(3)) AS DETERMINISTIC, CAST(DECODE(BITAND(R.FLAG, 16), 16, 'INVOKER', 'DEFINER') AS VARCHAR2(12)) AS AUTHID, R.TENANT_ID AS ORIGIN_CON_ID FROM (SELECT * FROM SYS.ALL_VIRTUAL_ROUTINE_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID())R LEFT JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D ON R.DATABASE_ID = D.DATABASE_ID AND D.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_PACKAGE_REAL_AGENT P ON R.PACKAGE_ID = P.PACKAGE_ID AND P.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T ON R.PACKAGE_ID = T.TYPE_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T1 ON R.TYPE_ID = T1.TYPE_ID AND T1.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D1 ON T1.DATABASE_ID = D1.DATABASE_ID AND T1.TENANT_ID = EFFECTIVE_TENANT_ID() WHERE D.IN_RECYCLEBIN = 0 UNION ALL SELECT CAST('SYS' AS VARCHAR2(30)) AS OWNER, CASE RS.ROUTINE_TYPE WHEN 1 THEN RS.ROUTINE_NAME WHEN 2 THEN RS.ROUTINE_NAME WHEN 3 THEN PS.PACKAGE_NAME WHEN 4 THEN TS.TYPE_NAME END AS OBJECT_NAME, CASE RS.ROUTINE_TYPE WHEN 1 THEN NULL WHEN 2 THEN NULL WHEN 3 THEN RS.ROUTINE_NAME WHEN 4 THEN RS.ROUTINE_NAME END AS PROCEDURE_NAME, CASE RS.ROUTINE_TYPE WHEN 1 THEN RS.ROUTINE_ID WHEN 2 THEN RS.ROUTINE_ID WHEN 3 THEN PS.PACKAGE_ID WHEN 4 THEN TS.TYPE_ID END AS OBJECT_ID, CASE RS.SUBPROGRAM_ID WHEN 0 THEN 1 ELSE RS.SUBPROGRAM_ID END AS SUBPROGRAM_ID, CASE RS.OVERLOAD WHEN 0 THEN NULL ELSE RS.OVERLOAD END AS OVERLOAD, CASE RS.ROUTINE_TYPE WHEN 1 THEN 'PROCEDURE' WHEN 2 THEN 'FUNCTION' WHEN 3 THEN 'PACKAGE' WHEN 4 THEN 'TYPE' END AS OBJECT_TYPE, CAST(DECODE(BITAND(RS.FLAG, 16384), 16384, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, CAST(DECODE(BITAND(RS.FLAG, 128), 128, 'YES', 'NO') AS VARCHAR2(3)) AS PIPELINED, CAST(CASE WHEN TS1.TYPE_NAME IS NULL THEN NULL ELSE 'SYS' END AS VARCHAR2(30)) AS IMPLTYPEOWNER, TS1.TYPE_NAME AS IMPLTYPENAME, CAST(DECODE(BITAND(RS.FLAG, 8), 8, 'YES', 'NO') AS VARCHAR2(3)) AS PARALLEL, CAST('NO' AS VARCHAR2(3)) AS INTERFACE, CAST(DECODE(BITAND(RS.FLAG, 4), 4, 'YES', 'NO') AS VARCHAR2(3)) AS DETERMINISTIC, CAST(DECODE(BITAND(RS.FLAG, 16), 16, 'INVOKER', 'DEFINER') AS VARCHAR2(12)) AS AUTHID, RS.TENANT_ID AS ORIGIN_CON_ID FROM SYS.ALL_VIRTUAL_ROUTINE_SYS_AGENT RS LEFT JOIN SYS.ALL_VIRTUAL_PACKAGE_SYS_AGENT PS ON RS.PACKAGE_ID = PS.PACKAGE_ID LEFT JOIN SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS ON RS.PACKAGE_ID = TS.TYPE_ID LEFT JOIN SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS1 ON RS.TYPE_ID = TS1.TYPE_ID WHERE RS.ROUTINE_TYPE != 1 AND RS.ROUTINE_TYPE != 2 UNION ALL SELECT DB.DATABASE_NAME AS OWNER, TRG.TRIGGER_NAME AS OBJECT_NAME, CAST(NULL AS VARCHAR2(128)) AS PROCEDURE_NAME, TRG.TRIGGER_ID AS OBJECT_ID, CAST(1 AS NUMBER) AS SUBPROGRAM_ID, CAST(NULL AS NUMBER) AS OVERLOAD, 'TRIGGER' AS OBJECT_TYPE, CAST('NO' AS VARCHAR2(3)) AS AGGREGATE, CAST('NO' AS VARCHAR2(3)) AS PIPELINED, CAST(NULL AS VARCHAR2(128)) AS IMPLTYPEOWNER, CAST(NULL AS VARCHAR2(64)) AS IMPLTYPENAME, CAST('NO' AS VARCHAR2(3)) AS PARALLEL, CAST('NO' AS VARCHAR2(3)) AS INTERFACE, CAST('NO' AS VARCHAR2(3)) AS DETERMINISTIC, CAST('DEFINER' AS VARCHAR2(12)) AS AUTHID, TRG.TENANT_ID AS ORIGIN_CON_ID FROM SYS.ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT TRG INNER JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB ON TRG.DATABASE_ID = DB.DATABASE_ID AND TRG.TENANT_ID = EFFECTIVE_TENANT_ID() AND DB.TENANT_ID = EFFECTIVE_TENANT_ID() AND DB.IN_RECYCLEBIN = 0 )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -360,7 +360,7 @@ int ObInnerTableSchema::all_procedures_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT D.DATABASE_NAME AS OWNER, CASE R.ROUTINE_TYPE WHEN 1 THEN R.ROUTINE_NAME WHEN 2 THEN R.ROUTINE_NAME WHEN 3 THEN P.PACKAGE_NAME WHEN 4 THEN T.TYPE_NAME END AS OBJECT_NAME, CASE R.ROUTINE_TYPE WHEN 1 THEN NULL WHEN 2 THEN NULL WHEN 3 THEN R.ROUTINE_NAME WHEN 4 THEN R.ROUTINE_NAME END AS PROCEDURE_NAME, CASE R.ROUTINE_TYPE WHEN 1 THEN R.ROUTINE_ID WHEN 2 THEN R.ROUTINE_ID WHEN 3 THEN P.PACKAGE_ID WHEN 4 THEN T.TYPE_ID END AS OBJECT_ID, CASE R.SUBPROGRAM_ID WHEN 0 THEN 1 ELSE R.SUBPROGRAM_ID END AS SUBPROGRAM_ID, CASE R.OVERLOAD WHEN 0 THEN NULL ELSE R.OVERLOAD END AS OVERLOAD, CASE R.ROUTINE_TYPE WHEN 1 THEN 'PROCEDURE' WHEN 2 THEN 'FUNCTION' WHEN 3 THEN 'PACKAGE' WHEN 4 THEN 'TYPE' END AS OBJECT_TYPE, CAST(DECODE(BITAND(R.FLAG, 16384), 16384, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, CAST(DECODE(BITAND(R.FLAG, 128), 128, 'YES', 'NO') AS VARCHAR2(3)) AS PIPELINED, D1.DATABASE_NAME AS IMPLTYPEOWNER, T1.TYPE_NAME AS IMPLTYPENAME, CAST(DECODE(BITAND(R.FLAG, 8), 8, 'YES', 'NO') AS VARCHAR2(3)) AS PARALLEL, CAST('NO' AS VARCHAR2(3)) AS INTERFACE, CAST(DECODE(BITAND(R.FLAG, 4), 4, 'YES', 'NO') AS VARCHAR2(3)) AS DETERMINISTIC, CAST(DECODE(BITAND(R.FLAG, 16), 16, 'INVOKER', 'DEFINER') AS VARCHAR2(12)) AS AUTHID, R.TENANT_ID AS ORIGIN_CON_ID FROM (SELECT * FROM SYS.ALL_VIRTUAL_ROUTINE_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID()) R LEFT JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D ON R.DATABASE_ID = D.DATABASE_ID AND D.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_PACKAGE_REAL_AGENT P ON R.PACKAGE_ID = P.PACKAGE_ID AND P.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T ON R.PACKAGE_ID = T.TYPE_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T1 ON R.TYPE_ID = T1.TYPE_ID AND T1.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D1 ON T1.DATABASE_ID = D1.DATABASE_ID AND T1.TENANT_ID = EFFECTIVE_TENANT_ID() WHERE (R.DATABASE_ID = USERENV('SCHEMAID') OR USER_CAN_ACCESS_OBJ(12, R.ROUTINE_ID, R.DATABASE_ID) = 1) AND D.IN_RECYCLEBIN = 0 UNION ALL SELECT CAST('SYS' AS VARCHAR2(30)) AS OWNER, CASE RS.ROUTINE_TYPE WHEN 1 THEN RS.ROUTINE_NAME WHEN 2 THEN RS.ROUTINE_NAME WHEN 3 THEN PS.PACKAGE_NAME WHEN 4 THEN TS.TYPE_NAME END AS OBJECT_NAME, CASE RS.ROUTINE_TYPE WHEN 1 THEN NULL WHEN 2 THEN NULL WHEN 3 THEN RS.ROUTINE_NAME WHEN 4 THEN RS.ROUTINE_NAME END AS PROCEDURE_NAME, CASE RS.ROUTINE_TYPE WHEN 1 THEN RS.ROUTINE_ID WHEN 2 THEN RS.ROUTINE_ID WHEN 3 THEN PS.PACKAGE_ID WHEN 4 THEN TS.TYPE_ID END AS OBJECT_ID, CASE RS.SUBPROGRAM_ID WHEN 0 THEN 1 ELSE RS.SUBPROGRAM_ID END AS SUBPROGRAM_ID, CASE RS.OVERLOAD WHEN 0 THEN NULL ELSE RS.OVERLOAD END AS OVERLOAD, CASE RS.ROUTINE_TYPE WHEN 1 THEN 'PROCEDURE' WHEN 2 THEN 'FUNCTION' WHEN 3 THEN 'PACKAGE' WHEN 4 THEN 'TYPE' END AS OBJECT_TYPE, CAST(DECODE(BITAND(RS.FLAG, 16384), 16484, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, CAST(DECODE(BITAND(RS.FLAG, 128), 128, 'YES', 'NO') AS VARCHAR2(3)) AS PIPELINED, CAST(CASE WHEN TS1.TYPE_NAME IS NULL THEN NULL ELSE 'SYS' END AS VARCHAR2(30)) AS IMPLTYPEOWNER, TS1.TYPE_NAME AS IMPLTYPENAME, CAST(DECODE(BITAND(RS.FLAG, 8), 8, 'YES', 'NO') AS VARCHAR2(3)) AS PARALLEL, CAST('NO' AS VARCHAR2(3)) AS INTERFACE, CAST(DECODE(BITAND(RS.FLAG, 4), 4, 'YES', 'NO') AS VARCHAR2(3)) AS DETERMINISTIC, CAST(DECODE(BITAND(RS.FLAG, 16), 16, 'INVOKER', 'DEFINER') AS VARCHAR2(12)) AS AUTHID, RS.TENANT_ID AS ORIGIN_CON_ID FROM SYS.ALL_VIRTUAL_ROUTINE_SYS_AGENT RS LEFT JOIN SYS.ALL_VIRTUAL_PACKAGE_SYS_AGENT PS ON RS.PACKAGE_ID = PS.PACKAGE_ID LEFT JOIN SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS ON RS.PACKAGE_ID = TS.TYPE_ID LEFT JOIN SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS1 ON RS.TYPE_ID = TS1.TYPE_ID WHERE RS.ROUTINE_TYPE != 1 AND RS.ROUTINE_TYPE != 2 -- sys tenant only have sys package and type. )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT D.DATABASE_NAME AS OWNER, CASE R.ROUTINE_TYPE WHEN 1 THEN R.ROUTINE_NAME WHEN 2 THEN R.ROUTINE_NAME WHEN 3 THEN P.PACKAGE_NAME WHEN 4 THEN T.TYPE_NAME END AS OBJECT_NAME, CASE R.ROUTINE_TYPE WHEN 1 THEN NULL WHEN 2 THEN NULL WHEN 3 THEN R.ROUTINE_NAME WHEN 4 THEN R.ROUTINE_NAME END AS PROCEDURE_NAME, CASE R.ROUTINE_TYPE WHEN 1 THEN R.ROUTINE_ID WHEN 2 THEN R.ROUTINE_ID WHEN 3 THEN P.PACKAGE_ID WHEN 4 THEN T.TYPE_ID END AS OBJECT_ID, CASE R.SUBPROGRAM_ID WHEN 0 THEN 1 ELSE R.SUBPROGRAM_ID END AS SUBPROGRAM_ID, CASE R.OVERLOAD WHEN 0 THEN NULL ELSE R.OVERLOAD END AS OVERLOAD, CASE R.ROUTINE_TYPE WHEN 1 THEN 'PROCEDURE' WHEN 2 THEN 'FUNCTION' WHEN 3 THEN 'PACKAGE' WHEN 4 THEN 'TYPE' END AS OBJECT_TYPE, CAST(DECODE(BITAND(R.FLAG, 16384), 16384, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, CAST(DECODE(BITAND(R.FLAG, 128), 128, 'YES', 'NO') AS VARCHAR2(3)) AS PIPELINED, D1.DATABASE_NAME AS IMPLTYPEOWNER, T1.TYPE_NAME AS IMPLTYPENAME, CAST(DECODE(BITAND(R.FLAG, 8), 8, 'YES', 'NO') AS VARCHAR2(3)) AS PARALLEL, CAST('NO' AS VARCHAR2(3)) AS INTERFACE, CAST(DECODE(BITAND(R.FLAG, 4), 4, 'YES', 'NO') AS VARCHAR2(3)) AS DETERMINISTIC, CAST(DECODE(BITAND(R.FLAG, 16), 16, 'INVOKER', 'DEFINER') AS VARCHAR2(12)) AS AUTHID, R.TENANT_ID AS ORIGIN_CON_ID FROM (SELECT * FROM SYS.ALL_VIRTUAL_ROUTINE_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID()) R LEFT JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D ON R.DATABASE_ID = D.DATABASE_ID AND D.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_PACKAGE_REAL_AGENT P ON R.PACKAGE_ID = P.PACKAGE_ID AND P.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T ON R.PACKAGE_ID = T.TYPE_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T1 ON R.TYPE_ID = T1.TYPE_ID AND T1.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D1 ON T1.DATABASE_ID = D1.DATABASE_ID AND T1.TENANT_ID = EFFECTIVE_TENANT_ID() WHERE (R.DATABASE_ID = USERENV('SCHEMAID') OR USER_CAN_ACCESS_OBJ(12, R.ROUTINE_ID, R.DATABASE_ID) = 1) AND D.IN_RECYCLEBIN = 0 UNION ALL SELECT CAST('SYS' AS VARCHAR2(30)) AS OWNER, CASE RS.ROUTINE_TYPE WHEN 1 THEN RS.ROUTINE_NAME WHEN 2 THEN RS.ROUTINE_NAME WHEN 3 THEN PS.PACKAGE_NAME WHEN 4 THEN TS.TYPE_NAME END AS OBJECT_NAME, CASE RS.ROUTINE_TYPE WHEN 1 THEN NULL WHEN 2 THEN NULL WHEN 3 THEN RS.ROUTINE_NAME WHEN 4 THEN RS.ROUTINE_NAME END AS PROCEDURE_NAME, CASE RS.ROUTINE_TYPE WHEN 1 THEN RS.ROUTINE_ID WHEN 2 THEN RS.ROUTINE_ID WHEN 3 THEN PS.PACKAGE_ID WHEN 4 THEN TS.TYPE_ID END AS OBJECT_ID, CASE RS.SUBPROGRAM_ID WHEN 0 THEN 1 ELSE RS.SUBPROGRAM_ID END AS SUBPROGRAM_ID, CASE RS.OVERLOAD WHEN 0 THEN NULL ELSE RS.OVERLOAD END AS OVERLOAD, CASE RS.ROUTINE_TYPE WHEN 1 THEN 'PROCEDURE' WHEN 2 THEN 'FUNCTION' WHEN 3 THEN 'PACKAGE' WHEN 4 THEN 'TYPE' END AS OBJECT_TYPE, CAST(DECODE(BITAND(RS.FLAG, 16384), 16384, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, CAST(DECODE(BITAND(RS.FLAG, 128), 128, 'YES', 'NO') AS VARCHAR2(3)) AS PIPELINED, CAST(CASE WHEN TS1.TYPE_NAME IS NULL THEN NULL ELSE 'SYS' END AS VARCHAR2(30)) AS IMPLTYPEOWNER, TS1.TYPE_NAME AS IMPLTYPENAME, CAST(DECODE(BITAND(RS.FLAG, 8), 8, 'YES', 'NO') AS VARCHAR2(3)) AS PARALLEL, CAST('NO' AS VARCHAR2(3)) AS INTERFACE, CAST(DECODE(BITAND(RS.FLAG, 4), 4, 'YES', 'NO') AS VARCHAR2(3)) AS DETERMINISTIC, CAST(DECODE(BITAND(RS.FLAG, 16), 16, 'INVOKER', 'DEFINER') AS VARCHAR2(12)) AS AUTHID, RS.TENANT_ID AS ORIGIN_CON_ID FROM SYS.ALL_VIRTUAL_ROUTINE_SYS_AGENT RS LEFT JOIN SYS.ALL_VIRTUAL_PACKAGE_SYS_AGENT PS ON RS.PACKAGE_ID = PS.PACKAGE_ID LEFT JOIN SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS ON RS.PACKAGE_ID = TS.TYPE_ID LEFT JOIN SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS1 ON RS.TYPE_ID = TS1.TYPE_ID WHERE RS.ROUTINE_TYPE != 1 AND RS.ROUTINE_TYPE != 2 UNION ALL SELECT DB.DATABASE_NAME AS OWNER, TRG.TRIGGER_NAME AS OBJECT_NAME, CAST(NULL AS VARCHAR2(128)) AS PROCEDURE_NAME, TRG.TRIGGER_ID AS OBJECT_ID, CAST(1 AS NUMBER) AS SUBPROGRAM_ID, CAST(NULL AS NUMBER) AS OVERLOAD, 'TRIGGER' AS OBJECT_TYPE, CAST('NO' AS VARCHAR2(3)) AS AGGREGATE, CAST('NO' AS VARCHAR2(3)) AS PIPELINED, CAST(NULL AS VARCHAR2(128)) AS IMPLTYPEOWNER, CAST(NULL AS VARCHAR2(64)) AS IMPLTYPENAME, CAST('NO' AS VARCHAR2(3)) AS PARALLEL, CAST('NO' AS VARCHAR2(3)) AS INTERFACE, CAST('NO' AS VARCHAR2(3)) AS DETERMINISTIC, CAST('DEFINER' AS VARCHAR2(12)) AS AUTHID, TRG.TENANT_ID AS ORIGIN_CON_ID FROM SYS.ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT TRG INNER JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB ON TRG.DATABASE_ID = DB.DATABASE_ID AND TRG.TENANT_ID = EFFECTIVE_TENANT_ID() AND DB.TENANT_ID = EFFECTIVE_TENANT_ID() AND (TRG.DATABASE_ID = USERENV('SCHEMAID') OR USER_CAN_ACCESS_OBJ(1, abs(nvl(TRG.BASE_OBJECT_ID,0)), TRG.DATABASE_ID) = 1) AND DB.IN_RECYCLEBIN = 0 )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -510,7 +510,7 @@ int ObInnerTableSchema::user_procedures_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CASE R.ROUTINE_TYPE WHEN 1 THEN R.ROUTINE_NAME WHEN 2 THEN R.ROUTINE_NAME WHEN 3 THEN P.PACKAGE_NAME WHEN 4 THEN T.TYPE_NAME END AS OBJECT_NAME, CASE R.ROUTINE_TYPE WHEN 1 THEN NULL WHEN 2 THEN NULL WHEN 3 THEN R.ROUTINE_NAME WHEN 4 THEN R.ROUTINE_NAME END AS PROCEDURE_NAME, CASE R.ROUTINE_TYPE WHEN 1 THEN R.ROUTINE_ID WHEN 2 THEN R.ROUTINE_ID WHEN 3 THEN P.PACKAGE_ID WHEN 4 THEN T.TYPE_ID END AS OBJECT_ID, CASE R.SUBPROGRAM_ID WHEN 0 THEN 1 ELSE R.SUBPROGRAM_ID END AS SUBPROGRAM_ID, CASE R.OVERLOAD WHEN 0 THEN NULL ELSE R.OVERLOAD END AS OVERLOAD, CASE R.ROUTINE_TYPE WHEN 1 THEN 'PROCEDURE' WHEN 2 THEN 'FUNCTION' WHEN 3 THEN 'PACKAGE' WHEN 4 THEN 'TYPE' END AS OBJECT_TYPE, CAST(DECODE(BITAND(R.FLAG, 16384), 16384, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, CAST(DECODE(BITAND(R.FLAG, 128), 128, 'YES', 'NO') AS VARCHAR2(3)) AS PIPELINED, D1.DATABASE_NAME AS IMPLTYPEOWNER, T1.TYPE_NAME AS IMPLTYPENAME, CAST(DECODE(BITAND(R.FLAG, 8), 8, 'YES', 'NO') AS VARCHAR2(3)) AS PARALLEL, CAST('NO' AS VARCHAR2(3)) AS INTERFACE, CAST(DECODE(BITAND(R.FLAG, 4), 4, 'YES', 'NO') AS VARCHAR2(3)) AS DETERMINISTIC, CAST(DECODE(BITAND(R.FLAG, 16), 16, 'INVOKER', 'DEFINER') AS VARCHAR2(12)) AS AUTHID, R.TENANT_ID AS ORIGIN_CON_ID FROM (SELECT * FROM SYS.ALL_VIRTUAL_ROUTINE_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID()) R LEFT JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D ON R.DATABASE_ID = D.DATABASE_ID AND D.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_PACKAGE_REAL_AGENT P ON R.PACKAGE_ID = P.PACKAGE_ID AND P.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T ON R.PACKAGE_ID = T.TYPE_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T1 ON R.TYPE_ID = T1.TYPE_ID AND T1.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D1 ON T1.DATABASE_ID = D1.DATABASE_ID AND T1.TENANT_ID = EFFECTIVE_TENANT_ID() WHERE D.IN_RECYCLEBIN = 0 AND R.DATABASE_ID = USERENV('SCHEMAID') )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CASE R.ROUTINE_TYPE WHEN 1 THEN R.ROUTINE_NAME WHEN 2 THEN R.ROUTINE_NAME WHEN 3 THEN P.PACKAGE_NAME WHEN 4 THEN T.TYPE_NAME END AS OBJECT_NAME, CASE R.ROUTINE_TYPE WHEN 1 THEN NULL WHEN 2 THEN NULL WHEN 3 THEN R.ROUTINE_NAME WHEN 4 THEN R.ROUTINE_NAME END AS PROCEDURE_NAME, CASE R.ROUTINE_TYPE WHEN 1 THEN R.ROUTINE_ID WHEN 2 THEN R.ROUTINE_ID WHEN 3 THEN P.PACKAGE_ID WHEN 4 THEN T.TYPE_ID END AS OBJECT_ID, CASE R.SUBPROGRAM_ID WHEN 0 THEN 1 ELSE R.SUBPROGRAM_ID END AS SUBPROGRAM_ID, CASE R.OVERLOAD WHEN 0 THEN NULL ELSE R.OVERLOAD END AS OVERLOAD, CASE R.ROUTINE_TYPE WHEN 1 THEN 'PROCEDURE' WHEN 2 THEN 'FUNCTION' WHEN 3 THEN 'PACKAGE' WHEN 4 THEN 'TYPE' END AS OBJECT_TYPE, CAST(DECODE(BITAND(R.FLAG, 16384), 16384, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, CAST(DECODE(BITAND(R.FLAG, 128), 128, 'YES', 'NO') AS VARCHAR2(3)) AS PIPELINED, D1.DATABASE_NAME AS IMPLTYPEOWNER, T1.TYPE_NAME AS IMPLTYPENAME, CAST(DECODE(BITAND(R.FLAG, 8), 8, 'YES', 'NO') AS VARCHAR2(3)) AS PARALLEL, CAST('NO' AS VARCHAR2(3)) AS INTERFACE, CAST(DECODE(BITAND(R.FLAG, 4), 4, 'YES', 'NO') AS VARCHAR2(3)) AS DETERMINISTIC, CAST(DECODE(BITAND(R.FLAG, 16), 16, 'INVOKER', 'DEFINER') AS VARCHAR2(12)) AS AUTHID, R.TENANT_ID AS ORIGIN_CON_ID FROM (SELECT * FROM SYS.ALL_VIRTUAL_ROUTINE_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID()) R LEFT JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D ON R.DATABASE_ID = D.DATABASE_ID AND D.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_PACKAGE_REAL_AGENT P ON R.PACKAGE_ID = P.PACKAGE_ID AND P.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T ON R.PACKAGE_ID = T.TYPE_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T1 ON R.TYPE_ID = T1.TYPE_ID AND T1.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D1 ON T1.DATABASE_ID = D1.DATABASE_ID AND T1.TENANT_ID = EFFECTIVE_TENANT_ID() WHERE D.IN_RECYCLEBIN = 0 AND R.DATABASE_ID = USERENV('SCHEMAID') UNION ALL SELECT TRG.TRIGGER_NAME AS OBJECT_NAME, CAST(NULL AS VARCHAR2(128)) AS PROCEDURE_NAME, TRG.TRIGGER_ID AS OBJECT_ID, CAST(1 AS NUMBER) AS SUBPROGRAM_ID, CAST(NULL AS NUMBER) AS OVERLOAD, 'TRIGGER' AS OBJECT_TYPE, CAST('NO' AS VARCHAR2(3)) AS AGGREGATE, CAST('NO' AS VARCHAR2(3)) AS PIPELINED, CAST(NULL AS VARCHAR2(128)) AS IMPLTYPEOWNER, CAST(NULL AS VARCHAR2(64)) AS IMPLTYPENAME, CAST('NO' AS VARCHAR2(3)) AS PARALLEL, CAST('NO' AS VARCHAR2(3)) AS INTERFACE, CAST('NO' AS VARCHAR2(3)) AS DETERMINISTIC, CAST('DEFINER' AS VARCHAR2(12)) AS AUTHID, TRG.TENANT_ID AS ORIGIN_CON_ID FROM SYS.ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT TRG INNER JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB ON TRG.DATABASE_ID = DB.DATABASE_ID AND TRG.TENANT_ID = EFFECTIVE_TENANT_ID() AND DB.TENANT_ID = EFFECTIVE_TENANT_ID() AND TRG.DATABASE_ID = USERENV('SCHEMAID') AND DB.IN_RECYCLEBIN = 0 )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index a5291f631..60567a068 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -42647,7 +42647,7 @@ def_table_schema( WHEN 2 THEN 'FUNCTION' WHEN 3 THEN 'PACKAGE' WHEN 4 THEN 'TYPE' END AS OBJECT_TYPE, - CAST(DECODE(BITAND(RS.FLAG, 16384), 16484, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, + CAST(DECODE(BITAND(RS.FLAG, 16384), 16384, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, CAST(DECODE(BITAND(RS.FLAG, 128), 128, 'YES', 'NO') AS VARCHAR2(3)) AS PIPELINED, CAST(CASE WHEN TS1.TYPE_NAME IS NULL THEN NULL ELSE 'SYS' END AS VARCHAR2(30)) AS IMPLTYPEOWNER, TS1.TYPE_NAME AS IMPLTYPENAME, @@ -42661,7 +42661,32 @@ def_table_schema( LEFT JOIN SYS.ALL_VIRTUAL_PACKAGE_SYS_AGENT PS ON RS.PACKAGE_ID = PS.PACKAGE_ID LEFT JOIN SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS ON RS.PACKAGE_ID = TS.TYPE_ID LEFT JOIN SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS1 ON RS.TYPE_ID = TS1.TYPE_ID - WHERE RS.ROUTINE_TYPE != 1 AND RS.ROUTINE_TYPE != 2 -- sys tenant only have sys package and type. + WHERE RS.ROUTINE_TYPE != 1 AND RS.ROUTINE_TYPE != 2 + UNION ALL + SELECT + DB.DATABASE_NAME AS OWNER, + TRG.TRIGGER_NAME AS OBJECT_NAME, + CAST(NULL AS VARCHAR2(128)) AS PROCEDURE_NAME, + TRG.TRIGGER_ID AS OBJECT_ID, + CAST(1 AS NUMBER) AS SUBPROGRAM_ID, + CAST(NULL AS NUMBER) AS OVERLOAD, + 'TRIGGER' AS OBJECT_TYPE, + CAST('NO' AS VARCHAR2(3)) AS AGGREGATE, + CAST('NO' AS VARCHAR2(3)) AS PIPELINED, + CAST(NULL AS VARCHAR2(128)) AS IMPLTYPEOWNER, + CAST(NULL AS VARCHAR2(64)) AS IMPLTYPENAME, + CAST('NO' AS VARCHAR2(3)) AS PARALLEL, + CAST('NO' AS VARCHAR2(3)) AS INTERFACE, + CAST('NO' AS VARCHAR2(3)) AS DETERMINISTIC, + CAST('DEFINER' AS VARCHAR2(12)) AS AUTHID, + TRG.TENANT_ID AS ORIGIN_CON_ID + FROM SYS.ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT TRG + INNER JOIN + SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB + ON TRG.DATABASE_ID = DB.DATABASE_ID + AND TRG.TENANT_ID = EFFECTIVE_TENANT_ID() + AND DB.TENANT_ID = EFFECTIVE_TENANT_ID() + AND DB.IN_RECYCLEBIN = 0 """.replace("\n", " ") ) @@ -42958,7 +42983,7 @@ def_table_schema( WHEN 2 THEN 'FUNCTION' WHEN 3 THEN 'PACKAGE' WHEN 4 THEN 'TYPE' END AS OBJECT_TYPE, - CAST(DECODE(BITAND(RS.FLAG, 16384), 16484, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, + CAST(DECODE(BITAND(RS.FLAG, 16384), 16384, 'YES', 'NO') AS VARCHAR(3)) AS AGGREGATE, CAST(DECODE(BITAND(RS.FLAG, 128), 128, 'YES', 'NO') AS VARCHAR2(3)) AS PIPELINED, CAST(CASE WHEN TS1.TYPE_NAME IS NULL THEN NULL ELSE 'SYS' END AS VARCHAR2(30)) AS IMPLTYPEOWNER, TS1.TYPE_NAME AS IMPLTYPENAME, @@ -42972,7 +42997,34 @@ def_table_schema( LEFT JOIN SYS.ALL_VIRTUAL_PACKAGE_SYS_AGENT PS ON RS.PACKAGE_ID = PS.PACKAGE_ID LEFT JOIN SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS ON RS.PACKAGE_ID = TS.TYPE_ID LEFT JOIN SYS.ALL_VIRTUAL_TYPE_SYS_AGENT TS1 ON RS.TYPE_ID = TS1.TYPE_ID - WHERE RS.ROUTINE_TYPE != 1 AND RS.ROUTINE_TYPE != 2 -- sys tenant only have sys package and type. + WHERE RS.ROUTINE_TYPE != 1 AND RS.ROUTINE_TYPE != 2 + UNION ALL + SELECT + DB.DATABASE_NAME AS OWNER, + TRG.TRIGGER_NAME AS OBJECT_NAME, + CAST(NULL AS VARCHAR2(128)) AS PROCEDURE_NAME, + TRG.TRIGGER_ID AS OBJECT_ID, + CAST(1 AS NUMBER) AS SUBPROGRAM_ID, + CAST(NULL AS NUMBER) AS OVERLOAD, + 'TRIGGER' AS OBJECT_TYPE, + CAST('NO' AS VARCHAR2(3)) AS AGGREGATE, + CAST('NO' AS VARCHAR2(3)) AS PIPELINED, + CAST(NULL AS VARCHAR2(128)) AS IMPLTYPEOWNER, + CAST(NULL AS VARCHAR2(64)) AS IMPLTYPENAME, + CAST('NO' AS VARCHAR2(3)) AS PARALLEL, + CAST('NO' AS VARCHAR2(3)) AS INTERFACE, + CAST('NO' AS VARCHAR2(3)) AS DETERMINISTIC, + CAST('DEFINER' AS VARCHAR2(12)) AS AUTHID, + TRG.TENANT_ID AS ORIGIN_CON_ID + FROM SYS.ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT TRG + INNER JOIN + SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB + ON TRG.DATABASE_ID = DB.DATABASE_ID + AND TRG.TENANT_ID = EFFECTIVE_TENANT_ID() + AND DB.TENANT_ID = EFFECTIVE_TENANT_ID() + AND (TRG.DATABASE_ID = USERENV('SCHEMAID') + OR USER_CAN_ACCESS_OBJ(1, abs(nvl(TRG.BASE_OBJECT_ID,0)), TRG.DATABASE_ID) = 1) + AND DB.IN_RECYCLEBIN = 0 """.replace("\n", " ") ) @@ -43253,6 +43305,31 @@ def_table_schema( WHERE D.IN_RECYCLEBIN = 0 AND R.DATABASE_ID = USERENV('SCHEMAID') + UNION ALL + SELECT + TRG.TRIGGER_NAME AS OBJECT_NAME, + CAST(NULL AS VARCHAR2(128)) AS PROCEDURE_NAME, + TRG.TRIGGER_ID AS OBJECT_ID, + CAST(1 AS NUMBER) AS SUBPROGRAM_ID, + CAST(NULL AS NUMBER) AS OVERLOAD, + 'TRIGGER' AS OBJECT_TYPE, + CAST('NO' AS VARCHAR2(3)) AS AGGREGATE, + CAST('NO' AS VARCHAR2(3)) AS PIPELINED, + CAST(NULL AS VARCHAR2(128)) AS IMPLTYPEOWNER, + CAST(NULL AS VARCHAR2(64)) AS IMPLTYPENAME, + CAST('NO' AS VARCHAR2(3)) AS PARALLEL, + CAST('NO' AS VARCHAR2(3)) AS INTERFACE, + CAST('NO' AS VARCHAR2(3)) AS DETERMINISTIC, + CAST('DEFINER' AS VARCHAR2(12)) AS AUTHID, + TRG.TENANT_ID AS ORIGIN_CON_ID + FROM SYS.ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT TRG + INNER JOIN + SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB + ON TRG.DATABASE_ID = DB.DATABASE_ID + AND TRG.TENANT_ID = EFFECTIVE_TENANT_ID() + AND DB.TENANT_ID = EFFECTIVE_TENANT_ID() + AND TRG.DATABASE_ID = USERENV('SCHEMAID') + AND DB.IN_RECYCLEBIN = 0 """.replace("\n", " ") ) From 653dfe0e0f271e5f7abbbd388a3b64c9a9b56e84 Mon Sep 17 00:00:00 2001 From: obdev Date: Mon, 19 Aug 2024 19:16:53 +0000 Subject: [PATCH 122/249] [CP] [to 2024072500103923587 and 2024072400103906226]fix bugs, mysql.proc and information_schema.parameters --- deps/oblib/src/common/object/ob_obj_type.cpp | 27 +++++++++ deps/oblib/src/common/object/ob_obj_type.h | 11 ++++ .../virtual_table/ob_mysql_proc_table.cpp | 18 +++--- .../ob_inner_table_schema.21301_21350.cpp | 2 +- .../inner_table/ob_inner_table_schema_def.py | 59 ++++++++++++++++--- 5 files changed, 98 insertions(+), 19 deletions(-) diff --git a/deps/oblib/src/common/object/ob_obj_type.cpp b/deps/oblib/src/common/object/ob_obj_type.cpp index 6a32e0615..a7c246e34 100644 --- a/deps/oblib/src/common/object/ob_obj_type.cpp +++ b/deps/oblib/src/common/object/ob_obj_type.cpp @@ -698,6 +698,33 @@ bool is_match_alter_string_column_online_ddl_rules(const common::ObObjMeta& src_ return is_online_ddl; } +int ob_sql_type_str_with_coll(char *buff, + int64_t buff_length, + int64_t &pos, + ObObjType type, + int64_t length, + int64_t precision, + int64_t scale, + ObCollationType coll_type, + const uint64_t sub_type/* common::ObGeoType::GEOTYPEMAX */) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ob_sql_type_str(buff, buff_length, pos, type, length, precision, scale, coll_type, sub_type))) { + LOG_WARN("fail to get data type str", K(ret), K(sub_type), K(buff), K(buff_length), K(pos)); + } else if (lib::is_mysql_mode() && ob_is_string_type(type) && CS_TYPE_BINARY != coll_type) { + if (ObCharset::is_default_collation(coll_type)) { + if (OB_FAIL(databuff_printf(buff, buff_length, pos, " CHARSET %s", ObCharset::charset_name(coll_type)))) { + LOG_WARN("fail to concat charset str", K(ret), K(sub_type), K(buff), K(buff_length), K(pos)); + } + } else { + if (OB_FAIL(databuff_printf(buff, buff_length, pos, " CHARSET %s COLLATE %s", ObCharset::charset_name(coll_type), ObCharset::collation_name(coll_type)))) { + LOG_WARN("fail to concat charset and coll_type str", K(ret), K(sub_type), K(buff), K(buff_length), K(pos)); + } + } + } + return ret; +} + int ob_sql_type_str(char *buff, int64_t buff_length, int64_t &pos, diff --git a/deps/oblib/src/common/object/ob_obj_type.h b/deps/oblib/src/common/object/ob_obj_type.h index 22dfd50d0..512597169 100644 --- a/deps/oblib/src/common/object/ob_obj_type.h +++ b/deps/oblib/src/common/object/ob_obj_type.h @@ -1333,6 +1333,17 @@ const char *ob_obj_type_str(ObObjType type); const char *inner_obj_type_str(ObObjType type); const char *ob_sql_type_str(ObObjType type); +//such as “char(20) CHARSET utf8mb4" or "char(20) CHARSET utf8mb4 COLLATE utf8mb4_bin". with charset and collate +int ob_sql_type_str_with_coll(char *buff, + int64_t buff_length, + int64_t &pos, + ObObjType type, + int64_t length, + int64_t precision, + int64_t scale, + ObCollationType coll_type, + const uint64_t sub_type = static_cast(common::ObGeoType::GEOTYPEMAX)); + //such as "double(10,7)". with accuracy int ob_sql_type_str(char *buff, int64_t buff_length, diff --git a/src/observer/virtual_table/ob_mysql_proc_table.cpp b/src/observer/virtual_table/ob_mysql_proc_table.cpp index d63c11877..52ac77e42 100644 --- a/src/observer/virtual_table/ob_mysql_proc_table.cpp +++ b/src/observer/virtual_table/ob_mysql_proc_table.cpp @@ -248,15 +248,15 @@ int ObMySQLProcTable::inner_get_next_row(common::ObNewRow *&row) SERVER_LOG(WARN, "fail to alloc returns_buf", K(ret)); } else { if (routine_info->is_function()) { - if (OB_FAIL(ob_sql_type_str(returns_buf, - returns_buf_size, - pos, - routine_info->get_ret_type()->get_obj_type(), - routine_info->get_ret_type()->get_length(), - routine_info->get_ret_type()->get_precision(), - routine_info->get_ret_type()->get_scale(), - routine_info->get_ret_type()->get_collation_type()))) { - SHARE_SCHEMA_LOG(WARN, "fail to get data type str", KPC(routine_info->get_ret_type())); + if (OB_FAIL(ob_sql_type_str_with_coll(returns_buf, + returns_buf_size, + pos, + routine_info->get_ret_type()->get_obj_type(), + routine_info->get_ret_type()->get_length(), + routine_info->get_ret_type()->get_precision(), + routine_info->get_ret_type()->get_scale(), + routine_info->get_ret_type()->get_collation_type()))) { + SHARE_SCHEMA_LOG(WARN, "fail to get data type str with coll", KPC(routine_info->get_ret_type())); } } else { // proc no returns, fill empty. diff --git a/src/share/inner_table/ob_inner_table_schema.21301_21350.cpp b/src/share/inner_table/ob_inner_table_schema.21301_21350.cpp index 50e2a1891..8f09df6f7 100644 --- a/src/share/inner_table/ob_inner_table_schema.21301_21350.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21301_21350.cpp @@ -1967,7 +1967,7 @@ int ObInnerTableSchema::parameters_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__(select CAST('def' AS CHAR(512)) AS SPECIFIC_CATALOG, CAST(d.database_name AS CHAR(128)) collate utf8mb4_name_case AS SPECIFIC_SCHEMA, CAST(r.routine_name AS CHAR(64)) AS SPECIFIC_NAME, CAST(rp.param_position AS signed) AS ORDINAL_POSITION, CAST(CASE rp.param_position WHEN 0 THEN NULL ELSE CASE rp.flag & 0x03 WHEN 1 THEN 'IN' WHEN 2 THEN 'OUT' WHEN 3 THEN 'INOUT' ELSE NULL END END AS CHAR(5)) AS PARAMETER_MODE, CAST(rp.param_name AS CHAR(64)) AS PARAMETER_NAME, CAST(lower(case v.data_type_str when 'TINYINT UNSIGNED' then 'TINYINT' when 'SMALLINT UNSIGNED' then 'SMALLINT' when 'MEDIUMINT UNSIGNED' then 'MEDIUMINT' when 'INT UNSIGNED' then 'INT' when 'BIGINT UNSIGNED' then 'BIGINT' when 'FLOAT UNSIGNED' then 'FLOAT' when 'DOUBLE UNSIGNED' then 'DOUBLE' when 'DECIMAL UNSIGNED' then 'DECIMAL' else v.data_type_str end) AS CHAR(64)) AS DATA_TYPE, CASE WHEN rp.param_type IN (22, 23, 27, 28, 29, 30) THEN CAST(rp.param_length AS SIGNED) ELSE CAST(NULL AS SIGNED) END AS CHARACTER_MAXIMUM_LENGTH, CASE WHEN rp.param_type IN (22, 23, 27, 28, 29, 30, 43, 44, 46) THEN CAST( rp.param_length * CASE rp.param_coll_type WHEN 63 THEN 1 WHEN 249 THEN 4 WHEN 248 THEN 4 WHEN 87 THEN 2 WHEN 28 THEN 2 WHEN 55 THEN 4 WHEN 54 THEN 4 WHEN 101 THEN 2 WHEN 46 THEN 4 WHEN 45 THEN 4 WHEN 224 THEN 4 ELSE 1 END AS SIGNED ) ELSE CAST(NULL AS SIGNED) END AS CHARACTER_OCTET_LENGTH, CASE WHEN rp.param_type IN (1, 2, 3, 4, 5, 15, 16, 50) THEN CAST(rp.param_precision AS UNSIGNED) ELSE CAST(NULL AS UNSIGNED) END AS NUMERIC_PRECISION, CASE WHEN rp.param_type IN (15, 16, 50) THEN CAST(rp.param_scale AS SIGNED) WHEN rp.param_type IN (1, 2, 3, 4, 5, 11, 12, 13, 14) THEN CAST(0 AS SIGNED) ELSE CAST(NULL AS SIGNED) END AS NUMERIC_SCALE, CASE WHEN rp.param_type IN (17, 18, 20) THEN CAST(rp.param_scale AS UNSIGNED) ELSE CAST(NULL AS UNSIGNED) END AS DATETIME_PRECISION, CAST(CASE rp.param_charset WHEN 1 THEN 'binary' WHEN 2 THEN 'utf8mb4' WHEN 3 THEN 'gbk' WHEN 4 THEN 'utf16' WHEN 5 THEN 'gb18030' WHEN 6 THEN 'latin1' WHEN 7 THEN 'gb18030_2022' ELSE NULL END AS CHAR(64)) AS CHARACTER_SET_NAME, CAST(CASE rp.param_coll_type WHEN 45 THEN 'utf8mb4_general_ci' WHEN 46 THEN 'utf8mb4_bin' WHEN 63 THEN 'binary' ELSE NULL END AS CHAR(64)) AS COLLATION_NAME, CAST(CASE WHEN rp.param_type IN (1, 2, 3, 4, 5) THEN CONCAT(lower(v.data_type_str),'(',rp.param_precision,')') WHEN rp.param_type IN (15,16,50) THEN CONCAT(lower(v.data_type_str),'(',rp.param_precision, ',', rp.param_scale,')') WHEN rp.param_type IN (18, 20) THEN CONCAT(lower(v.data_type_str),'(', rp.param_scale, ')') WHEN rp.param_type IN (22, 23) THEN CONCAT(lower(v.data_type_str),'(', rp.param_length, ')') ELSE lower(v.data_type_str) END AS char(4194304)) AS DTD_IDENTIFIER, CAST(CASE WHEN r.routine_type = 1 THEN 'PROCEDURE' WHEN ROUTINE_TYPE = 2 THEN 'FUNCTION' ELSE NULL END AS CHAR(9)) AS ROUTINE_TYPE from oceanbase.__all_routine_param as rp join oceanbase.__all_routine as r on rp.subprogram_id = r.subprogram_id and rp.tenant_id = r.tenant_id and rp.routine_id = r.routine_id join oceanbase.__all_database as d on r.database_id = d.database_id left join oceanbase.__all_virtual_data_type v on rp.param_type = v.data_type WHERE rp.tenant_id = 0 and in_recyclebin = 0 and database_name != '__recyclebin' order by SPECIFIC_SCHEMA, SPECIFIC_NAME, ORDINAL_POSITION )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__(select CAST('def' AS CHAR(512)) AS SPECIFIC_CATALOG, CAST(d.database_name AS CHAR(128)) collate utf8mb4_name_case AS SPECIFIC_SCHEMA, CAST(r.routine_name AS CHAR(64)) AS SPECIFIC_NAME, CAST(rp.param_position AS signed) AS ORDINAL_POSITION, CAST(CASE rp.param_position WHEN 0 THEN NULL ELSE CASE rp.flag & 0x03 WHEN 1 THEN 'IN' WHEN 2 THEN 'OUT' WHEN 3 THEN 'INOUT' ELSE NULL END END AS CHAR(5)) AS PARAMETER_MODE, CAST(rp.param_name AS CHAR(64)) AS PARAMETER_NAME, CAST(lower(case v.data_type_str when 'TINYINT UNSIGNED' then 'TINYINT' when 'SMALLINT UNSIGNED' then 'SMALLINT' when 'MEDIUMINT UNSIGNED' then 'MEDIUMINT' when 'INT UNSIGNED' then 'INT' when 'BIGINT UNSIGNED' then 'BIGINT' when 'FLOAT UNSIGNED' then 'FLOAT' when 'DOUBLE UNSIGNED' then 'DOUBLE' when 'DECIMAL UNSIGNED' then 'DECIMAL' when 'CHAR' then if(rp.param_charset = 1, 'BINARY', 'CHAR') when 'VARCHAR' then if(rp.param_charset = 1, 'VARBINARY', 'VARCHAR') when 'TINYTEXT' then if(rp.param_charset = 1, 'TINYBLOB', 'TINYTEXT') when 'TEXT' then if(rp.param_charset = 1, 'BLOB', 'TEXT') when 'MEDIUMTEXT' then if(rp.param_charset = 1, 'MEDIUMBLOB', 'MEDIUMTEXT') when 'LONGTEXT' then if(rp.param_charset = 1, 'LONGBLOB', 'LONGTEXT') else v.data_type_str end) AS CHAR(64)) AS DATA_TYPE, CASE WHEN rp.param_type IN (22, 23, 27, 28, 29, 30) THEN CAST(rp.param_length AS SIGNED) ELSE CAST(NULL AS SIGNED) END AS CHARACTER_MAXIMUM_LENGTH, CASE WHEN rp.param_type IN (22, 23, 27, 28, 29, 30, 43, 44, 46) THEN CAST( rp.param_length * CASE rp.param_coll_type WHEN 63 THEN 1 WHEN 249 THEN 4 WHEN 248 THEN 4 WHEN 87 THEN 2 WHEN 28 THEN 2 WHEN 55 THEN 4 WHEN 54 THEN 4 WHEN 101 THEN 2 WHEN 46 THEN 4 WHEN 45 THEN 4 WHEN 224 THEN 4 ELSE 1 END AS SIGNED ) ELSE CAST(NULL AS SIGNED) END AS CHARACTER_OCTET_LENGTH, CASE WHEN rp.param_type IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 16, 31, 50) THEN CAST(rp.param_precision AS UNSIGNED) WHEN rp.param_type IN (11, 13) THEN CAST(if(rp.param_scale = -1, 12, rp.param_precision) AS UNSIGNED) WHEN rp.param_type IN (12, 14) THEN CAST(if(rp.param_scale = -1, 22, rp.param_precision) AS UNSIGNED) ELSE CAST(NULL AS UNSIGNED) END AS NUMERIC_PRECISION, CASE WHEN rp.param_type IN (15, 16, 50) THEN CAST(rp.param_scale AS SIGNED) WHEN rp.param_type IN (11, 12, 13, 14) THEN CAST(if(rp.param_scale = -1, 0, rp.param_scale) AS SIGNED) WHEN rp.param_type IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 31) THEN CAST(0 AS SIGNED) ELSE CAST(NULL AS SIGNED) END AS NUMERIC_SCALE, CASE WHEN rp.param_type IN (17, 18, 20) THEN CAST(rp.param_scale AS UNSIGNED) ELSE CAST(NULL AS UNSIGNED) END AS DATETIME_PRECISION, CAST(CASE rp.param_charset WHEN 1 THEN 'binary' WHEN 2 THEN 'utf8mb4' WHEN 3 THEN 'gbk' WHEN 4 THEN 'utf16' WHEN 5 THEN 'gb18030' WHEN 6 THEN 'latin1' WHEN 7 THEN 'gb18030_2022' ELSE NULL END AS CHAR(64)) AS CHARACTER_SET_NAME, CAST(CASE rp.param_coll_type WHEN 8 THEN 'latin1_swedish_ci' WHEN 11 THEN 'ascii_general_ci' WHEN 18 THEN 'tis620_thai_ci' WHEN 28 THEN 'gbk_chinese_ci' WHEN 45 THEN 'utf8mb4_general_ci' WHEN 46 THEN 'utf8mb4_bin' WHEN 47 THEN 'latin1_bin' WHEN 54 THEN 'utf16_general_ci' WHEN 55 THEN 'utf16_bin' WHEN 63 THEN 'binary' WHEN 65 THEN 'ascii_bin' WHEN 87 THEN 'gbk_bin' WHEN 89 THEN 'tis620_bin' WHEN 101 THEN 'utf16_unicode_ci' WHEN 216 THEN 'gb18030_2022_bin' WHEN 217 THEN 'gb18030_2022_chinese_ci' WHEN 218 THEN 'gb18030_2022_chinese_cs' WHEN 219 THEN 'gb18030_2022_radical_ci' WHEN 220 THEN 'gb18030_2022_radical_cs' WHEN 221 THEN 'gb18030_2022_stroke_ci' WHEN 222 THEN 'gb18030_2022_stroke_cs' WHEN 224 THEN 'utf8mb4_unicode_ci' WHEN 234 THEN 'utf8mb4_czech_ci' WHEN 245 THEN 'utf8mb4_croatian_ci' WHEN 246 THEN 'utf8mb4_unicode_520_ci' WHEN 248 THEN 'gb18030_chinese_ci' WHEN 249 THEN 'gb18030_bin' WHEN 255 THEN 'utf8mb4_0900_ai_ci' ELSE NULL END AS CHAR(64)) AS COLLATION_NAME, CAST(CASE WHEN rp.param_type IN (1, 2, 3, 4, 5, 31) THEN CONCAT(lower(v.data_type_str),'(',rp.param_precision,')') WHEN (rp.param_type in (6, 7, 8, 9, 10) AND rp.param_zero_fill) THEN CONCAT(lower(v.data_type_str), ' zerofill') WHEN rp.param_type IN (15,16,50) THEN CONCAT(lower(v.data_type_str),'(',rp.param_precision, ',', rp.param_scale,')') WHEN rp.param_type IN (17, 18, 20) THEN CONCAT(lower(v.data_type_str),'(', rp.param_scale, ')') WHEN (rp.param_type IN (22, 23) AND rp.param_charset != 1) THEN CONCAT(lower(v.data_type_str),'(', rp.param_length, ')') WHEN (rp.param_type IN (22) AND rp.param_charset = 1) THEN CONCAT(lower('VARBINARY'),'(', rp.param_length, ')') WHEN (rp.param_type IN (23) AND rp.param_charset = 1) THEN CONCAT(lower('BINARY'),'(', rp.param_length, ')') WHEN (rp.param_type IN (27, 28, 29, 30) AND rp.param_charset = 1) THEN lower(REPLACE(v.data_type_str, 'TEXT', 'BLOB')) ELSE lower(v.data_type_str) END AS char(4194304)) AS DTD_IDENTIFIER, CAST(CASE WHEN r.routine_type = 1 THEN 'PROCEDURE' WHEN ROUTINE_TYPE = 2 THEN 'FUNCTION' ELSE NULL END AS CHAR(9)) AS ROUTINE_TYPE from oceanbase.__all_routine_param as rp join oceanbase.__all_routine as r on rp.subprogram_id = r.subprogram_id and rp.tenant_id = r.tenant_id and rp.routine_id = r.routine_id join oceanbase.__all_database as d on r.database_id = d.database_id left join oceanbase.__all_virtual_data_type v on rp.param_type = v.data_type WHERE rp.tenant_id = 0 and in_recyclebin = 0 and database_name != '__recyclebin' order by SPECIFIC_SCHEMA, SPECIFIC_NAME, ORDINAL_POSITION )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index 60567a068..7574608c5 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -29261,6 +29261,12 @@ def_table_schema( when 'FLOAT UNSIGNED' then 'FLOAT' when 'DOUBLE UNSIGNED' then 'DOUBLE' when 'DECIMAL UNSIGNED' then 'DECIMAL' + when 'CHAR' then if(rp.param_charset = 1, 'BINARY', 'CHAR') + when 'VARCHAR' then if(rp.param_charset = 1, 'VARBINARY', 'VARCHAR') + when 'TINYTEXT' then if(rp.param_charset = 1, 'TINYBLOB', 'TINYTEXT') + when 'TEXT' then if(rp.param_charset = 1, 'BLOB', 'TEXT') + when 'MEDIUMTEXT' then if(rp.param_charset = 1, 'MEDIUMBLOB', 'MEDIUMTEXT') + when 'LONGTEXT' then if(rp.param_charset = 1, 'LONGBLOB', 'LONGTEXT') else v.data_type_str end) AS CHAR(64)) AS DATA_TYPE, CASE WHEN rp.param_type IN (22, 23, 27, 28, 29, 30) THEN CAST(rp.param_length AS SIGNED) ELSE CAST(NULL AS SIGNED) @@ -29285,12 +29291,14 @@ def_table_schema( ) ELSE CAST(NULL AS SIGNED) END AS CHARACTER_OCTET_LENGTH, - CASE WHEN rp.param_type IN (1, 2, 3, 4, 5, 15, 16, 50) - THEN CAST(rp.param_precision AS UNSIGNED) + CASE WHEN rp.param_type IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 16, 31, 50) THEN CAST(rp.param_precision AS UNSIGNED) + WHEN rp.param_type IN (11, 13) THEN CAST(if(rp.param_scale = -1, 12, rp.param_precision) AS UNSIGNED) + WHEN rp.param_type IN (12, 14) THEN CAST(if(rp.param_scale = -1, 22, rp.param_precision) AS UNSIGNED) ELSE CAST(NULL AS UNSIGNED) END AS NUMERIC_PRECISION, CASE WHEN rp.param_type IN (15, 16, 50) THEN CAST(rp.param_scale AS SIGNED) - WHEN rp.param_type IN (1, 2, 3, 4, 5, 11, 12, 13, 14) THEN CAST(0 AS SIGNED) + WHEN rp.param_type IN (11, 12, 13, 14) THEN CAST(if(rp.param_scale = -1, 0, rp.param_scale) AS SIGNED) + WHEN rp.param_type IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 31) THEN CAST(0 AS SIGNED) ELSE CAST(NULL AS SIGNED) END AS NUMERIC_SCALE, CASE WHEN rp.param_type IN (17, 18, 20) THEN CAST(rp.param_scale AS UNSIGNED) @@ -29307,19 +29315,52 @@ def_table_schema( ELSE NULL END AS CHAR(64)) AS CHARACTER_SET_NAME, CAST(CASE rp.param_coll_type - WHEN 45 THEN 'utf8mb4_general_ci' - WHEN 46 THEN 'utf8mb4_bin' - WHEN 63 THEN 'binary' + WHEN 8 THEN 'latin1_swedish_ci' + WHEN 11 THEN 'ascii_general_ci' + WHEN 18 THEN 'tis620_thai_ci' + WHEN 28 THEN 'gbk_chinese_ci' + WHEN 45 THEN 'utf8mb4_general_ci' + WHEN 46 THEN 'utf8mb4_bin' + WHEN 47 THEN 'latin1_bin' + WHEN 54 THEN 'utf16_general_ci' + WHEN 55 THEN 'utf16_bin' + WHEN 63 THEN 'binary' + WHEN 65 THEN 'ascii_bin' + WHEN 87 THEN 'gbk_bin' + WHEN 89 THEN 'tis620_bin' + WHEN 101 THEN 'utf16_unicode_ci' + WHEN 216 THEN 'gb18030_2022_bin' + WHEN 217 THEN 'gb18030_2022_chinese_ci' + WHEN 218 THEN 'gb18030_2022_chinese_cs' + WHEN 219 THEN 'gb18030_2022_radical_ci' + WHEN 220 THEN 'gb18030_2022_radical_cs' + WHEN 221 THEN 'gb18030_2022_stroke_ci' + WHEN 222 THEN 'gb18030_2022_stroke_cs' + WHEN 224 THEN 'utf8mb4_unicode_ci' + WHEN 234 THEN 'utf8mb4_czech_ci' + WHEN 245 THEN 'utf8mb4_croatian_ci' + WHEN 246 THEN 'utf8mb4_unicode_520_ci' + WHEN 248 THEN 'gb18030_chinese_ci' + WHEN 249 THEN 'gb18030_bin' + WHEN 255 THEN 'utf8mb4_0900_ai_ci' ELSE NULL END AS CHAR(64)) AS COLLATION_NAME, - CAST(CASE WHEN rp.param_type IN (1, 2, 3, 4, 5) + CAST(CASE WHEN rp.param_type IN (1, 2, 3, 4, 5, 31) THEN CONCAT(lower(v.data_type_str),'(',rp.param_precision,')') + WHEN (rp.param_type in (6, 7, 8, 9, 10) AND rp.param_zero_fill) + THEN CONCAT(lower(v.data_type_str), ' zerofill') WHEN rp.param_type IN (15,16,50) THEN CONCAT(lower(v.data_type_str),'(',rp.param_precision, ',', rp.param_scale,')') - WHEN rp.param_type IN (18, 20) + WHEN rp.param_type IN (17, 18, 20) THEN CONCAT(lower(v.data_type_str),'(', rp.param_scale, ')') - WHEN rp.param_type IN (22, 23) + WHEN (rp.param_type IN (22, 23) AND rp.param_charset != 1) THEN CONCAT(lower(v.data_type_str),'(', rp.param_length, ')') + WHEN (rp.param_type IN (22) AND rp.param_charset = 1) + THEN CONCAT(lower('VARBINARY'),'(', rp.param_length, ')') + WHEN (rp.param_type IN (23) AND rp.param_charset = 1) + THEN CONCAT(lower('BINARY'),'(', rp.param_length, ')') + WHEN (rp.param_type IN (27, 28, 29, 30) AND rp.param_charset = 1) + THEN lower(REPLACE(v.data_type_str, 'TEXT', 'BLOB')) ELSE lower(v.data_type_str) END AS char(4194304)) AS DTD_IDENTIFIER, CAST(CASE WHEN r.routine_type = 1 THEN 'PROCEDURE' WHEN ROUTINE_TYPE = 2 THEN 'FUNCTION' From 9a630dcc6726f2d573f8271eecdd6a663556d4a6 Mon Sep 17 00:00:00 2001 From: jingtaoye35 <1255153887@qq.com> Date: Tue, 20 Aug 2024 04:47:45 +0000 Subject: [PATCH 123/249] disable sum(double/float) to groupby push down --- .../rewrite/ob_transform_groupby_pushdown.cpp | 23 ++++++++++++++++++- .../rewrite/ob_transform_groupby_pushdown.h | 6 ++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/sql/rewrite/ob_transform_groupby_pushdown.cpp b/src/sql/rewrite/ob_transform_groupby_pushdown.cpp index 5647cf75c..e247a3f00 100644 --- a/src/sql/rewrite/ob_transform_groupby_pushdown.cpp +++ b/src/sql/rewrite/ob_transform_groupby_pushdown.cpp @@ -226,7 +226,8 @@ int ObTransformGroupByPushdown::check_groupby_push_down_validity(ObSelectStmt *s aggr_expr->get_expr_type() != T_FUN_COUNT && aggr_expr->get_expr_type() != T_FUN_MIN && aggr_expr->get_expr_type() != T_FUN_MAX) || - aggr_expr->is_param_distinct()) { + aggr_expr->is_param_distinct() || + !can_sum_trans_to_sum_count(aggr_expr)) { is_valid = false; OPT_TRACE("invalid aggregation type for group by placement", aggr_expr); LOG_TRACE("invalid aggregation type for group by placement", K(is_valid), @@ -1695,3 +1696,23 @@ int ObTransformGroupByPushdown::check_hint_valid(ObDMLStmt &stmt, LOG_TRACE("succeed to check group by hint valid", K(is_valid), K(trans_tables), K(params), K(hint)); return ret; } + +bool ObTransformGroupByPushdown::can_sum_trans_to_sum_count(const ObRawExpr *expr) +{ + bool is_able = true; + if (OB_ISNULL(expr) || + expr->get_expr_type() != T_FUN_SUM || + OB_UNLIKELY(expr->get_param_count() != 1) || + OB_ISNULL(expr->get_param_expr(0))) { + /* do nothing */ + } else { + ObObjType obj_type = expr->get_param_expr(0)->get_result_type().get_type(); + if (ObFloatType == obj_type || ObUFloatType == obj_type || + ObDoubleType == obj_type || ObUDoubleType == obj_type) { + is_able = false; + OPT_TRACE("invalid aggregation param type"); + LOG_TRACE("invalid aggregation param type", K(obj_type)); + } + } + return is_able; +} \ No newline at end of file diff --git a/src/sql/rewrite/ob_transform_groupby_pushdown.h b/src/sql/rewrite/ob_transform_groupby_pushdown.h index 235ced7c8..54c7ea9e8 100644 --- a/src/sql/rewrite/ob_transform_groupby_pushdown.h +++ b/src/sql/rewrite/ob_transform_groupby_pushdown.h @@ -183,7 +183,11 @@ private: ObIArray ¶ms, bool &hint_force_pushdown, bool &is_valid); - + /* whether sum(c1) splift to sum(count(*) * c1) + * YES case: 1 + 1 + 1 + 2 + 2 + 2 = 3 * 1 + 3 * 2 + * NO case: 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 != 6 * 0.1 + */ + bool can_sum_trans_to_sum_count(const ObRawExpr *expr); private: // help functions int64_t get_count_sum_num(const ObIArray &exprs) From 0b0e5ecfb60172b34ff40ff56c0b07ef654de110 Mon Sep 17 00:00:00 2001 From: qingsuijiu <642782632@qq.com> Date: Tue, 20 Aug 2024 04:54:21 +0000 Subject: [PATCH 124/249] Clear the calculation flag of ObPxMSCoordOp. --- src/sql/engine/px/exchange/ob_px_receive_op.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sql/engine/px/exchange/ob_px_receive_op.cpp b/src/sql/engine/px/exchange/ob_px_receive_op.cpp index aded4a26a..8835a45d7 100644 --- a/src/sql/engine/px/exchange/ob_px_receive_op.cpp +++ b/src/sql/engine/px/exchange/ob_px_receive_op.cpp @@ -441,6 +441,7 @@ int ObPxReceiveOp::wrap_get_next_batch(const int64_t max_row_cnt) { const int64_t max_cnt = std::min(max_row_cnt, spec_.max_batch_size_); int ret = OB_SUCCESS; + clear_evaluated_flag(); int64_t idx = 0; ObEvalCtx::BatchInfoScopeGuard batch_info_guard(eval_ctx_); batch_info_guard.set_batch_size(max_cnt); From 4e2bb400c47b93990d22f65d9bb06d5e5e5bd15f Mon Sep 17 00:00:00 2001 From: tushicheng <18829573815@163.com> Date: Tue, 20 Aug 2024 07:16:31 +0000 Subject: [PATCH 125/249] malloc opt --- .../src/lib/alloc/ob_malloc_sample_struct.h | 8 +++-- .../src/lib/alloc/ob_malloc_time_monitor.cpp | 6 ++-- .../src/lib/alloc/ob_malloc_time_monitor.h | 31 +++++++++++-------- deps/oblib/src/lib/stat/ob_latch_define.h | 2 +- 4 files changed, 26 insertions(+), 21 deletions(-) diff --git a/deps/oblib/src/lib/alloc/ob_malloc_sample_struct.h b/deps/oblib/src/lib/alloc/ob_malloc_sample_struct.h index de1dc3b97..bdb5cac2b 100644 --- a/deps/oblib/src/lib/alloc/ob_malloc_sample_struct.h +++ b/deps/oblib/src/lib/alloc/ob_malloc_sample_struct.h @@ -87,9 +87,10 @@ struct ObMallocSamplePairCmp } }; -inline uint64_t ob_malloc_sample_hash(const char* data) +inline uint64_t ob_malloc_sample_hash(uint64_t v1, uint64_t v2) { - return (uint64_t)data * 0xdeece66d + 0xb; + uint64_t data = (v1<<32) | ((v2<<32)>>32); + return data * 0xdeece66d + 0xb; } inline ObMallocSampleLimiter::ObMallocSampleLimiter() @@ -124,7 +125,8 @@ inline bool ObMallocSampleLimiter::malloc_sample_allowed(const int64_t size, con // Full sample when size is bigger than 16M. ret = true; } else { - uint64_t hash_val = ob_malloc_sample_hash(attr.label_.str_); + int64_t tid = ob_gettid(); + uint64_t hash_val = ob_malloc_sample_hash((uint64_t)attr.label_.str_, tid); if (rate_limiters[hash_val & MAX_MALLOC_SAMPLER_NUM].try_acquire(size)) { ret = true; } diff --git a/deps/oblib/src/lib/alloc/ob_malloc_time_monitor.cpp b/deps/oblib/src/lib/alloc/ob_malloc_time_monitor.cpp index b3795e72a..631107f52 100644 --- a/deps/oblib/src/lib/alloc/ob_malloc_time_monitor.cpp +++ b/deps/oblib/src/lib/alloc/ob_malloc_time_monitor.cpp @@ -15,8 +15,6 @@ #include "lib/utility/ob_print_utils.h" using namespace oceanbase::lib; using namespace oceanbase::common; - -volatile int64_t ObMallocTimeMonitor::WARN_THRESHOLD = 100000; void ObMallocTimeMonitor::print() { char buf[1024] = {'\0'}; @@ -29,8 +27,8 @@ void ObMallocTimeMonitor::print() int64_t avg_cost_time = (0 == delta_count ? 0 : delta_total_cost_time / delta_count); last_total_cost_times_[i] = total_cost_time; last_counts_[i] = count; - int64_t left = (0 == i ? 0 : TIME_SLOT[i-1]); - int64_t right = TIME_SLOT[i]; + int64_t left = TIME_SLOT[i]; + int64_t right = TIME_SLOT[i + 1]; databuff_printf(buf, sizeof(buf), pos, "[MALLOC_TIME_MONITOR] [%8ld,%20ld): delta_total_cost_time=%15ld, delta_count=%15ld, avg_cost_time=%8ld\n", left, right, delta_total_cost_time, delta_count, avg_cost_time); diff --git a/deps/oblib/src/lib/alloc/ob_malloc_time_monitor.h b/deps/oblib/src/lib/alloc/ob_malloc_time_monitor.h index dcc20c429..72f727231 100644 --- a/deps/oblib/src/lib/alloc/ob_malloc_time_monitor.h +++ b/deps/oblib/src/lib/alloc/ob_malloc_time_monitor.h @@ -22,9 +22,10 @@ namespace lib class ObMallocTimeMonitor { public: - static volatile int64_t WARN_THRESHOLD; - static constexpr const int64_t TIME_SLOT[] = {10, 100, 1000, 10000, 100000, 1000000, INT64_MAX}; - static const int64_t TIME_SLOT_NUM = ARRAYSIZEOF(TIME_SLOT); + static const int64_t WARN_THRESHOLD = 100000; + static const int64_t RECORD_THRESHOLD = 1000; + static constexpr const int64_t TIME_SLOT[] = {1000, 10000, 100000, 1000000, INT64_MAX}; + static const int64_t TIME_SLOT_NUM = ARRAYSIZEOF(TIME_SLOT) - 1; ObMallocTimeMonitor() { MEMSET(this, 0, sizeof(*this)); @@ -35,28 +36,32 @@ public: static ObMallocTimeMonitor instance; return instance; } + void inc(int64_t cost_time) { for (int i = 0; i < TIME_SLOT_NUM; ++i) { - if (cost_time < TIME_SLOT[i]) { + if (cost_time < TIME_SLOT[i + 1]) { total_cost_times_[i].inc(cost_time); counts_[i].inc(1); break; } } } + void record_malloc_time(ObBasicTimeGuard& time_guard, const int64_t size, const ObMemAttr& attr) { const int64_t cost_time = time_guard.get_diff(); - inc(cost_time); - if (OB_UNLIKELY(cost_time > WARN_THRESHOLD)) { - const int64_t buf_len = 1024; - char buf[buf_len] = {'\0'}; - int64_t pos = attr.to_string(buf, buf_len); - (void)logdata_printf(buf, buf_len, pos, ", size=%ld, ", size); - pos += time_guard.to_string(buf + pos, buf_len - pos); - int64_t tid = GETTID(); - fprintf(stderr, "[%ld]OB_MALLOC COST TOO MUCH TIME, cost_time=%ld, %.*s\n", tid, cost_time, static_cast(pos), buf); + if (OB_UNLIKELY(cost_time >= RECORD_THRESHOLD)) { + inc(cost_time); + if (OB_UNLIKELY(cost_time > WARN_THRESHOLD)) { + const int64_t buf_len = 1024; + char buf[buf_len] = {'\0'}; + int64_t pos = attr.to_string(buf, buf_len); + (void)logdata_printf(buf, buf_len, pos, ", size=%ld, ", size); + pos += time_guard.to_string(buf + pos, buf_len - pos); + int64_t tid = GETTID(); + fprintf(stderr, "[%ld]OB_MALLOC COST TOO MUCH TIME, cost_time=%ld, %.*s\n", tid, cost_time, static_cast(pos), buf); + } } } void print(); diff --git a/deps/oblib/src/lib/stat/ob_latch_define.h b/deps/oblib/src/lib/stat/ob_latch_define.h index 89486c5b5..e7aa2e5e5 100644 --- a/deps/oblib/src/lib/stat/ob_latch_define.h +++ b/deps/oblib/src/lib/stat/ob_latch_define.h @@ -80,7 +80,7 @@ LATCH_DEF(SERVER_STATUS_LOCK, 53, "server status lock", LATCH_READ_PREFER, 2000, LATCH_DEF(SERVER_MAINTAINCE_LOCK, 54, "server maintaince lock", LATCH_READ_PREFER, 2000, 0, true) LATCH_DEF(UNIT_MANAGER_LOCK, 55, "unit manager lock", LATCH_READ_PREFER, 2000, 0, true) LATCH_DEF(ZONE_MANAGER_LOCK, 56, "zone manager maintaince lock", LATCH_READ_PREFER, 2000, 0, true) -LATCH_DEF(ALLOC_OBJECT_LOCK, 57, "object set lock", LATCH_READ_PREFER, 2000, 0, true) +LATCH_DEF(ALLOC_OBJECT_LOCK, 57, "object set lock", LATCH_READ_PREFER, 1, 0, true) LATCH_DEF(ALLOC_BLOCK_LOCK, 58, "block set lock", LATCH_READ_PREFER, 2000, 0, true) LATCH_DEF(TRACE_RECORDER_LOCK, 59, "normal trace recorder lock", LATCH_FIFO, 2000, 0, true) LATCH_DEF(SESSION_TRACE_RECORDER_LOCK, 60, "session trace recorder lock", LATCH_FIFO, 2000, 0, true) From 6833a96d6a6bd0f7fae0b4738c061493153a872a Mon Sep 17 00:00:00 2001 From: obdev Date: Tue, 20 Aug 2024 07:22:29 +0000 Subject: [PATCH 126/249] remove unused tmp file code --- src/storage/blocksstable/ob_block_manager.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/storage/blocksstable/ob_block_manager.cpp b/src/storage/blocksstable/ob_block_manager.cpp index c3239d91f..fc46108be 100644 --- a/src/storage/blocksstable/ob_block_manager.cpp +++ b/src/storage/blocksstable/ob_block_manager.cpp @@ -1502,11 +1502,7 @@ int ObBlockManager::mark_tmp_file_blocks( const uint64_t tenant_id = mtl_tenant_ids.at(i); MTL_SWITCH(tenant_id) { ObArray macro_block_list; - if (OB_FAIL(set_group_id(tenant_id))) { - LOG_WARN("isolate CPU and IOPS failed", K(ret)); - } else if (OB_FAIL(mark_tenant_blocks(mark_info, macro_id_set, tmp_status))) { - LOG_WARN("fail to mark tenant blocks", K(ret), K(tenant_id)); - } else if (OB_FALSE_IT(MTL(ObTenantTmpFileManager*)->get_macro_block_list(macro_block_list))) { + if (OB_FAIL(MTL(ObTenantTmpFileManager*)->get_macro_block_list(macro_block_list))) { LOG_WARN("fail to get macro block list", K(ret)); } else if (OB_FAIL(update_mark_info(macro_block_list, macro_id_set, mark_info))){ LOG_WARN("fail to update mark info", K(ret), K(macro_block_list.count())); From 32c124797e64937e235a37ff362ec5085f5585e2 Mon Sep 17 00:00:00 2001 From: coolfishchen Date: Tue, 20 Aug 2024 07:47:07 +0000 Subject: [PATCH 127/249] load data ignore first n lines of each file --- src/sql/engine/cmd/ob_load_data_direct_impl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sql/engine/cmd/ob_load_data_direct_impl.cpp b/src/sql/engine/cmd/ob_load_data_direct_impl.cpp index 361033e7b..1d44e089a 100644 --- a/src/sql/engine/cmd/ob_load_data_direct_impl.cpp +++ b/src/sql/engine/cmd/ob_load_data_direct_impl.cpp @@ -1553,7 +1553,7 @@ int ObLoadDataDirectImpl::MultiFilesLoadTaskProcessor::process() if (OB_FAIL(data_reader_.init(execute_param_->data_access_param_, *execute_ctx_, handle_->data_desc_, true))) { LOG_WARN("fail to init data reader", KR(ret)); - } else if (0 == handle_->data_desc_.file_idx_ && 0 == handle_->data_desc_.start_) { + } else if (0 == handle_->data_desc_.start_) { if (OB_FAIL(skip_ignore_rows(current_line_count))) { LOG_WARN("fail to skip ignore rows", KR(ret)); } else if (OB_UNLIKELY(current_line_count < execute_param_->ignore_row_num_)) { From f872bcb06e55afe82b107bb283d636bef61ab52d Mon Sep 17 00:00:00 2001 From: helloamateur Date: Tue, 20 Aug 2024 07:52:50 +0000 Subject: [PATCH 128/249] [CP] [GIS] fix srs leak in exception branch --- src/observer/omt/ob_tenant_srs.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/observer/omt/ob_tenant_srs.cpp b/src/observer/omt/ob_tenant_srs.cpp index 9813c12cd..d59f0822b 100644 --- a/src/observer/omt/ob_tenant_srs.cpp +++ b/src/observer/omt/ob_tenant_srs.cpp @@ -270,6 +270,7 @@ int ObTenantSrs::refresh_srs(bool is_sys) && OB_FAIL(srs_old_snapshots_.push_back(last_snapshot))) { LOG_WARN("failed to push last_snapshot to recycle queue", K(ret), K(tenant_id), K(is_sys)); } else { + last_snapshot->~ObSrsCacheSnapShot(); allocator_.free(last_snapshot); } } @@ -506,6 +507,7 @@ int ObTenantSrs::fetch_all_srs(ObSrsCacheSnapShot *&srs_snapshot, bool is_sys_sr } else { if (OB_FAIL(generate_pg_reserved_srs(snapshot))) { LOG_WARN("failed to geneate pg reserved srs", K(ret)); + snapshot->~ObSrsCacheSnapShot(); allocator_.free(snapshot); } else { snapshot->set_srs_version(srs_version); @@ -513,6 +515,7 @@ int ObTenantSrs::fetch_all_srs(ObSrsCacheSnapShot *&srs_snapshot, bool is_sys_sr } } } else if (snapshot != NULL) { + snapshot->~ObSrsCacheSnapShot(); allocator_.free(snapshot); LOG_WARN("failed to get all srs item, iter quit", K(ret)); } From 75d445c9dace5e9fe31d9e9a8c0b8878e0eafdc1 Mon Sep 17 00:00:00 2001 From: sdc Date: Tue, 20 Aug 2024 07:58:35 +0000 Subject: [PATCH 129/249] fix mysqltest cases --- tools/deploy/init_create_tenant_routines.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/deploy/init_create_tenant_routines.sql b/tools/deploy/init_create_tenant_routines.sql index b086be8ce..24b3f3fa7 100644 --- a/tools/deploy/init_create_tenant_routines.sql +++ b/tools/deploy/init_create_tenant_routines.sql @@ -214,13 +214,13 @@ end / drop procedure if exists create_oracle_tenant_mini;/ create procedure create_oracle_tenant_mini(tenant_name varchar(64)) begin - call oceanbase.create_oracle_tenant_1c1g(tenant_name); + call oceanbase.create_oracle_tenant_1c2g(tenant_name); end / drop procedure if exists create_oracle_tenant_mini_with_arg;/ create procedure create_oracle_tenant_mini_with_arg(tenant_name varchar(64), arg_list varchar(64)) begin - call oceanbase.create_oracle_tenant_1c1g_with_arg(tenant_name, arg_list); + call oceanbase.create_oracle_tenant_1c2g_with_arg(tenant_name, arg_list); end / -- create_oracle_tenant_1c1g / create_oracle_tenant_1c1g_with_arg: 创建一个1c1g的oracle租户 From a1dc54a38d69d5a66dc5eea7c768844bf83c686e Mon Sep 17 00:00:00 2001 From: "18523270951@163.com" <18523270951@163.com> Date: Tue, 20 Aug 2024 08:05:32 +0000 Subject: [PATCH 130/249] [CP] fix pkey-hash slice use wrong batch idx --- src/sql/executor/ob_slice_calc.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/sql/executor/ob_slice_calc.cpp b/src/sql/executor/ob_slice_calc.cpp index cdc5c66ea..2df93f17e 100644 --- a/src/sql/executor/ob_slice_calc.cpp +++ b/src/sql/executor/ob_slice_calc.cpp @@ -674,10 +674,13 @@ int ObAffinitizedRepartSliceIdxCalc::get_slice_idx_batch_inner(const ObIArray Date: Tue, 20 Aug 2024 08:11:37 +0000 Subject: [PATCH 131/249] Add option for check avx2 instruction for upgrade script --- tools/upgrade/upgrade_checker.py | 33 ++++++++++++++++++++++++++------ tools/upgrade/upgrade_post.py | 33 ++++++++++++++++++++++++++------ tools/upgrade/upgrade_pre.py | 33 ++++++++++++++++++++++++++------ 3 files changed, 81 insertions(+), 18 deletions(-) diff --git a/tools/upgrade/upgrade_checker.py b/tools/upgrade/upgrade_checker.py index 73243726b..154a132d6 100755 --- a/tools/upgrade/upgrade_checker.py +++ b/tools/upgrade/upgrade_checker.py @@ -114,6 +114,9 @@ sys.argv[0] + """ [OPTIONS]""" +\ ' that all modules should be run. They are splitted by ",".\n' +\ ' For example: -m all, or --module=ddl,normal_dml,special_action\n' +\ '-l, --log-file=name Log file path. If log file path is not given it\'s ' + os.path.splitext(sys.argv[0])[0] + '.log\n' +\ +'-arc, --cpu-arch=name CPU architecture. Whether machine in cluster support AVX2 arch or not.\n' +\ +' \'avx2\' for x86 avx2 instruction set supported\n' +\ +' \'avx2_not_support\' for x86 avx2 instruction set not supported\n' +\ '\n\n' +\ 'Maybe you want to run cmd like that:\n' +\ sys.argv[0] + ' -h 127.0.0.1 -P 3306 -u admin -p admin\n' @@ -173,7 +176,8 @@ Option('p', 'password', True, False, ''),\ # 要跑哪个模块，默认全跑 Option('m', 'module', True, False, 'all'),\ # 日志文件路径，不同脚本的main函数中中会改成不同的默认值 -Option('l', 'log-file', True, False) +Option('l', 'log-file', True, False),\ +Option('C', 'cpu-arch', True, False, 'unknown') ]\ def change_opt_defult_value(opt_long_name, opt_default_val): @@ -284,6 +288,12 @@ def get_opt_log_file(): for opt in g_opts: if 'log-file' == opt.get_long_name(): return opt.get_value() + +def get_opt_cpu_arch(): + global g_opts + for opt in g_opts: + if 'cpu-arch' == opt.get_long_name(): + return opt.get_value() #### ---------------end---------------------- #### --------------start : do_upgrade_pre.py-------------- @@ -878,9 +888,19 @@ def is_x86_arch(): # 检查cs_encoding格式是否兼容，对小于4.3.3版本的cpu不支持avx2指令集的集群，我们要求升级前schema上不存在cs_encoding的存储格式 # 注意：这里对混布集群 / schema上row_format进行了ddl变更的场景无法做到完全的防御 -def check_cs_encoding_arch_dependency_compatiblity(query_cur): +def check_cs_encoding_arch_dependency_compatiblity(query_cur, cpu_arch): can_upgrade = True need_check_schema = False + is_arch_support_avx2 = False + if 'unknown' == cpu_arch: + is_arch_support_avx2 = arch_support_avx2() + elif 'avx2' == cpu_arch: + is_arch_support_avx2 = True + elif 'avx2_not_support' == cpu_arch: + is_arch_support_avx2 = False + else: + fail_list.append("unexpected cpu_arch option value: {0}".format(cpu_arch)) + sql = """select distinct value from GV$OB_PARAMETERS where name='min_observer_version'""" (desc, results) = query_cur.exec_query(sql) if len(results) != 1: @@ -890,7 +910,7 @@ def check_cs_encoding_arch_dependency_compatiblity(query_cur): else: min_cluster_version = get_version(results[0][0]) if min_cluster_version < get_version("4.3.3.0"): - if (arch_support_avx2()): + if (is_arch_support_avx2): logging.info("current cpu support avx2 inst, no need to check cs_encoding format") else: get_data_version_sql = """select distinct value from oceanbase.__all_virtual_tenant_parameter_info where name='compatible'""" @@ -937,7 +957,7 @@ def check_cs_encoding_arch_dependency_compatiblity(query_cur): logging.info("check upgrade for arch-dependant cs_encoding format failed") # 开始升级前的检查 -def do_check(my_host, my_port, my_user, my_passwd, timeout, upgrade_params): +def do_check(my_host, my_port, my_user, my_passwd, timeout, upgrade_params, cpu_arch): try: conn = mysql.connector.connect(user = my_user, password = my_passwd, @@ -975,7 +995,7 @@ def do_check(my_host, my_port, my_user, my_passwd, timeout, upgrade_params): check_variable_binlog_row_image(query_cur) check_oracle_standby_replication_exist(query_cur) check_disk_space_for_mds_sstable_compat(query_cur) - check_cs_encoding_arch_dependency_compatiblity(query_cur) + check_cs_encoding_arch_dependency_compatiblity(query_cur, cpu_arch) # all check func should execute before check_fail_list check_fail_list() modify_server_permanent_offline_time(cur) @@ -1010,9 +1030,10 @@ if __name__ == '__main__': user = get_opt_user() password = get_opt_password() timeout = int(get_opt_timeout()) + cpu_arch = get_opt_cpu_arch() logging.info('parameters from cmd: host=\"%s\", port=%s, user=\"%s\", password=\"%s\", timeout=\"%s\", log-file=\"%s\"',\ host, port, user, password.replace('"', '\\"'), timeout, log_filename) - do_check(host, port, user, password, timeout, upgrade_params) + do_check(host, port, user, password, timeout, upgrade_params, cpu_arch) except mysql.connector.Error as e: logging.exception('mysql connctor error') raise diff --git a/tools/upgrade/upgrade_post.py b/tools/upgrade/upgrade_post.py index c6006f720..f837df482 100755 --- a/tools/upgrade/upgrade_post.py +++ b/tools/upgrade/upgrade_post.py @@ -1767,6 +1767,9 @@ #' that all modules should be run. They are splitted by ",".\n' +\ #' For example: -m all, or --module=ddl,normal_dml,special_action\n' +\ #'-l, --log-file=name Log file path. If log file path is not given it\'s ' + os.path.splitext(sys.argv[0])[0] + '.log\n' +\ +#'-arc, --cpu-arch=name CPU architecture. Whether machine in cluster support AVX2 arch or not.\n' +\ +#' \'avx2\' for x86 avx2 instruction set supported\n' +\ +#' \'avx2_not_support\' for x86 avx2 instruction set not supported\n' +\ #'\n\n' +\ #'Maybe you want to run cmd like that:\n' +\ #sys.argv[0] + ' -h 127.0.0.1 -P 3306 -u admin -p admin\n' @@ -1826,7 +1829,8 @@ ## 要跑哪个模块，默认全跑 #Option('m', 'module', True, False, 'all'),\ ## 日志文件路径，不同脚本的main函数中中会改成不同的默认值 -#Option('l', 'log-file', True, False) +#Option('l', 'log-file', True, False),\ +#Option('C', 'cpu-arch', True, False, 'unknown') #]\ # #def change_opt_defult_value(opt_long_name, opt_default_val): @@ -1937,6 +1941,12 @@ # for opt in g_opts: # if 'log-file' == opt.get_long_name(): # return opt.get_value() +# +#def get_opt_cpu_arch(): +# global g_opts +# for opt in g_opts: +# if 'cpu-arch' == opt.get_long_name(): +# return opt.get_value() ##### ---------------end---------------------- # ##### --------------start : do_upgrade_pre.py-------------- @@ -2531,9 +2541,19 @@ # ## 检查cs_encoding格式是否兼容，对小于4.3.3版本的cpu不支持avx2指令集的集群，我们要求升级前schema上不存在cs_encoding的存储格式 ## 注意：这里对混布集群 / schema上row_format进行了ddl变更的场景无法做到完全的防御 -#def check_cs_encoding_arch_dependency_compatiblity(query_cur): +#def check_cs_encoding_arch_dependency_compatiblity(query_cur, cpu_arch): # can_upgrade = True # need_check_schema = False +# is_arch_support_avx2 = False +# if 'unknown' == cpu_arch: +# is_arch_support_avx2 = arch_support_avx2() +# elif 'avx2' == cpu_arch: +# is_arch_support_avx2 = True +# elif 'avx2_not_support' == cpu_arch: +# is_arch_support_avx2 = False +# else: +# fail_list.append("unexpected cpu_arch option value: {0}".format(cpu_arch)) +# # sql = """select distinct value from GV$OB_PARAMETERS where name='min_observer_version'""" # (desc, results) = query_cur.exec_query(sql) # if len(results) != 1: @@ -2543,7 +2563,7 @@ # else: # min_cluster_version = get_version(results[0][0]) # if min_cluster_version < get_version("4.3.3.0"): -# if (arch_support_avx2()): +# if (is_arch_support_avx2): # logging.info("current cpu support avx2 inst, no need to check cs_encoding format") # else: # get_data_version_sql = """select distinct value from oceanbase.__all_virtual_tenant_parameter_info where name='compatible'""" @@ -2590,7 +2610,7 @@ # logging.info("check upgrade for arch-dependant cs_encoding format failed") # ## 开始升级前的检查 -#def do_check(my_host, my_port, my_user, my_passwd, timeout, upgrade_params): +#def do_check(my_host, my_port, my_user, my_passwd, timeout, upgrade_params, cpu_arch): # try: # conn = mysql.connector.connect(user = my_user, # password = my_passwd, @@ -2628,7 +2648,7 @@ # check_variable_binlog_row_image(query_cur) # check_oracle_standby_replication_exist(query_cur) # check_disk_space_for_mds_sstable_compat(query_cur) -# check_cs_encoding_arch_dependency_compatiblity(query_cur) +# check_cs_encoding_arch_dependency_compatiblity(query_cur, cpu_arch) # # all check func should execute before check_fail_list # check_fail_list() # modify_server_permanent_offline_time(cur) @@ -2663,9 +2683,10 @@ # user = get_opt_user() # password = get_opt_password() # timeout = int(get_opt_timeout()) +# cpu_arch = get_opt_cpu_arch() # logging.info('parameters from cmd: host=\"%s\", port=%s, user=\"%s\", password=\"%s\", timeout=\"%s\", log-file=\"%s\"',\ # host, port, user, password.replace('"', '\\"'), timeout, log_filename) -# do_check(host, port, user, password, timeout, upgrade_params) +# do_check(host, port, user, password, timeout, upgrade_params, cpu_arch) # except mysql.connector.Error as e: # logging.exception('mysql connctor error') # raise diff --git a/tools/upgrade/upgrade_pre.py b/tools/upgrade/upgrade_pre.py index 069c875ec..00e1ce181 100755 --- a/tools/upgrade/upgrade_pre.py +++ b/tools/upgrade/upgrade_pre.py @@ -1767,6 +1767,9 @@ #' that all modules should be run. They are splitted by ",".\n' +\ #' For example: -m all, or --module=ddl,normal_dml,special_action\n' +\ #'-l, --log-file=name Log file path. If log file path is not given it\'s ' + os.path.splitext(sys.argv[0])[0] + '.log\n' +\ +#'-arc, --cpu-arch=name CPU architecture. Whether machine in cluster support AVX2 arch or not.\n' +\ +#' \'avx2\' for x86 avx2 instruction set supported\n' +\ +#' \'avx2_not_support\' for x86 avx2 instruction set not supported\n' +\ #'\n\n' +\ #'Maybe you want to run cmd like that:\n' +\ #sys.argv[0] + ' -h 127.0.0.1 -P 3306 -u admin -p admin\n' @@ -1826,7 +1829,8 @@ ## 要跑哪个模块，默认全跑 #Option('m', 'module', True, False, 'all'),\ ## 日志文件路径，不同脚本的main函数中中会改成不同的默认值 -#Option('l', 'log-file', True, False) +#Option('l', 'log-file', True, False),\ +#Option('C', 'cpu-arch', True, False, 'unknown') #]\ # #def change_opt_defult_value(opt_long_name, opt_default_val): @@ -1937,6 +1941,12 @@ # for opt in g_opts: # if 'log-file' == opt.get_long_name(): # return opt.get_value() +# +#def get_opt_cpu_arch(): +# global g_opts +# for opt in g_opts: +# if 'cpu-arch' == opt.get_long_name(): +# return opt.get_value() ##### ---------------end---------------------- # ##### --------------start : do_upgrade_pre.py-------------- @@ -2531,9 +2541,19 @@ # ## 检查cs_encoding格式是否兼容，对小于4.3.3版本的cpu不支持avx2指令集的集群，我们要求升级前schema上不存在cs_encoding的存储格式 ## 注意：这里对混布集群 / schema上row_format进行了ddl变更的场景无法做到完全的防御 -#def check_cs_encoding_arch_dependency_compatiblity(query_cur): +#def check_cs_encoding_arch_dependency_compatiblity(query_cur, cpu_arch): # can_upgrade = True # need_check_schema = False +# is_arch_support_avx2 = False +# if 'unknown' == cpu_arch: +# is_arch_support_avx2 = arch_support_avx2() +# elif 'avx2' == cpu_arch: +# is_arch_support_avx2 = True +# elif 'avx2_not_support' == cpu_arch: +# is_arch_support_avx2 = False +# else: +# fail_list.append("unexpected cpu_arch option value: {0}".format(cpu_arch)) +# # sql = """select distinct value from GV$OB_PARAMETERS where name='min_observer_version'""" # (desc, results) = query_cur.exec_query(sql) # if len(results) != 1: @@ -2543,7 +2563,7 @@ # else: # min_cluster_version = get_version(results[0][0]) # if min_cluster_version < get_version("4.3.3.0"): -# if (arch_support_avx2()): +# if (is_arch_support_avx2): # logging.info("current cpu support avx2 inst, no need to check cs_encoding format") # else: # get_data_version_sql = """select distinct value from oceanbase.__all_virtual_tenant_parameter_info where name='compatible'""" @@ -2590,7 +2610,7 @@ # logging.info("check upgrade for arch-dependant cs_encoding format failed") # ## 开始升级前的检查 -#def do_check(my_host, my_port, my_user, my_passwd, timeout, upgrade_params): +#def do_check(my_host, my_port, my_user, my_passwd, timeout, upgrade_params, cpu_arch): # try: # conn = mysql.connector.connect(user = my_user, # password = my_passwd, @@ -2628,7 +2648,7 @@ # check_variable_binlog_row_image(query_cur) # check_oracle_standby_replication_exist(query_cur) # check_disk_space_for_mds_sstable_compat(query_cur) -# check_cs_encoding_arch_dependency_compatiblity(query_cur) +# check_cs_encoding_arch_dependency_compatiblity(query_cur, cpu_arch) # # all check func should execute before check_fail_list # check_fail_list() # modify_server_permanent_offline_time(cur) @@ -2663,9 +2683,10 @@ # user = get_opt_user() # password = get_opt_password() # timeout = int(get_opt_timeout()) +# cpu_arch = get_opt_cpu_arch() # logging.info('parameters from cmd: host=\"%s\", port=%s, user=\"%s\", password=\"%s\", timeout=\"%s\", log-file=\"%s\"',\ # host, port, user, password.replace('"', '\\"'), timeout, log_filename) -# do_check(host, port, user, password, timeout, upgrade_params) +# do_check(host, port, user, password, timeout, upgrade_params, cpu_arch) # except mysql.connector.Error as e: # logging.exception('mysql connctor error') # raise From 2eae4f3b29cb64c48884b8e9fd3740e6d8ea553b Mon Sep 17 00:00:00 2001 From: jw-guo Date: Tue, 20 Aug 2024 08:17:28 +0000 Subject: [PATCH 132/249] [xa][4.x] unregister timer task before set exiting --- src/storage/tx/ob_xa_ctx.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/storage/tx/ob_xa_ctx.cpp b/src/storage/tx/ob_xa_ctx.cpp index b1e74b998..1d7bd7c40 100644 --- a/src/storage/tx/ob_xa_ctx.cpp +++ b/src/storage/tx/ob_xa_ctx.cpp @@ -2327,6 +2327,7 @@ void ObXACtx::try_exit(const bool need_decrease_ref) void ObXACtx::try_exit_() { if (0 == xa_ref_count_) { + (void)unregister_timeout_task_(); set_exiting_(); } } From f5ddb04304d341af8f59c0a2b800259af9f6560b Mon Sep 17 00:00:00 2001 From: zhaoyiping0622 Date: Tue, 20 Aug 2024 08:35:56 +0000 Subject: [PATCH 133/249] upgrade virtual schema when change order of column --- src/rootserver/ob_root_inspection.cpp | 14 ++++---------- src/rootserver/ob_root_inspection.h | 3 +-- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/src/rootserver/ob_root_inspection.cpp b/src/rootserver/ob_root_inspection.cpp index fa9eb69cd..c713e394f 100755 --- a/src/rootserver/ob_root_inspection.cpp +++ b/src/rootserver/ob_root_inspection.cpp @@ -1286,9 +1286,8 @@ int ObRootInspection::check_table_schema(const ObTableSchema &hard_code_table, "table_name", hard_code_table.get_table_name(), "column", hard_code_column->get_column_name(), K(ret)); } else { - const bool ignore_column_id = is_virtual_table(hard_code_table.get_table_id()); if (OB_FAIL(check_column_schema_(hard_code_table.get_table_name(), - *column, *hard_code_column, ignore_column_id))) { + *column, *hard_code_column))) { LOG_WARN("column schema mismatch with hard code column schema", "table_name",inner_table.get_table_name(), "column", *column, "hard_code_column", *hard_code_column, K(ret)); @@ -1333,7 +1332,6 @@ int ObRootInspection::check_and_get_system_table_column_diff( const ObColumnSchemaV2 *column = NULL; const ObColumnSchemaV2 *hard_code_column = NULL; ObColumnSchemaV2 tmp_column; // check_column_can_be_altered_online() may change dst_column, is ugly. - bool ignore_column_id = false; // case 1. check if columns should be dropped. // case 2. check if column can be altered online. @@ -1353,8 +1351,7 @@ int ObRootInspection::check_and_get_system_table_column_diff( // case 2 int tmp_ret = check_column_schema_(table_schema.get_table_name_str(), *column, - *hard_code_column, - ignore_column_id); + *hard_code_column); if (OB_SUCCESS == tmp_ret) { // not changed } else if (OB_SCHEMA_ERROR != tmp_ret) { @@ -1654,8 +1651,7 @@ int ObRootInspection::check_table_options_(const ObTableSchema &table, int ObRootInspection::check_column_schema_(const ObString &table_name, const ObColumnSchemaV2 &column, - const ObColumnSchemaV2 &hard_code_column, - const bool ignore_column_id) + const ObColumnSchemaV2 &hard_code_column) { int ret = OB_SUCCESS; if (table_name.empty() || !column.is_valid() || !hard_code_column.is_valid()) { @@ -1689,9 +1685,7 @@ int ObRootInspection::check_column_schema_(const ObString &table_name, } } - if (!ignore_column_id) { - CMP_COLUMN_ATTR(column_id); - } + CMP_COLUMN_ATTR(column_id); CMP_COLUMN_ATTR(tenant_id); CMP_COLUMN_ATTR(table_id); // don't need to check schema version diff --git a/src/rootserver/ob_root_inspection.h b/src/rootserver/ob_root_inspection.h index 71b413016..d1d5d527e 100644 --- a/src/rootserver/ob_root_inspection.h +++ b/src/rootserver/ob_root_inspection.h @@ -210,8 +210,7 @@ private: const share::schema::ObTableSchema &hard_code_table); static int check_column_schema_(const common::ObString &table_name, const share::schema::ObColumnSchemaV2 &column, - const share::schema::ObColumnSchemaV2 &hard_code_column, - const bool ignore_column_id); + const share::schema::ObColumnSchemaV2 &hard_code_column); int check_data_version_(); int check_data_version_(const uint64_t tenant_id); From dce97deb2b1745551f7c470a063fd32b17103e5a Mon Sep 17 00:00:00 2001 From: obdev Date: Tue, 20 Aug 2024 08:53:59 +0000 Subject: [PATCH 134/249] [CP] [to #2024081300104107288] fix bulk collect with null composite value --- src/sql/ob_spi.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/sql/ob_spi.cpp b/src/sql/ob_spi.cpp index 98d37f91e..90165c020 100644 --- a/src/sql/ob_spi.cpp +++ b/src/sql/ob_spi.cpp @@ -7742,6 +7742,15 @@ int ObSPIService::store_result(ObPLExecCtx *ctx, *(table->get_allocator()), into_record_type->get_record_member_type(k)->get_type(), into_record_type->get_record_member_type(k)->get_user_type_id())); + } else if (obj_array.at(idx).is_null() && into_record_type->get_record_member_type(k)->is_composite_type()) { + const ObUserDefinedType *type = NULL; + int64_t ptr = 0; + int64_t init_size = OB_INVALID_SIZE; + OZ (ctx->get_user_type(into_record_type->get_record_member_type(k)->get_user_type_id(), type)); + CK (OB_NOT_NULL(type)); + OZ (type->newx(*(table->get_allocator()), ctx, ptr)); + OZ (type->get_size(PL_TYPE_INIT_SIZE, init_size)); + OX (tmp.set_extend(ptr, type->get_type(), init_size)); } else { OZ (deep_copy_obj(*table->get_allocator(), obj_array.at(idx), tmp)); } @@ -7771,6 +7780,15 @@ int ObSPIService::store_result(ObPLExecCtx *ctx, *(table->get_allocator()), table->get_element_desc().get_pl_type(), table->get_element_desc().get_udt_id())); + } else if (current_obj.is_null() && !table->get_element_desc().is_obj_type()) { + const ObUserDefinedType *type = NULL; + int64_t ptr = 0; + int64_t init_size = OB_INVALID_SIZE; + OZ (ctx->get_user_type(table->get_element_desc().get_udt_id(), type)); + CK (OB_NOT_NULL(type)); + OZ (type->newx(*(table->get_allocator()), ctx, ptr)); + OZ (type->get_size(PL_TYPE_INIT_SIZE, init_size)); + OX (tmp.set_extend(ptr, type->get_type(), init_size)); } else { OZ (deep_copy_obj(*table->get_allocator(), current_obj, tmp)); } From d3ec63e548fb3bdaf2c9385122204c090353033f Mon Sep 17 00:00:00 2001 From: obdev Date: Tue, 20 Aug 2024 12:02:39 +0000 Subject: [PATCH 135/249] =?UTF-8?q?=E5=BD=93=E5=8D=87=E7=BA=A7=E7=9A=84?= =?UTF-8?q?=E6=97=B6=E5=80=99=E7=A6=81=E6=AD=A2transfer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../report/ob_tablet_table_updater.cpp | 2 +- src/rootserver/ob_tenant_balance_service.cpp | 57 +++++++++++-------- .../ob_transfer_partition_command.cpp | 10 +--- src/share/ob_share_util.cpp | 20 ++++--- 4 files changed, 49 insertions(+), 40 deletions(-) diff --git a/src/observer/report/ob_tablet_table_updater.cpp b/src/observer/report/ob_tablet_table_updater.cpp index 6aa110b32..e1f965cc3 100644 --- a/src/observer/report/ob_tablet_table_updater.cpp +++ b/src/observer/report/ob_tablet_table_updater.cpp @@ -831,7 +831,7 @@ int ObTabletTableUpdater::do_batch_update_( } } } - LOG_INFO("REPORT: batch update tablets finished", KR(ret), K(replicas.count()), K(tasks), + LOG_TRACE("REPORT: batch update tablets finished", KR(ret), K(replicas.count()), K(tasks), "cost_time", ObTimeUtility::current_time() - batch_update_start_time); return ret; } diff --git a/src/rootserver/ob_tenant_balance_service.cpp b/src/rootserver/ob_tenant_balance_service.cpp index 49baff512..e3db7f14e 100755 --- a/src/rootserver/ob_tenant_balance_service.cpp +++ b/src/rootserver/ob_tenant_balance_service.cpp @@ -305,14 +305,25 @@ int ObTenantBalanceService::is_ls_balance_finished(const uint64_t &tenant_id, bo LOG_WARN("GCTX.sql_proxy_ is null", KR(ret), KP(GCTX.sql_proxy_)); } else if (ObAllTenantInfoProxy::is_primary_tenant(GCTX.sql_proxy_, tenant_id, is_primary)) { LOG_WARN("fail to execute is_primary_tenant", KR(ret), K(tenant_id)); - } else if (is_primary && ObShareUtil::is_tenant_enable_transfer(tenant_id)) { - if (OB_FAIL(is_primary_tenant_ls_balance_finished_(tenant_id, is_finished))) { - LOG_WARN("fail to execute is_primary_tenant_ls_balance_finished_", KR(ret), K(tenant_id)); + } else if (is_primary) { + if (!ObShareUtil::is_tenant_enable_rebalance(tenant_id)) { + // enable_rebalance = false + is_finished = true; + } else if (ObShareUtil::is_tenant_enable_transfer(tenant_id)) { + // primary tenant and enable_rebalance = true and enable_transfer = true + if (OB_FAIL(is_primary_tenant_ls_balance_finished_(tenant_id, is_finished))) { + LOG_WARN("fail to execute is_primary_tenant_ls_balance_finished_", KR(ret), K(tenant_id)); + } + } else { + // primary tenant and enable_rebalance = true and enable_transfer = false + if (OB_FAIL(is_standby_tenant_ls_balance_finished_(tenant_id, is_finished))) { + LOG_WARN("fail to execute is_standby_tenant_ls_balance_finished_", KR(ret), K(tenant_id), K(is_primary)); + } } } else { - // standby & restore & primary tenant and enable_transfer=false + // standby & restore if (OB_FAIL(is_standby_tenant_ls_balance_finished_(tenant_id, is_finished))) { - LOG_WARN("fail to execute is_standby_tenant_ls_balance_finished_", KR(ret), K(tenant_id)); + LOG_WARN("fail to execute is_standby_tenant_ls_balance_finished_", KR(ret), K(tenant_id), K(is_primary)); } } LOG_TRACE("check whether the tenant has balanced ls", K(ret), K(tenant_id), K(is_primary), K(is_finished)); @@ -660,25 +671,27 @@ int ObTenantBalanceService::check_ls_job_need_cancel_(const share::ObBalanceJob ret = OB_INVALID_ARGUMENT; LOG_WARN("job is invalid", KR(ret), K(job)); } else if (job.get_job_type().is_transfer_partition()) { - //手动transfer partition任务只需要看enable_transfer即可 - omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id_)); - if (OB_UNLIKELY(!tenant_config.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant config is invalid", K(tenant_id_)); - } else if (!tenant_config->enable_transfer) { + //手动transfer partition任务只需要看 enable_transfer 和 没有在升级状态中 即可 + if (!ObShareUtil::is_tenant_enable_transfer(tenant_id_)) { need_cancel = true; - if (OB_TMP_FAIL(comment.assign("Canceled due to tenant transfer being disabled"))) { + if (OB_TMP_FAIL(comment.assign("Canceled due to tenant transfer being disabled or tenant being in upgrade mode"))) { LOG_WARN("failed to assign fmt", KR(tmp_ret), K(job)); } - ISTAT("tenant transfer is disabled, need cancel current job", K(job), K(comment)); + ISTAT("tenant transfer is disabled or tenant is in upgrade mode; need cancel current job", K(job), K(comment)); } - } else if (!ObShareUtil::is_tenant_enable_transfer(tenant_id_)) { + } else if (!ObShareUtil::is_tenant_enable_rebalance(tenant_id_)) { need_cancel = true; - if (OB_TMP_FAIL(comment.assign_fmt("Canceled due to tenant balance or transfer being disabled"))) { + if (OB_TMP_FAIL(comment.assign_fmt("Canceled due to tenant balance being disabled"))) { LOG_WARN("failed to assign fmt", KR(tmp_ret), K(job)); } - ISTAT("tenant balance or transfer is disabled, need cancel current job", K(job), K(comment), - "enable_balance", ObShareUtil::is_tenant_enable_transfer(tenant_id_), + ISTAT("tenant balance is disabled; need cancel current job", K(job), K(comment), + "enable_rebalance", ObShareUtil::is_tenant_enable_rebalance(tenant_id_)); + } else if (!ObShareUtil::is_tenant_enable_transfer(tenant_id_)) { + need_cancel = true; + if (OB_TMP_FAIL(comment.assign_fmt("Canceled due to tenant transfer being disabled or tenant being in upgrade mode"))) { + LOG_WARN("failed to assign fmt", KR(tmp_ret), K(job)); + } + ISTAT("tenant transfer is disabled or tenant is in upgrade mode; need cancel current job", K(job), K(comment), "enable_transfer", ObShareUtil::is_tenant_enable_transfer(tenant_id_)); } else if (job.get_primary_zone_num() != primary_zone_num_) { need_cancel = true; @@ -975,7 +988,6 @@ int ObTenantBalanceService::transfer_partition_(int64_t &job_cnt) int ret = OB_SUCCESS; job_cnt = 0; uint64_t data_version = 0; - omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id_)); if (OB_UNLIKELY(!inited_ || !ATOMIC_LOAD(&loaded_))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(inited_), K(loaded_)); @@ -985,11 +997,8 @@ int ObTenantBalanceService::transfer_partition_(int64_t &job_cnt) //trasnsfer partition 功能提交到了4220分支，所以4220之后的42x分支不用判断兼容性 || (data_version >= DATA_VERSION_4_3_0_0 && DATA_VERSION_4_3_1_0 > data_version)) { LOG_TRACE("no need do transfer partition", K(data_version)); - } else if (OB_UNLIKELY(!tenant_config.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant config is invalid", K(tenant_id_)); - } else if (!tenant_config->enable_transfer) { - LOG_TRACE("can not transfer partition while can not transfer"); + } else if (!ObShareUtil::is_tenant_enable_transfer(tenant_id_)) { + LOG_TRACE("can not transfer partition due to transfer being disabled or tenant being in upgrade mode."); } else { ObTransferPartitionHelper tp_help(tenant_id_, GCTX.sql_proxy_); int64_t unit_num = 0; @@ -1028,7 +1037,7 @@ int ObTenantBalanceService::transfer_partition_(int64_t &job_cnt) } ISTAT("finish transfer partition", KR(ret), K(job_cnt), - "enable transfer", tenant_config->enable_transfer); + "enable transfer", ObShareUtil::is_tenant_enable_transfer(tenant_id_)); return ret; } diff --git a/src/rootserver/ob_transfer_partition_command.cpp b/src/rootserver/ob_transfer_partition_command.cpp index fde6e2969..14b4c6e14 100644 --- a/src/rootserver/ob_transfer_partition_command.cpp +++ b/src/rootserver/ob_transfer_partition_command.cpp @@ -226,19 +226,15 @@ int ObTransferPartitionCommand::execute_transfer_partition_(const ObTransferPart int ObTransferPartitionCommand::check_data_version_and_config_(const uint64_t tenant_id) { int ret = OB_SUCCESS; - omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id)); uint64_t compat_version = 0; bool bret = false; if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("tenant_id is invalid", KR(ret), K(tenant_id)); - } else if (OB_UNLIKELY(!tenant_config.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant config is invalid", KR(ret), K(tenant_id)); - } else if (!(bret = tenant_config->enable_transfer)) { + } else if (!(bret = ObShareUtil::is_tenant_enable_transfer(tenant_id))) { ret = OB_OP_NOT_ALLOW; - LOG_WARN("enable_transfer is off", KR(ret)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "Transfer is disabled, transfer partition is"); + LOG_WARN("transfer is disabled or tenant is in upgrade mode", KR(ret)); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "Transfer is disabled or tenant is in upgrade mode, transfer partition is"); } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) { LOG_WARN("fail to get data version", KR(ret), K(tenant_id)); } else if (compat_version < DATA_VERSION_4_2_1_2 diff --git a/src/share/ob_share_util.cpp b/src/share/ob_share_util.cpp index 79a057f35..8b18a8650 100644 --- a/src/share/ob_share_util.cpp +++ b/src/share/ob_share_util.cpp @@ -432,20 +432,24 @@ bool ObShareUtil::is_tenant_enable_rebalance(const uint64_t tenant_id) bool ObShareUtil::is_tenant_enable_transfer(const uint64_t tenant_id) { bool bret = false; - if (is_valid_tenant_id(tenant_id)) { + if (!is_valid_tenant_id(tenant_id)) { + bret = false; + } else { omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id)); - if (OB_UNLIKELY(!tenant_config.is_valid())) { - LOG_WARN_RET(OB_ERR_UNEXPECTED, "tenant config is invalid", K(tenant_id)); - } else if (!tenant_config->enable_rebalance) { - // if enable_rebalance is disabled, transfer is not allowed + if (OB_UNLIKELY(!tenant_config.is_valid())) { + LOG_WARN_RET(OB_ERR_UNEXPECTED, "tenant config is invalid", K(tenant_id)); + } else if (GCONF.in_upgrade_mode()) { bret = false; - } else { + LOG_TRACE("in upgrade, transfer is not allowed", K(tenant_id), K(bret)); + } else { bret = tenant_config->enable_transfer; - } + LOG_TRACE("show enable_transfer state", K(tenant_id), K(bret), + "enable_transfer", tenant_config->enable_transfer); + } } + return bret; } - int ObShareUtil::check_compat_version_for_tenant( const uint64_t tenant_id, const uint64_t target_data_version, From ab3f21264c0e2b588e2c4d7ace018a68411bca27 Mon Sep 17 00:00:00 2001 From: wjhh2008 Date: Tue, 20 Aug 2024 12:09:16 +0000 Subject: [PATCH 136/249] fix mysqltest --- src/sql/ob_result_set.cpp | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/sql/ob_result_set.cpp b/src/sql/ob_result_set.cpp index 5f96f0c25..9f0bc0039 100644 --- a/src/sql/ob_result_set.cpp +++ b/src/sql/ob_result_set.cpp @@ -1805,9 +1805,26 @@ int ObResultSet::construct_display_field_name(common::ObField &field, } else if (lib::is_oracle_mode()) { pos = pos < MAX_COLUMN_CHAR_LENGTH ? pos : MAX_COLUMN_CHAR_LENGTH; //field.cname_.assign(buf, pos); - if (OB_FAIL(ob_write_string(get_mem_pool(), ObString(pos, buf), field.cname_))) { - LOG_WARN("failed to copy paramed cname", K(ret)); + //if (OB_FAIL(ob_write_string(get_mem_pool(), ObString(pos, buf), field.cname_))) { + // LOG_WARN("failed to copy paramed cname", K(ret)); + //} + ObCollationType connection_collation = CS_TYPE_UTF8MB4_BIN; + ObCollationType nls_collation = CS_TYPE_UTF8MB4_BIN; + ObCollationType result_collation = CS_TYPE_UTF8MB4_BIN; + if (OB_NOT_NULL(get_exec_context().get_my_session())) { + ObCharsetType result_charset = CHARSET_INVALID; + connection_collation = get_exec_context().get_my_session()->get_local_collation_connection(); + nls_collation = get_exec_context().get_my_session()->get_nls_collation(); + get_exec_context().get_my_session()->get_character_set_results(result_charset); + if (ObCharset::is_valid_charset(result_charset)) { + result_collation = ObCharset::get_default_collation(result_charset); + } } + OZ (ObCharset::charset_convert(get_mem_pool(), ObString(pos, buf), connection_collation, + nls_collation, field.cname_, ObCharset::REPLACE_UNKNOWN_CHARACTER_ON_SAME_CHARSET)); + OZ (ObCharset::charset_convert(get_mem_pool(), field.cname_, nls_collation, + result_collation, field.cname_, ObCharset::REPLACE_UNKNOWN_CHARACTER_ON_SAME_CHARSET)); + LOG_DEBUG("check cname", K(field.cname_)); } else { // do nothing } From 6bb31f723906dc3702876e215db7204fdd974a13 Mon Sep 17 00:00:00 2001 From: simonjoylet Date: Tue, 20 Aug 2024 12:28:02 +0000 Subject: [PATCH 137/249] release all ddl kv when direct load mgr not needed --- src/storage/ddl/ob_ddl_merge_task.cpp | 2 ++ src/storage/ls/ob_ls_ddl_log_handler.cpp | 25 ++++++++++++------------ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/storage/ddl/ob_ddl_merge_task.cpp b/src/storage/ddl/ob_ddl_merge_task.cpp index 11cc27190..dc31c3f4f 100644 --- a/src/storage/ddl/ob_ddl_merge_task.cpp +++ b/src/storage/ddl/ob_ddl_merge_task.cpp @@ -473,6 +473,8 @@ int ObDDLTableMergeTask::merge_full_direct_load_ddl_kvs(ObLSHandle &ls_handle, O if (OB_SUCC(ret) && merge_param_.is_commit_ && is_major_exist) { if (OB_FAIL(MTL(ObTabletTableUpdater*)->submit_tablet_update_task(merge_param_.ls_id_, merge_param_.tablet_id_))) { LOG_WARN("fail to submit tablet update task", K(ret), K(tenant_id), K(merge_param_)); + } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->release_ddl_kvs(SCN::max_scn()))) { + LOG_WARN("release all ddl kv failed", K(ret), K(ddl_param)); } else if (OB_FAIL(tenant_direct_load_mgr->remove_tablet_direct_load(ObTabletDirectLoadMgrKey(merge_param_.tablet_id_, true)))) { if (OB_ENTRY_NOT_EXIST == ret) { ret = OB_SUCCESS; diff --git a/src/storage/ls/ob_ls_ddl_log_handler.cpp b/src/storage/ls/ob_ls_ddl_log_handler.cpp index 94120f24c..1ae8f91e4 100644 --- a/src/storage/ls/ob_ls_ddl_log_handler.cpp +++ b/src/storage/ls/ob_ls_ddl_log_handler.cpp @@ -373,6 +373,7 @@ int ObLSDDLLogHandler::flush(SCN &rec_scn) ObDDLKvMgrHandle ddl_kv_mgr_handle; ObTabletDirectLoadMgrHandle direct_load_mgr_hdl; bool is_major_sstable_exist = false; + int tmp_ret = OB_SUCCESS; if (OB_FAIL(tablet_iter.get_next_ddl_kv_mgr(ddl_kv_mgr_handle))) { if (OB_ITER_END == ret) { ret = OB_SUCCESS; @@ -381,21 +382,21 @@ int ObLSDDLLogHandler::flush(SCN &rec_scn) LOG_WARN("failed to get ddl kv mgr", K(ret), K(ddl_kv_mgr_handle)); } } else if (OB_UNLIKELY(!ddl_kv_mgr_handle.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid ddl kv mgr handle", K(ret), K(ddl_kv_mgr_handle)); - } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->check_has_effective_ddl_kv(has_ddl_kv))) { - LOG_WARN("failed to check ddl kv", K(ret)); + tmp_ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid ddl kv mgr handle", K(tmp_ret), K(ddl_kv_mgr_handle)); + } else if (OB_TMP_FAIL(ddl_kv_mgr_handle.get_obj()->check_has_effective_ddl_kv(has_ddl_kv))) { + LOG_WARN("failed to check ddl kv", K(tmp_ret)); } else if (!has_ddl_kv) { - } else if (OB_FAIL(tenant_direct_load_mgr->get_tablet_mgr_and_check_major( + } else if (OB_TMP_FAIL(tenant_direct_load_mgr->get_tablet_mgr_and_check_major( ls_->get_ls_id(), ddl_kv_mgr_handle.get_obj()->get_tablet_id(), true/* is_full_direct_load */, direct_load_mgr_hdl, is_major_sstable_exist))) { - if (OB_ENTRY_NOT_EXIST == ret && is_major_sstable_exist) { - LOG_WARN("major sstable already exist, ddl kv may leak", K(ret), "tablet_id", ddl_kv_mgr_handle.get_obj()->get_tablet_id()); + if (OB_ENTRY_NOT_EXIST == tmp_ret && is_major_sstable_exist) { + LOG_WARN("major sstable already exist, ddl kv may leak", K(tmp_ret), "tablet_id", ddl_kv_mgr_handle.get_obj()->get_tablet_id()); } else { - LOG_WARN("get tablet direct load mgr failed", K(ret), "tablet_id", ddl_kv_mgr_handle.get_obj()->get_tablet_id(), K(is_major_sstable_exist)); + LOG_WARN("get tablet direct load mgr failed", K(tmp_ret), "tablet_id", ddl_kv_mgr_handle.get_obj()->get_tablet_id(), K(is_major_sstable_exist)); } } else { DEBUG_SYNC(BEFORE_DDL_CHECKPOINT); @@ -409,10 +410,10 @@ int ObLSDDLLogHandler::flush(SCN &rec_scn) param.data_format_version_ = direct_load_mgr_hdl.get_full_obj()->get_data_format_version(); param.snapshot_version_ = direct_load_mgr_hdl.get_full_obj()->get_table_key().get_snapshot_version(); LOG_INFO("schedule ddl merge dag", K(param)); - if (OB_FAIL(ObTabletDDLUtil::freeze_ddl_kv(param))) { - LOG_WARN("try to freeze ddl kv failed", K(ret), K(param)); - } else if (OB_FAIL(compaction::ObScheduleDagFunc::schedule_ddl_table_merge_dag(param))) { - LOG_WARN("try schedule ddl merge dag failed when ddl kv is full ", K(ret), K(param)); + if (OB_TMP_FAIL(ObTabletDDLUtil::freeze_ddl_kv(param))) { + LOG_WARN("try to freeze ddl kv failed", K(tmp_ret), K(param)); + } else if (OB_TMP_FAIL(compaction::ObScheduleDagFunc::schedule_ddl_table_merge_dag(param))) { + LOG_WARN("try schedule ddl merge dag failed when ddl kv is full ", K(tmp_ret), K(param)); } } } From 51783965996d3abf46e940f6f975d3317b8e6757 Mon Sep 17 00:00:00 2001 From: dongb0 <708848999@qq.com> Date: Tue, 20 Aug 2024 13:18:59 +0000 Subject: [PATCH 138/249] fix tmp file not insert into flush prio mgr when there are flush tasks not completed --- .../mtlenv/storage/tmp_file/test_tmp_file.cpp | 220 +++++++++++++++++- .../tmp_file/ob_shared_nothing_tmp_file.cpp | 24 +- .../tmp_file/ob_shared_nothing_tmp_file.h | 10 +- .../tmp_file/ob_tmp_file_flush_ctx.cpp | 5 +- src/storage/tmp_file/ob_tmp_file_flush_ctx.h | 3 +- .../tmp_file/ob_tmp_file_flush_manager.cpp | 2 +- 6 files changed, 245 insertions(+), 19 deletions(-) diff --git a/mittest/mtlenv/storage/tmp_file/test_tmp_file.cpp b/mittest/mtlenv/storage/tmp_file/test_tmp_file.cpp index fb9137d0f..267ef8d61 100644 --- a/mittest/mtlenv/storage/tmp_file/test_tmp_file.cpp +++ b/mittest/mtlenv/storage/tmp_file/test_tmp_file.cpp @@ -1008,6 +1008,201 @@ TEST_F(TestTmpFile, test_tmp_file_truncate) LOG_INFO("test_tmp_file_truncate"); } +TEST_F(TestTmpFile, test_truncate_to_flushed_page_id) +{ + int ret = OB_SUCCESS; + const int64_t write_size = 4 * 1024 * 1024 + 12 * 1024; + const int64_t wbp_mem_limit = MTL(ObTenantTmpFileManager *)->page_cache_controller_.write_buffer_pool_.get_memory_limit(); + char *write_buf = new char [write_size]; + for (int64_t i = 0; i < write_size;) { + int64_t random_length = generate_random_int(1024, 8 * 1024); + int64_t random_int = generate_random_int(0, 256); + for (int64_t j = 0; j < random_length && i + j < write_size; ++j) { + write_buf[i + j] = random_int; + } + i += random_length; + } + + int64_t dir = -1; + int64_t fd = -1; + ret = MTL(ObTenantTmpFileManager *)->alloc_dir(dir); + ASSERT_EQ(OB_SUCCESS, ret); + ret = MTL(ObTenantTmpFileManager *)->open(fd, dir); + std::cout << "open temporary file: " << fd << std::endl; + ASSERT_EQ(OB_SUCCESS, ret); + ObTmpFileHandle file_handle; + ret = MTL(ObTenantTmpFileManager *)->get_tmp_file(fd, file_handle); + ASSERT_EQ(OB_SUCCESS, ret); + file_handle.get()->page_idx_cache_.max_bucket_array_capacity_ = SMALL_WBP_IDX_CACHE_MAX_CAPACITY; + + ObTmpFileIOInfo io_info; + io_info.fd_ = fd; + io_info.io_desc_.set_wait_event(2); + io_info.buf_ = write_buf; + io_info.size_ = write_size; + io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; + // Write data + ret = MTL(ObTenantTmpFileManager *)->write(io_info); + ASSERT_EQ(OB_SUCCESS, ret); + + sleep(2); // waits for flushing 4MB data pages, 12KB(2 pages) left + ret = MTL(ObTenantTmpFileManager *)->truncate(fd, 4 * 1024 * 1024); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(4 * 1024 * 1024, file_handle.get()->truncated_offset_); + + int64_t block_index = -1; + ret = MTL(ObTenantTmpFileManager *)->tmp_file_block_manager_.create_tmp_file_block(0/*begin_page_id*/, ObTmpFileGlobal::BLOCK_PAGE_NUMS, block_index); + int64_t flush_sequence = MTL(ObTenantTmpFileManager *)->page_cache_controller_.flush_mgr_.flush_ctx_.get_flush_sequence(); + ObTmpFileDataFlushContext data_flush_ctx; + ObTmpFileFlushTask flush_task; + flush_task.get_flush_infos().push_back(ObTmpFileFlushInfo()); + flush_task.set_block_index(block_index); + ret = flush_task.prealloc_block_buf(); + EXPECT_EQ(OB_SUCCESS, ret); + // copy first page after flush and truncate + int64_t last_info_idx = flush_task.get_flush_infos().count() - 1; + ret = file_handle.get()->generate_data_flush_info(flush_task, flush_task.get_flush_infos().at(last_info_idx), + data_flush_ctx, flush_sequence, false/*flush tail*/); + ASSERT_EQ(OB_SUCCESS, ret); + LOG_INFO("checking flush task", K(flush_task)); + LOG_INFO("checking data flush ctx", K(data_flush_ctx)); + int64_t PAGE_SIZE = 8 * 1024; + EXPECT_EQ(PAGE_SIZE, flush_task.get_data_length()); + + // simulate we have push 1 task to TFFT_INSERT_META_TREE and release truncate lock + // so that another truncate operation can come in. + file_handle.get()->truncate_lock_.unlock(); + + // truncate again + int64_t truncate_offset = 4 * 1024 * 1024 + PAGE_SIZE + PAGE_SIZE / 2; + ret = MTL(ObTenantTmpFileManager *)->truncate(fd, truncate_offset); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(truncate_offset, file_handle.get()->truncated_offset_); + + // append 8KB, 12KB left in wbp + io_info.size_ = PAGE_SIZE; + ret = MTL(ObTenantTmpFileManager *)->write(io_info); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(4 * 1024 * 1024 + 20 * 1024, file_handle.get()->file_size_); + + // copy second page + flush_task.get_flush_infos().push_back(ObTmpFileFlushInfo()); + last_info_idx = flush_task.get_flush_infos().count() - 1; + ret = file_handle.get()->generate_data_flush_info(flush_task, flush_task.get_flush_infos().at(last_info_idx), + data_flush_ctx, flush_sequence, false/*flush tail*/); + LOG_INFO("checking flush task", K(flush_task)); + LOG_INFO("checking data flush ctx", K(data_flush_ctx)); + EXPECT_EQ(OB_SUCCESS, ret); + EXPECT_EQ(PAGE_SIZE * 2, flush_task.get_data_length()); + + // copy third page + flush_task.get_flush_infos().push_back(ObTmpFileFlushInfo()); + last_info_idx = flush_task.get_flush_infos().count() - 1; + ret = file_handle.get()->generate_data_flush_info(flush_task, flush_task.get_flush_infos().at(last_info_idx), + data_flush_ctx, flush_sequence, true/*flush tail*/); + LOG_INFO("checking flush task", K(flush_task)); + LOG_INFO("checking data flush ctx", K(data_flush_ctx)); + EXPECT_EQ(OB_SUCCESS, ret); + EXPECT_EQ(PAGE_SIZE * 3, flush_task.get_data_length()); + + int64_t first_virtual_page_id = flush_task.get_flush_infos().at(0).flush_virtual_page_id_; + int64_t second_virtual_page_id = flush_task.get_flush_infos().at(1).flush_virtual_page_id_; + int64_t third_virtual_page_id = flush_task.get_flush_infos().at(2).flush_virtual_page_id_; + EXPECT_EQ(first_virtual_page_id + 1, second_virtual_page_id); + EXPECT_EQ(second_virtual_page_id + 1, third_virtual_page_id); + + file_handle.reset(); + flush_task.~ObTmpFileFlushTask(); + free(write_buf); + + ret = MTL(ObTenantTmpFileManager *)->remove(fd); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(0, MTL(ObTenantTmpFileManager *)->page_cache_controller_.flush_priority_mgr_.get_file_size()); + ASSERT_EQ(0, MTL(ObTenantTmpFileManager *)->page_cache_controller_.evict_mgr_.get_file_size()); + + LOG_INFO("test_truncate_offset_and_flushed_page_id_defensive_check"); +} + +TEST_F(TestTmpFile, test_write_last_page_during_flush) +{ + int ret = OB_SUCCESS; + const int64_t write_size = 64 * 1024 + 100; + char *write_buf = new char [write_size]; + for (int64_t i = 0; i < write_size;) { + int64_t random_length = generate_random_int(1024, 8 * 1024); + int64_t random_int = generate_random_int(0, 256); + for (int64_t j = 0; j < random_length && i + j < write_size; ++j) { + write_buf[i + j] = random_int; + } + i += random_length; + } + + int64_t dir = -1; + int64_t fd = -1; + ret = MTL(ObTenantTmpFileManager *)->alloc_dir(dir); + ASSERT_EQ(OB_SUCCESS, ret); + ret = MTL(ObTenantTmpFileManager *)->open(fd, dir); + std::cout << "open temporary file: " << fd << std::endl; + ASSERT_EQ(OB_SUCCESS, ret); + ObTmpFileHandle file_handle; + ret = MTL(ObTenantTmpFileManager *)->get_tmp_file(fd, file_handle); + ASSERT_EQ(OB_SUCCESS, ret); + file_handle.get()->page_idx_cache_.max_bucket_array_capacity_ = SMALL_WBP_IDX_CACHE_MAX_CAPACITY; + + ObTmpFileIOInfo io_info; + io_info.fd_ = fd; + io_info.io_desc_.set_wait_event(2); + io_info.buf_ = write_buf; + io_info.size_ = write_size; + io_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; + + ret = MTL(ObTenantTmpFileManager *)->write(io_info); + ASSERT_EQ(OB_SUCCESS, ret); + + printf("generate_data_flush_info\n"); + // hard code generate_data_flush_info to flush last page + int64_t block_index = -1; + ret = MTL(ObTenantTmpFileManager *)->tmp_file_block_manager_.create_tmp_file_block(0/*begin_page_id*/, ObTmpFileGlobal::BLOCK_PAGE_NUMS, block_index); + ASSERT_EQ(OB_SUCCESS, ret); + int64_t flush_sequence = MTL(ObTenantTmpFileManager *)->page_cache_controller_.flush_mgr_.flush_ctx_.get_flush_sequence(); + ObTmpFileDataFlushContext data_flush_ctx; + ObTmpFileFlushTask flush_task; + flush_task.get_flush_infos().push_back(ObTmpFileFlushInfo()); + flush_task.set_block_index(block_index); + ret = flush_task.prealloc_block_buf(); + EXPECT_EQ(OB_SUCCESS, ret); + // copy first page after flush and truncate + int64_t last_info_idx = flush_task.get_flush_infos().count() - 1; + ret = file_handle.get()->generate_data_flush_info(flush_task, flush_task.get_flush_infos().at(last_info_idx), + data_flush_ctx, flush_sequence, true/*flush tail*/); + ASSERT_EQ(OB_SUCCESS, ret); + + // insert_meta_tree + ret = file_handle.get()->insert_meta_tree_item(flush_task.get_flush_infos().at(0), block_index); + ASSERT_EQ(OB_SUCCESS, ret); + + // write before IO complete + ret = MTL(ObTenantTmpFileManager *)->write(io_info); + ASSERT_EQ(OB_SUCCESS, ret); + + // assume io complete, update file meta + bool reset_ctx = false; + ret = file_handle.get()->update_meta_after_flush(flush_task.get_flush_infos().at(0).batch_flush_idx_, false/*is_meta*/, reset_ctx); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(8, file_handle.get()->flushed_data_page_num_); + ASSERT_EQ(0, file_handle.get()->write_back_data_page_num_); + + flush_task.~ObTmpFileFlushTask(); + file_handle.reset(); + ret = MTL(ObTenantTmpFileManager *)->remove(fd); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(0, MTL(ObTenantTmpFileManager *)->page_cache_controller_.flush_priority_mgr_.get_file_size()); + ASSERT_EQ(0, MTL(ObTenantTmpFileManager *)->page_cache_controller_.evict_mgr_.get_file_size()); + + delete [] write_buf; + LOG_INFO("test_write_last_page_during_flush"); +} + // generate 750MB random data. // this test will trigger flush and evict logic for both data and meta pages. void test_big_file(const int64_t write_size, const int64_t wbp_mem_limit, ObTmpFileIOInfo io_info) @@ -1380,7 +1575,7 @@ TEST_F(TestTmpFile, test_multi_file_multi_thread_read_write_with_block_cache) TEST_F(TestTmpFile, test_more_files_more_threads_read_write) { int ret = OB_SUCCESS; - const int64_t read_thread_cnt = 1; + const int64_t read_thread_cnt = 2; const int64_t file_cnt = 128; const int64_t batch_size = 3 * 1024 * 1024; const int64_t batch_num = 2; // total 128 * 3MB * 2 = 768MB @@ -1400,6 +1595,29 @@ TEST_F(TestTmpFile, test_more_files_more_threads_read_write) STORAGE_LOG(INFO, "io time", K(io_time)); } +TEST_F(TestTmpFile, test_multiple_small_files) +{ + int ret = OB_SUCCESS; + const int64_t read_thread_cnt = 2; + const int64_t file_cnt = 256; + const int64_t batch_size = 10 * 1024; // 10KB + const int64_t batch_num = 3; + const bool disable_block_cache = true; + TestMultiTmpFileStress test(MTL_CTX()); + int64_t dir = -1; + ret = MTL(ObTenantTmpFileManager *)->alloc_dir(dir); + ASSERT_EQ(OB_SUCCESS, ret); + ret = test.init(file_cnt, dir, read_thread_cnt, batch_size, batch_num, disable_block_cache); + ASSERT_EQ(OB_SUCCESS, ret); + int64_t io_time = ObTimeUtility::current_time(); + test.start(); + test.wait(); + io_time = ObTimeUtility::current_time() - io_time; + + STORAGE_LOG(INFO, "test_multiple_small_files"); + STORAGE_LOG(INFO, "io time", K(io_time)); +} + TEST_F(TestTmpFile, test_big_file) { const int64_t write_size = 750 * 1024 * 1024; // write 750MB data diff --git a/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp b/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp index 76edc77ee..aed11d636 100644 --- a/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp +++ b/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp @@ -900,9 +900,7 @@ int ObSharedNothingTmpFile::aio_write(ObTmpFileIOCtx &io_ctx) if (OB_FAIL(inner_write_(io_ctx))) { if (OB_ALLOCATE_TMP_FILE_PAGE_FAILED == ret) { ret = OB_SUCCESS; - if (TC_REACH_COUNT_INTERVAL(10)) { - LOG_INFO("alloc mem failed, try to evict pages", K(fd_), K(file_size_), K(io_ctx)); - } + LOG_INFO("alloc mem failed, try to evict pages", K(fd_), K(file_size_), K(io_ctx), KPC(this)); if (OB_FAIL(page_cache_controller_->invoke_swap_and_wait( MIN(io_ctx.get_todo_size(), ObTmpFileGlobal::TMP_FILE_WRITE_BATCH_PAGE_NUM * ObTmpFileGlobal::PAGE_SIZE), io_ctx.get_io_timeout_ms()))) { @@ -1998,12 +1996,20 @@ int ObSharedNothingTmpFile::update_meta_after_flush(const int64_t info_idx, cons if (FAILEDx(inner_flush_ctx_.update_finished_continuous_flush_info_num(is_meta, end_pos))) { LOG_WARN("fail to update finished continuous flush info num", KR(ret), K(start_pos), K(end_pos), KPC(this)); - } else if (inner_flush_ctx_.is_all_finished()) { - if (OB_ISNULL(data_flush_node_.get_next()) && OB_FAIL(reinsert_flush_node_(false /*is_meta*/))) { - LOG_ERROR("fail to reinsert data flush node", KR(ret), K(is_meta), K(inner_flush_ctx_), KPC(this)); - } else if (OB_ISNULL(meta_flush_node_.get_next()) && OB_FAIL(reinsert_flush_node_(true /*is_meta*/))) { - LOG_ERROR("fail to reinsert meta flush node", KR(ret), K(is_meta), K(inner_flush_ctx_), KPC(this)); - } else { + } else { + int tmp_ret = OB_SUCCESS; + if (inner_flush_ctx_.is_data_finished()) { + if (OB_ISNULL(data_flush_node_.get_next()) && OB_TMP_FAIL(reinsert_flush_node_(false /*is_meta*/))) { + LOG_ERROR("fail to reinsert data flush node", KR(tmp_ret), K(is_meta), K(inner_flush_ctx_), KPC(this)); + } + } + if (inner_flush_ctx_.is_meta_finished()) { + if (OB_ISNULL(meta_flush_node_.get_next()) && OB_TMP_FAIL(reinsert_flush_node_(true /*is_meta*/))) { + LOG_ERROR("fail to reinsert meta flush node", KR(tmp_ret), K(is_meta), K(inner_flush_ctx_), KPC(this)); + } + } + + if (inner_flush_ctx_.is_all_finished()) { inner_flush_ctx_.reset(); } } diff --git a/src/storage/tmp_file/ob_shared_nothing_tmp_file.h b/src/storage/tmp_file/ob_shared_nothing_tmp_file.h index 03d05e1b9..441295dbb 100644 --- a/src/storage/tmp_file/ob_shared_nothing_tmp_file.h +++ b/src/storage/tmp_file/ob_shared_nothing_tmp_file.h @@ -85,10 +85,14 @@ public: } void reset(); int update_finished_continuous_flush_info_num(const bool is_meta, const int64_t end_pos); - bool is_all_finished() const + bool is_all_finished() const { return is_data_finished() && is_meta_finished(); } + bool is_data_finished() const { - return data_flush_infos_.size() == data_finished_continuous_flush_info_num_ && - meta_flush_infos_.size() == meta_finished_continuous_flush_info_num_; + return data_flush_infos_.size() == data_finished_continuous_flush_info_num_; + } + bool is_meta_finished() const + { + return meta_flush_infos_.size() == meta_finished_continuous_flush_info_num_; } bool is_flushing() const { diff --git a/src/storage/tmp_file/ob_tmp_file_flush_ctx.cpp b/src/storage/tmp_file/ob_tmp_file_flush_ctx.cpp index b2c759e3c..4aa258277 100644 --- a/src/storage/tmp_file/ob_tmp_file_flush_ctx.cpp +++ b/src/storage/tmp_file/ob_tmp_file_flush_ctx.cpp @@ -218,7 +218,7 @@ void ObTmpFileBatchFlushContext::record_flush_task(const int64_t data_length) // -------------- ObTmpFileFlushTask --------------- // -ObTmpFileFlushTask::ObTmpFileFlushTask(ObIAllocator &task_allocator) +ObTmpFileFlushTask::ObTmpFileFlushTask() : inst_handle_(), kvpair_(nullptr), block_handle_(), @@ -232,8 +232,7 @@ ObTmpFileFlushTask::ObTmpFileFlushTask(ObIAllocator &task_allocator) task_state_(ObTmpFileFlushTaskState::TFFT_INITED), tmp_file_block_handle_(), handle_(), - flush_infos_(), - task_allocator_(task_allocator) + flush_infos_() { flush_infos_.set_attr(ObMemAttr(MTL_ID(), "TFFlushInfos")); } diff --git a/src/storage/tmp_file/ob_tmp_file_flush_ctx.h b/src/storage/tmp_file/ob_tmp_file_flush_ctx.h index ad7a8c8ca..4c9726413 100644 --- a/src/storage/tmp_file/ob_tmp_file_flush_ctx.h +++ b/src/storage/tmp_file/ob_tmp_file_flush_ctx.h @@ -227,7 +227,7 @@ public: struct ObTmpFileFlushTask : public common::ObSpLinkQueue::Link { public: - ObTmpFileFlushTask(ObIAllocator &task_allocator); + ObTmpFileFlushTask(); ~ObTmpFileFlushTask() { destroy(); } enum ObTmpFileFlushTaskState { @@ -297,7 +297,6 @@ private: ObTmpFileBlockHandle tmp_file_block_handle_;// hold a reference to the corresponding tmp file block to prevent it from being released blocksstable::ObMacroBlockHandle handle_; ObArray flush_infos_; // multi file flush into one block if size > 0 - ObIAllocator &task_allocator_; // ref to ObTmpFilePageCacheController::task_allocator_, used to free data_buf_ }; } // end namespace tmp_file diff --git a/src/storage/tmp_file/ob_tmp_file_flush_manager.cpp b/src/storage/tmp_file/ob_tmp_file_flush_manager.cpp index d98eecf5f..2321df3ea 100644 --- a/src/storage/tmp_file/ob_tmp_file_flush_manager.cpp +++ b/src/storage/tmp_file/ob_tmp_file_flush_manager.cpp @@ -65,7 +65,7 @@ int ObTmpFileFlushManager::alloc_flush_task(ObTmpFileFlushTask *&flush_task) ret = OB_ALLOCATE_MEMORY_FAILED; STORAGE_LOG(WARN, "fail to allocate memory for flush callback", KR(ret)); } else { - flush_task = new (task_buf) ObTmpFileFlushTask(task_allocator_); + flush_task = new (task_buf) ObTmpFileFlushTask(); } return ret; } From ed07fd263ff74fbf6a28f181b2299480bf9ac0ec Mon Sep 17 00:00:00 2001 From: JiahuaChen Date: Tue, 20 Aug 2024 13:49:30 +0000 Subject: [PATCH 139/249] Placeholder for ObSSTableBasicMeta compat --- src/storage/blocksstable/ob_sstable_meta.cpp | 3 ++- src/storage/blocksstable/ob_sstable_meta.h | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/storage/blocksstable/ob_sstable_meta.cpp b/src/storage/blocksstable/ob_sstable_meta.cpp index 72e0874f6..90fac2193 100644 --- a/src/storage/blocksstable/ob_sstable_meta.cpp +++ b/src/storage/blocksstable/ob_sstable_meta.cpp @@ -64,7 +64,8 @@ ObSSTableBasicMeta::ObSSTableBasicMeta() master_key_id_(0), sstable_logic_seq_(0), latest_row_store_type_(ObRowStoreType::MAX_ROW_STORE), - table_flag_() + table_flag_(), + root_macro_seq_(0) { MEMSET(encrypt_key_, 0, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); } diff --git a/src/storage/blocksstable/ob_sstable_meta.h b/src/storage/blocksstable/ob_sstable_meta.h index 8f48fcd77..7f02da5c7 100644 --- a/src/storage/blocksstable/ob_sstable_meta.h +++ b/src/storage/blocksstable/ob_sstable_meta.h @@ -188,6 +188,7 @@ public: common::ObRowStoreType latest_row_store_type_; char encrypt_key_[share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH]; storage::ObTableFlag table_flag_; + int64_t root_macro_seq_; // placeholder, will be used after palf branch merged //Add new variable need consider ObSSTableMetaChecker }; From 3243eaf514a465df934c2d456ac9b9e8ff48d95e Mon Sep 17 00:00:00 2001 From: obdev Date: Tue, 20 Aug 2024 14:18:10 +0000 Subject: [PATCH 140/249] Minor, fix type of select_into op serialized value --- src/sql/engine/basic/ob_select_into_op.h | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/sql/engine/basic/ob_select_into_op.h b/src/sql/engine/basic/ob_select_into_op.h index 47c107e81..0fc024a67 100644 --- a/src/sql/engine/basic/ob_select_into_op.h +++ b/src/sql/engine/basic/ob_select_into_op.h @@ -13,11 +13,6 @@ #ifndef SRC_SQL_ENGINE_BASIC_OB_SELECT_INTO_OP_H_ #define SRC_SQL_ENGINE_BASIC_OB_SELECT_INTO_OP_H_ -#include -#include -#include -#include - #include "sql/engine/ob_operator.h" #include "lib/file/ob_file.h" #include "common/storage/ob_io_device.h" @@ -78,7 +73,7 @@ public: external_properties_(alloc), external_partition_(alloc) { - compression_algorithm_ = parquet::Compression::UNCOMPRESSED; + compression_algorithm_ = 0; /* parquet::Compression::UNCOMPRESSED */ per_row_group_size_ = DEFAULT_MAX_FILE_SIZE; cs_type_ = ObCharset::get_system_collation(); } @@ -103,7 +98,7 @@ public: ObExternalFileFormat::StringData external_properties_; ObExternalFileFormat::StringData external_partition_; int64_t per_row_group_size_; - parquet::Compression::type compression_algorithm_; + int64_t compression_algorithm_; static const int64_t DEFAULT_MAX_FILE_SIZE = 256LL * 1024 * 1024; static const int64_t DEFAULT_BUFFER_SIZE = 1LL * 1024 * 1024; }; From a1e7ad72670bab1026dfc00d77505a0625d699f2 Mon Sep 17 00:00:00 2001 From: xianyu-w <707512433@qq.com> Date: Wed, 21 Aug 2024 04:07:45 +0000 Subject: [PATCH 141/249] Add system variable range_index_dive_limit and partition_index_dive_limit --- .../ob_system_variable_init.json | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/share/system_variable/ob_system_variable_init.json b/src/share/system_variable/ob_system_variable_init.json index c1b4713ba..b498cbc38 100644 --- a/src/share/system_variable/ob_system_variable_init.json +++ b/src/share/system_variable/ob_system_variable_init.json @@ -11960,5 +11960,33 @@ "background_cn": "", "ref_url": "", "placeholder": true + }, + "range_index_dive_limit": { + "id": 10740, + "name": "range_index_dive_limit", + "default_value": "10", + "base_value": "10", + "data_type": "int", + "info": "Indicate the limit on the number of ranges when optimizer use storage cardinality estimation", + "flags": "SESSION | GLOBAL", + "publish_version": "425", + "info_cn": "", + "background_cn": "", + "ref_url": "", + "placeholder": true + }, + "partition_index_dive_limit": { + "id": 10741, + "name": "partition_index_dive_limit", + "default_value": "10", + "base_value": "10", + "data_type": "int", + "info": "Indicate the limit on the number of partitions when optimizer use storage cardinality estimation", + "flags": "SESSION | GLOBAL", + "publish_version": "425", + "info_cn": "", + "background_cn": "", + "ref_url": "", + "placeholder": true } } From 760f84b1ee32e58c44f982f831a2c67e81c38ed6 Mon Sep 17 00:00:00 2001 From: Hongqin-Li Date: Wed, 21 Aug 2024 04:14:01 +0000 Subject: [PATCH 142/249] Add rpc pcode placeholder --- deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h | 1 + 1 file changed, 1 insertion(+) diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h index 7ed1aaa1f..e75227d43 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h @@ -877,6 +877,7 @@ PCODE_DEF(OB_REMOTE_WRITE_DDL_INC_COMMIT_LOG, 0x967) PCODE_DEF(OB_BATCH_GET_TABLET_BINDING, 0x96D) //PCODE_DEF(OB_BATCH_UPGRADE_TABLE_SCHEMA, 0x96E) //PCODE_DEF(OB_SPLIT_TABLET_DATA_START_REQUEST, 0x96F) +//PCODE_DEF(OB_BATCH_GET_TABLET_SPLIT, 0x970) // Depedency Detector PCODE_DEF(OB_DETECTOR_LCL_MESSAGE, 0x9F0) From 2266d37762427b5cde1883fbd0219c18b32726de Mon Sep 17 00:00:00 2001 From: zhjc1124 Date: Wed, 21 Aug 2024 04:20:00 +0000 Subject: [PATCH 143/249] remove set_group_id limit --- deps/oblib/src/lib/worker.cpp | 9 --------- deps/oblib/src/lib/worker.h | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/deps/oblib/src/lib/worker.cpp b/deps/oblib/src/lib/worker.cpp index 8b3c089d3..02169ac95 100644 --- a/deps/oblib/src/lib/worker.cpp +++ b/deps/oblib/src/lib/worker.cpp @@ -85,15 +85,6 @@ Worker::Status Worker::check_wait() return ret_status; } -void Worker::set_group_id(int32_t group_id) -{ - if (OBCG_DEFAULT_GROUP_ID == group_id_ || (is_user_group(group_id_) && is_valid_resource_group(group_id))) { - group_id_ = group_id; - } else { - LOG_ERROR_RET(OB_INNER_STAT_ERROR, "group_id is unexpected", K(group_id_), K(group_id)); - } -} - bool Worker::sched_wait() { return true; diff --git a/deps/oblib/src/lib/worker.h b/deps/oblib/src/lib/worker.h index 4dda7ca79..48fc3c395 100644 --- a/deps/oblib/src/lib/worker.h +++ b/deps/oblib/src/lib/worker.h @@ -83,7 +83,7 @@ public: OB_INLINE void set_curr_request_level(const int32_t level) { curr_request_level_ = level; } OB_INLINE int32_t get_curr_request_level() const { return curr_request_level_; } - void set_group_id(int32_t group_id); + OB_INLINE void set_group_id(int32_t group_id) { group_id_ = group_id; } OB_INLINE int32_t get_group_id() const { return group_id_; } OB_INLINE void set_rpc_stat_srv(void *rpc_stat_srv) { rpc_stat_srv_ = rpc_stat_srv; } From 2df6814ad8733e67cca16144a6f6fac7e60b3b89 Mon Sep 17 00:00:00 2001 From: Minionyh Date: Wed, 21 Aug 2024 04:59:57 +0000 Subject: [PATCH 144/249] fix standby read blocked --- .../inner_table/ob_inner_table_schema_def.py | 2 +- src/storage/tx/ob_trans_part_ctx.cpp | 62 +++++--- src/storage/tx/ob_trans_service_v4.cpp | 22 ++- .../tx/test_ob_standby_read_transfer.cpp | 136 ++++++++++++++++-- 4 files changed, 179 insertions(+), 43 deletions(-) diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index 7574608c5..cfe1f457a 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -35103,7 +35103,7 @@ AND # 21558: V$OB_GROUP_IO_STAT # 21559: GV$OB_GROUP_IO_STAT # 21560: DBA_OB_STORAGE_IO_USAGE -# 21561: CDB_OB_STROAGE_IO_USAGE +# 21561: CDB_OB_STORAGE_IO_USAGE def_table_schema( owner = 'chaser.ch', diff --git a/src/storage/tx/ob_trans_part_ctx.cpp b/src/storage/tx/ob_trans_part_ctx.cpp index 1dd558383..3e3c9472e 100644 --- a/src/storage/tx/ob_trans_part_ctx.cpp +++ b/src/storage/tx/ob_trans_part_ctx.cpp @@ -9500,34 +9500,55 @@ int ObPartTransCtx::handle_trans_ask_state(const ObAskStateMsg &req, ObAskStateR if (IS_NOT_INIT) { TRANS_LOG(WARN, "ObPartTransCtx not inited"); ret = OB_NOT_INIT; - } else if (is_root()) { - if (exec_info_.participants_.empty()) { - // if coord not know any participants(before replay commit info log), - // fill self state to resp - ObStateInfo state_info; - state_info.ls_id_ = ls_id_; - state_info.state_ = exec_info_.state_; - state_info.snapshot_version_ = snapshot; - if (ObTxState::INIT == state_info.state_) { + } else { + ObStateInfo state_info; + state_info.ls_id_ = ls_id_; + state_info.state_ = exec_info_.state_; + state_info.snapshot_version_ = snapshot; + resp.state_info_array_.reset(); + + switch(exec_info_.state_) { + case ObTxState::UNKNOWN: + case ObTxState::INIT: + case ObTxState::REDO_COMPLETE: { if (OB_FAIL(get_ls_replica_readable_scn_(state_info.ls_id_, state_info.version_))) { TRANS_LOG(WARN, "get replica readable scn failed", K(ret), K(state_info), K(snapshot)); } else if (OB_FAIL(resp.state_info_array_.push_back(state_info))) { TRANS_LOG(WARN, "push back state info to resp msg failed", K(ret), K(snapshot), KPC(this)); } - } else { - ret = OB_STATE_NOT_MATCH; - TRANS_LOG(ERROR, "coord should not in other state befroe replay commit info log", K(ret), - KPC(this)); + break; } - TRANS_LOG(INFO, "coord not know any participants", K(ret), K(state_info), KPC(this)); - } else { - build_and_post_collect_state_msg_(snapshot); - if (OB_FAIL(resp.state_info_array_.assign(state_info_array_))) { - TRANS_LOG(WARN, "build ObAskStateRespMsg fail", K(ret), K(snapshot), KPC(this)); + case ObTxState::PREPARE: { + if (is_root()) { + build_and_post_collect_state_msg_(snapshot); + if (OB_FAIL(resp.state_info_array_.assign(state_info_array_))) { + TRANS_LOG(WARN, "build ObAskStateRespMsg fail", K(ret), K(snapshot), KPC(this)); + } + } else { + // to find root + build_and_post_ask_state_msg_(snapshot, req.ori_ls_id_, req.ori_addr_); + } + break; } + case ObTxState::PRE_COMMIT: + case ObTxState::COMMIT: + case ObTxState::CLEAR: { + state_info.version_ = ctx_tx_data_.get_commit_version(); + if (OB_FAIL(resp.state_info_array_.push_back(state_info))) { + TRANS_LOG(WARN, "push back state info to resp msg failed", K(ret), K(snapshot), KPC(this)); + } + break; + } + case ObTxState::ABORT: { + state_info.version_.set_min(); + if (OB_FAIL(resp.state_info_array_.push_back(state_info))) { + TRANS_LOG(WARN, "push back state info to resp msg failed", K(ret), K(snapshot), KPC(this)); + } + break; + } + default: + ret = OB_ERR_UNEXPECTED; } - } else { - build_and_post_ask_state_msg_(snapshot, req.ori_ls_id_, req.ori_addr_); } TRANS_LOG(INFO, "handle ask state msg", K(ret), K(req), K(resp)); return ret; @@ -9687,6 +9708,7 @@ int ObPartTransCtx::handle_trans_collect_state(ObCollectStateRespMsg &resp, do { state_info.state_ = exec_info_.state_; switch (state_info.state_) { + case ObTxState::UNKNOWN: case ObTxState::INIT: case ObTxState::REDO_COMPLETE: { if (need_loop) { diff --git a/src/storage/tx/ob_trans_service_v4.cpp b/src/storage/tx/ob_trans_service_v4.cpp index 1672108bc..b74f9d445 100644 --- a/src/storage/tx/ob_trans_service_v4.cpp +++ b/src/storage/tx/ob_trans_service_v4.cpp @@ -3631,18 +3631,16 @@ int ObTransService::handle_trans_ask_state(const ObAskStateMsg &msg, revert_tx_ctx_(ctx); } if (OB_SUCC(ret)) { - if (OB_ISNULL(ctx) || is_root) { - if (!resp.state_info_array_.empty()) { - build_tx_ask_state_resp_(resp, msg); - ObAddr send_to_addr; // for msg compat - if (msg.ori_addr_.is_valid()) { - send_to_addr = msg.ori_addr_; - } else { - send_to_addr = msg.sender_addr_; - } - if (OB_FAIL(rpc_->post_msg(send_to_addr, resp))) { - TRANS_LOG(WARN, "post ask state msg fail", K(ret), K(resp)); - } + if (!resp.state_info_array_.empty()) { + build_tx_ask_state_resp_(resp, msg); + ObAddr send_to_addr; // for msg compat + if (msg.ori_addr_.is_valid()) { + send_to_addr = msg.ori_addr_; + } else { + send_to_addr = msg.sender_addr_; + } + if (OB_FAIL(rpc_->post_msg(send_to_addr, resp))) { + TRANS_LOG(WARN, "post ask state msg fail", K(ret), K(resp)); } } } diff --git a/unittest/storage/tx/test_ob_standby_read_transfer.cpp b/unittest/storage/tx/test_ob_standby_read_transfer.cpp index de9edf1b9..5d0e1502e 100644 --- a/unittest/storage/tx/test_ob_standby_read_transfer.cpp +++ b/unittest/storage/tx/test_ob_standby_read_transfer.cpp @@ -138,18 +138,58 @@ public : return ret; } - int handle_trans_ask_state(const SCN &snapshot, ObAskStateRespMsg &resp) + int handle_trans_ask_state(const SCN &snapshot, ObAskStateRespMsg &resp, const SCN &state_version) { int ret = OB_SUCCESS; - CtxLockGuard guard(lock_); - build_and_post_collect_state_msg(snapshot); - - if (OB_FAIL(resp.state_info_array_.assign(state_info_array_))) { - TRANS_LOG(WARN, "build ObAskStateRespMsg fail", K(ret), K(snapshot), KPC(this)); + ObStateInfo state_info; + state_info.ls_id_ = ls_id_; + state_info.state_ = exec_info_.state_; + state_info.snapshot_version_ = snapshot; + resp.state_info_array_.reset(); + switch(exec_info_.state_) { + case ObTxState::UNKNOWN: + case ObTxState::INIT: { + state_info.version_ = state_version; + if (OB_FAIL(resp.state_info_array_.push_back(state_info))) { + TRANS_LOG(WARN, "push back state info to resp msg failed", K(ret), K(snapshot), KPC(this)); + } + break; + } + case ObTxState::REDO_COMPLETE: + case ObTxState::PREPARE: { + if (is_root()) { + build_and_post_collect_state_msg(snapshot); + if (OB_FAIL(resp.state_info_array_.assign(state_info_array_))) { + TRANS_LOG(WARN, "build ObAskStateRespMsg fail", K(ret), K(snapshot), KPC(this)); + } + } else { + // use coord handle_ask_state_func to test + } + break; + } + case ObTxState::PRE_COMMIT: + case ObTxState::COMMIT: + case ObTxState::CLEAR: { + state_info.version_ = state_version; + if (OB_FAIL(resp.state_info_array_.push_back(state_info))) { + TRANS_LOG(WARN, "push back state info to resp msg failed", K(ret), K(snapshot), KPC(this)); + } + break; + } + case ObTxState::ABORT: + { + state_info.version_.set_min(); + if (OB_FAIL(resp.state_info_array_.push_back(state_info))) { + TRANS_LOG(WARN, "push back state info to resp msg failed", K(ret), K(snapshot), KPC(this)); + } + break; + } + default: + ret = OB_ERR_UNEXPECTED; } - TRANS_LOG(INFO, "handle trans ask state", K(ret), K(resp), KPC(this)); + TRANS_LOG(INFO, "handle trans ask state", K(ret), K(resp), K(state_info), KPC(this)); return ret; } @@ -193,6 +233,7 @@ TEST_F(TestObStandbyReadTransfer, trans_check_for_standby_transfer) ASSERT_EQ(OB_SUCCESS, parts.push_back(ObTxExecPart(part3_ls, part3.epoch_, 0))); coord.set_2pc_participants_(parts); + coord.set_2pc_upstream_(coord_ls); part1.set_2pc_upstream_(coord_ls); part2.set_2pc_upstream_(coord_ls); part3.set_2pc_upstream_(coord_ls); @@ -222,7 +263,7 @@ TEST_F(TestObStandbyReadTransfer, trans_check_for_standby_transfer) transfer_part.exec_info_.prepare_version_.convert_for_tx(50); ASSERT_EQ(OB_ERR_SHARED_LOCK_CONFLICT, part1.check_for_standby(snapshot, can_read, trans_version)); - ASSERT_EQ(OB_SUCCESS, coord.handle_trans_ask_state(snapshot, resp)); + ASSERT_EQ(OB_SUCCESS, coord.handle_trans_ask_state(snapshot, resp, snapshot)); ObCollectStateMsg collect_state_req; ObCollectStateRespMsg collect_state_resp; @@ -246,7 +287,7 @@ TEST_F(TestObStandbyReadTransfer, trans_check_for_standby_transfer) ASSERT_EQ(OB_SUCCESS, coord.handle_trans_collect_state_resp(collect_state_resp)); ASSERT_EQ(5, coord.state_info_array_.count()); - ASSERT_EQ(OB_SUCCESS, coord.handle_trans_ask_state(snapshot, resp)); + ASSERT_EQ(OB_SUCCESS, coord.handle_trans_ask_state(snapshot, resp, snapshot)); ASSERT_EQ(OB_SUCCESS, part1.handle_trans_ask_state_resp(resp)); ASSERT_EQ(OB_ERR_SHARED_LOCK_CONFLICT, part1.check_for_standby(snapshot, can_read, trans_version)); @@ -262,7 +303,7 @@ TEST_F(TestObStandbyReadTransfer, trans_check_for_standby_transfer) ASSERT_EQ(OB_SUCCESS, coord.handle_trans_collect_state_resp(collect_state_resp)); ASSERT_EQ(5, coord.state_info_array_.count()); - ASSERT_EQ(OB_SUCCESS, coord.handle_trans_ask_state(snapshot, resp)); + ASSERT_EQ(OB_SUCCESS, coord.handle_trans_ask_state(snapshot, resp, snapshot)); ASSERT_EQ(OB_SUCCESS, part1.handle_trans_ask_state_resp(resp)); ASSERT_EQ(OB_SUCCESS, part1.check_for_standby(snapshot, can_read, trans_version)); ASSERT_EQ(true, can_read); @@ -270,6 +311,81 @@ TEST_F(TestObStandbyReadTransfer, trans_check_for_standby_transfer) ASSERT_EQ(compute_prepare_version, trans_version); } +TEST_F(TestObStandbyReadTransfer, trans_init_state) +{ + TRANS_LOG(INFO, "called", "func", test_info_->name()); + SCN snapshot; + snapshot.convert_for_tx(100); + SCN compute_prepare_version; + SCN max_decided_scn; + + bool can_read = false; + SCN trans_version = SCN::min_scn(); + ObStateInfo state_info; + ObAskStateRespMsg resp; + + share::ObLSID coord_ls = share::ObLSID(1); + share::ObLSID part1_ls = share::ObLSID(1001); + share::ObLSID part2_ls = share::ObLSID(1002); + share::ObLSID part3_ls = share::ObLSID(1003); + + MockObPartTransCtx coord(coord_ls); + MockObPartTransCtx part1(part1_ls), part2(part2_ls), part3(part3_ls); + + ObTxCommitParts parts; + ASSERT_EQ(OB_SUCCESS, parts.push_back(ObTxExecPart(coord_ls, coord.epoch_, 0))); + ASSERT_EQ(OB_SUCCESS, parts.push_back(ObTxExecPart(part1_ls, part1.epoch_, 0))); + ASSERT_EQ(OB_SUCCESS, parts.push_back(ObTxExecPart(part2_ls, part2.epoch_, 0))); + ASSERT_EQ(OB_SUCCESS, parts.push_back(ObTxExecPart(part3_ls, part3.epoch_, 0))); + coord.set_2pc_participants_(parts); + + part1.set_2pc_upstream_(coord_ls); + part2.set_2pc_upstream_(coord_ls); + part3.set_2pc_upstream_(coord_ls); + + share::ObLSID part_transfer_ls = share::ObLSID(1004); + MockObPartTransCtx transfer_part(part_transfer_ls); + transfer_part.set_2pc_upstream_(part1_ls); + ASSERT_EQ(OB_SUCCESS, transfer_part.exec_info_.transfer_parts_.push_back(ObTxExecPart(part1_ls, -1, 1))); + + ObTxCommitParts transfer_parts; + ASSERT_EQ(OB_SUCCESS, transfer_parts.push_back(ObTxExecPart(part_transfer_ls, -1, 1))); + part1.set_2pc_participants_(transfer_parts); + + TRANS_LOG(INFO, "test1:OB_ERR_SHARED_LOCK_CONFLICT with unknown prepare version"); + state_info.snapshot_version_ = snapshot; + + coord.set_downstream_state(ObTxState::PREPARE); + coord.exec_info_.prepare_version_.convert_for_tx(10); + part1.set_downstream_state(ObTxState::INIT); + // part1.exec_info_.prepare_version_.convert_for_tx(20); + part2.set_downstream_state(ObTxState::PREPARE); + part2.exec_info_.prepare_version_.convert_for_tx(30); + part3.set_downstream_state(ObTxState::PREPARE); + part3.exec_info_.prepare_version_.convert_for_tx(40); + + + transfer_part.set_downstream_state(ObTxState::PREPARE); + transfer_part.exec_info_.prepare_version_.convert_for_tx(50); + transfer_part.handle_trans_ask_state(snapshot, resp, snapshot); + + ASSERT_EQ(OB_ERR_SHARED_LOCK_CONFLICT, transfer_part.check_for_standby(snapshot, can_read, trans_version)); + ASSERT_EQ(false, can_read); + + max_decided_scn.convert_for_tx(50); + part1.handle_trans_ask_state(snapshot, resp, max_decided_scn); + ASSERT_EQ(1, resp.state_info_array_.count()); + transfer_part.handle_trans_ask_state_resp(resp); + ASSERT_EQ(OB_ERR_SHARED_LOCK_CONFLICT, transfer_part.check_for_standby(snapshot, can_read, trans_version)); + ASSERT_EQ(false, can_read); + + max_decided_scn.convert_for_tx(101); + part1.handle_trans_ask_state(snapshot, resp, max_decided_scn); + ASSERT_EQ(1, resp.state_info_array_.count()); + transfer_part.handle_trans_ask_state_resp(resp); + ASSERT_EQ(OB_SUCCESS, transfer_part.check_for_standby(snapshot, can_read, trans_version)); + ASSERT_EQ(false, can_read); +} }//end of unittest }//end of oceanbase From adef77e6212ff9a76e2eca7bb8ef1d52c3842c67 Mon Sep 17 00:00:00 2001 From: lalalafeier Date: Wed, 21 Aug 2024 05:06:53 +0000 Subject: [PATCH 145/249] placeholder about tmp file virtual table|view --- src/share/inner_table/ob_inner_table_schema_def.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index cfe1f457a..e393b6aba 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -14661,6 +14661,7 @@ def_table_schema(**gen_iterate_virtual_table_def( # 12502: __all_virtual_wr_res_mgr_sysstat # 12503: __all_virtual_kv_redis_table # 12504: __all_virtual_function_io_stat +# 12505: __all_virtual_temp_file # 余留位置（此行之前占位） # 本区域占位建议：采用真实表名进行占位 ################################################################################ @@ -15168,6 +15169,7 @@ def_table_schema(**no_direct_access(gen_oracle_mapping_real_virtual_table_def('1 # 15482: __all_virtual_res_mgr_sysstat # 15483: __all_virtual_wr_res_mgr_sysstat # 15484: __all_virtual_function_io_stat +# 15485: __all_virtual_temp_file # 余留位置（此行之前占位） # 本区域定义的Oracle表名比较复杂，一般都采用gen_xxx_table_def()方式定义，占位建议采用基表表名占位 # - 示例：def_table_schema(**no_direct_access(gen_oracle_mapping_virtual_table_def('15009', all_def_keywords['__all_virtual_sql_audit']))) @@ -35995,6 +35997,8 @@ def_table_schema( # 21619: CDB_OB_KV_REDIS_TABLE # 21620: GV$OB_FUNCTION_IO_STAT # 21621: V$OB_FUNCTION_IO_STAT +# 21622: DBA_OB_TEMP_FILES +# 21623: CDB_OB_TEMP_FILES # 余留位置（此行之前占位） # 本区域占位建议：采用真实视图名进行占位 ################################################################################ @@ -64109,6 +64113,7 @@ left join # 28261: DBA_OB_SPM_EVO_RESULT # 28262: GV$OB_FUNCTION_IO_STAT # 28263: V$OB_FUNCTION_IO_STAT +# 28264: DBA_OB_TEMP_FILES # 余留位置（此行之前占位） # 本区域占位建议：采用真实视图名进行占位 ################################################################################ From 5004a66cdb75b291cb9e2ee51d61750e37839174 Mon Sep 17 00:00:00 2001 From: xianyu-w <707512433@qq.com> Date: Wed, 21 Aug 2024 05:13:07 +0000 Subject: [PATCH 146/249] Fix mysqltest --- src/sql/optimizer/ob_access_path_estimation.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sql/optimizer/ob_access_path_estimation.cpp b/src/sql/optimizer/ob_access_path_estimation.cpp index 59362114c..343f65514 100644 --- a/src/sql/optimizer/ob_access_path_estimation.cpp +++ b/src/sql/optimizer/ob_access_path_estimation.cpp @@ -345,7 +345,7 @@ int ObAccessPathEstimation::choose_best_est_method(ObOptimizerContext &ctx, } // check is complex scene - if (OB_SUCC(ret) && !is_simple_scene && !is_complex_scene && (valid_methods | EST_DS_FULL)) { + if (OB_SUCC(ret) && !is_simple_scene && !is_complex_scene && (valid_methods & EST_DS_FULL)) { ObSelEstimatorFactory factory(ctx.get_session_info()->get_effective_tenant_id()); const OptSelectivityCtx* sel_ctx = NULL; if (OB_UNLIKELY(paths.empty()) || From f65f5e5d61a00df4347fd3dee1e2366845863cd4 Mon Sep 17 00:00:00 2001 From: xianyu-w <707512433@qq.com> Date: Wed, 21 Aug 2024 06:53:39 +0000 Subject: [PATCH 147/249] [CP] Fix lob selectivity bug --- src/sql/optimizer/ob_opt_selectivity.cpp | 4 +- src/sql/optimizer/ob_sel_estimator.cpp | 55 ++++++++++++++++-------- src/sql/optimizer/ob_sel_estimator.h | 4 +- 3 files changed, 42 insertions(+), 21 deletions(-) diff --git a/src/sql/optimizer/ob_opt_selectivity.cpp b/src/sql/optimizer/ob_opt_selectivity.cpp index bdf5a9f46..cb7684345 100644 --- a/src/sql/optimizer/ob_opt_selectivity.cpp +++ b/src/sql/optimizer/ob_opt_selectivity.cpp @@ -4473,7 +4473,9 @@ int ObOptSelectivity::calc_expr_min_max(const OptTableMetas &table_metas, // do nothing } else if (can_cmp && OB_FAIL(min_value.compare(max_value, cmp_result))) { - LOG_WARN("failed to compare", K(ret)); + ret = OB_SUCCESS; + min_value.set_min_value(); + max_value.set_max_value(); } else if (!can_cmp || 1 == cmp_result || min_value.is_null() || max_value.is_null()) { min_value.set_min_value(); diff --git a/src/sql/optimizer/ob_sel_estimator.cpp b/src/sql/optimizer/ob_sel_estimator.cpp index e41fd2fe8..732c28d73 100644 --- a/src/sql/optimizer/ob_sel_estimator.cpp +++ b/src/sql/optimizer/ob_sel_estimator.cpp @@ -54,17 +54,19 @@ void SimpleRange::set_false_range() inclusive_end_ = false; } -int SimpleRange::compare_with_end(const SimpleRange &r) const +int SimpleRange::compare_with_end(const SimpleRange &r, int &cmp) const { - int cmp = 0; + int ret = OB_SUCCESS; + cmp = 0; if (end_.is_max_value()) { if (!r.end_.is_max_value()) { cmp = 1; } } else if (r.end_.is_max_value()) { cmp = -1; + } else if (OB_FAIL(end_.compare(r.end_, cmp))) { + LOG_WARN("failed to compare", K(ret)); } else { - cmp = end_.compare(r.end_); if (0 == cmp) { if (inclusive_end_ && !r.inclusive_end_) { cmp = 1; @@ -73,20 +75,22 @@ int SimpleRange::compare_with_end(const SimpleRange &r) const } } } - return cmp; + return ret; } -int SimpleRange::compare_with_start(const SimpleRange &r) const +int SimpleRange::compare_with_start(const SimpleRange &r, int &cmp) const { - int cmp = 0; + int ret = OB_SUCCESS; + cmp = 0; if (start_.is_min_value()) { if (!r.start_.is_min_value()) { cmp = -1; } } else if (r.start_.is_min_value()) { cmp = 1; + } else if (OB_FAIL(start_.compare(r.start_, cmp))) { + LOG_WARN("failed to compare", K(ret)); } else { - cmp = start_.compare(r.start_); if (0 == cmp) { if (inclusive_start_ && !r.inclusive_start_) { cmp = -1; @@ -95,20 +99,27 @@ int SimpleRange::compare_with_start(const SimpleRange &r) const } } } - return cmp; + return ret; } bool SimpleRange::intersect(const SimpleRange &r) { bool bret = false; - if (start_.can_compare(r.start_) && end_.can_compare(r.end_)) { + int ret = OB_SUCCESS; + int cmp_start = 0; + int cmp_end = 0; + if (!start_.can_compare(r.start_) || !end_.can_compare(r.end_)) { + // do nothing + } else if (OB_FAIL(compare_with_start(r, cmp_start))) { + LOG_WARN("failed to compare start", K(ret)); + } else if (OB_FAIL(compare_with_end(r, cmp_end))) { + LOG_WARN("failed to compare end", K(ret)); + } else { bret = true; - int cmp_start = compare_with_start(r); if (cmp_start == -1) { start_ = r.start_; inclusive_start_ = r.inclusive_start_; } - int cmp_end = compare_with_end(r); if (cmp_end == 1) { end_ = r.end_; inclusive_end_ = r.inclusive_end_; @@ -160,6 +171,7 @@ void SimpleRange::set_bound(ObItemType item_type, double bound) bool SimpleRange::is_valid_range() { bool bret = false; + int cmp = 0; if (!start_.can_compare(end_)) { bret = false; } else if (start_.is_null() && end_.is_null()) { @@ -169,8 +181,9 @@ bool SimpleRange::is_valid_range() bret = false; } else if (start_.is_min_value() || end_.is_max_value()) { bret = true; + } else if (OB_SUCCESS != start_.compare(end_, cmp)) { + bret = false; } else { - int cmp = start_.compare(end_); if (-1 == cmp) { bret = true; } else if (1 == cmp) { @@ -189,10 +202,17 @@ bool SimpleRange::is_valid_range() bool SimpleRange::is_superset(const SimpleRange &r) const { bool bret = false; - if (start_.can_compare(r.start_) && end_.can_compare(r.end_)) { - int cmp1 = compare_with_start(r); - int cmp2 = compare_with_end(r); - bret = cmp1 <= 0 && cmp2 >= 0; + int ret = OB_SUCCESS; + int cmp_start = 0; + int cmp_end = 0; + if (!start_.can_compare(r.start_) || !end_.can_compare(r.end_)) { + // do nothing + } else if (OB_FAIL(compare_with_start(r, cmp_start))) { + LOG_WARN("failed to compare start", K(ret)); + } else if (OB_FAIL(compare_with_end(r, cmp_end))) { + LOG_WARN("failed to compare end", K(ret)); + } else { + bret = cmp_start <= 0 && cmp_end >= 0; } return bret; } @@ -3430,8 +3450,7 @@ int ObUniformRangeSelEstimator::get_sel(const OptTableMetas &table_metas, } else if (OB_UNLIKELY(!expr_min.can_compare(expr_max)) || OB_UNLIKELY(!expr_min.can_compare(range_.start_)) || OB_UNLIKELY(!expr_min.can_compare(range_.end_))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("obj type is not consistent", K(expr_min), K(expr_max), KPC(this)); + // ignore } else if (OB_FAIL(ObOptEstObjToScalar::convert_objs_to_scalars(&expr_min, &expr_max, &range_.start_, &range_.end_, &min_scalar, &max_scalar, diff --git a/src/sql/optimizer/ob_sel_estimator.h b/src/sql/optimizer/ob_sel_estimator.h index b394b5618..108767cd6 100644 --- a/src/sql/optimizer/ob_sel_estimator.h +++ b/src/sql/optimizer/ob_sel_estimator.h @@ -32,9 +32,9 @@ public: void set_false_range(); - int compare_with_end(const SimpleRange &r) const; + int compare_with_end(const SimpleRange &r, int &cmp) const; - int compare_with_start(const SimpleRange &r) const; + int compare_with_start(const SimpleRange &r, int &cmp) const; bool intersect(const SimpleRange &r); From d9203e2b94b1551b4bd005a2fdf79e8d5f46c28c Mon Sep 17 00:00:00 2001 From: zhjc1124 Date: Wed, 21 Aug 2024 07:18:52 +0000 Subject: [PATCH 148/249] add check_tg_id_leak case --- deps/oblib/src/lib/thread/thread_define.h | 7 +- .../logservice/env/ob_simple_log_server.cpp | 2 +- .../palf_cluster/env/ob_simple_log_server.cpp | 2 +- .../ob_sql_plan_monitor_node_list.cpp | 2 +- src/share/ob_thread_define.h | 69 ++++++++++--------- 5 files changed, 41 insertions(+), 41 deletions(-) diff --git a/deps/oblib/src/lib/thread/thread_define.h b/deps/oblib/src/lib/thread/thread_define.h index aa3cb4533..e56984a93 100644 --- a/deps/oblib/src/lib/thread/thread_define.h +++ b/deps/oblib/src/lib/thread/thread_define.h @@ -19,12 +19,11 @@ TG_DEF(TEST3, test3, DEDUP_QUEUE, 1, 8, 8, 16L << 20, 16L << 20,8192, "test") TG_DEF(TEST4, test4, THREAD_POOL, 1) TG_DEF(TEST5, test5, ASYNC_TASK_QUEUE, 1, 16) TG_DEF(TEST6, test6, MAP_QUEUE_THREAD, 2) -TG_DEF(TEST7, test7, QUEUE_THREAD, 10, 10) +// TG_DEF(TEST7, test7, QUEUE_THREAD, 10, 10) TG_DEF(TEST8, test8, REENTRANT_THREAD_POOL, 1) // other TG_DEF(MEMORY_DUMP, memDump, THREAD_POOL, 1) -TG_DEF(SchemaRefTask, SchemaRefTask, DEDUP_QUEUE, 1, 1024, 1024, 1L << 30, 512L << 20, common::OB_MALLOC_BIG_BLOCK_SIZE, "SchemaDedupQueu") -TG_DEF(ReqMemEvict, ReqMemEvict, TIMER) -TG_DEF(replica_control, replica_control, THREAD_POOL, 1) +// TG_DEF(SchemaRefTask, SchemaRefTask, DEDUP_QUEUE, 1, 1024, 1024, 1L << 30, 512L << 20, common::OB_MALLOC_BIG_BLOCK_SIZE, "SchemaDedupQueu") +// TG_DEF(replica_control, replica_control, THREAD_POOL, 1) TG_DEF(SYSLOG_COMPRESS, SyslogCompress, THREAD_POOL, 1) #endif diff --git a/mittest/logservice/env/ob_simple_log_server.cpp b/mittest/logservice/env/ob_simple_log_server.cpp index f8295d66f..abf8f0696 100644 --- a/mittest/logservice/env/ob_simple_log_server.cpp +++ b/mittest/logservice/env/ob_simple_log_server.cpp @@ -644,7 +644,7 @@ int ObLogDeliver::init(const common::ObAddr &self, const bool is_bootstrap) // init_all_propocessor_(); if (is_bootstrap && OB_FAIL(ObMittestBlacklist::init(self))) { SERVER_LOG(WARN, "ObMittestBlacklist init failed", K(ret)); - } else if (OB_FAIL(TG_CREATE_TENANT(lib::TGDefIDs::TEST7, tg_id_))) { + } else if (OB_FAIL(TG_CREATE_TENANT(lib::TGDefIDs::COMMON_QUEUE_THREAD, tg_id_))) { SERVER_LOG(WARN, "ObSimpleThreadPool::init failed", K(ret)); } else { is_inited_ = true; diff --git a/mittest/palf_cluster/env/ob_simple_log_server.cpp b/mittest/palf_cluster/env/ob_simple_log_server.cpp index bedc7d6e4..294009ccd 100644 --- a/mittest/palf_cluster/env/ob_simple_log_server.cpp +++ b/mittest/palf_cluster/env/ob_simple_log_server.cpp @@ -417,7 +417,7 @@ int ObLogDeliver::init() } else if (false == blacklist_.created()) { SERVER_LOG(WARN, "blacklist_ created failed"); } else if (OB_FAIL(create_queue_thread(lib::TGDefIDs::LogServerTest, "LogServerQue", req_queue_))) { - } else if (OB_FAIL(TG_CREATE_TENANT(lib::TGDefIDs::TEST7, tg_id_))) { + } else if (OB_FAIL(TG_CREATE_TENANT(lib::TGDefIDs::COMMON_QUEUE_THREAD, tg_id_))) { SERVER_LOG(WARN, "ObSimpleThreadPool::init failed", K(ret)); } else { is_inited_ = true; diff --git a/src/share/diagnosis/ob_sql_plan_monitor_node_list.cpp b/src/share/diagnosis/ob_sql_plan_monitor_node_list.cpp index 27d9363f5..3a4a697ff 100644 --- a/src/share/diagnosis/ob_sql_plan_monitor_node_list.cpp +++ b/src/share/diagnosis/ob_sql_plan_monitor_node_list.cpp @@ -13,7 +13,7 @@ #define USING_LOG_PREFIX SHARE #include "share/diagnosis/ob_sql_plan_monitor_node_list.h" #include "lib/rc/ob_rc.h" -#include "lib/thread/thread_mgr.h" +#include "share/ob_thread_mgr.h" using namespace oceanbase::common; using namespace oceanbase::sql; diff --git a/src/share/ob_thread_define.h b/src/share/ob_thread_define.h index c23a374ac..883883c03 100755 --- a/src/share/ob_thread_define.h +++ b/src/share/ob_thread_define.h @@ -19,18 +19,18 @@ TG_DEF(COMMON_THREAD_POOL, ComTh, THREAD_POOL, 1) TG_DEF(COMMON_QUEUE_THREAD, ComQueueTh, QUEUE_THREAD, 1, 100) TG_DEF(COMMON_TIMER_THREAD, ComTimerTh, TIMER) TG_DEF(Blacklist, Blacklist, THREAD_POOL, 1) -TG_DEF(PartSerMigRetryQt, PartSerMigRetryQt, THREAD_POOL, 1) +// TG_DEF(PartSerMigRetryQt, PartSerMigRetryQt, THREAD_POOL, 1) // TG_DEF(PartSerCb, PartSerCb, QUEUE_THREAD, ThreadCountPair(storage::ObCallbackQueueThread::QUEUE_THREAD_NUM, storage::ObCallbackQueueThread::MINI_MODE_QUEUE_THREAD_NUM), // (!lib::is_mini_mode() ? OB_MAX_PARTITION_NUM_PER_SERVER : OB_MINI_MODE_MAX_PARTITION_NUM_PER_SERVER) * 2) // TG_DEF(PartSerLargeCb, PartSerLargeCb, QUEUE_THREAD, ThreadCountPair(storage::ObCallbackQueueThread::QUEUE_THREAD_NUM, storage::ObCallbackQueueThread::MINI_MODE_QUEUE_THREAD_NUM), // (!lib::is_mini_mode() ? OB_MAX_PARTITION_NUM_PER_SERVER : OB_MINI_MODE_MAX_PARTITION_NUM_PER_SERVER) * 2) // TG_DEF(ReplayEngine, ReplayEngine, QUEUE_THREAD, ThreadCountPair(sysconf(_SC_NPROCESSORS_ONLN), 2), // !lib::is_mini_mode() ? (common::REPLAY_TASK_QUEUE_SIZE + 1) * OB_MAX_PARTITION_NUM_PER_SERVER : (common::REPLAY_TASK_QUEUE_SIZE + 1) * OB_MINI_MODE_MAX_PARTITION_NUM_PER_SERVER) -TG_DEF(TransMigrate, TransMigrate, QUEUE_THREAD, ThreadCountPair(GET_THREAD_NUM_BY_NPROCESSORS(24), 1), 10000) +// TG_DEF(TransMigrate, TransMigrate, QUEUE_THREAD, ThreadCountPair(GET_THREAD_NUM_BY_NPROCESSORS(24), 1), 10000) TG_DEF(StandbyTimestampService, StandbyTimestampService, THREAD_POOL, 1) TG_DEF(TSnapSvc, TSnapSvc, THREAD_POOL, 1) TG_DEF(WeakReadService, WeakRdSrv, THREAD_POOL, 1) -TG_DEF(TransTaskWork, TransTaskWork, QUEUE_THREAD, ThreadCountPair(GET_THREAD_NUM_BY_NPROCESSORS(12), 1), transaction::ObThreadLocalTransCtx::MAX_BIG_TRANS_TASK) +// TG_DEF(TransTaskWork, TransTaskWork, QUEUE_THREAD, ThreadCountPair(GET_THREAD_NUM_BY_NPROCESSORS(12), 1), transaction::ObThreadLocalTransCtx::MAX_BIG_TRANS_TASK) TG_DEF(DDLTaskExecutor3, DDLTaskExecutor3, THREAD_POOL, ThreadCountPair(8, 2)) TG_DEF(TSWorker, TSWorker, QUEUE_THREAD, ThreadCountPair(GET_THREAD_NUM_BY_NPROCESSORS(12), 1), transaction::ObTsWorker::MAX_TASK_NUM) TG_DEF(BRPC, BRPC, THREAD_POOL, ThreadCountPair(obrpc::ObBatchRpc::MAX_THREAD_COUNT, obrpc::ObBatchRpc::MINI_MODE_THREAD_COUNT)) @@ -44,12 +44,12 @@ TG_DEF(DdlBuild, DdlBuild, ASYNC_TASK_QUEUE, ThreadCountPair(16, 1), 4 << 10) TG_DEF(LSService, LSService, REENTRANT_THREAD_POOL, 2) TG_DEF(ObCreateStandbyFromNetActor, ObCreateStandbyFromNetActor, REENTRANT_THREAD_POOL, 1) TG_DEF(SimpleLSService, SimpleLSService, REENTRANT_THREAD_POOL, 1) -TG_DEF(IntermResGC, IntermResGC, TIMER) +// TG_DEF(IntermResGC, IntermResGC, TIMER) TG_DEF(ServerGTimer, ServerGTimer, TIMER) TG_DEF(FreezeTimer, FreezeTimer, TIMER) TG_DEF(SqlMemTimer, SqlMemTimer, TIMER) TG_DEF(ServerTracerTimer, ServerTracerTimer, TIMER) -TG_DEF(RSqlPool, RSqlPool, TIMER) +// TG_DEF(RSqlPool, RSqlPool, TIMER) TG_DEF(KVCacheWash, KVCacheWash, TIMER) TG_DEF(KVCacheRep, KVCacheRep, TIMER) TG_DEF(ObHeartbeat, ObHeartbeat, TIMER) @@ -61,40 +61,40 @@ TG_DEF(MergeLoop, MergeLoop, TIMER) TG_DEF(SSTableGC, SSTableGC, TIMER) TG_DEF(MediumLoop, MediumLoop, TIMER) TG_DEF(CompactionRefresh, CompactionRefresh, TIMER) -TG_DEF(MinorScan, MinorScan, TIMER) -TG_DEF(MajorScan, MajorScan, TIMER) +// TG_DEF(MinorScan, MinorScan, TIMER) +// TG_DEF(MajorScan, MajorScan, TIMER) TG_DEF(WriteCkpt, WriteCkpt, TIMER) -TG_DEF(EXTLogWash, EXTLogWash, TIMER) -TG_DEF(LineCache, LineCache, TIMER) +// TG_DEF(EXTLogWash, EXTLogWash, TIMER) +// TG_DEF(LineCache, LineCache, TIMER) TG_DEF(LocalityReload, LocalityReload, TIMER) -TG_DEF(MemstoreGC, MemstoreGC, TIMER) +// TG_DEF(MemstoreGC, MemstoreGC, TIMER) TG_DEF(DiskUseReport, DiskUseReport, TIMER) -TG_DEF(CLOGReqMinor, CLOGReqMinor, TIMER) -TG_DEF(PGArchiveLog, PGArchiveLog, TIMER) -TG_DEF(CKPTLogRep, CKPTLogRep, TIMER) -TG_DEF(RebuildRetry, RebuildRetry, TIMER) -TG_DEF(TableMgrGC, TableMgrGC, TIMER) -TG_DEF(IndexSche, IndexSche, TIMER) +// TG_DEF(CLOGReqMinor, CLOGReqMinor, TIMER) +// TG_DEF(PGArchiveLog, PGArchiveLog, TIMER) +// TG_DEF(CKPTLogRep, CKPTLogRep, TIMER) +// TG_DEF(RebuildRetry, RebuildRetry, TIMER) +// TG_DEF(TableMgrGC, TableMgrGC, TIMER) +// TG_DEF(IndexSche, IndexSche, TIMER) TG_DEF(FreInfoReload, FreInfoReload, TIMER) -TG_DEF(HAGtsMgr, HAGtsMgr, TIMER) -TG_DEF(HAGtsHB, HAGtsHB, TIMER) -TG_DEF(RebuildTask, RebuildTask, TIMER) -TG_DEF(LogDiskMon, LogDiskMon, TIMER) -TG_DEF(ILOGFlush, ILOGFlush, TIMER) -TG_DEF(ILOGPurge, ILOGPurge, TIMER) -TG_DEF(RLogClrCache, RLogClrCache, TIMER) +// TG_DEF(HAGtsMgr, HAGtsMgr, TIMER) +// TG_DEF(HAGtsHB, HAGtsHB, TIMER) +// TG_DEF(RebuildTask, RebuildTask, TIMER) +// TG_DEF(LogDiskMon, LogDiskMon, TIMER) +// TG_DEF(ILOGFlush, ILOGFlush, TIMER) +// TG_DEF(ILOGPurge, ILOGPurge, TIMER) +// TG_DEF(RLogClrCache, RLogClrCache, TIMER) TG_DEF(TableStatRpt, TableStatRpt, TIMER) -TG_DEF(MacroMetaMgr, MacroMetaMgr, TIMER) -TG_DEF(StoreFileGC, StoreFileGC, TIMER) -TG_DEF(LeaseHB, LeaseHB, TIMER) -TG_DEF(ClusterTimer, ClusterTimer, TIMER) -TG_DEF(MergeTimer, MergeTimer, TIMER) -TG_DEF(CFC, CFC, TIMER) -TG_DEF(CCDF, CCDF, TIMER) +// TG_DEF(MacroMetaMgr, MacroMetaMgr, TIMER) +// TG_DEF(StoreFileGC, StoreFileGC, TIMER) +// TG_DEF(LeaseHB, LeaseHB, TIMER) +// TG_DEF(ClusterTimer, ClusterTimer, TIMER) +// TG_DEF(MergeTimer, MergeTimer, TIMER) +// TG_DEF(CFC, CFC, TIMER) +// TG_DEF(CCDF, CCDF, TIMER) TG_DEF(LogMysqlPool, LogMysqlPool, TIMER) TG_DEF(TblCliSqlPool, TblCliSqlPool, TIMER) -TG_DEF(QueryExecCtxGC, QueryExecCtxGC, THREAD_POOL, 1) -TG_DEF(DtlDfc, DtlDfc, TIMER) +// TG_DEF(QueryExecCtxGC, QueryExecCtxGC, THREAD_POOL, 1) +// TG_DEF(DtlDfc, DtlDfc, TIMER) TG_DEF(LogIOTaskCbThreadPool, LogIOCb, QUEUE_THREAD, ThreadCountPair(palf::LogIOTaskCbThreadPool::THREAD_NUM, palf::LogIOTaskCbThreadPool::MINI_MODE_THREAD_NUM), @@ -125,6 +125,7 @@ TG_DEF(RCService, RCSrv, QUEUE_THREAD, TG_DEF(ApplyService, ApplySrv, QUEUE_THREAD, 1, (common::APPLY_TASK_QUEUE_SIZE + 1) * OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_CAN_BE_SET) TG_DEF(GlobalCtxTimer, GlobalCtxTimer, TIMER) TG_DEF(StorageLogWriter, StorageLogWriter, THREAD_POOL, 1) +TG_DEF(ReqMemEvict, ReqMemEvict, TIMER) TG_DEF(ReplayProcessStat, ReplayProcessStat, TIMER) TG_DEF(ActiveSessHist, ActiveSessHist, TIMER) TG_DEF(CTASCleanUpTimer, CTASCleanUpTimer, TIMER) @@ -159,8 +160,8 @@ TG_DEF(IO_HEALTH, IO_HEALTH, QUEUE_THREAD, 1, 100) TG_DEF(IO_BENCHMARK, IO_BENCHMARK, THREAD_POOL, 1) TG_DEF(TIMEZONE_MGR, TimezoneMgr, TIMER) TG_DEF(MASTER_KEY_MGR, MasterKeyMgr, QUEUE_THREAD, 1, 100) -TG_DEF(SRS_MGR, SrsMgr, TIMER, 128) -TG_DEF(InfoPoolResize, InfoPoolResize, TIMER) +// TG_DEF(SRS_MGR, SrsMgr, TIMER, 128) +// TG_DEF(InfoPoolResize, InfoPoolResize, TIMER) TG_DEF(TenantTransferService, TransferSrv, REENTRANT_THREAD_POOL, ThreadCountPair(4 ,1)) TG_DEF(WR_TIMER_THREAD, WrTimer, TIMER) From b263784bf1e770ad5cbfe444dec97dd6fa258b8d Mon Sep 17 00:00:00 2001 From: suz-yang Date: Wed, 21 Aug 2024 07:24:51 +0000 Subject: [PATCH 149/249] fix direct load build heap table multiple merge task --- .../direct_load/ob_direct_load_merge_ctx.cpp | 55 +++++++++---------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/src/storage/direct_load/ob_direct_load_merge_ctx.cpp b/src/storage/direct_load/ob_direct_load_merge_ctx.cpp index c28eff2fa..288d0eca6 100644 --- a/src/storage/direct_load/ob_direct_load_merge_ctx.cpp +++ b/src/storage/direct_load/ob_direct_load_merge_ctx.cpp @@ -634,38 +634,37 @@ int ObDirectLoadTabletMergeCtx::build_heap_table_multiple_merge_task( merge_task = nullptr; } } - if (OB_SUCC(ret)) { - // for imported data, construct task by multiple heap table - for (int64_t i = 0; OB_SUCC(ret) && !param_.is_fast_heap_table_ && i < multiple_heap_table_array_.count(); ++i) { - ObDirectLoadMultipleHeapTable *heap_table = multiple_heap_table_array_.at(i); - ObDirectLoadPartitionHeapTableMultipleMergeTask *merge_task = nullptr; - int64_t row_count = 0; - ObTabletCacheInterval pk_interval; - if (OB_FAIL(heap_table->get_tablet_row_count(tablet_id_, param_.table_data_desc_, row_count))) { - LOG_WARN("fail to get tablet row count", KR(ret), K(tablet_id_)); - } else if (0 == row_count) { - // ignore - } else if (OB_FAIL(get_autoincrement_value(row_count, pk_interval))) { - LOG_WARN("fail to get autoincrement value", KR(ret), K(row_count)); - } else if (OB_ISNULL(merge_task = OB_NEWx(ObDirectLoadPartitionHeapTableMultipleMergeTask, (&allocator_)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to new ObDirectLoadPartitionHeapTableMultipleMergeTask", KR(ret)); - } else if (OB_FAIL(merge_task->init(ctx_, param_, this, heap_table, pk_interval, parallel_idx++))) { - LOG_WARN("fail to init merge task", KR(ret)); - } else if (OB_FAIL(task_array_.push_back(merge_task))) { - LOG_WARN("fail to push back merge task", KR(ret)); - } - if (OB_FAIL(ret)) { - if (nullptr != merge_task) { - merge_task->~ObDirectLoadPartitionHeapTableMultipleMergeTask(); - allocator_.free(merge_task); - merge_task = nullptr; - } + } + if (OB_SUCC(ret)) { + // for imported data, construct task by multiple heap table + for (int64_t i = 0; OB_SUCC(ret) && !param_.is_fast_heap_table_ && i < multiple_heap_table_array_.count(); ++i) { + ObDirectLoadMultipleHeapTable *heap_table = multiple_heap_table_array_.at(i); + ObDirectLoadPartitionHeapTableMultipleMergeTask *merge_task = nullptr; + int64_t row_count = 0; + ObTabletCacheInterval pk_interval; + if (OB_FAIL(heap_table->get_tablet_row_count(tablet_id_, param_.table_data_desc_, row_count))) { + LOG_WARN("fail to get tablet row count", KR(ret), K(tablet_id_)); + } else if (0 == row_count) { + // ignore + } else if (OB_FAIL(get_autoincrement_value(row_count, pk_interval))) { + LOG_WARN("fail to get autoincrement value", KR(ret), K(row_count)); + } else if (OB_ISNULL(merge_task = OB_NEWx(ObDirectLoadPartitionHeapTableMultipleMergeTask, (&allocator_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObDirectLoadPartitionHeapTableMultipleMergeTask", KR(ret)); + } else if (OB_FAIL(merge_task->init(ctx_, param_, this, heap_table, pk_interval, parallel_idx++))) { + LOG_WARN("fail to init merge task", KR(ret)); + } else if (OB_FAIL(task_array_.push_back(merge_task))) { + LOG_WARN("fail to push back merge task", KR(ret)); + } + if (OB_FAIL(ret)) { + if (nullptr != merge_task) { + merge_task->~ObDirectLoadPartitionHeapTableMultipleMergeTask(); + allocator_.free(merge_task); + merge_task = nullptr; } } } } - return ret; } From c6abadd8cf1d6b287b251a08ccfdc615a96cc506 Mon Sep 17 00:00:00 2001 From: godyangfight Date: Wed, 21 Aug 2024 07:48:24 +0000 Subject: [PATCH 150/249] ObTableFlag split to ObTableBackFlag and ObTableSharedFlag --- .../blocksstable/test_index_dumper.cpp | 4 +- .../storage/blocksstable/test_index_tree.cpp | 76 +++++++-------- .../ob_all_virtual_table_mgr.cpp | 6 +- .../index_block/ob_index_block_builder.cpp | 26 ++--- .../index_block/ob_index_block_builder.h | 4 +- src/storage/blocksstable/ob_sstable_meta.cpp | 29 ++++-- src/storage/blocksstable/ob_sstable_meta.h | 8 +- src/storage/blocksstable/ob_table_flag.cpp | 55 ++++++++--- src/storage/blocksstable/ob_table_flag.h | 97 +++++++++++++------ .../ob_physical_copy_task.cpp | 16 +-- .../ob_storage_ha_reader.cpp | 2 +- .../ob_tablet_group_restore.cpp | 10 +- src/storage/ls/ob_ls_tablet_service.cpp | 3 +- src/storage/ob_i_table.cpp | 2 +- src/storage/tablet/ob_tablet.cpp | 2 +- .../tablet/ob_tablet_create_sstable_param.cpp | 22 +++-- .../tablet/ob_tablet_create_sstable_param.h | 6 +- src/storage/tablet/ob_tablet_table_store.cpp | 16 +-- 18 files changed, 236 insertions(+), 148 deletions(-) diff --git a/mittest/mtlenv/storage/blocksstable/test_index_dumper.cpp b/mittest/mtlenv/storage/blocksstable/test_index_dumper.cpp index 0c7138672..cdba2cba4 100644 --- a/mittest/mtlenv/storage/blocksstable/test_index_dumper.cpp +++ b/mittest/mtlenv/storage/blocksstable/test_index_dumper.cpp @@ -383,8 +383,8 @@ TEST_F(TestIndexDumper, get_from_disk) } } // close - ObTableFlag table_flag; - table_flag.clear(); + ObTableBackupFlag table_backup_flag; + table_backup_flag.clear(); ASSERT_EQ(OB_SUCCESS, macro_meta_dumper.close(meta_block_info)); STORAGE_LOG(INFO, "test print meta block info", K(meta_block_info)); ASSERT_FALSE(meta_block_info.in_mem_); diff --git a/mittest/mtlenv/storage/blocksstable/test_index_tree.cpp b/mittest/mtlenv/storage/blocksstable/test_index_tree.cpp index d4d3ef785..2f7e5632e 100644 --- a/mittest/mtlenv/storage/blocksstable/test_index_tree.cpp +++ b/mittest/mtlenv/storage/blocksstable/test_index_tree.cpp @@ -668,8 +668,8 @@ void TestIndexTree::mock_compaction(const int64_t test_row_num, ASSERT_EQ(sec_blk.original_size_, sec_blk.data_zsize_); ASSERT_EQ(OB_SUCCESS, data_writer.close()); ASSERT_EQ(OB_SUCCESS, sstable_builder->close(res)); - ASSERT_EQ(false, res.table_flag_.has_backup()); - ASSERT_EQ(true, res.table_flag_.has_local()); + ASSERT_EQ(false, res.table_backup_flag_.has_backup()); + ASSERT_EQ(true, res.table_backup_flag_.has_local()); ObSSTableMergeRes tmp_res; ASSERT_EQ(OB_ERR_UNEXPECTED, data_writer.close()); // not re-entrant @@ -677,8 +677,8 @@ void TestIndexTree::mock_compaction(const int64_t test_row_num, ASSERT_EQ(tmp_res.root_desc_.buf_, res.root_desc_.buf_); ASSERT_EQ(tmp_res.data_root_desc_.buf_, res.data_root_desc_.buf_); ASSERT_EQ(tmp_res.data_blocks_cnt_, res.data_blocks_cnt_); - ASSERT_EQ(false, tmp_res.table_flag_.has_backup()); - ASSERT_EQ(true, tmp_res.table_flag_.has_local()); + ASSERT_EQ(false, tmp_res.table_backup_flag_.has_backup()); + ASSERT_EQ(true, tmp_res.table_backup_flag_.has_local()); OK(data_write_ctxs.push_back(sstable_builder->roots_[0]->data_write_ctx_)); roots = &(sstable_builder->roots_); @@ -802,8 +802,8 @@ void TestIndexTree::mock_cg_compaction(const int64_t test_row_num, ASSERT_EQ(sec_blk.original_size_, sec_blk.data_zsize_); ASSERT_EQ(OB_SUCCESS, data_writer.close()); ASSERT_EQ(OB_SUCCESS, sstable_builder->close(res)); - ASSERT_EQ(false, res.table_flag_.has_backup()); - ASSERT_EQ(true, res.table_flag_.has_local()); + ASSERT_EQ(false, res.table_backup_flag_.has_backup()); + ASSERT_EQ(true, res.table_backup_flag_.has_local()); ObSSTableMergeRes tmp_res; ASSERT_EQ(OB_ERR_UNEXPECTED, data_writer.close()); // not re-entrant @@ -811,8 +811,8 @@ void TestIndexTree::mock_cg_compaction(const int64_t test_row_num, ASSERT_EQ(tmp_res.root_desc_.buf_, res.root_desc_.buf_); ASSERT_EQ(tmp_res.data_root_desc_.buf_, res.data_root_desc_.buf_); ASSERT_EQ(tmp_res.data_blocks_cnt_, res.data_blocks_cnt_); - ASSERT_EQ(false, tmp_res.table_flag_.has_backup()); - ASSERT_EQ(true, tmp_res.table_flag_.has_local()); + ASSERT_EQ(false, tmp_res.table_backup_flag_.has_backup()); + ASSERT_EQ(true, tmp_res.table_backup_flag_.has_local()); OK(data_write_ctxs.push_back(sstable_builder->roots_[0]->data_write_ctx_)); roots = &(sstable_builder->roots_); @@ -991,8 +991,8 @@ TEST_F(TestIndexTree, test_empty_index_tree) ASSERT_EQ(0, sstable_builder.roots_.count()); ASSERT_EQ(OB_SUCCESS, ret); ASSERT_TRUE(res.root_desc_.is_empty()); - ASSERT_EQ(false, res.table_flag_.has_backup()); - ASSERT_EQ(false, res.table_flag_.has_local()); + ASSERT_EQ(false, res.table_backup_flag_.has_backup()); + ASSERT_EQ(false, res.table_backup_flag_.has_local()); // test rebuild macro blocks ASSERT_EQ(OB_SUCCESS, data_writer.open(data_desc.get_desc(), data_seq)); @@ -1006,8 +1006,8 @@ TEST_F(TestIndexTree, test_empty_index_tree) ret = sstable_builder.close(res); ASSERT_EQ(OB_SUCCESS, ret); ASSERT_TRUE(res.root_desc_.is_empty()); - ASSERT_EQ(false, res.table_flag_.has_backup()); - ASSERT_EQ(false, res.table_flag_.has_local()); + ASSERT_EQ(false, res.table_backup_flag_.has_backup()); + ASSERT_EQ(false, res.table_backup_flag_.has_local()); // test easy index tree ObDatumRow row; @@ -1130,8 +1130,8 @@ TEST_F(TestIndexTree, test_multi_writers_with_close) res.reset(); sstable_builder.index_block_loader_.reset(); OK(sstable_builder.close(res)); - ASSERT_EQ(false, res.table_flag_.has_backup()); - ASSERT_EQ(true, res.table_flag_.has_local()); + ASSERT_EQ(false, res.table_backup_flag_.has_backup()); + ASSERT_EQ(true, res.table_backup_flag_.has_local()); ObIndexTreeRootBlockDesc &root_desc = res.root_desc_; ASSERT_TRUE(root_desc.is_valid()); @@ -1356,8 +1356,8 @@ TEST_F(TestIndexTree, test_meta_builder_mem_and_disk) ASSERT_EQ(10, sst_builder->roots_[1]->meta_block_info_.get_row_count()); ASSERT_EQ(OB_SUCCESS, sst_builder->close(res)); - ASSERT_EQ(false, res.table_flag_.has_backup()); - ASSERT_EQ(true, res.table_flag_.has_local()); + ASSERT_EQ(false, res.table_backup_flag_.has_backup()); + ASSERT_EQ(true, res.table_backup_flag_.has_local()); ObSSTableMergeRes tmp_res; @@ -1763,8 +1763,8 @@ TEST_F(TestIndexTree, test_reuse_macro_block) for (int64_t i = 0; i < res.data_blocks_cnt_; ++i) { ASSERT_EQ(res.data_block_ids_.at(i), reused_res.data_block_ids_.at(i)); } - ASSERT_EQ(false, res.table_flag_.has_backup()); - ASSERT_EQ(true, res.table_flag_.has_local()); + ASSERT_EQ(false, res.table_backup_flag_.has_backup()); + ASSERT_EQ(true, res.table_backup_flag_.has_local()); sst_builder->~ObSSTableIndexBuilder(); } @@ -1902,8 +1902,8 @@ TEST_F(TestIndexTree, test_rebuilder) OK(data_writer.close()); ObSSTableMergeRes res1; OK(sstable_builder1.close(res1)); - ASSERT_EQ(false, res1.table_flag_.has_backup()); - ASSERT_EQ(true, res1.table_flag_.has_local()); + ASSERT_EQ(false, res1.table_backup_flag_.has_backup()); + ASSERT_EQ(true, res1.table_backup_flag_.has_local()); ObIndexBlockRebuilder rebuilder; OK(rebuilder.init(sstable_builder2, nullptr, 0)); @@ -1934,8 +1934,8 @@ TEST_F(TestIndexTree, test_rebuilder) OK(rebuilder.close()); ObSSTableMergeRes res2; OK(sstable_builder2.close(res2)); - ASSERT_EQ(false, res2.table_flag_.has_backup()); - ASSERT_EQ(true, res2.table_flag_.has_local()); + ASSERT_EQ(false, res2.table_backup_flag_.has_backup()); + ASSERT_EQ(true, res2.table_backup_flag_.has_local()); // compare merge res ASSERT_EQ(res1.root_desc_.height_, res2.root_desc_.height_); ASSERT_EQ(res1.root_desc_.height_, 2); @@ -2128,8 +2128,8 @@ TEST_F(TestIndexTree, test_absolute_offset) ObSSTableMergeRes res2; OK(sstable_builder.close(res2)); ASSERT_GT(res2.root_desc_.height_, 2); - ASSERT_EQ(false, res2.table_flag_.has_backup()); - ASSERT_EQ(true, res2.table_flag_.has_local()); + ASSERT_EQ(false, res2.table_backup_flag_.has_backup()); + ASSERT_EQ(true, res2.table_backup_flag_.has_local()); ObMicroBlockReaderHelper reader_helper; ObIMicroBlockReader *micro_reader; @@ -2227,8 +2227,8 @@ TEST_F(TestIndexTree, test_rebuilder_backup_all_mem) OK(data_writer.close()); ObSSTableMergeRes res1; OK(sstable_builder1.close(res1)); - ASSERT_EQ(false, res1.table_flag_.has_backup()); - ASSERT_EQ(true, res1.table_flag_.has_local()); + ASSERT_EQ(false, res1.table_backup_flag_.has_backup()); + ASSERT_EQ(true, res1.table_backup_flag_.has_local()); ObIndexBlockRebuilder rebuilder; OK(mock_init_backup_rebuilder(rebuilder, sstable_builder2)); @@ -2265,8 +2265,8 @@ TEST_F(TestIndexTree, test_rebuilder_backup_all_mem) ASSERT_EQ(true, rebuilder.index_tree_root_ctx_->index_tree_info_.in_mem()); rebuilder.~ObIndexBlockRebuilder(); OK(sstable_builder2.close(res2)); - ASSERT_EQ(false, res2.table_flag_.has_backup()); - ASSERT_EQ(true, res2.table_flag_.has_local()); + ASSERT_EQ(false, res2.table_backup_flag_.has_backup()); + ASSERT_EQ(true, res2.table_backup_flag_.has_local()); ASSERT_EQ(true, res2.data_root_desc_.is_mem_type()); ASSERT_EQ(true, res2.root_desc_.is_mem_type()); // compare merge res @@ -2336,8 +2336,8 @@ TEST_F(TestIndexTree, test_rebuilder_backup_all_disk) OK(data_writer.close()); ObSSTableMergeRes res1; OK(sstable_builder1.close(res1)); - ASSERT_EQ(false, res1.table_flag_.has_backup()); - ASSERT_EQ(true, res1.table_flag_.has_local()); + ASSERT_EQ(false, res1.table_backup_flag_.has_backup()); + ASSERT_EQ(true, res1.table_backup_flag_.has_local()); ObIndexBlockRebuilder rebuilder; OK(mock_init_backup_rebuilder(rebuilder, sstable_builder2)); @@ -2374,8 +2374,8 @@ TEST_F(TestIndexTree, test_rebuilder_backup_all_disk) ObSSTableMergeRes res2; OK(sstable_builder2.close(res2)); - ASSERT_EQ(false, res2.table_flag_.has_backup()); - ASSERT_EQ(true, res2.table_flag_.has_local()); + ASSERT_EQ(false, res2.table_backup_flag_.has_backup()); + ASSERT_EQ(true, res2.table_backup_flag_.has_local()); // compare merge res ASSERT_EQ(res1.root_desc_.height_, res2.root_desc_.height_); // ASSERT_EQ(res1.root_desc_.height_, 2); @@ -2443,8 +2443,8 @@ TEST_F(TestIndexTree, test_rebuilder_backup_mem_and_disk) OK(data_writer.close()); ObSSTableMergeRes res1; OK(sstable_builder1.close(res1)); - ASSERT_EQ(false, res1.table_flag_.has_backup()); - ASSERT_EQ(true, res1.table_flag_.has_local()); + ASSERT_EQ(false, res1.table_backup_flag_.has_backup()); + ASSERT_EQ(true, res1.table_backup_flag_.has_local()); ObIndexBlockRebuilder rebuilder_mem; OK(mock_init_backup_rebuilder(rebuilder_mem, sstable_builder2)); ObIndexBlockRebuilder rebuilder; @@ -2496,8 +2496,8 @@ TEST_F(TestIndexTree, test_rebuilder_backup_mem_and_disk) rebuilder.~ObIndexBlockRebuilder(); ObSSTableMergeRes res2; OK(sstable_builder2.close(res2)); - ASSERT_EQ(false, res2.table_flag_.has_backup()); - ASSERT_EQ(true, res2.table_flag_.has_local()); + ASSERT_EQ(false, res2.table_backup_flag_.has_backup()); + ASSERT_EQ(true, res2.table_backup_flag_.has_local()); } TEST_F(TestIndexTree, test_cg_compaction_all_mem) @@ -2554,8 +2554,8 @@ TEST_F(TestIndexTree, test_cg_compaction_all_mem) OK(sstable_builder2.close(res2)); - ASSERT_EQ(false, res2.table_flag_.has_backup()); - ASSERT_EQ(true, res2.table_flag_.has_local()); + ASSERT_EQ(false, res2.table_backup_flag_.has_backup()); + ASSERT_EQ(true, res2.table_backup_flag_.has_local()); ASSERT_EQ(true, res2.data_root_desc_.is_mem_type()); ASSERT_EQ(true, res2.root_desc_.is_mem_type()); // compare merge res diff --git a/src/observer/virtual_table/ob_all_virtual_table_mgr.cpp b/src/observer/virtual_table/ob_all_virtual_table_mgr.cpp index 570315e34..95f6cce5a 100644 --- a/src/observer/virtual_table/ob_all_virtual_table_mgr.cpp +++ b/src/observer/virtual_table/ob_all_virtual_table_mgr.cpp @@ -315,7 +315,7 @@ int ObAllVirtualTableMgr::process_curr_tenant(common::ObNewRow *&row) break; } case TABLE_FLAG: { - ObTableFlag table_flag; + ObTableBackupFlag table_backup_flag; if (OB_ISNULL(table)) { ret = OB_ERR_UNEXPECTED; SERVER_LOG(WARN, "table should not be null", K(ret), KP(table)); @@ -324,10 +324,10 @@ int ObAllVirtualTableMgr::process_curr_tenant(common::ObNewRow *&row) if (OB_FAIL(static_cast(table)->get_meta(sst_meta_hdl))) { SERVER_LOG(WARN, "fail to get sstable meta handle", K(ret)); } else { - table_flag = sst_meta_hdl.get_sstable_meta().get_table_flag(); + table_backup_flag = sst_meta_hdl.get_sstable_meta().get_table_backup_flag(); } } - cur_row_.cells_[i].set_int(table_flag.flag_); + cur_row_.cells_[i].set_int(table_backup_flag.flag_); break; } default: diff --git a/src/storage/blocksstable/index_block/ob_index_block_builder.cpp b/src/storage/blocksstable/index_block/ob_index_block_builder.cpp index cbd599af3..351ec733a 100755 --- a/src/storage/blocksstable/index_block/ob_index_block_builder.cpp +++ b/src/storage/blocksstable/index_block/ob_index_block_builder.cpp @@ -170,11 +170,11 @@ ObSSTableMergeRes::ObSSTableMergeRes() master_key_id_(0), nested_offset_(0), nested_size_(0), - table_flag_(), + table_backup_flag_(), root_row_store_type_(ObRowStoreType::MAX_ROW_STORE) { MEMSET(encrypt_key_, 0, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); - table_flag_.clear(); + table_backup_flag_.clear(); if (share::is_reserve_mode()) { ObMemAttr attr(MTL_ID(), "SSTMrgeResArr", ObCtxIds::MERGE_RESERVE_CTX_ID); data_block_ids_.set_attr(attr); @@ -225,7 +225,7 @@ void ObSSTableMergeRes::reset() master_key_id_ = 0; nested_offset_ = 0; nested_size_ = 0; - table_flag_.clear(); + table_backup_flag_.clear(); root_row_store_type_ = ObRowStoreType::MAX_ROW_STORE; } @@ -239,7 +239,7 @@ bool ObSSTableMergeRes::is_valid() const && data_column_cnt_ > 0 && nested_offset_ >= 0 && nested_size_ >= 0 - && table_flag_.is_valid() + && table_backup_flag_.is_valid() && root_row_store_type_ < ObRowStoreType::MAX_ROW_STORE; } @@ -259,7 +259,7 @@ int ObSSTableMergeRes::assign(const ObSSTableMergeRes &src) row_count_ = src.row_count_; max_merged_trans_version_ = src.max_merged_trans_version_; contain_uncommitted_row_ = src.contain_uncommitted_row_; - table_flag_ = src.table_flag_; + table_backup_flag_ = src.table_backup_flag_; occupy_size_ = src.occupy_size_; original_size_ = src.original_size_; data_checksum_ = src.data_checksum_; @@ -327,27 +327,27 @@ int ObSSTableMergeRes::fill_column_checksum_for_empty_major( void ObSSTableMergeRes::set_table_flag_with_macro_id_array() { - table_flag_.set_no_backup(); - table_flag_.set_no_local(); + table_backup_flag_.set_no_backup(); + table_backup_flag_.set_no_local(); for (int64_t i = 0; i < other_block_ids_.count(); ++i) { const MacroBlockId &block_id = other_block_ids_.at(i); if (block_id.is_local_id()) { - table_flag_.set_has_local(); + table_backup_flag_.set_has_local(); } else if (block_id.is_backup_id()) { - table_flag_.set_has_backup(); + table_backup_flag_.set_has_backup(); } - if (table_flag_.has_backup() && table_flag_.has_local()) { + if (table_backup_flag_.has_backup() && table_backup_flag_.has_local()) { break; } } for (int64_t j = 0; j < data_block_ids_.count(); ++j) { const MacroBlockId &block_id = data_block_ids_.at(j); if (block_id.is_local_id()) { - table_flag_.set_has_local(); + table_backup_flag_.set_has_local(); } else if (block_id.is_backup_id()) { - table_flag_.set_has_backup(); + table_backup_flag_.set_has_backup(); } - if (table_flag_.has_backup() && table_flag_.has_local()) { + if (table_backup_flag_.has_backup() && table_backup_flag_.has_local()) { break; } } diff --git a/src/storage/blocksstable/index_block/ob_index_block_builder.h b/src/storage/blocksstable/index_block/ob_index_block_builder.h index 61e919678..d53710808 100755 --- a/src/storage/blocksstable/index_block/ob_index_block_builder.h +++ b/src/storage/blocksstable/index_block/ob_index_block_builder.h @@ -266,7 +266,7 @@ public: K_(data_column_cnt), K_(data_column_checksums), K_(row_count), K_(max_merged_trans_version), K_(contain_uncommitted_row), K_(occupy_size), K_(original_size), K_(data_checksum), K_(use_old_macro_block_count), - K_(compressor_type), K_(root_row_store_type), K_(nested_offset), K_(nested_size), K_(table_flag), + K_(compressor_type), K_(root_row_store_type), K_(nested_offset), K_(nested_size), K_(table_backup_flag), K_(encrypt_id), K_(master_key_id), KPHEX_(encrypt_key, sizeof(encrypt_key_))); public: ObIndexTreeRootBlockDesc root_desc_; @@ -290,7 +290,7 @@ public: int64_t master_key_id_; int64_t nested_offset_; int64_t nested_size_; - ObTableFlag table_flag_; + ObTableBackupFlag table_backup_flag_; ObRowStoreType root_row_store_type_; char encrypt_key_[share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH]; DISALLOW_COPY_AND_ASSIGN(ObSSTableMergeRes); diff --git a/src/storage/blocksstable/ob_sstable_meta.cpp b/src/storage/blocksstable/ob_sstable_meta.cpp index 90fac2193..565aae928 100644 --- a/src/storage/blocksstable/ob_sstable_meta.cpp +++ b/src/storage/blocksstable/ob_sstable_meta.cpp @@ -64,8 +64,9 @@ ObSSTableBasicMeta::ObSSTableBasicMeta() master_key_id_(0), sstable_logic_seq_(0), latest_row_store_type_(ObRowStoreType::MAX_ROW_STORE), - table_flag_(), - root_macro_seq_(0) + root_macro_seq_(0), + table_backup_flag_(), + table_shared_flag_() { MEMSET(encrypt_key_, 0, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); } @@ -118,7 +119,8 @@ bool ObSSTableBasicMeta::check_basic_meta_equality(const ObSSTableBasicMeta &oth && master_key_id_ == other.master_key_id_ && 0 == MEMCMP(encrypt_key_, other.encrypt_key_, sizeof(encrypt_key_)) && latest_row_store_type_ == other.latest_row_store_type_ - && table_flag_ == other.table_flag_; + && table_backup_flag_ == other.table_backup_flag_ + && table_shared_flag_ == other.table_shared_flag_; } bool ObSSTableBasicMeta::is_valid() const @@ -146,7 +148,8 @@ bool ObSSTableBasicMeta::is_valid() const && sstable_logic_seq_ >= 0 && root_row_store_type_ < ObRowStoreType::MAX_ROW_STORE && is_latest_row_store_type_valid()) - && table_flag_.is_valid(); + && table_backup_flag_.is_valid() + && table_shared_flag_.is_valid(); return ret; } @@ -186,7 +189,8 @@ void ObSSTableBasicMeta::reset() sstable_logic_seq_ = 0; MEMSET(encrypt_key_, 0, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); latest_row_store_type_ = ObRowStoreType::MAX_ROW_STORE; - table_flag_.reset(); + table_backup_flag_.reset(); + table_shared_flag_.reset(); } DEFINE_SERIALIZE(ObSSTableBasicMeta) @@ -243,7 +247,8 @@ DEFINE_SERIALIZE(ObSSTableBasicMeta) master_key_id_, sstable_logic_seq_, latest_row_store_type_, - table_flag_); + table_backup_flag_, + table_shared_flag_); if (OB_FAIL(ret)) { } else if (OB_UNLIKELY(length_ != pos - start_pos)) { ret = OB_ERR_UNEXPECTED; @@ -324,7 +329,8 @@ int ObSSTableBasicMeta::decode_for_compat(const char *buf, const int64_t data_le master_key_id_, sstable_logic_seq_, latest_row_store_type_, - table_flag_); + table_backup_flag_, + table_shared_flag_); return ret; } @@ -366,7 +372,8 @@ DEFINE_GET_SERIALIZE_SIZE(ObSSTableBasicMeta) master_key_id_, sstable_logic_seq_, latest_row_store_type_, - table_flag_); + table_backup_flag_, + table_shared_flag_); return len; } @@ -643,7 +650,8 @@ int ObSSTableMeta::init_base_meta( basic_meta_.encrypt_id_ = param.encrypt_id_; basic_meta_.master_key_id_ = param.master_key_id_; MEMCPY(basic_meta_.encrypt_key_, param.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); - basic_meta_.table_flag_ = param.table_flag_; + basic_meta_.table_backup_flag_ = param.table_backup_flag_; + basic_meta_.table_shared_flag_ = param.table_shared_flag_; basic_meta_.length_ = basic_meta_.get_serialize_size(); if (OB_FAIL(prepare_column_checksum(param.column_checksums_, allocator))) { LOG_WARN("fail to prepare column checksum", K(ret), K(param)); @@ -1389,6 +1397,9 @@ int ObSSTableMetaChecker::check_sstable_basic_meta( } else if (new_sstable_basic_meta.column_cnt_ != old_sstable_basic_meta.column_cnt_) { ret = OB_INVALID_DATA; LOG_WARN("column_cnt_ not match", K(ret), K(old_sstable_basic_meta), K(new_sstable_basic_meta)); + } else if (new_sstable_basic_meta.table_shared_flag_ != old_sstable_basic_meta.table_shared_flag_) { + ret = OB_INVALID_DATA; + LOG_WARN("table_shared_flag_ not match", K(ret), K(old_sstable_basic_meta), K(new_sstable_basic_meta)); } return ret; } diff --git a/src/storage/blocksstable/ob_sstable_meta.h b/src/storage/blocksstable/ob_sstable_meta.h index 7f02da5c7..1562a66ae 100644 --- a/src/storage/blocksstable/ob_sstable_meta.h +++ b/src/storage/blocksstable/ob_sstable_meta.h @@ -148,7 +148,7 @@ public: K(ddl_scn_), K(filled_tx_scn_), K(contain_uncommitted_row_), K(status_), K_(root_row_store_type), K_(compressor_type), K_(encrypt_id), K_(master_key_id), K_(sstable_logic_seq), KPHEX_(encrypt_key, sizeof(encrypt_key_)), - K_(latest_row_store_type), K_(table_flag)); + K_(latest_row_store_type), K_(table_backup_flag), K_(table_shared_flag)); public: int32_t version_; @@ -187,8 +187,10 @@ public: int16_t sstable_logic_seq_; common::ObRowStoreType latest_row_store_type_; char encrypt_key_[share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH]; - storage::ObTableFlag table_flag_; int64_t root_macro_seq_; // placeholder, will be used after palf branch merged + storage::ObTableBackupFlag table_backup_flag_; //cannot add backup flag to ObSSTableMetaChecker + //quick restore with rebuild replace major will has same key sstable + storage::ObTableSharedFlag table_shared_flag_; //Add new variable need consider ObSSTableMetaChecker }; @@ -280,7 +282,7 @@ public: OB_INLINE int64_t get_progressive_merge_step() const { return basic_meta_.progressive_merge_step_; } OB_INLINE const ObRootBlockInfo &get_root_info() const { return data_root_info_; } OB_INLINE const ObSSTableMacroInfo &get_macro_info() const { return macro_info_; } - OB_INLINE const ObTableFlag &get_table_flag() const { return basic_meta_.table_flag_; } + OB_INLINE const ObTableBackupFlag &get_table_backup_flag() const { return basic_meta_.table_backup_flag_; } int load_root_block_data(common::ObArenaAllocator &allocator); //TODO:@jinzhu remove me after using kv cache. inline int transform_root_block_extra_buf(common::ObArenaAllocator &allocator) { diff --git a/src/storage/blocksstable/ob_table_flag.cpp b/src/storage/blocksstable/ob_table_flag.cpp index 8bc567de2..a50496612 100644 --- a/src/storage/blocksstable/ob_table_flag.cpp +++ b/src/storage/blocksstable/ob_table_flag.cpp @@ -18,55 +18,82 @@ namespace oceanbase namespace storage { -ObTableFlag::ObTableFlag() +ObTableBackupFlag::ObTableBackupFlag() : has_backup_flag_(ObTableHasBackupFlag::NO_BACKUP), has_local_flag_(ObTableHasLocalFlag::HAS_LOCAL), - is_shared_flag_(ObTableIsSharedFlag::IS_NOT_SHARED), reserved_(0) { } -ObTableFlag::ObTableFlag(int64_t flag) +ObTableBackupFlag::ObTableBackupFlag(int64_t flag) : flag_(flag) { } -ObTableFlag::~ObTableFlag() +ObTableBackupFlag::~ObTableBackupFlag() { } -void ObTableFlag::reset() +void ObTableBackupFlag::reset() { has_backup_flag_ = ObTableHasBackupFlag::NO_BACKUP; has_local_flag_ = ObTableHasLocalFlag::HAS_LOCAL; - is_shared_flag_ = ObTableIsSharedFlag::IS_NOT_SHARED; reserved_ = 0; } -bool ObTableFlag::is_valid() const +bool ObTableBackupFlag::is_valid() const { return has_backup_flag_ >= ObTableHasBackupFlag::NO_BACKUP && has_backup_flag_ < ObTableHasBackupFlag::MAX - && has_local_flag_ >= ObTableHasLocalFlag::HAS_LOCAL && has_local_flag_ < ObTableHasLocalFlag::MAX - && is_shared_flag_ >= ObTableIsSharedFlag::IS_NOT_SHARED && is_shared_flag_ < ObTableIsSharedFlag::MAX; + && has_local_flag_ >= ObTableHasLocalFlag::HAS_LOCAL && has_local_flag_ < ObTableHasLocalFlag::MAX; } -void ObTableFlag::set_default() +void ObTableBackupFlag::set_default() { has_backup_flag_ = ObTableHasBackupFlag::NO_BACKUP; has_local_flag_ = ObTableHasLocalFlag::HAS_LOCAL; - is_shared_flag_ = ObTableIsSharedFlag::IS_NOT_SHARED; reserved_ = 0; } -void ObTableFlag::clear() +void ObTableBackupFlag::clear() { has_backup_flag_ = ObTableHasBackupFlag::NO_BACKUP; has_local_flag_ = ObTableHasLocalFlag::NO_LOCAL; - is_shared_flag_ = ObTableIsSharedFlag::IS_NOT_SHARED; reserved_ = 0; } -OB_SERIALIZE_MEMBER(ObTableFlag, flag_); +OB_SERIALIZE_MEMBER(ObTableBackupFlag, flag_); + + + +ObTableSharedFlag::ObTableSharedFlag() + : shared_flag_(PRIVATE), + reserved_(0) +{ +} + +ObTableSharedFlag::~ObTableSharedFlag() +{ +} + +void ObTableSharedFlag::reset() +{ + shared_flag_ = PRIVATE; + reserved_ = 0; +} + +bool ObTableSharedFlag::is_valid() const +{ + return shared_flag_ >= PRIVATE && shared_flag_ < MAX; +} + +void ObTableSharedFlag::set_default() +{ + shared_flag_ = PRIVATE; + reserved_ = 0; +} + +OB_SERIALIZE_MEMBER(ObTableSharedFlag, flag_); + } } diff --git a/src/storage/blocksstable/ob_table_flag.h b/src/storage/blocksstable/ob_table_flag.h index b434db332..fb4a59ad2 100644 --- a/src/storage/blocksstable/ob_table_flag.h +++ b/src/storage/blocksstable/ob_table_flag.h @@ -45,32 +45,21 @@ public: }; }; -class ObTableIsSharedFlag final -{ -public: - enum FLAG - { - IS_NOT_SHARED = 0, - IS_SHARED = 1, - MAX - }; -}; - -struct ObTableFlag final +struct ObTableBackupFlag final { OB_UNIS_VERSION(1); public: - ObTableFlag(); + ObTableBackupFlag(); // TODO: yangyi.yyy to Refactor - ObTableFlag(int64_t flag); - ~ObTableFlag(); + ObTableBackupFlag(int64_t flag); + ~ObTableBackupFlag(); void reset(); bool is_valid() const; - OB_INLINE bool operator==(const ObTableFlag &other) const; - OB_INLINE bool operator!=(const ObTableFlag &other) const; + OB_INLINE bool operator==(const ObTableBackupFlag &other) const; + OB_INLINE bool operator!=(const ObTableBackupFlag &other) const; void set_default(); void clear(); - TO_STRING_KV(K_(has_backup_flag), K_(has_local_flag), K_(is_shared_flag)); + TO_STRING_KV(K_(has_backup_flag), K_(has_local_flag)); public: bool has_backup() const { return ObTableHasBackupFlag::HAS_BACKUP == has_backup_flag_; } @@ -84,34 +73,86 @@ public: private: static const uint64_t SF_BIT_HAS_BACKUP = 1; static const uint64_t SF_BIT_HAS_LOCAL = 1; - static const uint64_t SF_BIT_IS_SHARED = 1; - static const uint64_t SF_BIT_RESERVED = 61; + static const uint64_t SF_BIT_RESERVED = 30; public: union { - int64_t flag_; + int32_t flag_; struct { ObTableHasBackupFlag::FLAG has_backup_flag_ : SF_BIT_HAS_BACKUP; ObTableHasLocalFlag::FLAG has_local_flag_ : SF_BIT_HAS_LOCAL; - ObTableIsSharedFlag::FLAG is_shared_flag_ : SF_BIT_IS_SHARED; int64_t reserved_: SF_BIT_RESERVED; }; }; }; -bool ObTableFlag::operator==(const ObTableFlag &other) const +bool ObTableBackupFlag::operator==(const ObTableBackupFlag &other) const { - return has_backup_flag_ == other.has_backup_flag_ - && has_local_flag_ == other.has_local_flag_ - && is_shared_flag_ == other.is_shared_flag_; + return flag_ == other.flag_; } -bool ObTableFlag::operator!=(const ObTableFlag &other) const +bool ObTableBackupFlag::operator!=(const ObTableBackupFlag &other) const { return !(this->operator==(other)); } +struct ObTableSharedFlag final +{ + OB_UNIS_VERSION(1); +public: + enum FLAG : uint8_t + { + PRIVATE = 0, //share nothing + SHARED_SSTABLE = 1, //sstable is public data, including meta tree and data + SHARED_MACRO_BLOCKS = 2, //only macro block is public data, meta tree is private + MAX + }; +public: + ObTableSharedFlag(); + ~ObTableSharedFlag(); + void reset(); + bool is_valid() const; + OB_INLINE bool operator==(const ObTableSharedFlag &other) const; + OB_INLINE bool operator!=(const ObTableSharedFlag &other) const; + void set_default(); + void clear(); + + void set_private() { shared_flag_ = PRIVATE; } + void set_shared_sstable() { shared_flag_ = SHARED_SSTABLE; } + void set_share_macro_blocks() { shared_flag_ = SHARED_MACRO_BLOCKS; } + bool is_shared_macro_blocks() const { + return SHARED_SSTABLE == shared_flag_ + || SHARED_MACRO_BLOCKS == shared_flag_; } + bool is_shared_sstable() const { return SHARED_SSTABLE == shared_flag_; } + bool is_only_shared_macro_blocks() const { + return SHARED_SSTABLE != shared_flag_ + && SHARED_MACRO_BLOCKS == shared_flag_; } + TO_STRING_KV(K_(shared_flag), K_(reserved)); + +private: + static const uint64_t SF_BIT_IS_SHARED = 8; + static const uint64_t SF_BIT_RESERVED = 24; + union { + int32_t flag_; + struct {; + FLAG shared_flag_ : SF_BIT_IS_SHARED; + int32_t reserved_: SF_BIT_RESERVED; + }; + }; +}; + +bool ObTableSharedFlag::operator==(const ObTableSharedFlag &other) const +{ + return flag_ == other.flag_; +} + +bool ObTableSharedFlag::operator!=(const ObTableSharedFlag &other) const +{ + return !(this->operator==(other)); +} + + } } -#endif \ No newline at end of file +#endif diff --git a/src/storage/high_availability/ob_physical_copy_task.cpp b/src/storage/high_availability/ob_physical_copy_task.cpp index aaf439bd9..366bf8d1e 100644 --- a/src/storage/high_availability/ob_physical_copy_task.cpp +++ b/src/storage/high_availability/ob_physical_copy_task.cpp @@ -917,7 +917,7 @@ int ObSSTableCopyFinishTask::get_space_optimization_mode_( mode = ObSSTableIndexBuilder::DISABLE; } else if (sstable_param->is_small_sstable_) { mode = ObSSTableIndexBuilder::ENABLE; - } else if (sstable_param->basic_meta_.table_flag_.has_backup()) { + } else if (sstable_param->basic_meta_.table_backup_flag_.has_backup()) { mode = ObSSTableIndexBuilder::ENABLE; } else { mode = ObSSTableIndexBuilder::DISABLE; @@ -1140,14 +1140,14 @@ int ObSSTableCopyFinishTask::build_create_empty_sstable_param_( } else if (ObTabletRestoreAction::is_restore_remote_sstable(copy_ctx_.restore_action_)) { // Donot set has no local, otherwise, the sstable cannot be scheduled to restore. Then column sstable // cannot be restored. - param.table_flag_.set_no_local(); - param.table_flag_.set_has_backup(); + param.table_backup_flag_.set_no_local(); + param.table_backup_flag_.set_has_backup(); } else if (ObTabletRestoreAction::is_restore_replace_remote_sstable(copy_ctx_.restore_action_)) { - param.table_flag_.set_has_local(); - param.table_flag_.set_no_backup(); + param.table_backup_flag_.set_has_local(); + param.table_backup_flag_.set_no_backup(); } else { // for migration - param.table_flag_ = sstable_param_->basic_meta_.table_flag_; + param.table_backup_flag_ = sstable_param_->basic_meta_.table_backup_flag_; } return ret; } @@ -1264,8 +1264,8 @@ int ObSSTableCopyFinishTask::build_create_pure_remote_sstable_param_( } else if (OB_FAIL(param.init_for_remote(*sstable_param_))) { LOG_WARN("failed to init for remote", K(ret)); } else { - param.table_flag_.set_has_backup(); - param.table_flag_.set_no_local(); + param.table_backup_flag_.set_has_backup(); + param.table_backup_flag_.set_no_local(); } return ret; } diff --git a/src/storage/high_availability/ob_storage_ha_reader.cpp b/src/storage/high_availability/ob_storage_ha_reader.cpp index d0c7829b6..f8e319f94 100644 --- a/src/storage/high_availability/ob_storage_ha_reader.cpp +++ b/src/storage/high_availability/ob_storage_ha_reader.cpp @@ -3147,7 +3147,7 @@ int ObCopyRemoteSSTableInfoObProducer::check_need_copy_sstable_( need_copy_sstable = true; } else if (OB_FAIL(sstable->get_meta(sst_meta_hdl))) { LOG_WARN("failed to get sstable meta handle", K(ret), KPC(sstable)); - } else if (sst_meta_hdl.get_sstable_meta().get_table_flag().has_backup()) { + } else if (sst_meta_hdl.get_sstable_meta().get_table_backup_flag().has_backup()) { need_copy_sstable = true; } diff --git a/src/storage/high_availability/ob_tablet_group_restore.cpp b/src/storage/high_availability/ob_tablet_group_restore.cpp index ed4fd80fe..fd5c199f2 100644 --- a/src/storage/high_availability/ob_tablet_group_restore.cpp +++ b/src/storage/high_availability/ob_tablet_group_restore.cpp @@ -2717,7 +2717,7 @@ int ObTabletRestoreTask::check_remote_sstable_exist_in_table_store_( need_copy = true; } else if (OB_FAIL(sstable->get_meta(sst_meta_hdl))) { LOG_WARN("failed to get sstable meta handle", K(ret), KPC(sstable)); - } else if (sst_meta_hdl.get_sstable_meta().get_table_flag().has_backup()) { + } else if (sst_meta_hdl.get_sstable_meta().get_table_backup_flag().has_backup()) { need_copy = true; } else { need_copy = false; @@ -2928,7 +2928,7 @@ int ObTabletFinishRestoreTask::verify_table_store_() // do nothing } else if (OB_FAIL(static_cast(table)->get_meta(sst_meta_hdl))) { LOG_WARN("failed to get new sstable meta handle", K(ret), KPC(table)); - } else if (sst_meta_hdl.get_sstable_meta().get_basic_meta().table_flag_.has_backup()) { + } else if (sst_meta_hdl.get_sstable_meta().get_basic_meta().table_backup_flag_.has_backup()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("remote table should not exist", K(ret), KPC(table)); } @@ -2949,7 +2949,7 @@ int ObTabletFinishRestoreTask::verify_table_store_() // do nothing } else if (OB_FAIL(static_cast(table)->get_meta(sst_meta_hdl))) { LOG_WARN("failed to get new sstable meta handle", K(ret), KPC(table)); - } else if (sst_meta_hdl.get_sstable_meta().get_basic_meta().table_flag_.has_backup()) { + } else if (sst_meta_hdl.get_sstable_meta().get_basic_meta().table_backup_flag_.has_backup()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("remote table should not exist", K(ret), KPC(table)); } @@ -2970,7 +2970,7 @@ int ObTabletFinishRestoreTask::verify_table_store_() // do nothing } else if (OB_FAIL(static_cast(table)->get_meta(sst_meta_hdl))) { LOG_WARN("failed to get new sstable meta handle", K(ret), KPC(table)); - } else if (sst_meta_hdl.get_sstable_meta().get_basic_meta().table_flag_.has_backup()) { + } else if (sst_meta_hdl.get_sstable_meta().get_basic_meta().table_backup_flag_.has_backup()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("remote table should not exist", K(ret), KPC(table)); } @@ -2995,7 +2995,7 @@ int ObTabletFinishRestoreTask::verify_table_store_() LOG_WARN("table is not sstable", K(ret), KPC(table)); } else if (OB_FAIL(static_cast(table)->get_meta(sst_meta_hdl))) { LOG_WARN("failed to get new sstable meta handle", K(ret), KPC(table)); - } else if (sst_meta_hdl.get_sstable_meta().get_basic_meta().table_flag_.has_backup()) { + } else if (sst_meta_hdl.get_sstable_meta().get_basic_meta().table_backup_flag_.has_backup()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("remote table should not exist", K(ret), KPC(table)); } diff --git a/src/storage/ls/ob_ls_tablet_service.cpp b/src/storage/ls/ob_ls_tablet_service.cpp index a5b0e1576..5b2b0e609 100644 --- a/src/storage/ls/ob_ls_tablet_service.cpp +++ b/src/storage/ls/ob_ls_tablet_service.cpp @@ -2590,7 +2590,8 @@ int ObLSTabletService::build_create_sstable_param_for_migration( MEMCPY(param.encrypt_key_, mig_param.basic_meta_.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); param.root_block_addr_.set_none_addr(); param.data_block_macro_meta_addr_.set_none_addr();; - param.table_flag_ = mig_param.basic_meta_.table_flag_; + param.table_backup_flag_ = mig_param.basic_meta_.table_backup_flag_; + param.table_shared_flag_ = mig_param.basic_meta_.table_shared_flag_; if (OB_FAIL(param.column_checksums_.assign(mig_param.column_checksums_))) { LOG_WARN("fail to assign column checksums", K(ret), K(mig_param)); } diff --git a/src/storage/ob_i_table.cpp b/src/storage/ob_i_table.cpp index 9abdaf542..6365cc466 100644 --- a/src/storage/ob_i_table.cpp +++ b/src/storage/ob_i_table.cpp @@ -890,7 +890,7 @@ int ObTablesHandleArray::get_all_remote_major_sstables(common::ObIArrayget_meta(sst_meta_hdl))) { STORAGE_LOG(WARN, "failed to get sstable meta handle", K(ret), K(i)); - } else if (!sst_meta_hdl.get_sstable_meta().get_table_flag().has_backup()) { + } else if (!sst_meta_hdl.get_sstable_meta().get_table_backup_flag().has_backup()) { // do nothing } else if (OB_FAIL(tables.push_back(table))) { STORAGE_LOG(WARN, "failed to add remote major sstable", K(ret), K(i)); diff --git a/src/storage/tablet/ob_tablet.cpp b/src/storage/tablet/ob_tablet.cpp index 165e68034..69fbbfbe4 100644 --- a/src/storage/tablet/ob_tablet.cpp +++ b/src/storage/tablet/ob_tablet.cpp @@ -7099,7 +7099,7 @@ int ObTablet::calc_sstable_occupy_size(int64_t &occupy_size, int64_t &pure_backu } else { const ObSSTableMeta &sstable_meta = meta_handle.get_sstable_meta(); occupy_size += sstable_meta.get_occupy_size(); - if (!meta_handle.get_sstable_meta().get_basic_meta().table_flag_.has_local()) { + if (!meta_handle.get_sstable_meta().get_basic_meta().table_backup_flag_.has_local()) { pure_backup_sstable_occupy_size += sstable_meta.get_occupy_size(); } } diff --git a/src/storage/tablet/ob_tablet_create_sstable_param.cpp b/src/storage/tablet/ob_tablet_create_sstable_param.cpp index 75e868b60..9e85fe8b5 100644 --- a/src/storage/tablet/ob_tablet_create_sstable_param.cpp +++ b/src/storage/tablet/ob_tablet_create_sstable_param.cpp @@ -76,7 +76,8 @@ ObTabletCreateSSTableParam::ObTabletCreateSSTableParam() nested_size_(0), data_block_ids_(), other_block_ids_(), - table_flag_(), + table_backup_flag_(), + table_shared_flag_(), uncommitted_tx_id_(0) { MEMSET(encrypt_key_, 0, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); @@ -91,9 +92,9 @@ bool ObTabletCreateSSTableParam::is_valid() const } else if (OB_UNLIKELY(!table_mode_.is_valid())) { ret = false; LOG_WARN("invalid table mode", K(table_mode_)); - } else if (OB_UNLIKELY(!table_flag_.is_valid())) { + } else if (OB_UNLIKELY(!table_backup_flag_.is_valid() || !table_shared_flag_.is_valid())) { ret = false; - LOG_WARN("invalid table flag", K_(table_flag)); + LOG_WARN("invalid table backup flag or invalid table shared flag", K_(table_backup_flag), K_(table_shared_flag)); } else if (!(schema_version_ >= 0 && sstable_logic_seq_ >= 0 && create_snapshot_version_ >= 0 @@ -170,7 +171,7 @@ int ObTabletCreateSSTableParam::inner_init_with_merge_res(const blocksstable::Ob STATIC_ASSERT(ARRAYSIZEOF(res.encrypt_key_) == share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH, "ObSSTableMergeRes encrypt_key_ array size mismatch OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH"); MEMCPY(encrypt_key_, res.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); - table_flag_ = res.table_flag_; + table_backup_flag_ = res.table_backup_flag_; if (OB_FAIL(data_block_ids_.assign(res.data_block_ids_))) { LOG_WARN("fail to fill data block ids", K(ret), K(res.data_block_ids_)); @@ -205,7 +206,7 @@ int ObTabletCreateSSTableParam::init_for_small_sstable(const blocksstable::ObSST max_merged_trans_version_ = res.max_merged_trans_version_; nested_offset_ = block_info.nested_offset_; nested_size_ = block_info.nested_size_; - table_flag_ = res.table_flag_; + table_shared_flag_.reset(); if (OB_FAIL(inner_init_with_merge_res(res))) { LOG_WARN("fail to inner init with merge res", K(ret), K(res)); } else if (table_key_.is_major_sstable()) { @@ -285,7 +286,7 @@ int ObTabletCreateSSTableParam::init_for_merge(const compaction::ObBasicTabletMe nested_size_ = res.nested_size_; nested_offset_ = res.nested_offset_; ddl_scn_.set_min(); - table_flag_ = res.table_flag_; + table_shared_flag_.reset(); if (OB_FAIL(inner_init_with_merge_res(res))) { LOG_WARN("fail to init with merge res", K(ret), K(res.data_block_ids_)); @@ -390,7 +391,7 @@ int ObTabletCreateSSTableParam::init_for_ddl(blocksstable::ObSSTableIndexBuilder max_merged_trans_version_ = ddl_param.snapshot_version_; nested_size_ = res.nested_size_; nested_offset_ = res.nested_offset_; - table_flag_ = res.table_flag_; + table_shared_flag_.reset(); if (OB_FAIL(inner_init_with_merge_res(res))) { LOG_WARN("fail to inner init with merge res", K(ret), K(res)); @@ -450,7 +451,7 @@ int ObTabletCreateSSTableParam::init_for_ha( max_merged_trans_version_ = res.max_merged_trans_version_; rowkey_column_cnt_ = sstable_param.basic_meta_.rowkey_column_count_; ddl_scn_ = sstable_param.basic_meta_.ddl_scn_; - table_flag_ = res.table_flag_; + table_shared_flag_ = sstable_param.basic_meta_.table_shared_flag_; if (table_key_.is_co_sstable()) { column_group_cnt_ = sstable_param.column_group_cnt_; full_column_cnt_ = sstable_param.full_column_cnt_; @@ -500,7 +501,8 @@ int ObTabletCreateSSTableParam::init_for_ha(const blocksstable::ObMigrationSSTab data_block_macro_meta_addr_.set_none_addr(); rowkey_column_cnt_ = sstable_param.basic_meta_.rowkey_column_count_; MEMCPY(encrypt_key_, sstable_param.basic_meta_.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); - table_flag_ = sstable_param.basic_meta_.table_flag_; + table_backup_flag_ = sstable_param.basic_meta_.table_backup_flag_; + table_shared_flag_ = sstable_param.basic_meta_.table_shared_flag_; if (table_key_.is_co_sstable()) { column_group_cnt_ = sstable_param.column_group_cnt_; is_co_table_without_cgs_ = table_key_.is_ddl_sstable() ? false : true; @@ -554,6 +556,7 @@ int ObTabletCreateSSTableParam::init_for_remote(const blocksstable::ObMigrationS rowkey_column_cnt_ = sstable_param.basic_meta_.rowkey_column_count_; ddl_scn_ = sstable_param.basic_meta_.ddl_scn_; + table_shared_flag_ = sstable_param.basic_meta_.table_shared_flag_; if (table_key_.is_co_sstable()) { column_group_cnt_ = sstable_param.column_group_cnt_; full_column_cnt_ = sstable_param.full_column_cnt_; @@ -610,6 +613,7 @@ int ObTabletCreateSSTableParam::init_for_mds( nested_size_ = res.nested_size_; nested_offset_ = res.nested_offset_; ddl_scn_.set_min(); + table_shared_flag_.reset(); if (OB_FAIL(inner_init_with_merge_res(res))) { LOG_WARN("fail to init with merge res", K(ret), K(res.data_block_ids_)); diff --git a/src/storage/tablet/ob_tablet_create_sstable_param.h b/src/storage/tablet/ob_tablet_create_sstable_param.h index 7c2755c83..17d1b6fa8 100644 --- a/src/storage/tablet/ob_tablet_create_sstable_param.h +++ b/src/storage/tablet/ob_tablet_create_sstable_param.h @@ -128,7 +128,8 @@ public: K_(nested_offset), K_(nested_size), KPHEX_(encrypt_key, sizeof(encrypt_key_)), - K_(table_flag), + K_(table_backup_flag), + K_(table_shared_flag), K_(uncommitted_tx_id)); private: static const int64_t DEFAULT_MACRO_BLOCK_CNT = 64; @@ -179,7 +180,8 @@ public: char encrypt_key_[share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH]; common::ObSEArray data_block_ids_; common::ObSEArray other_block_ids_; - storage::ObTableFlag table_flag_; + storage::ObTableBackupFlag table_backup_flag_; //ObTableBackupFlag will be updated by ObSSTableMergeRes + storage::ObTableSharedFlag table_shared_flag_; //ObTableSharedFlag will be updated by ObTabletCreateSSTableParam int64_t uncommitted_tx_id_; }; diff --git a/src/storage/tablet/ob_tablet_table_store.cpp b/src/storage/tablet/ob_tablet_table_store.cpp index c401329a0..b3dabeec3 100644 --- a/src/storage/tablet/ob_tablet_table_store.cpp +++ b/src/storage/tablet/ob_tablet_table_store.cpp @@ -1464,7 +1464,7 @@ int ObTabletTableStore::inner_replace_remote_major_sstable_( LOG_WARN("no major table exist", K(ret), K(old_store), KPC(new_table)); } else if (OB_FAIL(static_cast(new_table)->get_meta(new_sst_meta_hdl))) { LOG_WARN("failed to get new sstable meta handle", K(ret), KPC(new_table)); - } else if (new_sst_meta_hdl.get_sstable_meta().get_basic_meta().table_flag_.has_backup()) { + } else if (new_sst_meta_hdl.get_sstable_meta().get_basic_meta().table_backup_flag_.has_backup()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("new table still has backup macro block", K(ret), KPC(new_table), K(new_sst_meta_hdl)); } else if (OB_FAIL(old_store.major_tables_.get_all_tables(old_tables_array))) { @@ -1478,7 +1478,7 @@ int ObTabletTableStore::inner_replace_remote_major_sstable_( LOG_WARN("get unexpected null table", K(ret), K(old_store)); } else if (OB_FAIL(static_cast(old_table)->get_meta(old_sst_meta_hdl))) { LOG_WARN("failed to get old sstable meta handle", K(ret), KPC(old_table)); - } else if (old_sst_meta_hdl.get_sstable_meta().get_basic_meta().table_flag_.has_no_backup()) { + } else if (old_sst_meta_hdl.get_sstable_meta().get_basic_meta().table_backup_flag_.has_no_backup()) { if (OB_FAIL(new_tables_array.push_back(old_table))) { LOG_WARN("failed to push back", K(ret)); } @@ -2173,14 +2173,14 @@ int ObTabletTableStore::check_new_sstable_can_be_accepted_( ObSSTableMetaHandle old_sst_meta_hdl; if (OB_FAIL(static_cast(new_table)->get_meta(new_sst_meta_hdl))) { LOG_WARN("failed to get new sstable meta handle", K(ret), KPC(new_table)); - } else if (new_sst_meta_hdl.get_sstable_meta().get_basic_meta().table_flag_.has_backup()) { + } else if (new_sst_meta_hdl.get_sstable_meta().get_basic_meta().table_backup_flag_.has_backup()) { // compaction during restore, and backup macro block is reused by the new table. bool has_remote_sstable = false; for (int64_t i = 0; OB_SUCC(ret) && i < old_tables.count(); ++i) { old_table = old_tables.at(i); if (OB_FAIL(static_cast(old_table)->get_meta(old_sst_meta_hdl))) { LOG_WARN("failed to get old sstable meta handle", K(ret), KPC(old_table)); - } else if (old_sst_meta_hdl.get_sstable_meta().get_basic_meta().table_flag_.has_backup()) { + } else if (old_sst_meta_hdl.get_sstable_meta().get_basic_meta().table_backup_flag_.has_backup()) { has_remote_sstable = true; break; } @@ -2206,7 +2206,7 @@ int ObTabletTableStore::check_new_major_sstable_can_be_accepted_( ObSSTableMetaHandle sst_meta_hdl; if (OB_FAIL(static_cast(major_table)->get_meta(sst_meta_hdl))) { LOG_WARN("failed to get new sstable meta handle", K(ret), KPC(major_table)); - } else if (sst_meta_hdl.get_sstable_meta().get_basic_meta().table_flag_.has_no_backup()) { + } else if (sst_meta_hdl.get_sstable_meta().get_basic_meta().table_backup_flag_.has_no_backup()) { // do nothing } else if (OB_FAIL(old_store.ddl_sstables_.get_all_tables(ddl_sstables))) { LOG_WARN("get ddl dump sstables failed", K(ret), K(old_store)); @@ -2550,7 +2550,7 @@ int ObTabletTableStore::replace_ha_remote_ddl_tables_( LOG_WARN("table type is unexpected", K(ret), KPC(major_table)); } else if (OB_FAIL(static_cast(major_table)->get_meta(major_meta_handle))) { LOG_WARN("get major table meta fail", K(ret), KPC(major_table)); - } else if (major_meta_handle.get_sstable_meta().get_basic_meta().table_flag_.has_backup()) { + } else if (major_meta_handle.get_sstable_meta().get_basic_meta().table_backup_flag_.has_backup()) { // This remote major is generated with ddl sstables by ddl commit. In this case, replace cannot // continue, should schedule replace remote major action again. ret = OB_NO_NEED_MERGE; @@ -2921,7 +2921,7 @@ int ObTabletTableStore::replace_ha_remote_sstables_( LOG_WARN("new table is not sstable", K(ret), KPC(new_table)); } else if (OB_FAIL(static_cast(new_table)->get_meta(new_meta_handle))) { LOG_WARN("get table meta handle fail", K(ret), KPC(new_table)); - } else if (new_meta_handle.get_sstable_meta().get_basic_meta().table_flag_.has_backup()) { + } else if (new_meta_handle.get_sstable_meta().get_basic_meta().table_backup_flag_.has_backup()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("new table still has backup macro block", K(ret), KPC(new_table), K(new_meta_handle)); } else { @@ -2942,7 +2942,7 @@ int ObTabletTableStore::replace_ha_remote_sstables_( LOG_WARN("old table is not sstable", K(ret), KPC(old_table)); } else if (OB_FAIL(static_cast(old_table)->get_meta(old_meta_handle))) { LOG_WARN("get table meta handle fail", K(ret), KPC(old_table)); - } else if (old_meta_handle.get_sstable_meta().get_basic_meta().table_flag_.has_no_backup()) { + } else if (old_meta_handle.get_sstable_meta().get_basic_meta().table_backup_flag_.has_no_backup()) { // this table does not has backup macro block, no need to be replaced. if (check_continue && OB_NOT_NULL(last_table) && old_table->get_start_scn() != last_table->get_end_scn()) { ret = OB_ERR_UNEXPECTED; From 726e3a048e316651e3d66c8c30a450664f04a6b2 Mon Sep 17 00:00:00 2001 From: obdev Date: Wed, 21 Aug 2024 08:18:02 +0000 Subject: [PATCH 151/249] bugfix for ~GroupConcatExtraResult --- src/sql/engine/aggregate/ob_aggregate_processor.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sql/engine/aggregate/ob_aggregate_processor.cpp b/src/sql/engine/aggregate/ob_aggregate_processor.cpp index ea5fe15bf..3e9381ddc 100644 --- a/src/sql/engine/aggregate/ob_aggregate_processor.cpp +++ b/src/sql/engine/aggregate/ob_aggregate_processor.cpp @@ -509,6 +509,7 @@ ObAggregateProcessor::GroupConcatExtraResult::~GroupConcatExtraResult() alloc_.free(sort_op_); sort_op_ = NULL; } else { + row_store_iter_.reset(); row_store_.reset(); } } From d15a82d1224831e9f3f32aacbd552b95628a32d7 Mon Sep 17 00:00:00 2001 From: sdc Date: Wed, 21 Aug 2024 09:18:35 +0000 Subject: [PATCH 152/249] fix split granule report 4002 bug when tablet_size is zero --- src/sql/optimizer/ob_logical_operator.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sql/optimizer/ob_logical_operator.cpp b/src/sql/optimizer/ob_logical_operator.cpp index 23f35096d..4546a45c6 100644 --- a/src/sql/optimizer/ob_logical_operator.cpp +++ b/src/sql/optimizer/ob_logical_operator.cpp @@ -4334,7 +4334,7 @@ int ObLogicalOperator::allocate_granule_nodes_above(AllocGIContext &ctx) gi_op->set_parallel(get_parallel()); gi_op->set_partition_count(ctx.partition_count_); gi_op->set_hash_part(ctx.hash_part_); - gi_op->set_tablet_size(ctx.tablet_size_); + gi_op->set_tablet_size(ctx.tablet_size_ > 0 ? ctx.tablet_size_ : OB_DEFAULT_TABLET_SIZE); if (ctx.is_in_pw_affinity_state()) { gi_op->add_flag(GI_AFFINITIZE); @@ -6565,4 +6565,4 @@ bool ObLogicalOperator::is_parallel_more_than_part_cnt() const } else { return get_parallel() > strong_sharding_->get_part_cnt(); } -} \ No newline at end of file +} From ec846f1b2947a2765e611cff3240fbed290a0310 Mon Sep 17 00:00:00 2001 From: JLY2015 <1623359870@qq.com> Date: Wed, 21 Aug 2024 09:24:41 +0000 Subject: [PATCH 153/249] [auto split] fix serialize param error --- src/rootserver/ddl_task/ob_ddl_task.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/rootserver/ddl_task/ob_ddl_task.cpp b/src/rootserver/ddl_task/ob_ddl_task.cpp index dc139fbcd..fc2a4943b 100644 --- a/src/rootserver/ddl_task/ob_ddl_task.cpp +++ b/src/rootserver/ddl_task/ob_ddl_task.cpp @@ -966,7 +966,9 @@ int ObDDLTask::set_ddl_stmt_str(const ObString &ddl_stmt_str) int ObDDLTask::serialize_params_to_message(char *buf, const int64_t buf_size, int64_t &pos) const { int ret = OB_SUCCESS; - ObDDLTaskSerializeField serialize_field(task_version_, parallelism_, data_format_version_, consumer_group_id_, is_abort_, sub_task_trace_id_, is_pre_split_); + bool is_unique_index = false; + bool is_global_index = false; + ObDDLTaskSerializeField serialize_field(task_version_, parallelism_, data_format_version_, consumer_group_id_, is_abort_, sub_task_trace_id_, is_unique_index, is_global_index, is_pre_split_); if (OB_UNLIKELY(nullptr == buf || buf_size <= 0)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), KP(buf), K(buf_size)); @@ -1001,7 +1003,9 @@ int ObDDLTask::deserialize_params_from_message(const uint64_t tenant_id, const c int64_t ObDDLTask::get_serialize_param_size() const { - ObDDLTaskSerializeField serialize_field(task_version_, parallelism_, data_format_version_, consumer_group_id_, is_abort_, sub_task_trace_id_, is_pre_split_); + bool is_unique_index = false; + bool is_global_index = false; + ObDDLTaskSerializeField serialize_field(task_version_, parallelism_, data_format_version_, consumer_group_id_, is_abort_, sub_task_trace_id_, is_unique_index, is_global_index, is_pre_split_); return serialize_field.get_serialize_size(); } From 2baf07341ac1d5ba8c697d335b5fc8fcbd0a32de Mon Sep 17 00:00:00 2001 From: obdev Date: Wed, 21 Aug 2024 09:55:52 +0000 Subject: [PATCH 154/249] BUGFIX: do_trans_end always clean tablelock --- deps/oblib/src/lib/utility/ob_macro_utils.h | 27 +++++++ src/sql/ob_sql_trans_control.cpp | 3 + src/storage/memtable/ob_memtable_context.cpp | 40 +++++------ src/storage/memtable/ob_memtable_context.h | 8 +-- src/storage/ob_arena_object_pool.h | 3 +- src/storage/tx/ob_trans_part_ctx.cpp | 75 +++++++++++--------- 6 files changed, 93 insertions(+), 63 deletions(-) diff --git a/deps/oblib/src/lib/utility/ob_macro_utils.h b/deps/oblib/src/lib/utility/ob_macro_utils.h index a34816a9d..12a955293 100644 --- a/deps/oblib/src/lib/utility/ob_macro_utils.h +++ b/deps/oblib/src/lib/utility/ob_macro_utils.h @@ -685,6 +685,26 @@ for (__typeof__((c).at(0)) *it = ((extra_condition) && (c).count() > 0 ? &(c).at } while(false) #endif +#ifndef ENABLE_DEBUG_LOG +#define OB_SAFE_ASSERT(x) \ + do{ \ + bool v=(x); \ + if(OB_UNLIKELY(!(v))) { \ + _OB_LOG_RET(ERROR, oceanbase::common::OB_ERROR, "assert fail, exp=%s", #x); \ + BACKTRACE_RET(ERROR, oceanbase::common::OB_ERROR, 1, "assert fail"); \ + } \ + } while(false) +#else +#define OB_SAFE_ASSERT(x) \ + do{ \ + bool v=(x); \ + if(OB_UNLIKELY(!(v))) { \ + _OB_LOG_RET(ERROR, oceanbase::common::OB_ERROR, "assert fail, exp=%s", #x); \ + BACKTRACE_RET(ERROR, oceanbase::common::OB_ERROR, 1, "assert fail"); \ + assert(v); \ + } \ + } while(false) +#endif #define OB_ASSERT_MSG(x, msg...) \ do{ \ @@ -709,6 +729,13 @@ for (__typeof__((c).at(0)) *it = ((extra_condition) && (c).count() > 0 ? &(c).at //#define ob_assert(x) OB_ASSERT(x) #define ob_assert(x) ob_release_assert(x) + +#ifndef ENABLE_DEBUG_LOG +#define OB_SAFE_ABORT() +#else +#define OB_SAFE_ABORT() ob_abort() +#endif + //////////////////////////////////////////////////////////////// // interval diff --git a/src/sql/ob_sql_trans_control.cpp b/src/sql/ob_sql_trans_control.cpp index b4e9701eb..05b6fe573 100644 --- a/src/sql/ob_sql_trans_control.cpp +++ b/src/sql/ob_sql_trans_control.cpp @@ -428,6 +428,7 @@ int ObSqlTransControl::rollback_trans(ObSQLSessionInfo *session, return ret; } +ERRSIM_POINT_DEF(SQL_DO_END_TX_FAIL) int ObSqlTransControl::do_end_trans_(ObSQLSessionInfo *session, const bool is_rollback, const bool is_explicit, @@ -455,6 +456,8 @@ int ObSqlTransControl::do_end_trans_(ObSQLSessionInfo *session, if (!session->is_inner() && session->associated_xa() && !is_explicit) { ret = OB_TRANS_XA_RMFAIL; LOG_ERROR("executing do end trans in xa", K(ret), K(session->get_xid()), KPC(tx_ptr)); + } else if (OB_FAIL(SQL_DO_END_TX_FAIL)) { + LOG_WARN("do end trans failed", K(ret)); } else { /* * normal transaction control diff --git a/src/storage/memtable/ob_memtable_context.cpp b/src/storage/memtable/ob_memtable_context.cpp index 46dcf01a5..55d966169 100644 --- a/src/storage/memtable/ob_memtable_context.cpp +++ b/src/storage/memtable/ob_memtable_context.cpp @@ -130,15 +130,11 @@ void ObMemtableCtx::reset() } if (OB_UNLIKELY(callback_alloc_count_ != callback_free_count_)) { TRANS_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "callback alloc and free count not match", K(*this)); -#ifdef ENABLE_DEBUG_LOG - ob_abort(); -#endif + OB_SAFE_ABORT(); } if (OB_UNLIKELY(unsubmitted_cnt_ != 0)) { TRANS_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "txn unsubmitted cnt not zero", K(*this), K(unsubmitted_cnt_)); -#ifdef ENABLE_DEBUG_LOG - ob_abort(); -#endif + OB_SAFE_ABORT(); } if (OB_TRANS_KILLED != end_code_) { // _NOTE_: skip when txn was forcedly killed @@ -150,9 +146,7 @@ void ObMemtableCtx::reset() if (OB_UNLIKELY(fill != sync_succ + sync_fail)) { TRANS_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "redo filled_count != sync_succ + sync_fail", KPC(this), K(fill), K(sync_succ), K(sync_fail)); -#ifdef ENABLE_DEBUG_LOG - ob_abort(); -#endif + OB_SAFE_ABORT(); } } is_inited_ = false; @@ -525,18 +519,18 @@ int ObMemtableCtx::do_trans_end( // and check memory leakage if (OB_UNLIKELY(ATOMIC_LOAD(&callback_alloc_count_) != ATOMIC_LOAD(&callback_free_count_))) { TRANS_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "callback alloc and free count not match", KPC(this)); -#ifdef ENABLE_DEBUG_LOG - ob_abort(); -#endif - } - // release durable table lock - if (OB_FAIL(ret)) { - UNUSED(final_scn); - //commit or abort log ts for clear table lock - } else if (OB_FAIL(clear_table_lock_(commit, trans_version, final_scn))) { - TRANS_LOG(ERROR, "clear table lock failed.", K(ret), K(*this)); + OB_SAFE_ABORT(); } } + + // NOTE: the commit or abort log may replay many times(retry), but this function only do once now. + // we need always clear table lock because the retry process may create a new one. + // release durable table lock + if (OB_FAIL(ret)) { + //commit or abort log ts for clear table lock + } else if (OB_FAIL(clear_table_lock_(commit, trans_version, final_scn))) { + TRANS_LOG(ERROR, "clear table lock failed.", K(ret), K(*this)); + } return ret; } @@ -597,10 +591,7 @@ int ObMemtableCtx::trans_replay_end(const bool commit, "checksum_replayed", checksum_collapsed, "checksum_before_collapse", replay_checksum, K(checksum_signature), KPC(this)); -#ifdef ENABLE_DEBUG_LOG - ob_usleep(5000); - ob_abort(); -#endif + OB_SAFE_ABORT(); } } } @@ -958,12 +949,15 @@ int ObMemtableCtx::update_checksum(const ObIArray &checksum, return trans_mgr_.update_checksum(checksum, checksum_scn); } +ERRSIM_POINT_DEF(TX_FORCE_WRITE_CLOG) bool ObMemtableCtx::pending_log_size_too_large(const ObTxSEQ &write_seq_no) { bool ret = true; if (0 == GCONF._private_buffer_size) { ret = false; + } else if (TX_FORCE_WRITE_CLOG) { + ret = true; } else { ret = trans_mgr_.pending_log_size_too_large(write_seq_no, GCONF._private_buffer_size); } diff --git a/src/storage/memtable/ob_memtable_context.h b/src/storage/memtable/ob_memtable_context.h index 4d25cccb5..4dcc4c8f0 100644 --- a/src/storage/memtable/ob_memtable_context.h +++ b/src/storage/memtable/ob_memtable_context.h @@ -186,9 +186,7 @@ public: { if (OB_UNLIKELY(ATOMIC_LOAD(&free_count_) != ATOMIC_LOAD(&alloc_count_))) { TRANS_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "query allocator leak found", K(alloc_count_), K(free_count_), K(alloc_size_)); -#ifdef ENABLE_DEBUG_LOG - ob_abort(); -#endif + OB_SAFE_ABORT(); } if (!only_check) { allocator_.reset(); @@ -278,9 +276,7 @@ public: { if (OB_UNLIKELY(free_count_ != alloc_count_)) { TRANS_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "callback memory leak found", K(alloc_count_), K(free_count_), K(alloc_size_)); -#ifdef ENABLE_DEBUG_LOG - ob_abort(); -#endif + OB_SAFE_ABORT(); } if (!only_check) { allocator_.reset(); diff --git a/src/storage/ob_arena_object_pool.h b/src/storage/ob_arena_object_pool.h index 983f13daf..1bed81fd3 100644 --- a/src/storage/ob_arena_object_pool.h +++ b/src/storage/ob_arena_object_pool.h @@ -100,8 +100,9 @@ template void ObArenaObjPool::reset() { if (OB_UNLIKELY(alloc_count_ != free_count_)) { - TABLELOCK_LOG_RET(ERROR, common::OB_ERR_UNEXPECTED, "object alloc and free count not match", K(*this)); + STORAGE_LOG_RET(ERROR, common::OB_ERR_UNEXPECTED, "object alloc and free count not match", K(*this)); } + OB_SAFE_ASSERT(alloc_count_ == free_count_); alloc_count_ = 0; free_count_ = 0; diff --git a/src/storage/tx/ob_trans_part_ctx.cpp b/src/storage/tx/ob_trans_part_ctx.cpp index 3e3c9472e..4f2415588 100644 --- a/src/storage/tx/ob_trans_part_ctx.cpp +++ b/src/storage/tx/ob_trans_part_ctx.cpp @@ -2258,10 +2258,7 @@ int ObPartTransCtx::on_success(ObTxLogCb *log_cb) ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "cb arg array is empty", K(ret), KPC(this)); print_trace_log_(); -#ifdef ENABLE_DEBUG_LOG - usleep(5000); - ob_abort(); -#endif + OB_SAFE_ABORT(); } if (log_cb->is_callbacked()) { skip_on_succ_cnt++; @@ -2283,9 +2280,7 @@ int ObPartTransCtx::on_success(ObTxLogCb *log_cb) ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "callback was missed when tx ctx exiting", K(ret), KPC(log_cb), KPC(this)); print_trace_log_(); -#ifdef ENABLE_DEBUG_LOG - ob_abort(); -#endif + OB_SAFE_ABORT(); } } else { // save the first error code @@ -2901,9 +2896,7 @@ int ObPartTransCtx::on_failure(ObTxLogCb *log_cb) share::SCN max_committed_scn; if (OB_FAIL(ls_tx_ctx_mgr_->get_ls_log_adapter()->get_palf_committed_max_scn(max_committed_scn))) { TRANS_LOG(ERROR, "get palf max committed scn fail, need retry", K(ret), KPC(this)); -#ifdef ENABLE_DEBUG_LOG - ob_abort(); // fast abort for easy debug -#endif + OB_SAFE_ABORT(); } else { TRANS_LOG(INFO, "succ get palf max_commited_scn", K(max_committed_scn), KPC(log_cb)); } @@ -5426,10 +5419,7 @@ int ObPartTransCtx::replay_redo_in_ctx(const ObTxRedoLog &redo_log, if (!is_tx_log_queue) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "serial final redo must be in tx_log_queue", K(ret), KPC(this), K(timestamp)); -#ifdef ENABLE_DEBUG_LOG - usleep(50_ms); - ob_abort(); -#endif + OB_SAFE_ABORT(); } else if (!exec_info_.serial_final_scn_.is_valid()) { ret = switch_to_parallel_logging_(timestamp, max_seq_no); } @@ -5534,10 +5524,7 @@ int ObPartTransCtx::replay_rollback_to(const ObTxRollbackToLog &log, && !pre_barrier) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "missing pre barrier flag for parallel replay", KR(ret), K(*this)); -#ifdef ENABLE_DEBUG_LOG - usleep(5000); - ob_abort(); -#endif + OB_SAFE_ABORT(); } else if (OB_FAIL(check_replay_avaliable_(offset, timestamp, part_log_no, need_replay))) { TRANS_LOG(WARN, "check replay available failed", KR(ret), K(offset), K(timestamp), K(*this)); } else if (!need_replay) { @@ -5591,9 +5578,7 @@ int ObPartTransCtx::replay_rollback_to(const ObTxRollbackToLog &log, } else { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "code should not go here", K(ret), K(timestamp), K_(trans_id), KPC(this)); -#ifdef ENABLE_DEBUG_LOG - ob_abort(); -#endif + OB_SAFE_ABORT(); } } if (OB_SUCC(ret) && @@ -5758,10 +5743,7 @@ int ObPartTransCtx::replay_commit_info(const ObTxCommitInfoLog &commit_info_log, if (is_parallel_logging() && !pre_barrier) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "missing pre barrier flag for parallel replay", KR(ret), K(*this)); -#ifdef ENABLE_DEBUG_LOG - usleep(5000); - ob_abort(); -#endif + OB_SAFE_ABORT(); } else if (OB_FAIL(check_replay_avaliable_(offset, timestamp, part_log_no, need_replay))) { TRANS_LOG(WARN, "check replay available failed", KR(ret), K(offset), K(timestamp), K(*this)); } else if (!need_replay) { @@ -5951,6 +5933,10 @@ int ObPartTransCtx::replay_prepare(const ObTxPrepareLog &prepare_log, return ret; } +ERRSIM_POINT_DEF(TX_REPLAY_COMMIT_FAIL_BEFORE_NOTIFY_TABLELOCK) +ERRSIM_POINT_DEF(TX_REPLAY_COMMIT_FAIL_AFTER_NOTIFY_TABLELOCK) +ERRSIM_POINT_DEF(TX_REPLAY_COMMIT_FAIL_AFTER_NOTIFY_ON_COMMIT) +ERRSIM_POINT_DEF(TX_REPLAY_COMMIT_FAIL_AFTER_COMMIT) int ObPartTransCtx::replay_commit(const ObTxCommitLog &commit_log, const palf::LSN &offset, const SCN ×tamp, @@ -6034,21 +6020,29 @@ int ObPartTransCtx::replay_commit(const ObTxCommitLog &commit_log, (exec_info_.need_checksum_ && replay_completeness_.is_complete() ? commit_log.get_checksum() : 0); mt_ctx_.set_replay_compact_version(replay_compact_version); - if (OB_FAIL(notify_table_lock_(timestamp, - true, - exec_info_.multi_data_source_, - false /* not a force kill */))) { + if (OB_FAIL(TX_REPLAY_COMMIT_FAIL_BEFORE_NOTIFY_TABLELOCK)) { + TRANS_LOG(WARN, "errsim error", K(ret)); + } else if (OB_FAIL(notify_table_lock_(timestamp, + true, + exec_info_.multi_data_source_, + false /* not a force kill */))) { TRANS_LOG(WARN, "notify table lock failed", KR(ret), "context", *this); + } else if (OB_FAIL(TX_REPLAY_COMMIT_FAIL_AFTER_NOTIFY_TABLELOCK)) { + TRANS_LOG(WARN, "errsim error", K(ret)); } else if (OB_FAIL(notify_data_source_(NotifyType::ON_COMMIT, timestamp, true, exec_info_.multi_data_source_))) { TRANS_LOG(WARN, "notify data source failed", KR(ret), K(commit_log)); + } else if (OB_FAIL(TX_REPLAY_COMMIT_FAIL_AFTER_NOTIFY_ON_COMMIT)) { + TRANS_LOG(WARN, "errsim error", K(ret)); } else if (OB_FAIL(trans_replay_commit_(ctx_tx_data_.get_commit_version(), timestamp, cluster_version_, checksum))) { TRANS_LOG(WARN, "trans replay commit failed", KR(ret), K(commit_log), KPC(this)); + } else if (OB_FAIL(TX_REPLAY_COMMIT_FAIL_AFTER_COMMIT)) { + TRANS_LOG(WARN, "errsim error", K(ret)); } else if ((!ctx_tx_data_.is_read_only()) && OB_FAIL(ctx_tx_data_.insert_into_tx_table())) { TRANS_LOG(WARN, "insert to tx table failed", KR(ret), K(*this)); } else if (is_local_tx_()) { @@ -6155,6 +6149,11 @@ int ObPartTransCtx::replay_clear(const ObTxClearLog &clear_log, return ret; } +ERRSIM_POINT_DEF(TX_REPLAY_ABORT_FAIL_BEFORE_NOTIFY_TX_END) +ERRSIM_POINT_DEF(TX_REPLAY_ABORT_FAIL_AFTER_NOTIFY_TX_END) +ERRSIM_POINT_DEF(TX_REPLAY_ABORT_FAIL_AFTER_ABORT) +ERRSIM_POINT_DEF(TX_REPLAY_ABORT_FAIL_AFTER_CLEAR) +ERRSIM_POINT_DEF(TX_REPLAY_ABORT_FAIL_AFTER_NOTIFY_ON_ABORT) int ObPartTransCtx::replay_abort(const ObTxAbortLog &abort_log, const palf::LSN &offset, const SCN ×tamp, @@ -6222,16 +6221,26 @@ int ObPartTransCtx::replay_abort(const ObTxAbortLog &abort_log, if (OB_FAIL(mds_cache_.generate_final_notify_array( exec_info_.multi_data_source_, true /*need_merge_cache*/, true /*allow_log_overflow*/))) { TRANS_LOG(WARN, "gen total mds array failed", K(ret)); + } else if (OB_FAIL(TX_REPLAY_ABORT_FAIL_BEFORE_NOTIFY_TX_END)) { + TRANS_LOG(WARN, "errsim error", K(ret)); } else if (OB_FAIL(notify_data_source_(NotifyType::TX_END, timestamp, true, exec_info_.multi_data_source_))) { TRANS_LOG(WARN, "notify data source for TX_END failed", KR(ret), K(*this)); + } else if (OB_FAIL(TX_REPLAY_ABORT_FAIL_AFTER_NOTIFY_TX_END)) { + TRANS_LOG(WARN, "errsim error", K(ret)); } else if (OB_FAIL(trans_replay_abort_(timestamp))) { TRANS_LOG(WARN, "transaction replay end error", KR(ret), "context", *this); + } else if (OB_FAIL(TX_REPLAY_ABORT_FAIL_AFTER_ABORT)) { + TRANS_LOG(WARN, "errsim error", K(ret)); } else if (OB_FAIL(trans_clear_())) { TRANS_LOG(WARN, "transaction clear error", KR(ret), "context", *this); + } else if (OB_FAIL(TX_REPLAY_ABORT_FAIL_AFTER_CLEAR)) { + TRANS_LOG(WARN, "errsim error", K(ret)); } else if (OB_FAIL(notify_data_source_(NotifyType::ON_ABORT, timestamp, true, mds_cache_.get_final_notify_array()))) { TRANS_LOG(WARN, "notify data source failed", KR(ret), K(abort_log)); + } else if (OB_FAIL(TX_REPLAY_ABORT_FAIL_AFTER_NOTIFY_ON_ABORT)) { + TRANS_LOG(WARN, "errsim error", K(ret)); } else if (!ctx_tx_data_.is_read_only() && OB_FAIL(ctx_tx_data_.add_abort_op(timestamp))) { TRANS_LOG(WARN, "add tx data abort_op failed", K(ret), KPC(this)); } else if ((!ctx_tx_data_.is_read_only()) && OB_FAIL(ctx_tx_data_.insert_into_tx_table())) { @@ -7659,6 +7668,7 @@ int ObPartTransCtx::notify_data_source_(const NotifyType notify_type, return ret; } +ERRSIM_POINT_DEF(TX_FORCE_WRITE_CLOG) int ObPartTransCtx::register_multi_data_source(const ObTxDataSourceType data_source_type, const char *buf, const int64_t len, @@ -7755,7 +7765,8 @@ int ObPartTransCtx::register_multi_data_source(const ObTxDataSourceType data_sou TRANS_LOG(WARN, "notify data source for register_succ failed", K(tmp_ret)); } else if (mds_cache_.get_unsubmitted_size() < ObTxMultiDataSourceLog::MAX_PENDING_BUF_SIZE - && !register_flag.need_flush_redo_instantly_) { + && !register_flag.need_flush_redo_instantly_ + && (OB_SUCCESS == TX_FORCE_WRITE_CLOG)) { // do nothing } else if (OB_SUCCESS != (tmp_ret = submit_log_impl_(ObTxLogType::TX_MULTI_DATA_SOURCE_LOG))) { @@ -10607,9 +10618,7 @@ inline int ObPartTransCtx::switch_to_parallel_logging_(const share::SCN serial_f TRANS_LOG(ERROR, "max seq_no of serial final log is invalid", K(ret), K(serial_final_scn), K(max_seq_no), KPC(this)); print_trace_log_(); -#ifdef ENABLE_DEBUG_LOG - ob_abort(); -#endif + OB_SAFE_ABORT(); } if (OB_SUCC(ret)) { // when start replaying serial final redo log or submitted serial final redo log From 38896ca913b0781353ad5c720a7eb5e82a46d92e Mon Sep 17 00:00:00 2001 From: chimyue Date: Wed, 21 Aug 2024 12:26:57 +0000 Subject: [PATCH 155/249] add pq_gby/pq_distinct hint, fix some outline bug --- src/objit/include/objit/common/ob_item_type.h | 5 + src/sql/optimizer/ob_join_order.cpp | 20 +- src/sql/optimizer/ob_log_distinct.cpp | 26 +- src/sql/optimizer/ob_log_group_by.cpp | 12 + src/sql/optimizer/ob_log_group_by.h | 15 +- src/sql/optimizer/ob_log_plan.cpp | 132 +++++--- src/sql/optimizer/ob_log_plan.h | 31 +- src/sql/optimizer/ob_select_log_plan.cpp | 283 ++++++++++-------- src/sql/optimizer/ob_select_log_plan.h | 26 +- src/sql/parser/sql_parser_mysql_mode.l | 3 + src/sql/parser/sql_parser_mysql_mode.y | 14 +- src/sql/resolver/dml/ob_dml_resolver.cpp | 35 +++ src/sql/resolver/dml/ob_dml_resolver.h | 1 + src/sql/resolver/dml/ob_hint.cpp | 39 +++ src/sql/resolver/dml/ob_hint.h | 30 +- src/sql/resolver/dml/ob_sql_hint.cpp | 63 +++- src/sql/resolver/dml/ob_sql_hint.h | 18 +- 17 files changed, 539 insertions(+), 214 deletions(-) diff --git a/src/objit/include/objit/common/ob_item_type.h b/src/objit/include/objit/common/ob_item_type.h index fe8c733c0..46ed636d1 100755 --- a/src/objit/include/objit/common/ob_item_type.h +++ b/src/objit/include/objit/common/ob_item_type.h @@ -2588,6 +2588,11 @@ typedef enum ObItemType //restore sts T_RESTORE_WITH_CONFIG_LIST = 4732, T_STS_CREDENTIAL = 4733, + // optimizer hint + T_PQ_GBY_HINT = 4734, + T_PQ_DISTINCT_HINT = 4735, + T_DISTRIBUTE_BASIC = 4736, + T_MAX //Attention: add a new type before T_MAX } ObItemType; diff --git a/src/sql/optimizer/ob_join_order.cpp b/src/sql/optimizer/ob_join_order.cpp index f06e5ab72..92af4115b 100644 --- a/src/sql/optimizer/ob_join_order.cpp +++ b/src/sql/optimizer/ob_join_order.cpp @@ -9758,7 +9758,9 @@ int ObJoinOrder::get_distributed_join_method(Path &left_path, ObShardingInfo *left_sharding = NULL; ObShardingInfo *right_sharding = NULL; distributed_methods = path_info.distributed_methods_; - bool use_shared_hash_join = right_path.parallel_ > ObGlobalHint::DEFAULT_PARALLEL; + const bool is_force_dist_method = !path_info.ignore_hint_ + && (distributed_methods == get_dist_algo(distributed_methods)); + bool use_shared_hash_join = false; ObSQLSessionInfo *session = NULL; int64_t max_path_parallel = max(left_path.parallel_, right_path.parallel_); can_slave_mapping = @@ -9773,7 +9775,7 @@ int ObJoinOrder::get_distributed_join_method(Path &left_path, ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(get_plan()), K(left_sharding), K(right_sharding), K(left_path.parent_), K(ret)); - } else if (use_shared_hash_join && OB_FAIL(session->get_px_shared_hash_join( use_shared_hash_join))) { + } else if (OB_FAIL(session->get_px_shared_hash_join(use_shared_hash_join))) { LOG_WARN("get force parallel ddl dop failed", K(ret)); } else if (HASH_JOIN == join_algo && is_naaj) { distributed_methods &= ~DIST_PARTITION_WISE; @@ -9791,14 +9793,20 @@ int ObJoinOrder::get_distributed_join_method(Path &left_path, } if (OB_SUCC(ret)) { if (HASH_JOIN == join_algo) { - if (use_shared_hash_join) { - distributed_methods &= ~DIST_BROADCAST_NONE; - distributed_methods &= ~DIST_ALL_NONE; - OPT_TRACE("shared hash join will not use BROADCAST"); + if (use_shared_hash_join && right_path.parallel_ > ObGlobalHint::DEFAULT_PARALLEL) { + if (is_force_dist_method && (DIST_BROADCAST_NONE == distributed_methods + || DIST_ALL_NONE == distributed_methods)) { + /* do nothing */ + } else { + distributed_methods &= ~DIST_BROADCAST_NONE; + distributed_methods &= ~DIST_ALL_NONE; + OPT_TRACE("shared hash join will not use BROADCAST"); + } if (IS_LEFT_STYLE_JOIN(path_info.join_type_)) { distributed_methods &= ~DIST_BC2HOST_NONE; } } else { + use_shared_hash_join = false; distributed_methods &= ~DIST_BC2HOST_NONE; OPT_TRACE("hash join will not use BC2HOST"); } diff --git a/src/sql/optimizer/ob_log_distinct.cpp b/src/sql/optimizer/ob_log_distinct.cpp index fbb2c9310..2e147f75e 100644 --- a/src/sql/optimizer/ob_log_distinct.cpp +++ b/src/sql/optimizer/ob_log_distinct.cpp @@ -348,6 +348,13 @@ int ObLogDistinct::print_outline_data(PlanText &plan_text) qb_name.length(), qb_name.ptr()))) { LOG_WARN("fail to print buffer", K(ret), K(buf), K(buf_len), K(pos)); + } else if (NULL != op || is_partition_wise()) { + ObPQHint pq_hint(T_PQ_DISTINCT_HINT); + pq_hint.set_qb_name(qb_name); + pq_hint.set_dist_method(is_partition_wise() ? T_DISTRIBUTE_NONE : T_DISTRIBUTE_HASH); + if (OB_FAIL(pq_hint.print_hint(plan_text))) { + LOG_WARN("failed to print pq hint", K(ret), K(pq_hint)); + } } else {/*do nothing*/} return ret; } @@ -362,7 +369,8 @@ int ObLogDistinct::print_used_hint(PlanText &plan_text) LOG_WARN("unexpected NULL", K(ret), K(get_plan())); } else { const ObHint *use_hash = get_plan()->get_log_plan_hint().get_normal_hint(T_USE_HASH_DISTINCT); - const ObHint *pushdown = get_plan()->get_log_plan_hint().get_normal_hint(T_DISTINCT_PUSHDOWN); + const ObHint *pushdown_hint = get_plan()->get_log_plan_hint().get_normal_hint(T_DISTINCT_PUSHDOWN); + const ObPQHint *pq_hint = dynamic_cast(get_plan()->get_log_plan_hint().get_normal_hint(T_PQ_DISTINCT_HINT)); if (NULL != use_hash) { bool match_hint = (HASH_AGGREGATE == algo_ && use_hash->is_enable_hint()) || (MERGE_AGGREGATE == algo_ && use_hash->is_disable_hint()); @@ -370,7 +378,7 @@ int ObLogDistinct::print_used_hint(PlanText &plan_text) LOG_WARN("failed to print used hint for group by", K(ret), K(*use_hash)); } } - if (OB_SUCC(ret) && NULL != pushdown) { + if (OB_SUCC(ret) && (NULL != pushdown_hint || NULL != pq_hint)) { const ObLogicalOperator *child = NULL; const ObLogicalOperator *op = NULL; if (OB_ISNULL(child = get_child(ObLogicalOperator::first_child))) { @@ -378,12 +386,14 @@ int ObLogDistinct::print_used_hint(PlanText &plan_text) LOG_WARN("unexpected NULL", K(ret), K(child)); } else if (OB_FAIL(child->get_pushdown_op(log_op_def::LOG_DISTINCT, op))) { LOG_WARN("failed to get push down distinct", K(ret)); - } else { - bool match_hint = NULL == op ? pushdown->is_disable_hint() - : pushdown->is_enable_hint(); - if (match_hint && OB_FAIL(pushdown->print_hint(plan_text))) { - LOG_WARN("failed to print used hint for group by", K(ret), K(*pushdown)); - } + } else if (NULL != pushdown_hint && (NULL == op ? pushdown_hint->is_disable_hint() : pushdown_hint->is_enable_hint()) + && OB_FAIL(pushdown_hint->print_hint(plan_text))) { + LOG_WARN("failed to print used hint for group by", K(ret), KPC(pushdown_hint)); + } else if (NULL != pq_hint + && ((NULL != op && pq_hint->is_force_dist_hash()) + || (is_partition_wise() && pq_hint->is_force_partition_wise())) + && OB_FAIL(pq_hint->print_hint(plan_text))) { + LOG_WARN("failed to print used hint for pq group by", K(ret), KPC(pq_hint)); } } } diff --git a/src/sql/optimizer/ob_log_group_by.cpp b/src/sql/optimizer/ob_log_group_by.cpp index 83bece9f0..49df0d1b5 100644 --- a/src/sql/optimizer/ob_log_group_by.cpp +++ b/src/sql/optimizer/ob_log_group_by.cpp @@ -532,6 +532,14 @@ int ObLogGroupBy::print_outline_data(PlanText &plan_text) LOG_WARN("failed to print hint", K(ret), K(hint)); } } + if (OB_SUCC(ret) && T_DISTRIBUTE_BASIC != dist_method_) { + ObPQHint hint(T_PQ_GBY_HINT); + hint.set_qb_name(qb_name); + hint.set_dist_method(dist_method_); + if (OB_FAIL(hint.print_hint(plan_text))) { + LOG_WARN("failed to print hint", K(ret), K(hint)); + } + } } return ret; } @@ -554,6 +562,10 @@ int ObLogGroupBy::print_used_hint(PlanText &plan_text) && static_cast(hint)->force_partition_sort() == use_part_sort_ && OB_FAIL(hint->print_hint(plan_text))) { LOG_WARN("failed to print used hint for group by", K(ret), K(*hint)); + } else if (NULL != (hint = get_plan()->get_log_plan_hint().get_normal_hint(T_PQ_GBY_HINT)) + && static_cast(hint)->is_dist_method_match(dist_method_) + && OB_FAIL(hint->print_hint(plan_text))) { + LOG_WARN("failed to print used hint for group by", K(ret), K(*hint)); } return ret; } diff --git a/src/sql/optimizer/ob_log_group_by.h b/src/sql/optimizer/ob_log_group_by.h index 61462d2d1..0ac2f8eb4 100644 --- a/src/sql/optimizer/ob_log_group_by.h +++ b/src/sql/optimizer/ob_log_group_by.h @@ -99,6 +99,7 @@ public: use_hash_aggr_(false), has_push_down_(false), use_part_sort_(false), + dist_method_(T_INVALID), is_pushdown_scalar_aggr_(false) {} virtual ~ObLogGroupBy() @@ -218,8 +219,17 @@ public: { return ObRollupStatus::ROLLUP_COLLECTOR == rollup_adaptive_info_.rollup_status_; } inline void set_force_push_down(bool force_push_down) { force_push_down_ = force_push_down; } - void set_group_by_outline_info(bool use_hash_aggr, bool has_push_down, bool use_part_sort = false) - { use_hash_aggr_ = use_hash_aggr; has_push_down_ = has_push_down; use_part_sort_ = use_part_sort; } + void set_group_by_outline_info(bool is_basic, + bool is_partition_wise, + bool use_hash_aggr, + bool has_push_down, + bool use_part_sort = false) + { + dist_method_ = is_basic ? T_DISTRIBUTE_BASIC : (is_partition_wise ? T_DISTRIBUTE_NONE : T_DISTRIBUTE_HASH); + use_hash_aggr_ = use_hash_aggr; + has_push_down_ = has_push_down; + use_part_sort_ = use_part_sort; + } virtual int get_plan_item_info(PlanText &plan_text, ObSqlPlanItem &plan_item) override; @@ -265,6 +275,7 @@ private: bool use_hash_aggr_; bool has_push_down_; bool use_part_sort_; + ObItemType dist_method_; bool is_pushdown_scalar_aggr_; }; } // end of namespace sql diff --git a/src/sql/optimizer/ob_log_plan.cpp b/src/sql/optimizer/ob_log_plan.cpp index fe9fa6972..9ae0ef781 100644 --- a/src/sql/optimizer/ob_log_plan.cpp +++ b/src/sql/optimizer/ob_log_plan.cpp @@ -5201,7 +5201,7 @@ int ObLogPlan::create_three_stage_group_plan(const ObIArray &group_b helper.rollup_id_expr_))) { LOG_WARN("failed to set rollup parallel info", K(ret)); } else { - third_group_by->set_group_by_outline_info(HASH_AGGREGATE == second_aggr_algo, true); + third_group_by->set_group_by_outline_info(false, false, HASH_AGGREGATE == second_aggr_algo, true); } } return ret; @@ -5353,36 +5353,70 @@ int ObLogPlan::candi_allocate_scala_group_by(const ObIArray &a const bool is_from_povit) { int ret = OB_SUCCESS; - ObSEArray best_plans; + ObSEArray groupby_plans; ObSEArray dummy_exprs; SMART_VAR(GroupingOpHelper, groupby_helper) { if (OB_FAIL(init_groupby_helper(dummy_exprs, dummy_exprs, agg_items, is_from_povit, groupby_helper))) { LOG_WARN("failed to init group by helper", K(ret)); - } else if (OB_FAIL(get_minimal_cost_candidates(candidates_.candidate_plans_, best_plans))) { - LOG_WARN("failed to get minimal cost candidate", K(ret)); + } else if (OB_FAIL(inner_candi_allocate_scala_group_by(agg_items, + having_exprs, + groupby_helper, + groupby_plans))) { + LOG_WARN("failed to inner candi allocate scala group by", K(ret)); + } else if (!groupby_plans.empty()) { + LOG_TRACE("succeed to allocate scala group by using hint", K(groupby_plans.count()), K(groupby_helper)); + OPT_TRACE("success to generate scala group plan with hint"); + } else if (OB_FAIL(get_log_plan_hint().check_status())) { + LOG_WARN("failed to generate plans with hint", K(ret)); + } else if (OB_FALSE_IT(groupby_helper.set_ignore_hint())) { + } else if (OB_FAIL(inner_candi_allocate_scala_group_by(agg_items, + having_exprs, + groupby_helper, + groupby_plans))) { + LOG_WARN("failed to inner candi allocate scala group by", K(ret)); } else { - for (int64_t i = 0; OB_SUCC(ret) && i < best_plans.count(); i++) { - OPT_TRACE("generate scala group by for plan:", best_plans.at(i)); - if (OB_FAIL(create_scala_group_plan(agg_items, - having_exprs, - is_from_povit, - groupby_helper, - best_plans.at(i).plan_tree_))) { - LOG_WARN("failed to create scala group by plan", K(ret)); - } else { /*do nothing*/ } - } - if (OB_SUCC(ret)) { - int64_t check_scope = OrderingCheckScope::CHECK_WINFUNC | - OrderingCheckScope::CHECK_DISTINCT | - OrderingCheckScope::CHECK_SET | - OrderingCheckScope::CHECK_ORDERBY; - if (OB_FAIL(update_plans_interesting_order_info(best_plans, check_scope))) { - LOG_WARN("failed to update plans interesting order info", K(ret)); - } else if (OB_FAIL(prune_and_keep_best_plans(best_plans))) { - LOG_WARN("failed to add plan", K(ret)); - } else { /*do nothing*/ } - } + LOG_TRACE("succeed to allocate scala group by ignore hint", K(groupby_plans.count()), K(groupby_helper)); + OPT_TRACE("success to generate scala group plan without hint"); + } + + if (OB_SUCC(ret)) { + int64_t check_scope = OrderingCheckScope::CHECK_WINFUNC | + OrderingCheckScope::CHECK_DISTINCT | + OrderingCheckScope::CHECK_SET | + OrderingCheckScope::CHECK_ORDERBY; + if (OB_FAIL(update_plans_interesting_order_info(groupby_plans, check_scope))) { + LOG_WARN("failed to update plans interesting order info", K(ret)); + } else if (OB_FAIL(prune_and_keep_best_plans(groupby_plans))) { + LOG_WARN("failed to add plan", K(ret)); + } else { /*do nothing*/ } + } + } + return ret; +} + +int ObLogPlan::inner_candi_allocate_scala_group_by(const ObIArray &agg_items, + const ObIArray &having_exprs, + GroupingOpHelper &groupby_helper, + ObIArray &groupby_plans) + +{ + int ret = OB_SUCCESS; + ObSEArray best_plans; + if (OB_FAIL(get_minimal_cost_candidates(candidates_.candidate_plans_, best_plans))) { + LOG_WARN("failed to get minimal cost candidate", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < best_plans.count(); i++) { + OPT_TRACE("generate scala group by for plan:", best_plans.at(i)); + if (OB_FAIL(create_scala_group_plan(agg_items, + having_exprs, + groupby_helper, + best_plans.at(i).plan_tree_))) { + LOG_WARN("failed to create scala group by plan", K(ret)); + } else if (NULL != best_plans.at(i).plan_tree_ && + OB_FAIL(groupby_plans.push_back(best_plans.at(i)))) { + LOG_WARN("failed to push merge group by", K(ret)); + } else { /*do nothing*/ } } } return ret; @@ -5390,7 +5424,6 @@ int ObLogPlan::candi_allocate_scala_group_by(const ObIArray &a int ObLogPlan::create_scala_group_plan(const ObIArray &aggr_items, const ObIArray &having_exprs, - const bool is_from_povit, GroupingOpHelper &groupby_helper, ObLogicalOperator *&top) { @@ -5411,23 +5444,24 @@ int ObLogPlan::create_scala_group_plan(const ObIArray &aggr_it LOG_WARN("failed to push group by into table scan", K(ret)); } else if (!top->is_distributed()) { OPT_TRACE("generate scala group plan without pushdown"); - if (OB_FAIL(allocate_scala_group_by_as_top(top, - aggr_items, - having_exprs, - is_from_povit, - origin_child_card))) { + if (!groupby_helper.allow_basic()) { + top = NULL; + OPT_TRACE("ignore basic scala group by hint"); + } else if (OB_FAIL(allocate_scala_group_by_as_top(top, + aggr_items, + having_exprs, + groupby_helper.is_from_povit_, + origin_child_card))) { LOG_WARN("failed to allocate scala group by as top", K(ret)); } else { - static_cast(top)->set_group_by_outline_info(false, false); + static_cast(top)->set_group_by_outline_info(true, false, false, false); } } else if (!groupby_helper.distinct_exprs_.empty() && OB_FAIL(top->check_sharding_compatible_with_reduce_expr(groupby_helper.distinct_exprs_, is_partition_wise))) { LOG_WARN("failed to check if sharding compatible with distinct expr", K(ret)); } else if (groupby_helper.can_three_stage_pushdown_ && - !(is_partition_wise && - !(top->is_parallel_more_than_part_cnt() && - get_optimizer_context().get_query_ctx()->optimizer_features_enable_version_ > COMPAT_VERSION_4_3_2))) { + !(is_partition_wise && groupby_helper.allow_partition_wise(top->is_parallel_more_than_part_cnt()))) { OPT_TRACE("generate three stage group plan"); if (NULL == groupby_helper.aggr_code_expr_ && OB_FAIL(prepare_three_stage_info(dummy_exprs, dummy_exprs, groupby_helper))) { @@ -5447,7 +5481,7 @@ int ObLogPlan::create_scala_group_plan(const ObIArray &aggr_it dummy_exprs, aggr_items, dummy_exprs, - is_from_povit, + groupby_helper.is_from_povit_, groupby_helper.group_ndv_, origin_child_card, is_partition_wise, @@ -5461,11 +5495,11 @@ int ObLogPlan::create_scala_group_plan(const ObIArray &aggr_it } else if (OB_FAIL(allocate_scala_group_by_as_top(top, aggr_items, having_exprs, - is_from_povit, + groupby_helper.is_from_povit_, origin_child_card))) { LOG_WARN("failed to allocate scala group by as top", K(ret)); } else { - static_cast(top)->set_group_by_outline_info(false, groupby_helper.can_basic_pushdown_ || is_partition_wise); + static_cast(top)->set_group_by_outline_info(false, is_partition_wise, false, groupby_helper.can_basic_pushdown_ || is_partition_wise); } } @@ -5576,6 +5610,9 @@ int ObLogPlan::init_groupby_helper(const ObIArray &group_exprs, ObSEArray group_rollup_exprs; bool push_group = false; groupby_helper.is_scalar_group_by_ = true; + groupby_helper.is_from_povit_ = is_from_povit; + groupby_helper.clear_ignore_hint(); + groupby_helper.optimizer_features_enable_version_ = get_log_plan_hint().optimizer_features_enable_version_; if (OB_FAIL(candidates_.get_best_plan(best_plan))) { LOG_WARN("failed to get best plan", K(ret)); } else if (OB_ISNULL(best_plan) || @@ -5591,7 +5628,10 @@ int ObLogPlan::init_groupby_helper(const ObIArray &group_exprs, } else if (OB_FAIL(get_log_plan_hint().get_aggregation_info(groupby_helper.force_use_hash_, groupby_helper.force_use_merge_, groupby_helper.force_part_sort_, - groupby_helper.force_normal_sort_))) { + groupby_helper.force_normal_sort_, + groupby_helper.force_basic_, + groupby_helper.force_partition_wise_, + groupby_helper.force_dist_hash_))) { LOG_WARN("failed to get aggregation info from hint", K(ret)); } else if (OB_FAIL(check_storage_groupby_pushdown(aggr_items, group_exprs, @@ -5696,18 +5736,24 @@ int ObLogPlan::init_distinct_helper(const ObIArray &distinct_exprs, ObSQLSessionInfo *session_info = NULL; bool push_distinct = false; distinct_helper.can_basic_pushdown_ = false; - distinct_helper.force_use_hash_ = get_log_plan_hint().use_hash_distinct(); - distinct_helper.force_use_merge_ = get_log_plan_hint().use_merge_distinct(); + distinct_helper.clear_ignore_hint(); + distinct_helper.optimizer_features_enable_version_ = get_log_plan_hint().optimizer_features_enable_version_; if (OB_FAIL(candidates_.get_best_plan(best_plan))) { LOG_WARN("failed to get best plan", K(ret)); } else if (OB_ISNULL(best_plan) || OB_ISNULL(get_stmt())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret)); - } else if (get_log_plan_hint().no_pushdown_distinct()) { - OPT_TRACE("hint disable pushdown distinct"); + } else if (OB_FAIL(get_log_plan_hint().get_distinct_info(distinct_helper.force_use_hash_, + distinct_helper.force_use_merge_, + distinct_helper.force_basic_, + distinct_helper.force_partition_wise_, + distinct_helper.force_dist_hash_))) { + LOG_WARN("failed to get distinct info from hint", K(ret)); } else if (OB_FAIL(check_storage_distinct_pushdown(distinct_exprs, distinct_helper.can_storage_pushdown_))) { LOG_WARN("failed to check can storage distinct pushdown", K(ret)); + } else if (get_log_plan_hint().no_pushdown_distinct()) { + OPT_TRACE("hint disable pushdown distinct"); } else if (OB_ISNULL(session_info = get_optimizer_context().get_session_info())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(session_info), K(ret)); diff --git a/src/sql/optimizer/ob_log_plan.h b/src/sql/optimizer/ob_log_plan.h index a645e52f1..ba8e46f3d 100644 --- a/src/sql/optimizer/ob_log_plan.h +++ b/src/sql/optimizer/ob_log_plan.h @@ -485,6 +485,17 @@ public: } virtual ~GroupingOpHelper() {} + void set_ignore_hint() { ignore_hint_ = true; } + void clear_ignore_hint() { ignore_hint_ = false; } + inline bool allow_basic() const { return ignore_hint_ || (!force_partition_wise_ && !force_dist_hash_); } + inline bool allow_dist_hash() const { return ignore_hint_ || (!force_basic_ && !force_partition_wise_); } + inline bool allow_partition_wise(bool parallel_more_than_part_cnt) const + { + bool disable_by_rule = parallel_more_than_part_cnt && optimizer_features_enable_version_ > COMPAT_VERSION_4_3_2; + return ignore_hint_ ? !disable_by_rule + : (disable_by_rule ? force_partition_wise_ : (!force_basic_ && !force_dist_hash_)); + } + bool can_storage_pushdown_; bool can_basic_pushdown_; bool can_three_stage_pushdown_; @@ -493,7 +504,13 @@ public: bool force_use_merge_; // has no_use_hash_aggregation/no_use_hash_distinct hint bool force_part_sort_; // force use partition sort for merge group by bool force_normal_sort_; // disable use partition sort for merge group by + bool force_basic_; // pq hint force use basic plan + bool force_partition_wise_; // pq hint force use partition wise plan + bool force_dist_hash_; // pq hint force use hash distributed method plan bool is_scalar_group_by_; + bool is_from_povit_; + bool ignore_hint_; + uint64_t optimizer_features_enable_version_; ObSEArray distinct_exprs_; // context for three stage group by push down @@ -518,7 +535,15 @@ public: K_(can_rollup_pushdown), K_(force_use_hash), K_(force_use_merge), + K_(force_part_sort), + K_(force_normal_sort), + K_(force_basic), + K_(force_partition_wise), + K_(force_dist_hash), K_(is_scalar_group_by), + K_(is_from_povit), + K_(ignore_hint), + K_(optimizer_features_enable_version), K_(distinct_exprs), K_(pushdown_groupby_columns), K_(group_ndv), @@ -675,6 +700,11 @@ public: const ObIArray &having_exprs, const bool is_from_povit); + int inner_candi_allocate_scala_group_by(const ObIArray &agg_items, + const ObIArray &having_exprs, + GroupingOpHelper &groupby_helper, + ObIArray &groupby_plans); + int prepare_three_stage_info(const ObIArray &group_by_exprs, const ObIArray &rollup_exprs, GroupingOpHelper &helper); @@ -712,7 +742,6 @@ public: int create_scala_group_plan(const ObIArray &agg_items, const ObIArray &having_exprs, - const bool is_from_povit, GroupingOpHelper &groupby_helper, ObLogicalOperator *&top); diff --git a/src/sql/optimizer/ob_select_log_plan.cpp b/src/sql/optimizer/ob_select_log_plan.cpp index a8c5cc54a..e65c3aacb 100644 --- a/src/sql/optimizer/ob_select_log_plan.cpp +++ b/src/sql/optimizer/ob_select_log_plan.cpp @@ -265,10 +265,28 @@ int ObSelectLogPlan::candi_allocate_normal_group_by(const ObIArray & rollup_directions, aggr_items, having_exprs, - is_from_povit, groupby_helper, groupby_plans))) { LOG_WARN("failed to candi allocate three stage group by", K(ret)); + } else if (!groupby_plans.empty()) { + LOG_TRACE("succeed to allocate three stage group by using hint", K(groupby_plans.count()), K(groupby_helper)); + OPT_TRACE("success to generate three stage group plan with hint"); + } else if (OB_FAIL(get_log_plan_hint().check_status())) { + LOG_WARN("failed to generate plans with hint", K(ret)); + } else if (OB_FALSE_IT(groupby_helper.set_ignore_hint())) { + } else if (OB_FAIL(candi_allocate_three_stage_group_by(reduce_exprs, + group_by_exprs, + group_directions, + rollup_exprs, + rollup_directions, + aggr_items, + having_exprs, + groupby_helper, + groupby_plans))) { + LOG_WARN("failed to candi allocate three stage group by", K(ret)); + } else { + LOG_TRACE("succeed to allocate three stage group by ignore hint", K(groupby_plans.count()), K(groupby_helper)); + OPT_TRACE("success to generate three stage group plan without hint"); } } else if (OB_FAIL(candi_allocate_normal_group_by(reduce_exprs, group_by_exprs, @@ -277,9 +295,7 @@ int ObSelectLogPlan::candi_allocate_normal_group_by(const ObIArray & rollup_directions, having_exprs, aggr_items, - is_from_povit, groupby_helper, - false, groupby_plans))) { LOG_WARN("failed to inner allocate normal group by", K(ret)); } else if (!groupby_plans.empty()) { @@ -287,6 +303,7 @@ int ObSelectLogPlan::candi_allocate_normal_group_by(const ObIArray & OPT_TRACE("success to generate group plan with hint"); } else if (OB_FAIL(get_log_plan_hint().check_status())) { LOG_WARN("failed to generate plans with hint", K(ret)); + } else if (OB_FALSE_IT(groupby_helper.set_ignore_hint())) { } else if (OB_FAIL(candi_allocate_normal_group_by(reduce_exprs, group_by_exprs, group_directions, @@ -294,9 +311,7 @@ int ObSelectLogPlan::candi_allocate_normal_group_by(const ObIArray & rollup_directions, having_exprs, aggr_items, - is_from_povit, groupby_helper, - true, groupby_plans))) { LOG_WARN("failed to inner allocate normal group by", K(ret)); } else { @@ -328,7 +343,6 @@ int ObSelectLogPlan::candi_allocate_three_stage_group_by(const ObIArray &rollup_directions, const ObIArray &aggr_items, const ObIArray &having_exprs, - const bool is_from_povit, GroupingOpHelper &groupby_helper, ObIArray &groupby_plans) { @@ -344,14 +358,13 @@ int ObSelectLogPlan::candi_allocate_three_stage_group_by(const ObIArrayis_distributed() && !reduce_exprs.empty() && - OB_FAIL(candidate_plan.plan_tree_->check_sharding_compatible_with_reduce_expr(reduce_exprs, - is_partition_wise))) { + } else if (!candidate_plan.plan_tree_->is_distributed() && !groupby_helper.allow_basic()) { + OPT_TRACE("ignore basic group by hint"); + } else if (candidate_plan.plan_tree_->is_distributed() && !reduce_exprs.empty() + && groupby_helper.allow_partition_wise(candidate_plan.plan_tree_->is_parallel_more_than_part_cnt()) + && OB_FAIL(candidate_plan.plan_tree_->check_sharding_compatible_with_reduce_expr(reduce_exprs, is_partition_wise))) { LOG_WARN("failed to check if sharding compatible with distinct expr", K(ret)); - } else if (!candidate_plan.plan_tree_->is_distributed() || - (is_partition_wise && - !(candidate_plan.plan_tree_->is_parallel_more_than_part_cnt() && - get_optimizer_context().get_query_ctx()->optimizer_features_enable_version_ > COMPAT_VERSION_4_3_2))) { + } else if (!candidate_plan.plan_tree_->is_distributed() || is_partition_wise) { bool part_sort_valid = !groupby_helper.force_normal_sort_ && !group_by_exprs.empty(); bool normal_sort_valid = !groupby_helper.force_part_sort_; if (OB_FAIL(update_part_sort_method(part_sort_valid, normal_sort_valid))) { @@ -363,7 +376,6 @@ int ObSelectLogPlan::candi_allocate_three_stage_group_by(const ObIArray &group_by_exprs, const GroupingOpHelper &groupby_helper, - const bool ignore_hint, bool &use_hash_valid, bool &use_merge_valid, bool &part_sort_valid, @@ -401,7 +412,7 @@ int ObSelectLogPlan::get_valid_aggr_algo(const ObIArray &group_by_ex { int ret = OB_SUCCESS; bool has_keep_aggr = false; - if (ignore_hint) { + if (groupby_helper.ignore_hint_) { use_hash_valid = true; use_merge_valid = true; part_sort_valid = true; @@ -470,9 +481,7 @@ int ObSelectLogPlan::candi_allocate_normal_group_by(const ObIArray & const ObIArray &rollup_directions, const ObIArray &having_exprs, const ObIArray &aggr_items, - const bool is_from_povit, GroupingOpHelper &groupby_helper, - const bool ignore_hint, ObIArray &groupby_plans) { int ret = OB_SUCCESS; @@ -481,7 +490,7 @@ int ObSelectLogPlan::candi_allocate_normal_group_by(const ObIArray & bool use_merge_valid = false; bool part_sort_valid = false; bool normal_sort_valid = false; - if (OB_FAIL(get_valid_aggr_algo(group_by_exprs, groupby_helper, ignore_hint, + if (OB_FAIL(get_valid_aggr_algo(group_by_exprs, groupby_helper, use_hash_valid, use_merge_valid, part_sort_valid, normal_sort_valid))) { LOG_WARN("failed to get valid aggr algo", K(ret)); @@ -500,7 +509,6 @@ int ObSelectLogPlan::candi_allocate_normal_group_by(const ObIArray & rollup_exprs, aggr_items, having_exprs, - is_from_povit, groupby_helper, candidate_plan.plan_tree_))) { LOG_WARN("failed to create hash group by plan", K(ret)); @@ -543,7 +551,6 @@ int ObSelectLogPlan::candi_allocate_normal_group_by(const ObIArray & rollup_directions, aggr_items, having_exprs, - is_from_povit, groupby_helper, candidate_plan, groupby_plans, @@ -572,13 +579,13 @@ int ObSelectLogPlan::should_create_rollup_pushdown_plan(ObLogicalOperator *top, LOG_WARN("logical operator is null", K(ret), K(top), K(session)); } else if (rollup_exprs.empty() || !groupby_helper.can_rollup_pushdown_) { // do nothing - } else if (top->is_distributed() && - OB_FAIL(top->check_sharding_compatible_with_reduce_expr(reduce_exprs, - is_partition_wise))) { + } else if (!top->is_distributed() && !groupby_helper.allow_basic()) { + // do nothing + } else if (top->is_distributed() && groupby_helper.allow_partition_wise(top->is_parallel_more_than_part_cnt()) + && OB_FAIL(top->check_sharding_compatible_with_reduce_expr(reduce_exprs, + is_partition_wise))) { LOG_WARN("failed to check is partition wise", K(ret)); - } else if (!top->is_distributed() || (is_partition_wise && - !(top->is_parallel_more_than_part_cnt() && - get_optimizer_context().get_query_ctx()->optimizer_features_enable_version_ > COMPAT_VERSION_4_3_2))) { + } else if (!top->is_distributed() || is_partition_wise) { // do nothing } else if (NULL == groupby_helper.rollup_id_expr_ && OB_FAIL(ObRawExprUtils::build_pseudo_rollup_id(get_optimizer_context().get_expr_factory(), @@ -715,7 +722,7 @@ int ObSelectLogPlan::create_rollup_pushdown_plan(const ObIArray &gro groupby_helper.rollup_id_expr_))) { LOG_WARN("failed to set rollup id expr", K(ret)); } else { - rollup_collector->set_group_by_outline_info(false, true); + rollup_collector->set_group_by_outline_info(false, false, false, true); } return ret; } @@ -725,7 +732,6 @@ int ObSelectLogPlan::create_hash_group_plan(const ObIArray &reduce_e const ObIArray &rollup_exprs, const ObIArray &aggr_items, const ObIArray &having_exprs, - const bool is_from_povit, GroupingOpHelper &groupby_helper, ObLogicalOperator *&top) { @@ -736,26 +742,28 @@ int ObSelectLogPlan::create_hash_group_plan(const ObIArray &reduce_e ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(top), K(get_stmt()), K(ret)); } else if (OB_FALSE_IT(origin_child_card = top->get_card())) { - } else if (top->is_distributed() && - OB_FAIL(top->check_sharding_compatible_with_reduce_expr(reduce_exprs, - is_partition_wise))) { + } else if (!top->is_distributed() && !groupby_helper.allow_basic()) { + top = NULL; + OPT_TRACE("ignore basic hash group by hint"); + } else if (top->is_distributed() && groupby_helper.allow_partition_wise(top->is_parallel_more_than_part_cnt()) + && OB_FAIL(top->check_sharding_compatible_with_reduce_expr(reduce_exprs, + is_partition_wise))) { LOG_WARN("failed to check if sharding compatible", K(ret)); - } else if (!top->is_distributed() || (is_partition_wise && - !(top->is_parallel_more_than_part_cnt() && - get_optimizer_context().get_query_ctx()->optimizer_features_enable_version_ > COMPAT_VERSION_4_3_2))) { + } else if (!top->is_distributed() || is_partition_wise) { + bool is_basic = !top->is_distributed(); if (OB_FAIL(allocate_group_by_as_top(top, AggregateAlgo::HASH_AGGREGATE, group_by_exprs, rollup_exprs, aggr_items, having_exprs, - is_from_povit, + groupby_helper.is_from_povit_, groupby_helper.group_ndv_, origin_child_card, is_partition_wise))) { LOG_WARN("failed to allocate group by as top", K(ret)); } else { - static_cast(top)->set_group_by_outline_info(true, false); + static_cast(top)->set_group_by_outline_info(is_basic, is_partition_wise, true, false); } } else { // allocate push down group by @@ -776,7 +784,7 @@ int ObSelectLogPlan::create_hash_group_plan(const ObIArray &reduce_e dummy_exprs, aggr_items, dummy_exprs, - is_from_povit, + groupby_helper.is_from_povit_, groupby_helper.group_ndv_, origin_child_card, is_partition_wise, @@ -807,12 +815,12 @@ int ObSelectLogPlan::create_hash_group_plan(const ObIArray &reduce_e rollup_exprs, aggr_items, having_exprs, - is_from_povit, + groupby_helper.is_from_povit_, groupby_helper.group_ndv_, origin_child_card))) { LOG_WARN("failed to allocate scala group by as top", K(ret)); } else { - static_cast(top)->set_group_by_outline_info(true, groupby_helper.can_basic_pushdown_); + static_cast(top)->set_group_by_outline_info(false, false, true, groupby_helper.can_basic_pushdown_); } } } @@ -955,7 +963,6 @@ int ObSelectLogPlan::create_merge_group_plan(const ObIArray &reduce_ const ObIArray &rollup_directions, const ObIArray &aggr_items, const ObIArray &having_exprs, - const bool is_from_povit, GroupingOpHelper &groupby_helper, CandidatePlan &candidate_plan, ObIArray &candidate_plans, @@ -973,7 +980,6 @@ int ObSelectLogPlan::create_merge_group_plan(const ObIArray &reduce_ rollup_directions, aggr_items, having_exprs, - is_from_povit, groupby_helper, part_sort_mgb_plan.plan_tree_, true, @@ -990,7 +996,6 @@ int ObSelectLogPlan::create_merge_group_plan(const ObIArray &reduce_ rollup_directions, aggr_items, having_exprs, - is_from_povit, groupby_helper, candidate_plan.plan_tree_, false, @@ -1010,7 +1015,6 @@ int ObSelectLogPlan::inner_create_merge_group_plan(const ObIArray &r const ObIArray &rollup_directions, const ObIArray &aggr_items, const ObIArray &having_exprs, - const bool is_from_povit, GroupingOpHelper &groupby_helper, ObLogicalOperator *&top, bool use_part_sort, @@ -1086,13 +1090,15 @@ int ObSelectLogPlan::inner_create_merge_group_plan(const ObIArray &r // 1. need generate partition sort plan, but need not sort // 2. if need sort and no further op needs the output order, not generate merge groupby top = NULL; - } else if (top->is_distributed() && - OB_FAIL(top->check_sharding_compatible_with_reduce_expr(reduce_exprs, - is_partition_wise))) { + } else if (!top->is_distributed() && !groupby_helper.allow_basic()) { + top = NULL; + OPT_TRACE("ignore basic group by hint"); + } else if (top->is_distributed() && groupby_helper.allow_partition_wise(top->is_parallel_more_than_part_cnt()) + && OB_FAIL(top->check_sharding_compatible_with_reduce_expr(reduce_exprs, + is_partition_wise))) { LOG_WARN("failed to check if sharding compatible with reduce expr", K(ret)); - } else if (!top->is_distributed() || (is_partition_wise && - !(top->is_parallel_more_than_part_cnt() && - get_optimizer_context().get_query_ctx()->optimizer_features_enable_version_ > COMPAT_VERSION_4_3_2))) { + } else if (!top->is_distributed() || is_partition_wise) { + bool is_basic = !top->is_distributed(); if (OB_FAIL(try_allocate_sort_as_top(top, sort_keys, need_sort, prefix_pos, part_cnt))) { LOG_WARN("failed to allocate sort as top", K(ret)); } else if (OB_FAIL(allocate_group_by_as_top(top, @@ -1101,13 +1107,13 @@ int ObSelectLogPlan::inner_create_merge_group_plan(const ObIArray &r rollup_exprs, aggr_items, having_exprs, - is_from_povit, + groupby_helper.is_from_povit_, groupby_helper.group_ndv_, origin_child_card, is_partition_wise))) { LOG_WARN("failed to allocate group by as top", K(ret)); } else { - static_cast(top)->set_group_by_outline_info(false, false, use_part_sort); + static_cast(top)->set_group_by_outline_info(is_basic, is_partition_wise, false, false, use_part_sort); } } else if (use_part_sort && OB_FAIL(create_hash_sortkey(part_cnt, sort_keys, hash_sortkey))) { @@ -1149,7 +1155,7 @@ int ObSelectLogPlan::inner_create_merge_group_plan(const ObIArray &r dummy_exprs, aggr_items, dummy_exprs, - is_from_povit, + groupby_helper.is_from_povit_, groupby_helper.group_ndv_, origin_child_card, should_pullup_gi, @@ -1187,12 +1193,12 @@ int ObSelectLogPlan::inner_create_merge_group_plan(const ObIArray &r rollup_exprs, aggr_items, having_exprs, - is_from_povit, + groupby_helper.is_from_povit_, groupby_helper.group_ndv_, origin_child_card))) { LOG_WARN("failed to allocate group by as top", K(ret)); } else { - static_cast(top)->set_group_by_outline_info(false, + static_cast(top)->set_group_by_outline_info(false, false, false, groupby_helper.can_basic_pushdown_, use_part_sort); } } @@ -1440,63 +1446,37 @@ int ObSelectLogPlan::candi_allocate_distinct() LOG_TRACE("distinct exprs is unique, no need distinct", K(distinct_exprs)); } else { SMART_VAR(GroupingOpHelper, distinct_helper) { - CandidatePlan candidate_plan; ObSEArray distinct_plans; - ObSEArray distinct_directions; ObSEArray dummy_items; if (OB_FAIL(init_distinct_helper(distinct_exprs, distinct_helper))) { LOG_WARN("failed to init distinct helper", K(ret)); - } else if (OB_FAIL(ObOptimizerUtil::get_default_directions(distinct_exprs.count(), - distinct_directions))) { - LOG_WARN("failed to generate default directions", K(ret)); } else if (distinct_helper.can_storage_pushdown_ && OB_FAIL(try_push_aggr_into_table_scan(candidates_.candidate_plans_, dummy_items, distinct_exprs))) { LOG_WARN("failed to try push distinct exprs into table scan", K(ret)); + } else if (OB_FAIL(inner_candi_allocate_distinct(distinct_helper, + reduce_exprs, + distinct_exprs, + distinct_plans))) { + LOG_WARN("failed to inner candi allocate distinct", K(ret)); + } else if (!distinct_plans.empty()) { + LOG_TRACE("succeed to allocate distinct using hint", K(distinct_plans.count()), K(distinct_helper)); + OPT_TRACE("success to generate distinct plan with hint"); + } else if (OB_FAIL(get_log_plan_hint().check_status())) { + LOG_WARN("failed to generate plans with hint", K(ret)); + } else if (OB_FALSE_IT(distinct_helper.set_ignore_hint())) { + } else if (OB_FAIL(inner_candi_allocate_distinct(distinct_helper, + reduce_exprs, + distinct_exprs, + distinct_plans))) { + + LOG_WARN("failed to inner candi allocate distinct", K(ret)); + } else { + LOG_TRACE("succeed to allocate distinct ignore hint", K(distinct_plans.count()), K(distinct_helper)); + OPT_TRACE("success to generate distinct plan without hint"); } - // create hash distinct - if (OB_SUCC(ret) && !distinct_helper.force_use_merge_) { - ObSEArray best_candidates; - if (OB_FAIL(get_minimal_cost_candidates(candidates_.candidate_plans_, best_candidates))) { - LOG_WARN("failed to get minimal cost candidates", K(ret)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < best_candidates.count(); i++) { - candidate_plan = best_candidates.at(i); - OPT_TRACE("generate hash distinct for plan:", candidate_plan); - if (OB_FAIL(create_hash_distinct_plan(candidate_plan.plan_tree_, - distinct_helper, - reduce_exprs, - distinct_exprs))) { - LOG_WARN("failed to create hash distinct plan", K(ret)); - } else if (OB_FAIL(distinct_plans.push_back(candidate_plan))) { - LOG_WARN("failed to push back hash distinct candidate plan", K(ret)); - } else { /*do nothing*/ } - } - } - } - - //create merge distinct plan - if (OB_SUCC(ret) && !distinct_helper.force_use_hash_) { - bool can_ignore_merge_plan = !(distinct_plans.empty() || distinct_helper.force_use_merge_); - bool is_plan_valid = false; - for(int64_t i = 0; OB_SUCC(ret) && i < candidates_.candidate_plans_.count(); i++) { - candidate_plan = candidates_.candidate_plans_.at(i); - OPT_TRACE("generate merge distinct for plan:", candidate_plan); - if (OB_FAIL(create_merge_distinct_plan(candidate_plan.plan_tree_, - distinct_helper, - reduce_exprs, - distinct_exprs, - distinct_directions, - is_plan_valid, - can_ignore_merge_plan))) { - LOG_WARN("failed to allocate merge distinct plan", K(ret)); - } else if (is_plan_valid && OB_FAIL(distinct_plans.push_back(candidate_plan))) { - LOG_WARN("failed to add merge distinct candidate plan", K(ret)); - } else { /*do nothing*/ } - } - } if (OB_SUCC(ret)) { int64_t check_scope = OrderingCheckScope::CHECK_SET | OrderingCheckScope::CHECK_ORDERBY; if (OB_FAIL(update_plans_interesting_order_info(distinct_plans, check_scope))) { @@ -1510,6 +1490,56 @@ int ObSelectLogPlan::candi_allocate_distinct() return ret; } +int ObSelectLogPlan::inner_candi_allocate_distinct(const GroupingOpHelper &distinct_helper, + const ObIArray &reduce_exprs, + const ObIArray &distinct_exprs, + ObIArray &distinct_plans) +{ + int ret = OB_SUCCESS; + CandidatePlan candidate_plan; + // create hash distinct + if (OB_SUCC(ret) && !distinct_helper.force_use_merge_) { + ObSEArray best_candidates; + if (OB_FAIL(get_minimal_cost_candidates(candidates_.candidate_plans_, best_candidates))) { + LOG_WARN("failed to get minimal cost candidates", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < best_candidates.count(); i++) { + candidate_plan = best_candidates.at(i); + OPT_TRACE("generate hash distinct for plan:", candidate_plan); + if (OB_FAIL(create_hash_distinct_plan(candidate_plan.plan_tree_, + distinct_helper, + reduce_exprs, + distinct_exprs))) { + LOG_WARN("failed to create hash distinct plan", K(ret)); + } else if (NULL != candidate_plan.plan_tree_ + && OB_FAIL(distinct_plans.push_back(candidate_plan))) { + LOG_WARN("failed to push back hash distinct candidate plan", K(ret)); + } else { /*do nothing*/ } + } + } + } + + //create merge distinct plan + if (OB_SUCC(ret) && !distinct_helper.force_use_hash_) { + bool can_ignore_merge_plan = !(distinct_plans.empty() || distinct_helper.force_use_merge_); + for(int64_t i = 0; OB_SUCC(ret) && i < candidates_.candidate_plans_.count(); i++) { + candidate_plan = candidates_.candidate_plans_.at(i); + OPT_TRACE("generate merge distinct for plan:", candidate_plan); + if (OB_FAIL(create_merge_distinct_plan(candidate_plan.plan_tree_, + distinct_helper, + reduce_exprs, + distinct_exprs, + can_ignore_merge_plan))) { + LOG_WARN("failed to allocate merge distinct plan", K(ret)); + } else if (NULL != candidate_plan.plan_tree_ + && OB_FAIL(distinct_plans.push_back(candidate_plan))) { + LOG_WARN("failed to add merge distinct candidate plan", K(ret)); + } else { /*do nothing*/ } + } + } + return ret; +} + int ObSelectLogPlan::get_distinct_exprs(const ObLogicalOperator *top, ObIArray &reduce_exprs, ObIArray &distinct_exprs) @@ -1561,9 +1591,9 @@ int ObSelectLogPlan::get_distinct_exprs(const ObLogicalOperator *top, } int ObSelectLogPlan::create_hash_distinct_plan(ObLogicalOperator *&top, - GroupingOpHelper &distinct_helper, - ObIArray &reduce_exprs, - ObIArray &distinct_exprs) + const GroupingOpHelper &distinct_helper, + const ObIArray &reduce_exprs, + const ObIArray &distinct_exprs) { int ret = OB_SUCCESS; bool is_partition_wise = false; @@ -1571,14 +1601,14 @@ int ObSelectLogPlan::create_hash_distinct_plan(ObLogicalOperator *&top, if (OB_ISNULL(top)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(top), K(ret)); - } else if (top->is_distributed() && - OB_FAIL(top->check_sharding_compatible_with_reduce_expr(reduce_exprs, - is_partition_wise))) { + } else if (!top->is_distributed() && !distinct_helper.allow_basic()) { + top = NULL; + OPT_TRACE("ignore basic distinct by hint"); + } else if (top->is_distributed() && distinct_helper.allow_partition_wise(top->is_parallel_more_than_part_cnt()) + && OB_FAIL(top->check_sharding_compatible_with_reduce_expr(reduce_exprs, + is_partition_wise))) { LOG_WARN("failed to check sharding compatible with reduce expr", K(ret)); - } else if (!top->is_distributed() || - (is_partition_wise && - !(top->is_parallel_more_than_part_cnt() && - get_optimizer_context().get_query_ctx()->optimizer_features_enable_version_ > COMPAT_VERSION_4_3_2))) { + } else if (!top->is_distributed() || is_partition_wise) { OPT_TRACE("is basic distinct:", !top->is_distributed()); OPT_TRACE("is partition wise distinct", is_partition_wise); if (OB_FAIL(allocate_distinct_as_top(top, @@ -1613,11 +1643,9 @@ int ObSelectLogPlan::create_hash_distinct_plan(ObLogicalOperator *&top, } int ObSelectLogPlan::create_merge_distinct_plan(ObLogicalOperator *&top, - GroupingOpHelper &distinct_helper, - ObIArray &reduce_exprs, - ObIArray &distinct_exprs, - ObIArray &directions, - bool &is_plan_valid, + const GroupingOpHelper &distinct_helper, + const ObIArray &reduce_exprs, + const ObIArray &in_distinct_exprs, bool can_ignore_merge_plan) { int ret = OB_SUCCESS; @@ -1626,11 +1654,19 @@ int ObSelectLogPlan::create_merge_distinct_plan(ObLogicalOperator *&top, bool is_partition_wise = false; ObSEArray sort_keys; int64_t interesting_order_info = OrderingFlag::NOT_MATCH; - is_plan_valid = true; + ObSEArray distinct_exprs; + ObSEArray directions; OPT_TRACE("start generate merge distinct plan"); if (OB_ISNULL(top)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret)); + } else if (!top->is_distributed() && !distinct_helper.allow_basic()) { + top = NULL; + OPT_TRACE("ignore basic distinct by hint"); + } else if (OB_FAIL(distinct_exprs.assign(in_distinct_exprs))) { + LOG_WARN("failed to assign distinct exprs", K(ret)); + } else if (OB_FAIL(ObOptimizerUtil::get_default_directions(distinct_exprs.count(), directions))) { + LOG_WARN("failed to generate default directions", K(ret)); } else if (OB_FAIL(adjust_sort_expr_ordering(distinct_exprs, directions, *top, @@ -1660,15 +1696,12 @@ int ObSelectLogPlan::create_merge_distinct_plan(ObLogicalOperator *&top, LOG_WARN("failed to compute stmt interesting order", K(ret)); } else if (need_sort && can_ignore_merge_plan && OrderingFlag::NOT_MATCH == interesting_order_info) { // if no further order needed, not generate merge style distinct - is_plan_valid = false; - } else if (top->is_distributed() && - OB_FAIL(top->check_sharding_compatible_with_reduce_expr(reduce_exprs, - is_partition_wise))) { + top = NULL; + } else if (top->is_distributed() && distinct_helper.allow_partition_wise(top->is_parallel_more_than_part_cnt()) + && OB_FAIL(top->check_sharding_compatible_with_reduce_expr(reduce_exprs, + is_partition_wise))) { LOG_WARN("failed to check sharding compatible with reduce exprs", K(ret)); - } else if (!top->is_distributed() || - (is_partition_wise && - !(top->is_parallel_more_than_part_cnt() && - get_optimizer_context().get_query_ctx()->optimizer_features_enable_version_ > COMPAT_VERSION_4_3_2))) { + } else if (!top->is_distributed() || is_partition_wise) { OPT_TRACE("is basic distinct:", !top->is_distributed()); OPT_TRACE("is partition wise distinct", is_partition_wise); if (OB_FAIL(try_allocate_sort_as_top(top, sort_keys, need_sort, prefix_pos))) { diff --git a/src/sql/optimizer/ob_select_log_plan.h b/src/sql/optimizer/ob_select_log_plan.h index 0ae361ee9..cc292c6a8 100644 --- a/src/sql/optimizer/ob_select_log_plan.h +++ b/src/sql/optimizer/ob_select_log_plan.h @@ -83,7 +83,6 @@ private: int get_valid_aggr_algo(const ObIArray &group_by_exprs, const GroupingOpHelper &groupby_helper, - const bool ignore_hint, bool &use_hash_valid, bool &use_merge_valid, bool &part_sort_valid, @@ -96,9 +95,7 @@ private: const ObIArray &rollup_directions, const ObIArray &having_exprs, const ObIArray &aggr_items, - const bool is_from_povit, GroupingOpHelper &groupby_helper, - const bool ignore_hint, ObIArray &groupby_plans); int candi_allocate_three_stage_group_by(const ObIArray &reduce_exprs, @@ -108,7 +105,6 @@ private: const ObIArray &rollup_directions, const ObIArray &aggr_items, const ObIArray &having_exprs, - const bool is_from_povit, GroupingOpHelper &groupby_helper, ObIArray &groupby_plans); @@ -117,7 +113,6 @@ private: const ObIArray &rollup_exprs, const ObIArray &aggr_items, const ObIArray &having_exprs, - const bool is_from_povit, GroupingOpHelper &groupby_helper, ObLogicalOperator *&top); @@ -159,7 +154,6 @@ private: const ObIArray &rollup_directions, const ObIArray &aggr_items, const ObIArray &having_exprs, - const bool is_from_povit, GroupingOpHelper &groupby_helper, CandidatePlan &candidate_plan, ObIArray &candidate_plans, @@ -186,17 +180,20 @@ private: common::ObIArray &reduce_exprs, common::ObIArray &distinct_exprs); + int inner_candi_allocate_distinct(const GroupingOpHelper &distinct_helper, + const ObIArray &reduce_exprs, + const ObIArray &distinct_exprs, + ObIArray &distinct_plans); + int create_hash_distinct_plan(ObLogicalOperator *&top, - GroupingOpHelper &distinct_helper, - ObIArray &reduce_exprs, - ObIArray &distinct_exprs); + const GroupingOpHelper &distinct_helper, + const ObIArray &reduce_exprs, + const ObIArray &distinct_exprs); int create_merge_distinct_plan(ObLogicalOperator *&top, - GroupingOpHelper &distinct_helper, - ObIArray &reduce_exprs, - ObIArray &distinct_exprs, - ObIArray &directions, - bool &is_plan_valid, + const GroupingOpHelper &distinct_helper, + const ObIArray &reduce_exprs, + const ObIArray &distinct_exprs, bool can_ignore_merge_plan = false); int allocate_distinct_as_top(ObLogicalOperator *&top, @@ -924,7 +921,6 @@ int generate_window_functions_plan(WinFuncOpHelper &win_func_helper, const ObIArray &rollup_directions, const ObIArray &aggr_items, const ObIArray &having_exprs, - const bool is_from_povit, GroupingOpHelper &groupby_helper, ObLogicalOperator *&top, bool use_part_sort, diff --git a/src/sql/parser/sql_parser_mysql_mode.l b/src/sql/parser/sql_parser_mysql_mode.l index 47f2a1465..964e9715e 100644 --- a/src/sql/parser/sql_parser_mysql_mode.l +++ b/src/sql/parser/sql_parser_mysql_mode.l @@ -1086,6 +1086,7 @@ Timestamp{whitespace}?\"[^\"]*\" { BC2HOST { return BC2HOST; } RANGE { return RANGE; } LIST { return LIST; } +BASIC { return BASIC; } [-] { return NEG_SIGN; } MERGE { return MERGE_HINT; } NO_MERGE { return NO_MERGE_HINT; } @@ -1171,6 +1172,8 @@ Timestamp{whitespace}?\"[^\"]*\" { PUSHDOWN { return PUSHDOWN; } DECORRELATE { return DECORRELATE; } NO_DECORRELATE { return NO_DECORRELATE; } +PQ_GBY { return PQ_GBY; } +PQ_DISTINCT { return PQ_DISTINCT; } {identifier} { if (!(IS_FAST_PARAMETERIZE)) { check_value(yylval); diff --git a/src/sql/parser/sql_parser_mysql_mode.y b/src/sql/parser/sql_parser_mysql_mode.y index 695d2bc71..1b6a44557 100644 --- a/src/sql/parser/sql_parser_mysql_mode.y +++ b/src/sql/parser/sql_parser_mysql_mode.y @@ -190,7 +190,7 @@ USE_HASH_AGGREGATION NO_USE_HASH_AGGREGATION PARTITION_SORT NO_PARTITION_SORT WF_TOPN USE_LATE_MATERIALIZATION NO_USE_LATE_MATERIALIZATION PX_JOIN_FILTER NO_PX_JOIN_FILTER PX_PART_JOIN_FILTER NO_PX_PART_JOIN_FILTER -PQ_MAP PQ_DISTRIBUTE PQ_DISTRIBUTE_WINDOW PQ_SET PQ_SUBQUERY RANDOM_LOCAL BROADCAST BC2HOST LIST +PQ_MAP PQ_DISTRIBUTE PQ_DISTRIBUTE_WINDOW PQ_SET PQ_SUBQUERY PQ_GBY PQ_DISTINCT RANDOM_LOCAL BROADCAST BC2HOST LIST GBY_PUSHDOWN NO_GBY_PUSHDOWN USE_HASH_DISTINCT NO_USE_HASH_DISTINCT DISTINCT_PUSHDOWN NO_DISTINCT_PUSHDOWN @@ -11285,6 +11285,14 @@ INDEX_HINT '(' qb_name_option relation_factor_in_hint NAME_OB opt_index_prefix ' { malloc_non_terminal_node($$, result->malloc_pool_, T_NO_USE_COLUMN_STORE_HINT, 2, $3, $4); } +| PQ_GBY '(' qb_name_option distribute_method ')' +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_PQ_GBY_HINT, 2, $3, $4); +} +| PQ_DISTINCT '(' qb_name_option distribute_method ')' +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_PQ_DISTINCT_HINT, 2, $3, $4); +} ; opt_index_prefix: @@ -11531,6 +11539,10 @@ ALL { malloc_terminal_node($$, result->malloc_pool_, T_DISTRIBUTE_LIST); } +| BASIC +{ + malloc_terminal_node($$, result->malloc_pool_, T_DISTRIBUTE_BASIC); +} ; limit_expr: diff --git a/src/sql/resolver/dml/ob_dml_resolver.cpp b/src/sql/resolver/dml/ob_dml_resolver.cpp index 4ba31d2c3..01aaafd98 100755 --- a/src/sql/resolver/dml/ob_dml_resolver.cpp +++ b/src/sql/resolver/dml/ob_dml_resolver.cpp @@ -14617,6 +14617,13 @@ int ObDMLResolver::resolve_optimize_hint(const ParseNode &hint_node, } break; } + case T_PQ_GBY_HINT: + case T_PQ_DISTINCT_HINT: { + if (OB_FAIL(resolve_normal_pq_hint(hint_node, opt_hint))) { + LOG_WARN("failed to resolve normal pq hint.", K(ret)); + } + break; + } case T_USE_LATE_MATERIALIZATION: case T_NO_USE_LATE_MATERIALIZATION: case T_GBY_PUSHDOWN: @@ -15098,6 +15105,34 @@ int ObDMLResolver::resolve_pq_subquery_hint(const ParseNode &hint_node, return ret; } +int ObDMLResolver::resolve_normal_pq_hint(const ParseNode &hint_node, + ObOptHint *&opt_hint) +{ + int ret = OB_SUCCESS; + opt_hint = NULL; + const ParseNode *dist_method_node = NULL; + ObPQHint *pq_hint = NULL; + ObString qb_name; + if (OB_UNLIKELY(T_PQ_GBY_HINT != hint_node.type_ && T_PQ_DISTINCT_HINT != hint_node.type_) + || OB_UNLIKELY(2 != hint_node.num_child_) + || OB_ISNULL(dist_method_node = hint_node.children_[1])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected pq_gby/pq_distinct hint node", K(ret), K(hint_node.num_child_), + K(get_type_name(hint_node.type_))); + } else if (NULL == ObPQHint::get_dist_method_str(dist_method_node->type_)) { + LOG_TRACE("invalid normal pq hint dist_method_node", K(get_type_name(dist_method_node->type_))); + } else if (OB_FAIL(ObQueryHint::create_hint(allocator_, hint_node.type_, pq_hint))) { + LOG_WARN("failed to create hint", K(ret)); + } else if (OB_FAIL(resolve_qb_name_node(hint_node.children_[0], qb_name))) { + LOG_WARN("failed to resolve query block name", K(ret)); + } else { + pq_hint->set_qb_name(qb_name); + pq_hint->set_dist_method(dist_method_node->type_); + opt_hint = pq_hint; + } + return ret; +} + int ObDMLResolver::resolve_join_filter_hint(const ParseNode &hint_node, ObOptHint *&opt_hint) { diff --git a/src/sql/resolver/dml/ob_dml_resolver.h b/src/sql/resolver/dml/ob_dml_resolver.h index c29907b91..6e8f06689 100644 --- a/src/sql/resolver/dml/ob_dml_resolver.h +++ b/src/sql/resolver/dml/ob_dml_resolver.h @@ -946,6 +946,7 @@ private: int resolve_pq_distribute_hint(const ParseNode &hint_node, ObOptHint *&opt_hint); int resolve_pq_set_hint(const ParseNode &hint_node, ObOptHint *&opt_hint); int resolve_pq_subquery_hint(const ParseNode &hint_node, ObOptHint *&opt_hint); + int resolve_normal_pq_hint(const ParseNode &hint_node, ObOptHint *&opt_hint); int resolve_join_filter_hint(const ParseNode &join_node, ObOptHint *&opt_hint); int resolve_aggregation_hint(const ParseNode &hint_node, ObOptHint *&hint); int resolve_normal_transform_hint(const ParseNode &hint_node, ObTransHint *&hint); diff --git a/src/sql/resolver/dml/ob_hint.cpp b/src/sql/resolver/dml/ob_hint.cpp index b838c6051..1275f485b 100644 --- a/src/sql/resolver/dml/ob_hint.cpp +++ b/src/sql/resolver/dml/ob_hint.cpp @@ -1160,6 +1160,8 @@ const char* ObHint::get_hint_name(ObItemType type, bool is_enable_hint /* defaul case T_USE_DISTRIBUTED_DML: return is_enable_hint ? "USE_DISTRIBUTED_DML" : "NO_USE_DISTRIBUTED_DML"; case T_TABLE_DYNAMIC_SAMPLING: return "DYNAMIC_SAMPLING"; case T_PQ_SUBQUERY: return "PQ_SUBQUERY"; + case T_PQ_GBY_HINT: return "PQ_GBY"; + case T_PQ_DISTINCT_HINT: return "PQ_DISTINCT"; default: return NULL; } } @@ -2527,6 +2529,43 @@ int ObPQSubqueryHint::print_hint_desc(PlanText &plan_text) const return ret; } +int ObPQHint::assign(const ObPQHint &other) +{ + int ret = OB_SUCCESS; + dist_method_ = other.dist_method_; + if (OB_FAIL(ObOptHint::assign(other))) { + LOG_WARN("fail to assign hint", K(ret)); + } + return ret; +} + +int ObPQHint::print_hint_desc(PlanText &plan_text) const +{ + int ret = OB_SUCCESS; + char *buf = plan_text.buf_; + int64_t &buf_len = plan_text.buf_len_; + int64_t &pos = plan_text.pos_; + const char *str = NULL; + if (OB_ISNULL(str = ObPQHint::get_dist_method_str(dist_method_))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(dist_method_)); + } else if (OB_FAIL(BUF_PRINTF(" %s", str))) { + LOG_WARN("failed to print dist method", K(ret)); + } + return ret; +} + +const char *ObPQHint::get_dist_method_str(ObItemType dist_method) +{ + switch (dist_method) { + case T_DISTRIBUTE_BASIC: return "BASIC"; + case T_DISTRIBUTE_NONE: return "NONE"; + case T_DISTRIBUTE_HASH: return "HASH"; + default: return NULL; + } + return NULL; +}; + int ObJoinOrderHint::print_hint_desc(PlanText &plan_text) const { int ret = OB_SUCCESS; diff --git a/src/sql/resolver/dml/ob_hint.h b/src/sql/resolver/dml/ob_hint.h index 40afd02ca..d914dbe38 100644 --- a/src/sql/resolver/dml/ob_hint.h +++ b/src/sql/resolver/dml/ob_hint.h @@ -525,7 +525,8 @@ public: HINT_TABLE_PARALLEL, HINT_PQ_SET, HINT_JOIN_FILTER, - HINT_TABLE_DYNAMIC_SAMPLING + HINT_TABLE_DYNAMIC_SAMPLING, + HINT_PQ }; static const int64_t MAX_EXPR_STR_LENGTH_IN_HINT = 1024; @@ -992,7 +993,7 @@ public: const ObTableInHint &get_table() const { return table_; } int64_t get_parallel() const { return parallel_; } void set_parallel(int64_t parallel) { parallel_ = parallel; } - INHERIT_TO_STRING_KV("ObHint", ObHint, K_(table), K_(table), K_(parallel)); + INHERIT_TO_STRING_KV("ObHint", ObHint, K_(table), K_(parallel)); private: ObTableInHint table_; @@ -1114,6 +1115,31 @@ private: QbNameList sub_qb_names_; }; +// normal pq hint for single child op: group by/distinct +class ObPQHint : public ObOptHint +{ + public: + ObPQHint(ObItemType hint_type) + : ObOptHint(hint_type), + dist_method_(T_INVALID) + { + set_hint_class(HINT_PQ); + } + int assign(const ObPQHint &other); + virtual ~ObPQHint() {} + virtual int print_hint_desc(PlanText &plan_text) const override; + static const char* get_dist_method_str(ObItemType dist_method); + void set_dist_method(ObItemType dist_method) { dist_method_ = dist_method; } + inline bool is_dist_method_match(ObItemType dist_method) const { return dist_method_ == dist_method; } + inline bool is_force_basic() const { return T_DISTRIBUTE_BASIC == dist_method_; } + inline bool is_force_partition_wise() const { return T_DISTRIBUTE_NONE == dist_method_; } + inline bool is_force_dist_hash() const { return T_DISTRIBUTE_HASH == dist_method_; } + + INHERIT_TO_STRING_KV("ObHint", ObHint, K_(dist_method)); +private: + ObItemType dist_method_; +}; + class ObJoinOrderHint : public ObOptHint { public: ObJoinOrderHint(ObItemType hint_type = T_LEADING) diff --git a/src/sql/resolver/dml/ob_sql_hint.cpp b/src/sql/resolver/dml/ob_sql_hint.cpp index 0d6264a0f..64c6e1a9f 100644 --- a/src/sql/resolver/dml/ob_sql_hint.cpp +++ b/src/sql/resolver/dml/ob_sql_hint.cpp @@ -1405,7 +1405,7 @@ void ObLogPlanHint::reset() table_hints_.reuse(); join_hints_.reuse(); normal_hints_.reuse(); - enable_index_prefix_ = false; + optimizer_features_enable_version_ = LASTED_COMPAT_VERSION; } #ifndef OB_BUILD_SPM @@ -1425,7 +1425,7 @@ int ObLogPlanHint::init_log_plan_hint(ObSqlSchemaGuard &schema_guard, #ifdef OB_BUILD_SPM is_spm_evolution_ = is_spm_evolution; #endif - enable_index_prefix_ = (stmt.get_query_ctx()->optimizer_features_enable_version_ >= COMPAT_VERSION_4_2_3); + optimizer_features_enable_version_ = stmt.get_query_ctx()->optimizer_features_enable_version_; const ObStmtHint &stmt_hint = stmt.get_stmt_hint(); if (OB_FAIL(join_order_.init_leading_info(stmt, query_hint, stmt_hint.get_normal_hint(T_LEADING)))) { LOG_WARN("failed to get leading hint info", K(ret)); @@ -1721,14 +1721,23 @@ bool ObLogPlanHint::has_disable_hint(ObItemType hint_type) const int ObLogPlanHint::get_aggregation_info(bool &force_use_hash, bool &force_use_merge, bool &force_part_sort, - bool &force_normal_sort) const + bool &force_normal_sort, + bool &force_basic, + bool &force_partition_wise, + bool &force_dist_hash) const { int ret = OB_SUCCESS; force_use_hash = false; force_use_merge = false; force_part_sort = false; force_normal_sort = false; + force_basic = false; + force_partition_wise = false; + force_dist_hash = false; const ObAggHint *agg_hint = static_cast(get_normal_hint(T_USE_HASH_AGGREGATE)); + const ObPQHint *pq_hint = static_cast(get_normal_hint(T_PQ_GBY_HINT)); + const bool enable_pq_hint = COMPAT_VERSION_4_3_3 <= optimizer_features_enable_version_ + || COMPAT_VERSION_4_2_4 < optimizer_features_enable_version_; if (NULL != agg_hint) { force_use_hash = agg_hint->is_enable_hint(); force_use_merge = agg_hint->is_disable_hint(); @@ -1741,6 +1750,51 @@ int ObLogPlanHint::get_aggregation_info(bool &force_use_hash, force_use_merge = true; force_normal_sort = true; } + if (OB_FAIL(ret) || !enable_pq_hint) { + } else if (NULL != pq_hint) { + force_basic = pq_hint->is_force_basic(); + force_partition_wise = pq_hint->is_force_partition_wise(); + force_dist_hash = pq_hint->is_force_dist_hash(); + } else if (is_outline_data_) { + force_basic = true; + force_partition_wise = false; + force_dist_hash = false; + } + return ret; +} + +int ObLogPlanHint::get_distinct_info(bool &force_use_hash, + bool &force_use_merge, + bool &force_basic, + bool &force_partition_wise, + bool &force_dist_hash) const +{ + int ret = OB_SUCCESS; + force_use_hash = false; + force_use_merge = false; + force_basic = false; + force_partition_wise = false; + force_dist_hash = false; + const ObHint *method_hint = static_cast(get_normal_hint(T_USE_HASH_DISTINCT)); + const ObPQHint *pq_hint = static_cast(get_normal_hint(T_PQ_DISTINCT_HINT)); + const bool enable_pq_hint = COMPAT_VERSION_4_3_3 <= optimizer_features_enable_version_ + || COMPAT_VERSION_4_2_4 < optimizer_features_enable_version_; + if (NULL != method_hint) { + force_use_hash = method_hint->is_enable_hint(); + force_use_merge = method_hint->is_disable_hint(); + } else if (is_outline_data_) { + force_use_merge = true; + } + if (OB_FAIL(ret) || !enable_pq_hint) { + } else if (NULL != pq_hint) { + force_basic = pq_hint->is_force_basic(); + force_partition_wise = pq_hint->is_force_partition_wise(); + force_dist_hash = pq_hint->is_force_dist_hash(); + } else if (is_outline_data_) { + force_basic = true; + force_partition_wise = false; + force_dist_hash = false; + } return ret; } @@ -1941,7 +1995,8 @@ int ObLogPlanHint::get_index_prefix(const uint64_t table_id, { int ret = OB_SUCCESS; const LogTableHint *log_table_hint = NULL; - if (!enable_index_prefix_ || OB_ISNULL(log_table_hint = get_index_hint(table_id))) { + if (optimizer_features_enable_version_ < COMPAT_VERSION_4_2_3 + || OB_ISNULL(log_table_hint = get_index_hint(table_id))) { //do nothing } else if (!log_table_hint->is_use_index_hint()) { //do nothing diff --git a/src/sql/resolver/dml/ob_sql_hint.h b/src/sql/resolver/dml/ob_sql_hint.h index 9b41aabdf..5ac9ad1dc 100644 --- a/src/sql/resolver/dml/ob_sql_hint.h +++ b/src/sql/resolver/dml/ob_sql_hint.h @@ -470,7 +470,15 @@ struct ObLogPlanHint int get_aggregation_info(bool &force_use_hash, bool &force_use_merge, bool &force_part_sort, - bool &force_normal_sort) const; + bool &force_normal_sort, + bool &force_basic, + bool &force_partition_wise, + bool &force_dist_hash) const; + int get_distinct_info(bool &force_use_hash, + bool &force_use_merge, + bool &force_basic, + bool &force_partition_wise, + bool &force_dist_hash) const; int get_valid_pq_subquery_hint(const ObIArray &sub_qb_names, const ObPQSubqueryHint *&explicit_hint, const ObPQSubqueryHint *&implicit_hint) const; @@ -482,12 +490,8 @@ struct ObLogPlanHint bool use_late_material() const { return has_enable_hint(T_USE_LATE_MATERIALIZATION); } bool no_use_late_material() const { return has_disable_hint(T_USE_LATE_MATERIALIZATION); } - bool use_hash_aggregate() const { return has_enable_hint(T_USE_HASH_AGGREGATE); } - bool use_merge_aggregate() const { return has_disable_hint(T_USE_HASH_AGGREGATE); } bool pushdown_group_by() const { return has_enable_hint(T_GBY_PUSHDOWN); } bool no_pushdown_group_by() const { return has_disable_hint(T_GBY_PUSHDOWN); } - bool use_hash_distinct() const { return has_enable_hint(T_USE_HASH_DISTINCT); } - bool use_merge_distinct() const { return has_disable_hint(T_USE_HASH_DISTINCT); } bool pushdown_distinct() const { return has_enable_hint(T_DISTINCT_PUSHDOWN); } bool no_pushdown_distinct() const { return has_disable_hint(T_DISTINCT_PUSHDOWN); } bool use_distributed_dml() const { return has_enable_hint(T_USE_DISTRIBUTED_DML); } @@ -497,7 +501,7 @@ struct ObLogPlanHint TO_STRING_KV(K_(is_outline_data), K_(join_order), K_(table_hints), K_(join_hints), - K_(normal_hints), K_(enable_index_prefix)); + K_(normal_hints), K_(optimizer_features_enable_version)); bool is_outline_data_; #ifdef OB_BUILD_SPM @@ -507,7 +511,7 @@ struct ObLogPlanHint common::ObSEArray table_hints_; common::ObSEArray join_hints_; common::ObSEArray normal_hints_; - bool enable_index_prefix_; + uint64_t optimizer_features_enable_version_; }; } From a98241c7ecf28ab1c3f38e44453280b4adba2db7 Mon Sep 17 00:00:00 2001 From: qingzhu521 Date: Wed, 21 Aug 2024 12:45:43 +0000 Subject: [PATCH 156/249] =?UTF-8?q?fix=20user=20variable=E5=92=8Cmysql?= =?UTF-8?q?=E4=B8=8D=E5=90=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/sql/parser/sql_parser_mysql_mode.l | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sql/parser/sql_parser_mysql_mode.l b/src/sql/parser/sql_parser_mysql_mode.l index 964e9715e..6ab8c0ce8 100644 --- a/src/sql/parser/sql_parser_mysql_mode.l +++ b/src/sql/parser/sql_parser_mysql_mode.l @@ -65,7 +65,7 @@ comment ({sql_comment}) start_identifier (([A-Za-z0-9$_]*[A-Za-z$_][A-Za-z0-9$_]*|{NOTASCII_GB_CHAR})+) identifier (([A-Za-z0-9$_]|{NOTASCII_GB_CHAR})+) system_variable (@@[A-Za-z_][A-Za-z0-9_]*)|(@@[`][`A-Za-z_][`A-Za-z_]*) -user_variable (@[A-Za-z0-9_\.$]*)|(@[`'\"][`'\"A-Za-z0-9_\.$/%]*) +user_variable (@([A-Za-z0-9_\.$]|{NOTASCII_GB_CHAR})*)|(@[`'\"]([`'\"A-Za-z0-9_\.$/%]|{NOTASCII_GB_CHAR})*) version_num ([0-9]+\.+[0-9]*) int_num [0-9]+ client_version \([0-9]+(\.[0-9]+)*\) From f7c92208afa25161a45c2e269c485efdfa6c50ad Mon Sep 17 00:00:00 2001 From: obdev Date: Wed, 21 Aug 2024 12:51:35 +0000 Subject: [PATCH 157/249] Case of Tenant Backup Tasks Cancelled with Error 4016 --- src/share/backup/ob_backup_data_table_operator.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/share/backup/ob_backup_data_table_operator.cpp b/src/share/backup/ob_backup_data_table_operator.cpp index 620b03ba2..4cc09ecf0 100644 --- a/src/share/backup/ob_backup_data_table_operator.cpp +++ b/src/share/backup/ob_backup_data_table_operator.cpp @@ -818,9 +818,6 @@ int ObBackupJobOperator::cancel_jobs(common::ObISQLClient &proxy, const uint64_t LOG_WARN("[DATA_BACKUP]failed to append sql", K(ret), K(sql)); } else if (OB_FAIL(proxy.write(get_exec_tenant_id(tenant_id), sql.ptr(), affected_rows))) { LOG_WARN("[DATA_BACKUP]failed to exec sql", K(ret), K(sql)); - } else if (2 < affected_rows) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("[DATA_BACKUP]invalid affected_rows, disallow more than 2 jobs at the same", K(ret), K(affected_rows), K(sql)); } else { LOG_INFO("success cancel the backup jobs of tenant", K(ret), K(tenant_id)); } From a6cb5fe594b14ed2cdc639faa0cf6e0929e2b72a Mon Sep 17 00:00:00 2001 From: helloamateur Date: Wed, 21 Aug 2024 12:57:28 +0000 Subject: [PATCH 158/249] [CP] [GIS] fix gis index with high resolution --- .../src/lib/geo/ob_geo_to_s2_visitor.cpp | 119 +++++++++++++----- deps/oblib/src/lib/geo/ob_geo_to_s2_visitor.h | 8 +- deps/oblib/src/lib/geo/ob_s2adapter.cpp | 4 + .../r/mysql/geometry_bugfix_mysql.result | 103 +++++++++++++++ .../geometry/t/geometry_bugfix_mysql.test | 30 +++++ unittest/share/test_s2adapter.cpp | 7 +- 6 files changed, 239 insertions(+), 32 deletions(-) diff --git a/deps/oblib/src/lib/geo/ob_geo_to_s2_visitor.cpp b/deps/oblib/src/lib/geo/ob_geo_to_s2_visitor.cpp index 772878f69..494acbf38 100644 --- a/deps/oblib/src/lib/geo/ob_geo_to_s2_visitor.cpp +++ b/deps/oblib/src/lib/geo/ob_geo_to_s2_visitor.cpp @@ -63,6 +63,7 @@ int ObWkbToS2Visitor::MakeS2Point(T_IBIN *geo, S2Cell *&res) LOG_WARN("failed to alloc s2cell", K(ret)); } else { res = p; + bounder_.AddPoint(S2Point(latlng)); } } return ret; @@ -129,6 +130,7 @@ int ObWkbToS2Visitor::MakeProjS2Point(T_IBIN *geo, S2Cell *&res) LOG_WARN("failed to alloc s2cell", K(ret)); } else { res = p; + bounder_.AddPoint(point); } } } @@ -149,6 +151,8 @@ int ObWkbToS2Visitor::MakeS2Polyline(T_IBIN *geo, S2Polyline *&res) LOG_WARN("failed to add cell from point", K(ret)); } else if (OB_FAIL(vector_push_back(vertices, latlng))) { LOG_WARN("failed to add vertice", K(ret)); + } else { + bounder_.AddPoint(S2Point(latlng)); } } if (OB_SUCC(ret)) { @@ -178,6 +182,8 @@ int ObWkbToS2Visitor::MakeProjS2Polyline(T_IBIN *geo, S2Polyline *&res) LOG_WARN("failed to add cell from point", K(ret)); } else if (OB_FAIL(vector_push_back(vertices, p))) { LOG_WARN("failed to add vertice", K(ret)); + } else { + bounder_.AddPoint(p); } } @@ -212,6 +218,8 @@ int ObWkbToS2Visitor::MakeS2Polygon(T_IBIN *geo, S2Polygon *&res) LOG_WARN("failed to add cell from point", K(ret)); } else if (OB_FAIL(vector_push_back(vertices, tmp))) { LOG_WARN("failed to add vertice", K(ret)); + } else { + bounder_.AddPoint(tmp); } } if (OB_SUCC(ret)) { @@ -240,6 +248,8 @@ int ObWkbToS2Visitor::MakeS2Polygon(T_IBIN *geo, S2Polygon *&res) LOG_WARN("failed to add cell from point", K(ret)); } else if (OB_FAIL(vector_push_back(vertices, tmp))) { LOG_WARN("failed to add vertice", K(ret)); + } else { + bounder_.AddPoint(tmp); } } if (OB_SUCC(ret)) { @@ -286,6 +296,8 @@ int ObWkbToS2Visitor::MakeProjS2Polygon(T_IBIN *geo, S2Polygon *&res) LOG_WARN("failed to add cell from point", K(ret)); } else if (OB_FAIL(vector_push_back(vertices, tmp))) { LOG_WARN("failed to add vertice", K(ret)); + } else { + bounder_.AddPoint(tmp); } } if (OB_SUCC(ret)) { @@ -313,6 +325,8 @@ int ObWkbToS2Visitor::MakeProjS2Polygon(T_IBIN *geo, S2Polygon *&res) LOG_WARN("failed to add cell from point", K(ret)); } else if (OB_FAIL(vector_push_back(vertices, tmp))) { LOG_WARN("failed to add vertice", K(ret)); + } else { + bounder_.AddPoint(tmp); } } if (OB_SUCC(ret)) { @@ -448,30 +462,23 @@ int64_t ObWkbToS2Visitor::get_cellids(ObS2Cellids &cells, bool is_query, bool ne LOG_WARN("fail to push_back cellid", K(ret)); } } else { - S2CellUnion cellids; - S2RegionCoverer coverer(options_); uint32_t s2v_size = s2v_.size(); - - for (int i = 0; i < s2v_size; i++) { - S2CellUnion tmp = coverer.GetCovering(*s2v_[i]); - cellids = cellids.Union(tmp); - } if (need_buffer) { const int max_level_diff = 2; - cellids.Expand(distance, max_level_diff); + cell_union_.Expand(distance, max_level_diff); } if (s2v_size > 1) { - cellids.Normalize(); + cell_union_.Normalize(); } S2CellId prev_id = S2CellId::None(); - for (int i = 0; OB_SUCC(ret) && i < cellids.size(); i++) { - if (OB_FAIL(cells.push_back(cellids[i].id()))) { + for (int i = 0; OB_SUCC(ret) && i < cell_union_.size(); i++) { + if (OB_FAIL(cells.push_back(cell_union_[i].id()))) { LOG_WARN("fail to push_back cellid", K(ret)); } if (OB_SUCC(ret) && is_query) { - int level = cellids[i].level(); + int level = cell_union_[i].level(); while (OB_SUCC(ret) && (level -= options_.level_mod()) >= options_.min_level()) { - S2CellId ancestor_id = cellids[i].parent(level); + S2CellId ancestor_id = cell_union_[i].parent(level); if (prev_id != S2CellId::None() && prev_id.level() > level && prev_id.parent(level) == ancestor_id) { break; @@ -481,7 +488,7 @@ int64_t ObWkbToS2Visitor::get_cellids(ObS2Cellids &cells, bool is_query, bool ne } } } - prev_id = cellids[i]; + prev_id = cell_union_[i]; } if (OB_SUCC(ret) && has_reset_ && OB_FAIL(cells.push_back(exceedsBoundsCellID))) { LOG_WARN("fail to push_back cellid", K(ret)); @@ -490,6 +497,63 @@ int64_t ObWkbToS2Visitor::get_cellids(ObS2Cellids &cells, bool is_query, bool ne return ret; } +bool ObWkbToS2Visitor::is_full_range_cell_union(S2CellUnion &cellids) +{ + bool b_ret = false; + if (is_geog_) { + const uint8_t cell_faces = 6; + if (cellids.size() != cell_faces) { + // do nothing + } else { + uint8_t curr_faces = 0; + for (uint32_t i = 0; i < cellids.size(); i++) { + if (cellids[i].level() == 0) { + curr_faces++; + LOG_INFO("cell id", K(static_cast(cellids[i].id()))); + } + } + if (curr_faces == cell_faces) { + b_ret = true; + } + } + } else { + for (uint32_t i = 0; i < cellids.size() && !b_ret; i++) { + if (cellids[i].face() != 0) { + b_ret = true; + LOG_INFO("cell id", K(static_cast(cellids[i].id()))); + } + } + } + return b_ret; +} + +int ObWkbToS2Visitor::get_s2_cell_union() +{ + int ret = OB_SUCCESS; + if (!invalid_) { + S2RegionCoverer coverer(options_); + uint32_t s2v_size = s2v_.size(); + for (int i = 0; i < s2v_size; i++) { + S2CellUnion tmp = coverer.GetCovering(*s2v_[i]); + cell_union_ = cell_union_.Union(tmp); + } + if (is_full_range_cell_union(cell_union_)) { + S2LatLng margin = S2LatLng::FromDegrees(0.00001, 0.00001); + S2LatLngRect rect = bounder_.GetBound().Expanded(margin); + cell_union_ = coverer.GetCovering(rect); + mbr_ = rect; + LOG_INFO("generate new mbr: ", K(rect.lo().ToStringInDegrees().c_str()), K(rect.hi().ToStringInDegrees().c_str())); + S2cells_.clear(); + for (uint8_t i = 0; i < 4 && OB_SUCC(ret); i++) { + if (OB_FAIL(add_cell_from_point(rect.GetVertex(i)))) { + LOG_WARN("fail to push_back cellid", K(ret)); + } + } + } + } + return ret; +} + int64_t ObWkbToS2Visitor::get_cellids_and_unrepeated_ancestors(ObS2Cellids &cells, ObS2Cellids &ancestors, bool need_buffer, @@ -501,8 +565,6 @@ int64_t ObWkbToS2Visitor::get_cellids_and_unrepeated_ancestors(ObS2Cellids &cell LOG_WARN("fail to push_back cellid", K(ret)); } } else { - S2CellUnion cellids; - S2RegionCoverer coverer(options_); uint32_t s2v_size = s2v_.size(); hash::ObHashSet cellid_set; if (OB_FAIL(cellid_set.create(128, "CellidSet", "HashNode"))) { @@ -511,30 +573,26 @@ int64_t ObWkbToS2Visitor::get_cellids_and_unrepeated_ancestors(ObS2Cellids &cell ret = OB_NOT_INIT; LOG_WARN("fail to init cellid set", K(ret)); } else { - for (int i = 0; i < s2v_size && OB_SUCC(ret); i++) { - S2CellUnion tmp = coverer.GetCovering(*s2v_[i]); - cellids = cellids.Union(tmp); - } if (need_buffer) { const int max_level_diff = 2; - cellids.Expand(distance, max_level_diff); + cell_union_.Expand(distance, max_level_diff); } if (s2v_size > 1) { - cellids.Normalize(); + cell_union_.Normalize(); } S2CellId prev_id = S2CellId::None(); - for (int i = 0; OB_SUCC(ret) && i < cellids.size(); i++) { - int hash_ret = cellid_set.exist_refactored(cellids[i].id()); + for (int i = 0; OB_SUCC(ret) && i < cell_union_.size(); i++) { + int hash_ret = cellid_set.exist_refactored(cell_union_[i].id()); if (OB_HASH_NOT_EXIST == hash_ret) { - if (OB_FAIL(cellid_set.set_refactored(cellids[i].id()))) { + if (OB_FAIL(cellid_set.set_refactored(cell_union_[i].id()))) { LOG_WARN("failed to add cellid into set", K(ret)); - } else if (OB_FAIL(cells.push_back(cellids[i].id()))) { + } else if (OB_FAIL(cells.push_back(cell_union_[i].id()))) { LOG_WARN("fail to push_back cellid", K(ret)); } if (OB_SUCC(ret)) { - int level = cellids[i].level(); + int level = cell_union_[i].level(); while (OB_SUCC(ret) && (level -= options_.level_mod()) >= options_.min_level()) { - S2CellId ancestor_id = cellids[i].parent(level); + S2CellId ancestor_id = cell_union_[i].parent(level); if (prev_id != S2CellId::None() && prev_id.level() > level && prev_id.parent(level) == ancestor_id) { break; @@ -556,7 +614,7 @@ int64_t ObWkbToS2Visitor::get_cellids_and_unrepeated_ancestors(ObS2Cellids &cell ret = hash_ret; LOG_WARN("fail to check if key exist", K(ret), K(i)); } - prev_id = cellids[i]; + prev_id = cell_union_[i]; } if (OB_SUCC(ret) && has_reset_ && OB_FAIL(cells.push_back(exceedsBoundsCellID))) { LOG_WARN("fail to push_back cellid", K(ret)); @@ -607,6 +665,9 @@ void ObWkbToS2Visitor::reset() S2cells_.clear(); invalid_ = false; has_reset_ = true; + cell_union_.Clear(); + // reset to empty rectangle + bounder_.~S2LatLngRectBounder(); } template diff --git a/deps/oblib/src/lib/geo/ob_geo_to_s2_visitor.h b/deps/oblib/src/lib/geo/ob_geo_to_s2_visitor.h index 216b327f6..d2c0ed2da 100644 --- a/deps/oblib/src/lib/geo/ob_geo_to_s2_visitor.h +++ b/deps/oblib/src/lib/geo/ob_geo_to_s2_visitor.h @@ -46,7 +46,9 @@ public: options_(options), is_geog_(is_geog), invalid_(false), - has_reset_(false) + has_reset_(false), + cell_union_(), + bounder_() { mbr_ = S2LatLngRect::Empty(); } @@ -97,12 +99,14 @@ public: int64_t get_mbr(S2LatLngRect &mbr, bool need_buffer, S1Angle distance); bool is_invalid() { return invalid_; } void reset(); + int get_s2_cell_union(); private: double stToUV(double s); bool exceedsBounds(double x, double y); S2Point MakeS2PointFromXy(double x, double y); int add_cell_from_point(S2Point point); int add_cell_from_point(S2LatLng point); + bool is_full_range_cell_union(S2CellUnion &cellids); template static int vector_push_back(std::vector &vector, ElementType &element); static int vector_emplace_back(std::vector> &vector, S2Loop *element); @@ -117,6 +121,8 @@ private: bool is_geog_; bool invalid_; bool has_reset_; + S2CellUnion cell_union_; + S2LatLngRectBounder bounder_; DISALLOW_COPY_AND_ASSIGN(ObWkbToS2Visitor); }; diff --git a/deps/oblib/src/lib/geo/ob_s2adapter.cpp b/deps/oblib/src/lib/geo/ob_s2adapter.cpp index 84b2e12f7..e008e5af4 100644 --- a/deps/oblib/src/lib/geo/ob_s2adapter.cpp +++ b/deps/oblib/src/lib/geo/ob_s2adapter.cpp @@ -350,6 +350,8 @@ int64_t ObS2Adapter::init(const ObString &swkb, const ObSrsBoundsItem *bound) geo_ = geo; if (OB_FAIL(geo->do_visit(*visitor_))) { LOG_WARN("fail to do_visit by ObWkbToS2Visitor", K(ret)); + } else if (OB_FAIL(visitor_->get_s2_cell_union())) { + LOG_WARN("fail to get s2 cell union", K(ret)); } else if (visitor_->is_invalid()) { // 1. get valid geo inside bounds ObGeometry *corrected_geo = NULL; @@ -370,6 +372,8 @@ int64_t ObS2Adapter::init(const ObString &swkb, const ObSrsBoundsItem *bound) // 3. do_visit again if (OB_FAIL(corrected_geo->do_visit(*visitor_))) { LOG_WARN("fail to do_visit by ObWkbToS2Visitor", K(ret)); + } else if (OB_FAIL(visitor_->get_s2_cell_union())) { + LOG_WARN("fail to get s2 cell union", K(ret)); } } } diff --git a/tools/deploy/mysql_test/test_suite/geometry/r/mysql/geometry_bugfix_mysql.result b/tools/deploy/mysql_test/test_suite/geometry/r/mysql/geometry_bugfix_mysql.result index ee2cc46b3..803085cf4 100644 --- a/tools/deploy/mysql_test/test_suite/geometry/r/mysql/geometry_bugfix_mysql.result +++ b/tools/deploy/mysql_test/test_suite/geometry/r/mysql/geometry_bugfix_mysql.result @@ -650,6 +650,7 @@ Outputs & filters: (3747839314902384640,MIN,MIN ; 3747839314902384640,MAX,MAX), (3748120789879095296,MIN,MIN ; 3748120789879095296,MAX,MAX), (3751498489599623168,MIN,MIN ; 3751498489599623168,MAX,MAX), (3765009288481734656,MIN,MIN ; 3765009288481734656,MAX,MAX), (3819052484010180608,MIN,MIN ; 3819052484010180608,MAX,MAX), (3746994889972252672,MIN,MIN ; 3746994889972252672,MAX,MAX), (3458764513820540928,MIN,MIN ; 3458764513820540928,MAX,MAX) +drop table geek_poi_7; drop table if exists t1; create table t1( g geometry );// drop PROCEDURE IF EXISTS pro;// @@ -672,3 +673,105 @@ select getg(ST_SymDifference(point(1,0),point(1,6)));// getg(ST_SymDifference(point(1,0),point(1,6))) MULTIPOINT((1 0),(1 6)) drop table t1; +drop table if exists highway_621; +CREATE TABLE `highway_621` ( +`id` int(32) NOT NULL auto_increment, +`the_geom` geometry NOT NULL /*!80003 SRID 4326 */, +PRIMARY KEY (`id`), +SPATIAL KEY `idx_the_geom_highway_6` (`the_geom`) BLOCK_SIZE 16384 LOCAL +); +insert into highway_621(the_geom) values(ST_GeomFromText('point(120.34904267189361 30.320965261625222)',4326, 'axis-order=long-lat')); +insert into highway_621(the_geom) values(ST_GeomFromText('point(120.34904267189360 30.320965261625222)',4326, 'axis-order=long-lat')); +insert into highway_621(the_geom) values(ST_GeomFromText('linestring(120.34904267189360 30.320965261625222, 120.34904267189361 30.320965261625222)',4326, 'axis-order=long-lat')); +insert into highway_621(the_geom) values(ST_GeomFromText('linestring(120.34904267189361 30.320965261625222, 120.35612237862895 30.321118189268013)',4326, 'axis-order=long-lat')); +insert into highway_621(the_geom) values(ST_GeomFromText('linestring(120.34904267189360 30.320965261625222, 120.34904267189360 30.320800629134425)',4326, 'axis-order=long-lat')); +insert into highway_621(the_geom) values(ST_GeomFromText('MULTIPOLYGON (((120.34904267189361 30.320965261625222, 120.35612237862895 30.321118189268013, 120.35612237865006 30.32111818926693, 120.35460911503478 30.32080062911392, 120.3546091159729 30.320800629134425, 120.34904267189361 30.320965261625222)))',4326, 'axis-order=long-lat')); +insert into highway_621(the_geom) values(ST_GeomFromText('MULTIPOLYGON (((110.34904267189361 30.320965261625222, 110.35812237862895 30.321118189268013, 110.35812237865006 30.32111818926693, 110.35460911503478 30.32080062911392, 110.3546091159729 30.320800629134425, 110.34904267189361 30.320965261625222)))',4326, 'axis-order=long-lat')); +insert into highway_621(the_geom) values(ST_GeomFromText('GEOMETRYCOLLECTION(MULTIPOLYGON(((120.31599964855097 30.243634005580816,120.3160009095347 30.243635469782166,120.31600747846684 30.24364742764055,120.31601121779788 30.243660263421923,120.31601198353063 30.243673482839302,120.31599964855097 30.243634005580816)),((120.3160009095347 30.243635469782166,120.31600747846684 30.24364742764055,120.31601121779788 30.243660263421923,120.31601198353063 30.243673482839302,120.31601159806742 30.24367573874217,120.3160009095347 30.243635469782166))),POLYGON((120.31599964855097 30.243634005580816,120.3160009095347 30.243635469782166,120.31600747846684 30.24364742764055,120.31601121779788 30.243660263421923,120.31601198353063 30.243673482839302,120.31599964855097 30.243634005580816)),LINESTRING(120.31599964855097 30.243634005580816,120.3160009095347 30.243635469782166),MULTIPOINT(120.31599964855097 30.243634005580816,120.3160009095347 30.243635469782166,120.31600747846684 30.24364742764055),point(120.31599964855097 30.243634005580816))',4326, 'axis-order=long-lat')); +explain select id, st_astext(the_geom) from highway_621 where ST_Intersects(the_geom, ST_GeomFromText('MULTIPOLYGON (((120.34904267189361 30.320965261625222, 120.35612237862895 30.321118189268013, 120.35612237865006 30.32111818926693, 120.35460911503478 30.32080062911392, 120.3546091159729 30.320800629134425, 120.34904267189361 30.320965261625222)))',4326, 'axis-order=long-lat')); +Query Plan +============================================================================== +|ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| +------------------------------------------------------------------------------ +|0 |TABLE FULL SCAN|highway_621(idx_the_geom_highway_6)|12 |2947 | +============================================================================== +Outputs & filters: +------------------------------------- + 0 - output([highway_621.id], [st_astext(highway_621.the_geom)]), filter([ST_Intersects(highway_621.the_geom, ST_GeomFromText('MULTIPOLYGON (((120.34904267189361 + 30.320965261625222, 120.35612237862895 30.321118189268013, 120.35612237865006 30.32111818926693, 120.35460911503478 30.32080062911392, 120.3546091159729 + 30.320800629134425, 120.34904267189361 30.320965261625222)))', 4326, 'axis-order=long-lat'))]) + access([highway_621.id], [highway_621.the_geom]), partitions(p0) + is_index_back=true, is_global_index=false, filter_before_indexback[false], + range_key([highway_621.__cellid_17], [highway_621.__mbr_17], [highway_621.id]), range(3768553615552151552,MIN,MIN ; 3768553615552151552,MAX,MAX), + (3768553615485042688,MIN,MIN ; 3768553615485042688,MAX,MAX), (3768553614679736320,MIN,MIN ; 3768553614679736320,MAX,MAX), (3768553611458510848,MIN,MIN ; + 3768553611458510848,MAX,MAX), (3768553615753478144,MIN,MIN ; 3768553615753478144,MAX,MAX), (3768553632933347328,MIN,MIN ; 3768553632933347328,MAX,MAX), + (3768553839091777536,MIN,MIN ; 3768553839091777536,MAX,MAX), (3768553014458056704,MIN,MIN ; 3768553014458056704,MAX,MAX), (3768549715923173376,MIN,MIN ; + 3768549715923173376,MAX,MAX), (3768545317876662272,MIN,MIN ; 3768545317876662272,MAX,MAX), (3768598094434795520,MIN,MIN ; 3768598094434795520,MAX,MAX), + (3768668463178973184,MIN,MIN ; 3768668463178973184,MAX,MAX), (3768386988202262528,MIN,MIN ; 3768386988202262528,MAX,MAX), (3769512888109105152,MIN,MIN ; + 3769512888109105152,MAX,MAX), (3765009288481734656,MIN,MIN ; 3765009288481734656,MAX,MAX), (3819052484010180608,MIN,MIN ; 3819052484010180608,MAX,MAX), + (3746994889972252672,MIN,MIN ; 3746994889972252672,MAX,MAX), (3458764513820540928,MIN,MIN ; 3458764513820540928,MAX,MAX), (3768553615686369280,MIN,MIN ; + 3768553615686369280,MAX,MAX), (3768553615870918656,MIN,MIN ; 3768553615870918656,MAX,MAX), (3768553615820587008,MIN,MIN ; 3768553615820587008,MAX,MAX), + (3768553616021913600,MIN,MIN ; 3768553616021913600,MAX,MAX), (3768553616827219968,MIN,MIN ; 3768553616827219968,MAX,MAX), (3768553620048445440,MIN,MIN ; + 3768553620048445440,MAX,MAX), (3768553615954804736,MIN,MIN ; 3768553615954804736,MAX,MAX), (3768553615938027520,MIN,MIN ; 3768553615938027520,MAX,MAX), + (3768553615971581952,MIN,MIN ; 3768553615971581952,MAX,MAX), (3768553616642670592,MIN,MIN ; 3768553616642670592,MAX,MAX), (3768553616625893376,MIN,MIN ; + 3768553616625893376,MAX,MAX), (3768553616558784512,MIN,MIN ; 3768553616558784512,MAX,MAX), (3768553616760111104,MIN,MIN ; 3768553616760111104,MAX,MAX), + (3768553616776888320,MIN,MIN ; 3768553616776888320,MAX,MAX), (3768553616894328832,MIN,MIN ; 3768553616894328832,MAX,MAX), (3768553617095655424,MIN,MIN ; + 3768553617095655424,MAX,MAX), (3768553617028546560,MIN,MIN ; 3768553617028546560,MAX,MAX), (3768553617045323776,MIN,MIN ; 3768553617045323776,MAX,MAX), + (3768553622263037952,MIN,MIN ; 3768553622263037952,MAX,MAX), (3768553622464364544,MIN,MIN ; 3768553622464364544,MAX,MAX), (3768553623269670912,MIN,MIN ; + 3768553623269670912,MAX,MAX), (3768553622258843648,MIN,MIN ; 3768553622258843648,MAX,MAX), (3768553622246260736,MIN,MIN ; 3768553622246260736,MAX,MAX), + (3768553622397255680,MIN,MIN ; 3768553622397255680,MAX,MAX), (3768553622531473408,MIN,MIN ; 3768553622531473408,MAX,MAX), (3768553622665691136,MIN,MIN ; + 3768553622665691136,MAX,MAX), (3768553622799908864,MIN,MIN ; 3768553622799908864,MAX,MAX), (3768553623001235456,MIN,MIN ; 3768553623001235456,MAX,MAX), + (3768553623202562048,MIN,MIN ; 3768553623202562048,MAX,MAX), (3768553623252893696,MIN,MIN ; 3768553623252893696,MAX,MAX), (3768553643201003520,MIN,MIN ; + 3768553643201003520,MAX,MAX), (3768553643402330112,MIN,MIN ; 3768553643402330112,MAX,MAX), (3768553642597023744,MIN,MIN ; 3768553642597023744,MAX,MAX), + (3768553645818249216,MIN,MIN ; 3768553645818249216,MAX,MAX), (3768553650113216512,MIN,MIN ; 3768553650113216512,MAX,MAX), (3768553643808129024,MIN,MIN ; + 3768553643808129024,MAX,MAX), (3768553643809177600,MIN,MIN ; 3768553643809177600,MAX,MAX), (3768553643821760512,MIN,MIN ; 3768553643821760512,MAX,MAX), + (3768553643872092160,MIN,MIN ; 3768553643872092160,MAX,MAX), (3768553643939201024,MIN,MIN ; 3768553643939201024,MAX,MAX), (3768553644744507392,MIN,MIN ; + 3768553644744507392,MAX,MAX), (3768553615585705985,MIN,MIN ; 3768553615619260415,MAX,MAX), (3768553615619260417,MIN,MIN ; 3768553615652814847,MAX,MAX), + (3768553615854141441,MIN,MIN ; 3768553615862530047,MAX,MAX), (3768553615879307265,MIN,MIN ; 3768553615887695871,MAX,MAX), (3768553615887695873,MIN,MIN ; + 3768553615921250303,MAX,MAX), (3768553615946416129,MIN,MIN ; 3768553615954804735,MAX,MAX), (3768553615954804737,MIN,MIN ; 3768553615963193343,MAX,MAX), + (3768553615979970561,MIN,MIN ; 3768553615988359167,MAX,MAX), (3768553615988359169,MIN,MIN ; 3768553616021913599,MAX,MAX), (3768553616021913601,MIN,MIN ; + 3768553616156131327,MAX,MAX), (3768553616642670593,MIN,MIN ; 3768553616651059199,MAX,MAX), (3768553616651059201,MIN,MIN ; 3768553616659447807,MAX,MAX), + (3768553616659447809,MIN,MIN ; 3768553616693002239,MAX,MAX), (3768553616693002241,MIN,MIN ; 3768553616726556671,MAX,MAX), (3768553616785276929,MIN,MIN ; + 3768553616793665535,MAX,MAX), (3768553616793665537,MIN,MIN ; 3768553616827219967,MAX,MAX), (3768553616827219969,MIN,MIN ; 3768553616860774399,MAX,MAX), + (3768553616927883265,MIN,MIN ; 3768553616961437695,MAX,MAX), (3768553616961437697,MIN,MIN ; 3768553616994992127,MAX,MAX), (3768553616994992129,MIN,MIN ; + 3768553617028546559,MAX,MAX), (3768553617036935169,MIN,MIN ; 3768553617045323775,MAX,MAX), (3768553622195929089,MIN,MIN ; 3768553622229483519,MAX,MAX), + (3768553622260940801,MIN,MIN ; 3768553622263037951,MAX,MAX), (3768553622263037953,MIN,MIN ; 3768553622296592383,MAX,MAX), (3768553622296592385,MIN,MIN ; + 3768553622330146815,MAX,MAX), (3768553622430810113,MIN,MIN ; 3768553622464364543,MAX,MAX), (3768553622464364545,MIN,MIN ; 3768553622497918975,MAX,MAX), + (3768553622565027841,MIN,MIN ; 3768553622598582271,MAX,MAX), (3768553622598582273,MIN,MIN ; 3768553622632136703,MAX,MAX), (3768553622632136705,MIN,MIN ; + 3768553622665691135,MAX,MAX), (3768553622766354433,MIN,MIN ; 3768553622799908863,MAX,MAX), (3768553623001235457,MIN,MIN ; 3768553623135453183,MAX,MAX), + (3768553623135453185,MIN,MIN ; 3768553623169007615,MAX,MAX), (3768553623169007617,MIN,MIN ; 3768553623202562047,MAX,MAX), (3768553623202562049,MIN,MIN ; + 3768553623236116479,MAX,MAX), (3768553623244505089,MIN,MIN ; 3768553623252893695,MAX,MAX), (3768553643167449089,MIN,MIN ; 3768553643201003519,MAX,MAX), + (3768553643201003521,MIN,MIN ; 3768553643234557951,MAX,MAX), (3768553643536547841,MIN,MIN ; 3768553643670765567,MAX,MAX), (3768553643807604737,MIN,MIN ; + 3768553643808129023,MAX,MAX), (3768553643830149121,MIN,MIN ; 3768553643838537727,MAX,MAX) +select id, st_astext(the_geom) from highway_621 where ST_Intersects(the_geom, ST_GeomFromText('MULTIPOLYGON (((120.34904267189361 30.320965261625222, 120.35612237862895 30.321118189268013, 120.35612237865006 30.32111818926693, 120.35460911503478 30.32080062911392, 120.3546091159729 30.320800629134425, 120.34904267189361 30.320965261625222)))',4326, 'axis-order=long-lat')); +id st_astext(the_geom) +1 POINT(30.320965261625222 120.34904267189361) +3 LINESTRING(30.320965261625222 120.3490426718936,30.320965261625222 120.34904267189361) +4 LINESTRING(30.320965261625222 120.34904267189361,30.321118189268013 120.35612237862895) +5 LINESTRING(30.320965261625222 120.3490426718936,30.320800629134425 120.3490426718936) +6 MULTIPOLYGON(((30.320965261625222 120.34904267189361,30.321118189268013 120.35612237862895,30.32111818926693 120.35612237865006,30.32080062911392 120.35460911503478,30.320800629134425 120.3546091159729,30.320965261625222 120.34904267189361))) +select id, st_astext(the_geom) from highway_621 where _ST_covers(ST_GeomFromText('MULTIPOLYGON (((120.34904267189361 30.320965261625222, 120.35612237862895 30.321118189268013, 120.35612237865006 30.32111818926693, 120.35460911503478 30.32080062911392, 120.3546091159729 30.320800629134425, 120.34904267189361 30.320965261625222)))',4326, 'axis-order=long-lat'), the_geom); +id st_astext(the_geom) +1 POINT(30.320965261625222 120.34904267189361) +4 LINESTRING(30.320965261625222 120.34904267189361,30.321118189268013 120.35612237862895) +select id, st_astext(the_geom) from highway_621 ignore index(idx_the_geom_highway_6) where ST_Intersects(the_geom, ST_GeomFromText('MULTIPOLYGON (((120.34904267189361 30.320965261625222, 120.35612237862895 30.321118189268013, 120.35612237865006 30.32111818926693, 120.35460911503478 30.32080062911392, 120.3546091159729 30.320800629134425, 120.34904267189361 30.320965261625222)))',4326, 'axis-order=long-lat')); +id st_astext(the_geom) +1 POINT(30.320965261625222 120.34904267189361) +3 LINESTRING(30.320965261625222 120.3490426718936,30.320965261625222 120.34904267189361) +4 LINESTRING(30.320965261625222 120.34904267189361,30.321118189268013 120.35612237862895) +5 LINESTRING(30.320965261625222 120.3490426718936,30.320800629134425 120.3490426718936) +6 MULTIPOLYGON(((30.320965261625222 120.34904267189361,30.321118189268013 120.35612237862895,30.32111818926693 120.35612237865006,30.32080062911392 120.35460911503478,30.320800629134425 120.3546091159729,30.320965261625222 120.34904267189361))) +select id, st_astext(the_geom) from highway_621 ignore index(idx_the_geom_highway_6) where _ST_covers(ST_GeomFromText('MULTIPOLYGON (((120.34904267189361 30.320965261625222, 120.35612237862895 30.321118189268013, 120.35612237865006 30.32111818926693, 120.35460911503478 30.32080062911392, 120.3546091159729 30.320800629134425, 120.34904267189361 30.320965261625222)))',4326, 'axis-order=long-lat'), the_geom); +id st_astext(the_geom) +1 POINT(30.320965261625222 120.34904267189361) +4 LINESTRING(30.320965261625222 120.34904267189361,30.321118189268013 120.35612237862895) +select id, st_astext(the_geom) from highway_621 where ST_Intersects(the_geom, ST_GeomFromText('GEOMETRYCOLLECTION(MULTIPOLYGON (((120.34904267189361 30.320965261625222, 120.35612237862895 30.321118189268013, 120.35612237865006 30.32111818926693, 120.35460911503478 30.32080062911392, 120.3546091159729 30.320800629134425, 120.34904267189361 30.320965261625222))),POLYGON ((120.34904267189361 30.320965261625222, 120.35612237862895 30.321118189268013, 120.35612237865006 30.32111818926693, 120.35460911503478 30.32080062911392, 120.3546091159729 30.320800629134425, 120.34904267189361 30.320965261625222)),multipoint(120.34904267189361 30.320965261625222, 120.35612237862895 30.321118189268013, 120.35612237865006 30.32111818926693),point(120.31601198353063 30.243673482839302))',4326, 'axis-order=long-lat')); +id st_astext(the_geom) +1 POINT(30.320965261625222 120.34904267189361) +2 POINT(30.320965261625222 120.3490426718936) +3 LINESTRING(30.320965261625222 120.3490426718936,30.320965261625222 120.34904267189361) +4 LINESTRING(30.320965261625222 120.34904267189361,30.321118189268013 120.35612237862895) +5 LINESTRING(30.320965261625222 120.3490426718936,30.320800629134425 120.3490426718936) +6 MULTIPOLYGON(((30.320965261625222 120.34904267189361,30.321118189268013 120.35612237862895,30.32111818926693 120.35612237865006,30.32080062911392 120.35460911503478,30.320800629134425 120.3546091159729,30.320965261625222 120.34904267189361))) +8 GEOMETRYCOLLECTION(MULTIPOLYGON(((30.243634005580816 120.31599964855097,30.243635469782166 120.3160009095347,30.24364742764055 120.31600747846684,30.243660263421923 120.31601121779788,30.243673482839302 120.31601198353063,30.243634005580816 120.31599964855097)),((30.243635469782166 120.3160009095347,30.24364742764055 120.31600747846684,30.243660263421923 120.31601121779788,30.243673482839302 120.31601198353063,30.24367573874217 120.31601159806742,30.243635469782166 120.3160009095347))),POLYGON((30.243634005580816 120.31599964855097,30.243635469782166 120.3160009095347,30.24364742764055 120.31600747846684,30.243660263421923 120.31601121779788,30.243673482839302 120.31601198353063,30.243634005580816 120.31599964855097)),LINESTRING(30.243634005580816 120.31599964855097,30.243635469782166 120.3160009095347),MULTIPOINT((30.243634005580816 120.31599964855097),(30.243635469782166 120.3160009095347),(30.24364742764055 120.31600747846684)),POINT(30.243634005580816 120.31599964855097)) +drop table highway_621; diff --git a/tools/deploy/mysql_test/test_suite/geometry/t/geometry_bugfix_mysql.test b/tools/deploy/mysql_test/test_suite/geometry/t/geometry_bugfix_mysql.test index 3831e524f..69620bdbd 100644 --- a/tools/deploy/mysql_test/test_suite/geometry/t/geometry_bugfix_mysql.test +++ b/tools/deploy/mysql_test/test_suite/geometry/t/geometry_bugfix_mysql.test @@ -418,6 +418,9 @@ call dbms_stats.gather_table_stats('test','geek_poi_7'); --enable_result_log explain SELECT p.p_id,0 as distance FROM geek_poi_7 as p WHERE p.geohash_code like 'ws0emp%' and p.p_delete_time = 0 AND ST_Contains(p.geog_poi,_ST_GeogFromText('POINT(113.42416381835938 23.11138916015625)')); +drop table geek_poi_7; + +--disable_warnings drop table if exists t1; --enable_warnings delimiter //; @@ -443,3 +446,30 @@ select getg(ST_SymDifference(point(1,0),point(1,6)));// delimiter ;// drop table t1; + +--disable_warnings +drop table if exists highway_621; +--enable_warnings +CREATE TABLE `highway_621` ( + `id` int(32) NOT NULL auto_increment, + `the_geom` geometry NOT NULL /*!80003 SRID 4326 */, + PRIMARY KEY (`id`), + SPATIAL KEY `idx_the_geom_highway_6` (`the_geom`) BLOCK_SIZE 16384 LOCAL +); + +insert into highway_621(the_geom) values(ST_GeomFromText('point(120.34904267189361 30.320965261625222)',4326, 'axis-order=long-lat')); +insert into highway_621(the_geom) values(ST_GeomFromText('point(120.34904267189360 30.320965261625222)',4326, 'axis-order=long-lat')); +insert into highway_621(the_geom) values(ST_GeomFromText('linestring(120.34904267189360 30.320965261625222, 120.34904267189361 30.320965261625222)',4326, 'axis-order=long-lat')); +insert into highway_621(the_geom) values(ST_GeomFromText('linestring(120.34904267189361 30.320965261625222, 120.35612237862895 30.321118189268013)',4326, 'axis-order=long-lat')); +insert into highway_621(the_geom) values(ST_GeomFromText('linestring(120.34904267189360 30.320965261625222, 120.34904267189360 30.320800629134425)',4326, 'axis-order=long-lat')); +insert into highway_621(the_geom) values(ST_GeomFromText('MULTIPOLYGON (((120.34904267189361 30.320965261625222, 120.35612237862895 30.321118189268013, 120.35612237865006 30.32111818926693, 120.35460911503478 30.32080062911392, 120.3546091159729 30.320800629134425, 120.34904267189361 30.320965261625222)))',4326, 'axis-order=long-lat')); +insert into highway_621(the_geom) values(ST_GeomFromText('MULTIPOLYGON (((110.34904267189361 30.320965261625222, 110.35812237862895 30.321118189268013, 110.35812237865006 30.32111818926693, 110.35460911503478 30.32080062911392, 110.3546091159729 30.320800629134425, 110.34904267189361 30.320965261625222)))',4326, 'axis-order=long-lat')); +insert into highway_621(the_geom) values(ST_GeomFromText('GEOMETRYCOLLECTION(MULTIPOLYGON(((120.31599964855097 30.243634005580816,120.3160009095347 30.243635469782166,120.31600747846684 30.24364742764055,120.31601121779788 30.243660263421923,120.31601198353063 30.243673482839302,120.31599964855097 30.243634005580816)),((120.3160009095347 30.243635469782166,120.31600747846684 30.24364742764055,120.31601121779788 30.243660263421923,120.31601198353063 30.243673482839302,120.31601159806742 30.24367573874217,120.3160009095347 30.243635469782166))),POLYGON((120.31599964855097 30.243634005580816,120.3160009095347 30.243635469782166,120.31600747846684 30.24364742764055,120.31601121779788 30.243660263421923,120.31601198353063 30.243673482839302,120.31599964855097 30.243634005580816)),LINESTRING(120.31599964855097 30.243634005580816,120.3160009095347 30.243635469782166),MULTIPOINT(120.31599964855097 30.243634005580816,120.3160009095347 30.243635469782166,120.31600747846684 30.24364742764055),point(120.31599964855097 30.243634005580816))',4326, 'axis-order=long-lat')); +explain select id, st_astext(the_geom) from highway_621 where ST_Intersects(the_geom, ST_GeomFromText('MULTIPOLYGON (((120.34904267189361 30.320965261625222, 120.35612237862895 30.321118189268013, 120.35612237865006 30.32111818926693, 120.35460911503478 30.32080062911392, 120.3546091159729 30.320800629134425, 120.34904267189361 30.320965261625222)))',4326, 'axis-order=long-lat')); +select id, st_astext(the_geom) from highway_621 where ST_Intersects(the_geom, ST_GeomFromText('MULTIPOLYGON (((120.34904267189361 30.320965261625222, 120.35612237862895 30.321118189268013, 120.35612237865006 30.32111818926693, 120.35460911503478 30.32080062911392, 120.3546091159729 30.320800629134425, 120.34904267189361 30.320965261625222)))',4326, 'axis-order=long-lat')); +select id, st_astext(the_geom) from highway_621 where _ST_covers(ST_GeomFromText('MULTIPOLYGON (((120.34904267189361 30.320965261625222, 120.35612237862895 30.321118189268013, 120.35612237865006 30.32111818926693, 120.35460911503478 30.32080062911392, 120.3546091159729 30.320800629134425, 120.34904267189361 30.320965261625222)))',4326, 'axis-order=long-lat'), the_geom); +select id, st_astext(the_geom) from highway_621 ignore index(idx_the_geom_highway_6) where ST_Intersects(the_geom, ST_GeomFromText('MULTIPOLYGON (((120.34904267189361 30.320965261625222, 120.35612237862895 30.321118189268013, 120.35612237865006 30.32111818926693, 120.35460911503478 30.32080062911392, 120.3546091159729 30.320800629134425, 120.34904267189361 30.320965261625222)))',4326, 'axis-order=long-lat')); +select id, st_astext(the_geom) from highway_621 ignore index(idx_the_geom_highway_6) where _ST_covers(ST_GeomFromText('MULTIPOLYGON (((120.34904267189361 30.320965261625222, 120.35612237862895 30.321118189268013, 120.35612237865006 30.32111818926693, 120.35460911503478 30.32080062911392, 120.3546091159729 30.320800629134425, 120.34904267189361 30.320965261625222)))',4326, 'axis-order=long-lat'), the_geom); +select id, st_astext(the_geom) from highway_621 where ST_Intersects(the_geom, ST_GeomFromText('GEOMETRYCOLLECTION(MULTIPOLYGON (((120.34904267189361 30.320965261625222, 120.35612237862895 30.321118189268013, 120.35612237865006 30.32111818926693, 120.35460911503478 30.32080062911392, 120.3546091159729 30.320800629134425, 120.34904267189361 30.320965261625222))),POLYGON ((120.34904267189361 30.320965261625222, 120.35612237862895 30.321118189268013, 120.35612237865006 30.32111818926693, 120.35460911503478 30.32080062911392, 120.3546091159729 30.320800629134425, 120.34904267189361 30.320965261625222)),multipoint(120.34904267189361 30.320965261625222, 120.35612237862895 30.321118189268013, 120.35612237865006 30.32111818926693),point(120.31601198353063 30.243673482839302))',4326, 'axis-order=long-lat')); + +drop table highway_621; diff --git a/unittest/share/test_s2adapter.cpp b/unittest/share/test_s2adapter.cpp index 13dc9d4b3..8c21fb5f3 100644 --- a/unittest/share/test_s2adapter.cpp +++ b/unittest/share/test_s2adapter.cpp @@ -236,8 +236,11 @@ void printCellid(ObS2Cellids &cells) { S2CellId tmp(it); std::cout << "F" << tmp.face(); std::cout << "/L" << tmp.level() << "/"; - for (int level = 1; level <= tmp.level(); level++) { - std::cout << tmp.child_position(level); + for (int level = 1; level <= tmp.level() && !tmp.is_leaf(); level++) { + std::cout << tmp.child_position(level); + } + if (tmp.is_leaf()) { + std::cout << tmp.id(); } std::cout << std::endl; } From 7de3a485cb1f6b87685353c60042fd0e84e9bbd4 Mon Sep 17 00:00:00 2001 From: wanyue-wy <345657357@qq.com> Date: Wed, 21 Aug 2024 13:04:40 +0000 Subject: [PATCH 159/249] remove tmp_file_handle from tmp_file_io_handle --- .../tmp_file/ob_tmp_file_io_define.cpp | 35 +++++++++++-------- src/storage/tmp_file/ob_tmp_file_io_define.h | 10 +++--- src/storage/tmp_file/ob_tmp_file_manager.cpp | 20 +++++------ 3 files changed, 35 insertions(+), 30 deletions(-) diff --git a/src/storage/tmp_file/ob_tmp_file_io_define.cpp b/src/storage/tmp_file/ob_tmp_file_io_define.cpp index 6aab99c6e..dee3c6eb9 100644 --- a/src/storage/tmp_file/ob_tmp_file_io_define.cpp +++ b/src/storage/tmp_file/ob_tmp_file_io_define.cpp @@ -13,6 +13,7 @@ #define USING_LOG_PREFIX STORAGE #include "storage/tmp_file/ob_tmp_file_io_define.h" +#include "storage/tmp_file/ob_tmp_file_manager.h" namespace oceanbase { @@ -58,7 +59,7 @@ bool ObTmpFileIOInfo::is_valid() const ObTmpFileIOHandle::ObTmpFileIOHandle() : is_inited_(false), - tmp_file_handle_(), + fd_(ObTmpFileGlobal::INVALID_TMP_FILE_FD), ctx_(), buf_(nullptr), update_offset_in_file_(false), @@ -77,7 +78,7 @@ void ObTmpFileIOHandle::reset() { is_inited_ = false; ctx_.reset(); - tmp_file_handle_.reset(); + fd_ = ObTmpFileGlobal::INVALID_TMP_FILE_FD; buf_ = nullptr; update_offset_in_file_ = false; buf_size_ = -1; @@ -85,13 +86,13 @@ void ObTmpFileIOHandle::reset() read_offset_in_file_ = -1; } -int ObTmpFileIOHandle::init_write(const ObTmpFileIOInfo &io_info, ObTmpFileHandle &tmp_file_handle) +int ObTmpFileIOHandle::init_write(const ObTmpFileIOInfo &io_info) { int ret = OB_SUCCESS; if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("ObTmpFileIOHandle has been inited twice", KR(ret), KPC(this)); - } else if (OB_UNLIKELY(!io_info.is_valid()) || OB_ISNULL(tmp_file_handle.get())) { + } else if (OB_UNLIKELY(!io_info.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(io_info)); } else if (OB_FAIL(ctx_.init(io_info.fd_, io_info.dir_id_, false /*is_read*/, io_info.io_desc_, @@ -101,7 +102,7 @@ int ObTmpFileIOHandle::init_write(const ObTmpFileIOInfo &io_info, ObTmpFileHandl LOG_WARN("fail to prepare write context", KR(ret), KPC(this)); } else { is_inited_ = true; - tmp_file_handle_ = tmp_file_handle; + fd_ = io_info.fd_; buf_ = io_info.buf_; buf_size_ = io_info.size_; done_size_ = 0; @@ -110,13 +111,13 @@ int ObTmpFileIOHandle::init_write(const ObTmpFileIOInfo &io_info, ObTmpFileHandl return ret; } -int ObTmpFileIOHandle::init_pread(const ObTmpFileIOInfo &io_info, const int64_t read_offset, ObTmpFileHandle &tmp_file_handle) +int ObTmpFileIOHandle::init_pread(const ObTmpFileIOInfo &io_info, const int64_t read_offset) { int ret = OB_SUCCESS; if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("ObTmpFileIOHandle has been inited twice", KR(ret), KPC(this)); - } else if (OB_UNLIKELY(!io_info.is_valid()) || OB_ISNULL(tmp_file_handle.get())) { + } else if (OB_UNLIKELY(!io_info.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(io_info)); } else if (OB_UNLIKELY(read_offset < 0)) { @@ -129,7 +130,7 @@ int ObTmpFileIOHandle::init_pread(const ObTmpFileIOInfo &io_info, const int64_t LOG_WARN("fail to prepare read context", KR(ret), KPC(this), K(read_offset)); } else { is_inited_ = true; - tmp_file_handle_ = tmp_file_handle; + fd_ = io_info.fd_; buf_ = io_info.buf_; buf_size_ = io_info.size_; done_size_ = 0; @@ -139,13 +140,13 @@ int ObTmpFileIOHandle::init_pread(const ObTmpFileIOInfo &io_info, const int64_t return ret; } -int ObTmpFileIOHandle::init_read(const ObTmpFileIOInfo &io_info, ObTmpFileHandle &tmp_file_handle) +int ObTmpFileIOHandle::init_read(const ObTmpFileIOInfo &io_info) { int ret = OB_SUCCESS; if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("ObTmpFileIOHandle has been inited twice", KR(ret), KPC(this)); - } else if (OB_UNLIKELY(!io_info.is_valid()) || OB_ISNULL(tmp_file_handle.get())) { + } else if (OB_UNLIKELY(!io_info.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(io_info)); } else if (OB_FAIL(ctx_.init(io_info.fd_, io_info.dir_id_, true /*is_read*/, io_info.io_desc_, @@ -155,7 +156,7 @@ int ObTmpFileIOHandle::init_read(const ObTmpFileIOInfo &io_info, ObTmpFileHandle LOG_WARN("fail to prepare read context", KR(ret), KPC(this)); } else { is_inited_ = true; - tmp_file_handle_ = tmp_file_handle; + fd_ = io_info.fd_; buf_ = io_info.buf_; buf_size_ = io_info.size_; done_size_ = 0; @@ -177,6 +178,7 @@ bool ObTmpFileIOHandle::is_valid() const int ObTmpFileIOHandle::wait() { int ret = OB_SUCCESS; + ObTmpFileHandle file_handle; if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -193,6 +195,8 @@ int ObTmpFileIOHandle::wait() } else if (OB_UNLIKELY(done_size_ > buf_size_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("done size is larger than total todo size", KR(ret), KPC(this)); + } else if (OB_FAIL(MTL(ObTenantTmpFileManager*)->get_tmp_file(fd_, file_handle))) { + LOG_WARN("fail to get tmp file handle", KR(ret), K(fd_)); } else { while (OB_SUCC(ret) && !is_finished()) { if (OB_FAIL(ctx_.prepare_read(buf_ + done_size_, @@ -200,7 +204,7 @@ int ObTmpFileIOHandle::wait() ObTmpFileGlobal::TMP_FILE_READ_BATCH_SIZE), read_offset_in_file_))) { LOG_WARN("fail to generate read ctx", KR(ret), KPC(this)); - } else if (OB_FAIL(tmp_file_handle_.get()->aio_pread(ctx_))) { + } else if (OB_FAIL(file_handle.get()->aio_pread(ctx_))) { LOG_WARN("fail to continue read once batch", KR(ret), K(ctx_)); } else if (OB_FAIL(ctx_.wait())) { LOG_WARN("fail to wait tmp file io", KR(ret), K(ctx_)); @@ -208,11 +212,12 @@ int ObTmpFileIOHandle::wait() LOG_WARN("fail to handle finished ctx", KR(ret), KPC(this)); } } // end while + + if (update_offset_in_file_ && (OB_SUCC(ret) || OB_ITER_END == ret)) { + file_handle.get()->update_read_offset(read_offset_in_file_); + } } - if (update_offset_in_file_ && (OB_SUCC(ret) || OB_ITER_END == ret)) { - tmp_file_handle_.get()->update_read_offset(read_offset_in_file_); - } return ret; } diff --git a/src/storage/tmp_file/ob_tmp_file_io_define.h b/src/storage/tmp_file/ob_tmp_file_io_define.h index 4ab16df5b..7ced1481b 100644 --- a/src/storage/tmp_file/ob_tmp_file_io_define.h +++ b/src/storage/tmp_file/ob_tmp_file_io_define.h @@ -44,14 +44,14 @@ class ObTmpFileIOHandle final public: ObTmpFileIOHandle(); ~ObTmpFileIOHandle(); - int init_write(const ObTmpFileIOInfo &io_info, ObTmpFileHandle &tmp_file_handle); - int init_read(const ObTmpFileIOInfo &io_info, ObTmpFileHandle &tmp_file_handle); - int init_pread(const ObTmpFileIOInfo &io_info, const int64_t read_offset, ObTmpFileHandle &tmp_file_handle); + int init_write(const ObTmpFileIOInfo &io_info); + int init_read(const ObTmpFileIOInfo &io_info); + int init_pread(const ObTmpFileIOInfo &io_info, const int64_t read_offset); int wait(); void reset(); bool is_valid() const; - TO_STRING_KV(K(is_inited_), K(tmp_file_handle_), K(ctx_), + TO_STRING_KV(K(is_inited_), K(fd_), K(ctx_), KP(buf_), K(update_offset_in_file_), K(buf_size_), K(done_size_), K(read_offset_in_file_)); @@ -66,7 +66,7 @@ private: private: bool is_inited_; - ObTmpFileHandle tmp_file_handle_; + int64_t fd_; ObTmpFileIOCtx ctx_; char *buf_; bool update_offset_in_file_; diff --git a/src/storage/tmp_file/ob_tmp_file_manager.cpp b/src/storage/tmp_file/ob_tmp_file_manager.cpp index 790dbfd14..bed07322c 100644 --- a/src/storage/tmp_file/ob_tmp_file_manager.cpp +++ b/src/storage/tmp_file/ob_tmp_file_manager.cpp @@ -303,8 +303,8 @@ int ObTenantTmpFileManager::aio_read(const ObTmpFileIOInfo &io_info, ObTmpFileIO } else if (FALSE_IT(io_handle.reset())) { } else if (OB_FAIL(get_tmp_file(io_info.fd_, tmp_file_handle))) { LOG_WARN("fail to get tmp file io handle", KR(ret), K(io_info)); - } else if (OB_FAIL(io_handle.init_read(io_info, tmp_file_handle))) { - LOG_WARN("fail to init io handle", KR(ret), K(io_info), K(tmp_file_handle)); + } else if (OB_FAIL(io_handle.init_read(io_info))) { + LOG_WARN("fail to init io handle", KR(ret), K(io_info)); } else if (OB_FAIL(tmp_file_handle.get()->aio_pread(io_handle.get_io_ctx()))) { LOG_WARN("fail to aio pread", KR(ret), K(io_info)); } @@ -332,8 +332,8 @@ int ObTenantTmpFileManager::aio_pread(const ObTmpFileIOInfo &io_info, } else if (FALSE_IT(io_handle.reset())) { } else if (OB_FAIL(get_tmp_file(io_info.fd_, tmp_file_handle))) { LOG_WARN("fail to get tmp file io handle", KR(ret), K(io_info)); - } else if (OB_FAIL(io_handle.init_pread(io_info, offset, tmp_file_handle))) { - LOG_WARN("fail to init io handle", KR(ret), K(io_info), K(tmp_file_handle)); + } else if (OB_FAIL(io_handle.init_pread(io_info, offset))) { + LOG_WARN("fail to init io handle", KR(ret), K(io_info)); } else if (OB_FAIL(tmp_file_handle.get()->aio_pread(io_handle.get_io_ctx()))) { LOG_WARN("fail to aio pread", KR(ret), K(io_info)); } @@ -359,8 +359,8 @@ int ObTenantTmpFileManager::read(const ObTmpFileIOInfo &io_info, ObTmpFileIOHand } else if (FALSE_IT(io_handle.reset())) { } else if (OB_FAIL(get_tmp_file(io_info.fd_, tmp_file_handle))) { LOG_WARN("fail to get tmp file io handle", KR(ret), K(io_info)); - } else if (OB_FAIL(io_handle.init_read(io_info, tmp_file_handle))) { - LOG_WARN("fail to init io handle", KR(ret), K(io_info), K(tmp_file_handle)); + } else if (OB_FAIL(io_handle.init_read(io_info))) { + LOG_WARN("fail to init io handle", KR(ret), K(io_info)); } else if (OB_FAIL(tmp_file_handle.get()->aio_pread(io_handle.get_io_ctx()))) { LOG_WARN("fail to aio pread", KR(ret), K(io_info)); } @@ -394,8 +394,8 @@ int ObTenantTmpFileManager::pread(const ObTmpFileIOInfo &io_info, const int64_t } else if (FALSE_IT(io_handle.reset())) { } else if (OB_FAIL(get_tmp_file(io_info.fd_, tmp_file_handle))) { LOG_WARN("fail to get tmp file io handle", KR(ret), K(io_info)); - } else if (OB_FAIL(io_handle.init_pread(io_info, offset, tmp_file_handle))) { - LOG_WARN("fail to init io handle", KR(ret), K(io_info), K(tmp_file_handle)); + } else if (OB_FAIL(io_handle.init_pread(io_info, offset))) { + LOG_WARN("fail to init io handle", KR(ret), K(io_info)); } else if (OB_FAIL(tmp_file_handle.get()->aio_pread(io_handle.get_io_ctx()))) { LOG_WARN("fail to aio pread", KR(ret), K(io_info)); } @@ -426,8 +426,8 @@ int ObTenantTmpFileManager::aio_write(const ObTmpFileIOInfo &io_info, ObTmpFileI LOG_WARN("invalid argument", KR(ret), K(io_info)); } else if (OB_FAIL(get_tmp_file(io_info.fd_, tmp_file_handle))) { LOG_WARN("fail to get tmp file io handle", KR(ret), K(io_info)); - } else if (OB_FAIL(io_handle.init_write(io_info, tmp_file_handle))) { - LOG_WARN("fail to init io handle", KR(ret), K(io_info), K(tmp_file_handle)); + } else if (OB_FAIL(io_handle.init_write(io_info))) { + LOG_WARN("fail to init io handle", KR(ret), K(io_info)); } else if (OB_FAIL(tmp_file_handle.get()->aio_write(io_handle.get_io_ctx()))) { LOG_WARN("fail to aio write", KR(ret), K(io_info)); } From 10993c4915b088b20ddff4fdee351fa154f9e868 Mon Sep 17 00:00:00 2001 From: fengdeyiji <546976189@qq.com> Date: Wed, 21 Aug 2024 13:11:36 +0000 Subject: [PATCH 160/249] [CP] [MDS] deserialize buffer_ctx by multi_data_source id --- src/storage/multi_data_source/buffer_ctx.cpp | 63 ++++++++- .../ob_tablet_create_mds_ctx.cpp | 54 +++++--- .../ob_multi_data_source_tx_buffer_node.cpp | 3 + .../tx/ob_multi_data_source_tx_buffer_node.h | 8 +- src/storage/tx_table/ob_tx_ctx_table.cpp | 5 +- unittest/storage/CMakeLists.txt | 1 + .../test_mds_new_ctx_deserialized.cpp | 122 ++++++++++++++++++ 7 files changed, 227 insertions(+), 29 deletions(-) create mode 100644 unittest/storage/multi_data_source/test_mds_new_ctx_deserialized.cpp diff --git a/src/storage/multi_data_source/buffer_ctx.cpp b/src/storage/multi_data_source/buffer_ctx.cpp index 4cf4d4ce4..b7c9118fc 100644 --- a/src/storage/multi_data_source/buffer_ctx.cpp +++ b/src/storage/multi_data_source/buffer_ctx.cpp @@ -26,6 +26,7 @@ #include "storage/multi_data_source/runtime_utility/mds_tenant_service.h" #include "storage/tx/ob_trans_define.h" #include "storage/multi_data_source/runtime_utility/mds_tenant_service.h" +#include "storage/tx/ob_trans_define.h" namespace oceanbase { @@ -117,17 +118,65 @@ int deserialize_(BufferCtx *&ctx_, return ret; } +/****************************************************for compat********************************************************/ +transaction::ObTxBufferNode *get_current_tx_buffer_node() { + transaction::ObTxBufferNode *tx_buffer_node = nullptr; + if (transaction::TLOCAL_P_TX_BUFFER_NODE_ARRAY) { + transaction::ObTxBufferNodeArray &array = *transaction::TLOCAL_P_TX_BUFFER_NODE_ARRAY; + for (int64_t idx = 0; idx < array.count(); ++idx) { + if (!array[idx].has_deserialized_buffer_ctx()) { + tx_buffer_node = &array[idx]; + break; + } + } + } + return tx_buffer_node; +} +int get_ctx_type_id_by_multi_data_source_type_idx(const transaction::ObTxDataSourceType multi_data_source_type, int64_t &ctx_type_idx) { + int ret = OB_SUCCESS; + switch (multi_data_source_type) { + #define NEED_GENERATE_MDS_FRAME_CODE_FOR_TRANSACTION + #define _GENERATE_MDS_FRAME_CODE_FOR_TRANSACTION_(HELPER_CLASS, BUFFER_CTX_TYPE, ID, ENUM_NAME) \ + case transaction::ObTxDataSourceType::ENUM_NAME:\ + {\ + ctx_type_idx = TupleTypeIdx::value;\ + }\ + break; + #include "storage/multi_data_source/compile_utility/mds_register.h" + #undef _GENERATE_MDS_FRAME_CODE_FOR_TRANSACTION_ + #undef NEED_GENERATE_MDS_FRAME_CODE_FOR_TRANSACTION + default:// this is an old MDS out of FRAME code, for example: table lock + MDS_LOG(INFO, "this multi data source is out of frame", KR(ret), K(ctx_type_idx), K(multi_data_source_type)); + break; + } + return ret; +} +/**********************************************************************************************************************/ int BufferCtxNode::deserialize(const char *buf, const int64_t buf_len, int64_t &pos, ObIAllocator &allocator) { int ret = OB_SUCCESS; MDS_TG(10_ms); - int64_t type_idx = INVALID_VALUE; - if (MDS_FAIL(serialization::decode(buf, buf_len, pos, type_idx))) { - MDS_LOG(ERROR, "fail to deserialize buffer ctx id", KR(ret), K(type_idx)); - } else if (INVALID_VALUE == type_idx) { - MDS_LOG(DEBUG, "deserialized INVALD buffer ctx", KR(ret), K(type_idx), K(buf_len), K(pos)); - } else if (MDS_FAIL(deserialize_<0>(ctx_, type_idx, buf, buf_len, pos, allocator))) { - MDS_LOG(WARN, "deserialized buffer ctx failed", KR(ret), K(type_idx)); + int64_t ctx_type_idx = INVALID_VALUE; + transaction::ObTxBufferNode *tx_buffer_node = get_current_tx_buffer_node(); + if (MDS_FAIL(serialization::decode(buf, buf_len, pos, ctx_type_idx))) { + MDS_LOG(ERROR, "fail to deserialize buffer ctx id", KR(ret), K(ctx_type_idx)); + } else if (INVALID_VALUE == ctx_type_idx) { + MDS_LOG(DEBUG, "deserialized INVALD buffer ctx", KR(ret), K(ctx_type_idx), K(buf_len), K(pos)); + } else { + if (tx_buffer_node) { + if (OB_FAIL(get_ctx_type_id_by_multi_data_source_type_idx(tx_buffer_node->get_data_source_type(), ctx_type_idx))) { + MDS_LOG(ERROR, "fail get_ctx_type_id_by_multi_data_source_type_idx", KR(ret), K(ctx_type_idx)); + } + } + if (OB_FAIL(ret)) { + } else if (MDS_FAIL(deserialize_<0>(ctx_, ctx_type_idx, buf, buf_len, pos, allocator))) { + MDS_LOG(WARN, "deserialized buffer ctx failed", KR(ret), K(ctx_type_idx)); + } + } + if (OB_SUCC(ret)) { + if (tx_buffer_node) { + tx_buffer_node->set_has_deserialized_buffer_ctx(); + } } return ret; } diff --git a/src/storage/multi_data_source/ob_tablet_create_mds_ctx.cpp b/src/storage/multi_data_source/ob_tablet_create_mds_ctx.cpp index c83c70f4f..0469ca7c0 100644 --- a/src/storage/multi_data_source/ob_tablet_create_mds_ctx.cpp +++ b/src/storage/multi_data_source/ob_tablet_create_mds_ctx.cpp @@ -88,6 +88,7 @@ int ObTabletCreateMdsCtx::serialize(char *buf, const int64_t buf_len, int64_t &p int ObTabletCreateMdsCtx::deserialize(const char *buf, const int64_t buf_len, int64_t &pos) { int ret = OB_SUCCESS; + int64_t origin_pos = pos; int64_t tmp_pos = pos; int32_t magic = -1; int32_t version = -1; @@ -100,28 +101,41 @@ int ObTabletCreateMdsCtx::deserialize(const char *buf, const int64_t buf_len, in LOG_WARN("invalid args", K(ret), K(buf), K(buf_len), K(pos)); } else if (OB_FAIL(MdsCtx::deserialize(buf, buf_len, tmp_pos))) { LOG_WARN("fail to deserialize mds ctx", K(ret), K(buf_len), K(tmp_pos)); - } else if (OB_FAIL(serialization::decode(buf, buf_len, tmp_pos, magic))) { - LOG_WARN("failed to deserialize magic", K(ret), K(buf_len), K(tmp_pos)); - } else if (OB_UNLIKELY(magic != MAGIC)) { - FLOG_INFO("magic does not match, maybe this is old version data", K(ret), K(magic), LITERAL_K(MAGIC)); - version_ = VERSION; - ls_id_ = ObLSID::INVALID_LS_ID; - pos = tmp_pos; - } else if (OB_FAIL(serialization::decode(buf, buf_len, tmp_pos, version))) { - LOG_WARN("failed to deserialize version", K(ret), K(buf_len), K(tmp_pos)); - } else if (OB_UNLIKELY(VERSION != version)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("version does not match", K(ret), K(version)); - } else if (OB_FAIL(serialization::decode_i64(buf, buf_len, tmp_pos, &serialize_size))) { - LOG_WARN("failed to deserialize serialize size", K(ret), K(buf_len), K(tmp_pos)); - } else if (tmp_pos - pos < serialize_size && OB_FAIL(ls_id_.deserialize(buf, buf_len, tmp_pos))) { - LOG_WARN("failed to deserialize ls id", K(ret), K(buf_len), K(tmp_pos)); - } else if (OB_UNLIKELY(tmp_pos - pos != serialize_size)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("deserialize length does not match", K(ret), K(buf_len), K(pos), K(tmp_pos), K(serialize_size)); } else { - version_ = version; + int tmp_ret = OB_SUCCESS; pos = tmp_pos; + bool is_old_data = false; + if (tmp_pos == buf_len) { + LOG_WARN("buffer is not enough for magic deserialize", K(ret), K(buf_len), K(tmp_pos)); + is_old_data = true; + } else if (OB_TMP_FAIL(serialization::decode(buf, buf_len, tmp_pos, magic))) { + LOG_WARN("decode magic from buffer failed", K(tmp_ret), K(ret), K(buf_len), K(tmp_pos)); + is_old_data = true; + } else if (magic != MAGIC) { + LOG_WARN("magic not match", K(tmp_ret), K(ret), K(buf_len), K(tmp_pos)); + is_old_data = true; + } else if (OB_FAIL(serialization::decode(buf, buf_len, tmp_pos, version))) { + LOG_WARN("failed to deserialize version", K(ret), K(buf_len), K(tmp_pos)); + } else if (OB_UNLIKELY(VERSION != version)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("version does not match", K(ret), K(version)); + } else if (OB_FAIL(serialization::decode_i64(buf, buf_len, tmp_pos, &serialize_size))) { + LOG_WARN("failed to deserialize serialize size", K(ret), K(buf_len), K(tmp_pos)); + } else if (tmp_pos - origin_pos < serialize_size && OB_FAIL(ls_id_.deserialize(buf, buf_len, tmp_pos))) { + LOG_WARN("failed to deserialize ls id", K(ret), K(buf_len), K(tmp_pos)); + } else if (OB_UNLIKELY(tmp_pos - origin_pos != serialize_size)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("deserialize length does not match", K(ret), K(buf_len), K(pos), K(tmp_pos), K(serialize_size)); + } else { + version_ = version; + pos = tmp_pos; + } + + if (is_old_data) { + FLOG_INFO("maybe meet old version data", K(ret), K(magic), LITERAL_K(MAGIC)); + version_ = VERSION; + ls_id_ = ObLSID::INVALID_LS_ID; + } } return ret; diff --git a/src/storage/tx/ob_multi_data_source_tx_buffer_node.cpp b/src/storage/tx/ob_multi_data_source_tx_buffer_node.cpp index 718df3301..5b5fc5478 100644 --- a/src/storage/tx/ob_multi_data_source_tx_buffer_node.cpp +++ b/src/storage/tx/ob_multi_data_source_tx_buffer_node.cpp @@ -24,6 +24,8 @@ namespace oceanbase namespace transaction { +thread_local ObTxBufferNodeArray *TLOCAL_P_TX_BUFFER_NODE_ARRAY = nullptr;// FIXME: for compat issue, should be removed after barrier version + ObTxBufferNode::ObTxBufferNode() : seq_no_(), type_(ObTxDataSourceType::UNKNOWN), @@ -72,6 +74,7 @@ void ObTxBufferNode::reset() has_submitted_ = false; has_synced_ = false; mds_base_scn_.reset(); + has_deserialized_buffer_ctx_ = false; } int ObTxBufferNode::set_mds_register_no(const uint64_t register_no) diff --git a/src/storage/tx/ob_multi_data_source_tx_buffer_node.h b/src/storage/tx/ob_multi_data_source_tx_buffer_node.h index 8acb8430a..eb21b6889 100644 --- a/src/storage/tx/ob_multi_data_source_tx_buffer_node.h +++ b/src/storage/tx/ob_multi_data_source_tx_buffer_node.h @@ -77,6 +77,9 @@ public: void set_synced() { has_synced_ = true; } bool is_synced() const { return has_synced_; } + void set_has_deserialized_buffer_ctx() { has_deserialized_buffer_ctx_ = true; } + bool has_deserialized_buffer_ctx() const { return has_deserialized_buffer_ctx_; } + const share::SCN &get_base_scn() { return mds_base_scn_; } bool operator==(const ObTxBufferNode &buffer_node) const; @@ -92,12 +95,14 @@ public: K(has_submitted_), K(has_synced_), "type", ObMultiDataSourcePrinter::to_str_mds_type(type_), - K(data_.length())); + K(data_.length()), + K(has_deserialized_buffer_ctx_)); private: uint64_t register_no_; ObTxSEQ seq_no_; bool has_submitted_; bool has_synced_; + bool has_deserialized_buffer_ctx_;// FIXME: for compat issue, should be removed after barrier version share::SCN mds_base_scn_; ObTxDataSourceType type_; common::ObString data_; @@ -128,6 +133,7 @@ private: typedef common::ObSEArray ObTxBufferNodeArray; typedef common::ObSEArray ObTxBufferCtxArray; +extern thread_local ObTxBufferNodeArray *TLOCAL_P_TX_BUFFER_NODE_ARRAY;// FIXME: for compat issue, should be removed after barrier version } } diff --git a/src/storage/tx_table/ob_tx_ctx_table.cpp b/src/storage/tx_table/ob_tx_ctx_table.cpp index 255b03b92..cd12cb242 100644 --- a/src/storage/tx_table/ob_tx_ctx_table.cpp +++ b/src/storage/tx_table/ob_tx_ctx_table.cpp @@ -217,8 +217,11 @@ int ObTxCtxTableRecoverHelper::recover(const blocksstable::ObDatumRow &row, int64_t pos = 0; bool tx_ctx_existed = true; ctx_info_.set_compatible_version(curr_meta.get_version()); - if (OB_FAIL(ctx_info_.deserialize(deserialize_buf, deserialize_buf_length, pos, tx_data_table))) { + if (FALSE_IT(TLOCAL_P_TX_BUFFER_NODE_ARRAY = &ctx_info_.exec_info_.multi_data_source_)) {// FIXME: for compat issue, should be removed after barrier version + } else if (OB_FAIL(ctx_info_.deserialize(deserialize_buf, deserialize_buf_length, pos, tx_data_table))) { STORAGE_LOG(WARN, "failed to deserialize status_info", K(ret), K_(ctx_info)); + TLOCAL_P_TX_BUFFER_NODE_ARRAY = nullptr;// FIXME: for compat issue, should be removed after barrier version + } else if (FALSE_IT(TLOCAL_P_TX_BUFFER_NODE_ARRAY = nullptr)) {// FIXME: for compat issue, should be removed after barrier version } else if (FALSE_IT(ctx_info_.exec_info_.mrege_buffer_ctx_array_to_multi_data_source())) { } else if (OB_FAIL(recover_one_tx_ctx_(ls_tx_ctx_mgr, ctx_info_))) { // heap memory needed be freed, but can not do this in destruction, cause tx_buffer_node has no value sematics diff --git a/unittest/storage/CMakeLists.txt b/unittest/storage/CMakeLists.txt index 6a3f5e050..e553af48d 100644 --- a/unittest/storage/CMakeLists.txt +++ b/unittest/storage/CMakeLists.txt @@ -68,6 +68,7 @@ storage_unittest(test_mvcc_callback memtable/mvcc/test_mvcc_callback.cpp) # storage_unittest(test_mds_compile multi_data_source/test_mds_compile.cpp) storage_unittest(test_mds_list multi_data_source/test_mds_list.cpp) storage_unittest(test_mds_node multi_data_source/test_mds_node.cpp) +storage_unittest(test_mds_new_ctx_deserialized multi_data_source/test_mds_new_ctx_deserialized.cpp) # storage_unittest(test_mds_row multi_data_source/test_mds_row.cpp) # storage_unittest(test_mds_unit multi_data_source/test_mds_unit.cpp) storage_unittest(test_mds_table multi_data_source/test_mds_table.cpp) diff --git a/unittest/storage/multi_data_source/test_mds_new_ctx_deserialized.cpp b/unittest/storage/multi_data_source/test_mds_new_ctx_deserialized.cpp new file mode 100644 index 000000000..002abdbd1 --- /dev/null +++ b/unittest/storage/multi_data_source/test_mds_new_ctx_deserialized.cpp @@ -0,0 +1,122 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#include "storage/multi_data_source/compile_utility/compile_mapper.h" +#include "storage/multi_data_source/mds_ctx.h" +#include "storage/multi_data_source/mds_writer.h" +#include "storage/multi_data_source/ob_tablet_create_mds_ctx.h" +#include "storage/tx/ob_trans_define.h" +#define UNITTEST_DEBUG +#include "lib/utility/utility.h" +#include +#include +#define private public +#define protected public +#include "storage/multi_data_source/compile_utility/mds_dummy_key.h" +#include "storage/multi_data_source/runtime_utility/common_define.h" + +#include +#include +#include +#include +#include "storage/multi_data_source/runtime_utility/mds_factory.h" +#include "common/ob_clock_generator.h" +#include "storage/multi_data_source/mds_node.h" +#include "common/meta_programming/ob_type_traits.h" +#include "storage/multi_data_source/mds_row.h" +namespace oceanbase { +namespace storage +{ +namespace mds +{ +void *MdsAllocator::alloc(const int64_t size) +{ + void *ptr = ob_malloc(size, "MDS"); + ATOMIC_INC(&alloc_times_); + MDS_LOG(DEBUG, "alloc obj", KP(ptr), K(size), K(lbt())); + return ptr; +} +void MdsAllocator::free(void *ptr) { + ATOMIC_INC(&free_times_); + MDS_LOG(DEBUG, "free obj", KP(ptr), K(lbt())); + ob_free(ptr); +} +} +} +namespace unittest { + +using namespace common; +using namespace std; +using namespace storage; +using namespace mds; + +class TestMdsNewCtxDeserialized: public ::testing::Test +{ +public: + TestMdsNewCtxDeserialized() {}; + virtual ~TestMdsNewCtxDeserialized() {}; + virtual void SetUp() { + }; + virtual void TearDown() { + }; +private: + // disallow copy + DISALLOW_COPY_AND_ASSIGN(TestMdsNewCtxDeserialized); +}; + +TEST_F(TestMdsNewCtxDeserialized, deserialized_from_mds_ctx) { + MdsCtx old_ctx; + old_ctx.set_writer(MdsWriter(transaction::ObTransID(1))); + old_ctx.set_binding_type_id(TupleTypeIdx::value); + char buffer[1024]; + for (auto &ch : buffer) + ch = 0xff; + int64_t pos = 0; + ASSERT_EQ(OB_SUCCESS, old_ctx.serialize(buffer, 1024, pos)); + int64_t buffer_len = pos; + pos = 0; + ObTabletCreateMdsCtx new_ctx1; + ASSERT_EQ(OB_SUCCESS, new_ctx1.deserialize(buffer, buffer_len, pos)); + pos = 0; + ASSERT_EQ(OB_SUCCESS, new_ctx1.deserialize(buffer, buffer_len + 1, pos)); + pos = 0; + for (int idx = buffer_len; idx < 1024; ++idx) + buffer[idx] = 0; + ASSERT_EQ(OB_SUCCESS, new_ctx1.deserialize(buffer, buffer_len + 10, pos)); + pos = 0; + ObStartTransferInMdsCtx new_ctx2; + ASSERT_EQ(OB_SUCCESS, new_ctx2.deserialize(buffer, buffer_len, pos)); + pos = 0; + ObFinishTransferInMdsCtx new_ctx3; + ASSERT_EQ(OB_SUCCESS, new_ctx3.deserialize(buffer, buffer_len, pos)); +} + +} +} + +int main(int argc, char **argv) +{ + system("rm -rf test_mds_new_ctx_deserialized.log"); + oceanbase::common::ObLogger &logger = oceanbase::common::ObLogger::get_logger(); + logger.set_file_name("test_mds_new_ctx_deserialized.log", false); + logger.set_log_level(OB_LOG_LEVEL_DEBUG); + testing::InitGoogleTest(&argc, argv); + int ret = RUN_ALL_TESTS(); + int64_t alloc_times = oceanbase::storage::mds::MdsAllocator::get_alloc_times(); + int64_t free_times = oceanbase::storage::mds::MdsAllocator::get_free_times(); + if (alloc_times != free_times) { + MDS_LOG(ERROR, "memory may leak", K(free_times), K(alloc_times)); + ret = -1; + } else { + MDS_LOG(INFO, "all memory released", K(free_times), K(alloc_times)); + } + return ret; +} \ No newline at end of file From ee4a27aa398541ee627c427b38e05869ea0cb81f Mon Sep 17 00:00:00 2001 From: godyangfight Date: Wed, 21 Aug 2024 13:17:32 +0000 Subject: [PATCH 161/249] Fix restore transfer backfill check ddl major sstable bug. --- src/storage/tablet/ob_tablet_meta.cpp | 11 +++++++++-- src/storage/tablet/ob_tablet_table_store.cpp | 12 +++++++++--- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/storage/tablet/ob_tablet_meta.cpp b/src/storage/tablet/ob_tablet_meta.cpp index 08684d081..9aed3bf51 100644 --- a/src/storage/tablet/ob_tablet_meta.cpp +++ b/src/storage/tablet/ob_tablet_meta.cpp @@ -415,8 +415,15 @@ int ObTabletMeta::init( } ObTabletTableStoreFlag table_store_flag = old_tablet_meta.table_store_flag_; + SCN ddl_checkpoint_scn = old_tablet_meta.ddl_checkpoint_scn_; if (!table_store_flag.with_major_sstable()) { - table_store_flag = OB_ISNULL(tablet_meta) ? table_store_flag : tablet_meta->table_store_flag_; + if (OB_ISNULL(tablet_meta)) { + //do nothing + } else if (tablet_meta->table_store_flag_.with_major_sstable()) { + table_store_flag.set_with_major_sstable(); + ddl_checkpoint_scn = tablet_meta->ddl_checkpoint_scn_; + FLOG_INFO("update tablet table store flag with major", KPC(tablet_meta), K(table_store_flag), K(ddl_checkpoint_scn)); + } } const SCN mds_checkpoint_scn = OB_ISNULL(tablet_meta) ? old_tablet_meta.mds_checkpoint_scn_ : MAX(old_tablet_meta.mds_checkpoint_scn_, tablet_meta->mds_checkpoint_scn_); @@ -446,7 +453,7 @@ int ObTabletMeta::init( create_scn_ = old_tablet_meta.create_scn_; start_scn_ = old_tablet_meta.start_scn_; clog_checkpoint_scn_ = clog_checkpoint_scn; - ddl_checkpoint_scn_ = old_tablet_meta.ddl_checkpoint_scn_; + ddl_checkpoint_scn_ = ddl_checkpoint_scn; snapshot_version_ = snapshot_version; multi_version_start_ = multi_version_start; ha_status_ = new_ha_status; diff --git a/src/storage/tablet/ob_tablet_table_store.cpp b/src/storage/tablet/ob_tablet_table_store.cpp index b3dabeec3..876ca661a 100644 --- a/src/storage/tablet/ob_tablet_table_store.cpp +++ b/src/storage/tablet/ob_tablet_table_store.cpp @@ -1820,8 +1820,14 @@ int ObTabletTableStore::build_ddl_sstables( } } - if (OB_SUCC(ret) && !ddl_dump_sstables.empty() && major_tables_.empty()) { - if (OB_FAIL(ddl_sstables_.init(allocator, ddl_dump_sstables))) { + if (OB_SUCC(ret)) { + if (ddl_dump_sstables.empty()) { + //do nothing + } else if (!major_tables_.empty()) { + LOG_INFO("major sstables is not empty, no need update ddl sstable", K(ddl_dump_sstables), K(major_tables_)); + } else if (tablet.get_tablet_meta().table_store_flag_.with_major_sstable()) { + LOG_INFO("tablet has with major sstable flag, no need update ddl sstable", K(tablet), K(ddl_dump_sstables)); + } else if (OB_FAIL(ddl_sstables_.init(allocator, ddl_dump_sstables))) { LOG_WARN("failed to init ddl_sstables", K(ret)); } } @@ -2471,7 +2477,7 @@ int ObTabletTableStore::replace_ha_ddl_tables_( bool need_add_ddl_tables = true; ObSSTableMetaHandle new_meta_handle; - if (!old_store.major_tables_.empty()) { + if (!old_store.major_tables_.empty() || tablet.get_tablet_meta().table_store_flag_.with_major_sstable()) { need_add_ddl_tables = false; } From 54f6db574ae29a0df20be5b606db198fcbc3a980 Mon Sep 17 00:00:00 2001 From: dongb0 <708848999@qq.com> Date: Wed, 21 Aug 2024 13:36:06 +0000 Subject: [PATCH 162/249] fix external sort ut core dump --- src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp | 4 +++- unittest/storage/test_parallel_external_sort.cpp | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp b/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp index aed11d636..29f5f628b 100644 --- a/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp +++ b/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp @@ -900,7 +900,9 @@ int ObSharedNothingTmpFile::aio_write(ObTmpFileIOCtx &io_ctx) if (OB_FAIL(inner_write_(io_ctx))) { if (OB_ALLOCATE_TMP_FILE_PAGE_FAILED == ret) { ret = OB_SUCCESS; - LOG_INFO("alloc mem failed, try to evict pages", K(fd_), K(file_size_), K(io_ctx), KPC(this)); + if (TC_REACH_COUNT_INTERVAL(10)) { + LOG_INFO("alloc mem failed, try to evict pages", K(fd_), K(file_size_), K(io_ctx), KPC(this)); + } if (OB_FAIL(page_cache_controller_->invoke_swap_and_wait( MIN(io_ctx.get_todo_size(), ObTmpFileGlobal::TMP_FILE_WRITE_BATCH_PAGE_NUM * ObTmpFileGlobal::PAGE_SIZE), io_ctx.get_io_timeout_ms()))) { diff --git a/unittest/storage/test_parallel_external_sort.cpp b/unittest/storage/test_parallel_external_sort.cpp index e5e06cdd7..9a292ab18 100644 --- a/unittest/storage/test_parallel_external_sort.cpp +++ b/unittest/storage/test_parallel_external_sort.cpp @@ -211,6 +211,14 @@ void TestParallelExternalSort::SetUp() void TestParallelExternalSort::TearDown() { allocator_.reuse(); + // ObTenantTmpFileManager uses ObServerBlockManager, which is destroyed in TestDataFilePrepare::TearDown() + // so we need to destroy ObTenantTmpFileManager first + tmp_file::ObTenantTmpFileManager *tmp_file_mgr = MTL(tmp_file::ObTenantTmpFileManager *); + if (OB_NOT_NULL(tmp_file_mgr)) { + tmp_file_mgr->stop(); + tmp_file_mgr->wait(); + tmp_file_mgr->destroy(); + } tmp_file::ObTmpBlockCache::get_instance().destroy(); tmp_file::ObTmpPageCache::get_instance().destroy(); TestDataFilePrepare::TearDown(); From 3eb803a4d087dee1b252829533a62310d8735383 Mon Sep 17 00:00:00 2001 From: LoLolobster <949673574@qq.com> Date: Wed, 21 Aug 2024 13:42:32 +0000 Subject: [PATCH 163/249] Discard user restore job if sys restore job failed --- .../restore/ob_restore_scheduler.cpp | 33 ++++++++++++++++++- src/rootserver/restore/ob_restore_scheduler.h | 1 + src/share/ob_debug_sync_point.h | 1 + 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/rootserver/restore/ob_restore_scheduler.cpp b/src/rootserver/restore/ob_restore_scheduler.cpp index 988112a39..ab1b3397b 100644 --- a/src/rootserver/restore/ob_restore_scheduler.cpp +++ b/src/rootserver/restore/ob_restore_scheduler.cpp @@ -238,6 +238,7 @@ int ObRestoreScheduler::restore_tenant(const ObPhysicalRestoreJob &job_info) } else if (OB_FAIL(rpc_proxy_->timeout(timeout).create_tenant(arg, tenant_id))) { LOG_WARN("fail to create tenant", K(ret), K(arg)); } else { + DEBUG_SYNC(AFTER_PHYSICAL_RESTORE_CREATE_TENANT); ObPhysicalRestoreTableOperator restore_op; const int64_t job_id = job_info.get_job_id(); const uint64_t new_tenant_id = tenant_id; @@ -402,6 +403,7 @@ int ObRestoreScheduler::check_tenant_can_restore_(const uint64_t tenant_id) int ObRestoreScheduler::restore_pre(const ObPhysicalRestoreJob &job_info) { int ret = OB_SUCCESS; + bool is_sys_ready = true; if (!inited_) { ret = OB_NOT_INIT; LOG_WARN("not inited", K(ret)); @@ -411,6 +413,15 @@ int ObRestoreScheduler::restore_pre(const ObPhysicalRestoreJob &job_info) LOG_WARN("invalid tenant id", K(ret), K(tenant_id_)); } else if (OB_FAIL(restore_service_->check_stop())) { LOG_WARN("restore scheduler stopped", K(ret)); + } else if (OB_FAIL(wait_sys_job_ready_(job_info, is_sys_ready))) { + if (OB_ENTRY_NOT_EXIST == ret) { + ret = OB_CREATE_STANDBY_TENANT_FAILED; + LOG_WARN("sys restore job has failed, set user restore job in failure too", K(ret), K(job_info)); + } else { + LOG_WARN("fail to wait sys job ready", K(ret), K(job_info)); + } + } else if (!is_sys_ready) { // sys job not in WAIT_RETSTORE_TENANT_FINISH state + ret = OB_EAGAIN; } else if (share::ObBackupSetFileDesc::is_backup_set_support_quick_restore( static_cast(job_info.get_backup_compatible())) && OB_FAIL(update_tenant_restore_data_mode_to_remote_(tenant_id_))) { @@ -423,7 +434,7 @@ int ObRestoreScheduler::restore_pre(const ObPhysicalRestoreJob &job_info) LOG_WARN("fail to fill restore statistics", K(ret), K(job_info)); } - if (OB_IO_ERROR == ret || OB_SUCC(ret)) { + if (OB_IO_ERROR == ret || OB_CREATE_STANDBY_TENANT_FAILED == ret || OB_SUCC(ret)) { int tmp_ret = OB_SUCCESS; if (OB_TMP_FAIL(try_update_job_status(*sql_proxy_, ret, job_info))) { LOG_WARN("fail to update job status", K(ret), K(tmp_ret), K(job_info)); @@ -437,6 +448,26 @@ int ObRestoreScheduler::restore_pre(const ObPhysicalRestoreJob &job_info) return ret; } +int ObRestoreScheduler::wait_sys_job_ready_(const ObPhysicalRestoreJob &job, bool &is_ready) { + int ret = OB_SUCCESS; + ObPhysicalRestoreJob sys_job; + is_ready = false; + if (!job.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid job", K(ret), K(job)); + } else { + ObPhysicalRestoreTableOperator restore_op; + if (OB_FAIL(restore_op.init(sql_proxy_, OB_SYS_TENANT_ID, share::OBCG_STORAGE /*group_id*/))) { + LOG_WARN("failed to init restore op", KR(ret)); + } else if (OB_FAIL(restore_op.get_job(job.get_initiator_job_id(), sys_job))) { + LOG_WARN("failed to get sys restore job history", KR(ret), K(job)); + } else if (PHYSICAL_RESTORE_WAIT_TENANT_RESTORE_FINISH == sys_job.get_status()) { + is_ready = true; + } + } + return ret; +} + int ObRestoreScheduler::fill_restore_statistics(const share::ObPhysicalRestoreJob &job_info) { int ret = OB_SUCCESS; diff --git a/src/rootserver/restore/ob_restore_scheduler.h b/src/rootserver/restore/ob_restore_scheduler.h index 73537fb55..a942a5671 100644 --- a/src/rootserver/restore/ob_restore_scheduler.h +++ b/src/rootserver/restore/ob_restore_scheduler.h @@ -116,6 +116,7 @@ private: int update_tenant_restore_data_mode_to_remote_(const uint64_t tenant_id); int update_tenant_restore_data_mode_to_normal_(const uint64_t tenant_id); int update_tenant_restore_data_mode_(const uint64_t tenant_id, const share::ObRestoreDataMode &new_restore_data_mode); + int wait_sys_job_ready_(const ObPhysicalRestoreJob &job, bool &is_ready); private: bool inited_; share::schema::ObMultiVersionSchemaService *schema_service_; diff --git a/src/share/ob_debug_sync_point.h b/src/share/ob_debug_sync_point.h index 17736d12b..fb4aa4bc7 100755 --- a/src/share/ob_debug_sync_point.h +++ b/src/share/ob_debug_sync_point.h @@ -632,6 +632,7 @@ class ObString; ACT(BEFORE_CLOSE_BACKUP_INDEX_BUILDER,)\ ACT(BEFROE_UPDATE_DATA_VERSION,)\ ACT(BEFORE_DATA_DICT_DUMP_FINISH,)\ + ACT(AFTER_PHYSICAL_RESTORE_CREATE_TENANT,)\ ACT(MAX_DEBUG_SYNC_POINT,) DECLARE_ENUM(ObDebugSyncPoint, debug_sync_point, OB_DEBUG_SYNC_POINT_DEF); From 2677fd458b851241f2acd0904235271a5f073eeb Mon Sep 17 00:00:00 2001 From: "18523270951@163.com" <18523270951@163.com> Date: Wed, 21 Aug 2024 14:00:23 +0000 Subject: [PATCH 164/249] [FEAT MERGE] Optimizing external table direct load performance --- deps/oblib/src/lib/charset/ob_ctype_utf8.cc | 4 +- src/sql/engine/expr/ob_datum_cast.cpp | 2 +- src/sql/engine/expr/ob_expr_column_conv.cpp | 114 +++++++++++++++++- src/sql/engine/expr/ob_expr_column_conv.h | 11 ++ .../engine/expr/ob_expr_eval_functions.cpp | 6 +- src/sql/resolver/dml/ob_dml_resolver.cpp | 38 ++++-- src/sql/resolver/dml/ob_dml_resolver.h | 6 +- 7 files changed, 157 insertions(+), 24 deletions(-) diff --git a/deps/oblib/src/lib/charset/ob_ctype_utf8.cc b/deps/oblib/src/lib/charset/ob_ctype_utf8.cc index e7b2e844b..3a54e0e81 100644 --- a/deps/oblib/src/lib/charset/ob_ctype_utf8.cc +++ b/deps/oblib/src/lib/charset/ob_ctype_utf8.cc @@ -52,7 +52,7 @@ static int ob_valid_mbcharlen_utf8mb3(const uchar *s, const uchar *e) return 3; } -static int ob_valid_mbcharlen_utf8mb4(const ObCharsetInfo *cs __attribute__((unused)), const uchar *s, const uchar *e) +static inline int ob_valid_mbcharlen_utf8mb4(const ObCharsetInfo *cs __attribute__((unused)), const uchar *s, const uchar *e) { uchar c; if (s >= e) @@ -97,7 +97,7 @@ static uint ob_mbcharlen_utf8mb4(const ObCharsetInfo *cs __attribute__((unused)) return 0; /* Illegal mb head */; } -static size_t ob_well_formed_len_utf8mb4(const ObCharsetInfo *cs, +static inline size_t ob_well_formed_len_utf8mb4(const ObCharsetInfo *cs, const char *b, const char *e, size_t pos, int *error) { diff --git a/src/sql/engine/expr/ob_datum_cast.cpp b/src/sql/engine/expr/ob_datum_cast.cpp index 9b0089b58..0e06e812e 100644 --- a/src/sql/engine/expr/ob_datum_cast.cpp +++ b/src/sql/engine/expr/ob_datum_cast.cpp @@ -11522,7 +11522,7 @@ static int float_range_check(const ObCastMode &cast_mode, return ret; } -int string_length_check(const ObExpr &expr, +OB_INLINE int string_length_check(const ObExpr &expr, const ObCastMode &cast_mode, const ObAccuracy &accuracy, const ObObjType type, diff --git a/src/sql/engine/expr/ob_expr_column_conv.cpp b/src/sql/engine/expr/ob_expr_column_conv.cpp index 67c4aea79..54c8a20de 100644 --- a/src/sql/engine/expr/ob_expr_column_conv.cpp +++ b/src/sql/engine/expr/ob_expr_column_conv.cpp @@ -347,12 +347,27 @@ int ObExprColumnConv::cg_expr(ObExprCGCtx &op_cg_ctx, | CM_CHARSET_CONVERT_IGNORE_ERR; } enumset_info->cast_mode_ |= CM_COLUMN_CONVERT; - rt_expr.eval_func_ = column_convert; + if (GET_MIN_CLUSTER_VERSION() >= CLUSTER_VERSION_4_3_3_0 + && (enumset_info->cast_mode_ & CM_FAST_COLUMN_CONV) + && !ob_is_enum_or_set_type(rt_expr.datum_meta_.type_)) { + rt_expr.eval_func_ = column_convert_fast; + if (rt_expr.args_[4]->is_batch_result()) { + rt_expr.eval_batch_func_ = column_convert_batch_fast; + } + } else { + rt_expr.eval_func_ = column_convert; + if (rt_expr.args_[4]->is_batch_result() + && !ob_is_enum_or_set_type(rt_expr.datum_meta_.type_) + && !is_lob_storage(rt_expr.datum_meta_.type_) + && GET_MIN_CLUSTER_VERSION() >= CLUSTER_VERSION_4_3_3_0) { + rt_expr.eval_batch_func_ = column_convert_batch; + } + } } return ret; } -static inline int column_convert_datum_accuracy_check(const ObExpr &expr, +static OB_INLINE int column_convert_datum_accuracy_check(const ObExpr &expr, ObEvalCtx &ctx, bool has_lob_header, ObDatum &datum, @@ -563,6 +578,101 @@ int ObExprColumnConv::column_convert(const ObExpr &expr, return ret; } +int ObExprColumnConv::column_convert_fast(const ObExpr &expr, + ObEvalCtx &ctx, + ObDatum &datum) +{ + int ret = OB_SUCCESS; + ObDatum *val = nullptr; + if (OB_FAIL(expr.args_[4]->eval(ctx, val))) { + LOG_WARN("evaluate parameter failed", K(ret)); + } else { + datum.set_datum(*val); + } + return ret; +} + +int ObExprColumnConv::column_convert_batch(const ObExpr &expr, + ObEvalCtx &ctx, + const ObBitVector &skip, + const int64_t batch_size) +{ + int ret = OB_SUCCESS; + ObObjType out_type = expr.datum_meta_.type_; + ObCollationType out_cs_type = expr.datum_meta_.cs_type_; + const ObEnumSetInfo *enumset_info = static_cast(expr.extra_info_); + const uint64_t cast_mode = enumset_info->cast_mode_; + bool is_strict = CM_IS_STRICT_MODE(cast_mode); + if (OB_FAIL(expr.args_[4]->eval_batch(ctx, skip, batch_size))) { + LOG_WARN("failed to eval batch vals", K(ret)); + } else { + ObDatum *vals = expr.args_[4]->locate_batch_datums(ctx); + ObDatum *results = expr.locate_batch_datums(ctx); + ObBitVector &eval_flags = expr.get_evaluated_flags(ctx); + bool is_string_type = ob_is_string_type(out_type); + ObAccuracy accuracy; + accuracy.set_length(expr.max_length_); + accuracy.set_scale(expr.datum_meta_.scale_); + const ObObjTypeClass &dst_tc = ob_obj_type_class(expr.datum_meta_.type_); + const ObLength max_accuracy_len = accuracy.get_length(); + if (is_string_type) { + accuracy.set_length_semantics(expr.datum_meta_.length_semantics_); + } + ObEvalCtx::BatchInfoScopeGuard batch_info_guard(ctx); + batch_info_guard.set_batch_size(batch_size); + for (int64_t i = 0; OB_SUCC(ret) && i < batch_size; ++i) { + if (skip.at(i) || eval_flags.at(i)) { + continue; + } + if (vals[i].is_null()) { + results[i].set_null(); + } else { + batch_info_guard.set_batch_idx(i); + if (is_string_type) { + ObString str = vals[i].get_string(); + if (OB_FAIL(string_collation_check(is_strict, out_cs_type, out_type, str))) { + LOG_WARN("fail to check collation", K(ret), K(str), K(is_strict), K(expr)); + } else { + vals[i].set_string(str); + } + } + bool no_need_check_length = is_string_type && max_accuracy_len > 0 && vals[i].len_ < max_accuracy_len; + if (OB_FAIL(ret)) { + } else if (no_need_check_length) { + results[i].set_datum(vals[i]); + } else if (OB_FAIL(column_convert_datum_accuracy_check(expr, ctx, false, results[i], cast_mode, vals[i]))) { + LOG_WARN("fail do datum_accuracy_check for lob res", K(ret), K(expr)); + } + } + eval_flags.set(i); + } + } + return ret; +} + +int ObExprColumnConv::column_convert_batch_fast(const ObExpr &expr, + ObEvalCtx &ctx, + const ObBitVector &skip, + const int64_t batch_size) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(expr.args_[4]->eval_batch(ctx, skip, batch_size))) { + LOG_WARN("failed to eval batch vals", K(ret)); + } else { + ObDatum *vals = expr.args_[4]->locate_batch_datums(ctx); + ObDatum *results = expr.locate_batch_datums(ctx); + ObBitVector &eval_flags = expr.get_evaluated_flags(ctx); + for (int64_t i = 0; OB_SUCC(ret) && i < batch_size; ++i) { + if (skip.at(i) || eval_flags.at(i)) { + continue; + } + results[i].set_datum(vals[i]); + eval_flags.set(i); + } + } + return ret; +} + int ObExprColumnConv::eval_enumset(const ObExpr &expr, ObEvalCtx &ctx, common::ObDatum *&datum) { int ret = OB_SUCCESS; diff --git a/src/sql/engine/expr/ob_expr_column_conv.h b/src/sql/engine/expr/ob_expr_column_conv.h index 25a1ce1e7..b359b10a3 100644 --- a/src/sql/engine/expr/ob_expr_column_conv.h +++ b/src/sql/engine/expr/ob_expr_column_conv.h @@ -134,6 +134,17 @@ public: static int column_convert(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &datum); + static int column_convert_fast(const ObExpr &expr, + ObEvalCtx &ctx, + ObDatum &datum); + static int column_convert_batch(const ObExpr &expr, + ObEvalCtx &ctx, + const ObBitVector &skip, + const int64_t batch_size); + static int column_convert_batch_fast(const ObExpr &expr, + ObEvalCtx &ctx, + const ObBitVector &skip, + const int64_t batch_size); virtual bool need_rt_ctx() const override { return ob_is_enum_or_set_type(result_type_.get_type()); } diff --git a/src/sql/engine/expr/ob_expr_eval_functions.cpp b/src/sql/engine/expr/ob_expr_eval_functions.cpp index 86221d0b6..944211f54 100644 --- a/src/sql/engine/expr/ob_expr_eval_functions.cpp +++ b/src/sql/engine/expr/ob_expr_eval_functions.cpp @@ -1240,7 +1240,7 @@ static ObExpr::EvalFunc g_expr_eval_functions[] = { NULL, // ObExprRegexpLike::eval_hs_regexp_like, /* 746 */ NULL, // ObExprRegexpReplace::eval_hs_regexp_replace, /* 747 */ NULL, // ObExprRegexpSubstr::eval_hs_regexp_substr, /* 748 */ - NULL, // ObExprColumnConv::column_convert_fast, /* 749 */ + ObExprColumnConv::column_convert_fast, /* 749 */ NULL, //ObExprArrayContains::eval_array_contains_int64_t, /* 750 */ NULL, //ObExprArrayContains::eval_array_contains_float, /* 751 */ NULL, //ObExprArrayContains::eval_array_contains_double, /* 752 */ @@ -1386,8 +1386,8 @@ static ObExpr::EvalBatchFunc g_expr_eval_batch_functions[] = { NULL, // ObExprMinus::minus_vec_vec_batch, /* 134 */ NULL, // ObExprMul::mul_vec_vec_batch, /* 135 */ NULL, // ObExprDiv::div_vec_batch, /* 136 */ - NULL, // ObExprColumnConv::column_convert_batch, /* 137 */ - NULL, // ObExprColumnConv::column_convert_batch_fast, /* 138 */ + ObExprColumnConv::column_convert_batch, /* 137 */ + ObExprColumnConv::column_convert_batch_fast, /* 138 */ NULL, // ObExprArrayContains::eval_array_contains_batch_int64_t, /* 139 */ NULL, // ObExprArrayContains::eval_array_contains_batch_float, /* 140 */ NULL, // ObExprArrayContains::eval_array_contains_batch_double, /* 141 */ diff --git a/src/sql/resolver/dml/ob_dml_resolver.cpp b/src/sql/resolver/dml/ob_dml_resolver.cpp index 01aaafd98..f95897999 100755 --- a/src/sql/resolver/dml/ob_dml_resolver.cpp +++ b/src/sql/resolver/dml/ob_dml_resolver.cpp @@ -8118,14 +8118,15 @@ int ObDMLResolver::add_additional_function_according_to_type(const ColumnItem *c OZ(build_padding_expr(session_info_, column, expr)); } if (OB_SUCC(ret)) { - bool need_column_convert = true; - if (OB_FAIL(check_insert_into_select_need_column_convert(column->get_expr(), expr, need_column_convert))) { + bool fast_calc = false; + if (OB_FAIL(check_insert_into_select_use_fast_column_convert(column->get_expr(), expr, + fast_calc))) { LOG_WARN("fail to check insert into select need column conv expr", K(ret)); - } else if (need_column_convert && OB_FAIL(ObRawExprUtils::build_column_conv_expr(*params_.expr_factory_, - *params_.allocator_, - *column->get_expr(), - expr, - session_info_))) { + } else if (OB_FAIL(ObRawExprUtils::build_column_conv_expr(*params_.expr_factory_, + *params_.allocator_, + *column->get_expr(), + expr, + session_info_))) { LOG_WARN("fail to build column conv expr", K(ret)); } else if (column->is_geo_ && T_FUN_COLUMN_CONV == expr->get_expr_type()) { // 1. set geo sub type to cast mode to column covert expr when update @@ -8142,6 +8143,9 @@ int ObDMLResolver::add_additional_function_according_to_type(const ColumnItem *c type_expr->set_value(obj); } } + if (fast_calc && OB_SUCC(ret)) { + expr->set_extra(expr->get_extra() | CM_FAST_COLUMN_CONV); + } } } } //end else @@ -13744,12 +13748,12 @@ int ObDMLResolver::check_index_table_has_partition_keys(const ObTableSchema *ind return ret; } -int ObDMLResolver::check_insert_into_select_need_column_convert(const ObColumnRefRawExpr *target_expr, - const ObRawExpr *source_expr, - bool &need_column_convert) +int ObDMLResolver::check_insert_into_select_use_fast_column_convert(const ObColumnRefRawExpr *target_expr, + const ObRawExpr *source_expr, + bool &fast_calc) { int ret = OB_SUCCESS; - need_column_convert = true; + fast_calc = false; ObDMLStmt *stmt = get_stmt(); if (stmt->is_insert_stmt() && GCONF._ob_enable_direct_load) { const ObInsertStmt *insert_stmt = reinterpret_cast(stmt); @@ -13810,8 +13814,16 @@ int ObDMLResolver::check_insert_into_select_need_column_convert(const ObColumnRe // a.target column is nullable // or // b.target column is not allowed have NULL and can not read NULL from source column - // we just skip allocate column_conv expr - need_column_convert = false; + // we just allocate fast column_conv expr + fast_calc = true; + } else if (source_base_table_schema->is_external_table()) { + ObArenaAllocator alloc; + ObExternalFileFormat format; + if (OB_FAIL(format.load_from_string(source_base_table_schema->get_external_file_format(), alloc))) { + LOG_WARN("load from string failed", K(ret)); + } else if (format.format_type_ == ObExternalFileFormat::CSV_FORMAT) { + fast_calc = true; + } } } } diff --git a/src/sql/resolver/dml/ob_dml_resolver.h b/src/sql/resolver/dml/ob_dml_resolver.h index 6e8f06689..64decb895 100644 --- a/src/sql/resolver/dml/ob_dml_resolver.h +++ b/src/sql/resolver/dml/ob_dml_resolver.h @@ -919,9 +919,9 @@ private: bool &has_part_key); //This funcion used to optimize Bypass Import scenario so far - int check_insert_into_select_need_column_convert(const ObColumnRefRawExpr *target_expr, - const ObRawExpr *source_expr, - bool &need_column_convert); + int check_insert_into_select_use_fast_column_convert(const ObColumnRefRawExpr *target_expr, + const ObRawExpr *source_expr, + bool &fast_calc); ///////////functions for sql hint///////////// int resolve_global_hint(const ParseNode &hint_node, From b72d6286bd3d01f03556a66fc79534aba744f990 Mon Sep 17 00:00:00 2001 From: zhjc1124 Date: Thu, 22 Aug 2024 03:29:58 +0000 Subject: [PATCH 165/249] Hide add_self_to_cgroup and set_group_id --- .../src/lib/mysqlclient/ob_isql_connection.h | 6 +- deps/oblib/src/lib/worker.cpp | 20 +++- deps/oblib/src/lib/worker.h | 89 ++++++++++++++++- deps/oblib/src/rpc/obmysql/ob_sql_nio.cpp | 8 +- deps/oblib/src/rpc/obmysql/ob_sql_nio.h | 1 - src/observer/mysql/obmp_query.cpp | 2 +- src/observer/mysql/obmp_stmt_execute.cpp | 2 +- src/observer/ob_inner_sql_connection.cpp | 1 + src/observer/ob_inner_sql_rpc_processor.cpp | 21 +--- src/observer/ob_inner_sql_rpc_processor.h | 12 --- src/observer/ob_inner_sql_rpc_proxy.h | 6 +- src/observer/ob_srv_deliver.h | 7 +- src/observer/omt/ob_multi_tenant.cpp | 11 --- src/observer/omt/ob_tenant.cpp | 39 ++++---- src/observer/omt/ob_tenant.h | 22 ++--- src/observer/omt/ob_th_worker.cpp | 22 ++--- src/observer/omt/ob_th_worker.h | 11 +-- src/share/rc/ob_tenant_base.cpp | 8 +- src/share/resource_manager/ob_cgroup_ctrl.cpp | 97 ++++++++++++------- src/share/resource_manager/ob_cgroup_ctrl.h | 5 +- .../ob_resource_mapping_rule_manager.cpp | 12 ++- .../ob_resource_mapping_rule_manager.h | 16 ++- .../resource_manager/ob_resource_plan_info.h | 48 +++++++++ .../scheduler/ob_tenant_dag_scheduler.cpp | 38 +------- src/share/scheduler/ob_tenant_dag_scheduler.h | 7 +- src/sql/engine/px/ob_px_worker.cpp | 2 +- src/storage/blocksstable/ob_block_manager.cpp | 34 +------ src/storage/blocksstable/ob_block_manager.h | 1 - unittest/observer/omt/test_cgroup_ctrl.cpp | 4 +- 29 files changed, 306 insertions(+), 246 deletions(-) diff --git a/deps/oblib/src/lib/mysqlclient/ob_isql_connection.h b/deps/oblib/src/lib/mysqlclient/ob_isql_connection.h index 7d27a40b4..449bc27aa 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_isql_connection.h +++ b/deps/oblib/src/lib/mysqlclient/ob_isql_connection.h @@ -256,8 +256,8 @@ public: } return ret; } - void set_group_id(const int64_t v) {consumer_group_id_ = v; } - int64_t get_group_id() const {return consumer_group_id_; } + void set_group_id(const uint64_t v) {consumer_group_id_ = v; } + uint64_t get_group_id() const {return consumer_group_id_; } void set_reverse_link_creadentials(bool flag) { has_reverse_link_credentials_ = flag; } bool get_reverse_link_creadentials() { return has_reverse_link_credentials_; } void set_usable(bool flag) { usable_ = flag; } @@ -277,7 +277,7 @@ protected: uint64_t dblink_id_; // for dblink, record dblink_id of a connection used by dblink DblinkDriverProto dblink_driver_proto_; //for dblink, record DblinkDriverProto of a connection used by dblink uint32_t sessid_; - int64_t consumer_group_id_; //for resource isolation + uint64_t consumer_group_id_; //for resource isolation bool has_reverse_link_credentials_; // for dblink, mark if this link has credentials set bool usable_; // usable_ = false: connection is unusable, should not execute query again. char *last_set_sql_mode_cstr_; // for mysql dblink to set sql mode diff --git a/deps/oblib/src/lib/worker.cpp b/deps/oblib/src/lib/worker.cpp index 02169ac95..9e9027819 100644 --- a/deps/oblib/src/lib/worker.cpp +++ b/deps/oblib/src/lib/worker.cpp @@ -44,18 +44,34 @@ int __attribute__((weak)) common_yield() return OB_SUCCESS; } -} +int __attribute__((weak)) SET_GROUP_ID(uint64_t group_id) +{ + int ret = OB_SUCCESS; + THIS_WORKER.set_group_id_(group_id); + return ret; } +int __attribute__((weak)) CONVERT_FUNCTION_TYPE_TO_GROUP_ID(const uint8_t function_type, uint64_t &group_id) +{ + int ret = OB_SUCCESS; + UNUSED(function_type); + group_id = GET_GROUP_ID(); + return ret; +} + +} // namespace lib +} // namespace oceanbase __thread Worker *Worker::self_; Worker::Worker() - : allocator_(nullptr), + : group_(nullptr), + allocator_(nullptr), st_current_priority_(0), session_(nullptr), cur_request_(nullptr), worker_level_(INT32_MAX), curr_request_level_(0), + is_th_worker_(false), group_id_(0), rpc_stat_srv_(nullptr), timeout_ts_(INT64_MAX), diff --git a/deps/oblib/src/lib/worker.h b/deps/oblib/src/lib/worker.h index 48fc3c395..600f7592f 100644 --- a/deps/oblib/src/lib/worker.h +++ b/deps/oblib/src/lib/worker.h @@ -82,10 +82,15 @@ public: OB_INLINE void set_curr_request_level(const int32_t level) { curr_request_level_ = level; } OB_INLINE int32_t get_curr_request_level() const { return curr_request_level_; } + OB_INLINE bool is_th_worker() const { return is_th_worker_; } + OB_INLINE void set_group_id_(const uint64_t group_id) { group_id_ = group_id;} - OB_INLINE void set_group_id(int32_t group_id) { group_id_ = group_id; } - OB_INLINE int32_t get_group_id() const { return group_id_; } - + OB_INLINE uint64_t get_group_id() const { return group_id_; } + OB_INLINE void set_group(void *group) { group_ = group; }; + OB_INLINE void *get_group() { return group_;}; + OB_INLINE bool is_group_worker() const { return OB_NOT_NULL(group_); } + OB_INLINE void set_func_type_(uint8_t func_type) { func_type_ = func_type; } + OB_INLINE uint8_t get_func_type() const { return func_type_; } OB_INLINE void set_rpc_stat_srv(void *rpc_stat_srv) { rpc_stat_srv_ = rpc_stat_srv; } OB_INLINE void *get_rpc_stat_srv() const { return rpc_stat_srv_; } @@ -125,12 +130,14 @@ public: static void set_module_type(const ObErrsimModuleType &module_type); static ObErrsimModuleType get_module_type(); #endif - +protected: + OB_INLINE void set_is_th_worker(bool is_th_worker) { is_th_worker_ = is_th_worker; } public: static __thread Worker *self_; public: common::ObDLinkNode worker_node_; + void *group_; protected: // 线程运行时内存从此分配器分配 // 初始tenant_id=500, 在处理request时，tenant_id被更新成request的租户id @@ -143,7 +150,9 @@ private: // whether worker is in blocking int32_t worker_level_; int32_t curr_request_level_; - int32_t group_id_; + bool is_th_worker_; + uint64_t group_id_; + uint8_t func_type_; void *rpc_stat_srv_; int64_t timeout_ts_; @@ -190,6 +199,76 @@ inline Worker &this_worker() #define THIS_WORKER oceanbase::lib::Worker::self() +#define GET_FUNC_TYPE() (THIS_WORKER.get_func_type()) +#define GET_GROUP_ID() (THIS_WORKER.get_group_id()) + +int SET_GROUP_ID(uint64_t group_id); + +int CONVERT_FUNCTION_TYPE_TO_GROUP_ID(const uint8_t function_type, uint64_t &group_id); + +class ConsumerGroupIdGuard +{ +public: + ConsumerGroupIdGuard(uint64_t group_id) + : thread_group_id_(GET_GROUP_ID()), group_changed_(false), ret_(OB_SUCCESS) + { + group_changed_ = group_id != thread_group_id_; + if (group_changed_) { + ret_ = SET_GROUP_ID(group_id); + } + } + ~ConsumerGroupIdGuard() + { + if (group_changed_) { + // SET_GROUP_ID(thread_group_id_); + } + } + int get_ret() + { + return ret_; + } + +private: + uint64_t thread_group_id_; + bool group_changed_; + int ret_; +}; +#define CONSUMER_GROUP_ID_GUARD(group_id) oceanbase::lib::ConsumerGroupIdGuard consumer_group_id_guard_(group_id) + +class ConsumerGroupFuncGuard +{ +public: + ConsumerGroupFuncGuard(uint8_t func_type) + : thread_group_id_(GET_GROUP_ID()), thread_func_type_(GET_FUNC_TYPE()), group_changed_(false), ret_(OB_SUCCESS) + { + // THIS_WORKER.set_func_type_(func_type); + uint64_t group_id = 0; + ret_ = CONVERT_FUNCTION_TYPE_TO_GROUP_ID(func_type, group_id); + if (OB_SUCCESS == ret_ && is_user_group(group_id) && group_id != thread_group_id_) { + group_changed_ = true; + ret_ = SET_GROUP_ID(group_id); + } + } + ~ConsumerGroupFuncGuard() + { + if (group_changed_) { + // SET_GROUP_ID(thread_group_id_); + // THIS_WORKER.set_func_type_(thread_func_type_); + } + } + int get_ret() + { + return ret_; + } + +private: + uint64_t thread_group_id_; + uint8_t thread_func_type_; + bool group_changed_; + int ret_; +}; +#define CONSUMER_GROUP_FUNC_GUARD(func_type) oceanbase::lib::ConsumerGroupFuncGuard consumer_group_func_guard_(func_type) + class DisableSchedInterGuard { public: diff --git a/deps/oblib/src/rpc/obmysql/ob_sql_nio.cpp b/deps/oblib/src/rpc/obmysql/ob_sql_nio.cpp index 226932a02..02aa12429 100644 --- a/deps/oblib/src/rpc/obmysql/ob_sql_nio.cpp +++ b/deps/oblib/src/rpc/obmysql/ob_sql_nio.cpp @@ -1233,17 +1233,11 @@ void ObSqlNio::destroy() } } -int __attribute__((weak)) sql_nio_add_cgroup(const uint64_t tenant_id) -{ - return 0; -} void ObSqlNio::run(int64_t idx) { if (NULL != impl_) { lib::set_thread_name("sql_nio", idx); - // if (tenant_id_ != common::OB_INVALID_ID) { - // obmysql::sql_nio_add_cgroup(tenant_id_); - // } + // SET_GROUP_ID(OBCG_SQL_NIO); while(!has_set_stop() && !(OB_NOT_NULL(&lib::Thread::current()) ? lib::Thread::current().has_set_stop() : false)) { impl_[idx].do_work(); } diff --git a/deps/oblib/src/rpc/obmysql/ob_sql_nio.h b/deps/oblib/src/rpc/obmysql/ob_sql_nio.h index df40686dd..869d8ddf4 100644 --- a/deps/oblib/src/rpc/obmysql/ob_sql_nio.h +++ b/deps/oblib/src/rpc/obmysql/ob_sql_nio.h @@ -71,7 +71,6 @@ private: uint64_t dispatch_idx_; uint64_t tenant_id_; }; -extern int sql_nio_add_cgroup(const uint64_t tenant_id); }; // end namespace obmysql }; // end namespace oceanbase diff --git a/src/observer/mysql/obmp_query.cpp b/src/observer/mysql/obmp_query.cpp index 5485cbf92..52d9ced03 100644 --- a/src/observer/mysql/obmp_query.cpp +++ b/src/observer/mysql/obmp_query.cpp @@ -368,7 +368,7 @@ int ObMPQuery::process() IGNORE_RETURN record_flt_trace(session); } - if (OB_UNLIKELY(NULL != GCTX.cgroup_ctrl_) && GCTX.cgroup_ctrl_->is_valid() && is_conn_valid()) { + if (is_conn_valid()) { int tmp_ret = OB_SUCCESS; // Call setup_user_resource_group no matter OB_SUCC or OB_FAIL // because we have to reset conn.group_id_ according to user_name. diff --git a/src/observer/mysql/obmp_stmt_execute.cpp b/src/observer/mysql/obmp_stmt_execute.cpp index 82951dd7c..b12a92859 100644 --- a/src/observer/mysql/obmp_stmt_execute.cpp +++ b/src/observer/mysql/obmp_stmt_execute.cpp @@ -1837,7 +1837,7 @@ int ObMPStmtExecute::process_execute_stmt(const ObMultiStmtItem &multi_stmt_item if (OB_FAIL(do_process_single(session, params_, has_more_result, force_sync_resp, async_resp_used))) { LOG_WARN("fail to do process", K(ret), K(ctx_.cur_sql_)); } - if (OB_UNLIKELY(NULL != GCTX.cgroup_ctrl_) && GCTX.cgroup_ctrl_->is_valid() && is_conn_valid()) { + if (is_conn_valid()) { int bak_ret = ret; ObSQLSessionInfo *sess = NULL; if (OB_FAIL(get_session(sess))) { diff --git a/src/observer/ob_inner_sql_connection.cpp b/src/observer/ob_inner_sql_connection.cpp index 6d868d5cb..5ddec0629 100644 --- a/src/observer/ob_inner_sql_connection.cpp +++ b/src/observer/ob_inner_sql_connection.cpp @@ -697,6 +697,7 @@ int ObInnerSQLConnection::do_query(sqlclient::ObIExecutor &executor, ObInnerSQLR } else { ObSQLSessionInfo &session = res.result_set().get_session(); session.set_expect_group_id(group_id_); + CONSUMER_GROUP_ID_GUARD(consumer_group_id_); if (OB_ISNULL(res.sql_ctx().schema_guard_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("schema guard is null"); diff --git a/src/observer/ob_inner_sql_rpc_processor.cpp b/src/observer/ob_inner_sql_rpc_processor.cpp index a5af6d6a4..917bc47ca 100644 --- a/src/observer/ob_inner_sql_rpc_processor.cpp +++ b/src/observer/ob_inner_sql_rpc_processor.cpp @@ -108,7 +108,7 @@ int ObInnerSqlRpcP::process_write( { int ret = OB_SUCCESS; int64_t affected_rows = -1; - ResourceGroupGuard guard(transmit_arg.get_consumer_group_id()); + CONSUMER_GROUP_ID_GUARD(transmit_arg.get_consumer_group_id()); if (OB_FAIL(conn->execute_write(transmit_arg.get_tenant_id(), write_sql.ptr(), affected_rows))) { LOG_WARN("execute write failed", K(ret), K(transmit_arg), K(write_sql)); } else { @@ -129,7 +129,7 @@ int ObInnerSqlRpcP::process_read( int ret = OB_SUCCESS; common::ObScanner &scanner = transmit_result.get_scanner(); scanner.set_found_rows(0); - + CONSUMER_GROUP_ID_GUARD(transmit_arg.get_consumer_group_id()); SMART_VAR(ObMySQLProxy::MySQLResult, res) { sqlclient::ObMySQLResult *sql_result = NULL; if (OB_FAIL(conn->execute_read(GCONF.cluster_id, transmit_arg.get_tenant_id(), read_sql.ptr(), res))) { @@ -486,23 +486,6 @@ int ObInnerSqlRpcP::set_session_param_to_conn( return ret; } -ResourceGroupGuard::ResourceGroupGuard(const int32_t group_id) - : group_change_(false), old_group_id_(0) -{ - if (is_user_group(group_id)) { - old_group_id_ = THIS_WORKER.get_group_id(); - THIS_WORKER.set_group_id(group_id); - group_change_ = true; - } -} - -ResourceGroupGuard::~ResourceGroupGuard() -{ - if (group_change_) { - THIS_WORKER.set_group_id(old_group_id_); - } -} - } } // namespace oceanbase diff --git a/src/observer/ob_inner_sql_rpc_processor.h b/src/observer/ob_inner_sql_rpc_processor.h index 23674307a..e3679a0f2 100644 --- a/src/observer/ob_inner_sql_rpc_processor.h +++ b/src/observer/ob_inner_sql_rpc_processor.h @@ -75,18 +75,6 @@ private: DISALLOW_COPY_AND_ASSIGN(ObInnerSqlRpcP); }; -class ResourceGroupGuard -{ - //todo qilu:revert after ddl_back_threads are split under tenants -public: - ResourceGroupGuard(const int32_t group_id); - ~ResourceGroupGuard(); -public: - bool group_change_; - int32_t old_group_id_; - -}; - } } #endif /* OBDEV_SRC_OBSERVER_INNER_SQL_RPC_PROCESSOR_H_ */ diff --git a/src/observer/ob_inner_sql_rpc_proxy.h b/src/observer/ob_inner_sql_rpc_proxy.h index 1985ffe28..660bc13b0 100644 --- a/src/observer/ob_inner_sql_rpc_proxy.h +++ b/src/observer/ob_inner_sql_rpc_proxy.h @@ -124,10 +124,10 @@ public: int64_t get_trx_timeout() const { return trx_timeout_; } - void set_consumer_group_id(const int64_t consumer_group_id) { + void set_consumer_group_id(const uint64_t consumer_group_id) { consumer_group_id_ = consumer_group_id; } - int64_t get_consumer_group_id() const { + uint64_t get_consumer_group_id() const { return consumer_group_id_; } inline int set_tz_info_wrap(const ObTimeZoneInfoWrap &other) { return tz_info_wrap_.deep_copy(other); } @@ -184,7 +184,7 @@ private: bool is_load_data_exec_; common::ObString nls_formats_[common::ObNLSFormatEnum::NLS_MAX]; bool use_external_session_; - int64_t consumer_group_id_; + uint64_t consumer_group_id_; }; class ObInnerSQLTransmitResult diff --git a/src/observer/ob_srv_deliver.h b/src/observer/ob_srv_deliver.h index 6c43734f2..316c61686 100644 --- a/src/observer/ob_srv_deliver.h +++ b/src/observer/ob_srv_deliver.h @@ -72,11 +72,8 @@ public: if (thread_name_ != nullptr) { lib::set_thread_name(thread_name_, get_thread_idx()); } - if (GCONF._enable_new_sql_nio && GCONF._enable_tenant_sql_net_thread && - tenant_id_ != common::OB_INVALID_ID && nullptr != GCTX.cgroup_ctrl_ && - OB_LIKELY(GCTX.cgroup_ctrl_->is_valid())) { - GCTX.cgroup_ctrl_->add_self_to_cgroup(tenant_id_, - share::OBCG_MYSQL_LOGIN); + if (GCONF._enable_new_sql_nio && GCONF._enable_tenant_sql_net_thread) { + lib::SET_GROUP_ID(share::OBCG_MYSQL_LOGIN); } queue_.loop(); } diff --git a/src/observer/omt/ob_multi_tenant.cpp b/src/observer/omt/ob_multi_tenant.cpp index 442faaa4e..a4da3d6a8 100644 --- a/src/observer/omt/ob_multi_tenant.cpp +++ b/src/observer/omt/ob_multi_tenant.cpp @@ -2469,17 +2469,6 @@ int ObMultiTenant::check_if_unit_id_exist(const uint64_t unit_id, bool &exist) return ret; } -int obmysql::sql_nio_add_cgroup(const uint64_t tenant_id) -{ - int ret = OB_SUCCESS; - if (GCONF._enable_new_sql_nio && GCONF._enable_tenant_sql_net_thread && - nullptr != GCTX.cgroup_ctrl_ && - OB_LIKELY(GCTX.cgroup_ctrl_->is_valid())) { - ret = GCTX.cgroup_ctrl_->add_self_to_cgroup(tenant_id, OBCG_SQL_NIO); - } - return ret; -} - int ObSrvNetworkFrame::reload_tenant_sql_thread_config(const uint64_t tenant_id) { int ret = OB_SUCCESS; diff --git a/src/observer/omt/ob_tenant.cpp b/src/observer/omt/ob_tenant.cpp index 7f2e74bf9..0ae8266d2 100644 --- a/src/observer/omt/ob_tenant.cpp +++ b/src/observer/omt/ob_tenant.cpp @@ -291,10 +291,7 @@ void ObPxPool::run1() CLEAR_INTERRUPTABLE(); ObCgroupCtrl *cgroup_ctrl = GCTX.cgroup_ctrl_; LOG_INFO("run px pool", K(group_id_), K(tenant_id_), K_(active_threads)); - if (nullptr != cgroup_ctrl && OB_LIKELY(cgroup_ctrl->is_valid())) { - cgroup_ctrl->add_self_to_cgroup(tenant_id_, group_id_); - LOG_INFO("add thread to group succ", K(tenant_id_), K(group_id_)); - } + SET_GROUP_ID(group_id_); if (!is_inited_) { queue_.set_limit(common::ObServerConfig::get_instance().tenant_task_queue_size); @@ -357,7 +354,7 @@ void ObPxPool::stop() } } -ObResourceGroup::ObResourceGroup(int32_t group_id, ObTenant* tenant, share::ObCgroupCtrl *cgroup_ctrl): +ObResourceGroup::ObResourceGroup(uint64_t group_id, ObTenant* tenant, share::ObCgroupCtrl *cgroup_ctrl): ObResourceGroupNode(group_id), workers_lock_(tenant->workers_lock_), inited_(false), @@ -531,9 +528,6 @@ void ObResourceGroup::check_worker_count(ObThWorker &w) if (OB_UNLIKELY(ATOMIC_LOAD(&shrink_)) && OB_LIKELY(ATOMIC_BCAS(&shrink_, true, false))) { w.stop(); - if (cgroup_ctrl_->is_valid() && OB_FAIL(cgroup_ctrl_->remove_self_from_cgroup(tenant_->id()))) { - LOG_WARN("remove thread from cgroup failed", K(ret), "tenant:", tenant_->id(), K_(group_id)); - } LOG_INFO("worker thread exit", K(tenant_->id()), K(workers_.get_size())); } } @@ -624,7 +618,7 @@ int ObResourceGroup::get_throttled_time(int64_t &throttled_time) return ret; } -int GroupMap::create_and_insert_group(int32_t group_id, ObTenant *tenant, ObCgroupCtrl *cgroup_ctrl, ObResourceGroup *&group) +int GroupMap::create_and_insert_group(uint64_t group_id, ObTenant *tenant, ObCgroupCtrl *cgroup_ctrl, ObResourceGroup *&group) { int ret = OB_SUCCESS; if (nullptr == tenant @@ -1240,11 +1234,12 @@ int ObTenant::get_new_request( w.set_large_query(false); w.set_curr_request_level(0); wk_level = w.get_worker_level(); + ObResourceGroup *group = static_cast(w.get_group()); if (wk_level < 0 || wk_level >= MAX_REQUEST_LEVEL) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("unexpected level", K(wk_level), K(id_)); } else if (wk_level >= MAX_REQUEST_LEVEL - 1) { - ret = w.get_group()->multi_level_queue_.pop_timeup(task, wk_level, timeout); + ret = group->multi_level_queue_.pop_timeup(task, wk_level, timeout); if ((ret == OB_SUCCESS && nullptr == task) || ret == OB_ENTRY_NOT_EXIST) { ret = OB_ENTRY_NOT_EXIST; usleep(10 * 1000L); @@ -1255,17 +1250,17 @@ int ObTenant::get_new_request( LOG_ERROR("pop queue err", "tenant_id", id_, K(ret)); } } else if (w.is_level_worker()) { - ret = w.get_group()->multi_level_queue_.pop(task, wk_level, timeout); + ret = group->multi_level_queue_.pop(task, wk_level, timeout); } else { for (int32_t level = MAX_REQUEST_LEVEL - 1; level >= GROUP_MULTI_LEVEL_THRESHOLD; level--) { - IGNORE_RETURN w.get_group()->multi_level_queue_.try_pop(task, level); + IGNORE_RETURN group->multi_level_queue_.try_pop(task, level); if (nullptr != task) { ret = OB_SUCCESS; break; } } if (nullptr == task) { - ret = w.get_group()->req_queue_.pop(task, timeout); + ret = group->req_queue_.pop(task, timeout); } } } else { @@ -1628,7 +1623,13 @@ void ObTenant::print_throttled_time() LOG_WARN_RET(tmp_ret, "get throttled time failed", K(tmp_ret), K(group)); } else { tenant_throttled_time += group_throttled_time; - databuff_printf(buf, len, pos, "group_id: %d, group: %s, throttled_time: %ld;", group->group_id_, set.name_of_id(group->group_id_), group_throttled_time); + databuff_printf(buf, + len, + pos, + "group_id: %ld, group: %s, throttled_time: %ld;", + group->group_id_, + set.name_of_id(group->group_id_), + group_throttled_time); } } } @@ -1785,9 +1786,6 @@ void ObTenant::check_worker_count(ObThWorker &w) && OB_UNLIKELY(ATOMIC_LOAD(&shrink_)) && OB_LIKELY(ATOMIC_BCAS(&shrink_, true, false))) { w.stop(); - if (cgroup_ctrl_.is_valid() && OB_FAIL(cgroup_ctrl_.remove_self_from_cgroup(id_))) { - LOG_WARN("remove thread from cgroup failed", K(ret), K_(id)); - } LOG_INFO("worker thread exit", K(id_), K(workers_.get_size())); } } @@ -1856,7 +1854,7 @@ void ObTenant::lq_end(ObThWorker &w) { int ret = OB_SUCCESS; if (w.is_lq_yield()) { - if (OB_FAIL(cgroup_ctrl_.add_self_to_cgroup(id_, w.get_group_id()))) { + if (OB_FAIL(cgroup_ctrl_.add_self_to_cgroup_(id_, w.get_group_id()))) { LOG_WARN("move thread from lq group failed", K(ret), K(id_)); } else { w.set_lq_yield(false); @@ -1867,7 +1865,8 @@ void ObTenant::lq_end(ObThWorker &w) void ObTenant::lq_wait(ObThWorker &w) { int64_t last_query_us = ObTimeUtility::current_time() - w.get_last_wakeup_ts(); - int64_t lq_group_worker_cnt = w.get_group()->workers_.get_size(); + ObResourceGroup *group = static_cast(w.get_group()); + int64_t lq_group_worker_cnt = group->workers_.get_size(); int64_t default_group_worker_cnt = workers_.get_size(); double large_query_percentage = GCONF.large_query_worker_percentage / 100.0; int64_t wait_us = static_cast(last_query_us * lq_group_worker_cnt / @@ -1890,7 +1889,7 @@ int ObTenant::lq_yield(ObThWorker &w) } } else if (w.is_lq_yield()) { // avoid duplicate change group - } else if (OB_FAIL(cgroup_ctrl_.add_self_to_cgroup(id_, OBCG_LQ))) { + } else if (OB_FAIL(cgroup_ctrl_.add_self_to_cgroup_(id_, share::OBCG_LQ))) { LOG_WARN("move thread to lq group failed", K(ret), K(id_)); } else { w.set_lq_yield(); diff --git a/src/observer/omt/ob_tenant.h b/src/observer/omt/ob_tenant.h index 56546e3ee..bee1bee4f 100644 --- a/src/observer/omt/ob_tenant.h +++ b/src/observer/omt/ob_tenant.h @@ -63,7 +63,6 @@ public: ObPxPool() : tenant_id_(common::OB_INVALID_ID), group_id_(0), - cgroup_ctrl_(nullptr), is_inited_(false), concurrency_(0), active_threads_(0) @@ -71,7 +70,6 @@ public: virtual void stop(); void set_tenant_id(uint64_t tenant_id) { tenant_id_ = tenant_id; } void set_group_id(uint64_t group_id) { group_id_ = group_id; } - void set_cgroup_ctrl(share::ObCgroupCtrl *cgroup_ctrl) { cgroup_ctrl_ = cgroup_ctrl; } int64_t get_pool_size() const { return get_thread_count(); } int submit(const RunFuncT &func); void set_px_thread_name(); @@ -90,7 +88,6 @@ private: private: uint64_t tenant_id_; uint64_t group_id_; - share::ObCgroupCtrl *cgroup_ctrl_; common::ObPriorityQueue2<0, 1> queue_; bool is_inited_; int64_t concurrency_; @@ -232,12 +229,12 @@ private: class ObResourceGroupNode : public common::SpHashNode { public: - ObResourceGroupNode(int32_t group_id): + ObResourceGroupNode(uint64_t group_id): common::SpHashNode(calc_hash(group_id)), group_id_(group_id) {} ~ObResourceGroupNode() {} - int64_t calc_hash(int32_t group_id) + int64_t calc_hash(uint64_t group_id) { return (common::murmurhash(&group_id, sizeof(group_id), 0)) | 1; } @@ -258,10 +255,10 @@ public: } return ret; } - int32_t get_group_id() const { return group_id_; } - void set_group_id(const int32_t &group_id) { group_id_ = group_id; } + uint64_t get_group_id() const { return group_id_; } + void set_group_id(const uint64_t &group_id) { group_id_ = group_id; } protected: - int32_t group_id_; + uint64_t group_id_; }; class ObResourceGroup : public ObResourceGroupNode // group container, storing thread pool and queue, each group_id corresponds to one{ @@ -273,7 +270,7 @@ public: using WList = common::ObDList; static constexpr int64_t PRESERVE_INACTIVE_WORKER_TIME = 10 * 1000L * 1000L; - ObResourceGroup(int32_t group_id, ObTenant* tenant, share::ObCgroupCtrl *cgroup_ctrl); + ObResourceGroup(uint64_t group_id, ObTenant* tenant, share::ObCgroupCtrl *cgroup_ctrl); ~ObResourceGroup() {} bool is_inited() const { return inited_; } @@ -282,6 +279,8 @@ public: int64_t min_worker_cnt() const; int64_t max_worker_cnt() const; ObTenant *get_tenant() { return tenant_; } + WList &get_workers() { return workers_; } + lib::ObMutex &get_workers_lock() { return workers_lock_; } share::ObCgroupCtrl *get_cgroup_ctrl() { return cgroup_ctrl_; } int init(); @@ -329,7 +328,7 @@ public: { } ~GroupMap() {} - int create_and_insert_group(int32_t group_id, ObTenant *tenant, share::ObCgroupCtrl *cgroup_ctrl, ObResourceGroup *&group); + int create_and_insert_group(uint64_t group_id, ObTenant *tenant, share::ObCgroupCtrl *cgroup_ctrl, ObResourceGroup *&group); void wait_group(); void destroy_group(); int64_t to_string(char *buf, const int64_t buf_len) const @@ -369,7 +368,7 @@ class ObTenant : public share::ObTenantBase friend class observer::ObAllVirtualDumpTenantInfo; friend class ObResourceGroup; friend int ::select_dump_tenant_info(lua_State*); - friend int create_worker(ObThWorker* &worker, ObTenant *tenant, int32_t group_id, + friend int create_worker(ObThWorker* &worker, ObTenant *tenant, uint64_t group_id, int32_t level, bool force, ObResourceGroup *group); friend int destroy_worker(ObThWorker *worker); using WListNode = common::ObDLinkNode; @@ -508,6 +507,7 @@ public: { return 0; } + GroupMap& get_group_map() { return group_map_;} // OB_INLINE bool has_normal_request() const { return req_queue_.size() != 0; } // OB_INLINE bool has_level_request() const { return OB_NOT_NULL(multi_level_queue_) && multi_level_queue_->get_total_size() != 0; } private: diff --git a/src/observer/omt/ob_th_worker.cpp b/src/observer/omt/ob_th_worker.cpp index b2ddaa468..f8c5c4e56 100644 --- a/src/observer/omt/ob_th_worker.cpp +++ b/src/observer/omt/ob_th_worker.cpp @@ -44,7 +44,7 @@ namespace oceanbase namespace omt { -int create_worker(ObThWorker* &worker, ObTenant *tenant, int32_t group_id, +int create_worker(ObThWorker* &worker, ObTenant *tenant, uint64_t group_id, int32_t level, bool force, ObResourceGroup *group) { int ret = OB_SUCCESS; @@ -65,7 +65,7 @@ int create_worker(ObThWorker* &worker, ObTenant *tenant, int32_t group_id, } else { worker->reset(); worker->set_tenant(tenant); - worker->set_group_id(group_id); + worker->set_group_id_(group_id); worker->set_worker_level(level); worker->set_group(group); if (OB_FAIL(worker->start())) { @@ -101,12 +101,12 @@ int destroy_worker(ObThWorker *worker) ObThWorker::ObThWorker() : procor_(ObServer::get_instance().get_net_frame().get_xlator(), ObServer::get_instance().get_self()), is_inited_(false), tenant_(nullptr), - group_(nullptr), run_cond_(), + run_cond_(), pause_flag_(false), large_query_(false), priority_limit_(RQ_LOW), is_lq_yield_(false), query_start_time_(0), last_check_time_(0), can_retry_(true), need_retry_(false), - has_add_to_cgroup_(false), last_wakeup_ts_(0), blocking_ts_(nullptr), + last_wakeup_ts_(0), blocking_ts_(nullptr), idle_us_(0) { } @@ -125,6 +125,7 @@ int ObThWorker::init() } else if (OB_FAIL(run_cond_.init(ObWaitEventIds::TH_WORKER_COND_WAIT))) { LOG_ERROR("init run cond fail, ", K(ret)); } else { + set_is_th_worker(true); is_inited_ = true; } @@ -305,7 +306,7 @@ void ObThWorker::set_th_worker_thread_name() char buf[32]; if (serving_tenant_id != tenant_->id()) { serving_tenant_id = tenant_->id(); - snprintf(buf, 32, "L%d_G%d", get_worker_level(), get_group_id()); + snprintf(buf, sizeof(buf), "L%d_G%ld", get_worker_level(), get_group_id()); lib::set_thread_name(buf); } } @@ -332,11 +333,6 @@ void ObThWorker::worker(int64_t &tenant_id, int64_t &req_recv_timestamp, int32_t if (OB_NOT_NULL(tenant_)) { tenant_id = tenant_->id(); } - if (OB_NOT_NULL(GCTX.cgroup_ctrl_) && OB_LIKELY(GCTX.cgroup_ctrl_->is_valid()) && !has_add_to_cgroup_) { - if (OB_SUCC(GCTX.cgroup_ctrl_->add_self_to_cgroup(tenant_->id(), get_group_id()))) { - has_add_to_cgroup_ = true; - } - } if (OB_NOT_NULL(pm)) { if (pm->get_used() != 0) { LOG_ERROR("page manager's used should be 0, unexpected!!!", KP(pm)); @@ -411,10 +407,11 @@ void ObThWorker::worker(int64_t &tenant_id, int64_t &req_recv_timestamp, int32_t if (this->get_worker_level() != 0) { // nesting workers not allowed to calling check_worker_count } else if (this->get_group() == nullptr) { - tenant_->check_worker_count(*this); tenant_->lq_end(*this); + tenant_->check_worker_count(*this); } else { - group_->check_worker_count(*this); + ObResourceGroup *group = static_cast(group_); + group->check_worker_count(*this); } } } @@ -431,6 +428,7 @@ void ObThWorker::run(int64_t idx) int64_t tenant_id = -1; int64_t req_recv_timestamp = -1; int32_t worker_level = -1; + SET_GROUP_ID(get_group_id()); this->worker(tenant_id, req_recv_timestamp, worker_level); } diff --git a/src/observer/omt/ob_th_worker.h b/src/observer/omt/ob_th_worker.h index 4dcf10f55..a7a066d60 100644 --- a/src/observer/omt/ob_th_worker.h +++ b/src/observer/omt/ob_th_worker.h @@ -74,8 +74,6 @@ public: set_run_wrapper(MTL_CTX()); } - OB_INLINE void set_group(ObResourceGroup *group) { group_ = group; } - void worker(int64_t &tenant_id, int64_t &req_recv_timestamp, int32_t &worker_level); void run(int64_t idx) override; @@ -87,8 +85,6 @@ public: OB_INLINE bool large_query() const { return large_query_; } OB_INLINE void set_large_query(bool v=true) { large_query_ = v; } - - OB_INLINE bool is_group_worker() const { return OB_NOT_NULL(group_); } OB_INLINE bool is_level_worker() const { return get_worker_level() > 0; } OB_INLINE void set_priority_limit(uint8_t limit) { priority_limit_ = limit; } OB_INLINE bool is_high_priority() const { return priority_limit_ == QQ_HIGH; } @@ -100,7 +96,6 @@ public: OB_INLINE int64_t get_query_start_time() const { return query_start_time_; } OB_INLINE int64_t get_query_enqueue_time() const { return query_enqueue_time_; } OB_INLINE ObTenant* get_tenant() { return tenant_; } - OB_INLINE ObResourceGroup* get_group() { return group_; } OB_INLINE bool is_lq_yield() const { return is_lq_yield_; } OB_INLINE void set_lq_yield(bool v=true) { is_lq_yield_ = v; } OB_INLINE int64_t get_last_wakeup_ts() { return last_wakeup_ts_; } @@ -118,7 +113,6 @@ private: bool is_inited_; ObTenant *tenant_; - ObResourceGroup *group_; common::ObThreadCond run_cond_; bool pause_flag_; @@ -135,8 +129,6 @@ private: // if upper scheduler support retry, need this request retry? bool need_retry_; - bool has_add_to_cgroup_; - int64_t last_wakeup_ts_; int64_t* blocking_ts_; int64_t idle_us_; @@ -156,7 +148,6 @@ inline void ObThWorker::reset() query_enqueue_time_ = 0; can_retry_ = true; need_retry_ = false; - has_add_to_cgroup_ = false; last_wakeup_ts_ = 0; } @@ -168,7 +159,7 @@ group_id: set worker's group_id level: set worker's level, in ObResourceGroup level = INT32_MAX, in ObTenant level = 0, group: set worker's group, in ObResourceGroup level = this, in ObTenant level = nullptr, */ -int create_worker(ObThWorker* &worker, ObTenant *tenant, int32_t group_id, +int create_worker(ObThWorker* &worker, ObTenant *tenant, uint64_t group_id, int32_t level = INT32_MAX, bool force = false, ObResourceGroup *group = nullptr); // defalut level=INT32_MAX, group=nullptr int destroy_worker(ObThWorker *worker); diff --git a/src/share/rc/ob_tenant_base.cpp b/src/share/rc/ob_tenant_base.cpp index f0a52858e..e4afaf885 100644 --- a/src/share/rc/ob_tenant_base.cpp +++ b/src/share/rc/ob_tenant_base.cpp @@ -298,10 +298,7 @@ int ObTenantBase::pre_run() { int ret = OB_SUCCESS; ObTenantEnv::set_tenant(this); - ObCgroupCtrl *cgroup_ctrl = get_cgroup(); - if (cgroup_ctrl != nullptr && cgroup_ctrl->is_valid()) { - ret = cgroup_ctrl->add_self_to_cgroup(id_); - } + ret = lib::SET_GROUP_ID(OBCG_DEFAULT); { ThreadListNode *node = lib::Thread::current().get_thread_list_node(); lib::ObMutexGuard guard(thread_list_lock_); @@ -320,9 +317,6 @@ int ObTenantBase::end_run() int ret = OB_SUCCESS; ObTenantEnv::set_tenant(nullptr); ObCgroupCtrl *cgroup_ctrl = get_cgroup(); - if (cgroup_ctrl != nullptr && cgroup_ctrl->is_valid()) { - ret = cgroup_ctrl->remove_self_from_cgroup(id_); - } { ThreadListNode *node = lib::Thread::current().get_thread_list_node(); lib::ObMutexGuard guard(thread_list_lock_); diff --git a/src/share/resource_manager/ob_cgroup_ctrl.cpp b/src/share/resource_manager/ob_cgroup_ctrl.cpp index 3b9bbf184..9582d2355 100644 --- a/src/share/resource_manager/ob_cgroup_ctrl.cpp +++ b/src/share/resource_manager/ob_cgroup_ctrl.cpp @@ -22,20 +22,49 @@ #include "share/resource_manager/ob_resource_plan_info.h" #include "share/resource_manager/ob_resource_manager.h" #include "share/resource_manager/ob_cgroup_ctrl.h" +#include "observer/omt/ob_tenant.h" +#include "observer/omt/ob_multi_tenant.h" #include #include using namespace oceanbase::common; using namespace oceanbase::share; +using namespace oceanbase::omt; namespace oceanbase { + +namespace lib +{ + +int SET_GROUP_ID(uint64_t group_id) +{ + int ret = OB_SUCCESS; + + // to do switch group + + THIS_WORKER.set_group_id_(group_id); + int tmp_ret = OB_SUCCESS; + if (OB_NOT_NULL(GCTX.cgroup_ctrl_) + && OB_TMP_FAIL(GCTX.cgroup_ctrl_->add_self_to_cgroup_(MTL_ID(), group_id))) { + LOG_WARN("add self to cgroup fail", K(ret), K(MTL_ID()), K(group_id)); + } + return ret; +} + +int CONVERT_FUNCTION_TYPE_TO_GROUP_ID(const uint8_t function_type, uint64_t &group_id) +{ + return G_RES_MGR.get_mapping_rule_mgr().get_group_id_by_function_type(MTL_ID(), function_type, group_id); +} + +} // namespace lib + namespace share { -ObCgSet ObCgSet::instance_; -} -} +ObCgSet ObCgSet::instance_; +} // namespace share +} // namespace oceanbase // cgroup config name static const char *CPU_SHARES_FILE = "cpu.shares"; @@ -45,6 +74,7 @@ static const char *CPU_CFS_QUOTA_FILE = "cpu.cfs_quota_us"; static const char *CPU_CFS_PERIOD_FILE = "cpu.cfs_period_us"; static const char *CPUACCT_USAGE_FILE = "cpuacct.usage"; static const char *CPU_STAT_FILE = "cpu.stat"; +static const char *CGROUP_PROCS_FILE = "cgroup.procs"; //集成IO参数 int OBGroupIOInfo::init(int64_t min_percent, int64_t max_percent, int64_t weight_percent) @@ -100,20 +130,15 @@ int ObCgroupCtrl::init() } else { LOG_ERROR("init cgroup dir failed", K(ret), K(root_cgroup_)); } - } else if (OB_FAIL(init_dir_(other_cgroup_))) { - LOG_WARN("init other cgroup dir failed", K(ret), K_(other_cgroup)); } else { - char procs_path[PATH_BUFSIZE]; char pid_value[VALUE_BUFSIZE]; - snprintf(procs_path, PATH_BUFSIZE, "%s/cgroup.procs", other_cgroup_); snprintf(pid_value, VALUE_BUFSIZE, "%d", getpid()); - if(OB_FAIL(write_string_to_file_(procs_path, pid_value))) { - LOG_ERROR("add tid to cgroup failed", K(ret), K(procs_path), K(pid_value)); + if (OB_FAIL(set_cgroup_config_(other_cgroup_, CGROUP_PROCS_FILE, pid_value))) { + LOG_ERROR("add tid to cgroup failed", K(ret), K(other_cgroup_), K(pid_value)); } else { valid_ = true; } } - return ret; } @@ -363,19 +388,37 @@ int ObCgroupCtrl::get_group_path( return ret; } -int ObCgroupCtrl::add_self_to_cgroup(const uint64_t tenant_id, uint64_t group_id, const char *base_path) +int ObCgroupCtrl::add_self_to_cgroup_(const uint64_t tenant_id, const uint64_t group_id) { int ret = OB_SUCCESS; - char group_path[PATH_BUFSIZE]; - char tid_value[VALUE_BUFSIZE + 1]; + if (is_valid()) { + ResourceGroupType group_type = ResourceGroupType::INVALID_GROUP; + if (is_user_group(group_id) && + OB_FAIL(G_RES_MGR.get_mapping_rule_mgr().get_group_type_by_id(tenant_id, group_id, group_type))) { + if (OB_HASH_NOT_EXIST == ret) { + ret = OB_SUCCESS; + } else { + LOG_WARN("get group type by id failed", K(ret), K(tenant_id), K(group_id), K(group_type)); + } + } - snprintf(tid_value, VALUE_BUFSIZE, "%ld", gettid()); - if (OB_FAIL(get_group_path(group_path, PATH_BUFSIZE, tenant_id, group_id, base_path))) { - LOG_WARN("fail get group path", K(tenant_id), K(ret)); - } else if (OB_FAIL(set_cgroup_config_(group_path, TASKS_FILE, tid_value))) { - LOG_WARN("add tid to cgroup failed", K(ret), K(group_path), K(tid_value), K(tenant_id)); - } else { - LOG_INFO("add tid to cgroup success", K(group_path), K(tid_value), K(tenant_id), K(group_id)); + const char *base_path = + (GCONF.enable_global_background_resource_isolation && ResourceGroupType::FUNCTION_GROUP == group_type) + ? BACKGROUND_CGROUP + : ""; + char group_path[PATH_BUFSIZE]; + char tid_value[VALUE_BUFSIZE + 1]; + + snprintf(tid_value, VALUE_BUFSIZE, "%ld", gettid()); + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(get_group_path(group_path, PATH_BUFSIZE, tenant_id, group_id, base_path))) { + LOG_WARN("fail get group path", K(tenant_id), K(ret)); + } else if (OB_FAIL(set_cgroup_config_(group_path, TASKS_FILE, tid_value))) { + LOG_WARN("add tid to cgroup failed", K(ret), K(group_path), K(tid_value), K(tenant_id)); + } else { + LOG_INFO("add tid to cgroup success", K(group_path), K(tid_value), K(tenant_id), K(group_id)); + } } return ret; } @@ -394,20 +437,6 @@ int ObCgroupCtrl::get_group_info_by_group_id(const uint64_t tenant_id, return ret; } -int ObCgroupCtrl::remove_self_from_cgroup(const uint64_t tenant_id) -{ - int ret = OB_SUCCESS; - char tid_value[VALUE_BUFSIZE]; - // 把该tid加入other_cgroup目录的tasks文件中就会从其它tasks中删除 - snprintf(tid_value, VALUE_BUFSIZE, "%ld", gettid()); - if (OB_FAIL(set_cgroup_config_(other_cgroup_, TASKS_FILE, tid_value))) { - LOG_WARN("remove tid to cgroup failed", K(ret), K(other_cgroup_), K(tid_value), K(tenant_id)); - } else { - LOG_INFO("remove tid to cgroup success", K(other_cgroup_), K(tid_value), K(tenant_id)); - } - return ret; -} - int ObCgroupCtrl::get_cgroup_config_(const char *group_path, const char *config_name, char *config_value) { int ret = OB_SUCCESS; diff --git a/src/share/resource_manager/ob_cgroup_ctrl.h b/src/share/resource_manager/ob_cgroup_ctrl.h index 1be9b9244..4c86ee910 100644 --- a/src/share/resource_manager/ob_cgroup_ctrl.h +++ b/src/share/resource_manager/ob_cgroup_ctrl.h @@ -152,10 +152,7 @@ public: int remove_both_cgroup(const uint64_t tenant_id, const uint64_t group_id = OB_INVALID_GROUP_ID, const char *base_path = ""); static int remove_dir_(const char *curr_dir); - int add_self_to_cgroup(const uint64_t tenant_id, uint64_t group_id = OBCG_DEFAULT, const char *base_path = ""); - - // 从指定租户cgroup组移除指定tid - int remove_self_from_cgroup(const uint64_t tenant_id); + int add_self_to_cgroup_(const uint64_t tenant_id, const uint64_t group_id = OBCG_DEFAULT); static int get_cgroup_config_(const char *group_path, const char *config_name, char *config_value); static int set_cgroup_config_(const char *group_path, const char *config_name, char *config_value); diff --git a/src/share/resource_manager/ob_resource_mapping_rule_manager.cpp b/src/share/resource_manager/ob_resource_mapping_rule_manager.cpp index d81f7e9ee..a7d376756 100644 --- a/src/share/resource_manager/ob_resource_mapping_rule_manager.cpp +++ b/src/share/resource_manager/ob_resource_mapping_rule_manager.cpp @@ -27,7 +27,7 @@ int ObResourceMappingRuleManager::init() int rule_bucket_size = 4096; int group_bucket_size = 512; if (user_rule_map_.created() || group_id_name_map_.created() || - function_rule_map_.created() || group_name_id_map_.created()) { + function_rule_map_.created() || group_name_id_map_.created() || group_id_type_map_.created()) { ret = OB_INIT_TWICE; LOG_WARN("mapping rule manager should not init multiple times", K(ret)); } else if (OB_FAIL(user_rule_map_.create(rule_bucket_size, "UsrRuleMap", "UsrRuleMapNode"))) { @@ -40,7 +40,7 @@ int ObResourceMappingRuleManager::init() LOG_WARN("fail create function rule map", K(ret)); } else if (OB_FAIL(group_name_id_map_.create(group_bucket_size, "GrpNameIdMap", "GrpNameIdNode"))) { LOG_WARN("fail create name id map", K(ret)); - } + } else if (OB_FAIL(group_id_type_map_.create(group_bucket_size, "GrpIdTypeMap", "GrpIdTypeNode"))) LOG_INFO("resource mapping rule manager init ok"); return ret; } @@ -155,6 +155,10 @@ int ObResourceMappingRuleManager::refresh_resource_user_mapping_rule( 1 /* overwrite on dup key */))) { LOG_WARN("fail set user mapping rule to rule_map", K(rule), K(ret)); } + if (OB_SUCC(ret) && OB_FAIL(group_id_type_map_.set_refactored( + share::ObTenantGroupIdKey(rule.tenant_id_, group_id), ResourceGroupType::USER_GROUP))) { + LOG_WARN("group_id_type_map_ set_refactored failed", K(ret), K(group_id)); + } } LOG_INFO("refresh resource user mapping rule", K(tenant_id), K(plan), K(user_rules)); } @@ -192,6 +196,10 @@ int ObResourceMappingRuleManager::refresh_resource_function_mapping_rule( 1 /* overwrite on dup key */))) { LOG_WARN("fail set user mapping rule to rule_map", K(rule), K(ret)); } + if (OB_SUCC(ret) && OB_FAIL(group_id_type_map_.set_refactored( + share::ObTenantGroupIdKey(rule.tenant_id_, group_id), ResourceGroupType::FUNCTION_GROUP))) { + LOG_WARN("group_id_type_map_ set_refactored failed", K(ret), K(group_id)); + } } LOG_INFO("refresh_resource_function_mapping_rule", K(tenant_id), K(plan), K(rules)); } diff --git a/src/share/resource_manager/ob_resource_mapping_rule_manager.h b/src/share/resource_manager/ob_resource_mapping_rule_manager.h index 749a001b4..f547943fb 100644 --- a/src/share/resource_manager/ob_resource_mapping_rule_manager.h +++ b/src/share/resource_manager/ob_resource_mapping_rule_manager.h @@ -29,6 +29,15 @@ class ObString; } namespace share { + +enum class ResourceGroupType { + INVALID_GROUP, + USER_GROUP, + FUNCTION_GROUP, + SQL_GROUP, + END_GROUP +}; + class ObResourceManagerProxy; class ObResourceMappingRuleManager { @@ -126,7 +135,11 @@ public: int ret = function_rule_map_.set_refactored(share::ObTenantFunctionKey(tenant_id, func), 0, 1/*overwrite*/); return ret; } - + inline int get_group_type_by_id(const uint64_t tenant_id, uint64_t group_id, ResourceGroupType &group_type) + { + int ret = group_id_type_map_.get_refactored(share::ObTenantGroupIdKey(tenant_id, group_id), group_type); + return ret; + } private: int refresh_resource_function_mapping_rule( ObResourceManagerProxy &proxy, @@ -146,6 +159,7 @@ private: common::hash::ObHashMap group_id_name_map_; // 将 group_name 映射到 group_id, 用于快速根据group_name找到id(主要是用于io控制) common::hash::ObHashMap group_name_id_map_; + common::hash::ObHashMap group_id_type_map_; DISALLOW_COPY_AND_ASSIGN(ObResourceMappingRuleManager); }; } diff --git a/src/share/resource_manager/ob_resource_plan_info.h b/src/share/resource_manager/ob_resource_plan_info.h index 083b7c30a..70bb3c0bc 100644 --- a/src/share/resource_manager/ob_resource_plan_info.h +++ b/src/share/resource_manager/ob_resource_plan_info.h @@ -157,6 +157,54 @@ public: ObResMgrVarcharValue func_name_; }; +// ObTenantGroupIdKey +class ObTenantGroupIdKey { +public: + ObTenantGroupIdKey() : tenant_id_(OB_INVALID_TENANT_ID), group_id_(OB_INVALID_ID) + {} + ObTenantGroupIdKey(const uint64_t tenant_id, const uint64_t group_id) : + tenant_id_(tenant_id), group_id_(group_id) + {} + int assign(const ObTenantGroupIdKey &other) + { + tenant_id_ = other.tenant_id_; + group_id_ = other.group_id_; + return common::OB_SUCCESS; + } + uint64_t hash() const + { + uint64_t hash_val = 0; + hash_val = common::murmurhash(&tenant_id_, sizeof(tenant_id_), hash_val); + hash_val = common::murmurhash(&group_id_, sizeof(group_id_), hash_val); + return hash_val; + } + int hash(uint64_t &hash_val) const + { + hash_val = hash(); + return common::OB_SUCCESS; + } + int compare(const ObTenantGroupIdKey& r) const + { + int cmp = 0; + if (tenant_id_ < r.tenant_id_) { + cmp = -1; + } else if (tenant_id_ == r.tenant_id_) { + cmp = group_id_ < r.group_id_ ? -1 : (group_id_ > r.group_id_ ? 1 : 0); + } else { + cmp = 1; + } + return cmp; + } + bool operator== (const ObTenantGroupIdKey &other) const { return 0 == compare(other); } + bool operator!=(const ObTenantGroupIdKey &other) const { return !operator==(other); } + bool operator<(const ObTenantGroupIdKey &other) const { return -1 == compare(other); } + TO_STRING_KV(K_(tenant_id), K_(group_id)); + +public: + uint64_t tenant_id_; + uint64_t group_id_; +}; + class ObPlanDirective { public: diff --git a/src/share/scheduler/ob_tenant_dag_scheduler.cpp b/src/share/scheduler/ob_tenant_dag_scheduler.cpp index 82d909b9a..225a2f6df 100644 --- a/src/share/scheduler/ob_tenant_dag_scheduler.cpp +++ b/src/share/scheduler/ob_tenant_dag_scheduler.cpp @@ -1579,7 +1579,6 @@ ObTenantDagWorker::ObTenantDagWorker() check_period_(0), last_check_time_(0), function_type_(0), - group_id_(OB_INVALID_GROUP_ID), tg_id_(-1), hold_by_compaction_dag_(false), is_inited_(false) @@ -1650,7 +1649,6 @@ void ObTenantDagWorker::reset() check_period_ = 0; last_check_time_ = 0; function_type_ = 0; - group_id_ = OB_INVALID_GROUP_ID; self_ = NULL; is_inited_ = false; TG_DESTROY(tg_id_); @@ -1668,36 +1666,6 @@ void ObTenantDagWorker::resume() notify(DWS_RUNNABLE); } -int ObTenantDagWorker::set_dag_resource(const uint64_t group_id) -{ - int ret = OB_SUCCESS; - uint64_t consumer_group_id = USER_RESOURCE_OTHER_GROUP_ID; - if (is_user_group(group_id)) { - //user level - consumer_group_id = group_id; - } else if (OB_FAIL(G_RES_MGR.get_mapping_rule_mgr().get_group_id_by_function_type(MTL_ID(), function_type_, consumer_group_id))) { - //function level - LOG_WARN("fail to get group id by function", K(ret), K(MTL_ID()), K(function_type_), K(consumer_group_id)); - } - - if (OB_SUCC(ret) && consumer_group_id != group_id_) { - // for CPU isolation, depend on cgroup - if (OB_NOT_NULL(GCTX.cgroup_ctrl_) && GCTX.cgroup_ctrl_->is_valid() && - OB_FAIL(GCTX.cgroup_ctrl_->add_self_to_cgroup( - MTL_ID(), - consumer_group_id, - GCONF.enable_global_background_resource_isolation ? BACKGROUND_CGROUP - : ""))) { - LOG_WARN("bind back thread to group failed", K(ret), K(GETTID()), K(MTL_ID()), K(group_id)); - } else { - // for IOPS isolation, only depend on consumer_group_id - ATOMIC_SET(&group_id_, consumer_group_id); - THIS_WORKER.set_group_id(static_cast(consumer_group_id)); - } - } - return ret; -} - bool ObTenantDagWorker::need_wake_up() const { return (ObTimeUtility::fast_current_time() - last_check_time_) > check_period_ * 10; @@ -1758,6 +1726,7 @@ void ObTenantDagWorker::run1() } if (OB_SUCC(ret)) { + CONSUMER_GROUP_ID_GUARD(dag->get_consumer_group_id()); ObDagId dag_id = dag->get_dag_id(); if (task_->get_sub_task_id() > 0) { dag_id.set_sub_id(task_->get_sub_task_id()); @@ -1778,9 +1747,8 @@ void ObTenantDagWorker::run1() } else { THIS_WORKER.set_log_reduction_mode(LogReductionMode::NONE); } - if (OB_FAIL(set_dag_resource(dag->get_consumer_group_id()))) { - LOG_WARN("isolate dag CPU and IOPS failed", K(ret)); - } else if (OB_FAIL(task_->do_work())) { + CONSUMER_GROUP_FUNC_GUARD(function_type_); + if (OB_FAIL(task_->do_work())) { if (!dag->ignore_warning()) { COMMON_LOG(WARN, "failed to do work", K(ret), K(*task_), K(compat_mode)); } diff --git a/src/share/scheduler/ob_tenant_dag_scheduler.h b/src/share/scheduler/ob_tenant_dag_scheduler.h index 47eab7598..4b0bf4984 100644 --- a/src/share/scheduler/ob_tenant_dag_scheduler.h +++ b/src/share/scheduler/ob_tenant_dag_scheduler.h @@ -728,8 +728,7 @@ public: void run1() override; int yield(); void set_task(ObITask *task); - void set_function_type(const int64_t function_type) { function_type_ = function_type; } - int set_dag_resource(const uint64_t group_id); + void set_function_type(const uint8_t function_type) { function_type_ = function_type; } bool need_wake_up() const; ObITask *get_task() const { return task_; } DagWorkerStatus get_status() { return status_; } @@ -738,7 +737,6 @@ public: static bool is_reserve_mode() { return is_reserve_mode_; } static compaction::ObCompactionMemoryContext* get_mem_ctx() { return mem_ctx_; } static void set_mem_ctx(compaction::ObCompactionMemoryContext *mem_ctx) { if (nullptr == mem_ctx_) { mem_ctx_ = mem_ctx; } } - uint64_t get_group_id() { return group_id_; } bool get_force_cancel_flag(); bool hold_by_compaction_dag() const { return hold_by_compaction_dag_; } private: @@ -755,8 +753,7 @@ private: DagWorkerStatus status_; int64_t check_period_; int64_t last_check_time_; - int64_t function_type_; - uint64_t group_id_; + uint8_t function_type_; int tg_id_; bool hold_by_compaction_dag_; bool is_inited_; diff --git a/src/sql/engine/px/ob_px_worker.cpp b/src/sql/engine/px/ob_px_worker.cpp index 35a74e57e..00e6050ea 100644 --- a/src/sql/engine/px/ob_px_worker.cpp +++ b/src/sql/engine/px/ob_px_worker.cpp @@ -188,7 +188,7 @@ void PxWorkerFunctor::operator ()(bool need_exec) ObThreadLogLevelUtils::init(env_arg_.get_log_level()); } } - THIS_WORKER.set_group_id(env_arg_.get_group_id()); + SET_GROUP_ID(env_arg_.get_group_id()); // When deserialize expr, sql mode will affect basic function of expr. CompatModeGuard mode_guard(env_arg_.is_oracle_mode() ? Worker::CompatMode::ORACLE : Worker::CompatMode::MYSQL); MTL_SWITCH(sqc_handler->get_tenant_id()) { diff --git a/src/storage/blocksstable/ob_block_manager.cpp b/src/storage/blocksstable/ob_block_manager.cpp index fc46108be..1b71fb77c 100644 --- a/src/storage/blocksstable/ob_block_manager.cpp +++ b/src/storage/blocksstable/ob_block_manager.cpp @@ -1112,9 +1112,9 @@ int ObBlockManager::mark_macro_blocks( const uint64_t tenant_id = mtl_tenant_ids.at(i); MacroBlockId macro_id; MTL_SWITCH(tenant_id) { - if (OB_FAIL(set_group_id(tenant_id))) { - LOG_WARN("isolate CPU and IOPS failed", K(ret)); - } else if (OB_FAIL(mark_tenant_blocks(mark_info, macro_id_set, tmp_status))) { + CONSUMER_GROUP_FUNC_GUARD(ObFunctionType::PRIO_OTHER_BACKGROUND); + if (OB_FAIL(mark_tenant_blocks(mark_info, macro_id_set, + tmp_status))) { LOG_WARN("fail to mark tenant blocks", K(ret), K(tenant_id)); } else if (OB_FALSE_IT(MTL(ObSharedMacroBlockMgr*)->get_cur_shared_block(macro_id))) { } else if (OB_FAIL(mark_held_shared_block(macro_id, mark_info, macro_id_set, tmp_status))) { @@ -1598,34 +1598,6 @@ int ObBlockManager::update_mark_info(const MacroBlockId ¯o_id, return ret; } -int ObBlockManager::set_group_id(const uint64_t tenant_id) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(tenant_id <= 0)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid tenant id", K(ret), K(tenant_id)); - } else { - uint64_t consumer_group_id = 0; - if (OB_FAIL(G_RES_MGR.get_mapping_rule_mgr().get_group_id_by_function_type(tenant_id, ObFunctionType::PRIO_OTHER_BACKGROUND, consumer_group_id))) { - //function level - LOG_WARN("fail to get group id by function", K(ret), K(tenant_id), K(consumer_group_id)); - } else if (consumer_group_id != group_id_) { - // for CPU isolation, depend on cgroup - if (OB_NOT_NULL(GCTX.cgroup_ctrl_) && GCTX.cgroup_ctrl_->is_valid() && - OB_FAIL(GCTX.cgroup_ctrl_->add_self_to_cgroup(tenant_id, - consumer_group_id, - GCONF.enable_global_background_resource_isolation ? BACKGROUND_CGROUP : ""))) { - LOG_WARN("bind back thread to group failed", K(ret), K(GETTID()), K(tenant_id), K(consumer_group_id)); - } - } - if (OB_SUCC(ret)) { - group_id_ = consumer_group_id; - THIS_WORKER.set_group_id(static_cast(consumer_group_id)); - } - } - return ret; -} - int ObBlockManager::BlockMapIterator::get_next_block(common::ObIOFd &block_id) { int ret = OB_SUCCESS; diff --git a/src/storage/blocksstable/ob_block_manager.h b/src/storage/blocksstable/ob_block_manager.h index 6fa81efb4..568175639 100644 --- a/src/storage/blocksstable/ob_block_manager.h +++ b/src/storage/blocksstable/ob_block_manager.h @@ -405,7 +405,6 @@ private: MacroBlkIdMap &mark_info, common::hash::ObHashSet ¯o_id_set, ObMacroBlockMarkerStatus &tmp_status); - int set_group_id(const uint64_t tenant_id); int do_sweep(MacroBlkIdMap &mark_info); int sweep_one_block(const MacroBlockId& macro_id); diff --git a/unittest/observer/omt/test_cgroup_ctrl.cpp b/unittest/observer/omt/test_cgroup_ctrl.cpp index 29ab914d3..1121d1b0e 100644 --- a/unittest/observer/omt/test_cgroup_ctrl.cpp +++ b/unittest/observer/omt/test_cgroup_ctrl.cpp @@ -24,9 +24,9 @@ using namespace oceanbase::observer; void *thread_func(void *args) { - ASSERT_EQ(OB_SUCCESS, cg_ctrl.add_self_to_cgroup(1001)); + ASSERT_EQ(OB_SUCCESS, cg_ctrl.add_self_to_cgroup_(1001)); sleep(3); - ASSERT_EQ(OB_SUCCESS, cg_ctrl.remove_self_from_cgroup(1001)); + ASSERT_EQ(OB_SUCCESS, cg_ctrl.add_self_to_cgroup_(1001)); return nullptr; } From 78a2108dc655eeea2c8b2fa0344a034e1b873839 Mon Sep 17 00:00:00 2001 From: hy-guo Date: Thu, 22 Aug 2024 04:18:02 +0000 Subject: [PATCH 166/249] [CP] fix select * in exists --- src/sql/parser/sql_parser_mysql_mode.y | 4 --- src/sql/resolver/dml/ob_dml_resolver.cpp | 1 + src/sql/resolver/dml/ob_select_resolver.cpp | 30 +++++++++++++++++-- .../resolver/dml/ob_view_table_resolver.cpp | 1 + src/sql/resolver/expr/ob_expr_info_flag.h | 4 ++- .../expr/ob_raw_expr_info_extractor.cpp | 6 ++++ .../expr/ob_raw_expr_resolver_impl.cpp | 7 ++++- 7 files changed, 44 insertions(+), 9 deletions(-) diff --git a/src/sql/parser/sql_parser_mysql_mode.y b/src/sql/parser/sql_parser_mysql_mode.y index 1b6a44557..058382a46 100644 --- a/src/sql/parser/sql_parser_mysql_mode.y +++ b/src/sql/parser/sql_parser_mysql_mode.y @@ -1673,10 +1673,6 @@ simple_expr collation %prec NEG } | EXISTS select_with_parens { - /* mysql 允许 select * from dual 出现在 exists 中, 此处更改 from dual 的 select list 为常量 1 */ - if (NULL == $2->children_[PARSE_SELECT_FROM]) { - $2->value_ = 2; - } malloc_non_terminal_node($$, result->malloc_pool_, T_OP_EXISTS, 1, $2); } | MATCH '(' column_list ')' AGAINST '(' search_expr opt_mode_flag ')' diff --git a/src/sql/resolver/dml/ob_dml_resolver.cpp b/src/sql/resolver/dml/ob_dml_resolver.cpp index f95897999..c523a178d 100755 --- a/src/sql/resolver/dml/ob_dml_resolver.cpp +++ b/src/sql/resolver/dml/ob_dml_resolver.cpp @@ -7455,6 +7455,7 @@ int ObDMLResolver::resolve_subquery_info(const ObIArray &subquer subquery_resolver.set_current_level(current_level_ + 1); subquery_resolver.set_current_view_level(current_view_level_); subquery_resolver.set_parent_namespace_resolver(this); + subquery_resolver.set_in_exists_subquery(info.parents_expr_info_.has_member(IS_EXISTS)); set_query_ref_exec_params(info.ref_expr_ == NULL ? NULL : &info.ref_expr_->get_exec_params()); if (OB_FAIL(add_cte_table_to_children(subquery_resolver))) { LOG_WARN("add CTE table to children failed", K(ret)); diff --git a/src/sql/resolver/dml/ob_select_resolver.cpp b/src/sql/resolver/dml/ob_select_resolver.cpp index d88d32526..6509f47fc 100644 --- a/src/sql/resolver/dml/ob_select_resolver.cpp +++ b/src/sql/resolver/dml/ob_select_resolver.cpp @@ -53,6 +53,7 @@ ObSelectResolver::ObSelectResolver(ObResolverParams ¶ms) has_top_limit_(false), in_set_query_(false), is_sub_stmt_(false), + in_exists_subquery_(false), standard_group_checker_(), transpose_item_(NULL), is_left_child_(false), @@ -1312,8 +1313,6 @@ int ObSelectResolver::resolve_normal_query(const ParseNode &parse_tree) OB_NOT_NULL(session_info_), OB_NOT_NULL(select_stmt->get_query_ctx())); - set_in_exists_subquery(2 == parse_tree.value_); - /** * @muhang.zb * 定义了一个cte，无论最终是否在主句使用，都必须要进行解析 @@ -2963,6 +2962,8 @@ int ObSelectResolver::resolve_star(const ParseNode *node) SelectItem select_item; if (lib::is_mysql_mode()) { ObConstRawExpr *c_expr = NULL; + select_item.alias_name_ = "1"; + select_item.expr_name_ = "1"; if (!is_in_exists_subquery()) { ret = OB_ERR_NO_TABLES_USED; LOG_WARN("No tables used"); @@ -3024,7 +3025,29 @@ int ObSelectResolver::resolve_star(const ParseNode *node) bool is_column_name_equal = false; const TableItem* tab_item = NULL; ObNameCaseMode case_mode = OB_NAME_CASE_INVALID; - if (OB_FAIL(session_info_->get_name_case_mode(case_mode))) { + if (is_in_exists_subquery()) { + // Oracle and MySQL support use any.* as EXISTS subquery select item. + // Consider SQL: SELECT ... FROM T1 WHERE EXISTS (SELECT T3.* FROM T2); + // Even if T3 does not exist, this SQL statement can still be executed + // successfully. + // Here, we just simply resolve any.* in exists subquery as 1. + SelectItem select_item; + ObConstRawExpr *c_expr = NULL; + select_item.alias_name_ = "1"; + select_item.expr_name_ = "1"; + if (lib::is_oracle_mode() && + OB_FAIL(ObRawExprUtils::build_const_number_expr(*params_.expr_factory_, ObNumberType, + number::ObNumber::get_positive_one(), c_expr))) { + LOG_WARN("failed to build const number 1 expr", K(ret)); + } else if (!lib::is_oracle_mode() && + OB_FAIL(ObRawExprUtils::build_const_int_expr(*params_.expr_factory_, + ObIntType, 1, c_expr))) { + LOG_WARN("failed to build const int 1 expr", K(ret)); + } else if (OB_FALSE_IT(select_item.expr_ = c_expr)) { + } else if (OB_FAIL(select_stmt->add_select_item(select_item))) { + LOG_WARN("failed to add select item", K(ret)); + } + } else if (OB_FAIL(session_info_->get_name_case_mode(case_mode))) { LOG_WARN("fail to get name case mode", K(ret)); } else if (OB_FAIL(ObResolverUtils::resolve_column_ref(node, case_mode, column_ref))) { LOG_WARN("fail to resolve table name", K(ret)); @@ -5785,6 +5808,7 @@ int ObSelectResolver::resolve_subquery_info(const ObIArray &subq subquery_resolver.set_is_sub_stmt(true); subquery_resolver.set_parent_namespace_resolver(this); subquery_resolver.set_current_view_level(current_view_level_); + subquery_resolver.set_in_exists_subquery(info.parents_expr_info_.has_member(IS_EXISTS)); set_query_ref_exec_params(info.ref_expr_ == NULL ? NULL : &info.ref_expr_->get_exec_params()); resolve_alias_for_subquery_ = !(T_FIELD_LIST_SCOPE == current_scope_ && info.parents_expr_info_.has_member(IS_AGG)); diff --git a/src/sql/resolver/dml/ob_view_table_resolver.cpp b/src/sql/resolver/dml/ob_view_table_resolver.cpp index 4f4e37e4a..e3de18be1 100644 --- a/src/sql/resolver/dml/ob_view_table_resolver.cpp +++ b/src/sql/resolver/dml/ob_view_table_resolver.cpp @@ -257,6 +257,7 @@ int ObViewTableResolver::resolve_subquery_info(const ObIArray &s subquery_resolver.set_parent_namespace_resolver(this); subquery_resolver.set_parent_view_resolver(parent_view_resolver_); subquery_resolver.set_current_view_item(current_view_item); + subquery_resolver.set_in_exists_subquery(info.parents_expr_info_.has_member(IS_EXISTS)); set_query_ref_exec_params(info.ref_expr_ == NULL ? NULL : &info.ref_expr_->get_exec_params()); if (OB_FAIL(add_cte_table_to_children(subquery_resolver))) { LOG_WARN("add CTE table to children failed", K(ret)); diff --git a/src/sql/resolver/expr/ob_expr_info_flag.h b/src/sql/resolver/expr/ob_expr_info_flag.h index 84a5d97f5..b1ef79a98 100644 --- a/src/sql/resolver/expr/ob_expr_info_flag.h +++ b/src/sql/resolver/expr/ob_expr_info_flag.h @@ -130,7 +130,8 @@ enum ObExprInfoFlag IS_ROWID, IS_ROWID_SIMPLE_COND, // rowid = const IS_ROWID_RANGE_COND, // rowid belongs to a range - IS_TABLE_ASSIGN // update t1 set c1 = const + IS_TABLE_ASSIGN, // update t1 set c1 = const + IS_EXISTS, }; #define IS_INFO_MASK_BEGIN IS_CONST @@ -241,6 +242,7 @@ inline const char* get_expr_info_flag_str(const ObExprInfoFlag flag) case IS_ROWID_SIMPLE_COND: { ret = "IS_ROWID_SIMPLE_COND"; break; } case IS_ROWID_RANGE_COND: { ret = "IS_ROWID_RANGE_COND"; break; } case IS_TABLE_ASSIGN: { ret = "IS_TABLE_ASSIGN"; break; } + case IS_EXISTS: { ret = "IS_EXISTS"; break; } default: break; } diff --git a/src/sql/resolver/expr/ob_raw_expr_info_extractor.cpp b/src/sql/resolver/expr/ob_raw_expr_info_extractor.cpp index f3643454e..2dee8b78e 100644 --- a/src/sql/resolver/expr/ob_raw_expr_info_extractor.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_info_extractor.cpp @@ -234,6 +234,12 @@ int ObRawExprInfoExtractor::visit(ObOpRawExpr &expr) LOG_WARN("failed to add flag IS_CONNECT_BY_ROOT", K(ret)); } break; + case T_OP_EXISTS: + case T_OP_NOT_EXISTS: + if (OB_FAIL(expr.add_flag(IS_EXISTS))) { + LOG_WARN("failed to add flag IS_EXISTS", K(ret)); + } + break; default: break; } diff --git a/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp b/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp index aa27f57d1..6cb367390 100644 --- a/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp @@ -577,7 +577,12 @@ int ObRawExprResolverImpl::do_recursive_resolve(const ParseNode *node, break; } case T_OP_EXISTS: - //fall through + if (OB_FAIL(ctx_.parents_expr_info_.add_member(IS_EXISTS))) { + LOG_WARN("failed to add member", K(ret)); + } else if (OB_FAIL(process_any_or_all_node(node, expr))) { + LOG_WARN("fail to process exists node", K(ret), K(node)); + } + break; case T_ANY: //fall through case T_ALL: { From e378f7f147fe12f37220594886beeeace1d11cf3 Mon Sep 17 00:00:00 2001 From: liucc1997 <1192520566@qq.com> Date: Thu, 22 Aug 2024 05:11:17 +0000 Subject: [PATCH 167/249] move gtid/pkt_id to ObReqTransport::AsyncCB and support terminate rpc in ObAsyncRpcProxy --- deps/oblib/src/rpc/frame/ob_req_transport.h | 5 ++- deps/oblib/src/rpc/obrpc/ob_poc_rpc_proxy.cpp | 3 +- deps/oblib/src/rpc/obrpc/ob_poc_rpc_proxy.h | 14 ++++---- deps/oblib/src/rpc/obrpc/ob_rpc_proxy.h | 11 ++++++- deps/oblib/src/rpc/obrpc/ob_rpc_proxy.ipp | 4 +-- src/share/rpc/ob_async_rpc_proxy.h | 32 ++++++++++++++++++- src/sql/engine/px/ob_px_sqc_async_proxy.cpp | 23 ++++++------- .../tablelock/ob_table_lock_rpc_client.cpp | 2 ++ .../tablelock/ob_table_lock_service.cpp | 20 ++++++++---- 9 files changed, 80 insertions(+), 34 deletions(-) diff --git a/deps/oblib/src/rpc/frame/ob_req_transport.h b/deps/oblib/src/rpc/frame/ob_req_transport.h index 020f91a88..baa7d21b8 100644 --- a/deps/oblib/src/rpc/frame/ob_req_transport.h +++ b/deps/oblib/src/rpc/frame/ob_req_transport.h @@ -94,7 +94,8 @@ public: { public: AsyncCB(int pcode) - : low_level_cb_(NULL), dst_(), timeout_(0), tenant_id_(0), + : low_level_cb_(NULL), gtid_(0), pkt_id_(0), + dst_(), timeout_(0), tenant_id_(0), err_(0), pcode_(pcode), send_ts_(0), payload_(0) {} virtual ~AsyncCB() {} @@ -130,6 +131,8 @@ public: obrpc::ObRpcPacketCode get_pcode() const { return static_cast(pcode_); } void* low_level_cb_; + uint64_t gtid_; + uint32_t pkt_id_; private: static const int64_t REQUEST_ITEM_COST_RT = 100 * 1000; // 100ms protected: diff --git a/deps/oblib/src/rpc/obrpc/ob_poc_rpc_proxy.cpp b/deps/oblib/src/rpc/obrpc/ob_poc_rpc_proxy.cpp index 9bcfbf8d8..dc52110f8 100644 --- a/deps/oblib/src/rpc/obrpc/ob_poc_rpc_proxy.cpp +++ b/deps/oblib/src/rpc/obrpc/ob_poc_rpc_proxy.cpp @@ -67,7 +67,8 @@ int ObSyncRespCallback::wait(const int64_t wait_timeout_us, const int64_t pcode, const struct timespec ts = {1, 0}; bool has_terminated = false; while(ATOMIC_LOAD(&cond_) == 0) { - if (OB_UNLIKELY((obrpc::OB_REMOTE_SYNC_EXECUTE == pcode || obrpc::OB_REMOTE_EXECUTE == pcode) + if (OB_UNLIKELY((obrpc::OB_REMOTE_SYNC_EXECUTE == pcode || obrpc::OB_REMOTE_EXECUTE == pcode + || proxy_.is_detect_session_killed()) && !has_terminated && OB_ERR_SESSION_INTERRUPTED == THIS_WORKER.check_status())) { RPC_LOG(INFO, "check session killed, will execute pn_terminate_pkt", K(gtid_), K(pkt_id_)); diff --git a/deps/oblib/src/rpc/obrpc/ob_poc_rpc_proxy.h b/deps/oblib/src/rpc/obrpc/ob_poc_rpc_proxy.h index 19bb03310..13750a537 100644 --- a/deps/oblib/src/rpc/obrpc/ob_poc_rpc_proxy.h +++ b/deps/oblib/src/rpc/obrpc/ob_poc_rpc_proxy.h @@ -28,8 +28,8 @@ namespace obrpc class ObSyncRespCallback { public: - ObSyncRespCallback(ObRpcMemPool& pool) - : pkt_nio_cb_(NULL), pool_(pool), resp_(NULL), sz_(0), cond_(0), send_ret_(common::OB_SUCCESS), gtid_(0), pkt_id_(0){} + ObSyncRespCallback(ObRpcMemPool& pool, ObRpcProxy& proxy) + : pkt_nio_cb_(NULL), pool_(pool), proxy_(proxy), resp_(NULL), sz_(0), cond_(0), send_ret_(common::OB_SUCCESS){} ~ObSyncRespCallback() {} void* alloc(int64_t sz) { return pool_.alloc(sz); } int handle_resp(int io_err, const char* buf, int64_t sz); @@ -45,6 +45,7 @@ public: private: void* pkt_nio_cb_; ObRpcMemPool& pool_; + const ObRpcProxy& proxy_; char* resp_; int64_t sz_; int cond_; @@ -77,9 +78,6 @@ private: void* pkt_nio_cb_; ObRpcMemPool& pool_; UAsyncCB* ucb_; -public: - uint64_t gtid_; - uint32_t pkt_id_; }; void init_ucb(ObRpcProxy& proxy, UAsyncCB* ucb, const common::ObAddr& addr, int64_t send_ts, int64_t payload_sz); @@ -125,7 +123,7 @@ public: auto &set = obrpc::ObRpcPacketSet::instance(); const char* pcode_label = set.name_of_idx(set.idx_of_pcode(pcode)); ObRpcMemPool pool(src_tenant_id, pcode_label); - ObSyncRespCallback cb(pool); + ObSyncRespCallback cb(pool, proxy); char* req = NULL; int64_t req_sz = 0; const char* resp = NULL; @@ -244,8 +242,8 @@ public: set_ucb_args(newcb, args); init_ucb(proxy, cb->get_ucb(), addr, start_ts, req_sz); } - cb->gtid_ = (pnio_group_id<<32) + thread_id; - pkt_id_ptr = &cb->pkt_id_; + ucb->gtid_ = (pnio_group_id<<32) + thread_id; + pkt_id_ptr = &ucb->pkt_id_; } IGNORE_RETURN snprintf(rpc_timeguard_str, sizeof(rpc_timeguard_str), "sz=%ld,pcode=%x,id=%ld", req_sz, pcode, src_tenant_id); timeguard.click(rpc_timeguard_str); diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_proxy.h b/deps/oblib/src/rpc/obrpc/ob_rpc_proxy.h index 2d1982e00..9cf5f80b1 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_proxy.h +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_proxy.h @@ -119,7 +119,8 @@ public: max_process_handler_time_(0), compressor_type_(common::INVALID_COMPRESSOR), src_cluster_id_(common::OB_INVALID_CLUSTER_ID), dst_cluster_id_(common::OB_INVALID_CLUSTER_ID), init_(false), - active_(true), is_trace_time_(false), do_ratelimit_(false), is_bg_flow_(0), rcode_() {} + active_(true), is_trace_time_(false), do_ratelimit_(false), + do_detect_session_killed_(false), is_bg_flow_(0), rcode_() {} virtual ~ObRpcProxy() = default; int init(const rpc::frame::ObReqTransport *transport, @@ -133,6 +134,8 @@ public: int64_t get_timeout() const { return timeout_; } void set_trace_time(const bool is_trace_time) { is_trace_time_ = is_trace_time; } void set_ratelimit(const bool do_ratelimit) { do_ratelimit_ = do_ratelimit; } + void set_detect_session_killed(const bool do_detect) { do_detect_session_killed_ = do_detect; } + bool is_detect_session_killed() const { return do_detect_session_killed_; } void set_bg_flow(const int8_t is_bg_flow) { is_bg_flow_ = is_bg_flow;} void set_max_process_handler_time(const uint32_t max_process_handler_time) { max_process_handler_time_ = max_process_handler_time; } @@ -261,6 +264,7 @@ protected: bool active_; bool is_trace_time_; bool do_ratelimit_; + bool do_detect_session_killed_; int8_t is_bg_flow_; ObRpcResultCode rcode_; }; @@ -391,6 +395,11 @@ extern ObRpcProxy::NoneT None; set_ratelimit(do_ratelimit); \ return *this; \ } \ + inline CLS& detect_session_killed(const bool do_detect) \ + { \ + set_detect_session_killed(do_detect); \ + return *this; \ + } \ inline CLS& bg_flow(const uint32_t is_bg_flow) \ { \ set_bg_flow(is_bg_flow); \ diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_proxy.ipp b/deps/oblib/src/rpc/obrpc/ob_rpc_proxy.ipp index c28221675..527ec9b45 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_proxy.ipp +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_proxy.ipp @@ -55,7 +55,7 @@ int SSHandle::get_more(typename pcodeStruct::Response &result) auto &set = obrpc::ObRpcPacketSet::instance(); const char* pcode_label = set.name_of_idx(set.idx_of_pcode(pcode_)); ObRpcMemPool pool(src_tenant_id, pcode_label); - ObSyncRespCallback cb(pool); + ObSyncRespCallback cb(pool, proxy_); char* pnio_req = NULL; int64_t pnio_req_sz = 0, resp_sz = 0; const char* resp = NULL; @@ -205,7 +205,7 @@ int SSHandle::abort() auto &set = obrpc::ObRpcPacketSet::instance(); const char* pcode_label = set.name_of_idx(set.idx_of_pcode(pcode_)); ObRpcMemPool pool(src_tenant_id, pcode_label); - ObSyncRespCallback cb(pool); + ObSyncRespCallback cb(pool, proxy_); char* pnio_req = NULL; int64_t pnio_req_sz = 0, resp_sz = 0; const char* resp = NULL; diff --git a/src/share/rpc/ob_async_rpc_proxy.h b/src/share/rpc/ob_async_rpc_proxy.h index 5a07711d9..351b887b0 100644 --- a/src/share/rpc/ob_async_rpc_proxy.h +++ b/src/share/rpc/ob_async_rpc_proxy.h @@ -21,6 +21,8 @@ #include "rpc/obrpc/ob_rpc_packet.h" #include "rpc/obrpc/ob_rpc_result_code.h" #include "rpc/obrpc/ob_rpc_proxy.h" +#include "share/ob_errno.h" +#include "lib/worker.h" namespace oceanbase { @@ -403,8 +405,36 @@ int ObAsyncRpcProxy::wait( RPC_LOG(WARN, "inner stat error", K_(response_count), "cb_count", cb_list_.get_size(), K(ret)); } else { + bool has_terminated = false; while (response_count_ < cb_list_.get_size()) { - cond_.wait(); + cond_.wait(1000); + if (OB_UNLIKELY(rpc_proxy_.is_detect_session_killed() + && !has_terminated + && OB_ERR_SESSION_INTERRUPTED == THIS_WORKER.check_status())) { + RPC_LOG(INFO, "check session killed, will terminate all rpc locally", K(response_count_), K(cb_list_.get_size())); + int tmp_ret = OB_SUCCESS; + int index = 0; + ObAsyncCB *cb = cb_list_.get_first(); + while (common::OB_SUCCESS == tmp_ret && cb != cb_list_.get_header()) { + if (NULL == cb) { + tmp_ret = common::OB_ERR_UNEXPECTED; + RPC_LOG_RET(WARN, tmp_ret, "cb is null", KP(cb)); + } else { + RPC_LOG(INFO, "terminate the rpc of cb_list", K(cb->gtid_), K(cb->pkt_id_), K(index)); + int err = 0; + if ((err = pn_terminate_pkt(cb->gtid_, cb->pkt_id_)) != 0) { + tmp_ret = tranlate_to_ob_error(err); + RPC_LOG_RET(WARN, tmp_ret, "pn_terminate_pkt failed", K(err)); + } else { + cb = cb->get_next(); + ++index; + } + } + } + if (index == cb_list_.get_size()) { + has_terminated = true; + } + } } // set results diff --git a/src/sql/engine/px/ob_px_sqc_async_proxy.cpp b/src/sql/engine/px/ob_px_sqc_async_proxy.cpp index 126d3bd3b..a4ca0c73c 100644 --- a/src/sql/engine/px/ob_px_sqc_async_proxy.cpp +++ b/src/sql/engine/px/ob_px_sqc_async_proxy.cpp @@ -253,20 +253,15 @@ void ObPxSqcAsyncProxy::fail_process() { K(return_cb_count_), K(callbacks_.count())); ARRAY_FOREACH_X(callbacks_, idx, count, true) { ObSqcAsyncCB &callback = *callbacks_.at(idx); - { - // avoid rpc thread access the callback currently. - ObThreadCondGuard guard(callback.get_cond()); - if (!callback.is_visited() && - !(callback.is_processed() || callback.is_timeout() || callback.is_invalid())) { - // unregister async callbacks that have not received response. - ObAsyncRespCallback *async_cb = static_cast(callback.low_level_cb_); - uint64_t gtid = async_cb->gtid_; - uint32_t pkt_id = async_cb->pkt_id_; - int err = 0; - if ((err = pn_terminate_pkt(gtid, pkt_id)) != 0) { - int ret = tranlate_to_ob_error(err); - LOG_WARN("terminate pkt failed", K(ret), K(err)); - } + if (!callback.is_visited() && + !(callback.is_processed() || callback.is_timeout() || callback.is_invalid())) { + // unregister async callbacks that have not received response. + uint64_t gtid = callback.gtid_; + uint32_t pkt_id = callback.pkt_id_; + int err = 0; + if ((err = pn_terminate_pkt(gtid, pkt_id)) != 0) { + int ret = tranlate_to_ob_error(err); + LOG_WARN("terminate pkt failed", K(ret), K(err)); } } } diff --git a/src/storage/tablelock/ob_table_lock_rpc_client.cpp b/src/storage/tablelock/ob_table_lock_rpc_client.cpp index 180bcd200..9d5a94456 100644 --- a/src/storage/tablelock/ob_table_lock_rpc_client.cpp +++ b/src/storage/tablelock/ob_table_lock_rpc_client.cpp @@ -59,6 +59,8 @@ int ObTableLockRpcClient::init() if (OB_FAIL(table_lock_rpc_proxy_.init(GCTX.net_frame_->get_req_transport(), GCTX.self_addr()))) { LOG_WARN("failed to init rpc proxy", K(ret)); + } else { + table_lock_rpc_proxy_.set_detect_session_killed(true); } return ret; } diff --git a/src/storage/tablelock/ob_table_lock_service.cpp b/src/storage/tablelock/ob_table_lock_service.cpp index 56870ebf6..bdd5c44aa 100644 --- a/src/storage/tablelock/ob_table_lock_service.cpp +++ b/src/storage/tablelock/ob_table_lock_service.cpp @@ -1760,7 +1760,9 @@ int ObTableLockService::batch_pre_check_lock_(ObTableLockCtx &ctx, int last_ret = OB_SUCCESS; int64_t USLEEP_TIME = 100; // 0.1 ms bool need_retry = false; - ObBatchLockProxy proxy_batch(*GCTX.srv_rpc_proxy_, &obrpc::ObSrvRpcProxy::batch_lock_obj); + obrpc::ObSrvRpcProxy rpc_proxy(*GCTX.srv_rpc_proxy_); + rpc_proxy.set_detect_session_killed(true); + ObBatchLockProxy proxy_batch(rpc_proxy, &obrpc::ObSrvRpcProxy::batch_lock_obj); // only used in LOCK_TABLE/LOCK_PARTITION if (LOCK_TABLE == ctx.task_type_ || LOCK_PARTITION == ctx.task_type_) { @@ -1864,7 +1866,9 @@ int ObTableLockService::pre_check_lock_old_version_(ObTableLockCtx &ctx, ObRetryCtx retry_ctx; ObAddr addr; ObTableLockTaskRequest request; - ObTableLockProxy proxy_batch(*GCTX.srv_rpc_proxy_, &obrpc::ObSrvRpcProxy::lock_table); + obrpc::ObSrvRpcProxy rpc_proxy(*GCTX.srv_rpc_proxy_); + rpc_proxy.set_detect_session_killed(true); + ObTableLockProxy proxy_batch(rpc_proxy, &obrpc::ObSrvRpcProxy::lock_table); // only used in LOCK_TABLE/LOCK_PARTITION if (LOCK_TABLE == ctx.task_type_ || LOCK_PARTITION == ctx.task_type_) { @@ -2306,11 +2310,13 @@ int ObTableLockService::inner_process_obj_lock_batch_(ObTableLockCtx &ctx, const ObTableLockOwnerID lock_owner) { int ret = OB_SUCCESS; + obrpc::ObSrvRpcProxy rpc_proxy(*GCTX.srv_rpc_proxy_); + rpc_proxy.set_detect_session_killed(true); if (ctx.is_unlock_task()) { - ObHighPriorityBatchLockProxy proxy_batch(*GCTX.srv_rpc_proxy_, &obrpc::ObSrvRpcProxy::batch_unlock_obj); + ObHighPriorityBatchLockProxy proxy_batch(rpc_proxy, &obrpc::ObSrvRpcProxy::batch_unlock_obj); ret = batch_rpc_handle_(proxy_batch, ctx, lock_map, lock_mode, lock_owner); } else { - ObBatchLockProxy proxy_batch(*GCTX.srv_rpc_proxy_, &obrpc::ObSrvRpcProxy::batch_lock_obj); + ObBatchLockProxy proxy_batch(rpc_proxy, &obrpc::ObSrvRpcProxy::batch_lock_obj); ret = batch_rpc_handle_(proxy_batch, ctx, lock_map, lock_mode, lock_owner); } @@ -2324,12 +2330,14 @@ int ObTableLockService::inner_process_obj_lock_old_version_(ObTableLockCtx &ctx, const ObTableLockOwnerID lock_owner) { int ret = OB_SUCCESS; + obrpc::ObSrvRpcProxy rpc_proxy(*GCTX.srv_rpc_proxy_); + rpc_proxy.set_detect_session_killed(true); // TODO: yanyuan.cxf we process the rpc one by one and do parallel later. if (ctx.is_unlock_task()) { - ObHighPriorityTableLockProxy proxy_batch(*GCTX.srv_rpc_proxy_, &obrpc::ObSrvRpcProxy::unlock_table); + ObHighPriorityTableLockProxy proxy_batch(rpc_proxy, &obrpc::ObSrvRpcProxy::unlock_table); ret = parallel_rpc_handle_(proxy_batch, ctx, lock_map, ls_lock_map, lock_mode, lock_owner); } else { - ObTableLockProxy proxy_batch(*GCTX.srv_rpc_proxy_, &obrpc::ObSrvRpcProxy::lock_table); + ObTableLockProxy proxy_batch(rpc_proxy, &obrpc::ObSrvRpcProxy::lock_table); ret = parallel_rpc_handle_(proxy_batch, ctx, lock_map, ls_lock_map, lock_mode, lock_owner); } From b974a3dc7d72e0d875d1befe2c1de0c4cb2a61b1 Mon Sep 17 00:00:00 2001 From: WeiXinChan Date: Thu, 22 Aug 2024 05:29:55 +0000 Subject: [PATCH 168/249] [CP][FEAT MERGE] [Patch from 4.2.1.8] add ob_kv_mode variable to mark OBKV tenant and limit tenant capacity --- src/observer/CMakeLists.txt | 1 + .../table/ob_table_batch_execute_processor.h | 1 + .../table/ob_table_direct_load_processor.h | 1 + .../table/ob_table_execute_processor.h | 2 +- src/observer/table/ob_table_mode_control.cpp | 66 +++++++++++++++++++ src/observer/table/ob_table_mode_control.h | 38 +++++++++++ .../ob_table_query_and_mutate_processor.h | 2 +- .../table/ob_table_query_async_processor.h | 2 +- src/observer/table/ob_table_query_processor.h | 1 + src/observer/table/ob_table_rpc_processor.cpp | 24 +++++++ src/observer/table/ob_table_rpc_processor.h | 2 + 11 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 src/observer/table/ob_table_mode_control.cpp create mode 100644 src/observer/table/ob_table_mode_control.h diff --git a/src/observer/CMakeLists.txt b/src/observer/CMakeLists.txt index 35d07689e..77d4ff66d 100644 --- a/src/observer/CMakeLists.txt +++ b/src/observer/CMakeLists.txt @@ -175,6 +175,7 @@ ob_set_subtarget(ob_server table table/ob_table_move_response.cpp table/ob_table_connection_mgr.cpp table/redis/ob_redis_meta.cpp + table/ob_table_mode_control.cpp ) ob_set_subtarget(ob_server table_load diff --git a/src/observer/table/ob_table_batch_execute_processor.h b/src/observer/table/ob_table_batch_execute_processor.h index 982df4151..8bf2dbc43 100644 --- a/src/observer/table/ob_table_batch_execute_processor.h +++ b/src/observer/table/ob_table_batch_execute_processor.h @@ -50,6 +50,7 @@ protected: table::ObTableAPITransCb *new_callback(rpc::ObRequest *req) override; virtual void audit_on_finish() override; virtual uint64_t get_request_checksum() override; + virtual table::ObTableEntityType get_entity_type() override { return arg_.entity_type_; } virtual bool is_kv_processor() override { return true; } private: diff --git a/src/observer/table/ob_table_direct_load_processor.h b/src/observer/table/ob_table_direct_load_processor.h index a87e61833..a205b3b56 100644 --- a/src/observer/table/ob_table_direct_load_processor.h +++ b/src/observer/table/ob_table_direct_load_processor.h @@ -33,6 +33,7 @@ protected: int try_process() override; table::ObTableAPITransCb *new_callback(rpc::ObRequest *req) override; uint64_t get_request_checksum() override; + virtual table::ObTableEntityType get_entity_type() override { return table::ObTableEntityType::ET_DYNAMIC; } virtual bool is_kv_processor() override { return false; } private: common::ObArenaAllocator allocator_; diff --git a/src/observer/table/ob_table_execute_processor.h b/src/observer/table/ob_table_execute_processor.h index 15a191246..5f3e1557d 100644 --- a/src/observer/table/ob_table_execute_processor.h +++ b/src/observer/table/ob_table_execute_processor.h @@ -46,8 +46,8 @@ protected: virtual void audit_on_finish() override; virtual uint64_t get_request_checksum() override; virtual int before_response(int error_code) override; + virtual table::ObTableEntityType get_entity_type() override { return arg_.entity_type_; } virtual bool is_kv_processor() override { return true; } - private: int init_tb_ctx(); int check_arg2() const; diff --git a/src/observer/table/ob_table_mode_control.cpp b/src/observer/table/ob_table_mode_control.cpp new file mode 100644 index 000000000..6698df33d --- /dev/null +++ b/src/observer/table/ob_table_mode_control.cpp @@ -0,0 +1,66 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SERVER +#include "ob_table_mode_control.h" + +using namespace oceanbase::common; +using namespace oceanbase::table; + +int ObTableModeCtrl::check_mode(ObKvModeType tenant_mode, ObTableEntityType entity_type) +{ + int ret = OB_SUCCESS; + + switch (tenant_mode) { + case ObKvModeType::ALL: { + // all mode is supported + break; + } + case ObKvModeType::TABLEAPI: { + if (entity_type != ObTableEntityType::ET_KV && entity_type != ObTableEntityType::ET_DYNAMIC) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "As the ob_kv_mode variable has been set to 'TABLEAPI', your current interfaces"); + LOG_WARN("mode not matched", K(ret), K(entity_type), K(tenant_mode)); + } + break; + } + case ObKvModeType::HBASE: { + if (entity_type != ObTableEntityType::ET_HKV && entity_type != ObTableEntityType::ET_DYNAMIC) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "As the ob_kv_mode variable has been set to 'HBASE', your current interfaces"); + LOG_WARN("mode not matched", K(ret), K(entity_type), K(tenant_mode)); + } + break; + } + case ObKvModeType::REDIS: { + // OBKV-Redis model is not supported in 433 + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "As the ob_kv_mode variable has been set to 'REDIS', your current interfaces"); + LOG_WARN("redis is not supported now", K(ret), K(entity_type), K(tenant_mode)); + break; + } + case ObKvModeType::NONE: { + // all modes are not supported + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "As the ob_kv_mode variable has been set to 'NONE', your current interfaces"); + LOG_WARN("all OBKV modes are not supported", K(ret), K(entity_type), K(tenant_mode)); + break; + } + default: { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unknown ob_kv_mode", K(ret), K(tenant_mode)); + break; + } + } + + return ret; +} diff --git a/src/observer/table/ob_table_mode_control.h b/src/observer/table/ob_table_mode_control.h new file mode 100644 index 000000000..f898472b7 --- /dev/null +++ b/src/observer/table/ob_table_mode_control.h @@ -0,0 +1,38 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#ifndef _OB_TABLE_MODE_CONTROL_H +#define _OB_TABLE_MODE_CONTROL_H +#include "share/table/ob_table.h" +namespace oceanbase +{ +namespace table +{ + +enum class ObKvModeType +{ + ALL, + TABLEAPI, + HBASE, + REDIS, + NONE +}; + +class ObTableModeCtrl +{ +public: + static int check_mode(ObKvModeType tenant_mode, ObTableEntityType entity_type); +}; + +} // end namespace table +} // end namespace oceanbase + +#endif /* _OB_TABLE_MODE_CONTROL_H */ \ No newline at end of file diff --git a/src/observer/table/ob_table_query_and_mutate_processor.h b/src/observer/table/ob_table_query_and_mutate_processor.h index 9e97822ca..49f6562ba 100644 --- a/src/observer/table/ob_table_query_and_mutate_processor.h +++ b/src/observer/table/ob_table_query_and_mutate_processor.h @@ -45,7 +45,7 @@ protected: virtual void audit_on_finish() override; virtual uint64_t get_request_checksum() override; virtual bool is_kv_processor() override { return true; } - + virtual table::ObTableEntityType get_entity_type() override { return arg_.entity_type_; } private: typedef std::pair ColumnIdx; class ColumnIdxComparator; diff --git a/src/observer/table/ob_table_query_async_processor.h b/src/observer/table/ob_table_query_async_processor.h index ae115a3f4..95a37e90d 100644 --- a/src/observer/table/ob_table_query_async_processor.h +++ b/src/observer/table/ob_table_query_async_processor.h @@ -222,7 +222,7 @@ protected: virtual uint64_t get_request_checksum() override; virtual table::ObTableAPITransCb *new_callback(rpc::ObRequest *req) override; virtual bool is_kv_processor() override { return true; } - + virtual table::ObTableEntityType get_entity_type() override { return arg_.entity_type_; } private: int process_query_start(); int process_query_next(); diff --git a/src/observer/table/ob_table_query_processor.h b/src/observer/table/ob_table_query_processor.h index 648517f34..f83471ed8 100644 --- a/src/observer/table/ob_table_query_processor.h +++ b/src/observer/table/ob_table_query_processor.h @@ -39,6 +39,7 @@ protected: virtual table::ObTableAPITransCb *new_callback(rpc::ObRequest *req) override; virtual void audit_on_finish() override; virtual uint64_t get_request_checksum() override; + virtual table::ObTableEntityType get_entity_type() override { return arg_.entity_type_; } virtual bool is_kv_processor() override { return true; } private: diff --git a/src/observer/table/ob_table_rpc_processor.cpp b/src/observer/table/ob_table_rpc_processor.cpp index 073504e80..7a0f15a35 100644 --- a/src/observer/table/ob_table_rpc_processor.cpp +++ b/src/observer/table/ob_table_rpc_processor.cpp @@ -34,12 +34,14 @@ #include "share/table/ob_table_util.h" #include "observer/mysql/obmp_base.h" #include "lib/stat/ob_session_stat.h" +#include "ob_table_mode_control.h" using namespace oceanbase::observer; using namespace oceanbase::common; using namespace oceanbase::table; using namespace oceanbase::share; using namespace oceanbase::obrpc; +using namespace oceanbase::sql; int ObTableLoginP::process() { @@ -298,12 +300,34 @@ int ObTableApiProcessorBase::check_user_access(const ObString &credential_str) } else if (sess_credetial->cluster_id_ != credential_.cluster_id_) { ret = OB_ERR_NO_PRIVILEGE; LOG_WARN("invalid credential cluster id", K(ret), K_(credential), K(*sess_credetial)); + } else if (OB_FAIL(check_mode(guard.get_sess_info()))) { + LOG_WARN("fail to check mode", K(ret)); } else { LOG_DEBUG("user can access", K_(credential)); } return ret; } +int ObTableApiProcessorBase::check_mode(const ObSQLSessionInfo &sess_info) +{ + int ret = OB_SUCCESS; + int64_t sess_mode_val = 0; + + if (!is_kv_processor()) { + // do nothing + } else if (lib::is_oracle_mode()) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "OBKV running in oracle mode"); + LOG_WARN("OBKV running in oracle mode is not supported", K(ret)); + } else if (OB_FAIL(sess_info.get_sys_variable(SYS_VAR_OB_KV_MODE, sess_mode_val))) { + LOG_WARN("fail to get ob_kv_mode variable", K(ret)); + } else if (OB_FAIL(ObTableModeCtrl::check_mode(static_cast(sess_mode_val), get_entity_type()))) { + LOG_WARN("fail to check mode", K(ret), K(sess_mode_val), K(get_entity_type())); + } + + return ret; +} + oceanbase::sql::ObSQLSessionInfo &session() { static oceanbase::sql::ObSQLSessionInfo SESSION; diff --git a/src/observer/table/ob_table_rpc_processor.h b/src/observer/table/ob_table_rpc_processor.h index f20ef8513..f40d9320a 100644 --- a/src/observer/table/ob_table_rpc_processor.h +++ b/src/observer/table/ob_table_rpc_processor.h @@ -120,6 +120,7 @@ public: return bret; } int check_user_access(const ObString &credential_str); + int check_mode(const sql::ObSQLSessionInfo &sess_info); // transaction control int start_trans(bool is_readonly, const sql::stmt::StmtType stmt_type, const table::ObTableConsistencyLevel consistency_level, uint64_t table_id, @@ -156,6 +157,7 @@ protected: virtual void set_req_has_wokenup() = 0; virtual bool is_kv_processor() = 0; virtual void reset_ctx(); + virtual table::ObTableEntityType get_entity_type() = 0; int get_ls_id(const ObTabletID &tablet_id, share::ObLSID &ls_id); int process_with_retry(const ObString &credential, const int64_t timeout_ts); From 048b7033d7bec18d06919a16f531044b2dd9fe2a Mon Sep 17 00:00:00 2001 From: hnwyllmm Date: Thu, 22 Aug 2024 05:36:29 +0000 Subject: [PATCH 169/249] [FEAT MERGE] external table support compressed csv file --- .../engine/cmd/ob_load_data_file_reader.cpp | 44 ++- src/sql/engine/cmd/ob_load_data_file_reader.h | 11 +- src/sql/engine/cmd/ob_load_data_parser.cpp | 60 ++++ src/sql/engine/cmd/ob_load_data_parser.h | 23 +- .../ob_external_table_access_service.cpp | 267 ++++++++++++++++-- .../table/ob_external_table_access_service.h | 72 ++++- src/sql/parser/sql_parser_mysql_mode.y | 4 + src/sql/resolver/cmd/ob_load_data_stmt.h | 8 - src/sql/resolver/ddl/ob_ddl_resolver.cpp | 13 + 9 files changed, 429 insertions(+), 73 deletions(-) diff --git a/src/sql/engine/cmd/ob_load_data_file_reader.cpp b/src/sql/engine/cmd/ob_load_data_file_reader.cpp index ac9384871..f9fb22b24 100644 --- a/src/sql/engine/cmd/ob_load_data_file_reader.cpp +++ b/src/sql/engine/cmd/ob_load_data_file_reader.cpp @@ -44,27 +44,11 @@ ObFileReadParam::ObFileReadParam() int ObFileReadParam::parse_compression_format(ObString compression_name, ObString filename, ObLoadCompressionFormat &compression_format) { int ret = OB_SUCCESS; - if (compression_name.length() == 0 || - 0 == compression_name.case_compare("none")) { + if (compression_name.length() == 0) { compression_format = ObLoadCompressionFormat::NONE; - } else if (0 == compression_name.case_compare("gzip")) { - compression_format = ObLoadCompressionFormat::GZIP; - } else if (0 == compression_name.case_compare("deflate")) { - compression_format = ObLoadCompressionFormat::DEFLATE; - } else if (0 == compression_name.case_compare("zstd")) { - compression_format = ObLoadCompressionFormat::ZSTD; - } else if (0 == compression_name.case_compare("auto")) { - if (filename.suffix_match_ci(".gz")) { - compression_format = ObLoadCompressionFormat::GZIP; - } else if (filename.suffix_match_ci(".deflate")) { - compression_format = ObLoadCompressionFormat::DEFLATE; - } else if (filename.suffix_match_ci(".zst") || filename.suffix_match_ci(".zstd")) { - compression_format = ObLoadCompressionFormat::ZSTD; - } else { - ret = OB_INVALID_ARGUMENT; - } - } else { - ret = OB_INVALID_ARGUMENT; + } else if (OB_FAIL(compression_format_from_string(compression_name, compression_format))) { + } else if (ObLoadCompressionFormat::AUTO == compression_format) { + ret = compression_format_from_suffix(filename, compression_format); } return ret; } @@ -563,7 +547,7 @@ int ObDecompressor::create(ObLoadCompressionFormat format, ObIAllocator &allocat case ObLoadCompressionFormat::GZIP: case ObLoadCompressionFormat::DEFLATE: { - decompressor = OB_NEW(ObZlibDecompressor, MEMORY_ATTR, allocator); + decompressor = OB_NEW(ObZlibDecompressor, MEMORY_ATTR, allocator, format); } break; case ObLoadCompressionFormat::ZSTD: { @@ -579,14 +563,22 @@ int ObDecompressor::create(ObLoadCompressionFormat format, ObIAllocator &allocat if (OB_SUCC(ret) && OB_NOT_NULL(decompressor)) { if (OB_FAIL(decompressor->init())) { LOG_WARN("failed to init decompressor", KR(ret)); - decompressor->destroy(); - OB_DELETE(ObDecompressor, MEMORY_ATTR, decompressor); + ObDecompressor::destroy(decompressor); + decompressor = nullptr; } } return ret; } +void ObDecompressor::destroy(ObDecompressor *decompressor) +{ + if (OB_NOT_NULL(decompressor)) { + decompressor->destroy(); + OB_DELETE(ObDecompressor, MEMORY_ATTR, decompressor); + } +} + /** * ObDecompressFileReader */ @@ -603,7 +595,7 @@ ObDecompressFileReader::~ObDecompressFileReader() } if (OB_NOT_NULL(decompressor_)) { - OB_DELETE(ObDecompressor, MEMORY_ATTR, decompressor_); + ObDecompressor::destroy(decompressor_); } if (OB_NOT_NULL(compressed_data_)) { @@ -721,8 +713,8 @@ void zlib_free(voidpf opaque, voidpf address) } } -ObZlibDecompressor::ObZlibDecompressor(ObIAllocator &allocator) - : ObDecompressor(allocator) +ObZlibDecompressor::ObZlibDecompressor(ObIAllocator &allocator, ObLoadCompressionFormat compression_format) + : ObDecompressor(allocator), compression_format_(compression_format) {} ObZlibDecompressor::~ObZlibDecompressor() diff --git a/src/sql/engine/cmd/ob_load_data_file_reader.h b/src/sql/engine/cmd/ob_load_data_file_reader.h index b95050285..264aa97ba 100644 --- a/src/sql/engine/cmd/ob_load_data_file_reader.h +++ b/src/sql/engine/cmd/ob_load_data_file_reader.h @@ -18,6 +18,7 @@ #include "lib/allocator/ob_allocator.h" #include "lib/file/ob_file.h" #include "sql/resolver/cmd/ob_load_data_stmt.h" +#include "sql/engine/cmd/ob_load_data_parser.h" #include "share/backup/ob_backup_struct.h" #include "observer/mysql/obmp_packet_sender.h" @@ -225,7 +226,10 @@ public: virtual int decompress(const char *src, int64_t src_size, int64_t &consumed_size, char *dest, int64_t dest_capacity, int64_t &decompressed_size) = 0; + virtual ObLoadCompressionFormat compression_format() const = 0; + static int create(ObLoadCompressionFormat format, ObIAllocator &allocator, ObDecompressor *&decompressor); + static void destroy(ObDecompressor *decompressor); protected: ObIAllocator &allocator_; @@ -271,7 +275,7 @@ protected: class ObZlibDecompressor : public ObDecompressor { public: - explicit ObZlibDecompressor(ObIAllocator &allocator); + explicit ObZlibDecompressor(ObIAllocator &allocator, ObLoadCompressionFormat compression_format); virtual ~ObZlibDecompressor(); int init() override; @@ -280,9 +284,13 @@ public: int decompress(const char *src, int64_t src_size, int64_t &consumed_size, char *dest, int64_t dest_capacity, int64_t &decompressed_size) override; + ObLoadCompressionFormat compression_format() const override { return compression_format_; } + private: void *zlib_stream_ptr_ = nullptr; bool zstream_need_reset_ = false; // the zstreamptr should be reset if we got Z_STREAM_END + + ObLoadCompressionFormat compression_format_; }; /** @@ -300,6 +308,7 @@ public: int decompress(const char *src, int64_t src_size, int64_t &consumed_size, char *dest, int64_t dest_capacity, int64_t &decompressed_size) override; + ObLoadCompressionFormat compression_format() const override { return ObLoadCompressionFormat::ZSTD; } private: void *zstd_stream_context_ = nullptr; }; diff --git a/src/sql/engine/cmd/ob_load_data_parser.cpp b/src/sql/engine/cmd/ob_load_data_parser.cpp index 0b971c2ef..7a2f6b54c 100644 --- a/src/sql/engine/cmd/ob_load_data_parser.cpp +++ b/src/sql/engine/cmd/ob_load_data_parser.cpp @@ -336,6 +336,54 @@ int ObOriginFileFormat::load_from_json_data(json::Pair *&node, ObIAllocator &all return ret; } +const char *compression_format_to_string(ObLoadCompressionFormat compression_format) +{ + switch (compression_format) { + case ObLoadCompressionFormat::NONE: return "NONE"; + case ObLoadCompressionFormat::AUTO: return "AUTO"; + case ObLoadCompressionFormat::GZIP: return "GZIP"; + case ObLoadCompressionFormat::DEFLATE: return "DEFLATE"; + case ObLoadCompressionFormat::ZSTD: return "ZSTD"; + default: return "INVALID"; + } +} + +int compression_format_from_string(ObString compression_name, ObLoadCompressionFormat &compression_format) +{ + int ret = OB_SUCCESS; + + if (compression_name.length() == 0 || + 0 == compression_name.case_compare("none")) { + compression_format = ObLoadCompressionFormat::NONE; + } else if (0 == compression_name.case_compare("gzip")) { + compression_format = ObLoadCompressionFormat::GZIP; + } else if (0 == compression_name.case_compare("deflate")) { + compression_format = ObLoadCompressionFormat::DEFLATE; + } else if (0 == compression_name.case_compare("zstd")) { + compression_format = ObLoadCompressionFormat::ZSTD; + } else if (0 == compression_name.case_compare("auto")) { + compression_format = ObLoadCompressionFormat::AUTO; + } else { + ret = OB_INVALID_ARGUMENT; + compression_format = ObLoadCompressionFormat::INVALID; + } + return ret; +} + +int compression_format_from_suffix(ObString filename, ObLoadCompressionFormat &compression_format) +{ + int ret = OB_SUCCESS; + if (filename.suffix_match_ci(".gz")) { + compression_format = ObLoadCompressionFormat::GZIP; + } else if (filename.suffix_match_ci(".deflate")) { + compression_format = ObLoadCompressionFormat::DEFLATE; + } else if (filename.suffix_match_ci(".zst") || filename.suffix_match_ci(".zstd")) { + compression_format = ObLoadCompressionFormat::ZSTD; + } else { + compression_format = ObLoadCompressionFormat::NONE; + } + return ret; +} int64_t ObExternalFileFormat::to_string(char *buf, const int64_t buf_len) const { int64_t pos = 0; @@ -354,6 +402,11 @@ int64_t ObExternalFileFormat::to_string(char *buf, const int64_t buf_len) const pos += 0; } + if (compression_format_ != ObLoadCompressionFormat::NONE) { + J_COMMA(); + databuff_print_kv(buf, buf_len, pos, "\"COMPRESSION\"", compression_format_to_string(compression_format_)); + } + J_OBJ_END(); return pos; } @@ -400,6 +453,13 @@ int ObExternalFileFormat::load_from_string(const ObString &str, ObIAllocator &al LOG_WARN("invalid format type", K(ret), K(format_type_str)); break; } + + if (OB_SUCC(ret) && OB_NOT_NULL(format_type_node) + && 0 == format_type_node->name_.case_compare("COMPRESSION") + && format_type_node->value_->get_type() == json::JT_STRING) { + ObString compression_format_str = format_type_node->value_->get_string(); + OZ(compression_format_from_string(compression_format_str, compression_format_)); + } } } return ret; diff --git a/src/sql/engine/cmd/ob_load_data_parser.h b/src/sql/engine/cmd/ob_load_data_parser.h index 14870e89d..bf422b472 100644 --- a/src/sql/engine/cmd/ob_load_data_parser.h +++ b/src/sql/engine/cmd/ob_load_data_parser.h @@ -489,6 +489,26 @@ struct ObOriginFileFormat common::ObString origin_null_if_str_; }; +enum class ObLoadCompressionFormat +{ + INVALID, + NONE, + AUTO, + GZIP, + DEFLATE, + ZSTD, +}; + +const char *compression_format_to_string(ObLoadCompressionFormat compression_format); +int compression_format_from_string(ObString compression_name, ObLoadCompressionFormat &compression_format); + +/** + * guess compression format from filename suffix + * + * Return NONE if none of the known compression format matches. + */ +int compression_format_from_suffix(ObString filename, ObLoadCompressionFormat &compression_format); + struct ObExternalFileFormat { struct StringData { @@ -512,7 +532,7 @@ struct ObExternalFileFormat OPT_BINARY_AS_TEXT = 1 << 1, }; - ObExternalFileFormat() : format_type_(INVALID_FORMAT) {} + ObExternalFileFormat() : format_type_(INVALID_FORMAT), compression_format_(ObLoadCompressionFormat::NONE) {} int64_t to_string(char* buf, const int64_t buf_len) const; int load_from_string(const common::ObString &str, common::ObIAllocator &allocator); @@ -521,6 +541,7 @@ struct ObExternalFileFormat ObOriginFileFormat origin_file_format_str_; FormatType format_type_; sql::ObCSVGeneralFormat csv_format_; + ObLoadCompressionFormat compression_format_; uint64_t options_; static const char *FORMAT_TYPE_STR[]; diff --git a/src/sql/engine/table/ob_external_table_access_service.cpp b/src/sql/engine/table/ob_external_table_access_service.cpp index 31e826a73..673b3b707 100644 --- a/src/sql/engine/table/ob_external_table_access_service.cpp +++ b/src/sql/engine/table/ob_external_table_access_service.cpp @@ -22,6 +22,7 @@ #include "share/ob_device_manager.h" #include "lib/utility/ob_macro_utils.h" #include "sql/engine/table/ob_parquet_table_row_iter.h" +#include "sql/engine/cmd/ob_load_data_file_reader.h" namespace oceanbase { @@ -330,6 +331,215 @@ int ObExternalDataAccessDriver::init(const ObString &location, const ObString &a return ret; } +ObExternalStreamFileReader::~ObExternalStreamFileReader() +{ + reset(); +} + +const char * ObExternalStreamFileReader::MEMORY_LABEL = "ExternalReader"; +const int64_t ObExternalStreamFileReader::COMPRESSED_DATA_BUFFER_SIZE = 2 * 1024 * 1024; + +int ObExternalStreamFileReader::init(const common::ObString &location, + const ObString &access_info, + ObLoadCompressionFormat compression_format, + ObIAllocator &allocator) +{ + int ret = OB_SUCCESS; + if (OB_NOT_NULL(allocator_)) { + ret = OB_INIT_TWICE; + } else if (OB_FAIL(data_access_driver_.init(location, access_info))) { + LOG_WARN("failed to init data access driver", K(ret), K(location), K(access_info)); + } else { + allocator_ = &allocator; + compression_format_ = compression_format; + } + return ret; +} + +int ObExternalStreamFileReader::open(const ObString &filename) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(allocator_)) { + ret = OB_NOT_INIT; + } else if (data_access_driver_.is_opened()) { + ret = OB_INIT_TWICE; + } else if (OB_FAIL(data_access_driver_.open(filename.ptr()))) { + LOG_WARN("failed to open file", K(ret), K(filename)); + } else if (OB_FAIL(data_access_driver_.get_file_size(filename.ptr(), file_size_))) { + LOG_WARN("failed to get file size", K(ret), K(filename)); + } else { + ObLoadCompressionFormat this_file_compression_format = compression_format_; + if (this_file_compression_format == ObLoadCompressionFormat::AUTO + && OB_FAIL(compression_format_from_suffix(filename, this_file_compression_format))) { + LOG_WARN("failed to dectect compression format from filename", K(ret), K(filename)); + } + + if (OB_SUCC(ret) && OB_FAIL(create_decompressor(this_file_compression_format))) { + LOG_WARN("failed to create decompressor", K(ret)); + } + } + + LOG_TRACE("open file done", K(filename), K(ret)); + return ret; +} + +void ObExternalStreamFileReader::close() +{ + if (data_access_driver_.is_opened()) { + data_access_driver_.close(); + + is_file_end_ = true; + file_offset_ = 0; + file_size_ = 0; + LOG_DEBUG("close file"); + } +} + +void ObExternalStreamFileReader::reset() +{ + close(); + if (OB_NOT_NULL(compressed_data_) && OB_NOT_NULL(allocator_)) { + allocator_->free(compressed_data_); + compressed_data_ = nullptr; + } + + if (OB_NOT_NULL(decompressor_)) { + ObDecompressor::destroy(decompressor_); + decompressor_ = nullptr; + } + + allocator_ = nullptr; +} + +bool ObExternalStreamFileReader::eof() +{ + return is_file_end_; +} + +int ObExternalStreamFileReader::read(char *buf, int64_t buf_len, int64_t &read_size) +{ + int ret = OB_SUCCESS; + read_size = 0; + + if (OB_ISNULL(buf) || buf_len <= 0) { + ret = OB_INVALID_ARGUMENT; + } else if (OB_ISNULL(decompressor_)) { + ret = read_from_driver(buf, buf_len, read_size); + is_file_end_ = file_offset_ >= file_size_; + LOG_DEBUG("read file", K(is_file_end_), K(file_offset_), K(file_size_), K(read_size)); + } else { + ret = read_decompress(buf, buf_len, read_size); + } + return ret; +} + +int ObExternalStreamFileReader::read_from_driver(char *buf, int64_t buf_len, int64_t &read_size) +{ + int ret = OB_SUCCESS; + read_size = 0; + + if (OB_ISNULL(buf) || buf_len <= 0) { + ret = OB_INVALID_ARGUMENT; + } else if(OB_FAIL(data_access_driver_.pread(buf, buf_len, file_offset_, read_size))) { + LOG_WARN("failed to read data from data access driver", K(ret), K(file_offset_), K(buf_len)); + } else { + file_offset_ += read_size; + } + return ret; +} + +int ObExternalStreamFileReader::read_decompress(char *buf, int64_t buf_len, int64_t &read_size) +{ + int ret = OB_SUCCESS; + read_size = 0; + + if (!data_access_driver_.is_opened()) { + ret = OB_NOT_INIT; + } else if (OB_ISNULL(buf) || buf_len <= 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KP(buf), K(buf_len)); + } else if (consumed_data_size_ >= compress_data_size_) { + if (file_offset_ < file_size_) { + ret = read_compressed_data(); + } else { + is_file_end_ = true; + } + } + + if (OB_SUCC(ret) && compress_data_size_ > consumed_data_size_) { + int64_t consumed_size = 0; + ret = decompressor_->decompress(compressed_data_ + consumed_data_size_, + compress_data_size_ - consumed_data_size_, + consumed_size, + buf, + buf_len, + read_size); + if (OB_FAIL(ret)) { + LOG_WARN("failed to decompress", K(ret)); + } else { + consumed_data_size_ += consumed_size; + uncompressed_size_ += read_size; + } + } + return ret; +} + +int ObExternalStreamFileReader::read_compressed_data() +{ + int ret = OB_SUCCESS; + char *read_buffer = compressed_data_; + if (!data_access_driver_.is_opened()) { + ret = OB_NOT_INIT; + } else if (OB_UNLIKELY(consumed_data_size_ < compress_data_size_)) { + // backup data + const int64_t last_data_size = compress_data_size_ - consumed_data_size_; + MEMMOVE(compressed_data_, compressed_data_ + consumed_data_size_, last_data_size); + read_buffer = compressed_data_ + last_data_size; + consumed_data_size_ = 0; + compress_data_size_ = last_data_size; + } else if (consumed_data_size_ == compress_data_size_) { + consumed_data_size_ = 0; + compress_data_size_ = 0; + } + + if (OB_SUCC(ret)) { + // read data from source reader + int64_t read_size = 0; + int64_t capacity = COMPRESSED_DATA_BUFFER_SIZE - compress_data_size_; + ret = read_from_driver(read_buffer, capacity, read_size); + if (OB_SUCC(ret)) { + compress_data_size_ += read_size; + } + } + return ret; +} + +int ObExternalStreamFileReader::create_decompressor(ObLoadCompressionFormat compression_format) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(allocator_)) { + ret = OB_NOT_INIT; + } else if (compression_format == ObLoadCompressionFormat::NONE) { + ObDecompressor::destroy(decompressor_); + decompressor_ = nullptr; + } else if (OB_NOT_NULL(decompressor_) && decompressor_->compression_format() == compression_format) { + // do nothing + } else { + if (OB_NOT_NULL(decompressor_)) { + ObDecompressor::destroy(decompressor_); + decompressor_ = nullptr; + } + + if (OB_FAIL(ObDecompressor::create(compression_format, *allocator_, decompressor_))) { + LOG_WARN("failed to create decompressor", K(ret)); + } else if (OB_ISNULL(compressed_data_) && + OB_ISNULL(compressed_data_ = (char *)allocator_->alloc(COMPRESSED_DATA_BUFFER_SIZE))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory", K(COMPRESSED_DATA_BUFFER_SIZE)); + } + } + return ret; +} int ObExternalTableAccessService::table_scan( ObVTableScanParam ¶m, @@ -453,7 +663,7 @@ int ObCSVTableRowIterator::expand_buf() if (nullptr != old_buf) { new_buf_len = state_.buf_len_ * 2; } else { - if (data_access_driver_.get_storage_type() != OB_STORAGE_FILE) { + if (file_reader_.get_storage_type() != OB_STORAGE_FILE) { //for better performance new_buf_len = OB_MALLOC_BIG_BLOCK_SIZE; } else { @@ -548,11 +758,12 @@ int ObCSVTableRowIterator::init(const storage::ObTableScanParam *scan_param) arena_alloc_.set_attr(lib::ObMemAttr(scan_param->tenant_id_, "CSVRowIter")); OZ (ObExternalTableRowIterator::init(scan_param)); OZ (parser_.init(scan_param->external_file_format_.csv_format_)); - OZ (data_access_driver_.init(scan_param_->external_file_location_, scan_param->external_file_access_info_)); + OZ (file_reader_.init(scan_param_->external_file_location_, scan_param->external_file_access_info_, + scan_param_->external_file_format_.compression_format_, malloc_alloc_)); OZ (expand_buf()); if (OB_SUCC(ret)) { - if (data_access_driver_.get_storage_type() == OB_STORAGE_FILE) { + if (file_reader_.get_storage_type() == OB_STORAGE_FILE) { if (OB_ISNULL(state_.ip_port_buf_ = static_cast(arena_alloc_.alloc(max_ipv6_port_length)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to alloc memory", K(ret)); @@ -653,11 +864,9 @@ int ObCSVTableRowIterator::open_next_file() { int ret = OB_SUCCESS; ObString location = scan_param_->external_file_location_; + int64_t file_size = 0; - if (data_access_driver_.is_opened()) { - data_access_driver_.close(); - } - + file_reader_.close(); do { ObString file_url; int64_t file_id = 0; @@ -665,6 +874,8 @@ int ObCSVTableRowIterator::open_next_file() int64_t start_line = 0; int64_t end_line = 0; int64_t task_idx = state_.file_idx_++; + + file_size = 0; url_.reuse(); ret = get_next_file_and_line_number(task_idx, file_url, file_id, part_id, start_line, end_line); if (OB_FAIL(ret)) { @@ -695,8 +906,9 @@ int ObCSVTableRowIterator::open_next_file() OZ (url_.append_fmt("%.*s%s%.*s", location.length(), location.ptr(), (location.empty() || location[location.length() - 1] == '/') ? "" : split_char, file_url.length(), file_url.ptr())); - OZ (data_access_driver_.get_file_size(url_.string(), state_.file_size_)); - if (OB_SUCC(ret) && data_access_driver_.get_storage_type() == OB_STORAGE_FILE) { + // skip empty file and non-exist file + OZ (file_reader_.get_data_access_driver().get_file_size(url_.string(), file_size)); + if (OB_SUCC(ret) && file_reader_.get_storage_type() == OB_STORAGE_FILE) { ObSqlString full_name; if (state_.ip_port_len_ == 0) { OZ (GCONF.self_addr_.addr_to_buffer(state_.ip_port_buf_, max_ipv6_port_length, state_.ip_port_len_)); @@ -708,10 +920,11 @@ int ObCSVTableRowIterator::open_next_file() } } LOG_DEBUG("try next file", K(ret), K(url_), K(file_url), K(state_)); - } while (OB_SUCC(ret) && 0 >= state_.file_size_); //skip empty file - OZ (data_access_driver_.open(url_.ptr()), url_); + } while (OB_SUCC(ret) && file_size <= 0); - LOG_DEBUG("open external file", K(ret), K(url_), K(state_.file_size_), K(location)); + OZ(file_reader_.open(url_.ptr())); + + LOG_DEBUG("open external file", K(ret), K(url_), K(location)); return ret; } @@ -719,15 +932,14 @@ int ObCSVTableRowIterator::open_next_file() int ObCSVTableRowIterator::load_next_buf() { int ret = OB_SUCCESS; + int64_t read_size = 0; do { char *next_load_pos = NULL; int64_t next_buf_len = 0; - if (state_.is_end_file_) { + if (file_reader_.eof()) { if (OB_FAIL(open_next_file())) { //do not print log } else { - state_.is_end_file_ = false; - state_.file_offset_ = 0; next_load_pos = state_.buf_; next_buf_len = state_.buf_len_; } @@ -747,17 +959,20 @@ int ObCSVTableRowIterator::load_next_buf() } if (OB_SUCC(ret)) { - int64_t read_size = 0; - OZ (data_access_driver_.pread(next_load_pos, next_buf_len, state_.file_offset_, read_size)); + // `read` may return read_size 0. + // If we read a compressed empty file, we need to read it twice + // to know that we have reached the end of the file. The first + // time we read the original file data and decompress it, we get + // 0 bytes, and the second time we read it to know that we have + // reached the end of the file. + OZ (file_reader_.read(next_load_pos, next_buf_len, read_size)); if (OB_SUCC(ret)) { - state_.file_offset_ += read_size; state_.pos_ = state_.buf_; state_.data_end_ = next_load_pos + read_size; - state_.is_end_file_ = (state_.file_offset_ >= state_.file_size_); } } - } while (false); + } while (OB_SUCC(ret) && read_size <= 0); return ret; } @@ -773,7 +988,7 @@ int ObCSVTableRowIterator::skip_lines() do { nrows = state_.skip_lines_; OZ (parser_.scan(state_.pos_, state_.data_end_, nrows, nullptr, nullptr, - temp_handle, error_msgs, state_.is_end_file_)); + temp_handle, error_msgs, file_reader_.eof())); error_msgs.reuse(); state_.skip_lines_ -= nrows; } while (OB_SUCC(ret) && state_.skip_lines_ > 0 && OB_SUCC(load_next_buf())); @@ -820,7 +1035,7 @@ int ObCSVTableRowIterator::get_next_row() for (int i = 0; OB_SUCC(ret) && i < file_column_exprs_.count(); ++i) { ObDatum &datum = file_column_exprs_.at(i)->locate_datum_for_write(eval_ctx_); if (file_column_exprs_.at(i)->type_ == T_PSEUDO_EXTERNAL_FILE_URL) { - if (csv_iter_->data_access_driver_.get_storage_type() == OB_STORAGE_FILE) { + if (csv_iter_->file_reader_.get_storage_type() == OB_STORAGE_FILE) { datum.set_string(csv_iter_->state_.file_with_url_.ptr(), csv_iter_->state_.file_with_url_.length()); } else { datum.set_string(csv_iter_->state_.cur_file_name_.ptr(), csv_iter_->state_.cur_file_name_.length()); @@ -865,11 +1080,10 @@ int ObCSVTableRowIterator::get_next_row() nrows = MIN(1, state_.line_count_limit_); if (OB_UNLIKELY(0 == nrows)) { // if line_count_limit = 0, get next file. - state_.is_end_file_ = true; } else { ret = parser_.scan(state_.pos_, state_.data_end_, nrows, state_.escape_buf_, state_.escape_buf_end_, - handle_one_line, error_msgs, state_.is_end_file_); + handle_one_line, error_msgs, file_reader_.eof()); if (OB_FAIL(ret)) { LOG_WARN("fail to scan csv", K(ret)); } else if (OB_UNLIKELY(error_msgs.count() > 0)) { @@ -950,7 +1164,7 @@ int ObCSVTableRowIterator::get_next_rows(int64_t &count, int64_t capacity) for (int i = 0; OB_SUCC(ret) && i < file_column_exprs_.count(); ++i) { ObDatum *datums = file_column_exprs_.at(i)->locate_batch_datums(eval_ctx_); if (file_column_exprs_.at(i)->type_ == T_PSEUDO_EXTERNAL_FILE_URL) { - if (csv_iter_->data_access_driver_.get_storage_type() == OB_STORAGE_FILE) { + if (csv_iter_->file_reader_.get_storage_type() == OB_STORAGE_FILE) { datums[returned_row_cnt_].set_string(csv_iter_->state_.file_with_url_.ptr(), csv_iter_->state_.file_with_url_.length()); } else { datums[returned_row_cnt_].set_string(csv_iter_->state_.cur_file_name_.ptr(), csv_iter_->state_.cur_file_name_.length()); @@ -995,11 +1209,10 @@ int ObCSVTableRowIterator::get_next_rows(int64_t &count, int64_t capacity) nrows = MIN(batch_size, state_.line_count_limit_); if (OB_UNLIKELY(0 == nrows)) { // if line_count_limit = 0, get next file. - state_.is_end_file_ = true; } else { ret = parser_.scan(state_.pos_, state_.data_end_, nrows, state_.escape_buf_, state_.escape_buf_end_, handle_one_line, - error_msgs, state_.is_end_file_); + error_msgs, file_reader_.eof()); if (OB_FAIL(ret)) { LOG_WARN("fail to scan csv", K(ret)); } else if (OB_UNLIKELY(error_msgs.count() > 0)) { diff --git a/src/sql/engine/table/ob_external_table_access_service.h b/src/sql/engine/table/ob_external_table_access_service.h index bb5cd1ad4..33dacb849 100644 --- a/src/sql/engine/table/ob_external_table_access_service.h +++ b/src/sql/engine/table/ob_external_table_access_service.h @@ -30,6 +30,7 @@ namespace common namespace sql { class ObExprRegexpSessionVariables; +class ObDecompressor; class ObExternalDataAccessDriver { @@ -62,6 +63,63 @@ private: ObIOFd fd_; }; +class ObExternalStreamFileReader final +{ +public: + // ObExternalStreamFileReader(); + ~ObExternalStreamFileReader(); + void reset(); + + int init(const common::ObString &location, + const ObString &access_info, + ObLoadCompressionFormat compression_format, + ObIAllocator &allocator); + + int open(const ObString &filename); + void close(); + + /** + * read data into buffer. decompress source data if need + */ + int read(char *buf, int64_t buf_len, int64_t &read_size); + bool eof(); + + common::ObStorageType get_storage_type() { return data_access_driver_.get_storage_type(); } + + ObExternalDataAccessDriver &get_data_access_driver() { return data_access_driver_; } + +private: + int read_from_driver(char *buf, int64_t buf_len, int64_t &read_size); + int read_decompress(char *buf, int64_t buf_len, int64_t &read_size); + int read_compressed_data(); // read data from driver into compressed buffer + + /** + * create the decompressor if need + * + * It's no need to create new decompressor if the compression_format is the seem as decompressor's. + */ + int create_decompressor(ObLoadCompressionFormat compression_format); + +private: + ObExternalDataAccessDriver data_access_driver_; + bool is_file_end_ = true; + int64_t file_offset_ = 0; + int64_t file_size_ = 0; + + ObDecompressor *decompressor_ = nullptr; + char * compressed_data_ = nullptr; /// compressed data buffer + int64_t compress_data_size_ = 0; /// the valid data size in compressed data buffer + int64_t consumed_data_size_ = 0; /// handled buffer size in the compressed data buffer + int64_t uncompressed_size_ = 0; /// decompressed size from compressed data + + ObIAllocator *allocator_ = nullptr; + + /// the compression format specified in `create external table` statement + ObLoadCompressionFormat compression_format_ = ObLoadCompressionFormat::NONE; + + static const char * MEMORY_LABEL; + static const int64_t COMPRESSED_DATA_BUFFER_SIZE; +}; class ObExternalTableRowIterator : public common::ObNewRowIterator { public: @@ -109,7 +167,7 @@ public: StateValues() : buf_(nullptr), buf_len_(OB_MALLOC_NORMAL_BLOCK_SIZE), pos_(nullptr), data_end_(nullptr), escape_buf_(nullptr), escape_buf_end_(nullptr), - is_end_file_(true), file_idx_(0), file_offset_(0), file_size_(0), skip_lines_(0), + file_idx_(0), skip_lines_(0), cur_file_id_(MIN_EXTERNAL_TABLE_FILE_ID), cur_line_number_(MIN_EXTERNAL_TABLE_LINE_NUMBER), line_count_limit_(INT64_MAX), part_id_(0), part_list_val_(), ip_port_buf_(NULL), ip_port_len_(0), file_with_url_() {} char *buf_; @@ -118,10 +176,7 @@ public: const char *data_end_; char *escape_buf_; char *escape_buf_end_; - bool is_end_file_; int64_t file_idx_; - int64_t file_offset_; - int64_t file_size_; int64_t skip_lines_; common::ObString cur_file_name_; int64_t cur_file_id_; @@ -135,10 +190,7 @@ public: void reuse() { pos_ = buf_; data_end_ = buf_; - is_end_file_ = true; file_idx_ = 0; - file_offset_ = 0; - file_size_ = 0; skip_lines_ = 0; cur_file_name_.reset(); cur_file_id_ = MIN_EXTERNAL_TABLE_FILE_ID; @@ -149,8 +201,8 @@ public: ip_port_len_ = 0; file_with_url_.reset(); } - TO_STRING_KV(KP(buf_), K(buf_len_), KP(pos_), KP(data_end_), K(is_end_file_), K(file_idx_), - K(file_offset_), K(file_size_), K(skip_lines_), K(line_count_limit_), + TO_STRING_KV(KP(buf_), K(buf_len_), KP(pos_), KP(data_end_), K(file_idx_), + K(skip_lines_), K(line_count_limit_), K(cur_file_name_), K(cur_file_id_), K(cur_line_number_), K(line_count_limit_), K_(part_id), K_(ip_port_len), K_(file_with_url)); }; @@ -186,7 +238,7 @@ private: common::ObMalloc malloc_alloc_; //for internal data buffers common::ObArenaAllocator arena_alloc_; ObCSVGeneralParser parser_; - ObExternalDataAccessDriver data_access_driver_; + ObExternalStreamFileReader file_reader_; ObSqlString url_; ObExpr *file_name_expr_; }; diff --git a/src/sql/parser/sql_parser_mysql_mode.y b/src/sql/parser/sql_parser_mysql_mode.y index 058382a46..62cac381f 100644 --- a/src/sql/parser/sql_parser_mysql_mode.y +++ b/src/sql/parser/sql_parser_mysql_mode.y @@ -8233,6 +8233,10 @@ TYPE COMP_EQ STRING_VALUE { malloc_non_terminal_node($$, result->malloc_pool_, T_EMPTY_FIELD_AS_NULL, 1, $3); } +| COMPRESSION COMP_EQ compression_name +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_COMPRESSION, 1, $3); +} ; /***************************************************************************** diff --git a/src/sql/resolver/cmd/ob_load_data_stmt.h b/src/sql/resolver/cmd/ob_load_data_stmt.h index 3c7073f6c..1158d94ef 100644 --- a/src/sql/resolver/cmd/ob_load_data_stmt.h +++ b/src/sql/resolver/cmd/ob_load_data_stmt.h @@ -37,14 +37,6 @@ enum class ObLoadFileLocation { OSS, }; -enum class ObLoadCompressionFormat -{ - NONE, - GZIP, - DEFLATE, - ZSTD, -}; - class ObLoadFileIterator { public: diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.cpp b/src/sql/resolver/ddl/ob_ddl_resolver.cpp index e1c633e22..9258e2d12 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.cpp +++ b/src/sql/resolver/ddl/ob_ddl_resolver.cpp @@ -2826,11 +2826,24 @@ int ObDDLResolver::resolve_file_format(const ParseNode *node, ObExternalFileForm format.csv_format_.empty_field_as_null_ = node->children_[0]->value_; break; } + case T_COMPRESSION: { + ObString string_v = ObString(node->children_[0]->str_len_, node->children_[0]->str_value_).trim(); + ret = compression_format_from_string(string_v, format.compression_format_); + break; + } default: { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid file format option", K(ret), K(node->type_)); } } + + if (OB_SUCC(ret) + && format.format_type_ == ObExternalFileFormat::PARQUET_FORMAT + && format.compression_format_ != ObLoadCompressionFormat::NONE) { + LOG_WARN("parquet file doesn't support compression", K(format.compression_format_)); + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "parquet file with compression"); + } } return ret; } From 2e2c0d5c8611eb224ce6991537e275f31de0c118 Mon Sep 17 00:00:00 2001 From: obdev Date: Thu, 22 Aug 2024 05:55:27 +0000 Subject: [PATCH 170/249] Configuration item placeholder --- src/share/parameter/ob_parameter_seed.ipp | 5 ++++- .../r/mysql/all_virtual_sys_parameter_stat.result | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/share/parameter/ob_parameter_seed.ipp b/src/share/parameter/ob_parameter_seed.ipp index 9b52e13da..5b5cd45d2 100644 --- a/src/share/parameter/ob_parameter_seed.ipp +++ b/src/share/parameter/ob_parameter_seed.ipp @@ -2041,4 +2041,7 @@ DEF_STR_WITH_CHECKER(sql_plan_management_mode, OB_TENANT_PARAMETER, "Disable", ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); DEF_BOOL(_enable_check_trigger_const_variables_assign, OB_TENANT_PARAMETER, "True", "Used to control whether an error is reported when assigning a value to a const variable in a trigger under an Oracle tenant", - ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); \ No newline at end of file + ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +DEF_BOOL(_enable_unit_gc_wait, OB_CLUSTER_PARAMETER, "True", + "Used to control enable or disable the unit smooth gc feature, enabled by default.", + ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); \ No newline at end of file diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result index a53915b3f..d09558c3c 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result @@ -340,6 +340,7 @@ _enable_tenant_sql_net_thread _enable_trace_session_leak _enable_trace_tablet_leak _enable_transaction_internal_routing +_enable_unit_gc_wait _enable_values_table_folding _enable_var_assign_use_das _enable_wait_remote_lock From 83c293f357e86cf519ed1f585f3c5199804f94c9 Mon Sep 17 00:00:00 2001 From: leftgeek <1094669802@qq.com> Date: Thu, 22 Aug 2024 06:01:48 +0000 Subject: [PATCH 171/249] set default table mode to queuing for mlog --- src/rootserver/ob_mlog_builder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rootserver/ob_mlog_builder.cpp b/src/rootserver/ob_mlog_builder.cpp index 849e7eb5e..e6b552a81 100644 --- a/src/rootserver/ob_mlog_builder.cpp +++ b/src/rootserver/ob_mlog_builder.cpp @@ -640,7 +640,7 @@ int ObMLogBuilder::set_basic_infos( } else if (OB_FAIL(mlog_schema.set_table_name(mlog_table_name))) { LOG_WARN("failed to set table name", KR(ret), K(mlog_table_name)); } else { - mlog_schema.set_table_mode(base_table_schema.get_table_mode_flag()); + mlog_schema.set_table_mode_flag(ObTableModeFlag::TABLE_MODE_QUEUING_EXTREME); mlog_schema.set_table_type(MATERIALIZED_VIEW_LOG); mlog_schema.set_index_status(ObIndexStatus::INDEX_STATUS_UNAVAILABLE); mlog_schema.set_duplicate_scope(base_table_schema.get_duplicate_scope()); From cd124403e2cd63fa304ef7fc22a2145710bddbc2 Mon Sep 17 00:00:00 2001 From: hwx65 <1780011298@qq.com> Date: Thu, 22 Aug 2024 08:39:32 +0000 Subject: [PATCH 172/249] Fix regexp_replace core caused by incorrect template parameter passing --- src/sql/engine/expr/ob_expr_regexp_replace.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sql/engine/expr/ob_expr_regexp_replace.cpp b/src/sql/engine/expr/ob_expr_regexp_replace.cpp index a7d01e18e..4af87b97b 100644 --- a/src/sql/engine/expr/ob_expr_regexp_replace.cpp +++ b/src/sql/engine/expr/ob_expr_regexp_replace.cpp @@ -525,7 +525,7 @@ int ObExprRegexpReplace::vector_regexp_replace(VECTOR_EVAL_FUNC_ARG_DECL) { res_vec->set_null(i); } else { // if text is lob type, res_replace only get locator - ret = vector_regexp_replace_convert( + ret = vector_regexp_replace_convert( VECTOR_EVAL_FUNC_ARG_LIST, text_vec->get_string(i), true, expr.args_[0]->datum_meta_.cs_type_, out_alloc, tmp_alloc, i); } @@ -618,7 +618,7 @@ int ObExprRegexpReplace::vector_regexp_replace(VECTOR_EVAL_FUNC_ARG_DECL) { res_vec->set_null(i); } else { // if text is lob type, res_replace only get locator; - ret = vector_regexp_replace_convert( + ret = vector_regexp_replace_convert( VECTOR_EVAL_FUNC_ARG_LIST, text_vec->get_string(i), true, expr.args_[0]->datum_meta_.cs_type_, out_alloc, tmp_alloc, i); } @@ -661,7 +661,7 @@ int ObExprRegexpReplace::vector_regexp_replace(VECTOR_EVAL_FUNC_ARG_DECL) { } else if (res_replace.empty() && lib::is_oracle_mode()) { res_vec->set_null(i); } else { - ret = vector_regexp_replace_convert(VECTOR_EVAL_FUNC_ARG_LIST, + ret = vector_regexp_replace_convert(VECTOR_EVAL_FUNC_ARG_LIST, res_replace, is_no_pattern_to_replace, res_coll_type, out_alloc, tmp_alloc, i); } } From ba068fff43ac7a83c3d18182f4200898c728b8a4 Mon Sep 17 00:00:00 2001 From: obdev Date: Thu, 22 Aug 2024 08:51:22 +0000 Subject: [PATCH 173/249] fix mysqltest use db failed && modify dataversion for start_servers --- src/rootserver/ob_server_zone_op_service.cpp | 4 +++- src/share/ob_all_server_tracer.cpp | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/rootserver/ob_server_zone_op_service.cpp b/src/rootserver/ob_server_zone_op_service.cpp index 76b02a3e5..1d472a83f 100644 --- a/src/rootserver/ob_server_zone_op_service.cpp +++ b/src/rootserver/ob_server_zone_op_service.cpp @@ -314,7 +314,9 @@ int ObServerZoneOpService::start_servers( if (OB_FAIL(ObServerTableOperator::get(*GCTX.sql_proxy_, server, server_info))) { // make sure the server is in whitelist, then send rpc LOG_WARN("fail to get server_info", KR(ret), K(server)); - } else if (sys_tenant_data_version >= DATA_VERSION_4_3_2_0) { + } else if ((sys_tenant_data_version >= MOCK_DATA_VERSION_4_2_5_0 + && sys_tenant_data_version < DATA_VERSION_4_3_0_0) + || sys_tenant_data_version >= DATA_VERSION_4_3_2_0) { int64_t timeout = ctx.get_timeout(); const int64_t ERR_MSG_BUF_LEN = OB_MAX_SERVER_ADDR_SIZE + 150; char disk_error_server_err_msg[ERR_MSG_BUF_LEN] = ""; diff --git a/src/share/ob_all_server_tracer.cpp b/src/share/ob_all_server_tracer.cpp index a4d625d99..fde89defc 100644 --- a/src/share/ob_all_server_tracer.cpp +++ b/src/share/ob_all_server_tracer.cpp @@ -429,8 +429,7 @@ int ObServerTraceMap::get_alive_and_not_stopped_servers(const ObZone &zone, ObIA const ObAddr &server = server_info.get_server(); if ((server_info.get_zone() == zone || zone.is_empty()) && !server_info.is_stopped() - && server_info.is_alive() - && server_info.in_service()) { + && server_info.is_alive()) { if (OB_FAIL(server_list.push_back(server))) { LOG_WARN("fail to push an element into server_list", KR(ret), K(server)); } From 11d47a78ced5a4a5f12fe9a7876665e7aa446455 Mon Sep 17 00:00:00 2001 From: JiahuaChen Date: Thu, 22 Aug 2024 09:10:56 +0000 Subject: [PATCH 174/249] Fix root_macro_seq compat --- mittest/mtlenv/storage/test_ls_tablet_service.cpp | 4 ++++ src/storage/blocksstable/ob_sstable_meta.cpp | 13 ++++++++----- src/storage/blocksstable/ob_sstable_meta.h | 2 +- unittest/storage/blocksstable/test_sstable_meta.cpp | 9 +++++---- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/mittest/mtlenv/storage/test_ls_tablet_service.cpp b/mittest/mtlenv/storage/test_ls_tablet_service.cpp index 6bafbe2fc..a8130daba 100644 --- a/mittest/mtlenv/storage/test_ls_tablet_service.cpp +++ b/mittest/mtlenv/storage/test_ls_tablet_service.cpp @@ -1135,6 +1135,10 @@ TEST_F(TestLSTabletService, test_empty_shell_mds_compat) ASSERT_TRUE(nullptr == upgrade_tablet.mds_data_); ASSERT_TRUE(ObTabletStatus::Status::DELETED == upgrade_tablet.tablet_meta_.last_persisted_committed_tablet_status_.tablet_status_); + // release tmp memory and tablet + empty_shell_tablet.mds_data_->~ObTabletMdsData(); + compat_allocator.free(empty_shell_tablet.mds_data_); + empty_shell_tablet.mds_data_ = nullptr; ret = ls_tablet_service_->do_remove_tablet(ls_id_, tablet_id); ASSERT_EQ(OB_SUCCESS, ret); } diff --git a/src/storage/blocksstable/ob_sstable_meta.cpp b/src/storage/blocksstable/ob_sstable_meta.cpp index 565aae928..3fbeef57d 100644 --- a/src/storage/blocksstable/ob_sstable_meta.cpp +++ b/src/storage/blocksstable/ob_sstable_meta.cpp @@ -64,9 +64,9 @@ ObSSTableBasicMeta::ObSSTableBasicMeta() master_key_id_(0), sstable_logic_seq_(0), latest_row_store_type_(ObRowStoreType::MAX_ROW_STORE), - root_macro_seq_(0), table_backup_flag_(), - table_shared_flag_() + table_shared_flag_(), + root_macro_seq_(0) { MEMSET(encrypt_key_, 0, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); } @@ -248,7 +248,8 @@ DEFINE_SERIALIZE(ObSSTableBasicMeta) sstable_logic_seq_, latest_row_store_type_, table_backup_flag_, - table_shared_flag_); + table_shared_flag_, + root_macro_seq_); if (OB_FAIL(ret)) { } else if (OB_UNLIKELY(length_ != pos - start_pos)) { ret = OB_ERR_UNEXPECTED; @@ -330,7 +331,8 @@ int ObSSTableBasicMeta::decode_for_compat(const char *buf, const int64_t data_le sstable_logic_seq_, latest_row_store_type_, table_backup_flag_, - table_shared_flag_); + table_shared_flag_, + root_macro_seq_); return ret; } @@ -373,7 +375,8 @@ DEFINE_GET_SERIALIZE_SIZE(ObSSTableBasicMeta) sstable_logic_seq_, latest_row_store_type_, table_backup_flag_, - table_shared_flag_); + table_shared_flag_, + root_macro_seq_); return len; } diff --git a/src/storage/blocksstable/ob_sstable_meta.h b/src/storage/blocksstable/ob_sstable_meta.h index 1562a66ae..70092fc41 100644 --- a/src/storage/blocksstable/ob_sstable_meta.h +++ b/src/storage/blocksstable/ob_sstable_meta.h @@ -187,10 +187,10 @@ public: int16_t sstable_logic_seq_; common::ObRowStoreType latest_row_store_type_; char encrypt_key_[share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH]; - int64_t root_macro_seq_; // placeholder, will be used after palf branch merged storage::ObTableBackupFlag table_backup_flag_; //cannot add backup flag to ObSSTableMetaChecker //quick restore with rebuild replace major will has same key sstable storage::ObTableSharedFlag table_shared_flag_; + int64_t root_macro_seq_; // placeholder, will be used after palf branch merged //Add new variable need consider ObSSTableMetaChecker }; diff --git a/unittest/storage/blocksstable/test_sstable_meta.cpp b/unittest/storage/blocksstable/test_sstable_meta.cpp index 942570d15..e0a9abb41 100644 --- a/unittest/storage/blocksstable/test_sstable_meta.cpp +++ b/unittest/storage/blocksstable/test_sstable_meta.cpp @@ -630,6 +630,7 @@ TEST_F(TestSSTableMeta, test_sstable_meta_deep_copy) const int64_t buf_size = 8 << 10; //8K int64_t pos = 0; char *flat_buf_1 = (char*)ob_malloc(buf_size, ObMemAttr()); + MEMSET(flat_buf_1, 0, buf_size); int64_t deep_copy_size = src_meta.get_deep_copy_size(); ObSSTableMeta *flat_meta_1; ret = src_meta.deep_copy(flat_buf_1, deep_copy_size, pos, flat_meta_1); @@ -637,11 +638,10 @@ TEST_F(TestSSTableMeta, test_sstable_meta_deep_copy) ASSERT_EQ(deep_copy_size, pos); OB_LOG(INFO, "cooper", K(src_meta), K(sizeof(ObSSTableMeta)), K(deep_copy_size)); OB_LOG(INFO, "cooper", K(*flat_meta_1)); - // can't use MEMCMP between dynamic memory and flat memory, because one is stack, the other is heap ASSERT_EQ(src_meta.basic_meta_, flat_meta_1->basic_meta_); - // ASSERT_EQ(0, MEMCMP((char*)&src_meta.data_root_info_, (char*)&flat_meta_1->data_root_info_, sizeof(src_meta.data_root_info_))); - // ASSERT_EQ(0, MEMCMP((char*)&src_meta.macro_info_, (char*)&dst_meta->macro_info_, sizeof(src_meta.macro_info_))); - // ASSERT_EQ(0, MEMCMP((char*)&src_meta.cg_sstables_, (char*)&dst_meta->cg_sstables_, sizeof(src_meta.cg_sstables_))); + ASSERT_EQ(0, MEMCMP((char*)&src_meta.data_root_info_, (char*)&flat_meta_1->data_root_info_, sizeof(src_meta.data_root_info_))); + ASSERT_EQ(0, MEMCMP((char*)&src_meta.macro_info_, (char*)&flat_meta_1->macro_info_, sizeof(src_meta.macro_info_))); + // ASSERT_EQ(0, MEMCMP((char*)&src_meta.cg_sstables_, (char*)&flat_meta_1->cg_sstables_, sizeof(src_meta.cg_sstables_))); ASSERT_EQ(0, MEMCMP(src_meta.column_checksums_, flat_meta_1->column_checksums_, src_meta.column_checksum_count_ * sizeof(int64_t))); ASSERT_EQ(src_meta.tx_ctx_.len_, flat_meta_1->tx_ctx_.len_); ASSERT_EQ(src_meta.tx_ctx_.count_, flat_meta_1->tx_ctx_.count_); @@ -650,6 +650,7 @@ TEST_F(TestSSTableMeta, test_sstable_meta_deep_copy) // test deep copy from flat memory meta to flat memory meta pos = 0; char *flat_buf_2 = (char*)ob_malloc_align(4<<10, buf_size, ObMemAttr()); + MEMSET(flat_buf_2, 0, buf_size); deep_copy_size = flat_meta_1->get_deep_copy_size(); ObSSTableMeta *flat_meta_2; ret = flat_meta_1->deep_copy(flat_buf_2, deep_copy_size, pos, flat_meta_2); From 81e39dc173e0712d5746f6d4f17a46d20a35965f Mon Sep 17 00:00:00 2001 From: Tsunaou <895254752@qq.com> Date: Thu, 22 Aug 2024 11:26:15 +0000 Subject: [PATCH 175/249] [FEAT MERGE] Column Store Replica Co-authored-by: LeonChaoHi <15201161716@163.com> Co-authored-by: zzg19950727 <1071026277@qq.com> --- deps/oblib/src/common/ob_member.cpp | 103 +-- deps/oblib/src/common/ob_member.h | 68 +- deps/oblib/src/lib/ob_define.h | 105 +-- deps/oblib/src/lib/utility/utility.cpp | 27 - deps/oblib/src/lib/utility/utility.h | 2 - .../storage/test_ls_migration_param.cpp | 4 +- .../test_ls_tablet_info_writer_and_reader.cpp | 2 +- .../test_table_scan_pure_data_table.cpp | 2 + .../storage/test_tenant_meta_mem_mgr.cpp | 10 +- mittest/mtlenv/test_tx_data_table.cpp | 1 + mittest/simple_server/test_ls_recover.cpp | 6 + src/logservice/palf/palf_handle_impl.cpp | 12 +- src/observer/ob_service.cpp | 119 +-- src/observer/ob_service.h | 5 - src/observer/table/ob_table_cg_service.cpp | 22 +- src/observer/table/ob_table_cg_service.h | 6 +- src/observer/table/ob_table_context.cpp | 14 + src/observer/table/ob_table_context.h | 3 + .../virtual_table/ob_all_virtual_ls_info.cpp | 11 +- .../virtual_table/ob_all_virtual_ls_info.h | 1 - .../ob_all_virtual_proxy_schema.cpp | 52 +- .../ob_all_virtual_proxy_schema.h | 4 + .../freeze/ob_checksum_validator.cpp | 40 +- src/rootserver/freeze/ob_checksum_validator.h | 7 +- .../ob_major_merge_progress_checker.cpp | 14 +- .../freeze/ob_major_merge_progress_checker.h | 2 +- src/rootserver/ob_admin_drtask_util.cpp | 77 +- src/rootserver/ob_admin_drtask_util.h | 2 + src/rootserver/ob_balance_info.h | 13 - src/rootserver/ob_ddl_service.cpp | 25 +- src/rootserver/ob_ddl_service.h | 2 +- src/rootserver/ob_disaster_recovery_task.cpp | 109 +-- .../ob_disaster_recovery_worker.cpp | 279 +++++-- src/rootserver/ob_disaster_recovery_worker.h | 87 +-- src/rootserver/ob_locality_util.cpp | 112 +-- src/rootserver/ob_locality_util.h | 35 +- src/rootserver/ob_replica_addr.h | 2 +- src/rootserver/ob_root_service.cpp | 37 +- src/rootserver/ob_root_utils.cpp | 196 +---- src/rootserver/ob_root_utils.h | 30 - src/rootserver/ob_unit_manager.cpp | 2 +- .../ob_all_virtual_ls_replica_task_plan.cpp | 4 +- src/share/CMakeLists.txt | 1 - .../ob_feedback_partition_struct.h | 4 +- .../ob_compaction_locality_cache.cpp | 234 +++++- .../compaction/ob_compaction_locality_cache.h | 42 ++ .../ob_inner_table_schema.21151_21200.cpp | 6 +- .../ob_inner_table_schema.21301_21350.cpp | 2 +- .../ob_inner_table_schema.25151_25200.cpp | 2 +- .../inner_table/ob_inner_table_schema_def.py | 27 +- .../location_cache/ob_location_struct.cpp | 8 +- src/share/location_cache/ob_location_struct.h | 2 +- src/share/ls/ob_ls_creator.cpp | 40 +- src/share/ls/ob_ls_creator.h | 2 +- src/share/ls/ob_ls_info.cpp | 18 +- src/share/ls/ob_ls_info.h | 1 + src/share/ob_debug_sync_point.h | 2 + src/share/ob_locality_parser.cpp | 95 --- src/share/ob_locality_parser.h | 49 -- src/share/ob_replica_info.cpp | 180 ++--- src/share/ob_replica_info.h | 123 +--- src/share/ob_rpc_struct.cpp | 108 +-- src/share/ob_rpc_struct.h | 69 +- src/share/ob_share_util.cpp | 183 +++-- src/share/ob_share_util.h | 25 +- .../ob_tablet_replica_checksum_operator.cpp | 54 ++ .../ob_tablet_replica_checksum_operator.h | 13 + src/share/scheduler/ob_dag_scheduler_config.h | 2 + src/share/scheduler/ob_tenant_dag_scheduler.h | 2 + src/share/schema/ob_schema_struct.cpp | 275 +------ src/share/schema/ob_schema_struct.h | 26 - src/share/schema/ob_table_dml_param.cpp | 2 +- src/share/schema/ob_table_param.cpp | 34 +- src/share/schema/ob_table_param.h | 6 +- src/share/schema/ob_table_schema.cpp | 42 +- src/share/schema/ob_table_schema.h | 8 +- .../ob_system_variable_factory.cpp | 1 + .../ob_system_variable_init.cpp | 2 +- .../ob_system_variable_init.json | 3 +- src/share/table/ob_table.h | 2 +- src/sql/code_generator/ob_tsc_cg_service.cpp | 16 +- src/sql/das/ob_das_location_router.cpp | 15 +- src/sql/das/ob_das_location_router.h | 4 +- .../optimizer/ob_intersect_route_policy.cpp | 2 +- src/sql/optimizer/ob_join_order.cpp | 3 + src/sql/optimizer/ob_log_plan.cpp | 17 +- src/sql/optimizer/ob_log_plan.h | 1 + src/sql/optimizer/ob_optimizer.cpp | 20 + src/sql/optimizer/ob_optimizer.h | 1 + src/sql/optimizer/ob_optimizer_context.h | 6 +- src/sql/optimizer/ob_replica_compare.cpp | 6 +- src/sql/optimizer/ob_replica_compare.h | 1 + src/sql/optimizer/ob_route_policy.cpp | 15 + src/sql/optimizer/ob_route_policy.h | 5 +- src/sql/optimizer/ob_table_location.cpp | 13 + .../resolver/cmd/ob_alter_system_resolver.cpp | 45 +- .../resolver/cmd/ob_alter_system_resolver.h | 1 + src/sql/resolver/cmd/ob_resource_resolver.h | 4 + src/storage/CMakeLists.txt | 2 + src/storage/access/ob_table_access_param.cpp | 5 +- src/storage/access/ob_table_access_param.h | 1 + .../blocksstable/ob_data_store_desc.cpp | 11 +- src/storage/blocksstable/ob_data_store_desc.h | 13 +- src/storage/blocksstable/ob_macro_block.cpp | 26 +- .../column_store/ob_cg_iter_param_pool.cpp | 1 + src/storage/column_store/ob_co_merge_ctx.cpp | 61 +- src/storage/column_store/ob_co_merge_ctx.h | 1 + src/storage/column_store/ob_co_merge_dag.cpp | 7 +- .../column_store/ob_column_oriented_sstable.h | 12 +- .../ob_column_store_replica_util.cpp | 233 ++++++ .../ob_column_store_replica_util.h | 85 +++ .../column_store/ob_column_store_util.h | 1 + .../compaction/ob_basic_tablet_merge_ctx.cpp | 47 +- .../compaction/ob_basic_tablet_merge_ctx.h | 9 +- src/storage/compaction/ob_compaction_util.cpp | 1 + src/storage/compaction/ob_compaction_util.h | 9 +- .../compaction/ob_medium_compaction_func.cpp | 205 +++--- .../compaction/ob_medium_compaction_func.h | 17 +- .../compaction/ob_partition_merge_policy.cpp | 41 +- .../compaction/ob_partition_merge_policy.h | 7 +- .../compaction/ob_tenant_medium_checker.cpp | 30 +- .../compaction/ob_tenant_tablet_scheduler.cpp | 29 +- .../compaction/ob_tenant_tablet_scheduler.h | 8 +- src/storage/ddl/ob_complement_data_task.cpp | 2 +- src/storage/ddl/ob_ddl_clog.cpp | 7 +- src/storage/ddl/ob_ddl_clog.h | 4 +- src/storage/ddl/ob_ddl_merge_task.cpp | 7 + src/storage/ddl/ob_ddl_redo_log_writer.cpp | 18 +- src/storage/ddl/ob_ddl_redo_log_writer.h | 6 +- src/storage/ddl/ob_ddl_replay_executor.cpp | 93 ++- src/storage/ddl/ob_ddl_replay_executor.h | 11 +- src/storage/ddl/ob_ddl_struct.cpp | 15 + src/storage/ddl/ob_ddl_struct.h | 12 + .../ddl/ob_direct_insert_sstable_ctx_new.cpp | 82 ++- .../ddl/ob_direct_insert_sstable_ctx_new.h | 26 +- src/storage/ddl/ob_direct_load_struct.cpp | 223 +++++- src/storage/ddl/ob_direct_load_struct.h | 71 +- .../ob_cs_replica_migration.cpp | 693 ++++++++++++++++++ .../ob_cs_replica_migration.h | 160 ++++ .../ob_ls_complete_migration.cpp | 4 +- .../high_availability/ob_ls_migration.cpp | 274 ++++++- .../high_availability/ob_ls_migration.h | 21 +- .../ob_ls_remove_member_handler.cpp | 5 +- .../high_availability/ob_rebuild_service.cpp | 6 +- .../high_availability/ob_storage_ha_dag.cpp | 91 ++- .../high_availability/ob_storage_ha_dag.h | 27 +- .../ob_storage_ha_src_provider.cpp | 34 +- .../high_availability/ob_storage_ha_utils.cpp | 27 +- .../ob_tablet_group_restore.cpp | 4 +- src/storage/ls/ob_ls.cpp | 42 +- src/storage/ls/ob_ls.h | 6 + src/storage/ls/ob_ls_meta.cpp | 14 +- src/storage/ls/ob_ls_meta.h | 9 +- src/storage/ls/ob_ls_tablet_service.cpp | 9 +- src/storage/ob_i_table.h | 28 + src/storage/ob_storage_schema.cpp | 157 +++- src/storage/ob_storage_schema.h | 26 +- src/storage/ob_storage_struct.cpp | 6 +- src/storage/ob_storage_struct.h | 4 +- src/storage/ob_tenant_tablet_stat_mgr.cpp | 2 +- src/storage/restore/ob_ls_restore_handler.cpp | 36 +- src/storage/tablet/ob_mds_schema_helper.cpp | 8 + src/storage/tablet/ob_mds_schema_helper.h | 2 + src/storage/tablet/ob_table_store_util.cpp | 40 + src/storage/tablet/ob_table_store_util.h | 4 + src/storage/tablet/ob_tablet.cpp | 151 +++- src/storage/tablet/ob_tablet.h | 19 +- src/storage/tablet/ob_tablet_meta.cpp | 26 +- src/storage/tablet/ob_tablet_meta.h | 11 +- src/storage/tablet/ob_tablet_table_store.cpp | 36 +- src/storage/tablet/ob_tablet_table_store.h | 8 +- src/storage/tx_storage/ob_ls_service.cpp | 120 ++- src/storage/tx_storage/ob_ls_service.h | 12 + .../r/mysql/desc_sys_views_in_sys.result | 2 +- .../observer/table/test_create_executor.cpp | 3 +- unittest/storage/migration/test_migration.h | 1 + unittest/storage/test_compaction_policy.cpp | 4 +- unittest/storage/test_tablet_helper.h | 4 +- unittest/storage/test_tablet_pointer_map.cpp | 1 + 179 files changed, 4924 insertions(+), 2176 deletions(-) delete mode 100644 src/share/ob_locality_parser.cpp delete mode 100644 src/share/ob_locality_parser.h create mode 100644 src/storage/column_store/ob_column_store_replica_util.cpp create mode 100644 src/storage/column_store/ob_column_store_replica_util.h create mode 100644 src/storage/high_availability/ob_cs_replica_migration.cpp create mode 100644 src/storage/high_availability/ob_cs_replica_migration.h diff --git a/deps/oblib/src/common/ob_member.cpp b/deps/oblib/src/common/ob_member.cpp index 39499ce39..3dea591bf 100644 --- a/deps/oblib/src/common/ob_member.cpp +++ b/deps/oblib/src/common/ob_member.cpp @@ -86,6 +86,21 @@ void ObMember::reset_migrating() flag_ &= ~(1UL << MIGRATING_FLAG_BIT); } +bool ObMember::is_columnstore() const +{ + return (flag_ >> COLUMNSTORE_FLAG_BIT) & 1U; +} + +void ObMember::set_columnstore() +{ + flag_ |= (1UL << COLUMNSTORE_FLAG_BIT); +} + +void ObMember::reset_columnstore() +{ + flag_ &= ~(1UL << COLUMNSTORE_FLAG_BIT); +} + OB_SERIALIZE_MEMBER(ObMember, server_, timestamp_, flag_); bool ObReplicaMember::is_readonly_replica() const @@ -93,19 +108,63 @@ bool ObReplicaMember::is_readonly_replica() const return REPLICA_TYPE_READONLY == replica_type_; } +int ObReplicaMember::init( + const common::ObAddr &server, + const int64_t timestamp, + const common::ObReplicaType replica_type) +{ + int ret = OB_SUCCESS; + reset(); + if (OB_UNLIKELY(!server.is_valid() + || !ObReplicaTypeCheck::is_replica_type_valid(replica_type))) { + ret = OB_INVALID_ARGUMENT; + COMMON_LOG(WARN, "invalid argument", K(ret), K(server), K(replica_type)); + } else { + server_ = server; + timestamp_ = timestamp; + replica_type_ = replica_type; + if (REPLICA_TYPE_COLUMNSTORE == replica_type) { + ObMember::set_columnstore(); + } + } + return ret; +} + +int ObReplicaMember::init( + const ObMember &member, + const common::ObReplicaType replica_type) +{ + int ret = OB_SUCCESS; + reset(); + if (OB_UNLIKELY(!member.is_valid() + || !ObReplicaTypeCheck::is_replica_type_valid(replica_type))) { + ret = OB_INVALID_ARGUMENT; + COMMON_LOG(WARN, "invalid argument", K(ret), K(member), K(replica_type)); + } else if (OB_FAIL(ObMember::assign(member))) { + COMMON_LOG(WARN, "failed to assign member", K(ret), K(member)); + } else if (OB_FALSE_IT(replica_type_ = replica_type)) { + // should never be here + } else if (OB_UNLIKELY(! is_valid())) { // check flag_ and replica_type_ correct + ret = OB_INVALID_ARGUMENT; + COMMON_LOG(WARN, "invalid argument", K(ret), K(member), K(replica_type), KPC(this)); + } + return ret; +} + void ObReplicaMember::reset() { ObMember::reset(); replica_type_ = REPLICA_TYPE_FULL; - region_ = DEFAULT_REGION_NAME; memstore_percent_ = 100; } bool ObReplicaMember::is_valid() const { + // columnstore bit is 1 if and only if replica_type is C + bool is_flag_valid = (is_columnstore() == (REPLICA_TYPE_COLUMNSTORE == replica_type_)); return ObMember::is_valid() && ObReplicaTypeCheck::is_replica_type_valid(replica_type_) - && !region_.is_empty() + && is_flag_valid && memstore_percent_ <= 100 && memstore_percent_ >= 0; } @@ -115,46 +174,6 @@ common::ObReplicaType ObReplicaMember::get_replica_type() const return replica_type_; } -int ObReplicaMember::set_replica_type(const common::ObReplicaType replica_type) -{ - int ret = OB_SUCCESS; - if (!ObReplicaTypeCheck::is_replica_type_valid(replica_type)) { - ret = OB_INVALID_ARGUMENT; - } else { - replica_type_ = replica_type; - } - return ret; -} - -const common::ObRegion &ObReplicaMember::get_region() const -{ - return region_; -} - -int ObReplicaMember::set_member(const ObMember &member) -{ - int ret = OB_SUCCESS; - - if (!member.is_valid()) { - ret = OB_INVALID_ARGUMENT; - COMMON_LOG(WARN, "invalid args", K(ret), K(member)); - } else if (OB_FAIL(ObMember::assign(member))) { - COMMON_LOG(WARN, "failed to assign member", K(ret), K(member)); - } - return ret; -} - -int ObReplicaMember::set_region(const common::ObRegion ®ion) -{ - int ret = OB_SUCCESS; - if (region.is_empty()) { - ret = OB_INVALID_ARGUMENT; - } else { - region_ = region; - } - return ret; -} - ObReplicaMember &ObReplicaMember::operator=(const ObReplicaMember &rhs) { server_ = rhs.server_; diff --git a/deps/oblib/src/common/ob_member.h b/deps/oblib/src/common/ob_member.h index 25b8d005f..91e5ceec1 100644 --- a/deps/oblib/src/common/ob_member.h +++ b/deps/oblib/src/common/ob_member.h @@ -49,11 +49,16 @@ public: void set_migrating(); void reset_migrating(); + bool is_columnstore() const; + void set_columnstore(); + void reset_columnstore(); + TO_STRING_KV(K_(server), K_(timestamp), K_(flag)); TO_YSON_KV(OB_Y_(server), OB_ID(t), timestamp_, OB_Y_(flag)); OB_UNIS_VERSION(1); protected: static const int64_t MIGRATING_FLAG_BIT = 1; + static const int64_t COLUMNSTORE_FLAG_BIT = 0; common::ObAddr server_; int64_t timestamp_; int64_t flag_; @@ -90,61 +95,46 @@ inline int member_to_string(const common::ObMember &member, ObSqlString &member_ class ObReplicaMember : public ObMember { public: + // default constructor ObReplicaMember() : ObMember(), replica_type_(REPLICA_TYPE_FULL), - region_(DEFAULT_REGION_NAME), memstore_percent_(100) {} + // construct with only server and timestamp, when we don't know or care about replica_type + // TODO(cangming.zl): remove this constructor when DRTask do not use it. ObReplicaMember(const common::ObAddr &server, const int64_t timestamp) - : ObMember(server, timestamp), + : ObMember(ObMember(server, timestamp)), replica_type_(REPLICA_TYPE_FULL), - region_(DEFAULT_REGION_NAME), - memstore_percent_(100) - {} - ObReplicaMember(const ObMember &member) - : ObMember(member), - replica_type_(REPLICA_TYPE_FULL), - region_(DEFAULT_REGION_NAME), - memstore_percent_(100) - {} - /* After the subsequent type conversion code is completed, remove the constructor */ - ObReplicaMember(const common::ObAddr &server, - const int64_t timestamp, - const common::ObReplicaType replica_type) - : ObMember(server, timestamp), - replica_type_(replica_type), - region_(DEFAULT_REGION_NAME), memstore_percent_(100) {} + // construct with server, timestamp and replica_type, + // this func will set columnstore flag if replica_type is C. ObReplicaMember(const common::ObAddr &server, const int64_t timestamp, const common::ObReplicaType replica_type, - const int64_t memstore_percent) - : ObMember(server, timestamp), + const int64_t memstore_percent = 100) + : ObMember(ObMember(server, timestamp)), replica_type_(replica_type), - region_(DEFAULT_REGION_NAME), memstore_percent_(memstore_percent) - {} - ObReplicaMember(const common::ObAddr &server, - const int64_t timestamp, - const common::ObReplicaType replica_type, - const common::ObRegion ®ion, - const int64_t memstore_percent) - : ObMember(server, timestamp), - replica_type_(replica_type), - region_(region), - memstore_percent_(memstore_percent) - {} + { + if (REPLICA_TYPE_COLUMNSTORE == replica_type) { + ObMember::set_columnstore(); + } + } public: + // init with server, timestamp, replica_type. + // this func will set columnstore flag if replica_type is C. + int init(const common::ObAddr &server, + const int64_t timestamp, + const common::ObReplicaType replica_type); + // init with existing member whose flag_ may have been set. + // this function will check whether flag_ is consistent with replica_type. + int init(const ObMember &member, + const common::ObReplicaType replica_type); common::ObReplicaType get_replica_type() const; - int set_replica_type(const common::ObReplicaType replica_type); - const common::ObRegion &get_region() const; - int set_region(const common::ObRegion ®ion); - int set_member(const ObMember &member); int64_t get_memstore_percent() const { return memstore_percent_; } - void set_memstore_percent(const int64_t memstore_percent) { memstore_percent_ = memstore_percent; } virtual void reset(); virtual bool is_valid() const; virtual bool is_readonly_replica() const; @@ -154,8 +144,8 @@ public: OB_UNIS_VERSION(1); private: common::ObReplicaType replica_type_; - common::ObRegion region_; - int64_t memstore_percent_; + int64_t memstore_percent_; // obsolate, only as placeholder + common::ObRegion region_ = DEFAULT_REGION_NAME; // obsolate, only as placeholder }; } // namespace common } // namespace oceanbase diff --git a/deps/oblib/src/lib/ob_define.h b/deps/oblib/src/lib/ob_define.h index 5b98c87a3..457f8d571 100644 --- a/deps/oblib/src/lib/ob_define.h +++ b/deps/oblib/src/lib/ob_define.h @@ -2084,17 +2084,19 @@ enum ObFreezeStatus }; /* - * |---- 2 bits ---|--- 4 bits ---|--- 2 bits ---|--- 2 bits ---| LSB - * |-- encryption--|--- clog ---|-- SSStore ---|--- MemStore--| LSB + * |---- 2 bits ---|---- 2 bits ---|--- 4 bits ---|--- 2 bits ---|--- 2 bits ---| LSB + * |--column-store-|-- encryption--|--- clog ---|-- SSStore ---|--- MemStore--| LSB */ const int64_t MEMSTORE_BITS_SHIFT = 0; const int64_t SSSTORE_BITS_SHIFT = 2; const int64_t CLOG_BITS_SHIFT = 4; const int64_t ENCRYPTION_BITS_SHIFT = 8; +const int64_t COLUMNSTORE_BITS_SHIFT = 10; const int64_t REPLICA_TYPE_MEMSTORE_MASK = (0x3UL << MEMSTORE_BITS_SHIFT); const int64_t REPLICA_TYPE_SSSTORE_MASK = (0x3UL << SSSTORE_BITS_SHIFT); const int64_t REPLICA_TYPE_CLOG_MASK = (0xFUL << CLOG_BITS_SHIFT); const int64_t REPLICA_TYPE_ENCRYPTION_MASK = (0x3UL << ENCRYPTION_BITS_SHIFT); +const int64_t REPLICA_TYPE_COLUMNSTORE_MASK = (0x3UL << COLUMNSTORE_BITS_SHIFT); // replica type associated with memstore const int64_t WITH_MEMSTORE = 0; const int64_t WITHOUT_MEMSTORE = 1; @@ -2107,16 +2109,22 @@ const int64_t ASYNC_CLOG = 1 << CLOG_BITS_SHIFT; // replica type associated with encryption const int64_t WITHOUT_ENCRYPTION = 0 << ENCRYPTION_BITS_SHIFT; const int64_t WITH_ENCRYPTION = 1 << ENCRYPTION_BITS_SHIFT; +// replica type associated with columnstore +const int64_t NOT_COLUMNSTORE = 0 << COLUMNSTORE_BITS_SHIFT; +const int64_t COLUMNSTORE = 1 << COLUMNSTORE_BITS_SHIFT; // tracepoint, refer to OB_MAX_CONFIG_xxx const int64_t OB_MAX_TRACEPOINT_NAME_LEN = 128; const int64_t OB_MAX_TRACEPOINT_DESCRIBE_LEN = 4096; -// Need to manually maintain the replica_type_to_str function in utility.cpp, -// Currently there are only three types: REPLICA_TYPE_FULL, REPLICA_TYPE_READONLY, and REPLICA_TYPE_LOGONLY +// Please modify the replica_type_to_string and string_to_replica_type function +// in ob_share_util.cpp when adding new replica_type. enum ObReplicaType { - // Almighty copy: is a member of paxos; has ssstore; has memstore + // Invalid replica_type, value of which is -1. + // Attention: Please DO use REPLICA_TYPE_INVALID as initial value. DO NOT use REPLICA_TYPE_MAX. + REPLICA_TYPE_INVALID = -1, + // Fully functional copy: is a member of paxos; has ssstore; has memstore REPLICA_TYPE_FULL = (SYNC_CLOG | WITH_SSSTORE | WITH_MEMSTORE), // 0 // Backup copy: Paxos member; ssstore; no memstore REPLICA_TYPE_BACKUP = (SYNC_CLOG | WITH_SSSTORE | WITHOUT_MEMSTORE), // 1 @@ -2133,54 +2141,44 @@ enum ObReplicaType REPLICA_TYPE_ARBITRATION = (ASYNC_CLOG | WITHOUT_SSSTORE | WITHOUT_MEMSTORE), // 21 // Encrypted log copy: encrypted; paxos member; no sstore; no memstore REPLICA_TYPE_ENCRYPTION_LOGONLY = (WITH_ENCRYPTION | SYNC_CLOG | WITHOUT_SSSTORE | WITHOUT_MEMSTORE), // 261 - // invalid value + // Column-store copy: column-store, not a member of paxos; ssstore; memstore + REPLICA_TYPE_COLUMNSTORE = (COLUMNSTORE | ASYNC_CLOG | WITH_SSSTORE | WITH_MEMSTORE), // 1040 + // max value REPLICA_TYPE_MAX, }; -static inline int replica_type_to_string(const ObReplicaType replica_type, char *name_str, const int64_t str_len) -{ - int ret = OB_SUCCESS; - switch(replica_type) { - case REPLICA_TYPE_FULL: { - strncpy(name_str ,"FULL", str_len); - break; - } - case REPLICA_TYPE_BACKUP: { - strncpy(name_str ,"BACKUP", str_len); - break; - } - case REPLICA_TYPE_LOGONLY: { - strncpy(name_str ,"LOGONLY", str_len); - break; - } - case REPLICA_TYPE_READONLY: { - strncpy(name_str ,"READONLY", str_len); - break; - } - case REPLICA_TYPE_MEMONLY: { - strncpy(name_str ,"MEMONLY", str_len); - break; - } - case REPLICA_TYPE_ENCRYPTION_LOGONLY: { - strncpy(name_str ,"ENCRYPTION_LOGONLY", str_len); - break; - } - default: { - ret = OB_INVALID_ARGUMENT; - strncpy(name_str ,"INVALID", str_len); - break; - } // default - } // switch - return ret; -} +// full replica +const char *const FULL_REPLICA_STR = "FULL"; +const char *const F_REPLICA_STR = "F"; +// logonly replica +const char *const LOGONLY_REPLICA_STR = "LOGONLY"; +const char *const L_REPLICA_STR = "L"; +// backup replica +const char *const BACKUP_REPLICA_STR = "BACKUP"; +const char *const B_REPLICA_STR = "B"; +// readonly replica +const char *const READONLY_REPLICA_STR = "READONLY"; +const char *const R_REPLICA_STR = "R"; +// memonly replica +const char *const MEMONLY_REPLICA_STR = "MEMONLY"; +const char *const M_REPLICA_STR = "M"; +// encryption logonly replica +const char *const ENCRYPTION_LOGONLY_REPLICA_STR = "ENCRYPTION_LOGONLY"; +const char *const E_REPLICA_STR = "E"; +// columnstore replica +const char *const COLUMNSTORE_REPLICA_STR = "COLUMNSTORE"; +const char *const C_REPLICA_STR = "C"; class ObReplicaTypeCheck { public: + // Currently only three types are valid, + // including REPLICA_TYPE_FULL, REPLICA_TYPE_READONLY, and REPLICA_TYPE_COLUMNSTORE static bool is_replica_type_valid(const int32_t replica_type) { return REPLICA_TYPE_FULL == replica_type - || REPLICA_TYPE_READONLY == replica_type; + || REPLICA_TYPE_READONLY == replica_type + || REPLICA_TYPE_COLUMNSTORE == replica_type; } static bool is_can_elected_replica(const int32_t replica_type) { @@ -2194,6 +2192,10 @@ public: { return (REPLICA_TYPE_READONLY == replica_type); } + static bool is_columnstore_replica(const int32_t replica_type) + { + return (REPLICA_TYPE_COLUMNSTORE == replica_type); + } static bool is_log_replica(const int32_t replica_type) { return (REPLICA_TYPE_LOGONLY == replica_type || REPLICA_TYPE_ENCRYPTION_LOGONLY == replica_type); @@ -2208,13 +2210,18 @@ public: return (replica_type >= REPLICA_TYPE_FULL && replica_type <= REPLICA_TYPE_LOGONLY) || (REPLICA_TYPE_ENCRYPTION_LOGONLY == replica_type); } + static bool is_non_paxos_replica(const int32_t replica_type) + { + return (REPLICA_TYPE_READONLY == replica_type || REPLICA_TYPE_COLUMNSTORE == replica_type); + } static bool is_writable_replica(const int32_t replica_type) { return (REPLICA_TYPE_FULL == replica_type); } static bool is_readable_replica(const int32_t replica_type) { - return (REPLICA_TYPE_FULL == replica_type || REPLICA_TYPE_READONLY == replica_type); + return (REPLICA_TYPE_FULL == replica_type || REPLICA_TYPE_READONLY == replica_type + || REPLICA_TYPE_COLUMNSTORE == replica_type); } static bool is_replica_with_memstore(const ObReplicaType replica_type) { @@ -2228,15 +2235,11 @@ public: { return (REPLICA_TYPE_FULL == replica_type || REPLICA_TYPE_READONLY == replica_type); } - static bool can_as_data_source(const int32_t dest_replica_type, const int32_t src_replica_type) - { - return (dest_replica_type == src_replica_type - || REPLICA_TYPE_FULL == src_replica_type); // TODO temporarily only supports the same type or F as the data source - } //Currently only copies of F and R can be used for machine reading, not L static bool can_slave_read_replica(const int32_t replica_type) { - return (REPLICA_TYPE_FULL == replica_type || REPLICA_TYPE_READONLY == replica_type); + return (REPLICA_TYPE_FULL == replica_type || REPLICA_TYPE_READONLY == replica_type + || REPLICA_TYPE_COLUMNSTORE == replica_type); } static bool change_replica_op_allow(const ObReplicaType source, const ObReplicaType target) @@ -2245,6 +2248,8 @@ public: if (REPLICA_TYPE_LOGONLY == source || REPLICA_TYPE_LOGONLY == target) { bool_ret = false; + } else if (REPLICA_TYPE_COLUMNSTORE == source || REPLICA_TYPE_COLUMNSTORE == target) { + bool_ret = false; } else if (REPLICA_TYPE_FULL == source) { bool_ret = true; } else if (REPLICA_TYPE_READONLY == source && REPLICA_TYPE_FULL == target) { diff --git a/deps/oblib/src/lib/utility/utility.cpp b/deps/oblib/src/lib/utility/utility.cpp index 3dbe3d4bb..598592531 100644 --- a/deps/oblib/src/lib/utility/utility.cpp +++ b/deps/oblib/src/lib/utility/utility.cpp @@ -1658,33 +1658,6 @@ int long_to_str10(int64_t val,char *dst, const int64_t buf_len, const bool is_si //////////////////////////////////////////////////////////////////////////////////////////////////// - -const char *replica_type_to_str(const ObReplicaType &type) -{ - const char *str = ""; - - switch (type) { - case REPLICA_TYPE_FULL: - str = "REPLICA_TYPE_FULL"; - break; - case REPLICA_TYPE_BACKUP: - str = "REPLICA_TYPE_BACKUP"; - break; - case REPLICA_TYPE_LOGONLY: - str = "REPLICA_TYPE_LOGONLY"; - break; - case REPLICA_TYPE_READONLY: - str = "REPLICA_TYPE_READONLY"; - break; - case REPLICA_TYPE_MEMONLY: - str = "REPLICA_TYPE_MEMONLY"; - break; - default: - str = "REPLICA_TYPE_UNKNOWN"; - } - return str; -} - bool ez2ob_addr(ObAddr &addr, easy_addr_t& ez) { bool ret = false; diff --git a/deps/oblib/src/lib/utility/utility.h b/deps/oblib/src/lib/utility/utility.h index a049aa222..93dc35660 100644 --- a/deps/oblib/src/lib/utility/utility.h +++ b/deps/oblib/src/lib/utility/utility.h @@ -1273,8 +1273,6 @@ private: void get_addr_by_proxy_sessid(const uint64_t session_id, ObAddr &addr); -const char *replica_type_to_str(const ObReplicaType &type); - int ob_atoll(const char *str, int64_t &res); int ob_strtoll(const char *str, char *&endptr, int64_t &res); int ob_strtoull(const char *str, char *&endptr, uint64_t &res); diff --git a/mittest/mtlenv/storage/test_ls_migration_param.cpp b/mittest/mtlenv/storage/test_ls_migration_param.cpp index 6f7e162dc..e03038d4c 100644 --- a/mittest/mtlenv/storage/test_ls_migration_param.cpp +++ b/mittest/mtlenv/storage/test_ls_migration_param.cpp @@ -253,7 +253,7 @@ TEST_F(TestLSMigrationParam, test_migrate_tablet_param) SCN scn; scn.convert_from_ts(ObTimeUtility::current_time()); ret = src_handle.get_obj()->init_for_first_time_creation(allocator_, src_key.ls_id_, src_key.tablet_id_, src_key.tablet_id_, - scn, 2022, create_tablet_schema, true/*need_create_empty_major_sstable*/, ls_handle.get_ls()->get_freezer()); + scn, 2022, create_tablet_schema, true/*need_create_empty_major_sstable*/, false/*need_generate_cs_replica_cg_array*/, ls_handle.get_ls()->get_freezer()); ASSERT_EQ(common::OB_SUCCESS, ret); share::SCN create_commit_scn; @@ -334,7 +334,7 @@ TEST_F(TestLSMigrationParam, test_migration_param_compat) SCN scn; scn.convert_from_ts(ObTimeUtility::current_time()); ret = src_handle.get_obj()->init_for_first_time_creation(allocator_, src_key.ls_id_, src_key.tablet_id_, src_key.tablet_id_, - scn, 2022, create_tablet_schema, true/*need_create_empty_major_sstable*/, ls_handle.get_ls()->get_freezer()); + scn, 2022, create_tablet_schema, true/*need_create_empty_major_sstable*/, false/*need_generate_cs_replica_cg_array*/, ls_handle.get_ls()->get_freezer()); ASSERT_EQ(common::OB_SUCCESS, ret); share::SCN create_commit_scn; diff --git a/mittest/mtlenv/storage/test_ls_tablet_info_writer_and_reader.cpp b/mittest/mtlenv/storage/test_ls_tablet_info_writer_and_reader.cpp index 67b2a25ee..3461b2930 100644 --- a/mittest/mtlenv/storage/test_ls_tablet_info_writer_and_reader.cpp +++ b/mittest/mtlenv/storage/test_ls_tablet_info_writer_and_reader.cpp @@ -170,7 +170,7 @@ void TestLSTabletInfoWR::fill_tablet_meta() SCN scn; scn.convert_from_ts(ObTimeUtility::current_time()); ret = src_handle.get_obj()->init_for_first_time_creation(arena_allocator_, src_key.ls_id_, src_key.tablet_id_, src_key.tablet_id_, - scn, 2022, create_tablet_schema, true/*need_create_empty_major_sstable*/, ls_handle.get_ls()->get_freezer()); + scn, 2022, create_tablet_schema, true/*need_create_empty_major_sstable*/, false/*need_generate_cs_replica_cg_array*/, ls_handle.get_ls()->get_freezer()); ASSERT_EQ(common::OB_SUCCESS, ret); share::SCN create_commit_scn; diff --git a/mittest/mtlenv/storage/test_table_scan_pure_data_table.cpp b/mittest/mtlenv/storage/test_table_scan_pure_data_table.cpp index d5c29649b..91cb6d6b1 100644 --- a/mittest/mtlenv/storage/test_table_scan_pure_data_table.cpp +++ b/mittest/mtlenv/storage/test_table_scan_pure_data_table.cpp @@ -141,6 +141,7 @@ void TestTableScanPureDataTable::insert_data_to_tablet(MockObAccessService *acce share::schema::ObTableSchema table_schema; TestDmlCommon::build_data_table_schema(tenant_id_, table_schema); + table_schema.set_tablet_id(tablet_id_); ObSEArray index_schema_array; @@ -181,6 +182,7 @@ void TestTableScanPureDataTable::table_scan( // prepare table schema share::schema::ObTableSchema table_schema; TestDmlCommon::build_data_table_schema(tenant_id_, table_schema); + table_schema.set_tablet_id(tablet_id_); // 1. get tx desc transaction::ObTxDesc *tx_desc = nullptr; diff --git a/mittest/mtlenv/storage/test_tenant_meta_mem_mgr.cpp b/mittest/mtlenv/storage/test_tenant_meta_mem_mgr.cpp index a02718ad8..c6e8eeacc 100644 --- a/mittest/mtlenv/storage/test_tenant_meta_mem_mgr.cpp +++ b/mittest/mtlenv/storage/test_tenant_meta_mem_mgr.cpp @@ -718,7 +718,7 @@ TEST_F(TestTenantMetaMemMgr, test_wash_tablet) ObTabletID empty_tablet_id; ret = tablet->init_for_first_time_creation(allocator_, ls_id_, tablet_id, tablet_id, - create_scn, create_scn.get_val_for_tx(), create_tablet_schema, true/*need_create_empty_major_sstable*/, &freezer); + create_scn, create_scn.get_val_for_tx(), create_tablet_schema, true/*need_create_empty_major_sstable*/, false/*need_generate_cs_replica_cg_array*/, &freezer); ASSERT_EQ(common::OB_SUCCESS, ret); ASSERT_EQ(1, tablet->get_ref()); ObTabletPersister persister; @@ -817,7 +817,7 @@ TEST_F(TestTenantMetaMemMgr, test_wash_inner_tablet) bool make_empty_co_sstable = true; ret = tablet->init_for_first_time_creation(allocator_, ls_id_, tablet_id, tablet_id, create_scn, create_scn.get_val_for_tx(), create_tablet_schema, - make_empty_co_sstable/*need_create_empty_major_sstable*/, &freezer); + make_empty_co_sstable/*need_create_empty_major_sstable*/, false/*need_generate_cs_replica_cg_array*/, &freezer); ASSERT_EQ(common::OB_SUCCESS, ret); ASSERT_EQ(1, tablet->get_ref()); @@ -927,7 +927,7 @@ TEST_F(TestTenantMetaMemMgr, test_wash_no_sstable_tablet) bool make_empty_co_sstable = false; ret = tablet->init_for_first_time_creation(allocator_, ls_id_, tablet_id, tablet_id, create_scn, create_scn.get_val_for_tx(), create_tablet_schema, - make_empty_co_sstable/*need_create_empty_major_sstable*/, &freezer); + make_empty_co_sstable/*need_create_empty_major_sstable*/, false/*need_generate_cs_replica_cg_array*/, &freezer); ASSERT_EQ(common::OB_SUCCESS, ret); ASSERT_EQ(1, tablet->get_ref()); @@ -1024,7 +1024,7 @@ TEST_F(TestTenantMetaMemMgr, test_get_tablet_with_allocator) bool make_empty_co_sstable = true; ret = tablet->init_for_first_time_creation(allocator_, ls_id_, tablet_id, tablet_id, create_scn, create_scn.get_val_for_tx(), create_tablet_schema, - make_empty_co_sstable/*need_create_empty_major_sstable*/, &freezer); + make_empty_co_sstable/*need_create_empty_major_sstable*/, false/*need_generate_cs_replica_cg_array*/, &freezer); ASSERT_EQ(common::OB_SUCCESS, ret); ASSERT_EQ(1, tablet->get_ref()); @@ -1152,7 +1152,7 @@ TEST_F(TestTenantMetaMemMgr, test_wash_mem_tablet) bool make_empty_co_sstable = false; ret = tablet->init_for_first_time_creation(allocator_, ls_id_, tablet_id, tablet_id, create_scn, create_scn.get_val_for_tx(), create_tablet_schema, - make_empty_co_sstable/*need_create_empty_major_sstable*/, &freezer); + make_empty_co_sstable/*need_create_empty_major_sstable*/, false/*need_generate_cs_replica_cg_array*/, &freezer); ASSERT_EQ(common::OB_SUCCESS, ret); ASSERT_EQ(1, tablet->get_ref()); diff --git a/mittest/mtlenv/test_tx_data_table.cpp b/mittest/mtlenv/test_tx_data_table.cpp index 212516f49..0be8d15ca 100644 --- a/mittest/mtlenv/test_tx_data_table.cpp +++ b/mittest/mtlenv/test_tx_data_table.cpp @@ -727,6 +727,7 @@ void TestTxDataTable::fake_ls_(ObLS &ls) ls.ls_meta_.migration_status_ = ObMigrationStatus::OB_MIGRATION_STATUS_NONE; ls.ls_meta_.restore_status_ = ObLSRestoreStatus::NONE; ls.ls_meta_.rebuild_seq_ = 0; + ls.ls_meta_.store_format_ = common::ObLSStoreType::OB_LS_STORE_NORMAL; } void TestTxDataTable::do_print_leak_slice_test() diff --git a/mittest/simple_server/test_ls_recover.cpp b/mittest/simple_server/test_ls_recover.cpp index 544df0d00..9d0639d64 100644 --- a/mittest/simple_server/test_ls_recover.cpp +++ b/mittest/simple_server/test_ls_recover.cpp @@ -267,6 +267,7 @@ TEST_F(ObLSBeforeRestartTest, create_unfinished_ls_without_disk) migration_status, ObLSRestoreStatus(ObLSRestoreStatus::NONE), arg.get_create_scn(), + ObLSStoreFormat(ObLSStoreType::OB_LS_STORE_NORMAL), ls)); ObLSLockGuard lock_ls(ls); const ObLSMeta &ls_meta = ls->get_ls_meta(); @@ -295,6 +296,7 @@ TEST_F(ObLSBeforeRestartTest, create_unfinished_ls_with_disk) migration_status, ObLSRestoreStatus(ObLSRestoreStatus::NONE), arg.get_create_scn(), + ObLSStoreFormat(ObLSStoreType::OB_LS_STORE_NORMAL), ls)); const bool unused_allow_log_sync = true; prepare_palf_base_info(arg, palf_base_info); @@ -329,6 +331,7 @@ TEST_F(ObLSBeforeRestartTest, create_unfinished_ls_with_inner_tablet) migration_status, ObLSRestoreStatus(ObLSRestoreStatus::NONE), arg.get_create_scn(), + ObLSStoreFormat(ObLSStoreType::OB_LS_STORE_NORMAL), ls)); const bool unused_allow_log_sync = true; prepare_palf_base_info(arg, palf_base_info); @@ -365,6 +368,7 @@ TEST_F(ObLSBeforeRestartTest, create_unfinished_ls_with_commit_slog) migration_status, ObLSRestoreStatus(ObLSRestoreStatus::NONE), arg.get_create_scn(), + ObLSStoreFormat(ObLSStoreType::OB_LS_STORE_NORMAL), ls)); const bool unused_allow_log_sync = true; prepare_palf_base_info(arg, palf_base_info); @@ -404,6 +408,7 @@ TEST_F(ObLSBeforeRestartTest, create_restore_ls) migration_status, ObLSRestoreStatus(ObLSRestoreStatus::RESTORE_START), arg.get_create_scn(), + ObLSStoreFormat(ObLSStoreType::OB_LS_STORE_NORMAL), ls)); const bool unused_allow_log_sync = true; prepare_palf_base_info(arg, palf_base_info); @@ -448,6 +453,7 @@ TEST_F(ObLSBeforeRestartTest, create_rebuild_ls) migration_status, ObLSRestoreStatus(ObLSRestoreStatus::NONE), arg.get_create_scn(), + ObLSStoreFormat(ObLSStoreType::OB_LS_STORE_NORMAL), ls)); const bool unused_allow_log_sync = true; prepare_palf_base_info(arg, palf_base_info); diff --git a/src/logservice/palf/palf_handle_impl.cpp b/src/logservice/palf/palf_handle_impl.cpp index dfa18756f..eb735c266 100755 --- a/src/logservice/palf/palf_handle_impl.cpp +++ b/src/logservice/palf/palf_handle_impl.cpp @@ -5435,10 +5435,8 @@ void PalfHandleImpl::report_switch_learner_to_acceptor_(const common::ObMember & "member", member_buf, "curr_member_list", member_list_buf, "curr_replica_num", curr_replica_num); - char replica_readonly_name_[common::MAX_REPLICA_TYPE_LENGTH]; - char replica_full_name_[common::MAX_REPLICA_TYPE_LENGTH]; - replica_type_to_string(ObReplicaType::REPLICA_TYPE_READONLY, replica_readonly_name_, sizeof(replica_readonly_name_)); - replica_type_to_string(ObReplicaType::REPLICA_TYPE_FULL, replica_full_name_, sizeof(replica_full_name_)); + const char *replica_readonly_name_ = ObShareUtil::replica_type_to_string(ObReplicaType::REPLICA_TYPE_READONLY); + const char *replica_full_name_ = ObShareUtil::replica_type_to_string(ObReplicaType::REPLICA_TYPE_FULL); plugins_.record_replica_type_change_event(palf_id_, config_version, replica_readonly_name_, replica_full_name_, EXTRA_INFOS); } @@ -5457,10 +5455,8 @@ void PalfHandleImpl::report_switch_acceptor_to_learner_(const common::ObMember & "member", member_buf, "curr_member_list", member_list_buf, "curr_replica_num", curr_replica_num); - char replica_readonly_name_[common::MAX_REPLICA_TYPE_LENGTH]; - char replica_full_name_[common::MAX_REPLICA_TYPE_LENGTH]; - replica_type_to_string(ObReplicaType::REPLICA_TYPE_READONLY, replica_readonly_name_, sizeof(replica_readonly_name_)); - replica_type_to_string(ObReplicaType::REPLICA_TYPE_FULL, replica_full_name_, sizeof(replica_full_name_)); + const char *replica_readonly_name_ = ObShareUtil::replica_type_to_string(ObReplicaType::REPLICA_TYPE_READONLY); + const char *replica_full_name_ = ObShareUtil::replica_type_to_string(ObReplicaType::REPLICA_TYPE_FULL); plugins_.record_replica_type_change_event(palf_id_, config_version, replica_full_name_, replica_readonly_name_, EXTRA_INFOS); } diff --git a/src/observer/ob_service.cpp b/src/observer/ob_service.cpp index 7c4076df5..878ca9281 100644 --- a/src/observer/ob_service.cpp +++ b/src/observer/ob_service.cpp @@ -88,6 +88,8 @@ #include "share/ob_heartbeat_handler.h" #include "storage/slog/ob_storage_logger_manager.h" #include "storage/high_availability/ob_transfer_lock_utils.h" +#include "storage/column_store/ob_column_store_replica_util.h" +#include "common/ob_store_format.h" // ObLSStoreFormat namespace oceanbase { @@ -2704,7 +2706,17 @@ int ObService::inner_fill_tablet_info_( int64_t data_size = 0; int64_t required_size = 0; ObArray column_checksums; - if (OB_FAIL(tablet_handle.get_obj()->get_tablet_report_info(snapshot_version, column_checksums, + ObTablet *tablet = tablet_handle.get_obj(); + bool need_wait_major_convert_in_cs_replica = false; + if (OB_ISNULL(tablet)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tablet is unexpected nullptr", K(ret), K(tenant_id), K(tablet_id), K(tablet_handle)); + } else if (OB_FAIL(ObCSReplicaUtil::check_need_wait_major_convert(*ls, tablet_id, *tablet, need_wait_major_convert_in_cs_replica))) { + LOG_WARN("fail to check need wait major convert in cs replica", K(ret), KPC(ls), K(tablet)); + } else if (need_wait_major_convert_in_cs_replica) { + ret = OB_EAGAIN; + LOG_WARN("need wait major convert for cs replica", K(ret), K(tablet_id)); + } else if (OB_FAIL(tablet->get_tablet_report_info(snapshot_version, column_checksums, data_size, required_size, need_checksum))) { LOG_WARN("fail to get tablet report info from tablet", KR(ret), K(tenant_id), K(tablet_id)); } else if (OB_FAIL(tablet_replica.init( @@ -2788,118 +2800,27 @@ int ObService::fill_tablet_report_info( return ret; } -int ObService::get_role_from_palf_( - logservice::ObLogService &log_service, - const share::ObLSID &ls_id, - common::ObRole &role, - int64_t &proposal_id) -{ - int ret = OB_SUCCESS; - role = FOLLOWER; - proposal_id = 0; - palf::PalfHandleGuard palf_handle_guard; - if (OB_FAIL(log_service.open_palf(ls_id, palf_handle_guard))) { - LOG_WARN("open palf failed", KR(ret), K(ls_id)); - } else if (OB_FAIL(palf_handle_guard.get_role(role, proposal_id))) { - LOG_WARN("get role failed", KR(ret), K(ls_id)); - } - return ret; -} - int ObService::fill_ls_replica( const uint64_t tenant_id, const ObLSID &ls_id, share::ObLSReplica &replica) { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - uint64_t unit_id = common::OB_INVALID_ID; + replica.reset(); if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("service not inited", KR(ret)); - } else if (!ls_id.is_valid() - || OB_INVALID_TENANT_ID == tenant_id - || OB_ISNULL(gctx_.config_)) { + } else if (!ls_id.is_valid() || OB_INVALID_TENANT_ID == tenant_id) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(ls_id)); - } else if (OB_FAIL(GCTX.omt_->get_unit_id(tenant_id, unit_id))) { - LOG_WARN("get tenant unit id failed", KR(ret), K(tenant_id), K(ls_id)); } else { MTL_SWITCH(tenant_id) { - ObLSHandle ls_handle; - ObLSService *ls_svr = nullptr; - logservice::ObLogService *log_service = nullptr; - common::ObRole role = FOLLOWER; - ObMemberList ob_member_list; - ObLSReplica::MemberList member_list; - GlobalLearnerList learner_list; - int64_t proposal_id = 0; - int64_t paxos_replica_number = 0; - ObLSRestoreStatus restore_status; - ObReplicaStatus replica_status = REPLICA_STATUS_NORMAL; - ObReplicaType replica_type = REPLICA_TYPE_FULL; - bool is_compatible_with_readonly_replica = false; - ObMigrationStatus migration_status = OB_MIGRATION_STATUS_MAX; - if (OB_ISNULL(ls_svr = MTL(ObLSService*))) { + ObLSService *ls_svr = MTL(ObLSService*); + if (OB_ISNULL(ls_svr)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("MTL ObLSService is null", KR(ret), K(tenant_id)); - } else if (OB_FAIL(ls_svr->get_ls( - ObLSID(ls_id), - ls_handle, ObLSGetMod::OBSERVER_MOD))) { - LOG_WARN("get ls handle failed", KR(ret)); - } else if (OB_FAIL(ls_handle.get_ls()->get_paxos_member_list_and_learner_list(ob_member_list, paxos_replica_number, learner_list))) { - LOG_WARN("get member list and learner list from ObLS failed", KR(ret)); - } else if (OB_FAIL(ls_handle.get_ls()->get_restore_status(restore_status))) { - LOG_WARN("get restore status failed", KR(ret)); - } else if (OB_FAIL(ls_handle.get_ls()->get_migration_status(migration_status))) { - LOG_WARN("get migration status failed", KR(ret)); - } else if (OB_FAIL(ls_handle.get_ls()->get_replica_status(replica_status))) { - LOG_WARN("get replica status failed", KR(ret)); - } else if (OB_ISNULL(log_service = MTL(logservice::ObLogService*))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("MTL ObLogService is null", KR(ret), K(tenant_id)); - } else if (OB_FAIL(get_role_from_palf_(*log_service, ls_id, role, proposal_id))) { - LOG_WARN("failed to get role from palf", KR(ret), K(tenant_id), K(ls_id)); - } else if (OB_SUCCESS != (tmp_ret = ObShareUtil::check_compat_version_for_readonly_replica( - tenant_id, is_compatible_with_readonly_replica))) { - LOG_WARN("fail to check data version for read-only replica", KR(ret), K(tenant_id)); - } - - if (OB_FAIL(ret)) { - } else if (!is_compatible_with_readonly_replica) { - replica_type = REPLICA_TYPE_FULL; - } else if (learner_list.contains(gctx_.self_addr())) { - // if replica exists in learner_list, report it as R-replica. - // Otherwise, report as F-replica - replica_type = REPLICA_TYPE_READONLY; - } - - if (OB_FAIL(ret)) { - } else if (OB_FAIL(ObLSReplica::transform_ob_member_list(ob_member_list, member_list))) { - LOG_WARN("fail to transfrom ob_member_list into member_list", KR(ret), K(ob_member_list)); - } else if (OB_FAIL(replica.init( - 0, /*create_time_us*/ - 0, /*modify_time_us*/ - tenant_id, /*tenant_id*/ - ls_id, /*ls_id*/ - gctx_.self_addr(), /*server*/ - gctx_.config_->mysql_port, /*sql_port*/ - role, /*role*/ - replica_type, /*replica_type*/ - proposal_id, /*proposal_id*/ - is_strong_leader(role) ? REPLICA_STATUS_NORMAL : replica_status,/*replica_status*/ - restore_status, /*restore_status*/ - 100, /*memstore_percent*/ - unit_id, /*unit_id*/ - gctx_.config_->zone.str(), /*zone*/ - paxos_replica_number, /*paxos_replica_number*/ - 0, /*data_size*/ - 0, /*required_size*/ - member_list, - learner_list, - OB_MIGRATION_STATUS_REBUILD == migration_status /*is_rebuild*/))) { - LOG_WARN("fail to init a ls replica", KR(ret), K(tenant_id), K(ls_id), K(role), - K(proposal_id), K(unit_id), K(paxos_replica_number), K(member_list), K(learner_list)); + LOG_WARN("ObLSService is null", KR(ret)); + } else if (OB_FAIL(ls_svr->get_ls_replica(ls_id, ObLSGetMod::OBSERVER_MOD, replica))) { + LOG_WARN("fail to get_ls_replica", KR(ret), K(ls_id)); } else { LOG_TRACE("finish fill ls replica", KR(ret), K(tenant_id), K(ls_id), K(replica)); } diff --git a/src/observer/ob_service.h b/src/observer/ob_service.h index a4ed78244..c7f3dfea1 100644 --- a/src/observer/ob_service.h +++ b/src/observer/ob_service.h @@ -269,11 +269,6 @@ public: int check_server_empty(bool &server_empty); private: - int get_role_from_palf_( - logservice::ObLogService &log_service, - const share::ObLSID &ls_id, - common::ObRole &role, - int64_t &proposal_id); int inner_fill_tablet_info_( const int64_t tenant_id, const ObTabletID &tablet_id, diff --git a/src/observer/table/ob_table_cg_service.cpp b/src/observer/table/ob_table_cg_service.cpp index 1cc7014c2..e797165aa 100644 --- a/src/observer/table/ob_table_cg_service.cpp +++ b/src/observer/table/ob_table_cg_service.cpp @@ -688,6 +688,7 @@ int ObTableLocCgService::generate_table_loc_meta(const ObTableCtx &ctx, bool is_lookup) { int ret = OB_SUCCESS; + int64_t route_policy = 0; const ObTableSchema *table_schema = ctx.get_table_schema(); const ObTableSchema *index_schema = ctx.get_index_schema(); @@ -697,8 +698,11 @@ int ObTableLocCgService::generate_table_loc_meta(const ObTableCtx &ctx, } else if (ctx.is_index_scan() && OB_ISNULL(index_schema)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("index schema is null", K(ret)); + } else if (OB_FAIL(ctx.get_session_info().get_sys_variable(SYS_VAR_OB_ROUTE_POLICY, route_policy))) { + LOG_WARN("fail to get route policy from session", K(ret)); } else { loc_meta.reset(); + loc_meta.route_policy_ = route_policy; // is_lookup 有什么用？好像都是 false loc_meta.ref_table_id_ = is_lookup ? ctx.get_ref_table_id() : ctx.get_index_table_id(); loc_meta.table_loc_id_ = ctx.get_ref_table_id(); @@ -2541,7 +2545,8 @@ int ObTableTscCgService::generate_das_result_output(ObDASScanCtDef &das_tsc_ctde // 主表/索引回表/索引扫描不需要回表: select column ids // 索引表: rowkey column ids int ObTableTscCgService::generate_table_param(const ObTableCtx &ctx, - ObDASScanCtDef &das_tsc_ctdef) + ObDASScanCtDef &das_tsc_ctdef, + const bool query_cs_replica /*=false*/) { int ret = OB_SUCCESS; ObSEArray tsc_out_cols; @@ -2595,7 +2600,9 @@ int ObTableTscCgService::generate_table_param(const ObTableCtx &ctx, } else if (OB_FAIL(das_tsc_ctdef.table_param_.convert(*table_schema, das_tsc_ctdef.access_column_ids_, das_tsc_ctdef.pd_expr_spec_.pd_storage_flag_, - &tsc_out_cols))) { + &tsc_out_cols, + false /*force_mysql_mode*/, + query_cs_replica))) { LOG_WARN("fail to convert schema", K(ret), K(*table_schema)); } else if (OB_FAIL(generate_das_result_output(das_tsc_ctdef, tsc_out_cols))) { LOG_WARN("fail to generate das result outpur", K(ret), K(tsc_out_cols)); @@ -2606,7 +2613,8 @@ int ObTableTscCgService::generate_table_param(const ObTableCtx &ctx, int ObTableTscCgService::generate_das_tsc_ctdef(const ObTableCtx &ctx, ObIAllocator &allocator, - ObDASScanCtDef &das_tsc_ctdef) + ObDASScanCtDef &das_tsc_ctdef, + const bool query_cs_replica /*=false*/) { int ret = OB_SUCCESS; ObSchemaGetterGuard &schema_guard = (const_cast(ctx)).get_schema_guard(); @@ -2616,7 +2624,7 @@ int ObTableTscCgService::generate_das_tsc_ctdef(const ObTableCtx &ctx, if (OB_FAIL(generate_access_ctdef(ctx, allocator, das_tsc_ctdef))) { // init access_column_ids_,pd_expr_spec_.access_exprs_ LOG_WARN("fail to generate asccess ctdef", K(ret)); - } else if (OB_FAIL(generate_table_param(ctx, das_tsc_ctdef))) { // init table_param_, result_output_ + } else if (OB_FAIL(generate_table_param(ctx, das_tsc_ctdef, query_cs_replica))) { // init table_param_, result_output_ LOG_WARN("fail to generate table param", K(ret)); } @@ -2663,7 +2671,7 @@ int ObTableTscCgService::generate_tsc_ctdef(const ObTableCtx &ctx, int ret = OB_SUCCESS; ObStaticEngineCG cg(ctx.get_cur_cluster_version()); const int64_t filter_exprs_cnt = ctx.get_filter_exprs().count(); - + bool query_cs_replica = false; // init scan_ctdef_.ref_table_id_ tsc_ctdef.scan_ctdef_.ref_table_id_ = ctx.get_index_table_id(); if (OB_FAIL(tsc_ctdef.output_exprs_.init(ctx.get_select_exprs().count()))) { @@ -2674,7 +2682,9 @@ int ObTableTscCgService::generate_tsc_ctdef(const ObTableCtx &ctx, LOG_WARN("fail to generate output exprs", K(ret)); } else if (OB_FAIL(cg.generate_rt_exprs(ctx.get_filter_exprs(), tsc_ctdef.filter_exprs_))) { LOG_WARN("fail to generate filter rt exprs ", K(ret)); - } else if (OB_FAIL(generate_das_tsc_ctdef(ctx, allocator, tsc_ctdef.scan_ctdef_))) { // init scan_ctdef_ + } else if (OB_FAIL(ctx.check_is_cs_replica_query(query_cs_replica))) { + LOG_WARN("fail to check is cs replica query", K(ret)); + } else if (OB_FAIL(generate_das_tsc_ctdef(ctx, allocator, tsc_ctdef.scan_ctdef_, query_cs_replica))) { // init scan_ctdef_ LOG_WARN("fail to generate das scan ctdef", K(ret)); } else if (ctx.is_index_back()) { // init lookup_ctdef_,lookup_loc_meta_ diff --git a/src/observer/table/ob_table_cg_service.h b/src/observer/table/ob_table_cg_service.h index 5caf02217..ef0cba40a 100644 --- a/src/observer/table/ob_table_cg_service.h +++ b/src/observer/table/ob_table_cg_service.h @@ -362,7 +362,8 @@ public: private: static int generate_das_tsc_ctdef(const ObTableCtx &ctx, ObIAllocator &allocator, - sql::ObDASScanCtDef &das_tsc_ctdef); + sql::ObDASScanCtDef &das_tsc_ctdef, + const bool query_cs_replica = false); static int replace_gen_col_exprs(const ObTableCtx &ctx, common::ObIArray &access_exprs); static int generate_output_exprs(const ObTableCtx &ctx, @@ -371,7 +372,8 @@ private: ObIAllocator &allocator, sql::ObDASScanCtDef &das_tsc_ctdef); static int generate_table_param(const ObTableCtx &ctx, - sql::ObDASScanCtDef &das_tsc_ctdef); + sql::ObDASScanCtDef &das_tsc_ctdef, + const bool query_cs_replica = false); static OB_INLINE bool is_in_array(const common::ObIArray &array, const sql::ObRawExpr *expr) { diff --git a/src/observer/table/ob_table_context.cpp b/src/observer/table/ob_table_context.cpp index 8bb85b7b2..2fb5c0ffa 100644 --- a/src/observer/table/ob_table_context.cpp +++ b/src/observer/table/ob_table_context.cpp @@ -784,6 +784,20 @@ int ObTableCtx::adjust_entity() return ret; } +int ObTableCtx::check_is_cs_replica_query(bool &is_cs_replica_query) const +{ + int ret = OB_SUCCESS; + is_cs_replica_query = false; + if (ObRoutePolicyType::INVALID_POLICY == loc_meta_.route_policy_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid route policy", K(ret)); + } else { + is_cs_replica_query = ObRoutePolicyType::COLUMN_STORE_ONLY == loc_meta_.route_policy_; + } + LOG_TRACE("[CS-Replica] check cs replica query", K(ret), K(is_cs_replica_query), K_(loc_meta)); + return ret; +} + bool ObTableCtx::has_exist_in_columns(const ObIArray &columns, const ObString &name, int64_t *idx /* =nullptr */) const diff --git a/src/observer/table/ob_table_context.h b/src/observer/table/ob_table_context.h index af494a51f..7a797aa2c 100644 --- a/src/observer/table/ob_table_context.h +++ b/src/observer/table/ob_table_context.h @@ -413,6 +413,9 @@ public: // read lob的allocator需要保证obj序列化到rpc buffer后才能析构 static int read_real_lob(common::ObIAllocator &allocator, ObObj &obj); int adjust_entity(); +public: + // for column store replica query + int check_is_cs_replica_query(bool &is_cs_replica_query) const; private: // for common int get_tablet_by_rowkey(const common::ObRowkey &rowkey, diff --git a/src/observer/virtual_table/ob_all_virtual_ls_info.cpp b/src/observer/virtual_table/ob_all_virtual_ls_info.cpp index f568fccf6..361da89c3 100644 --- a/src/observer/virtual_table/ob_all_virtual_ls_info.cpp +++ b/src/observer/virtual_table/ob_all_virtual_ls_info.cpp @@ -130,15 +130,8 @@ int ObAllVirtualLSInfo::process_curr_tenant(ObNewRow *&row) break; case OB_APP_MIN_COLUMN_ID + 4: { // replica_type - if (OB_FAIL(replica_type_to_string(ls_info.replica_type_, - replica_type_name_, - sizeof(replica_type_name_)))) { - SERVER_LOG(WARN, "get replica type name failed", K(ret), K(ls_info.replica_type_)); - } else { - replica_type_name_[MAX_REPLICA_TYPE_LENGTH - 1] = '\0'; - cur_row_.cells_[i].set_varchar(replica_type_name_); - cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - } + cur_row_.cells_[i].set_varchar(ObShareUtil::replica_type_to_string(ls_info.replica_type_)); + cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); break; } case OB_APP_MIN_COLUMN_ID + 5: { diff --git a/src/observer/virtual_table/ob_all_virtual_ls_info.h b/src/observer/virtual_table/ob_all_virtual_ls_info.h index 23e922447..eb005701b 100644 --- a/src/observer/virtual_table/ob_all_virtual_ls_info.h +++ b/src/observer/virtual_table/ob_all_virtual_ls_info.h @@ -48,7 +48,6 @@ private: common::ObAddr addr_; char ip_buf_[common::OB_IP_STR_BUFF]; char state_name_[common::MAX_LS_STATE_LENGTH]; - char replica_type_name_[common::MAX_REPLICA_TYPE_LENGTH]; /* 跨租户访问的资源必须由ObMultiTenantOperator来处理释放*/ int64_t ls_id_; ObSharedGuard ls_iter_guard_; diff --git a/src/observer/virtual_table/ob_all_virtual_proxy_schema.cpp b/src/observer/virtual_table/ob_all_virtual_proxy_schema.cpp index 5a0e22873..91f434005 100644 --- a/src/observer/virtual_table/ob_all_virtual_proxy_schema.cpp +++ b/src/observer/virtual_table/ob_all_virtual_proxy_schema.cpp @@ -931,6 +931,43 @@ int ObAllVirtualProxySchema::get_next_tenant_server_( return ret; } +int ObAllVirtualProxySchema::get_replica_type_from_locality_( + const ZoneLocalityIArray &zone_locality_array, + const ObZone &zone, + ObReplicaType &replica_type) +{ + int ret = OB_SUCCESS; + replica_type = REPLICA_TYPE_FULL; + if (OB_UNLIKELY(zone_locality_array.empty() || zone.is_empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(zone_locality_array), K(zone)); + } else { + bool zone_found = false; + FOREACH_CNT_X(zone_locality, zone_locality_array, !zone_found && OB_SUCCESS == ret) { + if (zone_locality->get_zone_set().at(0) == zone) { + zone_found = true; + if (zone_locality->get_readonly_replica_num() > 0) { + replica_type = REPLICA_TYPE_READONLY; + } else if (zone_locality->get_columnstore_replica_num() > 0) { + replica_type = REPLICA_TYPE_COLUMNSTORE; + } else if (zone_locality->get_full_replica_num() > 0) { + replica_type = REPLICA_TYPE_FULL; + } else { + // unrecognized replica_type + ret = OB_ERR_UNEXPECTED; + replica_type = REPLICA_TYPE_INVALID; + LOG_WARN("unrecognized replica type", KR(ret), KPC(zone_locality)); + } + } + } + if (!zone_found) { + // tenant locality does not include this zone, regard as FULL + replica_type = REPLICA_TYPE_FULL; + } + } + return ret; +} + int ObAllVirtualProxySchema::fill_tenant_servers_( const uint64_t tenant_id, ObMySQLResult &result, @@ -952,6 +989,17 @@ int ObAllVirtualProxySchema::fill_tenant_servers_( int64_t svr_idx = 0; first_idx_in_zone.reset(); tenant_servers_.reset(); + const ObTenantSchema *tenant_schema = NULL; + ObArray zone_locality; + + if (OB_FAIL(schema_guard_.get_tenant_info(tenant_id, tenant_schema))) { + LOG_WARN("fail to get tenant info", KR(ret), K(tenant_id)); + } else if (OB_ISNULL(tenant_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant not exist", KR(ret), K(tenant_id)); + } else if (OB_FAIL(tenant_schema->get_zone_replica_attr_array(zone_locality))) { + LOG_WARN("fail to get zone_locality_array"); + } while (OB_SUCC(ret) && OB_SUCC(result.next())) { tenant_server.reset(); @@ -960,7 +1008,9 @@ int ObAllVirtualProxySchema::fill_tenant_servers_( EXTRACT_VARCHAR_FIELD_MYSQL(result, "svr_ip", svr_ip); EXTRACT_INT_FIELD_MYSQL(result, "inner_port", sql_port, int64_t); EXTRACT_VARCHAR_FIELD_MYSQL(result, "zone", zone); - if (OB_UNLIKELY(!server.set_ip_addr(svr_ip, svr_port))) { + if (FAILEDx(get_replica_type_from_locality_(zone_locality, zone, replica_type))) { + LOG_WARN("failed to get replica_type", KR(ret), K(zone_locality), K(zone)); + } else if (OB_UNLIKELY(!server.set_ip_addr(svr_ip, svr_port))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to set_ip_addr", KR(ret), K(svr_ip), K(svr_port)); } else if (OB_FAIL(replica_location.init( diff --git a/src/observer/virtual_table/ob_all_virtual_proxy_schema.h b/src/observer/virtual_table/ob_all_virtual_proxy_schema.h index 0cbdeca1d..c7e512c12 100644 --- a/src/observer/virtual_table/ob_all_virtual_proxy_schema.h +++ b/src/observer/virtual_table/ob_all_virtual_proxy_schema.h @@ -144,6 +144,10 @@ private: int get_next_tenant_server_(const common::ObString &table_name, const share::schema::ObTableSchema *table_schema); int get_tenant_servers_(const uint64_t tenant_id); + int get_replica_type_from_locality_( + const share::schema::ZoneLocalityIArray &zone_locality_array, + const ObZone &zone, + ObReplicaType &replica_type); int fill_tenant_servers_( const uint64_t tenant_id, common::sqlclient::ObMySQLResult &result, diff --git a/src/rootserver/freeze/ob_checksum_validator.cpp b/src/rootserver/freeze/ob_checksum_validator.cpp index 8694cedc8..bfbbdc8c2 100755 --- a/src/rootserver/freeze/ob_checksum_validator.cpp +++ b/src/rootserver/freeze/ob_checksum_validator.cpp @@ -28,6 +28,7 @@ #include "share/ob_zone_merge_info.h" #include "share/ob_freeze_info_manager.h" #include "rootserver/freeze/ob_fts_checksum_validate_util.h" +#include "storage/compaction/ob_medium_compaction_func.h" namespace oceanbase { @@ -422,42 +423,9 @@ int ObChecksumValidator::verify_tablet_replica_checksum() if (OB_UNLIKELY(replica_ckm_items_.empty())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(replica_ckm_items_)); - } else { - const ObTabletReplicaChecksumItem *prev_item = nullptr; - ObSEArray error_pairs; - error_pairs.set_attr(ObMemAttr(tenant_id_, "CkmErrPairs")); - ObLSID prev_error_ls_id; - ObTabletID prev_error_table_id; - int64_t affected_rows = 0; - for (int64_t i = 0; OB_SUCC(ret) && (i < replica_ckm_items_.count()); ++i) { - const ObTabletReplicaChecksumItem &curr_item = replica_ckm_items_.at(i); - if (OB_NOT_NULL(prev_item) - && curr_item.is_same_tablet(*prev_item)) { // same tablet - if (OB_FAIL(curr_item.verify_checksum(*prev_item))) { - if (OB_CHECKSUM_ERROR == ret) { - LOG_DBA_ERROR(OB_CHECKSUM_ERROR, "msg", "checksum error in tablet replica checksum", KR(ret), - K(curr_item), KPC(prev_item)); - if (curr_item.ls_id_ != prev_error_ls_id || curr_item.tablet_id_ != prev_error_table_id) { - prev_error_ls_id = curr_item.ls_id_; - prev_error_table_id = curr_item.tablet_id_; - if (OB_TMP_FAIL(error_pairs.push_back(ObTabletLSPair(curr_item.tablet_id_, curr_item.ls_id_)))) { - LOG_WARN("fail to push back error pair", K(tmp_ret), "tablet_id", curr_item.tablet_id_, "ls_id", curr_item.ls_id_); - } - } - } else { - LOG_WARN("unexpected error in tablet replica checksum", KR(ret), K(curr_item), KPC(prev_item)); - } - } - } - prev_item = &curr_item; - } - if (!error_pairs.empty()) { - if (OB_TMP_FAIL(ObTabletMetaTableCompactionOperator::batch_set_info_status(MTL_ID(), error_pairs, affected_rows))) { - LOG_WARN("fail to batch set info status", KR(tmp_ret)); - } else { - LOG_INFO("succ to batch set info status", K(tmp_ret), K(affected_rows), K(error_pairs)); - } - } + } else if (OB_FAIL(ObMediumCompactionScheduleFunc::check_replica_checksum_items( + replica_ckm_items_.array_, ls_locality_cache_.get_cs_replica_cache(), false /*is_medium_checker*/))) { + LOG_WARN("failed to verify tablet replica checksum", K(ret)); } return ret; } diff --git a/src/rootserver/freeze/ob_checksum_validator.h b/src/rootserver/freeze/ob_checksum_validator.h index fc8dcbf54..045d6c2b9 100644 --- a/src/rootserver/freeze/ob_checksum_validator.h +++ b/src/rootserver/freeze/ob_checksum_validator.h @@ -67,7 +67,8 @@ public: ObArray &finish_tablet_ls_pair_array, ObArray &finish_tablet_ckm_array, compaction::ObUncompactInfo &uncompact_info, - ObFTSGroupArray &fts_group_array) + ObFTSGroupArray &fts_group_array, + share::ObCompactionLocalityCache &ls_locality_cache) : is_inited_(false), is_primary_service_(false), need_validate_index_ckm_(false), @@ -93,7 +94,8 @@ public: simple_schema_(nullptr), table_compaction_info_(), replica_ckm_items_(), - last_table_ckm_items_(tenant_id) + last_table_ckm_items_(tenant_id), + ls_locality_cache_(ls_locality_cache) {} ~ObChecksumValidator() {} int init( @@ -198,6 +200,7 @@ private: ObArray cur_tablet_ls_pair_array_; ObReplicaCkmItems replica_ckm_items_; compaction::ObTableCkmItems last_table_ckm_items_; // only cached last data table with index + share::ObCompactionLocalityCache &ls_locality_cache_; }; } // end namespace rootserver diff --git a/src/rootserver/freeze/ob_major_merge_progress_checker.cpp b/src/rootserver/freeze/ob_major_merge_progress_checker.cpp index f3ac87708..46912199c 100644 --- a/src/rootserver/freeze/ob_major_merge_progress_checker.cpp +++ b/src/rootserver/freeze/ob_major_merge_progress_checker.cpp @@ -50,11 +50,11 @@ ObMajorMergeProgressChecker::ObMajorMergeProgressChecker( loop_cnt_(0), last_errno_(OB_SUCCESS), tenant_id_(tenant_id), compaction_scn_(), expected_epoch_(OB_INVALID_ID), sql_proxy_(nullptr), schema_service_(nullptr), server_trace_(nullptr), progress_(), - tablet_status_map_(), table_compaction_map_(), fts_group_array_(), + tablet_status_map_(), table_compaction_map_(), fts_group_array_(), ls_locality_cache_(), ckm_validator_(tenant_id, stop_, tablet_ls_pair_cache_, tablet_status_map_, table_compaction_map_, idx_ckm_validate_array_, validator_statistics_, - finish_tablet_ls_pair_array_, finish_tablet_ckm_array_, uncompact_info_, fts_group_array_), - uncompact_info_(), ls_locality_cache_(), total_time_guard_(), validator_statistics_(), batch_size_mgr_() {} + finish_tablet_ls_pair_array_, finish_tablet_ckm_array_, uncompact_info_, fts_group_array_, ls_locality_cache_), + uncompact_info_(), total_time_guard_(), validator_statistics_(), batch_size_mgr_() {} int ObMajorMergeProgressChecker::init( const bool is_primary_service, @@ -437,13 +437,11 @@ int ObMajorMergeProgressChecker::prepare_check_progress( bool &exist_uncompacted_table) { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; exist_uncompacted_table = true; table_ids_.start_looping(); - if (OB_TMP_FAIL(ls_locality_cache_.refresh_ls_locality(first_loop_in_cur_round_ /*force_refresh*/))) { - LOG_WARN("failed to refresh ls locality", K(tmp_ret)); - } - if (first_loop_in_cur_round_) { + if (OB_FAIL(ls_locality_cache_.refresh_ls_locality(first_loop_in_cur_round_ /*force_refresh*/))) { + LOG_WARN("failed to refresh ls locality", K(ret)); + } else if (first_loop_in_cur_round_) { total_time_guard_.reuse(); if (OB_FAIL(prepare_unfinish_table_ids())) { LOG_WARN("fail to prepare table_id_map", KR(ret), K_(tenant_id)); diff --git a/src/rootserver/freeze/ob_major_merge_progress_checker.h b/src/rootserver/freeze/ob_major_merge_progress_checker.h index f8a02da91..e4ad4c7ad 100644 --- a/src/rootserver/freeze/ob_major_merge_progress_checker.h +++ b/src/rootserver/freeze/ob_major_merge_progress_checker.h @@ -149,10 +149,10 @@ private: // record each table compaction/verify status compaction::ObTableCompactionInfoMap table_compaction_map_; // ObFTSGroupArray fts_group_array_; + share::ObCompactionLocalityCache ls_locality_cache_; ObChecksumValidator ckm_validator_; compaction::ObUncompactInfo uncompact_info_; // cache of ls_infos in __all_ls_meta_table - share::ObCompactionLocalityCache ls_locality_cache_; // statistics section compaction::ObRSCompactionTimeGuard total_time_guard_; compaction::ObCkmValidatorStatistics validator_statistics_; diff --git a/src/rootserver/ob_admin_drtask_util.cpp b/src/rootserver/ob_admin_drtask_util.cpp index fdea3695b..b34f6040b 100644 --- a/src/rootserver/ob_admin_drtask_util.cpp +++ b/src/rootserver/ob_admin_drtask_util.cpp @@ -13,7 +13,6 @@ #define USING_LOG_PREFIX RS #include "ob_admin_drtask_util.h" #include "logservice/ob_log_service.h" // for ObLogService -#include "share/ob_locality_parser.h" // for ObLocalityParser #include "storage/tx_storage/ob_ls_service.h" // for ObLSService #include "storage/ls/ob_ls.h" // for ObLS #include "observer/ob_server_event_history_table_operator.h" // for SERVER_EVENT_ADD @@ -146,8 +145,8 @@ int ObAdminDRTaskUtil::construct_arg_for_add_command_( ret = OB_INVALID_ARGUMENT; ret_comment = ObAdminDRTaskRetComment::TENANT_ID_OR_LS_ID_NOT_VALID; LOG_WARN("invalid tenant_id or ls_id", KR(ret), K(command_arg), K(tenant_id), K(ls_id)); - } else if (OB_UNLIKELY(!target_server.is_valid()) - || OB_UNLIKELY(REPLICA_TYPE_FULL != replica_type && REPLICA_TYPE_READONLY != replica_type)) { + } else if (OB_UNLIKELY(!target_server.is_valid() + || !ObReplicaTypeCheck::is_replica_type_valid(replica_type))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(replica_type), K(target_server)); // STEP 2: construct orig_paxos_replica_number and leader_server if not specified by ob_admin command @@ -164,8 +163,8 @@ int ObAdminDRTaskUtil::construct_arg_for_add_command_( new_paxos_replica_number = 0 == new_paxos_replica_number ? orig_paxos_replica_number : new_paxos_replica_number; - ObReplicaMember data_source_member(leader_server, 0/*timstamp*/); - ObReplicaMember force_data_source_member(force_data_source_server, 0/*timstamp*/); + ObReplicaMember data_source_member(leader_server, 0/*timstamp*/, REPLICA_TYPE_FULL/*dummy_replica_type*/); + ObReplicaMember force_data_source_member(force_data_source_server, 0/*timstamp*/, REPLICA_TYPE_FULL/*dummy_replica_type*/); ObReplicaMember add_member(target_server, ObTimeUtility::current_time(), replica_type); // STEP 3: construct arg if (OB_ISNULL(ObCurTraceId::get_trace_id())) { @@ -292,8 +291,8 @@ int ObAdminDRTaskUtil::handle_remove_command_( ret = OB_INVALID_ARGUMENT; ret_comment = ObAdminDRTaskRetComment::TENANT_ID_OR_LS_ID_NOT_VALID; LOG_WARN("invalid tenant_id or ls_id", KR(ret), K(command_arg), K(tenant_id), K(ls_id)); - } else if (OB_UNLIKELY(!target_server.is_valid()) - || OB_UNLIKELY(REPLICA_TYPE_FULL != replica_type && REPLICA_TYPE_READONLY != replica_type)) { + } else if (OB_UNLIKELY(!target_server.is_valid() + || !ObReplicaTypeCheck::is_replica_type_valid(replica_type))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(replica_type), K(target_server)); } else { @@ -311,10 +310,10 @@ int ObAdminDRTaskUtil::handle_remove_command_( } else { ret_comment = SUCCEED_TO_SEND_COMMAND; } - } else if (REPLICA_TYPE_READONLY == replica_type) { + } else if (ObReplicaTypeCheck::is_non_paxos_replica(replica_type)) { ObLSDropNonPaxosReplicaArg remove_nonpaxos_arg; if (OB_FAIL(construct_remove_nonpaxos_task_arg_( - tenant_id, ls_id, target_server, ret_comment, remove_nonpaxos_arg))) { + tenant_id, ls_id, target_server, replica_type, ret_comment, remove_nonpaxos_arg))) { LOG_WARN("fail to construct remove non-paxos replica task arg", KR(ret), K(tenant_id), K(ls_id), K(target_server), K(ret_comment), K(remove_nonpaxos_arg)); } else if (OB_FAIL(execute_remove_nonpaxos_task_(command_arg, remove_nonpaxos_arg))) { @@ -359,20 +358,17 @@ int ObAdminDRTaskUtil::construct_remove_paxos_task_arg_( LOG_WARN("replica not found in member_list", KR(ret), K(target_server), K(palf_stat)); } else if (OB_FAIL(palf_stat.paxos_member_list_.get_member_by_addr(target_server, member))) { LOG_WARN("fail to get member from paxos_member_list", KR(ret), K(palf_stat), K(target_server)); + } else if (OB_FAIL(member_to_remove.init(member, REPLICA_TYPE_FULL))) { + LOG_WARN("fail to init member_to_remove", KR(ret), K(member)); } else { - member_to_remove = ObReplicaMember(member); - if (OB_FAIL(member_to_remove.set_replica_type(REPLICA_TYPE_FULL))) { - LOG_WARN("fail to set replica type for member to remove", KR(ret)); - } else { - // If [orig_paxos_replica_number] not specified in obadmin command, - // use leader replica's info as default - orig_paxos_replica_number = 0 == orig_paxos_replica_number - ? palf_stat.paxos_replica_num_ - : orig_paxos_replica_number; - new_paxos_replica_number = 0 == new_paxos_replica_number - ? orig_paxos_replica_number - : new_paxos_replica_number; - } + // If [orig_paxos_replica_number] not specified in obadmin command, + // use leader replica's info as default + orig_paxos_replica_number = 0 == orig_paxos_replica_number + ? palf_stat.paxos_replica_num_ + : orig_paxos_replica_number; + new_paxos_replica_number = 0 == new_paxos_replica_number + ? orig_paxos_replica_number + : new_paxos_replica_number; } if (OB_FAIL(ret)) { } else if (OB_ISNULL(ObCurTraceId::get_trace_id())) { @@ -391,6 +387,7 @@ int ObAdminDRTaskUtil::construct_remove_nonpaxos_task_arg_( const uint64_t &tenant_id, const share::ObLSID &ls_id, const common::ObAddr &target_server, + const ObReplicaType &replica_type, ObAdminDRTaskRetComment &ret_comment, ObLSDropNonPaxosReplicaArg &remove_nonpaxos_arg) { @@ -401,9 +398,11 @@ int ObAdminDRTaskUtil::construct_remove_nonpaxos_task_arg_( palf::PalfStat palf_stat; if (OB_UNLIKELY(!ls_id.is_valid_with_tenant(tenant_id)) - || OB_UNLIKELY(!target_server.is_valid())) { + || OB_UNLIKELY(!target_server.is_valid()) + || OB_UNLIKELY(!ObReplicaTypeCheck::is_non_paxos_replica(replica_type))) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid tenant_id or ls_id", KR(ret), K(tenant_id), K(ls_id), K(target_server)); + LOG_WARN("invalid arguments for remove_non_paxos_task", KR(ret), K(tenant_id), K(ls_id), + K(target_server), K(replica_type)); } else if (OB_FAIL(get_local_palf_stat_(tenant_id, ls_id, palf_stat, ret_comment))) { LOG_WARN("fail to get local palf stat", KR(ret), K(tenant_id), K(ls_id)); } else if (OB_UNLIKELY(!palf_stat.is_valid())) { @@ -414,18 +413,15 @@ int ObAdminDRTaskUtil::construct_remove_nonpaxos_task_arg_( LOG_WARN("replica not found in learner_list", KR(ret), K(target_server), K(palf_stat)); } else if (OB_FAIL(palf_stat.learner_list_.get_learner_by_addr(target_server, member))) { LOG_WARN("fail to get member from learner_list", KR(ret), K(palf_stat), K(target_server)); - } else { - member_to_remove = ObReplicaMember(member); - if (OB_FAIL(member_to_remove.set_replica_type(REPLICA_TYPE_READONLY))) { - LOG_WARN("fail to set replica type for member to remove", KR(ret)); - } else if (OB_ISNULL(ObCurTraceId::get_trace_id())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret)); - } else if (OB_FAIL(remove_nonpaxos_arg.init( - *ObCurTraceId::get_trace_id()/*task_id*/, tenant_id, - ls_id, member_to_remove))) { - LOG_WARN("fail to init arg", KR(ret), K(tenant_id), K(ls_id), K(member_to_remove)); - } + } else if (OB_FAIL(member_to_remove.init(member, replica_type))) { + LOG_WARN("fail to init member_to_remove", KR(ret), K(member), K(replica_type)); + } else if (OB_ISNULL(ObCurTraceId::get_trace_id())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret)); + } else if (OB_FAIL(remove_nonpaxos_arg.init( + *ObCurTraceId::get_trace_id()/*task_id*/, tenant_id, + ls_id, member_to_remove))) { + LOG_WARN("fail to init arg", KR(ret), K(tenant_id), K(ls_id), K(member_to_remove)); } return ret; } @@ -594,11 +590,10 @@ int ObAdminDRTaskUtil::parse_params_from_obadmin_command_arg( ls_id = share::ObLSID(ls_id_to_set); } } else if (0 == param_name.string().case_compare("replica_type")) { - if (OB_FAIL(share::ObLocalityParser::parse_type( - param_value.ptr(), - param_value.length(), - replica_type))) { - LOG_WARN("fail to parse replica type", KR(ret), K(param_name_with_value), K(replica_type)); + replica_type = share::ObShareUtil::string_to_replica_type(param_value.ptr()); + if (! ObReplicaTypeCheck::is_replica_type_valid(replica_type)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid replica_type", KR(ret), K(param_name_with_value), K(replica_type)); } } else if (0 == param_name.string().case_compare("orig_paxos_replica_number")) { if (OB_FAIL(extract_int(param_value.string(), 0, pos, orig_paxos_replica_number))) { diff --git a/src/rootserver/ob_admin_drtask_util.h b/src/rootserver/ob_admin_drtask_util.h index d42f7e6a6..be3488231 100644 --- a/src/rootserver/ob_admin_drtask_util.h +++ b/src/rootserver/ob_admin_drtask_util.h @@ -89,12 +89,14 @@ private: // params[in] tenant_id, specified tenant_id // params[in] ls_id, specified ls_id // params[in] target_server, the replica to remove on which server + // params[in] replica_type, the replica type to remove, could be R or C // params[out] ret_comment, failed reason // params[out] remove_non_paxos_arg, arg for remove-R task static int construct_remove_nonpaxos_task_arg_( const uint64_t &tenant_id, const share::ObLSID &ls_id, const common::ObAddr &target_server, + const ObReplicaType &replica_type, ObAdminDRTaskRetComment &ret_comment, ObLSDropNonPaxosReplicaArg &remove_nonpaxos_arg); diff --git a/src/rootserver/ob_balance_info.h b/src/rootserver/ob_balance_info.h index 6c531f92a..ee159f9f3 100644 --- a/src/rootserver/ob_balance_info.h +++ b/src/rootserver/ob_balance_info.h @@ -44,19 +44,6 @@ class ObUnitManager; class ObZoneManager; -class ObDataSourceCandidateChecker -{ -public: - ObDataSourceCandidateChecker(common::ObReplicaType type) : this_type_(type) {} - inline bool is_candidate(common::ObReplicaType other_type) const - { - // TODO: Use a more refined way, such as this_type_ = F, you can have other_type = R - return common::ObReplicaTypeCheck::can_as_data_source(this_type_, other_type); - } -private: - common::ObReplicaType this_type_; -}; - class ObStatisticsCalculator { public: diff --git a/src/rootserver/ob_ddl_service.cpp b/src/rootserver/ob_ddl_service.cpp index f5f41395f..4ee875a8c 100755 --- a/src/rootserver/ob_ddl_service.cpp +++ b/src/rootserver/ob_ddl_service.cpp @@ -3413,15 +3413,22 @@ int ObDDLService::set_raw_table_options( } int ObDDLService::check_locality_compatible_( - ObTenantSchema &schema) + ObTenantSchema &schema, + const bool for_create_tenant) { int ret = OB_SUCCESS; common::ObArray zone_locality; + const uint64_t tenant_id = for_create_tenant ? OB_SYS_TENANT_ID : schema.get_tenant_id(); bool is_compatible_with_readonly_replica = false; + bool is_compatible_with_columnstore_replica = false; if (OB_FAIL(ObShareUtil::check_compat_version_for_readonly_replica( - schema.get_tenant_id(), is_compatible_with_readonly_replica))) { + tenant_id, is_compatible_with_readonly_replica))) { LOG_WARN("fail to check compatible with readonly replica", KR(ret), K(schema)); - } else if (is_compatible_with_readonly_replica) { + } else if (OB_FAIL(ObShareUtil::check_compat_version_for_columnstore_replica( + tenant_id, is_compatible_with_columnstore_replica))) { + LOG_WARN("fail to check compatible with columnstore replica", KR(ret), K(schema)); + } else if (is_compatible_with_readonly_replica && is_compatible_with_columnstore_replica) { + // check pass } else if (OB_FAIL(schema.get_zone_replica_attr_array(zone_locality))) { LOG_WARN("fail to get locality from schema", K(ret), K(schema)); } else { @@ -3430,10 +3437,16 @@ int ObDDLService::check_locality_compatible_( if (this_set.zone_set_.count() <= 0) { ret = OB_ERR_UNEXPECTED; LOG_WARN("zone set count unexpected", K(ret), "zone_set_cnt", this_set.zone_set_.count()); - } else if (0 != this_set.get_readonly_replica_num()) { + } else if (! is_compatible_with_readonly_replica + && 0 != this_set.get_readonly_replica_num()) { ret = OB_NOT_SUPPORTED; LOG_WARN("can not create tenant with read-only replica below data version 4.2", KR(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "Create tenant with R-replica in locality below data version 4.2"); + } else if (! is_compatible_with_columnstore_replica + && 0 != this_set.get_columnstore_replica_num()) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("can not create tenant with column-store replica below data version 4.3.3", KR(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "Create tenant with C-replica in locality below data version 4.3.3"); } } } @@ -27298,7 +27311,7 @@ int ObDDLService::set_new_tenant_options( } else if (OB_FAIL(parse_and_set_create_tenant_new_locality_options( schema_guard, new_tenant_schema, resource_pool_names, zones_in_pool, zone_region_list))) { LOG_WARN("fail to parse and set new locality option", K(ret)); - } else if (OB_FAIL(check_locality_compatible_(new_tenant_schema))) { + } else if (OB_FAIL(check_locality_compatible_(new_tenant_schema, false /*for_create_tenant*/))) { LOG_WARN("fail to check locality with data version", KR(ret), K(new_tenant_schema)); } else if (OB_FAIL(check_alter_tenant_locality_type( schema_guard, orig_tenant_schema, new_tenant_schema, alter_locality_type))) { @@ -30301,6 +30314,8 @@ int ObDDLService::check_create_tenant_locality( } else if (OB_FAIL(parse_and_set_create_tenant_new_locality_options( schema_guard, tenant_schema, pools, pool_zones, zone_region_list))) { LOG_WARN("fail to parse and set new locality option", K(ret)); + } else if (OB_FAIL(check_locality_compatible_(tenant_schema, true /*for_create_tenant*/))) { + LOG_WARN("fail to check locality with data version", KR(ret), K(tenant_schema)); } else if (OB_FAIL(check_pools_unit_num_enough_for_schema_locality( pools, schema_guard, tenant_schema))) { LOG_WARN("pools unit num is not enough for locality", K(ret)); diff --git a/src/rootserver/ob_ddl_service.h b/src/rootserver/ob_ddl_service.h index 9e4b62b28..f67c70144 100644 --- a/src/rootserver/ob_ddl_service.h +++ b/src/rootserver/ob_ddl_service.h @@ -2753,7 +2753,7 @@ private: common::ObIArray &init_configs); private: - int check_locality_compatible_(ObTenantSchema &schema); + int check_locality_compatible_(ObTenantSchema &schema, const bool for_create_tenant); int pre_rename_mysql_columns_online(const ObTableSchema &origin_table_schema, const AlterTableSchema &alter_table_schema, diff --git a/src/rootserver/ob_disaster_recovery_task.cpp b/src/rootserver/ob_disaster_recovery_task.cpp index 1ba8c83b3..834a42bce 100644 --- a/src/rootserver/ob_disaster_recovery_task.cpp +++ b/src/rootserver/ob_disaster_recovery_task.cpp @@ -254,34 +254,6 @@ const char *ob_disaster_recovery_task_type_strs(const rootserver::ObDRTaskType t return str; } -const char *ob_replica_type_strs(const ObReplicaType type) -{ - const char *str = NULL; - switch (type) { - case ObReplicaType::REPLICA_TYPE_FULL: { - str = "FULL"; - break; - } - case ObReplicaType::REPLICA_TYPE_LOGONLY: { - str = "LOGONLY"; - break; - } - case ObReplicaType::REPLICA_TYPE_READONLY: { - str = "READONLY"; - break; - } - case ObReplicaType::REPLICA_TYPE_ENCRYPTION_LOGONLY: { - str = "ENCRYPTION_LOGONLY"; - break; - } - default: { - LOG_WARN_RET(OB_ERR_UNEXPECTED, "invalid replica type", K(type)); - break; - } - } - return str; -} - bool ObDRTaskKey::is_valid() const { return key_type_ > ObDRTaskKeyType::INVALID @@ -558,8 +530,7 @@ int ObMigrateLSReplicaTask::get_execute_transmit_size( int ret = OB_SUCCESS; execute_transmit_size = 0; ObReplicaType dst_replica_type = dst_replica_.get_replica_type(); - if (REPLICA_TYPE_FULL == dst_replica_type - || REPLICA_TYPE_READONLY == dst_replica_type) { + if (ObReplicaTypeCheck::is_replica_type_valid(dst_replica_type)) { execute_transmit_size = transmit_data_size_; } else if (REPLICA_TYPE_LOGONLY == dst_replica_type || REPLICA_TYPE_ENCRYPTION_LOGONLY == dst_replica_type) { @@ -717,11 +688,11 @@ int ObMigrateLSReplicaTask::fill_dml_splicer( || OB_FAIL(dml_splicer.add_column("target_replica_svr_ip", dest_ip)) || OB_FAIL(dml_splicer.add_column("target_replica_svr_port", get_dst_server().get_port())) || OB_FAIL(dml_splicer.add_column("target_paxos_replica_number", get_paxos_replica_number())) - || OB_FAIL(dml_splicer.add_column("target_replica_type", ob_replica_type_strs(get_dst_replica().get_member().get_replica_type()))) + || OB_FAIL(dml_splicer.add_column("target_replica_type", ObShareUtil::replica_type_to_string(get_dst_replica().get_member().get_replica_type()))) || OB_FAIL(dml_splicer.add_column("source_replica_svr_ip", src_ip)) || OB_FAIL(dml_splicer.add_column("source_replica_svr_port", get_src_member().get_server().get_port())) || OB_FAIL(dml_splicer.add_column("source_paxos_replica_number", get_paxos_replica_number())) - || OB_FAIL(dml_splicer.add_column("source_replica_type", ob_replica_type_strs(get_src_member().get_replica_type()))) + || OB_FAIL(dml_splicer.add_column("source_replica_type", ObShareUtil::replica_type_to_string(get_src_member().get_replica_type()))) || OB_FAIL(dml_splicer.add_column("task_exec_svr_ip", dest_ip)) || OB_FAIL(dml_splicer.add_column("task_exec_svr_port", get_dst_server().get_port()))) { LOG_WARN("add column failed", KR(ret)); @@ -991,6 +962,7 @@ int ObMigrateLSReplicaTask::build_task_from_sql_result( (void)GET_COL_IGNORE_NULL(res.get_int, "target_replica_svr_port", dest_port); (void)GET_COL_IGNORE_NULL(res.get_int, "source_paxos_replica_number", src_paxos_replica_number); (void)GET_COL_IGNORE_NULL(res.get_varchar, "comment", comment); + // TODO(cangming.zl): get src_replica_type and dest_replica_type from result EXTRACT_INT_FIELD_MYSQL_WITH_DEFAULT_VALUE(res, "data_source_svr_port", data_source_port, int64_t, true/*skip null error*/, true/*skip column error*/, 0); EXTRACT_VARCHAR_FIELD_MYSQL_WITH_DEFAULT_VALUE(res, "data_source_svr_ip", data_source_ip, @@ -1081,8 +1053,7 @@ int ObAddLSReplicaTask::get_execute_transmit_size( int ret = OB_SUCCESS; execute_transmit_size = 0; ObReplicaType dst_replica_type = dst_replica_.get_replica_type(); - if (REPLICA_TYPE_FULL == dst_replica_type - || REPLICA_TYPE_READONLY == dst_replica_type) { + if (ObReplicaTypeCheck::is_replica_type_valid(dst_replica_type)) { execute_transmit_size = transmit_data_size_; } else if (REPLICA_TYPE_LOGONLY == dst_replica_type || REPLICA_TYPE_ENCRYPTION_LOGONLY == dst_replica_type) { @@ -1224,11 +1195,11 @@ int ObAddLSReplicaTask::fill_dml_splicer( || OB_FAIL(dml_splicer.add_column("target_replica_svr_ip", dest_ip)) || OB_FAIL(dml_splicer.add_column("target_replica_svr_port", get_dst_server().get_port())) || OB_FAIL(dml_splicer.add_column("target_paxos_replica_number", get_paxos_replica_number())) - || OB_FAIL(dml_splicer.add_column("target_replica_type", ob_replica_type_strs(get_dst_replica().get_member().get_replica_type()))) + || OB_FAIL(dml_splicer.add_column("target_replica_type", ObShareUtil::replica_type_to_string(get_dst_replica().get_member().get_replica_type()))) || OB_FAIL(dml_splicer.add_column("source_replica_svr_ip", src_ip)) || OB_FAIL(dml_splicer.add_column("source_replica_svr_port", get_data_src_member().get_server().get_port())) || OB_FAIL(dml_splicer.add_column("source_paxos_replica_number", get_orig_paxos_replica_number())) - || OB_FAIL(dml_splicer.add_column("source_replica_type", ob_replica_type_strs(get_dst_replica().get_member().get_replica_type()))) + || OB_FAIL(dml_splicer.add_column("source_replica_type", ObShareUtil::replica_type_to_string(get_dst_replica().get_member().get_replica_type()))) || OB_FAIL(dml_splicer.add_column("task_exec_svr_ip", dest_ip)) || OB_FAIL(dml_splicer.add_column("task_exec_svr_port", get_dst_server().get_port()))) { LOG_WARN("add column failed", KR(ret)); @@ -1505,6 +1476,7 @@ int ObAddLSReplicaTask::build_task_from_sql_result( (void)GET_COL_IGNORE_NULL(res.get_int, "source_paxos_replica_number", src_paxos_replica_number); (void)GET_COL_IGNORE_NULL(res.get_int, "target_paxos_replica_number", dest_paxos_replica_number); (void)GET_COL_IGNORE_NULL(res.get_varchar, "comment", comment); + // TODO(cangming.zl): get src_replica_type and dest_replica_type from result EXTRACT_INT_FIELD_MYSQL_WITH_DEFAULT_VALUE(res, "data_source_svr_port", data_source_port, int64_t, true/*skip null error*/, true/*skip column error*/, 0); EXTRACT_VARCHAR_FIELD_MYSQL_WITH_DEFAULT_VALUE(res, "data_source_svr_ip", data_source_ip, @@ -1751,11 +1723,11 @@ int ObLSTypeTransformTask::fill_dml_splicer( || OB_FAIL(dml_splicer.add_column("target_replica_svr_ip", dest_ip)) || OB_FAIL(dml_splicer.add_column("target_replica_svr_port", get_dst_server().get_port())) || OB_FAIL(dml_splicer.add_column("target_paxos_replica_number", get_paxos_replica_number())) - || OB_FAIL(dml_splicer.add_column("target_replica_type", ob_replica_type_strs(get_dst_replica().get_member().get_replica_type()))) + || OB_FAIL(dml_splicer.add_column("target_replica_type", ObShareUtil::replica_type_to_string(get_dst_replica().get_member().get_replica_type()))) || OB_FAIL(dml_splicer.add_column("source_replica_svr_ip", src_ip)) || OB_FAIL(dml_splicer.add_column("source_replica_svr_port", get_src_member().get_server().get_port())) || OB_FAIL(dml_splicer.add_column("source_paxos_replica_number", get_orig_paxos_replica_number())) - || OB_FAIL(dml_splicer.add_column("source_replica_type", ob_replica_type_strs(get_src_member().get_replica_type()))) + || OB_FAIL(dml_splicer.add_column("source_replica_type", ObShareUtil::replica_type_to_string(get_src_member().get_replica_type()))) || OB_FAIL(dml_splicer.add_column("task_exec_svr_ip", dest_ip)) || OB_FAIL(dml_splicer.add_column("task_exec_svr_port", get_dst_server().get_port()))) { LOG_WARN("add column failed", KR(ret)); @@ -1985,8 +1957,8 @@ int ObLSTypeTransformTask::build_task_from_sql_result( int64_t transmit_data_size = 0; int64_t src_paxos_replica_number = OB_INVALID_COUNT; int64_t dest_paxos_replica_number = OB_INVALID_COUNT; - common::ObString src_type; - common::ObString dest_type; + common::ObString src_type_str; + common::ObString dest_type_str; int64_t schedule_time_us = 0; int64_t generate_time_us = 0; common::ObString comment; @@ -2010,8 +1982,8 @@ int ObLSTypeTransformTask::build_task_from_sql_result( (void)GET_COL_IGNORE_NULL(res.get_int, "target_replica_svr_port", dest_port); (void)GET_COL_IGNORE_NULL(res.get_int, "source_paxos_replica_number", src_paxos_replica_number); (void)GET_COL_IGNORE_NULL(res.get_int, "target_paxos_replica_number", dest_paxos_replica_number); - (void)GET_COL_IGNORE_NULL(res.get_varchar, "source_replica_type", src_type); - (void)GET_COL_IGNORE_NULL(res.get_varchar, "target_replica_type", dest_type); + (void)GET_COL_IGNORE_NULL(res.get_varchar, "source_replica_type", src_type_str); + (void)GET_COL_IGNORE_NULL(res.get_varchar, "target_replica_type", dest_type_str); (void)GET_COL_IGNORE_NULL(res.get_varchar, "comment", comment); EXTRACT_BOOL_FIELD_MYSQL_SKIP_RET(res, "is_manual", is_manual); //STEP2_0: make necessary members to build a task @@ -2020,8 +1992,8 @@ int ObLSTypeTransformTask::build_task_from_sql_result( common::ObAddr dest_server; common::ObString zone; rootserver::ObDRTaskPriority priority_to_set; - ObReplicaType src_type_to_set = REPLICA_TYPE_MAX; - ObReplicaType dest_type_to_set = REPLICA_TYPE_MAX; + ObReplicaType src_type_to_set = ObShareUtil::string_to_replica_type(src_type_str); + ObReplicaType dest_type_to_set = ObShareUtil::string_to_replica_type(dest_type_str); ObDstReplica dst_replica; share::ObTaskId task_id_to_set; ObSqlString comment_to_set; @@ -2048,18 +2020,6 @@ int ObLSTypeTransformTask::build_task_from_sql_result( ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid server address", K(dest_ip), K(dest_port)); } else { - //transform replica_type(string) -> src_type_to_set(ObReplicaType) - if (src_type == common::ObString("FULL")) { - src_type_to_set = REPLICA_TYPE_FULL; - } else if (src_type == common::ObString("READONLY")) { - src_type_to_set = REPLICA_TYPE_READONLY; - } - //transform replica_type(string) -> dest_type_to_set(ObReplicaType) - if (dest_type == common::ObString("FULL")) { - dest_type_to_set = REPLICA_TYPE_FULL; - } else if (dest_type == common::ObString("READONLY")) { - dest_type_to_set = REPLICA_TYPE_READONLY; - } //transform priority(int) -> priority_to_set(ObDRTaskPriority) if (priority == 0) { priority_to_set = ObDRTaskPriority::HIGH_PRI; @@ -2068,12 +2028,12 @@ int ObLSTypeTransformTask::build_task_from_sql_result( } else { priority_to_set = ObDRTaskPriority::MAX_PRI; } - ObReplicaMember src_member(src_server, 0); - ObReplicaMember dest_member(dest_server, 0); - if (OB_FAIL(src_member.set_replica_type(src_type_to_set))) { - LOG_WARN("fail to set src replica type", KR(ret), K(src_type_to_set)); - } else if (OB_FAIL(dest_member.set_replica_type(dest_type_to_set))) { - LOG_WARN("fail to set dest replica type", KR(ret), K(dest_type_to_set)); + ObReplicaMember src_member; + ObReplicaMember dest_member; + if (OB_FAIL(src_member.init(src_server, 0, src_type_to_set))) { + LOG_WARN("failed to init src_member", KR(ret), K(src_server), K(src_type_str), K(src_type_to_set)); + } else if (OB_FAIL(dest_member.init(dest_server, 0, dest_type_to_set))) { + LOG_WARN("failed to init dest_member", KR(ret), K(dest_server), K(dest_type_str), K(dest_type_to_set)); } else if (OB_FAIL(dst_replica.assign( 0/*unit id*/, 0/*unit group id*/, @@ -2251,7 +2211,7 @@ int ObRemoveLSReplicaTask::fill_dml_splicer( || OB_FAIL(dml_splicer.add_column("target_replica_svr_ip", target_ip)) || OB_FAIL(dml_splicer.add_column("target_replica_svr_port", get_remove_server().get_server().get_port())) || OB_FAIL(dml_splicer.add_column("target_paxos_replica_number", get_paxos_replica_number())) - || OB_FAIL(dml_splicer.add_column("target_replica_type", ob_replica_type_strs(get_remove_server().get_replica_type()))) + || OB_FAIL(dml_splicer.add_column("target_replica_type", ObShareUtil::replica_type_to_string(get_remove_server().get_replica_type()))) || OB_FAIL(dml_splicer.add_column("source_replica_svr_ip", src_ip)) || OB_FAIL(dml_splicer.add_column("source_replica_svr_port", 0)) || OB_FAIL(dml_splicer.add_column("source_paxos_replica_number", get_orig_paxos_replica_number())) @@ -2374,7 +2334,7 @@ int ObRemoveLSReplicaTask::simple_build( || !remove_server.is_valid() || orig_paxos_replica_number <= 0 || paxos_replica_number <= 0 - || REPLICA_TYPE_MAX == replica_type)) { + || !ObReplicaTypeCheck::is_replica_type_valid(replica_type))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(ls_id), K(task_id), K(leader), K(remove_server), K(orig_paxos_replica_number), @@ -2422,7 +2382,7 @@ int ObRemoveLSReplicaTask::build_task_from_sql_result( int64_t schedule_time_us = 0; int64_t generate_time_us = 0; common::ObString comment; - ObReplicaType replica_type = REPLICA_TYPE_MAX; + common::ObString replica_type_str; bool is_manual = false; //STEP1_0: read certain members from sql result EXTRACT_INT_FIELD_MYSQL(res, "tenant_id", tenant_id, uint64_t); @@ -2442,6 +2402,7 @@ int ObRemoveLSReplicaTask::build_task_from_sql_result( (void)GET_COL_IGNORE_NULL(res.get_int, "task_exec_svr_port", dest_port); (void)GET_COL_IGNORE_NULL(res.get_varchar, "target_replica_svr_ip", target_ip); (void)GET_COL_IGNORE_NULL(res.get_int, "target_replica_svr_port", target_port); + (void)GET_COL_IGNORE_NULL(res.get_varchar, "target_replica_type", replica_type_str); (void)GET_COL_IGNORE_NULL(res.get_int, "source_paxos_replica_number", src_paxos_replica_number); (void)GET_COL_IGNORE_NULL(res.get_int, "target_paxos_replica_number", dest_paxos_replica_number); (void)GET_COL_IGNORE_NULL(res.get_varchar, "comment", comment); @@ -2453,7 +2414,7 @@ int ObRemoveLSReplicaTask::build_task_from_sql_result( rootserver::ObDRTaskPriority priority_to_set; share::ObTaskId task_id_to_set; ObSqlString comment_to_set; - ObSqlString task_id_sqlstring_format; + ObSqlString task_id_sqlstring_format; // for adding trailing null if (OB_FAIL(ret)) { } else if (OB_FAIL(comment_to_set.assign(comment))) { @@ -2484,13 +2445,21 @@ int ObRemoveLSReplicaTask::build_task_from_sql_result( } else { priority_to_set = ObDRTaskPriority::MAX_PRI; } - //transform task_type(string) -> replica_type(ObReplicaType) + // get replica_type_ from replica_type_str and check whether or not match with task_type. + replica_type_ = ObShareUtil::string_to_replica_type(replica_type_str); if (0 == task_type.case_compare(ob_disaster_recovery_task_type_strs(ObDRTaskType::LS_REMOVE_PAXOS_REPLICA))) { - replica_type_ = ObReplicaType::REPLICA_TYPE_FULL; + if (OB_UNLIKELY(REPLICA_TYPE_FULL != replica_type_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("task_type and replica_type do not match", KR(ret), K(task_type), K(replica_type_)); + } } else if (0 == task_type.case_compare(ob_disaster_recovery_task_type_strs(ObDRTaskType::LS_REMOVE_NON_PAXOS_REPLICA))) { - replica_type_ = ObReplicaType::REPLICA_TYPE_READONLY; + if (OB_UNLIKELY(! ObReplicaTypeCheck::is_non_paxos_replica(replica_type_))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("task_type and replica_type do not match", KR(ret), K(task_type), K(replica_type_)); + } } else { - replica_type_ = ObReplicaType::REPLICA_TYPE_MAX; + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected task_type", KR(ret), K(task_type)); } } //STEP3_0: to build a task diff --git a/src/rootserver/ob_disaster_recovery_worker.cpp b/src/rootserver/ob_disaster_recovery_worker.cpp index 678796cc0..df03fde97 100755 --- a/src/rootserver/ob_disaster_recovery_worker.cpp +++ b/src/rootserver/ob_disaster_recovery_worker.cpp @@ -128,7 +128,7 @@ bool ObLSReplicaTaskDisplayInfo::is_valid() const && task_type_ != ObDRTaskType::MAX_TYPE && task_priority_ != ObDRTaskPriority::MAX_PRI && target_server_.is_valid() - && target_replica_type_ != REPLICA_TYPE_MAX + && target_replica_type_ != REPLICA_TYPE_INVALID && target_replica_paxos_replica_number_ != OB_INVALID_COUNT && source_replica_paxos_replica_number_ != OB_INVALID_COUNT && execute_server_.is_valid(); @@ -293,6 +293,9 @@ int ObDRWorker::LocalityAlignment::build_locality_stat_map() // readonly locality const ObIArray &readonly_locality = zone_locality.replica_attr_set_.get_readonly_replica_attr_array(); + // columnstore locality + const ObIArray &columnstore_locality = + zone_locality.replica_attr_set_.get_columnstore_replica_attr_array(); if (OB_FAIL(locate_zone_locality(zone, zone_replica_desc))) { LOG_WARN("fail to locate zone locality", KR(ret), K(zone)); @@ -336,11 +339,17 @@ int ObDRWorker::LocalityAlignment::build_locality_stat_map() LOG_WARN("fail to push back", KR(ret)); } } - // readonly replica, all_server + // readonly or colmnstore replica, all_server if (dr_ls_info_.is_duplicate_ls()) { - // duplicate ls, should has R-replica all_server - zone_replica_desc->is_readonly_all_server_ = true; - zone_replica_desc->readonly_memstore_percent_ = 100; + // duplicate ls, should has R-replica or C-replica all_server + // if locality is C, then set columnstore_all_server. + // if locality is F/R, then set readonly_all_server, + if (columnstore_locality.count() > 0) { + // dup_ls must not be sys_ls + zone_replica_desc->set_columnstore_all_server(); + } else { + zone_replica_desc->set_readonly_all_server(); + } } else { // readonly replica, normal for (int64_t j = 0; OB_SUCC(ret) && j < readonly_locality.count(); ++j) { @@ -354,6 +363,22 @@ int ObDRWorker::LocalityAlignment::build_locality_stat_map() LOG_WARN("fail to push back", KR(ret)); } } + // columnstore replica, normal + if (ls_id.is_sys_ls()) { + // for sys ls, ignore C-replica in locality + } else { + for (int64_t j = 0; OB_SUCC(ret) && j < columnstore_locality.count(); ++j) { + const ReplicaAttr &replica_attr = columnstore_locality.at(j); + if (0 >= replica_attr.num_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("replica num unexpected", KR(ret), K(zone), K(columnstore_locality)); + } else if (OB_FAIL(zone_replica_desc->push_back(ReplicaDesc(REPLICA_TYPE_COLUMNSTORE, + replica_attr.memstore_percent_, + replica_attr.num_)))) { + LOG_WARN("fail to push back", KR(ret)); + } + } + } } } LOG_INFO("ls zone locality map info", K(zone), KPC(zone_replica_desc)); @@ -592,6 +617,16 @@ int ObDRWorker::LocalityAlignment::do_generate_locality_task_from_full_replica( } else { found = true; } + } else if (ObReplicaTypeCheck::is_columnstore_replica(replica_desc.replica_type_)) { + // if zone_locality is C-replica, remove this F-replica + // because transform from F to C is not supported + if (OB_FAIL(generate_remove_replica_task(replica_stat_desc))) { + LOG_WARN("fail to generate remove replica task", KR(ret)); + } else if (OB_FAIL(replica_stat_map_.remove(index))) { + LOG_WARN("fail to remove", KR(ret)); + } else { + found = true; + } } } // process not found @@ -599,11 +634,18 @@ int ObDRWorker::LocalityAlignment::do_generate_locality_task_from_full_replica( // failed } else if (found) { // found, bypass - } else if (zone_replica_desc->is_readonly_all_server_) { + } else if (zone_replica_desc->is_columnstore_all_server()) { + // remove this F, because transform from F to C is not supported + if (OB_FAIL(generate_remove_replica_task(replica_stat_desc))) { + LOG_WARN("fail to generate remove replica task", KR(ret)); + } else if (OB_FAIL(replica_stat_map_.remove(index))) { + LOG_WARN("fail to remove", KR(ret)); + } + } else if (zone_replica_desc->is_readonly_all_server()) { if (OB_FAIL(generate_type_transform_task( replica_stat_desc, REPLICA_TYPE_READONLY, - zone_replica_desc->readonly_memstore_percent_))) { + zone_replica_desc->get_readonly_memstore_percent()))) { LOG_WARN("fail to generate type transform task", KR(ret), K(replica_stat_desc)); } else if (OB_FAIL(replica_stat_map_.remove(index))) { LOG_WARN("fail to remove", KR(ret), K(index), K(replica), K(replica_stat_map_)); @@ -715,7 +757,7 @@ int ObDRWorker::LocalityAlignment::do_generate_locality_task_from_encryption_log return ret; } -int ObDRWorker::LocalityAlignment::try_generate_type_transform_task_for_readonly_replica_( +int ObDRWorker::LocalityAlignment::try_generate_task_for_readonly_replica_( ReplicaDescArray &zone_replica_desc_in_locality, ReplicaStatDesc &replica_stat_desc, const int64_t index, @@ -732,6 +774,15 @@ int ObDRWorker::LocalityAlignment::try_generate_type_transform_task_for_readonly if (REPLICA_TYPE_READONLY == replica_desc.replica_type_) { ret = OB_ERR_UNEXPECTED; LOG_WARN("replica type unexpected", KR(ret), K(dr_ls_info_)); + } else if (ObReplicaTypeCheck::is_columnstore_replica(replica_desc.replica_type_)) { + // if locality is C-replica, remove this R-replica because transform from R to C is not supported + if (OB_FAIL(generate_remove_replica_task(replica_stat_desc))) { + LOG_WARN("fail to generate remove replica task", KR(ret), K(replica_stat_desc)); + } else if (OB_FAIL(replica_stat_map_.remove(index))) { + LOG_WARN("fail to remove", KR(ret), K(replica_stat_map_), K(index)); + } else { + task_generated = true; + } } else if (REPLICA_TYPE_FULL == replica_desc.replica_type_) { if (OB_ISNULL(replica_stat_desc.unit_stat_info_)) { ret = OB_INVALID_ARGUMENT; @@ -764,7 +815,7 @@ int ObDRWorker::LocalityAlignment::try_generate_type_transform_task_for_readonly return ret; } -int ObDRWorker::LocalityAlignment::try_generate_remove_readonly_task_for_duplicate_log_stream_( +int ObDRWorker::LocalityAlignment::try_generate_remove_redundant_replica_task_for_dup_ls_( ReplicaStatDesc &replica_stat_desc, share::ObLSReplica &replica, const int64_t index) @@ -829,15 +880,23 @@ int ObDRWorker::LocalityAlignment::do_generate_locality_task_from_readonly_repli bool task_generated = false; lib::ob_sort(zone_replica_desc->begin(), zone_replica_desc->end()); // try to generate type_transform task if needed - if (OB_FAIL(try_generate_type_transform_task_for_readonly_replica_( + if (OB_FAIL(try_generate_task_for_readonly_replica_( *zone_replica_desc, replica_stat_desc, index, task_generated))) { LOG_WARN("fail to try generate type transform task", KR(ret), KPC(zone_replica_desc), K(replica_stat_desc), K(index), K(task_generated)); } else if (task_generated) { - // a type transform task generated, bypass - } else if (zone_replica_desc->is_readonly_all_server_) { - // for duplicate log stream, try to remove redudant R-replicas - if (OB_FAIL(try_generate_remove_readonly_task_for_duplicate_log_stream_(replica_stat_desc, replica, index))) { + // a type transform task or remove task generated, bypass + } else if (zone_replica_desc->is_columnstore_all_server()) { + // for duplicate log stream, if locality is C-replica, remove this R-replica + // because transform from R to C is not supported + if (OB_FAIL(generate_remove_replica_task(replica_stat_desc))) { + LOG_WARN("fail to generate remove replica task", KR(ret), K(replica_stat_desc)); + } else if (OB_FAIL(replica_stat_map_.remove(index))) { + LOG_WARN("fail to remove", KR(ret), K(replica_stat_map_), K(index)); + } + } else if (zone_replica_desc->is_readonly_all_server()) { + // for duplicate log stream, if locality is R-replica, try to remove redudant R-replicas + if (OB_FAIL(try_generate_remove_redundant_replica_task_for_dup_ls_(replica_stat_desc, replica, index))) { LOG_WARN("fail to generate remove replica task for duplicate log stream", KR(ret), K(replica_stat_desc), K(replica), K(index)); } @@ -890,6 +949,65 @@ int ObDRWorker::LocalityAlignment::try_generate_locality_task_from_paxos_replica return ret; } +int ObDRWorker::LocalityAlignment::do_generate_locality_task_from_columnstore_replica( + ReplicaStatDesc &replica_stat_desc, + share::ObLSReplica &replica, + const int64_t index) +{ + int ret = OB_SUCCESS; + const common::ObZone &zone = replica.get_zone(); + if (REPLICA_TYPE_COLUMNSTORE != replica.get_replica_type()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("replica type unexpected", KR(ret), K(replica)); + } else { + ReplicaDescArray *zone_replica_desc = nullptr; + int tmp_ret = locality_map_.get_refactored(zone, zone_replica_desc); + if (OB_HASH_NOT_EXIST == tmp_ret) { + if (OB_FAIL(generate_remove_replica_task(replica_stat_desc))) { + LOG_WARN("fail to generate remove replica task", KR(ret), K(replica_stat_desc)); + } else if (OB_FAIL(replica_stat_map_.remove(index))) { + LOG_WARN("fail to remove", KR(ret), K(replica_stat_map_), K(index)); + } + } else if (OB_SUCCESS == tmp_ret && OB_NOT_NULL(zone_replica_desc)) { + bool task_generated = false; + lib::ob_sort(zone_replica_desc->begin(), zone_replica_desc->end()); + // defensive check + for (int64_t i = zone_replica_desc->count() - 1; OB_SUCC(ret) && i >= 0; --i) { + ReplicaDesc &replica_desc = zone_replica_desc->at(i); + if (ObReplicaTypeCheck::is_columnstore_replica(replica_desc.replica_type_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("replica type unexpected", KR(ret), K(replica_desc), K(dr_ls_info_)); + } + } + // normal routine + if (OB_SUCC(ret)) { + if (zone_replica_desc->is_columnstore_all_server()) { + // for duplicate log stream, if locality is C-replica, try to remove redudant C-replicas + if (OB_FAIL(try_generate_remove_redundant_replica_task_for_dup_ls_(replica_stat_desc, replica, index))) { + LOG_WARN("fail to generate remove replica task for duplicate log stream", + KR(ret), K(replica_stat_desc), K(replica), K(index)); + } + } else { + // handle two abnormal occasions: + // 1. for duplicate ls, locality is R or F-replica; + // 2. for non-duplicate ls, locality is other type than C-replica + // in these both occasions, directly remove this C-replica, + // because transform from C-replica to R/F is not supported. + if (OB_FAIL(generate_remove_replica_task(replica_stat_desc))) { + LOG_WARN("fail to generate remove replica task", KR(ret), K(replica_stat_desc)); + } else if (OB_FAIL(replica_stat_map_.remove(index))) { + LOG_WARN("fail to remove", KR(ret), K(replica_stat_map_), K(index)); + } + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get refactored", KR(ret), K(zone)); + } + } + return ret; +} + void ObDRWorker::LocalityAlignment::print_locality_information() { for (LocalityMap::iterator iter = locality_map_.begin(); @@ -945,6 +1063,13 @@ int ObDRWorker::LocalityAlignment::do_generate_locality_task() i))) { LOG_WARN("fail to generate locality task from readonly replica", KR(ret)); } + } else if (REPLICA_TYPE_COLUMNSTORE == replica->get_replica_type()) { + if (OB_FAIL(do_generate_locality_task_from_columnstore_replica( + replica_stat_desc, + *replica, + i))) { + LOG_WARN("fail to generate locality task from columnstore replica", KR(ret)); + } } else { ret = OB_ERR_UNEXPECTED; LOG_WARN("replica type unexpected", KR(ret), KPC(replica)); @@ -1429,7 +1554,7 @@ int ObDRWorker::LocalityAlignment::try_get_normal_locality_alignment_task( return ret; } -int ObDRWorker::LocalityAlignment::try_get_readonly_all_server_locality_alignment_task( +int ObDRWorker::LocalityAlignment::try_get_readonly_or_columnstore_all_server_locality_alignment_task( UnitProvider &unit_provider, const LATask *&task) { @@ -1444,7 +1569,8 @@ int ObDRWorker::LocalityAlignment::try_get_readonly_all_server_locality_alignmen if (OB_UNLIKELY(nullptr == replica_desc_array)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("replica desc array ptr is null", KR(ret)); - } else if (!replica_desc_array->is_readonly_all_server_) { + } else if (!replica_desc_array->is_readonly_all_server() + && !replica_desc_array->is_columnstore_all_server()) { // bypass } else if (OB_FAIL(dr_ls_info_.get_ls_status_info(ls_status_info))) { LOG_WARN("fail to get log stream info", KR(ret)); @@ -1462,8 +1588,9 @@ int ObDRWorker::LocalityAlignment::try_get_readonly_all_server_locality_alignmen add_replica_task_.member_time_us_ = ObTimeUtility::current_time(); add_replica_task_.unit_id_ = unit.unit_id_; add_replica_task_.unit_group_id_ = ls_status_info->unit_group_id_; - add_replica_task_.replica_type_ = REPLICA_TYPE_READONLY; - add_replica_task_.memstore_percent_ = replica_desc_array->readonly_memstore_percent_; + add_replica_task_.replica_type_ = replica_desc_array->is_columnstore_all_server() ? + REPLICA_TYPE_COLUMNSTORE : REPLICA_TYPE_READONLY; + add_replica_task_.memstore_percent_ = replica_desc_array->get_readonly_memstore_percent(); add_replica_task_.orig_paxos_replica_number_ = curr_paxos_replica_number_; add_replica_task_.paxos_replica_number_ = curr_paxos_replica_number_; task = &add_replica_task_; @@ -1492,7 +1619,7 @@ int ObDRWorker::LocalityAlignment::get_next_locality_alignment_task( } else if (nullptr != task) { // got one LOG_INFO("success to get a normal task", KPC(task)); - } else if (OB_FAIL(try_get_readonly_all_server_locality_alignment_task( + } else if (OB_FAIL(try_get_readonly_or_columnstore_all_server_locality_alignment_task( unit_provider_, task))) { LOG_WARN("fail to get readonly all server locality alignment task", KR(ret)); @@ -2528,7 +2655,7 @@ int ObDRWorker::get_replica_type_by_leader_( // not leader replica get replica type may not right. when remove or modify replica, // replica type wrong may result in fatal error, so get it by leader int ret = OB_SUCCESS; - replica_type = REPLICA_TYPE_MAX; + replica_type = REPLICA_TYPE_INVALID; common::ObAddr leader_addr; // not used GlobalLearnerList learner_list; common::ObMemberList member_list; @@ -2546,7 +2673,16 @@ int ObDRWorker::get_replica_type_by_leader_( } else if (member_list.contains(server_addr)) { replica_type = REPLICA_TYPE_FULL; } else if (learner_list.contains(server_addr)) { - replica_type = REPLICA_TYPE_READONLY; + ObMember learner; + if (OB_FAIL(learner_list.get_learner_by_addr(server_addr, learner))) { + LOG_WARN("failed to get_learner_by_addr", KR(ret), K(server_addr)); + } else { + if (learner.is_columnstore()) { + replica_type = REPLICA_TYPE_COLUMNSTORE; + } else { + replica_type = REPLICA_TYPE_READONLY; + } + } } else { ret = OB_ERR_UNEXPECTED; LOG_WARN("fail to find server in leader member list and learner list", @@ -2628,7 +2764,7 @@ int ObDRWorker::build_remove_replica_task_( ObRemoveLSReplicaTask &remove_replica_task) { int ret = OB_SUCCESS; - common::ObReplicaType replica_type = REPLICA_TYPE_MAX; + common::ObReplicaType replica_type = REPLICA_TYPE_INVALID; common::ObAddr leader_addr; ObMember member_to_remove; if (OB_UNLIKELY(!inited_)) { @@ -2652,10 +2788,10 @@ int ObDRWorker::build_remove_replica_task_( share::ObTaskId task_id; int64_t new_paxos_replica_number = 0; bool has_leader = false; - ObReplicaMember remove_member(member_to_remove); + ObReplicaMember remove_member; if (FALSE_IT(task_id.init(self_addr_))) { - } else if (OB_FAIL(remove_member.set_replica_type(replica_type))) { - LOG_WARN("fail to set replica type", KR(ret), K(replica_type), K(remove_member)); + } else if (OB_FAIL(remove_member.init(member_to_remove, replica_type))) { + LOG_WARN("fail to init remove_member", KR(ret), K(member_to_remove), K(replica_type)); } else if (OB_FAIL(check_and_generate_new_paxos_replica_num_( arg, replica_type, dr_ls_info, new_paxos_replica_number))) { LOG_WARN("fail to check and generate new paxos replica num", KR(ret), K(arg), K(replica_type), K(dr_ls_info)); @@ -2687,7 +2823,7 @@ int ObDRWorker::build_modify_replica_type_task_( int ret = OB_SUCCESS; share::ObUnit unit; share::ObLSReplica ls_replica; - common::ObReplicaType replica_type = REPLICA_TYPE_MAX; + common::ObReplicaType replica_type = REPLICA_TYPE_INVALID; if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("DRWorker not init", KR(ret)); @@ -2713,6 +2849,12 @@ int ObDRWorker::build_modify_replica_type_task_( LOG_USER_ERROR(OB_ENTRY_EXIST, "Current replica type is same as the target type, no need to modify"); LOG_WARN("replica type is the same as the target type, no need to modify type", KR(ret), K(arg), K(replica_type)); + } else if (ObReplicaTypeCheck::is_columnstore_replica(arg.get_replica_type()) + || ObReplicaTypeCheck::is_columnstore_replica(replica_type)) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "Current or target replica-type is C-replica, type transform"); + LOG_WARN("Current or target replica_type is C-replica, type transform not supported", + KR(ret), K(arg), K(replica_type)); } else if (REPLICA_TYPE_FULL == arg.get_replica_type() && share::ObLSReplica::DEFAULT_REPLICA_COUNT == dr_ls_info.get_member_list_cnt()) { ret = OB_OP_NOT_ALLOW; @@ -2769,8 +2911,8 @@ int ObDRWorker::build_migrate_replica_task_( { int ret = OB_SUCCESS; share::ObUnit destination_unit; - common::ObReplicaType replica_type = REPLICA_TYPE_MAX; - share::ObLSReplica desti_ls_replica; + common::ObReplicaType replica_type = REPLICA_TYPE_INVALID; + share::ObLSReplica dest_ls_replica; share::ObLSReplica source_ls_replica; if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; @@ -2781,15 +2923,15 @@ int ObDRWorker::build_migrate_replica_task_( } else if (OB_FAIL(dr_ls_info.check_replica_exist_and_get_ls_replica( arg.get_server_addr(), source_ls_replica))) { LOG_WARN("fail to check and get replica by server", KR(ret), K(arg)); - } else if (!source_ls_replica.is_valid() || (source_ls_replica.is_valid() && !source_ls_replica.is_in_service())) { + } else if (!source_ls_replica.is_valid() || !source_ls_replica.is_in_service()) { ret = OB_ENTRY_NOT_EXIST; LOG_USER_ERROR(OB_ENTRY_NOT_EXIST, "Source server does not have a replica of this LS"); LOG_WARN("source server does not have a replica of this LS", KR(ret), K(arg), K(dr_ls_info), K(source_ls_replica)); } else if (OB_FAIL(dr_ls_info.check_replica_exist_and_get_ls_replica( - arg.get_destination_addr(), desti_ls_replica))) { + arg.get_destination_addr(), dest_ls_replica))) { LOG_WARN("fail to check and get replica by server", KR(ret), K(arg)); - } else if (desti_ls_replica.is_valid()) { + } else if (dest_ls_replica.is_valid()) { ret = OB_ENTRY_EXIST; LOG_USER_ERROR(OB_ENTRY_EXIST, "The destination server already has a replica"); LOG_WARN("target server already has a replica, no need migrate", KR(ret), K(arg), K(dr_ls_info)); @@ -2898,6 +3040,16 @@ int ObDRWorker::build_modify_paxos_replica_num_task_( return ret; } +// this func is only for basic checking. +// 1. if dest_replica_type is F/R replica, data_src can be same replica_type or F; +// 2. if dest_replica_type is C replica, data_src can be F/R/C. +bool can_as_data_source(const int32_t dest_replica_type, const int32_t src_replica_type) +{ + return (dest_replica_type == src_replica_type + || REPLICA_TYPE_FULL == src_replica_type + || ObReplicaTypeCheck::is_columnstore_replica(dest_replica_type)); +} + int ObDRWorker::check_data_source_available_and_init_( const obrpc::ObAdminAlterLSReplicaArg &arg, const common::ObReplicaType &replica_type, @@ -2916,7 +3068,7 @@ int ObDRWorker::check_data_source_available_and_init_( data_source.reset(); share::ObLSReplica ls_replica; ObServerInfoInTable server_info; - common::ObReplicaType provide_replica_type = REPLICA_TYPE_MAX; + common::ObReplicaType provide_replica_type = REPLICA_TYPE_INVALID; if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("DRWorker not init", KR(ret)); @@ -2924,11 +3076,10 @@ int ObDRWorker::check_data_source_available_and_init_( // passed LOG_INFO("data_source is not valid", KR(ret), K(arg)); } else if (OB_UNLIKELY(!arg.is_valid() - || OB_UNLIKELY(replica_type != REPLICA_TYPE_FULL && replica_type != REPLICA_TYPE_READONLY))) { + || OB_UNLIKELY(!ObReplicaTypeCheck::is_replica_type_valid(replica_type)))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(arg), K(replica_type)); } else { - ObDataSourceCandidateChecker type_checker(replica_type); if (OB_FAIL(dr_ls_info.check_replica_exist_and_get_ls_replica(arg.get_data_source(), ls_replica))) { LOG_WARN("fail to get ls replica", KR(ret), K(dr_ls_info)); } else if (!ls_replica.is_valid() || (ls_replica.is_valid() && !ls_replica.is_in_service())) { @@ -2941,10 +3092,10 @@ int ObDRWorker::check_data_source_available_and_init_( ret = OB_OP_NOT_ALLOW; LOG_USER_ERROR(OB_OP_NOT_ALLOW, "Data source replica restore or clone failed, which is"); LOG_WARN("ls replica restore failed", KR(ret), K(arg), K(ls_replica)); - } else if (!type_checker.is_candidate(provide_replica_type)) { + } else if (!can_as_data_source(replica_type, provide_replica_type)) { ret = OB_OP_NOT_ALLOW; LOG_USER_ERROR(OB_OP_NOT_ALLOW, "R replica is not supported as the source of F replica, which is"); - LOG_WARN("type_checker failed", KR(ret), K(arg), K(provide_replica_type)); + LOG_WARN("provided data_source replica_type not supported", KR(ret), K(arg), K(replica_type), K(provide_replica_type)); } else if (OB_FAIL(SVR_TRACER.get_server_info(arg.get_data_source(), server_info))) { LOG_WARN("fail to get server info", KR(ret), K(arg)); } else if (!server_info.is_alive()) { @@ -3013,7 +3164,7 @@ int ObDRWorker::check_and_generate_new_paxos_replica_num_( ret = OB_NOT_INIT; LOG_WARN("DRWorker not init", KR(ret)); } else if (OB_UNLIKELY(!arg.is_valid() - || (REPLICA_TYPE_FULL != replica_type && REPLICA_TYPE_READONLY != replica_type))) { + || !ObReplicaTypeCheck::is_replica_type_valid(replica_type))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(arg), K(replica_type)); } else if (std::abs(new_p - curr_p) > 1) { @@ -3023,13 +3174,13 @@ int ObDRWorker::check_and_generate_new_paxos_replica_num_( } else if (task_type.is_add_task()) { if (REPLICA_TYPE_FULL == replica_type) { member_change_type = MEMBER_CHANGE_ADD; - } else if (REPLICA_TYPE_READONLY == replica_type) { + } else if (ObReplicaTypeCheck::is_non_paxos_replica(replica_type)) { member_change_type = MEMBER_CHANGE_NOP; } } else if (task_type.is_remove_task()) { if (REPLICA_TYPE_FULL == replica_type) { member_change_type = MEMBER_CHANGE_SUB; - } else if (REPLICA_TYPE_READONLY == replica_type) { + } else if (ObReplicaTypeCheck::is_non_paxos_replica(replica_type)) { member_change_type = MEMBER_CHANGE_NOP; } } else if (task_type.is_modify_replica_task()) { @@ -3037,11 +3188,15 @@ int ObDRWorker::check_and_generate_new_paxos_replica_num_( member_change_type = MEMBER_CHANGE_ADD; } else if (REPLICA_TYPE_READONLY == replica_type) { member_change_type = MEMBER_CHANGE_SUB; + } else if (ObReplicaTypeCheck::is_columnstore_replica(replica_type)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid replica_type, do not support transforming to C-replica", KR(ret), K(arg)); } } else { ret = OB_ERR_UNEXPECTED; LOG_WARN("task type unexpected", KR(ret), K(arg), K(dr_ls_info), K(replica_type)); } + if (OB_FAIL(ret)) { } else if ((MEMBER_CHANGE_ADD == member_change_type)) { if (OB_FAIL(check_for_alter_full_replica_(member_list_count + 1, new_p))) { @@ -3192,7 +3347,7 @@ int ObDRWorker::check_has_leader_while_remove_replica( int64_t arb_replica_num = 0; uint64_t tenant_id = OB_INVALID_TENANT_ID; ObLSID ls_id; - ObReplicaType replica_type = REPLICA_TYPE_MAX; + ObReplicaType replica_type = REPLICA_TYPE_INVALID; for (int64_t index = 0; OB_SUCC(ret) && index < replica_cnt; ++index) { share::ObLSReplica *ls_replica = nullptr; DRServerStatInfo *server_stat_info = nullptr; @@ -3482,6 +3637,9 @@ int ObDRWorker::try_remove_permanent_offline_replicas( common::ObReplicaType replica_type = REPLICA_TYPE_READONLY; if (OB_FAIL(learner_list.get_member_by_index(index, learner_to_remove))) { LOG_WARN("fail to get learner by index", KR(ret), K(index)); + } else if (FALSE_IT(replica_type = learner_to_remove.is_columnstore() ? + REPLICA_TYPE_COLUMNSTORE : REPLICA_TYPE_READONLY)) { + // shall never be here } else if (OB_FAIL(do_single_replica_permanent_offline_( tenant_id, ls_id, @@ -3490,7 +3648,7 @@ int ObDRWorker::try_remove_permanent_offline_replicas( replica_type, learner_to_remove, acc_dr_task))) { - LOG_WARN("fail to do single replica permanent offline task for readonly replica", KR(ret), K(tenant_id), + LOG_WARN("fail to do single replica permanent offline task for non-paxos replica", KR(ret), K(tenant_id), K(ls_id), K(dr_ls_info), K(only_for_display), K(replica_type), K(learner_to_remove), K(acc_dr_task)); } } @@ -3532,12 +3690,12 @@ int ObDRWorker::do_single_replica_permanent_offline_( const int64_t memstore_percent = 100; ObDRTaskKey task_key; bool can_generate = false; - ObReplicaMember remove_member(member_to_remove); + ObReplicaMember remove_member; ObDRTaskType task_type = ObReplicaTypeCheck::is_paxos_replica_V2(replica_type) ? ObDRTaskType::LS_REMOVE_PAXOS_REPLICA : ObDRTaskType::LS_REMOVE_NON_PAXOS_REPLICA; - if (OB_FAIL(remove_member.set_replica_type(replica_type))) { - LOG_WARN("fail to set replica type", KR(ret), K(replica_type), K(remove_member)); + if (OB_FAIL(remove_member.init(member_to_remove, replica_type))) { + LOG_WARN("failed to init remove_member", KR(ret), K(member_to_remove), K(replica_type)); } else if (OB_FAIL(construct_extra_infos_to_build_remove_replica_task( dr_ls_info, task_id, @@ -3560,7 +3718,7 @@ int ObDRWorker::do_single_replica_permanent_offline_( replica_type, new_paxos_replica_number, source_server, - REPLICA_TYPE_MAX/*source_replica_type*/, + REPLICA_TYPE_INVALID/*source_replica_type*/, old_paxos_replica_number, leader_addr, "remove permanent offline replica"))) { @@ -4216,8 +4374,8 @@ int ObDRWorker::record_task_plan_for_locality_alignment( ObDRTaskType task_type = ObDRTaskType::MAX_TYPE; uint64_t tenant_id = OB_INVALID_ID; share::ObLSID ls_id; - ObReplicaType source_replica_type = REPLICA_TYPE_MAX; - ObReplicaType target_replica_type = REPLICA_TYPE_MAX; + ObReplicaType source_replica_type = REPLICA_TYPE_INVALID; + ObReplicaType target_replica_type = REPLICA_TYPE_INVALID; ObDRTaskPriority task_priority = ObDRTaskPriority::MAX_PRI; common::ObAddr leader_addr; common::ObAddr source_svr; @@ -4245,7 +4403,7 @@ int ObDRWorker::record_task_plan_for_locality_alignment( case RemoveNonPaxos: { const RemoveReplicaLATask *my_task = reinterpret_cast(task); task_type = RemovePaxos == task->get_task_type() ? ObDRTaskType::LS_REMOVE_PAXOS_REPLICA : ObDRTaskType::LS_REMOVE_NON_PAXOS_REPLICA; - source_replica_type = REPLICA_TYPE_MAX; + source_replica_type = REPLICA_TYPE_INVALID; target_replica_type = my_task->replica_type_; task_priority = task_type == ObDRTaskType::LS_REMOVE_PAXOS_REPLICA ? ObDRTaskPriority::HIGH_PRI : ObDRTaskPriority::LOW_PRI; target_svr = my_task->remove_server_; @@ -4476,15 +4634,15 @@ int ObDRWorker::try_shrink_resource_pools( && share::ObUnit::UNIT_STATUS_DELETING == unit_stat_info->get_unit().status_) { // replica is still in member_list, but unit is in DELETING status // If this is a duplicate log stream - // 1.1 for R-replica: execute remove_learner task directly + // 1.1 for non-paxos(R or C) replica: execute remove_learner task directly // 1.2 for F-replica: try to execute migrate-replica first, // if migrate-replica task can not generate then try to type_transform another R to F // If this is a normal log stream - // 2.1 try to execute migrate-replica task for both R-replica and F-replica + // 2.1 try to execute migrate-replica task for replica of any type if (dr_ls_info.is_duplicate_ls()) { - if (REPLICA_TYPE_READONLY == ls_replica->get_replica_type()) { + if (ObReplicaTypeCheck::is_non_paxos_replica(ls_replica->get_replica_type())) { // 1.1 try to generate and execute remove learner task - if (OB_FAIL(try_remove_readonly_replica_for_deleting_unit_( + if (OB_FAIL(try_remove_non_paxos_replica_for_deleting_unit_( *ls_replica, only_for_display, dr_ls_info, @@ -4554,7 +4712,7 @@ int ObDRWorker::try_shrink_resource_pools( return ret; } -int ObDRWorker::try_remove_readonly_replica_for_deleting_unit_( +int ObDRWorker::try_remove_non_paxos_replica_for_deleting_unit_( const share::ObLSReplica &ls_replica, const bool &only_for_display, DRLSInfo &dr_ls_info, @@ -4607,7 +4765,7 @@ int ObDRWorker::try_remove_readonly_replica_for_deleting_unit_( ls_replica.get_replica_type(), new_paxos_replica_number, source_server, - REPLICA_TYPE_MAX/*source_replica_type*/, + REPLICA_TYPE_INVALID/*source_replica_type*/, old_paxos_replica_number, leader_addr, "shrink unit task"))) { @@ -5055,7 +5213,7 @@ int ObDRWorker::check_need_generate_cancel_unit_migration_task( return ret; } -int ObDRWorker::construct_extra_info_to_build_cancael_migration_task( +int ObDRWorker::construct_extra_info_to_build_cancel_migration_task( const bool &is_paxos_replica_related, DRLSInfo &dr_ls_info, const share::ObLSReplica &ls_replica, @@ -5103,7 +5261,7 @@ int ObDRWorker::generate_cancel_unit_migration_task( int ret = OB_SUCCESS; ObRemoveLSReplicaTask remove_member_task; ObString comment_to_set = ""; - ObReplicaType replica_type = is_paxos_replica_related ? REPLICA_TYPE_FULL : REPLICA_TYPE_READONLY; + ObReplicaType replica_type = remove_member.get_replica_type(); if (is_paxos_replica_related) { comment_to_set.assign_ptr(drtask::CANCEL_MIGRATE_UNIT_WITH_PAXOS_REPLICA, strlen(drtask::CANCEL_MIGRATE_UNIT_WITH_PAXOS_REPLICA)); @@ -5132,7 +5290,7 @@ int ObDRWorker::generate_cancel_unit_migration_task( old_paxos_replica_number, new_paxos_replica_number, replica_type))) { - LOG_WARN("fail to build remove member task", KR(ret)); + LOG_WARN("fail to build remove member task", KR(ret), K(task_key), K(task_id)); } else if (OB_FAIL(disaster_recovery_task_mgr_->add_task(remove_member_task))) { LOG_WARN("fail to add task", KR(ret), K(remove_member_task)); } else { @@ -5218,7 +5376,7 @@ int ObDRWorker::try_cancel_unit_migration( strlen(drtask::CANCEL_MIGRATE_UNIT_WITH_NON_PAXOS_REPLICA)); } - if (OB_FAIL(construct_extra_info_to_build_cancael_migration_task( + if (OB_FAIL(construct_extra_info_to_build_cancel_migration_task( is_paxos_replica_related, dr_ls_info, *ls_replica, @@ -5240,7 +5398,7 @@ int ObDRWorker::try_cancel_unit_migration( ls_replica->get_replica_type(), new_paxos_replica_number, source_svr, - REPLICA_TYPE_MAX, + REPLICA_TYPE_INVALID, old_paxos_replica_number, leader_addr, comment_to_set))) { @@ -5773,7 +5931,8 @@ int ObDRWorker::check_ls_only_in_member_list_or_with_flag_( } } else { ret = OB_STATE_NOT_MATCH; - LOG_WARN("read only replica with flag should not appear in inner_ls_info", KR(ret), K(learner_to_check), K(inner_ls_info)); + LOG_WARN("read only replica with migrating-flag should not appear in inner_ls_info", + KR(ret), K(learner_to_check), K(inner_ls_info)); } } else if (OB_FAIL(inner_ls_info.find(learner_to_check.get_server(), replica))) { LOG_WARN("fail to find read only replica", KR(ret), K(inner_ls_info), K(learner_to_check)); diff --git a/src/rootserver/ob_disaster_recovery_worker.h b/src/rootserver/ob_disaster_recovery_worker.h index ec162fd32..97e73ee00 100755 --- a/src/rootserver/ob_disaster_recovery_worker.h +++ b/src/rootserver/ob_disaster_recovery_worker.h @@ -310,7 +310,7 @@ private: LA_P_ADD_LOGONLY, LA_P_ADD_ENCRYPTION, LA_P_FULL_TO_LOGONLY, - LA_P_ADD_READONLY, + LA_P_ADD_NON_PAXOS, LA_P_REMOVE_NON_PAXOS, LA_P_FULL_TO_READONLY, LA_P_REMOVE_PAXOS, @@ -348,7 +348,7 @@ private: RemoveReplicaLATask() : LATask(), remove_server_(), - replica_type_(REPLICA_TYPE_MAX), + replica_type_(REPLICA_TYPE_INVALID), memstore_percent_(100), member_time_us_(-1), orig_paxos_replica_number_(0), @@ -389,7 +389,7 @@ private: dst_server_(), unit_id_(OB_INVALID_ID), unit_group_id_(OB_INVALID_ID), - replica_type_(REPLICA_TYPE_MAX), + replica_type_(REPLICA_TYPE_INVALID), memstore_percent_(100), member_time_us_(-1), orig_paxos_replica_number_(0), @@ -405,8 +405,8 @@ private: priority = LATaskPrio::LA_P_ADD_LOGONLY; } else if (common::REPLICA_TYPE_ENCRYPTION_LOGONLY == replica_type_) { priority = LATaskPrio::LA_P_ADD_ENCRYPTION; - } else if (common::REPLICA_TYPE_READONLY == replica_type_) { - priority = LATaskPrio::LA_P_ADD_READONLY; + } else if (ObReplicaTypeCheck::is_non_paxos_replica(replica_type_)) { + priority = LATaskPrio::LA_P_ADD_NON_PAXOS; } else {} // default priority value return priority; } @@ -442,10 +442,10 @@ private: dst_server_(), unit_id_(OB_INVALID_ID), unit_group_id_(OB_INVALID_ID), - src_replica_type_(REPLICA_TYPE_MAX), + src_replica_type_(REPLICA_TYPE_INVALID), src_memstore_percent_(100), src_member_time_us_(-1), - dst_replica_type_(REPLICA_TYPE_MAX), + dst_replica_type_(REPLICA_TYPE_INVALID), dst_memstore_percent_(100), dst_member_time_us_(-1), orig_paxos_replica_number_(0), @@ -527,7 +527,7 @@ private: memstore_percent_(memstore_percent), replica_num_(replica_num) {} ReplicaDesc() - : replica_type_(REPLICA_TYPE_MAX), + : replica_type_(REPLICA_TYPE_INVALID), memstore_percent_(100), replica_num_(0) {} TO_STRING_KV(K(replica_type_), @@ -535,14 +535,16 @@ private: K(replica_num_)); int64_t cast(const common::ObReplicaType replica_type) { int64_t ret_val = 0; - if (REPLICA_TYPE_READONLY == replica_type) { + if (REPLICA_TYPE_COLUMNSTORE == replica_type) { ret_val = 1; - } else if (REPLICA_TYPE_ENCRYPTION_LOGONLY == replica_type) { + } else if (REPLICA_TYPE_READONLY == replica_type) { ret_val = 2; - } else if (REPLICA_TYPE_LOGONLY == replica_type) { + } else if (REPLICA_TYPE_ENCRYPTION_LOGONLY == replica_type) { ret_val = 3; - } else if (REPLICA_TYPE_FULL == replica_type) { + } else if (REPLICA_TYPE_LOGONLY == replica_type) { ret_val = 4; + } else if (REPLICA_TYPE_FULL == replica_type) { + ret_val = 5; } else { ret_val = 0; // invalid type, put it at the beginning } @@ -580,35 +582,6 @@ private: && nullptr != unit_stat_info_ && nullptr != unit_in_group_stat_info_; } - int64_t cast(const common::ObReplicaType replica_type) { - int64_t ret_val = 0; - if (REPLICA_TYPE_READONLY == replica_type) { - ret_val = 1; - } else if (REPLICA_TYPE_ENCRYPTION_LOGONLY == replica_type) { - ret_val = 2; - } else if (REPLICA_TYPE_LOGONLY == replica_type) { - ret_val = 3; - } else if (REPLICA_TYPE_FULL == replica_type) { - ret_val = 4; - } else { - ret_val = 0; // invalid type, put it at the beginning - } - return ret_val; - } - bool operator<(const ReplicaStatDesc &that) { - bool bool_ret = true; - if (nullptr == this->replica_ && nullptr != that.replica_) { - bool_ret = true; - } else if (nullptr != this->replica_ && nullptr == that.replica_) { - bool_ret = false; - } else if (nullptr == this->replica_ && nullptr == that.replica_) { - bool_ret = true; - } else { - bool_ret = cast(this->replica_->get_replica_type()) - < cast(that.replica_->get_replica_type()); - } - return bool_ret; - } TO_STRING_KV(KPC(replica_), KPC(server_stat_info_), KPC(unit_stat_info_), @@ -625,10 +598,22 @@ private: public: ReplicaDescArray() : common::ObSEArrayImpl(), is_readonly_all_server_(false), - readonly_memstore_percent_(100) {} - public: + is_columnstore_all_server_(false) {} + void set_readonly_all_server() { + is_readonly_all_server_ = true; + is_columnstore_all_server_ = false; + } + void set_columnstore_all_server() { + is_columnstore_all_server_ = true; + is_readonly_all_server_ = false; + } + bool is_readonly_all_server() { return is_readonly_all_server_; } + bool is_columnstore_all_server() { return is_columnstore_all_server_; } + int64_t get_readonly_memstore_percent() { return readonly_memstore_percent_; } + private: bool is_readonly_all_server_; - int64_t readonly_memstore_percent_; + bool is_columnstore_all_server_; + const int64_t readonly_memstore_percent_ = 100; // obsolete }; @@ -699,13 +684,17 @@ private: ReplicaStatDesc &replica_stat_desc, share::ObLSReplica &replica, const int64_t index); + int do_generate_locality_task_from_columnstore_replica( + ReplicaStatDesc &replica_stat_desc, + share::ObLSReplica &replica, + const int64_t index); - int try_generate_type_transform_task_for_readonly_replica_( + int try_generate_task_for_readonly_replica_( ReplicaDescArray &zone_replica_desc_in_locality, ReplicaStatDesc &replica_stat_desc, const int64_t index, bool &task_generated); - int try_generate_remove_readonly_task_for_duplicate_log_stream_( + int try_generate_remove_redundant_replica_task_for_dup_ls_( ReplicaStatDesc &replica_stat_desc, share::ObLSReplica &replica, const int64_t index); @@ -733,7 +722,7 @@ private: ReplicaDesc &replica_desc); int generate_modify_paxos_replica_number_task(); // private func for get_next_locality_alignment_task - int try_get_readonly_all_server_locality_alignment_task( + int try_get_readonly_or_columnstore_all_server_locality_alignment_task( UnitProvider &unit_provider, const LATask *&task); int try_get_normal_locality_alignment_task( @@ -933,7 +922,7 @@ private: bool &is_paxos_replica_related, bool &need_generate); - int construct_extra_info_to_build_cancael_migration_task( + int construct_extra_info_to_build_cancel_migration_task( const bool &is_paxos_replica_related, DRLSInfo &dr_ls_info, const share::ObLSReplica &ls_replica, @@ -1043,7 +1032,7 @@ private: // @params[in] only_for_display, whether just to display this task // @params[in] dr_ls_info, disaster recovery infos of this log stream // @params[out] acc_dr_task, accumulated disaster recovery task count - int try_remove_readonly_replica_for_deleting_unit_( + int try_remove_non_paxos_replica_for_deleting_unit_( const share::ObLSReplica &ls_replica, const bool &only_for_display, DRLSInfo &dr_ls_info, diff --git a/src/rootserver/ob_locality_util.cpp b/src/rootserver/ob_locality_util.cpp index e3d7e262c..64460d518 100644 --- a/src/rootserver/ob_locality_util.cpp +++ b/src/rootserver/ob_locality_util.cpp @@ -40,19 +40,6 @@ using namespace oceanbase::share::schema; ret = OB_INVALID_ARGUMENT; \ LOG_WARN("invalid locality", K(ret)); \ } while (0) -// full replica -const char *const ObLocalityDistribution::FULL_REPLICA_STR = "FULL"; -const char *const ObLocalityDistribution::F_REPLICA_STR = "F"; -// logonly replica -const char *const ObLocalityDistribution::LOGONLY_REPLICA_STR = "LOGONLY"; -const char *const ObLocalityDistribution::L_REPLICA_STR = "L"; -// readonly replica -const char *const ObLocalityDistribution::READONLY_REPLICA_STR = "READONLY"; -const char *const ObLocalityDistribution::R_REPLICA_STR = "R"; -// encryption logonly replica -const char *const ObLocalityDistribution::ENCRYPTION_LOGONLY_REPLICA_STR = "ENCRYPTION_LOGONLY"; -const char *const ObLocalityDistribution::E_REPLICA_STR = "E"; -// some other terminology const common::ObZone ObLocalityDistribution::EVERY_ZONE("everyzone"); const char *const ObLocalityDistribution::ALL_SERVER_STR = "ALL_SERVER"; const char *const ObLocalityDistribution::MEMSTORE_PERCENT_STR = "MEMSTORE_PERCENT"; @@ -149,6 +136,9 @@ int ObLocalityDistribution::ZoneSetReplicaDist::check_valid_replica_dist( for (int64_t i = 0; is_valid && i < all_replica_attr_array_[READONLY_REPLICA].count(); ++i) { is_valid = all_replica_attr_array_[READONLY_REPLICA].at(i).num_ >= 0; } + for (int64_t i = 0; is_valid && i < all_replica_attr_array_[COLUMNSTORE_REPLICA].count(); ++i) { + is_valid = all_replica_attr_array_[COLUMNSTORE_REPLICA].at(i).num_ >= 0; + } } } else { is_valid = false; // do not support mixed-zone locality from 3.2.1 and versions to come @@ -209,6 +199,12 @@ bool ObLocalityDistribution::ZoneSetReplicaDist::has_non_encryption_logonly() co const ReplicaAttr &attr = all_replica_attr_array_[READONLY_REPLICA].at(i); has = attr.num_ > 0; } + for (int64_t i = 0; + !has && i < all_replica_attr_array_[COLUMNSTORE_REPLICA].count(); + ++i) { + const ReplicaAttr &attr = all_replica_attr_array_[COLUMNSTORE_REPLICA].at(i); + has = attr.num_ > 0; + } return has; } @@ -436,6 +432,9 @@ int ObLocalityDistribution::ZoneSetReplicaDist::replica_type_to_str( case ENCRYPTION_LOGONLY_REPLICA: replica_type_str = "ENCRYPTION_LOGONLY"; break; + case COLUMNSTORE_REPLICA: + replica_type_str = "COLUMNSTORE"; + break; default: ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected replica type", K(ret), K(replica_type)); @@ -658,9 +657,11 @@ int ObLocalityDistribution::RawLocalityIter::get_replica_arrangements( && OB_SUCC(get_next_replica_arrangement( cursor, end, replica_type, replica_num, memstore_percent))) { if (OB_UNLIKELY(FULL_REPLICA != replica_type - && READONLY_REPLICA != replica_type)) { + && READONLY_REPLICA != replica_type + && COLUMNSTORE_REPLICA != replica_type)) { // TODO: F-replica is supported since 4.0, // R-replica is supported since 4.2, + // C-replica is supported since 4.3.2 // other types will be supported later INVALID_LOCALITY(); switch (replica_type) { @@ -775,6 +776,20 @@ int ObLocalityDistribution::RawLocalityIter::get_replica_type( cursor += strlen(R_REPLICA_STR); } else {} // not this type } + if (!type_found && remain >= strlen(COLUMNSTORE_REPLICA_STR)) { + if (0 == strncmp(COLUMNSTORE_REPLICA_STR, &locality_str_[cursor], strlen(COLUMNSTORE_REPLICA_STR))) { + replica_type = COLUMNSTORE_REPLICA; + type_found = true; + cursor += strlen(COLUMNSTORE_REPLICA_STR); + } else {} // not this type + } + if (!type_found && remain >= strlen(C_REPLICA_STR)) { + if (0 == strncmp(C_REPLICA_STR, &locality_str_[cursor], strlen(C_REPLICA_STR))) { + replica_type = COLUMNSTORE_REPLICA; + type_found = true; + cursor += strlen(C_REPLICA_STR); + } else {} // not this type + } if (!type_found && remain >= strlen(ENCRYPTION_LOGONLY_REPLICA_STR)) { if (0 == strncmp(ENCRYPTION_LOGONLY_REPLICA_STR, &locality_str_[cursor], @@ -1591,72 +1606,6 @@ int ObLocalityDistribution::convert_zone_list( return ret; } -int ObLocalityDistribution::get_zone_replica_num( - const common::ObZone &zone, - share::ObReplicaNumSet &replica_num_set) -{ - int ret = OB_SUCCESS; - if (zone.is_empty()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(zone)); - } else { - bool found = false; - for (int64_t i = 0; !found && OB_SUCC(ret) && i < zone_set_replica_dist_array_.count(); ++i) { - const ZoneSetReplicaDist &this_dist = zone_set_replica_dist_array_.at(i); - if (zone != this_dist.get_zone_set().at(0)) { - // bypass - } else{ - replica_num_set.set_replica_num(this_dist.get_full_replica_num(), - this_dist.get_logonly_replica_num(), - this_dist.get_readonly_replica_num(), - this_dist.get_encryption_logonly_replica_num()); - } - } - if (OB_FAIL(ret)) { - // failed - } else if (!found) { - ret = OB_ENTRY_NOT_EXIST; - } - } - return ret; -} - -int ObLocalityDistribution::get_zone_replica_num( - const common::ObZone &zone, - share::ObZoneReplicaAttrSet &zone_replica_attr_set) -{ - int ret = OB_SUCCESS; - if (zone.is_empty()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(zone)); - } else { - bool found = false; - for (int64_t i = 0; !found && OB_SUCC(ret) && i < zone_set_replica_dist_array_.count(); ++i) { - const ZoneSetReplicaDist &this_dist = zone_set_replica_dist_array_.at(i); - if (zone != this_dist.get_zone_set().at(0)) { - // bypass - } else if (OB_FAIL(zone_replica_attr_set.zone_set_.assign(this_dist.get_zone_set()))) { - LOG_WARN("fail to assign zone set", K(ret)); - } else { - zone_replica_attr_set.zone_ = zone; - if (OB_FAIL(zone_replica_attr_set.replica_attr_set_.set_replica_attr_array( - this_dist.get_full_replica_attr(), - this_dist.get_logonly_replica_attr(), - this_dist.get_readonly_replica_attr(), - this_dist.get_encryption_logonly_replica_attr()))) { - LOG_WARN("fail to set replica attr array", KR(ret)); - } - } - } - if (OB_FAIL(ret)) { - // failed - } else if (!found) { - ret = OB_ENTRY_NOT_EXIST; - } - } - return ret; -} - int ObLocalityDistribution::get_zone_replica_attr_array( common::ObIArray &zone_replica_num_array) { @@ -1675,7 +1624,8 @@ int ObLocalityDistribution::get_zone_replica_attr_array( this_dist.get_full_replica_attr(), this_dist.get_logonly_replica_attr(), this_dist.get_readonly_replica_attr(), - this_dist.get_encryption_logonly_replica_attr()))) { + this_dist.get_encryption_logonly_replica_attr(), + this_dist.get_columnstore_replica_attr()))) { LOG_WARN("fail to set replica attr array", KR(ret)); } } diff --git a/src/rootserver/ob_locality_util.h b/src/rootserver/ob_locality_util.h index 95b13eb79..5d70e37bf 100644 --- a/src/rootserver/ob_locality_util.h +++ b/src/rootserver/ob_locality_util.h @@ -35,7 +35,6 @@ namespace schema { struct ObZoneRegion; class ObSchemaGetterGuard; -class ObLocality; class ObSimpleTableSchemaV2; class ObTablegroupSchema; class ObTenantSchema; @@ -80,12 +79,6 @@ public: int64_t &pos); int get_zone_replica_attr_array( common::ObIArray &zone_replica_num_array); - int get_zone_replica_num( - const common::ObZone &zone, - share::ObReplicaNumSet &replica_num_set); - int get_zone_replica_num( - const common::ObZone &zone, - share::ObZoneReplicaAttrSet &zone_replica_num_set); public: static const int64_t ALL_SERVER_CNT = INT64_MAX; private: @@ -94,24 +87,12 @@ private: static const int32_t LOGONLY_REPLICA = 1; static const int32_t READONLY_REPLICA = 2; static const int32_t ENCRYPTION_LOGONLY_REPLICA = 3; - static const int32_t REPLICA_TYPE_MAX = 4; + static const int32_t COLUMNSTORE_REPLICA = 4; + static const int32_t REPLICA_TYPE_MAX = 5; private: static const int64_t MAX_BUCKET_NUM = 2 * common::MAX_ZONE_NUM; static const int64_t INVALID_CURSOR = -1; static const int64_t INVALID_COUNT = -1; - // full replica - static const char *const FULL_REPLICA_STR; - static const char *const F_REPLICA_STR; - // logonly replica - static const char *const LOGONLY_REPLICA_STR; - static const char *const L_REPLICA_STR; - // readonly replica - static const char *const READONLY_REPLICA_STR; - static const char *const R_REPLICA_STR; - // encryption logonly replica - static const char *const ENCRYPTION_LOGONLY_REPLICA_STR; - static const char *const E_REPLICA_STR; - // others static const common::ObZone EVERY_ZONE; static const char *const ALL_SERVER_STR; static const char *const MEMSTORE_PERCENT_STR; @@ -173,6 +154,12 @@ private: : 0); return num; } + inline int64_t get_columnstore_replica_num() const { + int64_t num = (all_replica_attr_array_[COLUMNSTORE_REPLICA].count() > 0 + ? all_replica_attr_array_[COLUMNSTORE_REPLICA].at(0).num_ + : 0); + return num; + } inline const ReplicaAttrArray &get_full_replica_attr() const { return all_replica_attr_array_[FULL_REPLICA]; } @@ -185,6 +172,9 @@ private: inline const ReplicaAttrArray &get_encryption_logonly_replica_attr() const { return all_replica_attr_array_[ENCRYPTION_LOGONLY_REPLICA]; } + inline const ReplicaAttrArray &get_columnstore_replica_attr() const { + return all_replica_attr_array_[COLUMNSTORE_REPLICA]; + } inline const common::ObIArray &get_zone_set() const { return zone_set_; } public: int format_to_locality_str(char *buf, int64_t buf_len, int64_t &pos) const; @@ -199,7 +189,8 @@ private: "full_replica_attr", all_replica_attr_array_[FULL_REPLICA], "logonly_replica_attr", all_replica_attr_array_[LOGONLY_REPLICA], "readonly_replica_attr", all_replica_attr_array_[READONLY_REPLICA], - "encryption_logonly_replica_attr", all_replica_attr_array_[ENCRYPTION_LOGONLY_REPLICA]); + "encryption_logonly_replica_attr", all_replica_attr_array_[ENCRYPTION_LOGONLY_REPLICA], + "columnstore_replica_attr", all_replica_attr_array_[COLUMNSTORE_REPLICA]); private: bool specific_replica_need_format( const ReplicaTypeID replica_type) const; diff --git a/src/rootserver/ob_replica_addr.h b/src/rootserver/ob_replica_addr.h index 02c75407e..06c17b70f 100644 --- a/src/rootserver/ob_replica_addr.h +++ b/src/rootserver/ob_replica_addr.h @@ -35,7 +35,7 @@ struct ObReplicaAddr initial_leader_(false), addr_(), zone_(), - replica_type_(common::REPLICA_TYPE_MAX), + replica_type_(common::REPLICA_TYPE_INVALID), replica_property_() {} void reset() { *this = ObReplicaAddr(); } int64_t get_memstore_percent() const {return replica_property_.get_memstore_percent();} diff --git a/src/rootserver/ob_root_service.cpp b/src/rootserver/ob_root_service.cpp index 80b342bb5..c2ee2855e 100755 --- a/src/rootserver/ob_root_service.cpp +++ b/src/rootserver/ob_root_service.cpp @@ -2565,14 +2565,10 @@ int ObRootService::create_resource_pool(const obrpc::ObCreateResourcePoolArg &ar LOG_USER_ERROR(OB_MISS_ARGUMENT, "unit_num"); } LOG_WARN("missing arg to create resource pool", K(arg), K(ret)); - } else if (REPLICA_TYPE_LOGONLY != arg.replica_type_ - && REPLICA_TYPE_FULL != arg.replica_type_) { + } else if (REPLICA_TYPE_FULL != arg.replica_type_) { ret = OB_NOT_SUPPORTED; - LOG_WARN("only full/logonly pool are supported", K(ret), K(arg)); - } else if (REPLICA_TYPE_LOGONLY == arg.replica_type_ - && arg.unit_num_> 1) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("logonly resource pool should only have one unit on one zone", K(ret), K(arg)); + LOG_WARN("only full replica pool are supported", K(ret), K(arg)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "replica_type of resource pool other than FULL replica"); } else if (0 == arg.unit_.case_compare(OB_STANDBY_UNIT_CONFIG_TEMPLATE_NAME)) { ret = OB_OP_NOT_ALLOW; LOG_WARN("can not create resource pool use standby unit config template", K(ret), K(arg)); @@ -9374,7 +9370,7 @@ int ObRootService::add_rs_event_for_alter_ls_replica_( ROOTSERVICE_EVENT_ADD(ADD_EVENT_FOR_ALTER_LS_REPLICA, "ls_id", arg.get_ls_id().id(), "target_replica", arg.get_server_addr(), - "replica_type", replica_type_to_str(arg.get_replica_type()), + "replica_type", share::ObShareUtil::replica_type_to_string(arg.get_replica_type()), "", NULL, extra_info); } else if (arg.get_alter_task_type().is_migrate_task()) { @@ -9641,6 +9637,31 @@ int ObRootService::check_restore_tenant_valid(const share::ObPhysicalRestoreJob } } } + // check if loclaity contains any C replica + ObLocalityDistribution locality_dist; + common::ObArray zone_region_list; + common::ObArray zone_replica_num_array; + if (OB_FAIL(ret)) { + // already failed + } else if (OB_FAIL(locality_dist.init())) { + LOG_WARN("fail to init locality dist", K(ret)); + } else if (OB_FAIL(ddl_service_.construct_zone_region_list(zone_region_list, zones))) { + LOG_WARN("fail to construct zone region list", K(ret)); + } else if (OB_FAIL(locality_dist.parse_locality( + job_info.get_locality(), zones, &zone_region_list))) { + LOG_WARN("fail to parse locality", K(ret)); + } else if (OB_FAIL(locality_dist.get_zone_replica_attr_array(zone_replica_num_array))) { + LOG_WARN("fail to get zone region replica num array", K(ret)); + } else { + FOREACH_X(zone_replica_attr, zone_replica_num_array, OB_SUCC(ret)) { + if (zone_replica_attr->get_columnstore_replica_num() > 0) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("restore tenant with C replica not supported", KR(ret), + "locality_str", job_info.get_locality(), K(zone_replica_num_array)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "restore tenant with COLUMNSTORE replica in locality is"); + } + } + } } } //TODO check if need check R replica diff --git a/src/rootserver/ob_root_utils.cpp b/src/rootserver/ob_root_utils.cpp index c0d269627..bb4d45e6c 100644 --- a/src/rootserver/ob_root_utils.cpp +++ b/src/rootserver/ob_root_utils.cpp @@ -569,188 +569,6 @@ int ObTenantGroupParser::jump_to_next_ttg( return ret; } -int ObLocalityTaskHelp::filter_logonly_task(const common::ObIArray &pools, - ObUnitManager &unit_mgr, - ObIArray &zone_locality) -{ - int ret = OB_SUCCESS; - ObArray logonly_unit_infos; - ObArray unit_infos; - if (pools.count() <= 0) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(pools)); - } else if (OB_FAIL(unit_mgr.get_unit_infos(pools, unit_infos))) { - LOG_WARN("fail to get unit infos", K(ret), K(pools)); - } else { - for (int64_t i = 0; i < unit_infos.count() && OB_SUCC(ret); ++i) { - if (REPLICA_TYPE_LOGONLY != unit_infos.at(i).unit_.replica_type_) { - // only L unit is counted - } else if (OB_FAIL(logonly_unit_infos.push_back(unit_infos.at(i)))) { - LOG_WARN("fail to push back", K(ret), K(i), K(unit_infos)); - } - } - for (int64_t i = 0; i < zone_locality.count() && OB_SUCC(ret); ++i) { - share::ObZoneReplicaAttrSet &zone_replica_attr_set = zone_locality.at(i); - if (zone_replica_attr_set.get_logonly_replica_num() - + zone_replica_attr_set.get_encryption_logonly_replica_num() <= 0) { - // no L replica : nothing todo - } else if (zone_replica_attr_set.zone_set_.count() <= 0) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("zone set unexpected", K(ret), K(zone_replica_attr_set)); - } else { - for (int64_t j = 0; j < logonly_unit_infos.count(); j++) { - const ObUnitInfo &unit_info = logonly_unit_infos.at(j); - if (!has_exist_in_array(zone_replica_attr_set.zone_set_, unit_info.unit_.zone_)) { - // bypass - } else if (zone_replica_attr_set.get_logonly_replica_num() - + zone_replica_attr_set.get_encryption_logonly_replica_num() <= 0) { - // bypass - } else if (zone_replica_attr_set.get_logonly_replica_num() > 0) { - ret = zone_replica_attr_set.sub_logonly_replica_num(ReplicaAttr(1, 100)); - } else { - ret = zone_replica_attr_set.sub_encryption_logonly_replica_num(ReplicaAttr(1, 100)); - } - } - } - } - } - return ret; -} - -int ObLocalityTaskHelp::get_logonly_task_with_logonly_unit(const uint64_t tenant_id, - ObUnitManager &unit_mgr, - share::schema::ObSchemaGetterGuard &schema_guard, - ObIArray &zone_locality) -{ - int ret = OB_SUCCESS; - ObArray logonly_unit_infos; - const ObTenantSchema *tenant_schema = NULL; - zone_locality.reset(); - common::ObArray tenant_zone_locality; - if (OB_INVALID_ID == tenant_id) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(tenant_id)); - } else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id, tenant_schema))) { - LOG_WARN("fail to get tenant info", K(ret), K(tenant_id)); - } else if (OB_ISNULL(tenant_schema)) { - ret = OB_TENANT_NOT_EXIST; - LOG_WARN("get invalid tenant schema", K(ret), K(tenant_schema)); - } else if (OB_FAIL(unit_mgr.get_logonly_unit_by_tenant(tenant_id, logonly_unit_infos))) { - LOG_WARN("fail to get logonly unit infos", K(ret), K(tenant_id)); - } else if (OB_FAIL(tenant_schema->get_zone_replica_attr_array(tenant_zone_locality))) { - LOG_WARN("fail to get zone replica attr array", K(ret)); - } else { - share::ObZoneReplicaNumSet logonly_set; - for (int64_t i = 0; i < logonly_unit_infos.count() && OB_SUCC(ret); i++) { - const ObUnitInfo &unit = logonly_unit_infos.at(i); - for (int64_t j = 0; j < tenant_zone_locality.count(); j++) { - logonly_set.reset(); - const ObZoneReplicaNumSet &zone_set = tenant_zone_locality.at(j); - if (zone_set.zone_ == unit.unit_.zone_ - && zone_set.get_logonly_replica_num() == 1) { - logonly_set.zone_ = zone_set.zone_; - if (OB_FAIL(logonly_set.replica_attr_set_.add_logonly_replica_num(ReplicaAttr(1, 100)))) { - LOG_WARN("fail to add logonly replica num", K(ret)); - } else if (OB_FAIL(zone_locality.push_back(logonly_set))) { - LOG_WARN("fail to push back", K(ret)); - } - } else if (zone_set.zone_ == unit.unit_.zone_ - && zone_set.get_encryption_logonly_replica_num() == 1) { - logonly_set.zone_ = zone_set.zone_; - if (OB_FAIL(logonly_set.replica_attr_set_.add_encryption_logonly_replica_num(ReplicaAttr(1, 100)))) { - LOG_WARN("fail to add logonly replica num", K(ret)); - } else if (OB_FAIL(zone_locality.push_back(logonly_set))) { - LOG_WARN("fail to push back", K(ret)); - } - } - } - } - } - return ret; -} - -int ObLocalityTaskHelp::filter_logonly_task(const uint64_t tenant_id, - ObUnitManager &unit_mgr, - share::schema::ObSchemaGetterGuard &schema_guard, - ObIArray &zone_locality) -{ - int ret = OB_SUCCESS; - ObArray logonly_unit_infos; - if (OB_FAIL(unit_mgr.get_logonly_unit_by_tenant(schema_guard, tenant_id, logonly_unit_infos))) { - LOG_WARN("fail to get loggonly unit by tenant", K(ret), K(tenant_id)); - } else { - LOG_DEBUG("get all logonly unit", K(tenant_id), K(logonly_unit_infos), K(zone_locality)); - for (int64_t i = 0; i < zone_locality.count() && OB_SUCC(ret); ++i) { - share::ObZoneReplicaAttrSet &zone_replica_attr_set = zone_locality.at(i); - if (zone_replica_attr_set.get_logonly_replica_num() - + zone_replica_attr_set.get_encryption_logonly_replica_num() <= 0) { - // no L replica : nothing todo - } else if (zone_replica_attr_set.zone_set_.count() <= 0) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("zone set unexpected", K(ret), K(zone_replica_attr_set)); - } else { - for (int64_t j = 0; j < logonly_unit_infos.count(); j++) { - const ObUnitInfo &unit_info = logonly_unit_infos.at(j); - if (!has_exist_in_array(zone_replica_attr_set.zone_set_, unit_info.unit_.zone_)) { - // bypass - } else if (zone_replica_attr_set.get_logonly_replica_num() - + zone_replica_attr_set.get_encryption_logonly_replica_num() <= 0) { - // bypass - } else if (zone_replica_attr_set.get_logonly_replica_num() > 0) { - ret = zone_replica_attr_set.sub_logonly_replica_num(ReplicaAttr(1, 100)); - } else { - ret = zone_replica_attr_set.sub_encryption_logonly_replica_num(ReplicaAttr(1, 100)); - } - } - } - } - } - return ret; -} - -int ObLocalityTaskHelp::alloc_logonly_replica(ObUnitManager &unit_mgr, - const ObIArray &pools, - const common::ObIArray &zone_locality, - ObPartitionAddr &partition_addr) -{ - int ret = OB_SUCCESS; - ObArray logonly_units; - ObArray unit_infos; - if (OB_FAIL(unit_mgr.get_unit_infos(pools, unit_infos))) { - LOG_WARN("fail to get unit infos", K(ret), K(pools)); - } else { - for (int64_t i = 0; i < unit_infos.count() && OB_SUCC(ret); i++) { - if (REPLICA_TYPE_LOGONLY != unit_infos.at(i).unit_.replica_type_) { - //nothing todo - } else if (OB_FAIL(logonly_units.push_back(unit_infos.at(i)))) { - LOG_WARN("fail to push back", K(ret), K(i), K(unit_infos)); - } - } - } - ObReplicaAddr raddr; - for (int64_t i = 0; i < logonly_units.count() && OB_SUCC(ret); i++) { - for (int64_t j = 0; j < zone_locality.count() && OB_SUCC(ret); j++) { - if (zone_locality.at(j).zone_ == logonly_units.at(i).unit_.zone_ - && (zone_locality.at(j).get_logonly_replica_num() == 1 - || zone_locality.at(j).get_encryption_logonly_replica_num() == 1)) { - raddr.reset(); - raddr.unit_id_ = logonly_units.at(i).unit_.unit_id_; - raddr.addr_ = logonly_units.at(i).unit_.server_; - raddr.zone_ = logonly_units.at(i).unit_.zone_; - raddr.replica_type_ = zone_locality.at(j).get_logonly_replica_num() == 1 - ? REPLICA_TYPE_LOGONLY - : REPLICA_TYPE_ENCRYPTION_LOGONLY; - if (OB_FAIL(partition_addr.push_back(raddr))) { - LOG_WARN("fail to push back", K(ret), K(raddr)); - } else { - LOG_INFO("alloc partition for logonly replica", K(raddr)); - } - } - } - } - return ret; -} - int ObLocalityCheckHelp::calc_paxos_replica_num( const common::ObIArray &zone_locality, int64_t &paxos_num) @@ -1562,6 +1380,7 @@ int ObLocalityCheckHelp::check_alter_single_zone_locality_valid( int ret = OB_SUCCESS; bool is_legal = true; // 1. check whether non_paxos member change + // check R-replica if (!non_paxos_locality_modified) { const ObIArray &pre_readonly_replica = orig_locality.replica_attr_set_.get_readonly_replica_attr_array(); const ObIArray &cur_readonly_replica = new_locality.replica_attr_set_.get_readonly_replica_attr_array(); @@ -1577,8 +1396,19 @@ int ObLocalityCheckHelp::check_alter_single_zone_locality_valid( } } } + // check C-replica + if (new_locality.get_columnstore_replica_num() != orig_locality.get_columnstore_replica_num()) { + if (new_locality.get_full_replica_num() != orig_locality.get_full_replica_num() + || new_locality.get_readonly_replica_num() != orig_locality.get_readonly_replica_num()) { + // transform between R/F and C is illegal + is_legal = false; + } else { + non_paxos_locality_modified = true; + } + } // 2. check whether alter locality is legal. - if (new_locality.get_logonly_replica_num() < orig_locality.get_logonly_replica_num()) { + if (!is_legal) { + } else if (new_locality.get_logonly_replica_num() < orig_locality.get_logonly_replica_num()) { // L-replica must not transfrom to other replica type. if (new_locality.get_full_replica_num() > orig_locality.get_full_replica_num()) { is_legal = false; // maybe L->F diff --git a/src/rootserver/ob_root_utils.h b/src/rootserver/ob_root_utils.h index c2f9300a2..7c84a209c 100644 --- a/src/rootserver/ob_root_utils.h +++ b/src/rootserver/ob_root_utils.h @@ -36,7 +36,6 @@ namespace schema { class ObMultiVersionSchemaService; class ObTableSchema; -class ObLocality; class ObSchemaGetterGuard; } } @@ -394,35 +393,6 @@ public: } }; -class ObLocalityTaskHelp -{ -public: - ObLocalityTaskHelp() {} - virtual ~ObLocalityTaskHelp() {} - static int filter_logonly_task( - const common::ObIArray &pools, - ObUnitManager &unit_mgr, - common::ObIArray &zone_locality); - - static int filter_logonly_task( - const uint64_t tenant_id, - ObUnitManager &unit_manager, - share::schema::ObSchemaGetterGuard &schema_guard, - common::ObIArray &zone_locality); - - static int alloc_logonly_replica( - ObUnitManager &unit_manager, - const common::ObIArray &pools, - const common::ObIArray &zone_locality, - ObPartitionAddr &partition_addr); - - static int get_logonly_task_with_logonly_unit( - const uint64_t tenant_id, - ObUnitManager &unit_mgr, - share::schema::ObSchemaGetterGuard &schema_guard, - common::ObIArray &zone_locality); -}; - enum PaxosReplicaNumberTaskType { NOP_PAXOS_REPLICA_NUMBER = 0, diff --git a/src/rootserver/ob_unit_manager.cpp b/src/rootserver/ob_unit_manager.cpp index 2f978a99d..47f3bee9a 100644 --- a/src/rootserver/ob_unit_manager.cpp +++ b/src/rootserver/ob_unit_manager.cpp @@ -2744,7 +2744,7 @@ int ObUnitManager::check_old_pool_name_condition( common::ObIArray &old_pool) { int ret = OB_SUCCESS; - common::ObReplicaType replica_type = REPLICA_TYPE_MAX; + common::ObReplicaType replica_type = REPLICA_TYPE_INVALID; uint64_t tenant_id = OB_INVALID_ID; share::ObUnitConfig *unit_config = NULL; int64_t unit_count = 0; diff --git a/src/rootserver/virtual_table/ob_all_virtual_ls_replica_task_plan.cpp b/src/rootserver/virtual_table/ob_all_virtual_ls_replica_task_plan.cpp index 589179b9c..46acb34c3 100644 --- a/src/rootserver/virtual_table/ob_all_virtual_ls_replica_task_plan.cpp +++ b/src/rootserver/virtual_table/ob_all_virtual_ls_replica_task_plan.cpp @@ -184,11 +184,11 @@ int ObAllVirtualLSReplicaTaskPlan::get_full_row_( ADD_COLUMN(set_varchar, table, "target_replica_svr_ip", target_ip_str, columns); ADD_COLUMN(set_int, table, "target_replica_svr_port", target_port, columns); ADD_COLUMN(set_int, table, "target_paxos_replica_number", task_stat.get_target_replica_paxos_replica_number(), columns); - ADD_COLUMN(set_varchar, table, "target_replica_type", ob_replica_type_strs(task_stat.get_target_replica_type()), columns); + ADD_COLUMN(set_varchar, table, "target_replica_type", ObShareUtil::replica_type_to_string(task_stat.get_target_replica_type()), columns); ADD_COLUMN(set_varchar, table, "source_replica_svr_ip", task_stat.get_source_server().is_valid() ? source_ip_str : "", columns); ADD_COLUMN(set_int, table, "source_replica_svr_port", source_port, columns); ADD_COLUMN(set_int, table, "source_paxos_replica_number", task_stat.get_source_replica_paxos_replica_number(), columns); - ADD_COLUMN(set_varchar, table, "source_replica_type", task_stat.get_source_server().is_valid() ? ob_replica_type_strs(task_stat.get_source_replica_type()) : "", columns); + ADD_COLUMN(set_varchar, table, "source_replica_type", task_stat.get_source_server().is_valid() ? ObShareUtil::replica_type_to_string(task_stat.get_source_replica_type()) : "", columns); ADD_COLUMN(set_varchar, table, "task_exec_svr_ip", execute_ip_str, columns); ADD_COLUMN(set_int, table, "task_exec_svr_port", execute_port, columns); ADD_COLUMN(set_varchar, table, "comment", task_stat.get_comment().string(), columns); diff --git a/src/share/CMakeLists.txt b/src/share/CMakeLists.txt index 8687880dd..b83f6be79 100644 --- a/src/share/CMakeLists.txt +++ b/src/share/CMakeLists.txt @@ -127,7 +127,6 @@ ob_set_subtarget(ob_share common ob_list_parser.cpp ob_local_device.cpp ob_locality_info.cpp - ob_locality_parser.cpp ob_locality_priority.cpp ob_locality_table_operator.cpp ob_ls_id.cpp diff --git a/src/share/client_feedback/ob_feedback_partition_struct.h b/src/share/client_feedback/ob_feedback_partition_struct.h index d3b226f9e..23ba2f526 100644 --- a/src/share/client_feedback/ob_feedback_partition_struct.h +++ b/src/share/client_feedback/ob_feedback_partition_struct.h @@ -61,7 +61,7 @@ inline bool ObFeedbackReplicaLocation::is_valid_obj() const { return server_.is_valid() && (common::INVALID_ROLE != role_) - && (common::REPLICA_TYPE_MAX != replica_type_); + && (common::REPLICA_TYPE_INVALID != replica_type_); } class ObFeedbackPartitionLocation : public ObAbstractFeedbackObject @@ -195,7 +195,7 @@ inline bool ObFeedbackRerouteInfo::is_valid_obj() const return (for_session_reroute_ && server_.is_valid()) || (!for_session_reroute_ && server_.is_valid() && (common::INVALID_ROLE != role_) - && (common::REPLICA_TYPE_MAX != replica_type_) + && (common::REPLICA_TYPE_INVALID != replica_type_) && tbl_name_len_ > 0 && tbl_name_len_ <= common::OB_MAX_TABLE_NAME_LENGTH && OB_INVALID_VERSION != tbl_schema_version_); diff --git a/src/share/compaction/ob_compaction_locality_cache.cpp b/src/share/compaction/ob_compaction_locality_cache.cpp index 5d5f3a257..a98370f7b 100644 --- a/src/share/compaction/ob_compaction_locality_cache.cpp +++ b/src/share/compaction/ob_compaction_locality_cache.cpp @@ -11,6 +11,7 @@ #include "share/compaction/ob_compaction_locality_cache.h" #include "src/storage/compaction/ob_medium_compaction_func.h" #include "src/storage/compaction/ob_compaction_util.h" +#include "src/storage/tx_storage/ob_ls_service.h" #include "src/share/ob_zone_merge_info.h" #include "observer/ob_server_struct.h" #include "src/share/ob_zone_merge_table_operator.h" @@ -22,11 +23,233 @@ namespace oceanbase namespace share { +/****************************** ObLSReplicaUniItem ******************************/ +ObLSReplicaUniItem::ObLSReplicaUniItem() + : ls_id_(), + server_() +{} + +ObLSReplicaUniItem::ObLSReplicaUniItem(const ObLSID &ls_id, const common::ObAddr &server) + : ls_id_(ls_id), + server_(server) +{} + +ObLSReplicaUniItem::~ObLSReplicaUniItem() +{ + reset(); +} + +void ObLSReplicaUniItem::reset() { + ls_id_.reset(); + server_.reset(); +} + +uint64_t ObLSReplicaUniItem::hash() const +{ + uint64_t hash_val = 0; + hash_val += ls_id_.hash(); + hash_val += server_.hash(); + return hash_val; +} + +int ObLSReplicaUniItem::hash(uint64_t &hash_val) const +{ + hash_val = hash(); + return OB_SUCCESS; +} + +bool ObLSReplicaUniItem::is_valid() const +{ + return ls_id_.is_valid() && server_.is_valid(); +} + +bool ObLSReplicaUniItem::operator == (const ObLSReplicaUniItem &other) const +{ + bool bret = true; + if (this == &other) { + } else if (ls_id_ != other.ls_id_ || server_ != other.server_) { + bret = false; + } + return bret; +} + +bool ObLSReplicaUniItem::operator != (const ObLSReplicaUniItem &other) const +{ + return !(*this == other); +} + +/****************************** ObLSColumnReplicaCache ******************************/ +ObLSColumnReplicaCache::ObLSColumnReplicaCache() + : is_inited_(false), + ls_id_set_(), + ls_replica_set_() +{} + +ObLSColumnReplicaCache::~ObLSColumnReplicaCache() +{ + destroy(); +} + +// init +int ObLSColumnReplicaCache::init() +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", K(ret)); + } else if (OB_FAIL(ls_id_set_.create(BUCKET_NUM_OF_LS_ID_SET, ObMemAttr(MTL_ID(), "LSIDsForCkm")))) { + LOG_WARN("fail to create ls id set", K(ret)); + } else if (OB_FAIL(ls_replica_set_.create(BUCKET_NUM_OF_LS_REPLICA_SET, ObMemAttr(MTL_ID(), "LSReplTypes")))) { + LOG_WARN("fail to create ls replica type map", K(ret)); + } else { + is_inited_ = true; + } + return ret; +} + +void ObLSColumnReplicaCache::destroy() +{ + int ret = OB_SUCCESS; // only for log + if (ls_replica_set_.created()) { + if (OB_FAIL(ls_replica_set_.destroy())) { + LOG_WARN("fail to destroy ls replica set", K(ret)); + } + } + if (ls_id_set_.created()) { + if (OB_FAIL(ls_id_set_.destroy())) { + LOG_WARN("fail to destroy ls replica set", K(ret)); + } + } + is_inited_ = false; +} + +void ObLSColumnReplicaCache::reuse() +{ + ls_id_set_.reuse(); + ls_replica_set_.reuse(); +} + +int ObLSColumnReplicaCache::check_contains_ls(const ObLSID &ls_id, bool &contained) const +{ + int ret = OB_SUCCESS; + contained = false; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", K(ret)); + } else if (OB_FAIL(ls_id_set_.exist_refactored(ls_id))) { + if (OB_HASH_EXIST == ret || OB_HASH_NOT_EXIST == ret) { + contained = (OB_HASH_EXIST == ret); + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to check contains ls", K(ret), K(ls_id), KPC(this)); + } + } + return ret; +} + +int ObLSColumnReplicaCache::mark_ls_finished(const ObLSID &ls_id) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", K(ret)); + } else if (OB_FAIL(ls_id_set_.set_refactored(ls_id))) { + if (OB_HASH_EXIST == ret) { + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to mark ls finish", K(ret), K(ls_id), KPC(this)); + } + } + return ret; +} + +int ObLSColumnReplicaCache::add_cs_replica(const ObLSReplicaUniItem &ls_item) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", K(ret)); + } else if (OB_FAIL(ls_replica_set_.set_refactored(ls_item))) { + LOG_WARN("fail to add col replica", K(ret), K(ls_item), KPC(this)); + } + return ret; +} + +int ObLSColumnReplicaCache::update(const ObLSID &ls_id, const ObAddr &server) +{ + int ret = OB_SUCCESS; + bool is_contained = false; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", K(ret)); + } else if (OB_FAIL(check_contains_ls(ls_id, is_contained))) { + LOG_WARN("fail to check exist for ls", K(ls_id), KPC(this)); + } else if (is_contained) { + } else if (OB_ISNULL(GCTX.lst_operator_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("lst operator is null", K(ret)); + } else { + const int64_t tenant_id = MTL_ID(); + const int64_t cluster_id = GCONF.cluster_id; + ObLSInfo ls_info; + if (OB_FAIL(GCTX.lst_operator_->get(cluster_id, tenant_id, ls_id, ObLSTable::DEFAULT_MODE, ls_info))) { + LOG_WARN("fail to get ls info", K(ret), K(cluster_id), K(tenant_id), K(ls_id)); + } else { + const ObLSInfo::ReplicaArray &all_replicas = ls_info.get_replicas(); + ObLSReplicaUniItem ls_item(ls_id, server); + for (int64_t i = 0; i < all_replicas.count() && OB_SUCC(ret); ++i) { + const ObLSReplica &replica = all_replicas.at(i); + if (ObRole::LEADER == replica.get_role()) { + const common::GlobalLearnerList &learner_list = replica.get_learner_list(); + for (int64_t i = 0; OB_SUCC(ret) && i < learner_list.get_member_number(); ++i) { + ObMember learner; + if (OB_FAIL(learner_list.get_learner(i, learner))) { + LOG_WARN("fail to get learner", K(ret), K(i), K(learner_list)); + } else if (learner.is_columnstore()) { + ls_item.server_ = learner.get_server(); + if (OB_FAIL(add_cs_replica(ls_item))) { + LOG_WARN("fail to add ls item", K(ret), K(ls_item)); + } + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(mark_ls_finished(ls_item.ls_id_))) { + LOG_WARN("fail to make ls finished", K(ret)); + } + LOG_TRACE("[CS-Replica] get learner list", K(ret), K(learner_list)); + } + } + } + } + return ret; +} + +int ObLSColumnReplicaCache::check_is_cs_replica(const ObLSReplicaUniItem &ls_item, bool &is_cs_replica) const +{ + int ret = OB_SUCCESS; + is_cs_replica = false; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", K(ret)); + } else if (OB_FAIL(ls_replica_set_.exist_refactored(ls_item))) { + if (OB_HASH_EXIST == ret || OB_HASH_NOT_EXIST == ret) { + is_cs_replica = (OB_HASH_EXIST == ret); + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to check contains ls", K(ret), K(ls_item), KPC(this)); + } + } + LOG_TRACE("[CS-Replica] check current ls is cs replica", K(ret), K(ls_item), K(is_cs_replica), KPC(this)); + return ret; +} + +/****************************** ObCompactionLocalityCache ******************************/ ObCompactionLocalityCache::ObCompactionLocalityCache() : is_inited_(false), tenant_id_(OB_INVALID_TENANT_ID), merge_info_mgr_(nullptr), - ls_infos_map_() + ls_infos_map_(), + ls_cs_replica_cache_() {} ObCompactionLocalityCache::~ObCompactionLocalityCache() @@ -45,6 +268,8 @@ int ObCompactionLocalityCache::init(const uint64_t tenant_id, rootserver::ObMajo LOG_WARN("invalid argument", K(ret), K(tenant_id)); } else if (OB_FAIL(ls_infos_map_.create(OB_MAX_LS_NUM_PER_TENANT_PER_SERVER, "CaLsInfoMap", "CaLsInfoNode", tenant_id))) { LOG_WARN("fail to create ls info map", K(ret)); + } else if (OB_FAIL(ls_cs_replica_cache_.init())) { + LOG_WARN("fail to init col replica cache", K(ret)); } if (OB_FAIL(ret)) { destroy(); @@ -100,6 +325,7 @@ int ObCompactionLocalityCache::inner_refresh_ls_locality() if (OB_SUCC(ret)) { // 1. clear ls_infos cached in memory ls_infos_map_.reuse(); + ls_cs_replica_cache_.reuse(); // 2. load ls_infos from __all_ls_meta_table ObArray ls_infos; ls_infos.set_attr(ObMemAttr(tenant_id_, "RefLSInfos")); @@ -230,12 +456,16 @@ int ObCompactionLocalityCache::refresh_by_zone( } else if (OB_FAIL(tmp_ls_info.init_by_replica(tmp_replica))) { LOG_WARN("fail to init ls_info by replica", KR(ret), K(tmp_replica)); } + if (OB_FAIL(ret)) { + } else if (tmp_replica.is_column_replica() && OB_FAIL(ls_cs_replica_cache_.add_cs_replica(ObLSReplicaUniItem(ls_id, tmp_replica.get_server())))) { + LOG_WARN("fail to add cs replica", K(ret), K(ls_id), K(tmp_replica), K_(ls_cs_replica_cache)); + } } } if (FAILEDx(ls_infos_map_.set_refactored(ls_id, tmp_ls_info, 1/*overwrite*/))) { LOG_WARN("fail to set refactored", KR(ret), K(ls_id), K(tmp_ls_info)); } else { - FLOG_INFO("success to refresh cached ls_info", K(ret), K(tmp_ls_info), K(zone_list), K(member_list_array)); + FLOG_INFO("success to refresh cached ls_info", K(ret), K(tmp_ls_info), K(zone_list)); } } return ret; diff --git a/src/share/compaction/ob_compaction_locality_cache.h b/src/share/compaction/ob_compaction_locality_cache.h index fac8249fa..e0c290fee 100644 --- a/src/share/compaction/ob_compaction_locality_cache.h +++ b/src/share/compaction/ob_compaction_locality_cache.h @@ -29,6 +29,46 @@ class ObMySQLResult; namespace share { +struct ObLSReplicaUniItem +{ + ObLSReplicaUniItem(); + ObLSReplicaUniItem(const ObLSID &ls_id, const common::ObAddr &server); + ~ObLSReplicaUniItem(); + void reset(); + uint64_t hash() const; + int hash(uint64_t &hash_val) const; + bool is_valid() const; + bool operator == (const ObLSReplicaUniItem &other) const; + bool operator != (const ObLSReplicaUniItem &other) const; + TO_STRING_KV(K_(ls_id), K_(server)); + + share::ObLSID ls_id_; + common::ObAddr server_; +}; + +class ObLSColumnReplicaCache +{ +public: + ObLSColumnReplicaCache(); + ~ObLSColumnReplicaCache(); + int init(); + void destroy(); + void reuse(); + int check_contains_ls(const ObLSID &ls_id, bool &contained) const; + int mark_ls_finished(const ObLSID &ls_id); + int add_cs_replica(const ObLSReplicaUniItem &ls_item); + int update(const ObLSID &ls_id, const ObAddr &server); + int check_is_cs_replica(const ObLSReplicaUniItem &ls_item, bool &is_cs_replica) const; + TO_STRING_KV(K_(is_inited), K_(ls_id_set), K_(ls_replica_set)); +private: + const static int64_t BUCKET_NUM_OF_LS_ID_SET = 15; + const static int64_t BUCKET_NUM_OF_LS_REPLICA_SET = 31; +private: + bool is_inited_; + hash::ObHashSet ls_id_set_; // pre-wamred ls id, unused + hash::ObHashSet ls_replica_set_; // cs-prelica ls +}; + class ObCompactionLocalityCache { public: @@ -39,6 +79,7 @@ public: bool empty() const { return ls_infos_map_.empty(); } int refresh_ls_locality(const bool force_refresh); int get_ls_info(const share::ObLSID &ls_id, share::ObLSInfo &ls_info); + const share::ObLSColumnReplicaCache& get_cs_replica_cache() const { return ls_cs_replica_cache_; } TO_STRING_KV(K_(is_inited), K_(tenant_id)); private: @@ -62,6 +103,7 @@ private: uint64_t tenant_id_; rootserver::ObMajorMergeInfoManager *merge_info_mgr_; common::hash::ObHashMap ls_infos_map_; + share::ObLSColumnReplicaCache ls_cs_replica_cache_; }; } // namespace share diff --git a/src/share/inner_table/ob_inner_table_schema.21151_21200.cpp b/src/share/inner_table/ob_inner_table_schema.21151_21200.cpp index 23e8a030e..8e2145813 100644 --- a/src/share/inner_table/ob_inner_table_schema.21151_21200.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21151_21200.cpp @@ -560,7 +560,7 @@ int ObInnerTableSchema::dba_ob_resource_pools_schema(ObTableSchema &table_schema table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT RESOURCE_POOL_ID, NAME, CASE TENANT_ID WHEN -1 THEN NULL ELSE TENANT_ID END AS TENANT_ID, gmt_create AS CREATE_TIME, gmt_modified AS MODIFY_TIME, UNIT_COUNT, UNIT_CONFIG_ID, ZONE_LIST, CASE replica_type WHEN 0 THEN "FULL" WHEN 5 THEN "LOGONLY" WHEN 16 THEN "READONLY" WHEN 261 THEN "ENCRYPTION LOGONLY" ELSE NULL END AS REPLICA_TYPE FROM oceanbase.__all_resource_pool )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT RESOURCE_POOL_ID, NAME, CASE TENANT_ID WHEN -1 THEN NULL ELSE TENANT_ID END AS TENANT_ID, gmt_create AS CREATE_TIME, gmt_modified AS MODIFY_TIME, UNIT_COUNT, UNIT_CONFIG_ID, ZONE_LIST, CASE replica_type WHEN 0 THEN "FULL" ELSE NULL END AS REPLICA_TYPE FROM oceanbase.__all_resource_pool )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -910,7 +910,7 @@ int ObInnerTableSchema::dba_ob_ls_locations_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( ( SELECT NOW(6) AS CREATE_TIME, NOW(6) AS MODIFY_TIME, LS_ID, SVR_IP, SVR_PORT, SQL_PORT, ZONE, (CASE ROLE WHEN 1 THEN "LEADER" ELSE "FOLLOWER" END) AS ROLE, (CASE ROLE WHEN 1 THEN MEMBER_LIST ELSE NULL END) AS MEMBER_LIST, (CASE ROLE WHEN 1 THEN PAXOS_REPLICA_NUMBER ELSE NULL END) AS PAXOS_REPLICA_NUMBER, (CASE REPLICA_TYPE WHEN 0 THEN "FULL" WHEN 5 THEN "LOGONLY" WHEN 16 THEN "READONLY" WHEN 261 THEN "ENCRYPTION LOGONLY" ELSE NULL END) AS REPLICA_TYPE, (CASE ROLE WHEN 1 THEN LEARNER_LIST ELSE "" END) AS LEARNER_LIST, (CASE REBUILD WHEN 0 THEN "FALSE" ELSE "TRUE" END) AS REBUILD FROM OCEANBASE.__ALL_VIRTUAL_CORE_META_TABLE WHERE EFFECTIVE_TENANT_ID() = 1 ) UNION ALL ( SELECT GMT_CREATE AS CREATE_TIME, GMT_MODIFIED AS MODIFY_TIME, LS_ID, SVR_IP, SVR_PORT, SQL_PORT, ZONE, (CASE ROLE WHEN 1 THEN "LEADER" ELSE "FOLLOWER" END) AS ROLE, (CASE ROLE WHEN 1 THEN MEMBER_LIST ELSE NULL END) AS MEMBER_LIST, (CASE ROLE WHEN 1 THEN PAXOS_REPLICA_NUMBER ELSE NULL END) AS PAXOS_REPLICA_NUMBER, (CASE REPLICA_TYPE WHEN 0 THEN "FULL" WHEN 5 THEN "LOGONLY" WHEN 16 THEN "READONLY" WHEN 261 THEN "ENCRYPTION LOGONLY" ELSE NULL END) AS REPLICA_TYPE, (CASE ROLE WHEN 1 THEN LEARNER_LIST ELSE "" END) AS LEARNER_LIST, (CASE REBUILD WHEN 0 THEN "FALSE" ELSE "TRUE" END) AS REBUILD FROM OCEANBASE.__ALL_VIRTUAL_LS_META_TABLE WHERE TENANT_ID = EFFECTIVE_TENANT_ID() AND TENANT_ID != 1 ) )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( ( SELECT NOW(6) AS CREATE_TIME, NOW(6) AS MODIFY_TIME, LS_ID, SVR_IP, SVR_PORT, SQL_PORT, ZONE, (CASE ROLE WHEN 1 THEN "LEADER" ELSE "FOLLOWER" END) AS ROLE, (CASE ROLE WHEN 1 THEN MEMBER_LIST ELSE NULL END) AS MEMBER_LIST, (CASE ROLE WHEN 1 THEN PAXOS_REPLICA_NUMBER ELSE NULL END) AS PAXOS_REPLICA_NUMBER, (CASE REPLICA_TYPE WHEN 0 THEN "FULL" WHEN 5 THEN "LOGONLY" WHEN 16 THEN "READONLY" WHEN 261 THEN "ENCRYPTION LOGONLY" WHEN 1040 THEN "COLUMNSTORE" ELSE NULL END) AS REPLICA_TYPE, (CASE ROLE WHEN 1 THEN LEARNER_LIST ELSE "" END) AS LEARNER_LIST, (CASE REBUILD WHEN 0 THEN "FALSE" ELSE "TRUE" END) AS REBUILD FROM OCEANBASE.__ALL_VIRTUAL_CORE_META_TABLE WHERE EFFECTIVE_TENANT_ID() = 1 ) UNION ALL ( SELECT GMT_CREATE AS CREATE_TIME, GMT_MODIFIED AS MODIFY_TIME, LS_ID, SVR_IP, SVR_PORT, SQL_PORT, ZONE, (CASE ROLE WHEN 1 THEN "LEADER" ELSE "FOLLOWER" END) AS ROLE, (CASE ROLE WHEN 1 THEN MEMBER_LIST ELSE NULL END) AS MEMBER_LIST, (CASE ROLE WHEN 1 THEN PAXOS_REPLICA_NUMBER ELSE NULL END) AS PAXOS_REPLICA_NUMBER, (CASE REPLICA_TYPE WHEN 0 THEN "FULL" WHEN 5 THEN "LOGONLY" WHEN 16 THEN "READONLY" WHEN 261 THEN "ENCRYPTION LOGONLY" WHEN 1040 THEN "COLUMNSTORE" ELSE NULL END) AS REPLICA_TYPE, (CASE ROLE WHEN 1 THEN LEARNER_LIST ELSE "" END) AS LEARNER_LIST, (CASE REBUILD WHEN 0 THEN "FALSE" ELSE "TRUE" END) AS REBUILD FROM OCEANBASE.__ALL_VIRTUAL_LS_META_TABLE WHERE TENANT_ID = EFFECTIVE_TENANT_ID() AND TENANT_ID != 1 ) )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -960,7 +960,7 @@ int ObInnerTableSchema::cdb_ob_ls_locations_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( ( SELECT NOW(6) AS CREATE_TIME, NOW(6) AS MODIFY_TIME, TENANT_ID, LS_ID, SVR_IP, SVR_PORT, SQL_PORT, ZONE, (CASE ROLE WHEN 1 THEN "LEADER" ELSE "FOLLOWER" END) AS ROLE, (CASE ROLE WHEN 1 THEN MEMBER_LIST ELSE NULL END) AS MEMBER_LIST, (CASE ROLE WHEN 1 THEN PAXOS_REPLICA_NUMBER ELSE NULL END) AS PAXOS_REPLICA_NUMBER, (CASE REPLICA_TYPE WHEN 0 THEN "FULL" WHEN 5 THEN "LOGONLY" WHEN 16 THEN "READONLY" WHEN 261 THEN "ENCRYPTION LOGONLY" ELSE NULL END) AS REPLICA_TYPE, (CASE ROLE WHEN 1 THEN LEARNER_LIST ELSE "" END) AS LEARNER_LIST, (CASE REBUILD WHEN 0 THEN "FALSE" ELSE "TRUE" END) AS REBUILD FROM OCEANBASE.__ALL_VIRTUAL_CORE_META_TABLE ) UNION ALL ( SELECT GMT_CREATE AS CREATE_TIME, GMT_MODIFIED AS MODIFY_TIME, TENANT_ID, LS_ID, SVR_IP, SVR_PORT, SQL_PORT, ZONE, (CASE ROLE WHEN 1 THEN "LEADER" ELSE "FOLLOWER" END) AS ROLE, (CASE ROLE WHEN 1 THEN MEMBER_LIST ELSE NULL END) AS MEMBER_LIST, (CASE ROLE WHEN 1 THEN PAXOS_REPLICA_NUMBER ELSE NULL END) AS PAXOS_REPLICA_NUMBER, (CASE REPLICA_TYPE WHEN 0 THEN "FULL" WHEN 5 THEN "LOGONLY" WHEN 16 THEN "READONLY" WHEN 261 THEN "ENCRYPTION LOGONLY" ELSE NULL END) AS REPLICA_TYPE, (CASE ROLE WHEN 1 THEN LEARNER_LIST ELSE "" END) AS LEARNER_LIST, (CASE REBUILD WHEN 0 THEN "FALSE" ELSE "TRUE" END) AS REBUILD FROM OCEANBASE.__ALL_VIRTUAL_LS_META_TABLE WHERE TENANT_ID != 1 ) )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( ( SELECT NOW(6) AS CREATE_TIME, NOW(6) AS MODIFY_TIME, TENANT_ID, LS_ID, SVR_IP, SVR_PORT, SQL_PORT, ZONE, (CASE ROLE WHEN 1 THEN "LEADER" ELSE "FOLLOWER" END) AS ROLE, (CASE ROLE WHEN 1 THEN MEMBER_LIST ELSE NULL END) AS MEMBER_LIST, (CASE ROLE WHEN 1 THEN PAXOS_REPLICA_NUMBER ELSE NULL END) AS PAXOS_REPLICA_NUMBER, (CASE REPLICA_TYPE WHEN 0 THEN "FULL" WHEN 5 THEN "LOGONLY" WHEN 16 THEN "READONLY" WHEN 261 THEN "ENCRYPTION LOGONLY" WHEN 1040 THEN "COLUMNSTORE" ELSE NULL END) AS REPLICA_TYPE, (CASE ROLE WHEN 1 THEN LEARNER_LIST ELSE "" END) AS LEARNER_LIST, (CASE REBUILD WHEN 0 THEN "FALSE" ELSE "TRUE" END) AS REBUILD FROM OCEANBASE.__ALL_VIRTUAL_CORE_META_TABLE ) UNION ALL ( SELECT GMT_CREATE AS CREATE_TIME, GMT_MODIFIED AS MODIFY_TIME, TENANT_ID, LS_ID, SVR_IP, SVR_PORT, SQL_PORT, ZONE, (CASE ROLE WHEN 1 THEN "LEADER" ELSE "FOLLOWER" END) AS ROLE, (CASE ROLE WHEN 1 THEN MEMBER_LIST ELSE NULL END) AS MEMBER_LIST, (CASE ROLE WHEN 1 THEN PAXOS_REPLICA_NUMBER ELSE NULL END) AS PAXOS_REPLICA_NUMBER, (CASE REPLICA_TYPE WHEN 0 THEN "FULL" WHEN 5 THEN "LOGONLY" WHEN 16 THEN "READONLY" WHEN 261 THEN "ENCRYPTION LOGONLY" WHEN 1040 THEN "COLUMNSTORE" ELSE NULL END) AS REPLICA_TYPE, (CASE ROLE WHEN 1 THEN LEARNER_LIST ELSE "" END) AS LEARNER_LIST, (CASE REBUILD WHEN 0 THEN "FALSE" ELSE "TRUE" END) AS REBUILD FROM OCEANBASE.__ALL_VIRTUAL_LS_META_TABLE WHERE TENANT_ID != 1 ) )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema.21301_21350.cpp b/src/share/inner_table/ob_inner_table_schema.21301_21350.cpp index 8f09df6f7..afc4d5913 100644 --- a/src/share/inner_table/ob_inner_table_schema.21301_21350.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21301_21350.cpp @@ -817,7 +817,7 @@ int ObInnerTableSchema::cdb_ob_tablet_checksum_error_info_schema(ObTableSchema & table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT TENANT_ID, TABLET_ID FROM ( SELECT TENANT_ID, TABLET_ID, ROW_COUNT, DATA_CHECKSUM, B_COLUMN_CHECKSUMS, COMPACTION_SCN FROM OCEANBASE.__ALL_VIRTUAL_TABLET_REPLICA_CHECKSUM ) J GROUP BY J.TENANT_ID, J.TABLET_ID, J.COMPACTION_SCN HAVING MIN(J.DATA_CHECKSUM) != MAX(J.DATA_CHECKSUM) OR MIN(J.ROW_COUNT) != MAX(J.ROW_COUNT) OR MIN(J.B_COLUMN_CHECKSUMS) != MAX(J.B_COLUMN_CHECKSUMS) )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT TENANT_ID, TABLET_ID FROM ( SELECT CKM.TENANT_ID, CKM.TABLET_ID, CKM.ROW_COUNT, CKM.DATA_CHECKSUM, CKM.B_COLUMN_CHECKSUMS, CKM.COMPACTION_SCN, M.REPLICA_TYPE FROM OCEANBASE.__ALL_VIRTUAL_TABLET_REPLICA_CHECKSUM CKM JOIN OCEANBASE.__ALL_VIRTUAL_LS_META_TABLE M ON CKM.TENANT_ID = M.TENANT_ID AND CKM.LS_ID = M.LS_ID AND CKM.SVR_IP = M.SVR_IP AND CKM.SVR_PORT = M.SVR_PORT ) J GROUP BY J.TENANT_ID, J.TABLET_ID, J.COMPACTION_SCN, J.REPLICA_TYPE HAVING MIN(J.DATA_CHECKSUM) != MAX(J.DATA_CHECKSUM) OR MIN(J.ROW_COUNT) != MAX(J.ROW_COUNT) OR MIN(J.B_COLUMN_CHECKSUMS) != MAX(J.B_COLUMN_CHECKSUMS) )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema.25151_25200.cpp b/src/share/inner_table/ob_inner_table_schema.25151_25200.cpp index 7e67bc32c..6df9edb3d 100644 --- a/src/share/inner_table/ob_inner_table_schema.25151_25200.cpp +++ b/src/share/inner_table/ob_inner_table_schema.25151_25200.cpp @@ -710,7 +710,7 @@ int ObInnerTableSchema::dba_ob_ls_locations_ora_schema(ObTableSchema &table_sche table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(TO_CHAR(GMT_CREATE) AS VARCHAR2(19)) AS CREATE_TIME, CAST(TO_CHAR(GMT_MODIFIED) AS VARCHAR2(19)) AS MODIFY_TIME, CAST(LS_ID AS NUMBER) AS LS_ID, SVR_IP, CAST(SVR_PORT AS NUMBER) AS SVR_PORT, CAST(SQL_PORT AS NUMBER) AS SQL_PORT, ZONE, (CASE ROLE WHEN 1 THEN 'LEADER' ELSE 'FOLLOWER' END) AS ROLE, (CASE ROLE WHEN 1 THEN MEMBER_LIST ELSE NULL END) AS MEMBER_LIST, CAST((CASE ROLE WHEN 1 THEN PAXOS_REPLICA_NUMBER ELSE NULL END) AS NUMBER) AS PAXOS_REPLICA_NUMBER, (CASE REPLICA_TYPE WHEN 0 THEN 'FULL' WHEN 5 THEN 'LOGONLY' WHEN 16 THEN 'READONLY' WHEN 261 THEN 'ENCRYPTION LOGONLY' ELSE NULL END) AS REPLICA_TYPE, (CASE ROLE WHEN 1 THEN LEARNER_LIST ELSE NULL END) AS LEARNER_LIST, (CASE REBUILD WHEN 0 THEN 'FALSE' ELSE 'TRUE' END) AS REBUILD FROM SYS.ALL_VIRTUAL_LS_META_TABLE WHERE TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(TO_CHAR(GMT_CREATE) AS VARCHAR2(19)) AS CREATE_TIME, CAST(TO_CHAR(GMT_MODIFIED) AS VARCHAR2(19)) AS MODIFY_TIME, CAST(LS_ID AS NUMBER) AS LS_ID, SVR_IP, CAST(SVR_PORT AS NUMBER) AS SVR_PORT, CAST(SQL_PORT AS NUMBER) AS SQL_PORT, ZONE, (CASE ROLE WHEN 1 THEN 'LEADER' ELSE 'FOLLOWER' END) AS ROLE, (CASE ROLE WHEN 1 THEN MEMBER_LIST ELSE NULL END) AS MEMBER_LIST, CAST((CASE ROLE WHEN 1 THEN PAXOS_REPLICA_NUMBER ELSE NULL END) AS NUMBER) AS PAXOS_REPLICA_NUMBER, (CASE REPLICA_TYPE WHEN 0 THEN 'FULL' WHEN 5 THEN 'LOGONLY' WHEN 16 THEN 'READONLY' WHEN 261 THEN 'ENCRYPTION LOGONLY' WHEN 1040 THEN 'COLUMNSTORE' ELSE NULL END) AS REPLICA_TYPE, (CASE ROLE WHEN 1 THEN LEARNER_LIST ELSE NULL END) AS LEARNER_LIST, (CASE REBUILD WHEN 0 THEN 'FALSE' ELSE 'TRUE' END) AS REBUILD FROM SYS.ALL_VIRTUAL_LS_META_TABLE WHERE TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index e393b6aba..4337c3efb 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -19630,9 +19630,6 @@ SELECT RESOURCE_POOL_ID, ZONE_LIST, CASE replica_type WHEN 0 THEN "FULL" - WHEN 5 THEN "LOGONLY" - WHEN 16 THEN "READONLY" - WHEN 261 THEN "ENCRYPTION LOGONLY" ELSE NULL END AS REPLICA_TYPE FROM oceanbase.__all_resource_pool @@ -19889,6 +19886,7 @@ def_table_schema( WHEN 5 THEN "LOGONLY" WHEN 16 THEN "READONLY" WHEN 261 THEN "ENCRYPTION LOGONLY" + WHEN 1040 THEN "COLUMNSTORE" ELSE NULL END) AS REPLICA_TYPE, (CASE ROLE WHEN 1 THEN LEARNER_LIST ELSE "" END) AS LEARNER_LIST, (CASE REBUILD @@ -19915,6 +19913,7 @@ def_table_schema( WHEN 5 THEN "LOGONLY" WHEN 16 THEN "READONLY" WHEN 261 THEN "ENCRYPTION LOGONLY" + WHEN 1040 THEN "COLUMNSTORE" ELSE NULL END) AS REPLICA_TYPE, (CASE ROLE WHEN 1 THEN LEARNER_LIST ELSE "" END) AS LEARNER_LIST, (CASE REBUILD @@ -19954,6 +19953,7 @@ def_table_schema( WHEN 5 THEN "LOGONLY" WHEN 16 THEN "READONLY" WHEN 261 THEN "ENCRYPTION LOGONLY" + WHEN 1040 THEN "COLUMNSTORE" ELSE NULL END) AS REPLICA_TYPE, (CASE ROLE WHEN 1 THEN LEARNER_LIST ELSE "" END) AS LEARNER_LIST, (CASE REBUILD @@ -19979,6 +19979,7 @@ def_table_schema( WHEN 5 THEN "LOGONLY" WHEN 16 THEN "READONLY" WHEN 261 THEN "ENCRYPTION LOGONLY" + WHEN 1040 THEN "COLUMNSTORE" ELSE NULL END) AS REPLICA_TYPE, (CASE ROLE WHEN 1 THEN LEARNER_LIST ELSE "" END) AS LEARNER_LIST, (CASE REBUILD @@ -28107,15 +28108,18 @@ def_table_schema( TABLET_ID FROM ( - SELECT TENANT_ID, - TABLET_ID, - ROW_COUNT, - DATA_CHECKSUM, - B_COLUMN_CHECKSUMS, - COMPACTION_SCN - FROM OCEANBASE.__ALL_VIRTUAL_TABLET_REPLICA_CHECKSUM + SELECT CKM.TENANT_ID, + CKM.TABLET_ID, + CKM.ROW_COUNT, + CKM.DATA_CHECKSUM, + CKM.B_COLUMN_CHECKSUMS, + CKM.COMPACTION_SCN, + M.REPLICA_TYPE + FROM OCEANBASE.__ALL_VIRTUAL_TABLET_REPLICA_CHECKSUM CKM + JOIN OCEANBASE.__ALL_VIRTUAL_LS_META_TABLE M + ON CKM.TENANT_ID = M.TENANT_ID AND CKM.LS_ID = M.LS_ID AND CKM.SVR_IP = M.SVR_IP AND CKM.SVR_PORT = M.SVR_PORT ) J - GROUP BY J.TENANT_ID, J.TABLET_ID, J.COMPACTION_SCN + GROUP BY J.TENANT_ID, J.TABLET_ID, J.COMPACTION_SCN, J.REPLICA_TYPE HAVING MIN(J.DATA_CHECKSUM) != MAX(J.DATA_CHECKSUM) OR MIN(J.ROW_COUNT) != MAX(J.ROW_COUNT) OR MIN(J.B_COLUMN_CHECKSUMS) != MAX(J.B_COLUMN_CHECKSUMS) @@ -50752,6 +50756,7 @@ def_table_schema( WHEN 5 THEN 'LOGONLY' WHEN 16 THEN 'READONLY' WHEN 261 THEN 'ENCRYPTION LOGONLY' + WHEN 1040 THEN 'COLUMNSTORE' ELSE NULL END) AS REPLICA_TYPE, (CASE ROLE WHEN 1 THEN LEARNER_LIST ELSE NULL END) AS LEARNER_LIST, (CASE REBUILD diff --git a/src/share/location_cache/ob_location_struct.cpp b/src/share/location_cache/ob_location_struct.cpp index 9a53ccaf8..8475c41e8 100644 --- a/src/share/location_cache/ob_location_struct.cpp +++ b/src/share/location_cache/ob_location_struct.cpp @@ -366,17 +366,17 @@ bool ObLSLocation::operator!=(const ObLSLocation &other) const return !(*this == other); } -int ObLSLocation::get_replica_count(int64_t &full_replica_cnt, int64_t &readonly_replica_cnt) +int ObLSLocation::get_replica_count(int64_t &full_replica_cnt, int64_t &non_paxos_replica_cnt) { int ret = OB_SUCCESS; full_replica_cnt = 0; - readonly_replica_cnt = 0; + non_paxos_replica_cnt = 0; for (int64_t i = 0; OB_SUCC(ret) && i < replica_locations_.count(); ++i) { const ObLSReplicaLocation &replica = replica_locations_.at(i); if (REPLICA_TYPE_FULL == replica.get_replica_type()) { full_replica_cnt++; - } else if (REPLICA_TYPE_READONLY == replica.get_replica_type()) { - readonly_replica_cnt++; + } else if (ObReplicaTypeCheck::is_non_paxos_replica(replica.get_replica_type())) { + non_paxos_replica_cnt++; } } return ret; diff --git a/src/share/location_cache/ob_location_struct.h b/src/share/location_cache/ob_location_struct.h index de5d448aa..fae0e6538 100644 --- a/src/share/location_cache/ob_location_struct.h +++ b/src/share/location_cache/ob_location_struct.h @@ -190,7 +190,7 @@ public: inline uint64_t get_tenant_id() const { return cache_key_.get_tenant_id(); } inline ObLSID get_ls_id() const { return cache_key_.get_ls_id(); } const ObLSLocationCacheKey &get_cache_key() const { return cache_key_; } - int get_replica_count(int64_t &full_replica_cnt, int64_t &readonly_replica_cnt); + int get_replica_count(int64_t &full_replica_cnt, int64_t &non_paxos_replica_cnt); inline const common::ObIArray &get_replica_locations() const { return replica_locations_; diff --git a/src/share/ls/ob_ls_creator.cpp b/src/share/ls/ob_ls_creator.cpp index 350821d31..b356af6fd 100644 --- a/src/share/ls/ob_ls_creator.cpp +++ b/src/share/ls/ob_ls_creator.cpp @@ -51,7 +51,7 @@ int ObLSReplicaAddr::init(const common::ObAddr &addr, { int ret = OB_SUCCESS; if (OB_UNLIKELY(!addr.is_valid() - || common::REPLICA_TYPE_MAX == replica_type)) { + || common::REPLICA_TYPE_INVALID == replica_type)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(addr), K(replica_type)); } else { @@ -662,6 +662,12 @@ int ObLSCreator::check_create_ls_result_( if (OB_FAIL(learner_list.add_learner(ObMember(addr, timestamp)))) { LOG_WARN("failed to add member", KR(ret), K(addr)); } + } else if (result->get_replica_type() == REPLICA_TYPE_COLUMNSTORE) { + ObMember member(addr, timestamp); + member.set_columnstore(); + if (OB_FAIL(learner_list.add_learner(member))) { + LOG_WARN("failed to add member", KR(ret), K(addr), K(member)); + } } LOG_TRACE("create ls result", KR(ret), K(i), K(addr), KPC(result)); } @@ -991,7 +997,9 @@ int ObLSCreator::construct_ls_addrs_according_to_locality_( for (int64_t i = 0; OB_SUCC(ret) && i < zone_locality_array.count(); ++i) { const share::ObZoneReplicaAttrSet &zone_locality = zone_locality_array.at(i); ObLSReplicaAddr replica_addr; - if (OB_FAIL(alloc_zone_ls_addr(is_sys_ls, zone_locality, unit_info_array, replica_addr))) { + if (is_sys_ls && zone_locality.get_columnstore_replica_num() > 0) { + // ignore, C-replica not applicable for sys-ls + } else if (OB_FAIL(alloc_zone_ls_addr(is_sys_ls, zone_locality, unit_info_array, replica_addr))) { LOG_WARN("fail to alloc zone ls addr", KR(ret), K(zone_locality), K(unit_info_array)); } else if (OB_FAIL(ls_addr.push_back(replica_addr))) { LOG_WARN("fail to push back", KR(ret), K(replica_addr)); @@ -1106,11 +1114,9 @@ int ObLSCreator::alloc_duplicate_ls_addr_( } else if (OB_FAIL(construct_ls_addrs_according_to_locality_( zone_locality_array, unit_info_array, - true/*is_sys_ls*/, + false/*is_sys_ls*/, true/*is_duplicate_ls*/, ls_addr))) { - // although duplicate log stream is a user log steam, we use the same logic to alloc addrs as sys log stream - // so set is_sys_ls to true when execute construct_ls_addrs_according_to_locality_ LOG_WARN("fail to construct ls addrs for tenant user ls", KR(ret), K(zone_locality_array), K(unit_info_array), K(ls_addr)); } @@ -1129,6 +1135,14 @@ int ObLSCreator::compensate_zone_readonly_replica_( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(unit_info_array)); } else { + ObReplicaType replica_type_to_add = ObReplicaType::REPLICA_TYPE_INVALID; + if (zlocality.get_columnstore_replica_num() > 0) { + // For C zone locality, compensate C-replica. + replica_type_to_add = ObReplicaType::REPLICA_TYPE_COLUMNSTORE; + } else { + // For other zone locality (F/R), compensate R-replica + replica_type_to_add = ObReplicaType::REPLICA_TYPE_READONLY; + } for (int64_t i = 0; OB_SUCC(ret) && i < unit_info_array.count(); ++i) { const share::ObUnit &unit = unit_info_array.at(i); if (locality_zone != unit.zone_) { @@ -1142,7 +1156,7 @@ int ObLSCreator::compensate_zone_readonly_replica_( ObLSReplicaAddr ls_replica_addr; if (OB_FAIL(ls_replica_addr.init( unit.server_, - ObReplicaType::REPLICA_TYPE_READONLY))) { + replica_type_to_add))) { LOG_WARN("fail to init ls replica addr", KR(ret), K(unit), K(locality_zone)); } else if (OB_FAIL(ls_addr.push_back(ls_replica_addr))) { LOG_WARN("fail to push back", KR(ret), K(ls_replica_addr)); @@ -1174,25 +1188,31 @@ int ObLSCreator::alloc_zone_ls_addr( if (OB_FAIL(ls_replica_addr.init( unit.server_, ObReplicaType::REPLICA_TYPE_FULL))) { - LOG_WARN("fail to init ls replica addr", KR(ret)); + LOG_WARN("fail to init ls replica addr", KR(ret), K(unit.server_)); } } else if (zlocality.replica_attr_set_.get_logonly_replica_attr_array().count() > 0) { if (OB_FAIL(ls_replica_addr.init( unit.server_, ObReplicaType::REPLICA_TYPE_LOGONLY))) { - LOG_WARN("fail to init ls replica addr", KR(ret)); + LOG_WARN("fail to init ls replica addr", KR(ret), K(unit.server_)); } } else if (zlocality.replica_attr_set_.get_encryption_logonly_replica_attr_array().count() > 0) { if (OB_FAIL(ls_replica_addr.init( unit.server_, ObReplicaType::REPLICA_TYPE_ENCRYPTION_LOGONLY))) { - LOG_WARN("fail to init ls replica addr", KR(ret)); + LOG_WARN("fail to init ls replica addr", KR(ret), K(unit.server_)); } } else if (zlocality.replica_attr_set_.get_readonly_replica_attr_array().count() > 0) { if (OB_FAIL(ls_replica_addr.init( unit.server_, ObReplicaType::REPLICA_TYPE_READONLY))) { - LOG_WARN("fail to init ls replica addr", KR(ret)); + LOG_WARN("fail to init ls replica addr", KR(ret), K(unit.server_)); + } + } else if (zlocality.replica_attr_set_.get_columnstore_replica_attr_array().count() > 0) { + if (OB_FAIL(ls_replica_addr.init( + unit.server_, + ObReplicaType::REPLICA_TYPE_COLUMNSTORE))) { + LOG_WARN("fail to init ls replica addr", KR(ret), K(unit.server_)); } } else { // zone locality shall has a paxos replica in 4.0 by // now(2021.10.25) diff --git a/src/share/ls/ob_ls_creator.h b/src/share/ls/ob_ls_creator.h index 2497d2234..3e227454f 100644 --- a/src/share/ls/ob_ls_creator.h +++ b/src/share/ls/ob_ls_creator.h @@ -52,7 +52,7 @@ struct ObLSReplicaAddr ObLSReplicaAddr() : addr_(), - replica_type_(common::REPLICA_TYPE_MAX) {} + replica_type_(common::REPLICA_TYPE_INVALID) {} void reset() { *this = ObLSReplicaAddr(); } int init(const common::ObAddr &addr, const common::ObReplicaType replica_type); diff --git a/src/share/ls/ob_ls_info.cpp b/src/share/ls/ob_ls_info.cpp index 625e33ffc..f6c1973e5 100644 --- a/src/share/ls/ob_ls_info.cpp +++ b/src/share/ls/ob_ls_info.cpp @@ -993,13 +993,18 @@ int ObLSInfo::update_replica_status() bool in_leader_learner_list = false; ObMember learner; // rectify replica_type_ + const ObReplicaType replica_type_before_rectify = r->get_replica_type(); if (OB_NOT_NULL(learner_list) && learner_list->contains(r->get_server())) { - r->set_replica_type(REPLICA_TYPE_READONLY); in_leader_learner_list = true; if (OB_FAIL(learner_list->get_learner_by_addr(r->get_server(), learner))) { LOG_WARN("fail to get learner by addr", KR(ret)); - } else if (in_leader_learner_list) { + } else { in_member_time_us = learner.get_timestamp(); + if (learner.is_columnstore()) { + r->set_replica_type(REPLICA_TYPE_COLUMNSTORE); + } else { + r->set_replica_type(REPLICA_TYPE_READONLY); + } } } else { r->set_replica_type(REPLICA_TYPE_FULL); @@ -1021,8 +1026,17 @@ int ObLSInfo::update_replica_status() // 2 non_paxos replicas (READONLY),NORMAL when in leader's learner_list otherwise offline // 3 if non_paxos replicas are deleted by partition service, status in meta table is set to REPLICA_STATUS_OFFLINE, // then set replica_status to REPLICA_STATUS_OFFLINE + // 4 COLUMNSTORE replica, if not in learner list or columnstore-flag is false, + // then set replica_status to REPLICA_STATUS_OFFLINE if (REPLICA_STATUS_OFFLINE == r->get_replica_status()) { // do nothing + } else if (REPLICA_TYPE_COLUMNSTORE == replica_type_before_rectify + && REPLICA_TYPE_COLUMNSTORE != r->get_replica_type()) { + // we set replica_type according to leader's member/learner_list, but need to log a warning. + LOG_WARN("replica_type before rectify is COLUMNSTORE, " + "but not match with leader's member_list and learner_list", + K(replica_type_before_rectify), K(member_list), K(learner_list), KPC(r)); + r->set_replica_status(REPLICA_STATUS_OFFLINE); } else if (in_leader_member_list || in_leader_learner_list) { r->set_replica_status(REPLICA_STATUS_NORMAL); } else { diff --git a/src/share/ls/ob_ls_info.h b/src/share/ls/ob_ls_info.h index 36bbb0e0d..b9a31fa37 100644 --- a/src/share/ls/ob_ls_info.h +++ b/src/share/ls/ob_ls_info.h @@ -135,6 +135,7 @@ public: inline bool is_paxos_replica() const { return common::REPLICA_TYPE_ENCRYPTION_LOGONLY == replica_type_ || common::REPLICA_TYPE_FULL == replica_type_ || common::REPLICA_TYPE_LOGONLY == replica_type_; } + inline bool is_column_replica() const { return common::REPLICA_TYPE_COLUMNSTORE == replica_type_; } int64_t to_string(char *buf, const int64_t buf_len) const; // operator-related functions int assign(const ObLSReplica &other); diff --git a/src/share/ob_debug_sync_point.h b/src/share/ob_debug_sync_point.h index fb4aa4bc7..0c3a9ca23 100755 --- a/src/share/ob_debug_sync_point.h +++ b/src/share/ob_debug_sync_point.h @@ -633,6 +633,8 @@ class ObString; ACT(BEFROE_UPDATE_DATA_VERSION,)\ ACT(BEFORE_DATA_DICT_DUMP_FINISH,)\ ACT(AFTER_PHYSICAL_RESTORE_CREATE_TENANT,)\ + ACT(BEFROE_UPDATE_MIG_TABLET_CONVERT_CO_PROGRESSING,)\ + ACT(AFTER_SET_CO_CONVERT_RETRY_EXHUASTED,)\ ACT(MAX_DEBUG_SYNC_POINT,) DECLARE_ENUM(ObDebugSyncPoint, debug_sync_point, OB_DEBUG_SYNC_POINT_DEF); diff --git a/src/share/ob_locality_parser.cpp b/src/share/ob_locality_parser.cpp deleted file mode 100644 index 9f40422ed..000000000 --- a/src/share/ob_locality_parser.cpp +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#define USING_LOG_PREFIX SHARE -#include "ob_locality_parser.h" -#include "lib/alloc/alloc_assist.h" - -using namespace oceanbase::common; -using namespace oceanbase::share; - -// full replica -const char *ObLocalityParser::FULL_REPLICA_STR = "FULL"; -const char *ObLocalityParser::F_REPLICA_STR = "F"; -// logonly replica -const char *ObLocalityParser::LOGONLY_REPLICA_STR = "LOGONLY"; -const char *ObLocalityParser::L_REPLICA_STR = "L"; -// backup replica -const char *ObLocalityParser::BACKUP_REPLICA_STR = "BACKUP"; -const char *ObLocalityParser::B_REPLICA_STR = "B"; -// readonly replica -const char *ObLocalityParser::READONLY_REPLICA_STR = "READONLY"; -const char *ObLocalityParser::R_REPLICA_STR = "R"; -// memonly replica -const char *ObLocalityParser::MEMONLY_REPLICA_STR = "MEMONLY"; -const char *ObLocalityParser::M_REPLICA_STR = "M"; -// encryption logonly replica -const char *ObLocalityParser::ENCRYPTION_LOGONLY_REPLICA_STR = "ENCRYPTION_LOGONLY"; -const char *ObLocalityParser::E_REPLICA_STR = "E"; - -int ObLocalityParser::parse_type(const char *str, int64_t len, ObReplicaType &replica_type) -{ - UNUSED(len); - // TODO: only support F-replica in 4.0 and R-replica in 4.2 for now, will support others in the future - int ret = OB_SUCCESS; - if (OB_ISNULL(str)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid replica type string. null!", K(ret)); - LOG_USER_ERROR(OB_INVALID_ARGUMENT, "replica_type, replica_type should not be null"); - } else if (0 == STRCASECMP(FULL_REPLICA_STR, str)) { - replica_type = REPLICA_TYPE_FULL; - } else if (0 == STRCASECMP(F_REPLICA_STR, str)) { - replica_type = REPLICA_TYPE_FULL; - } else if (0 == STRCASECMP(LOGONLY_REPLICA_STR, str)) { - replica_type = REPLICA_TYPE_LOGONLY; - ret = OB_NOT_SUPPORTED; - LOG_USER_ERROR(OB_NOT_SUPPORTED, "logonly-replica"); - } else if ( 0 == STRCASECMP(L_REPLICA_STR, str)) { - replica_type = REPLICA_TYPE_LOGONLY; - ret = OB_NOT_SUPPORTED; - LOG_USER_ERROR(OB_NOT_SUPPORTED, "logonly-replica"); - } else if (0 == STRCASECMP(ENCRYPTION_LOGONLY_REPLICA_STR, str)) { - replica_type = REPLICA_TYPE_ENCRYPTION_LOGONLY; - ret = OB_NOT_SUPPORTED; - LOG_USER_ERROR(OB_NOT_SUPPORTED, "encryption-logonly-replica"); - } else if ( 0 == STRCASECMP(E_REPLICA_STR, str)) { - replica_type = REPLICA_TYPE_ENCRYPTION_LOGONLY; - ret = OB_NOT_SUPPORTED; - LOG_USER_ERROR(OB_NOT_SUPPORTED, "encryption-logonly-replica"); - } else if ( 0 == STRCASECMP(BACKUP_REPLICA_STR, str)) { - replica_type = REPLICA_TYPE_BACKUP; - ret = OB_NOT_SUPPORTED; - LOG_USER_ERROR(OB_NOT_SUPPORTED, "backup-replica"); - } else if ( 0 == STRCASECMP(B_REPLICA_STR, str)) { - replica_type = REPLICA_TYPE_BACKUP; - ret = OB_NOT_SUPPORTED; - LOG_USER_ERROR(OB_NOT_SUPPORTED, "backup-replica"); - } else if ( 0 == STRCASECMP(READONLY_REPLICA_STR, str)) { - replica_type = REPLICA_TYPE_READONLY; - } else if ( 0 == STRCASECMP(R_REPLICA_STR, str)) { - replica_type = REPLICA_TYPE_READONLY; - } else if ( 0 == STRCASECMP(MEMONLY_REPLICA_STR, str)) { - replica_type = REPLICA_TYPE_MEMONLY; - ret = OB_NOT_SUPPORTED; - LOG_USER_ERROR(OB_NOT_SUPPORTED, "memonly-replica"); - } else if ( 0 == STRCASECMP(M_REPLICA_STR, str)) { - replica_type = REPLICA_TYPE_MEMONLY; - ret = OB_NOT_SUPPORTED; - LOG_USER_ERROR(OB_NOT_SUPPORTED, "memonly-replica"); - } else { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid replica type string", K(str), K(ret)); - LOG_USER_ERROR(OB_INVALID_ARGUMENT, "replica_type, unrecognized replica_type"); - } - return ret; -} - diff --git a/src/share/ob_locality_parser.h b/src/share/ob_locality_parser.h deleted file mode 100644 index 689befa43..000000000 --- a/src/share/ob_locality_parser.h +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#ifndef OCEANBASE_SHARE_OB_LOCALITY_PARSER_H_ -#define OCEANBASE_SHARE_OB_LOCALITY_PARSER_H_ - -#include "share/ob_define.h" - -namespace oceanbase -{ -namespace share -{ -class ObLocalityParser -{ -public: - static int parse_type(const char *str, int64_t len, common::ObReplicaType &type); -private: - // full replica - static const char *FULL_REPLICA_STR; - static const char *F_REPLICA_STR; - // logonly replica - static const char *LOGONLY_REPLICA_STR; - static const char *L_REPLICA_STR; - // backup replica - static const char *BACKUP_REPLICA_STR; - static const char *B_REPLICA_STR; - // readonly replica - static const char *READONLY_REPLICA_STR; - static const char *R_REPLICA_STR; - // memonly replica - static const char *MEMONLY_REPLICA_STR; - static const char *M_REPLICA_STR; - // encryption logonly replica - static const char *ENCRYPTION_LOGONLY_REPLICA_STR; - static const char *E_REPLICA_STR; -}; - -} // end namespace share -} // end namespace oceanbase -#endif diff --git a/src/share/ob_replica_info.cpp b/src/share/ob_replica_info.cpp index 0cfc1e2e0..8888e1b2c 100644 --- a/src/share/ob_replica_info.cpp +++ b/src/share/ob_replica_info.cpp @@ -59,6 +59,15 @@ int64_t BaseReplicaAttrSet::get_encryption_logonly_replica_num() const return num; } +int64_t BaseReplicaAttrSet::get_columnstore_replica_num() const +{ + int64_t num = 0; + for (int64_t i = 0; i < get_columnstore_replica_attr_array().count(); ++i) { + num += get_columnstore_replica_attr_array().at(i).num_; + } + return num; +} + int64_t BaseReplicaAttrSet::get_paxos_replica_num() const { return get_full_replica_num() @@ -81,6 +90,9 @@ int64_t BaseReplicaAttrSet::get_specific_replica_num() const if (ObLocalityDistribution::ALL_SERVER_CNT != get_encryption_logonly_replica_num()) { specific_replica_num += get_encryption_logonly_replica_num(); } + if (ObLocalityDistribution::ALL_SERVER_CNT != get_columnstore_replica_num()) { + specific_replica_num += get_columnstore_replica_num(); + } return specific_replica_num; } @@ -117,7 +129,8 @@ bool ObReplicaAttrSet::operator==(const ObReplicaAttrSet &that) const if (full_replica_attr_array_.count() != that.full_replica_attr_array_.count() || logonly_replica_attr_array_.count() != that.logonly_replica_attr_array_.count() || readonly_replica_attr_array_.count() != that.readonly_replica_attr_array_.count() - || encryption_logonly_replica_attr_array_.count() != that.encryption_logonly_replica_attr_array_.count()) { + || encryption_logonly_replica_attr_array_.count() != that.encryption_logonly_replica_attr_array_.count() + || columnstore_replica_attr_array_.count() != that.columnstore_replica_attr_array_.count()) { equal = false; } else { for (int64_t i = 0; equal && i < full_replica_attr_array_.count(); ++i) { @@ -140,6 +153,11 @@ bool ObReplicaAttrSet::operator==(const ObReplicaAttrSet &that) const equal = false; } } + for (int64_t i = 0; equal && i < columnstore_replica_attr_array_.count(); ++i) { + if (columnstore_replica_attr_array_.at(i) != that.columnstore_replica_attr_array_.at(i)) { + equal = false; + } + } } return equal; } @@ -162,6 +180,9 @@ int ObReplicaAttrSet::assign(const BaseReplicaAttrSet &that) } else if (OB_FAIL(encryption_logonly_replica_attr_array_.assign( that.get_encryption_logonly_replica_attr_array()))) { LOG_WARN("fail to assign encryption logonly replica attr array", KR(ret)); + } else if (OB_FAIL(columnstore_replica_attr_array_.assign( + that.get_columnstore_replica_attr_array()))) { + LOG_WARN("fail to assign columnstore replica attr array", KR(ret)); } return ret; } @@ -170,7 +191,8 @@ int ObReplicaAttrSet::set_replica_attr_array( const common::ObIArray &full_replica_attr_array, const common::ObIArray &logonly_replica_attr_array, const common::ObIArray &readonly_replica_attr_array, - const common::ObIArray &encryption_logonly_replica_attr_array) + const common::ObIArray &encryption_logonly_replica_attr_array, + const common::ObIArray &columnstore_replica_attr_array) { int ret = OB_SUCCESS; if (OB_FAIL(full_replica_attr_array_.assign(full_replica_attr_array))) { @@ -181,6 +203,8 @@ int ObReplicaAttrSet::set_replica_attr_array( LOG_WARN("fail to assign readonly replica attr array", KR(ret)); } else if (OB_FAIL(encryption_logonly_replica_attr_array_.assign(encryption_logonly_replica_attr_array))) { LOG_WARN("fail to assign encryption logonly replica attr array", KR(ret)); + } else if (OB_FAIL(columnstore_replica_attr_array_.assign(columnstore_replica_attr_array))) { + LOG_WARN("fail to assign columnstore replica attr array", KR(ret)); } return ret; } @@ -201,12 +225,15 @@ int ObReplicaAttrSet::set_paxos_replica_attr_array( return ret; } -int ObReplicaAttrSet::set_readonly_replica_attr_array( - const common::ObIArray &readonly_replica_attr_array) +int ObReplicaAttrSet::set_non_paxos_replica_attr_array( + const common::ObIArray &readonly_replica_attr_array, + const common::ObIArray &columnstore_replica_attr_array) { int ret = OB_SUCCESS; if (OB_FAIL(readonly_replica_attr_array_.assign(readonly_replica_attr_array))) { LOG_WARN("fail to assign full replica attr array", K(ret)); + } else if (OB_FAIL(columnstore_replica_attr_array_.assign(columnstore_replica_attr_array))) { + LOG_WARN("fail to assign columnstore replica attr array", KR(ret)); } return ret; } @@ -263,6 +290,10 @@ bool ObReplicaAttrSet::is_specific_replica_attr() const const ReplicaAttr &replica_attr = encryption_logonly_replica_attr_array_.at(i); bool_ret = 100 != replica_attr.memstore_percent_; } + for (int64_t i = 0; !bool_ret && i < columnstore_replica_attr_array_.count(); i++) { + const ReplicaAttr &replica_attr = columnstore_replica_attr_array_.at(i); + bool_ret = 100 != replica_attr.memstore_percent_; + } return bool_ret; } @@ -373,6 +404,30 @@ int ObReplicaAttrSet::add_encryption_logonly_replica_num(const ReplicaAttr &repl return ret; } +int ObReplicaAttrSet::add_columnstore_replica_num(const ReplicaAttr &replica_attr) +{ + int ret = OB_SUCCESS; + if (replica_attr.num_ > 0) { + if (columnstore_replica_attr_array_.count() <= 0) { + if (OB_FAIL(columnstore_replica_attr_array_.push_back( + ReplicaAttr(0, replica_attr.memstore_percent_)))) { + LOG_WARN("fail to push back", K(ret)); + } + } + if (OB_FAIL(ret)) { + // bypass + } else if (columnstore_replica_attr_array_.count() <= 0) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("index unexpected", K(ret), + "columnstore_replica_attr_array_count", + columnstore_replica_attr_array_.count()); + } else { + columnstore_replica_attr_array_.at(0).num_ += replica_attr.num_; + } + } + return ret; +} + int ObReplicaAttrSet::sub_full_replica_num(const ReplicaAttr &replica_attr) { int ret = OB_SUCCESS; @@ -485,56 +540,27 @@ int ObReplicaAttrSet::sub_encryption_logonly_replica_num(const ReplicaAttr &repl return ret; } -bool ObReplicaAttrSet::has_this_replica( - const common::ObReplicaType replica_type, - const int64_t memstore_percent) -{ - bool found = false; - if (common::REPLICA_TYPE_FULL == replica_type) { - for (int64_t i = 0; !found && i < full_replica_attr_array_.count(); ++i) { - ReplicaAttr &this_replica_attr = full_replica_attr_array_.at(i); - if (this_replica_attr.num_ > 0 && memstore_percent == this_replica_attr.memstore_percent_) { - found = true; - } - } - } else if (common::REPLICA_TYPE_LOGONLY == replica_type) { - for (int64_t i = 0; !found && i < logonly_replica_attr_array_.count(); ++i) { - ReplicaAttr &this_replica_attr = logonly_replica_attr_array_.at(i); - if (this_replica_attr.num_ > 0 && memstore_percent == this_replica_attr.memstore_percent_) { - found = true; - } - } - } else if (common::REPLICA_TYPE_READONLY == replica_type) { - for (int64_t i = 0; !found && i < readonly_replica_attr_array_.count(); ++i) { - ReplicaAttr &this_replica_attr = readonly_replica_attr_array_.at(i); - if (this_replica_attr.num_ > 0 && memstore_percent == this_replica_attr.memstore_percent_) { - found = true; - } - } - } else if (common::REPLICA_TYPE_ENCRYPTION_LOGONLY == replica_type) { - for (int64_t i = 0; !found && i < encryption_logonly_replica_attr_array_.count(); ++i) { - ReplicaAttr &this_replica_attr = encryption_logonly_replica_attr_array_.at(i); - if (this_replica_attr.num_ > 0 && memstore_percent == this_replica_attr.memstore_percent_) { - found = true; - } - } - } else { - found = false; - } - return found; -} - - -int ObReplicaAttrSet::get_readonly_memstore_percent(int64_t &memstore_percent) const +int ObReplicaAttrSet::sub_columnstore_replica_num(const ReplicaAttr &replica_attr) { int ret = OB_SUCCESS; - if (readonly_replica_attr_array_.count() <= 0 || readonly_replica_attr_array_.count() > 1) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("readonly replica attr array count unexpected", K(ret), - "array_count", readonly_replica_attr_array_.count()); - } else { - const ReplicaAttr &replica_attr = readonly_replica_attr_array_.at(0); - memstore_percent = replica_attr.memstore_percent_; + if (replica_attr.num_ > 0) { + if (columnstore_replica_attr_array_.count() <= 0) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("columnstore replica attr array empty", K(ret)); + } else if (columnstore_replica_attr_array_.at(0).num_ < replica_attr.num_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("columnstore replica num not enough", K(ret), + "columnstore_replica_num_in_array", + columnstore_replica_attr_array_.at(0).num_, + K(replica_attr)); + } else { + columnstore_replica_attr_array_.at(0).num_ -= replica_attr.num_; + if (columnstore_replica_attr_array_.at(0).num_ <= 0) { + if (OB_FAIL(columnstore_replica_attr_array_.remove(0))) { + LOG_WARN("fail to remove", K(ret)); + } + } + } } return ret; } @@ -552,7 +578,7 @@ int ObZoneReplicaAttrSet::append(const ObZoneReplicaAttrSet &that) OB_SUCC(ret) && i < that.replica_attr_set_.get_full_replica_attr_array().count(); ++i) { const ReplicaAttr &this_replica_attr = that.replica_attr_set_.get_full_replica_attr_array().at(i); - if (OB_FAIL(add_full_replica_num(this_replica_attr))) { + if (OB_FAIL(replica_attr_set_.add_full_replica_num(this_replica_attr))) { LOG_WARN("fail to add full replica num", K(ret)); } } @@ -562,7 +588,7 @@ int ObZoneReplicaAttrSet::append(const ObZoneReplicaAttrSet &that) OB_SUCC(ret) && i < that.replica_attr_set_.get_logonly_replica_attr_array().count(); ++i) { const ReplicaAttr &this_replica_attr = that.replica_attr_set_.get_logonly_replica_attr_array().at(i); - if (OB_FAIL(add_logonly_replica_num(this_replica_attr))) { + if (OB_FAIL(replica_attr_set_.add_logonly_replica_num(this_replica_attr))) { LOG_WARN("fail to add logonly replica num", K(ret)); } } @@ -572,7 +598,7 @@ int ObZoneReplicaAttrSet::append(const ObZoneReplicaAttrSet &that) OB_SUCC(ret) && i < that.replica_attr_set_.get_readonly_replica_attr_array().count(); ++i) { const ReplicaAttr &this_replica_attr = that.replica_attr_set_.get_readonly_replica_attr_array().at(i); - if (OB_FAIL(add_readonly_replica_num(this_replica_attr))) { + if (OB_FAIL(replica_attr_set_.add_readonly_replica_num(this_replica_attr))) { LOG_WARN("fail to add readonly replica num", K(ret)); } } @@ -583,12 +609,22 @@ int ObZoneReplicaAttrSet::append(const ObZoneReplicaAttrSet &that) ++i) { const ReplicaAttr &this_replica_attr = that.replica_attr_set_.get_encryption_logonly_replica_attr_array().at(i); - if (OB_FAIL(add_encryption_logonly_replica_num(this_replica_attr))) { + if (OB_FAIL(replica_attr_set_.add_encryption_logonly_replica_num(this_replica_attr))) { LOG_WARN("fail to add logonly replica num", K(ret)); } } lib::ob_sort(replica_attr_set_.get_encryption_logonly_replica_attr_array_for_sort().begin(), replica_attr_set_.get_encryption_logonly_replica_attr_array_for_sort().end()); + for (int64_t i = 0; + OB_SUCC(ret) && i < that.replica_attr_set_.get_columnstore_replica_attr_array().count(); + ++i) { + const ReplicaAttr &this_replica_attr = that.replica_attr_set_.get_columnstore_replica_attr_array().at(i); + if (OB_FAIL(replica_attr_set_.add_columnstore_replica_num(this_replica_attr))) { + LOG_WARN("fail to add columnstore replica num", K(ret)); + } + } + lib::ob_sort(replica_attr_set_.get_columnstore_replica_attr_array_for_sort().begin(), + replica_attr_set_.get_columnstore_replica_attr_array_for_sort().end()); return ret; } @@ -755,36 +791,6 @@ bool ObZoneReplicaAttrSet::check_paxos_num_valid() const } -void ObReplicaNumSet::set_replica_num( - int64_t full_replica_num, - int64_t logonly_replica_num, - int64_t readonly_replica_num, - int64_t encryption_logonly_replica_num) -{ - full_replica_num_ = full_replica_num; - logonly_replica_num_ = logonly_replica_num; - readonly_replica_num_ = readonly_replica_num; - encryption_logonly_replica_num_ = encryption_logonly_replica_num; -} - -int64_t ObReplicaNumSet::get_specific_replica_num() const -{ - int64_t specific_replica_num = 0; - if (ObLocalityDistribution::ALL_SERVER_CNT != full_replica_num_) { - specific_replica_num += full_replica_num_; - } - if (ObLocalityDistribution::ALL_SERVER_CNT != logonly_replica_num_) { - specific_replica_num += logonly_replica_num_; - } - if (ObLocalityDistribution::ALL_SERVER_CNT != readonly_replica_num_) { - specific_replica_num += readonly_replica_num_; - } - if (ObLocalityDistribution::ALL_SERVER_CNT != encryption_logonly_replica_num_) { - specific_replica_num += encryption_logonly_replica_num_; - } - return specific_replica_num; -} - int64_t SchemaReplicaAttrSet::get_convert_size() const { int64_t convert_size = sizeof(SchemaReplicaAttrSet); @@ -796,6 +802,8 @@ int64_t SchemaReplicaAttrSet::get_convert_size() const convert_size += readonly_set.count() * static_cast(sizeof(share::ReplicaAttr)); const ObIArray &encryption_logonly_set = get_encryption_logonly_replica_attr_array(); convert_size += encryption_logonly_set.count() * static_cast(sizeof(share::ReplicaAttr)); + const ObIArray &columnstore_set = get_columnstore_replica_attr_array(); + convert_size += columnstore_set.count() * static_cast(sizeof(share::ReplicaAttr)); return convert_size; } diff --git a/src/share/ob_replica_info.h b/src/share/ob_replica_info.h index f2359cbad..36656a6b0 100644 --- a/src/share/ob_replica_info.h +++ b/src/share/ob_replica_info.h @@ -72,22 +72,26 @@ public: virtual const common::ObIArray &get_logonly_replica_attr_array() const = 0; virtual const common::ObIArray &get_readonly_replica_attr_array() const = 0; virtual const common::ObIArray &get_encryption_logonly_replica_attr_array() const = 0; + virtual const common::ObIArray &get_columnstore_replica_attr_array() const = 0; virtual common::ObIArray &get_full_replica_attr_array() = 0; virtual common::ObIArray &get_logonly_replica_attr_array() = 0; virtual common::ObIArray &get_readonly_replica_attr_array() = 0; virtual common::ObIArray &get_encryption_logonly_replica_attr_array() = 0; + virtual common::ObIArray &get_columnstore_replica_attr_array() = 0; int64_t get_full_replica_num() const; int64_t get_logonly_replica_num() const; int64_t get_readonly_replica_num() const; int64_t get_encryption_logonly_replica_num() const; + int64_t get_columnstore_replica_num() const; int64_t get_paxos_replica_num() const; int64_t get_specific_replica_num() const; TO_STRING_KV("full_replica_attr_array", get_full_replica_attr_array(), "logonly_replica_attr_array", get_logonly_replica_attr_array(), "readonly_replica_attr_array", get_readonly_replica_attr_array(), - "encryption_logonly_replica_attr_array", get_encryption_logonly_replica_attr_array()); + "encryption_logonly_replica_attr_array", get_encryption_logonly_replica_attr_array(), + "columnstore_replica_attr_array", get_columnstore_replica_attr_array()); }; typedef common::ObArrayHelper SchemaReplicaAttrArray; @@ -100,7 +104,8 @@ public: full_replica_attr_array_(), logonly_replica_attr_array_(), readonly_replica_attr_array_(), - encryption_logonly_replica_attr_array_() {} + encryption_logonly_replica_attr_array_(), + columnstore_replica_attr_array_() {} virtual ~SchemaReplicaAttrSet() {} int64_t get_convert_size() const; public: @@ -116,6 +121,9 @@ public: virtual const common::ObIArray &get_encryption_logonly_replica_attr_array() const override { return encryption_logonly_replica_attr_array_; } + virtual const common::ObIArray &get_columnstore_replica_attr_array() const override { + return columnstore_replica_attr_array_; + } virtual common::ObIArray &get_full_replica_attr_array() override { return full_replica_attr_array_; } @@ -128,18 +136,23 @@ public: virtual common::ObIArray &get_encryption_logonly_replica_attr_array() override { return encryption_logonly_replica_attr_array_; } + virtual common::ObIArray &get_columnstore_replica_attr_array() override { + return columnstore_replica_attr_array_; + } public: void reset() { full_replica_attr_array_.reset(); logonly_replica_attr_array_.reset(); readonly_replica_attr_array_.reset(); encryption_logonly_replica_attr_array_.reset(); + columnstore_replica_attr_array_.reset(); } private: SchemaReplicaAttrArray full_replica_attr_array_; SchemaReplicaAttrArray logonly_replica_attr_array_; SchemaReplicaAttrArray readonly_replica_attr_array_; SchemaReplicaAttrArray encryption_logonly_replica_attr_array_; + SchemaReplicaAttrArray columnstore_replica_attr_array_; }; class ObReplicaAttrSet : public BaseReplicaAttrSet @@ -160,21 +173,20 @@ public: logonly_replica_attr_array_.reset(); readonly_replica_attr_array_.reset(); encryption_logonly_replica_attr_array_.reset(); + columnstore_replica_attr_array_.reset(); } int add_full_replica_num(const ReplicaAttr &replica_attr); int add_logonly_replica_num(const ReplicaAttr &replica_attr); int add_readonly_replica_num(const ReplicaAttr &replica_attr); int add_encryption_logonly_replica_num(const ReplicaAttr &replica_attr); + int add_columnstore_replica_num(const ReplicaAttr &replica_attr); int sub_full_replica_num(const ReplicaAttr &replica_attr); int sub_logonly_replica_num(const ReplicaAttr &replica_attr); int sub_readonly_replica_num(const ReplicaAttr &replica_attr); int sub_encryption_logonly_replica_num(const ReplicaAttr &replica_attr); - - bool has_this_replica( - const common::ObReplicaType replica_type, - const int64_t memstore_percent); + int sub_columnstore_replica_num(const ReplicaAttr &replica_attr); virtual const common::ObIArray &get_full_replica_attr_array() const override { return full_replica_attr_array_; @@ -188,6 +200,9 @@ public: virtual const common::ObIArray &get_encryption_logonly_replica_attr_array() const override { return encryption_logonly_replica_attr_array_; } + virtual const common::ObIArray &get_columnstore_replica_attr_array() const override { + return columnstore_replica_attr_array_; + } virtual common::ObIArray &get_full_replica_attr_array() override { return full_replica_attr_array_; } @@ -200,6 +215,9 @@ public: virtual common::ObIArray &get_encryption_logonly_replica_attr_array() override { return encryption_logonly_replica_attr_array_; } + virtual common::ObIArray &get_columnstore_replica_attr_array() override { + return columnstore_replica_attr_array_; + } ReplicaAttrArray &get_full_replica_attr_array_for_sort() { return full_replica_attr_array_; @@ -213,22 +231,25 @@ public: ReplicaAttrArray &get_encryption_logonly_replica_attr_array_for_sort() { return encryption_logonly_replica_attr_array_; } + ReplicaAttrArray &get_columnstore_replica_attr_array_for_sort() { + return columnstore_replica_attr_array_; + } int set_replica_attr_array( const common::ObIArray &full_replica_attr_array, const common::ObIArray &logonly_replica_attr_array, const common::ObIArray &readonly_replica_attr_array, - const common::ObIArray &encryption_logonly_replica_attr_array); + const common::ObIArray &encryption_logonly_replica_attr_array, + const common::ObIArray &columnstore_replica_attr_array); int set_paxos_replica_attr_array( const common::ObIArray &full_replica_attr_array, const common::ObIArray &logonly_replica_attr_array, const common::ObIArray &encryption_logonly_replica_attr_array); - int set_readonly_replica_attr_array( - const common::ObIArray &readonly_replica_attr_array); - - int get_readonly_memstore_percent(int64_t &memstore_percent) const; + int set_non_paxos_replica_attr_array( + const common::ObIArray &readonly_replica_attr_array, + const common::ObIArray &columnstore_replica_attr_array); bool has_paxos_replica() const; bool is_specific_readonly_replica() const; @@ -240,6 +261,7 @@ private: ReplicaAttrArray logonly_replica_attr_array_; ReplicaAttrArray readonly_replica_attr_array_; ReplicaAttrArray encryption_logonly_replica_attr_array_; + ReplicaAttrArray columnstore_replica_attr_array_; }; struct SchemaZoneReplicaAttrSet @@ -253,6 +275,7 @@ struct SchemaZoneReplicaAttrSet int64_t get_logonly_replica_num() const {return replica_attr_set_.get_logonly_replica_num();} int64_t get_readonly_replica_num() const {return replica_attr_set_.get_readonly_replica_num();} int64_t get_encryption_logonly_replica_num() const {return replica_attr_set_.get_encryption_logonly_replica_num();} + int64_t get_columnstore_replica_num() const {return replica_attr_set_.get_columnstore_replica_num();} int64_t get_paxos_replica_num() const { return get_full_replica_num() + get_logonly_replica_num() + get_encryption_logonly_replica_num(); } @@ -291,34 +314,9 @@ struct ObZoneReplicaAttrSet int64_t get_encryption_logonly_replica_num() const { return replica_attr_set_.get_encryption_logonly_replica_num(); } + int64_t get_columnstore_replica_num() const {return replica_attr_set_.get_columnstore_replica_num();} const common::ObIArray &get_zone_set() const { return zone_set_; } - int add_full_replica_num(const ReplicaAttr &replica_attr) { - return replica_attr_set_.add_full_replica_num(replica_attr); - } - int add_logonly_replica_num(const ReplicaAttr &replica_attr) { - return replica_attr_set_.add_logonly_replica_num(replica_attr); - } - int add_readonly_replica_num(const ReplicaAttr &replica_attr) { - return replica_attr_set_.add_readonly_replica_num(replica_attr); - } - int add_encryption_logonly_replica_num(const ReplicaAttr &replica_attr) { - return replica_attr_set_.add_encryption_logonly_replica_num(replica_attr); - } - - int sub_full_replica_num(const ReplicaAttr &replica_attr) { - return replica_attr_set_.sub_full_replica_num(replica_attr); - } - int sub_logonly_replica_num(const ReplicaAttr &replica_attr) { - return replica_attr_set_.sub_logonly_replica_num(replica_attr); - } - int sub_readonly_replica_num(const ReplicaAttr &replica_attr) { - return replica_attr_set_.sub_readonly_replica_num(replica_attr); - } - int sub_encryption_logonly_replica_num(const ReplicaAttr &replica_attr) { - return replica_attr_set_.sub_encryption_logonly_replica_num(replica_attr); - } - int64_t get_paxos_replica_num() const { return get_full_replica_num() + get_logonly_replica_num() + get_encryption_logonly_replica_num(); } @@ -359,57 +357,6 @@ struct ObZoneReplicaAttrSet typedef ObZoneReplicaAttrSet ObZoneReplicaNumSet; -struct ObReplicaNumSet -{ - ObReplicaNumSet() : full_replica_num_(0), - logonly_replica_num_(0), - readonly_replica_num_(0), - encryption_logonly_replica_num_(0) - { reset(); } - virtual ~ObReplicaNumSet() {} - bool operator==(const ObReplicaNumSet &that) { - return full_replica_num_ == that.full_replica_num_ - && logonly_replica_num_ == that.logonly_replica_num_ - && readonly_replica_num_ == that.readonly_replica_num_ - && encryption_logonly_replica_num_ == that.encryption_logonly_replica_num_; - } - bool operator!=(const ObReplicaNumSet &that) { - return (!(*this == that)); - } - void reset() { - full_replica_num_ = 0; - logonly_replica_num_ = 0; - readonly_replica_num_ = 0; - encryption_logonly_replica_num_ = 0; - } - ObReplicaNumSet &operator=(const ObReplicaNumSet &that) { - if (this != &that) { - full_replica_num_ = that.full_replica_num_; - logonly_replica_num_ = that.logonly_replica_num_; - readonly_replica_num_ = that.readonly_replica_num_; - encryption_logonly_replica_num_ = that.encryption_logonly_replica_num_; - } - return *this; - } - void set_replica_num( - int64_t full_replica_num, - int64_t logonly_replica_num, - int64_t readonly_replica_num, - int64_t encryption_logonly_replica_num); - - int64_t get_specific_replica_num() const; - - TO_STRING_KV(K(full_replica_num_), - K(logonly_replica_num_), - K(readonly_replica_num_), - K(logonly_replica_num_)); - - int64_t full_replica_num_; - int64_t logonly_replica_num_; - int64_t readonly_replica_num_; - int64_t encryption_logonly_replica_num_; -}; - } // end namespace share } // end namespace oceanbase #endif diff --git a/src/share/ob_rpc_struct.cpp b/src/share/ob_rpc_struct.cpp index b53da0777..20a4a510e 100644 --- a/src/share/ob_rpc_struct.cpp +++ b/src/share/ob_rpc_struct.cpp @@ -4078,7 +4078,7 @@ int ObAdminAlterLSReplicaArg::init_add( int ret = OB_SUCCESS; if (OB_UNLIKELY(!ls_id.is_valid()) || OB_UNLIKELY(!server_addr.is_valid()) - || OB_UNLIKELY(replica_type != REPLICA_TYPE_FULL && replica_type != REPLICA_TYPE_READONLY) + || OB_UNLIKELY(!ObReplicaTypeCheck::is_replica_type_valid(replica_type)) || OB_UNLIKELY(paxos_replica_num < 0) || OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { //data_source and paxos_replica_num is optional parameter @@ -4157,7 +4157,7 @@ int ObAdminAlterLSReplicaArg::init_modify_replica( int ret = OB_SUCCESS; if (OB_UNLIKELY(!ls_id.is_valid()) || OB_UNLIKELY(!server_addr.is_valid()) - || OB_UNLIKELY(replica_type != REPLICA_TYPE_FULL && replica_type != REPLICA_TYPE_READONLY) + || OB_UNLIKELY(!ObReplicaTypeCheck::is_replica_type_valid(replica_type)) || OB_UNLIKELY(paxos_replica_num < 0) || OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { ret = OB_INVALID_ARGUMENT; @@ -4218,7 +4218,7 @@ void ObAdminAlterLSReplicaArg::reset() ls_id_.reset(); server_addr_.reset(); destination_addr_.reset(); - replica_type_ = common::REPLICA_TYPE_MAX; + replica_type_ = common::REPLICA_TYPE_INVALID; tenant_id_ = OB_INVALID_TENANT_ID; task_id_.reset(); data_source_.reset(); @@ -6235,98 +6235,6 @@ OB_SERIALIZE_MEMBER(ObSyncPGPartitionMTFinishArg, server_, version_); OB_SERIALIZE_MEMBER(ObCheckDanglingReplicaFinishArg, server_, version_, dangling_count_); -OB_SERIALIZE_MEMBER(ObMemberListAndLeaderArg, - member_list_, - leader_, - self_, - lower_list_, - replica_type_, - property_, - role_); - -bool ObMemberListAndLeaderArg::is_valid() const -{ - return member_list_.count() > 0 - && self_.is_valid() - && common::REPLICA_TYPE_MAX != replica_type_ - && property_.is_valid() - && (common::INVALID_ROLE <= role_ && role_ <= common::STANDBY_LEADER); -} - -// If it is a leader, you need to ensure the consistency of role_, leader_/restore_leader_, and self_ -bool ObMemberListAndLeaderArg::check_leader_is_valid() const -{ - bool bret = true; - if (is_leader_by_election(role_)) { - bret = (leader_.is_valid() && self_ == leader_); - } - return bret; -} - -void ObMemberListAndLeaderArg::reset() -{ - member_list_.reset(); - leader_.reset(); - self_.reset(); - lower_list_.reset(); - replica_type_ = common::REPLICA_TYPE_MAX; - role_ = common::INVALID_ROLE; -} - -int ObMemberListAndLeaderArg::assign(const ObMemberListAndLeaderArg &other) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(member_list_.assign(other.member_list_))) { - LOG_WARN("fail to assign member_list", KR(ret), K_(member_list)); - } else if (OB_FAIL(lower_list_.assign(other.lower_list_))) { - LOG_WARN("fail to assign lower_list", KR(ret), K_(lower_list)); - } else { - leader_ = other.leader_; - self_ = other.self_; - replica_type_ = other.replica_type_; - property_ = other.property_; - role_ = other.role_; - } - return ret; -} - -OB_SERIALIZE_MEMBER(ObGetMemberListAndLeaderResult, - member_list_, - leader_, - self_, - lower_list_, - replica_type_, - property_); - -void ObGetMemberListAndLeaderResult::reset() -{ - member_list_.reset(); - leader_.reset(); - self_.reset(); - lower_list_.reset(); - replica_type_ = common::REPLICA_TYPE_MAX; -} - -int ObGetMemberListAndLeaderResult::assign(const ObGetMemberListAndLeaderResult &other) -{ - int ret = OB_SUCCESS; - reset(); - if (OB_UNLIKELY(!other.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(other)); - } else if (OB_FAIL(member_list_.assign(other.member_list_))) { - LOG_WARN("failed to assign member list", K(ret)); - } else if (OB_FAIL(lower_list_.assign(other.lower_list_))) { - LOG_WARN("fail to assign member list", K(ret)); - } else { - leader_ = other.leader_; - self_ = other.self_; - replica_type_ = other.replica_type_; - property_ = other.property_; - } - return ret; -} - OB_SERIALIZE_MEMBER(ObBatchGetRoleResult, results_); void ObBatchGetRoleResult::reset() @@ -7580,7 +7488,7 @@ bool TenantServerUnitConfig::is_valid() const return common::OB_INVALID_ID != tenant_id_ && ((lib::Worker::CompatMode::INVALID != compat_mode_ && unit_config_.is_valid() - && replica_type_ != common::ObReplicaType::REPLICA_TYPE_MAX) + && replica_type_ != common::ObReplicaType::REPLICA_TYPE_INVALID) #ifdef OB_BUILD_TDE_SECURITY // root_key can be invalid #endif @@ -7648,7 +7556,7 @@ void TenantServerUnitConfig::reset() unit_id_ = OB_INVALID_ID; compat_mode_ = lib::Worker::CompatMode::INVALID; unit_config_.reset(); - replica_type_ = common::ObReplicaType::REPLICA_TYPE_MAX; + replica_type_ = common::ObReplicaType::REPLICA_TYPE_INVALID; if_not_grant_ = false; is_delete_ = false; #ifdef OB_BUILD_TDE_SECURITY @@ -8909,7 +8817,7 @@ bool ObCreateLSArg::is_valid() const { return OB_INVALID_TENANT_ID != tenant_id_ && id_.is_valid() - && REPLICA_TYPE_MAX != replica_type_ + && ObReplicaTypeCheck::is_replica_type_valid(replica_type_) && replica_property_.is_valid() && tenant_info_.is_valid() && create_scn_.is_valid() @@ -8921,7 +8829,7 @@ void ObCreateLSArg::reset() { tenant_id_ = OB_INVALID_TENANT_ID; id_.reset(); - replica_type_ = REPLICA_TYPE_MAX; + replica_type_ = REPLICA_TYPE_INVALID; replica_property_.reset(); tenant_info_.reset(); create_scn_.reset(); @@ -8960,7 +8868,7 @@ int ObCreateLSArg::init(const int64_t tenant_id, int ret = OB_SUCCESS; if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id ||!id.is_valid() - || REPLICA_TYPE_MAX == replica_type + || !ObReplicaTypeCheck::is_replica_type_valid(replica_type) || !replica_property.is_valid() || !tenant_info.is_valid())) { ret = OB_INVALID_ARGUMENT; diff --git a/src/share/ob_rpc_struct.h b/src/share/ob_rpc_struct.h index 94b70c546..a9ce8dee4 100644 --- a/src/share/ob_rpc_struct.h +++ b/src/share/ob_rpc_struct.h @@ -3414,7 +3414,7 @@ public: CREATE_WITH_PALF, }; ObCreateLSArg() : tenant_id_(OB_INVALID_TENANT_ID), id_(), - replica_type_(REPLICA_TYPE_MAX), + replica_type_(REPLICA_TYPE_INVALID), replica_property_(), tenant_info_(), create_scn_(), compat_mode_(lib::Worker::CompatMode::INVALID), @@ -4339,7 +4339,7 @@ public: : ls_id_(), server_addr_(), destination_addr_(), - replica_type_(common::REPLICA_TYPE_MAX), + replica_type_(common::REPLICA_TYPE_INVALID), tenant_id_(OB_INVALID_TENANT_ID), task_id_(), data_source_(), @@ -4411,7 +4411,7 @@ private: bool is_add_valid_() const { return ls_id_.is_valid() && server_addr_.is_valid() - && REPLICA_TYPE_MAX != replica_type_ + && ObReplicaTypeCheck::is_replica_type_valid(replica_type_) && is_valid_tenant_id(tenant_id_) && paxos_replica_num_ >= 0; } @@ -4430,7 +4430,7 @@ private: bool is_modify_replica_valid_() const { return ls_id_.is_valid() && server_addr_.is_valid() - && REPLICA_TYPE_MAX != replica_type_ + && ObReplicaTypeCheck::is_replica_type_valid(replica_type_) && is_valid_tenant_id(tenant_id_) && paxos_replica_num_ >= 0; } @@ -6918,65 +6918,6 @@ public: int64_t dangling_count_; }; -struct ObGetMemberListAndLeaderResult final -{ - OB_UNIS_VERSION(1); -public: - ObGetMemberListAndLeaderResult() - : member_list_(), - leader_(), - self_(), - lower_list_(), - replica_type_(common::REPLICA_TYPE_MAX), - property_() {} - void reset(); - inline bool is_valid() const { - return member_list_.count() > 0 - && self_.is_valid() - && common::REPLICA_TYPE_MAX != replica_type_ - && property_.is_valid(); - } - - int assign(const ObGetMemberListAndLeaderResult &other); - TO_STRING_KV(K_(member_list), K_(leader), K_(self), K_(lower_list), K_(replica_type), K_(property)); - - common::ObSEArray member_list_; // copy won't fail - common::ObAddr leader_; - common::ObAddr self_; - common::ObSEArray lower_list_; //Cascaded downstream information - common::ObReplicaType replica_type_; //The type of copy actually stored in the local copy - common::ObReplicaProperty property_; -}; - -struct ObMemberListAndLeaderArg -{ - OB_UNIS_VERSION(1); -public: - ObMemberListAndLeaderArg() - : member_list_(), - leader_(), - self_(), - lower_list_(), - replica_type_(common::REPLICA_TYPE_MAX), - property_(), - role_(common::INVALID_ROLE) {} - void reset(); - bool is_valid() const; - bool check_leader_is_valid() const; - int assign(const ObMemberListAndLeaderArg &other); - TO_STRING_KV(K_(member_list), K_(leader), K_(self), K_(lower_list), - K_(replica_type), K_(property), K_(role)); - - common::ObSArray member_list_; // copy won't fail - common::ObAddr leader_; - common::ObAddr self_; - common::ObSArray lower_list_; //Cascaded downstream information - common::ObReplicaType replica_type_; //The type of copy actually stored in the local copy - common::ObReplicaProperty property_; - common::ObRole role_; -}; - struct ObBatchGetRoleResult { OB_UNIS_VERSION(1); @@ -9917,7 +9858,7 @@ public: unit_id_(common::OB_INVALID_ID), compat_mode_(lib::Worker::CompatMode::INVALID), unit_config_(), - replica_type_(common::ObReplicaType::REPLICA_TYPE_MAX), + replica_type_(common::ObReplicaType::REPLICA_TYPE_INVALID), if_not_grant_(false), is_delete_(false) #ifdef OB_BUILD_TDE_SECURITY diff --git a/src/share/ob_share_util.cpp b/src/share/ob_share_util.cpp index 8b18a8650..42e1c387a 100644 --- a/src/share/ob_share_util.cpp +++ b/src/share/ob_share_util.cpp @@ -186,30 +186,8 @@ int ObShareUtil::check_compat_version_for_arbitration_service( const uint64_t tenant_id, bool &is_compatible) { - int ret = OB_SUCCESS; - is_compatible = false; - uint64_t data_version = 0; - if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(tenant_id)); - } else if (OB_FAIL(GET_MIN_DATA_VERSION(OB_SYS_TENANT_ID, data_version))) { - LOG_WARN("fail to get sys tenant data version", KR(ret)); - } else if (DATA_VERSION_4_1_0_0 > data_version) { - is_compatible = false; - } else if (!is_sys_tenant(tenant_id) - && OB_FAIL(GET_MIN_DATA_VERSION(gen_user_tenant_id(tenant_id), data_version))) { - LOG_WARN("fail to get user tenant data version", KR(ret), "tenant_id", gen_user_tenant_id(tenant_id)); - } else if (!is_sys_tenant(tenant_id) && DATA_VERSION_4_1_0_0 > data_version) { - is_compatible = false; - } else if (!is_sys_tenant(tenant_id) - && OB_FAIL(GET_MIN_DATA_VERSION(gen_meta_tenant_id(tenant_id), data_version))) { - LOG_WARN("fail to get meta tenant data version", KR(ret), "tenant_id", gen_meta_tenant_id(tenant_id)); - } else if (!is_sys_tenant(tenant_id) && DATA_VERSION_4_1_0_0 > data_version) { - is_compatible = false; - } else { - is_compatible = true; - } - return ret; + return check_compat_data_version_(DATA_VERSION_4_1_0_0, true/*check_meta*/, true/*check_user*/, + tenant_id, is_compatible); } int ObShareUtil::generate_arb_replica_num( @@ -239,25 +217,16 @@ int ObShareUtil::check_compat_version_for_readonly_replica( const uint64_t tenant_id, bool &is_compatible) { - int ret = OB_SUCCESS; - uint64_t data_version = 0; - is_compatible = false; - if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(tenant_id)); - } else if (OB_FAIL(GET_MIN_DATA_VERSION(OB_SYS_TENANT_ID, data_version))) { - LOG_WARN("fail to get sys tenant data version", KR(ret)); - } else if (DATA_VERSION_4_2_0_0 > data_version) { - is_compatible = false; - } else if (!is_sys_tenant(tenant_id) - && OB_FAIL(GET_MIN_DATA_VERSION(gen_meta_tenant_id(tenant_id), data_version))) { - LOG_WARN("fail to get meta tenant data version", KR(ret), "tenant_id", gen_meta_tenant_id(tenant_id)); - } else if (!is_sys_tenant(tenant_id) && DATA_VERSION_4_2_0_0 > data_version) { - is_compatible = false; - } else { - is_compatible = true; - } - return ret; + return check_compat_data_version_(DATA_VERSION_4_2_0_0, true/*check_meta*/, false/*check_user*/, + tenant_id, is_compatible); +} + +int ObShareUtil::check_compat_version_for_columnstore_replica( + const uint64_t tenant_id, + bool &is_compatible) +{ + return check_compat_data_version_(DATA_VERSION_4_3_3_0, true/*check_meta*/, false/*check_user*/, + tenant_id, is_compatible); } int ObShareUtil::fetch_current_cluster_version( @@ -450,34 +419,42 @@ bool ObShareUtil::is_tenant_enable_transfer(const uint64_t tenant_id) return bret; } -int ObShareUtil::check_compat_version_for_tenant( - const uint64_t tenant_id, - const uint64_t target_data_version, - bool &is_compatible) + +int ObShareUtil::check_compat_data_version_( + const uint64_t required_data_version, + const bool check_meta_tenant, + const bool check_user_tenant, + const uint64_t tenant_id, + bool &is_compatible) { int ret = OB_SUCCESS; - is_compatible = false; + is_compatible = true; uint64_t data_version = 0; if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id) - || OB_UNLIKELY(0 == target_data_version)) { + || OB_UNLIKELY(0 == required_data_version)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(target_data_version)); + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(required_data_version)); } else if (OB_FAIL(GET_MIN_DATA_VERSION(OB_SYS_TENANT_ID, data_version))) { LOG_WARN("fail to get sys tenant data version", KR(ret)); - } else if (target_data_version > data_version) { + } else if (required_data_version > data_version) { is_compatible = false; - } else if (is_sys_tenant(tenant_id)) { - is_compatible = true; - } else if (OB_FAIL(GET_MIN_DATA_VERSION(gen_user_tenant_id(tenant_id), data_version))) { - LOG_WARN("fail to get user tenant data version", KR(ret), "tenant_id", gen_user_tenant_id(tenant_id)); - } else if (target_data_version > data_version) { - is_compatible = false; - } else if (OB_FAIL(GET_MIN_DATA_VERSION(gen_meta_tenant_id(tenant_id), data_version))) { - LOG_WARN("fail to get meta tenant data version", KR(ret), "tenant_id", gen_meta_tenant_id(tenant_id)); - } else if (target_data_version > data_version) { - is_compatible = false; - } else { - is_compatible = true; + } else if (!is_sys_tenant(tenant_id)) { + if (check_meta_tenant) { + if (OB_FAIL(GET_MIN_DATA_VERSION(gen_meta_tenant_id(tenant_id), data_version))) { + LOG_WARN("fail to get meta tenant data version", KR(ret), "tenant_id", gen_meta_tenant_id(tenant_id)); + } else if (required_data_version > data_version) { + is_compatible = false; + } + } + if (OB_FAIL(ret) || !is_compatible) { + // skip + } else if (check_user_tenant) { + if (OB_FAIL(GET_MIN_DATA_VERSION(gen_user_tenant_id(tenant_id), data_version))) { + LOG_WARN("fail to get user tenant data version", KR(ret), "tenant_id", gen_user_tenant_id(tenant_id)); + } else if (required_data_version > data_version) { + is_compatible = false; + } + } } return ret; } @@ -492,8 +469,8 @@ int ObShareUtil::check_compat_version_for_clone_standby_tenant( if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(tenant_id)); - } else if (OB_FAIL(check_compat_version_for_tenant( - tenant_id, target_data_version, is_compatible))) { + } else if (OB_FAIL(check_compat_data_version_(target_data_version, + true/*check_meta*/, true/*check_user*/, tenant_id, is_compatible))) { LOG_WARN("fail to check data version for clone tenant", KR(ret), K(tenant_id), K(target_data_version)); } @@ -510,8 +487,8 @@ int ObShareUtil::check_compat_version_for_clone_tenant( if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(tenant_id)); - } else if (OB_FAIL(check_compat_version_for_tenant( - tenant_id, target_data_version, is_compatible))) { + } else if (OB_FAIL(check_compat_data_version_(target_data_version, + true/*check_meta*/, true/*check_user*/, tenant_id, is_compatible))) { LOG_WARN("fail to check data version for clone tenant", KR(ret), K(tenant_id), K(target_data_version)); } @@ -551,5 +528,77 @@ int ObShareUtil::check_compat_version_for_clone_tenant_with_tenant_role( return ret; } +const char *ObShareUtil::replica_type_to_string(const ObReplicaType type) +{ + const char *str = NULL; + switch (type) { + case ObReplicaType::REPLICA_TYPE_FULL: { + str = FULL_REPLICA_STR; + break; + } + case ObReplicaType::REPLICA_TYPE_BACKUP: { + str = BACKUP_REPLICA_STR; + break; + } + case ObReplicaType::REPLICA_TYPE_LOGONLY: { + str = LOGONLY_REPLICA_STR; + break; + } + case ObReplicaType::REPLICA_TYPE_READONLY: { + str = READONLY_REPLICA_STR; + break; + } + case ObReplicaType::REPLICA_TYPE_MEMONLY: { + str = MEMONLY_REPLICA_STR; + break; + } + case ObReplicaType::REPLICA_TYPE_ENCRYPTION_LOGONLY: { + str = ENCRYPTION_LOGONLY_REPLICA_STR; + break; + } + case ObReplicaType::REPLICA_TYPE_COLUMNSTORE: { + str = COLUMNSTORE_REPLICA_STR; + break; + } + default: { + str = "INVALID"; + break; + } + } + return str; +} + +// retrun REPLICA_TYPE_INVALID if str is invaild +ObReplicaType ObShareUtil::string_to_replica_type(const char *str) +{ + return string_to_replica_type(ObString(str)); +} + +// retrun REPLICA_TYPE_INVALID if str is invaild +ObReplicaType ObShareUtil::string_to_replica_type(const ObString &str) +{ + ObReplicaType replica_type = REPLICA_TYPE_INVALID; + if (OB_UNLIKELY(str.empty())) { + replica_type = REPLICA_TYPE_INVALID; + } else if (0 == str.case_compare(FULL_REPLICA_STR) || 0 == str.case_compare(F_REPLICA_STR)) { + replica_type = REPLICA_TYPE_FULL; + } else if (0 == str.case_compare(READONLY_REPLICA_STR) || 0 == str.case_compare(R_REPLICA_STR)) { + replica_type = REPLICA_TYPE_READONLY; + } else if (0 == str.case_compare(COLUMNSTORE_REPLICA_STR) || 0 == str.case_compare(C_REPLICA_STR)) { + replica_type = REPLICA_TYPE_COLUMNSTORE; + } else if (0 == str.case_compare(LOGONLY_REPLICA_STR) || 0 == str.case_compare(L_REPLICA_STR)) { + replica_type = REPLICA_TYPE_LOGONLY; + } else if (0 == str.case_compare(ENCRYPTION_LOGONLY_REPLICA_STR) || 0 == str.case_compare(E_REPLICA_STR)) { + replica_type = REPLICA_TYPE_ENCRYPTION_LOGONLY; + } else if (0 == str.case_compare(BACKUP_REPLICA_STR) || 0 == str.case_compare(B_REPLICA_STR)) { + replica_type = REPLICA_TYPE_BACKUP; + } else if (0 == str.case_compare(MEMONLY_REPLICA_STR) || 0 == str.case_compare(M_REPLICA_STR)) { + replica_type = REPLICA_TYPE_MEMONLY; + } else { + replica_type = REPLICA_TYPE_INVALID; + } + return replica_type; +} + } //end namespace share } //end namespace oceanbase diff --git a/src/share/ob_share_util.h b/src/share/ob_share_util.h index 2294746ef..9f3ac170c 100644 --- a/src/share/ob_share_util.h +++ b/src/share/ob_share_util.h @@ -78,14 +78,6 @@ public: static int check_compat_version_for_arbitration_service( const uint64_t tenant_id, bool &is_compatible); - // check whether sys/meta/user tenant has been promoted to target data version - // params[in] tenant_id, which tenant to check - // params[in] target_data_version, data version to check - // params[out] is_compatible, whether tenants are promoted to target data version - static int check_compat_version_for_tenant( - const uint64_t tenant_id, - const uint64_t target_data_version, - bool &is_compatible); // tenant data version should up to 430 when cloning primary tenant // tenant data version should up to 432 when cloning standby tenant // params[in] tenant_id, which tenant to check @@ -122,6 +114,13 @@ public: const uint64_t tenant_id, bool &is_compatible); + // data version must up to 4.3.2 with column-store replica + // @params[in] tenant_id, which tenant to check + // @params[out] is_compatible, whether it is over 4.3.2 + static int check_compat_version_for_columnstore_replica( + const uint64_t tenant_id, + bool &is_compatible); + static int fetch_current_cluster_version( common::ObISQLClient &client, uint64_t &cluster_version); @@ -148,6 +147,16 @@ public: SCN &ora_rowscn); static bool is_tenant_enable_rebalance(const uint64_t tenant_id); static bool is_tenant_enable_transfer(const uint64_t tenant_id); + static const char *replica_type_to_string(const ObReplicaType type); + static ObReplicaType string_to_replica_type(const char *str); + static ObReplicaType string_to_replica_type(const ObString &str); +private: + static int check_compat_data_version_( + const uint64_t required_data_version, + const bool check_meta_tenant, + const bool check_user_tenant, + const uint64_t tenant_id, + bool &is_compatible); }; }//end namespace share }//end namespace oceanbase diff --git a/src/share/ob_tablet_replica_checksum_operator.cpp b/src/share/ob_tablet_replica_checksum_operator.cpp index e0ccaf6e5..bf845391a 100644 --- a/src/share/ob_tablet_replica_checksum_operator.cpp +++ b/src/share/ob_tablet_replica_checksum_operator.cpp @@ -311,6 +311,20 @@ int ObTabletReplicaChecksumItem::verify_checksum(const ObTabletReplicaChecksumIt return ret; } +int ObTabletReplicaChecksumItem::verify_column_checksum(const ObTabletReplicaChecksumItem &other) const +{ + int ret = OB_SUCCESS; + if (compaction_scn_ == other.compaction_scn_) { + bool column_meta_equal = false; + if (OB_FAIL(column_meta_.check_equal(other.column_meta_, column_meta_equal))) { + LOG_WARN("fail to check column meta equal", KR(ret), K(other), K(*this)); + } else if (!column_meta_equal) { + ret = OB_CHECKSUM_ERROR; + } + } + return ret; +} + int ObTabletReplicaChecksumItem::assign_key(const ObTabletReplicaChecksumItem &other) { int ret = OB_SUCCESS; @@ -901,5 +915,45 @@ int ObTabletReplicaChecksumOperator::get_hex_column_meta( return ret; } +// ----------------------- ObTabletDataChecksumChecker ----------------------- +ObTabletDataChecksumChecker::ObTabletDataChecksumChecker() + : normal_ckm_item_(nullptr), + cs_replica_ckm_item_(nullptr) +{} + +ObTabletDataChecksumChecker::~ObTabletDataChecksumChecker() +{ + reset(); +} + +void ObTabletDataChecksumChecker::reset() +{ + normal_ckm_item_ = nullptr; + cs_replica_ckm_item_ = nullptr; +} + +int ObTabletDataChecksumChecker::check_data_checksum(const ObTabletReplicaChecksumItem& curr_item, bool is_cs_replica) +{ + int ret = OB_SUCCESS; + if (is_cs_replica) { + if (OB_ISNULL(cs_replica_ckm_item_)) { + cs_replica_ckm_item_ = &curr_item; + } else if (cs_replica_ckm_item_->compaction_scn_ != curr_item.compaction_scn_) { + LOG_INFO("no need to check data checksum", K(curr_item), KPC(this)); + } else if (cs_replica_ckm_item_->data_checksum_ != curr_item.data_checksum_) { + ret = OB_CHECKSUM_ERROR; + } + } else { + if (OB_ISNULL(normal_ckm_item_)) { + normal_ckm_item_ = &curr_item; + } else if (normal_ckm_item_->compaction_scn_ != curr_item.compaction_scn_) { + LOG_INFO("no need to check data checksum", K(curr_item), KPC(this)); + } else if (normal_ckm_item_->data_checksum_ != curr_item.data_checksum_) { + ret = OB_CHECKSUM_ERROR; + } + } + return ret; +} + } // share } // oceanbase diff --git a/src/share/ob_tablet_replica_checksum_operator.h b/src/share/ob_tablet_replica_checksum_operator.h index 3153fd886..b918aafd1 100644 --- a/src/share/ob_tablet_replica_checksum_operator.h +++ b/src/share/ob_tablet_replica_checksum_operator.h @@ -83,6 +83,7 @@ public: bool is_valid() const; bool is_same_tablet(const ObTabletReplicaChecksumItem &other) const; int verify_checksum(const ObTabletReplicaChecksumItem &other) const; + int verify_column_checksum(const ObTabletReplicaChecksumItem &other) const; int assign_key(const ObTabletReplicaChecksumItem &other); int assign(const ObTabletReplicaChecksumItem &other); int set_tenant_id(const uint64_t tenant_id); @@ -270,6 +271,18 @@ int ObTabletReplicaChecksumOperator::construct_batch_get_sql_str_( return ret; } +class ObTabletDataChecksumChecker +{ +public: + ObTabletDataChecksumChecker(); + ~ObTabletDataChecksumChecker(); + void reset(); + int check_data_checksum(const ObTabletReplicaChecksumItem& curr_item, bool is_cs_replica); + TO_STRING_KV(KPC_(normal_ckm_item), KPC_(cs_replica_ckm_item)); +private: + const ObTabletReplicaChecksumItem *normal_ckm_item_; + const ObTabletReplicaChecksumItem *cs_replica_ckm_item_; +}; } // share } // oceanbase diff --git a/src/share/scheduler/ob_dag_scheduler_config.h b/src/share/scheduler/ob_dag_scheduler_config.h index d28fee492..f6a685f03 100644 --- a/src/share/scheduler/ob_dag_scheduler_config.h +++ b/src/share/scheduler/ob_dag_scheduler_config.h @@ -101,6 +101,8 @@ DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_START_PREPARE_MIGRATION, ObDagPrio::DAG_PRIO true, 3, {"tenant_id", "ls_id", "op_type"}) DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_FINISH_PREPARE_MIGRATION, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::MIGRATION_TASK, "FINISH_PREPARE_MIGRATION", "MIGRATE", true, 3, {"tenant_id", "ls_id", "op_type"}) +DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_TABLET_CHECK_CONVERT, ObDagPrio::DAG_PRIO_HA_HIGH, ObSysTaskType::MIGRATION_TASK, "TABLET_CHECKE_CONVERT", "MIGRATE", + true, 3, {"tenant_id", "ls_id", "op_type"}) // DAG_TYPE_MIGRATE END DAG_SCHEDULER_DAG_TYPE_DEF(DAG_TYPE_FAST_MIGRATE, ObDagPrio::DAG_PRIO_HA_MID, ObSysTaskType::MIGRATION_TASK, "FAST_MIGRATE", "MIGRATE", false, 0, {}) diff --git a/src/share/scheduler/ob_tenant_dag_scheduler.h b/src/share/scheduler/ob_tenant_dag_scheduler.h index 4b0bf4984..342df8cd0 100644 --- a/src/share/scheduler/ob_tenant_dag_scheduler.h +++ b/src/share/scheduler/ob_tenant_dag_scheduler.h @@ -226,6 +226,7 @@ public: TASK_TYPE_START_REBUILD_TABLET_TASK = 70, TASK_TYPE_TABLET_REBUILD_TASK = 71, TASK_TYPE_FINISH_REBUILD_TABLET_TASK = 72, + TASK_TYPE_CHECK_CONVERT_TABLET = 73, TASK_TYPE_MAX, }; @@ -599,6 +600,7 @@ public: void init_dag_id(); int set_dag_id(const ObDagId &dag_net_id); const ObDagId &get_dag_id() const { return dag_net_id_; } + void set_dag_net_id(const ObDagId &dag_net_id) { dag_net_id_ = dag_net_id; } void set_add_time() { add_time_ = ObTimeUtility::fast_current_time(); } int64_t get_add_time() const { return add_time_; } void set_start_time() { start_time_ = ObTimeUtility::fast_current_time(); } diff --git a/src/share/schema/ob_schema_struct.cpp b/src/share/schema/ob_schema_struct.cpp index f74da18a6..582d212c8 100644 --- a/src/share/schema/ob_schema_struct.cpp +++ b/src/share/schema/ob_schema_struct.cpp @@ -1835,6 +1835,12 @@ void ObTenantSchema::reset_zone_replica_attr_array() free(encryption_logonly_attr_set.get_base_address()); encryption_logonly_attr_set.reset(); } + SchemaReplicaAttrArray &columnstore_attr_set + = static_cast(zone_locality.replica_attr_set_.get_columnstore_replica_attr_array()); + if (nullptr != columnstore_attr_set.get_base_address()) { + free(columnstore_attr_set.get_base_address()); + columnstore_attr_set.reset(); + } } free(zone_replica_attr_array_.get_base_address()); zone_replica_attr_array_.reset(); @@ -1906,6 +1912,10 @@ int ObTenantSchema::set_zone_replica_attr_array( static_cast(this_schema_set->replica_attr_set_.get_encryption_logonly_replica_attr_array()), src_replica_attr_set.replica_attr_set_.get_encryption_logonly_replica_attr_array()))) { LOG_WARN("fail to set specific replica attr array", K(ret)); + } else if (OB_FAIL(set_specific_replica_attr_array( + static_cast(this_schema_set->replica_attr_set_.get_columnstore_replica_attr_array()), + src_replica_attr_set.replica_attr_set_.get_columnstore_replica_attr_array()))) { + LOG_WARN("fail to set specific replica attr array", K(ret)); } else if (OB_FAIL(deep_copy_string_array(src_replica_attr_set.zone_set_, this_schema_set->zone_set_))) { LOG_WARN("fail to copy schema replica attr set zone set", K(ret)); } else { @@ -1952,6 +1962,10 @@ int ObTenantSchema::set_zone_replica_attr_array( static_cast(this_schema_set->replica_attr_set_.get_encryption_logonly_replica_attr_array()), src_replica_attr_set.replica_attr_set_.get_encryption_logonly_replica_attr_array()))) { LOG_WARN("fail to set specific replica attr array", K(ret)); + } else if (OB_FAIL(set_specific_replica_attr_array( + static_cast(this_schema_set->replica_attr_set_.get_columnstore_replica_attr_array()), + src_replica_attr_set.replica_attr_set_.get_columnstore_replica_attr_array()))) { + LOG_WARN("fail to set specific replica attr array", K(ret)); } else { common::ObArray zone_set_ptrs; for (int64_t j = 0; OB_SUCC(ret) && j < src_replica_attr_set.zone_set_.count(); ++j) { @@ -2635,267 +2649,6 @@ int ObDatabaseSchema::get_primary_zone_inherit( } return ret; } -/*------------------------------------------------------------------------------------------------- - * ------------------------------ObLocality------------------------------------------- - ----------------------------------------------------------------------------------------------------*/ -void ObLocality::reset_zone_replica_attr_array() -{ - if (NULL != schema_ && NULL != zone_replica_attr_array_.get_base_address()) { - for (int64_t i = 0; i < zone_replica_attr_array_.count(); ++i) { - SchemaZoneReplicaAttrSet &zone_locality = zone_replica_attr_array_.at(i); - schema_->reset_string_array(zone_locality.zone_set_); - SchemaReplicaAttrArray &full_attr_set - = static_cast(zone_locality.replica_attr_set_.get_full_replica_attr_array()); - if (nullptr != full_attr_set.get_base_address()) { - schema_->free(full_attr_set.get_base_address()); - full_attr_set.reset(); - } - SchemaReplicaAttrArray &logonly_attr_set - = static_cast(zone_locality.replica_attr_set_.get_logonly_replica_attr_array()); - if (nullptr != logonly_attr_set.get_base_address()) { - schema_->free(logonly_attr_set.get_base_address()); - logonly_attr_set.reset(); - } - SchemaReplicaAttrArray &readonly_attr_set - = static_cast(zone_locality.replica_attr_set_.get_readonly_replica_attr_array()); - if (nullptr != readonly_attr_set.get_base_address()) { - schema_->free(readonly_attr_set.get_base_address()); - readonly_attr_set.reset(); - } - SchemaReplicaAttrArray &encryption_logonly_attr_set - = static_cast(zone_locality.replica_attr_set_.get_encryption_logonly_replica_attr_array()); - if (nullptr != encryption_logonly_attr_set.get_base_address()) { - schema_->free(encryption_logonly_attr_set.get_base_address()); - encryption_logonly_attr_set.reset(); - } - } - schema_->free(zone_replica_attr_array_.get_base_address()); - zone_replica_attr_array_.reset(); - } -} - -int ObLocality::set_specific_replica_attr_array( - SchemaReplicaAttrArray &this_schema_set, - const common::ObIArray &src) -{ - int ret = OB_SUCCESS; - const int64_t count = src.count(); - if (count > 0) { - const int64_t size = count * static_cast(sizeof(share::ReplicaAttr)); - void *ptr = nullptr; - if (nullptr == (ptr = schema_->alloc(size))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_ERROR("alloc failed", K(ret), K(size)); - } else if (FALSE_IT(this_schema_set.init(count, static_cast(ptr), count))) { - // shall never by here - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < src.count(); ++i) { - const share::ReplicaAttr &src_replica_attr = src.at(i); - ReplicaAttr *dst_replica_attr = &this_schema_set.at(i); - if (nullptr == (dst_replica_attr = new (dst_replica_attr) ReplicaAttr( - src_replica_attr.num_, src_replica_attr.memstore_percent_))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("placement new return nullptr", K(ret)); - } - } - } - } - return ret; -} - -int ObLocality::set_zone_replica_attr_array(const common::ObIArray &src) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(schema_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(schema_)); - } else { - reset_zone_replica_attr_array(); - const int64_t alloc_size = src.count() * static_cast(sizeof(SchemaZoneReplicaAttrSet)); - void *buf = NULL; - if (src.count() <= 0) { - // do nothing - } else if (NULL == (buf = schema_->alloc(alloc_size))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_ERROR("alloc failed", K(ret), K(alloc_size)); - } else { - zone_replica_attr_array_.init(src.count(), static_cast(buf), src.count()); - // call construct func in advance to avoid core status - // - ARRAY_NEW_CONSTRUCT(SchemaZoneReplicaAttrSet, zone_replica_attr_array_); - for (int64_t i = 0; i < src.count() && OB_SUCC(ret); ++i) { - const SchemaZoneReplicaAttrSet &src_replica_attr_set = src.at(i); - SchemaZoneReplicaAttrSet *this_schema_set = &zone_replica_attr_array_.at(i); - if (OB_FAIL(set_specific_replica_attr_array( - static_cast(this_schema_set->replica_attr_set_.get_full_replica_attr_array()), - src_replica_attr_set.replica_attr_set_.get_full_replica_attr_array()))) { - LOG_WARN("fail to set specific replica attr array", K(ret)); - } else if (OB_FAIL(set_specific_replica_attr_array( - static_cast(this_schema_set->replica_attr_set_.get_logonly_replica_attr_array()), - src_replica_attr_set.replica_attr_set_.get_logonly_replica_attr_array()))) { - LOG_WARN("fail to set specific replica attr array", K(ret)); - } else if (OB_FAIL(set_specific_replica_attr_array( - static_cast(this_schema_set->replica_attr_set_.get_readonly_replica_attr_array()), - src_replica_attr_set.replica_attr_set_.get_readonly_replica_attr_array()))) { - LOG_WARN("fail to set specific replica attr array", K(ret)); - } else if (OB_FAIL(set_specific_replica_attr_array( - static_cast(this_schema_set->replica_attr_set_.get_encryption_logonly_replica_attr_array()), - src_replica_attr_set.replica_attr_set_.get_encryption_logonly_replica_attr_array()))) { - LOG_WARN("fail to set specific replica attr array", K(ret)); - } else if (OB_FAIL(schema_->deep_copy_string_array( - src_replica_attr_set.zone_set_, this_schema_set->zone_set_))) { - LOG_WARN("fail to copy schema replica attr set zone set", K(ret)); - } else { - this_schema_set->zone_ = src_replica_attr_set.zone_; - } - } - } - } - return ret; -} - -int ObLocality::set_zone_replica_attr_array(const common::ObIArray &src) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(schema_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(schema_)); - } else { - reset_zone_replica_attr_array(); - const int64_t alloc_size = src.count() * static_cast(sizeof(SchemaZoneReplicaAttrSet)); - void *buf = NULL; - if (src.count() <= 0) { - // do nothing - } else if (NULL == (buf = schema_->alloc(alloc_size))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_ERROR("alloc failed", K(ret), K(alloc_size)); - } else { - zone_replica_attr_array_.init(src.count(), static_cast(buf), src.count()); - // call construct func in advance to avoid core status - // - ARRAY_NEW_CONSTRUCT(SchemaZoneReplicaAttrSet, zone_replica_attr_array_); - for (int64_t i = 0; i < src.count() && OB_SUCC(ret); ++i) { - const share::ObZoneReplicaAttrSet &src_replica_attr_set = src.at(i); - SchemaZoneReplicaAttrSet *this_schema_set = &zone_replica_attr_array_.at(i); - if (OB_FAIL(set_specific_replica_attr_array( - static_cast(this_schema_set->replica_attr_set_.get_full_replica_attr_array()), - src_replica_attr_set.replica_attr_set_.get_full_replica_attr_array()))) { - LOG_WARN("fail to set specific replica attr array", K(ret)); - } else if (OB_FAIL(set_specific_replica_attr_array( - static_cast(this_schema_set->replica_attr_set_.get_logonly_replica_attr_array()), - src_replica_attr_set.replica_attr_set_.get_logonly_replica_attr_array()))) { - LOG_WARN("fail to set specific replica attr array", K(ret)); - } else if (OB_FAIL(set_specific_replica_attr_array( - static_cast(this_schema_set->replica_attr_set_.get_readonly_replica_attr_array()), - src_replica_attr_set.replica_attr_set_.get_readonly_replica_attr_array()))) { - LOG_WARN("fail to set specific replica attr array", K(ret)); - } else if (OB_FAIL(set_specific_replica_attr_array( - static_cast(this_schema_set->replica_attr_set_.get_encryption_logonly_replica_attr_array()), - src_replica_attr_set.replica_attr_set_.get_encryption_logonly_replica_attr_array()))) { - LOG_WARN("fail to set specific replica attr array", K(ret)); - } else { - common::ObArray zone_set_ptrs; - for (int64_t j = 0; OB_SUCC(ret) && j < src_replica_attr_set.zone_set_.count(); ++j) { - const common::ObZone &zone = src_replica_attr_set.zone_set_.at(j); - if (OB_FAIL(zone_set_ptrs.push_back(common::ObString(zone.size(), zone.ptr())))) { - LOG_WARN("fail to push back", K(ret)); - } else {} // no more to do - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(schema_->deep_copy_string_array(zone_set_ptrs, this_schema_set->zone_set_))) { - LOG_WARN("fail to copy schema replica attr set zone set", K(ret)); - } else { - this_schema_set->zone_ = src_replica_attr_set.zone_; - } - } - } - } - } - return ret; -} - -int ObLocality::assign(const ObLocality &other) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(schema_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(schema_)); - } else if (OB_FAIL(schema_->deep_copy_str(other.locality_str_, locality_str_))) { - LOG_WARN("fail to assign locality info", K(ret)); - } else if (OB_FAIL(set_zone_replica_attr_array(other.zone_replica_attr_array_))) { - LOG_WARN("set zone replica attr array failed", K(ret)); - } - return ret; -} - -int ObLocality::set_locality_str(const ObString &other) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(schema_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(schema_)); - } else if (OB_FAIL(schema_->deep_copy_str(other, locality_str_))) { - LOG_WARN("fail to assign locality info", K(ret)); - } - return ret; -} - -int64_t ObLocality::get_convert_size() const -{ - int64_t convert_size = sizeof(*this); - convert_size += zone_replica_attr_array_.count() * static_cast(sizeof(SchemaZoneReplicaAttrSet)); - for (int64_t i = 0; i < zone_replica_attr_array_.count(); ++i) { - convert_size += zone_replica_attr_array_.at(i).get_convert_size(); - } - convert_size += locality_str_.length() + 1; - return convert_size; -} - -void ObLocality::reset() -{ - if (OB_ISNULL(schema_)) { - LOG_ERROR_RET(OB_ERR_UNEXPECTED, "invalid schema info", K(schema_)); - } else { - reset_zone_replica_attr_array(); - if (!OB_ISNULL(locality_str_.ptr())) { - schema_->free(locality_str_.ptr()); - } - locality_str_.reset(); - } -} - -OB_DEF_SERIALIZE(ObLocality) -{ - int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_ENCODE, locality_str_); - if (OB_FAIL(ret)) { - LOG_WARN("func_SERIALIZE failed", K(ret)); - } else {} // no more to do - return ret; -} - -OB_DEF_DESERIALIZE(ObLocality) -{ - int ret = OB_SUCCESS; - ObString locality; - LST_DO_CODE(OB_UNIS_DECODE, locality); - if (OB_FAIL(ret)) { - } else if (OB_ISNULL(schema_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get invalid schema_ info", K(ret), K(schema_)); - } else if (OB_FAIL(schema_->deep_copy_str(locality, locality_str_))) { - LOG_WARN("fail to deep copy str", K(ret)); - } - return ret; -} - -OB_DEF_SERIALIZE_SIZE(ObLocality) -{ - int64_t len = 0; - LST_DO_CODE(OB_UNIS_ADD_LEN, locality_str_); - return len; -} - /*------------------------------------------------------------------------------------------------- * ------------------------------ObPrimaryZone------------------------------------------- diff --git a/src/share/schema/ob_schema_struct.h b/src/share/schema/ob_schema_struct.h index d9cc7b028..ec720ce5d 100755 --- a/src/share/schema/ob_schema_struct.h +++ b/src/share/schema/ob_schema_struct.h @@ -1413,7 +1413,6 @@ typedef common::ObArray ObPrimaryZoneArray; class ObSchema { public: - friend class ObLocality; friend class ObPrimaryZone; ObSchema(); //explicit ObSchema(common::ObDataBuffer &buffer); @@ -1530,31 +1529,6 @@ struct SchemaObj TO_STRING_KV(K_(schema_type), K_(tenant_id), K_(schema_id), KP_(schema)); }; -class ObLocality -{ - OB_UNIS_VERSION(1); -public: - explicit ObLocality(ObSchema *schema) : schema_(schema) {} - int assign(const ObLocality &other); - int set_locality_str(const common::ObString &locality); - int set_zone_replica_attr_array( - const common::ObIArray &src); - int set_zone_replica_attr_array( - const common::ObIArray &src); - int set_specific_replica_attr_array( - share::SchemaReplicaAttrArray &schema_replica_set, - const common::ObIArray &src); - void reset_zone_replica_attr_array(); - int64_t get_convert_size() const; - inline const common::ObString &get_locality_str() const { return locality_str_; } - void reset(); - TO_STRING_KV(K_(locality_str), K_(zone_replica_attr_array)); -public: - common::ObString locality_str_; - ZoneLocalityArray zone_replica_attr_array_; - ObSchema *schema_; -}; - class ObPrimaryZone { OB_UNIS_VERSION(1); diff --git a/src/share/schema/ob_table_dml_param.cpp b/src/share/schema/ob_table_dml_param.cpp index 954f0095c..df97f5f53 100644 --- a/src/share/schema/ob_table_dml_param.cpp +++ b/src/share/schema/ob_table_dml_param.cpp @@ -212,7 +212,7 @@ int ObTableSchemaParam::convert(const ObTableSchema *schema) if (OB_SUCC(ret)) { if (OB_FAIL(tmp_cols_index.push_back(col_index))) { LOG_WARN("fail to push_back col_index", K(ret)); - } else if (use_cs && OB_FAIL(schema->get_column_group_index(*column, cg_idx))) { + } else if (use_cs && OB_FAIL(schema->get_column_group_index(*column, false /*need_calculate_cg_idx*/, cg_idx))) { LOG_WARN("Fail to get column group index", K(ret)); } else if (use_cs && OB_FAIL(tmp_cg_idxs.push_back(cg_idx))) { LOG_WARN("Fail to push back cg idx", K(ret)); diff --git a/src/share/schema/ob_table_param.cpp b/src/share/schema/ob_table_param.cpp index ffedd9483..b05f36736 100644 --- a/src/share/schema/ob_table_param.cpp +++ b/src/share/schema/ob_table_param.cpp @@ -18,6 +18,7 @@ #include "observer/ob_server.h" #include "storage/ob_storage_schema.h" #include "storage/access/ob_table_read_info.h" +#include "storage/column_store/ob_column_store_replica_util.h" #include "share/ob_lob_access_utils.h" namespace oceanbase @@ -916,7 +917,8 @@ int ObTableParam::construct_columns_and_projector( const common::ObIArray & output_column_ids, const common::ObIArray *tsc_out_cols, const bool force_mysql_mode, - const sql::ObStoragePushdownFlag &pd_pushdown_flag) + const sql::ObStoragePushdownFlag &pd_pushdown_flag, + const bool query_cs_replica /*=false*/) { int ret = OB_SUCCESS; static const int64_t COMMON_COLUMN_NUM = 16; @@ -933,22 +935,19 @@ int ObTableParam::construct_columns_and_projector( bool is_cs = false; bool has_all_column_group = false; int64_t rowkey_count = 0; + is_column_replica_table_ = false; // row store table schema does not contains cg, if true, need calculate cg idx by designed rules - if (OB_SUCC(ret)) { - bool is_table_row_store = false; - if (OB_FAIL(table_schema.get_is_row_store(is_table_row_store))) { - LOG_WARN("fail to get is talbe row store", K(ret)); - } else { - is_cs = !is_table_row_store; - } + if (OB_FAIL(table_schema.get_is_column_store(is_cs))) { + LOG_WARN("fail to get is table column store", K(ret), K(table_schema)); + } else if (!is_cs && query_cs_replica) { + is_cs = true; + is_column_replica_table_ = true; } if (OB_FAIL(ret)) { } else if (OB_FAIL(table_schema.has_all_column_group(has_all_column_group))) { LOG_WARN("Failed to check if has all column group", K(ret)); - } - - if (OB_SUCC(ret)) { + } else { // column array const ObRowkeyInfo &rowkey_info = table_schema.get_rowkey_info(); rowkey_count = rowkey_info.get_size(); @@ -992,7 +991,7 @@ int ObTableParam::construct_columns_and_projector( } else if (OB_FAIL(tmp_access_cols_extend.push_back(tmp_col_extend))) { LOG_WARN("fail to push_back tmp_access_cols_extend", K(ret)); } else if (is_cs) { - if (OB_FAIL(table_schema.get_column_group_index(*column, cg_idx))) { + if (OB_FAIL(table_schema.get_column_group_index(*column, is_column_replica_table_, cg_idx))) { LOG_WARN("Fail to get column group index", K(ret)); } else if (OB_FAIL(tmp_cg_idxs.push_back(cg_idx))) { LOG_WARN("Fail to push back cg idx", K(ret)); @@ -1066,7 +1065,7 @@ int ObTableParam::construct_columns_and_projector( } else if (OB_FAIL(tmp_access_cols_extend.push_back(tmp_col_extend))) { LOG_WARN("fail to push_back tmp_access_cols_extend", K(ret)); } else if (is_cs) { - if (OB_FAIL(table_schema.get_column_group_index(*column, cg_idx))) { + if (OB_FAIL(table_schema.get_column_group_index(*column, is_column_replica_table_, cg_idx))) { LOG_WARN("Fail to get column group index", K(ret)); } else if (OB_FAIL(tmp_cg_idxs.push_back(cg_idx))) { LOG_WARN("Fail to push back cg idx", K(ret)); @@ -1248,7 +1247,8 @@ int ObTableParam::convert(const ObTableSchema &table_schema, const ObIArray &access_column_ids, const sql::ObStoragePushdownFlag &pd_pushdown_flag, const common::ObIArray *tsc_out_cols, - const bool force_mysql_mode) + const bool force_mysql_mode, + const bool query_cs_replica /*=false*/) { int ret = OB_SUCCESS; // if mocked rowid index is used @@ -1256,7 +1256,8 @@ int ObTableParam::convert(const ObTableSchema &table_schema, table_id_ = table_schema.get_table_id(); bool is_oracle_mode = false; const common::ObIArray *cols_param = nullptr; - if (OB_FAIL(construct_columns_and_projector(table_schema, access_column_ids, tsc_out_cols, force_mysql_mode, pd_pushdown_flag))) { + + if (OB_FAIL(construct_columns_and_projector(table_schema, access_column_ids, tsc_out_cols, force_mysql_mode, pd_pushdown_flag, query_cs_replica))) { LOG_WARN("construct failed", K(ret)); } else if (OB_ISNULL(cols_param = main_read_info_.get_columns())) { ret = OB_ERR_UNEXPECTED; @@ -1546,7 +1547,8 @@ int64_t ObTableParam::to_string(char *buf, const int64_t buf_len) const K_(rowid_projector), K_(enable_lob_locator_v2), K_(is_fts_index), - K_(parser_name)); + K_(parser_name), + K_(is_column_replica_table)); J_OBJ_END(); return pos; diff --git a/src/share/schema/ob_table_param.h b/src/share/schema/ob_table_param.h index 4d0f0931f..9a5378f81 100644 --- a/src/share/schema/ob_table_param.h +++ b/src/share/schema/ob_table_param.h @@ -292,7 +292,8 @@ public: const common::ObIArray &output_column_ids, const sql::ObStoragePushdownFlag &pd_pushdown_flag, const common::ObIArray *tsc_out_cols = NULL, - const bool force_mysql_mode = false); + const bool force_mysql_mode = false, + const bool query_cs_replica = false); // convert aggregate column projector from 'aggregate_column_ids' and 'output_projector_' // convert group by column projector from 'group_by_column_ids' and 'output_projector_' @@ -344,7 +345,8 @@ private: const common::ObIArray &output_column_ids, const common::ObIArray *tsc_out_cols, const bool force_mysql_mode, - const sql::ObStoragePushdownFlag &pd_pushdown_flag); + const sql::ObStoragePushdownFlag &pd_pushdown_flag, + const bool query_cs_replica = false); int filter_common_columns(const common::ObIArray &columns, common::ObIArray &new_columns); diff --git a/src/share/schema/ob_table_schema.cpp b/src/share/schema/ob_table_schema.cpp index 252524550..e43315e71 100644 --- a/src/share/schema/ob_table_schema.cpp +++ b/src/share/schema/ob_table_schema.cpp @@ -8958,14 +8958,17 @@ int ObTableSchema::is_column_group_exist(const ObString &cg_name, bool &exist) c return ret; } -int ObTableSchema::get_column_group_index(const share::schema::ObColumnParam ¶m, int32_t &cg_idx) const +int ObTableSchema::get_column_group_index( + const share::schema::ObColumnParam ¶m, + const bool need_calculate_cg_idx, + int32_t &cg_idx) const { int ret = OB_SUCCESS; - uint64_t column_id = param.get_column_id(); + const uint64_t column_id = param.get_column_id(); cg_idx = -1; - if (OB_UNLIKELY(1 >= column_group_cnt_)) { + if (OB_UNLIKELY(1 >= column_group_cnt_ && !need_calculate_cg_idx)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("No column group exist", K(ret), K_(is_column_store_supported), K_(column_group_cnt)); + LOG_WARN("No column group exist", K(ret), K(need_calculate_cg_idx), K_(is_column_store_supported), K_(column_group_cnt)); } else if (param.is_virtual_gen_col()) { cg_idx = -1; } else if (column_id < OB_END_RESERVED_COLUMN_ID_NUM && @@ -8974,7 +8977,9 @@ int ObTableSchema::get_column_group_index(const share::schema::ObColumnParam &pa common::OB_HIDDEN_PK_INCREMENT_COLUMN_ID != column_id) { // this has its own column group now if (common::OB_HIDDEN_TRANS_VERSION_COLUMN_ID == column_id || common::OB_HIDDEN_SQL_SEQUENCE_COLUMN_ID == column_id) { - if (OB_FAIL(get_base_rowkey_column_group_index(cg_idx))) { + if (need_calculate_cg_idx) { + cg_idx = OB_CS_COLUMN_REPLICA_ROWKEY_CG_IDX; + } else if (OB_FAIL(get_base_rowkey_column_group_index(cg_idx))) { LOG_WARN("Fail to get base/rowkey column group index", K(ret), K(column_id)); } } else { @@ -8984,6 +8989,10 @@ int ObTableSchema::get_column_group_index(const share::schema::ObColumnParam &pa // common::OB_HIDDEN_GROUP_IDX_COLUMN_ID == column_id cg_idx = -1; } + } else if (need_calculate_cg_idx) { + if (OB_FAIL(calc_column_group_index_(column_id, cg_idx))) { + LOG_WARN("Fail to calc_column_group_index", K(ret), K(column_id)); + } } else { bool found = false; int64_t cg_column_cnt = 0; @@ -9018,6 +9027,29 @@ int ObTableSchema::get_column_group_index(const share::schema::ObColumnParam &pa LOG_WARN("Unexpected, can not find cg idx", K(ret), K(column_id)); } } + LOG_TRACE("[CS-Replica] get column group index", K(ret), K(need_calculate_cg_idx), K(cg_idx)); + return ret; +} + +int ObTableSchema::calc_column_group_index_(const uint64_t column_id, int32_t &cg_idx) const +{ + int ret = OB_SUCCESS; + cg_idx = -1; + // for cs replica, constructed cg schemas start with rowkey cg so the cg idx of row key cg is ALWAYS 0 + // and cg idx of normal cg is shifted by offset 1. + for (int64_t i = 0; i < column_cnt_; i++) { + ObColumnSchemaV2 *column = column_array_[i]; + if (OB_NOT_NULL(column) && column->get_column_id() == column_id) { + cg_idx = i + 1; + break; + } + } + + if (OB_UNLIKELY(-1 == cg_idx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected cg idx", K(ret)); + } + return ret; } diff --git a/src/share/schema/ob_table_schema.h b/src/share/schema/ob_table_schema.h index 8f908aded..cb985c888 100644 --- a/src/share/schema/ob_table_schema.h +++ b/src/share/schema/ob_table_schema.h @@ -975,6 +975,8 @@ public: inline bool is_mlog_table() const { return is_mlog_table(table_type_); } inline static bool is_mlog_table(share::schema::ObTableType table_type) { return MATERIALIZED_VIEW_LOG == table_type; } + inline static bool is_user_data_table(share::schema::ObTableType table_type) + { return USER_TABLE == table_type; } inline bool is_in_recyclebin() const { return common::OB_RECYCLEBIN_SCHEMA_ID == database_id_; } virtual inline bool is_external_table() const override { return EXTERNAL_TABLE == table_type_; } @@ -1681,8 +1683,9 @@ public: int get_all_cg_type_column_group(const ObColumnGroupSchema *&column_group) const; int get_each_column_group(ObIArray &each_cgs) const; int is_partition_key_match_rowkey_prefix(bool &is_prefix) const; - int get_column_group_index(const share::schema::ObColumnParam ¶m, int32_t &cg_idx) const; - + int get_column_group_index(const share::schema::ObColumnParam ¶m, + const bool need_calculate_cg_idx, + int32_t &cg_idx) const; int is_column_group_exist(const common::ObString &cg_name, bool &exist) const; int get_all_column_ids(ObIArray &column_ids) const; @@ -1877,6 +1880,7 @@ private: ObRowkeyInfo &rowkey_info); int alter_view_column_internal(ObColumnSchemaV2 &column_schema); int get_base_rowkey_column_group_index(int32_t &cg_idx) const; + int calc_column_group_index_(const uint64_t column_id, int32_t &cg_idx) const; protected: uint64_t max_used_column_id_; diff --git a/src/share/system_variable/ob_system_variable_factory.cpp b/src/share/system_variable/ob_system_variable_factory.cpp index 69ed42d45..cb8327dbc 100644 --- a/src/share/system_variable/ob_system_variable_factory.cpp +++ b/src/share/system_variable/ob_system_variable_factory.cpp @@ -62,6 +62,7 @@ const char *ObSysVarObRoutePolicy::OB_ROUTE_POLICY_NAMES[] = { "ONLY_READONLY_ZONE", "UNMERGE_ZONE_FIRST", "UNMERGE_FOLLOWER_FIRST", + "COLUMN_STORE_ONLY", 0 }; const char *ObSysVarObEnableJit::OB_ENABLE_JIT_NAMES[] = { diff --git a/src/share/system_variable/ob_system_variable_init.cpp b/src/share/system_variable/ob_system_variable_init.cpp index e6f40e2cd..8b73096c0 100644 --- a/src/share/system_variable/ob_system_variable_init.cpp +++ b/src/share/system_variable/ob_system_variable_init.cpp @@ -1852,7 +1852,7 @@ static struct VarsInit{ ObSysVars[127].info_ = "the routing policy of obproxy/java client and observer internal retry, 1=READONLY_ZONE_FIRST, 2=ONLY_READONLY_ZONE, 3=UNMERGE_ZONE_FIRST, 4=UNMERGE_FOLLOWER_FIRST" ; ObSysVars[127].name_ = "ob_route_policy" ; ObSysVars[127].data_type_ = ObIntType ; - ObSysVars[127].enum_names_ = "[u'', u'READONLY_ZONE_FIRST', u'ONLY_READONLY_ZONE', u'UNMERGE_ZONE_FIRST', u'UNMERGE_FOLLOWER_FIRST']" ; + ObSysVars[127].enum_names_ = "[u'', u'READONLY_ZONE_FIRST', u'ONLY_READONLY_ZONE', u'UNMERGE_ZONE_FIRST', u'UNMERGE_FOLLOWER_FIRST', u'COLUMN_STORE_ONLY']" ; ObSysVars[127].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::INFLUENCE_PLAN | ObSysVarFlag::NEED_SERIALIZE ; ObSysVars[127].id_ = SYS_VAR_OB_ROUTE_POLICY ; cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_OB_ROUTE_POLICY)) ; diff --git a/src/share/system_variable/ob_system_variable_init.json b/src/share/system_variable/ob_system_variable_init.json index b498cbc38..80381d5dd 100644 --- a/src/share/system_variable/ob_system_variable_init.json +++ b/src/share/system_variable/ob_system_variable_init.json @@ -1844,7 +1844,8 @@ "READONLY_ZONE_FIRST", "ONLY_READONLY_ZONE", "UNMERGE_ZONE_FIRST", - "UNMERGE_FOLLOWER_FIRST" + "UNMERGE_FOLLOWER_FIRST", + "COLUMN_STORE_ONLY" ], "publish_version": "", "info_cn": "", diff --git a/src/share/table/ob_table.h b/src/share/table/ob_table.h index ea04e1440..33789cf33 100644 --- a/src/share/table/ob_table.h +++ b/src/share/table/ob_table.h @@ -1036,7 +1036,7 @@ public: schema_version_(common::OB_INVALID_VERSION), tablet_id_(common::ObTabletID::INVALID_TABLET_ID), role_(common::ObRole::INVALID_ROLE), - replica_type_(common::ObReplicaType::REPLICA_TYPE_MAX), + replica_type_(common::ObReplicaType::REPLICA_TYPE_INVALID), part_renew_time_(0), reserved_(0) {} diff --git a/src/sql/code_generator/ob_tsc_cg_service.cpp b/src/sql/code_generator/ob_tsc_cg_service.cpp index 939ba1711..42bb8d0c9 100644 --- a/src/sql/code_generator/ob_tsc_cg_service.cpp +++ b/src/sql/code_generator/ob_tsc_cg_service.cpp @@ -212,7 +212,10 @@ int ObTscCgService::generate_table_param(const ObLogTableScan &op, const bool pd_agg = scan_ctdef.pd_expr_spec_.pd_storage_flag_.is_aggregate_pushdown(); const bool pd_group_by = scan_ctdef.pd_expr_spec_.pd_storage_flag_.is_group_by_pushdown(); ObSqlSchemaGuard *schema_guard = cg_.opt_ctx_->get_sql_schema_guard(); - CK(OB_NOT_NULL(schema_guard)); + ObBasicSessionInfo *session_info = cg_.opt_ctx_->get_session_info(); + int64_t route_policy = 0; + bool is_cs_replica_query = false; + CK(OB_NOT_NULL(schema_guard), OB_NOT_NULL(session_info)); if (OB_UNLIKELY(pd_agg && 0 == scan_ctdef.aggregate_column_ids_.count()) || OB_UNLIKELY(pd_group_by && 0 == scan_ctdef.group_by_column_ids_.count())) { ret = OB_INVALID_ARGUMENT; @@ -231,6 +234,10 @@ int ObTscCgService::generate_table_param(const ObLogTableScan &op, } else if (table_schema->is_multivalue_index_aux() && FALSE_IT(scan_ctdef.table_param_.set_is_multivalue_index(true))) { } else if (OB_FAIL(extract_das_output_column_ids(op, scan_ctdef, *table_schema, tsc_out_cols))) { LOG_WARN("extract tsc output column ids failed", K(ret)); + } else if (OB_FAIL(session_info->get_sys_variable(SYS_VAR_OB_ROUTE_POLICY, route_policy))) { + LOG_WARN("get route policy failed", K(ret)); + } else { + is_cs_replica_query = ObRoutePolicyType::COLUMN_STORE_ONLY == route_policy; } if (OB_FAIL(ret)) { @@ -240,7 +247,8 @@ int ObTscCgService::generate_table_param(const ObLogTableScan &op, scan_ctdef.access_column_ids_, scan_ctdef.pd_expr_spec_.pd_storage_flag_, &tsc_out_cols, - is_oracle_mapping_real_virtual_table(op.get_ref_table_id())))) {/* for real agent table , use mysql mode compulsory*/ + is_oracle_mapping_real_virtual_table(op.get_ref_table_id()), /* for real agent table , use mysql mode compulsory*/ + is_cs_replica_query))) { LOG_WARN("convert schema failed", K(ret), K(*table_schema), K(scan_ctdef.access_column_ids_), K(op.get_index_back())); } else if ((pd_agg || pd_group_by) && @@ -1080,9 +1088,12 @@ int ObTscCgService::generate_table_loc_meta(uint64_t table_loc_id, loc_meta.is_external_files_on_disk_ = ObSQLUtils::is_external_files_on_local_disk(table_schema.get_external_file_location()); bool is_weak_read = false; + int64_t route_policy = 0; if (OB_ISNULL(cg_.opt_ctx_) || OB_ISNULL(cg_.opt_ctx_->get_exec_ctx())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(cg_.opt_ctx_), K(ret)); + } else if (OB_FAIL(session.get_sys_variable(SYS_VAR_OB_ROUTE_POLICY, route_policy))) { + LOG_WARN("get route policy failed", K(ret)); } else if (stmt.get_query_ctx()->has_dml_write_stmt_) { loc_meta.select_leader_ = 1; loc_meta.is_weak_read_ = 0; @@ -1101,6 +1112,7 @@ int ObTscCgService::generate_table_loc_meta(uint64_t table_loc_id, loc_meta.select_leader_ = 1; loc_meta.is_weak_read_ = 0; } + loc_meta.route_policy_ = route_policy; if (OB_SUCC(ret) && !table_schema.is_global_index_table()) { TableLocRelInfo *rel_info = nullptr; ObTableID data_table_id = table_schema.is_index_table() && !table_schema.is_rowkey_doc_id() ? diff --git a/src/sql/das/ob_das_location_router.cpp b/src/sql/das/ob_das_location_router.cpp index 3d3e6eba1..f7238ec62 100755 --- a/src/sql/das/ob_das_location_router.cpp +++ b/src/sql/das/ob_das_location_router.cpp @@ -792,7 +792,8 @@ ObDASLocationRouter::~ObDASLocationRouter() int ObDASLocationRouter::nonblock_get_readable_replica(const uint64_t tenant_id, const ObTabletID &tablet_id, - ObDASTabletLoc &tablet_loc) + ObDASTabletLoc &tablet_loc, + const ObRoutePolicyType route_policy) { int ret = OB_SUCCESS; ObLSLocation ls_loc; @@ -836,7 +837,10 @@ int ObDASLocationRouter::nonblock_get_readable_replica(const uint64_t tenant_id, } else if (OB_FAIL(ObBLService::get_instance().check_in_black_list(bl_key, in_black_list))) { LOG_WARN("check in black list failed", K(ret)); } else if (!in_black_list) { - if (tmp_replica_loc.get_server() == GCTX.self_addr()) { + if (route_policy == COLUMN_STORE_ONLY && tmp_replica_loc.get_replica_type() != REPLICA_TYPE_COLUMNSTORE) { + // skip the tmp_replica_loc + LOG_TRACE("skip the replica due to the COLUMN_STORE_ONLY policy.", K(ret), K(tmp_replica_loc)); + } else if (tmp_replica_loc.get_server() == GCTX.self_addr()) { //prefer choose the local replica local_replica = &tmp_replica_loc; } else if (OB_FAIL(remote_replicas.push_back(&tmp_replica_loc))) { @@ -849,6 +853,10 @@ int ObDASLocationRouter::nonblock_get_readable_replica(const uint64_t tenant_id, if (OB_SUCC(ret)) { if (local_replica != nullptr) { tablet_loc.server_ = local_replica->get_server(); + } else if (route_policy == COLUMN_STORE_ONLY && remote_replicas.empty()) { + //do not retry + ret = OB_NO_REPLICA_VALID; + LOG_USER_ERROR(OB_NO_REPLICA_VALID); } else if (remote_replicas.empty()) { ret = OB_NO_READABLE_REPLICA; LOG_WARN("there has no readable replica", K(ret), K(tablet_id), K(ls_loc)); @@ -972,7 +980,8 @@ int ObDASLocationRouter::get_tablet_loc(const ObDASTableLocMeta &loc_meta, //if this statement is retried because of OB_NOT_MASTER, we will choose the leader directly ret = nonblock_get_leader(tenant_id, tablet_id, tablet_loc); } else { - ret = nonblock_get_readable_replica(tenant_id, tablet_id, tablet_loc); + ret = nonblock_get_readable_replica(tenant_id, tablet_id, tablet_loc, + static_cast(loc_meta.route_policy_)); } } return ret; diff --git a/src/sql/das/ob_das_location_router.h b/src/sql/das/ob_das_location_router.h index 6ea995e31..794193eb5 100644 --- a/src/sql/das/ob_das_location_router.h +++ b/src/sql/das/ob_das_location_router.h @@ -16,6 +16,7 @@ #include "share/schema/ob_schema_struct.h" #include "lib/container/ob_fixed_array.h" #include "sql/das/ob_das_define.h" +#include "sql/optimizer/ob_route_policy.h" namespace oceanbase { namespace common @@ -346,7 +347,8 @@ private: share::ObLSLocation &location); int nonblock_get_readable_replica(const uint64_t tenant_id, const common::ObTabletID &tablet_id, - ObDASTabletLoc &tablet_loc); + ObDASTabletLoc &tablet_loc, + const ObRoutePolicyType route_policy); private: int last_errno_; int cur_errno_; diff --git a/src/sql/optimizer/ob_intersect_route_policy.cpp b/src/sql/optimizer/ob_intersect_route_policy.cpp index 2efffc8f3..8982c5df1 100644 --- a/src/sql/optimizer/ob_intersect_route_policy.cpp +++ b/src/sql/optimizer/ob_intersect_route_policy.cpp @@ -34,7 +34,7 @@ int ObIntersectRoutePolicy::init_candidate_replicas(const ObListwill_use_column_store(OB_INVALID_ID, + ref_id, ref_id, index_back_will_use_column_store, index_back_will_use_row_store))) { @@ -2808,6 +2809,7 @@ int ObJoinOrder::create_access_paths(const uint64_t table_id, LOG_WARN("failed to check will use skip scan", K(ret)); } else if (OB_FAIL(get_plan()->will_use_column_store(table_id, valid_index_ids.at(i), + ref_table_id, use_column_store, use_row_store))) { LOG_WARN("failed to check will use column store", K(ret)); @@ -10891,6 +10893,7 @@ int ObJoinOrder::find_possible_join_filter_tables(const ObLogPlanHint &log_plan_ info.use_column_store_ = true; } else if (OB_FAIL(get_plan()->will_use_column_store(info.table_id_, info.index_id_, + info.ref_table_id_, will_use_column_store, will_use_row_store))) { LOG_WARN("failed to check will use column store", K(ret)); diff --git a/src/sql/optimizer/ob_log_plan.cpp b/src/sql/optimizer/ob_log_plan.cpp index 9ae0ef781..c93687cf3 100644 --- a/src/sql/optimizer/ob_log_plan.cpp +++ b/src/sql/optimizer/ob_log_plan.cpp @@ -630,12 +630,16 @@ int ObLogPlan::mock_base_rel_detectors(ObJoinOrder *&base_rel) int ObLogPlan::select_location(ObIArray &tbl_part_info_list) { int ret = OB_SUCCESS; + int64_t route_policy = 0; ObExecContext *exec_ctx = optimizer_context_.get_exec_ctx(); ObSEArray tbl_loc_list; ObSEArray phy_tbl_loc_info_list; - if (OB_ISNULL(exec_ctx)) { + ObSQLSessionInfo* session_info = optimizer_context_.get_session_info(); + if (OB_ISNULL(exec_ctx) || OB_ISNULL(session_info)) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("exec ctx is NULL", K(ret)); + } else if (OB_FAIL(session_info->get_sys_variable(SYS_VAR_OB_ROUTE_POLICY, route_policy))) { + LOG_WARN("get route policy failed", K(ret)); } for (int64_t i = 0; OB_SUCC(ret) && i < tbl_part_info_list.count(); ++i) { ObTablePartitionInfo *tbl_part_info = tbl_part_info_list.at(i); @@ -649,6 +653,8 @@ int ObLogPlan::select_location(ObIArray &tbl_part_info_l &tbl_part_info->get_phy_tbl_location_info_for_update()))) { LOG_WARN("fail to push back phy tble loc info", K(ret), K(tbl_part_info->get_phy_tbl_location_info_for_update())); + } else { + tbl_part_info->get_table_location().get_loc_meta().route_policy_ = route_policy; } } if (OB_FAIL(ret)) { @@ -742,6 +748,9 @@ int ObLogPlan::select_replicas(ObExecContext &exec_ctx, } } } + } else if (COLUMN_STORE_ONLY == route_policy_type) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "when route policy is COLUMN_STORE_ONLY, weak read request"); } else { const bool sess_in_retry = session->get_is_in_retry_for_dup_tbl(); //重试状态下不优化复制表的副本选择 if (OB_FAIL(ObLogPlan::strong_select_replicas(local_server, phy_tbl_loc_info_list, is_hit_partition, sess_in_retry))) { @@ -13202,6 +13211,7 @@ int ObLogPlan::find_possible_join_filter_tables(ObLogicalOperator *op, info.use_column_store_ = true; } else if (OB_FAIL(will_use_column_store(info.table_id_, info.index_id_, + info.ref_table_id_, use_column_store, use_row_store))) { LOG_WARN("failed to check will use column store", K(ret)); @@ -13359,6 +13369,7 @@ int ObLogPlan::find_possible_join_filter_tables(ObLogicalOperator *op, int ObLogPlan::will_use_column_store(const uint64_t table_id, const uint64_t index_id, + const uint64_t ref_table_id, bool &use_column_store, bool &use_row_store) { @@ -13378,6 +13389,10 @@ int ObLogPlan::will_use_column_store(const uint64_t table_id, OB_ISNULL(session_info=get_optimizer_context().get_session_info())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("NULL pointer error", K(stmt), K(schema_guard), K(ret)); + } else if (get_optimizer_context().use_column_store_replica() && + index_id == ref_table_id) { + use_column_store = true; + use_row_store = false; } else if (OB_FALSE_IT(session_disable_column_store=!session_info->is_enable_column_store())) { } else if (OB_FALSE_IT(is_link=ObSqlSchemaGuard::is_link_table(stmt, table_id))) { } else if (is_link) { diff --git a/src/sql/optimizer/ob_log_plan.h b/src/sql/optimizer/ob_log_plan.h index ba8e46f3d..98c0ad62a 100644 --- a/src/sql/optimizer/ob_log_plan.h +++ b/src/sql/optimizer/ob_log_plan.h @@ -1396,6 +1396,7 @@ public: int will_use_column_store(const uint64_t table_id, const uint64_t index_id, + const uint64_t ref_table_id, bool &use_column_store, bool &use_row_store); diff --git a/src/sql/optimizer/ob_optimizer.cpp b/src/sql/optimizer/ob_optimizer.cpp index 89dea29cc..27208a35e 100644 --- a/src/sql/optimizer/ob_optimizer.cpp +++ b/src/sql/optimizer/ob_optimizer.cpp @@ -543,6 +543,8 @@ int ObOptimizer::init_env_info(ObDMLStmt &stmt) LOG_WARN("fail to check enable pdml", K(ret)); } else if (OB_FAIL(init_parallel_policy(stmt, *session_info))) { // call after check pdml enabled LOG_WARN("fail to check enable pdml", K(ret)); + } else if (OB_FAIL(init_replica_policy(stmt, *session_info))) { + LOG_WARN("fail to check enable column store replica", K(ret)); } else if (OB_FAIL(init_correlation_model(stmt, *session_info))) { LOG_WARN("failed to init correlation model", K(ret)); } @@ -694,6 +696,24 @@ int ObOptimizer::init_parallel_policy(ObDMLStmt &stmt, const ObSQLSessionInfo &s return ret; } +int ObOptimizer::init_replica_policy(ObDMLStmt &dml_stmt, const ObSQLSessionInfo &session) +{ + int ret = OB_SUCCESS; + int64_t route_policy_type = 0; + if (OB_FAIL(session.get_sys_variable(SYS_VAR_OB_ROUTE_POLICY, route_policy_type))) { + LOG_WARN("fail to get sys variable", K(ret)); + } else if (COLUMN_STORE_ONLY == static_cast(route_policy_type)) { + if (dml_stmt.get_query_ctx()->has_dml_write_stmt_ || + dml_stmt.get_query_ctx()->is_contain_select_for_update_) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "when route policy is COLUMN_STORE_ONLY, read query request"); + } else { + ctx_.set_use_column_store_replica(true); + } + } + return ret; +} + int ObOptimizer::init_correlation_model(ObDMLStmt &stmt, const ObSQLSessionInfo &session) { int ret = OB_SUCCESS; diff --git a/src/sql/optimizer/ob_optimizer.h b/src/sql/optimizer/ob_optimizer.h index 24bdcfe94..270f7d92b 100644 --- a/src/sql/optimizer/ob_optimizer.h +++ b/src/sql/optimizer/ob_optimizer.h @@ -203,6 +203,7 @@ namespace sql int extract_opt_ctx_basic_flags(const ObDMLStmt &stmt, ObSQLSessionInfo &session); int init_parallel_policy(ObDMLStmt &stmt, const ObSQLSessionInfo &session); + int init_replica_policy(ObDMLStmt &stmt, const ObSQLSessionInfo &session); int set_auto_dop_params(const ObSQLSessionInfo &session); int check_pdml_enabled(const ObDMLStmt &stmt, const ObSQLSessionInfo &session); diff --git a/src/sql/optimizer/ob_optimizer_context.h b/src/sql/optimizer/ob_optimizer_context.h index 8e0ed049a..bf43140ba 100644 --- a/src/sql/optimizer/ob_optimizer_context.h +++ b/src/sql/optimizer/ob_optimizer_context.h @@ -244,7 +244,8 @@ ObOptimizerContext(ObSQLSessionInfo *session_info, storage_estimation_enabled_(false), das_keep_order_enabled_(true), generate_random_plan_(false), - correlation_type_(ObEstCorrelationType::MAX) + correlation_type_(ObEstCorrelationType::MAX), + use_column_store_replica_(false) { } inline common::ObOptStatManager *get_opt_stat_manager() { return opt_stat_manager_; } inline void set_opt_stat_manager(common::ObOptStatManager *sm) { opt_stat_manager_ = sm; } @@ -618,6 +619,8 @@ ObOptimizerContext(ObSQLSessionInfo *session_info, inline const OptSystemStat& get_system_stat() const { return system_stat_; } inline bool generate_random_plan() const { return generate_random_plan_; } inline void set_generate_random_plan(bool rand_plan) { generate_random_plan_ = rand_plan; } + inline bool use_column_store_replica() const { return use_column_store_replica_; } + inline void set_use_column_store_replica(bool use) { use_column_store_replica_ = use; } inline void set_correlation_type(ObEstCorrelationType type) { correlation_type_ = type; } inline ObEstCorrelationType get_correlation_type() const { return correlation_type_; } @@ -708,6 +711,7 @@ private: bool generate_random_plan_; ObEstCorrelationType correlation_type_; + bool use_column_store_replica_; }; } } diff --git a/src/sql/optimizer/ob_replica_compare.cpp b/src/sql/optimizer/ob_replica_compare.cpp index 3d4493af6..0aa8a6428 100644 --- a/src/sql/optimizer/ob_replica_compare.cpp +++ b/src/sql/optimizer/ob_replica_compare.cpp @@ -23,11 +23,13 @@ ObReplicaCompare::ObReplicaCompare(ObRoutePolicyType policy_type) policy_type_(policy_type), readonly_zone_first_{IS_OTHER_REGION, ZONE_TYPE, MERGE_STATUS, POS_TYPE}, only_readonly_zone_{ZONE_TYPE, IS_OTHER_REGION, MERGE_STATUS, POS_TYPE,}, - unmerge_zone_first_{IS_OTHER_REGION, MERGE_STATUS, ZONE_TYPE, POS_TYPE} + unmerge_zone_first_{IS_OTHER_REGION, MERGE_STATUS, ZONE_TYPE, POS_TYPE}, + column_store_only_{ZONE_TYPE, IS_OTHER_REGION, MERGE_STATUS, POS_TYPE} { static_assert(sizeof(readonly_zone_first_) == sizeof(only_readonly_zone_), "invalid array size"); static_assert(sizeof(readonly_zone_first_) == sizeof(unmerge_zone_first_), "invalid array size"); static_assert((sizeof(readonly_zone_first_)/sizeof(CompareType)) == (sizeof(cmp_func_array_)/sizeof(CmpFuncPtr)), "invalid array size"); + static_assert(sizeof(readonly_zone_first_) == sizeof(column_store_only_), "invalid array size"); cmp_func_array_[IS_OTHER_REGION] = &ObReplicaCompare::compare_other_region; cmp_func_array_[ZONE_TYPE] = &ObReplicaCompare::compare_zone_type; @@ -50,6 +52,8 @@ bool ObReplicaCompare::operator()(const ObRoutePolicy::CandidateReplica &replica cmp_type_array = only_readonly_zone_; } else if (UNMERGE_ZONE_FIRST == policy_type_) { cmp_type_array = unmerge_zone_first_; + } else if (COLUMN_STORE_ONLY == policy_type_) { + cmp_type_array = column_store_only_; } else { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected policy type", K(policy_type_), K(ret)); diff --git a/src/sql/optimizer/ob_replica_compare.h b/src/sql/optimizer/ob_replica_compare.h index 4a7fe94ce..a271602c7 100644 --- a/src/sql/optimizer/ob_replica_compare.h +++ b/src/sql/optimizer/ob_replica_compare.h @@ -59,6 +59,7 @@ private: CompareType readonly_zone_first_[CMP_CNT]; CompareType only_readonly_zone_[CMP_CNT]; CompareType unmerge_zone_first_[CMP_CNT]; + CompareType column_store_only_[CMP_CNT]; }; diff --git a/src/sql/optimizer/ob_route_policy.cpp b/src/sql/optimizer/ob_route_policy.cpp index 771fe04b1..6b622dd34 100644 --- a/src/sql/optimizer/ob_route_policy.cpp +++ b/src/sql/optimizer/ob_route_policy.cpp @@ -16,6 +16,7 @@ #include "sql/optimizer/ob_phy_table_location_info.h" #include "sql/optimizer/ob_log_plan.h" #include "storage/ob_locality_manager.h" +#include "lib/ob_define.h" using namespace oceanbase::common; using namespace oceanbase::share; using namespace oceanbase::storage; @@ -82,6 +83,8 @@ int ObRoutePolicy::filter_replica(const ObAddr &local_server, } else { LOG_TRACE("check ls readable", K(ctx), K(ls_id), K(cur_replica.get_server()), K(can_read)); if ((policy_type == ONLY_READONLY_ZONE && cur_replica.attr_.zone_type_ == ZONE_TYPE_READWRITE) + || (policy_type == COLUMN_STORE_ONLY && !ObReplicaTypeCheck::is_columnstore_replica(cur_replica.get_replica_type())) + || (policy_type != COLUMN_STORE_ONLY && ObReplicaTypeCheck::is_columnstore_replica(cur_replica.get_replica_type())) || cur_replica.attr_.zone_status_ == ObZoneStatus::INACTIVE || cur_replica.attr_.server_status_ != ObServerStatus::OB_SERVER_ACTIVE || cur_replica.attr_.start_service_time_ == 0 @@ -102,6 +105,18 @@ int ObRoutePolicy::filter_replica(const ObAddr &local_server, } } } + if (OB_SUCC(ret) && policy_type == COLUMN_STORE_ONLY) { + for (int64_t i = candi_replicas.count()-1; OB_SUCC(ret) && i >= 0; --i) { + CandidateReplica &cur_replica = candi_replicas.at(i); + if (cur_replica.is_filter_ && OB_FAIL(candi_replicas.remove(i))) { + LOG_WARN("failed to remove filted replica", K(ret)); + } + } + if (OB_SUCC(ret) && candi_replicas.count() == 0) { + ret = OB_NO_REPLICA_VALID; + LOG_USER_ERROR(OB_NO_REPLICA_VALID); + } + } return ret; } diff --git a/src/sql/optimizer/ob_route_policy.h b/src/sql/optimizer/ob_route_policy.h index ad188d863..c25c123ee 100644 --- a/src/sql/optimizer/ob_route_policy.h +++ b/src/sql/optimizer/ob_route_policy.h @@ -38,6 +38,7 @@ enum ObRoutePolicyType // 即使客户端将请求路由到partition主上, 也在本地执行， // 区别在于返回给OCJ && ObProxy的反馈不同; UNMERGE_FOLLOWER_FIRST = 4, + COLUMN_STORE_ONLY = 5, POLICY_TYPE_MAX }; @@ -228,7 +229,9 @@ protected: // 集群为读写zone时, 且ob_route_policy为UNMERGE_FOLLOWER_FIRST时，同样按照READONLY_ZONE_FIRST处理, 但会增加反馈内容 // 集群为有只读zone时，且ob_route_policy为UNMERGE_FOLLOWER_FIRST时, 同样按照READONLY_ZONE_FIRST处理，此时不会增加反馈内容 ObRoutePolicyType type = INVALID_POLICY; - if (has_readonly_zone_) { + if (COLUMN_STORE_ONLY == ctx.policy_type_) { + type = ctx.policy_type_; + } else if (has_readonly_zone_) { if (UNMERGE_FOLLOWER_FIRST == ctx.policy_type_) { type = READONLY_ZONE_FIRST; } else { diff --git a/src/sql/optimizer/ob_table_location.cpp b/src/sql/optimizer/ob_table_location.cpp index eb24bf3ba..9cc39a8e6 100644 --- a/src/sql/optimizer/ob_table_location.cpp +++ b/src/sql/optimizer/ob_table_location.cpp @@ -1386,6 +1386,19 @@ int ObTableLocation::get_is_weak_read(const ObDMLStmt &dml_stmt, is_weak_read = (ObTxConsistencyType::BOUNDED_STALENESS_READ == trans_consistency_type); } } + if (OB_SUCC(ret) && !is_weak_read) { + int64_t route_policy_type = 0; + if (OB_FAIL(session->get_sys_variable(SYS_VAR_OB_ROUTE_POLICY, route_policy_type))) { + LOG_WARN("fail to get sys variable", K(ret)); + } else if (COLUMN_STORE_ONLY == static_cast(route_policy_type)) { + if (dml_stmt.get_query_ctx()->is_contain_inner_table_) { + is_weak_read = true; + } else { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "when route policy is COLUMN_STORE_ONLY, weak read request"); + } + } + } return ret; } diff --git a/src/sql/resolver/cmd/ob_alter_system_resolver.cpp b/src/sql/resolver/cmd/ob_alter_system_resolver.cpp index 85a41fa94..503ca673c 100644 --- a/src/sql/resolver/cmd/ob_alter_system_resolver.cpp +++ b/src/sql/resolver/cmd/ob_alter_system_resolver.cpp @@ -16,7 +16,6 @@ #include "common/ob_region.h" #include "lib/string/ob_sql_string.h" #include "share/schema/ob_schema_getter_guard.h" -#include "share/ob_locality_parser.h" #include "share/ob_time_utility2.h" #include "share/ob_encryption_util.h" #ifdef OB_BUILD_TDE_SECURITY @@ -142,8 +141,40 @@ int ObAlterSystemResolverUtil::resolve_replica_type(const ParseNode *parse_tree, } else { int64_t len = parse_tree->str_len_; const char *str = parse_tree->str_value_; - if (OB_FAIL(ObLocalityParser::parse_type(str, len, replica_type))) { - // do nothing, error log will print inside parse_type + if (OB_ISNULL(str)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid replica type string. null!", K(ret)); + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "replica_type, replica_type should not be null"); + } else { + replica_type = share::ObShareUtil::string_to_replica_type(str); + if (REPLICA_TYPE_INVALID == replica_type) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid replica type string", K(str), K(ret)); + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "replica_type, unrecognized replica_type"); + } else if (! ObReplicaTypeCheck::is_replica_type_valid(replica_type)) { + ret = OB_NOT_SUPPORTED; + char err_msg[64] = {0}; + (void)snprintf(err_msg, sizeof(err_msg), "%s replica", ObShareUtil::replica_type_to_string(replica_type)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, err_msg); + } else { + // good, valid replica_type + } + } + } + return ret; +} + +int ObAlterSystemResolverUtil::check_compatibility_for_replica_type(const ObReplicaType replica_type, const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + if (ObReplicaTypeCheck::is_columnstore_replica(replica_type)) { + bool is_compatible = false; + if (OB_FAIL(ObShareUtil::check_compat_version_for_columnstore_replica(tenant_id, is_compatible))) { + LOG_WARN("failed to check compat version for C-replcia", KR(ret), K(tenant_id)); + } else if (OB_UNLIKELY(!is_compatible)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("data_version lower than 4.3.3, C-replica not supported"); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "data_version is lower than 4.3.3, C-replica"); } } return ret; @@ -2984,7 +3015,7 @@ int ObAddLSReplicaResolver::resolve(const ParseNode &parse_tree) int64_t ls_id = 0; common::ObAddr server_addr; - common::ObReplicaType replica_type = REPLICA_TYPE_MAX; + common::ObReplicaType replica_type = REPLICA_TYPE_INVALID; common::ObAddr data_source; int64_t paxos_replica_num = 0; uint64_t tenant_id = OB_INVALID_TENANT_ID; @@ -3000,6 +3031,8 @@ int ObAddLSReplicaResolver::resolve(const ParseNode &parse_tree) LOG_WARN("resolve server failed", KR(ret), KP(server_addr_node)); } else if (OB_FAIL(Util::resolve_replica_type(replica_type_node, replica_type))) { LOG_WARN("resolve replica type failed", KR(ret), KP(replica_type_node)); + } else if (OB_FAIL(Util::check_compatibility_for_replica_type(replica_type, tenant_id))) { + LOG_WARN("check compatibility for replica_type failed", KR(ret), K(replica_type), K(tenant_id)); } else if (OB_FAIL(Util::check_and_get_data_source(data_source_node, data_source))) { LOG_WARN("check and get data source failed", KR(ret), KP(data_source_node)); } else if (OB_FAIL(Util::check_and_get_paxos_replica_num(paxos_replica_num_node, paxos_replica_num))) { @@ -3178,7 +3211,7 @@ int ObModifyLSReplicaResolver::resolve(const ParseNode &parse_tree) ParseNode *tenant_name_node = parse_tree.children_[4]; int64_t ls_id = 0; common::ObAddr server_addr; - common::ObReplicaType replica_type = REPLICA_TYPE_MAX; + common::ObReplicaType replica_type = REPLICA_TYPE_INVALID; int64_t paxos_replica_num = 0; uint64_t tenant_id = OB_INVALID_TENANT_ID; if (OB_FAIL(Util::do_check_for_alter_ls_replica(tenant_name_node, @@ -3193,6 +3226,8 @@ int ObModifyLSReplicaResolver::resolve(const ParseNode &parse_tree) LOG_WARN("resolve server failed", KR(ret), KP(server_addr_node)); } else if (OB_FAIL(Util::resolve_replica_type(replica_type_node, replica_type))) { LOG_WARN("resolve replica type failed", KR(ret), KP(replica_type_node)); + } else if (OB_FAIL(Util::check_compatibility_for_replica_type(replica_type, tenant_id))) { + LOG_WARN("check compatibility for replica_type failed", KR(ret), K(replica_type), K(tenant_id)); } else if (OB_FAIL(Util::check_and_get_paxos_replica_num(paxos_replica_num_node, paxos_replica_num))) { LOG_WARN("check and get paxos replica num failed", KR(ret), KP(paxos_replica_num_node)); } diff --git a/src/sql/resolver/cmd/ob_alter_system_resolver.h b/src/sql/resolver/cmd/ob_alter_system_resolver.h index 682350bb1..839a2b62d 100644 --- a/src/sql/resolver/cmd/ob_alter_system_resolver.h +++ b/src/sql/resolver/cmd/ob_alter_system_resolver.h @@ -46,6 +46,7 @@ public: static int resolve_replica_type(const ParseNode *parse_tree, common::ObReplicaType &replica_type); + static int check_compatibility_for_replica_type(const ObReplicaType replica_type, const uint64_t tenant_id); static int resolve_memstore_percent(const ParseNode *parse_tree, ObReplicaProperty &replica_property); static int resolve_string(const ParseNode *parse_tree, common::ObString &string); diff --git a/src/sql/resolver/cmd/ob_resource_resolver.h b/src/sql/resolver/cmd/ob_resource_resolver.h index 92882bcaf..7bf4ca321 100644 --- a/src/sql/resolver/cmd/ob_resource_resolver.h +++ b/src/sql/resolver/cmd/ob_resource_resolver.h @@ -106,6 +106,10 @@ int ObResourcePoolOptionResolver::resolve_option(T *stmt, ParseNode *option_n SQL_RESV_LOG(WARN, "invalid replica type option_node", K(ret), K(option_node)); } else if (OB_FAIL(ObAlterSystemResolverUtil::resolve_replica_type(option_node->children_[0], type))) { SQL_RESV_LOG(WARN, "fail to resove repilca type", K(ret)); + } else if (REPLICA_TYPE_FULL != type) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("replica_type of resource pool other than FULL not supported.", KR(ret), K(type)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "replica_type of resource pool other than FULL replica"); } else { stmt->set_replica_type(type); } diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 23ac9cb28..6d89f6a7e 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -235,6 +235,7 @@ ob_set_subtarget(ob_storage high_availability high_availability/ob_ls_block_tx_service.cpp high_availability/ob_storage_ha_diagnose_mgr.cpp high_availability/ob_storage_ha_diagnose_service.cpp + high_availability/ob_cs_replica_migration.cpp ) ob_set_subtarget(ob_storage restore @@ -516,6 +517,7 @@ ob_set_subtarget(ob_storage column_store column_store/ob_virtual_cg_scanner.cpp column_store/ob_column_store_util.cpp column_store/ob_cg_group_by_scanner.cpp + column_store/ob_column_store_replica_util.cpp ) ob_set_subtarget(ob_storage access diff --git a/src/storage/access/ob_table_access_param.cpp b/src/storage/access/ob_table_access_param.cpp index 30c51b682..6305d16cb 100644 --- a/src/storage/access/ob_table_access_param.cpp +++ b/src/storage/access/ob_table_access_param.cpp @@ -59,7 +59,8 @@ ObTableIterParam::ObTableIterParam() auto_split_filter_type_(OB_INVALID_ID), auto_split_filter_(nullptr), auto_split_params_(nullptr), - is_tablet_spliting_(false) + is_tablet_spliting_(false), + is_column_replica_table_(false) { } @@ -111,6 +112,7 @@ void ObTableIterParam::reset() auto_split_filter_ = nullptr; auto_split_params_ = nullptr; is_tablet_spliting_ = false; + is_column_replica_table_ = false; ObSSTableIndexFilterFactory::destroy_sstable_index_filter(sstable_index_filter_); } @@ -316,6 +318,7 @@ int ObTableAccessParam::init( iter_param_.table_scan_opt_.storage_rowsets_size_ = 1; } iter_param_.pushdown_filter_ = scan_param.pd_storage_filters_; + iter_param_.is_column_replica_table_ = table_param.is_column_replica_table(); // disable blockscan if scan order is KeepOrder(for iterator iterator and table api) // disable blockscan if use index skip scan as no large range to scan if (OB_UNLIKELY(ObQueryFlag::KeepOrder == scan_param.scan_flag_.scan_order_ || diff --git a/src/storage/access/ob_table_access_param.h b/src/storage/access/ob_table_access_param.h index f7358902c..b21b524b4 100644 --- a/src/storage/access/ob_table_access_param.h +++ b/src/storage/access/ob_table_access_param.h @@ -230,6 +230,7 @@ public: const sql::ObExpr *auto_split_filter_; sql::ExprFixedArray *auto_split_params_; bool is_tablet_spliting_; + bool is_column_replica_table_; }; struct ObTableAccessParam diff --git a/src/storage/blocksstable/ob_data_store_desc.cpp b/src/storage/blocksstable/ob_data_store_desc.cpp index a6616c262..d28851bec 100644 --- a/src/storage/blocksstable/ob_data_store_desc.cpp +++ b/src/storage/blocksstable/ob_data_store_desc.cpp @@ -37,6 +37,7 @@ bool ObStaticDataStoreDesc::is_valid() const void ObStaticDataStoreDesc::reset() { MEMSET(this, 0, sizeof(*this)); + need_submit_io_ = true; } int ObStaticDataStoreDesc::assign(const ObStaticDataStoreDesc &desc) @@ -58,6 +59,7 @@ int ObStaticDataStoreDesc::assign(const ObStaticDataStoreDesc &desc) encrypt_id_ = desc.encrypt_id_; master_key_id_ = desc.master_key_id_; MEMCPY(encrypt_key_, desc.encrypt_key_, sizeof(encrypt_key_)); + need_submit_io_ = desc.need_submit_io_; return ret; } @@ -104,7 +106,8 @@ int ObStaticDataStoreDesc::init( const compaction::ObMergeType merge_type, const int64_t snapshot_version, const share::SCN &end_scn, - const int64_t cluster_version) + const int64_t cluster_version, + const bool need_submit_io) { int ret = OB_SUCCESS; const bool is_major = compaction::is_major_or_meta_merge_type(merge_type); @@ -118,6 +121,7 @@ int ObStaticDataStoreDesc::init( merge_type_ = merge_type; ls_id_ = ls_id; tablet_id_ = tablet_id; + need_submit_io_ = need_submit_io; if (!is_major) { end_scn_ = end_scn; @@ -863,11 +867,12 @@ int ObWholeDataStoreDesc::init( const int64_t cluster_version, const share::SCN &end_scn, const storage::ObStorageColumnGroupSchema *cg_schema, - const uint16_t table_cg_idx) + const uint16_t table_cg_idx, + const bool need_submit_io /*=true*/) { int ret = OB_SUCCESS; reset(); - if (OB_FAIL(static_desc_.init(is_ddl, merge_schema, ls_id, tablet_id, merge_type, snapshot_version, end_scn, cluster_version))) { + if (OB_FAIL(static_desc_.init(is_ddl, merge_schema, ls_id, tablet_id, merge_type, snapshot_version, end_scn, cluster_version, need_submit_io))) { STORAGE_LOG(WARN, "failed to init static desc", KR(ret)); } else if (OB_FAIL(inner_init(merge_schema, cg_schema, table_cg_idx))) { STORAGE_LOG(WARN, "failed to init", KR(ret), K(merge_schema), K(cg_schema), K(table_cg_idx)); diff --git a/src/storage/blocksstable/ob_data_store_desc.h b/src/storage/blocksstable/ob_data_store_desc.h index b12f4c7c9..d380fe760 100644 --- a/src/storage/blocksstable/ob_data_store_desc.h +++ b/src/storage/blocksstable/ob_data_store_desc.h @@ -58,7 +58,8 @@ public: const compaction::ObMergeType merge_type, const int64_t snapshot_version, const share::SCN &end_scn, - const int64_t cluster_version); + const int64_t cluster_version, + const bool need_submit_io = true); bool is_valid() const; void reset(); int assign(const ObStaticDataStoreDesc &desc); @@ -78,7 +79,8 @@ public: K_(master_key_id), KPHEX_(encrypt_key, sizeof(encrypt_key_)), K_(major_working_cluster_version), - K_(progressive_merge_round)); + K_(progressive_merge_round), + K_(need_submit_io)); private: OB_INLINE int init_encryption_info(const share::schema::ObMergeSchema &merge_schema); OB_INLINE void init_block_size(const share::schema::ObMergeSchema &merge_schema); @@ -106,6 +108,9 @@ public: int64_t encrypt_id_; int64_t master_key_id_; char encrypt_key_[share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH]; + // For ddl redo log for cs replica, leader write only macro block data in memory but do not flush to disk. + // indicate whether to submit io to write maroc block data to disk. + bool need_submit_io_; }; // ObColDataStoreDesc is same for every parallel task @@ -253,6 +258,7 @@ public: STATIC_DESC_FUNC(ObCompressorType, compressor_type); STATIC_DESC_FUNC(int64_t, major_working_cluster_version); STATIC_DESC_FUNC(const char *, encrypt_key); + STATIC_DESC_FUNC(bool, need_submit_io); COL_DESC_FUNC(bool, is_row_store); COL_DESC_FUNC(uint16_t, table_cg_idx); COL_DESC_FUNC(int64_t, row_column_count); @@ -335,7 +341,8 @@ struct ObWholeDataStoreDesc const int64_t cluster_version, const share::SCN &end_scn = share::SCN::invalid_scn(), const storage::ObStorageColumnGroupSchema *cg_schema = nullptr, - const uint16_t table_cg_idx = 0); + const uint16_t table_cg_idx = 0, + const bool need_submit_io = true); int gen_index_store_desc(const ObDataStoreDesc &data_desc); int assign(const ObDataStoreDesc &desc); ObStaticDataStoreDesc &get_static_desc() { return static_desc_; } diff --git a/src/storage/blocksstable/ob_macro_block.cpp b/src/storage/blocksstable/ob_macro_block.cpp index 1944a77ed..0cf0928e4 100644 --- a/src/storage/blocksstable/ob_macro_block.cpp +++ b/src/storage/blocksstable/ob_macro_block.cpp @@ -357,21 +357,10 @@ int ObMacroBlock::flush(ObMacroBlockHandle ¯o_handle, macro_header_.fixed_header_.data_checksum_ = 0; } #endif - + const bool need_flush_macro = spec_->get_need_submit_io(); if (OB_FAIL(write_macro_header())) { STORAGE_LOG(WARN, "fail to write macro header", K(ret), K_(macro_header)); - } else { - const int64_t common_header_size = common_header_.get_serialize_size(); - const char *payload_buf = data_.data() + common_header_size; - const int64_t payload_size = data_.length() - common_header_size; - common_header_.set_payload_size(static_cast(payload_size)); - common_header_.set_payload_checksum(static_cast(ob_crc64(payload_buf, payload_size))); - } - if (OB_FAIL(ret)) { - // do nothing - } else if (OB_FAIL(common_header_.build_serialized_header(data_.data(), data_.capacity()))) { - STORAGE_LOG(WARN, "Fail to build common header, ", K(ret), K_(common_header)); - } else { + } else if (need_flush_macro) { ObMacroBlockWriteInfo write_info; write_info.buffer_ = data_.data(); if (backup::ObBackupDeviceMacroBlockId::is_backup_block_file(macro_handle.get_macro_id().first_id())) { @@ -486,7 +475,18 @@ int ObMacroBlock::write_macro_header() int64_t pos = 0; if (OB_FAIL(macro_header_.serialize(data_.data() + common_header_size, buf_len, pos))) { STORAGE_LOG(WARN, "fail to serialize macro block", K(ret), K(macro_header_)); + } else { + const int64_t common_header_size = common_header_.get_serialize_size(); + const char *payload_buf = data_.data() + common_header_size; + const int64_t payload_size = data_.length() - common_header_size; + common_header_.set_payload_size(static_cast(payload_size)); + common_header_.set_payload_checksum(static_cast(ob_crc64(payload_buf, payload_size))); + + if (OB_FAIL(common_header_.build_serialized_header(data_.data(), data_.capacity()))) { + STORAGE_LOG(WARN, "Fail to build common header, ", K(ret), K_(common_header)); + } } + return ret; } diff --git a/src/storage/column_store/ob_cg_iter_param_pool.cpp b/src/storage/column_store/ob_cg_iter_param_pool.cpp index 6f2d657ca..cc88866f8 100644 --- a/src/storage/column_store/ob_cg_iter_param_pool.cpp +++ b/src/storage/column_store/ob_cg_iter_param_pool.cpp @@ -268,6 +268,7 @@ int ObCGIterParamPool::generate_for_column_store(const ObTableIterParam &row_par //cg_param.ss_rowkey_prefix_cnt_ = 0; cg_param.pd_storage_flag_ = row_param.pd_storage_flag_; cg_param.table_scan_opt_ = row_param.table_scan_opt_; + cg_param.is_column_replica_table_ = row_param.is_column_replica_table_; if (nullptr != row_param.cg_read_infos_) { if (OB_UNLIKELY(nullptr == row_param.cg_read_infos_->at(cg_pos))) { ret = OB_ERR_UNEXPECTED; diff --git a/src/storage/column_store/ob_co_merge_ctx.cpp b/src/storage/column_store/ob_co_merge_ctx.cpp index 3f4e82827..b4042fa64 100644 --- a/src/storage/column_store/ob_co_merge_ctx.cpp +++ b/src/storage/column_store/ob_co_merge_ctx.cpp @@ -152,17 +152,70 @@ int ObCOTabletMergeCtx::init_tablet_merge_info(const bool need_check) return ret; } +int ObCOTabletMergeCtx::prepare_cs_replica_param() +{ + int ret = OB_SUCCESS; + static_param_.is_cs_replica_ = false; + ObStorageSchema *schema_on_tablet = nullptr; + ObSSTable *sstable = nullptr; + if (static_param_.ls_handle_.get_ls()->is_cs_replica()) { + if (OB_FAIL(static_param_.tablet_schema_guard_.init(tablet_handle_, mem_ctx_))) { + LOG_WARN("failed to init cs replica schema guard", K(ret), KPC(this)); + } else if (OB_FAIL(static_param_.tablet_schema_guard_.load(schema_on_tablet))) { + LOG_WARN("failed to load schema on tablet", K(ret)); + } else if (schema_on_tablet->is_cs_replica_compat()) { + static_param_.is_cs_replica_ = true; + } else if (is_convert_co_major_merge(get_merge_type())) { + static_param_.is_cs_replica_ = true; + } else { + static_param_.is_cs_replica_ = static_param_.get_tablet_id().is_user_tablet() + && schema_on_tablet->is_user_data_table() + && schema_on_tablet->is_row_store(); + } + + if (OB_FAIL(ret) || !static_param_.is_cs_replica_) { + } else if (OB_ISNULL(sstable = static_cast(get_tables_handle().get_table(0)))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get sstable", K(ret), K(sstable), K(static_param_.tables_handle_)); + } else if (OB_UNLIKELY(!sstable->is_co_sstable())) { + // may be column store replica rebuild from full/read only row store replica + if (sstable->is_major_sstable()) { + static_param_.major_sstable_status_ = ObCOMajorSSTableStatus::COL_REPLICA_MAJOR; + static_param_.co_major_merge_type_ = ObCOMajorMergePolicy::USE_RS_BUILD_SCHEMA_MATCH_MERGE; + LOG_INFO("[CS-Replica] Decide rebuild column store from row major for cs replica", K(ret), KPC(sstable), K_(static_param)); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("first table should be major in cs replica", K(ret), KPC(sstable), K_(static_param)); + } + } else { + static_param_.co_major_merge_type_ = ObCOMajorMergePolicy::BUILD_COLUMN_STORE_MERGE; + } + } + LOG_INFO("[CS-Replica] prepare_cs_replica_param", K(ret), "merge_type", get_merge_type(), + "co_merge_type", ObCOMajorMergePolicy::co_major_merge_type_to_str(static_param_.co_major_merge_type_), + "is_cs_replica",static_param_.is_cs_replica_, KPC(schema_on_tablet)); + return ret; +} + int ObCOTabletMergeCtx::prepare_schema() { int ret = OB_SUCCESS; - - if (is_meta_major_merge(get_merge_type())) { + if (OB_FAIL(prepare_cs_replica_param())) { + LOG_WARN("failed to prepare cs replica param", K(ret), K_(static_param)); + } else if (is_meta_major_merge(get_merge_type())) { if (OB_FAIL(get_meta_compaction_info())) { LOG_WARN("failed to get meta compaction info", K(ret), KPC(this)); } + } else if (is_convert_co_major_merge(get_merge_type())) { + if (OB_FAIL(get_convert_compaction_info())) { + LOG_WARN("failed to get convert compaction info", K(ret), KPC(this)); + } } else if (OB_FAIL(get_medium_compaction_info())) { // have checked medium info inside LOG_WARN("failed to get medium compaction info", K(ret), KPC(this)); + } else { + LOG_INFO("[CS-Replica] finish prepare schema for co merge", K(ret), + "is_cs_replica", static_param_.is_cs_replica_, KPC(this)); } return ret; } @@ -665,9 +718,11 @@ int ObCOTabletMergeCtx::init_major_sstable_status() { int ret = OB_SUCCESS; ObSSTable *sstable = static_cast(get_tables_handle().get_table(0)); - if (OB_ISNULL(sstable) || OB_UNLIKELY(!sstable->is_co_sstable())) { + if (OB_ISNULL(sstable)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("fail to get sstable", K(ret), K(sstable), K(static_param_.tables_handle_)); + } else if (OB_UNLIKELY(!sstable->is_co_sstable())) { + // maybe cs replica, processed in ObCOTabletMergeCtx::prepare_cs_replica_param } else if (OB_FAIL(ObCOMajorMergePolicy::decide_co_major_sstable_status( *static_cast(sstable), *get_schema(), diff --git a/src/storage/column_store/ob_co_merge_ctx.h b/src/storage/column_store/ob_co_merge_ctx.h index cd8ccd0bf..83d150af0 100644 --- a/src/storage/column_store/ob_co_merge_ctx.h +++ b/src/storage/column_store/ob_co_merge_ctx.h @@ -136,6 +136,7 @@ struct ObCOTabletMergeCtx : public ObBasicTabletMergeCtx { return ObBasicTabletMergeCtx::swap_tablet(get_merge_table_result); } int prepare_mocked_row_store_cg_schema(); bool should_mock_row_store_cg_schema(); + int prepare_cs_replica_param(); OB_INLINE bool is_build_row_store_from_rowkey_cg() const { return static_param_.is_build_row_store_from_rowkey_cg(); } OB_INLINE bool is_build_row_store() const { return static_param_.is_build_row_store(); } int get_cg_schema_for_merge(const int64_t idx, const ObStorageColumnGroupSchema *&cg_schema_ptr); diff --git a/src/storage/column_store/ob_co_merge_dag.cpp b/src/storage/column_store/ob_co_merge_dag.cpp index db435c348..771dbcef1 100644 --- a/src/storage/column_store/ob_co_merge_dag.cpp +++ b/src/storage/column_store/ob_co_merge_dag.cpp @@ -183,7 +183,9 @@ int ObCOMergePrepareTask::create_schedule_dag(ObCOTabletMergeCtx &ctx) ObGetMergeTablesResult result; bool schedule_minor = false; - if (OB_FAIL(ctx.check_need_schedule_minor(schedule_minor))) { + if (is_convert_co_major_merge(ctx.get_merge_type())) { + // convert co major merge only rely on major sstable + } else if (OB_FAIL(ctx.check_need_schedule_minor(schedule_minor))) { LOG_WARN("failed to check need chedule minor", K(ret), K(schedule_minor)); } else if (schedule_minor) { ObTableHandleV2 tmp_table_handle; @@ -1056,6 +1058,7 @@ int ObCOMergeDagNet::init_by_param(const ObIDagInitParam *param) merge_type_ = merge_param->merge_type_; ls_id_ = merge_param->ls_id_; tablet_id_ = merge_param->tablet_id_; + (void) set_dag_net_id(merge_param->dag_net_id_); is_inited_ = true; } return ret; @@ -1317,7 +1320,7 @@ int ObCOMergeDagNet::inner_create_row_store_dag( common::ObIArray &exe_dag_array) { int ret = OB_SUCCESS; - LOG_DEBUG("chengkong debug: build row store in this compaction", "co_major_merge_type_", + LOG_DEBUG("build row store in this compaction", "co_major_merge_type_", ObCOMajorMergePolicy::co_major_merge_type_to_str(co_merge_ctx_->static_param_.co_major_merge_type_)); dag = nullptr; diff --git a/src/storage/column_store/ob_column_oriented_sstable.h b/src/storage/column_store/ob_column_oriented_sstable.h index fb2ce5c89..ecaa3d680 100644 --- a/src/storage/column_store/ob_column_oriented_sstable.h +++ b/src/storage/column_store/ob_column_oriented_sstable.h @@ -92,6 +92,7 @@ public: uint32_t full_column_cnt_; }; + enum ObCOSSTableBaseType : int32_t { INVALID_TYPE = 0, @@ -102,10 +103,11 @@ enum ObCOSSTableBaseType : int32_t enum ObCOMajorSSTableStatus: uint8_t { INVALID_CO_MAJOR_SSTABLE_STATUS = 0, - COL_WITH_ALL, // all cg + normal cg - COL_ONLY_ALL, // all cg only (schema have all cg) - PURE_COL, // rowkey cg + normal cg - PURE_COL_ONLY_ALL, // all cg only (schema do not have all cg) + COL_WITH_ALL = 1, // all cg + normal cg + COL_ONLY_ALL = 2, // all cg only (schema have all cg) + PURE_COL = 3, // rowkey cg + normal cg + PURE_COL_ONLY_ALL = 4, // all cg only (schema do not have all cg) + COL_REPLICA_MAJOR = 5, // temp status, row store major from F/R replica for column store replica MAX_CO_MAJOR_SSTABLE_STATUS }; /* @@ -120,6 +122,8 @@ enum ObCOMajorSSTableStatus: uint8_t { +-----------------+---------------+---------------+-------+ |PURE_COL_ONLY_ALL| EACH | ALL | NO | +-----------------+---------------+---------------+-------+ + |COL_REPLICA_MAJOR| ROW STORE | ROW STORE | YES | + +-----------------+---------------+---------------+-------+ */ inline bool is_valid_co_major_sstable_status(const ObCOMajorSSTableStatus& major_sstable_status) { diff --git a/src/storage/column_store/ob_column_store_replica_util.cpp b/src/storage/column_store/ob_column_store_replica_util.cpp new file mode 100644 index 000000000..1df0e938d --- /dev/null +++ b/src/storage/column_store/ob_column_store_replica_util.cpp @@ -0,0 +1,233 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + + #include "storage/tx_storage/ob_ls_service.h" + #include "storage/tablet/ob_mds_schema_helper.h" + #include "storage/column_store/ob_column_store_replica_util.h" + #define USING_LOG_PREFIX STORAGE + +namespace oceanbase +{ +namespace storage +{ + +int ObCSReplicaUtil::check_is_cs_replica( + const ObTableSchema &table_schema, + const ObTablet &tablet, + bool &is_cs_replica) +{ + int ret = OB_SUCCESS; + bool is_row_store = false; + is_cs_replica = false; + if (OB_FAIL(table_schema.get_is_row_store(is_row_store))) { + LOG_WARN("fail to get is row store", K(ret), K(table_schema)); + } else { + is_cs_replica = is_row_store + && table_schema.is_user_table() + && !tablet.is_row_store() // tablet in cs replica is not row store + && tablet.get_tablet_id().is_user_tablet(); + } + return ret; +} + +int ObCSReplicaUtil::check_local_is_cs_replica( + const share::ObLSID &ls_id, + bool &is_cs_replica) +{ + int ret = OB_SUCCESS; + is_cs_replica = false; + ObLS *ls = nullptr; + ObLSHandle ls_handle; + + if (OB_FAIL(MTL(ObLSService*)->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { + if (OB_LS_NOT_EXIST == ret) { + // Only use weak consistency locally read for cs replica, so local ls must exist. + // If ls not exist, table param or dml param is constructed remotely, ignore it. + ret = OB_SUCCESS; + } else { + LOG_WARN("failed to get ls", K(ret), K(ls_id)); + } + } else if (OB_ISNULL(ls = ls_handle.get_ls())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get nullptr ls", K(ret), K(ls_id), K(ls_handle)); + } else { + is_cs_replica = ls->is_cs_replica(); + } + return ret; +} + +bool ObCSReplicaUtil::check_need_convert_cs_when_migration( + const ObTablet &tablet, + const ObStorageSchema& schema_on_tablet) +{ + return schema_on_tablet.is_row_store() + && schema_on_tablet.is_user_data_table() + && tablet.is_row_store() + && tablet.get_tablet_id().is_user_tablet(); +} + +int ObCSReplicaUtil::check_has_cs_replica( + const share::ObLSID &ls_id, + bool &has_column_store_replica) +{ + int ret = OB_SUCCESS; + has_column_store_replica = false; + ObLS *ls = nullptr; + ObLSHandle ls_handle; + + if (OB_FAIL(MTL(ObLSService*)->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { + LOG_WARN("failed to get ls", K(ret), K(ls_id)); + } else if (OB_ISNULL(ls = ls_handle.get_ls())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get nullptr ls", K(ret), K(ls_id), K(ls_handle)); + } else if (OB_FAIL(ls->check_has_cs_replica(has_column_store_replica))) { + LOG_WARN("failed to check ls replica set", K(ret), KPC(ls)); + } + return ret; +} + +int ObCSReplicaUtil::check_need_process_cs_replica( + const ObLS &ls, + const ObTabletID &tablet_id, + const ObStorageSchema &schema, + bool &need_process_cs_replica) +{ + int ret = OB_SUCCESS; + need_process_cs_replica = ls.is_cs_replica() + && tablet_id.is_user_tablet() + && (schema.is_row_store() || schema.is_cs_replica_compat()) + && schema.is_user_data_table(); + return ret; +} + +int ObCSReplicaUtil::check_need_wait_major_convert( + const ObLS &ls, + const ObTabletID &tablet_id, + const ObTablet &tablet, + bool &need_wait_major_convert) +{ + int ret = OB_SUCCESS; + bool need_process_cs_replica = false; + ObStorageSchema *storage_schema = nullptr; + ObArenaAllocator arena_allocator(common::ObMemAttr(MTL_ID(), "CkMjrCvrt")); + ObTabletMemberWrapper wrapper; + const ObTabletTableStore *table_store = nullptr; + const ObITable *sstable = nullptr; + need_wait_major_convert = false; + if (OB_FAIL(tablet.load_storage_schema(arena_allocator, storage_schema))) { + LOG_WARN("fail to load storage schema", K(ret), K(tablet)); + } else if (OB_ISNULL(storage_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("storage schema is nullptr", K(ret), K(tablet)); + } else if (OB_FAIL(check_need_process_cs_replica(ls, tablet_id, *storage_schema, need_process_cs_replica))) { + LOG_WARN("fail to check need process cs replica", K(ret), K(ls), K(tablet_id), KPC(storage_schema)); + } else if (need_process_cs_replica) { + if (tablet.is_row_store()) { + // tablet migration but not do co convert + need_wait_major_convert = true; + } else if (OB_FAIL(tablet.fetch_table_store(wrapper))) { + LOG_WARN("failed to fetch table store", K(ret), K(tablet_id), K(tablet)); + } else if (OB_ISNULL(table_store = wrapper.get_member())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table store is nullptr", K(ret), K(tablet_id), K(tablet)); + } else if (OB_ISNULL(sstable = table_store->get_major_sstables().get_boundary_table(true /*is_last*/))) { + if (!tablet.get_tablet_meta().table_store_flag_.with_major_sstable()) { + ret = OB_SSTABLE_NOT_EXIST; + LOG_WARN("latest major is nullptr", K(ret), K(tablet_id), K(tablet)); + } + } else { + // ddl write row store major + need_wait_major_convert = ObITable::is_row_store_major_sstable(sstable->get_key().table_type_); + } + } + ObTabletObjLoadHelper::free(arena_allocator, storage_schema); + return ret; +} + +int ObCSReplicaUtil::check_replica_set_need_process_cs_replica( + const ObLS &ls, + const ObTabletID &tablet_id, + const ObStorageSchema &schema, + bool &need_process_cs_replica) +{ + int ret = OB_SUCCESS; + need_process_cs_replica = false; + if (OB_FAIL(ls.check_has_cs_replica(need_process_cs_replica))) { + LOG_WARN("failed to check ls replica set", K(ret), K(ls)); + } else if (need_process_cs_replica) { + need_process_cs_replica = tablet_id.is_user_tablet() + && schema.is_row_store() + && schema.is_user_data_table(); + } + return ret; +} + +ObCSReplicaStorageSchemaGuard::ObCSReplicaStorageSchemaGuard() + : is_inited_(false), + schema_(nullptr) +{ +} + +ObCSReplicaStorageSchemaGuard::~ObCSReplicaStorageSchemaGuard() +{ + reset(); +} + +int ObCSReplicaStorageSchemaGuard::init( + const ObTabletHandle &tablet_handle, + compaction::ObCompactionMemoryContext &mem_ctx) +{ + int ret = OB_SUCCESS; + ObStorageSchema *schema_on_tablet = nullptr; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", K(ret)); + } else if (OB_UNLIKELY(!tablet_handle.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(tablet_handle)); + } else if (OB_FAIL(tablet_handle.get_obj()->load_storage_schema(mem_ctx.get_allocator(), schema_on_tablet))) { + LOG_WARN("failed to load storage schema", K(ret), K(tablet_handle)); + } else { + schema_ = schema_on_tablet; + is_inited_ = true; + } + return ret; +} + +void ObCSReplicaStorageSchemaGuard::reset() +{ + if (IS_INIT) { + if (OB_NOT_NULL(schema_)) { + schema_->~ObStorageSchema(); + schema_ = nullptr; + } + is_inited_ = false; + } +} + +int ObCSReplicaStorageSchemaGuard::load(ObStorageSchema *&storage_schema) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("not init", K(ret)); + } else if (OB_ISNULL(schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("schema is nullptr", K(ret)); + } else { + storage_schema = schema_; + } + return ret; +} + +} // namespace storage +} // namespace oceanbase \ No newline at end of file diff --git a/src/storage/column_store/ob_column_store_replica_util.h b/src/storage/column_store/ob_column_store_replica_util.h new file mode 100644 index 000000000..1e2b93eea --- /dev/null +++ b/src/storage/column_store/ob_column_store_replica_util.h @@ -0,0 +1,85 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_STORAGE_COLUMN_OB_COLUMN_STORE_REPLICA_UTIL_H_ +#define OCEANBASE_STORAGE_COLUMN_OB_COLUMN_STORE_REPLICA_UTIL_H_ + +#include "storage/ls/ob_ls.h" +#include "storage/compaction/ob_compaction_memory_context.h" +#include "share/schema/ob_table_schema.h" + +namespace oceanbase +{ +namespace storage +{ + +class ObCSReplicaUtil +{ +public: + // for construct storage schema for merge from table_schema + static int check_is_cs_replica( + const ObTableSchema &table_schema, + const ObTablet &tablet, + bool &is_cs_replica); + // is local ls cs replica + static int check_local_is_cs_replica( + const share::ObLSID &ls_id, + bool &is_cs_replica); + // is migrated tablet need convert co major sstable + static bool check_need_convert_cs_when_migration( + const ObTablet &tablet, + const ObStorageSchema& schema_on_tablet); + static int check_has_cs_replica( + const share::ObLSID &ls_id, + bool &has_column_store_replica); + // local ls need process column store replica for specific tablet + static int check_need_process_cs_replica( + const ObLS &ls, + const ObTabletID &tablet_id, + const ObStorageSchema &schema, + bool &need_process_cs_replica); + static int check_need_wait_major_convert( + const ObLS &ls, + const ObTabletID &tablet_id, + const ObTablet &tablet, + bool &need_wait_major_convert); + // whole ls replica set need process column store replica for specific tablet + static int check_replica_set_need_process_cs_replica( + const ObLS &ls, + const ObTabletID &tablet_id, + const ObStorageSchema &schema, + bool &need_process_cs_replica); +public: + static const int64_t DEFAULT_CHECK_LS_REPLICA_LOCATION_TIMEOUT = 10 * 1000 * 1000L; // 10s +}; + +class ObCSReplicaStorageSchemaGuard +{ +public: + ObCSReplicaStorageSchemaGuard(); + ~ObCSReplicaStorageSchemaGuard(); + int init(const ObTabletHandle &tablet_handle, compaction::ObCompactionMemoryContext &mem_ctx); + void reset(); + OB_INLINE bool is_inited() const { return is_inited_; }; + int load(ObStorageSchema *&storage_schema); + TO_STRING_KV(K_(is_inited), KP_(schema)); +private: + bool is_inited_; + ObStorageSchema *schema_; + DISALLOW_COPY_AND_ASSIGN(ObCSReplicaStorageSchemaGuard); +}; + + +} // namespace storage +} // namespace oceanbase + +#endif \ No newline at end of file diff --git a/src/storage/column_store/ob_column_store_util.h b/src/storage/column_store/ob_column_store_util.h index 6eaf19a9e..91612b8dd 100644 --- a/src/storage/column_store/ob_column_store_util.h +++ b/src/storage/column_store/ob_column_store_util.h @@ -28,6 +28,7 @@ typedef int64_t ObCSRowId; const ObCSRowId OB_INVALID_CS_ROW_ID = -1; const uint32_t OB_CS_INVALID_CG_IDX = INT32_MAX; const uint32_t OB_CS_VIRTUAL_CG_IDX = INT32_MAX - 1; +const uint32_t OB_CS_COLUMN_REPLICA_ROWKEY_CG_IDX = 0; OB_INLINE bool is_virtual_cg(const uint32_t cg_idx) { diff --git a/src/storage/compaction/ob_basic_tablet_merge_ctx.cpp b/src/storage/compaction/ob_basic_tablet_merge_ctx.cpp index 7381db48d..8cb89ff46 100644 --- a/src/storage/compaction/ob_basic_tablet_merge_ctx.cpp +++ b/src/storage/compaction/ob_basic_tablet_merge_ctx.cpp @@ -39,6 +39,7 @@ ObStaticMergeParam::ObStaticMergeParam(ObTabletMergeDagParam &dag_param) is_schema_changed_(false), need_parallel_minor_merge_(true), is_tenant_major_merge_(false), + is_cs_replica_(false), is_backfill_(false), merge_level_(MICRO_BLOCK_MERGE_LEVEL), merge_reason_(ObAdaptiveMergePolicy::AdaptiveMergeReason::NONE), @@ -62,7 +63,8 @@ ObStaticMergeParam::ObStaticMergeParam(ObTabletMergeDagParam &dag_param) report_(nullptr), snapshot_info_(), tx_id_(0), - multi_version_column_descs_() + multi_version_column_descs_(), + tablet_schema_guard_() { merge_scn_.set_max(); } @@ -76,6 +78,7 @@ void ObStaticMergeParam::reset() multi_version_column_descs_.reset(); ls_handle_.reset(); // ls_handle could release before tablet_handle tx_id_ = 0; + tablet_schema_guard_.reset(); } bool ObStaticMergeParam::is_valid() const @@ -1095,7 +1098,9 @@ int ObBasicTabletMergeCtx::get_medium_compaction_info() ObStorageSchema *storage_schema = nullptr; if (OB_FAIL(ObStorageSchemaUtil::alloc_storage_schema(mem_ctx_.get_allocator(), storage_schema))) { LOG_WARN("failed to alloc storage schema", K(ret)); - } else if (OB_FAIL(storage_schema->init(mem_ctx_.get_allocator(), medium_info->storage_schema_))) { + } else if (OB_FAIL(storage_schema->init(mem_ctx_.get_allocator(), medium_info->storage_schema_, + false /*skip_column_info*/, nullptr /*column_group_schema*/, + medium_info->storage_schema_.is_row_store() && medium_info->storage_schema_.is_user_data_table() && static_param_.is_cs_replica_))) { LOG_WARN("failed to init storage schema from current medium info", K(ret), K(medium_info)); ObStorageSchemaUtil::free_storage_schema(mem_ctx_.get_allocator(), storage_schema); } else { @@ -1112,7 +1117,9 @@ int ObBasicTabletMergeCtx::get_medium_compaction_info() static_param_.is_schema_changed_ = medium_info->is_schema_changed_; } static_param_.merge_reason_ = (ObAdaptiveMergePolicy::AdaptiveMergeReason)medium_info->medium_merge_reason_; - static_param_.co_major_merge_type_ = static_cast(medium_info->co_major_merge_type_); + if (!static_param_.is_cs_replica_) { + static_param_.co_major_merge_type_ = static_cast(medium_info->co_major_merge_type_); + } FLOG_INFO("get storage schema to merge", "param", get_dag_param(), KPC(medium_info)); } @@ -1184,7 +1191,7 @@ int ObBasicTabletMergeCtx::get_meta_compaction_info() } else if (OB_FAIL(tablet->get_schema_version_from_storage_schema(schema_version))){ LOG_WARN("failed to get schema version from tablet", KR(ret), KPC(tablet)); } else if (OB_FAIL(ObMediumCompactionScheduleFunc::get_table_schema_to_merge( - *schema_service, *tablet, schema_version, ObMediumCompactionInfo::MEDIUM_COMPAT_VERSION_V3, mem_ctx_.get_allocator(), *storage_schema))) { + *schema_service, *tablet, schema_version, ObMediumCompactionInfo::MEDIUM_COMPAT_VERSION_LATEST, mem_ctx_.get_allocator(), *storage_schema))) { if (OB_TABLE_IS_DELETED != ret) { LOG_WARN("failed to get table schema", KR(ret), KPC(this)); } @@ -1214,5 +1221,37 @@ int ObBasicTabletMergeCtx::get_meta_compaction_info() return ret; } +int ObBasicTabletMergeCtx::get_convert_compaction_info() +{ + int ret = OB_SUCCESS; + ObTablet *tablet = get_tablet(); + ObStorageSchema *schema_on_tablet = nullptr; + ObStorageSchema *schema_for_merge = nullptr; + if (OB_FAIL(static_param_.tablet_schema_guard_.load(schema_on_tablet))) { + LOG_WARN("failed to load schema on tablet", K(ret), KPC(tablet)); + } else if (OB_UNLIKELY(!is_convert_co_major_merge(get_merge_type()) || OB_ISNULL(schema_on_tablet) || !schema_on_tablet->is_row_store())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected static param", K(ret), KPC(schema_on_tablet), K_(static_param)); + } else if (OB_FAIL(ObStorageSchemaUtil::alloc_storage_schema(mem_ctx_.get_allocator(), schema_for_merge))) { + LOG_WARN("failed to alloc storage schema", K(ret)); + } else if (OB_FAIL(schema_for_merge->init(mem_ctx_.get_allocator(), *schema_on_tablet, + false /*skip_column_info*/, nullptr /*column_group_schema*/, true /*generate_cs_replica_cg_array*/))) { + LOG_WARN("failed to init storage schema for convert co major merge", K(ret), K(tablet), KPC(schema_on_tablet)); + } else { + static_param_.schema_ = schema_for_merge; + static_param_.schema_version_ = static_param_.schema_->schema_version_; + static_param_.data_version_ = DATA_CURRENT_VERSION; + static_param_.is_rebuild_column_store_ = true; + static_param_.is_schema_changed_ = true; // use MACRO_BLOCK_MERGE_LEVEL + static_param_.merge_reason_ = ObAdaptiveMergePolicy::REBUILD_COLUMN_GROUP; + FLOG_INFO("[CS-Replica] get storage schema to convert co merge", "param", get_dag_param(), KPC_(static_param_.schema)); + } + + if (OB_FAIL(ret) && OB_NOT_NULL(schema_for_merge)) { + ObStorageSchemaUtil::free_storage_schema(mem_ctx_.get_allocator(), schema_for_merge); + } + return ret; +} + } // namespace compaction } // namespace oceanbase diff --git a/src/storage/compaction/ob_basic_tablet_merge_ctx.h b/src/storage/compaction/ob_basic_tablet_merge_ctx.h index 822462f77..affaeb06b 100644 --- a/src/storage/compaction/ob_basic_tablet_merge_ctx.h +++ b/src/storage/compaction/ob_basic_tablet_merge_ctx.h @@ -11,6 +11,7 @@ #define OB_STORAGE_COMPACTION_BASIC_TABLET_MERGE_CTX_H_ #include "storage/compaction/ob_tablet_merge_info.h" #include "storage/compaction/ob_partition_parallel_merge_ctx.h" +#include "storage/column_store/ob_column_store_replica_util.h" #include "storage/compaction/ob_progressive_merge_helper.h" namespace oceanbase { @@ -60,9 +61,9 @@ public: "merge_reason", ObAdaptiveMergePolicy::merge_reason_to_str(merge_reason_), "co_major_merge_type", ObCOMajorMergePolicy::co_major_merge_type_to_str(co_major_merge_type_), K_(sstable_logic_seq), K_(tables_handle), K_(is_rebuild_column_store), K_(is_schema_changed), K_(is_tenant_major_merge), - K_(read_base_version), K_(merge_scn), K_(need_parallel_minor_merge), + K_(is_cs_replica), K_(read_base_version), K_(merge_scn), K_(need_parallel_minor_merge), K_(schema_version), KP_(schema), "multi_version_column_descs_cnt", multi_version_column_descs_.count(), - K_(ls_handle), K_(snapshot_info), KP_(report), K_(is_backfill)); + K_(ls_handle), K_(snapshot_info), KP_(report), K_(is_backfill), K_(tablet_schema_guard)); ObTabletMergeDagParam &dag_param_; bool is_full_merge_; // full merge or increment merge @@ -70,6 +71,7 @@ public: bool is_schema_changed_; bool need_parallel_minor_merge_; bool is_tenant_major_merge_; + bool is_cs_replica_; bool is_backfill_; ObMergeLevel merge_level_; ObAdaptiveMergePolicy::AdaptiveMergeReason merge_reason_; @@ -94,6 +96,7 @@ public: ObStorageSnapshotInfo snapshot_info_; int64_t tx_id_; common::ObSEArray multi_version_column_descs_; + storage::ObCSReplicaStorageSchemaGuard tablet_schema_guard_; // original storage schema on tablet, used only in cs replcia DISALLOW_COPY_AND_ASSIGN(ObStaticMergeParam); }; @@ -205,6 +208,7 @@ public: STATIC_PARAM_FUNC(bool, is_tenant_major_merge); STATIC_PARAM_FUNC(bool, is_full_merge); STATIC_PARAM_FUNC(bool, need_parallel_minor_merge); + STATIC_PARAM_FUNC(bool, is_cs_replica); STATIC_PARAM_FUNC(int64_t, read_base_version); STATIC_PARAM_FUNC(int64_t, ls_rebuild_seq); STATIC_PARAM_FUNC(const storage::ObTablesHandleArray &, tables_handle); @@ -266,6 +270,7 @@ protected: int get_medium_compaction_info(); // for major int swap_tablet(ObGetMergeTablesResult &get_merge_table_result); // for major int get_meta_compaction_info(); // for meta major + int get_convert_compaction_info(); // for convert co major merge static const int64_t LARGE_VOLUME_DATA_ROW_COUNT_THREASHOLD = 1000L * 1000L; // 100w static const int64_t LARGE_VOLUME_DATA_MACRO_COUNT_THREASHOLD = 300L; public: diff --git a/src/storage/compaction/ob_compaction_util.cpp b/src/storage/compaction/ob_compaction_util.cpp index d28aba42e..bace8203c 100644 --- a/src/storage/compaction/ob_compaction_util.cpp +++ b/src/storage/compaction/ob_compaction_util.cpp @@ -29,6 +29,7 @@ const static char * ObMergeTypeStr[] = { "BACKFILL_TX_MERGE", "MDS_MINI_MERGE", "MDS_MINOR_MERGE", + "CONVERT_CO_MAJOR_MERGE", "EMPTY_MERGE_TYPE" }; diff --git a/src/storage/compaction/ob_compaction_util.h b/src/storage/compaction/ob_compaction_util.h index ec073eb9d..0435537f1 100644 --- a/src/storage/compaction/ob_compaction_util.h +++ b/src/storage/compaction/ob_compaction_util.h @@ -17,7 +17,7 @@ namespace oceanbase { namespace compaction { -enum ObMergeType : uint8_t +enum ObMergeType { INVALID_MERGE_TYPE = 0, MINOR_MERGE, // minor merge, compaction several mini sstable into one larger mini sstable @@ -30,6 +30,7 @@ enum ObMergeType : uint8_t BACKFILL_TX_MERGE, MDS_MINI_MERGE, MDS_MINOR_MERGE, + CONVERT_CO_MAJOR_MERGE, // convert row store major into columnar store cg sstables // add new merge type here // fix merge_type_to_str & ObPartitionMergePolicy::get_merge_tables MERGE_TYPE_MAX @@ -48,9 +49,13 @@ inline bool is_medium_merge(const ObMergeType &merge_type) { return MEDIUM_MERGE == merge_type; } +inline bool is_convert_co_major_merge(const ObMergeType &merge_type) +{ + return CONVERT_CO_MAJOR_MERGE == merge_type; +} inline bool is_major_merge_type(const ObMergeType &merge_type) { - return is_medium_merge(merge_type) || is_major_merge(merge_type); + return is_convert_co_major_merge(merge_type) || is_medium_merge(merge_type) || is_major_merge(merge_type); } inline bool is_mini_merge(const ObMergeType &merge_type) { diff --git a/src/storage/compaction/ob_medium_compaction_func.cpp b/src/storage/compaction/ob_medium_compaction_func.cpp index 5017fde86..e2a44486f 100644 --- a/src/storage/compaction/ob_medium_compaction_func.cpp +++ b/src/storage/compaction/ob_medium_compaction_func.cpp @@ -26,6 +26,7 @@ #include "storage/ob_partition_range_spliter.h" #include "storage/compaction/ob_compaction_diagnose.h" #include "src/storage/column_store/ob_column_oriented_sstable.h" +#include "storage/column_store/ob_column_store_replica_util.h" #include "storage/tablet/ob_tablet_medium_info_reader.h" namespace oceanbase @@ -865,8 +866,7 @@ int ObMediumCompactionScheduleFunc::init_co_major_merge_type( LOG_WARN("failed to decide co major merge type", K(ret)); } else { medium_info.co_major_merge_type_ = major_merge_type; - LOG_DEBUG("chengkong debug: success to get ", - "major_merge_type", ObCOMajorMergePolicy::co_major_merge_type_to_str(major_merge_type)); + LOG_DEBUG("success to get ", "major_merge_type", ObCOMajorMergePolicy::co_major_merge_type_to_str(major_merge_type)); } return ret; @@ -1058,7 +1058,7 @@ int ObMediumCompactionScheduleFunc::get_table_schema_to_merge( } } #endif - + bool is_cs_replica = false; int64_t storage_schema_version = ObStorageSchema::STORAGE_SCHEMA_VERSION_LATEST; if (medium_compat_version < ObMediumCompactionInfo::MEDIUM_COMPAT_VERSION_V2) { @@ -1067,10 +1067,10 @@ int ObMediumCompactionScheduleFunc::get_table_schema_to_merge( storage_schema_version = ObStorageSchema::STORAGE_SCHEMA_VERSION_V2; } // for old version medium info, need generate old version schema - if (FAILEDx(storage_schema.init( - allocator, *table_schema, tablet.get_tablet_meta().compat_mode_, false/*skip_column_info*/, - storage_schema_version))) { - LOG_WARN("failed to init storage schema", K(ret), K(schema_version)); + if (FAILEDx(ObCSReplicaUtil::check_is_cs_replica(*table_schema, tablet, is_cs_replica))) { + LOG_WARN("fail to get is row store", K(ret), K(table_id), KPC(table_schema)); + } else if (OB_FAIL(storage_schema.init(allocator, *table_schema, tablet.get_tablet_meta().compat_mode_, false/*skip_column_info*/, storage_schema_version, is_cs_replica))) { + LOG_WARN("failed to init storage schema", K(ret), K(schema_version), K(tablet), KPC(table_schema)); } else { LOG_INFO("get schema to merge", K(tablet_id), K(table_id), K(schema_version), K(save_schema_version), K(storage_schema), K(*reinterpret_cast(table_schema))); @@ -1224,105 +1224,135 @@ int ObMediumCompactionScheduleFunc::init_tablet_filters(share::ObTabletReplicaFi return ret; } -int ObMediumCompactionScheduleFunc::check_medium_checksum( + int ObMediumCompactionScheduleFunc::check_tablet_checksum( const ObIArray &checksum_items, + const ObLSColumnReplicaCache &ls_cs_replica_cache, + const int64_t start_idx, + const int64_t end_idx, + const bool is_medium_checker, ObIArray &error_pairs, - int64_t &item_idx, int &check_ret) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - check_ret = OB_SUCCESS; - int64_t items_cnt = checksum_items.count(); - ObTabletReplicaChecksumItem prev_item; - ObTabletReplicaChecksumItem curr_item; - while (OB_SUCC(ret) && item_idx < items_cnt) { - curr_item.reset(); - if (OB_FAIL(curr_item.assign(checksum_items.at(item_idx)))) { - LOG_WARN("fail to assign tablet replica checksum item", KR(ret), K(item_idx), "item", checksum_items.at(item_idx)); - } else { - if (prev_item.is_key_valid()) { - if (curr_item.is_same_tablet(prev_item)) { // same tablet - if (OB_TMP_FAIL(curr_item.verify_checksum(prev_item))) { - LOG_DBA_ERROR(OB_CHECKSUM_ERROR, "msg", "checksum error in tablet replica checksum", KR(tmp_ret), - K(curr_item), K(prev_item)); - if (OB_SUCCESS == check_ret) { - if (OB_TMP_FAIL(error_pairs.push_back(ObTabletLSPair(curr_item.tablet_id_, curr_item.ls_id_)))) { - LOG_WARN("fail to push back error pair", K(tmp_ret), "tablet_id", curr_item.tablet_id_, "ls_id", curr_item.ls_id_); - } - check_ret = OB_CHECKSUM_ERROR; + if (start_idx >= end_idx) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid idx range for check tablet checksums", K(ret), K(start_idx), K(end_idx)); + } else if (start_idx + 1 == end_idx) { + } else { + const ObTabletReplicaChecksumItem *prev_item = nullptr; + ObTabletDataChecksumChecker data_checksum_checker; + ObLSID prev_error_ls_id; + for (int64_t idx = start_idx; OB_SUCC(ret) && idx < end_idx; ++idx) { + const ObTabletReplicaChecksumItem &curr_item = checksum_items.at(idx); + bool is_cs_replica = false; + ObLSReplicaUniItem ls_item(curr_item.ls_id_, curr_item.server_); + if (OB_FAIL(ls_cs_replica_cache.check_is_cs_replica(ls_item, is_cs_replica))) { + LOG_WARN("fail to check is column replica", K(ret), K(ls_item), K(ls_cs_replica_cache)); + } else if (OB_ISNULL(prev_item)) { + } else if (!curr_item.is_same_tablet(*prev_item)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("not continuous same tablet id", K(ret), K(curr_item), KPC(prev_item)); + } else if (OB_TMP_FAIL(data_checksum_checker.check_data_checksum(curr_item, is_cs_replica)) + || OB_TMP_FAIL(curr_item.verify_column_checksum(*prev_item))) { + if (OB_CHECKSUM_ERROR == tmp_ret) { + LOG_DBA_ERROR(OB_CHECKSUM_ERROR, "msg", "checksum error in tablet replica checksum", KR(tmp_ret), + K(curr_item), KPC(prev_item), K(is_cs_replica), K(data_checksum_checker)); + check_ret = OB_CHECKSUM_ERROR; + if (curr_item.ls_id_ != prev_error_ls_id) { + prev_error_ls_id = curr_item.ls_id_; + if (OB_TMP_FAIL(error_pairs.push_back(ObTabletLSPair(curr_item.tablet_id_, curr_item.ls_id_)))) { + LOG_WARN("fail to push back error pair", K(tmp_ret), "tablet_id", curr_item.tablet_id_, "ls_id", curr_item.ls_id_); } } -#ifdef ERRSIM - if (OB_SUCC(ret)) { - ret = OB_E(EventTable::EN_MEDIUM_REPLICA_CHECKSUM_ERROR) OB_SUCCESS; - if (OB_FAIL(ret)) { - STORAGE_LOG(INFO, "ERRSIM EN_MEDIUM_REPLICA_CHECKSUM_ERROR", K(ret), - "tablet_id", curr_item.tablet_id_, "ls_id", curr_item.ls_id_); - if (OB_TMP_FAIL(error_pairs.push_back(ObTabletLSPair(curr_item.tablet_id_, curr_item.ls_id_)))) { - LOG_WARN("fail to push back error pair", K(tmp_ret), "tablet_id", curr_item.tablet_id_, "ls_id", curr_item.ls_id_); - } - check_ret = OB_CHECKSUM_ERROR; - } - } -#endif } else { - break; + ret = tmp_ret; + LOG_WARN("unexpected error in tablet replica checksum", KR(ret), K(curr_item), KPC(prev_item)); } - } else if (OB_FAIL(prev_item.assign(curr_item))) { - LOG_WARN("fail to assign tablet replica checksum item", KR(ret), K(item_idx), K(curr_item)); +#ifdef ERRSIM + if (is_medium_checker && OB_SUCC(ret)) { + ret = OB_E(EventTable::EN_MEDIUM_REPLICA_CHECKSUM_ERROR) OB_SUCCESS; + if (OB_FAIL(ret)) { + STORAGE_LOG(INFO, "ERRSIM EN_MEDIUM_REPLICA_CHECKSUM_ERROR", K(ret), "tablet_id", curr_item.tablet_id_, "ls_id", curr_item.ls_id_); + if (OB_TMP_FAIL(error_pairs.push_back(ObTabletLSPair(curr_item.tablet_id_, curr_item.ls_id_)))) { + LOG_WARN("fail to push back error pair", K(tmp_ret), "tablet_id", curr_item.tablet_id_, "ls_id", curr_item.ls_id_); + } + check_ret = OB_CHECKSUM_ERROR; + } + } +#endif } - ++item_idx; + prev_item = &curr_item; } - } // end for while + } return ret; } -int ObMediumCompactionScheduleFunc::batch_check_medium_checksum( - const ObIArray &checksum_items) +int ObMediumCompactionScheduleFunc::check_replica_checksum_items( + const ObIArray &checksum_items, + const ObLSColumnReplicaCache &ls_cs_replica_cache, + const bool is_medium_checker) { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - int check_ret = OB_SUCCESS; - int64_t pair_idx = 0; - int64_t item_idx = 0; - int64_t items_cnt = checksum_items.count(); - int64_t affected_rows = 0; - ObSEArray error_pairs; - while (OB_SUCC(ret) && item_idx < items_cnt) { - const ObTabletReplicaChecksumItem &tmp_item = checksum_items.at(item_idx); - const ObTabletID &tablet_id = tmp_item.tablet_id_; - const ObLSID &ls_id = tmp_item.ls_id_; - if (OB_FAIL(check_medium_checksum(checksum_items, error_pairs, item_idx, check_ret))) { - LOG_WARN("failed to check medium checksum", K(ret), K(item_idx)); - } else if (OB_SUCCESS == check_ret) { - ObLSHandle ls_handle; - ObTabletHandle unused_handle; - if (OB_TMP_FAIL((MTL(storage::ObLSService *)->get_ls(ls_id, ls_handle, ObLSGetMod::COMPACT_MODE)))) { - if (OB_LS_NOT_EXIST == tmp_ret) { - LOG_TRACE("ls not exist", K(tmp_ret), K(ls_id)); - } else { - LOG_WARN("failed to get ls", K(tmp_ret), K(ls_id)); - } - } else if (OB_TMP_FAIL(ls_handle.get_ls()->update_medium_compaction_info(tablet_id, unused_handle))) { - LOG_WARN("failed to update medium compaction info", K(tmp_ret), K(ls_id), K(tablet_id)); + if (checksum_items.empty()) { + } else { + int tmp_ret = OB_SUCCESS; + int check_ret = OB_SUCCESS; + int64_t affected_rows = 0; + const int64_t count = checksum_items.count(); + int64_t start_idx = 0; + int64_t end_idx = 0; + ObTabletID tablet_id = checksum_items.at(0).tablet_id_; + ObLSID ls_id = checksum_items.at(0).ls_id_; + ObSEArray error_pairs; + error_pairs.set_attr(ObMemAttr(MTL_ID(), "MedCkmErrs")); + + // [start_idx, end_idx share same tablet_id + while (OB_SUCC(ret) && end_idx < count) { + while (end_idx < count && tablet_id == checksum_items.at(end_idx).tablet_id_) { + end_idx++; + } + if (OB_FAIL(check_tablet_checksum(checksum_items, ls_cs_replica_cache, start_idx, end_idx, true /*is_medium_checker*/, error_pairs, check_ret))) { } else { - FLOG_INFO("finish check medium compaction info", K(tmp_ret), K(ls_id), K(tablet_id)); + // update medium compaction info + if (is_medium_checker && OB_SUCCESS == check_ret) { + ObLSHandle ls_handle; + ObTabletHandle unused_handle; + if (OB_TMP_FAIL((MTL(storage::ObLSService *)->get_ls(ls_id, ls_handle, ObLSGetMod::COMPACT_MODE)))) { + if (OB_LS_NOT_EXIST == tmp_ret) { + LOG_TRACE("ls not exist", K(tmp_ret), K(ls_id)); + } else { + LOG_WARN("failed to get ls", K(tmp_ret), K(ls_id)); + } + } else if (OB_TMP_FAIL(ls_handle.get_ls()->update_medium_compaction_info(tablet_id, unused_handle))) { + LOG_WARN("failed to update medium compaction info", K(tmp_ret), K(ls_id), K(tablet_id)); + } else { + FLOG_INFO("finish check medium compaction info", K(tmp_ret), K(ls_id), K(tablet_id)); + } + } + + // refresh sliding windows + if (OB_SUCC(ret) && end_idx < count) { + start_idx = end_idx; + tablet_id = checksum_items.at(end_idx).tablet_id_; + ls_id = checksum_items.at(end_idx).ls_id_; + check_ret = OB_SUCCESS; + } + } + } // end while + + if (!error_pairs.empty()) { + if (OB_TMP_FAIL(ObTabletMetaTableCompactionOperator::batch_set_info_status(MTL_ID(), error_pairs, affected_rows))) { + LOG_WARN("fail to batch set info status", KR(tmp_ret)); + } else { + LOG_INFO("succ to batch set info status", K(ret), K(affected_rows), K(error_pairs)); } } - ++pair_idx; - check_ret = OB_SUCCESS; - } // end for while - if (!error_pairs.empty()) { - if (OB_TMP_FAIL(ObTabletMetaTableCompactionOperator::batch_set_info_status(MTL_ID(), error_pairs, affected_rows))) { - LOG_WARN("fail to batch set info status", KR(tmp_ret)); - } else { - LOG_INFO("succ to batch set info status", K(ret), K(affected_rows), K(error_pairs)); + + if (is_medium_checker && affected_rows > 0) { + MTL(ObTenantTabletScheduler*)->update_error_tablet_cnt(affected_rows); } } - if (affected_rows > 0) { - MTL(ObTenantTabletScheduler*)->update_error_tablet_cnt(affected_rows); - } return ret; } @@ -1331,7 +1361,8 @@ int ObMediumCompactionScheduleFunc::batch_check_medium_finish( const hash::ObHashMap &ls_info_map, ObIArray &finish_tablet_ls_infos, const ObIArray &tablet_ls_infos, - ObCompactionTimeGuard &time_guard) + ObCompactionTimeGuard &time_guard, + const share::ObLSColumnReplicaCache &ls_cs_replica_cache) { int ret = OB_SUCCESS; if (tablet_ls_infos.empty()) { @@ -1348,8 +1379,8 @@ int ObMediumCompactionScheduleFunc::batch_check_medium_finish( MTL_ID(), finish_tablet_ls_infos, checksum_items))) { LOG_WARN("failed to get tablet checksum", K(ret)); } else if (FALSE_IT(time_guard.click(ObCompactionScheduleTimeGuard::SEARCH_CHECKSUM))) { - } else if (OB_FAIL(batch_check_medium_checksum(checksum_items))) { - LOG_WARN("failed to check medium tablets checksum", K(ret)); + } else if (OB_FAIL(check_replica_checksum_items(checksum_items, ls_cs_replica_cache, true /*is_medium_checker*/))) { + LOG_WARN("fail to check replica checksum items for medium checker", K(ret)); } else if (FALSE_IT(time_guard.click(ObCompactionScheduleTimeGuard::CHECK_CHECKSUM))) { } } diff --git a/src/storage/compaction/ob_medium_compaction_func.h b/src/storage/compaction/ob_medium_compaction_func.h index 2b210bd83..607354528 100644 --- a/src/storage/compaction/ob_medium_compaction_func.h +++ b/src/storage/compaction/ob_medium_compaction_func.h @@ -14,6 +14,7 @@ #include "storage/compaction/ob_partition_merge_policy.h" #include "share/tablet/ob_tablet_filter.h" #include "share/ob_tablet_meta_table_compaction_operator.h" +#include "share/ob_tablet_replica_checksum_operator.h" #include "storage/tablet/ob_tablet.h" #include "storage/compaction/ob_tenant_tablet_scheduler.h" #include "storage/compaction/ob_tenant_medium_checker.h" @@ -86,7 +87,12 @@ public: const hash::ObHashMap &ls_info_map, ObIArray &finish_tablet_ls_infos, const ObIArray &tablet_ls_infos, - ObCompactionTimeGuard &time_guard); + ObCompactionTimeGuard &time_guard, + const share::ObLSColumnReplicaCache &ls_cs_replica_cache); + static int check_replica_checksum_items( + const ObIArray &checksum_items, + const ObLSColumnReplicaCache &ls_cs_replica_cache, + const bool is_medium_checker); int schedule_next_medium_for_leader( const int64_t major_snapshot, @@ -132,13 +138,14 @@ protected: const hash::ObHashMap &ls_info_map, bool &merge_finish); static int init_tablet_filters(share::ObTabletReplicaFilterHolder &filters); - static int check_medium_checksum( + static int check_tablet_checksum( const ObIArray &checksum_items, + const ObLSColumnReplicaCache &ls_cs_replica_cache, + const int64_t start_idx, + const int64_t end_idx, + const bool is_medium_checker, ObIArray &error_pairs, - int64_t &item_idx, int &check_ret); - static int batch_check_medium_checksum( - const ObIArray &checksum_items); int choose_medium_snapshot( const int64_t max_sync_medium_scn, ObMediumCompactionInfo &medium_info, diff --git a/src/storage/compaction/ob_partition_merge_policy.cpp b/src/storage/compaction/ob_partition_merge_policy.cpp index 0d765b185..4cb2f19f9 100644 --- a/src/storage/compaction/ob_partition_merge_policy.cpp +++ b/src/storage/compaction/ob_partition_merge_policy.cpp @@ -63,7 +63,8 @@ ObPartitionMergePolicy::GetMergeTables ObPartitionMergePolicy::get_merge_tables[ ObPartitionMergePolicy::not_support_merge_type, ObPartitionMergePolicy::not_support_merge_type, ObPartitionMergePolicy::not_support_merge_type, - ObPartitionMergePolicy::get_mds_merge_tables + ObPartitionMergePolicy::get_mds_merge_tables, + ObPartitionMergePolicy::get_convert_co_major_merge_tables }; @@ -232,6 +233,44 @@ int ObPartitionMergePolicy::get_mds_merge_tables( return ret; } +int ObPartitionMergePolicy::get_convert_co_major_merge_tables( + const storage::ObGetMergeTablesParam ¶m, + storage::ObLS &ls, + const storage::ObTablet &tablet, + storage::ObGetMergeTablesResult &result) +{ + int ret = OB_SUCCESS; + ObSSTable *base_table = nullptr; + result.reset(); + result.merge_version_ = param.merge_version_; + ObTabletMemberWrapper table_store_wrapper; + if (OB_FAIL(tablet.fetch_table_store(table_store_wrapper))) { + LOG_WARN("fail to fetch table store", K(ret)); + } else if (OB_UNLIKELY(!table_store_wrapper.get_member()->is_valid() + || !param.is_valid() + || !is_convert_co_major_merge(param.merge_type_))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get invalid argument", K(ret), KPC(table_store_wrapper.get_member()), K(param)); + } else if (OB_ISNULL(base_table = static_cast( + table_store_wrapper.get_member()->get_major_sstables().get_boundary_table(true/*last*/)))) { + ret = OB_ENTRY_NOT_EXIST; + LOG_ERROR("major sstable not exist", K(ret), KPC(table_store_wrapper.get_member())); + } else if (OB_FAIL(result.handle_.add_sstable(base_table, table_store_wrapper.get_meta_handle()))) { + LOG_WARN("failed to add base_table to result", K(ret)); + } else if (OB_UNLIKELY(base_table->get_snapshot_version() != param.merge_version_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("convert co major merge should not change major snapshot version", K(ret), KPC(base_table), K(param), K(tablet)); + } else { + result.version_range_.base_version_ = 0; + result.version_range_.multi_version_start_ = tablet.get_multi_version_start(); + result.version_range_.snapshot_version_ = param.merge_version_; + if (OB_FAIL(get_multi_version_start(param.merge_type_, ls, tablet, result.version_range_, result.snapshot_info_))) { + LOG_WARN("failed to get multi version_start", K(ret)); + } + } + return ret; +} + int ObPartitionMergePolicy::get_result_by_snapshot( ObTablet &tablet, const int64_t snapshot, diff --git a/src/storage/compaction/ob_partition_merge_policy.h b/src/storage/compaction/ob_partition_merge_policy.h index 1848109d7..e2baca639 100644 --- a/src/storage/compaction/ob_partition_merge_policy.h +++ b/src/storage/compaction/ob_partition_merge_policy.h @@ -68,13 +68,16 @@ public: storage::ObLS &ls, const storage::ObTablet &tablet, storage::ObGetMergeTablesResult &result); - static int get_mds_merge_tables( const storage::ObGetMergeTablesParam ¶m, storage::ObLS &ls, const storage::ObTablet &tablet, storage::ObGetMergeTablesResult &result); - + static int get_convert_co_major_merge_tables( + const storage::ObGetMergeTablesParam ¶m, + storage::ObLS &ls, + const storage::ObTablet &tablet, + storage::ObGetMergeTablesResult &result); static int not_support_merge_type( const storage::ObGetMergeTablesParam ¶m, storage::ObLS &ls, diff --git a/src/storage/compaction/ob_tenant_medium_checker.cpp b/src/storage/compaction/ob_tenant_medium_checker.cpp index ca9749e23..3b05db2ff 100644 --- a/src/storage/compaction/ob_tenant_medium_checker.cpp +++ b/src/storage/compaction/ob_tenant_medium_checker.cpp @@ -122,11 +122,10 @@ int ObTenantMediumChecker::refresh_ls_status() int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; lib::ObMutexGuard guard(lock_); - if (OB_TMP_FAIL(ls_locality_cache_.refresh_ls_locality(true/*force_refresh*/))) { - LOG_WARN("failed to refresh ls locality", K(tmp_ret)); - } common::ObSEArray ls_ids; - if (OB_FAIL(MTL(ObLSService *)->get_ls_ids(ls_ids))) { + if (OB_FAIL(ls_locality_cache_.refresh_ls_locality(true/*force_refresh*/))) { + LOG_WARN("failed to refresh ls locality", K(ret)); + } else if (OB_FAIL(MTL(ObLSService *)->get_ls_ids(ls_ids))) { LOG_WARN("failed to get all ls id", K(ret)); } else { ls_info_map_.reuse(); @@ -155,19 +154,18 @@ int ObTenantMediumChecker::refresh_ls_status() int ObTenantMediumChecker::check_ls_status(const share::ObLSID &ls_id, bool &is_leader, bool need_check) { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; is_leader = false; if (need_check && CHECK_LS_LOCALITY_INTERVAL < ObTimeUtility::fast_current_time() - last_check_timestamp_) { - if (OB_TMP_FAIL(refresh_ls_status())) { - LOG_WARN("failed to refresh ls locality", K(tmp_ret)); + if (OB_FAIL(refresh_ls_status())) { + LOG_WARN("failed to refresh ls locality", K(ret)); } else { last_check_timestamp_ = ObTimeUtility::fast_current_time(); } } lib::ObMutexGuard guard(lock_); ObLSInfo ls_info; - if (OB_FAIL(ls_info_map_.get_refactored(ls_id, ls_info))) { + if (FAILEDx(ls_info_map_.get_refactored(ls_id, ls_info))) { if (OB_HASH_NOT_EXIST != ret) { LOG_WARN("fail to get map", K(ret), K(ls_id)); } else { @@ -219,16 +217,12 @@ int ObTenantMediumChecker::check_medium_finish_schedule() if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ObTenantMediumChecker is not inited", K(ret)); + } else if (OB_FAIL(ls_locality_cache_.refresh_ls_locality(false /*force_refresh*/))) { + LOG_WARN("failed to refresh ls locality"); + ADD_COMMON_SUSPECT_INFO(MEDIUM_MERGE, share::ObDiagnoseTabletType::TYPE_MEDIUM_MERGE, + SUSPECT_FAILED_TO_REFRESH_LS_LOCALITY, ret); } else { - // refresh ls locality cache - if (OB_TMP_FAIL(ls_locality_cache_.refresh_ls_locality(false /*force_refresh*/))) { - LOG_WARN("failed to refresh ls locality", K(tmp_ret)); - ADD_COMMON_SUSPECT_INFO(MEDIUM_MERGE, share::ObDiagnoseTabletType::TYPE_MEDIUM_MERGE, - SUSPECT_FAILED_TO_REFRESH_LS_LOCALITY, tmp_ret); - } else { - DEL_SUSPECT_INFO(MEDIUM_MERGE, UNKNOW_LS_ID, UNKNOW_TABLET_ID, ObDiagnoseTabletType::TYPE_MEDIUM_MERGE); - } - + DEL_SUSPECT_INFO(MEDIUM_MERGE, UNKNOW_LS_ID, UNKNOW_TABLET_ID, ObDiagnoseTabletType::TYPE_MEDIUM_MERGE); TabletLSArray tablet_ls_infos; tablet_ls_infos.set_attr(ObMemAttr(MTL_ID(), "CheckInfos")); TabletLSArray batch_tablet_ls_infos; @@ -319,7 +313,7 @@ int ObTenantMediumChecker::check_medium_finish( ObCompactionScheduleTimeGuard time_guard; stat.filter_cnt_ += (end_idx - start_idx - check_tablet_ls_infos.count()); if (FAILEDx(ObMediumCompactionScheduleFunc::batch_check_medium_finish( - ls_info_map_, finish_tablet_ls_infos, check_tablet_ls_infos, time_guard))) { + ls_info_map_, finish_tablet_ls_infos, check_tablet_ls_infos, time_guard, ls_locality_cache_.get_cs_replica_cache()))) { LOG_WARN("failed to batch check medium finish", K(ret), K(tablet_ls_infos.count()), K(check_tablet_ls_infos.count()), K(tablet_ls_infos), K(check_tablet_ls_infos)); stat.fail_cnt_ += check_tablet_ls_infos.count(); diff --git a/src/storage/compaction/ob_tenant_tablet_scheduler.cpp b/src/storage/compaction/ob_tenant_tablet_scheduler.cpp index 1a34973f4..24d518f84 100644 --- a/src/storage/compaction/ob_tenant_tablet_scheduler.cpp +++ b/src/storage/compaction/ob_tenant_tablet_scheduler.cpp @@ -1037,10 +1037,11 @@ int ObTenantTabletScheduler::schedule_merge_dag( const ObLSID &ls_id, const storage::ObTablet &tablet, const ObMergeType merge_type, - const int64_t &merge_snapshot_version) + const int64_t &merge_snapshot_version, + const ObDagId *dag_net_id /*= nullptr*/) { int ret = OB_SUCCESS; - if (is_major_merge_type(merge_type) && !tablet.is_row_store()) { + if (is_major_merge_type(merge_type) && (!tablet.is_row_store() || is_convert_co_major_merge(merge_type))) { ObCOMergeDagParam param; param.ls_id_ = ls_id; param.tablet_id_ = tablet.get_tablet_meta().tablet_id_; @@ -1048,6 +1049,9 @@ int ObTenantTabletScheduler::schedule_merge_dag( param.merge_version_ = merge_snapshot_version; param.compat_mode_ = tablet.get_tablet_meta().compat_mode_; param.transfer_seq_ = tablet.get_tablet_meta().transfer_info_.transfer_seq_; + if (OB_UNLIKELY(nullptr != dag_net_id)) { + param.dag_net_id_ = *dag_net_id; + } if (OB_FAIL(compaction::ObScheduleDagFunc::schedule_tablet_co_merge_dag_net(param))) { if (OB_EAGAIN != ret && OB_SIZE_OVERFLOW != ret) { LOG_WARN("failed to schedule tablet merge dag", K(ret)); @@ -1066,6 +1070,27 @@ int ObTenantTabletScheduler::schedule_merge_dag( LOG_WARN("failed to schedule tablet merge dag", K(ret)); } } + FLOG_INFO("schedule merge dag", K(ret), K(param), K(tablet.is_row_store())); + } + return ret; +} + +int ObTenantTabletScheduler::schedule_convert_co_merge_dag_net( + const ObLSID &ls_id, + const ObTablet &tablet, + const int64_t retry_times, + const ObDagId& curr_dag_net_id) +{ + int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(compaction::ObTenantTabletScheduler::schedule_merge_dag( + ls_id, tablet, compaction::ObMergeType::CONVERT_CO_MAJOR_MERGE, tablet.get_last_major_snapshot_version(), &curr_dag_net_id))) { + if (OB_SIZE_OVERFLOW != tmp_ret && OB_EAGAIN != tmp_ret) { + ret = tmp_ret; + LOG_WARN("failed to schedule co merge dag net for cs replica", K(ret), K(ls_id), "tablet_id", tablet.get_tablet_id()); + } + } else { + LOG_INFO("[CS-Replica] schedule COMergeDagNet to convert row store to column store", K(retry_times), K(ls_id), "tablet_id", tablet.get_tablet_id(), K(curr_dag_net_id)); } return ret; } diff --git a/src/storage/compaction/ob_tenant_tablet_scheduler.h b/src/storage/compaction/ob_tenant_tablet_scheduler.h index 30db53926..5764bc50e 100644 --- a/src/storage/compaction/ob_tenant_tablet_scheduler.h +++ b/src/storage/compaction/ob_tenant_tablet_scheduler.h @@ -257,7 +257,13 @@ public: const share::ObLSID &ls_id, const storage::ObTablet &tablet, const ObMergeType merge_type, - const int64_t &merge_snapshot_version); + const int64_t &merge_snapshot_version, + const ObDagId *dag_net_id = nullptr); + static int schedule_convert_co_merge_dag_net( + const ObLSID &ls_id, + const ObTablet &tablet, + const int64_t retry_times, + const ObDagId& curr_dag_net_id); static int schedule_tablet_ddl_major_merge( ObLSHandle &ls_handle, ObTabletHandle &tablet_handle); diff --git a/src/storage/ddl/ob_complement_data_task.cpp b/src/storage/ddl/ob_complement_data_task.cpp index 7ce784563..617012098 100644 --- a/src/storage/ddl/ob_complement_data_task.cpp +++ b/src/storage/ddl/ob_complement_data_task.cpp @@ -2059,7 +2059,7 @@ int ObLocalScan::construct_access_param( } else if (!has_all_cg) { for (int64_t i = 0; i < col_params_.count(); i++) { int32_t tmp_cg_idx = -1; - if (OB_FAIL(data_table_schema.get_column_group_index(*col_params_.at(i), tmp_cg_idx))) { + if (OB_FAIL(data_table_schema.get_column_group_index(*col_params_.at(i), false /*need_calculate_cg_idx*/, tmp_cg_idx))) { LOG_WARN("fail to get column group idx", K(ret), K(data_table_schema)); } else if (OB_FAIL(cg_idxs.push_back(tmp_cg_idx))) { LOG_WARN("fail to push back cg idx", K(ret)); diff --git a/src/storage/ddl/ob_ddl_clog.cpp b/src/storage/ddl/ob_ddl_clog.cpp index fecaeda8f..8927f155b 100644 --- a/src/storage/ddl/ob_ddl_clog.cpp +++ b/src/storage/ddl/ob_ddl_clog.cpp @@ -234,6 +234,8 @@ int ObDDLMacroBlockClogCb::on_success() if (is_data_buffer_freed_) { LOG_INFO("data buffer is freed, do not need to callback"); } else if (OB_FAIL(ret)) { + } else if (redo_info_.with_cs_replica_ && redo_info_.table_key_.is_column_store_sstable()) { + LOG_TRACE("[CS-Replica] skip replay cs replica redo clog in leader", K(ret), K_(redo_info)); } else if (OB_FAIL(macro_block.block_handle_.set_block_id(macro_block_id_))) { LOG_WARN("set macro block id failed", K(ret), K(macro_block_id_)); } else { @@ -406,7 +408,8 @@ int ObDDLStartLog::init( const uint64_t data_format_version, const int64_t execution_id, const ObDirectLoadType direct_load_type, - const ObTabletID &lob_meta_tablet_id) + const ObTabletID &lob_meta_tablet_id, + const bool with_cs_replica) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!table_key.is_valid() || execution_id < 0 || data_format_version <= 0 || !is_valid_direct_load(direct_load_type) @@ -419,7 +422,7 @@ int ObDDLStartLog::init( execution_id_ = execution_id; direct_load_type_ = direct_load_type; lob_meta_tablet_id_ = lob_meta_tablet_id; - with_cs_replica_ = false; // TODO(chengkong): placeholder for column store replica feature + with_cs_replica_ = with_cs_replica; } return ret; } diff --git a/src/storage/ddl/ob_ddl_clog.h b/src/storage/ddl/ob_ddl_clog.h index b786fd5bc..0e685fb38 100644 --- a/src/storage/ddl/ob_ddl_clog.h +++ b/src/storage/ddl/ob_ddl_clog.h @@ -196,13 +196,15 @@ public: const uint64_t data_format_version, const int64_t execution_id, const ObDirectLoadType direct_load_type, - const ObTabletID &lob_meta_tablet_id); + const ObTabletID &lob_meta_tablet_id, + const bool with_cs_replica); bool is_valid() const { return table_key_.is_valid() && data_format_version_ >= 0 && execution_id_ >= 0 && is_valid_direct_load(direct_load_type_); } ObITable::TableKey get_table_key() const { return table_key_; } uint64_t get_data_format_version() const { return data_format_version_; } int64_t get_execution_id() const { return execution_id_; } ObDirectLoadType get_direct_load_type() const { return direct_load_type_; } const ObTabletID &get_lob_meta_tablet_id() const { return lob_meta_tablet_id_; } + bool get_with_cs_replica() const { return with_cs_replica_; } TO_STRING_KV(K_(table_key), K_(data_format_version), K_(execution_id), K_(direct_load_type), K_(lob_meta_tablet_id), K_(with_cs_replica)); private: ObITable::TableKey table_key_; // use table type to distinguish column store, column group id is valid diff --git a/src/storage/ddl/ob_ddl_merge_task.cpp b/src/storage/ddl/ob_ddl_merge_task.cpp index dc31c3f4f..5ab22f679 100644 --- a/src/storage/ddl/ob_ddl_merge_task.cpp +++ b/src/storage/ddl/ob_ddl_merge_task.cpp @@ -914,6 +914,13 @@ int ObTabletDDLUtil::update_ddl_table_store( table_store_param.ddl_info_.data_format_version_ = ddl_param.data_format_version_; table_store_param.ddl_info_.ddl_commit_scn_ = ddl_param.commit_scn_; table_store_param.ddl_info_.ddl_checkpoint_scn_ = sstable->is_ddl_dump_sstable() ? sstable->get_end_scn() : ddl_param.commit_scn_; + if (ddl_param.table_key_.is_ddl_dump_sstable()) { + // data is not complete, now update ddl table store only for reducing count of ddl dump sstable. + table_store_param.ddl_info_.ddl_table_type_ = ddl_param.table_key_.table_type_; + } else { + // data is complete, make ddl table type to major sstable instead of ddl dump sstable (mark ddl finished). + table_store_param.ddl_info_.ddl_table_type_ = ddl_param.table_key_.is_co_sstable() ? ObITable::COLUMN_ORIENTED_SSTABLE : ObITable::MAJOR_SSTABLE; + } } else { // incremental direct load table_store_param.clog_checkpoint_scn_ = sstable->get_end_scn(); table_store_param.need_check_transfer_seq_ = true; diff --git a/src/storage/ddl/ob_ddl_redo_log_writer.cpp b/src/storage/ddl/ob_ddl_redo_log_writer.cpp index 2b2ce618c..c1882c66e 100644 --- a/src/storage/ddl/ob_ddl_redo_log_writer.cpp +++ b/src/storage/ddl/ob_ddl_redo_log_writer.cpp @@ -1175,7 +1175,7 @@ int ObDDLRedoLogWriter::write_start_log( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(table_key), K(execution_id), K(data_format_version), K(direct_load_type)); } else if (OB_FAIL(log.init(table_key, data_format_version, execution_id, direct_load_type, - lob_kv_mgr_handle.is_valid() ? lob_kv_mgr_handle.get_obj()->get_tablet_id() : ObTabletID()))) { + lob_kv_mgr_handle.is_valid() ? lob_kv_mgr_handle.get_obj()->get_tablet_id() : ObTabletID(), direct_load_mgr_handle.get_obj()->need_process_cs_replica()))) { LOG_WARN("fail to init DDLStartLog", K(ret), K(table_key), K(execution_id), K(data_format_version)); } else if (OB_FAIL(MTL(ObLSService *)->get_ls(ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { LOG_WARN("get ls failed", K(ret), K(ls_id_)); @@ -1528,7 +1528,8 @@ ObDDLRedoLogWriter::~ObDDLRedoLogWriter() ObDDLRedoLogWriterCallback::ObDDLRedoLogWriterCallback() : is_inited_(false), redo_info_(), block_type_(ObDDLMacroBlockType::DDL_MB_INVALID_TYPE), table_key_(), macro_block_id_(), task_id_(0), data_format_version_(0), - direct_load_type_(DIRECT_LOAD_INVALID), row_id_offset_(-1) + direct_load_type_(DIRECT_LOAD_INVALID), row_id_offset_(-1), + with_cs_replica_(false), need_submit_io_(true) { } @@ -1545,7 +1546,9 @@ int ObDDLRedoLogWriterCallback::init(const share::ObLSID &ls_id, const share::SCN &start_scn, const uint64_t data_format_version, const ObDirectLoadType direct_load_type, - const int64_t row_id_offset/*=-1*/) + const int64_t row_id_offset/*=-1*/, + const bool with_cs_replica/*=false*/, + const bool need_submit_io/*=true*/) { int ret = OB_SUCCESS; ObLS *ls = nullptr; @@ -1574,6 +1577,8 @@ int ObDDLRedoLogWriterCallback::init(const share::ObLSID &ls_id, data_format_version_ = data_format_version; direct_load_type_ = direct_load_type; row_id_offset_ = row_id_offset; + with_cs_replica_ = with_cs_replica; + need_submit_io_ = need_submit_io; is_inited_ = true; } return ret; @@ -1592,6 +1597,8 @@ void ObDDLRedoLogWriterCallback::reset() data_format_version_ = 0; direct_load_type_ = DIRECT_LOAD_INVALID; row_id_offset_ = -1; + with_cs_replica_ = false; + need_submit_io_ = true; } bool ObDDLRedoLogWriterCallback::is_column_group_info_valid() const @@ -1609,9 +1616,9 @@ int ObDDLRedoLogWriterCallback::write(ObMacroBlockHandle ¯o_handle, if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ObDDLRedoLogWriterCallback is not inited", K(ret)); - } else if (OB_UNLIKELY(!macro_handle.is_valid() || !logic_id.is_valid() || nullptr == buf || row_count <= 0)) { + } else if (OB_UNLIKELY((!macro_handle.is_valid() && need_submit_io_) || !logic_id.is_valid() || nullptr == buf || row_count <= 0)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(macro_handle), K(logic_id), KP(buf), K(row_count)); + LOG_WARN("invalid argument", K(ret), K(macro_handle), K_(need_submit_io), K(logic_id), KP(buf), K(row_count)); } else { macro_block_id_ = macro_handle.get_macro_id(); redo_info_.table_key_ = table_key_; @@ -1621,6 +1628,7 @@ int ObDDLRedoLogWriterCallback::write(ObMacroBlockHandle ¯o_handle, redo_info_.start_scn_ = start_scn_; redo_info_.data_format_version_ = data_format_version_; redo_info_.type_ = direct_load_type_; + redo_info_.with_cs_replica_ = with_cs_replica_; redo_info_.parallel_cnt_ = 0; // TODO @zhuoran.zzr, place holder for shared storage redo_info_.cg_cnt_ = 0; diff --git a/src/storage/ddl/ob_ddl_redo_log_writer.h b/src/storage/ddl/ob_ddl_redo_log_writer.h index ff20b411a..648650679 100644 --- a/src/storage/ddl/ob_ddl_redo_log_writer.h +++ b/src/storage/ddl/ob_ddl_redo_log_writer.h @@ -375,7 +375,9 @@ public: const share::SCN &start_scn, const uint64_t data_format_version, const storage::ObDirectLoadType direct_load_type, - const int64_t row_id_offset = -1); + const int64_t row_id_offset = -1, + const bool with_cs_replica = false, + const bool need_submit_io = true); void reset(); int write( blocksstable::ObMacroBlockHandle ¯o_handle, @@ -403,6 +405,8 @@ private: // if current macro block finish with 50 rows, current macro block's end_row_offset will be 149. // end_row_offset = ddl_start_row_offset + curr_row_count - 1. int64_t row_id_offset_; + bool with_cs_replica_; + bool need_submit_io_; }; } // end namespace storage diff --git a/src/storage/ddl/ob_ddl_replay_executor.cpp b/src/storage/ddl/ob_ddl_replay_executor.cpp index 77aa57ca1..e2198fc0a 100644 --- a/src/storage/ddl/ob_ddl_replay_executor.cpp +++ b/src/storage/ddl/ob_ddl_replay_executor.cpp @@ -26,6 +26,8 @@ using namespace oceanbase::blocksstable; using namespace oceanbase::storage; using namespace oceanbase::share; +ERRSIM_POINT_DEF(EN_REPLAY_REDO_DDL_LOG_WAIT); + ObDDLReplayExecutor::ObDDLReplayExecutor() : logservice::ObTabletReplayExecutor(), ls_(nullptr), scn_() {} @@ -265,7 +267,10 @@ int ObDDLStartReplayExecutor::replay_ddl_start(ObTabletHandle &tablet_handle, co } else { table_key = log_->get_table_key(); } - if (OB_FAIL(tenant_direct_load_mgr->replay_create_tablet_direct_load(tablet_handle, log_->get_execution_id(), direct_load_param))) { + + if (!is_lob_meta_tablet && ls_->is_cs_replica() && OB_FAIL(pre_process_for_cs_replica(direct_load_param, table_key, tablet_handle, tablet_id))) { + LOG_WARN("pre process for cs replica failed", K(ret), K(direct_load_param), K(table_key), K(tablet_id)); + } else if (OB_FAIL(tenant_direct_load_mgr->replay_create_tablet_direct_load(tablet_handle, log_->get_execution_id(), direct_load_param))) { LOG_WARN("create tablet manager failed", K(ret)); } else if (OB_FAIL(tenant_direct_load_mgr->get_tablet_mgr_and_check_major( ls_->get_ls_id(), @@ -284,7 +289,8 @@ int ObDDLStartReplayExecutor::replay_ddl_start(ObTabletHandle &tablet_handle, co direct_load_param))) { LOG_WARN("update direct load mgr failed", K(ret)); } else if (OB_FAIL(direct_load_mgr_handle.get_full_obj()->start(*tablet_handle.get_obj(), - table_key, scn_, log_->get_data_format_version(), log_->get_execution_id(), SCN::min_scn()/*checkpoint_scn*/))) { + table_key, scn_, log_->get_data_format_version(), log_->get_execution_id(), SCN::min_scn()/*checkpoint_scn*/, + direct_load_param.common_param_.replay_normal_in_cs_replica_))) { LOG_WARN("direct load start failed", K(ret)); if (OB_TASK_EXPIRED != ret) { LOG_WARN("start ddl log failed", K(ret), K_(log), K_(scn)); @@ -299,6 +305,48 @@ int ObDDLStartReplayExecutor::replay_ddl_start(ObTabletHandle &tablet_handle, co return ret; } +int ObDDLStartReplayExecutor::pre_process_for_cs_replica( + ObTabletDirectLoadInsertParam &direct_load_param, + ObITable::TableKey &table_key, + ObTabletHandle &tablet_handle, + const ObTabletID &tablet_id) +{ + int ret = OB_SUCCESS; + if (log_->get_with_cs_replica()) { + if (log_->get_table_key().is_row_store_major_sstable()) { + table_key.table_type_ = ObITable::COLUMN_ORIENTED_SSTABLE; // for passing defence + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid table key for replay ddl start in cs replica", K(ret), K_(log)); + } + } else { + // ddl is concurrent with adding C replica + const ObTablet *tablet = nullptr; + ObStorageSchema *schema_on_tablet = nullptr; + ObArenaAllocator tmp_arena("RplyStartTmp"); + if (OB_UNLIKELY(!tablet_handle.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tablet handle is invalid", K(ret), K(tablet_handle)); + } else if (OB_ISNULL(tablet = tablet_handle.get_obj())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tablet is null", K(ret), K(tablet_id)); + } else if (tablet->is_row_store()) { + // not be created to column store tablet in cs replica, means this tablet is not user data tablet, ignore + } else if (log_->get_table_key().is_column_store_major_sstable()) { + // column store tablet originally, ignore + } else if (OB_FAIL(tablet->load_storage_schema(tmp_arena, schema_on_tablet))) { + LOG_WARN("load storage schema failed", K(ret), KPC(tablet)); + } else if (schema_on_tablet->is_cs_replica_compat()) { + direct_load_param.common_param_.replay_normal_in_cs_replica_ = true; + LOG_TRACE("[CS-Replica] process concurrent ddl and ls migration", K(ret), K(tablet_id), KPC(tablet)); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid storage schema status", K(ret), KPC(tablet), KPC(schema_on_tablet)); + } + ObTabletObjLoadHelper::free(tmp_arena, schema_on_tablet); + } + return ret; +} // ObDDLRedoReplayExecutor ObDDLRedoReplayExecutor::ObDDLRedoReplayExecutor() @@ -344,6 +392,7 @@ int ObDDLRedoReplayExecutor::do_replay_(ObTabletHandle &tablet_handle) const ObDDLMacroBlockRedoInfo &redo_info = log_->get_redo_info(); ObMacroBlockWriteInfo write_info; ObDDLMacroBlock macro_block; + bool can_skip = false; write_info.buffer_ = redo_info.data_buffer_.ptr(); write_info.size_= redo_info.data_buffer_.length(); write_info.io_desc_.set_wait_event(ObWaitEventIds::DB_FILE_COMPACT_WRITE); @@ -368,6 +417,9 @@ int ObDDLRedoReplayExecutor::do_replay_(ObTabletHandle &tablet_handle) ret = OB_EAGAIN; } } + } else if (OB_FAIL(filter_redo_log_(redo_info, tablet_handle, can_skip))) { + LOG_WARN("fail to filter redo log", K(ret), K(redo_info), K_(ls)); + } else if (can_skip) { } else { if (OB_FAIL(do_full_replay_(tablet_handle, write_info, macro_block))) { LOG_WARN("fail to do full replay", K(ret)); @@ -491,6 +543,43 @@ int ObDDLRedoReplayExecutor::do_full_replay_( return ret; } +int ObDDLRedoReplayExecutor::filter_redo_log_( + const ObDDLMacroBlockRedoInfo &redo_info, + const ObTabletHandle &tablet_handle, + bool &can_skip) +{ + int ret = OB_SUCCESS; + ObTablet *tablet = nullptr; + bool is_cs_replica = ls_->is_cs_replica(); + can_skip = false; + if (redo_info.is_not_compat_cs_replica()) { + // normal + } else if (OB_UNLIKELY(!tablet_handle.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tablet handle", K(ret), K(tablet_handle)); + } else if (OB_ISNULL(tablet = tablet_handle.get_obj())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tablet is null", K(ret), K(tablet_handle)); + } else if (is_cs_replica && redo_info.is_cs_replica_row_store()) { + can_skip = true; + } else if (!is_cs_replica && redo_info.is_cs_replica_column_store()) { + can_skip = true; + } else if (is_cs_replica && redo_info.is_cs_replica_column_store() && tablet->is_row_store()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected tablet status", K(ret), K(redo_info), "ls_meta", ls_->get_ls_meta(), KPC(tablet)); + } + LOG_TRACE("[CS-Replica] Finish filter redo log", K(ret), K(redo_info), K(is_cs_replica), K(can_skip), KPC(tablet), K(ls_)); +#ifdef ERRSIM + if (OB_SUCC(ret)) { + ret = EN_REPLAY_REDO_DDL_LOG_WAIT; + if (OB_FAIL(ret)) { + LOG_INFO("EN_REPLAY_REDO_DDL_LOG_WAIT replay ddl redo failed", K(ret), K(redo_info), K(can_skip)); + } + } +#endif + return ret; +} + // ObDDLCommitReplayExecutor ObDDLCommitReplayExecutor::ObDDLCommitReplayExecutor() : ObDDLReplayExecutor(), log_(nullptr) diff --git a/src/storage/ddl/ob_ddl_replay_executor.h b/src/storage/ddl/ob_ddl_replay_executor.h index 28803f9bb..5ac50b2da 100644 --- a/src/storage/ddl/ob_ddl_replay_executor.h +++ b/src/storage/ddl/ob_ddl_replay_executor.h @@ -18,6 +18,7 @@ #include "storage/ddl/ob_ddl_clog.h" #include "storage/ddl/ob_ddl_inc_clog.h" #include "storage/ddl/ob_ddl_struct.h" +#include "storage/ddl/ob_direct_load_struct.h" #include "storage/blocksstable/ob_block_sstable_struct.h" namespace oceanbase @@ -93,7 +94,11 @@ protected: // @return other error codes, failed to replay. int do_replay_(ObTabletHandle &handle) override; int replay_ddl_start(ObTabletHandle &handle, const bool is_lob_meta_tablet); - + int pre_process_for_cs_replica( + ObTabletDirectLoadInsertParam &direct_load_param, + ObITable::TableKey &table_key, + ObTabletHandle &tablet_handle, + const ObTabletID &tablet_id); private: const ObDDLStartLog *log_; }; @@ -124,6 +129,10 @@ private: ObTabletHandle &tablet_handle, blocksstable::ObMacroBlockWriteInfo &write_info, storage::ObDDLMacroBlock ¯o_block); + int filter_redo_log_( + const ObDDLMacroBlockRedoInfo &redo_info, + const ObTabletHandle &tablet_handle, + bool &can_skip); private: const ObDDLRedoLog *log_; }; diff --git a/src/storage/ddl/ob_ddl_struct.cpp b/src/storage/ddl/ob_ddl_struct.cpp index 8282e849a..cae090d30 100644 --- a/src/storage/ddl/ob_ddl_struct.cpp +++ b/src/storage/ddl/ob_ddl_struct.cpp @@ -359,6 +359,21 @@ bool ObDDLMacroBlockRedoInfo::is_column_group_info_valid() const return table_key_.is_column_store_sstable() && end_row_id_ >= 0; } +bool ObDDLMacroBlockRedoInfo::is_not_compat_cs_replica() const +{ + return !with_cs_replica_; +} + +bool ObDDLMacroBlockRedoInfo::is_cs_replica_row_store() const +{ + return with_cs_replica_ && !table_key_.is_column_store_sstable(); +} + +bool ObDDLMacroBlockRedoInfo::is_cs_replica_column_store() const +{ + return with_cs_replica_ && table_key_.is_column_store_sstable(); +} + OB_SERIALIZE_MEMBER(ObDDLMacroBlockRedoInfo, table_key_, data_buffer_, diff --git a/src/storage/ddl/ob_ddl_struct.h b/src/storage/ddl/ob_ddl_struct.h index cae64144d..2a64bdb78 100644 --- a/src/storage/ddl/ob_ddl_struct.h +++ b/src/storage/ddl/ob_ddl_struct.h @@ -176,6 +176,18 @@ public: ~ObDDLMacroBlockRedoInfo() = default; bool is_valid() const; bool is_column_group_info_valid() const; + /* + * For tow conditions: + * 1. column store table, unnessasery to generate double redo clog. + * 2. row store table, but unnessasery to process cs replica. + * (a) cs replica not exist, may not be created or is creating. + * (b) table is not user data table. + */ + bool is_not_compat_cs_replica() const; + // If cs replica exist, this redo clog is suitable for F/R replica. + bool is_cs_replica_row_store() const; + // If cs replica exist, this redo clog is suitable for C replica. + bool is_cs_replica_column_store() const; void reset(); TO_STRING_KV(K_(table_key), K_(data_buffer), diff --git a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp index 464efab5d..bae9673fc 100644 --- a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp +++ b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp @@ -31,6 +31,7 @@ #include "storage/lob/ob_lob_util.h" #include "storage/tx_storage/ob_ls_service.h" #include "storage/column_store/ob_column_oriented_sstable.h" +#include "storage/column_store/ob_column_store_replica_util.h" #include "storage/direct_load/ob_direct_load_insert_table_row_iterator.h" using namespace oceanbase; @@ -1233,7 +1234,8 @@ void ObTabletDirectLoadBuildCtx::cleanup_slice_writer(const int64_t context_id) ObTabletDirectLoadMgr::ObTabletDirectLoadMgr() : is_inited_(false), is_schema_item_ready_(false), ls_id_(), tablet_id_(), table_key_(), data_format_version_(0), - lock_(), ref_cnt_(0), direct_load_type_(ObDirectLoadType::DIRECT_LOAD_INVALID), sqc_build_ctx_(), + lock_(), ref_cnt_(0), direct_load_type_(ObDirectLoadType::DIRECT_LOAD_INVALID), + need_process_cs_replica_(false), need_fill_column_group_(false), sqc_build_ctx_(), column_items_(), lob_column_idxs_(), lob_col_types_(), schema_item_(), dir_id_(0) { column_items_.set_attr(ObMemAttr(MTL_ID(), "DL_schema")); @@ -1252,6 +1254,8 @@ ObTabletDirectLoadMgr::~ObTabletDirectLoadMgr() data_format_version_ = 0; ATOMIC_STORE(&ref_cnt_, 0); direct_load_type_ = ObDirectLoadType::DIRECT_LOAD_INVALID; + need_process_cs_replica_ = false; + need_fill_column_group_ = false; column_items_.reset(); lob_column_idxs_.reset(); lob_col_types_.reset(); @@ -1794,21 +1798,26 @@ int ObTabletDirectLoadMgr::notify_all() return ret; } -struct SliceEndkeyCompareFunctor +struct CSSliceEndkeyCompareFunctor { public: - SliceEndkeyCompareFunctor(const ObStorageDatumUtils &datum_utils) : datum_utils_(datum_utils), ret_code_(OB_SUCCESS) {} + CSSliceEndkeyCompareFunctor(const ObStorageDatumUtils &datum_utils) : datum_utils_(datum_utils), ret_code_(OB_SUCCESS) {} bool operator ()(const ObDirectLoadSliceWriter *left, const ObDirectLoadSliceWriter *right) { bool bret = false; int ret = ret_code_; if (OB_FAIL(ret)) { - } else if (OB_ISNULL(left) || OB_ISNULL(right)) { + } else if (OB_ISNULL(left) || OB_ISNULL(right) || !left->need_column_store() + || !right->need_column_store() || left->get_writer_type() != right->get_writer_type()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret)); + LOG_WARN("invalid argument", K(ret), KPC(left), KPC(right)); } else if (!left->is_empty() && !right->is_empty()) { - const ObChunkSliceStore *left_slice_store = static_cast(left->get_slice_store()); - const ObChunkSliceStore *right_slice_store = static_cast(right->get_slice_store()); + const ObChunkSliceStore *left_slice_store = left->is_cs_replica_write() + ? static_cast(left->get_slice_store())->get_column_slice_store() + : static_cast(left->get_slice_store()); + const ObChunkSliceStore *right_slice_store = right->is_cs_replica_write() + ? static_cast(right->get_slice_store())->get_column_slice_store() + : static_cast(right->get_slice_store()); int cmp_ret = 0; if (OB_FAIL(left_slice_store->endkey_.compare(right_slice_store->endkey_, datum_utils_, cmp_ret))) { LOG_WARN("endkey compare failed", K(ret)); @@ -1865,7 +1874,7 @@ int ObTabletDirectLoadMgr::calc_range(const int64_t thread_cnt) } } if (OB_SUCC(ret)) { - SliceEndkeyCompareFunctor cmp(tablet_handle.get_obj()->get_rowkey_read_info().get_datum_utils()); + CSSliceEndkeyCompareFunctor cmp(tablet_handle.get_obj()->get_rowkey_read_info().get_datum_utils()); lib::ob_sort(sorted_slices.begin(), sorted_slices.end(), cmp); ret = cmp.ret_code_; if (OB_FAIL(ret)) { @@ -1878,11 +1887,8 @@ int ObTabletDirectLoadMgr::calc_range(const int64_t thread_cnt) offset += sorted_slices.at(i)->get_row_count(); } } - if (OB_SUCC(ret) && is_data_direct_load(direct_load_type_)) { - bool is_column_store = false; - if (OB_FAIL(ObCODDLUtil::need_column_group_store(*sqc_build_ctx_.storage_schema_, is_column_store))) { - LOG_WARN("fail to check need column group", K(ret)); - } else if (is_column_store) { + if (OB_SUCC(ret)) { + if (is_data_direct_load(direct_load_type_) && need_fill_column_group_) { if (thread_cnt <= 0) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invali thread cnt", K(ret), K(thread_cnt)); @@ -2007,6 +2013,7 @@ int ObTabletDirectLoadMgr::close_sstable_slice( } else if (OB_FALSE_IT(next_seq = slice_writer->get_next_block_start_seq())) { // block start seq after the close operation is the next availabled one. } else if (!slice_info.is_lob_slice_ && is_ddl_direct_load(direct_load_type_)) { + // for cs replica, full direct load all take the same way with offline ddl of column store int64_t task_finish_count = -1; { ObLatchRGuard guard(lock_, ObLatchIds::TABLET_DIRECT_LOAD_MGR_LOCK); @@ -2015,13 +2022,10 @@ int ObTabletDirectLoadMgr::close_sstable_slice( } } LOG_INFO("inc task finish count", K(tablet_id_), K(execution_id), K(task_finish_count), K(sqc_build_ctx_.task_total_cnt_)); - bool is_column_group_store = false; if (OB_ISNULL(sqc_build_ctx_.storage_schema_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid tablet handle", K(ret), KP(sqc_build_ctx_.storage_schema_)); - } else if (OB_FAIL(ObCODDLUtil::need_column_group_store(*sqc_build_ctx_.storage_schema_, is_column_group_store))) { - LOG_WARN("fail to check is column group store", K(ret)); - } else if (!is_column_group_store) { + } else if (!need_fill_column_group_) { if (task_finish_count >= sqc_build_ctx_.task_total_cnt_) { // for ddl, write commit log when all slices ready. if (OB_FAIL(close(execution_id, start_scn))) { @@ -2212,7 +2216,7 @@ int ObTabletDirectLoadMgr::fill_aggregated_column_group( } else if (OB_UNLIKELY(first_slice_writer->get_row_offset() < 0)) { ret = OB_ERR_SYS; LOG_WARN("invalid row offset", K(ret), K(first_slice_writer->get_row_offset())); - } else if (OB_FAIL(cur_writer->init(storage_schema, cg_idx, this, first_slice_writer->get_start_seq(), first_slice_writer->get_row_offset(), get_start_scn()))) { + } else if (OB_FAIL(cur_writer->init(storage_schema, cg_idx, this, first_slice_writer->get_start_seq(), first_slice_writer->get_row_offset(), get_start_scn(), need_process_cs_replica_))) { LOG_WARN("init co ddl writer failed", K(ret), KPC(cur_writer), K(cg_idx), KPC(this)); } else { for (int64_t i = start_idx; OB_SUCC(ret) && i < last_idx; ++i) { @@ -2359,6 +2363,27 @@ int ObTabletDirectLoadMgr::prepare_storage_schema(ObTabletHandle &tablet_handle) return ret; } +int ObTabletDirectLoadMgr::init_column_store_params( + const ObLSHandle &ls_handle, + const ObStorageSchema &storage_schema, + const ObTabletID &new_tablet_id, + const ObDirectLoadType new_direct_load_type) +{ + int ret = OB_SUCCESS; + const ObLS *ls = nullptr; + bool need_process = false; + if (OB_ISNULL(ls = ls_handle.get_ls())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get nullptr ls", K(ret), K(ls_handle)); + } else if (OB_FAIL(ObCSReplicaUtil::check_replica_set_need_process_cs_replica(*ls, new_tablet_id, storage_schema, need_process))) { + LOG_WARN("failed to check ls replica set", K(ret), K(new_tablet_id), KPC(sqc_build_ctx_.storage_schema_)); + } else { + need_process_cs_replica_ = need_process && is_ddl_direct_load(new_direct_load_type); + need_fill_column_group_ = !storage_schema.is_row_store() || need_process_cs_replica_; + } + return ret; +} + ObTabletFullDirectLoadMgr::ObTabletFullDirectLoadMgr() : ObTabletDirectLoadMgr(), start_scn_(share::SCN::min_scn()), commit_scn_(share::SCN::min_scn()), execution_id_(-1) @@ -2389,7 +2414,7 @@ int ObTabletFullDirectLoadMgr::update( LOG_WARN("null storage schema", K(ret)); } else if (OB_FAIL(ObCODDLUtil::need_column_group_store(*sqc_build_ctx_.storage_schema_, is_column_group_store))) { LOG_WARN("fail to get schema is column group store", K(ret)); - } else if (is_column_group_store) { + } else if (is_column_group_store && !build_param.common_param_.replay_normal_in_cs_replica_) { table_key_.table_type_ = ObITable::COLUMN_ORIENTED_SSTABLE; int64_t base_cg_idx = -1; if (OB_FAIL(ObCODDLUtil::get_base_cg_idx(sqc_build_ctx_.storage_schema_, base_cg_idx))) { @@ -2452,6 +2477,11 @@ int ObTabletFullDirectLoadMgr::open(const int64_t current_execution_id, share::S ret = OB_ERR_UNEXPECTED; LOG_WARN("start scn must be valid after commit", K(ret), K(start_scn)); } + } else if (OB_ISNULL(sqc_build_ctx_.storage_schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("null storage schema", K(ret), K(sqc_build_ctx_)); + } else if (OB_FAIL(init_column_store_params(ls_handle, *sqc_build_ctx_.storage_schema_, tablet_id_, direct_load_type_))) { + LOG_WARN("failed to refresh cs replica status", K(ret), K(ls_handle), K_(tablet_id)); } else { ObDDLKvMgrHandle ddl_kv_mgr_handle; ObDDLKvMgrHandle lob_kv_mgr_handle; @@ -2660,7 +2690,8 @@ int ObTabletFullDirectLoadMgr::start( const share::SCN &start_scn, const uint64_t data_format_version, const int64_t execution_id, - const share::SCN &checkpoint_scn) + const share::SCN &checkpoint_scn, + const bool replay_normal_in_cs_replica /*=false*/) { int ret = OB_SUCCESS; share::SCN saved_start_scn; @@ -2729,7 +2760,7 @@ int ObTabletFullDirectLoadMgr::start( if (lob_mgr_handle_.is_valid() && OB_FAIL(lob_mgr_handle_.get_full_obj()->init_ddl_table_store(saved_start_scn, saved_snapshot_version, saved_start_scn))) { LOG_WARN("clean up ddl sstable failed", K(ret)); - } else if (OB_FAIL(init_ddl_table_store(saved_start_scn, saved_snapshot_version, saved_start_scn))) { + } else if (OB_FAIL(init_ddl_table_store(saved_start_scn, saved_snapshot_version, saved_start_scn, replay_normal_in_cs_replica))) { LOG_WARN("clean up ddl sstable failed", K(ret), K(tablet_id_)); } } @@ -3157,7 +3188,8 @@ int ObTabletFullDirectLoadMgr::cleanup_unlock() int ObTabletFullDirectLoadMgr::init_ddl_table_store( const share::SCN &start_scn, const int64_t snapshot_version, - const share::SCN &ddl_checkpoint_scn) + const share::SCN &ddl_checkpoint_scn, + const bool replay_normal_in_cs_replica /*=false*/) { int ret = OB_SUCCESS; ObLSHandle ls_handle; @@ -3201,7 +3233,7 @@ int ObTabletFullDirectLoadMgr::init_ddl_table_store( ddl_param.commit_scn_ = commit_scn_; ddl_param.snapshot_version_ = table_key_.get_snapshot_version(); ddl_param.data_format_version_ = data_format_version_; - ddl_param.table_key_.table_type_ = is_column_group_store ? ObITable::DDL_MERGE_CO_SSTABLE : ObITable::DDL_DUMP_SSTABLE; + ddl_param.table_key_.table_type_ = (is_column_group_store && !replay_normal_in_cs_replica) ? ObITable::DDL_MERGE_CO_SSTABLE : ObITable::DDL_DUMP_SSTABLE; ddl_param.table_key_.scn_range_.start_scn_ = SCN::scn_dec(start_scn); ddl_param.table_key_.scn_range_.end_scn_ = start_scn; @@ -3215,6 +3247,7 @@ int ObTabletFullDirectLoadMgr::init_ddl_table_store( param.ddl_info_.ddl_checkpoint_scn_ = ddl_checkpoint_scn; param.ddl_info_.ddl_execution_id_ = execution_id_; param.ddl_info_.data_format_version_ = data_format_version_; + param.ddl_info_.ddl_table_type_ = ddl_param.table_key_.table_type_; if (OB_FAIL(ObTabletDDLUtil::create_ddl_sstable(*tablet_handle.get_obj(), ddl_param, empty_meta_array, nullptr/*first_ddl_sstable*/, storage_schema, tmp_arena, sstable_handle))) { LOG_WARN("create empty ddl sstable failed", K(ret)); @@ -3258,6 +3291,9 @@ int ObTabletFullDirectLoadMgr::init_ddl_table_store( } else { LOG_INFO("update tablet success", K(ls_id_), K(tablet_id_), "is_column_store", is_column_group_store, K(ddl_param), + "need_process_cs_replica", need_process_cs_replica_, + "need_fill_column_group", need_fill_column_group_, + "replay_normal_in_cs_replica", replay_normal_in_cs_replica, "column_group_schemas", storage_schema->get_column_groups(), "update_table_store_param", param, K(start_scn), K(snapshot_version), K(ddl_checkpoint_scn)); } diff --git a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.h b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.h index 63f97321f..0b2826150 100644 --- a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.h +++ b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.h @@ -388,6 +388,7 @@ public: inline const ObITable::TableKey &get_table_key() const { return table_key_; } inline uint64_t get_data_format_version() const { return data_format_version_; } inline ObDirectLoadType get_direct_load_type() const { return direct_load_type_; } + inline bool need_process_cs_replica() const { return need_process_cs_replica_; } inline ObTabletDirectLoadBuildCtx &get_sqc_build_ctx() { return sqc_build_ctx_; } inline const share::ObLSID &get_ls_id() const { return ls_id_; } inline const ObTabletID &get_tablet_id() const { return tablet_id_; } @@ -408,9 +409,23 @@ public: int calc_cg_range(ObArray &sorted_slices, const int64_t thread_cnt); const ObIArray &get_column_info() const { return column_items_; }; int prepare_storage_schema(ObTabletHandle &tablet_handle); + // init column store related parameters when open in leader + int init_column_store_params( + const ObLSHandle &ls_handle, + const ObStorageSchema &storage_schema, + const ObTabletID &new_tablet_id, + const ObDirectLoadType new_direct_load_type); + /* + * For full data direct load, row store table and column store table take diffrent way. + * 1. row store table: take the same way with offline ddl; + * 2. column store table: take PX to accelerate. + * 3. for table with cs replica, take the same way with offline ddl, and writing additional column store data. + * so if is data direct load type but need process cs replica, it should skip the originally column store load code. + */ + bool is_originally_column_store_data_direct_load() const { return is_data_direct_load(direct_load_type_) && !need_process_cs_replica_; } VIRTUAL_TO_STRING_KV(K_(is_inited), K_(is_schema_item_ready), K_(ls_id), K_(tablet_id), K_(table_key), K_(data_format_version), K_(ref_cnt), - K_(direct_load_type), K_(sqc_build_ctx), KPC(lob_mgr_handle_.get_obj()), K_(schema_item), K_(column_items), K_(lob_column_idxs)); + K_(direct_load_type), K_(need_process_cs_replica), K_(need_fill_column_group), K_(sqc_build_ctx), KPC(lob_mgr_handle_.get_obj()), K_(schema_item), K_(column_items), K_(lob_column_idxs)); private: int prepare_schema_item_on_demand(const uint64_t table_id, @@ -441,6 +456,10 @@ protected: common::ObLatch lock_; int64_t ref_cnt_; ObDirectLoadType direct_load_type_; + // only row store user tablet need process cs replica in leader, column store tablet do not need + bool need_process_cs_replica_; + // column store table, or need process cs replica + bool need_fill_column_group_; // sqc_build_ctx_ is just used for the observer node who receives the requests from the SQL Layer // to write the start log and the data redo log. And other observer nodes can not use it. ObTabletDirectLoadBuildCtx sqc_build_ctx_; @@ -480,7 +499,8 @@ public: const share::SCN &start_scn, const uint64_t data_format_version, const int64_t execution_id, - const share::SCN &checkpoint_scn); + const share::SCN &checkpoint_scn, + const bool replay_normal_in_cs_replica = false); int start_with_checkpoint( ObTablet &tablet, const share::SCN &start_scn, @@ -520,7 +540,7 @@ private: bool is_started() { return start_scn_.is_valid_and_not_min(); } int schedule_merge_task(const share::SCN &start_scn, const share::SCN &commit_scn, const bool wait_major_generated, const bool is_replay); // try wait build major sstable int cleanup_unlock(); - int init_ddl_table_store(const share::SCN &start_scn, const int64_t snapshot_version, const share::SCN &ddl_checkpoint_scn); + int init_ddl_table_store(const share::SCN &start_scn, const int64_t snapshot_version, const share::SCN &ddl_checkpoint_scn, const bool replay_normal_in_cs_replica = false); int update_major_sstable(); private: diff --git a/src/storage/ddl/ob_direct_load_struct.cpp b/src/storage/ddl/ob_direct_load_struct.cpp index 5dedd1e88..e54363c87 100644 --- a/src/storage/ddl/ob_direct_load_struct.cpp +++ b/src/storage/ddl/ob_direct_load_struct.cpp @@ -24,6 +24,7 @@ #include "storage/ddl/ob_direct_insert_sstable_ctx_new.h" #include "storage/lob/ob_lob_util.h" #include "storage/tablet/ob_tablet.h" +#include "storage/ob_storage_schema_util.h" #include "sql/engine/expr/ob_expr_lob_utils.h" #include "sql/das/ob_das_utils.h" #include "sql/engine/basic/chunk_store/ob_compact_store.h" @@ -638,7 +639,8 @@ int ObChunkSliceStore::close() int ObMacroBlockSliceStore::init( ObTabletDirectLoadMgr *tablet_direct_load_mgr, const blocksstable::ObMacroDataSeq &data_seq, - const SCN &start_scn) + const SCN &start_scn, + const bool need_process_cs_replica /*= false*/) { int ret = OB_SUCCESS; if (OB_UNLIKELY(is_inited_)) { @@ -669,7 +671,7 @@ int ObMacroBlockSliceStore::init( ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to alloc memory", K(ret)); } else if (OB_FAIL(static_cast(ddl_redo_callback_)->init( - ls_id, table_key.tablet_id_, DDL_MB_DATA_TYPE, table_key, ddl_task_id, start_scn, data_format_version, direct_load_type, -1/*row_id_offset*/))) { + ls_id, table_key.tablet_id_, DDL_MB_DATA_TYPE, table_key, ddl_task_id, start_scn, data_format_version, direct_load_type, -1/*row_id_offset*/, need_process_cs_replica))) { LOG_WARN("fail to init full ddl_redo_callback_", K(ret)); } } @@ -677,6 +679,7 @@ int ObMacroBlockSliceStore::init( if (OB_FAIL(macro_block_writer_.open(data_desc.get_desc(), data_seq, ddl_redo_callback_))) { LOG_WARN("open macro bock writer failed", K(ret)); } else { + need_process_cs_replica_ = need_process_cs_replica; is_inited_ = true; } } @@ -708,6 +711,147 @@ int ObMacroBlockSliceStore::close() return ret; } +ObMultiSliceStore::ObMultiSliceStore() +: is_inited_(false), + arena_allocator_(nullptr), + cs_replica_schema_(nullptr), + row_slice_store_(nullptr), + column_slice_store_(nullptr) +{} + +ObMultiSliceStore::~ObMultiSliceStore() +{ + reset(); +} + +int ObMultiSliceStore::init( + ObArenaAllocator &allocator, + ObTabletDirectLoadMgr *tablet_direct_load_mgr, + const blocksstable::ObMacroDataSeq &data_seq, + const share::SCN &start_scn, + const int64_t rowkey_column_count, + const ObStorageSchema *storage_schema, + const ObIArray &col_schema, + const int64_t dir_id, + const int64_t parallelism) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("multi slice store init twice", K(ret), K_(is_inited)); + } else if (OB_UNLIKELY(nullptr == tablet_direct_load_mgr + || !data_seq.is_valid() + || rowkey_column_count <= 0 + || nullptr == storage_schema + || !storage_schema->is_row_store() + || !storage_schema->is_user_data_table())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), KPC(tablet_direct_load_mgr), K(data_seq), K(rowkey_column_count), KPC(storage_schema)); + } else if (OB_FAIL(ObStorageSchemaUtil::alloc_storage_schema(allocator, cs_replica_schema_))) { + LOG_WARN("fail to alloc cs_replica_schema", K(ret)); + } else if (OB_FAIL(cs_replica_schema_->init(allocator, *storage_schema, false /*skip_column_info*/, nullptr /*column_group_schema*/, true /*generate_default_cg_array*/))) { + LOG_WARN("fail to init cs_replica_schema for multi slice store", K(ret), KPC(storage_schema)); + } else if (OB_ISNULL(row_slice_store_ = OB_NEWx(ObMacroBlockSliceStore, &allocator))){ + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate memory for row slice store", K(ret)); + } else if (OB_FAIL(row_slice_store_->init(tablet_direct_load_mgr, data_seq, start_scn, true /*need_process_cs_replica*/))) { + LOG_WARN("fail to init row slice store", K(ret), KPC(tablet_direct_load_mgr), K(data_seq), K(start_scn)); + } else if (OB_ISNULL(column_slice_store_ = OB_NEWx(ObChunkSliceStore, &allocator))){ + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate memory for column slice store", K(ret)); + } else if (OB_FAIL(column_slice_store_->init(rowkey_column_count, cs_replica_schema_, allocator, col_schema, dir_id, parallelism))) { + LOG_WARN("fail to init column slice store", K(ret), K(dir_id), K(parallelism), KPC(storage_schema), K(col_schema), K(rowkey_column_count)); + } else { + is_inited_ = true; + arena_allocator_ = &allocator; + LOG_DEBUG("[CS-Replica] Successfully init multi slice store", K(ret), KPC(this)); + } + + if (OB_FAIL(ret)) { + (void) free_memory(allocator); + } + return ret; +} + +int ObMultiSliceStore::append_row(const blocksstable::ObDatumRow &datum_row) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("multi slice store not init", K(ret)); + } else if (OB_UNLIKELY(nullptr == row_slice_store_ || nullptr == column_slice_store_)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("unexpected slice store", K(ret), KPC_(row_slice_store), KPC_(column_slice_store)); + } else if (OB_FAIL(row_slice_store_->append_row(datum_row))) { + LOG_WARN("fail to append row to row slice store", K(ret)); + } else if (OB_FAIL(column_slice_store_->append_row(datum_row))) { + LOG_WARN("fail to append row to column slice store", K(ret)); + } + return ret; +} + +int ObMultiSliceStore::close() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("multi slice store not init", K(ret)); + } else if (OB_UNLIKELY(nullptr == row_slice_store_ || nullptr == column_slice_store_)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("unexpected slice store", K(ret), KPC_(row_slice_store), KPC_(column_slice_store)); + } else if (OB_FAIL(row_slice_store_->close())) { + LOG_WARN("fail to close row slice store", K(ret)); + } else if (OB_FAIL(column_slice_store_->close())) { + LOG_WARN("fail to close column slice store", K(ret)); + } else { + LOG_DEBUG("[CS-Replica] Finish close multi slice store", K(ret), KPC(this)); + } + return ret; +} + +int64_t ObMultiSliceStore::get_row_count() const +{ + return column_slice_store_->get_row_count(); +} + +int64_t ObMultiSliceStore::get_next_block_start_seq() const +{ + return row_slice_store_->get_next_block_start_seq(); +} + +void ObMultiSliceStore::reset() +{ + if (OB_NOT_NULL(arena_allocator_)) { + (void) free_memory(*arena_allocator_); + } else if (nullptr == column_slice_store_ || nullptr == row_slice_store_ || nullptr == cs_replica_schema_) { + LOG_ERROR_RET(OB_ERR_UNEXPECTED, "unexpected status", KPC_(column_slice_store), KPC_(row_slice_store), KPC_(cs_replica_schema)); + } + column_slice_store_ = nullptr; + row_slice_store_ = nullptr; + cs_replica_schema_ = nullptr; + arena_allocator_ = nullptr; + is_inited_ = false; +} + +void ObMultiSliceStore::free_memory(ObArenaAllocator &allocator) +{ + if (OB_NOT_NULL(column_slice_store_)) { + column_slice_store_->~ObChunkSliceStore(); + allocator.free(column_slice_store_); + column_slice_store_ = nullptr; + } + if (OB_NOT_NULL(row_slice_store_)) { + row_slice_store_->~ObMacroBlockSliceStore(); + allocator.free(row_slice_store_); + row_slice_store_ = nullptr; + } + if (OB_NOT_NULL(cs_replica_schema_)) { + cs_replica_schema_->~ObStorageSchema(); + allocator.free(cs_replica_schema_); + cs_replica_schema_ = nullptr; + } +} + bool ObTabletDDLParam::is_valid() const { return is_valid_direct_load(direct_load_type_) @@ -721,7 +865,7 @@ bool ObTabletDDLParam::is_valid() const } ObDirectLoadSliceWriter::ObDirectLoadSliceWriter() - : is_inited_(false), need_column_store_(false), is_canceled_(false), start_seq_(), tablet_direct_load_mgr_(nullptr), + : is_inited_(false), writer_type_(ObDirectLoadSliceWriterType::WRITER_TYPE_MAX), is_canceled_(false), start_seq_(), tablet_direct_load_mgr_(nullptr), slice_store_(nullptr), meta_write_iter_(nullptr), row_iterator_(nullptr), allocator_(lib::ObLabel("SliceWriter"), OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), row_offset_(-1) { @@ -746,7 +890,7 @@ ObDirectLoadSliceWriter::~ObDirectLoadSliceWriter() } allocator_.reset(); row_offset_ = -1; - need_column_store_ = false; + writer_type_ = ObDirectLoadSliceWriterType::WRITER_TYPE_MAX; } //for test @@ -788,8 +932,28 @@ int ObDirectLoadSliceWriter::prepare_slice_store_if_need( LOG_WARN("not init", K(ret)); } else if (nullptr != slice_store_) { // do nothing + } else if (tablet_direct_load_mgr_->need_process_cs_replica()) { + writer_type_ = ObDirectLoadSliceWriterType::COL_REPLICA_WRITER; + ObMultiSliceStore *multi_slice_store = nullptr; + if (OB_ISNULL(storage_schema)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("null schema", K(ret), K(*this)); + } else if (OB_ISNULL(multi_slice_store = OB_NEWx(ObMultiSliceStore, &allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate memory for multi slice store failed", K(ret)); + } else if (OB_FAIL(multi_slice_store->init(allocator_, tablet_direct_load_mgr_, start_seq_, start_scn, + schema_rowkey_column_num + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(), + storage_schema, tablet_direct_load_mgr_->get_column_info(), dir_id, parallelism))) { + LOG_WARN("init multi slice store failed", K(ret), KPC_(tablet_direct_load_mgr), KPC(storage_schema)); + } else { + slice_store_ = multi_slice_store; + } + if (OB_FAIL(ret) && nullptr != multi_slice_store) { + multi_slice_store->~ObMultiSliceStore(); + allocator_.free(multi_slice_store); + } } else if (is_full_direct_load(tablet_direct_load_mgr_->get_direct_load_type()) && is_column_store) { - need_column_store_ = true; + writer_type_ = ObDirectLoadSliceWriterType::COL_STORE_WRITER; ObChunkSliceStore *chunk_slice_store = nullptr; if (OB_ISNULL(storage_schema)) { ret = OB_INVALID_ARGUMENT; @@ -808,6 +972,7 @@ int ObDirectLoadSliceWriter::prepare_slice_store_if_need( allocator_.free(chunk_slice_store); } } else { + writer_type_ = ObDirectLoadSliceWriterType::ROW_STORE_WRITER; ObMacroBlockSliceStore *macro_block_slice_store = nullptr; if (OB_ISNULL(macro_block_slice_store = OB_NEWx(ObMacroBlockSliceStore, &allocator_))) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -1307,7 +1472,9 @@ int ObDirectLoadSliceWriter::close() int ObDirectLoadSliceWriter::fill_column_group(const ObStorageSchema *storage_schema, const SCN &start_scn, ObInsertMonitor* insert_monitor) { int ret = OB_SUCCESS; - ObChunkSliceStore *chunk_slice_store = static_cast(slice_store_); + const bool need_process_cs_replica = tablet_direct_load_mgr_->need_process_cs_replica(); + const ObChunkSliceStore *chunk_slice_store = nullptr; + ObStorageSchema *cs_replica_storage_schema = nullptr; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); @@ -1317,14 +1484,42 @@ int ObDirectLoadSliceWriter::fill_column_group(const ObStorageSchema *storage_sc } else if (OB_UNLIKELY(row_offset_ < 0)) { ret = OB_ERR_SYS; LOG_WARN("row offset not set", K(ret), K(row_offset_)); - } else if (nullptr == chunk_slice_store || is_empty()) { + } else if (OB_ISNULL(slice_store_) || is_empty() + || OB_ISNULL(chunk_slice_store = need_process_cs_replica + ? static_cast(slice_store_)->get_column_slice_store() + : static_cast(slice_store_))) { // do nothing - LOG_INFO("chunk slice store is null or empty", K(ret), + LOG_INFO("slice_store_ is null or empty", K(ret), KPC_(slice_store), KPC(chunk_slice_store), KPC(tablet_direct_load_mgr_)); } else if (ATOMIC_LOAD(&is_canceled_)) { ret = OB_CANCELED; LOG_WARN("fil cg task canceled", K(ret), K(is_canceled_)); - } else { + } else if (need_process_cs_replica && OB_FAIL(ObStorageSchemaUtil::alloc_storage_schema(allocator_, cs_replica_storage_schema))) { + LOG_WARN("failed to alloc storage schema", K(ret)); + } else if (need_process_cs_replica && OB_FAIL(cs_replica_storage_schema->init(allocator_, *storage_schema, + false /*skip_column_info*/, nullptr /*column_group_schema*/, true /*generate_cs_replica_cg_array*/))) { + LOG_WARN("failed to init storage schema for cs replica", K(ret), KPC(storage_schema)); + } else if (OB_FAIL(inner_fill_column_group(chunk_slice_store, need_process_cs_replica ? cs_replica_storage_schema : storage_schema, start_scn, insert_monitor))) { + LOG_WARN("failed to fill column group", K(ret)); + } + + if (OB_NOT_NULL(cs_replica_storage_schema)) { + ObStorageSchemaUtil::free_storage_schema(allocator_, cs_replica_storage_schema); + cs_replica_storage_schema = nullptr; + } + + return ret; +} + + +int ObDirectLoadSliceWriter::inner_fill_column_group( + const ObChunkSliceStore *chunk_slice_store, + const ObStorageSchema *storage_schema, + const SCN &start_scn, + ObInsertMonitor* insert_monitor) +{ + int ret = OB_SUCCESS; + { // remain this {} pair to make git diff more readable const ObIArray &cg_schemas = storage_schema->get_column_groups(); FLOG_INFO("[DDL_FILL_CG] fill column group start", "tablet_id", tablet_direct_load_mgr_->get_tablet_id(), @@ -1340,7 +1535,7 @@ int ObDirectLoadSliceWriter::fill_column_group(const ObStorageSchema *storage_sc // 2. rescan and write for (int64_t cg_idx = 0; OB_SUCC(ret) && cg_idx < cg_schemas.count(); ++cg_idx) { cur_writer->reset(); - if (OB_FAIL(cur_writer->init(storage_schema, cg_idx, tablet_direct_load_mgr_, start_seq_, row_offset_, start_scn))) { + if (OB_FAIL(cur_writer->init(storage_schema, cg_idx, tablet_direct_load_mgr_, start_seq_, row_offset_, start_scn, tablet_direct_load_mgr_->need_process_cs_replica()))) { LOG_WARN("init co ddl writer failed", K(ret), KPC(cur_writer), K(cg_idx), KPC(this)); } else { sql::ObCompactStore *cur_datum_store = chunk_slice_store->datum_stores_.at(cg_idx); @@ -1401,7 +1596,7 @@ void ObCOSliceWriter::reset() int ObCOSliceWriter::init(const ObStorageSchema *storage_schema, const int64_t cg_idx, ObTabletDirectLoadMgr *tablet_direct_load_mgr, const ObMacroDataSeq &start_seq, const int64_t row_id_offset, - const SCN &start_scn) + const SCN &start_scn, const bool with_cs_replica) { int ret = OB_SUCCESS; if (OB_UNLIKELY(is_inited_)) { @@ -1420,6 +1615,7 @@ int ObCOSliceWriter::init(const ObStorageSchema *storage_schema, const int64_t c const int64_t ddl_task_id = tablet_direct_load_mgr->get_ddl_task_id(); const uint64_t data_format_version = tablet_direct_load_mgr->get_data_format_version(); ObLSID ls_id = tablet_direct_load_mgr->get_ls_id(); + const bool need_submit_io = !with_cs_replica; // if need to process cs replica, only write clog, not submit macro block to disk if (OB_FAIL(data_desc_.init(true/*is ddl*/, *storage_schema, ls_id, @@ -1429,13 +1625,14 @@ int ObCOSliceWriter::init(const ObStorageSchema *storage_schema, const int64_t c data_format_version, SCN::min_scn(), &cg_schema, - cg_idx))) { + cg_idx, + need_submit_io))) { LOG_WARN("init data store desc failed", K(ret)); } else if (OB_FAIL(index_builder_.init(data_desc_.get_desc(), nullptr/*macro block flush callback*/, ObSSTableIndexBuilder::ENABLE))) { // data_desc is deep copied LOG_WARN("init sstable index builder failed", K(ret), K(ls_id), K(table_key), K(data_desc_)); } else if (FALSE_IT(data_desc_.get_desc().sstable_index_builder_ = &index_builder_)) { // for build the tail index block in macro block } else if (OB_FAIL(flush_callback_.init(ls_id, table_key.tablet_id_, DDL_MB_DATA_TYPE, table_key, ddl_task_id, - start_scn, data_format_version, tablet_direct_load_mgr->get_direct_load_type(), row_id_offset))) { + start_scn, data_format_version, tablet_direct_load_mgr->get_direct_load_type(), row_id_offset, with_cs_replica, need_submit_io))) { LOG_WARN("fail to init redo log writer callback", KR(ret)); } else if (OB_FAIL(macro_block_writer_.open(data_desc_.get_desc(), start_seq, &flush_callback_))) { LOG_WARN("fail to open macro block writer", K(ret), K(ls_id), K(table_key), K(data_desc_), K(start_seq)); diff --git a/src/storage/ddl/ob_direct_load_struct.h b/src/storage/ddl/ob_direct_load_struct.h index 85f80a0be..f00e1f6ae 100644 --- a/src/storage/ddl/ob_direct_load_struct.h +++ b/src/storage/ddl/ob_direct_load_struct.h @@ -200,13 +200,13 @@ struct ObDirectInsertCommonParam final { public: ObDirectInsertCommonParam() - : ls_id_(), tablet_id_(), direct_load_type_(DIRECT_LOAD_INVALID), data_format_version_(0), read_snapshot_(0) + : ls_id_(), tablet_id_(), direct_load_type_(DIRECT_LOAD_INVALID), data_format_version_(0), read_snapshot_(0), replay_normal_in_cs_replica_(false) {} ~ObDirectInsertCommonParam() = default; bool is_valid() const { return ls_id_.is_valid() && tablet_id_.is_valid() && data_format_version_ >= 0 && read_snapshot_ >= 0 && DIRECT_LOAD_INVALID <= direct_load_type_ && direct_load_type_ <= DIRECT_LOAD_MAX; } - TO_STRING_KV(K_(ls_id), K_(tablet_id), K_(direct_load_type), K_(data_format_version), K_(read_snapshot)); + TO_STRING_KV(K_(ls_id), K_(tablet_id), K_(direct_load_type), K_(data_format_version), K_(read_snapshot), K_(replay_normal_in_cs_replica)); public: share::ObLSID ls_id_; common::ObTabletID tablet_id_; @@ -215,6 +215,7 @@ public: // read_snapshot_ is used to scan the source data. // For full direct load task, it is also the commit version of the target macro block. int64_t read_snapshot_; + bool replay_normal_in_cs_replica_; // when ddl and add cs replica are concurrent, leader may write normal clog }; // only used in runtime execution @@ -453,7 +454,7 @@ class ObMacroBlockSliceStore: public ObTabletSliceStore { public: ObMacroBlockSliceStore() - : is_inited_(false), ddl_redo_callback_(nullptr) {} + : is_inited_(false), need_process_cs_replica_(false), ddl_redo_callback_(nullptr) {} virtual ~ObMacroBlockSliceStore() { if (ddl_redo_callback_ != nullptr) { common::ob_delete(ddl_redo_callback_); @@ -462,17 +463,51 @@ public: int init( ObTabletDirectLoadMgr *tablet_direct_load_mgr, const blocksstable::ObMacroDataSeq &data_seq, - const share::SCN &start_scn); + const share::SCN &start_scn, + const bool need_process_cs_replica = false); virtual int append_row(const blocksstable::ObDatumRow &datum_row) override; virtual int close() override; virtual int64_t get_next_block_start_seq() const override { return macro_block_writer_.get_last_macro_seq(); } - TO_STRING_KV(K(is_inited_), K(macro_block_writer_)); + TO_STRING_KV(K(is_inited_), K_(need_process_cs_replica), K(macro_block_writer_)); private: bool is_inited_; + bool need_process_cs_replica_; blocksstable::ObIMacroBlockFlushCallback *ddl_redo_callback_; blocksstable::ObMacroBlockWriter macro_block_writer_; }; +class ObMultiSliceStore : public ObTabletSliceStore +{ +public: + ObMultiSliceStore(); + virtual ~ObMultiSliceStore(); + int init( + ObArenaAllocator &allocator, + ObTabletDirectLoadMgr *tablet_direct_load_mgr, + const blocksstable::ObMacroDataSeq &data_seq, + const share::SCN &start_scn, + const int64_t rowkey_column_count, + const ObStorageSchema *storage_schema, + const ObIArray &col_schema, + const int64_t dir_id, + const int64_t parallelism); + virtual int append_row(const blocksstable::ObDatumRow &datum_row) override; + virtual int close() override; + virtual int64_t get_row_count() const override; + virtual int64_t get_next_block_start_seq() const override; + void reset(); + const ObChunkSliceStore *get_column_slice_store() const { return column_slice_store_; } + TO_STRING_KV(K_(is_inited), KPC_(cs_replica_schema), KPC_(row_slice_store), KPC_(column_slice_store)); +private: + void free_memory(ObArenaAllocator &allocator); +private: + bool is_inited_; + ObArenaAllocator *arena_allocator_; + ObStorageSchema *cs_replica_schema_; + ObMacroBlockSliceStore *row_slice_store_; + ObChunkSliceStore *column_slice_store_; +}; + class ObTabletDirectLoadMgr; struct ObInsertMonitor final{ @@ -490,6 +525,14 @@ public: class ObCOSliceWriter; class ObDirectLoadSliceWriter final { +public: + enum class ObDirectLoadSliceWriterType: uint8_t + { + ROW_STORE_WRITER = 0, + COL_STORE_WRITER = 1, + COL_REPLICA_WRITER = 2, + WRITER_TYPE_MAX = 3 + }; public: ObDirectLoadSliceWriter(); ~ObDirectLoadSliceWriter(); @@ -541,11 +584,15 @@ public: int64_t get_row_offset() const { return row_offset_; } blocksstable::ObMacroDataSeq &get_start_seq() { return start_seq_; } bool is_empty() const { return 0 == get_row_count(); } - bool need_column_store() const { return need_column_store_; } + bool is_row_store_writer() const { return ObDirectLoadSliceWriterType::ROW_STORE_WRITER == writer_type_; } + bool is_col_store_writer() const { return ObDirectLoadSliceWriterType::COL_STORE_WRITER == writer_type_; } + bool is_cs_replica_write() const { return ObDirectLoadSliceWriterType::COL_REPLICA_WRITER == writer_type_; } + bool need_column_store() const { return is_col_store_writer() || is_cs_replica_write(); } ObTabletSliceStore *get_slice_store() const { return slice_store_; } + ObDirectLoadSliceWriterType get_writer_type() const { return writer_type_; } void cancel() { ATOMIC_SET(&is_canceled_, true); } int64_t get_next_block_start_seq() const { return nullptr == slice_store_ ? start_seq_.get_data_seq() /*slice empty*/ : slice_store_->get_next_block_start_seq(); } - TO_STRING_KV(K(is_inited_), K(need_column_store_), K(is_canceled_), K(start_seq_), KPC(slice_store_), K(row_offset_)); + TO_STRING_KV(K(is_inited_), K(writer_type_), K(is_canceled_), K(start_seq_), KPC(slice_store_), K(row_offset_)); private: int fill_lob_into_memtable( // for version < 4.3.0.0 ObIAllocator &allocator, @@ -601,9 +648,14 @@ private: share::ObTabletCacheInterval &pk_interval, ObLobMetaRowIterator *&row_iter); int mock_chunk_store(const int64_t row_cnt); + int inner_fill_column_group( + const ObChunkSliceStore *chunk_slice_store, + const ObStorageSchema *storage_schema, + const share::SCN &start_scn, + ObInsertMonitor *monitor_node = NULL); private: bool is_inited_; - bool need_column_store_; + ObDirectLoadSliceWriterType writer_type_; bool is_canceled_; blocksstable::ObMacroDataSeq start_seq_; ObTabletDirectLoadMgr *tablet_direct_load_mgr_; @@ -625,7 +677,8 @@ public: ObTabletDirectLoadMgr *tablet_direct_load_mgr, const blocksstable::ObMacroDataSeq &start_seq, const int64_t row_id_offset, - const share::SCN &start_scn); + const share::SCN &start_scn, + const bool with_cs_replica); void reset(); int append_row( const sql::ObChunkDatumStore::StoredRow *stored_row); diff --git a/src/storage/high_availability/ob_cs_replica_migration.cpp b/src/storage/high_availability/ob_cs_replica_migration.cpp new file mode 100644 index 000000000..1162df801 --- /dev/null +++ b/src/storage/high_availability/ob_cs_replica_migration.cpp @@ -0,0 +1,693 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#define USING_LOG_PREFIX STORAGE +#include "storage/high_availability/ob_cs_replica_migration.h" +#include "storage/column_store/ob_column_store_replica_util.h" +#include "storage/compaction/ob_tenant_tablet_scheduler.h" +#include "share/scheduler/ob_dag_warning_history_mgr.h" +#include "share/ob_debug_sync_point.h" + +namespace oceanbase +{ +namespace storage +{ +ERRSIM_POINT_DEF(EN_ALL_STATE_DETERMINISTIC_FALSE); + +/*----------------------------- ObTabletCOConvertCtx -----------------------------*/ +ObTabletCOConvertCtx::ObTabletCOConvertCtx() + : tablet_id_(), + co_dag_net_id_(), + status_(Status::MAX_STATUS), + retry_cnt_(0), + is_inited_(false) +{ +} + +ObTabletCOConvertCtx::~ObTabletCOConvertCtx() +{ + reset(); +} + +int ObTabletCOConvertCtx::init( + const ObTabletID &tablet_id, + const share::ObDagId &co_dag_net_id) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", K(ret)); + } else if (OB_UNLIKELY(!tablet_id.is_valid() || !co_dag_net_id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tablet_id ro co dag net id is invalid", K(ret), K(tablet_id), K(co_dag_net_id)); + } else { + tablet_id_ = tablet_id; + co_dag_net_id_ = co_dag_net_id; + status_ = Status::UNKNOWN; + is_inited_ = true; + } + return ret; +} + +void ObTabletCOConvertCtx::reset() +{ + tablet_id_.reset(); + co_dag_net_id_.reset(); + status_ = Status::MAX_STATUS; + retry_cnt_ = 0; + is_inited_ = false; +} + +bool ObTabletCOConvertCtx::is_valid() const +{ + return tablet_id_.is_valid() + && co_dag_net_id_.is_valid() + && status_ >= Status::UNKNOWN + && status_ < Status::MAX_STATUS + && retry_cnt_ >= 0 + && is_inited_; +} + +void ObTabletCOConvertCtx::set_progressing() +{ + int ret = OB_SUCCESS; + if (status_ != Status::RETRY_EXHAUSTED) { + status_ = Status::PROGRESSING; + } +} + +/*----------------------------- ObHATabletGroupCOConvertCtx -----------------------------*/ +ObHATabletGroupCOConvertCtx::ObHATabletGroupCOConvertCtx() + : ObHATabletGroupCtx(TabletGroupCtxType::CS_REPLICA_TYPE), + finish_migration_cnt_(0), + finish_check_cnt_(0), + retry_exhausted_cnt_(0), + idx_map_(), + convert_ctxs_() +{ +} + +ObHATabletGroupCOConvertCtx::~ObHATabletGroupCOConvertCtx() +{ + common::SpinWLockGuard guard(lock_); + if (idx_map_.created()) { + idx_map_.destroy(); + } +} + +void ObHATabletGroupCOConvertCtx::reuse() +{ + common::SpinWLockGuard guard(lock_); + inner_reuse(); + ObHATabletGroupCtx::inner_reuse(); +} + +void ObHATabletGroupCOConvertCtx::inner_reuse() +{ + finish_migration_cnt_ = 0; + finish_check_cnt_ = 0; + retry_exhausted_cnt_ = 0; + if (idx_map_.created()) { + idx_map_.destroy(); + } + convert_ctxs_.reuse(); +} + +int ObHATabletGroupCOConvertCtx::inner_init() +{ + int ret = OB_SUCCESS; + finish_migration_cnt_ = 0; + finish_check_cnt_ = 0; + retry_exhausted_cnt_ = 0; + const int64_t count = tablet_id_array_.count(); + if (OB_UNLIKELY(idx_map_.created() || !convert_ctxs_.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("idx map is created or convert ctxs not empty", K(ret), KPC(this)); + } else if (OB_FAIL(idx_map_.create(TABLET_CONVERT_CTX_MAP_BUCKED_NUM, ObMemAttr(MTL_ID(), "HATGCOCtx")))) { + LOG_WARN("failed to create tablet convert ctx idx map", K(ret)); + } else if (OB_FAIL(convert_ctxs_.reserve(count))) { + LOG_WARN("failed to reserve convert ctxs", K(ret), K(count)); + } else { + for (int64_t idx = 0; OB_SUCC(ret) && idx < count; ++idx) { + const ObTabletID &tablet_id = tablet_id_array_.at(idx).tablet_id_; + ObDagId co_dag_net_id; + co_dag_net_id.init(GCTX.self_addr()); + ObTabletCOConvertCtx co_convert_ctx; + if (OB_FAIL(co_convert_ctx.init(tablet_id, co_dag_net_id))) { + LOG_WARN("failed to init co convert ctx", K(ret), K(tablet_id), K(co_dag_net_id)); + } else if (OB_FAIL(convert_ctxs_.push_back(co_convert_ctx))) { + LOG_WARN("failed to push back co convert ctx", K(ret), K(co_convert_ctx)); + } else if (OB_FAIL(idx_map_.set_refactored(tablet_id, idx))) { + LOG_WARN("failed to set ctx idx into map", K(ret), K(tablet_id), K(idx)); + } + } + } + return ret; +} + +void ObHATabletGroupCOConvertCtx::inc_finish_migration_cnt() +{ + common::SpinWLockGuard guard(lock_); + finish_migration_cnt_++; +} + +bool ObHATabletGroupCOConvertCtx::ready_to_check() const +{ + common::SpinRLockGuard guard(lock_); + bool bret = false; + if (finish_migration_cnt_ < convert_ctxs_.count()) { + } else if (finish_migration_cnt_ > convert_ctxs_.count()) { + LOG_ERROR_RET(OB_ERR_UNEXPECTED, "invalid finish migration cnt", KPC(this)); + } else { + int ret = OB_SUCCESS; // ignore ret + for (int64_t idx = 0; OB_SUCC(ret) && idx < tablet_id_array_.count(); ++idx) { + const ObTabletID &tablet_id = tablet_id_array_[idx].tablet_id_; + bool is_exist = true; + int64_t ctx_idx = 0; + if (OB_FAIL(inner_get_valid_convert_ctx_idx(tablet_id, ctx_idx))) { + LOG_WARN("failed to get convert ctx idx", K(ret), K(tablet_id)); + } else if (convert_ctxs_[ctx_idx].is_progressing()) { + if (OB_FAIL(MTL(ObTenantDagScheduler *)->check_dag_net_exist(convert_ctxs_[ctx_idx].co_dag_net_id_, is_exist))) { + LOG_WARN("failed to check dag exists", K(ret), K(tablet_id), K(convert_ctxs_[ctx_idx])); + } else if (!is_exist) { + bret = true; + break; + } + } + } + } + LOG_TRACE("[CS-Replica] check ready to check", K(bret), KPC(this)); + return bret; +} + +bool ObHATabletGroupCOConvertCtx::is_all_state_deterministic() const +{ + common::SpinRLockGuard guard(lock_); + return convert_ctxs_.count() <= (finish_check_cnt_ + retry_exhausted_cnt_); +} + +int ObHATabletGroupCOConvertCtx::set_convert_status(const ObTabletID &tablet_id, const ObTabletCOConvertCtx::Status status) +{ + int ret = OB_SUCCESS; + int64_t idx = 0; + common::SpinWLockGuard guard(lock_); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_FAIL(inner_get_valid_convert_ctx_idx(tablet_id, idx))) { + LOG_WARN("failed to get convert ctx idx", K(ret), K(tablet_id)); + } else if (ObTabletCOConvertCtx::Status::FINISHED == status) { + inner_set_convert_finish(convert_ctxs_[idx]); + } else if (ObTabletCOConvertCtx::Status::RETRY_EXHAUSTED == status) { + inner_set_retry_exhausted(convert_ctxs_[idx]); + } else if (ObTabletCOConvertCtx::Status::PROGRESSING == status) { + convert_ctxs_[idx].set_progressing(); + } else { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid status to set", K(ret), K(tablet_id), K(status), K(convert_ctxs_[idx])); + } + return ret; +} + +int ObHATabletGroupCOConvertCtx::get_co_dag_net_id(const ObTabletID &tablet_id, share::ObDagId &co_dag_net_id) const +{ + int ret = OB_SUCCESS; + int64_t idx = 0; + common::SpinRLockGuard guard(lock_); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_FAIL(inner_get_valid_convert_ctx_idx(tablet_id, idx))) { + LOG_WARN("failed to get convert ctx idx", K(ret), K(tablet_id)); + } else { + co_dag_net_id = convert_ctxs_[idx].co_dag_net_id_; + } + return ret; +} + +int ObHATabletGroupCOConvertCtx::check_and_schedule(ObLS &ls) +{ + common::SpinWLockGuard guard(lock_); + int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else { + const int64_t count = tablet_id_array_.count(); + for (int64_t idx = 0; idx < count; ++idx) { + if (OB_TMP_FAIL(inner_check_and_schedule(ls, tablet_id_array_[idx].tablet_id_))) { + LOG_WARN("failed to check and schedule", K(tmp_ret)); + } + } + } + return ret; +} + +int ObHATabletGroupCOConvertCtx::check_need_convert(const ObTablet &tablet, bool &need_convert) +{ + int ret = OB_SUCCESS; + need_convert = false; + common::ObArenaAllocator tmp_allocator; // for schema_on_tablet + ObStorageSchema *schema_on_tablet = nullptr; + if (OB_FAIL(tablet.load_storage_schema(tmp_allocator, schema_on_tablet))) { + LOG_WARN("failed to load storage schema", K(ret),K(tablet)); + } else { + need_convert = ObCSReplicaUtil::check_need_convert_cs_when_migration(tablet, *schema_on_tablet); + } + + if (OB_NOT_NULL(schema_on_tablet)) { + schema_on_tablet->~ObStorageSchema(); + tmp_allocator.free(schema_on_tablet); + schema_on_tablet = nullptr; + } + return ret; +} + +int ObHATabletGroupCOConvertCtx::update_deleted_data_tablet_status( + ObHATabletGroupCtx *tablet_group_ctx, + const ObTabletID &tablet_id) +{ + int ret = OB_SUCCESS; + ObHATabletGroupCOConvertCtx *group_convert_ctx = nullptr; + if (OB_UNLIKELY(OB_ISNULL(tablet_group_ctx) + || !tablet_group_ctx->is_cs_replica_ctx())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tablet group ctx", K(ret), K(tablet_id), KPC(tablet_group_ctx)); + } else if (FALSE_IT(group_convert_ctx = static_cast(tablet_group_ctx))) { + } else if (OB_FAIL(group_convert_ctx->set_convert_finsih(tablet_id))) { + LOG_WARN("failed to set convert finish", K(ret), K(tablet_id)); + } else { + (void) group_convert_ctx->inc_finish_migration_cnt(); + } + return ret; +} + +void ObHATabletGroupCOConvertCtx::inner_set_convert_finish(ObTabletCOConvertCtx &convert_ctx) +{ + convert_ctx.set_finished(); + finish_check_cnt_++; +} + +void ObHATabletGroupCOConvertCtx::inner_set_retry_exhausted(ObTabletCOConvertCtx &convert_ctx) +{ + convert_ctx.set_retry_exhausted(); + retry_exhausted_cnt_++; +} + +int ObHATabletGroupCOConvertCtx::inner_get_valid_convert_ctx_idx(const ObTabletID &tablet_id, int64_t &idx) const +{ + int ret = OB_SUCCESS; + idx = 0; + if (OB_UNLIKELY(!tablet_id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tablet_id", K(ret), K(tablet_id)); + } else if (OB_FAIL(idx_map_.get_refactored(tablet_id, idx))) { + LOG_WARN("failed to get convert ctx idx", K(ret), K(tablet_id)); + } else if (OB_UNLIKELY(idx < 0 || idx >= convert_ctxs_.count())) { + ret = OB_INDEX_OUT_OF_RANGE; + LOG_WARN("convert ctx idx is invalid", K(ret), K(idx), K(tablet_id)); + } else if (OB_UNLIKELY(!convert_ctxs_[idx].is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("convert ctx is null or invalid", K(ret), K(convert_ctxs_[idx])); + } + return ret; +} + +int ObHATabletGroupCOConvertCtx::inner_check_and_schedule(ObLS &ls, const ObTabletID &tablet_id) +{ + int ret = OB_SUCCESS; + int64_t idx = 0; + const ObLSID &ls_id = ls.get_ls_id(); + ObTabletHandle tablet_handle; + ObTablet *tablet = nullptr; + bool need_convert = false; + bool is_dag_net_exist = false; + + if (OB_FAIL(inner_get_valid_convert_ctx_idx(tablet_id, idx))) { + LOG_WARN("failed to get convert ctx idx", K(ret), K(ls_id), K(tablet_id)); + } else if (convert_ctxs_[idx].is_finished() || convert_ctxs_[idx].is_retry_exhausted()) { + } else if (OB_FAIL(ls.get_tablet(tablet_id, tablet_handle))) { + if (OB_TABLET_NOT_EXIST == ret) { + LOG_INFO("tablet maybe deleted, skip it", K(ret), K(ls_id), K(tablet_id)); + inner_set_convert_finish(convert_ctxs_[idx]); + ret = OB_SUCCESS; + } else { + LOG_WARN("failed to get tablet handle", K(ret), K(ls_id), K(tablet_id)); + } + } else if (OB_ISNULL(tablet = tablet_handle.get_obj())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tablet should not be null", K(ret), K(ls_id), K(tablet_id)); + } else if (convert_ctxs_[idx].is_unknown()) { + // tablet in cs replcia != tablet need convert to column store, need take storage schema into consideration + if (OB_FAIL(check_need_convert(*tablet, need_convert))) { + LOG_WARN("failed to check need convert", K(ret), K(ls_id), K(tablet_id)); + } else if (need_convert) { + if (tablet->get_tablet_meta().ha_status_.is_data_status_complete()) { + convert_ctxs_[idx].set_progressing(); + } + } else { + inner_set_convert_finish(convert_ctxs_[idx]); + } + } + + if (OB_FAIL(ret)) { + } else if (!convert_ctxs_[idx].is_progressing()) { + } else if (OB_ISNULL(tablet)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("co_convert_ctx is progressing, but tablet is nullptr", K(ret), K(tablet_id), K(ls_id), K(need_convert), K(idx), K(convert_ctxs_[idx]), KPC(this)); + } else if (!tablet->is_row_store()) { + inner_set_convert_finish(convert_ctxs_[idx]); + LOG_INFO("[CS-Replica] Finish co merge dag net for switching column store", K(ls_id), K(tablet_id), K(tablet_handle)); + } else if (OB_FAIL(MTL(ObTenantDagScheduler *)->check_dag_net_exist(convert_ctxs_[idx].co_dag_net_id_, is_dag_net_exist))) { + LOG_WARN("failed to check dag exists", K(ret), K(convert_ctxs_[idx]), K(tablet_id)); + } else if (is_dag_net_exist) { + } else if (OB_FAIL(compaction::ObTenantTabletScheduler::schedule_convert_co_merge_dag_net(ls_id, *tablet, convert_ctxs_[idx].retry_cnt_, convert_ctxs_[idx].co_dag_net_id_))) { + LOG_WARN("failed to schedule convert co merge", K(ret), K(ls_id), K(tablet_id)); + } else { + convert_ctxs_[idx].inc_retry_cnt(); + if (convert_ctxs_[idx].is_retry_exhausted()) { + inner_set_retry_exhausted(convert_ctxs_[idx]); +#ifdef ERRSIM + LOG_INFO("[CS-Replica] set tablet co convert retry exhausted", K(ret), K(idx), K(tablet_id)); + DEBUG_SYNC(AFTER_SET_CO_CONVERT_RETRY_EXHUASTED); +#endif + } + } + + return ret; +} + +/*----------------------------- ObDataTabletsCheckCOConvertDag -----------------------------*/ +ObDataTabletsCheckCOConvertDag::ObDataTabletsCheckCOConvertDag() + : ObMigrationDag(ObDagType::DAG_TYPE_TABLET_CHECK_CONVERT), + ls_(nullptr), + first_start_time_(0), + is_inited_(false) +{ +} + +ObDataTabletsCheckCOConvertDag::~ObDataTabletsCheckCOConvertDag() +{ +} + +bool ObDataTabletsCheckCOConvertDag::check_can_schedule() +{ + int ret = OB_SUCCESS; + bool bret = false; + ObMigrationCtx *migration_ctx = nullptr; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_ISNULL(ha_dag_net_ctx_) || OB_UNLIKELY(ObIHADagNetCtx::LS_MIGRATION != ha_dag_net_ctx_->get_dag_net_ctx_type())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ha dag net ctx is null or unexpected type", K(ret), KPC(ha_dag_net_ctx_)); + } else if (OB_ISNULL(migration_ctx = get_migration_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("migration ctx is null", K(ret)); + } else if (OB_FAIL(inner_check_can_schedule(*migration_ctx, bret))) { + LOG_WARN("failed to check can schedule", K(ret)); + } + if (OB_FAIL(ret)) { + bret = true; + LOG_INFO("failed to check can schedule, allow dag running", K(ret), KPC_(ha_dag_net_ctx)); + } + return bret; +} + +int ObDataTabletsCheckCOConvertDag::inner_check_can_schedule( + ObMigrationCtx &migration_ctx, + bool &can_schedule) +{ + int ret = OB_SUCCESS; + bool all_state_deterministic = true; // all finish check or retry exhausted + ObCheckScheduleReason reason = ObCheckScheduleReason::MAX_NOT_SCHEDULE; + // time for diagnose. if dag has not be scheduled, start_time_ is 0 + first_start_time_ = (0 == first_start_time_) ? start_time_ : first_start_time_; + const int64_t current_time = ObTimeUtility::current_time(); + const int64_t wait_one_round_time = (0 == start_time_) ? current_time - add_time_ : current_time - start_time_; + const int64_t total_wait_time = current_time - first_start_time_; + + if (migration_ctx.is_failed()) { + // migration dag net failed, no need to check anymore + can_schedule = true; + reason = ObCheckScheduleReason::MIGRATION_FAILED; +#ifdef ERRSIM + LOG_INFO("migration dag net failed, make check dag schedule"); +#endif + } else { + const int64_t tablet_group_cnt = migration_ctx.tablet_group_mgr_.get_tablet_group_ctx_count(); + ObHATabletGroupCtx *ctx = nullptr; + ObHATabletGroupCOConvertCtx *group_convert_ctx = nullptr; + for (int64_t idx = 0; OB_SUCC(ret) && idx < tablet_group_cnt; ++idx) { + if (OB_FAIL(migration_ctx.tablet_group_mgr_.get_tablet_group_ctx(idx, ctx))) { + LOG_WARN("failed to get tablet group ctx", K(ret), K(idx)); + } else if (OB_ISNULL(ctx) || OB_UNLIKELY(!ctx->is_cs_replica_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ctx is null or invalid type", K(ret), KPC(ctx), K(migration_ctx)); + } else if (FALSE_IT(group_convert_ctx = static_cast(ctx))) { + } else if (group_convert_ctx->is_all_state_deterministic()) { + } else if (FALSE_IT(all_state_deterministic = false)) { + } else if (group_convert_ctx->ready_to_check()) { + can_schedule = true; + reason = ObCheckScheduleReason::READY_TO_CHECK; + break; + } + } + } + + if (!can_schedule) { + if (all_state_deterministic) { + can_schedule = true; + reason = ObCheckScheduleReason::ALL_DETERMINISTIC; + } else if (wait_one_round_time > OB_DATA_TABLETS_NOT_CHECK_CONVERT_THRESHOLD) { + can_schedule = true; + reason = ObCheckScheduleReason::WAIT_TIME_EXCEED; + } + } + + const int64_t cost_time = ObTimeUtility::current_time() - current_time; + if (REACH_TENANT_TIME_INTERVAL(OB_DATA_TABLETS_NOT_CHECK_CONVERT_THRESHOLD)) { + LOG_INFO("[CS-Replica] finish check_can_schedule", K(ret), K(can_schedule), K(reason), K(wait_one_round_time), K(total_wait_time), K(cost_time), KPC(this), K(migration_ctx.tablet_group_mgr_)); + } else { + LOG_TRACE("[CS-Replica] finish check_can_schedule", K(ret), K(can_schedule), K(reason), K(wait_one_round_time), K(total_wait_time), K(cost_time), KPC(this), K(migration_ctx.tablet_group_mgr_)); + } + return ret; +} + +int ObDataTabletsCheckCOConvertDag::init( + ObIHADagNetCtx *ha_dag_net_ctx, + ObLS *ls) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("can not init twice", K(ret)); + } else if (OB_UNLIKELY(OB_ISNULL(ha_dag_net_ctx) || OB_ISNULL(ls))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument to init ObDataTabletsCheckCOConvertDag", + K(ret), KPC(ha_dag_net_ctx), KPC(ls)); + } else if (OB_FAIL(check_convert_ctx_valid(ha_dag_net_ctx))) { + LOG_WARN("ha dag net ctx is invalid", K(ret), KPC(ha_dag_net_ctx)); + } else { + ha_dag_net_ctx_ = ha_dag_net_ctx; + ls_ = ls; + is_inited_ = true; + } + return ret; +} + +int ObDataTabletsCheckCOConvertDag::check_convert_ctx_valid(ObIHADagNetCtx *ha_dag_net_ctx) +{ + int ret = OB_SUCCESS; + ObMigrationCtx *migration_ctx = nullptr; + if (OB_UNLIKELY(OB_ISNULL(ha_dag_net_ctx) + || ObIHADagNetCtx::DagNetCtxType::LS_MIGRATION != ha_dag_net_ctx->get_dag_net_ctx_type())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid ha_dag_net_ctx", K(ret), KPC(ha_dag_net_ctx)); + } else if (OB_ISNULL(migration_ctx = static_cast(ha_dag_net_ctx))) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("migration ctx should not be NULL", K(ret), KP(migration_ctx)); + } else { + const int64_t count = migration_ctx->tablet_group_mgr_.get_tablet_group_ctx_count(); + ObHATabletGroupCtx *ctx = nullptr; + for (int64_t idx = 0; OB_SUCC(ret) && idx < count; ++idx) { + if (OB_FAIL(migration_ctx->tablet_group_mgr_.get_tablet_group_ctx(idx, ctx))) { + LOG_WARN("failed to get tablet group ctx", K(ret), K(idx)); + } else if (OB_ISNULL(ctx) || OB_UNLIKELY(!ctx->is_cs_replica_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ctx is null or not cs replica ctx", K(ret), KPC(ctx)); + } + } + } + return ret; +} + +int ObDataTabletsCheckCOConvertDag::create_first_task() +{ + int ret = OB_SUCCESS; + ObDataTabletsCheckConvertTask *task = nullptr; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("tablet check convert dag not init", K(ret)); + } else if (OB_FAIL(create_task(nullptr /*parent*/, task, ha_dag_net_ctx_, ls_))) { + LOG_WARN("failed to create tablet check convert task", K(ret)); + } else { + LOG_INFO("[CS-Replica] Success to create tablet check convert task", K(ret), KPC(this), KPC(task)); + } + return ret; +} + +bool ObDataTabletsCheckCOConvertDag::operator == (const ObIDag &other) const +{ + bool is_same = true; + if (this == &other) { + // same + } else if (get_type() != other.get_type() || ObDagType::DAG_TYPE_TABLET_CHECK_CONVERT != other.get_type()) { + is_same = false; + } else { + const ObDataTabletsCheckCOConvertDag &other_dag = static_cast(other); + ObMigrationCtx *ctx = get_migration_ctx(); + if (OB_ISNULL(ctx) || OB_ISNULL(other_dag.get_migration_ctx())) { + is_same = false; + } else { + is_same = ctx->arg_.ls_id_ == other_dag.get_migration_ctx()->arg_.ls_id_; + } + } + return is_same; +} + +int64_t ObDataTabletsCheckCOConvertDag::hash() const +{ + int64_t hash_value = 0; + ObMigrationCtx *ctx = nullptr; + if (IS_NOT_INIT) { + LOG_ERROR_RET(OB_NOT_INIT, "tablet check convert dag not init"); + } else if (OB_ISNULL(ctx = get_migration_ctx())) { + LOG_ERROR_RET(OB_ERR_UNEXPECTED, "migration ctx should not be NULL", KP(ctx)); + } else { + hash_value = common::murmurhash(&ctx->arg_.ls_id_, sizeof(ctx->arg_.ls_id_), hash_value); + ObDagType::ObDagTypeEnum dag_type = get_type(); + hash_value = common::murmurhash(&dag_type, sizeof(dag_type), hash_value); + } + return hash_value; +} + +int ObDataTabletsCheckCOConvertDag::fill_dag_key(char *buf, const int64_t buf_len) const +{ + int ret = OB_SUCCESS; + ObMigrationCtx *ctx = nullptr; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("tablet check convert dag not init", K(ret)); + } else if (OB_ISNULL(ctx = get_migration_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("migration ctx should not be NULL", K(ret), KP(ctx)); + } else if (OB_FAIL(databuff_printf(buf, buf_len, + "ObDataTabletsCheckCOConvertDag: ls_id = %s, migration_type = %s, dag_prio = %s", + to_cstring(ctx->arg_.ls_id_), ObMigrationOpType::get_str(ctx->arg_.type_), + ObIDag::get_dag_prio_str(this->get_priority())))) { + LOG_WARN("failed to fill comment", K(ret), KPC(ctx)); + } + return ret; +} + +/*----------------------------- ObDataTabletsCheckConvertTask -----------------------------*/ +ObDataTabletsCheckConvertTask::ObDataTabletsCheckConvertTask() + : ObITask(ObITask::TASK_TYPE_CHECK_CONVERT_TABLET), + is_inited_(false), + ctx_(nullptr), + ls_(nullptr) +{} + +ObDataTabletsCheckConvertTask::~ObDataTabletsCheckConvertTask() +{} + +int ObDataTabletsCheckConvertTask::init( + ObIHADagNetCtx *ha_dag_net_ctx, + ObLS *ls) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("can not init twice", K(ret)); + } else if (OB_UNLIKELY(OB_ISNULL(ha_dag_net_ctx) || OB_ISNULL(ls) + || ObIHADagNetCtx::LS_MIGRATION != ha_dag_net_ctx->get_dag_net_ctx_type())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument to init ObDataTabletsCheckConvertTask", + K(ret), KPC(ha_dag_net_ctx), KPC(ls)); + } else { + ctx_ = static_cast(ha_dag_net_ctx); + ls_ = ls; + is_inited_ = true; + } + return ret; +} + +int ObDataTabletsCheckConvertTask::process() +{ + int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + bool all_state_deterministic = true; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("task not init", K(ret)); + } else if (ctx_->is_failed()) { + // do nothing + } else if (OB_ISNULL(ls_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls is unexpected null", K(ret), KPC_(ls)); + } else { + const int64_t count = ctx_->tablet_group_mgr_.get_tablet_group_ctx_count(); + ObHATabletGroupCtx *group_ctx = nullptr; + ObHATabletGroupCOConvertCtx *group_convert_ctx = nullptr; + for (int64_t idx = 0; OB_SUCC(ret) && idx < count; ++idx) { + if (OB_FAIL(ctx_->tablet_group_mgr_.get_tablet_group_ctx(idx, group_ctx))) { + LOG_WARN("failed to get tablet group ctx", K(ret), K(idx)); + } else if (OB_ISNULL(group_ctx) || OB_UNLIKELY(!group_ctx->is_cs_replica_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("group_ctx is null or not cs replica ctx", K(ret), KPC(group_ctx)); + } else if (FALSE_IT(group_convert_ctx = static_cast(group_ctx))) { + } else if (OB_TMP_FAIL(group_convert_ctx->check_and_schedule(*ls_))) { + LOG_WARN("failed to check and schedule", K(tmp_ret), KPC(group_convert_ctx)); + } else if (group_convert_ctx->is_all_state_deterministic()) { +#ifdef ERRSIM + if (EN_ALL_STATE_DETERMINISTIC_FALSE) { + all_state_deterministic = false; + LOG_INFO("ERRSIM EN_ALL_STATE_DETERMINISTIC_FALSE make new round check", K(ret), K(all_state_deterministic)); + } +#endif + } else { + all_state_deterministic = false; + } + } + } + + if (OB_FAIL(ret)) { + if (OB_TMP_FAIL(ObStorageHADagUtils::deal_with_fo(ret, this->get_dag(), true /*alllow_retry*/))) { + LOG_WARN("failed to deal with fo", K(ret), K(tmp_ret), KPC_(ctx)); + } + } else if (ctx_->is_failed()) { +#ifdef ERRSIM + LOG_INFO("migration dag net failed, make check dag exit"); +#endif + } else if (!all_state_deterministic) { + ret = OB_EAGAIN; + LOG_WARN("not wait all tablets convert finish, failed this task, make dag retry", K(ret), K(all_state_deterministic), KPC_(ctx)); + } + LOG_TRACE("[CS-Replica] Finish process check data tablets convert to column store", K(ret), KPC_(ls), KPC_(ctx)); + return ret; +} + +} // namespace storage +} // namespace oceanbase \ No newline at end of file diff --git a/src/storage/high_availability/ob_cs_replica_migration.h b/src/storage/high_availability/ob_cs_replica_migration.h new file mode 100644 index 000000000..e30502e32 --- /dev/null +++ b/src/storage/high_availability/ob_cs_replica_migration.h @@ -0,0 +1,160 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEABASE_STORAGE_CS_REPLICA_MIGRATION_ +#define OCEABASE_STORAGE_CS_REPLICA_MIGRATION_ + +#include "storage/high_availability/ob_ls_migration.h" + +namespace oceanbase +{ +namespace storage +{ + +struct ObTabletCOConvertCtx +{ +public: + enum class Status + { + UNKNOWN = 0, // intial state, need take tablet_id and storage schema into consideration + PROGRESSING = 1, // need convert and to be check result + FINISHED = 2, // finish convert, or tablet is deleted + RETRY_EXHAUSTED = 3, // retry times >= MAX_RETRY_CNT + MAX_STATUS + }; +public: + ObTabletCOConvertCtx(); + virtual ~ObTabletCOConvertCtx(); + int init(const ObTabletID &tablet_id, const share::ObDagId &co_dag_net_id); + void reset(); + bool is_valid() const; + TO_STRING_KV(K_(tablet_id), K_(co_dag_net_id), K_(status), K_(retry_cnt), K_(is_inited)); +public: + OB_INLINE bool is_unknown() const { return Status::UNKNOWN == status_; } + OB_INLINE bool is_progressing() const { return Status::PROGRESSING == status_; } + OB_INLINE bool is_finished() const { return Status::FINISHED == status_; } + OB_INLINE bool is_retry_exhausted() const { return retry_cnt_ >= MAX_RETRY_CNT; } + void set_progressing(); + OB_INLINE void set_finished() { status_ = Status::FINISHED; } + OB_INLINE void set_retry_exhausted() { status_ = Status::RETRY_EXHAUSTED; } + OB_INLINE void inc_retry_cnt() { retry_cnt_++; } +public: + const static int64_t MAX_RETRY_CNT = 3; +public: + ObTabletID tablet_id_; + share::ObDagId co_dag_net_id_; + Status status_; + int64_t retry_cnt_; + bool is_inited_; +}; + +class ObHATabletGroupCOConvertCtx : public ObHATabletGroupCtx +{ +public: + ObHATabletGroupCOConvertCtx(); + virtual ~ObHATabletGroupCOConvertCtx(); +public: + virtual void reuse() override; + virtual void inner_reuse() override; + virtual int inner_init() override; + void inc_finish_migration_cnt(); + bool ready_to_check() const; + bool is_all_state_deterministic() const; + int set_convert_status(const ObTabletID &tablet_id, const ObTabletCOConvertCtx::Status status); + OB_INLINE int set_convert_finsih(const ObTabletID &tablet_id) { return set_convert_status(tablet_id, ObTabletCOConvertCtx::Status::FINISHED); } + OB_INLINE int set_convert_progressing(const ObTabletID &tablet_id) { return set_convert_status(tablet_id, ObTabletCOConvertCtx::Status::PROGRESSING); } + int get_co_dag_net_id(const ObTabletID &tablet_id, share::ObDagId &co_dag_net_id) const; + int check_and_schedule(ObLS &ls); + INHERIT_TO_STRING_KV("ObHATabletGroupCtx", ObHATabletGroupCtx, K_(finish_migration_cnt), K_(finish_check_cnt), K_(retry_exhausted_cnt), "map_size", idx_map_.size(), K_(convert_ctxs)); +public: + static int check_need_convert(const ObTablet &tablet, bool &need_convert); + static int update_deleted_data_tablet_status( + ObHATabletGroupCtx *tablet_group_ctx, + const ObTabletID &tablet_id); +private: + void inner_set_convert_finish(ObTabletCOConvertCtx &convert_ctx); + void inner_set_retry_exhausted(ObTabletCOConvertCtx &convert_ctx); + int inner_get_valid_convert_ctx_idx(const ObTabletID &tablet_id, int64_t &idx) const; + int inner_check_and_schedule(ObLS &ls, const ObTabletID &tablet_id); +private: + // refer to TransferTableMap + const static int64_t TABLET_CONVERT_CTX_MAP_BUCKED_NUM = 128; // a tablet group contains 2G size of tablets, 1024 macro block + typedef common::hash::ObHashMap TabletConvertCtxIndexMap; +public: + int64_t finish_migration_cnt_; + int64_t finish_check_cnt_; + int64_t retry_exhausted_cnt_; + TabletConvertCtxIndexMap idx_map_; + ObArray convert_ctxs_; + DISALLOW_COPY_AND_ASSIGN(ObHATabletGroupCOConvertCtx); +}; + +class ObDataTabletsCheckCOConvertDag : public ObMigrationDag +{ +public: + enum class ObCheckScheduleReason { + MIGRATION_FAILED = 0, + READY_TO_CHECK = 1, + ALL_DETERMINISTIC = 2, + WAIT_TIME_EXCEED = 3, + MAX_NOT_SCHEDULE = 4, + }; +public: + ObDataTabletsCheckCOConvertDag(); + virtual ~ObDataTabletsCheckCOConvertDag(); + virtual bool check_can_schedule() override; + virtual int create_first_task() override; + int init( + ObIHADagNetCtx *ha_dag_net_ctx, + ObLS *ls); + int check_convert_ctx_valid(ObIHADagNetCtx *ha_dag_net_ctx); +public: + virtual bool operator == (const share::ObIDag &other) const override; + virtual int64_t hash() const override; + virtual int fill_dag_key(char *buf, const int64_t buf_len) const override; + INHERIT_TO_STRING_KV("ObIMigrationDag", ObMigrationDag, KP(this), KPC_(ls), K_(first_start_time), K_(is_inited)); +private: + int inner_check_can_schedule(ObMigrationCtx &migration_ctx, bool &can_schedule); +public: +#ifdef ERRSIM + const static int64_t OB_DATA_TABLETS_NOT_CHECK_CONVERT_THRESHOLD = 30 * 1000 * 1000; /*30s*/ +#else + const static int64_t OB_DATA_TABLETS_NOT_CHECK_CONVERT_THRESHOLD = 10 * 60 * 1000 * 1000; /*10min*/ +#endif +private: + ObLS *ls_; + int64_t first_start_time_; + bool is_inited_; + DISALLOW_COPY_AND_ASSIGN(ObDataTabletsCheckCOConvertDag); +} ; + +class ObDataTabletsCheckConvertTask : public ObITask +{ +public: + ObDataTabletsCheckConvertTask(); + virtual ~ObDataTabletsCheckConvertTask(); + int init( + ObIHADagNetCtx *ha_dag_net_ctx, + ObLS *ls); +private: + virtual int process() override; +private: + bool is_inited_; + ObMigrationCtx *ctx_; + ObLS *ls_; + DISALLOW_COPY_AND_ASSIGN(ObDataTabletsCheckConvertTask); +}; + +} // namespace storage +} // namespace oceanbase + +#endif \ No newline at end of file diff --git a/src/storage/high_availability/ob_ls_complete_migration.cpp b/src/storage/high_availability/ob_ls_complete_migration.cpp index 75f3510da..e0723eee5 100644 --- a/src/storage/high_availability/ob_ls_complete_migration.cpp +++ b/src/storage/high_availability/ob_ls_complete_migration.cpp @@ -1444,7 +1444,7 @@ int ObStartCompleteMigrationTask::change_member_list_() LOG_WARN("failed to switch learner to acceptor", K(ret), K(leader_addr), K(ls_transfer_scn)); } } else { - // R-replica + // R-replica, C-replica if (OB_FAIL(replace_learners_for_add_(ls))) { LOG_WARN("failed to replace learners for add", K(ret), K(leader_addr), K(ls_transfer_scn)); } @@ -1455,7 +1455,7 @@ int ObStartCompleteMigrationTask::change_member_list_() LOG_WARN("failed to replace member with learner", K(ret), K(leader_addr), K(ls_transfer_scn)); } } else { - // R-replica + // R-replica, C-replica if (OB_FAIL(replace_learners_for_migration_(ls))) { LOG_WARN("failed to replace learners for migration", K(ret), K(leader_addr), K(ls_transfer_scn)); } diff --git a/src/storage/high_availability/ob_ls_migration.cpp b/src/storage/high_availability/ob_ls_migration.cpp index b773059db..5bac1cb7e 100644 --- a/src/storage/high_availability/ob_ls_migration.cpp +++ b/src/storage/high_availability/ob_ls_migration.cpp @@ -18,6 +18,8 @@ #include "share/scheduler/ob_dag_warning_history_mgr.h" #include "storage/tablet/ob_tablet_common.h" #include "storage/tx_storage/ob_ls_service.h" +#include "storage/compaction/ob_tenant_tablet_scheduler.h" +#include "storage/column_store/ob_column_store_replica_util.h" #include "logservice/ob_log_service.h" #include "lib/hash/ob_hashset.h" #include "lib/time/ob_time_utility.h" @@ -30,6 +32,7 @@ #include "share/ob_cluster_version.h" #include "ob_storage_ha_utils.h" #include "ob_storage_ha_src_provider.h" +#include "ob_cs_replica_migration.h" namespace oceanbase { @@ -41,6 +44,7 @@ ERRSIM_POINT_DEF(EN_DATA_TABLETS_MIGRATION_TASK_FATAL_FAILURE); ERRSIM_POINT_DEF(EN_BUILD_SYS_TABLETS_DAG_FAILED); ERRSIM_POINT_DEF(EN_UPDATE_LS_MIGRATION_STATUS_FAILED); ERRSIM_POINT_DEF(EN_JOIN_LEARNER_LIST_FAILED); +ERRSIM_POINT_DEF(EN_DATA_TABLET_MIGRATION_DAG_OUT_OF_RETRY); /******************ObMigrationCtx*********************/ ObMigrationCtx::ObMigrationCtx() @@ -2045,7 +2049,8 @@ ObTabletMigrationDag::ObTabletMigrationDag() is_inited_(false), ls_handle_(), copy_tablet_ctx_(), - tablet_group_ctx_(nullptr) + tablet_group_ctx_(nullptr), + tablet_type_(ObTabletType::MAX_TYPE) { } @@ -2126,7 +2131,8 @@ int ObTabletMigrationDag::init( const common::ObTabletID &tablet_id, ObTabletHandle &tablet_handle, ObIDagNet *dag_net, - ObHATabletGroupCtx *tablet_group_ctx) + ObHATabletGroupCtx *tablet_group_ctx /*=nullptr*/, + ObTabletType tablet_type /*=ObTabletType::SYS_TABLET_TYPE*/) { int ret = OB_SUCCESS; ObMigrationDagNet *migration_dag_net = nullptr; @@ -2162,11 +2168,40 @@ int ObTabletMigrationDag::init( } else { compat_mode_ = copy_tablet_ctx_.tablet_handle_.get_obj()->get_tablet_meta().compat_mode_; tablet_group_ctx_ = tablet_group_ctx; + tablet_type_ = tablet_type; is_inited_ = true; } return ret; } +int ObTabletMigrationDag::get_tablet_group_ctx(ObHATabletGroupCtx *&tablet_group_ctx) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("tablet migration dag do not init", K(ret)); + } else if (OB_ISNULL(tablet_group_ctx_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("only data tablet has tablet group ctx", K(ret), K_(tablet_type)); + } else { + tablet_group_ctx = tablet_group_ctx_; + } + return ret; +} + +int ObTabletMigrationDag::check_is_migrate_data_tablet(bool &is_migrate_data_tablet) +{ + int ret = OB_SUCCESS; + is_migrate_data_tablet = false; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("tablet migration dag do not init", K(ret)); + } else if (ObTabletType::DATA_TABLET_TYPE == tablet_type_) { + is_migrate_data_tablet = true; + } + return ret; +} + int ObTabletMigrationDag::create_first_task() { int ret = OB_SUCCESS; @@ -2279,6 +2314,10 @@ int ObTabletMigrationDag::generate_next_dag(share::ObIDag *&dag) } else if (OB_FAIL(ls->ha_get_tablet(logic_tablet_id.tablet_id_, tablet_handle))) { if (OB_TABLET_NOT_EXIST == ret) { ret = OB_SUCCESS; + if (ls->is_cs_replica() + && OB_FAIL(ObHATabletGroupCOConvertCtx::update_deleted_data_tablet_status(tablet_group_ctx_, logic_tablet_id.tablet_id_))) { + LOG_WARN("failed to update deleted tablet status", K(ret)); + } } else { LOG_WARN("failed to get tablet", K(ret), K(logic_tablet_id)); } @@ -2291,10 +2330,14 @@ int ObTabletMigrationDag::generate_next_dag(share::ObIDag *&dag) K(ret), K(logic_tablet_id), KPC(tablet)); } else if (logic_tablet_id.transfer_seq_ < tablet->get_tablet_meta().transfer_info_.transfer_seq_) { LOG_INFO("local tablet transfer seq is bigger than remote tablet, no need copy", K(logic_tablet_id), KPC(tablet)); + if (ls->is_cs_replica() + && OB_FAIL(ObHATabletGroupCOConvertCtx::update_deleted_data_tablet_status(tablet_group_ctx_, logic_tablet_id.tablet_id_))) { + LOG_WARN("failed to update deleted tablet status", K(ret)); + } } else if (OB_FAIL(scheduler->alloc_dag_with_priority(prio, tablet_migration_dag))) { LOG_WARN("failed to alloc tablet migration dag", K(ret)); } else { - if (OB_FAIL(tablet_migration_dag->init(logic_tablet_id.tablet_id_, tablet_handle, dag_net, tablet_group_ctx_))) { + if (OB_FAIL(tablet_migration_dag->init(logic_tablet_id.tablet_id_, tablet_handle, dag_net, tablet_group_ctx_, ObTabletMigrationDag::ObTabletType::DATA_TABLET_TYPE))) { LOG_WARN("failed to init tablet migration migration dag", K(ret), K(logic_tablet_id)); } else if (FALSE_IT(dag_id.init(MYADDR))) { } else if (OB_FAIL(tablet_migration_dag->set_dag_id(dag_id))) { @@ -3007,6 +3050,20 @@ int ObTabletMigrationTask::update_ha_expected_status_( LOG_WARN("failed to update tablet ha expected status", K(ret), K(expected_status), KPC(copy_tablet_ctx_)); } } + if (OB_FAIL(ret)) { + } else if (ls->is_cs_replica()) { + bool is_migrate_data_tablet = false; + ObHATabletGroupCtx *tablet_group_ctx = nullptr; + if (OB_FAIL(dag->check_is_migrate_data_tablet(is_migrate_data_tablet))) { + LOG_WARN("failed to check tablet type", K(ret)); + } else if (!is_migrate_data_tablet) { + // skip sys tablet migration + } else if (OB_FAIL(dag->get_tablet_group_ctx(tablet_group_ctx))) { + LOG_WARN("failed to get tablet group ctx", K(ret), KPC(dag)); + } else if (OB_FAIL(ObHATabletGroupCOConvertCtx::update_deleted_data_tablet_status(tablet_group_ctx, copy_tablet_ctx_->tablet_id_))) { + LOG_WARN("failed to update deleted tablet status", K(ret), KPC(tablet_group_ctx), KPC(copy_tablet_ctx_)); + } + } } return ret; } @@ -3176,6 +3233,9 @@ int ObTabletFinishMigrationTask::update_data_and_expected_status_() if (OB_TABLET_NOT_EXIST == ret) { LOG_INFO("migration tablet maybe deleted, skip it", K(ret), KPC(copy_tablet_ctx_)); ret = OB_SUCCESS; + if (OB_FAIL(update_co_convert_status_for_cs_replica(true /*tablet_is_deleted*/))) { + LOG_WARN("failed update convert status for cs replica", K(ret), KPC_(copy_tablet_ctx)); + } } else { LOG_WARN("failed to update tablet ha expected status", K(ret), K(expected_status), KPC(copy_tablet_ctx_)); } @@ -3197,15 +3257,35 @@ int ObTabletFinishMigrationTask::update_data_and_expected_status_() STORAGE_LOG(ERROR, "fake EN_UPDATE_TABLET_HA_STATUS_FAILED", K(ret)); } } + if (OB_SUCC(ret)) { + ObTabletMigrationDag *tablet_migration_dag = nullptr; + bool is_migrate_data_tablet = false; + if (OB_ISNULL(dag_) || OB_UNLIKELY(ObDagType::DAG_TYPE_TABLET_MIGRATION != dag_->get_type())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid null or wrong type", K(ret), KPC_(dag)); + } else if (FALSE_IT(tablet_migration_dag = static_cast(dag_))) { + } else if (OB_FAIL(tablet_migration_dag->check_is_migrate_data_tablet(is_migrate_data_tablet))) { + LOG_ERROR("failed to check is migrate data tablet", K(ret)); + } else if (!is_migrate_data_tablet) { + } else if (EN_DATA_TABLET_MIGRATION_DAG_OUT_OF_RETRY) { + ret = EN_DATA_TABLET_MIGRATION_DAG_OUT_OF_RETRY; + LOG_INFO("ERRSIM EN_DATA_TABLET_MIGRATION_DAG_OUT_OF_RETRY", K(ret)); + } + } #endif const ObTabletDataStatus::STATUS data_status = ObTabletDataStatus::COMPLETE; - if (OB_FAIL(ls_->update_tablet_ha_data_status(copy_tablet_ctx_->tablet_id_, data_status))) { + if (FAILEDx(ls_->update_tablet_ha_data_status(copy_tablet_ctx_->tablet_id_, data_status))) { if (OB_TABLET_NOT_EXIST == ret) { LOG_INFO("migration tablet maybe deleted, skip it", K(ret), KPC(copy_tablet_ctx_)); ret = OB_SUCCESS; + if (OB_FAIL(update_co_convert_status_for_cs_replica(true /*tablet_is_deleted*/))) { + LOG_WARN("failed update convert status for cs replica", K(ret), KPC_(copy_tablet_ctx)); + } } else { LOG_WARN("[HA]failed to update tablet ha data status", K(ret), KPC(copy_tablet_ctx_), K(data_status)); } + } else if (OB_FAIL(update_co_convert_status_for_cs_replica(false /*tablet_is_deleted*/))) { + LOG_WARN("failed to schedule convert merge if needed", K(ret), KPC_(copy_tablet_ctx)); } else { LOG_INFO("update tablet ha data status", KPC(copy_tablet_ctx_), K(data_status)); SERVER_EVENT_ADD("storage_ha", "tablet_finish_migration_task", @@ -3220,6 +3300,98 @@ int ObTabletFinishMigrationTask::update_data_and_expected_status_() return ret; } +int ObTabletFinishMigrationTask::prepare_co_convert_ctx(bool &is_migrate_data_tablet, ObHATabletGroupCOConvertCtx *&group_convert_ctx) +{ + int ret = OB_SUCCESS; + ObTabletMigrationDag *tablet_migration_dag = nullptr; + ObHATabletGroupCtx *ctx = nullptr; + is_migrate_data_tablet = false; + group_convert_ctx = nullptr; + + if (OB_ISNULL(dag_) || OB_UNLIKELY(ObDagType::DAG_TYPE_TABLET_MIGRATION != dag_->get_type())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid null or wrong type", K(ret), KPC_(dag)); + } else if (FALSE_IT(tablet_migration_dag = static_cast(dag_))) { + } else if (OB_FAIL(tablet_migration_dag->check_is_migrate_data_tablet(is_migrate_data_tablet))) { + LOG_WARN("failed to check tablet type", K(ret)); + } else if (!is_migrate_data_tablet) { + // skip sys tablet migration + } else if (OB_FAIL(tablet_migration_dag->get_tablet_group_ctx(ctx))) { + LOG_WARN("failed to get tablet group ctx", K(ret), KPC(tablet_migration_dag)); + } else if (OB_ISNULL(ctx) || OB_UNLIKELY(!ctx->is_cs_replica_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null ctx or invalid type", K(ret), KPC(ctx)); + } else { + group_convert_ctx = static_cast(ctx); + } + return ret; +} + +int ObTabletFinishMigrationTask::update_co_convert_status_for_cs_replica(const bool tablet_is_deleted) { + int ret = OB_SUCCESS; + bool is_migrate_data_tablet = false; + ObHATabletGroupCOConvertCtx *group_convert_ctx = nullptr; + + if (!ls_->is_cs_replica()) { + // skip F/R replica + } else if (OB_FAIL(prepare_co_convert_ctx(is_migrate_data_tablet, group_convert_ctx))) { + LOG_WARN("failed to prepare convert ctx", K(ret)); + } else if (!is_migrate_data_tablet) { + } else if (OB_ISNULL(group_convert_ctx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("convert ctx is null", K(ret)); + } else if (!tablet_is_deleted) { + (void) schedule_convert_co_merge(group_convert_ctx); // ignore ret + (void) group_convert_ctx->inc_finish_migration_cnt(); + } else if (OB_FAIL(group_convert_ctx->set_convert_finsih(copy_tablet_ctx_->tablet_id_))) { + LOG_WARN("failed to set convert finish", K(ret), K_(copy_tablet_ctx)); + } else { + (void) group_convert_ctx->inc_finish_migration_cnt(); + } + return ret; +} + +void ObTabletFinishMigrationTask::schedule_convert_co_merge( + ObHATabletGroupCOConvertCtx *group_convert_ctx) +{ + int ret = OB_SUCCESS; + const ObLSID &ls_id = ls_->get_ls_id(); + const ObTabletID &tablet_id = copy_tablet_ctx_->tablet_id_; + ObTablet *tablet = nullptr; + ObTabletHandle tablet_handle; + ObTabletCOConvertCtx *convert_ctx = nullptr; + bool need_convert = false; + + if (OB_FAIL(ls_->get_tablet(tablet_id, tablet_handle))) { + if (OB_TABLET_NOT_EXIST == ret) { + LOG_INFO("tablet maybe deleted, skip it", K(ret), K(ls_id), K(tablet_id)); + ret = OB_SUCCESS; + if (OB_FAIL(group_convert_ctx->set_convert_finsih(tablet_id))) { + LOG_WARN("failed to set convert finish", K(ret), K(tablet_id)); + } + } else { + LOG_WARN("failed to get tablet handle", K(ret), K(ls_id), K(tablet_id)); + } + } else if (FALSE_IT(tablet = tablet_handle.get_obj())) { + } else if (OB_FAIL(ObHATabletGroupCOConvertCtx::check_need_convert(*tablet, need_convert))) { + } else if (need_convert) { + DEBUG_SYNC(BEFROE_UPDATE_MIG_TABLET_CONVERT_CO_PROGRESSING); + LOG_INFO("[CS-Replica] Start schedule co merge dag to switch row to column store", K(ls_id), K(tablet_id)); + // Specific dag net id for co merge dag net to convert row store tablet into columnar store one. + // Use ObDataTabletsCheckCOConvertDag to check the convert result and re-schedule dag net if it failed, with the same dag net id. + ObDagId co_dag_net_id; + if (OB_FAIL(group_convert_ctx->get_co_dag_net_id(tablet_id, co_dag_net_id))) { + LOG_WARN("failed to get convert ctx", K(ret), K(ls_id), K(tablet_id)); + } else if (OB_FAIL(compaction::ObTenantTabletScheduler::schedule_convert_co_merge_dag_net(ls_id, *tablet, 0 /*retry_times*/, co_dag_net_id))) { + LOG_WARN("failed to schedule convert co merge for cs replica", K(ret), K(ls_id), K(tablet_id)); + } else if (OB_FAIL(group_convert_ctx->set_convert_progressing(tablet_id))) { + LOG_WARN("failed to set convert progressing", K(ret), K(tablet_id)); + } + } else if (OB_FAIL(group_convert_ctx->set_convert_finsih(tablet_id))) { + LOG_WARN("failed to set convert finish", K(ret), K(tablet_id)); + } +} + /******************ObDataTabletsMigrationDag*********************/ ObDataTabletsMigrationDag::ObDataTabletsMigrationDag() : ObMigrationDag(ObDagType::DAG_TYPE_DATA_TABLETS_MIGRATION), @@ -3463,6 +3635,8 @@ int ObDataTabletsMigrationTask::process() if (FAILEDx(generate_tablet_group_dag_())) { LOG_WARN("failed to generate tablet group dag", K(ret), KPC(ctx_)); + } else if (OB_FAIL(generate_check_co_convert_dag_if_needed())) { + LOG_WARN("failed to generate check convert dag", K(ret), KPC(ctx_)); } } @@ -3588,17 +3762,22 @@ int ObDataTabletsMigrationTask::build_tablet_group_info_() ObArray tablet_group_id_array; ObArray tablet_id_array; hash::ObHashSet remove_tablet_set; - + ObLS *ls = nullptr; DEBUG_SYNC(BEFORE_BUILD_TABLET_GROUP_INFO); if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("data tablets migration task do not init", K(ret)); + } else if (OB_ISNULL(ls = ls_handle_.get_ls())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls is nullptr", K(ret), K_(ls_handle)); } else { ctx_->tablet_group_mgr_.reuse(); const hash::ObHashMap &tablet_simple_info_map = ctx_->tablet_simple_info_map_; - + const ObHATabletGroupCtx::TabletGroupCtxType type = ls->is_cs_replica() + ? ObHATabletGroupCtx::TabletGroupCtxType::CS_REPLICA_TYPE + : ObHATabletGroupCtx::TabletGroupCtxType::NORMAL_TYPE; for (int64_t i = 0; OB_SUCC(ret) && i < ctx_->data_tablet_id_array_.count(); ++i) { tablet_simple_info.reset(); const ObLogicTabletID &logic_tablet_id = ctx_->data_tablet_id_array_.at(i); @@ -3618,7 +3797,7 @@ int ObDataTabletsMigrationTask::build_tablet_group_info_() && ObCopyTabletStatus::TABLET_EXIST == tablet_simple_info.status_) { if (OB_FAIL(tablet_group_id_array.push_back(logic_tablet_id))) { LOG_WARN("failed to push tablet id into array", K(ret), K(logic_tablet_id)); - } else if (OB_FAIL(ctx_->tablet_group_mgr_.build_tablet_group_ctx(tablet_group_id_array))) { + } else if (OB_FAIL(ctx_->tablet_group_mgr_.build_tablet_group_ctx(tablet_group_id_array, type))) { LOG_WARN("failed to build tablet group ctx", K(ret), KPC(ctx_)); } else { LOG_INFO("succeed build tablet group ctx", K(tablet_group_id_array)); @@ -3680,7 +3859,7 @@ int ObDataTabletsMigrationTask::build_tablet_group_info_() } if (OB_SUCC(ret)) { - if (OB_FAIL(ctx_->tablet_group_mgr_.build_tablet_group_ctx(tablet_group_id_array))) { + if (OB_FAIL(ctx_->tablet_group_mgr_.build_tablet_group_ctx(tablet_group_id_array, type))) { LOG_WARN("failed to build tablet group ctx", K(ret), K(tablet_group_id_array), KPC(ctx_)); } else { LOG_INFO("succeed build tablet group ctx", K(tablet_group_id_array), "count", tablet_group_id_array.count()); @@ -3761,6 +3940,75 @@ int ObDataTabletsMigrationTask::generate_tablet_group_dag_() return ret; } +int ObDataTabletsMigrationTask::generate_check_co_convert_dag_if_needed() { + int ret = OB_SUCCESS; + ObLS *ls = nullptr; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("data tablets migration task do not init", K(ret)); + } else if (OB_ISNULL(ctx_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ctx is nullptr", K(ret)); + } else if (OB_ISNULL(ls = ls_handle_.get_ls())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls is nullptr", K(ret), K_(ls_handle)); + } else if (!ls->is_cs_replica()) { + } else if (OB_FAIL(inner_generate_check_co_convert_dag(ls))) { + LOG_WARN("failed to generate check convert dag", K(ret)); + } else { + LOG_INFO("[CS-Replica] Finish generate check convert dag", K(ret), KPC(ls)); + SERVER_EVENT_ADD("storage_ha", "generage_check_co_convert_dag", + "tenant_id", MTL_ID(), + "ls_id", ctx_->arg_.ls_id_.id(), + "src", ctx_->arg_.src_.get_server(), + "dst", ctx_->arg_.dst_.get_server(), + "task_id", ctx_->task_id_); + } + return ret; +} + +int ObDataTabletsMigrationTask::inner_generate_check_co_convert_dag(ObLS *ls) +{ + int ret = OB_SUCCESS; + ObTenantDagScheduler *scheduler = MTL(ObTenantDagScheduler*); + ObDataTabletsCheckCOConvertDag *tablet_check_convert_dag = nullptr; + ObIDagNet *dag_net = nullptr; + ObMigrationDagNet *migration_dag_net = nullptr; + + if (OB_ISNULL(ls) || OB_ISNULL(dag_) || OB_ISNULL(dag_net = dag_->get_dag_net())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid nullptr", K(ret), KP(ls), KP_(dag), KP(dag_net)); + } else if (OB_FAIL(scheduler->alloc_dag_with_priority(dag_->get_priority(), tablet_check_convert_dag))) { + LOG_WARN("failed to alloc dag", K(ret)); + } else if (OB_UNLIKELY(ObDagNetType::DAG_NET_TYPE_MIGRATION != dag_net->get_type())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dag net type is unexpected", K(ret), KPC(dag_net)); + } else if (FALSE_IT(migration_dag_net = static_cast(dag_net))) { + } else if (OB_FAIL(tablet_check_convert_dag->init(migration_dag_net->get_migration_ctx(), ls))) { + LOG_WARN("failed to init tablet check convert dag", K(ret), "ls_id", ls->get_ls_id()); + } else if (OB_FAIL(dag_net->add_dag_into_dag_net(*(tablet_check_convert_dag)))) { + LOG_WARN("failed to add dag into dag net", K(ret)); + } else if (OB_FAIL(dag_->add_child_without_inheritance(*tablet_check_convert_dag))) { + LOG_WARN("failed to add child dag", K(ret), KPC(tablet_check_convert_dag), KPC_(dag)); + } else if (OB_FAIL(tablet_check_convert_dag->create_first_task())) { + LOG_WARN("failed to create first task", K(ret)); + } else if (OB_FAIL(tablet_check_convert_dag->add_child_without_inheritance(*finish_dag_))) { + LOG_WARN("failed to add finish dag as child", K(ret), KPC(tablet_check_convert_dag), KPC_(finish_dag)); + } else if (OB_FAIL(scheduler->add_dag(tablet_check_convert_dag))) { + LOG_WARN("failed to add tablet check convert dag", K(ret), KPC(tablet_check_convert_dag)); + if (OB_SIZE_OVERFLOW != ret && OB_EAGAIN != ret) { + LOG_WARN("Fail to add dag", K(ret), "ls_id", ls->get_ls_id()); + ret = OB_EAGAIN; + } + } + if (OB_FAIL(ret) && OB_NOT_NULL(tablet_check_convert_dag)) { + scheduler->free_dag(*tablet_check_convert_dag); + tablet_check_convert_dag = nullptr; + } + + return ret; +} + int ObDataTabletsMigrationTask::try_remove_unneeded_tablets_() { int ret = OB_SUCCESS; @@ -4286,6 +4534,10 @@ int ObTabletGroupMigrationTask::generate_tablet_migration_dag_() } else if (OB_FAIL(ls->ha_get_tablet(logic_tablet_id.tablet_id_, tablet_handle))) { if (OB_TABLET_NOT_EXIST == ret) { ret = OB_SUCCESS; + if (ls->is_cs_replica() + && OB_FAIL(ObHATabletGroupCOConvertCtx::update_deleted_data_tablet_status(tablet_group_ctx_, logic_tablet_id.tablet_id_))) { + LOG_WARN("failed to update deleted tablet status", K(ret)); + } } else { LOG_WARN("failed to get tablet", K(ret), K(logic_tablet_id)); } @@ -4298,9 +4550,13 @@ int ObTabletGroupMigrationTask::generate_tablet_migration_dag_() K(ret), K(logic_tablet_id), KPC(tablet)); } else if (logic_tablet_id.transfer_seq_ < tablet->get_tablet_meta().transfer_info_.transfer_seq_) { LOG_INFO("local tablet transfer seq is bigger than remote tablet, no need copy", K(logic_tablet_id), KPC(tablet)); + if (ls->is_cs_replica() + && OB_FAIL(ObHATabletGroupCOConvertCtx::update_deleted_data_tablet_status(tablet_group_ctx_, logic_tablet_id.tablet_id_))) { + LOG_WARN("failed to update deleted tablet status", K(ret)); + } } else if (OB_FAIL(scheduler->alloc_dag_with_priority(prio, tablet_migration_dag))) { LOG_WARN("failed to alloc tablet migration dag ", K(ret)); - } else if (OB_FAIL(tablet_migration_dag->init(logic_tablet_id.tablet_id_, tablet_handle, dag_net, tablet_group_ctx_))) { + } else if (OB_FAIL(tablet_migration_dag->init(logic_tablet_id.tablet_id_, tablet_handle, dag_net, tablet_group_ctx_, ObTabletMigrationDag::ObTabletType::DATA_TABLET_TYPE))) { LOG_WARN("failed to init tablet migration migration dag", K(ret), K(*ctx_)); } else if (OB_FAIL(dag_net->add_dag_into_dag_net(*tablet_migration_dag))) { LOG_WARN("failed to add dag into dag net", K(ret), K(*ctx_)); diff --git a/src/storage/high_availability/ob_ls_migration.h b/src/storage/high_availability/ob_ls_migration.h index 578fffad0..731c55025 100644 --- a/src/storage/high_availability/ob_ls_migration.h +++ b/src/storage/high_availability/ob_ls_migration.h @@ -305,6 +305,12 @@ private: class ObTabletMigrationDag : public ObMigrationDag { +public: + enum class ObTabletType { + SYS_TABLET_TYPE = 0, // sys tablet is unnecessary to processed in cs replica + DATA_TABLET_TYPE = 1, // only data tablet has ObHATabletGroupCOConvertCtx + MAX_TYPE + }; public: ObTabletMigrationDag(); virtual ~ObTabletMigrationDag(); @@ -319,15 +325,19 @@ public: const common::ObTabletID &tablet_id, ObTabletHandle &tablet_handle, share::ObIDagNet *dag_net, - ObHATabletGroupCtx *tablet_group_ctx = nullptr); + ObHATabletGroupCtx *tablet_group_ctx = nullptr, + ObTabletType tablet_type = ObTabletType::SYS_TABLET_TYPE); + int get_tablet_group_ctx(ObHATabletGroupCtx *&tablet_group_ctx); + int check_is_migrate_data_tablet(bool &is_migrate_data_tablet); int get_ls(ObLS *&ls); - INHERIT_TO_STRING_KV("ObIMigrationDag", ObMigrationDag, KP(this), K(copy_tablet_ctx_)); + INHERIT_TO_STRING_KV("ObIMigrationDag", ObMigrationDag, KP(this), K(copy_tablet_ctx_), K(tablet_type_)); protected: bool is_inited_; ObLSHandle ls_handle_; ObCopyTabletCtx copy_tablet_ctx_; ObHATabletGroupCtx *tablet_group_ctx_; + ObTabletType tablet_type_; DISALLOW_COPY_AND_ASSIGN(ObTabletMigrationDag); }; @@ -395,6 +405,7 @@ private: DISALLOW_COPY_AND_ASSIGN(ObTabletMigrationTask); }; +class ObHATabletGroupCOConvertCtx; class ObTabletFinishMigrationTask final : public share::ObITask { public: @@ -406,6 +417,10 @@ public: VIRTUAL_TO_STRING_KV(K("ObTabletFinishMigrationTask"), KP(this), KPC(ha_dag_net_ctx_), KPC(copy_tablet_ctx_), KPC(ls_)); private: int update_data_and_expected_status_(); + // handle cs replica + int prepare_co_convert_ctx(bool &is_migrate_data_tablet, ObHATabletGroupCOConvertCtx *&group_convert_ctx); + int update_co_convert_status_for_cs_replica(const bool tablet_is_deleted); + void schedule_convert_co_merge(ObHATabletGroupCOConvertCtx *group_convert_ctx); private: bool is_inited_; int64_t task_gen_time_; @@ -451,6 +466,8 @@ private: common::ObIArray &tablet_group_dag_array); int build_tablet_group_info_(); int generate_tablet_group_dag_(); + int generate_check_co_convert_dag_if_needed(); + int inner_generate_check_co_convert_dag(ObLS *ls); int try_remove_unneeded_tablets_(); int try_offline_ls_(); int record_server_event_(); diff --git a/src/storage/high_availability/ob_ls_remove_member_handler.cpp b/src/storage/high_availability/ob_ls_remove_member_handler.cpp index a102abac1..e7ba30f21 100644 --- a/src/storage/high_availability/ob_ls_remove_member_handler.cpp +++ b/src/storage/high_availability/ob_ls_remove_member_handler.cpp @@ -326,7 +326,6 @@ int ObLSRemoveMemberHandler::check_task_exist( ret = OB_ERR_UNEXPECTED; LOG_WARN("tenant dag scheduler should not be NULL", K(ret), KP(scheduler)); } else { - ObMember mock_member(MYADDR, OB_INVALID_TIMESTAMP); mock_remove_member_arg.tenant_id_ = MTL_ID(); mock_remove_member_arg.ls_id_ = ls_->get_ls_id(); mock_remove_member_arg.task_id_ = task_id; @@ -334,8 +333,8 @@ int ObLSRemoveMemberHandler::check_task_exist( mock_remove_member_arg.type_ = ObLSChangeMemberType::LS_REMOVE_MEMBER; param.arg_ = mock_remove_member_arg; - if (OB_FAIL(mock_remove_member_arg.remove_member_.set_member(mock_member))) { - LOG_WARN("failed to set member", K(ret), K(mock_member), K(mock_remove_member_arg)); + if (OB_FAIL(mock_remove_member_arg.remove_member_.init(MYADDR, OB_INVALID_TIMESTAMP, REPLICA_TYPE_FULL))) { + LOG_WARN("failed to init remove_member_", K(ret), K(MYADDR), K(mock_remove_member_arg)); } else if (OB_FAIL(scheduler->create_dag(¶m, exist_dag))) { LOG_WARN("failed to create ls remove member dag", K(ret)); } else if (OB_FAIL(scheduler->check_dag_exist(exist_dag, is_exist))) { diff --git a/src/storage/high_availability/ob_rebuild_service.cpp b/src/storage/high_availability/ob_rebuild_service.cpp index f5aaee998..1062db6b5 100644 --- a/src/storage/high_availability/ob_rebuild_service.cpp +++ b/src/storage/high_availability/ob_rebuild_service.cpp @@ -1092,8 +1092,10 @@ int ObLSRebuildMgr::generate_rebuild_task_() DEBUG_SYNC(BEFOR_EXEC_REBUILD_TASK); ObTaskId task_id; task_id.init(GCONF.self_addr_); - ObReplicaMember dst_replica_member(GCONF.self_addr_, timestamp); - ObReplicaMember src_replica_member(GCONF.self_addr_, timestamp); + ObReplicaMember dst_replica_member(GCONF.self_addr_, timestamp, + REPLICA_TYPE_FULL/*dummy_replica_type*/); + ObReplicaMember src_replica_member(GCONF.self_addr_, timestamp, + REPLICA_TYPE_FULL/*dummy_replica_type*/); ObMigrationOpArg arg; arg.cluster_id_ = GCONF.cluster_id; arg.data_src_ = src_replica_member; diff --git a/src/storage/high_availability/ob_storage_ha_dag.cpp b/src/storage/high_availability/ob_storage_ha_dag.cpp index 2a86eeb42..9a86eb1b0 100644 --- a/src/storage/high_availability/ob_storage_ha_dag.cpp +++ b/src/storage/high_availability/ob_storage_ha_dag.cpp @@ -18,6 +18,7 @@ #include "observer/ob_server_event_history_table_operator.h" #include "storage/tx_storage/ob_ls_service.h" #include "storage/tablet/ob_tablet.h" +#include "storage/high_availability/ob_cs_replica_migration.h" namespace oceanbase { @@ -476,11 +477,12 @@ int ObStorageHADagUtils::check_self_is_valid_member( } /******************ObHATabletGroupCtx*********************/ -ObHATabletGroupCtx::ObHATabletGroupCtx() +ObHATabletGroupCtx::ObHATabletGroupCtx(const TabletGroupCtxType type) : is_inited_(false), lock_(), tablet_id_array_(), - index_(0) + index_(0), + type_(type) { } @@ -499,6 +501,8 @@ int ObHATabletGroupCtx::init(const common::ObIArray &tablet_id_ LOG_WARN("init ha tablet group ctx get invalid argument", K(ret), K(tablet_id_array)); } else if (OB_FAIL(tablet_id_array_.assign(tablet_id_array))) { LOG_WARN("failed to assign tablet id array", K(ret), K(tablet_id_array)); + } else if (OB_FAIL(inner_init())) { + LOG_WARN("failed to inner init", K(ret)); } else { index_ = 0; is_inited_ = true; @@ -546,9 +550,19 @@ int ObHATabletGroupCtx::get_all_tablet_ids(ObIArray &tablet_id_ return ret; } +bool ObHATabletGroupCtx::is_cs_replica_ctx() const { + common::SpinRLockGuard guard(lock_); + return TabletGroupCtxType::CS_REPLICA_TYPE == type_; +} + void ObHATabletGroupCtx::reuse() { common::SpinWLockGuard guard(lock_); + inner_reuse(); +} + +void ObHATabletGroupCtx::inner_reuse() +{ tablet_id_array_.reuse(); index_ = 0; is_inited_ = false; @@ -608,10 +622,10 @@ int ObHATabletGroupMgr::get_next_tablet_group_ctx( } int ObHATabletGroupMgr::build_tablet_group_ctx( - const ObIArray &tablet_id_array) + const ObIArray &tablet_id_array, + const ObHATabletGroupCtx::TabletGroupCtxType type /*=NORMAL_TYPE*/) { int ret = OB_SUCCESS; - void *buf = nullptr; ObHATabletGroupCtx *tablet_group_ctx = nullptr; if (!is_inited_) { @@ -622,10 +636,11 @@ int ObHATabletGroupMgr::build_tablet_group_ctx( LOG_WARN("build tablet group ctx get invalid argument", K(ret), K(tablet_id_array)); } else { common::SpinWLockGuard guard(lock_); - if (OB_ISNULL(buf = allocator_.alloc(sizeof(ObHATabletGroupCtx)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc memory", K(ret), KP(buf)); - } else if (FALSE_IT(tablet_group_ctx = new (buf) ObHATabletGroupCtx())) { + if (OB_FAIL(alloc_and_new_tablet_group_ctx(type, tablet_group_ctx))) { + LOG_WARN("failed to alloc and new tablet group ctx", K(ret)); + } else if (OB_ISNULL(tablet_group_ctx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tablet group ctx should not be NULL", K(ret), KP(tablet_group_ctx)); } else if (OB_FAIL(tablet_group_ctx->init(tablet_id_array))) { LOG_WARN("failed to init tablet group ctx", K(ret), K(tablet_id_array)); } else if (OB_FAIL(tablet_group_ctx_array_.push_back(tablet_group_ctx))) { @@ -641,6 +656,38 @@ int ObHATabletGroupMgr::build_tablet_group_ctx( return ret; } +int ObHATabletGroupMgr::alloc_and_new_tablet_group_ctx( + const ObHATabletGroupCtx::TabletGroupCtxType type, + ObHATabletGroupCtx *&tablet_group_ctx) +{ + int ret = OB_SUCCESS; + void *buf = nullptr; + if (ObHATabletGroupCtx::TabletGroupCtxType::NORMAL_TYPE == type) { + if (OB_ISNULL(buf = allocator_.alloc(sizeof(ObHATabletGroupCtx)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory", K(ret), KP(buf)); + } else { + tablet_group_ctx = new (buf) ObHATabletGroupCtx(); + } + } else if (ObHATabletGroupCtx::TabletGroupCtxType::CS_REPLICA_TYPE == type) { + if (OB_ISNULL(buf = allocator_.alloc(sizeof(ObHATabletGroupCOConvertCtx)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory", K(ret), KP(buf)); + } else { + tablet_group_ctx = new (buf) ObHATabletGroupCOConvertCtx(); + } + } else { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid ctx type", K(ret), K(type)); + } + + if (OB_FAIL(ret) && OB_NOT_NULL(tablet_group_ctx)) { + tablet_group_ctx->~ObHATabletGroupCtx(); + tablet_group_ctx = nullptr; + } + return ret; +} + void ObHATabletGroupMgr::reuse() { common::SpinWLockGuard guard(lock_); @@ -655,6 +702,34 @@ void ObHATabletGroupMgr::reuse() index_ = 0; } +int64_t ObHATabletGroupMgr::get_tablet_group_ctx_count() const +{ + common::SpinRLockGuard guard(lock_); + return tablet_group_ctx_array_.count(); +} + +int ObHATabletGroupMgr::get_tablet_group_ctx( + const int64_t idx, + ObHATabletGroupCtx *&tablet_group_ctx) +{ + int ret = OB_SUCCESS; + tablet_group_ctx = nullptr; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ha tablet group mgr do not init", K(ret)); + } else { + common::SpinRLockGuard guard(lock_); + if (OB_UNLIKELY(idx < 0 || idx >= tablet_group_ctx_array_.count())) { + ret = OB_INDEX_OUT_OF_RANGE; + } else if (OB_ISNULL(tablet_group_ctx = tablet_group_ctx_array_.at(idx))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tablet group ctx is null", K(ret), K(idx), K_(tablet_group_ctx_array)); + } + } + return ret; +} + /******************ObStorageHATaskUtils*********************/ int ObStorageHATaskUtils::check_need_copy_sstable( const ObMigrationSSTableParam ¶m, diff --git a/src/storage/high_availability/ob_storage_ha_dag.h b/src/storage/high_availability/ob_storage_ha_dag.h index 75397dd3f..5af011892 100644 --- a/src/storage/high_availability/ob_storage_ha_dag.h +++ b/src/storage/high_availability/ob_storage_ha_dag.h @@ -133,19 +133,30 @@ public: class ObHATabletGroupCtx { public: - ObHATabletGroupCtx(); + enum class TabletGroupCtxType + { + NORMAL_TYPE = 0, + CS_REPLICA_TYPE = 1, + MAX_TYPE + }; +public: + ObHATabletGroupCtx(const TabletGroupCtxType type = TabletGroupCtxType::NORMAL_TYPE); virtual ~ObHATabletGroupCtx(); int init(const common::ObIArray &tablet_id_array); int get_next_tablet_id(ObLogicTabletID &logic_tablet_id); int get_all_tablet_ids(common::ObIArray &tablet_id); - void reuse(); - + bool is_cs_replica_ctx() const; +public: + virtual void reuse(); + virtual void inner_reuse(); + virtual int inner_init() { return OB_SUCCESS; } TO_STRING_KV(K_(tablet_id_array), K_(index)); -private: +protected: bool is_inited_; common::SpinRWLock lock_; ObArray tablet_id_array_; int64_t index_; + TabletGroupCtxType type_; DISALLOW_COPY_AND_ASSIGN(ObHATabletGroupCtx); }; @@ -158,8 +169,14 @@ public: int get_next_tablet_group_ctx( ObHATabletGroupCtx *&tablet_group_ctx); int build_tablet_group_ctx( - const common::ObIArray &tablet_id_array); + const common::ObIArray &tablet_id_array, + const ObHATabletGroupCtx::TabletGroupCtxType type = ObHATabletGroupCtx::TabletGroupCtxType::NORMAL_TYPE); + int alloc_and_new_tablet_group_ctx( + const ObHATabletGroupCtx::TabletGroupCtxType type, + ObHATabletGroupCtx *&tablet_group_ctx); void reuse(); + int64_t get_tablet_group_ctx_count() const; + int get_tablet_group_ctx(const int64_t idx, ObHATabletGroupCtx *&tablet_group_ctx); TO_STRING_KV(K_(tablet_group_ctx_array), K_(index)); private: diff --git a/src/storage/high_availability/ob_storage_ha_src_provider.cpp b/src/storage/high_availability/ob_storage_ha_src_provider.cpp index a602ca234..f67cc19a2 100644 --- a/src/storage/high_availability/ob_storage_ha_src_provider.cpp +++ b/src/storage/high_availability/ob_storage_ha_src_provider.cpp @@ -307,7 +307,7 @@ int ObStorageHASrcProvider::get_replica_addr_list( } else { if (common::ObReplicaType::REPLICA_TYPE_FULL == dst.get_replica_type()) { need_learner_list = false; - } else if (common::ObReplicaType::REPLICA_TYPE_READONLY == dst.get_replica_type()) { + } else if (ObReplicaTypeCheck::is_non_paxos_replica(dst.get_replica_type())) { need_learner_list = true; } else { ret = OB_ERR_UNEXPECTED; @@ -379,18 +379,28 @@ int ObStorageHASrcProvider::check_replica_type_( if (!addr.is_valid() || !dst.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument!", K(ret), K(addr), K(dst)); - } else if (learner_list.is_valid() && learner_list.contains(addr)) { // src is R - if (common::ObReplicaType::REPLICA_TYPE_FULL == dst.get_replica_type()) { // dst is F - is_replica_type_valid = false; - } else if (common::ObReplicaType::REPLICA_TYPE_READONLY == dst.get_replica_type()) { - is_replica_type_valid = true; + } else if (learner_list.is_valid() && learner_list.contains(addr)) { + // src is R/C + ObMember src; + if (OB_FAIL(learner_list.get_learner_by_addr(addr, src))) { + LOG_WARN("failed to get learner by addr", KR(ret), K(addr)); + } else if (src.is_columnstore()) { + // src is C, dst can only be C as well + is_replica_type_valid = REPLICA_TYPE_COLUMNSTORE == dst.get_replica_type(); } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected dst replica type", K(ret), K(dst), K(learner_list)); + // src is R, dst can be non-paxos replica-type (R or C) + if (common::ObReplicaType::REPLICA_TYPE_FULL == dst.get_replica_type()) { // dst is F + is_replica_type_valid = false; + } else if (ObReplicaTypeCheck::is_non_paxos_replica(dst.get_replica_type())) { + is_replica_type_valid = true; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected dst replica type", K(ret), K(dst), K(learner_list)); + } } - } else { // src is F - if (common::ObReplicaType::REPLICA_TYPE_FULL == dst.get_replica_type() - || common::ObReplicaType::REPLICA_TYPE_READONLY == dst.get_replica_type()) { + } else { + // src is F, dst can be all replica-type (F/R/C) + if (ObReplicaTypeCheck::is_replica_type_valid(dst.get_replica_type())) { is_replica_type_valid = true; } else { ret = OB_ERR_UNEXPECTED; @@ -480,7 +490,7 @@ int ObStorageHASrcProvider::get_palf_parent_addr_(const uint64_t tenant_id, cons if (OB_FAIL(member_helper_->get_ls_leader(tenant_id, ls_id, parent_addr))) { LOG_WARN("failed to get leader addr", K(ret), K(tenant_id), K(ls_id)); } - } else if (common::ObReplicaType::REPLICA_TYPE_READONLY == replica_type) { + } else if (ObReplicaTypeCheck::is_non_paxos_replica(replica_type)) { if (OB_FAIL(ls->get_log_handler()->get_parent(parent_addr))) { LOG_WARN("failed to get parent addr", K(ret), K(tenant_id), K(ls_id)); } diff --git a/src/storage/high_availability/ob_storage_ha_utils.cpp b/src/storage/high_availability/ob_storage_ha_utils.cpp index 0c8b8501a..f16ae3408 100644 --- a/src/storage/high_availability/ob_storage_ha_utils.cpp +++ b/src/storage/high_availability/ob_storage_ha_utils.cpp @@ -210,6 +210,8 @@ int ObStorageHAUtils::check_tablet_replica_checksum_(const uint64_t tenant_id, c LOG_WARN("failed to batch get replica checksum item", K(ret), K(tenant_id), K(pairs), K(compaction_scn)); } else { ObArray filter_items; + ObLSColumnReplicaCache ls_cs_replica_cache; + ObTabletDataChecksumChecker data_checksum_checker; for (int64_t i = 0; OB_SUCC(ret) && i < items.count(); ++i) { const ObTabletReplicaChecksumItem &item = items.at(i); if (item.compaction_scn_ == compaction_scn) { @@ -218,12 +220,31 @@ int ObStorageHAUtils::check_tablet_replica_checksum_(const uint64_t tenant_id, c } } } + + if (FAILEDx(ls_cs_replica_cache.init())) { + LOG_WARN("failed to init ls column replica cache", K(ret)); + } + + for (int64_t i = 0; OB_SUCC(ret) && i < filter_items.count(); ++i) { + const ObTabletReplicaChecksumItem &item = filter_items.at(i); + if (OB_FAIL(ls_cs_replica_cache.update(item.ls_id_, item.server_))) { + LOG_WARN("fail to update ls replica status", K(ret), K(item)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < filter_items.count(); ++i) { const ObTabletReplicaChecksumItem &first_item = filter_items.at(0); const ObTabletReplicaChecksumItem &item = filter_items.at(i); - if (OB_FAIL(first_item.verify_checksum(item))) { - LOG_ERROR("failed to verify checksum", K(ret), K(tenant_id), K(tablet_id), - K(ls_id), K(compaction_scn), K(first_item), K(item), K(filter_items)); + const ObLSReplicaUniItem ls_item(item.ls_id_, item.server_); + bool is_cs_replica = false; + if (OB_FAIL(ls_cs_replica_cache.check_is_cs_replica(ls_item, is_cs_replica))) { + LOG_WARN("fail to check is cs replica", K(ret), K(ls_item), K(ls_cs_replica_cache)); + } else if (OB_FAIL(data_checksum_checker.check_data_checksum(item, is_cs_replica))) { + LOG_ERROR("failed to verify data checksum", K(ret), K(tenant_id), K(tablet_id), + K(ls_id), K(compaction_scn), K(item), K(filter_items), K(is_cs_replica), K(ls_cs_replica_cache)); + } else if (OB_FAIL(item.verify_column_checksum(first_item))) { + LOG_ERROR("failed to verify column checksum", K(ret), K(tenant_id), K(tablet_id), + K(ls_id), K(compaction_scn), K(first_item), K(item), K(filter_items), K(is_cs_replica), K(ls_cs_replica_cache)); } } } diff --git a/src/storage/high_availability/ob_tablet_group_restore.cpp b/src/storage/high_availability/ob_tablet_group_restore.cpp index fd5c199f2..dbc8d40dd 100644 --- a/src/storage/high_availability/ob_tablet_group_restore.cpp +++ b/src/storage/high_availability/ob_tablet_group_restore.cpp @@ -136,7 +136,7 @@ ObTabletRestoreCtx::ObTabletRestoreCtx() action_(ObTabletRestoreAction::MAX), meta_index_store_(nullptr), second_meta_index_store_(nullptr), - replica_type_(ObReplicaType::REPLICA_TYPE_MAX), + replica_type_(ObReplicaType::REPLICA_TYPE_INVALID), ha_table_info_mgr_(nullptr), need_check_seq_(false), ls_rebuild_seq_(-1), @@ -174,7 +174,7 @@ void ObTabletRestoreCtx::reset() action_ = ObTabletRestoreAction::MAX; meta_index_store_ = nullptr; second_meta_index_store_ = nullptr; - replica_type_ = ObReplicaType::REPLICA_TYPE_MAX; + replica_type_ = ObReplicaType::REPLICA_TYPE_INVALID; need_check_seq_ = false; ls_rebuild_seq_ = -1; status_ = ObCopyTabletStatus::MAX_STATUS; diff --git a/src/storage/ls/ob_ls.cpp b/src/storage/ls/ob_ls.cpp index 8daaa75f1..05f9460c2 100755 --- a/src/storage/ls/ob_ls.cpp +++ b/src/storage/ls/ob_ls.cpp @@ -84,6 +84,8 @@ using namespace rootserver; namespace storage { +ERRSIM_POINT_DEF(EN_LS_NOT_SEE_CS_REPLICA); + using namespace checkpoint; using namespace mds; @@ -123,6 +125,7 @@ int ObLS::init(const share::ObLSID &ls_id, const ObMigrationStatus &migration_status, const ObLSRestoreStatus &restore_status, const SCN &create_scn, + const ObLSStoreFormat &store_format, observer::ObIMetaReport *reporter) { int ret = OB_SUCCESS; @@ -149,7 +152,8 @@ int ObLS::init(const share::ObLSID &ls_id, ls_id, migration_status, restore_status, - create_scn))) { + create_scn, + store_format))) { LOG_WARN("failed to init ls meta", K(ret), K(tenant_id), K(ls_id)); } else { rs_reporter_ = reporter; @@ -477,6 +481,42 @@ bool ObLS::is_restore_first_step() const return bool_ret; } +bool ObLS::is_cs_replica() const +{ + return ls_meta_.get_store_format().is_columnstore(); +} + +int ObLS::check_has_cs_replica(bool &has_cs_replica) const +{ + int ret = OB_SUCCESS; + has_cs_replica = false; + ObMemberList member_list; + GlobalLearnerList learner_list; + int64_t paxos_replica_number = 0; + if (OB_FAIL(get_paxos_member_list_and_learner_list(member_list, paxos_replica_number, learner_list))) { + LOG_WARN("fail to get member list and learner list", K(ret), K(ls_meta_.ls_id_)); + } else { + for (int64_t i = 0; i < learner_list.get_member_number(); i++) { + const ObMember &learner = learner_list.get_learner(i); + if (learner.is_columnstore()) { + has_cs_replica = true; + break; + } + } + } + +#ifdef ERRSIM + if (OB_SUCC(ret)) { + if (EN_LS_NOT_SEE_CS_REPLICA) { + has_cs_replica = false; + LOG_INFO("ERRSIM EN_LS_NOT_SEE_CS_REPLICA", K(ret), K(has_cs_replica)); + } + } +#endif + + return ret; +} + int ObLS::start() { int ret = OB_SUCCESS; diff --git a/src/storage/ls/ob_ls.h b/src/storage/ls/ob_ls.h index 621b9a605..3f93d1c64 100644 --- a/src/storage/ls/ob_ls.h +++ b/src/storage/ls/ob_ls.h @@ -231,6 +231,7 @@ public: const ObMigrationStatus &migration_status, const share::ObLSRestoreStatus &restore_status, const share::SCN &create_scn, + const ObLSStoreFormat &store_format, observer::ObIMetaReport *reporter); // I am ready to work now. int start(); @@ -309,6 +310,10 @@ public: bool is_in_gc(); bool is_restore_first_step() const; bool is_clone_first_step() const; + // is current ls replica a column store replica + bool is_cs_replica() const; + // is current ls replica set contains a column store replica + int check_has_cs_replica(bool &has_cs_replica) const; // for rebuild // remove inner tablet, the memtable and minor sstable of data tablet, disable replay // int prepare_rebuild(); @@ -497,6 +502,7 @@ public: DELEGATE_WITH_RET(ls_meta_, set_rebuild_info, int); DELEGATE_WITH_RET(ls_meta_, get_rebuild_info, int); DELEGATE_WITH_RET(ls_meta_, get_create_type, int); + DELEGATE_WITH_RET(ls_meta_, get_store_format, ObLSStoreFormat); // get ls_meta_package and sorted tablet_metas for backup. tablet gc is forbidden meanwhile. diff --git a/src/storage/ls/ob_ls_meta.cpp b/src/storage/ls/ob_ls_meta.cpp index 77190b6bb..5b24cedcd 100644 --- a/src/storage/ls/ob_ls_meta.cpp +++ b/src/storage/ls/ob_ls_meta.cpp @@ -91,7 +91,7 @@ ObLSMeta::ObLSMeta(const ObLSMeta &ls_meta) rebuild_info_(ls_meta.rebuild_info_), transfer_meta_info_(ls_meta.transfer_meta_info_), major_mv_merge_info_(ls_meta.major_mv_merge_info_), - store_format_() + store_format_(ls_meta.store_format_) { all_id_meta_.update_all_id_meta(ls_meta.all_id_meta_); } @@ -289,7 +289,8 @@ bool ObLSMeta::is_valid() const && OB_MIGRATION_STATUS_MAX != migration_status_ && ObGCHandler::is_valid_ls_gc_state(gc_state_) && restore_status_.is_valid() - && rebuild_seq_ >= 0; + && rebuild_seq_ >= 0 + && store_format_.is_valid(); } int64_t ObLSMeta::get_rebuild_seq() const @@ -708,7 +709,8 @@ int ObLSMeta::init( const share::ObLSID &ls_id, const ObMigrationStatus &migration_status, const share::ObLSRestoreStatus &restore_status, - const SCN &create_scn) + const SCN &create_scn, + const ObLSStoreFormat &store_format) { int ret = OB_SUCCESS; if (OB_INVALID_ID == tenant_id || !ls_id.is_valid() @@ -728,6 +730,7 @@ int ObLSMeta::init( gc_state_ = LSGCState::NORMAL; restore_status_ = restore_status; transfer_scn_ = SCN::min_scn(); + store_format_ = store_format; } return ret; } @@ -891,6 +894,11 @@ int ObLSMeta::check_ls_need_online(bool &need_online) const return ret; } +ObLSStoreFormat ObLSMeta::get_store_format() const +{ + return store_format_; +} + ObLSMeta::ObReentrantWLockGuard::ObReentrantWLockGuard(ObLatch &lock, const bool try_lock, const int64_t warn_threshold) diff --git a/src/storage/ls/ob_ls_meta.h b/src/storage/ls/ob_ls_meta.h index d2f96137b..14e1e1203 100644 --- a/src/storage/ls/ob_ls_meta.h +++ b/src/storage/ls/ob_ls_meta.h @@ -58,7 +58,8 @@ public: const share::ObLSID &ls_id, const ObMigrationStatus &migration_status, const share::ObLSRestoreStatus &restore_status, - const int64_t create_scn); + const int64_t create_scn, + const ObLSStoreFormat &store_format); void reset(); bool is_valid() const; int set_start_work_state(); @@ -112,13 +113,15 @@ public: int get_rebuild_info(ObLSRebuildInfo &rebuild_info) const; int get_create_type(int64_t &create_type) const; int check_ls_need_online(bool &need_online) const; + ObLSStoreFormat get_store_format() const; int init( const uint64_t tenant_id, const share::ObLSID &ls_id, const ObMigrationStatus &migration_status, const share::ObLSRestoreStatus &restore_status, - const share::SCN &create_scn); + const share::SCN &create_scn, + const ObLSStoreFormat &store_format); ObReplicaType get_replica_type() const { return unused_replica_type_; } @@ -206,7 +209,7 @@ private: ObLSRebuildInfo rebuild_info_; ObLSTransferMetaInfo transfer_meta_info_; //transfer_dml_ctrl_42x # placeholder ObMajorMVMergeInfo major_mv_merge_info_; - common::ObLSStoreFormat store_format_; //not used, only as placeholder + common::ObLSStoreFormat store_format_; // set on initialization and then remain unchanged }; } // namespace storage diff --git a/src/storage/ls/ob_ls_tablet_service.cpp b/src/storage/ls/ob_ls_tablet_service.cpp index 5b2b0e609..25f0fffb1 100644 --- a/src/storage/ls/ob_ls_tablet_service.cpp +++ b/src/storage/ls/ob_ls_tablet_service.cpp @@ -1999,6 +1999,9 @@ int ObLSTabletService::create_tablet( ObTenantMetaMemMgr *t3m = MTL(ObTenantMetaMemMgr*); ObTransService *tx_svr = MTL(ObTransService*); const ObTabletMapKey key(ls_id, tablet_id); + const bool need_generate_cs_replica_cg_array = ls_->is_cs_replica() + && create_tablet_schema.is_row_store() + && create_tablet_schema.is_user_data_table(); ObTablet *tablet = nullptr; ObFreezer *freezer = ls_->get_freezer(); tablet_handle.reset(); @@ -2015,7 +2018,7 @@ int ObLSTabletService::create_tablet( ret = OB_ERR_UNEXPECTED; LOG_ERROR("new tablet is null", K(ret), KP(tablet), KP(allocator), K(tablet_handle)); } else if (OB_FAIL(tablet->init_for_first_time_creation(*allocator, ls_id, tablet_id, data_tablet_id, - create_scn, snapshot_version, create_tablet_schema, need_create_empty_major_sstable, freezer))) { + create_scn, snapshot_version, create_tablet_schema, need_create_empty_major_sstable, need_generate_cs_replica_cg_array, freezer))) { LOG_WARN("failed to init tablet", K(ret), K(ls_id), K(tablet_id), K(data_tablet_id), K(create_scn), K(snapshot_version), K(create_tablet_schema)); } else if (OB_FAIL(tablet->get_updating_tablet_pointer_param(param))) { @@ -2071,7 +2074,7 @@ int ObLSTabletService::create_inner_tablet( LOG_ERROR("new tablet is null", K(ret), KPC(tmp_tablet), K(tmp_tablet_hdl)); } else if (FALSE_IT(time_guard.click("CreateTablet"))) { } else if (OB_FAIL(tmp_tablet->init_for_first_time_creation(allocator, ls_id, tablet_id, data_tablet_id, - create_scn, snapshot_version, create_tablet_schema, true/*need_create_empty_major_sstable*/, freezer))) { + create_scn, snapshot_version, create_tablet_schema, true/*need_create_empty_major_sstable*/, false/*need_generate_cs_replica_cg_array*/, freezer))) { LOG_WARN("failed to init tablet", K(ret), K(ls_id), K(tablet_id), K(data_tablet_id), K(create_scn), K(snapshot_version), K(create_tablet_schema)); } else if (FALSE_IT(time_guard.click("InitTablet"))) { @@ -2403,7 +2406,7 @@ int ObLSTabletService::get_read_tables( } else if (FALSE_IT(allow_to_read_mgr_.load_allow_to_read_info(allow_to_read))) { } else if (!allow_to_read) { ret = OB_REPLICA_NOT_READABLE; - LOG_WARN("ls is not allow to read", K(ret), KPC(ls_)); + LOG_WARN("ls is not allow to read", K(ret), KPC(ls_), K(lbt())); } else if (FALSE_IT(key.ls_id_ = ls_->get_ls_id())) { } else if (OB_FAIL(ObTabletCreateDeleteHelper::check_and_get_tablet(key, handle, timeout_us, diff --git a/src/storage/ob_i_table.h b/src/storage/ob_i_table.h index 1308fc817..ac6d79862 100644 --- a/src/storage/ob_i_table.h +++ b/src/storage/ob_i_table.h @@ -157,6 +157,10 @@ public: OB_INLINE bool is_normal_cg_sstable() const { return ObITable::is_normal_cg_sstable(table_type_); } OB_INLINE bool is_cg_sstable() const { return ObITable::is_cg_sstable(table_type_); } OB_INLINE bool is_column_store_sstable() const { return is_co_sstable() || is_cg_sstable(); } + OB_INLINE bool is_row_store_major_sstable() const { return ObITable::is_row_store_major_sstable(table_type_); } + OB_INLINE bool is_column_store_major_sstable() const { return ObITable::is_column_store_major_sstable(table_type_); } + OB_INLINE bool is_true_major_sstable() const { return is_row_store_major_sstable() || is_column_store_major_sstable(); } + OB_INLINE const common::ObTabletID &get_tablet_id() const { return tablet_id_; } share::SCN get_start_scn() const { return scn_range_.start_scn_; } share::SCN get_end_scn() const { return scn_range_.end_scn_; } @@ -462,10 +466,34 @@ public: { return is_mds_mini_sstable(table_type) || is_mds_minor_sstable(table_type); } + static bool is_row_store_major_sstable(const TableType table_type) + { + return ObITable::TableType::MAJOR_SSTABLE == table_type; + } + static bool is_column_store_major_sstable(const TableType table_type) + { + return ObITable::TableType::COLUMN_ORIENTED_SSTABLE == table_type; + } + static bool is_valid_ddl_table_type(const TableType table_type) + { + return ObITable::DDL_MEM_SSTABLE == table_type + || ObITable::MAJOR_SSTABLE == table_type + || ObITable::DDL_DUMP_SSTABLE == table_type + || ObITable::COLUMN_ORIENTED_SSTABLE == table_type + || ObITable::DDL_MERGE_CO_SSTABLE; + } static bool is_table_with_scn_range(const TableType table_type) { return is_multi_version_table(table_type) || is_meta_major_sstable(table_type); } + // row store sstable and corresponding column store sstable + static bool is_twin_major_sstable(const TableKey &rs_key, const TableKey &cs_key) + { + return rs_key.is_true_major_sstable() + && cs_key.is_true_major_sstable() + && rs_key.tablet_id_ == cs_key.tablet_id_ + && rs_key.scn_range_ == cs_key.scn_range_; + } OB_INLINE static const char* get_table_type_name(const TableType &table_type) { return is_table_type_valid(table_type) ? table_type_name_[table_type] : nullptr; diff --git a/src/storage/ob_storage_schema.cpp b/src/storage/ob_storage_schema.cpp index 5742dcda0..2ef2f7327 100644 --- a/src/storage/ob_storage_schema.cpp +++ b/src/storage/ob_storage_schema.cpp @@ -434,7 +434,8 @@ int ObStorageSchema::init( const ObTableSchema &input_schema, const lib::Worker::CompatMode compat_mode, const bool skip_column_info/* = false*/, - const int64_t compat_version/* = STORAGE_SCHEMA_VERSION_LATEST*/) + const int64_t compat_version/* = STORAGE_SCHEMA_VERSION_LATEST*/, + const bool generate_cs_replica_cg_array/* = false*/) { int ret = OB_SUCCESS; @@ -444,6 +445,10 @@ int ObStorageSchema::init( } else if (OB_UNLIKELY(!input_schema.is_valid())) { ret = OB_INVALID_ARGUMENT; STORAGE_LOG(WARN, "invalid args", K(ret), K(input_schema), K(skip_column_info)); + } else if (FALSE_IT(column_info_simplified_ = skip_column_info)) { + } else if (OB_UNLIKELY(generate_cs_replica_cg_array && column_info_simplified_)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument to init storage schema", K(ret)); } else { allocator_ = &allocator; rowkey_array_.set_allocator(&allocator); @@ -459,11 +464,17 @@ int ObStorageSchema::init( if (OB_FAIL(ret)) { } else if (OB_FAIL(generate_str(input_schema))) { STORAGE_LOG(WARN, "failed to generate string", K(ret), K(input_schema)); - } else if (FALSE_IT(column_info_simplified_ = skip_column_info)) { } else if (OB_FAIL(generate_column_array(input_schema))) { STORAGE_LOG(WARN, "failed to generate column array", K(ret), K(input_schema)); + } else if (generate_cs_replica_cg_array) { + if (OB_FAIL(ObStorageSchema::generate_cs_replica_cg_array())) { + STORAGE_LOG(WARN, "failed to generate_cs_replica_cg_array", K(ret)); + } } else if (OB_FAIL(generate_column_group_array(input_schema, allocator))) { STORAGE_LOG(WARN, "Failed to generate column group array", K(ret)); + } + + if (OB_FAIL(ret)) { } else if (OB_UNLIKELY(!ObStorageSchema::is_valid())) { ret = OB_ERR_UNEXPECTED; STORAGE_LOG(ERROR, "storage schema is invalid", K(ret)); @@ -482,7 +493,8 @@ int ObStorageSchema::init( common::ObIAllocator &allocator, const ObStorageSchema &old_schema, const bool skip_column_info/* = false*/, - const ObStorageSchema *column_group_schema) + const ObStorageSchema *column_group_schema/* = nullptr*/, + const bool generate_cs_replica_cg_array/* = false*/) { int ret = OB_SUCCESS; @@ -492,6 +504,10 @@ int ObStorageSchema::init( } else if (OB_UNLIKELY(!old_schema.is_valid())) { ret = OB_INVALID_ARGUMENT; STORAGE_LOG(WARN, "invalid args", K(ret), K(old_schema), K(skip_column_info)); + } else if (FALSE_IT(column_info_simplified_ = (skip_column_info || old_schema.column_info_simplified_))) { + } else if (OB_UNLIKELY(generate_cs_replica_cg_array && (column_info_simplified_ || column_group_schema != nullptr))) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument to init storage schema", K(ret), K(column_info_simplified_), K(column_group_schema)); } else { allocator_ = &allocator; rowkey_array_.set_allocator(&allocator); @@ -506,7 +522,6 @@ int ObStorageSchema::init( compressor_type_ = old_schema.compressor_type_; column_cnt_ = old_schema.column_cnt_; store_column_cnt_ = old_schema.store_column_cnt_; - column_info_simplified_ = (skip_column_info || old_schema.column_info_simplified_); if (OB_FAIL(deep_copy_str(old_schema.encryption_, encryption_))) { STORAGE_LOG(WARN, "failed to deep copy encryption", K(ret), K(old_schema)); @@ -522,10 +537,17 @@ int ObStorageSchema::init( STORAGE_LOG(WARN, "failed to copy skip idx attr array", K(ret), K(old_schema)); } else if (!column_info_simplified_ && OB_FAIL(deep_copy_column_array(allocator, old_schema, old_schema.column_array_.count()))) { STORAGE_LOG(WARN, "failed to deep copy column array", K(ret), K(old_schema)); + } else if (generate_cs_replica_cg_array) { + if (OB_FAIL(ObStorageSchema::generate_cs_replica_cg_array())) { + STORAGE_LOG(WARN, "failed to generate_cs_replica_cg_array", K(ret)); + } } else if (NULL != column_group_schema && OB_FAIL(deep_copy_column_group_array(allocator, *column_group_schema))) { STORAGE_LOG(WARN, "failed to deep copy column array from column group schema", K(ret), K(old_schema), KPC(column_group_schema)); } else if (NULL == column_group_schema && OB_FAIL(deep_copy_column_group_array(allocator, old_schema))) { STORAGE_LOG(WARN, "failed to deep copy column array", K(ret), K(old_schema)); + } + + if (OB_FAIL(ret)) { } else if (OB_UNLIKELY(!is_valid())) { ret = OB_ERR_UNEXPECTED; STORAGE_LOG(ERROR, "storage schema is invalid", K(ret)); @@ -1043,6 +1065,107 @@ int ObStorageSchema::generate_all_column_group_schema(ObStorageColumnGroupSchema return ret; } +int ObStorageSchema::generate_cs_replica_cg_array(common::ObIAllocator &allocator, ObIArray &cg_schemas) const +{ + int ret = OB_SUCCESS; + int schema_rowkey_column_cnt = get_rowkey_column_num(); + cg_schemas.reset(); + ObStorageColumnGroupSchema column_group; + + if (OB_FAIL(cg_schemas.reserve(store_column_cnt_ + 1))) { + STORAGE_LOG(WARN, "failed to reserve for column group array", K(ret), K_(store_column_cnt)); + } else if (OB_FAIL(generate_rowkey_column_group_schema(column_group, ObRowStoreType::CS_ENCODING_ROW_STORE, allocator))) { + STORAGE_LOG(WARN, "failed to generate_rowkey_column_group_schema", K(ret)); + } else if (OB_FAIL(cg_schemas.push_back(column_group))) { + STORAGE_LOG(WARN, "failed to add column group", K(ret), K(column_group)); + column_group.destroy(allocator); + } + + for (int64_t i = 0; OB_SUCC(ret) && i < store_column_cnt_; i++) { + if (OB_FAIL(generate_single_column_group_schema(column_group, ObRowStoreType::CS_ENCODING_ROW_STORE, + i + (i >= schema_rowkey_column_cnt ? ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt() : 0), allocator))) { + STORAGE_LOG(WARN, "failed to generate_single_column_group_schema", K(ret), K(i)); + } else if (OB_FAIL(cg_schemas.push_back(column_group))) { + STORAGE_LOG(WARN, "failed to add column group", K(ret), K(column_group)); + column_group.destroy(allocator); + } + } + + if (OB_FAIL(ret)) { + for (int64_t i = 0; i < cg_schemas.count(); i++) { + cg_schemas.at(i).destroy(allocator); + } + } + + return ret; +} + +int ObStorageSchema::generate_cs_replica_cg_array() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(generate_cs_replica_cg_array(*allocator_, column_group_array_))) { + STORAGE_LOG(WARN, "Failed to generate column store cg array", K(ret), KPC(this)); + } else { + is_cs_replica_compat_ = true; + STORAGE_LOG(INFO, "[CS-Replica] Success to generate cs replica cg array", K(ret), KPC(this)); + } + return ret; +} + +int ObStorageSchema::generate_single_column_group_schema(ObStorageColumnGroupSchema &column_group, const ObRowStoreType row_store_type, const uint16_t column_idx, common::ObIAllocator &allocator) const +{ + int ret = OB_SUCCESS; + column_group.reset(); + column_group.version_ = ObStorageColumnGroupSchema::COLUMN_GRUOP_SCHEMA_VERSION; + column_group.type_ = SINGLE_COLUMN_GROUP; + column_group.schema_column_cnt_ = 1; + column_group.rowkey_column_cnt_ = 0; + column_group.schema_rowkey_column_cnt_ = column_group.rowkey_column_cnt_; + column_group.column_cnt_ = column_group.schema_column_cnt_; + + uint16_t *column_idxs = nullptr; + if (OB_ISNULL(column_idxs = reinterpret_cast (allocator_->alloc(sizeof(uint16_t) * column_group.column_cnt_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + STORAGE_LOG(WARN, "Failed to alloc memory", K(ret), K(column_cnt_)); + } else { + column_idxs[0] = column_idx; + column_group.column_idxs_ = column_idxs; + column_group.block_size_ = block_size_; + column_group.compressor_type_ = compressor_type_; + column_group.row_store_type_ = row_store_type; + } + + return ret; +} + +int ObStorageSchema::generate_rowkey_column_group_schema(ObStorageColumnGroupSchema &column_group, const ObRowStoreType row_store_type, common::ObIAllocator &allocator) const +{ + int ret = OB_SUCCESS; + column_group.reset(); + column_group.version_ = ObStorageColumnGroupSchema::COLUMN_GRUOP_SCHEMA_VERSION; + column_group.type_ = ROWKEY_COLUMN_GROUP; + column_group.schema_column_cnt_ = get_rowkey_column_num(); + column_group.rowkey_column_cnt_ = get_rowkey_column_num() + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); + column_group.schema_rowkey_column_cnt_ = column_group.schema_column_cnt_; + column_group.column_cnt_ = column_group.rowkey_column_cnt_; + + uint16_t *column_idxs = nullptr; + if (OB_ISNULL(column_idxs = reinterpret_cast (allocator_->alloc(sizeof(uint16_t) * column_group.column_cnt_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + STORAGE_LOG(WARN, "Failed to alloc memory", K(ret), K(column_cnt_)); + } else { + for (int64_t i = 0; i < column_group.column_cnt_; ++i) { + column_idxs[i] = i; + } + column_group.column_idxs_ = column_idxs; + column_group.block_size_ = block_size_; + column_group.compressor_type_ = compressor_type_; + column_group.row_store_type_ = row_store_type; + } + + return ret; +} + int ObStorageSchema::mock_row_store_cg(ObStorageColumnGroupSchema &mocked_row_store_cg) const { // if cache mocked_row_store_cg in storage schema, cached value will become invalid when ddl happen, so re-build every time @@ -1053,6 +1176,32 @@ int ObStorageSchema::mock_row_store_cg(ObStorageColumnGroupSchema &mocked_row_st return ret; } +int ObStorageSchema::transform_from_row_to_columnar() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + STORAGE_LOG(WARN, "not inited", K(ret), K_(is_inited)); + } else if (!is_row_store()) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "only row store schema can be transformed", K(ret), KPC(this)); + } else { + is_inited_ = false; + (void) reset_column_group_array(); + has_all_column_group_ = false; + if (OB_FAIL(ObStorageSchema::generate_cs_replica_cg_array())) { + STORAGE_LOG(WARN, "failed to generate_cs_replica_cg_array", K(ret)); + } else if (OB_UNLIKELY(!is_valid())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "invalid storage schema", K(ret), KPC(this)); + } else { + is_inited_ = true; + } + } + STORAGE_LOG(INFO, "[CS-Replica] finish transform row store storage schema", K(ret), KPC(this)); + return ret; +} + /* * base_cg of column store schema can only be ROWKEY_CG OR ALL_CG * "with column group(all columns, each column)" -> ALL_CG + each_cg diff --git a/src/storage/ob_storage_schema.h b/src/storage/ob_storage_schema.h index 80f255e35..52b069280 100644 --- a/src/storage/ob_storage_schema.h +++ b/src/storage/ob_storage_schema.h @@ -110,6 +110,20 @@ public: column_cnt_(0), column_idxs_(nullptr) {} + ObStorageColumnGroupSchema(const share::schema::ObColumnGroupType type, const ObCompressorType compressor_type, + const ObRowStoreType row_store_type, const uint32_t block_size, const uint16_t schema_column_cnt, const uint16_t rowkey_column_cnt, + const uint16_t schema_rowkey_column_cnt, const uint16_t column_cnt, uint16_t *column_idxs) + : version_(COLUMN_GRUOP_SCHEMA_VERSION), + type_(type), + compressor_type_(compressor_type), + row_store_type_(row_store_type), + block_size_(block_size), + schema_column_cnt_(schema_column_cnt), + rowkey_column_cnt_(rowkey_column_cnt), + schema_rowkey_column_cnt_(schema_rowkey_column_cnt), + column_cnt_(column_cnt), + column_idxs_(column_idxs) + {} ~ObStorageColumnGroupSchema() = default; OB_INLINE void reset() { MEMSET(this, 0, sizeof(ObStorageColumnGroupSchema)); } void destroy(ObIAllocator &allocator); @@ -164,12 +178,14 @@ public: const share::schema::ObTableSchema &input_schema, const lib::Worker::CompatMode compat_mode, const bool skip_column_info = false, - const int64_t compat_version = STORAGE_SCHEMA_VERSION_LATEST); + const int64_t compat_version = STORAGE_SCHEMA_VERSION_LATEST, + const bool generate_cs_replica_cg_array = false); int init( common::ObIAllocator &allocator, const ObStorageSchema &old_schema, const bool skip_column_info = false, - const ObStorageSchema *column_group_schema = nullptr); + const ObStorageSchema *column_group_schema = nullptr, + const bool generate_cs_replica_cg_array = false); int deep_copy_column_array( common::ObIAllocator &allocator, const ObStorageSchema &src_schema, @@ -217,6 +233,7 @@ public: inline bool is_materialized_view() const { return share::schema::ObTableSchema::is_materialized_view(table_type_); } inline bool is_mlog_table() const { return share::schema::ObTableSchema::is_mlog_table(table_type_); } inline bool is_fts_index() const { return share::schema::is_fts_index(index_type_); } + inline bool is_user_data_table() const { return share::schema::ObTableSchema::is_user_data_table(table_type_); } virtual inline bool is_global_index_table() const override { return share::schema::ObSimpleTableSchemaV2::is_global_index_table(index_type_); } virtual inline int64_t get_block_size() const override { return block_size_; } @@ -250,6 +267,7 @@ public: blocksstable::ObDatumRow &default_row) const; const ObStorageColumnSchema *get_column_schema(const int64_t column_id) const; int mock_row_store_cg(ObStorageColumnGroupSchema &mocked_row_store_cg) const; + int transform_from_row_to_columnar(); // TODO(chengkong): is used? int get_base_rowkey_column_group_index(int32_t &cg_idx) const; // This function only get cg idx for actually stored column int get_column_group_index( @@ -289,6 +307,8 @@ private: int generate_str(const share::schema::ObTableSchema &input_schema); int generate_column_array(const share::schema::ObTableSchema &input_schema); int generate_column_group_array(const share::schema::ObTableSchema &input_schema, common::ObIAllocator &allocator); + int generate_cs_replica_cg_array(common::ObIAllocator &allocator, ObIArray &cg_schemas) const; // also used by ddl + int generate_cs_replica_cg_array(); int get_column_ids_without_rowkey( common::ObIArray &column_ids, bool no_virtual) const; @@ -306,6 +326,8 @@ private: int64_t get_column_array_serialize_length(const common::ObIArray &array) const; int deserialize_skip_idx_attr_array(const char *buf, const int64_t data_len, int64_t &pos); int generate_all_column_group_schema(ObStorageColumnGroupSchema &column_group, const ObRowStoreType row_store_type) const; + int generate_rowkey_column_group_schema(ObStorageColumnGroupSchema &column_group, const ObRowStoreType row_store_type, common::ObIAllocator &allocator) const; + int generate_single_column_group_schema(ObStorageColumnGroupSchema &column_group, const ObRowStoreType row_store_type, const uint16_t column_idx, common::ObIAllocator &allocator) const; template int64_t get_array_serialize_length(const common::ObIArray &array) const; template diff --git a/src/storage/ob_storage_struct.cpp b/src/storage/ob_storage_struct.cpp index 6fc7c2fde..6f88daed1 100644 --- a/src/storage/ob_storage_struct.cpp +++ b/src/storage/ob_storage_struct.cpp @@ -338,7 +338,8 @@ ObDDLTableStoreParam::ObDDLTableStoreParam() ddl_checkpoint_scn_(SCN::min_scn()), ddl_snapshot_version_(0), ddl_execution_id_(-1), - data_format_version_(0) + data_format_version_(0), + ddl_table_type_(ObITable::MAX_TABLE_TYPE) { } @@ -350,7 +351,8 @@ bool ObDDLTableStoreParam::is_valid() const && ddl_checkpoint_scn_.is_valid() && ddl_snapshot_version_ >= 0 && ddl_execution_id_ >= 0 - && data_format_version_ >= 0; + && data_format_version_ >= 0 + && ObITable::is_valid_ddl_table_type(ddl_table_type_); } UpdateUpperTransParam::UpdateUpperTransParam() diff --git a/src/storage/ob_storage_struct.h b/src/storage/ob_storage_struct.h index e1682f96c..6a1b2d9e6 100644 --- a/src/storage/ob_storage_struct.h +++ b/src/storage/ob_storage_struct.h @@ -336,7 +336,7 @@ public: ~ObDDLTableStoreParam() = default; bool is_valid() const; TO_STRING_KV(K_(keep_old_ddl_sstable), K_(ddl_start_scn), K_(ddl_commit_scn), K_(ddl_checkpoint_scn), - K_(ddl_snapshot_version), K_(ddl_execution_id), K_(data_format_version)); + K_(ddl_snapshot_version), K_(ddl_execution_id), K_(data_format_version), K_(ddl_table_type)); public: bool keep_old_ddl_sstable_; share::SCN ddl_start_scn_; @@ -345,6 +345,8 @@ public: int64_t ddl_snapshot_version_; int64_t ddl_execution_id_; int64_t data_format_version_; + // used to decide storage type for replaying ddl clog in cs replica, see ObTabletMeta::ddl_table_type_ for more detail + ObITable::TableType ddl_table_type_; }; struct UpdateUpperTransParam final diff --git a/src/storage/ob_tenant_tablet_stat_mgr.cpp b/src/storage/ob_tenant_tablet_stat_mgr.cpp index ebd408aaa..a8468c8ef 100644 --- a/src/storage/ob_tenant_tablet_stat_mgr.cpp +++ b/src/storage/ob_tenant_tablet_stat_mgr.cpp @@ -1184,7 +1184,7 @@ int ObTenantTabletStatMgr::get_queuing_cfg( } } else { queuing_cfg = ObTableQueuingModeCfg::get_basic_config(stream_node->mode_); - LOG_DEBUG("chengkong debug: success get queuing cfg", K(ret), K(ls_id), K(tablet_id), K(queuing_cfg)); + LOG_DEBUG("success get queuing cfg", K(ret), K(ls_id), K(tablet_id), K(queuing_cfg)); } } return ret; diff --git a/src/storage/restore/ob_ls_restore_handler.cpp b/src/storage/restore/ob_ls_restore_handler.cpp index 82ea61fa7..30ee43626 100644 --- a/src/storage/restore/ob_ls_restore_handler.cpp +++ b/src/storage/restore/ob_ls_restore_handler.cpp @@ -1102,14 +1102,12 @@ int ObILSRestoreState::follower_fill_tablet_group_restore_arg_( LOG_WARN("fail to get location", K(ret), KPC(ls_)); } else if (OB_FAIL(location.get_leader(leader))) { LOG_WARN("fail to get leader location", K(ret), K(location)); - } else if (OB_FAIL(tablet_group_restore_arg.src_.set_replica_type(leader.get_replica_type()))) { - LOG_WARN("fail to set src replica type", K(ret), K(leader)); - } else if (OB_FAIL(tablet_group_restore_arg.src_.set_member(ObMember(leader.get_server(), 0/*invalid timestamp is ok*/)))) { - LOG_WARN("fail to set src member", K(ret)); - } else if (OB_FAIL(tablet_group_restore_arg.dst_.set_replica_type(REPLICA_TYPE_FULL))) { - LOG_WARN("fail to set dst replica type", K(ret)); - } else if (OB_FAIL(tablet_group_restore_arg.dst_.set_member(ObMember(GCTX.self_addr(), 0/*invalid timestamp is ok*/)))) { - LOG_WARN("fail to set dst member", K(ret), "server", GCTX.self_addr()); + } else if (OB_FAIL(tablet_group_restore_arg.src_.init( + leader.get_server(), 0/*invalid timestamp is ok*/, leader.get_replica_type()))) { + LOG_WARN("fail to init src_", K(ret), K(leader)); + } else if (OB_FAIL(tablet_group_restore_arg.dst_.init( + GCTX.self_addr(), 0/*invalid timestamp is ok*/, REPLICA_TYPE_FULL))) { + LOG_WARN("fail to init dst_", K(ret), K(GCTX.self_addr())); } else if (OB_FAIL(append(tablet_group_restore_arg.tablet_id_array_, tablet_need_restore))) { LOG_WARN("fail to append tablet id", K(ret), K(tablet_need_restore)); } else if (OB_FAIL(tablet_group_restore_arg.restore_base_info_.copy_from(*ls_restore_arg_))) { @@ -1163,7 +1161,7 @@ int ObILSRestoreState::get_follower_server_(ObIArray &follow common::ObMemberList member_list; GlobalLearnerList learner_list; int64_t full_replica_count = 0; - int64_t readonly_replica_count = 0; + int64_t non_paxos_replica_count = 0; if (OB_ISNULL(log_handler = ls_->get_log_handler())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("log handler should not be NULL", K(ret)); @@ -1171,12 +1169,12 @@ int ObILSRestoreState::get_follower_server_(ObIArray &follow LOG_WARN("failed to get paxos member list and learner list", K(ret)); } else if (OB_FAIL(location_service_->get(follower_info.cluster_id_, tenant_id, ls_->get_ls_id(), expire_renew_time, is_cache_hit, location))) { LOG_WARN("fail to get location", K(ret), KPC(ls_)); - } else if (OB_FAIL(location.get_replica_count(full_replica_count, readonly_replica_count))) { - LOG_WARN("fail to get replica count in location", KR(ret), K(location), K(full_replica_count), K(readonly_replica_count)); - } else if (full_replica_count != paxos_replica_num || readonly_replica_count != learner_list.get_member_number()) { + } else if (OB_FAIL(location.get_replica_count(full_replica_count, non_paxos_replica_count))) { + LOG_WARN("fail to get replica count in location", KR(ret), K(location), K(full_replica_count), K(non_paxos_replica_count)); + } else if (full_replica_count != paxos_replica_num || non_paxos_replica_count != learner_list.get_member_number()) { ret = OB_REPLICA_NUM_NOT_MATCH; LOG_WARN("replica num not match, ls may in migration", K(ret), K(location), K(full_replica_count), - K(readonly_replica_count), K(member_list), K(paxos_replica_num), K(learner_list)); + K(non_paxos_replica_count), K(member_list), K(paxos_replica_num), K(learner_list)); } else { const ObIArray &replica_locations = location.get_replica_locations(); for (int64_t i = 0; OB_SUCC(ret) && i < replica_locations.count(); ++i) { @@ -1990,14 +1988,10 @@ int ObLSRestoreSysTabletState::follower_fill_ls_restore_arg_(ObLSRestoreArg &arg LOG_WARN("fail to get location", K(ret), KPC(ls_)); } else if (OB_FAIL(location.get_leader(leader))) { LOG_WARN("fail to get leader location", K(ret), K(location)); - } else if (OB_FAIL(arg.src_.set_replica_type(leader.get_replica_type()))) { - LOG_WARN("fail to set src replica type", K(ret), K(leader)); - } else if (OB_FAIL(arg.src_.set_member(ObMember(leader.get_server(), 0/*invalid timestamp is ok*/)))) { - LOG_WARN("fail to set src member", K(ret)); - } else if (OB_FAIL(arg.dst_.set_replica_type(REPLICA_TYPE_FULL))) { - LOG_WARN("fail to set dst replica type", K(ret)); - } else if (OB_FAIL(arg.dst_.set_member(ObMember(GCTX.self_addr(), 0/*invalid timestamp is ok*/)))) { - LOG_WARN("fail to set dst member", K(ret), "server", GCTX.self_addr()); + } else if (OB_FAIL(arg.src_.init(leader.get_server(), 0/*invalid timestamp is ok*/, leader.get_replica_type()))) { + LOG_WARN("fail to init src_", K(ret), K(leader)); + } else if (OB_FAIL(arg.dst_.init(GCTX.self_addr(), 0/*invalid timestamp is ok*/, REPLICA_TYPE_FULL))) { + LOG_WARN("fail to init dst_", K(ret), K(GCTX.self_addr())); } else if (OB_FAIL(arg.restore_base_info_.copy_from(*ls_restore_arg_))) { LOG_WARN("fail to fill restore base info from ls restore args", K(ret), KPC(ls_restore_arg_)); } diff --git a/src/storage/tablet/ob_mds_schema_helper.cpp b/src/storage/tablet/ob_mds_schema_helper.cpp index 514e6363d..079f314af 100644 --- a/src/storage/tablet/ob_mds_schema_helper.cpp +++ b/src/storage/tablet/ob_mds_schema_helper.cpp @@ -141,6 +141,14 @@ const ObRowkeyReadInfo *ObMdsSchemaHelper::get_rowkey_read_info() const return ptr; } +bool ObMdsSchemaHelper::is_mds_schema(const ObTableSchema &table_schema) +{ + bool bret = false; + const ObString &table_name = table_schema.get_table_name_str(); + const uint64_t table_id = table_schema.get_table_id(); + return MDS_TABLE_ID == table_id && 0 == table_name.case_compare(MDS_TABLE_NAME); +} + int ObMdsSchemaHelper::build_table_schema( const uint64_t tenant_id, const int64_t database_id, diff --git a/src/storage/tablet/ob_mds_schema_helper.h b/src/storage/tablet/ob_mds_schema_helper.h index 43055894c..c11457058 100644 --- a/src/storage/tablet/ob_mds_schema_helper.h +++ b/src/storage/tablet/ob_mds_schema_helper.h @@ -54,6 +54,8 @@ public: const share::schema::ObTableSchema *get_table_schema() const; const ObStorageSchema *get_storage_schema() const; const ObRowkeyReadInfo *get_rowkey_read_info() const; +public: + static bool is_mds_schema(const ObTableSchema &table_schema); private: static int build_table_schema( const uint64_t tenant_id, diff --git a/src/storage/tablet/ob_table_store_util.cpp b/src/storage/tablet/ob_table_store_util.cpp index 232d8373b..3281d7a87 100644 --- a/src/storage/tablet/ob_table_store_util.cpp +++ b/src/storage/tablet/ob_table_store_util.cpp @@ -560,6 +560,46 @@ int ObSSTableArray::get_all_tables(ObIArray &tables) const return ret; } +int ObSSTableArray::replace_twin_majors_and_build_new( + const ObIArray &tables_array, + ObIArray &major_tables) const +{ + int ret = OB_SUCCESS; + ObITable* new_co_major = nullptr; // new co major to replace old row store major + if (OB_UNLIKELY(!is_valid())) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (tables_array.count() != 1) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid major table cnt for replacing old row store", K(ret), K(tables_array)); + } else if (FALSE_IT(new_co_major = tables_array.at(0))) { + } else if (OB_UNLIKELY(nullptr == new_co_major || !new_co_major->is_column_store_sstable())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected new co table", K(ret), KPC(new_co_major)); + } else { + ObSSTable *table = nullptr; + for (int64_t i = 0; OB_SUCC(ret) && i < cnt_; ++i) { + if (OB_ISNULL(table = sstable_array_[i])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null table", K(ret)); + } else if (!table->get_key().is_row_store_major_sstable()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected sstable type", K(ret), KPC(table)); + } else if (ObITable::is_twin_major_sstable(table->get_key(), new_co_major->get_key())) { + // skip the old row store major + } else if (OB_FAIL(major_tables.push_back(table))) { + LOG_WARN("fail to push sstable address into array", K(ret), K(i), K(major_tables)); + } + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(major_tables.push_back(new_co_major))) { + LOG_WARN("fail to push sstable address into array", K(ret), KPC(new_co_major), K(major_tables)); + } + } + return ret; +} + int ObSSTableArray::get_all_table_wrappers( ObIArray &table_wrappers, const bool need_unpack) const diff --git a/src/storage/tablet/ob_table_store_util.h b/src/storage/tablet/ob_table_store_util.h index ca685b194..973c0ffb8 100644 --- a/src/storage/tablet/ob_table_store_util.h +++ b/src/storage/tablet/ob_table_store_util.h @@ -86,6 +86,10 @@ public: TO_STRING_KV(K_(cnt), KP_(sstable_array), K_(serialize_table_type), K_(is_inited)); private: int get_all_tables(ObIArray &tables) const; + // construct major_tables with old sstable array and input tables_array, but filter twin sstable of new_co_major + int replace_twin_majors_and_build_new( + const ObIArray &tables_array, + ObIArray &major_tables) const; int inc_meta_ref_cnt(bool &inc_success) const; int inc_data_ref_cnt(bool &inc_success) const; void dec_meta_ref_cnt() const; diff --git a/src/storage/tablet/ob_tablet.cpp b/src/storage/tablet/ob_tablet.cpp index 69fbbfbe4..b8b3d9686 100644 --- a/src/storage/tablet/ob_tablet.cpp +++ b/src/storage/tablet/ob_tablet.cpp @@ -258,7 +258,7 @@ ObTablet::ObTablet() table_store_cache_() { #if defined(__x86_64__) && !defined(ENABLE_OBJ_LEAK_CHECK) - check_size(); + check_size(); #endif MEMSET(memtables_, 0x0, sizeof(memtables_)); } @@ -331,6 +331,7 @@ int ObTablet::init_for_first_time_creation( const int64_t snapshot_version, const ObCreateTabletSchema &storage_schema, const bool need_create_empty_major_sstable, + const bool need_generate_cs_replica_cg_array, ObFreezer *freezer) { int ret = OB_SUCCESS; @@ -373,7 +374,8 @@ int ObTablet::init_for_first_time_creation( } else { if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, storage_schema_addr_.ptr_))) { LOG_WARN("fail to allocate and new object", K(ret)); - } else if (OB_FAIL(storage_schema_addr_.get_ptr()->init(allocator, storage_schema))) { + } else if (OB_FAIL(storage_schema_addr_.get_ptr()->init(allocator, storage_schema, false /*skip_column_info*/, + nullptr /*column_group_schema*/, need_generate_cs_replica_cg_array))) { LOG_WARN("fail to initialize tablet member", K(ret), K(storage_schema_addr_)); } } @@ -391,7 +393,7 @@ int ObTablet::init_for_first_time_creation( LOG_WARN("failed to update start scn", K(ret), K(table_store_addr_)); } else if (OB_FAIL(table_store_cache_.init(table_store_addr_.get_ptr()->get_major_sstables(), table_store_addr_.get_ptr()->get_minor_sstables(), - storage_schema.is_row_store()))) { + storage_schema_addr_.get_ptr()->is_row_store()))) { LOG_WARN("failed to init table store cache", K(ret), KPC(this)); } else if (OB_FAIL(check_sstable_column_checksum())) { LOG_WARN("failed to check sstable column checksum", K(ret), KPC(this)); @@ -504,9 +506,8 @@ int ObTablet::init_for_merge( } else if (FALSE_IT(set_initial_addr())) { } else if (OB_FAIL(inner_inc_macro_ref_cnt())) { LOG_WARN("failed to increase macro ref cnt", K(ret)); - } else if (OB_UNLIKELY(old_tablet.is_row_store() != param.storage_schema_->is_row_store())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("Unexpected schema for chaser debug", K(ret), KPC(param.storage_schema_), K(old_tablet)); + } else if (OB_FAIL(check_tablet_schema_mismatch(old_tablet, *param.storage_schema_, is_convert_co_major_merge(param.merge_type_)))) { + LOG_ERROR("find error while checking tablet schema mismatch", K(ret), KPC(param.storage_schema_), K(old_tablet), K(param.merge_type_)); } else if (OB_FAIL(check_table_store_flag_match_with_table_store_(table_store_addr_.get_ptr()))) { LOG_WARN("failed to check table store flag match with table store", K(ret), K(old_tablet), K_(table_store_addr)); } else { @@ -590,6 +591,7 @@ int ObTablet::init_with_migrate_param( ObTableHandleV2 mds_mini_sstable; const blocksstable::ObSSTable *sstable = nullptr; const bool need_compat = !tablet_id.is_ls_inner_tablet() && param.version_ < ObMigrationTabletParam::PARAM_VERSION_V3; + bool need_process_cs_replica = false; if (is_transfer) { // do nothing } else if (!need_compat) { @@ -640,13 +642,18 @@ int ObTablet::init_with_migrate_param( } if (OB_SUCC(ret)) { - ALLOC_AND_INIT(allocator, storage_schema_addr_, param.storage_schema_); + // since transfer use storage schema from ls leader, need convert into cs storage schema in cs replica + if (!is_transfer) { + ALLOC_AND_INIT(allocator, storage_schema_addr_, param.storage_schema_); + } else if (OB_FAIL(inner_alloc_and_init_storage_schema(allocator, ls_id, tablet_id, param.storage_schema_, need_process_cs_replica))) { + LOG_WARN("failed to int storage schema", K(ret), K(ls_id), K(tablet_id), K(param)); + } } if (OB_FAIL(ret)) { } else if (OB_FAIL(table_store_cache_.init(table_store_addr_.get_ptr()->get_major_sstables(), table_store_addr_.get_ptr()->get_minor_sstables(), - param.storage_schema_.is_row_store()))) { + param.storage_schema_.is_row_store() && !need_process_cs_replica))) { LOG_WARN("failed to init table store cache", K(ret), KPC(this)); } else if (OB_FAIL(build_read_info(allocator))) { LOG_WARN("fail to build read info", K(ret)); @@ -659,6 +666,8 @@ int ObTablet::init_with_migrate_param( } else if (FALSE_IT(set_initial_addr())) { } else if (OB_FAIL(inner_inc_macro_ref_cnt())) { LOG_WARN("failed to increase macro ref cnt", K(ret)); + } else if (OB_FAIL(check_tablet_schema_mismatch(*this, *storage_schema_addr_.ptr_, false/*is_convert_co_major_merge*/))) { + LOG_ERROR("find error while checking tablet schema mismatch", K(ret), KPC(storage_schema_addr_.ptr_), KPC(this)); } else { is_inited_ = true; LOG_INFO("succeeded to init tablet with migration tablet param", K(ret), K(param), KPC(this)); @@ -732,9 +741,6 @@ int ObTablet::init_for_defragment( } else if (FALSE_IT(set_initial_addr())) { } else if (OB_FAIL(inner_inc_macro_ref_cnt())) { LOG_WARN("failed to increase macro ref cnt", K(ret)); - } else if (OB_UNLIKELY(old_tablet.is_row_store() != old_storage_schema->is_row_store())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("Unexpected schema for chaser debug", K(ret), KPC(old_storage_schema), K(old_tablet)); } else if (OB_FAIL(table_store_cache_.init(table_store_addr_.get_ptr()->get_major_sstables(), table_store_addr_.get_ptr()->get_minor_sstables(), old_storage_schema->is_row_store()))) { @@ -844,9 +850,8 @@ int ObTablet::init_for_sstable_replace( LOG_WARN("failed to check sstable column checksum", K(ret), KPC(this)); } else if (param.is_transfer_replace_ && OB_FAIL(handle_transfer_replace_(param))) { LOG_WARN("failed to handle transfer replace", K(ret), K(param)); - } else if (OB_UNLIKELY(old_tablet.is_row_store() != storage_schema->is_row_store())) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("Unexpected schema for chaser debug", K(ret), KPC(storage_schema), K(old_tablet)); + } else if (OB_FAIL(check_tablet_schema_mismatch(old_tablet, *storage_schema, false/*is_convert_co_major_merge*/))) { + LOG_ERROR("find error while checking tablet schema mismatch", K(ret), KPC(storage_schema), K(old_tablet)); } else if (OB_FAIL(table_store_cache_.init(table_store_addr_.get_ptr()->get_major_sstables(), table_store_addr_.get_ptr()->get_minor_sstables(), storage_schema->is_row_store()))) { @@ -1131,6 +1136,45 @@ int ObTablet::init_with_update_medium_info( return ret; } +// delayed add column group and convert co major merge allow mismatch between old tablet and new schema +int ObTablet::check_tablet_schema_mismatch( + const ObTablet &old_tablet, + const ObStorageSchema &storage_schema, + const bool is_convert_co_major_merge) +{ + int ret = OB_SUCCESS; + ObLSHandle ls_handle; + ObLS *ls = nullptr; + const bool is_old_tablet_row_store = old_tablet.is_row_store(); + const bool is_storage_schema_row_store = storage_schema.is_row_store(); + if (OB_FAIL(MTL(ObLSService *)->get_ls(tablet_meta_.ls_id_, ls_handle, ObLSGetMod::TABLET_MOD))) { + LOG_WARN("failed to get ls", K(ret), "ls_id", tablet_meta_.ls_id_); + } else if (OB_UNLIKELY(!ls_handle.is_valid()) || OB_ISNULL(ls = ls_handle.get_ls())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls handle is invalid or nullptr", K(ret), K(ls_handle), KP(ls)); + } else if (ls->is_cs_replica()) { + LOG_INFO("For columns tore replica, allow old tablet and new schema mismatch", K(ret), K(old_tablet), K(storage_schema)); + } else if (is_old_tablet_row_store) { + if (is_storage_schema_row_store) { + // row store status match + } else if (is_convert_co_major_merge) { + // convert co major merge + LOG_INFO("convert co major merge, old tablet is row store and new storage schema is column store", K(ret), K(old_tablet), K(storage_schema)); + } else { + // delayed add column group + LOG_INFO("should be delayed add column group, old tablet is row store and new storage schema is column store", K(ret), K(old_tablet), K(storage_schema)); + } + } else { + if (!is_storage_schema_row_store) { + // column store status match + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected schema, old tablet is column store while new storage schema is column store", K(ret), K(old_tablet), K(storage_schema)); + } + } + return ret; +} + int ObTablet::update_meta_last_persisted_committed_tablet_status_from_sstable( const ObUpdateTableStoreParam ¶m, const ObTabletCreateDeleteMdsUserData &old_last_persisted_committed_tablet_status) @@ -4994,6 +5038,7 @@ int ObTablet::build_migration_tablet_param( mig_tablet_param.ddl_execution_id_ = tablet_meta_.ddl_execution_id_; mig_tablet_param.ddl_data_format_version_ = tablet_meta_.ddl_data_format_version_; mig_tablet_param.ddl_commit_scn_ = tablet_meta_.ddl_commit_scn_; + mig_tablet_param.ddl_table_type_ = tablet_meta_.ddl_table_type_; mig_tablet_param.report_status_ = tablet_meta_.report_status_; mig_tablet_param.mds_checkpoint_scn_ = tablet_meta_.mds_checkpoint_scn_; mig_tablet_param.transfer_info_ = tablet_meta_.transfer_info_; @@ -5480,6 +5525,49 @@ int ObTablet::update_tablet_autoinc_seq(const uint64_t autoinc_seq) return ret; } +int ObTablet::check_cs_replica_compat_schema(bool &is_cs_replica_compat) +{ + int ret = OB_SUCCESS; + is_cs_replica_compat = false; + ObStorageSchema *storage_schema = nullptr; + ObArenaAllocator arena_allocator(common::ObMemAttr(MTL_ID(), "CSReplSchema")); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", K(ret)); + } else if (OB_FAIL(load_storage_schema(arena_allocator, storage_schema))) { + LOG_WARN("fail to load storage schema", K(ret), K_(storage_schema_addr)); + } else { + // column storage schema + is_cs_replica_compat = storage_schema->is_cs_replica_compat_; + } + ObTabletObjLoadHelper::free(arena_allocator, storage_schema); + return ret; +} + +int ObTablet::pre_process_cs_replica(ObTabletDirectLoadInsertParam &direct_load_param) +{ + int ret = OB_SUCCESS; + ObLSHandle ls_handle; + ObLS *ls = nullptr; + bool is_cs_replica_compat = false; + if (OB_FAIL(MTL(ObLSService *)->get_ls(tablet_meta_.ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { + LOG_WARN("failed to get ls", K(ret), "ls_id", tablet_meta_.ls_id_); + } else if (OB_UNLIKELY(!ls_handle.is_valid()) || OB_ISNULL(ls = ls_handle.get_ls())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls handle is invalid or nullptr", K(ret), K(ls_handle), KP(ls)); + } else if (!ls->is_cs_replica()) { + } else if (is_row_store()) { + // do not need to process cs replica + } else if (OB_FAIL(check_cs_replica_compat_schema(is_cs_replica_compat))) { + LOG_WARN("failed to check cs replica compat", K(ret), KPC(this)); + } else if (is_cs_replica_compat && tablet_meta_.ddl_table_type_ == ObITable::DDL_DUMP_SSTABLE) { + direct_load_param.common_param_.replay_normal_in_cs_replica_ = true; + LOG_INFO("[CS-Replica] Set replay normal in cs replica", K(direct_load_param)); + } + LOG_TRACE("[CS-Replica] process cs replica when start direct load task", KPC(ls), K(is_cs_replica_compat), K(direct_load_param), K_(tablet_meta)); + return ret; +} + int ObTablet::start_direct_load_task_if_need() { int ret = OB_SUCCESS; @@ -5526,7 +5614,9 @@ int ObTablet::start_direct_load_task_if_need() direct_load_param.common_param_.direct_load_type_ = ObDirectLoadType::DIRECT_LOAD_DDL; direct_load_param.common_param_.read_snapshot_ = tablet_meta_.ddl_snapshot_version_; - if (OB_FAIL(tenant_direct_load_mgr->create_tablet_direct_load( + if (OB_FAIL(pre_process_cs_replica(direct_load_param))) { + LOG_WARN("failed to process cs replica", K(ret), KPC(this)); + } else if (OB_FAIL(tenant_direct_load_mgr->create_tablet_direct_load( unused_context_id, tablet_meta_.ddl_execution_id_, direct_load_param, @@ -6394,6 +6484,7 @@ int ObTablet::build_transfer_tablet_param_current_( mig_tablet_param.max_sync_storage_schema_version_ = mig_tablet_param.storage_schema_.schema_version_; mig_tablet_param.ddl_execution_id_ = tablet_meta_.ddl_execution_id_; mig_tablet_param.ddl_data_format_version_ = tablet_meta_.ddl_data_format_version_; + mig_tablet_param.ddl_table_type_ = tablet_meta_.ddl_table_type_; mig_tablet_param.mds_checkpoint_scn_ = user_data.transfer_scn_; mig_tablet_param.report_status_.reset(); @@ -6442,7 +6533,8 @@ int64_t ObTablet::to_string(char *buf, const int64_t buf_len) const K_(macro_info_addr), K_(mds_data), KP_(ddl_kvs), - K_(ddl_kv_count)); + K_(ddl_kv_count), + K_(table_store_cache)); J_COMMA(); BUF_PRINTF("memtables:"); J_ARRAY_START(); @@ -7870,6 +7962,33 @@ int ObTablet::build_transfer_backfill_tablet_param( return ret; } +int ObTablet::inner_alloc_and_init_storage_schema( + common::ObArenaAllocator &allocator, + const share::ObLSID &ls_id, + const ObTabletID &tablet_id, + const ObStorageSchema &input_storage_schema, + bool &need_process_cs_replica) +{ + int ret = OB_SUCCESS; + ObLSHandle ls_handle; + ObLS *ls = nullptr; + need_process_cs_replica = false; + if (OB_FAIL(MTL(ObLSService*)->get_ls(ls_id, ls_handle, ObLSGetMod::HA_MOD))) { + LOG_WARN("failed to get ls", K(ret), K(ls_id)); + } else if (OB_UNLIKELY(!ls_handle.is_valid()) || OB_ISNULL(ls = ls_handle.get_ls())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls is invalid or nullptr", K(ret), K(ls_id), K(ls_handle), KPC(ls)); + } else if (OB_FAIL(ObCSReplicaUtil::check_need_process_cs_replica(*ls, tablet_id, input_storage_schema, need_process_cs_replica))) { + LOG_WARN("failed to check need process cs replica", K(ret), K(ls_id), K(tablet_id), K(input_storage_schema)); + } else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, storage_schema_addr_.ptr_))) { + LOG_WARN("fail to alloc and new storage schema", K(ret)); + } else if (OB_FAIL(storage_schema_addr_.get_ptr()->init(allocator, input_storage_schema, + false /*skip_column_info*/, nullptr /*column_group_schema*/, need_process_cs_replica))) { + LOG_WARN("fail to init storage schema", K(ret), K(input_storage_schema), K(need_process_cs_replica)); + } + return ret; +} + int ObTablet::check_table_store_flag_match_with_table_store_(const ObTabletTableStore *table_store) { int ret = OB_SUCCESS; diff --git a/src/storage/tablet/ob_tablet.h b/src/storage/tablet/ob_tablet.h index 1e66ba0c1..b15a72e95 100644 --- a/src/storage/tablet/ob_tablet.h +++ b/src/storage/tablet/ob_tablet.h @@ -91,6 +91,7 @@ class ObCOSSTableV2; class ObMacroInfoIterator; class ObMdsRowIterator; class ObMdsMiniMergeOperator; +struct ObTabletDirectLoadInsertParam; struct ObTableStoreCache { @@ -114,6 +115,7 @@ public: int64_t recycle_version_; int64_t last_major_column_count_; bool is_row_store_; + // TODO(chengkong): add bool is_user_tablet_; common::ObCompressorType last_major_compressor_type_; common::ObRowStoreType last_major_latest_row_store_type_; }; @@ -179,6 +181,7 @@ public: const int64_t snapshot_version, const ObCreateTabletSchema &storage_schema, const bool need_create_empty_major_sstable, + const bool need_generate_cs_replica_cg_array, ObFreezer *freezer); // dump/merge build new multi version tablet int init_for_merge( @@ -428,7 +431,13 @@ public: const ObTabletMeta &src_tablet_meta, const ObStorageSchema &src_storage_schema, ObMigrationTabletParam ¶m) const; - + // transfer use storage schema from ls leader to create tablet, need specially process in cs replica + int inner_alloc_and_init_storage_schema( + common::ObArenaAllocator &allocator, + const share::ObLSID &ls_id, + const ObTabletID &tablet_id, + const ObStorageSchema &input_storage_schema, + bool &need_process_cs_replica); int get_ddl_kv_mgr(ObDDLKvMgrHandle &ddl_kv_mgr_handle, bool try_create = false); int set_ddl_kv_mgr(const ObDDLKvMgrHandle &ddl_kv_mgr_handle); int remove_ddl_kv_mgr(const ObDDLKvMgrHandle &ddl_kv_mgr_handle); @@ -440,6 +449,10 @@ public: int get_recycle_version(const int64_t multi_version_start, int64_t &recycle_version) const; int get_migration_sstable_size(int64_t &data_size); + // column store replica + int check_cs_replica_compat_schema(bool &is_cs_replica_compat); + int pre_process_cs_replica(ObTabletDirectLoadInsertParam &direct_load_param); + // other const ObMetaDiskAddr &get_tablet_addr() const { return tablet_addr_; } const ObTabletMeta &get_tablet_meta() const { return tablet_meta_; } @@ -560,6 +573,10 @@ protected:// for MDS use const bool create_if_not_exist) const override final; virtual ObTabletPointer *get_tablet_pointer_() const override final; private: + int check_tablet_schema_mismatch( + const ObTablet &old_tablet, + const ObStorageSchema &storage_schema, + const bool is_convert_co_major_merge); int update_meta_last_persisted_committed_tablet_status_from_sstable( const ObUpdateTableStoreParam ¶m, const ObTabletCreateDeleteMdsUserData &last_tablet_status); diff --git a/src/storage/tablet/ob_tablet_meta.cpp b/src/storage/tablet/ob_tablet_meta.cpp index 9aed3bf51..9d0712f67 100644 --- a/src/storage/tablet/ob_tablet_meta.cpp +++ b/src/storage/tablet/ob_tablet_meta.cpp @@ -64,6 +64,7 @@ ObTabletMeta::ObTabletMeta() last_persisted_committed_tablet_status_(), space_usage_(), create_schema_version_(0), + ddl_table_type_(ObITable::MAX_TABLE_TYPE), compat_mode_(lib::Worker::CompatMode::INVALID), has_next_tablet_(false), is_inited_(false) @@ -123,6 +124,7 @@ int ObTabletMeta::init( max_sync_storage_schema_version_ = create_schema_version; ddl_execution_id_ = -1; ddl_data_format_version_ = 0; + ddl_table_type_ = ObITable::DDL_MEM_SSTABLE; mds_checkpoint_scn_ = INIT_CLOG_CHECKPOINT_SCN; report_status_.merge_snapshot_version_ = snapshot_version; @@ -186,6 +188,7 @@ int ObTabletMeta::init( ddl_snapshot_version_ = MAX(old_tablet_meta.ddl_snapshot_version_, ddl_info.ddl_snapshot_version_); ddl_execution_id_ = MAX(old_tablet_meta.ddl_execution_id_, ddl_info.ddl_execution_id_); ddl_data_format_version_ = MAX(old_tablet_meta.ddl_data_format_version_, ddl_info.data_format_version_); + ddl_table_type_ = (ObITable::MAX_TABLE_TYPE == ddl_info.ddl_table_type_) ? old_tablet_meta.ddl_table_type_ : ddl_info.ddl_table_type_; mds_checkpoint_scn_ = old_tablet_meta.mds_checkpoint_scn_; transfer_info_ = old_tablet_meta.transfer_info_; extra_medium_info_ = old_tablet_meta.extra_medium_info_; @@ -227,6 +230,7 @@ int ObTabletMeta::init( ref_tablet_id_ = old_tablet_meta.ref_tablet_id_; create_scn_ = old_tablet_meta.create_scn_; create_schema_version_ = old_tablet_meta.create_schema_version_; + ddl_table_type_ = old_tablet_meta.ddl_table_type_; start_scn_ = old_tablet_meta.start_scn_; clog_checkpoint_scn_ = old_tablet_meta.clog_checkpoint_scn_; ddl_checkpoint_scn_ = old_tablet_meta.ddl_checkpoint_scn_; @@ -241,6 +245,7 @@ int ObTabletMeta::init( ddl_execution_id_ = old_tablet_meta.ddl_execution_id_; ddl_data_format_version_ = old_tablet_meta.ddl_data_format_version_; ddl_snapshot_version_ = old_tablet_meta.ddl_snapshot_version_; + ddl_table_type_ = old_tablet_meta.ddl_table_type_; max_sync_storage_schema_version_ = old_tablet_meta.max_sync_storage_schema_version_; max_serialized_medium_scn_ = old_tablet_meta.max_serialized_medium_scn_; mds_checkpoint_scn_ = SCN::max(flush_scn, old_tablet_meta.mds_checkpoint_scn_); @@ -300,6 +305,7 @@ int ObTabletMeta::init( max_serialized_medium_scn_ = param.max_serialized_medium_scn_; ddl_execution_id_ = param.ddl_execution_id_; ddl_data_format_version_ = param.ddl_data_format_version_; + ddl_table_type_ = param.ddl_table_type_; mds_checkpoint_scn_ = param.mds_checkpoint_scn_; transfer_info_ = param.transfer_info_; extra_medium_info_ = param.extra_medium_info_; @@ -360,6 +366,7 @@ int ObTabletMeta::assign(const ObTabletMeta &other) max_sync_storage_schema_version_ = other.max_sync_storage_schema_version_; ddl_execution_id_ = other.ddl_execution_id_; ddl_data_format_version_ = other.ddl_data_format_version_; + ddl_table_type_ = other.ddl_table_type_; max_serialized_medium_scn_ = other.max_serialized_medium_scn_; ddl_commit_scn_ = other.ddl_commit_scn_; mds_checkpoint_scn_ = other.mds_checkpoint_scn_; @@ -464,6 +471,7 @@ int ObTabletMeta::init( max_sync_storage_schema_version_ = max_sync_storage_schema_version; ddl_execution_id_ = old_tablet_meta.ddl_execution_id_; ddl_data_format_version_ = old_tablet_meta.ddl_data_format_version_; + ddl_table_type_ = old_tablet_meta.ddl_table_type_; max_serialized_medium_scn_ = MAX(old_tablet_meta.max_serialized_medium_scn_, OB_ISNULL(tablet_meta) ? 0 : tablet_meta->max_serialized_medium_scn_); ddl_commit_scn_ = old_tablet_meta.ddl_commit_scn_; @@ -517,6 +525,7 @@ void ObTabletMeta::reset() max_serialized_medium_scn_ = 0; ddl_execution_id_ = -1; ddl_data_format_version_ = 0; + ddl_table_type_ = ObITable::MAX_TABLE_TYPE; mds_checkpoint_scn_.reset(); transfer_info_.reset(); extra_medium_info_.reset(); @@ -548,7 +557,8 @@ bool ObTabletMeta::is_valid() const && start_scn_ >= INIT_CLOG_CHECKPOINT_SCN && start_scn_ <= clog_checkpoint_scn_)) && create_schema_version_ >= 0 - && space_usage_.is_valid(); + && space_usage_.is_valid() + && ObITable::is_valid_ddl_table_type(ddl_table_type_); } int ObTabletMeta::serialize(char *buf, const int64_t len, int64_t &pos) const @@ -634,6 +644,8 @@ int ObTabletMeta::serialize(char *buf, const int64_t len, int64_t &pos) const LOG_WARN("failed to serialize extra_medium_info", K(ret), K(len), K(new_pos), K_(extra_medium_info)); } else if (new_pos - pos < length && OB_FAIL(last_persisted_committed_tablet_status_.serialize(buf, len, new_pos))) { LOG_WARN("failed to serialize last_persisted_committed_tablet_status", K(ret), K(len), K(new_pos), K_(last_persisted_committed_tablet_status)); + } else if (new_pos - pos < length && OB_FAIL(serialization::encode(buf, len, new_pos, ddl_table_type_))) { + LOG_WARN("failed to serialize ddl table type", K(ret), K(len), K(new_pos), K_(ddl_table_type)); } else if (OB_UNLIKELY(length != new_pos - pos)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet meta's length doesn't match standard length", K(ret), K(new_pos), K(pos), K(length), K(length)); @@ -669,6 +681,7 @@ int ObTabletMeta::deserialize( } else if (TABLET_META_VERSION == version_) { int8_t compat_mode = -1; ddl_execution_id_ = 0; + ddl_table_type_ = ObITable::DDL_MEM_SSTABLE; if (OB_UNLIKELY(length_ > len - pos)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("buffer's length is not enough", K(ret), K(length_), K(len - new_pos)); @@ -728,6 +741,8 @@ int ObTabletMeta::deserialize( LOG_WARN("failed to deserialize extra_medium_info", K(ret), K(len), K(new_pos)); } else if (new_pos - pos < length_ && OB_FAIL(last_persisted_committed_tablet_status_.deserialize(buf, len, new_pos))) { LOG_WARN("failed to deserialize last_persisted_committed_tablet_status", K(ret), K(len), K(new_pos)); + } else if (new_pos - pos < length_ && OB_FAIL(serialization::decode(buf, len, new_pos, ddl_table_type_))) { + LOG_WARN("failed to deserialize ddl table type", K(ret), K(len), K(new_pos)); } else if (OB_UNLIKELY(length_ != new_pos - pos)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet's length doesn't match standard length", K(ret), K(new_pos), K(pos), K_(length)); @@ -778,6 +793,7 @@ int64_t ObTabletMeta::get_serialize_size() const size += space_usage_.get_serialize_size(); size += extra_medium_info_.get_serialize_size(); size += last_persisted_committed_tablet_status_.get_serialize_size(); + size += serialization::encoded_length(ddl_table_type_); return size; } @@ -978,6 +994,7 @@ ObMigrationTabletParam::ObMigrationTabletParam() mds_data_(), transfer_info_(), create_schema_version_(0), + ddl_table_type_(ObITable::MAX_TABLE_TYPE), allocator_("MigTblParam", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID(), ObCtxIds::DEFAULT_CTX_ID) { } @@ -1155,6 +1172,8 @@ int ObMigrationTabletParam::serialize(char *buf, const int64_t len, int64_t &pos LOG_WARN("failed to serialize extra_medium_info", K(ret), K(len), K(new_pos), K_(extra_medium_info)); } else if (PARAM_VERSION_V3 <= version_ && new_pos - pos < length && OB_FAIL(last_persisted_committed_tablet_status_.serialize(buf, len, new_pos))) { LOG_WARN("failed to serialize last_persisted_committed_tablet_status", K(ret), K(len), K(new_pos), K_(last_persisted_committed_tablet_status)); + } else if (PARAM_VERSION_V3 <= version_ && new_pos - pos < length && OB_FAIL(serialization::encode(buf, len, new_pos, ddl_table_type_))) { + LOG_WARN("failed to serialize ddk table type", K(ret), K(len), K(new_pos), K_(ddl_table_type)); } else if (OB_UNLIKELY(length != new_pos - pos)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("length doesn't match standard length", K(ret), K(new_pos), K(pos), K(length)); @@ -1242,6 +1261,8 @@ int ObMigrationTabletParam::deserialize_v2_v3(const char *buf, const int64_t len LOG_WARN("failed to deserialize extra_medium_info", K(ret), K(len), K(new_pos)); } else if (PARAM_VERSION_V3 <= version_ && new_pos - pos < length && OB_FAIL(last_persisted_committed_tablet_status_.deserialize(buf, len, new_pos))) { LOG_WARN("failed to deserialize last_persisted_committed_tablet_status", K(ret), K(len), K(new_pos)); + } else if (PARAM_VERSION_V3 <= version_ && new_pos - pos < length && OB_FAIL(serialization::decode(buf, len, new_pos, ddl_table_type_))) { + LOG_WARN("failed to deserialize ddl table type", K(ret), K(len), K(new_pos)); } else if (OB_UNLIKELY(length != new_pos - pos)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet's length doesn't match standard length", K(ret), K(new_pos), K(pos), K(length), KPC(this)); @@ -1440,6 +1461,7 @@ int64_t ObMigrationTabletParam::get_serialize_size() const if (PARAM_VERSION_V3 <= version_) { size += extra_medium_info_.get_serialize_size(); size += last_persisted_committed_tablet_status_.get_serialize_size(); + size += serialization::encoded_length(ddl_table_type_); } return size; } @@ -1479,6 +1501,7 @@ void ObMigrationTabletParam::reset() mds_data_.reset(); transfer_info_.reset(); create_schema_version_ = 0; + ddl_table_type_ = ObITable::MAX_TABLE_TYPE; allocator_.reset(); } @@ -1517,6 +1540,7 @@ int ObMigrationTabletParam::assign(const ObMigrationTabletParam ¶m) ddl_execution_id_ = param.ddl_execution_id_; ddl_data_format_version_ = param.ddl_data_format_version_; ddl_commit_scn_ = param.ddl_commit_scn_; + ddl_table_type_ = param.ddl_table_type_; mds_checkpoint_scn_ = param.mds_checkpoint_scn_; transfer_info_ = param.transfer_info_; extra_medium_info_ = param.extra_medium_info_; diff --git a/src/storage/tablet/ob_tablet_meta.h b/src/storage/tablet/ob_tablet_meta.h index 8b40011e7..ec7a1103d 100644 --- a/src/storage/tablet/ob_tablet_meta.h +++ b/src/storage/tablet/ob_tablet_meta.h @@ -149,7 +149,8 @@ public: K_(extra_medium_info), K_(last_persisted_committed_tablet_status), K_(create_schema_version), - K_(space_usage)); + K_(space_usage), + K_(ddl_table_type)); public: int32_t version_; @@ -185,6 +186,13 @@ public: ObTabletCreateDeleteMdsUserData last_persisted_committed_tablet_status_; // quick access for tablet status in sstables ObTabletSpaceUsage space_usage_; // calculated by tablet persist, ObMigrationTabletParam doesn't need it int64_t create_schema_version_; // add after 4.2, record schema_version when first create tablet. NEED COMPAT + // add after 4.3.3, is used to decide storage type for replaying ddl clog and create ddl dump sstable in cs replica. + // when offline ddl is concurrent with adding C-Replica, it may write row store clog, but storage schema in C-Replica is columnar. + // so need persist a field in tablet when replaying start log to decide table_type when restart from a checkpoint, or migrating, etc. + // - DDL_MEM_SSTABLE: initial state, tablet not doing offline ddl. only take this type for inital, unrelated to memtable. + // - DDL_DUMP_SSTABLE/DDL_MERGE_CO_SSTABLE: tablet is doing offline ddl, indicate target storage type for ddl dump sstable. + // - MAJOR_SSTABLE/COLUMN_ORIENTED_SSTABLE: tablet finish offline ddl, set when ddl merge task create major sstable. + ObITable::TableType ddl_table_type_; //ATTENTION : Add a new variable need consider ObMigrationTabletParam // and tablet meta init interface for migration. // yuque : @@ -322,6 +330,7 @@ public: ObTabletFullMemoryMdsData mds_data_; ObTabletTransferInfo transfer_info_; int64_t create_schema_version_; + ObITable::TableType ddl_table_type_; // Add new serialization member before this line, below members won't serialize common::ObArenaAllocator allocator_; // for storage schema diff --git a/src/storage/tablet/ob_tablet_table_store.cpp b/src/storage/tablet/ob_tablet_table_store.cpp index 876ca661a..ef56e9395 100644 --- a/src/storage/tablet/ob_tablet_table_store.cpp +++ b/src/storage/tablet/ob_tablet_table_store.cpp @@ -1356,24 +1356,20 @@ int ObTabletTableStore::build_major_tables( && OB_FAIL(major_tables.push_back(new_table))) { LOG_WARN("failed to add table into tables handle", K(ret), K(param)); } else if (OB_FAIL(inner_build_major_tables_(allocator, old_store, major_tables, - param.multi_version_start_, param.allow_duplicate_sstable_, inc_base_snapshot_version))) { + param.multi_version_start_, param.allow_duplicate_sstable_, inc_base_snapshot_version, is_convert_co_major_merge(param.merge_type_)))) { LOG_WARN("failed to inner build major tables", K(ret), K(param), K(major_tables)); } } return ret; } -int ObTabletTableStore::inner_build_major_tables_( - common::ObArenaAllocator &allocator, +int ObTabletTableStore::check_and_build_new_major_tables( const ObTabletTableStore &old_store, const ObIArray &tables_array, - const int64_t multi_version_start, const bool allow_duplicate_sstable, - int64_t &inc_base_snapshot_version) + ObIArray &major_tables) const { int ret = OB_SUCCESS; - inc_base_snapshot_version = -1; - ObSEArray major_tables; bool need_add = true; if (!old_store.major_tables_.empty() && OB_FAIL(old_store.major_tables_.get_all_tables(major_tables))) { @@ -1417,6 +1413,32 @@ int ObTabletTableStore::inner_build_major_tables_( } } } + return ret; +} + +int ObTabletTableStore::inner_build_major_tables_( + common::ObArenaAllocator &allocator, + const ObTabletTableStore &old_store, + const ObIArray &tables_array, + const int64_t multi_version_start, + const bool allow_duplicate_sstable, + int64_t &inc_base_snapshot_version, + bool replace_old_row_store_major /*= false*/) +{ + int ret = OB_SUCCESS; + inc_base_snapshot_version = -1; + ObSEArray major_tables; + + if (OB_UNLIKELY(!old_store.major_tables_.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("major in old store is invalid", K(ret), K(old_store)); + } else if (replace_old_row_store_major) { + if (OB_FAIL(old_store.major_tables_.replace_twin_majors_and_build_new(tables_array, major_tables))) { + LOG_WARN("failed to replace twin row store majors", K(ret), K(old_store), K(tables_array)); + } + } else if (OB_FAIL(check_and_build_new_major_tables(old_store, tables_array, allow_duplicate_sstable, major_tables))) { + LOG_WARN("failed to check and add new major tables", K(ret), K(old_store), K(tables_array)); + } if (OB_FAIL(ret)) { } else if (OB_FAIL(ObTableStoreUtil::sort_major_tables(major_tables))) { diff --git a/src/storage/tablet/ob_tablet_table_store.h b/src/storage/tablet/ob_tablet_table_store.h index 63263d572..cebf72002 100644 --- a/src/storage/tablet/ob_tablet_table_store.h +++ b/src/storage/tablet/ob_tablet_table_store.h @@ -269,7 +269,13 @@ private: const ObIArray &tables_array, const int64_t multi_version_start, const bool allow_duplicate_sstable, - int64_t &inc_base_snapshot_version); + int64_t &inc_base_snapshot_version, + bool replace_old_row_store_major = false); + int check_and_build_new_major_tables( + const ObTabletTableStore &old_store, + const ObIArray &tables_array, + const bool allow_duplicate_sstable, + ObIArray &major_tables) const; int inner_replace_remote_major_sstable_( common::ObArenaAllocator &allocator, const ObTabletTableStore &old_store, diff --git a/src/storage/tx_storage/ob_ls_service.cpp b/src/storage/tx_storage/ob_ls_service.cpp index 5f7566718..6f21c664f 100644 --- a/src/storage/tx_storage/ob_ls_service.cpp +++ b/src/storage/tx_storage/ob_ls_service.cpp @@ -14,6 +14,7 @@ #include "lib/guard/ob_shared_guard.h" #include "logservice/ob_garbage_collector.h" +#include "logservice/ob_log_service.h" #include "observer/ob_service.h" #include "observer/ob_srv_network_frame.h" #include "share/rc/ob_tenant_module_init_ctx.h" @@ -421,6 +422,7 @@ int ObLSService::inner_create_ls_(const share::ObLSID &lsid, const ObMigrationStatus &migration_status, const ObLSRestoreStatus &restore_status, const SCN &create_scn, + const ObLSStoreFormat &store_format, ObLS *&ls) { int ret = OB_SUCCESS; @@ -438,6 +440,7 @@ int ObLSService::inner_create_ls_(const share::ObLSID &lsid, migration_status, restore_status, create_scn, + store_format, rs_reporter_))) { LOG_WARN("fail to init ls", K(ret), K(lsid)); } @@ -980,6 +983,7 @@ int ObLSService::replay_create_ls_(const ObLSMeta &ls_meta) migration_status, restore_status, ls_meta.get_clog_checkpoint_scn(), + ls_meta.get_store_format(), ls))) { LOG_WARN("fail to inner create ls", K(ret), K(ls_meta.ls_id_)); } else if (FALSE_IT(state = ObLSCreateState::CREATE_STATE_INNER_CREATED)) { @@ -1036,6 +1040,86 @@ int ObLSService::get_ls( return ret; } +int ObLSService::get_ls_replica( + const ObLSID &ls_id, + ObLSGetMod mod, + share::ObLSReplica &replica) +{ + int ret = OB_SUCCESS; + replica.reset(); + const uint64_t tenant_id = MTL_ID(); + ObLSHandle ls_handle; + ObLS *ls = NULL; + ObLogService *log_service = MTL(ObLogService*); + common::ObRole role = FOLLOWER; + ObMemberList ob_member_list; + ObLSReplica::MemberList member_list; + GlobalLearnerList learner_list; + int64_t proposal_id = 0; + int64_t paxos_replica_number = 0; + ObLSRestoreStatus restore_status; + ObReplicaStatus replica_status = REPLICA_STATUS_NORMAL; + ObReplicaType replica_type = REPLICA_TYPE_FULL; + ObMigrationStatus migration_status = OB_MIGRATION_STATUS_MAX; + uint64_t unit_id = common::OB_INVALID_ID; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_UNLIKELY(!ls_id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(ls_id)); + } else if (OB_ISNULL(log_service) || OB_ISNULL(GCTX.config_) || OB_ISNULL(GCTX.omt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("null ptr", KR(ret), KP(log_service), KP(GCTX.config_), KP(GCTX.omt_)); + } else if (OB_FAIL(get_ls(ls_id, ls_handle, mod))) { + LOG_WARN("get ls handle failed", KR(ret), K(ls_id), K(mod)); + } else if (OB_ISNULL(ls = ls_handle.get_ls())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls is null", KR(ret), K(ls_id), KP(ls)); + } else if (OB_FAIL(ls->get_paxos_member_list_and_learner_list(ob_member_list, paxos_replica_number, learner_list))) { + LOG_WARN("get member list and learner list from ObLS failed", KR(ret)); + } else if (OB_FAIL(ls->get_restore_status(restore_status))) { + LOG_WARN("get restore status failed", KR(ret)); + } else if (OB_FAIL(ls->get_migration_status(migration_status))) { + LOG_WARN("get migration status failed", KR(ret)); + } else if (OB_FAIL(ls->get_replica_status(replica_status))) { + LOG_WARN("get replica status failed", KR(ret)); + } else if (OB_FAIL(log_service->get_palf_role(ls_id, role, proposal_id))) { + LOG_WARN("failed to get role from palf", KR(ret), K(tenant_id), K(ls_id)); + } else if (OB_FAIL(get_replica_type_(GCTX.self_addr(), ob_member_list, learner_list, + ls->get_store_format(), replica_type))) { + LOG_WARN("fail to get replica_type by member and learner list", KR(ret)); + } else if (OB_FAIL(ObLSReplica::transform_ob_member_list(ob_member_list, member_list))) { + LOG_WARN("fail to transfrom ob_member_list into member_list", KR(ret), K(ob_member_list)); + } else if (OB_FAIL(GCTX.omt_->get_unit_id(tenant_id, unit_id))) { + LOG_WARN("get tenant unit id failed", KR(ret), K(tenant_id), K(ls_id)); + } else if (OB_FAIL(replica.init( + 0, /*create_time_us*/ + 0, /*modify_time_us*/ + tenant_id, /*tenant_id*/ + ls_id, /*ls_id*/ + GCTX.self_addr(), /*server*/ + GCTX.config_->mysql_port, /*sql_port*/ + role, /*role*/ + replica_type, /*replica_type*/ + proposal_id, /*proposal_id*/ + is_strong_leader(role) ? REPLICA_STATUS_NORMAL : replica_status,/*replica_status*/ + restore_status, /*restore_status*/ + 100, /*memstore_percent*/ + unit_id, /*unit_id*/ + GCTX.config_->zone.str(), /*zone*/ + paxos_replica_number, /*paxos_replica_number*/ + 0, /*data_size*/ + 0, /*required_size*/ + member_list, + learner_list, + OB_MIGRATION_STATUS_REBUILD == migration_status /*is_rebuild*/))) { + LOG_WARN("fail to init a ls replica", KR(ret), K(tenant_id), K(ls_id), K(role), + K(proposal_id), K(unit_id), K(paxos_replica_number), K(member_list), K(learner_list)); + } + return ret; +} + int ObLSService::remove_ls(const share::ObLSID &ls_id) { int ret = OB_SUCCESS; @@ -1219,6 +1303,9 @@ int ObLSService::create_ls_(const ObCreateLSCommonArg &arg, bool ls_exist = false; bool waiting_destroy = false; int64_t process_point = 0; + const ObLSStoreFormat ls_store_format = (REPLICA_TYPE_COLUMNSTORE == arg.replica_type_) ? + common::ObLSStoreType::OB_LS_STORE_COLUMN_ONLY + : common::ObLSStoreType::OB_LS_STORE_NORMAL; if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -1254,8 +1341,9 @@ int ObLSService::create_ls_(const ObCreateLSCommonArg &arg, arg.migration_status_, arg.restore_status_, arg.create_scn_, + ls_store_format, ls))) { - LOG_WARN("create ls failed", K(ret), K(arg.ls_id_)); + LOG_WARN("create ls failed", K(ret), K(arg.ls_id_), K(ls_store_format)); } else { state = ObLSCreateState::CREATE_STATE_INNER_CREATED; ObLSLockGuard lock_ls(ls); @@ -1648,6 +1736,36 @@ int ObLSService::dump_ls_info() return ret; } +// this function is expected to not fail +int ObLSService::get_replica_type_( + const common::ObAddr &addr, + const ObMemberList &ob_member_list, + const GlobalLearnerList &learner_list, + const common::ObLSStoreFormat &ls_store_format, + ObReplicaType &replica_type) +{ + int ret = OB_SUCCESS; + const bool is_columnstore = ls_store_format.is_columnstore(); + const bool in_member_list = ob_member_list.contains(addr); + const bool in_learner_list = learner_list.contains(addr); + if (is_columnstore) { + replica_type = REPLICA_TYPE_COLUMNSTORE; + if (in_member_list) { + LOG_WARN("columnstore replica member in member_list is unexpected", + K(addr), K(ob_member_list), K(learner_list)); + } + } else { + // if replica exists in learner_list, report it as R-replica. + // Otherwise, report as F-replica + if (in_learner_list) { + replica_type = REPLICA_TYPE_READONLY; + } else { + replica_type = REPLICA_TYPE_FULL; + } + } + return ret; +} + } // storage } // oceanbase diff --git a/src/storage/tx_storage/ob_ls_service.h b/src/storage/tx_storage/ob_ls_service.h index 9e4bda068..902ac7ba2 100644 --- a/src/storage/tx_storage/ob_ls_service.h +++ b/src/storage/tx_storage/ob_ls_service.h @@ -107,6 +107,10 @@ public: int get_ls(const share::ObLSID &ls_id, ObLSHandle &handle, ObLSGetMod mod); + int get_ls_replica( + const ObLSID &ls_id, + ObLSGetMod mod, + share::ObLSReplica &replica); // @param [in] func, iterate all ls diagnose info int iterate_diagnose(const ObFunction &func); @@ -190,6 +194,7 @@ private: const ObMigrationStatus &migration_status, const share::ObLSRestoreStatus &restore_status, const share::SCN &create_scn, + const ObLSStoreFormat &store_format, ObLS *&ls); int inner_del_ls_(ObLS *&ls); int add_ls_to_map_(ObLS *ls); @@ -221,6 +226,13 @@ private: int cal_min_phy_resource_needed_(const int64_t ls_cnt, ObMinPhyResourceResult &min_phy_res); int get_resource_constraint_value_(ObResoureConstraintValue &constraint_value); + // for get_ls_replica + int get_replica_type_( + const common::ObAddr &addr, + const ObMemberList &ob_member_list, + const GlobalLearnerList &learner_list, + const common::ObLSStoreFormat &ls_store_format, + ObReplicaType &replica_type); private: bool is_inited_; diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result index 4eb37c6a4..c6f848b4d 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result @@ -2360,7 +2360,7 @@ MODIFY_TIME timestamp(6) YES UNIT_COUNT bigint(20) NO NULL UNIT_CONFIG_ID bigint(20) NO NULL ZONE_LIST varchar(8192) NO NULL -REPLICA_TYPE varchar(18) NO +REPLICA_TYPE varchar(4) NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_OB_RESOURCE_POOLS limit 1); cnt 1 diff --git a/unittest/observer/table/test_create_executor.cpp b/unittest/observer/table/test_create_executor.cpp index 3ec322276..71185c785 100644 --- a/unittest/observer/table/test_create_executor.cpp +++ b/unittest/observer/table/test_create_executor.cpp @@ -55,7 +55,7 @@ void fill_table_schema(ObTableSchema &table) table.set_part_level(PARTITION_LEVEL_TWO); table.set_charset_type(CHARSET_UTF8MB4); table.set_collation_type(CS_TYPE_UTF8MB4_BIN); - table.set_table_type(USER_TABLE); + table.set_table_type(USER_VIEW); table.set_index_type(INDEX_TYPE_IS_NOT); table.set_index_status(INDEX_STATUS_AVAILABLE); table.set_data_table_id(0); @@ -182,6 +182,7 @@ void TestCreateExecutor::fake_ctx_init_common(ObTableCtx &fake_ctx, ObTableSchem g_sess_node_val.sess_info_.test_init(0, 0, 0, NULL); g_sess_node_val.sess_info_.load_all_sys_vars(schema_guard_); fake_ctx.init_physical_plan_ctx(0, 1); + fake_ctx.loc_meta_.route_policy_ = ObRoutePolicyType::READONLY_ZONE_FIRST; ASSERT_EQ(OB_SUCCESS, fake_ctx.construct_column_items()); } diff --git a/unittest/storage/migration/test_migration.h b/unittest/storage/migration/test_migration.h index 1f23f8d12..19a64615c 100644 --- a/unittest/storage/migration/test_migration.h +++ b/unittest/storage/migration/test_migration.h @@ -385,6 +385,7 @@ static int mock_valid_ls_meta(obrpc::ObFetchLSMetaInfoResp &res) res.ls_meta_package_.ls_meta_.clog_checkpoint_scn_.set_base(); res.ls_meta_package_.ls_meta_.migration_status_ = ObMigrationStatus::OB_MIGRATION_STATUS_NONE; res.ls_meta_package_.ls_meta_.restore_status_ = share::ObLSRestoreStatus::NONE; + res.ls_meta_package_.ls_meta_.store_format_ = common::ObLSStoreType::OB_LS_STORE_NORMAL; res.ls_meta_package_.dup_ls_meta_.ls_id_ = ls_id; const palf::LSN lsn(184467440737095516); res.ls_meta_package_.palf_meta_.prev_log_info_.lsn_ = lsn; diff --git a/unittest/storage/test_compaction_policy.cpp b/unittest/storage/test_compaction_policy.cpp index 0ccfcf188..e6ca5f6dc 100644 --- a/unittest/storage/test_compaction_policy.cpp +++ b/unittest/storage/test_compaction_policy.cpp @@ -423,6 +423,7 @@ int TestCompactionPolicy::mock_tablet( ObArenaAllocator arena_allocator; ObCreateTabletSchema create_tablet_schema; bool need_empty_major_table = false; + bool need_generate_cs_replica_cg_array = false; if (OB_ISNULL(t3m)) { ret = OB_ERR_UNEXPECTED; @@ -439,8 +440,9 @@ int TestCompactionPolicy::mock_tablet( } else if (OB_FAIL(create_tablet_schema.init(arena_allocator, table_schema, compat_mode, false/*skip_column_info*/, ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_V3))) { LOG_WARN("failed to init storage schema", KR(ret), K(table_schema)); + } else if (FALSE_IT(need_generate_cs_replica_cg_array = ls_handle.get_ls()->is_cs_replica() && create_tablet_schema.is_row_store() && create_tablet_schema.is_user_data_table())) { } else if (OB_FAIL(tablet->init_for_first_time_creation(allocator, ls_id, tablet_id, tablet_id, - SCN::min_scn(), snapshot_version, create_tablet_schema, need_empty_major_table, ls_handle.get_ls()->get_freezer()))) { + SCN::min_scn(), snapshot_version, create_tablet_schema, need_empty_major_table, need_generate_cs_replica_cg_array, ls_handle.get_ls()->get_freezer()))) { LOG_WARN("failed to init tablet", K(ret), K(ls_id), K(tablet_id), K(snapshot_version), K(table_schema), K(compat_mode)); } else { diff --git a/unittest/storage/test_tablet_helper.h b/unittest/storage/test_tablet_helper.h index 6dc67bd66..064625bcf 100644 --- a/unittest/storage/test_tablet_helper.h +++ b/unittest/storage/test_tablet_helper.h @@ -130,12 +130,14 @@ inline int TestTabletHelper::create_tablet( const ObTabletMapKey key(ls_id, tablet_id); const bool need_create_empty_major_sstable = !(create_tablet_schema.is_user_hidden_table() || (create_tablet_schema.is_index_table() && !create_tablet_schema.can_read_index())); + const bool need_generate_cs_replica_cg_array = + ls_handle.get_ls()->is_cs_replica() && create_tablet_schema.is_row_store() && create_tablet_schema.is_user_data_table(); if (OB_FAIL(t3m->create_msd_tablet(WashTabletPriority::WTP_HIGH, key, ls_handle, tablet_handle))) { STORAGE_LOG(WARN, "t3m acquire tablet failed", K(ret), K(ls_id), K(tablet_id)); } else if (OB_FAIL(tablet_handle.get_obj()->init_for_first_time_creation( *tablet_handle.get_allocator(), ls_id, tablet_id, tablet_id, share::SCN::base_scn(), - snapshot_version, create_tablet_schema, need_create_empty_major_sstable, freezer))){ + snapshot_version, create_tablet_schema, need_create_empty_major_sstable, need_generate_cs_replica_cg_array, freezer))){ STORAGE_LOG(WARN, "failed to init tablet", K(ret), K(ls_id), K(tablet_id)); } else if (ObTabletStatus::Status::MAX != tablet_status) { ObTabletCreateDeleteMdsUserData data; diff --git a/unittest/storage/test_tablet_pointer_map.cpp b/unittest/storage/test_tablet_pointer_map.cpp index 6159f1749..33c5e4547 100644 --- a/unittest/storage/test_tablet_pointer_map.cpp +++ b/unittest/storage/test_tablet_pointer_map.cpp @@ -109,6 +109,7 @@ void TestMetaPointerMap::FakeLs(ObLS &ls) ls.ls_meta_.migration_status_ = ObMigrationStatus::OB_MIGRATION_STATUS_NONE; ls.ls_meta_.restore_status_ = ObLSRestoreStatus::NONE; ls.ls_meta_.rebuild_seq_ = 0; + ls.ls_meta_.store_format_ = common::ObLSStoreType::OB_LS_STORE_NORMAL; } class CalculateSize final From a96c34e695dbf3f479bb9891435a8aa5d0637b68 Mon Sep 17 00:00:00 2001 From: dongb0 <708848999@qq.com> Date: Thu, 22 Aug 2024 13:46:44 +0000 Subject: [PATCH 176/249] fix tmp file fail to insert_meta_tree when memory is not enough --- src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp b/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp index 29f5f628b..fa83f64cd 100644 --- a/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp +++ b/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp @@ -2516,10 +2516,10 @@ int ObSharedNothingTmpFile::insert_meta_tree_item(const ObTmpFileFlushInfo &info data_item.virtual_page_id_ = info.flush_virtual_page_id_; ObArray data_items; - if (OB_FAIL(meta_tree_.prepare_for_insert_items())) { - LOG_WARN("fail to prepare for insert items", KR(ret), K(info), K(block_index), KPC(this)); - } else if (OB_FAIL(data_items.push_back(data_item))) { + if (OB_FAIL(data_items.push_back(data_item))) { LOG_WARN("fail to push back data item", KR(ret), K(info), K(block_index), KPC(this)); + } else if (OB_FAIL(meta_tree_.prepare_for_insert_items())) { + LOG_WARN("fail to prepare for insert items", KR(ret), K(info), K(block_index), KPC(this)); } else if (OB_FAIL(meta_tree_.insert_items(data_items))) { LOG_WARN("fail to insert data items", KR(ret), K(info), K(block_index), KPC(this)); } From 962667c3bdb003ea62b764af3f2c2d59712a2668 Mon Sep 17 00:00:00 2001 From: haohao022 Date: Thu, 22 Aug 2024 13:58:31 +0000 Subject: [PATCH 177/249] [CP] [to #2024081300104110394] fix: skipping spi_result deconstruction in the exception path leads to memory leak --- src/sql/ob_spi.cpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/sql/ob_spi.cpp b/src/sql/ob_spi.cpp index 90165c020..dbdbe3f87 100644 --- a/src/sql/ob_spi.cpp +++ b/src/sql/ob_spi.cpp @@ -3467,13 +3467,12 @@ int ObSPIService::streaming_cursor_open(ObPLExecCtx *ctx, } while (RETRY_TYPE_NONE != retry_ctrl.get_retry_type()); spi_result->end_cursor_stmt(ctx, ret); cursor.set_last_execute_time(ObTimeUtility::current_time()); - // only care about end_cursor_stmt failed. - // if cursor already open, spi_result will released by cursor close. - // there is not a situation like '!cursor.is_open && spi_result not close'. - // so do not need call spi_result.close_result_set. - if (OB_FAIL(ret) && !cursor.isopen()) { - spi_result->~ObSPIResultSet(); - } + } + // if cursor already open, spi_result will released by cursor close. + // there is not a situation like '!cursor.is_open && spi_result not close'. + // so do not need call spi_result.close_result_set. + if (OB_FAIL(ret) && OB_NOT_NULL(spi_result) && !cursor.isopen()) { + spi_result->~ObSPIResultSet(); } return ret; } From 2ddad4afc0af7da4a8c2455b54c078bd47e3e1dc Mon Sep 17 00:00:00 2001 From: linqiucen Date: Thu, 22 Aug 2024 16:18:56 +0000 Subject: [PATCH 178/249] =?UTF-8?q?[FEAT=20MERGE]=20=E5=A4=87=E5=BA=93?= =?UTF-8?q?=E4=BC=98=E5=8C=96=20+=20is=5Fstandby=5Fcluster=E8=B0=83?= =?UTF-8?q?=E7=94=A8=E7=82=B9=E6=9B=BF=E6=8D=A2=20+=20service=5Fname?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mysqlclient/ob_mysql_connection_pool.cpp | 3 +- deps/oblib/src/lib/ob_define.h | 6 + deps/oblib/src/rpc/obmysql/obsm_struct.h | 2 + deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h | 2 +- .../ob_log_restore_driver_base.cpp | 29 +- .../ob_log_restore_driver_base.h | 2 + .../restoreservice/ob_log_restore_handler.cpp | 28 +- src/observer/dbms_job/ob_dbms_job_utils.cpp | 9 +- src/observer/mysql/obmp_connect.cpp | 83 +- src/observer/mysql/obmp_connect.h | 3 + src/observer/mysql/obsm_handler.cpp | 1 + src/observer/ob_rpc_processor_simple.cpp | 13 +- src/observer/ob_rpc_processor_simple.h | 1 + src/observer/ob_server.cpp | 10 +- src/observer/ob_server_struct.cpp | 44 - src/observer/ob_server_struct.h | 20 - src/observer/ob_service.cpp | 83 +- src/observer/ob_service.h | 2 + src/observer/ob_srv_xlator.cpp | 32 + src/observer/ob_srv_xlator.h | 1 + src/observer/ob_srv_xlator_partition.cpp | 1 + src/observer/omt/ob_multi_tenant.cpp | 2 +- src/observer/omt/ob_tenant_node_balancer.cpp | 7 - .../ob_all_virtual_ls_log_restore_status.cpp | 2 +- .../virtual_table/ob_show_processlist.cpp | 7 +- src/pl/sys_package/ob_dbms_stats.cpp | 19 +- src/rootserver/CMakeLists.txt | 10 +- .../backup/ob_backup_data_set_task_mgr.cpp | 4 +- .../ddl_task/ob_build_mview_task.cpp | 7 +- .../ddl_task/ob_index_build_task.cpp | 7 +- .../freeze/ob_major_merge_scheduler.cpp | 2 +- src/rootserver/ob_ddl_service.cpp | 111 +-- src/rootserver/ob_ddl_service.h | 3 - src/rootserver/ob_empty_server_checker.cpp | 22 +- src/rootserver/ob_ls_recovery_reportor.cpp | 2 +- src/rootserver/ob_ls_service_helper.cpp | 4 +- src/rootserver/ob_root_inspection.cpp | 23 +- src/rootserver/ob_root_service.cpp | 33 +- src/rootserver/ob_rs_async_rpc_proxy.h | 1 + src/rootserver/ob_rs_rpc_processor.h | 6 +- src/rootserver/ob_schema_history_recycler.cpp | 36 +- src/rootserver/ob_service_name_command.cpp | 460 ++++++++++ src/rootserver/ob_service_name_command.h | 82 ++ src/rootserver/ob_system_admin_util.cpp | 32 +- src/rootserver/ob_tenant_event_def.h | 252 ++++++ ...ob_tenant_event_history_table_operator.cpp | 47 ++ .../ob_tenant_event_history_table_operator.h | 36 + src/rootserver/ob_tenant_info_loader.cpp | 209 ++++- src/rootserver/ob_tenant_info_loader.h | 62 +- .../ob_transfer_partition_command.cpp | 4 +- src/rootserver/ob_upgrade_executor.cpp | 6 +- src/rootserver/restore/ob_clone_scheduler.cpp | 4 +- .../ob_recover_table_job_scheduler.cpp | 4 +- .../restore/ob_restore_common_util.cpp | 21 +- .../restore/ob_restore_scheduler.cpp | 6 +- .../{ => standby}/ob_recovery_ls_service.cpp | 7 +- .../{ => standby}/ob_recovery_ls_service.h | 2 +- .../standby/ob_standby_service.cpp} | 370 +++++--- .../standby/ob_standby_service.h} | 84 +- .../ob_tenant_role_transition_service.cpp | 787 +++++++++++++----- .../ob_tenant_role_transition_service.h | 115 ++- src/share/CMakeLists.txt | 2 +- .../ob_inner_table_schema.12451_12500.cpp | 136 +++ .../ob_inner_table_schema.15401_15450.cpp | 134 +++ .../ob_inner_table_schema.21201_21250.cpp | 4 +- .../ob_inner_table_schema.21501_21550.cpp | 100 +++ .../ob_inner_table_schema.25301_25350.cpp | 50 ++ .../ob_inner_table_schema.28101_28150.cpp | 4 +- .../ob_inner_table_schema.501_550.cpp | 152 ++++ .../ob_inner_table_schema.50501_50550.cpp | 135 +++ .../ob_inner_table_schema.60501_60550.cpp | 90 ++ src/share/inner_table/ob_inner_table_schema.h | 49 +- .../inner_table/ob_inner_table_schema.lob.cpp | 2 +- .../ob_inner_table_schema_constants.h | 16 + .../inner_table/ob_inner_table_schema_def.py | 108 ++- .../ob_inner_table_schema_misc.ipp | 24 +- src/share/inner_table/table_id_to_name | 9 + src/share/ob_autoincrement_service.cpp | 9 - src/share/ob_errno.cpp | 12 +- src/share/ob_errno.def | 2 +- src/share/ob_errno.h | 6 +- src/share/ob_event_history_table_operator.cpp | 51 +- src/share/ob_event_history_table_operator.h | 91 +- src/share/ob_log_restore_proxy.cpp | 41 +- src/share/ob_log_restore_proxy.h | 5 +- src/share/ob_max_id_fetcher.cpp | 9 +- src/share/ob_rpc_struct.cpp | 100 +++ src/share/ob_rpc_struct.h | 71 +- src/share/ob_service_epoch_proxy.cpp | 9 +- src/share/ob_service_epoch_proxy.h | 4 +- src/share/ob_service_name_proxy.cpp | 630 ++++++++++++++ src/share/ob_service_name_proxy.h | 222 +++++ src/share/ob_share_util.cpp | 102 +++ src/share/ob_share_util.h | 8 + src/share/ob_srv_rpc_proxy.h | 1 + src/share/ob_tenant_info_proxy.cpp | 55 +- src/share/ob_tenant_info_proxy.h | 18 +- src/share/ob_tenant_role.h | 3 +- src/share/ob_tenant_switchover_status.h | 2 +- src/share/ob_upgrade_utils.cpp | 29 +- src/share/ob_upgrade_utils.h | 1 + src/share/schema/ob_dependency_info.cpp | 12 +- src/share/schema/ob_dependency_info.h | 2 - .../ob_multi_version_schema_service.cpp | 20 +- src/share/schema/ob_schema_cache.cpp | 1 - src/share/schema/ob_schema_getter_guard.cpp | 24 +- src/share/schema/ob_schema_getter_guard.h | 6 +- src/share/schema/ob_schema_struct.cpp | 127 +-- src/share/schema/ob_server_schema_service.cpp | 10 +- src/share/schema/ob_table_schema.cpp | 85 +- src/share/sequence/ob_sequence_dml_proxy.cpp | 3 - src/share/stat/ob_dbms_stats_utils.cpp | 14 +- .../stat/ob_opt_stat_monitor_manager.cpp | 127 ++- src/share/table/ob_ttl_util.cpp | 7 +- .../engine/cmd/ob_alter_system_executor.cpp | 65 +- src/sql/engine/cmd/ob_alter_system_executor.h | 3 +- src/sql/engine/cmd/ob_analyze_executor.cpp | 25 +- src/sql/engine/cmd/ob_ddl_executor_util.cpp | 8 +- src/sql/engine/cmd/ob_index_executor.cpp | 23 +- src/sql/engine/cmd/ob_index_executor.h | 1 - src/sql/engine/cmd/ob_tenant_executor.cpp | 6 +- src/sql/executor/ob_cmd_executor.cpp | 4 + src/sql/parser/sql_parser_mysql_mode.y | 40 +- .../privilege_check/ob_privilege_check.cpp | 1 + .../resolver/cmd/ob_alter_system_resolver.cpp | 73 +- .../resolver/cmd/ob_alter_system_resolver.h | 3 +- src/sql/resolver/cmd/ob_alter_system_stmt.h | 11 + src/sql/resolver/ob_resolver.cpp | 4 + src/sql/session/ob_sql_session_info.cpp | 81 +- src/sql/session/ob_sql_session_info.h | 13 +- .../ob_multi_version_garbage_collector.cpp | 17 +- src/storage/ls/ob_ls.cpp | 2 +- src/storage/memtable/ob_memtable.cpp | 42 - src/storage/memtable/ob_memtable.h | 3 - .../tablelock/ob_table_lock_service.cpp | 16 +- .../r/mysql/information_schema.result | 6 + .../r/mysql/desc_sys_views_in_mysql.result | 16 +- .../r/mysql/desc_sys_views_in_sys.result | 27 +- .../mysql/desc_virtual_table_in_mysql.result | 11 + .../r/mysql/desc_virtual_table_in_sys.result | 11 + .../r/mysql/inner_table_overall.result | 4 + unittest/share/schema/mock_schema_service.h | 3 +- 142 files changed, 5495 insertions(+), 1231 deletions(-) create mode 100644 src/rootserver/ob_service_name_command.cpp create mode 100644 src/rootserver/ob_service_name_command.h create mode 100644 src/rootserver/ob_tenant_event_def.h create mode 100644 src/rootserver/ob_tenant_event_history_table_operator.cpp create mode 100644 src/rootserver/ob_tenant_event_history_table_operator.h rename src/rootserver/{ => standby}/ob_recovery_ls_service.cpp (99%) rename src/rootserver/{ => standby}/ob_recovery_ls_service.h (99%) rename src/{share/ob_primary_standby_service.cpp => rootserver/standby/ob_standby_service.cpp} (76%) rename src/{share/ob_primary_standby_service.h => rootserver/standby/ob_standby_service.h} (69%) rename src/rootserver/{ => standby}/ob_tenant_role_transition_service.cpp (73%) rename src/rootserver/{ => standby}/ob_tenant_role_transition_service.h (71%) create mode 100644 src/share/ob_service_name_proxy.cpp create mode 100644 src/share/ob_service_name_proxy.h diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_connection_pool.cpp b/deps/oblib/src/lib/mysqlclient/ob_mysql_connection_pool.cpp index 674402a49..e0a9d5a4a 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_connection_pool.cpp +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_connection_pool.cpp @@ -464,8 +464,7 @@ int ObMySQLConnectionPool::acquire(const uint64_t tenant_id, ObMySQLConnection * } if (OB_ISNULL(connection)) { - //overwrite ret - ret = OB_ERR_UNEXPECTED; + ret = OB_SUCC(ret) ? OB_ERR_UNEXPECTED: ret; LOG_WARN("failed to acquire connection", K(this), K(tenant_id), K(server_count), K(busy_conn_count_), K(ret)); obsys::ObRLockGuard lock(get_lock_); diff --git a/deps/oblib/src/lib/ob_define.h b/deps/oblib/src/lib/ob_define.h index 457f8d571..277c7f22b 100644 --- a/deps/oblib/src/lib/ob_define.h +++ b/deps/oblib/src/lib/ob_define.h @@ -409,6 +409,7 @@ const int64_t OB_MAX_DIRECTORY_NAME_LENGTH = 128; // Compatible with Oracle const int64_t OB_MAX_DIRECTORY_PATH_LENGTH = 4000; // Compatible with Oracle const uint64_t OB_MAX_INTERVAL_PARTITIONS = 1048575; // interval parted table max partitions const int64_t OB_MAX_BALANCE_GROUP_NAME_LENGTH = 512; +const int64_t OB_SERVICE_NAME_LENGTH = 64; //plan cache const int64_t OB_PC_NOT_PARAM_COUNT = 8; @@ -1525,6 +1526,11 @@ const char *const OB_MYSQL_PROXY_VEERSION = "__proxy_version"; const char *const OB_MYSQL_CLIENT_VERSION = "__ob_client_version"; const char *const OB_MYSQL_CLIENT_NAME = "__ob_client_name"; +const char *const OB_MYSQL_FAILOVER_MODE = "__proxy_failover_mode"; +const char *const OB_MYSQL_FAILOVER_MODE_OFF = "off"; +const char *const OB_MYSQL_FAILOVER_MODE_ON = "on"; +const char *const OB_MYSQL_SERVICE_NAME = "__proxy_service_name"; + const char *const OB_MYSQL_JDBC_CLIENT_NAME = "OceanBase Connector/J"; const char *const OB_MYSQL_OCI_CLIENT_NAME = "OceanBase Connector/C"; // for java client diff --git a/deps/oblib/src/rpc/obmysql/obsm_struct.h b/deps/oblib/src/rpc/obmysql/obsm_struct.h index 020b69447..a895ae748 100644 --- a/deps/oblib/src/rpc/obmysql/obsm_struct.h +++ b/deps/oblib/src/rpc/obmysql/obsm_struct.h @@ -80,6 +80,7 @@ public: client_sessid_ = INVALID_SESSID; client_addr_port_ = 0; client_create_time_ = 0; + has_service_name_ = false; } obmysql::ObCompressType get_compress_type() { @@ -215,6 +216,7 @@ public: uint32_t client_sessid_; int32_t client_addr_port_; int64_t client_create_time_; + bool has_service_name_; }; } // end of namespace observer } // end of namespace oceanbase diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h index e75227d43..e181d4891 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h @@ -285,7 +285,7 @@ PCODE_DEF(OB_REMOVE_CLUSTER_INFO_FROM_ARB_SERVER, 0x2A8) #endif PCODE_DEF(OB_DROP_LOB, 0x2A9) PCODE_DEF(OB_EXCHANGE_PARTITION, 0x2AA) -// PCODE_DEF(OB_REFRESH_SERVICE_NAME, 0x2AB) +PCODE_DEF(OB_REFRESH_SERVICE_NAME, 0x2AB) PCODE_DEF(OB_CREATE_OUTLINE, 0x350) PCODE_DEF(OB_DROP_OUTLINE, 0x351) diff --git a/src/logservice/restoreservice/ob_log_restore_driver_base.cpp b/src/logservice/restoreservice/ob_log_restore_driver_base.cpp index 2d831a369..7282c9cee 100644 --- a/src/logservice/restoreservice/ob_log_restore_driver_base.cpp +++ b/src/logservice/restoreservice/ob_log_restore_driver_base.cpp @@ -16,6 +16,7 @@ #include "lib/ob_define.h" #include "storage/tx_storage/ob_ls_map.h" // ObLSIterator #include "storage/tx_storage/ob_ls_service.h" // ObLSService +#include "rootserver/ob_tenant_info_loader.h" // ObTenantInfoLoader #include "logservice/ob_log_service.h" namespace oceanbase @@ -126,8 +127,13 @@ int ObLogRestoreDriverBase::get_upper_resotore_scn(share::SCN &scn) { int ret = OB_SUCCESS; SCN replayable_point; - if (OB_FAIL(log_service_->get_replayable_point(replayable_point))) { - ARCHIVE_LOG(WARN, "get replayable point failed", K(ret)); + bool restore_log_limit = true; + if (OB_FAIL(check_fetch_log_unlimited_(restore_log_limit))) { + LOG_WARN("check_fetch_log_unlimited_ failed"); + } else if (! restore_log_limit) { + scn = share::SCN::max_scn(); + } else if (OB_FAIL(log_service_->get_replayable_point(replayable_point))) { + LOG_WARN("get replayable point failed", K(ret)); } else { share::SCN advance_scn = share::SCN::plus(replayable_point, FETCH_LOG_AHEAD_THRESHOLD_NS); scn = global_recovery_scn_ <= advance_scn ? global_recovery_scn_ : advance_scn; @@ -135,5 +141,24 @@ int ObLogRestoreDriverBase::get_upper_resotore_scn(share::SCN &scn) return ret; } +int ObLogRestoreDriverBase::check_fetch_log_unlimited_(bool &limit) +{ + int ret = OB_SUCCESS; + share::ObAllTenantInfo tenant_info; + rootserver::ObTenantInfoLoader *tenant_info_loader = MTL(rootserver::ObTenantInfoLoader*); + limit = true; + bool is_prepare = false; + + if (OB_ISNULL(tenant_info_loader)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("tenant_info_loader is NULL", K(tenant_info_loader)); + } else if(OB_FAIL(tenant_info_loader->check_is_prepare_flashback_for_switch_to_primary_status(is_prepare))) { + LOG_WARN("fail to check tenant status", KR(ret), KPC(tenant_info_loader)); + } else if (is_prepare) { + limit = false; + } + return ret; +} + } // namespace logservice } // namespace oceanbase diff --git a/src/logservice/restoreservice/ob_log_restore_driver_base.h b/src/logservice/restoreservice/ob_log_restore_driver_base.h index 941e05521..ff99ff4a2 100644 --- a/src/logservice/restoreservice/ob_log_restore_driver_base.h +++ b/src/logservice/restoreservice/ob_log_restore_driver_base.h @@ -42,6 +42,8 @@ protected: virtual int do_fetch_log_(ObLS &ls) = 0; int check_replica_status_(storage::ObLS &ls, bool &can_fetch_log); int get_upper_resotore_scn(share::SCN &scn); +private: + int check_fetch_log_unlimited_(bool &limit); protected: bool inited_; uint64_t tenant_id_; diff --git a/src/logservice/restoreservice/ob_log_restore_handler.cpp b/src/logservice/restoreservice/ob_log_restore_handler.cpp index abd0dc633..e8fba4f45 100644 --- a/src/logservice/restoreservice/ob_log_restore_handler.cpp +++ b/src/logservice/restoreservice/ob_log_restore_handler.cpp @@ -856,7 +856,7 @@ bool ObLogRestoreHandler::restore_to_end() const RLockGuard guard(lock_); return restore_to_end_unlock_(); } - +ERRSIM_POINT_DEF(ERRSIM_LS_STATE_NOT_MATCH); int ObLogRestoreHandler::check_restore_to_newest_from_service_( const share::ObRestoreSourceServiceAttr &service_attr, const share::SCN &end_scn, @@ -864,27 +864,10 @@ int ObLogRestoreHandler::check_restore_to_newest_from_service_( { int ret = OB_SUCCESS; bool offline_log_exist = false; - share::ObTenantRole tenant_role; - share::schema::ObTenantStatus tenant_status; palf::AccessMode access_mode; - const char *db_name = service_attr.user_.mode_ == common::ObCompatibilityMode::MYSQL_MODE ? OB_SYS_DATABASE_NAME : OB_ORA_SYS_SCHEMA_NAME; - ObSqlString user; - char passwd[OB_MAX_PASSWORD_LENGTH + 1] = {0}; SMART_VAR(share::ObLogRestoreProxyUtil, proxy_util) { - if (!service_attr.is_valid()) { - ret = OB_ERR_UNEXPECTED; - } else if (OB_FAIL(service_attr.get_password(passwd, sizeof(passwd)))) { - CLOG_LOG(WARN, "get_password failed", K(id_), K(service_attr)); - } else if (OB_FAIL(service_attr.get_user_str_(user))) { - CLOG_LOG(WARN, "get user str failed", K(service_attr)); - } else if (OB_FAIL(proxy_util.init(MTL_ID(), service_attr.addr_, - user.ptr(), passwd, db_name))) { - CLOG_LOG(WARN, "proxy_util init failed", K(id_)); - } else if (OB_FAIL(proxy_util.get_tenant_info(tenant_role, tenant_status))) { - CLOG_LOG(WARN, "get tenant info failed", K(id_), K(service_attr)); - } else if (! tenant_role.is_standby() || share::schema::ObTenantStatus::TENANT_STATUS_NORMAL != tenant_status) { - ret = OB_SOURCE_TENANT_STATE_NOT_MATCH; - CLOG_LOG(WARN, "tenant role or status not match", K(id_), K(tenant_role), K(tenant_status), K(service_attr)); + if (OB_FAIL(proxy_util.init_with_service_attr(MTL_ID(), &service_attr))) { + CLOG_LOG(WARN, "proxy_util init failed", K(id_), K(service_attr)); } else if (OB_FAIL(proxy_util.get_max_log_info(share::ObLSID(id_), access_mode, archive_scn))) { // OB_ENTRY_NOT_EXIST, ls not exist in gv$ob_log_stat, a) ls has no leader; b) access virtual table failed; c) ls gc if (OB_ENTRY_NOT_EXIST == ret) { @@ -921,7 +904,10 @@ int ObLogRestoreHandler::check_restore_to_newest_from_service_( CLOG_LOG(INFO, "check_restore_to_newest succ", K(id_), K(archive_scn), K(end_scn)); } } - + if (OB_UNLIKELY(ERRSIM_LS_STATE_NOT_MATCH)) { + ret = OB_SUCC(ret) ? OB_SOURCE_LS_STATE_NOT_MATCH : ret; + CLOG_LOG(WARN, "ERRSIM_LS_STATE_NOT_MATCH is on", KR(ret)); + } // if connect to source tenant denied, rewrite ret_code if (-ER_ACCESS_DENIED_ERROR == ret) { ret = OB_PASSWORD_WRONG; diff --git a/src/observer/dbms_job/ob_dbms_job_utils.cpp b/src/observer/dbms_job/ob_dbms_job_utils.cpp index 62b3bb82f..b6c2beed3 100644 --- a/src/observer/dbms_job/ob_dbms_job_utils.cpp +++ b/src/observer/dbms_job/ob_dbms_job_utils.cpp @@ -218,10 +218,11 @@ int ObDBMSJobUtils::check_job_can_running(int64_t tenant_id, bool &can_running) OZ (GCTX.schema_service_->get_tenant_schema_guard(tenant_id, guard)); OZ (guard.check_tenant_is_restore(tenant_id, is_restore)); - // job can not run in standy cluster and restore. - if (OB_SUCC(ret) && job_queue_processor > 0 - && !GCTX.is_standby_cluster() - && !is_restore) { + // job can not run in standy and restore tenant. + bool is_primary = false; + if (FAILEDx(ObShareUtil::table_check_if_tenant_role_is_primary(tenant_id, is_primary))) { + LOG_WARN("fail to execute table_check_if_tenant_role_is_primary", KR(ret), K(tenant_id)); + } else if (is_primary && job_queue_processor > 0) { SMART_VAR(ObMySQLProxy::MySQLResult, result) { if (OB_FAIL(sql_proxy_->read(result, tenant_id, sql.ptr()))) { LOG_WARN("execute query failed", K(ret), K(sql), K(tenant_id)); diff --git a/src/observer/mysql/obmp_connect.cpp b/src/observer/mysql/obmp_connect.cpp index 178204602..550775055 100644 --- a/src/observer/mysql/obmp_connect.cpp +++ b/src/observer/mysql/obmp_connect.cpp @@ -312,6 +312,8 @@ int ObMPConnect::process() uint64_t tenant_id = OB_INVALID_ID; ObSQLSessionInfo *session = NULL; bool autocommit = false; + ObString service_name; + bool failover_mode = false; MAKE_TENANT_SWITCH_SCOPE_GUARD(guard); THIS_WORKER.set_timeout_ts(INT64_MAX); // avoid see a former timeout value if (THE_TRACE != nullptr) { @@ -342,8 +344,14 @@ int ObMPConnect::process() } else if (SS_STOPPING == GCTX.status_) { ret = OB_SERVER_IS_STOPPING; LOG_WARN("server is stopping", K(ret)); + } else if (OB_FAIL(extract_service_name(*conn, service_name, failover_mode))) { + LOG_WARN("fail to extraxt service name", KR(ret)); } else if (OB_FAIL(check_update_tenant_id(*conn, tenant_id))) { - LOG_WARN("fail to check update tenant id", K(ret)); + LOG_WARN("fail to check update tenant id", KR(ret), K(tenant_name_)); + if (OB_ERR_INVALID_TENANT_NAME == ret && !service_name.empty()) { + ret = OB_SERVICE_NAME_NOT_FOUND; + LOG_WARN("login via service_name but tenant not exist", KR(ret), K(service_name), K(tenant_name_)); + } } else if (OB_FAIL(guard.switch_to(tenant_id))) { LOG_WARN("switch to tenant fail", K(ret), K(tenant_id)); } else if (OB_FAIL(check_client_property(*conn))) { @@ -355,6 +363,8 @@ int ObMPConnect::process() } else if (OB_ISNULL(session)) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("null session", K(ret), K(session)); + } else if (OB_FAIL(set_service_name(tenant_id, *session, service_name, failover_mode))) { + LOG_WARN("fail to set service_name", KR(ret), KPC(session), K(service_name), K(failover_mode)); } else if (OB_FAIL(verify_identify(*conn, *session, tenant_id))) { LOG_WARN("fail to verify_identify", K(ret)); } else if (OB_FAIL(process_kill_client_session(*session, true))) { @@ -2341,3 +2351,74 @@ int ObMPConnect::set_client_version(ObSMConnection &conn) } return ret; } +ERRSIM_POINT_DEF(ERRSIM_MOCK_SERVICE_NAME); +int ObMPConnect::extract_service_name(ObSMConnection &conn, ObString &service_name, bool &failover_mode) +{ + int ret = OB_SUCCESS; + ObString failover_mode_key(OB_MYSQL_FAILOVER_MODE); + ObString failover_mode_off(OB_MYSQL_FAILOVER_MODE_OFF); + ObString failover_mode_on(OB_MYSQL_FAILOVER_MODE_ON); + ObString service_name_key(OB_MYSQL_SERVICE_NAME); + bool is_found_failover_mode = false; + bool is_found_service_name = false; + conn.has_service_name_ = false; + // extract failover_mode and service_name + for (int64_t i = 0; OB_SUCC(ret) && !is_found_failover_mode && i < hsr_.get_connect_attrs().count(); ++i) { + const ObStringKV &kv = hsr_.get_connect_attrs().at(i); + if (failover_mode_key == kv.key_) { + if (failover_mode_off == kv.value_) { + failover_mode = false; + } else if (failover_mode_on == kv.value_) { + failover_mode = true; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failover_mode should be on or off", KR(ret), K(kv)); + } + is_found_failover_mode = true; + } + } + for (int64_t i = 0; OB_SUCC(ret) && !is_found_service_name && i < hsr_.get_connect_attrs().count(); ++i) { + const ObStringKV &kv = hsr_.get_connect_attrs().at(i); + if (service_name_key == kv.key_) { + if (kv.value_.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("service_name should not be empty", KR(ret), K(kv)); + } else { + conn.has_service_name_ = true; + (void) service_name.assign_ptr(kv.value_.ptr(), kv.value_.length()); + } + is_found_service_name = true; + } + } + if (OB_SUCC(ret) && is_found_failover_mode != is_found_service_name) { + // The 'failover_mode' and 'service_name' must both be specified at the same time. + // The 'failover_mode' only matters if 'service_name' is not empty. + // If 'failover_mode' is 'on', it allows connection only to the main tenant. + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failover_mode or service_name is missing", KR(ret), K(is_found_failover_mode), K(is_found_service_name)); + } + if (OB_SUCC(ret) && ERRSIM_MOCK_SERVICE_NAME && !tenant_name_.empty() && 0 != tenant_name_.compare(OB_SYS_TENANT_NAME)) { + service_name = ObString::make_string("test_service"); + conn.has_service_name_ = true; + failover_mode = true; + LOG_INFO("ERRSIM_MOCK_SERVICE_NAME opened", KR(ret), K(service_name), K(tenant_name_)); + } + return ret; +} +int ObMPConnect::set_service_name(const uint64_t tenant_id, ObSQLSessionInfo &session, + const ObString &service_name, const bool failover_mode) +{ + int ret = OB_SUCCESS; + (void) session.set_failover_mode(failover_mode); + if (OB_FAIL(ret) || service_name.empty()) { + // If the connection is not established via 'service_name', the 'connection_attr' + // will not contain 'failover_mode' and 'service_name'. Consequently, 'service_name' + // in 'session_info' will be empty, indicating that any 'service_name' related logic + // will not be triggered. + } else if (OB_FAIL(session.set_service_name(service_name))) { + LOG_WARN("fail to set service_name", KR(ret), K(service_name), K(tenant_id)); + } else if (OB_FAIL(session.check_service_name_and_failover_mode(tenant_id))) { + LOG_WARN("fail to execute check_service_name_and_failover_mode", KR(ret), K(service_name), K(tenant_id)); + } + return ret; +} \ No newline at end of file diff --git a/src/observer/mysql/obmp_connect.h b/src/observer/mysql/obmp_connect.h index 07dcbb58f..e68739cef 100644 --- a/src/observer/mysql/obmp_connect.h +++ b/src/observer/mysql/obmp_connect.h @@ -126,6 +126,9 @@ private: int set_proxy_version(ObSMConnection &conn); int set_client_version(ObSMConnection &conn); + int extract_service_name(ObSMConnection &conn, ObString &service_name, bool &failover_mode); + int set_service_name(const uint64_t tenant_id, sql::ObSQLSessionInfo &session, + const ObString &service_name, const bool failover_mode); int get_proxy_user_name(ObString &real_user); private: DISALLOW_COPY_AND_ASSIGN(ObMPConnect); diff --git a/src/observer/mysql/obsm_handler.cpp b/src/observer/mysql/obsm_handler.cpp index 8546d8241..131870175 100644 --- a/src/observer/mysql/obsm_handler.cpp +++ b/src/observer/mysql/obsm_handler.cpp @@ -164,6 +164,7 @@ int ObSMHandler::on_disconnect(easy_connection_t *c) } else { sess_info->set_session_state(sql::SESSION_KILLED); sess_info->set_shadow(true); + conn->has_service_name_ = false; } } LOG_INFO("kill and revert session", K(conn->sessid_), diff --git a/src/observer/ob_rpc_processor_simple.cpp b/src/observer/ob_rpc_processor_simple.cpp index 61e9e97c6..5e2133803 100644 --- a/src/observer/ob_rpc_processor_simple.cpp +++ b/src/observer/ob_rpc_processor_simple.cpp @@ -85,7 +85,7 @@ #include "storage/tenant_snapshot/ob_tenant_snapshot_service.h" #include "storage/high_availability/ob_storage_ha_utils.h" #include "share/ob_rpc_struct.h" -#include "rootserver/ob_recovery_ls_service.h" +#include "rootserver/standby/ob_recovery_ls_service.h" #include "logservice/ob_server_log_block_mgr.h" #include "rootserver/ob_admin_drtask_util.h" #include "storage/ddl/ob_tablet_ddl_kv.h" @@ -3308,6 +3308,17 @@ int ObForceDumpServerUsageP::process() return ret; } +int ObRefreshServiceNameP::process() +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(gctx_.ob_service_)) { + ret = OB_ERR_UNEXPECTED; + COMMON_LOG(WARN, "ob_service is null", KR(ret)); + } else if (OB_FAIL(gctx_.ob_service_->refresh_service_name(arg_, result_))) { + COMMON_LOG(WARN, "fail to refresh_service_name", KR(ret), K(arg_)); + } + return ret; +} int ObResourceLimitCalculatorP::process() { int ret = OB_SUCCESS; diff --git a/src/observer/ob_rpc_processor_simple.h b/src/observer/ob_rpc_processor_simple.h index 7424f13f6..d0ef8a3a2 100644 --- a/src/observer/ob_rpc_processor_simple.h +++ b/src/observer/ob_rpc_processor_simple.h @@ -279,6 +279,7 @@ OB_DEFINE_PROCESSOR_S(Srv, OB_TABLET_LOCATION_BROADCAST, ObTabletLocationReceive OB_DEFINE_PROCESSOR_S(Srv, OB_CANCEL_GATHER_STATS, ObCancelGatherStatsP); OB_DEFINE_PROCESSOR_OBADMIN(Srv, OB_LOG_FORCE_SET_TENANT_LOG_DISK, ObForceSetTenantLogDiskP); OB_DEFINE_PROCESSOR_OBADMIN(Srv, OB_FORCE_DUMP_SERVER_USAGE, ObForceDumpServerUsageP); +OB_DEFINE_PROCESSOR_S(Srv, OB_REFRESH_SERVICE_NAME, ObRefreshServiceNameP); OB_DEFINE_PROCESSOR_S(Srv, OB_CAL_UNIT_PHY_RESOURCE, ObResourceLimitCalculatorP); OB_DEFINE_PROCESSOR_S(Srv, OB_CHECK_AND_CANCEL_DDL_COMPLEMENT_DAG, ObRpcCheckandCancelDDLComplementDagP); diff --git a/src/observer/ob_server.cpp b/src/observer/ob_server.cpp index ee898e0bc..c3782a7b2 100644 --- a/src/observer/ob_server.cpp +++ b/src/observer/ob_server.cpp @@ -102,7 +102,7 @@ #include "share/ash/ob_active_sess_hist_task.h" #include "share/ash/ob_active_sess_hist_list.h" #include "share/ob_server_blacklist.h" -#include "share/ob_primary_standby_service.h" // ObPrimaryStandbyService +#include "rootserver/standby/ob_standby_service.h" // ObStandbyService #include "share/scheduler/ob_dag_warning_history_mgr.h" #include "share/longops_mgr/ob_longops_mgr.h" #include "logservice/palf/election/interface/election.h" @@ -503,7 +503,7 @@ int ObServer::init(const ObServerOptions &opts, const ObPLogWriterCfg &log_cfg) } else if (OB_FAIL(imc_tasks_.init())) { LOG_ERROR("init imc tasks failed", KR(ret)); #endif - } else if (OB_FAIL(OB_PRIMARY_STANDBY_SERVICE.init(&sql_proxy_, &schema_service_))) { + } else if (OB_FAIL(OB_STANDBY_SERVICE.init(&sql_proxy_, &schema_service_))) { LOG_ERROR("init OB_PRIMARY_STANDBY_SERVICE failed", KR(ret)); } else if (OB_FAIL(init_px_target_mgr())) { LOG_ERROR("init px target mgr failed", KR(ret)); @@ -823,9 +823,9 @@ void ObServer::destroy() ObVirtualTenantManager::get_instance().destroy(); FLOG_INFO("virtual tenant manager destroyed"); - FLOG_INFO("begin to destroy OB_PRIMARY_STANDBY_SERVICE"); - OB_PRIMARY_STANDBY_SERVICE.destroy(); - FLOG_INFO("OB_PRIMARY_STANDBY_SERVICE destroyed"); + FLOG_INFO("begin to destroy OB_STANDBY_SERVICE"); + OB_STANDBY_SERVICE.destroy(); + FLOG_INFO("OB_STANDBY_SERVICE destroyed"); FLOG_INFO("begin to destroy rootservice event history"); ROOTSERVICE_EVENT_INSTANCE.destroy(); diff --git a/src/observer/ob_server_struct.cpp b/src/observer/ob_server_struct.cpp index d6ff9d80f..2438af405 100644 --- a/src/observer/ob_server_struct.cpp +++ b/src/observer/ob_server_struct.cpp @@ -35,16 +35,6 @@ ObGlobalContext &global_context() bool ObGlobalContext::is_observer() const { return !share::schema::ObSchemaService::g_liboblog_mode_; } -bool ObGlobalContext::is_primary_cluster() const -{ - return true;; -} - -bool ObGlobalContext::is_standby_cluster() const -{ - return false; -} - common::ObClusterRole ObGlobalContext::get_cluster_role() const { return PRIMARY_CLUSTER; @@ -147,39 +137,5 @@ ObGlobalContext &ObGlobalContext::operator=(const ObGlobalContext &other) return *this; } -ObUseWeakGuard::ObUseWeakGuard() -{ - auto *tsi_value = GET_TSI(TSIUseWeak); - if (NULL != tsi_value) { - tsi_value->inited_ = true; - tsi_value->did_use_weak_ = GCTX.is_standby_cluster_and_started();; - } else { - LOG_ERROR_RET(OB_ALLOCATE_MEMORY_FAILED, "tsi value is NULL"); - } -} - -ObUseWeakGuard::~ObUseWeakGuard() -{ - auto *tsi_value = GET_TSI(TSIUseWeak); - if (NULL != tsi_value) { - tsi_value->inited_ = false; - } -} - -bool ObUseWeakGuard::did_use_weak() -{ - bool use_weak = false; - auto *tsi_value = GET_TSI(TSIUseWeak); - if (NULL == tsi_value) { - LOG_ERROR_RET(OB_ALLOCATE_MEMORY_FAILED, "tsi value is NULL"); - use_weak = GCTX.is_standby_cluster_and_started(); - } else if (tsi_value->inited_) { - use_weak = tsi_value->did_use_weak_; - } else { - use_weak = GCTX.is_standby_cluster_and_started(); - } - return use_weak; -} - } // end of namespace observer } // end of namespace oceanbase diff --git a/src/observer/ob_server_struct.h b/src/observer/ob_server_struct.h index 7a27e767f..be14149c5 100644 --- a/src/observer/ob_server_struct.h +++ b/src/observer/ob_server_struct.h @@ -298,10 +298,6 @@ struct ObGlobalContext // Refer to the high availability zone design document // bool is_observer() const; - bool is_standby_cluster_and_started() { return is_observer() && is_standby_cluster() && has_start_service(); } - bool is_started_and_can_weak_read() { return is_observer() && has_start_service(); } - bool is_primary_cluster() const; - bool is_standby_cluster() const; common::ObClusterRole get_cluster_role() const; share::ServerServiceStatus get_server_service_status() const; void set_upgrade_stage(obrpc::ObUpgradeStage upgrade_stage) { upgrade_stage_ = upgrade_stage; } @@ -324,22 +320,6 @@ struct ObThreadContext }; ObGlobalContext &global_context(); - -struct ObUseWeakGuard -{ - ObUseWeakGuard(); - ~ObUseWeakGuard(); - static bool did_use_weak(); -private: - struct TSIUseWeak - { - bool inited_; - bool did_use_weak_; - TSIUseWeak() - :inited_(false), did_use_weak_(false) - {} - }; -}; } // end of namespace observer } // end of namespace oceanbase diff --git a/src/observer/ob_service.cpp b/src/observer/ob_service.cpp index 878ca9281..29dbe8457 100644 --- a/src/observer/ob_service.cpp +++ b/src/observer/ob_service.cpp @@ -48,6 +48,7 @@ #include "sql/optimizer/ob_join_order.h" #include "rootserver/ob_bootstrap.h" #include "rootserver/ob_tenant_info_loader.h" // ObTenantInfoLoader +#include "rootserver/ob_tenant_event_history_table_operator.h" // TENANT_EVENT_INSTANCE #include "observer/ob_server.h" #include "observer/ob_dump_task_generator.h" #include "observer/ob_server_schema_updater.h" @@ -78,6 +79,7 @@ #include "rootserver/backup/ob_backup_task_scheduler.h" // ObBackupTaskScheduler #include "rootserver/backup/ob_backup_schedule_task.h" // ObBackupScheduleTask #include "rootserver/ob_ls_recovery_stat_handler.h"//get_all_ls_replica_readbable_scn +#include "rootserver/ob_service_name_command.h" #ifdef OB_BUILD_TDE_SECURITY #include "share/ob_master_key_getter.h" #endif @@ -231,6 +233,8 @@ int ObService::init(common::ObMySQLProxy &sql_proxy, FLOG_WARN("client_manager_.initialize failed", "self_addr", gctx_.self_addr(), KR(ret)); } else if (OB_FAIL(CLUSTER_EVENT_INSTANCE.init(sql_proxy))) { FLOG_WARN("init cluster event history table failed", KR(ret)); + } else if (OB_FAIL(TENANT_EVENT_INSTANCE.init(sql_proxy, gctx_.self_addr()))) { + FLOG_WARN("init tenant event history table failed", KR(ret), K(gctx_.self_addr())); } else if (OB_FAIL(SERVER_EVENT_INSTANCE.init(sql_proxy, gctx_.self_addr()))) { FLOG_WARN("init server event history table failed", KR(ret)); } else if (OB_FAIL(DEALOCK_EVENT_INSTANCE.init(sql_proxy))) { @@ -359,6 +363,10 @@ void ObService::stop() FLOG_INFO("begin to stop cluster event instance"); CLUSTER_EVENT_INSTANCE.stop(); FLOG_INFO("cluster event instance stopped"); + + FLOG_INFO("begin to stop tenant event instance"); + TENANT_EVENT_INSTANCE.stop(); + FLOG_INFO("tenant event instance stopped"); } FLOG_INFO("[OBSERVICE_NOTICE] observice finish stop", K_(stopped)); } @@ -396,6 +404,10 @@ void ObService::wait() FLOG_INFO("begin to wait cluster event instance"); CLUSTER_EVENT_INSTANCE.wait(); FLOG_INFO("wait cluster event instance success"); + + FLOG_INFO("begin to wait tenant event instance"); + TENANT_EVENT_INSTANCE.wait(); + FLOG_INFO("wait tenant event instance success"); } FLOG_INFO("[OBSERVICE_NOTICE] wait ob_service end"); } @@ -420,6 +432,10 @@ int ObService::destroy() CLUSTER_EVENT_INSTANCE.destroy(); FLOG_INFO("cluster event instance destroyed"); + FLOG_INFO("begin to destroy tenant event instance"); + TENANT_EVENT_INSTANCE.destroy(); + FLOG_INFO("tenant event instance destroyed"); + FLOG_INFO("begin to destroy server event instance"); SERVER_EVENT_INSTANCE.destroy(); FLOG_INFO("server event instance destroyed"); @@ -3020,7 +3036,8 @@ int ObService::estimate_tablet_block_count(const obrpc::ObEstBlockArg &arg, } return ret; } - +ERRSIM_POINT_DEF(ERRSIM_GET_LS_SYNC_SCN_ERROR); +ERRSIM_POINT_DEF(ERRSIM_GET_SYS_LS_SYNC_SCN_ERROR); int ObService::get_ls_sync_scn( const ObGetLSSyncScnArg &arg, ObGetLSSyncScnRes &result) @@ -3089,6 +3106,16 @@ int ObService::get_ls_sync_scn( LOG_WARN("the ls not master", KR(ret), K(ls_id), K(first_leader_epoch), K(second_leader_epoch), K(role)); } + if (OB_SUCC(ret) && ERRSIM_GET_LS_SYNC_SCN_ERROR) { + cur_sync_scn = ls_id.is_sys_ls() ? cur_sync_scn : SCN::minus(cur_sync_scn, 1000); + ret = result.init(arg.get_tenant_id(), ls_id, cur_sync_scn, cur_restore_source_max_scn); + LOG_WARN("user ls errsim enabled", KR(ret), K(arg.get_tenant_id()), K(ls_id), K(cur_sync_scn), K(cur_restore_source_max_scn)); + } + if (OB_SUCC(ret) && ERRSIM_GET_SYS_LS_SYNC_SCN_ERROR) { + cur_sync_scn = ls_id.is_sys_ls() ? SCN::minus(cur_sync_scn, 1000) : cur_sync_scn; + ret = result.init(arg.get_tenant_id(), ls_id, cur_sync_scn, cur_restore_source_max_scn); + LOG_WARN("sys ls errsim enabled", KR(ret), K(arg.get_tenant_id()), K(ls_id), K(cur_sync_scn), K(cur_restore_source_max_scn)); + } } LOG_INFO("finish get_ls_sync_scn", KR(ret), K(cur_sync_scn), K(cur_restore_source_max_scn), K(arg), K(result)); return ret; @@ -3331,6 +3358,60 @@ int ObService::ob_admin_unlock_member_list( return ret; } +int ObService::refresh_service_name( + const ObRefreshServiceNameArg &arg, + ObRefreshServiceNameRes &result) +{ + // 1. epoch: + // 1.1 if the arg's epoch <= the tenant_info_loader's epoch, do nothing + // 1.2 otherwise, replace cache with the arg's service_name_list + // 2. kill local connections when the arg's service_op is STOP SERVICE + // and the target service_name's status in the tenant_info_loader is STOPPING + int ret = OB_SUCCESS; + const uint64_t tenant_id = arg.get_tenant_id(); + MAKE_TENANT_SWITCH_SCOPE_GUARD(guard); + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret), K(inited_), K(arg)); + } else if (OB_UNLIKELY(!arg.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("arg is invaild", KR(ret), K(arg)); + } else if (tenant_id != MTL_ID() && OB_FAIL(guard.switch_to(tenant_id))) { + LOG_WARN("switch tenant failed", KR(ret), K(arg)); + } + + if (OB_SUCC(ret)) { + rootserver::ObTenantInfoLoader *tenant_info_loader = MTL(rootserver::ObTenantInfoLoader*); + if (OB_ISNULL(tenant_info_loader)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant_info_loader should not be null", KR(ret), KP(tenant_info_loader)); + } else if (OB_FAIL(tenant_info_loader->update_service_name(arg.get_epoch(), arg.get_service_name_list()))) { + LOG_WARN("fail to execute update_service_name", KR(ret), K(arg)); + } else if (arg.is_start_service()) { + // When starting the service, it is expected that `service_name` is utilized. + // However, the ability for users to connect via `service_name` also depends on `tenant_info`, + // so it's crucial to ensure that `tenant_info` is up-to-date. + const ObUpdateTenantInfoCacheArg &u_arg = arg.get_update_tenant_info_arg(); + if (OB_FAIL(tenant_info_loader->update_tenant_info_cache(u_arg.get_ora_rowscn(), u_arg.get_tenant_info(), + u_arg.get_finish_data_version(), u_arg.get_data_version_barrier_scn()))) { + LOG_WARN("fail to execute update_tenant_info_cache", KR(ret), K(u_arg), K(arg)); + } + } else if (arg.is_stop_service()) { + ObServiceName service_name; + if (OB_FAIL(tenant_info_loader->get_service_name(arg.get_target_service_name_id(), service_name))) { + LOG_WARN("fail to get service name", KR(ret), K(arg)); + } else if (service_name.is_stopping() + && OB_FAIL(ObServiceNameCommand::kill_local_connections(tenant_id, service_name))) { + LOG_WARN("fail to kill local connections", KR(ret), K(arg), K(service_name)); + } + } + } + if (FAILEDx(result.init(tenant_id))) { + LOG_WARN("failed to init res", KR(ret), K(tenant_id)); + } + FLOG_INFO("finish refresh_service_name", KR(ret), K(arg), K(result)); + return ret; +} }// end namespace observer }// end namespace oceanbase diff --git a/src/observer/ob_service.h b/src/observer/ob_service.h index c7f3dfea1..41fa94f62 100644 --- a/src/observer/ob_service.h +++ b/src/observer/ob_service.h @@ -162,6 +162,8 @@ public: obrpc::ObEstBlockRes &res) const; int update_tenant_info_cache(const obrpc::ObUpdateTenantInfoCacheArg &arg, obrpc::ObUpdateTenantInfoCacheRes &result); + int refresh_service_name(const obrpc::ObRefreshServiceNameArg &arg, + obrpc::ObRefreshServiceNameRes &result); //////////////////////////////////////////////////////////////// // ObRpcMinorFreezeP @RS minor freeze int minor_freeze(const obrpc::ObMinorFreezeArg &arg, diff --git a/src/observer/ob_srv_xlator.cpp b/src/observer/ob_srv_xlator.cpp index bff67ad7e..1fc4d3230 100644 --- a/src/observer/ob_srv_xlator.cpp +++ b/src/observer/ob_srv_xlator.cpp @@ -203,6 +203,8 @@ int ObSrvMySQLXlator::translate(rpc::ObRequest &req, ObReqProcessor *&processor) } else { if (req.is_in_connected_phase()) { ret = get_mp_connect_processor(processor); + } else if (OB_FAIL(check_service_name_(req))) { + LOG_WARN("fail to execute check_service_name_", KR(ret)); } else { const ObMySQLRawPacket &pkt = reinterpret_cast(req.get_packet()); if (pkt.get_cmd() == obmysql::COM_QUERY) { @@ -338,6 +340,36 @@ int ObSrvMySQLXlator::translate(rpc::ObRequest &req, ObReqProcessor *&processor) return ret; } +int ObSrvMySQLXlator::check_service_name_(rpc::ObRequest &req) +{ + int ret = OB_SUCCESS; + ObSMConnection *conn = reinterpret_cast(SQL_REQ_OP.get_sql_session(&req)); + ObSQLSessionInfo *session = NULL; + uint32_t sess_id = 0; + uint64_t tenant_id = OB_INVALID_TENANT_ID; + if (OB_ISNULL(conn) || OB_ISNULL(GCTX.session_mgr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", KR(ret), KP(conn), KP(GCTX.session_mgr_), K(tenant_id)); + } else if (!conn->has_service_name_) { + // do nothing + } else if (FALSE_IT(sess_id = conn->sessid_)) { + } else if (FALSE_IT(tenant_id = conn->tenant_id_)) { + } else if (!is_user_tenant(tenant_id)) { + // do nothing + } else if (OB_FAIL(GCTX.session_mgr_->get_session(sess_id, session))) { + LOG_WARN("fail to get session", KR(ret), K(sess_id), K(tenant_id)); + } else if (OB_ISNULL(session)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", KR(ret), KP(session), K(tenant_id)); + } else if (OB_FAIL(session->check_service_name_and_failover_mode())) { + LOG_WARN("fail to execute check_service_name_and_failover_mode", KR(ret), K(tenant_id)); + } + if (NULL != session) { + GCTX.session_mgr_->revert_session(session); + } + return ret; +} + ObReqProcessor *ObSrvXlator::get_processor(ObRequest &req) { int ret = OB_SUCCESS; diff --git a/src/observer/ob_srv_xlator.h b/src/observer/ob_srv_xlator.h index 390cb6bbc..ed026df9c 100644 --- a/src/observer/ob_srv_xlator.h +++ b/src/observer/ob_srv_xlator.h @@ -157,6 +157,7 @@ protected: int get_mp_connect_processor(ObReqProcessor *&ret_proc); private: + int check_service_name_(rpc::ObRequest &req); const ObGlobalContext &gctx_; DISALLOW_COPY_AND_ASSIGN(ObSrvMySQLXlator); }; // end of class ObSrvMySQLXlator diff --git a/src/observer/ob_srv_xlator_partition.cpp b/src/observer/ob_srv_xlator_partition.cpp index 7dbf83ab7..c6001cda1 100644 --- a/src/observer/ob_srv_xlator_partition.cpp +++ b/src/observer/ob_srv_xlator_partition.cpp @@ -294,6 +294,7 @@ void oceanbase::observer::init_srv_xlator_for_others(ObSrvRpcXlator *xlator) { RPC_PROCESSOR(ObRefreshTenantInfoP, gctx_); RPC_PROCESSOR(ObRpcGetLSReplayedScnP, gctx_); RPC_PROCESSOR(ObUpdateTenantInfoCacheP, gctx_); + RPC_PROCESSOR(ObRefreshServiceNameP, gctx_); RPC_PROCESSOR(ObSyncRewriteRulesP, gctx_); diff --git a/src/observer/omt/ob_multi_tenant.cpp b/src/observer/omt/ob_multi_tenant.cpp index a4da3d6a8..c89d77162 100644 --- a/src/observer/omt/ob_multi_tenant.cpp +++ b/src/observer/omt/ob_multi_tenant.cpp @@ -110,7 +110,7 @@ #include "rootserver/ob_tenant_info_loader.h"//ObTenantInfoLoader #include "rootserver/ob_create_standby_from_net_actor.h" // ObCreateStandbyFromNetActor #include "rootserver/ob_primary_ls_service.h"//ObLSService -#include "rootserver/ob_recovery_ls_service.h"//ObRecoveryLSService +#include "rootserver/standby/ob_recovery_ls_service.h"//ObRecoveryLSService #include "rootserver/ob_common_ls_service.h"//ObCommonLSService #include "rootserver/restore/ob_restore_service.h" //ObRestoreService #include "rootserver/ob_tenant_transfer_service.h" // ObTenantTransferService diff --git a/src/observer/omt/ob_tenant_node_balancer.cpp b/src/observer/omt/ob_tenant_node_balancer.cpp index c0a75298f..c3c5e2283 100644 --- a/src/observer/omt/ob_tenant_node_balancer.cpp +++ b/src/observer/omt/ob_tenant_node_balancer.cpp @@ -242,13 +242,6 @@ int ObTenantNodeBalancer::notify_create_tenant(const obrpc::TenantServerUnitConf } } - // In standby cluster, may repeat create tenant, if if_not_grant_ is true, ignore OB_TENANT_EXIST - if (OB_TENANT_EXIST == ret && unit.if_not_grant_) { - if (GCTX.is_standby_cluster()) { - ret = OB_SUCCESS; - } - } - return ret; } diff --git a/src/observer/virtual_table/ob_all_virtual_ls_log_restore_status.cpp b/src/observer/virtual_table/ob_all_virtual_ls_log_restore_status.cpp index 8fe9b0ec9..9b4aaf1ad 100644 --- a/src/observer/virtual_table/ob_all_virtual_ls_log_restore_status.cpp +++ b/src/observer/virtual_table/ob_all_virtual_ls_log_restore_status.cpp @@ -22,7 +22,7 @@ #include "lib/mysqlclient/ob_mysql_proxy.h" #include "storage/tx_storage/ob_ls_map.h" // ObLSIterator #include "storage/tx_storage/ob_ls_service.h" // ObLSService -#include "rootserver/ob_recovery_ls_service.h" //ObLSRecoveryService +#include "rootserver/standby/ob_recovery_ls_service.h" //ObLSRecoveryService using namespace oceanbase::share; diff --git a/src/observer/virtual_table/ob_show_processlist.cpp b/src/observer/virtual_table/ob_show_processlist.cpp index 0e66a7be5..7a5015e35 100644 --- a/src/observer/virtual_table/ob_show_processlist.cpp +++ b/src/observer/virtual_table/ob_show_processlist.cpp @@ -487,7 +487,12 @@ bool ObShowProcesslist::FillScanner::operator()(sql::ObSQLSessionMgr::Key key, O break; } case SERVICE_NAME: { - cur_row_->cells_[cell_idx].set_null(); + if (!sess_info->get_service_name().is_empty()) { + cur_row_->cells_[cell_idx].set_varchar(sess_info->get_service_name().ptr()); + cur_row_->cells_[cell_idx].set_collation_type(default_collation); + } else { + cur_row_->cells_[cell_idx].set_null(); + } break; } case TOTAL_CPU_TIME: { diff --git a/src/pl/sys_package/ob_dbms_stats.cpp b/src/pl/sys_package/ob_dbms_stats.cpp index 8c88848e7..6ec9edd7b 100644 --- a/src/pl/sys_package/ob_dbms_stats.cpp +++ b/src/pl/sys_package/ob_dbms_stats.cpp @@ -5358,18 +5358,17 @@ int ObDbmsStats::flush_database_monitoring_info(sql::ObExecContext &ctx, int ObDbmsStats::check_statistic_table_writeable(sql::ObExecContext &ctx) { int ret = OB_SUCCESS; - share::schema::ObSchemaGetterGuard *schema_guard = ctx.get_virtual_table_ctx().schema_guard_; - bool in_restore = false; - if (OB_ISNULL(schema_guard) || OB_ISNULL(ctx.get_my_session())) { + uint64_t tenant_id = OB_INVALID_TENANT_ID; + bool is_primary = true; + if (OB_ISNULL(ctx.get_my_session())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(schema_guard)); - } else if (OB_FAIL(schema_guard->check_tenant_is_restore(ctx.get_my_session()->get_effective_tenant_id(), - in_restore))) { - LOG_WARN("failed to check tenant is restore", K(ret)); - } else if (OB_UNLIKELY(in_restore) || - GCTX.is_standby_cluster()) { + LOG_WARN("get unexpected null", KR(ret), KP(ctx.get_my_session())); + } else if (FALSE_IT(tenant_id = ctx.get_my_session()->get_effective_tenant_id())) { + } else if (OB_FAIL(ObShareUtil::mtl_check_if_tenant_role_is_primary(tenant_id, is_primary))) { + LOG_WARN("fail to execute mtl_check_if_tenant_role_is_primary", KR(ret), K(tenant_id)); + } else if (OB_UNLIKELY(!is_primary)) { ret = OB_NOT_SUPPORTED; - LOG_USER_ERROR(OB_NOT_SUPPORTED, "use dbms_stats during restore or standby cluster"); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "use dbms_stats during non-primary tenant"); } return ret; } diff --git a/src/rootserver/CMakeLists.txt b/src/rootserver/CMakeLists.txt index 4aa3609ae..cd45ae23f 100644 --- a/src/rootserver/CMakeLists.txt +++ b/src/rootserver/CMakeLists.txt @@ -54,6 +54,7 @@ ob_set_subtarget(ob_rootserver common ob_root_utils.cpp ob_root_inspection.cpp ob_rs_event_history_table_operator.cpp + ob_tenant_event_history_table_operator.cpp ob_rs_job_table_operator.cpp ob_rs_reentrant_thread.cpp ob_rs_thread_checker.cpp @@ -81,11 +82,9 @@ ob_set_subtarget(ob_rootserver common ob_tenant_info_loader.cpp ob_create_standby_from_net_actor.cpp ob_primary_ls_service.cpp - ob_recovery_ls_service.cpp ob_balance_ls_primary_zone.cpp ob_common_ls_service.cpp ob_ls_service_helper.cpp - ob_tenant_role_transition_service.cpp ob_tenant_transfer_service.cpp ob_transfer_partition_task.cpp ob_tenant_balance_service.cpp @@ -103,6 +102,7 @@ ob_set_subtarget(ob_rootserver common ob_shrink_expand_resource_pool_checker.cpp ob_transfer_partition_command.cpp ob_partition_exchange.cpp + ob_service_name_command.cpp ) ob_set_subtarget(ob_rootserver balance @@ -195,4 +195,10 @@ ob_set_subtarget(ob_rootserver mview mview/ob_mview_dependency_service.cpp ) +ob_set_subtarget(ob_rootserver standby + standby/ob_standby_service.cpp + standby/ob_recovery_ls_service.cpp + standby/ob_tenant_role_transition_service.cpp +) + ob_server_add_target(ob_rootserver) diff --git a/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp b/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp index c7da0efe0..e3f20dede 100644 --- a/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp +++ b/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp @@ -1104,7 +1104,7 @@ int ObBackupSetTaskMgr::get_backup_end_scn_(share::SCN &end_scn) const LOG_WARN("failed to get tenant info", K(ret), K(tenant_id)); } else if (OB_FAIL(ObBackupDataScheduler::get_backup_scn(*sql_proxy_, tenant_id, false/*is backup start*/, end_scn))) { LOG_WARN("failed to get end scn", K(ret), K(tenant_id)); - } else if (tenant_info.is_standby() && end_scn > tenant_info.get_standby_scn()) { + } else if (tenant_info.is_standby() && end_scn > tenant_info.get_readable_scn()) { // For standby tenant, make sure snapshot of end_scn is readable. Otherwise, we // can not backup table list. int64_t abs_timeout = ObTimeUtility::current_time() + 10 * 60 * 1000 * 1000; @@ -1115,7 +1115,7 @@ int ObBackupSetTaskMgr::get_backup_end_scn_(share::SCN &end_scn) const } else if (!tenant_info.is_standby()) { ret = OB_STATE_NOT_MATCH; LOG_WARN("tenant is not standby", K(ret), K(tenant_info)); - } else if (end_scn <= tenant_info.get_standby_scn()) { + } else if (end_scn <= tenant_info.get_readable_scn()) { break; } else if (ObTimeUtility::current_time() > abs_timeout) { ret = OB_TIMEOUT; diff --git a/src/rootserver/ddl_task/ob_build_mview_task.cpp b/src/rootserver/ddl_task/ob_build_mview_task.cpp index b614d83aa..49838c141 100644 --- a/src/rootserver/ddl_task/ob_build_mview_task.cpp +++ b/src/rootserver/ddl_task/ob_build_mview_task.cpp @@ -484,9 +484,12 @@ int ObBuildMViewTask::enable_mview() ObSchemaGetterGuard schema_guard; const ObTableSchema *mview_schema = nullptr; bool mview_table_exist = false; - if (GCTX.is_standby_cluster()) { + bool is_primary = false; + if (OB_FAIL(ObShareUtil::table_check_if_tenant_role_is_primary(tenant_id_, is_primary))) { + LOG_WARN("fail to execute table_check_if_tenant_role_is_primary", KR(ret), K(tenant_id_)); + } else if (!is_primary) { ret = OB_OP_NOT_ALLOW; - LOG_WARN("create mview in slave cluster is not allowed", KR(ret), K(mview_table_id_)); + LOG_WARN("create mview in non-primary tenant is not allowed", KR(ret), K(tenant_id_), K(is_primary), K(mview_table_id_)); } else if (OB_FAIL(schema_service.get_tenant_schema_guard(tenant_id_, schema_guard))) { LOG_WARN("failed to get schema guard", KR(ret), K(tenant_id_)); } else if (OB_FAIL(schema_guard.check_table_exist(tenant_id_, mview_table_id_, mview_table_exist))) { diff --git a/src/rootserver/ddl_task/ob_index_build_task.cpp b/src/rootserver/ddl_task/ob_index_build_task.cpp index ba7c7bbd9..b0ee65b10 100755 --- a/src/rootserver/ddl_task/ob_index_build_task.cpp +++ b/src/rootserver/ddl_task/ob_index_build_task.cpp @@ -1459,9 +1459,12 @@ int ObIndexBuildTask::enable_index() schema_status.tenant_id_ = tenant_id_; int64_t version_in_inner_table = OB_INVALID_VERSION; int64_t local_schema_version = OB_INVALID_VERSION; - if (GCTX.is_standby_cluster()) { + bool is_standby = false; + if (OB_FAIL(ObShareUtil::table_check_if_tenant_role_is_standby(tenant_id_, is_standby))) { + LOG_WARN("fail to execute table_check_if_tenant_role_is_standby", KR(ret), K(tenant_id_)); + } else if (is_standby) { ret = OB_OP_NOT_ALLOW; - LOG_WARN("create global index in slave cluster is not allowed", K(ret), K(index_table_id_)); + LOG_WARN("create global index in standby tenant is not allowed", K(ret), K(index_table_id_)); } else if (OB_FAIL(schema_service.get_tenant_schema_guard(tenant_id_, schema_guard))) { LOG_WARN("fail to get schema guard", K(ret), K(tenant_id_)); } else if (OB_FAIL(schema_guard.check_table_exist(tenant_id_, index_table_id_, index_table_exist))) { diff --git a/src/rootserver/freeze/ob_major_merge_scheduler.cpp b/src/rootserver/freeze/ob_major_merge_scheduler.cpp index 129d6594a..bcf1207dd 100644 --- a/src/rootserver/freeze/ob_major_merge_scheduler.cpp +++ b/src/rootserver/freeze/ob_major_merge_scheduler.cpp @@ -844,7 +844,7 @@ void ObMajorMergeScheduler::check_merge_interval_time(const bool is_merging) false, tenant_info))) { LOG_WARN("fail to load tenant info", KR(ret), K_(tenant_id)); } else if (tenant_info.is_standby() - && (tenant_info.get_standby_scn() >= tenant_info.get_recovery_until_scn())) { + && (tenant_info.get_readable_scn() >= tenant_info.get_recovery_until_scn())) { LOG_INFO("standby tenant do not sync from primary tenant any more, and do not" " major freeze any more"); } else { diff --git a/src/rootserver/ob_ddl_service.cpp b/src/rootserver/ob_ddl_service.cpp index 4ee875a8c..ae6f55d6a 100755 --- a/src/rootserver/ob_ddl_service.cpp +++ b/src/rootserver/ob_ddl_service.cpp @@ -52,7 +52,7 @@ #include "share/ob_global_stat_proxy.h" #include "share/ob_freeze_info_proxy.h" #include "share/ob_service_epoch_proxy.h" -#include "share/ob_primary_standby_service.h" // ObPrimaryStandbyService +#include "rootserver/standby/ob_standby_service.h" // ObStandbyService #include "sql/resolver/ob_stmt_type.h" #include "sql/resolver/ddl/ob_ddl_resolver.h" #include "sql/resolver/expr/ob_raw_expr_modify_column_name.h" @@ -257,7 +257,6 @@ int ObDDLService::get_zones_in_region( int ObDDLService::get_tenant_schema_guard_with_version_in_inner_table(const uint64_t tenant_id, ObSchemaGetterGuard &schema_guard) { int ret = OB_SUCCESS; - bool is_standby = false; bool is_restore = false; bool use_local = false; int64_t version_in_inner_table = OB_INVALID_VERSION; @@ -268,11 +267,9 @@ int ObDDLService::get_tenant_schema_guard_with_version_in_inner_table(const uint } else if (OB_ISNULL(schema_service_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("schema_service is null", K(ret)); - } else if (OB_FAIL(get_is_standby_cluster(is_standby))) { - LOG_WARN("failed to get is standby cluster", K(ret)); } else if (OB_FAIL(schema_service_->check_tenant_is_restore(NULL, tenant_id, is_restore))) { LOG_WARN("fail to check tenant is restore", KR(ret), K(tenant_id)); - } else if ((is_standby || is_restore) && OB_SYS_TENANT_ID != tenant_id) { + } else if (is_restore && OB_SYS_TENANT_ID != tenant_id) { ObSchemaStatusProxy *schema_status_proxy = GCTX.schema_status_proxy_; if (OB_ISNULL(schema_status_proxy)) { ret = OB_ERR_UNEXPECTED; @@ -280,12 +277,8 @@ int ObDDLService::get_tenant_schema_guard_with_version_in_inner_table(const uint } else if (OB_FAIL(schema_status_proxy->get_refresh_schema_status(tenant_id, schema_status))) { LOG_WARN("failed to get tenant refresh schema status", KR(ret), K(tenant_id)); } else if (OB_INVALID_VERSION == schema_status.readable_schema_version_) { - // 1. The standalone cluster: the schema status has been reset, it can use the internal table to refresh, - // in this time the standalone cluster already has a leader - // 2. The second of physical recovery, after reset schema status, modify_schema can be modified + // The second of physical recovery, after reset schema status, modify_schema can be modified use_local = false; - } else if (is_standby) { - use_local = true; } else if (is_restore) { ret = OB_NOT_SUPPORTED; LOG_WARN("tenant is still restoring, ddl not supported", KR(ret), K(tenant_id), K(schema_status)); @@ -2466,11 +2459,8 @@ int ObDDLService::create_tables_in_trans(const bool if_not_exist, uint64_t tenant_data_version = 0; ObArenaAllocator allocator(ObModIds::OB_RS_PARTITION_TABLE_TEMP); RS_TRACE(create_tables_in_trans_begin); - bool is_standby = false; if (OB_FAIL(check_inner_stat())) { LOG_WARN("variable is not init"); - } else if (OB_FAIL(get_is_standby_cluster(is_standby))) { - LOG_WARN("faile to get is standby cluster", K(ret)); } else if (table_schemas.count() < 1) { ret = OB_ERR_UNEXPECTED; LOG_WARN("table_schemas have no element", K(ret)); @@ -17271,7 +17261,6 @@ int ObDDLService::truncate_oracle_temp_table(const ObString &db_name, int ObDDLService::maintain_obj_dependency_info(const obrpc::ObDependencyObjDDLArg &arg) { int ret = OB_SUCCESS; - bool is_standby = false; const uint64_t tenant_id = arg.tenant_id_; ObSchemaService *schema_service = schema_service_->get_schema_service(); ObDDLOperator ddl_operator(*schema_service_, *sql_proxy_); @@ -17283,8 +17272,6 @@ int ObDDLService::maintain_obj_dependency_info(const obrpc::ObDependencyObjDDLAr } else if (OB_ISNULL(schema_service)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("schema_service must not null", K(ret)); - } else if (OB_FAIL(get_is_standby_cluster(is_standby))) { - LOG_WARN("failed to get is standby cluster", K(ret)); } else { ObDDLSQLTransaction trans(schema_service_); ObSchemaGetterGuard schema_guard; @@ -17296,15 +17283,15 @@ int ObDDLService::maintain_obj_dependency_info(const obrpc::ObDependencyObjDDLAr } else if (OB_FAIL(trans.start(sql_proxy_, tenant_id, refreshed_schema_version))) { LOG_WARN("failed to start trans, ", KR(ret), K(tenant_id), K(refreshed_schema_version)); } else if (!arg.update_dep_objs_.empty() - && OB_FAIL(process_schema_object_dependency(tenant_id, is_standby, arg.update_dep_objs_, + && OB_FAIL(process_schema_object_dependency(tenant_id, arg.update_dep_objs_, schema_guard, trans, ddl_operator, ObReferenceObjTable::UPDATE_OP))) { LOG_WARN("failed to process update object dependency", K(ret)); } else if (!arg.insert_dep_objs_.empty() - && OB_FAIL(process_schema_object_dependency(tenant_id, is_standby, arg.insert_dep_objs_, + && OB_FAIL(process_schema_object_dependency(tenant_id, arg.insert_dep_objs_, schema_guard, trans, ddl_operator, ObReferenceObjTable::INSERT_OP))) { LOG_WARN("failed to process insert object dependency", K(ret)); } else if (!arg.delete_dep_objs_.empty() - && OB_FAIL(process_schema_object_dependency(tenant_id, is_standby, arg.delete_dep_objs_, + && OB_FAIL(process_schema_object_dependency(tenant_id, arg.delete_dep_objs_, schema_guard, trans, ddl_operator, ObReferenceObjTable::DELETE_OP))) { LOG_WARN("failed to process delete object dependency", K(ret)); } else if (arg.schema_.is_valid() && OB_FAIL(recompile_view(arg.schema_, arg.reset_view_column_infos_, trans))) { @@ -17331,7 +17318,6 @@ int ObDDLService::maintain_obj_dependency_info(const obrpc::ObDependencyObjDDLAr int ObDDLService::process_schema_object_dependency( const uint64_t tenant_id, - const bool is_standby, const ObReferenceObjTable::DependencyObjKeyItemPairs &dep_objs, ObSchemaGetterGuard &schema_guard, ObMySQLTransaction &trans, @@ -17348,12 +17334,11 @@ int ObDDLService::process_schema_object_dependency( switch (op) { case ObReferenceObjTable::INSERT_OP: case ObReferenceObjTable::UPDATE_OP: - OZ (ObReferenceObjTable::batch_execute_insert_or_update_obj_dependency(tenant_id, is_standby, + OZ (ObReferenceObjTable::batch_execute_insert_or_update_obj_dependency(tenant_id, new_schema_version, dep_objs, trans, schema_guard, ddl_operator)); break; case ObReferenceObjTable::DELETE_OP: - OZ (ObReferenceObjTable::batch_execute_delete_obj_dependency(tenant_id, is_standby, - dep_objs, trans)); + OZ (ObReferenceObjTable::batch_execute_delete_obj_dependency(tenant_id, dep_objs, trans)); break; default: break; @@ -23826,30 +23811,23 @@ int ObDDLService::purge_recyclebin_tenant( ddl_stmt.reset(); const ObRecycleObject &recycle_obj = recycle_objs.at(i); if (ObRecycleObject::TENANT == recycle_obj.get_type()) { - bool is_standby = false; - if (OB_FAIL(get_is_standby_cluster(is_standby))) { - LOG_WARN("fail to get", K(ret)); - } else if (!is_standby) { - if (tenant_id != OB_SYS_TENANT_ID) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("purge tenant only in sys tenant", K(ret)); - } else if (OB_FAIL(ddl_stmt.assign_fmt("PURGE TENANT %.*s", - recycle_obj.get_object_name().length(), - recycle_obj.get_object_name().ptr()))) { - LOG_WARN("append sql failed", K(ret)); + if (tenant_id != OB_SYS_TENANT_ID) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("purge tenant only in sys tenant", K(ret)); + } else if (OB_FAIL(ddl_stmt.assign_fmt("PURGE TENANT %.*s", + recycle_obj.get_object_name().length(), + recycle_obj.get_object_name().ptr()))) { + LOG_WARN("append sql failed", K(ret)); + } else { + ObPurgeTenantArg purge_tenant_arg; + purge_tenant_arg.tenant_id_ = OB_SYS_TENANT_ID; + purge_tenant_arg.tenant_name_ = recycle_obj.get_object_name(); + purge_tenant_arg.ddl_stmt_str_ = ddl_stmt.string(); + if (OB_FAIL(purge_tenant(purge_tenant_arg))) { + LOG_WARN("purge tenant failed", K(purge_tenant_arg), K(recycle_obj), K(ret)); } else { - ObPurgeTenantArg purge_tenant_arg; - purge_tenant_arg.tenant_id_ = OB_SYS_TENANT_ID; - purge_tenant_arg.tenant_name_ = recycle_obj.get_object_name(); - purge_tenant_arg.ddl_stmt_str_ = ddl_stmt.string(); - if (OB_FAIL(purge_tenant(purge_tenant_arg))) { - LOG_WARN("purge tenant failed", K(purge_tenant_arg), K(recycle_obj), K(ret)); - } else { - ++purged_objects; - } + ++purged_objects; } - } else { // standalone cluster is not executed, but it should be counted normally - ++purged_objects; } } } @@ -25330,7 +25308,8 @@ int ObDDLService::create_sys_tenant( 0, /*freeze_service_epoch*/ 0, /*arbitration_service_epoch*/ 0, /*server_zone_op_service_epoch*/ - 0 /*heartbeat_service_epoch*/))) { + 0, /*heartbeat_service_epoch*/ + 0 /* service_name_epoch */))) { LOG_WARN("fail to init service epoch", KR(ret)); } if (trans.is_started()) { @@ -26766,11 +26745,11 @@ int ObDDLService::init_tenant_schema( LOG_WARN("fail to set tenant init global stat", KR(ret), K(tenant_id), K(core_schema_version), K(baseline_schema_version), K(snapshot_gc_scn), K(ddl_epoch), K(data_version)); - } else if (is_user_tenant(tenant_id) && OB_FAIL(OB_PRIMARY_STANDBY_SERVICE.write_upgrade_barrier_log( + } else if (is_user_tenant(tenant_id) && OB_FAIL(OB_STANDBY_SERVICE.write_upgrade_barrier_log( trans, tenant_id, data_version))) { LOG_WARN("fail to write_upgrade_barrier_log", KR(ret), K(tenant_id), K(data_version)); } else if (is_user_tenant(tenant_id) && - OB_FAIL(OB_PRIMARY_STANDBY_SERVICE.write_upgrade_data_version_barrier_log( + OB_FAIL(OB_STANDBY_SERVICE.write_upgrade_data_version_barrier_log( trans, tenant_id, data_version))) { LOG_WARN("fail to write_upgrade_data_version_barrier_log", KR(ret), K(tenant_id), K(data_version)); @@ -26815,7 +26794,8 @@ int ObDDLService::init_tenant_schema( 0, /*freeze_service_epoch*/ 0, /*arbitration_service_epoch*/ 0, /*server_zone_op_service_epoch*/ - 0 /*heartbeat_service_epoch*/))) { + 0, /*heartbeat_service_epoch*/ + 0 /* service_name_epoch */))) { LOG_WARN("fail to init service epoch", KR(ret)); } else if (is_creating_standby && OB_FAIL(set_log_restore_source(gen_user_tenant_id(tenant_id), log_restore_source, trans))) { LOG_WARN("fail to set_log_restore_source", KR(ret), K(tenant_id), K(log_restore_source)); @@ -28094,11 +28074,8 @@ int ObDDLService::modify_tenant(const ObModifyTenantArg &arg) const ObTenantSchema *orig_tenant_schema = NULL; const ObString &tenant_name = arg.tenant_schema_.get_tenant_name(); bool is_restore = false; - bool is_standby = false; if (OB_FAIL(check_inner_stat())) { LOG_WARN("variable is not init"); - } else if (OB_FAIL(get_is_standby_cluster(is_standby))) { - LOG_WARN("failed to get is standby", K(ret)); } else if (0 != arg.sys_var_list_.count() && !arg.alter_option_bitset_.is_empty()) { // After the schema is split, because __all_sys_variable is split under the tenant, in order to avoid @@ -28138,13 +28115,13 @@ int ObDDLService::modify_tenant(const ObModifyTenantArg &arg) } if (OB_FAIL(ret)) { - } else if (OB_FAIL(modify_tenant_inner_phase(arg, orig_tenant_schema, schema_guard, is_standby, is_restore))) { + } else if (OB_FAIL(modify_tenant_inner_phase(arg, orig_tenant_schema, schema_guard, is_restore))) { LOG_WARN("modify_tenant_inner_phase fail", K(ret)); } return ret; } -int ObDDLService::modify_tenant_inner_phase(const ObModifyTenantArg &arg, const ObTenantSchema *orig_tenant_schema, ObSchemaGetterGuard &schema_guard, bool is_standby, bool is_restore) +int ObDDLService::modify_tenant_inner_phase(const ObModifyTenantArg &arg, const ObTenantSchema *orig_tenant_schema, ObSchemaGetterGuard &schema_guard, bool is_restore) { int ret = OB_SUCCESS; if (OB_GTS_TENANT_ID == orig_tenant_schema->get_tenant_id()) { @@ -28159,7 +28136,7 @@ int ObDDLService::modify_tenant_inner_phase(const ObModifyTenantArg &arg, const ObDDLSQLTransaction trans(schema_service_); ObDDLOperator ddl_operator(*schema_service_, *sql_proxy_); bool value_changed = false; - if ((is_standby || is_restore) && is_user_tenant(tenant_id)) { + if (is_restore && is_user_tenant(tenant_id)) { ret = OB_OP_NOT_ALLOW; LOG_WARN("ddl operation is not allowed in standby cluster", K(ret)); LOG_USER_ERROR(OB_OP_NOT_ALLOW, "ddl operation in standby cluster"); @@ -28337,11 +28314,6 @@ int ObDDLService::modify_tenant_inner_phase(const ObModifyTenantArg &arg, const LOG_WARN("rename tenant while tenant is in physical restore status is not allowed", KR(ret), KPC(orig_tenant_schema)); LOG_USER_ERROR(OB_OP_NOT_ALLOW, "rename tenant while tenant is in physical restore status is"); - } else if (is_standby && is_user_tenant(orig_tenant_schema->get_tenant_id()) - && !arg.is_sync_from_primary()) { - ret = OB_OP_NOT_ALLOW; - LOG_WARN("rename user tenant in standby is not allowed", KR(ret), K(arg)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "rename user tenant in standby cluster"); } else if (orig_tenant_schema->get_tenant_id() <= OB_MAX_RESERVED_TENANT_ID) { ret = OB_NOT_SUPPORTED; LOG_WARN("rename special tenant not supported", @@ -36231,13 +36203,9 @@ int ObDDLService::check_create_schema_replica_options( } if (OB_SUCC(ret)) { int64_t paxos_num = 0; - bool is_standby = false; if (OB_FAIL(schema.get_paxos_replica_num(schema_guard, paxos_num))) { LOG_WARN("fail to get paxos replica num", K(ret)); - } else if (OB_FAIL(get_is_standby_cluster(is_standby))) { - LOG_WARN("failed to get is standby cluster", K(ret)); - } else if ((!is_standby && paxos_num <= 0) - || paxos_num > common::OB_MAX_MEMBER_NUMBER) { + } else if (paxos_num <= 0 || paxos_num > common::OB_MAX_MEMBER_NUMBER) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid paxos replica num", K(ret), K(schema)); LOG_USER_ERROR(OB_INVALID_ARGUMENT, "locality paxos replica num"); @@ -36288,13 +36256,9 @@ int ObDDLService::check_alter_schema_replica_options( if (OB_SUCC(ret)) { int64_t paxos_num = 0; - bool is_standby = false; if (OB_FAIL(new_schema.get_paxos_replica_num(schema_guard, paxos_num))) { LOG_WARN("fail to get paxos replica num", K(ret)); - } else if (OB_FAIL(get_is_standby_cluster(is_standby))) { - LOG_WARN("failed to get is standby cluster", K(ret)); - } else if ((!is_standby && paxos_num <= 0) - || paxos_num > common::OB_MAX_MEMBER_NUMBER) { + } else if (paxos_num <= 0 || paxos_num > common::OB_MAX_MEMBER_NUMBER) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid paxos replica num", K(ret)); LOG_USER_ERROR(OB_INVALID_ARGUMENT, "locality paxos replica num"); @@ -36472,13 +36436,6 @@ int ObDDLService::construct_zone_region_list( return ret; } -int ObDDLService::get_is_standby_cluster(bool &is_standby) const -{ - int ret = OB_SUCCESS; - is_standby = false; - return ret; -} - template int ObDDLService::set_schema_replica_num_options( SCHEMA &schema, diff --git a/src/rootserver/ob_ddl_service.h b/src/rootserver/ob_ddl_service.h index f67c70144..77f99faa8 100644 --- a/src/rootserver/ob_ddl_service.h +++ b/src/rootserver/ob_ddl_service.h @@ -654,7 +654,6 @@ public: int maintain_obj_dependency_info(const obrpc::ObDependencyObjDDLArg &arg); int process_schema_object_dependency( const uint64_t tenant_id, - const bool is_standby, const share::schema::ObReferenceObjTable::DependencyObjKeyItemPairs &dep_objs, share::schema::ObSchemaGetterGuard &schema_guard, ObMySQLTransaction &trans, @@ -1308,7 +1307,6 @@ private: int modify_tenant_inner_phase(const obrpc::ObModifyTenantArg &arg, const ObTenantSchema *orig_tenant_schema, ObSchemaGetterGuard &schema_guard, - bool is_standby, bool is_restore); int update_global_index(obrpc::ObAlterTableArg &arg, const uint64_t tenant_id, @@ -2629,7 +2627,6 @@ private: const share::schema::ObSysVariableSchema &sys_variable, share::schema::ObSysParam *sys_params, int64_t params_capacity); - int get_is_standby_cluster(bool &is_standby) const; int check_can_alter_column( const int64_t tenant_id, const share::schema::AlterTableSchema &alter_table_schema, diff --git a/src/rootserver/ob_empty_server_checker.cpp b/src/rootserver/ob_empty_server_checker.cpp index 5ce28b392..cc0fbc41f 100644 --- a/src/rootserver/ob_empty_server_checker.cpp +++ b/src/rootserver/ob_empty_server_checker.cpp @@ -290,16 +290,34 @@ int ObEmptyServerChecker::check_server_emtpy_by_ls_( LOG_WARN("NULL replica pointer", K(ret)); } else { // check whether has member on empty servers - FOREACH_CNT_X(m, replica->get_member_list(), OB_SUCC(ret)) { + FOREACH_CNT_X(m, replica->get_member_list(), OB_SUCC(ret) && empty_servers.count() > 0) { const ObAddr &addr = m->get_server(); if (has_exist_in_array(empty_servers, addr, &idx)) { //has member in server - LOG_INFO("ls replica has member on sever", K(ls_info), K(addr), K(empty_servers)); + LOG_INFO("ls replica has member on server", K(ls_info), K(addr), K(empty_servers)); if (OB_FAIL(empty_servers.remove(idx))) { LOG_WARN("failed to remove addr from empty servers", KR(ret), K(idx), K(empty_servers)); } } } // end FORECAH member_list + ObMember learner; + for (int64_t index = 0; + OB_SUCC(ret) && index < replica->get_learner_list().get_member_number() && empty_servers.count() > 0; + ++index) { + learner.reset(); + if (OB_FAIL(replica->get_learner_list().get_member_by_index(index, learner))) { + LOG_WARN("fail to get learner by index", KR(ret), K(index)); + } else { + const ObAddr &addr = learner.get_server(); + if (has_exist_in_array(empty_servers, addr, &idx)) { + //has learner in server + LOG_INFO("ls replica has learner on server", K(ls_info), K(addr), K(empty_servers)); + if (OB_FAIL(empty_servers.remove(idx))) { + LOG_WARN("failed to remove addr from empty servers", KR(ret), K(idx), K(empty_servers)); + } + } + } + } } // filter server of replicas for (int64_t i = 0; i < replica_array.count() && OB_SUCC(ret); ++i) { diff --git a/src/rootserver/ob_ls_recovery_reportor.cpp b/src/rootserver/ob_ls_recovery_reportor.cpp index 94e3fc796..c42a2029f 100755 --- a/src/rootserver/ob_ls_recovery_reportor.cpp +++ b/src/rootserver/ob_ls_recovery_reportor.cpp @@ -14,7 +14,7 @@ #include "rootserver/ob_ls_recovery_reportor.h" #include "rootserver/ob_tenant_info_loader.h" -#include "rootserver/ob_tenant_role_transition_service.h"//ObTenantRoleTransitionConstants +#include "rootserver/standby/ob_tenant_role_transition_service.h"//ObTenantRoleTransitionConstants #include "rootserver/ob_rs_async_rpc_proxy.h" //ObGetLSReplayedScnProxy #include "rootserver/ob_ls_recovery_stat_handler.h" //ObLSRecoveryStatHandler #include "rootserver/ob_ls_service_helper.h"//update_ls_stat_in_trans diff --git a/src/rootserver/ob_ls_service_helper.cpp b/src/rootserver/ob_ls_service_helper.cpp index 4462062d1..3ce7ce8c6 100755 --- a/src/rootserver/ob_ls_service_helper.cpp +++ b/src/rootserver/ob_ls_service_helper.cpp @@ -29,9 +29,9 @@ #include "share/ob_upgrade_utils.h"//ObUpgradeChecker #include "share/rc/ob_tenant_base.h"//MTL_SWITCH #include "observer/ob_server_struct.h"//GCTX -#include "rootserver/ob_recovery_ls_service.h"//ObRecoveryLSHelper +#include "rootserver/standby/ob_recovery_ls_service.h"//ObRecoveryLSHelper #include "rootserver/ob_tenant_thread_helper.h"//get_zone_priority -#include "rootserver/ob_tenant_role_transition_service.h"//get_checkpoint_by_rpc +#include "rootserver/standby/ob_tenant_role_transition_service.h"//get_checkpoint_by_rpc #include "storage/tx_storage/ob_ls_map.h" #include "storage/tx_storage/ob_ls_service.h" #include "storage/tx_storage/ob_ls_handle.h" //ObLSHandle diff --git a/src/rootserver/ob_root_inspection.cpp b/src/rootserver/ob_root_inspection.cpp index c713e394f..cadd67e75 100755 --- a/src/rootserver/ob_root_inspection.cpp +++ b/src/rootserver/ob_root_inspection.cpp @@ -128,8 +128,6 @@ int ObTenantChecker::check_create_tenant_end_() LOG_WARN("schema service not init", K(ret)); } else if (!schema_service_->is_sys_full_schema()) { // skip - } else if (GCTX.is_standby_cluster()) { - // skip } else if (OB_FAIL(schema_service_->get_tenant_ids(tenant_ids))) { LOG_WARN("get_tenant_ids failed", K(ret)); } else if (OB_ISNULL(GCTX.root_service_)) { @@ -991,8 +989,6 @@ int ObRootInspection::calc_diff_names(const uint64_t tenant_id, ObIArray &miss_names /* data inner table less than hard code*/) { int ret = OB_SUCCESS; - ObRefreshSchemaStatus schema_status; - schema_status.tenant_id_ = tenant_id; fetch_names.reset(); if (!inited_) { ret = OB_NOT_INIT; @@ -1007,28 +1003,21 @@ int ObRootInspection::calc_diff_names(const uint64_t tenant_id, ret = OB_INVALID_ARGUMENT; LOG_WARN("table_name is null or names is empty", KR(ret), K(tenant_id), KP(table_name), K(names)); - } else if (GCTX.is_standby_cluster() && is_user_tenant(tenant_id)) { - if (OB_ISNULL(GCTX.schema_status_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("schema status proxy is null", K(ret)); - } else if (OB_FAIL(GCTX.schema_status_proxy_->get_refresh_schema_status(tenant_id, schema_status))) { - LOG_WARN("fail to get schema status", KR(ret), K(tenant_id)); - } } if (OB_SUCC(ret)) { - const uint64_t exec_tenant_id = schema_status.tenant_id_; - int64_t snapshot_timestamp = schema_status.snapshot_timestamp_; + const uint64_t exec_tenant_id = tenant_id; ObSqlString sql; - ObSQLClientRetryWeak sql_client_retry_weak(sql_proxy_, - snapshot_timestamp); if (OB_FAIL(sql.append_fmt("SELECT name FROM %s%s%s", table_name, (extra_cond.empty()) ? "" : " WHERE ", extra_cond.ptr()))) { LOG_WARN("append_fmt failed", KR(ret), K(tenant_id), K(table_name), K(extra_cond)); + } else if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("GCTX.sql_proxy_ is null", KR(ret), KP(GCTX.sql_proxy_)); } else { SMART_VAR(ObMySQLProxy::MySQLResult, res) { ObMySQLResult *result = NULL; - if (OB_FAIL(sql_client_retry_weak.read(res, exec_tenant_id, sql.ptr()))) { + if (OB_FAIL(GCTX.sql_proxy_->read(res, exec_tenant_id, sql.ptr()))) { LOG_WARN("execute sql failed", KR(ret), K(tenant_id), K(sql)); can_retry_ = true; } else if (OB_ISNULL(result = res.get_result())) { @@ -1063,7 +1052,7 @@ int ObRootInspection::calc_diff_names(const uint64_t tenant_id, if (OB_SUCC(ret)) { if (fetch_names.count() <= 0) { LOG_WARN("maybe tenant or zone has been deleted, ignore it", - KR(ret), K(schema_status), K(table_name), K(extra_cond)); + KR(ret), K(table_name), K(extra_cond)); } else { extra_names.reset(); miss_names.reset(); diff --git a/src/rootserver/ob_root_service.cpp b/src/rootserver/ob_root_service.cpp index c2ee2855e..af5ea20b9 100755 --- a/src/rootserver/ob_root_service.cpp +++ b/src/rootserver/ob_root_service.cpp @@ -3029,19 +3029,6 @@ int ObRootService::alter_database(const ObAlterDatabaseArg &arg) } else if (!arg.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arg", K(arg), K(ret)); - } else if (common::STANDBY_CLUSTER == ObClusterInfoGetter::get_cluster_role_v2()) { - const int64_t tenant_id = arg.database_schema_.get_tenant_id(); - ObSchemaGetterGuard schema_guard; - uint64_t database_id = OB_INVALID_ID; - if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table( - tenant_id, schema_guard))) { - LOG_WARN("get_schema_guard with version in inner table failed", K(ret), K(tenant_id)); - } else if (OB_FAIL(schema_guard.get_database_id(tenant_id, - arg.database_schema_.get_database_name_str(), database_id))) { - LOG_WARN("failed to get database id", K(ret), K(tenant_id), K(arg)); - } - } - if (OB_FAIL(ret)) { } else if (OB_FAIL(ddl_service_.alter_database(arg))) { LOG_WARN("alter database failed", K(arg), K(ret)); } @@ -9493,7 +9480,6 @@ int ObRootService::admin_rolling_upgrade_cmd(const obrpc::ObAdminRollingUpgradeA int ObRootService::physical_restore_tenant(const obrpc::ObPhysicalRestoreTenantArg &arg, obrpc::Int64 &res_job_id) { int ret = OB_SUCCESS; - bool has_standby_cluster = false; res_job_id = OB_INVALID_ID; int64_t current_timestamp = ObTimeUtility::current_time(); int64_t start_ts = ObTimeUtility::current_time(); @@ -9515,12 +9501,10 @@ int ObRootService::physical_restore_tenant(const obrpc::ObPhysicalRestoreTenantA } else if (!arg.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arg", K(arg), K(ret)); - } else if (GCTX.is_standby_cluster() || GCONF.in_upgrade_mode()) { + } else if (GCONF.in_upgrade_mode()) { ret = OB_OP_NOT_ALLOW; - LOG_WARN("restore tenant while in standby cluster or " - "in upgrade mode is not allowed", KR(ret)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, - "restore tenant while in standby cluster or in upgrade mode"); + LOG_WARN("in upgrade mode is not allowed", KR(ret)); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "in upgrade mode"); } else if (0 == restore_concurrency) { ret = OB_OP_NOT_ALLOW; LOG_WARN("restore tenant when restore_concurrency is 0 not allowed", KR(ret)); @@ -11036,7 +11020,6 @@ int ObRootService::get_recycle_schema_versions( { int ret = OB_SUCCESS; LOG_INFO("receive get recycle schema versions request", K(arg)); - bool is_standby = GCTX.is_standby_cluster(); bool in_service = is_full_service(); if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; @@ -11044,10 +11027,10 @@ int ObRootService::get_recycle_schema_versions( } else if (!arg.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("arg is invalid", K(ret), K(arg)); - } else if (!is_standby || !in_service) { + } else if (!in_service) { ret = OB_STATE_NOT_MATCH; - LOG_WARN("should be standby cluster and rs in service", - KR(ret), K(is_standby), K(in_service)); + LOG_WARN("should be rs in service", + KR(ret), K(in_service)); } else if (OB_FAIL(schema_history_recycler_.get_recycle_schema_versions(arg, result))) { LOG_WARN("fail to get recycle schema versions", KR(ret), K(arg)); } @@ -11831,10 +11814,6 @@ int ObRootService::handle_recover_table(const obrpc::ObRecoverTableArg &arg) } else if (!arg.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(arg)); - } else if (GCTX.is_standby_cluster()) { - ret = OB_OP_NOT_ALLOW; - LOG_WARN("recover table in standby tenant is not allowed", K(ret), K(arg)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "recover table in standby tenant"); } else if (GCONF.in_upgrade_mode()) { ret = OB_OP_NOT_ALLOW; LOG_WARN("recover table in upgrade mode is not allowed", K(ret), K(arg)); diff --git a/src/rootserver/ob_rs_async_rpc_proxy.h b/src/rootserver/ob_rs_async_rpc_proxy.h index ab4315cce..4175a3c56 100644 --- a/src/rootserver/ob_rs_async_rpc_proxy.h +++ b/src/rootserver/ob_rs_async_rpc_proxy.h @@ -90,6 +90,7 @@ RPC_F(obrpc::OB_TRIM_KEY_LIST, obrpc::ObTrimKeyListArg, obrpc::ObTrimKeyListResu RPC_F(obrpc::OB_INNER_CREATE_TENANT_SNAPSHOT, obrpc::ObInnerCreateTenantSnapshotArg, obrpc::ObInnerCreateTenantSnapshotResult, ObTenantSnapshotCreatorProxy); RPC_F(obrpc::OB_INNER_DROP_TENANT_SNAPSHOT, obrpc::ObInnerDropTenantSnapshotArg, obrpc::ObInnerDropTenantSnapshotResult, ObTenantSnapshotDropperProxy); RPC_F(obrpc::OB_FLUSH_LS_ARCHIVE, obrpc::ObFlushLSArchiveArg, obrpc::Int64, ObFlushLSArchiveProxy); +RPC_F(obrpc::OB_REFRESH_SERVICE_NAME, obrpc::ObRefreshServiceNameArg, obrpc::ObRefreshServiceNameRes, ObRefreshServiceNameProxy); RPC_F(obrpc::OB_CAL_STANDBY_TENANT_PHY_RESOURCE, obrpc::ObGetTenantResArg, obrpc::ObTenantLogicalRes, ObGetTenantResProxy); RPC_F(obrpc::OB_KILL_QUERY_CLIENT_SESSION, obrpc::ObKillQueryClientSessionArg, obrpc::Int64, ObKillQueryClientSessionProxy); diff --git a/src/rootserver/ob_rs_rpc_processor.h b/src/rootserver/ob_rs_rpc_processor.h index 1d68251cd..8f305165b 100644 --- a/src/rootserver/ob_rs_rpc_processor.h +++ b/src/rootserver/ob_rs_rpc_processor.h @@ -124,11 +124,9 @@ protected: } else if (OB_INVALID_TENANT_ID == ddl_arg_->exec_tenant_id_) { ret = OB_INVALID_ARGUMENT; RS_LOG(WARN, "exec tenant id is invalid", K(ret), "arg", *ddl_arg_); - } else if (common::STANDBY_CLUSTER == ObClusterInfoGetter::get_cluster_role_v2() - && !ddl_arg_->is_allow_in_standby()) { - ret = OB_OP_NOT_ALLOW; - RS_LOG(WARN, "ddl operation not allow in standby", KR(ret), KPC(ddl_arg_)); } else { + // TODO (linqiucen.lqc): check whether the tenant is standby + // it will be done after DDL is executed by the tenant itself rather than sys tenant auto *tsi_value = GET_TSI(share::schema::TSIDDLVar); // used for parallel ddl auto *tsi_generator = GET_TSI(share::schema::TSISchemaVersionGenerator); diff --git a/src/rootserver/ob_schema_history_recycler.cpp b/src/rootserver/ob_schema_history_recycler.cpp index f965ba0e0..af25bf3fd 100644 --- a/src/rootserver/ob_schema_history_recycler.cpp +++ b/src/rootserver/ob_schema_history_recycler.cpp @@ -180,9 +180,6 @@ int ObSchemaHistoryRecycler::check_stop() int ret = OB_SUCCESS; if (OB_FAIL(check_inner_stat())) { LOG_WARN("fail to check inner stat", KR(ret)); - } else if (GCTX.is_standby_cluster()) { - ret = OB_CANCELED; - LOG_WARN("schema history recycler should stopped", KR(ret)); } return ret; } @@ -269,10 +266,20 @@ int ObSchemaHistoryRecycler::try_recycle_schema_history() LOG_WARN("fail to check inner stat", KR(ret)); } else if (OB_FAIL(schema_service_->get_tenant_ids(tenant_ids))) { LOG_WARN("fail to get schema_guard", KR(ret)); + } else { + for (int64_t i = tenant_ids.count() - 1; OB_SUCC(ret) && i >= 0; i--) { + const uint64_t tenant_id = tenant_ids.at(i); + bool skip = true; + if (OB_FAIL(check_can_skip_tenant(tenant_id, skip))) { + LOG_WARN("fail to check tenant can skip", KR(ret), K(tenant_id)); + } else if (skip && OB_FAIL(tenant_ids.remove(i))) { + LOG_WARN("fail to remove tenant_id", KR(ret), K(tenant_ids), K(i)); + } + } + } + if (OB_FAIL(ret) || tenant_ids.count() <= 0 ) { } else if (OB_FAIL(calc_recycle_schema_versions(tenant_ids))) { LOG_WARN("fail to fetch recycle schema version", KR(ret)); - } else if (GCTX.is_standby_cluster()) { - // standby cluster only calc recycle schema versions } else if (OB_FAIL(try_recycle_schema_history(tenant_ids))) { LOG_WARN("fail to recycle schema history", KR(ret)); } @@ -288,13 +295,8 @@ int ObSchemaHistoryRecycler::try_recycle_schema_history( } else { for (int64_t i = 0; OB_SUCC(ret) && i < tenant_ids.count(); i++) { const uint64_t tenant_id = tenant_ids.at(i); - bool skip = true; if (OB_FAIL(check_stop())) { LOG_WARN("schema history recycler is stopped", KR(ret)); - } else if (OB_FAIL(check_can_skip_tenant(tenant_id, skip))) { - LOG_WARN("fail to check tenant can skip", KR(ret), K(tenant_id)); - } else if (skip) { - // pass } else { int64_t recycle_schema_version = OB_INVALID_VERSION; if (OB_FAIL(recycle_schema_versions_.get_refactored(tenant_id, recycle_schema_version))) { @@ -319,6 +321,7 @@ int ObSchemaHistoryRecycler::check_can_skip_tenant( bool &skip) { int ret = OB_SUCCESS; + bool is_primary = false; skip = false; if (!inited_) { ret = OB_NOT_INIT; @@ -328,6 +331,10 @@ int ObSchemaHistoryRecycler::check_can_skip_tenant( // Additional schema history of system tenant should be recycled: // 1. Other tenant's schema history(except tenant schema history and system table's schema history) generated before schema split. skip = true; + } else if (OB_FAIL(ObShareUtil::table_check_if_tenant_role_is_primary(tenant_id, is_primary))) { + LOG_WARN("fail to execute table_check_if_tenant_role_is_primary", KR(ret), K(tenant_id)); + } else if (!is_primary) { + skip = true; } else { ObSchemaGetterGuard schema_guard; const ObSimpleTenantSchema *tenant_schema = NULL; @@ -479,7 +486,6 @@ int ObSchemaHistoryRecycler::get_recycle_schema_version_by_global_stat( if (OB_FAIL(check_inner_stat())) { LOG_WARN("fail to check inner stat", KR(ret)); } else { - bool is_standby = GCTX.is_standby_cluster(); if (OB_SUCC(ret)) { // step 1. calc by schema_history_expire_time int64_t conf_expire_time = GCONF.schema_history_expire_time; @@ -493,13 +499,7 @@ int ObSchemaHistoryRecycler::get_recycle_schema_version_by_global_stat( const uint64_t tenant_id = tenant_ids.at(i); int64_t expire_schema_version = OB_INVALID_VERSION; ObRefreshSchemaStatus schema_status; - if (!is_standby) { - schema_status.tenant_id_ = tenant_id; // use strong read - } else { - if (OB_FAIL(schema_status_proxy->get_refresh_schema_status(tenant_id, schema_status))) { - LOG_WARN("fail to get refresh schema status", KR(ret), K(tenant_id)); - } - } + schema_status.tenant_id_ = tenant_id; // use strong read if (FAILEDx(schema_service_->get_schema_version_by_timestamp( schema_status, tenant_id, expire_time, expire_schema_version))) { LOG_WARN("fail to get schema version by timestamp", diff --git a/src/rootserver/ob_service_name_command.cpp b/src/rootserver/ob_service_name_command.cpp new file mode 100644 index 000000000..5bc7a32a5 --- /dev/null +++ b/src/rootserver/ob_service_name_command.cpp @@ -0,0 +1,460 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#define USING_LOG_PREFIX RS + +#include "ob_service_name_command.h" + +#include "lib/oblog/ob_log_module.h" +#include "lib/string/ob_sql_string.h" +#include "lib/mysqlclient/ob_mysql_transaction.h" +#include "share/ob_unit_table_operator.h" +#include "share/ob_all_server_tracer.h" +#include "observer/ob_server_struct.h" +#include "rootserver/ob_rs_async_rpc_proxy.h" +#include "rootserver/ob_root_utils.h" +#include "rootserver/ob_tenant_event_def.h" + +using namespace oceanbase::common; +using namespace oceanbase::share; +using namespace oceanbase::tenant_event; +namespace oceanbase +{ +namespace rootserver +{ +int ObServiceNameKillSessionFunctor::init( + const uint64_t tenant_id, + const share::ObServiceNameString &service_name, + ObArray *killed_connection_list) +{ + int ret = OB_SUCCESS; + if (!is_valid_tenant_id(tenant_id) || !service_name.is_valid() || OB_ISNULL(killed_connection_list)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg", KR(ret), K(tenant_id), K(service_name), K(killed_connection_list)); + } else if (OB_FAIL(service_name_.assign(service_name))) { + LOG_WARN("fail to assign service_name", KR(ret), K(service_name)); + } + else { + tenant_id_ = tenant_id; + killed_connection_list_ = killed_connection_list; + killed_connection_list_->reset(); + } + return ret; +} +bool ObServiceNameKillSessionFunctor::operator()(sql::ObSQLSessionMgr::Key key, sql::ObSQLSessionInfo *sess_info) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(sess_info) || OB_ISNULL(GCTX.session_mgr_) || OB_ISNULL(killed_connection_list_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null pointer", KR(ret), KP(sess_info), KP(GCTX.session_mgr_), + KP(killed_connection_list_)); + } else if (sess_info->get_effective_tenant_id() == tenant_id_ + && sess_info->get_service_name().equal_to(service_name_)) { + uint64_t sess_id = sess_info->get_sessid(); + if (OB_FAIL(GCTX.session_mgr_->kill_session(*sess_info))) { + LOG_WARN("fail to kill session", KR(ret), K(sess_id)); + } else if (OB_FAIL(killed_connection_list_->push_back(sess_id))) { + LOG_WARN("fail to push back", KR(ret), K(sess_id)); + } + } + return OB_SUCCESS == ret; +} + +int ObServiceNameCommand::create_service( + const uint64_t tenant_id, + const ObServiceNameString &service_name_str) +{ + int ret = OB_SUCCESS; + ObArray target_servers; + ObServiceName service_name; + int64_t epoch = 0; + ObArray all_service_names; + int64_t begin_ts = ObTimeUtility::current_time(); + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || !service_name_str.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg", KR(ret), K(tenant_id), K(service_name_str)); + } else if (OB_FAIL(check_and_get_tenants_servers_(tenant_id, true /* include_temp_offline */, target_servers))) { + LOG_WARN("fail to execute check_and_get_tenants_online_servers_", KR(ret), K(tenant_id)); + } else if (OB_FAIL(ObServiceNameProxy::insert_service_name(tenant_id, service_name_str, + epoch, all_service_names))) { + // insert service_name into __all_service if the tenant's so_status is NORMAL + LOG_WARN("fail to insert service_name", KR(ret), K(tenant_id), K(service_name_str)); + } else if (OB_FAIL(extract_service_name_(all_service_names, service_name_str, service_name))) { + LOG_WARN("fail to execute extract_service_name_", KR(ret), K(all_service_names), K(service_name_str)); + if (OB_SERVICE_NAME_NOT_FOUND == ret) { + ret = OB_ERR_UNEXPECTED; + } + } else if (OB_FAIL(broadcast_refresh_( + tenant_id, + service_name.get_service_name_id(), + ObServiceNameArg::CREATE_SERVICE, + target_servers, + epoch, + all_service_names))) { + LOG_WARN("fail to broadcast", KR(ret), K(tenant_id), K(service_name), K(target_servers), + K(epoch), K(all_service_names)); + } + int64_t end_ts = ObTimeUtility::current_time(); + TENANT_EVENT(tenant_id, SERVICE_NAME, CREATE_SERVICE, end_ts, ret, end_ts - begin_ts, + service_name_str.ptr(), service_name); + return ret; +} +int ObServiceNameCommand::delete_service( + const uint64_t tenant_id, + const ObServiceNameString &service_name_str) +{ + int ret = OB_SUCCESS; + ObServiceName service_name; + int64_t begin_ts = ObTimeUtility::current_time(); + if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("GCTX.sql_proxy_ is null", KR(ret), KP(GCTX.sql_proxy_)); + } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || !service_name_str.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg", KR(ret), K(tenant_id), K(service_name_str)); + } else if (OB_FAIL(ObServiceNameProxy::select_service_name(*GCTX.sql_proxy_, tenant_id, service_name_str, service_name))) { + LOG_WARN("fail to select service_name", KR(ret), K(tenant_id), K(service_name_str)); + } else if (!service_name.is_stopped()) { + // simple check at first + // status will be checked again when the service_name is removed from the table + ret = OB_OP_NOT_ALLOW; + LOG_WARN("service_status is not STOPPED, delete_service is not allowed", KR(ret), K(service_name)); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "The service_status is not STOPPED, DELETE SERVICE is"); + } else if (OB_FAIL(ObServiceNameProxy::delete_service_name(service_name))) { + LOG_WARN("fail to delete service_name", KR(ret), K(tenant_id), K(service_name)); + } + int64_t end_ts = ObTimeUtility::current_time(); + TENANT_EVENT(tenant_id, SERVICE_NAME, DELETE_SERVICE, end_ts, ret, end_ts - begin_ts, service_name); + return ret; +} +int ObServiceNameCommand::start_service( + const uint64_t tenant_id, + const ObServiceNameString &service_name_str) +{ + int ret = OB_SUCCESS; + ObArray target_servers; + ObServiceName service_name; + ObServiceName service_name_before; + int64_t epoch = 0; + ObArray all_service_names; + int64_t begin_ts = ObTimeUtility::current_time(); + if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("GCTX.sql_proxy_ is null", KR(ret), KP(GCTX.sql_proxy_)); + } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || !service_name_str.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg", KR(ret), K(tenant_id), K(service_name_str)); + } else if (OB_FAIL(check_and_get_tenants_servers_(tenant_id, true /* include_temp_offline */, target_servers))) { + LOG_WARN("fail to execute check_and_get_tenants_servers_", KR(ret), K(tenant_id)); + } else if (OB_FAIL(ObServiceNameProxy::select_all_service_names_with_epoch(tenant_id, epoch, all_service_names))) { + LOG_WARN("fail to select service_name", KR(ret), K(tenant_id), K(service_name_str)); + } else if (OB_FAIL(extract_service_name_(all_service_names, service_name_str, service_name))) { + LOG_WARN("fail to execute extract_service_name_", KR(ret), K(all_service_names), K(service_name_str)); + } else if (OB_FAIL(service_name_before.assign(service_name))) { + LOG_WARN("fail to assign service_name_before", KR(ret), K(service_name)); + } else if (!service_name.is_started()) { + if (OB_FAIL(ObServiceNameProxy::update_service_status(service_name, ObServiceName::STARTED, + epoch, all_service_names))) { + LOG_WARN("fail to update service_status", KR(ret), K(tenant_id), K(service_name)); + } else if (OB_FAIL(extract_service_name_(all_service_names, service_name_str, service_name))) { + LOG_WARN("fail to execute extract_service_name_", KR(ret), K(all_service_names), K(service_name_str)); + if (OB_SERVICE_NAME_NOT_FOUND == ret) { + ret = OB_ERR_UNEXPECTED; + } + } + } + if (FAILEDx(broadcast_refresh_( + tenant_id, + service_name.get_service_name_id(), + ObServiceNameArg::START_SERVICE, + target_servers, + epoch, + all_service_names))) { + LOG_WARN("fail to broadcast", KR(ret), K(tenant_id), K(service_name), K(target_servers), + K(epoch), K(all_service_names)); + } + int64_t end_ts = ObTimeUtility::current_time(); + TENANT_EVENT(tenant_id, SERVICE_NAME, START_SERVICE, end_ts, ret, end_ts - begin_ts, service_name_before, service_name); + return ret; +} +int ObServiceNameCommand::stop_service( + const uint64_t tenant_id, + const ObServiceNameString &service_name_str) +{ + int ret = OB_SUCCESS; + ObArray tenant_online_servers; + ObServiceName service_name; + ObServiceName service_name_before; + int64_t epoch = 0; + ObArray all_service_names; + int64_t begin_ts = ObTimeUtility::current_time(); + if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("GCTX.sql_proxy_ is null", KR(ret), KP(GCTX.sql_proxy_)); + } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || !service_name_str.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg", KR(ret), K(tenant_id), K(service_name_str)); + } else if (OB_FAIL(ObServiceNameProxy::select_all_service_names_with_epoch(tenant_id, epoch, all_service_names))) { + LOG_WARN("fail to select service_name", KR(ret), K(tenant_id), K(service_name_str)); + } else if (OB_FAIL(extract_service_name_(all_service_names, service_name_str, service_name))) { + LOG_WARN("fail to execute extract_service_name_", KR(ret), K(all_service_names), K(service_name_str)); + } else if (OB_FAIL(service_name_before.assign(service_name))) { + LOG_WARN("fail to assign service_name_before", KR(ret), K(service_name)); + } else if (service_name.is_stopped()) { + // status has been already stopped, do nothing + } else if (OB_FAIL(check_and_get_tenants_servers_(tenant_id, false /* include_temp_offline */, tenant_online_servers))) { + // also ensure the tenant has no units on temp. offline servers + LOG_WARN("fail to execute check_and_get_tenants_online_servers_", KR(ret), K(tenant_id)); + } else { + if (service_name.is_started()) { + if (OB_FAIL(ObServiceNameProxy::update_service_status(service_name, ObServiceName::STOPPING, + epoch, all_service_names))) { + LOG_WARN("fail to update service_status", KR(ret), K(tenant_id), K(service_name)); + } else if (OB_FAIL(extract_service_name_(all_service_names, service_name_str, service_name))) { + LOG_WARN("fail to execute extract_service_name_", KR(ret), K(all_service_names), K(service_name_str)); + if (OB_SERVICE_NAME_NOT_FOUND == ret) { + ret = OB_ERR_UNEXPECTED; + } + } + } + if (FAILEDx(broadcast_refresh_( + tenant_id, + service_name.get_service_name_id(), + ObServiceNameArg::STOP_SERVICE, + tenant_online_servers, + epoch, + all_service_names))) { + LOG_WARN("fail to broadcast", KR(ret), K(tenant_id), K(service_name), K(tenant_online_servers), + K(epoch), K(all_service_names)); + } else if (OB_FAIL(ObServiceNameProxy::update_service_status(service_name, ObServiceName::STOPPED, + epoch, all_service_names))) { + LOG_WARN("fail to update service_status", KR(ret), K(tenant_id), K(service_name)); + } else if (OB_FAIL(extract_service_name_(all_service_names, service_name_str, service_name))) { + LOG_WARN("fail to execute extract_service_name_", KR(ret), K(all_service_names), K(service_name_str)); + if (OB_SERVICE_NAME_NOT_FOUND == ret) { + ret = OB_ERR_UNEXPECTED; + } + } + } + int64_t end_ts = ObTimeUtility::current_time(); + TENANT_EVENT(tenant_id, SERVICE_NAME, STOP_SERVICE, end_ts, ret, end_ts - begin_ts, service_name_before, service_name); + return ret; +} + +int ObServiceNameCommand::kill_local_connections( + const uint64_t tenant_id, + const share::ObServiceName &service_name) +{ + int ret = OB_SUCCESS; + ObArray killed_connection_list; + int64_t begin_ts = ObTimeUtility::current_time(); + ObServiceNameKillSessionFunctor kill_session_functor; + const ObServiceNameString & service_name_str = service_name.get_service_name_str(); + if (OB_ISNULL(GCTX.session_mgr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("GCTX.session_mgr_ is null", KR(ret), KP(GCTX.session_mgr_)); + } else if (OB_FAIL(kill_session_functor.init( + tenant_id, + service_name_str, + &killed_connection_list))) { + LOG_WARN("fail to init kill_session_functor", KR(ret), K(tenant_id), K(service_name_str)); + } else if (OB_FAIL(GCTX.session_mgr_->for_each_session(kill_session_functor))) { + LOG_WARN("fail to kill local sessions", KR(ret)); + } + if (killed_connection_list.count() > 0) { + int64_t end_ts = ObTimeUtility::current_time(); + TENANT_EVENT(tenant_id, SERVICE_NAME, KILL_CONNECTIONS_OF_SERVICE_NAME, end_ts, ret, end_ts - begin_ts, + service_name, killed_connection_list.count(), killed_connection_list); + } + return ret; +} + +int ObServiceNameCommand::check_and_get_tenants_servers_( + const uint64_t tenant_id, + const bool include_temp_offline, + common::ObIArray &target_servers) +{ + int ret = OB_SUCCESS; + ObArray units; + target_servers.reset(); + ObUnitTableOperator unit_operator; + if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("GCTX.sql_proxy_ is null", KR(ret), KP(GCTX.sql_proxy_)); + } else if (OB_FAIL(unit_operator.init(*GCTX.sql_proxy_))) { + LOG_WARN("failed to init unit operator", KR(ret)); + } else if (OB_FAIL(unit_operator.get_units_by_tenant(tenant_id, units))) { + LOG_WARN("failed to get tenant unit", KR(ret), K(tenant_id)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < units.count(); i++) { + const ObUnit &unit = units.at(i); + if (OB_FAIL(server_check_and_push_back_(unit.server_, include_temp_offline, target_servers))) { + LOG_WARN("fail to execute server_check_and_push_back_", KR(ret), K(unit.server_)); + } else if (unit.migrate_from_server_.is_valid() && + OB_FAIL(server_check_and_push_back_(unit.migrate_from_server_, include_temp_offline, target_servers))) { + LOG_WARN("fail to execute server_check_and_push_back_", KR(ret), K(unit.migrate_from_server_)); + } + } + return ret; +} + +int ObServiceNameCommand::server_check_and_push_back_( + const common::ObAddr &server, + const bool include_temp_offline, + common::ObIArray &target_servers) +{ + int ret = OB_SUCCESS; + ObServerInfoInTable server_info; + if (OB_UNLIKELY(!server.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid server", KR(ret), K(server)); + } else if (OB_FAIL(SVR_TRACER.get_server_info(server, server_info))) { + LOG_WARN("fail to execute get_server_info", KR(ret), K(server)); + } else if (server_info.is_permanent_offline()) { + // skip + } else if (!include_temp_offline && server_info.is_temporary_offline()) { + ret = OB_OP_NOT_ALLOW; + LOG_WARN("the tenant has units on temporary offline servers", KR(ret), K(server_info)); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "The tenant has units on temporary offline servers, STOP SERVICE is"); + } else if (OB_FAIL(target_servers.push_back(server))) { + LOG_WARN("fail to push back", KR(ret), K(server), K(server_info)); + } + return ret; +} + +int ObServiceNameCommand::broadcast_refresh_( + const uint64_t tenant_id, + const share::ObServiceNameID &target_service_name_id, + const share::ObServiceNameArg::ObServiceOp &service_op, + const common::ObIArray &target_servers, + const int64_t epoch, + const ObArray &all_service_names) +{ + // 1. Broadcasting to all servers restricts executing commands related to service_name + // when any server is permanently offline. + // 2. Users are required to ensure that processes on permanently offline servers are terminated. + // 3. Broadcasts target only online servers, following verification that no servers are temporarily offline. + int ret = OB_SUCCESS; + const ObAddr &from_server = GCTX.self_addr(); + ObTimeoutCtx ctx; + ObArray return_code_array; + obrpc::ObRefreshServiceNameArg arg; + share::ObAllTenantInfo tenant_info; + int64_t ora_rowscn = 0; + common::ObArray success_servers; + int64_t begin_ts = ObTimeUtility::current_time(); + if (OB_ISNULL(GCTX.srv_rpc_proxy_) || OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null ptr", KR(ret), KP(GCTX.srv_rpc_proxy_), KP(GCTX.sql_proxy_)); + } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { + // the validity of epoch and all_service_names will be checked when we init arg + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg", KR(ret), K(tenant_id)); + } else if (0 == target_servers.count()) { + LOG_INFO("no target servers, no need to broadcast", KR(ret), K(tenant_id), + K(all_service_names), K(target_servers)); + } else if (ObServiceNameArg::START_SERVICE == service_op && + OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id, GCTX.sql_proxy_, false, ora_rowscn, tenant_info))) { + // When starting the service, it is expected that `service_name` is utilized. + // However, the ability for users to connect via `service_name` also depends on `tenant_info`, + // so it's crucial to ensure that `tenant_info` is up-to-date. + LOG_WARN("fail to load tenant info", KR(ret), K(tenant_id)); + } else if (OB_FAIL(arg.init(tenant_id, epoch, from_server, target_service_name_id, + all_service_names, service_op, tenant_info, ora_rowscn))) { + LOG_WARN("failed to init arg", KR(ret), K(tenant_id), K(from_server), K(target_service_name_id), + K(all_service_names), K(service_op), K(tenant_info), K(ora_rowscn)); + } else { + const uint64_t group_id = share::OBCG_DBA_COMMAND; + ObRefreshServiceNameProxy proxy(*GCTX.srv_rpc_proxy_, &obrpc::ObSrvRpcProxy::refresh_service_name); + int tmp_ret = OB_SUCCESS; + // Try to send to all target servers, but return failure if at least one fails. + // The intention is to maximize the number of observers aware of the STARTED status when starting the service. + for (int64_t i = 0; OB_SUCC(ret) && i < target_servers.count(); i++) { + const ObAddr &server = target_servers.at(i); + if (OB_FAIL(ObRootUtils::get_rs_default_timeout_ctx(ctx))) { + LOG_WARN("fail to get timeout ctx", KR(ret), K(ctx)); + } else if (OB_TMP_FAIL(proxy.call(server, ctx.get_timeout(), GCONF.cluster_id, tenant_id, group_id, arg))) { + LOG_WARN("failed to send rpc", KR(ret), KR(tmp_ret), K(server), K(ctx), K(tenant_id), K(arg)); + } + } + + if (OB_TMP_FAIL(proxy.wait_all(return_code_array))) { + LOG_WARN("wait all batch result failed", KR(ret), KR(tmp_ret)); + ret = OB_SUCCESS == ret ? tmp_ret : ret; + } + int first_ret = OB_SUCCESS; + if (FAILEDx(proxy.check_return_cnt(return_code_array.count()))) { + LOG_WARN("fail to check return cnt", KR(ret), "return_cnt", return_code_array.count()); + } + ARRAY_FOREACH_X(proxy.get_results(), idx, cnt, OB_SUCC(ret)) { + const obrpc::ObRefreshServiceNameRes *result = proxy.get_results().at(idx); + const ObAddr &dest_addr = proxy.get_dests().at(idx); + tmp_ret = return_code_array.at(idx); + if (OB_TENANT_NOT_EXIST == tmp_ret) { + LOG_WARN("tenant not exist", KR(ret), KR(tmp_ret), K(dest_addr)); + tmp_ret = OB_SUCCESS; + } + if (OB_SUCCESS != tmp_ret) { + LOG_WARN("fail to send rpc", KR(ret), KR(tmp_ret), K(dest_addr), K(idx)); + } else if (OB_ISNULL(result)) { + tmp_ret = OB_ERR_UNEXPECTED; + LOG_WARN("result is null", KR(ret), KR(tmp_ret), KR(first_ret), KP(result)); + } else if (OB_UNLIKELY(!result->is_valid())) { + tmp_ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid result", + KR(ret), KR(tmp_ret), KR(first_ret), KPC(result), K(dest_addr)); + } else { + LOG_INFO("refresh_service_name success", KR(ret), KR(tmp_ret), KR(first_ret), K(dest_addr), K(arg), KPC(result)); + if (OB_FAIL(success_servers.push_back(dest_addr))) { + LOG_WARN("fail to push back", KR(ret), K(dest_addr)); + } + } + first_ret = OB_SUCCESS == first_ret ? tmp_ret : first_ret; + } + ret = OB_SUCCESS == ret ? first_ret : ret; + + if (OB_FAIL(ret)) { + int prev_ret = ret; + ret = ObServiceNameArg::STOP_SERVICE == service_op ? OB_NEED_RETRY : OB_SERVICE_NOT_FULLY_STARTED; + LOG_WARN("fail to broadcast to the tenant's all servers", KR(ret), KR(prev_ret), + K(target_servers), K(success_servers), K(service_op)); + } + int64_t end_ts = ObTimeUtility::current_time(); + TENANT_EVENT(tenant_id, SERVICE_NAME, BROADCAST_SERVICE_NAME, end_ts, ret, end_ts - begin_ts, + epoch, target_service_name_id, all_service_names, ObServiceNameArg::service_op_to_str(service_op), + target_servers, success_servers); + } + return ret; +} + +int ObServiceNameCommand::extract_service_name_( + const ObArray &all_service_names, + const share::ObServiceNameString &service_name_str, + share::ObServiceName &service_name) +{ + int ret = OB_SUCCESS; + bool is_found = false; + for (int64_t i = 0; OB_SUCC(ret) && i < all_service_names.count() && !is_found; ++i) { + if (all_service_names.at(i).get_service_name_str().equal_to(service_name_str)) { + is_found = true; + if (OB_FAIL(service_name.assign(all_service_names.at(i)))) { + LOG_WARN("fail to assign service_name", KR(ret), K(all_service_names.at(i))); + } + } + } + if (OB_SUCC(ret) && !is_found) { + ret = OB_SERVICE_NAME_NOT_FOUND; + LOG_WARN("service_name_str is not found", KR(ret), K(service_name_str), K(all_service_names)); + } + return ret; +} +} // end namespace rootserver +} // end namespace oceanbase \ No newline at end of file diff --git a/src/rootserver/ob_service_name_command.h b/src/rootserver/ob_service_name_command.h new file mode 100644 index 000000000..06db93d83 --- /dev/null +++ b/src/rootserver/ob_service_name_command.h @@ -0,0 +1,82 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_ROOTSERVER_OB_SERVICE_NAME_COMMAND_H +#define OCEANBASE_ROOTSERVER_OB_SERVICE_NAME_COMMAND_H + +#include "lib/ob_define.h" +#include "share/ob_define.h" +#include "share/ob_service_name_proxy.h" +#include "sql/session/ob_sql_session_mgr.h" +namespace oceanbase +{ +namespace rootserver +{ +class ObServiceNameKillSessionFunctor +{ +public: + ObServiceNameKillSessionFunctor() + : tenant_id_(OB_INVALID_TENANT_ID), service_name_(), killed_connection_list_(NULL) {}; + ~ObServiceNameKillSessionFunctor() {}; + int init(const uint64_t tenant_id, + const share::ObServiceNameString &service_name, + ObArray *killed_connection_list); + bool operator()(sql::ObSQLSessionMgr::Key key, sql::ObSQLSessionInfo *sess_info); +private: + uint64_t tenant_id_; + ObServiceNameString service_name_; + ObArray *killed_connection_list_; +}; +class ObServiceNameCommand +{ +public: + ObServiceNameCommand(); + ~ObServiceNameCommand(); + static int create_service( + const uint64_t tenant_id, + const share::ObServiceNameString &service_name_str); + static int delete_service( + const uint64_t tenant_id, + const share::ObServiceNameString &service_name_str); + static int start_service( + const uint64_t tenant_id, + const share::ObServiceNameString &service_name_str); + static int stop_service( + const uint64_t tenant_id, + const share::ObServiceNameString &service_name_str); + static int kill_local_connections( + const uint64_t tenant_id, + const share::ObServiceName &service_name); +private: + static int check_and_get_tenants_servers_( + const uint64_t tenant_id, + const bool include_temp_offline, + common::ObIArray &target_servers); + static int server_check_and_push_back_( + const common::ObAddr &server, + const bool include_temp_offline, + common::ObIArray &target_servers); + static int broadcast_refresh_( + const uint64_t tenant_id, + const share::ObServiceNameID &target_service_name_id, + const share::ObServiceNameArg::ObServiceOp &service_op, + const common::ObIArray &target_servers, + const int64_t epoch, + const ObArray &all_service_names); + static int extract_service_name_( + const ObArray &all_service_names, + const share::ObServiceNameString &service_name_str, + share::ObServiceName &service_name); +}; +} // end namespace rootserver +} // end namespace oceanbase +#endif \ No newline at end of file diff --git a/src/rootserver/ob_system_admin_util.cpp b/src/rootserver/ob_system_admin_util.cpp index 0f69e0b5a..d88304d8c 100644 --- a/src/rootserver/ob_system_admin_util.cpp +++ b/src/rootserver/ob_system_admin_util.cpp @@ -1435,26 +1435,36 @@ int ObAdminUpgradeVirtualSchema::execute() if (OB_UNLIKELY(!ctx_.is_inited())) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); - } else if (GCTX.is_standby_cluster()) { - // standby cluster cannot upgrade virtual schema independently, - // need to get these information from the primary cluster - ret = OB_OP_NOT_ALLOW; - LOG_WARN("upgrade virtual schema in standby cluster not allow", KR(ret)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "upgrade virtual schema in standby cluster"); - } else if (OB_ISNULL(ctx_.root_inspection_) - || OB_ISNULL(ctx_.ddl_service_)) { + } else if (OB_ISNULL(ctx_.root_inspection_) || OB_ISNULL(ctx_.ddl_service_) || OB_ISNULL(GCTX.sql_proxy_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("ptr is null", KR(ret), KP(ctx_.root_inspection_), KP(ctx_.ddl_service_)); + LOG_WARN("ptr is null", KR(ret), KP(ctx_.root_inspection_), KP(ctx_.ddl_service_), KP(GCTX.sql_proxy_)); } else if (OB_FAIL(ctx_.ddl_service_->get_tenant_schema_guard_with_version_in_inner_table( OB_SYS_TENANT_ID, schema_guard))) { LOG_WARN("get_schema_guard failed", KR(ret)); } else if (OB_FAIL(schema_guard.get_tenant_ids(tenant_ids))) { LOG_WARN("fail to get tenant ids", KR(ret)); } else { + share::ObTenantRole tenant_role; FOREACH(tenant_id, tenant_ids) { // ignore ret int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = execute(*tenant_id, upgrade_cnt))) { - LOG_WARN("fail to execute upgrade virtual table by tenant", KR(tmp_ret), K(*tenant_id)); + if (OB_TMP_FAIL(ObAllTenantInfoProxy::get_tenant_role(GCTX.sql_proxy_, *tenant_id, tenant_role))) { + LOG_WARN("fail to get tenant role", KR(ret), KP(GCTX.sql_proxy_), K(*tenant_id)); + } else if (tenant_role.is_invalid()) { + tmp_ret = OB_NEED_WAIT; + LOG_WARN("tenant role is not ready, need wait", KR(ret), K(*tenant_id), KR(tmp_ret), K(tenant_role)); + } else if (tenant_role.is_restore()) { + tmp_ret = OB_OP_NOT_ALLOW; + LOG_WARN("restore tenant cannot upgrade virtual schema", KR(ret), K(*tenant_id), KR(tmp_ret), K(tenant_role)); + } else if (tenant_role.is_standby()) { + // skip + } else if (tenant_role.is_primary()) { + if (OB_TMP_FAIL(execute(*tenant_id, upgrade_cnt))) { + LOG_WARN("fail to execute upgrade virtual table by tenant", KR(ret), K(*tenant_id), KR(tmp_ret), K(tenant_role)); + } + } else { + // Currently, clone tenant is not available, but it may be added later. + tmp_ret = OB_ERR_UNEXPECTED; + LOG_WARN("unknown tenant_role", KR(ret), K(*tenant_id), KR(tmp_ret), K(tenant_role)); } ret = OB_SUCC(ret) ? tmp_ret : ret; } diff --git a/src/rootserver/ob_tenant_event_def.h b/src/rootserver/ob_tenant_event_def.h new file mode 100644 index 000000000..d8c5e4959 --- /dev/null +++ b/src/rootserver/ob_tenant_event_def.h @@ -0,0 +1,252 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + /** + * @description: + * Use the DEF_MODULE macro in this file to define MODULE. Use the DEF_EVENT macro to define EVENT. + * Each MODULE can define multiple EVENTS of the same type. + * + * ****** NOTICE ********* + * To ensure compatibility, the already defined MODULE and EVENT cannot be modified. + * Only MODULE, EVENT, and EVENT fields can be added. + * + * DEF_MODULE(MODULE, MODULE_STR) + * @param[in] MODULE The label of MODULE + * @param[in] MODULE_STR The corresponding string of MODULE + * + * DEF_EVENT(MODULE, EVENT, EVENT_STR, NAME1 ...) + * @param[in] MODULE The label of MODULE + * @param[in] EVENT The label of EVENT + * @param[in] EVENT_STR The corresponding string of EVENT + * @param[in] NAME1 The field name of the key information of EVENT, + * up to 6 key information fields are supported + * + * Use the TENANT_EVENT macro to record the EVENT of the tenant, + * and fill in the field values in the order defined by DEF_EVENT. + * TENANT_EVENT(tenant_id, MODULE, EVENT, event_timestamp, user_ret, cost_us, VALUE1 ...) + * @param[in] tenant_id The tenant ID of the ordinary tenant + * @param[in] MODULE The label of MODULE + * @param[in] EVENT The label of EVENT + * @param[in] event_timestamp the timestamp when the event occurs + * @param[in] user_ret return code + * @param[in] cost_us cost time + * @param[in] VALUE1 The value of the field of the key information of EVENT, + * fill in the value of the field according to the order defined by DEF_EVENT + * The EVENT recorded by TENANT_EVENT will be stored in the inner table + * (__all_tenant_event_history) under the META tenant space corresponding to tenant_id, + * and displayed through the views CDB_OB_TENANT_EVENT_HISTORY/DBA_OB_TENANT_EVENT_HISTORY. + * + * ****** NOTICE ********* + * TENANT_EVENT is inserted into the table asynchronously, gmt_create is the input arg event_timestamp + * The EVENTS recorded in the inner table (__all_tenant_event_history) under the META tenant space + * will not be cleared until the tenant is deleted. + * Therefore, in order to prevent too many EVENTS, the interval between recording EVENTS should not be too frequent + */ +#ifdef DEF_MODULE +#ifdef DEF_EVENT + /** + * @description: + * Log events related to tenant role change + * failover to primary/switchover to primary/switchover to standby + */ + class TENANT_ROLE_CHANGE { + public: + DEF_MODULE(TENANT_ROLE_CHANGE, "TENANT ROLE CHANGE"); + DEF_EVENT(TENANT_ROLE_CHANGE, SWITCHOVER_TO_PRIMARY_START, "SWITCHOVER TO PRIMARY START", + STMT_STR, + TENANT_INFO); + DEF_EVENT(TENANT_ROLE_CHANGE, SWITCHOVER_TO_PRIMARY_END, "SWITCHOVER TO PRIMARY END", + STMT_STR, + TENANT_INFO, + SWITCHOVER_SCN, + COST_DETAIL, + ALL_LS); + DEF_EVENT(TENANT_ROLE_CHANGE, SWITCHOVER_TO_STANDBY_START, "SWITCHOVER TO STANDBY START", + STMT_STR, + TENANT_INFO); + DEF_EVENT(TENANT_ROLE_CHANGE, SWITCHOVER_TO_STANDBY_END, "SWITCHOVER TO STANDBY END", + STMT_STR, + TENANT_INFO, + SWITCHOVER_SCN, + COST_DETAIL, + ALL_LS); + DEF_EVENT(TENANT_ROLE_CHANGE, FAILOVER_TO_PRIMARY_START, "FAILOVER TO PRIMARY START", + STMT_STR, + TENANT_INFO); + DEF_EVENT(TENANT_ROLE_CHANGE, FAILOVER_TO_PRIMARY_END, "FAILOVER TO PRIMARY END", + STMT_STR, + TENANT_INFO, + FAILOVER_SCN, + COST_DETAIL, + ALL_LS); + DEF_EVENT(TENANT_ROLE_CHANGE, WAIT_LOG_SYNC, "WAIT LOG SYNC", + IS_SYS_LS_SYNCED, + IS_ALL_LS_SYNCED, + NON_SYNC_INFO); + }; + + class SERVICE_NAME { + public: + DEF_MODULE(SERVICE_NAME, "SERVICE NAME"); + + DEF_EVENT(SERVICE_NAME, CREATE_SERVICE, "CREATE SERVICE", + SERVICE_NAME_STRING, + CREATED_SERVICE_NAME); + + DEF_EVENT(SERVICE_NAME, DELETE_SERVICE, "DELETE SERVICE", + DELETED_SERVICE_NAME); + + DEF_EVENT(SERVICE_NAME, START_SERVICE, "START SERVICE", + SERVICE_NAME_BEFORE, + SERVICE_NAME_AFTER); + + DEF_EVENT(SERVICE_NAME, STOP_SERVICE, "STOP SERVICE", + SERVICE_NAME_BEFORE, + SERVICE_NAME_AFTER); + + DEF_EVENT(SERVICE_NAME, KILL_CONNECTIONS_OF_SERVICE_NAME, "KILL CONNECTIONS OF SERVICE NAME", + SERVICE_NAME, + KILLED_CONNECTIONS_COUNT, + KILLED_CONNECTIONS_LIST); + + DEF_EVENT(SERVICE_NAME, BROADCAST_SERVICE_NAME, "BROADCAST SERVICE NAME", + EPOCH, + TARGET_SERVICE_NAME_ID, + SERVICE_NAME_LIST, + SERVICE_NAME_COMMAND_TYPE, + TARGET_SERVERS_LIST, + SUCCESS_SERVERS_LIST); + }; +#endif +#endif +//////////////////////////////////////////////////////////////// +#ifndef _OB_TENANT_EVENT_DEF_H +#define _OB_TENANT_EVENT_DEF_H 1 +#include +#include "rootserver/ob_tenant_event_history_table_operator.h" // TENANT_EVENT_ADD +namespace oceanbase +{ +namespace tenant_event +{ +#define DEF_MODULE(MODULE, MODULE_STR) \ + static constexpr const char* const MODULE##_NAME = #MODULE; \ + static constexpr const char* const MODULE##_STR = MODULE_STR; +#define DEF_EVENT_COMMON(EVENT, EVENT_STR) \ + static constexpr const char* const EVENT##_NAME = #EVENT; \ + static constexpr const char* const EVENT##_STR = EVENT_STR; +#define OneArguments(MODULE) +#define TwoArguments(MODULE, EVENT) +#define ThreeArguments(MODULE, EVENT, EVENT_STR) \ + DEF_EVENT_COMMON(EVENT, EVENT_STR) \ + template<> \ + static void MODULE##_##EVENT##_func(const uint64_t tenant_id, const char * const module, const char * const event, \ + const int64_t event_timestamp, const int user_ret, const int64_t cost_us) \ + { \ + TENANT_EVENT_ADD(tenant_id, module, event, event_timestamp, user_ret, cost_us); \ + return ;\ + } +#define FourArguments(MODULE, EVENT, EVENT_STR, NAME1) \ + DEF_EVENT_COMMON(EVENT, EVENT_STR) \ + template \ + static void MODULE##_##EVENT##_func(const uint64_t tenant_id, const char * const module, const char * const event, \ + const int64_t event_timestamp, const int user_ret, const int64_t cost_us, \ + const T1 &value1) \ + { \ + TENANT_EVENT_ADD(tenant_id, module, event, event_timestamp, user_ret, cost_us, #NAME1, value1); \ + return ;\ + } +#define FiveArguments(MODULE, EVENT, EVENT_STR, NAME1, NAME2) \ + DEF_EVENT_COMMON(EVENT, EVENT_STR) \ + template \ + static void MODULE##_##EVENT##_func(const uint64_t tenant_id, const char * const module, const char * const event, \ + const int64_t event_timestamp, const int user_ret, const int64_t cost_us, \ + const T1 &value1, const T2 &value2) \ + { \ + TENANT_EVENT_ADD(tenant_id, module, event, event_timestamp, user_ret, cost_us, #NAME1, value1, #NAME2, value2); \ + return ;\ + } +#define SixArguments(MODULE, EVENT, EVENT_STR, NAME1, NAME2, NAME3) \ + DEF_EVENT_COMMON(EVENT, EVENT_STR) \ + template \ + static void MODULE##_##EVENT##_func(const uint64_t tenant_id, const char * const module, const char * const event, \ + const int64_t event_timestamp, const int user_ret, const int64_t cost_us, \ + const T1 &value1, const T2 &value2, \ + const T3 &value3) \ + { \ + TENANT_EVENT_ADD(tenant_id, module, event, event_timestamp, user_ret, cost_us, #NAME1, value1, #NAME2, value2, \ + #NAME3, value3); \ + return ;\ + } +#define SevenArguments(MODULE, EVENT, EVENT_STR, NAME1, NAME2, NAME3, NAME4) \ + DEF_EVENT_COMMON(EVENT, EVENT_STR) \ + template \ + static void MODULE##_##EVENT##_func(const uint64_t tenant_id, const char * const module, const char * const event, \ + const int64_t event_timestamp, const int user_ret, const int64_t cost_us, \ + const T1 &value1, const T2 &value2, \ + const T3 &value3, const T4 &value4) \ + { \ + TENANT_EVENT_ADD(tenant_id, module, event, event_timestamp, user_ret, cost_us, #NAME1, value1, #NAME2, value2, \ + #NAME3, value3, #NAME4, value4); \ + return ;\ + } +#define EightArguments(MODULE, EVENT, EVENT_STR, NAME1, NAME2, NAME3, NAME4, NAME5) \ + DEF_EVENT_COMMON(EVENT, EVENT_STR) \ + template \ + static void MODULE##_##EVENT##_func(const uint64_t tenant_id, const char * const module, const char * const event, \ + const int64_t event_timestamp, const int user_ret, const int64_t cost_us, \ + const T1 &value1, const T2 &value2, \ + const T3 &value3, const T4 &value4, \ + const T5 &value5) \ + { \ + TENANT_EVENT_ADD(tenant_id, module, event, event_timestamp, user_ret, cost_us, #NAME1, value1, #NAME2, value2, \ + #NAME3, value3, #NAME4, value4, #NAME5, value5); \ + return ;\ + } +#define NineArguments(MODULE, EVENT, EVENT_STR, NAME1, NAME2, NAME3, NAME4, NAME5, NAME6) \ + DEF_EVENT_COMMON(EVENT, EVENT_STR) \ + template \ + static void MODULE##_##EVENT##_func(const uint64_t tenant_id, const char * const module, const char * const event, \ + const int64_t event_timestamp, const int user_ret, const int64_t cost_us, \ + const T1 &value1, const T2 &value2, \ + const T3 &value3, const T4 &value4, \ + const T5 &value5, const T6 &value6) \ + { \ + TENANT_EVENT_ADD(tenant_id, module, event, event_timestamp, user_ret, cost_us, #NAME1, value1, #NAME2, value2, \ + #NAME3, value3, #NAME4, value4, #NAME5, value5, #NAME6, value6); \ + return ;\ + } + +#define GetMacro(_1, _2, _3, _4, _5, _6, _7, _8, _9, NAME, ...) NAME +#define DEF_EVENT(...) \ + GetMacro(__VA_ARGS__, NineArguments, EightArguments, SevenArguments, SixArguments, FiveArguments, FourArguments, ThreeArguments, TwoArguments, OneArgument, ...)(__VA_ARGS__) + +#define TENANT_EVENT(tenant_id, MODULE, EVENT, event_timestamp, user_ret, cost_us, args...) \ + MODULE::MODULE##_##EVENT##_func(tenant_id, MODULE::MODULE##_STR, MODULE::EVENT##_STR, event_timestamp, user_ret, cost_us, args) + +#include "ob_tenant_event_def.h" +#undef DEF_MODULE +#undef DEF_EVENT +#undef DEF_EVENT_COMMON +#undef OneArguments +#undef TwoArguments +#undef ThreeArguments +#undef FourArguments +#undef FiveArguments +#undef SixArguments +#undef SevenArguments +#undef EightArguments +#undef NineArguments +#undef GetMacro +} // end namespace tenant_event +} // end namespace oceanbase +#endif /* _OB_TENANT_EVENT_DEF_H */ \ No newline at end of file diff --git a/src/rootserver/ob_tenant_event_history_table_operator.cpp b/src/rootserver/ob_tenant_event_history_table_operator.cpp new file mode 100644 index 000000000..b33e52179 --- /dev/null +++ b/src/rootserver/ob_tenant_event_history_table_operator.cpp @@ -0,0 +1,47 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#define USING_LOG_PREFIX RS + +#include "ob_tenant_event_history_table_operator.h" + +#include "lib/oblog/ob_log_module.h" // LOG_* +namespace oceanbase +{ +namespace rootserver +{ +using namespace common; +using namespace share; +int ObTenantEventHistoryTableOperator::init(common::ObMySQLProxy &proxy, + const common::ObAddr &self_addr) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ObEventHistoryTableOperator::init(proxy))) { + LOG_WARN("fail to init event history table operator", KR(ret)); + } else { + const bool is_rs_event = false; + const bool is_server_event = false; + set_addr(self_addr, is_rs_event, is_server_event); + set_event_table(share::OB_ALL_TENANT_EVENT_HISTORY_TNAME); + } + return ret; +} +ObTenantEventHistoryTableOperator &ObTenantEventHistoryTableOperator::get_instance() +{ + static ObTenantEventHistoryTableOperator instance; + return instance; +} +int ObTenantEventHistoryTableOperator::async_delete() +{ + return OB_NOT_SUPPORTED; +} +}//end namespace rootserver +}//end namespace oceanbase \ No newline at end of file diff --git a/src/rootserver/ob_tenant_event_history_table_operator.h b/src/rootserver/ob_tenant_event_history_table_operator.h new file mode 100644 index 000000000..792ccd17d --- /dev/null +++ b/src/rootserver/ob_tenant_event_history_table_operator.h @@ -0,0 +1,36 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#ifndef OCEANBASE_ROOTSERVER_OB_TENANT_EVENT_HISTORY_TABLE_OPERATOR_H_ +#define OCEANBASE_ROOTSERVER_OB_TENANT_EVENT_HISTORY_TABLE_OPERATOR_H_ +#include "share/ob_event_history_table_operator.h" + +namespace oceanbase +{ +namespace rootserver +{ +class ObTenantEventHistoryTableOperator : public share::ObEventHistoryTableOperator +{ +public: + virtual ~ObTenantEventHistoryTableOperator() {} + int init(common::ObMySQLProxy &proxy, const common::ObAddr &self_addr); + virtual int async_delete() override; + static ObTenantEventHistoryTableOperator &get_instance(); +private: + ObTenantEventHistoryTableOperator() {} + DISALLOW_COPY_AND_ASSIGN(ObTenantEventHistoryTableOperator); +}; +} //end namespace rootserver +} //end namespace oceanbase +#define TENANT_EVENT_INSTANCE (::oceanbase::rootserver::ObTenantEventHistoryTableOperator::get_instance()) +#define TENANT_EVENT_ADD(args...) \ + TENANT_EVENT_INSTANCE.async_add_tenant_event(args) +#endif // OCEANBASE_ROOTSERVER_OB_TENANT_EVENT_HISTORY_TABLE_OPERATOR_H_ \ No newline at end of file diff --git a/src/rootserver/ob_tenant_info_loader.cpp b/src/rootserver/ob_tenant_info_loader.cpp index 631ced471..9ed9ffd00 100644 --- a/src/rootserver/ob_tenant_info_loader.cpp +++ b/src/rootserver/ob_tenant_info_loader.cpp @@ -56,6 +56,7 @@ int ObTenantInfoLoader::init() sql_proxy_ = GCTX.sql_proxy_; tenant_id_ = MTL_ID(); tenant_info_cache_.reset(); + service_names_cache_.reset(); ATOMIC_STORE(&broadcast_times_, 0); ATOMIC_STORE(&rpc_update_times_, 0); ATOMIC_STORE(&sql_update_times_, 0); @@ -65,6 +66,8 @@ int ObTenantInfoLoader::init() } else if (OB_ISNULL(GCTX.sql_proxy_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("sql proxy is null", KR(ret)); + } else if (OB_FAIL(service_names_cache_.init(tenant_id_))) { + LOG_WARN("fail to init service_name_cache_", KR(ret), K(tenant_id_)); } else if (OB_FAIL(create(thread_cnt, "TenantInf"))) { LOG_WARN("failed to create tenant info loader thread", KR(ret), K(thread_cnt)); } @@ -83,6 +86,7 @@ void ObTenantInfoLoader::destroy() is_inited_ = false; tenant_id_ = OB_INVALID_TENANT_ID; tenant_info_cache_.reset(); + service_names_cache_.reset(); sql_proxy_ = NULL; ATOMIC_STORE(&broadcast_times_, 0); ATOMIC_STORE(&rpc_update_times_, 0); @@ -148,8 +152,8 @@ void ObTenantInfoLoader::run2() const int64_t refresh_time_interval_us = act_as_standby_() && is_sys_ls_leader ? ObTenantRoleTransitionConstants::STS_TENANT_INFO_REFRESH_TIME_US : ObTenantRoleTransitionConstants::DEFAULT_TENANT_INFO_REFRESH_TIME_US; - - if (need_refresh(refresh_time_interval_us) + bool need_refresh_tenant_info = need_refresh(refresh_time_interval_us); + if (need_refresh_tenant_info && OB_FAIL(tenant_info_cache_.refresh_tenant_info(tenant_id_, sql_proxy_, content_changed))) { LOG_WARN("failed to update tenant info", KR(ret), K_(tenant_id), KP(sql_proxy_)); } @@ -168,7 +172,21 @@ void ObTenantInfoLoader::run2() if (content_changed) { (void)dump_tenant_info_(sql_update_cost_time, is_sys_ls_leader, broadcast_cost_time, end_time_us, last_dump_time_us); } - const int64_t idle_time = max(10 * 1000, refresh_time_interval_us - cost_time_us); + + // Positioned last to reduce the impact on tenant_info_cache. + int tmp_ret = OB_SUCCESS; + const int64_t start_time_us_service_name = ObTimeUtility::current_time(); + bool need_refresh_service_name = service_names_cache_.need_refresh(); + if (need_refresh_service_name + && OB_TMP_FAIL(service_names_cache_.refresh_service_name())) { + LOG_WARN("failed to refresh service_names", KR(ret), KR(tmp_ret), K_(tenant_id), KP(sql_proxy_)); + } + const int64_t cost_time_us_service_name = ObTimeUtility::current_time() - start_time_us_service_name; + + LOG_TRACE("tenant info loader cost info", KR(ret), K(need_refresh_tenant_info), + "cost_time_us_tenant_info", cost_time_us, + K(need_refresh_service_name), K(cost_time_us_service_name)); + const int64_t idle_time = max(10 * 1000, refresh_time_interval_us - cost_time_us - cost_time_us_service_name); //At least sleep 10ms, allowing the thread to release the lock if (!stop_) { get_cond().wait_us(idle_time); @@ -434,7 +452,7 @@ int ObTenantInfoLoader::get_valid_sts_after(const int64_t specified_time_us, sha ret = OB_NEED_WAIT; LOG_TRACE("sts can not work for current tenant status", KR(ret), K(tenant_info)); } else { - standby_scn = tenant_info.get_standby_scn(); + standby_scn = tenant_info.get_readable_scn(); } const int64_t PRINT_INTERVAL = 3 * 1000 * 1000L; @@ -445,6 +463,22 @@ int ObTenantInfoLoader::get_valid_sts_after(const int64_t specified_time_us, sha return ret; } +int ObTenantInfoLoader::check_if_sts_is_ready(bool &is_ready) +{ + int ret = OB_SUCCESS; + is_ready = false; + if (is_user_tenant(tenant_id_)) { + // user tenant + share::ObAllTenantInfo tenant_info; + if (OB_FAIL(tenant_info_cache_.get_tenant_info(tenant_info))) { + LOG_WARN("failed to get tenant info", KR(ret)); + } else { + is_ready = tenant_info.is_sts_ready(); + } + } + return ret; +} + int ObTenantInfoLoader::get_readable_scn(share::SCN &readable_scn) { int ret = OB_SUCCESS; @@ -495,6 +529,25 @@ int ObTenantInfoLoader::check_is_primary_normal_status(bool &is_primary_normal_s return ret; } +int ObTenantInfoLoader::check_is_prepare_flashback_for_switch_to_primary_status(bool &is_prepare) +{ + int ret = OB_SUCCESS; + is_prepare = false; + + if (OB_SYS_TENANT_ID == MTL_ID() || is_meta_tenant(MTL_ID())) { + is_prepare = false; + } else { + // user tenant + share::ObAllTenantInfo tenant_info; + if (OB_FAIL(tenant_info_cache_.get_tenant_info(tenant_info))) { + LOG_WARN("failed to get tenant info", KR(ret)); + } else { + is_prepare = tenant_info.is_prepare_flashback_for_switch_to_primary_status(); + } + } + return ret; +} + int ObTenantInfoLoader::get_global_replayable_scn(share::SCN &replayable_scn) { int ret = OB_SUCCESS; @@ -616,6 +669,30 @@ int ObTenantInfoLoader::refresh_tenant_info() return ret; } +int ObTenantInfoLoader::refresh_service_name() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (OB_FAIL(service_names_cache_.refresh_service_name())) { + LOG_WARN("failed to refresh_service_name", KR(ret), K_(tenant_id)); + } + return ret; +} + +int ObTenantInfoLoader::update_service_name(const int64_t epoch, const common::ObIArray &service_name_list) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (OB_FAIL(service_names_cache_.update_service_name(epoch, service_name_list))) { + LOG_WARN("fail to update_service_name", KR(ret), K_(tenant_id), K(service_name_list)); + } + return ret; +} + int ObTenantInfoLoader::update_tenant_info_cache(const int64_t new_ora_rowscn, const ObAllTenantInfo &new_tenant_info, const uint64_t new_finish_data_version, @@ -909,6 +986,130 @@ int ObAllTenantInfoCache::get_tenant_info(share::ObAllTenantInfo &tenant_info) } return ret; } +int ObAllServiceNamesCache::init(const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg", KR(ret), K(tenant_id)); + } else { + epoch_ = 0; + tenant_id_ = tenant_id; + all_service_names_.reset(); + last_refresh_time_ = OB_INVALID_TIMESTAMP; + ATOMIC_SET(&is_service_name_enabled_, false); + } + return ret; +} +int ObAllServiceNamesCache::refresh_service_name() +{ + int ret = OB_SUCCESS; + ObArray all_service_names; + int64_t epoch = 0; + if (!is_user_tenant(tenant_id_)) { + // do nothing + } else { + if (!ATOMIC_LOAD(&is_service_name_enabled_)) { + if (OB_FAIL(ObServiceNameProxy::check_is_service_name_enabled(tenant_id_))) { + LOG_WARN("fail to check whether service_name is enabled", KR(ret), K(tenant_id_)); + } else { + ATOMIC_SET(&is_service_name_enabled_, true); + LOG_INFO("service_name is enabled now", KR(ret), K(is_service_name_enabled_)); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(ObServiceNameProxy::select_all_service_names_with_epoch(tenant_id_, epoch, all_service_names))) { + LOG_WARN("fail to load", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(update_service_name(epoch, all_service_names))) { + LOG_WARN("fail to update service_name", KR(ret), K(all_service_names)); + } + if (dump_service_names_interval_.reach()) { + LOG_INFO("refresh service_names", KR(ret), K(tenant_id_), K(epoch_), K(all_service_names_)); + } + } + } + return ret; +} +int ObAllServiceNamesCache::update_service_name(const int64_t epoch, const common::ObIArray &service_name_list) +{ + int ret = OB_SUCCESS; + SpinWLockGuard guard(lock_); + if (epoch <= epoch_) { + // do nothing + } else { + LOG_INFO("try to update service_name", KR(ret), "local epoch", epoch_, + "local cache", all_service_names_, "new epoch", epoch, "new cache", service_name_list); + epoch_ = epoch; + all_service_names_.reset(); + if (OB_FAIL(all_service_names_.assign(service_name_list))) { + LOG_WARN("fail to assign all_service_names_", KR(ret), K(service_name_list)); + } + last_refresh_time_ = ObTimeUtility::current_time(); + } + return ret; +} +bool ObAllServiceNamesCache::need_refresh() +{ + bool need = false; + const int64_t now = ObTimeUtility::current_time(); + if (now - last_refresh_time_ >= REFRESH_INTERVAL) { + need = true; + } + return need; +} +void ObAllServiceNamesCache::reset() +{ + SpinWLockGuard guard(lock_); + all_service_names_.reset(); + last_refresh_time_ = OB_INVALID_TIMESTAMP; + ATOMIC_SET(&is_service_name_enabled_, false); + tenant_id_ = OB_INVALID_TENANT_ID; + epoch_ = 0; +} +int ObAllServiceNamesCache::check_if_the_service_name_is_stopped(const ObServiceNameString &service_name_str) const +{ + int ret = OB_SUCCESS; + bool is_found = false; + SpinRLockGuard guard(lock_); + for (int64_t i = 0; OB_SUCC(ret) && !is_found && i < all_service_names_.size(); i++) { + if (all_service_names_.at(i).get_service_name_str().equal_to(service_name_str)) { + const ObServiceName & service_name = all_service_names_.at(i); + is_found = true; + if (service_name.is_stopped() || service_name.is_stopping()) { + ret = OB_SERVICE_STOPPED; + LOG_WARN("service_status is stopped", KR(ret), K(service_name), K(epoch_), K(all_service_names_)); + } + } + } + if (OB_SUCC(ret) && !is_found) { + ret = OB_SERVICE_NAME_NOT_FOUND; + LOG_WARN("service_name_str is not found", KR(ret), K(service_name_str), K(epoch_), K(all_service_names_)); + } + return ret; +} + +int ObAllServiceNamesCache::get_service_name( + const ObServiceNameID &service_name_id, + ObServiceName &service_name) const +{ + int ret = OB_SUCCESS; + bool is_found = false; + SpinRLockGuard guard(lock_); + for (int64_t i = 0; OB_SUCC(ret) && !is_found && i < all_service_names_.size(); ++i) { + const ObServiceName & tmp_service_name = all_service_names_.at(i); + if (service_name_id == tmp_service_name.get_service_name_id()) { + is_found = true; + if (OB_FAIL(service_name.assign(tmp_service_name))) { + LOG_WARN("fail to assign service_name", KR(ret), K(tmp_service_name)); + } + } + } + if (OB_SUCC(ret) && !is_found) { + ret = OB_SERVICE_NAME_NOT_FOUND; + LOG_WARN("fail to find service_name", KR(ret), K(service_name_id), K(all_service_names_)); + } + return ret; +} } } diff --git a/src/rootserver/ob_tenant_info_loader.h b/src/rootserver/ob_tenant_info_loader.h index a27bc9448..da75220f6 100644 --- a/src/rootserver/ob_tenant_info_loader.h +++ b/src/rootserver/ob_tenant_info_loader.h @@ -16,8 +16,9 @@ #include "lib/thread/ob_reentrant_thread.h"//ObRsReentrantThread #include "lib/utility/ob_print_utils.h" //TO_STRING_KV #include "share/ob_tenant_info_proxy.h"//ObAllTenantInfo +#include "share/ob_service_name_proxy.h"//ObServiceName #include "lib/lock/ob_spin_rwlock.h" //lock -#include "rootserver/ob_tenant_role_transition_service.h"//ObTenantRoleTransitionConstants +#include "rootserver/standby/ob_tenant_role_transition_service.h"//ObTenantRoleTransitionConstants namespace oceanbase { namespace common @@ -87,6 +88,38 @@ private: DISALLOW_COPY_AND_ASSIGN(ObAllTenantInfoCache); }; +class ObAllServiceNamesCache +{ +public: + ObAllServiceNamesCache() + : lock_(), + tenant_id_(OB_INVALID_TENANT_ID), + epoch_(0), + all_service_names_(), + dump_service_names_interval_(DUMP_SERVICE_NAMES_INTERVAL), + last_refresh_time_(OB_INVALID_TIMESTAMP), + is_service_name_enabled_(false) {} + ~ObAllServiceNamesCache() {} + int init(const uint64_t tenant_id); + int refresh_service_name(); + int update_service_name(const int64_t epoch, const common::ObIArray &service_name_list); + bool need_refresh(); + int check_if_the_service_name_is_stopped(const ObServiceNameString &service_name_str) const; + int get_service_name(const ObServiceNameID &service_name_id, ObServiceName &service_name) const; + void reset(); +private: + static constexpr int64_t DUMP_SERVICE_NAMES_INTERVAL = 5 * 1000L * 1000L; // 5s + static constexpr int64_t REFRESH_INTERVAL = 2 * 1000L * 1000L; // 2s + common::SpinRWLock lock_; + uint64_t tenant_id_; + int64_t epoch_; + ObArray all_service_names_; + common::ObTimeInterval dump_service_names_interval_; + int64_t last_refresh_time_; + bool is_service_name_enabled_; + DISALLOW_COPY_AND_ASSIGN(ObAllServiceNamesCache); +}; + /*description: * Periodically cache tenant info.*/ class ObTenantInfoLoader : public share::ObReentrantThread @@ -102,7 +135,8 @@ public: rpc_update_times_(0), sql_update_times_(0), last_rpc_update_time_us_(OB_INVALID_TIMESTAMP), - dump_tenant_info_cache_update_action_interval_(DUMP_TENANT_INFO_CACHE_UPDATE_ACTION_INTERVAL) {} + dump_tenant_info_cache_update_action_interval_(DUMP_TENANT_INFO_CACHE_UPDATE_ACTION_INTERVAL), + service_names_cache_() {} ~ObTenantInfoLoader() {} static int mtl_init(ObTenantInfoLoader *&ka); int init(); @@ -127,6 +161,7 @@ public: * 3. sts can not work for current tenant status */ int get_valid_sts_after(const int64_t specified_time_us, share::SCN &standby_scn); + int check_if_sts_is_ready(bool &is_ready); /** * @description: * get tenant standby scn. @@ -200,13 +235,35 @@ public: * @param[out] is_primary_normal_status */ int check_is_primary_normal_status(bool &is_primary_normal_status); + int check_is_prepare_flashback_for_switch_to_primary_status(bool &is_prepare); int refresh_tenant_info(); + int refresh_service_name(); + int update_service_name(const int64_t epoch, const common::ObIArray &service_name_list); int update_tenant_info_cache(const int64_t new_ora_rowscn, const ObAllTenantInfo &new_tenant_info, const uint64_t new_finish_data_version, const share::SCN &new_data_version_barrier_scn); bool need_refresh(const int64_t refresh_time_interval_us); int get_max_ls_id(uint64_t &tenant_id, ObLSID &max_ls_id); + /** + * @description: + * check if service_status of the given service_name is STOPPED or STOPPING + * @param[in] service_name_str service_name string + * @return return code + * OB_SERVICE_NAME_NOT_FOUND service_name is not found, cannot check its service_status + * OB_SERVICE_STOPPED service_status is STOPPED or STOPPING + * OB_SUCCESS service_name exists and its service_status is STARTED + * others + */ + int check_if_the_service_name_is_stopped(const ObServiceNameString &service_name_str) const + { + return service_names_cache_.check_if_the_service_name_is_stopped(service_name_str); + } + bool get_service_name(const ObServiceNameID &service_name_id, ObServiceName &service_name) const + { + return service_names_cache_.get_service_name(service_name_id, service_name); + } + protected: /** @@ -245,6 +302,7 @@ private: uint64_t sql_update_times_; int64_t last_rpc_update_time_us_; common::ObTimeInterval dump_tenant_info_cache_update_action_interval_; + ObAllServiceNamesCache service_names_cache_; private: DISALLOW_COPY_AND_ASSIGN(ObTenantInfoLoader); }; diff --git a/src/rootserver/ob_transfer_partition_command.cpp b/src/rootserver/ob_transfer_partition_command.cpp index 14b4c6e14..e0b1a5828 100644 --- a/src/rootserver/ob_transfer_partition_command.cpp +++ b/src/rootserver/ob_transfer_partition_command.cpp @@ -19,7 +19,7 @@ #include "share/balance/ob_balance_job_table_operator.h"//ObBalanceJobTableOperator #include "share/transfer/ob_transfer_task_operator.h"//ObTransferTask #include "share/ob_tenant_info_proxy.h" -#include "share/ob_primary_standby_service.h" +#include "rootserver/standby/ob_standby_service.h" #include "observer/omt/ob_tenant_config_mgr.h" // ObTenantConfigGuard #include "share/ls/ob_ls_i_life_manager.h"//START/END_TRANSACTION #include "storage/tablelock/ob_lock_utils.h"//table_lock @@ -258,7 +258,7 @@ int ObTransferPartitionCommand::check_tenant_status_(const uint64_t tenant_id) } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(tenant_id)); - } else if (OB_FAIL(OB_PRIMARY_STANDBY_SERVICE.get_tenant_status(tenant_id, tenant_status))) { + } else if (OB_FAIL(OB_STANDBY_SERVICE.get_tenant_status(tenant_id, tenant_status))) { LOG_WARN("fail to get tenant status", KR(ret), K(tenant_id)); } else if (OB_UNLIKELY(!is_tenant_normal(tenant_status))) { ret = OB_OP_NOT_ALLOW; diff --git a/src/rootserver/ob_upgrade_executor.cpp b/src/rootserver/ob_upgrade_executor.cpp index 5b0b487b7..ed17489d5 100644 --- a/src/rootserver/ob_upgrade_executor.cpp +++ b/src/rootserver/ob_upgrade_executor.cpp @@ -18,7 +18,7 @@ #include "observer/ob_server_struct.h" #include "share/ob_global_stat_proxy.h" #include "share/ob_cluster_event_history_table_operator.h"//CLUSTER_EVENT_INSTANCE -#include "share/ob_primary_standby_service.h" // ObPrimaryStandbyService +#include "rootserver/standby/ob_standby_service.h" // ObStandbyService #include "share/ob_tenant_info_proxy.h" //ObAllTenantInfoProxy #include "observer/ob_service.h" @@ -650,7 +650,7 @@ int ObUpgradeExecutor::run_upgrade_begin_action_( LOG_WARN("fail to update target data version", KR(ret), K(tenant_id), "version", DVP(DATA_CURRENT_VERSION)); } else if (is_user_tenant(tenant_id) - && OB_FAIL(OB_PRIMARY_STANDBY_SERVICE.write_upgrade_barrier_log( + && OB_FAIL(OB_STANDBY_SERVICE.write_upgrade_barrier_log( trans, tenant_id, DATA_CURRENT_VERSION))) { LOG_WARN("fail to write_upgrade_barrier_log", KR(ret), K(tenant_id), "version", DVP(DATA_CURRENT_VERSION)); @@ -1125,7 +1125,7 @@ int ObUpgradeExecutor::update_final_current_data_version_(const uint64_t tenant_ if (OB_FAIL(end_proxy.update_current_data_version(version))) { LOG_WARN("fail to update current data version", KR(ret), K(tenant_id), KDV(version)); } else if (is_user_tenant(tenant_id) && - OB_FAIL(OB_PRIMARY_STANDBY_SERVICE.write_upgrade_data_version_barrier_log( + OB_FAIL(OB_STANDBY_SERVICE.write_upgrade_data_version_barrier_log( trans, tenant_id, version))) { LOG_WARN("fail to write_upgrade_data_version_barrier_log", KR(ret), K(tenant_id), KDV(version)); } diff --git a/src/rootserver/restore/ob_clone_scheduler.cpp b/src/rootserver/restore/ob_clone_scheduler.cpp index 0cb6daad3..cda05680d 100644 --- a/src/rootserver/restore/ob_clone_scheduler.cpp +++ b/src/rootserver/restore/ob_clone_scheduler.cpp @@ -1409,9 +1409,9 @@ int ObCloneScheduler::check_sys_tenant_(const uint64_t tenant_id) } else if (OB_ISNULL(GCTX.schema_service_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null schema service", KR(ret), KP(GCTX.schema_service_)); - } else if (GCTX.is_standby_cluster() || GCONF.in_upgrade_mode()) { + } else if (GCONF.in_upgrade_mode()) { ret = OB_OP_NOT_ALLOW; - LOG_WARN("clone tenant while in standby cluster or in upgrade mode is not allowed", KR(ret)); + LOG_WARN("clone tenant while in upgrade mode is not allowed", KR(ret)); } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard( OB_SYS_TENANT_ID, schema_guard))) { LOG_WARN("fail to get schema guard", KR(ret)); diff --git a/src/rootserver/restore/ob_recover_table_job_scheduler.cpp b/src/rootserver/restore/ob_recover_table_job_scheduler.cpp index 986664079..2ea8ce50a 100644 --- a/src/rootserver/restore/ob_recover_table_job_scheduler.cpp +++ b/src/rootserver/restore/ob_recover_table_job_scheduler.cpp @@ -17,7 +17,7 @@ #include "rootserver/restore/ob_recover_table_initiator.h" #include "rootserver/restore/ob_restore_service.h" #include "share/backup/ob_backup_data_table_operator.h" -#include "share/ob_primary_standby_service.h" +#include "rootserver/standby/ob_standby_service.h" #include "share/location_cache/ob_location_service.h" #include "share/restore/ob_physical_restore_table_operator.h" #include "share/restore/ob_import_util.h" @@ -652,7 +652,7 @@ int ObRecoverTableJobScheduler::failover_to_primary_( MTL_SWITCH(OB_SYS_TENANT_ID) { if (OB_FAIL(switch_tenant_arg.init(aux_tenant_id, obrpc::ObSwitchTenantArg::OpType::FAILOVER_TO_PRIMARY, "", false))) { LOG_WARN("failed to init switch tenant arg", K(ret), K(aux_tenant_id)); - } else if (OB_FAIL(OB_PRIMARY_STANDBY_SERVICE.switch_tenant(switch_tenant_arg))) { + } else if (OB_FAIL(OB_STANDBY_SERVICE.switch_tenant(switch_tenant_arg))) { LOG_WARN("failed to switch_tenant", KR(ret), K(switch_tenant_arg)); } else { LOG_INFO("[RECOVER_TABLE]succeed to switch aux tenant role to primary", K(aux_tenant_id), K(job)); diff --git a/src/rootserver/restore/ob_restore_common_util.cpp b/src/rootserver/restore/ob_restore_common_util.cpp index afd482188..947030003 100644 --- a/src/rootserver/restore/ob_restore_common_util.cpp +++ b/src/rootserver/restore/ob_restore_common_util.cpp @@ -16,7 +16,7 @@ #include "share/ls/ob_ls_status_operator.h" //ObLSStatusOperator #include "share/ls/ob_ls_operator.h"//ObLSAttr #include "rootserver/ob_ls_service_helper.h" -#include "rootserver/ob_tenant_role_transition_service.h" +#include "rootserver/standby/ob_tenant_role_transition_service.h" #include "src/share/ob_schema_status_proxy.h" #include "src/share/ob_rpc_struct.h" #include "rootserver/ob_ddl_service.h" @@ -204,6 +204,9 @@ int ObRestoreCommonUtil::try_update_tenant_role(common::ObMySQLProxy *sql_proxy, ObAllTenantInfo all_tenant_info; int64_t new_switch_ts = 0; bool need_update = false; + ObTenantRoleTransitionService role_transition_service; + ObTenantRoleTransCostDetail cost_detail; + ObTenantRoleTransAllLSInfo all_ls; if (OB_UNLIKELY(!is_user_tenant(tenant_id) || OB_ISNULL(sql_proxy) @@ -224,16 +227,24 @@ int ObRestoreCommonUtil::try_update_tenant_role(common::ObMySQLProxy *sql_proxy, //update tenant role to standby tenant if (all_tenant_info.get_sync_scn() != restore_scn) { sync_satisfied = false; - LOG_WARN("tenant sync scn not equal to restore scn", KR(ret), - K(all_tenant_info), K(restore_scn)); + LOG_WARN("tenant sync scn not equal to restore scn", KR(ret), K(all_tenant_info), K(restore_scn)); } else if (OB_FAIL(ObAllTenantInfoProxy::update_tenant_role( tenant_id, sql_proxy, all_tenant_info.get_switchover_epoch(), share::STANDBY_TENANT_ROLE, all_tenant_info.get_switchover_status(), share::NORMAL_SWITCHOVER_STATUS, new_switch_ts))) { LOG_WARN("failed to update tenant role", KR(ret), K(tenant_id), K(all_tenant_info)); + } else if (OB_FAIL(all_ls.init())) { + LOG_WARN("fail to init all_ls", KR(ret)); + } else if (OB_FAIL(role_transition_service.init( + tenant_id, + ObSwitchTenantArg::OpType::INVALID, + false, /* is_verify */ + sql_proxy, + GCTX.srv_rpc_proxy_, + &cost_detail, + &all_ls))) { + LOG_WARN("fail to init role_transition_service", KR(ret), K(tenant_id), KP(sql_proxy), KP(GCTX.srv_rpc_proxy_)); } else { - ObTenantRoleTransitionService role_transition_service(tenant_id, sql_proxy, - GCTX.srv_rpc_proxy_, obrpc::ObSwitchTenantArg::OpType::INVALID); (void)role_transition_service.broadcast_tenant_info( ObTenantRoleTransitionConstants::RESTORE_TO_STANDBY_LOG_MOD_STR); } diff --git a/src/rootserver/restore/ob_restore_scheduler.cpp b/src/rootserver/restore/ob_restore_scheduler.cpp index ab1b3397b..7dcbe7331 100644 --- a/src/rootserver/restore/ob_restore_scheduler.cpp +++ b/src/rootserver/restore/ob_restore_scheduler.cpp @@ -19,7 +19,7 @@ #include "rootserver/ob_unit_manager.h"//convert_pool_name_lis #include "rootserver/ob_ls_service_helper.h"//create_new_ls_in_trans #include "rootserver/ob_common_ls_service.h"//do_create_user_ls -#include "rootserver/ob_tenant_role_transition_service.h" +#include "rootserver/standby/ob_tenant_role_transition_service.h" #include "share/ob_schema_status_proxy.h" #include "share/schema/ob_schema_utils.h" #include "share/schema/ob_schema_mgr.h" @@ -34,7 +34,7 @@ #include "share/restore/ob_log_restore_source_mgr.h" #include "share/ls/ob_ls_recovery_stat_operator.h"//ObLSRecoveryStatOperator #include "share/ob_rpc_struct.h" -#include "share/ob_primary_standby_service.h" +#include "rootserver/standby/ob_standby_service.h" #include "logservice/palf/log_define.h"//scn #include "share/scn.h" #include "ob_restore_service.h" @@ -1281,7 +1281,7 @@ int ObRestoreScheduler::check_tenant_replay_to_consistent_scn(const uint64_t ten ret = OB_INVALID_ARGUMENT; LOG_WARN("unexpected recovery until scn", K(ret), K(tenant_info), K(scn)); } else { - is_replay_finish = (tenant_info.get_recovery_until_scn() <= tenant_info.get_standby_scn()); + is_replay_finish = (tenant_info.get_recovery_until_scn() <= tenant_info.get_readable_scn()); LOG_INFO("[RESTORE]tenant replay to consistent_scn", K(is_replay_finish)); } return ret; diff --git a/src/rootserver/ob_recovery_ls_service.cpp b/src/rootserver/standby/ob_recovery_ls_service.cpp similarity index 99% rename from src/rootserver/ob_recovery_ls_service.cpp rename to src/rootserver/standby/ob_recovery_ls_service.cpp index f7467b33f..98782473e 100755 --- a/src/rootserver/ob_recovery_ls_service.cpp +++ b/src/rootserver/standby/ob_recovery_ls_service.cpp @@ -40,7 +40,7 @@ #include "share/ob_errno.h" #include "share/ob_share_util.h" //ObShareUtil #include "share/schema/ob_multi_version_schema_service.h" //ObMultiSchemaService -#include "share/ob_primary_standby_service.h" // ObPrimaryStandbyService +#include "rootserver/standby/ob_standby_service.h" // ObStandbyService #include "share/ob_standby_upgrade.h" // ObStandbyUpgrade #include "share/ob_upgrade_utils.h" // ObUpgradeChecker #include "share/ob_global_stat_proxy.h" // ObGlobalStatProxy @@ -51,6 +51,7 @@ #include "share/ob_log_restore_proxy.h" // ObLogRestoreProxyUtil #include "share/ob_occam_time_guard.h"//ObTimeGuard #include "src/rootserver/ob_rs_event_history_table_operator.h" +#include "rootserver/tenant_snapshot/ob_tenant_snapshot_util.h" // ObTenantSnapshotUtil namespace oceanbase { @@ -1285,7 +1286,7 @@ int ObRecoveryLSService::do_ls_balance_task_() } else if (OB_FAIL(tenant_info_loader->get_tenant_info(tenant_info))) { LOG_WARN("get_tenant_info failed", K(ret)); } else if (OB_FAIL(ObBalanceTaskHelperTableOperator::load_tasks_order_by_scn( - tenant_id_, *proxy_, tenant_info.get_standby_scn(), + tenant_id_, *proxy_, tenant_info.get_readable_scn(), ls_balance_tasks))) { if (OB_ENTRY_NOT_EXIST == ret) { ret = OB_SUCCESS; @@ -1411,7 +1412,7 @@ int ObRecoveryLSService::check_transfer_begin_can_remove_( //check tenant_info status and check wait readable_scn is equal to sync_scn ret = OB_SUCCESS; transfer_scn = tenant_info.get_sync_scn(); - if (tenant_info.get_sync_scn() != tenant_info.get_standby_scn()) { + if (tenant_info.get_sync_scn() != tenant_info.get_readable_scn()) { can_remove = false; LOG_WARN("There are transfer tasks in progress. Must wait for replay to newest", KR(ret), K(tenant_id_), K(tenant_info), K(ls_balance_task)); diff --git a/src/rootserver/ob_recovery_ls_service.h b/src/rootserver/standby/ob_recovery_ls_service.h similarity index 99% rename from src/rootserver/ob_recovery_ls_service.h rename to src/rootserver/standby/ob_recovery_ls_service.h index dac7f3cb4..657bdb5a3 100755 --- a/src/rootserver/ob_recovery_ls_service.h +++ b/src/rootserver/standby/ob_recovery_ls_service.h @@ -17,7 +17,7 @@ #include "logservice/palf/lsn.h"//palf::LSN #include "logservice/palf/palf_iterator.h" //PalfBufferIterator #include "logservice/restoreservice/ob_log_restore_handler.h"//RestoreStatusInfo -#include "ob_primary_ls_service.h" //ObTenantThreadHelper +#include "rootserver/ob_primary_ls_service.h" //ObTenantThreadHelper #include "lib/lock/ob_spin_lock.h" //ObSpinLock #include "storage/tx/ob_multi_data_source.h" //ObTxBufferNode diff --git a/src/share/ob_primary_standby_service.cpp b/src/rootserver/standby/ob_standby_service.cpp similarity index 76% rename from src/share/ob_primary_standby_service.cpp rename to src/rootserver/standby/ob_standby_service.cpp index b4628c828..7562847ae 100644 --- a/src/share/ob_primary_standby_service.cpp +++ b/src/rootserver/standby/ob_standby_service.cpp @@ -12,12 +12,13 @@ #define USING_LOG_PREFIX STANDBY -#include "ob_primary_standby_service.h" // ObPrimaryStandbyService +#include "ob_standby_service.h" // ObStandbyService + #include "lib/oblog/ob_log_module.h" // LOG_* #include "lib/utility/ob_print_utils.h" // TO_STRING_KV #include "rootserver/ob_cluster_event.h" // CLUSTER_EVENT_ADD_CONTROL +#include "rootserver/ob_tenant_event_def.h" // TENANT_EVENT #include "rootserver/ob_rs_event_history_table_operator.h" // ROOTSERVICE_EVENT_ADD -#include "rootserver/ob_tenant_role_transition_service.h" // ObTenantRoleTransitionService #include "rootserver/ob_ls_service_helper.h"//ObTenantLSInfo #include "share/restore/ob_log_restore_source_mgr.h" // ObLogRestoreSourceMgr #include "share/ls/ob_ls_recovery_stat_operator.h"// ObLSRecoveryStatOperator @@ -42,11 +43,11 @@ using namespace obrpc; using namespace share; using namespace rootserver; using namespace storage; - +using namespace tenant_event; namespace standby { -int ObPrimaryStandbyService::init( +int ObStandbyService::init( ObMySQLProxy *sql_proxy, share::schema::ObMultiVersionSchemaService *schema_service) { @@ -63,20 +64,20 @@ int ObPrimaryStandbyService::init( return ret; } -void ObPrimaryStandbyService::destroy() +void ObStandbyService::destroy() { if (OB_UNLIKELY(!inited_)) { - LOG_INFO("ObPrimaryStandbyService has been destroyed", K_(inited)); + LOG_INFO("ObStandbyService has been destroyed", K_(inited)); } else { - LOG_INFO("ObPrimaryStandbyService begin to destroy", K_(inited)); + LOG_INFO("ObStandbyService begin to destroy", K_(inited)); sql_proxy_ = NULL; schema_service_ = NULL; inited_ = false; - LOG_INFO("ObPrimaryStandbyService destroyed", K_(inited)); + LOG_INFO("ObStandbyService destroyed", K_(inited)); } } -int ObPrimaryStandbyService::check_inner_stat_() +int ObStandbyService::check_inner_stat_() { int ret = OB_SUCCESS; if (OB_UNLIKELY(!inited_)) { @@ -88,117 +89,209 @@ int ObPrimaryStandbyService::check_inner_stat_() } return ret; } +#define PRINT_TENANT_INFO(tenant_info, tenant_info_buf) \ + do { \ + int64_t pos = 0; \ + size_t tenant_buf_size = sizeof(tenant_info_buf) / sizeof(tenant_info_buf[0]); \ + if ((tenant_info).is_valid()) { \ + (void)databuff_printf(tenant_info_buf, tenant_buf_size, pos, "%s", to_cstring((tenant_info))); \ + } else { \ + (void)databuff_printf(tenant_info_buf, tenant_buf_size, pos, "NULL"); \ + } \ + } while(0) -int ObPrimaryStandbyService::switch_tenant(const obrpc::ObSwitchTenantArg &arg) +void ObStandbyService::tenant_event_start_( + const uint64_t switch_tenant_id, const obrpc::ObSwitchTenantArg &arg, int ret, + int64_t begin_ts, const share::ObAllTenantInfo &tenant_info) +{ + char tenant_info_buf[1024] = ""; + PRINT_TENANT_INFO(tenant_info, tenant_info_buf); + switch (arg.get_op_type()) { + case ObSwitchTenantArg::SWITCH_TO_PRIMARY : + TENANT_EVENT(switch_tenant_id, TENANT_ROLE_CHANGE, SWITCHOVER_TO_PRIMARY_START, begin_ts, + ret, 0, ObHexEscapeSqlStr(arg.get_stmt_str()), ObHexEscapeSqlStr(tenant_info_buf)); + break; + case ObSwitchTenantArg::SWITCH_TO_STANDBY : + TENANT_EVENT(switch_tenant_id, TENANT_ROLE_CHANGE, SWITCHOVER_TO_STANDBY_START, begin_ts, + ret, 0, ObHexEscapeSqlStr(arg.get_stmt_str()), ObHexEscapeSqlStr(tenant_info_buf)); + break; + case ObSwitchTenantArg::FAILOVER_TO_PRIMARY : + TENANT_EVENT(switch_tenant_id, TENANT_ROLE_CHANGE, FAILOVER_TO_PRIMARY_START, begin_ts, + ret, 0, ObHexEscapeSqlStr(arg.get_stmt_str()), ObHexEscapeSqlStr(tenant_info_buf)); + break; + default :break; + } +} + +void ObStandbyService::tenant_event_end_( + const uint64_t switch_tenant_id, const obrpc::ObSwitchTenantArg &arg, + int ret, int64_t cost, int64_t end_ts, const share::SCN switch_scn, + ObTenantRoleTransCostDetail &cost_detail, ObTenantRoleTransAllLSInfo &all_ls) +{ + share::ObAllTenantInfo tenant_info; + if (!THIS_WORKER.is_timeout()) { + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(ObAllTenantInfoProxy::load_tenant_info( + switch_tenant_id, + sql_proxy_, + false, + tenant_info))) { + LOG_WARN("failed to load tenant info", KR(ret), K(switch_tenant_id)); + } + } + char tenant_info_buf[1024] = ""; + PRINT_TENANT_INFO(tenant_info, tenant_info_buf); + switch (arg.get_op_type()) { + case ObSwitchTenantArg::SWITCH_TO_PRIMARY : + TENANT_EVENT(switch_tenant_id, TENANT_ROLE_CHANGE, SWITCHOVER_TO_PRIMARY_END, end_ts, + ret, cost, ObHexEscapeSqlStr(arg.get_stmt_str()), ObHexEscapeSqlStr(tenant_info_buf), switch_scn.get_val_for_inner_table_field(), cost_detail, all_ls); + break; + case ObSwitchTenantArg::SWITCH_TO_STANDBY : + TENANT_EVENT(switch_tenant_id, TENANT_ROLE_CHANGE, SWITCHOVER_TO_STANDBY_END, end_ts, + ret, cost, ObHexEscapeSqlStr(arg.get_stmt_str()), ObHexEscapeSqlStr(tenant_info_buf), switch_scn.get_val_for_inner_table_field(), cost_detail, all_ls); + break; + case ObSwitchTenantArg::FAILOVER_TO_PRIMARY : + TENANT_EVENT(switch_tenant_id, TENANT_ROLE_CHANGE, FAILOVER_TO_PRIMARY_END, end_ts, + ret, cost, ObHexEscapeSqlStr(arg.get_stmt_str()), ObHexEscapeSqlStr(tenant_info_buf), switch_scn.get_val_for_inner_table_field(), cost_detail, all_ls); + break; + default :break; + } +} + +int ObStandbyService::switch_tenant(const obrpc::ObSwitchTenantArg &arg) { int ret = OB_SUCCESS; - int64_t begin_time = ObTimeUtility::current_time(); + int64_t begin_ts = ObTimeUtility::current_time(); uint64_t switch_tenant_id = OB_INVALID_ID; - const char *alter_cluster_event = arg.get_alter_type_str(); - ObTenantStatus tenant_status = TENANT_STATUS_MAX; + bool is_verify = arg.get_is_verify(); uint64_t compat_version = 0; - CLUSTER_EVENT_ADD_CONTROL_START(ret, alter_cluster_event, "stmt_str", arg.get_stmt_str()); + ObAllTenantInfo tenant_info; + share::SCN switch_scn = SCN::min_scn(); + ObTenantRoleTransCostDetail cost_detail; + ObTenantRoleTransAllLSInfo all_ls; + cost_detail.set_start(begin_ts); if (OB_FAIL(check_inner_stat_())) { LOG_WARN("inner stat error", KR(ret), K_(inited)); - } else if (!arg.is_valid()) { + } else if (OB_UNLIKELY(!arg.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arg", K(arg), KR(ret)); } else if (OB_FAIL(get_target_tenant_id(arg.get_tenant_name(), arg.get_exec_tenant_id(), switch_tenant_id))) { LOG_WARN("failed to get_target_tenant_id", KR(ret), K(switch_tenant_id), K(arg)); } else if (OB_FAIL(GET_MIN_DATA_VERSION(switch_tenant_id, compat_version))) { - LOG_WARN("fail to get data version", KR(ret), K(switch_tenant_id)); - } else if (compat_version < DATA_VERSION_4_1_0_0) { + LOG_WARN("fail to get data version", K(ret), K(arg)); + } else if (OB_UNLIKELY(compat_version < DATA_VERSION_4_1_0_0)) { ret = OB_NOT_SUPPORTED; LOG_WARN("Tenant COMPATIBLE is below 4.1.0.0, switch tenant is not supported", KR(ret)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "Tenant COMPATIBLE is below 4.1.0.0, switch tenant is"); - } else if (OB_FAIL(get_tenant_status(switch_tenant_id, tenant_status))) { - LOG_WARN("failed to get tenant status", KR(ret), K(switch_tenant_id)); - } else if (is_tenant_normal(tenant_status)) { + TENANT_ROLE_TRANS_USER_ERR_WITH_SUFFIX(OB_NOT_SUPPORTED, "Tenant COMPATIBLE is below 4.1.0.0", arg.get_op_type()); + } else if (OB_UNLIKELY(is_verify && !(compat_version >= DATA_VERSION_4_3_3_0 + || (compat_version >= DATA_VERSION_4_2_2_0 && compat_version < DATA_VERSION_4_3_0_0) + || (compat_version >= MOCK_DATA_VERSION_4_2_1_8 && compat_version < DATA_VERSION_4_2_2_0)))) { + ret = common::OB_NOT_SUPPORTED; + LOG_WARN("only (version >= 4_2_1_8 and version < 4_2_2_0) " + "or version >= 4_2_2_0 and version < 4_3_0_0 " + "or version >= 4_3_3_0 support this operation", KR(ret), K(compat_version)); + } else if (OB_FAIL(check_if_tenant_status_is_normal_(switch_tenant_id, arg.get_op_type()))) { + LOG_WARN("fail to check if tenant status is normal", KR(ret), K(switch_tenant_id), K(arg)); + } else if (OB_FAIL(all_ls.init())) { + LOG_WARN("fail to init all_ls", KR(ret)); + } else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info( + switch_tenant_id, + sql_proxy_, + false, + tenant_info))) { + LOG_WARN("failed to load tenant info", KR(ret), K(switch_tenant_id)); + } else { + if (!is_verify) { + (void) tenant_event_start_(switch_tenant_id, arg, ret, begin_ts, tenant_info); + } + switch (arg.get_op_type()) { case ObSwitchTenantArg::SWITCH_TO_PRIMARY : - if (OB_FAIL(switch_to_primary(switch_tenant_id, arg.get_op_type()))) { - LOG_WARN("failed to switch_to_primary", KR(ret), K(switch_tenant_id), K(arg), K(tenant_status)); + if (OB_FAIL(switch_to_primary(switch_tenant_id, arg.get_op_type(), is_verify, + switch_scn, cost_detail, all_ls))) { + LOG_WARN("failed to switch_to_primary", KR(ret), K(switch_tenant_id), K(arg)); } break; case ObSwitchTenantArg::SWITCH_TO_STANDBY : - if (OB_FAIL(switch_to_standby(switch_tenant_id, arg.get_op_type()))) { - LOG_WARN("failed to switch_to_standby", KR(ret), K(switch_tenant_id), K(arg), K(tenant_status)); + if (OB_FAIL(switch_to_standby(switch_tenant_id, arg.get_op_type(), is_verify, + tenant_info, switch_scn, cost_detail, all_ls))) { + LOG_WARN("failed to switch_to_standby", KR(ret), K(switch_tenant_id), K(arg)); } break; case ObSwitchTenantArg::FAILOVER_TO_PRIMARY : - if (OB_FAIL(failover_to_primary(switch_tenant_id, arg.get_op_type()))) { - LOG_WARN("failed to failover_to_primary", KR(ret), K(switch_tenant_id), K(arg), K(tenant_status)); + if (OB_FAIL(failover_to_primary(switch_tenant_id, arg.get_op_type(), is_verify, + tenant_info, switch_scn, cost_detail, all_ls))) { + LOG_WARN("failed to failover_to_primary", KR(ret), K(switch_tenant_id), K(arg)); } break; default : ret = OB_INVALID_ARGUMENT; - LOG_WARN("unkown op_type", K(arg)); + LOG_WARN("unkown op_type", KR(ret), K(arg)); + } + // reset return code to TIMEOUT, to prevent the error code which not user unfriendly + if (THIS_WORKER.is_timeout() && OB_ERR_EXCLUSIVE_LOCK_CONFLICT == ret) { + ret = OB_TIMEOUT; + } + int64_t end_ts = ObTimeUtility::current_time(); + int64_t cost = end_ts - begin_ts; + cost_detail.set_end(end_ts); + FLOG_INFO("switch tenant end", KR(ret), K(arg), K(cost), K(cost_detail), K(all_ls)); + if (!is_verify) { + (void) tenant_event_end_(switch_tenant_id, arg, ret, cost, end_ts, switch_scn, cost_detail, all_ls); } - } else { - ret = OB_OP_NOT_ALLOW; - LOG_WARN("tenant status is not normal, switch tenant is not allowed", KR(ret), K(switch_tenant_id), K(arg), K(tenant_status)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "tenant status is not normal, switch tenant is"); } - - // reset return code to TIMEOUT, to prevent the error code which not user unfriendly - if (THIS_WORKER.is_timeout() && OB_ERR_EXCLUSIVE_LOCK_CONFLICT == ret) { - ret = OB_TIMEOUT; - } - - int64_t cost = ObTimeUtility::current_time() - begin_time; - CLUSTER_EVENT_ADD_CONTROL_FINISH(ret, alter_cluster_event, - K(cost), - "stmt_str", arg.get_stmt_str()); - return ret; } -int ObPrimaryStandbyService::failover_to_primary(const uint64_t tenant_id, - const obrpc::ObSwitchTenantArg::OpType &switch_optype) +int ObStandbyService::failover_to_primary( + const uint64_t tenant_id, + const obrpc::ObSwitchTenantArg::OpType &switch_optype, + const bool is_verify, + const share::ObAllTenantInfo &tenant_info, + share::SCN &switch_scn, + ObTenantRoleTransCostDetail &cost_detail, + ObTenantRoleTransAllLSInfo &all_ls) { int ret = OB_SUCCESS; - ObAllTenantInfo tenant_info; - ObTenantStatus tenant_status = TENANT_STATUS_MAX; + ObTenantRoleTransitionService role_transition_service; if (OB_FAIL(check_inner_stat_())) { LOG_WARN("inner stat error", KR(ret), K_(inited)); - } else if (OB_ISNULL(GCTX.srv_rpc_proxy_) || OB_ISNULL(schema_service_)) { + } else if (OB_ISNULL(GCTX.srv_rpc_proxy_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("pointer is null", KR(ret), KP(GCTX.srv_rpc_proxy_), KP(schema_service_)); + LOG_WARN("pointer is null", KR(ret), KP(GCTX.srv_rpc_proxy_)); } else if (OB_UNLIKELY(obrpc::ObSwitchTenantArg::OpType::INVALID == switch_optype)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid switch_optype", KR(ret), K(switch_optype)); - } else if (!is_user_tenant(tenant_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("only support switch user tenant", KR(ret), K(tenant_id)); - LOG_USER_ERROR(OB_INVALID_ARGUMENT, "tenant id, only support operating user tenant"); - } else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id, sql_proxy_, - false, tenant_info))) { - LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id)); + } else if (OB_FAIL(role_transition_service.init( + tenant_id, + switch_optype, + is_verify, + sql_proxy_, + GCTX.srv_rpc_proxy_, + &cost_detail, + &all_ls))) { + LOG_WARN("fail to init role_transition_service", KR(ret), K(tenant_id), K(switch_optype), + KP(sql_proxy_), KP(GCTX.srv_rpc_proxy_), K(cost_detail), K(all_ls)); } else if (tenant_info.is_primary() && tenant_info.is_normal_status()) { LOG_INFO("already is primary tenant, no need switch", K(tenant_info)); - } else if (OB_FAIL(get_tenant_status(tenant_id, tenant_status))) { - LOG_WARN("failed to get tenant status", KR(ret), K(tenant_id)); - } else if (is_tenant_normal(tenant_status)) { - ObTenantRoleTransitionService role_transition_service(tenant_id, sql_proxy_, GCTX.srv_rpc_proxy_, switch_optype); - if (tenant_info.get_restore_data_mode().is_remote_mode()) { - ret = OB_OP_NOT_ALLOW; - LOG_WARN("tenant restore data mode is remote, failover is not allowed", KR(ret), K(tenant_id), K(tenant_info)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "Tenant restore data mode is remote. Operation is"); - } else if (OB_FAIL(role_transition_service.failover_to_primary())) { - LOG_WARN("failed to failover to primary", KR(ret), K(tenant_id)); - } - } else { + } else if (tenant_info.get_restore_data_mode().is_remote_mode()) { ret = OB_OP_NOT_ALLOW; - LOG_WARN("tenant status is not normal, failover is not allowed", KR(ret), K(tenant_id), K(tenant_status)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "tenant status is not normal, failover is"); + LOG_WARN("tenant restore data mode is remote, failover is not allowed", KR(ret), K(tenant_id), K(tenant_info)); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "Tenant restore data mode is remote. Operation is"); + } else { + if (OB_FAIL(role_transition_service.failover_to_primary())) { + LOG_WARN("failed to failover to primary", KR(ret), K(tenant_id)); + } + switch_scn = tenant_info.get_sync_scn(); } return ret; } -int ObPrimaryStandbyService::get_target_tenant_id(const ObString &tenant_name, - const uint64_t exec_tenant_id, - uint64_t &switch_tenant_id) +int ObStandbyService::get_target_tenant_id( + const ObString &tenant_name, + const uint64_t exec_tenant_id, + uint64_t &switch_tenant_id) { int ret = OB_SUCCESS; switch_tenant_id = OB_INVALID_ID; @@ -229,7 +322,7 @@ int ObPrimaryStandbyService::get_target_tenant_id(const ObString &tenant_name, LOG_WARN("get_schema_guard failed", KR(ret)); } else if (OB_FAIL(guard.get_tenant_id(tenant_name, switch_tenant_id))) { LOG_WARN("get_tenant_id failed", KR(ret), K(tenant_name), K(exec_tenant_id)); - } else if (!is_user_tenant(switch_tenant_id)) { + } else if (OB_UNLIKELY(!is_valid_tenant_id(switch_tenant_id) || !is_user_tenant(switch_tenant_id))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("only support switch user tenant", KR(ret), K(tenant_name), K(exec_tenant_id), K(switch_tenant_id)); LOG_USER_ERROR(OB_INVALID_ARGUMENT, "tenant name, only support operating user tenant"); @@ -240,7 +333,7 @@ int ObPrimaryStandbyService::get_target_tenant_id(const ObString &tenant_name, return ret; } -int ObPrimaryStandbyService::recover_tenant(const obrpc::ObRecoverTenantArg &arg) +int ObStandbyService::recover_tenant(const obrpc::ObRecoverTenantArg &arg) { int ret = OB_SUCCESS; int64_t begin_time = ObTimeUtility::current_time(); @@ -274,21 +367,19 @@ int ObPrimaryStandbyService::recover_tenant(const obrpc::ObRecoverTenantArg &arg return ret; } -int ObPrimaryStandbyService::get_tenant_status( +int ObStandbyService::get_tenant_status( const uint64_t tenant_id, ObTenantStatus &status) { int ret = OB_SUCCESS; status = TENANT_STATUS_MAX; - if (OB_FAIL(check_inner_stat_())) { - LOG_WARN("inner stat error", KR(ret), K_(inited)); - } else if (!is_user_tenant(tenant_id)) { + if (!is_user_tenant(tenant_id)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("only support get user tenant status", KR(ret), K(tenant_id)); LOG_USER_ERROR(OB_INVALID_ARGUMENT, "tenant id, only support operating user tenant"); - } else if (OB_ISNULL(sql_proxy_)) { + } else if (OB_ISNULL(GCTX.sql_proxy_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("pointer is null", KR(ret), KP(sql_proxy_)); + LOG_WARN("pointer is null", KR(ret), KP(GCTX.sql_proxy_)); } else { ObSqlString sql; SMART_VAR(ObISQLClient::ReadResult, result) { @@ -296,7 +387,7 @@ int ObPrimaryStandbyService::get_tenant_status( "SELECT status FROM %s WHERE tenant_id = %lu", OB_ALL_TENANT_TNAME, tenant_id))) { LOG_WARN("assign sql string failed", KR(ret), K(tenant_id)); - } else if (OB_FAIL(sql_proxy_->read(result, OB_SYS_TENANT_ID, sql.ptr()))) { + } else if (OB_FAIL(GCTX.sql_proxy_->read(result, OB_SYS_TENANT_ID, sql.ptr()))) { LOG_WARN("execute sql failed", KR(ret), K(tenant_id), K(sql)); } else if (OB_ISNULL(result.get_result())) { ret = OB_ERR_UNEXPECTED; @@ -328,7 +419,21 @@ int ObPrimaryStandbyService::get_tenant_status( return ret; } -int ObPrimaryStandbyService::do_recover_tenant( +int ObStandbyService::check_if_tenant_status_is_normal_(const uint64_t tenant_id, const RoleTransType op_type) +{ + int ret = OB_SUCCESS; + ObTenantStatus tenant_status = TENANT_STATUS_MAX; + if (OB_FAIL(get_tenant_status(tenant_id, tenant_status))) { + LOG_WARN("failed to get tenant status", KR(ret), K(tenant_id)); + } else if (OB_UNLIKELY(!is_tenant_normal(tenant_status))) { + ret = OB_OP_NOT_ALLOW; + LOG_WARN("tenant status is not normal", KR(ret), K(tenant_id), K(tenant_status)); + TENANT_ROLE_TRANS_USER_ERR_WITH_SUFFIX(OB_OP_NOT_ALLOW, "tenant status is not normal", op_type); + } + return ret; +} + +int ObStandbyService::do_recover_tenant( const uint64_t tenant_id, const share::ObTenantSwitchoverStatus &working_sw_status, const obrpc::ObRecoverTenantArg::RecoverType &recover_type, @@ -417,12 +522,17 @@ int ObPrimaryStandbyService::do_recover_tenant( return ret; } -int ObPrimaryStandbyService::switch_to_primary( +int ObStandbyService::switch_to_primary( const uint64_t tenant_id, - const obrpc::ObSwitchTenantArg::OpType &switch_optype) + const obrpc::ObSwitchTenantArg::OpType &switch_optype, + const bool is_verify, + share::SCN &switch_scn, + ObTenantRoleTransCostDetail &cost_detail, + ObTenantRoleTransAllLSInfo &all_ls) { int ret = OB_SUCCESS; int64_t begin_time = ObTimeUtility::current_time(); + ObTenantRoleTransitionService role_transition_service; ObAllTenantInfo tenant_info; if (OB_FAIL(check_inner_stat_())) { LOG_WARN("inner stat error", KR(ret), K_(inited)); @@ -432,29 +542,37 @@ int ObPrimaryStandbyService::switch_to_primary( } else if (OB_UNLIKELY(obrpc::ObSwitchTenantArg::OpType::INVALID == switch_optype)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid switch_optype", KR(ret), K(switch_optype)); - } else if (!is_user_tenant(tenant_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("only support switch user tenant", KR(ret), K(tenant_id)); - LOG_USER_ERROR(OB_INVALID_ARGUMENT, "tenant id, only support operating user tenant"); + } else if (OB_FAIL(role_transition_service.init( + tenant_id, + switch_optype, + is_verify, + sql_proxy_, + GCTX.srv_rpc_proxy_, + &cost_detail, + &all_ls))) { + LOG_WARN("fail to init role_transition_service", KR(ret), K(tenant_id), K(switch_optype), + KP(sql_proxy_), KP(GCTX.srv_rpc_proxy_), K(cost_detail), K(all_ls)); } else { - ObTenantRoleTransitionService role_transition_service(tenant_id, sql_proxy_, GCTX.srv_rpc_proxy_, switch_optype); (void)role_transition_service.set_switchover_epoch(tenant_info.get_switchover_epoch()); if (OB_FAIL(role_transition_service.failover_to_primary())) { - LOG_WARN("failed to failover to primary", KR(ret), K(tenant_id)); + LOG_WARN("fail to failover to primary", KR(ret), K(tenant_id)); } + switch_scn = role_transition_service.get_so_scn(); } - return ret; } -int ObPrimaryStandbyService::switch_to_standby( +int ObStandbyService::switch_to_standby( const uint64_t tenant_id, - const obrpc::ObSwitchTenantArg::OpType &switch_optype) + const obrpc::ObSwitchTenantArg::OpType &switch_optype, + const bool is_verify, + share::ObAllTenantInfo &tenant_info, + share::SCN &switch_scn, + ObTenantRoleTransCostDetail &cost_detail, + ObTenantRoleTransAllLSInfo &all_ls) { int ret = OB_SUCCESS; - ObAllTenantInfo tenant_info; const int32_t group_id = share::OBCG_DBA_COMMAND; - if (OB_FAIL(check_inner_stat_())) { LOG_WARN("inner stat error", KR(ret), K_(inited)); } else if (OB_ISNULL(GCTX.srv_rpc_proxy_)) { @@ -463,12 +581,6 @@ int ObPrimaryStandbyService::switch_to_standby( } else if (OB_UNLIKELY(obrpc::ObSwitchTenantArg::OpType::INVALID == switch_optype)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid switch_optype", KR(ret), K(switch_optype)); - } else if (!is_user_tenant(tenant_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("only support switch user tenant", KR(ret), K(tenant_id)); - LOG_USER_ERROR(OB_INVALID_ARGUMENT, "tenant id, only support operating user tenant"); - } else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id, sql_proxy_, false, tenant_info))) { - LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id)); } else if (tenant_info.is_standby() && tenant_info.is_normal_status()) { LOG_INFO("already is standby tenant, no need switch", K(tenant_id), K(tenant_info)); } else { @@ -479,6 +591,12 @@ int ObPrimaryStandbyService::switch_to_standby( ret = OB_OP_NOT_ALLOW; LOG_WARN("unexpected tenant role", KR(ret), K(tenant_info)); LOG_USER_ERROR(OB_OP_NOT_ALLOW, "tenant role is not PRIMARY, switchover to standby is"); + } else if (OB_UNLIKELY(!tenant_info.get_recovery_until_scn().is_max())) { + ret = OB_OP_NOT_ALLOW; + LOG_WARN("recovery_until_scn has been changed ", KR(ret), K(tenant_id), K(tenant_info)); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "recovery_until_scn has been changed, switchover to standby is"); + } else if (is_verify) { + // skip } else if (OB_FAIL(update_tenant_status_before_sw_to_standby_( tenant_info.get_switchover_status(), tenant_info.get_tenant_role(), @@ -490,7 +608,7 @@ int ObPrimaryStandbyService::switch_to_standby( } } case share::ObTenantSwitchoverStatus::PREPARE_SWITCHING_TO_STANDBY_STATUS: { - if (OB_FAIL(ret)) { + if (OB_FAIL(ret) || is_verify) { } else if (OB_FAIL(switch_to_standby_prepare_ls_status_(tenant_id, tenant_info.get_switchover_status(), tenant_info.get_switchover_epoch(), @@ -499,9 +617,19 @@ int ObPrimaryStandbyService::switch_to_standby( } } case share::ObTenantSwitchoverStatus::SWITCHING_TO_STANDBY_STATUS: { - if (OB_FAIL(ret)) { + ObTenantRoleTransitionService role_transition_service; + if (OB_FAIL(ret) || is_verify) { + } else if (OB_FAIL(role_transition_service.init( + tenant_id, + switch_optype, + is_verify, + sql_proxy_, + GCTX.srv_rpc_proxy_, + &cost_detail, + &all_ls))) { + LOG_WARN("fail to init role_transition_service", KR(ret), K(tenant_id), K(switch_optype), + KP(sql_proxy_), KP(GCTX.srv_rpc_proxy_), K(cost_detail), K(all_ls)); } else { - ObTenantRoleTransitionService role_transition_service(tenant_id, sql_proxy_, GCTX.srv_rpc_proxy_, switch_optype); uint64_t compat_version = 0; ObGlobalStatProxy global_proxy(*sql_proxy_, gen_meta_tenant_id(tenant_id)); (void)role_transition_service.set_switchover_epoch(tenant_info.get_switchover_epoch()); @@ -527,13 +655,14 @@ int ObPrimaryStandbyService::switch_to_standby( (void)role_transition_service.broadcast_tenant_info( ObTenantRoleTransitionConstants::SWITCH_TO_STANDBY_LOG_MOD_STR); } + switch_scn = role_transition_service.get_so_scn(); } break; } default: { ret = OB_OP_NOT_ALLOW; LOG_WARN("switchover status not match", KR(ret), K(tenant_info), K(tenant_id)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "switchover status not match, switchover to standby"); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "switchover status not match, switchover to standby is"); break; } } @@ -542,7 +671,7 @@ int ObPrimaryStandbyService::switch_to_standby( return ret; } -int ObPrimaryStandbyService::update_tenant_status_before_sw_to_standby_( +int ObStandbyService::update_tenant_status_before_sw_to_standby_( const ObTenantSwitchoverStatus cur_switchover_status, const ObTenantRole cur_tenant_role, const int64_t cur_switchover_epoch, @@ -571,7 +700,7 @@ int ObPrimaryStandbyService::update_tenant_status_before_sw_to_standby_( } else if (OB_UNLIKELY(!tenant_info.get_recovery_until_scn().is_max())) { ret = OB_OP_NOT_ALLOW; LOG_WARN("recovery_until_scn has been changed ", KR(ret), K(tenant_id), K(tenant_info)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "recovery_until_scn has been changed, switchover to standby"); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "recovery_until_scn has been changed, switchover to standby is"); } else if (cur_switchover_status != tenant_info.get_switchover_status()) { ret = OB_NEED_RETRY; LOG_WARN("tenant not expect switchover status", KR(ret), K(tenant_info), K(cur_switchover_status)); @@ -599,16 +728,15 @@ int ObPrimaryStandbyService::update_tenant_status_before_sw_to_standby_( ret = OB_SUCC(ret) ? temp_ret : ret; } } - CLUSTER_EVENT_ADD_LOG(ret, "update tenant before switchover to standby", - "tenant id", tenant_id, - "old switchover#", cur_switchover_epoch, - "new switchover#", tenant_info.get_switchover_epoch(), - K(cur_switchover_status), K(cur_tenant_role)); + "tenant id", tenant_id, + "old switchover#", cur_switchover_epoch, + "new switchover#", tenant_info.get_switchover_epoch(), + K(cur_switchover_status), K(cur_tenant_role)); return ret; } -int ObPrimaryStandbyService::switch_to_standby_prepare_ls_status_( +int ObStandbyService::switch_to_standby_prepare_ls_status_( const uint64_t tenant_id, const ObTenantSwitchoverStatus &status, const int64_t switchover_epoch, @@ -631,7 +759,7 @@ int ObPrimaryStandbyService::switch_to_standby_prepare_ls_status_( } else if (OB_UNLIKELY(!status.is_prepare_switching_to_standby_status())) { ret = OB_OP_NOT_ALLOW; LOG_WARN("switchover status not match, switchover to standby not allow", KR(ret), K(status)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "switchover status not match, switchover to standby"); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "switchover status not match, switchover to standby is"); } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, schema_guard))) { LOG_WARN("fail to get schema guard", KR(ret)); } else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id, tenant_schema))) { @@ -668,7 +796,7 @@ int ObPrimaryStandbyService::switch_to_standby_prepare_ls_status_( return ret; } -int ObPrimaryStandbyService::write_upgrade_barrier_log( +int ObStandbyService::write_upgrade_barrier_log( ObMySQLTransaction &trans, const uint64_t tenant_id, const uint64_t data_version) @@ -683,7 +811,7 @@ int ObPrimaryStandbyService::write_upgrade_barrier_log( return ret; } -int ObPrimaryStandbyService::write_upgrade_data_version_barrier_log( +int ObStandbyService::write_upgrade_data_version_barrier_log( ObMySQLTransaction &trans, const uint64_t tenant_id, const uint64_t data_version) @@ -700,7 +828,7 @@ int ObPrimaryStandbyService::write_upgrade_data_version_barrier_log( return ret; } -int ObPrimaryStandbyService::write_barrier_log_( +int ObStandbyService::write_barrier_log_( const transaction::ObTxDataSourceType type, ObMySQLTransaction &trans, const uint64_t tenant_id, @@ -744,7 +872,7 @@ int ObPrimaryStandbyService::write_barrier_log_( return ret; } -int ObPrimaryStandbyService::check_can_create_standby_tenant( +int ObStandbyService::check_can_create_standby_tenant( const common::ObString &log_restore_source, ObCompatibilityMode &compat_mode) { @@ -772,7 +900,7 @@ int ObPrimaryStandbyService::check_can_create_standby_tenant( return ret; } -int ObPrimaryStandbyService::wait_create_standby_tenant_end(const uint64_t tenant_id) +int ObStandbyService::wait_create_standby_tenant_end(const uint64_t tenant_id) { int ret = OB_SUCCESS; int64_t start_ts = ObTimeUtility::current_time(); @@ -860,7 +988,7 @@ int ObPrimaryStandbyService::wait_create_standby_tenant_end(const uint64_t tenan return ret; } -int ObPrimaryStandbyService::check_ls_restore_status_(const uint64_t tenant_id) +int ObStandbyService::check_ls_restore_status_(const uint64_t tenant_id) { int ret = OB_SUCCESS; ObMySQLProxy *sql_proxy = GCTX.sql_proxy_; diff --git a/src/share/ob_primary_standby_service.h b/src/rootserver/standby/ob_standby_service.h similarity index 69% rename from src/share/ob_primary_standby_service.h rename to src/rootserver/standby/ob_standby_service.h index 0eecf792b..582998607 100644 --- a/src/share/ob_primary_standby_service.h +++ b/src/rootserver/standby/ob_standby_service.h @@ -10,20 +10,46 @@ * See the Mulan PubL v2 for more details. */ -#ifndef OCEANBASE_STANDBY_OB_PRIMARY_STANDBY_SERVICE_H_ -#define OCEANBASE_STANDBY_OB_PRIMARY_STANDBY_SERVICE_H_ +#ifndef OCEANBASE_STANDBY_OB_STANDBY_SERVICE_H_ +#define OCEANBASE_STANDBY_OB_STANDBY_SERVICE_H_ #include "share/ob_rpc_struct.h" // ObAdminClusterArg #include "share/ob_rs_mgr.h" // ObRsMgr #include "lib/mysqlclient/ob_isql_client.h" // ObISQLClient #include "rootserver/ob_ddl_service.h" // ObDDLService #include "share/schema/ob_multi_version_schema_service.h" // ObMultiVersionSchemaService +#include "rootserver/standby/ob_tenant_role_transition_service.h" // ObTenantRoleTransitionService +// usage: TENANT_ROLE_TRANS_USER_ERR_WITH_SUFFIX(OB_OP_NOT_ALLOW, "tenant status is not normal") +// the output to user will be "tenant status is not normal, switchover to primary is not allowed" +#define TENANT_ROLE_TRANS_USER_ERR_WITH_SUFFIX(TRT_ERR_RET, TRT_ERR_MSG, TRT_OP) \ +({ \ + int tmp_ret = OB_SUCCESS; \ + ObSqlString err_msg; \ + if (OB_TMP_FAIL(err_msg.append_fmt(TRT_ERR_MSG))) { \ + LOG_WARN("fail to assign error message", KR(tmp_ret)); \ + } else { \ + if (obrpc::ObSwitchTenantArg::OpType::SWITCH_TO_PRIMARY == TRT_OP) { \ + tmp_ret = err_msg.append_fmt(", switchover to primary is"); \ + } else if (obrpc::ObSwitchTenantArg::OpType::SWITCH_TO_STANDBY == TRT_OP) { \ + tmp_ret = err_msg.append_fmt(", switchover to standby is"); \ + } else if (obrpc::ObSwitchTenantArg::OpType::FAILOVER_TO_PRIMARY == TRT_OP) { \ + tmp_ret = err_msg.append_fmt(", failover to primary is"); \ + } else { \ + tmp_ret = err_msg.append_fmt(", this operation is"); \ + } \ + if (OB_SUCCESS != tmp_ret) { \ + LOG_WARN("fail to assign error message", KR(tmp_ret)); \ + } else { \ + LOG_USER_ERROR(TRT_ERR_RET, err_msg.ptr()); \ + } \ + } \ +}) namespace oceanbase { using namespace share; - +using namespace rootserver; namespace share { namespace schema @@ -35,15 +61,15 @@ class ObMultiVersionSchemaService; namespace standby { -class ObPrimaryStandbyService +class ObStandbyService { public: - ObPrimaryStandbyService(): + ObStandbyService(): sql_proxy_(NULL), schema_service_(NULL), inited_(false) {} - virtual ~ObPrimaryStandbyService() {} - + virtual ~ObStandbyService() {} + typedef obrpc::ObSwitchTenantArg::OpType RoleTransType; int init(ObMySQLProxy *sql_proxy, share::schema::ObMultiVersionSchemaService *schema_service); void destroy(); @@ -122,7 +148,14 @@ private: * @param[in] arg tenant switch arguments * @return return code */ - int failover_to_primary(const uint64_t tenant_id, const obrpc::ObSwitchTenantArg::OpType &switch_optype); + int failover_to_primary( + const uint64_t tenant_id, + const obrpc::ObSwitchTenantArg::OpType &switch_optype, + const bool is_verify, + const share::ObAllTenantInfo &tenant_info, + share::SCN &switch_scn, + ObTenantRoleTransCostDetail &cost_detail, + ObTenantRoleTransAllLSInfo &all_ls); /** * @description: @@ -141,7 +174,13 @@ private: * @param[in] arg tenant switch arguments which include primary tenant switchover checkpoint * @return return code */ - int switch_to_primary(const uint64_t tenant_id, const obrpc::ObSwitchTenantArg::OpType &switch_optype); + int switch_to_primary( + const uint64_t tenant_id, + const obrpc::ObSwitchTenantArg::OpType &switch_optype, + const bool is_verify, + share::SCN &switch_scn, + ObTenantRoleTransCostDetail &cost_detail, + ObTenantRoleTransAllLSInfo &all_ls); /** * @description: @@ -149,7 +188,14 @@ private: * @param[in] tenant_id the primary tenant id to switch * @return return code */ - int switch_to_standby(const uint64_t tenant_id, const obrpc::ObSwitchTenantArg::OpType &switch_optype); + int switch_to_standby( + const uint64_t tenant_id, + const obrpc::ObSwitchTenantArg::OpType &switch_optype, + const bool is_verify, + share::ObAllTenantInfo &tenant_info, + share::SCN &switch_scn, + ObTenantRoleTransCostDetail &cost_detail, + ObTenantRoleTransAllLSInfo &all_ls); /** * @description: @@ -217,6 +263,12 @@ private: const uint64_t tenant_id, const uint64_t data_version); + int check_if_tenant_status_is_normal_(const uint64_t tenant_id, const RoleTransType op_type); + void tenant_event_start_(const uint64_t switch_tenant_id, const obrpc::ObSwitchTenantArg &arg, + int ret, int64_t begin_ts, const share::ObAllTenantInfo &tenant_info); + void tenant_event_end_(const uint64_t switch_tenant_id, const obrpc::ObSwitchTenantArg &arg, + int ret, int64_t cost, int64_t end_ts, const share::SCN switch_scn, + ObTenantRoleTransCostDetail &cost_detail, ObTenantRoleTransAllLSInfo &all_ls); private: const static int64_t SEC_UNIT = 1000L * 1000L; const static int64_t PRINT_INTERVAL = 10 * 1000 * 1000L; @@ -226,19 +278,19 @@ private: bool inited_; }; -class ObPrimaryStandbyServiceGetter +class ObStandbyServiceGetter { public: - static ObPrimaryStandbyService &get_instance() + static ObStandbyService &get_instance() { - static ObPrimaryStandbyService primary_standby_service; - return primary_standby_service; + static ObStandbyService standby_service; + return standby_service; } }; -#define OB_PRIMARY_STANDBY_SERVICE (oceanbase::standby::ObPrimaryStandbyServiceGetter::get_instance()) +#define OB_STANDBY_SERVICE (oceanbase::standby::ObStandbyServiceGetter::get_instance()) } // end namespace standby } // end namespace oceanbase -#endif // OCEANBASE_STANDBY_OB_PRIMARY_STANDBY_SERVICE_H_ +#endif // OCEANBASE_STANDBY_OB_STANDBY_SERVICE_H_ diff --git a/src/rootserver/ob_tenant_role_transition_service.cpp b/src/rootserver/standby/ob_tenant_role_transition_service.cpp similarity index 73% rename from src/rootserver/ob_tenant_role_transition_service.cpp rename to src/rootserver/standby/ob_tenant_role_transition_service.cpp index c02f5a794..bbc119963 100644 --- a/src/rootserver/ob_tenant_role_transition_service.cpp +++ b/src/rootserver/standby/ob_tenant_role_transition_service.cpp @@ -18,8 +18,10 @@ #include "rootserver/ob_rs_async_rpc_proxy.h"//ObChangeLSAccessModeProxy #include "lib/oblog/ob_log_module.h"// LOG_* #include "lib/utility/ob_print_utils.h"// TO_STRING_KV +#include "share/ls/ob_ls_operator.h" // ls_status_to_str #include "rootserver/ob_cluster_event.h"// CLUSTER_EVENT_ADD_CONTROL #include "rootserver/ob_rs_event_history_table_operator.h" // ROOTSERVICE_EVENT_ADD +#include "rootserver/ob_tenant_event_def.h" // TENANT_EVENT #include "rootserver/ob_ls_service_helper.h" // ObLSServiceHelper #include "rootserver/ob_empty_server_checker.h" // ObEmptyServerChecker #include "share/ob_rpc_struct.h"//ObLSAccessModeInfo @@ -28,20 +30,29 @@ #include "share/ob_global_stat_proxy.h"//ObGlobalStatProxy #include "share/ob_schema_status_proxy.h"//set_schema_status #include "storage/tx/ob_timestamp_service.h" // ObTimestampService -#include "share/ob_primary_standby_service.h" // ObPrimaryStandbyService +#include "rootserver/standby/ob_standby_service.h" // ObStandbyService #include "share/balance/ob_balance_task_helper_operator.h"//ObBalanceTaskHelper #include "lib/utility/ob_macro_utils.h" #include "lib/ob_errno.h" #include "share/oracle_errno.h"//oracle error code +#include "rootserver/ob_service_name_command.h" namespace oceanbase { using namespace share; using namespace palf; +using namespace common; +using namespace tenant_event; + namespace rootserver { - -#define TENANT_ROLE_TRANS_USER_ERROR \ +/* + The macro's usage scenario: It is used in the process of switchover to primary, + to translate the connection information and certain error checking messages + from the source primary tenant into USER ERROR, + facilitating the troubleshooting of cross-tenant connection issues. +*/ +#define SOURCE_TENANT_CHECK_USER_ERROR_FOR_SWITCHOVER_TO_PRIMARY \ int tmp_ret = OB_SUCCESS; \ ObSqlString str; \ switch (ret) { \ @@ -57,53 +68,208 @@ namespace rootserver case -ER_ACCESS_DENIED_ERROR: \ case OB_ERR_NO_LOGIN_PRIVILEGE: \ case -OER_INTERNAL_ERROR_CODE: \ - if (OB_TMP_FAIL(str.assign_fmt("query primary failed(original error code: %d), switch to primary", ret))) { \ + if (OB_TMP_FAIL(str.assign_fmt("query primary failed(original error code: %d), switchover to primary is", ret))) { \ LOG_WARN("tenant role trans user error str assign failed"); \ } else { \ ret = OB_OP_NOT_ALLOW; \ LOG_USER_ERROR(OB_OP_NOT_ALLOW, str.ptr()); \ } \ - break; \ + break; \ + case -ER_ACCOUNT_HAS_BEEN_LOCKED: \ + ret = OB_OP_NOT_ALLOW; \ + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "primary tenant's user account is locked, switchover to primary is"); \ + break; \ case OB_ERR_TENANT_IS_LOCKED: \ ret = OB_OP_NOT_ALLOW; \ - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "primary tenant is locked, role transition"); \ - break; \ - case OB_SOURCE_TENANT_STATE_NOT_MATCH: \ - LOG_USER_ERROR(OB_SOURCE_TENANT_STATE_NOT_MATCH); \ + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "primary tenant is locked, switchover to primary is"); \ break; \ case OB_SOURCE_LS_STATE_NOT_MATCH: \ LOG_USER_ERROR(OB_SOURCE_LS_STATE_NOT_MATCH); \ break; \ - default: \ - if (OB_TMP_FAIL(str.assign_fmt("wait tenant sync to latest failed(original error code: %d), switch to primary", ret))){ \ - LOG_WARN("tenant role trans user error str assign failed"); \ - } else { \ - ret = OB_OP_NOT_ALLOW; \ - LOG_USER_ERROR(OB_OP_NOT_ALLOW, str.ptr()); \ - } \ - } \ + } + const char* const ObTenantRoleTransitionConstants::SWITCH_TO_PRIMARY_LOG_MOD_STR = "SWITCH_TO_PRIMARY"; const char* const ObTenantRoleTransitionConstants::SWITCH_TO_STANDBY_LOG_MOD_STR = "SWITCH_TO_STANDBY"; const char* const ObTenantRoleTransitionConstants::RESTORE_TO_STANDBY_LOG_MOD_STR = "RESTORE_TO_STANDBY"; +///////////ObTenantRoleTransCostDetail///////////////// +const char* ObTenantRoleTransCostDetail::type_to_str(CostType type) const +{ + static const char *strs[] = { "WAIT_LOG_SYNC", "WAIT_BALANCE_TASK", "FLASHBACK_LOG", + "WAIT_LOG_END", "CHANGE_ACCESS_MODE" }; + STATIC_ASSERT(MAX_COST_TYPE == ARRAYSIZEOF(strs), "status string array size mismatch"); + const char* str = "UNKNOWN"; + if (type < 0 || type >= MAX_COST_TYPE) { + LOG_WARN_RET(OB_INVALID_ARGUMENT, "invalid type", K(type)); + } else { + str = strs[type]; + } + return str; +} +void ObTenantRoleTransCostDetail::add_cost(CostType type, int64_t cost) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(type < 0 || type >= MAX_COST_TYPE || cost < 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("invalid argument", KR(ret), K(type), K(cost)); + } else { + cost_type_[type] = cost; + } +} +int64_t ObTenantRoleTransCostDetail::to_string(char *buf, const int64_t buf_len) const +{ + int64_t pos = 0; + int64_t counted_cost = 0; + for (int i = 0 ; i < MAX_COST_TYPE; i++) { + if (cost_type_[i] > 0) { + counted_cost += cost_type_[i]; + CostType type = static_cast(i); + BUF_PRINTF("%s: %ld, ", type_to_str(type), cost_type_[i]); + } + } + BUF_PRINTF("OTHERS: %ld", end_ - start_ - counted_cost); + return pos; +} + +///////////ObTenantRoleTransAllLSInfo///////////////// +int ObTenantRoleTransAllLSInfo::init() +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < ObLSStatus::OB_LS_MAX_STATUS; i++) { + all_ls_[i].reset(); + } + return ret; +} +int ObTenantRoleTransAllLSInfo::add_ls(const ObLSID &ls_id, const ObLSStatus status) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!ls_id.is_valid() || status < 0 || status >= ObLSStatus::OB_LS_MAX_STATUS)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(ls_id), K(status)); + } else if (OB_FAIL(all_ls_[status].push_back(ls_id))) { + LOG_WARN("fail to push back", KR(ret), K(ls_id)); + } + return ret; +} +int64_t ObTenantRoleTransAllLSInfo::to_string(char *buf, const int64_t buf_len) const +{ + int64_t pos = 0; + bool meet_first = false; + for (int64_t i = 0; i < ObLSStatus::OB_LS_MAX_STATUS; i++) { + ObLSStatus status = static_cast(i); + if (all_ls_[i].count() > 0) { + if (!meet_first) { + BUF_PRINTF("%s: ", ls_status_to_str(status)); + meet_first = true; + } else { + BUF_PRINTF("; %s: ", ls_status_to_str(status)); + } + int64_t arr_end = all_ls_[i].count() - 1; + for (int64_t j=0; j < arr_end; j++) { + BUF_PRINTF("%ld, ", all_ls_[i].at(j).id()); + } + BUF_PRINTF("%ld", all_ls_[i].at(arr_end).id()); + } + } + if (!is_valid()) BUF_PRINTF("NULL"); + return pos; +} + +bool ObTenantRoleTransAllLSInfo::is_valid() const { + for (int64_t i = 0; i < ObLSStatus::OB_LS_MAX_STATUS; ++i) { + if (all_ls_[i].count() > 0) { + return true; + } + } + return false; +} + +////////////ObTenantRoleTransNonSyncInfo////////////// +int ObTenantRoleTransNonSyncInfo::init(const ObArray &switchover_checkpoints) +{ + int ret = OB_SUCCESS; + not_sync_checkpoints_.reset(); + is_sync_ = true; + for (int64_t i = 0; OB_SUCC(ret) && i < switchover_checkpoints.count(); i++) { + const obrpc::ObCheckpoint &checkpoint = switchover_checkpoints.at(i); + if (!checkpoint.is_sync_to_latest()) { + is_sync_ = false; + LOG_WARN("ls not sync, keep waiting", KR(ret), K(checkpoint)); + if (OB_FAIL(not_sync_checkpoints_.push_back(checkpoint))) { + LOG_WARN("fail to push back", KR(ret), K(checkpoint), K(not_sync_checkpoints_)); + } + } + } + return ret; +} + +int64_t ObTenantRoleTransNonSyncInfo::to_string(char *buf, const int64_t buf_len) const +{ + int64_t pos = 0; + int64_t ls_num = not_sync_checkpoints_.count(); + BUF_PRINTF("NON_SYNC_LS_CNT: %ld; TOP_%ld: ", ls_num, MAX_PRINT_LS_NUM); + if(ls_num > 0) { + int64_t arr_end = MAX_PRINT_LS_NUM > ls_num ? ls_num - 1 : MAX_PRINT_LS_NUM - 1; + for (int64_t i = 0; i < arr_end; i++) { + const obrpc::ObCheckpoint &checkpoint = not_sync_checkpoints_.at(i); + BUF_PRINTO(checkpoint); + J_COMMA(); + } + J_OBJ(not_sync_checkpoints_.at(arr_end)); + } + return pos; +} ////////////ObTenantRoleTransitionService////////////// + +int ObTenantRoleTransitionService::init( + uint64_t tenant_id, + const obrpc::ObSwitchTenantArg::OpType &switch_optype, + const bool is_verify, + common::ObMySQLProxy *sql_proxy, + obrpc::ObSrvRpcProxy *rpc_proxy, + ObTenantRoleTransCostDetail *cost_detail, + ObTenantRoleTransAllLSInfo *all_ls_info) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(sql_proxy) + || OB_ISNULL(rpc_proxy) + || OB_ISNULL(cost_detail) + || OB_ISNULL(all_ls_info) + || OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), KP(sql_proxy), KP(rpc_proxy), K(tenant_id), + K(switch_optype), KP(cost_detail), KP(all_ls_info)); + } else { + sql_proxy_= sql_proxy; + rpc_proxy_ = rpc_proxy; + tenant_id_ = tenant_id; + switch_optype_ = switch_optype; + switchover_epoch_ = OB_INVALID_VERSION; + so_scn_.set_min(); + cost_detail_ = cost_detail; + all_ls_info_ = all_ls_info; + has_restore_source_ = false; + is_verify_ = is_verify; + } + return ret; +} int ObTenantRoleTransitionService::check_inner_stat() { int ret = OB_SUCCESS; - if (OB_ISNULL(sql_proxy_) || OB_ISNULL(rpc_proxy_) || - OB_UNLIKELY(!is_user_tenant(tenant_id_))) { + if (OB_ISNULL(sql_proxy_) || OB_ISNULL(rpc_proxy_) || OB_ISNULL(cost_detail_) || OB_ISNULL(all_ls_info_) + || OB_UNLIKELY(!is_user_tenant(tenant_id_))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("error unexpected", KR(ret), K(tenant_id_), KP(sql_proxy_), KP(rpc_proxy_)); + LOG_WARN("error unexpected", KR(ret), K(tenant_id_), KP(sql_proxy_), KP(rpc_proxy_), + KP(cost_detail_), KP(all_ls_info_)); } return ret; -} - +} int ObTenantRoleTransitionService::failover_to_primary() { int ret = OB_SUCCESS; - LOG_INFO("[ROLE_TRANSITION] start to failover to primary", KR(ret), K(tenant_id_)); + LOG_INFO("[ROLE_TRANSITION] start to failover to primary", KR(ret), K(is_verify_), K(tenant_id_)); const int64_t start_service_time = ObTimeUtility::current_time(); ObAllTenantInfo tenant_info; if (OB_FAIL(check_inner_stat())) { @@ -118,35 +284,34 @@ int ObTenantRoleTransitionService::failover_to_primary() } else if (OB_UNLIKELY(!tenant_info.get_recovery_until_scn().is_valid_and_not_min())) { ret = OB_OP_NOT_ALLOW; LOG_WARN("invalid recovery_until_scn", KR(ret), K(tenant_info)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "recovery_until_scn is invalid, switch to primary"); + TENANT_ROLE_TRANS_USER_ERR_WITH_SUFFIX(OB_OP_NOT_ALLOW, "recovery_until_scn is invalid", switch_optype_); } else if (FALSE_IT(switchover_epoch_ = tenant_info.get_switchover_epoch())) { - } else if ((tenant_info.is_normal_status())) { - //do failover to primary + } else if (tenant_info.is_normal_status()) { if (OB_FAIL(do_failover_to_primary_(tenant_info))) { - LOG_WARN("failed to do failover to primary", KR(ret), K(tenant_info)); + LOG_WARN("fail to do failover to primary", KR(ret), K(tenant_info)); } } else if (tenant_info.is_prepare_flashback_for_failover_to_primary_status() - || tenant_info.is_prepare_flashback_for_switch_to_primary_status()) { + || tenant_info.is_prepare_flashback_for_switch_to_primary_status()) { //prepare flashback - if (OB_FAIL(do_prepare_flashback_(tenant_info))) { - LOG_WARN("failed to prepare flashback", KR(ret), K(tenant_info)); + if (!is_verify_ && OB_FAIL(do_prepare_flashback_(tenant_info))) { + LOG_WARN("fail to prepare flashback", KR(ret), K(tenant_info)); } } else if (tenant_info.is_flashback_status()) { - if (OB_FAIL(do_flashback_())) { - LOG_WARN("failed to flashback", KR(ret), K(tenant_info)); + if (!is_verify_ && OB_FAIL(do_flashback_())) { + LOG_WARN("fail to flashback", KR(ret), K(tenant_info)); } } else if (tenant_info.is_switching_to_primary_status()) { - if (OB_FAIL(do_switch_access_mode_to_append(tenant_info, share::PRIMARY_TENANT_ROLE))) { - LOG_WARN("failed to switch access mode", KR(ret), K(tenant_info)); + if (!is_verify_ && OB_FAIL(do_switch_access_mode_to_append(tenant_info, share::PRIMARY_TENANT_ROLE))) { + LOG_WARN("fail to switch access mode", KR(ret), K(tenant_info)); } } else { ret = OB_OP_NOT_ALLOW; - LOG_WARN("switchover status not match", KR(ret), K(tenant_info), K_(tenant_id)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "switchover status not match, switch to primary"); + LOG_WARN("switchover status not match", KR(ret), K(is_verify_), K(tenant_info), K_(tenant_id)); + TENANT_ROLE_TRANS_USER_ERR_WITH_SUFFIX(OB_OP_NOT_ALLOW, "switchover status not match", switch_optype_); } if (OB_FAIL(ret)) { - } else { + } else if (!is_verify_) { (void)broadcast_tenant_info(ObTenantRoleTransitionConstants::SWITCH_TO_PRIMARY_LOG_MOD_STR); ObBroadcastSchemaArg arg; arg.tenant_id_ = tenant_id_; @@ -159,15 +324,15 @@ int ObTenantRoleTransitionService::failover_to_primary() } const int64_t cost = ObTimeUtility::current_time() - start_service_time; - LOG_INFO("[ROLE_TRANSITION] finish failover to primary", KR(ret), K(tenant_info), K(cost)); + LOG_INFO("[ROLE_TRANSITION] finish failover to primary", KR(ret), K(tenant_info), K(is_verify_), K(cost)); return ret; } - ERRSIM_POINT_DEF(ERRSIM_TENANT_ROLE_TRANS_WAIT_SYNC_ERROR); int ObTenantRoleTransitionService::do_failover_to_primary_(const share::ObAllTenantInfo &tenant_info) { int ret = OB_SUCCESS; ObAllTenantInfo new_tenant_info; + (void) new_tenant_info.assign(tenant_info); if (OB_FAIL(check_inner_stat())) { LOG_WARN("error unexpected", KR(ret), K(tenant_id_), KP(sql_proxy_), KP(rpc_proxy_)); } else if (OB_UNLIKELY(obrpc::ObSwitchTenantArg::OpType::SWITCH_TO_PRIMARY != switch_optype_ @@ -176,32 +341,42 @@ int ObTenantRoleTransitionService::do_failover_to_primary_(const share::ObAllTen LOG_WARN("unexpected switch tenant action", KR(ret), K_(switch_optype), K(tenant_info), K_(tenant_id)); } else if (OB_UNLIKELY(!(tenant_info.is_normal_status()) - || tenant_info.is_primary() - || switchover_epoch_ != tenant_info.get_switchover_epoch())) { + || tenant_info.is_primary() + || switchover_epoch_ != tenant_info.get_switchover_epoch())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("tenant switchover status not valid", KR(ret), K(tenant_info), K(switchover_epoch_)); } else if (obrpc::ObSwitchTenantArg::OpType::SWITCH_TO_PRIMARY == switch_optype_ - && OB_FAIL(wait_tenant_sync_to_latest_until_timeout_(tenant_id_, tenant_info))) { - LOG_WARN("fail to wait_tenant_sync_to_latest_until_timeout_", KR(ret), K_(tenant_id), K(tenant_info)); - TENANT_ROLE_TRANS_USER_ERROR; - } else if (OB_SUCC(ret) && ERRSIM_TENANT_ROLE_TRANS_WAIT_SYNC_ERROR) { - ret = ERRSIM_TENANT_ROLE_TRANS_WAIT_SYNC_ERROR; - TENANT_ROLE_TRANS_USER_ERROR; - LOG_WARN("errsim wait_tenant_sync_to_latest_until_timeout", K(ret)); + && OB_FAIL(wait_sys_ls_sync_to_latest_until_timeout_(tenant_id_, new_tenant_info))) { + LOG_WARN("fail to execute wait_sys_ls_sync_to_latest_until_timeout_", KR(ret), K_(tenant_id), K(new_tenant_info)); + SOURCE_TENANT_CHECK_USER_ERROR_FOR_SWITCHOVER_TO_PRIMARY; + } + /*The switchover to primary verify command ends here. + This command cannot update the switchover status nor execute the further logic. + We update the switchover status right after sys ls being synced, The reason is as follows: + The tenant fetches log with reference to tenant_sync_scn + 3s. + If two ls' sync_scn have an extremely large difference, + e.g. tenant_sync_scn = ls_1001 sync_scn + 3s << ls_1002 sync_scn, + there is a possibility that ls_1002's log cannot be fetched completely. + To ensure all ls' log are fetched completely, we update the switchover status as PREPARE_xxx. + Then the tenant fetching log will no longer utilize tenant_sync_scn + 3s as a reference point. + **/ + if (OB_FAIL(ret) || is_verify_) { + } else if (OB_FAIL(clear_service_name_())) { + LOG_WARN("fail to execute clear_service_name", KR(ret), K(tenant_id_)); } else if (OB_FAIL(ObAllTenantInfoProxy::update_tenant_role( - tenant_id_, sql_proxy_, tenant_info.get_switchover_epoch(), - share::STANDBY_TENANT_ROLE, tenant_info.get_switchover_status(), - obrpc::ObSwitchTenantArg::OpType::SWITCH_TO_PRIMARY == switch_optype_ - ? share::PREPARE_FLASHBACK_FOR_SWITCH_TO_PRIMARY_SWITCHOVER_STATUS - : share::PREPARE_FLASHBACK_FOR_FAILOVER_TO_PRIMARY_SWITCHOVER_STATUS, - switchover_epoch_))) { + tenant_id_, sql_proxy_, tenant_info.get_switchover_epoch(), + share::STANDBY_TENANT_ROLE, tenant_info.get_switchover_status(), + obrpc::ObSwitchTenantArg::OpType::SWITCH_TO_PRIMARY == switch_optype_ ? + share::PREPARE_FLASHBACK_FOR_SWITCH_TO_PRIMARY_SWITCHOVER_STATUS : + share::PREPARE_FLASHBACK_FOR_FAILOVER_TO_PRIMARY_SWITCHOVER_STATUS, + switchover_epoch_))) { LOG_WARN("failed to update tenant role", KR(ret), K(tenant_id_), K(tenant_info)); } else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id_, sql_proxy_, - false, new_tenant_info))) { + false, new_tenant_info))) { LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id_)); } else if (OB_UNLIKELY(new_tenant_info.get_switchover_epoch() != switchover_epoch_)) { ret = OB_NEED_RETRY; - LOG_WARN("switchover is concurrency", KR(ret), K(switchover_epoch_), K(new_tenant_info)); + LOG_WARN("switchover is concurrency", KR(ret), K(switchover_epoch_), K(new_tenant_info)); } else if (OB_FAIL(do_prepare_flashback_(new_tenant_info))) { LOG_WARN("failed to prepare flashback", KR(ret), K(new_tenant_info)); } @@ -221,8 +396,8 @@ int ObTenantRoleTransitionService::do_prepare_flashback_(share::ObAllTenantInfo } else if (OB_UNLIKELY(!(tenant_info.is_prepare_flashback_for_failover_to_primary_status() || tenant_info.is_prepare_flashback_for_switch_to_primary_status()))) { ret = OB_OP_NOT_ALLOW; - LOG_WARN("switchover status not match, switch tenant not allow", KR(ret), K(tenant_info)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "switchover status not match, switchover tenant"); + LOG_WARN("switchover status not match, switch tenant is not allowed", KR(ret), K(tenant_info)); + TENANT_ROLE_TRANS_USER_ERR_WITH_SUFFIX(OB_OP_NOT_ALLOW, "switchover status not match", switch_optype_); } else if (OB_UNLIKELY(switchover_epoch_ != tenant_info.get_switchover_epoch())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("tenant switchover status not valid", KR(ret), K(tenant_info), K(switchover_epoch_)); @@ -231,7 +406,11 @@ int ObTenantRoleTransitionService::do_prepare_flashback_(share::ObAllTenantInfo LOG_WARN("failed to do_prepare_flashback_for_switch_to_primary_", KR(ret), K(tenant_info)); } } else if (obrpc::ObSwitchTenantArg::OpType::FAILOVER_TO_PRIMARY == switch_optype_) { - if (OB_FAIL(do_prepare_flashback_for_failover_to_primary_(tenant_info))) { + if (OB_FAIL(double_check_service_name_(tenant_info))) { + // do double check here + // so_status is not normal, service name related commands is not allowed + LOG_WARN("fail to execute double_check_service_name_", KR(ret), K(tenant_info)); + } else if (OB_FAIL(do_prepare_flashback_for_failover_to_primary_(tenant_info))) { LOG_WARN("failed to do_prepare_flashback_for_failover_to_primary_", KR(ret), K(tenant_info)); } } else { @@ -248,14 +427,12 @@ int ObTenantRoleTransitionService::do_prepare_flashback_(share::ObAllTenantInfo } return ret; } - int ObTenantRoleTransitionService::do_prepare_flashback_for_switch_to_primary_( share::ObAllTenantInfo &tenant_info) { int ret = OB_SUCCESS; - + ObLSStatusOperator status_op; DEBUG_SYNC(PREPARE_FLASHBACK_FOR_SWITCH_TO_PRIMARY); - LOG_INFO("start to do_prepare_flashback_for_switch_to_primary_", KR(ret), K_(tenant_id)); if (OB_FAIL(check_inner_stat())) { @@ -263,11 +440,28 @@ int ObTenantRoleTransitionService::do_prepare_flashback_for_switch_to_primary_( } else if (OB_UNLIKELY(!tenant_info.is_prepare_flashback_for_switch_to_primary_status())) { ret = OB_OP_NOT_ALLOW; LOG_WARN("switchover status not match, switch to primary not allow", KR(ret), K(tenant_info)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "switchover status not match, switchover to primary"); + TENANT_ROLE_TRANS_USER_ERR_WITH_SUFFIX(OB_OP_NOT_ALLOW, "switchover status not match", switch_optype_); + } else if (OB_UNLIKELY(obrpc::ObSwitchTenantArg::OpType::SWITCH_TO_PRIMARY != switch_optype_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("operation type is not SWITCH_TO_PRIMARY", KR(ret), K(switch_optype_)); } else if (OB_UNLIKELY(switchover_epoch_ != tenant_info.get_switchover_epoch())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("tenant switchover status not valid", KR(ret), K(tenant_info), K_(switchover_epoch)); - } else if (OB_FAIL(wait_ls_balance_task_finish_())) { + } else if (OB_FAIL(status_op.create_abort_ls_in_switch_tenant( + tenant_id_, tenant_info.get_switchover_status(), + tenant_info.get_switchover_epoch(), *sql_proxy_))) { + // SYS LS has been synced, all current CTREATING/CTREATED LS cannot become NORMAL + // These LS should become CREATE_ABORT, otherwise tenant cannot be synced + LOG_WARN("failed to create abort ls", KR(ret), K_(tenant_id), K(tenant_info)); + } else if (OB_FAIL(wait_tenant_sync_to_latest_until_timeout_(tenant_id_, tenant_info))) { + LOG_WARN("fail to execute wait_tenant_sync_to_latest_until_timeout_", KR(ret), K(tenant_info)); + SOURCE_TENANT_CHECK_USER_ERROR_FOR_SWITCHOVER_TO_PRIMARY; + } else if (OB_SUCC(ret) && ERRSIM_TENANT_ROLE_TRANS_WAIT_SYNC_ERROR) { + ret = ERRSIM_TENANT_ROLE_TRANS_WAIT_SYNC_ERROR; + SOURCE_TENANT_CHECK_USER_ERROR_FOR_SWITCHOVER_TO_PRIMARY; + LOG_WARN("errsim wait_tenant_sync_to_latest_until_timeout", K(ret)); + } + if (FAILEDx(wait_ls_balance_task_finish_())) { LOG_WARN("failed to wait ls balance task finish", KR(ret)); } else if (OB_FAIL(switchover_update_tenant_status(tenant_id_, true /* switch_to_primary */, @@ -278,7 +472,6 @@ int ObTenantRoleTransitionService::do_prepare_flashback_for_switch_to_primary_( tenant_info))) { LOG_WARN("failed to switchover_update_tenant_status", KR(ret), K_(tenant_id), K(tenant_info)); } - return ret; } @@ -292,11 +485,11 @@ int ObTenantRoleTransitionService::do_prepare_flashback_for_failover_to_primary_ } else if (OB_UNLIKELY(!tenant_info.is_prepare_flashback_for_failover_to_primary_status())) { ret = OB_OP_NOT_ALLOW; LOG_WARN("switchover status not match, failover to primary not allow", KR(ret), K(tenant_info)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "switchover status not match, failover to primary"); + TENANT_ROLE_TRANS_USER_ERR_WITH_SUFFIX(OB_OP_NOT_ALLOW, "switchover status not match", switch_optype_); } else if (OB_UNLIKELY(switchover_epoch_ != tenant_info.get_switchover_epoch())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("tenant switchover status not valid", KR(ret), K(tenant_info), K_(switchover_epoch)); - } else if (OB_FAIL(OB_PRIMARY_STANDBY_SERVICE.do_recover_tenant(tenant_id_, + } else if (OB_FAIL(OB_STANDBY_SERVICE.do_recover_tenant(tenant_id_, share::PREPARE_FLASHBACK_FOR_FAILOVER_TO_PRIMARY_SWITCHOVER_STATUS, obrpc::ObRecoverTenantArg::RecoverType::CANCEL, SCN::min_scn()))) { @@ -321,21 +514,58 @@ int ObTenantRoleTransitionService::do_prepare_flashback_for_failover_to_primary_ return ret; } -int ObTenantRoleTransitionService::do_switch_access_mode_to_flashback( - const share::ObAllTenantInfo &tenant_info) +int ObTenantRoleTransitionService::clear_service_name_() { int ret = OB_SUCCESS; - if (OB_FAIL(check_inner_stat())) { - LOG_WARN("error unexpected", KR(ret), K(tenant_id_), KP(sql_proxy_), KP(rpc_proxy_)); - } else if (OB_UNLIKELY(!tenant_info.is_flashback_status() - || switchover_epoch_ != tenant_info.get_switchover_epoch())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tenant switchover status not valid", KR(ret), K(tenant_info), K(switchover_epoch_)); - } else if (OB_FAIL(get_all_ls_status_and_change_access_mode_( - palf::AccessMode::FLASHBACK, - SCN::base_scn(), - SCN::min_scn()))) { - LOG_WARN("fail to execute get_all_ls_status_and_change_access_mode_", KR(ret), K(tenant_info)); + ObArray all_service_names; + int64_t epoch = 0; + if (obrpc::ObSwitchTenantArg::OpType::FAILOVER_TO_PRIMARY != switch_optype_) { + // do nothing + } else if (OB_FAIL(ObServiceNameProxy::check_is_service_name_enabled(tenant_id_))) { + if (OB_NOT_SUPPORTED == ret) { + ret = OB_SUCCESS; + } + LOG_WARN("service_name is not enabled, no need to execute clear_service_name", KR(ret), K(tenant_id_)); + } else if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("GCTX.sql_proxy_ is null", KR(ret), KP(GCTX.sql_proxy_)); + } else if (OB_FAIL(ObServiceNameProxy::select_all_service_names_with_epoch(tenant_id_, epoch, all_service_names))) { + LOG_WARN("fail to execute select_all_service_names_with_epoch", KR(ret), K(tenant_id_)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < all_service_names.count(); i++) { + const ObServiceName &service_name = all_service_names.at(i); + if (OB_FAIL(ObServiceNameCommand::stop_service(tenant_id_, service_name.get_service_name_str()))) { + LOG_WARN("fail to execute stop_service", KR(ret), K(tenant_id_), K(service_name)); + } else if (OB_FAIL(ObServiceNameCommand::delete_service(tenant_id_, service_name.get_service_name_str()))) { + LOG_WARN("fail to execute delete_service", KR(ret), K(tenant_id_), K(service_name)); + } + } + } + return ret; +} +int ObTenantRoleTransitionService::double_check_service_name_(const share::ObAllTenantInfo &tenant_info) +{ + int ret = OB_SUCCESS; + ObArray all_service_names; + int64_t service_num = 0; + if (obrpc::ObSwitchTenantArg::OpType::FAILOVER_TO_PRIMARY != switch_optype_) { + // do nothing + } else if (OB_FAIL(ObServiceNameProxy::check_is_service_name_enabled(tenant_id_))) { + if (OB_NOT_SUPPORTED == ret) { + ret = OB_SUCCESS; + } + LOG_WARN("service_name is not enabled, no need to execute double_check_service_name_", KR(ret), K(tenant_id_)); + } else if (OB_UNLIKELY(tenant_info.is_normal_status())) { + ret = OB_OP_NOT_ALLOW; + LOG_WARN("not allowed to do double check when switchover status is NORMAL", KR(ret), K(tenant_info)); + } else if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("GCTX.sql_proxy_ is null", KR(ret), KP(GCTX.sql_proxy_)); + } else if (OB_FAIL(ObServiceNameProxy::get_tenant_service_name_num(*GCTX.sql_proxy_, tenant_id_, service_num))) { + LOG_WARN("fail to execute get_tenant_service_name_num", KR(ret), K(tenant_id_)); + } else if (OB_UNLIKELY(0 != service_num)) { + ret = OB_NEED_RETRY; + LOG_WARN("the tenant should have zero service_name", KR(ret), K(service_num)); } return ret; } @@ -347,7 +577,6 @@ int ObTenantRoleTransitionService::do_flashback_() logservice::ObLogService *log_service = NULL; ObLSStatusOperator status_op; ObAllTenantInfo tenant_info; - if (OB_FAIL(check_inner_stat())) { LOG_WARN("error unexpected", KR(ret), K(tenant_id_), KP(sql_proxy_), KP(rpc_proxy_)); } else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info( @@ -358,9 +587,10 @@ int ObTenantRoleTransitionService::do_flashback_() tenant_info.get_switchover_epoch())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("tenant switchover status not valid", KR(ret), K(tenant_info), K(switchover_epoch_)); - } else if (OB_FAIL(status_op.create_abort_ls_in_switch_tenant( - tenant_id_, tenant_info.get_switchover_status(), - tenant_info.get_switchover_epoch(), *sql_proxy_))) { + } else if (obrpc::ObSwitchTenantArg::OpType::FAILOVER_TO_PRIMARY == switch_optype_ + && OB_FAIL(status_op.create_abort_ls_in_switch_tenant( + tenant_id_, tenant_info.get_switchover_status(), + tenant_info.get_switchover_epoch(), *sql_proxy_))) { LOG_WARN("failed to create abort ls", KR(ret), K_(tenant_id), K(tenant_info)); } else if (OB_FAIL(ObShareUtil::set_default_timeout_ctx( ctx, GCONF.internal_sql_execute_timeout))) { @@ -368,15 +598,23 @@ int ObTenantRoleTransitionService::do_flashback_() } else if (OB_ISNULL(log_service = MTL(logservice::ObLogService *))) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("failed to get MTL log_service", KR(ret), K(tenant_id_)); - } else if (OB_FAIL(log_service->flashback( - tenant_id_, tenant_info.get_sync_scn(), ctx.get_timeout()))) { - LOG_WARN("failed to flashback", KR(ret), K(tenant_id_), K(tenant_info)); + } else { + int64_t begin_time = ObTimeUtility::current_time(); + if (OB_FAIL(log_service->flashback(tenant_id_, tenant_info.get_sync_scn(), ctx.get_timeout()))) { + LOG_WARN("failed to flashback", KR(ret), K(tenant_id_), K(tenant_info)); + } + int64_t log_flashback = ObTimeUtility::current_time() - begin_time; + if (OB_LIKELY(NULL != cost_detail_)) { + (void) cost_detail_->add_cost(ObTenantRoleTransCostDetail::LOG_FLASHBACK, log_flashback); + } + } + + if (OB_FAIL(ret)) { } else { CLUSTER_EVENT_ADD_LOG(ret, "flashback end", - "tenant id", tenant_id_, - "switchover#", tenant_info.get_switchover_epoch(), - "flashback_scn#", tenant_info.get_sync_scn()); - + "tenant id", tenant_id_, + "switchover#", tenant_info.get_switchover_epoch(), + "flashback_scn#", tenant_info.get_sync_scn()); ObAllTenantInfo new_tenant_info; if (OB_FAIL(ObAllTenantInfoProxy::update_tenant_switchover_status( tenant_id_, sql_proxy_, tenant_info.get_switchover_epoch(), @@ -400,6 +638,7 @@ int ObTenantRoleTransitionService::do_switch_access_mode_to_append( const share::ObTenantRole &target_tenant_role) { int ret = OB_SUCCESS; + int64_t begin_time = ObTimeUtility::current_time(); palf::AccessMode access_mode = logservice::ObLogService::get_palf_access_mode(target_tenant_role); SCN ref_scn; if (OB_FAIL(check_inner_stat())) { @@ -430,13 +669,14 @@ int ObTenantRoleTransitionService::do_switch_access_mode_to_append( } else if (OB_UNLIKELY(tenant_info.get_switchover_status() != cur_tenant_info.get_switchover_status() || tenant_info.get_switchover_epoch() != cur_tenant_info.get_switchover_epoch())) { ret = OB_OP_NOT_ALLOW; - LOG_WARN("Tenant status changed by concurrent operation, switch to primary not allowed", - KR(ret), K(tenant_info), K(cur_tenant_info)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "tenant status changed by concurrent operation, switch to primary"); + LOG_WARN("Tenant status changed by concurrent operation, switch to primary is not allowed", + KR(ret), K(tenant_info), K(cur_tenant_info)); + TENANT_ROLE_TRANS_USER_ERR_WITH_SUFFIX(OB_OP_NOT_ALLOW, + "tenant status changed by concurrent operation", switch_optype_); } else if (OB_FAIL(ObAllTenantInfoProxy::update_tenant_role_in_trans( - tenant_id_, trans, tenant_info.get_switchover_epoch(), - share::PRIMARY_TENANT_ROLE, tenant_info.get_switchover_status(), - share::NORMAL_SWITCHOVER_STATUS, switchover_epoch_))) { + tenant_id_, trans, tenant_info.get_switchover_epoch(), + share::PRIMARY_TENANT_ROLE, tenant_info.get_switchover_status(), + share::NORMAL_SWITCHOVER_STATUS, switchover_epoch_))) { LOG_WARN("failed to update tenant switchover status", KR(ret), K(tenant_id_), K(tenant_info), K(cur_tenant_info)); } else if (cur_tenant_info.get_recovery_until_scn().is_max()) { LOG_INFO("recovery_until_scn already is max_scn", KR(ret), K_(tenant_id), K(cur_tenant_info)); @@ -452,6 +692,10 @@ int ObTenantRoleTransitionService::do_switch_access_mode_to_append( } } } + if (OB_LIKELY(NULL != cost_detail_)) { + int64_t log_mode_change = ObTimeUtility::current_time() - begin_time; + (void) cost_detail_->add_cost(ObTenantRoleTransCostDetail::CHANGE_ACCESS_MODE, log_mode_change); + } return ret; } @@ -502,7 +746,8 @@ int ObTenantRoleTransitionService::get_tenant_ref_scn_(const share::SCN &sync_sc int ObTenantRoleTransitionService::wait_ls_balance_task_finish_() { int ret = OB_SUCCESS; - + int64_t begin_time = ObTimeUtility::current_time(); + uint64_t compat_version = 0; if (OB_FAIL(check_inner_stat())) { LOG_WARN("error unexpected", KR(ret), K(tenant_id_), KP(sql_proxy_), KP(rpc_proxy_)); } else { @@ -556,7 +801,7 @@ int ObTenantRoleTransitionService::wait_ls_balance_task_finish_() } else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info( tenant_id_, sql_proxy_, false, cur_tenant_info))) { LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id_)); - } else if (cur_tenant_info.get_sync_scn() == cur_tenant_info.get_standby_scn()) { + } else if (cur_tenant_info.get_sync_scn() == cur_tenant_info.get_readable_scn()) { is_finish = true; for (int64_t i = 0; OB_SUCC(ret) && i < balance_task_array.count() && !is_finish; ++i) { const ObBalanceTask &task = balance_task_array.at(i); @@ -593,6 +838,10 @@ int ObTenantRoleTransitionService::wait_ls_balance_task_finish_() } } } + if (OB_LIKELY(NULL != cost_detail_)) { + int64_t wait_balance_task = ObTimeUtility::current_time() - begin_time; + (void) cost_detail_->add_cost(ObTenantRoleTransCostDetail::WAIT_BALANCE_TASK, wait_balance_task); + } return ret; } @@ -638,6 +887,7 @@ int ObTenantRoleTransitionService::do_switch_access_mode_to_raw_rw( const share::ObAllTenantInfo &tenant_info) { int ret = OB_SUCCESS; + int64_t begin_time = ObTimeUtility::current_time(); palf::AccessMode target_access_mode = logservice::ObLogService::get_palf_access_mode(STANDBY_TENANT_ROLE); ObLSStatusOperator status_op; share::ObLSStatusInfoArray sys_info_array; @@ -680,6 +930,12 @@ int ObTenantRoleTransitionService::do_switch_access_mode_to_raw_rw( LOG_WARN("fail to execute get_all_ls_status_and_change_access_mode_", KR(ret), K(tenant_info), K(target_access_mode), K(sys_ls_sync_scn)); } + + if (OB_LIKELY(NULL != cost_detail_)) { + int64_t log_mode_change = ObTimeUtility::current_time() - begin_time; + int64_t change_access_mode = log_mode_change - cost_detail_->get_wait_log_end(); + (void) cost_detail_->add_cost(ObTenantRoleTransCostDetail::CHANGE_ACCESS_MODE, change_access_mode); + } DEBUG_SYNC(AFTER_CHANGE_ACCESS_MODE); return ret; } @@ -690,6 +946,7 @@ int ObTenantRoleTransitionService::get_all_ls_status_and_change_access_mode_( const share::SCN &sys_ls_sync_scn) { int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; ObLSStatusOperator status_op; share::ObLSStatusInfoArray status_info_array; if (OB_FAIL(check_inner_stat())) { @@ -705,6 +962,9 @@ int ObTenantRoleTransitionService::get_all_ls_status_and_change_access_mode_( LOG_WARN("fail to execute change_ls_access_mode_", KR(ret), K(status_info_array), K(target_access_mode), K(ref_scn), K(sys_ls_sync_scn)); } + if (OB_TMP_FAIL(ls_status_stats_when_change_access_mode_(status_info_array))) { + LOG_WARN("fail to gather ls status", KR(ret), KR(tmp_ret), K(status_info_array)); + } return ret; } @@ -784,6 +1044,26 @@ int ObTenantRoleTransitionService::change_ls_access_mode_( return ret; } +int ObTenantRoleTransitionService::ls_status_stats_when_change_access_mode_( + const share::ObLSStatusInfoArray &status_info_array) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(check_inner_stat())) { + LOG_WARN("error unexpected", KR(ret), K(tenant_id_), KP(sql_proxy_), KP(rpc_proxy_)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < status_info_array.count(); ++i) { + const ObLSStatusInfo &status = status_info_array.at(i); + const ObLSID &ls_id = status.get_ls_id(); + const ObLSStatus &ls_status = status.get_status(); + if (OB_FAIL(all_ls_info_->add_ls(ls_id, ls_status))) { + LOG_WARN("fail to push back", KR(ret), K(ls_id), KPC(all_ls_info_)); + } + } + FLOG_INFO("gather ls_status_stats", KR(ret), K(all_ls_info_), K(status_info_array)); + } + return ret; +} + template int do_nonblock_renew(const ARRAY_L &array_l, const ARRAY_R &array_r, const uint64_t tenant_id) { @@ -970,9 +1250,10 @@ int ObTenantRoleTransitionService::do_change_ls_access_mode_( LOG_WARN("rpc count not equal to result count", KR(ret), K(rpc_count), K(return_code_array.count())); } else { + int64_t ls_wait_sync_scn_max = 0; for (int64_t i = 0; OB_SUCC(ret) && i < return_code_array.count(); ++i) { ret = return_code_array.at(i); - const auto *result = proxy.get_results().at(i); + const obrpc::ObChangeLSAccessModeRes *result = proxy.get_results().at(i); const obrpc::ObLSAccessModeInfo &info = ls_access_info.at(i); if (OB_FAIL(ret)) { LOG_WARN("send rpc is failed", KR(ret), K(i)); @@ -984,10 +1265,17 @@ int ObTenantRoleTransitionService::do_change_ls_access_mode_( } else if (OB_TMP_FAIL(success_ls_ids.push_back(result->get_ls_id()))) { LOG_WARN("fail to push back", KR(ret), KR(tmp_ret), K(success_ls_ids), K(result)); } - + if (OB_FAIL(ret)) { + } else { + int64_t wait_scn_t = result->get_wait_sync_scn_cost(); + ls_wait_sync_scn_max = + ls_wait_sync_scn_max > wait_scn_t ? ls_wait_sync_scn_max : wait_scn_t; + } LOG_INFO("[ROLE_TRANSITION] change ls access mode", KR(ret), K(info), KPC(result), K(proxy.get_dests())); }// end for - + if (OB_LIKELY(NULL != cost_detail_)) { + (void) cost_detail_->add_cost(ObTenantRoleTransCostDetail::WAIT_LOG_END, ls_wait_sync_scn_max); + } if (OB_FAIL(ret)) { if (OB_TMP_FAIL(do_nonblock_renew(ls_access_info, success_ls_ids, tenant_id_))) { LOG_WARN("failed to renew location", KR(ret), KR(tmp_ret), K(tenant_id_), K(ls_access_info), K(success_ls_ids)); @@ -1066,6 +1354,8 @@ int ObTenantRoleTransitionService::switchover_update_tenant_status( share::ObLSStatusInfoArray status_info_array; common::ObArray switchover_checkpoints; bool is_sync_to_latest = false; + SCN final_sync_scn; + final_sync_scn.set_min(); if (OB_UNLIKELY(!is_user_tenant(tenant_id) || !new_role.is_valid() @@ -1108,8 +1398,7 @@ int ObTenantRoleTransitionService::switchover_update_tenant_status( * Because replayable point is not support, set replayable_scn = sync_scn */ const SCN gts_upper_limit = transaction::ObTimestampService::get_sts_start_scn(max_sys_ls_sync_scn); - - const SCN final_sync_scn = MAX(max_checkpoint_scn, gts_upper_limit); + final_sync_scn = MAX(max_checkpoint_scn, gts_upper_limit); const SCN final_replayable_scn = final_sync_scn; SCN final_readable_scn = SCN::min_scn(); SCN final_recovery_until_scn = SCN::min_scn(); @@ -1131,7 +1420,7 @@ int ObTenantRoleTransitionService::switchover_update_tenant_status( if (switch_to_primary) { // switch_to_primary // Does not change STS - final_readable_scn = tmp_tenant_info.get_standby_scn(); + final_readable_scn = tmp_tenant_info.get_readable_scn(); // To prevent unexpected sync log, set recovery_until_scn = sync_scn final_recovery_until_scn = final_sync_scn; } else { @@ -1145,7 +1434,8 @@ int ObTenantRoleTransitionService::switchover_update_tenant_status( ret = OB_OP_NOT_ALLOW; LOG_WARN("recovery_until_scn is not max_scn ", KR(ret), K(tenant_id), K(final_recovery_until_scn), K(tmp_tenant_info)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "recovery_until_scn is not max_scn, switchover to standby"); + TENANT_ROLE_TRANS_USER_ERR_WITH_SUFFIX(OB_OP_NOT_ALLOW, + "recovery_until_scn is not max_scn", switch_optype_); } } } @@ -1177,24 +1467,85 @@ int ObTenantRoleTransitionService::switchover_update_tenant_status( ret = OB_SUCC(ret) ? temp_ret : ret; } } - + if (OB_SUCC(ret)) { + so_scn_ = final_sync_scn; + } CLUSTER_EVENT_ADD_LOG(ret, "update tenant status", - "tenant id", tenant_id, - "old switchover#", old_switchover_epoch, - "new switchover#", new_tenant_info.get_switchover_epoch(), - K(new_role), K(old_status), K(new_status)); + "tenant id", tenant_id, + "old switchover#", old_switchover_epoch, + "new switchover#", new_tenant_info.get_switchover_epoch(), + K(new_role), K(old_status), K(new_status)); + return ret; +} + +int ObTenantRoleTransitionService::wait_sys_ls_sync_to_latest_until_timeout_( + const uint64_t tenant_id, + ObAllTenantInfo &tenant_info) +{ + int ret = OB_SUCCESS; + bool only_check_sys_ls = true; + if (OB_FAIL(check_inner_stat())) { + LOG_WARN("inner stat error", KR(ret)); + } else if (obrpc::ObSwitchTenantArg::OpType::SWITCH_TO_PRIMARY != switch_optype_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("only switchover to primary can execute this logic", KR(ret), K(tenant_id), K(switch_optype_)); + } else if (OB_UNLIKELY(!tenant_info.is_normal_status())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant switchover status is not normal", KR(ret), K(tenant_id), K(tenant_info)); + } else if (OB_FAIL(check_restore_source_for_switchover_to_primary_(tenant_id_))) { + LOG_WARN("fail to check restore source", KR(ret), K_(tenant_id)); + } else if (!has_restore_source_) { + LOG_INFO("no restore source", K(tenant_id), K(tenant_info)); + } else if (OB_FAIL(check_sync_to_latest_do_while_(tenant_info, only_check_sys_ls))) { + LOG_WARN("fail to check whether sys ls is synced", KR(ret), K(tenant_info)); + } return ret; } int ObTenantRoleTransitionService::wait_tenant_sync_to_latest_until_timeout_( - const uint64_t tenant_id, - const ObAllTenantInfo &tenant_info) + const uint64_t tenant_id, + const ObAllTenantInfo &tenant_info) +{ + int ret = OB_SUCCESS; + bool only_check_sys_ls = false; + int64_t begin_time = ObTimeUtility::current_time(); + if (OB_FAIL(check_inner_stat())) { + LOG_WARN("inner stat error", KR(ret)); + } else if (obrpc::ObSwitchTenantArg::OpType::SWITCH_TO_PRIMARY != switch_optype_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("only switchover to primary can execute this logic", KR(ret), K(tenant_id), K(switch_optype_)); + } else if (OB_UNLIKELY(!tenant_info.is_prepare_flashback_for_switch_to_primary_status())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant switchover status is not prepare", KR(ret), K(tenant_id), K(tenant_info)); + } else if (OB_FAIL(check_restore_source_for_switchover_to_primary_(tenant_id_))) { + LOG_WARN("fail to check restore source", KR(ret), K_(tenant_id)); + } else if (!has_restore_source_) { + LOG_INFO("no restore source", K(tenant_id), K(tenant_info)); + } else if (OB_FAIL(check_sync_to_latest_do_while_(tenant_info, only_check_sys_ls))) { + LOG_WARN("fail to check whether all ls are synced", KR(ret), K(tenant_id), K(tenant_info)); + } + int64_t wait_log_sync = ObTimeUtility::current_time() - begin_time; + LOG_INFO("wait tenant sync to latest", KR(ret), K(has_restore_source_), K(wait_log_sync)); + if (OB_LIKELY(NULL != cost_detail_)) { + (void) cost_detail_->add_cost(ObTenantRoleTransCostDetail::WAIT_LOG_SYNC, wait_log_sync); + } + CLUSTER_EVENT_ADD_LOG(ret, "wait sync to latest end", + "tenant id", tenant_id, + "switchover#", tenant_info.get_switchover_epoch(), + "finished", OB_SUCC(ret) ? "yes" : "no", + "cost sec", wait_log_sync / SEC_UNIT); + return ret; +} + +int ObTenantRoleTransitionService::check_restore_source_for_switchover_to_primary_(const uint64_t tenant_id) { int ret = OB_SUCCESS; ObLogRestoreSourceMgr restore_source_mgr; ObLogRestoreSourceItem item; - bool has_restore_source = true; - int64_t begin_time = ObTimeUtility::current_time(); + ObSqlString standby_source_value; + ObRestoreSourceServiceAttr service_attr; + has_restore_source_ = true; + ObLogRestoreSourceType restore_type; if (OB_FAIL(check_inner_stat())) { LOG_WARN("inner stat error", KR(ret)); } else if (OB_FAIL(restore_source_mgr.init(tenant_id, sql_proxy_))) { @@ -1205,56 +1556,103 @@ int ObTenantRoleTransitionService::wait_tenant_sync_to_latest_until_timeout_( // When restore_source fails, in order to proceed switchover. If no restore_source is set, // do not check sync with restore_source LOG_INFO("failed to get_source", KR(ret), K(tenant_id), K(tenant_id)); - has_restore_source = false; + has_restore_source_ = false; ret = OB_SUCCESS; } - } - - if (OB_FAIL(ret) || !has_restore_source) { + } else if (OB_UNLIKELY(!item.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("log restore source item is invalid"); + } else if (share::is_location_log_source_type(item.type_)) { + // archive mode, cannot check whether previous primary tenant becomes standby + } else if (OB_FAIL(standby_source_value.assign(item.value_))) { + LOG_WARN("fail to assign standby source value", K(item.value_)); + } else if (OB_FAIL(service_attr.parse_service_attr_from_str(standby_source_value))) { + LOG_WARN("fail to parse service attr", K(item), K(standby_source_value)); } else { - bool has_sync_to_latest = false; - while (!THIS_WORKER.is_timeout() && !logservice::ObLogRestoreHandler::need_fail_when_switch_to_primary(ret)) { - has_sync_to_latest = false; - if (OB_FAIL(check_sync_to_latest_(tenant_id, tenant_info, has_sync_to_latest))) { - LOG_WARN("fail to check_sync_to_latest_", KR(ret), K(tenant_id), - K(tenant_info), K(has_sync_to_latest)); - } else if (has_sync_to_latest) { - LOG_INFO("sync to latest", K(has_sync_to_latest), K(tenant_id)); - break; - } else { - LOG_WARN("not sync to latest, wait a while", K(tenant_id)); + // net service mode, check whether previous primary tenant becomes standby + share::ObTenantRole tenant_role; + share::schema::ObTenantStatus tenant_status; + ObTenantSwitchoverStatus switchover_status; + SMART_VAR(share::ObLogRestoreProxyUtil, proxy_util) { + if (OB_FAIL(proxy_util.init_with_service_attr(tenant_id, &service_attr))) { + LOG_WARN("fail to init proxy_util", KR(ret), K(service_attr)); + } else if (OB_FAIL(proxy_util.get_tenant_info(tenant_role, tenant_status, switchover_status))) { + LOG_WARN("fail to get tenant info", KR(ret), K(service_attr)); + } else if (OB_UNLIKELY(!tenant_role.is_standby())) { + ret = OB_OP_NOT_ALLOW; + LOG_WARN("tenant role not match", KR(ret), K(tenant_role), K(service_attr)); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "log restore source is primary, switchover to primary is"); + } else if (OB_UNLIKELY(share::schema::ObTenantStatus::TENANT_STATUS_NORMAL != tenant_status)) { + ret = OB_OP_NOT_ALLOW; + LOG_WARN("tenant status not match", KR(ret), K(tenant_status), K(service_attr)); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "log restore source is not in normal status, switchover to primary is"); + } else if (OB_UNLIKELY(!switchover_status.is_normal_status())) { + ret = OB_OP_NOT_ALLOW; + LOG_WARN("tenant switchover status not match", KR(ret), K(switchover_status), K(service_attr)); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "log restore source is not in normal switchover status, switchover to primary is"); } - usleep(10L * 1000L); } - if (logservice::ObLogRestoreHandler::need_fail_when_switch_to_primary(ret)) { - } else if (THIS_WORKER.is_timeout() || !has_sync_to_latest) { - // return NOT_ALLOW instead of timeout - ret = OB_OP_NOT_ALLOW; - LOG_WARN("has not sync to latest, can not swithover to primary", KR(ret)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "not sync to latest, switchover to primary"); - } else if (OB_SUCC(ret)) { - LOG_INFO("finish check sync to latest", K(has_sync_to_latest)); - } - - int64_t cost = ObTimeUtility::current_time() - begin_time; - LOG_INFO("check sync to latest", KR(ret), K(cost), - "is_pass", has_sync_to_latest); - CLUSTER_EVENT_ADD_LOG(ret, "wait sync to latest end", - "tenant id", tenant_id, - "switchover#", tenant_info.get_switchover_epoch(), - "finished", OB_SUCC(ret) ? "yes" : "no", - "cost sec", cost / SEC_UNIT); } return ret; } -int ObTenantRoleTransitionService::check_sync_to_latest_(const uint64_t tenant_id, - const ObAllTenantInfo &tenant_info, - bool &has_sync_to_latest) +int ObTenantRoleTransitionService::check_sync_to_latest_do_while_( + const ObAllTenantInfo &tenant_info, + const bool only_check_sys_ls) { int ret = OB_SUCCESS; - int64_t begin_time = ObTimeUtility::current_time(); - has_sync_to_latest = false; + bool is_synced = false; + const uint64_t tenant_id = tenant_info.get_tenant_id(); + while (!THIS_WORKER.is_timeout() && !logservice::ObLogRestoreHandler::need_fail_when_switch_to_primary(ret)) { + bool is_all_ls_synced = false; + bool is_sys_ls_synced = false; + ret = OB_SUCCESS; + if (OB_FAIL(check_sync_to_latest_(tenant_id, only_check_sys_ls, tenant_info, is_sys_ls_synced, is_all_ls_synced))) { + LOG_WARN("fail to execute check_sync_to_latest_", KR(ret), K(tenant_id), K(only_check_sys_ls), K(tenant_info)); + } else { + is_synced = only_check_sys_ls ? is_sys_ls_synced : is_all_ls_synced; + if (is_synced) { + LOG_INFO("sync to latest", K(tenant_id), K(only_check_sys_ls), K(is_synced), + K(is_sys_ls_synced), K(is_all_ls_synced)); + break; + } else { + LOG_WARN("not sync to latest, wait a while", K(tenant_id), K(only_check_sys_ls)); + } + } + usleep(10L * 1000L); + } + if (logservice::ObLogRestoreHandler::need_fail_when_switch_to_primary(ret)) { + } else if (THIS_WORKER.is_timeout() || !is_synced) { + // return NOT_ALLOW instead of timeout + if (OB_SUCC(ret)) { + ret = OB_TIMEOUT; // to print in err_msg + } + ObSqlString err_msg; + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(err_msg.assign_fmt("wait tenant sync to latest failed(original error code: %d), switchover to primary is", ret))) { + LOG_WARN("fail to assign error msg", KR(ret), KR(tmp_ret)); + } else { + // convert OB_TIMEOUT or other failure code to OB_OP_NOT_ALLOW + ret = OB_OP_NOT_ALLOW; + LOG_WARN("has not sync to latest, can not swithover to primary", KR(ret), K(only_check_sys_ls)); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, err_msg.ptr()); + } + } + if (OB_SUCC(ret)) { + LOG_INFO("finish check sync to latest", K(only_check_sys_ls), K(is_synced)); + } + return ret; +} +int ObTenantRoleTransitionService::check_sync_to_latest_( + const uint64_t tenant_id, + const bool only_check_sys_ls, + const ObAllTenantInfo &tenant_info, + bool &is_sys_ls_synced, + bool &is_all_ls_synced) +{ + int ret = OB_SUCCESS; + int64_t begin_ts = ObTimeUtility::current_time(); + is_all_ls_synced = false; ObLSStatusOperator ls_status_op; share::ObLSStatusInfoArray all_ls_status_array; share::ObLSStatusInfoArray sys_ls_status_array; @@ -1262,11 +1660,11 @@ int ObTenantRoleTransitionService::check_sync_to_latest_(const uint64_t tenant_i ObLSRecoveryStatOperator ls_recovery_operator; ObLSRecoveryStat sys_ls_recovery_stat; SCN sys_ls_sync_scn = SCN::min_scn(); - bool sys_ls_sync_to_latest = false; + bool sys_ls_sync_has_all_log = false; share::ObLSStatusInfo ls_status; - + ObTenantRoleTransNonSyncInfo non_sync_info; LOG_INFO("start to check_sync_to_latest", KR(ret), K(tenant_id), K(tenant_info)); - + is_sys_ls_synced = false; if (!is_user_tenant(tenant_id)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(tenant_id)); @@ -1279,13 +1677,20 @@ int ObTenantRoleTransitionService::check_sync_to_latest_(const uint64_t tenant_i tenant_id, true /*need_check_sync_to_latest*/, sys_ls_sync_scn, - sys_ls_sync_to_latest))) { - LOG_WARN("failed to get_sys_ls_sync_scn_", KR(ret), K(switchover_checkpoints)); - } else if (!(sys_ls_sync_scn.is_valid_and_not_min() && sys_ls_sync_to_latest - && sys_ls_recovery_stat.get_sync_scn() == sys_ls_sync_scn)) { - LOG_WARN("sys ls not sync, keep waiting", KR(ret), K(sys_ls_sync_scn), K(sys_ls_sync_to_latest), - K(sys_ls_recovery_stat), K(switchover_checkpoints)); - // SYS LS is sync, check other LS + sys_ls_sync_has_all_log))) { + LOG_WARN("failed to get_sys_ls_sync_scn_", KR(ret)); + } else { + is_sys_ls_synced = (sys_ls_sync_scn.is_valid_and_not_min() && sys_ls_sync_has_all_log + && sys_ls_recovery_stat.get_sync_scn() == sys_ls_sync_scn); + } + if (OB_FAIL(ret)) { + } else if (only_check_sys_ls) { + // do nothing + } else if (OB_UNLIKELY(!is_sys_ls_synced)) { + // we have checked sys ls when only_check_sys_ls is true, + // now it should be synced + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sys ls not sync", KR(ret), K(only_check_sys_ls), K(is_sys_ls_synced), K(tenant_info)); } else if (OB_FAIL(ls_status_op.get_all_ls_status_by_order_for_switch_tenant(tenant_id, true/* ignore_need_create_abort */, all_ls_status_array, *sql_proxy_))) { LOG_WARN("failed to get_all_ls_status_by_order", KR(ret), K(tenant_id)); @@ -1297,28 +1702,28 @@ int ObTenantRoleTransitionService::check_sync_to_latest_(const uint64_t tenant_i LOG_WARN("unexpect checkpoints count", KR(ret), K(switchover_checkpoints), K(tenant_id), K(tenant_info), K(all_ls_status_array)); } else { - has_sync_to_latest = true; - for (int64_t i = 0; i < switchover_checkpoints.count() && OB_SUCC(ret) && has_sync_to_latest; i++) { - const auto &checkpoint = switchover_checkpoints.at(i); - if (checkpoint.is_sync_to_latest()) { - // LS is sync - } else { - has_sync_to_latest = false; - LOG_WARN("ls not sync, keep waiting", KR(ret), K(checkpoint)); - } - }//end for + lib::ob_sort(switchover_checkpoints.begin(), switchover_checkpoints.end()); + if (OB_FAIL(non_sync_info.init(switchover_checkpoints))) { + LOG_WARN("fail to init non_sync_info", KR(ret), K(switchover_checkpoints)); + } else { + is_all_ls_synced = non_sync_info.is_sync(); + } } - - LOG_INFO("check sync to latest", KR(ret)); - int64_t cost = ObTimeUtility::current_time() - begin_time; - if (REACH_TIME_INTERVAL(PRINT_INTERVAL) || has_sync_to_latest) { + int64_t cost = ObTimeUtility::current_time() - begin_ts; + LOG_INFO("check sync to latest", KR(ret), K(is_verify_), K(tenant_id), K(cost), K(only_check_sys_ls), + K(is_sys_ls_synced), K(is_all_ls_synced), K(non_sync_info)); + if (is_verify_ || only_check_sys_ls) { + } else if (REACH_TIME_INTERVAL(PRINT_INTERVAL) || is_all_ls_synced) { + TENANT_EVENT(tenant_id, TENANT_ROLE_CHANGE, WAIT_LOG_SYNC, begin_ts, + ret, cost, is_sys_ls_synced ? "YES" : "NO", + is_all_ls_synced ? "YES" : "NO", non_sync_info); CLUSTER_EVENT_ADD_LOG(ret, "wait tenant sync from latest", - "tenant id", tenant_id, - "is sync", has_sync_to_latest ? "yes" : "no", - "switchover#", tenant_info.get_switchover_epoch(), - "finished", OB_SUCC(ret) ? "yes" : "no", - "checkpoint", switchover_checkpoints, - "cost sec", cost / SEC_UNIT); + "tenant id", tenant_id, + "is sync", is_all_ls_synced ? "yes" : "no", + "switchover#", tenant_info.get_switchover_epoch(), + "finished", OB_SUCC(ret) ? "yes" : "no", + "checkpoint", switchover_checkpoints, + "cost sec", cost / SEC_UNIT); } return ret; } @@ -1603,25 +2008,27 @@ int ObTenantRoleTransitionService::check_tenant_server_online_() } else if (0 != temporary_offline_servers.count()) { ret = OB_OP_NOT_ALLOW; LOG_WARN("the tenant has units on temporary offline servers", KR(ret), K(tenant_id_), K(temporary_offline_servers)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "the tenant has units on temporary offline servers, switch to primary"); + TENANT_ROLE_TRANS_USER_ERR_WITH_SUFFIX(OB_OP_NOT_ALLOW, + "the tenant has units on temporary offline servers", switch_optype_); } else if (0 != permanent_offline_servers.count()) { bool exists = false; if (OB_FAIL(ObEmptyServerChecker::check_if_tenant_ls_replicas_exist_in_servers( tenant_id_, permanent_offline_servers, exists))) { - LOG_WARN("fail to check if the tenant's ls_replicas exist in permanent_offline_servers", + LOG_WARN("fail to check if the tenant's LS replicas exist in permanent_offline_servers", KR(ret), K(tenant_id_), K(permanent_offline_servers)); if (OB_LEADER_NOT_EXIST == ret) { ret = OB_OP_NOT_ALLOW; - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "the tenant has ls replicas without leader, switch to primary"); + TENANT_ROLE_TRANS_USER_ERR_WITH_SUFFIX(OB_OP_NOT_ALLOW, + "the tenant has LS replicas without leader", switch_optype_); } } else if (exists) { ret = OB_OP_NOT_ALLOW; - LOG_WARN("the tenant has ls replicas on at least one of the permanent offline servers", + LOG_WARN("the tenant has LS replicas on at least one of the permanent offline servers", KR(ret), K(tenant_id_), K(exists), K(permanent_offline_servers)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, - "the tenant has ls replicas on at least one of the permanent offline servers, switch to primary"); + TENANT_ROLE_TRANS_USER_ERR_WITH_SUFFIX(OB_OP_NOT_ALLOW, + "the tenant has LS replicas on at least one of the permanent offline servers", switch_optype_); } } return ret; diff --git a/src/rootserver/ob_tenant_role_transition_service.h b/src/rootserver/standby/ob_tenant_role_transition_service.h similarity index 71% rename from src/rootserver/ob_tenant_role_transition_service.h rename to src/rootserver/standby/ob_tenant_role_transition_service.h index 1a8662384..d21f8ec1a 100644 --- a/src/rootserver/ob_tenant_role_transition_service.h +++ b/src/rootserver/standby/ob_tenant_role_transition_service.h @@ -67,20 +67,83 @@ public: template int do_nonblock_renew(const ARRAY &array_l, const ARRAY &array_r, const uint64_t tenant_id); +struct ObTenantRoleTransCostDetail +{ +public: + enum CostType { + WAIT_LOG_SYNC = 0, + WAIT_BALANCE_TASK, + LOG_FLASHBACK, + WAIT_LOG_END, + CHANGE_ACCESS_MODE, + MAX_COST_TYPE + }; + const char* type_to_str(CostType type) const; +public: + ObTenantRoleTransCostDetail() : cost_type_{}, start_(0), end_(0) {} + ~ObTenantRoleTransCostDetail() {} + void set_start(int64_t start) { start_ = start; } + void add_cost(CostType type, int64_t cost); + void set_end(int64_t end) { end_ = end; } + int64_t get_wait_log_end () { return cost_type_[WAIT_LOG_END]; } + int64_t to_string (char *buf, const int64_t buf_len) const ; +private: + int64_t cost_type_[MAX_COST_TYPE]; + int64_t start_; + int64_t end_; +}; + +struct ObTenantRoleTransAllLSInfo +{ +public: + ObTenantRoleTransAllLSInfo() : all_ls_{} {} + ~ObTenantRoleTransAllLSInfo() {} + int init(); + int add_ls(const ObLSID &ls_id, const ObLSStatus status); + int64_t to_string (char *buf, const int64_t buf_len) const; + bool is_valid() const; +private: + ObArray all_ls_[ObLSStatus::OB_LS_MAX_STATUS]; +}; + +struct ObTenantRoleTransNonSyncInfo +{ +public: + ObTenantRoleTransNonSyncInfo() : is_sync_(true), not_sync_checkpoints_() {} + ~ObTenantRoleTransNonSyncInfo() {} + int init(const ObArray &switchover_checkpoints); + int64_t to_string (char *buf, const int64_t buf_len) const; + bool is_sync() const { return is_sync_; } +private: + static constexpr int64_t MAX_PRINT_LS_NUM = 5; + bool is_sync_; + ObArray not_sync_checkpoints_; +}; + /*description: * for primary to standby and standby to primary */ class ObTenantRoleTransitionService { public: - ObTenantRoleTransitionService(const uint64_t tenant_id, + ObTenantRoleTransitionService() + : tenant_id_(OB_INVALID_TENANT_ID), sql_proxy_(NULL), + rpc_proxy_(NULL), switchover_epoch_(OB_INVALID_VERSION), + switch_optype_(obrpc::ObSwitchTenantArg::OpType::INVALID), + so_scn_(), + cost_detail_(NULL), + all_ls_info_(NULL), + has_restore_source_(false), + is_verify_(false) {} + virtual ~ObTenantRoleTransitionService() {} + int init( + uint64_t tenant_id, + const obrpc::ObSwitchTenantArg::OpType &switch_optype, + const bool is_verify, common::ObMySQLProxy *sql_proxy, obrpc::ObSrvRpcProxy *rpc_proxy, - const obrpc::ObSwitchTenantArg::OpType &switch_optype) - : tenant_id_(tenant_id), sql_proxy_(sql_proxy), - rpc_proxy_(rpc_proxy), switchover_epoch_(OB_INVALID_VERSION), - switch_optype_(switch_optype) {} - virtual ~ObTenantRoleTransitionService() {} + ObTenantRoleTransCostDetail *cost_detail, + ObTenantRoleTransAllLSInfo *all_ls_info); int failover_to_primary(); int check_inner_stat(); int do_switch_access_mode_to_append(const share::ObAllTenantInfo &tenant_info, @@ -93,7 +156,6 @@ public: { switchover_epoch_ = switchover_epoch; } - /** * @description: * Update scn/tenant_role/switchover status when switchover is executed @@ -116,6 +178,7 @@ public: const int64_t old_switchover_epoch, ObAllTenantInfo &new_tenant_info); + int wait_sys_ls_sync_to_latest_until_timeout_(const uint64_t tenant_id, ObAllTenantInfo &tenant_info); /** * @description: * wait tenant sync to switchover checkpoint until timeout @@ -123,8 +186,8 @@ public: * @param[in] primary_checkpoints primary switchover checkpoint * @return return code */ - int wait_tenant_sync_to_latest_until_timeout_(const uint64_t tenant_id, - const ObAllTenantInfo &tenant_info); + int wait_tenant_sync_to_latest_until_timeout_(const uint64_t tenant_id, const ObAllTenantInfo &tenant_info); + int check_restore_source_for_switchover_to_primary_(const uint64_t tenant_id); /** * @description: @@ -155,6 +218,7 @@ public: const bool get_latest_scn, ObIArray &checkpoints ); + share::SCN get_so_scn() const { return so_scn_; } private: int do_failover_to_primary_(const share::ObAllTenantInfo &tenant_info); @@ -175,8 +239,6 @@ private: const palf::AccessMode target_access_mode, const share::SCN &ref_scn, const share::SCN &sys_ls_sync_scn); - int do_switch_access_mode_to_flashback( - const share::ObAllTenantInfo &tenant_info); /** * @description: @@ -204,21 +266,36 @@ private: const ObIArray &checkpoints, share::SCN &sys_ls_sync_scn, bool &is_sync_to_latest); + /** + * @description: + * wait tenant/sys ls sync to switchover checkpoint until timeout + * @param[in] tenant_id + * @param[in] only_check_sys_ls true: only wait sys ls sync; false: wait tenant sync + * @return return code + */ + int check_sync_to_latest_do_while_( + const ObAllTenantInfo &tenant_info, + const bool only_check_sys_ls); /** * @description: * when switch to primary, check all ls are sync to latest * @param[in] tenant_id the tenant id to check * @param[in] tenant_info - * @param[out] has_sync_to_latest whether sync to latest + * @param[out] is_all_ls_synced whether sync to latest * @return return code */ - int check_sync_to_latest_(const uint64_t tenant_id, - const ObAllTenantInfo &tenant_info, - bool &has_sync_to_latest); + int check_sync_to_latest_( + const uint64_t tenant_id, + const bool only_check_sys_ls, + const ObAllTenantInfo &tenant_info, + bool &is_sys_ls_synced, + bool &is_all_ls_synced); int do_prepare_flashback_for_switch_to_primary_(share::ObAllTenantInfo &tenant_info); int do_prepare_flashback_for_failover_to_primary_(share::ObAllTenantInfo &tenant_info); + int clear_service_name_(); + int double_check_service_name_(const share::ObAllTenantInfo &tenant_info); int check_and_update_sys_ls_recovery_stat_in_switchover_( const uint64_t tenant_id, const bool switch_to_primary, @@ -236,10 +313,11 @@ private: common::sqlclient::ObMySQLResult &res, ObArray &temporary_offline_servers, ObArray &permanent_offline_servers); + int ls_status_stats_when_change_access_mode_(const share::ObLSStatusInfoArray &status_info_array); private: const static int64_t SEC_UNIT = 1000L * 1000L; - const static int64_t PRINT_INTERVAL = 10 * 1000 * 1000L; + const static int64_t PRINT_INTERVAL = 1000L * 1000L; private: uint64_t tenant_id_; @@ -247,6 +325,11 @@ private: obrpc::ObSrvRpcProxy *rpc_proxy_; int64_t switchover_epoch_; obrpc::ObSwitchTenantArg::OpType switch_optype_; + share::SCN so_scn_; + ObTenantRoleTransCostDetail *cost_detail_; + ObTenantRoleTransAllLSInfo *all_ls_info_; + bool has_restore_source_; + bool is_verify_; }; } } diff --git a/src/share/CMakeLists.txt b/src/share/CMakeLists.txt index b83f6be79..eb2f4c2c1 100644 --- a/src/share/CMakeLists.txt +++ b/src/share/CMakeLists.txt @@ -188,13 +188,13 @@ ob_set_subtarget(ob_share common ob_inner_kv_table_operator.cpp ob_inner_table_operator.cpp ob_standby_upgrade.cpp - ob_primary_standby_service.cpp ob_common_id.cpp ob_cluster_event_history_table_operator.cpp scn.cpp ob_throttling_utils.cpp ob_storage_ha_diagnose_struct.cpp ob_storage_ha_diagnose_operator.cpp + ob_service_name_proxy.cpp ob_compatibility_control.cpp ) diff --git a/src/share/inner_table/ob_inner_table_schema.12451_12500.cpp b/src/share/inner_table/ob_inner_table_schema.12451_12500.cpp index 72b383d92..2a17c036a 100644 --- a/src/share/inner_table/ob_inner_table_schema.12451_12500.cpp +++ b/src/share/inner_table/ob_inner_table_schema.12451_12500.cpp @@ -3955,6 +3955,142 @@ int ObInnerTableSchema::all_virtual_user_proxy_role_info_history_schema(ObTableS return ret; } +int ObInnerTableSchema::all_virtual_service_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_SERVICE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(VIRTUAL_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_SERVICE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("service_name_id", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("gmt_create", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("gmt_modified", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("service_name", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_SERVICE_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("service_status", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + 64, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::all_virtual_tenant_resource_limit_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.15401_15450.cpp b/src/share/inner_table/ob_inner_table_schema.15401_15450.cpp index 818ee9021..75ad58195 100644 --- a/src/share/inner_table/ob_inner_table_schema.15401_15450.cpp +++ b/src/share/inner_table/ob_inner_table_schema.15401_15450.cpp @@ -7920,6 +7920,140 @@ int ObInnerTableSchema::all_virtual_user_proxy_role_info_real_agent_ora_schema(O return ret; } +int ObInnerTableSchema::all_virtual_service_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_SERVICE_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(VIRTUAL_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_SERVICE_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SERVICE_NAME_ID", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("GMT_CREATE", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampLTZType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("GMT_MODIFIED", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampLTZType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SERVICE_NAME", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_SERVICE_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SERVICE_STATUS", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + 64, //column_length + 2, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + } // end namespace share } // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema.21201_21250.cpp b/src/share/inner_table/ob_inner_table_schema.21201_21250.cpp index bd054eca2..569d150a6 100644 --- a/src/share/inner_table/ob_inner_table_schema.21201_21250.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21201_21250.cpp @@ -1060,7 +1060,7 @@ int ObInnerTableSchema::gv_ob_processlist_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT SVR_IP, SVR_PORT, SQL_PORT, ID, USER, HOST, DB, TENANT, COMMAND, TIME, TOTAL_TIME, STATE, INFO, PROXY_SESSID, MASTER_SESSID, USER_CLIENT_IP, USER_HOST, RETRY_CNT, RETRY_INFO, SQL_ID, TRANS_ID, THREAD_ID, SSL_CIPHER, TRACE_ID, TRANS_STATE, ACTION, MODULE, CLIENT_INFO, LEVEL, SAMPLE_PERCENTAGE, RECORD_POLICY, LB_VID, LB_VIP, LB_VPORT, IN_BYTES, OUT_BYTES, USER_CLIENT_PORT, cast(total_cpu_time as SIGNED) as TOTAL_CPU_TIME, PROXY_USER, TOP_INFO FROM oceanbase.__all_virtual_processlist )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT SVR_IP, SVR_PORT, SQL_PORT, ID, USER, HOST, DB, TENANT, COMMAND, TIME, TOTAL_TIME, STATE, INFO, PROXY_SESSID, MASTER_SESSID, USER_CLIENT_IP, USER_HOST, RETRY_CNT, RETRY_INFO, SQL_ID, TRANS_ID, THREAD_ID, SSL_CIPHER, TRACE_ID, TRANS_STATE, ACTION, MODULE, CLIENT_INFO, LEVEL, SAMPLE_PERCENTAGE, RECORD_POLICY, LB_VID, LB_VIP, LB_VPORT, IN_BYTES, OUT_BYTES, USER_CLIENT_PORT, PROXY_USER, SERVICE_NAME, cast(total_cpu_time as SIGNED) as TOTAL_CPU_TIME, TOP_INFO FROM oceanbase.__all_virtual_processlist )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -1110,7 +1110,7 @@ int ObInnerTableSchema::v_ob_processlist_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT SVR_IP, SVR_PORT, SQL_PORT, ID, USER, HOST, DB, TENANT, COMMAND, TIME, TOTAL_TIME, STATE, INFO, PROXY_SESSID, MASTER_SESSID, USER_CLIENT_IP, USER_HOST, RETRY_CNT, RETRY_INFO, SQL_ID, TRANS_ID, THREAD_ID, SSL_CIPHER, TRACE_ID, TRANS_STATE, ACTION, MODULE, CLIENT_INFO, LEVEL, SAMPLE_PERCENTAGE, RECORD_POLICY, LB_VID, LB_VIP, LB_VPORT, IN_BYTES, OUT_BYTES, USER_CLIENT_PORT, cast(total_cpu_time as SIGNED) as TOTAL_CPU_TIME, PROXY_USER, TOP_INFO FROM oceanbase.GV$OB_PROCESSLIST WHERE SVR_IP = host_ip() AND SVR_PORT = rpc_port() )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT SVR_IP, SVR_PORT, SQL_PORT, ID, USER, HOST, DB, TENANT, COMMAND, TIME, TOTAL_TIME, STATE, INFO, PROXY_SESSID, MASTER_SESSID, USER_CLIENT_IP, USER_HOST, RETRY_CNT, RETRY_INFO, SQL_ID, TRANS_ID, THREAD_ID, SSL_CIPHER, TRACE_ID, TRANS_STATE, ACTION, MODULE, CLIENT_INFO, LEVEL, SAMPLE_PERCENTAGE, RECORD_POLICY, LB_VID, LB_VIP, LB_VPORT, IN_BYTES, OUT_BYTES, USER_CLIENT_PORT, PROXY_USER, SERVICE_NAME, cast(total_cpu_time as SIGNED) as TOTAL_CPU_TIME, TOP_INFO FROM oceanbase.GV$OB_PROCESSLIST WHERE SVR_IP = host_ip() AND SVR_PORT = rpc_port() )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema.21501_21550.cpp b/src/share/inner_table/ob_inner_table_schema.21501_21550.cpp index 625cddb4f..edcf87ec1 100644 --- a/src/share/inner_table/ob_inner_table_schema.21501_21550.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21501_21550.cpp @@ -1875,6 +1875,106 @@ int ObInnerTableSchema::v_ob_compatibility_control_schema(ObTableSchema &table_s return ret; } +int ObInnerTableSchema::dba_ob_services_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_OB_SERVICES_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_OB_SERVICES_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT gmt_create AS CREATE_TIME, gmt_modified AS MODIFIED_TIME, SERVICE_NAME_ID, SERVICE_NAME, SERVICE_STATUS FROM oceanbase.__all_virtual_service WHERE TENANT_ID=EFFECTIVE_TENANT_ID(); )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::cdb_ob_services_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_CDB_OB_SERVICES_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_CDB_OB_SERVICES_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT TENANT_ID, gmt_create AS `CREATE_TIME`, gmt_modified AS 'MODIFIED_TIME', SERVICE_NAME_ID, SERVICE_NAME, SERVICE_STATUS FROM oceanbase.__all_virtual_service )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + } // end namespace share } // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema.25301_25350.cpp b/src/share/inner_table/ob_inner_table_schema.25301_25350.cpp index ce518e91a..eadddef4d 100644 --- a/src/share/inner_table/ob_inner_table_schema.25301_25350.cpp +++ b/src/share/inner_table/ob_inner_table_schema.25301_25350.cpp @@ -125,6 +125,56 @@ int ObInnerTableSchema::proxy_users_schema(ObTableSchema &table_schema) return ret; } +int ObInnerTableSchema::dba_ob_services_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_OB_SERVICES_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_OB_SERVICES_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT gmt_create AS "CREATE_TIME", gmt_modified AS "MODIFIED_TIME", SERVICE_NAME_ID, SERVICE_NAME, SERVICE_STATUS FROM SYS.ALL_VIRTUAL_SERVICE WHERE TENANT_ID=EFFECTIVE_TENANT_ID(); )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + } // end namespace share } // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema.28101_28150.cpp b/src/share/inner_table/ob_inner_table_schema.28101_28150.cpp index fd2f22643..40a0107dd 100644 --- a/src/share/inner_table/ob_inner_table_schema.28101_28150.cpp +++ b/src/share/inner_table/ob_inner_table_schema.28101_28150.cpp @@ -960,7 +960,7 @@ int ObInnerTableSchema::gv_ob_processlist_ora_schema(ObTableSchema &table_schema table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT SVR_IP, SVR_PORT, SQL_PORT, ID, "USER", HOST, DB, TENANT, COMMAND, TIME, TOTAL_TIME, STATE, INFO, PROXY_SESSID, MASTER_SESSID, USER_CLIENT_IP, USER_HOST, RETRY_CNT, RETRY_INFO, SQL_ID, TRANS_ID, THREAD_ID, SSL_CIPHER, TRACE_ID, TRANS_STATE, ACTION, MODULE, CLIENT_INFO, "LEVEL", SAMPLE_PERCENTAGE, RECORD_POLICY, LB_VID, LB_VIP, LB_VPORT, IN_BYTES, OUT_BYTES, USER_CLIENT_PORT, PROXY_USER, CAST(total_cpu_time AS INT) as TOTAL_CPU_TIME, TOP_INFO FROM SYS.ALL_VIRTUAL_PROCESSLIST )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT SVR_IP, SVR_PORT, SQL_PORT, ID, "USER", HOST, DB, TENANT, COMMAND, TIME, TOTAL_TIME, STATE, INFO, PROXY_SESSID, MASTER_SESSID, USER_CLIENT_IP, USER_HOST, RETRY_CNT, RETRY_INFO, SQL_ID, TRANS_ID, THREAD_ID, SSL_CIPHER, TRACE_ID, TRANS_STATE, ACTION, MODULE, CLIENT_INFO, "LEVEL", SAMPLE_PERCENTAGE, RECORD_POLICY, LB_VID, LB_VIP, LB_VPORT, IN_BYTES, OUT_BYTES, USER_CLIENT_PORT, PROXY_USER, SERVICE_NAME, CAST(total_cpu_time AS INT) as TOTAL_CPU_TIME, TOP_INFO FROM SYS.ALL_VIRTUAL_PROCESSLIST )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -1010,7 +1010,7 @@ int ObInnerTableSchema::v_ob_processlist_ora_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT SVR_IP, SVR_PORT, SQL_PORT, ID, "USER", HOST, DB, TENANT, COMMAND, TIME, TOTAL_TIME, STATE, INFO, PROXY_SESSID, MASTER_SESSID, USER_CLIENT_IP, USER_HOST, RETRY_CNT, RETRY_INFO, SQL_ID, TRANS_ID, THREAD_ID, SSL_CIPHER, TRACE_ID, TRANS_STATE, ACTION, MODULE, CLIENT_INFO, "LEVEL", SAMPLE_PERCENTAGE, RECORD_POLICY, LB_VID, LB_VIP, LB_VPORT, IN_BYTES, OUT_BYTES, USER_CLIENT_PORT, PROXY_USER, CAST(total_cpu_time AS INT) as TOTAL_CPU_TIME, TOP_INFO FROM SYS.GV$OB_PROCESSLIST WHERE SVR_IP = host_ip() AND SVR_PORT = rpc_port() )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT SVR_IP, SVR_PORT, SQL_PORT, ID, "USER", HOST, DB, TENANT, COMMAND, TIME, TOTAL_TIME, STATE, INFO, PROXY_SESSID, MASTER_SESSID, USER_CLIENT_IP, USER_HOST, RETRY_CNT, RETRY_INFO, SQL_ID, TRANS_ID, THREAD_ID, SSL_CIPHER, TRACE_ID, TRANS_STATE, ACTION, MODULE, CLIENT_INFO, "LEVEL", SAMPLE_PERCENTAGE, RECORD_POLICY, LB_VID, LB_VIP, LB_VPORT, IN_BYTES, OUT_BYTES, USER_CLIENT_PORT, PROXY_USER, SERVICE_NAME, CAST(total_cpu_time AS INT) as TOTAL_CPU_TIME, TOP_INFO FROM SYS.GV$OB_PROCESSLIST WHERE SVR_IP = host_ip() AND SVR_PORT = rpc_port() )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema.501_550.cpp b/src/share/inner_table/ob_inner_table_schema.501_550.cpp index 21590a612..15edca185 100644 --- a/src/share/inner_table/ob_inner_table_schema.501_550.cpp +++ b/src/share/inner_table/ob_inner_table_schema.501_550.cpp @@ -2233,6 +2233,158 @@ int ObInnerTableSchema::all_user_proxy_role_info_history_schema(ObTableSchema &t return ret; } +int ObInnerTableSchema::all_service_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_SERVICE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_SERVICE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ObObj gmt_create_default; + ObObj gmt_create_default_null; + + gmt_create_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_create_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("gmt_create", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_BINARY,//collation_type + 0, //column length + -1, //column_precision + 6, //column_scale + true,//is nullable + false, //is_autoincrement + false, //is_on_update_for_timestamp + gmt_create_default_null, + gmt_create_default) + } + + if (OB_SUCC(ret)) { + ObObj gmt_modified_default; + ObObj gmt_modified_default_null; + + gmt_modified_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_modified_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("gmt_modified", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_BINARY,//collation_type + 0, //column length + -1, //column_precision + 6, //column_scale + true,//is nullable + false, //is_autoincrement + true, //is_on_update_for_timestamp + gmt_modified_default_null, + gmt_modified_default) + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("service_name_id", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("service_name", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_SERVICE_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("service_status", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + 64, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_SERVICE_TID); + table_schema.set_aux_lob_meta_tid(OB_ALL_SERVICE_AUX_LOB_META_TID); + table_schema.set_aux_lob_piece_tid(OB_ALL_SERVICE_AUX_LOB_PIECE_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::all_mview_dep_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.50501_50550.cpp b/src/share/inner_table/ob_inner_table_schema.50501_50550.cpp index d60eb1b9a..a6abc694c 100644 --- a/src/share/inner_table/ob_inner_table_schema.50501_50550.cpp +++ b/src/share/inner_table/ob_inner_table_schema.50501_50550.cpp @@ -1375,6 +1375,141 @@ int ObInnerTableSchema::all_user_proxy_role_info_history_aux_lob_meta_schema(ObT return ret; } +int ObInnerTableSchema::all_service_aux_lob_meta_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_SERVICE_AUX_LOB_META_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(AUX_LOB_META); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_SERVICE_AUX_LOB_META_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 16, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("seq_id", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 8192, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("binary_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("char_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("piece_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_data", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 262144, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_SERVICE_AUX_LOB_META_TID); + table_schema.set_data_table_id(OB_ALL_SERVICE_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::all_mview_dep_aux_lob_meta_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.60501_60550.cpp b/src/share/inner_table/ob_inner_table_schema.60501_60550.cpp index 04d44f0e6..16db453ce 100644 --- a/src/share/inner_table/ob_inner_table_schema.60501_60550.cpp +++ b/src/share/inner_table/ob_inner_table_schema.60501_60550.cpp @@ -925,6 +925,96 @@ int ObInnerTableSchema::all_user_proxy_role_info_history_aux_lob_piece_schema(Ob return ret; } +int ObInnerTableSchema::all_service_aux_lob_piece_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_SERVICE_AUX_LOB_PIECE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(1); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(AUX_LOB_PIECE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_SERVICE_AUX_LOB_PIECE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("piece_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("data_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_data", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 32, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_SERVICE_AUX_LOB_PIECE_TID); + table_schema.set_data_table_id(OB_ALL_SERVICE_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::all_mview_dep_aux_lob_piece_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.h b/src/share/inner_table/ob_inner_table_schema.h index 039d421b8..ae3d0adc4 100644 --- a/src/share/inner_table/ob_inner_table_schema.h +++ b/src/share/inner_table/ob_inner_table_schema.h @@ -626,6 +626,7 @@ public: static int all_user_proxy_info_history_schema(share::schema::ObTableSchema &table_schema); static int all_user_proxy_role_info_schema(share::schema::ObTableSchema &table_schema); static int all_user_proxy_role_info_history_schema(share::schema::ObTableSchema &table_schema); + static int all_service_schema(share::schema::ObTableSchema &table_schema); static int all_mview_dep_schema(share::schema::ObTableSchema &table_schema); static int all_scheduler_job_run_detail_v2_schema(share::schema::ObTableSchema &table_schema); static int tenant_virtual_all_table_schema(share::schema::ObTableSchema &table_schema); @@ -1079,6 +1080,7 @@ public: static int all_virtual_user_proxy_info_history_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_user_proxy_role_info_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_user_proxy_role_info_history_schema(share::schema::ObTableSchema &table_schema); + static int all_virtual_service_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_tenant_resource_limit_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_tenant_resource_limit_detail_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_nic_info_schema(share::schema::ObTableSchema &table_schema); @@ -1354,6 +1356,7 @@ public: static int all_virtual_tracepoint_info_ora_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_user_proxy_info_real_agent_ora_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_user_proxy_role_info_real_agent_ora_schema(share::schema::ObTableSchema &table_schema); + static int all_virtual_service_ora_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_tenant_resource_limit_ora_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_tenant_resource_limit_detail_ora_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_nic_info_ora_schema(share::schema::ObTableSchema &table_schema); @@ -1778,6 +1781,8 @@ public: static int gv_ob_tracepoint_info_schema(share::schema::ObTableSchema &table_schema); static int v_ob_tracepoint_info_schema(share::schema::ObTableSchema &table_schema); static int v_ob_compatibility_control_schema(share::schema::ObTableSchema &table_schema); + static int dba_ob_services_schema(share::schema::ObTableSchema &table_schema); + static int cdb_ob_services_schema(share::schema::ObTableSchema &table_schema); static int gv_ob_tenant_resource_limit_schema(share::schema::ObTableSchema &table_schema); static int v_ob_tenant_resource_limit_schema(share::schema::ObTableSchema &table_schema); static int gv_ob_tenant_resource_limit_detail_schema(share::schema::ObTableSchema &table_schema); @@ -2086,6 +2091,7 @@ public: static int dba_mvref_stmt_stats_ora_schema(share::schema::ObTableSchema &table_schema); static int user_mvref_stmt_stats_ora_schema(share::schema::ObTableSchema &table_schema); static int proxy_users_schema(share::schema::ObTableSchema &table_schema); + static int dba_ob_services_ora_schema(share::schema::ObTableSchema &table_schema); static int gv_ob_sql_audit_ora_schema(share::schema::ObTableSchema &table_schema); static int v_ob_sql_audit_ora_schema(share::schema::ObTableSchema &table_schema); static int gv_instance_schema(share::schema::ObTableSchema &table_schema); @@ -2573,6 +2579,7 @@ public: static int all_user_proxy_info_history_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_user_proxy_role_info_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_user_proxy_role_info_history_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); + static int all_service_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_mview_dep_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_scheduler_job_run_detail_v2_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_table_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); @@ -2873,6 +2880,7 @@ public: static int all_user_proxy_info_history_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_user_proxy_role_info_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_user_proxy_role_info_history_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); + static int all_service_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_mview_dep_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_scheduler_job_run_detail_v2_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_ash_all_virtual_ash_i1_schema(share::schema::ObTableSchema &table_schema); @@ -3392,6 +3400,7 @@ const schema_create_func sys_table_schema_creators [] = { ObInnerTableSchema::all_user_proxy_info_history_schema, ObInnerTableSchema::all_user_proxy_role_info_schema, ObInnerTableSchema::all_user_proxy_role_info_history_schema, + ObInnerTableSchema::all_service_schema, ObInnerTableSchema::all_mview_dep_schema, ObInnerTableSchema::all_scheduler_job_run_detail_v2_schema, NULL,}; @@ -3848,6 +3857,7 @@ const schema_create_func virtual_table_schema_creators [] = { ObInnerTableSchema::all_virtual_user_proxy_info_history_schema, ObInnerTableSchema::all_virtual_user_proxy_role_info_schema, ObInnerTableSchema::all_virtual_user_proxy_role_info_history_schema, + ObInnerTableSchema::all_virtual_service_schema, ObInnerTableSchema::all_virtual_tenant_resource_limit_schema, ObInnerTableSchema::all_virtual_tenant_resource_limit_detail_schema, ObInnerTableSchema::all_virtual_nic_info_schema, @@ -4133,6 +4143,7 @@ const schema_create_func virtual_table_schema_creators [] = { ObInnerTableSchema::all_virtual_tracepoint_info_ora_schema, ObInnerTableSchema::all_virtual_user_proxy_info_real_agent_ora_schema, ObInnerTableSchema::all_virtual_user_proxy_role_info_real_agent_ora_schema, + ObInnerTableSchema::all_virtual_service_ora_schema, ObInnerTableSchema::all_virtual_tenant_resource_limit_ora_schema, ObInnerTableSchema::all_virtual_tenant_resource_limit_detail_ora_schema, ObInnerTableSchema::all_virtual_nic_info_ora_schema, @@ -4644,6 +4655,8 @@ const schema_create_func sys_view_schema_creators [] = { ObInnerTableSchema::gv_ob_tracepoint_info_schema, ObInnerTableSchema::v_ob_tracepoint_info_schema, ObInnerTableSchema::v_ob_compatibility_control_schema, + ObInnerTableSchema::dba_ob_services_schema, + ObInnerTableSchema::cdb_ob_services_schema, ObInnerTableSchema::gv_ob_tenant_resource_limit_schema, ObInnerTableSchema::v_ob_tenant_resource_limit_schema, ObInnerTableSchema::gv_ob_tenant_resource_limit_detail_schema, @@ -4952,6 +4965,7 @@ const schema_create_func sys_view_schema_creators [] = { ObInnerTableSchema::dba_mvref_stmt_stats_ora_schema, ObInnerTableSchema::user_mvref_stmt_stats_ora_schema, ObInnerTableSchema::proxy_users_schema, + ObInnerTableSchema::dba_ob_services_ora_schema, ObInnerTableSchema::gv_ob_sql_audit_ora_schema, ObInnerTableSchema::v_ob_sql_audit_ora_schema, ObInnerTableSchema::gv_instance_schema, @@ -5541,6 +5555,7 @@ const uint64_t tenant_space_tables [] = { OB_ALL_USER_PROXY_INFO_HISTORY_TID, OB_ALL_USER_PROXY_ROLE_INFO_TID, OB_ALL_USER_PROXY_ROLE_INFO_HISTORY_TID, + OB_ALL_SERVICE_TID, OB_ALL_MVIEW_DEP_TID, OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_TID, OB_TENANT_VIRTUAL_ALL_TABLE_TID, @@ -5774,6 +5789,7 @@ const uint64_t tenant_space_tables [] = { OB_ALL_VIRTUAL_SESSION_PS_INFO_TID, OB_ALL_VIRTUAL_TRACEPOINT_INFO_TID, OB_ALL_VIRTUAL_COMPATIBILITY_CONTROL_TID, + OB_ALL_VIRTUAL_SERVICE_TID, OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_TID, OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_DETAIL_TID, OB_ALL_VIRTUAL_NIC_INFO_TID, @@ -6056,6 +6072,7 @@ const uint64_t tenant_space_tables [] = { OB_ALL_VIRTUAL_TRACEPOINT_INFO_ORA_TID, OB_ALL_VIRTUAL_USER_PROXY_INFO_REAL_AGENT_ORA_TID, OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_REAL_AGENT_ORA_TID, + OB_ALL_VIRTUAL_SERVICE_ORA_TID, OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_ORA_TID, OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_DETAIL_ORA_TID, OB_ALL_VIRTUAL_NIC_INFO_ORA_TID, @@ -6360,6 +6377,7 @@ const uint64_t tenant_space_tables [] = { OB_GV_OB_TRACEPOINT_INFO_TID, OB_V_OB_TRACEPOINT_INFO_TID, OB_V_OB_COMPATIBILITY_CONTROL_TID, + OB_DBA_OB_SERVICES_TID, OB_GV_OB_TENANT_RESOURCE_LIMIT_TID, OB_V_OB_TENANT_RESOURCE_LIMIT_TID, OB_GV_OB_TENANT_RESOURCE_LIMIT_DETAIL_TID, @@ -6667,6 +6685,7 @@ const uint64_t tenant_space_tables [] = { OB_DBA_MVREF_STMT_STATS_ORA_TID, OB_USER_MVREF_STMT_STATS_ORA_TID, OB_PROXY_USERS_TID, + OB_DBA_OB_SERVICES_ORA_TID, OB_GV_OB_SQL_AUDIT_ORA_TID, OB_V_OB_SQL_AUDIT_ORA_TID, OB_GV_INSTANCE_TID, @@ -7309,6 +7328,7 @@ const uint64_t tenant_space_tables [] = { OB_ALL_USER_PROXY_INFO_HISTORY_AUX_LOB_META_TID, OB_ALL_USER_PROXY_ROLE_INFO_AUX_LOB_META_TID, OB_ALL_USER_PROXY_ROLE_INFO_HISTORY_AUX_LOB_META_TID, + OB_ALL_SERVICE_AUX_LOB_META_TID, OB_ALL_MVIEW_DEP_AUX_LOB_META_TID, OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_AUX_LOB_META_TID, OB_ALL_TABLE_AUX_LOB_PIECE_TID, @@ -7585,6 +7605,7 @@ const uint64_t tenant_space_tables [] = { OB_ALL_USER_PROXY_INFO_HISTORY_AUX_LOB_PIECE_TID, OB_ALL_USER_PROXY_ROLE_INFO_AUX_LOB_PIECE_TID, OB_ALL_USER_PROXY_ROLE_INFO_HISTORY_AUX_LOB_PIECE_TID, + OB_ALL_SERVICE_AUX_LOB_PIECE_TID, OB_ALL_MVIEW_DEP_AUX_LOB_PIECE_TID, OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_AUX_LOB_PIECE_TID, }; @@ -7733,6 +7754,7 @@ const uint64_t all_ora_mapping_virtual_table_org_tables [] = { OB_ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY_TID, OB_ALL_VIRTUAL_SESSION_PS_INFO_TID, OB_ALL_VIRTUAL_TRACEPOINT_INFO_TID, + OB_ALL_VIRTUAL_SERVICE_TID, OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_TID, OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_DETAIL_TID, OB_ALL_VIRTUAL_NIC_INFO_TID, }; @@ -7881,6 +7903,7 @@ const uint64_t all_ora_mapping_virtual_tables [] = { OB_ALL_VIRTUAL_SQL_AUDIT_O , OB_ALL_VIRTUAL_LS_REPLICA_TASK_HISTORY_ORA_TID , OB_ALL_VIRTUAL_SESSION_PS_INFO_ORA_TID , OB_ALL_VIRTUAL_TRACEPOINT_INFO_ORA_TID +, OB_ALL_VIRTUAL_SERVICE_ORA_TID , OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_ORA_TID , OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_DETAIL_ORA_TID , OB_ALL_VIRTUAL_NIC_INFO_ORA_TID @@ -8173,6 +8196,7 @@ const char* const tenant_space_table_names [] = { OB_ALL_USER_PROXY_INFO_HISTORY_TNAME, OB_ALL_USER_PROXY_ROLE_INFO_TNAME, OB_ALL_USER_PROXY_ROLE_INFO_HISTORY_TNAME, + OB_ALL_SERVICE_TNAME, OB_ALL_MVIEW_DEP_TNAME, OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_TNAME, OB_TENANT_VIRTUAL_ALL_TABLE_TNAME, @@ -8406,6 +8430,7 @@ const char* const tenant_space_table_names [] = { OB_ALL_VIRTUAL_SESSION_PS_INFO_TNAME, OB_ALL_VIRTUAL_TRACEPOINT_INFO_TNAME, OB_ALL_VIRTUAL_COMPATIBILITY_CONTROL_TNAME, + OB_ALL_VIRTUAL_SERVICE_TNAME, OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_TNAME, OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_DETAIL_TNAME, OB_ALL_VIRTUAL_NIC_INFO_TNAME, @@ -8688,6 +8713,7 @@ const char* const tenant_space_table_names [] = { OB_ALL_VIRTUAL_TRACEPOINT_INFO_ORA_TNAME, OB_ALL_VIRTUAL_USER_PROXY_INFO_REAL_AGENT_ORA_TNAME, OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_REAL_AGENT_ORA_TNAME, + OB_ALL_VIRTUAL_SERVICE_ORA_TNAME, OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_ORA_TNAME, OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_DETAIL_ORA_TNAME, OB_ALL_VIRTUAL_NIC_INFO_ORA_TNAME, @@ -8992,6 +9018,7 @@ const char* const tenant_space_table_names [] = { OB_GV_OB_TRACEPOINT_INFO_TNAME, OB_V_OB_TRACEPOINT_INFO_TNAME, OB_V_OB_COMPATIBILITY_CONTROL_TNAME, + OB_DBA_OB_SERVICES_TNAME, OB_GV_OB_TENANT_RESOURCE_LIMIT_TNAME, OB_V_OB_TENANT_RESOURCE_LIMIT_TNAME, OB_GV_OB_TENANT_RESOURCE_LIMIT_DETAIL_TNAME, @@ -9299,6 +9326,7 @@ const char* const tenant_space_table_names [] = { OB_DBA_MVREF_STMT_STATS_ORA_TNAME, OB_USER_MVREF_STMT_STATS_ORA_TNAME, OB_PROXY_USERS_TNAME, + OB_DBA_OB_SERVICES_ORA_TNAME, OB_GV_OB_SQL_AUDIT_ORA_TNAME, OB_V_OB_SQL_AUDIT_ORA_TNAME, OB_GV_INSTANCE_TNAME, @@ -9941,6 +9969,7 @@ const char* const tenant_space_table_names [] = { OB_ALL_USER_PROXY_INFO_HISTORY_AUX_LOB_META_TNAME, OB_ALL_USER_PROXY_ROLE_INFO_AUX_LOB_META_TNAME, OB_ALL_USER_PROXY_ROLE_INFO_HISTORY_AUX_LOB_META_TNAME, + OB_ALL_SERVICE_AUX_LOB_META_TNAME, OB_ALL_MVIEW_DEP_AUX_LOB_META_TNAME, OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_AUX_LOB_META_TNAME, OB_ALL_TABLE_AUX_LOB_PIECE_TNAME, @@ -10217,6 +10246,7 @@ const char* const tenant_space_table_names [] = { OB_ALL_USER_PROXY_INFO_HISTORY_AUX_LOB_PIECE_TNAME, OB_ALL_USER_PROXY_ROLE_INFO_AUX_LOB_PIECE_TNAME, OB_ALL_USER_PROXY_ROLE_INFO_HISTORY_AUX_LOB_PIECE_TNAME, + OB_ALL_SERVICE_AUX_LOB_PIECE_TNAME, OB_ALL_MVIEW_DEP_AUX_LOB_PIECE_TNAME, OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_AUX_LOB_PIECE_TNAME, }; @@ -10581,6 +10611,7 @@ const uint64_t restrict_access_virtual_tables[] = { OB_ALL_VIRTUAL_TRACEPOINT_INFO_ORA_TID, OB_ALL_VIRTUAL_USER_PROXY_INFO_REAL_AGENT_ORA_TID, OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_REAL_AGENT_ORA_TID, + OB_ALL_VIRTUAL_SERVICE_ORA_TID, OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_ORA_TID, OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_DETAIL_ORA_TID, OB_ALL_VIRTUAL_NIC_INFO_ORA_TID, @@ -13115,6 +13146,14 @@ LOBMapping const lob_aux_table_mappings [] = { ObInnerTableSchema::all_user_proxy_role_info_history_aux_lob_piece_schema }, + { + OB_ALL_SERVICE_TID, + OB_ALL_SERVICE_AUX_LOB_META_TID, + OB_ALL_SERVICE_AUX_LOB_PIECE_TID, + ObInnerTableSchema::all_service_aux_lob_meta_schema, + ObInnerTableSchema::all_service_aux_lob_piece_schema + }, + { OB_ALL_MVIEW_DEP_TID, OB_ALL_MVIEW_DEP_AUX_LOB_META_TID, @@ -13168,12 +13207,12 @@ static inline int get_sys_table_lob_aux_schema(const uint64_t tid, } const int64_t OB_CORE_TABLE_COUNT = 4; -const int64_t OB_SYS_TABLE_COUNT = 297; -const int64_t OB_VIRTUAL_TABLE_COUNT = 825; -const int64_t OB_SYS_VIEW_COUNT = 916; -const int64_t OB_SYS_TENANT_TABLE_COUNT = 2043; +const int64_t OB_SYS_TABLE_COUNT = 298; +const int64_t OB_VIRTUAL_TABLE_COUNT = 827; +const int64_t OB_SYS_VIEW_COUNT = 919; +const int64_t OB_SYS_TENANT_TABLE_COUNT = 2049; const int64_t OB_CORE_SCHEMA_VERSION = 1; -const int64_t OB_BOOTSTRAP_SCHEMA_VERSION = 2046; +const int64_t OB_BOOTSTRAP_SCHEMA_VERSION = 2052; } // end namespace share } // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema.lob.cpp b/src/share/inner_table/ob_inner_table_schema.lob.cpp index 9d142bcc0..f6df163d2 100644 --- a/src/share/inner_table/ob_inner_table_schema.lob.cpp +++ b/src/share/inner_table/ob_inner_table_schema.lob.cpp @@ -21,7 +21,7 @@ inner_lob_map_t inner_lob_map; bool lob_mapping_init() { int ret = OB_SUCCESS; - if (OB_FAIL(inner_lob_map.create(300, ObModIds::OB_INNER_LOB_HASH_SET))) { + if (OB_FAIL(inner_lob_map.create(301, ObModIds::OB_INNER_LOB_HASH_SET))) { SERVER_LOG(WARN, "fail to create inner lob map", K(ret)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < ARRAYSIZEOF(lob_aux_table_mappings); ++i) { diff --git a/src/share/inner_table/ob_inner_table_schema_constants.h b/src/share/inner_table/ob_inner_table_schema_constants.h index 26653165e..0938f63d6 100644 --- a/src/share/inner_table/ob_inner_table_schema_constants.h +++ b/src/share/inner_table/ob_inner_table_schema_constants.h @@ -326,6 +326,7 @@ const uint64_t OB_ALL_USER_PROXY_INFO_TID = 512; // "__all_user_proxy_info" const uint64_t OB_ALL_USER_PROXY_INFO_HISTORY_TID = 513; // "__all_user_proxy_info_history" const uint64_t OB_ALL_USER_PROXY_ROLE_INFO_TID = 514; // "__all_user_proxy_role_info" const uint64_t OB_ALL_USER_PROXY_ROLE_INFO_HISTORY_TID = 515; // "__all_user_proxy_role_info_history" +const uint64_t OB_ALL_SERVICE_TID = 516; // "__all_service" const uint64_t OB_ALL_MVIEW_DEP_TID = 518; // "__all_mview_dep" const uint64_t OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_TID = 519; // "__all_scheduler_job_run_detail_v2" const uint64_t OB_TENANT_VIRTUAL_ALL_TABLE_TID = 10001; // "__tenant_virtual_all_table" @@ -779,6 +780,7 @@ const uint64_t OB_ALL_VIRTUAL_USER_PROXY_INFO_TID = 12474; // "__all_virtual_use const uint64_t OB_ALL_VIRTUAL_USER_PROXY_INFO_HISTORY_TID = 12475; // "__all_virtual_user_proxy_info_history" const uint64_t OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_TID = 12476; // "__all_virtual_user_proxy_role_info" const uint64_t OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_HISTORY_TID = 12477; // "__all_virtual_user_proxy_role_info_history" +const uint64_t OB_ALL_VIRTUAL_SERVICE_TID = 12480; // "__all_virtual_service" const uint64_t OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_TID = 12481; // "__all_virtual_tenant_resource_limit" const uint64_t OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_DETAIL_TID = 12482; // "__all_virtual_tenant_resource_limit_detail" const uint64_t OB_ALL_VIRTUAL_NIC_INFO_TID = 12487; // "__all_virtual_nic_info" @@ -1054,6 +1056,7 @@ const uint64_t OB_ALL_VIRTUAL_SESSION_PS_INFO_ORA_TID = 15444; // "ALL_VIRTUAL_S const uint64_t OB_ALL_VIRTUAL_TRACEPOINT_INFO_ORA_TID = 15445; // "ALL_VIRTUAL_TRACEPOINT_INFO_ORA" const uint64_t OB_ALL_VIRTUAL_USER_PROXY_INFO_REAL_AGENT_ORA_TID = 15446; // "ALL_VIRTUAL_USER_PROXY_INFO_REAL_AGENT_ORA" const uint64_t OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_REAL_AGENT_ORA_TID = 15447; // "ALL_VIRTUAL_USER_PROXY_ROLE_INFO_REAL_AGENT_ORA" +const uint64_t OB_ALL_VIRTUAL_SERVICE_ORA_TID = 15449; // "ALL_VIRTUAL_SERVICE_ORA" const uint64_t OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_ORA_TID = 15450; // "ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_ORA" const uint64_t OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_DETAIL_ORA_TID = 15451; // "ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_DETAIL_ORA" const uint64_t OB_ALL_VIRTUAL_NIC_INFO_ORA_TID = 15456; // "ALL_VIRTUAL_NIC_INFO_ORA" @@ -1478,6 +1481,8 @@ const uint64_t OB_V_OB_SESSION_PS_INFO_TID = 21542; // "V$OB_SESSION_PS_INFO" const uint64_t OB_GV_OB_TRACEPOINT_INFO_TID = 21543; // "GV$OB_TRACEPOINT_INFO" const uint64_t OB_V_OB_TRACEPOINT_INFO_TID = 21544; // "V$OB_TRACEPOINT_INFO" const uint64_t OB_V_OB_COMPATIBILITY_CONTROL_TID = 21545; // "V$OB_COMPATIBILITY_CONTROL" +const uint64_t OB_DBA_OB_SERVICES_TID = 21548; // "DBA_OB_SERVICES" +const uint64_t OB_CDB_OB_SERVICES_TID = 21549; // "CDB_OB_SERVICES" const uint64_t OB_GV_OB_TENANT_RESOURCE_LIMIT_TID = 21550; // "GV$OB_TENANT_RESOURCE_LIMIT" const uint64_t OB_V_OB_TENANT_RESOURCE_LIMIT_TID = 21551; // "V$OB_TENANT_RESOURCE_LIMIT" const uint64_t OB_GV_OB_TENANT_RESOURCE_LIMIT_DETAIL_TID = 21552; // "GV$OB_TENANT_RESOURCE_LIMIT_DETAIL" @@ -1786,6 +1791,7 @@ const uint64_t OB_USER_MVREF_CHANGE_STATS_ORA_TID = 25298; // "USER_MVREF_CHANGE const uint64_t OB_DBA_MVREF_STMT_STATS_ORA_TID = 25299; // "DBA_MVREF_STMT_STATS_ORA" const uint64_t OB_USER_MVREF_STMT_STATS_ORA_TID = 25300; // "USER_MVREF_STMT_STATS_ORA" const uint64_t OB_PROXY_USERS_TID = 25301; // "PROXY_USERS" +const uint64_t OB_DBA_OB_SERVICES_ORA_TID = 25302; // "DBA_OB_SERVICES_ORA" const uint64_t OB_GV_OB_SQL_AUDIT_ORA_TID = 28002; // "GV$OB_SQL_AUDIT_ORA" const uint64_t OB_V_OB_SQL_AUDIT_ORA_TID = 28003; // "V$OB_SQL_AUDIT_ORA" const uint64_t OB_GV_INSTANCE_TID = 28004; // "GV$INSTANCE" @@ -2273,6 +2279,7 @@ const uint64_t OB_ALL_USER_PROXY_INFO_AUX_LOB_META_TID = 50512; // "__all_user_p const uint64_t OB_ALL_USER_PROXY_INFO_HISTORY_AUX_LOB_META_TID = 50513; // "__all_user_proxy_info_history_aux_lob_meta" const uint64_t OB_ALL_USER_PROXY_ROLE_INFO_AUX_LOB_META_TID = 50514; // "__all_user_proxy_role_info_aux_lob_meta" const uint64_t OB_ALL_USER_PROXY_ROLE_INFO_HISTORY_AUX_LOB_META_TID = 50515; // "__all_user_proxy_role_info_history_aux_lob_meta" +const uint64_t OB_ALL_SERVICE_AUX_LOB_META_TID = 50516; // "__all_service_aux_lob_meta" const uint64_t OB_ALL_MVIEW_DEP_AUX_LOB_META_TID = 50518; // "__all_mview_dep_aux_lob_meta" const uint64_t OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_AUX_LOB_META_TID = 50519; // "__all_scheduler_job_run_detail_v2_aux_lob_meta" const uint64_t OB_ALL_TABLE_AUX_LOB_PIECE_TID = 60003; // "__all_table_aux_lob_piece" @@ -2573,6 +2580,7 @@ const uint64_t OB_ALL_USER_PROXY_INFO_AUX_LOB_PIECE_TID = 60512; // "__all_user_ const uint64_t OB_ALL_USER_PROXY_INFO_HISTORY_AUX_LOB_PIECE_TID = 60513; // "__all_user_proxy_info_history_aux_lob_piece" const uint64_t OB_ALL_USER_PROXY_ROLE_INFO_AUX_LOB_PIECE_TID = 60514; // "__all_user_proxy_role_info_aux_lob_piece" const uint64_t OB_ALL_USER_PROXY_ROLE_INFO_HISTORY_AUX_LOB_PIECE_TID = 60515; // "__all_user_proxy_role_info_history_aux_lob_piece" +const uint64_t OB_ALL_SERVICE_AUX_LOB_PIECE_TID = 60516; // "__all_service_aux_lob_piece" const uint64_t OB_ALL_MVIEW_DEP_AUX_LOB_PIECE_TID = 60518; // "__all_mview_dep_aux_lob_piece" const uint64_t OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_AUX_LOB_PIECE_TID = 60519; // "__all_scheduler_job_run_detail_v2_aux_lob_piece" const uint64_t OB_ALL_VIRTUAL_PLAN_CACHE_STAT_ALL_VIRTUAL_PLAN_CACHE_STAT_I1_TID = 14999; // "__all_virtual_plan_cache_stat" @@ -3079,6 +3087,7 @@ const char *const OB_ALL_USER_PROXY_INFO_TNAME = "__all_user_proxy_info"; const char *const OB_ALL_USER_PROXY_INFO_HISTORY_TNAME = "__all_user_proxy_info_history"; const char *const OB_ALL_USER_PROXY_ROLE_INFO_TNAME = "__all_user_proxy_role_info"; const char *const OB_ALL_USER_PROXY_ROLE_INFO_HISTORY_TNAME = "__all_user_proxy_role_info_history"; +const char *const OB_ALL_SERVICE_TNAME = "__all_service"; const char *const OB_ALL_MVIEW_DEP_TNAME = "__all_mview_dep"; const char *const OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_TNAME = "__all_scheduler_job_run_detail_v2"; const char *const OB_TENANT_VIRTUAL_ALL_TABLE_TNAME = "__tenant_virtual_all_table"; @@ -3532,6 +3541,7 @@ const char *const OB_ALL_VIRTUAL_USER_PROXY_INFO_TNAME = "__all_virtual_user_pro const char *const OB_ALL_VIRTUAL_USER_PROXY_INFO_HISTORY_TNAME = "__all_virtual_user_proxy_info_history"; const char *const OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_TNAME = "__all_virtual_user_proxy_role_info"; const char *const OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_HISTORY_TNAME = "__all_virtual_user_proxy_role_info_history"; +const char *const OB_ALL_VIRTUAL_SERVICE_TNAME = "__all_virtual_service"; const char *const OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_TNAME = "__all_virtual_tenant_resource_limit"; const char *const OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_DETAIL_TNAME = "__all_virtual_tenant_resource_limit_detail"; const char *const OB_ALL_VIRTUAL_NIC_INFO_TNAME = "__all_virtual_nic_info"; @@ -3807,6 +3817,7 @@ const char *const OB_ALL_VIRTUAL_SESSION_PS_INFO_ORA_TNAME = "ALL_VIRTUAL_SESSIO const char *const OB_ALL_VIRTUAL_TRACEPOINT_INFO_ORA_TNAME = "ALL_VIRTUAL_TRACEPOINT_INFO"; const char *const OB_ALL_VIRTUAL_USER_PROXY_INFO_REAL_AGENT_ORA_TNAME = "ALL_VIRTUAL_USER_PROXY_INFO_REAL_AGENT"; const char *const OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_REAL_AGENT_ORA_TNAME = "ALL_VIRTUAL_USER_PROXY_ROLE_INFO_REAL_AGENT"; +const char *const OB_ALL_VIRTUAL_SERVICE_ORA_TNAME = "ALL_VIRTUAL_SERVICE"; const char *const OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_ORA_TNAME = "ALL_VIRTUAL_TENANT_RESOURCE_LIMIT"; const char *const OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_DETAIL_ORA_TNAME = "ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_DETAIL"; const char *const OB_ALL_VIRTUAL_NIC_INFO_ORA_TNAME = "ALL_VIRTUAL_NIC_INFO"; @@ -4231,6 +4242,8 @@ const char *const OB_V_OB_SESSION_PS_INFO_TNAME = "V$OB_SESSION_PS_INFO"; const char *const OB_GV_OB_TRACEPOINT_INFO_TNAME = "GV$OB_TRACEPOINT_INFO"; const char *const OB_V_OB_TRACEPOINT_INFO_TNAME = "V$OB_TRACEPOINT_INFO"; const char *const OB_V_OB_COMPATIBILITY_CONTROL_TNAME = "V$OB_COMPATIBILITY_CONTROL"; +const char *const OB_DBA_OB_SERVICES_TNAME = "DBA_OB_SERVICES"; +const char *const OB_CDB_OB_SERVICES_TNAME = "CDB_OB_SERVICES"; const char *const OB_GV_OB_TENANT_RESOURCE_LIMIT_TNAME = "GV$OB_TENANT_RESOURCE_LIMIT"; const char *const OB_V_OB_TENANT_RESOURCE_LIMIT_TNAME = "V$OB_TENANT_RESOURCE_LIMIT"; const char *const OB_GV_OB_TENANT_RESOURCE_LIMIT_DETAIL_TNAME = "GV$OB_TENANT_RESOURCE_LIMIT_DETAIL"; @@ -4539,6 +4552,7 @@ const char *const OB_USER_MVREF_CHANGE_STATS_ORA_TNAME = "USER_MVREF_CHANGE_STAT const char *const OB_DBA_MVREF_STMT_STATS_ORA_TNAME = "DBA_MVREF_STMT_STATS"; const char *const OB_USER_MVREF_STMT_STATS_ORA_TNAME = "USER_MVREF_STMT_STATS"; const char *const OB_PROXY_USERS_TNAME = "PROXY_USERS"; +const char *const OB_DBA_OB_SERVICES_ORA_TNAME = "DBA_OB_SERVICES"; const char *const OB_GV_OB_SQL_AUDIT_ORA_TNAME = "GV$OB_SQL_AUDIT"; const char *const OB_V_OB_SQL_AUDIT_ORA_TNAME = "V$OB_SQL_AUDIT"; const char *const OB_GV_INSTANCE_TNAME = "GV$INSTANCE"; @@ -5026,6 +5040,7 @@ const char *const OB_ALL_USER_PROXY_INFO_AUX_LOB_META_TNAME = "__all_user_proxy_ const char *const OB_ALL_USER_PROXY_INFO_HISTORY_AUX_LOB_META_TNAME = "__all_user_proxy_info_history_aux_lob_meta"; const char *const OB_ALL_USER_PROXY_ROLE_INFO_AUX_LOB_META_TNAME = "__all_user_proxy_role_info_aux_lob_meta"; const char *const OB_ALL_USER_PROXY_ROLE_INFO_HISTORY_AUX_LOB_META_TNAME = "__all_user_proxy_role_info_history_aux_lob_meta"; +const char *const OB_ALL_SERVICE_AUX_LOB_META_TNAME = "__all_service_aux_lob_meta"; const char *const OB_ALL_MVIEW_DEP_AUX_LOB_META_TNAME = "__all_mview_dep_aux_lob_meta"; const char *const OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_AUX_LOB_META_TNAME = "__all_scheduler_job_run_detail_v2_aux_lob_meta"; const char *const OB_ALL_TABLE_AUX_LOB_PIECE_TNAME = "__all_table_aux_lob_piece"; @@ -5326,6 +5341,7 @@ const char *const OB_ALL_USER_PROXY_INFO_AUX_LOB_PIECE_TNAME = "__all_user_proxy const char *const OB_ALL_USER_PROXY_INFO_HISTORY_AUX_LOB_PIECE_TNAME = "__all_user_proxy_info_history_aux_lob_piece"; const char *const OB_ALL_USER_PROXY_ROLE_INFO_AUX_LOB_PIECE_TNAME = "__all_user_proxy_role_info_aux_lob_piece"; const char *const OB_ALL_USER_PROXY_ROLE_INFO_HISTORY_AUX_LOB_PIECE_TNAME = "__all_user_proxy_role_info_history_aux_lob_piece"; +const char *const OB_ALL_SERVICE_AUX_LOB_PIECE_TNAME = "__all_service_aux_lob_piece"; const char *const OB_ALL_MVIEW_DEP_AUX_LOB_PIECE_TNAME = "__all_mview_dep_aux_lob_piece"; const char *const OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_AUX_LOB_PIECE_TNAME = "__all_scheduler_job_run_detail_v2_aux_lob_piece"; const char *const OB_ALL_VIRTUAL_PLAN_CACHE_STAT_ALL_VIRTUAL_PLAN_CACHE_STAT_I1_TNAME = "__idx_11003_all_virtual_plan_cache_stat_i1"; diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index 4337c3efb..af412a0a3 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -7278,7 +7278,26 @@ def_table_schema(**gen_history_table_def(515, all_user_proxy_role_info_def)) # 513 : __all_user_proxy_info_history # 514 : __all_user_proxy_role_info # 515 : __all_user_proxy_role_info_history -# 516 : __all_service +def_table_schema( + owner = 'linqiucen.lqc', + table_name = '__all_service', + table_id = '516', + table_type = 'SYSTEM_TABLE', + gm_columns = ['gmt_create', 'gmt_modified'], + rowkey_columns = [ + ('tenant_id', 'int'), + ('service_name_id', 'int'), + ], + in_tenant_space = True, + is_cluster_private = True, + meta_record_in_sys = False, + + normal_columns = [ + ('service_name', 'varchar:OB_SERVICE_NAME_LENGTH'), + ('service_status', 'varchar:64', 'false'), + ], +) + # 517 : __all_storage_io_usage def_table_schema( @@ -14559,7 +14578,11 @@ def_table_schema(**gen_iterate_virtual_table_def( # 12478: __all_virtual_tablet_reorganize_history # 12479: __all_virtual_res_mgr_directive -# 12480: __all_virtual_service +def_table_schema(**gen_iterate_private_virtual_table_def( + table_id = '12480', + table_name = '__all_virtual_service', + in_tenant_space = True, + keywords = all_def_keywords['__all_service'])) def_table_schema( owner = 'yanyuan.cxf', @@ -15131,7 +15154,7 @@ def_table_schema(**no_direct_access(gen_oracle_mapping_virtual_table_def('15445' def_table_schema(**no_direct_access(gen_oracle_mapping_real_virtual_table_def('15446', all_def_keywords['__all_user_proxy_info']))) def_table_schema(**no_direct_access(gen_oracle_mapping_real_virtual_table_def('15447', all_def_keywords['__all_user_proxy_role_info']))) # 15448: idx_user_proxy_info_proxy_user_id_real_agent -# 15449: __all_virtual_service +def_table_schema(**no_direct_access(gen_oracle_mapping_virtual_table_def('15449', all_def_keywords['__all_virtual_service']))) def_table_schema(**no_direct_access(gen_oracle_mapping_virtual_table_def('15450', all_def_keywords['__all_virtual_tenant_resource_limit']))) def_table_schema(**no_direct_access(gen_oracle_mapping_virtual_table_def('15451', all_def_keywords['__all_virtual_tenant_resource_limit_detail']))) @@ -24049,8 +24072,9 @@ SELECT IN_BYTES, OUT_BYTES, USER_CLIENT_PORT, - cast(total_cpu_time as SIGNED) as TOTAL_CPU_TIME, PROXY_USER, + SERVICE_NAME, + cast(total_cpu_time as SIGNED) as TOTAL_CPU_TIME, TOP_INFO FROM oceanbase.__all_virtual_processlist """.replace("\n", " ") @@ -24101,8 +24125,9 @@ def_table_schema( IN_BYTES, OUT_BYTES, USER_CLIENT_PORT, - cast(total_cpu_time as SIGNED) as TOTAL_CPU_TIME, PROXY_USER, + SERVICE_NAME, + cast(total_cpu_time as SIGNED) as TOTAL_CPU_TIME, TOP_INFO FROM oceanbase.GV$OB_PROCESSLIST WHERE SVR_IP = host_ip() AND SVR_PORT = rpc_port() @@ -35008,8 +35033,49 @@ def_table_schema( ) # 21546: DBA_OB_RSRC_DIRECTIVES # 21547: CDB_OB_RSRC_DIRECTIVES -# 21548: DBA_OB_SERVICES -# 21549: CDB_OB_SERVICES + +def_table_schema( + owner = 'linqiucen.lqc', + table_name = 'DBA_OB_SERVICES', + table_id = '21548', + table_type = 'SYSTEM_VIEW', + gm_columns = [], + rowkey_columns = [], + normal_columns = [], + in_tenant_space = True, + view_definition = + """ + SELECT + gmt_create AS CREATE_TIME, + gmt_modified AS MODIFIED_TIME, + SERVICE_NAME_ID, + SERVICE_NAME, + SERVICE_STATUS + FROM oceanbase.__all_virtual_service + WHERE TENANT_ID=EFFECTIVE_TENANT_ID(); + """.replace("\n", " ") +) + +def_table_schema( + owner = 'linqiucen.lqc', + table_name = 'CDB_OB_SERVICES', + table_id = '21549', + table_type = 'SYSTEM_VIEW', + gm_columns = [], + rowkey_columns = [], + normal_columns = [], + view_definition = + """ + SELECT + TENANT_ID, + gmt_create AS `CREATE_TIME`, + gmt_modified AS 'MODIFIED_TIME', + SERVICE_NAME_ID, + SERVICE_NAME, + SERVICE_STATUS + FROM oceanbase.__all_virtual_service + """.replace("\n", " ") +) def_table_schema( owner = 'cxf262476', @@ -56291,7 +56357,6 @@ def_table_schema( ) # 25301: PROXY_USERS -# 25302: DBA_OB_SERVICES # 25303: DBA_OB_STORAGE_IO_USAGE def_table_schema( owner = 'mingye.swj', @@ -56335,8 +56400,29 @@ where U1.TENANT_ID = U2.TENANT_ID and V.CLIENT_USER_ID = P.CLIENT_USER_ID """.replace("\n", " ") ) - - +def_table_schema( + owner = 'linqiucen.lqc', + table_name = 'DBA_OB_SERVICES', + name_postfix = '_ORA', + database_id = 'OB_ORA_SYS_DATABASE_ID', + table_id = '25302', + table_type = 'SYSTEM_VIEW', + gm_columns = [], + rowkey_columns = [], + normal_columns = [], + in_tenant_space = True, + view_definition = + """ + SELECT + gmt_create AS "CREATE_TIME", + gmt_modified AS "MODIFIED_TIME", + SERVICE_NAME_ID, + SERVICE_NAME, + SERVICE_STATUS + FROM SYS.ALL_VIRTUAL_SERVICE + WHERE TENANT_ID=EFFECTIVE_TENANT_ID(); + """.replace("\n", " ") +) # # 余留位置（此行之前占位） # 本区域占位建议：采用真实视图名进行占位 @@ -60996,6 +61082,7 @@ SELECT OUT_BYTES, USER_CLIENT_PORT, PROXY_USER, + SERVICE_NAME, CAST(total_cpu_time AS INT) as TOTAL_CPU_TIME, TOP_INFO FROM SYS.ALL_VIRTUAL_PROCESSLIST @@ -61050,6 +61137,7 @@ def_table_schema( OUT_BYTES, USER_CLIENT_PORT, PROXY_USER, + SERVICE_NAME, CAST(total_cpu_time AS INT) as TOTAL_CPU_TIME, TOP_INFO FROM SYS.GV$OB_PROCESSLIST diff --git a/src/share/inner_table/ob_inner_table_schema_misc.ipp b/src/share/inner_table/ob_inner_table_schema_misc.ipp index 8145ce45e..5e0fa3e0b 100644 --- a/src/share/inner_table/ob_inner_table_schema_misc.ipp +++ b/src/share/inner_table/ob_inner_table_schema_misc.ipp @@ -486,6 +486,7 @@ case OB_ALL_VIRTUAL_RECOVER_TABLE_JOB_HISTORY_TID: case OB_ALL_VIRTUAL_RESTORE_JOB_TID: case OB_ALL_VIRTUAL_RESTORE_JOB_HISTORY_TID: case OB_ALL_VIRTUAL_RESTORE_PROGRESS_TID: +case OB_ALL_VIRTUAL_SERVICE_TID: case OB_ALL_VIRTUAL_TABLE_OPT_STAT_GATHER_HISTORY_TID: case OB_ALL_VIRTUAL_TABLET_META_TABLE_TID: case OB_ALL_VIRTUAL_TABLET_REPLICA_CHECKSUM_TID: @@ -1460,6 +1461,24 @@ case OB_ALL_VIRTUAL_ZONE_MERGE_INFO_TID: break; } + case OB_ALL_VIRTUAL_SERVICE_TID: { + ObIteratePrivateVirtualTable *iter = NULL; + const bool meta_record_in_sys = false; + if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIteratePrivateVirtualTable, iter))) { + SERVER_LOG(WARN, "create iterate private virtual table iterator failed", KR(ret)); + } else if (OB_FAIL(iter->init(OB_ALL_SERVICE_TID, meta_record_in_sys, index_schema, params))) { + SERVER_LOG(WARN, "iterate private virtual table iter init failed", KR(ret)); + iter->~ObIteratePrivateVirtualTable(); + allocator.free(iter); + iter = NULL; + } else { + vt_iter = iter; + } + break; + } + END_CREATE_VT_ITER_SWITCH_LAMBDA + + BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA case OB_ALL_VIRTUAL_TABLE_OPT_STAT_GATHER_HISTORY_TID: { ObIteratePrivateVirtualTable *iter = NULL; const bool meta_record_in_sys = false; @@ -1475,9 +1494,7 @@ case OB_ALL_VIRTUAL_ZONE_MERGE_INFO_TID: } break; } - END_CREATE_VT_ITER_SWITCH_LAMBDA - BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA case OB_ALL_VIRTUAL_TABLET_META_TABLE_TID: { ObIteratePrivateVirtualTable *iter = NULL; const bool meta_record_in_sys = false; @@ -4849,6 +4866,9 @@ case OB_ALL_RESTORE_JOB_HISTORY_AUX_LOB_PIECE_TID: case OB_ALL_RESTORE_PROGRESS_TID: case OB_ALL_RESTORE_PROGRESS_AUX_LOB_META_TID: case OB_ALL_RESTORE_PROGRESS_AUX_LOB_PIECE_TID: +case OB_ALL_SERVICE_TID: +case OB_ALL_SERVICE_AUX_LOB_META_TID: +case OB_ALL_SERVICE_AUX_LOB_PIECE_TID: case OB_ALL_SERVICE_EPOCH_TID: case OB_ALL_SERVICE_EPOCH_AUX_LOB_META_TID: case OB_ALL_SERVICE_EPOCH_AUX_LOB_PIECE_TID: diff --git a/src/share/inner_table/table_id_to_name b/src/share/inner_table/table_id_to_name index ebfc2fc85..a84b7a15b 100644 --- a/src/share/inner_table/table_id_to_name +++ b/src/share/inner_table/table_id_to_name @@ -355,6 +355,7 @@ # 514: __all_user_proxy_role_info # 515: __all_user_proxy_role_info_history # 515: __all_user_proxy_role_info # BASE_TABLE_NAME +# 516: __all_service # 518: __all_mview_dep # 519: __all_scheduler_job_run_detail_v2 # 10001: __tenant_virtual_all_table @@ -1121,6 +1122,8 @@ # 12477: __all_virtual_user_proxy_role_info_history # 12477: __all_user_proxy_role_info # BASE_TABLE_NAME # 12477: __all_user_proxy_role_info_history # BASE_TABLE_NAME1 +# 12480: __all_virtual_service +# 12480: __all_service # BASE_TABLE_NAME # 12481: __all_virtual_tenant_resource_limit # 12482: __all_virtual_tenant_resource_limit_detail # 12487: __all_virtual_nic_info @@ -1726,6 +1729,9 @@ # 15446: __all_user_proxy_info # BASE_TABLE_NAME # 15447: ALL_VIRTUAL_USER_PROXY_ROLE_INFO_REAL_AGENT # 15447: __all_user_proxy_role_info # BASE_TABLE_NAME +# 15449: ALL_VIRTUAL_SERVICE +# 15449: __all_service # BASE_TABLE_NAME +# 15449: __all_virtual_service # BASE_TABLE_NAME1 # 15450: ALL_VIRTUAL_TENANT_RESOURCE_LIMIT # 15450: __all_virtual_tenant_resource_limit # BASE_TABLE_NAME # 15451: ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_DETAIL @@ -2155,6 +2161,8 @@ # 21543: GV$OB_TRACEPOINT_INFO # 21544: V$OB_TRACEPOINT_INFO # 21545: V$OB_COMPATIBILITY_CONTROL +# 21548: DBA_OB_SERVICES +# 21549: CDB_OB_SERVICES # 21550: GV$OB_TENANT_RESOURCE_LIMIT # 21551: V$OB_TENANT_RESOURCE_LIMIT # 21552: GV$OB_TENANT_RESOURCE_LIMIT_DETAIL @@ -2463,6 +2471,7 @@ # 25299: DBA_MVREF_STMT_STATS # 25300: USER_MVREF_STMT_STATS # 25301: PROXY_USERS +# 25302: DBA_OB_SERVICES # 28002: GV$OB_SQL_AUDIT # 28003: V$OB_SQL_AUDIT # 28004: GV$INSTANCE diff --git a/src/share/ob_autoincrement_service.cpp b/src/share/ob_autoincrement_service.cpp index a7e45969d..dfd32c12f 100644 --- a/src/share/ob_autoincrement_service.cpp +++ b/src/share/ob_autoincrement_service.cpp @@ -2017,9 +2017,6 @@ int ObAutoIncInnerTableProxy::next_autoinc_value(const AutoincKey &key, if (sql_len >= OB_MAX_SQL_LENGTH || sql_len <= 0) { ret = OB_SIZE_OVERFLOW; LOG_WARN("failed to format sql. size not enough", K(ret), K(sql_len)); - } else if (GCTX.is_standby_cluster() && OB_SYS_TENANT_ID != exec_tenant_id) { - ret = OB_OP_NOT_ALLOW; - LOG_WARN("can't write sys table now", K(ret), K(exec_tenant_id)); } else if (OB_FAIL(trans.write(exec_tenant_id, sql, affected_rows))) { LOG_WARN("failed to write data", K(ret)); } else if (affected_rows != 1) { @@ -2315,9 +2312,6 @@ int ObAutoIncInnerTableProxy::sync_autoinc_value(const AutoincKey &key, table_name, sync_value, new_seq_value, OB_INVALID_TENANT_ID, table_id, column_id, inner_autoinc_version))) { LOG_WARN("failed to assign sql", K(ret)); - } else if (GCTX.is_standby_cluster() && OB_SYS_TENANT_ID != exec_tenant_id) { - ret = OB_OP_NOT_ALLOW; - LOG_WARN("can't write sys table now", K(ret), K(exec_tenant_id)); } else if (OB_FAIL((trans.write(exec_tenant_id, sql.ptr(), affected_rows)))) { LOG_WARN("failed to execute", K(sql), K(ret)); } else if (!is_single_row(affected_rows)) { @@ -2458,9 +2452,6 @@ int ObAutoIncInnerTableProxy::read_and_push_inner_table(const AutoincKey &key, table_name, new_seq_value, OB_INVALID_TENANT_ID, table_id, column_id, inner_autoinc_version))) { LOG_WARN("failed to assign sql", K(ret)); - } else if (GCTX.is_standby_cluster() && OB_SYS_TENANT_ID != exec_tenant_id) { - ret = OB_OP_NOT_ALLOW; - LOG_WARN("can't write sys table now", K(ret), K(exec_tenant_id)); } else if (OB_FAIL((trans.write(exec_tenant_id, sql.ptr(), affected_rows)))) { LOG_WARN("failed to execute", K(sql), K(ret)); } else if (!is_single_row(affected_rows)) { diff --git a/src/share/ob_errno.cpp b/src/share/ob_errno.cpp index be507cc94..5445e6393 100644 --- a/src/share/ob_errno.cpp +++ b/src/share/ob_errno.cpp @@ -26997,13 +26997,13 @@ static const _error _error_OB_SOURCE_LS_STATE_NOT_MATCH = { .error_solution = "Contact OceanBase Support", .mysql_errno = -1, .sqlstate = "HY000", - .str_error = "log restore source ls state not match, switchover to primary not allowed", - .str_user_error = "log restore source ls state not match, switchover to primary not allowed", + .str_error = "log restore source LS state not match, switchover to primary is not allowed", + .str_user_error = "log restore source LS state not match, switchover to primary is not allowed", .oracle_errno = 600, - .oracle_str_error = "ORA-00600: internal error code, arguments: -9089, log restore source ls state not match, switchover to primary not allowed", - .oracle_str_user_error = "ORA-00600: internal error code, arguments: -9089, log restore source ls state not match, switchover to primary not allowed", - .ob_str_error = "OBE-00600: internal error code, arguments: -9089, log restore source ls state not match, switchover to primary not allowed", - .ob_str_user_error = "OBE-00600: internal error code, arguments: -9089, log restore source ls state not match, switchover to primary not allowed" + .oracle_str_error = "ORA-00600: internal error code, arguments: -9089, log restore source LS state not match, switchover to primary is not allowed", + .oracle_str_user_error = "ORA-00600: internal error code, arguments: -9089, log restore source LS state not match, switchover to primary is not allowed", + .ob_str_error = "OBE-00600: internal error code, arguments: -9089, log restore source LS state not match, switchover to primary is not allowed", + .ob_str_user_error = "OBE-00600: internal error code, arguments: -9089, log restore source LS state not match, switchover to primary is not allowed" }; static const _error _error_OB_ESI_SESSION_NOT_EXIST = { .error_name = "OB_ESI_SESSION_NOT_EXIST", diff --git a/src/share/ob_errno.def b/src/share/ob_errno.def index 9bd33f342..2bbc83c3b 100755 --- a/src/share/ob_errno.def +++ b/src/share/ob_errno.def @@ -2281,7 +2281,7 @@ DEFINE_ERROR(OB_BACKUP_MAJOR_NOT_COVER_MINOR, -9085, -1, "HY000", "backup major DEFINE_ERROR(OB_BACKUP_ADVANCE_CHECKPOINT_TIMEOUT, -9086, -1, "HY000", "backup advance checkpoint by flush timeout"); DEFINE_ERROR(OB_CLOG_RECYCLE_BEFORE_ARCHIVE, -9087, -1, "HY000", "observer clog is recycled before archive"); DEFINE_ERROR(OB_SOURCE_TENANT_STATE_NOT_MATCH, -9088, -1, "HY000", "log restore source tenant state not match, switchover to primary not allowed"); -DEFINE_ERROR(OB_SOURCE_LS_STATE_NOT_MATCH, -9089, -1, "HY000", "log restore source ls state not match, switchover to primary not allowed"); +DEFINE_ERROR(OB_SOURCE_LS_STATE_NOT_MATCH, -9089, -1, "HY000", "log restore source LS state not match, switchover to primary is not allowed"); DEFINE_ERROR(OB_ESI_SESSION_NOT_EXIST, -9090, -1, "HY000", "obesi process session not exist"); diff --git a/src/share/ob_errno.h b/src/share/ob_errno.h index 81c5c994a..2bb009656 100755 --- a/src/share/ob_errno.h +++ b/src/share/ob_errno.h @@ -3848,7 +3848,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_BACKUP_ADVANCE_CHECKPOINT_TIMEOUT__USER_ERROR_MSG "backup advance checkpoint by flush timeout" #define OB_CLOG_RECYCLE_BEFORE_ARCHIVE__USER_ERROR_MSG "observer clog is recycled before archive" #define OB_SOURCE_TENANT_STATE_NOT_MATCH__USER_ERROR_MSG "log restore source tenant state not match, switchover to primary not allowed" -#define OB_SOURCE_LS_STATE_NOT_MATCH__USER_ERROR_MSG "log restore source ls state not match, switchover to primary not allowed" +#define OB_SOURCE_LS_STATE_NOT_MATCH__USER_ERROR_MSG "log restore source LS state not match, switchover to primary is not allowed" #define OB_ESI_SESSION_NOT_EXIST__USER_ERROR_MSG "obesi process session not exist" #define OB_ALREADY_IN_ARCHIVE_MODE__USER_ERROR_MSG "Already in ARCHIVELOG mode" #define OB_ALREADY_IN_NOARCHIVE_MODE__USER_ERROR_MSG "Already in NOARCHIVELOG mode" @@ -8102,8 +8102,8 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_CLOG_RECYCLE_BEFORE_ARCHIVE__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -9087, observer clog is recycled before archive" #define OB_SOURCE_TENANT_STATE_NOT_MATCH__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9088, log restore source tenant state not match, switchover to primary not allowed" #define OB_SOURCE_TENANT_STATE_NOT_MATCH__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -9088, log restore source tenant state not match, switchover to primary not allowed" -#define OB_SOURCE_LS_STATE_NOT_MATCH__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9089, log restore source ls state not match, switchover to primary not allowed" -#define OB_SOURCE_LS_STATE_NOT_MATCH__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -9089, log restore source ls state not match, switchover to primary not allowed" +#define OB_SOURCE_LS_STATE_NOT_MATCH__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9089, log restore source LS state not match, switchover to primary is not allowed" +#define OB_SOURCE_LS_STATE_NOT_MATCH__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -9089, log restore source LS state not match, switchover to primary is not allowed" #define OB_ESI_SESSION_NOT_EXIST__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9090, obesi process session not exist" #define OB_ESI_SESSION_NOT_EXIST__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -9090, obesi process session not exist" #define OB_ALREADY_IN_ARCHIVE_MODE__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9091, Already in ARCHIVELOG mode" diff --git a/src/share/ob_event_history_table_operator.cpp b/src/share/ob_event_history_table_operator.cpp index 179269af8..b2e3f20e7 100644 --- a/src/share/ob_event_history_table_operator.cpp +++ b/src/share/ob_event_history_table_operator.cpp @@ -81,22 +81,24 @@ ObAsyncTask *ObEventTableClearTask::deep_copy(char *buf, const int64_t buf_size) //////////////////////////////////////////////////////////////// ObEventHistoryTableOperator::ObEventTableUpdateTask::ObEventTableUpdateTask( - ObEventHistoryTableOperator &table_operator, const bool is_delete, const int64_t create_time) + ObEventHistoryTableOperator &table_operator, const bool is_delete, + const int64_t create_time, const uint64_t exec_tenant_id) : IObDedupTask(T_RS_ET_UPDATE), table_operator_(table_operator), is_delete_(is_delete), - create_time_(create_time) + create_time_(create_time), exec_tenant_id_(exec_tenant_id) { } int ObEventHistoryTableOperator::ObEventTableUpdateTask::init(const char *ptr, - const int64_t buf_size) + const int64_t buf_size, const uint64_t exec_tenant_id) { int ret = OB_SUCCESS; - if (OB_ISNULL(ptr) || buf_size <= 0) { - LOG_WARN("invalid argument", KP(ptr), K(buf_size)); + if (OB_ISNULL(ptr) || OB_UNLIKELY(buf_size <= 0 || !is_valid_tenant_id(exec_tenant_id))) { + LOG_WARN("invalid argument", KP(ptr), K(buf_size), K(exec_tenant_id)); ret = OB_INVALID_ARGUMENT; } else { sql_.assign_ptr(ptr, static_cast(buf_size)); + exec_tenant_id_ = exec_tenant_id; } return ret; @@ -105,7 +107,7 @@ int ObEventHistoryTableOperator::ObEventTableUpdateTask::init(const char *ptr, bool ObEventHistoryTableOperator::ObEventTableUpdateTask::is_valid() const { - return table_operator_.is_inited() && !sql_.empty(); + return table_operator_.is_inited() && !sql_.empty() && is_valid_tenant_id(exec_tenant_id_); } int64_t ObEventHistoryTableOperator::ObEventTableUpdateTask::hash() const @@ -135,8 +137,9 @@ bool ObEventHistoryTableOperator::ObEventTableUpdateTask::operator==( is_equal = true; } else { is_equal = (&(this->table_operator_) == &(o.table_operator_)) - && this->sql_ == o.sql_ && this->is_delete_ == o.is_delete_; - //no need take care of create_time + && this->sql_ == o.sql_ && this->is_delete_ == o.is_delete_ + && this->exec_tenant_id_ == o.exec_tenant_id_; + //no need take care of create_time } } return is_equal; @@ -152,7 +155,7 @@ IObDedupTask *ObEventHistoryTableOperator::ObEventTableUpdateTask::deep_copy( LOG_WARN_RET(OB_INVALID_ARGUMENT, "invalid argument", "buf", reinterpret_cast(buf), K(buf_size), "need size", get_deep_copy_size()); } else { - task = new (buf) ObEventTableUpdateTask(table_operator_, is_delete_, create_time_); + task = new (buf) ObEventTableUpdateTask(table_operator_, is_delete_, create_time_, exec_tenant_id_); char *ptr = buf + sizeof(ObEventTableUpdateTask); MEMCPY(ptr, sql_.ptr(), sql_.length()); task->assign_ptr(ptr, sql_.length()); @@ -166,8 +169,8 @@ int ObEventHistoryTableOperator::ObEventTableUpdateTask::process() if (!this->is_valid()) { ret = OB_INNER_STAT_ERROR; LOG_WARN("invalid event task update task", "task", *this, K(ret)); - } else if (OB_FAIL(table_operator_.process_task(sql_, is_delete_, create_time_))) { - LOG_WARN("process_task failed", K_(sql), K_(is_delete), KR(ret), K(create_time_)); + } else if (OB_FAIL(table_operator_.process_task(sql_, is_delete_, create_time_, exec_tenant_id_))) { + LOG_WARN("process_task failed", KR(ret), K_(sql), K_(is_delete), K(create_time_), K_(exec_tenant_id)); } return ret; } @@ -269,25 +272,27 @@ int ObEventHistoryTableOperator::default_async_delete() return ret; } -int ObEventHistoryTableOperator::add_task(const ObSqlString &sql, const bool is_delete, const int64_t create_time) +int ObEventHistoryTableOperator::add_task(const ObSqlString &sql, const bool is_delete, + const int64_t create_time, const uint64_t exec_tenant_id) { int ret = OB_SUCCESS; if (!inited_) { ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret)); - } else if (sql.empty()) { + LOG_WARN("not iget_deep_copy_sizenit", K(ret)); + } else if (OB_UNLIKELY(sql.empty() || !is_valid_tenant_id(exec_tenant_id))) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("sql is empty", K(sql), K(ret)); + LOG_WARN("sql is empty", K(sql), K(exec_tenant_id), K(ret)); } else if (stopped_) { ret = OB_CANCELED; LOG_WARN("observer is stopped, cancel task", K(sql), K(is_delete), K(ret)); } else { int64_t new_create_time = OB_INVALID_TIMESTAMP == create_time ? ObTimeUtility::current_time() : create_time; - ObEventTableUpdateTask task(*this, is_delete, new_create_time); - if (OB_FAIL(task.init(sql.ptr(), sql.length() + 1))) { // extra byte for '\0' + ObEventTableUpdateTask task(*this, is_delete, new_create_time, exec_tenant_id); + if (OB_FAIL(task.init(sql.ptr(), sql.length() + 1, exec_tenant_id))) { // extra byte for '\0' LOG_WARN("task init error", K(ret)); - } else if (OB_FAIL(event_queue_.add_task(task))) { + } + if (FAILEDx(event_queue_.add_task(task))) { if (OB_EAGAIN == ret) { ret = OB_ERR_UNEXPECTED; LOG_WARN("duplicated task is not expected to exist", K(task), K(ret)); @@ -298,11 +303,11 @@ int ObEventHistoryTableOperator::add_task(const ObSqlString &sql, const bool is_ // do nothing } } - return ret; } -int ObEventHistoryTableOperator::process_task(const ObString &sql, const bool is_delete, const int64_t create_time) +int ObEventHistoryTableOperator::process_task(const ObString &sql, const bool is_delete, + const int64_t create_time, const uint64_t exec_tenant_id) { int ret = OB_SUCCESS; @@ -310,9 +315,9 @@ int ObEventHistoryTableOperator::process_task(const ObString &sql, const bool is if (!inited_) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); - } else if (sql.empty()) { + } else if (OB_UNLIKELY(sql.empty() || !is_valid_tenant_id(exec_tenant_id))) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("sql is empty", K(sql), K(ret)); + LOG_WARN("sql is empty", K(sql), K(exec_tenant_id), K(ret)); } else { if (stopped_) { ret = OB_CANCELED; @@ -320,7 +325,7 @@ int ObEventHistoryTableOperator::process_task(const ObString &sql, const bool is } else { int64_t affected_rows = 0; if (!is_delete) { - if (OB_FAIL(proxy_->write(sql.ptr(), affected_rows))) { + if (OB_FAIL(proxy_->write(exec_tenant_id, sql.ptr(), affected_rows))) { LOG_WARN("execute sql failed", K(sql), K(ret)); } else if (!is_single_row(affected_rows)) { ret = OB_ERR_UNEXPECTED; diff --git a/src/share/ob_event_history_table_operator.h b/src/share/ob_event_history_table_operator.h index 255c11895..5715bda86 100644 --- a/src/share/ob_event_history_table_operator.h +++ b/src/share/ob_event_history_table_operator.h @@ -71,26 +71,27 @@ public: { public: ObEventTableUpdateTask(ObEventHistoryTableOperator &table_operator, const bool is_delete, - const int64_t create_time); + const int64_t create_time, const uint64_t exec_tenant_id); virtual ~ObEventTableUpdateTask() {} - int init(const char *ptr, const int64_t buf_size); + int init(const char *ptr, const int64_t buf_size, const uint64_t exec_tenant_id = OB_SYS_TENANT_ID); bool is_valid() const; virtual int64_t hash() const; virtual bool operator==(const common::IObDedupTask &other) const; virtual int64_t get_deep_copy_size() const { return sizeof(*this) + sql_.length(); } virtual common::IObDedupTask *deep_copy(char *buf, const int64_t buf_size) const; virtual int64_t get_abs_expired_time() const { return 0; } + virtual uint64_t get_exec_tenant_id() const { return exec_tenant_id_; } virtual int process(); public: void assign_ptr(char *ptr, const int64_t buf_size) { sql_.assign_ptr(ptr, static_cast(buf_size));} - - TO_STRING_KV(K_(sql), K_(is_delete), K_(create_time)); + TO_STRING_KV(K_(sql), K_(is_delete), K_(create_time), K_(exec_tenant_id)); private: ObEventHistoryTableOperator &table_operator_; common::ObString sql_; bool is_delete_; int64_t create_time_; + uint64_t exec_tenant_id_; DISALLOW_COPY_AND_ASSIGN(ObEventTableUpdateTask); }; @@ -114,6 +115,10 @@ public: // number of others should not less than 0, or more than 13 // if number of others is not 13, should be even, every odd of them are name, every even of them are value template + int async_add_tenant_event(const uint64_t tenant_id, const char *module, const char *event, + const int64_t event_timestamp, const int user_ret, const int64_t cost_sec, Rest &&...others); + // number of others should not less than 0, or more than 13 + template int add_event_with_retry(const char *module, const char *event, Rest &&...others); virtual int async_delete() = 0; @@ -141,7 +146,7 @@ protected: void set_event_table(const char* tname) { event_table_name_ = tname; } const char *get_event_table() const { return event_table_name_; } int add_task(const common::ObSqlString &sql, const bool is_delete = false, - const int64_t create_time = OB_INVALID_TIMESTAMP); + const int64_t create_time = OB_INVALID_TIMESTAMP, const uint64_t exec_tenant_id = OB_SYS_TENANT_ID); int gen_event_ts(int64_t &event_ts); protected: static constexpr const char * names[7] = {"name1", "name2", "name3", "name4", "name5", "name6", "extra_info"}; // only valid in compile time @@ -153,7 +158,7 @@ protected: static const int64_t TASK_QUEUE_SIZE = 20 *1024; static const int64_t MAX_RETRY_COUNT = 12; - virtual int process_task(const common::ObString &sql, const bool is_delete, const int64_t create_time); + virtual int process_task(const common::ObString &sql, const bool is_delete, const int64_t create_time, const uint64_t exec_tenant_id); private: bool inited_; volatile bool stopped_; @@ -320,6 +325,80 @@ int ObEventHistoryTableOperator::sync_add_event(const char *module, const char * return ret; } +template +int ObEventHistoryTableOperator::async_add_tenant_event( + const uint64_t tenant_id, const char *module, const char *event, const int64_t event_timestamp, + const int user_ret, const int64_t cost_sec, Rest &&...others) +{ + static_assert(sizeof...(others) >= 0 && sizeof...(others) <= 13 && + (sizeof...(others) == 13 || (sizeof...(others) % 2 == 0)), + "max support 6 pair of name-value args and 1 extra info, if number of others is not 13, should be even"); + int ret = common::OB_SUCCESS; + int64_t affected_rows = 0; + common::ObSqlString sql; + share::ObDMLSqlSplicer dml; + char ip_buf[common::MAX_IP_ADDR_LENGTH]; + uint64_t compat_version = 0; + if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) { + SHARE_LOG(WARN, "fail to get data version", KR(ret), K(tenant_id)); + } else if (OB_UNLIKELY(!(compat_version >= DATA_VERSION_4_3_3_0 + || (compat_version >= DATA_VERSION_4_2_2_0 && compat_version < DATA_VERSION_4_3_0_0) + || (compat_version >= MOCK_DATA_VERSION_4_2_1_8 && compat_version < DATA_VERSION_4_2_2_0)))) { + ret = common::OB_NOT_SUPPORTED; + SHARE_LOG(WARN, "only (version >= 4_2_1_8 and version < 4_2_2_0) " + "or version >= 4_2_2_0 and version < 4_3_0_0 " + "or version >= 4_3_3_0 support this operation", KR(ret), K(compat_version)); + } else if (OB_UNLIKELY(!inited_)) { + ret = common::OB_NOT_INIT; + SHARE_LOG(WARN, "not init", KR(ret)); + } else if (OB_ISNULL(module) || OB_ISNULL(event)) { + ret = common::OB_INVALID_ARGUMENT; + SHARE_LOG(WARN, "neither module or event can be NULL", KR(ret), KP(module), KP(event)); + } else if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id)) { + ret = OB_INVALID_ARGUMENT; + SHARE_LOG(WARN, "tenant_id is invalid", KR(ret), K(tenant_id)); + } else if (OB_FAIL(dml.add_gmt_create(event_timestamp)) + || OB_FAIL(dml.add_column("module", module)) + || OB_FAIL(dml.add_column("event", event) + || OB_FAIL(dml.add_column("tenant_id", tenant_id)) + || OB_FAIL(dml.add_column("ret_code", user_ret)) + || OB_FAIL(dml.add_column("cost_time", cost_sec)))) { + SHARE_LOG(WARN, "add column failed", KR(ret), K(event_timestamp), "module", module, "event", event, K(tenant_id), KR(user_ret), K(cost_sec)); + } else if (OB_FAIL((add_event_helper_<0, false>(dml, std::forward(others)...)))) {// recursive call + } else if (common::OB_SUCCESS == ret && self_addr_.is_valid()) { + (void)self_addr_.ip_to_string(ip_buf, common::MAX_IP_ADDR_LENGTH); + if (OB_FAIL(dml.add_column("svr_ip", ip_buf)) + || OB_FAIL(dml.add_column("svr_port", self_addr_.get_port()))) { + SHARE_LOG(WARN, "add column failed", KR(ret), K(ip_buf), K(self_addr_.get_port())); + } + } + + if (OB_SUCC(ret)) { + const int64_t MAX_TRACE_ID_LENGTH = 64; + char trace_id_buf[MAX_TRACE_ID_LENGTH] = {0}; + ::oceanbase::common::ObCurTraceId::TraceId *trace_id = ObCurTraceId::get_trace_id(); + if (OB_NOT_NULL(trace_id)) { + if (FALSE_IT(trace_id->to_string(trace_id_buf, sizeof(trace_id_buf)))) { + } else if (OB_FAIL(dml.add_column("trace_id", trace_id_buf))) { + SHARE_LOG(WARN, "add trace_id column failed", KR(ret), K(trace_id_buf), KPC(trace_id)); + } + } + } + + if (OB_SUCC(ret)) { + uint64_t exec_tenant_id = gen_meta_tenant_id(tenant_id); + if (OB_FAIL(dml.splice_insert_sql(event_table_name_, sql))) { + SHARE_LOG(WARN, "splice_insert_sql failed", KR(ret), K(sql)); + } else if (OB_FAIL(add_task(sql, false, OB_INVALID_TIMESTAMP, exec_tenant_id))) { + SHARE_LOG(WARN, "add_task failed", K(sql), K(exec_tenant_id), KR(ret)); + } else { + ObTaskController::get().allow_next_syslog(); + SHARE_LOG(INFO, "event table async add event success", KR(ret), K_(event_table_name), K(sql), K(tenant_id), K(exec_tenant_id)); + } + } + return ret; +} + template int ObEventHistoryTableOperator::add_event_with_retry(const char *module, const char *event, Rest &&...others) { diff --git a/src/share/ob_log_restore_proxy.cpp b/src/share/ob_log_restore_proxy.cpp index 278804464..61b99f3f6 100644 --- a/src/share/ob_log_restore_proxy.cpp +++ b/src/share/ob_log_restore_proxy.cpp @@ -31,6 +31,7 @@ #include "share/config/ob_server_config.h" #include "logservice/palf/palf_options.h" #include "share/oracle_errno.h" +#include "share/backup/ob_log_restore_struct.h" #include namespace oceanbase @@ -265,6 +266,32 @@ int ObLogRestoreProxyUtil::init(const uint64_t tenant_id, } return ret; } +int ObLogRestoreProxyUtil::init_with_service_attr( + const uint64_t tenant_id, + const ObRestoreSourceServiceAttr *service_attr) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(service_attr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("service_attr is null", KR(ret), KP(service_attr)); + } else if (OB_UNLIKELY(!service_attr->is_valid() || !is_valid_tenant_id(tenant_id))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("service attr or tenant id is invalid", KR(ret), KPC(service_attr), K(tenant_id)); + } else { + const char *db_name = service_attr->user_.mode_ == common::ObCompatibilityMode::MYSQL_MODE ? OB_SYS_DATABASE_NAME : OB_ORA_SYS_SCHEMA_NAME; + ObSqlString user; + char passwd[OB_MAX_PASSWORD_LENGTH + 1] = {0}; + if (OB_FAIL(service_attr->get_password(passwd, sizeof(passwd)))) { + LOG_WARN("fail to get password", KR(ret), K(service_attr)); + } else if (OB_FAIL(service_attr->get_user_str_(user))) { + LOG_WARN("fail to get user str", KR(ret), K(service_attr)); + } else if (OB_FAIL(init(tenant_id, service_attr->addr_, + user.ptr(), passwd, db_name))) { + LOG_WARN("fail to init proxy_util", KR(ret), KPC(service_attr)); + } + } + return ret; +} int ObLogRestoreProxyUtil::get_sql_proxy(common::ObMySQLProxy *&proxy) { @@ -640,11 +667,12 @@ bool ObLogRestoreProxyUtil::is_user_changed_(const char *user_name, const char * return changed; } -int ObLogRestoreProxyUtil::get_tenant_info(ObTenantRole &role, schema::ObTenantStatus &status) +int ObLogRestoreProxyUtil::get_tenant_info(ObTenantRole &role, schema::ObTenantStatus &status, ObTenantSwitchoverStatus &switchover_status) { int ret = OB_SUCCESS; const char *TENANT_ROLE = "TENANT_ROLE"; const char *TENANT_STATUS = "STATUS"; + const char *SWITCHOVER_STATUS = "SWITCHOVER_STATUS"; common::ObMySQLProxy *proxy = &sql_proxy_; if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; @@ -653,8 +681,8 @@ int ObLogRestoreProxyUtil::get_tenant_info(ObTenantRole &role, schema::ObTenantS SMART_VAR(common::ObMySQLProxy::MySQLResult, res) { common::sqlclient::ObMySQLResult *result = NULL; common::ObSqlString sql; - const char *GET_TENANT_INFO_SQL = "SELECT %s, %s FROM %s"; - if (OB_FAIL(sql.append_fmt(GET_TENANT_INFO_SQL, TENANT_ROLE, TENANT_STATUS, OB_DBA_OB_TENANTS_TNAME))) { + const char *GET_TENANT_INFO_SQL = "SELECT %s, %s, %s FROM %s"; + if (OB_FAIL(sql.append_fmt(GET_TENANT_INFO_SQL, TENANT_ROLE, TENANT_STATUS, SWITCHOVER_STATUS, OB_DBA_OB_TENANTS_TNAME))) { LOG_WARN("append_fmt failed"); } else if (OB_FAIL(proxy->read(res, sql.ptr()))) { LOG_WARN("excute sql failed", K(sql)); @@ -664,13 +692,20 @@ int ObLogRestoreProxyUtil::get_tenant_info(ObTenantRole &role, schema::ObTenantS } else if (OB_FAIL(result->next())) { LOG_WARN("next failed"); } else { + ObString switchover_status_str; ObString status_str; ObString role_str; EXTRACT_VARCHAR_FIELD_MYSQL(*result, TENANT_ROLE, role_str); EXTRACT_VARCHAR_FIELD_MYSQL(*result, TENANT_STATUS, status_str); + EXTRACT_VARCHAR_FIELD_MYSQL(*result, SWITCHOVER_STATUS, switchover_status_str); + ObTenantSwitchoverStatus so_status(switchover_status_str); + switchover_status = so_status; if (OB_SUCC(ret)) { if (OB_FAIL(schema::get_tenant_status(status_str, status))) { LOG_WARN("get tenant status failed"); + } else if (OB_UNLIKELY(!switchover_status.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid switchover status", KR(ret), K(switchover_status_str), K(so_status), K(switchover_status)); } else { role = ObTenantRole(role_str.ptr()); } diff --git a/src/share/ob_log_restore_proxy.h b/src/share/ob_log_restore_proxy.h index 702c8418b..831c0e38e 100644 --- a/src/share/ob_log_restore_proxy.h +++ b/src/share/ob_log_restore_proxy.h @@ -26,6 +26,7 @@ #include "share/ob_tenant_role.h" #include "share/ob_root_addr_agent.h"//ObRootAddr #include "share/schema/ob_schema_struct.h" +#include "share/ob_tenant_switchover_status.h" #include "logservice/palf/palf_options.h" #include @@ -33,6 +34,7 @@ namespace oceanbase { namespace share { +class ObRestoreSourceServiceAttr; class ObLogRestoreMySQLProvider : public common::sqlclient::ObMySQLServerProvider { public: @@ -112,6 +114,7 @@ public: const char *user_name, const char *user_password, const char *db_name); + int init_with_service_attr(const uint64_t tenant_id, const ObRestoreSourceServiceAttr *service_attr); // destroy proxy, close all connections void destroy(); @@ -145,7 +148,7 @@ public: int get_server_addr(const uint64_t tenant_id, common::ObIArray &addrs); int check_begin_lsn(const uint64_t tenant_id); // get log restore source tenant info, includes tenant role and tennat status - int get_tenant_info(ObTenantRole &role, schema::ObTenantStatus &status); + int get_tenant_info(ObTenantRole &role, schema::ObTenantStatus &status, ObTenantSwitchoverStatus &switchover_status); // get the access_mode and max_scn of the specific LS in log restore source tenant int get_max_log_info(const ObLSID &id, palf::AccessMode &mode, SCN &scn); // get ls from dba_ob_ls diff --git a/src/share/ob_max_id_fetcher.cpp b/src/share/ob_max_id_fetcher.cpp index 3247138e0..1fdd77ce6 100755 --- a/src/share/ob_max_id_fetcher.cpp +++ b/src/share/ob_max_id_fetcher.cpp @@ -82,6 +82,7 @@ const char *ObMaxIdFetcher::max_id_name_info_[OB_MAX_ID_TYPE][2] = { { "ob_max_used_rls_policy_id", "max used ddl rls policy id"}, { "ob_max_used_rls_group_id", "max used ddl rls group id"}, { "ob_max_used_rls_context_id", "max used ddl rls context id"}, + { "ob_max_used_service_name_id", "max used service name id"} }; lib::ObMutex ObMaxIdFetcher::mutex_bucket_[MAX_TENANT_MUTEX_BUCKET_CNT]; @@ -123,6 +124,7 @@ int ObMaxIdFetcher::convert_id_type( case OB_MAX_USED_OBJECT_ID_TYPE: case OB_MAX_USED_LOCK_OWNER_ID_TYPE: case OB_MAX_USED_REWRITE_RULE_VERSION_TYPE: + case OB_MAX_USED_SERVICE_NAME_ID_TYPE: case OB_MAX_USED_TTL_TASK_ID_TYPE: { dst = src; break; @@ -314,6 +316,7 @@ int ObMaxIdFetcher::fetch_new_max_id(const uint64_t tenant_id, case OB_MAX_USED_LS_ID_TYPE: case OB_MAX_USED_LS_GROUP_ID_TYPE: case OB_MAX_USED_REWRITE_RULE_VERSION_TYPE: + case OB_MAX_USED_SERVICE_NAME_ID_TYPE: case OB_MAX_USED_TTL_TASK_ID_TYPE: { // won't check other id break; @@ -425,9 +428,6 @@ int ObMaxIdFetcher::update_max_id(ObISQLClient &sql_client, const uint64_t tenan || OB_INVALID_ID == max_id) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(tenant_id), K(max_id_type), K(max_id)); - } else if (GCTX.is_standby_cluster() && OB_SYS_TENANT_ID != tenant_id) { - ret = OB_OP_NOT_ALLOW; - LOG_WARN("can't write sys table now", K(ret), K(tenant_id)); } else if (OB_ISNULL(id_name = get_max_id_name(max_id_type))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("NULL name", K(ret)); @@ -528,9 +528,6 @@ int ObMaxIdFetcher::insert_initial_value(common::ObISQLClient &sql_client, uint6 if (OB_INVALID_ID == tenant_id || !valid_max_id_type(max_id_type) || UINT64_MAX == initial_value) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(tenant_id), K(max_id_type), K(initial_value)); - } else if (GCTX.is_standby_cluster() && OB_SYS_TENANT_ID != tenant_id) { - ret = OB_OP_NOT_ALLOW; - LOG_WARN("can't write sys table now", K(ret), K(tenant_id)); } else if (OB_ISNULL(name) || OB_ISNULL(info)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("NULL name or info", K(ret), KP(name), KP(info)); diff --git a/src/share/ob_rpc_struct.cpp b/src/share/ob_rpc_struct.cpp index 20a4a510e..346451fdc 100644 --- a/src/share/ob_rpc_struct.cpp +++ b/src/share/ob_rpc_struct.cpp @@ -11991,5 +11991,105 @@ int ObCheckServerMachineStatusResult::assign(const ObCheckServerMachineStatusRes } return ret; } +OB_SERIALIZE_MEMBER(ObRefreshServiceNameArg, tenant_id_, epoch_, from_server_, target_service_name_id_, + service_name_list_, service_op_, update_tenant_info_arg_); +int ObRefreshServiceNameArg::init( + const uint64_t tenant_id, + const uint64_t epoch, + const ObAddr &from_server, + const share::ObServiceNameID &target_service_name_id, + const common::ObIArray &service_name_list, + const share::ObServiceNameArg::ObServiceOp &service_op, + const share::ObAllTenantInfo &tenant_info, + const int64_t ora_rowscn) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || 0 == epoch || INT64_MAX == epoch + || !from_server.is_valid() || !target_service_name_id.is_valid() || service_name_list.count() <= 0 + || !ObServiceNameArg::is_valid_service_op(service_op))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg", KR(ret), K(tenant_id), K(epoch), K(from_server), K(target_service_name_id), + K(service_name_list), K(service_op)); + } else { + tenant_id_ = tenant_id; + epoch_ = epoch; + from_server_ = from_server; + target_service_name_id_ = target_service_name_id; + service_op_ = service_op; + for (int64_t i = 0; i < service_name_list.count(); ++i) { + const ObServiceName &service_name = service_name_list.at(i); + if (OB_UNLIKELY(!service_name.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid service_name", KR(ret), K(service_name), K(i), K(service_name_list)); + } else if (OB_FAIL(service_name_list_.push_back(service_name))) { + LOG_WARN("fail to push back", KR(ret), K(service_name), K(service_name_list_)); + } + } + } + if (OB_SUCC(ret) && is_start_service()) { + + if (OB_FAIL(update_tenant_info_arg_.init(tenant_id, tenant_info, ora_rowscn, 0 /*finish_data_version*/, SCN::min_scn()))) { + LOG_WARN("fail to init update_tenant_info_arg_", KR(ret), K(tenant_id), K(tenant_info), K(ora_rowscn)); + } + } + return ret; +} +bool ObRefreshServiceNameArg::is_valid() const +{ + bool service_name_list_valid = service_name_list_.count() > 0; + for (int64_t i = 0; i < service_name_list_.count() && service_name_list_valid; ++i) { + if (OB_UNLIKELY(!service_name_list_.at(i).is_valid())) { + service_name_list_valid = false; + } + } + return is_valid_tenant_id(tenant_id_) + && 0 != epoch_ && INT64_MAX != epoch_ && from_server_.is_valid() + && target_service_name_id_.is_valid() && service_name_list_valid + && ObServiceNameArg::is_valid_service_op(service_op_) + && (!is_start_service() || update_tenant_info_arg_.is_valid()); +} +int ObRefreshServiceNameArg::assign(const ObRefreshServiceNameArg &other) +{ + int ret = OB_SUCCESS; + if (this != &other) { + if (OB_FAIL(service_name_list_.assign(other.service_name_list_))) { + LOG_WARN("fail to assign service_name_str_", KR(ret), K(other)); + } else if (OB_FAIL(update_tenant_info_arg_.assign(other.update_tenant_info_arg_))) { + LOG_WARN("fail to assign update_tenant_info_arg_", KR(ret), K(other)); + } else { + tenant_id_ = other.tenant_id_; + epoch_ = other.epoch_; + from_server_ = other.from_server_; + target_service_name_id_ = other.target_service_name_id_; + service_op_ = other.service_op_; + } + } + return ret; +} +OB_SERIALIZE_MEMBER(ObRefreshServiceNameRes, tenant_id_); +int ObRefreshServiceNameRes::init(const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg", KR(ret), K(tenant_id)); + } else { + tenant_id_ = tenant_id; + } + return ret; +} +bool ObRefreshServiceNameRes::is_valid() const +{ + return is_valid_tenant_id(tenant_id_); +} +int ObRefreshServiceNameRes::assign(const ObRefreshServiceNameRes &other) +{ + int ret = OB_SUCCESS; + if (this != &other) { + tenant_id_ = other.tenant_id_; + } + return ret; +} + }//end namespace obrpc }//end namespace oceanbase diff --git a/src/share/ob_rpc_struct.h b/src/share/ob_rpc_struct.h index a9ce8dee4..759c85e76 100644 --- a/src/share/ob_rpc_struct.h +++ b/src/share/ob_rpc_struct.h @@ -78,6 +78,7 @@ #include "share/tenant_snapshot/ob_tenant_snapshot_id.h" #include "share/location_cache/ob_location_update_task.h" #include "share/resource_limit_calculator/ob_resource_limit_calculator.h"//ObUserResourceCalculateArg +#include "share/ob_service_name_proxy.h" #include "share/ob_heartbeat_handler.h" #include "storage/tablelock/ob_table_lock_common.h" //ObTableLockPriority @@ -8125,6 +8126,9 @@ public: bool is_valid() const; bool operator==(const obrpc::ObCheckpoint &r) const; + bool operator < (const ObCheckpoint &that) { + return this->cur_sync_scn_ < that.cur_sync_scn_; + } share::ObLSID get_ls_id() const { return ls_id_; @@ -8144,7 +8148,8 @@ public: && cur_sync_scn_.is_valid_and_not_min() && cur_restore_source_max_scn_.is_valid_and_not_min()); } - TO_STRING_KV(K_(ls_id), K_(cur_sync_scn), K_(cur_restore_source_max_scn)); + TO_STRING_KV("ls_id", ls_id_.id(), "cur_sync_scn", cur_sync_scn_.get_val_for_inner_table_field(), + "cur_restore_source_max_scn", cur_restore_source_max_scn_.get_val_for_inner_table_field()); share::ObLSID ls_id_; share::SCN cur_sync_scn_; @@ -12121,6 +12126,70 @@ public: private: share::ObServerHealthStatus server_health_status_; }; + +struct ObRefreshServiceNameArg +{ + OB_UNIS_VERSION(1); +public: + ObRefreshServiceNameArg() + : tenant_id_(OB_INVALID_TENANT_ID), + epoch_(0), + from_server_(), + target_service_name_id_(), + service_name_list_(), + service_op_(share::ObServiceNameArg::INVALID_SERVICE_OP), + update_tenant_info_arg_() {} + ~ObRefreshServiceNameArg() {} + int init( + const uint64_t tenant_id, + const uint64_t epoch, + const ObAddr &from_server, + const share::ObServiceNameID &target_service_name_id, + const common::ObIArray &service_name_list, + const share::ObServiceNameArg::ObServiceOp &service_op, + const share::ObAllTenantInfo &tenant_info, + const int64_t ora_rowscn); + bool is_valid() const; + int assign(const ObRefreshServiceNameArg &other); + uint64_t get_tenant_id() const { return tenant_id_; } + uint64_t get_epoch() const { return epoch_; } + const ObAddr &get_from_server() const { return from_server_; } + const share::ObServiceNameID &get_target_service_name_id() const { return target_service_name_id_; } + const common::ObSArray &get_service_name_list() const { return service_name_list_; } + const ObUpdateTenantInfoCacheArg &get_update_tenant_info_arg() const { return update_tenant_info_arg_; } + bool is_start_service() const {return share::ObServiceNameArg::START_SERVICE == service_op_; } + bool is_stop_service() const {return share::ObServiceNameArg::STOP_SERVICE == service_op_; } + TO_STRING_KV(K_(tenant_id), K_(epoch), K_(from_server), + "target_service_name_id", target_service_name_id_.id(), K_(service_name_list), + "service_op_to_str", share::ObServiceNameArg::service_op_to_str(service_op_), K_(update_tenant_info_arg)); +private: + uint64_t tenant_id_; + uint64_t epoch_; // __all_service snapshot corresponding epoch + ObAddr from_server_; // the server which sends broadcasting request + share::ObServiceNameID target_service_name_id_; // indicate which service name to operate + common::ObSArray service_name_list_; // __all_service snapshot + share::ObServiceNameArg::ObServiceOp service_op_; // the operation which triggered broadcasting + ObUpdateTenantInfoCacheArg update_tenant_info_arg_; // only used if the op is START_SERVICE +private: + DISALLOW_COPY_AND_ASSIGN(ObRefreshServiceNameArg); +}; + +struct ObRefreshServiceNameRes +{ + OB_UNIS_VERSION(1); +public: + ObRefreshServiceNameRes() : tenant_id_(OB_INVALID_TENANT_ID) {} + ~ObRefreshServiceNameRes() {} + int init(const uint64_t tenant_id); + bool is_valid() const; + int assign(const ObRefreshServiceNameRes &other); + uint64_t get_tenant_id() const { return tenant_id_; } + TO_STRING_KV(K_(tenant_id)); +private: + uint64_t tenant_id_; +private: + DISALLOW_COPY_AND_ASSIGN(ObRefreshServiceNameRes); +}; }//end namespace obrpc }//end namespace oceanbase #endif diff --git a/src/share/ob_service_epoch_proxy.cpp b/src/share/ob_service_epoch_proxy.cpp index 04267c635..59e713624 100644 --- a/src/share/ob_service_epoch_proxy.cpp +++ b/src/share/ob_service_epoch_proxy.cpp @@ -34,7 +34,8 @@ int ObServiceEpochProxy::init_service_epoch( const int64_t freeze_service_epoch, const int64_t arbitration_service_epoch, const int64_t server_zone_op_service_epoch, - const int64_t heartbeat_service_epoch) + const int64_t heartbeat_service_epoch, + const int64_t service_name_epoch) { int ret = OB_SUCCESS; if (is_user_tenant(tenant_id)) { @@ -94,6 +95,12 @@ int ObServiceEpochProxy::init_service_epoch( ARBITRATION_SERVICE_EPOCH, arbitration_service_epoch))) { LOG_WARN("fail to init arb service epoch", KR(ret), K(user_tenant_id), K(arbitration_service_epoch)); + } else if (OB_FAIL(ObServiceEpochProxy::insert_service_epoch( + sql_proxy, + user_tenant_id, + SERVICE_NAME_EPOCH, + service_name_epoch))) { + LOG_WARN("fail to init service name epoch", KR(ret), K(user_tenant_id), K(service_name_epoch)); } else {} } else {} return ret; diff --git a/src/share/ob_service_epoch_proxy.h b/src/share/ob_service_epoch_proxy.h index 9ebd83bce..273f6d935 100644 --- a/src/share/ob_service_epoch_proxy.h +++ b/src/share/ob_service_epoch_proxy.h @@ -51,7 +51,8 @@ public: const int64_t freeze_service_epoch, const int64_t arbitration_service_epoch, const int64_t server_zone_op_service_epoch, - const int64_t heartbeat_service_epoch); + const int64_t heartbeat_service_epoch, + const int64_t service_name_epoch); static int insert_service_epoch(common::ObISQLClient &sql_proxy, const int64_t tenant_id, @@ -94,6 +95,7 @@ public: constexpr static const char * const ARBITRATION_SERVICE_EPOCH = "arbitration_service_epoch"; constexpr static const char * const SERVER_ZONE_OP_SERVICE_EPOCH = "server_zone_op_service_epoch"; constexpr static const char * const HEARTBEAT_SERVICE_EPOCH = "heartbeat_service_epoch"; + constexpr static const char * const SERVICE_NAME_EPOCH = "service_name_epoch"; private: static int inner_get_service_epoch_(common::ObISQLClient &sql_proxy, diff --git a/src/share/ob_service_name_proxy.cpp b/src/share/ob_service_name_proxy.cpp new file mode 100644 index 000000000..e6e0368d4 --- /dev/null +++ b/src/share/ob_service_name_proxy.cpp @@ -0,0 +1,630 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#define USING_LOG_PREFIX SHARE + +#include "ob_service_name_proxy.h" + +#include "lib/mysqlclient/ob_mysql_result.h" +#include "lib/mysqlclient/ob_mysql_proxy.h" +#include "lib/mysqlclient/ob_mysql_transaction.h" +#include "lib/string/ob_sql_string.h" +#include "observer/ob_server_struct.h" +#include "common/ob_timeout_ctx.h" +#include "share/ob_share_util.h" +#include "share/inner_table/ob_inner_table_schema.h" +#include "share/config/ob_server_config.h" +#include "share/ob_tenant_info_proxy.h" +#include "rootserver/ob_tenant_event_def.h" +#include "share/ob_max_id_fetcher.h" +#include "share/ob_service_epoch_proxy.h" + +using namespace oceanbase::tenant_event; +using namespace oceanbase::common::sqlclient; +namespace oceanbase +{ +namespace share +{ +static const char *SERVICE_STATUS_STR[] = { + "INVALID SERVICE STATUS", + "STARTED", + "STOPPING", + "STOPPED", +}; +OB_SERIALIZE_MEMBER(ObServiceNameID,id_); +OB_SERIALIZE_MEMBER(ObServiceNameString,str_); +int ObServiceNameString::init(const ObString &str) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(check_service_name(str))) { + LOG_WARN("fail to execute check_service_name", KR(ret), K(str)); + } else if (OB_FAIL(str_.assign(str))) { + LOG_WARN("fail to assign str_", KR(ret), K(str)); + } + return ret; +} +int ObServiceNameString::assign(const ObServiceNameString &other) +{ + int ret = OB_SUCCESS; + if (this != &other) { + if (OB_FAIL(str_.assign(other.str_))) { + LOG_WARN("fail to assign str_", KR(ret), K(other)); + } + } + return ret; +} +bool ObServiceNameString::equal_to(const ObServiceNameString &service_name_string) const +{ + return 0 == str_.str().case_compare(service_name_string.str_.str()); +} +int ObServiceNameString::check_service_name(const ObString &service_name_str) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(service_name_str.empty() || service_name_str.length() > OB_SERVICE_NAME_LENGTH)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid service_name", KR(ret), K(service_name_str), K(service_name_str.length())); + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "service_name, whose length should be 1 - 64"); + } else if (!isalpha(service_name_str[0])) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid service_name", KR(ret), K(service_name_str)); + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "service_name, which should be start with letter"); + } else { + for (int i = 1; i < service_name_str.length() && OB_SUCC(ret); i++) + { + const char cur_char = service_name_str[i]; + if (!isalpha(cur_char) && !isdigit(cur_char) && cur_char != '_' ) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid service_name", KR(ret), K(service_name_str)); + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "service_name, which can only include letter, digit and underscore"); + } + } + } + return ret; +} + +static const char *SERVICE_OP_STR[] = { + "INVALID SERVICE OPERATION", + "CREATE SERVICE", + "DELETE SERVICE", + "START SERVICE", + "STOP SERVICE", +}; +const char *ObServiceNameArg::service_op_to_str(const ObServiceNameArg::ObServiceOp &service_op) +{ + STATIC_ASSERT(ARRAYSIZEOF(SERVICE_OP_STR) == MAX_SERVICE_OP, "array size mismatch"); + ObServiceOp returned_service_op = INVALID_SERVICE_OP; + if (OB_UNLIKELY(service_op >= MAX_SERVICE_OP + || service_op < INVALID_SERVICE_OP)) { + LOG_ERROR_RET(OB_ERR_UNEXPECTED, "fatal error, unknown service op", K(service_op)); + } else { + returned_service_op = service_op; + } + return SERVICE_OP_STR[returned_service_op]; +} +int ObServiceNameArg::init(const ObServiceOp op, const uint64_t target_tenant_id, const ObString &service_name_str) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_valid_service_op(op) + || !is_valid_tenant_id(target_tenant_id) + || service_name_str.empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("fail to init", KR(ret), K(op), K(target_tenant_id), K(service_name_str)); + } else if (OB_FAIL(service_name_str_.init(service_name_str))) { + LOG_WARN("fail to init service_name_str_", KR(ret), K(service_name_str)); + } else { + op_ = op; + target_tenant_id_ = target_tenant_id; + } + return ret; +} +bool ObServiceNameArg::is_valid() const +{ + return is_valid_service_op(op_) && is_valid_tenant_id(target_tenant_id_) && service_name_str_.is_valid(); +} +bool ObServiceNameArg::is_valid_service_op(ObServiceOp op) +{ + return op > INVALID_SERVICE_OP && op < MAX_SERVICE_OP; +} +int ObServiceNameArg::assign(const ObServiceNameArg &other) +{ + int ret = OB_SUCCESS; + if (this != &other) { + if (OB_FAIL(service_name_str_.assign(other.service_name_str_))) { + LOG_WARN("fail to assign service_name_str_", KR(ret), K(other)); + } else { + op_ = other.op_; + target_tenant_id_ = other.target_tenant_id_; + } + } + return ret; +} +void ObServiceNameArg::reset() +{ + op_ = INVALID_SERVICE_OP; + target_tenant_id_ = OB_INVALID_TENANT_ID; + service_name_str_.reset(); +} +const char *ObServiceName::service_status_to_str(const ObServiceName::ObServiceStatus &service_status) +{ + STATIC_ASSERT(ARRAYSIZEOF(SERVICE_STATUS_STR) == MAX_SERVICE_STATUS, "array size mismatch"); + ObServiceStatus returned_service_status = INVALID_SERVICE_STATUS; + if (OB_UNLIKELY(service_status >= MAX_SERVICE_STATUS + || service_status < INVALID_SERVICE_STATUS)) { + LOG_ERROR_RET(OB_ERR_UNEXPECTED, "fatal error, unknown service status", K(service_status)); + } else { + returned_service_status = service_status; + } + return SERVICE_STATUS_STR[returned_service_status]; +} +ObServiceName::ObServiceStatus ObServiceName::str_to_service_status(const ObString &service_status_str) +{ + ObServiceStatus service_status = INVALID_SERVICE_STATUS; + bool is_found = false; + for (int i = INVALID_SERVICE_STATUS; i < MAX_SERVICE_STATUS && !is_found; i++) + { + if (0 == service_status_str.case_compare(SERVICE_STATUS_STR[i])) { + service_status = static_cast(i); + is_found = true; + } + } + return service_status; +} +bool ObServiceName::is_valid_service_status(const ObServiceName::ObServiceStatus &service_status) +{ + return service_status > INVALID_SERVICE_STATUS && service_status < MAX_SERVICE_STATUS; +} +OB_SERIALIZE_MEMBER(ObServiceName, tenant_id_, service_name_id_, service_name_str_, service_status_); +int ObServiceName::init( + const uint64_t tenant_id, + const uint64_t service_name_id, + const ObString &service_name_str, + const ObString &service_status_str) +{ + int ret = OB_SUCCESS; + ObServiceStatus service_status = str_to_service_status(service_status_str); + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) + || !ObServiceNameID::is_valid_service_name_id(service_name_id) + || !is_valid_service_status(service_status) + || service_name_str.empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg", KR(ret), K(tenant_id), K(service_name_id), K(service_name_str), + K(service_status_str), K(service_status)); + } else if (OB_FAIL(service_name_str_.init(service_name_str))) { + LOG_WARN("fail to init service_name_str_", KR(ret), K(service_name_str)); + } else { + tenant_id_ = tenant_id; + service_name_id_ = service_name_id; + service_status_ = service_status; + } + return ret; +} +bool ObServiceName::is_valid() const +{ + return is_valid_tenant_id(tenant_id_) && service_name_id_.is_valid() + && !service_name_str_.is_empty() && is_valid_service_status(service_status_); +} +int ObServiceName::assign(const ObServiceName &other) +{ + int ret = OB_SUCCESS; + if (this != &other) { + if (OB_FAIL(service_name_str_.assign(other.service_name_str_))) { + LOG_WARN("fail to assign service_name_str_", KR(ret), K(other)); + } else { + service_name_id_ = other.service_name_id_; + service_status_ = other.service_status_; + tenant_id_ = other.tenant_id_; + } + } + return ret; +} +void ObServiceName::reset() +{ + tenant_id_ = OB_INVALID_TENANT_ID; + service_name_id_.reset(); + service_name_str_.reset(); + service_status_ = INVALID_SERVICE_STATUS; +} +int ObServiceNameProxy::select_all_service_names_with_epoch( + const int64_t tenant_id, + int64_t &epoch, + ObIArray &all_service_names) +{ + int ret = OB_SUCCESS; + const bool EXTRACT_EPOCH = true; + ObSqlString sql; + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tenant_id", KR(ret), K(tenant_id)); + } else if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("GCTX.sql_proxy_ is null", KR(ret), KP(GCTX.sql_proxy_)); + } else if (OB_FAIL(sql.assign_fmt("SELECT s.*, e.value as epoch FROM %s AS s " + "JOIN %s AS e ON s.tenant_id = e.tenant_id WHERE s.tenant_id = %lu and e.name='%s' ORDER BY s.gmt_create", + OB_ALL_SERVICE_TNAME, OB_ALL_SERVICE_EPOCH_TNAME, tenant_id, ObServiceEpochProxy::SERVICE_NAME_EPOCH))) { + // join the two tables to avoid add a row lock on __all_service_epoch + // otherwise there might be conflicts and too many retries in tenant_info_loader thread + // when the number of observers is large + LOG_WARN("sql assign_fmt failed", KR(ret), K(sql)); + } else if (OB_FAIL(select_service_name_sql_helper_(*GCTX.sql_proxy_, tenant_id, EXTRACT_EPOCH, + sql, epoch, all_service_names))) { + LOG_WARN("fail to execute select_service_name_sql_helper_", KR(ret), K(tenant_id), K(sql)); + } + return ret; +} +int ObServiceNameProxy::select_all_service_names_( + common::ObISQLClient &sql_proxy, + const int64_t tenant_id, + ObIArray &all_service_names) +{ + int ret = OB_SUCCESS; + ObSqlString sql; + const bool NOT_EXTRACT_EPOCH = false; + int64_t unused_epoch = 0; + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tenant_id", KR(ret), K(tenant_id)); + } else if (OB_FAIL(sql.assign_fmt("SELECT * FROM %s WHERE TENANT_ID = %lu ORDER BY gmt_create", + OB_ALL_SERVICE_TNAME, tenant_id))) { + LOG_WARN("sql assign_fmt failed", KR(ret), K(sql)); + } else if (OB_FAIL(select_service_name_sql_helper_(sql_proxy, tenant_id, NOT_EXTRACT_EPOCH, + sql, unused_epoch, all_service_names))) { + LOG_WARN("fail to execute select_service_name_sql_helper_", KR(ret), K(tenant_id), K(sql)); + } + return ret; +} + +int ObServiceNameProxy::select_service_name_sql_helper_( + common::ObISQLClient &sql_proxy, + const int64_t tenant_id, + const bool extract_epoch, + ObSqlString &sql, + int64_t &epoch, + ObIArray &all_service_names) +{ + int ret = OB_SUCCESS; + ObTimeoutCtx ctx; + const int64_t QUERY_TIMEOUT = 5 * GCONF.rpc_timeout; // 10s + all_service_names.reset(); + if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, QUERY_TIMEOUT))) { + LOG_WARN("fail to get timeout ctx", KR(ret), K(ctx), K(QUERY_TIMEOUT)); + } else { + SMART_VAR(ObMySQLProxy::MySQLResult, res) { + ObMySQLResult *result = NULL; + if (OB_FAIL(sql_proxy.read(res, gen_meta_tenant_id(tenant_id), sql.ptr()))) { + LOG_WARN("execute sql failed", KR(ret), K(sql)); + } else if (NULL == (result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("execute sql failed", K(sql), KR(ret)); + } else { + ObServiceName service_name; + while (OB_SUCC(ret)) { + service_name.reset(); + if (OB_FAIL(result->next())) { + if (OB_ITER_END != ret) { + LOG_WARN("result next failed", KR(ret)); + } else { + ret = OB_SUCCESS; + break; + } + } else if (OB_FAIL(build_service_name_(*result, service_name))) { + LOG_WARN("fail to build server status", KR(ret)); + } else if (OB_FAIL(all_service_names.push_back(service_name))) { + LOG_WARN("fail to build service_name", KR(ret)); + } else if (extract_epoch) { + // epoch can only be extracted when __all_service_epoch table is joined + EXTRACT_INT_FIELD_MYSQL(*result, "epoch", epoch, int64_t); + } + } + } + } + } + return ret; +} +int ObServiceNameProxy::select_service_name( + common::ObISQLClient &sql_proxy, + const uint64_t tenant_id, + const ObServiceNameString &service_name_str, + ObServiceName &service_name) +{ + int ret = OB_SUCCESS; + ObSqlString sql; + ObTimeoutCtx ctx; + const int64_t QUERY_TIMEOUT = 5 * GCONF.rpc_timeout; // 10s + service_name.reset(); + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tenant_id", KR(ret), K(tenant_id)); + } else if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, QUERY_TIMEOUT))) { + LOG_WARN("fail to get timeout ctx", KR(ret), K(ctx), K(QUERY_TIMEOUT)); + } else if (OB_FAIL(sql.assign_fmt("SELECT * FROM %s WHERE TENANT_ID = %lu AND SERVICE_NAME = '%s'", + OB_ALL_SERVICE_TNAME, tenant_id, service_name_str.ptr()))) { + LOG_WARN("sql assign_fmt failed", KR(ret), K(sql), K(tenant_id), K(service_name_str)); + } else { + SMART_VAR(ObMySQLProxy::MySQLResult, res) { + ObMySQLResult *result = NULL; + if (OB_FAIL(sql_proxy.read(res, gen_meta_tenant_id(tenant_id), sql.ptr()))) { + LOG_WARN("execute sql failed", K(sql), KR(ret)); + } else if (NULL == (result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("execute sql failed", K(sql), KR(ret)); + } else if (OB_FAIL(result->next())) { + if (OB_ITER_END == ret) { + ret = OB_SERVICE_NAME_NOT_FOUND; + } + LOG_WARN("fail to get next result", KR(ret)); + } else if (OB_FAIL(build_service_name_(*result, service_name))) { + LOG_WARN("fail to build server status", KR(ret)); + } else { + int tmp_ret = OB_SUCCESS; + if (OB_ITER_END != (tmp_ret = result->next())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get more row than one", KR(ret), KR(tmp_ret), K(sql)); + } + } + } + } + if (OB_SUCC(ret) && OB_UNLIKELY(!service_name.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("select invalid service_name", KR(ret), K(tenant_id), K(service_name_str), K(service_name)); + } + return ret; +} + +int ObServiceNameProxy::insert_service_name( + const uint64_t tenant_id, + const ObServiceNameString &service_name_str, + int64_t &epoch, + ObArray &all_service_names) +{ + // insert when so_status is normal + int ret = OB_SUCCESS; + int64_t service_name_num = INT64_MAX; + ObMySQLTransaction trans; + ObSqlString sql; + const char * service_status_str = ObServiceName::service_status_to_str(ObServiceName::STARTED); + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || !service_name_str.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg", KR(ret), K(tenant_id), K(service_name_str)); + } else if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("GCTX.sql_proxy_ is null", KR(ret), KP(GCTX.sql_proxy_)); + } else if (OB_FAIL(trans_start_and_precheck_(trans, tenant_id, epoch))) { + LOG_WARN("fail to execute trans_start_and_precheck_", KR(ret), K(tenant_id)); + } else if (OB_FAIL(get_tenant_service_name_num(trans, tenant_id, service_name_num))) { + LOG_WARN("fail to get the tenant's service_name_num", KR(ret), K(tenant_id)); + } else if (SERVICE_NAME_MAX_NUM <= service_name_num) { + ret = OB_OP_NOT_ALLOW; + LOG_WARN("The number of service_name for the tenant exceeds the limit", KR(ret), K(service_name_num)); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "The number of service_name for the tenant exceeds the limit, service name related command is"); + } else { + ObMaxIdFetcher id_fetcher(*GCTX.sql_proxy_); + uint64_t new_service_name_id = OB_INVALID_ID; + uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id); + if (OB_FAIL(id_fetcher.fetch_new_max_id( + meta_tenant_id, + OB_MAX_USED_SERVICE_NAME_ID_TYPE, + new_service_name_id, + 0 /* initial */))) { + LOG_WARN("fail to fetch new service_name id", KR(ret), K(tenant_id), K(meta_tenant_id)); + } else if (OB_FAIL(sql.assign_fmt("INSERT INTO %s " + "(tenant_id, service_name_id, service_name, service_status) value (%lu, %lu, '%s', '%s')", + OB_ALL_SERVICE_TNAME, tenant_id, new_service_name_id, service_name_str.ptr(), service_status_str))) { + LOG_WARN("fail to insert service_name", KR(ret), K(tenant_id), K(new_service_name_id), + K(service_name_str), K(service_status_str)); + } + } + (void) write_and_end_trans_(ret, trans, tenant_id, sql, all_service_names); + return ret; +} + +int ObServiceNameProxy::update_service_status( + const ObServiceName &service_name, + const ObServiceName::ObServiceStatus &new_status, + int64_t &epoch, + ObArray &all_service_names) +{ + // update when so_status is normal + int ret = OB_SUCCESS; + ObMySQLTransaction trans; + ObSqlString sql; + const char * old_service_status_str = ObServiceName::service_status_to_str(service_name.get_service_status()); + const char * new_service_status_str = ObServiceName::service_status_to_str(new_status); + const uint64_t service_name_id = service_name.get_service_name_id().id(); + const uint64_t tenant_id = service_name.get_tenant_id(); + if (OB_UNLIKELY(!service_name.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg", KR(ret), K(service_name)); + } else if (service_name.get_service_status() == new_status) { + ret = OB_OP_NOT_ALLOW; + LOG_WARN("the new service status is the same with the old one", KR(ret), K(service_name), K(new_status)); + } else if (OB_FAIL(trans_start_and_precheck_(trans, tenant_id, epoch))) { + LOG_WARN("fail to execute trans_start_and_precheck_", KR(ret), K(tenant_id)); + } else if (OB_FAIL(sql.assign_fmt("UPDATE %s SET service_status = '%s' " + "WHERE tenant_id = %lu AND service_name_id = '%lu' AND service_status = '%s'", + OB_ALL_SERVICE_TNAME, new_service_status_str, tenant_id, service_name_id, old_service_status_str))) { + LOG_WARN("fail to insert service_name", KR(ret), K(tenant_id), K(service_name), K(new_service_status_str)); + } + (void) write_and_end_trans_(ret, trans, tenant_id, sql, all_service_names); + return ret; +} + +int ObServiceNameProxy::delete_service_name(const ObServiceName &service_name) +{ + int ret = OB_SUCCESS; + ObMySQLTransaction trans; + ObSqlString sql; + int64_t epoch = 0; + ObArray all_service_names; + const char * stopped_status_str = ObServiceName::service_status_to_str(ObServiceName::STOPPED); + const uint64_t tenant_id = service_name.get_tenant_id(); + const uint64_t service_name_id = service_name.get_service_name_id().id(); + if (OB_UNLIKELY(!service_name.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid service_name", KR(ret), K(service_name)); + } else if (OB_FAIL(trans_start_and_precheck_(trans, tenant_id, epoch))) { + LOG_WARN("fail to execute trans_start_and_precheck_", KR(ret), K(tenant_id)); + } else if (OB_FAIL(sql.assign_fmt("DELETE FROM %s WHERE tenant_id = %lu AND service_name_id = '%lu' " + "AND service_status = '%s'", + OB_ALL_SERVICE_TNAME, service_name.get_tenant_id(), service_name_id, stopped_status_str))) { + LOG_WARN("fail to insert service_name", KR(ret), K(service_name)); + } + (void) write_and_end_trans_(ret, trans, tenant_id, sql, all_service_names); + return ret; +} + +int ObServiceNameProxy::check_is_service_name_enabled(const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + const uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id); + uint64_t tenant_data_version = 0; + uint64_t meta_tenant_data_version = 0; + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || !is_user_tenant(tenant_id))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("not user tenant", KR(ret), K(tenant_id)); + } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, tenant_data_version))) { + LOG_WARN("fail to get the tenant's min data version", KR(ret), K(tenant_id)); + } else if (!((tenant_data_version >= MOCK_DATA_VERSION_4_2_4_0 && tenant_data_version < DATA_VERSION_4_3_0_0) + || tenant_data_version >= DATA_VERSION_4_3_3_0)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("tenant_data_version should be [4.2.4.0, 4.3.0.0) or [4.3.3.0, +infinity)", KR(ret), K(tenant_data_version)); + } else if (OB_FAIL(GET_MIN_DATA_VERSION(meta_tenant_id, meta_tenant_data_version))) { + LOG_WARN("fail to get the meta tenant's min data version", KR(ret), K(meta_tenant_id)); + } else if (!((meta_tenant_data_version >= MOCK_DATA_VERSION_4_2_4_0 && meta_tenant_data_version < DATA_VERSION_4_3_0_0) + || meta_tenant_data_version >= DATA_VERSION_4_3_3_0)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("meta_tenant_data_version should be [4.2.4.0, 4.3.0.0) or [4.3.3.0, +infinity)", KR(ret), K(meta_tenant_data_version)); + } + return ret; +} + +int ObServiceNameProxy::trans_start_and_precheck_( + ObMySQLTransaction &trans, + const uint64_t tenant_id, + int64_t &epoch) +{ + int ret = OB_SUCCESS; + ObAllTenantInfo tenant_info; + const uint64_t exec_tenant_id = gen_meta_tenant_id(tenant_id); + int64_t affected_rows = 0; + if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("GCTX.sql_proxy_ is null", KR(ret), KP(GCTX.sql_proxy_)); + } else if (OB_FAIL(trans.start(GCTX.sql_proxy_, exec_tenant_id))) { + LOG_WARN("fail to start trans", KR(ret)); + } else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id, &trans, true /* for_update */, tenant_info))) { + LOG_WARN("fail to load tenant info", KR(ret), K(tenant_id)); + } else if (OB_UNLIKELY(!tenant_info.is_normal_status())) { + ret = OB_OP_NOT_ALLOW; + LOG_WARN("the tenant's switchover status is not normal", KR(ret), K(tenant_info)); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "The tenant's switchover status is not normal, service name related command is"); + } else if (OB_FAIL(ObServiceEpochProxy::get_service_epoch(trans, tenant_id, ObServiceEpochProxy::SERVICE_NAME_EPOCH, epoch))) { + LOG_WARN("fail to get service epoch", KR(ret), K(tenant_id)); + } else if (FALSE_IT(epoch += 1)) { + } else if (OB_FAIL(ObServiceEpochProxy::update_service_epoch(trans, tenant_id, + ObServiceEpochProxy::SERVICE_NAME_EPOCH, epoch, affected_rows))) { + LOG_WARN("fail to get service epoch", KR(ret), K(tenant_id)); + } else if (1 != affected_rows) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("affected_rows should be one", KR(ret), K(affected_rows), K(tenant_id)); + } + return ret; +} + +void ObServiceNameProxy::write_and_end_trans_( + int &ret, + ObMySQLTransaction &trans, + const uint64_t tenant_id, + const ObSqlString &sql, + ObArray &all_service_names) +{ + if (OB_SUCC(ret)) { + int64_t affected_rows = 0; + const uint64_t exec_tenant_id = gen_meta_tenant_id(tenant_id); + if (OB_FAIL(trans.write(exec_tenant_id, sql.ptr(), affected_rows))) { + LOG_WARN("failed to execute sql", KR(ret), K(exec_tenant_id), K(sql)); + } else if (1 != affected_rows) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("affected_rows should be one", KR(ret), K(affected_rows)); + } else if (OB_FAIL(select_all_service_names_(trans, tenant_id, all_service_names))) { + LOG_WARN("fail to execute select_all_service_names_", KR(ret), K(tenant_id)); + } + } + if (OB_UNLIKELY(!trans.is_started())) { + LOG_WARN("the transaction is not started"); + } else { + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(trans.end(OB_SUCC(ret)))) { + LOG_WARN("fail to commit the transaction", KR(ret), KR(tmp_ret), K(tenant_id)); + ret = OB_SUCC(ret) ? tmp_ret : ret; + } + } +} + +int ObServiceNameProxy::build_service_name_( + const common::sqlclient::ObMySQLResult &res, + ObServiceName &service_name) +{ + int ret = OB_SUCCESS; + uint64_t tenant_id = OB_INVALID_TENANT_ID; + uint64_t service_name_id = ObServiceNameID::INVALID_SERVICE_NAME_ID; + ObString service_name_str; + ObString service_status_str; + service_name.reset(); + EXTRACT_VARCHAR_FIELD_MYSQL(res, "service_name", service_name_str); + EXTRACT_VARCHAR_FIELD_MYSQL(res, "service_status", service_status_str); + EXTRACT_INT_FIELD_MYSQL(res, "tenant_id", tenant_id, uint64_t); + EXTRACT_INT_FIELD_MYSQL(res, "service_name_id", service_name_id, uint64_t); + if (FAILEDx(service_name.init(tenant_id, service_name_id, service_name_str, service_status_str))) { + LOG_WARN("fail to init service_name", KR(ret), K(tenant_id), K(service_name_id), K(service_name_str), + K(service_status_str)); + } else if (OB_UNLIKELY(!service_name.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("build invalid service_name", KR(ret), K(service_name)); + } + return ret; +} + +int ObServiceNameProxy::get_tenant_service_name_num( + common::ObISQLClient &sql_proxy, + const uint64_t tenant_id, + int64_t &service_name_num) +{ + int ret = OB_SUCCESS; + ObSqlString sql; + service_name_num = INT64_MAX; + const uint64_t exec_tenant_id = gen_meta_tenant_id(tenant_id); + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tenant_id", KR(ret), K(tenant_id)); + } else if (OB_FAIL(sql.assign_fmt("select count(*) as count from %s where tenant_id = %lu", OB_ALL_SERVICE_TNAME, tenant_id))) { + LOG_WARN("fail to assign sql", KR(ret), K(tenant_id)); + } else { + HEAP_VAR(ObMySQLProxy::MySQLResult, res) { + common::sqlclient::ObMySQLResult *result = NULL; + if (OB_FAIL(sql_proxy.read(res, exec_tenant_id, sql.ptr()))) { + LOG_WARN("failed to read", KR(ret), K(tenant_id), K(exec_tenant_id), K(sql)); + } else if (OB_ISNULL(result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get sql result", KR(ret)); + } else if (OB_FAIL(result->next())) { + LOG_WARN("fail to get next", KR(ret), K(sql)); + } else { + EXTRACT_INT_FIELD_MYSQL(*result, "count", service_name_num, int64_t); + if (OB_FAIL(ret)) { + LOG_WARN("fail to extract count", KR(ret), K(tenant_id), K(exec_tenant_id), K(sql)); + } + } + } + } + return ret; +} +} +} \ No newline at end of file diff --git a/src/share/ob_service_name_proxy.h b/src/share/ob_service_name_proxy.h new file mode 100644 index 000000000..c068f255f --- /dev/null +++ b/src/share/ob_service_name_proxy.h @@ -0,0 +1,222 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_SHARE_OB_SERVICE_NAME_PROXY_H_ +#define OCEANBASE_SHARE_OB_SERVICE_NAME_PROXY_H_ + +#include "lib/ob_define.h" +#include "share/ob_define.h" + +namespace oceanbase +{ +namespace common +{ +class ObMySQLProxy; +class ObMySQLTransaction; +class ObISQLClient; +namespace sqlclient +{ +class ObMySQLResult; +} +} +namespace share +{ +class ObServiceNameID final +{ + OB_UNIS_VERSION(1); +public: + static const uint64_t INVALID_SERVICE_NAME_ID = 0; + static bool is_valid_service_name_id(const uint64_t id) { return INVALID_SERVICE_NAME_ID != id && OB_INVALID_ID != id; } + + explicit ObServiceNameID(const uint64_t id = INVALID_SERVICE_NAME_ID) : id_(id) {} + ObServiceNameID(const ObServiceNameID &other) : id_(other.id_) {} + ~ObServiceNameID() { reset(); } + + uint64_t id() const { return id_; } + void reset() { id_ = INVALID_SERVICE_NAME_ID; } + // assignment + ObServiceNameID &operator=(const uint64_t id) { id_ = id; return *this; } + ObServiceNameID &operator=(const ObServiceNameID &other) { id_ = other.id_; return *this; } + bool operator==(const ObServiceNameID &other) const { return id_ == other.id_; } + bool operator!=(const ObServiceNameID &other) const { return id_ != other.id_; } + + bool is_valid() const { return is_valid_service_name_id(id_); } + TO_STRING_KV(K_(id)); +private: + uint64_t id_; +}; +class ObServiceNameString final +{ + OB_UNIS_VERSION(1); +public: + ObServiceNameString() : str_() {} + ~ObServiceNameString() {} + int init(const ObString &str); + bool equal_to(const ObServiceNameString &service_name_string) const; + bool is_valid() const { return OB_SUCCESS == check_service_name(str_.str()); } + int assign(const ObServiceNameString &other); + bool is_empty() const { return str_.is_empty(); } + void reset() { return str_.reset(); } + static int check_service_name(const ObString &service_name_str); + const char *ptr() const { return str_.ptr(); } + TO_STRING_KV(K_(str)); +private: + ObFixedLengthString str_; +}; +struct ObServiceNameArg +{ +public: + enum ObServiceOp { + INVALID_SERVICE_OP = 0, + CREATE_SERVICE, + DELETE_SERVICE, + START_SERVICE, + STOP_SERVICE, + MAX_SERVICE_OP + }; + static const char *service_op_to_str(const ObServiceOp &service_op); + ObServiceNameArg() + : op_(INVALID_SERVICE_OP), + target_tenant_id_(OB_INVALID_TENANT_ID), + service_name_str_() {}; + ~ObServiceNameArg() {}; + int init(const ObServiceOp op, const uint64_t target_tenant_id, const ObString &service_name_str); + bool is_valid() const; + static bool is_valid_service_op(ObServiceOp op); + int assign(const ObServiceNameArg &other); + void reset(); + bool is_create_service() const {return CREATE_SERVICE == op_; } + bool is_delete_service() const {return DELETE_SERVICE == op_; } + bool is_start_service() const {return START_SERVICE == op_; } + bool is_stop_service() const {return STOP_SERVICE == op_; } + const share::ObServiceNameString& get_service_name_str() const { return service_name_str_; } + uint64_t get_target_tenant_id() const { return target_tenant_id_; } + const ObServiceOp &get_service_op() const { return op_; } + TO_STRING_KV(K_(op), "service_op_to_str", service_op_to_str(op_), + K_(target_tenant_id), K_(service_name_str)); +private: + ObServiceOp op_; + uint64_t target_tenant_id_; + share::ObServiceNameString service_name_str_; +}; +struct ObServiceName +{ + OB_UNIS_VERSION(1); +public: + enum ObServiceStatus + { + INVALID_SERVICE_STATUS = 0, + STARTED, + STOPPING, + STOPPED, + MAX_SERVICE_STATUS + }; + static const char *service_status_to_str(const ObServiceStatus &service_status); + static ObServiceStatus str_to_service_status(const ObString &service_status_str); + static bool is_valid_service_status(const ObServiceStatus &service_status); + + ObServiceName() + : tenant_id_(OB_INVALID_TENANT_ID), + service_name_id_(), + service_name_str_(), + service_status_(INVALID_SERVICE_STATUS) {} + ~ObServiceName() {} + int init( + const uint64_t tenant_id, + const uint64_t service_name_id, + const ObString &service_name_str, + const ObString &service_status); + bool is_valid() const; + int assign(const ObServiceName &other); + void reset(); + bool is_started() const { return STARTED == service_status_; } + bool is_stopping() const { return STOPPING == service_status_; } + bool is_stopped() const {return STOPPED == service_status_; } + uint64_t get_tenant_id() const { return tenant_id_; } + const ObServiceNameID &get_service_name_id() const { return service_name_id_; } + const ObServiceNameString &get_service_name_str() const { return service_name_str_; } + const ObServiceStatus &get_service_status() const { return service_status_; } + TO_STRING_KV(K_(tenant_id), "service_name_id", service_name_id_.id(), K_(service_name_str), K_(service_status), + "service_status_str", service_status_to_str(service_status_)); +private: + uint64_t tenant_id_; + ObServiceNameID service_name_id_; + ObServiceNameString service_name_str_; + ObServiceStatus service_status_; +}; + +class ObServiceNameProxy +{ +public: + static constexpr int64_t SERVICE_NAME_MAX_NUM = 1; + static int check_is_service_name_enabled(const uint64_t tenant_id); + static int select_all_service_names_with_epoch( + const int64_t tenant_id, + int64_t &epoch, + ObIArray &all_service_names); + static int select_service_name( + common::ObISQLClient &sql_proxy, + const uint64_t tenant_id, + const ObServiceNameString &service_name_str, + ObServiceName &service_name); + static int insert_service_name( + const uint64_t tenant_id, + const ObServiceNameString &service_name_str, + int64_t &epoch, + ObArray &all_service_names); + static int update_service_status( + const ObServiceName &service_name, + const ObServiceName::ObServiceStatus &new_status, + int64_t &epoch, + ObArray &all_service_names); + static int delete_service_name(const ObServiceName &service_name); + static int get_tenant_service_name_num( + common::ObISQLClient &sql_proxy, + const uint64_t tenant_id, + int64_t &service_name_num); + + ObServiceNameProxy() {} + virtual ~ObServiceNameProxy() {} +private: + static int select_all_service_names_( + common::ObISQLClient &sql_proxy, + const int64_t tenant_id, + ObIArray &all_service_names); + static int select_service_name_sql_helper_( + common::ObISQLClient &sql_proxy, + const int64_t tenant_id, + const bool extract_epoch, + ObSqlString &sql, + int64_t &epoch, + ObIArray &all_service_names); + static int trans_start_and_precheck_( + ObMySQLTransaction &trans, + const uint64_t tenant_id, + int64_t &epoch); + static void write_and_end_trans_( + int &ret, + ObMySQLTransaction &trans, + const uint64_t tenant_id, + const ObSqlString &sql, + ObArray &all_service_names); + static int build_service_name_( + const common::sqlclient::ObMySQLResult &res, + ObServiceName &service_name); + static int get_tenant_service_name_num_( + common::ObISQLClient &sql_proxy, + const uint64_t tenant_id, + int64_t &service_name_num); + DISALLOW_COPY_AND_ASSIGN(ObServiceNameProxy); +}; +} // end namespace share +} // end namespace oceanbase +#endif // OCEANBASE_SHARE_OB_SERVICE_NAME_PROXY_H_ \ No newline at end of file diff --git a/src/share/ob_share_util.cpp b/src/share/ob_share_util.cpp index 42e1c387a..2ba4ba0f2 100644 --- a/src/share/ob_share_util.cpp +++ b/src/share/ob_share_util.cpp @@ -528,6 +528,108 @@ int ObShareUtil::check_compat_version_for_clone_tenant_with_tenant_role( return ret; } +int ObShareUtil::mtl_get_tenant_role(const uint64_t tenant_id, ObTenantRole::Role &tenant_role) +{ + int ret = OB_SUCCESS; + tenant_role = ObTenantRole::INVALID_TENANT; + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tenant_id", KR(ret), K(tenant_id)); + } else if (is_sys_tenant(tenant_id) || is_meta_tenant(tenant_id)) { + tenant_role = ObTenantRole::PRIMARY_TENANT; + } else { + MTL_SWITCH(tenant_id) { + tenant_role = MTL_GET_TENANT_ROLE_CACHE(); + } + } + if (OB_SUCC(ret) && OB_UNLIKELY(is_invalid_tenant(tenant_role))) { + ret = OB_NEED_WAIT; + LOG_WARN("tenant role is not ready, need wait", KR(ret), K(tenant_id), K(tenant_role)); + } + return ret; +} + +int ObShareUtil::mtl_check_if_tenant_role_is_primary(const uint64_t tenant_id, bool &is_primary) +{ + int ret = OB_SUCCESS; + is_primary = false; + ObTenantRole::Role tenant_role; + if (OB_FAIL(mtl_get_tenant_role(tenant_id, tenant_role))) { + LOG_WARN("fail to execute mtl_get_tenant_role", KR(ret), K(tenant_id)); + } else if (is_primary_tenant(tenant_role)) { + is_primary = true; + } + return ret; +} + +int ObShareUtil::mtl_check_if_tenant_role_is_standby(const uint64_t tenant_id, bool &is_standby) +{ + int ret = OB_SUCCESS; + is_standby = false; + ObTenantRole::Role tenant_role; + if (OB_FAIL(mtl_get_tenant_role(tenant_id, tenant_role))) { + LOG_WARN("fail to execute mtl_get_tenant_role", KR(ret), K(tenant_id)); + } else if (is_standby_tenant(tenant_role)) { + is_standby = true; + } + return ret; +} +int ObShareUtil::table_get_tenant_role(const uint64_t tenant_id, ObTenantRole &tenant_role) +{ + int ret = OB_SUCCESS; + tenant_role.reset(); + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tenant_id", KR(ret), K(tenant_id)); + } else if (is_sys_tenant(tenant_id) || is_meta_tenant(tenant_id)) { + tenant_role = ObTenantRole::PRIMARY_TENANT; + } else if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("GCTX.sql_proxy_ is null", KR(ret), KP(GCTX.sql_proxy_)); + } else if (OB_FAIL(ObAllTenantInfoProxy::get_tenant_role(GCTX.sql_proxy_, tenant_id, tenant_role))) { + LOG_WARN("fail to get tenant role", KR(ret), KP(GCTX.sql_proxy_), K(tenant_id)); + } else if (tenant_role.is_invalid()) { + ret = OB_NEED_WAIT; + LOG_WARN("tenant role is not ready, need wait", KR(ret), K(tenant_role)); + } + return ret; +} +int ObShareUtil::table_check_if_tenant_role_is_primary(const uint64_t tenant_id, bool &is_primary) +{ + int ret = OB_SUCCESS; + share::ObTenantRole tenant_role; + is_primary = false; + if (OB_FAIL(table_get_tenant_role(tenant_id, tenant_role))) { + LOG_WARN("fail to execute table_get_tenant_role", KR(ret), K(tenant_id)); + } else if (tenant_role.is_primary()) { + is_primary = true; + } + return ret; +} +int ObShareUtil::table_check_if_tenant_role_is_standby(const uint64_t tenant_id, bool &is_standby) +{ + int ret = OB_SUCCESS; + share::ObTenantRole tenant_role; + is_standby = false; + if (OB_FAIL(table_get_tenant_role(tenant_id, tenant_role))) { + LOG_WARN("fail to execute table_get_tenant_role", KR(ret), K(tenant_id)); + } else if (tenant_role.is_standby()) { + is_standby = true; + } + return ret; +} +int ObShareUtil::table_check_if_tenant_role_is_restore(const uint64_t tenant_id, bool &is_restore) +{ + int ret = OB_SUCCESS; + share::ObTenantRole tenant_role; + is_restore = false; + if (OB_FAIL(table_get_tenant_role(tenant_id, tenant_role))) { + LOG_WARN("fail to execute table_get_tenant_role", KR(ret), K(tenant_id)); + } else if (tenant_role.is_restore()) { + is_restore = true; + } + return ret; +} const char *ObShareUtil::replica_type_to_string(const ObReplicaType type) { const char *str = NULL; diff --git a/src/share/ob_share_util.h b/src/share/ob_share_util.h index 9f3ac170c..893d8fd6a 100644 --- a/src/share/ob_share_util.h +++ b/src/share/ob_share_util.h @@ -14,6 +14,7 @@ #define OCEANBASE_SHARE_OB_SHARE_UTIL_H_ #include "share/ob_define.h" #include "share/scn.h" +#include "share/ob_tenant_role.h" namespace oceanbase { namespace common @@ -147,6 +148,13 @@ public: SCN &ora_rowscn); static bool is_tenant_enable_rebalance(const uint64_t tenant_id); static bool is_tenant_enable_transfer(const uint64_t tenant_id); + static int mtl_get_tenant_role(const uint64_t tenant_id, ObTenantRole::Role &tenant_role); + static int mtl_check_if_tenant_role_is_primary(const uint64_t tenant_id, bool &is_primary); + static int mtl_check_if_tenant_role_is_standby(const uint64_t tenant_id, bool &is_standby); + static int table_get_tenant_role(const uint64_t tenant_id, ObTenantRole &tenant_role); + static int table_check_if_tenant_role_is_primary(const uint64_t tenant_id, bool &is_primary); + static int table_check_if_tenant_role_is_standby(const uint64_t tenant_id, bool &is_standby); + static int table_check_if_tenant_role_is_restore(const uint64_t tenant_id, bool &is_restore); static const char *replica_type_to_string(const ObReplicaType type); static ObReplicaType string_to_replica_type(const char *str); static ObReplicaType string_to_replica_type(const ObString &str); diff --git a/src/share/ob_srv_rpc_proxy.h b/src/share/ob_srv_rpc_proxy.h index 6880a2c86..006a98f10 100644 --- a/src/share/ob_srv_rpc_proxy.h +++ b/src/share/ob_srv_rpc_proxy.h @@ -265,6 +265,7 @@ public: RPC_S(PR5 cancel_gather_stats, OB_CANCEL_GATHER_STATS, (ObCancelGatherStatsArg)); RPC_S(PR5 force_set_tenant_log_disk, OB_LOG_FORCE_SET_TENANT_LOG_DISK, (obrpc::ObForceSetTenantLogDiskArg)); RPC_S(PR5 dump_server_usage, OB_FORCE_DUMP_SERVER_USAGE, (obrpc::ObDumpServerUsageRequest), obrpc::ObDumpServerUsageResult); + RPC_AP(PR5 refresh_service_name, OB_REFRESH_SERVICE_NAME, (obrpc::ObRefreshServiceNameArg), obrpc::ObRefreshServiceNameRes); RPC_AP(PR5 get_tenant_logical_resource, OB_CAL_STANDBY_TENANT_PHY_RESOURCE, (obrpc::ObGetTenantResArg), obrpc::ObTenantLogicalRes); RPC_S(PR5 phy_res_calculate_by_unit, OB_CAL_UNIT_PHY_RESOURCE, (obrpc::Int64), share::ObMinPhyResourceResult); RPC_S(PR5 rpc_reverse_keepalive, OB_RPC_REVERSE_KEEPALIVE, (obrpc::ObRpcReverseKeepaliveArg), obrpc::ObRpcReverseKeepaliveResp); diff --git a/src/share/ob_tenant_info_proxy.cpp b/src/share/ob_tenant_info_proxy.cpp index dce276cc9..e6eeea6b4 100755 --- a/src/share/ob_tenant_info_proxy.cpp +++ b/src/share/ob_tenant_info_proxy.cpp @@ -60,9 +60,9 @@ SCN gen_new_replayable_scn(const SCN &cur_replayable_scn, const SCN &desired_rep return MIN(MAX(cur_replayable_scn, desired_replayable_scn), new_sync_scn); } -SCN gen_new_standby_scn(const SCN &cur_standby_scn, const SCN &desired_standby_scn, const SCN &new_replayable_scn) +SCN gen_new_readable_scn(const SCN &cur_readable_scn, const SCN &desired_readable_scn, const SCN &new_replayable_scn) { - return MIN(MAX(cur_standby_scn, desired_standby_scn), new_replayable_scn); + return MIN(MAX(cur_readable_scn, desired_readable_scn), new_replayable_scn); } ////////////ObAllTenantInfo DEFINE_TO_YSON_KV(ObAllTenantInfo, @@ -70,7 +70,7 @@ DEFINE_TO_YSON_KV(ObAllTenantInfo, OB_ID(switchover_epoch), switchover_epoch_, OB_ID(sync_scn), sync_scn_, OB_ID(replayable_scn), replayable_scn_, - OB_ID(standby_scn), standby_scn_, + OB_ID(standby_scn), readable_scn_, OB_ID(recovery_until_scn), recovery_until_scn_, OB_ID(tenant_role), tenant_role_, OB_ID(switchover_status), switchover_status_); @@ -78,16 +78,16 @@ DEFINE_TO_YSON_KV(ObAllTenantInfo, bool ObAllTenantInfo::is_valid() const { return OB_INVALID_TENANT_ID != tenant_id_ - && 0 <= switchover_epoch_ - && sync_scn_.is_valid_and_not_min() - && replayable_scn_.is_valid_and_not_min() - && standby_scn_.is_valid_and_not_min() - && recovery_until_scn_.is_valid_and_not_min() - && tenant_role_.is_valid() - && switchover_status_.is_valid() - && log_mode_.is_valid() - && is_valid_tenant_scn(sync_scn_, replayable_scn_, standby_scn_, recovery_until_scn_) - && restore_data_mode_.is_valid(); + && 0 <= switchover_epoch_ + && sync_scn_.is_valid_and_not_min() + && replayable_scn_.is_valid_and_not_min() + && readable_scn_.is_valid_and_not_min() + && recovery_until_scn_.is_valid_and_not_min() + && tenant_role_.is_valid() + && switchover_status_.is_valid() + && log_mode_.is_valid() + && is_valid_tenant_scn(sync_scn_, replayable_scn_, readable_scn_, recovery_until_scn_) + && restore_data_mode_.is_valid(); } int ObAllTenantInfo::init( @@ -97,7 +97,7 @@ int ObAllTenantInfo::init( int64_t switchover_epoch, const SCN &sync_scn, const SCN &replayable_scn, - const SCN &standby_scn, + const SCN &readable_scn, const SCN &recovery_until_scn, const ObArchiveMode &log_mode, const share::ObLSID &max_ls_id, @@ -110,15 +110,15 @@ int ObAllTenantInfo::init( || 0 > switchover_epoch || !sync_scn.is_valid_and_not_min() || !replayable_scn.is_valid_and_not_min() - || !standby_scn.is_valid_and_not_min() + || !readable_scn.is_valid_and_not_min() || !recovery_until_scn.is_valid_and_not_min() || !log_mode.is_valid() - || !is_valid_tenant_scn(sync_scn, replayable_scn, standby_scn, recovery_until_scn) + || !is_valid_tenant_scn(sync_scn, replayable_scn, readable_scn, recovery_until_scn) || !max_ls_id.is_valid() || !restore_data_mode.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(tenant_role), K(switchover_status), - K(switchover_epoch), K(sync_scn), K(replayable_scn), K(standby_scn), K(recovery_until_scn), + K(switchover_epoch), K(sync_scn), K(replayable_scn), K(readable_scn), K(recovery_until_scn), K(log_mode), K(max_ls_id), K(restore_data_mode)); } else { tenant_id_ = tenant_id; @@ -127,7 +127,7 @@ int ObAllTenantInfo::init( switchover_epoch_ = switchover_epoch; sync_scn_ = sync_scn; replayable_scn_ = replayable_scn; - standby_scn_ = standby_scn; + readable_scn_ = readable_scn; recovery_until_scn_ = recovery_until_scn; log_mode_ = log_mode; max_ls_id_ = max_ls_id; @@ -146,7 +146,7 @@ void ObAllTenantInfo::assign(const ObAllTenantInfo &other) switchover_epoch_ = other.switchover_epoch_; sync_scn_ = other.sync_scn_; replayable_scn_ = other.replayable_scn_; - standby_scn_ = other.standby_scn_; + readable_scn_ = other.readable_scn_; recovery_until_scn_ = other.recovery_until_scn_; log_mode_ = other.log_mode_; max_ls_id_ = other.max_ls_id_; @@ -163,7 +163,7 @@ void ObAllTenantInfo::reset() switchover_epoch_ = OB_INVALID_VERSION; sync_scn_.set_min(); replayable_scn_.set_min(); - standby_scn_.set_min() ; + readable_scn_.set_min() ; recovery_until_scn_.set_min(); log_mode_.reset(); max_ls_id_.reset(); @@ -177,7 +177,7 @@ void ObAllTenantInfo::reset() OB_SERIALIZE_MEMBER(ObAllTenantInfo, tenant_id_, tenant_role_, switchover_status_, switchover_epoch_, sync_scn_, replayable_scn_, - standby_scn_, // FARM COMPAT WHITELIST + readable_scn_, // FARM COMPAT WHITELIST recovery_until_scn_, log_mode_, max_ls_id_, restore_data_mode_); @@ -231,7 +231,7 @@ int ObAllTenantInfoProxy::init_tenant_info( tenant_info.get_switchover_epoch(), tenant_info.get_sync_scn().get_val_for_inner_table_field(), tenant_info.get_replayable_scn().get_val_for_inner_table_field(), - tenant_info.get_standby_scn().get_val_for_inner_table_field(), + tenant_info.get_readable_scn().get_val_for_inner_table_field(), tenant_info.get_recovery_until_scn().get_val_for_inner_table_field(), tenant_info.get_log_mode().to_str(), tenant_info.get_max_ls_id().id()))) { @@ -475,7 +475,7 @@ int ObAllTenantInfoProxy::update_tenant_recovery_status_in_trans( } else { SCN new_sync_scn = gen_new_sync_scn(old_tenant_info.get_sync_scn(), sync_scn, old_tenant_info.get_recovery_until_scn()); SCN new_replayable_scn = gen_new_replayable_scn(old_tenant_info.get_replayable_scn(), replay_scn, new_sync_scn); - SCN new_readable_scn = gen_new_standby_scn(old_tenant_info.get_standby_scn(), readable_scn, new_replayable_scn); + SCN new_readable_scn = gen_new_readable_scn(old_tenant_info.get_readable_scn(), readable_scn, new_replayable_scn); omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id)); if (OB_UNLIKELY(!tenant_config.is_valid())) { @@ -495,14 +495,14 @@ int ObAllTenantInfoProxy::update_tenant_recovery_status_in_trans( && new_readable_scn_plus_gap.is_valid() && new_replayable_scn > new_readable_scn_plus_gap && new_readable_scn_plus_gap >= old_tenant_info.get_replayable_scn() - && old_tenant_info.get_standby_scn() > SCN::base_scn()) { + && old_tenant_info.get_readable_scn() > SCN::base_scn()) { // condition: !old_tenant_info.get_max_ls_id().is_sys_ls() // If max_ls_id is sys ls, this logic is not needed. // The goal of this logic is to minimize the difference of readable_scn among multiple ls - // condition: old_tenant_info.get_standby_scn() > SCN::base_scn() + // condition: old_tenant_info.get_readable_scn() > SCN::base_scn() // This condition is for restore tenant - // sys ls's readable_scn/standby_scn starts from base_scn + // sys ls's readable_scn starts from base_scn // replayable_scn cannot start from base_scn, it's too slow when we restore tenant // At the beginning time, replayable_scn should be sync_scn new_replayable_scn = new_readable_scn_plus_gap; @@ -511,7 +511,7 @@ int ObAllTenantInfoProxy::update_tenant_recovery_status_in_trans( if (old_tenant_info.get_sync_scn() == new_sync_scn && old_tenant_info.get_replayable_scn() == new_replayable_scn - && old_tenant_info.get_standby_scn() == new_readable_scn) { + && old_tenant_info.get_readable_scn() == new_readable_scn) { LOG_DEBUG("no need update", K(old_tenant_info), K(new_sync_scn), K(new_replayable_scn), K(new_readable_scn)); } else if (OB_FAIL(sql.assign_fmt( "update %s set sync_scn = %ld, replayable_scn = %ld, " @@ -746,7 +746,6 @@ int ObAllTenantInfoProxy::update_tenant_role_in_trans( int64_t cost = ObTimeUtility::current_time() - begin_time; ROOTSERVICE_EVENT_ADD("tenant_info", "update_tenant_role", K(ret), K(tenant_id), K(new_status), K(new_switchover_ts), K(old_status), K(cost)); - return ret; } diff --git a/src/share/ob_tenant_info_proxy.h b/src/share/ob_tenant_info_proxy.h index fa841287e..d0f60658b 100755 --- a/src/share/ob_tenant_info_proxy.h +++ b/src/share/ob_tenant_info_proxy.h @@ -44,7 +44,7 @@ namespace share bool is_valid_tenant_scn( const share::SCN &sync_scn, const share::SCN &replayable_scn, - const share::SCN &standby_scn, + const share::SCN &readable_scn, const share::SCN &recovery_until_scn); SCN gen_new_sync_scn(const share::SCN &cur_sync_scn, const share::SCN &desired_sync_scn, const share::SCN &cur_recovery_until_scn); @@ -63,7 +63,7 @@ public: * @param[in] switchover_epoch * @param[in] sync_scn * @param[in] replayable_scn - * @param[in] standby_scn + * @param[in] readable_scn * @param[in] recovery_until_scn * @param[in] log_mode * @param[in] restore_data_mode @@ -74,7 +74,7 @@ public: const int64_t switchover_epoch = 0, const SCN &sync_scn = SCN::base_scn(), const SCN &replayable_scn = SCN::base_scn(), - const SCN &standby_scn = SCN::base_scn(), + const SCN &readable_scn = SCN::base_scn(), const SCN &recovery_until_scn = SCN::base_scn(), const ObArchiveMode &log_mode = NOARCHIVE_MODE, const share::ObLSID &max_ls_id = share::SYS_LS, @@ -116,9 +116,13 @@ IS_TENANT_STATUS(prepare_switching_to_standby) IS_TENANT_STATUS(prepare_flashback_for_switch_to_primary) #undef IS_TENANT_STATUS - TO_STRING_KV(K_(tenant_id), K_(tenant_role), K_(switchover_status), - K_(switchover_epoch), K_(sync_scn), K_(replayable_scn), - K_(standby_scn), K_(recovery_until_scn), K_(log_mode), K_(max_ls_id), K_(restore_data_mode)); + TO_STRING_KV(K_(tenant_id), "tenant_role", tenant_role_.to_str(), + "switchover_status", switchover_status_.to_str(), + K_(switchover_epoch), "sync_scn", sync_scn_.get_val_for_inner_table_field(), + "replayable_scn", replayable_scn_.get_val_for_inner_table_field(), + "readable_scn", readable_scn_.get_val_for_inner_table_field(), + "recovery_until_scn", recovery_until_scn_.get_val_for_inner_table_field(), + "log_mode", log_mode_.to_str(), "max_ls_id", max_ls_id_.id(), K_(restore_data_mode)); DECLARE_TO_YSON_KV; // Getter&Setter @@ -140,7 +144,7 @@ public:\ Property_declare_var(int64_t, switchover_epoch) Property_declare_var(share::SCN, sync_scn) Property_declare_var(share::SCN, replayable_scn) - Property_declare_var(share::SCN, standby_scn) + Property_declare_var(share::SCN, readable_scn) Property_declare_var(share::SCN, recovery_until_scn) Property_declare_var(ObArchiveMode, log_mode) Property_declare_var(share::ObLSID, max_ls_id) diff --git a/src/share/ob_tenant_role.h b/src/share/ob_tenant_role.h index 693cc87fd..1d148dfd6 100644 --- a/src/share/ob_tenant_role.h +++ b/src/share/ob_tenant_role.h @@ -55,12 +55,13 @@ public: bool operator != (const ObTenantRole &other) const { return value_ != other.value_; } // ObTenantRole attribute interface + bool is_invalid() const { return INVALID_TENANT == value_; } bool is_primary() const { return PRIMARY_TENANT == value_; } bool is_standby() const { return STANDBY_TENANT == value_; } bool is_restore() const { return RESTORE_TENANT == value_; } bool is_clone() const { return CLONE_TENANT == value_; } - TO_STRING_KV(K_(value)); + TO_STRING_KV("tenant_role", to_str(), K_(value)); DECLARE_TO_YSON_KV; private: Role value_; diff --git a/src/share/ob_tenant_switchover_status.h b/src/share/ob_tenant_switchover_status.h index cd6f6620c..13c0084f8 100644 --- a/src/share/ob_tenant_switchover_status.h +++ b/src/share/ob_tenant_switchover_status.h @@ -74,7 +74,7 @@ IS_TENANT_STATUS(SWITCHING_TO_STANDBY_STATUS, switching_to_standby) IS_TENANT_STATUS(PREPARE_FLASHBACK_FOR_SWITCH_TO_PRIMARY_STATUS, prepare_flashback_for_switch_to_primary) #undef IS_TENANT_STATUS - TO_STRING_KV(K_(value)); + TO_STRING_KV("switchover_status", to_str(), K_(value)); DECLARE_TO_YSON_KV; private: ObTenantSwitchoverStatus::Status value_; diff --git a/src/share/ob_upgrade_utils.cpp b/src/share/ob_upgrade_utils.cpp index 5daa37d98..9a1ec2d55 100755 --- a/src/share/ob_upgrade_utils.cpp +++ b/src/share/ob_upgrade_utils.cpp @@ -812,10 +812,6 @@ int ObBaseUpgradeProcessor::check_inner_stat() const ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid processor status", KR(ret), K_(data_version), K_(tenant_id), K_(mode)); - } else if (GCTX.is_standby_cluster() && OB_SYS_TENANT_ID != tenant_id_) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("run upgrade job for non-sys tenant in standby cluster is not supported", - KR(ret), K_(tenant_id)); } else if (OB_ISNULL(check_stop_provider_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("check_stop_provider is null", KR(ret)); @@ -1462,6 +1458,8 @@ int ObUpgradeFor4330Processor::post_upgrade() LOG_WARN("fail to check inner stat", KR(ret)); } else if (OB_FAIL(post_upgrade_for_external_table_flag())) { LOG_WARN("fail to alter log external table flag", KR(ret)); + } else if (OB_FAIL(post_upgrade_for_service_name())) { + LOG_WARN("post upgrade for service name failed", KR(ret)); } else if (OB_FAIL(post_upgrade_for_optimizer_stats())) { LOG_WARN("fail to upgrade optimizer stats", KR(ret)); } @@ -1482,6 +1480,28 @@ int ObUpgradeFor4330Processor::post_upgrade_for_external_table_flag() return ret; } +int ObUpgradeFor4330Processor::post_upgrade_for_service_name() +{ + int ret = OB_SUCCESS; + int64_t affected_rows = 0; + if (OB_ISNULL(sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("error unexpected", KR(ret), KP(sql_proxy_)); + } else if (!is_meta_tenant(tenant_id_)) { + LOG_INFO("not meta tenant, skip", K(tenant_id_)); + } else { + ObSqlString sql; + uint64_t user_tenant_id = gen_user_tenant_id(tenant_id_); + if (OB_FAIL(sql.assign_fmt("INSERT IGNORE INTO %s (tenant_id, name, value) VALUES (%lu, '%s', 0)", + OB_ALL_SERVICE_EPOCH_TNAME, user_tenant_id, ObServiceEpochProxy::SERVICE_NAME_EPOCH))) { + LOG_WARN("fail to assign sql assign", KR(ret)); + } else if (OB_FAIL(sql_proxy_->write(tenant_id_, sql.ptr(), affected_rows))) { + LOG_WARN("fail to execute sql", KR(ret), K(sql)); + } else {} + } + FLOG_INFO("insert service name epoch", KR(ret), K(tenant_id_), K(affected_rows)); + return ret; +} int ObUpgradeFor4330Processor::post_upgrade_for_optimizer_stats() { int ret = OB_SUCCESS; @@ -1519,6 +1539,5 @@ int ObUpgradeFor4330Processor::post_upgrade_for_optimizer_stats() /* =========== 4330 upgrade processor end ============= */ -/* =========== special upgrade processor end ============= */ } // end share } // end oceanbase diff --git a/src/share/ob_upgrade_utils.h b/src/share/ob_upgrade_utils.h index 2a2fcd52a..35f6c767c 100755 --- a/src/share/ob_upgrade_utils.h +++ b/src/share/ob_upgrade_utils.h @@ -270,6 +270,7 @@ public: virtual int post_upgrade() override; private: int post_upgrade_for_external_table_flag(); + int post_upgrade_for_service_name(); int post_upgrade_for_optimizer_stats(); }; /* =========== special upgrade processor end ============= */ diff --git a/src/share/schema/ob_dependency_info.cpp b/src/share/schema/ob_dependency_info.cpp index 7c3f1e529..10ce4ff2d 100644 --- a/src/share/schema/ob_dependency_info.cpp +++ b/src/share/schema/ob_dependency_info.cpp @@ -1287,7 +1287,6 @@ int ObReferenceObjTable::fill_rowkey_pairs( int ObReferenceObjTable::batch_execute_insert_or_update_obj_dependency( const uint64_t tenant_id, - const bool is_standby, const int64_t new_schema_version, const ObReferenceObjTable::DependencyObjKeyItemPairs &dep_objs, ObMySQLTransaction &trans, @@ -1299,8 +1298,6 @@ int ObReferenceObjTable::batch_execute_insert_or_update_obj_dependency( if (OB_INVALID_ID == tenant_id) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid argument", K(ret), K(tenant_id)); - } else if (is_standby) { - // do nothing } else { ObSqlString sql; ObDMLSqlSplicer dml; @@ -1345,7 +1342,6 @@ int ObReferenceObjTable::batch_execute_insert_or_update_obj_dependency( int ObReferenceObjTable::batch_execute_delete_obj_dependency( const uint64_t tenant_id, - const bool is_standby, const ObReferenceObjTable::DependencyObjKeyItemPairs &dep_objs, ObMySQLTransaction &trans) { @@ -1354,8 +1350,6 @@ int ObReferenceObjTable::batch_execute_delete_obj_dependency( if (OB_INVALID_ID == tenant_id) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid argument", K(ret), K(tenant_id)); - } else if (is_standby) { - // do nothing } else { share::ObDMLSqlSplicer dml; ObSqlString sql; @@ -1510,7 +1504,11 @@ int ObReferenceObjTable::process_reference_obj_table(const uint64_t tenant_id, sql::ObMaintainDepInfoTaskQueue &task_queue) { int ret = OB_SUCCESS; - if (!is_inited() || GCTX.is_standby_cluster()) { + share::ObTenantRole::Role tenant_role; + bool is_standby = false; + if (OB_FAIL(ObShareUtil::mtl_check_if_tenant_role_is_standby(tenant_id, is_standby))) { + LOG_WARN("fail to execute mtl_check_if_tenant_role_is_standby", KR(ret), K(tenant_id)); + } else if (OB_UNLIKELY(!is_inited() || is_standby)) { if (OB_INVALID_ID != dep_obj_id) { OZ (task_queue.erase_view_id_from_set(dep_obj_id)); } diff --git a/src/share/schema/ob_dependency_info.h b/src/share/schema/ob_dependency_info.h index 348ac73e3..56bff18a1 100644 --- a/src/share/schema/ob_dependency_info.h +++ b/src/share/schema/ob_dependency_info.h @@ -467,7 +467,6 @@ public: inline const RefObjVersionMap &get_ref_obj_table() const { return ref_obj_version_table_; } static int batch_execute_insert_or_update_obj_dependency( const uint64_t tenant_id, - const bool is_standby, const int64_t new_schema_version, const ObReferenceObjTable::DependencyObjKeyItemPairs &dep_objs, ObMySQLTransaction &trans, @@ -475,7 +474,6 @@ public: rootserver::ObDDLOperator &ddl_operator); static int batch_execute_delete_obj_dependency( const uint64_t tenant_id, - const bool is_standby, const ObReferenceObjTable::DependencyObjKeyItemPairs &dep_objs, ObMySQLTransaction &trans); static int update_max_dependency_version( diff --git a/src/share/schema/ob_multi_version_schema_service.cpp b/src/share/schema/ob_multi_version_schema_service.cpp index ec18ed586..1e6a387f6 100644 --- a/src/share/schema/ob_multi_version_schema_service.cpp +++ b/src/share/schema/ob_multi_version_schema_service.cpp @@ -968,7 +968,7 @@ int ObMultiVersionSchemaService::get_cluster_schema_guard( // new schema refresh if (OB_FAIL(guard.fast_reset())) { LOG_WARN("fail to reset guard", K(ret)); - } else if (OB_FAIL(guard.init(GCTX.is_standby_cluster()))) { + } else if (OB_FAIL(guard.init())) { LOG_WARN("fail to init guard", K(ret)); } else { ObSEArray tenant_ids; @@ -1036,7 +1036,7 @@ int ObMultiVersionSchemaService::get_cluster_schema_guard( } else { // switchover/failover not clear schema_status, Cannot trust schema_status content unconditionally // bugfix: - if (guard.is_standby_cluster() || (*tenant)->is_restore()) { + if ((*tenant)->is_restore()) { if (OB_FAIL(get_schema_status(schema_status_array, tenant_id, schema_status))) { LOG_WARN("fail to get schema status", K(ret), KPC(*tenant)); } @@ -1093,7 +1093,7 @@ int ObMultiVersionSchemaService::get_tenant_schema_guard( LOG_WARN("invalid tenant_id", K(ret), K(tenant_id)); } else if (OB_FAIL(guard.fast_reset())) { LOG_WARN("fail to reset schema guard", K(ret)); - } else if (OB_FAIL(guard.init(GCTX.is_standby_cluster()))) { + } else if (OB_FAIL(guard.init())) { LOG_WARN("fail to init guard", K(ret)); } sys_schema_status.tenant_id_ = OB_SYS_TENANT_ID; @@ -1150,8 +1150,7 @@ int ObMultiVersionSchemaService::get_tenant_schema_guard( // Avoid circular dependencies } else if (ObSchemaService::g_liboblog_mode_) { tenant_schema_status.tenant_id_ = tenant_id; - } else if (!guard.is_standby_cluster() - && OB_FAIL(check_tenant_is_restore(&guard, tenant_id, guard.restore_tenant_exist_))) { + } else if (OB_FAIL(check_tenant_is_restore(&guard, tenant_id, guard.restore_tenant_exist_))) { LOG_WARN("fail to check restore tenant exist", K(ret), K(tenant_id)); } else if (guard.use_schema_status()) { ObSchemaStatusProxy *schema_status_proxy = GCTX.schema_status_proxy_; @@ -1482,10 +1481,9 @@ int ObMultiVersionSchemaService::retry_get_schema_guard( || is_meta_tenant(tenant_id) || ObSchemaService::g_liboblog_mode_) { // skip - } else if (!schema_guard.is_standby_cluster() - && OB_FAIL(check_tenant_is_restore(&schema_guard, tenant_id, is_restore))) { + } else if (OB_FAIL(check_tenant_is_restore(&schema_guard, tenant_id, is_restore))) { LOG_WARN("fail to check restore tenant exist", K(ret), K(tenant_id)); - } else if (schema_guard.is_standby_cluster() || is_restore) { + } else if (is_restore) { ObSchemaStatusProxy *schema_status_proxy = GCTX.schema_status_proxy_; if (OB_ISNULL(schema_status_proxy)) { ret = OB_ERR_UNEXPECTED; @@ -2392,7 +2390,6 @@ int ObMultiVersionSchemaService::refresh_and_add_schema(const ObIArray FLOG_INFO("[REFRESH_SCHEMA] start to refresh and add schema", K(tenant_ids)); const int64_t start = ObTimeUtility::current_time(); int ret = OB_SUCCESS; - bool is_standby_cluster = GCTX.is_standby_cluster(); if (!check_inner_stat()) { ret = OB_INNER_STAT_ERROR; LOG_WARN("inner stat error", K(ret)); @@ -2410,7 +2407,7 @@ int ObMultiVersionSchemaService::refresh_and_add_schema(const ObIArray LOG_WARN("fail to check restore tenant exist", K(ret), K(tmp_ret), K(tenant_ids)); restore_tenant_exist = true; } - if (is_standby_cluster || restore_tenant_exist) { + if (restore_tenant_exist) { if (OB_ISNULL(schema_status_proxy)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("schema_status_proxy is null", K(ret)); @@ -2723,7 +2720,6 @@ int ObMultiVersionSchemaService::refresh_tenant_schema( const int64_t start = ObTimeUtility::current_time(); int ret = OB_SUCCESS; bool refresh_full_schema = false; - bool is_standby_cluster = GCTX.is_standby_cluster(); bool is_restore = false; if (!check_inner_stat()) { ret = OB_INNER_STAT_ERROR; @@ -2743,7 +2739,7 @@ int ObMultiVersionSchemaService::refresh_tenant_schema( ObISQLClient &sql_client = *sql_proxy_; // read refresh_schema_status from inner table - if ((!is_standby_cluster && !is_restore) + if (!is_restore || is_sys_tenant(tenant_id) || is_meta_tenant(tenant_id)) { // 1. System tenants strengthen the consistency of reading and refresh schema diff --git a/src/share/schema/ob_schema_cache.cpp b/src/share/schema/ob_schema_cache.cpp index 91903752b..c2f50c717 100644 --- a/src/share/schema/ob_schema_cache.cpp +++ b/src/share/schema/ob_schema_cache.cpp @@ -933,7 +933,6 @@ int ObSchemaFetcher::fetch_schema(ObSchemaType schema_type, LOG_WARN("inner stat error", K(ret)); } else { do { - observer::ObUseWeakGuard use_weak_guard; if (INT64_MAX == schema_version) { // skip inspection while fetch latest schema } else if (OB_FAIL(schema_service_->can_read_schema_version(schema_status, schema_version))) { diff --git a/src/share/schema/ob_schema_getter_guard.cpp b/src/share/schema/ob_schema_getter_guard.cpp index 3658de665..fa08fc026 100644 --- a/src/share/schema/ob_schema_getter_guard.cpp +++ b/src/share/schema/ob_schema_getter_guard.cpp @@ -96,7 +96,6 @@ ObSchemaGetterGuard::ObSchemaGetterGuard() schema_objs_(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator(local_allocator_)), mod_(ObSchemaMgrItem::MOD_STACK), schema_guard_type_(INVALID_SCHEMA_GUARD_TYPE), - is_standby_cluster_(false), restore_tenant_exist_(false), is_inited_(false), pin_cache_size_(0) @@ -112,7 +111,6 @@ ObSchemaGetterGuard::ObSchemaGetterGuard(const ObSchemaMgrItem::Mod mod) schema_objs_(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator(local_allocator_)), mod_(mod), schema_guard_type_(INVALID_SCHEMA_GUARD_TYPE), - is_standby_cluster_(false), restore_tenant_exist_(false), is_inited_(false), pin_cache_size_(0) @@ -128,15 +126,13 @@ ObSchemaGetterGuard::~ObSchemaGetterGuard() } } -int ObSchemaGetterGuard::init( - const bool is_standby_cluster) +int ObSchemaGetterGuard::init() { int ret = OB_SUCCESS; if (is_inited_) { ret = OB_INIT_TWICE; LOG_WARN("init twice", KR(ret)); } else { - is_standby_cluster_ = is_standby_cluster; pin_cache_size_ = 0; is_inited_ = true; } @@ -149,7 +145,6 @@ int ObSchemaGetterGuard::reset() schema_service_ = NULL; schema_objs_.reset(); - is_standby_cluster_ = false; restore_tenant_exist_ = false; if (pin_cache_size_ >= FULL_SCHEMA_MEM_THREHOLD) { FLOG_WARN("hold too much full schema memory", K(tenant_id_), K(pin_cache_size_), K(lbt())); @@ -9890,18 +9885,13 @@ bool ObSchemaGetterGuard::ignore_tenant_not_exist_error( const uint64_t tenant_id) { bool bret = false; - if (is_standby_cluster()) { - // ingore error while standby cluster create tenant. + // ignore error when tenant is in physical restore. + bool is_restore = false; + int tmp_ret = check_tenant_is_restore(tenant_id, is_restore); + if (OB_SUCCESS != tmp_ret) { + LOG_WARN_RET(tmp_ret, "fail to check tenant is restore", K(bret), K(tmp_ret), K(tenant_id)); + } else if (is_restore) { bret = true; - } else { - // ignore error when tenant is in physical restore. - bool is_restore = false; - int tmp_ret = check_tenant_is_restore(tenant_id, is_restore); - if (OB_SUCCESS != tmp_ret) { - LOG_WARN_RET(tmp_ret, "fail to check tenant is restore", K(bret), K(tmp_ret), K(tenant_id)); - } else if (is_restore) { - bret = true; - } } return bret; } diff --git a/src/share/schema/ob_schema_getter_guard.h b/src/share/schema/ob_schema_getter_guard.h index 7ff14e6c4..220bbda01 100644 --- a/src/share/schema/ob_schema_getter_guard.h +++ b/src/share/schema/ob_schema_getter_guard.h @@ -1046,9 +1046,8 @@ public: SchemaGuardType get_schema_guard_type() const { return schema_guard_type_; } - bool is_standby_cluster() { return is_standby_cluster_; } bool restore_tenant_exist() { return restore_tenant_exist_; } - bool use_schema_status() { return is_standby_cluster() || restore_tenant_exist(); } + bool use_schema_status() { return restore_tenant_exist(); } int check_formal_guard() const; int is_lazy_mode(const uint64_t tenant_id, bool &is_lazy) const; @@ -1179,7 +1178,7 @@ private: const T *&schema, common::ObKVCacheHandle &handle); - int init(const bool is_standby_cluster); + int init(); int fast_reset() { return is_inited_? reset(): common::OB_SUCCESS; } @@ -1232,7 +1231,6 @@ private: ObSchemaMgrItem::Mod mod_; SchemaGuardType schema_guard_type_; - bool is_standby_cluster_; bool restore_tenant_exist_; bool is_inited_; int64_t pin_cache_size_; diff --git a/src/share/schema/ob_schema_struct.cpp b/src/share/schema/ob_schema_struct.cpp index 582d212c8..32a10fe68 100644 --- a/src/share/schema/ob_schema_struct.cpp +++ b/src/share/schema/ob_schema_struct.cpp @@ -2622,30 +2622,15 @@ int ObDatabaseSchema::get_primary_zone_inherit( ObPrimaryZone &primary_zone) const { int ret = OB_SUCCESS; - bool use_tenant_primary_zone = GCTX.is_standby_cluster() && OB_SYS_TENANT_ID != tenant_id_; primary_zone.reset(); - if (!use_tenant_primary_zone) { - const share::schema::ObSimpleTenantSchema *simple_tenant = nullptr; - if (OB_FAIL(schema_guard.get_tenant_info(tenant_id_, simple_tenant))) { - LOG_WARN("fail to get tenant info", K(ret), K_(tenant_id)); - } else if (OB_UNLIKELY(nullptr == simple_tenant)) { - ret = OB_TENANT_NOT_EXIST; - LOG_WARN("tenant schema ptr is null", K(ret), KPC(simple_tenant)); - } else { - use_tenant_primary_zone = simple_tenant->is_restore(); - } - } - if (OB_FAIL(ret)) { - } else { - const ObTenantSchema *tenant_schema = NULL; - if (OB_FAIL(schema_guard.get_tenant_info(tenant_id_, tenant_schema))) { - LOG_WARN("fail to get tenant schema", K(ret), K(database_id_), K(tenant_id_)); - } else if (OB_UNLIKELY(NULL == tenant_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant schema null", K(ret), K(database_id_), K(tenant_id_), KP(tenant_schema)); - } else if (OB_FAIL(tenant_schema->get_primary_zone_inherit(schema_guard, primary_zone))) { - LOG_WARN("fail to get primary zone array", K(ret), K(database_id_), K(tenant_id_)); - } + const ObTenantSchema *tenant_schema = NULL; + if (OB_FAIL(schema_guard.get_tenant_info(tenant_id_, tenant_schema))) { + LOG_WARN("fail to get tenant schema", K(ret), K(database_id_), K(tenant_id_)); + } else if (OB_UNLIKELY(NULL == tenant_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant schema null", K(ret), K(database_id_), K(tenant_id_), KP(tenant_schema)); + } else if (OB_FAIL(tenant_schema->get_primary_zone_inherit(schema_guard, primary_zone))) { + LOG_WARN("fail to get primary zone array", K(ret), K(database_id_), K(tenant_id_)); } return ret; } @@ -4757,30 +4742,15 @@ int ObTablegroupSchema::get_zone_replica_attr_array_inherit( ZoneLocalityIArray &locality) const { int ret = OB_SUCCESS; - bool use_tenant_locality = GCTX.is_standby_cluster() && OB_SYS_TENANT_ID != tenant_id_; locality.reset(); - if (!use_tenant_locality) { - const share::schema::ObSimpleTenantSchema *simple_tenant = nullptr; - if (OB_FAIL(schema_guard.get_tenant_info(tenant_id_, simple_tenant))) { - LOG_WARN("fail to get tenant info", K(ret), K_(tenant_id)); - } else if (OB_UNLIKELY(nullptr == simple_tenant)) { - ret = OB_TENANT_NOT_EXIST; - LOG_WARN("tenant schema ptr is null", K(ret), KPC(simple_tenant)); - } else { - use_tenant_locality = simple_tenant->is_restore(); - } - } - if (OB_FAIL(ret)) { - } else { - const ObTenantSchema *tenant_schema = NULL; - if (OB_FAIL(schema_guard.get_tenant_info(get_tenant_id(), tenant_schema))) { - LOG_WARN("fail to get tenant schema", K(ret), K(tablegroup_id_), K(tenant_id_)); - } else if (OB_UNLIKELY(NULL == tenant_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant schema null", K(ret), K(tablegroup_id_), K(tenant_id_), KP(tenant_schema)); - } else if (OB_FAIL(tenant_schema->get_zone_replica_attr_array_inherit(schema_guard, locality))) { - LOG_WARN("fail to get zone replica num array", K(ret), K(tablegroup_id_), K(tenant_id_)); - } + const ObTenantSchema *tenant_schema = NULL; + if (OB_FAIL(schema_guard.get_tenant_info(get_tenant_id(), tenant_schema))) { + LOG_WARN("fail to get tenant schema", K(ret), K(tablegroup_id_), K(tenant_id_)); + } else if (OB_UNLIKELY(NULL == tenant_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant schema null", K(ret), K(tablegroup_id_), K(tenant_id_), KP(tenant_schema)); + } else if (OB_FAIL(tenant_schema->get_zone_replica_attr_array_inherit(schema_guard, locality))) { + LOG_WARN("fail to get zone replica num array", K(ret), K(tablegroup_id_), K(tenant_id_)); } return ret; } @@ -4790,33 +4760,17 @@ int ObTablegroupSchema::get_locality_str_inherit( const common::ObString *&locality_str) const { int ret = OB_SUCCESS; - bool use_tenant_locality = OB_SYS_TENANT_ID != tenant_id_ && GCTX.is_standby_cluster(); locality_str = nullptr; - if (!use_tenant_locality) { - const share::schema::ObSimpleTenantSchema *simple_tenant = nullptr; - if (OB_FAIL(guard.get_tenant_info(tenant_id_, simple_tenant))) { - LOG_WARN("fail to get tenant info", K(ret), K_(tenant_id)); - } else if (OB_UNLIKELY(nullptr == simple_tenant)) { - ret = OB_TENANT_NOT_EXIST; - LOG_WARN("tenant schema ptr is null", K(ret), KPC(simple_tenant)); - } else { - use_tenant_locality = simple_tenant->is_restore(); - } - } - if (OB_FAIL(ret)) { - } else if (use_tenant_locality - || nullptr == locality_str - || locality_str->empty()) { - const ObSimpleTenantSchema *tenant_schema = nullptr; - if (OB_FAIL(guard.get_tenant_info(get_tenant_id(), tenant_schema))) { - LOG_WARN("fail to get tenant schema", K(ret), "tenant_id", get_tenant_id()); - } else if (OB_UNLIKELY(nullptr == tenant_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to get tenant schema", K(ret), "tenant_id", get_tenant_id()); - } else { - locality_str = &tenant_schema->get_locality_str(); - } + const ObSimpleTenantSchema *tenant_schema = nullptr; + if (OB_FAIL(guard.get_tenant_info(get_tenant_id(), tenant_schema))) { + LOG_WARN("fail to get tenant schema", K(ret), "tenant_id", get_tenant_id()); + } else if (OB_UNLIKELY(nullptr == tenant_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get tenant schema", K(ret), "tenant_id", get_tenant_id()); + } else { + locality_str = &tenant_schema->get_locality_str(); } + if (OB_SUCC(ret)) { if (OB_UNLIKELY(nullptr == locality_str || locality_str->empty())) { ret = OB_ERR_UNEXPECTED; @@ -4831,30 +4785,15 @@ int ObTablegroupSchema::get_primary_zone_inherit( ObPrimaryZone &primary_zone) const { int ret = OB_SUCCESS; - bool use_tenant_primary_zone = GCTX.is_standby_cluster() && OB_SYS_TENANT_ID != tenant_id_; primary_zone.reset(); - if (!use_tenant_primary_zone) { - const share::schema::ObSimpleTenantSchema *simple_tenant = nullptr; - if (OB_FAIL(schema_guard.get_tenant_info(tenant_id_, simple_tenant))) { - LOG_WARN("fail to get tenant info", K(ret), K_(tenant_id)); - } else if (OB_UNLIKELY(nullptr == simple_tenant)) { - ret = OB_TENANT_NOT_EXIST; - LOG_WARN("tenant schema ptr is null", K(ret), KPC(simple_tenant)); - } else { - use_tenant_primary_zone = simple_tenant->is_restore(); - } - } - if (OB_FAIL(ret)) { - } else { - const ObTenantSchema *tenant_schema = NULL; - if (OB_FAIL(schema_guard.get_tenant_info(get_tenant_id(), tenant_schema))) { - LOG_WARN("fail to get tenant schema", K(ret), K(tablegroup_id_), K(tenant_id_)); - } else if (OB_UNLIKELY(NULL == tenant_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant schema null", K(ret), K(tablegroup_id_), K(tenant_id_), KP(tenant_schema)); - } else if (OB_FAIL(tenant_schema->get_primary_zone_inherit(schema_guard, primary_zone))) { - LOG_WARN("fail to get primary zone array", K(ret), K(tablegroup_id_), K(tenant_id_)); - } + const ObTenantSchema *tenant_schema = NULL; + if (OB_FAIL(schema_guard.get_tenant_info(get_tenant_id(), tenant_schema))) { + LOG_WARN("fail to get tenant schema", K(ret), K(tablegroup_id_), K(tenant_id_)); + } else if (OB_UNLIKELY(NULL == tenant_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant schema null", K(ret), K(tablegroup_id_), K(tenant_id_), KP(tenant_schema)); + } else if (OB_FAIL(tenant_schema->get_primary_zone_inherit(schema_guard, primary_zone))) { + LOG_WARN("fail to get primary zone array", K(ret), K(tablegroup_id_), K(tenant_id_)); } return ret; } diff --git a/src/share/schema/ob_server_schema_service.cpp b/src/share/schema/ob_server_schema_service.cpp index 72ad8ce87..7d70c5256 100644 --- a/src/share/schema/ob_server_schema_service.cpp +++ b/src/share/schema/ob_server_schema_service.cpp @@ -4328,14 +4328,8 @@ int ObServerSchemaService::apply_##SCHEMA##_schema_to_cache( \ } else { \ FOREACH_CNT_X(schema_key, schema_keys, OB_SUCC(ret)) { \ if (OB_FAIL(mgr.del_##SCHEMA(schema_key->get_##SCHEMA##_key()))) { \ - if (GCTX.is_standby_cluster() \ - && OB_SYS_TENANT_ID == tenant_id \ - && OB_ENTRY_NOT_EXIST == ret) { \ - ret = OB_SUCCESS; \ - } else { \ - LOG_WARN("del "#SCHEMA" failed", K(ret), \ - #SCHEMA"_key", schema_key->get_##SCHEMA##_key()); \ - } \ + LOG_WARN("del "#SCHEMA" failed", K(ret), \ + #SCHEMA"_key", schema_key->get_##SCHEMA##_key()); \ } \ } \ ALLOW_NEXT_LOG(); \ diff --git a/src/share/schema/ob_table_schema.cpp b/src/share/schema/ob_table_schema.cpp index e43315e71..cbb8eb68c 100644 --- a/src/share/schema/ob_table_schema.cpp +++ b/src/share/schema/ob_table_schema.cpp @@ -417,33 +417,17 @@ int ObSimpleTableSchemaV2::get_zone_replica_attr_array_inherit( { int ret = OB_SUCCESS; const uint64_t tenant_id = get_tenant_id(); - bool use_tenant_locality = !is_sys_tenant(tenant_id) && GCTX.is_standby_cluster(); locality.reuse(); - if (!has_partition()) { - // No partition, no concept of locality - } else if (!use_tenant_locality) { - const share::schema::ObSimpleTenantSchema *simple_tenant = nullptr; - if (OB_FAIL(schema_guard.get_tenant_info(tenant_id, simple_tenant))) { - LOG_WARN("fail to get tenant info", K(ret), K(tenant_id)); - } else if (OB_UNLIKELY(nullptr == simple_tenant)) { - ret = OB_TENANT_NOT_EXIST; - LOG_WARN("tenant schema ptr is null", K(ret), KPC(simple_tenant)); - } else { - use_tenant_locality = simple_tenant->is_restore(); - } - } - if (OB_FAIL(ret)) { - } else { - // Locality is not set when creating table, take tenant's fill - const ObTenantSchema *tenant_schema = NULL; - if (OB_FAIL(schema_guard.get_tenant_info(tenant_id, tenant_schema))) { - LOG_WARN("fail to get tenant schema", K(ret), K(table_id_), K(tenant_id)); - } else if (OB_UNLIKELY(NULL == tenant_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant schema null", K(ret), K(table_id_), K(tenant_id), KP(tenant_schema)); - } else if (OB_FAIL(tenant_schema->get_zone_replica_attr_array_inherit(schema_guard, locality))) { - LOG_WARN("fail to get zone replica num array", K(ret), K(table_id_), K(tenant_id)); - } + + // Locality is not set when creating table, take tenant's fill + const ObTenantSchema *tenant_schema = NULL; + if (OB_FAIL(schema_guard.get_tenant_info(tenant_id, tenant_schema))) { + LOG_WARN("fail to get tenant schema", K(ret), K(table_id_), K(tenant_id)); + } else if (OB_UNLIKELY(NULL == tenant_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant schema null", K(ret), K(table_id_), K(tenant_id), KP(tenant_schema)); + } else if (OB_FAIL(tenant_schema->get_zone_replica_attr_array_inherit(schema_guard, locality))) { + LOG_WARN("fail to get zone replica num array", K(ret), K(table_id_), K(tenant_id)); } return ret; } @@ -454,32 +438,15 @@ int ObSimpleTableSchemaV2::get_primary_zone_inherit( { int ret = OB_SUCCESS; const uint64_t tenant_id = get_tenant_id(); - const uint64_t tablegroup_id = get_tablegroup_id(); - const uint64_t database_id = get_database_id(); - bool use_tenant_primary_zone = !is_sys_tenant(tenant_id) && GCTX.is_standby_cluster(); primary_zone.reset(); - if (!use_tenant_primary_zone) { - const share::schema::ObSimpleTenantSchema *simple_tenant = nullptr; - if (OB_FAIL(schema_guard.get_tenant_info(tenant_id, simple_tenant))) { - LOG_WARN("fail to get tenant info", K(ret), K(tenant_id)); - } else if (OB_UNLIKELY(nullptr == simple_tenant)) { - ret = OB_TENANT_NOT_EXIST; - LOG_WARN("tenant schema ptr is null", K(ret), KPC(simple_tenant)); - } else { - use_tenant_primary_zone = simple_tenant->is_restore(); - } - } - if (OB_FAIL(ret)) { - } else if (use_tenant_primary_zone) { - const ObTenantSchema *tenant_schema = NULL; - if (OB_FAIL(schema_guard.get_tenant_info(tenant_id, tenant_schema))) { - LOG_WARN("fail to get tenant schema", K(ret), K(database_id), K(tenant_id)); - } else if (OB_UNLIKELY(NULL == tenant_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant schema null", K(ret), K(database_id), K(tenant_id), KP(tenant_schema)); - } else if (OB_FAIL(tenant_schema->get_primary_zone_inherit(schema_guard, primary_zone))) { - LOG_WARN("fail to get primary zone array", K(ret), K(database_id), K(tenant_id)); - } + const ObTenantSchema *tenant_schema = NULL; + if (OB_FAIL(schema_guard.get_tenant_info(tenant_id, tenant_schema))) { + LOG_WARN("fail to get tenant schema", K(ret), K(tenant_id)); + } else if (OB_UNLIKELY(NULL == tenant_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant schema null", K(ret), K(tenant_id), KP(tenant_schema)); + } else if (OB_FAIL(tenant_schema->get_primary_zone_inherit(schema_guard, primary_zone))) { + LOG_WARN("fail to get primary zone array", K(ret), K(tenant_id)); } return ret; } @@ -1035,24 +1002,8 @@ int ObSimpleTableSchemaV2::get_locality_str_inherit( { int ret = OB_SUCCESS; const uint64_t tenant_id = get_tenant_id(); - bool use_tenant_locality = !is_sys_tenant(tenant_id) && GCTX.is_standby_cluster(); locality_str = NULL; - if (OB_INVALID_ID == get_table_id()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid table_id", K(ret), K_(table_id)); - } else if (!use_tenant_locality) { - const share::schema::ObSimpleTenantSchema *simple_tenant = nullptr; - if (OB_FAIL(guard.get_tenant_info(tenant_id, simple_tenant))) { - LOG_WARN("fail to get tenant info", K(ret), K(tenant_id)); - } else if (OB_UNLIKELY(nullptr == simple_tenant)) { - ret = OB_TENANT_NOT_EXIST; - LOG_WARN("tenant schema ptr is null", K(ret), KPC(simple_tenant)); - } else { - use_tenant_locality = simple_tenant->is_restore(); - } - } if (OB_FAIL(ret)) { - } else if (!OB_ISNULL(locality_str) && !locality_str->empty()) { } else { const ObSimpleTenantSchema *tenant = NULL; if (OB_FAIL(guard.get_tenant_info(get_tenant_id(), tenant))) { diff --git a/src/share/sequence/ob_sequence_dml_proxy.cpp b/src/share/sequence/ob_sequence_dml_proxy.cpp index a0fe3f33b..76d21af9e 100644 --- a/src/share/sequence/ob_sequence_dml_proxy.cpp +++ b/src/share/sequence/ob_sequence_dml_proxy.cpp @@ -286,9 +286,6 @@ int ObSequenceDMLProxy::next_batch( "WHERE SEQUENCE_ID = %lu", tname, next_value.format(), sequence_id))) { LOG_WARN("format update sql fail", K(ret)); - } else if (GCTX.is_standby_cluster() && OB_SYS_TENANT_ID != tenant_id) { - ret = OB_OP_NOT_ALLOW; - LOG_WARN("can't write sys table now", K(ret), K(tenant_id)); } else if (OB_FAIL(trans.write(tenant_id, sql.ptr(), affected_rows))) { LOG_WARN("fail to execute sql", K(sql), K(ret)); } else { diff --git a/src/share/stat/ob_dbms_stats_utils.cpp b/src/share/stat/ob_dbms_stats_utils.cpp index 38003395f..25763824c 100644 --- a/src/share/stat/ob_dbms_stats_utils.cpp +++ b/src/share/stat/ob_dbms_stats_utils.cpp @@ -149,19 +149,15 @@ int ObDbmsStatsUtils::cast_number_to_double(const number::ObNumber &src_val, dou return ret; } -// gather statistic related inner table should not read or write during tenant restore or on -// standby cluster. +// gather statistic related inner table should not read or write restore or standby tenant int ObDbmsStatsUtils::check_table_read_write_valid(const uint64_t tenant_id, bool &is_valid) { int ret = OB_SUCCESS; is_valid = true; - bool in_restore = false; - if (OB_ISNULL(GCTX.schema_service_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret)); - } else if (OB_FAIL(GCTX.schema_service_->check_tenant_is_restore(NULL, tenant_id, in_restore))) { - LOG_WARN("failed to check tenant is restore", K(ret)); - } else if (OB_UNLIKELY(in_restore) || GCTX.is_standby_cluster()) { + bool is_primary = true; + if (OB_FAIL(ObShareUtil::mtl_check_if_tenant_role_is_primary(tenant_id, is_primary))) { + LOG_WARN("fail to execute mtl_check_if_tenant_role_is_primary", KR(ret), K(tenant_id)); + } else if (OB_UNLIKELY(!is_primary)) { is_valid = false; } return ret; diff --git a/src/share/stat/ob_opt_stat_monitor_manager.cpp b/src/share/stat/ob_opt_stat_monitor_manager.cpp index 466aafd35..02f5b5339 100644 --- a/src/share/stat/ob_opt_stat_monitor_manager.cpp +++ b/src/share/stat/ob_opt_stat_monitor_manager.cpp @@ -103,15 +103,12 @@ void ObOptStatMonitorFlushAllTask::runTimerTask() int ret = OB_SUCCESS; if (OB_NOT_NULL(optstat_monitor_mgr_) && optstat_monitor_mgr_->inited_) { LOG_INFO("run opt stat monitor flush all task", K(optstat_monitor_mgr_->tenant_id_)); - share::schema::ObMultiVersionSchemaService &schema_service = share::schema::ObMultiVersionSchemaService::get_instance(); - share::schema::ObSchemaGetterGuard schema_guard; - bool in_restore = false; + uint64_t tenant_id = optstat_monitor_mgr_->tenant_id_; + bool is_primary = true; THIS_WORKER.set_timeout_ts(FLUSH_INTERVAL / 2 + ObTimeUtility::current_time()); - if (OB_FAIL(schema_service.get_tenant_schema_guard(optstat_monitor_mgr_->tenant_id_, schema_guard))) { - LOG_WARN("failed to get schema guard", K(ret)); - } else if (OB_FAIL(schema_guard.check_tenant_is_restore(optstat_monitor_mgr_->tenant_id_, in_restore))) { - LOG_WARN("failed to check tenant is restore", K(ret)); - } else if (in_restore || GCTX.is_standby_cluster()) { + if (OB_FAIL(ObShareUtil::mtl_check_if_tenant_role_is_primary(tenant_id, is_primary))) { + LOG_WARN("fail to execute mtl_check_if_tenant_role_is_primary", KR(ret), K(tenant_id)); + } else if (!is_primary) { // do nothing } else if (OB_FAIL(optstat_monitor_mgr_->update_column_usage_info(false))) { LOG_WARN("failed to update column usage info", K(ret)); @@ -126,15 +123,12 @@ void ObOptStatMonitorCheckTask::runTimerTask() int ret = OB_SUCCESS; if (OB_NOT_NULL(optstat_monitor_mgr_) && optstat_monitor_mgr_->inited_) { LOG_INFO("run opt stat monitor check task", K(optstat_monitor_mgr_->tenant_id_)); - share::schema::ObMultiVersionSchemaService &schema_service = share::schema::ObMultiVersionSchemaService::get_instance(); - share::schema::ObSchemaGetterGuard schema_guard; - bool in_restore = false; + uint64_t tenant_id = optstat_monitor_mgr_->tenant_id_; + bool is_primary = true; THIS_WORKER.set_timeout_ts(CHECK_INTERVAL + ObTimeUtility::current_time()); - if (OB_FAIL(schema_service.get_tenant_schema_guard(optstat_monitor_mgr_->tenant_id_, schema_guard))) { - LOG_WARN("failed to get schema guard", K(ret)); - } else if (OB_FAIL(schema_guard.check_tenant_is_restore(optstat_monitor_mgr_->tenant_id_, in_restore))) { - LOG_WARN("failed to check tenant is restore", K(ret)); - } else if (in_restore || GCTX.is_standby_cluster()) { + if (OB_FAIL(ObShareUtil::mtl_check_if_tenant_role_is_primary(tenant_id, is_primary))) { + LOG_WARN("fail to execute mtl_check_if_tenant_role_is_primary", KR(ret), K(tenant_id)); + } else if (!is_primary) { // do nothing } else if (OB_FAIL(optstat_monitor_mgr_->update_column_usage_info(true))) { LOG_WARN("failed to update column usage info", K(ret)); @@ -236,35 +230,31 @@ int ObOptStatMonitorManager::flush_database_monitoring_info(sql::ObExecContext & int ObOptStatMonitorManager::update_local_cache(common::ObIArray &args) { int ret = OB_SUCCESS; - if (GCTX.is_standby_cluster()) { - // standby cluster can't write __all_column_usage, so do not need to update local update - } else { - SpinRLockGuard guard(lock_); - for (int64_t i = 0; OB_SUCC(ret) && i < args.count(); ++i) { - ColumnUsageArg &arg = args.at(i); - StatKey col_key(arg.table_id_, arg.column_id_); - int64_t flags = 0; - if (OB_FAIL(column_usage_map_.get_refactored(col_key, flags))) { - if (OB_LIKELY(ret == OB_HASH_NOT_EXIST)) { - if (OB_FAIL(column_usage_map_.set_refactored(col_key, arg.flags_))) { - // other thread set the refactor, try update again - if (OB_FAIL(column_usage_map_.get_refactored(col_key, flags))) { - LOG_WARN("failed to get refactored", K(ret)); - } else if ((~flags) & arg.flags_) { - UpdateValueAtomicOp atomic_op(arg.flags_); - if (OB_FAIL(column_usage_map_.atomic_refactored(col_key, atomic_op))) { - LOG_WARN("failed to atomic refactored", K(ret)); - } + SpinRLockGuard guard(lock_); + for (int64_t i = 0; OB_SUCC(ret) && i < args.count(); ++i) { + ColumnUsageArg &arg = args.at(i); + StatKey col_key(arg.table_id_, arg.column_id_); + int64_t flags = 0; + if (OB_FAIL(column_usage_map_.get_refactored(col_key, flags))) { + if (OB_LIKELY(ret == OB_HASH_NOT_EXIST)) { + if (OB_FAIL(column_usage_map_.set_refactored(col_key, arg.flags_))) { + // other thread set the refactor, try update again + if (OB_FAIL(column_usage_map_.get_refactored(col_key, flags))) { + LOG_WARN("failed to get refactored", K(ret)); + } else if ((~flags) & arg.flags_) { + UpdateValueAtomicOp atomic_op(arg.flags_); + if (OB_FAIL(column_usage_map_.atomic_refactored(col_key, atomic_op))) { + LOG_WARN("failed to atomic refactored", K(ret)); } } - } else { - LOG_WARN("failed to get refactored", K(ret)); - } - } else if ((~flags) & arg.flags_) { - UpdateValueAtomicOp atomic_op(arg.flags_); - if (OB_FAIL(column_usage_map_.atomic_refactored(col_key, atomic_op))) { - LOG_WARN("failed to atomic refactored", K(ret)); } + } else { + LOG_WARN("failed to get refactored", K(ret)); + } + } else if ((~flags) & arg.flags_) { + UpdateValueAtomicOp atomic_op(arg.flags_); + if (OB_FAIL(column_usage_map_.atomic_refactored(col_key, atomic_op))) { + LOG_WARN("failed to atomic refactored", K(ret)); } } } @@ -274,34 +264,30 @@ int ObOptStatMonitorManager::update_local_cache(common::ObIArray int ObOptStatMonitorManager::update_local_cache(ObOptDmlStat &dml_stat) { int ret = OB_SUCCESS; - if (GCTX.is_standby_cluster()) { - // standby cluster can't write __all_monitor_modified, so do not need to update local update - } else { - SpinRLockGuard guard(lock_); - StatKey key(dml_stat.table_id_, dml_stat.tablet_id_); - ObOptDmlStat tmp_dml_stat; - if (OB_FAIL(dml_stat_map_.get_refactored(key, tmp_dml_stat))) { - if (OB_LIKELY(ret == OB_HASH_NOT_EXIST)) { - if (OB_FAIL(dml_stat_map_.set_refactored(key, dml_stat))) { - // other thread set the refactor, try update again - if (OB_FAIL(dml_stat_map_.get_refactored(key, tmp_dml_stat))) { - LOG_WARN("failed to get refactored", K(ret)); - } else { - UpdateValueAtomicOp atomic_op(dml_stat); - if (OB_FAIL(dml_stat_map_.atomic_refactored(key, atomic_op))) { - LOG_WARN("failed to atomic refactored", K(ret)); - } + SpinRLockGuard guard(lock_); + StatKey key(dml_stat.table_id_, dml_stat.tablet_id_); + ObOptDmlStat tmp_dml_stat; + if (OB_FAIL(dml_stat_map_.get_refactored(key, tmp_dml_stat))) { + if (OB_LIKELY(ret == OB_HASH_NOT_EXIST)) { + if (OB_FAIL(dml_stat_map_.set_refactored(key, dml_stat))) { + // other thread set the refactor, try update again + if (OB_FAIL(dml_stat_map_.get_refactored(key, tmp_dml_stat))) { + LOG_WARN("failed to get refactored", K(ret)); + } else { + UpdateValueAtomicOp atomic_op(dml_stat); + if (OB_FAIL(dml_stat_map_.atomic_refactored(key, atomic_op))) { + LOG_WARN("failed to atomic refactored", K(ret)); } } - } else { - LOG_WARN("failed to get refactored", K(ret)); } } else { - UpdateValueAtomicOp atomic_op(dml_stat); - if (OB_FAIL(dml_stat_map_.atomic_refactored(key, atomic_op))) { - LOG_WARN("failed to atomic refactored", K(ret)); - } else {/*do nothing*/} + LOG_WARN("failed to get refactored", K(ret)); } + } else { + UpdateValueAtomicOp atomic_op(dml_stat); + if (OB_FAIL(dml_stat_map_.atomic_refactored(key, atomic_op))) { + LOG_WARN("failed to atomic refactored", K(ret)); + } else {/*do nothing*/} } return ret; } @@ -578,13 +564,10 @@ int ObOptStatMonitorManager::check_table_writeable(bool &is_writeable) { int ret = OB_SUCCESS; is_writeable = true; - bool in_restore = false; - if (OB_ISNULL(GCTX.schema_service_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret)); - } else if (OB_FAIL(GCTX.schema_service_->check_tenant_is_restore(NULL, tenant_id_, in_restore))) { - LOG_WARN("failed to check tenant is restore", K(ret)); - } else if (OB_UNLIKELY(in_restore) || GCTX.is_standby_cluster()) { + bool is_primary = true; + if (OB_FAIL(ObShareUtil::mtl_check_if_tenant_role_is_primary(tenant_id_, is_primary))) { + LOG_WARN("fail to execute mtl_check_if_tenant_role_is_primary", KR(ret), K(tenant_id_)); + } else if (OB_UNLIKELY(!is_primary)) { is_writeable = false; } return ret; diff --git a/src/share/table/ob_ttl_util.cpp b/src/share/table/ob_ttl_util.cpp index 73725d32c..68bdbe888 100644 --- a/src/share/table/ob_ttl_util.cpp +++ b/src/share/table/ob_ttl_util.cpp @@ -18,6 +18,7 @@ #include "share/ob_server_status.h" #include "share/schema/ob_schema_utils.h" #include "rootserver/ob_root_service.h" +#include "rootserver/ob_tenant_info_loader.h" #include "observer/omt/ob_tenant_timezone_mgr.h" #include "share/schema/ob_multi_version_schema_service.h" #include "lib/stat/ob_diagnose_info.h" @@ -546,7 +547,11 @@ bool ObTTLUtil::check_can_do_work() { int ret = OB_SUCCESS; int64_t tenant_id = MTL_ID(); uint64_t tenant_data_version = 0;; - if (GCTX.is_standby_cluster()) { + bool is_primary = true; + if (OB_FAIL(ObShareUtil::mtl_check_if_tenant_role_is_primary(tenant_id, is_primary))) { + bret = false; + LOG_WARN("fail to execute mtl_check_if_tenant_role_is_primary", KR(ret), K(tenant_id)); + } else if (!is_primary) { bret = false; } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, tenant_data_version))) { bret = false; diff --git a/src/sql/engine/cmd/ob_alter_system_executor.cpp b/src/sql/engine/cmd/ob_alter_system_executor.cpp index 0f27eea70..2e7088f74 100644 --- a/src/sql/engine/cmd/ob_alter_system_executor.cpp +++ b/src/sql/engine/cmd/ob_alter_system_executor.cpp @@ -41,7 +41,7 @@ #include "share/scheduler/ob_dag_warning_history_mgr.h" #include "observer/omt/ob_tenant.h" //ObTenant #include "rootserver/freeze/ob_major_freeze_helper.h" //ObMajorFreezeHelper -#include "share/ob_primary_standby_service.h" // ObPrimaryStandbyService +#include "rootserver/standby/ob_standby_service.h" // ObStandbyService #include "rpc/obmysql/ob_sql_sock_session.h" #include "sql/plan_cache/ob_plan_cache.h" #include "pl/pl_cache/ob_pl_cache_mgr.h" @@ -50,6 +50,8 @@ #include "share/table/ob_ttl_util.h" #include "rootserver/restore/ob_tenant_clone_util.h" +#include "rootserver/ob_service_name_command.h" +#include "rootserver/ob_tenant_event_def.h" namespace oceanbase { using namespace common; @@ -57,7 +59,7 @@ using namespace obrpc; using namespace share; using namespace omt; using namespace obmysql; - +using namespace tenant_event; namespace sql { int ObFreezeExecutor::execute(ObExecContext &ctx, ObFreezeStmt &stmt) @@ -2146,6 +2148,7 @@ int ObSwitchTenantExecutor::execute(ObExecContext &ctx, ObSwitchTenantStmt &stmt } else { ObSwitchTenantArg &arg = stmt.get_arg(); arg.set_stmt_str(first_stmt); + ObSQLSessionInfo *session_info = ctx.get_my_session(); //left 200ms to return result const int64_t remain_timeout_interval_us = THIS_WORKER.get_timeout_remain(); @@ -2157,9 +2160,15 @@ int ObSwitchTenantExecutor::execute(ObExecContext &ctx, ObSwitchTenantStmt &stmt // TODO support specify ALL if (OB_FAIL(ret)) { - } else if (arg.get_is_verify()) { - //do nothing - } else if (OB_FAIL(OB_PRIMARY_STANDBY_SERVICE.switch_tenant(arg))) { + } else if (OB_ISNULL(session_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("session_info is null", KR(ret), KP(session_info)); + } else if (OB_UNLIKELY(!session_info->get_service_name().is_empty())) { + ret = OB_OP_NOT_ALLOW; + LOG_WARN("switching tenant role cannot be executed in the session which is created via service_name", + KR(ret), K(session_info->get_service_name())); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "This session is created via service_name, switching tenant is"); + } else if (OB_FAIL(OB_STANDBY_SERVICE.switch_tenant(arg))) { LOG_WARN("failed to switch_tenant", KR(ret), K(arg)); } @@ -2183,7 +2192,7 @@ int ObRecoverTenantExecutor::execute(ObExecContext &ctx, ObRecoverTenantStmt &st // TODO support specify ALL and tenant list if (OB_FAIL(ret)) { - } else if (OB_FAIL(OB_PRIMARY_STANDBY_SERVICE.recover_tenant(arg))) { + } else if (OB_FAIL(OB_STANDBY_SERVICE.recover_tenant(arg))) { LOG_WARN("failed to recover_tenant", KR(ret), K(arg)); } } @@ -2871,5 +2880,49 @@ int ObTransferPartitionExecutor::execute(ObExecContext& ctx, ObTransferPartition } return ret; } +int ObServiceNameExecutor::execute(ObExecContext& ctx, ObServiceNameStmt& stmt) +{ + int ret = OB_SUCCESS; + const ObServiceNameArg &arg = stmt.get_arg(); + const ObServiceNameString &service_name_str = arg.get_service_name_str(); + const ObServiceNameArg::ObServiceOp &service_op = arg.get_service_op(); + const uint64_t tenant_id = arg.get_target_tenant_id(); + ObSQLSessionInfo *session_info = ctx.get_my_session(); + if (OB_UNLIKELY(!arg.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg", KR(ret), K(arg)); + } else if (OB_FAIL(ObServiceNameProxy::check_is_service_name_enabled(tenant_id))) { + LOG_WARN("fail to execute check_is_service_name_enabled", KR(ret), K(tenant_id)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "The tenant's or meta tenant's data_version is smaller than 4_2_4_0, service name related command is"); + } else if (OB_ISNULL(session_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("session_info is null", KR(ret), KP(session_info)); + } else if (OB_UNLIKELY(!session_info->get_service_name().is_empty())) { + ret = OB_OP_NOT_ALLOW; + LOG_WARN("service_name related commands cannot be executed in the session which is created via service_name", + KR(ret), K(session_info->get_service_name())); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "This session is created via service_name, service name related command is"); + } else if (arg.is_create_service()) { + if (OB_FAIL(ObServiceNameCommand::create_service(tenant_id, service_name_str))) { + LOG_WARN("fail to create service", KR(ret), K(tenant_id), K(service_name_str)); + } + } else if (arg.is_delete_service()) { + if (OB_FAIL(ObServiceNameCommand::delete_service(tenant_id, service_name_str))) { + LOG_WARN("fail to delete service", KR(ret), K(tenant_id), K(service_name_str)); + } + } else if (arg.is_start_service()) { + if (OB_FAIL(ObServiceNameCommand::start_service(tenant_id, service_name_str))) { + LOG_WARN("fail to start service", KR(ret), K(tenant_id), K(service_name_str)); + } + } else if (arg.is_stop_service()) { + if (OB_FAIL(ObServiceNameCommand::stop_service(tenant_id, service_name_str))) { + LOG_WARN("fail to stop service", KR(ret), K(tenant_id), K(service_name_str)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unknown service operation", KR(ret), K(arg)); + } + return ret; +} } // end namespace sql } // end namespace oceanbase diff --git a/src/sql/engine/cmd/ob_alter_system_executor.h b/src/sql/engine/cmd/ob_alter_system_executor.h index 08cfdd67f..b1bec9616 100644 --- a/src/sql/engine/cmd/ob_alter_system_executor.h +++ b/src/sql/engine/cmd/ob_alter_system_executor.h @@ -142,8 +142,9 @@ DEF_SIMPLE_EXECUTOR(ObResetConfig); DEF_SIMPLE_EXECUTOR(ObCancelClone); -DEF_SIMPLE_EXECUTOR(ObTransferPartition); +DEF_SIMPLE_EXECUTOR(ObTransferPartition); +DEF_SIMPLE_EXECUTOR(ObServiceName); class ObCancelTaskExecutor { public: diff --git a/src/sql/engine/cmd/ob_analyze_executor.cpp b/src/sql/engine/cmd/ob_analyze_executor.cpp index 3786086e0..083cdd5a6 100644 --- a/src/sql/engine/cmd/ob_analyze_executor.cpp +++ b/src/sql/engine/cmd/ob_analyze_executor.cpp @@ -45,20 +45,21 @@ int ObAnalyzeExecutor::execute(ObExecContext &ctx, ObAnalyzeStmt &stmt) { int ret = OB_SUCCESS; ObSEArray params; - share::schema::ObSchemaGetterGuard *schema_guard = ctx.get_virtual_table_ctx().schema_guard_; ObSQLSessionInfo *session = ctx.get_my_session(); - bool in_restore = false; - if (OB_ISNULL(schema_guard) || OB_ISNULL(session)) { + if (OB_ISNULL(session)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(schema_guard), K(session)); - } else if (OB_FAIL(schema_guard->check_tenant_is_restore(session->get_effective_tenant_id(), - in_restore))) { - LOG_WARN("failed to check tenant is restore", K(ret)); - } else if (OB_UNLIKELY(in_restore) || - GCTX.is_standby_cluster()) { - ret = OB_NOT_SUPPORTED; - LOG_USER_ERROR(OB_NOT_SUPPORTED, "analyze table during restore or standby cluster"); - } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("get unexpected null", K(ret), K(session)); + } else { + uint64_t tenant_id = session->get_effective_tenant_id(); + bool is_primary = true; + if (OB_FAIL(ObShareUtil::mtl_check_if_tenant_role_is_primary(tenant_id, is_primary))) { + LOG_WARN("fail to execute mtl_check_if_tenant_role_is_primary", KR(ret), K(tenant_id)); + } else if (OB_UNLIKELY(!is_primary)) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "analyze table during non-primary tenant"); + } + } + if (FAILEDx(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (OB_FAIL(ObDbmsStatsUtils::cancel_async_gather_stats(ctx))) { LOG_WARN("failed to cancel async gather stats", K(ret)); diff --git a/src/sql/engine/cmd/ob_ddl_executor_util.cpp b/src/sql/engine/cmd/ob_ddl_executor_util.cpp index 63032f5ba..176484969 100644 --- a/src/sql/engine/cmd/ob_ddl_executor_util.cpp +++ b/src/sql/engine/cmd/ob_ddl_executor_util.cpp @@ -32,15 +32,19 @@ namespace sql int ObDDLExecutorUtil::handle_session_exception(ObSQLSessionInfo &session) { int ret = OB_SUCCESS; + const uint64_t tenant_id = session.get_effective_tenant_id(); + bool is_standby = false; if (OB_UNLIKELY(session.is_query_killed())) { ret = OB_ERR_QUERY_INTERRUPTED; LOG_WARN("query is killed", K(ret)); } else if (OB_UNLIKELY(session.is_zombie())) { ret = OB_SESSION_KILLED; LOG_WARN("session is killed", K(ret)); - } else if (GCTX.is_standby_cluster()) { + } else if (OB_FAIL(ObShareUtil::mtl_check_if_tenant_role_is_standby(tenant_id, is_standby))) { + LOG_WARN("fail to execute mtl_check_if_tenant_role_is_standby", KR(ret), K(tenant_id)); + } else if (is_standby) { ret = OB_SESSION_KILLED; - LOG_INFO("cluster switchoverd, kill session", KR(ret)); + LOG_WARN("session is killed", KR(ret)); } return ret; } diff --git a/src/sql/engine/cmd/ob_index_executor.cpp b/src/sql/engine/cmd/ob_index_executor.cpp index a95dc40ac..94ff6e028 100644 --- a/src/sql/engine/cmd/ob_index_executor.cpp +++ b/src/sql/engine/cmd/ob_index_executor.cpp @@ -234,17 +234,18 @@ int ObCreateIndexExecutor::sync_check_index_status(sql::ObSQLSessionInfo &my_ses LOG_WARN("failed to handle_session_exception", KR(ret)); } } - //处理主备库切换的场景，生效过程中发生切换的话，直接返回用户session_killed; //后续有备库来处理该索引； if (OB_FAIL(ret)) { } else if (OB_SYS_TENANT_ID == tenant_id) { //no need to process sys tenant - } else if (OB_FAIL(handle_switchover())) { - if (OB_SESSION_KILLED != ret) { - LOG_WARN("fail to handle switchover status", KR(ret)); - } else { - LOG_WARN("fail to add index while swithover", KR(ret)); + } else { + bool is_standby = false; + if (OB_FAIL(ObShareUtil::table_check_if_tenant_role_is_standby(tenant_id, is_standby))) { + LOG_WARN("fail to execute table_check_if_tenant_role_is_standby", KR(ret), K(tenant_id)); + } else if (is_standby) { + ret = OB_SESSION_KILLED; + LOG_WARN("create index while switchoverd, kill session", KR(ret)); } } @@ -267,16 +268,6 @@ int ObCreateIndexExecutor::handle_session_exception(ObSQLSessionInfo &session) return session.check_session_status(); } -int ObCreateIndexExecutor::handle_switchover() -{ - int ret = OB_SUCCESS; - if (GCTX.is_standby_cluster()) { - ret = OB_SESSION_KILLED; - LOG_INFO("create index while switchoverd, kill session", KR(ret)); - } - return ret; -} - ObDropIndexExecutor::ObDropIndexExecutor() { } diff --git a/src/sql/engine/cmd/ob_index_executor.h b/src/sql/engine/cmd/ob_index_executor.h index 6c19c46ff..d9c35c774 100644 --- a/src/sql/engine/cmd/ob_index_executor.h +++ b/src/sql/engine/cmd/ob_index_executor.h @@ -48,7 +48,6 @@ private: common::ObIAllocator &allocator, bool is_update_global_indexes = false); int handle_session_exception(ObSQLSessionInfo &session); - int handle_switchover(); }; class ObDropIndexStmt; diff --git a/src/sql/engine/cmd/ob_tenant_executor.cpp b/src/sql/engine/cmd/ob_tenant_executor.cpp index af884fed1..1dd9d4c55 100644 --- a/src/sql/engine/cmd/ob_tenant_executor.cpp +++ b/src/sql/engine/cmd/ob_tenant_executor.cpp @@ -24,7 +24,7 @@ #include "share/ls/ob_ls_operator.h" #include "share/ob_leader_election_waiter.h" #include "share/ls/ob_ls_status_operator.h" //ObLSStatusInfo, ObLSStatusOperator -#include "share/ob_primary_standby_service.h" // ObPrimaryStandbyService +#include "rootserver/standby/ob_standby_service.h" // ObStandbyService #include "sql/session/ob_sql_session_info.h" #include "sql/resolver/ddl/ob_create_tenant_stmt.h" #include "sql/resolver/ddl/ob_drop_tenant_stmt.h" @@ -250,7 +250,7 @@ int ObCreateStandbyTenantExecutor::execute(ObExecContext &ctx, ObCreateTenantStm } else if (OB_ISNULL(common_rpc_proxy = task_exec_ctx->get_common_rpc())) { ret = OB_NOT_INIT; LOG_WARN("get common rpc proxy failed"); - } else if (OB_FAIL(OB_PRIMARY_STANDBY_SERVICE.check_can_create_standby_tenant( + } else if (OB_FAIL(OB_STANDBY_SERVICE.check_can_create_standby_tenant( create_tenant_arg.log_restore_source_, compat_mode))) { LOG_WARN("check can create standby_tenant failed", KR(ret), K(create_tenant_arg)); } else { @@ -264,7 +264,7 @@ int ObCreateStandbyTenantExecutor::execute(ObExecContext &ctx, ObCreateTenantStm ret = OB_ERR_UNEXPECTED; LOG_WARN("if_not_exist not set and tenant_id invalid tenant_id", KR(ret), K(create_tenant_arg), K(tenant_id)); } else if (OB_INVALID_ID != tenant_id) { - if (OB_FAIL(OB_PRIMARY_STANDBY_SERVICE.wait_create_standby_tenant_end(tenant_id))) { + if (OB_FAIL(OB_STANDBY_SERVICE.wait_create_standby_tenant_end(tenant_id))) { LOG_WARN("failed to wait user create end", KR(ret), K(tenant_id)); } } diff --git a/src/sql/executor/ob_cmd_executor.cpp b/src/sql/executor/ob_cmd_executor.cpp index 317a6a9e5..293e34af9 100644 --- a/src/sql/executor/ob_cmd_executor.cpp +++ b/src/sql/executor/ob_cmd_executor.cpp @@ -1056,6 +1056,10 @@ int ObCmdExecutor::execute(ObExecContext &ctx, ObICmd &cmd) DEFINE_EXECUTE_CMD(ObCloneTenantStmt, ObCloneTenantExecutor); break; } + case stmt::T_SERVICE_NAME: { + DEFINE_EXECUTE_CMD(ObServiceNameStmt, ObServiceNameExecutor); + break; + } case stmt::T_ALTER_SYSTEM_RESET_PARAMETER: { DEFINE_EXECUTE_CMD(ObResetConfigStmt, ObResetConfigExecutor); break; diff --git a/src/sql/parser/sql_parser_mysql_mode.y b/src/sql/parser/sql_parser_mysql_mode.y index 62cac381f..fc476fe40 100644 --- a/src/sql/parser/sql_parser_mysql_mode.y +++ b/src/sql/parser/sql_parser_mysql_mode.y @@ -549,6 +549,7 @@ END_P SET_VAR DELIMITER %type transfer_partition_stmt transfer_partition_clause part_info cancel_transfer_partition_clause %type geometry_collection %type mock_stmt +%type service_name_stmt service_op %type ttl_definition ttl_expr ttl_unit %type id_dot_id id_dot_id_dot_id %type opt_table_list opt_repair_mode opt_repair_option_list repair_option repair_option_list opt_checksum_option @@ -727,6 +728,7 @@ stmt: | clone_tenant_stmt { $$ = $1; check_question_mark($$, result); } | transfer_partition_stmt { $$ = $1; check_question_mark($$, result); } | mock_stmt {$$ = $1; check_question_mark($$, result);} + | service_name_stmt { $$ = $1; check_question_mark($$, result); } ; /***************************************************************************** @@ -21158,11 +21160,47 @@ DAY dup_expr_string($$, result, @1.first_column, @1.last_column); } ; +/*=========================================================== + * + * 租户 SERVICE_NAME 管理 + * + *===========================================================*/ +service_name_stmt: +alter_with_opt_hint SYSTEM service_op SERVICE relation_name opt_tenant_name +{ + (void)($1); + malloc_non_terminal_node($$, result->malloc_pool_, T_SERVICE_NAME, 3, + $3, /* service operation */ + $5, /* service name */ + $6); /* tenant name */ +} +; +service_op : +CREATE +{ + malloc_terminal_node($$, result->malloc_pool_, T_INT); + $$->value_ = 1; +} +| DELETE +{ + malloc_terminal_node($$, result->malloc_pool_, T_INT); + $$->value_ = 2; +} +| START +{ + malloc_terminal_node($$, result->malloc_pool_, T_INT); + $$->value_ = 3; +} +| STOP +{ + malloc_terminal_node($$, result->malloc_pool_, T_INT); + $$->value_ = 4; +} +; /*=========================================================== * * JSON TABLE -* *============================================================*/ json_table_expr: diff --git a/src/sql/privilege_check/ob_privilege_check.cpp b/src/sql/privilege_check/ob_privilege_check.cpp index 62d7fe6f3..1284a6d51 100644 --- a/src/sql/privilege_check/ob_privilege_check.cpp +++ b/src/sql/privilege_check/ob_privilege_check.cpp @@ -2555,6 +2555,7 @@ int get_sys_tenant_alter_system_priv( stmt::T_TABLE_TTL != basic_stmt->get_stmt_type() && stmt::T_ALTER_SYSTEM_RESET_PARAMETER != basic_stmt->get_stmt_type() && stmt::T_TRANSFER_PARTITION != basic_stmt->get_stmt_type() && + stmt::T_SERVICE_NAME != basic_stmt->get_stmt_type() && stmt::T_ALTER_LS_REPLICA != basic_stmt->get_stmt_type()) { ret = OB_ERR_NO_PRIVILEGE; LOG_WARN("Only sys tenant can do this operation", diff --git a/src/sql/resolver/cmd/ob_alter_system_resolver.cpp b/src/sql/resolver/cmd/ob_alter_system_resolver.cpp index 503ca673c..ed6b45b18 100644 --- a/src/sql/resolver/cmd/ob_alter_system_resolver.cpp +++ b/src/sql/resolver/cmd/ob_alter_system_resolver.cpp @@ -2072,7 +2072,10 @@ static int set_reset_check_param_valid_oracle_mode(uint64_t tenant_id , ret = OB_OBJECT_NAME_NOT_EXIST; LOG_WARN("fail to get keystore schema", K(ret)); } else if (0 != keystore_schema->get_master_key_id()) { - if (!GCTX.is_standby_cluster()) { + bool is_standby = false; + if (OB_FAIL(ObShareUtil::table_check_if_tenant_role_is_standby(tenant_id, is_standby))) { + LOG_WARN("fail to execute table_check_if_tenant_role_is_standby", KR(ret), K(tenant_id)); + } else if (!is_standby) { ret = OB_NOT_SUPPORTED; LOG_WARN("alter tde method is not support", K(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter tde method with master key exists"); @@ -2470,7 +2473,10 @@ int ObSetConfigResolver::check_param_valid(int64_t tenant_id , ret = OB_OBJECT_NAME_NOT_EXIST; LOG_WARN("fail to get keystore schema", K(ret)); } else if (0 != keystore_schema->get_master_key_id()) { - if (!GCTX.is_standby_cluster()) { + bool is_standby = false; + if (OB_FAIL(ObShareUtil::table_check_if_tenant_role_is_standby(tenant_id, is_standby))) { + LOG_WARN("fail to execute table_check_if_tenant_role_is_standby", KR(ret), K(tenant_id)); + } else if (!is_standby) { ret = OB_NOT_SUPPORTED; LOG_WARN("alter tde method is not support", K(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter tde method with master key exists"); @@ -7045,5 +7051,68 @@ int ret = OB_SUCCESS; } return ret; } + +int ObServiceNameResolver::resolve(const ParseNode &parse_tree) +{ + int ret = OB_SUCCESS; + ObServiceNameStmt *stmt = create_stmt(); + uint64_t target_tenant_id = OB_INVALID_TENANT_ID; + ObString service_name_str; + if (OB_UNLIKELY(T_SERVICE_NAME != parse_tree.type_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid parse node, type is not T_SERVICE_NAME", KR(ret), "type", + get_type_name(parse_tree.type_)); + } else if (OB_ISNULL(stmt)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("create stmt fail", KR(ret)); + } else if (3 != parse_tree.num_child_ + || OB_ISNULL(parse_tree.children_[0]) + || OB_ISNULL(parse_tree.children_[1]) + || OB_ISNULL(session_info_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid parse tree or session info", KR(ret), "num_child", parse_tree.num_child_, + KP(parse_tree.children_[0]), KP(parse_tree.children_[1]), KP(session_info_)); + } else if (OB_FAIL(get_and_verify_tenant_name( + parse_tree.children_[2], + session_info_->get_effective_tenant_id(), + target_tenant_id, + "Service name related command"))) { + LOG_WARN("fail to execute get_and_verify_tenant_name", KR(ret), + K(session_info_->get_effective_tenant_id()), KP(parse_tree.children_[1])); + } else if (OB_UNLIKELY(T_INT != parse_tree.children_[0]->type_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid parse node, service_op is not T_INT", K(parse_tree.children_[0]->type_)); + } else if (OB_FAIL(Util::resolve_relation_name(parse_tree.children_[1], service_name_str))) { + LOG_WARN("fail to resolve service_name_str", KR(ret)); + } else { + ObServiceNameArg::ObServiceOp service_op = + static_cast(parse_tree.children_[0]->value_); + if (OB_FAIL(stmt->get_arg().init(service_op, target_tenant_id, service_name_str))) { + LOG_WARN("fail to init ObServiceNameArg", KR(ret), K(service_op), K(target_tenant_id), K(service_name_str)); + } + } + if (OB_SUCC(ret)) { + stmt_ = stmt; + } + if (OB_SUCC(ret) && ObSchemaChecker::is_ora_priv_check()) { + if (OB_ISNULL(schema_checker_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret)); + } else if (OB_FAIL(schema_checker_->check_ora_ddl_priv( + session_info_->get_effective_tenant_id(), + session_info_->get_priv_user_id(), + ObString(""), + // why use T_ALTER_SYSTEM_SET_PARAMETER? + // because T_ALTER_SYSTEM_SET_PARAMETER has following traits: + // T_ALTER_SYSTEM_SET_PARAMETER can allow dba to do an operation + // and prohibit other user to do this operation + // so we reuse this. + stmt::T_ALTER_SYSTEM_SET_PARAMETER, + session_info_->get_enable_role_array()))) { + LOG_WARN("failed to check privilege", K(session_info_->get_effective_tenant_id()), K(session_info_->get_user_id())); + } + } + return ret; +} } // end namespace sql } // end namespace oceanbase diff --git a/src/sql/resolver/cmd/ob_alter_system_resolver.h b/src/sql/resolver/cmd/ob_alter_system_resolver.h index 839a2b62d..796cf9b81 100644 --- a/src/sql/resolver/cmd/ob_alter_system_resolver.h +++ b/src/sql/resolver/cmd/ob_alter_system_resolver.h @@ -185,8 +185,7 @@ DEF_SIMPLE_CMD_RESOLVER(ObBackupSetDecryptionResolver); DEF_SIMPLE_CMD_RESOLVER(ObAddRestoreSourceResolver); DEF_SIMPLE_CMD_RESOLVER(ObClearRestoreSourceResolver); DEF_SIMPLE_CMD_RESOLVER(ObCheckpointSlogResolver); - - +DEF_SIMPLE_CMD_RESOLVER(ObServiceNameResolver); int resolve_restore_until(const ParseNode &time_node, const ObSQLSessionInfo *session_info, share::SCN &recovery_until_scn, diff --git a/src/sql/resolver/cmd/ob_alter_system_stmt.h b/src/sql/resolver/cmd/ob_alter_system_stmt.h index dd0a4add5..b927359e1 100644 --- a/src/sql/resolver/cmd/ob_alter_system_stmt.h +++ b/src/sql/resolver/cmd/ob_alter_system_stmt.h @@ -18,6 +18,7 @@ #include "share/scheduler/ob_sys_task_stat.h" #include "share/backup/ob_backup_clean_struct.h" #include "rootserver/ob_transfer_partition_command.h" +#include "share/ob_service_name_proxy.h" namespace oceanbase { @@ -1390,6 +1391,16 @@ private: rootserver::ObTransferPartitionArg arg_; }; +class ObServiceNameStmt : public ObSystemCmdStmt +{ +public: + ObServiceNameStmt() : ObSystemCmdStmt(stmt::T_SERVICE_NAME), arg_() {} + virtual ~ObServiceNameStmt() {} + share::ObServiceNameArg &get_arg() { return arg_; } +private: + share::ObServiceNameArg arg_; +}; + } // end namespace sql } // end namespace oceanbase diff --git a/src/sql/resolver/ob_resolver.cpp b/src/sql/resolver/ob_resolver.cpp index 7eb90e741..8239f1b07 100644 --- a/src/sql/resolver/ob_resolver.cpp +++ b/src/sql/resolver/ob_resolver.cpp @@ -1237,6 +1237,10 @@ int ObResolver::resolve(IsPrepared if_prepared, const ParseNode &parse_tree, ObS REGISTER_STMT_RESOLVER(TransferPartition); break; } + case T_SERVICE_NAME: { + REGISTER_STMT_RESOLVER(ServiceName); + break; + } case T_REPAIR_TABLE: { REGISTER_STMT_RESOLVER(Mock); break; diff --git a/src/sql/session/ob_sql_session_info.cpp b/src/sql/session/ob_sql_session_info.cpp index 3289708c1..a46d1d3ad 100644 --- a/src/sql/session/ob_sql_session_info.cpp +++ b/src/sql/session/ob_sql_session_info.cpp @@ -59,6 +59,7 @@ #include "ob_sess_info_verify.h" #include "share/schema/ob_schema_utils.h" #include "share/config/ob_config_helper.h" +#include "rootserver/ob_tenant_info_loader.h" using namespace oceanbase::sql; using namespace oceanbase::common; @@ -207,7 +208,9 @@ ObSQLSessionInfo::ObSQLSessionInfo(const uint64_t tenant_id) : current_dblink_sequence_id_(0), client_non_standard_(false), is_session_sync_support_(false), - job_info_(nullptr) + job_info_(nullptr), + failover_mode_(false), + service_name_() { MEMSET(tenant_buff_, 0, sizeof(share::ObTenantSpaceFetcher)); MEMSET(vip_buf_, 0, sizeof(vip_buf_)); @@ -405,6 +408,8 @@ void ObSQLSessionInfo::reset(bool skip_sys_var) need_send_feedback_proxy_info_ = false; is_lock_session_ = false; job_info_ = nullptr; + failover_mode_ = false; + service_name_.reset(); } void ObSQLSessionInfo::clean_status() @@ -986,32 +991,6 @@ int ObSQLSessionInfo::drop_temp_tables(const bool is_disconn, return ret; } -//清理oracle临时表中数据来源和当前session id相同, 但属于被重用的旧的session数据 -int ObSQLSessionInfo::drop_reused_oracle_temp_tables() -{ - int ret = OB_SUCCESS; - //obrpc::ObCommonRpcProxy *common_rpc_proxy = NULL; - if (false == get_is_deserialized() - && !is_inner() - && !GCTX.is_standby_cluster()) { - obrpc::ObDropTableArg drop_table_arg; - drop_table_arg.if_exist_ = true; - drop_table_arg.to_recyclebin_ = false; - drop_table_arg.table_type_ = share::schema::TMP_TABLE_ORA_SESS; - drop_table_arg.session_id_ = get_sessid_for_table(); - drop_table_arg.tenant_id_ = get_effective_tenant_id(); - drop_table_arg.sess_create_time_ = get_sess_create_time(); - //common_rpc_proxy = GCTX.rs_rpc_proxy_; - if (OB_FAIL(delete_from_oracle_temp_tables(drop_table_arg))) { - //if (OB_FAIL(common_rpc_proxy->drop_table(drop_table_arg))) { - LOG_WARN("failed to drop reused temporary table", K(drop_table_arg), K(ret)); - } else { - LOG_DEBUG("succeed to delete old rows for oracle temporary table", K(drop_table_arg)); - } - } - return ret; -} - //proxy方式下session创建、断开和后台定时task检查: //如果距离上次更新此session->last_refresh_temp_table_time_ 超过1hr //则更新session创建的临时表最后活动时间SESSION_ACTIVE_TIME @@ -4721,3 +4700,51 @@ int ObSQLSessionInfo::get_dblink_sequence_schema(int64_t sequence_id, const ObSe LOG_TRACE("get dblink sequence schema", K(sequence_id), K(dblink_sequence_schemas_)); return ret; } + +int ObSQLSessionInfo::set_service_name(const ObString& service_name) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(service_name_.init(service_name))) { + LOG_WARN("fail to init service_name", KR(ret), K(service_name)); + } + return ret; +} +int ObSQLSessionInfo::check_service_name_and_failover_mode(const uint64_t tenant_id) const +{ + // if failover_mode is on, and the session is created via service_name + // the tenant should be primary + // service name must exist and service status must be started + // if service_name is not empty, the version must be >= 4240 + int ret = OB_SUCCESS; + bool is_sts_ready = false; + if (service_name_.is_empty()) { + // do nothing + } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id)); + } else { + MTL_SWITCH(tenant_id) { + rootserver::ObTenantInfoLoader *tenant_info_loader = MTL(rootserver::ObTenantInfoLoader*); + if (OB_ISNULL(tenant_info_loader)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant_info_loader is null", KR(ret), KP(tenant_info_loader)); + } else if (OB_FAIL(tenant_info_loader->check_if_sts_is_ready(is_sts_ready))) { + LOG_WARN("fail to execute check_if_sts_is_ready", KR(ret)); + } else if (failover_mode_ && is_sts_ready) { + // 'sts_ready' indicates that the 'access_mode' is 'RAW_WRITE' + // The reason for using 'sts_ready' is that we believe all connections intending to reach the + // primary tenant should be accepted before the 'access_mode' switches to 'RAW_WRITE'. + ret = OB_NOT_PRIMARY_TENANT; + LOG_WARN("the tenant is not primary, the request is not allowed", KR(ret), K(is_sts_ready)); + } else if (OB_FAIL(tenant_info_loader->check_if_the_service_name_is_stopped(service_name_))) { + LOG_WARN("fail to execute check_if_the_service_name_is_stopped", KR(ret), K(service_name_)); + } + } + } + return ret; +} +int ObSQLSessionInfo::check_service_name_and_failover_mode() const +{ + uint64_t tenant_id = get_effective_tenant_id(); + return check_service_name_and_failover_mode(tenant_id); +} diff --git a/src/sql/session/ob_sql_session_info.h b/src/sql/session/ob_sql_session_info.h index 2ec293897..5f89af9b3 100644 --- a/src/sql/session/ob_sql_session_info.h +++ b/src/sql/session/ob_sql_session_info.h @@ -46,6 +46,7 @@ #include "sql/ob_optimizer_trace_impl.h" #include "sql/monitor/flt/ob_flt_span_mgr.h" #include "storage/tx/ob_tx_free_route.h" +#include "share/ob_service_name_proxy.h" #include "observer/dbms_scheduler/ob_dbms_sched_job_utils.h" namespace oceanbase @@ -84,6 +85,7 @@ namespace share { struct ObSequenceValue; } +using share::ObServiceNameString; using common::ObPsStmtId; namespace sql { @@ -880,7 +882,6 @@ public: const bool is_xa_trans = false, const bool is_reset_connection = false); void refresh_temp_tables_sess_active_time(); //更新临时表的sess active time - int drop_reused_oracle_temp_tables(); int delete_from_oracle_temp_tables(const obrpc::ObDropTableArg &const_drop_table_arg); //To generate an unique key for Oracle Global Temporary Table @@ -1451,6 +1452,13 @@ public: bool is_lock_session() const { return is_lock_session_; } int64_t get_plsql_exec_time(); void update_pure_sql_exec_time(int64_t elapsed_time); + const ObServiceNameString& get_service_name() const { return service_name_; } + bool get_failover_mode() const { return failover_mode_; } + void set_failover_mode(const bool failover_mode) { failover_mode_ = failover_mode; } + void reset_service_name() { service_name_.reset(); } + int set_service_name(const ObString& service_name); + int check_service_name_and_failover_mode() const; + int check_service_name_and_failover_mode(const uint64_t tenant_id) const; public: bool has_tx_level_temp_table() const { return tx_desc_ && tx_desc_->with_temporary_table(); } void set_affected_rows_is_changed(int64_t affected_rows); @@ -1720,8 +1728,11 @@ private: bool is_session_sync_support_; // session_sync_support flag. share::schema::ObUserLoginInfo login_info_; dbms_scheduler::ObDBMSSchedJobInfo *job_info_; // dbms_scheduler related. + bool failover_mode_; + ObServiceNameString service_name_; }; + inline bool ObSQLSessionInfo::is_terminate(int &ret) const { bool bret = false; diff --git a/src/storage/concurrency_control/ob_multi_version_garbage_collector.cpp b/src/storage/concurrency_control/ob_multi_version_garbage_collector.cpp index 786e61a67..6fb5e7b54 100644 --- a/src/storage/concurrency_control/ob_multi_version_garbage_collector.cpp +++ b/src/storage/concurrency_control/ob_multi_version_garbage_collector.cpp @@ -331,18 +331,19 @@ int ObMultiVersionGarbageCollector::study() timeguard.click("study_min_unallocated_GTS"); if (OB_SUCC(ret)) { - if (!GCTX.is_standby_cluster() && // standby cluster does not support WRS - OB_FAIL(study_min_unallocated_WRS(min_unallocated_WRS))) { - MVCC_LOG(WARN, "study min unallocated GTS failed", K(ret)); - } else if (!min_unallocated_WRS.is_valid() - || min_unallocated_WRS.is_min() - || min_unallocated_WRS.is_max()) { + bool is_primary = true; + const uint64_t tenant_id = MTL_ID(); + if (OB_FAIL(ObShareUtil::mtl_check_if_tenant_role_is_primary(tenant_id, is_primary))) { + MVCC_LOG(WARN, "fail to execute mtl_check_if_tenant_role_is_primary", KR(ret), K(tenant_id)); + } else if (is_primary && OB_FAIL(study_min_unallocated_WRS(min_unallocated_WRS))) { + MVCC_LOG(WARN, "study min unallocated GTS failed", K(ret), K(is_primary)); + } else if (!min_unallocated_WRS.is_valid() || min_unallocated_WRS.is_min()) { ret = OB_ERR_UNEXPECTED; MVCC_LOG(ERROR, "wrong min unallocated WRS", - K(ret), K(min_unallocated_WRS), KPC(this)); + K(ret), K(min_unallocated_WRS), KPC(this), K(is_primary)); } else { MVCC_LOG(INFO, "study min unallocated wrs succeed", - K(ret), K(min_unallocated_WRS), KPC(this)); + K(ret), K(min_unallocated_WRS), KPC(this), K(is_primary)); } } diff --git a/src/storage/ls/ob_ls.cpp b/src/storage/ls/ob_ls.cpp index 05f9460c2..9fddf641f 100755 --- a/src/storage/ls/ob_ls.cpp +++ b/src/storage/ls/ob_ls.cpp @@ -38,7 +38,7 @@ #include "rootserver/ob_create_standby_from_net_actor.h" #include "rootserver/ob_heartbeat_service.h" #include "rootserver/ob_primary_ls_service.h" -#include "rootserver/ob_recovery_ls_service.h" +#include "rootserver/standby/ob_recovery_ls_service.h" #include "rootserver/ob_tenant_transfer_service.h" // ObTenantTransferService #include "rootserver/ob_tenant_balance_service.h" #include "rootserver/restore/ob_restore_service.h" diff --git a/src/storage/memtable/ob_memtable.cpp b/src/storage/memtable/ob_memtable.cpp index 649282dcf..a11fd6fd7 100644 --- a/src/storage/memtable/ob_memtable.cpp +++ b/src/storage/memtable/ob_memtable.cpp @@ -1236,10 +1236,6 @@ int ObMemtable::replay_row(ObStoreCtx &ctx, TRANS_LOG(WARN, "get next row error", K(ret)); } } else if (FALSE_IT(timeguard.click("mutator_row copy"))) { - } else if (OB_FAIL(check_standby_cluster_schema_condition_(ctx, table_id, table_version))) { - TRANS_LOG(WARN, "failed to check standby_cluster_schema_condition", K(ret), K(table_id), - K(table_version)); - } else if (FALSE_IT(timeguard.click("check_standby_cluster_schema_condition"))) { } else if (OB_UNLIKELY(dml_flag == blocksstable::ObDmlFlag::DF_NOT_EXIST)) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "Unexpected not exist trans node", K(ret), K(dml_flag), K(rowkey)); @@ -2441,44 +2437,6 @@ void ObMemtable::set_minor_merged() minor_merged_time_ = ObTimeUtility::current_time(); } -int ObMemtable::check_standby_cluster_schema_condition_(ObStoreCtx &ctx, - const int64_t table_id, - const int64_t table_version) -{ - int ret = OB_SUCCESS; -#ifdef ERRSIM - ret = OB_E(EventTable::EN_CHECK_STANDBY_CLUSTER_SCHEMA_CONDITION) OB_SUCCESS; - if (OB_FAIL(ret) && !common::is_inner_table(table_id)) { - TRANS_LOG(WARN, "ERRSIM, replay row failed", K(ret)); - return ret; - } -#endif - if (GCTX.is_standby_cluster()) { - //only stand_by cluster need to be check - uint64_t tenant_id = MTL_ID(); - if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "invalid tenant_id", K(ret), K(tenant_id), K(table_id), K(table_version)); - } else if (OB_SYS_TENANT_ID == tenant_id) { - //sys tenant do not need check - } else { - int64_t tenant_schema_version = 0; - if (OB_FAIL(GSCHEMASERVICE.get_tenant_refreshed_schema_version(tenant_id, tenant_schema_version))) { - TRANS_LOG(WARN, "get_tenant_schema_version failed", K(ret), K(tenant_id), - K(table_id), K(tenant_id), K(table_version)); - if (OB_ENTRY_NOT_EXIST == ret) { - // tenant schema hasn't been flushed in the case of restart, rewrite OB_ENTRY_NOT_EXIST - ret = OB_TRANS_WAIT_SCHEMA_REFRESH; - } - } else if (table_version > tenant_schema_version) { - // replay is not allowed when data's table version is greater than tenant's schema version - //remove by msy164651, in 4.0 no need to check schema version - } else {/*do nothing*/} - } - } - return ret; -} - int64_t ObMemtable::get_upper_trans_version() const { return INT64_MAX; diff --git a/src/storage/memtable/ob_memtable.h b/src/storage/memtable/ob_memtable.h index 5a02ae1a9..dd05d9922 100644 --- a/src/storage/memtable/ob_memtable.h +++ b/src/storage/memtable/ob_memtable.h @@ -541,9 +541,6 @@ private: void set_begin(ObMvccAccessCtx &ctx); void set_end(ObMvccAccessCtx &ctx, int ret); - int check_standby_cluster_schema_condition_(storage::ObStoreCtx &ctx, - const int64_t table_id, - const int64_t table_version); int set_( const storage::ObTableIterParam ¶m, const common::ObIArray &columns, diff --git a/src/storage/tablelock/ob_table_lock_service.cpp b/src/storage/tablelock/ob_table_lock_service.cpp index bdd5c44aa..05ba584be 100644 --- a/src/storage/tablelock/ob_table_lock_service.cpp +++ b/src/storage/tablelock/ob_table_lock_service.cpp @@ -2566,20 +2566,14 @@ int ObTableLockService::check_op_allowed_(const uint64_t table_id, // all the tmp table is a normal table now, deal it as a normal user table // table lock not support virtual table/sys table(not in white list) etc. is_allowed = false; - } else if (GCTX.is_standby_cluster() && OB_SYS_TENANT_ID != tenant_id) { - is_allowed = false; - } else if (!GCTX.is_standby_cluster()) { - bool is_restore = false; - ObMultiVersionSchemaService *schema_service = MTL(ObTenantSchemaService*)->get_schema_service(); - if (OB_FAIL(schema_service->check_tenant_is_restore(NULL, - tenant_id, - is_restore))) { - LOG_WARN("failed to check tenant restore", K(ret), K(table_id)); - } else if (is_restore) { + } else { + bool is_primary = true; + if (OB_FAIL(ObShareUtil::mtl_check_if_tenant_role_is_primary(tenant_id, is_primary))) { + LOG_WARN("fail to execute mtl_check_if_tenant_role_is_primary", KR(ret), K(tenant_id)); + } else if (!is_primary) { is_allowed = false; } } - return ret; } diff --git a/tools/deploy/mysql_test/r/mysql/information_schema.result b/tools/deploy/mysql_test/r/mysql/information_schema.result index 41bfa5e16..5af54a352 100644 --- a/tools/deploy/mysql_test/r/mysql/information_schema.result +++ b/tools/deploy/mysql_test/r/mysql/information_schema.result @@ -331,6 +331,7 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | DBA_OB_RESTORE_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_RSRC_IO_DIRECTIVES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_SEQUENCE_OBJECTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_SERVICES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_SYS_VARIABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_TABLEGROUPS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_TABLEGROUP_PARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | @@ -848,6 +849,7 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | __all_virtual_server_compaction_event_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_server_compaction_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_server_schema_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_service | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_session_event | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_session_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_session_ps_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | @@ -1841,6 +1843,7 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | DBA_OB_RESTORE_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_RSRC_IO_DIRECTIVES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_SEQUENCE_OBJECTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_SERVICES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_SYS_VARIABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_TABLEGROUPS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_TABLEGROUP_PARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | @@ -2358,6 +2361,7 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | __all_virtual_server_compaction_event_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_server_compaction_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_server_schema_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_service | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_session_event | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_session_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_session_ps_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | @@ -3122,6 +3126,8 @@ select * from information_schema.statistics where table_schema in ('oceanbase', | def | oceanbase | __all_server_event_history | 0 | oceanbase | PRIMARY | 3 | svr_port | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_server_event_history | 1 | oceanbase | idx_server_event | 1 | event | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_server_event_history | 1 | oceanbase | idx_server_module | 1 | module | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_service | 0 | oceanbase | PRIMARY | 1 | tenant_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_service | 0 | oceanbase | PRIMARY | 2 | service_name_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_service_epoch | 0 | oceanbase | PRIMARY | 1 | tenant_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_service_epoch | 0 | oceanbase | PRIMARY | 2 | name | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_space_usage | 0 | oceanbase | PRIMARY | 1 | tenant_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result index b8d7f829d..3ca3c318e 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result @@ -2657,8 +2657,9 @@ LB_VPORT bigint(20) YES NULL IN_BYTES bigint(20) NO NULL OUT_BYTES bigint(20) NO NULL USER_CLIENT_PORT bigint(20) NO -TOTAL_CPU_TIME bigint(21) NO NULL PROXY_USER varchar(128) YES NULL +SERVICE_NAME varchar(64) YES NULL +TOTAL_CPU_TIME bigint(21) NO NULL TOP_INFO varchar(262143) YES NULL select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.GV$OB_PROCESSLIST limit 1); cnt @@ -2702,8 +2703,9 @@ LB_VPORT bigint(20) NO IN_BYTES bigint(20) NO OUT_BYTES bigint(20) NO USER_CLIENT_PORT bigint(20) NO -TOTAL_CPU_TIME bigint(21) NO PROXY_USER varchar(128) NO +SERVICE_NAME varchar(64) NO +TOTAL_CPU_TIME bigint(21) NO TOP_INFO varchar(262143) NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_PROCESSLIST limit 1); cnt @@ -6284,6 +6286,16 @@ ENABLE_VERSIONS longtext NO NULL select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_COMPATIBILITY_CONTROL limit 1); cnt 1 +desc oceanbase.DBA_OB_SERVICES; +Field Type Null Key Default Extra +CREATE_TIME timestamp(6) NO NULL +MODIFIED_TIME timestamp(6) NO NULL +SERVICE_NAME_ID bigint(20) NO NULL +SERVICE_NAME varchar(64) NO NULL +SERVICE_STATUS varchar(64) NO NULL +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_OB_SERVICES limit 1); +cnt +1 desc oceanbase.GV$OB_TENANT_RESOURCE_LIMIT; Field Type Null Key Default Extra SVR_IP varchar(46) NO NULL diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result index c6f848b4d..320c60d4f 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result @@ -3935,8 +3935,9 @@ LB_VPORT bigint(20) YES NULL IN_BYTES bigint(20) NO NULL OUT_BYTES bigint(20) NO NULL USER_CLIENT_PORT bigint(20) NO -TOTAL_CPU_TIME bigint(21) NO NULL PROXY_USER varchar(128) YES NULL +SERVICE_NAME varchar(64) YES NULL +TOTAL_CPU_TIME bigint(21) NO NULL TOP_INFO varchar(262143) YES NULL select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.GV$OB_PROCESSLIST limit 1); cnt @@ -3980,8 +3981,9 @@ LB_VPORT bigint(20) NO IN_BYTES bigint(20) NO OUT_BYTES bigint(20) NO USER_CLIENT_PORT bigint(20) NO -TOTAL_CPU_TIME bigint(21) NO PROXY_USER varchar(128) NO +SERVICE_NAME varchar(64) NO +TOTAL_CPU_TIME bigint(21) NO TOP_INFO varchar(262143) NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_PROCESSLIST limit 1); cnt @@ -9080,6 +9082,27 @@ ENABLE_VERSIONS longtext NO NULL select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_COMPATIBILITY_CONTROL limit 1); cnt 1 +desc oceanbase.DBA_OB_SERVICES; +Field Type Null Key Default Extra +CREATE_TIME timestamp(6) NO NULL +MODIFIED_TIME timestamp(6) NO NULL +SERVICE_NAME_ID bigint(20) NO NULL +SERVICE_NAME varchar(64) NO NULL +SERVICE_STATUS varchar(64) NO NULL +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_OB_SERVICES limit 1); +cnt +1 +desc oceanbase.CDB_OB_SERVICES; +Field Type Null Key Default Extra +TENANT_ID bigint(20) NO NULL +CREATE_TIME timestamp(6) NO NULL +MODIFIED_TIME timestamp(6) NO NULL +SERVICE_NAME_ID bigint(20) NO NULL +SERVICE_NAME varchar(64) NO NULL +SERVICE_STATUS varchar(64) NO NULL +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.CDB_OB_SERVICES limit 1); +cnt +1 desc oceanbase.GV$OB_TENANT_RESOURCE_LIMIT; Field Type Null Key Default Extra SVR_IP varchar(46) NO NULL diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result index dce795d9e..354f437c9 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result @@ -4984,6 +4984,17 @@ enable_versions longtext NO NULL select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_compatibility_control; IF(count(*) >= 0, 1, 0) 1 +desc oceanbase.__all_virtual_service; +Field Type Null Key Default Extra +tenant_id bigint(20) NO PRI NULL +service_name_id bigint(20) NO PRI NULL +gmt_create timestamp(6) NO NULL +gmt_modified timestamp(6) NO NULL +service_name varchar(64) NO NULL +service_status varchar(64) NO NULL +select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_service; +IF(count(*) >= 0, 1, 0) +1 desc oceanbase.__all_virtual_tenant_resource_limit; Field Type Null Key Default Extra svr_ip varchar(46) NO NULL diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result index b65613a55..3b381fcf2 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result @@ -9616,6 +9616,17 @@ is_deleted bigint(20) NO NULL select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_user_proxy_role_info_history; IF(count(*) >= 0, 1, 0) 1 +desc oceanbase.__all_virtual_service; +Field Type Null Key Default Extra +tenant_id bigint(20) NO PRI NULL +service_name_id bigint(20) NO PRI NULL +gmt_create timestamp(6) NO NULL +gmt_modified timestamp(6) NO NULL +service_name varchar(64) NO NULL +service_status varchar(64) NO NULL +select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_service; +IF(count(*) >= 0, 1, 0) +1 desc oceanbase.__all_virtual_tenant_resource_limit; Field Type Null Key Default Extra svr_ip varchar(46) NO NULL diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result index be1f568b1..57c7f8ef6 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result @@ -298,6 +298,7 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr 513 __all_user_proxy_info_history 0 201001 1 514 __all_user_proxy_role_info 0 201001 1 515 __all_user_proxy_role_info_history 0 201001 1 +516 __all_service 0 201001 1 518 __all_mview_dep 0 201001 1 519 __all_scheduler_job_run_detail_v2 0 201001 1 10001 __tenant_virtual_all_table 2 201001 1 @@ -751,6 +752,7 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr 12475 __all_virtual_user_proxy_info_history 2 201001 1 12476 __all_virtual_user_proxy_role_info 2 201001 1 12477 __all_virtual_user_proxy_role_info_history 2 201001 1 +12480 __all_virtual_service 2 201001 1 12481 __all_virtual_tenant_resource_limit 2 201001 1 12482 __all_virtual_tenant_resource_limit_detail 2 201001 1 12487 __all_virtual_nic_info 2 201001 1 @@ -1175,6 +1177,8 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr 21543 GV$OB_TRACEPOINT_INFO 1 201001 1 21544 V$OB_TRACEPOINT_INFO 1 201001 1 21545 V$OB_COMPATIBILITY_CONTROL 1 201001 1 +21548 DBA_OB_SERVICES 1 201001 1 +21549 CDB_OB_SERVICES 1 201001 1 21550 GV$OB_TENANT_RESOURCE_LIMIT 1 201001 1 21551 V$OB_TENANT_RESOURCE_LIMIT 1 201001 1 21552 GV$OB_TENANT_RESOURCE_LIMIT_DETAIL 1 201001 1 diff --git a/unittest/share/schema/mock_schema_service.h b/unittest/share/schema/mock_schema_service.h index 9a157ca4c..d9de903b3 100644 --- a/unittest/share/schema/mock_schema_service.h +++ b/unittest/share/schema/mock_schema_service.h @@ -110,10 +110,9 @@ public: int ret = OB_SUCCESS; UNUSED(force_fallback); - bool is_standby_cluster = false; if (OB_FAIL(guard.reset())) { SHARE_SCHEMA_LOG(WARN, "fail to reset guard", K(ret)); - } else if (OB_FAIL(guard.init(is_standby_cluster))) { + } else if (OB_FAIL(guard.init())) { SHARE_SCHEMA_LOG(WARN, "fail to init guard", K(ret)); } else { ObArray tenant_schemas; From 139fdda7b7bc7b45b91b03c993cc3a3be409440e Mon Sep 17 00:00:00 2001 From: leftgeek <1094669802@qq.com> Date: Thu, 22 Aug 2024 16:25:10 +0000 Subject: [PATCH 179/249] [FEAT MERGE]Support column store in creating materialized view Co-authored-by: medcll <527998250@qq.com> Co-authored-by: coolfishchen --- .../virtual_table/ob_show_create_table.cpp | 14 +- src/share/schema/ob_schema_printer.cpp | 226 ++++++++++++++++-- src/share/schema/ob_schema_printer.h | 9 + src/sql/parser/sql_parser_mysql_mode.y | 37 ++- .../resolver/ddl/ob_create_table_resolver.cpp | 95 -------- .../resolver/ddl/ob_create_table_resolver.h | 5 - .../ddl/ob_create_table_resolver_base.cpp | 105 +++++++- .../ddl/ob_create_table_resolver_base.h | 6 + .../resolver/ddl/ob_create_view_resolver.cpp | 29 ++- .../resolver/ddl/ob_create_view_resolver.h | 3 +- 10 files changed, 398 insertions(+), 131 deletions(-) diff --git a/src/observer/virtual_table/ob_show_create_table.cpp b/src/observer/virtual_table/ob_show_create_table.cpp index 462b126ec..e03fd03b5 100644 --- a/src/observer/virtual_table/ob_show_create_table.cpp +++ b/src/observer/virtual_table/ob_show_create_table.cpp @@ -216,7 +216,19 @@ int ObShowCreateTable::fill_row_cells_inner(const uint64_t show_table_id, // create_table ObSchemaPrinter schema_printer(*schema_guard_, strict_mode, sql_quote_show_create, ansi_quotes); int64_t pos = 0; - if (table_schema.is_view_table()) { + if (table_schema.is_materialized_view()) { + if (OB_FAIL(schema_printer.print_materialized_view_definition(effective_tenant_id_, + show_table_id, + table_def_buf, + table_def_buf_size, + pos, + TZ_INFO(session_), + false, + session_->get_sql_mode()))) { + SERVER_LOG(WARN, "Generate materialized view definition failed", + KR(ret), K(effective_tenant_id_), K(show_table_id)); + } + } else if (table_schema.is_view_table()) { if (OB_FAIL(schema_printer.print_view_definiton(effective_tenant_id_, show_table_id, table_def_buf, diff --git a/src/share/schema/ob_schema_printer.cpp b/src/share/schema/ob_schema_printer.cpp index 4af478c9d..e9c338ecd 100644 --- a/src/share/schema/ob_schema_printer.cpp +++ b/src/share/schema/ob_schema_printer.cpp @@ -37,6 +37,7 @@ #include "pl/ob_pl_resolver.h" #include "sql/ob_sql_utils.h" #include "share/ob_get_compat_mode.h" +#include "share/schema/ob_mview_info.h" @@ -2832,27 +2833,11 @@ int ObSchemaPrinter::print_view_definiton( SHARE_SCHEMA_LOG(WARN, "Unknow table", K(ret), K(table_id)); } else if (OB_FAIL(table_schema->check_if_oracle_compat_mode(is_oracle_mode))) { LOG_WARN("fail to check oracle mode", KR(ret), KPC(table_schema)); - } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "CREATE %s%sVIEW ", - is_oracle_mode && table_schema->is_view_created_by_or_replace_force() ? "OR REPLACE FORCE " : "", - table_schema->is_materialized_view() ? "MATERIALIZED " : ""))) { + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "CREATE %sVIEW ", + is_oracle_mode && table_schema->is_view_created_by_or_replace_force() ? "OR REPLACE FORCE " : ""))) { SHARE_SCHEMA_LOG(WARN, "fail to print view definition", K(ret)); } else if (OB_FAIL(print_identifier(buf, buf_len, pos, table_schema->get_table_name(), is_oracle_mode))) { SHARE_SCHEMA_LOG(WARN, "fail to print view definition", K(ret)); - } else if (table_schema->is_materialized_view()) { - const ObTableSchema *container_table_schema = nullptr; - - if (OB_FAIL(schema_guard_.get_table_schema(tenant_id, table_schema->get_data_table_id(), container_table_schema))) { - LOG_WARN("fail to get container_table_schema", KR(ret)); - } else if (NULL == container_table_schema) { - ret = OB_TABLE_NOT_EXIST; - SHARE_SCHEMA_LOG(WARN, "Unknow container table", K(ret), K(table_schema->get_data_table_id())); - } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, " "))) { - SHARE_SCHEMA_LOG(WARN, "fail to print space", K(ret)); - } else if (OB_FAIL(print_table_definition_table_options(*container_table_schema, buf, buf_len, pos, false, agent_mode, sql_mode))) { - SHARE_SCHEMA_LOG(WARN, "fail to print table options", K(ret), K(*container_table_schema)); - } else if (OB_FAIL(print_table_definition_partition_options(*container_table_schema, buf, buf_len, pos, agent_mode, tz_info))) { - SHARE_SCHEMA_LOG(WARN, "fail to print partition options", K(ret), K(*container_table_schema)); - } } if (OB_FAIL(ret)) { @@ -2894,6 +2879,211 @@ int ObSchemaPrinter::print_view_definiton( } return ret; } +int ObSchemaPrinter::print_materialized_view_definition( + const uint64_t tenant_id, + const uint64_t table_id, + char *buf, + const int64_t &buf_len, + int64_t &pos, + const ObTimeZoneInfo *tz_info, + bool agent_mode, + ObSQLMode sql_mode) const +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(buf)) { + ret = OB_ERR_UNEXPECTED; + SHARE_SCHEMA_LOG(WARN, "buf is null", KR(ret)); + } else if (buf_len <= 0) { + ret = OB_ERR_UNEXPECTED; + SHARE_SCHEMA_LOG(WARN, "buf_len should bigger than 0", KR(ret)); + } else { + const ObTableSchema *table_schema = nullptr; + const ObTableSchema *container_table_schema = nullptr; + uint64_t container_table_id = OB_INVALID_ID; + ObMViewInfo mview_info; + bool is_oracle_mode = false; + bool need_print_column_list = false; + common::ObSEArray column_ids; + if (OB_FAIL(schema_guard_.get_table_schema(tenant_id, table_id, table_schema))) { + SHARE_SCHEMA_LOG(WARN, "fail to get table schema", KR(ret), K(tenant_id), K(table_id)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_TABLE_NOT_EXIST; + SHARE_SCHEMA_LOG(WARN, "Unknow table", KR(ret), K(table_id)); + } else if (OB_INVALID_ID == (container_table_id = table_schema->get_data_table_id())) { + ret = OB_ERR_UNEXPECTED; + SHARE_SCHEMA_LOG(WARN, "fail to get container_table_id", KR(ret), K(tenant_id), K(table_id)); + } else if (OB_FAIL(schema_guard_.get_table_schema(tenant_id, container_table_id, container_table_schema))) { + SHARE_SCHEMA_LOG(WARN, "fail to get container_table_schema", KR(ret), K(tenant_id), K(table_id), K(container_table_id)); + } else if (OB_ISNULL(container_table_schema)) { + ret = OB_TABLE_NOT_EXIST; + SHARE_SCHEMA_LOG(WARN, "Unknow container table", KR(ret), K(table_id), K(container_table_id)); + } else if (OB_FAIL(table_schema->check_if_oracle_compat_mode(is_oracle_mode))) { + SHARE_SCHEMA_LOG(WARN, "fail to check oracle mode", KR(ret), KPC(table_schema)); + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "CREATE MATERIALIZED VIEW "))) { + SHARE_SCHEMA_LOG(WARN, "fail to print materialized view definition", KR(ret)); + } else if (OB_FAIL(print_identifier(buf, buf_len, pos, table_schema->get_table_name(), is_oracle_mode))) { + SHARE_SCHEMA_LOG(WARN, "fail to print materialized view name", KR(ret)); + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, " "))) { + SHARE_SCHEMA_LOG(WARN, "fail to print space", KR(ret)); + } else if (FALSE_IT(need_print_column_list = (is_oracle_mode && !is_inner_table(table_id)))) { + + } else if (need_print_column_list && OB_FAIL(databuff_printf(buf, buf_len, pos, "("))) { + SHARE_SCHEMA_LOG(WARN, "fail to print view definition", K(ret)); + } else if (need_print_column_list && OB_FAIL(table_schema->get_column_ids(column_ids))) { + SHARE_SCHEMA_LOG(WARN, "fail to print view definition", K(ret)); + } else if (need_print_column_list + && OB_FAIL(print_column_list(*table_schema, column_ids, buf, buf_len, pos))) { + SHARE_SCHEMA_LOG(WARN, "fail to print view definition", K(ret)); + } else if (need_print_column_list && OB_FAIL(databuff_printf(buf, buf_len, pos, ") "))) { + SHARE_SCHEMA_LOG(WARN, "fail to print view definition", K(ret)); + } else if (!strict_compat_) { + const ObRowkeyInfo &rowkey_info = container_table_schema->get_rowkey_info(); + bool is_first_col = true; + for (int64_t j = 0; OB_SUCC(ret) && j < rowkey_info.get_size(); ++j) { + const ObColumnSchemaV2 *col = nullptr; + if (OB_ISNULL(rowkey_info.get_column(j))) { + ret = OB_ERR_UNEXPECTED; + SHARE_SCHEMA_LOG(WARN, "fail to get column", KR(ret), K(tenant_id)); + } else if (OB_ISNULL(col = schema_guard_.get_column_schema(tenant_id, + container_table_id, + rowkey_info.get_column(j)->column_id_))) { + ret = OB_ERR_UNEXPECTED; + SHARE_SCHEMA_LOG(WARN, "fail to get column schema", KR(ret), K(tenant_id), + "column_id", rowkey_info.get_column(j)->column_id_); + } else if (OB_SUCC(ret) + && col->get_column_id() != OB_HIDDEN_SESSION_ID_COLUMN_ID + && !col->is_shadow_column() + && !col->is_hidden()) { + if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%s%.*s%s", + is_first_col ? "(PRIMARY KEY (" : "", + col->get_column_name_str().length(), + col->get_column_name_str().ptr(), + j < rowkey_info.get_size() - 1 ? ", " : ""))) { + SHARE_SCHEMA_LOG(WARN, "fail to print primary key", KR(ret), K(col->get_column_name())); + } + is_first_col = false; + if (OB_SUCC(ret) && rowkey_info.get_size() - 1 == j + && OB_FAIL(databuff_printf(buf, buf_len, pos, ")) "))) { + SHARE_SCHEMA_LOG(WARN, "fail to print materialized view rowkey", KR(ret)); + } + } + } + } + if (OB_SUCC(ret)) { + if (!strict_compat_ && OB_FAIL(print_table_definition_table_options(*container_table_schema, + buf, + buf_len, + pos, + false, + agent_mode, + sql_mode))) { + SHARE_SCHEMA_LOG(WARN, "fail to print table options", KR(ret), K(*container_table_schema)); + } else if (!strict_compat_ && OB_FAIL(print_table_definition_partition_options(*container_table_schema, + buf, + buf_len, + pos, + agent_mode, + tz_info))) { + SHARE_SCHEMA_LOG(WARN, "fail to print partition options", KR(ret), K(*container_table_schema)); + } else if (OB_FAIL(ObMViewInfo::fetch_mview_info(*GCTX.sql_proxy_, tenant_id, table_id, mview_info))) { + SHARE_SCHEMA_LOG(WARN, "fail to fecth materialized view info", KR(ret), K(table_id)); + } else if (!strict_compat_) { + switch (mview_info.get_refresh_method()) { + case ObMVRefreshMethod::NEVER: + if (OB_FAIL(databuff_printf(buf, buf_len, pos, " NEVER REFRESH "))) { + SHARE_SCHEMA_LOG(WARN, "fail to print refresh method never", KR(ret)); + } + break; + case ObMVRefreshMethod::COMPLETE: + if (OB_FAIL(databuff_printf(buf, buf_len, pos, " REFRESH COMPLETE "))) { + SHARE_SCHEMA_LOG(WARN, "fail to print refresh method complete", KR(ret)); + } + break; + case ObMVRefreshMethod::FAST: + if (OB_FAIL(databuff_printf(buf, buf_len, pos, " REFRESH FAST "))) { + SHARE_SCHEMA_LOG(WARN, "fail to print refresh method fast", KR(ret)); + } + break; + case ObMVRefreshMethod::FORCE: + if (OB_FAIL(databuff_printf(buf, buf_len, pos, " REFRESH FORCE "))) { + SHARE_SCHEMA_LOG(WARN, "fail to print refresh method force", KR(ret)); + } + break; + default: + ret = OB_NOT_SUPPORTED; + SHARE_SCHEMA_LOG(WARN, "unsupported refresh method", KR(ret)); + break; + } + if (OB_SUCC(ret)) { + switch (mview_info.get_refresh_mode()) { + case ObMVRefreshMode::NEVER: + // nothing to print + break; + case ObMVRefreshMode::DEMAND: + default: + if (OB_FAIL(databuff_printf(buf, buf_len, pos, "ON DEMAND "))) { + SHARE_SCHEMA_LOG(WARN, "fail to print refresh mode", KR(ret)); + } + break; + } + } + } + } + if (!strict_compat_ && OB_SUCC(ret)) { + if (OB_NOT_NULL(mview_info.get_refresh_next().ptr())) { + if (OB_FAIL(databuff_printf(buf, buf_len, pos, + "START WITH sysdate%s", + is_oracle_mode ? " + 0" : "()"))) { + SHARE_SCHEMA_LOG(WARN, "fail to print materialized view refresh start", KR(ret)); + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, " NEXT %.*s ", + mview_info.get_refresh_next().length(), + mview_info.get_refresh_next().ptr()))) { + SHARE_SCHEMA_LOG(WARN, "fail to print mv refresh next", KR(ret)); + } + } + if (OB_SUCC(ret)) { + if (table_schema->mv_enable_query_rewrite() + && OB_FAIL(databuff_printf(buf, buf_len, pos, + "ENABLE QUERY REWRITE "))) { + SHARE_SCHEMA_LOG(WARN, "fail to print materialized view enable qurey rewrite", KR(ret)); + } else if (table_schema->mv_on_query_computation() + && OB_FAIL(databuff_printf(buf, buf_len, pos, + "ENABLE ON QUERY COMPUTATION "))) { + SHARE_SCHEMA_LOG(WARN, "fail to print materialized view on query computation", KR(ret)); + } else if (OB_FAIL(print_table_definition_column_group(*container_table_schema, + buf, + buf_len, + pos))) { + SHARE_SCHEMA_LOG(WARN, "fail to print materialized view column group", KR(ret), K(*container_table_schema)); + } + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(databuff_printf(buf, buf_len, pos, "AS "))) { + SHARE_SCHEMA_LOG(WARN, "fail to print view definition", K(ret)); + } else if (OB_FAIL(print_view_define_str(buf, buf_len, pos, is_oracle_mode, + table_schema->get_view_schema().get_view_definition_str()))) { + SHARE_SCHEMA_LOG(WARN, "fail to print view definition", K(ret)); + } else if (!table_schema->get_view_schema().get_view_is_updatable() && is_oracle_mode) { + // with read only is only supported in syntax in oracle mode, but some inner system view is + // nonupdatable, so we have to check compatible mode here. + // view in oracle mode can't be both with read only and with check option. + if (OB_FAIL(databuff_printf(buf, buf_len, pos, " WITH READ ONLY"))) { + SHARE_SCHEMA_LOG(WARN, "fail to print view definition with read only", K(ret)); + } + } else if (VIEW_CHECK_OPTION_CASCADED == table_schema->get_view_schema().get_view_check_option()) { + if (OB_FAIL(databuff_printf(buf, buf_len, pos, " WITH CHECK OPTION"))) { + SHARE_SCHEMA_LOG(WARN, "fail to print view definition with check option", K(ret)); + } + } else if (VIEW_CHECK_OPTION_LOCAL == table_schema->get_view_schema().get_view_check_option()) { + if (OB_FAIL(databuff_printf(buf, buf_len, pos, " WITH LOCAL CHECK OPTION"))) { + SHARE_SCHEMA_LOG(WARN, "fail to print view definition with local check option", K(ret)); + } + } + } + } + return ret; +} int ObSchemaPrinter::print_tablegroup_definition( const uint64_t tenant_id, diff --git a/src/share/schema/ob_schema_printer.h b/src/share/schema/ob_schema_printer.h index 17a976187..e5d7df744 100644 --- a/src/share/schema/ob_schema_printer.h +++ b/src/share/schema/ob_schema_printer.h @@ -133,6 +133,15 @@ public: bool agent_mode, ObSQLMode sql_mode) const; + int print_materialized_view_definition(const uint64_t tenant_id, + const uint64_t table_id, + char *buf, + const int64_t &buf_len, + int64_t &pos, + const ObTimeZoneInfo *tz_info, + bool agent_mode, + ObSQLMode sql_mode) const; + int print_database_definiton(const uint64_t tenant_id, const uint64_t database_id, bool if_not_exists, diff --git a/src/sql/parser/sql_parser_mysql_mode.y b/src/sql/parser/sql_parser_mysql_mode.y index fc476fe40..71791563f 100644 --- a/src/sql/parser/sql_parser_mysql_mode.y +++ b/src/sql/parser/sql_parser_mysql_mode.y @@ -8418,7 +8418,7 @@ create_with_opt_hint opt_replace opt_algorithm opt_definer opt_sql_security VIEW UNUSED($3); UNUSED($4); UNUSED($5); - malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_VIEW, 12, + malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_VIEW, 13, NULL, /* opt_materialized, not support*/ $7, /* view name */ $8, /* column list */ @@ -8427,7 +8427,7 @@ create_with_opt_hint opt_replace opt_algorithm opt_definer opt_sql_security VIEW $2, $12, /* with option */ NULL, /* force view opt */ - NULL, NULL, NULL, NULL + NULL, NULL, NULL, NULL, NULL ); dup_expr_string($11, result, @11.first_column, @11.last_column); $$->reserved_ = 0; /* is create view */ @@ -8439,7 +8439,7 @@ create_with_opt_hint opt_replace opt_algorithm opt_definer opt_sql_security VIEW UNUSED($2); UNUSED($3); UNUSED($4); - malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_VIEW, 12, + malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_VIEW, 13, NULL, /* opt_materialized */ $6, /* view name */ $7, /* column list */ @@ -8448,7 +8448,7 @@ create_with_opt_hint opt_replace opt_algorithm opt_definer opt_sql_security VIEW NULL, $11, /* with option */ NULL, /* force view opt */ - NULL, NULL, NULL, NULL + NULL, NULL, NULL, NULL, NULL ); dup_expr_string($10, result, @10.first_column, @10.last_column); $$->reserved_ = 1; /* is alter view */ @@ -8467,7 +8467,7 @@ AS view_select_stmt opt_check_option ParseNode *table_options = NULL; merge_nodes(table_options, result, T_TABLE_OPTION_LIST, $6); - malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_VIEW, 12, + malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_VIEW, 13, NULL, /* opt_materialized, not support*/ $4, /* view name */ $5, /* column list */ @@ -8479,11 +8479,36 @@ AS view_select_stmt opt_check_option $8, /* mview options */ $7, /* partition option */ table_options, /* table options */ - $1 /* hint */ + $1, /* hint */ + NULL /* column group */ ); dup_expr_string($10, result, @10.first_column, @10.last_column); $$->reserved_ = 2; /* create materialized view */ } +| create_with_opt_hint MATERIALIZED VIEW view_name opt_mv_column_list opt_table_option_list opt_partition_option create_mview_opts with_column_group +AS view_select_stmt opt_check_option +{ + ParseNode *table_options = NULL; + merge_nodes(table_options, result, T_TABLE_OPTION_LIST, $6); + + malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_VIEW, 13, + NULL, /* opt_materialized, not support*/ + $4, /* view name */ + $5, /* column list */ + NULL, /* table_id */ + $11, /* select_stmt */ + NULL, + $12, /* with option */ + NULL, /* force view opt */ + $8, /* mview options */ + $7, /* partition option */ + table_options, /* table options */ + $1, /* hint */ + $9 /* column group */ + ); + dup_expr_string($11, result, @11.first_column, @11.last_column); + $$->reserved_ = 2; /* create materialized view */ +} ; create_mview_opts: diff --git a/src/sql/resolver/ddl/ob_create_table_resolver.cpp b/src/sql/resolver/ddl/ob_create_table_resolver.cpp index 91df9db21..6c3976d8d 100644 --- a/src/sql/resolver/ddl/ob_create_table_resolver.cpp +++ b/src/sql/resolver/ddl/ob_create_table_resolver.cpp @@ -54,7 +54,6 @@ namespace sql ObCreateTableResolver::ObCreateTableResolver(ObResolverParams ¶ms) : ObCreateTableResolverBase(params), cur_column_id_(OB_APP_MIN_COLUMN_ID - 1), - cur_column_group_id_(COLUMN_GROUP_START_ID), primary_keys_(), column_name_set_(), if_not_exist_(false), @@ -3348,100 +3347,6 @@ int ObCreateTableResolver::resolve_auto_partition(const ParseNode *partition_nod return ret; } -uint64_t ObCreateTableResolver::gen_column_group_id() -{ - return ++cur_column_group_id_; -} - -/* -* only when default columns store is column_store -* have to add each column group -*/ -int ObCreateTableResolver::resolve_column_group(const ParseNode *cg_node) -{ - int ret = OB_SUCCESS; - ObCreateTableStmt *create_table_stmt = static_cast(stmt_); - ObArray column_ids; // not include virtual column - uint64_t compat_version = 0; - ObTableStoreType table_store_type = OB_TABLE_STORE_INVALID; - - if (OB_ISNULL(create_table_stmt)) { - ret = OB_ERR_UNEXPECTED; - SQL_RESV_LOG(WARN, "create_table_stmt should not be null", KR(ret)); - } else { - ObTableSchema &table_schema = create_table_stmt->get_create_table_arg().schema_; - const uint64_t tenant_id = table_schema.get_tenant_id(); - const int64_t column_cnt = table_schema.get_column_count(); - if (OB_FAIL(column_ids.reserve(column_cnt))) { - LOG_WARN("fail to reserve", KR(ret), K(column_cnt)); - } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) { - LOG_WARN("fail to get min data version", KR(ret), K(tenant_id)); - } else if (!(compat_version >= DATA_VERSION_4_3_0_0)) { - if (OB_NOT_NULL(cg_node) && (T_COLUMN_GROUP == cg_node->type_)) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("can't support column store if version less than 4_1_0_0", KR(ret), K(compat_version)); - } - } else { - table_schema.set_column_store(true); - bool is_each_cg_exist = false; - if (OB_NOT_NULL(cg_node)) { - if (OB_FAIL(parse_column_group(cg_node, table_schema, table_schema))) { - LOG_WARN("fail to parse column group", K(ret)); - } - } - - /* build column group when cg node is null && tenant cg valid*/ - ObTenantConfigGuard tenant_config(TENANT_CONF(session_info_->get_effective_tenant_id())); - if (OB_FAIL(ret)) { - } else if ( OB_LIKELY(tenant_config.is_valid()) && nullptr == cg_node) { - /* force to build each cg*/ - if (!ObSchemaUtils::can_add_column_group(table_schema)) { - } else if (OB_FAIL(ObTableStoreFormat::find_table_store_type( - tenant_config->default_table_store_format.get_value_string(), - table_store_type))) { - LOG_WARN("fail to get table store format", K(ret), K(table_store_type)); - } else if (ObTableStoreFormat::is_with_column(table_store_type)) { - /* for default is column store, must add each column group*/ - if (OB_FAIL(ObSchemaUtils::build_add_each_column_group(table_schema, table_schema))) { - LOG_WARN("fail to add each column group", K(ret)); - } - } - - /* force to build all cg*/ - ObColumnGroupSchema all_cg; - if (OB_FAIL(ret)) { - } else if (!ObSchemaUtils::can_add_column_group(table_schema)) { - } else if (ObTableStoreFormat::is_row_with_column_store(table_store_type)) { - if (OB_FAIL(ObSchemaUtils::build_all_column_group(table_schema, table_schema.get_tenant_id(), - table_schema.get_max_used_column_group_id() + 1, all_cg))) { - LOG_WARN("fail to add all column group", K(ret)); - } else if (OB_FAIL(table_schema.add_column_group(all_cg))) { - LOG_WARN("fail to build all column group", K(ret)); - } - } - } - - // add default_type column_group, build a empty and then use alter_deafult_cg - if (OB_SUCC(ret)) { - ObColumnGroupSchema tmp_cg; - column_ids.reuse(); - if (OB_FAIL(build_column_group(table_schema, ObColumnGroupType::DEFAULT_COLUMN_GROUP, - OB_DEFAULT_COLUMN_GROUP_NAME, column_ids, DEFAULT_TYPE_COLUMN_GROUP_ID, tmp_cg))) { - LOG_WARN("fail to build default type column_group", KR(ret), K(table_store_type), - "table_id", table_schema.get_table_id()); - } else if (OB_FAIL(table_schema.add_column_group(tmp_cg))) { - LOG_WARN("fail to add default column group", KR(ret), "table_id", table_schema.get_table_id()); - } else if (OB_FAIL(ObSchemaUtils::alter_rowkey_column_group(table_schema))) { - LOG_WARN("fail to adjust rowkey column group when add column group", K(ret)); - } else if (OB_FAIL(ObSchemaUtils::alter_default_column_group(table_schema))) { - LOG_WARN("fail to adjust default column group", K(ret)); - } - } - } - } - return ret; -} - int ObCreateTableResolver::add_inner_index_for_heap_gtt() { int ret = OB_SUCCESS; diff --git a/src/sql/resolver/ddl/ob_create_table_resolver.h b/src/sql/resolver/ddl/ob_create_table_resolver.h index c0d86ee2b..b6318b7c9 100644 --- a/src/sql/resolver/ddl/ob_create_table_resolver.h +++ b/src/sql/resolver/ddl/ob_create_table_resolver.h @@ -132,10 +132,6 @@ private: int check_external_table_generated_partition_column_sanity(ObTableSchema &table_schema, ObRawExpr *dependant_expr, ObIArray &external_part_idx); typedef common::hash::ObPlacementHashSet VPColumnIdHashSet; - // check this type of table_schema should build column_group or not - uint64_t gen_column_group_id(); - int resolve_column_group(const ParseNode *cg_node); - int add_inner_index_for_heap_gtt(); int check_max_row_data_length(const ObTableSchema &table_schema); @@ -144,7 +140,6 @@ private: private: // data members uint64_t cur_column_id_; - uint64_t cur_column_group_id_; common::ObSEArray primary_keys_; common::hash::ObPlacementHashSet column_name_set_; bool if_not_exist_; diff --git a/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp b/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp index f7d8fb64c..002debb91 100644 --- a/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp +++ b/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp @@ -23,7 +23,8 @@ using namespace omt; namespace sql { ObCreateTableResolverBase::ObCreateTableResolverBase(ObResolverParams ¶ms) - : ObDDLResolver(params) + : ObDDLResolver(params), + cur_column_group_id_(COLUMN_GROUP_START_ID) { } @@ -458,5 +459,107 @@ int ObCreateTableResolverBase::add_primary_key_part(const ObString &column_name, return ret; } +uint64_t ObCreateTableResolverBase::gen_column_group_id() +{ + return ++cur_column_group_id_; +} + +int ObCreateTableResolverBase::resolve_column_group_helper(const ParseNode *cg_node, + ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + ObArray column_ids; // not include virtual column + uint64_t compat_version = 0; + ObTableStoreType table_store_type = OB_TABLE_STORE_INVALID; + const uint64_t tenant_id = table_schema.get_tenant_id(); + const int64_t column_cnt = table_schema.get_column_count(); + if (OB_FAIL(column_ids.reserve(column_cnt))) { + LOG_WARN("fail to reserve", KR(ret), K(column_cnt)); + } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) { + LOG_WARN("fail to get min data version", KR(ret), K(tenant_id)); + } else if (!(compat_version >= DATA_VERSION_4_3_0_0)) { + if (OB_NOT_NULL(cg_node) && (T_COLUMN_GROUP == cg_node->type_)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("can't support column store if version less than 4_1_0_0", KR(ret), K(compat_version)); + } + } else { + table_schema.set_column_store(true); + bool is_each_cg_exist = false; + if (OB_NOT_NULL(cg_node)) { + if (OB_FAIL(parse_column_group(cg_node, table_schema, table_schema))) { + LOG_WARN("fail to parse column group", K(ret)); + } + } + + /* build column group when cg node is null && tenant cg valid*/ + ObTenantConfigGuard tenant_config(TENANT_CONF(session_info_->get_effective_tenant_id())); + if (OB_FAIL(ret)) { + } else if ( OB_LIKELY(tenant_config.is_valid()) && nullptr == cg_node) { + /* force to build each cg*/ + if (!ObSchemaUtils::can_add_column_group(table_schema)) { + } else if (OB_FAIL(ObTableStoreFormat::find_table_store_type( + tenant_config->default_table_store_format.get_value_string(), + table_store_type))) { + LOG_WARN("fail to get table store format", K(ret), K(table_store_type)); + } else if (ObTableStoreFormat::is_with_column(table_store_type)) { + /* for default is column store, must add each column group*/ + if (OB_FAIL(ObSchemaUtils::build_add_each_column_group(table_schema, table_schema))) { + LOG_WARN("fail to add each column group", K(ret)); + } + } + + /* force to build all cg*/ + ObColumnGroupSchema all_cg; + if (OB_FAIL(ret)) { + } else if (!ObSchemaUtils::can_add_column_group(table_schema)) { + } else if (ObTableStoreFormat::is_row_with_column_store(table_store_type)) { + if (OB_FAIL(ObSchemaUtils::build_all_column_group(table_schema, table_schema.get_tenant_id(), + table_schema.get_max_used_column_group_id() + 1, all_cg))) { + LOG_WARN("fail to add all column group", K(ret)); + } else if (OB_FAIL(table_schema.add_column_group(all_cg))) { + LOG_WARN("fail to build all column group", K(ret)); + } + } + } + + // add default_type column_group, build a empty and then use alter_deafult_cg + if (OB_SUCC(ret)) { + ObColumnGroupSchema tmp_cg; + column_ids.reuse(); + if (OB_FAIL(build_column_group(table_schema, ObColumnGroupType::DEFAULT_COLUMN_GROUP, + OB_DEFAULT_COLUMN_GROUP_NAME, column_ids, DEFAULT_TYPE_COLUMN_GROUP_ID, tmp_cg))) { + LOG_WARN("fail to build default type column_group", KR(ret), K(table_store_type), + "table_id", table_schema.get_table_id()); + } else if (OB_FAIL(table_schema.add_column_group(tmp_cg))) { + LOG_WARN("fail to add default column group", KR(ret), "table_id", table_schema.get_table_id()); + } else if (OB_FAIL(ObSchemaUtils::alter_rowkey_column_group(table_schema))) { + LOG_WARN("fail to adjust rowkey column group when add column group", K(ret)); + } else if (OB_FAIL(ObSchemaUtils::alter_default_column_group(table_schema))) { + LOG_WARN("fail to adjust default column group", K(ret)); + } + } + } + return ret; +} +/* +* only when default columns store is column_store +* have to add each column group +*/ +int ObCreateTableResolverBase::resolve_column_group(const ParseNode *cg_node) +{ + int ret = OB_SUCCESS; + ObCreateTableStmt *create_table_stmt = static_cast(stmt_); + + if (OB_ISNULL(create_table_stmt)) { + ret = OB_ERR_UNEXPECTED; + SQL_RESV_LOG(WARN, "create_table_stmt should not be null", KR(ret)); + } else { + ObTableSchema &table_schema = create_table_stmt->get_create_table_arg().schema_; + if (OB_FAIL(resolve_column_group_helper(cg_node, table_schema))) { + LOG_WARN("fail to resolve column group helper", KR(ret)); + } + } + return ret; +} }//end namespace sql }//end namespace oceanbase diff --git a/src/sql/resolver/ddl/ob_create_table_resolver_base.h b/src/sql/resolver/ddl/ob_create_table_resolver_base.h index 532918e3e..2763a97b5 100644 --- a/src/sql/resolver/ddl/ob_create_table_resolver_base.h +++ b/src/sql/resolver/ddl/ob_create_table_resolver_base.h @@ -44,6 +44,12 @@ protected: int64_t &pk_data_length, ObColumnSchemaV2 *&col); + int resolve_column_group_helper(const ParseNode *cg_node, ObTableSchema &table_schema); + // check this type of table_schema should build column_group or not + uint64_t gen_column_group_id(); + int resolve_column_group(const ParseNode *cg_node); +protected: + uint64_t cur_column_group_id_; }; } // end namespace sql diff --git a/src/sql/resolver/ddl/ob_create_view_resolver.cpp b/src/sql/resolver/ddl/ob_create_view_resolver.cpp index e226937d6..1af72b392 100644 --- a/src/sql/resolver/ddl/ob_create_view_resolver.cpp +++ b/src/sql/resolver/ddl/ob_create_view_resolver.cpp @@ -27,12 +27,15 @@ #include "sql/resolver/mv/ob_mv_checker.h" #include "observer/virtual_table/ob_table_columns.h" #include "sql/rewrite/ob_transformer_impl.h" +#include "observer/omt/ob_tenant_config_mgr.h" +#include "common/ob_store_format.h" namespace oceanbase { using namespace common; using namespace obrpc; using namespace share::schema; +using namespace omt; namespace sql { ObCreateViewResolver::ObCreateViewResolver(ObResolverParams ¶ms) : ObCreateTableResolverBase(params) @@ -399,6 +402,8 @@ int ObCreateViewResolver::resolve(const ParseNode &parse_tree) ObMVAdditionalInfo *mv_ainfo = NULL; ObCreateTableStmt *create_table_stmt = static_cast(stmt_); ObSEArray &csts = create_table_stmt->get_create_table_arg().constraint_list_; + ObTenantConfigGuard tenant_config(TENANT_CONF(session_info_->get_effective_tenant_id())); + ObTableStoreType table_store_type = OB_TABLE_STORE_INVALID; if (OB_FAIL(ObResolverUtils::check_schema_valid_for_mview(table_schema))) { LOG_WARN("failed to check schema valid for mview", KR(ret), K(table_schema)); } else if (OB_FAIL(resolve_table_options(parse_tree.children_[TABLE_OPTION_NODE], false))) { @@ -418,10 +423,26 @@ int ObCreateViewResolver::resolve(const ParseNode &parse_tree) mv_ainfo->mv_refresh_info_, table_schema))) { LOG_WARN("fail to resolve mv options", K(ret)); - } else if (OB_FAIL(resolve_hints(parse_tree.children_[HINT_NODE], *stmt, mv_ainfo->container_table_schema_))) { - LOG_WARN("resolve hints failed", K(ret)); - } else { - mv_ainfo->mv_refresh_info_.parallel_ = stmt->get_parallelism(); + } else if (!tenant_config.is_valid()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant config is invalid", KR(ret)); + } else if (OB_FAIL(ObTableStoreFormat::find_table_store_type( + tenant_config->default_table_store_format.get_value_string(), + table_store_type))) { + LOG_WARN("fail to find table store type", KR(ret)); + } else if (ObTableStoreFormat::is_with_column(table_store_type) + || OB_NOT_NULL(parse_tree.children_[COLUMN_GROUP_NODE])) { + if (OB_FAIL(resolve_column_group_helper(parse_tree.children_[COLUMN_GROUP_NODE], + mv_ainfo->container_table_schema_))) { + LOG_WARN("fail to resolve column group", KR(ret)); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(resolve_hints(parse_tree.children_[HINT_NODE], *stmt, mv_ainfo->container_table_schema_))) { + LOG_WARN("resolve hints failed", K(ret)); + } else { + mv_ainfo->mv_refresh_info_.parallel_ = stmt->get_parallelism(); + } } } diff --git a/src/sql/resolver/ddl/ob_create_view_resolver.h b/src/sql/resolver/ddl/ob_create_view_resolver.h index 8d1b80e29..1b6c003eb 100644 --- a/src/sql/resolver/ddl/ob_create_view_resolver.h +++ b/src/sql/resolver/ddl/ob_create_view_resolver.h @@ -45,7 +45,8 @@ class ObCreateViewResolver : public ObCreateTableResolverBase static const int64_t PARTITION_NODE = 9; static const int64_t TABLE_OPTION_NODE = 10; static const int64_t HINT_NODE = 11; - static const int64_t ROOT_NUM_CHILD = 12; + static const int64_t COLUMN_GROUP_NODE = 12; + static const int64_t ROOT_NUM_CHILD = 13; public: explicit ObCreateViewResolver(ObResolverParams ¶ms); From cb613239172ab5a6080603cc8bce6859ca724eca Mon Sep 17 00:00:00 2001 From: jingtaoye35 <1255153887@qq.com> Date: Thu, 22 Aug 2024 16:31:05 +0000 Subject: [PATCH 180/249] [FEAT MERGE] enhance distributed late materialization [433] --- src/sql/CMakeLists.txt | 1 + src/sql/optimizer/ob_log_join.cpp | 3 +- src/sql/optimizer/ob_log_table_scan.cpp | 2 + src/sql/optimizer/ob_select_log_plan.cpp | 6 +- src/sql/parser/sql_parser_mysql_mode.y | 16 +- src/sql/resolver/dml/ob_dml_resolver.cpp | 6 +- src/sql/resolver/dml/ob_hint.cpp | 30 +- src/sql/resolver/dml/ob_hint.h | 11 +- src/sql/resolver/dml/ob_sql_hint.cpp | 16 +- src/sql/resolver/dml/ob_sql_hint.h | 2 +- src/sql/resolver/expr/ob_raw_expr_util.cpp | 60 + src/sql/resolver/expr/ob_raw_expr_util.h | 10 + .../ob_transform_late_materialization.cpp | 1194 +++++++++++++++++ .../ob_transform_late_materialization.h | 146 ++ src/sql/rewrite/ob_transform_rule.cpp | 1 + src/sql/rewrite/ob_transform_rule.h | 14 +- .../ob_transform_simplify_distinct.cpp | 3 +- src/sql/rewrite/ob_transform_utils.cpp | 1 + src/sql/rewrite/ob_transformer_impl.cpp | 2 + .../r/mysql/geometry_basic_mysql.result | 29 +- 20 files changed, 1505 insertions(+), 48 deletions(-) create mode 100644 src/sql/rewrite/ob_transform_late_materialization.cpp create mode 100644 src/sql/rewrite/ob_transform_late_materialization.h diff --git a/src/sql/CMakeLists.txt b/src/sql/CMakeLists.txt index 8def84013..63a37b295 100644 --- a/src/sql/CMakeLists.txt +++ b/src/sql/CMakeLists.txt @@ -1344,6 +1344,7 @@ ob_set_subtarget(ob_sql rewrite rewrite/ob_transform_mv_rewrite.cpp rewrite/ob_union_find.cpp rewrite/ob_transform_decorrelate.cpp + rewrite/ob_transform_late_materialization.cpp ) ob_set_subtarget(ob_sql session diff --git a/src/sql/optimizer/ob_log_join.cpp b/src/sql/optimizer/ob_log_join.cpp index c07996916..f53e18745 100644 --- a/src/sql/optimizer/ob_log_join.cpp +++ b/src/sql/optimizer/ob_log_join.cpp @@ -627,7 +627,8 @@ int ObLogJoin::print_used_hint(PlanText &plan_text) if (OB_ISNULL(hint = used_hints.at(i))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected NULL", K(ret), K(hint)); - } else if (OB_FAIL(hint->print_hint(plan_text))) { + } else if (!hint->is_trans_added() && + OB_FAIL(hint->print_hint(plan_text))) { LOG_WARN("failed to print hint in log join", K(ret), K(*hint)); } } diff --git a/src/sql/optimizer/ob_log_table_scan.cpp b/src/sql/optimizer/ob_log_table_scan.cpp index bb935d2a5..57cf5daca 100644 --- a/src/sql/optimizer/ob_log_table_scan.cpp +++ b/src/sql/optimizer/ob_log_table_scan.cpp @@ -2059,6 +2059,8 @@ int ObLogTableScan::print_used_hint(PlanText &plan_text) LOG_WARN("unexpected idx", K(ret), K(idx), K(table_hint->index_list_)); } else if (!is_skip_scan() && T_INDEX_SS_HINT == hint->get_hint_type()) { /* is not index skip scan but exist index_ss hint */ + } else if (hint->is_trans_added()) { + //do nothing } else if (OB_FAIL(hint->print_hint(plan_text))) { LOG_WARN("failed to print index hint", K(ret), K(*hint)); } diff --git a/src/sql/optimizer/ob_select_log_plan.cpp b/src/sql/optimizer/ob_select_log_plan.cpp index e65c3aacb..c2d0f89f4 100644 --- a/src/sql/optimizer/ob_select_log_plan.cpp +++ b/src/sql/optimizer/ob_select_log_plan.cpp @@ -4550,7 +4550,11 @@ int ObSelectLogPlan::allocate_plan_top() // step. allocate late materialization if needed if (OB_SUCC(ret) && select_stmt->has_limit()) { - if (OB_FAIL(candi_allocate_late_materialization())) { + if ((optimizer_context_.get_query_ctx()->optimizer_features_enable_version_ >= COMPAT_VERSION_4_2_5 && + optimizer_context_.get_query_ctx()->optimizer_features_enable_version_ < COMPAT_VERSION_4_3_0) || + (optimizer_context_.get_query_ctx()->optimizer_features_enable_version_ >= COMPAT_VERSION_4_3_3)) { + /* rewrite will enhance late materialization */ + } else if (OB_FAIL(candi_allocate_late_materialization())) { LOG_WARN("failed to allocate late-materialization operator", K(ret)); } else { LOG_TRACE("succeed to allocate late-materialization operator", diff --git a/src/sql/parser/sql_parser_mysql_mode.y b/src/sql/parser/sql_parser_mysql_mode.y index 71791563f..1f1a34647 100644 --- a/src/sql/parser/sql_parser_mysql_mode.y +++ b/src/sql/parser/sql_parser_mysql_mode.y @@ -11052,6 +11052,14 @@ NO_REWRITE opt_qb_name { malloc_non_terminal_node($$, result->malloc_pool_, T_NO_COALESCE_AGGR, 1, $2); } +| USE_LATE_MATERIALIZATION opt_qb_name +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_USE_LATE_MATERIALIZATION, 1, $2); +} +| NO_USE_LATE_MATERIALIZATION opt_qb_name +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_NO_USE_LATE_MATERIALIZATION, 1, $2); +} ; multi_qb_name_list: @@ -11190,14 +11198,6 @@ INDEX_HINT '(' qb_name_option relation_factor_in_hint NAME_OB opt_index_prefix ' malloc_non_terminal_node($$, result->malloc_pool_, T_NO_USE_HASH_AGGREGATE, 1, $3); $$->value_ = 1; } -| USE_LATE_MATERIALIZATION opt_qb_name -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_USE_LATE_MATERIALIZATION, 1, $2); -} -| NO_USE_LATE_MATERIALIZATION opt_qb_name -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_NO_USE_LATE_MATERIALIZATION, 1, $2); -} | PX_JOIN_FILTER '(' qb_name_option relation_factor_in_hint opt_relation_factor_in_hint_list ')' { malloc_non_terminal_node($$, result->malloc_pool_, T_PX_JOIN_FILTER, 4, $3, $4, $5, NULL); diff --git a/src/sql/resolver/dml/ob_dml_resolver.cpp b/src/sql/resolver/dml/ob_dml_resolver.cpp index c523a178d..dc6927b9e 100755 --- a/src/sql/resolver/dml/ob_dml_resolver.cpp +++ b/src/sql/resolver/dml/ob_dml_resolver.cpp @@ -14537,7 +14537,9 @@ int ObDMLResolver::resolve_transform_hint(const ParseNode &hint_node, case T_JOIN_FIRST_UNNEST: case T_NO_JOIN_FIRST_UNNEST: case T_DECORRELATE: - case T_NO_DECORRELATE: { + case T_NO_DECORRELATE: + case T_USE_LATE_MATERIALIZATION: + case T_NO_USE_LATE_MATERIALIZATION: { if (OB_FAIL(resolve_normal_transform_hint(hint_node, trans_hint))) { LOG_WARN("failed to resolve hint with qb name param.", K(ret)); } @@ -14637,8 +14639,6 @@ int ObDMLResolver::resolve_optimize_hint(const ParseNode &hint_node, } break; } - case T_USE_LATE_MATERIALIZATION: - case T_NO_USE_LATE_MATERIALIZATION: case T_GBY_PUSHDOWN: case T_NO_GBY_PUSHDOWN: case T_USE_HASH_DISTINCT: diff --git a/src/sql/resolver/dml/ob_hint.cpp b/src/sql/resolver/dml/ob_hint.cpp index 1275f485b..627ce582e 100644 --- a/src/sql/resolver/dml/ob_hint.cpp +++ b/src/sql/resolver/dml/ob_hint.cpp @@ -1068,6 +1068,7 @@ ObItemType ObHint::get_hint_type(ObItemType type) case T_NO_DECORRELATE : return T_DECORRELATE; case T_NO_COALESCE_AGGR: return T_COALESCE_AGGR; case T_MV_NO_REWRITE: return T_MV_REWRITE; + case T_NO_USE_LATE_MATERIALIZATION: return T_USE_LATE_MATERIALIZATION; // optimize hint case T_NO_USE_DAS_HINT: return T_USE_DAS_HINT; @@ -1079,7 +1080,6 @@ ObItemType ObHint::get_hint_type(ObItemType type) case T_NO_USE_NL_MATERIALIZATION: return T_USE_NL_MATERIALIZATION; case T_NO_PX_JOIN_FILTER: return T_PX_JOIN_FILTER; case T_NO_PX_PART_JOIN_FILTER: return T_PX_PART_JOIN_FILTER; - case T_NO_USE_LATE_MATERIALIZATION: return T_USE_LATE_MATERIALIZATION; case T_NO_USE_HASH_AGGREGATE: return T_USE_HASH_AGGREGATE; case T_NO_GBY_PUSHDOWN: return T_GBY_PUSHDOWN; case T_NO_USE_HASH_DISTINCT: return T_USE_HASH_DISTINCT; @@ -1127,6 +1127,8 @@ const char* ObHint::get_hint_name(ObItemType type, bool is_enable_hint /* defaul case T_DECORRELATE : return is_enable_hint ? "DECORRELATE" : "NO_DECORRELATE"; case T_COALESCE_AGGR: return is_enable_hint ? "COALESCE_AGGR" : "NO_COALESCE_AGGR"; case T_MV_REWRITE: return is_enable_hint ? "MV_REWRITE" : "NO_MV_REWRITE"; + case T_USE_LATE_MATERIALIZATION: + return is_enable_hint ? "USE_LATE_MATERIALIZATION" : "NO_USE_LATE_MATERIALIZATION"; // optimize hint case T_INDEX_HINT: return "INDEX"; case T_FULL_HINT: return "FULL"; @@ -1147,8 +1149,6 @@ const char* ObHint::get_hint_name(ObItemType type, bool is_enable_hint /* defaul case T_PQ_DISTRIBUTE: return "PQ_DISTRIBUTE"; case T_PQ_MAP: return "PQ_MAP"; case T_PQ_SET: return "PQ_SET"; - case T_USE_LATE_MATERIALIZATION: return is_enable_hint ? "USE_LATE_MATERIALIZATION" - : "NO_USE_LATE_MATERIALIZATION"; case T_USE_HASH_AGGREGATE: return is_enable_hint ? "USE_HASH_AGGREGATION" : "NO_USE_HASH_AGGREGATION"; case T_TABLE_PARALLEL: return "PARALLEL"; @@ -3158,5 +3158,29 @@ int ObDirectLoadHint::print_direct_load_hint(PlanText &plan_text) const return ret; } +bool ObIndexHint::is_match_index(const ObCollationType cs_type, + const TableItem &ref_table, + const ObTableSchema &index_schema) const +{ + bool match = false; + int ret = OB_SUCCESS; + ObString table_name; + ObString index_name; + if (OB_UNLIKELY(!index_schema.is_index_table())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("got unexpected param", K(ret)); + } else if (!table_.is_match_table_item(cs_type, ref_table)) { + /* do nothing */ + } else if (OB_FAIL(index_schema.get_index_name(index_name))) { + LOG_WARN("failed to get index name", K(ret)); + } else { + match = 0 == ObCharset::strcmp(cs_type, index_name_, index_name) ? true : false; + } + if (OB_FAIL(ret)) { + match = false; + } + return match; +} + }//end of namespace sql }//end of namespace oceanbase diff --git a/src/sql/resolver/dml/ob_hint.h b/src/sql/resolver/dml/ob_hint.h index d914dbe38..6c88a3356 100644 --- a/src/sql/resolver/dml/ob_hint.h +++ b/src/sql/resolver/dml/ob_hint.h @@ -222,6 +222,7 @@ struct ObGlobalHint { #define COMPAT_VERSION_4_2_2 (oceanbase::common::cal_version(4, 2, 2, 0)) #define COMPAT_VERSION_4_2_3 (oceanbase::common::cal_version(4, 2, 3, 0)) #define COMPAT_VERSION_4_2_4 (oceanbase::common::cal_version(4, 2, 4, 0)) +#define COMPAT_VERSION_4_2_5 (oceanbase::common::cal_version(4, 2, 5, 0)) #define COMPAT_VERSION_4_3_0 (oceanbase::common::cal_version(4, 3, 0, 0)) #define COMPAT_VERSION_4_3_1 (oceanbase::common::cal_version(4, 3, 1, 0)) #define COMPAT_VERSION_4_3_2 (oceanbase::common::cal_version(4, 3, 2, 0)) @@ -541,6 +542,7 @@ public: orig_hint_(NULL) { hint_type_ = get_hint_type(hint_type); is_enable_hint_ = (hint_type_ == hint_type); + is_trans_added_ = false; } virtual ~ObHint() {} int assign(const ObHint &other); @@ -600,10 +602,13 @@ public: bool is_pq_subquery_hint() const { return T_PQ_SUBQUERY == hint_type_; } bool is_decorrelate_hint() const { return T_DECORRELATE == hint_type_; } bool is_coalesce_aggr_hint() const {return HINT_COALESCE_AGGR == hint_class_; } + bool is_trans_added() const { return is_trans_added_; } + bool set_trans_added(bool is_trans_added) { return is_trans_added_ = is_trans_added; } VIRTUAL_TO_STRING_KV("hint_type", get_type_name(hint_type_), K_(hint_class), K_(qb_name), - K_(orig_hint), K_(is_enable_hint)); + K_(orig_hint), K_(is_enable_hint), + K_(is_trans_added)); private: // only used in create_push_down_hint @@ -615,6 +620,7 @@ protected: ObString qb_name_; const ObHint *orig_hint_; bool is_enable_hint_; + bool is_trans_added_; // It means that hint is added by rewriter, and it needn't print the hint when explain extended }; class ObTransHint : public ObHint @@ -968,6 +974,9 @@ public: const int64_t &get_index_prefix() const { return index_prefix_; } bool is_use_index_hint() const { return T_NO_INDEX_HINT != get_hint_type(); } bool use_skip_scan() const { return T_INDEX_SS_HINT == get_hint_type(); } + bool is_match_index(const ObCollationType cs_type, + const TableItem &ref_table, + const ObTableSchema &index_schema) const; INHERIT_TO_STRING_KV("ObHint", ObHint, K_(table), K_(index_name), K_(index_prefix)); diff --git a/src/sql/resolver/dml/ob_sql_hint.cpp b/src/sql/resolver/dml/ob_sql_hint.cpp index 64c6e1a9f..c6c94b752 100644 --- a/src/sql/resolver/dml/ob_sql_hint.cpp +++ b/src/sql/resolver/dml/ob_sql_hint.cpp @@ -1429,7 +1429,7 @@ int ObLogPlanHint::init_log_plan_hint(ObSqlSchemaGuard &schema_guard, const ObStmtHint &stmt_hint = stmt.get_stmt_hint(); if (OB_FAIL(join_order_.init_leading_info(stmt, query_hint, stmt_hint.get_normal_hint(T_LEADING)))) { LOG_WARN("failed to get leading hint info", K(ret)); - } else if (OB_FAIL(init_normal_hints(stmt_hint.normal_hints_))) { + } else if (OB_FAIL(init_normal_hints(stmt_hint.normal_hints_, *stmt.get_query_ctx()))) { LOG_WARN("failed to init normal hints", K(ret)); } else if (OB_FAIL(init_other_opt_hints(schema_guard, stmt, query_hint, stmt_hint.other_opt_hints_))) { @@ -1440,17 +1440,25 @@ int ObLogPlanHint::init_log_plan_hint(ObSqlSchemaGuard &schema_guard, return ret; } -int ObLogPlanHint::init_normal_hints(const ObIArray &normal_hints) +int ObLogPlanHint::init_normal_hints(const ObIArray &normal_hints, + const ObQueryCtx &query_ctx) { int ret = OB_SUCCESS; const ObHint *hint = NULL; for (int64_t i = 0; OB_SUCC(ret) && i < normal_hints.count(); ++i) { + bool need_add = true; if (OB_ISNULL(hint = normal_hints.at(i))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret), K(i), K(normal_hints)); + } else if (hint->get_hint_type() == T_USE_LATE_MATERIALIZATION && + (query_ctx.optimizer_features_enable_version_ < COMPAT_VERSION_4_2_5 || + (query_ctx.optimizer_features_enable_version_ >= COMPAT_VERSION_4_3_0 && + query_ctx.optimizer_features_enable_version_ < COMPAT_VERSION_4_3_3))) { + /* need_add = true */ } else if (hint->is_transform_hint() || hint->is_join_order_hint()) { - /* do nothing */ - } else if (OB_FAIL(normal_hints_.push_back(hint))) { + need_add = false; + } + if (OB_SUCC(ret) && need_add && OB_FAIL(normal_hints_.push_back(hint))) { LOG_WARN("failed to push back", K(ret)); } } diff --git a/src/sql/resolver/dml/ob_sql_hint.h b/src/sql/resolver/dml/ob_sql_hint.h index 5ac9ad1dc..e40271aef 100644 --- a/src/sql/resolver/dml/ob_sql_hint.h +++ b/src/sql/resolver/dml/ob_sql_hint.h @@ -397,7 +397,7 @@ struct ObLogPlanHint { ObLogPlanHint() { reset(); } void reset(); - int init_normal_hints(const ObIArray &normal_hints); + int init_normal_hints(const ObIArray &normal_hints, const ObQueryCtx &query_ctx); #ifndef OB_BUILD_SPM int init_log_plan_hint(ObSqlSchemaGuard &schema_guard, const ObDMLStmt &stmt, diff --git a/src/sql/resolver/expr/ob_raw_expr_util.cpp b/src/sql/resolver/expr/ob_raw_expr_util.cpp index bf94aa0a4..6562aaf4f 100644 --- a/src/sql/resolver/expr/ob_raw_expr_util.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_util.cpp @@ -6763,6 +6763,66 @@ int ObRawExprUtils::create_equal_expr(ObRawExprFactory &expr_factory, return ret; } +int ObRawExprUtils::create_null_safe_equal_expr(ObRawExprFactory &expr_factory, + const ObSQLSessionInfo *session_info, + const bool is_mysql_mode, + ObRawExpr *left_expr, + ObRawExpr *right_expr, + ObRawExpr *&expr) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(left_expr) || OB_ISNULL(right_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (is_mysql_mode) { + // left <=> right + ObOpRawExpr *nseq_expr = NULL; + if (OB_FAIL(expr_factory.create_raw_expr(T_OP_NSEQ, nseq_expr))) { + LOG_WARN("failed to create raw expr", K(ret)); + } else if (OB_ISNULL(expr = nseq_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_FAIL(nseq_expr->set_param_exprs(left_expr, right_expr))) { + LOG_WARN("failed to add param expr", K(ret)); + } else if (OB_FAIL(expr->formalize(session_info))) { + LOG_WARN("formalize equal expr failed", K(ret)); + } + } else { + // (left == right) or (left is null and right is null) + ObOpRawExpr *or_expr = NULL; + ObRawExpr *left_equal_expr = NULL; + ObOpRawExpr *and_expr = NULL; + ObRawExpr *left_is_null = NULL; + ObRawExpr *right_is_null = NULL; + if (OB_FAIL(expr_factory.create_raw_expr(T_OP_OR, or_expr))) { + LOG_WARN("failed to create raw expr", K(ret)); + } else if (OB_ISNULL(expr = or_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_FAIL(create_equal_expr(expr_factory, session_info, + left_expr, right_expr, left_equal_expr))) { + LOG_WARN("failed to create equal expr", K(ret)); + } else if (OB_FAIL(or_expr->add_param_expr(left_equal_expr))) { + LOG_WARN("add param expr to or expr failed", K(ret)); + } else if (OB_FAIL(build_is_not_null_expr(expr_factory, left_expr, false, left_is_null))) { + LOG_WARN("failed to build is not null expr", K(ret)); + } else if (OB_FAIL(build_is_not_null_expr(expr_factory, right_expr, false, right_is_null))) { + LOG_WARN("failed to build is not null expr", K(ret)); + } else if (OB_FAIL(expr_factory.create_raw_expr(T_OP_AND, and_expr))) { + LOG_WARN("failed to create a new expr", K(ret)); + } else if (OB_FAIL(and_expr->add_param_expr(left_is_null))) { + LOG_WARN("add param expr to or expr failed", K(ret)); + } else if (OB_FAIL(and_expr->add_param_expr(right_is_null))) { + LOG_WARN("add param expr to or expr failed", K(ret)); + } else if (OB_FAIL(or_expr->add_param_expr(and_expr))) { + LOG_WARN("add param expr to or expr failed", K(ret)); + } else if (OB_FAIL(expr->formalize(session_info))) { + LOG_WARN("formalize equal expr failed", K(ret)); + } + } + return ret; +} + int ObRawExprUtils::create_double_op_expr(ObRawExprFactory &expr_factory, const ObSQLSessionInfo *session_info, ObItemType expr_type, diff --git a/src/sql/resolver/expr/ob_raw_expr_util.h b/src/sql/resolver/expr/ob_raw_expr_util.h index cc62bfc11..5401bd317 100644 --- a/src/sql/resolver/expr/ob_raw_expr_util.h +++ b/src/sql/resolver/expr/ob_raw_expr_util.h @@ -1251,6 +1251,16 @@ public: const ObSQLMode sql_mode, ObColumnSchemaV2 &gen_col); static int check_contain_op_row_expr(const ObRawExpr *raw_expr, bool &contain); + /* + in mysql mode: ret left_expr <=> right_expr + in oracle mode: ret (left_expr = right_expr) or (left_expr is null and right_expr is null) + */ + static int create_null_safe_equal_expr(ObRawExprFactory &expr_factory, + const ObSQLSessionInfo *session_info, + const bool is_mysql_mode, + ObRawExpr *left_expr, + ObRawExpr *right_expr, + ObRawExpr *&expr); static int copy_and_formalize(ObRawExpr *&expr, ObRawExprCopier *copier, diff --git a/src/sql/rewrite/ob_transform_late_materialization.cpp b/src/sql/rewrite/ob_transform_late_materialization.cpp new file mode 100644 index 000000000..3cb3e6db7 --- /dev/null +++ b/src/sql/rewrite/ob_transform_late_materialization.cpp @@ -0,0 +1,1194 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_REWRITE +#include "sql/rewrite/ob_transform_late_materialization.h" +#include "sql/rewrite/ob_transform_utils.h" +#include "sql/optimizer/ob_optimizer_util.h" +#include "sql/optimizer/ob_log_table_scan.h" +#include "sql/optimizer/ob_log_sort.h" + +namespace oceanbase +{ +using namespace common; +namespace sql +{ +int ObTransformLateMaterialization::transform_one_stmt(ObIArray &parent_stmts, + ObDMLStmt *&stmt, + bool &trans_happened) +{ + int ret = OB_SUCCESS; + bool force_trans = false; + bool force_no_trans = false; + bool can_trans = false; + const ObSelectStmt *select_stmt = NULL; + ObLateMaterializationInfo info; + ObCostBasedLateMaterializationCtx check_ctx; + trans_happened = false; + OPT_TRACE_BEGIN_SECTION; + OPT_TRACE("try to perform late materialization"); + if (OB_ISNULL(stmt)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("stmt is NULL", K(ret)); + } else if (!stmt->is_select_stmt()) { + /* do nothing */ + } else if (OB_FAIL(check_hint_validity(*stmt, force_trans, force_no_trans))) { + LOG_WARN("failed to check hint validity", K(ret)); + } else if (force_no_trans) { + OPT_TRACE("hint reject transform"); + } else if (FALSE_IT(select_stmt = static_cast(stmt))) { + } else if (OB_FAIL(check_stmt_need_late_materialization(*select_stmt, force_trans, can_trans))) { + LOG_WARN("failed to check stmt need late materialization", K(ret)); + } else if (!can_trans) { + OPT_TRACE("there is no need to late materialization"); + } else if (OB_FAIL(generate_late_materialization_info(*select_stmt, info, check_ctx))) { + LOG_WARN("failed to check if table need late materialization", K(ret)); + } else if (info.candi_indexs_.empty()) { + OPT_TRACE("not accept transform because candidate index is empty"); + } else if (OB_FAIL(inner_accept_transform(parent_stmts, stmt, force_trans, info, check_ctx, + trans_happened))) { + LOG_WARN("failed to do inner accept transform", K(ret)); + } else if (trans_happened && OB_FAIL(add_transform_hint(*stmt, &info))) { + LOG_WARN("failed to add transform hint", K(ret)); + } else { + LOG_TRACE("succeed to do late materialization", K(trans_happened)); + } + OPT_TRACE_END_SECTION; + return ret; +} + +int ObTransformLateMaterialization::check_hint_validity(const ObDMLStmt &stmt, + bool &force_trans, + bool &force_no_trans) +{ + int ret = OB_SUCCESS; + force_trans = false; + force_no_trans = false; + const ObQueryHint *query_hint = NULL; + const ObHint *trans_hint = NULL; + if (OB_ISNULL(query_hint = stmt.get_stmt_hint().query_hint_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(query_hint)); + } else if (stmt.get_stmt_hint().enable_no_rewrite()) { + force_no_trans = true; + } else if (query_hint->has_outline_data() && + NULL != (trans_hint = query_hint->get_outline_trans_hint(ctx_->trans_list_loc_))) { + force_trans = trans_hint->get_hint_type() == T_USE_LATE_MATERIALIZATION; + } else if (NULL != (trans_hint = stmt.get_stmt_hint().get_normal_hint(T_USE_LATE_MATERIALIZATION))) { + force_no_trans = trans_hint->is_disable_hint(); + force_trans = trans_hint->is_enable_hint(); + } else { + /* it deals with T_FULL_HINT because we can also handle a large column with full path*/ + } + return ret; +} + +int ObTransformLateMaterialization::check_stmt_need_late_materialization(const ObSelectStmt &stmt, + const bool force_accept, + bool &need) +{ + int ret = OB_SUCCESS; + int64_t child_stmt_size = 0; + const TableItem *table_item = NULL; + const ObTableSchema *table_schema = NULL; + ObSqlSchemaGuard *schema_guard = NULL; + bool contain_enumset_rowkey = false; + need = false; + if (OB_ISNULL(stmt.get_query_ctx()) || + OB_ISNULL(ctx_) || OB_ISNULL(schema_guard = ctx_->sql_schema_guard_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (OB_UNLIKELY(COMPAT_VERSION_4_2_5 > stmt.get_query_ctx()->optimizer_features_enable_version_) || + OB_UNLIKELY(COMPAT_VERSION_4_3_3 > stmt.get_query_ctx()->optimizer_features_enable_version_ && + COMPAT_VERSION_4_3_0 <= stmt.get_query_ctx()->optimizer_features_enable_version_)) { + /* need_transform = false; */ + LOG_TRACE("not late materialization stmt, optimizer feature enable is lower than 4_2_5 or lower than 4_3_3"); + } else if (OB_FAIL(stmt.get_child_stmt_size(child_stmt_size))) { + LOG_WARN("failed to get child stmt size", K(ret)); + } else if (stmt.has_hierarchical_query() || + stmt.has_group_by()|| + stmt.has_rollup() || + stmt.has_having() || + stmt.has_window_function() || + stmt.has_distinct() || + stmt.is_unpivot_select() || + 1 != stmt.get_from_item_size() || + 1 != stmt.get_table_size() || + NULL == stmt.get_table_item(0) || + !stmt.get_table_item(0)->is_basic_table() || + stmt.get_table_item(0)->is_system_table_ || + 0 != child_stmt_size || + stmt.is_calc_found_rows() || + !stmt.has_limit()) { + /* need_transform = false; */ + } else if (OB_ISNULL(table_item = stmt.get_table_item(0))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (OB_FAIL(schema_guard->get_table_schema(table_item->ref_id_, table_schema))) { + LOG_WARN("fail to get table schema", K(ret), K(table_item->ref_id_)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table_schema is NULL", K(ret)); + } else if (!table_schema->get_rowkey_info().is_valid()) { + /* need_transform = false; */ + } else if (!(stmt.has_order_by()) && + !(force_accept && table_schema->get_part_level() != PARTITION_LEVEL_ZERO && + stmt.has_limit() && (NULL != stmt.get_offset_expr() || NULL != stmt.get_limit_percent_expr()))) { + /* need_transform = false; */ + } else if (OB_FAIL(contain_enum_set_rowkeys(table_schema->get_rowkey_info(), + contain_enumset_rowkey))) { + LOG_WARN("check contain enumset rowkey failed", K(ret)); + } else if (contain_enumset_rowkey) { + //if there are enumset rowkeys, don't use late materialization since 'enumset_col = ?' cannot be used to extract query ranges + } else { + need = true; + } + return ret; +} + +int ObTransformLateMaterialization::generate_late_materialization_info( + const ObSelectStmt &select_stmt, + ObLateMaterializationInfo &info, + ObCostBasedLateMaterializationCtx &check_ctx) +{ + int ret = OB_SUCCESS; + const TableItem *table_item = NULL; + const ObTableSchema *table_schema = NULL; + ObSqlSchemaGuard *schema_guard = NULL; + ObSEArray key_col_ids; + ObSEArray filter_col_ids; + ObSEArray orderby_col_ids; + ObSEArray select_col_ids; + ObSEArray index_column_ids; + ObSEArray common_select_cols; + ObSEArray index_schemas; + if (OB_ISNULL(ctx_) || OB_ISNULL(schema_guard = ctx_->sql_schema_guard_) || + OB_ISNULL(table_item = select_stmt.get_table_item(0))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (OB_FAIL(schema_guard->get_table_schema(table_item->ref_id_, table_schema))) { + LOG_WARN("fail to get table schema", K(ret), K(table_item->ref_id_)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table_schema is NULL", K(ret)); + } else if (OB_FAIL(extract_transform_column_ids(select_stmt, *table_schema, key_col_ids, + filter_col_ids, orderby_col_ids, + select_col_ids))) { + LOG_WARN("failed to extract column ids", K(ret)); + } else if (OB_FAIL(get_accessible_index(select_stmt, *table_item, index_schemas))) { + LOG_WARN("get valid index schema", K(ret)); + } else if (OB_FAIL(common_select_cols.assign(select_col_ids))) { + LOG_WARN("failed to assign predicate col in view", K(ret)); + } else { + bool is_partition_table = table_schema->get_part_level() != PARTITION_LEVEL_ZERO; + for (int64_t i = 0; OB_SUCC(ret) && i < index_schemas.count(); ++i) { + const ObTableSchema *index_schema = NULL; + bool is_match = false; + ObString index_name; + index_column_ids.reuse(); + if (OB_ISNULL(index_schema = index_schemas.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("got null ptr", K(ret)); + } else if (OB_FAIL(index_schema->get_column_ids(index_column_ids))) { + LOG_WARN("failed to get column ids", K(ret)); + } else if (OB_FAIL(check_index_match_late_materialization(index_schema->get_table_id(), + index_column_ids, key_col_ids, filter_col_ids, + orderby_col_ids, select_col_ids, + table_item->ref_id_, check_ctx, is_match))) { + LOG_WARN("failed to check index match", K(ret)); + } else if (!is_match) { + /* do nothing */ + } else if (OB_FAIL(info.candi_indexs_.push_back(index_schema->get_table_id()))) { + LOG_WARN("failed to push back", K(ret)); + } else if (OB_FAIL(index_schema->get_index_name(index_name))) { + LOG_WARN("failed to get index name", K(ret)); + } else if (OB_FAIL(info.candi_index_names_.push_back(index_name))) { + LOG_WARN("failed to push back", K(ret)); + } else if (select_stmt.has_order_by() && + !(is_partition_table && index_schema->is_global_index_table()) && + OB_FAIL(check_ctx.check_sort_indexs_.push_back(index_schema->get_table_id()))) { + LOG_WARN("failed to push back", K(ret)); + } else { + for (int64_t i = common_select_cols.count() - 1; OB_SUCC(ret) && i >= 0; --i) { + if (!ObOptimizerUtil::find_item(index_column_ids, common_select_cols.at(i)) && + OB_FAIL(common_select_cols.remove(i))) { + LOG_WARN("failed to remove", K(ret)); + } + } + } + } + if (OB_SUCC(ret) && !info.candi_indexs_.empty()) { + if (OB_FAIL(info.project_col_in_view_.assign(key_col_ids))) { + LOG_WARN("failed to assign predicate col in view", K(ret)); + } else if (OB_FAIL(append_array_no_dup(info.project_col_in_view_, orderby_col_ids))) { + LOG_WARN("failed to append array no dup", K(ret)); + } else if (OB_FAIL(append_array_no_dup(info.project_col_in_view_, common_select_cols))) { + LOG_WARN("failed to append array no dup", K(ret)); + } else { + check_ctx.late_table_id_ = table_item->table_id_; + } + } + } + return ret; +} + +int ObTransformLateMaterialization::extract_transform_column_ids(const ObSelectStmt &select_stmt, + const ObTableSchema &table_schema, + ObIArray &key_col_ids, + ObIArray &filter_col_ids, + ObIArray &orderby_col_ids, + ObIArray &select_col_ids) +{ + int ret = OB_SUCCESS; + ObSEArray temp_exprs; + ObSEArray part_col_ids; + if (OB_FAIL(table_schema.get_rowkey_info().get_column_ids(key_col_ids))) { + LOG_WARN("get rowkey column ids failed", K(ret)); + } else if (table_schema.get_partition_key_info().is_valid() && + OB_FAIL(table_schema.get_partition_key_info().get_column_ids(part_col_ids))) { + LOG_WARN("get partition column ids failed", K(ret)); + } else if (table_schema.get_subpartition_key_info().is_valid() && + OB_FAIL(table_schema.get_subpartition_key_info().get_column_ids(part_col_ids))) { + LOG_WARN("get subpartition column ids failed", K(ret)); + } else if (OB_FAIL(append_array_no_dup(key_col_ids, part_col_ids))) { + LOG_WARN("failed to append", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::extract_column_ids(select_stmt.get_condition_exprs(), + filter_col_ids))) { + LOG_WARN("failed to extract column ids", K(ret)); + } else if (OB_FAIL(select_stmt.get_order_exprs(temp_exprs))) { + LOG_WARN("ger order exprs failed", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::extract_column_ids(temp_exprs, orderby_col_ids))) { + LOG_WARN("failed to extract column ids", K(ret)); + } else if (FALSE_IT(temp_exprs.reuse())) { + } else if (OB_FAIL(select_stmt.get_select_exprs(temp_exprs))) { + LOG_WARN("get select exprs failed", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::extract_column_ids(temp_exprs, select_col_ids))) { + LOG_WARN("failed to get stored column ids", K(ret)); + } + return ret; +} + +/* + * index > full, index > no_index + * full > use_late_materialization +*/ +int ObTransformLateMaterialization::get_accessible_index(const ObSelectStmt &select_stmt, + const TableItem &table_item, + ObIArray &index_schemas) +{ + int ret = OB_SUCCESS; + ObSEArray hint_index_ids; + ObSEArray hint_no_index_ids; + ObSEArray index_ids; + ObSEArray tmp_index_schemas; + ObSqlSchemaGuard *schema_guard = NULL; + const ObQueryHint *query_hint = NULL; + bool has_full_hint = false; + if (OB_ISNULL(ctx_) || OB_ISNULL(schema_guard = ctx_->sql_schema_guard_) || + OB_ISNULL(query_hint = select_stmt.get_stmt_hint().query_hint_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (OB_FAIL(ObTransformUtils::get_vaild_index_id(schema_guard, &select_stmt, &table_item, + index_ids))) { + LOG_WARN("fail to get vaild index id", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < index_ids.count(); ++i) { + const ObTableSchema *index_schema = NULL; + if (OB_FAIL(schema_guard->get_table_schema(index_ids.at(i), index_schema))) { + LOG_WARN("fail to get index schema", K(ret), K(index_ids.at(i))); + } else if (OB_ISNULL(index_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get null index schema", K(ret)); + } else if (!index_schema->get_rowkey_info().is_valid() || + !index_schema->is_index_table()) { + /* do nothing */ + } else { + ObIndexType index_type = index_schema->get_index_type(); + if (INDEX_TYPE_NORMAL_LOCAL == index_type || + INDEX_TYPE_UNIQUE_LOCAL == index_type || + INDEX_TYPE_NORMAL_GLOBAL == index_type || + INDEX_TYPE_UNIQUE_GLOBAL == index_type || + INDEX_TYPE_PRIMARY == index_type || + INDEX_TYPE_NORMAL_GLOBAL_LOCAL_STORAGE == index_type || + INDEX_TYPE_UNIQUE_GLOBAL_LOCAL_STORAGE == index_type) { + if (OB_FAIL(tmp_index_schemas.push_back(index_schema))) { + LOG_WARN("failed to push back", K(ret)); + } + } + } + } + } + const ObIArray &opt_hints = select_stmt.get_stmt_hint().other_opt_hints_; + for (int64_t i = 0; OB_SUCC(ret) && i < opt_hints.count(); ++i) { + ObIndexHint *index_hint = NULL; + int64_t index_schema_i = 0; + if (OB_ISNULL(opt_hints.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (!opt_hints.at(i)->is_access_path_hint()) { + /* do nothing */ + } else if (FALSE_IT(index_hint = static_cast(opt_hints.at(i)))) { + /* do nothing */ + } else if (T_INDEX_HINT == index_hint->get_hint_type() || + T_INDEX_SS_HINT == index_hint->get_hint_type() || + T_INDEX_SS_ASC_HINT == index_hint->get_hint_type() || + T_INDEX_SS_DESC_HINT == index_hint->get_hint_type()) { + if (match_index_name(tmp_index_schemas, table_item, *index_hint, query_hint->cs_type_, index_schema_i)) { + if (OB_FAIL(hint_index_ids.push_back(tmp_index_schemas.at(index_schema_i)->get_table_id()))) { + LOG_WARN("push back hint index name failed", K(ret)); + } + } + } else if (T_NO_INDEX_HINT == index_hint->get_hint_type()) { + if (match_index_name(tmp_index_schemas, table_item, *index_hint, query_hint->cs_type_, index_schema_i)) { + if (OB_FAIL(hint_no_index_ids.push_back(tmp_index_schemas.at(index_schema_i)->get_table_id()))) { + LOG_WARN("push back hint index name failed", K(ret)); + } + } + } else if (T_FULL_HINT == index_hint->get_hint_type()) { + has_full_hint = true; + } + } + if (OB_SUCC(ret) && hint_index_ids.empty() && has_full_hint) { + tmp_index_schemas.reset(); + } + for (int64_t i = 0; OB_SUCC(ret) && i < tmp_index_schemas.count(); ++i) { + const ObTableSchema *index_schema = tmp_index_schemas.at(i); + if (OB_ISNULL(index_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get null index schema", K(ret)); + } else if (!hint_index_ids.empty() && + !ObOptimizerUtil::find_item(hint_index_ids, index_schema->get_table_id())) { + /* do nothing */ + } else if (!hint_no_index_ids.empty() && + ObOptimizerUtil::find_item(hint_no_index_ids, index_schema->get_table_id()) && + !ObOptimizerUtil::find_item(hint_index_ids, index_schema->get_table_id())) { + /* do nothing */ + } else if (OB_FAIL(index_schemas.push_back(index_schema))) { + LOG_WARN("failed to push back", K(ret)); + } + } + return ret; +} + +bool ObTransformLateMaterialization::match_index_name( + const ObIArray &index_schemas, + const TableItem &table_item, + const ObIndexHint &index_hint, + const ObCollationType cs_type, + int64_t &id) +{ + bool match = false; + id = -1; + for (int64_t i = 0; !match && i < index_schemas.count(); ++i) { + if (index_hint.is_match_index(cs_type, table_item, *index_schemas.at(i))) { + id = i; + match = true; + } + } + return match; +} + +int ObTransformLateMaterialization::check_index_match_late_materialization( + const uint64_t index_id, + const ObIArray &index_column_ids, + const ObIArray &key_col_ids, + const ObIArray &filter_col_ids, + const ObIArray &orderby_col_ids, + const ObIArray &select_col_ids, + const uint64_t ref_table_id, + ObCostBasedLateMaterializationCtx &check_ctx, + bool &is_match) +{ + int ret = OB_SUCCESS; + ObSEArray tmp_index_column_ids; + ObSqlSchemaGuard *schema_guard = NULL; + if (OB_ISNULL(ctx_) || OB_ISNULL(schema_guard = ctx_->sql_schema_guard_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (ObOptimizerUtil::is_subset(select_col_ids, index_column_ids) || + !ObOptimizerUtil::is_subset(key_col_ids, index_column_ids) || + !ObOptimizerUtil::is_subset(orderby_col_ids, index_column_ids)) { + /* do nothing */ + } else if (ObOptimizerUtil::is_subset(filter_col_ids, index_column_ids)) { + is_match = true; + if (OB_FAIL(check_ctx.late_material_indexs_.push_back(index_id))) { + LOG_WARN("failed to push back", K(ret)); + } + } else { + // it may match function index. + const int64_t cnt = index_column_ids.count(); + for (int64_t i = 0; OB_SUCC(ret) && i < cnt; ++i) { + const ObColumnSchemaV2 *index_col = NULL; + if (OB_FAIL(schema_guard->get_column_schema(ref_table_id, index_column_ids.at(i), index_col))) { + LOG_WARN("failed to get column schema", K(ret)); + } else if (NULL == index_col) { + /* do nothing : unique key will has a shadow_pk_0 column which no used here */ + } else if (index_col->is_func_idx_column()) { + if (OB_FAIL(index_col->get_cascaded_column_ids(tmp_index_column_ids))) { + LOG_WARN("failed to get column ref ids", K(ret)); + } + } + } + if (OB_SUCC(ret) && !tmp_index_column_ids.empty()) { + if (OB_FAIL(append(tmp_index_column_ids, index_column_ids))) { + LOG_WARN("failed to append array", K(ret)); + } else if (ObOptimizerUtil::is_subset(filter_col_ids, tmp_index_column_ids)) { + is_match = true; + } + } + } + return ret; +} + +int ObTransformLateMaterialization::evaluate_stmt_cost(ObDMLStmt *&stmt, + bool is_trans_stmt, + double &plan_cost, + bool &is_expected, + ObCostBasedLateMaterializationCtx &check_ctx) +{ + int ret = OB_SUCCESS; + ObEvalCostHelper eval_cost_helper; + is_expected = true; + if (OB_ISNULL(stmt) || OB_ISNULL(ctx_) || OB_UNLIKELY(!ctx_->is_valid()) || + OB_ISNULL(ctx_->exec_ctx_->get_physical_plan_ctx()) || + OB_ISNULL(ctx_->exec_ctx_->get_stmt_factory()) || + OB_ISNULL(ctx_->exec_ctx_->get_stmt_factory()->get_query_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("params are invalid", K(ret), K(stmt), K(ctx_)); + } else if (OB_FAIL(eval_cost_helper.fill_helper(*ctx_->exec_ctx_->get_physical_plan_ctx(), + *stmt->get_query_ctx(), *ctx_))) { + LOG_WARN("failed to fill eval cost helper", K(ret)); + } else if (!is_trans_stmt) { + /* do nothing */ + } else if (OB_FAIL(construct_transform_hint(*stmt, NULL))) { + LOG_WARN("failed to construct transform hint", K(ret)); + } else if (OB_FAIL(stmt->adjust_qb_name(ctx_->allocator_, + ctx_->src_qb_name_, + ctx_->src_hash_val_))) { + LOG_WARN("failed to adjust qb name", K(ret)); + } + if (OB_SUCC(ret)) { + ctx_->eval_cost_ = true; + lib::ContextParam param; + param.set_mem_attr(ctx_->session_info_->get_effective_tenant_id(), + "CostBasedRewrit", + ObCtxIds::DEFAULT_CTX_ID) + .set_properties(lib::USE_TL_PAGE_OPTIONAL) + .set_page_size(OB_MALLOC_NORMAL_BLOCK_SIZE); + CREATE_WITH_TEMP_CONTEXT(param) { + ObRawExprFactory tmp_expr_factory(CURRENT_CONTEXT->get_arena_allocator()); + HEAP_VAR(ObOptimizerContext, + optctx, + ctx_->session_info_, + ctx_->exec_ctx_, + ctx_->sql_schema_guard_, + ctx_->opt_stat_mgr_, + CURRENT_CONTEXT->get_arena_allocator(), + &ctx_->exec_ctx_->get_physical_plan_ctx()->get_param_store(), + *ctx_->self_addr_, + GCTX.srv_rpc_proxy_, + stmt->get_query_ctx()->get_global_hint(), + tmp_expr_factory, + stmt, + false, + ctx_->exec_ctx_->get_stmt_factory()->get_query_ctx()) { + ObOptimizer optimizer(optctx); + ObLogPlan *plan = NULL; + if (OB_FAIL(optimizer.get_optimization_cost(*stmt, plan, plan_cost))) { + LOG_WARN("failed to get optimization cost", K(ret)); + } else if (OB_FAIL(is_expected_plan(plan, &check_ctx, is_trans_stmt, is_expected))) { + LOG_WARN("failed to check transformed plan", K(ret)); + } else if (OB_FAIL(eval_cost_helper.recover_context( + *ctx_->exec_ctx_->get_physical_plan_ctx(), + *ctx_->exec_ctx_->get_stmt_factory()->get_query_ctx(), + *ctx_))) { + LOG_WARN("failed to recover context", K(ret)); + } + } + } + } + return ret; +} + +int ObTransformLateMaterialization::inner_accept_transform(ObIArray &parent_stmts, + ObDMLStmt *&stmt, + const bool force_accept, + ObLateMaterializationInfo &info, + ObCostBasedLateMaterializationCtx &check_ctx, + bool &trans_happened) +{ + int ret = OB_SUCCESS; + double trans_stmt_cost = -1; + double base_stmt_cost = -1; + bool is_expected = false; + bool is_base_expected = false; + ObDMLStmt *tmp1 = NULL; + ObDMLStmt *tmp2 = NULL; + ObDMLStmt *trans_stmt = NULL; + ObTryTransHelper try_trans_helper; + cost_based_trans_tried_ = true; + trans_happened = false; + STOP_OPT_TRACE; + if (OB_ISNULL(ctx_) || OB_ISNULL(stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("context is null", K(ret), K(ctx_), K(stmt)); + } else if (ctx_->in_accept_transform_) { + LOG_TRACE("not accept transform because already in one accepct transform"); + } else { + ctx_->in_accept_transform_ = true; + if (OB_SUCC(ret)) { + if (OB_FAIL(try_trans_helper.fill_helper(stmt->get_query_ctx()))) { + LOG_WARN("failed to fill try trans helper", K(ret)); + } else if (OB_FAIL(generate_late_materialization_stmt(info, stmt, trans_stmt))) { + LOG_WARN("perform late materialization failed", K(ret)); + } else if (force_accept && stmt->get_stmt_hint().query_hint_->has_outline_data()) { + trans_happened = true; + LOG_TRACE("force accept to use late materialization"); + } else if (OB_FAIL(evaluate_stmt_cost(trans_stmt, true, trans_stmt_cost, is_expected, check_ctx))) { + LOG_WARN("failed to evaluate cost for the transformed stmt", K(ret)); + } else if (!is_expected) { + trans_happened = false; + } else if (!force_accept && OB_FAIL(evaluate_stmt_cost(stmt, false, base_stmt_cost, + is_base_expected, check_ctx))) { + LOG_WARN("failed to evaluate cost of select_stmt"); + } else { + trans_happened = force_accept || + (base_stmt_cost > 0 && trans_stmt_cost > 0 && trans_stmt_cost < base_stmt_cost) || + // we always consider best late_materialization index is better to the full path + (check_ctx.base_index_ != common::OB_INVALID_ID && + ObOptimizerUtil::find_item(check_ctx.late_material_indexs_, check_ctx.base_index_)); + } + if (OB_SUCC(ret) && !trans_happened && + OB_FAIL(try_trans_helper.recover(stmt->get_query_ctx()))) { + LOG_WARN("failed to recover params", K(ret)); + } + } + ctx_->in_accept_transform_ = false; + } + RESUME_OPT_TRACE; + + if (OB_FAIL(ret)) { + } else if (!trans_happened) { + OPT_TRACE("reject transform because the cost is increased or the query plan is unexpected."); + OPT_TRACE("before transform cost:", base_stmt_cost); + OPT_TRACE("after transform cost:", trans_stmt_cost); + OPT_TRACE("is expected plan:", is_expected); + LOG_TRACE("reject transform because the cost is increased or the query plan is unexpected", + K(base_stmt_cost), K(trans_stmt_cost), K(is_expected), K(check_ctx)); + } else if (OB_FAIL(adjust_transformed_stmt(parent_stmts, trans_stmt, tmp1, tmp2))) { + LOG_WARN("failed to adjust transformed stmt", K(ret)); + } else if (force_accept) { + OPT_TRACE("hint or rule force cost based transform apply."); + LOG_TRACE("succeed force accept transform because hint/rule"); + stmt = trans_stmt; + reset_stmt_cost(); + } else { + OPT_TRACE("accept transform because the cost is decreased."); + OPT_TRACE("before transform cost:", base_stmt_cost); + OPT_TRACE("after transform cost:", trans_stmt_cost); + LOG_TRACE("accept transform because the cost is decreased", K(base_stmt_cost), K(trans_stmt_cost), + K(check_ctx)); + stmt = trans_stmt; + reset_stmt_cost(); + } + return ret; +} + +int ObTransformLateMaterialization::generate_late_materialization_stmt( + const ObLateMaterializationInfo &info, + ObDMLStmt *stmt, + ObDMLStmt *&trans_stmt) +{ + int ret = OB_SUCCESS; + ObSEArray old_col_exprs; + ObSEArray new_col_exprs; + const TableItem *table_item = NULL; + TableItem *view_table = NULL; + ObSelectStmt *view_stmt = NULL; + ObSelectStmt *select_stmt = NULL; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->stmt_factory_) || OB_ISNULL(ctx_->expr_factory_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_FAIL(ObTransformUtils::deep_copy_stmt(*ctx_->stmt_factory_, *ctx_->expr_factory_, + stmt, trans_stmt))) { + LOG_WARN("failed to deep copy select stmt", K(ret)); + } else if (FALSE_IT(select_stmt = static_cast(trans_stmt))) { + } else if (OB_ISNULL(table_item = select_stmt->get_table_item(0))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table item is NULL", K(ret)); + } else if (OB_FAIL(generate_late_materialization_view(info.project_col_in_view_, + select_stmt, view_stmt, view_table))) { + LOG_WARN("failed to generate late materialization view", K(ret)); + } else if (OB_FAIL(extract_replace_column_exprs(*select_stmt, *view_stmt, table_item->table_id_, + view_table->table_id_, old_col_exprs, + new_col_exprs))) { + LOG_WARN("failed to extract replace column exprs", K(ret)); + } else if (OB_FAIL(select_stmt->replace_relation_exprs(old_col_exprs, new_col_exprs))) { + LOG_WARN("failed to replace inner stmt expr", K(ret)); + } else if (OB_FAIL(generate_pk_join_conditions(table_item->ref_id_, table_item->table_id_, + old_col_exprs, new_col_exprs, *select_stmt))) { + LOG_WARN("failed generate pk join condition", K(ret)); + } else if (OB_FAIL(ObTransformUtils::adjust_pseudo_column_like_exprs(*select_stmt))) { + LOG_WARN("failed to adjust pseudo column like exprs", K(ret)); + } else if (OB_FAIL(select_stmt->formalize_stmt(ctx_->session_info_))) { + LOG_WARN("failed to formalize stmt", K(ret)); + } else if (OB_FAIL(select_stmt->formalize_stmt_expr_reference(ctx_->expr_factory_, + ctx_->session_info_))) { + LOG_WARN("failed to formalize stmt expr reference", K(ret)); + } else if (OB_FAIL(generate_late_materialization_hint(info, *table_item, *view_table, + *select_stmt, *view_stmt))) { + LOG_WARN("generate late materialization hint failed", K(ret)); + } else { + LOG_TRACE("got late materialization info", K(info)); + } + return ret; +} + +int ObTransformLateMaterialization::generate_late_materialization_view( + const ObIArray &select_col_ids, + ObSelectStmt *select_stmt, + ObSelectStmt *&view_stmt, + TableItem *&view_item) +{ + int ret = OB_SUCCESS; + const TableItem *table_item_inner = NULL; + ObDMLStmt *tmp_stmt = NULL; + ObSEArray select_col_exprs; + ObSEArray columns; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->stmt_factory_) || OB_ISNULL(ctx_->expr_factory_) || + OB_UNLIKELY(select_col_ids.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("got unexpected param", K(ret)); + } else if (OB_FAIL(ObTransformUtils::deep_copy_stmt(*ctx_->stmt_factory_, *ctx_->expr_factory_, + select_stmt, tmp_stmt))) { + LOG_WARN("failed to deep copy select stmt", K(ret)); + } else if (FALSE_IT(view_stmt = static_cast(tmp_stmt))) { + } else if (OB_FAIL(view_stmt->adjust_statement_id(ctx_->allocator_, ctx_->src_qb_name_, ctx_->src_hash_val_))) { + LOG_WARN("failed to recursive adjust statement id", K(ret)); + } else if (OB_FAIL(view_stmt->update_stmt_table_id(ctx_->allocator_, *select_stmt))) { + LOG_WARN("failed to update table id", K(ret)); + } else if (OB_ISNULL(table_item_inner = view_stmt->get_table_item(0))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else { + view_stmt->get_select_items().reset(); + for (int64_t i = 0; OB_SUCC(ret) && i < select_col_ids.count(); i++) { + ObRawExpr *raw_expr = view_stmt->get_column_expr_by_id(table_item_inner->table_id_, + select_col_ids.at(i)); + if (OB_ISNULL(raw_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (OB_FAIL(select_col_exprs.push_back(raw_expr))) { + LOG_WARN("failed to push back", K(ret)); + } + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(ObTransformUtils::create_select_item(*ctx_->allocator_, + select_col_exprs, + view_stmt))) { + LOG_WARN("failed to create select items", K(ret)); + } else if (OB_FAIL(ObTransformUtils::add_new_table_item(ctx_, select_stmt, view_stmt, view_item))) { + LOG_WARN("failed to add new table item", K(ret)); + } else if (OB_FAIL(select_stmt->add_from_item(view_item->table_id_))) { + LOG_WARN("failed to add from item", K(ret)); + } else if (OB_FAIL(select_stmt->rebuild_tables_hash())) { + LOG_WARN("failed to rebuid table hash", K(ret)); + } else if (OB_FAIL(ObTransformUtils::create_columns_for_view(ctx_, *view_item, select_stmt, columns))) { + LOG_WARN("failed to create columns for view", K(ret)); + } else { + select_stmt->get_condition_exprs().reset(); + select_stmt->set_limit_offset(NULL, NULL); + select_stmt->set_limit_percent_expr(NULL); + select_stmt->set_fetch_with_ties(false); + select_stmt->set_has_fetch(false); + } + return ret; +} + +int ObTransformLateMaterialization::extract_replace_column_exprs(const ObSelectStmt &select_stmt, + const ObSelectStmt &view_stmt, + const uint64_t table_id, + const uint64_t view_id, + ObIArray &old_col_exprs, + ObIArray &new_col_exprs) +{ + int ret = OB_SUCCESS; + ObRawExpr *col_expr = NULL; + ObRawExpr *old_col_expr = NULL; + ObRawExpr *new_col_expr = NULL; + old_col_exprs.reset(); + new_col_exprs.reset(); + for (int64_t i = 0; OB_SUCC(ret) && i < view_stmt.get_select_item_size(); ++i) { + if (OB_ISNULL(col_expr = view_stmt.get_select_item(i).expr_) || + OB_UNLIKELY(!col_expr->is_column_ref_expr())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_FAIL(old_col_exprs.push_back(col_expr))) { + LOG_WARN("failed to push back", K(ret)); + } else if (OB_ISNULL(new_col_expr = select_stmt.get_column_expr_by_id(view_id, + i + OB_APP_MIN_COLUMN_ID))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_FAIL(new_col_exprs.push_back(new_col_expr))) { + LOG_WARN("failed to push back", K(ret)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < old_col_exprs.count(); ++i) { + if (OB_ISNULL(old_col_exprs.at(i)) || + OB_ISNULL(old_col_expr = select_stmt.get_column_expr_by_id(table_id, + static_cast(old_col_exprs.at(i))->get_column_id()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("got unexpected param", K(ret)); + } else { + old_col_exprs.at(i) = old_col_expr; + } + } + return ret; +} + +int ObTransformLateMaterialization::generate_pk_join_conditions(const uint64_t ref_table_id, + const uint64_t table_id, + const ObIArray &old_col_exprs, + const ObIArray &new_col_exprs, + ObSelectStmt &select_stmt) +{ + int ret = OB_SUCCESS; + ObSqlSchemaGuard *schema_guard = NULL; + const ObTableSchema *table_schema = NULL; + ObSEArray key_col_ids; + ObSEArray part_col_ids; + const bool is_mysql_mode = lib::is_mysql_mode(); + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->expr_factory_) || + OB_ISNULL(schema_guard = ctx_->sql_schema_guard_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_FAIL(schema_guard->get_table_schema(ref_table_id, table_schema))) { + LOG_WARN("fail to get table schema", K(ret), K(ref_table_id)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table_schema is NULL", K(ret)); + } else if (OB_FAIL(table_schema->get_rowkey_info().get_column_ids(key_col_ids))) { + LOG_WARN("get rowkey column ids failed", K(ret)); + } else if (table_schema->get_partition_key_info().is_valid() && + OB_FAIL(table_schema->get_partition_key_info().get_column_ids(part_col_ids))) { + LOG_WARN("get partition column ids failed", K(ret)); + } else if (table_schema->get_subpartition_key_info().is_valid() && + OB_FAIL(table_schema->get_subpartition_key_info().get_column_ids(part_col_ids))) { + LOG_WARN("get subpartition column ids failed", K(ret)); + } else if (OB_FAIL(append_array_no_dup(key_col_ids, part_col_ids))) { + LOG_WARN("failed to append", K(ret)); + } else { + ObNotNullContext not_null_ctx(*ctx_, &select_stmt); + if (OB_FAIL(not_null_ctx.generate_stmt_context(NULLABLE_SCOPE::NS_FROM))){ + LOG_WARN("failed to generate not null context", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < key_col_ids.count(); ++i) { + uint64_t column_id = key_col_ids.at(i); + ObRawExpr *equal_expr = NULL; + ObRawExpr *view_col = NULL; + ObRawExpr *table_col = NULL; + int64_t idx = -1; + bool is_not_null = false; + ObArray constraints; + if (OB_ISNULL(table_col = select_stmt.get_column_expr_by_id(table_id, column_id)) || + OB_UNLIKELY(!ObOptimizerUtil::find_item(old_col_exprs, table_col, &idx)) || + OB_UNLIKELY(idx < 0 || idx >= new_col_exprs.count()) || + OB_ISNULL(view_col = new_col_exprs.at(idx))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error", K(ret)); + } else if (OB_FAIL(ObTransformUtils::is_expr_not_null(not_null_ctx, view_col, + is_not_null, &constraints))) { + LOG_WARN("failed to check expr not null", K(ret)); + } else if (is_not_null) { + if (OB_FAIL(ObRawExprUtils::create_equal_expr(*ctx_->expr_factory_, ctx_->session_info_, + view_col, table_col, equal_expr))) { + LOG_WARN("failed to create equal expr", K(ret)); + } else if (OB_FAIL(ObTransformUtils::add_param_not_null_constraint(*ctx_, constraints))) { + LOG_WARN("failed to add param not null constraint", K(ret)); + } else if (OB_FAIL(select_stmt.get_condition_exprs().push_back(equal_expr))) { + LOG_WARN("push back join condition failed", K(ret)); + } + } else { + if (OB_FAIL(ObRawExprUtils::create_null_safe_equal_expr(*ctx_->expr_factory_, + ctx_->session_info_, is_mysql_mode, table_col, view_col, equal_expr))) { + LOG_WARN("failed to create null safe equal expr", K(ret)); + } else if (OB_FAIL(select_stmt.get_condition_exprs().push_back(equal_expr))) { + LOG_WARN("push back join condition failed", K(ret)); + } + } + } + } + return ret; +} + +int ObTransformLateMaterialization::generate_late_materialization_hint( + const ObLateMaterializationInfo &info, + const TableItem &table_item, + const TableItem &view_table, + ObSelectStmt &select_stmt, + ObSelectStmt &view_stmt) +{ + int ret = OB_SUCCESS; + const TableItem *table_item_inner = NULL; + ObSqlSchemaGuard *schema_guard = NULL; + ObSEArray conflict_hints; + ObString parent_qb_name; + ObString view_qb_name; + if (OB_ISNULL(ctx_) || OB_ISNULL(schema_guard = ctx_->sql_schema_guard_) || + OB_ISNULL(select_stmt.get_stmt_hint().query_hint_) || + OB_ISNULL(table_item_inner = view_stmt.get_table_item(0))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (select_stmt.get_stmt_hint().query_hint_->has_outline_data()) { + /* do nothing */ + } else if (OB_FAIL(select_stmt.get_qb_name(parent_qb_name))) { + LOG_WARN("get qb name failed", K(ret)); + } else if (OB_FAIL(view_stmt.get_qb_name(view_qb_name))) { + LOG_WARN("get view qb name failed", K(ret)); + } else { + // nlj_hint + if (OB_SUCC(ret)) { + ObJoinHint *join_hint = NULL; + ObTableInHint table_in_hint(table_item.qb_name_, table_item.database_name_, table_item.get_object_name()); + if (OB_FAIL(ObQueryHint::create_hint(ctx_->allocator_, T_USE_NL, join_hint))) { + LOG_WARN("failed to create hint", K(ret)); + } else if (OB_ISNULL(join_hint)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_FAIL(join_hint->get_tables().push_back(table_in_hint))) { + LOG_WARN("failde to push back", K(ret)); + } else { + join_hint->set_qb_name(parent_qb_name); + join_hint->set_trans_added(true); + if (OB_FAIL(select_stmt.get_stmt_hint().merge_hint(*join_hint, + HINT_DOMINATED_EQUAL, + conflict_hints))) { + LOG_WARN("merge join hint failed", K(ret)); + } + } + } + // leading_hint + if (OB_SUCC(ret)) { + ObJoinOrderHint *join_order_hint = NULL; + ObLeadingTable *leading_table = NULL; + ObLeadingTable *left_leading_table = NULL; + ObLeadingTable *right_leading_table = NULL; + if (OB_FAIL(ObQueryHint::create_hint(ctx_->allocator_, T_LEADING, join_order_hint))) { + LOG_WARN("failed to create hint", K(ret)); + } else if (OB_ISNULL(join_order_hint)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_FAIL(ObQueryHint::create_leading_table(ctx_->allocator_, left_leading_table))) { + LOG_WARN("failed to create leading table", K(ret)); + } else if (OB_FAIL(ObQueryHint::create_leading_table(ctx_->allocator_, right_leading_table))) { + LOG_WARN("failed to create leading table", K(ret)); + } else if (OB_ISNULL(left_leading_table) || OB_ISNULL(right_leading_table)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_FAIL(ObQueryHint::create_hint_table(ctx_->allocator_, left_leading_table->table_))) { + LOG_WARN("fail to create hint table", K(ret)); + } else if (OB_FAIL(ObQueryHint::create_hint_table(ctx_->allocator_, right_leading_table->table_))) { + LOG_WARN("fail to create hint table", K(ret)); + } else if (OB_ISNULL(left_leading_table->table_) || OB_ISNULL(right_leading_table->table_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else { + left_leading_table->table_->qb_name_ = view_table.qb_name_; + left_leading_table->table_->db_name_ = view_table.database_name_; + left_leading_table->table_->table_name_ = view_table.get_object_name(); + right_leading_table->table_->qb_name_ = table_item.qb_name_; + right_leading_table->table_->db_name_ = table_item.database_name_; + right_leading_table->table_->table_name_ = table_item.get_object_name(); + join_order_hint->get_table().left_table_ = left_leading_table; + join_order_hint->get_table().right_table_ = right_leading_table; + join_order_hint->set_qb_name(parent_qb_name); + join_order_hint->set_trans_added(true); + if (OB_FAIL(select_stmt.get_stmt_hint().merge_hint(*join_order_hint, + HINT_DOMINATED_EQUAL, + conflict_hints))) { + LOG_WARN("merge join hint failed", K(ret)); + } + } + } + // index hint + if (OB_SUCC(ret)) { + ObIArray &opt_hints = select_stmt.get_stmt_hint().other_opt_hints_; + for (int64_t i = opt_hints.count() - 1; OB_SUCC(ret) && i >= 0; --i) { + ObHint* hint = opt_hints.at(i); + if (OB_ISNULL(hint)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), KP(hint)); + } else if (hint->is_access_path_hint()) { + ObIndexHint *index_hint = static_cast(hint); + if ((T_INDEX_HINT == index_hint->get_hint_type() || + T_INDEX_SS_HINT == index_hint->get_hint_type() || + T_INDEX_SS_ASC_HINT == index_hint->get_hint_type() || + T_INDEX_SS_DESC_HINT == index_hint->get_hint_type() || + T_NO_INDEX_HINT == index_hint->get_hint_type() || + T_FULL_HINT == index_hint->get_hint_type())) { + if (OB_FAIL(opt_hints.remove(i))) { + LOG_WARN("failed to remove opt hint"); + } + } + } + } + ObIArray &view_opt_hints = view_stmt.get_stmt_hint().other_opt_hints_; + for (int64_t i = view_opt_hints.count() - 1; OB_SUCC(ret) && i >= 0; --i) { + ObHint* hint = view_opt_hints.at(i); + if (OB_ISNULL(hint)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), KP(hint)); + } else if (hint->is_access_path_hint()) { + ObIndexHint *index_hint = static_cast(hint); + if ((T_INDEX_HINT == index_hint->get_hint_type() || + T_INDEX_SS_HINT == index_hint->get_hint_type() || + T_INDEX_SS_ASC_HINT == index_hint->get_hint_type() || + T_INDEX_SS_DESC_HINT == index_hint->get_hint_type() || + T_NO_INDEX_HINT == index_hint->get_hint_type() || + T_FULL_HINT == index_hint->get_hint_type())) { + if (OB_FAIL(view_opt_hints.remove(i))) { + LOG_WARN("failed to remove opt hint"); + } + } + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < info.candi_indexs_.count(); ++i) { + ObTableInHint table_in_hint(table_item_inner->qb_name_, + table_item_inner->database_name_, + table_item_inner->get_object_name()); + ObIndexHint *index_hint = NULL; + if (OB_FAIL(ObQueryHint::create_hint(ctx_->allocator_, T_INDEX_HINT, index_hint))) { + LOG_WARN("failed to create hint", K(ret)); + } else if (OB_FAIL(index_hint->get_table().assign(table_in_hint))) { + LOG_WARN("assign table in hint failed", K(ret)); + } else { + index_hint->get_index_name() = info.candi_index_names_.at(i); + index_hint->set_qb_name(view_qb_name); + index_hint->set_trans_added(true); + if (OB_FAIL(view_stmt.get_stmt_hint().merge_hint(*index_hint, + HINT_DOMINATED_EQUAL, + conflict_hints))) { + LOG_WARN("merge index hint failed", K(ret)); + } + } + } + } + if (OB_SUCC(ret)) { + // ObTransHint *trans_hint = NULL; + ObViewMergeHint *no_merge_hint = NULL; + // view_merge_hint->set_is_used_query_push_down(hint_node.value_ == 1); + if (OB_FAIL(ObQueryHint::create_hint(ctx_->allocator_, T_NO_MERGE_HINT, no_merge_hint))) { + LOG_WARN("failed to create hint", K(ret)); + } else { + no_merge_hint->set_parent_qb_name(parent_qb_name); + no_merge_hint->set_qb_name(view_qb_name); + if (OB_FAIL(select_stmt.get_stmt_hint().merge_hint(*no_merge_hint, + HINT_DOMINATED_EQUAL, + conflict_hints))) { + LOG_WARN("merge no expand hint failed", K(ret)); + } + } + } + } + return ret; +} + +int ObTransformLateMaterialization::is_expected_plan(ObLogPlan *plan, + void *check_ctx, + bool is_trans_plan, + bool &is_expected) +{ + int ret = OB_SUCCESS; + ObCostBasedLateMaterializationCtx *local_ctx = + static_cast(check_ctx); + is_expected = false; + if (OB_ISNULL(plan) || OB_ISNULL(local_ctx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect null param", K(ret)); + } else if (is_trans_plan) { + if (OB_FAIL(check_transform_plan_expected(plan->get_plan_root(), *local_ctx, is_expected))) { + LOG_WARN("failed to check transform plan expected", K(ret)); + } + } else if (!is_trans_plan) { + if (OB_FAIL(get_index_of_base_stmt_path(plan->get_plan_root(), *local_ctx))) { + LOG_WARN("failed to get base stmt best index", K(ret)); + } else { + is_expected = true; + } + } + return ret; +} + +int ObTransformLateMaterialization::get_index_of_base_stmt_path(ObLogicalOperator *top, + ObCostBasedLateMaterializationCtx &ctx) +{ + int ret = OB_SUCCESS; + ObLogicalOperator *limit_op = NULL; + ObLogicalOperator *sort_op = NULL; + if (OB_ISNULL(top)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), KP(top)); + } else { + while (OB_SUCC(ret) && 1 <= top->get_num_of_child() && log_op_def::LOG_TABLE_SCAN != top->get_type()) { + if (log_op_def::LOG_LIMIT == top->get_type()) { + limit_op = top; + } else if (log_op_def::LOG_SORT == top->get_type()) { + sort_op = top; + } + if (OB_ISNULL(top = top->get_child(ObLogicalOperator::first_child))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), KP(top)); + } + } + if (OB_SUCC(ret) && OB_UNLIKELY(log_op_def::LOG_TABLE_SCAN != top->get_type())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("the deeppest operator should be table scan", K(ret)); + } + if (OB_SUCC(ret)) { + ObLogTableScan *table_scan = static_cast(top); + if (table_scan->is_index_scan() && (NULL != limit_op || NULL != sort_op)) { + ctx.base_index_ = table_scan->get_index_table_id(); + } else { + // otherwise table full scan or can do filter、 sort、limt on index table + } + } + } + return ret; +} + +int ObTransformLateMaterialization::check_transform_plan_expected(ObLogicalOperator* top, + ObCostBasedLateMaterializationCtx &ctx, + bool &is_expected) +{ + int ret = OB_SUCCESS; + ObLogicalOperator *full_table_scan = NULL; + ObLogicalOperator *join_left_branch = NULL; + is_expected = false; + if (OB_ISNULL(top) || OB_ISNULL(ctx_) || OB_ISNULL(ctx_->sql_schema_guard_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), KP(top)); + } else { + while (OB_SUCC(ret) && log_op_def::LOG_JOIN != top->get_type() && 1 == top->get_num_of_child()) { + if (OB_ISNULL(top = top->get_child(ObLogicalOperator::first_child))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), KP(top)); + } + } + if (OB_SUCC(ret) && log_op_def::LOG_JOIN == top->get_type()) { + ObLogJoin *join_op = static_cast(top); + if (NESTED_LOOP_JOIN != join_op->get_join_algo()) { + LOG_TRACE("not nlj, reject transform", K(join_op->get_join_algo())); + } else if (OB_ISNULL(full_table_scan = join_op->get_child(ObLogicalOperator::second_child))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("operator is null", K(ret), KP(full_table_scan)); + } else if (log_op_def::LOG_TABLE_SCAN == full_table_scan->get_type() && + ctx.late_table_id_ == static_cast(full_table_scan)->get_table_id()) { + if (OB_ISNULL(join_left_branch = join_op->get_child(ObLogicalOperator::first_child))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), KP(join_left_branch)); + } else if (static_cast(full_table_scan)->use_column_store()) { + is_expected = false; + } else { + is_expected = true; + } + } + } + if (OB_SUCC(ret) && is_expected) { + ObLogTableScan *index_scan = NULL; + ObLogSort *sort_op = NULL; + while (OB_SUCC(ret) && log_op_def::LOG_TABLE_SCAN != join_left_branch->get_type() && + 1 == join_left_branch->get_num_of_child()) { + if (log_op_def::LOG_SORT == join_left_branch->get_type()) { + sort_op = static_cast(join_left_branch); + } + if (OB_ISNULL(join_left_branch = join_left_branch->get_child(ObLogicalOperator::first_child))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), KP(join_left_branch)); + } + } + if (OB_FAIL(ret)) { + } else if (log_op_def::LOG_TABLE_SCAN != join_left_branch->get_type()) { + is_expected = false; + LOG_TRACE("not table scan, reject transform"); + } else if (FALSE_IT(index_scan = static_cast(join_left_branch))) { + } else if (!index_scan->is_index_scan() || index_scan->use_column_store()) { + is_expected = false; + LOG_TRACE("not index scan, reject transform", K(index_scan->use_column_store()), K(index_scan->is_index_scan())); + } else if (NULL == sort_op && + ObOptimizerUtil::find_item(ctx.check_sort_indexs_, index_scan->get_index_table_id())) { + is_expected = false; + LOG_TRACE("check sort op, reject transform"); + } else if (index_scan->get_index_back()) { + if (ObOptimizerUtil::find_item(ctx.late_material_indexs_, index_scan->get_index_table_id())) { + is_expected = false; + LOG_TRACE("index back, reject transform"); + } else { + // inside evaluate_cost, the index back tag is inaccurate. Then do double check here + ObSEArray temp_exprs; + ObSEArray used_column_ids; + const ObTableSchema *index_schema = NULL; + ObSqlSchemaGuard *schema_guard = ctx_->sql_schema_guard_; + ObSEArray index_column_ids; + if (sort_op != NULL && OB_FAIL(sort_op->get_sort_exprs(temp_exprs))) { + LOG_WARN("failed to get sort exprs", K(ret)); + } else if (index_scan != NULL && OB_FAIL(append(temp_exprs, index_scan->get_filter_exprs()))) { + LOG_WARN("failed to get sort exprs", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::extract_column_ids(temp_exprs, used_column_ids))) { + LOG_WARN("failed to extract column ids", K(ret)); + } else if (OB_FAIL(schema_guard->get_table_schema(index_scan->get_index_table_id(), index_schema))) { + LOG_WARN("fail to get index schema", K(ret)); + } else if (OB_FAIL(index_schema->get_column_ids(index_column_ids))) { + LOG_WARN("failed to get column ids", K(ret)); + } else if (ObOptimizerUtil::is_subset(used_column_ids, index_column_ids)) { + is_expected = true; + } else { + is_expected = false; + LOG_TRACE("index back, reject transform"); + } + } + } + if (OB_SUCC(ret) && is_expected && NULL != index_scan) { + // check table range scan + bool is_get = false; + if (OB_FAIL(index_scan->is_table_get(is_get))) { + LOG_WARN("failed to consider is table get", K(ret)); + } else if (!is_get && index_scan->get_range_conditions().empty()) { + // if there is no range, then there is no need to late material + is_expected = false; + LOG_TRACE("there is no range", K(ret)); + } + } + } + } + return ret; +} + +int ObTransformLateMaterialization::contain_enum_set_rowkeys(const ObRowkeyInfo &rowkey_info, + bool &contain) +{ + int ret = OB_SUCCESS; + const ObRowkeyColumn *col = NULL; + contain = false; + for (int64_t i = 0; OB_SUCC(ret) && !contain && i < rowkey_info.get_size(); ++i) { + if (OB_ISNULL(col = rowkey_info.get_column(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (ob_is_enumset_tc(col->get_meta_type().get_type())) { + contain = true; + } + } + return ret; +} + +} +} diff --git a/src/sql/rewrite/ob_transform_late_materialization.h b/src/sql/rewrite/ob_transform_late_materialization.h new file mode 100644 index 000000000..452b1fb28 --- /dev/null +++ b/src/sql/rewrite/ob_transform_late_materialization.h @@ -0,0 +1,146 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OB_TRANSFORM_LATE_MATERIALIZATION_H +#define OB_TRANSFORM_LATE_MATERIALIZATION_H + +#include "sql/rewrite/ob_transform_rule.h" +#include "sql/resolver/dml/ob_select_stmt.h" +#include "sql/rewrite/ob_transform_utils.h" +#include "sql/optimizer/ob_optimizer_context.h" +#include "lib/container/ob_bit_set.h" + +namespace oceanbase +{ + + +namespace sql +{ + +class ObRawExpr; +class ObTransformLateMaterialization : public ObTransformRule +{ +public: + explicit ObTransformLateMaterialization(ObTransformerCtx *ctx) : + ObTransformRule(ctx, TransMethod::POST_ORDER, T_USE_LATE_MATERIALIZATION) {} + virtual ~ObTransformLateMaterialization() {} + virtual int transform_one_stmt(common::ObIArray &parent_stmts, + ObDMLStmt *&stmt, + bool &trans_happened) override; + +private: + struct ObCostBasedLateMaterializationCtx { + ObCostBasedLateMaterializationCtx() + : late_material_indexs_(), + check_sort_indexs_(), + late_table_id_(common::OB_INVALID_ID), + base_index_(common::OB_INVALID_ID) { } + ObSEArray late_material_indexs_; + ObSEArray check_sort_indexs_; // global index of partition table is ok even if there is no sort op + uint64_t late_table_id_; + uint64_t base_index_; + + TO_STRING_KV(K_(late_material_indexs), + K_(check_sort_indexs), + K_(late_table_id), + K_(base_index)); + }; + + struct ObLateMaterializationInfo + { + ObLateMaterializationInfo() + : + candi_index_names_(), + candi_indexs_(), + project_col_in_view_() { } + ObSEArray candi_index_names_; + ObSEArray candi_indexs_; // late materialization index + some may late materialization index + ObSEArray project_col_in_view_; + TO_STRING_KV(K_(candi_index_names), + K_(candi_indexs), + K_(project_col_in_view)); + }; + int check_hint_validity(const ObDMLStmt &stmt, bool &force_trans, bool &force_no_trans); + int check_stmt_need_late_materialization(const ObSelectStmt &stmt, const bool force_accept, bool &need); + int generate_late_materialization_info(const ObSelectStmt &stmt, + ObLateMaterializationInfo &info, + ObCostBasedLateMaterializationCtx &check_ctx); + int extract_transform_column_ids(const ObSelectStmt &select_stmt, + const ObTableSchema &table_schema, + ObIArray &key_col_ids, + ObIArray &filter_col_ids, + ObIArray &orderby_col_ids, + ObIArray &select_col_ids); + int get_accessible_index(const ObSelectStmt &select_stmt, + const TableItem &table_item, + ObIArray &index_schemas); + + bool match_index_name(const ObIArray &index_schemas, + const TableItem &table_item, + const ObIndexHint &index_hint, + const ObCollationType cs_type, + int64_t &id); + int contain_enum_set_rowkeys(const ObRowkeyInfo &rowkey_info, bool &contain); + int check_index_match_late_materialization(const uint64_t index_id, + const ObIArray &index_col_ids, + const ObIArray &key_col_ids, + const ObIArray &filter_col_ids, + const ObIArray &orderby_col_ids, + const ObIArray &select_col_ids, + const uint64_t ref_table_id, + ObCostBasedLateMaterializationCtx &check_ctx, + bool &is_match); + int generate_late_materialization_stmt(const ObLateMaterializationInfo &info, + ObDMLStmt *stmt, + ObDMLStmt *&trans_stmt); + int generate_late_materialization_view(const ObIArray &select_col_ids, + ObSelectStmt *select_stmt, + ObSelectStmt *&view_stmt, + TableItem *&table_item); + int extract_replace_column_exprs(const ObSelectStmt &select_stmt, + const ObSelectStmt &view_stmt, + const uint64_t table_id, + const uint64_t view_id, + ObIArray &old_col_exprs, + ObIArray &new_col_exprs); + int generate_pk_join_conditions(const uint64_t ref_table_id, + const uint64_t table_id, + const ObIArray &old_col_exprs, + const ObIArray &new_col_exprs, + ObSelectStmt &select_stmt); + int generate_late_materialization_hint(const ObLateMaterializationInfo &info, + const TableItem &table_item, + const TableItem &view_table, + ObSelectStmt &select_stmt, + ObSelectStmt &view_stmt); + int inner_accept_transform(common::ObIArray &parent_stmts, + ObDMLStmt *&stmt, + bool force_accept, + ObLateMaterializationInfo &info, + ObCostBasedLateMaterializationCtx &check_ctx, + bool &trans_happened); + int check_transform_plan_expected(ObLogicalOperator* top, + ObCostBasedLateMaterializationCtx &ctx, + bool &is_valid); + int get_index_of_base_stmt_path(ObLogicalOperator* top, ObCostBasedLateMaterializationCtx &ctx); + int evaluate_stmt_cost(ObDMLStmt *&stmt, + bool is_trans_stmt, + double &plan_cost, + bool &is_expected, + ObCostBasedLateMaterializationCtx &check_ctx); + virtual int is_expected_plan(ObLogPlan *plan, void *check_ctx, bool is_trans_plan, bool &is_valid) override; +}; + +} +} + +#endif // OB_TRANSFORM_LATE_MATERIALIZATION_H \ No newline at end of file diff --git a/src/sql/rewrite/ob_transform_rule.cpp b/src/sql/rewrite/ob_transform_rule.cpp index 94216c61b..5b50d8ab7 100644 --- a/src/sql/rewrite/ob_transform_rule.cpp +++ b/src/sql/rewrite/ob_transform_rule.cpp @@ -138,6 +138,7 @@ const char* ObTransformerCtx::get_trans_type_string(uint64_t trans_type) TRANS_TYPE_TO_STR(DECORRELATE) TRANS_TYPE_TO_STR(CONDITIONAL_AGGR_COALESCE) TRANS_TYPE_TO_STR(MV_REWRITE) + TRANS_TYPE_TO_STR(LATE_MATERIALIZATION) default: return NULL; } } diff --git a/src/sql/rewrite/ob_transform_rule.h b/src/sql/rewrite/ob_transform_rule.h index a2216b8b7..c541299b2 100644 --- a/src/sql/rewrite/ob_transform_rule.h +++ b/src/sql/rewrite/ob_transform_rule.h @@ -182,6 +182,7 @@ enum TRANSFORM_TYPE { DECORRELATE , CONDITIONAL_AGGR_COALESCE , MV_REWRITE , + LATE_MATERIALIZATION , TRANSFORM_TYPE_COUNT_PLUS_ONE , }; @@ -288,7 +289,8 @@ public: (1L << GROUPBY_PULLUP) | (1L << SUBQUERY_COALESCE) | (1L << SEMI_TO_INNER) | - (1L << MV_REWRITE); + (1L << MV_REWRITE) | + (1L << LATE_MATERIALIZATION); ObTransformRule(ObTransformerCtx *ctx, TransMethod transform_method, @@ -405,7 +407,11 @@ protected: ObRawExprFactory &expr_factory, ObIArray &old_temp_table_stmts, ObIArray &new_temp_table_stmts); - + int adjust_transformed_stmt(common::ObIArray &parent_stmts, + ObDMLStmt *stmt, + ObDMLStmt *&orgin_stmt, + ObDMLStmt *&root_stmt); + void reset_stmt_cost() { stmt_cost_ = -1; } private: // pre-order transformation int transform_pre_order(common::ObIArray &parent_stmts, @@ -428,10 +434,6 @@ private: int transform_temp_tables(ObIArray &parent_stmts, const int64_t current_level, ObDMLStmt *&stmt); - int adjust_transformed_stmt(common::ObIArray &parent_stmts, - ObDMLStmt *stmt, - ObDMLStmt *&orgin_stmt, - ObDMLStmt *&root_stmt); int evaluate_cost(common::ObIArray &parent_stms, ObDMLStmt *&stmt, diff --git a/src/sql/rewrite/ob_transform_simplify_distinct.cpp b/src/sql/rewrite/ob_transform_simplify_distinct.cpp index ff4404434..a71922dab 100644 --- a/src/sql/rewrite/ob_transform_simplify_distinct.cpp +++ b/src/sql/rewrite/ob_transform_simplify_distinct.cpp @@ -155,7 +155,8 @@ int ObTransformSimplifyDistinct::remove_distinct_on_unique_exprs(ObSelectStmt *s ctx_->schema_checker_, select_exprs, true /* strict */, - is_unique, true))) { + is_unique, + FLAGS_IGNORE_DISTINCT))) { LOG_WARN("failed to check stmt unique", K(ret)); } else if (is_unique) { stmt->assign_all(); diff --git a/src/sql/rewrite/ob_transform_utils.cpp b/src/sql/rewrite/ob_transform_utils.cpp index 65809be5c..dc014fe4c 100644 --- a/src/sql/rewrite/ob_transform_utils.cpp +++ b/src/sql/rewrite/ob_transform_utils.cpp @@ -2390,6 +2390,7 @@ int ObTransformUtils::is_column_expr_not_null(ObNotNullContext &ctx, is_not_null = expr->get_result_type().has_result_flag(NOT_NULL_FLAG); } } else if (table->is_generated_table() || table->is_temp_table() || table->is_lateral_table()) { + // TODO RUINAO NOTE : every col expr will go this path, and we can modify this logic oneday int64_t idx = expr->get_column_id() - OB_APP_MIN_COLUMN_ID; ObRawExpr *child_expr = NULL; ObSelectStmt *child_stmt = table->ref_query_; diff --git a/src/sql/rewrite/ob_transformer_impl.cpp b/src/sql/rewrite/ob_transformer_impl.cpp index d130d6f2c..5c1ede836 100644 --- a/src/sql/rewrite/ob_transformer_impl.cpp +++ b/src/sql/rewrite/ob_transformer_impl.cpp @@ -50,6 +50,7 @@ #include "sql/rewrite/ob_transform_conditional_aggr_coalesce.h" #include "sql/rewrite/ob_transform_mv_rewrite.h" #include "sql/rewrite/ob_transform_decorrelate.h" +#include "sql/rewrite/ob_transform_late_materialization.h" #include "common/ob_smart_call.h" #include "sql/engine/ob_exec_context.h" @@ -458,6 +459,7 @@ int ObTransformerImpl::transform_rule_set_in_one_iteration(ObDMLStmt *&stmt, APPLY_RULE_IF_NEEDED(FASTMINMAX, ObTransformMinMax); APPLY_RULE_IF_NEEDED(PREDICATE_MOVE_AROUND, ObTransformPredicateMoveAround); APPLY_RULE_IF_NEEDED(OR_EXPANSION, ObTransformOrExpansion); + APPLY_RULE_IF_NEEDED(LATE_MATERIALIZATION, ObTransformLateMaterialization); } return ret; } diff --git a/tools/deploy/mysql_test/test_suite/geometry/r/mysql/geometry_basic_mysql.result b/tools/deploy/mysql_test/test_suite/geometry/r/mysql/geometry_basic_mysql.result index e51575ff3..a5ed1e792 100644 --- a/tools/deploy/mysql_test/test_suite/geometry/r/mysql/geometry_basic_mysql.result +++ b/tools/deploy/mysql_test/test_suite/geometry/r/mysql/geometry_basic_mysql.result @@ -108,27 +108,18 @@ END// call geom_insert(10000); explain select x,y,st_astext(poi) from gis_point order by x,y desc limit ; Query Plan -================================================================== -|ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------------------------- -|0 |NESTED-LOOP JOIN | | | | -| |├─TOP-N SORT | | | | -| |│ └─TABLE FULL SCAN|gis_point(xy_index)| | | -| |└─TABLE GET |gis_point_alias | | | -================================================================== +================================================================ +|ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| +---------------------------------------------------------------- +|0 |TOP-N SORT | | | | +| |└─TABLE FULL SCAN|gis_point(xy_index)| | | +================================================================ Outputs & filters: ------------------------------------- - 0 - output([gis_point.x], [gis_point.y], [st_astext(gis_point_alias.poi)]), filter(nil) - conds(nil), nl_params_([gis_point.__pk_increment(:0)]), use_batch=false - - output([gis_point.__pk_increment], [gis_point.x], [gis_point.y]), filter(nil) + 0 - output([gis_point.x], [gis_point.y], [st_astext(gis_point.poi)]), filter(nil) sort_keys([gis_point.x, ASC], [gis_point.y, DESC]), topn( ), prefix_pos( ) - - output([gis_point.__pk_increment], [gis_point.x], [gis_point.y]), filter(nil) - access([gis_point.__pk_increment], [gis_point.x], [gis_point.y]), partitions(p0) - is_index_back=false, is_global_index=false, + - output([gis_point.poi], [gis_point.x], [gis_point.y]), filter(nil) + access([gis_point.__pk_increment], [gis_point.poi], [gis_point.x], [gis_point.y]), partitions(p0) + is_index_back=true, is_global_index=false, range_key([gis_point.x], [gis_point.y], [gis_point.__pk_increment]), range(MIN,MIN,MIN ; MAX,MAX,MAX)always true - - output([gis_point_alias.poi]), filter(nil) - access([gis_point_alias.poi]), partitions(p0) - is_index_back=false, is_global_index=false, - range_key([gis_point_alias.__pk_increment]), range(MIN ; MAX), - range_cond([gis_point_alias.__pk_increment = :0]) drop table if exists gis_point; From e537a0c479efd35e940aedbe7db52e6965600ff2 Mon Sep 17 00:00:00 2001 From: jingtaoye35 <1255153887@qq.com> Date: Fri, 23 Aug 2024 03:32:01 +0000 Subject: [PATCH 181/249] [FEAT MERGE] 424 SQL compatibility patch 433 Co-authored-by: wjhh2008 Co-authored-by: GongYusen <986957406@qq.com> Co-authored-by: hy-guo --- .gitignore | 10 +- deps/oblib/src/lib/CMakeLists.txt | 2 + deps/oblib/src/lib/charset/ob_charset.cpp | 484 ++- deps/oblib/src/lib/charset/ob_charset.h | 80 +- .../lib/charset/ob_charset_string_helper.h | 10 + deps/oblib/src/lib/charset/ob_ctype.h | 62 +- deps/oblib/src/lib/charset/ob_ctype_ascii.cc | 114 + .../src/lib/charset/ob_ctype_ascii_tab.h | 242 ++ deps/oblib/src/lib/charset/ob_ctype_bin.cc | 12 +- .../oblib/src/lib/charset/ob_ctype_gb18030.cc | 28 +- deps/oblib/src/lib/charset/ob_ctype_gbk.cc | 5 + deps/oblib/src/lib/charset/ob_ctype_latin1.cc | 12 +- deps/oblib/src/lib/charset/ob_ctype_simple.cc | 180 +- deps/oblib/src/lib/charset/ob_ctype_tis620.cc | 336 ++ .../src/lib/charset/ob_ctype_tis620_tab.h | 830 ++++ deps/oblib/src/lib/charset/ob_ctype_uca.cc | 120 +- deps/oblib/src/lib/charset/ob_ctype_utf16.cc | 6 +- deps/oblib/src/lib/charset/ob_ctype_utf8.cc | 6 + .../src/lib/mysqlclient/ob_mysql_result.h | 141 + deps/oblib/src/lib/ob_define.h | 6 +- deps/oblib/src/lib/ob_errno.h | 1 + deps/oblib/src/lib/ob_name_def.h | 6 + .../unittest/lib/charset/test_charset.cpp | 175 +- .../libobcdc/src/ob_obj2str_helper.cpp | 3 + src/objit/include/objit/common/ob_item_type.h | 3 +- src/observer/mysql/obmp_base.cpp | 6 - src/observer/mysql/obmp_connect.cpp | 33 + src/observer/mysql/obmp_connect.h | 5 +- src/observer/mysql/obmp_query.cpp | 4 +- src/observer/ob_server.cpp | 7 + src/observer/omt/ob_multi_tenant.cpp | 28 + src/observer/omt/ob_multi_tenant.h | 1 + .../ob_information_user_privileges_table.cpp | 6 + .../virtual_table/ob_mysql_db_table.cpp | 4 +- .../virtual_table/ob_mysql_user_table.cpp | 8 +- src/observer/virtual_table/ob_show_grants.cpp | 9 + src/pl/ob_pl.cpp | 36 +- src/pl/ob_pl_resolver.cpp | 20 +- src/rootserver/ob_ddl_operator.cpp | 50 +- src/rootserver/ob_ddl_operator.h | 16 +- src/rootserver/ob_ddl_service.cpp | 215 +- src/rootserver/ob_ddl_service.h | 26 +- src/rootserver/ob_ddl_sql_generator.cpp | 6 + src/rootserver/ob_root_service.cpp | 2 +- .../parallel_ddl/ob_create_table_helper.cpp | 4 +- src/share/config/ob_config_helper.cpp | 41 + src/share/config/ob_config_helper.h | 55 + src/share/datum/ob_datum_cmp_func_def.h | 10 +- .../ob_inner_table_schema.12451_12500.cpp | 287 ++ .../ob_inner_table_schema.20001_20050.cpp | 154 +- .../ob_inner_table_schema.21301_21350.cpp | 16 +- .../ob_inner_table_schema.21351_21400.cpp | 4 +- .../ob_inner_table_schema.21451_21500.cpp | 2 +- .../ob_inner_table_schema.21501_21550.cpp | 100 + .../ob_inner_table_schema.21551_21600.cpp | 150 + .../ob_inner_table_schema.501_550.cpp | 319 ++ .../ob_inner_table_schema.50501_50550.cpp | 270 ++ .../ob_inner_table_schema.60501_60550.cpp | 180 + src/share/inner_table/ob_inner_table_schema.h | 82 +- .../inner_table/ob_inner_table_schema.lob.cpp | 2 +- .../ob_inner_table_schema_constants.h | 32 + .../inner_table/ob_inner_table_schema_def.py | 417 +- .../ob_inner_table_schema_misc.ipp | 66 +- src/share/inner_table/table_id_to_name | 14 + .../ob_compatibility_security_feature_def.h | 6 + src/share/ob_encryption_util.h | 9 + src/share/ob_encryption_util_os.cpp | 38 + src/share/ob_errno.cpp | 24 +- src/share/ob_errno.def | 6 +- src/share/ob_errno.h | 19 +- src/share/ob_fts_index_builder_util.cpp | 18 +- src/share/ob_i_tablet_scan.h | 10 + src/share/ob_index_builder_util.cpp | 23 +- src/share/ob_rpc_struct.cpp | 1 - src/share/object/ob_obj_cast.cpp | 15 +- src/share/parameter/ob_parameter_seed.ipp | 44 + src/share/rc/ob_tenant_base.h | 6 +- src/share/schema/ob_column_schema.h | 5 +- src/share/schema/ob_priv_sql_service.cpp | 87 +- src/share/schema/ob_priv_sql_service.h | 28 +- src/share/schema/ob_priv_type.h | 15 +- src/share/schema/ob_schema_getter_guard.cpp | 26 +- src/share/schema/ob_schema_getter_guard.h | 3 +- src/share/schema/ob_schema_macro_define.cpp | 6 +- src/share/schema/ob_schema_printer.cpp | 68 +- src/share/schema/ob_schema_printer.h | 3 +- src/share/schema/ob_schema_retrieve_utils.ipp | 35 +- src/share/schema/ob_schema_struct.cpp | 9 + src/share/schema/ob_user_sql_service.cpp | 16 +- .../system_variable/gen_ob_sys_variables.py | 91 +- .../system_variable/ob_sys_var_class_type.h | 152 + .../system_variable/ob_system_variable.cpp | 94 + .../system_variable/ob_system_variable.h | 10 + .../ob_system_variable_alias.h | 152 + .../ob_system_variable_factory.cpp | 3796 ++++++++++++++++- .../ob_system_variable_factory.h | 1171 ++++- .../ob_system_variable_init.cpp | 2818 ++++++++++-- .../ob_system_variable_init.json | 631 ++- src/sql/CMakeLists.txt | 4 +- .../code_generator/ob_static_engine_cg.cpp | 9 +- src/sql/engine/cmd/ob_dcl_executor.cpp | 96 +- src/sql/engine/cmd/ob_dcl_executor.h | 7 +- src/sql/engine/cmd/ob_mock_executor.cpp | 34 +- .../engine/cmd/ob_variable_set_executor.cpp | 4 +- src/sql/engine/expr/ob_expr_aes_encrypt.cpp | 271 -- src/sql/engine/expr/ob_expr_aes_encrypt.h | 63 - src/sql/engine/expr/ob_expr_arg_case.cpp | 7 +- .../engine/expr/ob_expr_audit_log_func.cpp | 534 +++ src/sql/engine/expr/ob_expr_audit_log_func.h | 126 + .../expr/ob_expr_can_access_trigger.cpp | 108 + .../engine/expr/ob_expr_can_access_trigger.h | 38 + src/sql/engine/expr/ob_expr_case.cpp | 4 +- src/sql/engine/expr/ob_expr_cast.cpp | 12 +- src/sql/engine/expr/ob_expr_chr.cpp | 19 +- src/sql/engine/expr/ob_expr_chr.h | 3 +- src/sql/engine/expr/ob_expr_coalesce.cpp | 4 +- src/sql/engine/expr/ob_expr_collation.cpp | 2 + src/sql/engine/expr/ob_expr_compress.cpp | 2 +- src/sql/engine/expr/ob_expr_concat.cpp | 3 +- src/sql/engine/expr/ob_expr_concat_ws.cpp | 3 +- src/sql/engine/expr/ob_expr_convert.cpp | 2 +- .../engine/expr/ob_expr_current_user_priv.cpp | 81 + .../engine/expr/ob_expr_current_user_priv.h | 17 + src/sql/engine/expr/ob_expr_date_add.cpp | 4 +- src/sql/engine/expr/ob_expr_day_of_func.cpp | 4 +- src/sql/engine/expr/ob_expr_elt.cpp | 2 +- .../engine/expr/ob_expr_eval_functions.cpp | 22 +- src/sql/engine/expr/ob_expr_export_set.cpp | 2 +- src/sql/engine/expr/ob_expr_field.cpp | 8 +- src/sql/engine/expr/ob_expr_find_in_set.cpp | 7 +- src/sql/engine/expr/ob_expr_hex.cpp | 2 +- src/sql/engine/expr/ob_expr_ifnull.cpp | 16 +- src/sql/engine/expr/ob_expr_insert.cpp | 3 +- .../engine/expr/ob_expr_json_array_append.cpp | 1 + src/sql/engine/expr/ob_expr_least.cpp | 40 +- src/sql/engine/expr/ob_expr_left.cpp | 3 +- src/sql/engine/expr/ob_expr_like.cpp | 4 +- src/sql/engine/expr/ob_expr_lower.cpp | 3 +- src/sql/engine/expr/ob_expr_lrpad.cpp | 3 +- src/sql/engine/expr/ob_expr_make_set.cpp | 3 +- src/sql/engine/expr/ob_expr_nullif.cpp | 4 +- src/sql/engine/expr/ob_expr_nvl.cpp | 15 +- src/sql/engine/expr/ob_expr_operator.cpp | 295 +- src/sql/engine/expr/ob_expr_operator.h | 39 +- .../engine/expr/ob_expr_operator_factory.cpp | 15 +- src/sql/engine/expr/ob_expr_oracle_nullif.cpp | 3 +- src/sql/engine/expr/ob_expr_pad.cpp | 5 +- src/sql/engine/expr/ob_expr_password.cpp | 3 +- src/sql/engine/expr/ob_expr_quote.cpp | 3 +- src/sql/engine/expr/ob_expr_regexp.cpp | 68 +- src/sql/engine/expr/ob_expr_regexp_instr.cpp | 2 +- src/sql/engine/expr/ob_expr_regexp_like.cpp | 2 +- .../engine/expr/ob_expr_regexp_replace.cpp | 2 +- src/sql/engine/expr/ob_expr_regexp_substr.cpp | 2 +- src/sql/engine/expr/ob_expr_repeat.cpp | 2 +- src/sql/engine/expr/ob_expr_replace.cpp | 3 +- src/sql/engine/expr/ob_expr_reverse.h | 2 +- src/sql/engine/expr/ob_expr_right.cpp | 3 +- src/sql/engine/expr/ob_expr_sha.cpp | 74 +- src/sql/engine/expr/ob_expr_sha.h | 17 + .../engine/expr/ob_expr_statement_digest.cpp | 4 +- src/sql/engine/expr/ob_expr_strcmp.cpp | 30 +- src/sql/engine/expr/ob_expr_substr.cpp | 7 +- .../engine/expr/ob_expr_substring_index.cpp | 3 +- .../engine/expr/ob_expr_symmetric_encrypt.cpp | 390 ++ .../engine/expr/ob_expr_symmetric_encrypt.h | 118 + .../expr/ob_expr_sys_connect_by_path.cpp | 4 +- src/sql/engine/expr/ob_expr_to_multi_byte.cpp | 6 +- src/sql/engine/expr/ob_expr_trim.cpp | 4 +- src/sql/engine/expr/ob_expr_type_to_str.cpp | 7 + .../user_defined_function/ob_udf_util.cpp | 4 +- src/sql/executor/ob_cmd_executor.cpp | 14 +- src/sql/monitor/ob_sql_plan.cpp | 2 +- src/sql/ob_sql_init.h | 4 +- src/sql/ob_sql_utils.cpp | 49 +- src/sql/ob_sql_utils.h | 1 + src/sql/optimizer/ob_join_order.cpp | 50 +- src/sql/optimizer/ob_join_order.h | 3 +- src/sql/optimizer/ob_log_join.cpp | 9 +- src/sql/optimizer/ob_log_subplan_filter.cpp | 12 +- src/sql/optimizer/ob_opt_selectivity.cpp | 1 + src/sql/optimizer/ob_optimizer.cpp | 53 +- src/sql/optimizer/ob_optimizer_context.h | 25 + src/sql/optimizer/ob_optimizer_util.cpp | 11 +- src/sql/optimizer/ob_select_log_plan.cpp | 17 +- src/sql/optimizer/ob_table_location.cpp | 14 +- src/sql/optimizer/ob_table_location.h | 2 + src/sql/parser/CMakeLists.txt | 12 +- src/sql/parser/gen_parser.sh | 68 +- .../parser/non_reserved_keywords_mysql_mode.c | 1 + src/sql/parser/ob_fast_parser.cpp | 21 +- src/sql/parser/ob_fast_parser.h | 4 +- src/sql/parser/parse_malloc.cpp | 12 +- src/sql/parser/parse_node.c | 145 + src/sql/parser/parse_node.h | 7 +- src/sql/parser/sql_parser_base.c | 54 +- src/sql/parser/sql_parser_base.h | 83 +- src/sql/parser/sql_parser_mysql_mode.l | 3 + src/sql/parser/sql_parser_mysql_mode.y | 1121 ++++- .../ob_values_table_compression.cpp | 22 +- src/sql/printer/ob_dml_stmt_printer.cpp | 18 + src/sql/printer/ob_raw_expr_printer.cpp | 22 + .../privilege_check/ob_privilege_check.cpp | 85 +- .../resolver/cmd/ob_alter_system_resolver.cpp | 39 +- .../resolver/cmd/ob_alter_system_resolver.h | 1 + src/sql/resolver/cmd/ob_mock_resolver.cpp | 84 +- src/sql/resolver/cmd/ob_mock_resolver.h | 58 + src/sql/resolver/cmd/ob_mock_stmt.h | 9 + src/sql/resolver/cmd/ob_show_resolver.cpp | 490 ++- src/sql/resolver/cmd/ob_show_resolver.h | 51 +- .../resolver/cmd/ob_variable_set_resolver.cpp | 9 +- src/sql/resolver/dcl/ob_grant_resolver.cpp | 12 +- src/sql/resolver/dcl/ob_revoke_resolver.cpp | 8 + .../resolver/ddl/ob_alter_table_resolver.cpp | 35 +- .../resolver/ddl/ob_create_table_resolver.cpp | 10 +- src/sql/resolver/ddl/ob_ddl_resolver.cpp | 287 +- src/sql/resolver/ddl/ob_ddl_resolver.h | 11 +- src/sql/resolver/ddl/ob_trigger_resolver.cpp | 72 + src/sql/resolver/ddl/ob_trigger_resolver.h | 3 +- src/sql/resolver/ddl/ob_trigger_stmt.h | 8 + src/sql/resolver/dml/ob_dml_resolver.cpp | 47 +- src/sql/resolver/dml/ob_dml_resolver.h | 2 +- src/sql/resolver/dml/ob_dml_stmt.cpp | 24 +- src/sql/resolver/dml/ob_dml_stmt.h | 11 +- src/sql/resolver/dml/ob_hint.cpp | 129 +- src/sql/resolver/dml/ob_hint.h | 57 +- src/sql/resolver/dml/ob_inlist_resolver.cpp | 13 +- src/sql/resolver/dml/ob_select_resolver.cpp | 45 +- src/sql/resolver/dml/ob_select_resolver.h | 1 + src/sql/resolver/dml/ob_select_stmt.cpp | 30 - src/sql/resolver/dml/ob_select_stmt.h | 10 - src/sql/resolver/expr/ob_raw_expr.cpp | 7 +- .../resolver/expr/ob_raw_expr_deduce_type.cpp | 19 +- .../expr/ob_raw_expr_info_extractor.cpp | 4 + .../expr/ob_raw_expr_resolver_impl.cpp | 61 +- .../resolver/expr/ob_raw_expr_resolver_impl.h | 2 +- src/sql/resolver/expr/ob_raw_expr_util.cpp | 12 +- src/sql/resolver/ob_resolver.cpp | 19 +- src/sql/resolver/ob_resolver_utils.cpp | 28 +- src/sql/resolver/ob_schema_checker.cpp | 23 +- src/sql/resolver/ob_schema_checker.h | 3 +- src/sql/resolver/ob_stmt.h | 3 +- src/sql/resolver/ob_stmt_type.h | 4 +- src/sql/rewrite/ob_query_range.cpp | 29 +- src/sql/rewrite/ob_query_range.h | 6 + src/sql/rewrite/ob_stmt_comparer.cpp | 16 +- .../rewrite/ob_transform_groupby_pullup.cpp | 11 +- .../rewrite/ob_transform_groupby_pushdown.cpp | 11 +- .../rewrite/ob_transform_join_elimination.cpp | 5 + src/sql/rewrite/ob_transform_or_expansion.cpp | 2 + src/sql/rewrite/ob_transform_rule.cpp | 3 + src/sql/rewrite/ob_transform_rule.h | 9 + src/sql/rewrite/ob_transform_temp_table.cpp | 15 +- src/sql/rewrite/ob_transform_utils.cpp | 5 +- src/sql/rewrite/ob_transform_view_merge.cpp | 8 - src/sql/rewrite/ob_transformer_impl.cpp | 35 + src/sql/rewrite/ob_transformer_impl.h | 1 + src/sql/session/ob_basic_session_info.cpp | 17 + src/sql/session/ob_basic_session_info.h | 2 + src/sql/session/ob_sql_session_info.cpp | 22 + src/sql/session/ob_sql_session_info.h | 19 + src/sql/session/ob_sql_session_mgr.cpp | 4 + .../r/mysql/information_schema.result | 3157 +++++++------- .../mysql_test/r/mysql/special_hook.result | 16 +- .../mysql_test/r/mysql/special_stmt.result | 32 +- tools/deploy/mysql_test/r/mysql/view_2.result | 3 +- tools/deploy/mysql_test/t/view_2.test | 1 - .../expr/r/mysql/collation_expr.result | 4 +- .../expr/r/mysql/func_regexp.result | 8 +- .../r/mysql/information_schema_desc.result | 16 +- .../all_virtual_sys_parameter_stat.result | 10 + .../inner_table/r/mysql/character_sets.result | 6 + .../inner_table/r/mysql/collations.result | 34 +- .../r/mysql/desc_sys_views_in_mysql.result | 111 +- .../r/mysql/desc_sys_views_in_sys.result | 117 +- .../r/mysql/desc_virtual_table_in_sys.result | 23 + .../r/mysql/inner_table_overall.result | 12 + .../parameter_variable_conflict_check.result | 8 + .../inner_table/r/mysql/partitions.result | 10 +- .../r/mysql/table_privileges.result | 2 +- .../r/mysql/user_privileges.result | 10 +- .../inner_table/r/mysql/views.result | 4 +- .../r/mysql/expr_aes_encrypt.result | 259 +- .../static_engine/r/mysql/expr_and_or.result | 14 +- .../r/mysql/expr_collation.result | 2 +- .../static_engine/t/expr_aes_encrypt.test | 50 +- .../observer/table/test_create_executor.cpp | 2 +- unittest/share/schema/test_schema_service.cpp | 6 +- unittest/share/schema/test_table_schema.cpp | 2 +- unittest/sql/CMakeLists.txt | 3 + unittest/sql/audit/CMakeLists.txt | 1 + unittest/sql/audit/test_audit_log_utils.cpp | 112 + unittest/sql/parser/print_parser_tree.result | 10 +- unittest/sql/parser/test_parser.result | 10 +- unittest/sql/resolver/test_resolver.cpp | 4 +- unittest/sql/rewrite/test_query_range.cpp | 33 +- 296 files changed, 21588 insertions(+), 4122 deletions(-) create mode 100644 deps/oblib/src/lib/charset/ob_ctype_ascii.cc create mode 100644 deps/oblib/src/lib/charset/ob_ctype_ascii_tab.h create mode 100644 deps/oblib/src/lib/charset/ob_ctype_tis620.cc create mode 100644 deps/oblib/src/lib/charset/ob_ctype_tis620_tab.h delete mode 100644 src/sql/engine/expr/ob_expr_aes_encrypt.cpp delete mode 100644 src/sql/engine/expr/ob_expr_aes_encrypt.h create mode 100644 src/sql/engine/expr/ob_expr_audit_log_func.cpp create mode 100644 src/sql/engine/expr/ob_expr_audit_log_func.h create mode 100644 src/sql/engine/expr/ob_expr_can_access_trigger.cpp create mode 100644 src/sql/engine/expr/ob_expr_can_access_trigger.h create mode 100644 src/sql/engine/expr/ob_expr_symmetric_encrypt.cpp create mode 100644 src/sql/engine/expr/ob_expr_symmetric_encrypt.h create mode 100644 unittest/sql/audit/CMakeLists.txt create mode 100644 unittest/sql/audit/test_audit_log_utils.cpp diff --git a/.gitignore b/.gitignore index 06b531bae..6a6109a1d 100644 --- a/.gitignore +++ b/.gitignore @@ -152,11 +152,11 @@ src/sql/parser/sql_parser_oracle_utf8_mode_lex.c src/sql/parser/sql_parser_oracle_utf8_mode_lex.h src/sql/parser/sql_parser_oracle_utf8_mode_tab.c src/sql/parser/sql_parser_oracle_utf8_mode_tab.h -src/sql/parser/sql_parser_oracle_latin1_mode_lex.c -src/sql/parser/sql_parser_oracle_latin1_mode_lex.h -src/sql/parser/sql_parser_oracle_latin1_mode_tab.c -src/sql/parser/sql_parser_oracle_latin1_mode_tab.h -src/sql/parser/non_reserved_keywords_oracle_latin1_mode.c +src/sql/parser/sql_parser_oracle_single_byte_mode_lex.c +src/sql/parser/sql_parser_oracle_single_byte_mode_lex.h +src/sql/parser/sql_parser_oracle_single_byte_mode_tab.c +src/sql/parser/sql_parser_oracle_single_byte_mode_tab.h +src/sql/parser/non_reserved_keywords_oracle_single_byte_mode.c src/sql/parser/_gen_parser.output src/sql/parser/_gen_parser.error src/pl/parser/pl_parser_mysql_mode_lex.c diff --git a/deps/oblib/src/lib/CMakeLists.txt b/deps/oblib/src/lib/CMakeLists.txt index 120a98ddf..250affc1f 100644 --- a/deps/oblib/src/lib/CMakeLists.txt +++ b/deps/oblib/src/lib/CMakeLists.txt @@ -22,6 +22,8 @@ ob_set_subtarget(oblib_lib charset charset/ob_ctype_gb18030.cc charset/ob_ctype_gbk.cc charset/ob_ctype_latin1.cc + charset/ob_ctype_ascii.cc + charset/ob_ctype_tis620.cc charset/ob_ctype_mb.cc charset/ob_ctype_simple.cc charset/ob_ctype_uca.cc diff --git a/deps/oblib/src/lib/charset/ob_charset.cpp b/deps/oblib/src/lib/charset/ob_charset.cpp index c92a3ef88..fbe4b2cf8 100644 --- a/deps/oblib/src/lib/charset/ob_charset.cpp +++ b/deps/oblib/src/lib/charset/ob_charset.cpp @@ -282,6 +282,8 @@ const ObCharsetWrapper ObCharset::charset_wrap_arr_[ObCharset::VALID_CHARSET_TYP {CHARSET_GB18030, "GB18030 charset", CS_TYPE_GB18030_CHINESE_CI, 4}, {CHARSET_LATIN1, "cp1252 West European", CS_TYPE_LATIN1_SWEDISH_CI, 1}, {CHARSET_GB18030_2022, "GB18030-2022 charset", CS_TYPE_GB18030_2022_PINYIN_CI, 4}, + {CHARSET_ASCII, "US ASCII", CS_TYPE_ASCII_GENERAL_CI, 1}, + {CHARSET_TIS620, "TIS620 Thai", CS_TYPE_TIS620_THAI_CI, 1}, }; const ObCollationWrapper ObCharset::collation_wrap_arr_[ObCharset::VALID_COLLATION_TYPES] = @@ -294,9 +296,9 @@ const ObCollationWrapper ObCharset::collation_wrap_arr_[ObCharset::VALID_COLLATI {CS_TYPE_UTF16_GENERAL_CI, CHARSET_UTF16, CS_TYPE_UTF16_GENERAL_CI, true, true, 1}, {CS_TYPE_UTF16_BIN, CHARSET_UTF16, CS_TYPE_UTF16_BIN, false, true, 1}, //{CS_TYPE_UTF8MB4_ZH_0900_AS_CS, CHARSET_UTF8MB4, CS_TYPE_UTF8MB4_ZH_0900_AS_CS, false, true, 0}, - {CS_TYPE_UTF8MB4_UNICODE_CI, CHARSET_UTF8MB4, CS_TYPE_UTF8MB4_UNICODE_CI, false, true, 1}, - {CS_TYPE_UTF16_UNICODE_CI, CHARSET_UTF16, CS_TYPE_UTF16_UNICODE_CI, false, true, 1}, - {CS_TYPE_GB18030_CHINESE_CI, CHARSET_GB18030, CS_TYPE_GB18030_CHINESE_CI, true, true, 1}, + {CS_TYPE_UTF8MB4_UNICODE_CI, CHARSET_UTF8MB4, CS_TYPE_UTF8MB4_UNICODE_CI, false, true, 8}, + {CS_TYPE_UTF16_UNICODE_CI, CHARSET_UTF16, CS_TYPE_UTF16_UNICODE_CI, false, true, 8}, + {CS_TYPE_GB18030_CHINESE_CI, CHARSET_GB18030, CS_TYPE_GB18030_CHINESE_CI, true, true, 2}, {CS_TYPE_GB18030_BIN, CHARSET_GB18030, CS_TYPE_GB18030_BIN, false, true, 1}, {CS_TYPE_LATIN1_SWEDISH_CI, CHARSET_LATIN1, CS_TYPE_LATIN1_SWEDISH_CI,true, true, 1}, {CS_TYPE_LATIN1_BIN, CHARSET_LATIN1, CS_TYPE_LATIN1_BIN,false, true, 1}, @@ -307,7 +309,15 @@ const ObCollationWrapper ObCharset::collation_wrap_arr_[ObCharset::VALID_COLLATI {CS_TYPE_GB18030_2022_RADICAL_CS, CHARSET_GB18030_2022, CS_TYPE_GB18030_2022_RADICAL_CS, false, true, 1}, {CS_TYPE_GB18030_2022_STROKE_CI, CHARSET_GB18030_2022, CS_TYPE_GB18030_2022_STROKE_CI, false, true, 1}, {CS_TYPE_GB18030_2022_STROKE_CS, CHARSET_GB18030_2022, CS_TYPE_GB18030_2022_STROKE_CS, false, true, 1}, -}; + {CS_TYPE_UTF8MB4_CROATIAN_CI, CHARSET_UTF8MB4, CS_TYPE_UTF8MB4_CROATIAN_CI, false, true, 8}, + {CS_TYPE_UTF8MB4_UNICODE_520_CI, CHARSET_UTF8MB4, CS_TYPE_UTF8MB4_UNICODE_520_CI, false, true, 8}, + {CS_TYPE_UTF8MB4_CZECH_CI, CHARSET_UTF8MB4, CS_TYPE_UTF8MB4_CZECH_CI, false, true, 8}, + {CS_TYPE_ASCII_GENERAL_CI, CHARSET_ASCII, CS_TYPE_ASCII_GENERAL_CI,true, true, 1}, + {CS_TYPE_ASCII_BIN, CHARSET_ASCII, CS_TYPE_ASCII_BIN,false, true, 1}, + {CS_TYPE_TIS620_THAI_CI, CHARSET_TIS620, CS_TYPE_TIS620_THAI_CI,true, true, 1}, + {CS_TYPE_TIS620_BIN, CHARSET_TIS620, CS_TYPE_TIS620_BIN,false, true, 1}, + {CS_TYPE_UTF8MB4_0900_AI_CI, CHARSET_UTF8MB4, CS_TYPE_UTF8MB4_0900_AI_CI, false, true, 1}, + }; ObCharsetInfo *ObCharset::charset_arr[CS_TYPE_MAX] = { NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, // 0 ~ 7 @@ -353,8 +363,11 @@ ObCharsetInfo *ObCharset::charset_arr[CS_TYPE_MAX] = { &ob_charset_gb18030_2022_stroke_cs, NULL, // 222 &ob_charset_utf8mb4_unicode_ci, // 224 NULL, NULL, NULL, NULL, NULL, NULL, NULL, // 225 - NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, // 232 - NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, // 240 + NULL, NULL, &ob_charset_utf8mb4_czech_uca_ci, // 232 + NULL, NULL, NULL, NULL, NULL, // 235 + NULL, NULL, NULL, NULL, NULL, &ob_charset_utf8mb4_croatian_uca_ci, //240 + &ob_charset_utf8mb4_unicode_520_ci, //246 + NULL, // 247 &ob_charset_gb18030_chinese_ci, // 248 &ob_charset_gb18030_bin, // 249 NULL, &ob_charset_gb18030_chinese_cs, // 250 @@ -1396,6 +1409,14 @@ const char *ObCharset::charset_name(ObCharsetType charset_type) ret_name = "gb18030_2022"; break; } + case CHARSET_ASCII: { + ret_name = "ascii"; + break; + } + case CHARSET_TIS620: { + ret_name = "tis620"; + break; + } default: { break; } @@ -1493,7 +1514,7 @@ const char* ObCharset::collation_level(const ObCollationLevel cs_level) ObCharsetType ObCharset::charset_type(const ObString &cs_name) { ObCharsetType charset_type = CHARSET_INVALID; - if (0 == cs_name.case_compare("utf8")) { + if (0 == cs_name.case_compare("utf8") || 0 == cs_name.case_compare("utf8mb3")) { // utf8是utf8mb4的别名 charset_type = CHARSET_UTF8MB4; } else if (0 == cs_name.case_compare(ob_charset_utf8mb4_bin.csname)) { @@ -1510,6 +1531,10 @@ ObCharsetType ObCharset::charset_type(const ObString &cs_name) charset_type = CHARSET_LATIN1; } else if (0 == cs_name.case_compare(ob_charset_gb18030_2022_bin.csname)) { charset_type = CHARSET_GB18030_2022; + } else if (0 == cs_name.case_compare(ob_charset_ascii_bin.csname)) { + charset_type = CHARSET_ASCII; + } else if (0 == cs_name.case_compare(ob_charset_tis620_bin.csname)) { + charset_type = CHARSET_TIS620; } return charset_type; } @@ -1530,6 +1555,10 @@ ObCharsetType ObCharset::charset_type_by_name_oracle(const ObString &cs_name) charset_type = CHARSET_LATIN1; } else if (0 == cs_name.case_compare("ZHS32GB18030_2022")) { charset_type = CHARSET_GB18030_2022; + } else if (0 == cs_name.case_compare("US7ASCII")) { + charset_type = CHARSET_ASCII; + } else if (0 == cs_name.case_compare("TH8TISASCII")) { + charset_type = CHARSET_TIS620; } return charset_type; } @@ -1550,9 +1579,11 @@ ObCharsetType ObCharset::charset_type(const char *cs_name) ObCollationType ObCharset::collation_type(const ObString &cs_name) { ObCollationType collation_type = CS_TYPE_INVALID; - if (0 == cs_name.case_compare("utf8_bin")) { + if (0 == cs_name.case_compare("utf8_bin") || + 0 == cs_name.case_compare("utf8mb3_bin")) { collation_type = CS_TYPE_UTF8MB4_BIN; - } else if (0 == cs_name.case_compare("utf8_general_ci")) { + } else if (0 == cs_name.case_compare("utf8_general_ci") || + 0 == cs_name.case_compare("utf8mb3_general_ci")) { collation_type = CS_TYPE_UTF8MB4_GENERAL_CI; } else if (0 == cs_name.case_compare(ob_charset_utf8mb4_bin.name)) { collation_type = CS_TYPE_UTF8MB4_BIN; @@ -1570,6 +1601,8 @@ ObCollationType ObCharset::collation_type(const ObString &cs_name) collation_type = CS_TYPE_UTF16_BIN; } else if (0 == cs_name.case_compare("utf8_unicode_ci")) { collation_type = CS_TYPE_UTF8MB4_UNICODE_CI; + } else if (0 == cs_name.case_compare("utf8mb4_0900_ai_ci")) { + collation_type = CS_TYPE_UTF8MB4_0900_AI_CI; } else if (0 == cs_name.case_compare(ob_charset_utf16_unicode_ci.name)) { collation_type = CS_TYPE_UTF16_UNICODE_CI; } else if (0 == cs_name.case_compare(ob_charset_utf8mb4_unicode_ci.name)) { @@ -1600,6 +1633,26 @@ ObCollationType ObCharset::collation_type(const ObString &cs_name) collation_type = CS_TYPE_GB18030_2022_STROKE_CI; } else if (0 == cs_name.case_compare(ob_charset_gb18030_2022_stroke_cs.name)) { collation_type = CS_TYPE_GB18030_2022_STROKE_CS; + } else if (0 == cs_name.case_compare("utf8_croatian_ci")) { + collation_type = CS_TYPE_UTF8MB4_CROATIAN_CI; + } else if (0 == cs_name.case_compare(ob_charset_utf8mb4_croatian_uca_ci.name)) { + collation_type = CS_TYPE_UTF8MB4_CROATIAN_CI; + } else if (0 == cs_name.case_compare("utf8_unicode_520_ci")) { + collation_type = CS_TYPE_UTF8MB4_UNICODE_520_CI; + } else if (0 == cs_name.case_compare(ob_charset_utf8mb4_unicode_520_ci.name)) { + collation_type = CS_TYPE_UTF8MB4_UNICODE_520_CI; + } else if (0 == cs_name.case_compare("utf8_czech_ci")) { + collation_type = CS_TYPE_UTF8MB4_CZECH_CI; + } else if (0 == cs_name.case_compare(ob_charset_utf8mb4_czech_uca_ci.name)) { + collation_type = CS_TYPE_UTF8MB4_CZECH_CI; + } else if (0 == cs_name.case_compare(ob_charset_ascii_bin.name)) { + collation_type = CS_TYPE_ASCII_BIN; + } else if (0 == cs_name.case_compare(ob_charset_ascii.name)) { + collation_type = CS_TYPE_ASCII_GENERAL_CI; + } else if (0 == cs_name.case_compare(ob_charset_tis620_bin.name)) { + collation_type = CS_TYPE_TIS620_BIN; + } else if (0 == cs_name.case_compare(ob_charset_tis620_thai_ci.name)) { + collation_type = CS_TYPE_TIS620_THAI_CI; } return collation_type; } @@ -1617,6 +1670,10 @@ bool ObCharset::is_valid_collation(ObCharsetType charset_type, ObCollationType c if (CS_TYPE_UTF8MB4_BIN == collation_type || CS_TYPE_UTF8MB4_GENERAL_CI == collation_type || CS_TYPE_UTF8MB4_UNICODE_CI == collation_type + || CS_TYPE_UTF8MB4_CROATIAN_CI == collation_type + || CS_TYPE_UTF8MB4_UNICODE_520_CI == collation_type + || CS_TYPE_UTF8MB4_CZECH_CI == collation_type + || CS_TYPE_UTF8MB4_0900_AI_CI == collation_type ) { ret = true; } @@ -1645,9 +1702,18 @@ bool ObCharset::is_valid_collation(ObCharsetType charset_type, ObCollationType c } } else if (CHARSET_GB18030_2022 == charset_type) { ret = is_gb18030_2022(collation_type); + } else if (CHARSET_ASCII == charset_type) { + if (CS_TYPE_ASCII_GENERAL_CI == collation_type || CS_TYPE_ASCII_BIN == collation_type) { + ret = true; + } + } else if (CHARSET_TIS620 == charset_type) { + if (CS_TYPE_TIS620_THAI_CI == collation_type || CS_TYPE_TIS620_BIN == collation_type) { + ret = true; + } } return ret; } + ObCollationType ObCharset::get_coll_type_by_nlssort_param(ObCharsetType charset_type, const ObString &nlssort_param) { @@ -1662,6 +1728,8 @@ ObCollationType ObCharset::get_coll_type_by_nlssort_param(ObCharsetType charset_ CS_TYPE_GB18030_BIN, CS_TYPE_LATIN1_BIN, CS_TYPE_GB18030_2022_BIN, + CS_TYPE_ASCII_BIN, + CS_TYPE_TIS620_BIN, }; static ObCollationType non_bin_coll_marks[NLS_COLLATION_MAX] = { CS_TYPE_INVALID, @@ -1698,10 +1766,7 @@ ObCollationType ObCharset::get_coll_type_by_nlssort_param(ObCharsetType charset_ } else if (nls_coll_type == NLS_COLLATION_SCHINESE_STROKE2_M) { coll_type = CS_TYPE_GB18030_2022_STROKE_CS; } else { - if (charset_type != CHARSET_LATIN1) { - coll_type = static_cast( - non_bin_coll_marks[nls_coll_type] + (charset_type - CHARSET_BINARY)); - } + coll_type = static_cast(non_bin_coll_marks[nls_coll_type] + (charset_type - CHARSET_BINARY)); } } return coll_type; @@ -1724,7 +1789,15 @@ bool ObCharset::is_valid_collation(int64_t collation_type_int) || CS_TYPE_LATIN1_BIN == collation_type || is_gb18030_2022(collation_type) || CS_TYPE_UTF8MB4_UNICODE_CI == collation_type + || CS_TYPE_UTF8MB4_0900_AI_CI == collation_type || CS_TYPE_UTF16_UNICODE_CI == collation_type + || CS_TYPE_UTF8MB4_CROATIAN_CI == collation_type + || CS_TYPE_UTF8MB4_UNICODE_520_CI == collation_type + || CS_TYPE_UTF8MB4_CZECH_CI == collation_type + || CS_TYPE_ASCII_GENERAL_CI == collation_type + || CS_TYPE_ASCII_BIN == collation_type + || CS_TYPE_TIS620_THAI_CI == collation_type + || CS_TYPE_TIS620_BIN == collation_type || (CS_TYPE_EXTENDED_MARK < collation_type && collation_type < CS_TYPE_MAX) ; } @@ -1739,6 +1812,10 @@ ObCharsetType ObCharset::charset_type_by_coll(ObCollationType collation_type) case CS_TYPE_UTF8MB4_ZH_0900_AS_CS: case CS_TYPE_UTF8MB4_ZH2_0900_AS_CS: case CS_TYPE_UTF8MB4_ZH3_0900_AS_CS: + case CS_TYPE_UTF8MB4_CROATIAN_CI: + case CS_TYPE_UTF8MB4_UNICODE_520_CI: + case CS_TYPE_UTF8MB4_CZECH_CI: + case CS_TYPE_UTF8MB4_0900_AI_CI: case CS_TYPE_UTF8MB4_UNICODE_CI: { charset_type = CHARSET_UTF8MB4; break; @@ -1774,7 +1851,10 @@ ObCharsetType ObCharset::charset_type_by_coll(ObCollationType collation_type) break; } case CS_TYPE_LATIN1_SWEDISH_CI: - case CS_TYPE_LATIN1_BIN: { + case CS_TYPE_LATIN1_BIN: + case CS_TYPE_LATIN1_ZH_0900_AS_CS: + case CS_TYPE_LATIN1_ZH2_0900_AS_CS: + case CS_TYPE_LATIN1_ZH3_0900_AS_CS: { charset_type = CHARSET_LATIN1; break; } @@ -1791,6 +1871,22 @@ ObCharsetType ObCharset::charset_type_by_coll(ObCollationType collation_type) charset_type = CHARSET_GB18030_2022; break; } + case CS_TYPE_ASCII_GENERAL_CI: + case CS_TYPE_ASCII_BIN: + case CS_TYPE_ASCII_ZH_0900_AS_CS: + case CS_TYPE_ASCII_ZH2_0900_AS_CS: + case CS_TYPE_ASCII_ZH3_0900_AS_CS: { + charset_type = CHARSET_ASCII; + break; + } + case CS_TYPE_TIS620_THAI_CI: + case CS_TYPE_TIS620_BIN: + case CS_TYPE_TIS620_ZH_0900_AS_CS: + case CS_TYPE_TIS620_ZH2_0900_AS_CS: + case CS_TYPE_TIS620_ZH3_0900_AS_CS: { + charset_type = CHARSET_TIS620; + break; + } default: { break; } @@ -1821,6 +1917,12 @@ ObNlsCharsetId ObCharset::charset_type_to_ora_charset_id(ObCharsetType cs_type) case CHARSET_GB18030_2022: cs_id = CHARSET_ZHS32GB18030_2022_ID; break; + case CHARSET_ASCII: + cs_id = CHARSET_US7ASCII_ID; + break; + case CHARSET_TIS620: + cs_id = CHARSET_TH8TISASCII_ID; + break; default: break; } @@ -1833,28 +1935,77 @@ ObCharsetType ObCharset::ora_charset_type_to_charset_type(ObNlsCharsetId charset switch (charset_id) { case CHARSET_AL32UTF8_ID: - cs_type = CHARSET_UTF8MB4; - break; + case CHARSET_UTF8_ID: + cs_type = CHARSET_UTF8MB4; + break; case CHARSET_ZHS16GBK_ID: - cs_type = CHARSET_GBK; - break; + cs_type = CHARSET_GBK; + break; case CHARSET_ZHS32GB18030_ID: - cs_type = CHARSET_GB18030; - break; + cs_type = CHARSET_GB18030; + break; case CHARSET_AL16UTF16_ID: - cs_type = CHARSET_UTF16; - break; + cs_type = CHARSET_UTF16; + break; case CHARSET_WE8MSWIN1252_ID: - cs_type = CHARSET_LATIN1; + cs_type = CHARSET_LATIN1; + break; case CHARSET_ZHS32GB18030_2022_ID: - cs_type = CHARSET_GB18030_2022; - break; + cs_type = CHARSET_GB18030_2022; + break; + case CHARSET_US7ASCII_ID: + cs_type = CHARSET_ASCII; + break; + case CHARSET_TH8TISASCII_ID: + cs_type = CHARSET_TIS620; + break; default: - break; + break; } return cs_type; } +ObCollationType ObCharset::ora_charset_type_to_coll_type(ObNlsCharsetId charset_id) +{ + ObCollationType coll_type = CS_TYPE_INVALID; + switch (charset_id) + { + case CHARSET_AL32UTF8_ID: + case CHARSET_UTF8_ID: + coll_type = CS_TYPE_UTF8MB4_BIN; + break; + case CHARSET_ZHS16GBK_ID: + coll_type = CS_TYPE_GBK_BIN; + break; + case CHARSET_ZHS32GB18030_ID: + coll_type = CS_TYPE_GB18030_BIN; + break; + case CHARSET_AL16UTF16_ID: + coll_type = CS_TYPE_UTF16_BIN; + break; + case CHARSET_WE8MSWIN1252_ID: + coll_type = CS_TYPE_LATIN1_BIN; + break; + case CHARSET_ZHS32GB18030_2022_ID: + coll_type = CS_TYPE_GB18030_2022_BIN; + break; + case CHARSET_US7ASCII_ID: + coll_type = CS_TYPE_ASCII_BIN; + break; + case CHARSET_TH8TISASCII_ID: + coll_type = CS_TYPE_TIS620_BIN; + break; + default: + break; + } + return coll_type; +} + +bool ObCharset::is_valid_ora_charset_id(ObNlsCharsetId charset_id) +{ + return CS_TYPE_INVALID != ora_charset_type_to_coll_type(charset_id); +} + bool ObCharset::is_valid_nls_collation(ObNLSCollation nls_collation) { return nls_collation > NLS_COLLATION_INVALID && nls_collation < NLS_COLLATION_MAX; @@ -1946,7 +2097,7 @@ int ObCharset::result_collation( return ret; } -int ObCharset::aggregate_collation( +int ObCharset::aggregate_collation_old( const ObCollationLevel collation_level1, const ObCollationType collation_type1, const ObCollationLevel collation_level2, @@ -2049,6 +2200,12 @@ int ObCharset::aggregate_collation( } else if (charset_type_by_coll(collation_type1) == CHARSET_GB18030_2022) { res_type = CS_TYPE_GB18030_2022_BIN; res_level = (CS_TYPE_GB18030_2022_BIN == collation_type1) ? collation_level1 : collation_level2; + } else if (charset_type_by_coll(collation_type1) == CHARSET_ASCII) { + res_type = CS_TYPE_ASCII_BIN; + res_level = (CS_TYPE_ASCII_BIN == collation_type1) ? collation_level1 : collation_level2; + } else if (charset_type_by_coll(collation_type1) == CHARSET_TIS620) { + res_type = CS_TYPE_TIS620_BIN; + res_level = (CS_TYPE_TIS620_BIN == collation_type1) ? collation_level1 : collation_level2; } else { ret = OB_ERR_UNEXPECTED; LOG_WARN("Unexpected charset", K(ret), K(collation_type1), K(collation_type2), KCSTRING(lbt())); @@ -2079,6 +2236,180 @@ int ObCharset::aggregate_collation( return ret; } +int ObCharset::aggregate_collation_new( + const ObCollationLevel collation_level1, + const ObCollationType collation_type1, + const ObCollationLevel collation_level2, + const ObCollationType collation_type2, + ObCollationLevel &res_level, + ObCollationType &res_type, + uint32_t flags) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY( + CS_LEVEL_INVALID == collation_level1 + || CS_LEVEL_INVALID == collation_level2 + || !is_valid_collation(collation_type1) + || !is_valid_collation(collation_type2))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN ("invalid collation level or type", + K(ret), K(collation_level1), K(collation_type1), K(collation_level2), K(collation_type2)); + } else if (collation_type1 != collation_type2 && + CS_LEVEL_EXPLICIT == collation_level1 && + CS_LEVEL_EXPLICIT == collation_level2) { + ret = OB_CANT_AGGREGATE_2COLLATIONS; + } else { + + ObCharsetInfo *cs1 = static_cast(ObCharset::charset_arr[collation_type1]); + ObCharsetInfo *cs2 = static_cast(ObCharset::charset_arr[collation_type2]); + ObCharsetType charset_type1 = charset_type_by_coll(collation_type1); + ObCharsetType charset_type2 = charset_type_by_coll(collation_type2); + + if (charset_type1 != charset_type2) { + if (CS_TYPE_BINARY == collation_type1) { + if (collation_level1 <= collation_level2) { + res_type = collation_type1; + res_level = collation_level1; + } else { + res_type = collation_type2; + res_level = collation_level2; + } + } else if (CS_TYPE_BINARY == collation_type2) { + if (collation_level2 <= collation_level1) { + res_type = collation_type2; + res_level = collation_level2; + } else { + res_type = collation_type1; + res_level = collation_level1; + } + } else if ((flags & OB_COLL_ALLOW_SUPERSET_CONV) && + left_is_superset(collation_level1, + collation_type1, + collation_level2, + collation_type2)) { + res_type = collation_type1; + res_level = collation_level1; + } else if ((flags & OB_COLL_ALLOW_SUPERSET_CONV) && + left_is_superset(collation_level2, + collation_type2, + collation_level1, + collation_type1)) { + res_type = collation_type2; + res_level = collation_level2; + } else if ((flags & OB_COLL_ALLOW_COERCIBLE_CONV) && + collation_level1 < collation_level2 && + collation_level2 >= CS_LEVEL_SYSCONST) { + res_type = collation_type1; + res_level = collation_level1; + } else if ((flags & OB_COLL_ALLOW_COERCIBLE_CONV) && + collation_level2 < collation_level1 && + collation_level1 >= CS_LEVEL_SYSCONST) { + res_type = collation_type2; + res_level = collation_level2; + } else { + // Cannot apply conversion + res_type = CS_TYPE_BINARY; + res_level = CS_LEVEL_NONE; + } + if (lib::is_oracle_mode()) { + if (charset_type1 == CHARSET_UTF8MB4 && charset_type2 == CHARSET_UTF16) { + res_type = collation_type2; + res_level = collation_level2; + } else if (charset_type1 == CHARSET_UTF16 && charset_type2 == CHARSET_UTF8MB4) { + res_type = collation_type1; + res_level = collation_level1; + } + } + } else if (collation_level1 < collation_level2) { + res_type = collation_type1; + res_level = collation_level1; + } else if (collation_level2 < collation_level1) { + res_type = collation_type2; + res_level = collation_level2; + } else if (collation_type1 == collation_type2) { + res_type = collation_type1; + res_level = collation_level1; + } else if (CS_LEVEL_EXPLICIT == collation_level1) { + ret = OB_CANT_AGGREGATE_2COLLATIONS; + // ERROR 1267 (HY000): Illegal mix of collations (utf8_general_ci,EXPLICIT) and (utf8_bin,EXPLICIT) for operation '=' + } else if ((cs1->state & OB_CS_BINSORT) && (cs2->state & OB_CS_BINSORT)) { + // If we have two different binary collations for the same character set, + // and none of them is explicit, we don't know which to choose. For + // example: utf8mb4_bin is a binary padding collation, utf8mb4_0900_bin is + // a binary non-padding collation. Cannot determine if the resulting + // collation should be padding or non-padding, unless they are also + // aggregated with a third explicit collation. + res_type = CS_TYPE_BINARY; + res_level = CS_LEVEL_NONE; + ret = OB_CANT_AGGREGATE_2COLLATIONS; + } else if (cs1->state & OB_CS_BINSORT) { + res_type = collation_type1; + res_level = collation_level1; + } else if (cs2->state & OB_CS_BINSORT) { + res_type = collation_type2; + res_level = collation_level2; + } else { + /* + test (c1 char(10) COLLATE utf8mb4_unicode_ci, c2 char(10) COLLATE utf8mb4_general_ci) + collation(concat(c1,c2)) = utf8mb4_bin; + */ + res_type = ObCharset::get_bin_collation(charset_type1); + res_level = CS_LEVEL_NONE; + } + if (OB_SUCC(ret)) { + ObCharsetType res_cs = charset_type_by_coll(res_type); + if (CHARSET_GB18030 == res_cs) { + if (CHARSET_GB18030_2022 == charset_type1 || CHARSET_GB18030_2022 == charset_type2) { + ret = OB_CANT_AGGREGATE_2COLLATIONS; + } + } else if (CHARSET_GB18030_2022 == res_cs) { + if (CHARSET_GB18030 == charset_type1 || CHARSET_GB18030 == charset_type2) { + ret = OB_CANT_AGGREGATE_2COLLATIONS; + } + } + } + } + + if (OB_FAIL(ret)) { + LOG_WARN("Illegal mix of collations", K(ret), + "type1", ObCharset::collation_name(collation_type1), + "level1", ObCharset::collation_level(collation_level1), + "type2", ObCharset::collation_name(collation_type2), + "level2", ObCharset::collation_level(collation_level2)); + } + return ret; +} +bool ObCharset::left_is_superset(const ObCollationLevel collation_level1, + const ObCollationType collation_type1, + const ObCollationLevel collation_level2, + const ObCollationType collation_type2) +{ + ObCharsetInfo *cs1 = static_cast(ObCharset::charset_arr[collation_type1]); + ObCharsetInfo *cs2 = static_cast(ObCharset::charset_arr[collation_type2]); + bool bret = false; + if (cs1->state & OB_CS_UNICODE && + (collation_level1 < collation_level2 || + (collation_level1 == collation_level2 && + (!(cs2->state & OB_CS_UNICODE) || + /* The code below makes 4-byte utf8 a superset over 3-byte utf8 */ + (cs1->state & OB_CS_UNICODE_SUPPLEMENT && + !(cs2->state & OB_CS_UNICODE_SUPPLEMENT) && + cs1->mbmaxlen > cs2->mbmaxlen && + cs1->mbminlen == cs2->mbminlen))))) { + bret = true; + } else if (test_all_bits(cs1->state, OB_CS_UNICODE| OB_CS_UNICODE_SUPPLEMENT) && + (cs2->state & OB_CS_UNICODE) && + collation_level1 == collation_level2) { + /* Allow convert from any Unicode to utf32 or utf8mb4 */ + bret = true; + } else if ((cs2->state & OB_CS_PUREASCII) && + (collation_level1 < collation_level2 || + (collation_level1 == collation_level2 && !(cs1->state & OB_CS_PUREASCII)))) { + /* Allow convert from ASCII */ + bret = true; + } + return bret; +} bool ObCharset::is_bin_sort(ObCollationType collation_type) { bool ret = false; @@ -2094,6 +2425,7 @@ bool ObCharset::is_bin_sort(ObCollationType collation_type) return ret; } + bool ObCharset::is_ci_collate(ObCollationType collation_type) { bool ret = false; @@ -2149,6 +2481,14 @@ ObCollationType ObCharset::get_default_collation(ObCharsetType charset_type) collation_type = CS_TYPE_GB18030_2022_PINYIN_CI; break; } + case CHARSET_ASCII: { + collation_type = CS_TYPE_ASCII_GENERAL_CI; + break; + } + case CHARSET_TIS620: { + collation_type = CS_TYPE_TIS620_THAI_CI; + break; + } default: { break; } @@ -2195,6 +2535,14 @@ ObCollationType ObCharset::get_default_collation_oracle(ObCharsetType charset_ty collation_type = CS_TYPE_GB18030_2022_BIN; break; } + case CHARSET_ASCII: { + collation_type = CS_TYPE_ASCII_BIN; + break; + } + case CHARSET_TIS620: { + collation_type = CS_TYPE_TIS620_BIN; + break; + } default: { break; } @@ -2234,6 +2582,14 @@ int ObCharset::get_default_collation(ObCharsetType charset_type, ObCollationType collation_type = CS_TYPE_GB18030_2022_PINYIN_CI; break; } + case CHARSET_ASCII: { + collation_type = CS_TYPE_ASCII_GENERAL_CI; + break; + } + case CHARSET_TIS620: { + collation_type = CS_TYPE_TIS620_THAI_CI; + break; + } default: { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid charset type", K(ret), K(charset_type)); @@ -2275,6 +2631,14 @@ ObCollationType ObCharset::get_bin_collation(ObCharsetType charset_type) collation_type = CS_TYPE_GB18030_2022_BIN; break; } + case CHARSET_ASCII: { + collation_type = CS_TYPE_ASCII_BIN; + break; + } + case CHARSET_TIS620: { + collation_type = CS_TYPE_TIS620_BIN; + break; + } default: { break; } @@ -2407,6 +2771,8 @@ bool ObCharset::is_default_collation(ObCollationType collation_type) case CS_TYPE_UTF16_GENERAL_CI: case CS_TYPE_GB18030_CHINESE_CI: case CS_TYPE_LATIN1_SWEDISH_CI: + case CS_TYPE_ASCII_GENERAL_CI: + case CS_TYPE_TIS620_THAI_CI: case CS_TYPE_GB18030_2022_PINYIN_CI: case CS_TYPE_BINARY: { ret = true; @@ -2847,6 +3213,8 @@ int ObCharset::get_aggregate_len_unit(const ObCollationType collation_type, bool ObCharsetType res_charset = ObCharset::charset_type_by_coll(collation_type); if (CHARSET_UTF8MB4 == res_charset || CHARSET_LATIN1 == res_charset + || CHARSET_ASCII == res_charset + || CHARSET_TIS620 == res_charset || CHARSET_UTF16 == res_charset || CHARSET_GBK == res_charset || CHARSET_GB18030 == res_charset @@ -3114,6 +3482,8 @@ bool ObCharset::is_valid_connection_collation(ObCollationType collation_type) ObCharsetType cs_type = ObCharset::charset_type_by_coll(collation_type); return cs_type == CHARSET_UTF8MB4 || cs_type == CHARSET_LATIN1 + || cs_type == CHARSET_ASCII + || cs_type == CHARSET_TIS620 || cs_type == CHARSET_GBK || cs_type == CHARSET_GB18030 || cs_type == CHARSET_GB18030_2022 @@ -3142,6 +3512,12 @@ const char *ObCharset::get_oracle_charset_name_by_charset_type(ObCharsetType cha case CHARSET_LATIN1: ret = "WE8MSWIN1252"; break; + case CHARSET_ASCII: + ret = "US7ASCII"; + break; + case CHARSET_TIS620: + ret = "TH8TISASCII"; + break; default: break; } @@ -3170,6 +3546,12 @@ int ObCharset::get_nls_charset_id_by_charset_type(ObCharsetType charset_type) case CHARSET_GB18030_2022: ret_id = ObNlsCharsetId::CHARSET_ZHS32GB18030_2022_ID; break; + case CHARSET_ASCII: + ret_id = ObNlsCharsetId::CHARSET_US7ASCII_ID; + break; + case CHARSET_TIS620: + ret_id = ObNlsCharsetId::CHARSET_TH8TISASCII_ID; + break; default: break; } @@ -3278,6 +3660,37 @@ int ObCharset::init_charset() ObCharsetLoader loader; ob_charset_loader_init_mysys(&loader); + add_coll(CS_TYPE_UTF8MB4_0900_BIN, &ob_charset_utf8mb4_0900_bin); + add_coll(CS_TYPE_UTF8MB4_0900_AI_CI, &ob_charset_utf8mb4_0900_ai_ci); + add_coll(CS_TYPE_TIS620_BIN, &ob_charset_tis620_bin); + add_coll(CS_TYPE_TIS620_THAI_CI, &ob_charset_tis620_thai_ci); + //init charset_handler&collation_handler for some special charset + ObCharsetInfo *special_charset[] = {&ob_charset_ascii,&ob_charset_ascii_bin}; + for (int i = 0; OB_SUCC(ret) && i < array_elements(special_charset); i++) { + ObCharsetInfo *cs = special_charset[i]; + ObCharsetHandler *charset_handler = cs->cset; + ObCollationHandler *coll_handler = cs->coll; + if (OB_ISNULL(charset_handler) || OB_ISNULL(coll_handler)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null pointer", K(charset_handler), K(coll_handler), K(ret)); + } else if (charset_handler->init && + charset_handler->init(cs, &loader)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to init charset handler", K(ret)); + } else { + add_coll((ObCollationType)cs->number, cs); + /*debug + if (cs->tab_from_uni) { + OB_UNI_IDX test = cs->tab_from_uni[0]; + //int size = sizeof(test->tab)/sizeof(test->tab[0]); + LOG_INFO("charset debug", K(cs->name),K(test.from), K(test.to)); + for (int i=0;i<255;i++) { + LOG_INFO("charset debug",K(i),K(test.tab[i])); + } + } + */ + } + } if (OB_SUCC(ret)) { auto *utf8_pinyin = &ob_charset_utf8mb4_zh_0900_as_cs; ObCollationHandler *pinyin_coll = ob_charset_utf8mb4_zh_0900_as_cs.coll; @@ -3289,7 +3702,8 @@ int ObCharset::init_charset() ObCollationType pinyin_colls[] = { CS_TYPE_GBK_ZH_0900_AS_CS, CS_TYPE_UTF8MB4_ZH_0900_AS_CS, CS_TYPE_GB18030_ZH_0900_AS_CS, CS_TYPE_UTF16_ZH_0900_AS_CS, - CS_TYPE_GB18030_2022_ZH_0900_AS_CS + CS_TYPE_GB18030_2022_ZH_0900_AS_CS, CS_TYPE_LATIN1_ZH_0900_AS_CS, + CS_TYPE_ASCII_ZH_0900_AS_CS,CS_TYPE_TIS620_ZH_0900_AS_CS }; add_coll(CS_TYPE_UTF8MB4_ZH_0900_AS_CS, utf8_pinyin); @@ -3316,7 +3730,8 @@ int ObCharset::init_charset() ObCollationType radical_colls[] = { CS_TYPE_GBK_ZH2_0900_AS_CS, CS_TYPE_UTF8MB4_ZH2_0900_AS_CS, CS_TYPE_GB18030_ZH2_0900_AS_CS, CS_TYPE_UTF16_ZH2_0900_AS_CS, - CS_TYPE_GB18030_2022_ZH2_0900_AS_CS + CS_TYPE_GB18030_2022_ZH2_0900_AS_CS,CS_TYPE_LATIN1_ZH2_0900_AS_CS, + CS_TYPE_ASCII_ZH2_0900_AS_CS,CS_TYPE_TIS620_ZH2_0900_AS_CS }; add_coll(CS_TYPE_UTF8MB4_ZH2_0900_AS_CS, utf8_radical); @@ -3343,7 +3758,8 @@ int ObCharset::init_charset() ObCollationType stroke_colls[] = { CS_TYPE_GBK_ZH3_0900_AS_CS, CS_TYPE_UTF8MB4_ZH3_0900_AS_CS, CS_TYPE_GB18030_ZH3_0900_AS_CS, CS_TYPE_UTF16_ZH3_0900_AS_CS, - CS_TYPE_GB18030_2022_ZH3_0900_AS_CS + CS_TYPE_GB18030_2022_ZH3_0900_AS_CS, CS_TYPE_LATIN1_ZH3_0900_AS_CS, + CS_TYPE_ASCII_ZH3_0900_AS_CS,CS_TYPE_TIS620_ZH3_0900_AS_CS }; add_coll(CS_TYPE_UTF8MB4_ZH3_0900_AS_CS, utf8_stroke); @@ -3363,6 +3779,14 @@ int ObCharset::init_charset() //init utf8_0900_binary add_coll(CS_TYPE_UTF8MB4_0900_BIN, &ob_charset_utf8mb4_0900_bin); + if (OB_SUCC(ret)) { + if (OB_FAIL(ob_charset_utf8mb4_croatian_uca_ci.coll->init(&ob_charset_utf8mb4_croatian_uca_ci, &loader))) { + LOG_WARN("fail to init collation", K(ret)); + } else if (OB_FAIL(ob_charset_utf8mb4_czech_uca_ci.coll->init(&ob_charset_utf8mb4_czech_uca_ci, &loader))) { + LOG_WARN("fail to init collation", K(ret)); + } + } + return ret; } diff --git a/deps/oblib/src/lib/charset/ob_charset.h b/deps/oblib/src/lib/charset/ob_charset.h index 4cfb012d1..236c9c93d 100644 --- a/deps/oblib/src/lib/charset/ob_charset.h +++ b/deps/oblib/src/lib/charset/ob_charset.h @@ -38,6 +38,8 @@ enum ObCharsetType CHARSET_GB18030 = 5, CHARSET_LATIN1 = 6, CHARSET_GB18030_2022 = 7, + CHARSET_ASCII = 8, + CHARSET_TIS620 = 9, CHARSET_MAX, }; @@ -49,21 +51,25 @@ enum ObCharsetType *there is no possibly to reach AGGREGATE_2CHARSET[CHARSET_UTF8MB4][CHARSET_UTF8MB4] and so on */ static const int AGGREGATE_2CHARSET[CHARSET_MAX][CHARSET_MAX] = { - //CHARSET_INVALI,CHARSET_UTF8MB4... -{0,0,0,0,0,0,0,0},//CHARSET_INVALI -{0,0,0,0,0,0,0,0},//CHARSET_BINARY -{0,0,0,1,2,1,1,1},//CHARSET_UTF8MB4 -{0,0,2,0,2,0,1,0},//CHARSET_GBK -{0,0,1,1,0,1,1,1},//CHARSET_UTF16 -{0,0,2,0,2,0,1,0},//CHARSET_GB18030 -{0,0,2,2,2,2,0,2},//CHARSET_LATIN1 -{0,0,2,0,2,0,1,0} //CHARSET_GB18030_2022 +//CHARSET_INVALI,CHARSET_UTF8MB4... + {0,0,0,0,0,0,0,0,0,0},//CHARSET_INVALI + {0,0,0,0,0,0,0,0,0,0},//CHARSET_BINARY + {0,0,0,1,2,1,1,1,1,1},//CHARSET_UTF8MB4 + {0,0,2,0,2,0,1,0,1,0},//CHARSET_GBK + {0,0,1,1,0,1,1,1,1,1},//CHARSET_UTF16 + {0,0,2,0,2,0,1,0,1,0},//CHARSET_GB18030 + {0,0,2,2,2,2,0,2,1,0},//CHARSET_LATIN1 + {0,0,2,0,2,0,1,0,1,0}, //CHARSET_GB18030_2022 + {0,0,2,2,2,2,2,2,0,2},//CHARSET_ASCII + {0,0,2,0,2,0,0,0,1,0},//CHARSET_TIS620 }; enum ObCollationType { CS_TYPE_INVALID = 0, CS_TYPE_LATIN1_SWEDISH_CI = 8, + CS_TYPE_ASCII_GENERAL_CI = 11, + CS_TYPE_TIS620_THAI_CI = 18, CS_TYPE_GBK_CHINESE_CI = 28, CS_TYPE_UTF8MB4_GENERAL_CI = 45, CS_TYPE_UTF8MB4_BIN = 46, @@ -71,7 +77,9 @@ enum ObCollationType CS_TYPE_UTF16_GENERAL_CI = 54, CS_TYPE_UTF16_BIN = 55, CS_TYPE_BINARY = 63, + CS_TYPE_ASCII_BIN = 65, CS_TYPE_GBK_BIN = 87, + CS_TYPE_TIS620_BIN = 89, CS_TYPE_COLLATION_FREE = 100, // mysql中间没有使用这个 CS_TYPE_UTF16_UNICODE_CI = 101, CS_TYPE_ANY = 125, // unused in mysql @@ -83,10 +91,14 @@ enum ObCollationType CS_TYPE_GB18030_2022_STROKE_CI = 221, // unused in mysql CS_TYPE_GB18030_2022_STROKE_CS = 222, // unused in mysql CS_TYPE_UTF8MB4_UNICODE_CI = 224, + CS_TYPE_UTF8MB4_CZECH_CI = 234, + CS_TYPE_UTF8MB4_CROATIAN_CI = 245, + CS_TYPE_UTF8MB4_UNICODE_520_CI = 246, CS_TYPE_GB18030_CHINESE_CI = 248, CS_TYPE_GB18030_BIN = 249, CS_TYPE_GB18030_CHINESE_CS = 251, + CS_TYPE_UTF8MB4_0900_AI_CI = 255, CS_TYPE_EXTENDED_MARK = 256, //the cs types below can not used for storing CS_TYPE_UTF8MB4_0900_BIN, //309 in mysql 8.0 @@ -96,24 +108,33 @@ enum ObCollationType CS_TYPE_GBK_ZH_0900_AS_CS, CS_TYPE_UTF16_ZH_0900_AS_CS, CS_TYPE_GB18030_ZH_0900_AS_CS, - CS_TYPE_latin1_ZH_0900_AS_CS, //invaid, not really used + CS_TYPE_LATIN1_ZH_0900_AS_CS, CS_TYPE_GB18030_2022_ZH_0900_AS_CS, + CS_TYPE_ASCII_ZH_0900_AS_CS, + CS_TYPE_TIS620_ZH_0900_AS_CS, + //radical-stroke order CS_TYPE_RADICAL_BEGIN_MARK, CS_TYPE_UTF8MB4_ZH2_0900_AS_CS, CS_TYPE_GBK_ZH2_0900_AS_CS, CS_TYPE_UTF16_ZH2_0900_AS_CS, CS_TYPE_GB18030_ZH2_0900_AS_CS, - CS_TYPE_latin1_ZH2_0900_AS_CS ,//invaid + CS_TYPE_LATIN1_ZH2_0900_AS_CS, CS_TYPE_GB18030_2022_ZH2_0900_AS_CS, + CS_TYPE_ASCII_ZH2_0900_AS_CS, + CS_TYPE_TIS620_ZH2_0900_AS_CS, + //stroke order CS_TYPE_STROKE_BEGIN_MARK, CS_TYPE_UTF8MB4_ZH3_0900_AS_CS, CS_TYPE_GBK_ZH3_0900_AS_CS, CS_TYPE_UTF16_ZH3_0900_AS_CS, CS_TYPE_GB18030_ZH3_0900_AS_CS, - CS_TYPE_latin1_ZH3_0900_AS_CS, //invaid + CS_TYPE_LATIN1_ZH3_0900_AS_CS, CS_TYPE_GB18030_2022_ZH3_0900_AS_CS, + CS_TYPE_ASCII_ZH3_0900_AS_CS, + CS_TYPE_TIS620_ZH3_0900_AS_CS, + CS_TYPE_MAX }; @@ -122,7 +143,9 @@ enum ObCollationType enum ObNlsCharsetId { CHARSET_INVALID_ID = 0, + CHARSET_US7ASCII_ID = 1, CHARSET_WE8MSWIN1252_ID=31, + CHARSET_TH8TISASCII_ID = 41, CHARSET_ZHS16GBK_ID = 852, CHARSET_ZHS32GB18030_ID = 854, CHARSET_ZHS32GB18030_2022_ID = 859, // not used in oracle @@ -211,8 +234,8 @@ public: //比如latin1 1byte ,utf8mb4 4byte,转换因子为4，也可以理解为最多使用4字节存储一个字符 static const int32_t CharConvertFactorNum = 4; - static const int64_t VALID_CHARSET_TYPES = 7; - static const int64_t VALID_COLLATION_TYPES = 20; + static const int64_t VALID_CHARSET_TYPES = 9; + static const int64_t VALID_COLLATION_TYPES = 31; static int init_charset(); // strntodv2 is an enhanced version of strntod, @@ -393,7 +416,9 @@ public: || CHARSET_UTF16 == charset_type || CHARSET_GB18030 == charset_type || CHARSET_GB18030_2022 == charset_type - || CHARSET_LATIN1 == charset_type; + || CHARSET_LATIN1 == charset_type + || CHARSET_ASCII == charset_type + || CHARSET_TIS620 == charset_type; } static bool is_gb18030_2022(int64_t coll_type_int) { ObCollationType coll_type = static_cast(coll_type_int); @@ -422,12 +447,24 @@ public: const ObCollationType type2, ObCollationLevel &res_level, ObCollationType &res_type); - static int aggregate_collation(const ObCollationLevel level1, + static int aggregate_collation_old(const ObCollationLevel level1, const ObCollationType type1, const ObCollationLevel level2, const ObCollationType type2, ObCollationLevel &res_level, ObCollationType &res_type); + static int aggregate_collation_new( + const ObCollationLevel collation_level1, + const ObCollationType collation_type1, + const ObCollationLevel collation_level2, + const ObCollationType collation_type2, + ObCollationLevel &res_level, + ObCollationType &res_type, + uint32_t flags); + static bool left_is_superset(const ObCollationLevel collation_level1, + const ObCollationType collation_type1, + const ObCollationLevel collation_level2, + const ObCollationType collation_type2); static bool is_bin_sort(ObCollationType collation_type); static bool is_ci_collate(ObCollationType collation_type); @@ -552,6 +589,8 @@ public: static ObNlsCharsetId charset_type_to_ora_charset_id(ObCharsetType cs_type); static ObCharsetType ora_charset_type_to_charset_type(ObNlsCharsetId charset_id); static bool is_valid_nls_collation(ObNLSCollation nls_collation); + static bool is_valid_ora_charset_id(ObNlsCharsetId charset_id); + static ObCollationType ora_charset_type_to_coll_type(ObNlsCharsetId charset_id); static ObCollationType get_coll_type_by_nlssort_param(ObCharsetType charset_type, const ObString &nlssort_param); private: @@ -599,12 +638,9 @@ public: if (OB_ERR_INCORRECT_STRING_VALUE == ret && ignore_invalid_character) { ret = common::OB_SUCCESS; wchar = INT32_MAX; - length = ObCharset::is_mbchar(collation_type, temp_str.ptr(), temp_str.ptr() + temp_str.length()); - if (length <= 0) { - int64_t min_len = 0; - ObCharset::get_mbminlen_by_coll(collation_type, min_len); - length = static_cast(min_len); - } + int64_t min_len = 0; + ObCharset::get_mbminlen_by_coll(collation_type, min_len); + length = static_cast(min_len); } } if (OB_SUCC(ret)) { diff --git a/deps/oblib/src/lib/charset/ob_charset_string_helper.h b/deps/oblib/src/lib/charset/ob_charset_string_helper.h index 6b66031ce..59e2d43a7 100644 --- a/deps/oblib/src/lib/charset/ob_charset_string_helper.h +++ b/deps/oblib/src/lib/charset/ob_charset_string_helper.h @@ -705,6 +705,16 @@ public: foreach_char_prototype(str, func, ignore_convert_failed, stop_when_truncated, truncated_len) : foreach_char_prototype(str, func, ignore_convert_failed, stop_when_truncated, truncated_len); break; + case CHARSET_ASCII: + ret = convert_unicode ? + foreach_char_prototype(str, func, ignore_convert_failed, stop_when_truncated, truncated_len) + : foreach_char_prototype(str, func, ignore_convert_failed, stop_when_truncated, truncated_len); + break; + case CHARSET_TIS620: + ret = convert_unicode ? + foreach_char_prototype(str, func, ignore_convert_failed, stop_when_truncated, truncated_len) + : foreach_char_prototype(str, func, ignore_convert_failed, stop_when_truncated, truncated_len); + break; case CHARSET_BINARY: ret = convert_unicode ? foreach_char_prototype(str, func, ignore_convert_failed, stop_when_truncated, truncated_len) diff --git a/deps/oblib/src/lib/charset/ob_ctype.h b/deps/oblib/src/lib/charset/ob_ctype.h index 3fef65bf2..5f0969d07 100644 --- a/deps/oblib/src/lib/charset/ob_ctype.h +++ b/deps/oblib/src/lib/charset/ob_ctype.h @@ -23,6 +23,7 @@ #define OB_UTF8MB4_GENERAL_CS OB_UTF8MB4 "_general_cs" #define OB_UTF8MB4_BIN OB_UTF8MB4 "_bin" #define OB_UTF8MB4_UNICODE_CI OB_UTF8MB4 "_unicode_ci" +#define OB_UTF8MB4_0900_AI_CI OB_UTF8MB4 "_0900_ai_ci" #define OB_UTF16 "utf16" @@ -195,7 +196,7 @@ typedef struct ObUnicaseInfo typedef struct ObCharsetHandler { - //my_bool (*init)(struct ObCharsetInfo *, MY_CHARSET_LOADER *loader); + bool (*init)(struct ObCharsetInfo *, ObCharsetLoader *loader); /* Multibyte routines */ unsigned int (*ismbchar)(const struct ObCharsetInfo *, const char *, const char *); @@ -320,6 +321,17 @@ typedef struct ObCollationHandler size_t len); } ObCollationHandler; +typedef struct OB_UNI_IDX { + uint16 from; + uint16 to; + const uchar *tab; +} OB_UNI_IDX; + +typedef struct { + int nchars; + OB_UNI_IDX uidx; +} uni_idx; + struct ObCharsetInfo { unsigned int number; @@ -336,8 +348,8 @@ struct ObCharsetInfo unsigned char *to_upper; unsigned char *sort_order; ObUCAInfo *uca; - //uint16 *tab_to_uni; - //MY_UNI_IDX *tab_from_uni; + uint16 *tab_to_uni; + OB_UNI_IDX *tab_from_uni; ObUnicaseInfo *caseinfo; unsigned char *state_map; unsigned char *ident_map; @@ -407,6 +419,17 @@ static inline unsigned int ob_ismbchar(const ObCharsetInfo *cs, const unsigned c #define ob_mbcharlen_2(s, a, b) ((s)->cset->mbcharlen((s), ((((a)&0xFF) << 8) + ((b)&0xFF)))) #define ob_mbmaxlenlen(s) ((s)->mbmaxlenlen) +#define test_all_bits(a, b) (((a) & (b)) == (b)) + +#define OB_COLL_ALLOW_SUPERSET_CONV 1 +#define OB_COLL_ALLOW_COERCIBLE_CONV 2 +#define OB_COLL_DISALLOW_NONE 4 +#define OB_COLL_ALLOW_NUMERIC_CONV 8 +#define OB_COLL_ALLOW_NEW_CONV 16 //allow new rule set for charset aggregation + +#define OB_COLL_ALLOW_CONV \ + (OB_COLL_ALLOW_SUPERSET_CONV | OB_COLL_ALLOW_COERCIBLE_CONV | OB_COLL_ALLOW_NEW_CONV) +#define OB_COLL_CMP_CONV (OB_COLL_ALLOW_CONV | OB_COLL_DISALLOW_NONE) typedef struct ob_uni_ctype { @@ -426,7 +449,6 @@ extern ObUnicaseInfo ob_unicase_unicode520; extern ObCharsetInfo ob_charset_bin; extern ObCharsetInfo ob_charset_utf8mb4_bin; extern ObCharsetInfo ob_charset_utf8mb4_general_ci; -extern ObCharsetInfo ob_charset_latin1; extern ObCharsetInfo ob_charset_gbk_chinese_ci; extern ObCharsetInfo ob_charset_gbk_bin; extern ObCharsetInfo ob_charset_utf16_general_ci; @@ -441,15 +463,22 @@ extern ObCharsetInfo ob_charset_gb18030_2022_radical_cs; extern ObCharsetInfo ob_charset_gb18030_2022_stroke_ci; extern ObCharsetInfo ob_charset_gb18030_2022_stroke_cs; extern ObCharsetInfo ob_charset_gb18030_2022_bin; -extern ObCharsetInfo ob_charset_utf8mb4_unicode_ci; -extern ObCharsetInfo ob_charset_utf16_unicode_ci; extern ObCharsetInfo ob_charset_utf8mb4_zh_0900_as_cs; extern ObCharsetInfo ob_charset_utf8mb4_zh2_0900_as_cs; extern ObCharsetInfo ob_charset_utf8mb4_zh3_0900_as_cs; +extern ObCharsetInfo ob_charset_utf8mb4_unicode_ci; extern ObCharsetInfo ob_charset_utf8mb4_0900_bin; +extern ObCharsetInfo ob_charset_utf8mb4_0900_ai_ci; +extern ObCharsetInfo ob_charset_utf16_unicode_ci; extern ObCharsetInfo ob_charset_latin1; extern ObCharsetInfo ob_charset_latin1_bin; - +extern ObCharsetInfo ob_charset_utf8mb4_croatian_uca_ci; +extern ObCharsetInfo ob_charset_utf8mb4_unicode_520_ci; +extern ObCharsetInfo ob_charset_utf8mb4_czech_uca_ci; +extern ObCharsetInfo ob_charset_ascii; +extern ObCharsetInfo ob_charset_ascii_bin; +extern ObCharsetInfo ob_charset_tis620_thai_ci; +extern ObCharsetInfo ob_charset_tis620_bin; extern ObCollationHandler ob_collation_mb_bin_handler; extern ObCharsetHandler ob_charset_utf8mb4_handler; extern ObCharsetHandler ob_charset_utf16_handler; @@ -630,7 +659,10 @@ size_t ob_strxfrm_pad(const ObCharsetInfo *cs, unsigned char *str, unsigned char unsigned char *strend, unsigned int nweights, unsigned int flags); size_t ob_strnxfrmlen_simple(const struct ObCharsetInfo *, size_t); - +int ob_wildcmp_8bit(const ObCharsetInfo* cs, const char* str, const char* str_end, const char* wildstr, + const char* wildend, int escape, int w_one, int w_many); +uint32_t ob_instr_simple(const ObCharsetInfo* cs , const char* b, size_t b_length, + const char* s, size_t s_length, ob_match_t* match, unsigned int nmatch); size_t ob_strnxfrmlen_unicode_full_bin(const struct ObCharsetInfo *, size_t); size_t ob_strnxfrmlen_utf8mb4(const struct ObCharsetInfo *, size_t); @@ -667,11 +699,23 @@ char *strmake(char *, const char *, size_t); size_t ob_casedn_8bit(const ObCharsetInfo *cs __attribute__((unused)), char* str __attribute__((unused)), size_t srclen __attribute__((unused)), char* dst __attribute__((unused)), size_t dstlen __attribute__((unused))); - +int ob_strcasecmp_8bit(const ObCharsetInfo *cs, const char *s, const char *t); size_t ob_caseup_8bit(const ObCharsetInfo *cs __attribute__((unused)), char* str __attribute__((unused)), size_t srclen __attribute__((unused)), char* dst __attribute__((unused)), size_t dstlen __attribute__((unused))); +bool ob_cset_init_8bit(ObCharsetInfo *cs, ObCharsetLoader *loader); +bool ob_coll_init_simple(ObCharsetInfo *cs, ObCharsetLoader *loader); +size_t ob_well_formed_len_ascii(const ObCharsetInfo *cs, + const char *start, const char *end, + size_t nchars, + int *error); +int ob_8bit(const ObCharsetInfo *cs, ob_wc_t *wc, const uchar *str, + const uchar *end); +int ob_wc_mb_8bit(const ObCharsetInfo *cs, ob_wc_t wc, uchar *str, uchar *end); +int ob_mb_wc_8bit(const ObCharsetInfo *cs, ob_wc_t *wc, const uchar *str,const uchar *end); +unsigned int ob_ismbchar_8bit(const ObCharsetInfo *cs __attribute__((unused)), const char *p, const char *e); + extern "C" void right_to_die_or_duty_to_live_c(); diff --git a/deps/oblib/src/lib/charset/ob_ctype_ascii.cc b/deps/oblib/src/lib/charset/ob_ctype_ascii.cc new file mode 100644 index 000000000..d2482d047 --- /dev/null +++ b/deps/oblib/src/lib/charset/ob_ctype_ascii.cc @@ -0,0 +1,114 @@ +/** + * Copyright (code) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "lib/charset/ob_mysql_global.h" +#include "lib/charset/ob_ctype.h" +#include "lib/utility/ob_macro_utils.h" +#include "lib/charset/ob_ctype_ascii_tab.h" + +static ObCharsetHandler ob_charset_ascii_handler = { + ob_cset_init_8bit, + ob_ismbchar_8bit, + ob_mbcharlen_8bit, + ob_numchars_8bit, + ob_charpos_8bit, + ob_max_bytes_charpos_8bit, + ob_well_formed_len_ascii, + ob_lengthsp_8bit, + //ob_numcells_8bit, + ob_mb_wc_8bit, + ob_wc_mb_8bit, + ob_mb_ctype_8bit, + //ob_caseup_str_8bit, + //ob_casedn_str_8bit, + ob_caseup_8bit, + ob_casedn_8bit, + //ob_snprintf_8bit, + //ob_long10_to_str_8bit, + //ob_longlong10_to_str_8bit, + ob_fill_8bit, + ob_strntol_8bit, + ob_strntoul_8bit, + ob_strntoll_8bit, + ob_strntoull_8bit, + ob_strntod_8bit, + //ob_strtoll10_8bit, + ob_strntoull10rnd_8bit, + ob_scan_8bit}; + +ObCharsetInfo ob_charset_ascii = { + 11,0,0, + OB_CS_COMPILED | OB_CS_PRIMARY | OB_CS_PUREASCII, + "ascii", + "ascii_general_ci", + "US ASCII", + NULL, + NULL, + ctype_ascii_general_ci, + to_lower_ascii_general_ci, + to_upper_ascii_general_ci, + sort_order_ascii_general_ci, + NULL, + to_uni_ascii_general_ci, + NULL, + &ob_unicase_default, + NULL, + NULL, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 255, + ' ', + false, + 1, + 1, + &ob_charset_ascii_handler, + &ob_collation_8bit_simple_ci_handler, + PAD_SPACE}; + +ObCharsetInfo ob_charset_ascii_bin = { + 65,0,0, + OB_CS_COMPILED | OB_CS_BINSORT | OB_CS_PUREASCII, + "ascii", + "ascii_bin", + "US ASCII", + NULL, + NULL, + ctype_ascii_bin, + to_lower_ascii_bin, + to_upper_ascii_bin, + NULL, + NULL, + to_uni_ascii_bin, + nullptr, + &ob_unicase_default, + NULL, + NULL, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 255, + ' ', + false, + 1, + 1, + &ob_charset_ascii_handler, + &ob_collation_8bit_bin_handler, + PAD_SPACE}; \ No newline at end of file diff --git a/deps/oblib/src/lib/charset/ob_ctype_ascii_tab.h b/deps/oblib/src/lib/charset/ob_ctype_ascii_tab.h new file mode 100644 index 000000000..134c30036 --- /dev/null +++ b/deps/oblib/src/lib/charset/ob_ctype_ascii_tab.h @@ -0,0 +1,242 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +static unsigned char ctype_ascii_general_ci[] = { + 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x28, 0x28, + 0x28, 0x28, 0x28, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x48, 0x10, 0x10, + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, + 0x10, 0x84, 0x84, 0x84, 0x84, 0x84, 0x84, 0x84, 0x84, 0x84, 0x84, 0x10, + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x10, 0x10, 0x10, 0x10, + 0x10, 0x10, 0x82, 0x82, 0x82, 0x82, 0x82, 0x82, 0x02, 0x02, 0x02, 0x02, + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, + 0x02, 0x02, 0x02, 0x02, 0x10, 0x10, 0x10, 0x10, 0x20, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00}; + +static unsigned char to_lower_ascii_general_ci[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, + 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23, + 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, + 0x3C, 0x3D, 0x3E, 0x3F, 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F, + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, + 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F, 0x80, 0x81, 0x82, 0x83, + 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B, + 0x9C, 0x9D, 0x9E, 0x9F, 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, + 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0xB3, + 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF, + 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xCB, + 0xCC, 0xCD, 0xCE, 0xCF, 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, + 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 0xE0, 0xE1, 0xE2, 0xE3, + 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, + 0xFC, 0xFD, 0xFE, 0xFF}; + +static unsigned char to_upper_ascii_general_ci[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, + 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23, + 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, + 0x3C, 0x3D, 0x3E, 0x3F, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, + 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F, + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, + 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + 0x58, 0x59, 0x5A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F, 0x80, 0x81, 0x82, 0x83, + 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B, + 0x9C, 0x9D, 0x9E, 0x9F, 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, + 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0xB3, + 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF, + 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xCB, + 0xCC, 0xCD, 0xCE, 0xCF, 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, + 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 0xE0, 0xE1, 0xE2, 0xE3, + 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, + 0xFC, 0xFD, 0xFE, 0xFF}; + +static unsigned char sort_order_ascii_general_ci[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, + 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23, + 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, + 0x3C, 0x3D, 0x3E, 0x3F, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, + 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F, + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, + 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + 0x58, 0x59, 0x5A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F, 0x80, 0x81, 0x82, 0x83, + 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B, + 0x9C, 0x9D, 0x9E, 0x9F, 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, + 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0xB3, + 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF, + 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xCB, + 0xCC, 0xCD, 0xCE, 0xCF, 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, + 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 0xE0, 0xE1, 0xE2, 0xE3, + 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, + 0xFC, 0xFD, 0xFE, 0xFF}; + +static uint16 to_uni_ascii_general_ci[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, + 0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x000F, 0x0010, 0x0011, + 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001A, + 0x001B, 0x001C, 0x001D, 0x001E, 0x001F, 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002A, 0x002B, 0x002C, + 0x002D, 0x002E, 0x002F, 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, + 0x0036, 0x0037, 0x0038, 0x0039, 0x003A, 0x003B, 0x003C, 0x003D, 0x003E, + 0x003F, 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F, 0x0050, + 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, + 0x005A, 0x005B, 0x005C, 0x005D, 0x005E, 0x005F, 0x0060, 0x0061, 0x0062, + 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006A, 0x006B, + 0x006C, 0x006D, 0x006E, 0x006F, 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, + 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007A, 0x007B, 0x007C, 0x007D, + 0x007E, 0x007F, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000}; + + +static unsigned char ctype_ascii_bin[] = { + 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x28, 0x28, + 0x28, 0x28, 0x28, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x48, 0x10, 0x10, + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, + 0x10, 0x84, 0x84, 0x84, 0x84, 0x84, 0x84, 0x84, 0x84, 0x84, 0x84, 0x10, + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x10, 0x10, 0x10, 0x10, + 0x10, 0x10, 0x82, 0x82, 0x82, 0x82, 0x82, 0x82, 0x02, 0x02, 0x02, 0x02, + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, + 0x02, 0x02, 0x02, 0x02, 0x10, 0x10, 0x10, 0x10, 0x20, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00}; + +static unsigned char to_lower_ascii_bin[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, + 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23, + 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, + 0x3C, 0x3D, 0x3E, 0x3F, 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F, + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, + 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F, 0x80, 0x81, 0x82, 0x83, + 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B, + 0x9C, 0x9D, 0x9E, 0x9F, 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, + 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0xB3, + 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF, + 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xCB, + 0xCC, 0xCD, 0xCE, 0xCF, 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, + 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 0xE0, 0xE1, 0xE2, 0xE3, + 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, + 0xFC, 0xFD, 0xFE, 0xFF}; + +static unsigned char to_upper_ascii_bin[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, + 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23, + 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, + 0x3C, 0x3D, 0x3E, 0x3F, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, + 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F, + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, + 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + 0x58, 0x59, 0x5A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F, 0x80, 0x81, 0x82, 0x83, + 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B, + 0x9C, 0x9D, 0x9E, 0x9F, 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, + 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0xB3, + 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF, + 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xCB, + 0xCC, 0xCD, 0xCE, 0xCF, 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, + 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 0xE0, 0xE1, 0xE2, 0xE3, + 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, + 0xFC, 0xFD, 0xFE, 0xFF}; + +static uint16 to_uni_ascii_bin[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, + 0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x000F, 0x0010, 0x0011, + 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001A, + 0x001B, 0x001C, 0x001D, 0x001E, 0x001F, 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002A, 0x002B, 0x002C, + 0x002D, 0x002E, 0x002F, 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, + 0x0036, 0x0037, 0x0038, 0x0039, 0x003A, 0x003B, 0x003C, 0x003D, 0x003E, + 0x003F, 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F, 0x0050, + 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, + 0x005A, 0x005B, 0x005C, 0x005D, 0x005E, 0x005F, 0x0060, 0x0061, 0x0062, + 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006A, 0x006B, + 0x006C, 0x006D, 0x006E, 0x006F, 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, + 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007A, 0x007B, 0x007C, 0x007D, + 0x007E, 0x007F, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000}; diff --git a/deps/oblib/src/lib/charset/ob_ctype_bin.cc b/deps/oblib/src/lib/charset/ob_ctype_bin.cc index c5ad3a55a..229c690ba 100644 --- a/deps/oblib/src/lib/charset/ob_ctype_bin.cc +++ b/deps/oblib/src/lib/charset/ob_ctype_bin.cc @@ -384,10 +384,16 @@ void ob_hash_sort_bin(const ObCharsetInfo *cs __attribute__((unused)), } } - +extern "C" { +static bool ob_coll_init_8bit_bin(ObCharsetInfo *cs, ObCharsetLoader *loader) { + cs->max_sort_char = 255; + return false; +} +} static ObCharsetHandler ob_charset_handler= { + NULL, NULL, ob_mbcharlen_8bit, ob_numchars_8bit, @@ -412,7 +418,7 @@ static ObCharsetHandler ob_charset_handler= ObCollationHandler ob_collation_8bit_bin_handler = { - NULL, /* init */ + ob_coll_init_8bit_bin, /* init */ NULL, /* uninit */ ob_strnncoll_8bit_bin, ob_strnncollsp_8bit_bin, @@ -459,6 +465,8 @@ ObCharsetInfo ob_charset_bin = bin_char_array, NULL, NULL, + NULL, + NULL, &ob_unicase_default, NULL, NULL, diff --git a/deps/oblib/src/lib/charset/ob_ctype_gb18030.cc b/deps/oblib/src/lib/charset/ob_ctype_gb18030.cc index 87c08e14b..00729c01f 100644 --- a/deps/oblib/src/lib/charset/ob_ctype_gb18030.cc +++ b/deps/oblib/src/lib/charset/ob_ctype_gb18030.cc @@ -962,6 +962,7 @@ static ObCollationHandler ob_collation_cs_handler = }; static ObCharsetHandler ob_charset_gb18030_handler = { + NULL, ob_ismbchar_gb18030, ob_mbcharlen_gb18030, ob_numchars_mb, @@ -1005,8 +1006,8 @@ ObCharsetInfo ob_charset_gb18030_chinese_ci = { to_upper_gb18030, /* UPPER */ sort_order_gb18030_ci, /* sort */ NULL, /* uca */ - // NULL, /* tab_to_uni */ - // NULL, /* tab_from_uni */ + NULL, /* tab_to_uni */ + NULL, /* tab_from_uni */ &ob_caseinfo_gb18030, /* caseinfo */ NULL, /* state_map */ NULL, /* ident_map */ @@ -1041,8 +1042,8 @@ ObCharsetInfo ob_charset_gb18030_chinese_cs = { to_upper_gb18030, /* UPPER */ sort_order_gb18030, /* sort */ NULL, /* uca */ - // NULL, /* tab_to_uni */ - // NULL, /* tab_from_uni */ + NULL, /* tab_to_uni */ + NULL, /* tab_from_uni */ &ob_caseinfo_gb18030, /* caseinfo */ NULL, /* state_map */ NULL, /* ident_map */ @@ -1077,8 +1078,8 @@ ObCharsetInfo ob_charset_gb18030_bin = { to_upper_gb18030, NULL, NULL, - - + NULL, + NULL, &ob_caseinfo_gb18030, NULL, NULL, @@ -1599,6 +1600,7 @@ static ObCollationHandler ob_collation_2022_stroke_cs_handler = static ObCharsetHandler ob_charset_gb18030_2022_handler = { + NULL, ob_ismbchar_gb18030, ob_mbcharlen_gb18030, ob_numchars_mb, @@ -1637,6 +1639,8 @@ ObCharsetInfo ob_charset_gb18030_2022_bin = to_upper_gb18030, /* UPPER */ NULL, /* sort order */ NULL, /* uca */ + NULL, + NULL, &ob_caseinfo_gb18030, /* caseinfo */ NULL, /* state_map */ NULL, /* ident_map */ @@ -1673,6 +1677,8 @@ ObCharsetInfo ob_charset_gb18030_2022_pinyin_ci = to_upper_gb18030, /* UPPER */ sort_order_gb18030_ci, /* sort order */ NULL, /* uca */ + NULL, + NULL, &ob_caseinfo_gb18030, /* caseinfo */ NULL, /* state_map */ NULL, /* ident_map */ @@ -1709,6 +1715,8 @@ ObCharsetInfo ob_charset_gb18030_2022_pinyin_cs = to_upper_gb18030, /* UPPER */ sort_order_gb18030, /* sort order */ NULL, /* uca */ + NULL, + NULL, &ob_caseinfo_gb18030, /* caseinfo */ NULL, /* state_map */ NULL, /* ident_map */ @@ -1745,6 +1753,8 @@ ObCharsetInfo ob_charset_gb18030_2022_radical_ci = to_upper_gb18030, /* UPPER */ sort_order_gb18030_ci, /* sort order */ NULL, /* uca */ + NULL, + NULL, &ob_caseinfo_gb18030, /* caseinfo */ NULL, /* state_map */ NULL, /* ident_map */ @@ -1781,6 +1791,8 @@ ObCharsetInfo ob_charset_gb18030_2022_radical_cs = to_upper_gb18030, /* UPPER */ sort_order_gb18030, /* sort order */ NULL, /* uca */ + NULL, + NULL, &ob_caseinfo_gb18030, /* caseinfo */ NULL, /* state_map */ NULL, /* ident_map */ @@ -1817,6 +1829,8 @@ ObCharsetInfo ob_charset_gb18030_2022_stroke_ci = to_upper_gb18030, /* UPPER */ sort_order_gb18030_ci, /* sort order */ NULL, /* uca */ + NULL, + NULL, &ob_caseinfo_gb18030, /* caseinfo */ NULL, /* state_map */ NULL, /* ident_map */ @@ -1853,6 +1867,8 @@ ObCharsetInfo ob_charset_gb18030_2022_stroke_cs = to_upper_gb18030, /* UPPER */ sort_order_gb18030, /* sort order */ NULL, /* uca */ + NULL, + NULL, &ob_caseinfo_gb18030, /* caseinfo */ NULL, /* state_map */ NULL, /* ident_map */ diff --git a/deps/oblib/src/lib/charset/ob_ctype_gbk.cc b/deps/oblib/src/lib/charset/ob_ctype_gbk.cc index 51f46efdc..07510e0e3 100644 --- a/deps/oblib/src/lib/charset/ob_ctype_gbk.cc +++ b/deps/oblib/src/lib/charset/ob_ctype_gbk.cc @@ -345,6 +345,7 @@ static ObCollationHandler ob_collation_gbk_ci_handler = static ObCharsetHandler ob_charset_gbk_handler= { + NULL, ismbchar_gbk, mbcharlen_gbk, ob_numchars_mb, @@ -385,6 +386,8 @@ ObCharsetInfo ob_charset_gbk_chinese_ci= to_upper_gbk, sort_order_gbk, NULL, + NULL, + NULL, &ob_caseinfo_gbk, NULL, NULL, @@ -418,6 +421,8 @@ ObCharsetInfo ob_charset_gbk_bin= to_upper_gbk, NULL, NULL, + NULL, + NULL, &ob_caseinfo_gbk, NULL, NULL, diff --git a/deps/oblib/src/lib/charset/ob_ctype_latin1.cc b/deps/oblib/src/lib/charset/ob_ctype_latin1.cc index 50e122c4c..173d67f1a 100644 --- a/deps/oblib/src/lib/charset/ob_ctype_latin1.cc +++ b/deps/oblib/src/lib/charset/ob_ctype_latin1.cc @@ -40,8 +40,8 @@ static int ob_wc_mb_latin1(const ObCharsetInfo *cs __attribute__((unused)), static ObCharsetHandler ob_charset_latin1_handler= { - //NULL, NULL, + ob_ismbchar_8bit, ob_mbcharlen_8bit, ob_numchars_8bit, ob_charpos_8bit, @@ -83,8 +83,8 @@ ObCharsetInfo ob_charset_latin1 = { to_upper_latin1, sort_order_latin1, NULL, - //NULL, - //NULL, + NULL, + NULL, &ob_unicase_default, NULL, NULL, @@ -107,7 +107,7 @@ ObCharsetInfo ob_charset_latin1 = { ObCharsetInfo ob_charset_latin1_bin = { 47,0,0, OB_CS_COMPILED | OB_CS_BINSORT, - OB_LATIN1, + OB_LATIN1, OB_LATIN1_BIN, "cp1252 West European", NULL, @@ -117,8 +117,8 @@ ObCharsetInfo ob_charset_latin1_bin = { to_upper_latin1, NULL, NULL, - //NULL, - //NULL, + NULL, + NULL, &ob_unicase_default, NULL, NULL, diff --git a/deps/oblib/src/lib/charset/ob_ctype_simple.cc b/deps/oblib/src/lib/charset/ob_ctype_simple.cc index bae6727b3..31f3378ea 100644 --- a/deps/oblib/src/lib/charset/ob_ctype_simple.cc +++ b/deps/oblib/src/lib/charset/ob_ctype_simple.cc @@ -26,6 +26,9 @@ #define CUTLIM (UINT64_MAX % 10) #define SPACE_INT 0x20202020 #define DIGITS_IN_ULONGLONG 20 +#define PLANE_SIZE 0x100 +#define PLANE_NUM 0x100 +#define PLANE_NUMBER(x) (((x) >> 8) % PLANE_NUM) static ulonglong d10[DIGITS_IN_ULONGLONG]= { @@ -848,6 +851,13 @@ size_t ob_casedn_8bit(const ObCharsetInfo *cs __attribute__((unused)), return srclen; } +int ob_strcasecmp_8bit(const ObCharsetInfo *cs, const char *s, const char *t) { + const uchar *map = cs->to_upper; + while (map[(uchar)*s] == map[(uchar)*t++]) + if (!*s++) return 0; + return ((int)map[(uchar)s[0]] - (int)map[(uchar)t[-1]]); +} + int ob_strnncoll_simple(const ObCharsetInfo *cs __attribute__((unused)), const unsigned char *s, size_t slen, const unsigned char *t, size_t tlen, @@ -1073,8 +1083,176 @@ uint32_t ob_instr_simple(const ObCharsetInfo* cs , const char* b, size_t b_lengt return 0; } +static int pcmp(const void *f, const void *s) { + const uni_idx *F = (const uni_idx *)f; + const uni_idx *S = (const uni_idx *)s; + int res; + + if (!(res = ((S->nchars) - (F->nchars)))) + res = ((F->uidx.from) - (S->uidx.to)); + return res; +} + +static bool create_fromuni(ObCharsetInfo *cs, ObCharsetLoader *loader) +{ + uni_idx idx[PLANE_NUM]; + int i, n; + OB_UNI_IDX *tab_from_uni; + + /* + Check that Unicode map is loaded. + It can be not loaded when the collation is + listed in Index.xml but not specified + in the character set specific XML file. + */ + if (!cs->tab_to_uni) return true; + + /* Clear plane statistics */ + memset(idx, 0, sizeof(idx)); + + /* Count number of characters in each plane */ + for (i = 0; i < 0x100; i++) { + uint16 wc = cs->tab_to_uni[i]; + int pl = PLANE_NUMBER(wc); + + if (wc || !i) { + if (!idx[pl].nchars) { + idx[pl].uidx.from = wc; + idx[pl].uidx.to = wc; + } else { + idx[pl].uidx.from = wc < idx[pl].uidx.from ? wc : idx[pl].uidx.from; + idx[pl].uidx.to = wc > idx[pl].uidx.to ? wc : idx[pl].uidx.to; + } + idx[pl].nchars++; + } + } + + /* Sort planes in descending order */ + qsort(&idx, PLANE_NUM, sizeof(uni_idx), &pcmp); + + for (i = 0; i < PLANE_NUM; i++) { + int ch, numchars; + uchar *tab; + + /* Skip empty plane */ + if (!idx[i].nchars) break; + + numchars = idx[i].uidx.to - idx[i].uidx.from + 1; + if (!(idx[i].uidx.tab = tab = (uchar *)(loader->once_alloc)( + numchars * sizeof(*idx[i].uidx.tab)))) + return true; + + memset(tab, 0, numchars * sizeof(*idx[i].uidx.tab)); + + for (ch = 1; ch < PLANE_SIZE; ch++) { + uint16 wc = cs->tab_to_uni[ch]; + if (wc >= idx[i].uidx.from && wc <= idx[i].uidx.to && wc) { + int ofs = wc - idx[i].uidx.from; + /* + Character sets like armscii8 may have two code points for + one character. When converting from UNICODE back to + armscii8, select the lowest one, which is in the ASCII + range. + */ + if (tab[ofs] == '\0') tab[ofs] = ch; + } + } + } + + /* Allocate and fill reverse table for each plane */ + n = i; + if (!(cs->tab_from_uni = tab_from_uni = + (OB_UNI_IDX *)(loader->once_alloc)(sizeof(OB_UNI_IDX) * (n + 1)))) + return true; + + for (i = 0; i < n; i++) tab_from_uni[i] = idx[i].uidx; + + /* Set end-of-list marker */ + memset(&tab_from_uni[i], 0, sizeof(OB_UNI_IDX)); + return false; +} + +bool ob_cset_init_8bit(ObCharsetInfo *cs, ObCharsetLoader *loader) +{ + cs->caseup_multiply = 1; + cs->casedn_multiply = 1; + cs->pad_char = ' '; + return create_fromuni(cs, loader); +} + +static void set_max_sort_char(ObCharsetInfo *cs) { + uchar max_char; + uint i; + + if (!cs->sort_order) return; + + max_char = cs->sort_order[(uchar)cs->max_sort_char]; + for (i = 0; i < 256; i++) { + if ((uchar)cs->sort_order[i] > max_char) { + max_char = (uchar)cs->sort_order[i]; + cs->max_sort_char = i; + } + } +} + +bool ob_coll_init_simple(ObCharsetInfo *cs, ObCharsetLoader *loader) { + set_max_sort_char(cs); + return false; +} + +size_t ob_well_formed_len_ascii(const ObCharsetInfo *cs, + const char *start, const char *end, + size_t nchars, + int *error) +{ + const char *oldstart = start; + *error = 0; + while (start < end) { + if ((*start & 0x80) != 0) { + *error = 1; + break; + } + start++; + } + return start - oldstart; +} + +int ob_mb_wc_8bit(const ObCharsetInfo *cs, ob_wc_t *wc, const uchar *str, + const uchar *end) { + if (str >= end) return OB_CS_TOOSMALL; + + *wc = cs->tab_to_uni[*str]; + return (!wc[0] && str[0]) ? -1 : 1; +} + +int ob_wc_mb_8bit(const ObCharsetInfo *cs, ob_wc_t wc, uchar *str, uchar *end) { + const OB_UNI_IDX *idx; + + if (str >= end) return OB_CS_TOOSMALL; + + for (idx = cs->tab_from_uni; idx->tab; idx++) { + if (idx->from <= wc && idx->to >= wc) { + str[0] = idx->tab[wc - idx->from]; + return (!str[0] && wc) ? OB_CS_ILUNI : 1; + } + } + return OB_CS_ILUNI; +} + +unsigned int ob_ismbchar_8bit(const ObCharsetInfo *cs __attribute__((unused)), const char *p, const char *e) +{ + ob_charset_assert(e > p); + /*ob_ismbchar_8bit + *this is a mock interface, in mysql there is not is_mbchar for single byte charset + */ + if (0x00 <= (unsigned char)(*p) && (unsigned char)(*p) <= 0xFF) { + return 1; + } + return 0; +} + ObCollationHandler ob_collation_8bit_simple_ci_handler = { - NULL, /* init */ + ob_coll_init_simple, /* init */ NULL, ob_strnncoll_simple, ob_strnncollsp_simple, diff --git a/deps/oblib/src/lib/charset/ob_ctype_tis620.cc b/deps/oblib/src/lib/charset/ob_ctype_tis620.cc new file mode 100644 index 000000000..cd984c658 --- /dev/null +++ b/deps/oblib/src/lib/charset/ob_ctype_tis620.cc @@ -0,0 +1,336 @@ +/** + * Copyright (code) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "lib/charset/ob_mysql_global.h" +#include "lib/charset/ob_ctype.h" +#include "lib/utility/ob_macro_utils.h" +#include "lib/charset/ob_ctype_tis620_tab.h" +#include "lib/allocator/ob_malloc.h" + +/* + Convert thai string to "Standard C String Function" sortable string + + SYNOPSIS + thai2sortable() + tstr String to convert. Does not have to end with \0 + len Length of tstr +*/ + +static size_t thai2sortable(uchar *tstr, size_t len) { + uchar *p; + size_t tlen; + uchar l2bias; + + tlen = len; + l2bias = 256 - 8; + for (p = tstr; tlen > 0; p++, tlen--) { + uchar c = *p; + + if (isthai(c)) { + const int *t_ctype0 = t_ctype[c]; + + if (isconsnt(c)) l2bias -= 8; + if (isldvowel(c) && tlen != 1 && isconsnt(p[1])) { + /* simply swap between leading-vowel and consonant */ + *p = p[1]; + p[1] = c; + tlen--; + p++; + continue; + } + + /* if found level 2 char (L2_GARAN,L2_TONE*,L2_TYKHU) move to last */ + if (t_ctype0[1] >= L2_GARAN) { + /* + l2bias use to control position weight of l2char + example (*=l2char) XX*X must come before X*XX + */ + memmove((char *)p, (char *)(p + 1), tlen - 1); + tstr[len - 1] = l2bias + t_ctype0[1] - L2_GARAN + 1; + p--; + continue; + } + } else { + l2bias -= 8; + *p = to_lower_tis620[c]; + } + } + return len; +} + +/* + strncoll() replacement, compare 2 string, both are converted to sortable + string + + NOTE: + We can't cut strings at end \0 as this would break comparison with + LIKE characters, where the min range is stored as end \0 + + Arg: 2 Strings and it compare length + Ret: strcmp result +*/ + +extern "C" { +static int ob_strnncoll_tis620(const ObCharsetInfo *cs [[maybe_unused]], + const uchar *s1, size_t len1, const uchar *s2, + size_t len2, bool s2_is_prefix) { + uchar buf[80]; + uchar *tc1, *tc2; + int i; + + if (s2_is_prefix && len1 > len2) len1 = len2; + + tc1 = buf; + if ((len1 + len2 + 2) > (int)sizeof(buf)) + tc1 = static_cast(oceanbase::ob_malloc(len1 + len2 + 2, "CharsetInit")); + tc2 = tc1 + len1 + 1; + memcpy(tc1, s1, len1); + tc1[len1] = 0; /* if length(s1)> len1, need to put 'end of string' */ + memcpy(tc2, s2, len2); + tc2[len2] = 0; /* put end of string */ + thai2sortable(tc1, len1); + thai2sortable(tc2, len2); + i = strcmp((char *)tc1, (char *)tc2); + if (tc1 != buf) oceanbase::ob_free(tc1); + return i; +} + +static int ob_strnncollsp_tis620(const ObCharsetInfo *cs [[maybe_unused]], + const uchar *a0, size_t a_length, + const uchar *b0, size_t b_length, + bool diff_if_only_endspace_difference __attribute__((unused))) { + uchar buf[80], *end, *a, *b, *alloced = NULL; + size_t length; + int res = 0; + + a = buf; + if ((a_length + b_length + 2) > (int)sizeof(buf)) + alloced = a = (uchar *)oceanbase::ob_malloc(a_length + b_length + 2, "CharsetInit"); + + b = a + a_length + 1; + memcpy(a, a0, a_length); + a[a_length] = 0; /* if length(a0)> len1, need to put 'end of string' */ + memcpy(b, b0, b_length); + b[b_length] = 0; /* put end of string */ + a_length = thai2sortable(a, a_length); + b_length = thai2sortable(b, b_length); + + end = a + (length = std::min(a_length, b_length)); + while (a < end) { + if (*a++ != *b++) { + res = ((int)a[-1] - (int)b[-1]); + goto ret; + } + } + if (a_length != b_length) { + int swap = 1; + /* + Check the next not space character of the longer key. If it's < ' ', + then it's smaller than the other key. + */ + if (a_length < b_length) { + /* put shorter key in s */ + a_length = b_length; + a = b; + swap = -1; /* swap sign of result */ + res = -res; + } + for (end = a + a_length - length; a < end; a++) { + if (*a != ' ') { + res = (*a < ' ') ? -swap : swap; + goto ret; + } + } + } + +ret: + + if (alloced) oceanbase::ob_free(alloced); + return res; +} + +/* + strnxfrm replacement, convert Thai string to sortable string + + Arg: Destination buffer, source string, dest length and source length + Ret: Converted string size +*/ + +static size_t ob_strnxfrm_tis620(const ObCharsetInfo *cs, uchar *dst, + size_t dstlen, uint nweights, const uchar *src, + size_t srclen, uint flags,bool* is_valid_unicode) { + size_t dstlen0 = dstlen; + size_t min_len = std::min(dstlen, srclen); + size_t len = 0; + *is_valid_unicode = 1; + /* + We don't use strmake here, since it requires one more character for + the terminating '\0', while this function itself and the following calling + functions do not require it + */ + while (len < min_len) { + if (!(dst[len] = src[len])) break; + len++; + } + + len = thai2sortable(dst, len); + dstlen = std::min(dstlen, size_t(nweights)); + len = std::min(len, size_t(dstlen)); + len = ob_strxfrm_pad(cs, dst, dst + len, dst + dstlen, (uint)(dstlen - len), + flags); + if ((flags & OB_STRXFRM_PAD_TO_MAXLEN) && len < dstlen0) { + size_t fill_length = dstlen0 - len; + cs->cset->fill(cs, (char *)dst + len, fill_length, cs->pad_char); + len = dstlen0; + } + return len; +} +} // extern "C" + +extern "C" { +static int ob_mb_wc_tis620(const ObCharsetInfo *cs [[maybe_unused]], ob_wc_t *wc, + const uchar *str, const uchar *end) { + if (str >= end) return OB_CS_TOOSMALL; + + *wc = cs_to_uni_tis620[*str]; + return (!wc[0] && str[0]) ? -1 : 1; +} + +static int ob_wc_mb_tis620(const ObCharsetInfo *cs [[maybe_unused]], ob_wc_t wc, + uchar *str, uchar *end) { + unsigned char *pl; + + if (str >= end) return OB_CS_TOOSMALL; + + pl = uni_to_cs_tis620[(wc >> 8) & 0xFF]; + str[0] = pl ? pl[wc & 0xFF] : '\0'; + return (!str[0] && wc) ? OB_CS_ILUNI : 1; +} +} // extern "C" + +static ObCollationHandler ob_collation_tis620_handler = { + NULL, /* init */ + NULL, + ob_strnncoll_tis620, + ob_strnncollsp_tis620, + ob_strnxfrm_tis620, + ob_strnxfrmlen_simple, + NULL,//varlen + ob_like_range_simple, + ob_wildcmp_8bit, /* wildcmp */ + NULL,//ob_strcasecmp_8bit, + ob_instr_simple, /* QQ: To be fixed */ + ob_hash_sort_simple, + ob_propagate_simple}; + +static ObCharsetHandler ob_charset_tis620_handler = { + NULL, /* init */ + ob_ismbchar_8bit, /* ismbchar */ + ob_mbcharlen_8bit, /* mbcharlen */ + ob_numchars_8bit, + ob_charpos_8bit, + ob_max_bytes_charpos_8bit, + ob_well_formed_len_8bit, + ob_lengthsp_8bit, + //ob_numcells_8bit, + ob_mb_wc_tis620, /* mb_wc */ + ob_wc_mb_tis620, /* wc_mb */ + ob_mb_ctype_8bit, + //ob_caseup_str_8bit, + //ob_casedn_str_8bit, + ob_caseup_8bit, + ob_casedn_8bit, + //ob_snprintf_8bit, + //ob_long10_to_str_8bit, + //ob_longlong10_to_str_8bit, + ob_fill_8bit, + ob_strntol_8bit, + ob_strntoul_8bit, + ob_strntoll_8bit, + ob_strntoull_8bit, + ob_strntod_8bit, + //ob_strtoll10_8bit, + ob_strntoull10rnd_8bit, + ob_scan_8bit}; + + +ObCharsetInfo ob_charset_tis620_thai_ci = { + 18, + 0, + 0, /* number */ + OB_CS_COMPILED | OB_CS_PRIMARY | OB_CS_STRNXFRM, /* state */ + "tis620", /* cs name */ + "tis620_thai_ci", /* m_coll_name */ + "TIS620 Thai", /* comment */ + NULL, /* tailoring */ + NULL, /* coll_param */ + ctype_tis620, + to_lower_tis620, + to_upper_tis620, + sort_order_tis620, + NULL, /* uca */ + NULL, /* tab_to_uni */ + NULL, /* tab_from_uni */ + &ob_unicase_default, /* caseinfo */ + NULL, /* state_map */ + NULL, /* ident_map */ + 4, /* strxfrm_multiply */ + 1, /* caseup_multiply */ + 1, /* casedn_multiply */ + 1, /* mbminlen */ + 1, /* mbmaxlen */ + 1, + 0, /* min_sort_char */ + 255, /* max_sort_char */ + ' ', /* pad char */ + false, /* escape_with_backslash_is_dangerous */ + 1, /* levels_for_compare */ + 1, + &ob_charset_tis620_handler, + &ob_collation_tis620_handler, + PAD_SPACE}; + +ObCharsetInfo ob_charset_tis620_bin = { + 89, + 0, + 0, + OB_CS_COMPILED | OB_CS_BINSORT, + "tis620", + "tis620_bin", + "TIS620 Thai", + NULL, + NULL, + ctype_tis620, + to_lower_tis620, + to_upper_tis620, + NULL, + NULL, + NULL, + NULL, + &ob_unicase_default, + NULL, + NULL, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 255, + ' ', + false, + 1, + 1, + &ob_charset_tis620_handler, + &ob_collation_8bit_bin_handler, + PAD_SPACE}; \ No newline at end of file diff --git a/deps/oblib/src/lib/charset/ob_ctype_tis620_tab.h b/deps/oblib/src/lib/charset/ob_ctype_tis620_tab.h new file mode 100644 index 000000000..c7fae05db --- /dev/null +++ b/deps/oblib/src/lib/charset/ob_ctype_tis620_tab.h @@ -0,0 +1,830 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define TOT_LEVELS 5 +#define LAST_LEVEL 4 /* TOT_LEVELS - 1 */ + + +/* level 1 symbols & order */ +enum l1_symbols { + L1_08 = TOT_LEVELS, + L1_18, + L1_28, + L1_38, + L1_48, + L1_58, + L1_68, + L1_78, + L1_88, + L1_98, + L1_A8, + L1_B8, + L1_C8, + L1_D8, + L1_E8, + L1_F8, + L1_G8, + L1_H8, + L1_I8, + L1_J8, + L1_K8, + L1_L8, + L1_M8, + L1_N8, + L1_O8, + L1_P8, + L1_Q8, + L1_R8, + L1_S8, + L1_T8, + L1_U8, + L1_V8, + L1_W8, + L1_X8, + L1_Y8, + L1_Z8, + L1_KO_KAI, + L1_KHO_KHAI, + L1_KHO_KHUAT, + L1_KHO_KHWAI, + L1_KHO_KHON, + L1_KHO_RAKHANG, + L1_NGO_NGU, + L1_CHO_CHAN, + L1_CHO_CHING, + L1_CHO_CHANG, + L1_SO_SO, + L1_CHO_CHOE, + L1_YO_YING, + L1_DO_CHADA, + L1_TO_PATAK, + L1_THO_THAN, + L1_THO_NANGMONTHO, + L1_THO_PHUTHAO, + L1_NO_NEN, + L1_DO_DEK, + L1_TO_TAO, + L1_THO_THUNG, + L1_THO_THAHAN, + L1_THO_THONG, + L1_NO_NU, + L1_BO_BAIMAI, + L1_PO_PLA, + L1_PHO_PHUNG, + L1_FO_FA, + L1_PHO_PHAN, + L1_FO_FAN, + L1_PHO_SAMPHAO, + L1_MO_MA, + L1_YO_YAK, + L1_RO_RUA, + L1_RU, + L1_LO_LING, + L1_LU, + L1_WO_WAEN, + L1_SO_SALA, + L1_SO_RUSI, + L1_SO_SUA, + L1_HO_HIP, + L1_LO_CHULA, + L1_O_ANG, + L1_HO_NOKHUK, + L1_NKHIT, + L1_SARA_A, + L1_MAI_HAN_AKAT, + L1_SARA_AA, + L1_SARA_AM, + L1_SARA_I, + L1_SARA_II, + L1_SARA_UE, + L1_SARA_UEE, + L1_SARA_U, + L1_SARA_UU, + L1_SARA_E, + L1_SARA_AE, + L1_SARA_O, + L1_SARA_AI_MAIMUAN, + L1_SARA_AI_MAIMALAI +}; + +/* level 2 symbols & order */ +enum l2_symbols { + L2_BLANK = TOT_LEVELS, + L2_THAII, + L2_YAMAK, + L2_PINTHU, + L2_GARAN, + L2_TYKHU, + L2_TONE1, + L2_TONE2, + L2_TONE3, + L2_TONE4 +}; + +/* level 3 symbols & order */ +enum l3_symbols { + L3_BLANK = TOT_LEVELS, + L3_SPACE, + L3_NB_SACE, + L3_LOW_LINE, + L3_HYPHEN, + L3_COMMA, + L3_SEMICOLON, + L3_COLON, + L3_EXCLAMATION, + L3_QUESTION, + L3_SOLIDUS, + L3_FULL_STOP, + L3_PAIYAN_NOI, + L3_MAI_YAMOK, + L3_GRAVE, + L3_CIRCUMFLEX, + L3_TILDE, + L3_APOSTROPHE, + L3_QUOTATION, + L3_L_PARANTHESIS, + L3_L_BRACKET, + L3_L_BRACE, + L3_R_BRACE, + L3_R_BRACKET, + L3_R_PARENTHESIS, + L3_AT, + L3_BAHT, + L3_DOLLAR, + L3_FONGMAN, + L3_ANGKHANKHU, + L3_KHOMUT, + L3_ASTERISK, + L3_BK_SOLIDUS, + L3_AMPERSAND, + L3_NUMBER, + L3_PERCENT, + L3_PLUS, + L3_LESS_THAN, + L3_EQUAL, + L3_GREATER_THAN, + L3_V_LINE +}; + +/* level 4 symbols & order */ +enum l4_symbols { L4_BLANK = TOT_LEVELS, L4_MIN, L4_CAP, L4_EXT }; + +enum level_symbols { L_UPRUPR = TOT_LEVELS, L_UPPER, L_MIDDLE, L_LOWER }; + +#define _is(c) (t_ctype[(c)][LAST_LEVEL]) +#define _level 8 +#define _consnt 16 +#define _ldvowel 32 +#define _fllwvowel 64 +#define _uprvowel 128 +#define _lwrvowel 256 +#define _tone 512 +#define _diacrt1 1024 +#define _diacrt2 2048 +#define _combine 4096 +#define _stone 8192 +#define _tdig 16384 +#define _rearvowel (_fllwvowel | _uprvowel | _lwrvowel) +#define _diacrt (_diacrt1 | _diacrt2) +#define levelof(c) (_is(c) & _level) +#define isthai(c) ((c) >= 128) +#define istalpha(c) \ + (_is(c) & (_consnt | _ldvowel | _rearvowel | _tone | _diacrt1 | _diacrt2)) +#define isconsnt(c) (_is(c) & _consnt) +#define isldvowel(c) (_is(c) & _ldvowel) +#define isfllwvowel(c) (_is(c) & _fllwvowel) +#define ismidvowel(c) (_is(c) & (_ldvowel | _fllwvowel)) +#define isuprvowel(c) (_is(c) & _uprvowel) +#define islwrvowel(c) (_is(c) & _lwrvowel) +#define isuprlwrvowel(c) (_is(c) & (_lwrvowel | _uprvowel)) +#define isrearvowel(c) (_is(c) & _rearvowel) +#define isvowel(c) (_is(c) & (_ldvowel | _rearvowel)) +#define istone(c) (_is(c) & _tone) +#define isunldable(c) (_is(c) & (_rearvowel | _tone | _diacrt1 | _diacrt2)) +#define iscombinable(c) (_is(c) & _combine) +#define istdigit(c) (_is(c) & _tdig) +#define isstone(c) (_is(c) & _stone) +#define isdiacrt1(c) (_is(c) & _diacrt1) +#define isdiacrt2(c) (_is(c) & _diacrt2) +#define isdiacrt(c) (_is(c) & _diacrt) + + +#define M L_MIDDLE +//#define U L_UPPER +#define L L_LOWER +#define UU L_UPRUPR +#define X L_MIDDLE + +static const int t_ctype[][TOT_LEVELS] = { + /*0x00*/ {0, 0, 0, 0, X}, + /*0x01*/ {0, 0, 0, 0, X}, + /*0x02*/ {0, 0, 0, 0, X}, + /*0x03*/ {0, 0, 0, 0, X}, + /*0x04*/ {0, 0, 0, 0, X}, + /*0x05*/ {0, 0, 0, 0, X}, + /*0x06*/ {0, 0, 0, 0, X}, + /*0x07*/ {0, 0, 0, 0, X}, + /*0x08*/ {0, 0, 0, 0, X}, + /*0x09*/ {0, 0, 0, 0, X}, + /*0x0A*/ {0, 0, 0, 0, X}, + /*0x0B*/ {0, 0, 0, 0, X}, + /*0x0C*/ {0, 0, 0, 0, X}, + /*0x0D*/ {0, 0, 0, 0, X}, + /*0x0E*/ {0, 0, 0, 0, X}, + /*0x0F*/ {0, 0, 0, 0, X}, + /*0x10*/ {0, 0, 0, 0, X}, + /*0x11*/ {0, 0, 0, 0, X}, + /*0x12*/ {0, 0, 0, 0, X}, + /*0x13*/ {0, 0, 0, 0, X}, + /*0x14*/ {0, 0, 0, 0, X}, + /*0x15*/ {0, 0, 0, 0, X}, + /*0x16*/ {0, 0, 0, 0, X}, + /*0x17*/ {0, 0, 0, 0, X}, + /*0x18*/ {0, 0, 0, 0, X}, + /*0x19*/ {0, 0, 0, 0, X}, + /*0x1A*/ {0, 0, 0, 0, X}, + /*0x1B*/ {0, 0, 0, 0, X}, + /*0x1C*/ {0, 0, 0, 0, X}, + /*0x1D*/ {0, 0, 0, 0, X}, + /*0x1E*/ {0, 0, 0, 0, X}, + /*0x1F*/ {0, 0, 0, 0, X}, + /*0x20*/ {0, 0, L3_SPACE, 0, M}, + /*0x21*/ {0, 0, L3_EXCLAMATION, 0, M}, + /*0x22*/ {0, 0, L3_QUOTATION, 0, M}, + /*0x23*/ {0, 0, L3_NUMBER, 0, M}, + /*0x24*/ {0, 0, L3_DOLLAR, 0, M}, + /*0x25*/ {0, 0, L3_PERCENT, 0, M}, + /*0x26*/ {0, 0, L3_AMPERSAND, 0, M}, + /*0x27*/ {0, 0, L3_APOSTROPHE, 0, M}, + /*0x28*/ {0, 0, L3_L_PARANTHESIS, 0, M}, + /*0x29*/ {0, 0, L3_R_PARENTHESIS, 0, M}, + /*0x2A*/ {0, 0, L3_ASTERISK, 0, M}, + /*0x2B*/ {0, 0, L3_PLUS, 0, M}, + /*0x2C*/ {0, 0, L3_COMMA, 0, M}, + /*0x2D*/ {0, 0, L3_HYPHEN, 0, M}, + /*0x2E*/ {0, 0, L3_FULL_STOP, 0, M}, + /*0x2F*/ {0, 0, L3_SOLIDUS, 0, M}, + /*0x30*/ {L1_08, L2_BLANK, L3_BLANK, L4_BLANK, M}, + /*0x31*/ {L1_18, L2_BLANK, L3_BLANK, L4_BLANK, M}, + /*0x32*/ {L1_28, L2_BLANK, L3_BLANK, L4_BLANK, M}, + /*0x33*/ {L1_38, L2_BLANK, L3_BLANK, L4_BLANK, M}, + /*0x34*/ {L1_48, L2_BLANK, L3_BLANK, L4_BLANK, M}, + /*0x35*/ {L1_58, L2_BLANK, L3_BLANK, L4_BLANK, M}, + /*0x36*/ {L1_68, L2_BLANK, L3_BLANK, L4_BLANK, M}, + /*0x37*/ {L1_78, L2_BLANK, L3_BLANK, L4_BLANK, M}, + /*0x38*/ {L1_88, L2_BLANK, L3_BLANK, L4_BLANK, M}, + /*0x39*/ {L1_98, L2_BLANK, L3_BLANK, L4_BLANK, M}, + /*0x3A*/ {0, 0, L3_COLON, 0, M}, + /*0x3B*/ {0, 0, L3_SEMICOLON, 0, M}, + /*0x3C*/ {0, 0, L3_LESS_THAN, 0, M}, + /*0x3D*/ {0, 0, L3_EQUAL, 0, M}, + /*0x3E*/ {0, 0, L3_GREATER_THAN, 0, M}, + /*0x3F*/ {0, 0, L3_QUESTION, 0, M}, + /*0x40*/ {0, 0, L3_AT, 0, M}, + /*0x41*/ {L1_A8, L2_BLANK, L3_BLANK, L4_CAP, M}, + /*0x42*/ {L1_B8, L2_BLANK, L3_BLANK, L4_CAP, M}, + /*0x43*/ {L1_C8, L2_BLANK, L3_BLANK, L4_CAP, M}, + /*0x44*/ {L1_D8, L2_BLANK, L3_BLANK, L4_CAP, M}, + /*0x45*/ {L1_E8, L2_BLANK, L3_BLANK, L4_CAP, M}, + /*0x46*/ {L1_F8, L2_BLANK, L3_BLANK, L4_CAP, M}, + /*0x47*/ {L1_G8, L2_BLANK, L3_BLANK, L4_CAP, M}, + /*0x48*/ {L1_H8, L2_BLANK, L3_BLANK, L4_CAP, M}, + /*0x49*/ {L1_I8, L2_BLANK, L3_BLANK, L4_CAP, M}, + /*0x4A*/ {L1_J8, L2_BLANK, L3_BLANK, L4_CAP, M}, + /*0x4B*/ {L1_K8, L2_BLANK, L3_BLANK, L4_CAP, M}, + /*0x4C*/ {L1_L8, L2_BLANK, L3_BLANK, L4_CAP, M}, + /*0x4D*/ {L1_M8, L2_BLANK, L3_BLANK, L4_CAP, M}, + /*0x4E*/ {L1_N8, L2_BLANK, L3_BLANK, L4_CAP, M}, + /*0x4F*/ {L1_O8, L2_BLANK, L3_BLANK, L4_CAP, M}, + /*0x50*/ {L1_P8, L2_BLANK, L3_BLANK, L4_CAP, M}, + /*0x51*/ {L1_Q8, L2_BLANK, L3_BLANK, L4_CAP, M}, + /*0x52*/ {L1_R8, L2_BLANK, L3_BLANK, L4_CAP, M}, + /*0x53*/ {L1_S8, L2_BLANK, L3_BLANK, L4_CAP, M}, + /*0x54*/ {L1_T8, L2_BLANK, L3_BLANK, L4_CAP, M}, + /*0x55*/ {L1_U8, L2_BLANK, L3_BLANK, L4_CAP, M}, + /*0x56*/ {L1_V8, L2_BLANK, L3_BLANK, L4_CAP, M}, + /*0x57*/ {L1_W8, L2_BLANK, L3_BLANK, L4_CAP, M}, + /*0x58*/ {L1_X8, L2_BLANK, L3_BLANK, L4_CAP, M}, + /*0x59*/ {L1_Y8, L2_BLANK, L3_BLANK, L4_CAP, M}, + /*0x5A*/ {L1_Z8, L2_BLANK, L3_BLANK, L4_CAP, M}, + /*0x5B*/ {0, 0, L3_L_BRACKET, 0, M}, + /*0x5C*/ {0, 0, L3_BK_SOLIDUS, 0, M}, + /*0x5D*/ {0, 0, L3_R_BRACKET, 0, M}, + /*0x5E*/ {0, 0, L3_CIRCUMFLEX, 0, M}, + /*0x5F*/ {0, 0, L3_LOW_LINE, 0, M}, + /*0x60*/ {0, 0, L3_GRAVE, 0, M}, + /*0x61*/ {L1_A8, L2_BLANK, L3_BLANK, L4_MIN, M}, + /*0x62*/ {L1_B8, L2_BLANK, L3_BLANK, L4_MIN, M}, + /*0x63*/ {L1_C8, L2_BLANK, L3_BLANK, L4_MIN, M}, + /*0x64*/ {L1_D8, L2_BLANK, L3_BLANK, L4_MIN, M}, + /*0x65*/ {L1_E8, L2_BLANK, L3_BLANK, L4_MIN, M}, + /*0x66*/ {L1_F8, L2_BLANK, L3_BLANK, L4_MIN, M}, + /*0x67*/ {L1_G8, L2_BLANK, L3_BLANK, L4_MIN, M}, + /*0x68*/ {L1_H8, L2_BLANK, L3_BLANK, L4_MIN, M}, + /*0x69*/ {L1_I8, L2_BLANK, L3_BLANK, L4_MIN, M}, + /*0x6A*/ {L1_J8, L2_BLANK, L3_BLANK, L4_MIN, M}, + /*0x6B*/ {L1_K8, L2_BLANK, L3_BLANK, L4_MIN, M}, + /*0x6C*/ {L1_L8, L2_BLANK, L3_BLANK, L4_MIN, M}, + /*0x6D*/ {L1_M8, L2_BLANK, L3_BLANK, L4_MIN, M}, + /*0x6E*/ {L1_N8, L2_BLANK, L3_BLANK, L4_MIN, M}, + /*0x6F*/ {L1_O8, L2_BLANK, L3_BLANK, L4_MIN, M}, + /*0x70*/ {L1_P8, L2_BLANK, L3_BLANK, L4_MIN, M}, + /*0x71*/ {L1_Q8, L2_BLANK, L3_BLANK, L4_MIN, M}, + /*0x72*/ {L1_R8, L2_BLANK, L3_BLANK, L4_MIN, M}, + /*0x73*/ {L1_S8, L2_BLANK, L3_BLANK, L4_MIN, M}, + /*0x74*/ {L1_T8, L2_BLANK, L3_BLANK, L4_MIN, M}, + /*0x75*/ {L1_U8, L2_BLANK, L3_BLANK, L4_MIN, M}, + /*0x76*/ {L1_V8, L2_BLANK, L3_BLANK, L4_MIN, M}, + /*0x77*/ {L1_W8, L2_BLANK, L3_BLANK, L4_MIN, M}, + /*0x78*/ {L1_X8, L2_BLANK, L3_BLANK, L4_MIN, M}, + /*0x79*/ {L1_Y8, L2_BLANK, L3_BLANK, L4_MIN, M}, + /*0x7A*/ {L1_Z8, L2_BLANK, L3_BLANK, L4_MIN, M}, + /*0x7B*/ {0, 0, L3_L_BRACE, 0, M}, + /*0x7C*/ {0, 0, L3_V_LINE, 0, M}, + /*0x7D*/ {0, 0, L3_R_BRACE, 0, M}, + /*0x7E*/ {0, 0, L3_TILDE, 0, M}, + /*0x7F*/ {0, 0, 0, 0, X}, + /*0x80*/ {0, 0, 0, 0, X}, + /*0x81*/ {0, 0, 0, 0, X}, + /*0x82*/ {0, 0, 0, 0, X}, + /*0x83*/ {0, 0, 0, 0, X}, + /*0x84*/ {0, 0, 0, 0, X}, + /*0x85*/ {0, 0, 0, 0, X}, + /*0x86*/ {0, 0, 0, 0, X}, + /*0x87*/ {0, 0, 0, 0, X}, + /*0x88*/ {0, 0, 0, 0, X}, + /*0x89*/ {0, 0, 0, 0, X}, + /*0x8A*/ {0, 0, 0, 0, X}, + /*0x8B*/ {0, 0, 0, 0, X}, + /*0x8C*/ {0, 0, 0, 0, X}, + /*0x8D*/ {0, 0, 0, 0, X}, + /*0x8E*/ {0, 0, 0, 0, X}, + /*0x8F*/ {0, 0, 0, 0, X}, + /*0x90*/ {0, 0, 0, 0, X}, + /*0x91*/ {0, 0, 0, 0, X}, + /*0x92*/ {0, 0, 0, 0, X}, + /*0x93*/ {0, 0, 0, 0, X}, + /*0x94*/ {0, 0, 0, 0, X}, + /*0x95*/ {0, 0, 0, 0, X}, + /*0x96*/ {0, 0, 0, 0, X}, + /*0x97*/ {0, 0, 0, 0, X}, + /*0x98*/ {0, 0, 0, 0, X}, + /*0x99*/ {0, 0, 0, 0, X}, + /*0x9A*/ {0, 0, 0, 0, X}, + /*0x9B*/ {0, 0, 0, 0, X}, + /*0x9C*/ {0, 0, 0, 0, X}, + /*0x9D*/ {0, 0, 0, 0, X}, + /*0x9E*/ {0, 0, 0, 0, X}, + /*0x9F*/ {0, 0, 0, 0, X}, + /*0xA0*/ {0, 0, L3_NB_SACE, 0, X}, + /*0xA1*/ {L1_KO_KAI, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xA2*/ {L1_KHO_KHAI, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xA3*/ {L1_KHO_KHUAT, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xA4*/ {L1_KHO_KHWAI, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xA5*/ {L1_KHO_KHON, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xA6*/ {L1_KHO_RAKHANG, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xA7*/ {L1_NGO_NGU, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xA8*/ {L1_CHO_CHAN, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xA9*/ {L1_CHO_CHING, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xAA*/ {L1_CHO_CHANG, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xAB*/ {L1_SO_SO, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xAC*/ {L1_CHO_CHOE, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xAD*/ {L1_YO_YING, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xAE*/ {L1_DO_CHADA, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xAF*/ {L1_TO_PATAK, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xB0*/ {L1_THO_THAN, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xB1*/ {L1_THO_NANGMONTHO, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xB2*/ {L1_THO_PHUTHAO, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xB3*/ {L1_NO_NEN, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xB4*/ {L1_DO_DEK, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xB5*/ {L1_TO_TAO, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xB6*/ {L1_THO_THUNG, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xB7*/ {L1_THO_THAHAN, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xB8*/ {L1_THO_THONG, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xB9*/ {L1_NO_NU, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xBA*/ {L1_BO_BAIMAI, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xBB*/ {L1_PO_PLA, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xBC*/ {L1_PHO_PHUNG, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xBD*/ {L1_FO_FA, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xBE*/ {L1_PHO_PHAN, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xBF*/ {L1_FO_FAN, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xC0*/ {L1_PHO_SAMPHAO, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xC1*/ {L1_MO_MA, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xC2*/ {L1_YO_YAK, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xC3*/ {L1_RO_RUA, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xC4*/ {L1_RU, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xC5*/ {L1_LO_LING, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xC6*/ {L1_LU, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xC7*/ {L1_WO_WAEN, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xC8*/ {L1_SO_SALA, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xC9*/ {L1_SO_RUSI, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xCA*/ {L1_SO_SUA, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xCB*/ {L1_HO_HIP, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xCC*/ {L1_LO_CHULA, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xCD*/ {L1_O_ANG, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xCE*/ {L1_HO_NOKHUK, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt}, + /*0xCF*/ {0, 0, L3_PAIYAN_NOI, 0, M}, + /*0xD0*/ {L1_SARA_A, L2_BLANK, L3_BLANK, L4_BLANK, M | _fllwvowel}, + /*0xD1*/ {L1_MAI_HAN_AKAT, L2_BLANK, L3_BLANK, L4_BLANK, L_UPPER | _uprvowel}, + /*0xD2*/ {L1_SARA_AA, L2_BLANK, L3_BLANK, L4_BLANK, M | _fllwvowel}, + /*0xD3*/ {L1_SARA_AM, L2_BLANK, L3_BLANK, L4_BLANK, M | _fllwvowel}, + /*0xD4*/ {L1_SARA_I, L2_BLANK, L3_BLANK, L4_BLANK, L_UPPER | _uprvowel}, + /*0xD5*/ {L1_SARA_II, L2_BLANK, L3_BLANK, L4_BLANK, L_UPPER | _uprvowel}, + /*0xD6*/ {L1_SARA_UE, L2_BLANK, L3_BLANK, L4_BLANK, L_UPPER | _uprvowel}, + /*0xD7*/ {L1_SARA_UEE, L2_BLANK, L3_BLANK, L4_BLANK, L_UPPER | _uprvowel}, + /*0xD8*/ {L1_SARA_U, L2_BLANK, L3_BLANK, L4_BLANK, L | _lwrvowel}, + /*0xD9*/ {L1_SARA_UU, L2_BLANK, L3_BLANK, L4_BLANK, L | _lwrvowel}, + /*0xDA*/ {0, L2_PINTHU, L3_BLANK, L4_BLANK, L}, + /*0xDB*/ {0, 0, 0, 0, X}, + /*0xDC*/ {0, 0, 0, 0, X}, + /*0xDD*/ {0, 0, 0, 0, X}, + /*0xDE*/ {0, 0, 0, 0, X}, + /*0xDF*/ {0, 0, L3_BAHT, 0, M}, + /*0xE0*/ {L1_SARA_E, L2_BLANK, L3_BLANK, L4_BLANK, M | _ldvowel}, + /*0xE1*/ {L1_SARA_AE, L2_BLANK, L3_BLANK, L4_BLANK, M | _ldvowel}, + /*0xE2*/ {L1_SARA_O, L2_BLANK, L3_BLANK, L4_BLANK, M | _ldvowel}, + /*0xE3*/ {L1_SARA_AI_MAIMUAN, L2_BLANK, L3_BLANK, L4_BLANK, M | _ldvowel}, + /*0xE4*/ {L1_SARA_AI_MAIMALAI, L2_BLANK, L3_BLANK, L4_BLANK, M | _ldvowel}, + /*0xE5*/ {L1_SARA_AA, L2_BLANK, L3_BLANK, L4_EXT, M | _fllwvowel}, + /*0xE6*/ {0, 0, L3_MAI_YAMOK, 0, M | _stone}, + /*0xE7*/ {0, L2_TYKHU, L3_BLANK, L4_BLANK, L_UPPER | _diacrt1 | _stone}, + /*0xE8*/ + {0, L2_TONE1, L3_BLANK, L4_BLANK, UU | _tone | _combine | _stone}, + /*0xE9*/ + {0, L2_TONE2, L3_BLANK, L4_BLANK, UU | _tone | _combine | _stone}, + /*0xEA*/ + {0, L2_TONE3, L3_BLANK, L4_BLANK, UU | _tone | _combine | _stone}, + /*0xEB*/ + {0, L2_TONE4, L3_BLANK, L4_BLANK, UU | _tone | _combine | _stone}, + /*0xEC*/ + {0, L2_GARAN, L3_BLANK, L4_BLANK, UU | _diacrt2 | _combine | _stone}, + /*0xED*/ {L1_NKHIT, L2_BLANK, L3_BLANK, L4_BLANK, L_UPPER | _diacrt1}, + /*0xEE*/ {0, L2_YAMAK, L3_BLANK, L4_BLANK, L_UPPER | _diacrt1}, + /*0xEF*/ {0, 0, L3_FONGMAN, 0, M}, + /*0xF0*/ {L1_08, L2_THAII, L3_BLANK, L4_BLANK, M | _tdig}, + /*0xF1*/ {L1_18, L2_THAII, L3_BLANK, L4_BLANK, M | _tdig}, + /*0xF2*/ {L1_28, L2_THAII, L3_BLANK, L4_BLANK, M | _tdig}, + /*0xF3*/ {L1_38, L2_THAII, L3_BLANK, L4_BLANK, M | _tdig}, + /*0xF4*/ {L1_48, L2_THAII, L3_BLANK, L4_BLANK, M | _tdig}, + /*0xF5*/ {L1_58, L2_THAII, L3_BLANK, L4_BLANK, M | _tdig}, + /*0xF6*/ {L1_68, L2_THAII, L3_BLANK, L4_BLANK, M | _tdig}, + /*0xF7*/ {L1_78, L2_THAII, L3_BLANK, L4_BLANK, M | _tdig}, + /*0xF8*/ {L1_88, L2_THAII, L3_BLANK, L4_BLANK, M | _tdig}, + /*0xF9*/ {L1_98, L2_THAII, L3_BLANK, L4_BLANK, M | _tdig}, + /*0xFA*/ {0, 0, L3_ANGKHANKHU, 0, X}, + /*0xFB*/ {0, 0, L3_KHOMUT, 0, X}, + /*0xFC*/ {0, 0, 0, 0, X}, + /*0xFD*/ {0, 0, 0, 0, X}, + /*0xFE*/ {0, 0, 0, 0, X}, + /* Utilize 0xFF for max_sort_chr */ + /*0xFF*/ {255 /*0*/, 0, 0, 0, X}, +}; + +static unsigned char ctype_tis620[257] = { + 0, /* For standard library */ + 32, 32, 32, 32, 32, 32, 32, 32, 32, 40, 40, 40, 40, 40, 32, 32, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 72, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 16, 16, 16, 16, 16, 16, + 16, 129, 129, 129, 129, 129, 129, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 16, 16, 16, 16, 16, + 16, 130, 130, 130, 130, 130, 130, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 16, 16, 16, 16, 32, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; + +static unsigned char to_lower_tis620[] = { + '\000', '\001', '\002', '\003', '\004', + '\005', '\006', '\007', '\010', '\011', + '\012', '\013', '\014', '\015', '\016', + '\017', '\020', '\021', '\022', '\023', + '\024', '\025', '\026', '\027', '\030', + '\031', '\032', '\033', '\034', '\035', + '\036', '\037', ' ', '!', '"', + '#', '$', '%', '&', '\'', + '(', ')', '*', '+', ',', + '-', '.', '/', '0', '1', + '2', '3', '4', '5', '6', + '7', '8', '9', ':', ';', + '<', '=', '>', '?', '@', + 'a', 'b', 'c', 'd', 'e', + 'f', 'g', 'h', 'i', 'j', + 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', + 'u', 'v', 'w', 'x', 'y', + 'z', '[', '\\', ']', '^', + '_', '`', 'a', 'b', 'c', + 'd', 'e', 'f', 'g', 'h', + 'i', 'j', 'k', 'l', 'm', + 'n', 'o', 'p', 'q', 'r', + 's', 't', 'u', 'v', 'w', + 'x', 'y', 'z', '{', '|', + '}', '~', '\177', (uchar)'\200', (uchar)'\201', + (uchar)'\202', (uchar)'\203', (uchar)'\204', (uchar)'\205', (uchar)'\206', + (uchar)'\207', (uchar)'\210', (uchar)'\211', (uchar)'\212', (uchar)'\213', + (uchar)'\214', (uchar)'\215', (uchar)'\216', (uchar)'\217', (uchar)'\220', + (uchar)'\221', (uchar)'\222', (uchar)'\223', (uchar)'\224', (uchar)'\225', + (uchar)'\226', (uchar)'\227', (uchar)'\230', (uchar)'\231', (uchar)'\232', + (uchar)'\233', (uchar)'\234', (uchar)'\235', (uchar)'\236', (uchar)'\237', + (uchar)'\240', (uchar)'\241', (uchar)'\242', (uchar)'\243', (uchar)'\244', + (uchar)'\245', (uchar)'\246', (uchar)'\247', (uchar)'\250', (uchar)'\251', + (uchar)'\252', (uchar)'\253', (uchar)'\254', (uchar)'\255', (uchar)'\256', + (uchar)'\257', (uchar)'\260', (uchar)'\261', (uchar)'\262', (uchar)'\263', + (uchar)'\264', (uchar)'\265', (uchar)'\266', (uchar)'\267', (uchar)'\270', + (uchar)'\271', (uchar)'\272', (uchar)'\273', (uchar)'\274', (uchar)'\275', + (uchar)'\276', (uchar)'\277', (uchar)'\300', (uchar)'\301', (uchar)'\302', + (uchar)'\303', (uchar)'\304', (uchar)'\305', (uchar)'\306', (uchar)'\307', + (uchar)'\310', (uchar)'\311', (uchar)'\312', (uchar)'\313', (uchar)'\314', + (uchar)'\315', (uchar)'\316', (uchar)'\317', (uchar)'\320', (uchar)'\321', + (uchar)'\322', (uchar)'\323', (uchar)'\324', (uchar)'\325', (uchar)'\326', + (uchar)'\327', (uchar)'\330', (uchar)'\331', (uchar)'\332', (uchar)'\333', + (uchar)'\334', (uchar)'\335', (uchar)'\336', (uchar)'\337', (uchar)'\340', + (uchar)'\341', (uchar)'\342', (uchar)'\343', (uchar)'\344', (uchar)'\345', + (uchar)'\346', (uchar)'\347', (uchar)'\350', (uchar)'\351', (uchar)'\352', + (uchar)'\353', (uchar)'\354', (uchar)'\355', (uchar)'\356', (uchar)'\357', + (uchar)'\360', (uchar)'\361', (uchar)'\362', (uchar)'\363', (uchar)'\364', + (uchar)'\365', (uchar)'\366', (uchar)'\367', (uchar)'\370', (uchar)'\371', + (uchar)'\372', (uchar)'\373', (uchar)'\374', (uchar)'\375', (uchar)'\376', + (uchar)'\377', +}; + +static unsigned char to_upper_tis620[] = { + '\000', '\001', '\002', '\003', '\004', + '\005', '\006', '\007', '\010', '\011', + '\012', '\013', '\014', '\015', '\016', + '\017', '\020', '\021', '\022', '\023', + '\024', '\025', '\026', '\027', '\030', + '\031', '\032', '\033', '\034', '\035', + '\036', '\037', ' ', '!', '"', + '#', '$', '%', '&', '\'', + '(', ')', '*', '+', ',', + '-', '.', '/', '0', '1', + '2', '3', '4', '5', '6', + '7', '8', '9', ':', ';', + '<', '=', '>', '?', '@', + 'A', 'B', 'C', 'D', 'E', + 'F', 'G', 'H', 'I', 'J', + 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', + 'U', 'V', 'W', 'X', 'Y', + 'Z', '[', '\\', ']', '^', + '_', '`', 'A', 'B', 'C', + 'D', 'E', 'F', 'G', 'H', + 'I', 'J', 'K', 'L', 'M', + 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', + 'X', 'Y', 'Z', '{', '|', + '}', '~', '\177', (uchar)'\200', (uchar)'\201', + (uchar)'\202', (uchar)'\203', (uchar)'\204', (uchar)'\205', (uchar)'\206', + (uchar)'\207', (uchar)'\210', (uchar)'\211', (uchar)'\212', (uchar)'\213', + (uchar)'\214', (uchar)'\215', (uchar)'\216', (uchar)'\217', (uchar)'\220', + (uchar)'\221', (uchar)'\222', (uchar)'\223', (uchar)'\224', (uchar)'\225', + (uchar)'\226', (uchar)'\227', (uchar)'\230', (uchar)'\231', (uchar)'\232', + (uchar)'\233', (uchar)'\234', (uchar)'\235', (uchar)'\236', (uchar)'\237', + (uchar)'\240', (uchar)'\241', (uchar)'\242', (uchar)'\243', (uchar)'\244', + (uchar)'\245', (uchar)'\246', (uchar)'\247', (uchar)'\250', (uchar)'\251', + (uchar)'\252', (uchar)'\253', (uchar)'\254', (uchar)'\255', (uchar)'\256', + (uchar)'\257', (uchar)'\260', (uchar)'\261', (uchar)'\262', (uchar)'\263', + (uchar)'\264', (uchar)'\265', (uchar)'\266', (uchar)'\267', (uchar)'\270', + (uchar)'\271', (uchar)'\272', (uchar)'\273', (uchar)'\274', (uchar)'\275', + (uchar)'\276', (uchar)'\277', (uchar)'\300', (uchar)'\301', (uchar)'\302', + (uchar)'\303', (uchar)'\304', (uchar)'\305', (uchar)'\306', (uchar)'\307', + (uchar)'\310', (uchar)'\311', (uchar)'\312', (uchar)'\313', (uchar)'\314', + (uchar)'\315', (uchar)'\316', (uchar)'\317', (uchar)'\320', (uchar)'\321', + (uchar)'\322', (uchar)'\323', (uchar)'\324', (uchar)'\325', (uchar)'\326', + (uchar)'\327', (uchar)'\330', (uchar)'\331', (uchar)'\332', (uchar)'\333', + (uchar)'\334', (uchar)'\335', (uchar)'\336', (uchar)'\337', (uchar)'\340', + (uchar)'\341', (uchar)'\342', (uchar)'\343', (uchar)'\344', (uchar)'\345', + (uchar)'\346', (uchar)'\347', (uchar)'\350', (uchar)'\351', (uchar)'\352', + (uchar)'\353', (uchar)'\354', (uchar)'\355', (uchar)'\356', (uchar)'\357', + (uchar)'\360', (uchar)'\361', (uchar)'\362', (uchar)'\363', (uchar)'\364', + (uchar)'\365', (uchar)'\366', (uchar)'\367', (uchar)'\370', (uchar)'\371', + (uchar)'\372', (uchar)'\373', (uchar)'\374', (uchar)'\375', (uchar)'\376', + (uchar)'\377', +}; + +static unsigned char sort_order_tis620[] = { + '\000', '\001', '\002', '\003', '\004', + '\005', '\006', '\007', '\010', '\011', + '\012', '\013', '\014', '\015', '\016', + '\017', '\020', '\021', '\022', '\023', + '\024', '\025', '\026', '\027', '\030', + '\031', '\032', '\033', '\034', '\035', + '\036', '\037', ' ', '!', '"', + '#', '$', '%', '&', '\'', + '(', ')', '*', '+', ',', + '-', '.', '/', '0', '1', + '2', '3', '4', '5', '6', + '7', '8', '9', ':', ';', + '<', '=', '>', '?', '@', + 'A', 'B', 'C', 'D', 'E', + 'F', 'G', 'H', 'I', 'J', + 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', + 'U', 'V', 'W', 'X', 'Y', + 'Z', '\\', ']', '[', '^', + '_', 'E', 'A', 'B', 'C', + 'D', 'E', 'F', 'G', 'H', + 'I', 'J', 'K', 'L', 'M', + 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', + 'X', 'Y', 'Z', '{', '|', + '}', 'Y', '\177', (uchar)'\200', (uchar)'\201', + (uchar)'\202', (uchar)'\203', (uchar)'\204', (uchar)'\205', (uchar)'\206', + (uchar)'\207', (uchar)'\210', (uchar)'\211', (uchar)'\212', (uchar)'\213', + (uchar)'\214', (uchar)'\215', (uchar)'\216', (uchar)'\217', (uchar)'\220', + (uchar)'\221', (uchar)'\222', (uchar)'\223', (uchar)'\224', (uchar)'\225', + (uchar)'\226', (uchar)'\227', (uchar)'\230', (uchar)'\231', (uchar)'\232', + (uchar)'\233', (uchar)'\234', (uchar)'\235', (uchar)'\236', (uchar)'\237', + (uchar)'\240', (uchar)'\241', (uchar)'\242', (uchar)'\243', (uchar)'\244', + (uchar)'\245', (uchar)'\246', (uchar)'\247', (uchar)'\250', (uchar)'\251', + (uchar)'\252', (uchar)'\253', (uchar)'\254', (uchar)'\255', (uchar)'\256', + (uchar)'\257', (uchar)'\260', (uchar)'\261', (uchar)'\262', (uchar)'\263', + (uchar)'\264', (uchar)'\265', (uchar)'\266', (uchar)'\267', (uchar)'\270', + (uchar)'\271', (uchar)'\272', (uchar)'\273', (uchar)'\274', (uchar)'\275', + (uchar)'\276', (uchar)'\277', (uchar)'\300', (uchar)'\301', (uchar)'\302', + (uchar)'\303', (uchar)'\304', (uchar)'\305', (uchar)'\306', (uchar)'\307', + (uchar)'\310', (uchar)'\311', (uchar)'\312', (uchar)'\313', (uchar)'\314', + (uchar)'\315', (uchar)'\316', (uchar)'\317', (uchar)'\320', (uchar)'\321', + (uchar)'\322', (uchar)'\323', (uchar)'\324', (uchar)'\325', (uchar)'\326', + (uchar)'\327', (uchar)'\330', (uchar)'\331', (uchar)'\332', (uchar)'\333', + (uchar)'\334', (uchar)'\335', (uchar)'\336', (uchar)'\337', (uchar)'\340', + (uchar)'\341', (uchar)'\342', (uchar)'\343', (uchar)'\344', (uchar)'\345', + (uchar)'\346', (uchar)'\347', (uchar)'\350', (uchar)'\351', (uchar)'\352', + (uchar)'\353', (uchar)'\354', (uchar)'\355', (uchar)'\356', (uchar)'\357', + (uchar)'\360', (uchar)'\361', (uchar)'\362', (uchar)'\363', (uchar)'\364', + (uchar)'\365', (uchar)'\366', (uchar)'\367', (uchar)'\370', (uchar)'\371', + (uchar)'\372', (uchar)'\373', (uchar)'\374', (uchar)'\375', (uchar)'\376', + (uchar)'\377', +}; + + +static unsigned short cs_to_uni_tis620[256] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, + 0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x000F, 0x0010, 0x0011, + 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001A, + 0x001B, 0x001C, 0x001D, 0x001E, 0x001F, 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002A, 0x002B, 0x002C, + 0x002D, 0x002E, 0x002F, 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, + 0x0036, 0x0037, 0x0038, 0x0039, 0x003A, 0x003B, 0x003C, 0x003D, 0x003E, + 0x003F, 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F, 0x0050, + 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, + 0x005A, 0x005B, 0x005C, 0x005D, 0x005E, 0x005F, 0x0060, 0x0061, 0x0062, + 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006A, 0x006B, + 0x006C, 0x006D, 0x006E, 0x006F, 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, + 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007A, 0x007B, 0x007C, 0x007D, + 0x007E, 0x007F, 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, + 0x0087, 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, + 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F, 0xFFFD, 0x0E01, + 0x0E02, 0x0E03, 0x0E04, 0x0E05, 0x0E06, 0x0E07, 0x0E08, 0x0E09, 0x0E0A, + 0x0E0B, 0x0E0C, 0x0E0D, 0x0E0E, 0x0E0F, 0x0E10, 0x0E11, 0x0E12, 0x0E13, + 0x0E14, 0x0E15, 0x0E16, 0x0E17, 0x0E18, 0x0E19, 0x0E1A, 0x0E1B, 0x0E1C, + 0x0E1D, 0x0E1E, 0x0E1F, 0x0E20, 0x0E21, 0x0E22, 0x0E23, 0x0E24, 0x0E25, + 0x0E26, 0x0E27, 0x0E28, 0x0E29, 0x0E2A, 0x0E2B, 0x0E2C, 0x0E2D, 0x0E2E, + 0x0E2F, 0x0E30, 0x0E31, 0x0E32, 0x0E33, 0x0E34, 0x0E35, 0x0E36, 0x0E37, + 0x0E38, 0x0E39, 0x0E3A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x0E3F, 0x0E40, + 0x0E41, 0x0E42, 0x0E43, 0x0E44, 0x0E45, 0x0E46, 0x0E47, 0x0E48, 0x0E49, + 0x0E4A, 0x0E4B, 0x0E4C, 0x0E4D, 0x0E4E, 0x0E4F, 0x0E50, 0x0E51, 0x0E52, + 0x0E53, 0x0E54, 0x0E55, 0x0E56, 0x0E57, 0x0E58, 0x0E59, 0x0E5A, 0x0E5B, + 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD}; +static unsigned char pl00_tis620[256] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, + 0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x000F, 0x0010, 0x0011, + 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001A, + 0x001B, 0x001C, 0x001D, 0x001E, 0x001F, 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002A, 0x002B, 0x002C, + 0x002D, 0x002E, 0x002F, 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, + 0x0036, 0x0037, 0x0038, 0x0039, 0x003A, 0x003B, 0x003C, 0x003D, 0x003E, + 0x003F, 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F, 0x0050, + 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, + 0x005A, 0x005B, 0x005C, 0x005D, 0x005E, 0x005F, 0x0060, 0x0061, 0x0062, + 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006A, 0x006B, + 0x006C, 0x006D, 0x006E, 0x006F, 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, + 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007A, 0x007B, 0x007C, 0x007D, + 0x007E, 0x007F, 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, + 0x0087, 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, + 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000}; +static unsigned char pl0E[256] = { + 0x0000, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, + 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, 0x00B0, 0x00B1, + 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x00BA, + 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, 0x00C0, 0x00C1, 0x00C2, 0x00C3, + 0x00C4, 0x00C5, 0x00C6, 0x00C7, 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, + 0x00CD, 0x00CE, 0x00CF, 0x00D0, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, + 0x00D6, 0x00D7, 0x00D8, 0x00D9, 0x00DA, 0x0000, 0x0000, 0x0000, 0x0000, + 0x00DF, 0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7, + 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF, 0x00F0, + 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7, 0x00F8, 0x00F9, + 0x00FA, 0x00FB, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000}; +static unsigned char plFF[256] = { + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x00FF, 0x0000, 0x0000}; +static unsigned char *uni_to_cs_tis620[256] = { + pl00_tis620, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, pl0E, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, plFF}; \ No newline at end of file diff --git a/deps/oblib/src/lib/charset/ob_ctype_uca.cc b/deps/oblib/src/lib/charset/ob_ctype_uca.cc index c9a41217f..dede365cb 100644 --- a/deps/oblib/src/lib/charset/ob_ctype_uca.cc +++ b/deps/oblib/src/lib/charset/ob_ctype_uca.cc @@ -2906,8 +2906,8 @@ ObCharsetInfo ob_charset_utf8mb4_unicode_ci= NULL, NULL, &ob_uca_v400, - //NULL, - //NULL, + NULL, + NULL, &ob_unicase_default, NULL, NULL, @@ -2941,8 +2941,8 @@ ObCharsetInfo ob_charset_utf16_unicode_ci= NULL, NULL, &ob_uca_v400, - // NULL, - // NULL, + NULL, + NULL, &ob_unicase_default, NULL, NULL, @@ -2974,6 +2974,8 @@ ObCharsetInfo ob_charset_utf8mb4_0900_ai_ci = { nullptr, nullptr, &ob_uca_v900, + NULL, + NULL, &ob_unicase_unicode900, nullptr, nullptr, @@ -3007,6 +3009,8 @@ ObCharsetInfo ob_charset_utf8mb4_zh_0900_as_cs = { nullptr, nullptr, &ob_uca_v900, + NULL, + NULL, &ob_unicase_unicode900, nullptr, nullptr, @@ -3040,6 +3044,8 @@ ObCharsetInfo ob_charset_utf8mb4_zh2_0900_as_cs = { nullptr, nullptr, &ob_uca_v900, + NULL, + NULL, &ob_unicase_unicode900, nullptr, nullptr, @@ -3073,6 +3079,8 @@ ObCharsetInfo ob_charset_utf8mb4_zh3_0900_as_cs = { nullptr, nullptr, &ob_uca_v900, + NULL, + NULL, &ob_unicase_unicode900, nullptr, nullptr, @@ -3140,6 +3148,8 @@ ObCharsetInfo ob_charset_utf8mb4_0900_bin = { nullptr, // to_upper nullptr, // sort_order nullptr, // uca + NULL, + NULL, &ob_unicase_unicode900, // caseinfo nullptr, // state_map nullptr, // ident_map @@ -3158,3 +3168,105 @@ ObCharsetInfo ob_charset_utf8mb4_0900_bin = { &ob_charset_utf8mb4_handler, &ob_collation_utf8mb4_0900_bin_handler, NO_PAD}; + +ObCharsetInfo ob_charset_utf8mb4_croatian_uca_ci = { + 245, 0, 0, // numbers + OB_CS_UTF8MB4_UCA_FLAGS, // flags + OB_UTF8MB4, // cs name + OB_UTF8MB4 "_croatian_ci", // name + "", // comment + croatian, // tailoring + nullptr, // coll_param + ctype_utf8, // ctype + nullptr, // to_lower + nullptr, // to_upper + nullptr, // sort_order + nullptr, // uca + nullptr, // tab_to_uni + nullptr, // tab_from_uni + &ob_unicase_default, // caseinfo + nullptr, // state_map + nullptr, // ident_map + 8, // strxfrm_multiply + 1, // caseup_multiply + 1, // casedn_multiply + 1, // mbminlen + 4, // mbmaxlen + 1, // mbmaxlenlen + 9, // min_sort_char + 0xFFFF, // max_sort_char + ' ', // pad char + false, // escape_with_backslash_is_dangerous + 1, // levels_for_compare + 1, // levels_for_order + &ob_charset_utf8mb4_handler, + &ob_collation_any_uca_handler, + PAD_SPACE}; + +ObCharsetInfo ob_charset_utf8mb4_unicode_520_ci = { + 246, 0, 0, // numbers + OB_CS_UTF8MB4_UCA_FLAGS, // flags + OB_UTF8MB4, // cs name + OB_UTF8MB4 "_unicode_520_ci", // name + "", // comment + "", // tailoring + nullptr, // coll_param + ctype_utf8, // ctype + nullptr, // to_lower + nullptr, // to_upper + nullptr, // sort_order + &ob_uca_v520, // uca + nullptr, // tab_to_uni + nullptr, // tab_from_uni + &ob_unicase_unicode520, // caseinfo + nullptr, // state_map + nullptr, // ident_map + 8, // strxfrm_multiply + 1, // caseup_multiply + 1, // casedn_multiply + 1, // mbminlen + 4, // mbmaxlen + 1, // mbmaxlenlen + 9, // min_sort_char + 0x10FFFF, // max_sort_char + ' ', // pad char + false, // escape_with_backslash_is_dangerous + 1, // levels_for_compare + 1, // levels_for_order + &ob_charset_utf8mb4_handler, + &ob_collation_any_uca_handler, + PAD_SPACE}; + +ObCharsetInfo ob_charset_utf8mb4_czech_uca_ci = { + 234, 0, 0, // numbers + OB_CS_UTF8MB4_UCA_FLAGS, // flags + OB_UTF8MB4, // cs name + OB_UTF8MB4 "_czech_ci", // name + "", // comment + czech, // tailoring + nullptr, // coll_param + ctype_utf8, // ctype + nullptr, // to_lower + nullptr, // to_upper + nullptr, // sort_order + nullptr, // uca + nullptr, // tab_to_uni + nullptr, // tab_from_uni + &ob_unicase_default, // caseinfo + nullptr, // state_map + nullptr, // ident_map + 8, // strxfrm_multiply + 1, // caseup_multiply + 1, // casedn_multiply + 1, // mbminlen + 4, // mbmaxlen + 1, // mbmaxlenlen + 9, // min_sort_char + 0xFFFF, // max_sort_char + ' ', // pad char + false, // escape_with_backslash_is_dangerous + 1, // levels_for_compare + 1, // levels_for_order + &ob_charset_utf8mb4_handler, + &ob_collation_any_uca_handler, + PAD_SPACE}; \ No newline at end of file diff --git a/deps/oblib/src/lib/charset/ob_ctype_utf16.cc b/deps/oblib/src/lib/charset/ob_ctype_utf16.cc index 575366acf..2d2bf7778 100644 --- a/deps/oblib/src/lib/charset/ob_ctype_utf16.cc +++ b/deps/oblib/src/lib/charset/ob_ctype_utf16.cc @@ -48,7 +48,6 @@ ob_tosort_utf16(ObUnicaseInfo *uni_plane, ob_wc_t *wc) static unsigned int ob_mbcharlen_utf16(const ObCharsetInfo *cs __attribute__((unused)), unsigned int c __attribute__((unused))) { - ob_charset_assert(0); return OB_UTF16_HIGH_HEAD(c) ? 4 : 2; } @@ -1129,6 +1128,7 @@ PAD_MIN_MAX: ObCharsetHandler ob_charset_utf16_handler= { + NULL, ob_ismbchar_utf16, ob_mbcharlen_utf16, ob_numchars_utf16, @@ -1199,6 +1199,8 @@ ObCharsetInfo ob_charset_utf16_bin= NULL, NULL, NULL, + NULL, + NULL, &ob_unicase_default, NULL, NULL, @@ -1233,6 +1235,8 @@ ObCharsetInfo ob_charset_utf16_general_ci= NULL, NULL, NULL, + NULL, + NULL, &ob_unicase_default, NULL, NULL, diff --git a/deps/oblib/src/lib/charset/ob_ctype_utf8.cc b/deps/oblib/src/lib/charset/ob_ctype_utf8.cc index 3a54e0e81..c7e2b5aa9 100644 --- a/deps/oblib/src/lib/charset/ob_ctype_utf8.cc +++ b/deps/oblib/src/lib/charset/ob_ctype_utf8.cc @@ -443,6 +443,7 @@ void ob_strnxfrm_unicode_help(uchar **dst, if ((flags & OB_STRXFRM_PAD_TO_MAXLEN) && *dst < *de) *dst += ob_strxfrm_pad_unicode(*dst, *de); } + size_t ob_strnxfrm_unicode(const ObCharsetInfo *cs, uchar *dst, size_t dstlen, uint nweights, const uchar *src, size_t srclen, uint flags, bool *is_valid_unicode) @@ -944,6 +945,7 @@ int ob_mb_wc_utf8mb4_thunk(const ObCharsetInfo *cs __attribute__((unused)), ObCharsetHandler ob_charset_utf8mb4_handler= { + NULL, ob_ismbchar_utf8mb4, ob_mbcharlen_utf8mb4, ob_numchars_mb, @@ -1015,6 +1017,8 @@ ObCharsetInfo ob_charset_utf8mb4_general_ci= to_upper_utf8mb4, to_upper_utf8mb4, NULL, + NULL, + NULL, &ob_unicase_default, NULL, NULL, @@ -1050,6 +1054,8 @@ ObCharsetInfo ob_charset_utf8mb4_bin= to_upper_utf8mb4, NULL, NULL, + NULL, + NULL, &ob_unicase_default, NULL, NULL, diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_result.h b/deps/oblib/src/lib/mysqlclient/ob_mysql_result.h index 48ff8dc1b..d647907db 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_result.h +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_result.h @@ -1072,6 +1072,9 @@ ObCastCtx cast_ctx(&allocator, no_dtc_params ? NULL : &dtc_params, CM_NONE, column.get_collation_type());\ if (is_cur_default_value && column.is_default_expr_v2_column())\ { \ + if (lib::is_mysql_mode()) { \ + res_obj.set_collation_type(CS_TYPE_UTF8MB4_BIN); \ + } \ res_obj.set_varchar(str_value); \ ret = (class_obj).set_##column_name(res_obj); \ } \ @@ -1179,6 +1182,144 @@ } \ } () +#define EXTRACT_DEFAULT_VALUE_FIELD_MYSQL_V2(result, data_type, class_obj,is_cur_default_value, default_value_v2_version, tenant_id) \ +{ \ + if (OB_SUCC(ret)) \ + { \ + ObString str_value; \ + ObObj res_obj; \ + ret = (result).get_varchar("cur_default_value", str_value); \ + \ + if (OB_ERR_NULL_VALUE == ret) \ + { /* without default value */ \ + if (!default_value_v2_version) { \ + res_obj.set_null(); \ + ret = (class_obj).set_cur_default_value(res_obj, column.is_default_expr_v2_column()); \ + } \ + else { \ + ret = OB_SUCCESS; \ + } \ + } \ + else if (OB_ERR_COLUMN_NOT_FOUND == ret) \ + { \ + SQL_LOG(WARN, "column not found, ignore", "column_name", "cur_default_value"); \ + ret = OB_SUCCESS; \ + } \ + else if (OB_SUCC(ret)) \ + { /*big stack check ObSchemaRetrieveUtils::fill_column_schema*/ \ + ObObj def_obj; \ + ObArenaAllocator allocator(ObModIds::OB_SCHEMA); \ + ObTimeZoneInfo tz_info; \ + const ObDataTypeCastParams dtc_params(&tz_info); \ + bool no_dtc_params = (ob_is_bit_tc(data_type) || ob_is_enum_or_set_type(data_type)); \ + ObCastCtx cast_ctx(&allocator, no_dtc_params ? NULL : &dtc_params, CM_NONE, column.get_collation_type());\ + if (is_cur_default_value && column.is_default_expr_v2_column()) \ + { \ + if (lib::is_mysql_mode()) { \ + res_obj.set_collation_type(CS_TYPE_UTF8MB4_BIN); \ + } \ + res_obj.set_varchar(str_value); \ + ret = (class_obj).set_cur_default_value(res_obj, column.is_default_expr_v2_column()); \ + } \ + else if (IS_DEFAULT_NOW_STR(data_type, str_value)) \ + { \ + res_obj.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); \ + res_obj.set_scale(column.get_data_scale()); \ + ret = (class_obj).set_cur_default_value(res_obj, column.is_default_expr_v2_column()); \ + } \ + else if (column.is_generated_column()) { \ + res_obj.set_string(data_type, str_value); \ + res_obj.meta_.set_collation_type(CS_TYPE_UTF8MB4_BIN); \ + res_obj.meta_.set_collation_level(CS_LEVEL_IMPLICIT); \ + ret = (class_obj).set_cur_default_value(res_obj, column.is_default_expr_v2_column()); \ + } \ + else if (column.is_identity_column() || ob_is_string_type(data_type) || ob_is_geometry(data_type) || ob_is_collection_sql_type(data_type)) \ + { \ + res_obj.set_string(data_type, str_value); \ + res_obj.meta_.set_collation_type(column.get_collation_type()); \ + /* will override the collaction level set in set_varchar */ \ + res_obj.meta_.set_collation_level(CS_LEVEL_IMPLICIT); \ + /* only support full inrow data, all lobs from systable should be made inrow */ \ + if (res_obj.is_outrow_lob()) { \ + ret = OB_INVALID_ARGUMENT; \ + SQL_LOG(WARN, "outrow lob unsupported", "column_name", "cur_default_value"); \ + } \ + else { \ + if (ob_is_text_tc(data_type) || ob_is_geometry(data_type) || ob_is_collection_sql_type(data_type)) { res_obj.set_inrow(); } \ + ret = (class_obj).set_cur_default_value(res_obj, column.is_default_expr_v2_column()); \ + } \ + } \ + else { \ + if (ob_is_bit_tc(data_type) || ob_is_enum_or_set_type(data_type)) \ + { \ + def_obj.set_varchar(str_value); \ + ObObj tmp_obj; \ + if(OB_FAIL(ObObjCaster::to_type(ObUInt64Type, cast_ctx, def_obj, tmp_obj))) \ + { \ + SQL_LOG(WARN, "cast obj failed, ", "src type", def_obj.get_type()); \ + } \ + else \ + { \ + if (ObBitType == data_type) { \ + res_obj.set_bit(tmp_obj.get_uint64()); \ + res_obj.set_scale(column.get_data_precision()); \ + } else if (ObEnumType == data_type) { \ + res_obj.set_enum(tmp_obj.get_uint64()); \ + } else {/*set type*/ \ + res_obj.set_set(tmp_obj.get_uint64()); \ + } \ + ret = (class_obj).set_cur_default_value(res_obj, column.is_default_expr_v2_column());\ + } \ + } \ + else if (ob_is_json(data_type)) \ + { \ + def_obj.set_type(data_type); \ + if (is_mysql_mode()) { \ + if (OB_FAIL(def_obj.build_not_strict_default_value(column.get_data_precision()))) { \ + SQL_LOG(WARN, "failed to build not strict default json value", K(ret)); \ + } else { \ + res_obj.set_json_value(data_type, def_obj.get_string().ptr(), \ + def_obj.get_string().length()); \ + } \ + } else { \ + def_obj.set_json_value(data_type, str_value.ptr(), str_value.length()); \ + if (OB_FAIL(ObObjCaster::to_type(data_type, cast_ctx, def_obj, res_obj))) { \ + SQL_LOG(WARN, "cast obj failed, ", "src type", def_obj.get_type(), "dest type", data_type); \ + } else { \ + res_obj.set_inrow(); \ + } \ + } \ + if (OB_SUCC(ret)) { \ + res_obj.meta_.set_collation_level(CS_LEVEL_IMPLICIT); \ + ret = (class_obj).set_cur_default_value(res_obj, column.is_default_expr_v2_column());\ + } \ + } \ + else \ + { \ + def_obj.set_varchar(str_value); \ + if (OB_FAIL(OTTZ_MGR.get_tenant_tz(tenant_id, tz_info.get_tz_map_wrap()))) \ + { \ + SQL_LOG(WARN, "get tenant timezone map failed", K(ret)); \ + } \ + else if (OB_FAIL(ObObjCaster::to_type(data_type, cast_ctx, def_obj, res_obj))) \ + { \ + SQL_LOG(WARN, "cast obj failed, ", "src type", def_obj.get_type(), "dest type", data_type); \ + } \ + else \ + { \ + res_obj.set_scale(column.get_data_scale()); \ + ret = (class_obj).set_cur_default_value(res_obj, column.is_default_expr_v2_column());\ + } \ + } \ + } \ + } \ + else \ + { \ + SQL_LOG(WARN, "fail to default value field mysql. ", K(ret)); \ + } \ + } \ +} + #define EXTRACT_CREATE_TIME_FIELD_MYSQL(result, column_name, field, type) \ EXTRACT_INT_FIELD_MYSQL(result, column_name, field, type) diff --git a/deps/oblib/src/lib/ob_define.h b/deps/oblib/src/lib/ob_define.h index 277c7f22b..cb5388f0b 100644 --- a/deps/oblib/src/lib/ob_define.h +++ b/deps/oblib/src/lib/ob_define.h @@ -445,7 +445,7 @@ const int64_t OB_SCHEMA_START_VERSION = 100; const int64_t OB_SYS_PARAM_ROW_KEY_LENGTH = 192; const int64_t OB_MAX_SYS_PARAM_NAME_LENGTH = 128; const int64_t OB_MAX_SYS_PARAM_VALUE_LENGTH = 1024; -const int64_t OB_MAX_SYS_PARAM_NUM = 600; +const int64_t OB_MAX_SYS_PARAM_NUM = 700; const int64_t OB_MAX_PREPARE_STMT_NUM_PER_SESSION = 512; const uint32_t INVALID_SESSID = UINT32_MAX; const int64_t OB_MAX_VAR_NUM_PER_SESSION = 1024; @@ -856,6 +856,10 @@ const uint64_t DEFAULT_CUSTOMIZED_CG_NUM = 2; const int64_t OB_CG_NAME_PREFIX_LENGTH = 5; // length of cg prefix like "__cg_" const int64_t OB_MAX_COLUMN_GROUP_NAME_LENGTH = OB_MAX_COLUMN_NAME_LENGTH * OB_MAX_CHAR_LEN + OB_CG_NAME_PREFIX_LENGTH; //(max_column_name_length(128) * ob_max_char_len(3)) + prefix const int64_t MAX_NAME_CHAR_LEN = 64; +const int64_t MAX_AUDIT_FILTER_NAME_LENGTH = 64; +const int64_t MAX_AUDIT_FILTER_NAME_LENGTH_BYTE = 4 * MAX_AUDIT_FILTER_NAME_LENGTH; +const int64_t MAX_AUDIT_USER_NAME_LENGTH_BYTE = 4 * OB_MAX_USER_NAME_LENGTH_STORE; +const int64_t MAX_AUDIT_HOST_NAME_LENGTH_BYTE = 4 * OB_MAX_HOST_NAME_LENGTH; //Oracle const int64_t MAX_ORACLE_COMMENT_LENGTH = 4000; diff --git a/deps/oblib/src/lib/ob_errno.h b/deps/oblib/src/lib/ob_errno.h index 0edcefc8d..431fbc70e 100644 --- a/deps/oblib/src/lib/ob_errno.h +++ b/deps/oblib/src/lib/ob_errno.h @@ -182,6 +182,7 @@ constexpr int OB_ERR_UNEXPECTED_TZ_TRANSITION = -5297; constexpr int OB_ERR_INVALID_TIMEZONE_REGION_ID = -5341; constexpr int OB_ERR_INVALID_HEX_NUMBER = -5342; constexpr int OB_ERR_FIELD_NOT_FOUND_IN_DATETIME_OR_INTERVAL = -5352; +constexpr int OB_CANT_AGGREGATE_3COLLATIONS = -5356; constexpr int OB_ERR_INVALID_JSON_TEXT = -5411; constexpr int OB_ERR_INVALID_JSON_TEXT_IN_PARAM = -5412; constexpr int OB_ERR_INVALID_JSON_BINARY_DATA = -5413; diff --git a/deps/oblib/src/lib/ob_name_def.h b/deps/oblib/src/lib/ob_name_def.h index d61e2a1ae..1ca13f8e7 100644 --- a/deps/oblib/src/lib/ob_name_def.h +++ b/deps/oblib/src/lib/ob_name_def.h @@ -758,6 +758,7 @@ #define N_CURRENT_USER "current_user" #define N_CURRENT_USER_PRIV "current_user_priv" #define N_CURRENT_ROLE "current_role" +#define N_IS_ENABLED_ROLE "is_enabled_role" #define N_USER "user" #define N_HOST_IP "host_ip" #define N_RPC_PORT "rpc_port" @@ -1154,4 +1155,9 @@ #define N_RB_TO_STRING "rb_to_string" #define N_RB_FROM_STRING "rb_from_string" #define N_GET_PATH "get_path" +#define N_AUDIT_LOG_FILTER_SET_FILTER "audit_log_filter_set_filter" +#define N_AUDIT_LOG_FILTER_REMOVE_FILTER "audit_log_filter_remove_filter" +#define N_AUDIT_LOG_FILTER_SET_USER "audit_log_filter_set_user" +#define N_AUDIT_LOG_FILTER_REMOVE_USER "audit_log_filter_remove_user" +#define N_CAN_ACCESS_TRIGGER "can_access_trigger" #endif //OCEANBASE_LIB_OB_NAME_DEF_H_ diff --git a/deps/oblib/unittest/lib/charset/test_charset.cpp b/deps/oblib/unittest/lib/charset/test_charset.cpp index d8a3c1cb3..c6ec48ae6 100644 --- a/deps/oblib/unittest/lib/charset/test_charset.cpp +++ b/deps/oblib/unittest/lib/charset/test_charset.cpp @@ -22,6 +22,7 @@ #include "lib/allocator/page_arena.h" #include "lib/charset/ob_charset.h" +#include "lib/charset/ob_template_helper.h" #include "lib/string/ob_string.h" #include "lib/utility/ob_print_utils.h" #include "unicode_map.h" @@ -854,6 +855,89 @@ TEST_F(TestCharset, check_mbmaxlenlen) } } +std::vector test_strings = {"1", "abcdef", "ab1dc4", "你好", "b今a天", "1abad "}; + +TEST_F(TestCharset, basic_collation_handler_test) +{ + ObArenaAllocator alloc; + for (int i = CS_TYPE_INVALID; i < CS_TYPE_EXTENDED_MARK; i++) { + ObCollationType coll = static_cast(i); + const ObCharsetInfo * cs = ObCharset::get_charset(coll); + const char *coll_name = ObCharset::collation_name(coll); + if (OB_NOT_NULL(cs)) { + std::cout << "#TEST Coll = " << coll_name << std::endl; + for (const char* utf8_str:test_strings) { + ObString dst; + if (cs->mbmaxlen <= 1) { + dst = ObString(utf8_str); + } else { + ASSERT_EQ(0, ObCharset::charset_convert(alloc, ObString(utf8_str), CS_TYPE_UTF8MB4_BIN, coll, dst)); + } + char*str = dst.ptr(); + char*end = dst.ptr() + dst.length(); + + if (OB_NOT_NULL(cs->coll->strnncoll)) { + fprintf(stdout, ">> strnncoll = %d for text = \"%s\"\n", + cs->coll->strnncoll(cs, pointer_cast(str), end-str, pointer_cast(str), end-str, true), utf8_str); + } + if (OB_NOT_NULL(cs->coll->strnncollsp)) { + fprintf(stdout, ">> strnncollsp = %d for text = \"%s\"\n", + cs->coll->strnncollsp(cs, pointer_cast(str), end-str, pointer_cast(str), end-str, true), utf8_str); + } + if (OB_NOT_NULL(cs->coll->strnxfrm)) { + char temp[100]; + bool is_valid_unicode = false; + fprintf(stdout, ">> strnxfrm = %ld for text = \"%s\", is_valid_unicode = %d\n", + cs->coll->strnxfrm(cs, reinterpret_cast(temp), 100, UINT32_MAX, + reinterpret_cast(str), end-str, 0, &is_valid_unicode), utf8_str, is_valid_unicode); + } + if (OB_NOT_NULL(cs->coll->strnxfrmlen)) { + fprintf(stdout, ">> strnxfrmlen = %ld for text = \"%s\"\n", + cs->coll->strnxfrmlen(cs, end-str), utf8_str); + } + if (OB_NOT_NULL(cs->coll->strnxfrm_varlen)) { + fprintf(stdout, ">> strnxfrmlen = %ld for text = \"%s\"\n", + cs->coll->strnxfrmlen(cs, end-str), utf8_str); + } + if (OB_NOT_NULL(cs->coll->like_range)) { + char temp1[100]; + char temp2[100]; + size_t len1, len2; + fprintf(stdout, ">> like_range = %d for text = \"%s\", min = %.*s, max = %.*s\n", + cs->coll->like_range(cs, str, end-str, '\\', '_', '%', 100, temp1, temp2, &len1, &len2), utf8_str, + (int)len1, temp1, (int)len2, temp2); + } + if (OB_NOT_NULL(cs->coll->wildcmp)) { + const char *wild_str = "%"; + fprintf(stdout, ">> wildcmp = %d for text = \"%s\"\n", + cs->coll->wildcmp(cs, str, end, wild_str, wild_str + strlen(wild_str), '\\', '_', '%'), utf8_str); + } + if (OB_NOT_NULL(cs->coll->strcasecmp)) { + fprintf(stdout, ">> strcasecmp = %d for text = \"%s\"\n", + cs->coll->strcasecmp(cs, str, end), utf8_str); + } + if (OB_NOT_NULL(cs->coll->instr)) { + ob_match_t m_match_t[2]; + unsigned int nmatch = 1; + const char *temp = "1"; + fprintf(stdout, ">> instr = %d for text = \"%s\" nmatch = %u match_mb_len = %u\n", + cs->coll->instr(cs, temp, strlen(temp), str, end-str, m_match_t, nmatch), utf8_str, + nmatch, m_match_t[0].mb_len); + } + if (OB_NOT_NULL(cs->coll->hash_sort)) { + ulong nr1, nr2; + cs->coll->hash_sort(cs, pointer_cast(str), end-str, &nr1, &nr2, true, NULL); + fprintf(stdout, ">> hash_sort for text = \"%s\" nr1 = %lu nr1 = %lu\n", utf8_str, nr1, nr2); + } + if (OB_NOT_NULL(cs->coll->propagate)) { + fprintf(stdout, ">> propagate = %d for text = \"%s\"\n", + cs->coll->propagate(cs, pointer_cast(str), end-str), utf8_str); + } + } + } + } +} + TEST_F(TestCharset, foreach_char) { const char *data = "豫章故郡，洪都新府。星分翼轸，地接衡庐。襟三江而带五湖，控蛮荆而引瓯越。物华天宝，龙光射牛斗之墟" "人杰地灵，徐孺下陈蕃之榻。雄州雾列，俊采星驰。台隍枕夷夏之交，宾主尽东南之美。都督阎公之雅望，棨戟遥临" @@ -971,7 +1055,7 @@ TEST_F(TestCharset, foreach_char) { fprintf(stdout, "Raw Impl\n"); start_timer(); - ASSERT_EQ(OB_SUCCESS, ObCharsetUtils::foreach_char(input, test_collation_type, do_nothing)); + ASSERT_EQ(OB_SUCCESS, ObCharsetUtils::foreach_char(input, test_collation_type, do_nothing, true)); end_timer(); @@ -985,11 +1069,98 @@ TEST_F(TestCharset, foreach_char) { ASSERT_EQ(OB_SUCCESS, ObFastStringScanner::foreach_char(input, test_cs_type, do_nothing, false)); end_timer(); } +} - +TEST_F(TestCharset, basic_charset_handler_test) +{ + ObArenaAllocator alloc; + for (int i = CHARSET_INVALID; i < CHARSET_MAX; i++) { + if (ObCharset::is_valid_charset(static_cast(i))) { + ObCollationType coll = static_cast(ObCharset::get_default_collation(static_cast(i))); + const char *coll_name = ObCharset::collation_name(coll); + const ObCharsetInfo * cs = ObCharset::get_charset(coll); + std::cout << "#TEST Coll = " << coll_name << std::endl; + for (const char* utf8_str:test_strings) { + ObString dst; + if (cs->mbmaxlen <= 1) { + dst = ObString(utf8_str); + } else { + ASSERT_EQ(0, ObCharset::charset_convert(alloc, ObString(utf8_str), CS_TYPE_UTF8MB4_BIN, coll, dst)); + } + const char*str = dst.ptr(); + const char*end = dst.ptr() + dst.length(); + if (OB_NOT_NULL(cs->cset->ismbchar)) { + fprintf(stdout, ">> ismbchar = %d for text = \"%s\"\n", + cs->cset->ismbchar(cs, str, end), utf8_str); + } + if (OB_NOT_NULL(cs->cset->mbcharlen)) { + fprintf(stdout, ">> mbcharlen = %d for text = \"%s\"\n", + cs->cset->mbcharlen(cs, str[0]), utf8_str); + } + if (OB_NOT_NULL(cs->cset->numchars)) { + fprintf(stdout, ">> numchars = %ld for text = \"%s\"\n", + cs->cset->numchars(cs, str, end), utf8_str); + } + if (OB_NOT_NULL(cs->cset->charpos)) { + size_t pos = 3; + fprintf(stdout, ">> charpos = %ld pos = %ld for text = \"%s\"\n", + cs->cset->charpos(cs, str, end, pos), pos, utf8_str); + } + if (OB_NOT_NULL(cs->cset->max_bytes_charpos)) { + size_t max_bytes = 5; + size_t char_len = 0; + fprintf(stdout, ">> max_bytes_charpos = %ld max_bytes = %ld char_len = %ld for text = \"%s\"\n", + cs->cset->max_bytes_charpos(cs, str, end, max_bytes, &char_len), max_bytes, char_len, utf8_str); + } + if (OB_NOT_NULL(cs->cset->well_formed_len)) { + int error = 0; + fprintf(stdout, ">> well_formed_len = %ld error = %d text = \"%s\"\n", + cs->cset->well_formed_len(cs, str, end, INT64_MAX, &error), error, utf8_str); + } + if (OB_NOT_NULL(cs->cset->lengthsp)) { + fprintf(stdout, ">> lengthsp = %ld text = \"%s\"\n", + cs->cset->lengthsp(cs, str, end-str), utf8_str); + } + if (OB_NOT_NULL(cs->cset->mb_wc)) { + ob_wc_t wchar = 0; + fprintf(stdout, ">> mb_wc = %d wchar = %ld text = \"%s\"\n", + cs->cset->mb_wc(cs, &wchar, pointer_cast(str), pointer_cast(end)), wchar, utf8_str); + } + if (OB_NOT_NULL(cs->cset->wc_mb)) { + ob_wc_t wchar = 41; + unsigned char temp[10]; + MEMSET(temp, 0, 10); + fprintf(stdout, ">> wc_mb = %d A = %.*s text = \"%s\"\n", + cs->cset->wc_mb(cs, wchar, temp, temp + 10), 10, temp, utf8_str); + } + if (OB_NOT_NULL(cs->cset->ctype)) { + int ctype = 0; + fprintf(stdout, ">> ctype = %d ctype = %d text = \"%s\"\n", + cs->cset->ctype(cs, &ctype, pointer_cast(str), pointer_cast(end)), ctype, utf8_str); + } + if (cs->casedn_multiply <= 1 && OB_NOT_NULL(cs->cset->casedn)) { + ObString temp; + ASSERT_EQ(0, ob_write_string(alloc, dst, temp)); + fprintf(stdout, ">> casedn = %ld res = %.*s text = \"%s\"\n", + cs->cset->casedn(cs, temp.ptr(), temp.length(), temp.ptr(), temp.length()), temp.length(), temp.ptr(), utf8_str); + } + if (cs->caseup_multiply <= 1 && OB_NOT_NULL(cs->cset->caseup)) { + ObString temp; + ASSERT_EQ(0, ob_write_string(alloc, dst, temp)); + fprintf(stdout, ">> caseup = %ld res = %.*s text = \"%s\"\n", + cs->cset->caseup(cs, temp.ptr(), temp.length(), temp.ptr(), temp.length()), temp.length(), temp.ptr(), utf8_str); + } + if (OB_NOT_NULL(cs->cset->fill)) { + char temp[10]; + cs->cset->fill(cs, temp, 10, 0x42); + fprintf(stdout, ">> fill res = %.*s text = \"%s\"\n", 10, temp, utf8_str); + } + } + } + } } int main(int argc, char **argv) diff --git a/src/logservice/libobcdc/src/ob_obj2str_helper.cpp b/src/logservice/libobcdc/src/ob_obj2str_helper.cpp index 7b2f9697e..6df0c2224 100644 --- a/src/logservice/libobcdc/src/ob_obj2str_helper.cpp +++ b/src/logservice/libobcdc/src/ob_obj2str_helper.cpp @@ -92,6 +92,9 @@ int ObObj2strHelper::init_ob_charset_utils() OBLOG_LOG(ERROR, "failed to init ObNumberConstValue", KR(ret)); } else if (OB_FAIL(sql::ARITH_RESULT_TYPE_ORACLE.init())) { OBLOG_LOG(ERROR, "failed to init ORACLE_ARITH_RESULT_TYPE", KR(ret)); + } else if (OB_FAIL(ObCharset::init_charset())) { + SQL_LOG(ERROR, "fail to init charset", K(ret)); + //pre-generate charset const str tab should be done after init_charset } else if (OB_FAIL(ObCharsetUtils::init(*allocator))) { OBLOG_LOG(ERROR, "fail to init ObCharsetUtils", KR(ret)); } else if (OB_FAIL(wide::ObDecimalIntConstValue::init_const_values(*allocator, attr))) { diff --git a/src/objit/include/objit/common/ob_item_type.h b/src/objit/include/objit/common/ob_item_type.h index 46ed636d1..e916df7e8 100755 --- a/src/objit/include/objit/common/ob_item_type.h +++ b/src/objit/include/objit/common/ob_item_type.h @@ -495,7 +495,6 @@ typedef enum ObItemType T_FUN_SYS_DES_ENCRYPT = 763, T_FUN_SYS_ENCRYPT = 764, T_FUN_SYS_ICU_VERSION = 765, - T_FUN_SYS_CURRENT_USER_PRIV = 766, T_FUN_SYS_CURRENT_ROLE = 767, T_FUN_SYS_EXTRACT_CERT_EXPIRED_TIME = 768, @@ -2726,7 +2725,7 @@ typedef enum ObOutlineType #define IS_DML_STMT(op) \ ((op) == T_SELECT || (op) == T_DELETE || (op) == T_INSERT || (op) == T_MERGE || (op) == T_UPDATE || (op) == T_MULTI_INSERT) -#define IS_SHOW_STMT(op) (((op) >= T_SHOW_TABLES && (op) <= T_SHOW_GRANTS) || (op) == T_SHOW_TRIGGERS) +#define IS_SHOW_STMT(op) (((op) >= T_SHOW_TABLES && (op) <= T_SHOW_GRANTS) || (op) == T_SHOW_TRIGGERS || (op) == T_SHOW_CREATE_USER) #define EXPR_OP_NUM (T_MAX_OP-T_MIN_OP-1) extern const char *get_type_name(int type); diff --git a/src/observer/mysql/obmp_base.cpp b/src/observer/mysql/obmp_base.cpp index be39373a0..25a6ff486 100644 --- a/src/observer/mysql/obmp_base.cpp +++ b/src/observer/mysql/obmp_base.cpp @@ -676,12 +676,6 @@ int ObMPBase::update_charset_sys_vars(ObSMConnection &conn, ObSQLSessionInfo &se { int ret = OB_SUCCESS; int64_t cs_type = conn.client_cs_type_; - const int64_t LATIN1_CS = 8; - //background: mysqltest give a default connect_charset=latin1 - // but for history reason, oceanbase use utf8 as - // default charset for mysqltest - //TODO: after obclient&mysqltest support default charset = utf8 - // login for cs_type != LATIN1_CS would be deleted if (ObCharset::is_valid_collation(cs_type)) { if (OB_FAIL(sess_info.update_sys_variable(SYS_VAR_CHARACTER_SET_CLIENT, cs_type))) { SQL_ENG_LOG(WARN, "failed to update sys var", K(ret)); diff --git a/src/observer/mysql/obmp_connect.cpp b/src/observer/mysql/obmp_connect.cpp index 550775055..0f413e619 100644 --- a/src/observer/mysql/obmp_connect.cpp +++ b/src/observer/mysql/obmp_connect.cpp @@ -40,7 +40,10 @@ #include "observer/omt/ob_tenant.h" #include "observer/ob_req_time_service.h" #include "storage/tx/wrs/ob_weak_read_util.h" //ObWeakReadUtil +#ifdef OB_BUILD_AUDIT_SECURITY #include "sql/monitor/ob_security_audit_utils.h" +#include "sql/audit/ob_audit_log_utils.h" +#endif #include "sql/privilege_check/ob_privilege_check.h" #include "sql/privilege_check/ob_ora_priv_check.h" #include "lib/utility/ob_backtrace.h" @@ -482,6 +485,12 @@ int ObMPConnect::process() ObString::make_string("CONNECT"), comment_text.string(), proc_ret); + if (OB_SUCC(proc_ret)) { + ObAuditLogUtils::hanlde_connect_audit_log(*session, "LOGIN"); + } else { + ObAuditLogUtils::hanlde_connect_fail_audit_log(*session, user_name_, client_ip_, + db_name_, get_peer(), proc_ret); + } #endif // oracle temp table need to be refactored //if (OB_SUCCESS == proc_ret) { @@ -823,6 +832,11 @@ int ObMPConnect::load_privilege_info(ObSQLSessionInfo &session) } else if (OB_FAIL(check_audit_user(session_priv.tenant_id_, user_name_))) { LOG_WARN("fail to check audit user privilege", K(ret)); #endif + } else if (OB_FAIL(load_audit_log_filter(session_priv.tenant_id_, + user_name_, + client_ip_, + session))) { + LOG_WARN("failed to load audit log filter", K(ret)); } else if (OB_FAIL(get_client_attribute_capability(client_attr_cap_flags))) { LOG_WARN("failed to get client attribute capability", K(ret)); } else { @@ -1071,6 +1085,25 @@ int ObMPConnect::check_audit_user(const uint64_t tenant_id, ObString &user_name) } #endif +int ObMPConnect::load_audit_log_filter(const uint64_t tenant_id, + ObString &user_name, + ObString &client_ip, + sql::ObSQLSessionInfo &session) +{ + int ret = OB_SUCCESS; +#ifdef OB_BUILD_AUDIT_SECURITY + ObString filter_name; + ObArenaAllocator allocator(ObModIds::OB_SQL_SESSION); + if (OB_FAIL(ObAuditLogUtils::get_audit_filter_name(tenant_id, user_name, client_ip, allocator, + filter_name))) { + LOG_WARN("failed to get filter name", K(ret)); + } else if (OB_FAIL(session.set_audit_filter_name(filter_name))) { + LOG_WARN("failed to set filter name", K(ret)); + } +#endif + return ret; +} + int ObMPConnect::update_login_stat_in_trans(const uint64_t tenant_id, const bool is_login_succ, ObSchemaGetterGuard &schema_guard) diff --git a/src/observer/mysql/obmp_connect.h b/src/observer/mysql/obmp_connect.h index e68739cef..5405ea0b6 100644 --- a/src/observer/mysql/obmp_connect.h +++ b/src/observer/mysql/obmp_connect.h @@ -123,7 +123,10 @@ private: #ifdef OB_BUILD_AUDIT_SECURITY int check_audit_user(const uint64_t tenant_id, ObString &user_name); #endif - + int load_audit_log_filter(const uint64_t tenant_id, + ObString &user_name, + ObString &client_ip, + sql::ObSQLSessionInfo &session); int set_proxy_version(ObSMConnection &conn); int set_client_version(ObSMConnection &conn); int extract_service_name(ObSMConnection &conn, ObString &service_name, bool &failover_mode); diff --git a/src/observer/mysql/obmp_query.cpp b/src/observer/mysql/obmp_query.cpp index 52d9ced03..1599b881e 100644 --- a/src/observer/mysql/obmp_query.cpp +++ b/src/observer/mysql/obmp_query.cpp @@ -1229,7 +1229,9 @@ int ObMPQuery::is_readonly_stmt(ObMySQLResultSet &result, bool &is_readonly) case stmt::T_USE_DATABASE: case stmt::T_SET_NAMES: //read only not restrict it case stmt::T_START_TRANS: - case stmt::T_END_TRANS: { + case stmt::T_END_TRANS: + case stmt::T_SHOW_CHECK_TABLE: + case stmt::T_SHOW_CREATE_USER: { is_readonly = true; break; } diff --git a/src/observer/ob_server.cpp b/src/observer/ob_server.cpp index c3782a7b2..2e3b7db91 100644 --- a/src/observer/ob_server.cpp +++ b/src/observer/ob_server.cpp @@ -128,6 +128,9 @@ #endif #include "lib/xml/ob_libxml2_sax_handler.h" #include "ob_check_params.h" +#ifdef OB_BUILD_AUDIT_SECURITY +#include "sql/audit/ob_audit_log_mgr.h" +#endif using namespace oceanbase::lib; using namespace oceanbase::common; @@ -531,6 +534,10 @@ int ObServer::init(const ObServerOptions &opts, const ObPLogWriterCfg &log_cfg) #endif else if (OB_FAIL(ObDetectManagerThread::instance().init(GCTX.self_addr(), net_frame_.get_req_transport()))) { LOG_WARN("init ObDetectManagerThread failed", KR(ret)); +#ifdef OB_BUILD_AUDIT_SECURITY + } else if (OB_FAIL(ObAuditLogMgr::get_instance().init())) { + LOG_WARN("init ObAuditLogMgr failed", KR(ret)); +#endif } else if (OB_FAIL(wr_service_.init())) { LOG_WARN("failed to init wr service", K(ret)); } else if (OB_FAIL(ObStorageHADiagService::instance().init(GCTX.sql_proxy_))) { diff --git a/src/observer/omt/ob_multi_tenant.cpp b/src/observer/omt/ob_multi_tenant.cpp index c89d77162..1d7ad04e3 100644 --- a/src/observer/omt/ob_multi_tenant.cpp +++ b/src/observer/omt/ob_multi_tenant.cpp @@ -167,6 +167,10 @@ #include "storage/checkpoint/ob_checkpoint_diagnose.h" #include "storage/tmp_file/ob_tmp_file_manager.h" // ObTenantTmpFileManager #include "storage/restore/ob_tenant_restore_info_mgr.h" +#ifdef OB_BUILD_AUDIT_SECURITY +#include "sql/audit/ob_audit_logger.h" +#include "sql/audit/ob_audit_log_mgr.h" +#endif using namespace oceanbase; using namespace oceanbase::lib; @@ -591,6 +595,10 @@ int ObMultiTenant::init(ObAddr myaddr, #endif MTL_BIND2(mtl_new_default, storage::ObTenantRestoreInfoMgr::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); MTL_BIND2(mtl_new_default, ObGlobalIteratorPool::mtl_init, nullptr, nullptr, nullptr, ObGlobalIteratorPool::mtl_destroy); +#ifdef OB_BUILD_AUDIT_SECURITY + MTL_BIND2(mtl_new_default, ObAuditLogger::mtl_init, ObAuditLogger::mtl_start, ObAuditLogger::mtl_stop, ObAuditLogger::mtl_wait, mtl_destroy_default); + MTL_BIND2(mtl_new_default, ObAuditLogUpdater::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); +#endif } if (OB_SUCC(ret)) { @@ -1343,6 +1351,9 @@ int ObMultiTenant::update_tenant_config(uint64_t tenant_id) if (OB_TMP_FAIL(update_checkpoint_diagnose_config())) { LOG_WARN("failed to update tenant ddl config", K(tmp_ret), K(tenant_id)); } + if (OB_TMP_FAIL(update_tenant_audit_log_config())) { + LOG_WARN("failed to update tenant audit log config", K(tmp_ret), K(tenant_id)); + } } } LOG_INFO("update_tenant_config success", K(tenant_id)); @@ -1474,6 +1485,23 @@ int ObMultiTenant::update_tenant_decode_resource(const uint64_t tenant_id) return ret; } +int ObMultiTenant::update_tenant_audit_log_config() +{ + int ret = OB_SUCCESS; +#ifdef OB_BUILD_AUDIT_SECURITY + ObAuditLogger *audit_logger = MTL(ObAuditLogger*); + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_ISNULL(audit_logger)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("audit logger should not be null", K(ret)); + } else { + audit_logger->reload_config(); + } +#endif + return ret; +} + int ObMultiTenant::get_tenant_unit(const uint64_t tenant_id, ObUnitInfoGetter::ObTenantConfig &unit) { int ret = OB_SUCCESS; diff --git a/src/observer/omt/ob_multi_tenant.h b/src/observer/omt/ob_multi_tenant.h index 83116cb28..a152d2d56 100644 --- a/src/observer/omt/ob_multi_tenant.h +++ b/src/observer/omt/ob_multi_tenant.h @@ -119,6 +119,7 @@ public: int update_tenant_dag_scheduler_config(); int update_tenant_ddl_config(); int update_checkpoint_diagnose_config(); + int update_tenant_audit_log_config(); int get_tenant(const uint64_t tenant_id, ObTenant *&tenant) const; int get_tenant_with_tenant_lock(const uint64_t tenant_id, common::ObLDHandle &handle, ObTenant *&tenant) const; int get_active_tenant_with_tenant_lock(const uint64_t tenant_id, common::ObLDHandle &handle, ObTenant *&tenant) const; diff --git a/src/observer/virtual_table/ob_information_user_privileges_table.cpp b/src/observer/virtual_table/ob_information_user_privileges_table.cpp index aead47bc9..011355091 100644 --- a/src/observer/virtual_table/ob_information_user_privileges_table.cpp +++ b/src/observer/virtual_table/ob_information_user_privileges_table.cpp @@ -72,6 +72,12 @@ ObInfoSchemaUserPrivilegesTable::StaticInit::StaticInit() "SHUTDOWN"; ObInfoSchemaUserPrivilegesTable::priv_type_strs[OB_PRIV_RELOAD_SHIFT] = "RELOAD"; + ObInfoSchemaUserPrivilegesTable::priv_type_strs[OB_PRIV_REFERENCES_SHIFT] = + "REFERENCES"; + ObInfoSchemaUserPrivilegesTable::priv_type_strs[OB_PRIV_CREATE_ROLE_SHIFT] = + "CREATE ROLE"; + ObInfoSchemaUserPrivilegesTable::priv_type_strs[OB_PRIV_DROP_ROLE_SHIFT] = + "DROP ROLE"; } ObInfoSchemaUserPrivilegesTable::ObInfoSchemaUserPrivilegesTable() diff --git a/src/observer/virtual_table/ob_mysql_db_table.cpp b/src/observer/virtual_table/ob_mysql_db_table.cpp index f4260afa7..4cf0bd084 100644 --- a/src/observer/virtual_table/ob_mysql_db_table.cpp +++ b/src/observer/virtual_table/ob_mysql_db_table.cpp @@ -114,7 +114,7 @@ int ObMySQLDBTable::inner_get_next_row(common::ObNewRow *&row) EXIST_PRIV_CASE(CREATE); EXIST_PRIV_CASE(DROP); EXIST_PRIV_CASE(GRANT); - NO_EXIST_PRIV_CASE(REFERENCES); + EXIST_PRIV_CASE(REFERENCES); EXIST_PRIV_CASE(INDEX); EXIST_PRIV_CASE(ALTER); NO_EXIST_PRIV_CASE(CREATE_TMP_TABLE); @@ -125,7 +125,7 @@ int ObMySQLDBTable::inner_get_next_row(common::ObNewRow *&row) EXIST_PRIV_CASE(ALTER_ROUTINE); EXIST_PRIV_CASE(EXECUTE); NO_EXIST_PRIV_CASE(EVENT); - NO_EXIST_PRIV_CASE(TRIGGER); + EXIST_PRIV_CASE(TRIGGER); #undef EXIST_PRIV_CASE #undef NO_EXIST_PRIV_CASE diff --git a/src/observer/virtual_table/ob_mysql_user_table.cpp b/src/observer/virtual_table/ob_mysql_user_table.cpp index 10cd93091..138f1b028 100644 --- a/src/observer/virtual_table/ob_mysql_user_table.cpp +++ b/src/observer/virtual_table/ob_mysql_user_table.cpp @@ -128,7 +128,7 @@ int ObMySQLUserTable::inner_get_next_row(common::ObNewRow *&row) EXIST_PRIV_CASE(PROCESS); EXIST_PRIV_CASE(FILE); EXIST_PRIV_CASE(GRANT); - NO_EXIST_PRIV_CASE(REFERENCES); + EXIST_PRIV_CASE(REFERENCES); EXIST_PRIV_CASE(INDEX); EXIST_PRIV_CASE(ALTER); EXIST_PRIV_CASE(SHOW_DB); @@ -146,10 +146,10 @@ int ObMySQLUserTable::inner_get_next_row(common::ObNewRow *&row) EXIST_PRIV_CASE(ALTER_ROUTINE); EXIST_PRIV_CASE(CREATE_USER); NO_EXIST_PRIV_CASE(EVENT); - NO_EXIST_PRIV_CASE(TRIGGER); + EXIST_PRIV_CASE(TRIGGER); EXIST_PRIV_CASE(CREATE_TABLESPACE); - NO_EXIST_PRIV_CASE(CREATE_ROLE); - NO_EXIST_PRIV_CASE(DROP_ROLE); + EXIST_PRIV_CASE(CREATE_ROLE); + EXIST_PRIV_CASE(DROP_ROLE); COLUMN_SET_WITH_TYPE(SSL_TYPE, varchar, ssl_type_str); COLUMN_SET_WITH_TYPE(SSL_CIPHER, varchar, user_info->get_ssl_cipher()); COLUMN_SET_WITH_TYPE(X509_ISSUER, varchar, user_info->get_x509_issuer()); diff --git a/src/observer/virtual_table/ob_show_grants.cpp b/src/observer/virtual_table/ob_show_grants.cpp index 3ebc22468..5241f509f 100644 --- a/src/observer/virtual_table/ob_show_grants.cpp +++ b/src/observer/virtual_table/ob_show_grants.cpp @@ -895,6 +895,15 @@ int ObShowGrants::print_privs_to_buff( if ((priv_set & OB_PRIV_RELOAD) && OB_SUCCESS == ret) { ret = BUF_PRINTF(" RELOAD,"); } + if ((priv_set & OB_PRIV_CREATE_ROLE) && OB_SUCCESS == ret) { + ret = BUF_PRINTF(" CREATE ROLE,"); + } + if ((priv_set & OB_PRIV_DROP_ROLE) && OB_SUCCESS == ret) { + ret = BUF_PRINTF(" DROP ROLE,"); + } + if ((priv_set & OB_PRIV_TRIGGER) && OB_SUCCESS == ret) { + ret = BUF_PRINTF(" TRIGGER,"); + } if (OB_SUCCESS == ret && pos > 0) { pos--; //Delete last ',' } diff --git a/src/pl/ob_pl.cpp b/src/pl/ob_pl.cpp index 865730be5..ba2c76cd2 100644 --- a/src/pl/ob_pl.cpp +++ b/src/pl/ob_pl.cpp @@ -2233,7 +2233,8 @@ int ObPL::execute(ObExecContext &ctx, db_name = db_schema->get_database_name_str(); } } - if (OB_SUCC(ret) && !ObUDTObjectType::is_object_id(package_id)) { + if (OB_SUCC(ret) && !ObUDTObjectType::is_object_id(package_id) && !is_valid_id(dblink_id) + && !ObTriggerInfo::is_trigger_package_id(package_id)) { OZ (check_exec_priv(ctx, db_name, routine)); } } @@ -3879,6 +3880,7 @@ int ObPL::check_exec_priv( uint64_t func_id = OB_INVALID_ID; uint64_t db_id = OB_INVALID_ID; uint64_t tenant_id = OB_INVALID_ID; + bool need_check = false; CK (OB_NOT_NULL(routine)); OX (pkg_id = routine->get_package_id()); @@ -3948,6 +3950,38 @@ int ObPL::check_exec_priv( } } } + // add check trigger priv + if (OB_SUCC(ret) && lib::is_mysql_mode() && ObTriggerInfo::is_trigger_package_id(pkg_id) && + OB_FAIL(exec_ctx.get_my_session()->check_feature_enable( + ObCompatFeatureType::MYSQL_TRIGGER_PRIV_CHECK, need_check))) { + LOG_WARN("failed to check feature enable", K(ret)); + } else if (OB_SUCC(ret) && need_check) { + share::schema::ObSessionPrivInfo session_priv; + const ObTableSchema *table = NULL; + if (OB_FAIL(guard->get_session_priv_info( + exec_ctx.get_my_session()->get_priv_tenant_id(), + exec_ctx.get_my_session()->get_priv_user_id(), + exec_ctx.get_my_session()->get_database_name(), + session_priv))) { + LOG_WARN("fail to get_session_priv_info", K(ret)); + } else if (OB_UNLIKELY(!session_priv.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Session priv is invalid", "tenant_id", session_priv.tenant_id_, + "user_id", session_priv.user_id_, K(ret)); + } else { + ObNeedPriv need_priv; + need_priv.priv_level_ = OB_PRIV_TABLE_LEVEL; + need_priv.db_ = database_name; + need_priv.priv_set_ = OB_PRIV_TRIGGER; + const ObTriggerInfo *trigger_info = NULL; + OZ (guard->get_trigger_info(tenant_id, ObTriggerInfo::get_package_trigger_id(pkg_id), trigger_info)); + CK (OB_NOT_NULL(trigger_info)); + OZ (guard->get_table_schema(tenant_id, trigger_info->get_base_object_id(), table)); + CK (OB_NOT_NULL(table)); + OX (need_priv.table_ = table->get_table_name()); + OZ (guard->check_single_table_priv(session_priv, need_priv)); + } + } return ret; } diff --git a/src/pl/ob_pl_resolver.cpp b/src/pl/ob_pl_resolver.cpp index f0eae06a5..3f80f892c 100644 --- a/src/pl/ob_pl_resolver.cpp +++ b/src/pl/ob_pl_resolver.cpp @@ -5771,7 +5771,8 @@ int ObPLResolver::check_and_record_stmt_type(ObPLFunctionAST &func, case stmt::T_SHOW_CREATE_TABLEGROUP: case stmt::T_SHOW_CREATE_TRIGGER: case stmt::T_SHOW_QUERY_RESPONSE_TIME: - case stmt::T_SHOW_TRIGGERS: { + case stmt::T_SHOW_TRIGGERS: + case stmt::T_SHOW_CREATE_USER: { if (0 == prepare_result.into_exprs_.count()) { if (func.is_function() || in_tg) { ret = OB_ER_SP_NO_RETSET; @@ -9527,14 +9528,15 @@ int ObPLResolver::is_static_relation_expr(const ObRawExpr *expr, bool &is_static OZ (check_static_bool_expr(child, is_static_relation_expr)); } else if (T_OP_AND == expr->get_expr_type() || T_OP_OR == expr->get_expr_type()) { - const ObRawExpr *left = NULL; - const ObRawExpr *right = NULL; - CK (2 == expr->get_param_count()); - CK (OB_NOT_NULL(left = ObRawExprUtils::skip_implicit_cast(expr->get_param_expr(0)))); - CK (OB_NOT_NULL(right = ObRawExprUtils::skip_implicit_cast(expr->get_param_expr(1)))); - OZ (check_static_bool_expr(left, is_static_relation_expr)); - OZ (is_static_relation_expr - ? check_static_bool_expr(right, is_static_relation_expr) : OB_SUCCESS); + bool is_all_static = true; + for (int64_t i = 0; OB_SUCC(ret) && is_all_static && i < expr->get_param_count(); i++) { + const ObRawExpr *child_param = NULL; + CK (OB_NOT_NULL(child_param = ObRawExprUtils::skip_implicit_cast(expr->get_param_expr(i)))); + OZ (check_static_bool_expr(child_param, is_all_static)); + } + if (OB_SUCC(ret)) { + is_static_relation_expr = is_all_static; + } } else if (T_OP_IS == expr->get_expr_type() || T_OP_IS_NOT == expr->get_expr_type()) { const ObRawExpr *child = NULL; diff --git a/src/rootserver/ob_ddl_operator.cpp b/src/rootserver/ob_ddl_operator.cpp index de500493b..5c727bc96 100644 --- a/src/rootserver/ob_ddl_operator.cpp +++ b/src/rootserver/ob_ddl_operator.cpp @@ -1819,7 +1819,9 @@ int ObDDLOperator::create_sequence_in_create_table(ObTableSchema &table_schema, orig_default_value.set_collation_type(ObCharset::get_system_collation()); orig_default_value.set_collation_level(CS_LEVEL_IMPLICIT); orig_default_value.set_param_meta(); - if (OB_FAIL(column_schema.set_cur_default_value(cur_default_value))) { + if (OB_FAIL(column_schema.set_cur_default_value( + cur_default_value, + column_schema.is_default_expr_v2_column()))) { LOG_WARN("set current default value fail", K(ret)); } else if (OB_FAIL(column_schema.set_orig_default_value(orig_default_value))) { LOG_WARN("set origin default value fail", K(ret), K(column_schema)); @@ -1902,7 +1904,9 @@ int ObDDLOperator::create_sequence_in_add_column(const ObTableSchema &table_sche orig_default_value.set_collation_type(ObCharset::get_system_collation()); orig_default_value.set_collation_level(CS_LEVEL_IMPLICIT); orig_default_value.set_param_meta(); - if (OB_FAIL(column_schema.set_cur_default_value(cur_default_value))) { + if (OB_FAIL(column_schema.set_cur_default_value( + cur_default_value, + column_schema.is_default_expr_v2_column()))) { LOG_WARN("set current default value fail", K(ret)); } else if (OB_FAIL(column_schema.set_orig_default_value(orig_default_value))) { LOG_WARN("set origin default value fail", K(ret), K(column_schema)); @@ -6970,7 +6974,8 @@ int ObDDLOperator::drop_db_table_privs( } else if (OB_FAIL(schema_service_.gen_new_schema_version(tenant_id, new_schema_version))) { LOG_WARN("fail to gen new schema_version", K(ret), K(tenant_id)); } else if (OB_FAIL(schema_sql_service->get_priv_sql_service().grant_routine( - routine_priv->get_sort_key(), empty_priv, new_schema_version, &dcl_stmt, trans, 0, false))) { + routine_priv->get_sort_key(), empty_priv, new_schema_version, &dcl_stmt, trans, + 0, false, "", ""))) { LOG_WARN("Delete table privilege failed", K(routine_priv), K(ret)); } } @@ -7694,7 +7699,9 @@ int ObDDLOperator::grant_table( common::ObMySQLTransaction &trans, const share::ObRawObjPrivArray &obj_priv_array, const uint64_t option, - const share::schema::ObObjPrivSortKey &obj_priv_key) + const share::schema::ObObjPrivSortKey &obj_priv_key, + const common::ObString &grantor, + const common::ObString &grantor_host) { int ret = OB_SUCCESS; ObRawObjPrivArray new_obj_priv_array; @@ -7741,8 +7748,8 @@ int ObDDLOperator::grant_table( } OZ (schema_sql_service->get_priv_sql_service().grant_table( table_priv_key, new_priv, new_schema_version, ddl_stmt_str, trans, - new_obj_priv_array, option, obj_priv_key, new_schema_version_ora, true, false), - table_priv_key, ret, false); + new_obj_priv_array, option, obj_priv_key, new_schema_version_ora, true, false, + grantor, grantor_host), table_priv_key, ret, false); } else if (obj_priv_array.count() > 0) { OZ (set_need_flush_ora(schema_guard, obj_priv_key, option, obj_priv_array, new_obj_priv_array)); @@ -7765,7 +7772,9 @@ int ObDDLOperator::grant_routine( const ObPrivSet priv_set, common::ObMySQLTransaction &trans, const uint64_t option, - const bool gen_ddl_stmt) + const bool gen_ddl_stmt, + const common::ObString &grantor, + const common::ObString &grantor_host) { int ret = OB_SUCCESS; ObRawObjPrivArray new_obj_priv_array; @@ -7819,7 +7828,8 @@ int ObDDLOperator::grant_routine( if (OB_FAIL(schema_service_.gen_new_schema_version(tenant_id, new_schema_version))) { LOG_WARN("fail to gen new schema_version", K(ret), K(tenant_id)); } else if (OB_FAIL(schema_sql_service->get_priv_sql_service().grant_routine( - routine_priv_key, new_priv, new_schema_version, &ddl_sql, trans, option, true))) { + routine_priv_key, new_priv, new_schema_version, &ddl_sql, trans, option, true, + grantor, grantor_host))) { LOG_WARN("priv sql service grant routine failed", K(ret)); } } @@ -8174,13 +8184,16 @@ int ObDDLOperator::revoke_table( common::ObMySQLTransaction &trans, const ObObjPrivSortKey &obj_priv_key, const share::ObRawObjPrivArray &obj_priv_array, - const bool revoke_all_ora) + const bool revoke_all_ora, + const common::ObString &grantor, + const common::ObString &grantor_host) { int ret = OB_SUCCESS; const uint64_t tenant_id = table_priv_key.tenant_id_; ObSchemaGetterGuard schema_guard; ObSchemaService *schema_sql_service = schema_service_.get_schema_service(); bool is_oracle_mode = false; + uint64_t compat_version = 0; if (OB_ISNULL(schema_sql_service)) { ret = OB_ERR_SYS; LOG_ERROR("schama service_impl and schema manage must not null", @@ -8192,7 +8205,15 @@ int ObDDLOperator::revoke_table( } else if (OB_FAIL(schema_service_.get_tenant_schema_guard(tenant_id, schema_guard))) { LOG_WARN("failed to get schema guard", K(ret)); } else if (OB_FAIL(ObCompatModeGetter::check_is_oracle_mode_with_tenant_id(tenant_id, is_oracle_mode))) { - LOG_WARN("fail to get compat mode", K(ret)); + LOG_WARN("fail to check is oracle mode", K(ret)); + } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) { + LOG_WARN("fail to get data version", K(ret), K(tenant_id)); + } else if (!ObSQLUtils::is_data_version_ge_424_or_433(compat_version) + && !is_oracle_mode && 0 != (priv_set & OB_PRIV_REFERENCES)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("revoke references not supported when MIN_DATA_VERSION is below DATA_VERSION_4_2_4_0 or DATA_VERSION_4_3_3_0", + K(ret), K(priv_set), K(compat_version)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "revoke references"); } else { ObPrivSet table_priv_set = OB_PRIV_SET_EMPTY; if (OB_FAIL(schema_guard.get_table_priv_set(table_priv_key, table_priv_set))) { @@ -8263,7 +8284,8 @@ int ObDDLOperator::revoke_table( LOG_WARN("fail to gen new schema_version", K(ret), K(tenant_id)); } else if (OB_FAIL(schema_sql_service->get_priv_sql_service().revoke_table( table_priv_key, new_priv, new_schema_version, &ddl_sql, trans, - new_schema_version_ora, obj_priv_key, obj_priv_array, is_all))) { + new_schema_version_ora, obj_priv_key, obj_priv_array, is_all, + grantor, grantor_host))) { LOG_WARN("Failed to revoke table", K(table_priv_key), K(ret)); } else { OZ (revoke_obj_cascade(schema_guard, obj_priv_key.grantee_id_, @@ -8335,7 +8357,9 @@ int ObDDLOperator::revoke_routine( const ObPrivSet priv_set, common::ObMySQLTransaction &trans, bool report_error, - const bool gen_ddl_stmt) + const bool gen_ddl_stmt, + const common::ObString &grantor, + const common::ObString &grantor_host) { int ret = OB_SUCCESS; const uint64_t tenant_id = routine_priv_key.tenant_id_; @@ -8398,7 +8422,7 @@ int ObDDLOperator::revoke_routine( new_schema_version))) { LOG_WARN("fail to gen new schema_version", K(ret), K(tenant_id)); } else if (OB_FAIL(schema_sql_service->get_priv_sql_service().revoke_routine( - routine_priv_key, new_priv, new_schema_version, &ddl_sql, trans))) { + routine_priv_key, new_priv, new_schema_version, &ddl_sql, trans, grantor, grantor_host))) { LOG_WARN("Failed to revoke routine", K(routine_priv_key), K(ret)); } } diff --git a/src/rootserver/ob_ddl_operator.h b/src/rootserver/ob_ddl_operator.h index 28fb755f5..d06cf7e1c 100644 --- a/src/rootserver/ob_ddl_operator.h +++ b/src/rootserver/ob_ddl_operator.h @@ -630,24 +630,32 @@ public: common::ObMySQLTransaction &trans, const share::ObRawObjPrivArray &obj_priv_array, const uint64_t option, - const share::schema::ObObjPrivSortKey &obj_priv_key); + const share::schema::ObObjPrivSortKey &obj_priv_key, + const common::ObString &grantor = "", + const common::ObString &grantor_host = ""); virtual int revoke_table(const share::schema::ObTablePrivSortKey &table_priv_key, const ObPrivSet priv_set, common::ObMySQLTransaction &trans, const share::schema::ObObjPrivSortKey &obj_priv_key, const share::ObRawObjPrivArray &obj_priv_array, - const bool revoke_all_ora); + const bool revoke_all_ora, + const common::ObString &grantor = "", + const common::ObString &grantor_host = ""); virtual int grant_routine(const ObRoutinePrivSortKey &routine_priv_key, const ObPrivSet priv_set, common::ObMySQLTransaction &trans, const uint64_t option, - const bool gen_ddl_stmt = true); + const bool gen_ddl_stmt = true, + const common::ObString &grantor = "", + const common::ObString &grantor_host = ""); virtual int revoke_routine(const ObRoutinePrivSortKey &routine_priv_key, const ObPrivSet priv_set, common::ObMySQLTransaction &trans, const bool report_error = true, - const bool gen_ddl_stmt = true); + const bool gen_ddl_stmt = true, + const common::ObString &grantor = "", + const common::ObString &grantor_host = ""); virtual int grant_column(ObSchemaGetterGuard &schema_guard, const ObColumnPrivSortKey &column_priv_key, const ObPrivSet priv_set, diff --git a/src/rootserver/ob_ddl_service.cpp b/src/rootserver/ob_ddl_service.cpp index ae6f55d6a..6c287d327 100755 --- a/src/rootserver/ob_ddl_service.cpp +++ b/src/rootserver/ob_ddl_service.cpp @@ -8284,7 +8284,9 @@ int ObDDLService::modify_generated_column_default_value(ObColumnSchemaV2 &genera LOG_WARN("print expr definition failed", K(ret)); } else if (FALSE_IT(expr_def.assign_ptr(expr_str_buf, static_cast(pos)))) { } else if (FALSE_IT(default_value.set_varchar(expr_def))) { - } else if (OB_FAIL(generated_column.set_cur_default_value(default_value))) { + } else if (OB_FAIL(generated_column.set_cur_default_value( + default_value, + generated_column.is_default_expr_v2_column()))) { LOG_WARN("set cur default value failed", K(ret)); } else if (OB_FAIL(generated_column.set_orig_default_value(default_value))) { LOG_WARN("set original default value failed", K(ret)); @@ -8347,6 +8349,7 @@ int ObDDLService::modify_generated_column_local_vars(ObColumnSchemaV2 &generated ObExprResType dst_type; dst_type.set_meta(generated_column.get_meta_type()); dst_type.set_accuracy(generated_column.get_accuracy()); + dst_type.set_collation_level(CS_LEVEL_IMPLICIT); ObSQLMode sql_mode = default_session.get_sql_mode(); if (NULL != local_session_var) { share::schema::ObSessionSysVar *sys_var = NULL; @@ -9141,7 +9144,9 @@ int ObDDLService::fill_new_column_attributes( new_column_schema.set_data_precision(alter_column_schema.get_data_precision()); new_column_schema.set_data_scale(alter_column_schema.get_data_scale()); if (!is_oracle_mode() || alter_column_schema.is_set_default_) { - new_column_schema.set_cur_default_value(alter_column_schema.get_cur_default_value()); + new_column_schema.set_cur_default_value( + alter_column_schema.get_cur_default_value(), + alter_column_schema.is_default_expr_v2_column()); } new_column_schema.set_zero_fill(alter_column_schema.is_zero_fill()); new_column_schema.set_is_hidden(alter_column_schema.is_hidden()); @@ -10174,12 +10179,14 @@ int ObDDLService::gen_alter_column_new_table_schema_offline( ObObj default_value; if (alter_column_schema->is_drop_default_) { default_value.set_null(); - new_column_schema.del_column_flag(DEFAULT_EXPR_V2_COLUMN_FLAG); - if (OB_FAIL(new_column_schema.set_cur_default_value(default_value))) { - RS_LOG(WARN, "failed to set current default value"); + if (OB_FAIL(new_column_schema.set_cur_default_value(default_value, false))) { + RS_LOG(WARN, "failed to set current default value", K(ret), K(default_value)); + } else { + new_column_schema.del_column_flag(DEFAULT_EXPR_V2_COLUMN_FLAG); } } else { default_value = alter_column_schema->get_cur_default_value(); + bool is_default_expr_v2 = alter_column_schema->is_default_expr_v2_column(); if (!default_value.is_null() && ob_is_text_tc(new_column_schema.get_data_type())) { ret = OB_INVALID_DEFAULT; LOG_USER_ERROR(OB_INVALID_DEFAULT, new_column_schema.get_column_name_str().length(), @@ -10198,7 +10205,14 @@ int ObDDLService::gen_alter_column_new_table_schema_offline( LOG_USER_ERROR(OB_INVALID_DEFAULT, new_column_schema.get_column_name_str().length(), new_column_schema.get_column_name_str().ptr()); RS_LOG(WARN, "not null column with default value null!", K(ret)); - } else if (OB_FAIL(ObDDLResolver::check_default_value(default_value, + } else if (OB_FAIL(new_column_schema.set_cur_default_value(default_value, + is_default_expr_v2))) { + RS_LOG(WARN, "failed to set current default value", K(ret), K(default_value), K(is_default_expr_v2)); + // The check_default_value function not only verifies the + // default value but also performs type conversion on it. + // Therefore, the cur_default_value from the column_schema + // should be passed when calling this function. + } else if (OB_FAIL(ObDDLResolver::check_default_value(new_column_schema.get_cur_default_value(), tz_info_wrap, nls_formats, &alter_table_arg.local_session_var_, @@ -10210,8 +10224,6 @@ int ObDDLService::gen_alter_column_new_table_schema_offline( !alter_column_schema->is_generated_column(), /* allow_sequence */ &schema_checker))) { LOG_WARN("fail to check default value", KPC(alter_column_schema),K(ret)); - } else if (OB_FAIL(new_column_schema.set_cur_default_value(default_value))) { - RS_LOG(WARN, "failed to set current default value"); } } } @@ -10658,11 +10670,13 @@ int ObDDLService::alter_table_column(const ObTableSchema &origin_table_schema, ObObj default_value; if (alter_column_schema->is_drop_default_) { default_value.set_null(); - new_column_schema.del_column_flag(DEFAULT_EXPR_V2_COLUMN_FLAG); - if (OB_FAIL(new_column_schema.set_cur_default_value(default_value))) { - RS_LOG(WARN, "failed to set current default value"); + if (OB_FAIL(new_column_schema.set_cur_default_value(default_value, false))) { + RS_LOG(WARN, "failed to set current default value", K(ret), K(default_value)); + } else { + new_column_schema.del_column_flag(DEFAULT_EXPR_V2_COLUMN_FLAG); } } else { + bool is_default_expr_v2 = alter_column_schema->is_default_expr_v2_column(); default_value = alter_column_schema->get_cur_default_value(); if (!default_value.is_null() && ob_is_text_tc(new_column_schema.get_data_type())) { ret = OB_INVALID_DEFAULT; @@ -10682,7 +10696,14 @@ int ObDDLService::alter_table_column(const ObTableSchema &origin_table_schema, LOG_USER_ERROR(OB_INVALID_DEFAULT, new_column_schema.get_column_name_str().length(), new_column_schema.get_column_name_str().ptr()); RS_LOG(WARN, "not null column with default value null!", K(ret)); - } else if (OB_FAIL(ObDDLResolver::check_default_value(default_value, + } else if (OB_FAIL(new_column_schema.set_cur_default_value(default_value, + is_default_expr_v2))) { + RS_LOG(WARN, "failed to set current default value", K(ret), K(default_value), K(is_default_expr_v2)); + // The check_default_value function not only verifies the + // default value but also performs type conversion on it. + // Therefore, the cur_default_value from the column_schema + // should be passed when calling this function. + } else if (OB_FAIL(ObDDLResolver::check_default_value(new_column_schema.get_cur_default_value(), tz_info_wrap, nls_formats, &alter_table_arg.local_session_var_, @@ -10694,8 +10715,6 @@ int ObDDLService::alter_table_column(const ObTableSchema &origin_table_schema, !alter_column_schema->is_generated_column(), /* allow_sequence */ &schema_checker))) { LOG_WARN("fail to check default value", K(new_column_schema),K(ret)); - } else if (OB_FAIL(new_column_schema.set_cur_default_value(default_value))) { - RS_LOG(WARN, "failed to set current default value"); } } } @@ -31957,19 +31976,26 @@ int ObDDLService::grant(const ObGrantArg &arg) arg.grantor_id_, user_id); if (OB_FAIL(grant_priv_to_user(arg.tenant_id_, - user_id, - user_name, - host_name, - need_priv, - arg.obj_priv_array_, - arg.option_, - arg.is_inner_, - obj_priv_key, - schema_guard))) { + user_id, + user_name, + host_name, + need_priv, + arg.obj_priv_array_, + arg.option_, + arg.is_inner_, + obj_priv_key, + schema_guard, + arg.grantor_, + arg.grantor_host_))) { LOG_WARN("Grant priv to user failed", K(ret)); } } else if (lib::Worker::CompatMode::MYSQL == compat_mode) { - OZ (grant_table_and_column_mysql(arg, user_id, user_name, host_name, need_priv, schema_guard)); + OZ (grant_table_and_column_mysql(arg, + user_id, + user_name, + host_name, + need_priv, + schema_guard)); } else { OZ (grant_table_and_col_privs_to_user(arg, user_id, user_name, host_name, need_priv, schema_guard)); @@ -32109,7 +32135,9 @@ int ObDDLService::grant_priv_to_user(const uint64_t tenant_id, const uint64_t option, const bool is_from_inner_sql, ObObjPrivSortKey &obj_priv_key, - share::schema::ObSchemaGetterGuard &schema_guard) + share::schema::ObSchemaGetterGuard &schema_guard, + const common::ObString &grantor, + const common::ObString &grantor_host) { int ret = OB_SUCCESS; if (OB_INVALID_ID == tenant_id || OB_INVALID_ID == user_id) { @@ -32154,7 +32182,9 @@ int ObDDLService::grant_priv_to_user(const uint64_t tenant_id, obj_priv_array, option, obj_priv_key, - schema_guard))) { + schema_guard, + grantor, + grantor_host))) { LOG_WARN("Grant table error", K(ret)); } break; @@ -32171,7 +32201,9 @@ int ObDDLService::grant_priv_to_user(const uint64_t tenant_id, need_priv.priv_set_, &ddl_sql, option, - schema_guard))) { + schema_guard, + grantor, + grantor_host))) { LOG_WARN("Grant table error", K(ret)); } break; @@ -32186,11 +32218,11 @@ int ObDDLService::grant_priv_to_user(const uint64_t tenant_id, } int ObDDLService::grant_table_and_column_mysql(const obrpc::ObGrantArg &arg, - uint64_t user_id, - const ObString &user_name, - const ObString &host_name, - const ObNeedPriv &need_priv, - share::schema::ObSchemaGetterGuard &schema_guard) + uint64_t user_id, + const ObString &user_name, + const ObString &host_name, + const ObNeedPriv &need_priv, + share::schema::ObSchemaGetterGuard &schema_guard) { int ret = OB_SUCCESS; const uint64_t tenant_id = arg.tenant_id_; @@ -32224,12 +32256,14 @@ int ObDDLService::grant_table_and_column_mysql(const obrpc::ObGrantArg &arg, LOG_WARN("gen_table_priv sql failed", K(need_priv), K(ret)); } else if (OB_FALSE_IT(ddl_stmt = ddl_stmt_str.string())) { } else if (OB_FAIL(ddl_operator.grant_table(table_key, - arg.priv_set_, - &ddl_stmt, - trans, - obj_priv_array, - option, - obj_key))) { + arg.priv_set_, + &ddl_stmt, + trans, + obj_priv_array, + option, + obj_key, + arg.grantor_, + arg.grantor_host_))) { LOG_WARN("fail to grant table", K(ret)); } else if (OB_FAIL(grant_or_revoke_column_priv_mysql(tenant_id, arg.object_id_, user_id, user_name, host_name, need_priv.db_, @@ -32390,7 +32424,7 @@ int ObDDLService::grant_revoke_user( } else if (OB_INVALID_ID == tenant_id) { ret = OB_INVALID_ARGUMENT; LOG_WARN("Tenant id is invalid", K(ret)); - } else if (ObCompatModeGetter::check_is_oracle_mode_with_tenant_id(tenant_id, is_ora_mode)) { + } else if (OB_FAIL(ObCompatModeGetter::check_is_oracle_mode_with_tenant_id(tenant_id, is_ora_mode))) { LOG_WARN("fail to check is oracle mode", K(ret)); } else { ObDDLSQLTransaction trans(schema_service_); @@ -32414,6 +32448,14 @@ int ObDDLService::grant_revoke_user( ret = OB_NOT_SUPPORTED; LOG_WARN("some column of user info is not empty when MIN_DATA_VERSION is below DATA_VERSION_4_2_3_0 or DATA_VERSION_4_3_2_0", K(ret), K(priv_set)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "grant or revoke create tablespace/shutdown/reload privilege"); + } else if (!ObSQLUtils::is_data_version_ge_424_or_433(compat_version) && !is_ora_mode + && (0 != (priv_set & OB_PRIV_REFERENCES) || + 0 != (priv_set & OB_PRIV_CREATE_ROLE) || + 0 != (priv_set & OB_PRIV_DROP_ROLE) || + 0 != (priv_set & OB_PRIV_TRIGGER))) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("some column of user info is not empty when MIN_DATA_VERSION is below DATA_VERSION_4_2_4_0 or DATA_VERSION_4_3_3_0", K(ret), K(priv_set)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "grant or revoke references/create role/drop role/trigger"); } else if (OB_FAIL(trans.start(sql_proxy_, tenant_id, refreshed_schema_version))) { LOG_WARN("Start transaction failed", KR(ret), K(tenant_id), K(refreshed_schema_version)); } else { @@ -32894,7 +32936,7 @@ int ObDDLService::revoke_database( } else if (!db_key.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("db_key is invalid", K(db_key), K(ret)); - } else if (ObCompatModeGetter::check_is_oracle_mode_with_tenant_id(tenant_id, is_ora_mode)) { + } else if (OB_FAIL(ObCompatModeGetter::check_is_oracle_mode_with_tenant_id(tenant_id, is_ora_mode))) { LOG_WARN("fail to check is oracle mode", K(ret)); } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) { LOG_WARN("fail to get data version", K(ret), K(tenant_id)); @@ -32904,6 +32946,13 @@ int ObDDLService::revoke_database( 0 != (priv_set & OB_PRIV_CREATE_ROUTINE))) { ret = OB_NOT_SUPPORTED; LOG_WARN("some column of user info is not empty when MIN_DATA_VERSION is below DATA_VERSION_4_3_1_0 or DATA_VERSION_4_2_2_0", K(ret), K(priv_set)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "revoke execute/alter routine/create routine privilege"); + } else if (!ObSQLUtils::is_data_version_ge_424_or_433(compat_version) && !is_ora_mode + && (0 != (priv_set & OB_PRIV_REFERENCES) || + 0 != (priv_set & OB_PRIV_TRIGGER))) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("some column of user info is not empty when MIN_DATA_VERSION is below DATA_VERSION_4_2_4_0 or DATA_VERSION_4_3_3_0", K(ret), K(priv_set)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "revoke references/trigger"); } else if (OB_FAIL(get_tenant_schema_guard_with_version_in_inner_table(tenant_id, schema_guard))) { LOG_WARN("fail to get schema guard with version in inner table", K(ret), K(tenant_id)); } else { @@ -33158,7 +33207,9 @@ int ObDDLService::grant_table( const share::ObRawObjPrivArray &obj_priv_array, const uint64_t option, const share::schema::ObObjPrivSortKey &obj_key, - share::schema::ObSchemaGetterGuard &schema_guard) + share::schema::ObSchemaGetterGuard &schema_guard, + const common::ObString &grantor, + const common::ObString &grantor_host) { int ret = OB_SUCCESS; const uint64_t tenant_id = table_key.tenant_id_; @@ -33188,7 +33239,9 @@ int ObDDLService::grant_table( trans, obj_priv_array, option, - obj_key))) { + obj_key, + grantor, + grantor_host))) { LOG_WARN("fail to grant table", K(ret), K(table_key), K(priv_set)); } if (trans.is_started()) { @@ -33216,11 +33269,22 @@ int ObDDLService::revoke_table_and_column_mysql(const obrpc::ObRevokeTableArg& a const uint64_t tenant_id = arg.tenant_id_; ObSchemaGetterGuard schema_guard; int64_t refreshed_schema_version = 0; + uint64_t compat_version = 0; + bool is_ora_mode = false; if (OB_FAIL(check_inner_stat())) { LOG_WARN("variable is not init"); } else if (OB_UNLIKELY(!arg.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("arg is invalid", K(arg), K(ret)); + } else if (OB_FAIL(ObCompatModeGetter::check_is_oracle_mode_with_tenant_id(tenant_id, is_ora_mode))) { + LOG_WARN("fail to check is oracle mode", K(ret)); + } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) { + LOG_WARN("fail to get data version", K(ret), K(tenant_id)); + } else if (!ObSQLUtils::is_data_version_ge_424_or_433(compat_version) && !is_ora_mode + && (0 != (arg.priv_set_ & OB_PRIV_TRIGGER))) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("some column of user info is not empty when MIN_DATA_VERSION is below DATA_VERSION_4_2_4_0 or DATA_VERSION_4_3_3_0", K(ret), K(arg.priv_set_)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "revoke table level trigger privilege"); } else if (OB_FAIL(get_tenant_schema_guard_with_version_in_inner_table(tenant_id, schema_guard))) { LOG_WARN("fail to get schema guard with version in inner table", K(ret), K(tenant_id)); } else { @@ -33241,7 +33305,8 @@ int ObDDLService::revoke_table_and_column_mysql(const obrpc::ObRevokeTableArg& a arg.user_id_); share::ObRawObjPrivArray obj_priv_array; //useless if (priv_set != 0 && OB_FAIL(ddl_operator.revoke_table(table_priv_key, priv_set, trans, - obj_priv_key, obj_priv_array, false))) { + obj_priv_key, obj_priv_array, false, + arg.grantor_, arg.grantor_host_))) { LOG_WARN("fail to revoke table", K(ret), K(table_priv_key), K(priv_set)); } else { ObSEArray, 4> column_names_priv; @@ -33338,7 +33403,6 @@ int ObDDLService::revoke_table( const uint64_t tenant_id = table_key.tenant_id_; int64_t refreshed_schema_version = 0; ObSchemaGetterGuard schema_guard; - if (OB_FAIL(check_inner_stat())) { LOG_WARN("variable is not init"); } else if (!table_key.is_valid()) { @@ -33386,7 +33450,9 @@ int ObDDLService::revoke_table( int ObDDLService::revoke_routine( const share::schema::ObRoutinePrivSortKey &routine_key, - const ObPrivSet priv_set) + const ObPrivSet priv_set, + const common::ObString &grantor, + const common::ObString &grantor_host) { int ret = OB_SUCCESS; const uint64_t tenant_id = routine_key.tenant_id_; @@ -33418,7 +33484,7 @@ int ObDDLService::revoke_routine( LOG_WARN("Start transaction failed", KR(ret), K(tenant_id), K(refreshed_schema_version)); } else { ObDDLOperator ddl_operator(*schema_service_, *sql_proxy_); - if (OB_FAIL(ddl_operator.revoke_routine(routine_key, priv_set, trans))) { + if (OB_FAIL(ddl_operator.revoke_routine(routine_key, priv_set, trans, true, true, grantor, grantor_host))) { LOG_WARN("fail to revoke routine", K(ret), K(routine_key), K(priv_set)); } if (trans.is_started()) { @@ -33447,7 +33513,9 @@ int ObDDLService::grant_routine( const ObPrivSet priv_set, const ObString *ddl_stmt_str, const uint64_t option, - share::schema::ObSchemaGetterGuard &schema_guard) + share::schema::ObSchemaGetterGuard &schema_guard, + const common::ObString &grantor, + const common::ObString &grantor_host) { int ret = OB_SUCCESS; const uint64_t tenant_id = routine_key.tenant_id_; @@ -33479,7 +33547,10 @@ int ObDDLService::grant_routine( if (OB_FAIL(ddl_operator.grant_routine(routine_key, priv_set, trans, - option))) { + option, + true, + grantor, + grantor_host))) { LOG_WARN("fail to grant routine", K(ret), K(routine_key), K(priv_set)); } if (trans.is_started()) { @@ -33942,11 +34013,21 @@ int ObDDLService::create_routine(ObRoutineInfo &routine_info, ObPrivSet priv_set = (OB_PRIV_EXECUTE | OB_PRIV_ALTER_ROUTINE); int64_t option = 0; const bool gen_ddl_stmt = false; - if (OB_FAIL(ddl_operator.grant_routine(routine_key, - priv_set, - trans, - option, - gen_ddl_stmt))) { + const ObUserInfo *user_info = NULL; + if (OB_FAIL(schema_guard.get_user_info(tenant_id, + routine_info.get_owner_id(), + user_info))) { + LOG_WARN("failed to get user info", K(ret)); + } else if (OB_ISNULL(user_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("user info is unexpeced null", K(ret)); + } else if (OB_FAIL(ddl_operator.grant_routine(routine_key, + priv_set, + trans, + option, + gen_ddl_stmt, + user_info->get_user_name_str(), + user_info->get_host_name_str()))) { LOG_WARN("fail to grant routine", K(ret), K(routine_key), K(priv_set)); } } @@ -34838,7 +34919,7 @@ int ObDDLService::drop_trigger(const ObDropTriggerArg &arg) if (!arg.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arg", K(arg), K(ret)); - } else if (ObCompatModeGetter::check_is_oracle_mode_with_tenant_id(tenant_id, is_ora_mode)) { + } else if (OB_FAIL(ObCompatModeGetter::check_is_oracle_mode_with_tenant_id(tenant_id, is_ora_mode))) { LOG_WARN("fail to check is oracle mode", K(ret)); } else if (OB_FAIL(check_inner_stat())) { LOG_WARN("variable is not init"); @@ -37249,31 +37330,13 @@ int ObDDLService::update_oracle_tenant_sys_var( /* * In Oracle mode, we are only compatible with binary mode, so collate can only end with _bin */ - if (CS_TYPE_UTF8MB4_BIN == tenant_schema.get_collation_type() - || CS_TYPE_LATIN1_BIN == tenant_schema.get_collation_type() - || CS_TYPE_GBK_BIN == tenant_schema.get_collation_type() - || CS_TYPE_UTF16_BIN == tenant_schema.get_collation_type() - || CS_TYPE_GB18030_BIN == tenant_schema.get_collation_type() - || CS_TYPE_GB18030_2022_BIN == tenant_schema.get_collation_type()) { + if (ObCharset::is_bin_sort(tenant_schema.get_collation_type())) { VAR_INT_TO_STRING(val_buf, tenant_schema.get_collation_type()); SET_TENANT_VARIABLE(SYS_VAR_CHARACTER_SET_SERVER, val_buf); SET_TENANT_VARIABLE(SYS_VAR_CHARACTER_SET_DATABASE, val_buf); - if (CHARSET_UTF8MB4 == ObCharset::charset_type_by_coll(tenant_schema.get_collation_type())) { - OZ(databuff_printf(val_buf, OB_MAX_SYS_PARAM_VALUE_LENGTH, "%s", "AL32UTF8")); - } else if (CHARSET_GBK == ObCharset::charset_type_by_coll(tenant_schema.get_collation_type())) { - OZ(databuff_printf(val_buf, OB_MAX_SYS_PARAM_VALUE_LENGTH, "%s", "ZHS16GBK")); - } else if (CHARSET_UTF16 == ObCharset::charset_type_by_coll(tenant_schema.get_collation_type())) { - OZ(databuff_printf(val_buf, OB_MAX_SYS_PARAM_VALUE_LENGTH, "%s", "AL16UTF16")); - } else if (CHARSET_GB18030 == - ObCharset::charset_type_by_coll(tenant_schema.get_collation_type())) { - OZ(databuff_printf(val_buf, OB_MAX_SYS_PARAM_VALUE_LENGTH, "%s", "ZHS32GB18030")); - } else if (CHARSET_GB18030_2022 == - ObCharset::charset_type_by_coll(tenant_schema.get_collation_type())) { - OZ(databuff_printf(val_buf, OB_MAX_SYS_PARAM_VALUE_LENGTH, "%s", "ZHS32GB18030_2022")); - } else if (CHARSET_LATIN1 == - ObCharset::charset_type_by_coll(tenant_schema.get_collation_type())) { - OZ(databuff_printf(val_buf, OB_MAX_SYS_PARAM_VALUE_LENGTH, "%s", "WE8MSWIN1252")); - } + ObCharsetType charset_type = ObCharset::charset_type_by_coll(tenant_schema.get_collation_type()); + OZ(databuff_printf(val_buf, OB_MAX_SYS_PARAM_VALUE_LENGTH, "%s", + ObCharset::get_oracle_charset_name_by_charset_type(charset_type))); SET_TENANT_VARIABLE(SYS_VAR_NLS_CHARACTERSET, val_buf); } else { ret = OB_ERR_UNEXPECTED; diff --git a/src/rootserver/ob_ddl_service.h b/src/rootserver/ob_ddl_service.h index 77f99faa8..8281af7b7 100644 --- a/src/rootserver/ob_ddl_service.h +++ b/src/rootserver/ob_ddl_service.h @@ -836,7 +836,9 @@ int check_table_udt_id_is_exist(share::schema::ObSchemaGetterGuard &schema_guard const uint64_t option, const bool is_from_inner_sql, share::schema::ObObjPrivSortKey &obj_priv_key, - share::schema::ObSchemaGetterGuard &schema_guard); + share::schema::ObSchemaGetterGuard &schema_guard, + const common::ObString &grantor = "", + const common::ObString &grantor_host = ""); virtual int grant_table_and_col_privs_to_user( const obrpc::ObGrantArg &arg, uint64_t grantee_id, @@ -866,11 +868,11 @@ int check_table_udt_id_is_exist(share::schema::ObSchemaGetterGuard &schema_guard const bool is_grant); int grant_table_and_column_mysql(const obrpc::ObGrantArg &arg, - uint64_t user_id, - const ObString &user_name, - const ObString &host_name, - const ObNeedPriv &need_priv, - share::schema::ObSchemaGetterGuard &schema_guard); + uint64_t user_id, + const ObString &user_name, + const ObString &host_name, + const ObNeedPriv &need_priv, + share::schema::ObSchemaGetterGuard &schema_guard); int lock_user(const obrpc::ObLockUserArg &arg, common::ObIArray &failed_index); int standby_grant(const obrpc::ObStandbyGrantArg &arg); @@ -909,18 +911,24 @@ int check_table_udt_id_is_exist(share::schema::ObSchemaGetterGuard &schema_guard const share::ObRawObjPrivArray &obj_priv_array, const uint64_t option, const share::schema::ObObjPrivSortKey &obj_key, - share::schema::ObSchemaGetterGuard &schema_guard); + share::schema::ObSchemaGetterGuard &schema_guard, + const common::ObString &grantor = "", + const common::ObString &grantor_host = ""); virtual int grant_routine( const share::schema::ObRoutinePrivSortKey &routine_key, const ObPrivSet priv_set, const ObString *ddl_stmt_str, const uint64_t option, - share::schema::ObSchemaGetterGuard &schema_guard); + share::schema::ObSchemaGetterGuard &schema_guard, + const common::ObString &grantor = "", + const common::ObString &grantor_host = ""); virtual int revoke_routine( const share::schema::ObRoutinePrivSortKey &routine_key, - const ObPrivSet priv_set); + const ObPrivSet priv_set, + const common::ObString &grantor = "", + const common::ObString &grantor_host = ""); virtual int revoke_table(const obrpc::ObRevokeTableArg &arg, const share::schema::ObTablePrivSortKey &table_key, const ObPrivSet priv_set, diff --git a/src/rootserver/ob_ddl_sql_generator.cpp b/src/rootserver/ob_ddl_sql_generator.cpp index 03b0d3632..db8724a75 100644 --- a/src/rootserver/ob_ddl_sql_generator.cpp +++ b/src/rootserver/ob_ddl_sql_generator.cpp @@ -115,6 +115,12 @@ int ObDDLSqlGenerator::get_priv_name(const int64_t priv, const char *&name) name = "SHUTDOWN"; break; case OB_PRIV_RELOAD: name = "RELOAD"; break; + case OB_PRIV_CREATE_ROLE: + name = "CREATE ROLE"; break; + case OB_PRIV_DROP_ROLE: + name = "DROP ROLE"; break; + case OB_PRIV_TRIGGER: + name = "TRIGGER"; break; default: { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid priv", K(ret), K(priv)); diff --git a/src/rootserver/ob_root_service.cpp b/src/rootserver/ob_root_service.cpp index af5ea20b9..ed38e6643 100755 --- a/src/rootserver/ob_root_service.cpp +++ b/src/rootserver/ob_root_service.cpp @@ -6568,7 +6568,7 @@ int ObRootService::revoke_routine(const ObRevokeRoutineArg &arg) (arg.obj_type_ == (int64_t)ObObjectType::PROCEDURE) ? ObRoutineType::ROUTINE_PROCEDURE_TYPE : (arg.obj_type_ == (int64_t)ObObjectType::FUNCTION) ? ObRoutineType::ROUTINE_FUNCTION_TYPE : ObRoutineType::INVALID_ROUTINE_TYPE); - OZ (ddl_service_.revoke_routine(routine_priv_key, arg.priv_set_)); + OZ (ddl_service_.revoke_routine(routine_priv_key, arg.priv_set_, arg.grantor_, arg.grantor_host_)); } return ret; } diff --git a/src/rootserver/parallel_ddl/ob_create_table_helper.cpp b/src/rootserver/parallel_ddl/ob_create_table_helper.cpp index f76bd1d13..c8af98563 100644 --- a/src/rootserver/parallel_ddl/ob_create_table_helper.cpp +++ b/src/rootserver/parallel_ddl/ob_create_table_helper.cpp @@ -1942,7 +1942,9 @@ int ObCreateTableHelper::generate_sequence_object_() orig_default_value.set_collation_type(ObCharset::get_system_collation()); orig_default_value.set_collation_level(CS_LEVEL_IMPLICIT); orig_default_value.set_param_meta(); - if (OB_FAIL(column_schema->set_cur_default_value(cur_default_value))) { + if (OB_FAIL(column_schema->set_cur_default_value( + cur_default_value, + column_schema->is_default_expr_v2_column()))) { LOG_WARN("set current default value fail", KR(ret)); } else if (OB_FAIL(column_schema->set_orig_default_value(orig_default_value))) { LOG_WARN("set origin default value fail", KR(ret), K(column_schema)); diff --git a/src/share/config/ob_config_helper.cpp b/src/share/config/ob_config_helper.cpp index 3a1383f13..febb1877f 100644 --- a/src/share/config/ob_config_helper.cpp +++ b/src/share/config/ob_config_helper.cpp @@ -465,6 +465,47 @@ bool ObConfigAuditTrailChecker::check(const ObConfigItem &t) const return sql::get_audit_trail_type_from_string(tmp_string) != sql::ObAuditTrailType::INVALID ; } +bool ObConfigAuditLogCompressionChecker::check(const ObConfigItem &t) const +{ + common::ObString tmp_string(t.str()); + return 0 == tmp_string.case_compare("NONE") + || 0 == tmp_string.case_compare("ZSTD"); +} + +bool ObConfigAuditLogPathChecker::check(const ObConfigItem &t) const +{ + int ret = OB_SUCCESS; + common::ObString tmp_string(t.str()); + ObBackupDest dest; + if (tmp_string.empty()) { + // do nothing + } else if (OB_FAIL(dest.set(tmp_string))) { + OB_LOG(WARN, "failed to set backup dest", K(ret)); + } + return OB_SUCCESS == ret; +} + +bool ObConfigAuditLogFormatChecker::check(const ObConfigItem &t) const +{ + common::ObString tmp_string(t.str()); + return 0 == tmp_string.case_compare("CSV"); +} + +bool ObConfigAuditLogQuerySQLChecker::check(const ObConfigItem &t) const +{ + common::ObString tmp_string(t.str()); + return 0 == tmp_string.case_compare("ALL") + || 0 == tmp_string.case_compare("NONE"); +} + +bool ObConfigAuditLogStrategyChecker::check(const ObConfigItem &t) const +{ + common::ObString tmp_string(t.str()); + return 0 == tmp_string.case_compare("ASYNCHRONOUS") + || 0 == tmp_string.case_compare("PERFORMANCE") + || 0 == tmp_string.case_compare("SYNCHRONOUS"); +} + bool ObConfigWorkAreaPolicyChecker::check(const ObConfigItem &t) const { const ObString tmp_str(t.str()); diff --git a/src/share/config/ob_config_helper.h b/src/share/config/ob_config_helper.h index aeecef6f0..c35c1a929 100644 --- a/src/share/config/ob_config_helper.h +++ b/src/share/config/ob_config_helper.h @@ -347,6 +347,61 @@ private: DISALLOW_COPY_AND_ASSIGN(ObConfigAuditTrailChecker); }; +class ObConfigAuditLogCompressionChecker + : public ObConfigChecker +{ +public: + ObConfigAuditLogCompressionChecker() {} + virtual ~ObConfigAuditLogCompressionChecker() {} + bool check(const ObConfigItem &t) const; +private: + DISALLOW_COPY_AND_ASSIGN(ObConfigAuditLogCompressionChecker); +}; + +class ObConfigAuditLogPathChecker + : public ObConfigChecker +{ +public: + ObConfigAuditLogPathChecker() {} + virtual ~ObConfigAuditLogPathChecker() {} + bool check(const ObConfigItem &t) const; +private: + DISALLOW_COPY_AND_ASSIGN(ObConfigAuditLogPathChecker); +}; + +class ObConfigAuditLogFormatChecker + : public ObConfigChecker +{ +public: + ObConfigAuditLogFormatChecker() {} + virtual ~ObConfigAuditLogFormatChecker() {} + bool check(const ObConfigItem &t) const; +private: + DISALLOW_COPY_AND_ASSIGN(ObConfigAuditLogFormatChecker); +}; + +class ObConfigAuditLogQuerySQLChecker + : public ObConfigChecker +{ +public: + ObConfigAuditLogQuerySQLChecker() {} + virtual ~ObConfigAuditLogQuerySQLChecker() {} + bool check(const ObConfigItem &t) const; +private: + DISALLOW_COPY_AND_ASSIGN(ObConfigAuditLogQuerySQLChecker); +}; + +class ObConfigAuditLogStrategyChecker + : public ObConfigChecker +{ +public: + ObConfigAuditLogStrategyChecker() {} + virtual ~ObConfigAuditLogStrategyChecker() {} + bool check(const ObConfigItem &t) const; +private: + DISALLOW_COPY_AND_ASSIGN(ObConfigAuditLogStrategyChecker); +}; + class ObConfigWorkAreaPolicyChecker : public ObConfigChecker { diff --git a/src/share/datum/ob_datum_cmp_func_def.h b/src/share/datum/ob_datum_cmp_func_def.h index 2d9d3e870..e10d2bb4f 100644 --- a/src/share/datum/ob_datum_cmp_func_def.h +++ b/src/share/datum/ob_datum_cmp_func_def.h @@ -456,7 +456,15 @@ typedef ObConstIntMapping<0, CS_TYPE_GB18030_2022_RADICAL_CI, 1, CS_TYPE_GB18030_2022_RADICAL_CS, 1, CS_TYPE_GB18030_2022_STROKE_CI, 1, - CS_TYPE_GB18030_2022_STROKE_CS, 1 > SupportedCollections; + CS_TYPE_GB18030_2022_STROKE_CS, 1, + CS_TYPE_UTF8MB4_CROATIAN_CI, 1, + CS_TYPE_UTF8MB4_UNICODE_520_CI, 1, + CS_TYPE_UTF8MB4_CZECH_CI, 1, + CS_TYPE_ASCII_GENERAL_CI,1, + CS_TYPE_ASCII_BIN,1, + CS_TYPE_TIS620_THAI_CI,1, + CS_TYPE_TIS620_BIN,1, + CS_TYPE_UTF8MB4_0900_AI_CI, 1> SupportedCollections; // bool is_calc_with_end_space(ObObjType type1, ObObjType type2, // bool is_oracle_mode, diff --git a/src/share/inner_table/ob_inner_table_schema.12451_12500.cpp b/src/share/inner_table/ob_inner_table_schema.12451_12500.cpp index 2a17c036a..926b750eb 100644 --- a/src/share/inner_table/ob_inner_table_schema.12451_12500.cpp +++ b/src/share/inner_table/ob_inner_table_schema.12451_12500.cpp @@ -1534,6 +1534,293 @@ int ObInnerTableSchema::all_virtual_index_usage_info_schema(ObTableSchema &table return ret; } +int ObInnerTableSchema::all_virtual_audit_log_filter_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_AUDIT_LOG_FILTER_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(VIRTUAL_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_AUDIT_LOG_FILTER_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("filter_name", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + MAX_AUDIT_FILTER_NAME_LENGTH_BYTE, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("gmt_create", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("gmt_modified", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("definition", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObLongTextType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("is_deleted", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::all_virtual_audit_log_user_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_AUDIT_LOG_USER_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(VIRTUAL_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_AUDIT_LOG_USER_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("user_name", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + MAX_AUDIT_USER_NAME_LENGTH_BYTE, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("host", //column_name + ++column_id, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_HOST_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("gmt_create", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("gmt_modified", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("filter_name", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + MAX_AUDIT_FILTER_NAME_LENGTH_BYTE, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("is_deleted", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::all_virtual_column_privilege_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.20001_20050.cpp b/src/share/inner_table/ob_inner_table_schema.20001_20050.cpp index 60042ba5b..cd374978c 100644 --- a/src/share/inner_table/ob_inner_table_schema.20001_20050.cpp +++ b/src/share/inner_table/ob_inner_table_schema.20001_20050.cpp @@ -410,7 +410,7 @@ int ObInnerTableSchema::tables_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( select /*+ leading(a) no_use_nl(ts)*/ cast('def' as char(512)) as TABLE_CATALOG, cast(b.database_name as char(64)) collate utf8mb4_name_case as TABLE_SCHEMA, cast(a.table_name as char(64)) collate utf8mb4_name_case as TABLE_NAME, cast(case when (a.database_id = 201002 or a.table_type = 1) then 'SYSTEM VIEW' when a.table_type in (0, 2) then 'SYSTEM TABLE' when a.table_type = 4 then 'VIEW' when a.table_type = 14 then 'EXTERNAL TABLE' else 'BASE TABLE' end as char(64)) as TABLE_TYPE, cast(case when a.table_type in (0,3,5,6,7,11,12,13,15) then 'InnoDB' else 'MEMORY' end as char(64)) as ENGINE, cast(NULL as unsigned) as VERSION, cast(a.store_format as char(10)) as ROW_FORMAT, cast( coalesce(ts.row_cnt,0) as unsigned) as TABLE_ROWS, cast( coalesce(ts.avg_row_len,0) as unsigned) as AVG_ROW_LENGTH, cast( coalesce(ts.data_size,0) as unsigned) as DATA_LENGTH, cast(NULL as unsigned) as MAX_DATA_LENGTH, cast(NULL as unsigned) as INDEX_LENGTH, cast(NULL as unsigned) as DATA_FREE, cast(NULL as unsigned) as AUTO_INCREMENT, cast(a.gmt_create as datetime) as CREATE_TIME, cast(a.gmt_modified as datetime) as UPDATE_TIME, cast(NULL as datetime) as CHECK_TIME, cast(d.collation as char(32)) as TABLE_COLLATION, cast(NULL as unsigned) as CHECKSUM, cast(NULL as char(255)) as CREATE_OPTIONS, cast(case when a.table_type = 4 then 'VIEW' else a.comment end as char(2048)) as TABLE_COMMENT from ( select cast(0 as signed) as tenant_id, c.database_id, c.table_id, c.table_name, c.collation_type, c.table_type, usec_to_time(d.schema_version) as gmt_create, usec_to_time(c.schema_version) as gmt_modified, c.comment, c.store_format from oceanbase.__all_virtual_core_all_table c join oceanbase.__all_virtual_core_all_table d on c.tenant_id = d.tenant_id and d.table_name = '__all_core_table' where c.tenant_id = effective_tenant_id() union all select tenant_id, database_id, table_id, table_name, collation_type, table_type, gmt_create, gmt_modified, comment, store_format from oceanbase.__all_table where table_mode >> 12 & 15 in (0,1)) a join oceanbase.__all_database b on a.database_id = b.database_id and a.tenant_id = b.tenant_id join oceanbase.__tenant_virtual_collation d on a.collation_type = d.collation_type left join ( select tenant_id, table_id, row_cnt, avg_row_len, (macro_blk_cnt * 2 * 1024 * 1024) as data_size from oceanbase.__all_table_stat where partition_id = -1 or partition_id = table_id) ts on a.table_id = ts.table_id and a.tenant_id = ts.tenant_id where a.tenant_id = 0 and a.table_type in (0, 1, 2, 3, 4, 14, 15) and b.database_name != '__recyclebin' and b.in_recyclebin = 0 and 0 = sys_privilege_check('table_acc', effective_tenant_id(), b.database_name, a.table_name) )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( select /*+ leading(a) no_use_nl(ts)*/ cast('def' as char(512)) as TABLE_CATALOG, cast(b.database_name as char(64)) collate utf8mb4_name_case as TABLE_SCHEMA, cast(a.table_name as char(64)) collate utf8mb4_name_case as TABLE_NAME, cast(case when (a.database_id = 201002 or a.table_type = 1) then 'SYSTEM VIEW' when a.table_type in (0, 2) then 'SYSTEM TABLE' when a.table_type = 4 then 'VIEW' when a.table_type = 14 then 'EXTERNAL TABLE' else 'BASE TABLE' end as char(64)) as TABLE_TYPE, cast(case when a.table_type in (0,3,5,6,7,11,12,13,15) then 'InnoDB' else 'MEMORY' end as char(64)) as ENGINE, cast(NULL as unsigned) as VERSION, cast(a.store_format as char(10)) as ROW_FORMAT, cast( coalesce(ts.row_cnt,0) as unsigned) as TABLE_ROWS, cast( coalesce(ts.avg_row_len,0) as unsigned) as AVG_ROW_LENGTH, cast( coalesce(ts.data_size,0) as unsigned) as DATA_LENGTH, cast(NULL as unsigned) as MAX_DATA_LENGTH, cast( coalesce(idx_stat.index_length, 0) as unsigned) as INDEX_LENGTH, cast(NULL as unsigned) as DATA_FREE, cast(NULL as unsigned) as AUTO_INCREMENT, cast(a.gmt_create as datetime) as CREATE_TIME, cast(a.gmt_modified as datetime) as UPDATE_TIME, cast(NULL as datetime) as CHECK_TIME, cast(d.collation as char(32)) as TABLE_COLLATION, cast(NULL as unsigned) as CHECKSUM, cast(NULL as char(255)) as CREATE_OPTIONS, cast(case when a.table_type = 4 then 'VIEW' else a.comment end as char(2048)) as TABLE_COMMENT from ( select cast(0 as signed) as tenant_id, c.database_id, c.table_id, c.table_name, c.collation_type, c.table_type, usec_to_time(d.schema_version) as gmt_create, usec_to_time(c.schema_version) as gmt_modified, c.comment, c.store_format from oceanbase.__all_virtual_core_all_table c join oceanbase.__all_virtual_core_all_table d on c.tenant_id = d.tenant_id and d.table_name = '__all_core_table' where c.tenant_id = effective_tenant_id() union all select tenant_id, database_id, table_id, table_name, collation_type, table_type, gmt_create, gmt_modified, comment, store_format from oceanbase.__all_table where table_mode >> 12 & 15 in (0,1)) a join oceanbase.__all_database b on a.database_id = b.database_id and a.tenant_id = b.tenant_id join oceanbase.__tenant_virtual_collation d on a.collation_type = d.collation_type left join ( select tenant_id, table_id, row_cnt, avg_row_len, (macro_blk_cnt * 2 * 1024 * 1024) as data_size from oceanbase.__all_table_stat where partition_id = -1 or partition_id = table_id) ts on a.table_id = ts.table_id and a.tenant_id = ts.tenant_id left join ( select e.tenant_id as tenant_id, e.data_table_id as data_table_id, SUM(f.macro_blk_cnt * 2 * 1024 * 1024) AS index_length FROM oceanbase.__all_table e JOIN oceanbase.__all_table_stat f ON e.tenant_id = f.tenant_id and e.table_id = f.table_id and (f.partition_id = -1 or f.partition_id = e.table_id) WHERE e.index_type in (1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12) and e.table_type = 5 group by tenant_id, data_table_id ) idx_stat on idx_stat.tenant_id = a.tenant_id and idx_stat.data_table_id = a.table_id where a.tenant_id = 0 and a.table_type in (0, 1, 2, 3, 4, 14, 15) and b.database_name != '__recyclebin' and b.in_recyclebin = 0 and 0 = sys_privilege_check('table_acc', effective_tenant_id(), b.database_name, a.table_name) )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -710,7 +710,7 @@ int ObInnerTableSchema::routines_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__(select CAST(mp.specific_name AS CHAR(64)) AS SPECIFIC_NAME, CAST('def' AS CHAR(512)) as ROUTINE_CATALOG, CAST(mp.db AS CHAR(64)) collate utf8mb4_name_case as ROUTINE_SCHEMA, CAST(mp.name AS CHAR(64)) as ROUTINE_NAME, CAST(mp.type AS CHAR(9)) as ROUTINE_TYPE, CAST(lower(v.data_type_str) AS CHAR(64)) AS DATA_TYPE, CAST( CASE WHEN mp.type = 'FUNCTION' THEN CASE WHEN rp.param_type IN (22, 23, 27, 28, 29, 30) THEN rp.param_length ELSE NULL END ELSE NULL END AS SIGNED ) as CHARACTER_MAXIMUM_LENGTH, CASE WHEN rp.param_type IN (22, 23, 27, 28, 29, 30, 43, 44, 46) THEN CAST( rp.param_length * CASE rp.param_coll_type WHEN 63 THEN 1 WHEN 249 THEN 4 WHEN 248 THEN 4 WHEN 87 THEN 2 WHEN 28 THEN 2 WHEN 55 THEN 4 WHEN 54 THEN 4 WHEN 101 THEN 2 WHEN 46 THEN 4 WHEN 45 THEN 4 WHEN 224 THEN 4 ELSE 1 END AS SIGNED ) ELSE CAST(NULL AS SIGNED) END AS CHARACTER_OCTET_LENGTH, CASE WHEN rp.param_type IN (1, 2, 3, 4, 5, 15, 16, 50) THEN CAST(rp.param_precision AS UNSIGNED) ELSE CAST(NULL AS UNSIGNED) END AS NUMERIC_PRECISION, CASE WHEN rp.param_type IN (15, 16, 50) THEN CAST(rp.param_scale AS SIGNED) WHEN rp.param_type IN (1, 2, 3, 4, 5, 11, 12, 13, 14) THEN CAST(0 AS SIGNED) ELSE CAST(NULL AS SIGNED) END AS NUMERIC_SCALE, CASE WHEN rp.param_type IN (17, 18, 20) THEN CAST(rp.param_scale AS UNSIGNED) ELSE CAST(NULL AS UNSIGNED) END AS DATETIME_PRECISION, CAST( CASE rp.param_charset WHEN 1 THEN 'binary' WHEN 2 THEN 'utf8mb4' WHEN 3 THEN 'gbk' WHEN 4 THEN 'utf16' WHEN 5 THEN 'gb18030' WHEN 6 THEN 'latin1' WHEN 7 THEN 'gb18030_2022' ELSE NULL END AS CHAR(64) ) AS CHARACTER_SET_NAME, CAST( CASE rp.param_coll_type WHEN 45 THEN 'utf8mb4_general_ci' WHEN 46 THEN 'utf8mb4_bin' WHEN 63 THEN 'binary' ELSE NULL END AS CHAR(64) ) AS COLLATION_NAME, CAST( CASE WHEN rp.param_type IN (1, 2, 3, 4, 5) THEN CONCAT( lower(v.data_type_str), '(', rp.param_precision, ')' ) WHEN rp.param_type IN (15, 16, 50) THEN CONCAT( lower(v.data_type_str), '(', rp.param_precision, ',', rp.param_scale, ')' ) WHEN rp.param_type IN (18, 20) THEN CONCAT(lower(v.data_type_str), '(', rp.param_scale, ')') WHEN rp.param_type IN (22, 23) and rp.param_length > 0 THEN CONCAT(lower(v.data_type_str), '(', rp.param_length, ')') ELSE lower(v.data_type_str) END AS CHAR(4194304) ) AS DTD_IDENTIFIER, CAST('SQL' AS CHAR(8)) as ROUTINE_BODY, CAST(mp.body AS CHAR(4194304)) as ROUTINE_DEFINITION, CAST(NULL AS CHAR(64)) as EXTERNAL_NAME, CAST(NULL AS CHAR(64)) as EXTERNAL_LANGUAGE, CAST('SQL' AS CHAR(8)) as PARAMETER_STYLE, CAST(mp.IS_DETERMINISTIC AS CHAR(3)) AS IS_DETERMINISTIC, CAST(mp.SQL_DATA_ACCESS AS CHAR(64)) AS SQL_DATA_ACCESS, CAST(NULL AS CHAR(64)) as SQL_PATH, CAST(mp.SECURITY_TYPE AS CHAR(7)) as SECURITY_TYPE, CAST(r.gmt_create AS datetime) as CREATED, CAST(r.gmt_modified AS datetime) as LAST_ALTERED, CAST(mp.SQL_MODE AS CHAR(8192)) as SQL_MODE, CAST(mp.comment AS CHAR(4194304)) as ROUTINE_COMMENT, CAST(mp.DEFINER AS CHAR(93)) as DEFINER, CAST(mp.CHARACTER_SET_CLIENT AS CHAR(32)) as CHARACTER_SET_CLIENT, CAST(mp.COLLATION_CONNECTION AS CHAR(32)) as COLLATION_CONNECTION, CAST(mp.db_collation AS CHAR(32)) as DATABASE_COLLATION from mysql.proc as mp join oceanbase.__all_database a on mp.DB = a.DATABASE_NAME and a.in_recyclebin = 0 join oceanbase.__all_routine as r on mp.specific_name = r.routine_name and r.DATABASE_ID = a.DATABASE_ID and CAST( CASE r.routine_type WHEN 1 THEN 'PROCEDURE' WHEN 2 THEN 'FUNCTION' ELSE NULL END AS CHAR(9) ) = mp.type left join oceanbase.__all_routine_param as rp on rp.subprogram_id = r.subprogram_id and rp.tenant_id = r.tenant_id and rp.routine_id = r.routine_id and rp.param_position = 0 left join oceanbase.__all_virtual_data_type v on rp.param_type = v.data_type )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__(select CAST(mp.specific_name AS CHAR(64)) AS SPECIFIC_NAME, CAST('def' AS CHAR(512)) as ROUTINE_CATALOG, CAST(mp.db AS CHAR(64)) collate utf8mb4_name_case as ROUTINE_SCHEMA, CAST(mp.name AS CHAR(64)) as ROUTINE_NAME, CAST(mp.type AS CHAR(9)) as ROUTINE_TYPE, CAST(lower(v.data_type_str) AS CHAR(64)) AS DATA_TYPE, CAST( CASE WHEN mp.type = 'FUNCTION' THEN CASE WHEN rp.param_type IN (22, 23, 27, 28, 29, 30) THEN rp.param_length ELSE NULL END ELSE NULL END AS SIGNED ) as CHARACTER_MAXIMUM_LENGTH, CASE WHEN rp.param_type IN (22, 23, 27, 28, 29, 30, 43, 44, 46) THEN CAST( rp.param_length * CASE rp.param_coll_type WHEN 63 THEN 1 WHEN 249 THEN 4 WHEN 248 THEN 4 WHEN 87 THEN 2 WHEN 28 THEN 2 WHEN 55 THEN 4 WHEN 54 THEN 4 WHEN 101 THEN 2 WHEN 46 THEN 4 WHEN 45 THEN 4 WHEN 224 THEN 4 ELSE 1 END AS SIGNED ) ELSE CAST(NULL AS SIGNED) END AS CHARACTER_OCTET_LENGTH, CASE WHEN rp.param_type IN (1, 2, 3, 4, 5, 15, 16, 50) THEN CAST(rp.param_precision AS UNSIGNED) ELSE CAST(NULL AS UNSIGNED) END AS NUMERIC_PRECISION, CASE WHEN rp.param_type IN (15, 16, 50) THEN CAST(rp.param_scale AS SIGNED) WHEN rp.param_type IN (1, 2, 3, 4, 5, 11, 12, 13, 14) THEN CAST(0 AS SIGNED) ELSE CAST(NULL AS SIGNED) END AS NUMERIC_SCALE, CASE WHEN rp.param_type IN (17, 18, 20) THEN CAST(rp.param_scale AS UNSIGNED) ELSE CAST(NULL AS UNSIGNED) END AS DATETIME_PRECISION, CAST( CASE rp.param_charset WHEN 1 THEN 'binary' WHEN 2 THEN 'utf8mb4' WHEN 3 THEN 'gbk' WHEN 4 THEN 'utf16' WHEN 5 THEN 'gb18030' WHEN 6 THEN 'latin1' WHEN 7 THEN 'gb18030_2022' WHEN 8 THEN 'ascii' WHEN 9 THEN 'tis620' ELSE NULL END AS CHAR(64) ) AS CHARACTER_SET_NAME, CAST( CASE rp.param_coll_type WHEN 45 THEN 'utf8mb4_general_ci' WHEN 46 THEN 'utf8mb4_bin' WHEN 63 THEN 'binary' ELSE NULL END AS CHAR(64) ) AS COLLATION_NAME, CAST( CASE WHEN rp.param_type IN (1, 2, 3, 4, 5) THEN CONCAT( lower(v.data_type_str), '(', rp.param_precision, ')' ) WHEN rp.param_type IN (15, 16, 50) THEN CONCAT( lower(v.data_type_str), '(', rp.param_precision, ',', rp.param_scale, ')' ) WHEN rp.param_type IN (18, 20) THEN CONCAT(lower(v.data_type_str), '(', rp.param_scale, ')') WHEN rp.param_type IN (22, 23) and rp.param_length > 0 THEN CONCAT(lower(v.data_type_str), '(', rp.param_length, ')') ELSE lower(v.data_type_str) END AS CHAR(4194304) ) AS DTD_IDENTIFIER, CAST('SQL' AS CHAR(8)) as ROUTINE_BODY, CAST(mp.body AS CHAR(4194304)) as ROUTINE_DEFINITION, CAST(NULL AS CHAR(64)) as EXTERNAL_NAME, CAST(NULL AS CHAR(64)) as EXTERNAL_LANGUAGE, CAST('SQL' AS CHAR(8)) as PARAMETER_STYLE, CAST(mp.IS_DETERMINISTIC AS CHAR(3)) AS IS_DETERMINISTIC, CAST(mp.SQL_DATA_ACCESS AS CHAR(64)) AS SQL_DATA_ACCESS, CAST(NULL AS CHAR(64)) as SQL_PATH, CAST(mp.SECURITY_TYPE AS CHAR(7)) as SECURITY_TYPE, CAST(r.gmt_create AS datetime) as CREATED, CAST(r.gmt_modified AS datetime) as LAST_ALTERED, CAST(mp.SQL_MODE AS CHAR(8192)) as SQL_MODE, CAST(mp.comment AS CHAR(4194304)) as ROUTINE_COMMENT, CAST(mp.DEFINER AS CHAR(93)) as DEFINER, CAST(mp.CHARACTER_SET_CLIENT AS CHAR(32)) as CHARACTER_SET_CLIENT, CAST(mp.COLLATION_CONNECTION AS CHAR(32)) as COLLATION_CONNECTION, CAST(mp.db_collation AS CHAR(32)) as DATABASE_COLLATION from mysql.proc as mp join oceanbase.__all_database a on mp.DB = a.DATABASE_NAME and a.in_recyclebin = 0 join oceanbase.__all_routine as r on mp.specific_name = r.routine_name and r.DATABASE_ID = a.DATABASE_ID and CAST( CASE r.routine_type WHEN 1 THEN 'PROCEDURE' WHEN 2 THEN 'FUNCTION' ELSE NULL END AS CHAR(9) ) = mp.type left join oceanbase.__all_routine_param as rp on rp.subprogram_id = r.subprogram_id and rp.tenant_id = r.tenant_id and rp.routine_id = r.routine_id and rp.param_position = 0 left join oceanbase.__all_virtual_data_type v on rp.param_type = v.data_type )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -775,6 +775,156 @@ int ObInnerTableSchema::profiling_schema(ObTableSchema &table_schema) return ret; } +int ObInnerTableSchema::optimizer_trace_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_INFORMATION_SCHEMA_ID); + table_schema.set_table_id(OB_OPTIMIZER_TRACE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_OPTIMIZER_TRACE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST('query' as CHAR(200)) as QUERY, CAST('trace' as CHAR(200)) as TRACE, CAST(00000000000000000000 as SIGNED) as MISSING_BYTES_MAX_MEM_SIZE, CAST(0 as SIGNED) as INSUFFICIENT_PRIVILEGES FROM DUAL limit 0; )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::plugins_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_INFORMATION_SCHEMA_ID); + table_schema.set_table_id(OB_PLUGINS_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_PLUGINS_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST('plugin name' as CHAR(64)) as PLUGIN_NAME, CAST('version' as CHAR(20)) as PLUGIN, CAST('plugin status' as CHAR(10)) as PLUGIN_STATUS, CAST('type' as CHAR(80)) as PLUGIN_TYPE, CAST('version' as CHAR(20)) as PLUGIN_TYPE_VERSION, CAST('library' as CHAR(64)) as PLUGIN_LIBRARY, CAST('lib version' as CHAR(20)) as PLUGIN_LIBRARY_VERSION, CAST('author' as CHAR(64)) as PLUGIN_AUTHOR, CAST('description' as CHAR(200)) as PLUGIN_DESCRIPTION, CAST('license' as CHAR(80)) as PLUGIN_LICENSE, CAST('load option' as CHAR(64)) as LOAD_OPTION FROM DUAL limit 0; )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::innodb_sys_columns_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_INFORMATION_SCHEMA_ID); + table_schema.set_table_id(OB_INNODB_SYS_COLUMNS_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_INNODB_SYS_COLUMNS_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(000000000000000000000 as UNSIGNED) AS TABLE_ID, CAST('name' as CHAR(193)) AS NAME, CAST(000000000000000000000 as UNSIGNED) AS POS, CAST(00000000000 as SIGNED) AS MTYPE, CAST(00000000000 as SIGNED) AS PRTYPE, CAST(00000000000 as SIGNED) AS LEN FROM DUAL limit 0; )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::innodb_ft_being_deleted_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.21301_21350.cpp b/src/share/inner_table/ob_inner_table_schema.21301_21350.cpp index afc4d5913..60d96d78a 100644 --- a/src/share/inner_table/ob_inner_table_schema.21301_21350.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21301_21350.cpp @@ -1567,7 +1567,7 @@ int ObInnerTableSchema::dba_ob_users_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT USER_NAME, HOST, PASSWD, INFO, (CASE WHEN PRIV_ALTER = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_ALTER, (CASE WHEN PRIV_CREATE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE, (CASE WHEN PRIV_DELETE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_DELETE, (CASE WHEN PRIV_DROP = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_DROP, (CASE WHEN PRIV_GRANT_OPTION = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_GRANT_OPTION, (CASE WHEN PRIV_INSERT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_INSERT, (CASE WHEN PRIV_UPDATE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_UPDATE, (CASE WHEN PRIV_SELECT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_SELECT, (CASE WHEN PRIV_INDEX = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_INDEX, (CASE WHEN PRIV_CREATE_VIEW = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_VIEW, (CASE WHEN PRIV_SHOW_VIEW = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_SHOW_VIEW, (CASE WHEN PRIV_SHOW_DB = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_SHOW_DB, (CASE WHEN PRIV_CREATE_USER = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_USER, (CASE WHEN PRIV_SUPER = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_SUPER, (CASE WHEN IS_LOCKED = 0 THEN 'NO' ELSE 'YES' END) AS IS_LOCKED, (CASE WHEN PRIV_PROCESS = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_PROCESS, (CASE WHEN PRIV_CREATE_SYNONYM = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_SYNONYM, SSL_TYPE, SSL_CIPHER, X509_ISSUER, X509_SUBJECT, (CASE WHEN TYPE = 0 THEN 'USER' ELSE 'ROLE' END) AS TYPE, PROFILE_ID, PASSWORD_LAST_CHANGED, (CASE WHEN PRIV_FILE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_FILE, (CASE WHEN PRIV_ALTER_TENANT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_ALTER_TENANT, (CASE WHEN PRIV_ALTER_SYSTEM = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_ALTER_SYSTEM, (CASE WHEN PRIV_CREATE_RESOURCE_POOL = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_RESOURCE_POOL, (CASE WHEN PRIV_CREATE_RESOURCE_UNIT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_RESOURCE_UNIT, MAX_CONNECTIONS, MAX_USER_CONNECTIONS, (CASE WHEN PRIV_REPL_SLAVE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_REPL_SLAVE, (CASE WHEN PRIV_REPL_CLIENT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_REPL_CLIENT, (CASE WHEN PRIV_DROP_DATABASE_LINK = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_DROP_DATABASE_LINK, (CASE WHEN PRIV_CREATE_DATABASE_LINK = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_DATABASE_LINK, (CASE WHEN (PRIV_OTHERS & (1 << 0)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_EXECUTE, (CASE WHEN (PRIV_OTHERS & (1 << 1)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_ALTER_ROUTINE, (CASE WHEN (PRIV_OTHERS & (1 << 2)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_CREATE_ROUTINE, (CASE WHEN (PRIV_OTHERS & (1 << 3)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_CREATE_TABLESPACE, (CASE WHEN (PRIV_OTHERS & (1 << 4)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_SHUTDOWN, (CASE WHEN (PRIV_OTHERS & (1 << 5)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_RELOAD FROM OCEANBASE.__all_user; )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT USER_NAME, HOST, PASSWD, INFO, (CASE WHEN PRIV_ALTER = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_ALTER, (CASE WHEN PRIV_CREATE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE, (CASE WHEN PRIV_DELETE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_DELETE, (CASE WHEN PRIV_DROP = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_DROP, (CASE WHEN PRIV_GRANT_OPTION = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_GRANT_OPTION, (CASE WHEN PRIV_INSERT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_INSERT, (CASE WHEN PRIV_UPDATE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_UPDATE, (CASE WHEN PRIV_SELECT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_SELECT, (CASE WHEN PRIV_INDEX = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_INDEX, (CASE WHEN PRIV_CREATE_VIEW = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_VIEW, (CASE WHEN PRIV_SHOW_VIEW = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_SHOW_VIEW, (CASE WHEN PRIV_SHOW_DB = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_SHOW_DB, (CASE WHEN PRIV_CREATE_USER = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_USER, (CASE WHEN PRIV_SUPER = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_SUPER, (CASE WHEN IS_LOCKED = 0 THEN 'NO' ELSE 'YES' END) AS IS_LOCKED, (CASE WHEN PRIV_PROCESS = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_PROCESS, (CASE WHEN PRIV_CREATE_SYNONYM = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_SYNONYM, SSL_TYPE, SSL_CIPHER, X509_ISSUER, X509_SUBJECT, (CASE WHEN TYPE = 0 THEN 'USER' ELSE 'ROLE' END) AS TYPE, PROFILE_ID, PASSWORD_LAST_CHANGED, (CASE WHEN PRIV_FILE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_FILE, (CASE WHEN PRIV_ALTER_TENANT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_ALTER_TENANT, (CASE WHEN PRIV_ALTER_SYSTEM = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_ALTER_SYSTEM, (CASE WHEN PRIV_CREATE_RESOURCE_POOL = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_RESOURCE_POOL, (CASE WHEN PRIV_CREATE_RESOURCE_UNIT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_RESOURCE_UNIT, MAX_CONNECTIONS, MAX_USER_CONNECTIONS, (CASE WHEN PRIV_REPL_SLAVE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_REPL_SLAVE, (CASE WHEN PRIV_REPL_CLIENT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_REPL_CLIENT, (CASE WHEN PRIV_DROP_DATABASE_LINK = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_DROP_DATABASE_LINK, (CASE WHEN PRIV_CREATE_DATABASE_LINK = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_DATABASE_LINK, (CASE WHEN (PRIV_OTHERS & (1 << 0)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_EXECUTE, (CASE WHEN (PRIV_OTHERS & (1 << 1)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_ALTER_ROUTINE, (CASE WHEN (PRIV_OTHERS & (1 << 2)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_CREATE_ROUTINE, (CASE WHEN (PRIV_OTHERS & (1 << 3)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_CREATE_TABLESPACE, (CASE WHEN (PRIV_OTHERS & (1 << 4)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_SHUTDOWN, (CASE WHEN (PRIV_OTHERS & (1 << 5)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_RELOAD, (CASE WHEN (PRIV_OTHERS & (1 << 6)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_REFERENCES, (CASE WHEN (PRIV_OTHERS & (1 << 7)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_CREATE_ROLE, (CASE WHEN (PRIV_OTHERS & (1 << 8)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_DROP_ROLE, (CASE WHEN (PRIV_OTHERS & (1 << 9)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_TRIGGER FROM OCEANBASE.__all_user; )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -1617,7 +1617,7 @@ int ObInnerTableSchema::cdb_ob_users_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT TENANT_ID, USER_NAME, HOST, PASSWD, INFO, (CASE WHEN PRIV_ALTER = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_ALTER, (CASE WHEN PRIV_CREATE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE, (CASE WHEN PRIV_DELETE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_DELETE, (CASE WHEN PRIV_DROP = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_DROP, (CASE WHEN PRIV_GRANT_OPTION = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_GRANT_OPTION, (CASE WHEN PRIV_INSERT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_INSERT, (CASE WHEN PRIV_UPDATE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_UPDATE, (CASE WHEN PRIV_SELECT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_SELECT, (CASE WHEN PRIV_INDEX = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_INDEX, (CASE WHEN PRIV_CREATE_VIEW = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_VIEW, (CASE WHEN PRIV_SHOW_VIEW = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_SHOW_VIEW, (CASE WHEN PRIV_SHOW_DB = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_SHOW_DB, (CASE WHEN PRIV_CREATE_USER = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_USER, (CASE WHEN PRIV_SUPER = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_SUPER, (CASE WHEN IS_LOCKED = 0 THEN 'NO' ELSE 'YES' END) AS IS_LOCKED, (CASE WHEN PRIV_PROCESS = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_PROCESS, (CASE WHEN PRIV_CREATE_SYNONYM = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_SYNONYM, SSL_TYPE, SSL_CIPHER, X509_ISSUER, X509_SUBJECT, (CASE WHEN TYPE = 0 THEN 'USER' ELSE 'ROLE' END) AS TYPE, PROFILE_ID, PASSWORD_LAST_CHANGED, (CASE WHEN PRIV_FILE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_FILE, (CASE WHEN PRIV_ALTER_TENANT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_ALTER_TENANT, (CASE WHEN PRIV_ALTER_SYSTEM = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_ALTER_SYSTEM, (CASE WHEN PRIV_CREATE_RESOURCE_POOL = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_RESOURCE_POOL, (CASE WHEN PRIV_CREATE_RESOURCE_UNIT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_RESOURCE_UNIT, MAX_CONNECTIONS, MAX_USER_CONNECTIONS, (CASE WHEN PRIV_REPL_SLAVE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_REPL_SLAVE, (CASE WHEN PRIV_REPL_CLIENT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_REPL_CLIENT, (CASE WHEN PRIV_DROP_DATABASE_LINK = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_DROP_DATABASE_LINK, (CASE WHEN PRIV_CREATE_DATABASE_LINK = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_DATABASE_LINK, (CASE WHEN (PRIV_OTHERS & (1 << 0)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_EXECUTE, (CASE WHEN (PRIV_OTHERS & (1 << 1)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_ALTER_ROUTINE, (CASE WHEN (PRIV_OTHERS & (1 << 2)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_CREATE_ROUTINE, (CASE WHEN (PRIV_OTHERS & (1 << 3)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_CREATE_TABLESPACE, (CASE WHEN (PRIV_OTHERS & (1 << 4)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_SHUTDOWN, (CASE WHEN (PRIV_OTHERS & (1 << 5)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_RELOAD FROM OCEANBASE.__all_virtual_user; )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT TENANT_ID, USER_NAME, HOST, PASSWD, INFO, (CASE WHEN PRIV_ALTER = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_ALTER, (CASE WHEN PRIV_CREATE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE, (CASE WHEN PRIV_DELETE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_DELETE, (CASE WHEN PRIV_DROP = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_DROP, (CASE WHEN PRIV_GRANT_OPTION = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_GRANT_OPTION, (CASE WHEN PRIV_INSERT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_INSERT, (CASE WHEN PRIV_UPDATE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_UPDATE, (CASE WHEN PRIV_SELECT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_SELECT, (CASE WHEN PRIV_INDEX = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_INDEX, (CASE WHEN PRIV_CREATE_VIEW = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_VIEW, (CASE WHEN PRIV_SHOW_VIEW = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_SHOW_VIEW, (CASE WHEN PRIV_SHOW_DB = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_SHOW_DB, (CASE WHEN PRIV_CREATE_USER = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_USER, (CASE WHEN PRIV_SUPER = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_SUPER, (CASE WHEN IS_LOCKED = 0 THEN 'NO' ELSE 'YES' END) AS IS_LOCKED, (CASE WHEN PRIV_PROCESS = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_PROCESS, (CASE WHEN PRIV_CREATE_SYNONYM = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_SYNONYM, SSL_TYPE, SSL_CIPHER, X509_ISSUER, X509_SUBJECT, (CASE WHEN TYPE = 0 THEN 'USER' ELSE 'ROLE' END) AS TYPE, PROFILE_ID, PASSWORD_LAST_CHANGED, (CASE WHEN PRIV_FILE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_FILE, (CASE WHEN PRIV_ALTER_TENANT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_ALTER_TENANT, (CASE WHEN PRIV_ALTER_SYSTEM = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_ALTER_SYSTEM, (CASE WHEN PRIV_CREATE_RESOURCE_POOL = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_RESOURCE_POOL, (CASE WHEN PRIV_CREATE_RESOURCE_UNIT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_RESOURCE_UNIT, MAX_CONNECTIONS, MAX_USER_CONNECTIONS, (CASE WHEN PRIV_REPL_SLAVE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_REPL_SLAVE, (CASE WHEN PRIV_REPL_CLIENT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_REPL_CLIENT, (CASE WHEN PRIV_DROP_DATABASE_LINK = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_DROP_DATABASE_LINK, (CASE WHEN PRIV_CREATE_DATABASE_LINK = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_DATABASE_LINK, (CASE WHEN (PRIV_OTHERS & (1 << 0)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_EXECUTE, (CASE WHEN (PRIV_OTHERS & (1 << 1)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_ALTER_ROUTINE, (CASE WHEN (PRIV_OTHERS & (1 << 2)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_CREATE_ROUTINE, (CASE WHEN (PRIV_OTHERS & (1 << 3)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_CREATE_TABLESPACE, (CASE WHEN (PRIV_OTHERS & (1 << 4)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_SHUTDOWN, (CASE WHEN (PRIV_OTHERS & (1 << 5)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_RELOAD, (CASE WHEN (PRIV_OTHERS & (1 << 6)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_REFERENCES, (CASE WHEN (PRIV_OTHERS & (1 << 7)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_CREATE_ROLE, (CASE WHEN (PRIV_OTHERS & (1 << 8)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_DROP_ROLE, (CASE WHEN (PRIV_OTHERS & (1 << 9)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_TRIGGER FROM OCEANBASE.__all_virtual_user; )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -1667,7 +1667,7 @@ int ObInnerTableSchema::dba_ob_database_privilege_schema(ObTableSchema &table_sc table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( WITH DB_PRIV AS ( select A.tenant_id TENANT_ID, A.user_id USER_ID, A.database_name DATABASE_NAME, A.priv_alter PRIV_ALTER, A.priv_create PRIV_CREATE, A.priv_delete PRIV_DELETE, A.priv_drop PRIV_DROP, A.priv_grant_option PRIV_GRANT_OPTION, A.priv_insert PRIV_INSERT, A.priv_update PRIV_UPDATE, A.priv_select PRIV_SELECT, A.priv_index PRIV_INDEX, A.priv_create_view PRIV_CREATE_VIEW, A.priv_show_view PRIV_SHOW_VIEW, A.GMT_CREATE GMT_CREATE, A.GMT_MODIFIED GMT_MODIFIED, A.priv_others PRIV_OTHERS from oceanbase.__all_database_privilege_history A, (select tenant_id, user_id, database_name, max(schema_version) schema_version from oceanbase.__all_database_privilege_history group by tenant_id, user_id, database_name, database_name collate utf8mb4_bin) B where A.tenant_id = B.tenant_id and A.user_id = B.user_id and A.database_name collate utf8mb4_bin = B.database_name collate utf8mb4_bin and A.schema_version = B.schema_version and A.is_deleted = 0 ) SELECT A.USER_ID USER_ID, B.USER_NAME USERNAME, A.DATABASE_NAME DATABASE_NAME, A.GMT_CREATE GMT_CREATE, A.GMT_MODIFIED GMT_MODIFIED, (CASE WHEN A.PRIV_ALTER = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_ALTER, (CASE WHEN A.PRIV_CREATE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE, (CASE WHEN A.PRIV_DELETE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_DELETE, (CASE WHEN A.PRIV_DROP = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_DROP, (CASE WHEN A.PRIV_GRANT_OPTION = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_GRANT_OPTION, (CASE WHEN A.PRIV_INSERT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_INSERT, (CASE WHEN A.PRIV_UPDATE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_UPDATE, (CASE WHEN A.PRIV_SELECT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_SELECT, (CASE WHEN A.PRIV_INDEX = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_INDEX, (CASE WHEN A.PRIV_CREATE_VIEW = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_VIEW, (CASE WHEN A.PRIV_SHOW_VIEW = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_SHOW_VIEW, (CASE WHEN (A.PRIV_OTHERS & (1 << 0)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_EXECUTE, (CASE WHEN (A.PRIV_OTHERS & (1 << 1)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_ALTER_ROUTINE, (CASE WHEN (A.PRIV_OTHERS & (1 << 2)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_CREATE_ROUTINE FROM DB_PRIV A INNER JOIN OCEANBASE.__all_user B ON A.TENANT_ID = B.TENANT_ID AND A.USER_ID = B.USER_ID; )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( WITH DB_PRIV AS ( select A.tenant_id TENANT_ID, A.user_id USER_ID, A.database_name DATABASE_NAME, A.priv_alter PRIV_ALTER, A.priv_create PRIV_CREATE, A.priv_delete PRIV_DELETE, A.priv_drop PRIV_DROP, A.priv_grant_option PRIV_GRANT_OPTION, A.priv_insert PRIV_INSERT, A.priv_update PRIV_UPDATE, A.priv_select PRIV_SELECT, A.priv_index PRIV_INDEX, A.priv_create_view PRIV_CREATE_VIEW, A.priv_show_view PRIV_SHOW_VIEW, A.GMT_CREATE GMT_CREATE, A.GMT_MODIFIED GMT_MODIFIED, A.priv_others PRIV_OTHERS from oceanbase.__all_database_privilege_history A, (select tenant_id, user_id, database_name, max(schema_version) schema_version from oceanbase.__all_database_privilege_history group by tenant_id, user_id, database_name, database_name collate utf8mb4_bin) B where A.tenant_id = B.tenant_id and A.user_id = B.user_id and A.database_name collate utf8mb4_bin = B.database_name collate utf8mb4_bin and A.schema_version = B.schema_version and A.is_deleted = 0 ) SELECT A.USER_ID USER_ID, B.USER_NAME USERNAME, A.DATABASE_NAME DATABASE_NAME, A.GMT_CREATE GMT_CREATE, A.GMT_MODIFIED GMT_MODIFIED, (CASE WHEN A.PRIV_ALTER = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_ALTER, (CASE WHEN A.PRIV_CREATE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE, (CASE WHEN A.PRIV_DELETE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_DELETE, (CASE WHEN A.PRIV_DROP = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_DROP, (CASE WHEN A.PRIV_GRANT_OPTION = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_GRANT_OPTION, (CASE WHEN A.PRIV_INSERT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_INSERT, (CASE WHEN A.PRIV_UPDATE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_UPDATE, (CASE WHEN A.PRIV_SELECT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_SELECT, (CASE WHEN A.PRIV_INDEX = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_INDEX, (CASE WHEN A.PRIV_CREATE_VIEW = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_VIEW, (CASE WHEN A.PRIV_SHOW_VIEW = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_SHOW_VIEW, (CASE WHEN (A.PRIV_OTHERS & (1 << 0)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_EXECUTE, (CASE WHEN (A.PRIV_OTHERS & (1 << 1)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_ALTER_ROUTINE, (CASE WHEN (A.PRIV_OTHERS & (1 << 2)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_CREATE_ROUTINE, (CASE WHEN (A.PRIV_OTHERS & (1 << 6)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_REFERENCES, (CASE WHEN (A.PRIV_OTHERS & (1 << 9)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_TRIGGER FROM DB_PRIV A INNER JOIN OCEANBASE.__all_user B ON A.TENANT_ID = B.TENANT_ID AND A.USER_ID = B.USER_ID; )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -1717,7 +1717,7 @@ int ObInnerTableSchema::cdb_ob_database_privilege_schema(ObTableSchema &table_sc table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( WITH DB_PRIV AS ( select A.tenant_id TENANT_ID, A.user_id USER_ID, A.database_name DATABASE_NAME, A.priv_alter PRIV_ALTER, A.priv_create PRIV_CREATE, A.priv_delete PRIV_DELETE, A.priv_drop PRIV_DROP, A.priv_grant_option PRIV_GRANT_OPTION, A.priv_insert PRIV_INSERT, A.priv_update PRIV_UPDATE, A.priv_select PRIV_SELECT, A.priv_index PRIV_INDEX, A.priv_create_view PRIV_CREATE_VIEW, A.priv_show_view PRIV_SHOW_VIEW, A.GMT_CREATE GMT_CREATE, A.GMT_MODIFIED GMT_MODIFIED, A.PRIV_OTHERS PRIV_OTHERS from oceanbase.__all_virtual_database_privilege_history A, (select tenant_id, user_id, database_name, max(schema_version) schema_version from oceanbase.__all_virtual_database_privilege_history group by tenant_id, user_id, database_name, database_name collate utf8mb4_bin) B where A.tenant_id = B.tenant_id and A.user_id = B.user_id and A.database_name collate utf8mb4_bin = B.database_name collate utf8mb4_bin and A.schema_version = B.schema_version and A.is_deleted = 0 ) SELECT A.TENANT_ID, A.USER_ID USER_ID, B.USER_NAME USERNAME, A.DATABASE_NAME DATABASE_NAME, A.GMT_CREATE GMT_CREATE, A.GMT_MODIFIED GMT_MODIFIED, (CASE WHEN A.PRIV_ALTER = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_ALTER, (CASE WHEN A.PRIV_CREATE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE, (CASE WHEN A.PRIV_DELETE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_DELETE, (CASE WHEN A.PRIV_DROP = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_DROP, (CASE WHEN A.PRIV_GRANT_OPTION = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_GRANT_OPTION, (CASE WHEN A.PRIV_INSERT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_INSERT, (CASE WHEN A.PRIV_UPDATE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_UPDATE, (CASE WHEN A.PRIV_SELECT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_SELECT, (CASE WHEN A.PRIV_INDEX = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_INDEX, (CASE WHEN A.PRIV_CREATE_VIEW = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_VIEW, (CASE WHEN A.PRIV_SHOW_VIEW = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_SHOW_VIEW, (CASE WHEN (A.PRIV_OTHERS & (1 << 0)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_EXECUTE, (CASE WHEN (A.PRIV_OTHERS & (1 << 1)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_ALTER_ROUTINE, (CASE WHEN (A.PRIV_OTHERS & (1 << 2)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_CREATE_ROUTINE FROM DB_PRIV A INNER JOIN OCEANBASE.__all_virtual_user B ON A.USER_ID = B.USER_ID AND A.TENANT_ID = B.TENANT_ID; )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( WITH DB_PRIV AS ( select A.tenant_id TENANT_ID, A.user_id USER_ID, A.database_name DATABASE_NAME, A.priv_alter PRIV_ALTER, A.priv_create PRIV_CREATE, A.priv_delete PRIV_DELETE, A.priv_drop PRIV_DROP, A.priv_grant_option PRIV_GRANT_OPTION, A.priv_insert PRIV_INSERT, A.priv_update PRIV_UPDATE, A.priv_select PRIV_SELECT, A.priv_index PRIV_INDEX, A.priv_create_view PRIV_CREATE_VIEW, A.priv_show_view PRIV_SHOW_VIEW, A.GMT_CREATE GMT_CREATE, A.GMT_MODIFIED GMT_MODIFIED, A.PRIV_OTHERS PRIV_OTHERS from oceanbase.__all_virtual_database_privilege_history A, (select tenant_id, user_id, database_name, max(schema_version) schema_version from oceanbase.__all_virtual_database_privilege_history group by tenant_id, user_id, database_name, database_name collate utf8mb4_bin) B where A.tenant_id = B.tenant_id and A.user_id = B.user_id and A.database_name collate utf8mb4_bin = B.database_name collate utf8mb4_bin and A.schema_version = B.schema_version and A.is_deleted = 0 ) SELECT A.TENANT_ID, A.USER_ID USER_ID, B.USER_NAME USERNAME, A.DATABASE_NAME DATABASE_NAME, A.GMT_CREATE GMT_CREATE, A.GMT_MODIFIED GMT_MODIFIED, (CASE WHEN A.PRIV_ALTER = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_ALTER, (CASE WHEN A.PRIV_CREATE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE, (CASE WHEN A.PRIV_DELETE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_DELETE, (CASE WHEN A.PRIV_DROP = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_DROP, (CASE WHEN A.PRIV_GRANT_OPTION = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_GRANT_OPTION, (CASE WHEN A.PRIV_INSERT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_INSERT, (CASE WHEN A.PRIV_UPDATE = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_UPDATE, (CASE WHEN A.PRIV_SELECT = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_SELECT, (CASE WHEN A.PRIV_INDEX = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_INDEX, (CASE WHEN A.PRIV_CREATE_VIEW = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_CREATE_VIEW, (CASE WHEN A.PRIV_SHOW_VIEW = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_SHOW_VIEW, (CASE WHEN (A.PRIV_OTHERS & (1 << 0)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_EXECUTE, (CASE WHEN (A.PRIV_OTHERS & (1 << 1)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_ALTER_ROUTINE, (CASE WHEN (A.PRIV_OTHERS & (1 << 2)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_CREATE_ROUTINE, (CASE WHEN (A.PRIV_OTHERS & (1 << 6)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_REFERENCES, (CASE WHEN (A.PRIV_OTHERS & (1 << 9)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_TRIGGER FROM DB_PRIV A INNER JOIN OCEANBASE.__all_virtual_user B ON A.USER_ID = B.USER_ID AND A.TENANT_ID = B.TENANT_ID; )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -1967,7 +1967,7 @@ int ObInnerTableSchema::parameters_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__(select CAST('def' AS CHAR(512)) AS SPECIFIC_CATALOG, CAST(d.database_name AS CHAR(128)) collate utf8mb4_name_case AS SPECIFIC_SCHEMA, CAST(r.routine_name AS CHAR(64)) AS SPECIFIC_NAME, CAST(rp.param_position AS signed) AS ORDINAL_POSITION, CAST(CASE rp.param_position WHEN 0 THEN NULL ELSE CASE rp.flag & 0x03 WHEN 1 THEN 'IN' WHEN 2 THEN 'OUT' WHEN 3 THEN 'INOUT' ELSE NULL END END AS CHAR(5)) AS PARAMETER_MODE, CAST(rp.param_name AS CHAR(64)) AS PARAMETER_NAME, CAST(lower(case v.data_type_str when 'TINYINT UNSIGNED' then 'TINYINT' when 'SMALLINT UNSIGNED' then 'SMALLINT' when 'MEDIUMINT UNSIGNED' then 'MEDIUMINT' when 'INT UNSIGNED' then 'INT' when 'BIGINT UNSIGNED' then 'BIGINT' when 'FLOAT UNSIGNED' then 'FLOAT' when 'DOUBLE UNSIGNED' then 'DOUBLE' when 'DECIMAL UNSIGNED' then 'DECIMAL' when 'CHAR' then if(rp.param_charset = 1, 'BINARY', 'CHAR') when 'VARCHAR' then if(rp.param_charset = 1, 'VARBINARY', 'VARCHAR') when 'TINYTEXT' then if(rp.param_charset = 1, 'TINYBLOB', 'TINYTEXT') when 'TEXT' then if(rp.param_charset = 1, 'BLOB', 'TEXT') when 'MEDIUMTEXT' then if(rp.param_charset = 1, 'MEDIUMBLOB', 'MEDIUMTEXT') when 'LONGTEXT' then if(rp.param_charset = 1, 'LONGBLOB', 'LONGTEXT') else v.data_type_str end) AS CHAR(64)) AS DATA_TYPE, CASE WHEN rp.param_type IN (22, 23, 27, 28, 29, 30) THEN CAST(rp.param_length AS SIGNED) ELSE CAST(NULL AS SIGNED) END AS CHARACTER_MAXIMUM_LENGTH, CASE WHEN rp.param_type IN (22, 23, 27, 28, 29, 30, 43, 44, 46) THEN CAST( rp.param_length * CASE rp.param_coll_type WHEN 63 THEN 1 WHEN 249 THEN 4 WHEN 248 THEN 4 WHEN 87 THEN 2 WHEN 28 THEN 2 WHEN 55 THEN 4 WHEN 54 THEN 4 WHEN 101 THEN 2 WHEN 46 THEN 4 WHEN 45 THEN 4 WHEN 224 THEN 4 ELSE 1 END AS SIGNED ) ELSE CAST(NULL AS SIGNED) END AS CHARACTER_OCTET_LENGTH, CASE WHEN rp.param_type IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 16, 31, 50) THEN CAST(rp.param_precision AS UNSIGNED) WHEN rp.param_type IN (11, 13) THEN CAST(if(rp.param_scale = -1, 12, rp.param_precision) AS UNSIGNED) WHEN rp.param_type IN (12, 14) THEN CAST(if(rp.param_scale = -1, 22, rp.param_precision) AS UNSIGNED) ELSE CAST(NULL AS UNSIGNED) END AS NUMERIC_PRECISION, CASE WHEN rp.param_type IN (15, 16, 50) THEN CAST(rp.param_scale AS SIGNED) WHEN rp.param_type IN (11, 12, 13, 14) THEN CAST(if(rp.param_scale = -1, 0, rp.param_scale) AS SIGNED) WHEN rp.param_type IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 31) THEN CAST(0 AS SIGNED) ELSE CAST(NULL AS SIGNED) END AS NUMERIC_SCALE, CASE WHEN rp.param_type IN (17, 18, 20) THEN CAST(rp.param_scale AS UNSIGNED) ELSE CAST(NULL AS UNSIGNED) END AS DATETIME_PRECISION, CAST(CASE rp.param_charset WHEN 1 THEN 'binary' WHEN 2 THEN 'utf8mb4' WHEN 3 THEN 'gbk' WHEN 4 THEN 'utf16' WHEN 5 THEN 'gb18030' WHEN 6 THEN 'latin1' WHEN 7 THEN 'gb18030_2022' ELSE NULL END AS CHAR(64)) AS CHARACTER_SET_NAME, CAST(CASE rp.param_coll_type WHEN 8 THEN 'latin1_swedish_ci' WHEN 11 THEN 'ascii_general_ci' WHEN 18 THEN 'tis620_thai_ci' WHEN 28 THEN 'gbk_chinese_ci' WHEN 45 THEN 'utf8mb4_general_ci' WHEN 46 THEN 'utf8mb4_bin' WHEN 47 THEN 'latin1_bin' WHEN 54 THEN 'utf16_general_ci' WHEN 55 THEN 'utf16_bin' WHEN 63 THEN 'binary' WHEN 65 THEN 'ascii_bin' WHEN 87 THEN 'gbk_bin' WHEN 89 THEN 'tis620_bin' WHEN 101 THEN 'utf16_unicode_ci' WHEN 216 THEN 'gb18030_2022_bin' WHEN 217 THEN 'gb18030_2022_chinese_ci' WHEN 218 THEN 'gb18030_2022_chinese_cs' WHEN 219 THEN 'gb18030_2022_radical_ci' WHEN 220 THEN 'gb18030_2022_radical_cs' WHEN 221 THEN 'gb18030_2022_stroke_ci' WHEN 222 THEN 'gb18030_2022_stroke_cs' WHEN 224 THEN 'utf8mb4_unicode_ci' WHEN 234 THEN 'utf8mb4_czech_ci' WHEN 245 THEN 'utf8mb4_croatian_ci' WHEN 246 THEN 'utf8mb4_unicode_520_ci' WHEN 248 THEN 'gb18030_chinese_ci' WHEN 249 THEN 'gb18030_bin' WHEN 255 THEN 'utf8mb4_0900_ai_ci' ELSE NULL END AS CHAR(64)) AS COLLATION_NAME, CAST(CASE WHEN rp.param_type IN (1, 2, 3, 4, 5, 31) THEN CONCAT(lower(v.data_type_str),'(',rp.param_precision,')') WHEN (rp.param_type in (6, 7, 8, 9, 10) AND rp.param_zero_fill) THEN CONCAT(lower(v.data_type_str), ' zerofill') WHEN rp.param_type IN (15,16,50) THEN CONCAT(lower(v.data_type_str),'(',rp.param_precision, ',', rp.param_scale,')') WHEN rp.param_type IN (17, 18, 20) THEN CONCAT(lower(v.data_type_str),'(', rp.param_scale, ')') WHEN (rp.param_type IN (22, 23) AND rp.param_charset != 1) THEN CONCAT(lower(v.data_type_str),'(', rp.param_length, ')') WHEN (rp.param_type IN (22) AND rp.param_charset = 1) THEN CONCAT(lower('VARBINARY'),'(', rp.param_length, ')') WHEN (rp.param_type IN (23) AND rp.param_charset = 1) THEN CONCAT(lower('BINARY'),'(', rp.param_length, ')') WHEN (rp.param_type IN (27, 28, 29, 30) AND rp.param_charset = 1) THEN lower(REPLACE(v.data_type_str, 'TEXT', 'BLOB')) ELSE lower(v.data_type_str) END AS char(4194304)) AS DTD_IDENTIFIER, CAST(CASE WHEN r.routine_type = 1 THEN 'PROCEDURE' WHEN ROUTINE_TYPE = 2 THEN 'FUNCTION' ELSE NULL END AS CHAR(9)) AS ROUTINE_TYPE from oceanbase.__all_routine_param as rp join oceanbase.__all_routine as r on rp.subprogram_id = r.subprogram_id and rp.tenant_id = r.tenant_id and rp.routine_id = r.routine_id join oceanbase.__all_database as d on r.database_id = d.database_id left join oceanbase.__all_virtual_data_type v on rp.param_type = v.data_type WHERE rp.tenant_id = 0 and in_recyclebin = 0 and database_name != '__recyclebin' order by SPECIFIC_SCHEMA, SPECIFIC_NAME, ORDINAL_POSITION )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__(select CAST('def' AS CHAR(512)) AS SPECIFIC_CATALOG, CAST(d.database_name AS CHAR(128)) collate utf8mb4_name_case AS SPECIFIC_SCHEMA, CAST(r.routine_name AS CHAR(64)) AS SPECIFIC_NAME, CAST(rp.param_position AS signed) AS ORDINAL_POSITION, CAST(CASE rp.param_position WHEN 0 THEN NULL ELSE CASE rp.flag & 0x03 WHEN 1 THEN 'IN' WHEN 2 THEN 'OUT' WHEN 3 THEN 'INOUT' ELSE NULL END END AS CHAR(5)) AS PARAMETER_MODE, CAST(rp.param_name AS CHAR(64)) AS PARAMETER_NAME, CAST(lower(case v.data_type_str when 'TINYINT UNSIGNED' then 'TINYINT' when 'SMALLINT UNSIGNED' then 'SMALLINT' when 'MEDIUMINT UNSIGNED' then 'MEDIUMINT' when 'INT UNSIGNED' then 'INT' when 'BIGINT UNSIGNED' then 'BIGINT' when 'FLOAT UNSIGNED' then 'FLOAT' when 'DOUBLE UNSIGNED' then 'DOUBLE' when 'DECIMAL UNSIGNED' then 'DECIMAL' when 'CHAR' then if(rp.param_charset = 1, 'BINARY', 'CHAR') when 'VARCHAR' then if(rp.param_charset = 1, 'VARBINARY', 'VARCHAR') when 'TINYTEXT' then if(rp.param_charset = 1, 'TINYBLOB', 'TINYTEXT') when 'TEXT' then if(rp.param_charset = 1, 'BLOB', 'TEXT') when 'MEDIUMTEXT' then if(rp.param_charset = 1, 'MEDIUMBLOB', 'MEDIUMTEXT') when 'LONGTEXT' then if(rp.param_charset = 1, 'LONGBLOB', 'LONGTEXT') else v.data_type_str end) AS CHAR(64)) AS DATA_TYPE, CASE WHEN rp.param_type IN (22, 23, 27, 28, 29, 30) THEN CAST(rp.param_length AS SIGNED) ELSE CAST(NULL AS SIGNED) END AS CHARACTER_MAXIMUM_LENGTH, CASE WHEN rp.param_type IN (22, 23, 27, 28, 29, 30, 43, 44, 46) THEN CAST( rp.param_length * CASE rp.param_coll_type WHEN 63 THEN 1 WHEN 249 THEN 4 WHEN 248 THEN 4 WHEN 87 THEN 2 WHEN 28 THEN 2 WHEN 55 THEN 4 WHEN 54 THEN 4 WHEN 101 THEN 2 WHEN 46 THEN 4 WHEN 45 THEN 4 WHEN 224 THEN 4 ELSE 1 END AS SIGNED ) ELSE CAST(NULL AS SIGNED) END AS CHARACTER_OCTET_LENGTH, CASE WHEN rp.param_type IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 16, 31, 50) THEN CAST(rp.param_precision AS UNSIGNED) WHEN rp.param_type IN (11, 13) THEN CAST(if(rp.param_scale = -1, 12, rp.param_precision) AS UNSIGNED) WHEN rp.param_type IN (12, 14) THEN CAST(if(rp.param_scale = -1, 22, rp.param_precision) AS UNSIGNED) ELSE CAST(NULL AS UNSIGNED) END AS NUMERIC_PRECISION, CASE WHEN rp.param_type IN (15, 16, 50) THEN CAST(rp.param_scale AS SIGNED) WHEN rp.param_type IN (11, 12, 13, 14) THEN CAST(if(rp.param_scale = -1, 0, rp.param_scale) AS SIGNED) WHEN rp.param_type IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 31) THEN CAST(0 AS SIGNED) ELSE CAST(NULL AS SIGNED) END AS NUMERIC_SCALE, CASE WHEN rp.param_type IN (17, 18, 20) THEN CAST(rp.param_scale AS UNSIGNED) ELSE CAST(NULL AS UNSIGNED) END AS DATETIME_PRECISION, CAST(CASE rp.param_charset WHEN 1 THEN 'binary' WHEN 2 THEN 'utf8mb4' WHEN 3 THEN 'gbk' WHEN 4 THEN 'utf16' WHEN 5 THEN 'gb18030' WHEN 6 THEN 'latin1' WHEN 7 THEN 'gb18030_2022' WHEN 8 THEN 'ascii' WHEN 9 THEN 'tis620' ELSE NULL END AS CHAR(64)) AS CHARACTER_SET_NAME, CAST(CASE rp.param_coll_type WHEN 8 THEN 'latin1_swedish_ci' WHEN 11 THEN 'ascii_general_ci' WHEN 18 THEN 'tis620_thai_ci' WHEN 28 THEN 'gbk_chinese_ci' WHEN 45 THEN 'utf8mb4_general_ci' WHEN 46 THEN 'utf8mb4_bin' WHEN 47 THEN 'latin1_bin' WHEN 54 THEN 'utf16_general_ci' WHEN 55 THEN 'utf16_bin' WHEN 63 THEN 'binary' WHEN 65 THEN 'ascii_bin' WHEN 87 THEN 'gbk_bin' WHEN 89 THEN 'tis620_bin' WHEN 101 THEN 'utf16_unicode_ci' WHEN 216 THEN 'gb18030_2022_bin' WHEN 217 THEN 'gb18030_2022_chinese_ci' WHEN 218 THEN 'gb18030_2022_chinese_cs' WHEN 219 THEN 'gb18030_2022_radical_ci' WHEN 220 THEN 'gb18030_2022_radical_cs' WHEN 221 THEN 'gb18030_2022_stroke_ci' WHEN 222 THEN 'gb18030_2022_stroke_cs' WHEN 224 THEN 'utf8mb4_unicode_ci' WHEN 234 THEN 'utf8mb4_czech_ci' WHEN 245 THEN 'utf8mb4_croatian_ci' WHEN 246 THEN 'utf8mb4_unicode_520_ci' WHEN 248 THEN 'gb18030_chinese_ci' WHEN 249 THEN 'gb18030_bin' WHEN 255 THEN 'utf8mb4_0900_ai_ci' ELSE NULL END AS CHAR(64)) AS COLLATION_NAME, CAST(CASE WHEN rp.param_type IN (1, 2, 3, 4, 5, 31) THEN CONCAT(lower(v.data_type_str),'(',rp.param_precision,')') WHEN (rp.param_type in (6, 7, 8, 9, 10) AND rp.param_zero_fill) THEN CONCAT(lower(v.data_type_str), ' zerofill') WHEN rp.param_type IN (15,16,50) THEN CONCAT(lower(v.data_type_str),'(',rp.param_precision, ',', rp.param_scale,')') WHEN rp.param_type IN (17, 18, 20) THEN CONCAT(lower(v.data_type_str),'(', rp.param_scale, ')') WHEN (rp.param_type IN (22, 23) AND rp.param_charset != 1) THEN CONCAT(lower(v.data_type_str),'(', rp.param_length, ')') WHEN (rp.param_type IN (22) AND rp.param_charset = 1) THEN CONCAT(lower('VARBINARY'),'(', rp.param_length, ')') WHEN (rp.param_type IN (23) AND rp.param_charset = 1) THEN CONCAT(lower('BINARY'),'(', rp.param_length, ')') WHEN (rp.param_type IN (27, 28, 29, 30) AND rp.param_charset = 1) THEN lower(REPLACE(v.data_type_str, 'TEXT', 'BLOB')) ELSE lower(v.data_type_str) END AS char(4194304)) AS DTD_IDENTIFIER, CAST(CASE WHEN r.routine_type = 1 THEN 'PROCEDURE' WHEN ROUTINE_TYPE = 2 THEN 'FUNCTION' ELSE NULL END AS CHAR(9)) AS ROUTINE_TYPE from oceanbase.__all_routine_param as rp join oceanbase.__all_routine as r on rp.subprogram_id = r.subprogram_id and rp.tenant_id = r.tenant_id and rp.routine_id = r.routine_id join oceanbase.__all_database as d on r.database_id = d.database_id left join oceanbase.__all_virtual_data_type v on rp.param_type = v.data_type WHERE rp.tenant_id = 0 and in_recyclebin = 0 and database_name != '__recyclebin' order by SPECIFIC_SCHEMA, SPECIFIC_NAME, ORDINAL_POSITION )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -2017,7 +2017,7 @@ int ObInnerTableSchema::table_privileges_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( WITH DB_PRIV AS ( select A.tenant_id TENANT_ID, A.user_id USER_ID, A.database_name DATABASE_NAME, A.priv_alter PRIV_ALTER, A.priv_create PRIV_CREATE, A.priv_delete PRIV_DELETE, A.priv_drop PRIV_DROP, A.priv_grant_option PRIV_GRANT_OPTION, A.priv_insert PRIV_INSERT, A.priv_update PRIV_UPDATE, A.priv_select PRIV_SELECT, A.priv_index PRIV_INDEX, A.priv_create_view PRIV_CREATE_VIEW, A.priv_show_view PRIV_SHOW_VIEW, A.GMT_CREATE GMT_CREATE, A.GMT_MODIFIED GMT_MODIFIED, A.PRIV_OTHERS PRIV_OTHERS from oceanbase.__all_database_privilege_history A, (select tenant_id, user_id, database_name, max(schema_version) schema_version from oceanbase.__all_database_privilege_history group by tenant_id, user_id, database_name, database_name collate utf8mb4_bin) B where A.tenant_id = B.tenant_id and A.user_id = B.user_id and A.database_name collate utf8mb4_bin = B.database_name collate utf8mb4_bin and A.schema_version = B.schema_version and A.is_deleted = 0 ), TABLE_PRIV AS ( select A.tenant_id TENANT_ID, A.user_id USER_ID, A.database_name DATABASE_NAME, A.table_name TABLE_NAME, A.priv_alter PRIV_ALTER, A.priv_create PRIV_CREATE, A.priv_delete PRIV_DELETE, A.priv_drop PRIV_DROP, A.priv_grant_option PRIV_GRANT_OPTION, A.priv_insert PRIV_INSERT, A.priv_update PRIV_UPDATE, A.priv_select PRIV_SELECT, A.priv_index PRIV_INDEX, A.priv_create_view PRIV_CREATE_VIEW, A.priv_show_view PRIV_SHOW_VIEW, A.PRIV_OTHERS PRIV_OTHERS from oceanbase.__all_table_privilege_history A, (select tenant_id, user_id, database_name, table_name, max(schema_version) schema_version from oceanbase.__all_table_privilege_history group by tenant_id, user_id, database_name, database_name collate utf8mb4_bin, table_name, table_name collate utf8mb4_bin) B where A.tenant_id = B.tenant_id and A.user_id = B.user_id and A.database_name collate utf8mb4_bin = B.database_name collate utf8mb4_bin and A.schema_version = B.schema_version and A.table_name collate utf8mb4_bin = B.table_name collate utf8mb4_bin and A.is_deleted = 0 ) SELECT CAST(CONCAT('''', V.USER_NAME, '''', '@', '''', V.HOST, '''') AS CHAR(81)) AS GRANTEE , CAST('def' AS CHAR(512)) AS TABLE_CATALOG , CAST(V.DATABASE_NAME AS CHAR(128)) collate utf8mb4_name_case AS TABLE_SCHEMA , CAST(V.TABLE_NAME AS CHAR(64)) collate utf8mb4_name_case AS TABLE_NAME, CAST(V.PRIVILEGE_TYPE AS CHAR(64)) AS PRIVILEGE_TYPE , CAST(V.IS_GRANTABLE AS CHAR(3)) AS IS_GRANTABLE FROM (SELECT TP.DATABASE_NAME AS DATABASE_NAME, TP.TABLE_NAME AS TABLE_NAME, U.USER_NAME AS USER_NAME, U.HOST AS HOST, CASE WHEN V1.C1 = 1 AND TP.PRIV_ALTER = 1 THEN 'ALTER' WHEN V1.C1 = 2 AND TP.PRIV_CREATE = 1 THEN 'CREATE' WHEN V1.C1 = 4 AND TP.PRIV_DELETE = 1 THEN 'DELETE' WHEN V1.C1 = 5 AND TP.PRIV_DROP = 1 THEN 'DROP' WHEN V1.C1 = 7 AND TP.PRIV_INSERT = 1 THEN 'INSERT' WHEN V1.C1 = 8 AND TP.PRIV_UPDATE = 1 THEN 'UPDATE' WHEN V1.C1 = 9 AND TP.PRIV_SELECT = 1 THEN 'SELECT' WHEN V1.C1 = 10 AND TP.PRIV_INDEX = 1 THEN 'INDEX' WHEN V1.C1 = 11 AND TP.PRIV_CREATE_VIEW = 1 THEN 'CREATE VIEW' WHEN V1.C1 = 12 AND TP.PRIV_SHOW_VIEW = 1 THEN 'SHOW VIEW' ELSE NULL END PRIVILEGE_TYPE , CASE WHEN TP.PRIV_GRANT_OPTION = 1 THEN 'YES' WHEN TP.PRIV_GRANT_OPTION = 0 THEN 'NO' END IS_GRANTABLE FROM TABLE_PRIV TP, oceanbase.__all_user U, (SELECT 1 AS C1 UNION ALL SELECT 2 AS C1 UNION ALL SELECT 4 AS C1 UNION ALL SELECT 5 AS C1 UNION ALL SELECT 7 AS C1 UNION ALL SELECT 8 AS C1 UNION ALL SELECT 9 AS C1 UNION ALL SELECT 10 AS C1 UNION ALL SELECT 11 AS C1 UNION ALL SELECT 12 AS C1) V1, (SELECT USER_ID FROM oceanbase.__all_user WHERE TENANT_ID = 0 AND CONCAT(USER_NAME, '@', HOST) = CURRENT_USER()) CURR LEFT JOIN (SELECT USER_ID FROM DB_PRIV WHERE TENANT_ID = 0 AND DATABASE_NAME = 'mysql' AND PRIV_SELECT = 1) DB ON CURR.USER_ID = DB.USER_ID WHERE TP.TENANT_ID = 0 AND TP.TENANT_ID = U.TENANT_ID AND TP.USER_ID = U.USER_ID AND (DB.USER_ID IS NOT NULL OR 512 & CURRENT_USER_PRIV() = 512 OR TP.USER_ID = CURR.USER_ID)) V WHERE V.PRIVILEGE_TYPE IS NOT NULL )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( WITH DB_PRIV AS ( select A.tenant_id TENANT_ID, A.user_id USER_ID, A.database_name DATABASE_NAME, A.priv_alter PRIV_ALTER, A.priv_create PRIV_CREATE, A.priv_delete PRIV_DELETE, A.priv_drop PRIV_DROP, A.priv_grant_option PRIV_GRANT_OPTION, A.priv_insert PRIV_INSERT, A.priv_update PRIV_UPDATE, A.priv_select PRIV_SELECT, A.priv_index PRIV_INDEX, A.priv_create_view PRIV_CREATE_VIEW, A.priv_show_view PRIV_SHOW_VIEW, A.GMT_CREATE GMT_CREATE, A.GMT_MODIFIED GMT_MODIFIED, A.PRIV_OTHERS PRIV_OTHERS from oceanbase.__all_database_privilege_history A, (select tenant_id, user_id, database_name, max(schema_version) schema_version from oceanbase.__all_database_privilege_history group by tenant_id, user_id, database_name, database_name collate utf8mb4_bin) B where A.tenant_id = B.tenant_id and A.user_id = B.user_id and A.database_name collate utf8mb4_bin = B.database_name collate utf8mb4_bin and A.schema_version = B.schema_version and A.is_deleted = 0 ), TABLE_PRIV AS ( select A.tenant_id TENANT_ID, A.user_id USER_ID, A.database_name DATABASE_NAME, A.table_name TABLE_NAME, A.priv_alter PRIV_ALTER, A.priv_create PRIV_CREATE, A.priv_delete PRIV_DELETE, A.priv_drop PRIV_DROP, A.priv_grant_option PRIV_GRANT_OPTION, A.priv_insert PRIV_INSERT, A.priv_update PRIV_UPDATE, A.priv_select PRIV_SELECT, A.priv_index PRIV_INDEX, A.priv_create_view PRIV_CREATE_VIEW, A.priv_show_view PRIV_SHOW_VIEW, A.PRIV_OTHERS PRIV_OTHERS from oceanbase.__all_table_privilege_history A, (select tenant_id, user_id, database_name, table_name, max(schema_version) schema_version from oceanbase.__all_table_privilege_history group by tenant_id, user_id, database_name, database_name collate utf8mb4_bin, table_name, table_name collate utf8mb4_bin) B where A.tenant_id = B.tenant_id and A.user_id = B.user_id and A.database_name collate utf8mb4_bin = B.database_name collate utf8mb4_bin and A.schema_version = B.schema_version and A.table_name collate utf8mb4_bin = B.table_name collate utf8mb4_bin and A.is_deleted = 0 ) SELECT CAST(CONCAT('''', V.USER_NAME, '''', '@', '''', V.HOST, '''') AS CHAR(81)) AS GRANTEE , CAST('def' AS CHAR(512)) AS TABLE_CATALOG , CAST(V.DATABASE_NAME AS CHAR(128)) collate utf8mb4_name_case AS TABLE_SCHEMA , CAST(V.TABLE_NAME AS CHAR(64)) collate utf8mb4_name_case AS TABLE_NAME, CAST(V.PRIVILEGE_TYPE AS CHAR(64)) AS PRIVILEGE_TYPE , CAST(V.IS_GRANTABLE AS CHAR(3)) AS IS_GRANTABLE FROM (SELECT TP.DATABASE_NAME AS DATABASE_NAME, TP.TABLE_NAME AS TABLE_NAME, U.USER_NAME AS USER_NAME, U.HOST AS HOST, CASE WHEN V1.C1 = 1 AND TP.PRIV_ALTER = 1 THEN 'ALTER' WHEN V1.C1 = 2 AND TP.PRIV_CREATE = 1 THEN 'CREATE' WHEN V1.C1 = 4 AND TP.PRIV_DELETE = 1 THEN 'DELETE' WHEN V1.C1 = 5 AND TP.PRIV_DROP = 1 THEN 'DROP' WHEN V1.C1 = 7 AND TP.PRIV_INSERT = 1 THEN 'INSERT' WHEN V1.C1 = 8 AND TP.PRIV_UPDATE = 1 THEN 'UPDATE' WHEN V1.C1 = 9 AND TP.PRIV_SELECT = 1 THEN 'SELECT' WHEN V1.C1 = 10 AND TP.PRIV_INDEX = 1 THEN 'INDEX' WHEN V1.C1 = 11 AND TP.PRIV_CREATE_VIEW = 1 THEN 'CREATE VIEW' WHEN V1.C1 = 12 AND TP.PRIV_SHOW_VIEW = 1 THEN 'SHOW VIEW' WHEN V1.C1 = 22 AND (TP.PRIV_OTHERS & (1 << 6)) != 0 THEN 'REFERENCES' WHEN V1.C1 = 44 AND (TP.PRIV_OTHERS & (1 << 9)) != 0 THEN 'TRIGGER' ELSE NULL END PRIVILEGE_TYPE , CASE WHEN TP.PRIV_GRANT_OPTION = 1 THEN 'YES' WHEN TP.PRIV_GRANT_OPTION = 0 THEN 'NO' END IS_GRANTABLE FROM TABLE_PRIV TP, oceanbase.__all_user U, (SELECT 1 AS C1 UNION ALL SELECT 2 AS C1 UNION ALL SELECT 4 AS C1 UNION ALL SELECT 5 AS C1 UNION ALL SELECT 7 AS C1 UNION ALL SELECT 8 AS C1 UNION ALL SELECT 9 AS C1 UNION ALL SELECT 10 AS C1 UNION ALL SELECT 11 AS C1 UNION ALL SELECT 12 AS C1 UNION ALL SELECT 22 AS C1 UNION ALL SELECT 44 AS C1) V1, (SELECT USER_ID FROM oceanbase.__all_user WHERE TENANT_ID = 0 AND CONCAT(USER_NAME, '@', HOST) = CURRENT_USER()) CURR LEFT JOIN (SELECT USER_ID FROM DB_PRIV WHERE TENANT_ID = 0 AND DATABASE_NAME = 'mysql' AND PRIV_SELECT = 1) DB ON CURR.USER_ID = DB.USER_ID WHERE TP.TENANT_ID = 0 AND TP.TENANT_ID = U.TENANT_ID AND TP.USER_ID = U.USER_ID AND (DB.USER_ID IS NOT NULL OR 512 & CURRENT_USER_PRIV() = 512 OR TP.USER_ID = CURR.USER_ID)) V WHERE V.PRIVILEGE_TYPE IS NOT NULL )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -2067,7 +2067,7 @@ int ObInnerTableSchema::user_privileges_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(CONCAT('''', V.USER_NAME, '''', '@', '''', V.HOST, '''') AS CHAR(81)) AS GRANTEE , CAST('def' AS CHAR(512)) AS TABLE_CATALOG , CAST(V.PRIVILEGE_TYPE AS CHAR(64)) AS PRIVILEGE_TYPE , CAST(V.IS_GRANTABLE AS CHAR(3)) AS IS_GRANTABLE FROM (SELECT U.USER_NAME AS USER_NAME, U.HOST AS HOST, CASE WHEN V1.C1 = 1 AND U.PRIV_ALTER = 1 THEN 'ALTER' WHEN V1.C1 = 2 AND U.PRIV_CREATE = 1 THEN 'CREATE' WHEN V1.C1 = 3 AND U.PRIV_CREATE_USER = 1 THEN 'CREATE USER' WHEN V1.C1 = 4 AND U.PRIV_DELETE = 1 THEN 'DELETE' WHEN V1.C1 = 5 AND U.PRIV_DROP = 1 THEN 'DROP' WHEN V1.C1 = 7 AND U.PRIV_INSERT = 1 THEN 'INSERT' WHEN V1.C1 = 8 AND U.PRIV_UPDATE = 1 THEN 'UPDATE' WHEN V1.C1 = 9 AND U.PRIV_SELECT = 1 THEN 'SELECT' WHEN V1.C1 = 10 AND U.PRIV_INDEX = 1 THEN 'INDEX' WHEN V1.C1 = 11 AND U.PRIV_CREATE_VIEW = 1 THEN 'CREATE VIEW' WHEN V1.C1 = 12 AND U.PRIV_SHOW_VIEW = 1 THEN 'SHOW VIEW' WHEN V1.C1 = 13 AND U.PRIV_SHOW_DB = 1 THEN 'SHOW DATABASES' WHEN V1.C1 = 14 AND U.PRIV_SUPER = 1 THEN 'SUPER' WHEN V1.C1 = 15 AND U.PRIV_PROCESS = 1 THEN 'PROCESS' WHEN V1.C1 = 17 AND U.PRIV_CREATE_SYNONYM = 1 THEN 'CREATE SYNONYM' WHEN V1.C1 = 23 AND (U.PRIV_OTHERS & (1 << 0)) != 0 THEN 'EXECUTE' WHEN V1.C1 = 27 AND U.PRIV_FILE = 1 THEN 'FILE' WHEN V1.C1 = 28 AND U.PRIV_ALTER_TENANT = 1 THEN 'ALTER TENANT' WHEN V1.C1 = 29 AND U.PRIV_ALTER_SYSTEM = 1 THEN 'ALTER SYSTEM' WHEN V1.C1 = 30 AND U.PRIV_CREATE_RESOURCE_POOL = 1 THEN 'CREATE RESOURCE POOL' WHEN V1.C1 = 31 AND U.PRIV_CREATE_RESOURCE_UNIT = 1 THEN 'CREATE RESOURCE UNIT' WHEN V1.C1 = 33 AND U.PRIV_REPL_SLAVE = 1 THEN 'REPLICATION SLAVE' WHEN V1.C1 = 34 AND U.PRIV_REPL_CLIENT = 1 THEN 'REPLICATION CLIENT' WHEN V1.C1 = 35 AND U.PRIV_DROP_DATABASE_LINK = 1 THEN 'DROP DATABASE LINK' WHEN V1.C1 = 36 AND U.PRIV_CREATE_DATABASE_LINK = 1 THEN 'CREATE DATABASE LINK' WHEN V1.C1 = 37 AND (U.PRIV_OTHERS & (1 << 1)) != 0 THEN 'ALTER ROUTINE' WHEN V1.C1 = 38 AND (U.PRIV_OTHERS & (1 << 2)) != 0 THEN 'CREATE ROUTINE' WHEN V1.C1 = 39 AND (U.PRIV_OTHERS & (1 << 3)) != 0 THEN 'CREATE TABLESPACE' WHEN V1.C1 = 40 AND (U.PRIV_OTHERS & (1 << 4)) != 0 THEN 'SHUTDOWN' WHEN V1.C1 = 41 AND (U.PRIV_OTHERS & (1 << 5)) != 0 THEN 'RELOAD' WHEN V1.C1 = 0 AND U.PRIV_ALTER = 0 AND U.PRIV_CREATE = 0 AND U.PRIV_CREATE_USER = 0 AND U.PRIV_DELETE = 0 AND U.PRIV_DROP = 0 AND U.PRIV_INSERT = 0 AND U.PRIV_UPDATE = 0 AND U.PRIV_SELECT = 0 AND U.PRIV_INDEX = 0 AND U.PRIV_CREATE_VIEW = 0 AND U.PRIV_SHOW_VIEW = 0 AND U.PRIV_SHOW_DB = 0 AND U.PRIV_SUPER = 0 AND U.PRIV_PROCESS = 0 AND U.PRIV_CREATE_SYNONYM = 0 AND U.PRIV_FILE = 0 AND U.PRIV_ALTER_TENANT = 0 AND U.PRIV_ALTER_SYSTEM = 0 AND U.PRIV_CREATE_RESOURCE_POOL = 0 AND U.PRIV_CREATE_RESOURCE_UNIT = 0 AND U.PRIV_REPL_SLAVE = 0 AND U.PRIV_REPL_CLIENT = 0 AND U.PRIV_DROP_DATABASE_LINK = 0 AND U.PRIV_CREATE_DATABASE_LINK = 0 AND U.PRIV_OTHERS = 0 THEN 'USAGE' END PRIVILEGE_TYPE , CASE WHEN U.PRIV_GRANT_OPTION = 0 THEN 'NO' WHEN U.PRIV_ALTER = 0 AND U.PRIV_CREATE = 0 AND U.PRIV_CREATE_USER = 0 AND U.PRIV_DELETE = 0 AND U.PRIV_DROP = 0 AND U.PRIV_INSERT = 0 AND U.PRIV_UPDATE = 0 AND U.PRIV_SELECT = 0 AND U.PRIV_INDEX = 0 AND U.PRIV_CREATE_VIEW = 0 AND U.PRIV_SHOW_VIEW = 0 AND U.PRIV_SHOW_DB = 0 AND U.PRIV_SUPER = 0 AND U.PRIV_PROCESS = 0 AND U.PRIV_CREATE_SYNONYM = 0 AND U.PRIV_FILE = 0 AND U.PRIV_ALTER_TENANT = 0 AND U.PRIV_ALTER_SYSTEM = 0 AND U.PRIV_CREATE_RESOURCE_POOL = 0 AND U.PRIV_CREATE_RESOURCE_UNIT = 0 AND U.PRIV_REPL_SLAVE = 0 AND U.PRIV_REPL_CLIENT = 0 AND U.PRIV_DROP_DATABASE_LINK = 0 AND U.PRIV_CREATE_DATABASE_LINK = 0 AND U.PRIV_OTHERS = 0 THEN 'NO' WHEN U.PRIV_GRANT_OPTION = 1 THEN 'YES' END IS_GRANTABLE FROM oceanbase.__all_user U, (SELECT 0 AS C1 UNION ALL SELECT 1 AS C1 UNION ALL SELECT 2 AS C1 UNION ALL SELECT 3 AS C1 UNION ALL SELECT 4 AS C1 UNION ALL SELECT 5 AS C1 UNION ALL SELECT 7 AS C1 UNION ALL SELECT 8 AS C1 UNION ALL SELECT 9 AS C1 UNION ALL SELECT 10 AS C1 UNION ALL SELECT 11 AS C1 UNION ALL SELECT 12 AS C1 UNION ALL SELECT 13 AS C1 UNION ALL SELECT 14 AS C1 UNION ALL SELECT 15 AS C1 UNION ALL SELECT 17 AS C1 UNION ALL SELECT 23 AS C1 UNION ALL SELECT 27 AS C1 UNION ALL SELECT 28 AS C1 UNION ALL SELECT 29 AS C1 UNION ALL SELECT 30 AS C1 UNION ALL SELECT 31 AS C1 UNION ALL SELECT 33 AS C1 UNION ALL SELECT 34 AS C1 UNION ALL SELECT 35 AS C1 UNION ALL SELECT 36 AS C1 UNION ALL SELECT 37 AS C1 UNION ALL SELECT 38 AS C1 UNION ALL SELECT 39 AS C1 UNION ALL SELECT 40 AS C1 UNION ALL SELECT 41 AS C1) V1, (SELECT USER_ID FROM oceanbase.__all_user WHERE TENANT_ID = 0 AND CONCAT(USER_NAME, '@', HOST) = CURRENT_USER()) CURR LEFT JOIN (SELECT USER_ID FROM oceanbase.__all_database_privilege WHERE TENANT_ID = 0 AND DATABASE_NAME = 'mysql' AND PRIV_SELECT = 1) DB ON CURR.USER_ID = DB.USER_ID WHERE U.TENANT_ID = 0 AND (DB.USER_ID IS NOT NULL OR 512 & CURRENT_USER_PRIV() = 512 OR U.USER_ID = CURR.USER_ID)) V WHERE V.PRIVILEGE_TYPE IS NOT NULL )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(CONCAT('''', V.USER_NAME, '''', '@', '''', V.HOST, '''') AS CHAR(81)) AS GRANTEE , CAST('def' AS CHAR(512)) AS TABLE_CATALOG , CAST(V.PRIVILEGE_TYPE AS CHAR(64)) AS PRIVILEGE_TYPE , CAST(V.IS_GRANTABLE AS CHAR(3)) AS IS_GRANTABLE FROM (SELECT U.USER_NAME AS USER_NAME, U.HOST AS HOST, CASE WHEN V1.C1 = 1 AND U.PRIV_ALTER = 1 THEN 'ALTER' WHEN V1.C1 = 2 AND U.PRIV_CREATE = 1 THEN 'CREATE' WHEN V1.C1 = 3 AND U.PRIV_CREATE_USER = 1 THEN 'CREATE USER' WHEN V1.C1 = 4 AND U.PRIV_DELETE = 1 THEN 'DELETE' WHEN V1.C1 = 5 AND U.PRIV_DROP = 1 THEN 'DROP' WHEN V1.C1 = 7 AND U.PRIV_INSERT = 1 THEN 'INSERT' WHEN V1.C1 = 8 AND U.PRIV_UPDATE = 1 THEN 'UPDATE' WHEN V1.C1 = 9 AND U.PRIV_SELECT = 1 THEN 'SELECT' WHEN V1.C1 = 10 AND U.PRIV_INDEX = 1 THEN 'INDEX' WHEN V1.C1 = 11 AND U.PRIV_CREATE_VIEW = 1 THEN 'CREATE VIEW' WHEN V1.C1 = 12 AND U.PRIV_SHOW_VIEW = 1 THEN 'SHOW VIEW' WHEN V1.C1 = 13 AND U.PRIV_SHOW_DB = 1 THEN 'SHOW DATABASES' WHEN V1.C1 = 14 AND U.PRIV_SUPER = 1 THEN 'SUPER' WHEN V1.C1 = 15 AND U.PRIV_PROCESS = 1 THEN 'PROCESS' WHEN V1.C1 = 17 AND U.PRIV_CREATE_SYNONYM = 1 THEN 'CREATE SYNONYM' WHEN V1.C1 = 22 AND (U.PRIV_OTHERS & (1 << 6)) != 0 THEN 'REFERENCES' WHEN V1.C1 = 23 AND (U.PRIV_OTHERS & (1 << 0)) != 0 THEN 'EXECUTE' WHEN V1.C1 = 27 AND U.PRIV_FILE = 1 THEN 'FILE' WHEN V1.C1 = 28 AND U.PRIV_ALTER_TENANT = 1 THEN 'ALTER TENANT' WHEN V1.C1 = 29 AND U.PRIV_ALTER_SYSTEM = 1 THEN 'ALTER SYSTEM' WHEN V1.C1 = 30 AND U.PRIV_CREATE_RESOURCE_POOL = 1 THEN 'CREATE RESOURCE POOL' WHEN V1.C1 = 31 AND U.PRIV_CREATE_RESOURCE_UNIT = 1 THEN 'CREATE RESOURCE UNIT' WHEN V1.C1 = 33 AND U.PRIV_REPL_SLAVE = 1 THEN 'REPLICATION SLAVE' WHEN V1.C1 = 34 AND U.PRIV_REPL_CLIENT = 1 THEN 'REPLICATION CLIENT' WHEN V1.C1 = 35 AND U.PRIV_DROP_DATABASE_LINK = 1 THEN 'DROP DATABASE LINK' WHEN V1.C1 = 36 AND U.PRIV_CREATE_DATABASE_LINK = 1 THEN 'CREATE DATABASE LINK' WHEN V1.C1 = 37 AND (U.PRIV_OTHERS & (1 << 1)) != 0 THEN 'ALTER ROUTINE' WHEN V1.C1 = 38 AND (U.PRIV_OTHERS & (1 << 2)) != 0 THEN 'CREATE ROUTINE' WHEN V1.C1 = 39 AND (U.PRIV_OTHERS & (1 << 3)) != 0 THEN 'CREATE TABLESPACE' WHEN V1.C1 = 40 AND (U.PRIV_OTHERS & (1 << 4)) != 0 THEN 'SHUTDOWN' WHEN V1.C1 = 41 AND (U.PRIV_OTHERS & (1 << 5)) != 0 THEN 'RELOAD' WHEN V1.C1 = 42 AND (U.PRIV_OTHERS & (1 << 7)) != 0 THEN 'CREATE ROLE' WHEN V1.C1 = 43 AND (U.PRIV_OTHERS & (1 << 8)) != 0 THEN 'DROP ROLE' WHEN V1.C1 = 44 AND (U.PRIV_OTHERS & (1 << 9)) != 0 THEN 'TRIGGER' WHEN V1.C1 = 0 AND U.PRIV_ALTER = 0 AND U.PRIV_CREATE = 0 AND U.PRIV_CREATE_USER = 0 AND U.PRIV_DELETE = 0 AND U.PRIV_DROP = 0 AND U.PRIV_INSERT = 0 AND U.PRIV_UPDATE = 0 AND U.PRIV_SELECT = 0 AND U.PRIV_INDEX = 0 AND U.PRIV_CREATE_VIEW = 0 AND U.PRIV_SHOW_VIEW = 0 AND U.PRIV_SHOW_DB = 0 AND U.PRIV_SUPER = 0 AND U.PRIV_PROCESS = 0 AND U.PRIV_CREATE_SYNONYM = 0 AND U.PRIV_FILE = 0 AND U.PRIV_ALTER_TENANT = 0 AND U.PRIV_ALTER_SYSTEM = 0 AND U.PRIV_CREATE_RESOURCE_POOL = 0 AND U.PRIV_CREATE_RESOURCE_UNIT = 0 AND U.PRIV_REPL_SLAVE = 0 AND U.PRIV_REPL_CLIENT = 0 AND U.PRIV_DROP_DATABASE_LINK = 0 AND U.PRIV_CREATE_DATABASE_LINK = 0 AND U.PRIV_OTHERS = 0 THEN 'USAGE' END PRIVILEGE_TYPE , CASE WHEN U.PRIV_GRANT_OPTION = 0 THEN 'NO' WHEN U.PRIV_ALTER = 0 AND U.PRIV_CREATE = 0 AND U.PRIV_CREATE_USER = 0 AND U.PRIV_DELETE = 0 AND U.PRIV_DROP = 0 AND U.PRIV_INSERT = 0 AND U.PRIV_UPDATE = 0 AND U.PRIV_SELECT = 0 AND U.PRIV_INDEX = 0 AND U.PRIV_CREATE_VIEW = 0 AND U.PRIV_SHOW_VIEW = 0 AND U.PRIV_SHOW_DB = 0 AND U.PRIV_SUPER = 0 AND U.PRIV_PROCESS = 0 AND U.PRIV_CREATE_SYNONYM = 0 AND U.PRIV_FILE = 0 AND U.PRIV_ALTER_TENANT = 0 AND U.PRIV_ALTER_SYSTEM = 0 AND U.PRIV_CREATE_RESOURCE_POOL = 0 AND U.PRIV_CREATE_RESOURCE_UNIT = 0 AND U.PRIV_REPL_SLAVE = 0 AND U.PRIV_REPL_CLIENT = 0 AND U.PRIV_DROP_DATABASE_LINK = 0 AND U.PRIV_CREATE_DATABASE_LINK = 0 AND U.PRIV_OTHERS = 0 THEN 'NO' WHEN U.PRIV_GRANT_OPTION = 1 THEN 'YES' END IS_GRANTABLE FROM oceanbase.__all_user U, (SELECT 0 AS C1 UNION ALL SELECT 1 AS C1 UNION ALL SELECT 2 AS C1 UNION ALL SELECT 3 AS C1 UNION ALL SELECT 4 AS C1 UNION ALL SELECT 5 AS C1 UNION ALL SELECT 7 AS C1 UNION ALL SELECT 8 AS C1 UNION ALL SELECT 9 AS C1 UNION ALL SELECT 10 AS C1 UNION ALL SELECT 11 AS C1 UNION ALL SELECT 12 AS C1 UNION ALL SELECT 13 AS C1 UNION ALL SELECT 14 AS C1 UNION ALL SELECT 15 AS C1 UNION ALL SELECT 17 AS C1 UNION ALL SELECT 22 AS C1 UNION ALL SELECT 23 AS C1 UNION ALL SELECT 27 AS C1 UNION ALL SELECT 28 AS C1 UNION ALL SELECT 29 AS C1 UNION ALL SELECT 30 AS C1 UNION ALL SELECT 31 AS C1 UNION ALL SELECT 33 AS C1 UNION ALL SELECT 34 AS C1 UNION ALL SELECT 35 AS C1 UNION ALL SELECT 36 AS C1 UNION ALL SELECT 37 AS C1 UNION ALL SELECT 38 AS C1 UNION ALL SELECT 39 AS C1 UNION ALL SELECT 40 AS C1 UNION ALL SELECT 41 AS C1 UNION ALL SELECT 42 AS C1 UNION ALL SELECT 43 AS C1 UNION ALL SELECT 44 AS C1) V1, (SELECT USER_ID FROM oceanbase.__all_user WHERE TENANT_ID = 0 AND CONCAT(USER_NAME, '@', HOST) = CURRENT_USER()) CURR LEFT JOIN (SELECT USER_ID FROM oceanbase.__all_database_privilege WHERE TENANT_ID = 0 AND DATABASE_NAME = 'mysql' AND PRIV_SELECT = 1) DB ON CURR.USER_ID = DB.USER_ID WHERE U.TENANT_ID = 0 AND (DB.USER_ID IS NOT NULL OR 512 & CURRENT_USER_PRIV() = 512 OR U.USER_ID = CURR.USER_ID)) V WHERE V.PRIVILEGE_TYPE IS NOT NULL )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -2117,7 +2117,7 @@ int ObInnerTableSchema::schema_privileges_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( WITH DB_PRIV AS ( select A.tenant_id TENANT_ID, A.user_id USER_ID, A.database_name DATABASE_NAME, A.priv_alter PRIV_ALTER, A.priv_create PRIV_CREATE, A.priv_delete PRIV_DELETE, A.priv_drop PRIV_DROP, A.priv_grant_option PRIV_GRANT_OPTION, A.priv_insert PRIV_INSERT, A.priv_update PRIV_UPDATE, A.priv_select PRIV_SELECT, A.priv_index PRIV_INDEX, A.priv_create_view PRIV_CREATE_VIEW, A.priv_show_view PRIV_SHOW_VIEW, A.priv_others PRIV_OTHERS from oceanbase.__all_database_privilege_history A, (select tenant_id, user_id, database_name, max(schema_version) schema_version from oceanbase.__all_database_privilege_history group by tenant_id, user_id, database_name, database_name collate utf8mb4_bin) B where A.tenant_id = B.tenant_id and A.user_id = B.user_id and A.database_name collate utf8mb4_bin = B.database_name collate utf8mb4_bin and A.schema_version = B.schema_version and A.is_deleted = 0 ) SELECT CAST(CONCAT('''', V.USER_NAME, '''', '@', '''', V.HOST, '''') AS CHAR(81)) AS GRANTEE , CAST('def' AS CHAR(512)) AS TABLE_CATALOG , CAST(V.DATABASE_NAME AS CHAR(128)) collate utf8mb4_name_case AS TABLE_SCHEMA , CAST(V.PRIVILEGE_TYPE AS CHAR(64)) AS PRIVILEGE_TYPE , CAST(V.IS_GRANTABLE AS CHAR(3)) AS IS_GRANTABLE FROM (SELECT DP.DATABASE_NAME DATABASE_NAME, U.USER_NAME AS USER_NAME, U.HOST AS HOST, CASE WHEN V1.C1 = 1 AND DP.PRIV_ALTER = 1 THEN 'ALTER' WHEN V1.C1 = 2 AND DP.PRIV_CREATE = 1 THEN 'CREATE' WHEN V1.C1 = 4 AND DP.PRIV_DELETE = 1 THEN 'DELETE' WHEN V1.C1 = 5 AND DP.PRIV_DROP = 1 THEN 'DROP' WHEN V1.C1 = 7 AND DP.PRIV_INSERT = 1 THEN 'INSERT' WHEN V1.C1 = 8 AND DP.PRIV_UPDATE = 1 THEN 'UPDATE' WHEN V1.C1 = 9 AND DP.PRIV_SELECT = 1 THEN 'SELECT' WHEN V1.C1 = 10 AND DP.PRIV_INDEX = 1 THEN 'INDEX' WHEN V1.C1 = 11 AND DP.PRIV_CREATE_VIEW = 1 THEN 'CREATE VIEW' WHEN V1.C1 = 12 AND DP.PRIV_SHOW_VIEW = 1 THEN 'SHOW VIEW' WHEN V1.C1 = 13 AND (U.PRIV_OTHERS & (1 << 0)) != 0 THEN 'EXECUTE' WHEN V1.C1 = 14 AND (U.PRIV_OTHERS & (1 << 1)) != 0 THEN 'ALTER ROUTINE' WHEN V1.C1 = 15 AND (U.PRIV_OTHERS & (1 << 2)) != 0 THEN 'CREATE ROUTINE' ELSE NULL END PRIVILEGE_TYPE , CASE WHEN DP.PRIV_GRANT_OPTION = 1 THEN 'YES' WHEN DP.PRIV_GRANT_OPTION = 0 THEN 'NO' END IS_GRANTABLE FROM DB_PRIV DP, oceanbase.__all_user U, (SELECT 1 AS C1 UNION ALL SELECT 2 AS C1 UNION ALL SELECT 4 AS C1 UNION ALL SELECT 5 AS C1 UNION ALL SELECT 7 AS C1 UNION ALL SELECT 8 AS C1 UNION ALL SELECT 9 AS C1 UNION ALL SELECT 10 AS C1 UNION ALL SELECT 11 AS C1 UNION ALL SELECT 12 AS C1 UNION ALL SELECT 13 AS C1 UNION ALL SELECT 14 AS C1 UNION ALL SELECT 15 AS C1) V1, (SELECT USER_ID FROM oceanbase.__all_user WHERE TENANT_ID= 0 AND CONCAT(USER_NAME, '@', HOST) = CURRENT_USER()) CURR LEFT JOIN (SELECT USER_ID FROM DB_PRIV WHERE TENANT_ID = 0 AND DATABASE_NAME = 'mysql' AND PRIV_SELECT = 1) DB ON CURR.USER_ID = DB.USER_ID WHERE DP.TENANT_ID = 0 AND DP.TENANT_ID = U.TENANT_ID AND DP.USER_ID = U.USER_ID AND DP.DATABASE_NAME != '__recyclebin' AND DP.DATABASE_NAME != '__public' AND DP.DATABASE_NAME != 'SYS' AND DP.DATABASE_NAME != 'LBACSYS' AND DP.DATABASE_NAME != 'ORAAUDITOR' AND (DB.USER_ID IS NOT NULL OR 512 & CURRENT_USER_PRIV() = 512 OR DP.USER_ID = CURR.USER_ID)) V WHERE V.PRIVILEGE_TYPE IS NOT NULL )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( WITH DB_PRIV AS ( select A.tenant_id TENANT_ID, A.user_id USER_ID, A.database_name DATABASE_NAME, A.priv_alter PRIV_ALTER, A.priv_create PRIV_CREATE, A.priv_delete PRIV_DELETE, A.priv_drop PRIV_DROP, A.priv_grant_option PRIV_GRANT_OPTION, A.priv_insert PRIV_INSERT, A.priv_update PRIV_UPDATE, A.priv_select PRIV_SELECT, A.priv_index PRIV_INDEX, A.priv_create_view PRIV_CREATE_VIEW, A.priv_show_view PRIV_SHOW_VIEW, A.priv_others PRIV_OTHERS from oceanbase.__all_database_privilege_history A, (select tenant_id, user_id, database_name, max(schema_version) schema_version from oceanbase.__all_database_privilege_history group by tenant_id, user_id, database_name, database_name collate utf8mb4_bin) B where A.tenant_id = B.tenant_id and A.user_id = B.user_id and A.database_name collate utf8mb4_bin = B.database_name collate utf8mb4_bin and A.schema_version = B.schema_version and A.is_deleted = 0 ) SELECT CAST(CONCAT('''', V.USER_NAME, '''', '@', '''', V.HOST, '''') AS CHAR(81)) AS GRANTEE , CAST('def' AS CHAR(512)) AS TABLE_CATALOG , CAST(V.DATABASE_NAME AS CHAR(128)) collate utf8mb4_name_case AS TABLE_SCHEMA , CAST(V.PRIVILEGE_TYPE AS CHAR(64)) AS PRIVILEGE_TYPE , CAST(V.IS_GRANTABLE AS CHAR(3)) AS IS_GRANTABLE FROM (SELECT DP.DATABASE_NAME DATABASE_NAME, U.USER_NAME AS USER_NAME, U.HOST AS HOST, CASE WHEN V1.C1 = 1 AND DP.PRIV_ALTER = 1 THEN 'ALTER' WHEN V1.C1 = 2 AND DP.PRIV_CREATE = 1 THEN 'CREATE' WHEN V1.C1 = 4 AND DP.PRIV_DELETE = 1 THEN 'DELETE' WHEN V1.C1 = 5 AND DP.PRIV_DROP = 1 THEN 'DROP' WHEN V1.C1 = 7 AND DP.PRIV_INSERT = 1 THEN 'INSERT' WHEN V1.C1 = 8 AND DP.PRIV_UPDATE = 1 THEN 'UPDATE' WHEN V1.C1 = 9 AND DP.PRIV_SELECT = 1 THEN 'SELECT' WHEN V1.C1 = 10 AND DP.PRIV_INDEX = 1 THEN 'INDEX' WHEN V1.C1 = 11 AND DP.PRIV_CREATE_VIEW = 1 THEN 'CREATE VIEW' WHEN V1.C1 = 12 AND DP.PRIV_SHOW_VIEW = 1 THEN 'SHOW VIEW' WHEN V1.C1 = 22 AND (DP.PRIV_OTHERS & (1 << 6)) != 0 THEN 'REFERENCES' WHEN V1.C1 = 23 AND (DP.PRIV_OTHERS & (1 << 0)) != 0 THEN 'EXECUTE' WHEN V1.C1 = 37 AND (DP.PRIV_OTHERS & (1 << 1)) != 0 THEN 'ALTER ROUTINE' WHEN V1.C1 = 38 AND (DP.PRIV_OTHERS & (1 << 2)) != 0 THEN 'CREATE ROUTINE' WHEN V1.C1 = 44 AND (DP.PRIV_OTHERS & (1 << 9)) != 0 THEN 'TRIGGER' ELSE NULL END PRIVILEGE_TYPE , CASE WHEN DP.PRIV_GRANT_OPTION = 1 THEN 'YES' WHEN DP.PRIV_GRANT_OPTION = 0 THEN 'NO' END IS_GRANTABLE FROM DB_PRIV DP, oceanbase.__all_user U, (SELECT 1 AS C1 UNION ALL SELECT 2 AS C1 UNION ALL SELECT 4 AS C1 UNION ALL SELECT 5 AS C1 UNION ALL SELECT 7 AS C1 UNION ALL SELECT 8 AS C1 UNION ALL SELECT 9 AS C1 UNION ALL SELECT 10 AS C1 UNION ALL SELECT 11 AS C1 UNION ALL SELECT 12 AS C1 UNION ALL SELECT 22 AS C1 UNION ALL SELECT 23 AS C1 UNION ALL SELECT 37 AS C1 UNION ALL SELECT 38 AS C1 UNION ALL SELECT 44 AS C1) V1, (SELECT USER_ID FROM oceanbase.__all_user WHERE TENANT_ID= 0 AND CONCAT(USER_NAME, '@', HOST) = CURRENT_USER()) CURR LEFT JOIN (SELECT USER_ID FROM DB_PRIV WHERE TENANT_ID = 0 AND DATABASE_NAME = 'mysql' AND PRIV_SELECT = 1) DB ON CURR.USER_ID = DB.USER_ID WHERE DP.TENANT_ID = 0 AND DP.TENANT_ID = U.TENANT_ID AND DP.USER_ID = U.USER_ID AND DP.DATABASE_NAME != '__recyclebin' AND DP.DATABASE_NAME != '__public' AND DP.DATABASE_NAME != 'SYS' AND DP.DATABASE_NAME != 'LBACSYS' AND DP.DATABASE_NAME != 'ORAAUDITOR' AND (DB.USER_ID IS NOT NULL OR 512 & CURRENT_USER_PRIV() = 512 OR DP.USER_ID = CURR.USER_ID)) V WHERE V.PRIVILEGE_TYPE IS NOT NULL )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema.21351_21400.cpp b/src/share/inner_table/ob_inner_table_schema.21351_21400.cpp index cc47533d5..0e6b75c05 100644 --- a/src/share/inner_table/ob_inner_table_schema.21351_21400.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21351_21400.cpp @@ -310,7 +310,7 @@ int ObInnerTableSchema::triggers_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT CAST('def' AS CHAR(512)) AS TRIGGER_CATALOG, CAST(db.database_name AS CHAR(64)) collate utf8mb4_name_case AS TRIGGER_SCHEMA, CAST(trg.trigger_name AS CHAR(64)) AS TRIGGER_NAME, CAST((case when trg.trigger_events=1 then 'INSERT' when trg.trigger_events=2 then 'UPDATE' when trg.trigger_events=4 then 'DELETE' end) AS CHAR(6)) AS EVENT_MANIPULATION, CAST('def' AS CHAR(512)) AS EVENT_OBJECT_CATALOG, CAST(db.database_name AS CHAR(64)) collate utf8mb4_name_case AS EVENT_OBJECT_SCHEMA, CAST(t.table_name AS CHAR(64)) collate utf8mb4_name_case AS EVENT_OBJECT_TABLE, CAST(trg.action_order AS SIGNED) AS ACTION_ORDER, CAST(NULL AS CHAR(4194304)) AS ACTION_CONDITION, CAST(trg.trigger_body AS CHAR(4194304)) AS ACTION_STATEMENT, CAST('ROW' AS CHAR(9)) AS ACTION_ORIENTATION, CAST((case when trg.TIMING_POINTS=4 then 'BEFORE' when trg.TIMING_POINTS=8 then 'AFTER' end) AS CHAR(6)) AS ACTION_TIMING, CAST(NULL AS CHAR(64)) AS ACTION_REFERENCE_OLD_TABLE, CAST(NULL AS CHAR(64)) AS ACTION_REFERENCE_NEW_TABLE, CAST('OLD' AS CHAR(3)) AS ACTION_REFERENCE_OLD_ROW, CAST('NEW' AS CHAR(3)) AS ACTION_REFERENCE_NEW_ROW, CAST(trg.gmt_create AS DATETIME(2)) AS CREATED, CAST(sql_mode_convert(trg.sql_mode) AS CHAR(8192)) AS SQL_MODE, CAST(trg.trigger_priv_user AS CHAR(93)) AS DEFINER, CAST((select charset from oceanbase.__tenant_virtual_collation where id = substring_index(substring_index(trg.package_exec_env, ',', 2), ',', -1)) AS CHAR(32) ) AS CHARACTER_SET_CLIENT, CAST((select collation from oceanbase.__tenant_virtual_collation where collation_type = substring_index(substring_index(trg.package_exec_env, ',', 3), ',', -1)) AS CHAR(32) ) AS COLLATION_CONNECTION, CAST((select collation from oceanbase.__tenant_virtual_collation where collation_type = substring_index(substring_index(trg.package_exec_env, ',', 4), ',', -1)) AS CHAR(32) ) AS DATABASE_COLLATION FROM oceanbase.__all_tenant_trigger trg JOIN oceanbase.__all_database db on trg.database_id = db.database_id JOIN oceanbase.__all_table t on trg.base_object_id = t.table_id WHERE db.database_name != '__recyclebin' and db.in_recyclebin = 0 and t.table_mode >> 12 & 15 in (0,1) )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT CAST('def' AS CHAR(512)) AS TRIGGER_CATALOG, CAST(db.database_name AS CHAR(64)) collate utf8mb4_name_case AS TRIGGER_SCHEMA, CAST(trg.trigger_name AS CHAR(64)) AS TRIGGER_NAME, CAST((case when trg.trigger_events=1 then 'INSERT' when trg.trigger_events=2 then 'UPDATE' when trg.trigger_events=4 then 'DELETE' end) AS CHAR(6)) AS EVENT_MANIPULATION, CAST('def' AS CHAR(512)) AS EVENT_OBJECT_CATALOG, CAST(db.database_name AS CHAR(64)) collate utf8mb4_name_case AS EVENT_OBJECT_SCHEMA, CAST(t.table_name AS CHAR(64)) collate utf8mb4_name_case AS EVENT_OBJECT_TABLE, CAST(trg.action_order AS SIGNED) AS ACTION_ORDER, CAST(NULL AS CHAR(4194304)) AS ACTION_CONDITION, CAST(trg.trigger_body AS CHAR(4194304)) AS ACTION_STATEMENT, CAST('ROW' AS CHAR(9)) AS ACTION_ORIENTATION, CAST((case when trg.TIMING_POINTS=4 then 'BEFORE' when trg.TIMING_POINTS=8 then 'AFTER' end) AS CHAR(6)) AS ACTION_TIMING, CAST(NULL AS CHAR(64)) AS ACTION_REFERENCE_OLD_TABLE, CAST(NULL AS CHAR(64)) AS ACTION_REFERENCE_NEW_TABLE, CAST('OLD' AS CHAR(3)) AS ACTION_REFERENCE_OLD_ROW, CAST('NEW' AS CHAR(3)) AS ACTION_REFERENCE_NEW_ROW, CAST(trg.gmt_create AS DATETIME(2)) AS CREATED, CAST(sql_mode_convert(trg.sql_mode) AS CHAR(8192)) AS SQL_MODE, CAST(trg.trigger_priv_user AS CHAR(93)) AS DEFINER, CAST((select charset from oceanbase.__tenant_virtual_collation where id = substring_index(substring_index(trg.package_exec_env, ',', 2), ',', -1)) AS CHAR(32) ) AS CHARACTER_SET_CLIENT, CAST((select collation from oceanbase.__tenant_virtual_collation where collation_type = substring_index(substring_index(trg.package_exec_env, ',', 3), ',', -1)) AS CHAR(32) ) AS COLLATION_CONNECTION, CAST((select collation from oceanbase.__tenant_virtual_collation where collation_type = substring_index(substring_index(trg.package_exec_env, ',', 4), ',', -1)) AS CHAR(32) ) AS DATABASE_COLLATION FROM oceanbase.__all_tenant_trigger trg JOIN oceanbase.__all_database db on trg.database_id = db.database_id JOIN oceanbase.__all_table t on trg.base_object_id = t.table_id WHERE db.database_name != '__recyclebin' and db.in_recyclebin = 0 and t.table_mode >> 12 & 15 in (0,1) and can_access_trigger(db.database_name, t.table_name) )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -360,7 +360,7 @@ int ObInnerTableSchema::partitions_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT CAST('def' as CHAR(4096)) AS TABLE_CATALOG, DB.DATABASE_NAME collate utf8mb4_name_case AS TABLE_SCHEMA, T.TABLE_NAME collate utf8mb4_name_case AS TABLE_NAME, P.PART_NAME AS PARTITION_NAME, SP.SUB_PART_NAME AS SUBPARTITION_NAME, CAST(PART_POSITION AS UNSIGNED) AS PARTITION_ORDINAL_POSITION, CAST(SUB_PART_POSITION AS UNSIGNED) AS SUBPARTITION_ORDINAL_POSITION, CAST(CASE WHEN T.PART_LEVEL = 0 THEN NULL ELSE (CASE T.PART_FUNC_TYPE WHEN 0 THEN 'HASH' WHEN 1 THEN 'KEY' WHEN 2 THEN 'KEY' WHEN 3 THEN 'RANGE' WHEN 4 THEN 'RANGE COLUMNS' WHEN 5 THEN 'LIST' WHEN 6 THEN 'LIST COLUMNS' WHEN 7 THEN 'RANGE' END) END AS CHAR(13)) PARTITION_METHOD, CAST(CASE WHEN (T.PART_LEVEL = 0 OR T.PART_LEVEL = 1) THEN NULL ELSE (CASE T.SUB_PART_FUNC_TYPE WHEN 0 THEN 'HASH' WHEN 1 THEN 'KEY' WHEN 2 THEN 'KEY' WHEN 3 THEN 'RANGE' WHEN 4 THEN 'RANGE COLUMNS' WHEN 5 THEN 'LIST' WHEN 6 THEN 'LIST COLUMNS' WHEN 7 THEN 'RANGE' END) END AS CHAR(13)) SUBPARTITION_METHOD, CAST(CASE WHEN (T.PART_LEVEL = 0) THEN NULL ELSE T.PART_FUNC_EXPR END AS CHAR(2048)) PARTITION_EXPRESSION, CAST(CASE WHEN (T.PART_LEVEL = 0 OR T.PART_LEVEL = 1) THEN NULL ELSE T.SUB_PART_FUNC_EXPR END AS CHAR(2048)) SUBPARTITION_EXPRESSION, CAST(CASE WHEN (T.PART_LEVEL = 0) THEN NULL ELSE (CASE WHEN LENGTH(P.HIGH_BOUND_VAL) > 0 THEN P.HIGH_BOUND_VAL ELSE P.LIST_VAL END) END AS CHAR(4096)) AS PARTITION_DESCRIPTION, CAST(CASE WHEN (T.PART_LEVEL = 0 OR T.PART_LEVEL = 1) THEN NULL ELSE (CASE WHEN LENGTH(SP.HIGH_BOUND_VAL) > 0 THEN SP.HIGH_BOUND_VAL ELSE SP.LIST_VAL END) END AS CHAR(4096)) AS SUBPARTITION_DESCRIPTION, CAST(TS.ROW_CNT AS UNSIGNED) AS TABLE_ROWS, CAST(TS.AVG_ROW_LEN AS UNSIGNED) AS AVG_ROW_LENGTH, CAST(NULL AS UNSIGNED) AS DATA_LENGTH, CAST(NULL AS UNSIGNED) AS MAX_DATA_LENGTH, CAST(NULL AS UNSIGNED) AS INDEX_LENGTH, CAST(NULL AS UNSIGNED) AS DATA_FREE, CASE T.PART_LEVEL WHEN 0 THEN T.GMT_CREATE WHEN 1 THEN P.GMT_CREATE WHEN 2 THEN SP.GMT_CREATE END AS CREATE_TIME, CAST(NULL AS DATETIME) AS UPDATE_TIME, CAST(NULL AS DATETIME) AS CHECK_TIME, CAST(NULL AS SIGNED) AS CHECKSUM, CAST(CASE T.PART_LEVEL WHEN 0 THEN NULL WHEN 1 THEN P.COMMENT WHEN 2 THEN SP.COMMENT END AS CHAR(1024)) AS PARTITION_COMMENT, CAST('default' AS CHAR(256)) NODEGROUP, CAST(TP.TABLESPACE_NAME AS CHAR(268)) AS TABLESPACE_NAME FROM OCEANBASE.__ALL_TABLE T JOIN OCEANBASE.__ALL_DATABASE DB ON T.DATABASE_ID = DB.DATABASE_ID AND T.TENANT_ID = DB.TENANT_ID AND T.TABLE_MODE >> 12 & 15 in (0,1) LEFT JOIN ( SELECT TENANT_ID, TABLE_ID, PART_ID, PART_NAME, HIGH_BOUND_VAL, LIST_VAL, TABLESPACE_ID, GMT_CREATE, COMMENT, ROW_NUMBER() OVER(PARTITION BY TENANT_ID,TABLE_ID ORDER BY PART_IDX) AS PART_POSITION FROM OCEANBASE.__ALL_PART ) P ON T.TABLE_ID = P.TABLE_ID AND T.TENANT_ID = P.TENANT_ID LEFT JOIN ( SELECT TENANT_ID, TABLE_ID, PART_ID, SUB_PART_ID, SUB_PART_NAME, HIGH_BOUND_VAL, LIST_VAL, TABLESPACE_ID, GMT_CREATE, COMMENT, ROW_NUMBER() OVER(PARTITION BY TENANT_ID,TABLE_ID,PART_ID ORDER BY SUB_PART_IDX) AS SUB_PART_POSITION FROM OCEANBASE.__ALL_SUB_PART ) SP ON T.TABLE_ID = SP.TABLE_ID AND P.PART_ID = SP.PART_ID AND T.TENANT_ID = SP.TENANT_ID LEFT JOIN OCEANBASE.__ALL_TENANT_TABLESPACE TP ON TP.TABLESPACE_ID = IFNULL(SP.TABLESPACE_ID, P.TABLESPACE_ID) AND TP.TENANT_ID = T.TENANT_ID LEFT JOIN OCEANBASE.__ALL_TABLE_STAT TS ON T.TENANT_ID = TS.TENANT_ID AND TS.TABLE_ID = T.TABLE_ID AND TS.PARTITION_ID = CASE T.PART_LEVEL WHEN 0 THEN T.TABLE_ID WHEN 1 THEN P.PART_ID WHEN 2 THEN SP.SUB_PART_ID END WHERE T.TABLE_TYPE IN (3,6,8,9,14,15) )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT CAST('def' as CHAR(4096)) AS TABLE_CATALOG, DB.DATABASE_NAME collate utf8mb4_name_case AS TABLE_SCHEMA, T.TABLE_NAME collate utf8mb4_name_case AS TABLE_NAME, P.PART_NAME AS PARTITION_NAME, SP.SUB_PART_NAME AS SUBPARTITION_NAME, CAST(PART_POSITION AS UNSIGNED) AS PARTITION_ORDINAL_POSITION, CAST(SUB_PART_POSITION AS UNSIGNED) AS SUBPARTITION_ORDINAL_POSITION, CAST(CASE WHEN T.PART_LEVEL = 0 THEN NULL ELSE (CASE T.PART_FUNC_TYPE WHEN 0 THEN 'HASH' WHEN 1 THEN 'KEY' WHEN 2 THEN 'KEY' WHEN 3 THEN 'RANGE' WHEN 4 THEN 'RANGE COLUMNS' WHEN 5 THEN 'LIST' WHEN 6 THEN 'LIST COLUMNS' WHEN 7 THEN 'RANGE' END) END AS CHAR(13)) PARTITION_METHOD, CAST(CASE WHEN (T.PART_LEVEL = 0 OR T.PART_LEVEL = 1) THEN NULL ELSE (CASE T.SUB_PART_FUNC_TYPE WHEN 0 THEN 'HASH' WHEN 1 THEN 'KEY' WHEN 2 THEN 'KEY' WHEN 3 THEN 'RANGE' WHEN 4 THEN 'RANGE COLUMNS' WHEN 5 THEN 'LIST' WHEN 6 THEN 'LIST COLUMNS' WHEN 7 THEN 'RANGE' END) END AS CHAR(13)) SUBPARTITION_METHOD, CAST(CASE WHEN (T.PART_LEVEL = 0) THEN NULL ELSE T.PART_FUNC_EXPR END AS CHAR(2048)) PARTITION_EXPRESSION, CAST(CASE WHEN (T.PART_LEVEL = 0 OR T.PART_LEVEL = 1) THEN NULL ELSE T.SUB_PART_FUNC_EXPR END AS CHAR(2048)) SUBPARTITION_EXPRESSION, CAST(CASE WHEN (T.PART_LEVEL = 0) THEN NULL ELSE (CASE WHEN LENGTH(P.HIGH_BOUND_VAL) > 0 THEN P.HIGH_BOUND_VAL ELSE P.LIST_VAL END) END AS CHAR(4096)) AS PARTITION_DESCRIPTION, CAST(CASE WHEN (T.PART_LEVEL = 0 OR T.PART_LEVEL = 1) THEN NULL ELSE (CASE WHEN LENGTH(SP.HIGH_BOUND_VAL) > 0 THEN SP.HIGH_BOUND_VAL ELSE SP.LIST_VAL END) END AS CHAR(4096)) AS SUBPARTITION_DESCRIPTION, CAST(TS.ROW_CNT AS UNSIGNED) AS TABLE_ROWS, CAST(TS.AVG_ROW_LEN AS UNSIGNED) AS AVG_ROW_LENGTH, CAST(COALESCE(TS.MACRO_BLK_CNT * 2 * 1024 * 1024, 0) AS UNSIGNED) AS DATA_LENGTH, CAST(NULL AS UNSIGNED) AS MAX_DATA_LENGTH, CAST(COALESCE(IDX_STAT.INDEX_LENGTH, 0) AS UNSIGNED) AS INDEX_LENGTH, CAST(NULL AS UNSIGNED) AS DATA_FREE, CASE T.PART_LEVEL WHEN 0 THEN T.GMT_CREATE WHEN 1 THEN P.GMT_CREATE WHEN 2 THEN SP.GMT_CREATE END AS CREATE_TIME, CAST(NULL AS DATETIME) AS UPDATE_TIME, CAST(NULL AS DATETIME) AS CHECK_TIME, CAST(NULL AS SIGNED) AS CHECKSUM, CAST(CASE T.PART_LEVEL WHEN 0 THEN NULL WHEN 1 THEN P.COMMENT WHEN 2 THEN SP.COMMENT END AS CHAR(1024)) AS PARTITION_COMMENT, CAST('default' AS CHAR(256)) NODEGROUP, CAST(TP.TABLESPACE_NAME AS CHAR(268)) AS TABLESPACE_NAME FROM OCEANBASE.__ALL_TABLE T JOIN OCEANBASE.__ALL_DATABASE DB ON T.DATABASE_ID = DB.DATABASE_ID AND T.TENANT_ID = DB.TENANT_ID AND T.TABLE_MODE >> 12 & 15 in (0,1) LEFT JOIN ( SELECT TENANT_ID, TABLE_ID, PART_ID, PART_NAME, HIGH_BOUND_VAL, LIST_VAL, TABLESPACE_ID, GMT_CREATE, COMMENT, PART_IDX, ROW_NUMBER() OVER(PARTITION BY TENANT_ID,TABLE_ID ORDER BY PART_IDX) AS PART_POSITION FROM OCEANBASE.__ALL_PART ) P ON T.TABLE_ID = P.TABLE_ID AND T.TENANT_ID = P.TENANT_ID LEFT JOIN ( SELECT TENANT_ID, TABLE_ID, PART_ID, SUB_PART_ID, SUB_PART_NAME, HIGH_BOUND_VAL, LIST_VAL, TABLESPACE_ID, GMT_CREATE, COMMENT, SUB_PART_IDX, ROW_NUMBER() OVER(PARTITION BY TENANT_ID,TABLE_ID,PART_ID ORDER BY SUB_PART_IDX) AS SUB_PART_POSITION FROM OCEANBASE.__ALL_SUB_PART ) SP ON T.TABLE_ID = SP.TABLE_ID AND P.PART_ID = SP.PART_ID AND T.TENANT_ID = SP.TENANT_ID LEFT JOIN OCEANBASE.__ALL_TENANT_TABLESPACE TP ON TP.TABLESPACE_ID = IFNULL(SP.TABLESPACE_ID, P.TABLESPACE_ID) AND TP.TENANT_ID = T.TENANT_ID LEFT JOIN OCEANBASE.__ALL_TABLE_STAT TS ON T.TENANT_ID = TS.TENANT_ID AND TS.TABLE_ID = T.TABLE_ID AND TS.PARTITION_ID = CASE T.PART_LEVEL WHEN 0 THEN T.TABLE_ID WHEN 1 THEN P.PART_ID WHEN 2 THEN SP.SUB_PART_ID END LEFT JOIN ( SELECT E.TENANT_ID AS TENANT_ID, E.DATA_TABLE_ID AS DATA_TABLE_ID, F.PART_IDX AS PART_IDX, SF.SUB_PART_IDX AS SUB_PART_IDX, SUM(G.macro_blk_cnt * 2 * 1024 * 1024) AS INDEX_LENGTH FROM OCEANBASE.__ALL_TABLE E LEFT JOIN OCEANBASE.__ALL_PART F ON E.TENANT_ID = F.TENANT_ID AND E.TABLE_ID = F.TABLE_ID LEFT JOIN OCEANBASE.__ALL_SUB_PART SF ON E.TENANT_ID = SF.TENANT_ID AND E.TABLE_ID = SF.TABLE_ID AND F.PART_ID = SF.PART_ID JOIN OCEANBASE.__ALL_TABLE_STAT G ON E.TENANT_ID = G.TENANT_ID AND E.TABLE_ID = G.TABLE_ID AND G.PARTITION_ID = CASE E.PART_LEVEL WHEN 0 THEN E.TABLE_ID WHEN 1 THEN F.PART_ID WHEN 2 THEN SF.SUB_PART_ID END WHERE E.INDEX_TYPE in (1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12) AND E.TABLE_TYPE = 5 GROUP BY TENANT_ID, DATA_TABLE_ID, PART_IDX, SUB_PART_IDX ) IDX_STAT ON IDX_STAT.TENANT_ID = T.TENANT_ID AND IDX_STAT.DATA_TABLE_ID = T.TABLE_ID AND CASE T.PART_LEVEL WHEN 0 THEN 1 WHEN 1 THEN P.PART_IDX = IDX_STAT.PART_IDX WHEN 2 THEN P.PART_IDX = IDX_STAT.PART_IDX AND SP.SUB_PART_IDX = IDX_STAT.SUB_PART_IDX END WHERE T.TABLE_TYPE IN (3,6,8,9,14,15) )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema.21451_21500.cpp b/src/share/inner_table/ob_inner_table_schema.21451_21500.cpp index 2d2efbb1b..4a85005be 100644 --- a/src/share/inner_table/ob_inner_table_schema.21451_21500.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21451_21500.cpp @@ -1060,7 +1060,7 @@ int ObInnerTableSchema::procs_priv_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT cast(b.host as char(60)) as Host, cast(a.database_name as char(64)) as Db, cast(b.user_name as char(32)) as User, cast(a.routine_name as char(64)) as Routine_name, case when a.routine_type = 1 then 'PROCEDURE' else 'FUNCTION' end as Routine_type, cast(c.priv_user as char(93)) as Grantor, substr(concat(case when (a.all_priv & 1) > 0 then ',Execute' else '' end, case when (a.all_priv & 2) > 0 then ',Alter Routine' else '' end, case when (a.all_priv & 4) > 0 then ',Grant' else '' end), 2) as Proc_priv, cast(a.gmt_modified as date) as Timestamp FROM oceanbase.__all_routine_privilege a, oceanbase.__all_user b, oceanbase.__all_routine c, oceanbase.__all_database d WHERE a.tenant_id = b.tenant_id AND a.user_id = b.user_id AND a.tenant_id = d.tenant_id and a.database_name = d.database_name AND a.tenant_id = c.tenant_id AND a.routine_name = c.routine_name AND a.routine_type = c.routine_type AND c.database_id = d.database_id AND c.package_id = -1; )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT cast(b.host as char(60)) as Host, cast(a.database_name as char(64)) as Db, cast(b.user_name as char(32)) as User, cast(a.routine_name as char(64)) as Routine_name, case when a.routine_type = 1 then 'PROCEDURE' else 'FUNCTION' end as Routine_type, cast(concat(a.grantor, '@', a.grantor_host) as char(93)) as Grantor, substr(concat(case when (a.all_priv & 1) > 0 then ',Execute' else '' end, case when (a.all_priv & 2) > 0 then ',Alter Routine' else '' end, case when (a.all_priv & 4) > 0 then ',Grant' else '' end), 2) as Proc_priv, cast(a.gmt_modified as date) as Timestamp FROM oceanbase.__all_routine_privilege a, oceanbase.__all_user b WHERE a.tenant_id = 0 and a.tenant_id = b.tenant_id AND a.user_id = b.user_id; )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema.21501_21550.cpp b/src/share/inner_table/ob_inner_table_schema.21501_21550.cpp index edcf87ec1..ca2f5f111 100644 --- a/src/share/inner_table/ob_inner_table_schema.21501_21550.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21501_21550.cpp @@ -525,6 +525,106 @@ int ObInnerTableSchema::cdb_index_usage_schema(ObTableSchema &table_schema) return ret; } +int ObInnerTableSchema::audit_log_filter_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_MYSQL_SCHEMA_ID); + table_schema.set_table_id(OB_AUDIT_LOG_FILTER_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_AUDIT_LOG_FILTER_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(filter_name AS CHAR(64)) collate utf8mb4_bin as NAME, definition as FILTER FROM oceanbase.__all_audit_log_filter WHERE tenant_id = 0 and is_deleted = 0; )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::audit_log_user_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_MYSQL_SCHEMA_ID); + table_schema.set_table_id(OB_AUDIT_LOG_USER_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_AUDIT_LOG_USER_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(user_name AS CHAR(128)) collate utf8mb4_bin as USER, host as HOST, CAST(filter_name AS CHAR(64)) collate utf8mb4_bin as FILTERNAME FROM oceanbase.__all_audit_log_user WHERE tenant_id = 0 and is_deleted = 0; )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::columns_priv_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.21551_21600.cpp b/src/share/inner_table/ob_inner_table_schema.21551_21600.cpp index 410bf9af7..1edd5ec9b 100644 --- a/src/share/inner_table/ob_inner_table_schema.21551_21600.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21551_21600.cpp @@ -1225,6 +1225,156 @@ int ObInnerTableSchema::v_ob_nic_info_schema(ObTableSchema &table_schema) return ret; } +int ObInnerTableSchema::role_table_grants_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_INFORMATION_SCHEMA_ID); + table_schema.set_table_id(OB_ROLE_TABLE_GRANTS_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ROLE_TABLE_GRANTS_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( with recursive role_graph (from_user, from_host, to_user, to_host, is_enabled) as ( select user_name, host, cast('' as char(128)), cast('' as char(128)), false from oceanbase.__all_user where tenant_id=0 and concat(user_name, '@', host)=current_user() union all select role_edges.from_user, role_edges.from_host, role_edges.to_user, role_edges.to_host, if ((role_graph.is_enabled or is_enabled_role(role_edges.from_user, role_edges.from_host)), true, false) from mysql.role_edges role_edges join role_graph on role_edges.to_user = role_graph.from_user and role_edges.to_host = role_graph.from_host ) select distinct cast(tp.grantor as char(97)) as GRANTOR, cast(tp.grantor_host as char(256)) as GRANTOR_HOST, cast(u.user_name as char(32)) as GRANTEE, cast(u.host as char(255)) as GRANTEE_HOST, cast('def' as char(3)) as TABLE_CATALOG, cast(tp.database_name as char(64)) as TABLE_SCHEMA, cast(tp.table_name as char(64)) as TABLE_NAME, substr(concat(case when tp.priv_alter > 0 then ',Alter' else '' end, case when tp.priv_create > 0 then ',Create' else '' end, case when tp.priv_delete > 0 then ',Delete' else '' end, case when tp.priv_drop > 0 then ',Drop' else '' end, case when tp.priv_grant_option > 0 then ',Grant' else '' end, case when tp.priv_insert > 0 then ',Insert' else '' end, case when tp.priv_update > 0 then ',Update' else '' end, case when tp.priv_select > 0 then ',Select' else '' end, case when tp.priv_index > 0 then ',Index' else '' end, case when tp.priv_create_view > 0 then ',Create View' else '' end, case when tp.priv_show_view > 0 then ',Show View' else '' end, case when (tp.priv_others & 64) > 0 then ',References' else '' end),2) as PRIVILEGE_TYPE, cast(if (tp.priv_grant_option > 0,'YES','NO') as char(3)) AS IS_GRANTABLE from (select distinct from_user, from_host, to_user, to_host, is_enabled from role_graph) rg join oceanbase.__all_table_privilege tp join oceanbase.__all_user u on tp.user_id = u.user_id and rg.from_user = u.user_name and rg.from_host = u.host where rg.is_enabled and rg.to_user <> '' and tp.tenant_id = 0 and u.tenant_id = 0 )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::role_column_grants_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_INFORMATION_SCHEMA_ID); + table_schema.set_table_id(OB_ROLE_COLUMN_GRANTS_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ROLE_COLUMN_GRANTS_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( with recursive role_graph (from_user, from_host, to_user, to_host, is_enabled) as ( select user_name, host, cast('' as char(128)), cast('' as char(128)), false from oceanbase.__all_user where tenant_id=0 and concat(user_name, '@', host)=current_user() union all select role_edges.from_user, role_edges.from_host, role_edges.to_user, role_edges.to_host, if ((role_graph.is_enabled or is_enabled_role(role_edges.from_user, role_edges.from_host)), true, false) from mysql.role_edges role_edges join role_graph on role_edges.to_user = role_graph.from_user and role_edges.to_host = role_graph.from_host ) select distinct NULL as GRANTOR, NULL as GRANTOR_HOST, cast(u.user_name as char(32)) as GRANTEE, cast(u.host as char(255)) as GRANTEE_HOST, cast('def' as char(3)) as TABLE_CATALOG, cast(cp.database_name as char(64)) as TABLE_SCHEMA, cast(cp.table_name as char(64)) as TABLE_NAME, cast(cp.column_name as char(64)) as COLUMN_NAME, substr(concat(case when (cp.all_priv & 1) > 0 then ',Select' else '' end, case when (cp.all_priv & 2) > 0 then ',Insert' else '' end, case when (cp.all_priv & 4) > 0 then ',Update' else '' end, case when (cp.all_priv & 8) > 0 then ',References' else '' end), 2) as PRIVILEGE_TYPE, cast(if (tp.priv_grant_option > 0,'YES','NO') as char(3)) AS IS_GRANTABLE from ((select distinct from_user, from_host, to_user, to_host, is_enabled from role_graph) rg join oceanbase.__all_user u join oceanbase.__all_column_privilege cp on cp.user_id = u.user_id and rg.from_user = u.user_name and rg.from_host = u.host and rg.is_enabled and rg.to_user <> '' and cp.tenant_id = 0 and u.tenant_id = 0) left join oceanbase.__all_table_privilege tp on cp.database_name = tp.database_name and cp.table_name = tp.table_name and cp.user_id = tp.user_id and tp.tenant_id = 0 )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::role_routine_grants_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_INFORMATION_SCHEMA_ID); + table_schema.set_table_id(OB_ROLE_ROUTINE_GRANTS_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ROLE_ROUTINE_GRANTS_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( with recursive role_graph (from_user, from_host, to_user, to_host, is_enabled) as ( select user_name, host, cast('' as char(128)), cast('' as char(128)), false from oceanbase.__all_user where tenant_id=0 and concat(user_name, '@', host)=current_user() union all select role_edges.from_user, role_edges.from_host, role_edges.to_user, role_edges.to_host, if ((role_graph.is_enabled or is_enabled_role(role_edges.from_user, role_edges.from_host)), true, false) from mysql.role_edges role_edges join role_graph on role_edges.to_user = role_graph.from_user and role_edges.to_host = role_graph.from_host ) select distinct cast(rp.grantor as char(97)) as GRANTOR, cast(rp.grantor_host as char(256)) as GRANTOR_HOST, cast(u.user_name as char(32)) as GRANTEE, cast(u.host as char(255)) as GRANTEE_HOST, cast('def' as char(3)) AS SPECIFIC_CATALOG, cast(rp.database_name as char(64)) AS SPECIFIC_SCHEMA, cast(rp.routine_name as char(64)) AS SPECIFIC_NAME, cast('def' as char(3)) AS ROUTINE_CATALOG, cast(rp.database_name as char(64)) AS ROUTINE_SCHEMA, cast(rp.routine_name as char(64)) AS ROUTINE_NAME, substr(concat(case when (rp.all_priv & 1) > 0 then ',Execute' else '' end, case when (rp.all_priv & 2) > 0 then ',Alter Routine' else '' end, case when (rp.all_priv & 4) > 0 then ',Grant' else '' end), 2) AS PRIVILEGE_TYPE, cast(if ((rp.all_priv & 4) > 0,'YES','NO') as char(3)) AS `IS_GRANTABLE` from (select distinct from_user, from_host, to_user, to_host, is_enabled from role_graph) rg join oceanbase.__all_routine_privilege rp join oceanbase.__all_user u on rp.user_id = u.user_id and rg.from_user = u.user_name and rg.from_host = u.host where rg.to_user <> '' and rg.is_enabled and u.tenant_id = 0 and rp.tenant_id = 0 )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::func_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.501_550.cpp b/src/share/inner_table/ob_inner_table_schema.501_550.cpp index 15edca185..50dc85778 100644 --- a/src/share/inner_table/ob_inner_table_schema.501_550.cpp +++ b/src/share/inner_table/ob_inner_table_schema.501_550.cpp @@ -346,6 +346,325 @@ int ObInnerTableSchema::all_trusted_root_certificate_schema(ObTableSchema &table return ret; } +int ObInnerTableSchema::all_audit_log_filter_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_AUDIT_LOG_FILTER_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_AUDIT_LOG_FILTER_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ObObj gmt_create_default; + ObObj gmt_create_default_null; + + gmt_create_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_create_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("gmt_create", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_BINARY,//collation_type + 0, //column length + -1, //column_precision + 6, //column_scale + true,//is nullable + false, //is_autoincrement + false, //is_on_update_for_timestamp + gmt_create_default_null, + gmt_create_default) + } + + if (OB_SUCC(ret)) { + ObObj gmt_modified_default; + ObObj gmt_modified_default_null; + + gmt_modified_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_modified_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("gmt_modified", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_BINARY,//collation_type + 0, //column length + -1, //column_precision + 6, //column_scale + true,//is nullable + false, //is_autoincrement + true, //is_on_update_for_timestamp + gmt_modified_default_null, + gmt_modified_default) + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("filter_name", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + MAX_AUDIT_FILTER_NAME_LENGTH_BYTE, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("definition", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObLongTextType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("is_deleted", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_AUDIT_LOG_FILTER_TID); + table_schema.set_aux_lob_meta_tid(OB_ALL_AUDIT_LOG_FILTER_AUX_LOB_META_TID); + table_schema.set_aux_lob_piece_tid(OB_ALL_AUDIT_LOG_FILTER_AUX_LOB_PIECE_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::all_audit_log_user_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_AUDIT_LOG_USER_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_AUDIT_LOG_USER_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ObObj gmt_create_default; + ObObj gmt_create_default_null; + + gmt_create_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_create_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("gmt_create", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_BINARY,//collation_type + 0, //column length + -1, //column_precision + 6, //column_scale + true,//is nullable + false, //is_autoincrement + false, //is_on_update_for_timestamp + gmt_create_default_null, + gmt_create_default) + } + + if (OB_SUCC(ret)) { + ObObj gmt_modified_default; + ObObj gmt_modified_default_null; + + gmt_modified_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_modified_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("gmt_modified", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_BINARY,//collation_type + 0, //column length + -1, //column_precision + 6, //column_scale + true,//is nullable + false, //is_autoincrement + true, //is_on_update_for_timestamp + gmt_modified_default_null, + gmt_modified_default) + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("user_name", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + MAX_AUDIT_USER_NAME_LENGTH_BYTE, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("host", //column_name + ++column_id, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_HOST_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("filter_name", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + MAX_AUDIT_FILTER_NAME_LENGTH_BYTE, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("is_deleted", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_AUDIT_LOG_USER_TID); + table_schema.set_aux_lob_meta_tid(OB_ALL_AUDIT_LOG_USER_AUX_LOB_META_TID); + table_schema.set_aux_lob_piece_tid(OB_ALL_AUDIT_LOG_USER_AUX_LOB_PIECE_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::all_column_privilege_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.50501_50550.cpp b/src/share/inner_table/ob_inner_table_schema.50501_50550.cpp index a6abc694c..21b093c27 100644 --- a/src/share/inner_table/ob_inner_table_schema.50501_50550.cpp +++ b/src/share/inner_table/ob_inner_table_schema.50501_50550.cpp @@ -295,6 +295,276 @@ int ObInnerTableSchema::all_trusted_root_certificate_aux_lob_meta_schema(ObTable return ret; } +int ObInnerTableSchema::all_audit_log_filter_aux_lob_meta_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_AUDIT_LOG_FILTER_AUX_LOB_META_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(AUX_LOB_META); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_AUDIT_LOG_FILTER_AUX_LOB_META_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 16, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("seq_id", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 8192, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("binary_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("char_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("piece_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_data", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 262144, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_AUDIT_LOG_FILTER_AUX_LOB_META_TID); + table_schema.set_data_table_id(OB_ALL_AUDIT_LOG_FILTER_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::all_audit_log_user_aux_lob_meta_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_AUDIT_LOG_USER_AUX_LOB_META_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(AUX_LOB_META); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_AUDIT_LOG_USER_AUX_LOB_META_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 16, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("seq_id", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 8192, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("binary_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("char_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("piece_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_data", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 262144, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_AUDIT_LOG_USER_AUX_LOB_META_TID); + table_schema.set_data_table_id(OB_ALL_AUDIT_LOG_USER_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::all_column_privilege_aux_lob_meta_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.60501_60550.cpp b/src/share/inner_table/ob_inner_table_schema.60501_60550.cpp index 16db453ce..9b0477d37 100644 --- a/src/share/inner_table/ob_inner_table_schema.60501_60550.cpp +++ b/src/share/inner_table/ob_inner_table_schema.60501_60550.cpp @@ -205,6 +205,186 @@ int ObInnerTableSchema::all_trusted_root_certificate_aux_lob_piece_schema(ObTabl return ret; } +int ObInnerTableSchema::all_audit_log_filter_aux_lob_piece_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_AUDIT_LOG_FILTER_AUX_LOB_PIECE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(1); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(AUX_LOB_PIECE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_AUDIT_LOG_FILTER_AUX_LOB_PIECE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("piece_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("data_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_data", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 32, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_AUDIT_LOG_FILTER_AUX_LOB_PIECE_TID); + table_schema.set_data_table_id(OB_ALL_AUDIT_LOG_FILTER_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::all_audit_log_user_aux_lob_piece_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_AUDIT_LOG_USER_AUX_LOB_PIECE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(1); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(AUX_LOB_PIECE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_AUDIT_LOG_USER_AUX_LOB_PIECE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("piece_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("data_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_data", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 32, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_AUDIT_LOG_USER_AUX_LOB_PIECE_TID); + table_schema.set_data_table_id(OB_ALL_AUDIT_LOG_USER_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::all_column_privilege_aux_lob_piece_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.h b/src/share/inner_table/ob_inner_table_schema.h index ae3d0adc4..60f945002 100644 --- a/src/share/inner_table/ob_inner_table_schema.h +++ b/src/share/inner_table/ob_inner_table_schema.h @@ -618,6 +618,8 @@ public: static int all_transfer_partition_task_history_schema(share::schema::ObTableSchema &table_schema); static int all_tenant_snapshot_job_schema(share::schema::ObTableSchema &table_schema); static int all_trusted_root_certificate_schema(share::schema::ObTableSchema &table_schema); + static int all_audit_log_filter_schema(share::schema::ObTableSchema &table_schema); + static int all_audit_log_user_schema(share::schema::ObTableSchema &table_schema); static int all_column_privilege_schema(share::schema::ObTableSchema &table_schema); static int all_column_privilege_history_schema(share::schema::ObTableSchema &table_schema); static int all_tenant_snapshot_ls_replica_history_schema(share::schema::ObTableSchema &table_schema); @@ -1068,6 +1070,8 @@ public: static int all_virtual_dbms_lock_allocated_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_ls_snapshot_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_index_usage_info_schema(share::schema::ObTableSchema &table_schema); + static int all_virtual_audit_log_filter_schema(share::schema::ObTableSchema &table_schema); + static int all_virtual_audit_log_user_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_column_privilege_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_column_privilege_history_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_tenant_snapshot_ls_replica_history_schema(share::schema::ObTableSchema &table_schema); @@ -1377,6 +1381,9 @@ public: static int engines_schema(share::schema::ObTableSchema &table_schema); static int routines_schema(share::schema::ObTableSchema &table_schema); static int profiling_schema(share::schema::ObTableSchema &table_schema); + static int optimizer_trace_schema(share::schema::ObTableSchema &table_schema); + static int plugins_schema(share::schema::ObTableSchema &table_schema); + static int innodb_sys_columns_schema(share::schema::ObTableSchema &table_schema); static int innodb_ft_being_deleted_schema(share::schema::ObTableSchema &table_schema); static int innodb_ft_config_schema(share::schema::ObTableSchema &table_schema); static int innodb_ft_deleted_schema(share::schema::ObTableSchema &table_schema); @@ -1754,6 +1761,8 @@ public: static int role_edges_schema(share::schema::ObTableSchema &table_schema); static int default_roles_schema(share::schema::ObTableSchema &table_schema); static int cdb_index_usage_schema(share::schema::ObTableSchema &table_schema); + static int audit_log_filter_schema(share::schema::ObTableSchema &table_schema); + static int audit_log_user_schema(share::schema::ObTableSchema &table_schema); static int columns_priv_schema(share::schema::ObTableSchema &table_schema); static int gv_ob_ls_snapshots_schema(share::schema::ObTableSchema &table_schema); static int v_ob_ls_snapshots_schema(share::schema::ObTableSchema &table_schema); @@ -1807,6 +1816,9 @@ public: static int innodb_metrics_schema(share::schema::ObTableSchema &table_schema); static int events_schema(share::schema::ObTableSchema &table_schema); static int v_ob_nic_info_schema(share::schema::ObTableSchema &table_schema); + static int role_table_grants_schema(share::schema::ObTableSchema &table_schema); + static int role_column_grants_schema(share::schema::ObTableSchema &table_schema); + static int role_routine_grants_schema(share::schema::ObTableSchema &table_schema); static int func_schema(share::schema::ObTableSchema &table_schema); static int gv_ob_nic_info_schema(share::schema::ObTableSchema &table_schema); static int dba_scheduler_job_run_details_schema(share::schema::ObTableSchema &table_schema); @@ -2571,6 +2583,8 @@ public: static int all_transfer_partition_task_history_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_tenant_snapshot_job_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_trusted_root_certificate_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); + static int all_audit_log_filter_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); + static int all_audit_log_user_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_column_privilege_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_column_privilege_history_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_tenant_snapshot_ls_replica_history_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); @@ -2872,6 +2886,8 @@ public: static int all_transfer_partition_task_history_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_tenant_snapshot_job_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_trusted_root_certificate_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); + static int all_audit_log_filter_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); + static int all_audit_log_user_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_column_privilege_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_column_privilege_history_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_tenant_snapshot_ls_replica_history_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); @@ -3392,6 +3408,8 @@ const schema_create_func sys_table_schema_creators [] = { ObInnerTableSchema::all_transfer_partition_task_history_schema, ObInnerTableSchema::all_tenant_snapshot_job_schema, ObInnerTableSchema::all_trusted_root_certificate_schema, + ObInnerTableSchema::all_audit_log_filter_schema, + ObInnerTableSchema::all_audit_log_user_schema, ObInnerTableSchema::all_column_privilege_schema, ObInnerTableSchema::all_column_privilege_history_schema, ObInnerTableSchema::all_tenant_snapshot_ls_replica_history_schema, @@ -3845,6 +3863,8 @@ const schema_create_func virtual_table_schema_creators [] = { ObInnerTableSchema::all_virtual_dbms_lock_allocated_schema, ObInnerTableSchema::all_virtual_ls_snapshot_schema, ObInnerTableSchema::all_virtual_index_usage_info_schema, + ObInnerTableSchema::all_virtual_audit_log_filter_schema, + ObInnerTableSchema::all_virtual_audit_log_user_schema, ObInnerTableSchema::all_virtual_column_privilege_schema, ObInnerTableSchema::all_virtual_column_privilege_history_schema, ObInnerTableSchema::all_virtual_tenant_snapshot_ls_replica_history_schema, @@ -4251,6 +4271,9 @@ const schema_create_func sys_view_schema_creators [] = { ObInnerTableSchema::engines_schema, ObInnerTableSchema::routines_schema, ObInnerTableSchema::profiling_schema, + ObInnerTableSchema::optimizer_trace_schema, + ObInnerTableSchema::plugins_schema, + ObInnerTableSchema::innodb_sys_columns_schema, ObInnerTableSchema::innodb_ft_being_deleted_schema, ObInnerTableSchema::innodb_ft_config_schema, ObInnerTableSchema::innodb_ft_deleted_schema, @@ -4628,6 +4651,8 @@ const schema_create_func sys_view_schema_creators [] = { ObInnerTableSchema::role_edges_schema, ObInnerTableSchema::default_roles_schema, ObInnerTableSchema::cdb_index_usage_schema, + ObInnerTableSchema::audit_log_filter_schema, + ObInnerTableSchema::audit_log_user_schema, ObInnerTableSchema::columns_priv_schema, ObInnerTableSchema::gv_ob_ls_snapshots_schema, ObInnerTableSchema::v_ob_ls_snapshots_schema, @@ -4681,6 +4706,9 @@ const schema_create_func sys_view_schema_creators [] = { ObInnerTableSchema::innodb_metrics_schema, ObInnerTableSchema::events_schema, ObInnerTableSchema::v_ob_nic_info_schema, + ObInnerTableSchema::role_table_grants_schema, + ObInnerTableSchema::role_column_grants_schema, + ObInnerTableSchema::role_routine_grants_schema, ObInnerTableSchema::func_schema, ObInnerTableSchema::gv_ob_nic_info_schema, ObInnerTableSchema::dba_scheduler_job_run_details_schema, @@ -5547,6 +5575,8 @@ const uint64_t tenant_space_tables [] = { OB_ALL_TRANSFER_PARTITION_TASK_TID, OB_ALL_TRANSFER_PARTITION_TASK_HISTORY_TID, OB_ALL_TENANT_SNAPSHOT_JOB_TID, + OB_ALL_AUDIT_LOG_FILTER_TID, + OB_ALL_AUDIT_LOG_USER_TID, OB_ALL_COLUMN_PRIVILEGE_TID, OB_ALL_COLUMN_PRIVILEGE_HISTORY_TID, OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_TID, @@ -6093,6 +6123,9 @@ const uint64_t tenant_space_tables [] = { OB_ENGINES_TID, OB_ROUTINES_TID, OB_PROFILING_TID, + OB_OPTIMIZER_TRACE_TID, + OB_PLUGINS_TID, + OB_INNODB_SYS_COLUMNS_TID, OB_INNODB_FT_BEING_DELETED_TID, OB_INNODB_FT_CONFIG_TID, OB_INNODB_FT_DELETED_TID, @@ -6360,6 +6393,8 @@ const uint64_t tenant_space_tables [] = { OB_DBA_OB_TRANSFER_PARTITION_TASK_HISTORY_TID, OB_ROLE_EDGES_TID, OB_DEFAULT_ROLES_TID, + OB_AUDIT_LOG_FILTER_TID, + OB_AUDIT_LOG_USER_TID, OB_COLUMNS_PRIV_TID, OB_GV_OB_LS_SNAPSHOTS_TID, OB_V_OB_LS_SNAPSHOTS_TID, @@ -6402,6 +6437,9 @@ const uint64_t tenant_space_tables [] = { OB_INNODB_METRICS_TID, OB_EVENTS_TID, OB_V_OB_NIC_INFO_TID, + OB_ROLE_TABLE_GRANTS_TID, + OB_ROLE_COLUMN_GRANTS_TID, + OB_ROLE_ROUTINE_GRANTS_TID, OB_FUNC_TID, OB_GV_OB_NIC_INFO_TID, OB_DBA_SCHEDULER_JOB_RUN_DETAILS_TID, @@ -7320,6 +7358,8 @@ const uint64_t tenant_space_tables [] = { OB_ALL_TRANSFER_PARTITION_TASK_AUX_LOB_META_TID, OB_ALL_TRANSFER_PARTITION_TASK_HISTORY_AUX_LOB_META_TID, OB_ALL_TENANT_SNAPSHOT_JOB_AUX_LOB_META_TID, + OB_ALL_AUDIT_LOG_FILTER_AUX_LOB_META_TID, + OB_ALL_AUDIT_LOG_USER_AUX_LOB_META_TID, OB_ALL_COLUMN_PRIVILEGE_AUX_LOB_META_TID, OB_ALL_COLUMN_PRIVILEGE_HISTORY_AUX_LOB_META_TID, OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_AUX_LOB_META_TID, @@ -7597,6 +7637,8 @@ const uint64_t tenant_space_tables [] = { OB_ALL_TRANSFER_PARTITION_TASK_AUX_LOB_PIECE_TID, OB_ALL_TRANSFER_PARTITION_TASK_HISTORY_AUX_LOB_PIECE_TID, OB_ALL_TENANT_SNAPSHOT_JOB_AUX_LOB_PIECE_TID, + OB_ALL_AUDIT_LOG_FILTER_AUX_LOB_PIECE_TID, + OB_ALL_AUDIT_LOG_USER_AUX_LOB_PIECE_TID, OB_ALL_COLUMN_PRIVILEGE_AUX_LOB_PIECE_TID, OB_ALL_COLUMN_PRIVILEGE_HISTORY_AUX_LOB_PIECE_TID, OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_AUX_LOB_PIECE_TID, @@ -8188,6 +8230,8 @@ const char* const tenant_space_table_names [] = { OB_ALL_TRANSFER_PARTITION_TASK_TNAME, OB_ALL_TRANSFER_PARTITION_TASK_HISTORY_TNAME, OB_ALL_TENANT_SNAPSHOT_JOB_TNAME, + OB_ALL_AUDIT_LOG_FILTER_TNAME, + OB_ALL_AUDIT_LOG_USER_TNAME, OB_ALL_COLUMN_PRIVILEGE_TNAME, OB_ALL_COLUMN_PRIVILEGE_HISTORY_TNAME, OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_TNAME, @@ -8734,6 +8778,9 @@ const char* const tenant_space_table_names [] = { OB_ENGINES_TNAME, OB_ROUTINES_TNAME, OB_PROFILING_TNAME, + OB_OPTIMIZER_TRACE_TNAME, + OB_PLUGINS_TNAME, + OB_INNODB_SYS_COLUMNS_TNAME, OB_INNODB_FT_BEING_DELETED_TNAME, OB_INNODB_FT_CONFIG_TNAME, OB_INNODB_FT_DELETED_TNAME, @@ -9001,6 +9048,8 @@ const char* const tenant_space_table_names [] = { OB_DBA_OB_TRANSFER_PARTITION_TASK_HISTORY_TNAME, OB_ROLE_EDGES_TNAME, OB_DEFAULT_ROLES_TNAME, + OB_AUDIT_LOG_FILTER_TNAME, + OB_AUDIT_LOG_USER_TNAME, OB_COLUMNS_PRIV_TNAME, OB_GV_OB_LS_SNAPSHOTS_TNAME, OB_V_OB_LS_SNAPSHOTS_TNAME, @@ -9043,6 +9092,9 @@ const char* const tenant_space_table_names [] = { OB_INNODB_METRICS_TNAME, OB_EVENTS_TNAME, OB_V_OB_NIC_INFO_TNAME, + OB_ROLE_TABLE_GRANTS_TNAME, + OB_ROLE_COLUMN_GRANTS_TNAME, + OB_ROLE_ROUTINE_GRANTS_TNAME, OB_FUNC_TNAME, OB_GV_OB_NIC_INFO_TNAME, OB_DBA_SCHEDULER_JOB_RUN_DETAILS_TNAME, @@ -9961,6 +10013,8 @@ const char* const tenant_space_table_names [] = { OB_ALL_TRANSFER_PARTITION_TASK_AUX_LOB_META_TNAME, OB_ALL_TRANSFER_PARTITION_TASK_HISTORY_AUX_LOB_META_TNAME, OB_ALL_TENANT_SNAPSHOT_JOB_AUX_LOB_META_TNAME, + OB_ALL_AUDIT_LOG_FILTER_AUX_LOB_META_TNAME, + OB_ALL_AUDIT_LOG_USER_AUX_LOB_META_TNAME, OB_ALL_COLUMN_PRIVILEGE_AUX_LOB_META_TNAME, OB_ALL_COLUMN_PRIVILEGE_HISTORY_AUX_LOB_META_TNAME, OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_AUX_LOB_META_TNAME, @@ -10238,6 +10292,8 @@ const char* const tenant_space_table_names [] = { OB_ALL_TRANSFER_PARTITION_TASK_AUX_LOB_PIECE_TNAME, OB_ALL_TRANSFER_PARTITION_TASK_HISTORY_AUX_LOB_PIECE_TNAME, OB_ALL_TENANT_SNAPSHOT_JOB_AUX_LOB_PIECE_TNAME, + OB_ALL_AUDIT_LOG_FILTER_AUX_LOB_PIECE_TNAME, + OB_ALL_AUDIT_LOG_USER_AUX_LOB_PIECE_TNAME, OB_ALL_COLUMN_PRIVILEGE_AUX_LOB_PIECE_TNAME, OB_ALL_COLUMN_PRIVILEGE_HISTORY_AUX_LOB_PIECE_TNAME, OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_AUX_LOB_PIECE_TNAME, @@ -13082,6 +13138,22 @@ LOBMapping const lob_aux_table_mappings [] = { ObInnerTableSchema::all_trusted_root_certificate_aux_lob_piece_schema }, + { + OB_ALL_AUDIT_LOG_FILTER_TID, + OB_ALL_AUDIT_LOG_FILTER_AUX_LOB_META_TID, + OB_ALL_AUDIT_LOG_FILTER_AUX_LOB_PIECE_TID, + ObInnerTableSchema::all_audit_log_filter_aux_lob_meta_schema, + ObInnerTableSchema::all_audit_log_filter_aux_lob_piece_schema + }, + + { + OB_ALL_AUDIT_LOG_USER_TID, + OB_ALL_AUDIT_LOG_USER_AUX_LOB_META_TID, + OB_ALL_AUDIT_LOG_USER_AUX_LOB_PIECE_TID, + ObInnerTableSchema::all_audit_log_user_aux_lob_meta_schema, + ObInnerTableSchema::all_audit_log_user_aux_lob_piece_schema + }, + { OB_ALL_COLUMN_PRIVILEGE_TID, OB_ALL_COLUMN_PRIVILEGE_AUX_LOB_META_TID, @@ -13207,12 +13279,12 @@ static inline int get_sys_table_lob_aux_schema(const uint64_t tid, } const int64_t OB_CORE_TABLE_COUNT = 4; -const int64_t OB_SYS_TABLE_COUNT = 298; -const int64_t OB_VIRTUAL_TABLE_COUNT = 827; -const int64_t OB_SYS_VIEW_COUNT = 919; -const int64_t OB_SYS_TENANT_TABLE_COUNT = 2049; +const int64_t OB_SYS_TABLE_COUNT = 300; +const int64_t OB_VIRTUAL_TABLE_COUNT = 829; +const int64_t OB_SYS_VIEW_COUNT = 927; +const int64_t OB_SYS_TENANT_TABLE_COUNT = 2061; const int64_t OB_CORE_SCHEMA_VERSION = 1; -const int64_t OB_BOOTSTRAP_SCHEMA_VERSION = 2052; +const int64_t OB_BOOTSTRAP_SCHEMA_VERSION = 2064; } // end namespace share } // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema.lob.cpp b/src/share/inner_table/ob_inner_table_schema.lob.cpp index f6df163d2..92faa9ad2 100644 --- a/src/share/inner_table/ob_inner_table_schema.lob.cpp +++ b/src/share/inner_table/ob_inner_table_schema.lob.cpp @@ -21,7 +21,7 @@ inner_lob_map_t inner_lob_map; bool lob_mapping_init() { int ret = OB_SUCCESS; - if (OB_FAIL(inner_lob_map.create(301, ObModIds::OB_INNER_LOB_HASH_SET))) { + if (OB_FAIL(inner_lob_map.create(303, ObModIds::OB_INNER_LOB_HASH_SET))) { SERVER_LOG(WARN, "fail to create inner lob map", K(ret)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < ARRAYSIZEOF(lob_aux_table_mappings); ++i) { diff --git a/src/share/inner_table/ob_inner_table_schema_constants.h b/src/share/inner_table/ob_inner_table_schema_constants.h index 0938f63d6..04e062152 100644 --- a/src/share/inner_table/ob_inner_table_schema_constants.h +++ b/src/share/inner_table/ob_inner_table_schema_constants.h @@ -318,6 +318,8 @@ const uint64_t OB_ALL_TRANSFER_PARTITION_TASK_TID = 498; // "__all_transfer_part const uint64_t OB_ALL_TRANSFER_PARTITION_TASK_HISTORY_TID = 499; // "__all_transfer_partition_task_history" const uint64_t OB_ALL_TENANT_SNAPSHOT_JOB_TID = 500; // "__all_tenant_snapshot_job" const uint64_t OB_ALL_TRUSTED_ROOT_CERTIFICATE_TID = 502; // "__all_trusted_root_certificate" +const uint64_t OB_ALL_AUDIT_LOG_FILTER_TID = 503; // "__all_audit_log_filter" +const uint64_t OB_ALL_AUDIT_LOG_USER_TID = 504; // "__all_audit_log_user" const uint64_t OB_ALL_COLUMN_PRIVILEGE_TID = 505; // "__all_column_privilege" const uint64_t OB_ALL_COLUMN_PRIVILEGE_HISTORY_TID = 506; // "__all_column_privilege_history" const uint64_t OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_TID = 507; // "__all_tenant_snapshot_ls_replica_history" @@ -768,6 +770,8 @@ const uint64_t OB_ALL_VIRTUAL_TENANT_SNAPSHOT_JOB_TID = 12453; // "__all_virtual const uint64_t OB_ALL_VIRTUAL_DBMS_LOCK_ALLOCATED_TID = 12456; // "__all_virtual_dbms_lock_allocated" const uint64_t OB_ALL_VIRTUAL_LS_SNAPSHOT_TID = 12458; // "__all_virtual_ls_snapshot" const uint64_t OB_ALL_VIRTUAL_INDEX_USAGE_INFO_TID = 12459; // "__all_virtual_index_usage_info" +const uint64_t OB_ALL_VIRTUAL_AUDIT_LOG_FILTER_TID = 12460; // "__all_virtual_audit_log_filter" +const uint64_t OB_ALL_VIRTUAL_AUDIT_LOG_USER_TID = 12461; // "__all_virtual_audit_log_user" const uint64_t OB_ALL_VIRTUAL_COLUMN_PRIVILEGE_TID = 12462; // "__all_virtual_column_privilege" const uint64_t OB_ALL_VIRTUAL_COLUMN_PRIVILEGE_HISTORY_TID = 12463; // "__all_virtual_column_privilege_history" const uint64_t OB_ALL_VIRTUAL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_TID = 12464; // "__all_virtual_tenant_snapshot_ls_replica_history" @@ -1077,6 +1081,9 @@ const uint64_t OB_KEY_COLUMN_USAGE_TID = 20012; // "KEY_COLUMN_USAGE" const uint64_t OB_ENGINES_TID = 20014; // "ENGINES" const uint64_t OB_ROUTINES_TID = 20015; // "ROUTINES" const uint64_t OB_PROFILING_TID = 20016; // "PROFILING" +const uint64_t OB_OPTIMIZER_TRACE_TID = 20017; // "OPTIMIZER_TRACE" +const uint64_t OB_PLUGINS_TID = 20018; // "PLUGINS" +const uint64_t OB_INNODB_SYS_COLUMNS_TID = 20019; // "INNODB_SYS_COLUMNS" const uint64_t OB_INNODB_FT_BEING_DELETED_TID = 20020; // "INNODB_FT_BEING_DELETED" const uint64_t OB_INNODB_FT_CONFIG_TID = 20021; // "INNODB_FT_CONFIG" const uint64_t OB_INNODB_FT_DELETED_TID = 20022; // "INNODB_FT_DELETED" @@ -1454,6 +1461,8 @@ const uint64_t OB_DBA_OB_CLONE_PROGRESS_TID = 21510; // "DBA_OB_CLONE_PROGRESS" const uint64_t OB_ROLE_EDGES_TID = 21511; // "role_edges" const uint64_t OB_DEFAULT_ROLES_TID = 21512; // "default_roles" const uint64_t OB_CDB_INDEX_USAGE_TID = 21513; // "CDB_INDEX_USAGE" +const uint64_t OB_AUDIT_LOG_FILTER_TID = 21514; // "audit_log_filter" +const uint64_t OB_AUDIT_LOG_USER_TID = 21515; // "audit_log_user" const uint64_t OB_COLUMNS_PRIV_TID = 21516; // "columns_priv" const uint64_t OB_GV_OB_LS_SNAPSHOTS_TID = 21517; // "GV$OB_LS_SNAPSHOTS" const uint64_t OB_V_OB_LS_SNAPSHOTS_TID = 21518; // "V$OB_LS_SNAPSHOTS" @@ -1507,6 +1516,9 @@ const uint64_t OB_INNODB_TEMP_TABLE_INFO_TID = 21578; // "INNODB_TEMP_TABLE_INFO const uint64_t OB_INNODB_METRICS_TID = 21579; // "INNODB_METRICS" const uint64_t OB_EVENTS_TID = 21580; // "EVENTS" const uint64_t OB_V_OB_NIC_INFO_TID = 21581; // "V$OB_NIC_INFO" +const uint64_t OB_ROLE_TABLE_GRANTS_TID = 21582; // "ROLE_TABLE_GRANTS" +const uint64_t OB_ROLE_COLUMN_GRANTS_TID = 21583; // "ROLE_COLUMN_GRANTS" +const uint64_t OB_ROLE_ROUTINE_GRANTS_TID = 21584; // "ROLE_ROUTINE_GRANTS" const uint64_t OB_FUNC_TID = 21585; // "func" const uint64_t OB_GV_OB_NIC_INFO_TID = 21586; // "GV$OB_NIC_INFO" const uint64_t OB_DBA_SCHEDULER_JOB_RUN_DETAILS_TID = 21589; // "DBA_SCHEDULER_JOB_RUN_DETAILS" @@ -2271,6 +2283,8 @@ const uint64_t OB_ALL_TRANSFER_PARTITION_TASK_AUX_LOB_META_TID = 50498; // "__al const uint64_t OB_ALL_TRANSFER_PARTITION_TASK_HISTORY_AUX_LOB_META_TID = 50499; // "__all_transfer_partition_task_history_aux_lob_meta" const uint64_t OB_ALL_TENANT_SNAPSHOT_JOB_AUX_LOB_META_TID = 50500; // "__all_tenant_snapshot_job_aux_lob_meta" const uint64_t OB_ALL_TRUSTED_ROOT_CERTIFICATE_AUX_LOB_META_TID = 50502; // "__all_trusted_root_certificate_aux_lob_meta" +const uint64_t OB_ALL_AUDIT_LOG_FILTER_AUX_LOB_META_TID = 50503; // "__all_audit_log_filter_aux_lob_meta" +const uint64_t OB_ALL_AUDIT_LOG_USER_AUX_LOB_META_TID = 50504; // "__all_audit_log_user_aux_lob_meta" const uint64_t OB_ALL_COLUMN_PRIVILEGE_AUX_LOB_META_TID = 50505; // "__all_column_privilege_aux_lob_meta" const uint64_t OB_ALL_COLUMN_PRIVILEGE_HISTORY_AUX_LOB_META_TID = 50506; // "__all_column_privilege_history_aux_lob_meta" const uint64_t OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_AUX_LOB_META_TID = 50507; // "__all_tenant_snapshot_ls_replica_history_aux_lob_meta" @@ -2572,6 +2586,8 @@ const uint64_t OB_ALL_TRANSFER_PARTITION_TASK_AUX_LOB_PIECE_TID = 60498; // "__a const uint64_t OB_ALL_TRANSFER_PARTITION_TASK_HISTORY_AUX_LOB_PIECE_TID = 60499; // "__all_transfer_partition_task_history_aux_lob_piece" const uint64_t OB_ALL_TENANT_SNAPSHOT_JOB_AUX_LOB_PIECE_TID = 60500; // "__all_tenant_snapshot_job_aux_lob_piece" const uint64_t OB_ALL_TRUSTED_ROOT_CERTIFICATE_AUX_LOB_PIECE_TID = 60502; // "__all_trusted_root_certificate_aux_lob_piece" +const uint64_t OB_ALL_AUDIT_LOG_FILTER_AUX_LOB_PIECE_TID = 60503; // "__all_audit_log_filter_aux_lob_piece" +const uint64_t OB_ALL_AUDIT_LOG_USER_AUX_LOB_PIECE_TID = 60504; // "__all_audit_log_user_aux_lob_piece" const uint64_t OB_ALL_COLUMN_PRIVILEGE_AUX_LOB_PIECE_TID = 60505; // "__all_column_privilege_aux_lob_piece" const uint64_t OB_ALL_COLUMN_PRIVILEGE_HISTORY_AUX_LOB_PIECE_TID = 60506; // "__all_column_privilege_history_aux_lob_piece" const uint64_t OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_AUX_LOB_PIECE_TID = 60507; // "__all_tenant_snapshot_ls_replica_history_aux_lob_piece" @@ -3079,6 +3095,8 @@ const char *const OB_ALL_TRANSFER_PARTITION_TASK_TNAME = "__all_transfer_partiti const char *const OB_ALL_TRANSFER_PARTITION_TASK_HISTORY_TNAME = "__all_transfer_partition_task_history"; const char *const OB_ALL_TENANT_SNAPSHOT_JOB_TNAME = "__all_tenant_snapshot_job"; const char *const OB_ALL_TRUSTED_ROOT_CERTIFICATE_TNAME = "__all_trusted_root_certificate"; +const char *const OB_ALL_AUDIT_LOG_FILTER_TNAME = "__all_audit_log_filter"; +const char *const OB_ALL_AUDIT_LOG_USER_TNAME = "__all_audit_log_user"; const char *const OB_ALL_COLUMN_PRIVILEGE_TNAME = "__all_column_privilege"; const char *const OB_ALL_COLUMN_PRIVILEGE_HISTORY_TNAME = "__all_column_privilege_history"; const char *const OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_TNAME = "__all_tenant_snapshot_ls_replica_history"; @@ -3529,6 +3547,8 @@ const char *const OB_ALL_VIRTUAL_TENANT_SNAPSHOT_JOB_TNAME = "__all_virtual_tena const char *const OB_ALL_VIRTUAL_DBMS_LOCK_ALLOCATED_TNAME = "__all_virtual_dbms_lock_allocated"; const char *const OB_ALL_VIRTUAL_LS_SNAPSHOT_TNAME = "__all_virtual_ls_snapshot"; const char *const OB_ALL_VIRTUAL_INDEX_USAGE_INFO_TNAME = "__all_virtual_index_usage_info"; +const char *const OB_ALL_VIRTUAL_AUDIT_LOG_FILTER_TNAME = "__all_virtual_audit_log_filter"; +const char *const OB_ALL_VIRTUAL_AUDIT_LOG_USER_TNAME = "__all_virtual_audit_log_user"; const char *const OB_ALL_VIRTUAL_COLUMN_PRIVILEGE_TNAME = "__all_virtual_column_privilege"; const char *const OB_ALL_VIRTUAL_COLUMN_PRIVILEGE_HISTORY_TNAME = "__all_virtual_column_privilege_history"; const char *const OB_ALL_VIRTUAL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_TNAME = "__all_virtual_tenant_snapshot_ls_replica_history"; @@ -3838,6 +3858,9 @@ const char *const OB_KEY_COLUMN_USAGE_TNAME = "KEY_COLUMN_USAGE"; const char *const OB_ENGINES_TNAME = "ENGINES"; const char *const OB_ROUTINES_TNAME = "ROUTINES"; const char *const OB_PROFILING_TNAME = "PROFILING"; +const char *const OB_OPTIMIZER_TRACE_TNAME = "OPTIMIZER_TRACE"; +const char *const OB_PLUGINS_TNAME = "PLUGINS"; +const char *const OB_INNODB_SYS_COLUMNS_TNAME = "INNODB_SYS_COLUMNS"; const char *const OB_INNODB_FT_BEING_DELETED_TNAME = "INNODB_FT_BEING_DELETED"; const char *const OB_INNODB_FT_CONFIG_TNAME = "INNODB_FT_CONFIG"; const char *const OB_INNODB_FT_DELETED_TNAME = "INNODB_FT_DELETED"; @@ -4215,6 +4238,8 @@ const char *const OB_DBA_OB_CLONE_PROGRESS_TNAME = "DBA_OB_CLONE_PROGRESS"; const char *const OB_ROLE_EDGES_TNAME = "role_edges"; const char *const OB_DEFAULT_ROLES_TNAME = "default_roles"; const char *const OB_CDB_INDEX_USAGE_TNAME = "CDB_INDEX_USAGE"; +const char *const OB_AUDIT_LOG_FILTER_TNAME = "audit_log_filter"; +const char *const OB_AUDIT_LOG_USER_TNAME = "audit_log_user"; const char *const OB_COLUMNS_PRIV_TNAME = "columns_priv"; const char *const OB_GV_OB_LS_SNAPSHOTS_TNAME = "GV$OB_LS_SNAPSHOTS"; const char *const OB_V_OB_LS_SNAPSHOTS_TNAME = "V$OB_LS_SNAPSHOTS"; @@ -4268,6 +4293,9 @@ const char *const OB_INNODB_TEMP_TABLE_INFO_TNAME = "INNODB_TEMP_TABLE_INFO"; const char *const OB_INNODB_METRICS_TNAME = "INNODB_METRICS"; const char *const OB_EVENTS_TNAME = "EVENTS"; const char *const OB_V_OB_NIC_INFO_TNAME = "V$OB_NIC_INFO"; +const char *const OB_ROLE_TABLE_GRANTS_TNAME = "ROLE_TABLE_GRANTS"; +const char *const OB_ROLE_COLUMN_GRANTS_TNAME = "ROLE_COLUMN_GRANTS"; +const char *const OB_ROLE_ROUTINE_GRANTS_TNAME = "ROLE_ROUTINE_GRANTS"; const char *const OB_FUNC_TNAME = "func"; const char *const OB_GV_OB_NIC_INFO_TNAME = "GV$OB_NIC_INFO"; const char *const OB_DBA_SCHEDULER_JOB_RUN_DETAILS_TNAME = "DBA_SCHEDULER_JOB_RUN_DETAILS"; @@ -5032,6 +5060,8 @@ const char *const OB_ALL_TRANSFER_PARTITION_TASK_AUX_LOB_META_TNAME = "__all_tra const char *const OB_ALL_TRANSFER_PARTITION_TASK_HISTORY_AUX_LOB_META_TNAME = "__all_transfer_partition_task_history_aux_lob_meta"; const char *const OB_ALL_TENANT_SNAPSHOT_JOB_AUX_LOB_META_TNAME = "__all_tenant_snapshot_job_aux_lob_meta"; const char *const OB_ALL_TRUSTED_ROOT_CERTIFICATE_AUX_LOB_META_TNAME = "__all_trusted_root_certificate_aux_lob_meta"; +const char *const OB_ALL_AUDIT_LOG_FILTER_AUX_LOB_META_TNAME = "__all_audit_log_filter_aux_lob_meta"; +const char *const OB_ALL_AUDIT_LOG_USER_AUX_LOB_META_TNAME = "__all_audit_log_user_aux_lob_meta"; const char *const OB_ALL_COLUMN_PRIVILEGE_AUX_LOB_META_TNAME = "__all_column_privilege_aux_lob_meta"; const char *const OB_ALL_COLUMN_PRIVILEGE_HISTORY_AUX_LOB_META_TNAME = "__all_column_privilege_history_aux_lob_meta"; const char *const OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_AUX_LOB_META_TNAME = "__all_tenant_snapshot_ls_replica_history_aux_lob_meta"; @@ -5333,6 +5363,8 @@ const char *const OB_ALL_TRANSFER_PARTITION_TASK_AUX_LOB_PIECE_TNAME = "__all_tr const char *const OB_ALL_TRANSFER_PARTITION_TASK_HISTORY_AUX_LOB_PIECE_TNAME = "__all_transfer_partition_task_history_aux_lob_piece"; const char *const OB_ALL_TENANT_SNAPSHOT_JOB_AUX_LOB_PIECE_TNAME = "__all_tenant_snapshot_job_aux_lob_piece"; const char *const OB_ALL_TRUSTED_ROOT_CERTIFICATE_AUX_LOB_PIECE_TNAME = "__all_trusted_root_certificate_aux_lob_piece"; +const char *const OB_ALL_AUDIT_LOG_FILTER_AUX_LOB_PIECE_TNAME = "__all_audit_log_filter_aux_lob_piece"; +const char *const OB_ALL_AUDIT_LOG_USER_AUX_LOB_PIECE_TNAME = "__all_audit_log_user_aux_lob_piece"; const char *const OB_ALL_COLUMN_PRIVILEGE_AUX_LOB_PIECE_TNAME = "__all_column_privilege_aux_lob_piece"; const char *const OB_ALL_COLUMN_PRIVILEGE_HISTORY_AUX_LOB_PIECE_TNAME = "__all_column_privilege_history_aux_lob_piece"; const char *const OB_ALL_TENANT_SNAPSHOT_LS_REPLICA_HISTORY_AUX_LOB_PIECE_TNAME = "__all_tenant_snapshot_ls_replica_history_aux_lob_piece"; diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index af412a0a3..cdcb373cb 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -7136,8 +7136,43 @@ def_table_schema( ], ) -# 503 : __all_audit_log_filter -# 504 : __all_audit_log_user +def_table_schema( + owner = 'sean.yyj', + table_name = '__all_audit_log_filter', + table_id = '503', + table_type = 'SYSTEM_TABLE', + gm_columns = ['gmt_create', 'gmt_modified'], + rowkey_columns = [ + ('tenant_id', 'int'), + ('filter_name', 'varbinary:MAX_AUDIT_FILTER_NAME_LENGTH_BYTE'), + ], + in_tenant_space = True, + + normal_columns = [ + ('definition', 'longtext'), + ('is_deleted', 'int'), + ], +) + +def_table_schema( + owner = 'sean.yyj', + table_name = '__all_audit_log_user', + table_id = '504', + table_type = 'SYSTEM_TABLE', + gm_columns = ['gmt_create', 'gmt_modified'], + rowkey_columns = [ + ('tenant_id', 'int'), + ('user_name', 'varbinary:MAX_AUDIT_USER_NAME_LENGTH_BYTE'), + ('host', 'varchar:OB_MAX_HOST_NAME_LENGTH'), + ], + in_tenant_space = True, + + normal_columns = [ + ('filter_name', 'varbinary:MAX_AUDIT_FILTER_NAME_LENGTH_BYTE'), + ('is_deleted', 'int'), + ], +) + all_column_privilege_def = dict( owner = 'mingye.swj', table_name = '__all_column_privilege', @@ -14443,6 +14478,16 @@ def_table_schema(**gen_iterate_virtual_table_def( # 12460: __all_virtual_audit_log_filter # 12461: __all_virtual_audit_log_user +def_table_schema(**gen_iterate_virtual_table_def( + table_id = '12460', + table_name = '__all_virtual_audit_log_filter', + keywords = all_def_keywords['__all_audit_log_filter'])) + +def_table_schema(**gen_iterate_virtual_table_def( + table_id = '12461', + table_name = '__all_virtual_audit_log_user', + keywords = all_def_keywords['__all_audit_log_user'])) + def_table_schema(**gen_iterate_virtual_table_def( table_id = '12462', table_name = '__all_virtual_column_privilege', @@ -15619,7 +15664,7 @@ def_table_schema( cast( coalesce(ts.avg_row_len,0) as unsigned) as AVG_ROW_LENGTH, cast( coalesce(ts.data_size,0) as unsigned) as DATA_LENGTH, cast(NULL as unsigned) as MAX_DATA_LENGTH, - cast(NULL as unsigned) as INDEX_LENGTH, + cast( coalesce(idx_stat.index_length, 0) as unsigned) as INDEX_LENGTH, cast(NULL as unsigned) as DATA_FREE, cast(NULL as unsigned) as AUTO_INCREMENT, cast(a.gmt_create as datetime) as CREATE_TIME, @@ -15673,6 +15718,15 @@ def_table_schema( where partition_id = -1 or partition_id = table_id) ts on a.table_id = ts.table_id and a.tenant_id = ts.tenant_id + left join ( + select e.tenant_id as tenant_id, + e.data_table_id as data_table_id, + SUM(f.macro_blk_cnt * 2 * 1024 * 1024) AS index_length + FROM oceanbase.__all_table e JOIN oceanbase.__all_table_stat f + ON e.tenant_id = f.tenant_id and e.table_id = f.table_id and (f.partition_id = -1 or f.partition_id = e.table_id) + WHERE e.index_type in (1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12) and e.table_type = 5 + group by tenant_id, data_table_id + ) idx_stat on idx_stat.tenant_id = a.tenant_id and idx_stat.data_table_id = a.table_id where a.tenant_id = 0 and a.table_type in (0, 1, 2, 3, 4, 14, 15) and b.database_name != '__recyclebin' @@ -15965,6 +16019,8 @@ def_table_schema( WHEN 5 THEN 'gb18030' WHEN 6 THEN 'latin1' WHEN 7 THEN 'gb18030_2022' + WHEN 8 THEN 'ascii' + WHEN 9 THEN 'tis620' ELSE NULL END AS CHAR(64) @@ -16080,6 +16136,77 @@ def_table_schema( """.replace("\n", " ") ) + +def_table_schema( + owner = 'sanquan.qz', + tablegroup_id = 'OB_INVALID_ID', + database_id = 'OB_INFORMATION_SCHEMA_ID', + table_name = 'OPTIMIZER_TRACE', + table_id = '20017', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT CAST('query' as CHAR(200)) as QUERY, + CAST('trace' as CHAR(200)) as TRACE, + CAST(00000000000000000000 as SIGNED) as MISSING_BYTES_MAX_MEM_SIZE, + CAST(0 as SIGNED) as INSUFFICIENT_PRIVILEGES + FROM DUAL limit 0; + """.replace("\n", " ") +) + +def_table_schema( + owner = 'sanquan.qz', + tablegroup_id = 'OB_INVALID_ID', + database_id = 'OB_INFORMATION_SCHEMA_ID', + table_name = 'PLUGINS', + table_id = '20018', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT CAST('plugin name' as CHAR(64)) as PLUGIN_NAME, + CAST('version' as CHAR(20)) as PLUGIN, + CAST('plugin status' as CHAR(10)) as PLUGIN_STATUS, + CAST('type' as CHAR(80)) as PLUGIN_TYPE, + CAST('version' as CHAR(20)) as PLUGIN_TYPE_VERSION, + CAST('library' as CHAR(64)) as PLUGIN_LIBRARY, + CAST('lib version' as CHAR(20)) as PLUGIN_LIBRARY_VERSION, + CAST('author' as CHAR(64)) as PLUGIN_AUTHOR, + CAST('description' as CHAR(200)) as PLUGIN_DESCRIPTION, + CAST('license' as CHAR(80)) as PLUGIN_LICENSE, + CAST('load option' as CHAR(64)) as LOAD_OPTION + FROM DUAL limit 0; + """.replace("\n", " ") +) + +def_table_schema( + owner = 'sanquan.qz', + tablegroup_id = 'OB_INVALID_ID', + database_id = 'OB_INFORMATION_SCHEMA_ID', + table_name = 'INNODB_SYS_COLUMNS', + table_id = '20019', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT CAST(000000000000000000000 as UNSIGNED) AS TABLE_ID, + CAST('name' as CHAR(193)) AS NAME, + CAST(000000000000000000000 as UNSIGNED) AS POS, + CAST(00000000000 as SIGNED) AS MTYPE, + CAST(00000000000 as SIGNED) AS PRTYPE, + CAST(00000000000 as SIGNED) AS LEN + FROM DUAL limit 0; + """.replace("\n", " ") +) + + def_table_schema( owner = 'sanquan.qz', tablegroup_id = 'OB_INVALID_ID', @@ -28915,7 +29042,11 @@ def_table_schema( (CASE WHEN (PRIV_OTHERS & (1 << 2)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_CREATE_ROUTINE, (CASE WHEN (PRIV_OTHERS & (1 << 3)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_CREATE_TABLESPACE, (CASE WHEN (PRIV_OTHERS & (1 << 4)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_SHUTDOWN, - (CASE WHEN (PRIV_OTHERS & (1 << 5)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_RELOAD + (CASE WHEN (PRIV_OTHERS & (1 << 5)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_RELOAD, + (CASE WHEN (PRIV_OTHERS & (1 << 6)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_REFERENCES, + (CASE WHEN (PRIV_OTHERS & (1 << 7)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_CREATE_ROLE, + (CASE WHEN (PRIV_OTHERS & (1 << 8)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_DROP_ROLE, + (CASE WHEN (PRIV_OTHERS & (1 << 9)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_TRIGGER FROM OCEANBASE.__all_user; """.replace("\n", " ") ) @@ -28975,7 +29106,11 @@ def_table_schema( (CASE WHEN (PRIV_OTHERS & (1 << 2)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_CREATE_ROUTINE, (CASE WHEN (PRIV_OTHERS & (1 << 3)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_CREATE_TABLESPACE, (CASE WHEN (PRIV_OTHERS & (1 << 4)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_SHUTDOWN, - (CASE WHEN (PRIV_OTHERS & (1 << 5)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_RELOAD + (CASE WHEN (PRIV_OTHERS & (1 << 5)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_RELOAD, + (CASE WHEN (PRIV_OTHERS & (1 << 6)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_REFERENCES, + (CASE WHEN (PRIV_OTHERS & (1 << 7)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_CREATE_ROLE, + (CASE WHEN (PRIV_OTHERS & (1 << 8)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_DROP_ROLE, + (CASE WHEN (PRIV_OTHERS & (1 << 9)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_TRIGGER FROM OCEANBASE.__all_virtual_user; """.replace("\n", " ") ) @@ -29032,7 +29167,9 @@ def_table_schema( (CASE WHEN A.PRIV_SHOW_VIEW = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_SHOW_VIEW, (CASE WHEN (A.PRIV_OTHERS & (1 << 0)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_EXECUTE, (CASE WHEN (A.PRIV_OTHERS & (1 << 1)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_ALTER_ROUTINE, - (CASE WHEN (A.PRIV_OTHERS & (1 << 2)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_CREATE_ROUTINE + (CASE WHEN (A.PRIV_OTHERS & (1 << 2)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_CREATE_ROUTINE, + (CASE WHEN (A.PRIV_OTHERS & (1 << 6)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_REFERENCES, + (CASE WHEN (A.PRIV_OTHERS & (1 << 9)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_TRIGGER FROM DB_PRIV A INNER JOIN OCEANBASE.__all_user B ON A.TENANT_ID = B.TENANT_ID AND A.USER_ID = B.USER_ID; """.replace("\n", " ") @@ -29089,7 +29226,9 @@ def_table_schema( (CASE WHEN A.PRIV_SHOW_VIEW = 0 THEN 'NO' ELSE 'YES' END) AS PRIV_SHOW_VIEW, (CASE WHEN (A.PRIV_OTHERS & (1 << 0)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_EXECUTE, (CASE WHEN (A.PRIV_OTHERS & (1 << 1)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_ALTER_ROUTINE, - (CASE WHEN (A.PRIV_OTHERS & (1 << 2)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_CREATE_ROUTINE + (CASE WHEN (A.PRIV_OTHERS & (1 << 2)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_CREATE_ROUTINE, + (CASE WHEN (A.PRIV_OTHERS & (1 << 6)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_REFERENCES, + (CASE WHEN (A.PRIV_OTHERS & (1 << 9)) != 0 THEN 'YES' ELSE 'NO' END) AS PRIV_TRIGGER FROM DB_PRIV A INNER JOIN OCEANBASE.__all_virtual_user B ON A.USER_ID = B.USER_ID AND A.TENANT_ID = B.TENANT_ID; """.replace("\n", " ") @@ -29343,6 +29482,8 @@ def_table_schema( WHEN 5 THEN 'gb18030' WHEN 6 THEN 'latin1' WHEN 7 THEN 'gb18030_2022' + WHEN 8 THEN 'ascii' + WHEN 9 THEN 'tis620' ELSE NULL END AS CHAR(64)) AS CHARACTER_SET_NAME, CAST(CASE rp.param_coll_type @@ -29502,6 +29643,10 @@ def_table_schema( AND TP.PRIV_CREATE_VIEW = 1 THEN 'CREATE VIEW' WHEN V1.C1 = 12 AND TP.PRIV_SHOW_VIEW = 1 THEN 'SHOW VIEW' + WHEN V1.C1 = 22 + AND (TP.PRIV_OTHERS & (1 << 6)) != 0 THEN 'REFERENCES' + WHEN V1.C1 = 44 + AND (TP.PRIV_OTHERS & (1 << 9)) != 0 THEN 'TRIGGER' ELSE NULL END PRIVILEGE_TYPE , CASE @@ -29519,7 +29664,9 @@ def_table_schema( UNION ALL SELECT 9 AS C1 UNION ALL SELECT 10 AS C1 UNION ALL SELECT 11 AS C1 - UNION ALL SELECT 12 AS C1) V1, + UNION ALL SELECT 12 AS C1 + UNION ALL SELECT 22 AS C1 + UNION ALL SELECT 44 AS C1) V1, (SELECT USER_ID FROM oceanbase.__all_user WHERE TENANT_ID = 0 @@ -29592,6 +29739,8 @@ def_table_schema( AND U.PRIV_PROCESS = 1 THEN 'PROCESS' WHEN V1.C1 = 17 AND U.PRIV_CREATE_SYNONYM = 1 THEN 'CREATE SYNONYM' + WHEN V1.C1 = 22 + AND (U.PRIV_OTHERS & (1 << 6)) != 0 THEN 'REFERENCES' WHEN V1.C1 = 23 AND (U.PRIV_OTHERS & (1 << 0)) != 0 THEN 'EXECUTE' WHEN V1.C1 = 27 @@ -29622,6 +29771,12 @@ def_table_schema( AND (U.PRIV_OTHERS & (1 << 4)) != 0 THEN 'SHUTDOWN' WHEN V1.C1 = 41 AND (U.PRIV_OTHERS & (1 << 5)) != 0 THEN 'RELOAD' + WHEN V1.C1 = 42 + AND (U.PRIV_OTHERS & (1 << 7)) != 0 THEN 'CREATE ROLE' + WHEN V1.C1 = 43 + AND (U.PRIV_OTHERS & (1 << 8)) != 0 THEN 'DROP ROLE' + WHEN V1.C1 = 44 + AND (U.PRIV_OTHERS & (1 << 9)) != 0 THEN 'TRIGGER' WHEN V1.C1 = 0 AND U.PRIV_ALTER = 0 AND U.PRIV_CREATE = 0 @@ -29695,6 +29850,7 @@ def_table_schema( UNION ALL SELECT 14 AS C1 UNION ALL SELECT 15 AS C1 UNION ALL SELECT 17 AS C1 + UNION ALL SELECT 22 AS C1 UNION ALL SELECT 23 AS C1 UNION ALL SELECT 27 AS C1 UNION ALL SELECT 28 AS C1 @@ -29709,7 +29865,10 @@ def_table_schema( UNION ALL SELECT 38 AS C1 UNION ALL SELECT 39 AS C1 UNION ALL SELECT 40 AS C1 - UNION ALL SELECT 41 AS C1) V1, + UNION ALL SELECT 41 AS C1 + UNION ALL SELECT 42 AS C1 + UNION ALL SELECT 43 AS C1 + UNION ALL SELECT 44 AS C1) V1, (SELECT USER_ID FROM oceanbase.__all_user WHERE TENANT_ID = 0 @@ -29792,12 +29951,16 @@ def_table_schema( AND DP.PRIV_CREATE_VIEW = 1 THEN 'CREATE VIEW' WHEN V1.C1 = 12 AND DP.PRIV_SHOW_VIEW = 1 THEN 'SHOW VIEW' - WHEN V1.C1 = 13 - AND (U.PRIV_OTHERS & (1 << 0)) != 0 THEN 'EXECUTE' - WHEN V1.C1 = 14 - AND (U.PRIV_OTHERS & (1 << 1)) != 0 THEN 'ALTER ROUTINE' - WHEN V1.C1 = 15 - AND (U.PRIV_OTHERS & (1 << 2)) != 0 THEN 'CREATE ROUTINE' + WHEN V1.C1 = 22 + AND (DP.PRIV_OTHERS & (1 << 6)) != 0 THEN 'REFERENCES' + WHEN V1.C1 = 23 + AND (DP.PRIV_OTHERS & (1 << 0)) != 0 THEN 'EXECUTE' + WHEN V1.C1 = 37 + AND (DP.PRIV_OTHERS & (1 << 1)) != 0 THEN 'ALTER ROUTINE' + WHEN V1.C1 = 38 + AND (DP.PRIV_OTHERS & (1 << 2)) != 0 THEN 'CREATE ROUTINE' + WHEN V1.C1 = 44 + AND (DP.PRIV_OTHERS & (1 << 9)) != 0 THEN 'TRIGGER' ELSE NULL END PRIVILEGE_TYPE , CASE @@ -29816,9 +29979,11 @@ def_table_schema( UNION ALL SELECT 10 AS C1 UNION ALL SELECT 11 AS C1 UNION ALL SELECT 12 AS C1 - UNION ALL SELECT 13 AS C1 - UNION ALL SELECT 14 AS C1 - UNION ALL SELECT 15 AS C1) V1, + UNION ALL SELECT 22 AS C1 + UNION ALL SELECT 23 AS C1 + UNION ALL SELECT 37 AS C1 + UNION ALL SELECT 38 AS C1 + UNION ALL SELECT 44 AS C1) V1, (SELECT USER_ID FROM oceanbase.__all_user WHERE TENANT_ID= 0 @@ -30213,6 +30378,7 @@ def_table_schema( JOIN oceanbase.__all_table t on trg.base_object_id = t.table_id WHERE db.database_name != '__recyclebin' and db.in_recyclebin = 0 and t.table_mode >> 12 & 15 in (0,1) + and can_access_trigger(db.database_name, t.table_name) """.replace("\n", " "), ) @@ -30285,9 +30451,9 @@ def_table_schema( END AS CHAR(4096)) AS SUBPARTITION_DESCRIPTION, CAST(TS.ROW_CNT AS UNSIGNED) AS TABLE_ROWS, CAST(TS.AVG_ROW_LEN AS UNSIGNED) AS AVG_ROW_LENGTH, - CAST(NULL AS UNSIGNED) AS DATA_LENGTH, + CAST(COALESCE(TS.MACRO_BLK_CNT * 2 * 1024 * 1024, 0) AS UNSIGNED) AS DATA_LENGTH, CAST(NULL AS UNSIGNED) AS MAX_DATA_LENGTH, - CAST(NULL AS UNSIGNED) AS INDEX_LENGTH, + CAST(COALESCE(IDX_STAT.INDEX_LENGTH, 0) AS UNSIGNED) AS INDEX_LENGTH, CAST(NULL AS UNSIGNED) AS DATA_FREE, CASE T.PART_LEVEL WHEN 0 THEN T.GMT_CREATE @@ -30318,6 +30484,7 @@ FROM TABLESPACE_ID, GMT_CREATE, COMMENT, + PART_IDX, ROW_NUMBER() OVER(PARTITION BY TENANT_ID,TABLE_ID ORDER BY PART_IDX) AS PART_POSITION FROM OCEANBASE.__ALL_PART ) P ON T.TABLE_ID = P.TABLE_ID AND T.TENANT_ID = P.TENANT_ID @@ -30333,12 +30500,26 @@ FROM TABLESPACE_ID, GMT_CREATE, COMMENT, + SUB_PART_IDX, ROW_NUMBER() OVER(PARTITION BY TENANT_ID,TABLE_ID,PART_ID ORDER BY SUB_PART_IDX) AS SUB_PART_POSITION FROM OCEANBASE.__ALL_SUB_PART ) SP ON T.TABLE_ID = SP.TABLE_ID AND P.PART_ID = SP.PART_ID AND T.TENANT_ID = SP.TENANT_ID LEFT JOIN OCEANBASE.__ALL_TENANT_TABLESPACE TP ON TP.TABLESPACE_ID = IFNULL(SP.TABLESPACE_ID, P.TABLESPACE_ID) AND TP.TENANT_ID = T.TENANT_ID LEFT JOIN OCEANBASE.__ALL_TABLE_STAT TS ON T.TENANT_ID = TS.TENANT_ID AND TS.TABLE_ID = T.TABLE_ID AND TS.PARTITION_ID = CASE T.PART_LEVEL WHEN 0 THEN T.TABLE_ID WHEN 1 THEN P.PART_ID WHEN 2 THEN SP.SUB_PART_ID END + LEFT JOIN ( + SELECT E.TENANT_ID AS TENANT_ID, + E.DATA_TABLE_ID AS DATA_TABLE_ID, + F.PART_IDX AS PART_IDX, + SF.SUB_PART_IDX AS SUB_PART_IDX, + SUM(G.macro_blk_cnt * 2 * 1024 * 1024) AS INDEX_LENGTH + FROM OCEANBASE.__ALL_TABLE E LEFT JOIN OCEANBASE.__ALL_PART F ON E.TENANT_ID = F.TENANT_ID AND E.TABLE_ID = F.TABLE_ID + LEFT JOIN OCEANBASE.__ALL_SUB_PART SF ON E.TENANT_ID = SF.TENANT_ID AND E.TABLE_ID = SF.TABLE_ID AND F.PART_ID = SF.PART_ID + JOIN OCEANBASE.__ALL_TABLE_STAT G ON E.TENANT_ID = G.TENANT_ID AND E.TABLE_ID = G.TABLE_ID AND G.PARTITION_ID = CASE E.PART_LEVEL WHEN 0 THEN E.TABLE_ID WHEN 1 THEN F.PART_ID WHEN 2 THEN SF.SUB_PART_ID END + WHERE E.INDEX_TYPE in (1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12) AND E.TABLE_TYPE = 5 GROUP BY TENANT_ID, DATA_TABLE_ID, PART_IDX, SUB_PART_IDX + ) IDX_STAT ON IDX_STAT.TENANT_ID = T.TENANT_ID AND + IDX_STAT.DATA_TABLE_ID = T.TABLE_ID AND + CASE T.PART_LEVEL WHEN 0 THEN 1 WHEN 1 THEN P.PART_IDX = IDX_STAT.PART_IDX WHEN 2 THEN P.PART_IDX = IDX_STAT.PART_IDX AND SP.SUB_PART_IDX = IDX_STAT.SUB_PART_IDX END WHERE T.TABLE_TYPE IN (3,6,8,9,14,15) """.replace("\n", " "), @@ -33404,16 +33585,13 @@ def_table_schema( cast(b.user_name as char(32)) as User, cast(a.routine_name as char(64)) as Routine_name, case when a.routine_type = 1 then 'PROCEDURE' else 'FUNCTION' end as Routine_type, - cast(c.priv_user as char(93)) as Grantor, + cast(concat(a.grantor, '@', a.grantor_host) as char(93)) as Grantor, substr(concat(case when (a.all_priv & 1) > 0 then ',Execute' else '' end, case when (a.all_priv & 2) > 0 then ',Alter Routine' else '' end, case when (a.all_priv & 4) > 0 then ',Grant' else '' end), 2) as Proc_priv, cast(a.gmt_modified as date) as Timestamp - FROM oceanbase.__all_routine_privilege a, oceanbase.__all_user b, oceanbase.__all_routine c, oceanbase.__all_database d - WHERE a.tenant_id = b.tenant_id AND a.user_id = b.user_id - AND a.tenant_id = d.tenant_id and a.database_name = d.database_name - AND a.tenant_id = c.tenant_id AND a.routine_name = c.routine_name - AND a.routine_type = c.routine_type AND c.database_id = d.database_id AND c.package_id = -1; + FROM oceanbase.__all_routine_privilege a, oceanbase.__all_user b + WHERE a.tenant_id = 0 and a.tenant_id = b.tenant_id AND a.user_id = b.user_id; """.replace("\n", " ") ) @@ -33786,6 +33964,40 @@ def_table_schema( #21514: audit_log_filter #21515: audit_log_user +def_table_schema( + owner = 'sean.yyj', + database_id = 'OB_MYSQL_SCHEMA_ID', + table_name = 'audit_log_filter', + table_id = '21514', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT CAST(filter_name AS CHAR(64)) collate utf8mb4_bin as NAME, + definition as FILTER + FROM oceanbase.__all_audit_log_filter WHERE tenant_id = 0 and is_deleted = 0; +""".replace("\n", " "), +) +def_table_schema( + owner = 'sean.yyj', + database_id = 'OB_MYSQL_SCHEMA_ID', + table_name = 'audit_log_user', + table_id = '21515', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT CAST(user_name AS CHAR(128)) collate utf8mb4_bin as USER, + host as HOST, + CAST(filter_name AS CHAR(64)) collate utf8mb4_bin as FILTERNAME + FROM oceanbase.__all_audit_log_user WHERE tenant_id = 0 and is_deleted = 0; +""".replace("\n", " "), +) + def_table_schema( owner = 'mingye.swj', database_id = 'OB_MYSQL_SCHEMA_ID', @@ -35783,9 +35995,156 @@ def_table_schema( """.replace("\n", " ") ) -# 21582: ROLE_TABLE_GRANTS -# 21583: ROLE_COLUMN_GRANTS -# 21584: ROLE_ROUTINE_GRANTS +def_table_schema( + owner = 'linyi.cl', + database_id = 'OB_INFORMATION_SCHEMA_ID', + table_name = 'ROLE_TABLE_GRANTS', + table_id = '21582', + table_type = 'SYSTEM_VIEW', + gm_columns = [], + rowkey_columns = [], + in_tenant_space = True, + + view_definition = """ + with recursive role_graph (from_user, from_host, to_user, to_host, is_enabled) + as ( + select user_name, host, cast('' as char(128)), cast('' as char(128)), false + from oceanbase.__all_user + where tenant_id=0 and concat(user_name, '@', host)=current_user() + union all + select role_edges.from_user, role_edges.from_host, role_edges.to_user, role_edges.to_host, + if ((role_graph.is_enabled + or is_enabled_role(role_edges.from_user, role_edges.from_host)), + true, + false) + from mysql.role_edges role_edges join role_graph + on role_edges.to_user = role_graph.from_user and role_edges.to_host = role_graph.from_host + ) + select distinct + cast(tp.grantor as char(97)) as GRANTOR, + cast(tp.grantor_host as char(256)) as GRANTOR_HOST, + cast(u.user_name as char(32)) as GRANTEE, + cast(u.host as char(255)) as GRANTEE_HOST, + cast('def' as char(3)) as TABLE_CATALOG, + cast(tp.database_name as char(64)) as TABLE_SCHEMA, + cast(tp.table_name as char(64)) as TABLE_NAME, + substr(concat(case when tp.priv_alter > 0 then ',Alter' else '' end, + case when tp.priv_create > 0 then ',Create' else '' end, + case when tp.priv_delete > 0 then ',Delete' else '' end, + case when tp.priv_drop > 0 then ',Drop' else '' end, + case when tp.priv_grant_option > 0 then ',Grant' else '' end, + case when tp.priv_insert > 0 then ',Insert' else '' end, + case when tp.priv_update > 0 then ',Update' else '' end, + case when tp.priv_select > 0 then ',Select' else '' end, + case when tp.priv_index > 0 then ',Index' else '' end, + case when tp.priv_create_view > 0 then ',Create View' else '' end, + case when tp.priv_show_view > 0 then ',Show View' else '' end, + case when (tp.priv_others & 64) > 0 then ',References' else '' end),2) as PRIVILEGE_TYPE, + cast(if (tp.priv_grant_option > 0,'YES','NO') as char(3)) AS IS_GRANTABLE + from (select distinct from_user, from_host, to_user, to_host, is_enabled from role_graph) rg + join oceanbase.__all_table_privilege tp join oceanbase.__all_user u + on tp.user_id = u.user_id and rg.from_user = u.user_name and rg.from_host = u.host + where rg.is_enabled and rg.to_user <> '' and tp.tenant_id = 0 and u.tenant_id = 0 + """.replace("\n", " "), + + normal_columns = [], +) + +def_table_schema( + owner = 'linyi.cl', + database_id = 'OB_INFORMATION_SCHEMA_ID', + table_name = 'ROLE_COLUMN_GRANTS', + table_id = '21583', + table_type = 'SYSTEM_VIEW', + gm_columns = [], + rowkey_columns = [], + in_tenant_space = True, + + view_definition = """ + with recursive role_graph (from_user, from_host, to_user, to_host, is_enabled) + as ( + select user_name, host, cast('' as char(128)), cast('' as char(128)), false + from oceanbase.__all_user + where tenant_id=0 and concat(user_name, '@', host)=current_user() + union all + select role_edges.from_user, role_edges.from_host, role_edges.to_user, role_edges.to_host, + if ((role_graph.is_enabled or is_enabled_role(role_edges.from_user, role_edges.from_host)), true, false) + from mysql.role_edges role_edges join role_graph + on role_edges.to_user = role_graph.from_user and role_edges.to_host = role_graph.from_host + ) + select distinct + NULL as GRANTOR, + NULL as GRANTOR_HOST, + cast(u.user_name as char(32)) as GRANTEE, + cast(u.host as char(255)) as GRANTEE_HOST, + cast('def' as char(3)) as TABLE_CATALOG, + cast(cp.database_name as char(64)) as TABLE_SCHEMA, + cast(cp.table_name as char(64)) as TABLE_NAME, + cast(cp.column_name as char(64)) as COLUMN_NAME, + substr(concat(case when (cp.all_priv & 1) > 0 then ',Select' else '' end, + case when (cp.all_priv & 2) > 0 then ',Insert' else '' end, + case when (cp.all_priv & 4) > 0 then ',Update' else '' end, + case when (cp.all_priv & 8) > 0 then ',References' else '' end), 2) as PRIVILEGE_TYPE, + cast(if (tp.priv_grant_option > 0,'YES','NO') as char(3)) AS IS_GRANTABLE + from ((select distinct from_user, from_host, to_user, to_host, is_enabled from role_graph) rg + join oceanbase.__all_user u join oceanbase.__all_column_privilege cp + on cp.user_id = u.user_id and rg.from_user = u.user_name and rg.from_host = u.host + and rg.is_enabled and rg.to_user <> '' and cp.tenant_id = 0 and u.tenant_id = 0) + left join + oceanbase.__all_table_privilege tp + on cp.database_name = tp.database_name and cp.table_name = tp.table_name + and cp.user_id = tp.user_id and tp.tenant_id = 0 + """.replace("\n", " "), + + normal_columns = [], +) + +def_table_schema( + owner = 'linyi.cl', + database_id = 'OB_INFORMATION_SCHEMA_ID', + table_name = 'ROLE_ROUTINE_GRANTS', + table_id = '21584', + table_type = 'SYSTEM_VIEW', + gm_columns = [], + rowkey_columns = [], + in_tenant_space = True, + + view_definition = """ + with recursive role_graph (from_user, from_host, to_user, to_host, is_enabled) + as ( + select user_name, host, cast('' as char(128)), cast('' as char(128)), false + from oceanbase.__all_user + where tenant_id=0 and concat(user_name, '@', host)=current_user() + union all + select role_edges.from_user, role_edges.from_host, role_edges.to_user, role_edges.to_host, + if ((role_graph.is_enabled or is_enabled_role(role_edges.from_user, role_edges.from_host)), true, false) + from mysql.role_edges role_edges join role_graph + on role_edges.to_user = role_graph.from_user and role_edges.to_host = role_graph.from_host + ) + select distinct + cast(rp.grantor as char(97)) as GRANTOR, + cast(rp.grantor_host as char(256)) as GRANTOR_HOST, + cast(u.user_name as char(32)) as GRANTEE, + cast(u.host as char(255)) as GRANTEE_HOST, + cast('def' as char(3)) AS SPECIFIC_CATALOG, + cast(rp.database_name as char(64)) AS SPECIFIC_SCHEMA, + cast(rp.routine_name as char(64)) AS SPECIFIC_NAME, + cast('def' as char(3)) AS ROUTINE_CATALOG, + cast(rp.database_name as char(64)) AS ROUTINE_SCHEMA, + cast(rp.routine_name as char(64)) AS ROUTINE_NAME, + substr(concat(case when (rp.all_priv & 1) > 0 then ',Execute' else '' end, + case when (rp.all_priv & 2) > 0 then ',Alter Routine' else '' end, + case when (rp.all_priv & 4) > 0 then ',Grant' else '' end), 2) AS PRIVILEGE_TYPE, + cast(if ((rp.all_priv & 4) > 0,'YES','NO') as char(3)) AS `IS_GRANTABLE` + from (select distinct from_user, from_host, to_user, to_host, is_enabled from role_graph) rg + join oceanbase.__all_routine_privilege rp join oceanbase.__all_user u + on rp.user_id = u.user_id and rg.from_user = u.user_name and rg.from_host = u.host + where rg.to_user <> '' and rg.is_enabled and u.tenant_id = 0 and rp.tenant_id = 0 + """.replace("\n", " "), + + normal_columns = [], +) + def_table_schema( owner = 'wangbai.wx', database_id = 'OB_MYSQL_SCHEMA_ID', diff --git a/src/share/inner_table/ob_inner_table_schema_misc.ipp b/src/share/inner_table/ob_inner_table_schema_misc.ipp index 5e0fa3e0b..3a0615b06 100644 --- a/src/share/inner_table/ob_inner_table_schema_misc.ipp +++ b/src/share/inner_table/ob_inner_table_schema_misc.ipp @@ -1789,6 +1789,8 @@ case OB_ALL_VIRTUAL_ZONE_MERGE_INFO_TID: #ifdef ITERATE_VIRTUAL_TABLE_LOCATION_SWITCH +case OB_ALL_VIRTUAL_AUDIT_LOG_FILTER_TID: +case OB_ALL_VIRTUAL_AUDIT_LOG_USER_TID: case OB_ALL_VIRTUAL_AUTO_INCREMENT_TID: case OB_ALL_VIRTUAL_AUX_STAT_TID: case OB_ALL_VIRTUAL_BALANCE_JOB_TID: @@ -1975,6 +1977,36 @@ case OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_HISTORY_TID: #ifdef ITERATE_VIRTUAL_TABLE_CREATE_ITER BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA + case OB_ALL_VIRTUAL_AUDIT_LOG_FILTER_TID: { + ObIterateVirtualTable *iter = NULL; + if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) { + SERVER_LOG(WARN, "create virtual table iterator failed", K(ret)); + } else if (OB_FAIL(iter->init(OB_ALL_AUDIT_LOG_FILTER_TID, index_schema, params))) { + SERVER_LOG(WARN, "virtual table iter init failed", K(ret)); + iter->~ObIterateVirtualTable(); + allocator.free(iter); + iter = NULL; + } else { + vt_iter = iter; + } + break; + } + + case OB_ALL_VIRTUAL_AUDIT_LOG_USER_TID: { + ObIterateVirtualTable *iter = NULL; + if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) { + SERVER_LOG(WARN, "create virtual table iterator failed", K(ret)); + } else if (OB_FAIL(iter->init(OB_ALL_AUDIT_LOG_USER_TID, index_schema, params))) { + SERVER_LOG(WARN, "virtual table iter init failed", K(ret)); + iter->~ObIterateVirtualTable(); + allocator.free(iter); + iter = NULL; + } else { + vt_iter = iter; + } + break; + } + case OB_ALL_VIRTUAL_AUTO_INCREMENT_TID: { ObIterateVirtualTable *iter = NULL; if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) { @@ -2244,7 +2276,9 @@ case OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_HISTORY_TID: } break; } + END_CREATE_VT_ITER_SWITCH_LAMBDA + BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA case OB_ALL_VIRTUAL_COLUMN_STAT_HISTORY_TID: { ObIterateVirtualTable *iter = NULL; if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) { @@ -2274,9 +2308,7 @@ case OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_HISTORY_TID: } break; } - END_CREATE_VT_ITER_SWITCH_LAMBDA - BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA case OB_ALL_VIRTUAL_CONSTRAINT_TID: { ObIterateVirtualTable *iter = NULL; if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) { @@ -2546,7 +2578,9 @@ case OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_HISTORY_TID: } break; } + END_CREATE_VT_ITER_SWITCH_LAMBDA + BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA case OB_ALL_VIRTUAL_DDL_TASK_STATUS_TID: { ObIterateVirtualTable *iter = NULL; if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) { @@ -2576,9 +2610,7 @@ case OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_HISTORY_TID: } break; } - END_CREATE_VT_ITER_SWITCH_LAMBDA - BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA case OB_ALL_VIRTUAL_DEF_SUB_PART_HISTORY_TID: { ObIterateVirtualTable *iter = NULL; if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) { @@ -2848,7 +2880,9 @@ case OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_HISTORY_TID: } break; } + END_CREATE_VT_ITER_SWITCH_LAMBDA + BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA case OB_ALL_VIRTUAL_MLOG_TID: { ObIterateVirtualTable *iter = NULL; if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) { @@ -2878,9 +2912,7 @@ case OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_HISTORY_TID: } break; } - END_CREATE_VT_ITER_SWITCH_LAMBDA - BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA case OB_ALL_VIRTUAL_MOCK_FK_PARENT_TABLE_COLUMN_TID: { ObIterateVirtualTable *iter = NULL; if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) { @@ -3150,7 +3182,9 @@ case OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_HISTORY_TID: } break; } + END_CREATE_VT_ITER_SWITCH_LAMBDA + BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA case OB_ALL_VIRTUAL_OUTLINE_HISTORY_TID: { ObIterateVirtualTable *iter = NULL; if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) { @@ -3180,9 +3214,7 @@ case OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_HISTORY_TID: } break; } - END_CREATE_VT_ITER_SWITCH_LAMBDA - BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA case OB_ALL_VIRTUAL_PACKAGE_HISTORY_TID: { ObIterateVirtualTable *iter = NULL; if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) { @@ -3452,7 +3484,9 @@ case OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_HISTORY_TID: } break; } + END_CREATE_VT_ITER_SWITCH_LAMBDA + BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA case OB_ALL_VIRTUAL_RLS_SECURITY_COLUMN_HISTORY_TID: { ObIterateVirtualTable *iter = NULL; if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) { @@ -3482,9 +3516,7 @@ case OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_HISTORY_TID: } break; } - END_CREATE_VT_ITER_SWITCH_LAMBDA - BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA case OB_ALL_VIRTUAL_ROUTINE_HISTORY_TID: { ObIterateVirtualTable *iter = NULL; if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) { @@ -3754,7 +3786,9 @@ case OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_HISTORY_TID: } break; } + END_CREATE_VT_ITER_SWITCH_LAMBDA + BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA case OB_ALL_VIRTUAL_SYS_STAT_TID: { ObIterateVirtualTable *iter = NULL; if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) { @@ -3784,9 +3818,7 @@ case OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_HISTORY_TID: } break; } - END_CREATE_VT_ITER_SWITCH_LAMBDA - BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA case OB_ALL_VIRTUAL_SYS_VARIABLE_HISTORY_TID: { ObIterateVirtualTable *iter = NULL; if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) { @@ -4056,7 +4088,9 @@ case OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_HISTORY_TID: } break; } + END_CREATE_VT_ITER_SWITCH_LAMBDA + BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA case OB_ALL_VIRTUAL_TENANT_KEYSTORE_TID: { ObIterateVirtualTable *iter = NULL; if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) { @@ -4086,9 +4120,7 @@ case OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_HISTORY_TID: } break; } - END_CREATE_VT_ITER_SWITCH_LAMBDA - BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA case OB_ALL_VIRTUAL_TENANT_OLS_COMPONENT_TID: { ObIterateVirtualTable *iter = NULL; if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) { @@ -4358,7 +4390,9 @@ case OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_HISTORY_TID: } break; } + END_CREATE_VT_ITER_SWITCH_LAMBDA + BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA case OB_ALL_VIRTUAL_TENANT_TABLESPACE_HISTORY_TID: { ObIterateVirtualTable *iter = NULL; if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) { @@ -4388,9 +4422,7 @@ case OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_HISTORY_TID: } break; } - END_CREATE_VT_ITER_SWITCH_LAMBDA - BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA case OB_ALL_VIRTUAL_TIME_ZONE_NAME_TID: { ObIterateVirtualTable *iter = NULL; if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) { @@ -4660,7 +4692,9 @@ case OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_HISTORY_TID: } break; } + END_CREATE_VT_ITER_SWITCH_LAMBDA + BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA case OB_ALL_VIRTUAL_USER_PROXY_ROLE_INFO_HISTORY_TID: { ObIterateVirtualTable *iter = NULL; if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) { diff --git a/src/share/inner_table/table_id_to_name b/src/share/inner_table/table_id_to_name index a84b7a15b..f558bfc9d 100644 --- a/src/share/inner_table/table_id_to_name +++ b/src/share/inner_table/table_id_to_name @@ -344,6 +344,8 @@ # 499: __all_transfer_partition_task # BASE_TABLE_NAME # 500: __all_tenant_snapshot_job # 502: __all_trusted_root_certificate +# 503: __all_audit_log_filter +# 504: __all_audit_log_user # 505: __all_column_privilege # 506: __all_column_privilege_history # 506: __all_column_privilege # BASE_TABLE_NAME @@ -1099,6 +1101,10 @@ # 12458: __all_virtual_ls_snapshot # 12459: __all_virtual_index_usage_info # 12459: __all_index_usage_info # BASE_TABLE_NAME +# 12460: __all_virtual_audit_log_filter +# 12460: __all_audit_log_filter # BASE_TABLE_NAME +# 12461: __all_virtual_audit_log_user +# 12461: __all_audit_log_user # BASE_TABLE_NAME # 12462: __all_virtual_column_privilege # 12462: __all_column_privilege # BASE_TABLE_NAME # 12463: __all_virtual_column_privilege_history @@ -1757,6 +1763,9 @@ # 20014: ENGINES # 20015: ROUTINES # 20016: PROFILING +# 20017: OPTIMIZER_TRACE +# 20018: PLUGINS +# 20019: INNODB_SYS_COLUMNS # 20020: INNODB_FT_BEING_DELETED # 20021: INNODB_FT_CONFIG # 20022: INNODB_FT_DELETED @@ -2134,6 +2143,8 @@ # 21511: role_edges # 21512: default_roles # 21513: CDB_INDEX_USAGE +# 21514: audit_log_filter +# 21515: audit_log_user # 21516: columns_priv # 21517: GV$OB_LS_SNAPSHOTS # 21518: V$OB_LS_SNAPSHOTS @@ -2187,6 +2198,9 @@ # 21579: INNODB_METRICS # 21580: EVENTS # 21581: V$OB_NIC_INFO +# 21582: ROLE_TABLE_GRANTS +# 21583: ROLE_COLUMN_GRANTS +# 21584: ROLE_ROUTINE_GRANTS # 21585: func # 21586: GV$OB_NIC_INFO # 21589: DBA_SCHEDULER_JOB_RUN_DETAILS diff --git a/src/share/ob_compatibility_security_feature_def.h b/src/share/ob_compatibility_security_feature_def.h index d46c0baca..2add33000 100644 --- a/src/share/ob_compatibility_security_feature_def.h +++ b/src/share/ob_compatibility_security_feature_def.h @@ -26,4 +26,10 @@ DEF_COMPAT_CONTROL_FEATURE(MYSQL_USER_REVOKE_ALL_ENHANCE, "use create_user to ch DEF_COMPAT_CONTROL_FEATURE(MYSQL_USER_REVOKE_ALL_WITH_PL_PRIV_CHECK, "revoke all on db.* need check pl privilege", MOCK_CLUSTER_VERSION_4_2_4_0, CLUSTER_VERSION_4_3_0_0, CLUSTER_VERSION_4_3_2_0) +DEF_COMPAT_CONTROL_FEATURE(MYSQL_REFERENCES_PRIV_ENHANCE, "add privilege check to references", + MOCK_CLUSTER_VERSION_4_2_4_0, CLUSTER_VERSION_4_3_0_0, + CLUSTER_VERSION_4_3_3_0) +DEF_COMPAT_CONTROL_FEATURE(MYSQL_TRIGGER_PRIV_CHECK, "add trigger privilege check", + MOCK_CLUSTER_VERSION_4_2_4_0, CLUSTER_VERSION_4_3_0_0, + CLUSTER_VERSION_4_3_3_0) #endif diff --git a/src/share/ob_encryption_util.h b/src/share/ob_encryption_util.h index 940b3979e..d88876027 100644 --- a/src/share/ob_encryption_util.h +++ b/src/share/ob_encryption_util.h @@ -23,6 +23,10 @@ namespace common { class ObString; } +namespace sql +{ +class ObSQLSessionInfo; +} namespace share { struct ObEncryptMeta; @@ -140,6 +144,8 @@ public: static int parse_encryption_id(const common::ObString &str, int64_t &encrypt_id); static bool is_aes_encryption(const ObCipherOpMode opmode); static bool is_sm4_encryption(const ObCipherOpMode opmode); + static bool is_ecb_mode(const ObCipherOpMode opmode); + static int get_cipher_op_mode(ObCipherOpMode &op_mode, sql::ObSQLSessionInfo *session); #ifdef OB_BUILD_TDE_SECURITY static const ObCipherOpMode DEFAULT_TABLE_KEY_AES_ENCRYPT_ALGORITHM = @@ -256,9 +262,12 @@ enum ObHashAlgorithm { OB_HASH_SH256, OB_HASH_SH384, OB_HASH_SH512, + OB_HASH_SM3, OB_HASH_MAX }; +const int64_t OB_SM3_DIGEST_LENGTH = 32; // 256 bits + class ObHashUtil { public: diff --git a/src/share/ob_encryption_util_os.cpp b/src/share/ob_encryption_util_os.cpp index 1ec9817df..25fa5eb07 100644 --- a/src/share/ob_encryption_util_os.cpp +++ b/src/share/ob_encryption_util_os.cpp @@ -416,6 +416,40 @@ bool ObEncryptionUtil::is_sm4_encryption(const ObCipherOpMode opmode) { (opmode >= ObCipherOpMode::ob_sm4_cbc && opmode <= ObCipherOpMode::ob_sm4_gcm); } +bool ObEncryptionUtil::is_ecb_mode(const ObCipherOpMode opmode) +{ + return (opmode >= ObCipherOpMode::ob_aes_128_ecb && opmode <= ob_aes_256_ecb) || + (opmode == ObCipherOpMode::ob_sm4_ecb); +} + +int ObEncryptionUtil::get_cipher_op_mode(ObCipherOpMode &op_mode, ObSQLSessionInfo *session) +{ + int ret = OB_SUCCESS; + int64_t encryption = -1; + if (OB_ISNULL(session)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("session is NULL", K(ret)); + } else if (OB_FAIL(session->get_sys_variable(SYS_VAR_BLOCK_ENCRYPTION_MODE, encryption))) { + LOG_WARN("fail to get block encryption variable", K(ret)); + } else if (encryption >= 0 && encryption <= 17) { + encryption++; + op_mode = static_cast(encryption); + } else if (18 == encryption) { + op_mode = ObCipherOpMode::ob_sm4_ecb; + } else if (19 == encryption) { + op_mode = ObCipherOpMode::ob_sm4_cbc; + } else if (20 == encryption) { + op_mode = ObCipherOpMode::ob_sm4_cfb128; + } else if (21 == encryption) { + op_mode = ObCipherOpMode::ob_sm4_ofb; + } else { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "Variable block_encryption_mode used"); + LOG_WARN("got unsupported block encryption mode", K(ret), K(encryption)); + } + return ret; +} + bool ObBackupEncryptionMode::is_valid(const EncryptionMode &mode) { return mode >= NONE && mode < MAX_MODE; @@ -562,6 +596,9 @@ int ObHashUtil::get_hash_output_len(const ObHashAlgorithm algo, int64_t &output_ case OB_HASH_SH512: output_len = SHA512_DIGEST_LENGTH; break; + case OB_HASH_SM3: + output_len = OB_SM3_DIGEST_LENGTH; + break; default: ret = OB_NOT_SUPPORTED; LOG_USER_ERROR(OB_NOT_SUPPORTED, "cipher type passed"); @@ -606,6 +643,7 @@ const EVP_MD* ObHashUtil::get_hash_evp_md(const ObHashAlgorithm algo) case OB_HASH_SH256: return EVP_sha256(); case OB_HASH_SH384: return EVP_sha384(); case OB_HASH_SH512: return EVP_sha512(); + case OB_HASH_SM3: return EVP_sm3(); default: return NULL; } } diff --git a/src/share/ob_errno.cpp b/src/share/ob_errno.cpp index 5445e6393..c5b7fefbf 100644 --- a/src/share/ob_errno.cpp +++ b/src/share/ob_errno.cpp @@ -4205,13 +4205,13 @@ static const _error _error_OB_ERR_AES_IV_LENGTH = { .error_solution = "Contact OceanBase Support", .mysql_errno = ER_AES_INVALID_IV, .sqlstate = "HY000", - .str_error = "The initialization vector supplied to aes_encrypt is too short. Must be at least 16 bytes long", - .str_user_error = "The initialization vector supplied to aes_encrypt is too short. Must be at least 16 bytes long", + .str_error = "The initialization vector supplied is too short. Must be at least 16 bytes long", + .str_user_error = "The initialization vector supplied is too short. Must be at least 16 bytes long", .oracle_errno = 600, - .oracle_str_error = "ORA-00600: internal error code, arguments: -4336, The initialization vector supplied to aes_encrypt is too short. Must be at least 16 bytes long", - .oracle_str_user_error = "ORA-00600: internal error code, arguments: -4336, The initialization vector supplied to aes_encrypt is too short. Must be at least 16 bytes long", - .ob_str_error = "OBE-00600: internal error code, arguments: -4336, The initialization vector supplied to aes_encrypt is too short. Must be at least 16 bytes long", - .ob_str_user_error = "OBE-00600: internal error code, arguments: -4336, The initialization vector supplied to aes_encrypt is too short. Must be at least 16 bytes long" + .oracle_str_error = "ORA-00600: internal error code, arguments: -4336, The initialization vector supplied is too short. Must be at least 16 bytes long", + .oracle_str_user_error = "ORA-00600: internal error code, arguments: -4336, The initialization vector supplied is too short. Must be at least 16 bytes long", + .ob_str_error = "OBE-00600: internal error code, arguments: -4336, The initialization vector supplied is too short. Must be at least 16 bytes long", + .ob_str_user_error = "OBE-00600: internal error code, arguments: -4336, The initialization vector supplied is too short. Must be at least 16 bytes long" }; static const _error _error_OB_STORE_DIR_ERROR = { .error_name = "OB_STORE_DIR_ERROR", @@ -10296,12 +10296,12 @@ static const _error _error_OB_CANT_AGGREGATE_2COLLATIONS = { .mysql_errno = ER_CANT_AGGREGATE_2COLLATIONS, .sqlstate = "HY000", .str_error = "Illegal mix of collations", - .str_user_error = "Illegal mix of collations", + .str_user_error = "Illegal mix of collations (%s,%s), (%s,%s)", .oracle_errno = 600, .oracle_str_error = "ORA-00600: internal error code, arguments: -5177, Illegal mix of collations", - .oracle_str_user_error = "ORA-00600: internal error code, arguments: -5177, Illegal mix of collations", + .oracle_str_user_error = "ORA-00600: internal error code, arguments: -5177, Illegal mix of collations (%s,%s), (%s,%s)", .ob_str_error = "OBE-00600: internal error code, arguments: -5177, Illegal mix of collations", - .ob_str_user_error = "OBE-00600: internal error code, arguments: -5177, Illegal mix of collations" + .ob_str_user_error = "OBE-00600: internal error code, arguments: -5177, Illegal mix of collations (%s,%s), (%s,%s)" }; static const _error _error_OB_ERR_FIELD_TYPE_NOT_ALLOWED_AS_PARTITION_FIELD = { .error_name = "OB_ERR_FIELD_TYPE_NOT_ALLOWED_AS_PARTITION_FIELD", @@ -12774,12 +12774,12 @@ static const _error _error_OB_CANT_AGGREGATE_3COLLATIONS = { .mysql_errno = ER_CANT_AGGREGATE_3COLLATIONS, .sqlstate = "HY000", .str_error = "Illegal mix of collations", - .str_user_error = "Illegal mix of collations", + .str_user_error = "Illegal mix of collations (%s, %s),(%s, %s),(%s, %s)", .oracle_errno = 600, .oracle_str_error = "ORA-00600: internal error code, arguments: -5356, Illegal mix of collations", - .oracle_str_user_error = "ORA-00600: internal error code, arguments: -5356, Illegal mix of collations", + .oracle_str_user_error = "ORA-00600: internal error code, arguments: -5356, Illegal mix of collations (%s, %s),(%s, %s),(%s, %s)", .ob_str_error = "OBE-00600: internal error code, arguments: -5356, Illegal mix of collations", - .ob_str_user_error = "OBE-00600: internal error code, arguments: -5356, Illegal mix of collations" + .ob_str_user_error = "OBE-00600: internal error code, arguments: -5356, Illegal mix of collations (%s, %s),(%s, %s),(%s, %s)" }; static const _error _error_OB_CANT_AGGREGATE_NCOLLATIONS = { .error_name = "OB_CANT_AGGREGATE_NCOLLATIONS", diff --git a/src/share/ob_errno.def b/src/share/ob_errno.def index 2bbc83c3b..eee41eee9 100755 --- a/src/share/ob_errno.def +++ b/src/share/ob_errno.def @@ -406,7 +406,7 @@ DEFINE_ORACLE_ERROR_EXT(OB_ERR_SEQ_VALUE_EXCEED_LIMIT, -4332, -1, "HY000", "sequ DEFINE_ORACLE_ERROR(OB_ERR_DIVISOR_IS_ZERO, -4333,-1, "HY000", "divisor is equal to zero", 1476, "divisor is equal to zero"); DEFINE_ERROR(OB_ERR_AES_DECRYPT, -4334, -1, "HY000", "fail to decrypt data"); DEFINE_ERROR(OB_ERR_AES_ENCRYPT, -4335, -1, "HY000", "fail to encrypt data"); -DEFINE_ERROR(OB_ERR_AES_IV_LENGTH, -4336, ER_AES_INVALID_IV, "HY000", "The initialization vector supplied to aes_encrypt is too short. Must be at least 16 bytes long"); +DEFINE_ERROR(OB_ERR_AES_IV_LENGTH, -4336, ER_AES_INVALID_IV, "HY000", "The initialization vector supplied is too short. Must be at least 16 bytes long"); DEFINE_ERROR(OB_STORE_DIR_ERROR, -4337, -1, "HY000", "store directory structure error"); DEFINE_ERROR(OB_OPEN_TWICE, -4338, -1, "HY000", "open twice"); DEFINE_ERROR(OB_RAID_SUPER_BLOCK_NOT_MACTH, -4339, -1, "HY000", "raid super block not match"); @@ -883,7 +883,7 @@ DEFINE_ERROR_EXT(OB_INVALID_DEFAULT, -5173, ER_INVALID_DEFAULT, "42000", "Invali DEFINE_ERROR_EXT(OB_ERR_UPDATE_TABLE_USED, -5174, ER_UPDATE_TABLE_USED, "HY000", "Update table used", "You can\'t specify target table \'%s\' for update in FROM clause"); DEFINE_ERROR_EXT(OB_ERR_COULUMN_VALUE_NOT_MATCH, -5175, ER_WRONG_VALUE_COUNT_ON_ROW, "21S01", "Column count doesn\'t match value count", "Column count doesn\'t match value count at row %ld"); DEFINE_ERROR(OB_ERR_INVALID_GROUP_FUNC_USE, -5176, ER_INVALID_GROUP_FUNC_USE, "HY000", "Invalid use of group function"); -DEFINE_ERROR_DEP(OB_CANT_AGGREGATE_2COLLATIONS, -5177, ER_CANT_AGGREGATE_2COLLATIONS, "HY000", "Illegal mix of collations"); +DEFINE_ERROR_EXT_DEP(OB_CANT_AGGREGATE_2COLLATIONS, -5177, ER_CANT_AGGREGATE_2COLLATIONS, "HY000", "Illegal mix of collations","Illegal mix of collations (%s,%s), (%s,%s)"); DEFINE_ERROR_EXT(OB_ERR_FIELD_TYPE_NOT_ALLOWED_AS_PARTITION_FIELD, -5178, ER_FIELD_TYPE_NOT_ALLOWED_AS_PARTITION_FIELD, "HY000", "Field is of a not allowed type for this type of partitioning", "Field \'%.*s\' is of a not allowed type for this type of partitioning"); DEFINE_ORACLE_ERROR_EXT(OB_ERR_TOO_LONG_IDENT, -5179, ER_TOO_LONG_IDENT, "42000", "Identifier name is too long", "Identifier name \'%.*s\' is too long", 972, "identifier is too long", "identifier \'%.*s\' is too long"); DEFINE_ERROR_EXT(OB_ERR_WRONG_TYPE_FOR_VAR, -5180, ER_WRONG_TYPE_FOR_VAR, "42000", "Incorrect argument type to variable", "Incorrect argument type to variable '%.*s'"); @@ -1060,7 +1060,7 @@ DEFINE_ORACLE_ERROR_DEP(OB_ERR_FIELD_NOT_FOUND_IN_DATETIME_OR_INTERVAL, -5352, - DEFINE_ORACLE_ERROR(OB_ERR_ADD_PART_BOUN_NOT_INC, -5353, ER_RANGE_NOT_INCREASING_ERROR, "HY000", "VALUES LESS THAN value must be strictly increasing for each partition", 14074, "partition bound must collate higher than that of the last partition"); DEFINE_ORACLE_ERROR(OB_ERR_DATA_TOO_LONG_IN_PART_CHECK,-5354, ER_DATA_TOO_LONG, "22001", "Data too long for column", 14036, "partition bound value too large for column"); DEFINE_ORACLE_ERROR(OB_ERR_WRONG_TYPE_COLUMN_VALUE_V2_ERROR, -5355, ER_WRONG_TYPE_COLUMN_VALUE_ERROR, "HY000", "Partition column values of incorrect type", 14308, "partition bound element must be one of: string, datetime or interval literal, number, or NULL"); -DEFINE_ERROR(OB_CANT_AGGREGATE_3COLLATIONS, -5356, ER_CANT_AGGREGATE_3COLLATIONS, "HY000", "Illegal mix of collations"); +DEFINE_ERROR_EXT_DEP(OB_CANT_AGGREGATE_3COLLATIONS, -5356, ER_CANT_AGGREGATE_3COLLATIONS, "HY000", "Illegal mix of collations","Illegal mix of collations (%s, %s),(%s, %s),(%s, %s)"); DEFINE_ERROR(OB_CANT_AGGREGATE_NCOLLATIONS, -5357, ER_CANT_AGGREGATE_NCOLLATIONS, "HY000", "Illegal mix of collations"); DEFINE_ORACLE_ERROR_EXT(OB_ERR_DUPLICATED_UNIQUE_KEY, -5358, ER_DUP_ENTRY, "23000", "Duplicated primary key", "Duplicate entry \'%s\' for key \'%.*s\'", 1452, "cannot CREATE UNIQUE INDEX; duplicate keys found", "cannot CREATE UNIQUE INDEX; duplicate keys found"); DEFINE_ORACLE_ERROR(OB_DOUBLE_OVERFLOW, -5359, ER_WARN_DATA_OUT_OF_RANGE, "22003", "result Double value is out of range", 1426, "Double value overflow"); diff --git a/src/share/ob_errno.h b/src/share/ob_errno.h index 2bb009656..588f0adf7 100755 --- a/src/share/ob_errno.h +++ b/src/share/ob_errno.h @@ -773,7 +773,6 @@ constexpr int OB_ERR_INVALID_TYPE_FOR_ARGUMENT = -5351; constexpr int OB_ERR_ADD_PART_BOUN_NOT_INC = -5353; constexpr int OB_ERR_DATA_TOO_LONG_IN_PART_CHECK = -5354; constexpr int OB_ERR_WRONG_TYPE_COLUMN_VALUE_V2_ERROR = -5355; -constexpr int OB_CANT_AGGREGATE_3COLLATIONS = -5356; constexpr int OB_CANT_AGGREGATE_NCOLLATIONS = -5357; constexpr int OB_ERR_DUPLICATED_UNIQUE_KEY = -5358; constexpr int OB_DOUBLE_OVERFLOW = -5359; @@ -2220,7 +2219,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_DIVISOR_IS_ZERO__USER_ERROR_MSG "divisor is equal to zero" #define OB_ERR_AES_DECRYPT__USER_ERROR_MSG "fail to decrypt data" #define OB_ERR_AES_ENCRYPT__USER_ERROR_MSG "fail to encrypt data" -#define OB_ERR_AES_IV_LENGTH__USER_ERROR_MSG "The initialization vector supplied to aes_encrypt is too short. Must be at least 16 bytes long" +#define OB_ERR_AES_IV_LENGTH__USER_ERROR_MSG "The initialization vector supplied is too short. Must be at least 16 bytes long" #define OB_STORE_DIR_ERROR__USER_ERROR_MSG "store directory structure error" #define OB_OPEN_TWICE__USER_ERROR_MSG "open twice" #define OB_RAID_SUPER_BLOCK_NOT_MACTH__USER_ERROR_MSG "raid super block not match" @@ -2655,7 +2654,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_UPDATE_TABLE_USED__USER_ERROR_MSG "You can\'t specify target table \'%s\' for update in FROM clause" #define OB_ERR_COULUMN_VALUE_NOT_MATCH__USER_ERROR_MSG "Column count doesn\'t match value count at row %ld" #define OB_ERR_INVALID_GROUP_FUNC_USE__USER_ERROR_MSG "Invalid use of group function" -#define OB_CANT_AGGREGATE_2COLLATIONS__USER_ERROR_MSG "Illegal mix of collations" +#define OB_CANT_AGGREGATE_2COLLATIONS__USER_ERROR_MSG "Illegal mix of collations (%s,%s), (%s,%s)" #define OB_ERR_FIELD_TYPE_NOT_ALLOWED_AS_PARTITION_FIELD__USER_ERROR_MSG "Field \'%.*s\' is of a not allowed type for this type of partitioning" #define OB_ERR_TOO_LONG_IDENT__USER_ERROR_MSG "Identifier name \'%.*s\' is too long" #define OB_ERR_WRONG_TYPE_FOR_VAR__USER_ERROR_MSG "Incorrect argument type to variable '%.*s'" @@ -2832,7 +2831,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_ADD_PART_BOUN_NOT_INC__USER_ERROR_MSG "VALUES LESS THAN value must be strictly increasing for each partition" #define OB_ERR_DATA_TOO_LONG_IN_PART_CHECK__USER_ERROR_MSG "Data too long for column" #define OB_ERR_WRONG_TYPE_COLUMN_VALUE_V2_ERROR__USER_ERROR_MSG "Partition column values of incorrect type" -#define OB_CANT_AGGREGATE_3COLLATIONS__USER_ERROR_MSG "Illegal mix of collations" +#define OB_CANT_AGGREGATE_3COLLATIONS__USER_ERROR_MSG "Illegal mix of collations (%s, %s),(%s, %s),(%s, %s)" #define OB_CANT_AGGREGATE_NCOLLATIONS__USER_ERROR_MSG "Illegal mix of collations" #define OB_ERR_DUPLICATED_UNIQUE_KEY__USER_ERROR_MSG "Duplicate entry \'%s\' for key \'%.*s\'" #define OB_DOUBLE_OVERFLOW__USER_ERROR_MSG "result Double value is out of range" @@ -4846,8 +4845,8 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_AES_DECRYPT__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -4334, fail to decrypt data" #define OB_ERR_AES_ENCRYPT__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -4335, fail to encrypt data" #define OB_ERR_AES_ENCRYPT__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -4335, fail to encrypt data" -#define OB_ERR_AES_IV_LENGTH__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -4336, The initialization vector supplied to aes_encrypt is too short. Must be at least 16 bytes long" -#define OB_ERR_AES_IV_LENGTH__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -4336, The initialization vector supplied to aes_encrypt is too short. Must be at least 16 bytes long" +#define OB_ERR_AES_IV_LENGTH__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -4336, The initialization vector supplied is too short. Must be at least 16 bytes long" +#define OB_ERR_AES_IV_LENGTH__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -4336, The initialization vector supplied is too short. Must be at least 16 bytes long" #define OB_STORE_DIR_ERROR__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -4337, store directory structure error" #define OB_STORE_DIR_ERROR__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -4337, store directory structure error" #define OB_OPEN_TWICE__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -4338, open twice" @@ -5716,8 +5715,8 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_COULUMN_VALUE_NOT_MATCH__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -5175, Column count doesn\'t match value count at row %ld" #define OB_ERR_INVALID_GROUP_FUNC_USE__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -5176, Invalid use of group function" #define OB_ERR_INVALID_GROUP_FUNC_USE__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -5176, Invalid use of group function" -#define OB_CANT_AGGREGATE_2COLLATIONS__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -5177, Illegal mix of collations" -#define OB_CANT_AGGREGATE_2COLLATIONS__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -5177, Illegal mix of collations" +#define OB_CANT_AGGREGATE_2COLLATIONS__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -5177, Illegal mix of collations (%s,%s), (%s,%s)" +#define OB_CANT_AGGREGATE_2COLLATIONS__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -5177, Illegal mix of collations (%s,%s), (%s,%s)" #define OB_ERR_FIELD_TYPE_NOT_ALLOWED_AS_PARTITION_FIELD__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -5178, Field \'%.*s\' is of a not allowed type for this type of partitioning" #define OB_ERR_FIELD_TYPE_NOT_ALLOWED_AS_PARTITION_FIELD__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -5178, Field \'%.*s\' is of a not allowed type for this type of partitioning" #define OB_ERR_TOO_LONG_IDENT__ORA_USER_ERROR_MSG "ORA-00972: identifier \'%.*s\' is too long" @@ -6070,8 +6069,8 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_DATA_TOO_LONG_IN_PART_CHECK__OBE_USER_ERROR_MSG "OBE-14036: partition bound value too large for column" #define OB_ERR_WRONG_TYPE_COLUMN_VALUE_V2_ERROR__ORA_USER_ERROR_MSG "ORA-14308: partition bound element must be one of: string, datetime or interval literal, number, or NULL" #define OB_ERR_WRONG_TYPE_COLUMN_VALUE_V2_ERROR__OBE_USER_ERROR_MSG "OBE-14308: partition bound element must be one of: string, datetime or interval literal, number, or NULL" -#define OB_CANT_AGGREGATE_3COLLATIONS__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -5356, Illegal mix of collations" -#define OB_CANT_AGGREGATE_3COLLATIONS__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -5356, Illegal mix of collations" +#define OB_CANT_AGGREGATE_3COLLATIONS__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -5356, Illegal mix of collations (%s, %s),(%s, %s),(%s, %s)" +#define OB_CANT_AGGREGATE_3COLLATIONS__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -5356, Illegal mix of collations (%s, %s),(%s, %s),(%s, %s)" #define OB_CANT_AGGREGATE_NCOLLATIONS__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -5357, Illegal mix of collations" #define OB_CANT_AGGREGATE_NCOLLATIONS__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -5357, Illegal mix of collations" #define OB_ERR_DUPLICATED_UNIQUE_KEY__ORA_USER_ERROR_MSG "ORA-01452: cannot CREATE UNIQUE INDEX; duplicate keys found" diff --git a/src/share/ob_fts_index_builder_util.cpp b/src/share/ob_fts_index_builder_util.cpp index 01d71aa57..259dbce40 100644 --- a/src/share/ob_fts_index_builder_util.cpp +++ b/src/share/ob_fts_index_builder_util.cpp @@ -940,7 +940,8 @@ int ObFtsIndexBuilderUtil::generate_doc_id_column( LOG_WARN("set column name failed", K(ret)); } else if (OB_FAIL(column_schema.set_orig_default_value(default_value))) { LOG_WARN("set orig default value failed", K(ret)); - } else if (OB_FAIL(column_schema.set_cur_default_value(default_value))) { + } else if (OB_FAIL(column_schema.set_cur_default_value(default_value, + column_schema.is_default_expr_v2_column()))) { LOG_WARN("set current default value failed", K(ret)); } else if (OB_FAIL(data_schema.add_column(column_schema))) { LOG_WARN("add column schema to data table failed", K(ret)); @@ -1062,7 +1063,8 @@ int ObFtsIndexBuilderUtil::generate_word_segment_column( LOG_WARN("set column name failed", K(ret)); } else if (OB_FAIL(column_schema.set_orig_default_value(default_value))) { LOG_WARN("set orig default value failed", K(ret)); - } else if (OB_FAIL(column_schema.set_cur_default_value(default_value))) { + } else if (OB_FAIL(column_schema.set_cur_default_value(default_value, + column_schema.is_default_expr_v2_column()))) { LOG_WARN("set current default value failed", K(ret)); } else if (OB_FAIL(data_schema.add_column(column_schema))) { LOG_WARN("add column schema to data table failed", K(ret)); @@ -1176,7 +1178,8 @@ int ObFtsIndexBuilderUtil::generate_word_count_column( LOG_WARN("set column name failed", K(ret)); } else if (OB_FAIL(column_schema.set_orig_default_value(default_value))) { LOG_WARN("set orig default value failed", K(ret)); - } else if (OB_FAIL(column_schema.set_cur_default_value(default_value))) { + } else if (OB_FAIL(column_schema.set_cur_default_value(default_value, + column_schema.is_default_expr_v2_column()))) { LOG_WARN("set current default value failed", K(ret)); } else if (OB_FAIL(data_schema.add_column(column_schema))) { LOG_WARN("add word_count column schema to data table failed", K(ret)); @@ -1286,7 +1289,8 @@ int ObFtsIndexBuilderUtil::generate_doc_length_column( LOG_WARN("set column name failed", K(ret)); } else if (OB_FAIL(column_schema.set_orig_default_value(default_value))) { LOG_WARN("set orig default value failed", K(ret)); - } else if (OB_FAIL(column_schema.set_cur_default_value(default_value))) { + } else if (OB_FAIL(column_schema.set_cur_default_value(default_value, + column_schema.is_default_expr_v2_column()))) { LOG_WARN("set current default value failed", K(ret)); } else if (OB_FAIL(data_schema.add_column(column_schema))) { LOG_WARN("add word_count column schema to data table failed", K(ret)); @@ -2277,7 +2281,8 @@ int ObMulValueIndexBuilderUtil::generate_multivalue_column( LOG_WARN("set column name failed", K(ret)); } else if (OB_FAIL(multival_col.set_orig_default_value(default_value))) { LOG_WARN("set orig default value failed", K(ret)); - } else if (OB_FAIL(multival_col.set_cur_default_value(default_value))) { + } else if (OB_FAIL(multival_col.set_cur_default_value(default_value, + multival_col.is_default_expr_v2_column()))) { LOG_WARN("set current default value failed", K(ret)); } else if (OB_FAIL(data_schema.add_column(multival_col))) { LOG_WARN("add column schema to data table failed", K(ret)); @@ -2315,7 +2320,8 @@ int ObMulValueIndexBuilderUtil::generate_multivalue_column( LOG_WARN("set column name failed", K(ret)); } else if (OB_FAIL(multival_arr_col.set_orig_default_value(default_value))) { LOG_WARN("set orig default value failed", K(ret)); - } else if (OB_FAIL(multival_arr_col.set_cur_default_value(default_value))) { + } else if (OB_FAIL(multival_arr_col.set_cur_default_value(default_value, + multival_arr_col.is_default_expr_v2_column()))) { LOG_WARN("set current default value failed", K(ret)); } else if (OB_FAIL(data_schema.add_column(multival_arr_col))) { LOG_WARN("add column schema to data table failed", K(ret)); diff --git a/src/share/ob_i_tablet_scan.h b/src/share/ob_i_tablet_scan.h index f11854de9..3a367c878 100644 --- a/src/share/ob_i_tablet_scan.h +++ b/src/share/ob_i_tablet_scan.h @@ -86,6 +86,16 @@ struct SampleInfo force_block_ = false; } + bool same_as(const SampleInfo &oth) const { + return table_id_ == oth.table_id_ + && method_ == oth.method_ + && scope_ == oth.scope_ + && percent_ == oth.percent_ + && seed_ == oth.seed_ + && force_block_ == oth.force_block_ + && seed_ != -1; + } + uint64_t table_id_; SampleMethod method_; SampleScope scope_; diff --git a/src/share/ob_index_builder_util.cpp b/src/share/ob_index_builder_util.cpp index eaed68369..9fc1d23f1 100644 --- a/src/share/ob_index_builder_util.cpp +++ b/src/share/ob_index_builder_util.cpp @@ -54,7 +54,7 @@ void ObIndexBuilderUtil::del_column_flags_and_default_value(ObColumnSchemaV2 &co } ObObj obj; obj.set_null(); - column.set_cur_default_value(obj); + column.set_cur_default_value(obj, column.is_default_expr_v2_column()); column.set_orig_default_value(obj); } return; @@ -148,7 +148,7 @@ int ObIndexBuilderUtil::add_column( column.set_is_hidden(false); if (FAILEDx(column.set_orig_default_value(default_value))) { LOG_WARN("set orig default value failed", K(ret)); - } else if (OB_FAIL(column.set_cur_default_value(default_value))) { + } else if (OB_FAIL(column.set_cur_default_value(default_value, column.is_default_expr_v2_column()))) { LOG_WARN("set current default value failed", K(ret)); } } @@ -171,7 +171,8 @@ int ObIndexBuilderUtil::set_shadow_column_info( shadow_column_schema.set_column_flags(0); ObObj default_obj; default_obj.set_null(); - shadow_column_schema.set_cur_default_value(default_obj); + shadow_column_schema.set_cur_default_value( + default_obj, shadow_column_schema.is_default_expr_v2_column()); shadow_column_schema.set_orig_default_value(default_obj); shadow_column_schema.set_tbl_part_key_pos(0); shadow_column_schema.set_column_id(src_column_id); @@ -992,7 +993,9 @@ int ObIndexBuilderUtil::generate_ordinary_generated_column( LOG_WARN("set column name failed", K(ret)); } else if (OB_FAIL(tmp_gen_col.set_orig_default_value(default_value))) { LOG_WARN("set orig default value failed", K(ret)); - } else if (OB_FAIL(tmp_gen_col.set_cur_default_value(default_value))) { + } else if (OB_FAIL(tmp_gen_col.set_cur_default_value( + default_value, + tmp_gen_col.is_default_expr_v2_column()))) { LOG_WARN("set current default value failed", K(ret)); } else if (OB_FAIL(data_schema.add_column(tmp_gen_col))) { LOG_WARN("add column schema to data table failed", K(ret)); @@ -1063,7 +1066,9 @@ int ObIndexBuilderUtil::generate_prefix_column( LOG_WARN("set column name failed", K(ret)); } else if (OB_FAIL(prefix_column.set_orig_default_value(default_value))) { LOG_WARN("set orig default value failed", K(ret)); - } else if (OB_FAIL(prefix_column.set_cur_default_value(default_value))) { + } else if (OB_FAIL(prefix_column.set_cur_default_value( + default_value, + prefix_column.is_default_expr_v2_column()))) { LOG_WARN("set current default value to prefix column failed", K(ret)); } else if (OB_FAIL(prefix_column.add_cascaded_column_id(old_column->get_column_id()))) { LOG_WARN("add cascaded column id failed", K(ret)); @@ -1238,7 +1243,9 @@ int ObIndexBuilderUtil::generate_spatial_cellid_column( LOG_WARN("set column name failed", K(ret)); } else if (OB_FAIL(column_schema.set_orig_default_value(default_value))) { LOG_WARN("set orig default value failed", K(ret)); - } else if (OB_FAIL(column_schema.set_cur_default_value(default_value))) { + } else if (OB_FAIL(column_schema.set_cur_default_value( + default_value, + column_schema.is_default_expr_v2_column()))) { LOG_WARN("set current default value failed", K(ret)); } else if (OB_FAIL(data_schema.add_column(column_schema))) { LOG_WARN("add cellid column schema to data table failed", K(ret)); @@ -1302,7 +1309,9 @@ int ObIndexBuilderUtil::generate_spatial_mbr_column( LOG_WARN("set column name failed", K(ret)); } else if (OB_FAIL(column_schema.set_orig_default_value(default_value))) { LOG_WARN("set orig default value failed", K(ret)); - } else if (OB_FAIL(column_schema.set_cur_default_value(default_value))) { + } else if (OB_FAIL(column_schema.set_cur_default_value( + default_value, + column_schema.is_default_expr_v2_column()))) { LOG_WARN("set current default value failed", K(ret)); } else if (OB_FAIL(data_schema.add_column(column_schema))) { LOG_WARN("add mbr column schema to data table failed", K(ret)); diff --git a/src/share/ob_rpc_struct.cpp b/src/share/ob_rpc_struct.cpp index 346451fdc..081ba3df2 100644 --- a/src/share/ob_rpc_struct.cpp +++ b/src/share/ob_rpc_struct.cpp @@ -5522,7 +5522,6 @@ int ObRevokeTableArg::assign(const ObRevokeTableArg &other) revoke_all_ora_ = other.revoke_all_ora_; grantor_ = other.grantor_; grantor_host_ = other.grantor_host_; - if (OB_FAIL(ObDDLArg::assign(other))) { LOG_WARN("fail to assign ddl arg", K(ret)); } else if (OB_FAIL(obj_priv_array_.assign(other.obj_priv_array_))) { diff --git a/src/share/object/ob_obj_cast.cpp b/src/share/object/ob_obj_cast.cpp index c5c53b898..9d9cc9be6 100644 --- a/src/share/object/ob_obj_cast.cpp +++ b/src/share/object/ob_obj_cast.cpp @@ -15654,12 +15654,15 @@ int ObObjCaster::is_order_consistent(const ObObjMeta &from, ObCollationLevel res_cs_level = CS_LEVEL_INVALID; ObCollationType from_cs_type = from.get_collation_type(); ObCollationType to_cs_type = to.get_collation_type(); - if (OB_FAIL(ObCharset::aggregate_collation(from.get_collation_level(), - from_cs_type, - to.get_collation_level(), - to_cs_type, - res_cs_level, - res_cs_type))) { + uint32_t flags = OB_COLL_ALLOW_SUPERSET_CONV | OB_COLL_ALLOW_COERCIBLE_CONV | + OB_COLL_ALLOW_NUMERIC_CONV | OB_COLL_ALLOW_NEW_CONV; + if (OB_FAIL(ObExprOperator::aggregate_two_collation(from.get_collation_level(), + from_cs_type, + to.get_collation_level(), + to_cs_type, + res_cs_level, + res_cs_type, + flags))) { LOG_WARN("fail to aggregate collation", K(ret), K(from), K(to)); } else { int64_t idx_from = get_idx_of_collate(from_cs_type); diff --git a/src/share/parameter/ob_parameter_seed.ipp b/src/share/parameter/ob_parameter_seed.ipp index 5b5cd45d2..9fa369222 100644 --- a/src/share/parameter/ob_parameter_seed.ipp +++ b/src/share/parameter/ob_parameter_seed.ipp @@ -1249,6 +1249,50 @@ DEF_STR_WITH_CHECKER(audit_trail, OB_TENANT_PARAMETER, "None", "enables or disables database auditing, support NONE;OS;DB;DB,EXTENDED;DB_EXTENDED", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +DEF_BOOL(audit_log_enable, OB_TENANT_PARAMETER, "False", + "whether enable audit log", + ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +DEF_CAP(audit_log_buffer_size, OB_TENANT_PARAMETER, "16M", "[16M,)" + "the buffer size of async audit log" + "Range: [16M, total size of memory]", + ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +DEF_STR_WITH_CHECKER(audit_log_compression, OB_TENANT_PARAMETER, "NONE", + common::ObConfigAuditLogCompressionChecker, + "the type of compression for the audit log file, values: NONE, ZSTD", + ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +DEF_STR_WITH_CHECKER(audit_log_path, OB_TENANT_PARAMETER, "", + common::ObConfigAuditLogPathChecker, + "the directory of the audit log", + ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +DEF_STR_WITH_CHECKER(audit_log_format, OB_TENANT_PARAMETER, "CSV", + common::ObConfigAuditLogFormatChecker, + "the audit log file format, values: CSV", + ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +DEF_CAP(audit_log_max_size, OB_TENANT_PARAMETER, "0M", "[0M,)", + "the maximum combined size of the audit log files", + ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +DEF_INT(audit_log_prune_seconds, OB_TENANT_PARAMETER, "0", "[0,)", + "the number of seconds after which audit log files become subject to pruning, range: [0,)", + ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +DEF_STR_WITH_CHECKER(audit_log_query_sql, OB_TENANT_PARAMETER, "ALL", + common::ObConfigAuditLogQuerySQLChecker, + "how to record the query sql. " + "ALL: record the original query sql. " + "DESENSITIVE: record the desensitive query sql. " + "NONE: not to record query sql.", + ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +DEF_CAP(audit_log_rotate_on_size, OB_TENANT_PARAMETER, "256M", "[0,)" + "whenever a write to the audit log file causes its size to exceed the config value, " + "it will be renamed and a new audit log file using is opened, range: [0,)", + ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +DEF_STR_WITH_CHECKER(audit_log_strategy, OB_TENANT_PARAMETER, "ASYNCHRONOUS", + common::ObConfigAuditLogStrategyChecker, + "the logging method used by the audit log plugin. " + "ASYNCHRONOUS: Log asynchronously. Wait for space in the output buffer. " + "PERFORMANCE: Log asynchronously. Drop requests when there is insufficient buffer. " + "SYNCHRONOUS: Log synchronously.", + ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); + //ddl 超时时间 DEF_TIME(_ob_ddl_timeout, OB_CLUSTER_PARAMETER, "1000s", "[1s,)", "the config parameter of ddl timeout" diff --git a/src/share/rc/ob_tenant_base.h b/src/share/rc/ob_tenant_base.h index 170e06c2d..40c4699aa 100755 --- a/src/share/rc/ob_tenant_base.h +++ b/src/share/rc/ob_tenant_base.h @@ -61,6 +61,8 @@ namespace sql { class ObUDRMgr; class ObPlanCache; class ObPsCache; + class ObAuditLogger; + class ObAuditLogUpdater; } namespace blocksstable { class ObSharedMacroBlockMgr; @@ -372,7 +374,9 @@ using ObTableScanIteratorObjPool = common::ObServerObjectPool 0 && OB_FAIL(dml.add_column("GRANTOR", grantor))) { + LOG_WARN("add column failed", K(ret)); + } else if (grantor_host.length() > 0 && OB_FAIL(dml.add_column("GRANTOR_HOST", grantor_host))) { + LOG_WARN("add column failed", K(ret)); } return ret; } @@ -953,7 +988,9 @@ int ObPrivSqlService::gen_routine_priv_dml( const uint64_t exec_tenant_id, const ObRoutinePrivSortKey &routine_priv_key, const ObPrivSet &priv_set, - ObDMLSqlSplicer &dml) + ObDMLSqlSplicer &dml, + const common::ObString &grantor, + const common::ObString &grantor_host) { int ret = OB_SUCCESS; int64_t all_priv = 0; @@ -976,6 +1013,14 @@ int ObPrivSqlService::gen_routine_priv_dml( LOG_WARN("add column failed", K(ret)); } } + if (OB_FAIL(ret)) { + } else if (ObSQLUtils::is_data_version_ge_424_or_433(compat_version)) { + if (grantor.length() > 0 && OB_FAIL(dml.add_column("GRANTOR", grantor))) { + LOG_WARN("add column failed", K(ret)); + } else if (grantor_host.length() > 0 && OB_FAIL(dml.add_column("GRANTOR_HOST", grantor_host))) { + LOG_WARN("add column failed", K(ret)); + } + } return ret; } @@ -987,9 +1032,11 @@ int ObPrivSqlService::gen_db_priv_dml( { int ret = OB_SUCCESS; ObPrivSet priv_others = 0; - priv_others |= (priv_set & OB_PRIV_EXECUTE) != 0 ? 1 : 0; - priv_others |= (priv_set & OB_PRIV_ALTER_ROUTINE) != 0 ? 2 : 0; - priv_others |= (priv_set & OB_PRIV_CREATE_ROUTINE) != 0 ? 4 : 0; + priv_others |= (priv_set & OB_PRIV_EXECUTE) != 0 ? OB_PRIV_OTHERS_EXECUTE : 0; + priv_others |= (priv_set & OB_PRIV_ALTER_ROUTINE) != 0 ? OB_PRIV_OTHERS_ALTER_ROUTINE : 0; + priv_others |= (priv_set & OB_PRIV_CREATE_ROUTINE) != 0 ? OB_PRIV_OTHERS_CREATE_ROUTINE : 0; + priv_others |= (priv_set & OB_PRIV_REFERENCES) != 0 ? OB_PRIV_OTHERS_REFERENCES : 0; + priv_others |= (priv_set & OB_PRIV_TRIGGER) != 0 ? OB_PRIV_OTHERS_TRIGGER : 0; uint64_t compat_version = 0; if (OB_FAIL(GET_MIN_DATA_VERSION(exec_tenant_id, compat_version))) { LOG_WARN("fail to get data version", KR(ret), K(exec_tenant_id)); @@ -1015,10 +1062,18 @@ int ObPrivSqlService::gen_db_priv_dml( if (priv_others != 0) { ret = OB_NOT_SUPPORTED; LOG_WARN("priv others is not suppported when tenant's data version is below 4.3.1.0 or 4.2.2.0", KR(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "grant or revoke database level priv other privilege"); } } else if (OB_FAIL(dml.add_column("PRIV_OTHERS", priv_others))) { LOG_WARN("add column failed", K(ret)); } + if (OB_FAIL(ret)) { + } else if (!ObSQLUtils::is_data_version_ge_424_or_433(compat_version) && + ((priv_set & OB_PRIV_REFERENCES) != 0 || (priv_set & OB_PRIV_TRIGGER) != 0)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("priv references/priv trigger is not suppported when tenant's data version is below 4.2.4.0 or 4.3.3.0", KR(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "grant or revoke database level trigger privilege/references privilege"); + } return ret; } diff --git a/src/share/schema/ob_priv_sql_service.h b/src/share/schema/ob_priv_sql_service.h index 9a1a5abe9..21862cbe7 100644 --- a/src/share/schema/ob_priv_sql_service.h +++ b/src/share/schema/ob_priv_sql_service.h @@ -76,7 +76,9 @@ public: const ObObjPrivSortKey &obj_priv_key, const int64_t new_schema_version_ora, const bool is_grant, - bool is_revoke_all_ora); + bool is_revoke_all_ora, + const common::ObString &grantor, + const common::ObString &grantor_host); virtual int revoke_table( const ObTablePrivSortKey &table_priv_key, @@ -87,7 +89,9 @@ public: const int64_t new_schema_version_ora, const ObObjPrivSortKey &obj_priv_key, const share::ObRawObjPrivArray &obj_priv_array, - bool is_revoke_all); + bool is_revoke_all, + const common::ObString &grantor, + const common::ObString &grantor_host); virtual int grant_column( const ObColumnPrivSortKey &column_priv_key, @@ -117,18 +121,24 @@ public: const ObString *ddl_stmt_str, ObISQLClient &sql_client, const uint64_t option, - const bool is_grant); + const bool is_grant, + const common::ObString &grantor, + const common::ObString &grantor_host); virtual int revoke_routine( const ObRoutinePrivSortKey &routine_priv_key, const ObPrivSet priv_set, const int64_t new_schema_version, const ObString *ddl_stmt_str, - ObISQLClient &sql_client); + ObISQLClient &sql_client, + const common::ObString &grantor, + const common::ObString &grantor_host); virtual int gen_routine_priv_dml( const uint64_t exec_tenant_id, const ObRoutinePrivSortKey &routine_priv_key, const ObPrivSet &priv_set, - ObDMLSqlSplicer &dml); + ObDMLSqlSplicer &dml, + const common::ObString &grantor, + const common::ObString &grantor_host); virtual int alter_user_default_role( const share::schema::ObUserInfo &user_info, const int64_t new_schema_version, @@ -199,7 +209,9 @@ private: const ObTablePrivSortKey &table_priv_key, const ObPrivSet &priv_set, const int64_t schema_version, - common::ObISQLClient &sql_client); + common::ObISQLClient &sql_client, + const common::ObString &grantor, + const common::ObString &grantor_host); int gen_db_priv_dml( const uint64_t exec_tenant_id, @@ -211,7 +223,9 @@ private: const uint64_t exec_tenant_id, const ObTablePrivSortKey &table_priv_key, const ObPrivSet &priv_set, - share::ObDMLSqlSplicer &dml); + share::ObDMLSqlSplicer &dml, + const common::ObString &grantor, + const common::ObString &grantor_host); int gen_column_priv_dml( const uint64_t exec_tenant_id, diff --git a/src/share/schema/ob_priv_type.h b/src/share/schema/ob_priv_type.h index 9bdebdd9d..0e03825e6 100644 --- a/src/share/schema/ob_priv_type.h +++ b/src/share/schema/ob_priv_type.h @@ -123,6 +123,9 @@ enum OB_PRIV_SHIFT #define OB_PRIV_CREATE_TABLESPACE OB_PRIV_GET_TYPE(OB_PRIV_CREATE_TABLESPACE_SHIFT) #define OB_PRIV_SHUTDOWN OB_PRIV_GET_TYPE(OB_PRIV_SHUTDOWN_SHIFT) #define OB_PRIV_RELOAD OB_PRIV_GET_TYPE(OB_PRIV_RELOAD_SHIFT) +#define OB_PRIV_CREATE_ROLE OB_PRIV_GET_TYPE(OB_PRIV_CREATE_ROLE_SHIFT) +#define OB_PRIV_DROP_ROLE OB_PRIV_GET_TYPE(OB_PRIV_DROP_ROLE_SHIFT) +#define OB_PRIV_TRIGGER OB_PRIV_GET_TYPE(OB_PRIV_TRIGGER_SHIFT) #define OB_PRIV_ALL \ (OB_PRIV_ALTER | OB_PRIV_CREATE | OB_PRIV_CREATE_USER | OB_PRIV_DELETE | \ @@ -134,24 +137,28 @@ enum OB_PRIV_SHIFT OB_PRIV_REPL_SLAVE | OB_PRIV_REPL_CLIENT | \ OB_PRIV_DROP_DATABASE_LINK | OB_PRIV_CREATE_DATABASE_LINK | \ OB_PRIV_EXECUTE | OB_PRIV_ALTER_ROUTINE | OB_PRIV_CREATE_ROUTINE | \ - OB_PRIV_CREATE_TABLESPACE | OB_PRIV_SHUTDOWN | OB_PRIV_RELOAD) + OB_PRIV_CREATE_TABLESPACE | OB_PRIV_SHUTDOWN | OB_PRIV_RELOAD | \ + OB_PRIV_REFERENCES | OB_PRIV_CREATE_ROLE | OB_PRIV_DROP_ROLE | OB_PRIV_TRIGGER) #define OB_PRIV_DB_ACC \ (OB_PRIV_ALTER | OB_PRIV_CREATE | OB_PRIV_DELETE | \ OB_PRIV_DROP | OB_PRIV_INSERT | OB_PRIV_UPDATE | OB_PRIV_SELECT | \ OB_PRIV_INDEX | OB_PRIV_CREATE_VIEW | OB_PRIV_SHOW_VIEW | \ - OB_PRIV_EXECUTE | OB_PRIV_ALTER_ROUTINE | OB_PRIV_CREATE_ROUTINE) + OB_PRIV_EXECUTE | OB_PRIV_ALTER_ROUTINE | OB_PRIV_CREATE_ROUTINE | \ + OB_PRIV_REFERENCES | OB_PRIV_TRIGGER) #define OB_PRIV_TABLE_ACC \ (OB_PRIV_ALTER | OB_PRIV_CREATE | OB_PRIV_DELETE | \ OB_PRIV_DROP | OB_PRIV_INSERT | OB_PRIV_UPDATE | OB_PRIV_SELECT | \ - OB_PRIV_INDEX | OB_PRIV_CREATE_VIEW | OB_PRIV_SHOW_VIEW) + OB_PRIV_INDEX | OB_PRIV_CREATE_VIEW | OB_PRIV_SHOW_VIEW | \ + OB_PRIV_REFERENCES | OB_PRIV_TRIGGER) #define OB_PRIV_ROUTINE_ACC \ (OB_PRIV_ALTER_ROUTINE | OB_PRIV_EXECUTE) #define OB_PRIV_COLUMN_ACC \ - (OB_PRIV_INSERT | OB_PRIV_UPDATE | OB_PRIV_SELECT) + (OB_PRIV_INSERT | OB_PRIV_UPDATE | OB_PRIV_SELECT | OB_PRIV_REFERENCES) + enum OB_PRIV_OTHERS_TYPE { diff --git a/src/share/schema/ob_schema_getter_guard.cpp b/src/share/schema/ob_schema_getter_guard.cpp index fa08fc026..6d6ffe6e4 100644 --- a/src/share/schema/ob_schema_getter_guard.cpp +++ b/src/share/schema/ob_schema_getter_guard.cpp @@ -3634,10 +3634,9 @@ int ObSchemaGetterGuard::check_table_show(const ObSessionPrivInfo &session_priv, return ret; } - - int ObSchemaGetterGuard::check_user_priv(const ObSessionPrivInfo &session_priv, - const ObPrivSet priv_set) + const ObPrivSet priv_set, + bool check_all) { int ret = OB_SUCCESS; uint64_t tenant_id = session_priv.tenant_id_; @@ -3649,7 +3648,8 @@ int ObSchemaGetterGuard::check_user_priv(const ObSessionPrivInfo &session_priv, LOG_WARN("fail to check tenant schema guard", KR(ret), K(tenant_id), K_(tenant_id)); } else if (OB_FAIL(check_lazy_guard(tenant_id, mgr))) { LOG_WARN("fail to check lazy guard", KR(ret), K(tenant_id)); - } else if (!OB_TEST_PRIVS(user_priv_set, priv_set)) { + } else if ((!OB_TEST_PRIVS(user_priv_set, priv_set) && check_all) + || (!OB_PRIV_HAS_ANY(user_priv_set, priv_set) && !check_all)) { if ((priv_set == OB_PRIV_ALTER_TENANT || priv_set == OB_PRIV_ALTER_SYSTEM || priv_set == OB_PRIV_CREATE_RESOURCE_POOL @@ -3666,7 +3666,7 @@ int ObSchemaGetterGuard::check_user_priv(const ObSessionPrivInfo &session_priv, LOG_WARN("fail to collect privs in roles", K(ret)); } else { user_priv_set |= collected_privs.priv_set_; - if (!check_succ) { + if ((!check_succ && check_all) || (!OB_PRIV_HAS_ANY(user_priv_set, priv_set) && !check_all)) { ret = OB_ERR_NO_PRIVILEGE; } } @@ -3692,6 +3692,10 @@ int ObSchemaGetterGuard::check_user_priv(const ObSessionPrivInfo &session_priv, } else { LOG_USER_ERROR(OB_ERR_NO_PRIVILEGE, priv_name_with_prefix.ptr()); } + } else if (priv_set == (OB_PRIV_CREATE_ROLE | OB_PRIV_CREATE_USER) && !check_all) { + LOG_USER_ERROR(OB_ERR_NO_PRIVILEGE, "CREATE USER or CREATE ROLE"); + } else if (priv_set == (OB_PRIV_DROP_ROLE | OB_PRIV_CREATE_USER) && !check_all) { + LOG_USER_ERROR(OB_ERR_NO_PRIVILEGE, "CREATE USER or DROP ROLE"); } else { LOG_USER_ERROR(OB_ERR_NO_PRIVILEGE, priv_name); } @@ -4384,12 +4388,14 @@ int ObSchemaGetterGuard::check_priv(const ObSessionPrivInfo &session_priv, const ObNeedPriv &need_priv = need_privs.at(i); switch (need_priv.priv_level_) { case OB_PRIV_USER_LEVEL: { - if (OB_FAIL(check_user_priv(session_priv, need_priv.priv_set_))) { + if (OB_FAIL(check_user_priv(session_priv, + need_priv.priv_set_, + OB_PRIV_CHECK_ALL == need_priv.priv_check_type_))) { LOG_WARN("No privilege", "tenant_id", session_priv.tenant_id_, - "user_id", session_priv.user_id_, - "need_priv", need_priv.priv_set_, - "user_priv", session_priv.user_priv_set_, - KR(ret));//need print priv + "user_id", session_priv.user_id_, + "need_priv", need_priv.priv_set_, + "user_priv", session_priv.user_priv_set_, + KR(ret));//need print priv } break; } diff --git a/src/share/schema/ob_schema_getter_guard.h b/src/share/schema/ob_schema_getter_guard.h index 220bbda01..c3d301b14 100644 --- a/src/share/schema/ob_schema_getter_guard.h +++ b/src/share/schema/ob_schema_getter_guard.h @@ -1105,7 +1105,8 @@ private: const common::ObString &db, const ObPrivSet need_priv_set); int check_user_priv(const ObSessionPrivInfo &session_priv, - const ObPrivSet priv_set); + const ObPrivSet priv_set, + bool check_all = true); int verify_db_read_only(const uint64_t tenant_id, const ObNeedPriv &need_priv); int verify_table_read_only(const uint64_t tenant_id, diff --git a/src/share/schema/ob_schema_macro_define.cpp b/src/share/schema/ob_schema_macro_define.cpp index aa50c7c38..2890a626c 100644 --- a/src/share/schema/ob_schema_macro_define.cpp +++ b/src/share/schema/ob_schema_macro_define.cpp @@ -228,7 +228,8 @@ int ADD_COLUMN_SCHEMA_WITH_DEFAULT_VALUE(share::schema::ObTableSchema &table_sch cur_default_value.set_collation_type(column.get_collation_type()); if (OB_FAIL(column.set_orig_default_value(orig_default_value))) { SHARE_SCHEMA_LOG(WARN, "Fail to set original default value, ", K(ret)); - } else if (OB_FAIL(column.set_cur_default_value(cur_default_value))) { + // Internal tables only support constant default values. + } else if (OB_FAIL(column.set_cur_default_value(cur_default_value, false))) { SHARE_SCHEMA_LOG(WARN, "Fail to set cur default value, ", K(ret)); } } @@ -338,7 +339,8 @@ int ADD_COLUMN_SCHEMA_TS_WITH_DEFAULT_VALUE(share::schema::ObTableSchema &table_ cur_default_value.set_collation_type(column.get_collation_type()); if (OB_FAIL(column.set_orig_default_value(orig_default_value))) { SHARE_SCHEMA_LOG(WARN, "Fail to set original default value, ", K(ret)); - } else if (OB_FAIL(column.set_cur_default_value(cur_default_value))) { + // Internal tables only support constant default values. + } else if (OB_FAIL(column.set_cur_default_value(cur_default_value, false))) { SHARE_SCHEMA_LOG(WARN, "Fail to set cur default value, ", K(ret)); } } diff --git a/src/share/schema/ob_schema_printer.cpp b/src/share/schema/ob_schema_printer.cpp index e9c338ecd..8f2fddebd 100644 --- a/src/share/schema/ob_schema_printer.cpp +++ b/src/share/schema/ob_schema_printer.cpp @@ -338,6 +338,11 @@ int ObSchemaPrinter::print_table_definition_columns(const ObTableSchema &table_s SHARE_SCHEMA_LOG(WARN, "fail to print DEFAULT now()", K(ret)); } } + } else if (col->is_default_expr_v2_column()) { + ObString default_value = col->get_cur_default_value().get_string(); + if (OB_FAIL(databuff_printf(buf, buf_len, pos, " DEFAULT (%.*s)", default_value.length(), default_value.ptr()))) { + SHARE_SCHEMA_LOG(WARN, "fail to print sql literal", K(default_value), K(ret)); + } } else { if (OB_FAIL(databuff_printf(buf, buf_len, pos, " DEFAULT "))) { SHARE_SCHEMA_LOG(WARN, "fail to print DEFAULT", K(ret)); @@ -5284,7 +5289,8 @@ int ObSchemaPrinter::print_user_definition(uint64_t tenant_id, char *buf, const int64_t &buf_len, int64_t &pos, - bool is_role) + bool is_role, + bool print_password_secret /* default = false */) { int ret = OB_SUCCESS; common::ObArray user_schemas; @@ -5300,23 +5306,29 @@ int ObSchemaPrinter::print_user_definition(uint64_t tenant_id, SHARE_SCHEMA_LOG(WARN, "fail to print user name", K(user_name), K(ret)); } else if (OB_FAIL(print_identifier(buf, buf_len, pos, user_name, is_oracle_mode))) { SHARE_SCHEMA_LOG(WARN, "fail to print user name", K(user_name), K(ret)); - } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, " "))) { - SHARE_SCHEMA_LOG(WARN, "fail to print user name", K(user_name), K(ret)); - } else if (host_name.compare(OB_DEFAULT_HOST_NAME) != 0) { + } else if (!is_oracle_mode) { if (OB_FAIL(databuff_printf(buf, buf_len, pos, "@"))) { SHARE_SCHEMA_LOG(WARN, "fail to print host name", K(host_name), K(ret)); } else if (OB_FAIL(print_identifier(buf, buf_len, pos, host_name, is_oracle_mode))) { SHARE_SCHEMA_LOG(WARN, "fail to print host name", K(host_name), K(ret)); - } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, " "))) { - SHARE_SCHEMA_LOG(WARN, "fail to print host name", K(host_name), K(ret)); } } if (OB_SUCC(ret) && user_passwd.length() > 0) { - if (OB_FAIL(databuff_printf(buf, buf_len, pos, - is_oracle_mode ? "IDENTIFIED BY VALUES \"%.*s\" " : "IDENTIFIED BY PASSWORD '%.*s' ", - user_passwd.length(), user_passwd.ptr()))) { - SHARE_SCHEMA_LOG(WARN, "fail to print user passwd", K(user_name), K(user_passwd), K(ret)); + if (is_oracle_mode) { + if (OB_FAIL(databuff_printf(buf, buf_len, pos, " IDENTIFIED BY VALUES \"%.*s\"", + user_passwd.length(), user_passwd.ptr()))) { + SHARE_SCHEMA_LOG(WARN, "fail to print user passwd", K(user_name), K(user_passwd), K(ret)); + } + } else { + if (print_password_secret && + OB_FAIL(databuff_printf(buf, buf_len, pos, " IDENTIFIED BY PASSWORD ''"))) { + SHARE_SCHEMA_LOG(WARN, "fail to print user passwd", K(user_name), K(user_passwd), K(ret)); + } else if (!print_password_secret && + OB_FAIL(databuff_printf(buf, buf_len, pos, " IDENTIFIED BY PASSWORD '%.*s'", + user_passwd.length(), user_passwd.ptr()))) { + SHARE_SCHEMA_LOG(WARN, "fail to print user passwd", K(user_name), K(user_passwd), K(ret)); + } } } @@ -5331,14 +5343,12 @@ int ObSchemaPrinter::print_user_definition(uint64_t tenant_id, SHARE_SCHEMA_LOG(WARN, "get profile schena failed", K(ret)); } } else { - if (OB_FAIL(databuff_printf(buf, buf_len, pos, "PROFILE "))) { + if (OB_FAIL(databuff_printf(buf, buf_len, pos, " PROFILE "))) { SHARE_SCHEMA_LOG(WARN, "fail to print profile", K(ret)); } else if (OB_FAIL(print_identifier(buf, buf_len, pos, profile_schema->get_profile_name_str(), is_oracle_mode))) { SHARE_SCHEMA_LOG(WARN, "fail to print profile", K(ret)); - } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, " "))) { - SHARE_SCHEMA_LOG(WARN, "fail to print profile", K(ret)); } } } @@ -5349,36 +5359,36 @@ int ObSchemaPrinter::print_user_definition(uint64_t tenant_id, break; } case ObSSLType::SSL_TYPE_NONE: { - if (OB_FAIL(databuff_printf(buf, buf_len, pos, " REQUIRE NONE "))) { + if (OB_FAIL(databuff_printf(buf, buf_len, pos, " REQUIRE NONE"))) { SHARE_SCHEMA_LOG(WARN, "fail to print ssl info", K(user_name), K(ret)); } break; } case ObSSLType::SSL_TYPE_ANY: { - if (OB_FAIL(databuff_printf(buf, buf_len, pos, " REQUIRE SSL "))) { + if (OB_FAIL(databuff_printf(buf, buf_len, pos, " REQUIRE SSL"))) { SHARE_SCHEMA_LOG(WARN, "fail to print ssl info", K(user_name), K(ret)); } break; } case ObSSLType::SSL_TYPE_X509: { - if (OB_FAIL(databuff_printf(buf, buf_len, pos, " REQUIRE X509 "))) { + if (OB_FAIL(databuff_printf(buf, buf_len, pos, " REQUIRE X509"))) { SHARE_SCHEMA_LOG(WARN, "fail to print ssl info", K(user_name), K(ret)); } break; } case ObSSLType::SSL_TYPE_SPECIFIED: { - if (OB_FAIL(databuff_printf(buf, buf_len, pos, " REQUIRE "))) { + if (OB_FAIL(databuff_printf(buf, buf_len, pos, " REQUIRE"))) { SHARE_SCHEMA_LOG(WARN, "fail to print ssl info", K(user_name), K(ret)); } else if (!user_info.get_ssl_cipher_str().empty() - && OB_FAIL(databuff_printf(buf, buf_len, pos, "CIPHER '%s' ", + && OB_FAIL(databuff_printf(buf, buf_len, pos, " CIPHER '%s'", user_info.get_ssl_cipher()))) { SHARE_SCHEMA_LOG(WARN, "fail to print ssl info", K(user_name), K(ret)); } else if (!user_info.get_x509_issuer_str().empty() - && OB_FAIL(databuff_printf(buf, buf_len, pos, "ISSUER '%s' ", + && OB_FAIL(databuff_printf(buf, buf_len, pos, " ISSUER '%s'", user_info.get_x509_issuer()))) { SHARE_SCHEMA_LOG(WARN, "fail to print ssl info", K(user_name), K(ret)); } else if (!user_info.get_x509_subject_str().empty() - && OB_FAIL(databuff_printf(buf, buf_len, pos, "SUBJECT '%s' ", + && OB_FAIL(databuff_printf(buf, buf_len, pos, " SUBJECT '%s'", user_info.get_x509_subject()))) { SHARE_SCHEMA_LOG(WARN, "fail to print ssl info", K(user_name), K(ret)); } @@ -5390,6 +5400,24 @@ int ObSchemaPrinter::print_user_definition(uint64_t tenant_id, } } } + + if (OB_SUCC(ret)) { + if (0 != user_info.get_max_connections() || 0 != user_info.get_max_user_connections()) { + if (OB_FAIL(databuff_printf(buf, buf_len, pos, " WITH"))) { + SHARE_SCHEMA_LOG(WARN, "fail to print with info", K(ret)); + } else if (0 != user_info.get_max_connections() && + OB_FAIL(databuff_printf(buf, buf_len, pos, " MAX_CONNECTIONS_PER_HOUR %lu", + user_info.get_max_connections()))) { + SHARE_SCHEMA_LOG(WARN, "fail to print get max_connections", + K(user_info.get_max_connections()), K(ret)); + } else if (0 != user_info.get_max_user_connections() && + OB_FAIL(databuff_printf(buf, buf_len, pos, " MAX_USER_CONNECTIONS %lu", + user_info.get_max_user_connections()))) { + SHARE_SCHEMA_LOG(WARN, "fail to print get_max_user_connections", + K(user_info.get_max_user_connections()), K(ret)); + } + } + } return ret; } diff --git a/src/share/schema/ob_schema_printer.h b/src/share/schema/ob_schema_printer.h index e5d7df744..a717030a7 100644 --- a/src/share/schema/ob_schema_printer.h +++ b/src/share/schema/ob_schema_printer.h @@ -464,7 +464,8 @@ public: char *buf, const int64_t &buf_len, int64_t &pos, - bool is_role); + bool is_role, + bool print_password_secret = false); int print_synonym_definition(const ObSynonymInfo &synonym_info, char *buf, const int64_t &buf_len, diff --git a/src/share/schema/ob_schema_retrieve_utils.ipp b/src/share/schema/ob_schema_retrieve_utils.ipp index 91c88dffe..915b2d169 100644 --- a/src/share/schema/ob_schema_retrieve_utils.ipp +++ b/src/share/schema/ob_schema_retrieve_utils.ipp @@ -1528,7 +1528,7 @@ int fill_column_schema_default_value(T &result, lib::CompatModeGuard guard(compat_mode); EXTRACT_DEFAULT_VALUE_FIELD_MYSQL(result, orig_default_value, default_type, column,false, false, tenant_id); - EXTRACT_DEFAULT_VALUE_FIELD_MYSQL(result, cur_default_value, default_type, + EXTRACT_DEFAULT_VALUE_FIELD_MYSQL_V2(result, default_type, column, true, false, tenant_id); EXTRACT_DEFAULT_VALUE_FIELD_MYSQL(result, orig_default_value_v2, default_type, column, false, true, tenant_id); @@ -1911,12 +1911,16 @@ int ObSchemaRetrieveUtils::fill_user_schema( ObPrivSet priv_others = 0; EXTRACT_INT_FIELD_MYSQL_WITH_DEFAULT_VALUE(result, "priv_others", priv_others, uint64_t, true /* skip null error*/, ignore_column_error, 0); - user_info.set_priv((priv_others & 1) != 0 ? OB_PRIV_EXECUTE : 0); - user_info.set_priv((priv_others & 2) != 0 ? OB_PRIV_ALTER_ROUTINE : 0); - user_info.set_priv((priv_others & 4) != 0 ? OB_PRIV_CREATE_ROUTINE : 0); - user_info.set_priv((priv_others & 8) != 0 ? OB_PRIV_CREATE_TABLESPACE : 0); - user_info.set_priv((priv_others & 16) != 0 ? OB_PRIV_SHUTDOWN : 0); - user_info.set_priv((priv_others & 32) != 0 ? OB_PRIV_RELOAD : 0); + user_info.set_priv((priv_others & OB_PRIV_OTHERS_EXECUTE) != 0 ? OB_PRIV_EXECUTE : 0); + user_info.set_priv((priv_others & OB_PRIV_OTHERS_ALTER_ROUTINE) != 0 ? OB_PRIV_ALTER_ROUTINE : 0); + user_info.set_priv((priv_others & OB_PRIV_OTHERS_CREATE_ROUTINE) != 0 ? OB_PRIV_CREATE_ROUTINE : 0); + user_info.set_priv((priv_others & OB_PRIV_OTHERS_CREATE_TABLESPACE) != 0 ? OB_PRIV_CREATE_TABLESPACE : 0); + user_info.set_priv((priv_others & OB_PRIV_OTHERS_SHUTDOWN) != 0 ? OB_PRIV_SHUTDOWN : 0); + user_info.set_priv((priv_others & OB_PRIV_OTHERS_RELOAD) != 0 ? OB_PRIV_RELOAD : 0); + user_info.set_priv((priv_others & OB_PRIV_OTHERS_REFERENCES) != 0 ? OB_PRIV_REFERENCES : 0); + user_info.set_priv((priv_others & OB_PRIV_OTHERS_CREATE_ROLE) != 0 ? OB_PRIV_CREATE_ROLE : 0); + user_info.set_priv((priv_others & OB_PRIV_OTHERS_DROP_ROLE) != 0 ? OB_PRIV_DROP_ROLE : 0); + user_info.set_priv((priv_others & OB_PRIV_OTHERS_TRIGGER) != 0 ? OB_PRIV_TRIGGER : 0); if (OB_SUCC(ret)) { int64_t default_flags = 0; @@ -2215,9 +2219,11 @@ int ObSchemaRetrieveUtils::fill_db_priv_schema( ignore_column_error, 0); if (OB_FAIL(ret)) { } else { - db_priv.set_priv((priv_others & 1) != 0 ? OB_PRIV_EXECUTE : 0); - db_priv.set_priv((priv_others & 2) != 0 ? OB_PRIV_ALTER_ROUTINE : 0); - db_priv.set_priv((priv_others & 4) != 0 ? OB_PRIV_CREATE_ROUTINE : 0); + db_priv.set_priv((priv_others & OB_PRIV_OTHERS_EXECUTE) != 0 ? OB_PRIV_EXECUTE : 0); + db_priv.set_priv((priv_others & OB_PRIV_OTHERS_ALTER_ROUTINE) != 0 ? OB_PRIV_ALTER_ROUTINE : 0); + db_priv.set_priv((priv_others & OB_PRIV_OTHERS_CREATE_ROUTINE) != 0 ? OB_PRIV_CREATE_ROUTINE : 0); + db_priv.set_priv((priv_others & OB_PRIV_OTHERS_REFERENCES) != 0 ? OB_PRIV_REFERENCES : 0); + db_priv.set_priv((priv_others & OB_PRIV_OTHERS_TRIGGER) != 0 ? OB_PRIV_TRIGGER : 0); } } @@ -2272,6 +2278,15 @@ int ObSchemaRetrieveUtils::fill_table_priv_schema( EXTRACT_PRIV_FROM_MYSQL_RESULT(result, priv_create_view, table_priv, PRIV_CREATE_VIEW); EXTRACT_PRIV_FROM_MYSQL_RESULT(result, priv_show_view, table_priv, PRIV_SHOW_VIEW); EXTRACT_INT_FIELD_TO_CLASS_MYSQL(result, schema_version, table_priv, int64_t); + bool ignore_column_error = true; + ObPrivSet priv_others = 0; + EXTRACT_INT_FIELD_MYSQL_WITH_DEFAULT_VALUE(result, "priv_others", priv_others, uint64_t, true /* skip null error*/, + ignore_column_error, 0); + if (OB_FAIL(ret)) { + } else { + table_priv.set_priv((priv_others & OB_PRIV_OTHERS_REFERENCES) != 0 ? OB_PRIV_REFERENCES : 0); + table_priv.set_priv((priv_others & OB_PRIV_OTHERS_TRIGGER) != 0 ? OB_PRIV_TRIGGER : 0); + } } return ret; diff --git a/src/share/schema/ob_schema_struct.cpp b/src/share/schema/ob_schema_struct.cpp index 32a10fe68..d8207c274 100644 --- a/src/share/schema/ob_schema_struct.cpp +++ b/src/share/schema/ob_schema_struct.cpp @@ -8072,6 +8072,15 @@ DEF_TO_STRING(ObPrintPrivSet) if ((priv_set_ & OB_PRIV_RELOAD) && OB_SUCCESS == ret) { ret = BUF_PRINTF(" RELOAD,"); } + if ((priv_set_ & OB_PRIV_CREATE_ROLE) && OB_SUCCESS == ret) { + ret = BUF_PRINTF(" CREATE ROLE,"); + } + if ((priv_set_ & OB_PRIV_DROP_ROLE) && OB_SUCCESS == ret) { + ret = BUF_PRINTF(" DROP ROLE,"); + } + if ((priv_set_ & OB_PRIV_TRIGGER) && OB_SUCCESS == ret) { + ret = BUF_PRINTF(" TRIGGER,"); + } if (OB_SUCCESS == ret && pos > 1) { pos--; //Delete last ',' } diff --git a/src/share/schema/ob_user_sql_service.cpp b/src/share/schema/ob_user_sql_service.cpp index a86697fb2..3c044536c 100644 --- a/src/share/schema/ob_user_sql_service.cpp +++ b/src/share/schema/ob_user_sql_service.cpp @@ -1070,12 +1070,16 @@ int ObUserSqlService::gen_user_dml( } int64_t priv_others = 0; if (OB_SUCC(ret)) { - if ((user.get_priv_set() & OB_PRIV_EXECUTE) != 0) { priv_others |= 1; } - if ((user.get_priv_set() & OB_PRIV_ALTER_ROUTINE) != 0) { priv_others |= 2; } - if ((user.get_priv_set() & OB_PRIV_CREATE_ROUTINE) != 0) { priv_others |= 4; } - if ((user.get_priv_set() & OB_PRIV_CREATE_TABLESPACE) != 0) { priv_others |= 8; } - if ((user.get_priv_set() & OB_PRIV_SHUTDOWN) != 0) { priv_others |= 16; } - if ((user.get_priv_set() & OB_PRIV_RELOAD) != 0) { priv_others |= 32; } + if ((user.get_priv_set() & OB_PRIV_EXECUTE) != 0) { priv_others |= OB_PRIV_OTHERS_EXECUTE; } + if ((user.get_priv_set() & OB_PRIV_ALTER_ROUTINE) != 0) { priv_others |= OB_PRIV_OTHERS_ALTER_ROUTINE; } + if ((user.get_priv_set() & OB_PRIV_CREATE_ROUTINE) != 0) { priv_others |= OB_PRIV_OTHERS_CREATE_ROUTINE; } + if ((user.get_priv_set() & OB_PRIV_CREATE_TABLESPACE) != 0) { priv_others |= OB_PRIV_OTHERS_CREATE_TABLESPACE; } + if ((user.get_priv_set() & OB_PRIV_SHUTDOWN) != 0) { priv_others |= OB_PRIV_OTHERS_SHUTDOWN; } + if ((user.get_priv_set() & OB_PRIV_RELOAD) != 0) { priv_others |= OB_PRIV_OTHERS_RELOAD; } + if ((user.get_priv_set() & OB_PRIV_REFERENCES) != 0) { priv_others |= OB_PRIV_OTHERS_REFERENCES; } + if ((user.get_priv_set() & OB_PRIV_CREATE_ROLE) != 0) { priv_others |= OB_PRIV_OTHERS_CREATE_ROLE; } + if ((user.get_priv_set() & OB_PRIV_DROP_ROLE) != 0) { priv_others |= OB_PRIV_OTHERS_DROP_ROLE; } + if ((user.get_priv_set() & OB_PRIV_TRIGGER) != 0) { priv_others |= OB_PRIV_OTHERS_TRIGGER; } } if (OB_FAIL(ret)) { } else if (!sql::ObSQLUtils::is_data_version_ge_422_or_431(compat_version)) { diff --git a/src/share/system_variable/gen_ob_sys_variables.py b/src/share/system_variable/gen_ob_sys_variables.py index 831dd76d3..948076a92 100755 --- a/src/share/system_variable/gen_ob_sys_variables.py +++ b/src/share/system_variable/gen_ob_sys_variables.py @@ -97,7 +97,7 @@ def parse_json(json_file_name): # If it is true, it means it is a placeholder variable. filtered_json = filter(lambda d: 'placeholder' not in d[1] or d[1]['placeholder'] is False, json_Dict.iteritems()) filtered_dict = dict(filtered_json) - list_sorted_by_name= sorted(filtered_dict.iteritems(), key=lambda d:d[0]) + list_sorted_by_name= sorted(filtered_dict.iteritems(), key=lambda d:d[0].lower()) list_sorted_by_id= sorted(filtered_dict.iteritems(), key=lambda d:d[1]['id']) json_file.close() return (json_Dict, list_sorted_by_name, list_sorted_by_id) @@ -502,7 +502,10 @@ public: static ObSysVarClassType find_sys_var_id_by_name(const common::ObString &sys_var_name, bool is_from_sys_table = false); //二分查找 static int get_sys_var_name_by_id(ObSysVarClassType sys_var_id, common::ObString &sys_var_name); static const common::ObString get_sys_var_name_by_id(ObSysVarClassType sys_var_id); +private: + int try_init_store_mem(); +public: const static int64_t MYSQL_SYS_VARS_COUNT = """) wfile.write(str(mysql_sys_var_names_count) + ";") wfile.write(""" @@ -525,8 +528,8 @@ private: const static ObSysVarClassType SYS_VAR_IDS_SORTED_BY_NAME[ALL_SYS_VARS_COUNT]; const static char *SYS_VAR_NAMES_SORTED_BY_ID[ALL_SYS_VARS_COUNT]; common::ObArenaAllocator allocator_; - ObBasicSysVar *store_[ALL_SYS_VARS_COUNT]; - ObBasicSysVar *store_buf_[ALL_SYS_VARS_COUNT]; + ObBasicSysVar **store_; + ObBasicSysVar **store_buf_; bool all_sys_vars_created_; }; """) @@ -842,10 +845,34 @@ const ObString ObSysVarFactory::get_sys_var_name_by_id(ObSysVarClassType sys_var ObSysVarFactory::ObSysVarFactory(const int64_t tenant_id) : allocator_(ObMemAttr(tenant_id, ObModIds::OB_COMMON_SYS_VAR_FAC)), - all_sys_vars_created_(false) + store_(nullptr), store_buf_(nullptr), all_sys_vars_created_(false) { - MEMSET(store_, 0, sizeof(store_)); - MEMSET(store_buf_, 0, sizeof(store_buf_)); +} + +int ObSysVarFactory::try_init_store_mem() +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(store_)) { + void *store_ptr = NULL; + if (OB_ISNULL(store_ptr = allocator_.alloc(sizeof(ObBasicSysVar *) * ALL_SYS_VARS_COUNT))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc store_.", K(ret)); + } else { + store_ = static_cast(store_ptr); + MEMSET(store_, 0, sizeof(ObBasicSysVar *) * ALL_SYS_VARS_COUNT); + } + } + if (OB_ISNULL(store_buf_)) { + void *store_buf_ptr = NULL; + if (OB_ISNULL(store_buf_ptr = allocator_.alloc(sizeof(ObBasicSysVar *) * ALL_SYS_VARS_COUNT))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc store_buf_.", K(ret)); + } else { + store_buf_ = static_cast(store_buf_ptr); + MEMSET(store_buf_, 0, sizeof(ObBasicSysVar *) * ALL_SYS_VARS_COUNT); + } + } + return ret; } ObSysVarFactory::~ObSysVarFactory() @@ -856,15 +883,23 @@ ObSysVarFactory::~ObSysVarFactory() void ObSysVarFactory::destroy() { int ret = OB_SUCCESS; - for (int64_t i = 0; i < ALL_SYS_VARS_COUNT; ++i) { - if (OB_NOT_NULL(store_[i])) { - store_[i]->~ObBasicSysVar(); - store_[i] = nullptr; + if (OB_NOT_NULL(store_)) { + for (int64_t i = 0; i < ALL_SYS_VARS_COUNT; ++i) { + if (OB_NOT_NULL(store_[i])) { + store_[i]->~ObBasicSysVar(); + store_[i] = nullptr; + } } - if (OB_NOT_NULL(store_buf_[i])) { - store_buf_[i]->~ObBasicSysVar(); - store_buf_[i] = nullptr; + store_ = nullptr; + } + if (OB_NOT_NULL(store_buf_)) { + for (int64_t i = 0; i < ALL_SYS_VARS_COUNT; ++i) { + if (OB_NOT_NULL(store_buf_[i])) { + store_buf_[i]->~ObBasicSysVar(); + store_buf_[i] = nullptr; + } } + store_buf_ = nullptr; } allocator_.reset(); all_sys_vars_created_ = false; @@ -873,24 +908,28 @@ void ObSysVarFactory::destroy() int ObSysVarFactory::free_sys_var(ObBasicSysVar *sys_var, int64_t sys_var_idx) { int ret = OB_SUCCESS; - OV (OB_NOT_NULL(sys_var)); - OV (is_valid_sys_var_store_idx(sys_var_idx)); - OV (sys_var == store_[sys_var_idx], OB_ERR_UNEXPECTED, sys_var, sys_var_idx); - if (OB_NOT_NULL(store_buf_[sys_var_idx])) { - OX (store_buf_[sys_var_idx]->~ObBasicSysVar()); - OX (allocator_.free(store_buf_[sys_var_idx])); - OX (store_buf_[sys_var_idx] = nullptr); + if (OB_NOT_NULL(store_) && OB_NOT_NULL(store_buf_)) { + OV (OB_NOT_NULL(sys_var)); + OV (is_valid_sys_var_store_idx(sys_var_idx)); + OV (sys_var == store_[sys_var_idx], OB_ERR_UNEXPECTED, sys_var, sys_var_idx); + if (OB_NOT_NULL(store_buf_[sys_var_idx])) { + OX (store_buf_[sys_var_idx]->~ObBasicSysVar()); + OX (allocator_.free(store_buf_[sys_var_idx])); + OX (store_buf_[sys_var_idx] = nullptr); + } + OX (store_buf_[sys_var_idx] = store_[sys_var_idx]); + OX (store_buf_[sys_var_idx]->clean_value()); + OX (store_[sys_var_idx] = nullptr); } - OX (store_buf_[sys_var_idx] = store_[sys_var_idx]); - OX (store_buf_[sys_var_idx]->clean_value()); - OX (store_[sys_var_idx] = nullptr); return ret; } int ObSysVarFactory::create_all_sys_vars() { int ret = OB_SUCCESS; - if (!all_sys_vars_created_) { + if (OB_FAIL(try_init_store_mem())) { + LOG_WARN("Fail to init", K(ret)); + } else if (!all_sys_vars_created_) { int64_t store_idx = -1; ObBasicSysVar *sys_var_ptr = NULL; int64_t total_mem_size = 0 @@ -967,7 +1006,9 @@ int ObSysVarFactory::create_sys_var(ObSysVarClassType sys_var_id, ObBasicSysVar int ret = OB_SUCCESS; int64_t store_idx = -1; ObBasicSysVar *sys_var_ptr = NULL; - if (OB_FAIL(calc_sys_var_store_idx(sys_var_id, store_idx))) { + if (OB_FAIL(try_init_store_mem())) { + LOG_WARN("fail to init", K(ret)); + } else if (OB_FAIL(calc_sys_var_store_idx(sys_var_id, store_idx))) { LOG_WARN("fail to calc sys var store idx", K(ret), K(sys_var_id)); } else if (store_idx < 0 || store_idx >= ALL_SYS_VARS_COUNT) { ret = OB_ERR_UNEXPECTED; diff --git a/src/share/system_variable/ob_sys_var_class_type.h b/src/share/system_variable/ob_sys_var_class_type.h index a399b6ea1..45489c69d 100644 --- a/src/share/system_variable/ob_sys_var_class_type.h +++ b/src/share/system_variable/ob_sys_var_class_type.h @@ -432,6 +432,64 @@ enum ObSysVarClassType SYS_VAR_INNODB_SYNC_DEBUG = 10324, SYS_VAR_DEFAULT_COLLATION_FOR_UTF8MB4 = 10325, SYS_VAR__ENABLE_OLD_CHARSET_AGGREGATION = 10326, + SYS_VAR_INSERT_ID = 10328, + SYS_VAR_JOIN_BUFFER_SIZE = 10329, + SYS_VAR_MAX_JOIN_SIZE = 10330, + SYS_VAR_MAX_LENGTH_FOR_SORT_DATA = 10331, + SYS_VAR_MAX_PREPARED_STMT_COUNT = 10332, + SYS_VAR_MAX_SORT_LENGTH = 10333, + SYS_VAR_MIN_EXAMINED_ROW_LIMIT = 10334, + SYS_VAR_MULTI_RANGE_COUNT = 10335, + SYS_VAR_MYSQLX_CONNECT_TIMEOUT = 10336, + SYS_VAR_MYSQLX_IDLE_WORKER_THREAD_TIMEOUT = 10337, + SYS_VAR_MYSQLX_MAX_ALLOWED_PACKET = 10338, + SYS_VAR_MYSQLX_MAX_CONNECTIONS = 10339, + SYS_VAR_MYSQLX_MIN_WORKER_THREADS = 10340, + SYS_VAR_PERFORMANCE_SCHEMA_SHOW_PROCESSLIST = 10341, + SYS_VAR_QUERY_ALLOC_BLOCK_SIZE = 10342, + SYS_VAR_QUERY_PREALLOC_SIZE = 10343, + SYS_VAR_SLOW_QUERY_LOG = 10344, + SYS_VAR_SLOW_QUERY_LOG_FILE = 10345, + SYS_VAR_SORT_BUFFER_SIZE = 10346, + SYS_VAR_SQL_BUFFER_RESULT = 10347, + SYS_VAR_BINLOG_CACHE_SIZE = 10348, + SYS_VAR_BINLOG_DIRECT_NON_TRANSACTIONAL_UPDATES = 10349, + SYS_VAR_BINLOG_ERROR_ACTION = 10350, + SYS_VAR_BINLOG_GROUP_COMMIT_SYNC_DELAY = 10351, + SYS_VAR_BINLOG_GROUP_COMMIT_SYNC_NO_DELAY_COUNT = 10352, + SYS_VAR_BINLOG_MAX_FLUSH_QUEUE_TIME = 10353, + SYS_VAR_BINLOG_ORDER_COMMITS = 10354, + SYS_VAR_BINLOG_STMT_CACHE_SIZE = 10355, + SYS_VAR_BINLOG_TRANSACTION_DEPENDENCY_HISTORY_SIZE = 10356, + SYS_VAR_BINLOG_TRANSACTION_DEPENDENCY_TRACKING = 10357, + SYS_VAR_EXPIRE_LOGS_DAYS = 10358, + SYS_VAR_INNODB_FLUSH_LOG_AT_TIMEOUT = 10359, + SYS_VAR_INNODB_FLUSH_LOG_AT_TRX_COMMIT = 10360, + SYS_VAR_INNODB_LOG_CHECKPOINT_NOW = 10361, + SYS_VAR_INNODB_LOG_CHECKSUMS = 10362, + SYS_VAR_INNODB_LOG_COMPRESSED_PAGES = 10363, + SYS_VAR_INNODB_LOG_WRITE_AHEAD_SIZE = 10364, + SYS_VAR_INNODB_MAX_UNDO_LOG_SIZE = 10365, + SYS_VAR_INNODB_ONLINE_ALTER_LOG_MAX_SIZE = 10366, + SYS_VAR_INNODB_UNDO_LOG_TRUNCATE = 10367, + SYS_VAR_INNODB_UNDO_LOGS = 10368, + SYS_VAR_LOG_BIN_TRUST_FUNCTION_CREATORS = 10369, + SYS_VAR_LOG_BIN_USE_V1_ROW_EVENTS = 10370, + SYS_VAR_LOG_BUILTIN_AS_IDENTIFIED_BY_PASSWORD = 10371, + SYS_VAR_MAX_BINLOG_CACHE_SIZE = 10372, + SYS_VAR_MAX_BINLOG_SIZE = 10373, + SYS_VAR_MAX_BINLOG_STMT_CACHE_SIZE = 10374, + SYS_VAR_MAX_RELAY_LOG_SIZE = 10375, + SYS_VAR_RELAY_LOG_INFO_REPOSITORY = 10376, + SYS_VAR_RELAY_LOG_PURGE = 10377, + SYS_VAR_SYNC_BINLOG = 10378, + SYS_VAR_SYNC_RELAY_LOG = 10379, + SYS_VAR_SYNC_RELAY_LOG_INFO = 10380, + SYS_VAR_INNODB_DEADLOCK_DETECT = 10381, + SYS_VAR_INNODB_LOCK_WAIT_TIMEOUT = 10382, + SYS_VAR_INNODB_PRINT_ALL_DEADLOCKS = 10383, + SYS_VAR_INNODB_TABLE_LOCKS = 10384, + SYS_VAR_MAX_WRITE_LOCK_COUNT = 10385, SYS_VAR__OB_ENABLE_ROLE_IDS = 10386, SYS_VAR_INNODB_READ_ONLY = 10387, SYS_VAR_INNODB_API_DISABLE_ROWLOCK = 10388, @@ -473,6 +531,100 @@ enum ObSysVarClassType SYS_VAR_KEY_CACHE_BLOCK_SIZE = 10641, SYS_VAR_OB_KV_MODE = 10642, SYS_VAR_OB_ENABLE_PARAMETER_ANONYMOUS_BLOCK = 10644, + SYS_VAR_CHARACTER_SETS_DIR = 10645, + SYS_VAR_DATE_FORMAT = 10646, + SYS_VAR_DATETIME_FORMAT = 10647, + SYS_VAR_DISCONNECT_ON_EXPIRED_PASSWORD = 10648, + SYS_VAR_EXTERNAL_USER = 10649, + SYS_VAR_HAVE_CRYPT = 10650, + SYS_VAR_HAVE_DYNAMIC_LOADING = 10651, + SYS_VAR_KEYRING_AWS_CONF_FILE = 10652, + SYS_VAR_KEYRING_AWS_DATA_FILE = 10653, + SYS_VAR_LANGUAGE = 10654, + SYS_VAR_LC_MESSAGES_DIR = 10655, + SYS_VAR_LOWER_CASE_FILE_SYSTEM = 10656, + SYS_VAR_MAX_DIGEST_LENGTH = 10657, + SYS_VAR_NDBINFO_DATABASE = 10658, + SYS_VAR_NDBINFO_TABLE_PREFIX = 10659, + SYS_VAR_NDBINFO_VERSION = 10660, + SYS_VAR_NDB_BATCH_SIZE = 10661, + SYS_VAR_NDB_CLUSTER_CONNECTION_POOL = 10662, + SYS_VAR_NDB_CLUSTER_CONNECTION_POOL_NODEIDS = 10663, + SYS_VAR_NDB_LOG_APPLY_STATUS = 10664, + SYS_VAR_NDB_LOG_BIN = 10665, + SYS_VAR_NDB_LOG_FAIL_TERMINATE = 10666, + SYS_VAR_NDB_LOG_ORIG = 10667, + SYS_VAR_NDB_LOG_TRANSACTION_ID = 10668, + SYS_VAR_NDB_OPTIMIZED_NODE_SELECTION = 10669, + SYS_VAR_NDB_SYSTEM_NAME = 10670, + SYS_VAR_NDB_USE_COPYING_ALTER_TABLE = 10671, + SYS_VAR_NDB_VERSION_STRING = 10672, + SYS_VAR_NDB_WAIT_CONNECTED = 10673, + SYS_VAR_NDB_WAIT_SETUP = 10674, + SYS_VAR_PROXY_USER = 10675, + SYS_VAR_SHA256_PASSWORD_AUTO_GENERATE_RSA_KEYS = 10676, + SYS_VAR_SHA256_PASSWORD_PRIVATE_KEY_PATH = 10677, + SYS_VAR_SHA256_PASSWORD_PUBLIC_KEY_PATH = 10678, + SYS_VAR_SKIP_SHOW_DATABASE = 10679, + SYS_VAR_PLUGIN_LOAD = 10680, + SYS_VAR_PLUGIN_LOAD_ADD = 10681, + SYS_VAR_BIG_TABLES = 10682, + SYS_VAR_CHECK_PROXY_USERS = 10683, + SYS_VAR_CONNECTION_CONTROL_FAILED_CONNECTIONS_THRESHOLD = 10684, + SYS_VAR_CONNECTION_CONTROL_MAX_CONNECTION_DELAY = 10685, + SYS_VAR_CONNECTION_CONTROL_MIN_CONNECTION_DELAY = 10686, + SYS_VAR_DEFAULT_WEEK_FORMAT = 10687, + SYS_VAR_DELAYED_INSERT_TIMEOUT = 10688, + SYS_VAR_DELAYED_QUEUE_SIZE = 10689, + SYS_VAR_EQ_RANGE_INDEX_DIVE_LIMIT = 10690, + SYS_VAR_INNODB_STATS_AUTO_RECALC = 10691, + SYS_VAR_INNODB_STATS_INCLUDE_DELETE_MARKED = 10692, + SYS_VAR_INNODB_STATS_METHOD = 10693, + SYS_VAR_INNODB_STATS_ON_METADATA = 10694, + SYS_VAR_VERSION_TOKENS_SESSION = 10695, + SYS_VAR_INNODB_STATS_PERSISTENT_SAMPLE_PAGES = 10696, + SYS_VAR_INNODB_STATS_SAMPLE_PAGES = 10697, + SYS_VAR_INNODB_STATS_TRANSIENT_SAMPLE_PAGES = 10698, + SYS_VAR_KEYRING_AWS_CMK_ID = 10699, + SYS_VAR_KEYRING_AWS_REGION = 10700, + SYS_VAR_KEYRING_ENCRYPTED_FILE_DATA = 10701, + SYS_VAR_KEYRING_ENCRYPTED_FILE_PASSWORD = 10702, + SYS_VAR_KEYRING_FILE_DATA = 10703, + SYS_VAR_KEYRING_OKV_CONF_DIR = 10704, + SYS_VAR_KEYRING_OPERATIONS = 10705, + SYS_VAR_OPTIMIZER_SWITCH = 10706, + SYS_VAR_MAX_CONNECT_ERRORS = 10707, + SYS_VAR_MYSQL_FIREWALL_MODE = 10708, + SYS_VAR_MYSQL_FIREWALL_TRACE = 10709, + SYS_VAR_MYSQL_NATIVE_PASSWORD_PROXY_USERS = 10710, + SYS_VAR_NET_RETRY_COUNT = 10711, + SYS_VAR_NEW = 10712, + SYS_VAR_OLD_PASSWORDS = 10713, + SYS_VAR_OPTIMIZER_PRUNE_LEVEL = 10714, + SYS_VAR_OPTIMIZER_SEARCH_DEPTH = 10715, + SYS_VAR_OPTIMIZER_TRACE = 10716, + SYS_VAR_OPTIMIZER_TRACE_FEATURES = 10717, + SYS_VAR_OPTIMIZER_TRACE_LIMIT = 10718, + SYS_VAR_OPTIMIZER_TRACE_MAX_MEM_SIZE = 10719, + SYS_VAR_OPTIMIZER_TRACE_OFFSET = 10720, + SYS_VAR_PARSER_MAX_MEM_SIZE = 10721, + SYS_VAR_RAND_SEED1 = 10722, + SYS_VAR_RAND_SEED2 = 10723, + SYS_VAR_RANGE_ALLOC_BLOCK_SIZE = 10724, + SYS_VAR_RANGE_OPTIMIZER_MAX_MEM_SIZE = 10725, + SYS_VAR_REWRITER_ENABLED = 10726, + SYS_VAR_REWRITER_VERBOSE = 10727, + SYS_VAR_SECURE_AUTH = 10728, + SYS_VAR_SHA256_PASSWORD_PROXY_USERS = 10729, + SYS_VAR_SHOW_COMPATIBILITY_56 = 10730, + SYS_VAR_SHOW_CREATE_TABLE_VERBOSITY = 10731, + SYS_VAR_SHOW_OLD_TEMPORALS = 10732, + SYS_VAR_SQL_BIG_SELECTS = 10733, + SYS_VAR_UPDATABLE_VIEWS_WITH_LIMIT = 10734, + SYS_VAR_VALIDATE_PASSWORD_DICTIONARY_FILE = 10735, + SYS_VAR_DELAYED_INSERT_LIMIT = 10736, + SYS_VAR_NDB_VERSION = 10737, + SYS_VAR_AUTO_GENERATE_CERTS = 10738, }; } diff --git a/src/share/system_variable/ob_system_variable.cpp b/src/share/system_variable/ob_system_variable.cpp index e562fb35a..4413d4152 100644 --- a/src/share/system_variable/ob_system_variable.cpp +++ b/src/share/system_variable/ob_system_variable.cpp @@ -2001,6 +2001,64 @@ int ObSysVarOnCheckFuncs::check_and_convert_collation_not_null(ObExecContext &ct return ret; } +int ObSysVarOnCheckFuncs::check_default_value_for_utf8mb4(ObExecContext &ctx, + const ObSetVar &set_var, + const ObBasicSysVar &sys_var, + const ObObj &in_val, + ObObj &out_val) +{ + UNUSED(sys_var); + UNUSED(ctx); + int ret = OB_SUCCESS; + if (true == set_var.is_set_default_) { + // do nothing + } else if (true == in_val.is_null()) { + ret = OB_ERR_WRONG_VALUE_FOR_VAR; + LOG_USER_ERROR(OB_ERR_WRONG_VALUE_FOR_VAR, sys_var.get_name().length(), sys_var.get_name().ptr(), + (int)strlen("NULL"), "NULL"); + } else { + ObString coll_name; + if (ObVarcharType == in_val.get_type()) { + coll_name = in_val.get_varchar(); + ObCollationType coll_type = CS_TYPE_INVALID; + if (CS_TYPE_INVALID == (coll_type = ObCharset::collation_type(coll_name))) { + ret = OB_ERR_UNKNOWN_COLLATION; + LOG_USER_ERROR(OB_ERR_UNKNOWN_COLLATION, coll_name.length(), coll_name.ptr()); + } else if (coll_type != CS_TYPE_UTF8MB4_GENERAL_CI) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED,"only utf8mb4_general_ci is supported, other collation"); + } else { + out_val.set_int(static_cast(coll_type)); + } + } else if (ObIntType == in_val.get_type()) { + int64_t int64_val = in_val.get_int(); + if (false == ObCharset::is_valid_collation(int64_val)) { + ret = OB_ERR_UNKNOWN_COLLATION; + int p_ret = OB_SUCCESS; + const static int64_t val_buf_len = 1024; + char val_buf[val_buf_len]; + int64_t pos = 0; + if (OB_SUCCESS != (p_ret = databuff_printf(val_buf, val_buf_len, pos, "%ld", int64_val))) { + // p_ret不覆盖ret + LOG_WARN("fail to databuff_printf", K(ret), K(p_ret), K(int64_val)); + } else { + ObString coll_name_str(val_buf); + LOG_USER_ERROR(OB_ERR_UNKNOWN_COLLATION, coll_name_str.length(), coll_name_str.ptr()); + } + } else if (int64_val != 45) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED,"only utf8mb4_general_ci is supported, other collation"); + } else { + out_val = in_val; + } + } else { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("invalid type", K(ret), K(in_val)); + } + } + return ret; +} + int ObSysVarOnCheckFuncs::check_and_convert_timeout_too_large(ObExecContext &ctx, const ObSetVar &set_var, const ObBasicSysVar &sys_var, @@ -2092,6 +2150,7 @@ int ObSysVarOnCheckFuncs::check_and_convert_tx_read_only(ObExecContext &ctx, return ret; } + int ObSysVarOnCheckFuncs::check_update_resource_manager_plan(ObExecContext &ctx, const ObSetVar &set_var, const ObBasicSysVar &sys_var, @@ -2693,6 +2752,41 @@ int ObSysVarOnCheckFuncs::check_and_convert_version(sql::ObExecContext &ctx, return ret; } +int ObSysVarOnCheckFuncs::check_and_convert_block_encryption_mode(sql::ObExecContext &ctx, + const ObSetVar &set_var, + const ObBasicSysVar &sys_var, + const common::ObObj &in_val, + common::ObObj &out_val) +{ + int ret = OB_SUCCESS; + if (set_var.is_set_default_ || in_val.is_null()) { + /* do nothing */ + } else if (ObIntType == in_val.get_type()) { +#ifndef OB_USE_BABASSL + int64_t op_mode = in_val.get_int(); + if (op_mode >= 18) { + SMART_VAR(char[OB_MAX_SQL_LENGTH], val_str_buf) { + int64_t pos = 0; + int log_ret = in_val.print_plain_str_literal(val_str_buf, OB_MAX_SQL_LENGTH, pos); + if (OB_SUCCESS != log_ret) { + LOG_WARN("fail to print_plain_str_literal", K(log_ret), K(OB_MAX_SQL_LENGTH), K(pos), K(lbt())); + } else { + LOG_USER_ERROR(OB_ERR_WRONG_VALUE_FOR_VAR, + sys_var.get_name().length(), sys_var.get_name().ptr(), + static_cast(pos), val_str_buf); + } + } + ret = OB_ERR_WRONG_VALUE_FOR_VAR; + LOG_WARN("in opensource mode, we can use aes-128-ecb ~ aes-256-ofb only"); + } +#endif + } else { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid type", K(ret), K(in_val)); + } + return ret; +} + int ObSysVarOnUpdateFuncs::update_tx_isolation(ObExecContext &ctx, const ObSetVar &set_var, const ObBasicSysVar &sys_var, diff --git a/src/share/system_variable/ob_system_variable.h b/src/share/system_variable/ob_system_variable.h index 9af52da55..c78abdd18 100644 --- a/src/share/system_variable/ob_system_variable.h +++ b/src/share/system_variable/ob_system_variable.h @@ -838,6 +838,11 @@ public: const ObBasicSysVar &sys_var, const common::ObObj &in_val, common::ObObj &out_val); + static int check_default_value_for_utf8mb4(sql::ObExecContext &ctx, + const ObSetVar &set_var, + const ObBasicSysVar &sys_var, + const ObObj &in_val, + ObObj &out_val); static int check_and_convert_tx_isolation(sql::ObExecContext &ctx, const ObSetVar &set_var, const ObBasicSysVar &sys_var, @@ -950,6 +955,11 @@ public: const common::ObObj &in_val, const uint64_t tenant_id, uint64_t &version); + static int check_and_convert_block_encryption_mode(sql::ObExecContext &ctx, + const ObSetVar &set_var, + const ObBasicSysVar &sys_var, + const common::ObObj &in_val, + common::ObObj &out_val); private: static int check_session_readonly(sql::ObExecContext &ctx, const ObSetVar &set_var, diff --git a/src/share/system_variable/ob_system_variable_alias.h b/src/share/system_variable/ob_system_variable_alias.h index 0beb7f563..1524bf679 100644 --- a/src/share/system_variable/ob_system_variable_alias.h +++ b/src/share/system_variable/ob_system_variable_alias.h @@ -427,6 +427,64 @@ namespace share static const char* const OB_SV_INNODB_SYNC_DEBUG = "innodb_sync_debug"; static const char* const OB_SV_DEFAULT_COLLATION_FOR_UTF8MB4 = "default_collation_for_utf8mb4"; static const char* const OB_SV__ENABLE_OLD_CHARSET_AGGREGATION = "_enable_old_charset_aggregation"; + static const char* const OB_SV_INSERT_ID = "insert_id"; + static const char* const OB_SV_JOIN_BUFFER_SIZE = "join_buffer_size"; + static const char* const OB_SV_MAX_JOIN_SIZE = "max_join_size"; + static const char* const OB_SV_MAX_LENGTH_FOR_SORT_DATA = "max_length_for_sort_data"; + static const char* const OB_SV_MAX_PREPARED_STMT_COUNT = "max_prepared_stmt_count"; + static const char* const OB_SV_MAX_SORT_LENGTH = "max_sort_length"; + static const char* const OB_SV_MIN_EXAMINED_ROW_LIMIT = "min_examined_row_limit"; + static const char* const OB_SV_MULTI_RANGE_COUNT = "multi_range_count"; + static const char* const OB_SV_MYSQLX_CONNECT_TIMEOUT = "mysqlx_connect_timeout"; + static const char* const OB_SV_MYSQLX_IDLE_WORKER_THREAD_TIMEOUT = "mysqlx_idle_worker_thread_timeout"; + static const char* const OB_SV_MYSQLX_MAX_ALLOWED_PACKET = "mysqlx_max_allowed_packet"; + static const char* const OB_SV_MYSQLX_MAX_CONNECTIONS = "mysqlx_max_connections"; + static const char* const OB_SV_MYSQLX_MIN_WORKER_THREADS = "mysqlx_min_worker_threads"; + static const char* const OB_SV_PERFORMANCE_SCHEMA_SHOW_PROCESSLIST = "performance_schema_show_processlist"; + static const char* const OB_SV_QUERY_ALLOC_BLOCK_SIZE = "query_alloc_block_size"; + static const char* const OB_SV_QUERY_PREALLOC_SIZE = "query_prealloc_size"; + static const char* const OB_SV_SLOW_QUERY_LOG = "slow_query_log"; + static const char* const OB_SV_SLOW_QUERY_LOG_FILE = "slow_query_log_file"; + static const char* const OB_SV_SORT_BUFFER_SIZE = "sort_buffer_size"; + static const char* const OB_SV_SQL_BUFFER_RESULT = "sql_buffer_result"; + static const char* const OB_SV_BINLOG_CACHE_SIZE = "binlog_cache_size"; + static const char* const OB_SV_BINLOG_DIRECT_NON_TRANSACTIONAL_UPDATES = "binlog_direct_non_transactional_updates"; + static const char* const OB_SV_BINLOG_ERROR_ACTION = "binlog_error_action"; + static const char* const OB_SV_BINLOG_GROUP_COMMIT_SYNC_DELAY = "binlog_group_commit_sync_delay"; + static const char* const OB_SV_BINLOG_GROUP_COMMIT_SYNC_NO_DELAY_COUNT = "binlog_group_commit_sync_no_delay_count"; + static const char* const OB_SV_BINLOG_MAX_FLUSH_QUEUE_TIME = "binlog_max_flush_queue_time"; + static const char* const OB_SV_BINLOG_ORDER_COMMITS = "binlog_order_commits"; + static const char* const OB_SV_BINLOG_STMT_CACHE_SIZE = "binlog_stmt_cache_size"; + static const char* const OB_SV_BINLOG_TRANSACTION_DEPENDENCY_HISTORY_SIZE = "binlog_transaction_dependency_history_size"; + static const char* const OB_SV_BINLOG_TRANSACTION_DEPENDENCY_TRACKING = "binlog_transaction_dependency_tracking"; + static const char* const OB_SV_EXPIRE_LOGS_DAYS = "expire_logs_days"; + static const char* const OB_SV_INNODB_FLUSH_LOG_AT_TIMEOUT = "innodb_flush_log_at_timeout"; + static const char* const OB_SV_INNODB_FLUSH_LOG_AT_TRX_COMMIT = "innodb_flush_log_at_trx_commit"; + static const char* const OB_SV_INNODB_LOG_CHECKPOINT_NOW = "innodb_log_checkpoint_now"; + static const char* const OB_SV_INNODB_LOG_CHECKSUMS = "innodb_log_checksums"; + static const char* const OB_SV_INNODB_LOG_COMPRESSED_PAGES = "innodb_log_compressed_pages"; + static const char* const OB_SV_INNODB_LOG_WRITE_AHEAD_SIZE = "innodb_log_write_ahead_size"; + static const char* const OB_SV_INNODB_MAX_UNDO_LOG_SIZE = "innodb_max_undo_log_size"; + static const char* const OB_SV_INNODB_ONLINE_ALTER_LOG_MAX_SIZE = "innodb_online_alter_log_max_size"; + static const char* const OB_SV_INNODB_UNDO_LOG_TRUNCATE = "innodb_undo_log_truncate"; + static const char* const OB_SV_INNODB_UNDO_LOGS = "innodb_undo_logs"; + static const char* const OB_SV_LOG_BIN_TRUST_FUNCTION_CREATORS = "log_bin_trust_function_creators"; + static const char* const OB_SV_LOG_BIN_USE_V1_ROW_EVENTS = "log_bin_use_v1_row_events"; + static const char* const OB_SV_LOG_BUILTIN_AS_IDENTIFIED_BY_PASSWORD = "log_builtin_as_identified_by_password"; + static const char* const OB_SV_MAX_BINLOG_CACHE_SIZE = "max_binlog_cache_size"; + static const char* const OB_SV_MAX_BINLOG_SIZE = "max_binlog_size"; + static const char* const OB_SV_MAX_BINLOG_STMT_CACHE_SIZE = "max_binlog_stmt_cache_size"; + static const char* const OB_SV_MAX_RELAY_LOG_SIZE = "max_relay_log_size"; + static const char* const OB_SV_RELAY_LOG_INFO_REPOSITORY = "relay_log_info_repository"; + static const char* const OB_SV_RELAY_LOG_PURGE = "relay_log_purge"; + static const char* const OB_SV_SYNC_BINLOG = "sync_binlog"; + static const char* const OB_SV_SYNC_RELAY_LOG = "sync_relay_log"; + static const char* const OB_SV_SYNC_RELAY_LOG_INFO = "sync_relay_log_info"; + static const char* const OB_SV_INNODB_DEADLOCK_DETECT = "innodb_deadlock_detect"; + static const char* const OB_SV_INNODB_LOCK_WAIT_TIMEOUT = "innodb_lock_wait_timeout"; + static const char* const OB_SV_INNODB_PRINT_ALL_DEADLOCKS = "innodb_print_all_deadlocks"; + static const char* const OB_SV_INNODB_TABLE_LOCKS = "innodb_table_locks"; + static const char* const OB_SV_MAX_WRITE_LOCK_COUNT = "max_write_lock_count"; static const char* const OB_SV__OB_ENABLE_ROLE_IDS = "_ob_enable_role_ids"; static const char* const OB_SV_INNODB_READ_ONLY = "innodb_read_only"; static const char* const OB_SV_INNODB_API_DISABLE_ROWLOCK = "innodb_api_disable_rowlock"; @@ -468,6 +526,100 @@ namespace share static const char* const OB_SV_KEY_CACHE_BLOCK_SIZE = "key_cache_block_size"; static const char* const OB_SV_KV_MODE = "ob_kv_mode"; static const char* const OB_SV_ENABLE_PARAMETER_ANONYMOUS_BLOCK = "ob_enable_parameter_anonymous_block"; + static const char* const OB_SV_CHARACTER_SETS_DIR = "character_sets_dir"; + static const char* const OB_SV_DATE_FORMAT = "date_format"; + static const char* const OB_SV_DATETIME_FORMAT = "datetime_format"; + static const char* const OB_SV_DISCONNECT_ON_EXPIRED_PASSWORD = "disconnect_on_expired_password"; + static const char* const OB_SV_EXTERNAL_USER = "external_user"; + static const char* const OB_SV_HAVE_CRYPT = "have_crypt"; + static const char* const OB_SV_HAVE_DYNAMIC_LOADING = "have_dynamic_loading"; + static const char* const OB_SV_KEYRING_AWS_CONF_FILE = "keyring_aws_conf_file"; + static const char* const OB_SV_KEYRING_AWS_DATA_FILE = "keyring_aws_data_file"; + static const char* const OB_SV_LANGUAGE = "language"; + static const char* const OB_SV_LC_MESSAGES_DIR = "lc_messages_dir"; + static const char* const OB_SV_LOWER_CASE_FILE_SYSTEM = "lower_case_file_system"; + static const char* const OB_SV_MAX_DIGEST_LENGTH = "max_digest_length"; + static const char* const OB_SV_NDBINFO_DATABASE = "ndbinfo_database"; + static const char* const OB_SV_NDBINFO_TABLE_PREFIX = "ndbinfo_table_prefix"; + static const char* const OB_SV_NDBINFO_VERSION = "ndbinfo_version"; + static const char* const OB_SV_NDB_BATCH_SIZE = "ndb_batch_size"; + static const char* const OB_SV_NDB_CLUSTER_CONNECTION_POOL = "ndb_cluster_connection_pool"; + static const char* const OB_SV_NDB_CLUSTER_CONNECTION_POOL_NODEIDS = "ndb_cluster_connection_pool_nodeids"; + static const char* const OB_SV_NDB_LOG_APPLY_STATUS = "ndb_log_apply_status"; + static const char* const OB_SV_NDB_LOG_BIN = "ndb_log_bin"; + static const char* const OB_SV_NDB_LOG_FAIL_TERMINATE = "ndb_log_fail_terminate"; + static const char* const OB_SV_NDB_LOG_ORIG = "ndb_log_orig"; + static const char* const OB_SV_NDB_LOG_TRANSACTION_ID = "ndb_log_transaction_id"; + static const char* const OB_SV_NDB_OPTIMIZED_NODE_SELECTION = "ndb_optimized_node_selection"; + static const char* const OB_SV_NDB_SYSTEM_NAME = "Ndb_system_name"; + static const char* const OB_SV_NDB_USE_COPYING_ALTER_TABLE = "ndb_use_copying_alter_table"; + static const char* const OB_SV_NDB_VERSION_STRING = "ndb_version_string"; + static const char* const OB_SV_NDB_WAIT_CONNECTED = "ndb_wait_connected"; + static const char* const OB_SV_NDB_WAIT_SETUP = "ndb_wait_setup"; + static const char* const OB_SV_PROXY_USER = "proxy_user"; + static const char* const OB_SV_SHA256_PASSWORD_AUTO_GENERATE_RSA_KEYS = "sha256_password_auto_generate_rsa_keys"; + static const char* const OB_SV_SHA256_PASSWORD_PRIVATE_KEY_PATH = "sha256_password_private_key_path"; + static const char* const OB_SV_SHA256_PASSWORD_PUBLIC_KEY_PATH = "sha256_password_public_key_path"; + static const char* const OB_SV_SKIP_SHOW_DATABASE = "skip_show_database"; + static const char* const OB_SV_PLUGIN_LOAD = "plugin_load"; + static const char* const OB_SV_PLUGIN_LOAD_ADD = "plugin_load_add"; + static const char* const OB_SV_BIG_TABLES = "big_tables"; + static const char* const OB_SV_CHECK_PROXY_USERS = "check_proxy_users"; + static const char* const OB_SV_CONNECTION_CONTROL_FAILED_CONNECTIONS_THRESHOLD = "connection_control_failed_connections_threshold"; + static const char* const OB_SV_CONNECTION_CONTROL_MAX_CONNECTION_DELAY = "connection_control_max_connection_delay"; + static const char* const OB_SV_CONNECTION_CONTROL_MIN_CONNECTION_DELAY = "connection_control_min_connection_delay"; + static const char* const OB_SV_DEFAULT_WEEK_FORMAT = "default_week_format"; + static const char* const OB_SV_DELAYED_INSERT_TIMEOUT = "delayed_insert_timeout"; + static const char* const OB_SV_DELAYED_QUEUE_SIZE = "delayed_queue_size"; + static const char* const OB_SV_EQ_RANGE_INDEX_DIVE_LIMIT = "eq_range_index_dive_limit"; + static const char* const OB_SV_INNODB_STATS_AUTO_RECALC = "innodb_stats_auto_recalc"; + static const char* const OB_SV_INNODB_STATS_INCLUDE_DELETE_MARKED = "innodb_stats_include_delete_marked"; + static const char* const OB_SV_INNODB_STATS_METHOD = "innodb_stats_method"; + static const char* const OB_SV_INNODB_STATS_ON_METADATA = "innodb_stats_on_metadata"; + static const char* const OB_SV_VERSION_TOKENS_SESSION = "version_tokens_session"; + static const char* const OB_SV_INNODB_STATS_PERSISTENT_SAMPLE_PAGES = "innodb_stats_persistent_sample_pages"; + static const char* const OB_SV_INNODB_STATS_SAMPLE_PAGES = "innodb_stats_sample_pages"; + static const char* const OB_SV_INNODB_STATS_TRANSIENT_SAMPLE_PAGES = "innodb_stats_transient_sample_pages"; + static const char* const OB_SV_KEYRING_AWS_CMK_ID = "keyring_aws_cmk_id"; + static const char* const OB_SV_KEYRING_AWS_REGION = "keyring_aws_region"; + static const char* const OB_SV_KEYRING_ENCRYPTED_FILE_DATA = "keyring_encrypted_file_data"; + static const char* const OB_SV_KEYRING_ENCRYPTED_FILE_PASSWORD = "keyring_encrypted_file_password"; + static const char* const OB_SV_KEYRING_FILE_DATA = "keyring_file_data"; + static const char* const OB_SV_KEYRING_OKV_CONF_DIR = "keyring_okv_conf_dir"; + static const char* const OB_SV_KEYRING_OPERATIONS = "keyring_operations"; + static const char* const OB_SV_OPTIMIZER_SWITCH = "optimizer_switch"; + static const char* const OB_SV_MAX_CONNECT_ERRORS = "max_connect_errors"; + static const char* const OB_SV_MYSQL_FIREWALL_MODE = "mysql_firewall_mode"; + static const char* const OB_SV_MYSQL_FIREWALL_TRACE = "mysql_firewall_trace"; + static const char* const OB_SV_MYSQL_NATIVE_PASSWORD_PROXY_USERS = "mysql_native_password_proxy_users"; + static const char* const OB_SV_NET_RETRY_COUNT = "net_retry_count"; + static const char* const OB_SV_NEW = "new"; + static const char* const OB_SV_OLD_PASSWORDS = "old_passwords"; + static const char* const OB_SV_OPTIMIZER_PRUNE_LEVEL = "optimizer_prune_level"; + static const char* const OB_SV_OPTIMIZER_SEARCH_DEPTH = "optimizer_search_depth"; + static const char* const OB_SV_OPTIMIZER_TRACE = "optimizer_trace"; + static const char* const OB_SV_OPTIMIZER_TRACE_FEATURES = "optimizer_trace_features"; + static const char* const OB_SV_OPTIMIZER_TRACE_LIMIT = "optimizer_trace_limit"; + static const char* const OB_SV_OPTIMIZER_TRACE_MAX_MEM_SIZE = "optimizer_trace_max_mem_size"; + static const char* const OB_SV_OPTIMIZER_TRACE_OFFSET = "optimizer_trace_offset"; + static const char* const OB_SV_PARSER_MAX_MEM_SIZE = "parser_max_mem_size"; + static const char* const OB_SV_RAND_SEED1 = "rand_seed1"; + static const char* const OB_SV_RAND_SEED2 = "rand_seed2"; + static const char* const OB_SV_RANGE_ALLOC_BLOCK_SIZE = "range_alloc_block_size"; + static const char* const OB_SV_RANGE_OPTIMIZER_MAX_MEM_SIZE = "range_optimizer_max_mem_size"; + static const char* const OB_SV_REWRITER_ENABLED = "rewriter_enabled"; + static const char* const OB_SV_REWRITER_VERBOSE = "rewriter_verbose"; + static const char* const OB_SV_SECURE_AUTH = "secure_auth"; + static const char* const OB_SV_SHA256_PASSWORD_PROXY_USERS = "sha256_password_proxy_users"; + static const char* const OB_SV_SHOW_COMPATIBILITY_56 = "show_compatibility_56"; + static const char* const OB_SV_SHOW_CREATE_TABLE_VERBOSITY = "show_create_table_verbosity"; + static const char* const OB_SV_SHOW_OLD_TEMPORALS = "show_old_temporals"; + static const char* const OB_SV_SQL_BIG_SELECTS = "sql_big_selects"; + static const char* const OB_SV_UPDATABLE_VIEWS_WITH_LIMIT = "updatable_views_with_limit"; + static const char* const OB_SV_VALIDATE_PASSWORD_DICTIONARY_FILE = "validate_password_dictionary_file"; + static const char* const OB_SV_DELAYED_INSERT_LIMIT = "delayed_insert_limit"; + static const char* const OB_SV_NDB_VERSION = "ndb_version"; + static const char* const OB_SV_AUTO_GENERATE_CERTS = "auto_generate_certs"; } } diff --git a/src/share/system_variable/ob_system_variable_factory.cpp b/src/share/system_variable/ob_system_variable_factory.cpp index cb8327dbc..765430bcd 100644 --- a/src/share/system_variable/ob_system_variable_factory.cpp +++ b/src/share/system_variable/ob_system_variable_factory.cpp @@ -90,6 +90,10 @@ const char *ObSysVarBlockEncryptionMode::BLOCK_ENCRYPTION_MODE_NAMES[] = { "aes-128-ofb", "aes-192-ofb", "aes-256-ofb", + "sm4-ecb", + "sm4-cbc", + "sm4-cfb", + "sm4-ofb", 0 }; const char *ObSysVarValidatePasswordCheckUserName::VALIDATE_PASSWORD_CHECK_USER_NAME_NAMES[] = { @@ -402,6 +406,17 @@ const char *ObSysVarSlaveParallelType::SLAVE_PARALLEL_TYPE_NAMES[] = { "LOGICAL_CLOCK", 0 }; +const char *ObSysVarBinlogErrorAction::BINLOG_ERROR_ACTION_NAMES[] = { + "IGNORE_ERROR", + "ABORT_SERVER", + 0 +}; +const char *ObSysVarBinlogTransactionDependencyTracking::BINLOG_TRANSACTION_DEPENDENCY_TRACKING_NAMES[] = { + "COMMIT_ORDER", + "WRITESET", + "WRITESET_SESSION", + 0 +}; const char *ObSysVarDefaultTmpStorageEngine::DEFAULT_TMP_STORAGE_ENGINE_NAMES[] = { "InnoDB", 0 @@ -444,6 +459,55 @@ const char *ObSysVarObKvMode::OB_KV_MODE_NAMES[] = { "NONE", 0 }; +const char *ObSysVarInnodbStatsMethod::INNODB_STATS_METHOD_NAMES[] = { + "nulls_equal", + "nulls_unequal", + "nulls_ignored", + 0 +}; +const char *ObSysVarKeyringAwsRegion::KEYRING_AWS_REGION_NAMES[] = { + "af-south-1", + "ap-east-1", + "ap-northeast-1", + "ap-northeast-2", + "ap-northeast-3", + "ap-south-1", + "ap-southeast-1", + "ap-southeast-2", + "ca-central-1", + "cn-north-1", + "cn-northwest-1", + "eu-central-1", + "eu-north-1", + "eu-south-1", + "eu-west-1", + "eu-west-2", + "eu-west-3", + "me-south-1", + "sa-east-1", + "us-east-1", + "us-east-2", + "us-gov-east-1", + "us-iso-east-1", + "us-iso-west-1", + "us-isob-east-1", + "us-west-1", + "us-west-2", + 0 +}; +const char *ObSysVarOldPasswords::OLD_PASSWORDS_NAMES[] = { + "0", + "1", + "2", + 0 +}; +const char *ObSysVarUpdatableViewsWithLimit::UPDATABLE_VIEWS_WITH_LIMIT_NAMES[] = { + "OFF", + "ON", + "NO", + "YES", + 0 +}; const char *ObSysVarFactory::SYS_VAR_NAMES_SORTED_BY_NAME[] = { "_aggregation_optimization_settings", @@ -486,16 +550,28 @@ const char *ObSysVarFactory::SYS_VAR_NAMES_SORTED_BY_NAME[] = { "_show_ddl_in_compat_mode", "_windowfunc_optimization_settings", "activate_all_roles_on_login", + "auto_generate_certs", "auto_increment_cache_size", "auto_increment_increment", "auto_increment_offset", "autocommit", "automatic_sp_privileges", "avoid_temporal_upgrade", + "big_tables", + "binlog_cache_size", "binlog_checksum", + "binlog_direct_non_transactional_updates", + "binlog_error_action", "binlog_format", + "binlog_group_commit_sync_delay", + "binlog_group_commit_sync_no_delay_count", + "binlog_max_flush_queue_time", + "binlog_order_commits", "binlog_row_image", "binlog_rows_query_log_events", + "binlog_stmt_cache_size", + "binlog_transaction_dependency_history_size", + "binlog_transaction_dependency_tracking", "block_encryption_mode", "cardinality_estimation_model", "character_set_client", @@ -505,15 +581,22 @@ const char *ObSysVarFactory::SYS_VAR_NAMES_SORTED_BY_NAME[] = { "character_set_results", "character_set_server", "character_set_system", + "character_sets_dir", + "check_proxy_users", "collation_connection", "collation_database", "collation_server", "completion_type", "concurrent_insert", "connect_timeout", + "connection_control_failed_connections_threshold", + "connection_control_max_connection_delay", + "connection_control_min_connection_delay", "cte_max_recursion_depth", "cursor_sharing", "datadir", + "date_format", + "datetime_format", "debug", "debug_sync", "default_authentication_plugin", @@ -521,13 +604,21 @@ const char *ObSysVarFactory::SYS_VAR_NAMES_SORTED_BY_NAME[] = { "default_password_lifetime", "default_storage_engine", "default_tmp_storage_engine", + "default_week_format", "delay_key_write", + "delayed_insert_limit", + "delayed_insert_timeout", + "delayed_queue_size", "disabled_storage_engines", + "disconnect_on_expired_password", "div_precision_increment", "enforce_gtid_consistency", + "eq_range_index_dive_limit", "error_count", "error_on_overlap_time", + "expire_logs_days", "explicit_defaults_for_timestamp", + "external_user", "flush", "flush_time", "foreign_key_checks", @@ -576,6 +667,8 @@ const char *ObSysVarFactory::SYS_VAR_NAMES_SORTED_BY_NAME[] = { "gtid_next", "gtid_owned", "gtid_purged", + "have_crypt", + "have_dynamic_loading", "have_openssl", "have_profiling", "have_query_cache", @@ -621,6 +714,7 @@ const char *ObSysVarFactory::SYS_VAR_NAMES_SORTED_BY_NAME[] = { "innodb_concurrency_tickets", "innodb_data_file_path", "innodb_data_home_dir", + "innodb_deadlock_detect", "innodb_default_row_format", "innodb_disable_resize_buffer_pool_debug", "innodb_disable_sort_file_cache", @@ -631,6 +725,8 @@ const char *ObSysVarFactory::SYS_VAR_NAMES_SORTED_BY_NAME[] = { "innodb_file_format_max", "innodb_file_per_table", "innodb_fill_factor", + "innodb_flush_log_at_timeout", + "innodb_flush_log_at_trx_commit", "innodb_flush_method", "innodb_flush_neighbors", "innodb_flush_sync", @@ -646,61 +742,137 @@ const char *ObSysVarFactory::SYS_VAR_NAMES_SORTED_BY_NAME[] = { "innodb_ft_total_cache_size", "innodb_large_prefix", "innodb_limit_optimistic_insert_debug", + "innodb_lock_wait_timeout", + "innodb_log_checkpoint_now", + "innodb_log_checksums", + "innodb_log_compressed_pages", + "innodb_log_write_ahead_size", "innodb_lru_scan_depth", "innodb_max_dirty_pages_pct", "innodb_max_dirty_pages_pct_lwm", "innodb_max_purge_lag", "innodb_max_purge_lag_delay", + "innodb_max_undo_log_size", "innodb_merge_threshold_set_all_debug", + "innodb_online_alter_log_max_size", "innodb_optimize_fulltext_only", "innodb_page_size", + "innodb_print_all_deadlocks", "innodb_read_only", "innodb_replication_delay", "innodb_rollback_on_timeout", "innodb_saved_page_number_debug", "innodb_sort_buffer_size", + "innodb_stats_auto_recalc", + "innodb_stats_include_delete_marked", + "innodb_stats_method", + "innodb_stats_on_metadata", "innodb_stats_persistent", + "innodb_stats_persistent_sample_pages", + "innodb_stats_sample_pages", + "innodb_stats_transient_sample_pages", "innodb_strict_mode", "innodb_support_xa", "innodb_sync_debug", + "innodb_table_locks", "innodb_temp_data_file_path", "innodb_tmpdir", "innodb_trx_purge_view_update_only_debug", "innodb_trx_rseg_n_slots_debug", + "innodb_undo_log_truncate", + "innodb_undo_logs", "innodb_version", + "insert_id", "interactive_timeout", "is_result_accurate", + "join_buffer_size", "key_buffer_size", "key_cache_age_threshold", "key_cache_block_size", "key_cache_division_limit", + "keyring_aws_cmk_id", + "keyring_aws_conf_file", + "keyring_aws_data_file", + "keyring_aws_region", + "keyring_encrypted_file_data", + "keyring_encrypted_file_password", + "keyring_file_data", + "keyring_okv_conf_dir", + "keyring_operations", + "language", "last_insert_id", "lc_messages", + "lc_messages_dir", "lc_time_names", "license", "local_infile", "lock_wait_timeout", "log_bin", + "log_bin_trust_function_creators", + "log_bin_use_v1_row_events", + "log_builtin_as_identified_by_password", "log_row_value_options", "long_query_time", + "lower_case_file_system", "lower_case_table_names", "master_info_repository", "master_verify_checksum", "max_allowed_packet", + "max_binlog_cache_size", + "max_binlog_size", + "max_binlog_stmt_cache_size", + "max_connect_errors", "max_connections", + "max_digest_length", "max_execution_time", + "max_join_size", + "max_length_for_sort_data", + "max_prepared_stmt_count", + "max_relay_log_size", "max_seeks_for_key", + "max_sort_length", "max_sp_recursion_depth", "max_tmp_tables", "max_user_connections", + "max_write_lock_count", "mecab_rc_file", "metadata_locks_cache_size", "metadata_locks_hash_instances", + "min_examined_row_limit", + "multi_range_count", "myisam_mmap_size", + "mysql_firewall_mode", + "mysql_firewall_trace", + "mysql_native_password_proxy_users", + "mysqlx_connect_timeout", + "mysqlx_idle_worker_thread_timeout", + "mysqlx_max_allowed_packet", + "mysqlx_max_connections", + "mysqlx_min_worker_threads", "ncharacter_set_connection", + "ndb_batch_size", + "ndb_cluster_connection_pool", + "ndb_cluster_connection_pool_nodeids", + "ndb_log_apply_status", + "ndb_log_bin", + "ndb_log_fail_terminate", + "ndb_log_orig", + "ndb_log_transaction_id", + "ndb_optimized_node_selection", + "Ndb_system_name", + "ndb_use_copying_alter_table", + "ndb_version", + "ndb_version_string", + "ndb_wait_connected", + "ndb_wait_setup", + "ndbinfo_database", + "ndbinfo_table_prefix", + "ndbinfo_version", "net_buffer_length", "net_read_timeout", + "net_retry_count", "net_write_timeout", + "new", "nls_calendar", "nls_characterset", "nls_comp", @@ -769,40 +941,64 @@ const char *ObSysVarFactory::SYS_VAR_NAMES_SORTED_BY_NAME[] = { "ob_trx_lock_timeout", "ob_trx_timeout", "old_alter_table", + "old_passwords", "optimizer_capture_sql_plan_baselines", "optimizer_dynamic_sampling", "optimizer_features_enable", + "optimizer_prune_level", + "optimizer_search_depth", + "optimizer_switch", + "optimizer_trace", + "optimizer_trace_features", + "optimizer_trace_limit", + "optimizer_trace_max_mem_size", + "optimizer_trace_offset", "optimizer_use_sql_plan_baselines", "parallel_degree_limit", "parallel_degree_policy", "parallel_min_scan_time_threshold", "parallel_servers_target", + "parser_max_mem_size", "performance_schema", + "performance_schema_show_processlist", "plsql_ccflags", "plsql_optimize_level", "plsql_warnings", "plugin_dir", + "plugin_load", + "plugin_load_add", "privilege_features_enable", "profiling", "profiling_history_size", "protocol_version", + "proxy_user", "pseudo_slave_mode", "pseudo_thread_id", + "query_alloc_block_size", "query_cache_limit", "query_cache_min_res_unit", "query_cache_size", "query_cache_type", "query_cache_wlock_invalidate", + "query_prealloc_size", "query_rewrite_enabled", "query_rewrite_integrity", + "rand_seed1", + "rand_seed2", + "range_alloc_block_size", + "range_optimizer_max_mem_size", "rbr_exec_mode", "read_only", "recyclebin", "regexp_stack_limit", "regexp_time_limit", + "relay_log_info_repository", + "relay_log_purge", "replication_optimize_for_static_plugin_config", "replication_sender_observe_commit_only", "resource_manager_plan", + "rewriter_enabled", + "rewriter_verbose", "rpl_semi_sync_master_enabled", "rpl_semi_sync_master_timeout", "rpl_semi_sync_master_trace_level", @@ -816,6 +1012,7 @@ const char *ObSysVarFactory::SYS_VAR_NAMES_SORTED_BY_NAME[] = { "runtime_filter_max_in_num", "runtime_filter_type", "runtime_filter_wait_time_ms", + "secure_auth", "secure_file_priv", "server_id", "server_uuid", @@ -824,7 +1021,15 @@ const char *ObSysVarFactory::SYS_VAR_NAMES_SORTED_BY_NAME[] = { "session_track_state_change", "session_track_system_variables", "session_track_transaction_info", + "sha256_password_auto_generate_rsa_keys", + "sha256_password_private_key_path", + "sha256_password_proxy_users", + "sha256_password_public_key_path", + "show_compatibility_56", + "show_create_table_verbosity", + "show_old_temporals", "skip_external_locking", + "skip_show_database", "skip_slave_start", "slave_allow_batching", "slave_checkpoint_group", @@ -843,7 +1048,12 @@ const char *ObSysVarFactory::SYS_VAR_NAMES_SORTED_BY_NAME[] = { "slave_sql_verify_checksum", "slave_transaction_retries", "slave_type_conversions", + "slow_query_log", + "slow_query_log_file", + "sort_buffer_size", "sql_auto_is_null", + "sql_big_selects", + "sql_buffer_result", "sql_mode", "sql_notes", "sql_quote_show_create", @@ -867,6 +1077,9 @@ const char *ObSysVarFactory::SYS_VAR_NAMES_SORTED_BY_NAME[] = { "ssl_key", "stored_program_cache", "super_read_only", + "sync_binlog", + "sync_relay_log", + "sync_relay_log_info", "system_time_zone", "table_definition_cache", "table_open_cache_instances", @@ -886,7 +1099,9 @@ const char *ObSysVarFactory::SYS_VAR_NAMES_SORTED_BY_NAME[] = { "tx_isolation", "tx_read_only", "unique_checks", + "updatable_views_with_limit", "validate_password_check_user_name", + "validate_password_dictionary_file", "validate_password_length", "validate_password_mixed_case_count", "validate_password_number_count", @@ -896,6 +1111,7 @@ const char *ObSysVarFactory::SYS_VAR_NAMES_SORTED_BY_NAME[] = { "version_comment", "version_compile_machine", "version_compile_os", + "version_tokens_session", "wait_timeout", "warning_count" }; @@ -941,16 +1157,28 @@ const ObSysVarClassType ObSysVarFactory::SYS_VAR_IDS_SORTED_BY_NAME[] = { SYS_VAR__SHOW_DDL_IN_COMPAT_MODE, SYS_VAR__WINDOWFUNC_OPTIMIZATION_SETTINGS, SYS_VAR_ACTIVATE_ALL_ROLES_ON_LOGIN, + SYS_VAR_AUTO_GENERATE_CERTS, SYS_VAR_AUTO_INCREMENT_CACHE_SIZE, SYS_VAR_AUTO_INCREMENT_INCREMENT, SYS_VAR_AUTO_INCREMENT_OFFSET, SYS_VAR_AUTOCOMMIT, SYS_VAR_AUTOMATIC_SP_PRIVILEGES, SYS_VAR_AVOID_TEMPORAL_UPGRADE, + SYS_VAR_BIG_TABLES, + SYS_VAR_BINLOG_CACHE_SIZE, SYS_VAR_BINLOG_CHECKSUM, + SYS_VAR_BINLOG_DIRECT_NON_TRANSACTIONAL_UPDATES, + SYS_VAR_BINLOG_ERROR_ACTION, SYS_VAR_BINLOG_FORMAT, + SYS_VAR_BINLOG_GROUP_COMMIT_SYNC_DELAY, + SYS_VAR_BINLOG_GROUP_COMMIT_SYNC_NO_DELAY_COUNT, + SYS_VAR_BINLOG_MAX_FLUSH_QUEUE_TIME, + SYS_VAR_BINLOG_ORDER_COMMITS, SYS_VAR_BINLOG_ROW_IMAGE, SYS_VAR_BINLOG_ROWS_QUERY_LOG_EVENTS, + SYS_VAR_BINLOG_STMT_CACHE_SIZE, + SYS_VAR_BINLOG_TRANSACTION_DEPENDENCY_HISTORY_SIZE, + SYS_VAR_BINLOG_TRANSACTION_DEPENDENCY_TRACKING, SYS_VAR_BLOCK_ENCRYPTION_MODE, SYS_VAR_CARDINALITY_ESTIMATION_MODEL, SYS_VAR_CHARACTER_SET_CLIENT, @@ -960,15 +1188,22 @@ const ObSysVarClassType ObSysVarFactory::SYS_VAR_IDS_SORTED_BY_NAME[] = { SYS_VAR_CHARACTER_SET_RESULTS, SYS_VAR_CHARACTER_SET_SERVER, SYS_VAR_CHARACTER_SET_SYSTEM, + SYS_VAR_CHARACTER_SETS_DIR, + SYS_VAR_CHECK_PROXY_USERS, SYS_VAR_COLLATION_CONNECTION, SYS_VAR_COLLATION_DATABASE, SYS_VAR_COLLATION_SERVER, SYS_VAR_COMPLETION_TYPE, SYS_VAR_CONCURRENT_INSERT, SYS_VAR_CONNECT_TIMEOUT, + SYS_VAR_CONNECTION_CONTROL_FAILED_CONNECTIONS_THRESHOLD, + SYS_VAR_CONNECTION_CONTROL_MAX_CONNECTION_DELAY, + SYS_VAR_CONNECTION_CONTROL_MIN_CONNECTION_DELAY, SYS_VAR_CTE_MAX_RECURSION_DEPTH, SYS_VAR_CURSOR_SHARING, SYS_VAR_DATADIR, + SYS_VAR_DATE_FORMAT, + SYS_VAR_DATETIME_FORMAT, SYS_VAR_DEBUG, SYS_VAR_DEBUG_SYNC, SYS_VAR_DEFAULT_AUTHENTICATION_PLUGIN, @@ -976,13 +1211,21 @@ const ObSysVarClassType ObSysVarFactory::SYS_VAR_IDS_SORTED_BY_NAME[] = { SYS_VAR_DEFAULT_PASSWORD_LIFETIME, SYS_VAR_DEFAULT_STORAGE_ENGINE, SYS_VAR_DEFAULT_TMP_STORAGE_ENGINE, + SYS_VAR_DEFAULT_WEEK_FORMAT, SYS_VAR_DELAY_KEY_WRITE, + SYS_VAR_DELAYED_INSERT_LIMIT, + SYS_VAR_DELAYED_INSERT_TIMEOUT, + SYS_VAR_DELAYED_QUEUE_SIZE, SYS_VAR_DISABLED_STORAGE_ENGINES, + SYS_VAR_DISCONNECT_ON_EXPIRED_PASSWORD, SYS_VAR_DIV_PRECISION_INCREMENT, SYS_VAR_ENFORCE_GTID_CONSISTENCY, + SYS_VAR_EQ_RANGE_INDEX_DIVE_LIMIT, SYS_VAR_ERROR_COUNT, SYS_VAR_ERROR_ON_OVERLAP_TIME, + SYS_VAR_EXPIRE_LOGS_DAYS, SYS_VAR_EXPLICIT_DEFAULTS_FOR_TIMESTAMP, + SYS_VAR_EXTERNAL_USER, SYS_VAR_FLUSH, SYS_VAR_FLUSH_TIME, SYS_VAR_FOREIGN_KEY_CHECKS, @@ -1031,6 +1274,8 @@ const ObSysVarClassType ObSysVarFactory::SYS_VAR_IDS_SORTED_BY_NAME[] = { SYS_VAR_GTID_NEXT, SYS_VAR_GTID_OWNED, SYS_VAR_GTID_PURGED, + SYS_VAR_HAVE_CRYPT, + SYS_VAR_HAVE_DYNAMIC_LOADING, SYS_VAR_HAVE_OPENSSL, SYS_VAR_HAVE_PROFILING, SYS_VAR_HAVE_QUERY_CACHE, @@ -1076,6 +1321,7 @@ const ObSysVarClassType ObSysVarFactory::SYS_VAR_IDS_SORTED_BY_NAME[] = { SYS_VAR_INNODB_CONCURRENCY_TICKETS, SYS_VAR_INNODB_DATA_FILE_PATH, SYS_VAR_INNODB_DATA_HOME_DIR, + SYS_VAR_INNODB_DEADLOCK_DETECT, SYS_VAR_INNODB_DEFAULT_ROW_FORMAT, SYS_VAR_INNODB_DISABLE_RESIZE_BUFFER_POOL_DEBUG, SYS_VAR_INNODB_DISABLE_SORT_FILE_CACHE, @@ -1086,6 +1332,8 @@ const ObSysVarClassType ObSysVarFactory::SYS_VAR_IDS_SORTED_BY_NAME[] = { SYS_VAR_INNODB_FILE_FORMAT_MAX, SYS_VAR_INNODB_FILE_PER_TABLE, SYS_VAR_INNODB_FILL_FACTOR, + SYS_VAR_INNODB_FLUSH_LOG_AT_TIMEOUT, + SYS_VAR_INNODB_FLUSH_LOG_AT_TRX_COMMIT, SYS_VAR_INNODB_FLUSH_METHOD, SYS_VAR_INNODB_FLUSH_NEIGHBORS, SYS_VAR_INNODB_FLUSH_SYNC, @@ -1101,61 +1349,137 @@ const ObSysVarClassType ObSysVarFactory::SYS_VAR_IDS_SORTED_BY_NAME[] = { SYS_VAR_INNODB_FT_TOTAL_CACHE_SIZE, SYS_VAR_INNODB_LARGE_PREFIX, SYS_VAR_INNODB_LIMIT_OPTIMISTIC_INSERT_DEBUG, + SYS_VAR_INNODB_LOCK_WAIT_TIMEOUT, + SYS_VAR_INNODB_LOG_CHECKPOINT_NOW, + SYS_VAR_INNODB_LOG_CHECKSUMS, + SYS_VAR_INNODB_LOG_COMPRESSED_PAGES, + SYS_VAR_INNODB_LOG_WRITE_AHEAD_SIZE, SYS_VAR_INNODB_LRU_SCAN_DEPTH, SYS_VAR_INNODB_MAX_DIRTY_PAGES_PCT, SYS_VAR_INNODB_MAX_DIRTY_PAGES_PCT_LWM, SYS_VAR_INNODB_MAX_PURGE_LAG, SYS_VAR_INNODB_MAX_PURGE_LAG_DELAY, + SYS_VAR_INNODB_MAX_UNDO_LOG_SIZE, SYS_VAR_INNODB_MERGE_THRESHOLD_SET_ALL_DEBUG, + SYS_VAR_INNODB_ONLINE_ALTER_LOG_MAX_SIZE, SYS_VAR_INNODB_OPTIMIZE_FULLTEXT_ONLY, SYS_VAR_INNODB_PAGE_SIZE, + SYS_VAR_INNODB_PRINT_ALL_DEADLOCKS, SYS_VAR_INNODB_READ_ONLY, SYS_VAR_INNODB_REPLICATION_DELAY, SYS_VAR_INNODB_ROLLBACK_ON_TIMEOUT, SYS_VAR_INNODB_SAVED_PAGE_NUMBER_DEBUG, SYS_VAR_INNODB_SORT_BUFFER_SIZE, + SYS_VAR_INNODB_STATS_AUTO_RECALC, + SYS_VAR_INNODB_STATS_INCLUDE_DELETE_MARKED, + SYS_VAR_INNODB_STATS_METHOD, + SYS_VAR_INNODB_STATS_ON_METADATA, SYS_VAR_INNODB_STATS_PERSISTENT, + SYS_VAR_INNODB_STATS_PERSISTENT_SAMPLE_PAGES, + SYS_VAR_INNODB_STATS_SAMPLE_PAGES, + SYS_VAR_INNODB_STATS_TRANSIENT_SAMPLE_PAGES, SYS_VAR_INNODB_STRICT_MODE, SYS_VAR_INNODB_SUPPORT_XA, SYS_VAR_INNODB_SYNC_DEBUG, + SYS_VAR_INNODB_TABLE_LOCKS, SYS_VAR_INNODB_TEMP_DATA_FILE_PATH, SYS_VAR_INNODB_TMPDIR, SYS_VAR_INNODB_TRX_PURGE_VIEW_UPDATE_ONLY_DEBUG, SYS_VAR_INNODB_TRX_RSEG_N_SLOTS_DEBUG, + SYS_VAR_INNODB_UNDO_LOG_TRUNCATE, + SYS_VAR_INNODB_UNDO_LOGS, SYS_VAR_INNODB_VERSION, + SYS_VAR_INSERT_ID, SYS_VAR_INTERACTIVE_TIMEOUT, SYS_VAR_IS_RESULT_ACCURATE, + SYS_VAR_JOIN_BUFFER_SIZE, SYS_VAR_KEY_BUFFER_SIZE, SYS_VAR_KEY_CACHE_AGE_THRESHOLD, SYS_VAR_KEY_CACHE_BLOCK_SIZE, SYS_VAR_KEY_CACHE_DIVISION_LIMIT, + SYS_VAR_KEYRING_AWS_CMK_ID, + SYS_VAR_KEYRING_AWS_CONF_FILE, + SYS_VAR_KEYRING_AWS_DATA_FILE, + SYS_VAR_KEYRING_AWS_REGION, + SYS_VAR_KEYRING_ENCRYPTED_FILE_DATA, + SYS_VAR_KEYRING_ENCRYPTED_FILE_PASSWORD, + SYS_VAR_KEYRING_FILE_DATA, + SYS_VAR_KEYRING_OKV_CONF_DIR, + SYS_VAR_KEYRING_OPERATIONS, + SYS_VAR_LANGUAGE, SYS_VAR_LAST_INSERT_ID, SYS_VAR_LC_MESSAGES, + SYS_VAR_LC_MESSAGES_DIR, SYS_VAR_LC_TIME_NAMES, SYS_VAR_LICENSE, SYS_VAR_LOCAL_INFILE, SYS_VAR_LOCK_WAIT_TIMEOUT, SYS_VAR_LOG_BIN, + SYS_VAR_LOG_BIN_TRUST_FUNCTION_CREATORS, + SYS_VAR_LOG_BIN_USE_V1_ROW_EVENTS, + SYS_VAR_LOG_BUILTIN_AS_IDENTIFIED_BY_PASSWORD, SYS_VAR_LOG_ROW_VALUE_OPTIONS, SYS_VAR_LONG_QUERY_TIME, + SYS_VAR_LOWER_CASE_FILE_SYSTEM, SYS_VAR_LOWER_CASE_TABLE_NAMES, SYS_VAR_MASTER_INFO_REPOSITORY, SYS_VAR_MASTER_VERIFY_CHECKSUM, SYS_VAR_MAX_ALLOWED_PACKET, + SYS_VAR_MAX_BINLOG_CACHE_SIZE, + SYS_VAR_MAX_BINLOG_SIZE, + SYS_VAR_MAX_BINLOG_STMT_CACHE_SIZE, + SYS_VAR_MAX_CONNECT_ERRORS, SYS_VAR_MAX_CONNECTIONS, + SYS_VAR_MAX_DIGEST_LENGTH, SYS_VAR_MAX_EXECUTION_TIME, + SYS_VAR_MAX_JOIN_SIZE, + SYS_VAR_MAX_LENGTH_FOR_SORT_DATA, + SYS_VAR_MAX_PREPARED_STMT_COUNT, + SYS_VAR_MAX_RELAY_LOG_SIZE, SYS_VAR_MAX_SEEKS_FOR_KEY, + SYS_VAR_MAX_SORT_LENGTH, SYS_VAR_MAX_SP_RECURSION_DEPTH, SYS_VAR_MAX_TMP_TABLES, SYS_VAR_MAX_USER_CONNECTIONS, + SYS_VAR_MAX_WRITE_LOCK_COUNT, SYS_VAR_MECAB_RC_FILE, SYS_VAR_METADATA_LOCKS_CACHE_SIZE, SYS_VAR_METADATA_LOCKS_HASH_INSTANCES, + SYS_VAR_MIN_EXAMINED_ROW_LIMIT, + SYS_VAR_MULTI_RANGE_COUNT, SYS_VAR_MYISAM_MMAP_SIZE, + SYS_VAR_MYSQL_FIREWALL_MODE, + SYS_VAR_MYSQL_FIREWALL_TRACE, + SYS_VAR_MYSQL_NATIVE_PASSWORD_PROXY_USERS, + SYS_VAR_MYSQLX_CONNECT_TIMEOUT, + SYS_VAR_MYSQLX_IDLE_WORKER_THREAD_TIMEOUT, + SYS_VAR_MYSQLX_MAX_ALLOWED_PACKET, + SYS_VAR_MYSQLX_MAX_CONNECTIONS, + SYS_VAR_MYSQLX_MIN_WORKER_THREADS, SYS_VAR_NCHARACTER_SET_CONNECTION, + SYS_VAR_NDB_BATCH_SIZE, + SYS_VAR_NDB_CLUSTER_CONNECTION_POOL, + SYS_VAR_NDB_CLUSTER_CONNECTION_POOL_NODEIDS, + SYS_VAR_NDB_LOG_APPLY_STATUS, + SYS_VAR_NDB_LOG_BIN, + SYS_VAR_NDB_LOG_FAIL_TERMINATE, + SYS_VAR_NDB_LOG_ORIG, + SYS_VAR_NDB_LOG_TRANSACTION_ID, + SYS_VAR_NDB_OPTIMIZED_NODE_SELECTION, + SYS_VAR_NDB_SYSTEM_NAME, + SYS_VAR_NDB_USE_COPYING_ALTER_TABLE, + SYS_VAR_NDB_VERSION, + SYS_VAR_NDB_VERSION_STRING, + SYS_VAR_NDB_WAIT_CONNECTED, + SYS_VAR_NDB_WAIT_SETUP, + SYS_VAR_NDBINFO_DATABASE, + SYS_VAR_NDBINFO_TABLE_PREFIX, + SYS_VAR_NDBINFO_VERSION, SYS_VAR_NET_BUFFER_LENGTH, SYS_VAR_NET_READ_TIMEOUT, + SYS_VAR_NET_RETRY_COUNT, SYS_VAR_NET_WRITE_TIMEOUT, + SYS_VAR_NEW, SYS_VAR_NLS_CALENDAR, SYS_VAR_NLS_CHARACTERSET, SYS_VAR_NLS_COMP, @@ -1224,40 +1548,64 @@ const ObSysVarClassType ObSysVarFactory::SYS_VAR_IDS_SORTED_BY_NAME[] = { SYS_VAR_OB_TRX_LOCK_TIMEOUT, SYS_VAR_OB_TRX_TIMEOUT, SYS_VAR_OLD_ALTER_TABLE, + SYS_VAR_OLD_PASSWORDS, SYS_VAR_OPTIMIZER_CAPTURE_SQL_PLAN_BASELINES, SYS_VAR_OPTIMIZER_DYNAMIC_SAMPLING, SYS_VAR_OPTIMIZER_FEATURES_ENABLE, + SYS_VAR_OPTIMIZER_PRUNE_LEVEL, + SYS_VAR_OPTIMIZER_SEARCH_DEPTH, + SYS_VAR_OPTIMIZER_SWITCH, + SYS_VAR_OPTIMIZER_TRACE, + SYS_VAR_OPTIMIZER_TRACE_FEATURES, + SYS_VAR_OPTIMIZER_TRACE_LIMIT, + SYS_VAR_OPTIMIZER_TRACE_MAX_MEM_SIZE, + SYS_VAR_OPTIMIZER_TRACE_OFFSET, SYS_VAR_OPTIMIZER_USE_SQL_PLAN_BASELINES, SYS_VAR_PARALLEL_DEGREE_LIMIT, SYS_VAR_PARALLEL_DEGREE_POLICY, SYS_VAR_PARALLEL_MIN_SCAN_TIME_THRESHOLD, SYS_VAR_PARALLEL_SERVERS_TARGET, + SYS_VAR_PARSER_MAX_MEM_SIZE, SYS_VAR_PERFORMANCE_SCHEMA, + SYS_VAR_PERFORMANCE_SCHEMA_SHOW_PROCESSLIST, SYS_VAR_PLSQL_CCFLAGS, SYS_VAR_PLSQL_OPTIMIZE_LEVEL, SYS_VAR_PLSQL_WARNINGS, SYS_VAR_PLUGIN_DIR, + SYS_VAR_PLUGIN_LOAD, + SYS_VAR_PLUGIN_LOAD_ADD, SYS_VAR_PRIVILEGE_FEATURES_ENABLE, SYS_VAR_PROFILING, SYS_VAR_PROFILING_HISTORY_SIZE, SYS_VAR_PROTOCOL_VERSION, + SYS_VAR_PROXY_USER, SYS_VAR_PSEUDO_SLAVE_MODE, SYS_VAR_PSEUDO_THREAD_ID, + SYS_VAR_QUERY_ALLOC_BLOCK_SIZE, SYS_VAR_QUERY_CACHE_LIMIT, SYS_VAR_QUERY_CACHE_MIN_RES_UNIT, SYS_VAR_QUERY_CACHE_SIZE, SYS_VAR_QUERY_CACHE_TYPE, SYS_VAR_QUERY_CACHE_WLOCK_INVALIDATE, + SYS_VAR_QUERY_PREALLOC_SIZE, SYS_VAR_QUERY_REWRITE_ENABLED, SYS_VAR_QUERY_REWRITE_INTEGRITY, + SYS_VAR_RAND_SEED1, + SYS_VAR_RAND_SEED2, + SYS_VAR_RANGE_ALLOC_BLOCK_SIZE, + SYS_VAR_RANGE_OPTIMIZER_MAX_MEM_SIZE, SYS_VAR_RBR_EXEC_MODE, SYS_VAR_READ_ONLY, SYS_VAR_RECYCLEBIN, SYS_VAR_REGEXP_STACK_LIMIT, SYS_VAR_REGEXP_TIME_LIMIT, + SYS_VAR_RELAY_LOG_INFO_REPOSITORY, + SYS_VAR_RELAY_LOG_PURGE, SYS_VAR_REPLICATION_OPTIMIZE_FOR_STATIC_PLUGIN_CONFIG, SYS_VAR_REPLICATION_SENDER_OBSERVE_COMMIT_ONLY, SYS_VAR_RESOURCE_MANAGER_PLAN, + SYS_VAR_REWRITER_ENABLED, + SYS_VAR_REWRITER_VERBOSE, SYS_VAR_RPL_SEMI_SYNC_MASTER_ENABLED, SYS_VAR_RPL_SEMI_SYNC_MASTER_TIMEOUT, SYS_VAR_RPL_SEMI_SYNC_MASTER_TRACE_LEVEL, @@ -1271,6 +1619,7 @@ const ObSysVarClassType ObSysVarFactory::SYS_VAR_IDS_SORTED_BY_NAME[] = { SYS_VAR_RUNTIME_FILTER_MAX_IN_NUM, SYS_VAR_RUNTIME_FILTER_TYPE, SYS_VAR_RUNTIME_FILTER_WAIT_TIME_MS, + SYS_VAR_SECURE_AUTH, SYS_VAR_SECURE_FILE_PRIV, SYS_VAR_SERVER_ID, SYS_VAR_SERVER_UUID, @@ -1279,7 +1628,15 @@ const ObSysVarClassType ObSysVarFactory::SYS_VAR_IDS_SORTED_BY_NAME[] = { SYS_VAR_SESSION_TRACK_STATE_CHANGE, SYS_VAR_SESSION_TRACK_SYSTEM_VARIABLES, SYS_VAR_SESSION_TRACK_TRANSACTION_INFO, + SYS_VAR_SHA256_PASSWORD_AUTO_GENERATE_RSA_KEYS, + SYS_VAR_SHA256_PASSWORD_PRIVATE_KEY_PATH, + SYS_VAR_SHA256_PASSWORD_PROXY_USERS, + SYS_VAR_SHA256_PASSWORD_PUBLIC_KEY_PATH, + SYS_VAR_SHOW_COMPATIBILITY_56, + SYS_VAR_SHOW_CREATE_TABLE_VERBOSITY, + SYS_VAR_SHOW_OLD_TEMPORALS, SYS_VAR_SKIP_EXTERNAL_LOCKING, + SYS_VAR_SKIP_SHOW_DATABASE, SYS_VAR_SKIP_SLAVE_START, SYS_VAR_SLAVE_ALLOW_BATCHING, SYS_VAR_SLAVE_CHECKPOINT_GROUP, @@ -1298,7 +1655,12 @@ const ObSysVarClassType ObSysVarFactory::SYS_VAR_IDS_SORTED_BY_NAME[] = { SYS_VAR_SLAVE_SQL_VERIFY_CHECKSUM, SYS_VAR_SLAVE_TRANSACTION_RETRIES, SYS_VAR_SLAVE_TYPE_CONVERSIONS, + SYS_VAR_SLOW_QUERY_LOG, + SYS_VAR_SLOW_QUERY_LOG_FILE, + SYS_VAR_SORT_BUFFER_SIZE, SYS_VAR_SQL_AUTO_IS_NULL, + SYS_VAR_SQL_BIG_SELECTS, + SYS_VAR_SQL_BUFFER_RESULT, SYS_VAR_SQL_MODE, SYS_VAR_SQL_NOTES, SYS_VAR_SQL_QUOTE_SHOW_CREATE, @@ -1322,6 +1684,9 @@ const ObSysVarClassType ObSysVarFactory::SYS_VAR_IDS_SORTED_BY_NAME[] = { SYS_VAR_SSL_KEY, SYS_VAR_STORED_PROGRAM_CACHE, SYS_VAR_SUPER_READ_ONLY, + SYS_VAR_SYNC_BINLOG, + SYS_VAR_SYNC_RELAY_LOG, + SYS_VAR_SYNC_RELAY_LOG_INFO, SYS_VAR_SYSTEM_TIME_ZONE, SYS_VAR_TABLE_DEFINITION_CACHE, SYS_VAR_TABLE_OPEN_CACHE_INSTANCES, @@ -1341,7 +1706,9 @@ const ObSysVarClassType ObSysVarFactory::SYS_VAR_IDS_SORTED_BY_NAME[] = { SYS_VAR_TX_ISOLATION, SYS_VAR_TX_READ_ONLY, SYS_VAR_UNIQUE_CHECKS, + SYS_VAR_UPDATABLE_VIEWS_WITH_LIMIT, SYS_VAR_VALIDATE_PASSWORD_CHECK_USER_NAME, + SYS_VAR_VALIDATE_PASSWORD_DICTIONARY_FILE, SYS_VAR_VALIDATE_PASSWORD_LENGTH, SYS_VAR_VALIDATE_PASSWORD_MIXED_CASE_COUNT, SYS_VAR_VALIDATE_PASSWORD_NUMBER_COUNT, @@ -1351,6 +1718,7 @@ const ObSysVarClassType ObSysVarFactory::SYS_VAR_IDS_SORTED_BY_NAME[] = { SYS_VAR_VERSION_COMMENT, SYS_VAR_VERSION_COMPILE_MACHINE, SYS_VAR_VERSION_COMPILE_OS, + SYS_VAR_VERSION_TOKENS_SESSION, SYS_VAR_WAIT_TIMEOUT, SYS_VAR_WARNING_COUNT }; @@ -1767,6 +2135,64 @@ const char *ObSysVarFactory::SYS_VAR_NAMES_SORTED_BY_ID[] = { "innodb_sync_debug", "default_collation_for_utf8mb4", "_enable_old_charset_aggregation", + "insert_id", + "join_buffer_size", + "max_join_size", + "max_length_for_sort_data", + "max_prepared_stmt_count", + "max_sort_length", + "min_examined_row_limit", + "multi_range_count", + "mysqlx_connect_timeout", + "mysqlx_idle_worker_thread_timeout", + "mysqlx_max_allowed_packet", + "mysqlx_max_connections", + "mysqlx_min_worker_threads", + "performance_schema_show_processlist", + "query_alloc_block_size", + "query_prealloc_size", + "slow_query_log", + "slow_query_log_file", + "sort_buffer_size", + "sql_buffer_result", + "binlog_cache_size", + "binlog_direct_non_transactional_updates", + "binlog_error_action", + "binlog_group_commit_sync_delay", + "binlog_group_commit_sync_no_delay_count", + "binlog_max_flush_queue_time", + "binlog_order_commits", + "binlog_stmt_cache_size", + "binlog_transaction_dependency_history_size", + "binlog_transaction_dependency_tracking", + "expire_logs_days", + "innodb_flush_log_at_timeout", + "innodb_flush_log_at_trx_commit", + "innodb_log_checkpoint_now", + "innodb_log_checksums", + "innodb_log_compressed_pages", + "innodb_log_write_ahead_size", + "innodb_max_undo_log_size", + "innodb_online_alter_log_max_size", + "innodb_undo_log_truncate", + "innodb_undo_logs", + "log_bin_trust_function_creators", + "log_bin_use_v1_row_events", + "log_builtin_as_identified_by_password", + "max_binlog_cache_size", + "max_binlog_size", + "max_binlog_stmt_cache_size", + "max_relay_log_size", + "relay_log_info_repository", + "relay_log_purge", + "sync_binlog", + "sync_relay_log", + "sync_relay_log_info", + "innodb_deadlock_detect", + "innodb_lock_wait_timeout", + "innodb_print_all_deadlocks", + "innodb_table_locks", + "max_write_lock_count", "_ob_enable_role_ids", "innodb_read_only", "innodb_api_disable_rowlock", @@ -1807,7 +2233,101 @@ const char *ObSysVarFactory::SYS_VAR_NAMES_SORTED_BY_ID[] = { "innodb_sort_buffer_size", "key_cache_block_size", "ob_kv_mode", - "ob_enable_parameter_anonymous_block" + "ob_enable_parameter_anonymous_block", + "character_sets_dir", + "date_format", + "datetime_format", + "disconnect_on_expired_password", + "external_user", + "have_crypt", + "have_dynamic_loading", + "keyring_aws_conf_file", + "keyring_aws_data_file", + "language", + "lc_messages_dir", + "lower_case_file_system", + "max_digest_length", + "ndbinfo_database", + "ndbinfo_table_prefix", + "ndbinfo_version", + "ndb_batch_size", + "ndb_cluster_connection_pool", + "ndb_cluster_connection_pool_nodeids", + "ndb_log_apply_status", + "ndb_log_bin", + "ndb_log_fail_terminate", + "ndb_log_orig", + "ndb_log_transaction_id", + "ndb_optimized_node_selection", + "Ndb_system_name", + "ndb_use_copying_alter_table", + "ndb_version_string", + "ndb_wait_connected", + "ndb_wait_setup", + "proxy_user", + "sha256_password_auto_generate_rsa_keys", + "sha256_password_private_key_path", + "sha256_password_public_key_path", + "skip_show_database", + "plugin_load", + "plugin_load_add", + "big_tables", + "check_proxy_users", + "connection_control_failed_connections_threshold", + "connection_control_max_connection_delay", + "connection_control_min_connection_delay", + "default_week_format", + "delayed_insert_timeout", + "delayed_queue_size", + "eq_range_index_dive_limit", + "innodb_stats_auto_recalc", + "innodb_stats_include_delete_marked", + "innodb_stats_method", + "innodb_stats_on_metadata", + "version_tokens_session", + "innodb_stats_persistent_sample_pages", + "innodb_stats_sample_pages", + "innodb_stats_transient_sample_pages", + "keyring_aws_cmk_id", + "keyring_aws_region", + "keyring_encrypted_file_data", + "keyring_encrypted_file_password", + "keyring_file_data", + "keyring_okv_conf_dir", + "keyring_operations", + "optimizer_switch", + "max_connect_errors", + "mysql_firewall_mode", + "mysql_firewall_trace", + "mysql_native_password_proxy_users", + "net_retry_count", + "new", + "old_passwords", + "optimizer_prune_level", + "optimizer_search_depth", + "optimizer_trace", + "optimizer_trace_features", + "optimizer_trace_limit", + "optimizer_trace_max_mem_size", + "optimizer_trace_offset", + "parser_max_mem_size", + "rand_seed1", + "rand_seed2", + "range_alloc_block_size", + "range_optimizer_max_mem_size", + "rewriter_enabled", + "rewriter_verbose", + "secure_auth", + "sha256_password_proxy_users", + "show_compatibility_56", + "show_create_table_verbosity", + "show_old_temporals", + "sql_big_selects", + "updatable_views_with_limit", + "validate_password_dictionary_file", + "delayed_insert_limit", + "ndb_version", + "auto_generate_certs" }; bool ObSysVarFactory::sys_var_name_case_cmp(const char *name1, const ObString &name2) @@ -1924,10 +2444,34 @@ const ObString ObSysVarFactory::get_sys_var_name_by_id(ObSysVarClassType sys_var ObSysVarFactory::ObSysVarFactory(const int64_t tenant_id) : allocator_(ObMemAttr(tenant_id, ObModIds::OB_COMMON_SYS_VAR_FAC)), - all_sys_vars_created_(false) + store_(nullptr), store_buf_(nullptr), all_sys_vars_created_(false) { - MEMSET(store_, 0, sizeof(store_)); - MEMSET(store_buf_, 0, sizeof(store_buf_)); +} + +int ObSysVarFactory::try_init_store_mem() +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(store_)) { + void *store_ptr = NULL; + if (OB_ISNULL(store_ptr = allocator_.alloc(sizeof(ObBasicSysVar *) * ALL_SYS_VARS_COUNT))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc store_.", K(ret)); + } else { + store_ = static_cast(store_ptr); + MEMSET(store_, 0, sizeof(ObBasicSysVar *) * ALL_SYS_VARS_COUNT); + } + } + if (OB_ISNULL(store_buf_)) { + void *store_buf_ptr = NULL; + if (OB_ISNULL(store_buf_ptr = allocator_.alloc(sizeof(ObBasicSysVar *) * ALL_SYS_VARS_COUNT))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc store_buf_.", K(ret)); + } else { + store_buf_ = static_cast(store_buf_ptr); + MEMSET(store_buf_, 0, sizeof(ObBasicSysVar *) * ALL_SYS_VARS_COUNT); + } + } + return ret; } ObSysVarFactory::~ObSysVarFactory() @@ -1938,15 +2482,23 @@ ObSysVarFactory::~ObSysVarFactory() void ObSysVarFactory::destroy() { int ret = OB_SUCCESS; - for (int64_t i = 0; i < ALL_SYS_VARS_COUNT; ++i) { - if (OB_NOT_NULL(store_[i])) { - store_[i]->~ObBasicSysVar(); - store_[i] = nullptr; + if (OB_NOT_NULL(store_)) { + for (int64_t i = 0; i < ALL_SYS_VARS_COUNT; ++i) { + if (OB_NOT_NULL(store_[i])) { + store_[i]->~ObBasicSysVar(); + store_[i] = nullptr; + } } - if (OB_NOT_NULL(store_buf_[i])) { - store_buf_[i]->~ObBasicSysVar(); - store_buf_[i] = nullptr; + store_ = nullptr; + } + if (OB_NOT_NULL(store_buf_)) { + for (int64_t i = 0; i < ALL_SYS_VARS_COUNT; ++i) { + if (OB_NOT_NULL(store_buf_[i])) { + store_buf_[i]->~ObBasicSysVar(); + store_buf_[i] = nullptr; + } } + store_buf_ = nullptr; } allocator_.reset(); all_sys_vars_created_ = false; @@ -1955,24 +2507,28 @@ void ObSysVarFactory::destroy() int ObSysVarFactory::free_sys_var(ObBasicSysVar *sys_var, int64_t sys_var_idx) { int ret = OB_SUCCESS; - OV (OB_NOT_NULL(sys_var)); - OV (is_valid_sys_var_store_idx(sys_var_idx)); - OV (sys_var == store_[sys_var_idx], OB_ERR_UNEXPECTED, sys_var, sys_var_idx); - if (OB_NOT_NULL(store_buf_[sys_var_idx])) { - OX (store_buf_[sys_var_idx]->~ObBasicSysVar()); - OX (allocator_.free(store_buf_[sys_var_idx])); - OX (store_buf_[sys_var_idx] = nullptr); + if (OB_NOT_NULL(store_) && OB_NOT_NULL(store_buf_)) { + OV (OB_NOT_NULL(sys_var)); + OV (is_valid_sys_var_store_idx(sys_var_idx)); + OV (sys_var == store_[sys_var_idx], OB_ERR_UNEXPECTED, sys_var, sys_var_idx); + if (OB_NOT_NULL(store_buf_[sys_var_idx])) { + OX (store_buf_[sys_var_idx]->~ObBasicSysVar()); + OX (allocator_.free(store_buf_[sys_var_idx])); + OX (store_buf_[sys_var_idx] = nullptr); + } + OX (store_buf_[sys_var_idx] = store_[sys_var_idx]); + OX (store_buf_[sys_var_idx]->clean_value()); + OX (store_[sys_var_idx] = nullptr); } - OX (store_buf_[sys_var_idx] = store_[sys_var_idx]); - OX (store_buf_[sys_var_idx]->clean_value()); - OX (store_[sys_var_idx] = nullptr); return ret; } int ObSysVarFactory::create_all_sys_vars() { int ret = OB_SUCCESS; - if (!all_sys_vars_created_) { + if (OB_FAIL(try_init_store_mem())) { + LOG_WARN("Fail to init", K(ret)); + } else if (!all_sys_vars_created_) { int64_t store_idx = -1; ObBasicSysVar *sys_var_ptr = NULL; int64_t total_mem_size = 0 @@ -2387,6 +2943,64 @@ int ObSysVarFactory::create_all_sys_vars() + sizeof(ObSysVarInnodbSyncDebug) + sizeof(ObSysVarDefaultCollationForUtf8mb4) + sizeof(ObSysVarEnableOldCharsetAggregation) + + sizeof(ObSysVarInsertId) + + sizeof(ObSysVarJoinBufferSize) + + sizeof(ObSysVarMaxJoinSize) + + sizeof(ObSysVarMaxLengthForSortData) + + sizeof(ObSysVarMaxPreparedStmtCount) + + sizeof(ObSysVarMaxSortLength) + + sizeof(ObSysVarMinExaminedRowLimit) + + sizeof(ObSysVarMultiRangeCount) + + sizeof(ObSysVarMysqlxConnectTimeout) + + sizeof(ObSysVarMysqlxIdleWorkerThreadTimeout) + + sizeof(ObSysVarMysqlxMaxAllowedPacket) + + sizeof(ObSysVarMysqlxMaxConnections) + + sizeof(ObSysVarMysqlxMinWorkerThreads) + + sizeof(ObSysVarPerformanceSchemaShowProcesslist) + + sizeof(ObSysVarQueryAllocBlockSize) + + sizeof(ObSysVarQueryPreallocSize) + + sizeof(ObSysVarSlowQueryLog) + + sizeof(ObSysVarSlowQueryLogFile) + + sizeof(ObSysVarSortBufferSize) + + sizeof(ObSysVarSqlBufferResult) + + sizeof(ObSysVarBinlogCacheSize) + + sizeof(ObSysVarBinlogDirectNonTransactionalUpdates) + + sizeof(ObSysVarBinlogErrorAction) + + sizeof(ObSysVarBinlogGroupCommitSyncDelay) + + sizeof(ObSysVarBinlogGroupCommitSyncNoDelayCount) + + sizeof(ObSysVarBinlogMaxFlushQueueTime) + + sizeof(ObSysVarBinlogOrderCommits) + + sizeof(ObSysVarBinlogStmtCacheSize) + + sizeof(ObSysVarBinlogTransactionDependencyHistorySize) + + sizeof(ObSysVarBinlogTransactionDependencyTracking) + + sizeof(ObSysVarExpireLogsDays) + + sizeof(ObSysVarInnodbFlushLogAtTimeout) + + sizeof(ObSysVarInnodbFlushLogAtTrxCommit) + + sizeof(ObSysVarInnodbLogCheckpointNow) + + sizeof(ObSysVarInnodbLogChecksums) + + sizeof(ObSysVarInnodbLogCompressedPages) + + sizeof(ObSysVarInnodbLogWriteAheadSize) + + sizeof(ObSysVarInnodbMaxUndoLogSize) + + sizeof(ObSysVarInnodbOnlineAlterLogMaxSize) + + sizeof(ObSysVarInnodbUndoLogTruncate) + + sizeof(ObSysVarInnodbUndoLogs) + + sizeof(ObSysVarLogBinTrustFunctionCreators) + + sizeof(ObSysVarLogBinUseV1RowEvents) + + sizeof(ObSysVarLogBuiltinAsIdentifiedByPassword) + + sizeof(ObSysVarMaxBinlogCacheSize) + + sizeof(ObSysVarMaxBinlogSize) + + sizeof(ObSysVarMaxBinlogStmtCacheSize) + + sizeof(ObSysVarMaxRelayLogSize) + + sizeof(ObSysVarRelayLogInfoRepository) + + sizeof(ObSysVarRelayLogPurge) + + sizeof(ObSysVarSyncBinlog) + + sizeof(ObSysVarSyncRelayLog) + + sizeof(ObSysVarSyncRelayLogInfo) + + sizeof(ObSysVarInnodbDeadlockDetect) + + sizeof(ObSysVarInnodbLockWaitTimeout) + + sizeof(ObSysVarInnodbPrintAllDeadlocks) + + sizeof(ObSysVarInnodbTableLocks) + + sizeof(ObSysVarMaxWriteLockCount) + sizeof(ObSysVarObEnableRoleIds) + sizeof(ObSysVarInnodbReadOnly) + sizeof(ObSysVarInnodbApiDisableRowlock) @@ -2428,6 +3042,100 @@ int ObSysVarFactory::create_all_sys_vars() + sizeof(ObSysVarKeyCacheBlockSize) + sizeof(ObSysVarObKvMode) + sizeof(ObSysVarObEnableParameterAnonymousBlock) + + sizeof(ObSysVarCharacterSetsDir) + + sizeof(ObSysVarDateFormat) + + sizeof(ObSysVarDatetimeFormat) + + sizeof(ObSysVarDisconnectOnExpiredPassword) + + sizeof(ObSysVarExternalUser) + + sizeof(ObSysVarHaveCrypt) + + sizeof(ObSysVarHaveDynamicLoading) + + sizeof(ObSysVarKeyringAwsConfFile) + + sizeof(ObSysVarKeyringAwsDataFile) + + sizeof(ObSysVarLanguage) + + sizeof(ObSysVarLcMessagesDir) + + sizeof(ObSysVarLowerCaseFileSystem) + + sizeof(ObSysVarMaxDigestLength) + + sizeof(ObSysVarNdbinfoDatabase) + + sizeof(ObSysVarNdbinfoTablePrefix) + + sizeof(ObSysVarNdbinfoVersion) + + sizeof(ObSysVarNdbBatchSize) + + sizeof(ObSysVarNdbClusterConnectionPool) + + sizeof(ObSysVarNdbClusterConnectionPoolNodeids) + + sizeof(ObSysVarNdbLogApplyStatus) + + sizeof(ObSysVarNdbLogBin) + + sizeof(ObSysVarNdbLogFailTerminate) + + sizeof(ObSysVarNdbLogOrig) + + sizeof(ObSysVarNdbLogTransactionId) + + sizeof(ObSysVarNdbOptimizedNodeSelection) + + sizeof(ObSysVarNdbSystemName) + + sizeof(ObSysVarNdbUseCopyingAlterTable) + + sizeof(ObSysVarNdbVersionString) + + sizeof(ObSysVarNdbWaitConnected) + + sizeof(ObSysVarNdbWaitSetup) + + sizeof(ObSysVarProxyUser) + + sizeof(ObSysVarSha256PasswordAutoGenerateRsaKeys) + + sizeof(ObSysVarSha256PasswordPrivateKeyPath) + + sizeof(ObSysVarSha256PasswordPublicKeyPath) + + sizeof(ObSysVarSkipShowDatabase) + + sizeof(ObSysVarPluginLoad) + + sizeof(ObSysVarPluginLoadAdd) + + sizeof(ObSysVarBigTables) + + sizeof(ObSysVarCheckProxyUsers) + + sizeof(ObSysVarConnectionControlFailedConnectionsThreshold) + + sizeof(ObSysVarConnectionControlMaxConnectionDelay) + + sizeof(ObSysVarConnectionControlMinConnectionDelay) + + sizeof(ObSysVarDefaultWeekFormat) + + sizeof(ObSysVarDelayedInsertTimeout) + + sizeof(ObSysVarDelayedQueueSize) + + sizeof(ObSysVarEqRangeIndexDiveLimit) + + sizeof(ObSysVarInnodbStatsAutoRecalc) + + sizeof(ObSysVarInnodbStatsIncludeDeleteMarked) + + sizeof(ObSysVarInnodbStatsMethod) + + sizeof(ObSysVarInnodbStatsOnMetadata) + + sizeof(ObSysVarVersionTokensSession) + + sizeof(ObSysVarInnodbStatsPersistentSamplePages) + + sizeof(ObSysVarInnodbStatsSamplePages) + + sizeof(ObSysVarInnodbStatsTransientSamplePages) + + sizeof(ObSysVarKeyringAwsCmkId) + + sizeof(ObSysVarKeyringAwsRegion) + + sizeof(ObSysVarKeyringEncryptedFileData) + + sizeof(ObSysVarKeyringEncryptedFilePassword) + + sizeof(ObSysVarKeyringFileData) + + sizeof(ObSysVarKeyringOkvConfDir) + + sizeof(ObSysVarKeyringOperations) + + sizeof(ObSysVarOptimizerSwitch) + + sizeof(ObSysVarMaxConnectErrors) + + sizeof(ObSysVarMysqlFirewallMode) + + sizeof(ObSysVarMysqlFirewallTrace) + + sizeof(ObSysVarMysqlNativePasswordProxyUsers) + + sizeof(ObSysVarNetRetryCount) + + sizeof(ObSysVarNew) + + sizeof(ObSysVarOldPasswords) + + sizeof(ObSysVarOptimizerPruneLevel) + + sizeof(ObSysVarOptimizerSearchDepth) + + sizeof(ObSysVarOptimizerTrace) + + sizeof(ObSysVarOptimizerTraceFeatures) + + sizeof(ObSysVarOptimizerTraceLimit) + + sizeof(ObSysVarOptimizerTraceMaxMemSize) + + sizeof(ObSysVarOptimizerTraceOffset) + + sizeof(ObSysVarParserMaxMemSize) + + sizeof(ObSysVarRandSeed1) + + sizeof(ObSysVarRandSeed2) + + sizeof(ObSysVarRangeAllocBlockSize) + + sizeof(ObSysVarRangeOptimizerMaxMemSize) + + sizeof(ObSysVarRewriterEnabled) + + sizeof(ObSysVarRewriterVerbose) + + sizeof(ObSysVarSecureAuth) + + sizeof(ObSysVarSha256PasswordProxyUsers) + + sizeof(ObSysVarShowCompatibility56) + + sizeof(ObSysVarShowCreateTableVerbosity) + + sizeof(ObSysVarShowOldTemporals) + + sizeof(ObSysVarSqlBigSelects) + + sizeof(ObSysVarUpdatableViewsWithLimit) + + sizeof(ObSysVarValidatePasswordDictionaryFile) + + sizeof(ObSysVarDelayedInsertLimit) + + sizeof(ObSysVarNdbVersion) + + sizeof(ObSysVarAutoGenerateCerts) ; void *ptr = NULL; if (OB_ISNULL(ptr = allocator_.alloc(total_mem_size))) { @@ -6135,6 +6843,528 @@ int ObSysVarFactory::create_all_sys_vars() ptr = (void *)((char *)ptr + sizeof(ObSysVarEnableOldCharsetAggregation)); } } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInsertId())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInsertId", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_INSERT_ID))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarInsertId)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarJoinBufferSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarJoinBufferSize", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_JOIN_BUFFER_SIZE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarJoinBufferSize)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMaxJoinSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMaxJoinSize", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_MAX_JOIN_SIZE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarMaxJoinSize)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMaxLengthForSortData())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMaxLengthForSortData", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_MAX_LENGTH_FOR_SORT_DATA))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarMaxLengthForSortData)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMaxPreparedStmtCount())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMaxPreparedStmtCount", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_MAX_PREPARED_STMT_COUNT))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarMaxPreparedStmtCount)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMaxSortLength())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMaxSortLength", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_MAX_SORT_LENGTH))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarMaxSortLength)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMinExaminedRowLimit())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMinExaminedRowLimit", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_MIN_EXAMINED_ROW_LIMIT))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarMinExaminedRowLimit)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMultiRangeCount())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMultiRangeCount", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_MULTI_RANGE_COUNT))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarMultiRangeCount)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMysqlxConnectTimeout())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMysqlxConnectTimeout", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_MYSQLX_CONNECT_TIMEOUT))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarMysqlxConnectTimeout)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMysqlxIdleWorkerThreadTimeout())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMysqlxIdleWorkerThreadTimeout", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_MYSQLX_IDLE_WORKER_THREAD_TIMEOUT))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarMysqlxIdleWorkerThreadTimeout)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMysqlxMaxAllowedPacket())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMysqlxMaxAllowedPacket", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_MYSQLX_MAX_ALLOWED_PACKET))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarMysqlxMaxAllowedPacket)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMysqlxMaxConnections())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMysqlxMaxConnections", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_MYSQLX_MAX_CONNECTIONS))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarMysqlxMaxConnections)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMysqlxMinWorkerThreads())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMysqlxMinWorkerThreads", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_MYSQLX_MIN_WORKER_THREADS))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarMysqlxMinWorkerThreads)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarPerformanceSchemaShowProcesslist())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarPerformanceSchemaShowProcesslist", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_PERFORMANCE_SCHEMA_SHOW_PROCESSLIST))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarPerformanceSchemaShowProcesslist)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarQueryAllocBlockSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarQueryAllocBlockSize", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_QUERY_ALLOC_BLOCK_SIZE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarQueryAllocBlockSize)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarQueryPreallocSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarQueryPreallocSize", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_QUERY_PREALLOC_SIZE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarQueryPreallocSize)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSlowQueryLog())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSlowQueryLog", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_SLOW_QUERY_LOG))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarSlowQueryLog)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSlowQueryLogFile())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSlowQueryLogFile", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_SLOW_QUERY_LOG_FILE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarSlowQueryLogFile)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSortBufferSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSortBufferSize", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_SORT_BUFFER_SIZE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarSortBufferSize)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSqlBufferResult())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSqlBufferResult", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_SQL_BUFFER_RESULT))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarSqlBufferResult)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarBinlogCacheSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarBinlogCacheSize", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_BINLOG_CACHE_SIZE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarBinlogCacheSize)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarBinlogDirectNonTransactionalUpdates())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarBinlogDirectNonTransactionalUpdates", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_BINLOG_DIRECT_NON_TRANSACTIONAL_UPDATES))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarBinlogDirectNonTransactionalUpdates)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarBinlogErrorAction())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarBinlogErrorAction", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_BINLOG_ERROR_ACTION))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarBinlogErrorAction)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarBinlogGroupCommitSyncDelay())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarBinlogGroupCommitSyncDelay", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_BINLOG_GROUP_COMMIT_SYNC_DELAY))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarBinlogGroupCommitSyncDelay)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarBinlogGroupCommitSyncNoDelayCount())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarBinlogGroupCommitSyncNoDelayCount", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_BINLOG_GROUP_COMMIT_SYNC_NO_DELAY_COUNT))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarBinlogGroupCommitSyncNoDelayCount)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarBinlogMaxFlushQueueTime())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarBinlogMaxFlushQueueTime", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_BINLOG_MAX_FLUSH_QUEUE_TIME))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarBinlogMaxFlushQueueTime)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarBinlogOrderCommits())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarBinlogOrderCommits", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_BINLOG_ORDER_COMMITS))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarBinlogOrderCommits)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarBinlogStmtCacheSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarBinlogStmtCacheSize", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_BINLOG_STMT_CACHE_SIZE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarBinlogStmtCacheSize)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarBinlogTransactionDependencyHistorySize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarBinlogTransactionDependencyHistorySize", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_BINLOG_TRANSACTION_DEPENDENCY_HISTORY_SIZE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarBinlogTransactionDependencyHistorySize)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarBinlogTransactionDependencyTracking())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarBinlogTransactionDependencyTracking", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_BINLOG_TRANSACTION_DEPENDENCY_TRACKING))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarBinlogTransactionDependencyTracking)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarExpireLogsDays())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarExpireLogsDays", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_EXPIRE_LOGS_DAYS))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarExpireLogsDays)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbFlushLogAtTimeout())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbFlushLogAtTimeout", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_INNODB_FLUSH_LOG_AT_TIMEOUT))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarInnodbFlushLogAtTimeout)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbFlushLogAtTrxCommit())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbFlushLogAtTrxCommit", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_INNODB_FLUSH_LOG_AT_TRX_COMMIT))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarInnodbFlushLogAtTrxCommit)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbLogCheckpointNow())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbLogCheckpointNow", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_INNODB_LOG_CHECKPOINT_NOW))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarInnodbLogCheckpointNow)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbLogChecksums())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbLogChecksums", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_INNODB_LOG_CHECKSUMS))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarInnodbLogChecksums)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbLogCompressedPages())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbLogCompressedPages", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_INNODB_LOG_COMPRESSED_PAGES))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarInnodbLogCompressedPages)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbLogWriteAheadSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbLogWriteAheadSize", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_INNODB_LOG_WRITE_AHEAD_SIZE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarInnodbLogWriteAheadSize)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbMaxUndoLogSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbMaxUndoLogSize", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_INNODB_MAX_UNDO_LOG_SIZE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarInnodbMaxUndoLogSize)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbOnlineAlterLogMaxSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbOnlineAlterLogMaxSize", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_INNODB_ONLINE_ALTER_LOG_MAX_SIZE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarInnodbOnlineAlterLogMaxSize)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbUndoLogTruncate())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbUndoLogTruncate", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_INNODB_UNDO_LOG_TRUNCATE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarInnodbUndoLogTruncate)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbUndoLogs())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbUndoLogs", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_INNODB_UNDO_LOGS))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarInnodbUndoLogs)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarLogBinTrustFunctionCreators())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarLogBinTrustFunctionCreators", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_LOG_BIN_TRUST_FUNCTION_CREATORS))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarLogBinTrustFunctionCreators)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarLogBinUseV1RowEvents())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarLogBinUseV1RowEvents", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_LOG_BIN_USE_V1_ROW_EVENTS))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarLogBinUseV1RowEvents)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarLogBuiltinAsIdentifiedByPassword())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarLogBuiltinAsIdentifiedByPassword", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_LOG_BUILTIN_AS_IDENTIFIED_BY_PASSWORD))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarLogBuiltinAsIdentifiedByPassword)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMaxBinlogCacheSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMaxBinlogCacheSize", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_MAX_BINLOG_CACHE_SIZE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarMaxBinlogCacheSize)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMaxBinlogSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMaxBinlogSize", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_MAX_BINLOG_SIZE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarMaxBinlogSize)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMaxBinlogStmtCacheSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMaxBinlogStmtCacheSize", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_MAX_BINLOG_STMT_CACHE_SIZE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarMaxBinlogStmtCacheSize)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMaxRelayLogSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMaxRelayLogSize", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_MAX_RELAY_LOG_SIZE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarMaxRelayLogSize)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarRelayLogInfoRepository())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarRelayLogInfoRepository", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_RELAY_LOG_INFO_REPOSITORY))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarRelayLogInfoRepository)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarRelayLogPurge())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarRelayLogPurge", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_RELAY_LOG_PURGE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarRelayLogPurge)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSyncBinlog())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSyncBinlog", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_SYNC_BINLOG))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarSyncBinlog)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSyncRelayLog())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSyncRelayLog", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_SYNC_RELAY_LOG))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarSyncRelayLog)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSyncRelayLogInfo())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSyncRelayLogInfo", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_SYNC_RELAY_LOG_INFO))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarSyncRelayLogInfo)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbDeadlockDetect())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbDeadlockDetect", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_INNODB_DEADLOCK_DETECT))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarInnodbDeadlockDetect)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbLockWaitTimeout())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbLockWaitTimeout", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_INNODB_LOCK_WAIT_TIMEOUT))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarInnodbLockWaitTimeout)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbPrintAllDeadlocks())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbPrintAllDeadlocks", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_INNODB_PRINT_ALL_DEADLOCKS))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarInnodbPrintAllDeadlocks)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbTableLocks())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbTableLocks", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_INNODB_TABLE_LOCKS))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarInnodbTableLocks)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMaxWriteLockCount())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMaxWriteLockCount", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_MAX_WRITE_LOCK_COUNT))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarMaxWriteLockCount)); + } + } if (OB_SUCC(ret)) { if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarObEnableRoleIds())) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -6504,6 +7734,852 @@ int ObSysVarFactory::create_all_sys_vars() ptr = (void *)((char *)ptr + sizeof(ObSysVarObEnableParameterAnonymousBlock)); } } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarCharacterSetsDir())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarCharacterSetsDir", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_CHARACTER_SETS_DIR))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarCharacterSetsDir)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarDateFormat())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarDateFormat", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_DATE_FORMAT))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarDateFormat)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarDatetimeFormat())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarDatetimeFormat", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_DATETIME_FORMAT))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarDatetimeFormat)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarDisconnectOnExpiredPassword())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarDisconnectOnExpiredPassword", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_DISCONNECT_ON_EXPIRED_PASSWORD))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarDisconnectOnExpiredPassword)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarExternalUser())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarExternalUser", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_EXTERNAL_USER))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarExternalUser)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarHaveCrypt())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarHaveCrypt", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_HAVE_CRYPT))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarHaveCrypt)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarHaveDynamicLoading())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarHaveDynamicLoading", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_HAVE_DYNAMIC_LOADING))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarHaveDynamicLoading)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarKeyringAwsConfFile())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarKeyringAwsConfFile", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_KEYRING_AWS_CONF_FILE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarKeyringAwsConfFile)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarKeyringAwsDataFile())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarKeyringAwsDataFile", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_KEYRING_AWS_DATA_FILE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarKeyringAwsDataFile)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarLanguage())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarLanguage", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_LANGUAGE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarLanguage)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarLcMessagesDir())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarLcMessagesDir", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_LC_MESSAGES_DIR))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarLcMessagesDir)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarLowerCaseFileSystem())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarLowerCaseFileSystem", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_LOWER_CASE_FILE_SYSTEM))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarLowerCaseFileSystem)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMaxDigestLength())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMaxDigestLength", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_MAX_DIGEST_LENGTH))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarMaxDigestLength)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbinfoDatabase())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbinfoDatabase", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_NDBINFO_DATABASE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarNdbinfoDatabase)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbinfoTablePrefix())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbinfoTablePrefix", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_NDBINFO_TABLE_PREFIX))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarNdbinfoTablePrefix)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbinfoVersion())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbinfoVersion", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_NDBINFO_VERSION))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarNdbinfoVersion)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbBatchSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbBatchSize", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_NDB_BATCH_SIZE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarNdbBatchSize)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbClusterConnectionPool())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbClusterConnectionPool", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_NDB_CLUSTER_CONNECTION_POOL))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarNdbClusterConnectionPool)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbClusterConnectionPoolNodeids())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbClusterConnectionPoolNodeids", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_NDB_CLUSTER_CONNECTION_POOL_NODEIDS))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarNdbClusterConnectionPoolNodeids)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbLogApplyStatus())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbLogApplyStatus", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_NDB_LOG_APPLY_STATUS))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarNdbLogApplyStatus)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbLogBin())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbLogBin", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_NDB_LOG_BIN))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarNdbLogBin)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbLogFailTerminate())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbLogFailTerminate", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_NDB_LOG_FAIL_TERMINATE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarNdbLogFailTerminate)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbLogOrig())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbLogOrig", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_NDB_LOG_ORIG))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarNdbLogOrig)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbLogTransactionId())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbLogTransactionId", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_NDB_LOG_TRANSACTION_ID))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarNdbLogTransactionId)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbOptimizedNodeSelection())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbOptimizedNodeSelection", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_NDB_OPTIMIZED_NODE_SELECTION))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarNdbOptimizedNodeSelection)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbSystemName())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbSystemName", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_NDB_SYSTEM_NAME))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarNdbSystemName)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbUseCopyingAlterTable())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbUseCopyingAlterTable", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_NDB_USE_COPYING_ALTER_TABLE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarNdbUseCopyingAlterTable)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbVersionString())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbVersionString", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_NDB_VERSION_STRING))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarNdbVersionString)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbWaitConnected())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbWaitConnected", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_NDB_WAIT_CONNECTED))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarNdbWaitConnected)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbWaitSetup())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbWaitSetup", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_NDB_WAIT_SETUP))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarNdbWaitSetup)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarProxyUser())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarProxyUser", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_PROXY_USER))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarProxyUser)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSha256PasswordAutoGenerateRsaKeys())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSha256PasswordAutoGenerateRsaKeys", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_SHA256_PASSWORD_AUTO_GENERATE_RSA_KEYS))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarSha256PasswordAutoGenerateRsaKeys)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSha256PasswordPrivateKeyPath())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSha256PasswordPrivateKeyPath", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_SHA256_PASSWORD_PRIVATE_KEY_PATH))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarSha256PasswordPrivateKeyPath)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSha256PasswordPublicKeyPath())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSha256PasswordPublicKeyPath", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_SHA256_PASSWORD_PUBLIC_KEY_PATH))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarSha256PasswordPublicKeyPath)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSkipShowDatabase())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSkipShowDatabase", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_SKIP_SHOW_DATABASE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarSkipShowDatabase)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarPluginLoad())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarPluginLoad", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_PLUGIN_LOAD))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarPluginLoad)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarPluginLoadAdd())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarPluginLoadAdd", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_PLUGIN_LOAD_ADD))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarPluginLoadAdd)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarBigTables())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarBigTables", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_BIG_TABLES))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarBigTables)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarCheckProxyUsers())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarCheckProxyUsers", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_CHECK_PROXY_USERS))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarCheckProxyUsers)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarConnectionControlFailedConnectionsThreshold())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarConnectionControlFailedConnectionsThreshold", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_CONNECTION_CONTROL_FAILED_CONNECTIONS_THRESHOLD))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarConnectionControlFailedConnectionsThreshold)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarConnectionControlMaxConnectionDelay())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarConnectionControlMaxConnectionDelay", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_CONNECTION_CONTROL_MAX_CONNECTION_DELAY))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarConnectionControlMaxConnectionDelay)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarConnectionControlMinConnectionDelay())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarConnectionControlMinConnectionDelay", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_CONNECTION_CONTROL_MIN_CONNECTION_DELAY))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarConnectionControlMinConnectionDelay)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarDefaultWeekFormat())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarDefaultWeekFormat", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_DEFAULT_WEEK_FORMAT))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarDefaultWeekFormat)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarDelayedInsertTimeout())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarDelayedInsertTimeout", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_DELAYED_INSERT_TIMEOUT))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarDelayedInsertTimeout)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarDelayedQueueSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarDelayedQueueSize", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_DELAYED_QUEUE_SIZE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarDelayedQueueSize)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarEqRangeIndexDiveLimit())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarEqRangeIndexDiveLimit", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_EQ_RANGE_INDEX_DIVE_LIMIT))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarEqRangeIndexDiveLimit)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbStatsAutoRecalc())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbStatsAutoRecalc", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_INNODB_STATS_AUTO_RECALC))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarInnodbStatsAutoRecalc)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbStatsIncludeDeleteMarked())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbStatsIncludeDeleteMarked", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_INNODB_STATS_INCLUDE_DELETE_MARKED))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarInnodbStatsIncludeDeleteMarked)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbStatsMethod())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbStatsMethod", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_INNODB_STATS_METHOD))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarInnodbStatsMethod)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbStatsOnMetadata())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbStatsOnMetadata", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_INNODB_STATS_ON_METADATA))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarInnodbStatsOnMetadata)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarVersionTokensSession())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarVersionTokensSession", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_VERSION_TOKENS_SESSION))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarVersionTokensSession)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbStatsPersistentSamplePages())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbStatsPersistentSamplePages", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_INNODB_STATS_PERSISTENT_SAMPLE_PAGES))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarInnodbStatsPersistentSamplePages)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbStatsSamplePages())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbStatsSamplePages", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_INNODB_STATS_SAMPLE_PAGES))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarInnodbStatsSamplePages)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbStatsTransientSamplePages())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbStatsTransientSamplePages", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_INNODB_STATS_TRANSIENT_SAMPLE_PAGES))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarInnodbStatsTransientSamplePages)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarKeyringAwsCmkId())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarKeyringAwsCmkId", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_KEYRING_AWS_CMK_ID))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarKeyringAwsCmkId)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarKeyringAwsRegion())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarKeyringAwsRegion", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_KEYRING_AWS_REGION))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarKeyringAwsRegion)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarKeyringEncryptedFileData())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarKeyringEncryptedFileData", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_KEYRING_ENCRYPTED_FILE_DATA))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarKeyringEncryptedFileData)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarKeyringEncryptedFilePassword())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarKeyringEncryptedFilePassword", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_KEYRING_ENCRYPTED_FILE_PASSWORD))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarKeyringEncryptedFilePassword)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarKeyringFileData())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarKeyringFileData", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_KEYRING_FILE_DATA))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarKeyringFileData)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarKeyringOkvConfDir())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarKeyringOkvConfDir", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_KEYRING_OKV_CONF_DIR))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarKeyringOkvConfDir)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarKeyringOperations())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarKeyringOperations", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_KEYRING_OPERATIONS))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarKeyringOperations)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarOptimizerSwitch())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarOptimizerSwitch", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_OPTIMIZER_SWITCH))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarOptimizerSwitch)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMaxConnectErrors())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMaxConnectErrors", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_MAX_CONNECT_ERRORS))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarMaxConnectErrors)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMysqlFirewallMode())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMysqlFirewallMode", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_MYSQL_FIREWALL_MODE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarMysqlFirewallMode)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMysqlFirewallTrace())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMysqlFirewallTrace", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_MYSQL_FIREWALL_TRACE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarMysqlFirewallTrace)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMysqlNativePasswordProxyUsers())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMysqlNativePasswordProxyUsers", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_MYSQL_NATIVE_PASSWORD_PROXY_USERS))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarMysqlNativePasswordProxyUsers)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNetRetryCount())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNetRetryCount", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_NET_RETRY_COUNT))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarNetRetryCount)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNew())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNew", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_NEW))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarNew)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarOldPasswords())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarOldPasswords", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_OLD_PASSWORDS))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarOldPasswords)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarOptimizerPruneLevel())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarOptimizerPruneLevel", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_OPTIMIZER_PRUNE_LEVEL))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarOptimizerPruneLevel)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarOptimizerSearchDepth())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarOptimizerSearchDepth", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_OPTIMIZER_SEARCH_DEPTH))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarOptimizerSearchDepth)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarOptimizerTrace())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarOptimizerTrace", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_OPTIMIZER_TRACE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarOptimizerTrace)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarOptimizerTraceFeatures())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarOptimizerTraceFeatures", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_OPTIMIZER_TRACE_FEATURES))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarOptimizerTraceFeatures)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarOptimizerTraceLimit())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarOptimizerTraceLimit", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_OPTIMIZER_TRACE_LIMIT))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarOptimizerTraceLimit)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarOptimizerTraceMaxMemSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarOptimizerTraceMaxMemSize", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_OPTIMIZER_TRACE_MAX_MEM_SIZE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarOptimizerTraceMaxMemSize)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarOptimizerTraceOffset())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarOptimizerTraceOffset", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_OPTIMIZER_TRACE_OFFSET))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarOptimizerTraceOffset)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarParserMaxMemSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarParserMaxMemSize", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_PARSER_MAX_MEM_SIZE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarParserMaxMemSize)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarRandSeed1())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarRandSeed1", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_RAND_SEED1))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarRandSeed1)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarRandSeed2())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarRandSeed2", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_RAND_SEED2))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarRandSeed2)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarRangeAllocBlockSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarRangeAllocBlockSize", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_RANGE_ALLOC_BLOCK_SIZE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarRangeAllocBlockSize)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarRangeOptimizerMaxMemSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarRangeOptimizerMaxMemSize", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_RANGE_OPTIMIZER_MAX_MEM_SIZE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarRangeOptimizerMaxMemSize)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarRewriterEnabled())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarRewriterEnabled", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_REWRITER_ENABLED))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarRewriterEnabled)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarRewriterVerbose())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarRewriterVerbose", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_REWRITER_VERBOSE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarRewriterVerbose)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSecureAuth())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSecureAuth", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_SECURE_AUTH))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarSecureAuth)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSha256PasswordProxyUsers())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSha256PasswordProxyUsers", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_SHA256_PASSWORD_PROXY_USERS))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarSha256PasswordProxyUsers)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarShowCompatibility56())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarShowCompatibility56", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_SHOW_COMPATIBILITY_56))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarShowCompatibility56)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarShowCreateTableVerbosity())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarShowCreateTableVerbosity", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_SHOW_CREATE_TABLE_VERBOSITY))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarShowCreateTableVerbosity)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarShowOldTemporals())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarShowOldTemporals", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_SHOW_OLD_TEMPORALS))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarShowOldTemporals)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSqlBigSelects())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSqlBigSelects", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_SQL_BIG_SELECTS))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarSqlBigSelects)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarUpdatableViewsWithLimit())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarUpdatableViewsWithLimit", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_UPDATABLE_VIEWS_WITH_LIMIT))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarUpdatableViewsWithLimit)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarValidatePasswordDictionaryFile())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarValidatePasswordDictionaryFile", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_VALIDATE_PASSWORD_DICTIONARY_FILE))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarValidatePasswordDictionaryFile)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarDelayedInsertLimit())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarDelayedInsertLimit", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_DELAYED_INSERT_LIMIT))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarDelayedInsertLimit)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbVersion())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbVersion", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_NDB_VERSION))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarNdbVersion)); + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarAutoGenerateCerts())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarAutoGenerateCerts", K(ret)); + } else { + store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast(SYS_VAR_AUTO_GENERATE_CERTS))] = sys_var_ptr; + ptr = (void *)((char *)ptr + sizeof(ObSysVarAutoGenerateCerts)); + } + } } return ret; @@ -11035,6 +13111,644 @@ int ObSysVarFactory::create_sys_var(ObIAllocator &allocator_, ObSysVarClassType } break; } + case SYS_VAR_INSERT_ID: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarInsertId)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarInsertId))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInsertId())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInsertId", K(ret)); + } + break; + } + case SYS_VAR_JOIN_BUFFER_SIZE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarJoinBufferSize)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarJoinBufferSize))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarJoinBufferSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarJoinBufferSize", K(ret)); + } + break; + } + case SYS_VAR_MAX_JOIN_SIZE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarMaxJoinSize)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarMaxJoinSize))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMaxJoinSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMaxJoinSize", K(ret)); + } + break; + } + case SYS_VAR_MAX_LENGTH_FOR_SORT_DATA: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarMaxLengthForSortData)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarMaxLengthForSortData))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMaxLengthForSortData())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMaxLengthForSortData", K(ret)); + } + break; + } + case SYS_VAR_MAX_PREPARED_STMT_COUNT: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarMaxPreparedStmtCount)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarMaxPreparedStmtCount))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMaxPreparedStmtCount())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMaxPreparedStmtCount", K(ret)); + } + break; + } + case SYS_VAR_MAX_SORT_LENGTH: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarMaxSortLength)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarMaxSortLength))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMaxSortLength())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMaxSortLength", K(ret)); + } + break; + } + case SYS_VAR_MIN_EXAMINED_ROW_LIMIT: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarMinExaminedRowLimit)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarMinExaminedRowLimit))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMinExaminedRowLimit())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMinExaminedRowLimit", K(ret)); + } + break; + } + case SYS_VAR_MULTI_RANGE_COUNT: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarMultiRangeCount)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarMultiRangeCount))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMultiRangeCount())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMultiRangeCount", K(ret)); + } + break; + } + case SYS_VAR_MYSQLX_CONNECT_TIMEOUT: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarMysqlxConnectTimeout)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarMysqlxConnectTimeout))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMysqlxConnectTimeout())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMysqlxConnectTimeout", K(ret)); + } + break; + } + case SYS_VAR_MYSQLX_IDLE_WORKER_THREAD_TIMEOUT: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarMysqlxIdleWorkerThreadTimeout)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarMysqlxIdleWorkerThreadTimeout))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMysqlxIdleWorkerThreadTimeout())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMysqlxIdleWorkerThreadTimeout", K(ret)); + } + break; + } + case SYS_VAR_MYSQLX_MAX_ALLOWED_PACKET: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarMysqlxMaxAllowedPacket)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarMysqlxMaxAllowedPacket))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMysqlxMaxAllowedPacket())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMysqlxMaxAllowedPacket", K(ret)); + } + break; + } + case SYS_VAR_MYSQLX_MAX_CONNECTIONS: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarMysqlxMaxConnections)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarMysqlxMaxConnections))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMysqlxMaxConnections())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMysqlxMaxConnections", K(ret)); + } + break; + } + case SYS_VAR_MYSQLX_MIN_WORKER_THREADS: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarMysqlxMinWorkerThreads)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarMysqlxMinWorkerThreads))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMysqlxMinWorkerThreads())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMysqlxMinWorkerThreads", K(ret)); + } + break; + } + case SYS_VAR_PERFORMANCE_SCHEMA_SHOW_PROCESSLIST: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarPerformanceSchemaShowProcesslist)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarPerformanceSchemaShowProcesslist))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarPerformanceSchemaShowProcesslist())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarPerformanceSchemaShowProcesslist", K(ret)); + } + break; + } + case SYS_VAR_QUERY_ALLOC_BLOCK_SIZE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarQueryAllocBlockSize)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarQueryAllocBlockSize))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarQueryAllocBlockSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarQueryAllocBlockSize", K(ret)); + } + break; + } + case SYS_VAR_QUERY_PREALLOC_SIZE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarQueryPreallocSize)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarQueryPreallocSize))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarQueryPreallocSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarQueryPreallocSize", K(ret)); + } + break; + } + case SYS_VAR_SLOW_QUERY_LOG: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarSlowQueryLog)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarSlowQueryLog))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSlowQueryLog())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSlowQueryLog", K(ret)); + } + break; + } + case SYS_VAR_SLOW_QUERY_LOG_FILE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarSlowQueryLogFile)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarSlowQueryLogFile))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSlowQueryLogFile())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSlowQueryLogFile", K(ret)); + } + break; + } + case SYS_VAR_SORT_BUFFER_SIZE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarSortBufferSize)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarSortBufferSize))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSortBufferSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSortBufferSize", K(ret)); + } + break; + } + case SYS_VAR_SQL_BUFFER_RESULT: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarSqlBufferResult)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarSqlBufferResult))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSqlBufferResult())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSqlBufferResult", K(ret)); + } + break; + } + case SYS_VAR_BINLOG_CACHE_SIZE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarBinlogCacheSize)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarBinlogCacheSize))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarBinlogCacheSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarBinlogCacheSize", K(ret)); + } + break; + } + case SYS_VAR_BINLOG_DIRECT_NON_TRANSACTIONAL_UPDATES: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarBinlogDirectNonTransactionalUpdates)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarBinlogDirectNonTransactionalUpdates))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarBinlogDirectNonTransactionalUpdates())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarBinlogDirectNonTransactionalUpdates", K(ret)); + } + break; + } + case SYS_VAR_BINLOG_ERROR_ACTION: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarBinlogErrorAction)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarBinlogErrorAction))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarBinlogErrorAction())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarBinlogErrorAction", K(ret)); + } + break; + } + case SYS_VAR_BINLOG_GROUP_COMMIT_SYNC_DELAY: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarBinlogGroupCommitSyncDelay)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarBinlogGroupCommitSyncDelay))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarBinlogGroupCommitSyncDelay())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarBinlogGroupCommitSyncDelay", K(ret)); + } + break; + } + case SYS_VAR_BINLOG_GROUP_COMMIT_SYNC_NO_DELAY_COUNT: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarBinlogGroupCommitSyncNoDelayCount)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarBinlogGroupCommitSyncNoDelayCount))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarBinlogGroupCommitSyncNoDelayCount())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarBinlogGroupCommitSyncNoDelayCount", K(ret)); + } + break; + } + case SYS_VAR_BINLOG_MAX_FLUSH_QUEUE_TIME: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarBinlogMaxFlushQueueTime)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarBinlogMaxFlushQueueTime))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarBinlogMaxFlushQueueTime())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarBinlogMaxFlushQueueTime", K(ret)); + } + break; + } + case SYS_VAR_BINLOG_ORDER_COMMITS: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarBinlogOrderCommits)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarBinlogOrderCommits))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarBinlogOrderCommits())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarBinlogOrderCommits", K(ret)); + } + break; + } + case SYS_VAR_BINLOG_STMT_CACHE_SIZE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarBinlogStmtCacheSize)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarBinlogStmtCacheSize))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarBinlogStmtCacheSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarBinlogStmtCacheSize", K(ret)); + } + break; + } + case SYS_VAR_BINLOG_TRANSACTION_DEPENDENCY_HISTORY_SIZE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarBinlogTransactionDependencyHistorySize)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarBinlogTransactionDependencyHistorySize))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarBinlogTransactionDependencyHistorySize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarBinlogTransactionDependencyHistorySize", K(ret)); + } + break; + } + case SYS_VAR_BINLOG_TRANSACTION_DEPENDENCY_TRACKING: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarBinlogTransactionDependencyTracking)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarBinlogTransactionDependencyTracking))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarBinlogTransactionDependencyTracking())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarBinlogTransactionDependencyTracking", K(ret)); + } + break; + } + case SYS_VAR_EXPIRE_LOGS_DAYS: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarExpireLogsDays)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarExpireLogsDays))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarExpireLogsDays())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarExpireLogsDays", K(ret)); + } + break; + } + case SYS_VAR_INNODB_FLUSH_LOG_AT_TIMEOUT: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarInnodbFlushLogAtTimeout)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarInnodbFlushLogAtTimeout))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbFlushLogAtTimeout())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbFlushLogAtTimeout", K(ret)); + } + break; + } + case SYS_VAR_INNODB_FLUSH_LOG_AT_TRX_COMMIT: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarInnodbFlushLogAtTrxCommit)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarInnodbFlushLogAtTrxCommit))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbFlushLogAtTrxCommit())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbFlushLogAtTrxCommit", K(ret)); + } + break; + } + case SYS_VAR_INNODB_LOG_CHECKPOINT_NOW: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarInnodbLogCheckpointNow)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarInnodbLogCheckpointNow))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbLogCheckpointNow())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbLogCheckpointNow", K(ret)); + } + break; + } + case SYS_VAR_INNODB_LOG_CHECKSUMS: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarInnodbLogChecksums)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarInnodbLogChecksums))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbLogChecksums())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbLogChecksums", K(ret)); + } + break; + } + case SYS_VAR_INNODB_LOG_COMPRESSED_PAGES: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarInnodbLogCompressedPages)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarInnodbLogCompressedPages))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbLogCompressedPages())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbLogCompressedPages", K(ret)); + } + break; + } + case SYS_VAR_INNODB_LOG_WRITE_AHEAD_SIZE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarInnodbLogWriteAheadSize)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarInnodbLogWriteAheadSize))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbLogWriteAheadSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbLogWriteAheadSize", K(ret)); + } + break; + } + case SYS_VAR_INNODB_MAX_UNDO_LOG_SIZE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarInnodbMaxUndoLogSize)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarInnodbMaxUndoLogSize))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbMaxUndoLogSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbMaxUndoLogSize", K(ret)); + } + break; + } + case SYS_VAR_INNODB_ONLINE_ALTER_LOG_MAX_SIZE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarInnodbOnlineAlterLogMaxSize)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarInnodbOnlineAlterLogMaxSize))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbOnlineAlterLogMaxSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbOnlineAlterLogMaxSize", K(ret)); + } + break; + } + case SYS_VAR_INNODB_UNDO_LOG_TRUNCATE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarInnodbUndoLogTruncate)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarInnodbUndoLogTruncate))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbUndoLogTruncate())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbUndoLogTruncate", K(ret)); + } + break; + } + case SYS_VAR_INNODB_UNDO_LOGS: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarInnodbUndoLogs)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarInnodbUndoLogs))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbUndoLogs())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbUndoLogs", K(ret)); + } + break; + } + case SYS_VAR_LOG_BIN_TRUST_FUNCTION_CREATORS: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarLogBinTrustFunctionCreators)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarLogBinTrustFunctionCreators))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarLogBinTrustFunctionCreators())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarLogBinTrustFunctionCreators", K(ret)); + } + break; + } + case SYS_VAR_LOG_BIN_USE_V1_ROW_EVENTS: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarLogBinUseV1RowEvents)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarLogBinUseV1RowEvents))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarLogBinUseV1RowEvents())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarLogBinUseV1RowEvents", K(ret)); + } + break; + } + case SYS_VAR_LOG_BUILTIN_AS_IDENTIFIED_BY_PASSWORD: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarLogBuiltinAsIdentifiedByPassword)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarLogBuiltinAsIdentifiedByPassword))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarLogBuiltinAsIdentifiedByPassword())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarLogBuiltinAsIdentifiedByPassword", K(ret)); + } + break; + } + case SYS_VAR_MAX_BINLOG_CACHE_SIZE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarMaxBinlogCacheSize)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarMaxBinlogCacheSize))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMaxBinlogCacheSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMaxBinlogCacheSize", K(ret)); + } + break; + } + case SYS_VAR_MAX_BINLOG_SIZE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarMaxBinlogSize)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarMaxBinlogSize))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMaxBinlogSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMaxBinlogSize", K(ret)); + } + break; + } + case SYS_VAR_MAX_BINLOG_STMT_CACHE_SIZE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarMaxBinlogStmtCacheSize)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarMaxBinlogStmtCacheSize))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMaxBinlogStmtCacheSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMaxBinlogStmtCacheSize", K(ret)); + } + break; + } + case SYS_VAR_MAX_RELAY_LOG_SIZE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarMaxRelayLogSize)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarMaxRelayLogSize))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMaxRelayLogSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMaxRelayLogSize", K(ret)); + } + break; + } + case SYS_VAR_RELAY_LOG_INFO_REPOSITORY: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarRelayLogInfoRepository)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarRelayLogInfoRepository))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarRelayLogInfoRepository())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarRelayLogInfoRepository", K(ret)); + } + break; + } + case SYS_VAR_RELAY_LOG_PURGE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarRelayLogPurge)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarRelayLogPurge))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarRelayLogPurge())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarRelayLogPurge", K(ret)); + } + break; + } + case SYS_VAR_SYNC_BINLOG: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarSyncBinlog)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarSyncBinlog))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSyncBinlog())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSyncBinlog", K(ret)); + } + break; + } + case SYS_VAR_SYNC_RELAY_LOG: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarSyncRelayLog)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarSyncRelayLog))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSyncRelayLog())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSyncRelayLog", K(ret)); + } + break; + } + case SYS_VAR_SYNC_RELAY_LOG_INFO: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarSyncRelayLogInfo)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarSyncRelayLogInfo))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSyncRelayLogInfo())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSyncRelayLogInfo", K(ret)); + } + break; + } + case SYS_VAR_INNODB_DEADLOCK_DETECT: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarInnodbDeadlockDetect)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarInnodbDeadlockDetect))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbDeadlockDetect())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbDeadlockDetect", K(ret)); + } + break; + } + case SYS_VAR_INNODB_LOCK_WAIT_TIMEOUT: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarInnodbLockWaitTimeout)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarInnodbLockWaitTimeout))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbLockWaitTimeout())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbLockWaitTimeout", K(ret)); + } + break; + } + case SYS_VAR_INNODB_PRINT_ALL_DEADLOCKS: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarInnodbPrintAllDeadlocks)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarInnodbPrintAllDeadlocks))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbPrintAllDeadlocks())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbPrintAllDeadlocks", K(ret)); + } + break; + } + case SYS_VAR_INNODB_TABLE_LOCKS: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarInnodbTableLocks)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarInnodbTableLocks))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbTableLocks())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbTableLocks", K(ret)); + } + break; + } + case SYS_VAR_MAX_WRITE_LOCK_COUNT: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarMaxWriteLockCount)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarMaxWriteLockCount))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMaxWriteLockCount())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMaxWriteLockCount", K(ret)); + } + break; + } case SYS_VAR__OB_ENABLE_ROLE_IDS: { void *ptr = NULL; if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarObEnableRoleIds)))) { @@ -11486,6 +14200,1040 @@ int ObSysVarFactory::create_sys_var(ObIAllocator &allocator_, ObSysVarClassType } break; } + case SYS_VAR_CHARACTER_SETS_DIR: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarCharacterSetsDir)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarCharacterSetsDir))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarCharacterSetsDir())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarCharacterSetsDir", K(ret)); + } + break; + } + case SYS_VAR_DATE_FORMAT: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarDateFormat)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarDateFormat))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarDateFormat())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarDateFormat", K(ret)); + } + break; + } + case SYS_VAR_DATETIME_FORMAT: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarDatetimeFormat)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarDatetimeFormat))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarDatetimeFormat())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarDatetimeFormat", K(ret)); + } + break; + } + case SYS_VAR_DISCONNECT_ON_EXPIRED_PASSWORD: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarDisconnectOnExpiredPassword)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarDisconnectOnExpiredPassword))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarDisconnectOnExpiredPassword())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarDisconnectOnExpiredPassword", K(ret)); + } + break; + } + case SYS_VAR_EXTERNAL_USER: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarExternalUser)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarExternalUser))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarExternalUser())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarExternalUser", K(ret)); + } + break; + } + case SYS_VAR_HAVE_CRYPT: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarHaveCrypt)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarHaveCrypt))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarHaveCrypt())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarHaveCrypt", K(ret)); + } + break; + } + case SYS_VAR_HAVE_DYNAMIC_LOADING: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarHaveDynamicLoading)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarHaveDynamicLoading))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarHaveDynamicLoading())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarHaveDynamicLoading", K(ret)); + } + break; + } + case SYS_VAR_KEYRING_AWS_CONF_FILE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarKeyringAwsConfFile)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarKeyringAwsConfFile))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarKeyringAwsConfFile())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarKeyringAwsConfFile", K(ret)); + } + break; + } + case SYS_VAR_KEYRING_AWS_DATA_FILE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarKeyringAwsDataFile)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarKeyringAwsDataFile))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarKeyringAwsDataFile())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarKeyringAwsDataFile", K(ret)); + } + break; + } + case SYS_VAR_LANGUAGE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarLanguage)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarLanguage))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarLanguage())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarLanguage", K(ret)); + } + break; + } + case SYS_VAR_LC_MESSAGES_DIR: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarLcMessagesDir)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarLcMessagesDir))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarLcMessagesDir())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarLcMessagesDir", K(ret)); + } + break; + } + case SYS_VAR_LOWER_CASE_FILE_SYSTEM: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarLowerCaseFileSystem)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarLowerCaseFileSystem))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarLowerCaseFileSystem())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarLowerCaseFileSystem", K(ret)); + } + break; + } + case SYS_VAR_MAX_DIGEST_LENGTH: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarMaxDigestLength)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarMaxDigestLength))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMaxDigestLength())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMaxDigestLength", K(ret)); + } + break; + } + case SYS_VAR_NDBINFO_DATABASE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarNdbinfoDatabase)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarNdbinfoDatabase))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbinfoDatabase())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbinfoDatabase", K(ret)); + } + break; + } + case SYS_VAR_NDBINFO_TABLE_PREFIX: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarNdbinfoTablePrefix)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarNdbinfoTablePrefix))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbinfoTablePrefix())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbinfoTablePrefix", K(ret)); + } + break; + } + case SYS_VAR_NDBINFO_VERSION: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarNdbinfoVersion)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarNdbinfoVersion))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbinfoVersion())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbinfoVersion", K(ret)); + } + break; + } + case SYS_VAR_NDB_BATCH_SIZE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarNdbBatchSize)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarNdbBatchSize))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbBatchSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbBatchSize", K(ret)); + } + break; + } + case SYS_VAR_NDB_CLUSTER_CONNECTION_POOL: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarNdbClusterConnectionPool)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarNdbClusterConnectionPool))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbClusterConnectionPool())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbClusterConnectionPool", K(ret)); + } + break; + } + case SYS_VAR_NDB_CLUSTER_CONNECTION_POOL_NODEIDS: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarNdbClusterConnectionPoolNodeids)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarNdbClusterConnectionPoolNodeids))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbClusterConnectionPoolNodeids())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbClusterConnectionPoolNodeids", K(ret)); + } + break; + } + case SYS_VAR_NDB_LOG_APPLY_STATUS: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarNdbLogApplyStatus)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarNdbLogApplyStatus))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbLogApplyStatus())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbLogApplyStatus", K(ret)); + } + break; + } + case SYS_VAR_NDB_LOG_BIN: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarNdbLogBin)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarNdbLogBin))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbLogBin())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbLogBin", K(ret)); + } + break; + } + case SYS_VAR_NDB_LOG_FAIL_TERMINATE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarNdbLogFailTerminate)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarNdbLogFailTerminate))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbLogFailTerminate())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbLogFailTerminate", K(ret)); + } + break; + } + case SYS_VAR_NDB_LOG_ORIG: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarNdbLogOrig)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarNdbLogOrig))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbLogOrig())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbLogOrig", K(ret)); + } + break; + } + case SYS_VAR_NDB_LOG_TRANSACTION_ID: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarNdbLogTransactionId)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarNdbLogTransactionId))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbLogTransactionId())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbLogTransactionId", K(ret)); + } + break; + } + case SYS_VAR_NDB_OPTIMIZED_NODE_SELECTION: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarNdbOptimizedNodeSelection)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarNdbOptimizedNodeSelection))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbOptimizedNodeSelection())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbOptimizedNodeSelection", K(ret)); + } + break; + } + case SYS_VAR_NDB_SYSTEM_NAME: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarNdbSystemName)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarNdbSystemName))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbSystemName())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbSystemName", K(ret)); + } + break; + } + case SYS_VAR_NDB_USE_COPYING_ALTER_TABLE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarNdbUseCopyingAlterTable)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarNdbUseCopyingAlterTable))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbUseCopyingAlterTable())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbUseCopyingAlterTable", K(ret)); + } + break; + } + case SYS_VAR_NDB_VERSION_STRING: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarNdbVersionString)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarNdbVersionString))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbVersionString())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbVersionString", K(ret)); + } + break; + } + case SYS_VAR_NDB_WAIT_CONNECTED: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarNdbWaitConnected)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarNdbWaitConnected))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbWaitConnected())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbWaitConnected", K(ret)); + } + break; + } + case SYS_VAR_NDB_WAIT_SETUP: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarNdbWaitSetup)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarNdbWaitSetup))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbWaitSetup())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbWaitSetup", K(ret)); + } + break; + } + case SYS_VAR_PROXY_USER: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarProxyUser)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarProxyUser))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarProxyUser())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarProxyUser", K(ret)); + } + break; + } + case SYS_VAR_SHA256_PASSWORD_AUTO_GENERATE_RSA_KEYS: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarSha256PasswordAutoGenerateRsaKeys)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarSha256PasswordAutoGenerateRsaKeys))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSha256PasswordAutoGenerateRsaKeys())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSha256PasswordAutoGenerateRsaKeys", K(ret)); + } + break; + } + case SYS_VAR_SHA256_PASSWORD_PRIVATE_KEY_PATH: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarSha256PasswordPrivateKeyPath)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarSha256PasswordPrivateKeyPath))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSha256PasswordPrivateKeyPath())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSha256PasswordPrivateKeyPath", K(ret)); + } + break; + } + case SYS_VAR_SHA256_PASSWORD_PUBLIC_KEY_PATH: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarSha256PasswordPublicKeyPath)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarSha256PasswordPublicKeyPath))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSha256PasswordPublicKeyPath())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSha256PasswordPublicKeyPath", K(ret)); + } + break; + } + case SYS_VAR_SKIP_SHOW_DATABASE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarSkipShowDatabase)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarSkipShowDatabase))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSkipShowDatabase())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSkipShowDatabase", K(ret)); + } + break; + } + case SYS_VAR_PLUGIN_LOAD: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarPluginLoad)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarPluginLoad))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarPluginLoad())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarPluginLoad", K(ret)); + } + break; + } + case SYS_VAR_PLUGIN_LOAD_ADD: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarPluginLoadAdd)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarPluginLoadAdd))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarPluginLoadAdd())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarPluginLoadAdd", K(ret)); + } + break; + } + case SYS_VAR_BIG_TABLES: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarBigTables)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarBigTables))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarBigTables())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarBigTables", K(ret)); + } + break; + } + case SYS_VAR_CHECK_PROXY_USERS: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarCheckProxyUsers)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarCheckProxyUsers))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarCheckProxyUsers())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarCheckProxyUsers", K(ret)); + } + break; + } + case SYS_VAR_CONNECTION_CONTROL_FAILED_CONNECTIONS_THRESHOLD: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarConnectionControlFailedConnectionsThreshold)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarConnectionControlFailedConnectionsThreshold))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarConnectionControlFailedConnectionsThreshold())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarConnectionControlFailedConnectionsThreshold", K(ret)); + } + break; + } + case SYS_VAR_CONNECTION_CONTROL_MAX_CONNECTION_DELAY: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarConnectionControlMaxConnectionDelay)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarConnectionControlMaxConnectionDelay))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarConnectionControlMaxConnectionDelay())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarConnectionControlMaxConnectionDelay", K(ret)); + } + break; + } + case SYS_VAR_CONNECTION_CONTROL_MIN_CONNECTION_DELAY: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarConnectionControlMinConnectionDelay)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarConnectionControlMinConnectionDelay))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarConnectionControlMinConnectionDelay())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarConnectionControlMinConnectionDelay", K(ret)); + } + break; + } + case SYS_VAR_DEFAULT_WEEK_FORMAT: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarDefaultWeekFormat)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarDefaultWeekFormat))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarDefaultWeekFormat())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarDefaultWeekFormat", K(ret)); + } + break; + } + case SYS_VAR_DELAYED_INSERT_TIMEOUT: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarDelayedInsertTimeout)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarDelayedInsertTimeout))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarDelayedInsertTimeout())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarDelayedInsertTimeout", K(ret)); + } + break; + } + case SYS_VAR_DELAYED_QUEUE_SIZE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarDelayedQueueSize)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarDelayedQueueSize))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarDelayedQueueSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarDelayedQueueSize", K(ret)); + } + break; + } + case SYS_VAR_EQ_RANGE_INDEX_DIVE_LIMIT: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarEqRangeIndexDiveLimit)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarEqRangeIndexDiveLimit))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarEqRangeIndexDiveLimit())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarEqRangeIndexDiveLimit", K(ret)); + } + break; + } + case SYS_VAR_INNODB_STATS_AUTO_RECALC: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarInnodbStatsAutoRecalc)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarInnodbStatsAutoRecalc))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbStatsAutoRecalc())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbStatsAutoRecalc", K(ret)); + } + break; + } + case SYS_VAR_INNODB_STATS_INCLUDE_DELETE_MARKED: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarInnodbStatsIncludeDeleteMarked)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarInnodbStatsIncludeDeleteMarked))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbStatsIncludeDeleteMarked())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbStatsIncludeDeleteMarked", K(ret)); + } + break; + } + case SYS_VAR_INNODB_STATS_METHOD: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarInnodbStatsMethod)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarInnodbStatsMethod))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbStatsMethod())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbStatsMethod", K(ret)); + } + break; + } + case SYS_VAR_INNODB_STATS_ON_METADATA: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarInnodbStatsOnMetadata)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarInnodbStatsOnMetadata))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbStatsOnMetadata())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbStatsOnMetadata", K(ret)); + } + break; + } + case SYS_VAR_VERSION_TOKENS_SESSION: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarVersionTokensSession)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarVersionTokensSession))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarVersionTokensSession())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarVersionTokensSession", K(ret)); + } + break; + } + case SYS_VAR_INNODB_STATS_PERSISTENT_SAMPLE_PAGES: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarInnodbStatsPersistentSamplePages)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarInnodbStatsPersistentSamplePages))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbStatsPersistentSamplePages())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbStatsPersistentSamplePages", K(ret)); + } + break; + } + case SYS_VAR_INNODB_STATS_SAMPLE_PAGES: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarInnodbStatsSamplePages)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarInnodbStatsSamplePages))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbStatsSamplePages())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbStatsSamplePages", K(ret)); + } + break; + } + case SYS_VAR_INNODB_STATS_TRANSIENT_SAMPLE_PAGES: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarInnodbStatsTransientSamplePages)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarInnodbStatsTransientSamplePages))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarInnodbStatsTransientSamplePages())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarInnodbStatsTransientSamplePages", K(ret)); + } + break; + } + case SYS_VAR_KEYRING_AWS_CMK_ID: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarKeyringAwsCmkId)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarKeyringAwsCmkId))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarKeyringAwsCmkId())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarKeyringAwsCmkId", K(ret)); + } + break; + } + case SYS_VAR_KEYRING_AWS_REGION: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarKeyringAwsRegion)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarKeyringAwsRegion))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarKeyringAwsRegion())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarKeyringAwsRegion", K(ret)); + } + break; + } + case SYS_VAR_KEYRING_ENCRYPTED_FILE_DATA: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarKeyringEncryptedFileData)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarKeyringEncryptedFileData))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarKeyringEncryptedFileData())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarKeyringEncryptedFileData", K(ret)); + } + break; + } + case SYS_VAR_KEYRING_ENCRYPTED_FILE_PASSWORD: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarKeyringEncryptedFilePassword)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarKeyringEncryptedFilePassword))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarKeyringEncryptedFilePassword())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarKeyringEncryptedFilePassword", K(ret)); + } + break; + } + case SYS_VAR_KEYRING_FILE_DATA: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarKeyringFileData)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarKeyringFileData))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarKeyringFileData())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarKeyringFileData", K(ret)); + } + break; + } + case SYS_VAR_KEYRING_OKV_CONF_DIR: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarKeyringOkvConfDir)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarKeyringOkvConfDir))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarKeyringOkvConfDir())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarKeyringOkvConfDir", K(ret)); + } + break; + } + case SYS_VAR_KEYRING_OPERATIONS: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarKeyringOperations)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarKeyringOperations))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarKeyringOperations())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarKeyringOperations", K(ret)); + } + break; + } + case SYS_VAR_OPTIMIZER_SWITCH: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarOptimizerSwitch)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarOptimizerSwitch))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarOptimizerSwitch())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarOptimizerSwitch", K(ret)); + } + break; + } + case SYS_VAR_MAX_CONNECT_ERRORS: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarMaxConnectErrors)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarMaxConnectErrors))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMaxConnectErrors())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMaxConnectErrors", K(ret)); + } + break; + } + case SYS_VAR_MYSQL_FIREWALL_MODE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarMysqlFirewallMode)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarMysqlFirewallMode))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMysqlFirewallMode())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMysqlFirewallMode", K(ret)); + } + break; + } + case SYS_VAR_MYSQL_FIREWALL_TRACE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarMysqlFirewallTrace)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarMysqlFirewallTrace))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMysqlFirewallTrace())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMysqlFirewallTrace", K(ret)); + } + break; + } + case SYS_VAR_MYSQL_NATIVE_PASSWORD_PROXY_USERS: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarMysqlNativePasswordProxyUsers)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarMysqlNativePasswordProxyUsers))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMysqlNativePasswordProxyUsers())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarMysqlNativePasswordProxyUsers", K(ret)); + } + break; + } + case SYS_VAR_NET_RETRY_COUNT: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarNetRetryCount)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarNetRetryCount))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNetRetryCount())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNetRetryCount", K(ret)); + } + break; + } + case SYS_VAR_NEW: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarNew)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarNew))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNew())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNew", K(ret)); + } + break; + } + case SYS_VAR_OLD_PASSWORDS: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarOldPasswords)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarOldPasswords))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarOldPasswords())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarOldPasswords", K(ret)); + } + break; + } + case SYS_VAR_OPTIMIZER_PRUNE_LEVEL: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarOptimizerPruneLevel)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarOptimizerPruneLevel))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarOptimizerPruneLevel())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarOptimizerPruneLevel", K(ret)); + } + break; + } + case SYS_VAR_OPTIMIZER_SEARCH_DEPTH: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarOptimizerSearchDepth)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarOptimizerSearchDepth))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarOptimizerSearchDepth())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarOptimizerSearchDepth", K(ret)); + } + break; + } + case SYS_VAR_OPTIMIZER_TRACE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarOptimizerTrace)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarOptimizerTrace))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarOptimizerTrace())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarOptimizerTrace", K(ret)); + } + break; + } + case SYS_VAR_OPTIMIZER_TRACE_FEATURES: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarOptimizerTraceFeatures)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarOptimizerTraceFeatures))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarOptimizerTraceFeatures())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarOptimizerTraceFeatures", K(ret)); + } + break; + } + case SYS_VAR_OPTIMIZER_TRACE_LIMIT: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarOptimizerTraceLimit)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarOptimizerTraceLimit))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarOptimizerTraceLimit())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarOptimizerTraceLimit", K(ret)); + } + break; + } + case SYS_VAR_OPTIMIZER_TRACE_MAX_MEM_SIZE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarOptimizerTraceMaxMemSize)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarOptimizerTraceMaxMemSize))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarOptimizerTraceMaxMemSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarOptimizerTraceMaxMemSize", K(ret)); + } + break; + } + case SYS_VAR_OPTIMIZER_TRACE_OFFSET: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarOptimizerTraceOffset)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarOptimizerTraceOffset))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarOptimizerTraceOffset())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarOptimizerTraceOffset", K(ret)); + } + break; + } + case SYS_VAR_PARSER_MAX_MEM_SIZE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarParserMaxMemSize)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarParserMaxMemSize))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarParserMaxMemSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarParserMaxMemSize", K(ret)); + } + break; + } + case SYS_VAR_RAND_SEED1: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarRandSeed1)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarRandSeed1))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarRandSeed1())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarRandSeed1", K(ret)); + } + break; + } + case SYS_VAR_RAND_SEED2: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarRandSeed2)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarRandSeed2))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarRandSeed2())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarRandSeed2", K(ret)); + } + break; + } + case SYS_VAR_RANGE_ALLOC_BLOCK_SIZE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarRangeAllocBlockSize)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarRangeAllocBlockSize))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarRangeAllocBlockSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarRangeAllocBlockSize", K(ret)); + } + break; + } + case SYS_VAR_RANGE_OPTIMIZER_MAX_MEM_SIZE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarRangeOptimizerMaxMemSize)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarRangeOptimizerMaxMemSize))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarRangeOptimizerMaxMemSize())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarRangeOptimizerMaxMemSize", K(ret)); + } + break; + } + case SYS_VAR_REWRITER_ENABLED: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarRewriterEnabled)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarRewriterEnabled))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarRewriterEnabled())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarRewriterEnabled", K(ret)); + } + break; + } + case SYS_VAR_REWRITER_VERBOSE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarRewriterVerbose)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarRewriterVerbose))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarRewriterVerbose())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarRewriterVerbose", K(ret)); + } + break; + } + case SYS_VAR_SECURE_AUTH: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarSecureAuth)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarSecureAuth))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSecureAuth())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSecureAuth", K(ret)); + } + break; + } + case SYS_VAR_SHA256_PASSWORD_PROXY_USERS: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarSha256PasswordProxyUsers)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarSha256PasswordProxyUsers))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSha256PasswordProxyUsers())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSha256PasswordProxyUsers", K(ret)); + } + break; + } + case SYS_VAR_SHOW_COMPATIBILITY_56: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarShowCompatibility56)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarShowCompatibility56))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarShowCompatibility56())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarShowCompatibility56", K(ret)); + } + break; + } + case SYS_VAR_SHOW_CREATE_TABLE_VERBOSITY: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarShowCreateTableVerbosity)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarShowCreateTableVerbosity))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarShowCreateTableVerbosity())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarShowCreateTableVerbosity", K(ret)); + } + break; + } + case SYS_VAR_SHOW_OLD_TEMPORALS: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarShowOldTemporals)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarShowOldTemporals))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarShowOldTemporals())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarShowOldTemporals", K(ret)); + } + break; + } + case SYS_VAR_SQL_BIG_SELECTS: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarSqlBigSelects)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarSqlBigSelects))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarSqlBigSelects())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarSqlBigSelects", K(ret)); + } + break; + } + case SYS_VAR_UPDATABLE_VIEWS_WITH_LIMIT: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarUpdatableViewsWithLimit)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarUpdatableViewsWithLimit))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarUpdatableViewsWithLimit())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarUpdatableViewsWithLimit", K(ret)); + } + break; + } + case SYS_VAR_VALIDATE_PASSWORD_DICTIONARY_FILE: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarValidatePasswordDictionaryFile)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarValidatePasswordDictionaryFile))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarValidatePasswordDictionaryFile())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarValidatePasswordDictionaryFile", K(ret)); + } + break; + } + case SYS_VAR_DELAYED_INSERT_LIMIT: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarDelayedInsertLimit)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarDelayedInsertLimit))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarDelayedInsertLimit())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarDelayedInsertLimit", K(ret)); + } + break; + } + case SYS_VAR_NDB_VERSION: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarNdbVersion)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarNdbVersion))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarNdbVersion())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarNdbVersion", K(ret)); + } + break; + } + case SYS_VAR_AUTO_GENERATE_CERTS: { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarAutoGenerateCerts)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarAutoGenerateCerts))); + } else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarAutoGenerateCerts())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to new ObSysVarAutoGenerateCerts", K(ret)); + } + break; + } default: { ret = OB_ERR_UNEXPECTED; @@ -11501,7 +15249,9 @@ int ObSysVarFactory::create_sys_var(ObSysVarClassType sys_var_id, ObBasicSysVar int ret = OB_SUCCESS; int64_t store_idx = -1; ObBasicSysVar *sys_var_ptr = NULL; - if (OB_FAIL(calc_sys_var_store_idx(sys_var_id, store_idx))) { + if (OB_FAIL(try_init_store_mem())) { + LOG_WARN("fail to init", K(ret)); + } else if (OB_FAIL(calc_sys_var_store_idx(sys_var_id, store_idx))) { LOG_WARN("fail to calc sys var store idx", K(ret), K(sys_var_id)); } else if (store_idx < 0 || store_idx >= ALL_SYS_VARS_COUNT) { ret = OB_ERR_UNEXPECTED; diff --git a/src/share/system_variable/ob_system_variable_factory.h b/src/share/system_variable/ob_system_variable_factory.h index cc6457c6e..f4a7c15d3 100644 --- a/src/share/system_variable/ob_system_variable_factory.h +++ b/src/share/system_variable/ob_system_variable_factory.h @@ -1101,7 +1101,7 @@ class ObSysVarBlockEncryptionMode : public ObEnumSysVar public: const static char * BLOCK_ENCRYPTION_MODE_NAMES[]; public: - ObSysVarBlockEncryptionMode() : ObEnumSysVar(BLOCK_ENCRYPTION_MODE_NAMES, NULL, NULL, NULL, NULL, NULL) {} + ObSysVarBlockEncryptionMode() : ObEnumSysVar(BLOCK_ENCRYPTION_MODE_NAMES, ObSysVarOnCheckFuncs::check_and_convert_block_encryption_mode, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_BLOCK_ENCRYPTION_MODE; } inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(151); } }; @@ -3019,7 +3019,7 @@ public: class ObSysVarDefaultCollationForUtf8mb4 : public ObCharsetSysVar { public: - ObSysVarDefaultCollationForUtf8mb4() : ObCharsetSysVar(NULL, NULL, ObSysVarToObjFuncs::to_obj_collation, ObSysVarToStrFuncs::to_str_collation, ObSysVarGetMetaTypeFuncs::get_meta_type_varchar) {} + ObSysVarDefaultCollationForUtf8mb4() : ObCharsetSysVar(ObSysVarOnCheckFuncs::check_default_value_for_utf8mb4, NULL, ObSysVarToObjFuncs::to_obj_collation, ObSysVarToStrFuncs::to_str_collation, ObSysVarGetMetaTypeFuncs::get_meta_type_varchar) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_DEFAULT_COLLATION_FOR_UTF8MB4; } inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(409); } }; @@ -3030,131 +3030,541 @@ public: inline virtual ObSysVarClassType get_type() const { return SYS_VAR__ENABLE_OLD_CHARSET_AGGREGATION; } inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(410); } }; +class ObSysVarInsertId : public ObIntSysVar +{ +public: + ObSysVarInsertId() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INSERT_ID; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(411); } +}; +class ObSysVarJoinBufferSize : public ObIntSysVar +{ +public: + ObSysVarJoinBufferSize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_JOIN_BUFFER_SIZE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(412); } +}; +class ObSysVarMaxJoinSize : public ObIntSysVar +{ +public: + ObSysVarMaxJoinSize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_MAX_JOIN_SIZE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(413); } +}; +class ObSysVarMaxLengthForSortData : public ObIntSysVar +{ +public: + ObSysVarMaxLengthForSortData() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_MAX_LENGTH_FOR_SORT_DATA; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(414); } +}; +class ObSysVarMaxPreparedStmtCount : public ObIntSysVar +{ +public: + ObSysVarMaxPreparedStmtCount() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_MAX_PREPARED_STMT_COUNT; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(415); } +}; +class ObSysVarMaxSortLength : public ObIntSysVar +{ +public: + ObSysVarMaxSortLength() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_MAX_SORT_LENGTH; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(416); } +}; +class ObSysVarMinExaminedRowLimit : public ObIntSysVar +{ +public: + ObSysVarMinExaminedRowLimit() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_MIN_EXAMINED_ROW_LIMIT; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(417); } +}; +class ObSysVarMultiRangeCount : public ObIntSysVar +{ +public: + ObSysVarMultiRangeCount() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_MULTI_RANGE_COUNT; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(418); } +}; +class ObSysVarMysqlxConnectTimeout : public ObIntSysVar +{ +public: + ObSysVarMysqlxConnectTimeout() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_MYSQLX_CONNECT_TIMEOUT; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(419); } +}; +class ObSysVarMysqlxIdleWorkerThreadTimeout : public ObIntSysVar +{ +public: + ObSysVarMysqlxIdleWorkerThreadTimeout() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_MYSQLX_IDLE_WORKER_THREAD_TIMEOUT; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(420); } +}; +class ObSysVarMysqlxMaxAllowedPacket : public ObIntSysVar +{ +public: + ObSysVarMysqlxMaxAllowedPacket() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_MYSQLX_MAX_ALLOWED_PACKET; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(421); } +}; +class ObSysVarMysqlxMaxConnections : public ObIntSysVar +{ +public: + ObSysVarMysqlxMaxConnections() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_MYSQLX_MAX_CONNECTIONS; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(422); } +}; +class ObSysVarMysqlxMinWorkerThreads : public ObIntSysVar +{ +public: + ObSysVarMysqlxMinWorkerThreads() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_MYSQLX_MIN_WORKER_THREADS; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(423); } +}; +class ObSysVarPerformanceSchemaShowProcesslist : public ObBoolSysVar +{ +public: + ObSysVarPerformanceSchemaShowProcesslist() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_PERFORMANCE_SCHEMA_SHOW_PROCESSLIST; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(424); } +}; +class ObSysVarQueryAllocBlockSize : public ObIntSysVar +{ +public: + ObSysVarQueryAllocBlockSize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_QUERY_ALLOC_BLOCK_SIZE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(425); } +}; +class ObSysVarQueryPreallocSize : public ObIntSysVar +{ +public: + ObSysVarQueryPreallocSize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_QUERY_PREALLOC_SIZE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(426); } +}; +class ObSysVarSlowQueryLog : public ObIntSysVar +{ +public: + ObSysVarSlowQueryLog() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_SLOW_QUERY_LOG; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(427); } +}; +class ObSysVarSlowQueryLogFile : public ObVarcharSysVar +{ +public: + ObSysVarSlowQueryLogFile() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_SLOW_QUERY_LOG_FILE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(428); } +}; +class ObSysVarSortBufferSize : public ObIntSysVar +{ +public: + ObSysVarSortBufferSize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_SORT_BUFFER_SIZE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(429); } +}; +class ObSysVarSqlBufferResult : public ObIntSysVar +{ +public: + ObSysVarSqlBufferResult() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_SQL_BUFFER_RESULT; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(430); } +}; +class ObSysVarBinlogCacheSize : public ObIntSysVar +{ +public: + ObSysVarBinlogCacheSize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_BINLOG_CACHE_SIZE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(431); } +}; +class ObSysVarBinlogDirectNonTransactionalUpdates : public ObIntSysVar +{ +public: + ObSysVarBinlogDirectNonTransactionalUpdates() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_BINLOG_DIRECT_NON_TRANSACTIONAL_UPDATES; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(432); } +}; +class ObSysVarBinlogErrorAction : public ObEnumSysVar +{ +public: + const static char * BINLOG_ERROR_ACTION_NAMES[]; +public: + ObSysVarBinlogErrorAction() : ObEnumSysVar(BINLOG_ERROR_ACTION_NAMES, NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_BINLOG_ERROR_ACTION; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(433); } +}; +class ObSysVarBinlogGroupCommitSyncDelay : public ObIntSysVar +{ +public: + ObSysVarBinlogGroupCommitSyncDelay() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_BINLOG_GROUP_COMMIT_SYNC_DELAY; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(434); } +}; +class ObSysVarBinlogGroupCommitSyncNoDelayCount : public ObIntSysVar +{ +public: + ObSysVarBinlogGroupCommitSyncNoDelayCount() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_BINLOG_GROUP_COMMIT_SYNC_NO_DELAY_COUNT; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(435); } +}; +class ObSysVarBinlogMaxFlushQueueTime : public ObIntSysVar +{ +public: + ObSysVarBinlogMaxFlushQueueTime() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_BINLOG_MAX_FLUSH_QUEUE_TIME; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(436); } +}; +class ObSysVarBinlogOrderCommits : public ObIntSysVar +{ +public: + ObSysVarBinlogOrderCommits() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_BINLOG_ORDER_COMMITS; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(437); } +}; +class ObSysVarBinlogStmtCacheSize : public ObIntSysVar +{ +public: + ObSysVarBinlogStmtCacheSize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_BINLOG_STMT_CACHE_SIZE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(438); } +}; +class ObSysVarBinlogTransactionDependencyHistorySize : public ObIntSysVar +{ +public: + ObSysVarBinlogTransactionDependencyHistorySize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_BINLOG_TRANSACTION_DEPENDENCY_HISTORY_SIZE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(439); } +}; +class ObSysVarBinlogTransactionDependencyTracking : public ObEnumSysVar +{ +public: + const static char * BINLOG_TRANSACTION_DEPENDENCY_TRACKING_NAMES[]; +public: + ObSysVarBinlogTransactionDependencyTracking() : ObEnumSysVar(BINLOG_TRANSACTION_DEPENDENCY_TRACKING_NAMES, NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_BINLOG_TRANSACTION_DEPENDENCY_TRACKING; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(440); } +}; +class ObSysVarExpireLogsDays : public ObIntSysVar +{ +public: + ObSysVarExpireLogsDays() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_EXPIRE_LOGS_DAYS; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(441); } +}; +class ObSysVarInnodbFlushLogAtTimeout : public ObIntSysVar +{ +public: + ObSysVarInnodbFlushLogAtTimeout() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_FLUSH_LOG_AT_TIMEOUT; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(442); } +}; +class ObSysVarInnodbFlushLogAtTrxCommit : public ObIntSysVar +{ +public: + ObSysVarInnodbFlushLogAtTrxCommit() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_FLUSH_LOG_AT_TRX_COMMIT; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(443); } +}; +class ObSysVarInnodbLogCheckpointNow : public ObBoolSysVar +{ +public: + ObSysVarInnodbLogCheckpointNow() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_LOG_CHECKPOINT_NOW; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(444); } +}; +class ObSysVarInnodbLogChecksums : public ObIntSysVar +{ +public: + ObSysVarInnodbLogChecksums() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_LOG_CHECKSUMS; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(445); } +}; +class ObSysVarInnodbLogCompressedPages : public ObIntSysVar +{ +public: + ObSysVarInnodbLogCompressedPages() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_LOG_COMPRESSED_PAGES; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(446); } +}; +class ObSysVarInnodbLogWriteAheadSize : public ObIntSysVar +{ +public: + ObSysVarInnodbLogWriteAheadSize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_LOG_WRITE_AHEAD_SIZE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(447); } +}; +class ObSysVarInnodbMaxUndoLogSize : public ObIntSysVar +{ +public: + ObSysVarInnodbMaxUndoLogSize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_MAX_UNDO_LOG_SIZE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(448); } +}; +class ObSysVarInnodbOnlineAlterLogMaxSize : public ObIntSysVar +{ +public: + ObSysVarInnodbOnlineAlterLogMaxSize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_ONLINE_ALTER_LOG_MAX_SIZE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(449); } +}; +class ObSysVarInnodbUndoLogTruncate : public ObIntSysVar +{ +public: + ObSysVarInnodbUndoLogTruncate() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_UNDO_LOG_TRUNCATE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(450); } +}; +class ObSysVarInnodbUndoLogs : public ObIntSysVar +{ +public: + ObSysVarInnodbUndoLogs() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_UNDO_LOGS; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(451); } +}; +class ObSysVarLogBinTrustFunctionCreators : public ObIntSysVar +{ +public: + ObSysVarLogBinTrustFunctionCreators() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_LOG_BIN_TRUST_FUNCTION_CREATORS; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(452); } +}; +class ObSysVarLogBinUseV1RowEvents : public ObIntSysVar +{ +public: + ObSysVarLogBinUseV1RowEvents() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_LOG_BIN_USE_V1_ROW_EVENTS; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(453); } +}; +class ObSysVarLogBuiltinAsIdentifiedByPassword : public ObIntSysVar +{ +public: + ObSysVarLogBuiltinAsIdentifiedByPassword() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_LOG_BUILTIN_AS_IDENTIFIED_BY_PASSWORD; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(454); } +}; +class ObSysVarMaxBinlogCacheSize : public ObIntSysVar +{ +public: + ObSysVarMaxBinlogCacheSize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_MAX_BINLOG_CACHE_SIZE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(455); } +}; +class ObSysVarMaxBinlogSize : public ObIntSysVar +{ +public: + ObSysVarMaxBinlogSize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_MAX_BINLOG_SIZE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(456); } +}; +class ObSysVarMaxBinlogStmtCacheSize : public ObIntSysVar +{ +public: + ObSysVarMaxBinlogStmtCacheSize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_MAX_BINLOG_STMT_CACHE_SIZE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(457); } +}; +class ObSysVarMaxRelayLogSize : public ObIntSysVar +{ +public: + ObSysVarMaxRelayLogSize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_MAX_RELAY_LOG_SIZE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(458); } +}; +class ObSysVarRelayLogInfoRepository : public ObVarcharSysVar +{ +public: + ObSysVarRelayLogInfoRepository() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_RELAY_LOG_INFO_REPOSITORY; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(459); } +}; +class ObSysVarRelayLogPurge : public ObIntSysVar +{ +public: + ObSysVarRelayLogPurge() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_RELAY_LOG_PURGE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(460); } +}; +class ObSysVarSyncBinlog : public ObIntSysVar +{ +public: + ObSysVarSyncBinlog() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_SYNC_BINLOG; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(461); } +}; +class ObSysVarSyncRelayLog : public ObIntSysVar +{ +public: + ObSysVarSyncRelayLog() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_SYNC_RELAY_LOG; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(462); } +}; +class ObSysVarSyncRelayLogInfo : public ObIntSysVar +{ +public: + ObSysVarSyncRelayLogInfo() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_SYNC_RELAY_LOG_INFO; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(463); } +}; +class ObSysVarInnodbDeadlockDetect : public ObIntSysVar +{ +public: + ObSysVarInnodbDeadlockDetect() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_DEADLOCK_DETECT; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(464); } +}; +class ObSysVarInnodbLockWaitTimeout : public ObIntSysVar +{ +public: + ObSysVarInnodbLockWaitTimeout() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_LOCK_WAIT_TIMEOUT; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(465); } +}; +class ObSysVarInnodbPrintAllDeadlocks : public ObIntSysVar +{ +public: + ObSysVarInnodbPrintAllDeadlocks() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_PRINT_ALL_DEADLOCKS; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(466); } +}; +class ObSysVarInnodbTableLocks : public ObIntSysVar +{ +public: + ObSysVarInnodbTableLocks() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_TABLE_LOCKS; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(467); } +}; +class ObSysVarMaxWriteLockCount : public ObIntSysVar +{ +public: + ObSysVarMaxWriteLockCount() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_MAX_WRITE_LOCK_COUNT; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(468); } +}; class ObSysVarObEnableRoleIds : public ObVarcharSysVar { public: ObSysVarObEnableRoleIds() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR__OB_ENABLE_ROLE_IDS; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(411); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(469); } }; class ObSysVarInnodbReadOnly : public ObBoolSysVar { public: ObSysVarInnodbReadOnly() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_READ_ONLY; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(412); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(470); } }; class ObSysVarInnodbApiDisableRowlock : public ObBoolSysVar { public: ObSysVarInnodbApiDisableRowlock() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_API_DISABLE_ROWLOCK; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(413); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(471); } }; class ObSysVarInnodbAutoincLockMode : public ObIntSysVar { public: ObSysVarInnodbAutoincLockMode() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_AUTOINC_LOCK_MODE; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(414); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(472); } }; class ObSysVarSkipExternalLocking : public ObBoolSysVar { public: ObSysVarSkipExternalLocking() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_SKIP_EXTERNAL_LOCKING; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(415); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(473); } }; class ObSysVarSuperReadOnly : public ObBoolSysVar { public: ObSysVarSuperReadOnly() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_SUPER_READ_ONLY; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(416); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(474); } }; class ObSysVarPlsqlOptimizeLevel : public ObIntSysVar { public: ObSysVarPlsqlOptimizeLevel() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_PLSQL_OPTIMIZE_LEVEL; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(417); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(475); } }; class ObSysVarFtStopwordFile : public ObVarcharSysVar { public: ObSysVarFtStopwordFile() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_FT_STOPWORD_FILE; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(418); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(476); } }; class ObSysVarInnodbFtCacheSize : public ObIntSysVar { public: ObSysVarInnodbFtCacheSize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_FT_CACHE_SIZE; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(419); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(477); } }; class ObSysVarInnodbFtSortPllDegree : public ObIntSysVar { public: ObSysVarInnodbFtSortPllDegree() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_FT_SORT_PLL_DEGREE; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(420); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(478); } }; class ObSysVarInnodbFtTotalCacheSize : public ObIntSysVar { public: ObSysVarInnodbFtTotalCacheSize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_FT_TOTAL_CACHE_SIZE; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(421); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(479); } }; class ObSysVarMecabRcFile : public ObVarcharSysVar { public: ObSysVarMecabRcFile() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_MECAB_RC_FILE; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(422); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(480); } }; class ObSysVarMetadataLocksCacheSize : public ObIntSysVar { public: ObSysVarMetadataLocksCacheSize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_METADATA_LOCKS_CACHE_SIZE; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(423); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(481); } }; class ObSysVarMetadataLocksHashInstances : public ObIntSysVar { public: ObSysVarMetadataLocksHashInstances() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_METADATA_LOCKS_HASH_INSTANCES; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(424); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(482); } }; class ObSysVarInnodbTempDataFilePath : public ObVarcharSysVar { public: ObSysVarInnodbTempDataFilePath() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_TEMP_DATA_FILE_PATH; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(425); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(483); } }; class ObSysVarInnodbDataFilePath : public ObVarcharSysVar { public: ObSysVarInnodbDataFilePath() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_DATA_FILE_PATH; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(426); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(484); } }; class ObSysVarInnodbDataHomeDir : public ObVarcharSysVar { public: ObSysVarInnodbDataHomeDir() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_DATA_HOME_DIR; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(427); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(485); } }; class ObSysVarAvoidTemporalUpgrade : public ObBoolSysVar { public: ObSysVarAvoidTemporalUpgrade() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_AVOID_TEMPORAL_UPGRADE; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(428); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(486); } }; class ObSysVarDefaultTmpStorageEngine : public ObEnumSysVar { @@ -3163,63 +3573,63 @@ public: public: ObSysVarDefaultTmpStorageEngine() : ObEnumSysVar(DEFAULT_TMP_STORAGE_ENGINE_NAMES, NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_DEFAULT_TMP_STORAGE_ENGINE; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(429); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(487); } }; class ObSysVarInnodbFtEnableDiagPrint : public ObBoolSysVar { public: ObSysVarInnodbFtEnableDiagPrint() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_FT_ENABLE_DIAG_PRINT; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(430); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(488); } }; class ObSysVarInnodbFtNumWordOptimize : public ObIntSysVar { public: ObSysVarInnodbFtNumWordOptimize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_FT_NUM_WORD_OPTIMIZE; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(431); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(489); } }; class ObSysVarInnodbFtResultCacheLimit : public ObIntSysVar { public: ObSysVarInnodbFtResultCacheLimit() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_FT_RESULT_CACHE_LIMIT; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(432); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(490); } }; class ObSysVarInnodbFtServerStopwordTable : public ObVarcharSysVar { public: ObSysVarInnodbFtServerStopwordTable() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_FT_SERVER_STOPWORD_TABLE; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(433); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(491); } }; class ObSysVarInnodbOptimizeFulltextOnly : public ObBoolSysVar { public: ObSysVarInnodbOptimizeFulltextOnly() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_OPTIMIZE_FULLTEXT_ONLY; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(434); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(492); } }; class ObSysVarMaxTmpTables : public ObIntSysVar { public: ObSysVarMaxTmpTables() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_MAX_TMP_TABLES; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(435); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(493); } }; class ObSysVarInnodbTmpdir : public ObVarcharSysVar { public: ObSysVarInnodbTmpdir() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_TMPDIR; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(436); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(494); } }; class ObSysVarGroupReplicationGroupSeeds : public ObVarcharSysVar { public: ObSysVarGroupReplicationGroupSeeds() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_GROUP_REPLICATION_GROUP_SEEDS; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(437); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(495); } }; class ObSysVarSlaveRowsSearchAlgorithms : public ObEnumSysVar { @@ -3228,7 +3638,7 @@ public: public: ObSysVarSlaveRowsSearchAlgorithms() : ObEnumSysVar(SLAVE_ROWS_SEARCH_ALGORITHMS_NAMES, NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_SLAVE_ROWS_SEARCH_ALGORITHMS; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(438); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(496); } }; class ObSysVarSlaveTypeConversions : public ObEnumSysVar { @@ -3237,7 +3647,7 @@ public: public: ObSysVarSlaveTypeConversions() : ObEnumSysVar(SLAVE_TYPE_CONVERSIONS_NAMES, NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_SLAVE_TYPE_CONVERSIONS; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(439); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(497); } }; class ObSysVarDelayKeyWrite : public ObEnumSysVar { @@ -3246,7 +3656,7 @@ public: public: ObSysVarDelayKeyWrite() : ObEnumSysVar(DELAY_KEY_WRITE_NAMES, NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_DELAY_KEY_WRITE; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(440); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(498); } }; class ObSysVarInnodbLargePrefix : public ObEnumSysVar { @@ -3255,35 +3665,35 @@ public: public: ObSysVarInnodbLargePrefix() : ObEnumSysVar(INNODB_LARGE_PREFIX_NAMES, NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_LARGE_PREFIX; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(441); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(499); } }; class ObSysVarKeyBufferSize : public ObIntSysVar { public: ObSysVarKeyBufferSize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_KEY_BUFFER_SIZE; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(442); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(500); } }; class ObSysVarKeyCacheAgeThreshold : public ObIntSysVar { public: ObSysVarKeyCacheAgeThreshold() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_KEY_CACHE_AGE_THRESHOLD; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(443); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(501); } }; class ObSysVarKeyCacheDivisionLimit : public ObIntSysVar { public: ObSysVarKeyCacheDivisionLimit() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_KEY_CACHE_DIVISION_LIMIT; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(444); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(502); } }; class ObSysVarMaxSeeksForKey : public ObIntSysVar { public: ObSysVarMaxSeeksForKey() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_MAX_SEEKS_FOR_KEY; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(445); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(503); } }; class ObSysVarOldAlterTable : public ObEnumSysVar { @@ -3292,28 +3702,28 @@ public: public: ObSysVarOldAlterTable() : ObEnumSysVar(OLD_ALTER_TABLE_NAMES, NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_OLD_ALTER_TABLE; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(446); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(504); } }; class ObSysVarTableDefinitionCache : public ObIntSysVar { public: ObSysVarTableDefinitionCache() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_TABLE_DEFINITION_CACHE; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(447); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(505); } }; class ObSysVarInnodbSortBufferSize : public ObIntSysVar { public: ObSysVarInnodbSortBufferSize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_SORT_BUFFER_SIZE; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(448); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(506); } }; class ObSysVarKeyCacheBlockSize : public ObIntSysVar { public: ObSysVarKeyCacheBlockSize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_KEY_CACHE_BLOCK_SIZE; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(449); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(507); } }; class ObSysVarObKvMode : public ObEnumSysVar { @@ -3322,14 +3732,680 @@ public: public: ObSysVarObKvMode() : ObEnumSysVar(OB_KV_MODE_NAMES, NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_OB_KV_MODE; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(450); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(508); } }; class ObSysVarObEnableParameterAnonymousBlock : public ObBoolSysVar { public: ObSysVarObEnableParameterAnonymousBlock() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} inline virtual ObSysVarClassType get_type() const { return SYS_VAR_OB_ENABLE_PARAMETER_ANONYMOUS_BLOCK; } - inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(451); } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(509); } +}; +class ObSysVarCharacterSetsDir : public ObVarcharSysVar +{ +public: + ObSysVarCharacterSetsDir() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_CHARACTER_SETS_DIR; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(510); } +}; +class ObSysVarDateFormat : public ObVarcharSysVar +{ +public: + ObSysVarDateFormat() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_DATE_FORMAT; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(511); } +}; +class ObSysVarDatetimeFormat : public ObVarcharSysVar +{ +public: + ObSysVarDatetimeFormat() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_DATETIME_FORMAT; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(512); } +}; +class ObSysVarDisconnectOnExpiredPassword : public ObBoolSysVar +{ +public: + ObSysVarDisconnectOnExpiredPassword() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_DISCONNECT_ON_EXPIRED_PASSWORD; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(513); } +}; +class ObSysVarExternalUser : public ObVarcharSysVar +{ +public: + ObSysVarExternalUser() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_EXTERNAL_USER; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(514); } +}; +class ObSysVarHaveCrypt : public ObVarcharSysVar +{ +public: + ObSysVarHaveCrypt() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_HAVE_CRYPT; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(515); } +}; +class ObSysVarHaveDynamicLoading : public ObVarcharSysVar +{ +public: + ObSysVarHaveDynamicLoading() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_HAVE_DYNAMIC_LOADING; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(516); } +}; +class ObSysVarKeyringAwsConfFile : public ObVarcharSysVar +{ +public: + ObSysVarKeyringAwsConfFile() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_KEYRING_AWS_CONF_FILE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(517); } +}; +class ObSysVarKeyringAwsDataFile : public ObVarcharSysVar +{ +public: + ObSysVarKeyringAwsDataFile() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_KEYRING_AWS_DATA_FILE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(518); } +}; +class ObSysVarLanguage : public ObVarcharSysVar +{ +public: + ObSysVarLanguage() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_LANGUAGE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(519); } +}; +class ObSysVarLcMessagesDir : public ObVarcharSysVar +{ +public: + ObSysVarLcMessagesDir() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_LC_MESSAGES_DIR; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(520); } +}; +class ObSysVarLowerCaseFileSystem : public ObBoolSysVar +{ +public: + ObSysVarLowerCaseFileSystem() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_LOWER_CASE_FILE_SYSTEM; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(521); } +}; +class ObSysVarMaxDigestLength : public ObIntSysVar +{ +public: + ObSysVarMaxDigestLength() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_MAX_DIGEST_LENGTH; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(522); } +}; +class ObSysVarNdbinfoDatabase : public ObVarcharSysVar +{ +public: + ObSysVarNdbinfoDatabase() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_NDBINFO_DATABASE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(523); } +}; +class ObSysVarNdbinfoTablePrefix : public ObVarcharSysVar +{ +public: + ObSysVarNdbinfoTablePrefix() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_NDBINFO_TABLE_PREFIX; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(524); } +}; +class ObSysVarNdbinfoVersion : public ObVarcharSysVar +{ +public: + ObSysVarNdbinfoVersion() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_NDBINFO_VERSION; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(525); } +}; +class ObSysVarNdbBatchSize : public ObIntSysVar +{ +public: + ObSysVarNdbBatchSize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_NDB_BATCH_SIZE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(526); } +}; +class ObSysVarNdbClusterConnectionPool : public ObIntSysVar +{ +public: + ObSysVarNdbClusterConnectionPool() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_NDB_CLUSTER_CONNECTION_POOL; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(527); } +}; +class ObSysVarNdbClusterConnectionPoolNodeids : public ObVarcharSysVar +{ +public: + ObSysVarNdbClusterConnectionPoolNodeids() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_NDB_CLUSTER_CONNECTION_POOL_NODEIDS; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(528); } +}; +class ObSysVarNdbLogApplyStatus : public ObBoolSysVar +{ +public: + ObSysVarNdbLogApplyStatus() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_NDB_LOG_APPLY_STATUS; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(529); } +}; +class ObSysVarNdbLogBin : public ObBoolSysVar +{ +public: + ObSysVarNdbLogBin() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_NDB_LOG_BIN; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(530); } +}; +class ObSysVarNdbLogFailTerminate : public ObBoolSysVar +{ +public: + ObSysVarNdbLogFailTerminate() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_NDB_LOG_FAIL_TERMINATE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(531); } +}; +class ObSysVarNdbLogOrig : public ObBoolSysVar +{ +public: + ObSysVarNdbLogOrig() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_NDB_LOG_ORIG; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(532); } +}; +class ObSysVarNdbLogTransactionId : public ObBoolSysVar +{ +public: + ObSysVarNdbLogTransactionId() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_NDB_LOG_TRANSACTION_ID; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(533); } +}; +class ObSysVarNdbOptimizedNodeSelection : public ObIntSysVar +{ +public: + ObSysVarNdbOptimizedNodeSelection() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_NDB_OPTIMIZED_NODE_SELECTION; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(534); } +}; +class ObSysVarNdbSystemName : public ObVarcharSysVar +{ +public: + ObSysVarNdbSystemName() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_NDB_SYSTEM_NAME; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(535); } +}; +class ObSysVarNdbUseCopyingAlterTable : public ObBoolSysVar +{ +public: + ObSysVarNdbUseCopyingAlterTable() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_NDB_USE_COPYING_ALTER_TABLE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(536); } +}; +class ObSysVarNdbVersionString : public ObVarcharSysVar +{ +public: + ObSysVarNdbVersionString() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_NDB_VERSION_STRING; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(537); } +}; +class ObSysVarNdbWaitConnected : public ObIntSysVar +{ +public: + ObSysVarNdbWaitConnected() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_NDB_WAIT_CONNECTED; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(538); } +}; +class ObSysVarNdbWaitSetup : public ObIntSysVar +{ +public: + ObSysVarNdbWaitSetup() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_NDB_WAIT_SETUP; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(539); } +}; +class ObSysVarProxyUser : public ObVarcharSysVar +{ +public: + ObSysVarProxyUser() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_PROXY_USER; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(540); } +}; +class ObSysVarSha256PasswordAutoGenerateRsaKeys : public ObBoolSysVar +{ +public: + ObSysVarSha256PasswordAutoGenerateRsaKeys() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_SHA256_PASSWORD_AUTO_GENERATE_RSA_KEYS; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(541); } +}; +class ObSysVarSha256PasswordPrivateKeyPath : public ObVarcharSysVar +{ +public: + ObSysVarSha256PasswordPrivateKeyPath() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_SHA256_PASSWORD_PRIVATE_KEY_PATH; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(542); } +}; +class ObSysVarSha256PasswordPublicKeyPath : public ObVarcharSysVar +{ +public: + ObSysVarSha256PasswordPublicKeyPath() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_SHA256_PASSWORD_PUBLIC_KEY_PATH; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(543); } +}; +class ObSysVarSkipShowDatabase : public ObVarcharSysVar +{ +public: + ObSysVarSkipShowDatabase() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_SKIP_SHOW_DATABASE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(544); } +}; +class ObSysVarPluginLoad : public ObVarcharSysVar +{ +public: + ObSysVarPluginLoad() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_PLUGIN_LOAD; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(545); } +}; +class ObSysVarPluginLoadAdd : public ObVarcharSysVar +{ +public: + ObSysVarPluginLoadAdd() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_PLUGIN_LOAD_ADD; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(546); } +}; +class ObSysVarBigTables : public ObBoolSysVar +{ +public: + ObSysVarBigTables() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_BIG_TABLES; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(547); } +}; +class ObSysVarCheckProxyUsers : public ObBoolSysVar +{ +public: + ObSysVarCheckProxyUsers() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_CHECK_PROXY_USERS; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(548); } +}; +class ObSysVarConnectionControlFailedConnectionsThreshold : public ObIntSysVar +{ +public: + ObSysVarConnectionControlFailedConnectionsThreshold() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_CONNECTION_CONTROL_FAILED_CONNECTIONS_THRESHOLD; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(549); } +}; +class ObSysVarConnectionControlMaxConnectionDelay : public ObIntSysVar +{ +public: + ObSysVarConnectionControlMaxConnectionDelay() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_CONNECTION_CONTROL_MAX_CONNECTION_DELAY; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(550); } +}; +class ObSysVarConnectionControlMinConnectionDelay : public ObIntSysVar +{ +public: + ObSysVarConnectionControlMinConnectionDelay() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_CONNECTION_CONTROL_MIN_CONNECTION_DELAY; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(551); } +}; +class ObSysVarDefaultWeekFormat : public ObIntSysVar +{ +public: + ObSysVarDefaultWeekFormat() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_DEFAULT_WEEK_FORMAT; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(552); } +}; +class ObSysVarDelayedInsertTimeout : public ObIntSysVar +{ +public: + ObSysVarDelayedInsertTimeout() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_DELAYED_INSERT_TIMEOUT; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(553); } +}; +class ObSysVarDelayedQueueSize : public ObIntSysVar +{ +public: + ObSysVarDelayedQueueSize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_DELAYED_QUEUE_SIZE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(554); } +}; +class ObSysVarEqRangeIndexDiveLimit : public ObIntSysVar +{ +public: + ObSysVarEqRangeIndexDiveLimit() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_EQ_RANGE_INDEX_DIVE_LIMIT; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(555); } +}; +class ObSysVarInnodbStatsAutoRecalc : public ObBoolSysVar +{ +public: + ObSysVarInnodbStatsAutoRecalc() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_STATS_AUTO_RECALC; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(556); } +}; +class ObSysVarInnodbStatsIncludeDeleteMarked : public ObBoolSysVar +{ +public: + ObSysVarInnodbStatsIncludeDeleteMarked() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_STATS_INCLUDE_DELETE_MARKED; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(557); } +}; +class ObSysVarInnodbStatsMethod : public ObEnumSysVar +{ +public: + const static char * INNODB_STATS_METHOD_NAMES[]; +public: + ObSysVarInnodbStatsMethod() : ObEnumSysVar(INNODB_STATS_METHOD_NAMES, NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_STATS_METHOD; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(558); } +}; +class ObSysVarInnodbStatsOnMetadata : public ObBoolSysVar +{ +public: + ObSysVarInnodbStatsOnMetadata() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_STATS_ON_METADATA; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(559); } +}; +class ObSysVarVersionTokensSession : public ObVarcharSysVar +{ +public: + ObSysVarVersionTokensSession() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_VERSION_TOKENS_SESSION; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(560); } +}; +class ObSysVarInnodbStatsPersistentSamplePages : public ObIntSysVar +{ +public: + ObSysVarInnodbStatsPersistentSamplePages() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_STATS_PERSISTENT_SAMPLE_PAGES; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(561); } +}; +class ObSysVarInnodbStatsSamplePages : public ObIntSysVar +{ +public: + ObSysVarInnodbStatsSamplePages() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_STATS_SAMPLE_PAGES; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(562); } +}; +class ObSysVarInnodbStatsTransientSamplePages : public ObIntSysVar +{ +public: + ObSysVarInnodbStatsTransientSamplePages() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_INNODB_STATS_TRANSIENT_SAMPLE_PAGES; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(563); } +}; +class ObSysVarKeyringAwsCmkId : public ObVarcharSysVar +{ +public: + ObSysVarKeyringAwsCmkId() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_KEYRING_AWS_CMK_ID; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(564); } +}; +class ObSysVarKeyringAwsRegion : public ObEnumSysVar +{ +public: + const static char * KEYRING_AWS_REGION_NAMES[]; +public: + ObSysVarKeyringAwsRegion() : ObEnumSysVar(KEYRING_AWS_REGION_NAMES, NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_KEYRING_AWS_REGION; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(565); } +}; +class ObSysVarKeyringEncryptedFileData : public ObVarcharSysVar +{ +public: + ObSysVarKeyringEncryptedFileData() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_KEYRING_ENCRYPTED_FILE_DATA; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(566); } +}; +class ObSysVarKeyringEncryptedFilePassword : public ObVarcharSysVar +{ +public: + ObSysVarKeyringEncryptedFilePassword() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_KEYRING_ENCRYPTED_FILE_PASSWORD; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(567); } +}; +class ObSysVarKeyringFileData : public ObVarcharSysVar +{ +public: + ObSysVarKeyringFileData() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_KEYRING_FILE_DATA; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(568); } +}; +class ObSysVarKeyringOkvConfDir : public ObVarcharSysVar +{ +public: + ObSysVarKeyringOkvConfDir() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_KEYRING_OKV_CONF_DIR; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(569); } +}; +class ObSysVarKeyringOperations : public ObBoolSysVar +{ +public: + ObSysVarKeyringOperations() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_KEYRING_OPERATIONS; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(570); } +}; +class ObSysVarOptimizerSwitch : public ObVarcharSysVar +{ +public: + ObSysVarOptimizerSwitch() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_OPTIMIZER_SWITCH; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(571); } +}; +class ObSysVarMaxConnectErrors : public ObIntSysVar +{ +public: + ObSysVarMaxConnectErrors() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_MAX_CONNECT_ERRORS; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(572); } +}; +class ObSysVarMysqlFirewallMode : public ObBoolSysVar +{ +public: + ObSysVarMysqlFirewallMode() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_MYSQL_FIREWALL_MODE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(573); } +}; +class ObSysVarMysqlFirewallTrace : public ObBoolSysVar +{ +public: + ObSysVarMysqlFirewallTrace() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_MYSQL_FIREWALL_TRACE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(574); } +}; +class ObSysVarMysqlNativePasswordProxyUsers : public ObBoolSysVar +{ +public: + ObSysVarMysqlNativePasswordProxyUsers() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_MYSQL_NATIVE_PASSWORD_PROXY_USERS; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(575); } +}; +class ObSysVarNetRetryCount : public ObIntSysVar +{ +public: + ObSysVarNetRetryCount() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_NET_RETRY_COUNT; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(576); } +}; +class ObSysVarNew : public ObBoolSysVar +{ +public: + ObSysVarNew() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_NEW; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(577); } +}; +class ObSysVarOldPasswords : public ObEnumSysVar +{ +public: + const static char * OLD_PASSWORDS_NAMES[]; +public: + ObSysVarOldPasswords() : ObEnumSysVar(OLD_PASSWORDS_NAMES, NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_OLD_PASSWORDS; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(578); } +}; +class ObSysVarOptimizerPruneLevel : public ObIntSysVar +{ +public: + ObSysVarOptimizerPruneLevel() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_OPTIMIZER_PRUNE_LEVEL; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(579); } +}; +class ObSysVarOptimizerSearchDepth : public ObIntSysVar +{ +public: + ObSysVarOptimizerSearchDepth() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_OPTIMIZER_SEARCH_DEPTH; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(580); } +}; +class ObSysVarOptimizerTrace : public ObVarcharSysVar +{ +public: + ObSysVarOptimizerTrace() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_OPTIMIZER_TRACE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(581); } +}; +class ObSysVarOptimizerTraceFeatures : public ObVarcharSysVar +{ +public: + ObSysVarOptimizerTraceFeatures() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_OPTIMIZER_TRACE_FEATURES; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(582); } +}; +class ObSysVarOptimizerTraceLimit : public ObIntSysVar +{ +public: + ObSysVarOptimizerTraceLimit() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_OPTIMIZER_TRACE_LIMIT; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(583); } +}; +class ObSysVarOptimizerTraceMaxMemSize : public ObIntSysVar +{ +public: + ObSysVarOptimizerTraceMaxMemSize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_OPTIMIZER_TRACE_MAX_MEM_SIZE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(584); } +}; +class ObSysVarOptimizerTraceOffset : public ObIntSysVar +{ +public: + ObSysVarOptimizerTraceOffset() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_OPTIMIZER_TRACE_OFFSET; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(585); } +}; +class ObSysVarParserMaxMemSize : public ObIntSysVar +{ +public: + ObSysVarParserMaxMemSize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_PARSER_MAX_MEM_SIZE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(586); } +}; +class ObSysVarRandSeed1 : public ObIntSysVar +{ +public: + ObSysVarRandSeed1() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_RAND_SEED1; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(587); } +}; +class ObSysVarRandSeed2 : public ObIntSysVar +{ +public: + ObSysVarRandSeed2() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_RAND_SEED2; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(588); } +}; +class ObSysVarRangeAllocBlockSize : public ObIntSysVar +{ +public: + ObSysVarRangeAllocBlockSize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_RANGE_ALLOC_BLOCK_SIZE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(589); } +}; +class ObSysVarRangeOptimizerMaxMemSize : public ObIntSysVar +{ +public: + ObSysVarRangeOptimizerMaxMemSize() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_RANGE_OPTIMIZER_MAX_MEM_SIZE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(590); } +}; +class ObSysVarRewriterEnabled : public ObBoolSysVar +{ +public: + ObSysVarRewriterEnabled() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_REWRITER_ENABLED; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(591); } +}; +class ObSysVarRewriterVerbose : public ObIntSysVar +{ +public: + ObSysVarRewriterVerbose() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_REWRITER_VERBOSE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(592); } +}; +class ObSysVarSecureAuth : public ObBoolSysVar +{ +public: + ObSysVarSecureAuth() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_SECURE_AUTH; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(593); } +}; +class ObSysVarSha256PasswordProxyUsers : public ObBoolSysVar +{ +public: + ObSysVarSha256PasswordProxyUsers() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_SHA256_PASSWORD_PROXY_USERS; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(594); } +}; +class ObSysVarShowCompatibility56 : public ObBoolSysVar +{ +public: + ObSysVarShowCompatibility56() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_SHOW_COMPATIBILITY_56; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(595); } +}; +class ObSysVarShowCreateTableVerbosity : public ObBoolSysVar +{ +public: + ObSysVarShowCreateTableVerbosity() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_SHOW_CREATE_TABLE_VERBOSITY; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(596); } +}; +class ObSysVarShowOldTemporals : public ObBoolSysVar +{ +public: + ObSysVarShowOldTemporals() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_SHOW_OLD_TEMPORALS; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(597); } +}; +class ObSysVarSqlBigSelects : public ObBoolSysVar +{ +public: + ObSysVarSqlBigSelects() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_SQL_BIG_SELECTS; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(598); } +}; +class ObSysVarUpdatableViewsWithLimit : public ObEnumSysVar +{ +public: + const static char * UPDATABLE_VIEWS_WITH_LIMIT_NAMES[]; +public: + ObSysVarUpdatableViewsWithLimit() : ObEnumSysVar(UPDATABLE_VIEWS_WITH_LIMIT_NAMES, NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_UPDATABLE_VIEWS_WITH_LIMIT; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(599); } +}; +class ObSysVarValidatePasswordDictionaryFile : public ObVarcharSysVar +{ +public: + ObSysVarValidatePasswordDictionaryFile() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_VALIDATE_PASSWORD_DICTIONARY_FILE; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(600); } +}; +class ObSysVarDelayedInsertLimit : public ObIntSysVar +{ +public: + ObSysVarDelayedInsertLimit() : ObIntSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_DELAYED_INSERT_LIMIT; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(601); } +}; +class ObSysVarNdbVersion : public ObVarcharSysVar +{ +public: + ObSysVarNdbVersion() : ObVarcharSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_NDB_VERSION; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(602); } +}; +class ObSysVarAutoGenerateCerts : public ObBoolSysVar +{ +public: + ObSysVarAutoGenerateCerts() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {} + inline virtual ObSysVarClassType get_type() const { return SYS_VAR_AUTO_GENERATE_CERTS; } + inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(603); } }; @@ -3349,9 +4425,12 @@ public: static ObSysVarClassType find_sys_var_id_by_name(const common::ObString &sys_var_name, bool is_from_sys_table = false); //二分查找 static int get_sys_var_name_by_id(ObSysVarClassType sys_var_id, common::ObString &sys_var_name); static const common::ObString get_sys_var_name_by_id(ObSysVarClassType sys_var_id); +private: + int try_init_store_mem(); +public: const static int64_t MYSQL_SYS_VARS_COUNT = 99; - const static int64_t OB_SYS_VARS_COUNT = 353; + const static int64_t OB_SYS_VARS_COUNT = 505; const static int64_t ALL_SYS_VARS_COUNT = MYSQL_SYS_VARS_COUNT + OB_SYS_VARS_COUNT; const static int64_t INVALID_MAX_READ_STALE_TIME = -1; @@ -3366,8 +4445,8 @@ private: const static ObSysVarClassType SYS_VAR_IDS_SORTED_BY_NAME[ALL_SYS_VARS_COUNT]; const static char *SYS_VAR_NAMES_SORTED_BY_ID[ALL_SYS_VARS_COUNT]; common::ObArenaAllocator allocator_; - ObBasicSysVar *store_[ALL_SYS_VARS_COUNT]; - ObBasicSysVar *store_buf_[ALL_SYS_VARS_COUNT]; + ObBasicSysVar **store_; + ObBasicSysVar **store_buf_; bool all_sys_vars_created_; }; diff --git a/src/share/system_variable/ob_system_variable_init.cpp b/src/share/system_variable/ob_system_variable_init.cpp index 8b73096c0..085bec977 100644 --- a/src/share/system_variable/ob_system_variable_init.cpp +++ b/src/share/system_variable/ob_system_variable_init.cpp @@ -2173,8 +2173,9 @@ static struct VarsInit{ ObSysVars[151].info_ = "specifies the encryption algorithm used in the functions aes_encrypt and aes_decrypt" ; ObSysVars[151].name_ = "block_encryption_mode" ; ObSysVars[151].data_type_ = ObIntType ; - ObSysVars[151].enum_names_ = "[u'aes-128-ecb', u'aes-192-ecb', u'aes-256-ecb', u'aes-128-cbc', u'aes-192-cbc', u'aes-256-cbc', u'aes-128-cfb1', u'aes-192-cfb1', u'aes-256-cfb1', u'aes-128-cfb8', u'aes-192-cfb8', u'aes-256-cfb8', u'aes-128-cfb128', u'aes-192-cfb128', u'aes-256-cfb128', u'aes-128-ofb', u'aes-192-ofb', u'aes-256-ofb']" ; + ObSysVars[151].enum_names_ = "[u'aes-128-ecb', u'aes-192-ecb', u'aes-256-ecb', u'aes-128-cbc', u'aes-192-cbc', u'aes-256-cbc', u'aes-128-cfb1', u'aes-192-cfb1', u'aes-256-cfb1', u'aes-128-cfb8', u'aes-192-cfb8', u'aes-256-cfb8', u'aes-128-cfb128', u'aes-192-cfb128', u'aes-256-cfb128', u'aes-128-ofb', u'aes-192-ofb', u'aes-256-ofb', u'sm4-ecb', u'sm4-cbc', u'sm4-cfb', u'sm4-ofb']" ; ObSysVars[151].flags_ = ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::NEED_SERIALIZE ; + ObSysVars[151].on_check_and_convert_func_ = "ObSysVarOnCheckFuncs::check_and_convert_block_encryption_mode" ; ObSysVars[151].id_ = SYS_VAR_BLOCK_ENCRYPTION_MODE ; cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_BLOCK_ENCRYPTION_MODE)) ; ObSysVarsIdToArrayIdx[SYS_VAR_BLOCK_ENCRYPTION_MODE] = 151 ; @@ -5785,6 +5786,7 @@ static struct VarsInit{ ObSysVars[409].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::INFLUENCE_PLAN | ObSysVarFlag::NEED_SERIALIZE | ObSysVarFlag::MYSQL_ONLY ; ObSysVars[409].base_class_ = "ObCharsetSysVar" ; ObSysVars[409].to_select_obj_func_ = "ObSysVarToObjFuncs::to_obj_collation" ; + ObSysVars[409].on_check_and_convert_func_ = "ObSysVarOnCheckFuncs::check_default_value_for_utf8mb4" ; ObSysVars[409].id_ = SYS_VAR_DEFAULT_COLLATION_FOR_UTF8MB4 ; cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_DEFAULT_COLLATION_FOR_UTF8MB4)) ; ObSysVarsIdToArrayIdx[SYS_VAR_DEFAULT_COLLATION_FOR_UTF8MB4] = 409 ; @@ -5807,573 +5809,2605 @@ static struct VarsInit{ }(); [&] (){ - ObSysVars[411].default_value_ = "" ; - ObSysVars[411].info_ = "enabled roles for current session" ; - ObSysVars[411].name_ = "_ob_enable_role_ids" ; - ObSysVars[411].data_type_ = ObVarcharType ; - ObSysVars[411].flags_ = ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::NEED_SERIALIZE | ObSysVarFlag::INVISIBLE ; - ObSysVars[411].id_ = SYS_VAR__OB_ENABLE_ROLE_IDS ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR__OB_ENABLE_ROLE_IDS)) ; - ObSysVarsIdToArrayIdx[SYS_VAR__OB_ENABLE_ROLE_IDS] = 411 ; - ObSysVars[411].base_value_ = "" ; - ObSysVars[411].alias_ = "OB_SV__OB_ENABLE_ROLE_IDS" ; + ObSysVars[411].default_value_ = "0" ; + ObSysVars[411].info_ = "The value to be used by the following INSERT or ALTER TABLE statement when inserting an AUTO_INCREMENT value. Merely simulates MySQL 5.7." ; + ObSysVars[411].name_ = "insert_id" ; + ObSysVars[411].data_type_ = ObIntType ; + ObSysVars[411].flags_ = ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[411].id_ = SYS_VAR_INSERT_ID ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INSERT_ID)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INSERT_ID] = 411 ; + ObSysVars[411].base_value_ = "0" ; + ObSysVars[411].alias_ = "OB_SV_INSERT_ID" ; }(); [&] (){ - ObSysVars[412].default_value_ = "0" ; - ObSysVars[412].info_ = "Starts InnoDB in read-only mode. For distributing database applications or data sets on read-only media. Can also be used in data warehouses to share the same data directory between multiple instances." ; - ObSysVars[412].name_ = "innodb_read_only" ; + ObSysVars[412].default_value_ = "262144" ; + ObSysVars[412].info_ = "The minimum size of the buffer that is used for plain index scans, range index scans, and joins that do not use indexes and thus perform full table scans. Merely simulates MySQL 5.7." ; + ObSysVars[412].name_ = "join_buffer_size" ; ObSysVars[412].data_type_ = ObIntType ; - ObSysVars[412].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; - ObSysVars[412].id_ = SYS_VAR_INNODB_READ_ONLY ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_READ_ONLY)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_READ_ONLY] = 412 ; - ObSysVars[412].base_value_ = "0" ; - ObSysVars[412].alias_ = "OB_SV_INNODB_READ_ONLY" ; + ObSysVars[412].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[412].id_ = SYS_VAR_JOIN_BUFFER_SIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_JOIN_BUFFER_SIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_JOIN_BUFFER_SIZE] = 412 ; + ObSysVars[412].base_value_ = "262144" ; + ObSysVars[412].alias_ = "OB_SV_JOIN_BUFFER_SIZE" ; }(); [&] (){ - ObSysVars[413].default_value_ = "0" ; - ObSysVars[413].info_ = "Use this option to disable row locks when InnoDB memcached performs DML operations. By default, innodb_api_disable_rowlock is disabled, which means that memcached requests row locks for get and set operations." ; - ObSysVars[413].name_ = "innodb_api_disable_rowlock" ; - ObSysVars[413].data_type_ = ObIntType ; - ObSysVars[413].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; - ObSysVars[413].id_ = SYS_VAR_INNODB_API_DISABLE_ROWLOCK ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_API_DISABLE_ROWLOCK)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_API_DISABLE_ROWLOCK] = 413 ; - ObSysVars[413].base_value_ = "0" ; - ObSysVars[413].alias_ = "OB_SV_INNODB_API_DISABLE_ROWLOCK" ; + ObSysVars[413].default_value_ = "18446744073709547520" ; + ObSysVars[413].info_ = "Do not permit statements that probably need to examine more than max_join_size rows (for single-table statements) or row combinations (for multiple-table statements) or that are likely to do more than max_join_size disk seeks. Merely simulates MySQL 5.7." ; + ObSysVars[413].name_ = "max_join_size" ; + ObSysVars[413].data_type_ = ObUInt64Type ; + ObSysVars[413].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[413].id_ = SYS_VAR_MAX_JOIN_SIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_MAX_JOIN_SIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_MAX_JOIN_SIZE] = 413 ; + ObSysVars[413].base_value_ = "18446744073709547520" ; + ObSysVars[413].alias_ = "OB_SV_MAX_JOIN_SIZE" ; }(); [&] (){ - ObSysVars[414].default_value_ = "1" ; - ObSysVars[414].info_ = "The lock mode to use for generating auto-increment values. Permissible values are 0, 1, or 2, for traditional, consecutive, or interleaved, respectively. The default setting is 1 (consecutive)." ; - ObSysVars[414].name_ = "innodb_autoinc_lock_mode" ; + ObSysVars[414].default_value_ = "1024" ; + ObSysVars[414].info_ = "The cutoff on the size of index values that determines which filesort algorithm to use. Merely simulates MySQL 5.7." ; + ObSysVars[414].name_ = "max_length_for_sort_data" ; ObSysVars[414].data_type_ = ObIntType ; - ObSysVars[414].min_val_ = "0" ; - ObSysVars[414].max_val_ = "2" ; - ObSysVars[414].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; - ObSysVars[414].id_ = SYS_VAR_INNODB_AUTOINC_LOCK_MODE ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_AUTOINC_LOCK_MODE)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_AUTOINC_LOCK_MODE] = 414 ; - ObSysVars[414].base_value_ = "1" ; - ObSysVars[414].alias_ = "OB_SV_INNODB_AUTOINC_LOCK_MODE" ; + ObSysVars[414].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[414].id_ = SYS_VAR_MAX_LENGTH_FOR_SORT_DATA ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_MAX_LENGTH_FOR_SORT_DATA)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_MAX_LENGTH_FOR_SORT_DATA] = 414 ; + ObSysVars[414].base_value_ = "1024" ; + ObSysVars[414].alias_ = "OB_SV_MAX_LENGTH_FOR_SORT_DATA" ; }(); [&] (){ - ObSysVars[415].default_value_ = "1" ; - ObSysVars[415].info_ = "This is OFF if mysqld uses external locking (system locking), ON if external locking is disabled." ; - ObSysVars[415].name_ = "skip_external_locking" ; + ObSysVars[415].default_value_ = "16382" ; + ObSysVars[415].info_ = "This variable limits the total number of prepared statements in the server. Merely simulates MySQL 5.7." ; + ObSysVars[415].name_ = "max_prepared_stmt_count" ; ObSysVars[415].data_type_ = ObIntType ; - ObSysVars[415].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; - ObSysVars[415].id_ = SYS_VAR_SKIP_EXTERNAL_LOCKING ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_SKIP_EXTERNAL_LOCKING)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_SKIP_EXTERNAL_LOCKING] = 415 ; - ObSysVars[415].base_value_ = "1" ; - ObSysVars[415].alias_ = "OB_SV_SKIP_EXTERNAL_LOCKING" ; + ObSysVars[415].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[415].id_ = SYS_VAR_MAX_PREPARED_STMT_COUNT ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_MAX_PREPARED_STMT_COUNT)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_MAX_PREPARED_STMT_COUNT] = 415 ; + ObSysVars[415].base_value_ = "16382" ; + ObSysVars[415].alias_ = "OB_SV_MAX_PREPARED_STMT_COUNT" ; }(); [&] (){ - ObSysVars[416].default_value_ = "0" ; - ObSysVars[416].info_ = "If the read_only system variable is enabled, the server permits no client updates except from users who have the SUPER privilege. If the super_read_only system variable is also enabled, the server prohibits client updates even from users who have SUPER." ; - ObSysVars[416].name_ = "super_read_only" ; + ObSysVars[416].default_value_ = "1024" ; + ObSysVars[416].info_ = "The number of bytes to use when sorting data values. Merely simulates MySQL 5.7." ; + ObSysVars[416].name_ = "max_sort_length" ; ObSysVars[416].data_type_ = ObIntType ; - ObSysVars[416].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; - ObSysVars[416].id_ = SYS_VAR_SUPER_READ_ONLY ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_SUPER_READ_ONLY)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_SUPER_READ_ONLY] = 416 ; - ObSysVars[416].base_value_ = "0" ; - ObSysVars[416].alias_ = "OB_SV_SUPER_READ_ONLY" ; + ObSysVars[416].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[416].id_ = SYS_VAR_MAX_SORT_LENGTH ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_MAX_SORT_LENGTH)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_MAX_SORT_LENGTH] = 416 ; + ObSysVars[416].base_value_ = "1024" ; + ObSysVars[416].alias_ = "OB_SV_MAX_SORT_LENGTH" ; }(); [&] (){ - ObSysVars[417].default_value_ = "2" ; - ObSysVars[417].info_ = "PLSQL_OPTIMIZE_LEVEL specifies the optimization level that will be used to compile PL/SQL library units. The higher the setting of this parameter, the more effort the compiler makes to optimize PL/SQL library units." ; - ObSysVars[417].name_ = "plsql_optimize_level" ; + ObSysVars[417].default_value_ = "0" ; + ObSysVars[417].info_ = "Queries that examine fewer than this number of rows are not logged to the slow query log. Merely simulates MySQL 5.7." ; + ObSysVars[417].name_ = "min_examined_row_limit" ; ObSysVars[417].data_type_ = ObIntType ; - ObSysVars[417].flags_ = ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::NEED_SERIALIZE ; - ObSysVars[417].id_ = SYS_VAR_PLSQL_OPTIMIZE_LEVEL ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_PLSQL_OPTIMIZE_LEVEL)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_PLSQL_OPTIMIZE_LEVEL] = 417 ; - ObSysVars[417].base_value_ = "2" ; - ObSysVars[417].alias_ = "OB_SV_PLSQL_OPTIMIZE_LEVEL" ; + ObSysVars[417].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[417].id_ = SYS_VAR_MIN_EXAMINED_ROW_LIMIT ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_MIN_EXAMINED_ROW_LIMIT)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_MIN_EXAMINED_ROW_LIMIT] = 417 ; + ObSysVars[417].base_value_ = "0" ; + ObSysVars[417].alias_ = "OB_SV_MIN_EXAMINED_ROW_LIMIT" ; }(); [&] (){ - ObSysVars[418].default_value_ = "built-in" ; - ObSysVars[418].info_ = "mock for mysql5.7" ; - ObSysVars[418].name_ = "ft_stopword_file" ; - ObSysVars[418].data_type_ = ObVarcharType ; - ObSysVars[418].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[418].id_ = SYS_VAR_FT_STOPWORD_FILE ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_FT_STOPWORD_FILE)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_FT_STOPWORD_FILE] = 418 ; - ObSysVars[418].base_value_ = "built-in" ; - ObSysVars[418].alias_ = "OB_SV_FT_STOPWORD_FILE" ; + ObSysVars[418].default_value_ = "256" ; + ObSysVars[418].info_ = "This variable has no effect. Merely simulates MySQL 5.7." ; + ObSysVars[418].name_ = "multi_range_count" ; + ObSysVars[418].data_type_ = ObIntType ; + ObSysVars[418].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[418].id_ = SYS_VAR_MULTI_RANGE_COUNT ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_MULTI_RANGE_COUNT)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_MULTI_RANGE_COUNT] = 418 ; + ObSysVars[418].base_value_ = "256" ; + ObSysVars[418].alias_ = "OB_SV_MULTI_RANGE_COUNT" ; }(); [&] (){ - ObSysVars[419].default_value_ = "8000000" ; - ObSysVars[419].info_ = "mock for mysql5.7" ; - ObSysVars[419].name_ = "innodb_ft_cache_size" ; + ObSysVars[419].default_value_ = "30" ; + ObSysVars[419].info_ = "The number of seconds X Plugin waits for the first packet to be received from newly connected clients. Merely simulates MySQL 5.7." ; + ObSysVars[419].name_ = "mysqlx_connect_timeout" ; ObSysVars[419].data_type_ = ObIntType ; - ObSysVars[419].min_val_ = "1600000" ; - ObSysVars[419].max_val_ = "80000000" ; ObSysVars[419].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[419].id_ = SYS_VAR_INNODB_FT_CACHE_SIZE ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_FT_CACHE_SIZE)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_FT_CACHE_SIZE] = 419 ; - ObSysVars[419].base_value_ = "8000000" ; - ObSysVars[419].alias_ = "OB_SV_INNODB_FT_CACHE_SIZE" ; + ObSysVars[419].id_ = SYS_VAR_MYSQLX_CONNECT_TIMEOUT ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_MYSQLX_CONNECT_TIMEOUT)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_MYSQLX_CONNECT_TIMEOUT] = 419 ; + ObSysVars[419].base_value_ = "30" ; + ObSysVars[419].alias_ = "OB_SV_MYSQLX_CONNECT_TIMEOUT" ; }(); [&] (){ - ObSysVars[420].default_value_ = "2" ; - ObSysVars[420].info_ = "mock for mysql5.7" ; - ObSysVars[420].name_ = "innodb_ft_sort_pll_degree" ; + ObSysVars[420].default_value_ = "60" ; + ObSysVars[420].info_ = "The number of seconds after which idle worker threads are terminated. Merely simulates MySQL 5.7." ; + ObSysVars[420].name_ = "mysqlx_idle_worker_thread_timeout" ; ObSysVars[420].data_type_ = ObIntType ; - ObSysVars[420].min_val_ = "1" ; - ObSysVars[420].max_val_ = "16" ; ObSysVars[420].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[420].id_ = SYS_VAR_INNODB_FT_SORT_PLL_DEGREE ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_FT_SORT_PLL_DEGREE)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_FT_SORT_PLL_DEGREE] = 420 ; - ObSysVars[420].base_value_ = "2" ; - ObSysVars[420].alias_ = "OB_SV_INNODB_FT_SORT_PLL_DEGREE" ; + ObSysVars[420].id_ = SYS_VAR_MYSQLX_IDLE_WORKER_THREAD_TIMEOUT ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_MYSQLX_IDLE_WORKER_THREAD_TIMEOUT)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_MYSQLX_IDLE_WORKER_THREAD_TIMEOUT] = 420 ; + ObSysVars[420].base_value_ = "60" ; + ObSysVars[420].alias_ = "OB_SV_MYSQLX_IDLE_WORKER_THREAD_TIMEOUT" ; }(); [&] (){ - ObSysVars[421].default_value_ = "640000000" ; - ObSysVars[421].info_ = "mock for mysql5.7" ; - ObSysVars[421].name_ = "innodb_ft_total_cache_size" ; + ObSysVars[421].default_value_ = "67108864" ; + ObSysVars[421].info_ = "The maximum size of network packets that can be received by X Plugin. Merely simulates MySQL 5.7." ; + ObSysVars[421].name_ = "mysqlx_max_allowed_packet" ; ObSysVars[421].data_type_ = ObIntType ; - ObSysVars[421].min_val_ = "32000000" ; - ObSysVars[421].max_val_ = "1600000000" ; ObSysVars[421].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[421].id_ = SYS_VAR_INNODB_FT_TOTAL_CACHE_SIZE ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_FT_TOTAL_CACHE_SIZE)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_FT_TOTAL_CACHE_SIZE] = 421 ; - ObSysVars[421].base_value_ = "640000000" ; - ObSysVars[421].alias_ = "OB_SV_INNODB_FT_TOTAL_CACHE_SIZE" ; + ObSysVars[421].id_ = SYS_VAR_MYSQLX_MAX_ALLOWED_PACKET ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_MYSQLX_MAX_ALLOWED_PACKET)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_MYSQLX_MAX_ALLOWED_PACKET] = 421 ; + ObSysVars[421].base_value_ = "67108864" ; + ObSysVars[421].alias_ = "OB_SV_MYSQLX_MAX_ALLOWED_PACKET" ; }(); [&] (){ - ObSysVars[422].default_value_ = "" ; - ObSysVars[422].info_ = "mock for mysql5.7" ; - ObSysVars[422].name_ = "mecab_rc_file" ; - ObSysVars[422].data_type_ = ObVarcharType ; + ObSysVars[422].default_value_ = "100" ; + ObSysVars[422].info_ = "The maximum number of concurrent client connections X Plugin can accept. Merely simulates MySQL 5.7." ; + ObSysVars[422].name_ = "mysqlx_max_connections" ; + ObSysVars[422].data_type_ = ObIntType ; ObSysVars[422].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[422].id_ = SYS_VAR_MECAB_RC_FILE ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_MECAB_RC_FILE)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_MECAB_RC_FILE] = 422 ; - ObSysVars[422].base_value_ = "" ; - ObSysVars[422].alias_ = "OB_SV_MECAB_RC_FILE" ; + ObSysVars[422].id_ = SYS_VAR_MYSQLX_MAX_CONNECTIONS ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_MYSQLX_MAX_CONNECTIONS)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_MYSQLX_MAX_CONNECTIONS] = 422 ; + ObSysVars[422].base_value_ = "100" ; + ObSysVars[422].alias_ = "OB_SV_MYSQLX_MAX_CONNECTIONS" ; }(); [&] (){ - ObSysVars[423].default_value_ = "1024" ; - ObSysVars[423].info_ = "mock for mysql5.7" ; - ObSysVars[423].name_ = "metadata_locks_cache_size" ; + ObSysVars[423].default_value_ = "2" ; + ObSysVars[423].info_ = "The minimum number of worker threads used by X Plugin for handling client requests. Merely simulates MySQL 5.7." ; + ObSysVars[423].name_ = "mysqlx_min_worker_threads" ; ObSysVars[423].data_type_ = ObIntType ; - ObSysVars[423].min_val_ = "1" ; - ObSysVars[423].max_val_ = "1048576" ; ObSysVars[423].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[423].id_ = SYS_VAR_METADATA_LOCKS_CACHE_SIZE ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_METADATA_LOCKS_CACHE_SIZE)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_METADATA_LOCKS_CACHE_SIZE] = 423 ; - ObSysVars[423].base_value_ = "1024" ; - ObSysVars[423].alias_ = "OB_SV_METADATA_LOCKS_CACHE_SIZE" ; + ObSysVars[423].id_ = SYS_VAR_MYSQLX_MIN_WORKER_THREADS ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_MYSQLX_MIN_WORKER_THREADS)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_MYSQLX_MIN_WORKER_THREADS] = 423 ; + ObSysVars[423].base_value_ = "2" ; + ObSysVars[423].alias_ = "OB_SV_MYSQLX_MIN_WORKER_THREADS" ; }(); [&] (){ - ObSysVars[424].default_value_ = "8" ; - ObSysVars[424].info_ = "mock for mysql5.7" ; - ObSysVars[424].name_ = "metadata_locks_hash_instances" ; + ObSysVars[424].default_value_ = "0" ; + ObSysVars[424].info_ = "The variable determines which SHOW PROCESSLIST implementation to use. Merely simulates MySQL 5.7." ; + ObSysVars[424].name_ = "performance_schema_show_processlist" ; ObSysVars[424].data_type_ = ObIntType ; - ObSysVars[424].min_val_ = "1" ; - ObSysVars[424].max_val_ = "1024" ; ObSysVars[424].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[424].id_ = SYS_VAR_METADATA_LOCKS_HASH_INSTANCES ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_METADATA_LOCKS_HASH_INSTANCES)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_METADATA_LOCKS_HASH_INSTANCES] = 424 ; - ObSysVars[424].base_value_ = "8" ; - ObSysVars[424].alias_ = "OB_SV_METADATA_LOCKS_HASH_INSTANCES" ; + ObSysVars[424].id_ = SYS_VAR_PERFORMANCE_SCHEMA_SHOW_PROCESSLIST ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_PERFORMANCE_SCHEMA_SHOW_PROCESSLIST)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_PERFORMANCE_SCHEMA_SHOW_PROCESSLIST] = 424 ; + ObSysVars[424].base_value_ = "0" ; + ObSysVars[424].alias_ = "OB_SV_PERFORMANCE_SCHEMA_SHOW_PROCESSLIST" ; }(); [&] (){ - ObSysVars[425].default_value_ = "ibtmp1:12M:autoextend" ; - ObSysVars[425].info_ = "mock for mysql5.7" ; - ObSysVars[425].name_ = "innodb_temp_data_file_path" ; - ObSysVars[425].data_type_ = ObVarcharType ; - ObSysVars[425].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[425].id_ = SYS_VAR_INNODB_TEMP_DATA_FILE_PATH ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_TEMP_DATA_FILE_PATH)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_TEMP_DATA_FILE_PATH] = 425 ; - ObSysVars[425].base_value_ = "ibtmp1:12M:autoextend" ; - ObSysVars[425].alias_ = "OB_SV_INNODB_TEMP_DATA_FILE_PATH" ; + ObSysVars[425].default_value_ = "8192" ; + ObSysVars[425].info_ = "The allocation size in bytes of memory blocks that are allocated for objects created during statement parsing and execution. Merely simulates MySQL 5.7." ; + ObSysVars[425].name_ = "query_alloc_block_size" ; + ObSysVars[425].data_type_ = ObIntType ; + ObSysVars[425].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[425].id_ = SYS_VAR_QUERY_ALLOC_BLOCK_SIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_QUERY_ALLOC_BLOCK_SIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_QUERY_ALLOC_BLOCK_SIZE] = 425 ; + ObSysVars[425].base_value_ = "8192" ; + ObSysVars[425].alias_ = "OB_SV_QUERY_ALLOC_BLOCK_SIZE" ; }(); [&] (){ - ObSysVars[426].default_value_ = "ibdata1:12M:autoextend" ; - ObSysVars[426].info_ = "mock for mysql5.7" ; - ObSysVars[426].name_ = "innodb_data_file_path" ; - ObSysVars[426].data_type_ = ObVarcharType ; - ObSysVars[426].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[426].id_ = SYS_VAR_INNODB_DATA_FILE_PATH ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_DATA_FILE_PATH)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_DATA_FILE_PATH] = 426 ; - ObSysVars[426].base_value_ = "ibdata1:12M:autoextend" ; - ObSysVars[426].alias_ = "OB_SV_INNODB_DATA_FILE_PATH" ; + ObSysVars[426].default_value_ = "8192" ; + ObSysVars[426].info_ = "The size in bytes of the persistent buffer used for statement parsing and execution. Merely simulates MySQL 5.7." ; + ObSysVars[426].name_ = "query_prealloc_size" ; + ObSysVars[426].data_type_ = ObIntType ; + ObSysVars[426].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[426].id_ = SYS_VAR_QUERY_PREALLOC_SIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_QUERY_PREALLOC_SIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_QUERY_PREALLOC_SIZE] = 426 ; + ObSysVars[426].base_value_ = "8192" ; + ObSysVars[426].alias_ = "OB_SV_QUERY_PREALLOC_SIZE" ; }(); [&] (){ - ObSysVars[427].default_value_ = "" ; - ObSysVars[427].info_ = "mock for mysql5.7" ; - ObSysVars[427].name_ = "innodb_data_home_dir" ; - ObSysVars[427].data_type_ = ObVarcharType ; + ObSysVars[427].default_value_ = "0" ; + ObSysVars[427].info_ = "Whether the slow query log is enabled. Merely simulates MySQL 5.7." ; + ObSysVars[427].name_ = "slow_query_log" ; + ObSysVars[427].data_type_ = ObIntType ; ObSysVars[427].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[427].id_ = SYS_VAR_INNODB_DATA_HOME_DIR ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_DATA_HOME_DIR)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_DATA_HOME_DIR] = 427 ; - ObSysVars[427].base_value_ = "" ; - ObSysVars[427].alias_ = "OB_SV_INNODB_DATA_HOME_DIR" ; + ObSysVars[427].id_ = SYS_VAR_SLOW_QUERY_LOG ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_SLOW_QUERY_LOG)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_SLOW_QUERY_LOG] = 427 ; + ObSysVars[427].base_value_ = "0" ; + ObSysVars[427].alias_ = "OB_SV_SLOW_QUERY_LOG" ; }(); [&] (){ - ObSysVars[428].default_value_ = "0" ; - ObSysVars[428].info_ = "mock for mysql5.7" ; - ObSysVars[428].name_ = "avoid_temporal_upgrade" ; - ObSysVars[428].data_type_ = ObIntType ; + ObSysVars[428].default_value_ = "/usr/local/mysql/data/obrd-slow.log" ; + ObSysVars[428].info_ = "The name of the slow query log file. Merely simulates MySQL 5.7." ; + ObSysVars[428].name_ = "slow_query_log_file" ; + ObSysVars[428].data_type_ = ObVarcharType ; ObSysVars[428].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[428].id_ = SYS_VAR_AVOID_TEMPORAL_UPGRADE ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_AVOID_TEMPORAL_UPGRADE)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_AVOID_TEMPORAL_UPGRADE] = 428 ; - ObSysVars[428].base_value_ = "0" ; - ObSysVars[428].alias_ = "OB_SV_AVOID_TEMPORAL_UPGRADE" ; + ObSysVars[428].id_ = SYS_VAR_SLOW_QUERY_LOG_FILE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_SLOW_QUERY_LOG_FILE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_SLOW_QUERY_LOG_FILE] = 428 ; + ObSysVars[428].base_value_ = "/usr/local/mysql/data/obrd-slow.log" ; + ObSysVars[428].alias_ = "OB_SV_SLOW_QUERY_LOG_FILE" ; }(); [&] (){ - ObSysVars[429].default_value_ = "0" ; - ObSysVars[429].info_ = "mock for mysql5.7" ; - ObSysVars[429].name_ = "default_tmp_storage_engine" ; + ObSysVars[429].default_value_ = "262144" ; + ObSysVars[429].info_ = "Each session that must perform a sort allocates a buffer of this size. Merely simulates MySQL 5.7." ; + ObSysVars[429].name_ = "sort_buffer_size" ; ObSysVars[429].data_type_ = ObIntType ; - ObSysVars[429].enum_names_ = "[u'InnoDB']" ; ObSysVars[429].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[429].id_ = SYS_VAR_DEFAULT_TMP_STORAGE_ENGINE ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_DEFAULT_TMP_STORAGE_ENGINE)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_DEFAULT_TMP_STORAGE_ENGINE] = 429 ; - ObSysVars[429].base_value_ = "0" ; - ObSysVars[429].alias_ = "OB_SV_DEFAULT_TMP_STORAGE_ENGINE" ; + ObSysVars[429].id_ = SYS_VAR_SORT_BUFFER_SIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_SORT_BUFFER_SIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_SORT_BUFFER_SIZE] = 429 ; + ObSysVars[429].base_value_ = "262144" ; + ObSysVars[429].alias_ = "OB_SV_SORT_BUFFER_SIZE" ; }(); [&] (){ ObSysVars[430].default_value_ = "0" ; - ObSysVars[430].info_ = "mock for mysql5.7" ; - ObSysVars[430].name_ = "innodb_ft_enable_diag_print" ; + ObSysVars[430].info_ = "If enabled, sql_buffer_result forces results from SELECT statements to be put into temporary tables. Merely simulates MySQL 5.7." ; + ObSysVars[430].name_ = "sql_buffer_result" ; ObSysVars[430].data_type_ = ObIntType ; - ObSysVars[430].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[430].id_ = SYS_VAR_INNODB_FT_ENABLE_DIAG_PRINT ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_FT_ENABLE_DIAG_PRINT)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_FT_ENABLE_DIAG_PRINT] = 430 ; + ObSysVars[430].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[430].id_ = SYS_VAR_SQL_BUFFER_RESULT ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_SQL_BUFFER_RESULT)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_SQL_BUFFER_RESULT] = 430 ; ObSysVars[430].base_value_ = "0" ; - ObSysVars[430].alias_ = "OB_SV_INNODB_FT_ENABLE_DIAG_PRINT" ; + ObSysVars[430].alias_ = "OB_SV_SQL_BUFFER_RESULT" ; }(); [&] (){ - ObSysVars[431].default_value_ = "2000" ; - ObSysVars[431].info_ = "mock for mysql5.7" ; - ObSysVars[431].name_ = "innodb_ft_num_word_optimize" ; + ObSysVars[431].default_value_ = "32768" ; + ObSysVars[431].info_ = "binlog_cache_size sets the size for the transaction cache only. Merely simulates MySQL 5.7." ; + ObSysVars[431].name_ = "binlog_cache_size" ; ObSysVars[431].data_type_ = ObIntType ; - ObSysVars[431].min_val_ = "1000" ; - ObSysVars[431].max_val_ = "10000" ; ObSysVars[431].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[431].id_ = SYS_VAR_INNODB_FT_NUM_WORD_OPTIMIZE ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_FT_NUM_WORD_OPTIMIZE)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_FT_NUM_WORD_OPTIMIZE] = 431 ; - ObSysVars[431].base_value_ = "2000" ; - ObSysVars[431].alias_ = "OB_SV_INNODB_FT_NUM_WORD_OPTIMIZE" ; + ObSysVars[431].id_ = SYS_VAR_BINLOG_CACHE_SIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_BINLOG_CACHE_SIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_BINLOG_CACHE_SIZE] = 431 ; + ObSysVars[431].base_value_ = "32768" ; + ObSysVars[431].alias_ = "OB_SV_BINLOG_CACHE_SIZE" ; }(); [&] (){ - ObSysVars[432].default_value_ = "2000000000" ; - ObSysVars[432].info_ = "mock for mysql5.7" ; - ObSysVars[432].name_ = "innodb_ft_result_cache_limit" ; - ObSysVars[432].data_type_ = ObUInt64Type ; - ObSysVars[432].min_val_ = "1000000" ; - ObSysVars[432].max_val_ = "4294967295" ; - ObSysVars[432].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[432].id_ = SYS_VAR_INNODB_FT_RESULT_CACHE_LIMIT ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_FT_RESULT_CACHE_LIMIT)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_FT_RESULT_CACHE_LIMIT] = 432 ; - ObSysVars[432].base_value_ = "2000000000" ; - ObSysVars[432].alias_ = "OB_SV_INNODB_FT_RESULT_CACHE_LIMIT" ; + ObSysVars[432].default_value_ = "0" ; + ObSysVars[432].info_ = "Enabling binlog_direct_non_transactional_updates causes updates to nontransactional tables to be written directly to the binary log, rather than to the transaction cache. Merely simulates MySQL 5.7." ; + ObSysVars[432].name_ = "binlog_direct_non_transactional_updates" ; + ObSysVars[432].data_type_ = ObIntType ; + ObSysVars[432].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[432].id_ = SYS_VAR_BINLOG_DIRECT_NON_TRANSACTIONAL_UPDATES ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_BINLOG_DIRECT_NON_TRANSACTIONAL_UPDATES)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_BINLOG_DIRECT_NON_TRANSACTIONAL_UPDATES] = 432 ; + ObSysVars[432].base_value_ = "0" ; + ObSysVars[432].alias_ = "OB_SV_BINLOG_DIRECT_NON_TRANSACTIONAL_UPDATES" ; }(); [&] (){ - ObSysVars[433].default_value_ = "" ; - ObSysVars[433].info_ = "mock for mysql5.7" ; - ObSysVars[433].name_ = "innodb_ft_server_stopword_table" ; - ObSysVars[433].data_type_ = ObVarcharType ; + ObSysVars[433].default_value_ = "1" ; + ObSysVars[433].info_ = "Controls what happens when the server encounters an error. Merely simulates MySQL 5.7." ; + ObSysVars[433].name_ = "binlog_error_action" ; + ObSysVars[433].data_type_ = ObIntType ; + ObSysVars[433].enum_names_ = "[u'IGNORE_ERROR', u'ABORT_SERVER']" ; ObSysVars[433].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[433].id_ = SYS_VAR_INNODB_FT_SERVER_STOPWORD_TABLE ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_FT_SERVER_STOPWORD_TABLE)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_FT_SERVER_STOPWORD_TABLE] = 433 ; - ObSysVars[433].base_value_ = "" ; - ObSysVars[433].alias_ = "OB_SV_INNODB_FT_SERVER_STOPWORD_TABLE" ; + ObSysVars[433].id_ = SYS_VAR_BINLOG_ERROR_ACTION ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_BINLOG_ERROR_ACTION)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_BINLOG_ERROR_ACTION] = 433 ; + ObSysVars[433].base_value_ = "1" ; + ObSysVars[433].alias_ = "OB_SV_BINLOG_ERROR_ACTION" ; }(); [&] (){ ObSysVars[434].default_value_ = "0" ; - ObSysVars[434].info_ = "mock for mysql5.7" ; - ObSysVars[434].name_ = "innodb_optimize_fulltext_only" ; + ObSysVars[434].info_ = "Controls how many microseconds the binary log commit waits before synchronizing the binary log file to disk. Merely simulates MySQL 5.7." ; + ObSysVars[434].name_ = "binlog_group_commit_sync_delay" ; ObSysVars[434].data_type_ = ObIntType ; ObSysVars[434].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[434].id_ = SYS_VAR_INNODB_OPTIMIZE_FULLTEXT_ONLY ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_OPTIMIZE_FULLTEXT_ONLY)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_OPTIMIZE_FULLTEXT_ONLY] = 434 ; + ObSysVars[434].id_ = SYS_VAR_BINLOG_GROUP_COMMIT_SYNC_DELAY ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_BINLOG_GROUP_COMMIT_SYNC_DELAY)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_BINLOG_GROUP_COMMIT_SYNC_DELAY] = 434 ; ObSysVars[434].base_value_ = "0" ; - ObSysVars[434].alias_ = "OB_SV_INNODB_OPTIMIZE_FULLTEXT_ONLY" ; + ObSysVars[434].alias_ = "OB_SV_BINLOG_GROUP_COMMIT_SYNC_DELAY" ; }(); [&] (){ - ObSysVars[435].default_value_ = "32" ; - ObSysVars[435].info_ = "mock for mysql5.7" ; - ObSysVars[435].name_ = "max_tmp_tables" ; + ObSysVars[435].default_value_ = "0" ; + ObSysVars[435].info_ = "The maximum number of transactions to wait for before aborting the current delay as specified by binlog_group_commit_sync_delay. Merely simulates MySQL 5.7." ; + ObSysVars[435].name_ = "binlog_group_commit_sync_no_delay_count" ; ObSysVars[435].data_type_ = ObIntType ; ObSysVars[435].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[435].id_ = SYS_VAR_MAX_TMP_TABLES ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_MAX_TMP_TABLES)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_MAX_TMP_TABLES] = 435 ; - ObSysVars[435].base_value_ = "32" ; - ObSysVars[435].alias_ = "OB_SV_MAX_TMP_TABLES" ; + ObSysVars[435].id_ = SYS_VAR_BINLOG_GROUP_COMMIT_SYNC_NO_DELAY_COUNT ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_BINLOG_GROUP_COMMIT_SYNC_NO_DELAY_COUNT)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_BINLOG_GROUP_COMMIT_SYNC_NO_DELAY_COUNT] = 435 ; + ObSysVars[435].base_value_ = "0" ; + ObSysVars[435].alias_ = "OB_SV_BINLOG_GROUP_COMMIT_SYNC_NO_DELAY_COUNT" ; }(); [&] (){ - ObSysVars[436].default_value_ = "" ; - ObSysVars[436].info_ = "mock for mysql5.7" ; - ObSysVars[436].name_ = "innodb_tmpdir" ; - ObSysVars[436].data_type_ = ObVarcharType ; - ObSysVars[436].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[436].id_ = SYS_VAR_INNODB_TMPDIR ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_TMPDIR)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_TMPDIR] = 436 ; - ObSysVars[436].base_value_ = "" ; - ObSysVars[436].alias_ = "OB_SV_INNODB_TMPDIR" ; + ObSysVars[436].default_value_ = "0" ; + ObSysVars[436].info_ = "This variable no longer has any effect. Merely simulates MySQL 5.7." ; + ObSysVars[436].name_ = "binlog_max_flush_queue_time" ; + ObSysVars[436].data_type_ = ObIntType ; + ObSysVars[436].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[436].id_ = SYS_VAR_BINLOG_MAX_FLUSH_QUEUE_TIME ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_BINLOG_MAX_FLUSH_QUEUE_TIME)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_BINLOG_MAX_FLUSH_QUEUE_TIME] = 436 ; + ObSysVars[436].base_value_ = "0" ; + ObSysVars[436].alias_ = "OB_SV_BINLOG_MAX_FLUSH_QUEUE_TIME" ; }(); [&] (){ - ObSysVars[437].default_value_ = "" ; - ObSysVars[437].info_ = "A list of group members to which a joining member can connect to obtain details of all the current group members" ; - ObSysVars[437].name_ = "group_replication_group_seeds" ; - ObSysVars[437].data_type_ = ObVarcharType ; - ObSysVars[437].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE ; - ObSysVars[437].id_ = SYS_VAR_GROUP_REPLICATION_GROUP_SEEDS ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_GROUP_REPLICATION_GROUP_SEEDS)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_GROUP_REPLICATION_GROUP_SEEDS] = 437 ; - ObSysVars[437].base_value_ = "" ; - ObSysVars[437].alias_ = "OB_SV_GROUP_REPLICATION_GROUP_SEEDS" ; + ObSysVars[437].default_value_ = "1" ; + ObSysVars[437].info_ = "When this variable is enabled on a replication source server, transaction commit instructions issued to storage engines are serialized on a single thread. Merely simulates MySQL 5.7." ; + ObSysVars[437].name_ = "binlog_order_commits" ; + ObSysVars[437].data_type_ = ObIntType ; + ObSysVars[437].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[437].id_ = SYS_VAR_BINLOG_ORDER_COMMITS ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_BINLOG_ORDER_COMMITS)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_BINLOG_ORDER_COMMITS] = 437 ; + ObSysVars[437].base_value_ = "1" ; + ObSysVars[437].alias_ = "OB_SV_BINLOG_ORDER_COMMITS" ; }(); [&] (){ - ObSysVars[438].default_value_ = "0" ; - ObSysVars[438].info_ = "When preparing batches of rows for row-based logging and replication, this variable controls how the rows are searched for matches" ; - ObSysVars[438].name_ = "slave_rows_search_algorithms" ; + ObSysVars[438].default_value_ = "32768" ; + ObSysVars[438].info_ = "This variable determines the size of the cache for the binary log to hold nontransactional statements issued during a transaction. Merely simulates MySQL 5.7." ; + ObSysVars[438].name_ = "binlog_stmt_cache_size" ; ObSysVars[438].data_type_ = ObIntType ; - ObSysVars[438].enum_names_ = "[u'TABLE_SCAN,INDEX_SCAN', u'INDEX_SCAN,HASH_SCAN', u'TABLE_SCAN,HASH_SCAN', u'TABLE_SCAN,INDEX_SCAN,HASH_SCAN']" ; - ObSysVars[438].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE ; - ObSysVars[438].id_ = SYS_VAR_SLAVE_ROWS_SEARCH_ALGORITHMS ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_SLAVE_ROWS_SEARCH_ALGORITHMS)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_SLAVE_ROWS_SEARCH_ALGORITHMS] = 438 ; - ObSysVars[438].base_value_ = "0" ; - ObSysVars[438].alias_ = "OB_SV_SLAVE_ROWS_SEARCH_ALGORITHMS" ; + ObSysVars[438].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[438].id_ = SYS_VAR_BINLOG_STMT_CACHE_SIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_BINLOG_STMT_CACHE_SIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_BINLOG_STMT_CACHE_SIZE] = 438 ; + ObSysVars[438].base_value_ = "32768" ; + ObSysVars[438].alias_ = "OB_SV_BINLOG_STMT_CACHE_SIZE" ; }(); [&] (){ - ObSysVars[439].default_value_ = "0" ; - ObSysVars[439].info_ = "Controls the type conversion mode in effect on the replica when using row-based replication" ; - ObSysVars[439].name_ = "slave_type_conversions" ; + ObSysVars[439].default_value_ = "25000" ; + ObSysVars[439].info_ = "Sets an upper limit on the number of row hashes which are kept in memory and used for looking up the transaction that last modified a given row. Merely simulates MySQL 5.7." ; + ObSysVars[439].name_ = "binlog_transaction_dependency_history_size" ; ObSysVars[439].data_type_ = ObIntType ; - ObSysVars[439].enum_names_ = "[u'ALL_LOSSY', u'ALL_NON_LOSSY', u'ALL_SIGNED', u'ALL_UNSIGNED']" ; - ObSysVars[439].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE ; - ObSysVars[439].id_ = SYS_VAR_SLAVE_TYPE_CONVERSIONS ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_SLAVE_TYPE_CONVERSIONS)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_SLAVE_TYPE_CONVERSIONS] = 439 ; - ObSysVars[439].base_value_ = "0" ; - ObSysVars[439].alias_ = "OB_SV_SLAVE_TYPE_CONVERSIONS" ; + ObSysVars[439].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[439].id_ = SYS_VAR_BINLOG_TRANSACTION_DEPENDENCY_HISTORY_SIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_BINLOG_TRANSACTION_DEPENDENCY_HISTORY_SIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_BINLOG_TRANSACTION_DEPENDENCY_HISTORY_SIZE] = 439 ; + ObSysVars[439].base_value_ = "25000" ; + ObSysVars[439].alias_ = "OB_SV_BINLOG_TRANSACTION_DEPENDENCY_HISTORY_SIZE" ; }(); [&] (){ ObSysVars[440].default_value_ = "0" ; - ObSysVars[440].info_ = "This variable specifies how to use delayed key writes. It applies only to MyISAM tables. Delayed key writing causes key buffers not to be flushed between writes, merely simulates MySQL 5.7" ; - ObSysVars[440].name_ = "delay_key_write" ; + ObSysVars[440].info_ = "The source of dependency information that the source uses to determine which transactions can be executed in parallel by the replica's multithreaded applier. Merely simulates MySQL 5.7." ; + ObSysVars[440].name_ = "binlog_transaction_dependency_tracking" ; ObSysVars[440].data_type_ = ObIntType ; - ObSysVars[440].enum_names_ = "[u'ON', u'OFF', u'ALL']" ; + ObSysVars[440].enum_names_ = "[u'COMMIT_ORDER', u'WRITESET', u'WRITESET_SESSION']" ; ObSysVars[440].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[440].id_ = SYS_VAR_DELAY_KEY_WRITE ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_DELAY_KEY_WRITE)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_DELAY_KEY_WRITE] = 440 ; + ObSysVars[440].id_ = SYS_VAR_BINLOG_TRANSACTION_DEPENDENCY_TRACKING ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_BINLOG_TRANSACTION_DEPENDENCY_TRACKING)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_BINLOG_TRANSACTION_DEPENDENCY_TRACKING] = 440 ; ObSysVars[440].base_value_ = "0" ; - ObSysVars[440].alias_ = "OB_SV_DELAY_KEY_WRITE" ; + ObSysVars[440].alias_ = "OB_SV_BINLOG_TRANSACTION_DEPENDENCY_TRACKING" ; }(); [&] (){ ObSysVars[441].default_value_ = "0" ; - ObSysVars[441].info_ = "When this option is enabled, index key prefixes longer than 767 bytes (up to 3072 bytes) are allowed for InnoDB tables that use DYNAMIC or COMPRESSED row format, merely simulates MySQL 5.7" ; - ObSysVars[441].name_ = "innodb_large_prefix" ; + ObSysVars[441].info_ = "The number of days for automatic binary log file removal. Merely simulates MySQL 5.7." ; + ObSysVars[441].name_ = "expire_logs_days" ; ObSysVars[441].data_type_ = ObIntType ; - ObSysVars[441].enum_names_ = "[u'ON', u'OFF']" ; ObSysVars[441].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[441].id_ = SYS_VAR_INNODB_LARGE_PREFIX ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_LARGE_PREFIX)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_LARGE_PREFIX] = 441 ; + ObSysVars[441].id_ = SYS_VAR_EXPIRE_LOGS_DAYS ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_EXPIRE_LOGS_DAYS)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_EXPIRE_LOGS_DAYS] = 441 ; ObSysVars[441].base_value_ = "0" ; - ObSysVars[441].alias_ = "OB_SV_INNODB_LARGE_PREFIX" ; + ObSysVars[441].alias_ = "OB_SV_EXPIRE_LOGS_DAYS" ; }(); [&] (){ - ObSysVars[442].default_value_ = "8388608" ; - ObSysVars[442].info_ = "The size of the buffer used for index blocks, merely simulates MySQL 5.7" ; - ObSysVars[442].name_ = "key_buffer_size" ; + ObSysVars[442].default_value_ = "1" ; + ObSysVars[442].info_ = "Write and flush the logs every N seconds. Merely simulates MySQL 5.7." ; + ObSysVars[442].name_ = "innodb_flush_log_at_timeout" ; ObSysVars[442].data_type_ = ObIntType ; - ObSysVars[442].min_val_ = "0" ; - ObSysVars[442].max_val_ = "4294967295" ; ObSysVars[442].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[442].id_ = SYS_VAR_KEY_BUFFER_SIZE ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_KEY_BUFFER_SIZE)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_KEY_BUFFER_SIZE] = 442 ; - ObSysVars[442].base_value_ = "8388608" ; - ObSysVars[442].alias_ = "OB_SV_KEY_BUFFER_SIZE" ; + ObSysVars[442].id_ = SYS_VAR_INNODB_FLUSH_LOG_AT_TIMEOUT ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_FLUSH_LOG_AT_TIMEOUT)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_FLUSH_LOG_AT_TIMEOUT] = 442 ; + ObSysVars[442].base_value_ = "1" ; + ObSysVars[442].alias_ = "OB_SV_INNODB_FLUSH_LOG_AT_TIMEOUT" ; }(); [&] (){ - ObSysVars[443].default_value_ = "300" ; - ObSysVars[443].info_ = "This value controls the demotion of buffers from the hot sublist of a key cache to the warm sublist. Lower values cause demotion to happen more quickly, merely simulates MySQL 5.7" ; - ObSysVars[443].name_ = "key_cache_age_threshold" ; - ObSysVars[443].data_type_ = ObUInt64Type ; - ObSysVars[443].min_val_ = "100" ; - ObSysVars[443].max_val_ = "18446744073709551516" ; + ObSysVars[443].default_value_ = "1" ; + ObSysVars[443].info_ = "Controls the balance between strict ACID compliance for commit operations and higher performance. Merely simulates MySQL 5.7." ; + ObSysVars[443].name_ = "innodb_flush_log_at_trx_commit" ; + ObSysVars[443].data_type_ = ObIntType ; ObSysVars[443].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[443].id_ = SYS_VAR_KEY_CACHE_AGE_THRESHOLD ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_KEY_CACHE_AGE_THRESHOLD)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_KEY_CACHE_AGE_THRESHOLD] = 443 ; - ObSysVars[443].base_value_ = "300" ; - ObSysVars[443].alias_ = "OB_SV_KEY_CACHE_AGE_THRESHOLD" ; + ObSysVars[443].id_ = SYS_VAR_INNODB_FLUSH_LOG_AT_TRX_COMMIT ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_FLUSH_LOG_AT_TRX_COMMIT)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_FLUSH_LOG_AT_TRX_COMMIT] = 443 ; + ObSysVars[443].base_value_ = "1" ; + ObSysVars[443].alias_ = "OB_SV_INNODB_FLUSH_LOG_AT_TRX_COMMIT" ; }(); [&] (){ - ObSysVars[444].default_value_ = "100" ; - ObSysVars[444].info_ = "The division point between the hot and warm sublists of the key cache buffer list, merely simulates MySQL 5.7" ; - ObSysVars[444].name_ = "key_cache_division_limit" ; + ObSysVars[444].default_value_ = "0" ; + ObSysVars[444].info_ = "Enable this debug option to force InnoDB to write a checkpoint. Merely simulates MySQL 5.7." ; + ObSysVars[444].name_ = "innodb_log_checkpoint_now" ; ObSysVars[444].data_type_ = ObIntType ; - ObSysVars[444].min_val_ = "1" ; - ObSysVars[444].max_val_ = "100" ; ObSysVars[444].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[444].id_ = SYS_VAR_KEY_CACHE_DIVISION_LIMIT ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_KEY_CACHE_DIVISION_LIMIT)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_KEY_CACHE_DIVISION_LIMIT] = 444 ; - ObSysVars[444].base_value_ = "100" ; - ObSysVars[444].alias_ = "OB_SV_KEY_CACHE_DIVISION_LIMIT" ; + ObSysVars[444].id_ = SYS_VAR_INNODB_LOG_CHECKPOINT_NOW ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_LOG_CHECKPOINT_NOW)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_LOG_CHECKPOINT_NOW] = 444 ; + ObSysVars[444].base_value_ = "0" ; + ObSysVars[444].alias_ = "OB_SV_INNODB_LOG_CHECKPOINT_NOW" ; }(); [&] (){ - ObSysVars[445].default_value_ = "18446744073709551615" ; - ObSysVars[445].info_ = "Limit the assumed maximum number of seeks when looking up rows based on a key, merely simulates MySQL 5.7" ; - ObSysVars[445].name_ = "max_seeks_for_key" ; - ObSysVars[445].data_type_ = ObUInt64Type ; - ObSysVars[445].min_val_ = "1" ; - ObSysVars[445].max_val_ = "18446744073709551615" ; - ObSysVars[445].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[445].id_ = SYS_VAR_MAX_SEEKS_FOR_KEY ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_MAX_SEEKS_FOR_KEY)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_MAX_SEEKS_FOR_KEY] = 445 ; - ObSysVars[445].base_value_ = "18446744073709551615" ; - ObSysVars[445].alias_ = "OB_SV_MAX_SEEKS_FOR_KEY" ; + ObSysVars[445].default_value_ = "1" ; + ObSysVars[445].info_ = "Enables or disables checksums for redo log pages. Merely simulates MySQL 5.7." ; + ObSysVars[445].name_ = "innodb_log_checksums" ; + ObSysVars[445].data_type_ = ObIntType ; + ObSysVars[445].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[445].id_ = SYS_VAR_INNODB_LOG_CHECKSUMS ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_LOG_CHECKSUMS)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_LOG_CHECKSUMS] = 445 ; + ObSysVars[445].base_value_ = "1" ; + ObSysVars[445].alias_ = "OB_SV_INNODB_LOG_CHECKSUMS" ; }(); [&] (){ - ObSysVars[446].default_value_ = "0" ; - ObSysVars[446].info_ = "When this variable is enabled, the server does not use the optimized method of processing an ALTER TABLE operation, merely simulates MySQL 5.7" ; - ObSysVars[446].name_ = "old_alter_table" ; + ObSysVars[446].default_value_ = "1" ; + ObSysVars[446].info_ = "Specifies whether images of re-compressed pages are written to the redo log. Merely simulates MySQL 5.7." ; + ObSysVars[446].name_ = "innodb_log_compressed_pages" ; ObSysVars[446].data_type_ = ObIntType ; - ObSysVars[446].enum_names_ = "[u'OFF', u'ON']" ; - ObSysVars[446].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[446].id_ = SYS_VAR_OLD_ALTER_TABLE ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_OLD_ALTER_TABLE)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_OLD_ALTER_TABLE] = 446 ; - ObSysVars[446].base_value_ = "0" ; - ObSysVars[446].alias_ = "OB_SV_OLD_ALTER_TABLE" ; + ObSysVars[446].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[446].id_ = SYS_VAR_INNODB_LOG_COMPRESSED_PAGES ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_LOG_COMPRESSED_PAGES)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_LOG_COMPRESSED_PAGES] = 446 ; + ObSysVars[446].base_value_ = "1" ; + ObSysVars[446].alias_ = "OB_SV_INNODB_LOG_COMPRESSED_PAGES" ; }(); [&] (){ - ObSysVars[447].default_value_ = "-1" ; - ObSysVars[447].info_ = "The number of table definitions that can be stored in the table definition cache, merely simulates MySQL 5.7" ; - ObSysVars[447].name_ = "table_definition_cache" ; + ObSysVars[447].default_value_ = "8192" ; + ObSysVars[447].info_ = "Defines the write-ahead block size for the redo log, in bytes. Merely simulates MySQL 5.7." ; + ObSysVars[447].name_ = "innodb_log_write_ahead_size" ; ObSysVars[447].data_type_ = ObIntType ; - ObSysVars[447].min_val_ = "400" ; - ObSysVars[447].max_val_ = "524288" ; - ObSysVars[447].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[447].id_ = SYS_VAR_TABLE_DEFINITION_CACHE ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_TABLE_DEFINITION_CACHE)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_TABLE_DEFINITION_CACHE] = 447 ; - ObSysVars[447].base_value_ = "-1" ; - ObSysVars[447].alias_ = "OB_SV_TABLE_DEFINITION_CACHE" ; + ObSysVars[447].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[447].id_ = SYS_VAR_INNODB_LOG_WRITE_AHEAD_SIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_LOG_WRITE_AHEAD_SIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_LOG_WRITE_AHEAD_SIZE] = 447 ; + ObSysVars[447].base_value_ = "8192" ; + ObSysVars[447].alias_ = "OB_SV_INNODB_LOG_WRITE_AHEAD_SIZE" ; }(); [&] (){ - ObSysVars[448].default_value_ = "1048576" ; - ObSysVars[448].info_ = "The sort buffer size for online DDL operations that create or rebuild secondary indexes, merely simulates MySQL 5.7" ; - ObSysVars[448].name_ = "innodb_sort_buffer_size" ; + ObSysVars[448].default_value_ = "1073741824" ; + ObSysVars[448].info_ = "Defines a threshold size for undo tablespaces. Merely simulates MySQL 5.7." ; + ObSysVars[448].name_ = "innodb_max_undo_log_size" ; ObSysVars[448].data_type_ = ObIntType ; - ObSysVars[448].min_val_ = "65536" ; - ObSysVars[448].max_val_ = "67108864" ; - ObSysVars[448].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; - ObSysVars[448].id_ = SYS_VAR_INNODB_SORT_BUFFER_SIZE ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_SORT_BUFFER_SIZE)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_SORT_BUFFER_SIZE] = 448 ; - ObSysVars[448].base_value_ = "1048576" ; - ObSysVars[448].alias_ = "OB_SV_INNODB_SORT_BUFFER_SIZE" ; + ObSysVars[448].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[448].id_ = SYS_VAR_INNODB_MAX_UNDO_LOG_SIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_MAX_UNDO_LOG_SIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_MAX_UNDO_LOG_SIZE] = 448 ; + ObSysVars[448].base_value_ = "1073741824" ; + ObSysVars[448].alias_ = "OB_SV_INNODB_MAX_UNDO_LOG_SIZE" ; }(); [&] (){ - ObSysVars[449].default_value_ = "1024" ; - ObSysVars[449].info_ = "The size in bytes of blocks in the key cache, merely simulates MySQL 5.7" ; - ObSysVars[449].name_ = "key_cache_block_size" ; + ObSysVars[449].default_value_ = "134217728" ; + ObSysVars[449].info_ = "Specifies an upper limit in bytes on the size of the temporary log files used during online DDL operations for InnoDB tables. Merely simulates MySQL 5.7." ; + ObSysVars[449].name_ = "innodb_online_alter_log_max_size" ; ObSysVars[449].data_type_ = ObIntType ; - ObSysVars[449].min_val_ = "512" ; - ObSysVars[449].max_val_ = "16384" ; ObSysVars[449].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; - ObSysVars[449].id_ = SYS_VAR_KEY_CACHE_BLOCK_SIZE ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_KEY_CACHE_BLOCK_SIZE)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_KEY_CACHE_BLOCK_SIZE] = 449 ; - ObSysVars[449].base_value_ = "1024" ; - ObSysVars[449].alias_ = "OB_SV_KEY_CACHE_BLOCK_SIZE" ; + ObSysVars[449].id_ = SYS_VAR_INNODB_ONLINE_ALTER_LOG_MAX_SIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_ONLINE_ALTER_LOG_MAX_SIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_ONLINE_ALTER_LOG_MAX_SIZE] = 449 ; + ObSysVars[449].base_value_ = "134217728" ; + ObSysVars[449].alias_ = "OB_SV_INNODB_ONLINE_ALTER_LOG_MAX_SIZE" ; }(); [&] (){ ObSysVars[450].default_value_ = "0" ; - ObSysVars[450].info_ = "Use this variable to select the interface mode for the OBKV tenant. You can select one of 'ALL, TABLEAPI, HBASE, REDIS, NONE', where 'ALL' is the default and 'NONE' represents the non-OBKV interface mode." ; - ObSysVars[450].name_ = "ob_kv_mode" ; + ObSysVars[450].info_ = "When enabled, undo tablespaces that exceed the threshold value defined by innodb_max_undo_log_size are marked for truncation. Merely simulates MySQL 5.7." ; + ObSysVars[450].name_ = "innodb_undo_log_truncate" ; ObSysVars[450].data_type_ = ObIntType ; - ObSysVars[450].enum_names_ = "[u'ALL', u'TABLEAPI', u'HBASE', u'REDIS', u'NONE']" ; - ObSysVars[450].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::READONLY ; - ObSysVars[450].id_ = SYS_VAR_OB_KV_MODE ; - cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_OB_KV_MODE)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_OB_KV_MODE] = 450 ; + ObSysVars[450].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[450].id_ = SYS_VAR_INNODB_UNDO_LOG_TRUNCATE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_UNDO_LOG_TRUNCATE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_UNDO_LOG_TRUNCATE] = 450 ; ObSysVars[450].base_value_ = "0" ; - ObSysVars[450].alias_ = "OB_SV_KV_MODE" ; + ObSysVars[450].alias_ = "OB_SV_INNODB_UNDO_LOG_TRUNCATE" ; }(); [&] (){ - ObSysVars[451].default_value_ = "1" ; - ObSysVars[451].info_ = "wether use parameter anonymous_block" ; - ObSysVars[451].name_ = "ob_enable_parameter_anonymous_block" ; + ObSysVars[451].default_value_ = "128" ; + ObSysVars[451].info_ = "Defines the number of rollback segments used by InnoDB. Merely simulates MySQL 5.7." ; + ObSysVars[451].name_ = "innodb_undo_logs" ; ObSysVars[451].data_type_ = ObIntType ; - ObSysVars[451].flags_ = ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::NEED_SERIALIZE ; - ObSysVars[451].id_ = SYS_VAR_OB_ENABLE_PARAMETER_ANONYMOUS_BLOCK ; + ObSysVars[451].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[451].id_ = SYS_VAR_INNODB_UNDO_LOGS ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_UNDO_LOGS)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_UNDO_LOGS] = 451 ; + ObSysVars[451].base_value_ = "128" ; + ObSysVars[451].alias_ = "OB_SV_INNODB_UNDO_LOGS" ; + }(); + + [&] (){ + ObSysVars[452].default_value_ = "0" ; + ObSysVars[452].info_ = "It controls whether stored function creators can be trusted not to create stored functions that causes unsafe events to be written to the binary log. Merely simulates MySQL 5.7." ; + ObSysVars[452].name_ = "log_bin_trust_function_creators" ; + ObSysVars[452].data_type_ = ObIntType ; + ObSysVars[452].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[452].id_ = SYS_VAR_LOG_BIN_TRUST_FUNCTION_CREATORS ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_LOG_BIN_TRUST_FUNCTION_CREATORS)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_LOG_BIN_TRUST_FUNCTION_CREATORS] = 452 ; + ObSysVars[452].base_value_ = "0" ; + ObSysVars[452].alias_ = "OB_SV_LOG_BIN_TRUST_FUNCTION_CREATORS" ; + }(); + + [&] (){ + ObSysVars[453].default_value_ = "0" ; + ObSysVars[453].info_ = "Whether Version 2 binary logging is in use. Merely simulates MySQL 5.7." ; + ObSysVars[453].name_ = "log_bin_use_v1_row_events" ; + ObSysVars[453].data_type_ = ObIntType ; + ObSysVars[453].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[453].id_ = SYS_VAR_LOG_BIN_USE_V1_ROW_EVENTS ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_LOG_BIN_USE_V1_ROW_EVENTS)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_LOG_BIN_USE_V1_ROW_EVENTS] = 453 ; + ObSysVars[453].base_value_ = "0" ; + ObSysVars[453].alias_ = "OB_SV_LOG_BIN_USE_V1_ROW_EVENTS" ; + }(); + + [&] (){ + ObSysVars[454].default_value_ = "0" ; + ObSysVars[454].info_ = "This variable affects binary logging of user-management statements. Merely simulates MySQL 5.7." ; + ObSysVars[454].name_ = "log_builtin_as_identified_by_password" ; + ObSysVars[454].data_type_ = ObIntType ; + ObSysVars[454].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[454].id_ = SYS_VAR_LOG_BUILTIN_AS_IDENTIFIED_BY_PASSWORD ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_LOG_BUILTIN_AS_IDENTIFIED_BY_PASSWORD)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_LOG_BUILTIN_AS_IDENTIFIED_BY_PASSWORD] = 454 ; + ObSysVars[454].base_value_ = "0" ; + ObSysVars[454].alias_ = "OB_SV_LOG_BUILTIN_AS_IDENTIFIED_BY_PASSWORD" ; + }(); + + [&] (){ + ObSysVars[455].default_value_ = "18446744073709500416" ; + ObSysVars[455].info_ = "If a transaction requires more than this many bytes, the server generates a Multi-statement transaction required more than 'max_binlog_cache_size' bytes of storage error. Merely simulates MySQL 5.7." ; + ObSysVars[455].name_ = "max_binlog_cache_size" ; + ObSysVars[455].data_type_ = ObUInt64Type ; + ObSysVars[455].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[455].id_ = SYS_VAR_MAX_BINLOG_CACHE_SIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_MAX_BINLOG_CACHE_SIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_MAX_BINLOG_CACHE_SIZE] = 455 ; + ObSysVars[455].base_value_ = "18446744073709500416" ; + ObSysVars[455].alias_ = "OB_SV_MAX_BINLOG_CACHE_SIZE" ; + }(); + + [&] (){ + ObSysVars[456].default_value_ = "1073741824" ; + ObSysVars[456].info_ = "If a write to the binary log causes the current log file size to exceed the value of this variable, the server rotates the binary logs. Merely simulates MySQL 5.7." ; + ObSysVars[456].name_ = "max_binlog_size" ; + ObSysVars[456].data_type_ = ObIntType ; + ObSysVars[456].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[456].id_ = SYS_VAR_MAX_BINLOG_SIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_MAX_BINLOG_SIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_MAX_BINLOG_SIZE] = 456 ; + ObSysVars[456].base_value_ = "1073741824" ; + ObSysVars[456].alias_ = "OB_SV_MAX_BINLOG_SIZE" ; + }(); + + [&] (){ + ObSysVars[457].default_value_ = "18446744073709500416" ; + ObSysVars[457].info_ = "If nontransactional statements within a transaction require more than this many bytes of memory, the server generates an error. Merely simulates MySQL 5.7." ; + ObSysVars[457].name_ = "max_binlog_stmt_cache_size" ; + ObSysVars[457].data_type_ = ObUInt64Type ; + ObSysVars[457].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[457].id_ = SYS_VAR_MAX_BINLOG_STMT_CACHE_SIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_MAX_BINLOG_STMT_CACHE_SIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_MAX_BINLOG_STMT_CACHE_SIZE] = 457 ; + ObSysVars[457].base_value_ = "18446744073709500416" ; + ObSysVars[457].alias_ = "OB_SV_MAX_BINLOG_STMT_CACHE_SIZE" ; + }(); + + [&] (){ + ObSysVars[458].default_value_ = "0" ; + ObSysVars[458].info_ = "The size at which the server rotates relay log files automatically. Merely simulates MySQL 5.7." ; + ObSysVars[458].name_ = "max_relay_log_size" ; + ObSysVars[458].data_type_ = ObIntType ; + ObSysVars[458].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[458].id_ = SYS_VAR_MAX_RELAY_LOG_SIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_MAX_RELAY_LOG_SIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_MAX_RELAY_LOG_SIZE] = 458 ; + ObSysVars[458].base_value_ = "0" ; + ObSysVars[458].alias_ = "OB_SV_MAX_RELAY_LOG_SIZE" ; + }(); + + [&] (){ + ObSysVars[459].default_value_ = "FILE" ; + ObSysVars[459].info_ = "The setting of this variable determines whether the replica server stores its applier metadata repository as an InnoDB table in the mysql system database, or as a file in the data directory. Merely simulates MySQL 5.7." ; + ObSysVars[459].name_ = "relay_log_info_repository" ; + ObSysVars[459].data_type_ = ObVarcharType ; + ObSysVars[459].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[459].id_ = SYS_VAR_RELAY_LOG_INFO_REPOSITORY ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_RELAY_LOG_INFO_REPOSITORY)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_RELAY_LOG_INFO_REPOSITORY] = 459 ; + ObSysVars[459].base_value_ = "FILE" ; + ObSysVars[459].alias_ = "OB_SV_RELAY_LOG_INFO_REPOSITORY" ; + }(); + + [&] (){ + ObSysVars[460].default_value_ = "1" ; + ObSysVars[460].info_ = "Disables or enables automatic purging of relay log files as soon as they are not needed any more. Merely simulates MySQL 5.7." ; + ObSysVars[460].name_ = "relay_log_purge" ; + ObSysVars[460].data_type_ = ObIntType ; + ObSysVars[460].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[460].id_ = SYS_VAR_RELAY_LOG_PURGE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_RELAY_LOG_PURGE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_RELAY_LOG_PURGE] = 460 ; + ObSysVars[460].base_value_ = "1" ; + ObSysVars[460].alias_ = "OB_SV_RELAY_LOG_PURGE" ; + }(); + + [&] (){ + ObSysVars[461].default_value_ = "1" ; + ObSysVars[461].info_ = "Controls how often the MySQL server synchronizes the binary log to disk. Merely simulates MySQL 5.7." ; + ObSysVars[461].name_ = "sync_binlog" ; + ObSysVars[461].data_type_ = ObIntType ; + ObSysVars[461].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[461].id_ = SYS_VAR_SYNC_BINLOG ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_SYNC_BINLOG)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_SYNC_BINLOG] = 461 ; + ObSysVars[461].base_value_ = "1" ; + ObSysVars[461].alias_ = "OB_SV_SYNC_BINLOG" ; + }(); + + [&] (){ + ObSysVars[462].default_value_ = "10000" ; + ObSysVars[462].info_ = "If the value of this variable is greater than 0, the MySQL server synchronizes its relay log to disk after every sync_relay_log events are written to the relay log. Merely simulates MySQL 5.7." ; + ObSysVars[462].name_ = "sync_relay_log" ; + ObSysVars[462].data_type_ = ObIntType ; + ObSysVars[462].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[462].id_ = SYS_VAR_SYNC_RELAY_LOG ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_SYNC_RELAY_LOG)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_SYNC_RELAY_LOG] = 462 ; + ObSysVars[462].base_value_ = "10000" ; + ObSysVars[462].alias_ = "OB_SV_SYNC_RELAY_LOG" ; + }(); + + [&] (){ + ObSysVars[463].default_value_ = "10000" ; + ObSysVars[463].info_ = "Setting this variable takes effect for all replication channels immediately, including running channels. Merely simulates MySQL 5.7." ; + ObSysVars[463].name_ = "sync_relay_log_info" ; + ObSysVars[463].data_type_ = ObIntType ; + ObSysVars[463].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[463].id_ = SYS_VAR_SYNC_RELAY_LOG_INFO ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_SYNC_RELAY_LOG_INFO)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_SYNC_RELAY_LOG_INFO] = 463 ; + ObSysVars[463].base_value_ = "10000" ; + ObSysVars[463].alias_ = "OB_SV_SYNC_RELAY_LOG_INFO" ; + }(); + + [&] (){ + ObSysVars[464].default_value_ = "1" ; + ObSysVars[464].info_ = "This option is used to disable deadlock detection. Merely simulates MySQL 5.7." ; + ObSysVars[464].name_ = "innodb_deadlock_detect" ; + ObSysVars[464].data_type_ = ObIntType ; + ObSysVars[464].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[464].id_ = SYS_VAR_INNODB_DEADLOCK_DETECT ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_DEADLOCK_DETECT)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_DEADLOCK_DETECT] = 464 ; + ObSysVars[464].base_value_ = "1" ; + ObSysVars[464].alias_ = "OB_SV_INNODB_DEADLOCK_DETECT" ; + }(); + + [&] (){ + ObSysVars[465].default_value_ = "50" ; + ObSysVars[465].info_ = "The length of time in seconds an InnoDB transaction waits for a row lock before giving up. Merely simulates MySQL 5.7." ; + ObSysVars[465].name_ = "innodb_lock_wait_timeout" ; + ObSysVars[465].data_type_ = ObIntType ; + ObSysVars[465].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[465].id_ = SYS_VAR_INNODB_LOCK_WAIT_TIMEOUT ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_LOCK_WAIT_TIMEOUT)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_LOCK_WAIT_TIMEOUT] = 465 ; + ObSysVars[465].base_value_ = "50" ; + ObSysVars[465].alias_ = "OB_SV_INNODB_LOCK_WAIT_TIMEOUT" ; + }(); + + [&] (){ + ObSysVars[466].default_value_ = "0" ; + ObSysVars[466].info_ = "When this option is enabled, information about all deadlocks in InnoDB user transactions is recorded in the mysqld error log. Merely simulates MySQL 5.7." ; + ObSysVars[466].name_ = "innodb_print_all_deadlocks" ; + ObSysVars[466].data_type_ = ObIntType ; + ObSysVars[466].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[466].id_ = SYS_VAR_INNODB_PRINT_ALL_DEADLOCKS ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_PRINT_ALL_DEADLOCKS)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_PRINT_ALL_DEADLOCKS] = 466 ; + ObSysVars[466].base_value_ = "0" ; + ObSysVars[466].alias_ = "OB_SV_INNODB_PRINT_ALL_DEADLOCKS" ; + }(); + + [&] (){ + ObSysVars[467].default_value_ = "1" ; + ObSysVars[467].info_ = "The default value is 1, which means that LOCK TABLES causes InnoDB to lock a table internally if autocommit = 0. Merely simulates MySQL 5.7." ; + ObSysVars[467].name_ = "innodb_table_locks" ; + ObSysVars[467].data_type_ = ObIntType ; + ObSysVars[467].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[467].id_ = SYS_VAR_INNODB_TABLE_LOCKS ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_TABLE_LOCKS)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_TABLE_LOCKS] = 467 ; + ObSysVars[467].base_value_ = "1" ; + ObSysVars[467].alias_ = "OB_SV_INNODB_TABLE_LOCKS" ; + }(); + + [&] (){ + ObSysVars[468].default_value_ = "18446744073709500416" ; + ObSysVars[468].info_ = "if max_write_lock_count is set to some low value (say, 10), read lock requests may be preferred over pending write lock requests if the read lock requests have already been passed over in favor of 10 write lock requests. Merely simulates MySQL 5.7." ; + ObSysVars[468].name_ = "max_write_lock_count" ; + ObSysVars[468].data_type_ = ObUInt64Type ; + ObSysVars[468].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[468].id_ = SYS_VAR_MAX_WRITE_LOCK_COUNT ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_MAX_WRITE_LOCK_COUNT)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_MAX_WRITE_LOCK_COUNT] = 468 ; + ObSysVars[468].base_value_ = "18446744073709500416" ; + ObSysVars[468].alias_ = "OB_SV_MAX_WRITE_LOCK_COUNT" ; + }(); + + [&] (){ + ObSysVars[469].default_value_ = "" ; + ObSysVars[469].info_ = "enabled roles for current session" ; + ObSysVars[469].name_ = "_ob_enable_role_ids" ; + ObSysVars[469].data_type_ = ObVarcharType ; + ObSysVars[469].flags_ = ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::NEED_SERIALIZE | ObSysVarFlag::INVISIBLE ; + ObSysVars[469].id_ = SYS_VAR__OB_ENABLE_ROLE_IDS ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR__OB_ENABLE_ROLE_IDS)) ; + ObSysVarsIdToArrayIdx[SYS_VAR__OB_ENABLE_ROLE_IDS] = 469 ; + ObSysVars[469].base_value_ = "" ; + ObSysVars[469].alias_ = "OB_SV__OB_ENABLE_ROLE_IDS" ; + }(); + + [&] (){ + ObSysVars[470].default_value_ = "0" ; + ObSysVars[470].info_ = "Starts InnoDB in read-only mode. For distributing database applications or data sets on read-only media. Can also be used in data warehouses to share the same data directory between multiple instances." ; + ObSysVars[470].name_ = "innodb_read_only" ; + ObSysVars[470].data_type_ = ObIntType ; + ObSysVars[470].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[470].id_ = SYS_VAR_INNODB_READ_ONLY ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_READ_ONLY)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_READ_ONLY] = 470 ; + ObSysVars[470].base_value_ = "0" ; + ObSysVars[470].alias_ = "OB_SV_INNODB_READ_ONLY" ; + }(); + + [&] (){ + ObSysVars[471].default_value_ = "0" ; + ObSysVars[471].info_ = "Use this option to disable row locks when InnoDB memcached performs DML operations. By default, innodb_api_disable_rowlock is disabled, which means that memcached requests row locks for get and set operations." ; + ObSysVars[471].name_ = "innodb_api_disable_rowlock" ; + ObSysVars[471].data_type_ = ObIntType ; + ObSysVars[471].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[471].id_ = SYS_VAR_INNODB_API_DISABLE_ROWLOCK ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_API_DISABLE_ROWLOCK)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_API_DISABLE_ROWLOCK] = 471 ; + ObSysVars[471].base_value_ = "0" ; + ObSysVars[471].alias_ = "OB_SV_INNODB_API_DISABLE_ROWLOCK" ; + }(); + + [&] (){ + ObSysVars[472].default_value_ = "1" ; + ObSysVars[472].info_ = "The lock mode to use for generating auto-increment values. Permissible values are 0, 1, or 2, for traditional, consecutive, or interleaved, respectively. The default setting is 1 (consecutive)." ; + ObSysVars[472].name_ = "innodb_autoinc_lock_mode" ; + ObSysVars[472].data_type_ = ObIntType ; + ObSysVars[472].min_val_ = "0" ; + ObSysVars[472].max_val_ = "2" ; + ObSysVars[472].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[472].id_ = SYS_VAR_INNODB_AUTOINC_LOCK_MODE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_AUTOINC_LOCK_MODE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_AUTOINC_LOCK_MODE] = 472 ; + ObSysVars[472].base_value_ = "1" ; + ObSysVars[472].alias_ = "OB_SV_INNODB_AUTOINC_LOCK_MODE" ; + }(); + + [&] (){ + ObSysVars[473].default_value_ = "1" ; + ObSysVars[473].info_ = "This is OFF if mysqld uses external locking (system locking), ON if external locking is disabled." ; + ObSysVars[473].name_ = "skip_external_locking" ; + ObSysVars[473].data_type_ = ObIntType ; + ObSysVars[473].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[473].id_ = SYS_VAR_SKIP_EXTERNAL_LOCKING ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_SKIP_EXTERNAL_LOCKING)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_SKIP_EXTERNAL_LOCKING] = 473 ; + ObSysVars[473].base_value_ = "1" ; + ObSysVars[473].alias_ = "OB_SV_SKIP_EXTERNAL_LOCKING" ; + }(); + + [&] (){ + ObSysVars[474].default_value_ = "0" ; + ObSysVars[474].info_ = "If the read_only system variable is enabled, the server permits no client updates except from users who have the SUPER privilege. If the super_read_only system variable is also enabled, the server prohibits client updates even from users who have SUPER." ; + ObSysVars[474].name_ = "super_read_only" ; + ObSysVars[474].data_type_ = ObIntType ; + ObSysVars[474].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[474].id_ = SYS_VAR_SUPER_READ_ONLY ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_SUPER_READ_ONLY)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_SUPER_READ_ONLY] = 474 ; + ObSysVars[474].base_value_ = "0" ; + ObSysVars[474].alias_ = "OB_SV_SUPER_READ_ONLY" ; + }(); + + [&] (){ + ObSysVars[475].default_value_ = "2" ; + ObSysVars[475].info_ = "PLSQL_OPTIMIZE_LEVEL specifies the optimization level that will be used to compile PL/SQL library units. The higher the setting of this parameter, the more effort the compiler makes to optimize PL/SQL library units." ; + ObSysVars[475].name_ = "plsql_optimize_level" ; + ObSysVars[475].data_type_ = ObIntType ; + ObSysVars[475].flags_ = ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::NEED_SERIALIZE ; + ObSysVars[475].id_ = SYS_VAR_PLSQL_OPTIMIZE_LEVEL ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_PLSQL_OPTIMIZE_LEVEL)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_PLSQL_OPTIMIZE_LEVEL] = 475 ; + ObSysVars[475].base_value_ = "2" ; + ObSysVars[475].alias_ = "OB_SV_PLSQL_OPTIMIZE_LEVEL" ; + }(); + + [&] (){ + ObSysVars[476].default_value_ = "built-in" ; + ObSysVars[476].info_ = "mock for mysql5.7" ; + ObSysVars[476].name_ = "ft_stopword_file" ; + ObSysVars[476].data_type_ = ObVarcharType ; + ObSysVars[476].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[476].id_ = SYS_VAR_FT_STOPWORD_FILE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_FT_STOPWORD_FILE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_FT_STOPWORD_FILE] = 476 ; + ObSysVars[476].base_value_ = "built-in" ; + ObSysVars[476].alias_ = "OB_SV_FT_STOPWORD_FILE" ; + }(); + + [&] (){ + ObSysVars[477].default_value_ = "8000000" ; + ObSysVars[477].info_ = "mock for mysql5.7" ; + ObSysVars[477].name_ = "innodb_ft_cache_size" ; + ObSysVars[477].data_type_ = ObIntType ; + ObSysVars[477].min_val_ = "1600000" ; + ObSysVars[477].max_val_ = "80000000" ; + ObSysVars[477].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[477].id_ = SYS_VAR_INNODB_FT_CACHE_SIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_FT_CACHE_SIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_FT_CACHE_SIZE] = 477 ; + ObSysVars[477].base_value_ = "8000000" ; + ObSysVars[477].alias_ = "OB_SV_INNODB_FT_CACHE_SIZE" ; + }(); + + [&] (){ + ObSysVars[478].default_value_ = "2" ; + ObSysVars[478].info_ = "mock for mysql5.7" ; + ObSysVars[478].name_ = "innodb_ft_sort_pll_degree" ; + ObSysVars[478].data_type_ = ObIntType ; + ObSysVars[478].min_val_ = "1" ; + ObSysVars[478].max_val_ = "16" ; + ObSysVars[478].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[478].id_ = SYS_VAR_INNODB_FT_SORT_PLL_DEGREE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_FT_SORT_PLL_DEGREE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_FT_SORT_PLL_DEGREE] = 478 ; + ObSysVars[478].base_value_ = "2" ; + ObSysVars[478].alias_ = "OB_SV_INNODB_FT_SORT_PLL_DEGREE" ; + }(); + + [&] (){ + ObSysVars[479].default_value_ = "640000000" ; + ObSysVars[479].info_ = "mock for mysql5.7" ; + ObSysVars[479].name_ = "innodb_ft_total_cache_size" ; + ObSysVars[479].data_type_ = ObIntType ; + ObSysVars[479].min_val_ = "32000000" ; + ObSysVars[479].max_val_ = "1600000000" ; + ObSysVars[479].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[479].id_ = SYS_VAR_INNODB_FT_TOTAL_CACHE_SIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_FT_TOTAL_CACHE_SIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_FT_TOTAL_CACHE_SIZE] = 479 ; + ObSysVars[479].base_value_ = "640000000" ; + ObSysVars[479].alias_ = "OB_SV_INNODB_FT_TOTAL_CACHE_SIZE" ; + }(); + + [&] (){ + ObSysVars[480].default_value_ = "" ; + ObSysVars[480].info_ = "mock for mysql5.7" ; + ObSysVars[480].name_ = "mecab_rc_file" ; + ObSysVars[480].data_type_ = ObVarcharType ; + ObSysVars[480].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[480].id_ = SYS_VAR_MECAB_RC_FILE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_MECAB_RC_FILE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_MECAB_RC_FILE] = 480 ; + ObSysVars[480].base_value_ = "" ; + ObSysVars[480].alias_ = "OB_SV_MECAB_RC_FILE" ; + }(); + + [&] (){ + ObSysVars[481].default_value_ = "1024" ; + ObSysVars[481].info_ = "mock for mysql5.7" ; + ObSysVars[481].name_ = "metadata_locks_cache_size" ; + ObSysVars[481].data_type_ = ObIntType ; + ObSysVars[481].min_val_ = "1" ; + ObSysVars[481].max_val_ = "1048576" ; + ObSysVars[481].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[481].id_ = SYS_VAR_METADATA_LOCKS_CACHE_SIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_METADATA_LOCKS_CACHE_SIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_METADATA_LOCKS_CACHE_SIZE] = 481 ; + ObSysVars[481].base_value_ = "1024" ; + ObSysVars[481].alias_ = "OB_SV_METADATA_LOCKS_CACHE_SIZE" ; + }(); + + [&] (){ + ObSysVars[482].default_value_ = "8" ; + ObSysVars[482].info_ = "mock for mysql5.7" ; + ObSysVars[482].name_ = "metadata_locks_hash_instances" ; + ObSysVars[482].data_type_ = ObIntType ; + ObSysVars[482].min_val_ = "1" ; + ObSysVars[482].max_val_ = "1024" ; + ObSysVars[482].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[482].id_ = SYS_VAR_METADATA_LOCKS_HASH_INSTANCES ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_METADATA_LOCKS_HASH_INSTANCES)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_METADATA_LOCKS_HASH_INSTANCES] = 482 ; + ObSysVars[482].base_value_ = "8" ; + ObSysVars[482].alias_ = "OB_SV_METADATA_LOCKS_HASH_INSTANCES" ; + }(); + + [&] (){ + ObSysVars[483].default_value_ = "ibtmp1:12M:autoextend" ; + ObSysVars[483].info_ = "mock for mysql5.7" ; + ObSysVars[483].name_ = "innodb_temp_data_file_path" ; + ObSysVars[483].data_type_ = ObVarcharType ; + ObSysVars[483].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[483].id_ = SYS_VAR_INNODB_TEMP_DATA_FILE_PATH ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_TEMP_DATA_FILE_PATH)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_TEMP_DATA_FILE_PATH] = 483 ; + ObSysVars[483].base_value_ = "ibtmp1:12M:autoextend" ; + ObSysVars[483].alias_ = "OB_SV_INNODB_TEMP_DATA_FILE_PATH" ; + }(); + + [&] (){ + ObSysVars[484].default_value_ = "ibdata1:12M:autoextend" ; + ObSysVars[484].info_ = "mock for mysql5.7" ; + ObSysVars[484].name_ = "innodb_data_file_path" ; + ObSysVars[484].data_type_ = ObVarcharType ; + ObSysVars[484].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[484].id_ = SYS_VAR_INNODB_DATA_FILE_PATH ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_DATA_FILE_PATH)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_DATA_FILE_PATH] = 484 ; + ObSysVars[484].base_value_ = "ibdata1:12M:autoextend" ; + ObSysVars[484].alias_ = "OB_SV_INNODB_DATA_FILE_PATH" ; + }(); + + [&] (){ + ObSysVars[485].default_value_ = "" ; + ObSysVars[485].info_ = "mock for mysql5.7" ; + ObSysVars[485].name_ = "innodb_data_home_dir" ; + ObSysVars[485].data_type_ = ObVarcharType ; + ObSysVars[485].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[485].id_ = SYS_VAR_INNODB_DATA_HOME_DIR ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_DATA_HOME_DIR)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_DATA_HOME_DIR] = 485 ; + ObSysVars[485].base_value_ = "" ; + ObSysVars[485].alias_ = "OB_SV_INNODB_DATA_HOME_DIR" ; + }(); + + [&] (){ + ObSysVars[486].default_value_ = "0" ; + ObSysVars[486].info_ = "mock for mysql5.7" ; + ObSysVars[486].name_ = "avoid_temporal_upgrade" ; + ObSysVars[486].data_type_ = ObIntType ; + ObSysVars[486].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[486].id_ = SYS_VAR_AVOID_TEMPORAL_UPGRADE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_AVOID_TEMPORAL_UPGRADE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_AVOID_TEMPORAL_UPGRADE] = 486 ; + ObSysVars[486].base_value_ = "0" ; + ObSysVars[486].alias_ = "OB_SV_AVOID_TEMPORAL_UPGRADE" ; + }(); + + [&] (){ + ObSysVars[487].default_value_ = "0" ; + ObSysVars[487].info_ = "mock for mysql5.7" ; + ObSysVars[487].name_ = "default_tmp_storage_engine" ; + ObSysVars[487].data_type_ = ObIntType ; + ObSysVars[487].enum_names_ = "[u'InnoDB']" ; + ObSysVars[487].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[487].id_ = SYS_VAR_DEFAULT_TMP_STORAGE_ENGINE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_DEFAULT_TMP_STORAGE_ENGINE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_DEFAULT_TMP_STORAGE_ENGINE] = 487 ; + ObSysVars[487].base_value_ = "0" ; + ObSysVars[487].alias_ = "OB_SV_DEFAULT_TMP_STORAGE_ENGINE" ; + }(); + + [&] (){ + ObSysVars[488].default_value_ = "0" ; + ObSysVars[488].info_ = "mock for mysql5.7" ; + ObSysVars[488].name_ = "innodb_ft_enable_diag_print" ; + ObSysVars[488].data_type_ = ObIntType ; + ObSysVars[488].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[488].id_ = SYS_VAR_INNODB_FT_ENABLE_DIAG_PRINT ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_FT_ENABLE_DIAG_PRINT)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_FT_ENABLE_DIAG_PRINT] = 488 ; + ObSysVars[488].base_value_ = "0" ; + ObSysVars[488].alias_ = "OB_SV_INNODB_FT_ENABLE_DIAG_PRINT" ; + }(); + + [&] (){ + ObSysVars[489].default_value_ = "2000" ; + ObSysVars[489].info_ = "mock for mysql5.7" ; + ObSysVars[489].name_ = "innodb_ft_num_word_optimize" ; + ObSysVars[489].data_type_ = ObIntType ; + ObSysVars[489].min_val_ = "1000" ; + ObSysVars[489].max_val_ = "10000" ; + ObSysVars[489].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[489].id_ = SYS_VAR_INNODB_FT_NUM_WORD_OPTIMIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_FT_NUM_WORD_OPTIMIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_FT_NUM_WORD_OPTIMIZE] = 489 ; + ObSysVars[489].base_value_ = "2000" ; + ObSysVars[489].alias_ = "OB_SV_INNODB_FT_NUM_WORD_OPTIMIZE" ; + }(); + + [&] (){ + ObSysVars[490].default_value_ = "2000000000" ; + ObSysVars[490].info_ = "mock for mysql5.7" ; + ObSysVars[490].name_ = "innodb_ft_result_cache_limit" ; + ObSysVars[490].data_type_ = ObUInt64Type ; + ObSysVars[490].min_val_ = "1000000" ; + ObSysVars[490].max_val_ = "4294967295" ; + ObSysVars[490].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[490].id_ = SYS_VAR_INNODB_FT_RESULT_CACHE_LIMIT ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_FT_RESULT_CACHE_LIMIT)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_FT_RESULT_CACHE_LIMIT] = 490 ; + ObSysVars[490].base_value_ = "2000000000" ; + ObSysVars[490].alias_ = "OB_SV_INNODB_FT_RESULT_CACHE_LIMIT" ; + }(); + + [&] (){ + ObSysVars[491].default_value_ = "" ; + ObSysVars[491].info_ = "mock for mysql5.7" ; + ObSysVars[491].name_ = "innodb_ft_server_stopword_table" ; + ObSysVars[491].data_type_ = ObVarcharType ; + ObSysVars[491].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[491].id_ = SYS_VAR_INNODB_FT_SERVER_STOPWORD_TABLE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_FT_SERVER_STOPWORD_TABLE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_FT_SERVER_STOPWORD_TABLE] = 491 ; + ObSysVars[491].base_value_ = "" ; + ObSysVars[491].alias_ = "OB_SV_INNODB_FT_SERVER_STOPWORD_TABLE" ; + }(); + + [&] (){ + ObSysVars[492].default_value_ = "0" ; + ObSysVars[492].info_ = "mock for mysql5.7" ; + ObSysVars[492].name_ = "innodb_optimize_fulltext_only" ; + ObSysVars[492].data_type_ = ObIntType ; + ObSysVars[492].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[492].id_ = SYS_VAR_INNODB_OPTIMIZE_FULLTEXT_ONLY ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_OPTIMIZE_FULLTEXT_ONLY)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_OPTIMIZE_FULLTEXT_ONLY] = 492 ; + ObSysVars[492].base_value_ = "0" ; + ObSysVars[492].alias_ = "OB_SV_INNODB_OPTIMIZE_FULLTEXT_ONLY" ; + }(); + + [&] (){ + ObSysVars[493].default_value_ = "32" ; + ObSysVars[493].info_ = "mock for mysql5.7" ; + ObSysVars[493].name_ = "max_tmp_tables" ; + ObSysVars[493].data_type_ = ObIntType ; + ObSysVars[493].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[493].id_ = SYS_VAR_MAX_TMP_TABLES ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_MAX_TMP_TABLES)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_MAX_TMP_TABLES] = 493 ; + ObSysVars[493].base_value_ = "32" ; + ObSysVars[493].alias_ = "OB_SV_MAX_TMP_TABLES" ; + }(); + + [&] (){ + ObSysVars[494].default_value_ = "" ; + ObSysVars[494].info_ = "mock for mysql5.7" ; + ObSysVars[494].name_ = "innodb_tmpdir" ; + ObSysVars[494].data_type_ = ObVarcharType ; + ObSysVars[494].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[494].id_ = SYS_VAR_INNODB_TMPDIR ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_TMPDIR)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_TMPDIR] = 494 ; + ObSysVars[494].base_value_ = "" ; + ObSysVars[494].alias_ = "OB_SV_INNODB_TMPDIR" ; + }(); + + [&] (){ + ObSysVars[495].default_value_ = "" ; + ObSysVars[495].info_ = "A list of group members to which a joining member can connect to obtain details of all the current group members" ; + ObSysVars[495].name_ = "group_replication_group_seeds" ; + ObSysVars[495].data_type_ = ObVarcharType ; + ObSysVars[495].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE ; + ObSysVars[495].id_ = SYS_VAR_GROUP_REPLICATION_GROUP_SEEDS ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_GROUP_REPLICATION_GROUP_SEEDS)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_GROUP_REPLICATION_GROUP_SEEDS] = 495 ; + ObSysVars[495].base_value_ = "" ; + ObSysVars[495].alias_ = "OB_SV_GROUP_REPLICATION_GROUP_SEEDS" ; + }(); + + [&] (){ + ObSysVars[496].default_value_ = "0" ; + ObSysVars[496].info_ = "When preparing batches of rows for row-based logging and replication, this variable controls how the rows are searched for matches" ; + ObSysVars[496].name_ = "slave_rows_search_algorithms" ; + ObSysVars[496].data_type_ = ObIntType ; + ObSysVars[496].enum_names_ = "[u'TABLE_SCAN,INDEX_SCAN', u'INDEX_SCAN,HASH_SCAN', u'TABLE_SCAN,HASH_SCAN', u'TABLE_SCAN,INDEX_SCAN,HASH_SCAN']" ; + ObSysVars[496].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE ; + ObSysVars[496].id_ = SYS_VAR_SLAVE_ROWS_SEARCH_ALGORITHMS ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_SLAVE_ROWS_SEARCH_ALGORITHMS)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_SLAVE_ROWS_SEARCH_ALGORITHMS] = 496 ; + ObSysVars[496].base_value_ = "0" ; + ObSysVars[496].alias_ = "OB_SV_SLAVE_ROWS_SEARCH_ALGORITHMS" ; + }(); + + [&] (){ + ObSysVars[497].default_value_ = "0" ; + ObSysVars[497].info_ = "Controls the type conversion mode in effect on the replica when using row-based replication" ; + ObSysVars[497].name_ = "slave_type_conversions" ; + ObSysVars[497].data_type_ = ObIntType ; + ObSysVars[497].enum_names_ = "[u'ALL_LOSSY', u'ALL_NON_LOSSY', u'ALL_SIGNED', u'ALL_UNSIGNED']" ; + ObSysVars[497].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE ; + ObSysVars[497].id_ = SYS_VAR_SLAVE_TYPE_CONVERSIONS ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_SLAVE_TYPE_CONVERSIONS)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_SLAVE_TYPE_CONVERSIONS] = 497 ; + ObSysVars[497].base_value_ = "0" ; + ObSysVars[497].alias_ = "OB_SV_SLAVE_TYPE_CONVERSIONS" ; + }(); + + [&] (){ + ObSysVars[498].default_value_ = "0" ; + ObSysVars[498].info_ = "This variable specifies how to use delayed key writes. It applies only to MyISAM tables. Delayed key writing causes key buffers not to be flushed between writes, merely simulates MySQL 5.7" ; + ObSysVars[498].name_ = "delay_key_write" ; + ObSysVars[498].data_type_ = ObIntType ; + ObSysVars[498].enum_names_ = "[u'ON', u'OFF', u'ALL']" ; + ObSysVars[498].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[498].id_ = SYS_VAR_DELAY_KEY_WRITE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_DELAY_KEY_WRITE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_DELAY_KEY_WRITE] = 498 ; + ObSysVars[498].base_value_ = "0" ; + ObSysVars[498].alias_ = "OB_SV_DELAY_KEY_WRITE" ; + }(); + + [&] (){ + ObSysVars[499].default_value_ = "0" ; + ObSysVars[499].info_ = "When this option is enabled, index key prefixes longer than 767 bytes (up to 3072 bytes) are allowed for InnoDB tables that use DYNAMIC or COMPRESSED row format, merely simulates MySQL 5.7" ; + ObSysVars[499].name_ = "innodb_large_prefix" ; + ObSysVars[499].data_type_ = ObIntType ; + ObSysVars[499].enum_names_ = "[u'ON', u'OFF']" ; + ObSysVars[499].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[499].id_ = SYS_VAR_INNODB_LARGE_PREFIX ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_LARGE_PREFIX)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_LARGE_PREFIX] = 499 ; + ObSysVars[499].base_value_ = "0" ; + ObSysVars[499].alias_ = "OB_SV_INNODB_LARGE_PREFIX" ; + }(); + + [&] (){ + ObSysVars[500].default_value_ = "8388608" ; + ObSysVars[500].info_ = "The size of the buffer used for index blocks, merely simulates MySQL 5.7" ; + ObSysVars[500].name_ = "key_buffer_size" ; + ObSysVars[500].data_type_ = ObIntType ; + ObSysVars[500].min_val_ = "0" ; + ObSysVars[500].max_val_ = "4294967295" ; + ObSysVars[500].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[500].id_ = SYS_VAR_KEY_BUFFER_SIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_KEY_BUFFER_SIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_KEY_BUFFER_SIZE] = 500 ; + ObSysVars[500].base_value_ = "8388608" ; + ObSysVars[500].alias_ = "OB_SV_KEY_BUFFER_SIZE" ; + }(); + + [&] (){ + ObSysVars[501].default_value_ = "300" ; + ObSysVars[501].info_ = "This value controls the demotion of buffers from the hot sublist of a key cache to the warm sublist. Lower values cause demotion to happen more quickly, merely simulates MySQL 5.7" ; + ObSysVars[501].name_ = "key_cache_age_threshold" ; + ObSysVars[501].data_type_ = ObUInt64Type ; + ObSysVars[501].min_val_ = "100" ; + ObSysVars[501].max_val_ = "18446744073709551516" ; + ObSysVars[501].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[501].id_ = SYS_VAR_KEY_CACHE_AGE_THRESHOLD ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_KEY_CACHE_AGE_THRESHOLD)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_KEY_CACHE_AGE_THRESHOLD] = 501 ; + ObSysVars[501].base_value_ = "300" ; + ObSysVars[501].alias_ = "OB_SV_KEY_CACHE_AGE_THRESHOLD" ; + }(); + + [&] (){ + ObSysVars[502].default_value_ = "100" ; + ObSysVars[502].info_ = "The division point between the hot and warm sublists of the key cache buffer list, merely simulates MySQL 5.7" ; + ObSysVars[502].name_ = "key_cache_division_limit" ; + ObSysVars[502].data_type_ = ObIntType ; + ObSysVars[502].min_val_ = "1" ; + ObSysVars[502].max_val_ = "100" ; + ObSysVars[502].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[502].id_ = SYS_VAR_KEY_CACHE_DIVISION_LIMIT ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_KEY_CACHE_DIVISION_LIMIT)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_KEY_CACHE_DIVISION_LIMIT] = 502 ; + ObSysVars[502].base_value_ = "100" ; + ObSysVars[502].alias_ = "OB_SV_KEY_CACHE_DIVISION_LIMIT" ; + }(); + + [&] (){ + ObSysVars[503].default_value_ = "18446744073709551615" ; + ObSysVars[503].info_ = "Limit the assumed maximum number of seeks when looking up rows based on a key, merely simulates MySQL 5.7" ; + ObSysVars[503].name_ = "max_seeks_for_key" ; + ObSysVars[503].data_type_ = ObUInt64Type ; + ObSysVars[503].min_val_ = "1" ; + ObSysVars[503].max_val_ = "18446744073709551615" ; + ObSysVars[503].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[503].id_ = SYS_VAR_MAX_SEEKS_FOR_KEY ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_MAX_SEEKS_FOR_KEY)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_MAX_SEEKS_FOR_KEY] = 503 ; + ObSysVars[503].base_value_ = "18446744073709551615" ; + ObSysVars[503].alias_ = "OB_SV_MAX_SEEKS_FOR_KEY" ; + }(); + + [&] (){ + ObSysVars[504].default_value_ = "0" ; + ObSysVars[504].info_ = "When this variable is enabled, the server does not use the optimized method of processing an ALTER TABLE operation, merely simulates MySQL 5.7" ; + ObSysVars[504].name_ = "old_alter_table" ; + ObSysVars[504].data_type_ = ObIntType ; + ObSysVars[504].enum_names_ = "[u'OFF', u'ON']" ; + ObSysVars[504].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[504].id_ = SYS_VAR_OLD_ALTER_TABLE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_OLD_ALTER_TABLE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_OLD_ALTER_TABLE] = 504 ; + ObSysVars[504].base_value_ = "0" ; + ObSysVars[504].alias_ = "OB_SV_OLD_ALTER_TABLE" ; + }(); + + [&] (){ + ObSysVars[505].default_value_ = "-1" ; + ObSysVars[505].info_ = "The number of table definitions that can be stored in the table definition cache, merely simulates MySQL 5.7" ; + ObSysVars[505].name_ = "table_definition_cache" ; + ObSysVars[505].data_type_ = ObIntType ; + ObSysVars[505].min_val_ = "400" ; + ObSysVars[505].max_val_ = "524288" ; + ObSysVars[505].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[505].id_ = SYS_VAR_TABLE_DEFINITION_CACHE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_TABLE_DEFINITION_CACHE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_TABLE_DEFINITION_CACHE] = 505 ; + ObSysVars[505].base_value_ = "-1" ; + ObSysVars[505].alias_ = "OB_SV_TABLE_DEFINITION_CACHE" ; + }(); + + [&] (){ + ObSysVars[506].default_value_ = "1048576" ; + ObSysVars[506].info_ = "The sort buffer size for online DDL operations that create or rebuild secondary indexes, merely simulates MySQL 5.7" ; + ObSysVars[506].name_ = "innodb_sort_buffer_size" ; + ObSysVars[506].data_type_ = ObIntType ; + ObSysVars[506].min_val_ = "65536" ; + ObSysVars[506].max_val_ = "67108864" ; + ObSysVars[506].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[506].id_ = SYS_VAR_INNODB_SORT_BUFFER_SIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_SORT_BUFFER_SIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_SORT_BUFFER_SIZE] = 506 ; + ObSysVars[506].base_value_ = "1048576" ; + ObSysVars[506].alias_ = "OB_SV_INNODB_SORT_BUFFER_SIZE" ; + }(); + + [&] (){ + ObSysVars[507].default_value_ = "1024" ; + ObSysVars[507].info_ = "The size in bytes of blocks in the key cache, merely simulates MySQL 5.7" ; + ObSysVars[507].name_ = "key_cache_block_size" ; + ObSysVars[507].data_type_ = ObIntType ; + ObSysVars[507].min_val_ = "512" ; + ObSysVars[507].max_val_ = "16384" ; + ObSysVars[507].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[507].id_ = SYS_VAR_KEY_CACHE_BLOCK_SIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_KEY_CACHE_BLOCK_SIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_KEY_CACHE_BLOCK_SIZE] = 507 ; + ObSysVars[507].base_value_ = "1024" ; + ObSysVars[507].alias_ = "OB_SV_KEY_CACHE_BLOCK_SIZE" ; + }(); + + [&] (){ + ObSysVars[508].default_value_ = "0" ; + ObSysVars[508].info_ = "Use this variable to select the interface mode for the OBKV tenant. You can select one of 'ALL, TABLEAPI, HBASE, REDIS, NONE', where 'ALL' is the default and 'NONE' represents the non-OBKV interface mode." ; + ObSysVars[508].name_ = "ob_kv_mode" ; + ObSysVars[508].data_type_ = ObIntType ; + ObSysVars[508].enum_names_ = "[u'ALL', u'TABLEAPI', u'HBASE', u'REDIS', u'NONE']" ; + ObSysVars[508].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::READONLY ; + ObSysVars[508].id_ = SYS_VAR_OB_KV_MODE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_OB_KV_MODE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_OB_KV_MODE] = 508 ; + ObSysVars[508].base_value_ = "0" ; + ObSysVars[508].alias_ = "OB_SV_KV_MODE" ; + }(); + + [&] (){ + ObSysVars[509].default_value_ = "1" ; + ObSysVars[509].info_ = "wether use parameter anonymous_block" ; + ObSysVars[509].name_ = "ob_enable_parameter_anonymous_block" ; + ObSysVars[509].data_type_ = ObIntType ; + ObSysVars[509].flags_ = ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::NEED_SERIALIZE ; + ObSysVars[509].id_ = SYS_VAR_OB_ENABLE_PARAMETER_ANONYMOUS_BLOCK ; cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_OB_ENABLE_PARAMETER_ANONYMOUS_BLOCK)) ; - ObSysVarsIdToArrayIdx[SYS_VAR_OB_ENABLE_PARAMETER_ANONYMOUS_BLOCK] = 451 ; - ObSysVars[451].base_value_ = "1" ; - ObSysVars[451].alias_ = "OB_SV_ENABLE_PARAMETER_ANONYMOUS_BLOCK" ; + ObSysVarsIdToArrayIdx[SYS_VAR_OB_ENABLE_PARAMETER_ANONYMOUS_BLOCK] = 509 ; + ObSysVars[509].base_value_ = "1" ; + ObSysVars[509].alias_ = "OB_SV_ENABLE_PARAMETER_ANONYMOUS_BLOCK" ; + }(); + + [&] (){ + ObSysVars[510].default_value_ = "" ; + ObSysVars[510].info_ = "The directory where character sets are installed" ; + ObSysVars[510].name_ = "character_sets_dir" ; + ObSysVars[510].data_type_ = ObVarcharType ; + ObSysVars[510].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[510].id_ = SYS_VAR_CHARACTER_SETS_DIR ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_CHARACTER_SETS_DIR)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_CHARACTER_SETS_DIR] = 510 ; + ObSysVars[510].base_value_ = "" ; + ObSysVars[510].alias_ = "OB_SV_CHARACTER_SETS_DIR" ; + }(); + + [&] (){ + ObSysVars[511].default_value_ = "%Y-%m-%d" ; + ObSysVars[511].info_ = "" ; + ObSysVars[511].name_ = "date_format" ; + ObSysVars[511].data_type_ = ObVarcharType ; + ObSysVars[511].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[511].id_ = SYS_VAR_DATE_FORMAT ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_DATE_FORMAT)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_DATE_FORMAT] = 511 ; + ObSysVars[511].base_value_ = "%Y-%m-%d" ; + ObSysVars[511].alias_ = "OB_SV_DATE_FORMAT" ; + }(); + + [&] (){ + ObSysVars[512].default_value_ = "%Y-%m-%d %H:%i:%s" ; + ObSysVars[512].info_ = "" ; + ObSysVars[512].name_ = "datetime_format" ; + ObSysVars[512].data_type_ = ObVarcharType ; + ObSysVars[512].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[512].id_ = SYS_VAR_DATETIME_FORMAT ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_DATETIME_FORMAT)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_DATETIME_FORMAT] = 512 ; + ObSysVars[512].base_value_ = "%Y-%m-%d %H:%i:%s" ; + ObSysVars[512].alias_ = "OB_SV_DATETIME_FORMAT" ; + }(); + + [&] (){ + ObSysVars[513].default_value_ = "1" ; + ObSysVars[513].info_ = "This variable controls how the server handles clients with expired passwords" ; + ObSysVars[513].name_ = "disconnect_on_expired_password" ; + ObSysVars[513].data_type_ = ObIntType ; + ObSysVars[513].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[513].id_ = SYS_VAR_DISCONNECT_ON_EXPIRED_PASSWORD ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_DISCONNECT_ON_EXPIRED_PASSWORD)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_DISCONNECT_ON_EXPIRED_PASSWORD] = 513 ; + ObSysVars[513].base_value_ = "1" ; + ObSysVars[513].alias_ = "OB_SV_DISCONNECT_ON_EXPIRED_PASSWORD" ; + }(); + + [&] (){ + ObSysVars[514].default_value_ = "" ; + ObSysVars[514].info_ = "The external user name used during the authentication process, as set by the plugin used to authenticate the client" ; + ObSysVars[514].name_ = "external_user" ; + ObSysVars[514].data_type_ = ObVarcharType ; + ObSysVars[514].flags_ = ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::NULLABLE | ObSysVarFlag::READONLY ; + ObSysVars[514].id_ = SYS_VAR_EXTERNAL_USER ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_EXTERNAL_USER)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_EXTERNAL_USER] = 514 ; + ObSysVars[514].base_value_ = "" ; + ObSysVars[514].alias_ = "OB_SV_EXTERNAL_USER" ; + }(); + + [&] (){ + ObSysVars[515].default_value_ = "YES" ; + ObSysVars[515].info_ = "The external user name used during the authentication process, as set by the plugin used to authenticate the client" ; + ObSysVars[515].name_ = "have_crypt" ; + ObSysVars[515].data_type_ = ObVarcharType ; + ObSysVars[515].flags_ = ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[515].id_ = SYS_VAR_HAVE_CRYPT ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_HAVE_CRYPT)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_HAVE_CRYPT] = 515 ; + ObSysVars[515].base_value_ = "YES" ; + ObSysVars[515].alias_ = "OB_SV_HAVE_CRYPT" ; + }(); + + [&] (){ + ObSysVars[516].default_value_ = "YES" ; + ObSysVars[516].info_ = "YES if mysqld supports dynamic loading of plugins, NO if not. If the value is NO, you cannot use options such as --plugin-load to load plugins at server startup, or the INSTALL PLUGIN statement to load plugins at runtime" ; + ObSysVars[516].name_ = "have_dynamic_loading" ; + ObSysVars[516].data_type_ = ObVarcharType ; + ObSysVars[516].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[516].id_ = SYS_VAR_HAVE_DYNAMIC_LOADING ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_HAVE_DYNAMIC_LOADING)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_HAVE_DYNAMIC_LOADING] = 516 ; + ObSysVars[516].base_value_ = "YES" ; + ObSysVars[516].alias_ = "OB_SV_HAVE_DYNAMIC_LOADING" ; + }(); + + [&] (){ + ObSysVars[517].default_value_ = "" ; + ObSysVars[517].info_ = "The location of the configuration file for the keyring_aws plugin. This variable is unavailable unless that plugin is installed" ; + ObSysVars[517].name_ = "keyring_aws_conf_file" ; + ObSysVars[517].data_type_ = ObVarcharType ; + ObSysVars[517].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[517].id_ = SYS_VAR_KEYRING_AWS_CONF_FILE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_KEYRING_AWS_CONF_FILE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_KEYRING_AWS_CONF_FILE] = 517 ; + ObSysVars[517].base_value_ = "" ; + ObSysVars[517].alias_ = "OB_SV_KEYRING_AWS_CONF_FILE" ; + }(); + + [&] (){ + ObSysVars[518].default_value_ = "" ; + ObSysVars[518].info_ = "The location of the storage file for the keyring_aws plugin. This variable is unavailable unless that plugin is installed" ; + ObSysVars[518].name_ = "keyring_aws_data_file" ; + ObSysVars[518].data_type_ = ObVarcharType ; + ObSysVars[518].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[518].id_ = SYS_VAR_KEYRING_AWS_DATA_FILE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_KEYRING_AWS_DATA_FILE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_KEYRING_AWS_DATA_FILE] = 518 ; + ObSysVars[518].base_value_ = "" ; + ObSysVars[518].alias_ = "OB_SV_KEYRING_AWS_DATA_FILE" ; + }(); + + [&] (){ + ObSysVars[519].default_value_ = "" ; + ObSysVars[519].info_ = "The language to use for error messages" ; + ObSysVars[519].name_ = "language" ; + ObSysVars[519].data_type_ = ObVarcharType ; + ObSysVars[519].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[519].id_ = SYS_VAR_LANGUAGE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_LANGUAGE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_LANGUAGE] = 519 ; + ObSysVars[519].base_value_ = "" ; + ObSysVars[519].alias_ = "OB_SV_LANGUAGE" ; + }(); + + [&] (){ + ObSysVars[520].default_value_ = "" ; + ObSysVars[520].info_ = "The language to use for error messages" ; + ObSysVars[520].name_ = "lc_messages_dir" ; + ObSysVars[520].data_type_ = ObVarcharType ; + ObSysVars[520].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[520].id_ = SYS_VAR_LC_MESSAGES_DIR ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_LC_MESSAGES_DIR)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_LC_MESSAGES_DIR] = 520 ; + ObSysVars[520].base_value_ = "" ; + ObSysVars[520].alias_ = "OB_SV_LC_MESSAGES_DIR" ; + }(); + + [&] (){ + ObSysVars[521].default_value_ = "0" ; + ObSysVars[521].info_ = "This variable describes the case sensitivity of file names on the file system where the data directory is located" ; + ObSysVars[521].name_ = "lower_case_file_system" ; + ObSysVars[521].data_type_ = ObIntType ; + ObSysVars[521].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[521].id_ = SYS_VAR_LOWER_CASE_FILE_SYSTEM ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_LOWER_CASE_FILE_SYSTEM)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_LOWER_CASE_FILE_SYSTEM] = 521 ; + ObSysVars[521].base_value_ = "0" ; + ObSysVars[521].alias_ = "OB_SV_LOWER_CASE_FILE_SYSTEM" ; + }(); + + [&] (){ + ObSysVars[522].default_value_ = "1024" ; + ObSysVars[522].info_ = "The maximum number of bytes of memory reserved per session for computation of normalized statement digests" ; + ObSysVars[522].name_ = "max_digest_length" ; + ObSysVars[522].data_type_ = ObIntType ; + ObSysVars[522].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[522].id_ = SYS_VAR_MAX_DIGEST_LENGTH ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_MAX_DIGEST_LENGTH)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_MAX_DIGEST_LENGTH] = 522 ; + ObSysVars[522].base_value_ = "1024" ; + ObSysVars[522].alias_ = "OB_SV_MAX_DIGEST_LENGTH" ; + }(); + + [&] (){ + ObSysVars[523].default_value_ = "ndbinfo" ; + ObSysVars[523].info_ = "Shows the name used for the NDB information database" ; + ObSysVars[523].name_ = "ndbinfo_database" ; + ObSysVars[523].data_type_ = ObVarcharType ; + ObSysVars[523].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[523].id_ = SYS_VAR_NDBINFO_DATABASE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_NDBINFO_DATABASE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_NDBINFO_DATABASE] = 523 ; + ObSysVars[523].base_value_ = "ndbinfo" ; + ObSysVars[523].alias_ = "OB_SV_NDBINFO_DATABASE" ; + }(); + + [&] (){ + ObSysVars[524].default_value_ = "ndb$" ; + ObSysVars[524].info_ = "The prefix used in naming the ndbinfo database's base tables (normally hidden, unless exposed by setting ndbinfo_show_hidden" ; + ObSysVars[524].name_ = "ndbinfo_table_prefix" ; + ObSysVars[524].data_type_ = ObVarcharType ; + ObSysVars[524].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[524].id_ = SYS_VAR_NDBINFO_TABLE_PREFIX ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_NDBINFO_TABLE_PREFIX)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_NDBINFO_TABLE_PREFIX] = 524 ; + ObSysVars[524].base_value_ = "ndb$" ; + ObSysVars[524].alias_ = "OB_SV_NDBINFO_TABLE_PREFIX" ; + }(); + + [&] (){ + ObSysVars[525].default_value_ = "" ; + ObSysVars[525].info_ = "Shows the version of the ndbinfo engine in use" ; + ObSysVars[525].name_ = "ndbinfo_version" ; + ObSysVars[525].data_type_ = ObVarcharType ; + ObSysVars[525].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[525].id_ = SYS_VAR_NDBINFO_VERSION ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_NDBINFO_VERSION)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_NDBINFO_VERSION] = 525 ; + ObSysVars[525].base_value_ = "" ; + ObSysVars[525].alias_ = "OB_SV_NDBINFO_VERSION" ; + }(); + + [&] (){ + ObSysVars[526].default_value_ = "32768" ; + ObSysVars[526].info_ = "This sets the size in bytes that is used for NDB transaction batches" ; + ObSysVars[526].name_ = "ndb_batch_size" ; + ObSysVars[526].data_type_ = ObIntType ; + ObSysVars[526].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[526].id_ = SYS_VAR_NDB_BATCH_SIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_NDB_BATCH_SIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_NDB_BATCH_SIZE] = 526 ; + ObSysVars[526].base_value_ = "32768" ; + ObSysVars[526].alias_ = "OB_SV_NDB_BATCH_SIZE" ; + }(); + + [&] (){ + ObSysVars[527].default_value_ = "1" ; + ObSysVars[527].info_ = "a mysqld process can use multiple connections to the cluster, effectively mimicking several SQL nodes" ; + ObSysVars[527].name_ = "ndb_cluster_connection_pool" ; + ObSysVars[527].data_type_ = ObIntType ; + ObSysVars[527].min_val_ = "1" ; + ObSysVars[527].max_val_ = "63" ; + ObSysVars[527].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[527].id_ = SYS_VAR_NDB_CLUSTER_CONNECTION_POOL ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_NDB_CLUSTER_CONNECTION_POOL)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_NDB_CLUSTER_CONNECTION_POOL] = 527 ; + ObSysVars[527].base_value_ = "1" ; + ObSysVars[527].alias_ = "OB_SV_NDB_CLUSTER_CONNECTION_POOL" ; + }(); + + [&] (){ + ObSysVars[528].default_value_ = "" ; + ObSysVars[528].info_ = "Specifies a comma-separated list of node IDs for connections to the cluster used by an SQL node" ; + ObSysVars[528].name_ = "ndb_cluster_connection_pool_nodeids" ; + ObSysVars[528].data_type_ = ObVarcharType ; + ObSysVars[528].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[528].id_ = SYS_VAR_NDB_CLUSTER_CONNECTION_POOL_NODEIDS ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_NDB_CLUSTER_CONNECTION_POOL_NODEIDS)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_NDB_CLUSTER_CONNECTION_POOL_NODEIDS] = 528 ; + ObSysVars[528].base_value_ = "" ; + ObSysVars[528].alias_ = "OB_SV_NDB_CLUSTER_CONNECTION_POOL_NODEIDS" ; + }(); + + [&] (){ + ObSysVars[529].default_value_ = "0" ; + ObSysVars[529].info_ = "Causes a replica mysqld to log any updates received from its immediate source to the mysql.ndb_apply_status table in its own binary log using its own server ID rather than the server ID of the source" ; + ObSysVars[529].name_ = "ndb_log_apply_status" ; + ObSysVars[529].data_type_ = ObIntType ; + ObSysVars[529].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[529].id_ = SYS_VAR_NDB_LOG_APPLY_STATUS ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_NDB_LOG_APPLY_STATUS)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_NDB_LOG_APPLY_STATUS] = 529 ; + ObSysVars[529].base_value_ = "0" ; + ObSysVars[529].alias_ = "OB_SV_NDB_LOG_APPLY_STATUS" ; + }(); + + [&] (){ + ObSysVars[530].default_value_ = "1" ; + ObSysVars[530].info_ = "Causes updates to NDB tables to be written to the binary log. Setting this variable has no effect if binary logging is not already enabled for the server using log_bin" ; + ObSysVars[530].name_ = "ndb_log_bin" ; + ObSysVars[530].data_type_ = ObIntType ; + ObSysVars[530].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[530].id_ = SYS_VAR_NDB_LOG_BIN ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_NDB_LOG_BIN)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_NDB_LOG_BIN] = 530 ; + ObSysVars[530].base_value_ = "1" ; + ObSysVars[530].alias_ = "OB_SV_NDB_LOG_BIN" ; + }(); + + [&] (){ + ObSysVars[531].default_value_ = "0" ; + ObSysVars[531].info_ = "When this option is specified, and complete logging of all found row events is not possible, the mysqld process is terminated" ; + ObSysVars[531].name_ = "ndb_log_fail_terminate" ; + ObSysVars[531].data_type_ = ObIntType ; + ObSysVars[531].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[531].id_ = SYS_VAR_NDB_LOG_FAIL_TERMINATE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_NDB_LOG_FAIL_TERMINATE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_NDB_LOG_FAIL_TERMINATE] = 531 ; + ObSysVars[531].base_value_ = "0" ; + ObSysVars[531].alias_ = "OB_SV_NDB_LOG_FAIL_TERMINATE" ; + }(); + + [&] (){ + ObSysVars[532].default_value_ = "0" ; + ObSysVars[532].info_ = "Shows whether the originating server ID and epoch are logged in the ndb_binlog_index table" ; + ObSysVars[532].name_ = "ndb_log_orig" ; + ObSysVars[532].data_type_ = ObIntType ; + ObSysVars[532].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[532].id_ = SYS_VAR_NDB_LOG_ORIG ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_NDB_LOG_ORIG)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_NDB_LOG_ORIG] = 532 ; + ObSysVars[532].base_value_ = "0" ; + ObSysVars[532].alias_ = "OB_SV_NDB_LOG_ORIG" ; + }(); + + [&] (){ + ObSysVars[533].default_value_ = "0" ; + ObSysVars[533].info_ = "shows whether a replica mysqld writes NDB transaction IDs in the binary log" ; + ObSysVars[533].name_ = "ndb_log_transaction_id" ; + ObSysVars[533].data_type_ = ObIntType ; + ObSysVars[533].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[533].id_ = SYS_VAR_NDB_LOG_TRANSACTION_ID ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_NDB_LOG_TRANSACTION_ID)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_NDB_LOG_TRANSACTION_ID] = 533 ; + ObSysVars[533].base_value_ = "0" ; + ObSysVars[533].alias_ = "OB_SV_NDB_LOG_TRANSACTION_ID" ; + }(); + + [&] (){ + ObSysVars[534].default_value_ = "3" ; + ObSysVars[534].info_ = "" ; + ObSysVars[534].name_ = "ndb_optimized_node_selection" ; + ObSysVars[534].data_type_ = ObIntType ; + ObSysVars[534].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[534].id_ = SYS_VAR_NDB_OPTIMIZED_NODE_SELECTION ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_NDB_OPTIMIZED_NODE_SELECTION)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_NDB_OPTIMIZED_NODE_SELECTION] = 534 ; + ObSysVars[534].base_value_ = "3" ; + ObSysVars[534].alias_ = "OB_SV_NDB_OPTIMIZED_NODE_SELECTION" ; + }(); + + [&] (){ + ObSysVars[535].default_value_ = "" ; + ObSysVars[535].info_ = "If this MySQL Server is connected to an NDB cluster, this read-only variable shows the cluster system name. Otherwise, the value is an empty string" ; + ObSysVars[535].name_ = "Ndb_system_name" ; + ObSysVars[535].data_type_ = ObVarcharType ; + ObSysVars[535].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[535].id_ = SYS_VAR_NDB_SYSTEM_NAME ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_NDB_SYSTEM_NAME)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_NDB_SYSTEM_NAME] = 535 ; + ObSysVars[535].base_value_ = "" ; + ObSysVars[535].alias_ = "OB_SV_NDB_SYSTEM_NAME" ; + }(); + + [&] (){ + ObSysVars[536].default_value_ = "0" ; + ObSysVars[536].info_ = "Forces NDB to use copying of tables in the event of problems with online ALTER TABLE operations" ; + ObSysVars[536].name_ = "ndb_use_copying_alter_table" ; + ObSysVars[536].data_type_ = ObIntType ; + ObSysVars[536].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[536].id_ = SYS_VAR_NDB_USE_COPYING_ALTER_TABLE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_NDB_USE_COPYING_ALTER_TABLE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_NDB_USE_COPYING_ALTER_TABLE] = 536 ; + ObSysVars[536].base_value_ = "0" ; + ObSysVars[536].alias_ = "OB_SV_NDB_USE_COPYING_ALTER_TABLE" ; + }(); + + [&] (){ + ObSysVars[537].default_value_ = "" ; + ObSysVars[537].info_ = "NDB engine version in ndb-x.y.z format" ; + ObSysVars[537].name_ = "ndb_version_string" ; + ObSysVars[537].data_type_ = ObVarcharType ; + ObSysVars[537].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[537].id_ = SYS_VAR_NDB_VERSION_STRING ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_NDB_VERSION_STRING)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_NDB_VERSION_STRING] = 537 ; + ObSysVars[537].base_value_ = "" ; + ObSysVars[537].alias_ = "OB_SV_NDB_VERSION_STRING" ; + }(); + + [&] (){ + ObSysVars[538].default_value_ = "30" ; + ObSysVars[538].info_ = "This option sets the period of time that the MySQL server waits for connections to NDB Cluster management and data nodes to be established before accepting MySQL client connections." ; + ObSysVars[538].name_ = "ndb_wait_connected" ; + ObSysVars[538].data_type_ = ObIntType ; + ObSysVars[538].min_val_ = "0" ; + ObSysVars[538].max_val_ = "31536000" ; + ObSysVars[538].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[538].id_ = SYS_VAR_NDB_WAIT_CONNECTED ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_NDB_WAIT_CONNECTED)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_NDB_WAIT_CONNECTED] = 538 ; + ObSysVars[538].base_value_ = "30" ; + ObSysVars[538].alias_ = "OB_SV_NDB_WAIT_CONNECTED" ; + }(); + + [&] (){ + ObSysVars[539].default_value_ = "30" ; + ObSysVars[539].info_ = "This variable shows the period of time that the MySQL server waits for the NDB storage engine to complete setup before timing out and treating NDB as unavailable. The time is specified in seconds." ; + ObSysVars[539].name_ = "ndb_wait_setup" ; + ObSysVars[539].data_type_ = ObIntType ; + ObSysVars[539].min_val_ = "0" ; + ObSysVars[539].max_val_ = "31536000" ; + ObSysVars[539].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[539].id_ = SYS_VAR_NDB_WAIT_SETUP ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_NDB_WAIT_SETUP)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_NDB_WAIT_SETUP] = 539 ; + ObSysVars[539].base_value_ = "30" ; + ObSysVars[539].alias_ = "OB_SV_NDB_WAIT_SETUP" ; + }(); + + [&] (){ + ObSysVars[540].default_value_ = "" ; + ObSysVars[540].info_ = "If the current client is a proxy for another user, this variable is the proxy user account name" ; + ObSysVars[540].name_ = "proxy_user" ; + ObSysVars[540].data_type_ = ObVarcharType ; + ObSysVars[540].flags_ = ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[540].id_ = SYS_VAR_PROXY_USER ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_PROXY_USER)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_PROXY_USER] = 540 ; + ObSysVars[540].base_value_ = "" ; + ObSysVars[540].alias_ = "OB_SV_PROXY_USER" ; + }(); + + [&] (){ + ObSysVars[541].default_value_ = "1" ; + ObSysVars[541].info_ = "It controls whether the server autogenerates RSA private/public key-pair files in the data directory" ; + ObSysVars[541].name_ = "sha256_password_auto_generate_rsa_keys" ; + ObSysVars[541].data_type_ = ObIntType ; + ObSysVars[541].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[541].id_ = SYS_VAR_SHA256_PASSWORD_AUTO_GENERATE_RSA_KEYS ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_SHA256_PASSWORD_AUTO_GENERATE_RSA_KEYS)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_SHA256_PASSWORD_AUTO_GENERATE_RSA_KEYS] = 541 ; + ObSysVars[541].base_value_ = "1" ; + ObSysVars[541].alias_ = "OB_SV_SHA256_PASSWORD_AUTO_GENERATE_RSA_KEYS" ; + }(); + + [&] (){ + ObSysVars[542].default_value_ = "private_key.pem" ; + ObSysVars[542].info_ = "Its value is the path name of the RSA private key file for the sha256_password authentication plugin" ; + ObSysVars[542].name_ = "sha256_password_private_key_path" ; + ObSysVars[542].data_type_ = ObVarcharType ; + ObSysVars[542].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[542].id_ = SYS_VAR_SHA256_PASSWORD_PRIVATE_KEY_PATH ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_SHA256_PASSWORD_PRIVATE_KEY_PATH)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_SHA256_PASSWORD_PRIVATE_KEY_PATH] = 542 ; + ObSysVars[542].base_value_ = "private_key.pem" ; + ObSysVars[542].alias_ = "OB_SV_SHA256_PASSWORD_PRIVATE_KEY_PATH" ; + }(); + + [&] (){ + ObSysVars[543].default_value_ = "public_key.pem" ; + ObSysVars[543].info_ = "Its value is the path name of the RSA public key file for the sha256_password authentication plugin" ; + ObSysVars[543].name_ = "sha256_password_public_key_path" ; + ObSysVars[543].data_type_ = ObVarcharType ; + ObSysVars[543].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[543].id_ = SYS_VAR_SHA256_PASSWORD_PUBLIC_KEY_PATH ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_SHA256_PASSWORD_PUBLIC_KEY_PATH)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_SHA256_PASSWORD_PUBLIC_KEY_PATH] = 543 ; + ObSysVars[543].base_value_ = "public_key.pem" ; + ObSysVars[543].alias_ = "OB_SV_SHA256_PASSWORD_PUBLIC_KEY_PATH" ; + }(); + + [&] (){ + ObSysVars[544].default_value_ = "0" ; + ObSysVars[544].info_ = " If the variable value is ON, the SHOW DATABASES statement is permitted only to users who have the SHOW DATABASES privilege, and the statement displays all database names" ; + ObSysVars[544].name_ = "skip_show_database" ; + ObSysVars[544].data_type_ = ObVarcharType ; + ObSysVars[544].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[544].id_ = SYS_VAR_SKIP_SHOW_DATABASE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_SKIP_SHOW_DATABASE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_SKIP_SHOW_DATABASE] = 544 ; + ObSysVars[544].base_value_ = "0" ; + ObSysVars[544].alias_ = "OB_SV_SKIP_SHOW_DATABASE" ; + }(); + + [&] (){ + ObSysVars[545].default_value_ = "" ; + ObSysVars[545].info_ = "This option tells the server to load the named plugins at startup" ; + ObSysVars[545].name_ = "plugin_load" ; + ObSysVars[545].data_type_ = ObVarcharType ; + ObSysVars[545].flags_ = ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[545].id_ = SYS_VAR_PLUGIN_LOAD ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_PLUGIN_LOAD)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_PLUGIN_LOAD] = 545 ; + ObSysVars[545].base_value_ = "" ; + ObSysVars[545].alias_ = "OB_SV_PLUGIN_LOAD" ; + }(); + + [&] (){ + ObSysVars[546].default_value_ = "" ; + ObSysVars[546].info_ = "adds a plugin or plugins to the set of plugins to be loaded at startup" ; + ObSysVars[546].name_ = "plugin_load_add" ; + ObSysVars[546].data_type_ = ObVarcharType ; + ObSysVars[546].flags_ = ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[546].id_ = SYS_VAR_PLUGIN_LOAD_ADD ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_PLUGIN_LOAD_ADD)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_PLUGIN_LOAD_ADD] = 546 ; + ObSysVars[546].base_value_ = "" ; + ObSysVars[546].alias_ = "OB_SV_PLUGIN_LOAD_ADD" ; + }(); + + [&] (){ + ObSysVars[547].default_value_ = "0" ; + ObSysVars[547].info_ = "the server stores all temporary tables on disk rather than in memory" ; + ObSysVars[547].name_ = "big_tables" ; + ObSysVars[547].data_type_ = ObIntType ; + ObSysVars[547].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[547].id_ = SYS_VAR_BIG_TABLES ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_BIG_TABLES)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_BIG_TABLES] = 547 ; + ObSysVars[547].base_value_ = "0" ; + ObSysVars[547].alias_ = "OB_SV_BIG_TABLES" ; + }(); + + [&] (){ + ObSysVars[548].default_value_ = "0" ; + ObSysVars[548].info_ = "If the check_proxy_users system variable is enabled, the server performs proxy user mapping for any authentication plugins that make such a request" ; + ObSysVars[548].name_ = "check_proxy_users" ; + ObSysVars[548].data_type_ = ObIntType ; + ObSysVars[548].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[548].id_ = SYS_VAR_CHECK_PROXY_USERS ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_CHECK_PROXY_USERS)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_CHECK_PROXY_USERS] = 548 ; + ObSysVars[548].base_value_ = "0" ; + ObSysVars[548].alias_ = "OB_SV_CHECK_PROXY_USERS" ; + }(); + + [&] (){ + ObSysVars[549].default_value_ = "0" ; + ObSysVars[549].info_ = "The number of consecutive failed connection attempts permitted to accounts before the server adds a delay for subsequent connection attempts" ; + ObSysVars[549].name_ = "connection_control_failed_connections_threshold" ; + ObSysVars[549].data_type_ = ObIntType ; + ObSysVars[549].min_val_ = "0" ; + ObSysVars[549].max_val_ = "2147483647" ; + ObSysVars[549].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[549].id_ = SYS_VAR_CONNECTION_CONTROL_FAILED_CONNECTIONS_THRESHOLD ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_CONNECTION_CONTROL_FAILED_CONNECTIONS_THRESHOLD)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_CONNECTION_CONTROL_FAILED_CONNECTIONS_THRESHOLD] = 549 ; + ObSysVars[549].base_value_ = "0" ; + ObSysVars[549].alias_ = "OB_SV_CONNECTION_CONTROL_FAILED_CONNECTIONS_THRESHOLD" ; + }(); + + [&] (){ + ObSysVars[550].default_value_ = "2147483647" ; + ObSysVars[550].info_ = "The maximum delay in milliseconds for server response to failed connection attempts, if connection_control_failed_connections_threshold is greater than zero" ; + ObSysVars[550].name_ = "connection_control_max_connection_delay" ; + ObSysVars[550].data_type_ = ObIntType ; + ObSysVars[550].min_val_ = "1000" ; + ObSysVars[550].max_val_ = "2147483647" ; + ObSysVars[550].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[550].id_ = SYS_VAR_CONNECTION_CONTROL_MAX_CONNECTION_DELAY ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_CONNECTION_CONTROL_MAX_CONNECTION_DELAY)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_CONNECTION_CONTROL_MAX_CONNECTION_DELAY] = 550 ; + ObSysVars[550].base_value_ = "2147483647" ; + ObSysVars[550].alias_ = "OB_SV_CONNECTION_CONTROL_MAX_CONNECTION_DELAY" ; + }(); + + [&] (){ + ObSysVars[551].default_value_ = "1000" ; + ObSysVars[551].info_ = "The minmum delay in milliseconds for server response to failed connection attempts, if connection_control_failed_connections_threshold is greater than zero" ; + ObSysVars[551].name_ = "connection_control_min_connection_delay" ; + ObSysVars[551].data_type_ = ObIntType ; + ObSysVars[551].min_val_ = "1000" ; + ObSysVars[551].max_val_ = "2147483647" ; + ObSysVars[551].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[551].id_ = SYS_VAR_CONNECTION_CONTROL_MIN_CONNECTION_DELAY ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_CONNECTION_CONTROL_MIN_CONNECTION_DELAY)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_CONNECTION_CONTROL_MIN_CONNECTION_DELAY] = 551 ; + ObSysVars[551].base_value_ = "1000" ; + ObSysVars[551].alias_ = "OB_SV_CONNECTION_CONTROL_MIN_CONNECTION_DELAY" ; + }(); + + [&] (){ + ObSysVars[552].default_value_ = "0" ; + ObSysVars[552].info_ = "The default mode value to use for the WEEK() function" ; + ObSysVars[552].name_ = "default_week_format" ; + ObSysVars[552].data_type_ = ObIntType ; + ObSysVars[552].min_val_ = "0" ; + ObSysVars[552].max_val_ = "7" ; + ObSysVars[552].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[552].id_ = SYS_VAR_DEFAULT_WEEK_FORMAT ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_DEFAULT_WEEK_FORMAT)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_DEFAULT_WEEK_FORMAT] = 552 ; + ObSysVars[552].base_value_ = "0" ; + ObSysVars[552].alias_ = "OB_SV_DEFAULT_WEEK_FORMAT" ; + }(); + + [&] (){ + ObSysVars[553].default_value_ = "300" ; + ObSysVars[553].info_ = "" ; + ObSysVars[553].name_ = "delayed_insert_timeout" ; + ObSysVars[553].data_type_ = ObIntType ; + ObSysVars[553].min_val_ = "1" ; + ObSysVars[553].max_val_ = "31536000" ; + ObSysVars[553].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[553].id_ = SYS_VAR_DELAYED_INSERT_TIMEOUT ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_DELAYED_INSERT_TIMEOUT)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_DELAYED_INSERT_TIMEOUT] = 553 ; + ObSysVars[553].base_value_ = "300" ; + ObSysVars[553].alias_ = "OB_SV_DELAYED_INSERT_TIMEOUT" ; + }(); + + [&] (){ + ObSysVars[554].default_value_ = "1000" ; + ObSysVars[554].info_ = "" ; + ObSysVars[554].name_ = "delayed_queue_size" ; + ObSysVars[554].data_type_ = ObUInt64Type ; + ObSysVars[554].min_val_ = "1" ; + ObSysVars[554].max_val_ = "18446744073709551615" ; + ObSysVars[554].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[554].id_ = SYS_VAR_DELAYED_QUEUE_SIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_DELAYED_QUEUE_SIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_DELAYED_QUEUE_SIZE] = 554 ; + ObSysVars[554].base_value_ = "1000" ; + ObSysVars[554].alias_ = "OB_SV_DELAYED_QUEUE_SIZE" ; + }(); + + [&] (){ + ObSysVars[555].default_value_ = "200" ; + ObSysVars[555].info_ = "This variable indicates the number of equality ranges in an equality comparison condition when the optimizer should switch from using index dives to index statistics in estimating the number of qualifying rows" ; + ObSysVars[555].name_ = "eq_range_index_dive_limit" ; + ObSysVars[555].data_type_ = ObIntType ; + ObSysVars[555].min_val_ = "0" ; + ObSysVars[555].max_val_ = "4294967295" ; + ObSysVars[555].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[555].id_ = SYS_VAR_EQ_RANGE_INDEX_DIVE_LIMIT ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_EQ_RANGE_INDEX_DIVE_LIMIT)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_EQ_RANGE_INDEX_DIVE_LIMIT] = 555 ; + ObSysVars[555].base_value_ = "200" ; + ObSysVars[555].alias_ = "OB_SV_EQ_RANGE_INDEX_DIVE_LIMIT" ; + }(); + + [&] (){ + ObSysVars[556].default_value_ = "1" ; + ObSysVars[556].info_ = "Causes InnoDB to automatically recalculate persistent statistics after the data in a table is changed substantially, merely simulates MySQL 5.7" ; + ObSysVars[556].name_ = "innodb_stats_auto_recalc" ; + ObSysVars[556].data_type_ = ObIntType ; + ObSysVars[556].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[556].id_ = SYS_VAR_INNODB_STATS_AUTO_RECALC ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_STATS_AUTO_RECALC)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_STATS_AUTO_RECALC] = 556 ; + ObSysVars[556].base_value_ = "1" ; + ObSysVars[556].alias_ = "OB_SV_INNODB_STATS_AUTO_RECALC" ; + }(); + + [&] (){ + ObSysVars[557].default_value_ = "0" ; + ObSysVars[557].info_ = "When innodb_stats_include_delete_marked is enabled, ANALYZE TABLE considers delete-marked records when recalculating statistics" ; + ObSysVars[557].name_ = "innodb_stats_include_delete_marked" ; + ObSysVars[557].data_type_ = ObIntType ; + ObSysVars[557].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[557].id_ = SYS_VAR_INNODB_STATS_INCLUDE_DELETE_MARKED ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_STATS_INCLUDE_DELETE_MARKED)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_STATS_INCLUDE_DELETE_MARKED] = 557 ; + ObSysVars[557].base_value_ = "0" ; + ObSysVars[557].alias_ = "OB_SV_INNODB_STATS_INCLUDE_DELETE_MARKED" ; + }(); + + [&] (){ + ObSysVars[558].default_value_ = "0" ; + ObSysVars[558].info_ = "How the server treats NULL values when collecting statistics about the distribution of index values for InnoDB tables" ; + ObSysVars[558].name_ = "innodb_stats_method" ; + ObSysVars[558].data_type_ = ObIntType ; + ObSysVars[558].enum_names_ = "[u'nulls_equal', u'nulls_unequal', u'nulls_ignored']" ; + ObSysVars[558].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[558].id_ = SYS_VAR_INNODB_STATS_METHOD ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_STATS_METHOD)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_STATS_METHOD] = 558 ; + ObSysVars[558].base_value_ = "0" ; + ObSysVars[558].alias_ = "OB_SV_INNODB_STATS_METHOD" ; + }(); + + [&] (){ + ObSysVars[559].default_value_ = "0" ; + ObSysVars[559].info_ = "When innodb_stats_on_metadata is enabled, InnoDB updates non-persistent statistics when metadata statements such as SHOW TABLE STATUS or when accessing the Information Schema TABLES or STATISTICS tables" ; + ObSysVars[559].name_ = "innodb_stats_on_metadata" ; + ObSysVars[559].data_type_ = ObIntType ; + ObSysVars[559].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[559].id_ = SYS_VAR_INNODB_STATS_ON_METADATA ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_STATS_ON_METADATA)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_STATS_ON_METADATA] = 559 ; + ObSysVars[559].base_value_ = "0" ; + ObSysVars[559].alias_ = "OB_SV_INNODB_STATS_ON_METADATA" ; + }(); + + [&] (){ + ObSysVars[560].default_value_ = "" ; + ObSysVars[560].info_ = "The session value of this variable specifies the client version token list and indicates the tokens that the client session requires the server version token list to have, merely simulates MySQL 5.7" ; + ObSysVars[560].name_ = "version_tokens_session" ; + ObSysVars[560].data_type_ = ObVarcharType ; + ObSysVars[560].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[560].id_ = SYS_VAR_VERSION_TOKENS_SESSION ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_VERSION_TOKENS_SESSION)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_VERSION_TOKENS_SESSION] = 560 ; + ObSysVars[560].base_value_ = "" ; + ObSysVars[560].alias_ = "OB_SV_VERSION_TOKENS_SESSION" ; + }(); + + [&] (){ + ObSysVars[561].default_value_ = "20" ; + ObSysVars[561].info_ = "The number of index pages to sample when estimating cardinality and other statistics for an indexed column, such as those calculated by ANALYZE TABLE" ; + ObSysVars[561].name_ = "innodb_stats_persistent_sample_pages" ; + ObSysVars[561].data_type_ = ObUInt64Type ; + ObSysVars[561].min_val_ = "1" ; + ObSysVars[561].max_val_ = "18446744073709551615" ; + ObSysVars[561].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[561].id_ = SYS_VAR_INNODB_STATS_PERSISTENT_SAMPLE_PAGES ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_STATS_PERSISTENT_SAMPLE_PAGES)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_STATS_PERSISTENT_SAMPLE_PAGES] = 561 ; + ObSysVars[561].base_value_ = "20" ; + ObSysVars[561].alias_ = "OB_SV_INNODB_STATS_PERSISTENT_SAMPLE_PAGES" ; + }(); + + [&] (){ + ObSysVars[562].default_value_ = "8" ; + ObSysVars[562].info_ = "The number of index pages to sample when estimating cardinality and other statistics for an indexed column, such as those calculated by ANALYZE TABLE" ; + ObSysVars[562].name_ = "innodb_stats_sample_pages" ; + ObSysVars[562].data_type_ = ObUInt64Type ; + ObSysVars[562].min_val_ = "1" ; + ObSysVars[562].max_val_ = "18446744073709551615" ; + ObSysVars[562].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[562].id_ = SYS_VAR_INNODB_STATS_SAMPLE_PAGES ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_STATS_SAMPLE_PAGES)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_STATS_SAMPLE_PAGES] = 562 ; + ObSysVars[562].base_value_ = "8" ; + ObSysVars[562].alias_ = "OB_SV_INNODB_STATS_SAMPLE_PAGES" ; + }(); + + [&] (){ + ObSysVars[563].default_value_ = "8" ; + ObSysVars[563].info_ = "The number of index pages to sample when estimating cardinality and other statistics for an indexed column, such as those calculated by ANALYZE TABLE" ; + ObSysVars[563].name_ = "innodb_stats_transient_sample_pages" ; + ObSysVars[563].data_type_ = ObUInt64Type ; + ObSysVars[563].min_val_ = "1" ; + ObSysVars[563].max_val_ = "18446744073709551615" ; + ObSysVars[563].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[563].id_ = SYS_VAR_INNODB_STATS_TRANSIENT_SAMPLE_PAGES ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_INNODB_STATS_TRANSIENT_SAMPLE_PAGES)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_INNODB_STATS_TRANSIENT_SAMPLE_PAGES] = 563 ; + ObSysVars[563].base_value_ = "8" ; + ObSysVars[563].alias_ = "OB_SV_INNODB_STATS_TRANSIENT_SAMPLE_PAGES" ; + }(); + + [&] (){ + ObSysVars[564].default_value_ = "" ; + ObSysVars[564].info_ = "The customer master key (CMK) ID obtained from the AWS KMS server and used by the keyring_aws plugin" ; + ObSysVars[564].name_ = "keyring_aws_cmk_id" ; + ObSysVars[564].data_type_ = ObVarcharType ; + ObSysVars[564].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[564].id_ = SYS_VAR_KEYRING_AWS_CMK_ID ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_KEYRING_AWS_CMK_ID)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_KEYRING_AWS_CMK_ID] = 564 ; + ObSysVars[564].base_value_ = "" ; + ObSysVars[564].alias_ = "OB_SV_KEYRING_AWS_CMK_ID" ; + }(); + + [&] (){ + ObSysVars[565].default_value_ = "19" ; + ObSysVars[565].info_ = "The AWS region for the keyring_aws plugin. This variable is unavailable unless that plugin is installed" ; + ObSysVars[565].name_ = "keyring_aws_region" ; + ObSysVars[565].data_type_ = ObIntType ; + ObSysVars[565].enum_names_ = "[u'af-south-1', u'ap-east-1', u'ap-northeast-1', u'ap-northeast-2', u'ap-northeast-3', u'ap-south-1', u'ap-southeast-1', u'ap-southeast-2', u'ca-central-1', u'cn-north-1', u'cn-northwest-1', u'eu-central-1', u'eu-north-1', u'eu-south-1', u'eu-west-1', u'eu-west-2', u'eu-west-3', u'me-south-1', u'sa-east-1', u'us-east-1', u'us-east-2', u'us-gov-east-1', u'us-iso-east-1', u'us-iso-west-1', u'us-isob-east-1', u'us-west-1', u'us-west-2']" ; + ObSysVars[565].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[565].id_ = SYS_VAR_KEYRING_AWS_REGION ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_KEYRING_AWS_REGION)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_KEYRING_AWS_REGION] = 565 ; + ObSysVars[565].base_value_ = "19" ; + ObSysVars[565].alias_ = "OB_SV_KEYRING_AWS_REGION" ; + }(); + + [&] (){ + ObSysVars[566].default_value_ = "" ; + ObSysVars[566].info_ = "The path name of the data file used for secure data storage by the keyring_encrypted_file plugin" ; + ObSysVars[566].name_ = "keyring_encrypted_file_data" ; + ObSysVars[566].data_type_ = ObVarcharType ; + ObSysVars[566].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[566].id_ = SYS_VAR_KEYRING_ENCRYPTED_FILE_DATA ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_KEYRING_ENCRYPTED_FILE_DATA)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_KEYRING_ENCRYPTED_FILE_DATA] = 566 ; + ObSysVars[566].base_value_ = "" ; + ObSysVars[566].alias_ = "OB_SV_KEYRING_ENCRYPTED_FILE_DATA" ; + }(); + + [&] (){ + ObSysVars[567].default_value_ = "" ; + ObSysVars[567].info_ = "The password used by the keyring_encrypted_file pluginn" ; + ObSysVars[567].name_ = "keyring_encrypted_file_password" ; + ObSysVars[567].data_type_ = ObVarcharType ; + ObSysVars[567].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[567].id_ = SYS_VAR_KEYRING_ENCRYPTED_FILE_PASSWORD ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_KEYRING_ENCRYPTED_FILE_PASSWORD)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_KEYRING_ENCRYPTED_FILE_PASSWORD] = 567 ; + ObSysVars[567].base_value_ = "" ; + ObSysVars[567].alias_ = "OB_SV_KEYRING_ENCRYPTED_FILE_PASSWORD" ; + }(); + + [&] (){ + ObSysVars[568].default_value_ = "" ; + ObSysVars[568].info_ = "The path name of the data file used for secure data storage by the keyring_file plugin" ; + ObSysVars[568].name_ = "keyring_file_data" ; + ObSysVars[568].data_type_ = ObVarcharType ; + ObSysVars[568].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[568].id_ = SYS_VAR_KEYRING_FILE_DATA ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_KEYRING_FILE_DATA)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_KEYRING_FILE_DATA] = 568 ; + ObSysVars[568].base_value_ = "" ; + ObSysVars[568].alias_ = "OB_SV_KEYRING_FILE_DATA" ; + }(); + + [&] (){ + ObSysVars[569].default_value_ = "" ; + ObSysVars[569].info_ = "The path name of the directory that stores configuration information used by the keyring_okv plugin" ; + ObSysVars[569].name_ = "keyring_okv_conf_dir" ; + ObSysVars[569].data_type_ = ObVarcharType ; + ObSysVars[569].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[569].id_ = SYS_VAR_KEYRING_OKV_CONF_DIR ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_KEYRING_OKV_CONF_DIR)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_KEYRING_OKV_CONF_DIR] = 569 ; + ObSysVars[569].base_value_ = "" ; + ObSysVars[569].alias_ = "OB_SV_KEYRING_OKV_CONF_DIR" ; + }(); + + [&] (){ + ObSysVars[570].default_value_ = "1" ; + ObSysVars[570].info_ = "Whether keyring operations are enabled. This variable is used during key migration operations" ; + ObSysVars[570].name_ = "keyring_operations" ; + ObSysVars[570].data_type_ = ObIntType ; + ObSysVars[570].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[570].id_ = SYS_VAR_KEYRING_OPERATIONS ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_KEYRING_OPERATIONS)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_KEYRING_OPERATIONS] = 570 ; + ObSysVars[570].base_value_ = "1" ; + ObSysVars[570].alias_ = "OB_SV_KEYRING_OPERATIONS" ; + }(); + + [&] (){ + ObSysVars[571].default_value_ = "" ; + ObSysVars[571].info_ = "enables control over optimizer behavior" ; + ObSysVars[571].name_ = "optimizer_switch" ; + ObSysVars[571].data_type_ = ObVarcharType ; + ObSysVars[571].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[571].id_ = SYS_VAR_OPTIMIZER_SWITCH ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_OPTIMIZER_SWITCH)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_OPTIMIZER_SWITCH] = 571 ; + ObSysVars[571].base_value_ = "" ; + ObSysVars[571].alias_ = "OB_SV_OPTIMIZER_SWITCH" ; + }(); + + [&] (){ + ObSysVars[572].default_value_ = "100" ; + ObSysVars[572].info_ = "After max_connect_errors successive connection requests from a host are interrupted without a successful connection, the server blocks that host from further connections" ; + ObSysVars[572].name_ = "max_connect_errors" ; + ObSysVars[572].data_type_ = ObUInt64Type ; + ObSysVars[572].min_val_ = "1" ; + ObSysVars[572].max_val_ = "18446744073709551615" ; + ObSysVars[572].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[572].id_ = SYS_VAR_MAX_CONNECT_ERRORS ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_MAX_CONNECT_ERRORS)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_MAX_CONNECT_ERRORS] = 572 ; + ObSysVars[572].base_value_ = "100" ; + ObSysVars[572].alias_ = "OB_SV_MAX_CONNECT_ERRORS" ; + }(); + + [&] (){ + ObSysVars[573].default_value_ = "0" ; + ObSysVars[573].info_ = "Whether MySQL Enterprise Firewall is enabled (the default) or disabled" ; + ObSysVars[573].name_ = "mysql_firewall_mode" ; + ObSysVars[573].data_type_ = ObIntType ; + ObSysVars[573].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[573].id_ = SYS_VAR_MYSQL_FIREWALL_MODE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_MYSQL_FIREWALL_MODE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_MYSQL_FIREWALL_MODE] = 573 ; + ObSysVars[573].base_value_ = "0" ; + ObSysVars[573].alias_ = "OB_SV_MYSQL_FIREWALL_MODE" ; + }(); + + [&] (){ + ObSysVars[574].default_value_ = "0" ; + ObSysVars[574].info_ = "Whether the MySQL Enterprise Firewall trace is enabled or disabled (the default)" ; + ObSysVars[574].name_ = "mysql_firewall_trace" ; + ObSysVars[574].data_type_ = ObIntType ; + ObSysVars[574].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[574].id_ = SYS_VAR_MYSQL_FIREWALL_TRACE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_MYSQL_FIREWALL_TRACE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_MYSQL_FIREWALL_TRACE] = 574 ; + ObSysVars[574].base_value_ = "0" ; + ObSysVars[574].alias_ = "OB_SV_MYSQL_FIREWALL_TRACE" ; + }(); + + [&] (){ + ObSysVars[575].default_value_ = "0" ; + ObSysVars[575].info_ = "This variable controls whether the mysql_native_password built-in authentication plugin supports proxy users" ; + ObSysVars[575].name_ = "mysql_native_password_proxy_users" ; + ObSysVars[575].data_type_ = ObIntType ; + ObSysVars[575].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[575].id_ = SYS_VAR_MYSQL_NATIVE_PASSWORD_PROXY_USERS ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_MYSQL_NATIVE_PASSWORD_PROXY_USERS)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_MYSQL_NATIVE_PASSWORD_PROXY_USERS] = 575 ; + ObSysVars[575].base_value_ = "0" ; + ObSysVars[575].alias_ = "OB_SV_MYSQL_NATIVE_PASSWORD_PROXY_USERS" ; + }(); + + [&] (){ + ObSysVars[576].default_value_ = "10" ; + ObSysVars[576].info_ = "If a read or write on a communication port is interrupted, retry this many times before giving up. This value should be set quite high on FreeBSD because internal interrupts are sent to all threads" ; + ObSysVars[576].name_ = "net_retry_count" ; + ObSysVars[576].data_type_ = ObUInt64Type ; + ObSysVars[576].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[576].id_ = SYS_VAR_NET_RETRY_COUNT ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_NET_RETRY_COUNT)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_NET_RETRY_COUNT] = 576 ; + ObSysVars[576].base_value_ = "10" ; + ObSysVars[576].alias_ = "OB_SV_NET_RETRY_COUNT" ; + }(); + + [&] (){ + ObSysVars[577].default_value_ = "0" ; + ObSysVars[577].info_ = "This variable was used in MySQL 4.0 to turn on some 4.1 behaviors, and is retained for backward compatibility. Its value is always OFF" ; + ObSysVars[577].name_ = "new" ; + ObSysVars[577].data_type_ = ObIntType ; + ObSysVars[577].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[577].id_ = SYS_VAR_NEW ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_NEW)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_NEW] = 577 ; + ObSysVars[577].base_value_ = "0" ; + ObSysVars[577].alias_ = "OB_SV_NEW" ; + }(); + + [&] (){ + ObSysVars[578].default_value_ = "0" ; + ObSysVars[578].info_ = "This variable controls the password hashing method used by the PASSWORD() function. It also influences password hashing performed by CREATE USER and GRANT statements that specify a password using an IDENTIFIED BY clause" ; + ObSysVars[578].name_ = "old_passwords" ; + ObSysVars[578].data_type_ = ObIntType ; + ObSysVars[578].enum_names_ = "[u'0', u'1', u'2']" ; + ObSysVars[578].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[578].id_ = SYS_VAR_OLD_PASSWORDS ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_OLD_PASSWORDS)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_OLD_PASSWORDS] = 578 ; + ObSysVars[578].base_value_ = "0" ; + ObSysVars[578].alias_ = "OB_SV_OLD_PASSWORDS" ; + }(); + + [&] (){ + ObSysVars[579].default_value_ = "1" ; + ObSysVars[579].info_ = "Controls the heuristics applied during query optimization to prune less-promising partial plans from the optimizer search space" ; + ObSysVars[579].name_ = "optimizer_prune_level" ; + ObSysVars[579].data_type_ = ObIntType ; + ObSysVars[579].min_val_ = "0" ; + ObSysVars[579].max_val_ = "1" ; + ObSysVars[579].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[579].id_ = SYS_VAR_OPTIMIZER_PRUNE_LEVEL ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_OPTIMIZER_PRUNE_LEVEL)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_OPTIMIZER_PRUNE_LEVEL] = 579 ; + ObSysVars[579].base_value_ = "1" ; + ObSysVars[579].alias_ = "OB_SV_OPTIMIZER_PRUNE_LEVEL" ; + }(); + + [&] (){ + ObSysVars[580].default_value_ = "62" ; + ObSysVars[580].info_ = "The maximum depth of search performed by the query optimizer" ; + ObSysVars[580].name_ = "optimizer_search_depth" ; + ObSysVars[580].data_type_ = ObIntType ; + ObSysVars[580].min_val_ = "0" ; + ObSysVars[580].max_val_ = "62" ; + ObSysVars[580].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[580].id_ = SYS_VAR_OPTIMIZER_SEARCH_DEPTH ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_OPTIMIZER_SEARCH_DEPTH)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_OPTIMIZER_SEARCH_DEPTH] = 580 ; + ObSysVars[580].base_value_ = "62" ; + ObSysVars[580].alias_ = "OB_SV_OPTIMIZER_SEARCH_DEPTH" ; + }(); + + [&] (){ + ObSysVars[581].default_value_ = "" ; + ObSysVars[581].info_ = "This variable controls optimizer tracing" ; + ObSysVars[581].name_ = "optimizer_trace" ; + ObSysVars[581].data_type_ = ObVarcharType ; + ObSysVars[581].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[581].id_ = SYS_VAR_OPTIMIZER_TRACE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_OPTIMIZER_TRACE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_OPTIMIZER_TRACE] = 581 ; + ObSysVars[581].base_value_ = "" ; + ObSysVars[581].alias_ = "OB_SV_OPTIMIZER_TRACE" ; + }(); + + [&] (){ + ObSysVars[582].default_value_ = "" ; + ObSysVars[582].info_ = "This variable enables or disables selected optimizer tracing features" ; + ObSysVars[582].name_ = "optimizer_trace_features" ; + ObSysVars[582].data_type_ = ObVarcharType ; + ObSysVars[582].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[582].id_ = SYS_VAR_OPTIMIZER_TRACE_FEATURES ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_OPTIMIZER_TRACE_FEATURES)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_OPTIMIZER_TRACE_FEATURES] = 582 ; + ObSysVars[582].base_value_ = "" ; + ObSysVars[582].alias_ = "OB_SV_OPTIMIZER_TRACE_FEATURES" ; + }(); + + [&] (){ + ObSysVars[583].default_value_ = "1" ; + ObSysVars[583].info_ = "The maximum number of optimizer traces to display" ; + ObSysVars[583].name_ = "optimizer_trace_limit" ; + ObSysVars[583].data_type_ = ObIntType ; + ObSysVars[583].min_val_ = "0" ; + ObSysVars[583].max_val_ = "2147483647" ; + ObSysVars[583].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[583].id_ = SYS_VAR_OPTIMIZER_TRACE_LIMIT ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_OPTIMIZER_TRACE_LIMIT)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_OPTIMIZER_TRACE_LIMIT] = 583 ; + ObSysVars[583].base_value_ = "1" ; + ObSysVars[583].alias_ = "OB_SV_OPTIMIZER_TRACE_LIMIT" ; + }(); + + [&] (){ + ObSysVars[584].default_value_ = "16384" ; + ObSysVars[584].info_ = "The maximum cumulative size of stored optimizer traces" ; + ObSysVars[584].name_ = "optimizer_trace_max_mem_size" ; + ObSysVars[584].data_type_ = ObUInt64Type ; + ObSysVars[584].min_val_ = "0" ; + ObSysVars[584].max_val_ = "4294967295" ; + ObSysVars[584].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[584].id_ = SYS_VAR_OPTIMIZER_TRACE_MAX_MEM_SIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_OPTIMIZER_TRACE_MAX_MEM_SIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_OPTIMIZER_TRACE_MAX_MEM_SIZE] = 584 ; + ObSysVars[584].base_value_ = "16384" ; + ObSysVars[584].alias_ = "OB_SV_OPTIMIZER_TRACE_MAX_MEM_SIZE" ; + }(); + + [&] (){ + ObSysVars[585].default_value_ = "-1" ; + ObSysVars[585].info_ = "The offset of optimizer traces to display" ; + ObSysVars[585].name_ = "optimizer_trace_offset" ; + ObSysVars[585].data_type_ = ObIntType ; + ObSysVars[585].min_val_ = "-2147483647" ; + ObSysVars[585].max_val_ = "2147483647" ; + ObSysVars[585].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[585].id_ = SYS_VAR_OPTIMIZER_TRACE_OFFSET ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_OPTIMIZER_TRACE_OFFSET)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_OPTIMIZER_TRACE_OFFSET] = 585 ; + ObSysVars[585].base_value_ = "-1" ; + ObSysVars[585].alias_ = "OB_SV_OPTIMIZER_TRACE_OFFSET" ; + }(); + + [&] (){ + ObSysVars[586].default_value_ = "18446744073709551615" ; + ObSysVars[586].info_ = "The maximum amount of memory available to the parser" ; + ObSysVars[586].name_ = "parser_max_mem_size" ; + ObSysVars[586].data_type_ = ObUInt64Type ; + ObSysVars[586].min_val_ = "10000000" ; + ObSysVars[586].max_val_ = "18446744073709551615" ; + ObSysVars[586].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[586].id_ = SYS_VAR_PARSER_MAX_MEM_SIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_PARSER_MAX_MEM_SIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_PARSER_MAX_MEM_SIZE] = 586 ; + ObSysVars[586].base_value_ = "18446744073709551615" ; + ObSysVars[586].alias_ = "OB_SV_PARSER_MAX_MEM_SIZE" ; + }(); + + [&] (){ + ObSysVars[587].default_value_ = "0" ; + ObSysVars[587].info_ = "For statements that invoke RAND(), the source passes two values to the replica, where they are used to seed the random number generator" ; + ObSysVars[587].name_ = "rand_seed1" ; + ObSysVars[587].data_type_ = ObUInt64Type ; + ObSysVars[587].min_val_ = "0" ; + ObSysVars[587].max_val_ = "4294967295" ; + ObSysVars[587].flags_ = ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[587].id_ = SYS_VAR_RAND_SEED1 ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_RAND_SEED1)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_RAND_SEED1] = 587 ; + ObSysVars[587].base_value_ = "0" ; + ObSysVars[587].alias_ = "OB_SV_RAND_SEED1" ; + }(); + + [&] (){ + ObSysVars[588].default_value_ = "0" ; + ObSysVars[588].info_ = "For statements that invoke RAND(), the source passes two values to the replica, where they are used to seed the random number generator" ; + ObSysVars[588].name_ = "rand_seed2" ; + ObSysVars[588].data_type_ = ObUInt64Type ; + ObSysVars[588].min_val_ = "0" ; + ObSysVars[588].max_val_ = "4294967295" ; + ObSysVars[588].flags_ = ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[588].id_ = SYS_VAR_RAND_SEED2 ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_RAND_SEED2)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_RAND_SEED2] = 588 ; + ObSysVars[588].base_value_ = "0" ; + ObSysVars[588].alias_ = "OB_SV_RAND_SEED2" ; + }(); + + [&] (){ + ObSysVars[589].default_value_ = "4096" ; + ObSysVars[589].info_ = "The size in bytes of blocks that are allocated when doing range optimization" ; + ObSysVars[589].name_ = "range_alloc_block_size" ; + ObSysVars[589].data_type_ = ObUInt64Type ; + ObSysVars[589].min_val_ = "4096" ; + ObSysVars[589].max_val_ = "18446744073709550592" ; + ObSysVars[589].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[589].id_ = SYS_VAR_RANGE_ALLOC_BLOCK_SIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_RANGE_ALLOC_BLOCK_SIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_RANGE_ALLOC_BLOCK_SIZE] = 589 ; + ObSysVars[589].base_value_ = "4096" ; + ObSysVars[589].alias_ = "OB_SV_RANGE_ALLOC_BLOCK_SIZE" ; + }(); + + [&] (){ + ObSysVars[590].default_value_ = "8388608" ; + ObSysVars[590].info_ = "The limit on memory consumption for the range optimizer" ; + ObSysVars[590].name_ = "range_optimizer_max_mem_size" ; + ObSysVars[590].data_type_ = ObUInt64Type ; + ObSysVars[590].min_val_ = "0" ; + ObSysVars[590].max_val_ = "18446744073709551615" ; + ObSysVars[590].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[590].id_ = SYS_VAR_RANGE_OPTIMIZER_MAX_MEM_SIZE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_RANGE_OPTIMIZER_MAX_MEM_SIZE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_RANGE_OPTIMIZER_MAX_MEM_SIZE] = 590 ; + ObSysVars[590].base_value_ = "8388608" ; + ObSysVars[590].alias_ = "OB_SV_RANGE_OPTIMIZER_MAX_MEM_SIZE" ; + }(); + + [&] (){ + ObSysVars[591].default_value_ = "1" ; + ObSysVars[591].info_ = "Whether the Rewriter query rewrite plugin is enabled" ; + ObSysVars[591].name_ = "rewriter_enabled" ; + ObSysVars[591].data_type_ = ObIntType ; + ObSysVars[591].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[591].id_ = SYS_VAR_REWRITER_ENABLED ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_REWRITER_ENABLED)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_REWRITER_ENABLED] = 591 ; + ObSysVars[591].base_value_ = "1" ; + ObSysVars[591].alias_ = "OB_SV_REWRITER_ENABLED" ; + }(); + + [&] (){ + ObSysVars[592].default_value_ = "0" ; + ObSysVars[592].info_ = "For internal use in MySQL" ; + ObSysVars[592].name_ = "rewriter_verbose" ; + ObSysVars[592].data_type_ = ObIntType ; + ObSysVars[592].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[592].id_ = SYS_VAR_REWRITER_VERBOSE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_REWRITER_VERBOSE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_REWRITER_VERBOSE] = 592 ; + ObSysVars[592].base_value_ = "0" ; + ObSysVars[592].alias_ = "OB_SV_REWRITER_VERBOSE" ; + }(); + + [&] (){ + ObSysVars[593].default_value_ = "1" ; + ObSysVars[593].info_ = "If this variable is enabled, the server blocks connections by clients that attempt to use accounts that have passwords stored in the old (pre-4.1) format" ; + ObSysVars[593].name_ = "secure_auth" ; + ObSysVars[593].data_type_ = ObIntType ; + ObSysVars[593].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[593].id_ = SYS_VAR_SECURE_AUTH ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_SECURE_AUTH)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_SECURE_AUTH] = 593 ; + ObSysVars[593].base_value_ = "1" ; + ObSysVars[593].alias_ = "OB_SV_SECURE_AUTH" ; + }(); + + [&] (){ + ObSysVars[594].default_value_ = "0" ; + ObSysVars[594].info_ = "This variable controls whether the sha256_password built-in authentication plugin supports proxy users" ; + ObSysVars[594].name_ = "sha256_password_proxy_users" ; + ObSysVars[594].data_type_ = ObIntType ; + ObSysVars[594].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[594].id_ = SYS_VAR_SHA256_PASSWORD_PROXY_USERS ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_SHA256_PASSWORD_PROXY_USERS)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_SHA256_PASSWORD_PROXY_USERS] = 594 ; + ObSysVars[594].base_value_ = "0" ; + ObSysVars[594].alias_ = "OB_SV_SHA256_PASSWORD_PROXY_USERS" ; + }(); + + [&] (){ + ObSysVars[595].default_value_ = "0" ; + ObSysVars[595].info_ = "which affects whether MySQL 5.6 compatibility is enabled with respect to how system and status variable information is provided by the INFORMATION_SCHEMA and Performance Schema tables, and also by the SHOW VARIABLES and SHOW STATUS statements" ; + ObSysVars[595].name_ = "show_compatibility_56" ; + ObSysVars[595].data_type_ = ObIntType ; + ObSysVars[595].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[595].id_ = SYS_VAR_SHOW_COMPATIBILITY_56 ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_SHOW_COMPATIBILITY_56)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_SHOW_COMPATIBILITY_56] = 595 ; + ObSysVars[595].base_value_ = "0" ; + ObSysVars[595].alias_ = "OB_SV_SHOW_COMPATIBILITY_56" ; + }(); + + [&] (){ + ObSysVars[596].default_value_ = "0" ; + ObSysVars[596].info_ = "Enabling this variable causes SHOW CREATE TABLE to display ROW_FORMAT regardless of whether it is the default format" ; + ObSysVars[596].name_ = "show_create_table_verbosity" ; + ObSysVars[596].data_type_ = ObIntType ; + ObSysVars[596].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[596].id_ = SYS_VAR_SHOW_CREATE_TABLE_VERBOSITY ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_SHOW_CREATE_TABLE_VERBOSITY)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_SHOW_CREATE_TABLE_VERBOSITY] = 596 ; + ObSysVars[596].base_value_ = "0" ; + ObSysVars[596].alias_ = "OB_SV_SHOW_CREATE_TABLE_VERBOSITY" ; + }(); + + [&] (){ + ObSysVars[597].default_value_ = "0" ; + ObSysVars[597].info_ = "Whether SHOW CREATE TABLE output includes comments" ; + ObSysVars[597].name_ = "show_old_temporals" ; + ObSysVars[597].data_type_ = ObIntType ; + ObSysVars[597].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[597].id_ = SYS_VAR_SHOW_OLD_TEMPORALS ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_SHOW_OLD_TEMPORALS)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_SHOW_OLD_TEMPORALS] = 597 ; + ObSysVars[597].base_value_ = "0" ; + ObSysVars[597].alias_ = "OB_SV_SHOW_OLD_TEMPORALS" ; + }(); + + [&] (){ + ObSysVars[598].default_value_ = "1" ; + ObSysVars[598].info_ = "If set to OFF, MySQL aborts SELECT statements that are likely to take a very long time to execute" ; + ObSysVars[598].name_ = "sql_big_selects" ; + ObSysVars[598].data_type_ = ObIntType ; + ObSysVars[598].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[598].id_ = SYS_VAR_SQL_BIG_SELECTS ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_SQL_BIG_SELECTS)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_SQL_BIG_SELECTS] = 598 ; + ObSysVars[598].base_value_ = "1" ; + ObSysVars[598].alias_ = "OB_SV_SQL_BIG_SELECTS" ; + }(); + + [&] (){ + ObSysVars[599].default_value_ = "1" ; + ObSysVars[599].info_ = "This variable controls whether updates to a view can be made when the view does not contain all columns of the primary key defined in the underlying table, if the update statement contains a LIMIT clause" ; + ObSysVars[599].name_ = "updatable_views_with_limit" ; + ObSysVars[599].data_type_ = ObIntType ; + ObSysVars[599].enum_names_ = "[u'OFF', u'ON', u'NO', u'YES']" ; + ObSysVars[599].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[599].id_ = SYS_VAR_UPDATABLE_VIEWS_WITH_LIMIT ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_UPDATABLE_VIEWS_WITH_LIMIT)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_UPDATABLE_VIEWS_WITH_LIMIT] = 599 ; + ObSysVars[599].base_value_ = "1" ; + ObSysVars[599].alias_ = "OB_SV_UPDATABLE_VIEWS_WITH_LIMIT" ; + }(); + + [&] (){ + ObSysVars[600].default_value_ = "" ; + ObSysVars[600].info_ = "The path name of the dictionary file that validate_password uses for checking passwords." ; + ObSysVars[600].name_ = "validate_password_dictionary_file" ; + ObSysVars[600].data_type_ = ObVarcharType ; + ObSysVars[600].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[600].id_ = SYS_VAR_VALIDATE_PASSWORD_DICTIONARY_FILE ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_VALIDATE_PASSWORD_DICTIONARY_FILE)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_VALIDATE_PASSWORD_DICTIONARY_FILE] = 600 ; + ObSysVars[600].base_value_ = "" ; + ObSysVars[600].alias_ = "OB_SV_VALIDATE_PASSWORD_DICTIONARY_FILE" ; + }(); + + [&] (){ + ObSysVars[601].default_value_ = "100" ; + ObSysVars[601].info_ = "" ; + ObSysVars[601].name_ = "delayed_insert_limit" ; + ObSysVars[601].data_type_ = ObUInt64Type ; + ObSysVars[601].min_val_ = "1" ; + ObSysVars[601].max_val_ = "18446744073709551615" ; + ObSysVars[601].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY ; + ObSysVars[601].id_ = SYS_VAR_DELAYED_INSERT_LIMIT ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_DELAYED_INSERT_LIMIT)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_DELAYED_INSERT_LIMIT] = 601 ; + ObSysVars[601].base_value_ = "100" ; + ObSysVars[601].alias_ = "OB_SV_DELAYED_INSERT_LIMIT" ; + }(); + + [&] (){ + ObSysVars[602].default_value_ = "" ; + ObSysVars[602].info_ = "NDB engine version in ndb-x.y.z format" ; + ObSysVars[602].name_ = "ndb_version" ; + ObSysVars[602].data_type_ = ObVarcharType ; + ObSysVars[602].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[602].id_ = SYS_VAR_NDB_VERSION ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_NDB_VERSION)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_NDB_VERSION] = 602 ; + ObSysVars[602].base_value_ = "" ; + ObSysVars[602].alias_ = "OB_SV_NDB_VERSION" ; + }(); + + [&] (){ + ObSysVars[603].default_value_ = "1" ; + ObSysVars[603].info_ = "This variable is available if the server was compiled using OpenSSL. It controls whether the server autogenerates SSL key and certificate files in the data directory, if they do not already exist" ; + ObSysVars[603].name_ = "auto_generate_certs" ; + ObSysVars[603].data_type_ = ObIntType ; + ObSysVars[603].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::MYSQL_ONLY | ObSysVarFlag::READONLY ; + ObSysVars[603].id_ = SYS_VAR_AUTO_GENERATE_CERTS ; + cur_max_var_id = MAX(cur_max_var_id, static_cast(SYS_VAR_AUTO_GENERATE_CERTS)) ; + ObSysVarsIdToArrayIdx[SYS_VAR_AUTO_GENERATE_CERTS] = 603 ; + ObSysVars[603].base_value_ = "1" ; + ObSysVars[603].alias_ = "OB_SV_AUTO_GENERATE_CERTS" ; }(); if (cur_max_var_id >= ObSysVarFactory::OB_MAX_SYS_VAR_ID) { @@ -6382,7 +8416,7 @@ static struct VarsInit{ } }vars_init; -static int64_t var_amount = 452; +static int64_t var_amount = 604; int64_t ObSysVariables::get_all_sys_var_count(){ return ObSysVarFactory::ALL_SYS_VARS_COUNT;} ObSysVarClassType ObSysVariables::get_sys_var_id(int64_t i){ return ObSysVars[i].id_;} diff --git a/src/share/system_variable/ob_system_variable_init.json b/src/share/system_variable/ob_system_variable_init.json index 80381d5dd..96a3146b5 100644 --- a/src/share/system_variable/ob_system_variable_init.json +++ b/src/share/system_variable/ob_system_variable_init.json @@ -2189,12 +2189,17 @@ "aes-256-cfb128", "aes-128-ofb", "aes-192-ofb", - "aes-256-ofb" + "aes-256-ofb", + "sm4-ecb", + "sm4-cbc", + "sm4-cfb", + "sm4-ofb" ], "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "" + "ref_url": "", + "on_check_and_convert_func": "ObSysVarOnCheckFuncs::check_and_convert_block_encryption_mode" }, "nls_date_format": { "id": 10064, @@ -5998,6 +6003,7 @@ "data_type": "int", "info": "control default collation for utf8mb4", "flags": "GLOBAL | SESSION | INFLUENCE_PLAN | NEED_SERIALIZE | MYSQL_ONLY", + "on_check_and_convert_func": "ObSysVarOnCheckFuncs::check_default_value_for_utf8mb4", "get_meta_type_func": "ObSysVarGetMetaTypeFuncs::get_meta_type_varchar", "to_select_obj_func": "ObSysVarToObjFuncs::to_obj_collation", "to_show_str_func": "ObSysVarToStrFuncs::to_str_collation", @@ -6040,13 +6046,12 @@ "default_value": "0", "base_value": "0", "data_type": "int", - "info": "mock for mysql5.7", + "info": "The value to be used by the following INSERT or ALTER TABLE statement when inserting an AUTO_INCREMENT value. Merely simulates MySQL 5.7.", "flags": "SESSION | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html" }, "join_buffer_size": { "id": 10329, @@ -6054,27 +6059,25 @@ "default_value": "262144", "base_value": "262144", "data_type": "int", - "info": "mock for mysql5.7", + "info": "The minimum size of the buffer that is used for plain index scans, range index scans, and joins that do not use indexes and thus perform full table scans. Merely simulates MySQL 5.7.", "flags": "GLOBAL | SESSION | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html" }, "max_join_size": { "id": 10330, "name": "max_join_size", - "default_value": "18446744073709500416", - "base_value": "18446744073709500416", - "data_type": "int", - "info": "mock for mysql5.7", + "default_value": "18446744073709547520", + "base_value": "18446744073709547520", + "data_type": "uint", + "info": "Do not permit statements that probably need to examine more than max_join_size rows (for single-table statements) or row combinations (for multiple-table statements) or that are likely to do more than max_join_size disk seeks. Merely simulates MySQL 5.7.", "flags": "GLOBAL | SESSION | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html" }, "max_length_for_sort_data": { "id": 10331, @@ -6082,13 +6085,12 @@ "default_value": "1024", "base_value": "1024", "data_type": "int", - "info": "mock for mysql5.7", + "info": "The cutoff on the size of index values that determines which filesort algorithm to use. Merely simulates MySQL 5.7.", "flags": "GLOBAL | SESSION | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html" }, "max_prepared_stmt_count": { "id": 10332, @@ -6096,13 +6098,12 @@ "default_value": "16382", "base_value": "16382", "data_type": "int", - "info": "mock for mysql5.7", + "info": "This variable limits the total number of prepared statements in the server. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html" }, "max_sort_length": { "id": 10333, @@ -6110,13 +6111,12 @@ "default_value": "1024", "base_value": "1024", "data_type": "int", - "info": "mock for mysql5.7", + "info": "The number of bytes to use when sorting data values. Merely simulates MySQL 5.7.", "flags": "GLOBAL | SESSION | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html" }, "min_examined_row_limit": { "id": 10334, @@ -6124,13 +6124,12 @@ "default_value": "0", "base_value": "0", "data_type": "int", - "info": "mock for mysql5.7", + "info": "Queries that examine fewer than this number of rows are not logged to the slow query log. Merely simulates MySQL 5.7.", "flags": "GLOBAL | SESSION | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html" }, "multi_range_count": { "id": 10335, @@ -6138,13 +6137,12 @@ "default_value": "256", "base_value": "256", "data_type": "int", - "info": "mock for mysql5.7", + "info": "This variable has no effect. Merely simulates MySQL 5.7.", "flags": "GLOBAL | SESSION | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html" }, "mysqlx_connect_timeout": { "id": 10336, @@ -6152,13 +6150,12 @@ "default_value": "30", "base_value": "30", "data_type": "int", - "info": "mock for mysql5.7", + "info": "The number of seconds X Plugin waits for the first packet to be received from newly connected clients. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/x-plugin-options-system-variables.html" }, "mysqlx_idle_worker_thread_timeout": { "id": 10337, @@ -6166,13 +6163,12 @@ "default_value": "60", "base_value": "60", "data_type": "int", - "info": "mock for mysql5.7", + "info": "The number of seconds after which idle worker threads are terminated. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/x-plugin-options-system-variables.html" }, "mysqlx_max_allowed_packet": { "id": 10338, @@ -6180,13 +6176,12 @@ "default_value": "67108864", "base_value": "67108864", "data_type": "int", - "info": "mock for mysql5.7", + "info": "The maximum size of network packets that can be received by X Plugin. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/x-plugin-options-system-variables.html" }, "mysqlx_max_connections": { "id": 10339, @@ -6194,13 +6189,12 @@ "default_value": "100", "base_value": "100", "data_type": "int", - "info": "mock for mysql5.7", + "info": "The maximum number of concurrent client connections X Plugin can accept. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/x-plugin-options-system-variables.html" }, "mysqlx_min_worker_threads": { "id": 10340, @@ -6208,27 +6202,25 @@ "default_value": "2", "base_value": "2", "data_type": "int", - "info": "mock for mysql5.7", + "info": "The minimum number of worker threads used by X Plugin for handling client requests. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/x-plugin-options-system-variables.html" }, "performance_schema_show_processlist": { "id": 10341, "name": "performance_schema_show_processlist", - "default_value": "OFF", - "base_value": "OFF", - "data_type": "int", - "info": "mock for mysql5.7", + "default_value": "0", + "base_value": "0", + "data_type": "bool", + "info": "The variable determines which SHOW PROCESSLIST implementation to use. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/performance-schema-system-variables.html" }, "query_alloc_block_size": { "id": 10342, @@ -6236,13 +6228,12 @@ "default_value": "8192", "base_value": "8192", "data_type": "int", - "info": "mock for mysql5.7", + "info": "The allocation size in bytes of memory blocks that are allocated for objects created during statement parsing and execution. Merely simulates MySQL 5.7.", "flags": "GLOBAL | SESSION | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html" }, "query_prealloc_size": { "id": 10343, @@ -6250,13 +6241,12 @@ "default_value": "8192", "base_value": "8192", "data_type": "int", - "info": "mock for mysql5.7", + "info": "The size in bytes of the persistent buffer used for statement parsing and execution. Merely simulates MySQL 5.7.", "flags": "GLOBAL | SESSION | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html" }, "slow_query_log": { "id": 10344, @@ -6264,27 +6254,25 @@ "default_value": "0", "base_value": "0", "data_type": "int", - "info": "mock for mysql5.7", + "info": "Whether the slow query log is enabled. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html" }, "slow_query_log_file": { "id": 10345, "name": "slow_query_log_file", "default_value": "/usr/local/mysql/data/obrd-slow.log", "base_value": "/usr/local/mysql/data/obrd-slow.log", - "data_type": "int", - "info": "mock for mysql5.7", + "data_type": "varchar", + "info": "The name of the slow query log file. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html" }, "sort_buffer_size": { "id": 10346, @@ -6292,13 +6280,12 @@ "default_value": "262144", "base_value": "262144", "data_type": "int", - "info": "mock for mysql5.7", + "info": "Each session that must perform a sort allocates a buffer of this size. Merely simulates MySQL 5.7.", "flags": "GLOBAL | SESSION | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html" }, "sql_buffer_result": { "id": 10347, @@ -6306,13 +6293,12 @@ "default_value": "0", "base_value": "0", "data_type": "int", - "info": "mock for mysql5.7", + "info": "If enabled, sql_buffer_result forces results from SELECT statements to be put into temporary tables. Merely simulates MySQL 5.7.", "flags": "GLOBAL | SESSION | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html" }, "binlog_cache_size": { "id": 10348, @@ -6320,13 +6306,12 @@ "default_value": "32768", "base_value": "32768", "data_type": "int", - "info": "mock for mysql5.7", + "info": "binlog_cache_size sets the size for the transaction cache only. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/replication-options-binary-log.html" }, "binlog_direct_non_transactional_updates": { "id": 10349, @@ -6334,27 +6319,29 @@ "default_value": "0", "base_value": "0", "data_type": "int", - "info": "mock for mysql5.7", + "info": "Enabling binlog_direct_non_transactional_updates causes updates to nontransactional tables to be written directly to the binary log, rather than to the transaction cache. Merely simulates MySQL 5.7.", "flags": "GLOBAL | SESSION | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/replication-options-binary-log.html" }, "binlog_error_action": { "id": 10350, "name": "binlog_error_action", - "default_value": "ABORT_SERVER", - "base_value": "ABORT_SERVER", - "data_type": "int", - "info": "mock for mysql5.7", + "default_value": "1", + "base_value": "1", + "data_type": "enum", + "info": "Controls what happens when the server encounters an error. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", + "enum_names": [ + "IGNORE_ERROR", + "ABORT_SERVER" + ], "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/replication-options-binary-log.html" }, "binlog_group_commit_sync_delay": { "id": 10351, @@ -6362,13 +6349,12 @@ "default_value": "0", "base_value": "0", "data_type": "int", - "info": "mock for mysql5.7", + "info": "Controls how many microseconds the binary log commit waits before synchronizing the binary log file to disk. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/replication-options-binary-log.html" }, "binlog_group_commit_sync_no_delay_count": { "id": 10352, @@ -6376,13 +6362,12 @@ "default_value": "0", "base_value": "0", "data_type": "int", - "info": "mock for mysql5.7", + "info": "The maximum number of transactions to wait for before aborting the current delay as specified by binlog_group_commit_sync_delay. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/replication-options-binary-log.html" }, "binlog_max_flush_queue_time": { "id": 10353, @@ -6390,13 +6375,12 @@ "default_value": "0", "base_value": "0", "data_type": "int", - "info": "mock for mysql5.7", + "info": "This variable no longer has any effect. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/replication-options-binary-log.html" }, "binlog_order_commits": { "id": 10354, @@ -6404,13 +6388,12 @@ "default_value": "1", "base_value": "1", "data_type": "int", - "info": "mock for mysql5.7", + "info": "When this variable is enabled on a replication source server, transaction commit instructions issued to storage engines are serialized on a single thread. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/replication-options-binary-log.html" }, "binlog_stmt_cache_size": { "id": 10355, @@ -6418,13 +6401,12 @@ "default_value": "32768", "base_value": "32768", "data_type": "int", - "info": "mock for mysql5.7", + "info": "This variable determines the size of the cache for the binary log to hold nontransactional statements issued during a transaction. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/replication-options-binary-log.html" }, "binlog_transaction_dependency_history_size": { "id": 10356, @@ -6432,27 +6414,30 @@ "default_value": "25000", "base_value": "25000", "data_type": "int", - "info": "mock for mysql5.7", + "info": "Sets an upper limit on the number of row hashes which are kept in memory and used for looking up the transaction that last modified a given row. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/replication-options-binary-log.html" }, "binlog_transaction_dependency_tracking": { "id": 10357, "name": "binlog_transaction_dependency_tracking", - "default_value": "COMMIT_ORDER", - "base_value": "COMMIT_ORDER", - "data_type": "int", - "info": "mock for mysql5.7", + "default_value": "0", + "base_value": "0", + "data_type": "enum", + "info": "The source of dependency information that the source uses to determine which transactions can be executed in parallel by the replica's multithreaded applier. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", + "enum_names": [ + "COMMIT_ORDER", + "WRITESET", + "WRITESET_SESSION" + ], "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/replication-options-binary-log.html" }, "expire_logs_days": { "id": 10358, @@ -6460,13 +6445,12 @@ "default_value": "0", "base_value": "0", "data_type": "int", - "info": "mock for mysql5.7", + "info": "The number of days for automatic binary log file removal. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/replication-options-binary-log.html" }, "innodb_flush_log_at_timeout": { "id": 10359, @@ -6474,13 +6458,12 @@ "default_value": "1", "base_value": "1", "data_type": "int", - "info": "mock for mysql5.7", + "info": "Write and flush the logs every N seconds. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html" }, "innodb_flush_log_at_trx_commit": { "id": 10360, @@ -6488,27 +6471,25 @@ "default_value": "1", "base_value": "1", "data_type": "int", - "info": "mock for mysql5.7", + "info": "Controls the balance between strict ACID compliance for commit operations and higher performance. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html" }, "innodb_log_checkpoint_now": { "id": 10361, "name": "innodb_log_checkpoint_now", - "default_value": "OFF", - "base_value": "OFF", - "data_type": "int", - "info": "mock for mysql5.7", + "default_value": "0", + "base_value": "0", + "data_type": "bool", + "info": "Enable this debug option to force InnoDB to write a checkpoint. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html" }, "innodb_log_checksums": { "id": 10362, @@ -6516,13 +6497,12 @@ "default_value": "1", "base_value": "1", "data_type": "int", - "info": "mock for mysql5.7", + "info": "Enables or disables checksums for redo log pages. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html" }, "innodb_log_compressed_pages": { "id": 10363, @@ -6530,13 +6510,12 @@ "default_value": "1", "base_value": "1", "data_type": "int", - "info": "mock for mysql5.7", + "info": "Specifies whether images of re-compressed pages are written to the redo log. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html" }, "innodb_log_write_ahead_size": { "id": 10364, @@ -6544,13 +6523,12 @@ "default_value": "8192", "base_value": "8192", "data_type": "int", - "info": "mock for mysql5.7", + "info": "Defines the write-ahead block size for the redo log, in bytes. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html" }, "innodb_max_undo_log_size": { "id": 10365, @@ -6558,13 +6536,12 @@ "default_value": "1073741824", "base_value": "1073741824", "data_type": "int", - "info": "mock for mysql5.7", + "info": "Defines a threshold size for undo tablespaces. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html" }, "innodb_online_alter_log_max_size": { "id": 10366, @@ -6572,13 +6549,12 @@ "default_value": "134217728", "base_value": "134217728", "data_type": "int", - "info": "mock for mysql5.7", + "info": "Specifies an upper limit in bytes on the size of the temporary log files used during online DDL operations for InnoDB tables. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html" }, "innodb_undo_log_truncate": { "id": 10367, @@ -6586,13 +6562,12 @@ "default_value": "0", "base_value": "0", "data_type": "int", - "info": "mock for mysql5.7", + "info": "When enabled, undo tablespaces that exceed the threshold value defined by innodb_max_undo_log_size are marked for truncation. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html" }, "innodb_undo_logs": { "id": 10368, @@ -6600,13 +6575,12 @@ "default_value": "128", "base_value": "128", "data_type": "int", - "info": "mock for mysql5.7", + "info": "Defines the number of rollback segments used by InnoDB. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html" }, "log_bin_trust_function_creators": { "id": 10369, @@ -6614,13 +6588,12 @@ "default_value": "0", "base_value": "0", "data_type": "int", - "info": "mock for mysql5.7", + "info": "It controls whether stored function creators can be trusted not to create stored functions that causes unsafe events to be written to the binary log. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/replication-options-binary-log.html" }, "log_bin_use_v1_row_events": { "id": 10370, @@ -6628,13 +6601,12 @@ "default_value": "0", "base_value": "0", "data_type": "int", - "info": "mock for mysql5.7", + "info": "Whether Version 2 binary logging is in use. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/replication-options-binary-log.html" }, "log_builtin_as_identified_by_password": { "id": 10371, @@ -6642,27 +6614,25 @@ "default_value": "0", "base_value": "0", "data_type": "int", - "info": "mock for mysql5.7", + "info": "This variable affects binary logging of user-management statements. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/replication-options-binary-log.html" }, "max_binlog_cache_size": { "id": 10372, "name": "max_binlog_cache_size", "default_value": "18446744073709500416", "base_value": "18446744073709500416", - "data_type": "int", - "info": "mock for mysql5.7", + "data_type": "uint", + "info": "If a transaction requires more than this many bytes, the server generates a Multi-statement transaction required more than 'max_binlog_cache_size' bytes of storage error. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/replication-options-binary-log.html" }, "max_binlog_size": { "id": 10373, @@ -6670,27 +6640,25 @@ "default_value": "1073741824", "base_value": "1073741824", "data_type": "int", - "info": "mock for mysql5.7", + "info": "If a write to the binary log causes the current log file size to exceed the value of this variable, the server rotates the binary logs. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/replication-options-binary-log.html" }, "max_binlog_stmt_cache_size": { "id": 10374, "name": "max_binlog_stmt_cache_size", "default_value": "18446744073709500416", "base_value": "18446744073709500416", - "data_type": "int", - "info": "mock for mysql5.7", + "data_type": "uint", + "info": "If nontransactional statements within a transaction require more than this many bytes of memory, the server generates an error. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/replication-options-binary-log.html" }, "max_relay_log_size": { "id": 10375, @@ -6698,27 +6666,25 @@ "default_value": "0", "base_value": "0", "data_type": "int", - "info": "mock for mysql5.7", + "info": "The size at which the server rotates relay log files automatically. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/replication-options-replica.html" }, "relay_log_info_repository": { "id": 10376, "name": "relay_log_info_repository", "default_value": "FILE", "base_value": "FILE", - "data_type": "int", - "info": "mock for mysql5.7", + "data_type": "varchar", + "info": "The setting of this variable determines whether the replica server stores its applier metadata repository as an InnoDB table in the mysql system database, or as a file in the data directory. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/replication-options-replica.html" }, "relay_log_purge": { "id": 10377, @@ -6726,13 +6692,12 @@ "default_value": "1", "base_value": "1", "data_type": "int", - "info": "mock for mysql5.7", + "info": "Disables or enables automatic purging of relay log files as soon as they are not needed any more. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/replication-options-replica.html" }, "sync_binlog": { "id": 10378, @@ -6740,13 +6705,12 @@ "default_value": "1", "base_value": "1", "data_type": "int", - "info": "mock for mysql5.7", + "info": "Controls how often the MySQL server synchronizes the binary log to disk. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/replication-options-binary-log.html" }, "sync_relay_log": { "id": 10379, @@ -6754,13 +6718,12 @@ "default_value": "10000", "base_value": "10000", "data_type": "int", - "info": "mock for mysql5.7", + "info": "If the value of this variable is greater than 0, the MySQL server synchronizes its relay log to disk after every sync_relay_log events are written to the relay log. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/replication-options-replica.html" }, "sync_relay_log_info": { "id": 10380, @@ -6768,13 +6731,12 @@ "default_value": "10000", "base_value": "10000", "data_type": "int", - "info": "mock for mysql5.7", + "info": "Setting this variable takes effect for all replication channels immediately, including running channels. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/replication-options-replica.html" }, "innodb_deadlock_detect": { "id": 10381, @@ -6782,13 +6744,12 @@ "default_value": "1", "base_value": "1", "data_type": "int", - "info": "mock for mysql5.7", + "info": "This option is used to disable deadlock detection. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html" }, "innodb_lock_wait_timeout": { "id": 10382, @@ -6796,13 +6757,12 @@ "default_value": "50", "base_value": "50", "data_type": "int", - "info": "mock for mysql5.7", + "info": "The length of time in seconds an InnoDB transaction waits for a row lock before giving up. Merely simulates MySQL 5.7.", "flags": "GLOBAL | SESSION | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html" }, "innodb_print_all_deadlocks": { "id": 10383, @@ -6810,13 +6770,12 @@ "default_value": "0", "base_value": "0", "data_type": "int", - "info": "mock for mysql5.7", + "info": "When this option is enabled, information about all deadlocks in InnoDB user transactions is recorded in the mysqld error log. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html" }, "innodb_table_locks": { "id": 10384, @@ -6824,27 +6783,25 @@ "default_value": "1", "base_value": "1", "data_type": "int", - "info": "mock for mysql5.7", + "info": "The default value is 1, which means that LOCK TABLES causes InnoDB to lock a table internally if autocommit = 0. Merely simulates MySQL 5.7.", "flags": "GLOBAL | SESSION | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html" }, "max_write_lock_count": { "id": 10385, "name": "max_write_lock_count", "default_value": "18446744073709500416", "base_value": "18446744073709500416", - "data_type": "int", - "info": "mock for mysql5.7", + "data_type": "uint", + "info": "if max_write_lock_count is set to some low value (say, 10), read lock requests may be preferred over pending write lock requests if the read lock requests have already been passed over in favor of 10 write lock requests. Merely simulates MySQL 5.7.", "flags": "GLOBAL | MYSQL_ONLY", "publish_version": "424", "info_cn": "", "background_cn": "", - "ref_url": "", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html" }, "_ob_enable_role_ids": { "id": 10386, @@ -10546,8 +10503,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_character_sets_dir", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_character_sets_dir" }, "date_format": { "id": 10646, @@ -10560,8 +10516,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_date_format", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_date_format" }, "datetime_format": { "id": 10647, @@ -10574,8 +10529,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_datetime_format", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_datetime_format" }, "disconnect_on_expired_password": { "id": 10648, @@ -10588,8 +10542,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_disconnect_on_expired_password", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_disconnect_on_expired_password" }, "external_user": { "id": 10649, @@ -10602,8 +10555,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_external_user", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_external_user" }, "have_crypt": { "id": 10650, @@ -10616,8 +10568,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_have_crypt", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_have_crypt" }, "have_dynamic_loading": { "id": 10651, @@ -10630,8 +10581,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_have_dynamic_loading", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_have_dynamic_loading" }, "keyring_aws_conf_file": { "id": 10652, @@ -10644,8 +10594,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/keyring-system-variables.html#sysvar_keyring_aws_conf_file", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/keyring-system-variables.html#sysvar_keyring_aws_conf_file" }, "keyring_aws_data_file": { "id": 10653, @@ -10658,8 +10607,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/keyring-system-variables.html#sysvar_keyring_aws_data_file", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/keyring-system-variables.html#sysvar_keyring_aws_data_file" }, "language": { "id": 10654, @@ -10672,8 +10620,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_language", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_language" }, "lc_messages_dir": { "id": 10655, @@ -10686,8 +10633,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_lc_messages_dir", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_lc_messages_dir" }, "lower_case_file_system": { "id": 10656, @@ -10700,8 +10646,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_lower_case_file_system", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_lower_case_file_system" }, "max_digest_length": { "id": 10657, @@ -10714,8 +10659,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_max_digest_length", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_max_digest_length" }, "ndbinfo_database": { "id": 10658, @@ -10728,8 +10672,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html#sysvar_ndbinfo_database", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html#sysvar_ndbinfo_database" }, "ndbinfo_table_prefix": { "id": 10659, @@ -10742,8 +10685,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html#sysvar_ndbinfo_database", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html#sysvar_ndbinfo_database" }, "ndbinfo_version": { "id": 10660, @@ -10756,8 +10698,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html#sysvar_ndbinfo_version", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html#sysvar_ndbinfo_version" }, "ndb_batch_size": { "id": 10661, @@ -10770,8 +10711,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html" }, "ndb_cluster_connection_pool": { "id": 10662, @@ -10786,8 +10726,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html" }, "ndb_cluster_connection_pool_nodeids": { "id": 10663, @@ -10800,8 +10739,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html" }, "ndb_log_apply_status": { "id": 10664, @@ -10814,8 +10752,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html" }, "ndb_log_bin": { "id": 10665, @@ -10828,8 +10765,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html#sysvar_ndb_log_bin", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html#sysvar_ndb_log_bin" }, "ndb_log_fail_terminate": { "id": 10666, @@ -10842,8 +10778,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html" }, "ndb_log_orig": { "id": 10667, @@ -10856,8 +10791,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html#sysvar_ndb_log_orig", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html#sysvar_ndb_log_orig" }, "ndb_log_transaction_id": { "id": 10668, @@ -10870,8 +10804,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html#sysvar_ndb_log_transaction_id", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html#sysvar_ndb_log_transaction_id" }, "ndb_optimized_node_selection": { "id": 10669, @@ -10884,8 +10817,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html#sysvar_ndb_optimized_node_selection", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html#sysvar_ndb_optimized_node_selection" }, "Ndb_system_name": { "id": 10670, @@ -10898,8 +10830,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html#statvar_Ndb_system_name", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html#statvar_Ndb_system_name" }, "ndb_use_copying_alter_table": { "id": 10671, @@ -10912,8 +10843,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html#sysvar_ndb_use_copying_alter_table", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html#sysvar_ndb_use_copying_alter_table" }, "ndb_version_string": { "id": 10672, @@ -10926,8 +10856,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html#sysvar_ndb_version_string", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html#sysvar_ndb_version_string" }, "ndb_wait_connected": { "id": 10673, @@ -10942,8 +10871,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html" }, "ndb_wait_setup": { "id": 10674, @@ -10958,8 +10886,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html" }, "proxy_user": { "id": 10675, @@ -10972,8 +10899,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_proxy_user", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_proxy_user" }, "sha256_password_auto_generate_rsa_keys": { "id": 10676, @@ -10986,8 +10912,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_sha256_password_auto_generate_rsa_keys", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_sha256_password_auto_generate_rsa_keys" }, "sha256_password_private_key_path": { "id": 10677, @@ -11000,8 +10925,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_sha256_password_private_key_path", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_sha256_password_private_key_path" }, "sha256_password_public_key_path": { "id": 10678, @@ -11014,8 +10938,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_sha256_password_public_key_path", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_sha256_password_public_key_path" }, "skip_show_database": { "id": 10679, @@ -11028,8 +10951,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_skip_show_database", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_skip_show_database" }, "plugin_load": { "id": 10680, @@ -11042,8 +10964,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_plugin-load", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_plugin-load" }, "plugin_load_add": { "id": 10681, @@ -11056,8 +10977,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_plugin-load-add", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_plugin-load-add" }, "big_tables": { "id": 10682, @@ -11070,8 +10990,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_big_tables", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_big_tables" }, "check_proxy_users": { "id": 10683, @@ -11084,8 +11003,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_check_proxy_users", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_check_proxy_users" }, "connection_control_failed_connections_threshold": { "id": 10684, @@ -11100,8 +11018,7 @@ "max_val": "2147483647", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/connection-control-variables.html#sysvar_connection_control_failed_connections_threshold", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/connection-control-variables.html#sysvar_connection_control_failed_connections_threshold" }, "connection_control_max_connection_delay": { "id": 10685, @@ -11116,8 +11033,7 @@ "max_val": "2147483647", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/connection-control-variables.html#sysvar_connection_control_max_connection_delay", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/connection-control-variables.html#sysvar_connection_control_max_connection_delay" }, "connection_control_min_connection_delay": { "id": 10686, @@ -11132,8 +11048,7 @@ "max_val": "2147483647", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/connection-control-variables.html#sysvar_connection_control_min_connection_delay", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/connection-control-variables.html#sysvar_connection_control_min_connection_delay" }, "default_week_format": { "id": 10687, @@ -11148,8 +11063,7 @@ "max_val": "7", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_default_week_format", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_default_week_format" }, "delayed_insert_timeout": { "id": 10688, @@ -11164,8 +11078,7 @@ "max_val": "31536000", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_delayed_insert_timeout", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_delayed_insert_timeout" }, "delayed_queue_size": { "id": 10689, @@ -11180,8 +11093,7 @@ "max_val": "18446744073709551615", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_delayed_queue_size", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_delayed_queue_size" }, "eq_range_index_dive_limit": { "id": 10690, @@ -11196,8 +11108,7 @@ "max_val": "4294967295", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_eq_range_index_dive_limit", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_eq_range_index_dive_limit" }, "innodb_stats_auto_recalc": { "id": 10691, @@ -11210,8 +11121,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_stats_auto_recalc", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_stats_auto_recalc" }, "innodb_stats_include_delete_marked": { "id": 10692, @@ -11224,8 +11134,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_stats_include_delete_marked", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_stats_include_delete_marked" }, "innodb_stats_method": { "id": 10693, @@ -11243,8 +11152,7 @@ ], "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_stats_method", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_stats_method" }, "innodb_stats_on_metadata": { "id": 10694, @@ -11257,8 +11165,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_stats_on_metadata", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_stats_on_metadata" }, "version_tokens_session": { "id": 10695, @@ -11271,8 +11178,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/version-tokens-reference.html#sysvar_version_tokens_session", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/version-tokens-reference.html#sysvar_version_tokens_session" }, "innodb_stats_persistent_sample_pages": { "id": 10696, @@ -11287,8 +11193,7 @@ "max_val": "18446744073709551615", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_stats_persistent_sample_pages", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_stats_persistent_sample_pages" }, "innodb_stats_sample_pages": { "id": 10697, @@ -11303,8 +11208,7 @@ "max_val": "18446744073709551615", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_stats_sample_pages", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_stats_sample_pages" }, "innodb_stats_transient_sample_pages": { "id": 10698, @@ -11319,8 +11223,7 @@ "max_val": "18446744073709551615", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_stats_transient_sample_pagess", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_stats_transient_sample_pagess" }, "keyring_aws_cmk_id": { "id": 10699, @@ -11333,8 +11236,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/keyring-system-variables.html#sysvar_keyring_aws_cmk_id", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/keyring-system-variables.html#sysvar_keyring_aws_cmk_id" }, "keyring_aws_region": { "id": 10700, @@ -11376,8 +11278,7 @@ ], "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/keyring-system-variables.html#sysvar_keyring_aws_region", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/keyring-system-variables.html#sysvar_keyring_aws_region" }, "keyring_encrypted_file_data": { "id": 10701, @@ -11390,8 +11291,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/keyring-system-variables.html#sysvar_keyring_encrypted_file_data", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/keyring-system-variables.html#sysvar_keyring_encrypted_file_data" }, "keyring_encrypted_file_password": { "id": 10702, @@ -11404,8 +11304,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/keyring-system-variables.html#sysvar_keyring_encrypted_file_password", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/keyring-system-variables.html#sysvar_keyring_encrypted_file_password" }, "keyring_file_data": { "id": 10703, @@ -11418,8 +11317,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/keyring-system-variables.html#sysvar_keyring_file_data", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/keyring-system-variables.html#sysvar_keyring_file_data" }, "keyring_okv_conf_dir": { "id": 10704, @@ -11432,8 +11330,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/keyring-system-variables.html#sysvar_keyring_okv_conf_dir", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/keyring-system-variables.html#sysvar_keyring_okv_conf_dir" }, "keyring_operations": { "id": 10705, @@ -11446,8 +11343,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/keyring-system-variables.html#sysvar_keyring_operations", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/keyring-system-variables.html#sysvar_keyring_operations" }, "optimizer_switch": { "id": 10706, @@ -11460,8 +11356,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_optimizer_switch", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_optimizer_switch" }, "max_connect_errors": { "id": 10707, @@ -11476,8 +11371,7 @@ "max_val": "18446744073709551615", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_max_connect_errors", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_max_connect_errors" }, "mysql_firewall_mode": { "id": 10708, @@ -11490,8 +11384,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/firewall-reference.html#sysvar_mysql_firewall_mode", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/firewall-reference.html#sysvar_mysql_firewall_mode" }, "mysql_firewall_trace": { "id": 10709, @@ -11504,8 +11397,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/firewall-reference.html#sysvar_mysql_firewall_trace", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/firewall-reference.html#sysvar_mysql_firewall_trace" }, "mysql_native_password_proxy_users": { "id": 10710, @@ -11518,8 +11410,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_mysql_native_password_proxy_users", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_mysql_native_password_proxy_users" }, "net_retry_count": { "id": 10711, @@ -11532,8 +11423,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_net_retry_count", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_net_retry_count" }, "new": { "id": 10712, @@ -11546,8 +11436,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_new", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_new" }, "old_passwords": { "id": 10713, @@ -11565,8 +11454,7 @@ ], "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_old_passwords", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_old_passwords" }, "optimizer_prune_level": { "id": 10714, @@ -11581,8 +11469,7 @@ "max_val": "1", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_optimizer_prune_level", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_optimizer_prune_level" }, "optimizer_search_depth": { "id": 10715, @@ -11597,8 +11484,7 @@ "max_val": "62", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_optimizer_search_depth", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_optimizer_search_depth" }, "optimizer_trace": { "id": 10716, @@ -11611,8 +11497,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_optimizer_trace", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_optimizer_trace" }, "optimizer_trace_features": { "id": 10717, @@ -11625,8 +11510,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_optimizer_trace_features", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_optimizer_trace_features" }, "optimizer_trace_limit": { "id": 10718, @@ -11641,8 +11525,7 @@ "max_val": "2147483647", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_optimizer_trace_limit", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_optimizer_trace_limit" }, "optimizer_trace_max_mem_size": { "id": 10719, @@ -11657,8 +11540,7 @@ "max_val": "4294967295", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_optimizer_trace_max_mem_size", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_optimizer_trace_max_mem_size" }, "optimizer_trace_offset": { "id": 10720, @@ -11673,8 +11555,7 @@ "max_val": "2147483647", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_optimizer_trace_offset", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_optimizer_trace_offset" }, "parser_max_mem_size": { "id": 10721, @@ -11689,8 +11570,7 @@ "max_val": "18446744073709551615", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_parser_max_mem_size", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_parser_max_mem_size" }, "rand_seed1": { "id": 10722, @@ -11705,8 +11585,7 @@ "max_val": "4294967295", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_rand_seed1", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_rand_seed1" }, "rand_seed2": { "id": 10723, @@ -11721,8 +11600,7 @@ "max_val": "4294967295", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_rand_seed2", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_rand_seed2" }, "range_alloc_block_size": { "id": 10724, @@ -11737,8 +11615,7 @@ "max_val": "18446744073709550592", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_range_alloc_block_size", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_range_alloc_block_size" }, "range_optimizer_max_mem_size": { "id": 10725, @@ -11753,8 +11630,7 @@ "max_val": "18446744073709551615", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_range_optimizer_max_mem_size", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_range_optimizer_max_mem_size" }, "rewriter_enabled": { "id": 10726, @@ -11767,8 +11643,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/rewriter-query-rewrite-plugin-reference.html#sysvar_rewriter_enabled", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/rewriter-query-rewrite-plugin-reference.html#sysvar_rewriter_enabled" }, "rewriter_verbose": { "id": 10727, @@ -11781,8 +11656,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/rewriter-query-rewrite-plugin-reference.html#sysvar_rewriter_verbose", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/rewriter-query-rewrite-plugin-reference.html#sysvar_rewriter_verbose" }, "secure_auth": { "id": 10728, @@ -11795,8 +11669,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_secure_auth", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_secure_auth" }, "sha256_password_proxy_users": { "id": 10729, @@ -11809,8 +11682,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_sha256_password_proxy_users", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_sha256_password_proxy_users" }, "show_compatibility_56": { "id": 10730, @@ -11823,8 +11695,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_sha256_password_proxy_users", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_sha256_password_proxy_users" }, "show_create_table_verbosity": { "id": 10731, @@ -11837,8 +11708,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_show_create_table_verbosity", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_show_create_table_verbosity" }, "show_old_temporals": { "id": 10732, @@ -11851,8 +11721,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_show_old_temporals", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_show_old_temporals" }, "sql_big_selects": { "id": 10733, @@ -11865,8 +11734,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_sql_big_selects", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_sql_big_selects" }, "updatable_views_with_limit": { "id": 10734, @@ -11885,8 +11753,7 @@ ], "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_updatable_views_with_limit", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_updatable_views_with_limit" }, "validate_password_dictionary_file": { "id": 10735, @@ -11899,8 +11766,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/validate-password-options-variables.html#sysvar_validate_password_dictionary_file", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/validate-password-options-variables.html#sysvar_validate_password_dictionary_file" }, "delayed_insert_limit": { "id": 10736, @@ -11915,8 +11781,7 @@ "max_val": "18446744073709551615", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_delayed_insert_limit", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_delayed_insert_limit" }, "ndb_version": { "id": 10737, @@ -11929,8 +11794,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html#sysvar_ndb_version", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/mysql-cluster-options-variables.html#sysvar_ndb_version" }, "auto_generate_certs": { "id": 10738, @@ -11943,8 +11807,7 @@ "publish_version": "", "info_cn": "", "background_cn": "", - "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_auto_generate_certs", - "placeholder": true + "ref_url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_auto_generate_certs" }, "_optimizer_cost_based_transformation": { "id": 10739, diff --git a/src/sql/CMakeLists.txt b/src/sql/CMakeLists.txt index 63a37b295..dd0c596b2 100644 --- a/src/sql/CMakeLists.txt +++ b/src/sql/CMakeLists.txt @@ -302,7 +302,7 @@ ob_set_subtarget(ob_sql engine_expr engine/expr/ob_expr_abs.cpp engine/expr/ob_expr_acos.cpp engine/expr/ob_expr_add.cpp - engine/expr/ob_expr_aes_encrypt.cpp + engine/expr/ob_expr_symmetric_encrypt.cpp engine/expr/ob_expr_agg_param_list.cpp engine/expr/ob_expr_and.cpp engine/expr/ob_expr_any_value.cpp @@ -313,6 +313,7 @@ ob_set_subtarget(ob_sql engine_expr engine/expr/ob_expr_at_time_zone.cpp engine/expr/ob_expr_atan.cpp engine/expr/ob_expr_atan2.cpp + engine/expr/ob_expr_audit_log_func.cpp engine/expr/ob_expr_autoinc_nextval.cpp engine/expr/ob_expr_benchmark.cpp engine/expr/ob_expr_between.cpp @@ -706,6 +707,7 @@ ob_set_subtarget(ob_sql engine_expr engine/expr/ob_expr_st_contains.cpp engine/expr/ob_expr_st_within.cpp engine/expr/ob_expr_sql_mode_convert.cpp + engine/expr/ob_expr_can_access_trigger.cpp engine/expr/ob_expr_prefix_pattern.cpp engine/expr/ob_expr_sys_makexml.cpp engine/expr/ob_expr_xml_func_helper.cpp diff --git a/src/sql/code_generator/ob_static_engine_cg.cpp b/src/sql/code_generator/ob_static_engine_cg.cpp index 3fb300412..fd129c36b 100644 --- a/src/sql/code_generator/ob_static_engine_cg.cpp +++ b/src/sql/code_generator/ob_static_engine_cg.cpp @@ -3678,13 +3678,20 @@ int ObStaticEngineCG::generate_spec(ObLogJoinFilter &op, ObJoinFilterSpec &spec, || (min_ver > MOCK_CLUSTER_VERSION_4_2_1_4 && min_ver < CLUSTER_VERSION_4_2_2_0)) { spec.bloom_filter_ratio_ = GCONF._bloom_filter_ratio; spec.send_bloom_filter_size_ = GCONF._send_bloom_filter_size; + if (OB_ISNULL(opt_ctx_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("opt_ctx_ is null", K(ret)); + } else if (OB_FAIL(opt_ctx_->get_global_hint().opt_params_.get_integer_opt_param(ObOptParamHint::BLOOM_FILTER_RATIO, spec.bloom_filter_ratio_))) { + LOG_WARN("failed to get opt param bloom filter ratio", K(ret)); + } } else { // for compatibility, if the cluseter is upgrading, set them as default value 0 spec.bloom_filter_ratio_ = 0; spec.send_bloom_filter_size_ = 0; } - if (OB_FAIL(spec.join_keys_.init(op.get_join_exprs().count()))) { + if (OB_FAIL(ret)) { + } else if (OB_FAIL(spec.join_keys_.init(op.get_join_exprs().count()))) { LOG_WARN("failed to init join keys", K(ret)); } else if (OB_NOT_NULL(op.get_tablet_id_expr()) && OB_FAIL(generate_calc_part_id_expr(*op.get_tablet_id_expr(), nullptr, spec.calc_tablet_id_expr_))) { diff --git a/src/sql/engine/cmd/ob_dcl_executor.cpp b/src/sql/engine/cmd/ob_dcl_executor.cpp index 64a772e7c..2d1a7d4fd 100644 --- a/src/sql/engine/cmd/ob_dcl_executor.cpp +++ b/src/sql/engine/cmd/ob_dcl_executor.cpp @@ -41,6 +41,8 @@ int ObGrantExecutor::execute(ObExecContext &ctx, ObGrantStmt &stmt) ObIAllocator &allocator = ctx.get_allocator(); obrpc::ObGrantArg &arg = static_cast(stmt.get_ddl_arg()); const bool is_role = arg.roles_.count() > 0; + ObSchemaGetterGuard schema_guard; + const ObUserInfo *user_info = NULL; if (OB_ISNULL(task_exec_ctx = GET_TASK_EXECUTOR_CTX(ctx))) { ret = OB_NOT_INIT; @@ -147,6 +149,28 @@ int ObGrantExecutor::execute(ObExecContext &ctx, ObGrantStmt &stmt) } } if (OB_FAIL(ret)) { + } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(tenant_id, schema_guard))) { + LOG_WARN("failed to get tenant schema guard", K(ret)); + } else if (OB_FAIL(schema_guard.get_user_info(tenant_id, + session_info->get_priv_user_id(), + user_info))) { + LOG_WARN("failed to get user info", K(ret)); + } else if (OB_ISNULL(user_info)) { + // ignore ret + LOG_WARN("user info is unexpected null", K(ret)); + } else if (OB_FAIL(ob_write_string(allocator, user_info->get_user_name_str(), arg.grantor_))) { + LOG_WARN("failed to write string", K(ret)); + } else if (OB_FAIL(ob_write_string(allocator, user_info->get_host_name_str(), arg.grantor_host_))) { + LOG_WARN("failed to write string", K(ret)); + } + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(schema_guard.reset())) { + LOG_WARN("failed to reset schema guard", K(tmp_ret)); + if (OB_SUCC(ret)) { + ret = tmp_ret; + } + } + if (OB_FAIL(ret)) { } else if (OB_FAIL(common_rpc_proxy->grant(arg))) { LOG_WARN("Grant privileges to user error", K(ret), K(arg)); } @@ -183,13 +207,13 @@ int ObRevokeExecutor::execute(ObExecContext &ctx, ObRevokeStmt &stmt) break; } case OB_PRIV_TABLE_LEVEL: { - if (OB_FAIL(revoke_table(common_rpc_proxy, stmt))) { + if (OB_FAIL(revoke_table(common_rpc_proxy, stmt, ctx))) { LOG_WARN("grant_revoke_user error", K(ret)); } break; } case OB_PRIV_ROUTINE_LEVEL: { - if (OB_FAIL(revoke_routine(common_rpc_proxy, stmt))) { + if (OB_FAIL(revoke_routine(common_rpc_proxy, stmt, ctx))) { LOG_WARN("grant_revoke_user error", K(ret)); } break; @@ -273,12 +297,20 @@ int ObRevokeExecutor::revoke_db(obrpc::ObCommonRpcProxy *rpc_proxy, ObRevokeStmt return ret; } -int ObRevokeExecutor::revoke_table(obrpc::ObCommonRpcProxy *rpc_proxy, ObRevokeStmt &stmt) +int ObRevokeExecutor::revoke_table(obrpc::ObCommonRpcProxy *rpc_proxy, + ObRevokeStmt &stmt, + ObExecContext &ctx) { int ret = OB_SUCCESS; + ObSQLSessionInfo *session_info = NULL; + ObSchemaGetterGuard schema_guard; + const ObUserInfo *user_info = NULL; if (OB_ISNULL(rpc_proxy) || OB_ISNULL(GCTX.schema_service_)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("Input argument error", K(rpc_proxy), K(ret)); + } else if (OB_ISNULL(session_info = ctx.get_my_session())) { + ret = OB_NOT_INIT; + LOG_WARN("Get my session error"); } else { obrpc::ObRevokeTableArg &arg = static_cast(stmt.get_ddl_arg()); arg.tenant_id_ = stmt.get_tenant_id(); @@ -306,6 +338,28 @@ int ObRevokeExecutor::revoke_table(obrpc::ObCommonRpcProxy *rpc_proxy, ObRevokeS } else { //todo: pl routine and others } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(arg.tenant_id_, schema_guard))) { + LOG_WARN("failed to get tenant schema guard", K(ret)); + } else if (OB_FAIL(schema_guard.get_user_info(arg.tenant_id_, + session_info->get_priv_user_id(), + user_info))) { + LOG_WARN("failed to get user info", K(ret)); + } else if (OB_ISNULL(user_info)) { + // ignore ret + LOG_WARN("user info is unexpected null", K(ret)); + } else if (OB_FAIL(ob_write_string(ctx.get_allocator(), user_info->get_user_name_str(), arg.grantor_))) { + LOG_WARN("failed to write string", K(ret)); + } else if (OB_FAIL(ob_write_string(ctx.get_allocator(), user_info->get_host_name_str(), arg.grantor_host_))) { + LOG_WARN("failed to write string", K(ret)); + } + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(schema_guard.reset())) { + LOG_WARN("failed to reset schema guard", K(tmp_ret)); + if (OB_SUCC(ret)) { + ret = tmp_ret; + } + } const ObIArray &user_ids = stmt.get_users(); if (OB_FAIL(ret)) { } else if (0 == user_ids.count()) { @@ -323,12 +377,20 @@ int ObRevokeExecutor::revoke_table(obrpc::ObCommonRpcProxy *rpc_proxy, ObRevokeS return ret; } -int ObRevokeExecutor::revoke_routine(obrpc::ObCommonRpcProxy *rpc_proxy, ObRevokeStmt &stmt) +int ObRevokeExecutor::revoke_routine(obrpc::ObCommonRpcProxy *rpc_proxy, + ObRevokeStmt &stmt, + ObExecContext &ctx) { int ret = OB_SUCCESS; + ObSQLSessionInfo *session_info = NULL; + ObSchemaGetterGuard schema_guard; + const ObUserInfo *user_info = NULL; if (OB_ISNULL(rpc_proxy)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("Input argument error", K(rpc_proxy), K(ret)); + } else if (OB_ISNULL(session_info = ctx.get_my_session())) { + ret = OB_NOT_INIT; + LOG_WARN("Get my session error"); } else { obrpc::ObRevokeRoutineArg &arg = static_cast(stmt.get_ddl_arg()); arg.tenant_id_ = stmt.get_tenant_id(); @@ -339,9 +401,31 @@ int ObRevokeExecutor::revoke_routine(obrpc::ObCommonRpcProxy *rpc_proxy, ObRevok arg.obj_type_ = static_cast(stmt.get_object_type()); arg.grantor_id_ = stmt.get_grantor_id(); arg.revoke_all_ora_ = stmt.get_revoke_all_ora(); - + if (OB_FAIL(ret)) { + } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(arg.tenant_id_, schema_guard))) { + LOG_WARN("failed to get tenant schema guard", K(ret)); + } else if (OB_FAIL(schema_guard.get_user_info(arg.tenant_id_, + session_info->get_priv_user_id(), + user_info))) { + LOG_WARN("failed to get user info", K(ret)); + } else if (OB_ISNULL(user_info)) { + // ignore ret + LOG_WARN("user info is unexpected null", K(ret)); + } else if (OB_FAIL(ob_write_string(ctx.get_allocator(), user_info->get_user_name_str(), arg.grantor_))) { + LOG_WARN("failed to write string", K(ret)); + } else if (OB_FAIL(ob_write_string(ctx.get_allocator(), user_info->get_host_name_str(), arg.grantor_host_))) { + LOG_WARN("failed to write string", K(ret)); + } + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(schema_guard.reset())) { + LOG_WARN("failed to reset schema guard", K(tmp_ret)); + if (OB_SUCC(ret)) { + ret = tmp_ret; + } + } const ObIArray &user_ids = stmt.get_users(); - if (0 == user_ids.count()) { + if (OB_FAIL(ret)) { + } else if (0 == user_ids.count()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("User ids is empty, resolver may be error", K(ret)); } else { diff --git a/src/sql/engine/cmd/ob_dcl_executor.h b/src/sql/engine/cmd/ob_dcl_executor.h index 66162547b..fa638573c 100644 --- a/src/sql/engine/cmd/ob_dcl_executor.h +++ b/src/sql/engine/cmd/ob_dcl_executor.h @@ -53,9 +53,12 @@ private: int revoke_db(obrpc::ObCommonRpcProxy *rpc_proxy, ObRevokeStmt &stmt); int revoke_table(obrpc::ObCommonRpcProxy *rpc_proxy, - ObRevokeStmt &stmt); + ObRevokeStmt &stmt, + ObExecContext &ctx); - int revoke_routine(obrpc::ObCommonRpcProxy *rpc_proxy, ObRevokeStmt &stmt); + int revoke_routine(obrpc::ObCommonRpcProxy *rpc_proxy, + ObRevokeStmt &stmt, + ObExecContext &ctx); int revoke_sys_priv(obrpc::ObCommonRpcProxy *rpc_proxy, ObRevokeStmt &stmt); private: diff --git a/src/sql/engine/cmd/ob_mock_executor.cpp b/src/sql/engine/cmd/ob_mock_executor.cpp index bf16fe593..626309ad8 100644 --- a/src/sql/engine/cmd/ob_mock_executor.cpp +++ b/src/sql/engine/cmd/ob_mock_executor.cpp @@ -21,14 +21,40 @@ namespace sql int ObMockExecutor::execute(ObExecContext &exec_ctx, ObMockStmt &stmt) { int ret = OB_SUCCESS; - if (stmt::T_FLUSH_PRIVILEGES == stmt.get_stmt_type()) { - LOG_USER_WARN(OB_NOT_SUPPORTED, "After executing the GRANT and REVOKE statements, " - "the permissions will be automatically applied and take effect. " - "There is no need to execute the FLUSH PRIVILEGES command. FLUSH PRIVILEGES"); + + if (stmt::T_INSTALL_PLUGIN == stmt.get_stmt_type() + || stmt::T_UNINSTALL_PLUGIN == stmt.get_stmt_type() + || stmt::T_FLUSH_MOCK == stmt.get_stmt_type() + || stmt::T_HANDLER_MOCK == stmt.get_stmt_type() + || stmt::T_SHOW_PLUGINS == stmt.get_stmt_type() + || stmt::T_CREATE_SERVER == stmt.get_stmt_type() + || stmt::T_ALTER_SERVER == stmt.get_stmt_type() + || stmt::T_DROP_SERVER == stmt.get_stmt_type() + || stmt::T_CREATE_LOGFILE_GROUP == stmt.get_stmt_type() + || stmt::T_ALTER_LOGFILE_GROUP == stmt.get_stmt_type() + || stmt::T_DROP_LOGFILE_GROUP == stmt.get_stmt_type()) { + LOG_USER_WARN(OB_NOT_SUPPORTED, "This statement is"); + } else if (stmt::T_FLUSH_MOCK_LIST == stmt.get_stmt_type()) { + const ObIArray &type_list = stmt.get_stmt_type_list(); + for (int64_t i = 0; OB_SUCC(ret) && i < type_list.count(); ++i) { + if (stmt::T_FLUSH_MOCK == type_list.at(i)) { + LOG_USER_WARN(OB_NOT_SUPPORTED, "This statement is"); + } else if (stmt::T_FLUSH_PRIVILEGES == type_list.at(i)) { + LOG_USER_WARN(OB_NOT_SUPPORTED, "After executing the GRANT and REVOKE statements, " + "the permissions will be automatically applied and take effect. " + "There is no need to execute the FLUSH PRIVILEGES command. FLUSH PRIVILEGES"); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_USER_WARN(OB_NOT_SUPPORTED, "unknown stmt type"); + } + } } else if (stmt::T_REPAIR_TABLE == stmt.get_stmt_type()) { LOG_USER_WARN(OB_NOT_SUPPORTED, "Repair table Statement just mocks the syntax of MySQL without supporting specific realization"); } else if (stmt::T_CHECKSUM_TABLE == stmt.get_stmt_type()) { LOG_USER_WARN(OB_NOT_SUPPORTED, "Checksum table Statement just mocks the syntax of MySQL without supporting specific realization"); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_USER_WARN(OB_NOT_SUPPORTED, "unknown stmt type"); } return ret; } diff --git a/src/sql/engine/cmd/ob_variable_set_executor.cpp b/src/sql/engine/cmd/ob_variable_set_executor.cpp index ba1c20e22..5fef66754 100644 --- a/src/sql/engine/cmd/ob_variable_set_executor.cpp +++ b/src/sql/engine/cmd/ob_variable_set_executor.cpp @@ -1378,7 +1378,9 @@ int ObVariableSetExecutor::is_support(const share::ObSetVar &set_var) if(SYS_VAR_INVALID == (var_id = ObSysVarFactory::find_sys_var_id_by_name(set_var.var_name_))) { ret = OB_ERR_SYS_VARIABLE_UNKNOWN; LOG_WARN("unknown variable", K(set_var.var_name_), K(ret)); - } else if (SYS_VAR_DEBUG <= var_id && SYS_VAR_STORED_PROGRAM_CACHE >= var_id) { + } else if ((SYS_VAR_DEBUG <= var_id && SYS_VAR_STORED_PROGRAM_CACHE >= var_id) || + (SYS_VAR_INSERT_ID <= var_id && SYS_VAR_MAX_WRITE_LOCK_COUNT >= var_id) || + (SYS_VAR_BIG_TABLES <= var_id && SYS_VAR_DELAYED_INSERT_LIMIT >= var_id)) { ret = OB_NOT_SUPPORTED; LOG_WARN("This variable not support, just mock", K(set_var.var_name_), K(var_id), K(ret)); } diff --git a/src/sql/engine/expr/ob_expr_aes_encrypt.cpp b/src/sql/engine/expr/ob_expr_aes_encrypt.cpp deleted file mode 100644 index a8e110a35..000000000 --- a/src/sql/engine/expr/ob_expr_aes_encrypt.cpp +++ /dev/null @@ -1,271 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#define USING_LOG_PREFIX SQL_EXE -#include "ob_expr_aes_encrypt.h" -#include "share/object/ob_obj_cast.h" -#include "share/ob_encryption_util.h" -#include "ob_expr_extract.h" -#include "sql/session/ob_sql_session_info.h" -#include "sql/engine/ob_exec_context.h" - -using namespace oceanbase::share; -using namespace oceanbase::common; - -namespace oceanbase -{ -namespace sql -{ - -int get_encryption_value(int64_t &encryption, ObSQLSessionInfo *session) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(session)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("session is NULL", K(ret)); - } else if (OB_FAIL(session->get_sys_variable(SYS_VAR_BLOCK_ENCRYPTION_MODE, encryption))) { - LOG_WARN("fail to get block encryption variable", K(ret)); - } else { - ++encryption; //0为invalid_mdoe,所以需要+1 - } - return ret; -} - -ObExprAesEncrypt::ObExprAesEncrypt(ObIAllocator& alloc) - : ObFuncExprOperator(alloc, - T_FUN_AES_ENCRYPT, - N_AES_ENCRYPT, - TWO_OR_THREE, - NOT_VALID_FOR_GENERATED_COL, - NOT_ROW_DIMENSION) -{ -} -ObExprAesEncrypt::~ObExprAesEncrypt() {} - -int ObExprAesEncrypt::calc_result_typeN(ObExprResType& type, - ObExprResType* types_stack, - int64_t param_num, - ObExprTypeCtx& type_ctx) const -{ - UNUSED(type_ctx); - int ret = OB_SUCCESS; - if (OB_ISNULL(types_stack)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("null types",K(ret)); - } else if (OB_UNLIKELY(param_num > 3 || param_num < 2)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("param num is not correct", K(param_num)); - } else { - for (int i = 0; i < param_num; ++i) { - types_stack[i].set_calc_type(common::ObVarcharType); - types_stack[i].set_calc_collation_type(types_stack[i].get_collation_type()); - types_stack[i].set_calc_collation_level(types_stack[i].get_collation_level()); - } - type.set_varbinary(); - type.set_length((types_stack[0].get_length() * 3 / ObBlockCipher::OB_CIPHER_BLOCK_LENGTH + 1) * - ObBlockCipher::OB_CIPHER_BLOCK_LENGTH); - type.set_collation_level(CS_LEVEL_COERCIBLE); - } - return ret; -} - -int ObExprAesEncrypt::eval_aes_encrypt(const ObExpr &expr, ObEvalCtx &ctx, - ObDatum &res) -{ - int ret = OB_SUCCESS; - ObDatum *src = NULL; - ObDatum *key = NULL; - if (OB_FAIL(expr.eval_param_value(ctx, src, key))) { - LOG_WARN("eval arg failed", K(ret)); - } else if (src->is_null() || key->is_null()) { - res.set_null(); - } else { - int64_t encryption = 0; - OZ(get_encryption_value(encryption, ctx.exec_ctx_.get_my_session())); - CK(2 == expr.arg_cnt_ || 3 == expr.arg_cnt_); - if (OB_SUCC(ret)) { - const ObString &src_str = expr.locate_param_datum(ctx, 0).get_string(); - const ObString &key_str = expr.locate_param_datum(ctx, 1).get_string(); - int64_t out_len = 0; - ObEvalCtx::TempAllocGuard alloc_guard(ctx); - ObIAllocator &calc_alloc = alloc_guard.get_allocator(); - ObCipherOpMode opmode = static_cast(encryption); - int buf_length = (src_str.length() / ObBlockCipher::OB_CIPHER_BLOCK_LENGTH + 1) * ObBlockCipher::OB_CIPHER_BLOCK_LENGTH; - char *buf = static_cast(calc_alloc.alloc(buf_length)); - bool is_ecb = opmode <= ObCipherOpMode::ob_aes_256_ecb && - opmode >= ObCipherOpMode::ob_aes_128_ecb; - if (OB_ISNULL(buf)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("alloc memory failed", K(ret)); - } else if (!is_ecb && 2 == expr.arg_cnt_) { - ret = OB_ERR_PARAM_SIZE; - LOG_WARN("param num error", K(ret), K(expr.arg_cnt_), K(encryption)); - } else if (3 == expr.arg_cnt_ && is_ecb) { - // just user warn, not set ret error. - LOG_USER_WARN(OB_ERR_INVALID_INPUT_STRING, "iv"); - } - if (OB_SUCC(ret)) { - if (2 == expr.arg_cnt_ || is_ecb) { - OZ(ObBlockCipher::encrypt(key_str.ptr(), key_str.length(), src_str.ptr(), - src_str.length(), buf_length, NULL, 0, NULL, 0, 0, opmode, buf, - out_len, NULL)); - } else { - ObString iv_str = expr.locate_param_datum(ctx, 2).get_string(); - OV(iv_str.length() >= ObBlockCipher::OB_DEFAULT_IV_LENGTH, OB_ERR_AES_IV_LENGTH); - OX(iv_str.assign(iv_str.ptr(), ObBlockCipher::OB_DEFAULT_IV_LENGTH)); - OZ(ObBlockCipher::encrypt(key_str.ptr(), key_str.length(), src_str.ptr(), src_str.length(), - buf_length, iv_str.ptr(), iv_str.length(), NULL, 0, 0, opmode, - buf, out_len, NULL)); - } - } - if (OB_SUCC(ret)) { - ObExprStrResAlloc res_alloc(expr, ctx); - char *res_buf = static_cast(res_alloc.alloc(out_len)); - OV(OB_NOT_NULL(res_buf), OB_ALLOCATE_MEMORY_FAILED); - OX(MEMCPY(res_buf, buf, out_len)); - OX(res.set_string(res_buf, out_len)); - } - } - } - - return ret; -} - -int ObExprAesEncrypt::cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, - ObExpr &rt_expr) const -{ - int ret = OB_SUCCESS; - UNUSED(expr_cg_ctx); - UNUSED(raw_expr); - rt_expr.eval_func_ = eval_aes_encrypt; - return ret; -} - -//---------------------------------------分割线 -ObExprAesDecrypt::ObExprAesDecrypt(ObIAllocator& alloc) - : ObFuncExprOperator(alloc, - T_FUN_AES_DECRYPT, - N_AES_DECRYPT, - TWO_OR_THREE, - NOT_VALID_FOR_GENERATED_COL, - NOT_ROW_DIMENSION) -{ -} -ObExprAesDecrypt::~ObExprAesDecrypt() {} - -int ObExprAesDecrypt::calc_result_typeN(ObExprResType& type, - ObExprResType* types_stack, - int64_t param_num, - ObExprTypeCtx& type_ctx) const -{ - UNUSED(type_ctx); - int ret = OB_SUCCESS; - if (OB_ISNULL(types_stack)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("null types",K(ret)); - } else if (OB_UNLIKELY(param_num > 3 || param_num < 2)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("param num is not correct", K(param_num)); - } else { - for (int i = 0; i < param_num; ++i) { - types_stack[i].set_calc_type(common::ObVarcharType); - } - type.set_varbinary(); - type.set_length(types_stack[0].get_length() * 3); - type.set_collation_level(CS_LEVEL_COERCIBLE); - } - return ret; -} - -int ObExprAesDecrypt::eval_aes_decrypt(const ObExpr &expr, ObEvalCtx &ctx, - ObDatum &res) -{ - int ret = OB_SUCCESS; - ObDatum *src = NULL; - ObDatum *key = NULL; - if (OB_FAIL(expr.eval_param_value(ctx, src, key))) { - LOG_WARN("eval arg failed", K(ret)); - } else if (src->is_null() || key->is_null()) { - res.set_null(); - } else { - int64_t encryption = 0; - OZ(get_encryption_value(encryption, ctx.exec_ctx_.get_my_session())); - CK(2 == expr.arg_cnt_ || 3 == expr.arg_cnt_); - bool is_null = false; - if (OB_SUCC(ret)) { - const ObString &src_str = expr.locate_param_datum(ctx, 0).get_string(); - const ObString &key_str = expr.locate_param_datum(ctx, 1).get_string(); - int64_t out_len = 0; - ObEvalCtx::TempAllocGuard alloc_guard(ctx); - ObIAllocator &calc_alloc = alloc_guard.get_allocator(); - ObCipherOpMode opmode = static_cast(encryption); - const int64_t buf_len = src_str.length() + 1; - char *buf = static_cast(calc_alloc.alloc(buf_len)); - bool is_ecb = opmode <= ObCipherOpMode::ob_aes_256_ecb && - opmode >= ObCipherOpMode::ob_aes_128_ecb; - if (OB_ISNULL(buf)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("alloc mem failed", K(ret)); - } else if (!is_ecb && 2 == expr.arg_cnt_) { - ret = OB_ERR_PARAM_SIZE; - LOG_WARN("param num error", K(ret), K(expr.arg_cnt_), K(encryption)); - } else if (3 == expr.arg_cnt_ && is_ecb) { - // just user warn, not set ret error. - LOG_USER_WARN(OB_ERR_INVALID_INPUT_STRING, "iv"); - } - if (OB_SUCC(ret)) { - if (2 == expr.arg_cnt_ || is_ecb) { - OZ(ObBlockCipher::decrypt(key_str.ptr(), key_str.length(), src_str.ptr(), - src_str.length(), src_str.length(), NULL, 0, NULL, 0, NULL, 0, - opmode, buf, out_len)); - } else { - ObString iv_str = expr.locate_param_datum(ctx, 2).get_string(); - OV(iv_str.length() >= ObBlockCipher::OB_DEFAULT_IV_LENGTH, OB_ERR_AES_IV_LENGTH); - OX(iv_str.assign(iv_str.ptr(), ObBlockCipher::OB_DEFAULT_IV_LENGTH)); - OZ(ObBlockCipher::decrypt(key_str.ptr(), key_str.length(), src_str.ptr(), src_str.length(), - src_str.length(), iv_str.ptr(), iv_str.length(), NULL, 0, NULL, 0, - opmode, buf, out_len)); - } - if (OB_ERR_AES_DECRYPT == ret) { - //按照mysql兼容的做法,如果解密失败,则将结果设置为null - is_null = true; - ret = OB_SUCCESS; - } - } - if (OB_SUCC(ret)) { - if (is_null) { - res.set_null(); - } else { - ObExprStrResAlloc res_alloc(expr, ctx); - char *res_buf = static_cast(res_alloc.alloc(out_len)); - OV(OB_NOT_NULL(res_buf), OB_ALLOCATE_MEMORY_FAILED); - OX(MEMCPY(res_buf, buf, out_len)); - OX(res.set_string(res_buf, out_len)); - } - } - } - } - return ret; -} - -int ObExprAesDecrypt::cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, - ObExpr &rt_expr) const -{ - int ret = OB_SUCCESS; - UNUSED(expr_cg_ctx); - UNUSED(raw_expr); - rt_expr.eval_func_ = eval_aes_decrypt; - return ret; -} - -} -} diff --git a/src/sql/engine/expr/ob_expr_aes_encrypt.h b/src/sql/engine/expr/ob_expr_aes_encrypt.h deleted file mode 100644 index 485420108..000000000 --- a/src/sql/engine/expr/ob_expr_aes_encrypt.h +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#ifndef SRC_SQL_ENGINE_EXPR_OB_EXPR_AES_ENCRYPT_H_ -#define SRC_SQL_ENGINE_EXPR_OB_EXPR_AES_ENCRYPT_H_ -#include "sql/engine/expr/ob_expr_operator.h" -namespace oceanbase -{ -namespace sql -{ -class ObExprAesEncrypt : public ObFuncExprOperator -{ -public: - ObExprAesEncrypt(); - explicit ObExprAesEncrypt(common::ObIAllocator& alloc); - virtual ~ObExprAesEncrypt(); - virtual int calc_result_typeN(ObExprResType& type, - ObExprResType* types, - int64_t param_num, - common::ObExprTypeCtx& type_ctx) - const override; - static int eval_aes_encrypt(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res); - virtual int cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, - ObExpr &rt_expr) const override; -private: - DISALLOW_COPY_AND_ASSIGN(ObExprAesEncrypt); -}; - -class ObExprAesDecrypt : public ObFuncExprOperator -{ -public: - ObExprAesDecrypt(); - explicit ObExprAesDecrypt(common::ObIAllocator& alloc); - virtual ~ObExprAesDecrypt(); - virtual int calc_result_typeN(ObExprResType& type, - ObExprResType* types, - int64_t param_num, - common::ObExprTypeCtx& type_ctx) - const override; - static int eval_aes_decrypt(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res); - virtual int cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, - ObExpr &rt_expr) const override; -private: - DISALLOW_COPY_AND_ASSIGN(ObExprAesDecrypt); -}; - -} -} - - - - - -#endif diff --git a/src/sql/engine/expr/ob_expr_arg_case.cpp b/src/sql/engine/expr/ob_expr_arg_case.cpp index 6d46fe669..a86165860 100644 --- a/src/sql/engine/expr/ob_expr_arg_case.cpp +++ b/src/sql/engine/expr/ob_expr_arg_case.cpp @@ -130,14 +130,12 @@ int ObExprArgCase::calc_result_typeN(ObExprResType &type, int64_t cond_type_count = param_num / 2; int64_t val_type_count = param_num / 2; ObExprResType tmp_res_type; - const ObLengthSemantics default_length_semantics = (OB_NOT_NULL(type_ctx.get_session()) ? type_ctx.get_session()->get_actual_nls_length_semantics() : LS_BYTE); if (OB_FAIL(aggregate_result_type_for_case( tmp_res_type, types_stack, cond_type_count, - type_ctx.get_coll_type(), lib::is_oracle_mode(), - default_length_semantics, + type_ctx, FALSE, FALSE, is_called_in_sql_))) { LOG_WARN("failed to get result type for cmp", K(ret)); @@ -145,9 +143,8 @@ int ObExprArgCase::calc_result_typeN(ObExprResType &type, type, types_stack + cond_type_count, val_type_count, - type_ctx.get_coll_type(), lib::is_oracle_mode(), - default_length_semantics, + type_ctx, true, false, is_called_in_sql_))) { LOG_WARN("failed to get result type", K(ret)); diff --git a/src/sql/engine/expr/ob_expr_audit_log_func.cpp b/src/sql/engine/expr/ob_expr_audit_log_func.cpp new file mode 100644 index 000000000..0ebb9d902 --- /dev/null +++ b/src/sql/engine/expr/ob_expr_audit_log_func.cpp @@ -0,0 +1,534 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_ENG +#include "sql/engine/expr/ob_expr_audit_log_func.h" +#include "lib/charset/ob_charset.h" +#ifdef OB_BUILD_AUDIT_SECURITY +#include "sql/audit/ob_audit_log_info.h" +#include "sql/audit/ob_audit_log_table_operator.h" +#include "sql/audit/ob_audit_log_utils.h" +#endif +#include "lib/ob_name_def.h" +#include "sql/engine/ob_physical_plan_ctx.h" +#include "sql/engine/ob_exec_context.h" + +namespace oceanbase +{ +using namespace common; +using namespace share; +namespace sql +{ + +ObExprAuditLogFunc::ObExprAuditLogFunc(ObIAllocator &alloc, + ObExprOperatorType type, + const char *name, + int32_t param_num) + : ObStringExprOperator(alloc, type, name, param_num, NOT_VALID_FOR_GENERATED_COL) +{ +} + +int ObExprAuditLogFunc::calc_result_type1(ObExprResType &type, + ObExprResType &type1, + ObExprTypeCtx &type_ctx) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(check_data_version(type_ctx))) { + LOG_WARN("failed to check data version"); + } else if (OB_FAIL(check_param_type(type1))) { + LOG_WARN("failed to check param type"); + } else { + type.set_varchar(); + type.set_collation_type(type_ctx.get_coll_type()); + type.set_collation_level(common::CS_LEVEL_COERCIBLE); + type.set_length(OB_MAX_MYSQL_VARCHAR_LENGTH); + + type1.set_calc_type(ObVarcharType); + type1.set_calc_collation_type(ObCharset::get_system_collation()); + } + return ret; +} + +int ObExprAuditLogFunc::calc_result_type2(ObExprResType &type, + ObExprResType &type1, + ObExprResType &type2, + ObExprTypeCtx &type_ctx) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(check_data_version(type_ctx))) { + LOG_WARN("failed to check data version"); + } else if (OB_FAIL(check_param_type(type1)) || OB_FAIL(check_param_type(type2))) { + LOG_WARN("failed to check param type"); + } else { + type.set_varchar(); + type.set_collation_type(type_ctx.get_coll_type()); + type.set_collation_level(common::CS_LEVEL_COERCIBLE); + type.set_length(OB_MAX_MYSQL_VARCHAR_LENGTH); + + type1.set_calc_type(ObVarcharType); + type1.set_calc_collation_type(ObCharset::get_system_collation()); + type2.set_calc_type(ObVarcharType); + type2.set_calc_collation_type(ObCharset::get_system_collation()); + } + return ret; +} + +int ObExprAuditLogFunc::check_data_version(ObExprTypeCtx &type_ctx) const +{ + int ret = OB_SUCCESS; + const ObSQLSessionInfo *session = type_ctx.get_session(); + uint64_t data_version = 0; + if (OB_ISNULL(session)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null session", K(ret)); + } else if (OB_FAIL(GET_MIN_DATA_VERSION(session->get_effective_tenant_id(), data_version))) { + LOG_WARN("failed to get min data version", K(ret)); + } else if (data_version < MOCK_DATA_VERSION_4_2_4_0 || + (data_version >= DATA_VERSION_4_3_0_0 && data_version < DATA_VERSION_4_3_3_0)) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, get_name()); + } + return ret; +} + +int ObExprAuditLogFunc::check_param_type(const ObExprResType &type) const +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!type.is_varchar_or_char())) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, get_name()); + LOG_WARN("invalid param type", K(ret), K(type)); + } + return ret; +} + +int ObExprAuditLogFunc::check_privilege(ObSQLSessionInfo &session, + bool &is_valid, + ObString &error_info) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!session.has_user_super_privilege())) { + is_valid = false; + error_info = "Request ignored. SUPER needed to perform operation"; + } + return ret; +} + +int ObExprAuditLogFunc::parse_user_name(const ObString &str, + ObString &user_name, + ObString &host, + bool &is_valid, + ObString &error_info) +{ + int ret = OB_SUCCESS; + is_valid = true; + if (NULL == str.find('@')) { + user_name = str; + user_name = user_name.trim(); + host.reset(); + } else { + host = str; + user_name = host.split_on('@'); + user_name = user_name.trim(); + host = host.trim(); + } + if (OB_SUCC(ret) && is_valid) { + int64_t name_len = ObCharset::strlen_char(ObCharset::get_system_collation(), + user_name.ptr(), user_name.length()); + if (OB_UNLIKELY(user_name.empty())) { + is_valid = false; + error_info = "User cannot be empty"; + } else if (OB_UNLIKELY(name_len > OB_MAX_USER_NAME_LENGTH)) { + is_valid = false; + error_info = "User name is too long"; + } else if (OB_UNLIKELY(0 == user_name.compare("%"))) { + // do nothing + } else if (OB_NOT_NULL(user_name.find('_')) || OB_NOT_NULL(user_name.find('%'))) { + is_valid = false; + error_info = "Invalid character in the user name"; + LOG_WARN("invalid user name", K(user_name)); + } + } + if (OB_SUCC(ret) && is_valid) { + if (host.empty()) { + host = "%"; + } else if (OB_UNLIKELY(0 != host.compare("%"))) { + is_valid = false; + error_info = "Invalid host name"; + LOG_WARN("invalid host name", K(host)); + } + } + return ret; +} + +int ObExprAuditLogFunc::parse_filter_name(const ObString &str, + ObString &filter_name, + bool &is_valid, + ObString &error_info) +{ + int ret = OB_SUCCESS; + int64_t name_len = 0; + is_valid = true; + filter_name = str; + filter_name.trim(); + name_len = ObCharset::strlen_char(ObCharset::get_system_collation(), + filter_name.ptr(), filter_name.length()); + if (OB_UNLIKELY(filter_name.empty())) { + is_valid = false; + error_info = "Filter name cannot be empty"; + } else if (OB_UNLIKELY(name_len > MAX_AUDIT_FILTER_NAME_LENGTH)) { + is_valid = false; + error_info = "Could not store field of the audit_log_filter table"; + } + return ret; +} + +int ObExprAuditLogFunc::parse_definition(const ObString &str, + ObString &definition, + bool &is_valid, + ObString &error_info) +{ + int ret = OB_SUCCESS; +#ifdef OB_BUILD_AUDIT_SECURITY + definition = str; + definition = definition.trim(); + uint64_t audit_class = 0; + if (OB_UNLIKELY(definition.empty())) { + is_valid = false; + error_info = "JSON parsing error"; + } else if (OB_FAIL(ObAuditLogUtils::parse_filter_definition(definition, is_valid, audit_class))) { + LOG_WARN("failed to parse filter definition", K(ret)); + } else if (!is_valid) { + error_info = "JSON parsing error"; + } +#endif + return ret; +} + +int ObExprAuditLogFunc::fill_res_datum(const ObExpr &expr, + ObEvalCtx &ctx, + const bool is_valid, + const common::ObString &error_info, + ObDatum &expr_datum) +{ + int ret = OB_SUCCESS; + if (is_valid) { + expr_datum.set_string("OK"); + } else { + int64_t buf_len = error_info.length() + strlen("ERROR: .") + 1; + int64_t res_len = 0; + char *buf = expr.get_str_res_mem(ctx, buf_len); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory", K(ret), K(buf_len)); + } else if (OB_FAIL(databuff_printf(buf, buf_len, res_len, "ERROR: %.*s.", + error_info.length(), error_info.ptr()))) { + LOG_WARN("failed to print", K(ret), K(error_info)); + } else { + expr_datum.set_string(buf, res_len); + } + } + return ret; +} + +ObExprAuditLogSetFilter::ObExprAuditLogSetFilter(ObIAllocator &alloc) + : ObExprAuditLogFunc(alloc, T_FUN_SYS_AUDIT_LOG_SET_FILTER, N_AUDIT_LOG_FILTER_SET_FILTER, 2) +{ +} + +int ObExprAuditLogSetFilter::cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, + ObExpr &expr) const +{ + int ret = OB_SUCCESS; + UNUSED(expr_cg_ctx); + UNUSED(raw_expr); + expr.eval_func_ = eval_set_filter; + return ret; +} + +int ObExprAuditLogSetFilter::eval_set_filter(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &expr_datum) +{ + int ret = OB_SUCCESS; +#ifdef OB_BUILD_AUDIT_SECURITY + ObDatum *arg0 = NULL; + ObDatum *arg1 = NULL; + ObSQLSessionInfo *session = ctx.exec_ctx_.get_my_session(); + if (OB_ISNULL(session)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null session", K(ret)); + } else if (OB_FAIL(expr.eval_param_value(ctx, arg0, arg1))) { + LOG_WARN("evaluate parameters values failed", K(ret)); + } else { + ObString filter_name; + ObString definition; + ObString error_info; + ObSEArray params; + bool is_valid = true; + if (OB_FAIL(check_privilege(*session, is_valid, error_info))) { + LOG_WARN("failed to check privilege", K(ret)); + } else if (is_valid && OB_FAIL(parse_filter_name(arg0->get_string(), filter_name, + is_valid, error_info))) { + } else if (is_valid && OB_FAIL(parse_definition(arg1->get_string(), definition, + is_valid, error_info))) { + LOG_WARN("failed to parse definition", KPC(arg1)); + } else if (is_valid) { + uint64_t tenant_id = session->get_effective_tenant_id(); + ObAuditLogFilterInfo audit_filter; + ObAuditLogTableOperator table_operator; + ObISQLClient *sql_client = ctx.exec_ctx_.get_sql_proxy(); + audit_filter.set_tenant_id(tenant_id); + audit_filter.set_filter_name(filter_name); + audit_filter.set_definition(definition); + if (OB_ISNULL(sql_client)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null sql client", K(ret)); + } else if (OB_FAIL(table_operator.init(tenant_id, sql_client))) { + LOG_WARN("failed to init audit log table operator", K(ret), K(tenant_id)); + } else if (OB_FAIL(table_operator.set_audit_filter(audit_filter))) { + LOG_WARN("failed to set audit log filter", K(ret), K(audit_filter)); + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(fill_res_datum(expr, ctx, is_valid, error_info, expr_datum))) { + LOG_WARN("failed to fill expr_datum", K(ret), K(is_valid), K(error_info)); + } + } +#endif + return ret; +} + +ObExprAuditLogRemoveFilter::ObExprAuditLogRemoveFilter(ObIAllocator &alloc) + : ObExprAuditLogFunc(alloc, T_FUN_SYS_AUDIT_LOG_REMOVE_FILTER, N_AUDIT_LOG_FILTER_REMOVE_FILTER, 1) +{ +} + +int ObExprAuditLogRemoveFilter::cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, + ObExpr &expr) const +{ + int ret = OB_SUCCESS; + UNUSED(expr_cg_ctx); + UNUSED(raw_expr); + expr.eval_func_ = eval_remove_filter; + return ret; +} + +int ObExprAuditLogRemoveFilter::eval_remove_filter(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &expr_datum) +{ + int ret = OB_SUCCESS; +#ifdef OB_BUILD_AUDIT_SECURITY + ObDatum *arg0 = NULL; + ObSQLSessionInfo *session = ctx.exec_ctx_.get_my_session(); + if (OB_ISNULL(session)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null session", K(ret)); + } else if (OB_FAIL(expr.eval_param_value(ctx, arg0))) { + LOG_WARN("evaluate parameters values failed", K(ret)); + } else { + ObString filter_name; + ObString error_info; + ObSEArray params; + bool is_valid = true; + if (OB_FAIL(check_privilege(*session, is_valid, error_info))) { + LOG_WARN("failed to check privilege", K(ret)); + } else if (is_valid && OB_FAIL(parse_filter_name(arg0->get_string(), filter_name, + is_valid, error_info))) { + LOG_WARN("failed to parse filter name", KPC(arg0)); + } else if (is_valid) { + uint64_t tenant_id = session->get_effective_tenant_id(); + ObAuditLogFilterInfo audit_filter; + ObAuditLogTableOperator table_operator; + ObMySQLTransaction trans; + ObEvalCtx::TempAllocGuard alloc_guard(ctx); + ObIAllocator &tmp_allocator = alloc_guard.get_allocator(); + bool is_exist = false; + audit_filter.set_tenant_id(tenant_id); + audit_filter.set_filter_name(filter_name); + if (OB_FAIL(trans.start(ctx.exec_ctx_.get_sql_proxy(), tenant_id))) { + LOG_WARN("fail to start transaction", K(ret)); + } else if (OB_FAIL(table_operator.init(tenant_id, &trans))) { + LOG_WARN("failed to init audit log table operator", K(ret), K(tenant_id)); + } else if (OB_FAIL(table_operator.get_audit_filter(audit_filter, tmp_allocator, + true /*for_update*/, is_exist))) { + LOG_WARN("failed to set audit log filter", K(ret), K(audit_filter)); + } else if (!is_exist) { + // do nothing + } else if (OB_FAIL(table_operator.mark_audit_user_delete_by_filter(audit_filter))) { + LOG_WARN("failed to delete audit log user by filter", K(ret), K(audit_filter)); + } else if (OB_FAIL(table_operator.mark_audit_filter_delete(audit_filter))) { + LOG_WARN("failed to delete audit log filter", K(ret), K(audit_filter)); + } + if (trans.is_started()) { + int temp_ret = OB_SUCCESS; + if (OB_SUCCESS != (temp_ret = trans.end(OB_SUCC(ret)))) { + LOG_WARN("trans end failed", "is_commit", OB_SUCCESS == ret, K(ret), K(temp_ret)); + ret = OB_SUCC(ret) ? temp_ret : ret; + } + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(fill_res_datum(expr, ctx, is_valid, error_info, expr_datum))) { + LOG_WARN("failed to fill expr_datum", K(ret), K(is_valid), K(error_info)); + } + } +#endif + return ret; +} + +ObExprAuditLogSetUser::ObExprAuditLogSetUser(ObIAllocator &alloc) + : ObExprAuditLogFunc(alloc, T_FUN_SYS_AUDIT_LOG_SET_USER, N_AUDIT_LOG_FILTER_SET_USER, 2) +{ +} + +int ObExprAuditLogSetUser::cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, + ObExpr &expr) const +{ + int ret = OB_SUCCESS; + UNUSED(expr_cg_ctx); + UNUSED(raw_expr); + expr.eval_func_ = eval_set_user; + return ret; +} + +int ObExprAuditLogSetUser::eval_set_user(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &expr_datum) +{ + int ret = OB_SUCCESS; +#ifdef OB_BUILD_AUDIT_SECURITY + ObDatum *arg0 = NULL; + ObDatum *arg1 = NULL; + ObSQLSessionInfo *session = ctx.exec_ctx_.get_my_session(); + if (OB_ISNULL(session)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null session", K(ret)); + } else if (OB_FAIL(expr.eval_param_value(ctx, arg0, arg1))) { + LOG_WARN("evaluate parameters values failed", K(ret)); + } else { + ObString user_name; + ObString host; + ObString filter_name; + ObString error_info; + ObSEArray params; + bool is_valid = true; + if (OB_FAIL(check_privilege(*session, is_valid, error_info))) { + LOG_WARN("failed to check privilege", K(ret)); + } else if (is_valid && OB_FAIL(parse_user_name(arg0->get_string(), user_name, host, + is_valid, error_info))) { + LOG_WARN("failed to parse user name", KPC(arg0)); + } else if (is_valid && OB_FAIL(parse_filter_name(arg1->get_string(), filter_name, + is_valid, error_info))) { + LOG_WARN("failed to parse filter name", KPC(arg1)); + } else if (is_valid) { + uint64_t tenant_id = session->get_effective_tenant_id(); + ObAuditLogFilterInfo audit_filter; + ObAuditLogUserInfo audit_user; + ObAuditLogTableOperator table_operator; + ObMySQLTransaction trans; + ObEvalCtx::TempAllocGuard alloc_guard(ctx); + ObIAllocator &tmp_allocator = alloc_guard.get_allocator(); + bool is_exist = false; + audit_filter.set_tenant_id(tenant_id); + audit_user.set_tenant_id(tenant_id); + audit_filter.set_filter_name(filter_name); + audit_user.set_user_name(user_name); + audit_user.set_host(host); + audit_user.set_filter_name(filter_name); + if (OB_FAIL(trans.start(ctx.exec_ctx_.get_sql_proxy(), tenant_id))) { + LOG_WARN("fail to start transaction", K(ret)); + } else if (OB_FAIL(table_operator.init(tenant_id, &trans))) { + LOG_WARN("failed to init audit log table operator", K(ret), K(tenant_id)); + } else if (OB_FAIL(table_operator.get_audit_filter(audit_filter, tmp_allocator, + true /*for_update*/, is_exist))) { + LOG_WARN("failed to set audit log filter", K(ret), K(audit_filter)); + } else if (!is_exist) { + // do nothing + } else if (OB_FAIL(table_operator.set_audit_user(audit_user))) { + LOG_WARN("failed to set audit log user", K(ret), K(audit_user)); + } + if (trans.is_started()) { + int temp_ret = OB_SUCCESS; + if (OB_SUCCESS != (temp_ret = trans.end(OB_SUCC(ret)))) { + LOG_WARN("trans end failed", "is_commit", OB_SUCCESS == ret, K(ret), K(temp_ret)); + ret = OB_SUCC(ret) ? temp_ret : ret; + } + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(fill_res_datum(expr, ctx, is_valid, error_info, expr_datum))) { + LOG_WARN("failed to fill expr_datum", K(ret), K(is_valid), K(error_info)); + } + } +#endif + return ret; +} +ObExprAuditLogRemoveUser::ObExprAuditLogRemoveUser(ObIAllocator &alloc) + : ObExprAuditLogFunc(alloc, T_FUN_SYS_AUDIT_LOG_REMOVE_USER, N_AUDIT_LOG_FILTER_REMOVE_USER, 1) +{ +} + +int ObExprAuditLogRemoveUser::cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, + ObExpr &expr) const +{ + int ret = OB_SUCCESS; + UNUSED(expr_cg_ctx); + UNUSED(raw_expr); + expr.eval_func_ = eval_remove_user; + return ret; +} + +int ObExprAuditLogRemoveUser::eval_remove_user(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &expr_datum) +{ + int ret = OB_SUCCESS; +#ifdef OB_BUILD_AUDIT_SECURITY + ObDatum *arg0 = NULL; + ObSQLSessionInfo *session = ctx.exec_ctx_.get_my_session(); + if (OB_ISNULL(session)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null session", K(ret)); + } else if (OB_FAIL(expr.eval_param_value(ctx, arg0))) { + LOG_WARN("evaluate parameters values failed", K(ret)); + } else { + ObString user_name; + ObString host; + ObString error_info; + ObSEArray params; + bool is_valid = true; + if (OB_FAIL(check_privilege(*session, is_valid, error_info))) { + LOG_WARN("failed to check privilege", K(ret)); + } else if (is_valid && OB_FAIL(parse_user_name(arg0->get_string(), user_name, host, + is_valid, error_info))) { + } else if (is_valid) { + uint64_t tenant_id = session->get_effective_tenant_id(); + ObAuditLogUserInfo audit_user; + ObAuditLogTableOperator table_operator; + ObISQLClient *sql_client = ctx.exec_ctx_.get_sql_proxy(); + audit_user.set_tenant_id(tenant_id); + audit_user.set_user_name(user_name); + audit_user.set_host(host); + if (OB_ISNULL(sql_client)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null sql client", K(ret)); + } else if (OB_FAIL(table_operator.init(tenant_id, sql_client))) { + LOG_WARN("failed to init audit log table operator", K(ret), K(tenant_id)); + } else if (OB_FAIL(table_operator.mark_audit_user_delete(audit_user))) { + LOG_WARN("failed to mark audit log user delete", K(ret), K(audit_user)); + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(fill_res_datum(expr, ctx, is_valid, error_info, expr_datum))) { + LOG_WARN("failed to fill expr_datum", K(ret), K(is_valid), K(error_info)); + } + } +#endif + return ret; +} + +} +} diff --git a/src/sql/engine/expr/ob_expr_audit_log_func.h b/src/sql/engine/expr/ob_expr_audit_log_func.h new file mode 100644 index 000000000..33ad0370b --- /dev/null +++ b/src/sql/engine/expr/ob_expr_audit_log_func.h @@ -0,0 +1,126 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_SQL_OB_EXPR_AUDIT_LOG_FUNC_H +#define OCEANBASE_SQL_OB_EXPR_AUDIT_LOG_FUNC_H + +#include "sql/engine/expr/ob_expr_operator.h" + +namespace oceanbase +{ +namespace sql +{ + +class ObExprAuditLogFunc : public ObStringExprOperator +{ +public: + explicit ObExprAuditLogFunc(common::ObIAllocator &alloc, + ObExprOperatorType type, + const char *name, + int32_t param_num); + virtual ~ObExprAuditLogFunc() {} + virtual int calc_result_type1(ObExprResType &type, + ObExprResType &type1, + common::ObExprTypeCtx &type_ctx) const override; + virtual int calc_result_type2(ObExprResType &type, + ObExprResType &type1, + ObExprResType &type2, + common::ObExprTypeCtx &type_ctx) const override; +protected: + static int check_privilege(ObSQLSessionInfo &session, + bool &is_valid, + common::ObString &error_info); + static int parse_user_name(const common::ObString &str, + common::ObString &user_name, + common::ObString &host, + bool &is_valid, + common::ObString &error_info); + static int parse_filter_name(const common::ObString &str, + common::ObString &filter_name, + bool &is_valid, + common::ObString &error_info); + static int parse_definition(const common::ObString &str, + common::ObString &definition, + bool &is_valid, + common::ObString &error_info); + static int fill_res_datum(const ObExpr &expr, + ObEvalCtx &ctx, + const bool is_valid, + const common::ObString &error_info, + ObDatum &expr_datum); +private: + int check_data_version(common::ObExprTypeCtx &type_ctx) const; + int check_param_type(const ObExprResType &type) const; +private: + //disallow copy + DISALLOW_COPY_AND_ASSIGN(ObExprAuditLogFunc); +}; + +class ObExprAuditLogSetFilter : public ObExprAuditLogFunc +{ +public: + explicit ObExprAuditLogSetFilter(common::ObIAllocator &alloc); + virtual ~ObExprAuditLogSetFilter() {} + virtual int cg_expr(ObExprCGCtx &op_cg_ctx, + const ObRawExpr &raw_expr, + ObExpr &rt_expr) const override; + static int eval_set_filter(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &expr_datum); +private : + //disallow copy + DISALLOW_COPY_AND_ASSIGN(ObExprAuditLogSetFilter); +}; + +class ObExprAuditLogRemoveFilter : public ObExprAuditLogFunc +{ +public: + explicit ObExprAuditLogRemoveFilter(common::ObIAllocator &alloc); + virtual ~ObExprAuditLogRemoveFilter() {} + virtual int cg_expr(ObExprCGCtx &op_cg_ctx, + const ObRawExpr &raw_expr, + ObExpr &rt_expr) const override; + static int eval_remove_filter(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &expr_datum); +private : + //disallow copy + DISALLOW_COPY_AND_ASSIGN(ObExprAuditLogRemoveFilter); +}; + +class ObExprAuditLogSetUser : public ObExprAuditLogFunc +{ +public: + explicit ObExprAuditLogSetUser(common::ObIAllocator &alloc); + virtual ~ObExprAuditLogSetUser() {} + virtual int cg_expr(ObExprCGCtx &op_cg_ctx, + const ObRawExpr &raw_expr, + ObExpr &rt_expr) const override; + static int eval_set_user(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &expr_datum); +private : + //disallow copy + DISALLOW_COPY_AND_ASSIGN(ObExprAuditLogSetUser); +}; + +class ObExprAuditLogRemoveUser : public ObExprAuditLogFunc +{ +public: + explicit ObExprAuditLogRemoveUser(common::ObIAllocator &alloc); + virtual ~ObExprAuditLogRemoveUser() {} + virtual int cg_expr(ObExprCGCtx &op_cg_ctx, + const ObRawExpr &raw_expr, + ObExpr &rt_expr) const override; + static int eval_remove_user(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &expr_datum); +private : + //disallow copy + DISALLOW_COPY_AND_ASSIGN(ObExprAuditLogRemoveUser); +}; + +} +} +#endif diff --git a/src/sql/engine/expr/ob_expr_can_access_trigger.cpp b/src/sql/engine/expr/ob_expr_can_access_trigger.cpp new file mode 100644 index 000000000..686516280 --- /dev/null +++ b/src/sql/engine/expr/ob_expr_can_access_trigger.cpp @@ -0,0 +1,108 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_ENG + +#include "sql/engine/expr/ob_expr_can_access_trigger.h" +#include "share/schema/ob_schema_getter_guard.h" +#include "objit/common/ob_item_type.h" +#include "sql/session/ob_sql_session_info.h" +#include "sql/engine/ob_exec_context.h" + +using namespace oceanbase::common; +namespace oceanbase +{ +namespace sql +{ +ObExprCanAccessTrigger::ObExprCanAccessTrigger(ObIAllocator &alloc) + : ObExprOperator(alloc, T_FUN_SYS_CAN_ACCESS_TRIGGER, N_CAN_ACCESS_TRIGGER, 2, NOT_VALID_FOR_GENERATED_COL, NOT_ROW_DIMENSION, true) +{ +} + +ObExprCanAccessTrigger::~ObExprCanAccessTrigger() +{ +} + +int ObExprCanAccessTrigger::calc_result_type2(ObExprResType &type, + ObExprResType &type1, + ObExprResType &type2, + ObExprTypeCtx &type_ctx) const +{ + UNUSED(type_ctx); + int ret = OB_SUCCESS; + type1.set_calc_type(ObVarcharType); + type1.set_calc_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + type2.set_calc_type(ObVarcharType); + type2.set_calc_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + type.set_type(ObInt32Type); + return ret; +} + +int ObExprCanAccessTrigger::can_access_trigger(const ObExpr &expr, + ObEvalCtx &ctx, + ObDatum &res_datum) +{ + int ret = OB_SUCCESS; + bool need_check = false; + if (OB_FAIL(expr.eval_param_value(ctx))) { + LOG_WARN("eval arg failed", K(ret)); + } else { + ObDatum &database_name = expr.locate_param_datum(ctx, 0); + ObDatum &table_name = expr.locate_param_datum(ctx, 1); + OX (res_datum.set_true()); + ObSQLSessionInfo *session = ctx.exec_ctx_.get_my_session(); + ObSessionPrivInfo session_priv; + if (OB_ISNULL(session)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("session is NULL", K(ret)); + } else if (OB_FAIL(session->check_feature_enable( + ObCompatFeatureType::MYSQL_TRIGGER_PRIV_CHECK, need_check))) { + LOG_WARN("failed to check feature enable", K(ret)); + } else if (need_check && lib::is_mysql_mode()) { + if (OB_FAIL(session->get_session_priv_info(session_priv))) { + LOG_WARN("faile to get session priv info", K(ret)); + } else { + ObNeedPriv need_priv; + need_priv.priv_level_ = OB_PRIV_TABLE_LEVEL; + need_priv.db_ = database_name.get_string(); + need_priv.priv_set_ = OB_PRIV_TRIGGER; + need_priv.table_ = table_name.get_string(); + share::schema::ObSchemaGetterGuard *schema_guard = + ctx.exec_ctx_.get_sql_ctx()->schema_guard_; + if (OB_ISNULL(schema_guard)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("schema guard is NULL", K(ret)); + } else { + OZ (schema_guard->check_single_table_priv(session_priv, need_priv), K(need_priv), K(ret)); + if(OB_ERR_NO_TABLE_PRIVILEGE == ret) { + ret = OB_SUCCESS; + OX (res_datum.set_false()); + } + } + } + } + } + return ret; +} + +int ObExprCanAccessTrigger::cg_expr(ObExprCGCtx &op_cg_ctx, + const ObRawExpr &raw_expr, + ObExpr &rt_expr) const +{ + UNUSED(raw_expr); + UNUSED(op_cg_ctx); + rt_expr.eval_func_ = ObExprCanAccessTrigger::can_access_trigger; + return OB_SUCCESS; +} + +} +} \ No newline at end of file diff --git a/src/sql/engine/expr/ob_expr_can_access_trigger.h b/src/sql/engine/expr/ob_expr_can_access_trigger.h new file mode 100644 index 000000000..ad3ac3948 --- /dev/null +++ b/src/sql/engine/expr/ob_expr_can_access_trigger.h @@ -0,0 +1,38 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef _OB_EXPR_CAN_ACCESS_TRIGGER_H +#define _OB_EXPR_CAN_ACCESS_TRIGGER_H + +#include "sql/engine/expr/ob_expr_operator.h" + +namespace oceanbase +{ +namespace sql +{ +class ObExprCanAccessTrigger : public ObExprOperator +{ +public: + explicit ObExprCanAccessTrigger(common::ObIAllocator &alloc); + virtual ~ObExprCanAccessTrigger(); + virtual int calc_result_type2(ObExprResType &type, + ObExprResType &type1, + ObExprResType &type2, + common::ObExprTypeCtx &type_ctx) const; + static int can_access_trigger(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &expr_datum); + virtual int cg_expr(ObExprCGCtx &op_cg_ctx, const ObRawExpr &raw_expr, ObExpr &rt_expr) const override; +private: + DISALLOW_COPY_AND_ASSIGN(ObExprCanAccessTrigger); +}; +} +} +#endif \ No newline at end of file diff --git a/src/sql/engine/expr/ob_expr_case.cpp b/src/sql/engine/expr/ob_expr_case.cpp index ba6c65d92..8dd09949c 100644 --- a/src/sql/engine/expr/ob_expr_case.cpp +++ b/src/sql/engine/expr/ob_expr_case.cpp @@ -63,15 +63,13 @@ int ObExprCase::calc_result_typeN(ObExprResType &type, */ const int64_t cond_type_count = param_num / 2; const int64_t val_type_count = param_num - cond_type_count; - const ObLengthSemantics default_length_semantics = (OB_NOT_NULL(type_ctx.get_session()) ? type_ctx.get_session()->get_actual_nls_length_semantics() : LS_BYTE); if (OB_FAIL(aggregate_result_type_for_case( type, types_stack + cond_type_count, val_type_count, - type_ctx.get_coll_type(), lib::is_oracle_mode(), - default_length_semantics, + type_ctx, true, false, is_called_in_sql_))) { LOG_WARN("failed to aggregate result type"); diff --git a/src/sql/engine/expr/ob_expr_cast.cpp b/src/sql/engine/expr/ob_expr_cast.cpp index 43e7d0f84..9230ad584 100644 --- a/src/sql/engine/expr/ob_expr_cast.cpp +++ b/src/sql/engine/expr/ob_expr_cast.cpp @@ -348,6 +348,11 @@ int ObExprCast::calc_result_type2(ObExprResType &type, } else if (OB_FAIL(get_cast_type(enable_decimalint, type2, cast_raw_expr->get_extra(), dst_type))) { LOG_WARN("get cast dest type failed", K(ret)); + } else if (OB_FAIL(ObSQLUtils::get_cs_level_from_cast_mode(cast_raw_expr->get_extra(), + type1.get_collation_level(), + cs_level))) { + LOG_WARN("failed to get collation level", K(ret)); + } else if (FALSE_IT(dst_type.set_collation_level(cs_level))) { } else if (OB_FAIL(adjust_udt_cast_type(type1, dst_type, type_ctx))) { LOG_WARN("adjust udt cast sub type failed", K(ret)); } else if (OB_UNLIKELY(!cast_supported(type1.get_type(), type1.get_collation_type(), @@ -368,10 +373,6 @@ int ObExprCast::calc_result_type2(ObExprResType &type, } } else if (FALSE_IT(is_explicit_cast = CM_IS_EXPLICIT_CAST(cast_raw_expr->get_extra()))) { // check cast supported in cast_map but not support here. - } else if (OB_FAIL(ObSQLUtils::get_cs_level_from_cast_mode(cast_raw_expr->get_extra(), - type1.get_collation_level(), - cs_level))) { - LOG_WARN("failed to get collation level", K(ret)); } else if (!check_cast_allowed(type1.get_type(), type1.get_collation_type(), dst_type.get_type(), dst_type.get_collation_type(), is_explicit_cast)) { @@ -438,7 +439,7 @@ int ObExprCast::calc_result_type2(ObExprResType &type, int32_t length = 0; if (ob_is_string_or_lob_type(dst_type.get_type()) || ob_is_raw(dst_type.get_type()) || ob_is_json(dst_type.get_type()) || ob_is_geometry(dst_type.get_type())) { - type.set_collation_level(cs_level); + type.set_collation_level(dst_type.get_collation_level()); int32_t len = dst_type.get_length(); int16_t length_semantics = ((dst_type.is_string_or_lob_locator_type() || dst_type.is_json()) ? dst_type.get_length_semantics() @@ -603,6 +604,7 @@ int ObExprCast::calc_result_type2(ObExprResType &type, // to add enum_to_str(), so we still set the calc type but skip add implicit cast in decuding. type1.set_calc_type(type.get_type()); type1.set_calc_collation_type(type.get_collation_type()); + type1.set_calc_collation_level(type.get_collation_level()); type1.set_calc_accuracy(type.get_accuracy()); } } diff --git a/src/sql/engine/expr/ob_expr_chr.cpp b/src/sql/engine/expr/ob_expr_chr.cpp index 7effd3454..8ebbe9718 100644 --- a/src/sql/engine/expr/ob_expr_chr.cpp +++ b/src/sql/engine/expr/ob_expr_chr.cpp @@ -63,7 +63,8 @@ inline int ObExprChr::calc_result_type1(ObExprResType &type, int ObExprChr::number2varchar(ObString &str_result, const double double_val, - ObIAllocator &alloc) + ObIAllocator &alloc, + bool is_single_byte) { int ret = OB_SUCCESS; int64_t int_val = static_cast(double_val); @@ -77,6 +78,9 @@ int ObExprChr::number2varchar(ObString &str_result, str_result.reset(); } else { uint32_t num = static_cast(int_val); + if (is_single_byte) { + num %= 256; + } int length = 0; if (num & 0xFF000000UL) { length = 4; @@ -105,7 +109,11 @@ int ObExprChr::calc_chr_expr(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res_da { int ret = OB_SUCCESS; ObDatum *arg = NULL; - if (OB_FAIL(expr.args_[0]->eval(ctx, arg))) { + sql::ObSQLSessionInfo *my_session_ = NULL; + if (OB_ISNULL(my_session_ = ctx.exec_ctx_.get_my_session())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null pointer", K(ret)); + } else if (OB_FAIL(expr.args_[0]->eval(ctx, arg))) { LOG_WARN("eval arg failed", K(ret)); } else if (arg->is_null()) { res_datum.set_null(); @@ -113,7 +121,12 @@ int ObExprChr::calc_chr_expr(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res_da ObExprStrResAlloc res_alloc(expr, ctx); double arg_double = arg->get_double(); ObString res_str; - if (OB_FAIL(ObExprChr::number2varchar(res_str, arg_double, res_alloc))) { + bool is_single_byte = false; + int64_t maxmb_len = 0; + if (OB_FAIL(ObCharset::get_mbmaxlen_by_coll(my_session_->get_nls_collation(), maxmb_len))) { + LOG_WARN("failed to get mbmaxlen by coll", K(my_session_->get_nls_collation())); + } else if (FALSE_IT(is_single_byte = (maxmb_len==1))) { + } else if (OB_FAIL(ObExprChr::number2varchar(res_str, arg_double, res_alloc, is_single_byte))) { LOG_WARN("number2varchar failed", K(ret), K(arg_double)); } else if (res_str.empty()) { res_datum.set_null(); diff --git a/src/sql/engine/expr/ob_expr_chr.h b/src/sql/engine/expr/ob_expr_chr.h index f3d88238c..ccadda77d 100644 --- a/src/sql/engine/expr/ob_expr_chr.h +++ b/src/sql/engine/expr/ob_expr_chr.h @@ -31,7 +31,8 @@ public: ObExpr &rt_expr) const; static int number2varchar(common::ObString &str_result, const double text, - common::ObIAllocator &alloc); + common::ObIAllocator &alloc, + bool is_single_byte); static int calc_chr_expr(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res_datum); private: DISALLOW_COPY_AND_ASSIGN(ObExprChr); diff --git a/src/sql/engine/expr/ob_expr_coalesce.cpp b/src/sql/engine/expr/ob_expr_coalesce.cpp index 608f4926f..4bb950dbd 100644 --- a/src/sql/engine/expr/ob_expr_coalesce.cpp +++ b/src/sql/engine/expr/ob_expr_coalesce.cpp @@ -38,7 +38,6 @@ int ObExprCoalesce::calc_result_typeN(ObExprResType &type, ObExprTypeCtx &type_ctx) const { int ret = OB_SUCCESS; - const ObLengthSemantics default_length_semantics = (OB_NOT_NULL(type_ctx.get_session()) ? type_ctx.get_session()->get_actual_nls_length_semantics() : LS_BYTE); if (OB_ISNULL(types)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("null types", K(ret)); @@ -49,9 +48,8 @@ int ObExprCoalesce::calc_result_typeN(ObExprResType &type, type, types, param_num, - type_ctx.get_coll_type(), lib::is_oracle_mode(), - default_length_semantics, + type_ctx, true, true, is_called_in_sql_))) { diff --git a/src/sql/engine/expr/ob_expr_collation.cpp b/src/sql/engine/expr/ob_expr_collation.cpp index ee21fe0ef..0f4f62baa 100644 --- a/src/sql/engine/expr/ob_expr_collation.cpp +++ b/src/sql/engine/expr/ob_expr_collation.cpp @@ -194,6 +194,8 @@ int ObExprCoercibility::calc_result_type1(ObExprResType &type, type.set_int(); type.set_precision(ObAccuracy::DDL_DEFAULT_ACCURACY[ObIntType].precision_); type.set_scale(ObAccuracy::DDL_DEFAULT_ACCURACY[ObIntType].scale_); + type.set_collation_type(CS_TYPE_BINARY); + type.set_collation_level(CS_LEVEL_NUMERIC); return ret; } diff --git a/src/sql/engine/expr/ob_expr_compress.cpp b/src/sql/engine/expr/ob_expr_compress.cpp index d11534db7..994638085 100644 --- a/src/sql/engine/expr/ob_expr_compress.cpp +++ b/src/sql/engine/expr/ob_expr_compress.cpp @@ -49,7 +49,7 @@ int ObExprCompress::calc_result_type1(ObExprResType &type, type.set_varbinary(); type.set_collation_level(CS_LEVEL_COERCIBLE); type1.set_calc_type(ObVarcharType); - OZ (aggregate_charsets_for_string_result(tmp_type, &type1, 1, type_ctx.get_coll_type())); + OZ (aggregate_charsets_for_string_result(tmp_type, &type1, 1, type_ctx)); OX (type1.set_calc_collation_type(tmp_type.get_collation_type())); OX (type1.set_calc_collation_level(tmp_type.get_collation_level())); OZ (ObCharset::get_mbmaxlen_by_coll(tmp_type.get_collation_type(), mbmaxlen)); diff --git a/src/sql/engine/expr/ob_expr_concat.cpp b/src/sql/engine/expr/ob_expr_concat.cpp index 694e78aa2..1a9f292a1 100644 --- a/src/sql/engine/expr/ob_expr_concat.cpp +++ b/src/sql/engine/expr/ob_expr_concat.cpp @@ -232,10 +232,11 @@ int ObExprConcat::calc_result_typeN(ObExprResType &type, OZ (aggregate_charsets_for_string_result(type, types, param_num, - type_ctx.get_coll_type())); + type_ctx)); for (int64_t i = 0; i < param_num; ++i) { types[i].set_calc_type(type.get_type()); types[i].set_calc_collation_type(type.get_collation_type()); + types[i].set_calc_collation_level(type.get_collation_level()); } } diff --git a/src/sql/engine/expr/ob_expr_concat_ws.cpp b/src/sql/engine/expr/ob_expr_concat_ws.cpp index 0a4dff102..c93f92802 100644 --- a/src/sql/engine/expr/ob_expr_concat_ws.cpp +++ b/src/sql/engine/expr/ob_expr_concat_ws.cpp @@ -58,11 +58,12 @@ int ObExprConcatWs::calc_result_typeN(ObExprResType &type, types[0].set_calc_type(ObVarcharType); type.set_length(len); type.set_varchar(); - if (OB_FAIL(aggregate_charsets_for_string_result(type, types, param_num, type_ctx.get_coll_type()))) { + if (OB_FAIL(aggregate_charsets_for_string_result(type, types, param_num, type_ctx))) { LOG_WARN("aggregate_charsets_for_string_result failed", K(ret)); } else { for (int64_t i = 0; i < param_num; i++) { types[i].set_calc_collation_type(type.get_collation_type()); + types[i].set_calc_collation_level(type.get_collation_level()); } } } diff --git a/src/sql/engine/expr/ob_expr_convert.cpp b/src/sql/engine/expr/ob_expr_convert.cpp index 0b7d314fd..69f4d1402 100644 --- a/src/sql/engine/expr/ob_expr_convert.cpp +++ b/src/sql/engine/expr/ob_expr_convert.cpp @@ -60,7 +60,7 @@ int ObExprConvert::calc_result_type2(ObExprResType &type, ret = OB_ERR_UNKNOWN_CHARSET; LOG_WARN("unknown charset", K(ret), K(cs_name)); } else { - type.set_collation_level(CS_LEVEL_EXPLICIT); + type.set_collation_level(CS_LEVEL_IMPLICIT); type.set_collation_type(ObCharset::get_default_collation(charset_type)); //set calc type //only set type2 here. diff --git a/src/sql/engine/expr/ob_expr_current_user_priv.cpp b/src/sql/engine/expr/ob_expr_current_user_priv.cpp index cd30e9d1b..2fdc635df 100644 --- a/src/sql/engine/expr/ob_expr_current_user_priv.cpp +++ b/src/sql/engine/expr/ob_expr_current_user_priv.cpp @@ -152,5 +152,86 @@ int ObExprCurrentRole::eval_current_role(const ObExpr &expr, ObEvalCtx &ctx, ObD return ret; } +ObExprIsEnabledRole::ObExprIsEnabledRole(ObIAllocator &alloc) + : ObFuncExprOperator(alloc, T_FUN_SYS_IS_ENABLED_ROLE, N_IS_ENABLED_ROLE, 2, + NOT_VALID_FOR_GENERATED_COL, NOT_ROW_DIMENSION, true) { +} + +ObExprIsEnabledRole::~ObExprIsEnabledRole() { +} + +int ObExprIsEnabledRole::cg_expr(ObExprCGCtx &op_cg_ctx, + const ObRawExpr &raw_expr, + ObExpr &rt_expr) const +{ + UNUSED(raw_expr); + UNUSED(op_cg_ctx); + rt_expr.eval_func_ = ObExprIsEnabledRole::eval_is_enabled_role; + return OB_SUCCESS; +} + +int ObExprIsEnabledRole::calc_result_type2(ObExprResType &type, + ObExprResType &type1, + ObExprResType &type2, + ObExprTypeCtx &type_ctx) const +{ + UNUSED(type_ctx); + type.set_tinyint(); + type.set_precision(DEFAULT_PRECISION_FOR_BOOL); + type.set_scale(DEFAULT_SCALE_FOR_INTEGER); + type1.set_calc_type(ObVarcharType); + type1.set_calc_collation_type(common::ObCharset::get_default_collation(common::ObCharset::get_default_charset())); + type1.set_calc_length(type1.get_length()); + type2.set_calc_type(ObVarcharType); + type2.set_calc_collation_type(common::ObCharset::get_default_collation(common::ObCharset::get_default_charset())); + type2.set_calc_length(type2.get_length()); + return OB_SUCCESS; +} + +int ObExprIsEnabledRole::eval_is_enabled_role(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &expr_datum) +{ + int ret = OB_SUCCESS; + uint64_t tenant_id = MTL_ID(); + ObSchemaGetterGuard schema_guard; + ObSQLSessionInfo *session_info = NULL; + ObDatum *user = NULL; + ObDatum *host = NULL; + ObString user_name; + ObString host_name; + bool is_enabled_role = false; + if (OB_ISNULL(session_info = ctx.exec_ctx_.get_my_session())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("session info is null", K(ret)); + } else if (OB_ISNULL(GCTX.schema_service_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get schema_service", K(ret)); + } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(tenant_id, schema_guard))) { + LOG_WARN("failed to get schema guard", K(ret)); + } else if (OB_UNLIKELY(2 != expr.arg_cnt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected arg cnt", K(ret)); + } else if (OB_FAIL(expr.eval_param_value(ctx, user, host))) { + LOG_WARN("eval arg failed", K(ret)); + } else if (user->is_null() || host->is_null()) { + } else { + user_name = user->get_string(); + host_name = host->get_string(); + const ObIArray &roles = session_info->get_enable_role_array(); + const ObUserInfo *user_info = NULL; + for (int i = 0; OB_SUCC(ret) && !is_enabled_role && i < roles.count(); i++) { + if (OB_FAIL(schema_guard.get_user_info(tenant_id, roles.at(i), user_info))) { + LOG_WARN("failed to get user info", K(ret)); + } else if (OB_ISNULL(user_info)) { + //ignored + } else if ((user_info->get_user_name_str().compare(user_name) == 0) + && (user_info->get_host_name_str().compare(host_name) == 0)) { + is_enabled_role = true; + } + } + } + expr_datum.set_bool(is_enabled_role); + return ret; +} + }/* ns sql*/ }/* ns oceanbase */ diff --git a/src/sql/engine/expr/ob_expr_current_user_priv.h b/src/sql/engine/expr/ob_expr_current_user_priv.h index 541fbd551..63c5ec963 100644 --- a/src/sql/engine/expr/ob_expr_current_user_priv.h +++ b/src/sql/engine/expr/ob_expr_current_user_priv.h @@ -48,6 +48,23 @@ public: private: DISALLOW_COPY_AND_ASSIGN(ObExprCurrentRole); }; + +class ObExprIsEnabledRole : public ObFuncExprOperator +{ +public: + explicit ObExprIsEnabledRole(common::ObIAllocator &alloc); + virtual ~ObExprIsEnabledRole(); + virtual int calc_result_type2(ObExprResType &type, + ObExprResType &type1, + ObExprResType &type2, + common::ObExprTypeCtx &type_ctx) const; + static int eval_is_enabled_role(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &expr_datum); + virtual int cg_expr(ObExprCGCtx &op_cg_ctx, + const ObRawExpr &raw_expr, + ObExpr &rt_expr) const override; +private: + DISALLOW_COPY_AND_ASSIGN(ObExprIsEnabledRole); +}; } } #endif /* OCEANBASE_SQL_ENGINE_EXPR_OB_EXPR_CURRENT_USER_PRIV_ */ diff --git a/src/sql/engine/expr/ob_expr_date_add.cpp b/src/sql/engine/expr/ob_expr_date_add.cpp index b6d34835b..e26837d94 100644 --- a/src/sql/engine/expr/ob_expr_date_add.cpp +++ b/src/sql/engine/expr/ob_expr_date_add.cpp @@ -80,7 +80,9 @@ int ObExprDateAdjust::calc_result_type3(ObExprResType &type, date.set_calc_type(ObVarcharType); type.set_varchar(); type.set_length(DATETIME_MAX_LENGTH); - ret = aggregate_charsets_for_string_result(type, &date, 1, type_ctx.get_coll_type()); + ret = aggregate_charsets_for_string_result(type, &date, 1, type_ctx); + date.set_calc_collation_type(type.get_collation_type()); + date.set_calc_collation_level(type.get_collation_level()); } interval.set_calc_type(ObVarcharType); if (OB_SUCC(ret)) { diff --git a/src/sql/engine/expr/ob_expr_day_of_func.cpp b/src/sql/engine/expr/ob_expr_day_of_func.cpp index 60cd8cda9..8f252565f 100644 --- a/src/sql/engine/expr/ob_expr_day_of_func.cpp +++ b/src/sql/engine/expr/ob_expr_day_of_func.cpp @@ -333,7 +333,9 @@ int ObExprSubAddtime::calc_result_type2(ObExprResType &type, type.set_varchar(); type.set_length(DATETIME_MAX_LENGTH); type.set_scale(-1); - ret = aggregate_charsets_for_string_result(type, &date_arg, 1, type_ctx.get_coll_type()); + ret = aggregate_charsets_for_string_result(type, &date_arg, 1, type_ctx); + date_arg.set_calc_collation_type(type.get_collation_type()); + date_arg.set_calc_collation_level(type.get_collation_level()); } if (OB_SUCC(ret)) { // date_arg无法设置calc_type的原因是，date_arg类型是varchar时，设置calc_type为time和datetime都不合适 diff --git a/src/sql/engine/expr/ob_expr_elt.cpp b/src/sql/engine/expr/ob_expr_elt.cpp index 6efa46d23..4ef3cce8a 100644 --- a/src/sql/engine/expr/ob_expr_elt.cpp +++ b/src/sql/engine/expr/ob_expr_elt.cpp @@ -44,7 +44,7 @@ int ObExprElt::calc_result_typeN( } else { type.set_varchar(); ret = aggregate_charsets_for_string_result( - type, types_stack + 1, param_num - 1, type_ctx.get_coll_type()); + type, types_stack + 1, param_num - 1, type_ctx); if (OB_SUCC(ret)) { int32_t length = 0; for (int64_t i = 1; i < param_num; ++i) { diff --git a/src/sql/engine/expr/ob_expr_eval_functions.cpp b/src/sql/engine/expr/ob_expr_eval_functions.cpp index 944211f54..8b4034395 100644 --- a/src/sql/engine/expr/ob_expr_eval_functions.cpp +++ b/src/sql/engine/expr/ob_expr_eval_functions.cpp @@ -151,7 +151,7 @@ #include "ob_expr_shadow_uk_project.h" #include "ob_expr_char_length.h" #include "ob_expr_unix_timestamp.h" -#include "ob_expr_aes_encrypt.h" +#include "ob_expr_symmetric_encrypt.h" #include "ob_expr_case.h" #include "ob_expr_oracle_decode.h" #include "ob_expr_remove_const.h" @@ -390,6 +390,8 @@ #include "ob_expr_rb_calc.h" #include "ob_expr_rb_to_string.h" #include "ob_expr_rb_from_string.h" +#include "ob_expr_audit_log_func.h" +#include "ob_expr_can_access_trigger.h" namespace oceanbase { @@ -1144,10 +1146,10 @@ static ObExpr::EvalFunc g_expr_eval_functions[] = { ObExprSTSymDifference::eval_st_symdifference, /* 650 */ ObExprPrivSTAsMVTGeom::eval_priv_st_asmvtgeom, /* 651 */ ObExprPrivSTMakeValid::eval_priv_st_makevalid, /* 652 */ - NULL, //ObExprAuditLogSetFilter::eval_set_filter, /* 653 */ - NULL, //ObExprAuditLogRemoveFilter::eval_remove_filter, /* 654 */ - NULL, //ObExprAuditLogSetUser::eval_set_user, /* 655 */ - NULL, //ObExprAuditLogRemoveUser::eval_remove_user, /* 656 */ + ObExprAuditLogSetFilter::eval_set_filter, /* 653 */ + ObExprAuditLogRemoveFilter::eval_remove_filter, /* 654 */ + ObExprAuditLogSetUser::eval_set_user, /* 655 */ + ObExprAuditLogRemoveUser::eval_remove_user, /* 656 */ eval_questionmark_decint2nmb, /* 657 */ eval_questionmark_nmb2decint_eqcast, /* 658 */ eval_questionmark_decint2decint_eqcast, /* 659 */ @@ -1184,8 +1186,8 @@ static ObExpr::EvalFunc g_expr_eval_functions[] = { ObExprLastRefreshScn::eval_last_refresh_scn, /* 690 */ ObExprDocLength::generate_doc_length, /* 691 */ ObExprTopNFilter::eval_topn_filter, /* 692 */ - NULL, // ObExprIsEnabledRole::eval_is_enabled_role, /* 693 */ - NULL, // ObExprCanAccessTrigger::can_access_trigger, /* 694 */ + ObExprIsEnabledRole::eval_is_enabled_role, /* 693 */ + ObExprCanAccessTrigger::can_access_trigger, /* 694 */ NULL, //ObRelationalExprOperator::eval_min_max_compare, /* 695 */ NULL, //ObRelationalExprOperator::min_max_row_eval, /* 696 */ ObExprRbBuildEmpty::eval_rb_build_empty, /* 697 */ @@ -1221,9 +1223,9 @@ static ObExpr::EvalFunc g_expr_eval_functions[] = { NULL, // ObExprVectorDistance::calc_distance, /* 727 */ NULL, // ObExprInnerDoubleToInt::eval_inner_double_to_int /* 728 */ NULL, // ObExprInnerDecimalToYear::eval_inner_decimal_to_year /* 729 */ - NULL, // ObExprSm3::eval_sm3, /* 730 */ - NULL, // ObExprSm4Encrypt::eval_sm4_encrypt, /* 731 */ - NULL, // ObExprSm4Decrypt::eval_sm4_decrypt, /* 732 */ + ObExprSm3::eval_sm3, /* 730 */ + ObExprSm4Encrypt::eval_sm4_encrypt, /* 731 */ + ObExprSm4Decrypt::eval_sm4_decrypt, /* 732 */ NULL, // ObExprAdd::add_vec_vec, /* 733 */ NULL, // ObExprMinus::minus_vec_vec, /* 734 */ NULL, // ObExprMul::mul_vec_vec, /* 735 */ diff --git a/src/sql/engine/expr/ob_expr_export_set.cpp b/src/sql/engine/expr/ob_expr_export_set.cpp index b6e9cf576..fff31eec2 100644 --- a/src/sql/engine/expr/ob_expr_export_set.cpp +++ b/src/sql/engine/expr/ob_expr_export_set.cpp @@ -70,7 +70,7 @@ int ObExprExportSet::calc_result_typeN(ObExprResType& type, ObExprResType* types type.set_varchar(); // set collation_type for string type OZ(ObExprOperator::aggregate_charsets_for_string_result_with_comparison( - type, &types_array[1], str_num, type_ctx.get_coll_type())); + type, &types_array[1], str_num, type_ctx)); for (int64_t i = 1; OB_SUCC(ret) && i <= str_num; ++i) { types_array[i].set_calc_meta(type); } diff --git a/src/sql/engine/expr/ob_expr_field.cpp b/src/sql/engine/expr/ob_expr_field.cpp index 8333cadc0..c9f3874d8 100644 --- a/src/sql/engine/expr/ob_expr_field.cpp +++ b/src/sql/engine/expr/ob_expr_field.cpp @@ -117,11 +117,13 @@ int ObExprField::calc_result_typeN(ObExprResType &type, type.set_precision(3); //calc comparison collation,etc type.set_scale(DEFAULT_SCALE_FOR_INTEGER); - if (ob_is_string_type(type.get_calc_type())) { - ret = aggregate_charsets_for_comparison(type, types_stack, param_num, type_ctx.get_coll_type()); - } + OZ (aggregate_charsets_for_comparison(type, + types_stack, + param_num, + type_ctx)); if (OB_SUCC(ret) && !types_stack[0].is_null()) { for (int64_t i = 0; OB_SUCC(ret) && i < param_num; ++i) { + types_stack[i].set_calc_type(type.get_type()); types_stack[i].set_calc_meta(type.get_calc_meta()); types_stack[i].set_calc_accuracy(types_stack[0].get_accuracy()); if (type.get_calc_meta().is_decimal_int()) { diff --git a/src/sql/engine/expr/ob_expr_find_in_set.cpp b/src/sql/engine/expr/ob_expr_find_in_set.cpp index c514b10c7..f48856196 100644 --- a/src/sql/engine/expr/ob_expr_find_in_set.cpp +++ b/src/sql/engine/expr/ob_expr_find_in_set.cpp @@ -47,14 +47,13 @@ int ObExprFindInSet::calc_result_type2(ObExprResType &type, ObObjMeta coll_types[2]; coll_types[0] = type1.get_obj_meta(); coll_types[1] = type2.get_obj_meta(); - if (OB_FAIL(aggregate_charsets_for_comparison(type.get_calc_meta(), - coll_types, 2, type_ctx.get_coll_type()))) { + if (OB_FAIL(aggregate_charsets_for_comparison(type.get_calc_meta(), coll_types, 2, type_ctx))) { LOG_WARN("failed to aggregate_charsets_for_comparison", K(ret)); } else { type1.set_calc_type(ObVarcharType); - type1.set_calc_collation_type(type.get_calc_collation_type()); + type1.set_calc_collation(type); type2.set_calc_type(ObVarcharType); - type2.set_calc_collation_type(type.get_calc_collation_type()); + type2.set_calc_collation(type); } } else { ret = OB_ERR_INVALID_TYPE_FOR_OP; diff --git a/src/sql/engine/expr/ob_expr_hex.cpp b/src/sql/engine/expr/ob_expr_hex.cpp index 970603ab6..6b6b158e1 100644 --- a/src/sql/engine/expr/ob_expr_hex.cpp +++ b/src/sql/engine/expr/ob_expr_hex.cpp @@ -92,7 +92,7 @@ int ObExprHex::calc_result_type1(ObExprResType &type, text.set_calc_type(ObVarcharType); } ObExprResType tmp_type; - OZ(aggregate_charsets_for_string_result(tmp_type, &text, 1, type_ctx.get_coll_type())); + OZ(aggregate_charsets_for_string_result(tmp_type, &text, 1, type_ctx)); if (OB_SUCC(ret)) { text.set_calc_collation_type(tmp_type.get_collation_type()); text.set_calc_collation_level(tmp_type.get_collation_level()); diff --git a/src/sql/engine/expr/ob_expr_ifnull.cpp b/src/sql/engine/expr/ob_expr_ifnull.cpp index 777f77352..861adbdaa 100644 --- a/src/sql/engine/expr/ob_expr_ifnull.cpp +++ b/src/sql/engine/expr/ob_expr_ifnull.cpp @@ -47,15 +47,13 @@ int ObExprIfNull::calc_result_type2(ObExprResType &type, } else if (OB_FAIL(ObExprPromotionUtil::get_nvl_type(type, type1, type2))) { LOG_WARN("failed to get nvl type", K(ret)); } else if (ob_is_string_type(type.get_type()) || ob_is_json_tc(type.get_type())) { - ObCollationLevel res_cs_level = CS_LEVEL_INVALID; - ObCollationType res_cs_type = CS_TYPE_INVALID; - if (OB_FAIL(ObCharset::aggregate_collation(type1.get_collation_level(), type1.get_collation_type(), - type2.get_collation_level(), type2.get_collation_type(), - res_cs_level, res_cs_type))) { - LOG_WARN("failed to calc collation", K(ret)); - } else { - type.set_collation_level(res_cs_level); - type.set_collation_type(res_cs_type); + ObExprResTypes res_types; + if (OB_FAIL(res_types.push_back(type1))) { + LOG_WARN("fail to push back res type", K(ret)); + } else if (OB_FAIL(res_types.push_back(type2))) { + LOG_WARN("fail to push back res type", K(ret)); + } else if (OB_FAIL(aggregate_charsets_for_string_result(type, &res_types.at(0), 2, type_ctx))) { + LOG_WARN("failed to aggregate_charsets_for_comparison", K(ret)); } } else if (ob_is_roaringbitmap_tc(type.get_type())) { type.set_collation_level(CS_LEVEL_IMPLICIT); diff --git a/src/sql/engine/expr/ob_expr_insert.cpp b/src/sql/engine/expr/ob_expr_insert.cpp index f6daead3e..ca8b2cb1b 100644 --- a/src/sql/engine/expr/ob_expr_insert.cpp +++ b/src/sql/engine/expr/ob_expr_insert.cpp @@ -52,8 +52,7 @@ int ObExprInsert::calc_result_typeN(ObExprResType &type, LOG_WARN("fail push col", K(coll0), K(ret)); } else if (OB_FAIL(coll_types.push_back(coll3))) { LOG_WARN("fail push col", K(coll3), K(ret)); - } else if (OB_FAIL(aggregate_charsets_for_string_result( - type, &coll_types.at(0), 2, type_ctx.get_coll_type()))) { + } else if (OB_FAIL(aggregate_charsets_for_string_result(type, &coll_types.at(0), 2, type_ctx))) { LOG_WARN("aggregate cahrset for string result failed", K(ret)); } else { types_array[0].set_calc_type(ObVarcharType); diff --git a/src/sql/engine/expr/ob_expr_json_array_append.cpp b/src/sql/engine/expr/ob_expr_json_array_append.cpp index 7f63149fc..5d359a29f 100644 --- a/src/sql/engine/expr/ob_expr_json_array_append.cpp +++ b/src/sql/engine/expr/ob_expr_json_array_append.cpp @@ -16,6 +16,7 @@ #include "ob_expr_json_array_append.h" #include "share/ob_json_access_utils.h" #include "sql/engine/expr/ob_expr_json_func_helper.h" +#include "sql/engine/ob_exec_context.h" using namespace oceanbase::common; using namespace oceanbase::sql; diff --git a/src/sql/engine/expr/ob_expr_least.cpp b/src/sql/engine/expr/ob_expr_least.cpp index 6d85396eb..add058883 100644 --- a/src/sql/engine/expr/ob_expr_least.cpp +++ b/src/sql/engine/expr/ob_expr_least.cpp @@ -244,36 +244,30 @@ int ObExprLeastGreatest::calc_result_typeN_mysql(ObExprResType &type, all_integer = false; } } - const ObLengthSemantics default_length_semantics = (OB_NOT_NULL(type_ctx.get_session()) - ? type_ctx.get_session()->get_actual_nls_length_semantics() : LS_BYTE); bool enable_decimalint = false; - if (OB_FAIL(ObSQLUtils::check_enable_decimalint(session, enable_decimalint))) { - LOG_WARN("fail to check_enable_decimalint", K(ret), K(session->get_effective_tenant_id())); - } else if (OB_FAIL(calc_result_meta_for_comparison( - type, types, param_num, - type_ctx.get_coll_type(), - default_length_semantics, - enable_decimalint))) { + if (OB_FAIL(calc_result_meta_for_comparison(type, types, param_num, type_ctx, enable_decimalint))) { LOG_WARN("calc result meta for comparison failed"); } - // can't cast origin parameters. - for (int64_t i = 0; i < param_num; i++) { - types[i].set_calc_meta(types[i].get_obj_meta()); - } - if (all_integer && type.is_integer_type()) { - type.set_calc_type(ObNullType); - } else if (all_integer && ob_is_number_or_decimal_int_tc(type.get_type())) { - // the args type is integer and result type is number/decimal, there are unsigned bigint in - // args, set expr meta here. - type.set_accuracy(common::ObAccuracy::DDL_DEFAULT_ACCURACY2[0/*is_oracle*/][ObUInt64Type]); - } else { + if (OB_SUCC(ret)) { + // can't cast origin parameters. for (int64_t i = 0; i < param_num; i++) { - if (ob_is_enum_or_set_type(types[i].get_type())) { - types[i].set_calc_type(type.get_calc_type()); + types[i].set_calc_meta(types[i].get_obj_meta()); + } + if (all_integer && type.is_integer_type()) { + type.set_calc_type(ObNullType); + } else if (all_integer && ob_is_number_or_decimal_int_tc(type.get_type())) { + // the args type is integer and result type is number/decimal, there are unsigned bigint in + // args, set expr meta here. + type.set_accuracy(common::ObAccuracy::DDL_DEFAULT_ACCURACY2[0/*is_oracle*/][ObUInt64Type]); + } else { + for (int64_t i = 0; i < param_num; i++) { + if (ob_is_enum_or_set_type(types[i].get_type())) { + types[i].set_calc_type(type.get_calc_type()); + } } } + LOG_DEBUG("least calc_result_typeN", K(type), K(type.get_calc_accuracy())); } - LOG_DEBUG("least calc_result_typeN", K(type), K(type.get_calc_accuracy())); } return ret; } diff --git a/src/sql/engine/expr/ob_expr_left.cpp b/src/sql/engine/expr/ob_expr_left.cpp index eddaae2a6..8593205ec 100644 --- a/src/sql/engine/expr/ob_expr_left.cpp +++ b/src/sql/engine/expr/ob_expr_left.cpp @@ -97,9 +97,10 @@ int ObExprLeft::calc_result_type2(ObExprResType &type, type.set_length(int_obj.get_int()); } - OZ(aggregate_charsets_for_string_result(type, &type1, 1, type_ctx.get_coll_type())); + OZ(aggregate_charsets_for_string_result(type, &type1, 1, type_ctx)); OX(type1.set_calc_type(ObVarcharType)); OX(type1.set_calc_collation_type(type.get_collation_type())); + OX(type1.set_calc_collation_level(type.get_collation_level())); } return ret; } diff --git a/src/sql/engine/expr/ob_expr_like.cpp b/src/sql/engine/expr/ob_expr_like.cpp index 6dfd138ff..9023ce830 100644 --- a/src/sql/engine/expr/ob_expr_like.cpp +++ b/src/sql/engine/expr/ob_expr_like.cpp @@ -293,10 +293,12 @@ int ObExprLike::calc_result_type3(ObExprResType &type, type.set_calc_collation_type(type_ctx.get_session()->get_nls_collation()); } } else { - ret = aggregate_charsets_for_comparison(type.get_calc_meta(), types, 2, type_ctx.get_coll_type()); + ret = aggregate_charsets_for_comparison(type.get_calc_meta(), types, 2, type_ctx); } type1.set_calc_collation_type(type.get_calc_collation_type()); type2.set_calc_collation_type(type.get_calc_collation_type()); + type1.set_calc_collation_level(type.get_calc_collation_level()); + type2.set_calc_collation_level(type.get_calc_collation_level()); ObExprOperator::calc_result_flag2(type, type1, type2); // ESCAPE is ignored } return ret; diff --git a/src/sql/engine/expr/ob_expr_lower.cpp b/src/sql/engine/expr/ob_expr_lower.cpp index eea89d615..743aac6cd 100644 --- a/src/sql/engine/expr/ob_expr_lower.cpp +++ b/src/sql/engine/expr/ob_expr_lower.cpp @@ -77,8 +77,9 @@ int ObExprLowerUpper::calc_result_type1(ObExprResType &type, ObExprResType &text const common::ObLengthSemantics default_length_semantics = (OB_NOT_NULL(type_ctx.get_session()) ? type_ctx.get_session()->get_actual_nls_length_semantics() : common::LS_BYTE); - ret = aggregate_charsets_for_string_result(type, &text, 1, type_ctx.get_coll_type()); + ret = aggregate_charsets_for_string_result(type, &text, 1, type_ctx); OX(text.set_calc_collation_type(type.get_collation_type())); + OX(text.set_calc_collation_level(type.get_collation_level())); OX(type.set_length(text.get_length())); } } diff --git a/src/sql/engine/expr/ob_expr_lrpad.cpp b/src/sql/engine/expr/ob_expr_lrpad.cpp index 0c0fc2785..a685aeab6 100644 --- a/src/sql/engine/expr/ob_expr_lrpad.cpp +++ b/src/sql/engine/expr/ob_expr_lrpad.cpp @@ -269,8 +269,7 @@ int ObExprBaseLRpad::calc_type(ObExprResType &type, ObSEArray types; OZ(types.push_back(text)); OZ(types.push_back(*pad_text)); - OZ(aggregate_charsets_for_string_result(type, &types.at(0), 2, - type_ctx.get_coll_type())); + OZ(aggregate_charsets_for_string_result(type, &types.at(0), 2, type_ctx)); OX(text.set_calc_collation_type(type.get_collation_type())); OX(pad_text->set_calc_collation_type(type.get_collation_type())); if (OB_SUCC(ret)) { diff --git a/src/sql/engine/expr/ob_expr_make_set.cpp b/src/sql/engine/expr/ob_expr_make_set.cpp index aafa21f0b..5d5570244 100644 --- a/src/sql/engine/expr/ob_expr_make_set.cpp +++ b/src/sql/engine/expr/ob_expr_make_set.cpp @@ -53,11 +53,12 @@ int ObExprMakeSet::calc_result_typeN(ObExprResType &type, // set expected type of results type.set_varchar(); type.set_length(max_len); - if OB_FAIL(aggregate_charsets_for_string_result(type, &types[1], param_num - 1, type_ctx.get_coll_type())) { + if OB_FAIL(aggregate_charsets_for_string_result(type, &types[1], param_num - 1, type_ctx)) { LOG_WARN("aggregate charset for string result failed", K(ret)); } else { for (int64_t i = 1; i < param_num; i++) { types[i].set_calc_collation_type(type.get_collation_type()); + types[i].set_calc_collation_level(type.get_collation_level()); } } } diff --git a/src/sql/engine/expr/ob_expr_nullif.cpp b/src/sql/engine/expr/ob_expr_nullif.cpp index 8e22b8b3b..918615f28 100644 --- a/src/sql/engine/expr/ob_expr_nullif.cpp +++ b/src/sql/engine/expr/ob_expr_nullif.cpp @@ -86,7 +86,7 @@ int ObExprNullif::se_deduce_type(ObExprResType &type, } else if (ob_is_enumset_tc(type.get_type()) || ob_is_enumset_inner_tc(type.get_type())) { type.set_varchar(); } - OZ(calc_cmp_type2(cmp_type, type1, type2, type_ctx.get_coll_type())); + OZ(calc_cmp_type2(cmp_type, type1, type2, type_ctx)); if (OB_SUCC(ret)) { if (ob_is_enumset_tc(type1.get_type()) || ob_is_enumset_tc(type2.get_type())) { ObObjType calc_type = enumset_calc_types_[OBJ_TYPE_TO_CLASS[cmp_type.get_calc_type()]]; @@ -102,6 +102,7 @@ int ObExprNullif::se_deduce_type(ObExprResType &type, // implicit cast from enum_inner to uint64 will be added on EnumToInner expression. type1.set_calc_type(calc_type); type1.set_calc_collation_type(cmp_type.get_calc_collation_type()); + type1.set_calc_collation_level(cmp_type.get_calc_collation_level()); } } if (type1.is_decimal_int()) { @@ -110,6 +111,7 @@ int ObExprNullif::se_deduce_type(ObExprResType &type, // set calc type for type2 no matter whether calc_type is varchar or not, and no matther which param is enum. type2.set_calc_type(calc_type); type2.set_calc_collation_type(cmp_type.get_calc_collation_type()); + type2.set_calc_collation_level(cmp_type.get_calc_collation_level()); } } if (OB_FAIL(ret)) { diff --git a/src/sql/engine/expr/ob_expr_nvl.cpp b/src/sql/engine/expr/ob_expr_nvl.cpp index 19e34258f..3cc08c858 100644 --- a/src/sql/engine/expr/ob_expr_nvl.cpp +++ b/src/sql/engine/expr/ob_expr_nvl.cpp @@ -27,7 +27,6 @@ namespace sql { int ObExprNvlUtil::calc_result_type(ObExprResType &type, - ObExprResType &type1, ObExprResType &type2, ObExprTypeCtx &type_ctx) @@ -49,10 +48,16 @@ int ObExprNvlUtil::calc_result_type(ObExprResType &type, res_cs_type = type_ctx.get_session()->get_dtc_params().nls_collation_; } } else { - if (OB_FAIL(ObCharset::aggregate_collation(type1.get_collation_level(), type1.get_collation_type(), - type2.get_collation_level(), type2.get_collation_type(), - res_cs_level, res_cs_type))) { - LOG_WARN("aggregate collation failed", K(ret), K(type1), K(type2)); + ObExprResTypes res_types; + if (OB_FAIL(res_types.push_back(type1))) { + LOG_WARN("fail to push back res type", K(ret)); + } else if (OB_FAIL(res_types.push_back(type2))) { + LOG_WARN("fail to push back res type", K(ret)); + } else if (OB_FAIL(ObExprOperator::aggregate_charsets_for_comparison(type, &res_types.at(0), 2, type_ctx))) { + LOG_WARN("failed to aggregate_charsets_for_comparison", K(ret)); + } else { + res_cs_type = type.get_calc_collation_type(); + res_cs_level = type.get_calc_collation_level(); } } if (OB_SUCC(ret)) { diff --git a/src/sql/engine/expr/ob_expr_operator.cpp b/src/sql/engine/expr/ob_expr_operator.cpp index 79cc01b3c..7543f9c53 100644 --- a/src/sql/engine/expr/ob_expr_operator.cpp +++ b/src/sql/engine/expr/ob_expr_operator.cpp @@ -717,16 +717,31 @@ int ObExprOperator::aggregate_collations(ObObjMeta &type, } else { ObCollationType coll_type = types[0].get_collation_type(); ObCollationLevel coll_level = types[0].get_collation_level(); + bool unknown_cs = false; for (int64_t i = 1; OB_SUCC(ret) && i < param_num; ++i) { - ret = ObCharset::aggregate_collation(coll_level, coll_type, - types[i].get_collation_level(), - types[i].get_collation_type(), - coll_level, coll_type); + ret = aggregate_two_collation(coll_level, coll_type, + types[i].get_collation_level(), + types[i].get_collation_type(), + coll_level, coll_type, flags); + if (OB_SUCC(ret) && + coll_type == CS_TYPE_BINARY && + coll_level == CS_LEVEL_NONE) { + unknown_cs = true; + } + } + if (OB_SUCC(ret)) { + if ((flags & OB_COLL_ALLOW_NEW_CONV) && + unknown_cs && + coll_level != CS_LEVEL_EXPLICIT) { + ret = OB_CANT_AGGREGATE_2COLLATIONS; + LOG_WARN("Illegal mix of collations",K(ret), K(unknown_cs)); + } } if (OB_SUCC(ret)) { if (OB_UNLIKELY((flags & OB_COLL_DISALLOW_NONE) && CS_LEVEL_NONE == coll_level)) { // @todo (zhuweng.yzf) correct error code is OB_CANT_AGGREGATE_NCOLLATIONS ret = OB_CANT_AGGREGATE_2COLLATIONS; + LOG_WARN("Illegal mix of collations",K(ret), K(flags),K(coll_type), K(coll_level)); } } if (OB_SUCC(ret)) { @@ -736,55 +751,156 @@ int ObExprOperator::aggregate_collations(ObObjMeta &type, coll_level = CS_LEVEL_COERCIBLE; // MySQL need charset converter here. We consider only two charset(binary // and utf8mb4), so no conversion is actually needed - } else {} + } if (OB_SUCC(ret)) { type.set_collation_type(coll_type); type.set_collation_level(coll_level); - } else {} - } else {} + } + } if (OB_CANT_AGGREGATE_2COLLATIONS == ret) { if (3 == param_num) { ret = OB_CANT_AGGREGATE_3COLLATIONS; + const char* coll_type0 = ObCharset::collation_name(types[0].get_collation_type()); + const char* coll_level0 = ObCharset::collation_level(types[0].get_collation_level()); + const char* coll_type1 = ObCharset::collation_name(types[1].get_collation_type()); + const char* coll_level1 = ObCharset::collation_level(types[1].get_collation_level()); + const char* coll_type2 = ObCharset::collation_name(types[2].get_collation_type()); + const char* coll_level2 = ObCharset::collation_level(types[2].get_collation_level()); + LOG_USER_ERROR(OB_CANT_AGGREGATE_3COLLATIONS, coll_type0, + coll_level0, + coll_type1, + coll_level1, + coll_type2, + coll_level2); } else if (3 < param_num) { ret = OB_CANT_AGGREGATE_NCOLLATIONS; + } else { + const char* coll_type0 = ObCharset::collation_name(types[0].get_collation_type()); + const char* coll_level0 = ObCharset::collation_level(types[0].get_collation_level()); + const char* coll_type1 = ObCharset::collation_name(types[1].get_collation_type()); + const char* coll_level1 = ObCharset::collation_level(types[1].get_collation_level()); + LOG_USER_ERROR(OB_CANT_AGGREGATE_2COLLATIONS, coll_type0, + coll_level0, + coll_type1, + coll_level1); + } + LOG_WARN("Illegal mix of collations",K(ret),K(param_num), K(coll_type), K(coll_level)); + for (int64_t i = 0; OB_SUCC(ret) && i < param_num; ++i) { + LOG_WARN("Illegal mix of collations", K(ret),K(i), + "type", ObCharset::collation_name(types[i].get_collation_type()), + "level", ObCharset::collation_level(types[i].get_collation_level()) ); } } } return ret; } +int ObExprOperator::aggregate_two_collation(const ObCollationLevel level1, + const ObCollationType type1, + const ObCollationLevel level2, + const ObCollationType type2, + ObCollationLevel &res_level, + ObCollationType &res_type, + uint32_t flags) +{ + int ret = OB_SUCCESS; + if (flags & OB_COLL_ALLOW_NEW_CONV) { + ret = ObCharset::aggregate_collation_new(level1, type1, + level2, type2, + res_level, res_type, + flags); + } else { + ret = ObCharset::aggregate_collation_old(level1, type1, + level2, type2, + res_level, res_type); + } + if (OB_CANT_AGGREGATE_2COLLATIONS == ret) { + const char* coll_type1 = ObCharset::collation_name(type1); + const char* coll_level1 = ObCharset::collation_level(level1); + const char* coll_type2 = ObCharset::collation_name(type2); + const char* coll_level2 = ObCharset::collation_level(level2); + LOG_USER_ERROR(OB_CANT_AGGREGATE_2COLLATIONS, coll_type1, + coll_level1, + coll_type2, + coll_level2); + } + LOG_TRACE("aggregate_two_collation debug",K(ret), K(flags), K(res_type), K(res_level), + K(type1), K(level1), K(type2), K(level2)); + return ret; +} -// We consider only two charset: binary and utf8mb4, so no conversion is actually needed - +int ObExprOperator::enable_old_charset_aggregation(const ObBasicSessionInfo *session, uint32_t &flags) +{ + int ret = OB_SUCCESS; + bool enable_old_rule = false; + bool version_not_allow = ((GET_MIN_CLUSTER_VERSION() < MOCK_CLUSTER_VERSION_4_2_4_0) || + (CLUSTER_VERSION_4_3_0_0 <= GET_MIN_CLUSTER_VERSION() && (GET_MIN_CLUSTER_VERSION() < CLUSTER_VERSION_4_3_3_0))); + if (version_not_allow) { + flags &= ~OB_COLL_ALLOW_NEW_CONV; + LOG_TRACE("old charset aggregation rule is enabled because version is less then 424", K(version_not_allow)); + } else if (OB_ISNULL(session)) { + LOG_TRACE("use new charset aggregation rule"); + } else if (OB_FAIL(session->is_old_charset_aggregation_enabled(enable_old_rule))) { + LOG_WARN("failed to check is_old_charset_aggregation_enabled", K(ret)); + } else if (enable_old_rule) { + flags &= ~OB_COLL_ALLOW_NEW_CONV; + LOG_TRACE("old charset aggregation rule is enabled because variables control", K(enable_old_rule)); + } + return ret; +} int ObExprOperator::aggregate_charsets_for_string_result( ObObjMeta &type, const ObObjMeta *types, int64_t param_num, - const ObCollationType conn_coll_type) + common::ObExprTypeCtx &type_ctx) { - uint32_t flags = OB_COLL_ALLOW_NUMERIC_CONV; - return aggregate_charsets(type, types, param_num, flags, conn_coll_type); + int ret = OB_SUCCESS; + bool enable_old_rule = false; + uint32_t flags = OB_COLL_ALLOW_SUPERSET_CONV | OB_COLL_ALLOW_COERCIBLE_CONV | + OB_COLL_ALLOW_NUMERIC_CONV | OB_COLL_ALLOW_NEW_CONV; + if (OB_FAIL(enable_old_charset_aggregation(type_ctx.get_session(), flags))) { + LOG_WARN("failed to check is_old_charset_aggregation_enabled", K(ret)); + } else { + ret = aggregate_charsets(type, types, param_num, flags, type_ctx.get_coll_type()); + } + return ret; } int ObExprOperator::aggregate_charsets_for_string_result( ObExprResType &type, const ObExprResType *types, int64_t param_num, - const ObCollationType conn_coll_type) + common::ObExprTypeCtx &type_ctx) { - uint32_t flags = OB_COLL_ALLOW_NUMERIC_CONV; - return aggregate_charsets(type, types, param_num, flags, conn_coll_type); + int ret = OB_SUCCESS; + bool enable_old_rule = false; + uint32_t flags = OB_COLL_ALLOW_SUPERSET_CONV | OB_COLL_ALLOW_COERCIBLE_CONV | + OB_COLL_ALLOW_NUMERIC_CONV | OB_COLL_ALLOW_NEW_CONV; + if (OB_FAIL(enable_old_charset_aggregation(type_ctx.get_session(), flags))) { + LOG_WARN("failed to check is_old_charset_aggregation_enabled", K(ret)); + } else { + ret = aggregate_charsets(type, types, param_num, flags, type_ctx.get_coll_type()); + } + return ret; } int ObExprOperator::aggregate_charsets_for_comparison( ObObjMeta &type, const ObObjMeta *types, int64_t param_num, - const ObCollationType conn_coll_type) + common::ObExprTypeCtx &type_ctx) { - uint32_t flags = OB_COLL_DISALLOW_NONE; - return aggregate_charsets(type, types, param_num, flags, conn_coll_type); + int ret = OB_SUCCESS; + bool enable_old_rule = false; + uint32_t flags = OB_COLL_ALLOW_SUPERSET_CONV | OB_COLL_ALLOW_COERCIBLE_CONV | + OB_COLL_DISALLOW_NONE | OB_COLL_ALLOW_NEW_CONV; + if (OB_FAIL(enable_old_charset_aggregation(type_ctx.get_session(), flags))) { + LOG_WARN("failed to check is_old_charset_aggregation_enabled", K(ret)); + } else { + ret = aggregate_charsets(type, types, param_num, flags, type_ctx.get_coll_type()); + } + return ret; } @@ -792,30 +908,56 @@ int ObExprOperator::aggregate_charsets_for_comparison( ObExprResType &type, const ObExprResType *types, int64_t param_num, - const ObCollationType conn_coll_type) + common::ObExprTypeCtx &type_ctx) { - uint32_t flags = OB_COLL_DISALLOW_NONE; - return aggregate_charsets(type.get_calc_meta(), types, param_num, flags, conn_coll_type); + int ret = OB_SUCCESS; + bool enable_old_rule = false; + uint32_t flags = OB_COLL_ALLOW_SUPERSET_CONV | OB_COLL_ALLOW_COERCIBLE_CONV | + OB_COLL_DISALLOW_NONE | OB_COLL_ALLOW_NEW_CONV; + if (OB_FAIL(enable_old_charset_aggregation(type_ctx.get_session(), flags))) { + LOG_WARN("failed to check is_old_charset_aggregation_enabled", K(ret)); + } else { + ret = aggregate_charsets(type.get_calc_meta(), types, param_num, flags, type_ctx.get_coll_type()); + } + return ret; } int ObExprOperator::aggregate_charsets_for_string_result_with_comparison( ObObjMeta &type, const ObObjMeta *types, int64_t param_num, - const ObCollationType conn_coll_type) + common::ObExprTypeCtx &type_ctx) { - uint32_t flags = OB_COLL_DISALLOW_NONE | OB_COLL_ALLOW_NUMERIC_CONV; - return aggregate_charsets(type, types, param_num, flags, conn_coll_type); + int ret = OB_SUCCESS; + bool enable_old_rule = false; + uint32_t flags = OB_COLL_ALLOW_SUPERSET_CONV | OB_COLL_ALLOW_COERCIBLE_CONV | + OB_COLL_ALLOW_NUMERIC_CONV | OB_COLL_ALLOW_NEW_CONV | + OB_COLL_DISALLOW_NONE; + if (OB_FAIL(enable_old_charset_aggregation(type_ctx.get_session(), flags))) { + LOG_WARN("failed to check is_old_charset_aggregation_enabled", K(ret)); + } else { + ret = aggregate_charsets(type, types, param_num, flags, type_ctx.get_coll_type()); + } + return ret; } int ObExprOperator::aggregate_charsets_for_string_result_with_comparison( ObObjMeta &type, const ObExprResType *types, int64_t param_num, - const ObCollationType coll_type) + common::ObExprTypeCtx &type_ctx) { - uint32_t flags = OB_COLL_DISALLOW_NONE | OB_COLL_ALLOW_NUMERIC_CONV; - return aggregate_charsets(type, types, param_num, flags, coll_type); + int ret = OB_SUCCESS; + bool enable_old_rule = false; + uint32_t flags = OB_COLL_ALLOW_SUPERSET_CONV | OB_COLL_ALLOW_COERCIBLE_CONV | + OB_COLL_ALLOW_NUMERIC_CONV | OB_COLL_ALLOW_NEW_CONV | + OB_COLL_DISALLOW_NONE; + if (OB_FAIL(enable_old_charset_aggregation(type_ctx.get_session(), flags))) { + LOG_WARN("failed to check is_old_charset_aggregation_enabled", K(ret)); + } else { + ret = aggregate_charsets(type, types, param_num, flags, type_ctx.get_coll_type()); + } + return ret; } int ObExprOperator::aggregate_charsets( @@ -1080,9 +1222,8 @@ int ObExprOperator::aggregate_result_type_for_case( ObExprResType &type, const ObExprResType *types, int64_t param_num, - const ObCollationType conn_coll_type, bool is_oracle_mode, - const ObLengthSemantics default_length_semantics, + common::ObExprTypeCtx &type_ctx, bool need_merge_type, bool skip_null, bool is_called_in_sql) @@ -1129,8 +1270,8 @@ int ObExprOperator::aggregate_result_type_for_case( if (OB_FAIL(aggregate_numeric_accuracy_for_merge(type, types, param_num, is_oracle_mode))) { LOG_WARN("fail to aggregate numeric accuracy", K(ret)); } - } else if (OB_FAIL(aggregate_result_type_for_merge(type, types, param_num, conn_coll_type, - is_oracle_mode, default_length_semantics, need_merge_type, skip_null, + } else if (OB_FAIL(aggregate_result_type_for_merge(type, types, param_num, + is_oracle_mode, type_ctx, need_merge_type, skip_null, is_called_in_sql))) { LOG_WARN("fail to aggregate result type", K(ret)); } else if (ObFloatType == type.get_type() && !is_oracle_mode) { @@ -1144,9 +1285,8 @@ int ObExprOperator::aggregate_result_type_for_merge( ObExprResType &type, const ObExprResType *types, int64_t param_num, - const ObCollationType conn_coll_type, bool is_oracle_mode, - const ObLengthSemantics default_length_semantics, + common::ObExprTypeCtx &type_ctx, bool need_merge_type, bool skip_null, bool is_called_in_sql) @@ -1159,11 +1299,13 @@ int ObExprOperator::aggregate_result_type_for_merge( //this is for case when clause like case when 1 then c1 end; type.set_type(ObVarcharType); type.set_collation_level(common::CS_LEVEL_IMPLICIT); - type.set_collation_type(conn_coll_type); + type.set_collation_type(type_ctx.get_coll_type()); } else { ObObjType res_type = types[0].get_type(); - bool is_oracle_all_same_number = is_oracle_mode && - ob_is_number_or_decimal_int_tc(types[0].get_type()); + bool is_oracle_all_same_number = is_oracle_mode && ob_is_number_or_decimal_int_tc(types[0].get_type()); + const ObLengthSemantics default_length_semantics = ((OB_NOT_NULL(type_ctx.get_session())) ? + type_ctx.get_session()->get_actual_nls_length_semantics() : LS_BYTE); + for (int64_t i = 1; OB_SUCC(ret) && i < param_num; ++i) { if (OB_FAIL(ObExprResultTypeUtil::get_merge_result_type(res_type, res_type, @@ -1200,7 +1342,7 @@ int ObExprOperator::aggregate_result_type_for_merge( } else if (ob_is_temporal_type(res_type) || ob_is_otimestamp_type(res_type)) { ret = aggregate_temporal_accuracy_for_merge(type, types, param_num); } else if (ob_is_string_or_lob_type(res_type)) { - if (OB_FAIL(aggregate_charsets_for_string_result(type, types, param_num, conn_coll_type))) { + if (OB_FAIL(aggregate_charsets_for_string_result(type, types, param_num, type_ctx))) { } else if (OB_FAIL(aggregate_max_length_for_string_result(type, types, param_num, is_oracle_mode, default_length_semantics, need_merge_type, skip_null, is_called_in_sql))) { @@ -2059,13 +2201,14 @@ int ObRelationalExprOperator::get_equal_meta(ObObjMeta &meta, { int ret = OB_SUCCESS; ObObjType type = ObMaxType; + ObExprTypeCtx type_ctx; if (OB_FAIL(ObExprResultTypeUtil::get_relational_equal_type(type, meta1.get_type(), meta2.get_type()))) { LOG_WARN("get equal type failed", K(ret), K(meta1), K(meta2)); } else if (ob_is_string_or_lob_type(type)) { ObObjMeta coll_types[2] = {meta1, meta2}; - ret = aggregate_charsets_for_comparison(meta, coll_types, 2, CS_TYPE_INVALID); + ret = aggregate_charsets_for_comparison(meta, coll_types, 2, type_ctx); } if (OB_SUCC(ret)) { meta.set_type(type); @@ -2078,7 +2221,7 @@ int ObRelationalExprOperator::get_equal_meta(ObObjMeta &meta, int ObExprOperator::calc_cmp_type2(ObExprResType &type, const ObExprResType &type1, const ObExprResType &type2, - const ObCollationType coll_type, + common::ObExprTypeCtx &type_ctx, const bool left_is_const, /* false */ const bool right_is_const /* false*/) const { @@ -2187,7 +2330,7 @@ int ObExprOperator::calc_cmp_type2(ObExprResType &type, ObObjMeta coll_types[2]; coll_types[0].set_collation(type1); coll_types[1].set_collation(type2); - ret = aggregate_charsets_for_comparison(type.get_calc_meta(), coll_types, 2, coll_type); + ret = aggregate_charsets_for_comparison(type.get_calc_meta(), coll_types, 2, type_ctx); if (OB_FAIL(ret)) { LOG_WARN("aggregate charset failed", K(type1), K(type2), K(type.get_calc_meta())); } @@ -2203,7 +2346,7 @@ int ObExprOperator::calc_cmp_type3(ObExprResType &type, const ObExprResType &type1, const ObExprResType &type2, const ObExprResType &type3, - const ObCollationType coll_type) const + common::ObExprTypeCtx &type_ctx) const { int ret = OB_SUCCESS; // cmp type @@ -2231,7 +2374,7 @@ int ObExprOperator::calc_cmp_type3(ObExprResType &type, coll_types[0].set_collation(type1); coll_types[1].set_collation(type2); coll_types[2].set_collation(type3); - ret = aggregate_charsets_for_comparison(type.get_calc_meta(), coll_types, 3, coll_type); + ret = aggregate_charsets_for_comparison(type.get_calc_meta(), coll_types, 3, type_ctx); } else if (ObRawType == cmp_type) { type.get_calc_meta().set_collation_type(CS_TYPE_BINARY); } @@ -2440,7 +2583,7 @@ int ObRelationalExprOperator::deduce_cmp_type(const ObExprOperator &expr, right_param = op_expr->get_param_expr(1); } if (OB_FAIL(ret)) { - } else if (OB_SUCC(expr.calc_cmp_type2(cmp_type, type1, type2, type_ctx.get_coll_type(), + } else if (OB_SUCC(expr.calc_cmp_type2(cmp_type, type1, type2, type_ctx, left_param->is_static_const_expr(), right_param->is_static_const_expr()))) { type.set_int32(); // not tinyint, compatible with MySQL @@ -2467,8 +2610,8 @@ int ObRelationalExprOperator::deduce_cmp_type(const ObExprOperator &expr, } LOG_DEBUG("decimalint vs number compare", K(type1), K(type2)); } else if (ob_is_string_or_lob_type(cmp_type.get_calc_type())) { - type1.set_calc_collation_type(cmp_type.get_calc_collation_type()); - type2.set_calc_collation_type(cmp_type.get_calc_collation_type()); + type1.set_calc_collation(cmp_type); + type2.set_calc_collation(cmp_type); } else if (ObRawType == cmp_type.get_calc_type()) { type1.set_calc_collation_type(CS_TYPE_BINARY); type2.set_calc_collation_type(CS_TYPE_BINARY); @@ -2500,7 +2643,7 @@ int ObRelationalExprOperator::calc_result_type3(ObExprResType &type, int ret = OB_SUCCESS; ObExprResType cmp_type; - if (OB_SUCC(calc_cmp_type3(cmp_type, type1, type2, type3, type_ctx.get_coll_type()))) { + if (OB_SUCC(calc_cmp_type3(cmp_type, type1, type2, type3, type_ctx))) { type.set_int32(); type.set_calc_collation(cmp_type); type.set_calc_type(cmp_type.get_calc_type()); @@ -2511,9 +2654,9 @@ int ObRelationalExprOperator::calc_result_type3(ObExprResType &type, LOG_WARN("set calc type failed", K(type1), K(type2), K(type3), K(cmp_type), K(ret)); } if (ob_is_string_or_lob_type(cmp_type.get_calc_type())) { - type1.set_calc_collation_type(cmp_type.get_calc_collation_type()); - type2.set_calc_collation_type(cmp_type.get_calc_collation_type()); - type3.set_calc_collation_type(cmp_type.get_calc_collation_type()); + type1.set_calc_collation(cmp_type); + type2.set_calc_collation(cmp_type); + type3.set_calc_collation(cmp_type); } else if (ObRawType == cmp_type.get_calc_type()) { type1.set_calc_collation_type(CS_TYPE_BINARY); type2.set_calc_collation_type(CS_TYPE_BINARY); @@ -2554,8 +2697,8 @@ int ObRelationalExprOperator::calc_result_typeN(ObExprResType &type, LOG_WARN("failed to push back cmp type", K(ret)); } else { if (ob_is_string_or_lob_type(tmp_res_type.get_calc_type())) { - types[i].set_calc_collation_type(tmp_res_type.get_calc_collation_type()); - types[i + row_dimension_].set_calc_collation_type(tmp_res_type.get_calc_collation_type()); + types[i].set_calc_collation(tmp_res_type); + types[i + row_dimension_].set_calc_collation(tmp_res_type); } else if (ObRawType == tmp_res_type.get_calc_type()) { types[i].set_calc_collation_type(CS_TYPE_BINARY); types[i + row_dimension_].set_calc_collation_type(CS_TYPE_BINARY); @@ -2588,9 +2731,9 @@ int ObRelationalExprOperator::calc_calc_type3(ObExprResType &type1, CK(NULL != type_ctx.get_session()); CK(NULL != raw_expr_); if (OB_FAIL(ret)) { - } else if (OB_FAIL(calc_cmp_type2(cmp_type21, type2, type1, type_ctx.get_coll_type()))) { + } else if (OB_FAIL(calc_cmp_type2(cmp_type21, type2, type1, type_ctx))) { LOG_WARN("get cmp failed", K(ret), K(type2), K(type1)); - } else if (OB_FAIL(calc_cmp_type2(cmp_type13, type1, type3, type_ctx.get_coll_type()))) { + } else if (OB_FAIL(calc_cmp_type2(cmp_type13, type1, type3, type_ctx))) { LOG_WARN("get cmp failed", K(ret), K(type1), K(type3)); } else { ObObjType type21 = cmp_type21.get_calc_type(); @@ -2676,10 +2819,12 @@ int ObRelationalExprOperator::calc_calc_type3(ObExprResType &type1, int ObRelationalExprOperator::get_cmp_result_type3(ObExprResType &type, bool &need_no_cast, ObExprResType *types, const int64_t param_num, const bool has_lower, - const sql::ObSQLSessionInfo &my_session) + common::ObExprTypeCtx &type_ctx) { int ret = OB_SUCCESS; need_no_cast = false; + const sql::ObSQLSessionInfo *my_session = NULL; + CK (OB_NOT_NULL(my_session = type_ctx.get_session())); if (OB_ISNULL(types) || OB_UNLIKELY(param_num < 1) || OB_UNLIKELY(param_num > 3)) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("types is null or param_num is wrong", K(types), K(param_num), K(ret)); @@ -2689,12 +2834,12 @@ int ObRelationalExprOperator::get_cmp_result_type3(ObExprResType &type, bool &ne need_no_cast = true; } else { ObCollationType coll_type = CS_TYPE_INVALID; - if (OB_FAIL(my_session.get_collation_connection(coll_type))) { + if (OB_FAIL(my_session->get_collation_connection(coll_type))) { LOG_WARN("fail to get_collation_connection", K(ret)); } else if (2 == param_num) { need_no_cast = can_cmp_without_cast(types[1], types[0], CO_CMP); if (!need_no_cast) { - if (OB_FAIL(calc_cmp_type2(type, types[0], types[1], coll_type))) { + if (OB_FAIL(calc_cmp_type2(type, types[0], types[1], type_ctx))) { LOG_WARN("calc_cmp_type2 failed", K(ret)); } else if (type.get_calc_type() == ObDecimalIntType) { if (types[0].is_decimal_int()) { @@ -2729,11 +2874,11 @@ int ObRelationalExprOperator::get_cmp_result_type3(ObExprResType &type, bool &ne //a(type1) between b(type2) and c(type3) <==> b<=a && a <=c ObExprResType cmp_type21; // b <= a ObExprResType cmp_type13; // a <= c - if (OB_FAIL(calc_cmp_type3(type, type1, type2, type3, coll_type))) { + if (OB_FAIL(calc_cmp_type3(type, type1, type2, type3, type_ctx))) { LOG_WARN("fail to calc_cmp_type3", K(ret)); - } else if (OB_FAIL(calc_cmp_type2(cmp_type21, type2, type1, coll_type))) { + } else if (OB_FAIL(calc_cmp_type2(cmp_type21, type2, type1, type_ctx))) { LOG_WARN("get cmp failed", K(ret), K(type2), K(type1)); - } else if (OB_FAIL(calc_cmp_type2(cmp_type13, type1, type3, coll_type))) { + } else if (OB_FAIL(calc_cmp_type2(cmp_type13, type1, type3, type_ctx))) { LOG_WARN("get cmp failed", K(ret), K(type1), K(type3)); } else if (cmp_type21.get_calc_type() == cmp_type13.get_calc_type() && cmp_type13.get_calc_type() == type.get_calc_type()) { @@ -2745,7 +2890,7 @@ int ObRelationalExprOperator::get_cmp_result_type3(ObExprResType &type, bool &ne if (OB_SUCC(ret)) { ObExprTypeCtx dummy_ctx; dummy_ctx.set_raw_expr(raw_expr_); - ObSQLUtils::init_type_ctx(&my_session, dummy_ctx); + ObSQLUtils::init_type_ctx(my_session, dummy_ctx); ObExprResType dummy_res; if (OB_FAIL(calc_result_type3(dummy_res, type1, type2, type3, dummy_ctx))) { LOG_WARN("calc_result_type3 failed", K(ret)); @@ -3190,8 +3335,8 @@ int ObSubQueryRelationalExpr::calc_result_typeN(ObExprResType &type, OZ(type.get_row_calc_cmp_types().push_back(tmp_res_type.get_calc_meta())); if (OB_SUCC(ret)) { if (ob_is_string_type(tmp_res_type.get_calc_type())) { - types[i].set_calc_collation_type(tmp_res_type.get_calc_collation_type()); - types[i + row_dimension_].set_calc_collation_type(tmp_res_type.get_calc_collation_type()); + types[i].set_calc_collation(tmp_res_type); + types[i + row_dimension_].set_calc_collation(tmp_res_type); } else if (ObRawType == tmp_res_type.get_calc_type()) { types[i].set_calc_collation_type(CS_TYPE_BINARY); types[i + row_dimension_].set_calc_collation_type(CS_TYPE_BINARY); @@ -4133,7 +4278,7 @@ int ObSubQueryRelationalExpr::calc_result_type2_(ObExprResType &type, { int ret = OB_SUCCESS; ObExprResType cmp_type; - if (OB_SUCC(calc_cmp_type2(cmp_type, type1, type2, type_ctx.get_coll_type()))) { + if (OB_SUCC(calc_cmp_type2(cmp_type, type1, type2, type_ctx))) { type.set_tinyint(); type.set_calc_collation(cmp_type); type.set_calc_type(cmp_type.get_calc_type()); @@ -4345,7 +4490,7 @@ int ObVectorExprOperator::calc_result_type2_(ObExprResType &type, } else if (lib::is_oracle_mode() && (type1.is_json() || type2.is_json())) { ret = OB_ERR_INVALID_CMP_OP; LOG_USER_ERROR(OB_ERR_INVALID_CMP_OP); - } else if (OB_SUCC(calc_cmp_type2(cmp_type, type1, type2, type_ctx.get_coll_type(), left_is_const, + } else if (OB_SUCC(calc_cmp_type2(cmp_type, type1, type2, type_ctx, left_is_const, right_is_const))) { type.set_int(); // not tinyint, compatiable with MySQL type.set_calc_collation(cmp_type); @@ -4392,12 +4537,14 @@ int ObVectorExprOperator::calc_result_type2_(ObExprResType &type, LOG_WARN("unexpected failed", K(ret)); } } else { - type1.set_calc_collation_type(cmp_type.get_calc_collation_type()); - type2.set_calc_collation_type(cmp_type.get_calc_collation_type()); + type1.set_calc_collation(cmp_type); + type2.set_calc_collation(cmp_type); } } else if (ObRawType == cmp_type.get_calc_type()) { type1.set_calc_collation_type(CS_TYPE_BINARY); + type1.set_calc_collation_level(cmp_type.get_calc_collation_level()); type2.set_calc_collation_type(CS_TYPE_BINARY); + type2.set_calc_collation_level(cmp_type.get_calc_collation_level()); } else if (is_mysql_mode() && ob_is_double_tc(cmp_type.get_calc_type())) { if (ob_is_numeric_tc(type1.get_type_class()) && ob_is_numeric_tc(type2.get_type_class()) && SCALE_UNKNOWN_YET != type1.get_scale() && SCALE_UNKNOWN_YET != type2.get_scale()) { @@ -4406,10 +4553,14 @@ int ObVectorExprOperator::calc_result_type2_(ObExprResType &type, ObAccuracy calc_acc(precision, scale); type1.set_calc_accuracy(calc_acc); type2.set_calc_accuracy(calc_acc); + type1.set_calc_collation(cmp_type); + type2.set_calc_collation(cmp_type); } else { ObAccuracy calc_acc(PRECISION_UNKNOWN_YET, SCALE_UNKNOWN_YET); type1.set_calc_accuracy(calc_acc); type2.set_calc_accuracy(calc_acc); + type1.set_calc_collation(cmp_type); + type2.set_calc_collation(cmp_type); } } } @@ -5359,11 +5510,9 @@ int ObMinMaxExprOperator::calc_result_meta_for_comparison( ObExprResType &type, ObExprResType *types_stack, int64_t param_num, - const ObCollationType coll_type, - const ObLengthSemantics default_length_semantics, + common::ObExprTypeCtx &type_ctx, const bool enable_decimal_int) const { - UNUSED(default_length_semantics); int ret = OB_SUCCESS; int64_t i = 0; //bool all_string = true; @@ -5389,12 +5538,12 @@ int ObMinMaxExprOperator::calc_result_meta_for_comparison( || ob_is_text_tc(type.get_calc_type()); // compare collation if (OB_SUCC(ret) && string_cmp) { - ret = aggregate_charsets_for_comparison(type, types_stack, param_num, coll_type); + ret = aggregate_charsets_for_comparison(type, types_stack, param_num, type_ctx); } // result collation if (OB_SUCC(ret) && string_result) { - ret = aggregate_charsets_for_string_result(type, types_stack, param_num, coll_type); + ret = aggregate_charsets_for_string_result(type, types_stack, param_num, type_ctx); } if (OB_SUCC(ret)) { @@ -5753,9 +5902,11 @@ int ObLocationExprOperator::calc_result_type2(ObExprResType &type, type2.set_calc_type(ObVarcharType); type_ctx.set_cast_mode(type_ctx.get_cast_mode() | CM_STRING_INTEGER_TRUNC); ObObjMeta types[2] = {type1, type2}; - OZ(aggregate_charsets_for_comparison(type.get_calc_meta(), types, 2, type_ctx.get_coll_type())); + OZ(aggregate_charsets_for_comparison(type.get_calc_meta(), types, 2, type_ctx)); OX(type1.set_calc_collation_type(type.get_calc_collation_type())); + OX(type1.set_calc_collation_level(type.get_calc_collation_level())); OX(type2.set_calc_collation_type(type.get_calc_collation_type())); + OX(type2.set_calc_collation_level(type.get_calc_collation_level())); return ret; } diff --git a/src/sql/engine/expr/ob_expr_operator.h b/src/sql/engine/expr/ob_expr_operator.h index c59f56543..8df436ba3 100644 --- a/src/sql/engine/expr/ob_expr_operator.h +++ b/src/sql/engine/expr/ob_expr_operator.h @@ -515,29 +515,36 @@ public: common::ObObjMeta &type, const common::ObObjMeta *types, int64_t param_num, - const common::ObCollationType conn_coll_type); + common::ObExprTypeCtx &type_ctx); static int aggregate_charsets_for_comparison( ObExprResType &type, const ObExprResType *types, int64_t param_num, - const common::ObCollationType conn_coll_type); + common::ObExprTypeCtx &type_ctx); /* Aggregate arguments for string result, e.g: CONCAT(a,b) - convert to @@character_set_connection if all arguments are numbers - allow DERIVATION_NONE */ + static int enable_old_charset_aggregation(const ObBasicSessionInfo *session, uint32_t &flags); static int aggregate_charsets_for_string_result( common::ObObjMeta &type, const common::ObObjMeta *types, int64_t param_num, - const common::ObCollationType conn_coll_type); + common::ObExprTypeCtx &type_ctx); static int aggregate_charsets_for_string_result( ObExprResType &type, const ObExprResType *types, int64_t param_num, - const common::ObCollationType conn_coll_type); - + common::ObExprTypeCtx &type_ctx); + static int aggregate_two_collation(const ObCollationLevel level1, + const ObCollationType type1, + const ObCollationLevel level2, + const ObCollationType type2, + ObCollationLevel &res_level, + ObCollationType &res_type, + uint32_t flags); static int aggregate_max_length_for_string_result(ObExprResType &type, const ObExprResType *types, int64_t param_num, @@ -558,20 +565,19 @@ public: common::ObObjMeta &type, const common::ObObjMeta *types, int64_t param_num, - const common::ObCollationType conn_coll_type); + common::ObExprTypeCtx &type_ctx); static int aggregate_charsets_for_string_result_with_comparison( common::ObObjMeta &type, const ObExprResType *types, int64_t param_num, - const common::ObCollationType conn_coll_type); + common::ObExprTypeCtx &type_ctx); //skip_null for expr COALESCE static int aggregate_result_type_for_merge( ObExprResType &type, const ObExprResType *types, int64_t param_num, - const common::ObCollationType conn_coll_type, bool is_oracle_mode, - const common::ObLengthSemantics default_length_semantics, + common::ObExprTypeCtx &type_ctx, bool need_merge_type = TRUE, bool skip_null = FALSE, bool is_called_in_sql = TRUE); @@ -579,9 +585,8 @@ public: ObExprResType &type, const ObExprResType *types, int64_t param_num, - const common::ObCollationType conn_coll_type, bool is_oracle_mode, - const common::ObLengthSemantics default_length_semantics, + common::ObExprTypeCtx &type_ctx, bool need_merge_type = TRUE, bool skip_null = FALSE, bool is_called_in_sql = TRUE); @@ -656,7 +661,7 @@ public: int calc_cmp_type2(ObExprResType &type, const ObExprResType &type1, const ObExprResType &type2, - const common::ObCollationType coll_type, + common::ObExprTypeCtx &type_ctx, const bool left_is_const = false, const bool right_is_const = false) const; @@ -664,7 +669,7 @@ public: const ObExprResType &type1, const ObExprResType &type2, const ObExprResType &type3, - const common::ObCollationType coll_type) const; + common::ObExprTypeCtx &type_ctx) const; int calc_trig_function_result_type1(ObExprResType &type, ObExprResType &type1, common::ObExprTypeCtx &type_ctx) const; @@ -716,9 +721,6 @@ private: DISALLOW_COPY_AND_ASSIGN(ObExprOperator); // types and constants - static const uint32_t OB_COLL_DISALLOW_NONE = 1; - static const uint32_t OB_COLL_ALLOW_NUMERIC_CONV = 2; - protected: static int aggregate_collations( common::ObObjMeta &type, @@ -1231,7 +1233,7 @@ public: ObExprResType *types, const int64_t param_num, const bool has_lower, - const sql::ObSQLSessionInfo &my_session); + common::ObExprTypeCtx &type_ctx); // vector comparison, e.g. (a,b,c) > (1,2,3) virtual int calc_result_typeN(ObExprResType &type, @@ -2071,8 +2073,7 @@ protected: int calc_result_meta_for_comparison(ObExprResType &type, ObExprResType *types, int64_t param_num, - const common::ObCollationType coll_type, - const common::ObLengthSemantics default_length_semantics, + common::ObExprTypeCtx &type_ctx, const bool enable_decimal_int) const; protected: diff --git a/src/sql/engine/expr/ob_expr_operator_factory.cpp b/src/sql/engine/expr/ob_expr_operator_factory.cpp index 594e82130..58e77f3dc 100644 --- a/src/sql/engine/expr/ob_expr_operator_factory.cpp +++ b/src/sql/engine/expr/ob_expr_operator_factory.cpp @@ -252,7 +252,7 @@ #include "sql/engine/expr/ob_expr_plsql_variable.h" #include "sql/engine/expr/ob_expr_pl_associative_index.h" #include "sql/engine/expr/ob_expr_chr.h" -#include "sql/engine/expr/ob_expr_aes_encrypt.h" +#include "sql/engine/expr/ob_expr_symmetric_encrypt.h" #include "sql/engine/expr/ob_expr_timezone.h" #include "sql/engine/expr/ob_expr_sys_extract_utc.h" #include "sql/engine/expr/ob_expr_tz_offset.h" @@ -450,11 +450,13 @@ #include "sql/engine/expr/ob_expr_rb_calc.h" #include "sql/engine/expr/ob_expr_rb_to_string.h" #include "sql/engine/expr/ob_expr_rb_from_string.h" - #include "sql/engine/expr/ob_expr_lock_func.h" #include "sql/engine/expr/ob_expr_decode_trace_id.h" #include "sql/engine/expr/ob_expr_topn_filter.h" #include "sql/engine/expr/ob_expr_get_path.h" +#include "sql/engine/expr/ob_expr_transaction_id.h" +#include "sql/engine/expr/ob_expr_audit_log_func.h" +#include "sql/engine/expr/ob_expr_can_access_trigger.h" using namespace oceanbase::common; namespace oceanbase @@ -989,6 +991,7 @@ void ObExprOperatorFactory::register_expr_operators() REG_OP(ObExprTimestampToScn); REG_OP(ObExprScnToTimestamp); REG_OP(ObExprSqlModeConvert); + REG_OP(ObExprCanAccessTrigger); #if defined(ENABLE_DEBUG_LOG) || !defined(NDEBUG) // convert input value into an OceanBase error number and throw out as exception REG_OP(ObExprErrno); @@ -1118,6 +1121,14 @@ void ObExprOperatorFactory::register_expr_operators() REG_OP(ObExprRbFromString); REG_OP(ObExprGetPath); REG_OP(ObExprDecodeTraceId); + REG_OP(ObExprAuditLogSetFilter); + REG_OP(ObExprAuditLogRemoveFilter); + REG_OP(ObExprAuditLogSetUser); + REG_OP(ObExprAuditLogRemoveUser); + REG_OP(ObExprIsEnabledRole); + REG_OP(ObExprSm3); + REG_OP(ObExprSm4Encrypt); + REG_OP(ObExprSm4Decrypt); }(); // 注册oracle系统函数 REG_OP_ORCL(ObExprSysConnectByPath); diff --git a/src/sql/engine/expr/ob_expr_oracle_nullif.cpp b/src/sql/engine/expr/ob_expr_oracle_nullif.cpp index 6ec857943..b24efdba7 100644 --- a/src/sql/engine/expr/ob_expr_oracle_nullif.cpp +++ b/src/sql/engine/expr/ob_expr_oracle_nullif.cpp @@ -62,8 +62,7 @@ int ObExprOracleNullif::calc_result_type2(ObExprResType &type, type.set_length(type1.get_length()); } else { ObExprResType cmp_type; - if (OB_FAIL(calc_cmp_type2(cmp_type, type1, type2, - type_ctx.get_coll_type()))) { + if (OB_FAIL(calc_cmp_type2(cmp_type, type1, type2, type_ctx))) { LOG_WARN("failed to calc cmp type", K(ret), K(type1), K(type2)); } else { type.set_type(type1.get_type()); diff --git a/src/sql/engine/expr/ob_expr_pad.cpp b/src/sql/engine/expr/ob_expr_pad.cpp index 5ad30458c..290108061 100644 --- a/src/sql/engine/expr/ob_expr_pad.cpp +++ b/src/sql/engine/expr/ob_expr_pad.cpp @@ -84,12 +84,13 @@ int ObExprPad::calc_result_type3(ObExprResType &type, LOG_WARN("failed to push back source type", K(ret)); } else if (OB_FAIL(types.push_back(padding_str))) { LOG_WARN("failed to push back padding source type", K(ret)); - } else if (OB_FAIL(aggregate_charsets_for_string_result( - type, &types.at(0), 2, type_ctx.get_coll_type()))) { + } else if (OB_FAIL(aggregate_charsets_for_string_result(type, &types.at(0), 2, type_ctx))) { LOG_WARN("failed to set collation", K(ret)); } else { source.set_calc_collation_type(type.get_collation_type()); + source.set_calc_collation_level(type.get_collation_level()); padding_str.set_calc_collation_type(type.get_collation_type()); + padding_str.set_calc_collation_level(type.get_collation_level()); } } } diff --git a/src/sql/engine/expr/ob_expr_password.cpp b/src/sql/engine/expr/ob_expr_password.cpp index 3cf43357f..da9ada3e8 100644 --- a/src/sql/engine/expr/ob_expr_password.cpp +++ b/src/sql/engine/expr/ob_expr_password.cpp @@ -45,8 +45,7 @@ int ObExprPassword::calc_result_type1(ObExprResType &type, ObExprResType &text, text.set_calc_type(ObVarcharType); ObExprResType tmp_type; - OZ(ObExprOperator::aggregate_charsets_for_string_result(tmp_type, &text, 1, - type_ctx.get_coll_type())); + OZ(ObExprOperator::aggregate_charsets_for_string_result(tmp_type, &text, 1, type_ctx)); OX(text.set_calc_collation_type(tmp_type.get_collation_type())); OX(text.set_calc_collation_level(tmp_type.get_collation_level())); return ret; diff --git a/src/sql/engine/expr/ob_expr_quote.cpp b/src/sql/engine/expr/ob_expr_quote.cpp index 417619ecb..d672bce39 100644 --- a/src/sql/engine/expr/ob_expr_quote.cpp +++ b/src/sql/engine/expr/ob_expr_quote.cpp @@ -52,11 +52,12 @@ int ObExprQuote::calc_result_type1(ObExprResType &type, ObExprResType &type1, } else { type.set_varchar(); type.set_length(2 * type1.get_length() + 2); - if OB_FAIL(aggregate_charsets_for_string_result(type, &type1, 1, type_ctx.get_coll_type())) { + if OB_FAIL(aggregate_charsets_for_string_result(type, &type1, 1, type_ctx)) { LOG_WARN("aggregate charset for res failed", K(ret)); } else { type1.set_calc_type(ObVarcharType); type1.set_calc_collation_type(type.get_collation_type()); + type1.set_calc_collation_level(type.get_collation_level()); } } return ret; diff --git a/src/sql/engine/expr/ob_expr_regexp.cpp b/src/sql/engine/expr/ob_expr_regexp.cpp index 40c379d85..b555e0427 100644 --- a/src/sql/engine/expr/ob_expr_regexp.cpp +++ b/src/sql/engine/expr/ob_expr_regexp.cpp @@ -66,8 +66,6 @@ int ObExprRegexp::calc_result_type2(ObExprResType &type, { int ret = OB_SUCCESS; ObRawExpr * raw_expr = type_ctx.get_raw_expr(); - ObCollationType res_cs_type = CS_TYPE_INVALID; - ObCollationLevel res_cs_level = CS_LEVEL_INVALID; CK(NULL != type_ctx.get_raw_expr()); if (type1.is_null() || type2.is_null()) { type.set_int32(); @@ -85,41 +83,43 @@ int ObExprRegexp::calc_result_type2(ObExprResType &type, ret = OB_ERR_MYSQL_CHARACTER_SET_MISMATCH; LOG_USER_ERROR(OB_ERR_MYSQL_CHARACTER_SET_MISMATCH, collation1.length(), collation1.ptr(), collation2.length(), collation2.ptr()); LOG_WARN("If one of the params is binary string, all of the params should be implicitly castable to binary charset.", K(ret), K(type1), K(type2)); - } else if (OB_FAIL(ObCharset::aggregate_collation(type1.get_collation_level(), - type1.get_collation_type(), - type2.get_collation_level(), - type2.get_collation_type(), - res_cs_level, - res_cs_type))) { - LOG_WARN("fail to aggregate collation", K(ret), K(type1), K(type2)); } else { - type.set_int32(); - type.set_precision(DEFAULT_PRECISION_FOR_BOOL); - type.set_scale(DEFAULT_SCALE_FOR_INTEGER); - //why we set the calc collation type is utf16, because the ICU regexp engine is used uft16, - //we need convert it the need collation in advance, and no need to think about in regexp. - bool is_case_sensitive = ObCharset::is_bin_sort(res_cs_type); - bool need_utf8 = false; - type1.set_calc_type(ObVarcharType); - type1.set_calc_collation_level(type.get_collation_level()); - type2.set_calc_type(ObVarcharType); - type2.set_calc_collation_level(type.get_collation_level()); - if (OB_FAIL(ObExprRegexContext::check_need_utf8(raw_expr->get_param_expr(1), need_utf8))) { - LOG_WARN("fail to check need utf8", K(ret)); - } else if (need_utf8) { - type2.set_calc_collation_type(is_case_sensitive ? CS_TYPE_UTF8MB4_BIN : CS_TYPE_UTF8MB4_GENERAL_CI); + ObExprResTypes res_types; + if (OB_FAIL(res_types.push_back(type1))) { + LOG_WARN("fail to push back res type", K(ret)); + } else if (OB_FAIL(res_types.push_back(type2))) { + LOG_WARN("fail to push back res type", K(ret)); + } else if (OB_FAIL(aggregate_charsets_for_comparison(type, &res_types.at(0), 2, type_ctx))) { + LOG_WARN("failed to aggregate_charsets_for_comparison", K(ret)); } else { - type2.set_calc_collation_type(is_case_sensitive ? CS_TYPE_UTF16_BIN : CS_TYPE_UTF16_GENERAL_CI); - } + //why we set the calc collation type is utf16, because the ICU regexp engine is used uft16, + //we need convert it the need collation in advance, and no need to think about in regexp. + bool is_case_sensitive = ObCharset::is_bin_sort(type.get_calc_collation_type()); + type.set_int32(); + type.set_precision(DEFAULT_PRECISION_FOR_BOOL); + type.set_scale(DEFAULT_SCALE_FOR_INTEGER); + bool need_utf8 = false; + type1.set_calc_type(ObVarcharType); + type1.set_calc_collation_level(type.get_collation_level()); + type2.set_calc_type(ObVarcharType); + type2.set_calc_collation_level(type.get_collation_level()); + if (OB_FAIL(ObExprRegexContext::check_need_utf8(raw_expr->get_param_expr(1), need_utf8))) { + LOG_WARN("fail to check need utf8", K(ret)); + } else if (need_utf8) { + type2.set_calc_collation_type(is_case_sensitive ? CS_TYPE_UTF8MB4_BIN : CS_TYPE_UTF8MB4_GENERAL_CI); + } else { + type2.set_calc_collation_type(is_case_sensitive ? CS_TYPE_UTF16_BIN : CS_TYPE_UTF16_GENERAL_CI); + } - need_utf8 = false; - if (OB_FAIL(ret)) { - } else if (OB_FAIL(ObExprRegexContext::check_need_utf8(raw_expr->get_param_expr(0), need_utf8))) { - LOG_WARN("fail to check need utf8", K(ret)); - } else if (need_utf8) { - type1.set_calc_collation_type(is_case_sensitive ? CS_TYPE_UTF8MB4_BIN : CS_TYPE_UTF8MB4_GENERAL_CI); - } else { - type1.set_calc_collation_type(is_case_sensitive ? CS_TYPE_UTF16_BIN : CS_TYPE_UTF16_GENERAL_CI); + need_utf8 = false; + if (OB_FAIL(ret)) { + } else if (OB_FAIL(ObExprRegexContext::check_need_utf8(raw_expr->get_param_expr(0), need_utf8))) { + LOG_WARN("fail to check need utf8", K(ret)); + } else if (need_utf8) { + type1.set_calc_collation_type(is_case_sensitive ? CS_TYPE_UTF8MB4_BIN : CS_TYPE_UTF8MB4_GENERAL_CI); + } else { + type1.set_calc_collation_type(is_case_sensitive ? CS_TYPE_UTF16_BIN : CS_TYPE_UTF16_GENERAL_CI); + } } } return ret; diff --git a/src/sql/engine/expr/ob_expr_regexp_instr.cpp b/src/sql/engine/expr/ob_expr_regexp_instr.cpp index 6b8a99e83..c32116f23 100644 --- a/src/sql/engine/expr/ob_expr_regexp_instr.cpp +++ b/src/sql/engine/expr/ob_expr_regexp_instr.cpp @@ -61,7 +61,7 @@ int ObExprRegexpInstr::calc_result_typeN(ObExprResType &type, ObExprResType cmp_type; if (OB_FAIL(ObExprRegexContext::check_binary_compatible(types, 2))) { LOG_WARN("types are not compatible with binary.", K(ret)); - } else if (OB_FAIL(aggregate_charsets_for_comparison(cmp_type, types, 2, type_ctx.get_coll_type()))) { + } else if (OB_FAIL(aggregate_charsets_for_comparison(cmp_type, types, 2, type_ctx))) { LOG_WARN("fail to aggregate charsets for comparison"); } else { is_case_sensitive = ObCharset::is_bin_sort(cmp_type.get_calc_collation_type()); diff --git a/src/sql/engine/expr/ob_expr_regexp_like.cpp b/src/sql/engine/expr/ob_expr_regexp_like.cpp index f33f91b88..8fa42c20d 100644 --- a/src/sql/engine/expr/ob_expr_regexp_like.cpp +++ b/src/sql/engine/expr/ob_expr_regexp_like.cpp @@ -62,7 +62,7 @@ int ObExprRegexpLike::calc_result_typeN(ObExprResType &type, ObExprResType cmp_type; if (OB_FAIL(ObExprRegexContext::check_binary_compatible(types, 2))) { LOG_WARN("types are not compatible with binary.", K(ret)); - } else if (OB_FAIL(aggregate_charsets_for_comparison(cmp_type, types, 2, type_ctx.get_coll_type()))) { + } else if (OB_FAIL(aggregate_charsets_for_comparison(cmp_type, types, 2, type_ctx))) { LOG_WARN("fail to aggregate charsets for comparison"); } else { is_case_sensitive = ObCharset::is_bin_sort(cmp_type.get_calc_collation_type()); diff --git a/src/sql/engine/expr/ob_expr_regexp_replace.cpp b/src/sql/engine/expr/ob_expr_regexp_replace.cpp index 4af87b97b..54684391c 100644 --- a/src/sql/engine/expr/ob_expr_regexp_replace.cpp +++ b/src/sql/engine/expr/ob_expr_regexp_replace.cpp @@ -110,7 +110,7 @@ int ObExprRegexpReplace::calc_result_typeN(ObExprResType &type, if (OB_FAIL(ObExprRegexContext::check_binary_compatible(types, 3))) { LOG_WARN("types are not compatible with binary.", K(ret)); } else { - ret = aggregate_charsets_for_string_result(type, real_types, 2, type_ctx.get_coll_type()); + ret = aggregate_charsets_for_string_result(type, real_types, 2, type_ctx); is_case_sensitive = ObCharset::is_bin_sort(type.get_collation_type()); } } diff --git a/src/sql/engine/expr/ob_expr_regexp_substr.cpp b/src/sql/engine/expr/ob_expr_regexp_substr.cpp index d0bbd7c19..21b5547f9 100644 --- a/src/sql/engine/expr/ob_expr_regexp_substr.cpp +++ b/src/sql/engine/expr/ob_expr_regexp_substr.cpp @@ -98,7 +98,7 @@ int ObExprRegexpSubstr::calc_result_typeN(ObExprResType &type, if (OB_FAIL(ObExprRegexContext::check_binary_compatible(types, 2))) { LOG_WARN("types are not compatible with binary.", K(ret)); } else { - ret = aggregate_charsets_for_string_result(type, real_types, 2, type_ctx.get_coll_type()); + ret = aggregate_charsets_for_string_result(type, real_types, 2, type_ctx); is_case_sensitive = ObCharset::is_bin_sort(type.get_collation_type()); } } diff --git a/src/sql/engine/expr/ob_expr_repeat.cpp b/src/sql/engine/expr/ob_expr_repeat.cpp index cfed0651a..8d0b41d1b 100644 --- a/src/sql/engine/expr/ob_expr_repeat.cpp +++ b/src/sql/engine/expr/ob_expr_repeat.cpp @@ -102,7 +102,7 @@ int ObExprRepeat::calc_result_type2(ObExprResType &type, } } if (OB_SUCC(ret)) { - if (OB_FAIL(aggregate_charsets_for_string_result(type, &text, 1, type_ctx.get_coll_type()))) { + if (OB_FAIL(aggregate_charsets_for_string_result(type, &text, 1, type_ctx))) { LOG_WARN("failed to aggregate charsets for string result", K(ret)); } else { text.set_calc_collation_level(type.get_collation_level()); diff --git a/src/sql/engine/expr/ob_expr_replace.cpp b/src/sql/engine/expr/ob_expr_replace.cpp index d272b12bd..2704e5f1a 100644 --- a/src/sql/engine/expr/ob_expr_replace.cpp +++ b/src/sql/engine/expr/ob_expr_replace.cpp @@ -81,8 +81,7 @@ int ObExprReplace::calc_result_typeN(ObExprResType &type, if (3 == param_num) { types_array[2].set_calc_type(ObVarcharType); } - OZ(ObExprOperator::aggregate_charsets_for_string_result_with_comparison( - type, types_array, param_num, type_ctx.get_coll_type())); + OZ(ObExprOperator::aggregate_charsets_for_string_result_with_comparison(type, types_array, 1, type_ctx)); for (int64_t i = 0; OB_SUCC(ret) && i < param_num; i++) { types_array[i].set_calc_meta(type); } diff --git a/src/sql/engine/expr/ob_expr_reverse.h b/src/sql/engine/expr/ob_expr_reverse.h index c9801e66c..eb5d9f38c 100644 --- a/src/sql/engine/expr/ob_expr_reverse.h +++ b/src/sql/engine/expr/ob_expr_reverse.h @@ -65,7 +65,7 @@ inline int ObExprReverse::calc_result_type1(ObExprResType &type, type.set_length(type1.get_length()); } } - ret = aggregate_charsets_for_string_result(type, &type1, 1, type_ctx.get_coll_type()); + ret = aggregate_charsets_for_string_result(type, &type1, 1, type_ctx); } else { if (ob_is_character_type(type1.get_type(), type1.get_collation_type()) || ob_is_varbinary_or_binary(type1.get_type(), type1.get_collation_type()) diff --git a/src/sql/engine/expr/ob_expr_right.cpp b/src/sql/engine/expr/ob_expr_right.cpp index 8dcc20908..eb2e9023d 100644 --- a/src/sql/engine/expr/ob_expr_right.cpp +++ b/src/sql/engine/expr/ob_expr_right.cpp @@ -43,9 +43,10 @@ int ObExprRight::calc_result_type2(ObExprResType &type, ObExprResType &type1, type_ctx.set_cast_mode(type_ctx.get_cast_mode() | CM_STRING_INTEGER_TRUNC); type.set_varchar(); type.set_length(type1.get_length()); - OZ(aggregate_charsets_for_string_result(type, &type1, 1, type_ctx.get_coll_type())); + OZ(aggregate_charsets_for_string_result(type, &type1, 1, type_ctx)); OX(type1.set_calc_type(ObVarcharType)); OX(type1.set_calc_collation_type(type.get_collation_type())); + OX(type1.set_calc_collation_level(type.get_collation_level())); return ret; } diff --git a/src/sql/engine/expr/ob_expr_sha.cpp b/src/sql/engine/expr/ob_expr_sha.cpp index d9c591130..8848c1887 100644 --- a/src/sql/engine/expr/ob_expr_sha.cpp +++ b/src/sql/engine/expr/ob_expr_sha.cpp @@ -47,7 +47,7 @@ int ObExprSha::calc_result_type1(ObExprResType &type, type.set_collation_type(type_ctx.get_coll_type()); type.set_collation_level(common::CS_LEVEL_COERCIBLE); type1.set_calc_type(ObVarcharType); - OZ (aggregate_charsets_for_string_result(tmp_type, &type1, 1, type_ctx.get_coll_type())); + OZ (aggregate_charsets_for_string_result(tmp_type, &type1, 1, type_ctx)); OX (type1.set_calc_collation_type(tmp_type.get_collation_type())); OX (type1.set_calc_collation_level(tmp_type.get_collation_level())); @@ -69,6 +69,9 @@ int ObExprSha::eval_sha(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &expr_datum) ObDatum *arg = NULL; if (OB_FAIL(expr.eval_param_value(ctx, arg))) { LOG_WARN("evaluate parameter value failed", K(ret)); + } else if (OB_ISNULL(arg)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("arg is null", K(ret)); } else if (arg->is_null()) { expr_datum.set_null(); } else { @@ -113,7 +116,7 @@ int ObExprSha2::calc_result_type2(ObExprResType &type, type.set_collation_type(type_ctx.get_coll_type()); type.set_collation_level(common::CS_LEVEL_COERCIBLE); type1.set_calc_type(ObVarcharType); - OZ (aggregate_charsets_for_string_result(tmp_type, &type1, 1, type_ctx.get_coll_type())); + OZ (aggregate_charsets_for_string_result(tmp_type, &type1, 1, type_ctx)); OX (type1.set_calc_collation_type(tmp_type.get_collation_type())); OX (type1.set_calc_collation_level(tmp_type.get_collation_level())); OX (type2.set_calc_type(ObIntType)); @@ -136,6 +139,9 @@ int ObExprSha2::eval_sha2(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &expr_datu ObDatum *arg1 = NULL; if (OB_FAIL(expr.eval_param_value(ctx, arg0, arg1))) { LOG_WARN("evaluate parameter value failed", K(ret)); + } else if (OB_ISNULL(arg0) || OB_ISNULL(arg1)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("arg is null", K(ret)); } else if (arg0->is_null() || arg1->is_null()) { expr_datum.set_null(); } else { @@ -168,5 +174,69 @@ DEF_SET_LOCAL_SESSION_VARS(ObExprSha2, raw_expr) { return ret; } + +ObExprSm3::ObExprSm3(ObIAllocator &alloc) + : ObStringExprOperator(alloc, T_FUN_SYS_SM3, N_SM3, 1, VALID_FOR_GENERATED_COL) { } + +ObExprSm3::~ObExprSm3() { } + +int ObExprSm3::calc_result_type1(ObExprResType &type, + ObExprResType &type1, + common::ObExprTypeCtx &type_ctx) const +{ + int ret = OB_SUCCESS; + ObExprResType tmp_type; + int64_t length = 0; + type.set_varchar(); + type.set_collation_type(type_ctx.get_coll_type()); + type.set_collation_level(common::CS_LEVEL_COERCIBLE); + type1.set_calc_type(ObVarcharType); + OZ (aggregate_charsets_for_string_result(tmp_type, &type1, 1, type_ctx)); + OX (type1.set_calc_collation_type(tmp_type.get_collation_type())); + OX (type1.set_calc_collation_level(tmp_type.get_collation_level())); + OZ (ObHashUtil::get_hash_output_len(OB_HASH_SM3, length)); + OX (type.set_length(length * 2)); + return ret; +} + +int ObExprSm3::cg_expr(ObExprCGCtx &, const ObRawExpr &, ObExpr &rt_expr) const +{ + int ret = OB_SUCCESS; + rt_expr.eval_func_ = &ObExprSm3::eval_sm3; + return ret; +} + +int ObExprSm3::eval_sm3(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &expr_datum) +{ + int ret = OB_SUCCESS; + ObDatum *arg = NULL; + if (OB_FAIL(expr.eval_param_value(ctx, arg))) { + LOG_WARN("evaluate parameter value failed", K(ret)); + } else if (OB_ISNULL(arg)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("arg is null", K(ret)); + } else if (arg->is_null()) { + expr_datum.set_null(); + } else { + ObString text = arg->get_string(); + ObString sha_str; + ObEvalCtx::TempAllocGuard alloc_guard(ctx); + if (OB_FAIL(ObHashUtil::hash(OB_HASH_SM3, text, alloc_guard.get_allocator(), sha_str))) { + LOG_WARN("fail to calc sha", K(text), K(ret)); + } else if (OB_FAIL(ObDatumHexUtils::hex(expr, sha_str, ctx, alloc_guard.get_allocator(), + expr_datum, false))) { + LOG_WARN("fail to conver sha_str to hex", K(sha_str), K(ret)); + } + } + return ret; +} + +DEF_SET_LOCAL_SESSION_VARS(ObExprSm3, raw_expr) { + int ret = OB_SUCCESS; + SET_LOCAL_SYSVAR_CAPACITY(1); + EXPR_ADD_LOCAL_SYSVAR(share::SYS_VAR_COLLATION_CONNECTION); + return ret; +} + } } diff --git a/src/sql/engine/expr/ob_expr_sha.h b/src/sql/engine/expr/ob_expr_sha.h index c23a69979..f110e7fc4 100644 --- a/src/sql/engine/expr/ob_expr_sha.h +++ b/src/sql/engine/expr/ob_expr_sha.h @@ -58,6 +58,23 @@ private: DISALLOW_COPY_AND_ASSIGN(ObExprSha2); }; +class ObExprSm3 : public ObStringExprOperator +{ +public: + explicit ObExprSm3(common::ObIAllocator &alloc); + virtual ~ObExprSm3(); + virtual int calc_result_type1(ObExprResType &type, + ObExprResType &type1, + common::ObExprTypeCtx &type_ctx) const; + virtual int cg_expr(ObExprCGCtx &op_cg_ctx, + const ObRawExpr &raw_expr, + ObExpr &rt_expr) const override; + static int eval_sm3(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &expr_datum); + DECLARE_SET_LOCAL_SESSION_VARS; +private: + DISALLOW_COPY_AND_ASSIGN(ObExprSm3); +}; + } } #endif /* SRC_SQL_ENGINE_EXPR_OB_EXPR_SHA_H_ */ diff --git a/src/sql/engine/expr/ob_expr_statement_digest.cpp b/src/sql/engine/expr/ob_expr_statement_digest.cpp index 626d0b35c..0cd607395 100644 --- a/src/sql/engine/expr/ob_expr_statement_digest.cpp +++ b/src/sql/engine/expr/ob_expr_statement_digest.cpp @@ -126,7 +126,7 @@ int ObExprStatementDigest::calc_result_type1(ObExprResType &type, type.set_collation_level(CS_LEVEL_COERCIBLE); type.set_length(OB_MAX_SQL_ID_LENGTH); type1.set_calc_type(ObVarcharType); - OZ (aggregate_charsets_for_string_result(tmp_type, &type1, 1, type_ctx.get_coll_type())); + OZ (aggregate_charsets_for_string_result(tmp_type, &type1, 1, type_ctx)); OX (type1.set_calc_collation_type(tmp_type.get_collation_type())); OX (type1.set_calc_collation_level(tmp_type.get_collation_level())); return ret; @@ -190,7 +190,7 @@ int ObExprStatementDigestText::calc_result_type1(ObExprResType &type, int ret = OB_SUCCESS; type.set_clob(); type1.set_calc_type(ObVarcharType); - OZ (aggregate_charsets_for_string_result(type, &type1, 1, type_ctx.get_coll_type())); + OZ (aggregate_charsets_for_string_result(type, &type1, 1, type_ctx)); OX (type1.set_calc_collation_type(type.get_collation_type())); OX (type1.set_calc_collation_level(type.get_collation_level())); type.set_length(OB_MAX_MYSQL_VARCHAR_LENGTH); diff --git a/src/sql/engine/expr/ob_expr_strcmp.cpp b/src/sql/engine/expr/ob_expr_strcmp.cpp index 4aec91f96..709df8e53 100644 --- a/src/sql/engine/expr/ob_expr_strcmp.cpp +++ b/src/sql/engine/expr/ob_expr_strcmp.cpp @@ -35,28 +35,26 @@ int ObExprStrcmp::calc_result_type2(ObExprResType &type, ObExprResType &type2, ObExprTypeCtx &type_ctx) const { - UNUSED(type2); - UNUSED(type_ctx); int ret = OB_SUCCESS; type.set_int(); type.set_scale(ObAccuracy::DDL_DEFAULT_ACCURACY[ObIntType].scale_); type.set_precision(ObAccuracy::DDL_DEFAULT_ACCURACY[ObIntType].precision_); ObCollationType coll_type = CS_TYPE_INVALID; ObCollationLevel coll_level = CS_LEVEL_EXPLICIT; - ret = ObCharset::aggregate_collation(type1.get_collation_level(), - type1.get_collation_type(), - type2.get_collation_level(), - type2.get_collation_type(), - coll_level, - coll_type); - type.set_calc_type(ObVarcharType); - type.set_calc_collation_type(coll_type); - type.set_calc_collation_level(coll_level); - type1.set_calc_type(ObVarcharType); - type2.set_calc_type(ObVarcharType); - type1.set_calc_collation_type(coll_type); - type2.set_calc_collation_type(coll_type); - + ObExprResTypes res_types; + if (OB_FAIL(res_types.push_back(type1))) { + LOG_WARN("fail to push back res type", K(ret)); + } else if (OB_FAIL(res_types.push_back(type2))) { + LOG_WARN("fail to push back res type", K(ret)); + } else if (OB_FAIL(aggregate_charsets_for_comparison(type, &res_types.at(0), 2, type_ctx))) { + LOG_WARN("failed to aggregate_charsets_for_comparison", K(ret)); + } else { + type.set_calc_type(ObVarcharType); + type1.set_calc_type(ObVarcharType); + type2.set_calc_type(ObVarcharType); + type1.set_calc_collation(type); + type2.set_calc_collation(type); + } return ret; } diff --git a/src/sql/engine/expr/ob_expr_substr.cpp b/src/sql/engine/expr/ob_expr_substr.cpp index 6f2b8e8f3..cd3dd83bc 100644 --- a/src/sql/engine/expr/ob_expr_substr.cpp +++ b/src/sql/engine/expr/ob_expr_substr.cpp @@ -254,19 +254,20 @@ int ObExprSubstr::calc_result_typeN(ObExprResType &type, type.set_varchar(); } } - OZ(aggregate_charsets_for_string_result(type, types_array, 1, type_ctx.get_coll_type())); + OZ(aggregate_charsets_for_string_result(type, types_array, 1, type_ctx)); if (OB_SUCC(ret)) { if (is_mysql_mode() && (types_array[0].is_text() || types_array[0].is_blob())) { // do nothing } else { types_array[0].set_calc_type(ObVarcharType); + types_array[0].set_calc_collation_level(type.get_collation_level()); + types_array[0].set_calc_collation_type(type.get_collation_type()); } - types_array[0].set_calc_collation_level(type.get_calc_collation_level()); - types_array[0].set_calc_collation_type(type.get_collation_type()); } if (OB_SUCC(ret)) { for (int i = 1; i < param_num; i++) { types_array[i].set_calc_type(ObIntType); + types_array[i].set_calc_collation_level(type.get_collation_level()); } } if (OB_SUCC(ret) && !ob_is_text_tc(type.get_type())) { diff --git a/src/sql/engine/expr/ob_expr_substring_index.cpp b/src/sql/engine/expr/ob_expr_substring_index.cpp index 42a9808a8..2e93c3217 100644 --- a/src/sql/engine/expr/ob_expr_substring_index.cpp +++ b/src/sql/engine/expr/ob_expr_substring_index.cpp @@ -70,8 +70,7 @@ inline int ObExprSubstringIndex::calc_result_type3(ObExprResType &type, ObExprResType types[2] = {alloc, alloc}; types[0] = str; types[1] = delim; - if (OB_FAIL(aggregate_charsets_for_string_result_with_comparison( - type, types, 2, type_ctx.get_coll_type()))) { + if (OB_FAIL(aggregate_charsets_for_string_result_with_comparison(type, types, 2, type_ctx))) { LOG_WARN("aggregate_charsets_for_string_result_with_comparison failed", K(ret)); } else { diff --git a/src/sql/engine/expr/ob_expr_symmetric_encrypt.cpp b/src/sql/engine/expr/ob_expr_symmetric_encrypt.cpp new file mode 100644 index 000000000..08ff9d868 --- /dev/null +++ b/src/sql/engine/expr/ob_expr_symmetric_encrypt.cpp @@ -0,0 +1,390 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_EXE +#include "ob_expr_symmetric_encrypt.h" +#include "share/object/ob_obj_cast.h" +#include "share/ob_encryption_util.h" +#include "ob_expr_extract.h" +#include "sql/session/ob_sql_session_info.h" +#include "sql/engine/ob_exec_context.h" + +using namespace oceanbase::share; +using namespace oceanbase::common; + +namespace oceanbase +{ +namespace sql +{ +ObExprBaseEncrypt::ObExprBaseEncrypt(ObIAllocator& alloc, ObItemType func_type, const char* name) + : ObFuncExprOperator(alloc, + func_type, + name, + TWO_OR_THREE, + NOT_VALID_FOR_GENERATED_COL, + NOT_ROW_DIMENSION) { } + +ObExprBaseEncrypt::~ObExprBaseEncrypt() { } + +int ObExprBaseEncrypt::calc_result_typeN(ObExprResType& type, + ObExprResType* types_stack, + int64_t param_num, + ObExprTypeCtx& type_ctx) const +{ + UNUSED(type_ctx); + int ret = OB_SUCCESS; + if (OB_ISNULL(types_stack)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("null types",K(ret)); + } else if (OB_UNLIKELY(param_num > 3 || param_num < 2)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("param num is not correct", K(param_num)); + } else { + for (int i = 0; i < param_num; ++i) { + types_stack[i].set_calc_type(common::ObVarcharType); + types_stack[i].set_calc_collation_type(types_stack[i].get_collation_type()); + types_stack[i].set_calc_collation_level(types_stack[i].get_collation_level()); + } + type.set_varbinary(); + type.set_length((types_stack[0].get_length() * 3 / ObBlockCipher::OB_CIPHER_BLOCK_LENGTH + 1) * + ObBlockCipher::OB_CIPHER_BLOCK_LENGTH); + type.set_collation_level(CS_LEVEL_COERCIBLE); + } + return ret; +} + +int ObExprBaseEncrypt::eval_encrypt(const ObExpr &expr, + ObEvalCtx &ctx, + const ObCipherOpMode op_mode, + const ObString &func_name, + ObDatum &res) +{ + int ret = OB_SUCCESS; + ObDatum *src = NULL; + ObDatum *key = NULL; + bool is_ecb = true; + if (OB_UNLIKELY(2 != expr.arg_cnt_ && 3 != expr.arg_cnt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error", K(ret), K(expr.arg_cnt_)); + } else if (OB_FAIL(expr.eval_param_value(ctx, src, key))) { + LOG_WARN("eval arg failed", K(ret)); + } else if (OB_ISNULL(src) || OB_ISNULL(key)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("got null ptr", K(ret)); + } else if (src->is_null() || key->is_null()) { + res.set_null(); + } else if (FALSE_IT(is_ecb = ObEncryptionUtil::is_ecb_mode(op_mode))) { + } else if (!is_ecb && 3 != expr.arg_cnt_) { + ret = OB_ERR_PARAM_SIZE; + LOG_USER_ERROR(OB_ERR_PARAM_SIZE, func_name.length(), func_name.ptr()); + } else { + if (is_ecb && 3 == expr.arg_cnt_) { + LOG_USER_WARN(OB_ERR_INVALID_INPUT_STRING, "iv"); // just user warn, not set ret error. + } + const ObString &src_str = expr.locate_param_datum(ctx, 0).get_string(); + const ObString &key_str = expr.locate_param_datum(ctx, 1).get_string(); + const int64_t buf_length = (src_str.length() / ObBlockCipher::OB_CIPHER_BLOCK_LENGTH + 1) * + ObBlockCipher::OB_CIPHER_BLOCK_LENGTH; + char *buf = NULL; + int64_t out_len = 0; + ObEvalCtx::TempAllocGuard alloc_guard(ctx); + ObIAllocator &calc_alloc = alloc_guard.get_allocator(); + if (OB_ISNULL(buf = static_cast(calc_alloc.alloc(buf_length)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc memory failed", K(ret), K(buf_length)); + } else if (is_ecb) { + if (OB_FAIL(ObBlockCipher::encrypt(key_str.ptr(), key_str.length(), + src_str.ptr(), src_str.length(), + buf_length, NULL, 0, NULL, 0, 0, op_mode, + buf, out_len, NULL))) { + LOG_WARN("failed to encrypt", K(ret)); + } + } else if (!is_ecb) { + ObString iv_str = expr.locate_param_datum(ctx, 2).get_string(); + if (OB_UNLIKELY(iv_str.length() < ObBlockCipher::OB_DEFAULT_IV_LENGTH)) { + ret = OB_ERR_AES_IV_LENGTH; + LOG_USER_ERROR(OB_ERR_AES_IV_LENGTH); + } else if (FALSE_IT(iv_str.assign(iv_str.ptr(), ObBlockCipher::OB_DEFAULT_IV_LENGTH))) { + } else if (OB_FAIL(ObBlockCipher::encrypt(key_str.ptr(), key_str.length(), + src_str.ptr(), src_str.length(), + buf_length, iv_str.ptr(), iv_str.length(), NULL, 0, + 0, op_mode, buf, out_len, NULL))) { + LOG_WARN("failed to encrypt", K(ret)); + } + } + if (OB_SUCC(ret)) { + ObExprStrResAlloc res_alloc(expr, ctx); + char *res_buf = NULL; + if (OB_ISNULL(res_buf = static_cast(res_alloc.alloc(out_len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc memory failed", K(ret), K(out_len)); + } else { + MEMCPY(res_buf, buf, out_len); + res.set_string(res_buf, out_len); + } + } + } + return ret; +} + +ObExprBaseDecrypt::ObExprBaseDecrypt(ObIAllocator& alloc, ObItemType func_type, const char *name) + : ObFuncExprOperator(alloc, + func_type, + name, + TWO_OR_THREE, + NOT_VALID_FOR_GENERATED_COL, + NOT_ROW_DIMENSION) { } + +ObExprBaseDecrypt::~ObExprBaseDecrypt() { } + +int ObExprBaseDecrypt::calc_result_typeN(ObExprResType& type, + ObExprResType* types_stack, + int64_t param_num, + ObExprTypeCtx& type_ctx) const +{ + UNUSED(type_ctx); + int ret = OB_SUCCESS; + if (OB_ISNULL(types_stack)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("null types",K(ret)); + } else if (OB_UNLIKELY(param_num > 3 || param_num < 2)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("param num is not correct", K(param_num)); + } else { + for (int i = 0; i < param_num; ++i) { + types_stack[i].set_calc_type(common::ObVarcharType); + } + type.set_varbinary(); + type.set_length(types_stack[0].get_length() * 3); + type.set_collation_level(CS_LEVEL_COERCIBLE); + } + return ret; +} + +int ObExprBaseDecrypt::eval_decrypt(const ObExpr &expr, + ObEvalCtx &ctx, + const ObCipherOpMode op_mode, + const ObString &func_name, + ObDatum &res) +{ + int ret = OB_SUCCESS; + ObDatum *src = NULL; + ObDatum *key = NULL; + bool is_null = false; + bool is_ecb = true; + if (OB_FAIL(expr.eval_param_value(ctx, src, key))) { + LOG_WARN("eval arg failed", K(ret)); + } else if (src->is_null() || key->is_null()) { + res.set_null(); + } else if (OB_UNLIKELY(2 != expr.arg_cnt_ && 3 != expr.arg_cnt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error", K(ret), K(expr.arg_cnt_)); + } else if (FALSE_IT(is_ecb= ObEncryptionUtil::is_ecb_mode(op_mode))) { + } else if (!is_ecb && 3 != expr.arg_cnt_) { + ret = OB_ERR_PARAM_SIZE; + LOG_USER_ERROR(OB_ERR_PARAM_SIZE, func_name.length(), func_name.ptr()); + } else { + if (is_ecb && 3 == expr.arg_cnt_) { + LOG_USER_WARN(OB_ERR_INVALID_INPUT_STRING, "iv"); + } + const ObString &src_str = expr.locate_param_datum(ctx, 0).get_string(); + const ObString &key_str = expr.locate_param_datum(ctx, 1).get_string(); + const int64_t buf_len = src_str.length() + 1; + char *buf = NULL; + int64_t out_len = 0; + ObEvalCtx::TempAllocGuard alloc_guard(ctx); + ObIAllocator &calc_alloc = alloc_guard.get_allocator(); + if (OB_ISNULL(buf = static_cast(calc_alloc.alloc(buf_len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc mem failed", K(ret), K(buf_len)); + } else if (is_ecb) { + if (OB_FAIL(ObBlockCipher::decrypt(key_str.ptr(), key_str.length(), + src_str.ptr(), src_str.length(), buf_len, NULL, 0, NULL, 0, + NULL, 0, op_mode, buf, out_len))) { + LOG_WARN("failed to decrypt", K(ret)); + } + } else { + ObString iv_str = expr.locate_param_datum(ctx, 2).get_string(); + if (OB_UNLIKELY(iv_str.length() < ObBlockCipher::OB_DEFAULT_IV_LENGTH)) { + ret = OB_ERR_AES_IV_LENGTH; + LOG_USER_ERROR(OB_ERR_AES_IV_LENGTH); + } else if (FALSE_IT(iv_str.assign(iv_str.ptr(), ObBlockCipher::OB_DEFAULT_IV_LENGTH))) { + } else if (OB_FAIL(ObBlockCipher::decrypt(key_str.ptr(), key_str.length(), + src_str.ptr(), src_str.length(), buf_len, + iv_str.ptr(), iv_str.length(), NULL, 0, NULL, 0, + op_mode, buf, out_len))) { + LOG_WARN("failed to decrypt", K(ret)); + } + } + if (OB_ERR_AES_DECRYPT == ret) { + is_null = true; + ret = OB_SUCCESS; + } + if (OB_SUCC(ret)) { + ObExprStrResAlloc res_alloc(expr, ctx); + char *res_buf = NULL; + if (is_null) { + res.set_null(); + } else if (OB_ISNULL(res_buf = static_cast(res_alloc.alloc(out_len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc memory failed", K(ret), K(out_len)); + } else { + MEMCPY(res_buf, buf, out_len); + res.set_string(res_buf, out_len); + } + } + } + return ret; +} + +ObExprAesEncrypt::ObExprAesEncrypt(ObIAllocator& alloc) + : ObExprBaseEncrypt(alloc, T_FUN_AES_ENCRYPT, N_AES_ENCRYPT) { } + +ObExprAesEncrypt::~ObExprAesEncrypt() { } + +int ObExprAesEncrypt::cg_expr(ObExprCGCtx &expr_cg_ctx, + const ObRawExpr &raw_expr, + ObExpr &rt_expr) const +{ + int ret = OB_SUCCESS; + UNUSED(expr_cg_ctx); + UNUSED(raw_expr); + rt_expr.eval_func_ = eval_aes_encrypt; + return ret; +} + +int ObExprAesEncrypt::eval_aes_encrypt(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) +{ + int ret = OB_SUCCESS; + ObCipherOpMode op_mode = ObCipherOpMode::ob_invalid_mode; + ObString func_name(strlen(N_AES_ENCRYPT), N_AES_ENCRYPT); + if (OB_FAIL(ObEncryptionUtil::get_cipher_op_mode(op_mode, ctx.exec_ctx_.get_my_session()))) { + LOG_WARN("fail to get cipher mode", K(ret)); + } else if (!ObEncryptionUtil::is_aes_encryption(op_mode)) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "using aes_encrypt with not aes block_encryption_mode"); + } else if (OB_FAIL(eval_encrypt(expr, ctx, op_mode, func_name, res))) { + LOG_WARN("failed to eval aes encrypt", K(ret)); + } else { /* do nothing */ } + return ret; +} + +ObExprAesDecrypt::ObExprAesDecrypt(ObIAllocator& alloc) + : ObExprBaseDecrypt(alloc, T_FUN_AES_DECRYPT, N_AES_DECRYPT) { } + +ObExprAesDecrypt::~ObExprAesDecrypt() { } + +int ObExprAesDecrypt::cg_expr(ObExprCGCtx &expr_cg_ctx, + const ObRawExpr &raw_expr, + ObExpr &rt_expr) const +{ + int ret = OB_SUCCESS; + UNUSED(expr_cg_ctx); + UNUSED(raw_expr); + rt_expr.eval_func_ = eval_aes_decrypt; + return ret; +} + +int ObExprAesDecrypt::eval_aes_decrypt(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) +{ + int ret = OB_SUCCESS; + ObCipherOpMode op_mode = ObCipherOpMode::ob_invalid_mode; + ObString func_name(strlen(N_AES_DECRYPT), N_AES_DECRYPT); + if (OB_FAIL(ObEncryptionUtil::get_cipher_op_mode(op_mode, ctx.exec_ctx_.get_my_session()))) { + LOG_WARN("fail to get cipher mode", K(ret)); + } else if (!ObEncryptionUtil::is_aes_encryption(op_mode)) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "using aes_decrypt with not aes block_encryption_mode"); + } else if (OB_FAIL(eval_decrypt(expr, ctx, op_mode, func_name, res))) { + LOG_WARN("failed to eval aes decrypt", K(ret)); + } else { /* do nothing */ } + return ret; +} + +ObExprSm4Encrypt::ObExprSm4Encrypt(ObIAllocator& alloc) + : ObExprBaseEncrypt(alloc, T_FUN_SYS_SM4_ENCRYPT, N_SM4_ENCRYPT) { } + +ObExprSm4Encrypt::~ObExprSm4Encrypt() { } + +int ObExprSm4Encrypt::cg_expr(ObExprCGCtx &expr_cg_ctx, + const ObRawExpr &raw_expr, + ObExpr &rt_expr) const +{ + int ret = OB_SUCCESS; + UNUSED(expr_cg_ctx); + UNUSED(raw_expr); + rt_expr.eval_func_ = eval_sm4_encrypt; + return ret; +} + +int ObExprSm4Encrypt::eval_sm4_encrypt(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) +{ + int ret = OB_SUCCESS; + ObCipherOpMode op_mode = ObCipherOpMode::ob_invalid_mode; + ObString func_name(strlen(N_SM4_ENCRYPT), N_SM4_ENCRYPT); +#ifdef OB_USE_BABASSL + if (OB_FAIL(ObEncryptionUtil::get_cipher_op_mode(op_mode, ctx.exec_ctx_.get_my_session()))) { + LOG_WARN("fail to get cipher mode", K(ret)); + } else if (!ObEncryptionUtil::is_sm4_encryption(op_mode)) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "using sm4_encrypt with not sm4 block_encryption_mode"); + } else if (OB_FAIL(eval_encrypt(expr, ctx, op_mode, func_name, res))) { + LOG_WARN("failed to eval sm4 encrypt", K(ret)); + } else { /* do nothing */ } +#else + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "Function sm4_encrypt"); + // we use openssl 1.1.1 in opensource mode, which has no sm4 encryption until in 3.0 version +#endif + return ret; +} + +ObExprSm4Decrypt::ObExprSm4Decrypt(ObIAllocator& alloc) + : ObExprBaseDecrypt(alloc, T_FUN_SYS_SM4_DECRYPT, N_SM4_DECRYPT) { } + +ObExprSm4Decrypt::~ObExprSm4Decrypt() { } + +int ObExprSm4Decrypt::cg_expr(ObExprCGCtx &expr_cg_ctx, + const ObRawExpr &raw_expr, + ObExpr &rt_expr) const +{ + int ret = OB_SUCCESS; + UNUSED(expr_cg_ctx); + UNUSED(raw_expr); + rt_expr.eval_func_ = eval_sm4_decrypt; + return ret; +} + +int ObExprSm4Decrypt::eval_sm4_decrypt(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) +{ + int ret = OB_SUCCESS; + ObCipherOpMode op_mode = ObCipherOpMode::ob_invalid_mode; + ObString func_name(strlen(N_SM4_DECRYPT), N_SM4_DECRYPT); +#ifdef OB_USE_BABASSL + if (OB_FAIL(ObEncryptionUtil::get_cipher_op_mode(op_mode, ctx.exec_ctx_.get_my_session()))) { + LOG_WARN("fail to get cipher mode", K(ret)); + } else if (!ObEncryptionUtil::is_sm4_encryption(op_mode)) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "using sm4_decrypt with not sm4 block_encryption_mode"); + } else if (OB_FAIL(eval_decrypt(expr, ctx, op_mode, func_name, res))) { + LOG_WARN("failed to eval aes decrypt", K(ret)); + } else { /* do nothing */ } +#else + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "Function sm4_decrypt"); + // we use openssl 1.1.1 in opensource mode, which has no sm4 encryption until in 3.0 version +#endif + return ret; +} + +} +} diff --git a/src/sql/engine/expr/ob_expr_symmetric_encrypt.h b/src/sql/engine/expr/ob_expr_symmetric_encrypt.h new file mode 100644 index 000000000..e6fb29591 --- /dev/null +++ b/src/sql/engine/expr/ob_expr_symmetric_encrypt.h @@ -0,0 +1,118 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef SRC_SQL_ENGINE_EXPR_OB_EXPR_SYMMETRIC_ENCRYPT_H_ +#define SRC_SQL_ENGINE_EXPR_OB_EXPR_SYMMETRIC_ENCRYPT_H_ +#include "sql/engine/expr/ob_expr_operator.h" + +namespace oceanbase +{ +namespace sql +{ +class ObExprBaseEncrypt : public ObFuncExprOperator +{ +public: + ObExprBaseEncrypt(); + explicit ObExprBaseEncrypt(common::ObIAllocator& alloc, ObItemType func_type, const char* name); + virtual ~ObExprBaseEncrypt(); + virtual int calc_result_typeN(ObExprResType& type, + ObExprResType* types, + int64_t param_num, + common::ObExprTypeCtx& type_ctx) const override; + static int eval_encrypt(const ObExpr &expr, + ObEvalCtx &ctx, + const ObCipherOpMode op_mode, + const ObString &func_name, + ObDatum &res); +}; + +class ObExprBaseDecrypt : public ObFuncExprOperator +{ +public: + ObExprBaseDecrypt(); + explicit ObExprBaseDecrypt(common::ObIAllocator& alloc, ObItemType func_type, const char* name); + virtual ~ObExprBaseDecrypt(); + virtual int calc_result_typeN(ObExprResType& type, + ObExprResType* types, + int64_t param_num, + common::ObExprTypeCtx& type_ctx) const override; + static int eval_decrypt(const ObExpr &expr, + ObEvalCtx &ctx, + const ObCipherOpMode op_mode, + const ObString &func_name, + ObDatum &res); +}; + +class ObExprAesEncrypt : public ObExprBaseEncrypt +{ +public: + ObExprAesEncrypt(); + explicit ObExprAesEncrypt(common::ObIAllocator& alloc); + virtual ~ObExprAesEncrypt(); + virtual int cg_expr(ObExprCGCtx &expr_cg_ctx, + const ObRawExpr &raw_expr, + ObExpr &rt_expr) const override; + static int eval_aes_encrypt(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res); +private: + DISALLOW_COPY_AND_ASSIGN(ObExprAesEncrypt); +}; + +class ObExprAesDecrypt : public ObExprBaseDecrypt +{ +public: + ObExprAesDecrypt(); + explicit ObExprAesDecrypt(common::ObIAllocator& alloc); + virtual ~ObExprAesDecrypt(); + virtual int cg_expr(ObExprCGCtx &expr_cg_ctx, + const ObRawExpr &raw_expr, + ObExpr &rt_expr) const override; + static int eval_aes_decrypt(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res); +private: + DISALLOW_COPY_AND_ASSIGN(ObExprAesDecrypt); +}; + +class ObExprSm4Encrypt : public ObExprBaseEncrypt +{ +public: + ObExprSm4Encrypt(); + explicit ObExprSm4Encrypt(common::ObIAllocator& alloc); + virtual ~ObExprSm4Encrypt(); + virtual int cg_expr(ObExprCGCtx &expr_cg_ctx, + const ObRawExpr &raw_expr, + ObExpr &rt_expr) const override; + static int eval_sm4_encrypt(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res); +private: + DISALLOW_COPY_AND_ASSIGN(ObExprSm4Encrypt); +}; + +class ObExprSm4Decrypt : public ObExprBaseDecrypt +{ +public: + ObExprSm4Decrypt(); + explicit ObExprSm4Decrypt(common::ObIAllocator& alloc); + virtual ~ObExprSm4Decrypt(); + virtual int cg_expr(ObExprCGCtx &expr_cg_ctx, + const ObRawExpr &raw_expr, + ObExpr &rt_expr) const override; + static int eval_sm4_decrypt(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res); +private: + DISALLOW_COPY_AND_ASSIGN(ObExprSm4Decrypt); +}; + +} +} + + + + + +#endif diff --git a/src/sql/engine/expr/ob_expr_sys_connect_by_path.cpp b/src/sql/engine/expr/ob_expr_sys_connect_by_path.cpp index 1bf8b3ae7..8b8b8dafa 100644 --- a/src/sql/engine/expr/ob_expr_sys_connect_by_path.cpp +++ b/src/sql/engine/expr/ob_expr_sys_connect_by_path.cpp @@ -45,9 +45,9 @@ int ObExprSysConnectByPath::calc_result_type2(ObExprResType &type, type2.set_calc_type(ObVarcharType); types[0] = type1; types[1] = type2; - if (OB_FAIL(aggregate_charsets_for_string_result(type, types, 2, type_ctx.get_coll_type()))) { + if (OB_FAIL(aggregate_charsets_for_string_result(type, types, 2, type_ctx))) { LOG_WARN("fail to aggregate charset", K(type1), K(type2), K(ret)); - } else if (OB_FAIL(aggregate_charsets_for_comparison(type, types, 2, type_ctx.get_coll_type()))) { + } else if (OB_FAIL(aggregate_charsets_for_comparison(type, types, 2, type_ctx))) { LOG_WARN("fail to aggregate charset", K(type1), K(type2), K(ret)); } } diff --git a/src/sql/engine/expr/ob_expr_to_multi_byte.cpp b/src/sql/engine/expr/ob_expr_to_multi_byte.cpp index 5dca9dd7a..adc395e27 100644 --- a/src/sql/engine/expr/ob_expr_to_multi_byte.cpp +++ b/src/sql/engine/expr/ob_expr_to_multi_byte.cpp @@ -75,10 +75,12 @@ int calc_to_multi_byte_expr(const ObString &input, const ObCollationType cs_type { int ret = OB_SUCCESS; int64_t min_char_width = 0; - + int64_t max_char_width = 0; if (OB_FAIL(ObCharset::get_mbminlen_by_coll(cs_type, min_char_width))) { LOG_WARN("fail to get mbminlen", K(ret)); - } else if (min_char_width > 1) { + } else if (OB_FAIL(ObCharset::get_mbmaxlen_by_coll(cs_type, max_char_width))) { + LOG_WARN("fail to get mbminlen", K(ret)); + } else if (min_char_width > 1 || max_char_width == 1) { ObDataBuffer allocator(buf, buf_len); ObString output; diff --git a/src/sql/engine/expr/ob_expr_trim.cpp b/src/sql/engine/expr/ob_expr_trim.cpp index 0571ac6de..7ee482665 100644 --- a/src/sql/engine/expr/ob_expr_trim.cpp +++ b/src/sql/engine/expr/ob_expr_trim.cpp @@ -356,7 +356,7 @@ inline int ObExprTrim::deduce_result_type(ObExprResType &type, // OZ(tmp_types.push_back(*pattern_type)); //} OZ(aggregate_charsets_for_string_result_with_comparison( - type, &tmp_types.at(0), tmp_types.count(), type_ctx.get_coll_type())); + type, &tmp_types.at(0), tmp_types.count(), type_ctx)); str_type->set_calc_collation_type(type.get_collation_type()); str_type->set_calc_collation_level(type.get_collation_level()); if (NULL != pattern_type) { @@ -977,7 +977,7 @@ inline int ObExprLtrim::deduce_result_type(ObExprResType &type, OZ(tmp_types.push_back(*pattern_type)); } OZ(aggregate_charsets_for_string_result_with_comparison( - type, &tmp_types.at(0), tmp_types.count(), type_ctx.get_coll_type())); + type, &tmp_types.at(0), tmp_types.count(), type_ctx)); str_type->set_calc_collation_type(type.get_collation_type()); str_type->set_calc_collation_level(type.get_collation_level()); if (NULL != pattern_type) { diff --git a/src/sql/engine/expr/ob_expr_type_to_str.cpp b/src/sql/engine/expr/ob_expr_type_to_str.cpp index c5205631c..2c7b884d0 100644 --- a/src/sql/engine/expr/ob_expr_type_to_str.cpp +++ b/src/sql/engine/expr/ob_expr_type_to_str.cpp @@ -72,6 +72,8 @@ int ObExprTypeToStr::calc_result_type2(ObExprResType &type, type.set_collation_type(type1.get_collation_type()); type.set_collation_level(CS_LEVEL_IMPLICIT); type.set_length(type1.get_length()); + type1.set_calc_collation_type(type.get_collation_type()); + type1.set_calc_collation_level(type.get_collation_level()); return ret; } @@ -417,7 +419,10 @@ int ObExprSetToInnerType::calc_result_type2(ObExprResType &type, int ret = OB_SUCCESS; type.set_type(ObSetInnerType); type.set_collation_type(type1.get_collation_type()); + type.set_collation_level(type1.get_collation_level()); type.set_length(type1.get_length()); + type1.set_calc_collation_type(type.get_collation_type()); + type1.set_calc_collation_level(type.get_collation_level()); return ret; } @@ -522,6 +527,8 @@ int ObExprEnumToInnerType::calc_result_type2(ObExprResType &type, type.set_type(ObEnumInnerType); type.set_collation_type(type1.get_collation_type()); type.set_length(type1.get_length()); + type1.set_calc_collation_type(type.get_collation_type()); + type1.set_calc_collation_level(type.get_collation_level()); return ret; } diff --git a/src/sql/engine/user_defined_function/ob_udf_util.cpp b/src/sql/engine/user_defined_function/ob_udf_util.cpp index e0c895c3e..d7ff4df11 100644 --- a/src/sql/engine/user_defined_function/ob_udf_util.cpp +++ b/src/sql/engine/user_defined_function/ob_udf_util.cpp @@ -201,11 +201,11 @@ int ObUdfUtil::calc_udf_result_type(common::ObIAllocator &allocator, case STRING_RESULT: { ObExprResType calc_type; calc_type.set_varchar(); - OZ(ObExprOperator::aggregate_charsets_for_string_result( - calc_type, param_types + i, 1, type_ctx.get_coll_type())); + OZ(ObExprOperator::aggregate_charsets_for_string_result(calc_type, param_types + i, 1, type_ctx)); if (OB_SUCC(ret)) { param_types[i].set_calc_type(ObVarcharType); param_types[i].set_calc_collation_type(calc_type.get_collation_type()); + param_types[i].set_calc_collation_level(calc_type.get_collation_level()); } break; } diff --git a/src/sql/executor/ob_cmd_executor.cpp b/src/sql/executor/ob_cmd_executor.cpp index 293e34af9..3fcedd28d 100644 --- a/src/sql/executor/ob_cmd_executor.cpp +++ b/src/sql/executor/ob_cmd_executor.cpp @@ -574,7 +574,19 @@ int ObCmdExecutor::execute(ObExecContext &ctx, ObICmd &cmd) case stmt::T_CHECKSUM_TABLE: case stmt::T_CACHE_INDEX: case stmt::T_LOAD_INDEX_INTO_CACHE: - case stmt::T_FLUSH_PRIVILEGES: { + case stmt::T_FLUSH_PRIVILEGES: + case stmt::T_INSTALL_PLUGIN: + case stmt::T_UNINSTALL_PLUGIN: + case stmt::T_FLUSH_MOCK: + case stmt::T_FLUSH_MOCK_LIST: + case stmt::T_HANDLER_MOCK: + case stmt::T_SHOW_PLUGINS: + case stmt::T_CREATE_SERVER: + case stmt::T_ALTER_SERVER: + case stmt::T_DROP_SERVER: + case stmt::T_CREATE_LOGFILE_GROUP: + case stmt::T_ALTER_LOGFILE_GROUP: + case stmt::T_DROP_LOGFILE_GROUP: { DEFINE_EXECUTE_CMD(ObMockStmt, ObMockExecutor); break; } diff --git a/src/sql/monitor/ob_sql_plan.cpp b/src/sql/monitor/ob_sql_plan.cpp index 61aa5a776..77284ca27 100644 --- a/src/sql/monitor/ob_sql_plan.cpp +++ b/src/sql/monitor/ob_sql_plan.cpp @@ -886,7 +886,7 @@ int ObSqlPlan::get_plan_other_info(PlanText &plan_text, } else if (OB_FAIL(BUF_PRINTF(" Parameters:"))) { } else if (OB_FAIL(BUF_PRINTF(NEW_LINE))) { } else { /* Do nothing */ } - for (int64_t i = 0; OB_SUCC(ret) && NULL != params && i < params->count(); i++) { + for (int64_t i = 0; OB_SUCC(ret) && NULL != params && i < params->count() && i < 100; i++) { if (OB_FAIL(BUF_PRINTF(OUTPUT_PREFIX))) { } else if (OB_FAIL(BUF_PRINTF(":%ld => ", i))) { } else { diff --git a/src/sql/ob_sql_init.h b/src/sql/ob_sql_init.h index 60325105f..b1b04970a 100644 --- a/src/sql/ob_sql_init.h +++ b/src/sql/ob_sql_init.h @@ -75,10 +75,10 @@ inline int init_sql_expr_static_var() SQL_LOG(ERROR, "failed to init ObNumberConstValue", K(ret)); } else if (OB_FAIL(ARITH_RESULT_TYPE_ORACLE.init())) { SQL_LOG(ERROR, "failed to init ORACLE_ARITH_RESULT_TYPE", K(ret)); - } else if (OB_FAIL(ObCharsetUtils::init(*allocator))) { - SQL_LOG(ERROR, "fail to init ObCharsetUtils", K(ret)); } else if (OB_FAIL(ObCharset::init_charset())) { SQL_LOG(ERROR, "fail to init charset", K(ret)); + } else if (OB_FAIL(ObCharsetUtils::init(*allocator))) { + SQL_LOG(ERROR, "fail to init ObCharsetUtils", K(ret)); } else if (OB_FAIL(wide::ObDecimalIntConstValue::init_const_values(*allocator, attr))) { SQL_LOG(ERROR, "failed to init ObDecimalIntConstValue", K(ret)); } diff --git a/src/sql/ob_sql_utils.cpp b/src/sql/ob_sql_utils.cpp index 0066f3cd7..16229f1a4 100644 --- a/src/sql/ob_sql_utils.cpp +++ b/src/sql/ob_sql_utils.cpp @@ -60,6 +60,9 @@ #include "sql/executor/ob_maintain_dependency_info_task.h" #include "sql/resolver/ddl/ob_create_view_resolver.h" #include "sql/resolver/dcl/ob_dcl_resolver.h" +#ifdef OB_BUILD_AUDIT_SECURITY +#include "sql/audit/ob_audit_log_utils.h" +#endif extern "C" { #include "sql/parser/ob_non_reserved_keywords.h" } @@ -628,6 +631,12 @@ int ObSQLUtils::is_charset_data_version_valid(ObCharsetType charset_type, const ret = OB_NOT_SUPPORTED; SQL_LOG(WARN, "GB18030_2022 not supported when data_version < 4_2_0_0", K(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "tenant data version is less than 4.2, charset GB18030_2022 is"); + } else if ((CHARSET_ASCII == charset_type || CHARSET_TIS620 == charset_type) && + ((data_version < MOCK_DATA_VERSION_4_2_4_0) || + (DATA_VERSION_4_3_0_0 <= data_version && data_version < DATA_VERSION_4_3_3_0))) { + ret = OB_NOT_SUPPORTED; + SQL_LOG(WARN, "charset not supported when data_version < 4_2_4_0 or between [430,433)",K(charset_type), K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "tenant data version is less than 4.2.4 or between [430,433), charset is"); } return ret; } @@ -635,18 +644,38 @@ int ObSQLUtils::is_charset_data_version_valid(ObCharsetType charset_type, const int ObSQLUtils::is_collation_data_version_valid(ObCollationType collation_type, const int64_t tenant_id) { int ret = OB_SUCCESS; -#ifndef OB_BUILD_CLOSE_MODULES uint64_t data_version = 0; if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, data_version))) { SQL_LOG(WARN, "failed to GET_MIN_DATA_VERSION", K(ret)); - } else if (data_version < DATA_VERSION_4_2_2_0 && - (CS_TYPE_UTF16_UNICODE_CI == collation_type || - CS_TYPE_UTF8MB4_UNICODE_CI == collation_type)) { + } else if ((data_version < MOCK_DATA_VERSION_4_2_4_0 + || (data_version >= DATA_VERSION_4_3_0_0 && data_version < DATA_VERSION_4_3_3_0)) + && (CS_TYPE_UTF8MB4_CROATIAN_CI == collation_type + || CS_TYPE_UTF8MB4_UNICODE_520_CI == collation_type + || CS_TYPE_UTF8MB4_CZECH_CI == collation_type)) { ret = OB_NOT_SUPPORTED; SQL_LOG(WARN, "Unicode collation not supported when data_version < 4_2_2_0", K(collation_type), K(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "tenant data version is less than 4.2.2, unicode collation is"); } +#ifndef OB_BUILD_CLOSE_MODULES + if (OB_SUCC(ret)) { + if (data_version < DATA_VERSION_4_2_2_0 && + (CS_TYPE_UTF16_UNICODE_CI == collation_type || + CS_TYPE_UTF8MB4_UNICODE_CI == collation_type)) { + ret = OB_NOT_SUPPORTED; + SQL_LOG(WARN, "Unicode collation not supported when data_version < 4_2_2_0", K(collation_type), K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "tenant data version is less than 4.2.2, unicode collation is"); + } + } #endif + if (OB_SUCC(ret)) { + if ((CS_TYPE_UTF8MB4_0900_AI_CI == collation_type) && + ((data_version < MOCK_DATA_VERSION_4_2_4_0) || + (DATA_VERSION_4_3_0_0 <= data_version && data_version < DATA_VERSION_4_3_3_0))) { + ret = OB_NOT_SUPPORTED; + SQL_LOG(WARN, "Unicode collation not supported when data_version < 4_2_4_0 or between [430,433)", K(collation_type), K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "tenant data version is less than 4.2.4 or between [430,433), collation is"); + } + } return ret; } @@ -1650,7 +1679,8 @@ bool ObSQLUtils::is_readonly_stmt(ParseResult &result) || T_SHOW_SEQUENCES == type || T_SHOW_ENGINE == type || T_SHOW_OPEN_TABLES == type - || (T_SET_ROLE == type && lib::is_mysql_mode())) { + || (T_SET_ROLE == type && lib::is_mysql_mode()) + || T_SHOW_CREATE_USER == type) { ret = true; } } @@ -1927,6 +1957,7 @@ int ObSQLUtils::get_cs_level_from_cast_mode(const ObCastMode cast_mode, } else { cs_level = tmp_cs_level; } + LOG_TRACE(" get_cs_level_from_cast_mode debug",K(default_level),K(cs_level)); } return ret; } @@ -4636,6 +4667,9 @@ int ObSQLUtils::handle_audit_record(bool need_retry, ret = OB_SUCCESS; } } +#ifdef OB_BUILD_AUDIT_SECURITY + (void) ObAuditLogUtils::handle_sql_audit_log(session, audit_record, is_sensitive); +#endif } } if (lib::is_diagnose_info_enabled()) { @@ -5900,3 +5934,8 @@ bool ObSQLUtils::is_data_version_ge_423_or_432(uint64_t data_version) { return ((MOCK_DATA_VERSION_4_2_3_0 <= data_version && data_version < DATA_VERSION_4_3_0_0) || data_version >= DATA_VERSION_4_3_2_0); } + +bool ObSQLUtils::is_data_version_ge_424_or_433(uint64_t data_version) +{ + return ((MOCK_DATA_VERSION_4_2_4_0 <= data_version && data_version < DATA_VERSION_4_3_0_0) || data_version >= DATA_VERSION_4_3_3_0); +} diff --git a/src/sql/ob_sql_utils.h b/src/sql/ob_sql_utils.h index ccee984bd..78fee24a4 100644 --- a/src/sql/ob_sql_utils.h +++ b/src/sql/ob_sql_utils.h @@ -722,6 +722,7 @@ public: static bool is_data_version_ge_422_or_431(uint64_t data_version); static bool is_data_version_ge_423_or_431(uint64_t data_version); static bool is_data_version_ge_423_or_432(uint64_t data_version); + static bool is_data_version_ge_424_or_433(uint64_t data_version); static int get_proxy_can_activate_role(const ObIArray &role_id_array, const ObIArray &role_id_option_array, diff --git a/src/sql/optimizer/ob_join_order.cpp b/src/sql/optimizer/ob_join_order.cpp index e56c5e993..516d223b0 100644 --- a/src/sql/optimizer/ob_join_order.cpp +++ b/src/sql/optimizer/ob_join_order.cpp @@ -1629,7 +1629,7 @@ int ObJoinOrder::will_use_das(const uint64_t table_id, } if (OB_SUCC(ret) && get_plan()->get_stmt()->is_select_stmt()) { const ObSelectStmt *stmt = static_cast(get_plan()->get_stmt()); - const SampleInfo *sample_info = stmt->get_sample_info_by_table_id(table_id); + const SampleInfo *sample_info = table_item->sample_info_; if (sample_info != NULL && !sample_info->is_no_sample()) { is_sample_stmt = true; } @@ -1778,8 +1778,9 @@ int ObJoinOrder::create_one_access_path(const uint64_t table_id, ap->use_skip_scan_ = use_skip_scan; ap->use_column_store_ = use_column_store; ap->est_cost_info_.use_column_store_ = use_column_store; + ap->contain_das_op_ = ap->use_das_; - if (OB_FAIL(init_sample_info_for_access_path(ap, table_id))) { + if (OB_FAIL(init_sample_info_for_access_path(ap, table_id, table_item))) { LOG_WARN("failed to init sample info", K(ret)); } else if (OB_FAIL(add_access_filters(ap, ordering_info.get_index_keys(), @@ -1852,7 +1853,8 @@ int ObJoinOrder::create_one_access_path(const uint64_t table_id, } int ObJoinOrder::init_sample_info_for_access_path(AccessPath *ap, - const uint64_t table_id) + const uint64_t table_id, + const TableItem *table_item) { int ret = OB_SUCCESS; if (OB_ISNULL(get_plan()) || OB_ISNULL(get_plan()->get_stmt())) { @@ -1863,7 +1865,8 @@ int ObJoinOrder::init_sample_info_for_access_path(AccessPath *ap, // sample scan doesn't support DML other than SELECT. } else { const ObSelectStmt *stmt = static_cast(get_plan()->get_stmt()); - const SampleInfo *sample_info = stmt->get_sample_info_by_table_id(table_id); + const SampleInfo *sample_info = table_item->sample_info_; + if (sample_info != NULL) { ap->sample_info_ = *sample_info; ap->sample_info_.table_id_ = ap->get_index_table_id(); @@ -2891,12 +2894,13 @@ int ObJoinOrder::will_use_skip_scan(const uint64_t table_id, use_skip_scan = OptSkipScanState::SS_UNSET; IndexInfoEntry *index_info_entry = NULL; const ObQueryRange *query_range = NULL; + ObQueryCtx *query_ctx = NULL; bool hint_force_skip_scan = false; bool hint_force_no_skip_scan = false; if (OB_UNLIKELY(OB_INVALID_ID == ref_id) || OB_UNLIKELY(OB_INVALID_ID == index_id) || - OB_ISNULL(get_plan())) { + OB_ISNULL(get_plan()) || OB_ISNULL(query_ctx = get_plan()->get_optimizer_context().get_query_ctx())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ref_id), K(index_id), K(get_plan()), K(ret)); + LOG_WARN("get unexpected null", K(ref_id), K(index_id), K(get_plan()), K(query_ctx), K(ret)); } else if (is_virtual_table(ref_id)) { use_skip_scan = OptSkipScanState::SS_DISABLE; } else if (OB_FAIL(index_info_cache.get_index_info_entry(table_id, index_id, @@ -2917,7 +2921,7 @@ int ObJoinOrder::will_use_skip_scan(const uint64_t table_id, use_skip_scan = OptSkipScanState::SS_HINT_ENABLE; } else if (hint_force_no_skip_scan) { use_skip_scan = OptSkipScanState::SS_DISABLE; - } else if (!session_info->is_index_skip_scan_enabled()) { + } else if (!OPT_CTX.get_is_skip_scan_enabled()) { use_skip_scan = OptSkipScanState::SS_DISABLE; } else if (helper.is_inner_path_ || get_tables().is_subset(get_plan()->get_subq_pdfilter_tset())) { use_skip_scan = OptSkipScanState::SS_DISABLE; @@ -3816,6 +3820,7 @@ int ObJoinOrder::extract_preliminary_query_range(const ObIArray &ran LOG_WARN("failed to check in range optimization enabled", K(ret)); } else if (OB_FAIL(tmp_qr->preliminary_extract_query_range(range_columns, range_predicates, dtc_params, opt_ctx->get_exec_ctx(), + opt_ctx->get_query_ctx(), &expr_constraints, params, false, true, is_in_range_optimization_enabled, @@ -3840,17 +3845,18 @@ int ObJoinOrder::check_enable_better_inlist(int64_t table_id, bool &enable) int ret = OB_SUCCESS; enable = false; ObOptimizerContext *opt_ctx = NULL; + ObQueryCtx *query_ctx = NULL; ObSQLSessionInfo *session_info = NULL; OptTableMeta *table_meta = NULL; if (OB_ISNULL(get_plan()) || OB_ISNULL(opt_ctx = &get_plan()->get_optimizer_context()) || - OB_ISNULL(session_info = opt_ctx->get_session_info())) { + OB_ISNULL(session_info = opt_ctx->get_session_info()) || + OB_ISNULL(query_ctx = get_plan()->get_optimizer_context().get_query_ctx())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("get unexpected null", K(get_plan()), K(opt_ctx), K(ret)); + LOG_WARN("get unexpected null", K(get_plan()), K(opt_ctx), K(session_info), K(query_ctx), K(ret)); } else if (!session_info->is_user_session()) { enable = false; - } else if (session_info->is_better_inlist_enabled(enable)) { - LOG_WARN("failed to check better inlist enabled", K(ret)); + } else if (OB_FALSE_IT(enable = opt_ctx->get_enable_better_inlist_costing())) { } else if (!enable) { //do nothing } else if (OB_ISNULL(table_meta=get_plan()->get_basic_table_metas(). @@ -3902,6 +3908,7 @@ int ObJoinOrder::extract_geo_preliminary_query_range(const ObIArray if (OB_FAIL(ret)) { } else if (OB_FAIL(tmp_qr->preliminary_extract_query_range(range_columns, predicates, dtc_params, opt_ctx->get_exec_ctx(), + opt_ctx->get_query_ctx(), NULL, params))) { LOG_WARN("failed to preliminary extract query range", K(ret)); } @@ -3949,6 +3956,7 @@ int ObJoinOrder::extract_multivalue_preliminary_query_range(const ObIArraypreliminary_extract_query_range(range_columns, predicates, dtc_params, opt_ctx->get_exec_ctx(), + opt_ctx->get_query_ctx(), NULL, params, false, true, is_in_range_optimization_enabled))) { LOG_WARN("failed to preliminary extract query range", K(ret)); @@ -5996,6 +6004,7 @@ int AccessPath::check_adj_index_cost_valid(double &stats_phy_query_range_row_cou ObLogPlan *plan = NULL; ObOptimizerContext *opt_ctx = NULL; ObSQLSessionInfo *session_info = NULL; + ObQueryCtx *query_ctx = NULL; const OptTableMeta* table_meta = NULL; bool enable_adj_index_cost = false; opt_stats_cost_percent = 0; @@ -6003,12 +6012,12 @@ int AccessPath::check_adj_index_cost_valid(double &stats_phy_query_range_row_cou if (OB_ISNULL(parent_) || OB_ISNULL(plan = parent_->get_plan()) || OB_ISNULL(opt_ctx = &plan->get_optimizer_context()) || OB_ISNULL(session_info = opt_ctx->get_session_info()) || + OB_ISNULL(query_ctx = opt_ctx->get_query_ctx()) || OB_ISNULL(table_meta = plan->get_basic_table_metas().get_table_meta_by_table_id(table_id_))) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("get unexpected null", K(plan), K(opt_ctx), K(ret)); - } else if (session_info->is_adj_index_cost_enabled(enable_adj_index_cost, opt_stats_cost_percent)) { - LOG_WARN("failed to check adjust scan enabled", K(ret)); - } else if (!enable_adj_index_cost || //session disable adjust + LOG_WARN("get unexpected null", K(plan), K(opt_ctx), K(query_ctx), K(ret)); + } else if (OB_FALSE_IT(opt_stats_cost_percent = opt_ctx->get_optimizer_index_cost_adj())) { + } else if (0 == opt_stats_cost_percent || //session disable adjust est_cost_info_.prefix_filters_.empty() || //not have query range !est_cost_info_.pushdown_prefix_filters_.empty() || //can not use storage estimate table_meta->use_default_stat()) { //not have optimzier stats @@ -6890,16 +6899,17 @@ int JoinPath::can_use_batch_nlj(bool &use_batch_nlj) int ret = OB_SUCCESS; const ParamStore *params = NULL; ObSQLSessionInfo *session_info = NULL; - bool enable_use_batch_nlj = false; const AccessPath *access_path = NULL; const SubQueryPath *subq_path = NULL; ObLogPlan *plan = NULL; + ObQueryCtx *query_ctx = NULL; use_batch_nlj = false; if (OB_ISNULL(parent_) || OB_ISNULL(plan = parent_->get_plan()) || OB_ISNULL(right_path_) || OB_ISNULL(session_info = plan->get_optimizer_context().get_session_info()) || - OB_ISNULL(params = plan->get_optimizer_context().get_params())) { + OB_ISNULL(params = plan->get_optimizer_context().get_params()) || + OB_ISNULL(query_ctx = plan->get_optimizer_context().get_query_ctx())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid argument", K(session_info), K(ret)); + LOG_WARN("invalid argument", K(session_info), K(params), K(query_ctx), K(ret)); } else if (NESTED_LOOP_JOIN == join_algo_ && CONNECT_BY_JOIN != join_type_ && (!IS_SEMI_ANTI_JOIN(join_type_)) @@ -6920,9 +6930,7 @@ int JoinPath::can_use_batch_nlj(bool &use_batch_nlj) } else { right_has_gi_or_exchange = true; } - if (OB_FAIL(session_info->get_nlj_batching_enabled(enable_use_batch_nlj))) { - LOG_WARN("failed to get join cache size variable", K(ret)); - } else if (!enable_use_batch_nlj) { + if (!plan->get_optimizer_context().get_nlj_batching_enabled()) { //do nothing } else if ((!right_path_->is_access_path() && !right_path_->is_subquery_path()) || right_has_gi_or_exchange) { diff --git a/src/sql/optimizer/ob_join_order.h b/src/sql/optimizer/ob_join_order.h index 846e2c27e..a7adfa22c 100644 --- a/src/sql/optimizer/ob_join_order.h +++ b/src/sql/optimizer/ob_join_order.h @@ -1516,7 +1516,8 @@ struct NullAwareAntiJoinInfo { OptSkipScanState use_skip_scan); int init_sample_info_for_access_path(AccessPath *ap, - const uint64_t table_id); + const uint64_t table_id, + const TableItem *table_item); int init_filter_selectivity(ObCostTableScanInfo &est_cost_info); diff --git a/src/sql/optimizer/ob_log_join.cpp b/src/sql/optimizer/ob_log_join.cpp index f53e18745..707ab3184 100644 --- a/src/sql/optimizer/ob_log_join.cpp +++ b/src/sql/optimizer/ob_log_join.cpp @@ -1349,14 +1349,15 @@ int ObLogJoin::check_and_set_use_batch() int ret = OB_SUCCESS; ObSQLSessionInfo *session_info = NULL; ObLogPlan *plan = NULL; + ObQueryCtx *query_ctx = NULL; if (OB_ISNULL(plan = get_plan()) - || OB_ISNULL(session_info = plan->get_optimizer_context().get_session_info())) { + || OB_ISNULL(session_info = plan->get_optimizer_context().get_session_info()) + || OB_ISNULL(query_ctx = plan->get_optimizer_context().get_query_ctx())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", K(ret)); + LOG_WARN("unexpected null", K(ret), K(plan), K(session_info), K(query_ctx)); } else if (!can_use_batch_nlj_) { // do nothing - } else if (OB_FAIL(session_info->get_nlj_batching_enabled(can_use_batch_nlj_))) { - LOG_WARN("failed to get enable batch variable", K(ret)); + } else if (OB_FALSE_IT(can_use_batch_nlj_ = plan->get_optimizer_context().get_nlj_batching_enabled())) { } else if (NESTED_LOOP_JOIN != get_join_algo()) { can_use_batch_nlj_ = false; } diff --git a/src/sql/optimizer/ob_log_subplan_filter.cpp b/src/sql/optimizer/ob_log_subplan_filter.cpp index b62c58abe..eaffed1b7 100644 --- a/src/sql/optimizer/ob_log_subplan_filter.cpp +++ b/src/sql/optimizer/ob_log_subplan_filter.cpp @@ -523,15 +523,17 @@ int ObLogSubPlanFilter::check_and_set_das_group_rescan() { int ret = OB_SUCCESS; ObSQLSessionInfo *session_info = NULL; + ObQueryCtx *query_ctx = NULL; ObLogPlan *plan = NULL; if (OB_ISNULL(plan = get_plan()) - || OB_ISNULL(session_info = plan->get_optimizer_context().get_session_info())) { + || OB_ISNULL(session_info = plan->get_optimizer_context().get_session_info()) + || OB_ISNULL(query_ctx = plan->get_optimizer_context().get_query_ctx())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", K(ret)); - } else if (!session_info->is_spf_mlj_group_rescan_enabled()) { + LOG_WARN("get unexpected null", K(ret), K(plan), K(session_info), K(query_ctx)); + } else if (!plan->get_optimizer_context().get_enable_spf_batch_rescan()) { enable_das_group_rescan_ = false; - } else if (OB_FAIL(session_info->get_nlj_batching_enabled(enable_das_group_rescan_))) { - LOG_WARN("failed to get enable batch variable", K(ret)); + } else { + enable_das_group_rescan_ = plan->get_optimizer_context().get_nlj_batching_enabled(); } // check use batch for (int64_t i = 1; OB_SUCC(ret) && enable_das_group_rescan_ && i < get_num_of_child(); i++) { diff --git a/src/sql/optimizer/ob_opt_selectivity.cpp b/src/sql/optimizer/ob_opt_selectivity.cpp index cb7684345..6721a1a35 100644 --- a/src/sql/optimizer/ob_opt_selectivity.cpp +++ b/src/sql/optimizer/ob_opt_selectivity.cpp @@ -2644,6 +2644,7 @@ int ObOptSelectivity::get_column_query_range(const OptSelectivityCtx &ctx, quals, dtc_params, ctx.get_opt_ctx().get_exec_ctx(), + ctx.get_opt_ctx().get_query_ctx(), NULL, params, false, diff --git a/src/sql/optimizer/ob_optimizer.cpp b/src/sql/optimizer/ob_optimizer.cpp index 27208a35e..52444966d 100644 --- a/src/sql/optimizer/ob_optimizer.cpp +++ b/src/sql/optimizer/ob_optimizer.cpp @@ -566,6 +566,15 @@ int ObOptimizer::extract_opt_ctx_basic_flags(const ObDMLStmt &stmt, ObSQLSession bool rowsets_enabled = tenant_config.is_valid() && tenant_config->_rowsets_enabled; ctx_.set_is_online_ddl(session.get_ddl_info().is_ddl()); // set is online ddl first, is used by other extract operations bool das_keep_order_enabled = tenant_config.is_valid() && tenant_config->_enable_das_keep_order; + bool hash_join_enabled = tenant_config.is_valid() && tenant_config->_hash_join_enabled; + bool optimizer_sortmerge_join_enabled = tenant_config.is_valid() && tenant_config->_optimizer_sortmerge_join_enabled; + bool nested_loop_join_enabled = tenant_config.is_valid() && tenant_config->_nested_loop_join_enabled; + bool enable_adj_index_cost = false; + int64_t optimizer_index_cost_adj = 0; + bool is_skip_scan_enable = session.is_index_skip_scan_enabled(); + bool enable_use_batch_nlj = false; + bool better_inlist_costing = false; + bool enable_spf_batch_rescan = session.is_spf_mlj_group_rescan_enabled(); const ObOptParamHint &opt_params = ctx_.get_global_hint().opt_params_; if (OB_FAIL(check_whether_contain_nested_sql(stmt))) { LOG_WARN("check whether contain nested sql failed", K(ret)); @@ -583,7 +592,7 @@ int ObOptimizer::extract_opt_ctx_basic_flags(const ObDMLStmt &stmt, ObSQLSession LOG_WARN("calc link stmt count failed", K(ret)); } else if (OB_FAIL(ObDblinkUtils::has_reverse_link_or_any_dblink(&stmt, has_dblink, true))) { LOG_WARN("failed to find dblink in stmt", K(ret)); - } else if (OB_FAIL(ctx_.get_global_hint().opt_params_.get_bool_opt_param(ObOptParamHint::ROWSETS_ENABLED, rowsets_enabled))) { + } else if (OB_FAIL(opt_params.get_bool_opt_param(ObOptParamHint::ROWSETS_ENABLED, rowsets_enabled))) { LOG_WARN("fail to check rowsets enabled", K(ret)); } else if (OB_FAIL(stmt.check_has_cursor_expression(has_cursor_expr))) { LOG_WARN("fail to check cursor expression info", K(ret)); @@ -591,6 +600,28 @@ int ObOptimizer::extract_opt_ctx_basic_flags(const ObDMLStmt &stmt, ObSQLSession LOG_WARN("fail to get storage_estimation_enabled", K(ret)); } else if (OB_FAIL(opt_params.get_bool_opt_param(ObOptParamHint::ENABLE_DAS_KEEP_ORDER, das_keep_order_enabled))) { LOG_WARN("failed to check das keep order enabled", K(ret)); + } else if (OB_FAIL(opt_params.get_bool_opt_param(ObOptParamHint::HASH_JOIN_ENABLED, hash_join_enabled))) { + LOG_WARN("failed to check hash join enabled", K(ret)); + } else if (OB_FAIL(opt_params.get_bool_opt_param(ObOptParamHint::OPTIMIZER_SORTMERGE_JOIN_ENABLED, optimizer_sortmerge_join_enabled))) { + LOG_WARN("failed to check merge join enabled", K(ret)); + } else if (OB_FAIL(opt_params.get_bool_opt_param(ObOptParamHint::NESTED_LOOP_JOIN_ENABLED, nested_loop_join_enabled))) { + LOG_WARN("failed to check nested loop join enabled", K(ret)); + } else if (OB_FAIL(session.is_adj_index_cost_enabled(enable_adj_index_cost, optimizer_index_cost_adj))) { + LOG_WARN("failed to check adjust index cost", K(ret)); + } else if (OB_FAIL(opt_params.get_integer_opt_param(ObOptParamHint::OPTIMIZER_INDEX_COST_ADJ, optimizer_index_cost_adj))) { + LOG_WARN("fail to check opt param adjust index cost", K(ret)); + } else if (OB_FAIL(opt_params.get_bool_opt_param(ObOptParamHint::OPTIMIZER_SKIP_SCAN_ENABLED, is_skip_scan_enable))) { + LOG_WARN("failed to get opt param skip scan enabled", K(ret)); + } else if (OB_FAIL(session.is_better_inlist_enabled(better_inlist_costing))) { + LOG_WARN("failed to check better inlist enabled", K(ret)); + } else if (OB_FAIL(opt_params.get_bool_opt_param(ObOptParamHint::OPTIMIZER_BETTER_INLIST_COSTING, better_inlist_costing))) { + LOG_WARN("failed to get opt param better inlist costing", K(ret)); + } else if (OB_FAIL(session.get_nlj_batching_enabled(enable_use_batch_nlj))) { + LOG_WARN("failed to get join cache size variable", K(ret)); + } else if (OB_FAIL(opt_params.get_bool_opt_param(ObOptParamHint::NLJ_BATCHING_ENABLED, enable_use_batch_nlj))) { + LOG_WARN("failed to get opt param nlj batching enable", K(ret)); + } else if (OB_FAIL(opt_params.get_bool_opt_param(ObOptParamHint::ENABLE_SPF_BATCH_RESCAN, enable_spf_batch_rescan))) { + LOG_WARN("failed to get opt param enable spf batch rescan", K(ret)); } else { ctx_.set_storage_estimation_enabled(storage_estimation_enabled); ctx_.set_serial_set_order(force_serial_set_order); @@ -602,22 +633,26 @@ int ObOptimizer::extract_opt_ctx_basic_flags(const ObDMLStmt &stmt, ObSQLSession ctx_.set_cost_model_type(rowsets_enabled ? ObOptEstCost::VECTOR_MODEL : ObOptEstCost::NORMAL_MODEL); ctx_.set_has_cursor_expression(has_cursor_expr); ctx_.set_das_keep_order_enabled(GET_MIN_CLUSTER_VERSION() < CLUSTER_VERSION_4_3_2_0 ? false : das_keep_order_enabled); - if (!tenant_config.is_valid() || - (!tenant_config->_hash_join_enabled && - !tenant_config->_optimizer_sortmerge_join_enabled && - !tenant_config->_nested_loop_join_enabled)) { + ctx_.set_optimizer_index_cost_adj(optimizer_index_cost_adj); + ctx_.set_is_skip_scan_enabled(is_skip_scan_enable); + ctx_.set_enable_better_inlist_costing(better_inlist_costing); + ctx_.set_nlj_batching_enabled(enable_use_batch_nlj); + ctx_.set_enable_spf_batch_rescan(enable_spf_batch_rescan); + if (!hash_join_enabled + && !optimizer_sortmerge_join_enabled + && !nested_loop_join_enabled) { ctx_.set_hash_join_enabled(true); ctx_.set_merge_join_enabled(true); ctx_.set_nested_join_enabled(true); + LOG_TRACE("all join types are set to disable"); } else { - ctx_.set_hash_join_enabled(tenant_config->_hash_join_enabled); - ctx_.set_merge_join_enabled(tenant_config->_optimizer_sortmerge_join_enabled); - ctx_.set_nested_join_enabled(tenant_config->_nested_loop_join_enabled); + ctx_.set_hash_join_enabled(hash_join_enabled); + ctx_.set_merge_join_enabled(optimizer_sortmerge_join_enabled); + ctx_.set_nested_join_enabled(nested_loop_join_enabled); } if (!session.is_inner() && stmt.get_query_ctx()->get_injected_random_status()) { ctx_.set_generate_random_plan(true); } - //do nothing } return ret; } diff --git a/src/sql/optimizer/ob_optimizer_context.h b/src/sql/optimizer/ob_optimizer_context.h index bf43140ba..e43a5ef5a 100644 --- a/src/sql/optimizer/ob_optimizer_context.h +++ b/src/sql/optimizer/ob_optimizer_context.h @@ -244,6 +244,11 @@ ObOptimizerContext(ObSQLSessionInfo *session_info, storage_estimation_enabled_(false), das_keep_order_enabled_(true), generate_random_plan_(false), + optimizer_index_cost_adj_(0), + is_skip_scan_enabled_(false), + enable_better_inlist_costing_(false), + nlj_batching_enabled_(false), + enable_spf_batch_rescan_(false), correlation_type_(ObEstCorrelationType::MAX), use_column_store_replica_(false) { } @@ -443,6 +448,11 @@ ObOptimizerContext(ObSQLSessionInfo *session_info, } void get_runtime_filter_type() { runtime_filter_type_ = session_info_->get_runtime_filter_type(); + if (OB_ISNULL(query_ctx_)) { + // do nothing + } else { + query_ctx_->get_global_hint().opt_params_.get_opt_param_runtime_filter_type(runtime_filter_type_); + } } int64_t get_batch_size() const { return batch_size_; } @@ -619,6 +629,16 @@ ObOptimizerContext(ObSQLSessionInfo *session_info, inline const OptSystemStat& get_system_stat() const { return system_stat_; } inline bool generate_random_plan() const { return generate_random_plan_; } inline void set_generate_random_plan(bool rand_plan) { generate_random_plan_ = rand_plan; } + inline int64_t get_optimizer_index_cost_adj() const { return optimizer_index_cost_adj_; } + inline void set_optimizer_index_cost_adj(int64_t v) { optimizer_index_cost_adj_ = v; } + inline bool get_is_skip_scan_enabled() const { return is_skip_scan_enabled_; } + inline void set_is_skip_scan_enabled(bool v) { is_skip_scan_enabled_ = v; } + inline bool get_enable_better_inlist_costing() const { return enable_better_inlist_costing_; } + inline void set_enable_better_inlist_costing(bool v) { enable_better_inlist_costing_ = v; } + inline bool get_nlj_batching_enabled() const { return nlj_batching_enabled_; } + inline void set_nlj_batching_enabled(bool v) { nlj_batching_enabled_ = v; } + inline bool get_enable_spf_batch_rescan() const { return enable_spf_batch_rescan_; } + inline void set_enable_spf_batch_rescan(bool v) { enable_spf_batch_rescan_ = v; } inline bool use_column_store_replica() const { return use_column_store_replica_; } inline void set_use_column_store_replica(bool use) { use_column_store_replica_ = use; } @@ -710,6 +730,11 @@ private: bool das_keep_order_enabled_; bool generate_random_plan_; + int64_t optimizer_index_cost_adj_; + bool is_skip_scan_enabled_; + bool enable_better_inlist_costing_; + bool nlj_batching_enabled_; + bool enable_spf_batch_rescan_; ObEstCorrelationType correlation_type_; bool use_column_store_replica_; }; diff --git a/src/sql/optimizer/ob_optimizer_util.cpp b/src/sql/optimizer/ob_optimizer_util.cpp index 05b60f3da..3f8588596 100644 --- a/src/sql/optimizer/ob_optimizer_util.cpp +++ b/src/sql/optimizer/ob_optimizer_util.cpp @@ -6717,9 +6717,10 @@ int ObOptimizerUtil::get_set_res_types(ObIAllocator *allocator, LOG_WARN("failed to get collation connection", K(ret)); } else { ObExprVersion dummy_op(*allocator); - const ObLengthSemantics length_semantics = session_info->get_actual_nls_length_semantics(); ObSEArray types; ObExprResType res_type; + ObExprTypeCtx type_ctx; + ObSQLUtils::init_type_ctx(session_info, type_ctx); const int64_t child_num = child_querys.count(); const int64_t select_num = select_stmt->get_select_item_size(); ObSelectStmt *cur_stmt = NULL; @@ -6770,8 +6771,8 @@ int ObOptimizerUtil::get_set_res_types(ObIAllocator *allocator, } else if (1 == types.count() || all_types_is_decint) { ret = res_types.push_back(types.at(0)); } else if (OB_FAIL(dummy_op.aggregate_result_type_for_merge(res_type, &types.at(0), - types.count(), coll_type, is_oracle_mode(), - length_semantics))) { + types.count(), is_oracle_mode(), + type_ctx))) { LOG_WARN("failed to aggregate result type for merge", K(ret)); } else if (OB_FAIL(res_types.push_back(res_type))) { LOG_WARN("failed to pushback res type", K(ret)); @@ -6822,6 +6823,8 @@ int ObOptimizerUtil::try_add_cast_to_set_child_list(ObIAllocator *allocator, res_types->reuse(); } const int64_t num = left_types.count(); + ObExprTypeCtx type_ctx; + ObSQLUtils::init_type_ctx(session_info, type_ctx); for (int64_t i = 0; OB_SUCC(ret) && i < num; i++) { res_type.reset(); ObExprResType &left_type = left_types.at(i); @@ -6909,7 +6912,7 @@ int ObOptimizerUtil::try_add_cast_to_set_child_list(ObIAllocator *allocator, } else if (OB_FAIL(session_info->get_collation_connection(coll_type))) { LOG_WARN("failed to get collation connection", K(ret)); } else if (OB_FAIL(dummy_op.aggregate_result_type_for_merge(res_type, &types.at(0), 2, - coll_type, is_oracle_mode(), length_semantics))) { + is_oracle_mode(), type_ctx))) { if (session_info->is_varparams_sql_prepare()) { skip_add_cast = true; res_type = left_type; diff --git a/src/sql/optimizer/ob_select_log_plan.cpp b/src/sql/optimizer/ob_select_log_plan.cpp index c2d0f89f4..ec6eaf903 100644 --- a/src/sql/optimizer/ob_select_log_plan.cpp +++ b/src/sql/optimizer/ob_select_log_plan.cpp @@ -7268,14 +7268,15 @@ int ObSelectLogPlan::adjust_late_materialization_plan_structure(ObLogicalOperato is_in_range_optimization_enabled))) { LOG_WARN("failed to check in range optimization enabled", K(ret)); } else if (OB_FAIL(query_range->preliminary_extract_query_range(range_columns, - join_conditions, - dtc_params, - optimizer_context_.get_exec_ctx(), - NULL, - params, - false, - true, - is_in_range_optimization_enabled))) { + join_conditions, + dtc_params, + optimizer_context_.get_exec_ctx(), + optimizer_context_.get_query_ctx(), + NULL, + params, + false, + true, + is_in_range_optimization_enabled))) { LOG_WARN("failed to preliminary extract query range", K(ret)); } else if (OB_FAIL(table_scan->set_range_columns(range_columns))) { LOG_WARN("failed to set range columns", K(ret)); diff --git a/src/sql/optimizer/ob_table_location.cpp b/src/sql/optimizer/ob_table_location.cpp index 9cc39a8e6..260207f70 100644 --- a/src/sql/optimizer/ob_table_location.cpp +++ b/src/sql/optimizer/ob_table_location.cpp @@ -2063,9 +2063,10 @@ int ObTableLocation::set_location_calc_node(const ObDMLStmt &stmt, ObSEArray part_columns; ObSEArray gen_cols; const ObRawExpr *part_raw_expr = NULL; - if (OB_ISNULL(exec_ctx)) { + ObQueryCtx *query_ctx = NULL; + if (OB_ISNULL(exec_ctx) || OB_ISNULL(query_ctx = stmt.get_query_ctx())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", K(exec_ctx), K(ret)); + LOG_WARN("unexpected null", K(exec_ctx), K(query_ctx), K(ret)); } else if (OB_FAIL(get_partition_column_info(stmt, part_level, part_columns, @@ -2087,6 +2088,7 @@ int ObTableLocation::set_location_calc_node(const ObDMLStmt &stmt, is_range_get, dtc_params, exec_ctx, + query_ctx, is_in_range_optimization_enabled))) { LOG_WARN("Failed to get location calc node", K(ret)); } else if (gen_cols.count() > 0) { @@ -2099,6 +2101,7 @@ int ObTableLocation::set_location_calc_node(const ObDMLStmt &stmt, gen_col_node, dtc_params, exec_ctx, + query_ctx, is_in_range_optimization_enabled))) { LOG_WARN("Get query range node error", K(ret)); } else if (always_true) { @@ -2484,6 +2487,7 @@ int ObTableLocation::get_location_calc_node(const ObPartitionLevel part_level, bool &is_range_get, const ObDataTypeCastParams &dtc_params, ObExecContext *exec_ctx, + ObQueryCtx *query_ctx, const bool is_in_range_optimization_enabled) { int ret = OB_SUCCESS; @@ -2513,6 +2517,7 @@ int ObTableLocation::get_location_calc_node(const ObPartitionLevel part_level, calc_node, dtc_params, exec_ctx, + query_ctx, is_in_range_optimization_enabled))) { LOG_WARN("Get query range node error", K(ret)); } else if (always_true) { @@ -2556,7 +2561,8 @@ int ObTableLocation::get_location_calc_node(const ObPartitionLevel part_level, if (normal_filters.count() > 0) { column_always_true = false; if (OB_FAIL(get_query_range_node(part_level, partition_columns, filter_exprs, column_always_true, - column_node, dtc_params, exec_ctx, is_in_range_optimization_enabled))) { + column_node, dtc_params, exec_ctx, query_ctx, + is_in_range_optimization_enabled))) { LOG_WARN("Failed to get query range node", K(ret)); } else if (OB_NOT_NULL(column_node)) { is_column_range_get = static_cast(column_node)->pre_query_range_.is_precise_get(); @@ -2587,6 +2593,7 @@ int ObTableLocation::get_query_range_node(const ObPartitionLevel part_level, ObPartLocCalcNode *&calc_node, const ObDataTypeCastParams &dtc_params, ObExecContext *exec_ctx, + ObQueryCtx *query_ctx, const bool is_in_range_optimization_enabled) { int ret = OB_SUCCESS; @@ -2603,6 +2610,7 @@ int ObTableLocation::get_query_range_node(const ObPartitionLevel part_level, filter_exprs, dtc_params, exec_ctx, + query_ctx, NULL, NULL, phy_rowid_for_table_loc, diff --git a/src/sql/optimizer/ob_table_location.h b/src/sql/optimizer/ob_table_location.h index 50fb1e9d2..4516f497d 100644 --- a/src/sql/optimizer/ob_table_location.h +++ b/src/sql/optimizer/ob_table_location.h @@ -973,6 +973,7 @@ private: bool &is_range_get, const common::ObDataTypeCastParams &dtc_params, ObExecContext *exec_ctx, + ObQueryCtx *query_ctx, const bool is_in_range_optimization_enabled); int analyze_filter(const common::ObIArray &partition_columns, @@ -992,6 +993,7 @@ private: ObPartLocCalcNode *&calc_node, const common::ObDataTypeCastParams &dtc_params, ObExecContext *exec_ctx, + ObQueryCtx *query_ctx, const bool is_in_range_optimization_enabled); int extract_eq_op(ObExecContext *exec_ctx, diff --git a/src/sql/parser/CMakeLists.txt b/src/sql/parser/CMakeLists.txt index ff717566b..888f82a27 100644 --- a/src/sql/parser/CMakeLists.txt +++ b/src/sql/parser/CMakeLists.txt @@ -8,6 +8,8 @@ set(ob_sql_parser_charset_object_list ob_ctype.cc ob_ctype_gbk.cc ob_ctype_latin1.cc + ob_ctype_ascii.cc + ob_ctype_tis620.cc ob_ctype_mb.cc ob_ctype_simple.cc ob_ctype_utf8.cc @@ -50,17 +52,17 @@ if (OB_BUILD_ORACLE_PARSER) sql_parser_oracle_gbk_mode_lex.h sql_parser_oracle_gbk_mode_tab.c sql_parser_oracle_gbk_mode_tab.h - sql_parser_oracle_latin1_mode_lex.c - sql_parser_oracle_latin1_mode_lex.h - sql_parser_oracle_latin1_mode_tab.c - sql_parser_oracle_latin1_mode_tab.h + sql_parser_oracle_single_byte_mode_lex.c + sql_parser_oracle_single_byte_mode_lex.h + sql_parser_oracle_single_byte_mode_tab.c + sql_parser_oracle_single_byte_mode_tab.h ) set(ob_inner_sql_parser_object_list ${ob_inner_sql_parser_object_list} non_reserved_keywords_oracle_utf8_mode.c non_reserved_keywords_oracle_gbk_mode.c - non_reserved_keywords_oracle_latin1_mode.c + non_reserved_keywords_oracle_single_byte_mode.c ) endif() diff --git a/src/sql/parser/gen_parser.sh b/src/sql/parser/gen_parser.sh index 6fe26d54f..24e4d8bdf 100755 --- a/src/sql/parser/gen_parser.sh +++ b/src/sql/parser/gen_parser.sh @@ -50,47 +50,47 @@ ln -sf ../../../close_modules/oracle_parser/sql/parser/sql_parser_oracle_mode.l # generate oracle latin1 sql_parser(do not support multi_byte_space、multi_byte_comma、multi_byte_left_parenthesis、multi_byte_right_parenthesis) ##1.copy lex and yacc files -cat ../../../src/sql/parser/sql_parser_oracle_mode.y > ../../../src/sql/parser/sql_parser_oracle_latin1_mode.y -cat ../../../src/sql/parser/sql_parser_oracle_mode.l > ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l +cat ../../../src/sql/parser/sql_parser_oracle_mode.y > ../../../src/sql/parser/sql_parser_oracle_single_byte_mode.y +cat ../../../src/sql/parser/sql_parser_oracle_mode.l > ../../../src/sql/parser/sql_parser_oracle_single_byte_mode.l ##2.replace name -sed "s/obsql_oracle_yy/obsql_oracle_latin1_yy/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.y -sed "s/obsql_oracle_yy/obsql_oracle_latin1_yy/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l -sed "s/sql_parser_oracle_mode/sql_parser_oracle_latin1_mode/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.y -sed "s/sql_parser_oracle_mode/sql_parser_oracle_latin1_mode/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l -sed "s/obsql_oracle_parser_fatal_error/obsql_oracle_latin1_parser_fatal_error/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.y -sed "s/obsql_oracle_parser_fatal_error/obsql_oracle_latin1_parser_fatal_error/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l -sed "s/obsql_oracle_fast_parse/obsql_oracle_latin1_fast_parse/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.y -sed "s/obsql_oracle_multi_fast_parse/obsql_oracle_latin1_multi_fast_parse/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.y -sed "s/obsql_oracle_multi_values_parse/obsql_oracle_latin1_multi_values_parse/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.y +sed "s/obsql_oracle_yy/obsql_oracle_single_byte_yy/g" -i ../../../src/sql/parser/sql_parser_oracle_single_byte_mode.y +sed "s/obsql_oracle_yy/obsql_oracle_single_byte_yy/g" -i ../../../src/sql/parser/sql_parser_oracle_single_byte_mode.l +sed "s/sql_parser_oracle_mode/sql_parser_oracle_single_byte_mode/g" -i ../../../src/sql/parser/sql_parser_oracle_single_byte_mode.y +sed "s/sql_parser_oracle_mode/sql_parser_oracle_single_byte_mode/g" -i ../../../src/sql/parser/sql_parser_oracle_single_byte_mode.l +sed "s/obsql_oracle_parser_fatal_error/obsql_oracle_single_byte_parser_fatal_error/g" -i ../../../src/sql/parser/sql_parser_oracle_single_byte_mode.y +sed "s/obsql_oracle_parser_fatal_error/obsql_oracle_single_byte_parser_fatal_error/g" -i ../../../src/sql/parser/sql_parser_oracle_single_byte_mode.l +sed "s/obsql_oracle_fast_parse/obsql_oracle_single_byte_fast_parse/g" -i ../../../src/sql/parser/sql_parser_oracle_single_byte_mode.y +sed "s/obsql_oracle_multi_fast_parse/obsql_oracle_single_byte_multi_fast_parse/g" -i ../../../src/sql/parser/sql_parser_oracle_single_byte_mode.y +sed "s/obsql_oracle_multi_values_parse/obsql_oracle_single_byte_multi_values_parse/g" -i ../../../src/sql/parser/sql_parser_oracle_single_byte_mode.y ##3.do not need to replace multi_byte_space、multi_byte_comma、multi_byte_left_parenthesis、multi_byte_right_parenthesis code -sed "s/multi_byte_space \[\\\u3000\]/multi_byte_space \[\\\x20]/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l -sed "s/multi_byte_comma \[\\\uff0c\]/multi_byte_comma \[\\\x2c]/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l -sed "s/multi_byte_left_parenthesis \[\\\uff08\]/multi_byte_left_parenthesis \[\\\x28]/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l -sed "s/multi_byte_right_parenthesis \[\\\uff09\]/multi_byte_right_parenthesis \[\\\x29]/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l +sed "s/multi_byte_space \[\\\u3000\]/multi_byte_space \[\\\x20]/g" -i ../../../src/sql/parser/sql_parser_oracle_single_byte_mode.l +sed "s/multi_byte_comma \[\\\uff0c\]/multi_byte_comma \[\\\x2c]/g" -i ../../../src/sql/parser/sql_parser_oracle_single_byte_mode.l +sed "s/multi_byte_left_parenthesis \[\\\uff08\]/multi_byte_left_parenthesis \[\\\x28]/g" -i ../../../src/sql/parser/sql_parser_oracle_single_byte_mode.l +sed "s/multi_byte_right_parenthesis \[\\\uff09\]/multi_byte_right_parenthesis \[\\\x29]/g" -i ../../../src/sql/parser/sql_parser_oracle_single_byte_mode.l echo "LATIN1_CHAR [\x80-\xFF]" > ../../../src/sql/parser/latin1.txt -sed '/following character status will be rewrite by gen_parse.sh according to connection character/d' -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l -sed '/multi_byte_connect_char \/\*According to connection character to set by gen_parse.sh\*\//r ../../../src/sql/parser/latin1.txt' -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l -sed '/multi_byte_connect_char \/\*According to connection character to set by gen_parse.sh\*\//d' -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l -sed 's/multi_byte_connect_char/LATIN1_CHAR/g' -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l -sed -i '/{multi_byte_space}/,+5d' sql_parser_oracle_latin1_mode.l -sed -i '/{multi_byte_comma}/,+35d' sql_parser_oracle_latin1_mode.l -sed -i '/{multi_byte_comma}/,+23d' sql_parser_oracle_latin1_mode.l -sed -i '/{multi_byte_space}/,+4d' sql_parser_oracle_latin1_mode.l +sed '/following character status will be rewrite by gen_parse.sh according to connection character/d' -i ../../../src/sql/parser/sql_parser_oracle_single_byte_mode.l +sed '/multi_byte_connect_char \/\*According to connection character to set by gen_parse.sh\*\//r ../../../src/sql/parser/latin1.txt' -i ../../../src/sql/parser/sql_parser_oracle_single_byte_mode.l +sed '/multi_byte_connect_char \/\*According to connection character to set by gen_parse.sh\*\//d' -i ../../../src/sql/parser/sql_parser_oracle_single_byte_mode.l +sed 's/multi_byte_connect_char/LATIN1_CHAR/g' -i ../../../src/sql/parser/sql_parser_oracle_single_byte_mode.l +sed -i '/{multi_byte_space}/,+5d' sql_parser_oracle_single_byte_mode.l +sed -i '/{multi_byte_comma}/,+35d' sql_parser_oracle_single_byte_mode.l +sed -i '/{multi_byte_comma}/,+23d' sql_parser_oracle_single_byte_mode.l +sed -i '/{multi_byte_space}/,+4d' sql_parser_oracle_single_byte_mode.l ##4.generate oracle latin1 parser files -bison_parser ../../../src/sql/parser/sql_parser_oracle_latin1_mode.y ../../../src/sql/parser/sql_parser_oracle_latin1_mode_tab.c -flex -o ../../../src/sql/parser/sql_parser_oracle_latin1_mode_lex.c ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l ../../../src/sql/parser/sql_parser_oracle_latin1_mode_tab.h +bison_parser ../../../src/sql/parser/sql_parser_oracle_single_byte_mode.y ../../../src/sql/parser/sql_parser_oracle_single_byte_mode_tab.c +flex -o ../../../src/sql/parser/sql_parser_oracle_single_byte_mode_lex.c ../../../src/sql/parser/sql_parser_oracle_single_byte_mode.l ../../../src/sql/parser/sql_parser_oracle_single_byte_mode_tab.h ##5.replace other info -sed "/Setup the input buffer state to scan the given bytes/,/}/{/int i/d}" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode_lex.c -sed "/Setup the input buffer state to scan the given bytes/,/}/{/for ( i = 0; i < _yybytes_len; ++i )/d}" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode_lex.c -sed "/Setup the input buffer state to scan the given bytes/,/}/{s/\tbuf\[i\] = yybytes\[i\]/memcpy(buf, yybytes, _yybytes_len)/g}" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode_lex.c -sed "/obsql_oracle_latin1_yylex_init is special because it creates the scanner itself/,/Initialization is the same as for the non-reentrant scanner/{s/return 1/return errno/g}" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode_lex.c -cat ../../../src/sql/parser/non_reserved_keywords_oracle_mode.c > ../../../src/sql/parser/non_reserved_keywords_oracle_latin1_mode.c -sed '/#include "ob_non_reserved_keywords.h"/a\#include "sql/parser/sql_parser_oracle_latin1_mode_tab.h\"' -i ../../../src/sql/parser/non_reserved_keywords_oracle_latin1_mode.c -sed "s/non_reserved_keywords_oracle_mode.c is for …/non_reserved_keywords_oracle_latin1_mode.c is auto generated by gen_parser.sh/g" -i ../../../src/sql/parser/non_reserved_keywords_oracle_latin1_mode.c +sed "/Setup the input buffer state to scan the given bytes/,/}/{/int i/d}" -i ../../../src/sql/parser/sql_parser_oracle_single_byte_mode_lex.c +sed "/Setup the input buffer state to scan the given bytes/,/}/{/for ( i = 0; i < _yybytes_len; ++i )/d}" -i ../../../src/sql/parser/sql_parser_oracle_single_byte_mode_lex.c +sed "/Setup the input buffer state to scan the given bytes/,/}/{s/\tbuf\[i\] = yybytes\[i\]/memcpy(buf, yybytes, _yybytes_len)/g}" -i ../../../src/sql/parser/sql_parser_oracle_single_byte_mode_lex.c +sed "/obsql_oracle_single_byte_yylex_init is special because it creates the scanner itself/,/Initialization is the same as for the non-reentrant scanner/{s/return 1/return errno/g}" -i ../../../src/sql/parser/sql_parser_oracle_single_byte_mode_lex.c +cat ../../../src/sql/parser/non_reserved_keywords_oracle_mode.c > ../../../src/sql/parser/non_reserved_keywords_oracle_single_byte_mode.c +sed '/#include "ob_non_reserved_keywords.h"/a\#include "sql/parser/sql_parser_oracle_single_byte_mode_tab.h\"' -i ../../../src/sql/parser/non_reserved_keywords_oracle_single_byte_mode.c +sed "s/non_reserved_keywords_oracle_mode.c is for …/non_reserved_keywords_oracle_single_byte_mode.c is auto generated by gen_parser.sh/g" -i ../../../src/sql/parser/non_reserved_keywords_oracle_single_byte_mode.c ##6.clean useless files rm -f ../../../src/sql/parser/latin1.txt -rm -f ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l -rm -f ../../../src/sql/parser/sql_parser_oracle_latin1_mode.y +rm -f ../../../src/sql/parser/sql_parser_oracle_single_byte_mode.l +rm -f ../../../src/sql/parser/sql_parser_oracle_single_byte_mode.y # generate oracle utf8 sql_parser(support multi_byte_space、multi_byte_comma、multi_byte_left_parenthesis、multi_byte_right_parenthesis) ##1.copy lex and yacc files diff --git a/src/sql/parser/non_reserved_keywords_mysql_mode.c b/src/sql/parser/non_reserved_keywords_mysql_mode.c index 9f846978b..78eec77c6 100644 --- a/src/sql/parser/non_reserved_keywords_mysql_mode.c +++ b/src/sql/parser/non_reserved_keywords_mysql_mode.c @@ -1057,6 +1057,7 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] = {"rb_build_agg", RB_BUILD_AGG}, {"rb_or_agg", RB_OR_AGG}, {"rb_and_agg", RB_AND_AGG}, + {"optimizer_costs", OPTIMIZER_COSTS} }; /** https://dev.mysql.com/doc/refman/5.7/en/sql-syntax-prepared-statements.html diff --git a/src/sql/parser/ob_fast_parser.cpp b/src/sql/parser/ob_fast_parser.cpp index 3c9735a9c..75aec99f6 100644 --- a/src/sql/parser/ob_fast_parser.cpp +++ b/src/sql/parser/ob_fast_parser.cpp @@ -130,6 +130,9 @@ int ObFastParserBase::parse(const ObString &stmt, static_cast(allocator_.alloc((len + 1))))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to alloc memory", K(ret), K(len)); + } else if (OB_ISNULL(charset_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error", K(ret), K(charset_info_)); } else { no_param_sql_[0] = '\0'; while (len > 0 && is_space(stmt[len - 1])) { @@ -457,8 +460,8 @@ inline int64_t ObFastParserBase::is_identifier_flags(const int64_t pos) idf_pos = is_utf8_char(pos); } else if (ObCharset::is_gb_charset(charset_type_)) { idf_pos = is_gbk_char(pos); - } else if (CHARSET_LATIN1 == charset_type_) { - idf_pos = is_latin1_char(pos); + } else if (charset_info_->mbmaxlen == 1) { + idf_pos = is_single_byte_char(pos); } return idf_pos; } @@ -930,18 +933,18 @@ int ObFastParserBase::get_one_insert_row_str(ObRawSql &raw_sql, inline int64_t ObFastParserBase::notascii_gb_char(const int64_t pos) { int64_t idf_pos = -1; - if (notascii(raw_sql_.char_at(pos))) { + if ((idf_pos = is_gbk_char(pos)) != -1) { + //do nothing + } else if (notascii(raw_sql_.char_at(pos))) { idf_pos = pos + 1; - } else { - idf_pos = is_gbk_char(pos); } return idf_pos; } -inline int64_t ObFastParserBase::is_latin1_char(const int64_t pos) +inline int64_t ObFastParserBase::is_single_byte_char(const int64_t pos) { int64_t idf_pos = -1; - if (is_latin1(raw_sql_.char_at(pos))) { + if (is_single_byte(raw_sql_.char_at(pos))) { idf_pos = pos + 1; } return idf_pos; @@ -1622,8 +1625,8 @@ inline int64_t ObFastParserBase::is_first_identifier_flags(const int64_t pos) idf_pos = is_utf8_char(pos); } else if (ObCharset::is_gb_charset(charset_type_)) { idf_pos = is_gbk_char(pos); - } else if (CHARSET_LATIN1 == charset_type_) { - idf_pos = is_latin1_char(pos); + } else if (charset_info_->mbmaxlen == 1) { + idf_pos = is_single_byte_char(pos); } return idf_pos; } diff --git a/src/sql/parser/ob_fast_parser.h b/src/sql/parser/ob_fast_parser.h index e611aea47..a5f7e42eb 100644 --- a/src/sql/parser/ob_fast_parser.h +++ b/src/sql/parser/ob_fast_parser.h @@ -383,7 +383,7 @@ protected: void reset_parser_node(ParseNode *node); int64_t notascii_gb_char(const int64_t pos); //{U} - int64_t is_latin1_char(const int64_t pos); + int64_t is_single_byte_char(const int64_t pos); // ({U_2}{U}|{U_3}{U}{U}|{U_4}{U}{U}{U} int64_t is_utf8_char(const int64_t pos); // NOTES: No boundary check, the caller guarantees safety!!! @@ -446,7 +446,7 @@ protected: return is_valid_char(ch) && (static_cast(ch) >= 0x80 && static_cast(ch) <= 0xFF); } - inline bool is_latin1(char ch) + inline bool is_single_byte(char ch) { return is_valid_char(ch) && static_cast(ch) >= 0x80 && static_cast(ch) <= 0xFF; diff --git a/src/sql/parser/parse_malloc.cpp b/src/sql/parser/parse_malloc.cpp index 7b73baee5..b6921ad36 100644 --- a/src/sql/parser/parse_malloc.cpp +++ b/src/sql/parser/parse_malloc.cpp @@ -120,11 +120,13 @@ char *replace_invalid_character(const struct ObCharsetInfo* src_cs, const struct const char *str, int64_t *out_len, void *malloc_pool, int *extra_errno) { char *out_str = NULL; - if (OB_ISNULL(str) || OB_ISNULL(extra_errno) || OB_ISNULL(out_len)) { + if (OB_ISNULL(str) || OB_ISNULL(extra_errno) || OB_ISNULL(out_len) || OB_ISNULL(src_cs)) { } else if (NULL == oracle_db_cs) { out_str = const_cast(str); } else { - ob_wc_t replace_char = !!(oracle_db_cs->state & OB_CS_UNICODE) ? 0xFFFD : '?'; + ob_wc_t replace_char = (!!(oracle_db_cs->state & OB_CS_UNICODE)) && + (!!(src_cs->state & OB_CS_UNICODE)) + ? 0xFFFD : '?'; uint errors = 0; size_t str_len = STRLEN(str); char *temp_str = NULL; @@ -269,8 +271,10 @@ char *parse_strdup_with_replace_multi_byte_char(const char *str, int *connection case 46/*CS_TYPE_UTF8MB4_BIN*/: case 63/*CS_TYPE_BINARY*/: case 224/*CS_TYPE_UTF8MB4_UNICODE_CI*/: - //case 8/*CS_TYPE_LATIN1_SWEDISH_CI*/: - //case 47/*CS_TYPE_LATIN1_BIN*/: + case 245/*CS_TYPE_UTF8MB4_CROATIAN_CI*/: + case 246/*CS_TYPE_UTF8MB4_UNICODE_520_CI*/: + case 234/*CS_TYPE_UTF8MB4_CZECH_CI*/: + case 255/*CS_TYPE_UTF8MB4_0900_AI_CI*/: { if (i + 2 < dup_len) { if (str[i] == (char)0xe3 && str[i+1] == (char)0x80 && str[i+2] == (char)0x80) { diff --git a/src/sql/parser/parse_node.c b/src/sql/parser/parse_node.c index 042296b6b..82b7666e4 100644 --- a/src/sql/parser/parse_node.c +++ b/src/sql/parser/parse_node.c @@ -384,6 +384,28 @@ ParseNode *new_non_terminal_node(void *malloc_pool, ObItemType node_tag, int num return ret_node; } +ParseNode *new_list_node(void *malloc_pool, ObItemType node_tag, int capacity, int num, ...) +{ + ParseNode *ret_node = NULL; + if (OB_UNLIKELY(capacity <= 0 || num <= 0 || num > capacity)) { + (void)fprintf(stderr, "ERROR invalid num:%d capacity:%d\n", num, capacity); + } else { + int32_t i = 0; + va_list va; + ret_node = new_node(malloc_pool, node_tag, capacity); + if (OB_LIKELY(NULL != ret_node)) { + ret_node->value_ = capacity; + ret_node->num_child_ = num; + va_start(va, num); + for (; i < num; ++i) { + ret_node->children_[i] = va_arg(va, ParseNode *); + } + va_end(va); + } + } + return ret_node; +} + char *copy_expr_string(ParseResult *p, int expr_start, int expr_end) { char *expr_string = NULL; @@ -826,6 +848,129 @@ extern bool nodename_is_sdo_geometry_type(const ParseNode *node) return result; } +int64_t get_need_reserve_capacity(int64_t n) +{ + int64_t capacity = 0; + // equal to OB_MALLOC_BIG_BLOCK_SIZE in ob_define.h + const int64_t max_delta_capacity = (1LL << 21) / sizeof(ParseNode*); // 2MB + if (n <= 2) { + capacity = 2; + } else if ((n & (n - 1)) == 0) { + capacity = n; + } else if (n > max_delta_capacity) { + int64_t i = n / max_delta_capacity; + capacity = max_delta_capacity * (i + 1); + } else { + capacity = 4; + while (capacity < n) { + capacity <<= 1; + } + } + return capacity; +} + +// (A OR B) OR C --> OR (A, B, C) +ParseNode *push_back_child(void *malloc_pool, int *error_code, ParseNode *left_node, ParseNode *node) +{ + ParseNode *ret_node = NULL; + if (OB_ISNULL(malloc_pool) || OB_ISNULL(error_code)) { + (void)fprintf(stderr, "ERROR parser result is NULL\n"); + } else if (NULL == left_node || NULL == node) { + /* do nothing */ + } else if ((left_node->type_ != T_OP_OR && + left_node->type_ != T_OP_AND && + left_node->type_ != T_EXPR_LIST) || + left_node->value_ == INT64_MAX) { + *error_code = OB_PARSER_ERR_UNEXPECTED; + } else { + int64_t capacity = get_need_reserve_capacity(left_node->num_child_ + 1); + if (left_node->value_ < capacity) { + ParseNode *new_op = new_node(malloc_pool, left_node->type_, capacity); + if (OB_ISNULL(new_op)) { + *error_code = OB_PARSER_ERR_NO_MEMORY; + } else { + MEMCPY(new_op->children_, left_node->children_, sizeof(ParseNode*) * left_node->num_child_); + new_op->children_[left_node->num_child_] = node; + new_op->num_child_ = left_node->num_child_ + 1; + new_op->value_ = capacity; + ret_node = new_op; + } + } else { + left_node->children_[left_node->num_child_] = node; + left_node->num_child_ += 1; + ret_node = left_node; + } + } + return ret_node; +} + +// A OR (B OR C) --> OR (A, B, C) +ParseNode *push_front_child(void *malloc_pool, int *error_code, ParseNode *right_node, ParseNode *node) +{ + ParseNode *ret_node = NULL; + if (OB_ISNULL(malloc_pool) || OB_ISNULL(error_code)) { + (void)fprintf(stderr, "ERROR parser result is NULL\n"); + } else if (NULL == right_node || NULL == node) { + /* do nothing */ + } else if ((right_node->type_ != T_OP_OR && + right_node->type_ != T_OP_AND && + right_node->type_ != T_EXPR_LIST) || + right_node->value_ == INT64_MAX) { + *error_code = OB_PARSER_ERR_UNEXPECTED; + } else { + int64_t capacity = get_need_reserve_capacity(right_node->num_child_ + 1); + ParseNode *new_op = new_node(malloc_pool, right_node->type_, capacity); + if (OB_ISNULL(new_op)) { + *error_code = OB_PARSER_ERR_NO_MEMORY; + } else { + new_op->children_[0] = node; + MEMCPY(new_op->children_ + 1, right_node->children_, sizeof(ParseNode*) * right_node->num_child_); + new_op->value_ = capacity; + new_op->num_child_ = right_node->num_child_ + 1; + ret_node = new_op; + } + } + return ret_node; +} + +// (A OR B) OR (C OR D) --> OR (A, B, C, D) +ParseNode *append_child(void *malloc_pool, int *error_code, ParseNode *left_node, ParseNode *right_node) +{ + ParseNode *ret_node = NULL; + if (OB_ISNULL(malloc_pool) || OB_ISNULL(error_code)) { + (void)fprintf(stderr, "ERROR parser result is NULL\n"); + } else if (NULL == left_node || NULL == right_node) { + /* do nothing */ + } else if (left_node->type_ != right_node->type_ || + (left_node->type_ != T_OP_OR && + left_node->type_ != T_OP_AND && + left_node->type_ != T_EXPR_LIST) || + left_node->value_ == INT64_MAX || + right_node->value_ == INT64_MAX) { + *error_code = OB_PARSER_ERR_UNEXPECTED; + } else { + int64_t num_child = left_node->num_child_ + right_node->num_child_; + int64_t capacity = get_need_reserve_capacity(num_child); + if (left_node->value_ < capacity) { + ParseNode *new_op = new_node(malloc_pool, left_node->type_, capacity); + if (OB_ISNULL(new_op)) { + *error_code = OB_PARSER_ERR_NO_MEMORY; + } else { + MEMCPY(new_op->children_, left_node->children_, sizeof(ParseNode*) * left_node->num_child_); + MEMCPY(new_op->children_ + left_node->num_child_, right_node->children_, sizeof(ParseNode*) * right_node->num_child_); + new_op->num_child_ = num_child; + new_op->value_ = capacity; + ret_node = new_op; + } + } else { + MEMCPY(left_node->children_ + left_node->num_child_, right_node->children_, sizeof(ParseNode*) * right_node->num_child_); + left_node->num_child_ = num_child; + ret_node = left_node; + } + } + return ret_node; +} + ParseNode *adjust_inner_join_inner(int *error_code, ParseNode *inner_join, ParseNode *table_node) { ParseNode *ret_node = NULL; diff --git a/src/sql/parser/parse_node.h b/src/sql/parser/parse_node.h index 6d98a9bb0..3c3200cc6 100644 --- a/src/sql/parser/parse_node.h +++ b/src/sql/parser/parse_node.h @@ -56,7 +56,7 @@ enum SelectParserOffset PARSE_SELECT_LIMIT, PARSE_SELECT_FOR_UPD, PARSE_SELECT_HINTS, - PARSE_SELECT_WHEN, + PARSE_SELECT_WHEN, // I find that it is no longer used. PARSE_SELECT_FETCH, PARSE_SELECT_FETCH_TEMP, //use to temporary store fetch clause in parser PARSE_SELECT_WITH_CHECK_OPTION, @@ -386,6 +386,7 @@ extern int64_t str_remove_space(char *buff, int64_t len); extern ParseNode *new_node(void *malloc_pool, ObItemType type, int num); extern ParseNode *new_non_terminal_node(void *malloc_pool, ObItemType node_tag, int num, ...); extern ParseNode *new_terminal_node(void *malloc_pool, ObItemType type); +extern ParseNode *new_list_node(void *malloc_pool, ObItemType node_tag, int capacity, int num, ...); extern int obpl_parser_check_stack_overflow(); @@ -412,6 +413,10 @@ extern bool parsenode_equal(const ParseNode *node1, const ParseNode *node2, int extern int64_t get_question_mark(ObQuestionMarkCtx *ctx, void *malloc_pool, const char *name); extern int64_t get_question_mark_by_defined_name(ObQuestionMarkCtx *ctx, const char *name); +extern int64_t get_need_reserve_capacity(int64_t n); +extern ParseNode *push_back_child(void *malloc_pool, int *error_code, ParseNode *left_node, ParseNode *node); +extern ParseNode *push_front_child(void *malloc_pool, int *error_code, ParseNode *right_node, ParseNode *node); +extern ParseNode *append_child(void *malloc_pool, int *error_code, ParseNode *left_node, ParseNode *right_node); extern ParseNode *adjust_inner_join_inner(int *error_code, ParseNode *inner_join, ParseNode *table_node); // compare ParseNode str_value_ to pattern diff --git a/src/sql/parser/sql_parser_base.c b/src/sql/parser/sql_parser_base.c index be85268dd..5b3d8c18c 100644 --- a/src/sql/parser/sql_parser_base.c +++ b/src/sql/parser/sql_parser_base.c @@ -28,15 +28,15 @@ extern YY_BUFFER_STATE obsql_mysql_yy_scan_bytes (yyconst char *bytes,int len ,y extern void obsql_mysql_yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner ); extern void obsql_mysql_yy_delete_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner ); #ifdef OB_BUILD_ORACLE_PARSER -extern int obsql_oracle_latin1_yylex_init_extra(YY_EXTRA_TYPE yy_user_defined,yyscan_t* ptr_yy_globals ); -extern int obsql_oracle_latin1_yyparse(ParseResult *result); -extern int obsql_oracle_latin1_multi_fast_parse(ParseResult *p); -extern int obsql_oracle_latin1_multi_values_parse(ParseResult *p); -extern int obsql_oracle_latin1_fast_parse(ParseResult *p); -extern int obsql_oracle_latin1_yylex_destroy (yyscan_t yyscanner ); -extern YY_BUFFER_STATE obsql_oracle_latin1_yy_scan_bytes (yyconst char *bytes,int len ,yyscan_t yyscanner ); -extern void obsql_oracle_latin1_yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner ); -extern void obsql_oracle_latin1_yy_delete_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner ); +extern int obsql_oracle_single_byte_yylex_init_extra(YY_EXTRA_TYPE yy_user_defined,yyscan_t* ptr_yy_globals ); +extern int obsql_oracle_single_byte_yyparse(ParseResult *result); +extern int obsql_oracle_single_byte_multi_fast_parse(ParseResult *p); +extern int obsql_oracle_single_byte_multi_values_parse(ParseResult *p); +extern int obsql_oracle_single_byte_fast_parse(ParseResult *p); +extern int obsql_oracle_single_byte_yylex_destroy (yyscan_t yyscanner ); +extern YY_BUFFER_STATE obsql_oracle_single_byte_yy_scan_bytes (yyconst char *bytes,int len ,yyscan_t yyscanner ); +extern void obsql_oracle_single_byte_yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner ); +extern void obsql_oracle_single_byte_yy_delete_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner ); extern int obsql_oracle_utf8_yylex_init_extra(YY_EXTRA_TYPE yy_user_defined,yyscan_t* ptr_yy_globals ); extern int obsql_oracle_utf8_yyparse(ParseResult *result); extern int obsql_oracle_utf8_multi_fast_parse(ParseResult *p); @@ -89,11 +89,16 @@ int parse_init(ParseResult *p) case 46/*CS_TYPE_UTF8MB4_BIN*/: case 63/*CS_TYPE_BINARY*/: case 224/*CS_TYPE_UTF8MB4_UNICODE_CI*/: + case 255/*CS_TYPE_UTF8MB4_0900_AI_CI*/: ret = obsql_oracle_utf8_yylex_init_extra(p, &(p->yyscan_info_)); break; case 8/*CS_TYPE_LATIN1_SWEDISH_CI*/: case 47/*CS_TYPE_LATIN1_BIN*/: - ret = obsql_oracle_latin1_yylex_init_extra(p, &(p->yyscan_info_)); + case 11/*CS_TYPE_ASCII_GENERAL_CI*/: + case 65/*CS_TYPE_ASCII_BIN*/: + case 18/*CS_TYPE_TIS620_THAI_CI*/: + case 89/*CS_TYPE_TIS620_BIN*/: + ret = obsql_oracle_single_byte_yylex_init_extra(p, &(p->yyscan_info_)); break; default: { ret = -1; @@ -145,11 +150,16 @@ int parse_terminate(ParseResult *p) case 46/*CS_TYPE_UTF8MB4_BIN*/: case 63/*CS_TYPE_BINARY*/: case 224/*CS_TYPE_UTF8MB4_UNICODE_CI*/: + case 255/*CS_TYPE_UTF8MB4_0900_AI_CI*/: ret = obsql_oracle_utf8_yylex_destroy(p->yyscan_info_); break; case 8/*CS_TYPE_LATIN1_SWEDISH_CI*/: case 47/*CS_TYPE_LATIN1_BIN*/: - ret = obsql_oracle_latin1_yylex_destroy(p->yyscan_info_); + case 11/*CS_TYPE_ASCII_GENERAL_CI*/: + case 65/*CS_TYPE_ASCII_BIN*/: + case 18/*CS_TYPE_TIS620_THAI_CI*/: + case 89/*CS_TYPE_TIS620_BIN*/: + ret = obsql_oracle_single_byte_yylex_destroy(p->yyscan_info_); break; default: { ret = -1; @@ -264,7 +274,9 @@ int parse_sql(ParseResult *p, const char *buf, size_t input_len) case 45/*CS_TYPE_UTF8MB4_GENERAL_CI*/: case 46/*CS_TYPE_UTF8MB4_BIN*/: case 63/*CS_TYPE_BINARY*/: - case 224/*CS_TYPE_UTF8MB4_UNICODE_CI*/:{ + case 224/*CS_TYPE_UTF8MB4_UNICODE_CI*/: + case 255/*CS_TYPE_UTF8MB4_0900_AI_CI*/: + { YY_BUFFER_STATE bp = obsql_oracle_utf8_yy_scan_bytes(buf, len, p->yyscan_info_); obsql_oracle_utf8_yy_switch_to_buffer(bp, p->yyscan_info_); int tmp_ret = -1; @@ -291,19 +303,23 @@ int parse_sql(ParseResult *p, const char *buf, size_t input_len) obsql_oracle_utf8_yy_delete_buffer(bp, p->yyscan_info_); break; } + case 11/*CS_TYPE_ASCII_GENERAL_CI*/: + case 65/*CS_TYPE_ASCII_BIN*/: + case 18/*CS_TYPE_TIS620_THAI_CI*/: + case 89/*CS_TYPE_TIS620_BIN*/: case 8/*CS_TYPE_LATIN1_SWEDISH_CI*/: case 47/*CS_TYPE_LATIN1_BIN*/:{ - YY_BUFFER_STATE bp = obsql_oracle_latin1_yy_scan_bytes(buf, len, p->yyscan_info_); - obsql_oracle_latin1_yy_switch_to_buffer(bp, p->yyscan_info_); + YY_BUFFER_STATE bp = obsql_oracle_single_byte_yy_scan_bytes(buf, len, p->yyscan_info_); + obsql_oracle_single_byte_yy_switch_to_buffer(bp, p->yyscan_info_); int tmp_ret = -1; if (p->is_fp_) { - tmp_ret = obsql_oracle_latin1_fast_parse(p); + tmp_ret = obsql_oracle_single_byte_fast_parse(p); } else if (p->is_multi_query_) { - tmp_ret = obsql_oracle_latin1_multi_fast_parse(p); + tmp_ret = obsql_oracle_single_byte_multi_fast_parse(p); } else if (p->is_multi_values_parser_) { - tmp_ret = obsql_oracle_latin1_multi_values_parse(p); + tmp_ret = obsql_oracle_single_byte_multi_values_parse(p); } else { - tmp_ret = obsql_oracle_latin1_yyparse(p); + tmp_ret = obsql_oracle_single_byte_yyparse(p); } if (0 == tmp_ret) { ret = OB_PARSER_SUCCESS; @@ -316,7 +332,7 @@ int parse_sql(ParseResult *p, const char *buf, size_t input_len) ret = OB_PARSER_ERR_PARSE_SQL; } } - obsql_oracle_latin1_yy_delete_buffer(bp, p->yyscan_info_); + obsql_oracle_single_byte_yy_delete_buffer(bp, p->yyscan_info_); break; } default: { diff --git a/src/sql/parser/sql_parser_base.h b/src/sql/parser/sql_parser_base.h index 7787e0396..6cdfd9212 100644 --- a/src/sql/parser/sql_parser_base.h +++ b/src/sql/parser/sql_parser_base.h @@ -74,7 +74,7 @@ int add_alias_name(ParseNode *node, ParseResult *result, int end); do { \ if (OB_UNLIKELY(NULL == result)) { \ (void)fprintf(stderr, "ERROR : result is NULL\n"); \ - } else if (0 == result->extra_errno_) { \ + } else if (OB_PARSER_SUCCESS == result->extra_errno_) { \ result->extra_errno_ = OB_PARSER_ERR_NO_MEMORY; \ } else {/*do nothing*/} \ YYABORT; \ @@ -84,7 +84,7 @@ int add_alias_name(ParseNode *node, ParseResult *result, int end); do { \ if (OB_UNLIKELY(NULL == result)) { \ (void)fprintf(stderr, "ERROR : result is NULL\n"); \ - } else if (0 == result->extra_errno_) { \ + } else if (OB_PARSER_SUCCESS == result->extra_errno_) { \ result->extra_errno_ = OB_PARSER_ERR_UNEXPECTED; \ } else {/*do nothing*/} \ YYABORT; \ @@ -94,7 +94,7 @@ int add_alias_name(ParseNode *node, ParseResult *result, int end); do { \ if (OB_UNLIKELY(NULL == result)) { \ (void)fprintf(stderr, "ERROR : result is NULL\n"); \ - } else if (0 == result->extra_errno_) { \ + } else if (OB_PARSER_SUCCESS == result->extra_errno_) { \ result->extra_errno_ = OB_PARSER_ERR_TOO_BIG_DISPLAYWIDTH; \ } else {/*do nothing*/} \ YYABORT; \ @@ -104,7 +104,7 @@ int add_alias_name(ParseNode *node, ParseResult *result, int end); do { \ if (OB_UNLIKELY(NULL == result)) { \ (void)fprintf(stderr, "ERROR : result is NULL\n"); \ - } else if (0 == result->extra_errno_) { \ + } else if (OB_PARSER_SUCCESS == result->extra_errno_) { \ result->extra_errno_ = OB_PARSER_ERR_STR_LITERAL_TOO_LONG;\ } else {/*do nothing*/} \ yyerror(yylloc, yyextra, "string literal is too long\n", yytext); \ @@ -115,7 +115,7 @@ int add_alias_name(ParseNode *node, ParseResult *result, int end); do { \ if (OB_UNLIKELY(NULL == result)) { \ (void)fprintf(stderr, "ERROR : result is NULL\n"); \ - } else if (0 == result->extra_errno_) { \ + } else if (OB_PARSER_SUCCESS == result->extra_errno_) { \ result->extra_errno_ = OB_PARSER_ERR_UNDECLARED_VAR;\ } else {/*do nothing*/} \ YYABORT; \ @@ -125,7 +125,7 @@ int add_alias_name(ParseNode *node, ParseResult *result, int end); do { \ if (OB_UNLIKELY(NULL == result)) { \ (void)fprintf(stderr, "ERROR : result is NULL\n"); \ - } else if (0 == result->extra_errno_) { \ + } else if (OB_PARSER_SUCCESS == result->extra_errno_) { \ result->extra_errno_ = OB_PARSER_ERR_NOT_VALID_ROUTINE_NAME;\ } else {/*do nothing*/} \ YYABORT; \ @@ -159,6 +159,14 @@ do { } \ } while(0) +#define malloc_list_node(node, malloc_pool, node_tag, ...) \ + do { \ + if (OB_UNLIKELY(NULL == (node = new_list_node(malloc_pool, node_tag, ##__VA_ARGS__)))) {\ + yyerror(NULL, result, "No more space for malloc\n"); \ + YYABORT_NO_MEMORY; \ + } \ + } while(0) + #define merge_nodes(node, result, node_tag, source_tree) \ do { \ if (OB_UNLIKELY(NULL == source_tree)) { \ @@ -1211,6 +1219,69 @@ do {\ }\ } while(0);\ +#define push_back_list(malloc_pool, result, ret_node, left_node, right_node) \ + do { \ + ret_node = push_back_child(malloc_pool, &result->extra_errno_, left_node, right_node); \ + if (OB_UNLIKELY(NULL == ret_node)) { \ + if (OB_PARSER_SUCCESS == result->extra_errno_) { \ + result->extra_errno_ = OB_PARSER_ERR_UNEXPECTED; \ + } \ + yyerror(NULL, result, "error happened\n"); \ + YYABORT; \ + } \ + } while(0); \ + +#define push_front_list(malloc_pool, result, ret_node, left_node, right_node) \ + do { \ + ret_node = push_front_child(malloc_pool, &result->extra_errno_, left_node, right_node); \ + if (OB_UNLIKELY(NULL == ret_node)) { \ + if (OB_PARSER_SUCCESS == result->extra_errno_) { \ + result->extra_errno_ = OB_PARSER_ERR_UNEXPECTED; \ + } \ + yyerror(NULL, result, "error happened\n"); \ + YYABORT; \ + } \ + } while(0); \ + +#define append_list(malloc_pool, result, ret_node, left_node, right_node) \ + do { \ + ret_node = append_child(malloc_pool, &result->extra_errno_, left_node, right_node); \ + if (OB_UNLIKELY(NULL == ret_node)) { \ + if (OB_PARSER_SUCCESS == result->extra_errno_) { \ + result->extra_errno_ = OB_PARSER_ERR_UNEXPECTED; \ + } \ + yyerror(NULL, result, "error happened\n"); \ + YYABORT; \ + } \ + } while(0); \ + +#define flatten_and_or(malloc_pool, result, ret_node, left_node, right_node, type) \ + do { \ + ret_node = NULL; \ + if (NULL == left_node || NULL == right_node || (T_OP_OR != type && T_OP_AND != type)) { \ + result->extra_errno_ = OB_PARSER_ERR_UNEXPECTED; \ + yyerror(NULL, result, "unexpected param\n"); \ + YYABORT; \ + } else if (left_node->type_ == type && right_node->type_ == type) { \ + /* (A OR B) OR (C OR D) */ \ + append_list(malloc_pool, result, ret_node, left_node, right_node); \ + } else if (left_node->type_ == type && right_node->type_ != type) { \ + /* (A OR B) OR C */ \ + push_back_list(malloc_pool, result, ret_node, left_node, right_node); \ + } else if (left_node->type_ != type && right_node->type_ == type) { \ + /* A OR (B OR C) */ \ + push_front_list(malloc_pool, result, ret_node, right_node, left_node); \ + } else { \ + ret_node = new_list_node(malloc_pool, type, 2, 2, left_node, right_node); \ + if (OB_UNLIKELY(NULL == ret_node)) \ + { \ + result->extra_errno_ = OB_PARSER_ERR_NO_MEMORY; \ + yyerror(NULL, result, "No more space for malloc\n"); \ + YYABORT; \ + } \ + } \ + } while(0); \ + #define adjust_inner_join(result, ret_node, inner_join, table_node) \ do { \ ret_node = NULL; \ diff --git a/src/sql/parser/sql_parser_mysql_mode.l b/src/sql/parser/sql_parser_mysql_mode.l index 6ab8c0ce8..b2e7a52c3 100644 --- a/src/sql/parser/sql_parser_mysql_mode.l +++ b/src/sql/parser/sql_parser_mysql_mode.l @@ -112,10 +112,13 @@ INTERVAL { _UTF8 { REPUT_TOKEN_NEG_SIGN(_UTF8); } _UTF8MB4 { REPUT_TOKEN_NEG_SIGN(_UTF8MB4); } +_UTF8MB3 { REPUT_TOKEN_NEG_SIGN(_UTF8MB3); } _GBK { REPUT_TOKEN_NEG_SIGN(_GBK); } _GB18030 { REPUT_TOKEN_NEG_SIGN(_GB18030); } _GB18030_2022 { REPUT_TOKEN_NEG_SIGN(_GB18030_2022); } _LATIN1 { REPUT_TOKEN_NEG_SIGN(_LATIN1); } +_ASCII { REPUT_TOKEN_NEG_SIGN(_ASCII); } +_TIS620 { REPUT_TOKEN_NEG_SIGN(_TIS620); } _BINARY { REPUT_TOKEN_NEG_SIGN(_BINARY); } _UTF16 { REPUT_TOKEN_NEG_SIGN(_UTF16); } NOT { diff --git a/src/sql/parser/sql_parser_mysql_mode.y b/src/sql/parser/sql_parser_mysql_mode.y index 1f1a34647..14a79e512 100644 --- a/src/sql/parser/sql_parser_mysql_mode.y +++ b/src/sql/parser/sql_parser_mysql_mode.y @@ -26,7 +26,7 @@ const struct _NonReservedKeyword *non_reserved_keyword; const struct _NonReservedKeyword *reserved_keyword; int32_t ival[2]; //ival[0]表示value, ival[1]表示fast parse在对应的该node及其子node可识别的常量个数 - } +} %{ #include "../../../src/sql/parser/sql_parser_mysql_mode_lex.h" @@ -207,7 +207,7 @@ DYNAMIC_SAMPLING NEG_SIGN %token /*can not be relation name*/ -_BINARY _UTF8 _UTF8MB4 _GBK _UTF16 _GB18030 _GB18030_2022 _LATIN1 CNNOP +_BINARY _UTF8 _UTF8MB4 _UTF8MB3 _GBK _UTF16 _GB18030 _GB18030_2022 _LATIN1 _ASCII _TIS620 CNNOP SELECT_HINT_BEGIN UPDATE_HINT_BEGIN DELETE_HINT_BEGIN INSERT_HINT_BEGIN REPLACE_HINT_BEGIN HINT_HINT_BEGIN HINT_END LOAD_DATA_HINT_BEGIN CREATE_HINT_BEGIN ALTER_HINT_BEGIN END_P SET_VAR DELIMITER @@ -374,7 +374,7 @@ END_P SET_VAR DELIMITER YEAR - ZONE ZONE_LIST ZONE_TYPE + ZONE ZONE_LIST ZONE_TYPE OPTIMIZER_COSTS OVERWRITE //-----------------------------non_reserved keyword end--------------------------------------------- @@ -466,7 +466,7 @@ END_P SET_VAR DELIMITER %type alter_column_behavior opt_set opt_position_column %type alter_system_stmt alter_system_set_parameter_actions alter_system_settp_actions settp_option alter_system_set_parameter_action server_info_list server_info alter_system_reset_parameter_actions alter_system_reset_parameter_action %type opt_comment opt_as -%type column_name relation_name function_name column_label var_name relation_name_or_string row_format_option compression_name +%type column_name relation_name opt_relation_name function_name column_label var_name relation_name_or_string row_format_option compression_name %type audit_stmt audit_clause op_audit_tail_clause audit_operation_clause audit_all_shortcut_list audit_all_shortcut auditing_on_clause auditing_by_user_clause audit_user_list audit_user audit_user_with_host_name %type opt_hint_list hint_option select_with_opt_hint update_with_opt_hint delete_with_opt_hint hint_list_with_end global_hint transform_hint optimize_hint %type create_index_stmt index_name sort_column_list sort_column_key opt_index_option_list index_option opt_sort_column_key_length opt_index_using_algorithm index_using_algorithm visibility_option opt_constraint_name constraint_name create_with_opt_hint index_expr alter_with_opt_hint @@ -544,15 +544,18 @@ END_P SET_VAR DELIMITER %type json_table_expr mock_jt_on_error_on_empty jt_column_list json_table_column_def %type json_table_ordinality_column_def json_table_exists_column_def json_table_value_column_def json_table_nested_column_def %type opt_value_on_empty_or_error_or_mismatch opt_on_mismatch -%type table_values_caluse table_values_caluse_with_order_by_and_limit values_row_list row_value +%type table_values_clause table_values_clause_with_order_by_and_limit values_row_list row_value %type create_tenant_snapshot_stmt snapshot_name drop_tenant_snapshot_stmt clone_tenant_stmt clone_snapshot_option clone_tenant_option clone_tenant_option_list %type transfer_partition_stmt transfer_partition_clause part_info cancel_transfer_partition_clause %type geometry_collection -%type mock_stmt +%type mock_stmt check_table_options check_table_option user_host_or_current_user install_plugin_stmt plugin_name uninstall_plugin_stmt flush_stmt flush_options flush_options_list flush_option opt_no_write_to_binlog handler_stmt handler_read_or_scan handler_scan_function handler_rkey_function handler_rkey_mode +%type show_plugin_stmt merge_insert_types opt_table_list +%type create_server_stmt server_options_list server_option alter_server_stmt drop_server_stmt create_logfile_group_stmt logfile_group_info add_log_file lg_undofile lg_redofile logfile_group_options opt_ts_initial_size opt_ts_undo_buffer_size opt_ts_redo_buffer_size opt_ts_engine opt_ts_comment +%type alter_logfile_group_stmt alter_logfile_group_info alter_logfile_group_option_list alter_logfile_group_options alter_logfile_group_option drop_logfile_group_stmt drop_ts_options_list drop_ts_options drop_ts_option opt_ts_nodegroup logfile_group_option logfile_group_option_list %type service_name_stmt service_op %type ttl_definition ttl_expr ttl_unit %type id_dot_id id_dot_id_dot_id -%type opt_table_list opt_repair_mode opt_repair_option_list repair_option repair_option_list opt_checksum_option +%type opt_empty_table_list opt_repair_mode opt_repair_option_list repair_option repair_option_list opt_checksum_option %type cache_index_stmt load_index_into_cache_stmt tbl_index_list tbl_index tbl_partition_list opt_tbl_partition_list tbl_index_or_partition_list tbl_index_or_partition opt_ignore_leaves key_cache_name %start sql_stmt %% @@ -760,15 +763,15 @@ pl_expr_stmt: expr_list: expr %prec LOWER_COMMA { - $$ = $1; /* every mysql's item(same as ob's expr) has its own name */ - if (OB_UNLIKELY((NULL == $$->str_value_)) && $$->type_ != T_VARCHAR) { - dup_string($$, result, @1.first_column, @1.last_column); + if (OB_UNLIKELY((NULL == $1->str_value_)) && $1->type_ != T_VARCHAR) { + dup_string($1, result, @1.first_column, @1.last_column); } + malloc_list_node($$, result->malloc_pool_, T_EXPR_LIST, 2, 1, $1); } | expr_list ',' expr { - malloc_non_terminal_node($$, result->malloc_pool_, T_LINK_NODE, 2, $1, $3); + push_back_list(result->malloc_pool_, result, $$, $1, $3); } ; @@ -1092,6 +1095,15 @@ _UTF8 YYABORT_NO_MEMORY; } } +| _UTF8MB3 +{ + malloc_terminal_node($$, result->malloc_pool_, T_CHARSET); + $$->str_value_ = parse_strdup("utf8mb4", result->malloc_pool_, &($$->str_len_)); + if (OB_UNLIKELY(NULL == $$->str_value_)) { + yyerror(NULL, result, "No more space for mallocing string\n"); + YYABORT_NO_MEMORY; + } +} | _BINARY { malloc_terminal_node($$, result->malloc_pool_, T_CHARSET); @@ -1119,6 +1131,24 @@ _UTF8 YYABORT_NO_MEMORY; } } +| _ASCII +{ + malloc_terminal_node($$, result->malloc_pool_, T_CHARSET); + $$->str_value_ = parse_strdup("ascii", result->malloc_pool_, &($$->str_len_)); + if (OB_UNLIKELY(NULL == $$->str_value_)) { + yyerror(NULL, result, "No more space for mallocing string"); + YYABORT_NO_MEMORY; + } +} +| _TIS620 +{ + malloc_terminal_node($$, result->malloc_pool_, T_CHARSET); + $$->str_value_ = parse_strdup("tis620", result->malloc_pool_, &($$->str_len_)); + if (OB_UNLIKELY(NULL == $$->str_value_)) { + yyerror(NULL, result, "No more space for mallocing string"); + YYABORT_NO_MEMORY; + } +} | _GB18030 { malloc_terminal_node($$, result->malloc_pool_, T_CHARSET); @@ -1663,15 +1693,11 @@ simple_expr collation %prec NEG { $$ = $2; $$->is_assigned_from_child_ = 1; } | '(' expr_list ',' expr ')' { - ParseNode *node = NULL; - malloc_non_terminal_node(node, result->malloc_pool_, T_LINK_NODE, 2, $2, $4); - merge_nodes($$, result, T_EXPR_LIST, node); + push_back_list(result->malloc_pool_, result, $$, $2, $4); } | ROW '(' expr_list ',' expr ')' { - ParseNode *node = NULL; - malloc_non_terminal_node(node, result->malloc_pool_, T_LINK_NODE, 2, $3, $5); - merge_nodes($$, result, T_EXPR_LIST, node); + push_back_list(result->malloc_pool_, result, $$, $3, $5); } | EXISTS select_with_parens { @@ -1783,28 +1809,28 @@ IN NATURAL LANGUAGE MODE expr: expr AND expr %prec AND { - malloc_non_terminal_node($$, result->malloc_pool_, T_OP_AND, 2, $1, $3); + flatten_and_or(result->malloc_pool_, result, $$, $1, $3, T_OP_AND); if (result->pl_parse_info_.is_pl_parse_) { dup_expr_string($$, result, @1.first_column, @3.last_column); } } | expr AND_OP expr %prec AND { - malloc_non_terminal_node($$, result->malloc_pool_, T_OP_AND, 2, $1, $3); + flatten_and_or(result->malloc_pool_, result, $$, $1, $3, T_OP_AND); if (result->pl_parse_info_.is_pl_parse_) { dup_expr_string($$, result, @1.first_column, @3.last_column); } } | expr OR expr %prec OR { - malloc_non_terminal_node($$, result->malloc_pool_, T_OP_OR, 2, $1, $3); + flatten_and_or(result->malloc_pool_, result, $$, $1, $3, T_OP_OR); if (result->pl_parse_info_.is_pl_parse_) { dup_expr_string($$, result, @1.first_column, @3.last_column); } } | expr OR_OP expr %prec OR { - malloc_non_terminal_node($$, result->malloc_pool_, T_OP_OR, 2, $1, $3); + flatten_and_or(result->malloc_pool_, result, $$, $1, $3, T_OP_OR); if (result->pl_parse_info_.is_pl_parse_) { dup_expr_string($$, result, @1.first_column, @3.last_column); } @@ -1910,7 +1936,9 @@ select_with_parens %prec NEG $$ = $1; } | '(' expr_list ')' -{ merge_nodes($$, result, T_EXPR_LIST, $2); } +{ + $$ = $2; +} ; case_expr: @@ -1940,22 +1968,19 @@ COUNT '(' opt_all '*' ')' OVER new_generalized_window_clause { ParseNode *distinct = NULL; malloc_terminal_node(distinct, result->malloc_pool_, T_DISTINCT); - ParseNode *expr_list = NULL; - merge_nodes(expr_list, result, T_EXPR_LIST, $4); + ParseNode *expr_list = $4; malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_COUNT, 2, distinct, expr_list); malloc_non_terminal_node($$, result->malloc_pool_, T_WINDOW_FUNCTION, 2, $$, $7); } | APPROX_COUNT_DISTINCT '(' expr_list ')' OVER new_generalized_window_clause { - ParseNode *expr_list = NULL; - merge_nodes(expr_list, result, T_EXPR_LIST, $3); + ParseNode *expr_list = $3; malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_APPROX_COUNT_DISTINCT, 1, expr_list); malloc_non_terminal_node($$, result->malloc_pool_, T_WINDOW_FUNCTION, 2, $$, $6); } | APPROX_COUNT_DISTINCT_SYNOPSIS '(' expr_list ')' OVER new_generalized_window_clause { - ParseNode *expr_list = NULL; - merge_nodes(expr_list, result, T_EXPR_LIST, $3); + ParseNode *expr_list = $3; malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_APPROX_COUNT_DISTINCT_SYNOPSIS, 1, expr_list); malloc_non_terminal_node($$, result->malloc_pool_, T_WINDOW_FUNCTION, 2, $$, $6); } @@ -2031,15 +2056,13 @@ COUNT '(' opt_all '*' ')' OVER new_generalized_window_clause } | GROUP_CONCAT '(' opt_distinct expr_list opt_order_by opt_separator ')' OVER new_generalized_window_clause { - ParseNode *group_concat_exprs = NULL; - merge_nodes(group_concat_exprs, result, T_EXPR_LIST, $4); + ParseNode *group_concat_exprs = $4; malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_GROUP_CONCAT, 4, $3, group_concat_exprs, $5, $6); malloc_non_terminal_node($$, result->malloc_pool_, T_WINDOW_FUNCTION, 2, $$, $9); } | LISTAGG '(' opt_distinct expr_list opt_order_by opt_separator ')' OVER new_generalized_window_clause { - ParseNode *group_concat_exprs = NULL; - merge_nodes(group_concat_exprs, result, T_EXPR_LIST, $4); + ParseNode *group_concat_exprs = $4; malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_GROUP_CONCAT, 4, $3, group_concat_exprs, $5, $6); malloc_non_terminal_node($$, result->malloc_pool_, T_WINDOW_FUNCTION, 2, $$, $9); } @@ -2197,14 +2220,12 @@ win_fun_lead_lag_params: '(' expr respect_or_ignore NULLS ',' expr_list ')' { ParseNode *params_node = NULL; - malloc_non_terminal_node(params_node, result->malloc_pool_, T_LINK_NODE, 2, $2, $6); - merge_nodes(params_node, result, T_EXPR_LIST, params_node); + push_front_list(result->malloc_pool_, result, params_node, $6, $2); malloc_non_terminal_node($$, result->malloc_pool_, T_INVALID, 2, params_node, $3); } | '(' expr_list ')' opt_respect_or_ignore_nulls { - ParseNode *params_node = NULL; - merge_nodes(params_node, result, T_EXPR_LIST, $2); + ParseNode *params_node = $2; malloc_non_terminal_node($$, result->malloc_pool_, T_INVALID, 2, params_node, $4); } ; @@ -2272,7 +2293,7 @@ opt_partition_by: { $$ = NULL;} | PARTITION BY expr_list { - merge_nodes($$, result, T_EXPR_LIST, $3); + $$ = $3; }; win_rows_or_range: @@ -2423,28 +2444,24 @@ MOD '(' expr ',' expr ')' { ParseNode *distinct = NULL; malloc_terminal_node(distinct, result->malloc_pool_, T_DISTINCT); - ParseNode *expr_list = NULL; - merge_nodes(expr_list, result, T_EXPR_LIST, $4); + ParseNode *expr_list = $4; malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_COUNT, 2, distinct, expr_list); } | COUNT '(' UNIQUE expr_list ')' { ParseNode *distinct = NULL; malloc_terminal_node(distinct, result->malloc_pool_, T_DISTINCT); - ParseNode *expr_list = NULL; - merge_nodes(expr_list, result, T_EXPR_LIST, $4); + ParseNode *expr_list = $4; malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_COUNT, 2, distinct, expr_list); } | APPROX_COUNT_DISTINCT '(' expr_list ')' { - ParseNode *expr_list = NULL; - merge_nodes(expr_list, result, T_EXPR_LIST, $3); + ParseNode *expr_list = $3; malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_APPROX_COUNT_DISTINCT, 1, expr_list); } | APPROX_COUNT_DISTINCT_SYNOPSIS '(' expr_list ')' { - ParseNode *expr_list = NULL; - merge_nodes(expr_list, result, T_EXPR_LIST, $3); + ParseNode *expr_list = $3; malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_APPROX_COUNT_DISTINCT_SYNOPSIS, 1, expr_list); } | APPROX_COUNT_DISTINCT_SYNOPSIS_MERGE '(' expr ')' @@ -2521,8 +2538,7 @@ MOD '(' expr ',' expr ')' } | GROUP_CONCAT '(' opt_distinct expr_list opt_order_by opt_separator ')' { - ParseNode *group_concat_exprs = NULL; - merge_nodes(group_concat_exprs, result, T_EXPR_LIST, $4); + ParseNode *group_concat_exprs = $4; malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_GROUP_CONCAT, 4, $3, group_concat_exprs, $5, $6); } | TOP_K_FRE_HIST '(' DECIMAL_VAL ',' bit_expr ',' INTNUM ',' expr_const ')' @@ -2958,11 +2974,8 @@ MOD '(' expr ',' expr ')' yyerror(NULL, result, "No more space for mallocing string\n"); YYABORT_NO_MEMORY; } - ParseNode *params_node = NULL; - malloc_non_terminal_node(params_node, result->malloc_pool_, T_LINK_NODE, 2, $3, charset_node); - merge_nodes(params_node, result, T_EXPR_LIST, params_node); - + push_back_list(result->malloc_pool_, result, params_node, $3, charset_node); make_name_node($$, result->malloc_pool_, "char"); malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_SYS, 2, $$, params_node); @@ -2970,9 +2983,7 @@ MOD '(' expr ',' expr ')' | CHARACTER '(' expr_list USING charset_name')' { ParseNode *params_node = NULL; - malloc_non_terminal_node(params_node, result->malloc_pool_, T_LINK_NODE, 2, $3, $5); - merge_nodes(params_node, result, T_EXPR_LIST, params_node); - + push_back_list(result->malloc_pool_, result, params_node, $3, $5); make_name_node($$, result->malloc_pool_, "char"); malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_SYS, 2, $$, params_node); } @@ -3157,39 +3168,33 @@ MOD '(' expr ',' expr ')' } | LINESTRING '(' expr_list ')' { - ParseNode *expr_list = NULL; - merge_nodes(expr_list, result, T_EXPR_LIST, $3); + ParseNode *expr_list = $3; malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_SYS_LINESTRING, 1, expr_list); } | MULTIPOINT '(' expr_list ')' { - ParseNode *expr_list = NULL; - merge_nodes(expr_list, result, T_EXPR_LIST, $3); + ParseNode *expr_list = $3; malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_SYS_MULTIPOINT, 1, expr_list); } | MULTILINESTRING '(' expr_list ')' { - ParseNode *expr_list = NULL; - merge_nodes(expr_list, result, T_EXPR_LIST, $3); + ParseNode *expr_list = $3; malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_SYS_MULTILINESTRING, 1, expr_list); } | POLYGON '(' expr_list ')' { - ParseNode *expr_list = NULL; - merge_nodes(expr_list, result, T_EXPR_LIST, $3); + ParseNode *expr_list = $3; malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_SYS_POLYGON, 1, expr_list); } | MULTIPOLYGON '(' expr_list ')' { - ParseNode *expr_list = NULL; - merge_nodes(expr_list, result, T_EXPR_LIST, $3); + ParseNode *expr_list = $3; malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_SYS_MULTIPOLYGON, 1, expr_list); } | geometry_collection '(' expr_list ')' { UNUSED($1); - ParseNode *expr_list = NULL; - merge_nodes(expr_list, result, T_EXPR_LIST, $3); + ParseNode *expr_list = $3; malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_SYS_GEOMCOLLECTION, 1, expr_list); } | geometry_collection '(' ')' @@ -3267,11 +3272,9 @@ INTERVAL '(' expr ',' expr ')' } | INTERVAL '(' expr ',' expr ',' expr_list ')' { - ParseNode *params = NULL; - ParseNode *params_node = NULL; make_name_node($$, result->malloc_pool_, "interval"); - malloc_non_terminal_node(params, result->malloc_pool_, T_LINK_NODE, 2, $5, $7); - merge_nodes(params_node, result, T_EXPR_LIST, params); + ParseNode *params_node = NULL; + push_front_list(result->malloc_pool_, result, params_node, $7, $5); malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_SYS_INTERVAL, 2, $3, params_node); } | CHECK '(' expr ')' @@ -6719,6 +6722,18 @@ NAME_OB $$->param_num_ = 0; $$->is_hidden_const_ = 1; } +| ASCII +{ + malloc_terminal_node($$, result->malloc_pool_, T_VARCHAR); + $$->str_value_ = parse_strdup("ascii", result->malloc_pool_, &($$->str_len_)); + if (OB_UNLIKELY(NULL == $$->str_value_)) { + yyerror(NULL, result, "No more space for mallocing string\n"); + YYABORT_NO_MEMORY; + } + $$->type_ = T_CHAR_CHARSET; + $$->param_num_ = 0; + $$->is_hidden_const_ = 1; +} ; charset_name_or_default: @@ -6860,6 +6875,30 @@ not NULLX { $$ = $1; } +| COLUMN_FORMAT DEFAULT +{ + $$ = NULL; +} +| COLUMN_FORMAT FIXED +{ + $$ = NULL; +} +| COLUMN_FORMAT DYNAMIC +{ + $$ = NULL; +} +| STORAGE DEFAULT +{ + $$ = NULL; +} +| STORAGE DISK +{ + $$ = NULL; +} +| STORAGE MEMORY +{ + $$ = NULL; +} ; opt_column_default_value_list: @@ -6878,6 +6917,12 @@ DEFAULT now_or_signed_literal { malloc_non_terminal_node($$, result->malloc_pool_, T_CONSTR_DEFAULT, 1, $2); } +| DEFAULT '(' expr ')' +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_CONSTR_DEFAULT, 1, $3); + dup_expr_string($$, result, @3.first_column, @3.last_column); +} +; now_or_signed_literal: cur_timestamp_func @@ -7256,6 +7301,110 @@ TABLE_MODE opt_equal_mark STRING_VALUE int_node->value_ = 2; malloc_non_terminal_node($$, result->malloc_pool_, T_EXTERNAL_TABLE_AUTO_REFRESH, 1, int_node); } +| MAX_ROWS opt_equal_mark INTNUM +{ + (void)($2); + (void)($3); + $$ = NULL; +} +| MIN_ROWS opt_equal_mark INTNUM +{ + (void)($2); + (void)($3); + $$ = NULL; +} +| PASSWORD opt_equal_mark STRING_VALUE +{ + (void)($2); + (void)($3); + $$ = NULL; +} +| PACK_KEYS opt_equal_mark INTNUM +{ + (void)($2); + (void)($3); + $$ = NULL; +} +| PACK_KEYS opt_equal_mark DEFAULT +{ + (void)($2); + $$ = NULL; +} +| CONNECTION opt_equal_mark STRING_VALUE +{ + (void)($2); + (void)($3); + $$ = NULL; +} +| DATA DIRECTORY opt_equal_mark STRING_VALUE +{ + (void)($3); + (void)($4); + $$ = NULL; +} +| INDEX DIRECTORY opt_equal_mark STRING_VALUE +{ + (void)($3); + (void)($4); + $$ = NULL; +} +| ENCRYPTION opt_equal_mark STRING_VALUE +{ + (void)($2); + (void)($3); + $$ = NULL; +} +| STATS_AUTO_RECALC opt_equal_mark INTNUM +{ + (void)($2); + (void)($3); + $$ = NULL; +} +| STATS_AUTO_RECALC opt_equal_mark DEFAULT +{ + (void)($2); + $$ = NULL; +} +| STATS_PERSISTENT opt_equal_mark INTNUM +{ + (void)($2); + (void)($3); + $$ = NULL; +} +| STATS_PERSISTENT opt_equal_mark DEFAULT +{ + (void)($2); + $$ = NULL; +} +| STATS_SAMPLE_PAGES opt_equal_mark INTNUM +{ + (void)($2); + (void)($3); + $$ = NULL; +} +| STATS_SAMPLE_PAGES opt_equal_mark DEFAULT +{ + (void)($2); + $$ = NULL; +} +| UNION opt_equal_mark '(' opt_empty_table_list ')' +{ + (void)($2); + (void)($4); + $$ = NULL; +} +| INSERT_METHOD opt_equal_mark merge_insert_types +{ + (void)($2); + (void)($3); + $$ = NULL; +} +; + +merge_insert_types: +NO { $$ = NULL; } +| FIRST { $$ = NULL; } +| LAST { $$ = NULL; } ; parallel_option: @@ -8226,8 +8375,7 @@ TYPE COMP_EQ STRING_VALUE } | NULL_IF_EXETERNAL COMP_EQ '(' expr_list ')' { - ParseNode *expr_list_node = NULL; - merge_nodes(expr_list_node, result, T_EXPR_LIST, $4); + ParseNode *expr_list_node = $4; malloc_non_terminal_node($$, result->malloc_pool_, T_NULL_IF_EXETERNAL, 1, expr_list_node); dup_expr_string($$, result, @4.first_column, @4.last_column); } @@ -9974,11 +10122,11 @@ no_table_select { $$ = $1; } -| table_values_caluse +| table_values_clause { $$ = $1; } -| table_values_caluse_with_order_by_and_limit +| table_values_clause_with_order_by_and_limit { $$ = $1; } @@ -10031,7 +10179,7 @@ no_table_select { $$ = $1; } -| table_values_caluse +| table_values_clause { $$ = $1; } @@ -13416,7 +13564,7 @@ column_name opt_asc_desc * https://dev.mysql.com/doc/refman/8.0/en/values.html * *****************************************************************************/ -table_values_caluse: +table_values_clause: VALUES values_row_list { ParseNode *values_node = NULL; @@ -13427,7 +13575,7 @@ VALUES values_row_list } ; -table_values_caluse_with_order_by_and_limit: +table_values_clause_with_order_by_and_limit: VALUES values_row_list order_by { ParseNode *values_node = NULL; @@ -14114,6 +14262,11 @@ SHOW opt_extended_or_full TABLES opt_from_or_in_database_clause opt_show_conditi (void)($2); malloc_non_terminal_node($$, result->malloc_pool_, T_SHOW_CREATE_TRIGGER, 1, $4); } +| SHOW create_with_opt_hint USER user_host_or_current_user +{ + (void)($2); + malloc_non_terminal_node($$, result->malloc_pool_, T_SHOW_CREATE_USER, 1, $4); +} | SHOW WARNINGS opt_limit { malloc_non_terminal_node($$, result->malloc_pool_, T_SHOW_WARNINGS, 1, $3); @@ -14287,6 +14440,53 @@ SHOW opt_extended_or_full TABLES opt_from_or_in_database_clause opt_show_conditi (void)($4); malloc_non_terminal_node($$, result->malloc_pool_, T_SHOW_OPEN_TABLES, 1, $5); } +| CHECK TABLE table_list check_table_options +{ + (void) ($4); + malloc_non_terminal_node($$, result->malloc_pool_, T_SHOW_CHECK_TABLE, 1, $3, $4); +} +| CHECK TABLE table_list +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_SHOW_CHECK_TABLE, 1, $3); +} +; + +check_table_options: +check_table_option +{ + $$ = $1; +} +| check_table_options check_table_option +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_LINK_NODE, 2, $1, $2); +} +; + +check_table_option: +FOR UPGRADE +{ + $$ = NULL; +} +| QUICK +{ + $$ = NULL; +} +| FAST +{ + $$ = NULL; +} +| MEDIUM +{ + $$ = NULL; +} +| EXTENDED +{ + $$ = NULL; +} +| CHANGED +{ + $$ = NULL; +} ; databases_or_schemas: @@ -14988,6 +15188,21 @@ USER_VARIABLE } ; +user_host_or_current_user: +user opt_host_name +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_USER_WITH_HOST_NAME, 2, $1, $2); +} +| CURRENT_USER +{ + malloc_terminal_node($$, result->malloc_pool_, T_FUN_SYS_CURRENT_USER); +} +| CURRENT_USER '(' ')' +{ + malloc_terminal_node($$, result->malloc_pool_, T_FUN_SYS_CURRENT_USER); +} +; + user_with_host_name: user opt_host_name { @@ -15984,6 +16199,33 @@ role_with_host malloc_terminal_node($$, result->malloc_pool_, T_PRIV_TYPE); $$->value_ = OB_PRIV_RELOAD; } +| REFERENCES +{ + malloc_terminal_node($$, result->malloc_pool_, T_PRIV_TYPE); + $$->value_ = OB_PRIV_REFERENCES; +} +| REFERENCES '(' column_name_list ')' +{ + ParseNode *col_list = NULL; + merge_nodes(col_list, result, T_COLUMN_LIST, $3); + malloc_non_terminal_node($$, result->malloc_pool_, T_PRIV_TYPE, 1, col_list); + $$->value_ = OB_PRIV_REFERENCES; +} +| CREATE ROLE +{ + malloc_terminal_node($$, result->malloc_pool_, T_PRIV_TYPE); + $$->value_ = OB_PRIV_CREATE_ROLE; +} +| DROP ROLE +{ + malloc_terminal_node($$, result->malloc_pool_, T_PRIV_TYPE); + $$->value_ = OB_PRIV_DROP_ROLE; +} +| TRIGGER +{ + malloc_terminal_node($$, result->malloc_pool_, T_PRIV_TYPE); + $$->value_ = OB_PRIV_TRIGGER; +} ; opt_privilege: @@ -17624,6 +17866,11 @@ SET DEFAULT signed_literal { malloc_non_terminal_node($$, result->malloc_pool_, T_CONSTR_DEFAULT, 1, $3); } +| SET DEFAULT '(' expr ')' +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_CONSTR_DEFAULT, 1, $4); + dup_expr_string($$, result, @4.first_column, @4.last_column); +} | DROP DEFAULT { malloc_terminal_node($$, result->malloc_pool_, T_CONSTR_NULL); @@ -18984,7 +19231,6 @@ repair_option_list { $$ = NULL; } ; - opt_checksum_option: QUICK { $$ = NULL; } @@ -20340,12 +20586,716 @@ BEGI * *===========================================================*/ mock_stmt: -FLUSH PRIVILEGES +install_plugin_stmt +{ + $$ = $1; +} +| uninstall_plugin_stmt +{ + $$ = $1; +} +| flush_stmt +{ + $$ = $1; +} +| handler_stmt +{ + (void)($$); + (void)($1); +} +| show_plugin_stmt +{ + $$ = $1; +} +| create_server_stmt +{ + $$ = $1; +} +| alter_server_stmt +{ + $$ = $1; +} +| drop_server_stmt +{ + $$ = $1; +} +| create_logfile_group_stmt +{ + $$ = $1; +} +| alter_logfile_group_stmt +{ + $$ = $1; +} +| drop_logfile_group_stmt +{ + $$ = $1; +} +; + +plugin_name: +NAME_OB +{ + $$ = $1; +} +; + +install_plugin_stmt: +INSTALL PLUGIN plugin_name SONAME STRING_VALUE +{ + (void) ($3); + (void) ($5); + malloc_terminal_node($$, result->malloc_pool_, T_INSTALL_PLUGIN); +} +; + +uninstall_plugin_stmt: +UNINSTALL PLUGIN plugin_name +{ + (void) ($3); + malloc_terminal_node($$, result->malloc_pool_, T_UNINSTALL_PLUGIN); +} +; + +flush_stmt: +FLUSH opt_no_write_to_binlog flush_options +{ + (void) ($2); + $$ = $3; +} +; + +/* In mysql opt table list will add table name to global list */ + +flush_options: +flush_options_list +{ + merge_nodes($$, result, T_FLUSH_MOCK_LIST, $1); +} +/* +| +table_or_tables opt_table_list opt_flush_lock +{ + (void) ($1); + (void) ($2); + (void) ($3); + malloc_terminal_node($$, result->malloc_pool_, T_FLUSH_MOCK); +} +*/ +; + +opt_no_write_to_binlog: +/* empty */ +{ + (void) ($$); +} +| NO_WRITE_TO_BINLOG +{ + (void) ($$); +} +| LOCAL +{ + (void) ($$); +} +; + +/* In mysql the will give a flag to the table */ +// opt_flush_lock: +// /* empty */ +// { +// (void) ($$); +// } +// | WITH READ LOCK_ +// { +// (void) ($$); +// } +// | FOR EXPORT +// { +// (void) ($$); +// } +// ; + + +flush_options_list: +flush_options_list ',' flush_option +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_LINK_NODE, 2, $1, $3); +} +| flush_option +{ + $$ = $1; +} +; + +flush_option: +ERROR_P LOGS +{ + malloc_terminal_node($$, result->malloc_pool_, T_FLUSH_MOCK); +} +| ENGINE_ LOGS +{ + malloc_terminal_node($$, result->malloc_pool_, T_FLUSH_MOCK); +} +| GENERAL LOGS +{ + malloc_terminal_node($$, result->malloc_pool_, T_FLUSH_MOCK); +} +| SLOW LOGS +{ + malloc_terminal_node($$, result->malloc_pool_, T_FLUSH_MOCK); +} +| BINARY LOGS +{ + malloc_terminal_node($$, result->malloc_pool_, T_FLUSH_MOCK); +} +| RELAY LOGS /*opt channle*/ +{ + malloc_terminal_node($$, result->malloc_pool_, T_FLUSH_MOCK); +} +| RELAY LOG NAME_OB +{ + (void)($3); + malloc_terminal_node($$, result->malloc_pool_, T_FLUSH_MOCK); +} +| QUERY CACHE +{ + malloc_terminal_node($$, result->malloc_pool_, T_FLUSH_MOCK); +} +| HOSTS +{ + malloc_terminal_node($$, result->malloc_pool_, T_FLUSH_MOCK); +} +| LOGS +{ + malloc_terminal_node($$, result->malloc_pool_, T_FLUSH_MOCK); +} +| STATUS +{ + malloc_terminal_node($$, result->malloc_pool_, T_FLUSH_MOCK); +} +| DES_KEY_FILE +{ + malloc_terminal_node($$, result->malloc_pool_, T_FLUSH_MOCK); +} +| USER_RESOURCES +{ + malloc_terminal_node($$, result->malloc_pool_, T_FLUSH_MOCK); +} +| OPTIMIZER_COSTS +{ + malloc_terminal_node($$, result->malloc_pool_, T_FLUSH_MOCK); +} +| PRIVILEGES { malloc_terminal_node($$, result->malloc_pool_, T_FLUSH_PRIVILEGES); } ; +handler_stmt: +HANDLER relation_name OPEN opt_relation_name +{ + (void)($$); + (void)($2); + (void)($4); + malloc_terminal_node($$, result->malloc_pool_, T_HANDLER_MOCK); +} +| HANDLER relation_name CLOSE +{ + (void)($$); + (void)($2); + malloc_terminal_node($$, result->malloc_pool_, T_HANDLER_MOCK); +} +| HANDLER relation_name READ +{ + (void)($$); + (void)($2); + malloc_terminal_node($$, result->malloc_pool_, T_HANDLER_MOCK); +} +| HANDLER relation_name READ handler_read_or_scan opt_where opt_limit +{ + (void)($2); + (void)($4); + (void)($5); + (void)($6); + malloc_non_terminal_node($$, result->malloc_pool_, T_HANDLER_MOCK, 3, $2, $4, $5); +} +; + +handler_read_or_scan: +handler_scan_function { + (void)($1); + (void)($$); +} +| NAME_OB handler_rkey_function { + (void)($1); + (void)($2); + (void)($$); +} +; + +handler_scan_function: +FIRST { + (void)($1); + (void)($$); + // malloc_terminal_node($$, result->malloc_pool_, T_HANDLER_FIRST); +} +| NEXT { + (void)($1); + (void)($$); + // malloc_terminal_node($$, result->malloc_pool_, T_HANDLER_NEXT); +} +; + +handler_rkey_function: +FIRST { + (void)($1); + (void)($$); +} +| NEXT { + (void)($1); + (void)($$); +} +| PREV { + (void)($1); + (void)($$); +} +| LAST { + (void)($1); + (void)($$); +} +| handler_rkey_mode '(' expr_list ')' { + (void)($1); + (void)($3); + (void)($$); +} +; + +handler_rkey_mode: +COMP_EQ { + (void)($$); +} +| COMP_GE { + (void)($$); +} +| COMP_LE { + (void)($$); +} +| COMP_GT { + (void)($$); +} +| COMP_LT { + (void)($$); +} +; + +opt_empty_table_list: +/* empty */ +{ + (void) ($$); +} +| table_list +{ + (void) ($$); + (void) ($1); +} +; + +show_plugin_stmt: +SHOW PLUGINS +{ + malloc_terminal_node($$, result->malloc_pool_, T_SHOW_PLUGINS); +} +; + +create_server_stmt: +CREATE SERVER NAME_OB FOREIGN DATA WRAPPER NAME_OB OPTIONS '(' server_options_list ')' +{ + (void)($3); + (void)($7); + (void)($10); + malloc_terminal_node($$, result->malloc_pool_, T_CREATE_SERVER); +} +; + +server_options_list: +server_option +{ + (void)($1); + $$ = NULL; +} +| server_options_list ',' server_option +{ + (void)($1); + (void)($3); + $$ = NULL; +} +; + +server_option: +USER STRING_VALUE +{ + (void)($2); + $$ = NULL; +} +| HOST STRING_VALUE +{ + (void)($2); + $$ = NULL; +} +| DATABASE STRING_VALUE +{ + (void)($2); + $$ = NULL; +} +| OWNER STRING_VALUE +{ + (void)($2); + $$ = NULL; +} +| PASSWORD STRING_VALUE +{ + (void)($2); + $$ = NULL; +} +| SOCKET STRING_VALUE +{ + (void)($2); + $$ = NULL; +} +| PORT INTNUM +{ + (void)($2); + $$ = NULL; +}; + +alter_server_stmt: +ALTER SERVER NAME_OB OPTIONS '(' server_options_list ')' +{ + (void)($3); + (void)($6); + malloc_terminal_node($$, result->malloc_pool_, T_ALTER_SERVER); +} +; + +drop_server_stmt: +DROP SERVER opt_if_exists NAME_OB +{ + (void)($3); + (void)($4); + malloc_terminal_node($$, result->malloc_pool_, T_DROP_SERVER); +} +; + +create_logfile_group_stmt: +CREATE LOGFILE GROUP logfile_group_info +{ + (void)($4); + malloc_terminal_node($$, result->malloc_pool_, T_CREATE_LOGFILE_GROUP); +} +; + +logfile_group_info: +NAME_OB +add_log_file +logfile_group_option_list +{ + (void)($1); + (void)($2); + (void)($3); + $$ = NULL; +} +; + +add_log_file: +ADD lg_undofile +{ + (void)($2); + $$ = NULL; +} +| ADD lg_redofile +{ + (void)($2); + $$ = NULL; +} +; + +lg_undofile: +UNDOFILE STRING_VALUE +{ + (void)($2); + $$ = NULL; +} +; + +lg_redofile: +REDOFILE STRING_VALUE +{ + (void)($2); + $$ = NULL; +} +; + +logfile_group_option_list: +/* empty */ +{ + $$ = NULL; +} +| logfile_group_options +{ + (void)($1); + $$ = NULL; +} +; + +logfile_group_options: +logfile_group_option +{ + (void)($1); + $$ = NULL; +} +| logfile_group_options logfile_group_option +{ + (void)($1); + (void)($2); + $$ = NULL; +} +| logfile_group_options ',' logfile_group_option +{ + (void)($1); + (void)($3); + $$ = NULL; +} +; + +logfile_group_option: +opt_ts_initial_size +{ + (void)($1); + $$ = NULL; +} +| opt_ts_undo_buffer_size +{ + (void)($1); + $$ = NULL; +} +| opt_ts_redo_buffer_size +{ + (void)($1); + $$ = NULL; +} +| opt_ts_nodegroup +{ + (void)($1); + $$ = NULL; +} +| opt_ts_engine +{ + (void)($1); + $$ = NULL; +} +| ts_wait +{ + $$ = NULL; +} +| opt_ts_comment +{ + (void)($1); + $$ = NULL; +} +; + +opt_ts_initial_size: +INITIAL_SIZE opt_equal_mark INTNUM +{ + (void)($2); + (void)($3); + $$ = NULL; +} +; + +opt_ts_undo_buffer_size: +UNDO_BUFFER_SIZE opt_equal_mark INTNUM +{ + (void)($2); + (void)($3); + $$ = NULL; +} +; + +opt_ts_redo_buffer_size: +REDO_BUFFER_SIZE opt_equal_mark INTNUM +{ + (void)($2); + (void)($3); + $$ = NULL; +} +; + +opt_ts_nodegroup: +NODEGROUP opt_equal_mark INTNUM +{ + (void)($2); + (void)($3); + $$ = NULL; +} +; + +opt_ts_comment: +COMMENT opt_equal_mark STRING_VALUE +{ + (void)($2); + (void)($3); + $$ = NULL; +} +; + +alter_logfile_group_stmt: +ALTER LOGFILE GROUP alter_logfile_group_info +{ + (void)($4); + malloc_terminal_node($$, result->malloc_pool_, T_ALTER_LOGFILE_GROUP); +}; + +alter_logfile_group_info: +NAME_OB +add_log_file +alter_logfile_group_option_list +{ + (void)($1); + (void)($2); + (void)($3); + $$ = NULL; +} +; + +alter_logfile_group_option_list: +/* empty */ +{ + $$ = NULL; +} +| alter_logfile_group_options +{ + (void)($1); + $$ = NULL; +} +; + +alter_logfile_group_options: +alter_logfile_group_option +{ + (void)($1); + $$ = NULL; +} +| alter_logfile_group_options alter_logfile_group_option +{ + (void)($1); + (void)($2); + $$ = NULL; +} +| alter_logfile_group_options ',' alter_logfile_group_option +{ + (void)($1); + (void)($3); + $$ = NULL; +} +; + +alter_logfile_group_option: +opt_ts_initial_size +{ + (void)($1); + $$ = NULL; +} +| opt_ts_engine +{ + (void)($1); + $$ = NULL; +} +| ts_wait +{ + $$ = NULL; +} +; + +drop_logfile_group_stmt: +DROP LOGFILE GROUP NAME_OB drop_ts_options_list +{ + (void)($4); + (void)($5); + malloc_terminal_node($$, result->malloc_pool_, T_DROP_LOGFILE_GROUP); +} +; + +drop_ts_options_list: +/* empty */ +{ + $$ = NULL; +} +| drop_ts_options +{ + (void)($1); + $$ = NULL; +} +; + +drop_ts_options: +drop_ts_option +{ + (void)($1); + $$ = NULL; +} +| drop_ts_options drop_ts_option +{ + (void)($1); + (void)($2); + $$ = NULL; +} +| drop_ts_options_list ',' drop_ts_option +{ + (void)($1); + (void)($3); + $$ = NULL; +} +; + +drop_ts_option: +opt_ts_engine +{ + (void)($1); + $$ = NULL; +} +| ts_wait +{ + $$ = NULL; +} +; + +ts_wait: +WAIT +{} +| NO_WAIT +{} +; + +opt_ts_engine: +opt_storage ENGINE_ opt_equal_mark NAME_OB +{ + (void)($1); + (void)($3); + (void)($4); + $$ = NULL; +} +| opt_storage ENGINE_ opt_equal_mark STRING_VALUE +{ + (void)($1); + (void)($3); + (void)($4); + $$ = NULL; +} +; + +/*===========================END=============================*/ + /***************************************************************************** METHOD_OPT grammar ==> used for GatherTableStats METHOD_OPT - The value controls column statistics collection and histogram creation. It accepts @@ -20908,6 +21858,17 @@ NAME_OB { $$ = $1; } } ; +opt_relation_name: +/* empty */ +{ + $$ = NULL; +} +| relation_name +{ + $$ = $1; +} +; + id_dot_id: ID_DOT_ID { $$ = $1; } @@ -22792,6 +23753,7 @@ ACCOUNT | RB_OR_AGG | RB_AND_AGG | OVERWRITE +| OPTIMIZER_COSTS ; unreserved_keyword_special: @@ -23051,7 +24013,6 @@ ACCESSIBLE | XOR | YEAR_MONTH | ZEROFILL - %% //////////////////////////////////////////////////////////////// void yyerror(void *yylloc, ParseResult *p, char *s, ...) diff --git a/src/sql/plan_cache/ob_values_table_compression.cpp b/src/sql/plan_cache/ob_values_table_compression.cpp index 18c4a2448..0f0d138b4 100644 --- a/src/sql/plan_cache/ob_values_table_compression.cpp +++ b/src/sql/plan_cache/ob_values_table_compression.cpp @@ -505,16 +505,13 @@ int ObValuesTableCompression::resolve_params_for_values_clause(ObPlanCacheCtx &p if (OB_SUCC(ret) && !is_same) { new_res_type.reset(); ObExprVersion dummy_op(pc_ctx.allocator_); - const ObLengthSemantics length_semantics = session->get_actual_nls_length_semantics(); - ObCollationType coll_type = CS_TYPE_INVALID; - if (OB_FAIL(session->get_collation_connection(coll_type))) { - LOG_WARN("fail to get_collation_connection", K(ret)); - } else if (OB_FAIL(dummy_op.aggregate_result_type_for_merge(new_res_type, + ObExprTypeCtx type_ctx; + ObSQLUtils::init_type_ctx(session, type_ctx); + if (OB_FAIL(dummy_op.aggregate_result_type_for_merge(new_res_type, &res_types.at(0), res_types.count(), - coll_type, false, - length_semantics))) { + type_ctx))) { LOG_WARN("failed to aggregate result type for merge", K(ret)); } } @@ -657,16 +654,13 @@ int ObValuesTableCompression::resolve_params_for_values_clause(ObPlanCacheCtx &p if (OB_SUCC(ret) && !is_same) { new_res_type.reset(); ObExprVersion dummy_op(pc_ctx.allocator_); - const ObLengthSemantics length_semantics = session->get_actual_nls_length_semantics(); - ObCollationType coll_type = CS_TYPE_INVALID; - if (OB_FAIL(session->get_collation_connection(coll_type))) { - LOG_WARN("fail to get_collation_connection", K(ret)); - } else if (OB_FAIL(dummy_op.aggregate_result_type_for_merge(new_res_type, + ObExprTypeCtx type_ctx; + ObSQLUtils::init_type_ctx(session, type_ctx); + if (OB_FAIL(dummy_op.aggregate_result_type_for_merge(new_res_type, &res_types.at(0), res_types.count(), - coll_type, false, - length_semantics))) { + type_ctx))) { LOG_WARN("failed to aggregate result type for merge", K(ret)); } else { LOG_TRACE("get result type", K(new_res_type), K(res_types)); diff --git a/src/sql/printer/ob_dml_stmt_printer.cpp b/src/sql/printer/ob_dml_stmt_printer.cpp index 6a4456d6e..46724cd6f 100644 --- a/src/sql/printer/ob_dml_stmt_printer.cpp +++ b/src/sql/printer/ob_dml_stmt_printer.cpp @@ -1709,6 +1709,24 @@ int ObDMLStmtPrinter::print_base_table(const TableItem *table_item) DATA_PRINTF(")"); } } + + if (OB_SUCC(ret) && OB_NOT_NULL(table_item->sample_info_)) { + if (SampleInfo::SampleMethod::BLOCK_SAMPLE == table_item->sample_info_->method_) { + DATA_PRINTF(" sample block("); + DATA_PRINTF("%lf", table_item->sample_info_->percent_); + DATA_PRINTF(")"); + } else if (SampleInfo::SampleMethod::ROW_SAMPLE == table_item->sample_info_->method_) { + DATA_PRINTF(" sample("); + DATA_PRINTF("%lf", table_item->sample_info_->percent_); + DATA_PRINTF(")"); + } + if (table_item->sample_info_->seed_ != -1) { + DATA_PRINTF(" seed("); + DATA_PRINTF(" %ld ", table_item->sample_info_->seed_); + DATA_PRINTF(")"); + } + } + // flashback query if (OB_SUCC(ret)) { bool explain_non_extend = false; diff --git a/src/sql/printer/ob_raw_expr_printer.cpp b/src/sql/printer/ob_raw_expr_printer.cpp index d45999219..e07fe8382 100644 --- a/src/sql/printer/ob_raw_expr_printer.cpp +++ b/src/sql/printer/ob_raw_expr_printer.cpp @@ -2967,6 +2967,28 @@ int ObRawExprPrinter::print(ObSysFunRawExpr *expr) } break; } + case T_OP_GET_SYS_VAR: { + int64_t param_num = expr->get_param_count(); + ObRawExpr *name_expr = NULL; + ObRawExpr *scope_expr = NULL; + if (OB_UNLIKELY(2 != param_num) || + OB_ISNULL(name_expr = expr->get_param_expr(0)) || + OB_ISNULL(scope_expr = expr->get_param_expr(1))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid param count", K(ret), K(expr->get_param_count())); + } else if (OB_UNLIKELY(!name_expr->is_const_raw_expr()) || + OB_UNLIKELY(!static_cast(name_expr)->get_value().is_varchar()) || + OB_UNLIKELY(!scope_expr->is_const_raw_expr()) || + OB_UNLIKELY(!static_cast(scope_expr)->get_value().is_int())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid user variable name", K(ret)); + } else { + DATA_PRINTF("@@"); + func_name = static_cast(expr->get_param_expr(0))->get_value().get_varchar(); + DATA_PRINTF("%.*s", LEN_AND_PTR(func_name)); + } + break; + } case T_FUN_COLUMN_CONV: { int64_t param_num = expr->get_param_count(); if ((param_num != ObExprColumnConv::PARAMS_COUNT_WITH_COLUMN_INFO diff --git a/src/sql/privilege_check/ob_privilege_check.cpp b/src/sql/privilege_check/ob_privilege_check.cpp index 1284a6d51..d1d36fb19 100644 --- a/src/sql/privilege_check/ob_privilege_check.cpp +++ b/src/sql/privilege_check/ob_privilege_check.cpp @@ -65,6 +65,7 @@ #include "sql/resolver/ddl/ob_create_routine_stmt.h" #include "sql/resolver/ddl/ob_alter_routine_stmt.h" #include "sql/resolver/ddl/ob_drop_routine_stmt.h" +#include "sql/resolver/ddl/ob_trigger_stmt.h" #include "rootserver/ob_ddl_service.h" #include "sql/resolver/dml/ob_merge_stmt.h" #include "sql/privilege_check/ob_ora_priv_check.h" @@ -1233,6 +1234,7 @@ int get_alter_table_stmt_need_privs( ObIArray &need_privs) { int ret = OB_SUCCESS; + bool need_check = false; if (OB_ISNULL(basic_stmt)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("Basic stmt should be not be NULL", K(ret)); @@ -1243,6 +1245,7 @@ int get_alter_table_stmt_need_privs( } else { ObNeedPriv need_priv; const ObAlterTableStmt *stmt = static_cast(basic_stmt); + const ObSArray &foreign_keys = stmt->get_read_only_foreign_key_arg_list(); if (OB_FAIL(ObPrivilegeCheck::can_do_operation_on_db(session_priv, stmt->get_org_database_name()))) { LOG_WARN("Can not alter table in the database", K(session_priv), K(ret), "database_name", stmt->get_org_database_name()); @@ -1284,6 +1287,22 @@ int get_alter_table_stmt_need_privs( LOG_USER_ERROR(OB_ERR_NO_PRIVILEGE, "SUPER"); } } + // check references privilege + if (OB_SUCC(ret)) { + if (OB_FAIL(ObPrivilegeCheck::get_priv_need_check(session_priv, + ObCompatFeatureType::MYSQL_REFERENCES_PRIV_ENHANCE, + need_check))) { + LOG_WARN("failed to get priv need check", K(ret)); + } else if (lib::is_mysql_mode() && need_check) { + for (int64_t i = 0; OB_SUCC(ret) && i < foreign_keys.count(); i++) { + need_priv.db_ = foreign_keys.at(i).parent_database_; + need_priv.table_ = foreign_keys.at(i).parent_table_; + need_priv.priv_level_ = OB_PRIV_TABLE_LEVEL; + need_priv.priv_set_ = OB_PRIV_REFERENCES; + ADD_NEED_PRIV(need_priv); + } + } + } } return ret; } @@ -1385,6 +1404,7 @@ int get_create_table_stmt_need_privs( ObIArray &need_privs) { int ret = OB_SUCCESS; + bool need_check = false; if (OB_ISNULL(basic_stmt)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("Basic stmt should be not be NULL", K(ret)); @@ -1406,6 +1426,7 @@ int get_create_table_stmt_need_privs( } } else { const ObSelectStmt *select_stmt = stmt->get_sub_select(); + const ObSArray &foreign_keys = stmt->get_read_only_foreign_key_arg_list(); if (NULL != select_stmt) { need_priv.priv_set_ = OB_PRIV_CREATE | OB_PRIV_INSERT; } else { @@ -1418,6 +1439,22 @@ int get_create_table_stmt_need_privs( if (OB_SUCC(ret) && NULL != select_stmt) { OZ (ObPrivilegeCheck::get_stmt_need_privs(session_priv, select_stmt, need_privs)); } + // check references privilege + if (OB_SUCC(ret)) { + if (OB_FAIL(ObPrivilegeCheck::get_priv_need_check(session_priv, + ObCompatFeatureType::MYSQL_REFERENCES_PRIV_ENHANCE, + need_check))) { + LOG_WARN("failed to get priv need check", K(ret)); + } else if (lib::is_mysql_mode() && need_check) { + for (int64_t i = 0; OB_SUCC(ret) && i < foreign_keys.count(); i++) { + need_priv.db_ = foreign_keys.at(i).parent_database_; + need_priv.table_ = foreign_keys.at(i).parent_table_; + need_priv.priv_level_ = OB_PRIV_TABLE_LEVEL; + need_priv.priv_set_ = OB_PRIV_REFERENCES; + ADD_NEED_PRIV(need_priv); + } + } + } } } return ret; @@ -2166,14 +2203,16 @@ int get_role_privs( stmt::StmtType stmt_type = basic_stmt->get_stmt_type(); switch (stmt_type) { case stmt::T_CREATE_ROLE: { - need_priv.priv_set_ = OB_PRIV_CREATE_USER; //[TODO ROLE] + need_priv.priv_set_ = OB_PRIV_CREATE_USER | OB_PRIV_CREATE_ROLE; need_priv.priv_level_ = OB_PRIV_USER_LEVEL; + need_priv.priv_check_type_ = OB_PRIV_CHECK_ANY; ADD_NEED_PRIV(need_priv); break; } case stmt::T_DROP_ROLE: { - need_priv.priv_set_ = OB_PRIV_CREATE_USER; + need_priv.priv_set_ = OB_PRIV_CREATE_USER | OB_PRIV_DROP_ROLE; need_priv.priv_level_ = OB_PRIV_USER_LEVEL; + need_priv.priv_check_type_ = OB_PRIV_CHECK_ANY; ADD_NEED_PRIV(need_priv); break; } @@ -2353,6 +2392,48 @@ int get_routine_stmt_need_privs( return ret; } +int get_trigger_stmt_need_privs( + const ObSessionPrivInfo &session_priv, + const ObStmt *basic_stmt, + ObIArray &need_privs) +{ + int ret = OB_SUCCESS; + bool need_check = false; + if (OB_ISNULL(basic_stmt)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Basic stmt should be not be NULL", K(ret)); + } else if (OB_UNLIKELY(stmt::T_CREATE_TRIGGER != basic_stmt->get_stmt_type() + && stmt::T_DROP_TRIGGER != basic_stmt->get_stmt_type())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Stmt type should be trigger stmt", + K(ret), "stmt type", basic_stmt->get_stmt_type()); + } else if (OB_FAIL(ObPrivilegeCheck::get_priv_need_check(session_priv, + ObCompatFeatureType::MYSQL_TRIGGER_PRIV_CHECK, need_check))) { + LOG_WARN("failed to get priv need check", K(ret)); + } else if (lib::is_mysql_mode() && need_check) { + if (stmt::T_CREATE_TRIGGER == basic_stmt->get_stmt_type()) { + const ObCreateTriggerStmt *stmt = static_cast(basic_stmt); + ObNeedPriv need_priv; + need_priv.table_ = stmt->get_trigger_arg().base_object_name_; + need_priv.db_ = stmt->get_trigger_arg().base_object_database_; + need_priv.priv_level_ = OB_PRIV_TABLE_LEVEL; + need_priv.priv_set_ = OB_PRIV_TRIGGER; + ADD_NEED_PRIV(need_priv); + } else if (stmt::T_DROP_TRIGGER == basic_stmt->get_stmt_type()) { + const ObDropTriggerStmt *stmt = static_cast(basic_stmt); + if(stmt->is_exist) { + ObNeedPriv need_priv; + need_priv.table_ = stmt->trigger_table_name_; + need_priv.db_ = stmt->get_trigger_arg().trigger_database_; + need_priv.priv_level_ = OB_PRIV_TABLE_LEVEL; + need_priv.priv_set_ = OB_PRIV_TRIGGER; + ADD_NEED_PRIV(need_priv); + } + } + } + return ret; +} + int get_drop_tenant_stmt_need_privs( const ObSessionPrivInfo &session_priv, const ObStmt *basic_stmt, diff --git a/src/sql/resolver/cmd/ob_alter_system_resolver.cpp b/src/sql/resolver/cmd/ob_alter_system_resolver.cpp index ed6b45b18..e05008b48 100644 --- a/src/sql/resolver/cmd/ob_alter_system_resolver.cpp +++ b/src/sql/resolver/cmd/ob_alter_system_resolver.cpp @@ -2212,13 +2212,18 @@ int ObSetConfigResolver::resolve(const ParseNode &parse_tree) // config value ObObjParam val; + ObDefaultValueRes resolve_res(val); if (OB_UNLIKELY(NULL == action_node->children_[1])) { ret = OB_ERR_UNEXPECTED; LOG_WARN("children[1] should not be null"); break; - } else if (OB_FAIL(ddl_resolver.resolve_default_value(action_node->children_[1], val))) { + } else if (OB_FAIL(ddl_resolver.resolve_default_value(action_node->children_[1], resolve_res))) { LOG_WARN("resolve config value failed", K(ret)); break; + } else if (!resolve_res.is_literal_) { + ret = OB_ERR_ILLEGAL_TYPE; + LOG_WARN("resolve config value failed", K(ret), K(resolve_res.is_literal_)); + break; } ObString str_val; ObCollationType cast_coll_type = CS_TYPE_INVALID; @@ -2245,6 +2250,8 @@ int ObSetConfigResolver::resolve(const ParseNode &parse_tree) } else if (OB_FAIL(item.value_.assign(str_val))) { LOG_WARN("assign config value failed", K(ret), K(str_val)); break; + } else if (OB_FAIL(convert_param_value(item))) { + LOG_WARN("convert config value failed", K(ret)); } else if (session_info_ != NULL && action_node->children_[4] == NULL && OB_FAIL(check_param_valid(session_info_->get_effective_tenant_id(), ObString(item.name_.size(), item.name_.ptr()), @@ -2514,6 +2521,29 @@ int ObSetConfigResolver::check_param_valid(int64_t tenant_id , return ret; } +int ObSetConfigResolver::convert_param_value(ObAdminSetConfigItem &item) +{ + int ret = OB_SUCCESS; + if (0 == item.name_.str().case_compare("audit_log_path")) { + ObBackupDest dest; + ObBackupPathString path; + if (item.value_.str().empty()) { + // do nothing + } else if (OB_FAIL(dest.set(item.value_.str()))) { + LOG_WARN("failed to set backup dest", K(ret)); + if (OB_INVALID_BACKUP_DEST == ret) { + // let config checker return the actual error info + ret = OB_SUCCESS; + } + } else if (OB_FAIL(dest.get_backup_dest_str(path.ptr(), path.capacity()))) { + LOG_WARN("failed to get backup dest", K(ret)); + } else if (OB_FAIL(item.value_.assign(path.str()))) { + LOG_WARN("failed to assign config value", K(ret)); + } + } + return ret; +} + int ObSetTPResolver::resolve(const ParseNode &parse_tree) { int ret = OB_SUCCESS; @@ -4625,9 +4655,14 @@ int ObAlterSystemSetResolver::resolve(const ParseNode &parse_tree) LOG_WARN("value node is NULL", K(ret)); } else { ObObjParam val; - if (OB_FAIL(ddl_resolver.resolve_default_value(value_node, val))) { + ObDefaultValueRes resolve_res(val); + if (OB_FAIL(ddl_resolver.resolve_default_value(value_node, resolve_res))) { LOG_WARN("resolve config value failed", K(ret)); break; + } else if (!resolve_res.is_literal_) { + ret = OB_ERR_ILLEGAL_TYPE; + LOG_WARN("resolve config value failed", K(ret), K(resolve_res.is_literal_)); + break; } ObString str_val; ObCollationType cast_coll_type = CS_TYPE_INVALID; diff --git a/src/sql/resolver/cmd/ob_alter_system_resolver.h b/src/sql/resolver/cmd/ob_alter_system_resolver.h index 796cf9b81..a7159ab33 100644 --- a/src/sql/resolver/cmd/ob_alter_system_resolver.h +++ b/src/sql/resolver/cmd/ob_alter_system_resolver.h @@ -246,6 +246,7 @@ public: private: int check_param_valid(int64_t tenant_id, const common::ObString &name_node, const common::ObString &value_node); + int convert_param_value(ObAdminSetConfigItem &item); }; class ObTransferPartitionResolver : public ObSystemCmdResolver { diff --git a/src/sql/resolver/cmd/ob_mock_resolver.cpp b/src/sql/resolver/cmd/ob_mock_resolver.cpp index 86c0dee58..fbcb1df65 100644 --- a/src/sql/resolver/cmd/ob_mock_resolver.cpp +++ b/src/sql/resolver/cmd/ob_mock_resolver.cpp @@ -22,32 +22,66 @@ namespace sql int ObMockResolver::resolve(const ParseNode& parse_tree) { int ret = OB_SUCCESS; - ObMockStmt *mock_stmt = NULL; - if (OB_UNLIKELY(NULL == (mock_stmt = create_stmt()))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to create mock stmt"); - } else { - switch (parse_tree.type_) { - case T_FLUSH_PRIVILEGES: - mock_stmt->set_stmt_type(stmt::T_FLUSH_PRIVILEGES); - break; - case T_REPAIR_TABLE: - mock_stmt->set_stmt_type(stmt::T_REPAIR_TABLE); - break; - case T_CHECKSUM_TABLE: - mock_stmt->set_stmt_type(stmt::T_CHECKSUM_TABLE); - break; - case T_CACHE_INDEX: - mock_stmt->set_stmt_type(stmt::T_CACHE_INDEX); - break; - case T_LOAD_INDEX_INTO_CACHE: - mock_stmt->set_stmt_type(stmt::T_LOAD_INDEX_INTO_CACHE); - break; - default: - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected parse tree type", K(ret), K(parse_tree.type_)); - break; + switch (parse_tree.type_) { + case T_FLUSH_PRIVILEGES: + case T_INSTALL_PLUGIN: + case T_UNINSTALL_PLUGIN: + case T_FLUSH_MOCK: + case T_HANDLER_MOCK: + case T_SHOW_PLUGINS: + case T_REPAIR_TABLE: + case T_CHECKSUM_TABLE: + case T_CACHE_INDEX: + case T_LOAD_INDEX_INTO_CACHE: + case T_CREATE_SERVER: + case T_ALTER_SERVER: + case T_DROP_SERVER: + case T_CREATE_LOGFILE_GROUP: + case T_ALTER_LOGFILE_GROUP: + case T_DROP_LOGFILE_GROUP: + { + ObMockStmt *mock_stmt = NULL; + if (OB_UNLIKELY(NULL == (mock_stmt = create_stmt()))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to create mock stmt"); + } else { + const stmt::StmtType stmt_code = type_convert(parse_tree.type_); + if (stmt_code == stmt::T_NONE) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("stmt code is null", K(ret)); + } else { + mock_stmt->set_stmt_type(stmt_code); + } + } + break; } + case T_FLUSH_MOCK_LIST: + { + ObMockStmt *mock_stmt = NULL; + if (OB_UNLIKELY(NULL == (mock_stmt = create_stmt()))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to create mock stmt"); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < parse_tree.num_child_; ++i) { + ParseNode *node = parse_tree.children_[i]; + if (OB_NOT_NULL(node)) { + if (node->type_ != T_FLUSH_MOCK && node->type_ != T_FLUSH_PRIVILEGES) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected parse tree type", K(ret), K(node->type_)); + } else if (OB_FAIL(mock_stmt->add_stmt(type_convert(node->type_)))) { + LOG_WARN("failed to add stmt", K(ret), K(node->type_)); + } + } + } + if (OB_SUCC(ret)) { + mock_stmt->set_stmt_type(stmt::T_FLUSH_MOCK_LIST); + } + } + break; + } + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected parse tree type", K(ret), K(parse_tree.type_)); } return ret; } diff --git a/src/sql/resolver/cmd/ob_mock_resolver.h b/src/sql/resolver/cmd/ob_mock_resolver.h index 808f41748..6ec1c0eaa 100644 --- a/src/sql/resolver/cmd/ob_mock_resolver.h +++ b/src/sql/resolver/cmd/ob_mock_resolver.h @@ -25,12 +25,70 @@ public: ObCMDResolver(params) { } + virtual ~ObMockResolver() { } virtual int resolve(const ParseNode &parse_tree); private: DISALLOW_COPY_AND_ASSIGN(ObMockResolver); + + stmt::StmtType type_convert(ObItemType type) { + stmt::StmtType ret = stmt::T_NONE; + switch (type) { + case T_FLUSH_PRIVILEGES: + ret = stmt::T_FLUSH_PRIVILEGES; + break; + case T_INSTALL_PLUGIN: + ret = stmt::T_INSTALL_PLUGIN; + break; + case T_UNINSTALL_PLUGIN: + ret = stmt::T_UNINSTALL_PLUGIN; + break; + case T_FLUSH_MOCK: + ret = stmt::T_FLUSH_MOCK; + break; + case T_HANDLER_MOCK: + ret = stmt::T_HANDLER_MOCK; + break; + case T_SHOW_PLUGINS: + ret = stmt::T_SHOW_PLUGINS; + break; + case T_REPAIR_TABLE: + ret = stmt::T_REPAIR_TABLE; + break; + case T_CHECKSUM_TABLE: + ret = stmt::T_CHECKSUM_TABLE; + break; + case T_CACHE_INDEX: + ret = stmt::T_CACHE_INDEX; + break; + case T_LOAD_INDEX_INTO_CACHE: + ret = stmt::T_LOAD_INDEX_INTO_CACHE; + break; + case T_CREATE_SERVER: + ret = stmt::T_CREATE_SERVER; + break; + case T_ALTER_SERVER: + ret = stmt::T_ALTER_SERVER; + break; + case T_DROP_SERVER: + ret = stmt::T_DROP_SERVER; + break; + case T_CREATE_LOGFILE_GROUP: + ret = stmt::T_CREATE_LOGFILE_GROUP; + break; + case T_ALTER_LOGFILE_GROUP: + ret = stmt::T_ALTER_LOGFILE_GROUP; + break; + case T_DROP_LOGFILE_GROUP: + ret = stmt::T_DROP_LOGFILE_GROUP; + break; + default: + ret = stmt::T_NONE; + } + return ret; + } }; } // namespace sql diff --git a/src/sql/resolver/cmd/ob_mock_stmt.h b/src/sql/resolver/cmd/ob_mock_stmt.h index 6fc5eb170..f11139009 100644 --- a/src/sql/resolver/cmd/ob_mock_stmt.h +++ b/src/sql/resolver/cmd/ob_mock_stmt.h @@ -27,8 +27,17 @@ public: } virtual ~ObMockStmt() {} + int add_stmt(stmt::StmtType stmt_type) { + return stmt_types_.push_back(stmt_type); + } + + const ObIArray &get_stmt_type_list() const { + return stmt_types_; + } + private: DISALLOW_COPY_AND_ASSIGN(ObMockStmt); + ObSEArray stmt_types_; }; } } diff --git a/src/sql/resolver/cmd/ob_show_resolver.cpp b/src/sql/resolver/cmd/ob_show_resolver.cpp index 856801c26..a487674b0 100644 --- a/src/sql/resolver/cmd/ob_show_resolver.cpp +++ b/src/sql/resolver/cmd/ob_show_resolver.cpp @@ -22,6 +22,7 @@ #include "observer/ob_server_struct.h" #include "sql/resolver/dcl/ob_grant_resolver.h" #include "observer/virtual_table/ob_tenant_all_tables.h" +#include "share/schema/ob_schema_printer.h" using namespace oceanbase::common; using namespace oceanbase::share; using namespace oceanbase::share::schema; @@ -81,6 +82,54 @@ ObShowResolver::~ObShowResolver() #define REAL_NAME(a, b) ((!is_oracle_mode) ? (a) : (b)) +static constexpr char check_table_ok_template[] = + "(SELECT \'%s.%s\' As \"Table\", " + "'check' As Op, " + "'status' As Msg_type, " + "'OK' As Msg_text " + "FROM dual)"; +static constexpr char check_table_db_not_exists1[] = + "(SELECT \'%s.%s\' As \"Table\", " + "'check' As Op, " + "'Error' As Msg_type, " + "\"Unknown database \'%s\'\" As Msg_text " + "FROM dual)"; + +static constexpr char check_table_db_not_exists2[] = + "(SELECT \'%s.%s\' As \"Table\", " + "'check' As Op, " + "'error' As Msg_type, " + "'Corrupt' As Msg_text " + "FROM dual)"; + +static constexpr char check_table_table_not_exists1[] = + "(SELECT \'%s.%s\' As \"Table\", " + "'check' As Op, " + "'Error' As Msg_type, " + "\"Table \'%s.%s\' doesn't exist\" As Msg_text " + "FROM dual)"; + +static constexpr char check_table_table_not_exists2[] = + "(SELECT \'%s.%s\' As \"Table\", " + "'check' As Op, " + "'status' As Msg_type, " + "'Operation failed' As Msg_text " + "FROM dual)"; + +static constexpr char check_table_view_invalid1[] = + "(SELECT \'%s.%s\' As \"Table\", " + "'check' As Op, " + "'Error' As Msg_type, " + "\"View \'%s.%s\' references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them\" As Msg_text " + "FROM dual)"; + +static constexpr char check_table_view_invalid2[] = + "(SELECT \'%s.%s\' As \"Table\", " + "'check' As Op, " + "'error' As Msg_type, " + "'Corrupt' As Msg_text " + "FROM dual)"; + int ObShowResolver::resolve(const ParseNode &parse_tree) { int ret = OB_SUCCESS; @@ -105,9 +154,14 @@ int ObShowResolver::resolve(const ParseNode &parse_tree) K(params_.allocator_), K(schema_checker_)); } else if (OB_UNLIKELY(parse_tree.type_ < T_SHOW_TABLES || parse_tree.type_ > T_SHOW_GRANTS) - && (parse_tree.type_ != T_SHOW_TRIGGERS) && (parse_tree.type_ != T_SHOW_PROFILE) - && (parse_tree.type_ != T_SHOW_PROCEDURE_CODE) && (parse_tree.type_ != T_SHOW_FUNCTION_CODE) - && (parse_tree.type_ != T_SHOW_ENGINE) && (parse_tree.type_ != T_SHOW_OPEN_TABLES)) { + && OB_UNLIKELY(parse_tree.type_ != T_SHOW_TRIGGERS) + && OB_UNLIKELY(parse_tree.type_ != T_SHOW_PROFILE) + && OB_UNLIKELY(parse_tree.type_ != T_SHOW_PROCEDURE_CODE) + && OB_UNLIKELY(parse_tree.type_ != T_SHOW_FUNCTION_CODE) + && OB_UNLIKELY(parse_tree.type_ != T_SHOW_ENGINE) + && OB_UNLIKELY(parse_tree.type_ != T_SHOW_OPEN_TABLES) + && OB_UNLIKELY(parse_tree.type_ != T_SHOW_CREATE_USER) + && OB_UNLIKELY(parse_tree.type_ != T_SHOW_CHECK_TABLE)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected parse tree type", K(ret), K(parse_tree.type_)); } else { @@ -549,6 +603,20 @@ int ObShowResolver::resolve(const ParseNode &parse_tree) }(); break; } + case T_SHOW_CHECK_TABLE: + { + if (OB_FAIL(resolve_show_check_table(parse_tree, show_resv_ctx, select_sql))) { + LOG_WARN("failed to resolve show check table", K(ret)); + } + break; + } + case T_SHOW_CREATE_USER: + { + if (OB_FAIL(resolve_show_create_user(parse_tree, show_resv_ctx, session_priv, stmt_need_privs, select_sql, sql_gen))) { + LOG_WARN("failed to resolve show create user", K(ret)); + } + break; + } case T_SHOW_CREATE_PROCEDURE: case T_SHOW_CREATE_FUNCTION: { [&] { @@ -651,23 +719,22 @@ int ObShowResolver::resolve(const ParseNode &parse_tree) uint64_t show_db_id = OB_INVALID_ID; ObString show_tg_name; uint64_t show_tg_id = OB_INVALID_ID; + ObString show_table_name; bool allow_show = false; if (OB_FAIL(resolve_show_from_trigger(parse_tree.children_[0], NULL, database_name.empty(), real_tenant_id, show_db_name, - show_db_id, show_tg_name, show_tg_id))) { + show_db_id, show_tg_name, show_tg_id, show_table_name))) { LOG_WARN("fail to resolve show from trigger", K(ret)); } if (OB_FAIL(ret)) { } else if (OB_FAIL(schema_checker_->check_trigger_show(session_priv, show_db_name, - show_tg_name, allow_show))) { + show_tg_name, allow_show, show_table_name))) { LOG_WARN("Check trigger show error", K(ret)); } else if (!allow_show) { - ret = OB_ERR_NO_TABLE_PRIVILEGE; - LOG_USER_ERROR(OB_ERR_NO_TABLE_PRIVILEGE, static_cast(strlen("SHOW")), "SHOW", - session_priv.user_name_.length(), session_priv.user_name_.ptr(), - session_priv.host_name_.length(), session_priv.host_name_.ptr(), - show_tg_name.length(), show_tg_name.ptr()); + ret = OB_ERR_NO_PRIVILEGE; + LOG_WARN("trigger not has priv", K(ret), K(show_db_name), K(show_table_name), K(show_tg_name)); + LOG_USER_ERROR(OB_ERR_NO_PRIVILEGE, "TRIGGER"); } else { }//do nothing if (OB_SUCC(ret)) { show_resv_ctx.stmt_type_ = stmt::T_SHOW_CREATE_TRIGGER; @@ -1703,6 +1770,306 @@ int ObShowResolver::resolve(const ParseNode &parse_tree) return ret; } +int ObShowResolver::resolve_show_check_table(const ParseNode &parse_tree, + ObShowResolverContext &show_resv_ctx, + ObString &select_sql) +{ + int ret = OB_SUCCESS; + const bool is_oracle_mode = lib::is_oracle_mode(); + show_resv_ctx.stmt_type_ = stmt::T_SHOW_CHECK_TABLE; + static const int64_t MAX_CHECK_TABLE_CNT = 10000; + ObArenaAllocator alloc("ShowCKTable", OB_MALLOC_NORMAL_BLOCK_SIZE, session_info_->get_effective_tenant_id()); + ObSEArray infos; + TableInfoSet tables_set; + if (OB_UNLIKELY(is_oracle_mode)) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "check table oracle mode is"); + } else if (OB_FAIL(tables_set.create(MAX_CHECK_TABLE_CNT / 2))) { + LOG_WARN("failed to create hash set", K(ret)); + } else if (OB_UNLIKELY(1 != parse_tree.num_child_) || OB_ISNULL(parse_tree.children_[0])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("parse tree is wrong", K(ret)); + } else if (OB_FAIL(recursive_resolve_table_info(parse_tree.children_[0], alloc, infos, tables_set))) { + LOG_WARN("failed to resolve table info", K(ret)); + } else if (infos.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get error info", K(ret)); + } else if (infos.count() >= MAX_CHECK_TABLE_CNT) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "check table count exceed 10000 is"); + } + if (OB_SUCC(ret)) { + ObSqlString check_table_str; + for (int64_t i = 0; OB_SUCC(ret) && i < infos.count(); ++i) { + if (i > 0) { + OZ (check_table_str.append(" UNION ALL ")); + } + if (!infos.at(i).db_exist_) { + OZ (check_table_str.append_fmt(check_table_db_not_exists1, infos.at(i).db_name_.ptr(), infos.at(i).table_name_.ptr(), + infos.at(i).db_name_.ptr())); + OZ (check_table_str.append(" UNION ALL ")); + OZ (check_table_str.append_fmt(check_table_db_not_exists2, infos.at(i).db_name_.ptr(), infos.at(i).table_name_.ptr())); + } else if (!infos.at(i).table_exist_) { + OZ (check_table_str.append_fmt(check_table_table_not_exists1, infos.at(i).db_name_.ptr(), infos.at(i).table_name_.ptr(), + infos.at(i).db_name_.ptr(), infos.at(i).table_name_.ptr())); + OZ (check_table_str.append(" UNION ALL ")); + OZ (check_table_str.append_fmt(check_table_table_not_exists2, infos.at(i).db_name_.ptr(), infos.at(i).table_name_.ptr())); + } else if (infos.at(i).is_view_ && !infos.at(i).valid_) { + OZ (check_table_str.append_fmt(check_table_view_invalid1, infos.at(i).db_name_.ptr(), infos.at(i).table_name_.ptr(), + infos.at(i).db_name_.ptr(), infos.at(i).table_name_.ptr())); + OZ (check_table_str.append(" UNION ALL ")); + OZ (check_table_str.append_fmt(check_table_view_invalid2, infos.at(i).db_name_.ptr(), infos.at(i).table_name_.ptr())); + } else { + OZ (check_table_str.append_fmt(check_table_ok_template, infos.at(i).db_name_.ptr(), infos.at(i).table_name_.ptr())); + } + } + OZ (check_table_str.append(";")); + OZ (ob_write_string(*allocator_, check_table_str.string(), select_sql)); + } + return ret; +} + +int ObShowResolver::resolve_show_create_user(const ParseNode &parse_tree, + ObShowResolverContext &show_resv_ctx, + ObSessionPrivInfo &session_priv, + ObStmtNeedPrivs &stmt_need_privs, + ObString &select_sql, + ObSqlStrGenerator &sql_gen) +{ + int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + uint64_t tenant_id = OB_INVALID_ID; + uint64_t user_id = OB_INVALID_ID; + ObString user_name; + ObString host_name; + bool has_select_privilege = false; + bool show_current_user = false; + const ObUserInfo *user_info = NULL; + ObSchemaGetterGuard *schema_guard = NULL; + const bool is_oracle_mode = lib::is_oracle_mode(); + show_resv_ctx.stmt_type_ = stmt::T_SHOW_CREATE_USER; + if (OB_UNLIKELY(is_oracle_mode)) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "show table in oracle mode is"); + } else if (OB_ISNULL(session_info_) || + OB_ISNULL(params_.schema_checker_) || + OB_ISNULL(schema_guard = params_.schema_checker_->get_schema_guard())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("got null ptr", K(ret)); + } else if (OB_UNLIKELY(1 != parse_tree.num_child_) || OB_ISNULL(parse_tree.children_[0])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("parse tree is wrong", K(ret)); + } else if (T_FUN_SYS_CURRENT_USER == parse_tree.children_[0]->type_) { + tenant_id = session_info_->get_effective_tenant_id(); + user_id = session_info_->get_priv_user_id(); + show_current_user = true; + if (OB_FAIL(params_.schema_checker_->get_user_info(tenant_id, user_id, user_info))) { + if (ret == OB_USER_NOT_EXIST) { + tmp_ret = OB_USER_NOT_EXIST; + ret = OB_SUCCESS; + } else { + LOG_WARN("failed to got user info", K(ret), K(tenant_id), K(user_id)); + } + } else if (OB_ISNULL(user_info)) { + tmp_ret = OB_USER_NOT_EXIST; + ret = OB_SUCCESS; + } else { + user_name = user_info->get_user_name_str(); + host_name = user_info->get_host_name_str(); + } + } else if (OB_UNLIKELY(T_USER_WITH_HOST_NAME != parse_tree.children_[0]->type_) || + OB_UNLIKELY(2 != parse_tree.children_[0]->num_child_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("parse tree is wrong", K(ret)); + } else { + ParseNode *user_name_node = parse_tree.children_[0]->children_[0]; + ParseNode *host_name_node = parse_tree.children_[0]->children_[1]; + tenant_id = session_info_->get_effective_tenant_id(); + if (OB_ISNULL(user_name_node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("user_name is NULL", K(ret), K(user_name)); + } else { + user_name = ObString(user_name_node->str_len_, user_name_node->str_value_); + if (NULL != host_name_node) { + host_name = ObString(host_name_node->str_len_, host_name_node->str_value_); + } else { + host_name = ObString(OB_DEFAULT_HOST_NAME); + } + if (OB_FAIL(params_.schema_checker_->get_user_info(tenant_id, user_name, host_name, user_info))) { + if (ret == OB_USER_NOT_EXIST) { + tmp_ret = OB_USER_NOT_EXIST; + ret = OB_SUCCESS; + } else { + LOG_WARN("failed to got user info", K(ret), K(tenant_id), K(user_name), K(host_name)); + } + } else if (OB_ISNULL(user_info)) { + tmp_ret = OB_USER_NOT_EXIST; + ret = OB_SUCCESS; + } else { + user_id = user_info->get_user_id(); + if (user_id == session_info_->get_user_id()) { + show_current_user = true; + } + } + } + } + // check privileges + if (OB_SUCC(ret)) { + int ret_code = OB_SUCCESS; + if (OB_FAIL(check_show_create_user_privilege(show_current_user, + stmt_need_privs, + session_priv, + ret_code, + has_select_privilege))) { + LOG_WARN("failed to check show create user privileges", K(ret)); + } else if (!has_select_privilege && !show_current_user) { + ret = ret_code; + if (OB_ERR_NO_DB_PRIVILEGE == ret) { + LOG_USER_ERROR(OB_ERR_NO_DB_PRIVILEGE, + session_info_->get_user_name().length(), session_info_->get_user_name().ptr(), + session_info_->get_host_name().length(), session_info_->get_host_name().ptr(), + (int)strlen(OB_SYS_DATABASE_NAME), OB_SYS_DATABASE_NAME); + } else if (OB_ERR_NO_TABLE_PRIVILEGE == ret) { + LOG_USER_ERROR(OB_ERR_NO_TABLE_PRIVILEGE, (int)strlen("SELECT"), "SELECT", + session_info_->get_user_name().length(), session_info_->get_user_name().ptr(), + session_info_->get_host_name().length(), session_info_->get_host_name().ptr(), + (int)strlen("OCEANBASE.__ALL_USER"), "OCEANBASE.__ALL_USER"); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("got unexpected ret code", K(ret), K(ret_code)); + } + } else if (has_select_privilege && OB_USER_NOT_EXIST == tmp_ret) { + ObSqlString msg; + if (show_current_user && + OB_FAIL(msg.append_fmt("CURRENT_USER()"))) { + LOG_WARN("Build msg fail", K(ret)); + } else if (!show_current_user && + OB_FAIL(msg.append_fmt("`%.*s`@`%.*s`", user_name.length(), user_name.ptr(), + host_name.length(), host_name.ptr()))) { + LOG_WARN("Build msg fail", K(user_name), K(host_name), K(ret)); + } else { + ret = OB_CANNOT_USER; + LOG_USER_ERROR(OB_CANNOT_USER, (int)strlen("SHOW CREATE USER"), "SHOW CREATE USER", + (int)msg.length(), msg.ptr()); + } + } + } + // fanpin create user sql + if (OB_SUCC(ret)) { + char *user_def_buf = NULL; + int64_t user_def_buf_size = OB_MAX_VARCHAR_LENGTH; + int64_t pos = 0; + ObSchemaPrinter schema_printer(*schema_guard); + if (OB_ISNULL(user_def_buf = static_cast(allocator_->alloc(user_def_buf_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + SERVER_LOG(ERROR, "fail to alloc user_def_buf", K(ret), K(user_def_buf_size)); + } else if (OB_FAIL(schema_printer.print_user_definition(tenant_id, *user_info, user_def_buf, + user_def_buf_size, pos, false, + !has_select_privilege && show_current_user))) { + SERVER_LOG(WARN, "Generate user definition failed"); + } else { + GEN_SQL_STEP_1(ObShowSqlSet::SHOW_CREATE_USER); + GEN_SQL_STEP_2(ObShowSqlSet::SHOW_CREATE_USER, + pos, user_def_buf, + user_name.length(), user_name.ptr(), + host_name.length(), host_name.ptr()); + } + } + return ret; +} + +int ObShowResolver::check_show_create_user_privilege(const bool show_current_user, + ObStmtNeedPrivs &stmt_need_privs, + ObSessionPrivInfo &session_priv, + int &ret_code, + bool &has_select_privilege) +{ + int ret = OB_SUCCESS; + ret_code = OB_SUCCESS; + has_select_privilege = false; + if (OB_ISNULL(session_info_) || OB_ISNULL(params_.schema_checker_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("got unexpected NULL ptr", K(ret)); + } else if (show_current_user) { + // current_user require select privileges on mysql.user or oceanbase.__all_user + // user level + ObNeedPriv need_priv_0; + need_priv_0.priv_set_ = OB_PRIV_SELECT; + need_priv_0.priv_level_ = OB_PRIV_USER_LEVEL; + // db level + ObNeedPriv need_priv_1; + need_priv_1.db_ = OB_SYS_DATABASE_NAME; + need_priv_1.priv_set_ = OB_PRIV_SELECT; + need_priv_1.priv_level_ = OB_PRIV_DB_LEVEL; + ObNeedPriv need_priv_2; + need_priv_2.db_ = OB_MYSQL_SCHEMA_NAME; + need_priv_2.priv_set_ = OB_PRIV_SELECT; + need_priv_2.priv_level_ = OB_PRIV_DB_LEVEL; + // table level + ObNeedPriv need_priv_3; + need_priv_3.priv_set_ = OB_PRIV_SELECT; + need_priv_3.db_ = OB_SYS_DATABASE_NAME; + need_priv_3.table_ = OB_ALL_USER_TNAME; + need_priv_3.priv_level_ = OB_PRIV_TABLE_LEVEL; + ObNeedPriv need_priv_4; + need_priv_4.db_ = OB_MYSQL_SCHEMA_NAME; + need_priv_4.table_ = "user"; + need_priv_4.priv_set_ = OB_PRIV_SELECT; + need_priv_4.priv_level_ = OB_PRIV_TABLE_LEVEL; + if (OB_FAIL(stmt_need_privs.need_privs_.init(5))) { + LOG_WARN("Failed to init stmt need priv", K(ret)); + } else if (OB_FAIL(stmt_need_privs.need_privs_.push_back(need_priv_0))) { + LOG_WARN("Failed to add need priv", K(ret)); + } else if (OB_FAIL(stmt_need_privs.need_privs_.push_back(need_priv_1))) { + LOG_WARN("Failed to add need priv", K(ret)); + } else if (OB_FAIL(stmt_need_privs.need_privs_.push_back(need_priv_2))) { + LOG_WARN("Failed to add need priv", K(ret)); + } else if (OB_FAIL(stmt_need_privs.need_privs_.push_back(need_priv_3))) { + LOG_WARN("Failed to add need priv", K(ret)); + } else if (OB_FAIL(stmt_need_privs.need_privs_.push_back(need_priv_4))) { + LOG_WARN("Failed to add need priv", K(ret)); + } + } else { + // not current_user require select privileges on mysql or oceanbase + // user level + ObNeedPriv need_priv_0; + need_priv_0.priv_set_ = OB_PRIV_SELECT; + need_priv_0.priv_level_ = OB_PRIV_USER_LEVEL; + // db level + ObNeedPriv need_priv_1; + need_priv_1.db_ = OB_SYS_DATABASE_NAME; + need_priv_1.priv_set_ = OB_PRIV_SELECT; + need_priv_1.priv_level_ = OB_PRIV_DB_LEVEL; + ObNeedPriv need_priv_2; + need_priv_2.db_ = OB_MYSQL_SCHEMA_NAME; + need_priv_2.priv_set_ = OB_PRIV_SELECT; + need_priv_2.priv_level_ = OB_PRIV_DB_LEVEL; + if (OB_FAIL(stmt_need_privs.need_privs_.init(3))) { + LOG_WARN("Failed to init stmt need priv", K(ret)); + } else if (OB_FAIL(stmt_need_privs.need_privs_.push_back(need_priv_0))) { + LOG_WARN("Failed to add need priv", K(ret)); + } else if (OB_FAIL(stmt_need_privs.need_privs_.push_back(need_priv_1))) { + LOG_WARN("Failed to add need priv", K(ret)); + } else if (OB_FAIL(stmt_need_privs.need_privs_.push_back(need_priv_2))) { + LOG_WARN("Failed to add need priv", K(ret)); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(schema_checker_->check_priv_or(session_priv, stmt_need_privs))) { + if (OB_ERR_NO_DB_PRIVILEGE == ret || OB_ERR_NO_TABLE_PRIVILEGE == ret) { + ret_code = ret; + ret = OB_SUCCESS; + has_select_privilege = false; + } else { + LOG_WARN("Failed to check acc", K(ret)); + } + } else { + has_select_privilege = true; + } + } + return ret; +} + int ObShowResolver::get_database_info(const ParseNode *database_node, const ObString &database_name, uint64_t real_tenant_id, @@ -1769,7 +2136,8 @@ int ObShowResolver::process_select_type(ObSelectStmt *select_stmt, //|| stmt_type == stmt::T_SHOW_PRIVILEGES || stmt_type == stmt::T_SHOW_PROCESSLIST //|| stmt_type == stmt::T_SHOW_PROFILES - || stmt_type == stmt::T_SHOW_WARNINGS) { + || stmt_type == stmt::T_SHOW_WARNINGS + || stmt_type == stmt::T_SHOW_CREATE_USER) { select_stmt->set_select_type(NOT_AFFECT_FOUND_ROWS); } else { select_stmt->set_select_type(AFFECT_FOUND_ROWS); @@ -2173,9 +2541,11 @@ int ObShowResolver::resolve_show_from_trigger(const ParseNode *from_tg_node, ObString &show_database_name, uint64_t &show_database_id, ObString &show_tg_name, - uint64_t &show_tg_id) + uint64_t &show_tg_id, + ObString &show_table_name) { int ret = OB_SUCCESS; + const ObTableSchema *table = NULL; if (OB_UNLIKELY(NULL == schema_checker_)) { ret = OB_ERR_SCHEMA_UNSET; LOG_WARN("some data member is not init", K(ret), K(schema_checker_)); @@ -2241,6 +2611,9 @@ int ObShowResolver::resolve_show_from_trigger(const ParseNode *from_tg_node, real_tenant_id, show_database_name, show_tg_name); OV (OB_NOT_NULL(tg_info), OB_ERR_TRIGGER_NOT_EXIST); OX (show_tg_id = tg_info->get_trigger_id()); + OZ (schema_checker_->get_table_schema(real_tenant_id, tg_info->get_base_object_id(), table)); + CK (OB_NOT_NULL(table)); + OX (show_table_name = table->get_table_name()); } return ret; } @@ -2748,6 +3121,94 @@ int ObShowResolver::resolve_column_ref_expr(const ObQualifiedName &q_name, ObRaw return ret; } +int ObShowResolver::recursive_resolve_table_info(const ParseNode *table_list_node, + ObIAllocator &alloc, + ObIArray &table_infos, + TableInfoSet &tables_set) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(table_list_node)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("null point", K(ret), KP(table_list_node)); + } else if (T_LINK_NODE == table_list_node->type_) { + if (OB_FAIL(SMART_CALL(recursive_resolve_table_info(table_list_node->children_[0], alloc, table_infos, tables_set)))) { + LOG_WARN("recursive resolve table list node failed", K(ret)); + } else if (OB_FAIL(SMART_CALL(recursive_resolve_table_info(table_list_node->children_[1], alloc, table_infos, tables_set)))) { + LOG_WARN("recursive resolve table list node failed", K(ret)); + } + } else if (OB_FAIL(resolve_table_info(table_list_node, alloc, table_infos, tables_set))) { + LOG_WARN("resolve table info failed", K(ret)); + } + return ret; +} + +int ObShowResolver::resolve_table_info(const ParseNode *table_node, + ObIAllocator &alloc, + ObIArray &table_infos, + TableInfoSet &tables_set) +{ + int ret = OB_SUCCESS; + ObString table_name; + ObString database_name; + const ObTableSchema *table_schema = NULL; + int64_t tenant_id = session_info_->get_effective_tenant_id(); + uint64_t database_id = OB_INVALID_ID; + ObCheckTableInfo curr_info; + if (OB_ISNULL(table_node) || OB_ISNULL(schema_checker_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("null point", K(table_node), K(schema_checker_), K(ret)); + } else if (OB_FAIL(resolve_table_relation_node(table_node, table_name, database_name))) { + LOG_WARN("failed to resolve table relation node", K(ret)); + } else if (OB_FAIL(ob_write_string(alloc, database_name, curr_info.db_name_, true))) { + LOG_WARN("failed to copy db name", K(ret)); + } else if (OB_FAIL(ob_write_string(alloc, table_name, curr_info.table_name_, true))) { + LOG_WARN("failed to copy table name", K(ret)); + } else { + const ObTableSchema *table_schema = nullptr; + if (OB_FAIL(schema_checker_->get_database_id(tenant_id, database_name, database_id))) { + if (OB_ERR_BAD_DATABASE != ret) { + LOG_WARN("failed to get database id", K(ret)); + } else { + ret = OB_SUCCESS; + curr_info.db_exist_ = false; + } + } else { + curr_info.db_exist_ = true; + } + if (OB_SUCC(ret) + && curr_info.db_exist_ + && OB_FAIL(schema_checker_->get_table_schema(tenant_id, database_name, + table_name, false, table_schema))) { + if (OB_TABLE_NOT_EXIST != ret) { + LOG_WARN("failed to get schema", K(ret)); + } else { + curr_info.table_exist_ = false; + ret = OB_SUCCESS; + } + } else if (nullptr == table_schema) { + curr_info.table_exist_ = false; + } else { + curr_info.table_exist_ = true; + curr_info.is_view_ = table_schema->is_view_table(); + curr_info.valid_ = ObObjectStatus::VALID == table_schema->get_object_status(); + } + if (OB_SUCC(ret)) { + if (OB_FAIL(tables_set.set_refactored(curr_info, 0))) { + if (OB_HASH_EXIST != ret) { + LOG_WARN("failed to set info", K(ret)); + } else { + ret = OB_ERR_NONUNIQ_TABLE; + LOG_USER_ERROR(OB_ERR_NONUNIQ_TABLE, curr_info.table_name_.length(), curr_info.table_name_.ptr()); + } + } else { + OZ (table_infos.push_back(curr_info)); + } + } + } + return ret; +} + + int ObShowResolver::ObSqlStrGenerator::init(common::ObIAllocator *allocator) { int ret = OB_SUCCESS; @@ -3264,5 +3725,10 @@ DEFINE_SHOW_CLAUSE_SET(SHOW_SEQUENCES_LIKE, "SELECT sequence_name FROM %s.%s WHERE database_id = %ld ORDER BY sequence_name COLLATE utf8mb4_bin ASC", NULL, "sequence_name"); +DEFINE_SHOW_CLAUSE_SET(SHOW_CREATE_USER, + NULL, + "SELECT \"%.*s\" AS `CREATE USER for %.*s@%.*s` FROM DUAL", + NULL, + NULL); }/* ns sql*/ }/* ns oceanbase */ diff --git a/src/sql/resolver/cmd/ob_show_resolver.h b/src/sql/resolver/cmd/ob_show_resolver.h index bdc1b6a5b..78a0f1dd1 100644 --- a/src/sql/resolver/cmd/ob_show_resolver.h +++ b/src/sql/resolver/cmd/ob_show_resolver.h @@ -13,6 +13,7 @@ #ifndef OCEANBASE_SQL_RESOLVER_CMD_OB_SHOW_RESOLVER_ #define OCEANBASE_SQL_RESOLVER_CMD_OB_SHOW_RESOLVER_ #include "sql/resolver/dml/ob_select_resolver.h" +#include "lib/hash/ob_hashset.h" namespace oceanbase { namespace sql @@ -29,6 +30,30 @@ protected: private: class ObSqlStrGenerator; struct ObShowSqlSet; + struct ObCheckTableInfo + { + ObCheckTableInfo() : db_name_(), table_name_(), db_exist_(false), + table_exist_(false), is_view_(false), + valid_(true) {} + int hash(uint64_t &res) const { res = db_name_.hash(table_name_.hash()); return OB_SUCCESS; } + bool operator==(const ObCheckTableInfo &other) const + { + return db_name_ == other.db_name_ && table_name_ == other.table_name_; + } + ObString db_name_; + ObString table_name_; + bool db_exist_; + bool table_exist_; + bool is_view_; + bool valid_; + TO_STRING_KV(K_(db_name), + K_(table_name), + K_(table_exist), + K_(db_exist), + K_(is_view), + K_(valid)); + }; + typedef common::hash::ObHashSet TableInfoSet; int get_database_info(const ParseNode *databse_node, const common::ObString &database_name, uint64_t real_tenant_id, @@ -73,14 +98,37 @@ private: ObString &show_database_name, uint64_t &show_database_id, ObString &show_tg_name, - uint64_t &show_tg_id); + uint64_t &show_tg_id, + ObString &show_table_name); int parse_and_resolve_select_sql(const common::ObString &select_sql); int resolve_like_or_where_clause(ObShowResolverContext &ctx); int replace_where_clause(ParseNode* expr_node, const ObShowResolverContext &show_resv_ctx); int process_select_type( ObSelectStmt *select_stmt, stmt::StmtType stmt_type, const ParseNode &parse_tree); virtual int resolve_column_ref_expr(const ObQualifiedName &q_name, ObRawExpr *&real_ref_expr); + int resolve_show_check_table(const ParseNode &parse_tree, + ObShowResolverContext &show_resv_ctx, + ObString &select_sql); + int resolve_show_create_user(const ParseNode &parse_tree, + ObShowResolverContext &show_resv_ctx, + ObSessionPrivInfo &session_priv, + ObStmtNeedPrivs &stmt_need_privs, + ObString &select_sql, + ObSqlStrGenerator &sql_gen); + int check_show_create_user_privilege(const bool show_current_user, + ObStmtNeedPrivs &stmt_need_privs, + ObSessionPrivInfo &session_priv, + int &ret_code, + bool &has_select_privilege); private: + int recursive_resolve_table_info(const ParseNode *table_list_node, + ObIAllocator &alloc, + ObIArray &infos, + TableInfoSet &tables_set); + int resolve_table_info(const ParseNode *table_node, + ObIAllocator &alloc, + ObIArray &infos, + TableInfoSet &tables_set); DISALLOW_COPY_AND_ASSIGN(ObShowResolver); };// ObShowresolver @@ -161,6 +209,7 @@ struct ObShowResolver::ObShowSqlSet DECLARE_SHOW_CLAUSE_SET(SHOW_SEQUENCES_LIKE); DECLARE_SHOW_CLAUSE_SET(SHOW_ENGINE); DECLARE_SHOW_CLAUSE_SET(SHOW_OPEN_TABLES); + DECLARE_SHOW_CLAUSE_SET(SHOW_CREATE_USER); };// ObShowSqlSet class ObShowResolver::ObSqlStrGenerator diff --git a/src/sql/resolver/cmd/ob_variable_set_resolver.cpp b/src/sql/resolver/cmd/ob_variable_set_resolver.cpp index 8f892dade..bbc3c2920 100644 --- a/src/sql/resolver/cmd/ob_variable_set_resolver.cpp +++ b/src/sql/resolver/cmd/ob_variable_set_resolver.cpp @@ -194,10 +194,6 @@ int ObVariableSetResolver::resolve(const ParseNode &parse_tree) LOG_USER_ERROR(OB_NOT_SUPPORTED, "Not support oracle mode"); } #endif - if (OB_SUCC(ret) && - OB_FAIL(ObResolverUtils::resolve_const_expr(params_, value_node, var_node.value_expr_, NULL))) { - LOG_WARN("resolve variable value failed", K(ret)); - } } if (OB_SUCC(ret)) { if (0 == var_node.variable_name_.case_compare("_enable_mysql_pl_priv_check")) { @@ -210,6 +206,11 @@ int ObVariableSetResolver::resolve(const ParseNode &parse_tree) } } } + if (OB_SUCC(ret)) { + if (OB_FAIL(resolve_value_expr(value_node, var_node.value_expr_))) { + LOG_WARN("failed to resolve value expr", K(ret)); + } + } } else { if (lib::is_mysql_mode() && check_var_name_length) { if (OB_FAIL(ObResolverUtils::check_user_variable_length(var_node.variable_name_.ptr(), diff --git a/src/sql/resolver/dcl/ob_grant_resolver.cpp b/src/sql/resolver/dcl/ob_grant_resolver.cpp index 7caaab5f7..3f752612d 100644 --- a/src/sql/resolver/dcl/ob_grant_resolver.cpp +++ b/src/sql/resolver/dcl/ob_grant_resolver.cpp @@ -777,10 +777,6 @@ int ObGrantResolver::resolve_col_names_mysql( || OB_ISNULL(schema_checker->get_schema_guard())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected error", K(ret)); - } else if (priv_type == OB_PRIV_REFERENCES) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("reference is not supported", K(ret)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "references privilege"); } else { ObString column_name; for (int32_t i = 0; OB_SUCCESS == ret && i < column_list->num_child_; ++i) { @@ -1427,6 +1423,14 @@ int ObGrantResolver::resolve_mysql(const ParseNode &parse_tree) ret = OB_NOT_SUPPORTED; LOG_WARN("grammar is not support when MIN_DATA_VERSION is below DATA_VERSION_4_2_3_0 or 4_3_2_0", K(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "grant create tablespace/shutdown/reload privilege"); + } else if (!sql::ObSQLUtils::is_data_version_ge_424_or_433(compat_version) + && ((priv_set & OB_PRIV_REFERENCES) != 0 || + (priv_set & OB_PRIV_CREATE_ROLE) != 0 || + (priv_set & OB_PRIV_DROP_ROLE) != 0 || + (priv_set & OB_PRIV_TRIGGER) != 0)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("grammar is not support when MIN_DATA_VERSION is below DATA_VERSION_4_2_4_0 or 4_3_3_0", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "grant references/create role/drop role/trigger"); } if (OB_FAIL(ret)) { } else { diff --git a/src/sql/resolver/dcl/ob_revoke_resolver.cpp b/src/sql/resolver/dcl/ob_revoke_resolver.cpp index 62638d94d..acc9de97d 100644 --- a/src/sql/resolver/dcl/ob_revoke_resolver.cpp +++ b/src/sql/resolver/dcl/ob_revoke_resolver.cpp @@ -473,6 +473,14 @@ int ObRevokeResolver::resolve_mysql(const ParseNode &parse_tree) ret = OB_NOT_SUPPORTED; LOG_WARN("grammar is not support when MIN_DATA_VERSION is below DATA_VERSION_4_2_3_0 or 4_3_2_0", K(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "revoke create tablespace/shutdown/reload privilege"); + } else if (!sql::ObSQLUtils::is_data_version_ge_424_or_433(compat_version) + && ((priv_set & OB_PRIV_REFERENCES) != 0 || + (priv_set & OB_PRIV_CREATE_ROLE) != 0 || + (priv_set & OB_PRIV_DROP_ROLE) != 0 || + (priv_set & OB_PRIV_TRIGGER) != 0)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("grammar is not support when MIN_DATA_VERSION is below DATA_VERSION_4_2_4_0 or 4_3_3_0", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "revoke references/create role/drop role/trigger"); } if (OB_FAIL(ret)) { } else { diff --git a/src/sql/resolver/ddl/ob_alter_table_resolver.cpp b/src/sql/resolver/ddl/ob_alter_table_resolver.cpp index 4fbcbb7a9..fe6744afd 100644 --- a/src/sql/resolver/ddl/ob_alter_table_resolver.cpp +++ b/src/sql/resolver/ddl/ob_alter_table_resolver.cpp @@ -5519,7 +5519,9 @@ int ObAlterTableResolver::resolve_alter_table_column_definition(AlterColumnSchem schema_checker_, NULL == node->children_[1]))) { SQL_RESV_LOG(WARN, "failed to check default value", K(column), K(ret)); - } else if (OB_FAIL(column.set_cur_default_value(dummy_column.get_cur_default_value()))) { + } else if (OB_FAIL(column.set_cur_default_value( + dummy_column.get_cur_default_value(), + dummy_column.is_default_expr_v2_column()))) { LOG_WARN("failed to set default value", K(ret)); } // else if (OB_FAIL(process_default_value(stat, column))) { @@ -5599,6 +5601,7 @@ int ObAlterTableResolver::resolve_alter_column(const ParseNode &node) SQL_RESV_LOG(WARN, "invalid parse tree", K(ret)); } else { ObObjParam default_value; + ObDefaultValueRes resolve_res(default_value); if (T_CONSTR_NULL == default_node->type_) { default_value.set_null(); alter_column_schema.is_drop_default_ = true; @@ -5611,11 +5614,37 @@ int ObAlterTableResolver::resolve_alter_column(const ParseNode &node) } else if (!lib::is_oracle_mode() && ob_is_geometry_tc(alter_column_schema.get_data_type())) { ret = OB_ERR_BLOB_CANT_HAVE_DEFAULT; SQL_RESV_LOG(WARN, "GEOMETRY can't set default value!", K(ret)); - } else if (OB_FAIL(resolve_default_value(default_node, default_value))) { + } else if (OB_FAIL(resolve_default_value(default_node, resolve_res))) { SQL_RESV_LOG(WARN, "failed to resolve default value!", K(ret)); } + if (OB_SUCC(ret) && !resolve_res.is_literal_) { + uint64_t data_version = 0; + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(session_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("session info is NULL", KR(ret)); + } else if (OB_FAIL(GET_MIN_DATA_VERSION(session_info_->get_effective_tenant_id(), data_version))) { + LOG_WARN("fail to get tenant data version", KR(ret), K(session_info_->get_effective_tenant_id()), K(data_version)); + } else if ((MOCK_DATA_VERSION_4_2_4_0 <= data_version && + data_version < DATA_VERSION_4_3_0_0) || + data_version >= DATA_VERSION_4_3_3_0) { + ObString expr_str(default_node->str_len_, default_node->str_value_); + if (OB_FAIL(ObSQLUtils::convert_sql_text_to_schema_for_storing( + *allocator_, session_info_->get_dtc_params(), expr_str))) { + LOG_WARN("fail to copy and convert string charset", K(ret)); + } else { + default_value.set_varchar(expr_str); + default_value.set_collation_type(CS_TYPE_UTF8MB4_BIN); + default_value.set_param_meta(); + } + } else { + ret = OB_ERR_ILLEGAL_TYPE; + SQL_RESV_LOG(WARN, "Illegal type of default value",K(ret)); + } + } + if (OB_SUCCESS == ret && - OB_FAIL(alter_column_schema.set_cur_default_value(default_value))) { + OB_FAIL(alter_column_schema.set_cur_default_value(default_value, !resolve_res.is_literal_))) { SQL_RESV_LOG(WARN, "failed to set current default to alter column schema!", K(ret)); } } diff --git a/src/sql/resolver/ddl/ob_create_table_resolver.cpp b/src/sql/resolver/ddl/ob_create_table_resolver.cpp index 6c3976d8d..c0cf73701 100644 --- a/src/sql/resolver/ddl/ob_create_table_resolver.cpp +++ b/src/sql/resolver/ddl/ob_create_table_resolver.cpp @@ -1858,12 +1858,12 @@ int ObCreateTableResolver::set_nullable_for_cta_column(ObSelectStmt *select_stmt T_WIN_FUN_ROW_NUMBER == win_expr->get_func_type()) { ObObj temp_default; temp_default.set_uint64(0); - column.set_cur_default_value(temp_default); + column.set_cur_default_value(temp_default, false); } else if (T_WIN_FUN_CUME_DIST == win_expr->get_func_type() || T_WIN_FUN_PERCENT_RANK == win_expr->get_func_type()) { ObObj temp_default; temp_default.set_double(0); - column.set_cur_default_value(temp_default); + column.set_cur_default_value(temp_default, false); } else {} } else {} column.set_nullable(!is_not_null); @@ -2046,7 +2046,9 @@ int ObCreateTableResolver::resolve_table_elements_from_select(const ParseNode &p if (NULL != org_column && !org_column->is_generated_column() && !org_column->get_cur_default_value().is_null()) { - column.set_cur_default_value(org_column->get_cur_default_value()); + column.set_cur_default_value( + org_column->get_cur_default_value(), + org_column->is_default_expr_v2_column()); } } } else if (new_table_item == NULL && @@ -2057,7 +2059,7 @@ int ObCreateTableResolver::resolve_table_elements_from_select(const ParseNode &p common::ObObjType result_type = expr->get_result_type().get_obj_meta().get_type(); if (ob_is_numeric_type(result_type) || ob_is_string_tc(result_type) || ob_is_time_tc(result_type)) { common::ObObj zero_obj(0); - if (OB_FAIL(column.set_cur_default_value(zero_obj))) { + if (OB_FAIL(column.set_cur_default_value(zero_obj, false))) { LOG_WARN("set default value failed", K(ret)); } } diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.cpp b/src/sql/resolver/ddl/ob_ddl_resolver.cpp index 9258e2d12..2f9f683fd 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.cpp +++ b/src/sql/resolver/ddl/ob_ddl_resolver.cpp @@ -44,6 +44,7 @@ #include "sql/engine/expr/ob_expr_lob_utils.h" #include "pl/ob_pl_stmt.h" #include "share/table/ob_ttl_util.h" +#include "common/ob_smart_call.h" namespace oceanbase { using namespace common; @@ -559,7 +560,7 @@ int update_datetime_default_value(ObObjParam &default_value, ParseNode &def_val, int ObDDLResolver::resolve_default_value(ParseNode *def_node, // const ObObjType column_data_type, - ObObjParam &default_value) + ObDefaultValueRes &resolve_res) { int ret = OB_SUCCESS; ObString tmp_str; @@ -568,6 +569,8 @@ int ObDDLResolver::resolve_default_value(ParseNode *def_node, int16_t scale = -1; ObIAllocator *name_pool = NULL; ParseNode *def_val = NULL; + ObObjParam &default_value = resolve_res.value_; + resolve_res.is_literal_ = true; if (NULL != def_node) { def_val = def_node; if (def_node->type_ == T_CONSTR_DEFAULT @@ -786,6 +789,19 @@ int ObDDLResolver::resolve_default_value(ParseNode *def_node, ret = update_datetime_default_value(default_value, *def_val, ObTimestampType, ObActionFlag::OP_DEFAULT_NOW_FLAG); break; } + case T_DEFAULT_INT: { + default_value.set_int(def_val->value_); + default_value.set_param_meta(); + break; + } + case T_OP_POS: + case T_OP_NEG: { + if (OB_FAIL(resolve_sign_in_default_value( + name_pool, def_val, resolve_res, T_OP_NEG == def_val->type_))) { + SQL_RESV_LOG(WARN, "resolve_sign_in_default_value failed", K(ret), K(def_val->type_)); + } + break; + } case T_INTERVAL_YM: case T_INTERVAL_DS: case T_NVARCHAR2: @@ -795,14 +811,177 @@ int ObDDLResolver::resolve_default_value(ParseNode *def_node, LOG_USER_ERROR(OB_NOT_SUPPORTED, "specify default value for interval_ym/interval_ds/urowid"); break; } - case T_OP_POS: - case T_OP_NEG: { - if (OB_FAIL(resolve_sign_in_default_value( - name_pool, def_val, default_value, T_OP_NEG == def_val->type_))) { - SQL_RESV_LOG(WARN, "resolve_sign_in_default_value failed", K(ret), K(def_val->type_)); + case T_FUN_SUM: + case T_FUN_MAX: + case T_FUN_MIN: + case T_FUN_AVG: { + resolve_res.is_literal_ = false; + ObObjParam arg; + ObDefaultValueRes arg_res(arg); + if (def_val->num_child_ != 2) { + ret = OB_INVALID_ARGUMENT; + SQL_RESV_LOG(WARN, "Invalid expression", K(ret), K(def_val->type_), K(def_val->num_child_)); + } else if (OB_FAIL(SMART_CALL(resolve_default_value(def_val->children_[1], arg_res)))) { + SQL_RESV_LOG(WARN, "Resolve expression failed", K(ret), K(def_val->type_)); } break; } + case T_OP_ADD: + case T_OP_MINUS: + case T_OP_MUL: + case T_OP_DIV: + case T_OP_MOD: + case T_OP_EQ: + case T_OP_NSEQ: + case T_OP_LE: + case T_OP_LT: + case T_OP_GE: + case T_OP_GT: + case T_OP_NE: + case T_OP_AND: + case T_OP_OR: + case T_OP_IN: + case T_OP_NOT_IN: + case T_OP_BIT_OR: + case T_OP_BIT_AND: + case T_OP_BIT_XOR: + case T_OP_BIT_LEFT_SHIFT: + case T_OP_BIT_RIGHT_SHIFT: + case T_WHEN: + case T_FUN_SYS_INTERVAL: + case T_FUN_SYS_IF: + case T_FUN_SYS_ISNULL: + case T_FUN_SYS_POINT: { + resolve_res.is_literal_ = false; + ObObjParam first; + ObDefaultValueRes first_res(first); + ObObjParam second; + ObDefaultValueRes second_res(second); + if (def_val->num_child_ != 2) { + ret = OB_INVALID_ARGUMENT; + SQL_RESV_LOG(WARN, "Invalid expression", K(ret), K(def_val->type_), K(def_val->num_child_)); + } else if (OB_FAIL(SMART_CALL(resolve_default_value(def_val->children_[0], first_res)))) { + SQL_RESV_LOG(WARN, "Resolve expression failed", K(ret), K(def_val->type_)); + } else if (OB_FAIL(SMART_CALL(resolve_default_value(def_val->children_[1], second_res)))) { + SQL_RESV_LOG(WARN, "Resolve expression failed", K(ret), K(def_val->type_)); + } + break; + } + case T_OP_BTW: + case T_OP_NOT_BTW: + case T_CASE: { + resolve_res.is_literal_ = false; + ObObjParam first; + ObDefaultValueRes first_res(first); + ObObjParam second; + ObDefaultValueRes second_res(second); + ObObjParam third; + ObDefaultValueRes third_res(third); + if (def_val->num_child_ != 3) { + ret = OB_INVALID_ARGUMENT; + SQL_RESV_LOG(WARN, "Invalid expression", K(ret), K(def_val->type_), K(def_val->num_child_)); + } else if (OB_FAIL(SMART_CALL(resolve_default_value(def_val->children_[0], first_res)))) { + SQL_RESV_LOG(WARN, "Resolve expression failed", K(ret), K(def_val->type_)); + } else if (OB_FAIL(SMART_CALL(resolve_default_value(def_val->children_[1], second_res)))) { + SQL_RESV_LOG(WARN, "Resolve expression failed", K(ret), K(def_val->type_)); + } else if (OB_FAIL(SMART_CALL(resolve_default_value(def_val->children_[2], third_res)))) { + SQL_RESV_LOG(WARN, "Resolve expression failed", K(ret), K(def_val->type_)); + } + break; + } + case T_OP_NOT: + case T_OP_BIT_NEG: + case T_FUN_SYS_UTC_TIMESTAMP: + case T_FUN_SYS_CUR_TIME: + case T_FUN_SYS_UTC_TIME: + case T_FUN_SYS_SYSDATE: { + resolve_res.is_literal_ = false; + ObObjParam first; + ObDefaultValueRes first_res(first); + if (def_val->num_child_ != 1) { + ret = OB_INVALID_ARGUMENT; + SQL_RESV_LOG(WARN, "Invalid expression", K(ret), K(def_val->type_), K(def_val->num_child_)); + } else if (OB_FAIL(SMART_CALL(resolve_default_value(def_val->children_[0], first_res)))) { + SQL_RESV_LOG(WARN, "Resolve expression failed", K(ret), K(def_val->type_)); + } + break; + } + case T_FUN_SYS: { + resolve_res.is_literal_ = false; + ObObjParam func_ident; + ObDefaultValueRes func_res(func_ident); + if (def_val->num_child_ < 1) { + ret = OB_INVALID_ARGUMENT; + SQL_RESV_LOG(WARN, "Invalid expression", K(ret), K(def_val->type_), K(def_val->num_child_)); + } else if (OB_FAIL(SMART_CALL(resolve_default_value(def_val->children_[0], func_res)))) { + SQL_RESV_LOG(WARN, "Resolve expression failed", K(ret), K(def_val->type_)); + } else if (def_val->num_child_ == 2) { + ObObjParam argument; + ObDefaultValueRes argument_res(argument); + if (OB_FAIL(SMART_CALL(resolve_default_value(def_val->children_[1], argument_res)))) { + SQL_RESV_LOG(WARN, "Resolve expression failed", K(ret), K(def_val->type_)); + } + } + break; + } + case T_OP_LIKE: { + resolve_res.is_literal_ = false; + int child_num = def_val->num_child_; + ObObjParam value; + ObDefaultValueRes value_res(value); + ObObjParam pattern; + ObDefaultValueRes pattern_res(pattern); + if (def_val->num_child_ < 2) { + ret = OB_INVALID_ARGUMENT; + SQL_RESV_LOG(WARN, "Invalid expression", K(ret), K(def_val->type_), K(def_val->num_child_)); + } else if (OB_FAIL(SMART_CALL(resolve_default_value(def_val->children_[0], value_res)))) { + SQL_RESV_LOG(WARN, "Resolve expression failed", K(ret), K(def_val->type_)); + } else if (OB_FAIL(SMART_CALL(resolve_default_value(def_val->children_[1], pattern_res)))) { + SQL_RESV_LOG(WARN, "Resolve expression failed", K(ret), K(def_val->type_)); + } + if (child_num == 3) { + ObObjParam escape; + ObDefaultValueRes escape_res(escape); + if (OB_FAIL(SMART_CALL(resolve_default_value(def_val->children_[2], escape_res)))) { + SQL_RESV_LOG(WARN, "Resolve expression failed", K(ret), K(def_val->type_)); + } + } + break; + } + case T_EXPR_LIST: + case T_LINK_NODE: + case T_WHEN_LIST: { + resolve_res.is_literal_ = false; + int num_child = def_val->num_child_; + for (int i = 0; OB_SUCC(ret) && ichildren_[i], expr_res)))) { + SQL_RESV_LOG(WARN, "Resolve expression list failed", K(ret)); + } + } + break; + } + case T_FUN_SYS_UTC_DATE: + case T_FUN_SYS_CUR_DATE: + case T_IDENT: { + resolve_res.is_literal_ = false; + break; + } + case T_OP_CNN: { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "Connect operator in default value expression"); + break; + } + case T_COLUMN_REF: { + resolve_res.is_literal_ = false; + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "Column reference in default value expression"); + break; + } + case T_SFU_DOUBLE: { + break; + } default: ret = OB_ERR_ILLEGAL_TYPE; SQL_RESV_LOG(WARN, "Illegal type of default value",K(ret), K(def_val->type_)); @@ -820,12 +999,15 @@ int ObDDLResolver::resolve_default_value(ParseNode *def_node, } int ObDDLResolver::resolve_sign_in_default_value( - ObIAllocator *name_pool, ParseNode *def_val, ObObjParam &default_value, const bool is_neg) + ObIAllocator *name_pool, ParseNode *def_val, ObDefaultValueRes &resolve_res, const bool is_neg) { int ret = OB_SUCCESS; if (is_neg) { ObObjParam old_obj; - ret = resolve_default_value(def_val->children_[0], old_obj); + ObDefaultValueRes old_res(old_obj); + ObObjParam &default_value = resolve_res.value_; + ret = resolve_default_value(def_val->children_[0], old_res); + resolve_res.is_literal_ = old_res.is_literal_; if (OB_FAIL(ret)) { SQL_RESV_LOG(WARN, "Resolve default const value failed", K(ret)); } else if (ObIntType == old_obj.get_type()) { @@ -866,7 +1048,7 @@ int ObDDLResolver::resolve_sign_in_default_value( SQL_RESV_LOG(WARN, "Invalid flag '-' in default value",K(ret)); } } else { // is_pos - if (OB_FAIL(resolve_default_value(def_val->children_[0], default_value))) { + if (OB_FAIL(resolve_default_value(def_val->children_[0], resolve_res))) { SQL_RESV_LOG(WARN, "resolve_default_value failed",K(ret), K(def_val->children_[0]->type_)); } } @@ -3362,7 +3544,10 @@ int ObDDLResolver::resolve_column_definition(ObColumnSchemaV2 &column, } else { default_value.set_varchar(expr_str); default_value.set_collation_type(ObCharset::get_system_collation()); - if (OB_FAIL(column.set_cur_default_value(default_value))) { + default_value.set_collation_level(CS_LEVEL_COERCIBLE); + if (OB_FAIL(column.set_cur_default_value( + default_value, + column.is_default_expr_v2_column()))) { LOG_WARN("set current default value failed", K(ret)); } else if ((node->children_[4] != NULL && node->children_[4]->type_ == T_STORED_COLUMN) || (node->children_[4] == NULL && is_external_table)) { @@ -3392,7 +3577,8 @@ int ObDDLResolver::resolve_column_definition(ObColumnSchemaV2 &column, ObObj default_value; default_value.set_varchar(mock_gen_column_str); default_value.set_collation_type(ObCharset::get_system_collation()); - if (OB_FAIL(column.set_cur_default_value(default_value))) { + default_value.set_collation_level(CS_LEVEL_COERCIBLE); + if (OB_FAIL(column.set_cur_default_value(default_value, column.is_default_expr_v2_column()))) { LOG_WARN("set current default value failed", K(ret)); } else { column.add_column_flag(STORED_GENERATED_COLUMN_FLAG); @@ -3606,24 +3792,49 @@ int ObDDLResolver::resolve_normal_column_attribute_constr_default(ObColumnSchema } else { default_value.set_varchar(expr_str); default_value.set_collation_type(CS_TYPE_UTF8MB4_BIN); + default_value.set_collation_level(CS_LEVEL_COERCIBLE); default_value.set_param_meta(); if (T_CONSTR_ORIG_DEFAULT == attr_node->type_) { ret = OB_NOT_SUPPORTED; //TODO:@yanhua do it next LOG_WARN("not support set orig default now", K(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "specify orig default value for column"); - } else if (OB_FAIL(column.set_cur_default_value(default_value))) { + // In oracle mode, default value always be expression + } else if (OB_FAIL(column.set_cur_default_value(default_value, true))) { LOG_WARN("set current default value failed", K(ret)); - } else { - column.add_column_flag(DEFAULT_EXPR_V2_COLUMN_FLAG); } } } else { + ObDefaultValueRes resolve_res(default_value); if (1 != attr_node->num_child_ || NULL == attr_node->children_[0]) { ret = OB_ERR_UNEXPECTED; SQL_RESV_LOG(WARN, "resolve default value failed", K(ret)); - } else if (OB_FAIL(resolve_default_value(attr_node, default_value))) { + } else if (OB_FAIL(resolve_default_value(attr_node, resolve_res))) { SQL_RESV_LOG(WARN, "resolve default value failed", K(ret)); + } else if (!resolve_res.is_literal_) { + // default value expression is not literal + uint64_t data_version = 0; + if (OB_ISNULL(session_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("session info is NULL", KR(ret)); + } else if (OB_FAIL(GET_MIN_DATA_VERSION(session_info_->get_effective_tenant_id(), data_version))) { + LOG_WARN("fail to get tenant data version", KR(ret), K(session_info_->get_effective_tenant_id()), K(data_version)); + } else if ((MOCK_DATA_VERSION_4_2_4_0 <= data_version && + data_version < DATA_VERSION_4_3_0_0) || + data_version >= DATA_VERSION_4_3_3_0) { + ObString expr_str(attr_node->str_len_, attr_node->str_value_); + if (OB_FAIL(ObSQLUtils::convert_sql_text_to_schema_for_storing( + *allocator_, session_info_->get_dtc_params(), expr_str))) { + LOG_WARN("fail to copy and convert string charset", K(ret)); + } else { + default_value.set_varchar(expr_str); + default_value.set_collation_type(CS_TYPE_UTF8MB4_BIN); + default_value.set_param_meta(); + } + } else { + ret = OB_ERR_ILLEGAL_TYPE; + SQL_RESV_LOG(WARN, "Illegal type of default value",K(ret)); + } } else if (IS_DEFAULT_NOW_OBJ(default_value)) { if ((ObDateTimeTC != column.get_data_type_class() && ObOTimestampTC != column.get_data_type_class()) || default_value.get_scale() != column.get_accuracy().get_scale()) { @@ -3656,7 +3867,7 @@ int ObDDLResolver::resolve_normal_column_attribute_constr_default(ObColumnSchema SQL_RESV_LOG(WARN, "cannot set current default value twice", K(ret)); } else { is_set_cur_default = true; - column.set_cur_default_value(default_value); + column.set_cur_default_value(default_value, !resolve_res.is_literal_); } } else { // T_CONSTR_ORIG_DEFAULT == column.get_data_type_class() @@ -4901,7 +5112,8 @@ int ObDDLResolver::cast_default_value(ObObj &default_value, } } else if (IS_DEFAULT_NOW_OBJ(default_value)) { if (ObDateTimeTC == column_schema.get_data_type_class()) { - if (OB_FAIL(column_schema.set_cur_default_value(default_value))) { + if (OB_FAIL(column_schema.set_cur_default_value( + default_value, column_schema.is_default_expr_v2_column()))) { SQL_RESV_LOG(WARN, "set current default value failed", K(ret)); } } else { @@ -5408,7 +5620,7 @@ int ObDDLResolver::check_and_fill_column_charset_info(ObColumnSchemaV2 &column, const ObCollationType table_collation_type) { int ret = OB_SUCCESS; ObCharsetType charset_type = CHARSET_INVALID; - ObCollationType collation_type = CS_TYPE_INVALID;; + ObCollationType collation_type = CS_TYPE_INVALID; if (CHARSET_INVALID == table_charset_type || CS_TYPE_INVALID == table_collation_type) { ret = OB_ERR_UNEXPECTED; SQL_RESV_LOG(WARN, "invalid column charset info!", K(ret)); @@ -6077,7 +6289,10 @@ int ObDDLResolver::print_expr_to_default_value(ObRawExpr &expr, } else if (FALSE_IT(expr_def.assign_ptr(expr_str_buf, static_cast(pos)))) { } else if (FALSE_IT(default_value.set_varchar(expr_def))) { } else if (FALSE_IT(default_value.set_collation_type(ObCharset::get_system_collation()))) { - } else if (OB_FAIL(column.set_cur_default_value(default_value))) { + } else if (FALSE_IT(default_value.set_collation_level(CS_LEVEL_COERCIBLE))) { + } else if (OB_FAIL(column.set_cur_default_value( + default_value, + column.is_default_expr_v2_column()))) { LOG_WARN("set orig default value failed", K(ret)); } else { LOG_DEBUG("succ to print_expr_to_default_value", K(expr), K(column), K(default_value), K(expr_def), K(ret)); @@ -6226,6 +6441,7 @@ int ObDDLResolver::resolve_generated_column_expr(ObString &expr_str, return ret; } +// Verifies the default value and performs type conversion on it. // this function is called in ddl service, which can not access user session_info, so need to // construct an empty session int ObDDLResolver::check_default_value(ObObj &default_value, @@ -6381,8 +6597,19 @@ int ObDDLResolver::check_default_value(ObObj &default_value, } else if (lib::is_oracle_mode() && column.get_meta_type().is_blob() && ob_is_numeric_type(tmp_default_value.get_type())) { ret = OB_ERR_INVALID_TYPE_FOR_OP; LOG_WARN("inconsistent datatypes", "expected", data_type, "got", tmp_default_value.get_type(), K(ret)); - } else if(OB_FAIL(ObObjCaster::to_type(data_type, cast_ctx, tmp_default_value, tmp_dest_obj, tmp_res_obj))) { - LOG_WARN("cast obj failed, ", "src type", tmp_default_value.get_type(), "dest type", data_type, K(tmp_default_value), K(ret)); + } else if (ob_is_enumset_tc(column.get_data_type())) { + if (OB_FAIL(cast_enum_or_set_default_value(column, cast_ctx, tmp_default_value))) { + LOG_WARN("fail to cast enum or set default value", K(input_default_value), K(column), K(ret)); + } else { + tmp_res_obj = &tmp_default_value; + } + } else { + if (OB_FAIL(ObObjCaster::to_type(data_type, cast_ctx, tmp_default_value, tmp_dest_obj, tmp_res_obj))) { + LOG_WARN("cast obj failed, ", "src type", tmp_default_value.get_type(), "dest type", data_type, K(tmp_default_value), K(ret)); + } + } + + if (OB_FAIL(ret)) { } else if (OB_ISNULL(tmp_res_obj)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("cast obj failed, ", "src type", tmp_default_value.get_type(), "dest type", data_type, K(tmp_default_value), K(ret)); @@ -6395,7 +6622,8 @@ int ObDDLResolver::check_default_value(ObObj &default_value, //FIXME::when observer differentiate '' and null, we can delete this code @yanhua tmp_dest_obj_null.set_varchar(input_default_value.get_string()); tmp_dest_obj_null.set_collation_type(ObCharset::get_system_collation()); - if (OB_FAIL(column.set_cur_default_value(tmp_dest_obj_null))) { + if (OB_FAIL(column.set_cur_default_value( + tmp_dest_obj_null, column.is_default_expr_v2_column()))) { LOG_WARN("set orig default value failed", K(ret)); } } else if (OB_FAIL(print_expr_to_default_value(*expr, column, schema_checker, tz_info_wrap.get_time_zone_info()))) { @@ -6529,10 +6757,18 @@ int ObDDLResolver::calc_default_value(share::schema::ObColumnSchemaV2 &column, ObObj dest_obj; const ObDataTypeCastParams dtc_params(tz_info_wrap.get_time_zone_info(), nls_formats, CS_TYPE_INVALID, CS_TYPE_INVALID, CS_TYPE_UTF8MB4_GENERAL_CI); ObCastCtx cast_ctx(&allocator, &dtc_params, CM_NONE, collation_type); - ObAccuracy res_acc = column.get_accuracy(); + if (ob_is_enumset_tc(data_type)) { + if (OB_FAIL(cast_enum_or_set_default_value(column, cast_ctx, default_value))) { + LOG_WARN("fail to cast enum or set default value", K(default_value), K(column), K(ret)); + } + } else { + ObAccuracy res_acc = column.get_accuracy(); cast_ctx.res_accuracy_ = &res_acc; if (OB_FAIL(ObObjCaster::to_type(data_type, cast_ctx, default_value, dest_obj))) { - LOG_WARN("cast obj failed, ", "src type", default_value.get_type(), "dest type", data_type, K(default_value), K(ret)); + LOG_WARN("cast obj failed, ", "src type", default_value.get_type(), "dest type", data_type, K(default_value), K(ret)); + } + } + if (OB_FAIL(ret)) { } else { // remove lob header for lob if (dest_obj.has_lob_header()) { @@ -6821,7 +7057,8 @@ int ObDDLResolver::get_udt_column_default_values(const ObObj &default_value, } else if (0 == input_default_value.get_string().compare("''")) { tmp_dest_obj_null.set_varchar(input_default_value.get_string()); tmp_dest_obj_null.set_collation_type(ObCharset::get_system_collation()); - if (OB_FAIL(column.set_cur_default_value(tmp_dest_obj_null))) { + if (OB_FAIL(column.set_cur_default_value( + tmp_dest_obj_null, column.is_default_expr_v2_column()))) { LOG_WARN("set orig default value failed", K(ret)); } } else if (OB_FAIL(print_expr_to_default_value(*expr, column, schema_checker, tz_info_wrap.get_time_zone_info()))) { diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.h b/src/sql/resolver/ddl/ob_ddl_resolver.h index 13861f8db..83232c508 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.h +++ b/src/sql/resolver/ddl/ob_ddl_resolver.h @@ -99,6 +99,13 @@ struct ObColumnResolveStat bool is_set_orig_default_value_; }; +struct ObDefaultValueRes +{ + ObDefaultValueRes(common::ObObjParam &value): is_literal_(true), value_(value) {} + bool is_literal_; + common::ObObjParam &value_; +}; + class ObDDLResolver : public ObStmtResolver { public: @@ -219,9 +226,9 @@ public: const share::schema::ObColumnSchemaV2 &column_schema); int resolve_default_value( ParseNode *def_node, - common::ObObjParam &default_value); + ObDefaultValueRes &resolve_res); int resolve_sign_in_default_value( - ObIAllocator *name_pool, ParseNode *def_val, ObObjParam &default_value, const bool is_neg); + ObIAllocator *name_pool, ParseNode *def_val, ObDefaultValueRes &resolve_res, const bool is_neg); static int check_and_fill_column_charset_info( share::schema::ObColumnSchemaV2 &column, const common::ObCharsetType table_charset_type, diff --git a/src/sql/resolver/ddl/ob_trigger_resolver.cpp b/src/sql/resolver/ddl/ob_trigger_resolver.cpp index a02e13041..e9a2da31f 100644 --- a/src/sql/resolver/ddl/ob_trigger_resolver.cpp +++ b/src/sql/resolver/ddl/ob_trigger_resolver.cpp @@ -46,6 +46,9 @@ int ObTriggerResolver::resolve(const ParseNode &parse_tree) ObDropTriggerStmt *stmt = create_stmt(); OV (OB_NOT_NULL(stmt), OB_ALLOCATE_MEMORY_FAILED); OZ (resolve_drop_trigger_stmt(parse_tree, stmt->get_trigger_arg())); + if (lib::is_mysql_mode()) { + OZ (get_drop_trigger_stmt_table_name(stmt)); + } break; } case T_TG_ALTER: { @@ -61,6 +64,75 @@ int ObTriggerResolver::resolve(const ParseNode &parse_tree) return ret; } +int ObTriggerResolver::get_drop_trigger_stmt_table_name(ObDropTriggerStmt *stmt) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("drop trigger stmt is NULL", K(ret)); + } else { + const obrpc::ObDropTriggerArg &arg = stmt->get_trigger_arg(); + uint64_t tenant_id = arg.tenant_id_; + const ObString &trigger_database = arg.trigger_database_; + const ObString &trigger_name = arg.trigger_name_; + ObSchemaGetterGuard *schema_guard = NULL; + const ObDatabaseSchema *db_schema = NULL; + uint64_t trigger_database_id = OB_INVALID_ID; + const ObTriggerInfo *trigger_info = NULL; + const ObTableSchema *table = NULL; + + CK (OB_NOT_NULL(schema_checker_)); + CK (OB_NOT_NULL(schema_checker_->get_schema_guard())); + OX (schema_guard = schema_checker_->get_schema_guard()); + if (OB_SUCC(ret)) { + if(OB_FAIL(schema_guard->get_database_schema(tenant_id, trigger_database, db_schema))) { + LOG_WARN("get database schema failed", K(ret)); + } else if (NULL == db_schema) { + ret = OB_ERR_BAD_DATABASE; + LOG_USER_ERROR(OB_ERR_BAD_DATABASE, trigger_database.length(), trigger_database.ptr()); + } else if (db_schema->is_or_in_recyclebin()) { + ret = OB_ERR_OPERATION_ON_RECYCLE_OBJECT; + LOG_WARN("Can't not operate db in recyclebin", + K(tenant_id), K(trigger_database), K(trigger_database_id), K(*db_schema), K(ret)); + } else if (OB_INVALID_ID == (trigger_database_id = db_schema->get_database_id())) { + ret = OB_ERR_BAD_DATABASE; + LOG_WARN("database id is invalid", + K(tenant_id), K(trigger_database), K(trigger_database_id), K(*db_schema), K(ret)); + } else if (OB_FAIL(schema_guard->get_trigger_info(tenant_id, trigger_database_id, + trigger_name, trigger_info))) { + LOG_WARN("get trigger info failed", K(ret), K(trigger_database), K(trigger_name)); + } else if (OB_ISNULL(trigger_info)) { + ret = OB_ERR_TRIGGER_NOT_EXIST; + } else if (trigger_info->is_in_recyclebin()) { + ret = OB_ERR_OPERATION_ON_RECYCLE_OBJECT; + LOG_WARN("trigger is in recyclebin", K(ret), + K(trigger_info->get_trigger_id()), K(trigger_info->get_trigger_name())); + } else if (OB_FAIL(schema_guard->get_table_schema(tenant_id, + trigger_info->get_base_object_id(), + table))) { + LOG_WARN("Failed to get table schema", K(tenant_id), + K(trigger_info->get_base_object_id()), K(ret)); + } else if (OB_ISNULL(table)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Table schema should not be NULL", K(ret)); + } else { + stmt->trigger_table_name_ = table->get_table_name_str(); + } + if (OB_ERR_TRIGGER_NOT_EXIST == ret || OB_ERR_BAD_DATABASE == ret) { + ret = OB_ERR_TRIGGER_NOT_EXIST; + stmt->is_exist = false; + if (arg.if_exist_) { + ret = OB_SUCCESS; + } else { + LOG_MYSQL_USER_ERROR(OB_ERR_TRIGGER_NOT_EXIST); + } + LOG_WARN("trigger not exist", K(arg.trigger_database_), K(arg.trigger_name_), K(ret)); + } + } + } + return ret; +} + int ObTriggerResolver::resolve_sp_definer(const ParseNode *parse_node, ObCreateTriggerArg &trigger_arg) { diff --git a/src/sql/resolver/ddl/ob_trigger_resolver.h b/src/sql/resolver/ddl/ob_trigger_resolver.h index 17854e155..87e4bf377 100644 --- a/src/sql/resolver/ddl/ob_trigger_resolver.h +++ b/src/sql/resolver/ddl/ob_trigger_resolver.h @@ -14,7 +14,7 @@ #define OCEANBASE_SQL_RESOLVER_DDL_OB_TRIGGER_RESOLVER_ #include "sql/resolver/ob_stmt_resolver.h" - +#include "sql/resolver/ddl/ob_trigger_stmt.h" namespace oceanbase { namespace obrpc @@ -95,6 +95,7 @@ private: int resolve_base_object(obrpc::ObCreateTriggerArg &trigger_arg, bool search_public_schema); int resolve_order_clause(const ParseNode *parse_node, obrpc::ObCreateTriggerArg &trigger_arg); + int get_drop_trigger_stmt_table_name(ObDropTriggerStmt *stmt); #ifdef OB_BUILD_ORACLE_PL int resolve_rename_trigger(const ParseNode &rename_clause, ObSchemaGetterGuard &schema_guard, diff --git a/src/sql/resolver/ddl/ob_trigger_stmt.h b/src/sql/resolver/ddl/ob_trigger_stmt.h index 8c4f7b2f4..a6367650a 100644 --- a/src/sql/resolver/ddl/ob_trigger_stmt.h +++ b/src/sql/resolver/ddl/ob_trigger_stmt.h @@ -36,6 +36,7 @@ public: {} virtual obrpc::ObDDLArg &get_ddl_arg() { return trigger_arg_; } obrpc::ObCreateTriggerArg &get_trigger_arg() { return trigger_arg_; } + const obrpc::ObCreateTriggerArg &get_trigger_arg() const { return trigger_arg_; } private: obrpc::ObCreateTriggerArg trigger_arg_; DISALLOW_COPY_AND_ASSIGN(ObCreateTriggerStmt); @@ -46,16 +47,23 @@ class ObDropTriggerStmt : public ObDDLStmt public: explicit ObDropTriggerStmt(common::ObIAllocator *name_pool) : ObDDLStmt(name_pool, stmt::T_DROP_TRIGGER), + trigger_table_name_(), + is_exist(true), trigger_arg_() {} explicit ObDropTriggerStmt() : ObDDLStmt(stmt::T_DROP_TRIGGER), + trigger_table_name_(), + is_exist(true), trigger_arg_() {} virtual ~ObDropTriggerStmt() {} virtual obrpc::ObDDLArg &get_ddl_arg() { return trigger_arg_; } obrpc::ObDropTriggerArg &get_trigger_arg() { return trigger_arg_; } + const obrpc::ObDropTriggerArg &get_trigger_arg() const { return trigger_arg_; } + common::ObString trigger_table_name_; + bool is_exist; private: obrpc::ObDropTriggerArg trigger_arg_; DISALLOW_COPY_AND_ASSIGN(ObDropTriggerStmt); diff --git a/src/sql/resolver/dml/ob_dml_resolver.cpp b/src/sql/resolver/dml/ob_dml_resolver.cpp index dc6927b9e..537f6c646 100755 --- a/src/sql/resolver/dml/ob_dml_resolver.cpp +++ b/src/sql/resolver/dml/ob_dml_resolver.cpp @@ -3767,7 +3767,7 @@ int ObDMLResolver::resolve_basic_table_without_cte(const ParseNode &parse_tree, !is_oracle_mapping_real_virtual_table(table_item->ref_id_)) { ret = OB_NOT_SUPPORTED; LOG_USER_ERROR(OB_NOT_SUPPORTED, "sampling virtual table"); - } else if (OB_FAIL(resolve_sample_clause(sample_node, table_item->table_id_))) { + } else if (OB_FAIL(resolve_sample_clause(sample_node, *table_item))) { LOG_WARN("resolve sample clause failed", K(ret)); } else { } } @@ -3965,7 +3965,7 @@ int ObDMLResolver::resolve_flashback_query_node(const ParseNode *time_node, Tabl } else if (TableItem::USING_SCN == table_item->flashback_query_type_ && ObUInt64Type != expr->get_result_type().get_type()) { ObExprResType res_type; - res_type.set_type(ObUInt64Type); + res_type.set_uint64(); res_type.set_accuracy(ObAccuracy::DDL_DEFAULT_ACCURACY2[ORACLE_MODE][ObUInt64Type]); OZ(ObRawExprUtils::create_cast_expr(*params_.expr_factory_, expr, res_type, dst_expr, session_info_, use_default_cm, cm)); @@ -6570,10 +6570,22 @@ int ObDMLResolver::resolve_special_expr(ObRawExpr *&expr, ObStmtScope scope) LOG_WARN("resolve special expr failed", K(ret), K(i)); } } -} + } if (OB_SUCC(ret) && expr->has_flag(CNT_CUR_TIME)) { stmt->get_query_ctx()->fetch_cur_time_ = true; } + if (OB_FAIL(ret)) { + // do nothing + } else if (stmt->is_select_stmt() && T_FIELD_LIST_SCOPE == scope) { + // do nothing + } else if (T_FUN_SYS_AUDIT_LOG_SET_FILTER == expr->get_expr_type() || + T_FUN_SYS_AUDIT_LOG_REMOVE_FILTER == expr->get_expr_type() || + T_FUN_SYS_AUDIT_LOG_SET_USER == expr->get_expr_type() || + T_FUN_SYS_AUDIT_LOG_REMOVE_USER == expr->get_expr_type()) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("use audit log function in dml stmt is not supported", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "use audit log function in dml stmt"); + } return ret; } @@ -11049,11 +11061,11 @@ int ObDMLResolver::build_prefix_index_compare_expr(ObRawExpr &column_expr, } int ObDMLResolver::resolve_sample_clause(const ParseNode *sample_node, - const uint64_t table_id) + TableItem &table_item) { int ret = OB_SUCCESS; ObSelectStmt *stmt; - if (OB_ISNULL(get_stmt())) { + if (OB_ISNULL(get_stmt()) || OB_ISNULL(allocator_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("stmt should not be NULL", K(ret)); } else if (OB_UNLIKELY(!get_stmt()->is_select_stmt())) { @@ -11062,13 +11074,19 @@ int ObDMLResolver::resolve_sample_clause(const ParseNode *sample_node, } else { stmt = static_cast(get_stmt()); enum SampleNode { METHOD = 0, PERCENT = 1, SEED = 2, SCOPE = 3}; - if (OB_ISNULL(sample_node) || OB_ISNULL(sample_node->children_[METHOD]) - || OB_ISNULL(sample_node->children_[PERCENT])) { + void *buf = allocator_->alloc(sizeof(SampleInfo)); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory for sample info", K(ret)); + } else if (OB_ISNULL(sample_node) + || OB_ISNULL(sample_node->children_[METHOD]) + || OB_ISNULL(sample_node->children_[PERCENT])) { ret = OB_ERR_UNEXPECTED; LOG_WARN("sample node should not be NULL", K(ret)); } else { - SampleInfo sample_info; - sample_info.table_id_ = table_id; + SampleInfo *samp_ptr = new(buf)SampleInfo(); + SampleInfo &sample_info = *samp_ptr; + sample_info.table_id_ = table_item.table_id_; if (sample_node->children_[METHOD]->value_ == 2) { sample_info.method_ = SampleInfo::BLOCK_SAMPLE; } else { @@ -11115,8 +11133,8 @@ int ObDMLResolver::resolve_sample_clause(const ParseNode *sample_node, } if (OB_FAIL(ret)) { // do nothing - } else if (OB_FAIL(stmt->add_sample_info(sample_info))) { - LOG_WARN("add sample info failed", K(ret), K(sample_info)); + } else { + table_item.sample_info_ = samp_ptr; } } if (OB_FAIL(ret)) { @@ -17394,11 +17412,12 @@ int ObDMLResolver::get_values_res_types(const ObIArray &cur_value LOG_WARN("get unexpected error", K(res_types), K(cur_values_types), K(session_info_), K(allocator_), K(ret)); } else { + ObExprTypeCtx type_ctx; + ObSQLUtils::init_type_ctx(session_info_, type_ctx); for (int64_t i = 0; OB_SUCC(ret) && i < res_types.count(); ++i) { ObExprVersion dummy_op(*allocator_); ObExprResType new_res_type; ObSEArray tmp_types; - const ObLengthSemantics length_semantics = session_info_->get_actual_nls_length_semantics(); ObCollationType coll_type = CS_TYPE_INVALID; if (OB_FAIL(tmp_types.push_back(res_types.at(i))) || OB_FAIL(tmp_types.push_back(cur_values_types.at(i)))) { @@ -17406,8 +17425,8 @@ int ObDMLResolver::get_values_res_types(const ObIArray &cur_value } else if (OB_FAIL(session_info_->get_collation_connection(coll_type))) { LOG_WARN("fail to get_collation_connection", K(ret)); } else if (OB_FAIL(dummy_op.aggregate_result_type_for_merge(new_res_type, &tmp_types.at(0), - tmp_types.count(), coll_type, false, - length_semantics))) { + tmp_types.count(), false, + type_ctx))) { LOG_WARN("failed to aggregate result type for merge", K(ret)); } else { res_types.at(i) = new_res_type; diff --git a/src/sql/resolver/dml/ob_dml_resolver.h b/src/sql/resolver/dml/ob_dml_resolver.h index 64decb895..4b77219fd 100644 --- a/src/sql/resolver/dml/ob_dml_resolver.h +++ b/src/sql/resolver/dml/ob_dml_resolver.h @@ -487,7 +487,7 @@ protected: const share::schema::ObTableSchema &table_schema, TableItem &table_item); int resolve_sample_clause(const ParseNode *part_node, - const uint64_t table_id); + TableItem &table_item); int check_pivot_aggr_expr(ObRawExpr *expr) const; int resolve_transpose_table(const ParseNode *transpose_node, TableItem *&table_item); diff --git a/src/sql/resolver/dml/ob_dml_stmt.cpp b/src/sql/resolver/dml/ob_dml_stmt.cpp index b2b8ff088..113306de0 100644 --- a/src/sql/resolver/dml/ob_dml_stmt.cpp +++ b/src/sql/resolver/dml/ob_dml_stmt.cpp @@ -270,6 +270,7 @@ int TableItem::deep_copy(ObIRawExprCopier &expr_copier, ddl_schema_version_ = other.ddl_schema_version_; ddl_table_id_ = other.ddl_table_id_; ref_query_ = other.ref_query_; + SampleInfo *buf = NULL; if (is_json_table() && OB_FAIL(deep_copy_json_table_def(*other.json_table_def_, expr_copier, allocator))) { LOG_WARN("failed to deep copy json table define", K(ret)); @@ -284,7 +285,26 @@ int TableItem::deep_copy(ObIRawExprCopier &expr_copier, } else if (is_values_table() && OB_FAIL(deep_copy_values_table_def(*other.values_table_def_, expr_copier, allocator))) { LOG_WARN("failed to deep copy values table def", K(ret)); - } else { + } else { /* do nothing */ } + if (OB_SUCC(ret)) { + if (OB_ISNULL(other.sample_info_)) { + // do nothing + } else { + if (OB_ISNULL(sample_info_)) { + buf = static_cast(allocator->alloc(sizeof(SampleInfo))); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory for sample info", K(ret)); + } else { + sample_info_ = new(buf) SampleInfo(); + } + } + if (OB_SUCC(ret)) { + *sample_info_ = *other.sample_info_; + } + } + } + if (OB_SUCC(ret)) { exec_params_.reuse(); for (int64_t i = 0; OB_SUCC(ret) && i < other.exec_params_.count(); ++i) { ObRawExpr *exec_param = other.exec_params_.at(i); @@ -292,7 +312,7 @@ int TableItem::deep_copy(ObIRawExprCopier &expr_copier, if (OB_FAIL(expr_copier.do_copy_expr(exec_param, new_expr))) { LOG_WARN("failed to copy exec param", K(ret)); } else if (OB_ISNULL(new_expr) || - OB_UNLIKELY(!new_expr->is_exec_param_expr())) { + OB_UNLIKELY(!new_expr->is_exec_param_expr())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("exec param is invalid", K(ret), K(new_expr)); } else if (OB_FAIL(expr_copier.copy(static_cast(new_expr)->get_ref_expr()))) { diff --git a/src/sql/resolver/dml/ob_dml_stmt.h b/src/sql/resolver/dml/ob_dml_stmt.h index 9c52250e9..b66faca6a 100644 --- a/src/sql/resolver/dml/ob_dml_stmt.h +++ b/src/sql/resolver/dml/ob_dml_stmt.h @@ -267,6 +267,7 @@ struct TableItem json_table_def_ = nullptr; table_type_ = MAX_TABLE_TYPE; values_table_def_ = NULL; + sample_info_ = nullptr; } virtual TO_STRING_KV(N_TID, table_id_, @@ -289,7 +290,10 @@ struct TableItem K_(is_view_table), K_(part_ids), K_(part_names), K_(cte_type), KPC_(function_table_expr), K_(flashback_query_type), KPC_(flashback_query_expr), K_(table_type), - K_(exec_params), K_(mview_id), K_(need_expand_rt_mv)); + K_(exec_params), + KPC_(sample_info), + K_(mview_id), + K_(need_expand_rt_mv)); enum TableType { @@ -336,6 +340,7 @@ struct TableItem bool is_generated_table() const { return GENERATED_TABLE == type_; } bool is_temp_table() const { return TEMP_TABLE == type_; } bool is_fake_cte_table() const { return CTE_TABLE == type_; } + bool is_has_sample_info() const { return sample_info_ != nullptr; } bool is_joined_table() const { return JOINED_TABLE == type_; } bool is_function_table() const { return FUNCTION_TABLE == type_; } bool is_link_table() const { return OB_INVALID_ID != dblink_id_; } // why not use type_, cause type_ will be changed in dblink transform rule, but dblink id don't change @@ -433,9 +438,11 @@ struct TableItem common::ObSEArray part_names_; common::ObSEArray exec_params_; // json table - ObJsonTableDef* json_table_def_; + ObJsonTableDef *json_table_def_; // values table ObValuesTableDef *values_table_def_; + // sample scan infos + SampleInfo *sample_info_; }; struct ColumnItem diff --git a/src/sql/resolver/dml/ob_hint.cpp b/src/sql/resolver/dml/ob_hint.cpp index 627ce582e..5f1b59f3c 100644 --- a/src/sql/resolver/dml/ob_hint.cpp +++ b/src/sql/resolver/dml/ob_hint.cpp @@ -16,6 +16,7 @@ #include "sql/optimizer/ob_log_plan.h" #include "common/ob_smart_call.h" #include "sql/monitor/ob_sql_plan.h" +#include "share/config/ob_config_helper.h" namespace oceanbase { @@ -784,12 +785,26 @@ bool ObOptParamHint::is_param_val_valid(const OptParamType param_type, const ObO { bool is_valid = false; switch (param_type) { - case HIDDEN_COLUMN_VISIBLE: { - is_valid = val.is_varchar() && (0 == val.get_varchar().case_compare("true") - || 0 == val.get_varchar().case_compare("false")); - break; - } - case ROWSETS_ENABLED: { + case HIDDEN_COLUMN_VISIBLE: + case ROWSETS_ENABLED: + case ENABLE_NEWSORT: + case USE_PART_SORT_MGB: + case USE_DEFAULT_OPT_STAT: + case ENABLE_IN_RANGE_OPTIMIZATION: + case XSOLAPI_GENERATE_WITH_CLAUSE: + case ENABLE_RICH_VECTOR_FORMAT: + case _ENABLE_STORAGE_CARDINALITY_ESTIMATION: + case PRESERVE_ORDER_FOR_PAGINATION: + case ENABLE_DAS_KEEP_ORDER: + case HASH_JOIN_ENABLED: + case OPTIMIZER_SORTMERGE_JOIN_ENABLED: + case NESTED_LOOP_JOIN_ENABLED: + case ENABLE_RANGE_EXTRACTION_FOR_NOT_IN: + case OPTIMIZER_SKIP_SCAN_ENABLED: + case OPTIMIZER_BETTER_INLIST_COSTING: + case OPTIMIZER_GROUP_BY_PLACEMENT: + case ENABLE_SPF_BATCH_RESCAN: + case NLJ_BATCHING_ENABLED: { is_valid = val.is_varchar() && (0 == val.get_varchar().case_compare("true") || 0 == val.get_varchar().case_compare("false")); break; @@ -798,63 +813,28 @@ bool ObOptParamHint::is_param_val_valid(const OptParamType param_type, const ObO is_valid = val.is_int() && (0 <= val.get_int() && 65535 >= val.get_int()); break; } + case OPTIMIZER_INDEX_COST_ADJ: + case BLOOM_FILTER_RATIO: { + is_valid = val.is_int() && (0 <= val.get_int() && 100 >= val.get_int()); + break; + } + case WITH_SUBQUERY: { + is_valid = val.is_int() && (0 <= val.get_int() && 2 >= val.get_int()); + break; + } case DDL_EXECUTION_ID: { is_valid = val.is_int() && (0 <= val.get_int()); break; } - case DDL_TASK_ID: { + case DDL_TASK_ID: + case INLIST_REWRITE_THRESHOLD: { is_valid = val.is_int() && (0 < val.get_int()); break; } - case ENABLE_NEWSORT: { - is_valid = val.is_varchar() && (0 == val.get_varchar().case_compare("true") - || 0 == val.get_varchar().case_compare("false")); - break; - } - case USE_PART_SORT_MGB: { - is_valid = val.is_varchar() && (0 == val.get_varchar().case_compare("true") - || 0 == val.get_varchar().case_compare("false")); - break; - } - case USE_DEFAULT_OPT_STAT: { - is_valid = val.is_varchar() && (0 == val.get_varchar().case_compare("true") - || 0 == val.get_varchar().case_compare("false")); - break; - } - case ENABLE_IN_RANGE_OPTIMIZATION: { - is_valid = val.is_varchar() && (0 == val.get_varchar().case_compare("true") - || 0 == val.get_varchar().case_compare("false")); - break; - } - case XSOLAPI_GENERATE_WITH_CLAUSE: { - is_valid = val.is_varchar() && (0 == val.get_varchar().case_compare("true") - || 0 == val.get_varchar().case_compare("false")); - break; - } case WORKAREA_SIZE_POLICY: { is_valid = val.is_varchar() && (0 == val.get_varchar().case_compare("MANULE")); break; } - case ENABLE_RICH_VECTOR_FORMAT: { - is_valid = val.is_varchar() && (0 == val.get_varchar().case_compare("true") - || 0 == val.get_varchar().case_compare("false")); - break; - } - case _ENABLE_STORAGE_CARDINALITY_ESTIMATION: { - is_valid = val.is_varchar() && (0 == val.get_varchar().case_compare("true") - || 0 == val.get_varchar().case_compare("false")); - break; - } - case PRESERVE_ORDER_FOR_PAGINATION: { - is_valid = val.is_varchar() && (0 == val.get_varchar().case_compare("true") - || 0 == val.get_varchar().case_compare("false")); - break; - } - case ENABLE_DAS_KEEP_ORDER : { - is_valid = val.is_varchar() && (0 == val.get_varchar().case_compare("true") - || 0 == val.get_varchar().case_compare("false")); - break; - } case SPILL_COMPRESSION_CODEC: { is_valid = val.is_varchar(); if (is_valid) { @@ -868,8 +848,16 @@ bool ObOptParamHint::is_param_val_valid(const OptParamType param_type, const ObO } break; } - case INLIST_REWRITE_THRESHOLD: { - is_valid = val.is_int() && (0 < val.get_int()); + case RUNTIME_FILTER_TYPE: { + is_valid = false; + if (!val.is_varchar()) { + // do nothing + } else { + ObString str_val = val.get_varchar(); + int64_t rf_type = ObConfigRuntimeFilterChecker::get_runtime_filter_type(str_val.ptr(), + str_val.length()); + is_valid = rf_type >= 0; + } break; } case PUSHDOWN_STORAGE_LEVEL: { @@ -934,7 +922,7 @@ int ObOptParamHint::get_bool_opt_param(const OptParamType param_type, bool &val, is_exists = false; ObObj obj; if (OB_FAIL(get_opt_param(param_type, obj))) { - LOG_WARN("fail to get rowsets_enabled opt_param", K(ret)); + LOG_WARN("fail to get bool opt_param", K(ret)); } else if (obj.is_nop_value()) { // do nothing } else if (!obj.is_varchar()) { @@ -953,12 +941,13 @@ int ObOptParamHint::get_bool_opt_param(const OptParamType param_type, bool &val) return get_bool_opt_param(param_type, val, is_exists); } -int ObOptParamHint::get_integer_opt_param(const OptParamType param_type, int64_t &val) const +int ObOptParamHint::get_integer_opt_param(const OptParamType param_type, int64_t &val, bool &is_exists) const { int ret = OB_SUCCESS; + is_exists = false; ObObj obj; if (OB_FAIL(get_opt_param(param_type, obj))) { - LOG_WARN("fail to get rowsets_enabled opt_param", K(ret)); + LOG_WARN("fail to get integer opt_param", K(ret)); } else if (obj.is_nop_value()) { // do nothing } else if (!obj.is_int()) { @@ -966,6 +955,32 @@ int ObOptParamHint::get_integer_opt_param(const OptParamType param_type, int64_t LOG_WARN("param obj is invalid", K(ret), K(obj)); } else { val = obj.get_int(); + is_exists = true; + } + return ret; +} + +int ObOptParamHint::get_integer_opt_param(const OptParamType param_type, int64_t &val) const +{ + bool is_exists = false; + return get_integer_opt_param(param_type, val, is_exists); +} + +int ObOptParamHint::get_opt_param_runtime_filter_type(int64_t &rf_type) const +{ + int ret = OB_SUCCESS; + ObObj obj; + if (OB_FAIL(get_opt_param(OptParamType::RUNTIME_FILTER_TYPE, obj))) { + LOG_WARN("fail to get runtime filter opt param", K(ret)); + } else if (obj.is_nop_value()) { + // do nothing + } else if (!obj.is_varchar()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("runtime filter opt param obj is invalid", K(ret), K(obj)); + } else { + ObString str_val = obj.get_varchar(); + rf_type = ObConfigRuntimeFilterChecker::get_runtime_filter_type(str_val.ptr(), + str_val.length()); } return ret; } diff --git a/src/sql/resolver/dml/ob_hint.h b/src/sql/resolver/dml/ob_hint.h index 6c88a3356..5988285d3 100644 --- a/src/sql/resolver/dml/ob_hint.h +++ b/src/sql/resolver/dml/ob_hint.h @@ -140,26 +140,39 @@ struct ObOptParamHint { ObOptParamHint() {}; - #define OPT_PARAM_TYPE_DEF(DEF) \ - DEF(INVALID_OPT_PARAM_TYPE, = 0) \ - DEF(HIDDEN_COLUMN_VISIBLE,) \ - DEF(ROWSETS_ENABLED,) \ - DEF(ROWSETS_MAX_ROWS,) \ - DEF(DDL_EXECUTION_ID,) \ - DEF(DDL_TASK_ID,) \ - DEF(ENABLE_NEWSORT,) \ - DEF(USE_PART_SORT_MGB,) \ - DEF(USE_DEFAULT_OPT_STAT,) \ - DEF(ENABLE_IN_RANGE_OPTIMIZATION,) \ - DEF(XSOLAPI_GENERATE_WITH_CLAUSE,) \ - DEF(WORKAREA_SIZE_POLICY,) \ - DEF(ENABLE_RICH_VECTOR_FORMAT,) \ - DEF(_ENABLE_STORAGE_CARDINALITY_ESTIMATION,) \ - DEF(PRESERVE_ORDER_FOR_PAGINATION,) \ - DEF(ENABLE_DAS_KEEP_ORDER,) \ - DEF(SPILL_COMPRESSION_CODEC,) \ - DEF(INLIST_REWRITE_THRESHOLD,) \ - DEF(PUSHDOWN_STORAGE_LEVEL,) \ + #define OPT_PARAM_TYPE_DEF(DEF) \ + DEF(INVALID_OPT_PARAM_TYPE, = 0) \ + DEF(HIDDEN_COLUMN_VISIBLE,) \ + DEF(ROWSETS_ENABLED,) \ + DEF(ROWSETS_MAX_ROWS,) \ + DEF(DDL_EXECUTION_ID,) \ + DEF(DDL_TASK_ID,) \ + DEF(ENABLE_NEWSORT,) \ + DEF(USE_PART_SORT_MGB,) \ + DEF(USE_DEFAULT_OPT_STAT,) \ + DEF(ENABLE_IN_RANGE_OPTIMIZATION,) \ + DEF(XSOLAPI_GENERATE_WITH_CLAUSE,) \ + DEF(WORKAREA_SIZE_POLICY,) \ + DEF(ENABLE_RICH_VECTOR_FORMAT,) \ + DEF(_ENABLE_STORAGE_CARDINALITY_ESTIMATION,) \ + DEF(PRESERVE_ORDER_FOR_PAGINATION,) \ + DEF(ENABLE_DAS_KEEP_ORDER,) \ + DEF(SPILL_COMPRESSION_CODEC,) \ + DEF(INLIST_REWRITE_THRESHOLD,) \ + DEF(PUSHDOWN_STORAGE_LEVEL,) \ + DEF(HASH_JOIN_ENABLED,) \ + DEF(OPTIMIZER_SORTMERGE_JOIN_ENABLED,) \ + DEF(NESTED_LOOP_JOIN_ENABLED,) \ + DEF(ENABLE_RANGE_EXTRACTION_FOR_NOT_IN,) \ + DEF(OPTIMIZER_INDEX_COST_ADJ,) \ + DEF(OPTIMIZER_SKIP_SCAN_ENABLED,) \ + DEF(OPTIMIZER_BETTER_INLIST_COSTING,) \ + DEF(OPTIMIZER_GROUP_BY_PLACEMENT,) \ + DEF(WITH_SUBQUERY,) \ + DEF(ENABLE_SPF_BATCH_RESCAN,) \ + DEF(NLJ_BATCHING_ENABLED,) \ + DEF(RUNTIME_FILTER_TYPE,) \ + DEF(BLOOM_FILTER_RATIO,) \ DEF(CORRELATION_FOR_CARDINALITY_ESTIMATION,) \ DECLARE_ENUM(OptParamType, opt_param, OPT_PARAM_TYPE_DEF, static); @@ -172,10 +185,12 @@ struct ObOptParamHint int get_opt_param(const OptParamType param_type, ObObj &val) const; int has_enable_opt_param(const OptParamType param_type, bool &enabled) const; int print_opt_param_hint(PlanText &plan_text) const; - int get_bool_opt_param(const OptParamType param_type, bool &val, bool& is_exists) const; + int get_bool_opt_param(const OptParamType param_type, bool &val, bool &is_exists) const; // if the corresponding opt_param is specified, the `val` will be overwritten int get_bool_opt_param(const OptParamType param_type, bool &val) const; + int get_integer_opt_param(const OptParamType param_type, int64_t &val, bool &is_exists) const; int get_integer_opt_param(const OptParamType param_type, int64_t &val) const; + int get_opt_param_runtime_filter_type(int64_t &rf_type) const; int get_enum_opt_param(const OptParamType param_type, int64_t &val) const; int has_opt_param(const OptParamType param_type, bool &has_hint) const; bool empty() const { return param_types_.empty(); } diff --git a/src/sql/resolver/dml/ob_inlist_resolver.cpp b/src/sql/resolver/dml/ob_inlist_resolver.cpp index 9580377fc..a72f464fd 100644 --- a/src/sql/resolver/dml/ob_inlist_resolver.cpp +++ b/src/sql/resolver/dml/ob_inlist_resolver.cpp @@ -334,8 +334,6 @@ int ObInListResolver::resolve_access_param_values_table(const ParseNode &in_list LOG_WARN("got unexpected NULL ptr", K(ret)); } else if (OB_FAIL(session_info->get_collation_connection(coll_type))) { LOG_WARN("fail to get collation_connection", K(ret)); - } else { - length_semantics = session_info->get_actual_nls_length_semantics(); } for (int64_t i = 0; OB_SUCC(ret) && i < row_cnt; i++) { @@ -360,13 +358,14 @@ int ObInListResolver::resolve_access_param_values_table(const ParseNode &in_list ObExprResType new_res_type; ObExprVersion dummy_op(*allocator); ObSEArray tmp_res_types; + ObExprTypeCtx type_ctx; + ObSQLUtils::init_type_ctx(session_info, type_ctx); if (OB_FAIL(tmp_res_types.push_back(table_def.column_types_.at(j)))) { LOG_WARN("failed to push back res type", K(ret)); } else if (OB_FAIL(tmp_res_types.push_back(res_type))) { LOG_WARN("failed to push back res type", K(ret)); } else if (OB_FAIL(dummy_op.aggregate_result_type_for_merge(new_res_type, - &tmp_res_types.at(0), 2, coll_type, lib::is_oracle_mode(), length_semantics, - session_info))) { + &tmp_res_types.at(0), 2, lib::is_oracle_mode(), type_ctx))) { LOG_WARN("failed to aggregate result type for merge", K(ret)); } else { table_def.column_types_.at(j) = new_res_type; @@ -462,13 +461,15 @@ int ObInListResolver::resolve_access_obj_values_table(const ParseNode &in_list, ObExprResType new_res_type; ObExprVersion dummy_op(*allocator); ObSEArray tmp_res_types; + ObExprTypeCtx type_ctx; + ObSQLUtils::init_type_ctx(session_info, type_ctx); if (OB_FAIL(tmp_res_types.push_back(table_def.column_types_.at(j)))) { LOG_WARN("failed to push back res type", K(ret)); } else if (OB_FAIL(tmp_res_types.push_back(res_type))) { LOG_WARN("failed to push back res type", K(ret)); } else if (OB_FAIL(dummy_op.aggregate_result_type_for_merge(new_res_type, - &tmp_res_types.at(0), 2, coll_type, is_oracle_mode, length_semantics, - session_info))) { + &tmp_res_types.at(0), 2, is_oracle_mode, + type_ctx))) { LOG_WARN("failed to aggregate result type for merge", K(ret)); } else { table_def.column_types_.at(j) = new_res_type; diff --git a/src/sql/resolver/dml/ob_select_resolver.cpp b/src/sql/resolver/dml/ob_select_resolver.cpp index 6509f47fc..838df5e76 100644 --- a/src/sql/resolver/dml/ob_select_resolver.cpp +++ b/src/sql/resolver/dml/ob_select_resolver.cpp @@ -1497,6 +1497,11 @@ int ObSelectResolver::resolve_normal_query(const ParseNode &parse_tree) } } } + if (OB_SUCC(ret)) { + if (OB_FAIL(check_audit_log_stmt(select_stmt))) { + LOG_WARN("failed to check audit log stmt"); + } + } return ret; } @@ -3969,11 +3974,11 @@ int ObSelectResolver::gen_unpivot_target_column(const int64_t table_count, } else if (OB_FAIL(session_info_->get_collation_connection(coll_type))) { LOG_WARN("fail to get_collation_connection", K(ret)); } else { - const ObLengthSemantics default_ls = session_info_->get_actual_nls_length_semantics(); ObSEArray types; ObExprResType res_type; ObExprVersion dummy_op(*allocator_); - + ObExprTypeCtx type_ctx; + ObSQLUtils::init_type_ctx(session_info_, type_ctx); for (int64_t colum_idx = 0; OB_SUCC(ret) && colum_idx < new_column_count; colum_idx++) { res_type.reset(); types.reset(); @@ -4024,9 +4029,8 @@ int ObSelectResolver::gen_unpivot_target_column(const int64_t table_count, } else if (OB_FAIL(dummy_op.aggregate_result_type_for_merge(res_type, &types.at(0), types.count(), - coll_type, true, - default_ls))) { + type_ctx))) { LOG_WARN("fail to aggregate_result_type_for_merge", K(ret), K(types)); } } @@ -7290,5 +7294,38 @@ int ObSelectResolver::check_listagg_aggr_param_valid(ObAggFunRawExpr *aggr_expr) return ret; } +int ObSelectResolver::check_audit_log_stmt(ObSelectStmt *select_stmt) +{ + int ret = OB_SUCCESS; + bool is_contain = false; + if (OB_ISNULL(select_stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (lib::is_mysql_mode()) { + for (int64_t i = 0; OB_SUCC(ret) && !is_contain && i < select_stmt->get_select_item_size(); i++) { + ObRawExpr *expr = select_stmt->get_select_item(i).expr_; + if (OB_ISNULL(expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (T_FUN_SYS_AUDIT_LOG_SET_FILTER == expr->get_expr_type() || + T_FUN_SYS_AUDIT_LOG_REMOVE_FILTER == expr->get_expr_type() || + T_FUN_SYS_AUDIT_LOG_SET_USER == expr->get_expr_type() || + T_FUN_SYS_AUDIT_LOG_REMOVE_USER == expr->get_expr_type()) { + is_contain = true; + } + } + if (OB_SUCC(ret) && is_contain) { + if (current_level_ > 0 || is_substmt() || select_stmt->get_table_size() > 0 || + select_stmt->get_condition_size() > 0 || select_stmt->has_group_by() || + select_stmt->has_having() || select_stmt->get_select_item_size() > 1 || + select_stmt->has_order_by() || select_stmt->has_limit()) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "use audit log function in complex query"); + } + } + } + return ret; +} + } // namespace sql } // namespace oceanbase diff --git a/src/sql/resolver/dml/ob_select_resolver.h b/src/sql/resolver/dml/ob_select_resolver.h index a4a01ef8c..e467507dd 100644 --- a/src/sql/resolver/dml/ob_select_resolver.h +++ b/src/sql/resolver/dml/ob_select_resolver.h @@ -363,6 +363,7 @@ private: int check_aggr_in_select_scope(ObSelectStmt *select_stmt); int mark_aggr_in_select_scope(ObSelectStmt *select_stmt); + int check_audit_log_stmt(ObSelectStmt *select_stmt); protected: // data members /*these member is only for with clause*/ diff --git a/src/sql/resolver/dml/ob_select_stmt.cpp b/src/sql/resolver/dml/ob_select_stmt.cpp index 409250bfc..470945d2e 100644 --- a/src/sql/resolver/dml/ob_select_stmt.cpp +++ b/src/sql/resolver/dml/ob_select_stmt.cpp @@ -204,8 +204,6 @@ int ObSelectStmt::assign(const ObSelectStmt &other) LOG_WARN("assign other rollup directions.", K(ret)); } else if (OB_FAIL(search_by_items_.assign(other.search_by_items_))) { LOG_WARN("assign search by items failed", K(ret)); - } else if (OB_FAIL(sample_infos_.assign(other.sample_infos_))) { - LOG_WARN("assign sample scan infos failed", K(ret)); } else if (OB_FAIL(set_query_.assign(other.set_query_))) { LOG_WARN("assign set query failed", K(ret)); } else if (OB_FAIL(for_update_dml_info_.assign(other.for_update_dml_info_))) { @@ -289,8 +287,6 @@ int ObSelectStmt::deep_copy_stmt_struct(ObIAllocator &allocator, other.order_items_, order_items_))) { LOG_WARN("deep copy order items failed", K(ret)); - } else if (OB_FAIL(sample_infos_.assign(other.sample_infos_))) { - LOG_WARN("failed to assign sample infos", K(ret)); } else if (OB_FAIL(deep_copy_stmt_objects(expr_copier, other.grouping_sets_items_, grouping_sets_items_))) { @@ -898,32 +894,6 @@ int ObSelectStmt::remove_useless_sharable_expr(ObRawExprFactory *expr_factory, return ret; } -const SampleInfo *ObSelectStmt::get_sample_info_by_table_id(uint64_t table_id) const -{ - const SampleInfo *sample_info = nullptr; - int64_t num = sample_infos_.count(); - for (int64_t i = 0; i < num; ++i) { - if (sample_infos_.at(i).table_id_ == table_id) { - sample_info = &sample_infos_.at(i); - break; - } - } - return sample_info; -} - -SampleInfo *ObSelectStmt::get_sample_info_by_table_id(uint64_t table_id) -{ - SampleInfo *sample_info = nullptr; - int64_t num = sample_infos_.count(); - for (int64_t i = 0; i < num; ++i) { - if (sample_infos_.at(i).table_id_ == table_id) { - sample_info = &sample_infos_.at(i); - break; - } - } - return sample_info; -} - bool ObSelectStmt::is_spj() const { bool has_rownum_expr = false; diff --git a/src/sql/resolver/dml/ob_select_stmt.h b/src/sql/resolver/dml/ob_select_stmt.h index f40fa56f6..985389371 100644 --- a/src/sql/resolver/dml/ob_select_stmt.h +++ b/src/sql/resolver/dml/ob_select_stmt.h @@ -638,13 +638,6 @@ public: int add_search_item(const OrderItem &order_item) { return search_by_items_.push_back(order_item); } int add_cycle_item(const ColumnItem &col_item) { return cycle_by_items_.push_back(col_item); } - int add_sample_info(const SampleInfo &sample_info) { return sample_infos_.push_back(sample_info); } - common::ObIArray &get_sample_infos() { return sample_infos_; } - const common::ObIArray &get_sample_infos() const { return sample_infos_; } - const SampleInfo *get_sample_info_by_table_id(uint64_t table_id) const; - SampleInfo *get_sample_info_by_table_id(uint64_t table_id); - // check if a table is using sample scan - bool is_sample_scan(uint64_t table_id) const { return get_sample_info_by_table_id(table_id) != nullptr; } virtual int check_table_be_modified(uint64_t ref_table_id, bool& is_modified) const override; // check aggregation has distinct or group concat e.g.: @@ -719,9 +712,6 @@ private: common::ObSEArray rollup_items_; common::ObSEArray cube_items_; - // sample scan infos - common::ObSEArray sample_infos_; - // for oracle mode only, for stmt print only common::ObSEArray for_update_columns_; diff --git a/src/sql/resolver/expr/ob_raw_expr.cpp b/src/sql/resolver/expr/ob_raw_expr.cpp index 86d5bbbc4..267f3da70 100644 --- a/src/sql/resolver/expr/ob_raw_expr.cpp +++ b/src/sql/resolver/expr/ob_raw_expr.cpp @@ -952,6 +952,10 @@ int ObRawExpr::is_const_inherit_expr(bool &is_const_inherit, || T_FUN_LABEL_SE_SESSION_LABEL == type_ || T_FUN_LABEL_SE_SESSION_ROW_LABEL == type_ || T_FUN_SYS_LAST_REFRESH_SCN == type_ + || T_FUN_SYS_AUDIT_LOG_SET_FILTER == type_ + || T_FUN_SYS_AUDIT_LOG_REMOVE_FILTER == type_ + || T_FUN_SYS_AUDIT_LOG_SET_USER == type_ + || T_FUN_SYS_AUDIT_LOG_REMOVE_USER == type_ || (T_FUN_UDF == type_ && !static_cast(this)->is_deterministic()) || T_FUN_SYS_GET_LOCK == type_ @@ -1062,7 +1066,8 @@ int ObRawExpr::is_non_pure_sys_func_expr(bool &is_non_pure) const || T_FUN_SYS_IS_USED_LOCK == type_ || T_FUN_SYS_RELEASE_LOCK == type_ || T_FUN_SYS_RELEASE_ALL_LOCKS == type_ - || T_FUN_SYS_CURRENT_ROLE == type_) { + || T_FUN_SYS_CURRENT_ROLE == type_ + || T_FUN_SYS_IS_ENABLED_ROLE == type_) { is_non_pure = true; } } diff --git a/src/sql/resolver/expr/ob_raw_expr_deduce_type.cpp b/src/sql/resolver/expr/ob_raw_expr_deduce_type.cpp index c9147c56c..36e517a65 100644 --- a/src/sql/resolver/expr/ob_raw_expr_deduce_type.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_deduce_type.cpp @@ -529,6 +529,7 @@ int ObRawExprDeduceType::calc_result_type(ObNonTerminalRawExpr &expr, } } } + if (!IS_CLUSTER_VERSION_BEFORE_4_1_0_0) { result_type.set_has_lob_header(); } @@ -2430,7 +2431,6 @@ int ObRawExprDeduceType::visit(ObSysFunRawExpr &expr) LOG_WARN("fail to calc result type", K(ret), K(types)); } } - if (OB_SUCC(ret) && T_FUN_SYS_ANY_VALUE == expr.get_expr_type()) { ObRawExpr *first_param = NULL; if (OB_ISNULL(first_param = expr.get_param_expr(0))) { @@ -2560,6 +2560,8 @@ int ObRawExprDeduceType::visit(ObWinFunRawExpr &expr) result_number_type.set_number(); common::ObIArray &func_params = expr.get_func_params(); + ObExprTypeCtx type_ctx; + ObSQLUtils::init_type_ctx(my_session_, type_ctx); if (func_params.count() <= 0) { if (NULL == expr.get_agg_expr()) { ObExprResType result_type(alloc_); @@ -2678,7 +2680,6 @@ int ObRawExprDeduceType::visit(ObWinFunRawExpr &expr) } else if (T_WIN_FUN_LEAD == expr.get_func_type() || T_WIN_FUN_LAG == expr.get_func_type()) { if (is_mysql_mode() && func_params.count() == 3) { //compatiable with mysql - const ObLengthSemantics default_ls = my_session_->get_actual_nls_length_semantics(); ObExprResType res_type; ObSEArray types; ObCollationType coll_type = CS_TYPE_INVALID; @@ -2691,9 +2692,8 @@ int ObRawExprDeduceType::visit(ObWinFunRawExpr &expr) } else if (OB_FAIL(ObExprOperator::aggregate_result_type_for_merge(res_type, &types.at(0), types.count(), - coll_type, false, - default_ls))) { + type_ctx))) { LOG_WARN("fail to aggregate_result_type_for_merge", K(ret), K(types)); } else { if (res_type.is_json()) { @@ -2925,7 +2925,7 @@ int ObRawExprDeduceType::visit(ObWinFunRawExpr &expr) LOG_WARN("fail to push_back", K(ret)); } else if (OB_FAIL(dummy_op.get_cmp_result_type3(result_type, need_no_cast, &types.at(0), types.count(), has_lower, - *my_session_))) { + type_ctx))) { LOG_WARN("fail to get_cmp_result_type3", K(ret)); } else { result_type.set_accuracy(result_type.get_calc_accuracy()); @@ -3143,9 +3143,13 @@ int ObRawExprDeduceType::set_agg_group_concat_result_type(ObAggFunRawExpr &expr, ObExprResType &result_type) { int ret = OB_SUCCESS; + CK(OB_NOT_NULL(my_session_)); + CK(OB_NOT_NULL(expr_factory_)); ObArray types; expr.set_data_type(ObVarcharType); const ObIArray &real_parm_exprs = expr.get_real_param_exprs(); + ObExprTypeCtx type_ctx; + ObSQLUtils::init_type_ctx(my_session_, type_ctx); for (int64_t i = 0; OB_SUCC(ret) && i < real_parm_exprs.count(); ++i) { ObRawExpr *real_param_expr = real_parm_exprs.at(i); if (OB_ISNULL(real_param_expr)) { @@ -3159,8 +3163,6 @@ int ObRawExprDeduceType::set_agg_group_concat_result_type(ObAggFunRawExpr &expr, K(real_param_expr->get_result_type())); } } - CK(OB_NOT_NULL(my_session_)); - CK(OB_NOT_NULL(expr_factory_)); ObCollationType coll_type = CS_TYPE_INVALID; OC( (my_session_->get_collation_connection)(coll_type) ); @@ -3196,7 +3198,7 @@ int ObRawExprDeduceType::set_agg_group_concat_result_type(ObAggFunRawExpr &expr, if (OB_FAIL(ret)) { } else if (OB_FAIL(dummy_op.aggregate_charsets_for_string_result( result_type, (types.count() == 0 ? NULL : &(types.at(0))), - types.count(), coll_type))) { + types.count(), type_ctx))) { LOG_WARN("fail to aggregate charsets for string result", K(ret), K(types)); } else { expr.set_result_type(result_type); @@ -3212,6 +3214,7 @@ int ObRawExprDeduceType::set_agg_group_concat_result_type(ObAggFunRawExpr &expr, ObExprResType result_type(alloc_); result_type.set_varchar(); result_type.set_collation_type(expr.get_result_type().get_collation_type()); + result_type.set_collation_level(expr.get_result_type().get_collation_level()); if (lib::is_oracle_mode()) { result_type.set_length_semantics(expr.get_result_type().get_length_semantics()); } diff --git a/src/sql/resolver/expr/ob_raw_expr_info_extractor.cpp b/src/sql/resolver/expr/ob_raw_expr_info_extractor.cpp index 2dee8b78e..48a80c594 100644 --- a/src/sql/resolver/expr/ob_raw_expr_info_extractor.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_info_extractor.cpp @@ -491,6 +491,10 @@ int ObRawExprInfoExtractor::visit(ObSysFunRawExpr &expr) || T_FUN_NORMAL_UDF == expr.get_expr_type() || T_FUN_SYS_GENERATOR == expr.get_expr_type() || T_FUN_SYS_LAST_REFRESH_SCN == expr.get_expr_type() + || T_FUN_SYS_AUDIT_LOG_SET_FILTER == expr.get_expr_type() + || T_FUN_SYS_AUDIT_LOG_REMOVE_FILTER == expr.get_expr_type() + || T_FUN_SYS_AUDIT_LOG_SET_USER == expr.get_expr_type() + || T_FUN_SYS_AUDIT_LOG_REMOVE_USER == expr.get_expr_type() || (T_FUN_UDF == expr.get_expr_type() && !static_cast(expr).is_deterministic()) || T_FUN_SYS_GET_LOCK == expr.get_expr_type() diff --git a/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp b/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp index 6cb367390..721269f99 100644 --- a/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp @@ -288,6 +288,10 @@ int ObRawExprResolverImpl::do_recursive_resolve(const ParseNode *node, if (OB_FAIL(ctx_.expr_factory_.create_raw_expr(node->type_, c_expr))) { LOG_WARN("fail to create raw expr", K(ret)); } else { + ObObj val; + val.set_null(); + c_expr->set_value(val); + c_expr->set_param(val); expr = c_expr; } break; @@ -617,12 +621,24 @@ int ObRawExprResolverImpl::do_recursive_resolve(const ParseNode *node, break; } case T_OP_AND: - case T_OP_OR: + case T_OP_OR: { + ObOpRawExpr *m_expr = NULL; + if (OB_FAIL(process_node_with_children(node, node->num_child_, m_expr, is_root_expr))) { + LOG_WARN("fail to process node with children", K(ret), K(node)); + } else if (OB_ISNULL(ctx_.session_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("session_info_ is NULL", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::try_add_bool_expr(m_expr, ctx_.expr_factory_))) { + LOG_WARN("try_add_bool_expr for add or expr failed", K(ret)); + } else { + expr = m_expr; + } + break; + } case T_OP_XOR: { ObOpRawExpr *m_expr = NULL; int64_t num_child = 2; - if (OB_FAIL(process_node_with_children(node, num_child, m_expr, - node->type_ == T_OP_XOR ? false : is_root_expr))) { + if (OB_FAIL(process_node_with_children(node, num_child, m_expr, false))) { LOG_WARN("fail to process node with children", K(ret), K(node)); } else if (OB_ISNULL(ctx_.session_info_)) { ret = OB_ERR_UNEXPECTED; @@ -1105,7 +1121,7 @@ int ObRawExprResolverImpl::do_recursive_resolve(const ParseNode *node, } case T_FUN_SYS_REGEXP_LIKE: case T_FUN_SYS: { - if (OB_FAIL(process_fun_sys_node(node, expr))) { + if (OB_FAIL(process_fun_sys_node(node, expr, is_root_expr))) { if (ret != OB_ERR_FUNCTION_UNKNOWN) { LOG_WARN("fail to process system function node", K(ret), K(node)); } else if (OB_FAIL(process_dll_udf_node(node, expr))) { @@ -2537,7 +2553,7 @@ int ObRawExprResolverImpl::resolve_func_node_of_obj_access_idents(const ParseNod } else if (func_node.type_ == T_FUN_ORA_XMLAGG) { OZ (process_agg_node(&func_node, func_expr)); }else { - OZ (process_fun_sys_node(&func_node, func_expr)); + OZ (process_fun_sys_node(&func_node, func_expr, false)); } CK (OB_NOT_NULL(func_expr)); OX (access_ident.sys_func_expr_ = func_expr); @@ -6769,7 +6785,9 @@ int ObRawExprResolverImpl::cast_accuracy_check(const ParseNode *node, const char return ret; } -int ObRawExprResolverImpl::process_fun_sys_node(const ParseNode *node, ObRawExpr *&expr) +int ObRawExprResolverImpl::process_fun_sys_node(const ParseNode *node, + ObRawExpr *&expr, + const bool is_root_expr) { int ret = OB_SUCCESS; ObSysFunRawExpr *func_expr = NULL; @@ -6926,6 +6944,18 @@ int ObRawExprResolverImpl::process_fun_sys_node(const ParseNode *node, ObRawExpr } } + if (OB_SUCC(ret)) { + if (T_FUN_SYS_AUDIT_LOG_SET_FILTER == func_expr->get_expr_type() + || T_FUN_SYS_AUDIT_LOG_REMOVE_FILTER == func_expr->get_expr_type() + || T_FUN_SYS_AUDIT_LOG_SET_USER == func_expr->get_expr_type() + || T_FUN_SYS_AUDIT_LOG_REMOVE_USER == func_expr->get_expr_type()) { + if (OB_UNLIKELY(!is_root_expr)) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "use audit log function as param is"); + } + } + } + //mark expr is deterministic or not(default deterministic) if (OB_SUCC(ret)) { //bug: @@ -7021,6 +7051,21 @@ int ObRawExprResolverImpl::process_sys_func_params(ObSysFunRawExpr &func_expr, i } } break; + case T_FUN_SYS_AUDIT_LOG_SET_FILTER: + case T_FUN_SYS_AUDIT_LOG_REMOVE_FILTER: + case T_FUN_SYS_AUDIT_LOG_SET_USER: + case T_FUN_SYS_AUDIT_LOG_REMOVE_USER: + for (int64_t i = 0; OB_SUCC(ret) && i < func_expr.get_param_count(); ++i) { + ObRawExpr *param_expr = func_expr.get_param_expr(i); + if (OB_ISNULL(param_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null epxr", K(ret)); + } else if (OB_UNLIKELY(!param_expr->is_const_raw_expr())) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "non-const param of audit log function"); + } + } + break; default: break; } @@ -8563,8 +8608,8 @@ int ObRawExprResolverImpl::check_internal_function(const ObString &name) bool is_internal = false; if (OB_FAIL(ret)) { } else if (ctx_.session_info_->is_inner() - || is_sys_view(ctx_.view_ref_id_) - || ctx_.is_from_show_resolver_) { + || ctx_.is_in_system_view_ + || ctx_.is_from_show_resolver_) { // ignore } else if (FALSE_IT(ObExprOperatorFactory::get_internal_info_by_name(name, exist, is_internal))) { } else if (exist && is_internal) { diff --git a/src/sql/resolver/expr/ob_raw_expr_resolver_impl.h b/src/sql/resolver/expr/ob_raw_expr_resolver_impl.h index 0d0e8ff45..1472d3f24 100644 --- a/src/sql/resolver/expr/ob_raw_expr_resolver_impl.h +++ b/src/sql/resolver/expr/ob_raw_expr_resolver_impl.h @@ -129,7 +129,7 @@ private: int process_json_array_node(const ParseNode *node, ObRawExpr *&expr); int process_json_mergepatch_node(const ParseNode *node, ObRawExpr *&expr); static void modification_type_to_int(ParseNode &node); - int process_fun_sys_node(const ParseNode *node, ObRawExpr *&expr); + int process_fun_sys_node(const ParseNode *node, ObRawExpr *&expr, const bool is_root_expr); int process_dll_udf_node(const ParseNode *node, ObRawExpr *&expr); int process_agg_udf_node(const ParseNode *node, const share::schema::ObUDF &udf_info, diff --git a/src/sql/resolver/expr/ob_raw_expr_util.cpp b/src/sql/resolver/expr/ob_raw_expr_util.cpp index 6562aaf4f..3efddd811 100644 --- a/src/sql/resolver/expr/ob_raw_expr_util.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_util.cpp @@ -4165,7 +4165,7 @@ int ObRawExprUtils::try_add_cast_expr_above(ObRawExprFactory *expr_factory, session, false, cm_zf, local_vars, local_var_id)); CK(OB_NOT_NULL(new_expr = dynamic_cast(cast_expr))); } - LOG_DEBUG("in try_add_cast", K(ret), K(dst_type), K(cm)); + LOG_DEBUG("in try_add_cast", K(ret), K(dst_type), K(src_type) ,K(cm)); } return ret; } @@ -4342,10 +4342,18 @@ int ObRawExprUtils::create_cast_expr(ObRawExprFactory &expr_factory, } else { OZ(create_real_cast_expr(expr_factory, src_expr, dst_type, func_expr, session)); } + if (OB_SUCC(ret) && lib::is_mysql_mode()) { + if (dst_type.get_collation_level() == CS_LEVEL_INVALID) { + LOG_WARN("aggregation level is CS_TYPE_INVALID", K(dst_type)); + } else if (OB_FAIL(ObSQLUtils::set_cs_level_cast_mode(dst_type.get_collation_level(), cm))) { + LOG_WARN("failed to set cs level cast mode", K(ret)); + } + } if (NULL != extra_cast) { OX(extra_cast->set_extra(cm)); OZ(extra_cast->add_flag(IS_INNER_ADDED_EXPR)); } + CK(OB_NOT_NULL(func_expr)); OX(func_expr->set_extra(cm)); OZ(func_expr->add_flag(IS_INNER_ADDED_EXPR)); @@ -4395,6 +4403,7 @@ int ObRawExprUtils::setup_extra_cast_utf8_type(const ObExprResType &type, } else { utf8_type = type; utf8_type.set_collation_type(ObCharset::get_system_collation()); + utf8_type.set_collation_level(type.get_collation_level()); if (ObNVarchar2Type == type.get_type()) { utf8_type.set_type(ObVarcharType); } else if (ObNCharType == type.get_type()) { @@ -4892,6 +4901,7 @@ int ObRawExprUtils::create_type_to_str_expr(ObRawExprFactory &expr_factory, LOG_WARN("col_accuracy_expr is NULL", K(ret)); } else { col_accuracy_expr->set_collation_type(src_expr->get_collation_type()); + col_accuracy_expr->set_collation_level(src_expr->get_collation_level()); col_accuracy_expr->set_accuracy(src_expr->get_accuracy()); } } diff --git a/src/sql/resolver/ob_resolver.cpp b/src/sql/resolver/ob_resolver.cpp index 8239f1b07..347635c63 100644 --- a/src/sql/resolver/ob_resolver.cpp +++ b/src/sql/resolver/ob_resolver.cpp @@ -421,7 +421,20 @@ int ObResolver::resolve(IsPrepared if_prepared, const ParseNode &parse_tree, ObS REGISTER_STMT_RESOLVER(FlushDagWarnings); break; } - case T_FLUSH_PRIVILEGES: { + case T_FLUSH_PRIVILEGES: + case T_INSTALL_PLUGIN: + case T_UNINSTALL_PLUGIN: + case T_FLUSH_MOCK: + case T_HANDLER_MOCK: + case T_FLUSH_MOCK_LIST: + case T_SHOW_PLUGINS: + case T_CREATE_SERVER: + case T_ALTER_SERVER: + case T_DROP_SERVER: + case T_CREATE_LOGFILE_GROUP: + case T_ALTER_LOGFILE_GROUP: + case T_DROP_LOGFILE_GROUP: + { REGISTER_STMT_RESOLVER(Mock); break; } @@ -747,7 +760,9 @@ int ObResolver::resolve(IsPrepared if_prepared, const ParseNode &parse_tree, ObS case T_SHOW_CREATE_TRIGGER: case T_SHOW_ENGINE: case T_SHOW_OPEN_TABLES: - case T_SHOW_SEQUENCES: { + case T_SHOW_SEQUENCES: + case T_SHOW_CHECK_TABLE: + case T_SHOW_CREATE_USER: { REGISTER_STMT_RESOLVER(Show); break; } diff --git a/src/sql/resolver/ob_resolver_utils.cpp b/src/sql/resolver/ob_resolver_utils.cpp index 5e7226130..ee0d1aabc 100644 --- a/src/sql/resolver/ob_resolver_utils.cpp +++ b/src/sql/resolver/ob_resolver_utils.cpp @@ -2344,6 +2344,7 @@ stmt::StmtType ObResolverUtils::get_stmt_type_by_item_type(const ObItemType item SET_STMT_TYPE(T_CHECKSUM_TABLE); SET_STMT_TYPE(T_CACHE_INDEX); SET_STMT_TYPE(T_LOAD_INDEX_INTO_CACHE); + SET_STMT_TYPE(T_SHOW_CREATE_USER); #undef SET_STMT_TYPE case T_ROLLBACK: case T_COMMIT: { @@ -2581,7 +2582,6 @@ int ObResolverUtils::resolve_const(const ParseNode *node, Otherwise, the character set and collation given by the character_set_connection and collation_connection system variables are used. */ - val.set_collation_level(CS_LEVEL_COERCIBLE); //TODO::@yanhua raw // if (lib::is_oracle_mode()) { // val.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); @@ -2621,6 +2621,7 @@ int ObResolverUtils::resolve_const(const ParseNode *node, } } } + val.set_collation_level(CS_LEVEL_COERCIBLE); ObLengthSemantics length_semantics = LS_DEFAULT; if (OB_SUCC(ret)) { if (lib::is_oracle_mode() && (T_NVARCHAR2 == node->type_ || T_NCHAR == node->type_)) { @@ -5430,6 +5431,7 @@ int ObResolverUtils::resolve_generated_column_expr(ObResolverParams ¶ms, ObExprResType cast_dst_type; cast_dst_type.set_meta(generated_column.get_meta_type()); cast_dst_type.set_accuracy(generated_column.get_accuracy()); + cast_dst_type.set_collation_level(CS_LEVEL_IMPLICIT); ObRawExpr *expr_with_implicit_cast = NULL; //only formalize once if (OB_FAIL(ObRawExprUtils::erase_operand_implicit_cast(expr, expr))) { @@ -5737,11 +5739,26 @@ int ObResolverUtils::resolve_default_expr_v2_column_expr(ObResolverParams ¶m KPC(expr), K(ret)); } } else if (OB_UNLIKELY(!columns.empty())) { - ret = OB_ERR_BAD_FIELD_ERROR; const ObQualifiedName &q_name = columns.at(0); - ObString scope_name = default_expr_v2_column.get_column_name_str(); - ObString col_name = concat_qualified_name(q_name.database_name_, q_name.tbl_name_, q_name.col_name_); - LOG_USER_ERROR(OB_ERR_BAD_FIELD_ERROR, col_name.length(), col_name.ptr(), scope_name.length(), scope_name.ptr()); + bool contain_udf = false; + int64_t udf_idx = -1; + for (int64_t i = 0; iformalize(session_info))) { LOG_WARN("formalize expr failed", K(ret)); } else if (OB_UNLIKELY(expr->has_flag(CNT_ROWNUM))) { @@ -9060,6 +9077,7 @@ int ObResolverUtils::resolve_file_format_string_value(const ParseNode *node, cast_dst_type.set_length(max_len); cast_dst_type.set_calc_meta(ObObjMeta()); cast_dst_type.set_collation_type(result_collation_type); + cast_dst_type.set_collation_level(expr->get_collation_level()); if (!(expr_output_type.is_varchar() || expr_output_type.is_nvarchar2() || expr_output_type.is_char() || diff --git a/src/sql/resolver/ob_schema_checker.cpp b/src/sql/resolver/ob_schema_checker.cpp index b7b6b9ff4..7e0de0a18 100644 --- a/src/sql/resolver/ob_schema_checker.cpp +++ b/src/sql/resolver/ob_schema_checker.cpp @@ -276,7 +276,8 @@ int ObSchemaChecker::check_routine_show(const share::schema::ObSessionPrivInfo & int ObSchemaChecker::check_trigger_show(const share::schema::ObSessionPrivInfo &s_priv, const ObString &db, const ObString &trigger, - bool &allow_show) const + bool &allow_show, + const ObString &table) const { int ret = OB_SUCCESS; allow_show = true; @@ -286,7 +287,25 @@ int ObSchemaChecker::check_trigger_show(const share::schema::ObSessionPrivInfo & } else if (OB_UNLIKELY(!s_priv.is_valid() || db.empty() || trigger.empty())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(s_priv), K(db), K(trigger), K(ret)); - } else {} + } else { + bool need_check = false; + if(OB_FAIL(ObCompatControl::check_feature_enable(s_priv.security_version_, + ObCompatFeatureType::MYSQL_TRIGGER_PRIV_CHECK, need_check))) { + LOG_WARN("failed to check feature enable", K(ret)); + } else if(need_check && lib::is_mysql_mode()) { + ObNeedPriv need_priv; + need_priv.priv_level_ = OB_PRIV_TABLE_LEVEL; + need_priv.db_ = db; + need_priv.priv_set_ = OB_PRIV_TRIGGER; + need_priv.table_ = table; + OZ (schema_mgr_->check_single_table_priv(s_priv, need_priv)); + if(OB_FAIL(ret)) { + allow_show = false; + ret = OB_SUCCESS; + LOG_WARN("show create trigger not has trigger priv", K(s_priv), K(db), K(trigger), K(table), K(ret)); + } + } + } return ret; } diff --git a/src/sql/resolver/ob_schema_checker.h b/src/sql/resolver/ob_schema_checker.h index 9cf2b0924..8ba7b9f38 100644 --- a/src/sql/resolver/ob_schema_checker.h +++ b/src/sql/resolver/ob_schema_checker.h @@ -99,7 +99,8 @@ public: int check_trigger_show(const share::schema::ObSessionPrivInfo &s_priv, const common::ObString &db, const common::ObString &trigger, - bool &allow_show) const; + bool &allow_show, + const ObString &table) const; int check_column_exists(const uint64_t tenant_id, const uint64_t table_id, const common::ObString &column_name, bool &is_exist, diff --git a/src/sql/resolver/ob_stmt.h b/src/sql/resolver/ob_stmt.h index f8fbaebbc..a899838f9 100644 --- a/src/sql/resolver/ob_stmt.h +++ b/src/sql/resolver/ob_stmt.h @@ -175,7 +175,8 @@ public: static inline bool is_show_stmt(stmt::StmtType stmt_type) { return (stmt_type >= stmt::T_SHOW_TABLES && stmt_type <= stmt::T_SHOW_GRANTS) - || stmt_type == stmt::T_SHOW_TRIGGERS; + || stmt_type == stmt::T_SHOW_TRIGGERS + || stmt_type == stmt::T_SHOW_CREATE_USER; } static inline bool is_dml_write_stmt(stmt::StmtType stmt_type) diff --git a/src/sql/resolver/ob_stmt_type.h b/src/sql/resolver/ob_stmt_type.h index 5a24fa63d..2971881dc 100644 --- a/src/sql/resolver/ob_stmt_type.h +++ b/src/sql/resolver/ob_stmt_type.h @@ -194,8 +194,8 @@ OB_STMT_TYPE_DEF_UNKNOWN_AT(T_CREATE_KEYSTORE, no_priv_needed, 184) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_ALTER_KEYSTORE, no_priv_needed, 185) OB_STMT_TYPE_DEF(T_CREATE_TABLESPACE, get_create_tablespace_priv, 186, ACTION_TYPE_CREATE_TABLESPACE) OB_STMT_TYPE_DEF(T_DROP_TABLESPACE, get_create_tablespace_priv, 187, ACTION_TYPE_DROP_TABLESPACE) -OB_STMT_TYPE_DEF(T_CREATE_TRIGGER, no_priv_needed, 188, ACTION_TYPE_CREATE_TRIGGER) -OB_STMT_TYPE_DEF(T_DROP_TRIGGER, no_priv_needed, 189, ACTION_TYPE_DROP_TRIGGER) +OB_STMT_TYPE_DEF(T_CREATE_TRIGGER, get_trigger_stmt_need_privs, 188, ACTION_TYPE_CREATE_TRIGGER) +OB_STMT_TYPE_DEF(T_DROP_TRIGGER, get_trigger_stmt_need_privs, 189, ACTION_TYPE_DROP_TRIGGER) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_FLASHBACK_TABLE_TO_SCN, get_flashback_table_stmt_need_privs, 190) OB_STMT_TYPE_DEF(T_CREATE_ROLE, get_role_privs, 191, ACTION_TYPE_CREATE_ROLE) OB_STMT_TYPE_DEF(T_DROP_ROLE, get_role_privs, 192, ACTION_TYPE_DROP_ROLE) diff --git a/src/sql/rewrite/ob_query_range.cpp b/src/sql/rewrite/ob_query_range.cpp index 7e31dd10d..66fb2da7d 100644 --- a/src/sql/rewrite/ob_query_range.cpp +++ b/src/sql/rewrite/ob_query_range.cpp @@ -160,6 +160,7 @@ void ObQueryRange::reset() int ObQueryRange::init_query_range_ctx(ObIAllocator &allocator, const ColumnIArray &range_columns, ObExecContext *exec_ctx, + ObQueryCtx *query_ctx, ExprConstrantArray *expr_constraints, const ParamsIArray *params, const bool phy_rowid_for_table_loc, @@ -177,11 +178,12 @@ int ObQueryRange::init_query_range_ctx(ObIAllocator &allocator, ret = OB_ALLOCATE_MEMORY_FAILED; LOG_ERROR("alloc query range context failed", K(ret)); } else if (OB_ISNULL(exec_ctx) || OB_ISNULL(exec_ctx->get_my_session()) || - OB_ISNULL(exec_ctx->get_physical_plan_ctx())) { + OB_ISNULL(exec_ctx->get_physical_plan_ctx()) + || OB_ISNULL(query_ctx)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(exec_ctx)); + LOG_WARN("get unexpected null", K(ret), K(exec_ctx), K(query_ctx)); } else { - query_range_ctx_ = new(ptr) ObQueryRangeCtx(exec_ctx, expr_constraints, params); + query_range_ctx_ = new(ptr) ObQueryRangeCtx(exec_ctx, query_ctx, expr_constraints, params); query_range_ctx_->phy_rowid_for_table_loc_ = phy_rowid_for_table_loc; query_range_ctx_->ignore_calc_failure_ = ignore_calc_failure; query_range_ctx_->index_prefix_ = index_prefix; @@ -272,6 +274,7 @@ int ObQueryRange::preliminary_extract_query_range(const ColumnIArray &range_colu const ObRawExpr *expr_root, const ObDataTypeCastParams &dtc_params, ObExecContext *exec_ctx, + ObQueryCtx *query_ctx, ExprConstrantArray *expr_constraints /* = NULL */, const ParamsIArray *params /* = NULL */, const bool use_in_optimization /* = false */, @@ -279,8 +282,9 @@ int ObQueryRange::preliminary_extract_query_range(const ColumnIArray &range_colu { int ret = OB_SUCCESS; ObArenaAllocator ctx_allocator(ObModIds::OB_QUERY_RANGE_CTX); - if (OB_FAIL(init_query_range_ctx(ctx_allocator, range_columns, exec_ctx, expr_constraints, params, - false, true, use_in_optimization, index_prefix))) { + if (OB_FAIL(init_query_range_ctx(ctx_allocator, range_columns, exec_ctx, query_ctx, + expr_constraints, params, false, true, + use_in_optimization, index_prefix))) { LOG_WARN("init query range context failed", K(ret)); } else if (OB_ISNULL(query_range_ctx_)) { ret = OB_NOT_INIT; @@ -741,6 +745,7 @@ int ObQueryRange::preliminary_extract_query_range(const ColumnIArray &range_colu const ExprIArray &root_exprs, const ObDataTypeCastParams &dtc_params, ObExecContext *exec_ctx, + ObQueryCtx *query_ctx, ExprConstrantArray *expr_constraints /* = NULL */, const ParamsIArray *params /* = NULL */, const bool phy_rowid_for_table_loc /* = false*/, @@ -757,7 +762,7 @@ int ObQueryRange::preliminary_extract_query_range(const ColumnIArray &range_colu SQL_REWRITE_LOG(DEBUG, "preliminary extract", K(range_columns), K(root_exprs), K(use_in_optimization)); ObSEArray candi_exprs; ObArenaAllocator ctx_allocator(ObModIds::OB_QUERY_RANGE_CTX); - if (OB_FAIL(init_query_range_ctx(ctx_allocator, range_columns, exec_ctx, + if (OB_FAIL(init_query_range_ctx(ctx_allocator, range_columns, exec_ctx, query_ctx, expr_constraints, params, phy_rowid_for_table_loc, ignore_calc_failure, use_in_optimization, index_prefix))) { LOG_WARN("init query range context failed", K(ret)); @@ -3503,11 +3508,13 @@ int ObQueryRange::pre_extract_not_in_op(const ObOpRawExpr *b_expr, const ObRawExpr *l_expr = NULL; const ObOpRawExpr *r_expr = NULL; ObSQLSessionInfo *session = NULL; + ObQueryCtx *query_ctx = NULL; bool enable_not_in_range = false; if (OB_ISNULL(b_expr) || OB_ISNULL(query_range_ctx_) || OB_ISNULL(query_range_ctx_->exec_ctx_) - || OB_ISNULL(session = query_range_ctx_->exec_ctx_->get_my_session())) { + || OB_ISNULL(session = query_range_ctx_->exec_ctx_->get_my_session()) + || OB_ISNULL(query_ctx = query_range_ctx_->query_ctx_)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("unexpected null", K(b_expr), K_(query_range_ctx), K(session)); + LOG_WARN("get unexpected null", K(b_expr), K_(query_range_ctx), K(query_ctx), K(session)); } else if (2 != b_expr->get_param_count()) {//binary op expr ret = OB_ERR_UNEXPECTED; LOG_WARN("b_expr must has 2 arguments", K(ret)); @@ -3517,6 +3524,8 @@ int ObQueryRange::pre_extract_not_in_op(const ObOpRawExpr *b_expr, LOG_WARN("r_expr is null.", K(ret)); } else if (OB_FAIL(session->is_enable_range_extraction_for_not_in(enable_not_in_range))) { LOG_WARN("failed to check not in range enabled", K(ret)); + } else if (OB_FAIL(query_ctx->get_global_hint().opt_params_.get_bool_opt_param(ObOptParamHint::ENABLE_RANGE_EXTRACTION_FOR_NOT_IN, enable_not_in_range))) { + LOG_WARN("fail to check opt param not in range enabled", K(ret)); } else if (!enable_not_in_range || r_expr->get_param_count() > MAX_NOT_IN_SIZE || l_expr->get_expr_type() == T_OP_ROW) { // do not extract range: 1. not in range is disabled; 2. not in size over MAX_NOT_IN_SIZE @@ -10008,6 +10017,7 @@ int ObQueryRange::get_geo_coveredby_keypart(uint32_t input_srid, const ObSrsBoundsItem *srs_bound = NULL; ObS2Adapter *s2object = NULL; ObExecContext *exec_ctx = NULL; + ObQueryCtx *query_ctx = NULL; ObString buffer_geo; if ((input_srid != 0) && OB_FAIL(OTSRS_MGR->get_tenant_srs_guard(srs_guard))) { @@ -10051,6 +10061,7 @@ int ObQueryRange::get_geo_coveredby_keypart(uint32_t input_srid, if (NULL != query_range_ctx_) { query_range_ctx_->cur_expr_is_precise_ = false; exec_ctx = query_range_ctx_->exec_ctx_; + query_ctx = query_range_ctx_->query_ctx_; } ObKeyPart *head = nullptr; ObKeyPart *last = nullptr; @@ -10136,7 +10147,7 @@ int ObQueryRange::get_geo_coveredby_keypart(uint32_t input_srid, ret = OB_ALLOCATE_MEMORY_FAILED; LOG_ERROR("alloc query range context failed"); } else { - query_range_ctx_ = new(ptr) ObQueryRangeCtx(exec_ctx, NULL, NULL); + query_range_ctx_ = new(ptr) ObQueryRangeCtx(exec_ctx, query_ctx, NULL, NULL); } // copy temp_result to out_key_part diff --git a/src/sql/rewrite/ob_query_range.h b/src/sql/rewrite/ob_query_range.h index bac6844e4..40d3d28ee 100644 --- a/src/sql/rewrite/ob_query_range.h +++ b/src/sql/rewrite/ob_query_range.h @@ -87,6 +87,7 @@ private: struct ObQueryRangeCtx { ObQueryRangeCtx(ObExecContext *exec_ctx, + ObQueryCtx *query_ctx, ExprConstrantArray *expr_constraints, const ParamsIArray *params) : need_final_extract_(false), @@ -96,6 +97,7 @@ private: ignore_calc_failure_(false), range_optimizer_max_mem_size_(100*1024*1024), exec_ctx_(exec_ctx), + query_ctx_(query_ctx), expr_constraints_(expr_constraints), params_(params), use_in_optimization_(false), @@ -119,6 +121,7 @@ private: int64_t range_optimizer_max_mem_size_; common::ObSEArray precise_range_exprs_; ObExecContext *exec_ctx_; + ObQueryCtx *query_ctx_; ExprConstrantArray *expr_constraints_; const ParamsIArray *params_; common::ObSEArray final_exprs_; @@ -387,6 +390,7 @@ public: const ObRawExpr *expr_root, const common::ObDataTypeCastParams &dtc_params, ObExecContext *exec_ctx, + ObQueryCtx *query_ctx, ExprConstrantArray *expr_constraints = NULL, const ParamsIArray *params = NULL, const bool use_in_optimization = false, @@ -409,6 +413,7 @@ public: const ExprIArray &root_exprs, const common::ObDataTypeCastParams &dtc_params, ObExecContext *exec_ctx, + ObQueryCtx *query_ctx, ExprConstrantArray *expr_constraints = NULL, const ParamsIArray *params = NULL, const bool phy_rowid_for_table_loc = false, @@ -529,6 +534,7 @@ private: int init_query_range_ctx(common::ObIAllocator &allocator, const ColumnIArray &range_columns, ObExecContext *exec_ctx, + ObQueryCtx *query_ctx, ExprConstrantArray *expr_constraints, const ParamsIArray *params, const bool phy_rowid_for_table_loc, diff --git a/src/sql/rewrite/ob_stmt_comparer.cpp b/src/sql/rewrite/ob_stmt_comparer.cpp index 618331619..8c51c6043 100644 --- a/src/sql/rewrite/ob_stmt_comparer.cpp +++ b/src/sql/rewrite/ob_stmt_comparer.cpp @@ -1396,12 +1396,16 @@ int ObStmtComparer::compare_basic_table_item(const ObDMLStmt *first, || OB_ISNULL(second) || OB_ISNULL(second_table)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("param has null", K(first), K(first_table), K(second), K(second_table)); - } else if ((first_table->is_basic_table() || first_table->is_link_table()) && - (second_table->is_basic_table() || second_table->is_link_table()) && - first_table->ref_id_ == second_table->ref_id_ && - first_table->flashback_query_type_ == second_table->flashback_query_type_ && - (first_table->flashback_query_expr_ == second_table->flashback_query_expr_ || - first_table->flashback_query_expr_->same_as(*second_table->flashback_query_expr_))) { + } else if ((first_table->is_basic_table() || first_table->is_link_table()) + && (second_table->is_basic_table() || second_table->is_link_table()) + && first_table->ref_id_ == second_table->ref_id_ + && first_table->flashback_query_type_ == second_table->flashback_query_type_ + && (first_table->flashback_query_expr_ == second_table->flashback_query_expr_ + || first_table->flashback_query_expr_->same_as(*second_table->flashback_query_expr_)) + && ((first_table->sample_info_ == NULL && second_table->sample_info_ == NULL) + || (first_table->sample_info_ != NULL && second_table->sample_info_ != NULL + && first_table->sample_info_->same_as(*second_table->sample_info_)))) { + // if sample info is not null the seed != 1 && seed is same then sample info is same if (OB_LIKELY(first_table->access_all_part() && second_table->access_all_part())) { relation = QueryRelation::QUERY_EQUAL; } else if (first_table->access_all_part()) { diff --git a/src/sql/rewrite/ob_transform_groupby_pullup.cpp b/src/sql/rewrite/ob_transform_groupby_pullup.cpp index caebe79b4..8d797af45 100644 --- a/src/sql/rewrite/ob_transform_groupby_pullup.cpp +++ b/src/sql/rewrite/ob_transform_groupby_pullup.cpp @@ -304,14 +304,13 @@ int ObTransformGroupByPullup::check_groupby_pullup_validity(ObDMLStmt *stmt, ObString dummy_str; const ObViewMergeHint *myhint = NULL; ObSQLSessionInfo *session_info = NULL; - bool enable_group_by_placement_transform = false; + ObQueryCtx *query_ctx = NULL; OPT_TRACE("try", table); if (OB_ISNULL(ctx_) || - OB_ISNULL(session_info = ctx_->session_info_)) { + OB_ISNULL(session_info = ctx_->session_info_) || + OB_ISNULL(query_ctx = stmt->get_query_ctx())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect null param", K(ctx_), K(ret)); - } else if (OB_FAIL(session_info->is_groupby_placement_transformation_enabled(enable_group_by_placement_transform))) { - LOG_WARN("failed to check group by placement transform enabled", K(ret)); + LOG_WARN("unexpect null param", K(ctx_), K(session_info), K(query_ctx), K(ret)); } else if (OB_ISNULL(sub_stmt = table->ref_query_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid generated table item", K(ret), K(*table)); @@ -321,7 +320,7 @@ int ObTransformGroupByPullup::check_groupby_pullup_validity(ObDMLStmt *stmt, // can not set is_valid as false, may pullup other table OPT_TRACE("hint reject transform"); } else if (OB_FALSE_IT(myhint = static_cast(sub_stmt->get_stmt_hint().get_normal_hint(T_MERGE_HINT)))) { - } else if (!enable_group_by_placement_transform && (NULL == myhint || myhint->enable_no_group_by_pull_up())) { + } else if (!ctx_->is_groupby_placement_enabled_ && (NULL == myhint || myhint->enable_no_group_by_pull_up())) { OPT_TRACE("system var disable group by placemebt"); } else if (ignore_tables.has_member(stmt->get_table_bit_index(table->table_id_))) { // skip the generated table diff --git a/src/sql/rewrite/ob_transform_groupby_pushdown.cpp b/src/sql/rewrite/ob_transform_groupby_pushdown.cpp index e247a3f00..fda8bdd7c 100644 --- a/src/sql/rewrite/ob_transform_groupby_pushdown.cpp +++ b/src/sql/rewrite/ob_transform_groupby_pushdown.cpp @@ -663,12 +663,13 @@ int ObTransformGroupByPushdown::do_groupby_push_down(ObSelectStmt *stmt, ObSqlBitSet<> outer_table_set; trans_happend = false; ObSQLSessionInfo *session_info = NULL; - bool enable_group_by_placement_transform = false; + ObQueryCtx *query_ctx = NULL; if (OB_ISNULL(stmt) || OB_ISNULL(ctx_) || OB_ISNULL(ctx_->stmt_factory_) || OB_ISNULL(ctx_->expr_factory_) || - OB_ISNULL(session_info = ctx_->session_info_)) { + OB_ISNULL(session_info = ctx_->session_info_) || + OB_ISNULL(query_ctx = stmt->get_query_ctx())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("params are invalid", K(ret), K(ctx_), K(stmt)); + LOG_WARN("params are invalid", K(ret), K(ctx_), K(stmt), K(query_ctx)); } else if (OB_FAIL(ctx_->stmt_factory_->create_stmt(trans_stmt))) { LOG_WARN("failed to create stmt", K(ret)); } else if (OB_FAIL(trans_stmt->deep_copy(*ctx_->stmt_factory_, @@ -729,8 +730,6 @@ int ObTransformGroupByPushdown::do_groupby_push_down(ObSelectStmt *stmt, LOG_TRACE("push down params", K(ret)); bool hint_force_pushdown = false; if (OB_FAIL(ret) || !is_valid) { - } else if (OB_FAIL(session_info->is_groupby_placement_transformation_enabled(enable_group_by_placement_transform))) { - LOG_WARN("failed to check group by placement transform enabled", K(ret)); } else if (OB_FAIL(check_hint_valid(static_cast(*stmt), params, hint_force_pushdown, @@ -738,7 +737,7 @@ int ObTransformGroupByPushdown::do_groupby_push_down(ObSelectStmt *stmt, LOG_WARN("check hint failed", K(ret)); } else if (!is_valid) { OPT_TRACE("hint disable group by pushdown"); - } else if (!enable_group_by_placement_transform && !hint_force_pushdown) { + } else if (!ctx_->is_groupby_placement_enabled_ && !hint_force_pushdown) { OPT_TRACE("system variable disable group by pushdown"); } else if (OB_FAIL(transform_groupby_push_down(trans_stmt, flattern_joined_tables, diff --git a/src/sql/rewrite/ob_transform_join_elimination.cpp b/src/sql/rewrite/ob_transform_join_elimination.cpp index bb23928ae..2ea98d9d3 100644 --- a/src/sql/rewrite/ob_transform_join_elimination.cpp +++ b/src/sql/rewrite/ob_transform_join_elimination.cpp @@ -1545,6 +1545,9 @@ int ObTransformJoinElimination::check_transform_validity_foreign_key(const ObDML foreign_key_info, all_primary_key))) { LOG_WARN("failed to check all column primary key", K(ret)); + } else if (source_table->is_has_sample_info()) { + // do nothing + OPT_TRACE("table has sample info", K(source_table->table_id_)); } else if (all_primary_key) { can_be_eliminated = true; } else { @@ -3225,6 +3228,8 @@ int ObTransformJoinElimination::check_transform_validity_foreign_key(const ObDML } else if (OB_FAIL(check_all_column_primary_key(target_stmt, target_table->table_id_, foreign_key_info, all_primary_key))) { LOG_WARN("failed to check all column primary key", K(ret)); + } else if (source_table->is_has_sample_info()) { + OPT_TRACE("table has sample info", K(source_table->table_id_)); } else if (all_primary_key) { can_be_eliminated = true; } else { diff --git a/src/sql/rewrite/ob_transform_or_expansion.cpp b/src/sql/rewrite/ob_transform_or_expansion.cpp index 9b7868a9b..c4dc8e852 100644 --- a/src/sql/rewrite/ob_transform_or_expansion.cpp +++ b/src/sql/rewrite/ob_transform_or_expansion.cpp @@ -3461,6 +3461,8 @@ int ObTransformOrExpansion::check_stmt_valid_for_expansion(ObDMLStmt *stmt, bool } else if (table_item->is_fake_cte_table()) { is_stmt_valid = false; OPT_TRACE("contain fake cte table"); + } else if (table_item->is_has_sample_info()) { + is_stmt_valid = false; } } return ret; diff --git a/src/sql/rewrite/ob_transform_rule.cpp b/src/sql/rewrite/ob_transform_rule.cpp index 5b50d8ab7..5df4bdd3b 100644 --- a/src/sql/rewrite/ob_transform_rule.cpp +++ b/src/sql/rewrite/ob_transform_rule.cpp @@ -65,6 +65,9 @@ void ObTransformerCtx::reset() outline_trans_hints_.reset(); used_trans_hints_.reset(); groupby_pushdown_stmts_.reset(); + is_groupby_placement_enabled_ = true; + is_force_inline_ = false; + is_force_materialize_ = false; is_spm_outline_ = false; push_down_filters_.reset(); iteration_level_ = 0; diff --git a/src/sql/rewrite/ob_transform_rule.h b/src/sql/rewrite/ob_transform_rule.h index c541299b2..63b131f9f 100644 --- a/src/sql/rewrite/ob_transform_rule.h +++ b/src/sql/rewrite/ob_transform_rule.h @@ -68,6 +68,9 @@ struct ObTransformerCtx outline_trans_hints_(), used_trans_hints_(), groupby_pushdown_stmts_(), + is_groupby_placement_enabled_(true), + is_force_inline_(false), + is_force_materialize_(false), is_spm_outline_(false), push_down_filters_(), in_accept_transform_(false), @@ -130,6 +133,12 @@ struct ObTransformerCtx ObSEArray used_trans_hints_; ObSEArray groupby_pushdown_stmts_; /* end used for hint and outline below */ + + // used for transform parameters + bool is_groupby_placement_enabled_; + bool is_force_inline_; + bool is_force_materialize_; + bool is_spm_outline_; ObSEArray push_down_filters_; bool in_accept_transform_; diff --git a/src/sql/rewrite/ob_transform_temp_table.cpp b/src/sql/rewrite/ob_transform_temp_table.cpp index 4ac67b07d..e05edaf52 100644 --- a/src/sql/rewrite/ob_transform_temp_table.cpp +++ b/src/sql/rewrite/ob_transform_temp_table.cpp @@ -140,7 +140,6 @@ int ObTransformTempTable::generate_with_clause(ObDMLStmt *&stmt, bool &trans_hap hash::ObHashMap parent_map; trans_happened = false; bool enable_temp_table_transform = false; - bool force_temp_table_inline = false; bool has_hint = false; bool is_hint_enabled = false; ObSQLSessionInfo *session_info = NULL; @@ -150,8 +149,6 @@ int ObTransformTempTable::generate_with_clause(ObDMLStmt *&stmt, bool &trans_hap LOG_WARN("unexpect null param", K(ctx_), K(ret)); } else if (OB_FAIL(session_info->is_temp_table_transformation_enabled(enable_temp_table_transform))) { LOG_WARN("failed to check temp table transform enabled", K(ret)); - } else if (OB_FAIL(session_info->is_force_temp_table_inline(force_temp_table_inline))) { - LOG_WARN("failed to check temp table force inline", K(ret)); } else if (OB_FAIL(stmt->get_query_ctx()->get_global_hint().opt_params_.get_bool_opt_param( ObOptParamHint::XSOLAPI_GENERATE_WITH_CLAUSE, is_hint_enabled, has_hint))) { LOG_WARN("failed to check has opt param", K(ret)); @@ -161,7 +158,7 @@ int ObTransformTempTable::generate_with_clause(ObDMLStmt *&stmt, bool &trans_hap if (OB_FAIL(ret)) { } else if (ctx_->is_set_stmt_oversize_) { OPT_TRACE("stmt containt oversize set stmt"); - } else if (!enable_temp_table_transform || force_temp_table_inline) { + } else if (!enable_temp_table_transform || ctx_->is_force_inline_) { OPT_TRACE("session variable disable temp table transform"); } else if (stmt->has_for_update()) { OPT_TRACE("stmt has for update, can not extract CTE"); @@ -192,18 +189,12 @@ int ObTransformTempTable::expand_temp_table(ObIArray &temp_table_ { int ret = OB_SUCCESS; trans_happened = false; - bool system_force_inline_cte = false; - bool system_force_materialize_cte = false; ObSQLSessionInfo *session_info = NULL; if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->stmt_factory_) || OB_ISNULL(ctx_->allocator_) || OB_ISNULL(ctx_->expr_factory_) || OB_ISNULL(session_info = ctx_->session_info_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpect null param", K(ctx_), K(ret)); - } else if (OB_FAIL(session_info->is_force_temp_table_inline(system_force_inline_cte))) { - LOG_WARN("failed to check temp table force inline", K(ret)); - } else if (OB_FAIL(session_info->is_force_temp_table_materialize(system_force_materialize_cte))) { - LOG_WARN("failed to check temp table force materialize", K(ret)); } for (int64_t i = 0; OB_SUCC(ret) && i < temp_table_info.count(); ++i) { TempTableInfo &helper = temp_table_info.at(i); @@ -244,10 +235,10 @@ int ObTransformTempTable::expand_temp_table(ObIArray &temp_table_ } else if (is_oversize_stmt) { //do nothing OPT_TRACE("CTE too large to expand"); - } else if (system_force_materialize_cte) { + } else if (ctx_->is_force_materialize_) { //do nothing OPT_TRACE("system variable force materialize CTE"); - } else if (system_force_inline_cte) { + } else if (ctx_->is_force_inline_) { need_expand = true; OPT_TRACE("system variable force inline CTE"); } else if (1 == helper.table_items_.count()) { diff --git a/src/sql/rewrite/ob_transform_utils.cpp b/src/sql/rewrite/ob_transform_utils.cpp index dc014fe4c..9b602285a 100644 --- a/src/sql/rewrite/ob_transform_utils.cpp +++ b/src/sql/rewrite/ob_transform_utils.cpp @@ -3484,8 +3484,10 @@ int ObTransformUtils::check_index_extract_query_range(const ObDMLStmt *stmt, void *tmp_ptr = NULL; ObQueryRange *query_range = NULL; const ParamStore *params = NULL; + ObQueryCtx *query_ctx = NULL; if (OB_ISNULL(stmt) || OB_ISNULL(ctx) || OB_ISNULL(ctx->exec_ctx_) - || OB_ISNULL(ctx->exec_ctx_->get_physical_plan_ctx())) { + || OB_ISNULL(ctx->exec_ctx_->get_physical_plan_ctx()) + || OB_ISNULL(query_ctx = stmt->get_query_ctx())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("params have null", K(ret), K(stmt), K(ctx)); } else { @@ -3505,6 +3507,7 @@ int ObTransformUtils::check_index_extract_query_range(const ObDMLStmt *stmt, predicate_exprs, dtc_params, ctx->exec_ctx_, + query_ctx, NULL, params))) { LOG_WARN("failed to extract query range", K(ret)); diff --git a/src/sql/rewrite/ob_transform_view_merge.cpp b/src/sql/rewrite/ob_transform_view_merge.cpp index 90d9ad664..2e575edb7 100644 --- a/src/sql/rewrite/ob_transform_view_merge.cpp +++ b/src/sql/rewrite/ob_transform_view_merge.cpp @@ -325,10 +325,6 @@ int ObTransformViewMerge::do_view_merge_for_semi_right_table(ObDMLStmt *parent_s } else if (OB_FAIL(append(parent_stmt->get_check_constraint_items(), child_stmt->get_check_constraint_items()))) { LOG_WARN("failed to append check constraint items", K(ret)); - } else if (parent_stmt->is_select_stmt() && - OB_FAIL(append(static_cast(parent_stmt)->get_sample_infos(), - child_stmt->get_sample_infos()))) { - LOG_WARN("failed to append exprs", K(ret)); } else if (OB_FAIL(ObTransformUtils::adjust_pseudo_column_like_exprs(*parent_stmt))) { LOG_WARN("failed to adjust pseudo column like exprs", K(ret)); } else if (OB_FAIL(parent_stmt->rebuild_tables_hash())) { @@ -1117,10 +1113,6 @@ int ObTransformViewMerge::do_view_merge(ObDMLStmt *parent_stmt, } else if (OB_FAIL(append(parent_stmt->get_check_constraint_items(), child_stmt->get_check_constraint_items()))) { LOG_WARN("failed to append check constraint items", K(ret)); - } else if (parent_stmt->is_select_stmt() && - OB_FAIL(append(static_cast(parent_stmt)->get_sample_infos(), - child_stmt->get_sample_infos()))) { - LOG_WARN("failed to append exprs", K(ret)); } else if (OB_FAIL(ObTransformUtils::pull_up_subquery(parent_stmt, child_stmt))) { LOG_WARN("failed to pull up subquery", K(ret)); } else if (OB_FAIL(parent_stmt->remove_from_item(table_item->table_id_))) { diff --git a/src/sql/rewrite/ob_transformer_impl.cpp b/src/sql/rewrite/ob_transformer_impl.cpp index 5c1ede836..b0a4a6a8b 100644 --- a/src/sql/rewrite/ob_transformer_impl.cpp +++ b/src/sql/rewrite/ob_transformer_impl.cpp @@ -68,6 +68,8 @@ int ObTransformerImpl::transform(ObDMLStmt *&stmt) if (OB_ISNULL(stmt)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret)); + } else if (OB_FAIL(set_transformation_parameters(stmt->get_query_ctx()))) { + LOG_WARN("failed to extract trans ctx param", K(ret)); } else if (OB_FAIL(do_transform_dblink_write(stmt, trans_happended))) { LOG_WARN("failed to do transform dblink write", K(ret)); } else if (trans_happended) { @@ -94,6 +96,39 @@ int ObTransformerImpl::transform(ObDMLStmt *&stmt) return ret; } +int ObTransformerImpl::set_transformation_parameters(ObQueryCtx *query_ctx) +{ + int ret = OB_SUCCESS; + ObSQLSessionInfo *session_info = NULL; + bool enable_group_by_placement_transform = false; + bool opt_param_exists = false; + int64_t opt_param_val = 0; + if (OB_ISNULL(query_ctx) || OB_ISNULL(ctx_) || OB_ISNULL(session_info = ctx_->session_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(query_ctx), K(ctx_)); + } else if (OB_FAIL(session_info->is_groupby_placement_transformation_enabled(enable_group_by_placement_transform))) { + LOG_WARN("failed to check group by placement transform enabled", K(ret)); + } else if (OB_FAIL(query_ctx->get_global_hint().opt_params_.get_bool_opt_param(ObOptParamHint::OPTIMIZER_GROUP_BY_PLACEMENT, enable_group_by_placement_transform))) { + LOG_WARN("fail to check opt param group by placement", K(ret)); + } else { + ctx_->is_groupby_placement_enabled_ = enable_group_by_placement_transform; + } + + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(query_ctx->get_global_hint().opt_params_.get_integer_opt_param(ObOptParamHint::WITH_SUBQUERY, opt_param_val, opt_param_exists))) { + LOG_WARN("fail to check opt param with subquery", K(ret)); + } else if (opt_param_exists) { + ctx_->is_force_inline_ = 2 == opt_param_val; + ctx_->is_force_materialize_ = 1 == opt_param_val; + } else if (OB_FAIL(session_info->is_force_temp_table_inline(ctx_->is_force_inline_))) { + LOG_WARN("failed to check temp table force inline", K(ret)); + } else if (OB_FAIL(session_info->is_force_temp_table_materialize(ctx_->is_force_materialize_))) { + LOG_WARN("failed to check temp table force materialize", K(ret)); + } + return ret; +} + int ObTransformerImpl::get_stmt_trans_info(ObDMLStmt *stmt, bool is_root) { int ret = OB_SUCCESS; diff --git a/src/sql/rewrite/ob_transformer_impl.h b/src/sql/rewrite/ob_transformer_impl.h index 1b011d36f..ac25688f9 100644 --- a/src/sql/rewrite/ob_transformer_impl.h +++ b/src/sql/rewrite/ob_transformer_impl.h @@ -153,6 +153,7 @@ public: static int check_stmt_functions(const ObDMLStmt *stmt, StmtFunc &func); int check_temp_table_functions(ObDMLStmt *stmt, StmtFunc &func); inline ObTransformerCtx *get_trans_ctx() { return ctx_; } + int set_transformation_parameters(ObQueryCtx *query_ctx); private: int collect_trans_stat(const ObTransformRule &rule); diff --git a/src/sql/session/ob_basic_session_info.cpp b/src/sql/session/ob_basic_session_info.cpp index 00c995d8b..e6c8bfc19 100644 --- a/src/sql/session/ob_basic_session_info.cpp +++ b/src/sql/session/ob_basic_session_info.cpp @@ -3776,6 +3776,18 @@ int ObBasicSessionInfo::is_serial_set_order_forced(bool &force_set_order, bool i return ret; } +int ObBasicSessionInfo::is_old_charset_aggregation_enabled(bool &is_enable) const +{ + int ret = OB_SUCCESS; + is_enable = false; + if (lib::is_oracle_mode()) { + //do nothing + } else { + ret = get_bool_sys_var(SYS_VAR__ENABLE_OLD_CHARSET_AGGREGATION, is_enable); + } + return ret; +} + int ObBasicSessionInfo::is_storage_estimation_enabled(bool &storage_estimation_enabled) const { int ret = OB_SUCCESS; @@ -5619,6 +5631,11 @@ int ObBasicSessionInfo::get_collation_server(ObCollationType &collation_server) return get_collation_sys_var(SYS_VAR_COLLATION_SERVER, collation_server); } +int ObBasicSessionInfo::get_default_collation_for_utf8mb4(ObCollationType &collation_server) const +{ + return get_collation_sys_var(SYS_VAR_DEFAULT_COLLATION_FOR_UTF8MB4, collation_server); +} + int ObBasicSessionInfo::get_capture_plan_baseline(bool &v) const { v = sys_vars_cache_.get_optimizer_capture_sql_plan_baselines(); diff --git a/src/sql/session/ob_basic_session_info.h b/src/sql/session/ob_basic_session_info.h index 11ef06b2a..7e2220e6f 100644 --- a/src/sql/session/ob_basic_session_info.h +++ b/src/sql/session/ob_basic_session_info.h @@ -683,6 +683,7 @@ public: } int get_collation_database(common::ObCollationType &collation_database) const; int get_collation_server(common::ObCollationType &collation_server) const; + int get_default_collation_for_utf8mb4(ObCollationType &collation_server) const; int get_foreign_key_checks(int64_t &foreign_key_checks) const { foreign_key_checks = sys_vars_cache_.get_foreign_key_checks(); @@ -1060,6 +1061,7 @@ public: int get_query_rewrite_enabled(int64_t &query_rewrite_enabled) const; int get_query_rewrite_integrity(int64_t &query_rewrite_integrity) const; int is_serial_set_order_forced(bool &force_set_order, bool is_oracle_mode) const; + int is_old_charset_aggregation_enabled(bool &is_enable) const; int is_storage_estimation_enabled(bool &storage_estimation_enabled) const; bool is_use_trace_log() const { diff --git a/src/sql/session/ob_sql_session_info.cpp b/src/sql/session/ob_sql_session_info.cpp index a46d1d3ad..a9d167a11 100644 --- a/src/sql/session/ob_sql_session_info.cpp +++ b/src/sql/session/ob_sql_session_info.cpp @@ -60,6 +60,9 @@ #include "share/schema/ob_schema_utils.h" #include "share/config/ob_config_helper.h" #include "rootserver/ob_tenant_info_loader.h" +#ifdef OB_BUILD_AUDIT_SECURITY +#include "sql/audit/ob_audit_log_utils.h" +#endif using namespace oceanbase::sql; using namespace oceanbase::common; @@ -3006,6 +3009,14 @@ void ObSQLSessionInfo::ObCachedTenantConfigInfo::refresh() K(effective_tenant_id), K(lbt())); } +#ifdef OB_BUILD_AUDIT_SECURITY + audit_log_enable_ = tenant_config->audit_log_enable; + if (OB_SUCCESS != (tmp_ret = ObAuditLogUtils::parse_query_sql_conf( + tenant_config->audit_log_query_sql.str(), + audit_log_query_sql_))) { + LOG_WARN_RET(tmp_ret, "fail parse query sql conf", "ret", tmp_ret, K(lbt())); + } +#endif // 6. enable extended SQL syntax in the MySQL mode enable_sql_extension_ = tenant_config->enable_sql_extension; px_join_skew_handling_ = tenant_config->_px_join_skew_handling; @@ -4748,3 +4759,14 @@ int ObSQLSessionInfo::check_service_name_and_failover_mode() const uint64_t tenant_id = get_effective_tenant_id(); return check_service_name_and_failover_mode(tenant_id); } + +int ObSQLSessionInfo::set_audit_filter_name(const common::ObString &filter_name) +{ + int ret = OB_SUCCESS; + if (filter_name.empty()) { + audit_filter_name_.reset(); + } else if (OB_FAIL(sess_level_name_pool_.write_string(filter_name, &audit_filter_name_))) { + LOG_WARN("failed to write filter_name to string_buf_", K(ret)); + } + return ret; +} diff --git a/src/sql/session/ob_sql_session_info.h b/src/sql/session/ob_sql_session_info.h index 5f89af9b3..3bb79ed50 100644 --- a/src/sql/session/ob_sql_session_info.h +++ b/src/sql/session/ob_sql_session_info.h @@ -698,6 +698,8 @@ public: px_join_skew_handling_(true), px_join_skew_minfreq_(30), at_type_(ObAuditTrailType::NONE), + audit_log_enable_(false), + audit_log_query_sql_(0), sort_area_size_(128*1024*1024), hash_area_size_(128*1024*1024), data_version_(0), @@ -720,6 +722,8 @@ public: bool get_enable_bloom_filter() const { return enable_bloom_filter_; } bool get_enable_sql_extension() const { return enable_sql_extension_; } ObAuditTrailType get_at_type() const { return at_type_; } + bool enable_audit_log() const { return audit_log_enable_; } + int64_t get_audit_log_query_sql() const { return audit_log_query_sql_; } int64_t get_sort_area_size() const { return ATOMIC_LOAD(&sort_area_size_); } int64_t get_hash_area_size() const { return ATOMIC_LOAD(&hash_area_size_); } uint64_t get_data_version() const { return ATOMIC_LOAD(&data_version_); } @@ -743,6 +747,8 @@ public: bool px_join_skew_handling_; int64_t px_join_skew_minfreq_; ObAuditTrailType at_type_; + bool audit_log_enable_; + int64_t audit_log_query_sql_; int64_t sort_area_size_; int64_t hash_area_size_; uint64_t data_version_; @@ -1250,6 +1256,8 @@ public: int64_t get_current_dblink_sequence_id() const { return current_dblink_sequence_id_; } void set_client_non_standard(bool client_non_standard) { client_non_standard_ = client_non_standard; } bool client_non_standard() { return client_non_standard_; } + int set_audit_filter_name(const common::ObString &filter_name); + const common::ObString &get_audit_filter_name() const { return audit_filter_name_; } int get_mem_ctx_alloc(common::ObIAllocator *&alloc); int update_sess_sync_info(const SessionSyncInfoType sess_sync_info_type, const char *buf, const int64_t length, int64_t &pos); @@ -1407,6 +1415,16 @@ public: cached_tenant_config_info_.refresh(); return cached_tenant_config_info_.get_sql_plan_management_mode(); } + bool enable_audit_log() + { + cached_tenant_config_info_.refresh(); + return cached_tenant_config_info_.enable_audit_log(); + } + int64_t get_audit_log_query_sql() + { + cached_tenant_config_info_.refresh(); + return cached_tenant_config_info_.get_audit_log_query_sql(); + } int get_tmp_table_size(uint64_t &size); int ps_use_stream_result_set(bool &use_stream); void set_proxy_version(uint64_t v) { proxy_version_ = v; } @@ -1730,6 +1748,7 @@ private: dbms_scheduler::ObDBMSSchedJobInfo *job_info_; // dbms_scheduler related. bool failover_mode_; ObServiceNameString service_name_; + common::ObString audit_filter_name_; }; diff --git a/src/sql/session/ob_sql_session_mgr.cpp b/src/sql/session/ob_sql_session_mgr.cpp index 351df6741..277082f65 100644 --- a/src/sql/session/ob_sql_session_mgr.cpp +++ b/src/sql/session/ob_sql_session_mgr.cpp @@ -25,7 +25,10 @@ #include "observer/mysql/obsm_handler.h" #include "rpc/obmysql/obsm_struct.h" #include "observer/ob_server_struct.h" +#ifdef OB_BUILD_AUDIT_SECURITY #include "sql/monitor/ob_security_audit_utils.h" +#include "sql/audit/ob_audit_log_utils.h" +#endif #include "sql/session/ob_user_resource_mgr.h" #include "sql/monitor/flt/ob_flt_control_info_mgr.h" #include "storage/concurrency_control/ob_multi_version_garbage_collector.h" @@ -590,6 +593,7 @@ int ObSQLSessionMgr::free_session(const ObFreeSessionCtx &ctx) ObString::make_string("DISCONNECT"), empty_comment_text, ret); + ObAuditLogUtils::hanlde_connect_audit_log(*sess_info, "LOGOFF"); THIS_WORKER.set_timeout_ts(cur_timeout_backup); } #endif diff --git a/tools/deploy/mysql_test/r/mysql/information_schema.result b/tools/deploy/mysql_test/r/mysql/information_schema.result index 5af54a352..a712732cd 100644 --- a/tools/deploy/mysql_test/r/mysql/information_schema.result +++ b/tools/deploy/mysql_test/r/mysql/information_schema.result @@ -42,7 +42,7 @@ desc information_schema.tables; | AVG_ROW_LENGTH | bigint(21) unsigned | NO | | | | | DATA_LENGTH | bigint(20) unsigned | NO | | | | | MAX_DATA_LENGTH | bigint(0) unsigned | NO | | | | -| INDEX_LENGTH | bigint(0) unsigned | NO | | | | +| INDEX_LENGTH | bigint(21) unsigned | NO | | | | | DATA_FREE | bigint(0) unsigned | NO | | | | | AUTO_INCREMENT | bigint(0) unsigned | NO | | | | | CREATE_TIME | datetime | NO | | | | @@ -185,756 +185,766 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys +---------------+--------------------+--------------------------------------------------------+--------------+--------+---------+------------+------------+----------------+-------------+-----------------+--------------+-----------+----------------+---------------------+---------------------+------------+--------------------+----------+----------------+---------------+ | TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | TABLE_TYPE | ENGINE | VERSION | ROW_FORMAT | TABLE_ROWS | AVG_ROW_LENGTH | DATA_LENGTH | MAX_DATA_LENGTH | INDEX_LENGTH | DATA_FREE | AUTO_INCREMENT | CREATE_TIME | UPDATE_TIME | CHECK_TIME | TABLE_COLLATION | CHECKSUM | CREATE_OPTIONS | TABLE_COMMENT | +---------------+--------------------+--------------------------------------------------------+--------------+--------+---------+------------+------------+----------------+-------------+-----------------+--------------+-----------+----------------+---------------------+---------------------+------------+--------------------+----------+----------------+---------------+ -| def | information_schema | CHARACTER_SETS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | CHECK_CONSTRAINTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | COLLATIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | COLLATION_CHARACTER_SET_APPLICABILITY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | COLUMNS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | COLUMN_PRIVILEGES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | CONNECTION_CONTROL_FAILED_LOGIN_ATTEMPTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | ENABLED_ROLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | ENGINES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | EVENTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | FILES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | GLOBAL_STATUS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | GLOBAL_VARIABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_BUFFER_PAGE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_BUFFER_PAGE_LRU | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_BUFFER_POOL_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_CMP | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_CMPMEM | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_CMPMEM_RESET | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_CMP_PER_INDEX | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_CMP_PER_INDEX_RESET | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_CMP_RESET | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_FT_BEING_DELETED | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_FT_CONFIG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_FT_DELETED | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_FT_INDEX_CACHE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_METRICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_SYS_DATAFILES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_SYS_FIELDS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_SYS_FOREIGN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_SYS_FOREIGN_COLS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_SYS_INDEXES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_SYS_TABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_SYS_TABLESPACES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_SYS_TABLESTATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_SYS_VIRTUAL | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_TEMP_TABLE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | KEY_COLUMN_USAGE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | PARAMETERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | PARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | PROCESSLIST | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | PROFILING | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | QUERY_RESPONSE_TIME | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | REFERENTIAL_CONSTRAINTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | ROUTINES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | SCHEMATA | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | SCHEMA_PRIVILEGES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | SESSION_STATUS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | SESSION_VARIABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | ST_GEOMETRY_COLUMNS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | ST_SPATIAL_REFERENCE_SYSTEMS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | TABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | TABLESPACES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | TABLE_CONSTRAINTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | TABLE_PRIVILEGES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | TRIGGERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | USER_PRIVILEGES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | VIEWS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | VIEW_TABLE_USAGE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | columns_priv | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | db | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | default_roles | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | func | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | help_category | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | help_keyword | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | help_relation | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | help_topic | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | proc | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | procs_priv | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | role_edges | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | time_zone | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | time_zone_name | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | time_zone_transition | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | time_zone_transition_type | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | user | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | ALL_OB_EXTERNAL_TABLE_FILES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_DB_LINKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_INDEX_USAGE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_IND_PARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_IND_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_IND_SUBPARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_MVIEWS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_MVIEW_LOGS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_MVREF_CHANGE_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_MVREF_RUN_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_MVREF_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_MVREF_STATS_PARAMS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_MVREF_STATS_SYS_DEFAULTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_MVREF_STMT_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OBJECTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_ACCESS_POINT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_ARCHIVELOG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_ARCHIVELOG_PIECE_FILES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_ARCHIVELOG_SUMMARY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_ARCHIVE_DEST | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_AUTO_INCREMENT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_AUX_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BACKUP_DELETE_JOBS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BACKUP_DELETE_JOB_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BACKUP_DELETE_POLICY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BACKUP_DELETE_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BACKUP_DELETE_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BACKUP_JOBS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BACKUP_JOB_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BACKUP_PARAMETER | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BACKUP_SET_FILES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BACKUP_STORAGE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BACKUP_STORAGE_INFO_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BACKUP_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BACKUP_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BALANCE_JOBS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BALANCE_JOB_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BALANCE_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BALANCE_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_CONCURRENT_LIMIT_SQL | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_DATABASES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_DATABASE_PRIVILEGE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_DATA_DICTIONARY_IN_LOG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_DEADLOCK_EVENT_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_EXTERNAL_TABLE_FILES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_FREEZE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_IMPORT_TABLE_JOBS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_IMPORT_TABLE_JOB_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_IMPORT_TABLE_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_IMPORT_TABLE_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_KV_TTL_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_KV_TTL_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_LOG_RESTORE_SOURCE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_LS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_LS_ARB_REPLICA_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_LS_ARB_REPLICA_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_LS_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_LS_LOCATIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_LS_LOG_ARCHIVE_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_LS_REPLICA_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_LS_REPLICA_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_MAJOR_COMPACTION | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_OUTLINES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_OUTLINE_CONCURRENT_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_RECOVER_TABLE_JOBS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_RECOVER_TABLE_JOB_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_RESTORE_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_RESTORE_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_RSRC_IO_DIRECTIVES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_SEQUENCE_OBJECTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_SERVICES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_SYS_VARIABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TABLEGROUPS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TABLEGROUP_PARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TABLEGROUP_SUBPARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TABLEGROUP_TABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TABLET_REPLICAS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TABLET_TO_LS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TABLE_LOCATIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TABLE_STAT_STALE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TASK_OPT_STAT_GATHER_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TENANTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TENANT_EVENT_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TRANSFER_PARTITION_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TRANSFER_PARTITION_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TRANSFER_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TRANSFER_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_USERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_USER_DEFINED_RULES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_ZONE_MAJOR_COMPACTION | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_PART_COL_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_PART_HISTOGRAMS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_PART_INDEXES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_PART_KEY_COLUMNS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_PART_TABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_RECYCLEBIN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_RSRC_CONSUMER_GROUPS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_RSRC_GROUP_MAPPINGS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_RSRC_PLANS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_RSRC_PLAN_DIRECTIVES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_SCHEDULER_JOBS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_SCHEDULER_JOB_RUN_DETAILS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_SCHEDULER_WINDOWS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_SEQUENCES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_SQL_MANAGEMENT_CONFIG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_SQL_PLAN_BASELINES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_SUBPARTITION_TEMPLATES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_SUBPART_COL_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_SUBPART_HISTOGRAMS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_SUBPART_KEY_COLUMNS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_TAB_COL_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_TAB_HISTOGRAMS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_TAB_MODIFICATIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_TAB_PARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_TAB_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_TAB_STATS_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_TAB_SUBPARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_WR_ACTIVE_SESSION_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_WR_CONTROL | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_WR_SNAPSHOT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_WR_STATNAME | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_WR_SYSSTAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$ACTIVE_SESSION_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$DML_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$LATCH | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_ARBITRATION_MEMBER_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_ARBITRATION_SERVICE_STATUS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_CGROUP_CONFIG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_COMPACTION_DIAGNOSE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_COMPACTION_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_COMPACTION_SUGGESTIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_DTL_INTERM_RESULT_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_FLT_TRACE_CONFIG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_KVCACHE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_KV_CONNECTIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_LOCKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_LOG_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_LS_SNAPSHOTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_MEMORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_MEMSTORE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_MEMSTORE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_MERGE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_NIC_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_OPT_STAT_GATHER_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_PARAMETERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_PLAN_CACHE_PLAN_EXPLAIN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_PLAN_CACHE_PLAN_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_PLAN_CACHE_REFERENCE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_PLAN_CACHE_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_PL_CACHE_OBJECT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_PROCESSLIST | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_PS_ITEM_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_PS_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_PX_P2P_DATAHUB | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_PX_TARGET_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_PX_WORKER_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_RPC_INCOMING | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_RPC_OUTGOING | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_SERVER_SCHEMA_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_SESSION | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_SESSION_PS_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_SQL_AUDIT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_SQL_PLAN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_SQL_WORKAREA_MEMORY_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_SSTABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_TABLET_COMPACTION_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_TABLET_COMPACTION_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_TABLET_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_TENANT_MEMORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_TENANT_RESOURCE_LIMIT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_TENANT_RESOURCE_LIMIT_DETAIL | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_THREAD | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_TRACEPOINT_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_TRANSACTION_PARTICIPANTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_TRANSACTION_SCHEDULERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_UNITS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$SESSION_EVENT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$SESSION_LONGOPS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$SESSION_WAIT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$SESSION_WAIT_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$SESSTAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$SQL_JOIN_FILTER | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$SQL_PLAN_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$SQL_WORKAREA | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$SQL_WORKAREA_ACTIVE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$SQL_WORKAREA_HISTOGRAM | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$SYSSTAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$SYSTEM_EVENT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$ACTIVE_SESSION_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$DML_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$ENCRYPTED_TABLESPACES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$EVENT_NAME | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$LATCH | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_ARBITRATION_MEMBER_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_ARBITRATION_SERVICE_STATUS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_ARCHIVE_DEST_STATUS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_CGROUP_CONFIG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_COMPACTION_DIAGNOSE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_COMPACTION_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_COMPACTION_SUGGESTIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_COMPATIBILITY_CONTROL | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_DTL_INTERM_RESULT_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_ENCRYPTED_TABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_KVCACHE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_KV_CONNECTIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_LOCKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_LOG_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_LS_LOG_RESTORE_STATUS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_LS_REPLICA_TASK_PLAN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_LS_SNAPSHOTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_MEMORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_MEMSTORE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_MEMSTORE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_MERGE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_NIC_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_OPT_STAT_GATHER_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_PARAMETERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_PLAN_CACHE_PLAN_EXPLAIN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_PLAN_CACHE_PLAN_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_PLAN_CACHE_REFERENCE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_PLAN_CACHE_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_PL_CACHE_OBJECT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_PROCESSLIST | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_PS_ITEM_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_PS_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_PX_P2P_DATAHUB | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_PX_TARGET_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_PX_WORKER_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_RPC_INCOMING | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_RPC_OUTGOING | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_SERVER_SCHEMA_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_SESSION | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_SESSION_PS_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_SQL_AUDIT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_SQL_PLAN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_SQL_WORKAREA_MEMORY_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_SSTABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_TABLET_COMPACTION_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_TABLET_COMPACTION_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_TABLET_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_TENANT_MEMORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_TENANT_RESOURCE_LIMIT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_TENANT_RESOURCE_LIMIT_DETAIL | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_THREAD | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_TIMESTAMP_SERVICE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_TRACEPOINT_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_TRANSACTION_PARTICIPANTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_TRANSACTION_SCHEDULERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_UNITS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$RSRC_PLAN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$SESSION_EVENT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$SESSION_LONGOPS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$SESSION_WAIT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$SESSION_WAIT_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$SESSTAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$SQL_JOIN_FILTER | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$SQL_MONITOR_STATNAME | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$SQL_PLAN_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$SQL_WORKAREA | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$SQL_WORKAREA_ACTIVE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$SQL_WORKAREA_HISTOGRAM | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$STATNAME | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$SYSSTAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$SYSTEM_EVENT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __ALL_VIRTUAL_INFORMATION_COLUMNS | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_acquired_snapshot | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_auto_increment | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_aux_stat | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_balance_job | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_balance_job_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_balance_task | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_balance_task_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_client_to_server_session_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_coll_type | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_coll_type_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_column | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_column_group | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_column_group_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_column_group_mapping | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_column_group_mapping_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_column_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_column_privilege | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_column_privilege_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_column_stat | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_column_stat_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_column_usage | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_constraint | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_constraint_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_context | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_context_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_core_table | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_dam_cleanup_jobs | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_dam_last_arch_ts | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_data_dictionary_in_log | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_database | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_database_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_database_privilege | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_database_privilege_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_dblink | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_dblink_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_dbms_lock_allocated | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_ddl_checksum | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_ddl_error_message | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_ddl_id | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_ddl_operation | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_ddl_task_status | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_def_sub_part | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_def_sub_part_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_detect_lock_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_dummy | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_external_table_file | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_foreign_key | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_foreign_key_column | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_foreign_key_column_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_foreign_key_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_freeze_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_func | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_func_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_histogram_stat | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_histogram_stat_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_index_usage_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_job | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_job_log | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_ls | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_mlog | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_mock_fk_parent_table | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_mock_fk_parent_table_column | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_mock_fk_parent_table_column_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_mock_fk_parent_table_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_monitor_modified | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_mview | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_mview_dep | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_mview_refresh_change_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_mview_refresh_run_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_mview_refresh_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_mview_refresh_stats_params | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_mview_refresh_stats_sys_defaults | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_mview_refresh_stmt_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_ncomp_dll | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_optstat_global_prefs | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_optstat_user_prefs | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_ori_schema_version | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_outline | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_outline_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_package | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_package_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_part | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_part_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_part_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_part_info_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_pending_transaction | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_plan_baseline | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_plan_baseline_item | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_recyclebin | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_res_mgr_consumer_group | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_res_mgr_directive | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_res_mgr_mapping_rule | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_res_mgr_plan | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_rls_attribute | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_rls_attribute_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_rls_context | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_rls_context_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_rls_group | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_rls_group_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_rls_policy | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_rls_policy_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_rls_security_column | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_rls_security_column_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_routine | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_routine_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_routine_param | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_routine_param_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_routine_privilege | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_routine_privilege_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_scheduler_job_run_detail_v2 | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_sequence_object | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_sequence_object_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_sequence_value | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_spatial_reference_systems | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_spm_config | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_sub_part | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_sub_part_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_synonym | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_synonym_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_sys_stat | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_sys_variable | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_sys_variable_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_table | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_table_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_table_privilege | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_table_privilege_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_table_stat | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_table_stat_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tablegroup | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tablegroup_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tablet_checksum | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tablet_to_ls | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tablet_to_table_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_temp_table | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_constraint_column | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_constraint_column_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_dependency | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_directory | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_directory_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_error | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_keystore | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_keystore_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_objauth | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_objauth_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_object_type | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_object_type_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_ols_component | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_ols_component_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_ols_label | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_ols_label_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_ols_policy | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_ols_policy_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_ols_user_level | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_ols_user_level_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_profile | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_profile_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_rewrite_rules | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_role_grantee_map | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_role_grantee_map_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_scheduler_job | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_scheduler_job_class | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_scheduler_job_run_detail | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_scheduler_program | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_scheduler_program_argument | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_security_audit | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_security_audit_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_security_audit_record | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_sysauth | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_sysauth_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_tablespace | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_tablespace_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_time_zone | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_time_zone_name | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_time_zone_transition | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_time_zone_transition_type | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_trigger | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_trigger_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_transfer_partition_task | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_transfer_partition_task_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_transfer_task | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_transfer_task_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_type | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_type_attr | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_type_attr_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_type_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_user | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_user_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_user_proxy_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_user_proxy_info_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_user_proxy_role_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_user_proxy_role_info_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_apply_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_arbitration_member_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_arbitration_service_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_archive_dest_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_archive_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ash | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_audit_action | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_audit_operation | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_delete_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_delete_job_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_delete_ls_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_delete_ls_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_delete_policy | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_delete_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_delete_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_job_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_parameter | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_set_files | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_storage_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_storage_info_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_cgroup_config | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_checkpoint | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_checkpoint_diagnose_checkpoint_unit_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_checkpoint_diagnose_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_checkpoint_diagnose_memtable_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_client_to_server_session_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_clone_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_clone_job_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_column_group_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_column_group_mapping | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_column_group_mapping_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_compaction_diagnose_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_compaction_suggestion | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_compatibility_control | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_core_all_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_core_column_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_core_meta_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_data_type | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_data_type_class | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_deadlock_event_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_detect_lock_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_dml_stats | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_dtl_interm_result_monitor | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_dup_ls_lease_mgr | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_dup_ls_tablet_set | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_dup_ls_tablets | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_engine | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_files | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_flt_config | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_freeze_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_global_transaction | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_import_table_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_import_table_job_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_import_table_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_import_table_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_kv_connection | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_kv_ttl_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_kv_ttl_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_kvcache_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_latch | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_lock_wait_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_log_archive_dest_parameter | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_log_archive_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_log_archive_piece_files | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_log_archive_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_log_restore_source | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_log_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_arb_replica_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_arb_replica_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_election_reference_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_log_archive_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_log_restore_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_meta_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_recovery_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_replica_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_replica_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_replica_task_plan | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_restore_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_restore_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_snapshot | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_transfer_member_list_lock_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_malloc_sample_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_mds_event_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_mds_node_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_memory_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_memstore_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_merge_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_mlog | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_mview | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_mview_refresh_change_stats | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_mview_refresh_run_stats | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_mview_refresh_stats | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_mview_refresh_stats_params | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_mview_refresh_stats_sys_defaults | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_mview_refresh_stmt_stats | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_nic_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_obj_lock | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_obrpc_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_open_cursor | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_opt_stat_gather_monitor | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_plan_cache_plan_explain | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_plan_cache_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_plan_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_privilege | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_processlist | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_proxy_partition | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_proxy_partition_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_proxy_schema | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_proxy_sub_partition | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ps_item_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ps_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_px_p2p_datahub | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_px_target_monitor | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_px_worker_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_query_response_time | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_recover_table_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_recover_table_job_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_replay_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_resource_pool_mysql_sys_agent | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_restore_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_restore_job_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_restore_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_server_compaction_event_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_server_compaction_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_server_schema_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_service | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_session_event | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_session_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_session_ps_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_session_wait | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_session_wait_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_sesstat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_show_trace | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_sql_audit | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_sql_monitor_statname | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_sql_plan | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_sql_plan_monitor | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_sql_workarea_active | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_sql_workarea_histogram | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_sql_workarea_history_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_sql_workarea_memory_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_sys_variable_default_value | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_sysstat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_system_event | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_table_mgr | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_table_opt_stat_gather_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tablet_compaction_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tablet_compaction_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tablet_compaction_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tablet_encrypt_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tablet_meta_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tablet_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_task_opt_stat_gather_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_event_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_memory_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_memstore_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_mysql_sys_agent | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_parameter | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_parameter_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_resource_limit | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_resource_limit_detail | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_snapshot | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_snapshot_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_snapshot_ls | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_snapshot_ls_replica | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_snapshot_ls_replica_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_tablespace | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_user_failed_login_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_thread | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_timestamp_service | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_trace_span_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tracepoint_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_trans_lock_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_trans_scheduler | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_trans_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_transaction_checkpoint | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_transaction_freeze_checkpoint | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_unit | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_user | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_virtual_long_ops_status_mysql_sys_agent | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_wr_active_session_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_wr_control | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_wr_snapshot | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_wr_statname | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_wr_sysstat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_zone_merge_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_all_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_charset | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_collation | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_concurrent_limit_sql | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_current_tenant | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_database_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_event_name | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_global_variable | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_object_definition | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_outline | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_privilege_grant | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_session_variable | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_show_create_database | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_show_create_procedure | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_show_create_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_show_create_tablegroup | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_show_create_trigger | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_show_tables | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_statname | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_table_column | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_table_index | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_tenant_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_warning | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | CHARACTER_SETS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | CHECK_CONSTRAINTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | COLLATIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | COLLATION_CHARACTER_SET_APPLICABILITY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | COLUMNS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | COLUMN_PRIVILEGES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | CONNECTION_CONTROL_FAILED_LOGIN_ATTEMPTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | ENABLED_ROLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | ENGINES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | EVENTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | FILES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | GLOBAL_STATUS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | GLOBAL_VARIABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_BUFFER_PAGE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_BUFFER_PAGE_LRU | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_BUFFER_POOL_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_CMP | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_CMPMEM | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_CMPMEM_RESET | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_CMP_PER_INDEX | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_CMP_PER_INDEX_RESET | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_CMP_RESET | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_FT_BEING_DELETED | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_FT_CONFIG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_FT_DELETED | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_FT_INDEX_CACHE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_METRICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_SYS_COLUMNS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_SYS_DATAFILES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_SYS_FIELDS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_SYS_FOREIGN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_SYS_FOREIGN_COLS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_SYS_INDEXES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_SYS_TABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_SYS_TABLESPACES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_SYS_TABLESTATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_SYS_VIRTUAL | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_TEMP_TABLE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | KEY_COLUMN_USAGE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | OPTIMIZER_TRACE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | PARAMETERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | PARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | PLUGINS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | PROCESSLIST | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | PROFILING | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | QUERY_RESPONSE_TIME | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | REFERENTIAL_CONSTRAINTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | ROLE_COLUMN_GRANTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | ROLE_ROUTINE_GRANTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | ROLE_TABLE_GRANTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | ROUTINES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | SCHEMATA | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | SCHEMA_PRIVILEGES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | SESSION_STATUS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | SESSION_VARIABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | ST_GEOMETRY_COLUMNS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | ST_SPATIAL_REFERENCE_SYSTEMS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | TABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | TABLESPACES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | TABLE_CONSTRAINTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | TABLE_PRIVILEGES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | TRIGGERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | USER_PRIVILEGES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | VIEWS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | VIEW_TABLE_USAGE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | audit_log_filter | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | audit_log_user | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | columns_priv | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | db | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | default_roles | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | func | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | help_category | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | help_keyword | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | help_relation | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | help_topic | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | proc | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | procs_priv | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | role_edges | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | time_zone | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | time_zone_name | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | time_zone_transition | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | time_zone_transition_type | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | user | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | ALL_OB_EXTERNAL_TABLE_FILES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_DB_LINKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_INDEX_USAGE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_IND_PARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_IND_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_IND_SUBPARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVIEWS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVIEW_LOGS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVREF_CHANGE_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVREF_RUN_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVREF_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVREF_STATS_PARAMS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVREF_STATS_SYS_DEFAULTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVREF_STMT_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OBJECTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_ACCESS_POINT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_ARCHIVELOG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_ARCHIVELOG_PIECE_FILES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_ARCHIVELOG_SUMMARY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_ARCHIVE_DEST | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_AUTO_INCREMENT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_AUX_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BACKUP_DELETE_JOBS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BACKUP_DELETE_JOB_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BACKUP_DELETE_POLICY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BACKUP_DELETE_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BACKUP_DELETE_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BACKUP_JOBS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BACKUP_JOB_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BACKUP_PARAMETER | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BACKUP_SET_FILES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BACKUP_STORAGE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BACKUP_STORAGE_INFO_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BACKUP_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BACKUP_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BALANCE_JOBS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BALANCE_JOB_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BALANCE_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BALANCE_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_CONCURRENT_LIMIT_SQL | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_DATABASES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_DATABASE_PRIVILEGE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_DATA_DICTIONARY_IN_LOG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_DEADLOCK_EVENT_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_EXTERNAL_TABLE_FILES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_FREEZE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_IMPORT_TABLE_JOBS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_IMPORT_TABLE_JOB_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_IMPORT_TABLE_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_IMPORT_TABLE_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_KV_TTL_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_KV_TTL_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_LOG_RESTORE_SOURCE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_LS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_LS_ARB_REPLICA_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_LS_ARB_REPLICA_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_LS_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_LS_LOCATIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_LS_LOG_ARCHIVE_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_LS_REPLICA_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_LS_REPLICA_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_MAJOR_COMPACTION | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_OUTLINES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_OUTLINE_CONCURRENT_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_RECOVER_TABLE_JOBS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_RECOVER_TABLE_JOB_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_RESTORE_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_RESTORE_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_RSRC_IO_DIRECTIVES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_SEQUENCE_OBJECTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_SERVICES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_SYS_VARIABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TABLEGROUPS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TABLEGROUP_PARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TABLEGROUP_SUBPARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TABLEGROUP_TABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TABLET_REPLICAS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TABLET_TO_LS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TABLE_LOCATIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TABLE_STAT_STALE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TASK_OPT_STAT_GATHER_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TENANTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TENANT_EVENT_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TRANSFER_PARTITION_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TRANSFER_PARTITION_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TRANSFER_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TRANSFER_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_USERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_USER_DEFINED_RULES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_ZONE_MAJOR_COMPACTION | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_PART_COL_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_PART_HISTOGRAMS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_PART_INDEXES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_PART_KEY_COLUMNS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_PART_TABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_RECYCLEBIN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_RSRC_CONSUMER_GROUPS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_RSRC_GROUP_MAPPINGS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_RSRC_PLANS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_RSRC_PLAN_DIRECTIVES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_SCHEDULER_JOBS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_SCHEDULER_JOB_RUN_DETAILS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_SCHEDULER_WINDOWS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_SEQUENCES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_SQL_MANAGEMENT_CONFIG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_SQL_PLAN_BASELINES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_SUBPARTITION_TEMPLATES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_SUBPART_COL_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_SUBPART_HISTOGRAMS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_SUBPART_KEY_COLUMNS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_TAB_COL_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_TAB_HISTOGRAMS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_TAB_MODIFICATIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_TAB_PARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_TAB_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_TAB_STATS_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_TAB_SUBPARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_WR_ACTIVE_SESSION_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_WR_CONTROL | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_WR_SNAPSHOT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_WR_STATNAME | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_WR_SYSSTAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$ACTIVE_SESSION_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$DML_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$LATCH | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_ARBITRATION_MEMBER_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_ARBITRATION_SERVICE_STATUS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_CGROUP_CONFIG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_COMPACTION_DIAGNOSE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_COMPACTION_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_COMPACTION_SUGGESTIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_DTL_INTERM_RESULT_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_FLT_TRACE_CONFIG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_KVCACHE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_KV_CONNECTIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_LOCKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_LOG_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_LS_SNAPSHOTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_MEMORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_MEMSTORE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_MEMSTORE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_MERGE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_NIC_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_OPT_STAT_GATHER_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_PARAMETERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_PLAN_CACHE_PLAN_EXPLAIN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_PLAN_CACHE_PLAN_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_PLAN_CACHE_REFERENCE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_PLAN_CACHE_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_PL_CACHE_OBJECT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_PROCESSLIST | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_PS_ITEM_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_PS_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_PX_P2P_DATAHUB | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_PX_TARGET_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_PX_WORKER_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_RPC_INCOMING | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_RPC_OUTGOING | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_SERVER_SCHEMA_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_SESSION | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_SESSION_PS_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_SQL_AUDIT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_SQL_PLAN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_SQL_WORKAREA_MEMORY_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_SSTABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_TABLET_COMPACTION_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_TABLET_COMPACTION_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_TABLET_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_TENANT_MEMORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_TENANT_RESOURCE_LIMIT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_TENANT_RESOURCE_LIMIT_DETAIL | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_THREAD | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_TRACEPOINT_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_TRANSACTION_PARTICIPANTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_TRANSACTION_SCHEDULERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_UNITS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$SESSION_EVENT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$SESSION_LONGOPS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$SESSION_WAIT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$SESSION_WAIT_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$SESSTAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$SQL_JOIN_FILTER | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$SQL_PLAN_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$SQL_WORKAREA | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$SQL_WORKAREA_ACTIVE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$SQL_WORKAREA_HISTOGRAM | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$SYSSTAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$SYSTEM_EVENT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$ACTIVE_SESSION_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$DML_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$ENCRYPTED_TABLESPACES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$EVENT_NAME | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$LATCH | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_ARBITRATION_MEMBER_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_ARBITRATION_SERVICE_STATUS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_ARCHIVE_DEST_STATUS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_CGROUP_CONFIG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_COMPACTION_DIAGNOSE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_COMPACTION_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_COMPACTION_SUGGESTIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_COMPATIBILITY_CONTROL | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_DTL_INTERM_RESULT_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_ENCRYPTED_TABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_KVCACHE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_KV_CONNECTIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_LOCKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_LOG_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_LS_LOG_RESTORE_STATUS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_LS_REPLICA_TASK_PLAN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_LS_SNAPSHOTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_MEMORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_MEMSTORE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_MEMSTORE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_MERGE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_NIC_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_OPT_STAT_GATHER_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_PARAMETERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_PLAN_CACHE_PLAN_EXPLAIN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_PLAN_CACHE_PLAN_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_PLAN_CACHE_REFERENCE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_PLAN_CACHE_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_PL_CACHE_OBJECT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_PROCESSLIST | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_PS_ITEM_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_PS_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_PX_P2P_DATAHUB | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_PX_TARGET_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_PX_WORKER_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_RPC_INCOMING | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_RPC_OUTGOING | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_SERVER_SCHEMA_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_SESSION | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_SESSION_PS_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_SQL_AUDIT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_SQL_PLAN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_SQL_WORKAREA_MEMORY_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_SSTABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_TABLET_COMPACTION_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_TABLET_COMPACTION_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_TABLET_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_TENANT_MEMORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_TENANT_RESOURCE_LIMIT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_TENANT_RESOURCE_LIMIT_DETAIL | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_THREAD | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_TIMESTAMP_SERVICE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_TRACEPOINT_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_TRANSACTION_PARTICIPANTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_TRANSACTION_SCHEDULERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_UNITS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$RSRC_PLAN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$SESSION_EVENT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$SESSION_LONGOPS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$SESSION_WAIT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$SESSION_WAIT_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$SESSTAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$SQL_JOIN_FILTER | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$SQL_MONITOR_STATNAME | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$SQL_PLAN_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$SQL_WORKAREA | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$SQL_WORKAREA_ACTIVE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$SQL_WORKAREA_HISTOGRAM | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$STATNAME | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$SYSSTAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$SYSTEM_EVENT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __ALL_VIRTUAL_INFORMATION_COLUMNS | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_acquired_snapshot | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_audit_log_filter | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_audit_log_user | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_auto_increment | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_aux_stat | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_balance_job | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_balance_job_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_balance_task | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_balance_task_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_client_to_server_session_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_coll_type | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_coll_type_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_column | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_column_group | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_column_group_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_column_group_mapping | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_column_group_mapping_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_column_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_column_privilege | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_column_privilege_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_column_stat | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_column_stat_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_column_usage | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_constraint | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_constraint_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_context | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_context_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_core_table | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_dam_cleanup_jobs | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_dam_last_arch_ts | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_data_dictionary_in_log | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_database | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_database_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_database_privilege | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_database_privilege_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_dblink | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_dblink_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_dbms_lock_allocated | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_ddl_checksum | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_ddl_error_message | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_ddl_id | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_ddl_operation | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_ddl_task_status | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_def_sub_part | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_def_sub_part_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_detect_lock_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_dummy | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_external_table_file | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_foreign_key | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_foreign_key_column | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_foreign_key_column_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_foreign_key_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_freeze_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_func | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_func_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_histogram_stat | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_histogram_stat_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_index_usage_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_job | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_job_log | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_ls | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mlog | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mock_fk_parent_table | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mock_fk_parent_table_column | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mock_fk_parent_table_column_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mock_fk_parent_table_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_monitor_modified | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mview | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mview_dep | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mview_refresh_change_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mview_refresh_run_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mview_refresh_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mview_refresh_stats_params | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mview_refresh_stats_sys_defaults | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mview_refresh_stmt_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_ncomp_dll | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_optstat_global_prefs | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_optstat_user_prefs | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_ori_schema_version | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_outline | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_outline_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_package | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_package_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_part | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_part_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_part_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_part_info_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_pending_transaction | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_plan_baseline | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_plan_baseline_item | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_recyclebin | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_res_mgr_consumer_group | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_res_mgr_directive | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_res_mgr_mapping_rule | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_res_mgr_plan | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_rls_attribute | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_rls_attribute_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_rls_context | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_rls_context_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_rls_group | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_rls_group_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_rls_policy | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_rls_policy_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_rls_security_column | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_rls_security_column_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_routine | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_routine_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_routine_param | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_routine_param_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_routine_privilege | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_routine_privilege_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_scheduler_job_run_detail_v2 | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_sequence_object | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_sequence_object_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_sequence_value | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_spatial_reference_systems | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_spm_config | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_sub_part | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_sub_part_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_synonym | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_synonym_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_sys_stat | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_sys_variable | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_sys_variable_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_table | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_table_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_table_privilege | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_table_privilege_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_table_stat | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_table_stat_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tablegroup | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tablegroup_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tablet_checksum | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tablet_to_ls | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tablet_to_table_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_temp_table | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_constraint_column | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_constraint_column_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_dependency | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_directory | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_directory_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_error | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_keystore | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_keystore_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_objauth | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_objauth_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_object_type | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_object_type_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_ols_component | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_ols_component_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_ols_label | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_ols_label_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_ols_policy | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_ols_policy_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_ols_user_level | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_ols_user_level_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_profile | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_profile_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_rewrite_rules | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_role_grantee_map | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_role_grantee_map_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_scheduler_job | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_scheduler_job_class | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_scheduler_job_run_detail | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_scheduler_program | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_scheduler_program_argument | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_security_audit | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_security_audit_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_security_audit_record | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_sysauth | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_sysauth_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_tablespace | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_tablespace_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_time_zone | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_time_zone_name | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_time_zone_transition | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_time_zone_transition_type | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_trigger | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_trigger_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_transfer_partition_task | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_transfer_partition_task_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_transfer_task | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_transfer_task_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_type | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_type_attr | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_type_attr_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_type_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_user | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_user_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_user_proxy_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_user_proxy_info_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_user_proxy_role_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_user_proxy_role_info_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_apply_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_arbitration_member_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_arbitration_service_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_archive_dest_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_archive_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ash | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_audit_action | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_audit_operation | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_delete_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_delete_job_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_delete_ls_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_delete_ls_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_delete_policy | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_delete_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_delete_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_job_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_parameter | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_set_files | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_storage_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_storage_info_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_cgroup_config | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_checkpoint | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_checkpoint_diagnose_checkpoint_unit_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_checkpoint_diagnose_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_checkpoint_diagnose_memtable_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_client_to_server_session_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_clone_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_clone_job_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_column_group_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_column_group_mapping | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_column_group_mapping_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_compaction_diagnose_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_compaction_suggestion | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_compatibility_control | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_core_all_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_core_column_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_core_meta_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_data_type | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_data_type_class | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_deadlock_event_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_detect_lock_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_dml_stats | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_dtl_interm_result_monitor | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_dup_ls_lease_mgr | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_dup_ls_tablet_set | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_dup_ls_tablets | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_engine | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_files | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_flt_config | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_freeze_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_global_transaction | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_import_table_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_import_table_job_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_import_table_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_import_table_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_kv_connection | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_kv_ttl_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_kv_ttl_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_kvcache_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_latch | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_lock_wait_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_log_archive_dest_parameter | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_log_archive_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_log_archive_piece_files | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_log_archive_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_log_restore_source | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_log_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_arb_replica_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_arb_replica_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_election_reference_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_log_archive_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_log_restore_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_meta_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_recovery_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_replica_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_replica_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_replica_task_plan | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_restore_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_restore_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_snapshot | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_transfer_member_list_lock_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_malloc_sample_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_mds_event_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_mds_node_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_memory_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_memstore_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_merge_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_mlog | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_mview | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_mview_refresh_change_stats | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_mview_refresh_run_stats | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_mview_refresh_stats | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_mview_refresh_stats_params | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_mview_refresh_stats_sys_defaults | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_mview_refresh_stmt_stats | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_nic_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_obj_lock | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_obrpc_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_open_cursor | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_opt_stat_gather_monitor | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_plan_cache_plan_explain | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_plan_cache_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_plan_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_privilege | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_processlist | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_proxy_partition | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_proxy_partition_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_proxy_schema | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_proxy_sub_partition | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ps_item_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ps_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_px_p2p_datahub | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_px_target_monitor | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_px_worker_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_query_response_time | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_recover_table_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_recover_table_job_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_replay_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_resource_pool_mysql_sys_agent | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_restore_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_restore_job_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_restore_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_server_compaction_event_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_server_compaction_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_server_schema_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_service | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_session_event | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_session_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_session_ps_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_session_wait | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_session_wait_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_sesstat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_show_trace | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_sql_audit | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_sql_monitor_statname | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_sql_plan | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_sql_plan_monitor | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_sql_workarea_active | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_sql_workarea_histogram | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_sql_workarea_history_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_sql_workarea_memory_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_sys_variable_default_value | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_sysstat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_system_event | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_table_mgr | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_table_opt_stat_gather_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tablet_compaction_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tablet_compaction_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tablet_compaction_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tablet_encrypt_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tablet_meta_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tablet_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_task_opt_stat_gather_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_event_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_memory_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_memstore_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_mysql_sys_agent | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_parameter | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_parameter_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_resource_limit | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_resource_limit_detail | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_snapshot | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_snapshot_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_snapshot_ls | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_snapshot_ls_replica | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_snapshot_ls_replica_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_tablespace | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_user_failed_login_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_thread | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_timestamp_service | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_trace_span_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tracepoint_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_trans_lock_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_trans_scheduler | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_trans_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_transaction_checkpoint | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_transaction_freeze_checkpoint | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_unit | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_user | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_virtual_long_ops_status_mysql_sys_agent | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_wr_active_session_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_wr_control | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_wr_snapshot | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_wr_statname | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_wr_sysstat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_zone_merge_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_all_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_charset | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_collation | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_concurrent_limit_sql | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_current_tenant | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_database_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_event_name | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_global_variable | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_object_definition | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_outline | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_privilege_grant | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_session_variable | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_show_create_database | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_show_create_procedure | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_show_create_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_show_create_tablegroup | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_show_create_trigger | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_show_tables | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_statname | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_table_column | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_table_index | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_tenant_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_warning | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +---------------+--------------------+--------------------------------------------------------+--------------+--------+---------+------------+------------+----------------+-------------+-----------------+--------------+-----------+----------------+---------------------+---------------------+------------+--------------------+----------+----------------+---------------+ test select * from information_schema.table_constraints where table_schema in ('oceanbase', 'mysql', 'information_schema') order by CONSTRAINT_CATALOG, CONSTRAINT_SCHEMA, CONSTRAINT_NAME; @@ -946,11 +956,13 @@ select * from information_schema.character_sets; +--------------------+-------------------------+-----------------------+--------+ | CHARACTER_SET_NAME | DEFAULT_COLLATE_NAME | DESCRIPTION | MAXLEN | +--------------------+-------------------------+-----------------------+--------+ +| ascii | ascii_general_ci | US ASCII | 1 | | binary | binary | Binary pseudo charset | 1 | | gb18030 | gb18030_chinese_ci | GB18030 charset | 4 | | gb18030_2022 | gb18030_2022_chinese_ci | GB18030-2022 charset | 4 | | gbk | gbk_chinese_ci | GBK charset | 2 | | latin1 | latin1_swedish_ci | cp1252 West European | 1 | +| tis620 | tis620_thai_ci | TIS620 Thai | 1 | | utf16 | utf16_general_ci | UTF-16 Unicode | 2 | | utf8mb4 | utf8mb4_general_ci | UTF-8 Unicode | 4 | +--------------------+-------------------------+-----------------------+--------+ @@ -966,6 +978,11 @@ select * from information_schema.statistics where table_schema in ('oceanbase', | def | oceanbase | __all_acquired_snapshot | 0 | oceanbase | PRIMARY | 1 | tenant_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_acquired_snapshot | 0 | oceanbase | PRIMARY | 2 | gmt_create | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_acquired_snapshot | 1 | oceanbase | idx_snapshot_tablet | 1 | tablet_id | A | NULL | NULL | NULL | YES | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_audit_log_filter | 0 | oceanbase | PRIMARY | 1 | tenant_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_audit_log_filter | 0 | oceanbase | PRIMARY | 2 | filter_name | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_audit_log_user | 0 | oceanbase | PRIMARY | 1 | tenant_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_audit_log_user | 0 | oceanbase | PRIMARY | 2 | user_name | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_audit_log_user | 0 | oceanbase | PRIMARY | 3 | host | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_auto_increment | 0 | oceanbase | PRIMARY | 1 | sequence_key | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_auto_increment | 0 | oceanbase | PRIMARY | 2 | column_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_aux_stat | 0 | oceanbase | PRIMARY | 1 | tenant_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | @@ -1671,19 +1688,21 @@ select * from information_schema.schemata where schema_name in ('oceanbase', 'my | def | oceanbase | utf8mb4 | utf8mb4_general_ci | NULL | NO | +--------------+--------------------+----------------------------+------------------------+----------+--------------------+ select * from information_schema.views where table_schema in ('oceanbase', 'mysql', 'information_schema') order by TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME; -+---------------+--------------+---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ -| TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | VIEW_DEFINITION | CHECK_OPTION | IS_UPDATABLE | DEFINER | SECURITY_TYPE | CHARACTER_SET_CLIENT | COLLATION_CONNECTION | -+---------------+--------------+---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ -| def | mysql | columns_priv | SELECT cast(b.host as char(255)) as Host, cast(a.database_name as char(128)) as Db, cast(b.user_name as char(128)) as User, cast(a.table_name as char(128)) as Table_name, cast(a.column_name as char(128)) as Column_name, substr(concat(case when (a.all_priv & 1) > 0 then ',Select' else '' end, case when (a.all_priv & 2) > 0 then ',Insert' else '' end, case when (a.all_priv & 4) > 0 then ',Update' else '' end, case when (a.all_priv & 8) > 0 then ',References' else '' end), 2) as Column_priv, cast(a.gmt_modified as datetime) as Timestamp FROM oceanbase.__all_column_privilege a, oceanbase.__all_user b WHERE a.tenant_id = 0 and a.tenant_id = b.tenant_id AND a.user_id = b.user_id | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | default_roles | SELECT cast(to_user.host AS char(255)) HOST, cast(to_user.user_name AS char(128)) USER, cast(from_user.host AS char(255)) DEFAULT_ROLE_HOST, cast(from_user.user_name AS char(128)) DEFAULT_ROLE_USER FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id AND role_map.disable_flag = 0; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | func | SELECT name, ret, dl, type FROM oceanbase.__all_func | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | procs_priv | SELECT cast(b.host as char(60)) as Host, cast(a.database_name as char(64)) as Db, cast(b.user_name as char(32)) as User, cast(a.routine_name as char(64)) as Routine_name, case when a.routine_type = 1 then 'PROCEDURE' else 'FUNCTION' end as Routine_type, cast(c.priv_user as char(93)) as Grantor, substr(concat(case when (a.all_priv & 1) > 0 then ',Execute' else '' end, case when (a.all_priv & 2) > 0 then ',Alter Routine' else '' end, case when (a.all_priv & 4) > 0 then ',Grant' else '' end), 2) as Proc_priv, cast(a.gmt_modified as date) as Timestamp FROM oceanbase.__all_routine_privilege a, oceanbase.__all_user b, oceanbase.__all_routine c, oceanbase.__all_database d WHERE a.tenant_id = b.tenant_id AND a.user_id = b.user_id AND a.tenant_id = d.tenant_id and a.database_name = d.database_name AND a.tenant_id = c.tenant_id AND a.routine_name = c.routine_name AND a.routine_type = c.routine_type AND c.database_id = d.database_id AND c.package_id = -1; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | role_edges | SELECT cast(from_user.host AS char(255)) FROM_HOST, cast(from_user.user_name AS char(128)) FROM_USER, cast(to_user.host AS char(255)) TO_HOST, cast(to_user.user_name AS char(128)) TO_USER, cast(CASE role_map.admin_option WHEN 1 THEN 'Y' ELSE 'N' END AS char(1)) WITH_ADMIN_OPTION FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | time_zone | SELECT time_zone_id as Time_zone_id, use_leap_seconds as Use_leap_seconds FROM oceanbase.__all_tenant_time_zone | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | time_zone_name | SELECT name as Name, time_zone_id as Time_zone_id FROM oceanbase.__all_tenant_time_zone_name | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | time_zone_transition | SELECT time_zone_id as Time_zone_id, transition_time as Transition_time, transition_type_id as Transition_type_id FROM oceanbase.__all_tenant_time_zone_transition | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | time_zone_transition_type | SELECT time_zone_id as Time_zone_id, transition_type_id as Transition_type_id, offset as Offset, is_dst as Is_DST, abbreviation as Abbreviation FROM oceanbase.__all_tenant_time_zone_transition_type | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -+---------------+--------------+---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ ++---------------+--------------+---------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ +| TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | VIEW_DEFINITION | CHECK_OPTION | IS_UPDATABLE | DEFINER | SECURITY_TYPE | CHARACTER_SET_CLIENT | COLLATION_CONNECTION | ++---------------+--------------+---------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ +| def | mysql | audit_log_filter | SELECT CAST(filter_name AS CHAR(64)) collate utf8mb4_bin as NAME, definition as FILTER FROM oceanbase.__all_audit_log_filter WHERE tenant_id = 0 and is_deleted = 0; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | audit_log_user | SELECT CAST(user_name AS CHAR(128)) collate utf8mb4_bin as USER, host as HOST, CAST(filter_name AS CHAR(64)) collate utf8mb4_bin as FILTERNAME FROM oceanbase.__all_audit_log_user WHERE tenant_id = 0 and is_deleted = 0; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | columns_priv | SELECT cast(b.host as char(255)) as Host, cast(a.database_name as char(128)) as Db, cast(b.user_name as char(128)) as User, cast(a.table_name as char(128)) as Table_name, cast(a.column_name as char(128)) as Column_name, substr(concat(case when (a.all_priv & 1) > 0 then ',Select' else '' end, case when (a.all_priv & 2) > 0 then ',Insert' else '' end, case when (a.all_priv & 4) > 0 then ',Update' else '' end, case when (a.all_priv & 8) > 0 then ',References' else '' end), 2) as Column_priv, cast(a.gmt_modified as datetime) as Timestamp FROM oceanbase.__all_column_privilege a, oceanbase.__all_user b WHERE a.tenant_id = 0 and a.tenant_id = b.tenant_id AND a.user_id = b.user_id | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | default_roles | SELECT cast(to_user.host AS char(255)) HOST, cast(to_user.user_name AS char(128)) USER, cast(from_user.host AS char(255)) DEFAULT_ROLE_HOST, cast(from_user.user_name AS char(128)) DEFAULT_ROLE_USER FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id AND role_map.disable_flag = 0; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | func | SELECT name, ret, dl, type FROM oceanbase.__all_func | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | procs_priv | SELECT cast(b.host as char(60)) as Host, cast(a.database_name as char(64)) as Db, cast(b.user_name as char(32)) as User, cast(a.routine_name as char(64)) as Routine_name, case when a.routine_type = 1 then 'PROCEDURE' else 'FUNCTION' end as Routine_type, cast(concat(a.grantor, '@', a.grantor_host) as char(93)) as Grantor, substr(concat(case when (a.all_priv & 1) > 0 then ',Execute' else '' end, case when (a.all_priv & 2) > 0 then ',Alter Routine' else '' end, case when (a.all_priv & 4) > 0 then ',Grant' else '' end), 2) as Proc_priv, cast(a.gmt_modified as date) as Timestamp FROM oceanbase.__all_routine_privilege a, oceanbase.__all_user b WHERE a.tenant_id = 0 and a.tenant_id = b.tenant_id AND a.user_id = b.user_id; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | role_edges | SELECT cast(from_user.host AS char(255)) FROM_HOST, cast(from_user.user_name AS char(128)) FROM_USER, cast(to_user.host AS char(255)) TO_HOST, cast(to_user.user_name AS char(128)) TO_USER, cast(CASE role_map.admin_option WHEN 1 THEN 'Y' ELSE 'N' END AS char(1)) WITH_ADMIN_OPTION FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | time_zone | SELECT time_zone_id as Time_zone_id, use_leap_seconds as Use_leap_seconds FROM oceanbase.__all_tenant_time_zone | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | time_zone_name | SELECT name as Name, time_zone_id as Time_zone_id FROM oceanbase.__all_tenant_time_zone_name | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | time_zone_transition | SELECT time_zone_id as Time_zone_id, transition_time as Transition_time, transition_type_id as Transition_type_id FROM oceanbase.__all_tenant_time_zone_transition | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | time_zone_transition_type | SELECT time_zone_id as Time_zone_id, transition_type_id as Transition_type_id, offset as Offset, is_dst as Is_DST, abbreviation as Abbreviation FROM oceanbase.__all_tenant_time_zone_transition_type | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | ++---------------+--------------+---------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ select * from information_schema.files; +---------+-----------+-----------+-----------------+---------------+--------------+------------+--------------------+----------------------+--------+---------------+--------------+--------------+--------------+---------------+-------------+--------------+--------------+-----------------+---------------+------------------+------------------+--------------+---------------------+---------+------------+------------+----------------+-------------+-----------------+--------------+-----------+-------------+-------------+------------+----------+--------+-------+ | FILE_ID | FILE_NAME | FILE_TYPE | TABLESPACE_NAME | TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | LOGFILE_GROUP_NAME | LOGFILE_GROUP_NUMBER | ENGINE | FULLTEXT_KEYS | DELETED_ROWS | UPDATE_COUNT | FREE_EXTENTS | TOTAL_EXTENTS | EXTENT_SIZE | INITIAL_SIZE | MAXIMUM_SIZE | AUTOEXTEND_SIZE | CREATION_TIME | LAST_UPDATE_TIME | LAST_ACCESS_TIME | RECOVER_TIME | TRANSACTION_COUNTER | VERSION | ROW_FORMAT | TABLE_ROWS | AVG_ROW_LENGTH | DATA_LENGTH | MAX_DATA_LENGTH | INDEX_LENGTH | DATA_FREE | CREATE_TIME | UPDATE_TIME | CHECK_TIME | CHECKSUM | STATUS | EXTRA | @@ -1697,757 +1716,767 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys +---------------+--------------------+--------------------------------------------------------+--------------+--------+---------+------------+------------+----------------+-------------+-----------------+--------------+-----------+----------------+---------------------+---------------------+------------+--------------------+----------+----------------+---------------+ | TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | TABLE_TYPE | ENGINE | VERSION | ROW_FORMAT | TABLE_ROWS | AVG_ROW_LENGTH | DATA_LENGTH | MAX_DATA_LENGTH | INDEX_LENGTH | DATA_FREE | AUTO_INCREMENT | CREATE_TIME | UPDATE_TIME | CHECK_TIME | TABLE_COLLATION | CHECKSUM | CREATE_OPTIONS | TABLE_COMMENT | +---------------+--------------------+--------------------------------------------------------+--------------+--------+---------+------------+------------+----------------+-------------+-----------------+--------------+-----------+----------------+---------------------+---------------------+------------+--------------------+----------+----------------+---------------+ -| def | information_schema | CHARACTER_SETS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | CHECK_CONSTRAINTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | COLLATIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | COLLATION_CHARACTER_SET_APPLICABILITY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | COLUMNS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | COLUMN_PRIVILEGES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | CONNECTION_CONTROL_FAILED_LOGIN_ATTEMPTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | ENABLED_ROLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | ENGINES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | EVENTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | FILES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | GLOBAL_STATUS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | GLOBAL_VARIABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_BUFFER_PAGE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_BUFFER_PAGE_LRU | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_BUFFER_POOL_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_CMP | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_CMPMEM | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_CMPMEM_RESET | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_CMP_PER_INDEX | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_CMP_PER_INDEX_RESET | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_CMP_RESET | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_FT_BEING_DELETED | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_FT_CONFIG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_FT_DELETED | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_FT_INDEX_CACHE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_METRICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_SYS_DATAFILES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_SYS_FIELDS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_SYS_FOREIGN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_SYS_FOREIGN_COLS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_SYS_INDEXES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_SYS_TABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_SYS_TABLESPACES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_SYS_TABLESTATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_SYS_VIRTUAL | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | INNODB_TEMP_TABLE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | KEY_COLUMN_USAGE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | PARAMETERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | PARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | PROCESSLIST | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | PROFILING | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | QUERY_RESPONSE_TIME | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | REFERENTIAL_CONSTRAINTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | ROUTINES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | SCHEMATA | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | SCHEMA_PRIVILEGES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | SESSION_STATUS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | SESSION_VARIABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | ST_GEOMETRY_COLUMNS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | ST_SPATIAL_REFERENCE_SYSTEMS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | TABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | TABLESPACES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | TABLE_CONSTRAINTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | TABLE_PRIVILEGES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | TRIGGERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | USER_PRIVILEGES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | VIEWS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | information_schema | VIEW_TABLE_USAGE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | columns_priv | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | db | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | default_roles | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | func | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | help_category | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | help_keyword | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | help_relation | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | help_topic | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | proc | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | procs_priv | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | role_edges | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | time_zone | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | time_zone_name | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | time_zone_transition | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | time_zone_transition_type | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | mysql | user | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | ALL_OB_EXTERNAL_TABLE_FILES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_DB_LINKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_INDEX_USAGE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_IND_PARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_IND_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_IND_SUBPARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_MVIEWS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_MVIEW_LOGS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_MVREF_CHANGE_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_MVREF_RUN_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_MVREF_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_MVREF_STATS_PARAMS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_MVREF_STATS_SYS_DEFAULTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_MVREF_STMT_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OBJECTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_ACCESS_POINT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_ARCHIVELOG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_ARCHIVELOG_PIECE_FILES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_ARCHIVELOG_SUMMARY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_ARCHIVE_DEST | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_AUTO_INCREMENT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_AUX_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BACKUP_DELETE_JOBS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BACKUP_DELETE_JOB_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BACKUP_DELETE_POLICY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BACKUP_DELETE_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BACKUP_DELETE_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BACKUP_JOBS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BACKUP_JOB_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BACKUP_PARAMETER | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BACKUP_SET_FILES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BACKUP_STORAGE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BACKUP_STORAGE_INFO_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BACKUP_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BACKUP_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BALANCE_JOBS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BALANCE_JOB_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BALANCE_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_BALANCE_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_CONCURRENT_LIMIT_SQL | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_DATABASES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_DATABASE_PRIVILEGE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_DATA_DICTIONARY_IN_LOG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_DEADLOCK_EVENT_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_EXTERNAL_TABLE_FILES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_FREEZE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_IMPORT_TABLE_JOBS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_IMPORT_TABLE_JOB_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_IMPORT_TABLE_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_IMPORT_TABLE_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_KV_TTL_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_KV_TTL_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_LOG_RESTORE_SOURCE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_LS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_LS_ARB_REPLICA_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_LS_ARB_REPLICA_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_LS_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_LS_LOCATIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_LS_LOG_ARCHIVE_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_LS_REPLICA_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_LS_REPLICA_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_MAJOR_COMPACTION | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_OUTLINES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_OUTLINE_CONCURRENT_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_RECOVER_TABLE_JOBS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_RECOVER_TABLE_JOB_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_RESTORE_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_RESTORE_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_RSRC_IO_DIRECTIVES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_SEQUENCE_OBJECTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_SERVICES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_SYS_VARIABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TABLEGROUPS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TABLEGROUP_PARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TABLEGROUP_SUBPARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TABLEGROUP_TABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TABLET_REPLICAS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TABLET_TO_LS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TABLE_LOCATIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TABLE_STAT_STALE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TASK_OPT_STAT_GATHER_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TENANTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TENANT_EVENT_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TRANSFER_PARTITION_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TRANSFER_PARTITION_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TRANSFER_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_TRANSFER_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_USERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_USER_DEFINED_RULES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_OB_ZONE_MAJOR_COMPACTION | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_PART_COL_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_PART_HISTOGRAMS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_PART_INDEXES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_PART_KEY_COLUMNS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_PART_TABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_RECYCLEBIN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_RSRC_CONSUMER_GROUPS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_RSRC_GROUP_MAPPINGS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_RSRC_PLANS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_RSRC_PLAN_DIRECTIVES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_SCHEDULER_JOBS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_SCHEDULER_JOB_RUN_DETAILS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_SCHEDULER_WINDOWS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_SEQUENCES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_SQL_MANAGEMENT_CONFIG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_SQL_PLAN_BASELINES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_SUBPARTITION_TEMPLATES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_SUBPART_COL_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_SUBPART_HISTOGRAMS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_SUBPART_KEY_COLUMNS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_TAB_COL_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_TAB_HISTOGRAMS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_TAB_MODIFICATIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_TAB_PARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_TAB_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_TAB_STATS_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_TAB_SUBPARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_WR_ACTIVE_SESSION_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_WR_CONTROL | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_WR_SNAPSHOT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_WR_STATNAME | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | DBA_WR_SYSSTAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$ACTIVE_SESSION_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$DML_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$LATCH | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_ARBITRATION_MEMBER_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_ARBITRATION_SERVICE_STATUS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_CGROUP_CONFIG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_COMPACTION_DIAGNOSE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_COMPACTION_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_COMPACTION_SUGGESTIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_DTL_INTERM_RESULT_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_FLT_TRACE_CONFIG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_KVCACHE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_KV_CONNECTIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_LOCKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_LOG_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_LS_SNAPSHOTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_MEMORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_MEMSTORE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_MEMSTORE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_MERGE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_NIC_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_OPT_STAT_GATHER_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_PARAMETERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_PLAN_CACHE_PLAN_EXPLAIN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_PLAN_CACHE_PLAN_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_PLAN_CACHE_REFERENCE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_PLAN_CACHE_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_PL_CACHE_OBJECT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_PROCESSLIST | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_PS_ITEM_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_PS_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_PX_P2P_DATAHUB | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_PX_TARGET_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_PX_WORKER_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_RPC_INCOMING | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_RPC_OUTGOING | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_SERVER_SCHEMA_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_SESSION | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_SESSION_PS_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_SQL_AUDIT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_SQL_PLAN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_SQL_WORKAREA_MEMORY_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_SSTABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_TABLET_COMPACTION_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_TABLET_COMPACTION_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_TABLET_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_TENANT_MEMORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_TENANT_RESOURCE_LIMIT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_TENANT_RESOURCE_LIMIT_DETAIL | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_THREAD | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_TRACEPOINT_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_TRANSACTION_PARTICIPANTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_TRANSACTION_SCHEDULERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$OB_UNITS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$SESSION_EVENT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$SESSION_LONGOPS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$SESSION_WAIT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$SESSION_WAIT_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$SESSTAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$SQL_JOIN_FILTER | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$SQL_PLAN_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$SQL_WORKAREA | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$SQL_WORKAREA_ACTIVE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$SQL_WORKAREA_HISTOGRAM | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$SYSSTAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | GV$SYSTEM_EVENT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$ACTIVE_SESSION_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$DML_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$ENCRYPTED_TABLESPACES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$EVENT_NAME | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$LATCH | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_ARBITRATION_MEMBER_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_ARBITRATION_SERVICE_STATUS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_ARCHIVE_DEST_STATUS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_CGROUP_CONFIG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_COMPACTION_DIAGNOSE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_COMPACTION_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_COMPACTION_SUGGESTIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_COMPATIBILITY_CONTROL | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_DTL_INTERM_RESULT_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_ENCRYPTED_TABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_KVCACHE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_KV_CONNECTIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_LOCKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_LOG_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_LS_LOG_RESTORE_STATUS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_LS_REPLICA_TASK_PLAN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_LS_SNAPSHOTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_MEMORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_MEMSTORE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_MEMSTORE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_MERGE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_NIC_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_OPT_STAT_GATHER_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_PARAMETERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_PLAN_CACHE_PLAN_EXPLAIN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_PLAN_CACHE_PLAN_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_PLAN_CACHE_REFERENCE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_PLAN_CACHE_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_PL_CACHE_OBJECT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_PROCESSLIST | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_PS_ITEM_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_PS_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_PX_P2P_DATAHUB | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_PX_TARGET_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_PX_WORKER_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_RPC_INCOMING | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_RPC_OUTGOING | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_SERVER_SCHEMA_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_SESSION | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_SESSION_PS_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_SQL_AUDIT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_SQL_PLAN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_SQL_WORKAREA_MEMORY_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_SSTABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_TABLET_COMPACTION_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_TABLET_COMPACTION_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_TABLET_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_TENANT_MEMORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_TENANT_RESOURCE_LIMIT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_TENANT_RESOURCE_LIMIT_DETAIL | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_THREAD | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_TIMESTAMP_SERVICE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_TRACEPOINT_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_TRANSACTION_PARTICIPANTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_TRANSACTION_SCHEDULERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$OB_UNITS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$RSRC_PLAN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$SESSION_EVENT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$SESSION_LONGOPS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$SESSION_WAIT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$SESSION_WAIT_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$SESSTAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$SQL_JOIN_FILTER | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$SQL_MONITOR_STATNAME | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$SQL_PLAN_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$SQL_WORKAREA | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$SQL_WORKAREA_ACTIVE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$SQL_WORKAREA_HISTOGRAM | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$STATNAME | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$SYSSTAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | V$SYSTEM_EVENT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_acquired_snapshot | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_auto_increment | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_aux_stat | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_balance_job | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_balance_job_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_balance_task | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_balance_task_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_client_to_server_session_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_coll_type | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_coll_type_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_column | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_column_group | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_column_group_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_column_group_mapping | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_column_group_mapping_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_column_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_column_privilege | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_column_privilege_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_column_stat | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_column_stat_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_column_usage | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_constraint | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_constraint_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_context | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_context_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_core_table | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_dam_cleanup_jobs | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_dam_last_arch_ts | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_database | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_database_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_database_privilege | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_database_privilege_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_data_dictionary_in_log | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_dblink | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_dblink_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_dbms_lock_allocated | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_ddl_checksum | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_ddl_error_message | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_ddl_id | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_ddl_operation | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_ddl_task_status | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_def_sub_part | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_def_sub_part_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_detect_lock_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_dummy | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_external_table_file | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_foreign_key | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_foreign_key_column | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_foreign_key_column_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_foreign_key_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_freeze_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_func | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_func_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_histogram_stat | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_histogram_stat_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_index_usage_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_job | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_job_log | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_ls | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_mlog | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_mock_fk_parent_table | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_mock_fk_parent_table_column | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_mock_fk_parent_table_column_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_mock_fk_parent_table_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_monitor_modified | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_mview | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_mview_dep | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_mview_refresh_change_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_mview_refresh_run_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_mview_refresh_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_mview_refresh_stats_params | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_mview_refresh_stats_sys_defaults | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_mview_refresh_stmt_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_ncomp_dll | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_optstat_global_prefs | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_optstat_user_prefs | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_ori_schema_version | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_outline | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_outline_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_package | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_package_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_part | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_part_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_part_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_part_info_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_pending_transaction | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_plan_baseline | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_plan_baseline_item | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_recyclebin | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_res_mgr_consumer_group | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_res_mgr_directive | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_res_mgr_mapping_rule | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_res_mgr_plan | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_rls_attribute | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_rls_attribute_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_rls_context | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_rls_context_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_rls_group | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_rls_group_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_rls_policy | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_rls_policy_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_rls_security_column | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_rls_security_column_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_routine | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_routine_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_routine_param | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_routine_param_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_routine_privilege | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_routine_privilege_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_scheduler_job_run_detail_v2 | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_sequence_object | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_sequence_object_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_sequence_value | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_spatial_reference_systems | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_spm_config | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_sub_part | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_sub_part_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_synonym | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_synonym_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_sys_stat | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_sys_variable | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_sys_variable_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_table | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tablegroup | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tablegroup_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tablet_checksum | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tablet_to_ls | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tablet_to_table_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_table_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_table_privilege | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_table_privilege_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_table_stat | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_table_stat_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_temp_table | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_constraint_column | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_constraint_column_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_dependency | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_directory | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_directory_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_error | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_keystore | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_keystore_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_objauth | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_objauth_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_object_type | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_object_type_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_ols_component | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_ols_component_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_ols_label | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_ols_label_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_ols_policy | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_ols_policy_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_ols_user_level | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_ols_user_level_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_profile | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_profile_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_rewrite_rules | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_role_grantee_map | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_role_grantee_map_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_scheduler_job | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_scheduler_job_class | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_scheduler_job_run_detail | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_scheduler_program | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_scheduler_program_argument | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_security_audit | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_security_audit_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_security_audit_record | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_sysauth | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_sysauth_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_tablespace | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_tablespace_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_time_zone | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_time_zone_name | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_time_zone_transition | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_time_zone_transition_type | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_trigger | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_tenant_trigger_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_transfer_partition_task | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_transfer_partition_task_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_transfer_task | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_transfer_task_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_type | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_type_attr | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_type_attr_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_type_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_user | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_user_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_user_proxy_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_user_proxy_info_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_user_proxy_role_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_user_proxy_role_info_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_apply_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_arbitration_member_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_arbitration_service_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_archive_dest_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_archive_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ash | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_audit_action | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_audit_operation | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_delete_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_delete_job_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_delete_ls_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_delete_ls_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_delete_policy | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_delete_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_delete_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_job_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_parameter | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_set_files | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_storage_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_storage_info_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_backup_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_cgroup_config | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_checkpoint | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_checkpoint_diagnose_checkpoint_unit_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_checkpoint_diagnose_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_checkpoint_diagnose_memtable_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_client_to_server_session_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_clone_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_clone_job_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_column_group_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_column_group_mapping | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_column_group_mapping_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_compaction_diagnose_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_compaction_suggestion | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_compatibility_control | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_core_all_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_core_column_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_core_meta_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_data_type | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_data_type_class | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_deadlock_event_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_detect_lock_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_dml_stats | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_dtl_interm_result_monitor | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_dup_ls_lease_mgr | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_dup_ls_tablets | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_dup_ls_tablet_set | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_engine | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_files | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_flt_config | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_freeze_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_global_transaction | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_import_table_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_import_table_job_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_import_table_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_import_table_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __ALL_VIRTUAL_INFORMATION_COLUMNS | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_kvcache_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_kv_connection | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_kv_ttl_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_kv_ttl_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_latch | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_lock_wait_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_log_archive_dest_parameter | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_log_archive_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_log_archive_piece_files | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_log_archive_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_log_restore_source | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_log_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_arb_replica_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_arb_replica_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_election_reference_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_log_archive_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_log_restore_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_meta_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_recovery_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_replica_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_replica_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_replica_task_plan | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_restore_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_restore_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_snapshot | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ls_transfer_member_list_lock_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_malloc_sample_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_mds_event_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_mds_node_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_memory_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_memstore_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_merge_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_mlog | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_mview | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_mview_refresh_change_stats | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_mview_refresh_run_stats | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_mview_refresh_stats | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_mview_refresh_stats_params | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_mview_refresh_stats_sys_defaults | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_mview_refresh_stmt_stats | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_nic_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_obj_lock | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_obrpc_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_open_cursor | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_opt_stat_gather_monitor | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_plan_cache_plan_explain | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_plan_cache_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_plan_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_privilege | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_processlist | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_proxy_partition | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_proxy_partition_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_proxy_schema | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_proxy_sub_partition | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ps_item_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_ps_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_px_p2p_datahub | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_px_target_monitor | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_px_worker_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_query_response_time | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_recover_table_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_recover_table_job_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_replay_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_resource_pool_mysql_sys_agent | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_restore_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_restore_job_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_restore_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_server_compaction_event_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_server_compaction_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_server_schema_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_service | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_session_event | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_session_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_session_ps_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_session_wait | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_session_wait_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_sesstat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_show_trace | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_sql_audit | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_sql_monitor_statname | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_sql_plan | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_sql_plan_monitor | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_sql_workarea_active | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_sql_workarea_histogram | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_sql_workarea_history_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_sql_workarea_memory_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_sysstat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_system_event | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_sys_variable_default_value | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tablet_compaction_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tablet_compaction_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tablet_compaction_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tablet_encrypt_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tablet_meta_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tablet_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_table_mgr | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_table_opt_stat_gather_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_task_opt_stat_gather_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_event_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_memory_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_memstore_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_mysql_sys_agent | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_parameter | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_parameter_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_resource_limit | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_resource_limit_detail | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_snapshot | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_snapshot_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_snapshot_ls | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_snapshot_ls_replica | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_snapshot_ls_replica_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_tablespace | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tenant_user_failed_login_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_thread | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_timestamp_service | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_tracepoint_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_trace_span_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_transaction_checkpoint | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_transaction_freeze_checkpoint | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_trans_lock_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_trans_scheduler | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_trans_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_unit | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_user | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_virtual_long_ops_status_mysql_sys_agent | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_wr_active_session_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_wr_control | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_wr_snapshot | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_wr_statname | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_wr_sysstat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __all_virtual_zone_merge_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_all_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_charset | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_collation | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_concurrent_limit_sql | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_current_tenant | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_database_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_event_name | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_global_variable | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_object_definition | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_outline | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_privilege_grant | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_session_variable | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_show_create_database | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_show_create_procedure | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_show_create_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_show_create_tablegroup | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_show_create_trigger | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_show_tables | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_statname | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_table_column | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_table_index | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_tenant_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | oceanbase | __tenant_virtual_warning | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | -| def | test1 | t1tenant1 | BASE TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | CHARACTER_SETS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | CHECK_CONSTRAINTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | COLLATIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | COLLATION_CHARACTER_SET_APPLICABILITY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | COLUMNS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | COLUMN_PRIVILEGES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | CONNECTION_CONTROL_FAILED_LOGIN_ATTEMPTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | ENABLED_ROLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | ENGINES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | EVENTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | FILES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | GLOBAL_STATUS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | GLOBAL_VARIABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_BUFFER_PAGE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_BUFFER_PAGE_LRU | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_BUFFER_POOL_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_CMP | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_CMPMEM | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_CMPMEM_RESET | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_CMP_PER_INDEX | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_CMP_PER_INDEX_RESET | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_CMP_RESET | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_FT_BEING_DELETED | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_FT_CONFIG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_FT_DELETED | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_FT_INDEX_CACHE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_METRICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_SYS_COLUMNS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_SYS_DATAFILES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_SYS_FIELDS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_SYS_FOREIGN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_SYS_FOREIGN_COLS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_SYS_INDEXES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_SYS_TABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_SYS_TABLESPACES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_SYS_TABLESTATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_SYS_VIRTUAL | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | INNODB_TEMP_TABLE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | KEY_COLUMN_USAGE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | OPTIMIZER_TRACE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | PARAMETERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | PARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | PLUGINS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | PROCESSLIST | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | PROFILING | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | QUERY_RESPONSE_TIME | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | REFERENTIAL_CONSTRAINTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | ROLE_COLUMN_GRANTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | ROLE_ROUTINE_GRANTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | ROLE_TABLE_GRANTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | ROUTINES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | SCHEMATA | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | SCHEMA_PRIVILEGES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | SESSION_STATUS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | SESSION_VARIABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | ST_GEOMETRY_COLUMNS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | ST_SPATIAL_REFERENCE_SYSTEMS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | TABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | TABLESPACES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | TABLE_CONSTRAINTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | TABLE_PRIVILEGES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | TRIGGERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | USER_PRIVILEGES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | VIEWS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | information_schema | VIEW_TABLE_USAGE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | audit_log_filter | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | audit_log_user | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | columns_priv | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | db | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | default_roles | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | func | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | help_category | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | help_keyword | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | help_relation | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | help_topic | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | proc | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | procs_priv | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | role_edges | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | time_zone | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | time_zone_name | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | time_zone_transition | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | time_zone_transition_type | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | mysql | user | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | ALL_OB_EXTERNAL_TABLE_FILES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_DB_LINKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_INDEX_USAGE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_IND_PARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_IND_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_IND_SUBPARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVIEWS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVIEW_LOGS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVREF_CHANGE_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVREF_RUN_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVREF_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVREF_STATS_PARAMS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVREF_STATS_SYS_DEFAULTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVREF_STMT_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OBJECTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_ACCESS_POINT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_ARCHIVELOG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_ARCHIVELOG_PIECE_FILES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_ARCHIVELOG_SUMMARY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_ARCHIVE_DEST | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_AUTO_INCREMENT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_AUX_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BACKUP_DELETE_JOBS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BACKUP_DELETE_JOB_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BACKUP_DELETE_POLICY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BACKUP_DELETE_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BACKUP_DELETE_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BACKUP_JOBS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BACKUP_JOB_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BACKUP_PARAMETER | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BACKUP_SET_FILES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BACKUP_STORAGE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BACKUP_STORAGE_INFO_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BACKUP_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BACKUP_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BALANCE_JOBS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BALANCE_JOB_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BALANCE_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_BALANCE_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_CONCURRENT_LIMIT_SQL | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_DATABASES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_DATABASE_PRIVILEGE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_DATA_DICTIONARY_IN_LOG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_DEADLOCK_EVENT_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_EXTERNAL_TABLE_FILES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_FREEZE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_IMPORT_TABLE_JOBS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_IMPORT_TABLE_JOB_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_IMPORT_TABLE_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_IMPORT_TABLE_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_KV_TTL_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_KV_TTL_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_LOG_RESTORE_SOURCE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_LS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_LS_ARB_REPLICA_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_LS_ARB_REPLICA_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_LS_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_LS_LOCATIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_LS_LOG_ARCHIVE_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_LS_REPLICA_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_LS_REPLICA_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_MAJOR_COMPACTION | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_OUTLINES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_OUTLINE_CONCURRENT_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_RECOVER_TABLE_JOBS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_RECOVER_TABLE_JOB_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_RESTORE_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_RESTORE_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_RSRC_IO_DIRECTIVES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_SEQUENCE_OBJECTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_SERVICES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_SYS_VARIABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TABLEGROUPS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TABLEGROUP_PARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TABLEGROUP_SUBPARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TABLEGROUP_TABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TABLET_REPLICAS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TABLET_TO_LS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TABLE_LOCATIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TABLE_STAT_STALE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TASK_OPT_STAT_GATHER_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TENANTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TENANT_EVENT_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TRANSFER_PARTITION_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TRANSFER_PARTITION_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TRANSFER_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TRANSFER_TASK_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_USERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_USER_DEFINED_RULES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_ZONE_MAJOR_COMPACTION | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_PART_COL_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_PART_HISTOGRAMS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_PART_INDEXES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_PART_KEY_COLUMNS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_PART_TABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_RECYCLEBIN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_RSRC_CONSUMER_GROUPS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_RSRC_GROUP_MAPPINGS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_RSRC_PLANS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_RSRC_PLAN_DIRECTIVES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_SCHEDULER_JOBS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_SCHEDULER_JOB_RUN_DETAILS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_SCHEDULER_WINDOWS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_SEQUENCES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_SQL_MANAGEMENT_CONFIG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_SQL_PLAN_BASELINES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_SUBPARTITION_TEMPLATES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_SUBPART_COL_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_SUBPART_HISTOGRAMS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_SUBPART_KEY_COLUMNS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_TAB_COL_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_TAB_HISTOGRAMS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_TAB_MODIFICATIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_TAB_PARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_TAB_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_TAB_STATS_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_TAB_SUBPARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_WR_ACTIVE_SESSION_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_WR_CONTROL | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_WR_SNAPSHOT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_WR_STATNAME | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_WR_SYSSTAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$ACTIVE_SESSION_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$DML_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$LATCH | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_ARBITRATION_MEMBER_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_ARBITRATION_SERVICE_STATUS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_CGROUP_CONFIG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_COMPACTION_DIAGNOSE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_COMPACTION_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_COMPACTION_SUGGESTIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_DTL_INTERM_RESULT_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_FLT_TRACE_CONFIG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_KVCACHE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_KV_CONNECTIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_LOCKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_LOG_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_LS_SNAPSHOTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_MEMORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_MEMSTORE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_MEMSTORE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_MERGE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_NIC_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_OPT_STAT_GATHER_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_PARAMETERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_PLAN_CACHE_PLAN_EXPLAIN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_PLAN_CACHE_PLAN_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_PLAN_CACHE_REFERENCE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_PLAN_CACHE_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_PL_CACHE_OBJECT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_PROCESSLIST | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_PS_ITEM_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_PS_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_PX_P2P_DATAHUB | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_PX_TARGET_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_PX_WORKER_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_RPC_INCOMING | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_RPC_OUTGOING | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_SERVER_SCHEMA_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_SESSION | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_SESSION_PS_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_SQL_AUDIT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_SQL_PLAN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_SQL_WORKAREA_MEMORY_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_SSTABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_TABLET_COMPACTION_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_TABLET_COMPACTION_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_TABLET_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_TENANT_MEMORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_TENANT_RESOURCE_LIMIT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_TENANT_RESOURCE_LIMIT_DETAIL | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_THREAD | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_TRACEPOINT_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_TRANSACTION_PARTICIPANTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_TRANSACTION_SCHEDULERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$OB_UNITS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$SESSION_EVENT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$SESSION_LONGOPS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$SESSION_WAIT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$SESSION_WAIT_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$SESSTAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$SQL_JOIN_FILTER | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$SQL_PLAN_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$SQL_WORKAREA | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$SQL_WORKAREA_ACTIVE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$SQL_WORKAREA_HISTOGRAM | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$SYSSTAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | GV$SYSTEM_EVENT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$ACTIVE_SESSION_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$DML_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$ENCRYPTED_TABLESPACES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$EVENT_NAME | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$LATCH | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_ARBITRATION_MEMBER_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_ARBITRATION_SERVICE_STATUS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_ARCHIVE_DEST_STATUS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_CGROUP_CONFIG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_COMPACTION_DIAGNOSE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_COMPACTION_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_COMPACTION_SUGGESTIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_COMPATIBILITY_CONTROL | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_DTL_INTERM_RESULT_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_ENCRYPTED_TABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_KVCACHE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_KV_CONNECTIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_LOCKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_LOG_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_LS_LOG_RESTORE_STATUS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_LS_REPLICA_TASK_PLAN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_LS_SNAPSHOTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_MEMORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_MEMSTORE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_MEMSTORE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_MERGE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_NIC_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_OPT_STAT_GATHER_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_PARAMETERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_PLAN_CACHE_PLAN_EXPLAIN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_PLAN_CACHE_PLAN_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_PLAN_CACHE_REFERENCE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_PLAN_CACHE_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_PL_CACHE_OBJECT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_PROCESSLIST | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_PS_ITEM_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_PS_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_PX_P2P_DATAHUB | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_PX_TARGET_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_PX_WORKER_STAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_RPC_INCOMING | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_RPC_OUTGOING | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_SERVER_SCHEMA_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_SESSION | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_SESSION_PS_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_SQL_AUDIT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_SQL_PLAN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_SQL_WORKAREA_MEMORY_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_SSTABLES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_TABLET_COMPACTION_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_TABLET_COMPACTION_PROGRESS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_TABLET_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_TENANT_MEMORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_TENANT_RESOURCE_LIMIT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_TENANT_RESOURCE_LIMIT_DETAIL | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_THREAD | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_TIMESTAMP_SERVICE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_TRACEPOINT_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_TRANSACTION_PARTICIPANTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_TRANSACTION_SCHEDULERS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$OB_UNITS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$RSRC_PLAN | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$SESSION_EVENT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$SESSION_LONGOPS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$SESSION_WAIT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$SESSION_WAIT_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$SESSTAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$SQL_JOIN_FILTER | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$SQL_MONITOR_STATNAME | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$SQL_PLAN_MONITOR | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$SQL_WORKAREA | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$SQL_WORKAREA_ACTIVE | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$SQL_WORKAREA_HISTOGRAM | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$STATNAME | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$SYSSTAT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | V$SYSTEM_EVENT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_acquired_snapshot | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_audit_log_filter | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_audit_log_user | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_auto_increment | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_aux_stat | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_balance_job | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_balance_job_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_balance_task | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_balance_task_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_client_to_server_session_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_coll_type | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_coll_type_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_column | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_column_group | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_column_group_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_column_group_mapping | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_column_group_mapping_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_column_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_column_privilege | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_column_privilege_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_column_stat | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_column_stat_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_column_usage | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_constraint | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_constraint_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_context | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_context_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_core_table | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_dam_cleanup_jobs | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_dam_last_arch_ts | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_database | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_database_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_database_privilege | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_database_privilege_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_data_dictionary_in_log | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_dblink | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_dblink_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_dbms_lock_allocated | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_ddl_checksum | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_ddl_error_message | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_ddl_id | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_ddl_operation | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_ddl_task_status | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_def_sub_part | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_def_sub_part_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_detect_lock_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_dummy | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_external_table_file | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_foreign_key | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_foreign_key_column | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_foreign_key_column_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_foreign_key_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_freeze_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_func | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_func_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_histogram_stat | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_histogram_stat_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_index_usage_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_job | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_job_log | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_ls | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mlog | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mock_fk_parent_table | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mock_fk_parent_table_column | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mock_fk_parent_table_column_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mock_fk_parent_table_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_monitor_modified | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mview | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mview_dep | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mview_refresh_change_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mview_refresh_run_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mview_refresh_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mview_refresh_stats_params | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mview_refresh_stats_sys_defaults | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_mview_refresh_stmt_stats | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_ncomp_dll | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_optstat_global_prefs | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_optstat_user_prefs | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_ori_schema_version | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_outline | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_outline_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_package | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_package_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_part | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_part_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_part_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_part_info_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_pending_transaction | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_plan_baseline | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_plan_baseline_item | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_recyclebin | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_res_mgr_consumer_group | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_res_mgr_directive | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_res_mgr_mapping_rule | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_res_mgr_plan | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_rls_attribute | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_rls_attribute_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_rls_context | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_rls_context_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_rls_group | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_rls_group_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_rls_policy | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_rls_policy_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_rls_security_column | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_rls_security_column_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_routine | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_routine_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_routine_param | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_routine_param_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_routine_privilege | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_routine_privilege_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_scheduler_job_run_detail_v2 | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_sequence_object | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_sequence_object_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_sequence_value | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_spatial_reference_systems | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_spm_config | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_sub_part | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_sub_part_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_synonym | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_synonym_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_sys_stat | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_sys_variable | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_sys_variable_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_table | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tablegroup | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tablegroup_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tablet_checksum | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tablet_to_ls | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tablet_to_table_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_table_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_table_privilege | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_table_privilege_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_table_stat | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_table_stat_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_temp_table | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_constraint_column | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_constraint_column_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_dependency | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_directory | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_directory_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_error | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_keystore | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_keystore_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_objauth | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_objauth_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_object_type | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_object_type_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_ols_component | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_ols_component_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_ols_label | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_ols_label_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_ols_policy | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_ols_policy_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_ols_user_level | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_ols_user_level_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_profile | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_profile_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_rewrite_rules | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_role_grantee_map | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_role_grantee_map_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_scheduler_job | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_scheduler_job_class | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_scheduler_job_run_detail | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_scheduler_program | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_scheduler_program_argument | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_security_audit | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_security_audit_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_security_audit_record | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_sysauth | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_sysauth_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_tablespace | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_tablespace_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_time_zone | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_time_zone_name | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_time_zone_transition | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_time_zone_transition_type | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_trigger | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_tenant_trigger_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_transfer_partition_task | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_transfer_partition_task_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_transfer_task | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_transfer_task_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_type | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_type_attr | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_type_attr_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_type_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_user | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_user_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_user_proxy_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_user_proxy_info_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_user_proxy_role_info | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_user_proxy_role_info_history | SYSTEM TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_apply_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_arbitration_member_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_arbitration_service_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_archive_dest_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_archive_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ash | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_audit_action | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_audit_operation | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_delete_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_delete_job_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_delete_ls_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_delete_ls_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_delete_policy | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_delete_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_delete_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_job_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_parameter | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_set_files | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_storage_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_storage_info_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_backup_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_cgroup_config | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_checkpoint | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_checkpoint_diagnose_checkpoint_unit_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_checkpoint_diagnose_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_checkpoint_diagnose_memtable_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_client_to_server_session_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_clone_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_clone_job_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_column_group_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_column_group_mapping | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_column_group_mapping_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_compaction_diagnose_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_compaction_suggestion | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_compatibility_control | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_core_all_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_core_column_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_core_meta_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_data_type | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_data_type_class | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_deadlock_event_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_detect_lock_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_dml_stats | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_dtl_interm_result_monitor | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_dup_ls_lease_mgr | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_dup_ls_tablets | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_dup_ls_tablet_set | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_engine | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_files | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_flt_config | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_freeze_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_global_transaction | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_import_table_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_import_table_job_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_import_table_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_import_table_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __ALL_VIRTUAL_INFORMATION_COLUMNS | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_kvcache_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_kv_connection | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_kv_ttl_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_kv_ttl_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_latch | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_lock_wait_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_log_archive_dest_parameter | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_log_archive_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_log_archive_piece_files | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_log_archive_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_log_restore_source | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_log_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_arb_replica_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_arb_replica_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_election_reference_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_log_archive_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_log_restore_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_meta_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_recovery_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_replica_task | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_replica_task_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_replica_task_plan | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_restore_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_restore_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_snapshot | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ls_transfer_member_list_lock_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_malloc_sample_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_mds_event_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_mds_node_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_memory_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_memstore_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_merge_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_mlog | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_mview | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_mview_refresh_change_stats | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_mview_refresh_run_stats | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_mview_refresh_stats | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_mview_refresh_stats_params | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_mview_refresh_stats_sys_defaults | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_mview_refresh_stmt_stats | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_nic_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_obj_lock | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_obrpc_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_open_cursor | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_opt_stat_gather_monitor | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_plan_cache_plan_explain | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_plan_cache_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_plan_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_privilege | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_processlist | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_proxy_partition | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_proxy_partition_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_proxy_schema | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_proxy_sub_partition | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ps_item_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_ps_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_px_p2p_datahub | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_px_target_monitor | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_px_worker_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_query_response_time | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_recover_table_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_recover_table_job_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_replay_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_resource_pool_mysql_sys_agent | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_restore_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_restore_job_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_restore_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_server_compaction_event_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_server_compaction_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_server_schema_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_service | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_session_event | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_session_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_session_ps_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_session_wait | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_session_wait_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_sesstat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_show_trace | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_sql_audit | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_sql_monitor_statname | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_sql_plan | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_sql_plan_monitor | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_sql_workarea_active | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_sql_workarea_histogram | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_sql_workarea_history_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_sql_workarea_memory_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_sysstat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_system_event | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_sys_variable_default_value | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tablet_compaction_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tablet_compaction_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tablet_compaction_progress | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tablet_encrypt_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tablet_meta_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tablet_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_table_mgr | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_table_opt_stat_gather_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_task_opt_stat_gather_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_event_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_memory_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_memstore_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_mysql_sys_agent | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_parameter | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_parameter_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_resource_limit | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_resource_limit_detail | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_snapshot | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_snapshot_job | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_snapshot_ls | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_snapshot_ls_replica | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_snapshot_ls_replica_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_tablespace | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tenant_user_failed_login_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_thread | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_timestamp_service | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_tracepoint_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_trace_span_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_transaction_checkpoint | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_transaction_freeze_checkpoint | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_trans_lock_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_trans_scheduler | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_trans_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_unit | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_user | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_virtual_long_ops_status_mysql_sys_agent | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_wr_active_session_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_wr_control | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_wr_snapshot | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_wr_statname | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_wr_sysstat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_zone_merge_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_all_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_charset | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_collation | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_concurrent_limit_sql | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_current_tenant | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_database_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_event_name | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_global_variable | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_object_definition | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_outline | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_privilege_grant | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_session_variable | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_show_create_database | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_show_create_procedure | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_show_create_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_show_create_tablegroup | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_show_create_trigger | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_show_tables | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_statname | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_table_column | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_table_index | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_tenant_status | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __tenant_virtual_warning | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | test1 | t1tenant1 | BASE TABLE | InnoDB | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +---------------+--------------------+--------------------------------------------------------+--------------+--------+---------+------------+------------+----------------+-------------+-----------------+--------------+-----------+----------------+---------------------+---------------------+------------+--------------------+----------+----------------+---------------+ create index t1tenant1_idx1 on t1tenant1(c2, c1); @@ -2473,20 +2502,22 @@ select * from information_schema.schemata where schema_name in ('oceanbase', 'my | def | test1 | utf8mb4 | utf8mb4_general_ci | NULL | NO | +--------------+--------------------+----------------------------+------------------------+----------+--------------------+ select * from information_schema.views where table_schema in ('oceanbase', 'mysql', 'information_schema', 'test1') order by TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME; -+---------------+--------------+---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ -| TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | VIEW_DEFINITION | CHECK_OPTION | IS_UPDATABLE | DEFINER | SECURITY_TYPE | CHARACTER_SET_CLIENT | COLLATION_CONNECTION | -+---------------+--------------+---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ -| def | mysql | columns_priv | SELECT cast(b.host as char(255)) as Host, cast(a.database_name as char(128)) as Db, cast(b.user_name as char(128)) as User, cast(a.table_name as char(128)) as Table_name, cast(a.column_name as char(128)) as Column_name, substr(concat(case when (a.all_priv & 1) > 0 then ',Select' else '' end, case when (a.all_priv & 2) > 0 then ',Insert' else '' end, case when (a.all_priv & 4) > 0 then ',Update' else '' end, case when (a.all_priv & 8) > 0 then ',References' else '' end), 2) as Column_priv, cast(a.gmt_modified as datetime) as Timestamp FROM oceanbase.__all_column_privilege a, oceanbase.__all_user b WHERE a.tenant_id = 0 and a.tenant_id = b.tenant_id AND a.user_id = b.user_id | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | default_roles | SELECT cast(to_user.host AS char(255)) HOST, cast(to_user.user_name AS char(128)) USER, cast(from_user.host AS char(255)) DEFAULT_ROLE_HOST, cast(from_user.user_name AS char(128)) DEFAULT_ROLE_USER FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id AND role_map.disable_flag = 0; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | func | SELECT name, ret, dl, type FROM oceanbase.__all_func | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | procs_priv | SELECT cast(b.host as char(60)) as Host, cast(a.database_name as char(64)) as Db, cast(b.user_name as char(32)) as User, cast(a.routine_name as char(64)) as Routine_name, case when a.routine_type = 1 then 'PROCEDURE' else 'FUNCTION' end as Routine_type, cast(c.priv_user as char(93)) as Grantor, substr(concat(case when (a.all_priv & 1) > 0 then ',Execute' else '' end, case when (a.all_priv & 2) > 0 then ',Alter Routine' else '' end, case when (a.all_priv & 4) > 0 then ',Grant' else '' end), 2) as Proc_priv, cast(a.gmt_modified as date) as Timestamp FROM oceanbase.__all_routine_privilege a, oceanbase.__all_user b, oceanbase.__all_routine c, oceanbase.__all_database d WHERE a.tenant_id = b.tenant_id AND a.user_id = b.user_id AND a.tenant_id = d.tenant_id and a.database_name = d.database_name AND a.tenant_id = c.tenant_id AND a.routine_name = c.routine_name AND a.routine_type = c.routine_type AND c.database_id = d.database_id AND c.package_id = -1; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | role_edges | SELECT cast(from_user.host AS char(255)) FROM_HOST, cast(from_user.user_name AS char(128)) FROM_USER, cast(to_user.host AS char(255)) TO_HOST, cast(to_user.user_name AS char(128)) TO_USER, cast(CASE role_map.admin_option WHEN 1 THEN 'Y' ELSE 'N' END AS char(1)) WITH_ADMIN_OPTION FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | time_zone | SELECT time_zone_id as Time_zone_id, use_leap_seconds as Use_leap_seconds FROM oceanbase.__all_tenant_time_zone | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | time_zone_name | SELECT name as Name, time_zone_id as Time_zone_id FROM oceanbase.__all_tenant_time_zone_name | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | time_zone_transition | SELECT time_zone_id as Time_zone_id, transition_time as Transition_time, transition_type_id as Transition_type_id FROM oceanbase.__all_tenant_time_zone_transition | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | time_zone_transition_type | SELECT time_zone_id as Time_zone_id, transition_type_id as Transition_type_id, offset as Offset, is_dst as Is_DST, abbreviation as Abbreviation FROM oceanbase.__all_tenant_time_zone_transition_type | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | test1 | vt1tenant1 | select `test1`.`t1tenant1`.`c1` AS `c1`,`test1`.`t1tenant1`.`c2` AS `c2` from `test1`.`t1tenant1` | NONE | YES | admin@% | NONE | utf8mb4 | utf8mb4_general_ci | -+---------------+--------------+---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ ++---------------+--------------+---------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ +| TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | VIEW_DEFINITION | CHECK_OPTION | IS_UPDATABLE | DEFINER | SECURITY_TYPE | CHARACTER_SET_CLIENT | COLLATION_CONNECTION | ++---------------+--------------+---------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ +| def | mysql | audit_log_filter | SELECT CAST(filter_name AS CHAR(64)) collate utf8mb4_bin as NAME, definition as FILTER FROM oceanbase.__all_audit_log_filter WHERE tenant_id = 0 and is_deleted = 0; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | audit_log_user | SELECT CAST(user_name AS CHAR(128)) collate utf8mb4_bin as USER, host as HOST, CAST(filter_name AS CHAR(64)) collate utf8mb4_bin as FILTERNAME FROM oceanbase.__all_audit_log_user WHERE tenant_id = 0 and is_deleted = 0; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | columns_priv | SELECT cast(b.host as char(255)) as Host, cast(a.database_name as char(128)) as Db, cast(b.user_name as char(128)) as User, cast(a.table_name as char(128)) as Table_name, cast(a.column_name as char(128)) as Column_name, substr(concat(case when (a.all_priv & 1) > 0 then ',Select' else '' end, case when (a.all_priv & 2) > 0 then ',Insert' else '' end, case when (a.all_priv & 4) > 0 then ',Update' else '' end, case when (a.all_priv & 8) > 0 then ',References' else '' end), 2) as Column_priv, cast(a.gmt_modified as datetime) as Timestamp FROM oceanbase.__all_column_privilege a, oceanbase.__all_user b WHERE a.tenant_id = 0 and a.tenant_id = b.tenant_id AND a.user_id = b.user_id | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | default_roles | SELECT cast(to_user.host AS char(255)) HOST, cast(to_user.user_name AS char(128)) USER, cast(from_user.host AS char(255)) DEFAULT_ROLE_HOST, cast(from_user.user_name AS char(128)) DEFAULT_ROLE_USER FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id AND role_map.disable_flag = 0; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | func | SELECT name, ret, dl, type FROM oceanbase.__all_func | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | procs_priv | SELECT cast(b.host as char(60)) as Host, cast(a.database_name as char(64)) as Db, cast(b.user_name as char(32)) as User, cast(a.routine_name as char(64)) as Routine_name, case when a.routine_type = 1 then 'PROCEDURE' else 'FUNCTION' end as Routine_type, cast(concat(a.grantor, '@', a.grantor_host) as char(93)) as Grantor, substr(concat(case when (a.all_priv & 1) > 0 then ',Execute' else '' end, case when (a.all_priv & 2) > 0 then ',Alter Routine' else '' end, case when (a.all_priv & 4) > 0 then ',Grant' else '' end), 2) as Proc_priv, cast(a.gmt_modified as date) as Timestamp FROM oceanbase.__all_routine_privilege a, oceanbase.__all_user b WHERE a.tenant_id = 0 and a.tenant_id = b.tenant_id AND a.user_id = b.user_id; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | role_edges | SELECT cast(from_user.host AS char(255)) FROM_HOST, cast(from_user.user_name AS char(128)) FROM_USER, cast(to_user.host AS char(255)) TO_HOST, cast(to_user.user_name AS char(128)) TO_USER, cast(CASE role_map.admin_option WHEN 1 THEN 'Y' ELSE 'N' END AS char(1)) WITH_ADMIN_OPTION FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | time_zone | SELECT time_zone_id as Time_zone_id, use_leap_seconds as Use_leap_seconds FROM oceanbase.__all_tenant_time_zone | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | time_zone_name | SELECT name as Name, time_zone_id as Time_zone_id FROM oceanbase.__all_tenant_time_zone_name | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | time_zone_transition | SELECT time_zone_id as Time_zone_id, transition_time as Transition_time, transition_type_id as Transition_type_id FROM oceanbase.__all_tenant_time_zone_transition | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | time_zone_transition_type | SELECT time_zone_id as Time_zone_id, transition_type_id as Transition_type_id, offset as Offset, is_dst as Is_DST, abbreviation as Abbreviation FROM oceanbase.__all_tenant_time_zone_transition_type | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | test1 | vt1tenant1 | select `test1`.`t1tenant1`.`c1` AS `c1`,`test1`.`t1tenant1`.`c2` AS `c2` from `test1`.`t1tenant1` | NONE | YES | admin@% | NONE | utf8mb4 | utf8mb4_general_ci | ++---------------+--------------+---------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ set @@session.ob_query_timeout = 60000000; use information_schema; @@ -2509,6 +2540,11 @@ select * from information_schema.statistics where table_schema in ('oceanbase', | def | oceanbase | __all_acquired_snapshot | 0 | oceanbase | PRIMARY | 2 | gmt_create | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_acquired_snapshot | 1 | oceanbase | idx_snapshot_tablet | 1 | tablet_id | A | NULL | NULL | NULL | YES | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_arbitration_service | 0 | oceanbase | PRIMARY | 1 | arbitration_service_key | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_audit_log_filter | 0 | oceanbase | PRIMARY | 1 | tenant_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_audit_log_filter | 0 | oceanbase | PRIMARY | 2 | filter_name | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_audit_log_user | 0 | oceanbase | PRIMARY | 1 | tenant_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_audit_log_user | 0 | oceanbase | PRIMARY | 2 | user_name | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_audit_log_user | 0 | oceanbase | PRIMARY | 3 | host | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_auto_increment | 0 | oceanbase | PRIMARY | 1 | sequence_key | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_auto_increment | 0 | oceanbase | PRIMARY | 2 | column_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_aux_stat | 0 | oceanbase | PRIMARY | 1 | tenant_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | @@ -3530,19 +3566,21 @@ select * from information_schema.statistics where table_schema in ('oceanbase', | def | oceanbase | __wr_sysstat | 0 | oceanbase | PRIMARY | 6 | stat_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +---------------+--------------+-------------------------------------------+------------+--------------+-----------------------------------------------------+--------------+-------------------------+-----------+-------------+----------+--------+----------+------------+---------+---------------+------------+------------+ select * from information_schema.views where table_schema in ('oceanbase', 'mysql', 'information_schema', 'SYS', 'LBACSYS', 'ORAAUDITOR') order by TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME; -+---------------+--------------+---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ -| TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | VIEW_DEFINITION | CHECK_OPTION | IS_UPDATABLE | DEFINER | SECURITY_TYPE | CHARACTER_SET_CLIENT | COLLATION_CONNECTION | -+---------------+--------------+---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ -| def | mysql | columns_priv | SELECT cast(b.host as char(255)) as Host, cast(a.database_name as char(128)) as Db, cast(b.user_name as char(128)) as User, cast(a.table_name as char(128)) as Table_name, cast(a.column_name as char(128)) as Column_name, substr(concat(case when (a.all_priv & 1) > 0 then ',Select' else '' end, case when (a.all_priv & 2) > 0 then ',Insert' else '' end, case when (a.all_priv & 4) > 0 then ',Update' else '' end, case when (a.all_priv & 8) > 0 then ',References' else '' end), 2) as Column_priv, cast(a.gmt_modified as datetime) as Timestamp FROM oceanbase.__all_column_privilege a, oceanbase.__all_user b WHERE a.tenant_id = 0 and a.tenant_id = b.tenant_id AND a.user_id = b.user_id | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | default_roles | SELECT cast(to_user.host AS char(255)) HOST, cast(to_user.user_name AS char(128)) USER, cast(from_user.host AS char(255)) DEFAULT_ROLE_HOST, cast(from_user.user_name AS char(128)) DEFAULT_ROLE_USER FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id AND role_map.disable_flag = 0; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | func | SELECT name, ret, dl, type FROM oceanbase.__all_func | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | procs_priv | SELECT cast(b.host as char(60)) as Host, cast(a.database_name as char(64)) as Db, cast(b.user_name as char(32)) as User, cast(a.routine_name as char(64)) as Routine_name, case when a.routine_type = 1 then 'PROCEDURE' else 'FUNCTION' end as Routine_type, cast(c.priv_user as char(93)) as Grantor, substr(concat(case when (a.all_priv & 1) > 0 then ',Execute' else '' end, case when (a.all_priv & 2) > 0 then ',Alter Routine' else '' end, case when (a.all_priv & 4) > 0 then ',Grant' else '' end), 2) as Proc_priv, cast(a.gmt_modified as date) as Timestamp FROM oceanbase.__all_routine_privilege a, oceanbase.__all_user b, oceanbase.__all_routine c, oceanbase.__all_database d WHERE a.tenant_id = b.tenant_id AND a.user_id = b.user_id AND a.tenant_id = d.tenant_id and a.database_name = d.database_name AND a.tenant_id = c.tenant_id AND a.routine_name = c.routine_name AND a.routine_type = c.routine_type AND c.database_id = d.database_id AND c.package_id = -1; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | role_edges | SELECT cast(from_user.host AS char(255)) FROM_HOST, cast(from_user.user_name AS char(128)) FROM_USER, cast(to_user.host AS char(255)) TO_HOST, cast(to_user.user_name AS char(128)) TO_USER, cast(CASE role_map.admin_option WHEN 1 THEN 'Y' ELSE 'N' END AS char(1)) WITH_ADMIN_OPTION FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | time_zone | SELECT time_zone_id as Time_zone_id, use_leap_seconds as Use_leap_seconds FROM oceanbase.__all_tenant_time_zone | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | time_zone_name | SELECT name as Name, time_zone_id as Time_zone_id FROM oceanbase.__all_tenant_time_zone_name | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | time_zone_transition | SELECT time_zone_id as Time_zone_id, transition_time as Transition_time, transition_type_id as Transition_type_id FROM oceanbase.__all_tenant_time_zone_transition | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | time_zone_transition_type | SELECT time_zone_id as Time_zone_id, transition_type_id as Transition_type_id, offset as Offset, is_dst as Is_DST, abbreviation as Abbreviation FROM oceanbase.__all_tenant_time_zone_transition_type | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -+---------------+--------------+---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ ++---------------+--------------+---------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ +| TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | VIEW_DEFINITION | CHECK_OPTION | IS_UPDATABLE | DEFINER | SECURITY_TYPE | CHARACTER_SET_CLIENT | COLLATION_CONNECTION | ++---------------+--------------+---------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ +| def | mysql | audit_log_filter | SELECT CAST(filter_name AS CHAR(64)) collate utf8mb4_bin as NAME, definition as FILTER FROM oceanbase.__all_audit_log_filter WHERE tenant_id = 0 and is_deleted = 0; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | audit_log_user | SELECT CAST(user_name AS CHAR(128)) collate utf8mb4_bin as USER, host as HOST, CAST(filter_name AS CHAR(64)) collate utf8mb4_bin as FILTERNAME FROM oceanbase.__all_audit_log_user WHERE tenant_id = 0 and is_deleted = 0; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | columns_priv | SELECT cast(b.host as char(255)) as Host, cast(a.database_name as char(128)) as Db, cast(b.user_name as char(128)) as User, cast(a.table_name as char(128)) as Table_name, cast(a.column_name as char(128)) as Column_name, substr(concat(case when (a.all_priv & 1) > 0 then ',Select' else '' end, case when (a.all_priv & 2) > 0 then ',Insert' else '' end, case when (a.all_priv & 4) > 0 then ',Update' else '' end, case when (a.all_priv & 8) > 0 then ',References' else '' end), 2) as Column_priv, cast(a.gmt_modified as datetime) as Timestamp FROM oceanbase.__all_column_privilege a, oceanbase.__all_user b WHERE a.tenant_id = 0 and a.tenant_id = b.tenant_id AND a.user_id = b.user_id | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | default_roles | SELECT cast(to_user.host AS char(255)) HOST, cast(to_user.user_name AS char(128)) USER, cast(from_user.host AS char(255)) DEFAULT_ROLE_HOST, cast(from_user.user_name AS char(128)) DEFAULT_ROLE_USER FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id AND role_map.disable_flag = 0; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | func | SELECT name, ret, dl, type FROM oceanbase.__all_func | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | procs_priv | SELECT cast(b.host as char(60)) as Host, cast(a.database_name as char(64)) as Db, cast(b.user_name as char(32)) as User, cast(a.routine_name as char(64)) as Routine_name, case when a.routine_type = 1 then 'PROCEDURE' else 'FUNCTION' end as Routine_type, cast(concat(a.grantor, '@', a.grantor_host) as char(93)) as Grantor, substr(concat(case when (a.all_priv & 1) > 0 then ',Execute' else '' end, case when (a.all_priv & 2) > 0 then ',Alter Routine' else '' end, case when (a.all_priv & 4) > 0 then ',Grant' else '' end), 2) as Proc_priv, cast(a.gmt_modified as date) as Timestamp FROM oceanbase.__all_routine_privilege a, oceanbase.__all_user b WHERE a.tenant_id = 0 and a.tenant_id = b.tenant_id AND a.user_id = b.user_id; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | role_edges | SELECT cast(from_user.host AS char(255)) FROM_HOST, cast(from_user.user_name AS char(128)) FROM_USER, cast(to_user.host AS char(255)) TO_HOST, cast(to_user.user_name AS char(128)) TO_USER, cast(CASE role_map.admin_option WHEN 1 THEN 'Y' ELSE 'N' END AS char(1)) WITH_ADMIN_OPTION FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | time_zone | SELECT time_zone_id as Time_zone_id, use_leap_seconds as Use_leap_seconds FROM oceanbase.__all_tenant_time_zone | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | time_zone_name | SELECT name as Name, time_zone_id as Time_zone_id FROM oceanbase.__all_tenant_time_zone_name | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | time_zone_transition | SELECT time_zone_id as Time_zone_id, transition_time as Transition_time, transition_type_id as Transition_type_id FROM oceanbase.__all_tenant_time_zone_transition | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | time_zone_transition_type | SELECT time_zone_id as Time_zone_id, transition_type_id as Transition_type_id, offset as Offset, is_dst as Is_DST, abbreviation as Abbreviation FROM oceanbase.__all_tenant_time_zone_transition_type | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | ++---------------+--------------+---------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ use test1; ERROR 42000: Unknown database 'test1' @@ -3566,6 +3604,11 @@ select * from information_schema.statistics where table_schema in ('oceanbase', | def | oceanbase | __all_acquired_snapshot | 0 | oceanbase | PRIMARY | 1 | tenant_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_acquired_snapshot | 0 | oceanbase | PRIMARY | 2 | gmt_create | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_acquired_snapshot | 1 | oceanbase | idx_snapshot_tablet | 1 | tablet_id | A | NULL | NULL | NULL | YES | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_audit_log_filter | 0 | oceanbase | PRIMARY | 1 | tenant_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_audit_log_filter | 0 | oceanbase | PRIMARY | 2 | filter_name | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_audit_log_user | 0 | oceanbase | PRIMARY | 1 | tenant_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_audit_log_user | 0 | oceanbase | PRIMARY | 2 | user_name | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_audit_log_user | 0 | oceanbase | PRIMARY | 3 | host | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_auto_increment | 0 | oceanbase | PRIMARY | 1 | sequence_key | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_auto_increment | 0 | oceanbase | PRIMARY | 2 | column_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_aux_stat | 0 | oceanbase | PRIMARY | 1 | tenant_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | @@ -4267,19 +4310,21 @@ select * from information_schema.statistics where table_schema in ('oceanbase', | def | test1 | t1tenant1 | 1 | test1 | t1tenant1_idx1 | 2 | c1 | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +---------------+--------------+-------------------------------------------+------------+--------------+-----------------------------------------------------+--------------+-------------------------+-----------+-------------+----------+--------+----------+------------+---------+---------------+------------+------------+ select * from information_schema.views where table_schema in ('oceanbase', 'mysql', 'information_schema', 'test1') order by TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME; -+---------------+--------------+---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ -| TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | VIEW_DEFINITION | CHECK_OPTION | IS_UPDATABLE | DEFINER | SECURITY_TYPE | CHARACTER_SET_CLIENT | COLLATION_CONNECTION | -+---------------+--------------+---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ -| def | mysql | columns_priv | SELECT cast(b.host as char(255)) as Host, cast(a.database_name as char(128)) as Db, cast(b.user_name as char(128)) as User, cast(a.table_name as char(128)) as Table_name, cast(a.column_name as char(128)) as Column_name, substr(concat(case when (a.all_priv & 1) > 0 then ',Select' else '' end, case when (a.all_priv & 2) > 0 then ',Insert' else '' end, case when (a.all_priv & 4) > 0 then ',Update' else '' end, case when (a.all_priv & 8) > 0 then ',References' else '' end), 2) as Column_priv, cast(a.gmt_modified as datetime) as Timestamp FROM oceanbase.__all_column_privilege a, oceanbase.__all_user b WHERE a.tenant_id = 0 and a.tenant_id = b.tenant_id AND a.user_id = b.user_id | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | default_roles | SELECT cast(to_user.host AS char(255)) HOST, cast(to_user.user_name AS char(128)) USER, cast(from_user.host AS char(255)) DEFAULT_ROLE_HOST, cast(from_user.user_name AS char(128)) DEFAULT_ROLE_USER FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id AND role_map.disable_flag = 0; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | func | SELECT name, ret, dl, type FROM oceanbase.__all_func | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | procs_priv | SELECT cast(b.host as char(60)) as Host, cast(a.database_name as char(64)) as Db, cast(b.user_name as char(32)) as User, cast(a.routine_name as char(64)) as Routine_name, case when a.routine_type = 1 then 'PROCEDURE' else 'FUNCTION' end as Routine_type, cast(c.priv_user as char(93)) as Grantor, substr(concat(case when (a.all_priv & 1) > 0 then ',Execute' else '' end, case when (a.all_priv & 2) > 0 then ',Alter Routine' else '' end, case when (a.all_priv & 4) > 0 then ',Grant' else '' end), 2) as Proc_priv, cast(a.gmt_modified as date) as Timestamp FROM oceanbase.__all_routine_privilege a, oceanbase.__all_user b, oceanbase.__all_routine c, oceanbase.__all_database d WHERE a.tenant_id = b.tenant_id AND a.user_id = b.user_id AND a.tenant_id = d.tenant_id and a.database_name = d.database_name AND a.tenant_id = c.tenant_id AND a.routine_name = c.routine_name AND a.routine_type = c.routine_type AND c.database_id = d.database_id AND c.package_id = -1; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | role_edges | SELECT cast(from_user.host AS char(255)) FROM_HOST, cast(from_user.user_name AS char(128)) FROM_USER, cast(to_user.host AS char(255)) TO_HOST, cast(to_user.user_name AS char(128)) TO_USER, cast(CASE role_map.admin_option WHEN 1 THEN 'Y' ELSE 'N' END AS char(1)) WITH_ADMIN_OPTION FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | time_zone | SELECT time_zone_id as Time_zone_id, use_leap_seconds as Use_leap_seconds FROM oceanbase.__all_tenant_time_zone | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | time_zone_name | SELECT name as Name, time_zone_id as Time_zone_id FROM oceanbase.__all_tenant_time_zone_name | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | time_zone_transition | SELECT time_zone_id as Time_zone_id, transition_time as Transition_time, transition_type_id as Transition_type_id FROM oceanbase.__all_tenant_time_zone_transition | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | mysql | time_zone_transition_type | SELECT time_zone_id as Time_zone_id, transition_type_id as Transition_type_id, offset as Offset, is_dst as Is_DST, abbreviation as Abbreviation FROM oceanbase.__all_tenant_time_zone_transition_type | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | -| def | test1 | vt1tenant1 | select `test1`.`t1tenant1`.`c1` AS `c1`,`test1`.`t1tenant1`.`c2` AS `c2` from `test1`.`t1tenant1` | NONE | YES | admin@% | NONE | utf8mb4 | utf8mb4_general_ci | -+---------------+--------------+---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ ++---------------+--------------+---------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ +| TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | VIEW_DEFINITION | CHECK_OPTION | IS_UPDATABLE | DEFINER | SECURITY_TYPE | CHARACTER_SET_CLIENT | COLLATION_CONNECTION | ++---------------+--------------+---------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ +| def | mysql | audit_log_filter | SELECT CAST(filter_name AS CHAR(64)) collate utf8mb4_bin as NAME, definition as FILTER FROM oceanbase.__all_audit_log_filter WHERE tenant_id = 0 and is_deleted = 0; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | audit_log_user | SELECT CAST(user_name AS CHAR(128)) collate utf8mb4_bin as USER, host as HOST, CAST(filter_name AS CHAR(64)) collate utf8mb4_bin as FILTERNAME FROM oceanbase.__all_audit_log_user WHERE tenant_id = 0 and is_deleted = 0; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | columns_priv | SELECT cast(b.host as char(255)) as Host, cast(a.database_name as char(128)) as Db, cast(b.user_name as char(128)) as User, cast(a.table_name as char(128)) as Table_name, cast(a.column_name as char(128)) as Column_name, substr(concat(case when (a.all_priv & 1) > 0 then ',Select' else '' end, case when (a.all_priv & 2) > 0 then ',Insert' else '' end, case when (a.all_priv & 4) > 0 then ',Update' else '' end, case when (a.all_priv & 8) > 0 then ',References' else '' end), 2) as Column_priv, cast(a.gmt_modified as datetime) as Timestamp FROM oceanbase.__all_column_privilege a, oceanbase.__all_user b WHERE a.tenant_id = 0 and a.tenant_id = b.tenant_id AND a.user_id = b.user_id | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | default_roles | SELECT cast(to_user.host AS char(255)) HOST, cast(to_user.user_name AS char(128)) USER, cast(from_user.host AS char(255)) DEFAULT_ROLE_HOST, cast(from_user.user_name AS char(128)) DEFAULT_ROLE_USER FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id AND role_map.disable_flag = 0; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | func | SELECT name, ret, dl, type FROM oceanbase.__all_func | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | procs_priv | SELECT cast(b.host as char(60)) as Host, cast(a.database_name as char(64)) as Db, cast(b.user_name as char(32)) as User, cast(a.routine_name as char(64)) as Routine_name, case when a.routine_type = 1 then 'PROCEDURE' else 'FUNCTION' end as Routine_type, cast(concat(a.grantor, '@', a.grantor_host) as char(93)) as Grantor, substr(concat(case when (a.all_priv & 1) > 0 then ',Execute' else '' end, case when (a.all_priv & 2) > 0 then ',Alter Routine' else '' end, case when (a.all_priv & 4) > 0 then ',Grant' else '' end), 2) as Proc_priv, cast(a.gmt_modified as date) as Timestamp FROM oceanbase.__all_routine_privilege a, oceanbase.__all_user b WHERE a.tenant_id = 0 and a.tenant_id = b.tenant_id AND a.user_id = b.user_id; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | role_edges | SELECT cast(from_user.host AS char(255)) FROM_HOST, cast(from_user.user_name AS char(128)) FROM_USER, cast(to_user.host AS char(255)) TO_HOST, cast(to_user.user_name AS char(128)) TO_USER, cast(CASE role_map.admin_option WHEN 1 THEN 'Y' ELSE 'N' END AS char(1)) WITH_ADMIN_OPTION FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id; | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | time_zone | SELECT time_zone_id as Time_zone_id, use_leap_seconds as Use_leap_seconds FROM oceanbase.__all_tenant_time_zone | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | time_zone_name | SELECT name as Name, time_zone_id as Time_zone_id FROM oceanbase.__all_tenant_time_zone_name | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | time_zone_transition | SELECT time_zone_id as Time_zone_id, transition_time as Transition_time, transition_type_id as Transition_type_id FROM oceanbase.__all_tenant_time_zone_transition | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | mysql | time_zone_transition_type | SELECT time_zone_id as Time_zone_id, transition_type_id as Transition_type_id, offset as Offset, is_dst as Is_DST, abbreviation as Abbreviation FROM oceanbase.__all_tenant_time_zone_transition_type | NONE | NO | NONE | NONE | utf8mb4 | utf8mb4_general_ci | +| def | test1 | vt1tenant1 | select `test1`.`t1tenant1`.`c1` AS `c1`,`test1`.`t1tenant1`.`c2` AS `c2` from `test1`.`t1tenant1` | NONE | YES | admin@% | NONE | utf8mb4 | utf8mb4_general_ci | ++---------------+--------------+---------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------+---------+---------------+----------------------+----------------------+ drop database test1; diff --git a/tools/deploy/mysql_test/r/mysql/special_hook.result b/tools/deploy/mysql_test/r/mysql/special_hook.result index 140be67fb..9b6eea308 100644 --- a/tools/deploy/mysql_test/r/mysql/special_hook.result +++ b/tools/deploy/mysql_test/r/mysql/special_hook.result @@ -13,9 +13,9 @@ gbk_chinese_ci gbk 28 Yes Yes 1 gbk_bin gbk 87 Yes 1 utf16_general_ci utf16 54 Yes Yes 1 utf16_bin utf16 55 Yes 1 -utf8mb4_unicode_ci utf8mb4 224 Yes 1 -utf16_unicode_ci utf16 101 Yes 1 -gb18030_chinese_ci gb18030 248 Yes Yes 1 +utf8mb4_unicode_ci utf8mb4 224 Yes 8 +utf16_unicode_ci utf16 101 Yes 8 +gb18030_chinese_ci gb18030 248 Yes Yes 2 gb18030_bin gb18030 249 Yes 1 latin1_swedish_ci latin1 8 Yes Yes 1 latin1_bin latin1 47 Yes 1 @@ -26,6 +26,14 @@ gb18030_2022_radical_ci gb18030_2022 219 Yes 1 gb18030_2022_radical_cs gb18030_2022 220 Yes 1 gb18030_2022_stroke_ci gb18030_2022 221 Yes 1 gb18030_2022_stroke_cs gb18030_2022 222 Yes 1 +utf8mb4_croatian_ci utf8mb4 245 Yes 8 +utf8mb4_unicode_520_ci utf8mb4 246 Yes 8 +utf8mb4_czech_ci utf8mb4 234 Yes 8 +ascii_general_ci ascii 11 Yes Yes 1 +ascii_bin ascii 65 Yes 1 +tis620_thai_ci tis620 18 Yes Yes 1 +tis620_bin tis620 89 Yes 1 +utf8mb4_0900_ai_ci utf8mb4 255 Yes 1 SHOW CHARACTER SET; Charset Description Default collation Maxlen binary Binary pseudo charset binary 1 @@ -35,4 +43,6 @@ utf16 UTF-16 Unicode utf16_general_ci 2 gb18030 GB18030 charset gb18030_chinese_ci 4 latin1 cp1252 West European latin1_swedish_ci 1 gb18030_2022 GB18030-2022 charset gb18030_2022_chinese_ci 4 +ascii US ASCII ascii_general_ci 1 +tis620 TIS620 Thai tis620_thai_ci 1 SET NAMES latin1; diff --git a/tools/deploy/mysql_test/r/mysql/special_stmt.result b/tools/deploy/mysql_test/r/mysql/special_stmt.result index f92229d17..325b951ea 100644 --- a/tools/deploy/mysql_test/r/mysql/special_stmt.result +++ b/tools/deploy/mysql_test/r/mysql/special_stmt.result @@ -7,9 +7,9 @@ gbk_chinese_ci gbk 28 Yes Yes 1 gbk_bin gbk 87 Yes 1 utf16_general_ci utf16 54 Yes Yes 1 utf16_bin utf16 55 Yes 1 -utf8mb4_unicode_ci utf8mb4 224 Yes 1 -utf16_unicode_ci utf16 101 Yes 1 -gb18030_chinese_ci gb18030 248 Yes Yes 1 +utf8mb4_unicode_ci utf8mb4 224 Yes 8 +utf16_unicode_ci utf16 101 Yes 8 +gb18030_chinese_ci gb18030 248 Yes Yes 2 gb18030_bin gb18030 249 Yes 1 latin1_swedish_ci latin1 8 Yes Yes 1 latin1_bin latin1 47 Yes 1 @@ -20,6 +20,14 @@ gb18030_2022_radical_ci gb18030_2022 219 Yes 1 gb18030_2022_radical_cs gb18030_2022 220 Yes 1 gb18030_2022_stroke_ci gb18030_2022 221 Yes 1 gb18030_2022_stroke_cs gb18030_2022 222 Yes 1 +utf8mb4_croatian_ci utf8mb4 245 Yes 8 +utf8mb4_unicode_520_ci utf8mb4 246 Yes 8 +utf8mb4_czech_ci utf8mb4 234 Yes 8 +ascii_general_ci ascii 11 Yes Yes 1 +ascii_bin ascii 65 Yes 1 +tis620_thai_ci tis620 18 Yes Yes 1 +tis620_bin tis620 89 Yes 1 +utf8mb4_0900_ai_ci utf8mb4 255 Yes 1 show collation; Collation Charset Id Default Compiled Sortlen utf8mb4_general_ci utf8mb4 45 Yes Yes 1 @@ -29,9 +37,9 @@ gbk_chinese_ci gbk 28 Yes Yes 1 gbk_bin gbk 87 Yes 1 utf16_general_ci utf16 54 Yes Yes 1 utf16_bin utf16 55 Yes 1 -utf8mb4_unicode_ci utf8mb4 224 Yes 1 -utf16_unicode_ci utf16 101 Yes 1 -gb18030_chinese_ci gb18030 248 Yes Yes 1 +utf8mb4_unicode_ci utf8mb4 224 Yes 8 +utf16_unicode_ci utf16 101 Yes 8 +gb18030_chinese_ci gb18030 248 Yes Yes 2 gb18030_bin gb18030 249 Yes 1 latin1_swedish_ci latin1 8 Yes Yes 1 latin1_bin latin1 47 Yes 1 @@ -42,6 +50,14 @@ gb18030_2022_radical_ci gb18030_2022 219 Yes 1 gb18030_2022_radical_cs gb18030_2022 220 Yes 1 gb18030_2022_stroke_ci gb18030_2022 221 Yes 1 gb18030_2022_stroke_cs gb18030_2022 222 Yes 1 +utf8mb4_croatian_ci utf8mb4 245 Yes 8 +utf8mb4_unicode_520_ci utf8mb4 246 Yes 8 +utf8mb4_czech_ci utf8mb4 234 Yes 8 +ascii_general_ci ascii 11 Yes Yes 1 +ascii_bin ascii 65 Yes 1 +tis620_thai_ci tis620 18 Yes Yes 1 +tis620_bin tis620 89 Yes 1 +utf8mb4_0900_ai_ci utf8mb4 255 Yes 1 show collation test; ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your OceanBase version for the right syntax to use near 'test' at line 1 SHOW CHARACTER SET; @@ -53,6 +69,8 @@ utf16 UTF-16 Unicode utf16_general_ci 2 gb18030 GB18030 charset gb18030_chinese_ci 4 latin1 cp1252 West European latin1_swedish_ci 1 gb18030_2022 GB18030-2022 charset gb18030_2022_chinese_ci 4 +ascii US ASCII ascii_general_ci 1 +tis620 TIS620 Thai tis620_thai_ci 1 SHOW CHARACTER SET; Charset Description Default collation Maxlen binary Binary pseudo charset binary 1 @@ -62,6 +80,8 @@ utf16 UTF-16 Unicode utf16_general_ci 2 gb18030 GB18030 charset gb18030_chinese_ci 4 latin1 cp1252 West European latin1_swedish_ci 1 gb18030_2022 GB18030-2022 charset gb18030_2022_chinese_ci 4 +ascii US ASCII ascii_general_ci 1 +tis620 TIS620 Thai tis620_thai_ci 1 SHOW CHARACTER SET test; ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your OceanBase version for the right syntax to use near 'test' at line 1 set names utf8; diff --git a/tools/deploy/mysql_test/r/mysql/view_2.result b/tools/deploy/mysql_test/r/mysql/view_2.result index f8040d647..5137d6ae8 100644 --- a/tools/deploy/mysql_test/r/mysql/view_2.result +++ b/tools/deploy/mysql_test/r/mysql/view_2.result @@ -354,7 +354,8 @@ ERROR 42S02: 'view.v2' contains view recursion select * from v3; ERROR 42S02: 'view.v2' contains view recursion SET optimizer_switch = (SELECT variable_value FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE VARIABLE_NAME = 'optimizer_switch'); -ERROR 0A000: subqueries or stored function calls here not supported +Warnings: +Warning 1235 This system variable now is mock not supported drop table if exists t1,t2; drop view if exists v1; create table t1(c1 int,c2 int); diff --git a/tools/deploy/mysql_test/t/view_2.test b/tools/deploy/mysql_test/t/view_2.test index 4f67fb84a..0929dae14 100644 --- a/tools/deploy/mysql_test/t/view_2.test +++ b/tools/deploy/mysql_test/t/view_2.test @@ -435,7 +435,6 @@ select * from v3; ## 主要测试在非select语句引用select_stmt(view展开)的问题，select结果作为其它stmt的输入暂时不支持， ## 这里仅仅针对bug加入回归case # ---error 1235 SET optimizer_switch = (SELECT variable_value FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE VARIABLE_NAME = 'optimizer_switch'); --disable_warnings drop table if exists t1,t2; diff --git a/tools/deploy/mysql_test/test_suite/expr/r/mysql/collation_expr.result b/tools/deploy/mysql_test/test_suite/expr/r/mysql/collation_expr.result index 83a42a678..ced153f22 100644 --- a/tools/deploy/mysql_test/test_suite/expr/r/mysql/collation_expr.result +++ b/tools/deploy/mysql_test/test_suite/expr/r/mysql/collation_expr.result @@ -234,7 +234,7 @@ collation(replace(null, b, 'a')) binary select collation(replace(uc, b, ub)) from coll_test; collation(replace(uc, b, ub)) -binary +utf8mb4_general_ci select collation(replace(ub, uc, ub)) from coll_test; collation(replace(ub, uc, ub)) utf8mb4_bin @@ -252,7 +252,7 @@ collation(replace(null, b, 'a')) binary select collation(replace(uc, b, ub)) from coll_test; collation(replace(uc, b, ub)) -binary +utf8mb4_general_ci select collation(replace(ub, uc, ub)) from coll_test; collation(replace(ub, uc, ub)) utf8mb4_bin diff --git a/tools/deploy/mysql_test/test_suite/expr/r/mysql/func_regexp.result b/tools/deploy/mysql_test/test_suite/expr/r/mysql/func_regexp.result index 5e0c58c57..09d6ae776 100644 --- a/tools/deploy/mysql_test/test_suite/expr/r/mysql/func_regexp.result +++ b/tools/deploy/mysql_test/test_suite/expr/r/mysql/func_regexp.result @@ -6348,9 +6348,9 @@ select 'a' collate gbk_chinese_ci regexp 'A' collate gbk_chinese_ci; 'a' collate gbk_chinese_ci regexp 'A' collate gbk_chinese_ci 1 select 'a' collate gbk_bin regexp 'A' collate gbk_chinese_ci; -ERROR HY000: Illegal mix of collations +ERROR HY000: Illegal mix of collations (gbk_bin,EXPLICIT), (gbk_chinese_ci,EXPLICIT) select 'a' collate gbk_chinese_ci regexp 'A' collate gbk_bin; -ERROR HY000: Illegal mix of collations +ERROR HY000: Illegal mix of collations (gbk_chinese_ci,EXPLICIT), (gbk_bin,EXPLICIT) set names latin1; select 'a' collate latin1_bin regexp 'A'; 'a' collate latin1_bin regexp 'A' @@ -6371,9 +6371,9 @@ select 'a' collate latin1_swedish_ci regexp 'A' collate latin1_swedish_ci; 'a' collate latin1_swedish_ci regexp 'A' collate latin1_swedish_ci 1 select 'a' collate latin1_bin regexp 'A' collate latin1_swedish_ci; -ERROR HY000: Illegal mix of collations +ERROR HY000: Illegal mix of collations (latin1_bin,EXPLICIT), (latin1_swedish_ci,EXPLICIT) select 'a' collate latin1_swedish_ci regexp 'A' collate latin1_bin; -ERROR HY000: Illegal mix of collations +ERROR HY000: Illegal mix of collations (latin1_swedish_ci,EXPLICIT), (latin1_bin,EXPLICIT) drop database ly; create database ly character set GB18030; use ly; diff --git a/tools/deploy/mysql_test/test_suite/information_schema/r/mysql/information_schema_desc.result b/tools/deploy/mysql_test/test_suite/information_schema/r/mysql/information_schema_desc.result index 8c151d09a..047d5ba5a 100644 --- a/tools/deploy/mysql_test/test_suite/information_schema/r/mysql/information_schema_desc.result +++ b/tools/deploy/mysql_test/test_suite/information_schema/r/mysql/information_schema_desc.result @@ -79,9 +79,9 @@ PARTITION_DESCRIPTION text NO SUBPARTITION_DESCRIPTION text NO TABLE_ROWS bigint(20) unsigned NO NULL AVG_ROW_LENGTH bigint(21) unsigned NO NULL -DATA_LENGTH bigint(0) unsigned NO +DATA_LENGTH bigint(20) unsigned NO MAX_DATA_LENGTH bigint(0) unsigned NO -INDEX_LENGTH bigint(0) unsigned NO +INDEX_LENGTH bigint(21) unsigned NO DATA_FREE bigint(0) unsigned NO CREATE_TIME timestamp(6) YES NULL UPDATE_TIME datetime NO @@ -173,7 +173,7 @@ TABLE_ROWS bigint(20) unsigned NO AVG_ROW_LENGTH bigint(21) unsigned NO DATA_LENGTH bigint(20) unsigned NO MAX_DATA_LENGTH bigint(0) unsigned NO -INDEX_LENGTH bigint(0) unsigned NO +INDEX_LENGTH bigint(21) unsigned NO DATA_FREE bigint(0) unsigned NO AUTO_INCREMENT bigint(0) unsigned NO CREATE_TIME datetime NO @@ -248,13 +248,13 @@ View Create View character_set_client collation_connection KEY_COLUMN_USAGE CREATE VIEW `KEY_COLUMN_USAGE` AS (select 'def' as CONSTRAINT_CATALOG, c.database_name collate utf8mb4_name_case as CONSTRAINT_SCHEMA, 'PRIMARY' as CONSTRAINT_NAME, 'def' as TABLE_CATALOG, c.database_name collate utf8mb4_name_case as TABLE_SCHEMA, a.table_name collate utf8mb4_name_case as TABLE_NAME, b.column_name as COLUMN_NAME, b.rowkey_position as ORDINAL_POSITION, CAST(NULL AS UNSIGNED) as POSITION_IN_UNIQUE_CONSTRAINT, CAST(NULL AS CHAR(64)) as REFERENCED_TABLE_SCHEMA, CAST(NULL AS CHAR(64)) as REFERENCED_TABLE_NAME, CAST(NULL AS CHAR(64)) as REFERENCED_COLUMN_NAME from oceanbase.__all_table a join oceanbase.__all_column b on a.tenant_id = b.tenant_id and a.table_id = b.table_id join oceanbase.__all_database c on a.tenant_id = c.tenant_id and a.database_id = c.database_id where a.tenant_id = 0 and a.table_mode >> 12 & 15 in (0,1) and c.in_recyclebin = 0 and c.database_name != '__recyclebin' and b.rowkey_position > 0 and b.column_id >= 16 and a.table_type != 5 and a.table_type != 12 and a.table_type != 13 and b.column_flags & (0x1 << 8) = 0) union all (select 'def' as CONSTRAINT_CATALOG, d.database_name collate utf8mb4_name_case as CONSTRAINT_SCHEMA, substr(a.table_name, 2 + length(substring_index(a.table_name,'_',4))) as CONSTRAINT_NAME, 'def' as TABLE_CATALOG, d.database_name collate utf8mb4_name_case as TABLE_SCHEMA, c.table_name collate utf8mb4_name_case as TABLE_NAME, b.column_name as COLUMN_NAME, b.index_position as ORDINAL_POSITION, CAST(NULL AS UNSIGNED) as POSITION_IN_UNIQUE_CONSTRAINT, CAST(NULL AS CHAR(64)) as REFERENCED_TABLE_SCHEMA, CAST(NULL AS CHAR(64)) as REFERENCED_TABLE_NAME, CAST(NULL AS CHAR(64)) as REFERENCED_COLUMN_NAME from oceanbase.__all_table a join oceanbase.__all_column b on a.tenant_id = b.tenant_id and a.table_id = b.table_id join oceanbase.__all_table c on a.tenant_id = c.tenant_id and a.data_table_id = c.table_id join oceanbase.__all_database d on a.tenant_id = d.tenant_id and c.database_id = d.database_id where a.tenant_id = 0 and d.in_recyclebin = 0 and d.database_name != '__recyclebin' and a.table_type = 5 and a.index_type in (2, 4, 8) and b.index_position > 0) union all (select 'def' as CONSTRAINT_CATALOG, d.database_name collate utf8mb4_name_case as CONSTRAINT_SCHEMA, f.foreign_key_name as CONSTRAINT_NAME, 'def' as TABLE_CATALOG, d.database_name collate utf8mb4_name_case as TABLE_SCHEMA, t.table_name collate utf8mb4_name_case as TABLE_NAME, c.column_name as COLUMN_NAME, fc.position as ORDINAL_POSITION, CAST(fc.position AS UNSIGNED) as POSITION_IN_UNIQUE_CONSTRAINT, d2.database_name as REFERENCED_TABLE_SCHEMA, t2.table_name as REFERENCED_TABLE_NAME, c2.column_name as REFERENCED_COLUMN_NAME from oceanbase.__all_foreign_key f join oceanbase.__all_table t on f.tenant_id = t.tenant_id and f.child_table_id = t.table_id join oceanbase.__all_database d on f.tenant_id = d.tenant_id and t.database_id = d.database_id join oceanbase.__all_foreign_key_column fc on f.tenant_id = fc.tenant_id and f.foreign_key_id = fc.foreign_key_id join oceanbase.__all_column c on f.tenant_id = c.tenant_id and fc.child_column_id = c.column_id and t.table_id = c.table_id join oceanbase.__all_table t2 on f.tenant_id = t2.tenant_id and f.parent_table_id = t2.table_id join oceanbase.__all_database d2 on f.tenant_id = d2.tenant_id and t2.database_id = d2.database_id join oceanbase.__all_column c2 on f.tenant_id = c2.tenant_id and fc.parent_column_id = c2.column_id and t2.table_id = c2.table_id where f.tenant_id = 0) union all (select 'def' as CONSTRAINT_CATALOG, d.database_name collate utf8mb4_name_case as CONSTRAINT_SCHEMA, f.foreign_key_name as CONSTRAINT_NAME, 'def' as TABLE_CATALOG, d.database_name collate utf8mb4_name_case as TABLE_SCHEMA, t.table_name collate utf8mb4_name_case as TABLE_NAME, c.column_name as COLUMN_NAME, fc.position as ORDINAL_POSITION, CAST(fc.position AS UNSIGNED) as POSITION_IN_UNIQUE_CONSTRAINT, d.database_name as REFERENCED_TABLE_SCHEMA, t2.mock_fk_parent_table_name as REFERENCED_TABLE_NAME, c2.parent_column_name as REFERENCED_COLUMN_NAME from oceanbase.__all_foreign_key f join oceanbase.__all_table t on f.tenant_id = t.tenant_id and f.child_table_id = t.table_id join oceanbase.__all_database d on f.tenant_id = d.tenant_id and t.database_id = d.database_id join oceanbase.__all_foreign_key_column fc on f.tenant_id = fc.tenant_id and f.foreign_key_id = fc.foreign_key_id join oceanbase.__all_column c on f.tenant_id = c.tenant_id and fc.child_column_id = c.column_id and t.table_id = c.table_id join oceanbase.__all_mock_fk_parent_table t2 on f.tenant_id = t2.tenant_id and f.parent_table_id = t2.mock_fk_parent_table_id join oceanbase.__all_mock_fk_parent_table_column c2 on f.tenant_id = c2.tenant_id and fc.parent_column_id = c2.parent_column_id and t2.mock_fk_parent_table_id = c2.mock_fk_parent_table_id where f.tenant_id = 0) utf8mb4 utf8mb4_general_ci show create table partitions; View Create View character_set_client collation_connection -PARTITIONS CREATE VIEW `PARTITIONS` AS SELECT CAST('def' as CHAR(4096)) AS TABLE_CATALOG, DB.DATABASE_NAME collate utf8mb4_name_case AS TABLE_SCHEMA, T.TABLE_NAME collate utf8mb4_name_case AS TABLE_NAME, P.PART_NAME AS PARTITION_NAME, SP.SUB_PART_NAME AS SUBPARTITION_NAME, CAST(PART_POSITION AS UNSIGNED) AS PARTITION_ORDINAL_POSITION, CAST(SUB_PART_POSITION AS UNSIGNED) AS SUBPARTITION_ORDINAL_POSITION, CAST(CASE WHEN T.PART_LEVEL = 0 THEN NULL ELSE (CASE T.PART_FUNC_TYPE WHEN 0 THEN 'HASH' WHEN 1 THEN 'KEY' WHEN 2 THEN 'KEY' WHEN 3 THEN 'RANGE' WHEN 4 THEN 'RANGE COLUMNS' WHEN 5 THEN 'LIST' WHEN 6 THEN 'LIST COLUMNS' WHEN 7 THEN 'RANGE' END) END AS CHAR(13)) PARTITION_METHOD, CAST(CASE WHEN (T.PART_LEVEL = 0 OR T.PART_LEVEL = 1) THEN NULL ELSE (CASE T.SUB_PART_FUNC_TYPE WHEN 0 THEN 'HASH' WHEN 1 THEN 'KEY' WHEN 2 THEN 'KEY' WHEN 3 THEN 'RANGE' WHEN 4 THEN 'RANGE COLUMNS' WHEN 5 THEN 'LIST' WHEN 6 THEN 'LIST COLUMNS' WHEN 7 THEN 'RANGE' END) END AS CHAR(13)) SUBPARTITION_METHOD, CAST(CASE WHEN (T.PART_LEVEL = 0) THEN NULL ELSE T.PART_FUNC_EXPR END AS CHAR(2048)) PARTITION_EXPRESSION, CAST(CASE WHEN (T.PART_LEVEL = 0 OR T.PART_LEVEL = 1) THEN NULL ELSE T.SUB_PART_FUNC_EXPR END AS CHAR(2048)) SUBPARTITION_EXPRESSION, CAST(CASE WHEN (T.PART_LEVEL = 0) THEN NULL ELSE (CASE WHEN LENGTH(P.HIGH_BOUND_VAL) > 0 THEN P.HIGH_BOUND_VAL ELSE P.LIST_VAL END) END AS CHAR(4096)) AS PARTITION_DESCRIPTION, CAST(CASE WHEN (T.PART_LEVEL = 0 OR T.PART_LEVEL = 1) THEN NULL ELSE (CASE WHEN LENGTH(SP.HIGH_BOUND_VAL) > 0 THEN SP.HIGH_BOUND_VAL ELSE SP.LIST_VAL END) END AS CHAR(4096)) AS SUBPARTITION_DESCRIPTION, CAST(TS.ROW_CNT AS UNSIGNED) AS TABLE_ROWS, CAST(TS.AVG_ROW_LEN AS UNSIGNED) AS AVG_ROW_LENGTH, CAST(NULL AS UNSIGNED) AS DATA_LENGTH, CAST(NULL AS UNSIGNED) AS MAX_DATA_LENGTH, CAST(NULL AS UNSIGNED) AS INDEX_LENGTH, CAST(NULL AS UNSIGNED) AS DATA_FREE, CASE T.PART_LEVEL WHEN 0 THEN T.GMT_CREATE WHEN 1 THEN P.GMT_CREATE WHEN 2 THEN SP.GMT_CREATE END AS CREATE_TIME, CAST(NULL AS DATETIME) AS UPDATE_TIME, CAST(NULL AS DATETIME) AS CHECK_TIME, CAST(NULL AS SIGNED) AS CHECKSUM, CAST(CASE T.PART_LEVEL WHEN 0 THEN NULL WHEN 1 THEN P.COMMENT WHEN 2 THEN SP.COMMENT END AS CHAR(1024)) AS PARTITION_COMMENT, CAST('default' AS CHAR(256)) NODEGROUP, CAST(TP.TABLESPACE_NAME AS CHAR(268)) AS TABLESPACE_NAME FROM OCEANBASE.__ALL_TABLE T JOIN OCEANBASE.__ALL_DATABASE DB ON T.DATABASE_ID = DB.DATABASE_ID AND T.TENANT_ID = DB.TENANT_ID AND T.TABLE_MODE >> 12 & 15 in (0,1) LEFT JOIN ( SELECT TENANT_ID, TABLE_ID, PART_ID, PART_NAME, HIGH_BOUND_VAL, LIST_VAL, TABLESPACE_ID, GMT_CREATE, COMMENT, ROW_NUMBER() OVER(PARTITION BY TENANT_ID,TABLE_ID ORDER BY PART_IDX) AS PART_POSITION FROM OCEANBASE.__ALL_PART ) P ON T.TABLE_ID = P.TABLE_ID AND T.TENANT_ID = P.TENANT_ID LEFT JOIN ( SELECT TENANT_ID, TABLE_ID, PART_ID, SUB_PART_ID, SUB_PART_NAME, HIGH_BOUND_VAL, LIST_VAL, TABLESPACE_ID, GMT_CREATE, COMMENT, ROW_NUMBER() OVER(PARTITION BY TENANT_ID,TABLE_ID,PART_ID ORDER BY SUB_PART_IDX) AS SUB_PART_POSITION FROM OCEANBASE.__ALL_SUB_PART ) SP ON T.TABLE_ID = SP.TABLE_ID AND P.PART_ID = SP.PART_ID AND T.TENANT_ID = SP.TENANT_ID LEFT JOIN OCEANBASE.__ALL_TENANT_TABLESPACE TP ON TP.TABLESPACE_ID = IFNULL(SP.TABLESPACE_ID, P.TABLESPACE_ID) AND TP.TENANT_ID = T.TENANT_ID LEFT JOIN OCEANBASE.__ALL_TABLE_STAT TS ON T.TENANT_ID = TS.TENANT_ID AND TS.TABLE_ID = T.TABLE_ID AND TS.PARTITION_ID = CASE T.PART_LEVEL WHEN 0 THEN T.TABLE_ID WHEN 1 THEN P.PART_ID WHEN 2 THEN SP.SUB_PART_ID END WHERE T.TABLE_TYPE IN (3,6,8,9,14,15) utf8mb4 utf8mb4_general_ci +PARTITIONS CREATE VIEW `PARTITIONS` AS SELECT CAST('def' as CHAR(4096)) AS TABLE_CATALOG, DB.DATABASE_NAME collate utf8mb4_name_case AS TABLE_SCHEMA, T.TABLE_NAME collate utf8mb4_name_case AS TABLE_NAME, P.PART_NAME AS PARTITION_NAME, SP.SUB_PART_NAME AS SUBPARTITION_NAME, CAST(PART_POSITION AS UNSIGNED) AS PARTITION_ORDINAL_POSITION, CAST(SUB_PART_POSITION AS UNSIGNED) AS SUBPARTITION_ORDINAL_POSITION, CAST(CASE WHEN T.PART_LEVEL = 0 THEN NULL ELSE (CASE T.PART_FUNC_TYPE WHEN 0 THEN 'HASH' WHEN 1 THEN 'KEY' WHEN 2 THEN 'KEY' WHEN 3 THEN 'RANGE' WHEN 4 THEN 'RANGE COLUMNS' WHEN 5 THEN 'LIST' WHEN 6 THEN 'LIST COLUMNS' WHEN 7 THEN 'RANGE' END) END AS CHAR(13)) PARTITION_METHOD, CAST(CASE WHEN (T.PART_LEVEL = 0 OR T.PART_LEVEL = 1) THEN NULL ELSE (CASE T.SUB_PART_FUNC_TYPE WHEN 0 THEN 'HASH' WHEN 1 THEN 'KEY' WHEN 2 THEN 'KEY' WHEN 3 THEN 'RANGE' WHEN 4 THEN 'RANGE COLUMNS' WHEN 5 THEN 'LIST' WHEN 6 THEN 'LIST COLUMNS' WHEN 7 THEN 'RANGE' END) END AS CHAR(13)) SUBPARTITION_METHOD, CAST(CASE WHEN (T.PART_LEVEL = 0) THEN NULL ELSE T.PART_FUNC_EXPR END AS CHAR(2048)) PARTITION_EXPRESSION, CAST(CASE WHEN (T.PART_LEVEL = 0 OR T.PART_LEVEL = 1) THEN NULL ELSE T.SUB_PART_FUNC_EXPR END AS CHAR(2048)) SUBPARTITION_EXPRESSION, CAST(CASE WHEN (T.PART_LEVEL = 0) THEN NULL ELSE (CASE WHEN LENGTH(P.HIGH_BOUND_VAL) > 0 THEN P.HIGH_BOUND_VAL ELSE P.LIST_VAL END) END AS CHAR(4096)) AS PARTITION_DESCRIPTION, CAST(CASE WHEN (T.PART_LEVEL = 0 OR T.PART_LEVEL = 1) THEN NULL ELSE (CASE WHEN LENGTH(SP.HIGH_BOUND_VAL) > 0 THEN SP.HIGH_BOUND_VAL ELSE SP.LIST_VAL END) END AS CHAR(4096)) AS SUBPARTITION_DESCRIPTION, CAST(TS.ROW_CNT AS UNSIGNED) AS TABLE_ROWS, CAST(TS.AVG_ROW_LEN AS UNSIGNED) AS AVG_ROW_LENGTH, CAST(COALESCE(TS.MACRO_BLK_CNT * 2 * 1024 * 1024, 0) AS UNSIGNED) AS DATA_LENGTH, CAST(NULL AS UNSIGNED) AS MAX_DATA_LENGTH, CAST(COALESCE(IDX_STAT.INDEX_LENGTH, 0) AS UNSIGNED) AS INDEX_LENGTH, CAST(NULL AS UNSIGNED) AS DATA_FREE, CASE T.PART_LEVEL WHEN 0 THEN T.GMT_CREATE WHEN 1 THEN P.GMT_CREATE WHEN 2 THEN SP.GMT_CREATE END AS CREATE_TIME, CAST(NULL AS DATETIME) AS UPDATE_TIME, CAST(NULL AS DATETIME) AS CHECK_TIME, CAST(NULL AS SIGNED) AS CHECKSUM, CAST(CASE T.PART_LEVEL WHEN 0 THEN NULL WHEN 1 THEN P.COMMENT WHEN 2 THEN SP.COMMENT END AS CHAR(1024)) AS PARTITION_COMMENT, CAST('default' AS CHAR(256)) NODEGROUP, CAST(TP.TABLESPACE_NAME AS CHAR(268)) AS TABLESPACE_NAME FROM OCEANBASE.__ALL_TABLE T JOIN OCEANBASE.__ALL_DATABASE DB ON T.DATABASE_ID = DB.DATABASE_ID AND T.TENANT_ID = DB.TENANT_ID AND T.TABLE_MODE >> 12 & 15 in (0,1) LEFT JOIN ( SELECT TENANT_ID, TABLE_ID, PART_ID, PART_NAME, HIGH_BOUND_VAL, LIST_VAL, TABLESPACE_ID, GMT_CREATE, COMMENT, PART_IDX, ROW_NUMBER() OVER(PARTITION BY TENANT_ID,TABLE_ID ORDER BY PART_IDX) AS PART_POSITION FROM OCEANBASE.__ALL_PART ) P ON T.TABLE_ID = P.TABLE_ID AND T.TENANT_ID = P.TENANT_ID LEFT JOIN ( SELECT TENANT_ID, TABLE_ID, PART_ID, SUB_PART_ID, SUB_PART_NAME, HIGH_BOUND_VAL, LIST_VAL, TABLESPACE_ID, GMT_CREATE, COMMENT, SUB_PART_IDX, ROW_NUMBER() OVER(PARTITION BY TENANT_ID,TABLE_ID,PART_ID ORDER BY SUB_PART_IDX) AS SUB_PART_POSITION FROM OCEANBASE.__ALL_SUB_PART ) SP ON T.TABLE_ID = SP.TABLE_ID AND P.PART_ID = SP.PART_ID AND T.TENANT_ID = SP.TENANT_ID LEFT JOIN OCEANBASE.__ALL_TENANT_TABLESPACE TP ON TP.TABLESPACE_ID = IFNULL(SP.TABLESPACE_ID, P.TABLESPACE_ID) AND TP.TENANT_ID = T.TENANT_ID LEFT JOIN OCEANBASE.__ALL_TABLE_STAT TS ON T.TENANT_ID = TS.TENANT_ID AND TS.TABLE_ID = T.TABLE_ID AND TS.PARTITION_ID = CASE T.PART_LEVEL WHEN 0 THEN T.TABLE_ID WHEN 1 THEN P.PART_ID WHEN 2 THEN SP.SUB_PART_ID END LEFT JOIN ( SELECT E.TENANT_ID AS TENANT_ID, E.DATA_TABLE_ID AS DATA_TABLE_ID, F.PART_IDX AS PART_IDX, SF.SUB_PART_IDX AS SUB_PART_IDX, SUM(G.macro_blk_cnt * 2 * 1024 * 1024) AS INDEX_LENGTH FROM OCEANBASE.__ALL_TABLE E LEFT JOIN OCEANBASE.__ALL_PART F ON E.TENANT_ID = F.TENANT_ID AND E.TABLE_ID = F.TABLE_ID LEFT JOIN OCEANBASE.__ALL_SUB_PART SF ON E.TENANT_ID = SF.TENANT_ID AND E.TABLE_ID = SF.TABLE_ID AND F.PART_ID = SF.PART_ID JOIN OCEANBASE.__ALL_TABLE_STAT G ON E.TENANT_ID = G.TENANT_ID AND E.TABLE_ID = G.TABLE_ID AND G.PARTITION_ID = CASE E.PART_LEVEL WHEN 0 THEN E.TABLE_ID WHEN 1 THEN F.PART_ID WHEN 2 THEN SF.SUB_PART_ID END WHERE E.INDEX_TYPE in (1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12) AND E.TABLE_TYPE = 5 GROUP BY TENANT_ID, DATA_TABLE_ID, PART_IDX, SUB_PART_IDX ) IDX_STAT ON IDX_STAT.TENANT_ID = T.TENANT_ID AND IDX_STAT.DATA_TABLE_ID = T.TABLE_ID AND CASE T.PART_LEVEL WHEN 0 THEN 1 WHEN 1 THEN P.PART_IDX = IDX_STAT.PART_IDX WHEN 2 THEN P.PART_IDX = IDX_STAT.PART_IDX AND SP.SUB_PART_IDX = IDX_STAT.SUB_PART_IDX END WHERE T.TABLE_TYPE IN (3,6,8,9,14,15) utf8mb4 utf8mb4_general_ci show create table processlist; View Create View character_set_client collation_connection PROCESSLIST CREATE VIEW `PROCESSLIST` AS SELECT id AS ID, user AS USER, concat(user_client_ip, ':', user_client_port) AS HOST, db AS DB, command AS COMMAND, cast(time as SIGNED) AS TIME, state AS STATE, info AS INFO FROM oceanbase.__all_virtual_processlist WHERE is_serving_tenant(svr_ip, svr_port, effective_tenant_id()) utf8mb4 utf8mb4_general_ci show create table schema_privileges; View Create View character_set_client collation_connection -SCHEMA_PRIVILEGES CREATE VIEW `SCHEMA_PRIVILEGES` AS WITH DB_PRIV AS ( select A.tenant_id TENANT_ID, A.user_id USER_ID, A.database_name DATABASE_NAME, A.priv_alter PRIV_ALTER, A.priv_create PRIV_CREATE, A.priv_delete PRIV_DELETE, A.priv_drop PRIV_DROP, A.priv_grant_option PRIV_GRANT_OPTION, A.priv_insert PRIV_INSERT, A.priv_update PRIV_UPDATE, A.priv_select PRIV_SELECT, A.priv_index PRIV_INDEX, A.priv_create_view PRIV_CREATE_VIEW, A.priv_show_view PRIV_SHOW_VIEW, A.priv_others PRIV_OTHERS from oceanbase.__all_database_privilege_history A, (select tenant_id, user_id, database_name, max(schema_version) schema_version from oceanbase.__all_database_privilege_history group by tenant_id, user_id, database_name, database_name collate utf8mb4_bin) B where A.tenant_id = B.tenant_id and A.user_id = B.user_id and A.database_name collate utf8mb4_bin = B.database_name collate utf8mb4_bin and A.schema_version = B.schema_version and A.is_deleted = 0 ) SELECT CAST(CONCAT('''', V.USER_NAME, '''', '@', '''', V.HOST, '''') AS CHAR(81)) AS GRANTEE , CAST('def' AS CHAR(512)) AS TABLE_CATALOG , CAST(V.DATABASE_NAME AS CHAR(128)) collate utf8mb4_name_case AS TABLE_SCHEMA , CAST(V.PRIVILEGE_TYPE AS CHAR(64)) AS PRIVILEGE_TYPE , CAST(V.IS_GRANTABLE AS CHAR(3)) AS IS_GRANTABLE FROM (SELECT DP.DATABASE_NAME DATABASE_NAME, U.USER_NAME AS USER_NAME, U.HOST AS HOST, CASE WHEN V1.C1 = 1 AND DP.PRIV_ALTER = 1 THEN 'ALTER' WHEN V1.C1 = 2 AND DP.PRIV_CREATE = 1 THEN 'CREATE' WHEN V1.C1 = 4 AND DP.PRIV_DELETE = 1 THEN 'DELETE' WHEN V1.C1 = 5 AND DP.PRIV_DROP = 1 THEN 'DROP' WHEN V1.C1 = 7 AND DP.PRIV_INSERT = 1 THEN 'INSERT' WHEN V1.C1 = 8 AND DP.PRIV_UPDATE = 1 THEN 'UPDATE' WHEN V1.C1 = 9 AND DP.PRIV_SELECT = 1 THEN 'SELECT' WHEN V1.C1 = 10 AND DP.PRIV_INDEX = 1 THEN 'INDEX' WHEN V1.C1 = 11 AND DP.PRIV_CREATE_VIEW = 1 THEN 'CREATE VIEW' WHEN V1.C1 = 12 AND DP.PRIV_SHOW_VIEW = 1 THEN 'SHOW VIEW' WHEN V1.C1 = 13 AND (U.PRIV_OTHERS & (1 << 0)) != 0 THEN 'EXECUTE' WHEN V1.C1 = 14 AND (U.PRIV_OTHERS & (1 << 1)) != 0 THEN 'ALTER ROUTINE' WHEN V1.C1 = 15 AND (U.PRIV_OTHERS & (1 << 2)) != 0 THEN 'CREATE ROUTINE' ELSE NULL END PRIVILEGE_TYPE , CASE WHEN DP.PRIV_GRANT_OPTION = 1 THEN 'YES' WHEN DP.PRIV_GRANT_OPTION = 0 THEN 'NO' END IS_GRANTABLE FROM DB_PRIV DP, oceanbase.__all_user U, (SELECT 1 AS C1 UNION ALL SELECT 2 AS C1 UNION ALL SELECT 4 AS C1 UNION ALL SELECT 5 AS C1 UNION ALL SELECT 7 AS C1 UNION ALL SELECT 8 AS C1 UNION ALL SELECT 9 AS C1 UNION ALL SELECT 10 AS C1 UNION ALL SELECT 11 AS C1 UNION ALL SELECT 12 AS C1 UNION ALL SELECT 13 AS C1 UNION ALL SELECT 14 AS C1 UNION ALL SELECT 15 AS C1) V1, (SELECT USER_ID FROM oceanbase.__all_user WHERE TENANT_ID= 0 AND CONCAT(USER_NAME, '@', HOST) = CURRENT_USER()) CURR LEFT JOIN (SELECT USER_ID FROM DB_PRIV WHERE TENANT_ID = 0 AND DATABASE_NAME = 'mysql' AND PRIV_SELECT = 1) DB ON CURR.USER_ID = DB.USER_ID WHERE DP.TENANT_ID = 0 AND DP.TENANT_ID = U.TENANT_ID AND DP.USER_ID = U.USER_ID AND DP.DATABASE_NAME != '__recyclebin' AND DP.DATABASE_NAME != '__public' AND DP.DATABASE_NAME != 'SYS' AND DP.DATABASE_NAME != 'LBACSYS' AND DP.DATABASE_NAME != 'ORAAUDITOR' AND (DB.USER_ID IS NOT NULL OR 512 & CURRENT_USER_PRIV() = 512 OR DP.USER_ID = CURR.USER_ID)) V WHERE V.PRIVILEGE_TYPE IS NOT NULL utf8mb4 utf8mb4_general_ci +SCHEMA_PRIVILEGES CREATE VIEW `SCHEMA_PRIVILEGES` AS WITH DB_PRIV AS ( select A.tenant_id TENANT_ID, A.user_id USER_ID, A.database_name DATABASE_NAME, A.priv_alter PRIV_ALTER, A.priv_create PRIV_CREATE, A.priv_delete PRIV_DELETE, A.priv_drop PRIV_DROP, A.priv_grant_option PRIV_GRANT_OPTION, A.priv_insert PRIV_INSERT, A.priv_update PRIV_UPDATE, A.priv_select PRIV_SELECT, A.priv_index PRIV_INDEX, A.priv_create_view PRIV_CREATE_VIEW, A.priv_show_view PRIV_SHOW_VIEW, A.priv_others PRIV_OTHERS from oceanbase.__all_database_privilege_history A, (select tenant_id, user_id, database_name, max(schema_version) schema_version from oceanbase.__all_database_privilege_history group by tenant_id, user_id, database_name, database_name collate utf8mb4_bin) B where A.tenant_id = B.tenant_id and A.user_id = B.user_id and A.database_name collate utf8mb4_bin = B.database_name collate utf8mb4_bin and A.schema_version = B.schema_version and A.is_deleted = 0 ) SELECT CAST(CONCAT('''', V.USER_NAME, '''', '@', '''', V.HOST, '''') AS CHAR(81)) AS GRANTEE , CAST('def' AS CHAR(512)) AS TABLE_CATALOG , CAST(V.DATABASE_NAME AS CHAR(128)) collate utf8mb4_name_case AS TABLE_SCHEMA , CAST(V.PRIVILEGE_TYPE AS CHAR(64)) AS PRIVILEGE_TYPE , CAST(V.IS_GRANTABLE AS CHAR(3)) AS IS_GRANTABLE FROM (SELECT DP.DATABASE_NAME DATABASE_NAME, U.USER_NAME AS USER_NAME, U.HOST AS HOST, CASE WHEN V1.C1 = 1 AND DP.PRIV_ALTER = 1 THEN 'ALTER' WHEN V1.C1 = 2 AND DP.PRIV_CREATE = 1 THEN 'CREATE' WHEN V1.C1 = 4 AND DP.PRIV_DELETE = 1 THEN 'DELETE' WHEN V1.C1 = 5 AND DP.PRIV_DROP = 1 THEN 'DROP' WHEN V1.C1 = 7 AND DP.PRIV_INSERT = 1 THEN 'INSERT' WHEN V1.C1 = 8 AND DP.PRIV_UPDATE = 1 THEN 'UPDATE' WHEN V1.C1 = 9 AND DP.PRIV_SELECT = 1 THEN 'SELECT' WHEN V1.C1 = 10 AND DP.PRIV_INDEX = 1 THEN 'INDEX' WHEN V1.C1 = 11 AND DP.PRIV_CREATE_VIEW = 1 THEN 'CREATE VIEW' WHEN V1.C1 = 12 AND DP.PRIV_SHOW_VIEW = 1 THEN 'SHOW VIEW' WHEN V1.C1 = 22 AND (DP.PRIV_OTHERS & (1 << 6)) != 0 THEN 'REFERENCES' WHEN V1.C1 = 23 AND (DP.PRIV_OTHERS & (1 << 0)) != 0 THEN 'EXECUTE' WHEN V1.C1 = 37 AND (DP.PRIV_OTHERS & (1 << 1)) != 0 THEN 'ALTER ROUTINE' WHEN V1.C1 = 38 AND (DP.PRIV_OTHERS & (1 << 2)) != 0 THEN 'CREATE ROUTINE' WHEN V1.C1 = 44 AND (DP.PRIV_OTHERS & (1 << 9)) != 0 THEN 'TRIGGER' ELSE NULL END PRIVILEGE_TYPE , CASE WHEN DP.PRIV_GRANT_OPTION = 1 THEN 'YES' WHEN DP.PRIV_GRANT_OPTION = 0 THEN 'NO' END IS_GRANTABLE FROM DB_PRIV DP, oceanbase.__all_user U, (SELECT 1 AS C1 UNION ALL SELECT 2 AS C1 UNION ALL SELECT 4 AS C1 UNION ALL SELECT 5 AS C1 UNION ALL SELECT 7 AS C1 UNION ALL SELECT 8 AS C1 UNION ALL SELECT 9 AS C1 UNION ALL SELECT 10 AS C1 UNION ALL SELECT 11 AS C1 UNION ALL SELECT 12 AS C1 UNION ALL SELECT 22 AS C1 UNION ALL SELECT 23 AS C1 UNION ALL SELECT 37 AS C1 UNION ALL SELECT 38 AS C1 UNION ALL SELECT 44 AS C1) V1, (SELECT USER_ID FROM oceanbase.__all_user WHERE TENANT_ID= 0 AND CONCAT(USER_NAME, '@', HOST) = CURRENT_USER()) CURR LEFT JOIN (SELECT USER_ID FROM DB_PRIV WHERE TENANT_ID = 0 AND DATABASE_NAME = 'mysql' AND PRIV_SELECT = 1) DB ON CURR.USER_ID = DB.USER_ID WHERE DP.TENANT_ID = 0 AND DP.TENANT_ID = U.TENANT_ID AND DP.USER_ID = U.USER_ID AND DP.DATABASE_NAME != '__recyclebin' AND DP.DATABASE_NAME != '__public' AND DP.DATABASE_NAME != 'SYS' AND DP.DATABASE_NAME != 'LBACSYS' AND DP.DATABASE_NAME != 'ORAAUDITOR' AND (DB.USER_ID IS NOT NULL OR 512 & CURRENT_USER_PRIV() = 512 OR DP.USER_ID = CURR.USER_ID)) V WHERE V.PRIVILEGE_TYPE IS NOT NULL utf8mb4 utf8mb4_general_ci show create table schemata; View Create View character_set_client collation_connection SCHEMATA CREATE VIEW `SCHEMATA` AS SELECT 'def' AS CATALOG_NAME, DATABASE_NAME collate utf8mb4_name_case AS SCHEMA_NAME, b.charset AS DEFAULT_CHARACTER_SET_NAME, b.collation AS DEFAULT_COLLATION_NAME, CAST(NULL AS CHAR(512)) as SQL_PATH, 'NO' as DEFAULT_ENCRYPTION FROM oceanbase.__all_database a inner join oceanbase.__tenant_virtual_collation b ON a.collation_type = b.collation_type WHERE a.tenant_id = 0 and in_recyclebin = 0 and a.database_name not in ('__recyclebin', '__public') and 0 = sys_privilege_check('db_acc', 0, a.database_name, '') ORDER BY a.database_id utf8mb4 utf8mb4_general_ci @@ -278,13 +278,13 @@ View Create View character_set_client collation_connection TABLE_CONSTRAINTS CREATE VIEW `TABLE_CONSTRAINTS` AS SELECT CAST('def' AS CHAR(64)) AS CONSTRAINT_CATALOG, CAST(d.database_name AS CHAR(128)) collate utf8mb4_name_case AS CONSTRAINT_SCHEMA, CAST('PRIMARY' AS CHAR(256)) AS CONSTRAINT_NAME, CAST(d.database_name AS CHAR(128)) collate utf8mb4_name_case AS TABLE_SCHEMA, CAST(t.table_name AS CHAR(256)) collate utf8mb4_name_case AS TABLE_NAME, CAST('PRIMARY KEY' AS CHAR(11)) AS CONSTRAINT_TYPE, CAST('YES' AS CHAR(3)) AS ENFORCED FROM oceanbase.__all_database d JOIN oceanbase.__all_table t ON d.database_id = t.database_id WHERE (d.database_id = 201003 OR d.database_id > 500000) AND d.in_recyclebin = 0 AND t.table_type = 3 AND t.table_mode >> 16 & 1 = 0 AND t.table_mode >> 12 & 15 in (0,1) union all SELECT CAST('def' AS CHAR(64)) AS CONSTRAINT_CATALOG, CAST(d.database_name AS CHAR(128)) collate utf8mb4_name_case AS CONSTRAINT_SCHEMA, CAST(SUBSTR(it.table_name, 7 + INSTR(SUBSTR(it.table_name, 7), '_')) AS CHAR(256)) AS CONSTRAINT_NAME, CAST(d.database_name AS CHAR(128)) collate utf8mb4_name_case AS TABLE_SCHEMA, CAST(ut.table_name AS CHAR(256)) collate utf8mb4_name_case AS TABLE_NAME, CAST('UNIQUE' AS CHAR(11)) AS CONSTRAINT_TYPE, CAST('YES' AS CHAR(3)) AS ENFORCED FROM oceanbase.__all_database d JOIN oceanbase.__all_table it ON d.database_id = it.database_id JOIN oceanbase.__all_table ut ON it.data_table_id = ut.table_id WHERE d.database_id > 500000 AND d.in_recyclebin = 0 AND it.table_type = 5 AND it.index_type IN (2, 4, 8) union all SELECT CAST('def' AS CHAR(64)) AS CONSTRAINT_CATALOG, CAST(d.database_name AS CHAR(128)) collate utf8mb4_name_case AS CONSTRAINT_SCHEMA, CAST(c.constraint_name AS CHAR(256)) AS CONSTRAINT_NAME, CAST(d.database_name AS CHAR(128)) collate utf8mb4_name_case AS TABLE_SCHEMA, CAST(t.table_name AS CHAR(256)) collate utf8mb4_name_case AS TABLE_NAME, CAST('CHECK' AS CHAR(11)) AS CONSTRAINT_TYPE, CAST(CASE WHEN c.enable_flag = 1 THEN 'YES' ELSE 'NO' END AS CHAR(3)) AS ENFORCED FROM oceanbase.__all_database d JOIN oceanbase.__all_table t ON d.database_id = t.database_id JOIN oceanbase.__all_constraint c ON t.table_id = c.table_id WHERE d.database_id > 500000 AND d.in_recyclebin = 0 AND t.table_type = 3 AND c.constraint_type = 3 union all SELECT CAST('def' AS CHAR(64)) AS CONSTRAINT_CATALOG, CAST(f.constraint_schema AS CHAR(128)) collate utf8mb4_name_case AS CONSTRAINT_SCHEMA, CAST(f.constraint_name AS CHAR(256)) AS CONSTRAINT_NAME, CAST(f.constraint_schema AS CHAR(128)) collate utf8mb4_name_case AS TABLE_SCHEMA, CAST(f.table_name AS CHAR(256)) collate utf8mb4_name_case AS TABLE_NAME, CAST('FOREIGN KEY' AS CHAR(11)) AS CONSTRAINT_TYPE, CAST('YES' AS CHAR(3)) AS ENFORCED FROM information_schema.REFERENTIAL_CONSTRAINTS f utf8mb4 utf8mb4_general_ci show create table table_privileges; View Create View character_set_client collation_connection -TABLE_PRIVILEGES CREATE VIEW `TABLE_PRIVILEGES` AS WITH DB_PRIV AS ( select A.tenant_id TENANT_ID, A.user_id USER_ID, A.database_name DATABASE_NAME, A.priv_alter PRIV_ALTER, A.priv_create PRIV_CREATE, A.priv_delete PRIV_DELETE, A.priv_drop PRIV_DROP, A.priv_grant_option PRIV_GRANT_OPTION, A.priv_insert PRIV_INSERT, A.priv_update PRIV_UPDATE, A.priv_select PRIV_SELECT, A.priv_index PRIV_INDEX, A.priv_create_view PRIV_CREATE_VIEW, A.priv_show_view PRIV_SHOW_VIEW, A.GMT_CREATE GMT_CREATE, A.GMT_MODIFIED GMT_MODIFIED, A.PRIV_OTHERS PRIV_OTHERS from oceanbase.__all_database_privilege_history A, (select tenant_id, user_id, database_name, max(schema_version) schema_version from oceanbase.__all_database_privilege_history group by tenant_id, user_id, database_name, database_name collate utf8mb4_bin) B where A.tenant_id = B.tenant_id and A.user_id = B.user_id and A.database_name collate utf8mb4_bin = B.database_name collate utf8mb4_bin and A.schema_version = B.schema_version and A.is_deleted = 0 ), TABLE_PRIV AS ( select A.tenant_id TENANT_ID, A.user_id USER_ID, A.database_name DATABASE_NAME, A.table_name TABLE_NAME, A.priv_alter PRIV_ALTER, A.priv_create PRIV_CREATE, A.priv_delete PRIV_DELETE, A.priv_drop PRIV_DROP, A.priv_grant_option PRIV_GRANT_OPTION, A.priv_insert PRIV_INSERT, A.priv_update PRIV_UPDATE, A.priv_select PRIV_SELECT, A.priv_index PRIV_INDEX, A.priv_create_view PRIV_CREATE_VIEW, A.priv_show_view PRIV_SHOW_VIEW, A.PRIV_OTHERS PRIV_OTHERS from oceanbase.__all_table_privilege_history A, (select tenant_id, user_id, database_name, table_name, max(schema_version) schema_version from oceanbase.__all_table_privilege_history group by tenant_id, user_id, database_name, database_name collate utf8mb4_bin, table_name, table_name collate utf8mb4_bin) B where A.tenant_id = B.tenant_id and A.user_id = B.user_id and A.database_name collate utf8mb4_bin = B.database_name collate utf8mb4_bin and A.schema_version = B.schema_version and A.table_name collate utf8mb4_bin = B.table_name collate utf8mb4_bin and A.is_deleted = 0 ) SELECT CAST(CONCAT('''', V.USER_NAME, '''', '@', '''', V.HOST, '''') AS CHAR(81)) AS GRANTEE , CAST('def' AS CHAR(512)) AS TABLE_CATALOG , CAST(V.DATABASE_NAME AS CHAR(128)) collate utf8mb4_name_case AS TABLE_SCHEMA , CAST(V.TABLE_NAME AS CHAR(64)) collate utf8mb4_name_case AS TABLE_NAME, CAST(V.PRIVILEGE_TYPE AS CHAR(64)) AS PRIVILEGE_TYPE , CAST(V.IS_GRANTABLE AS CHAR(3)) AS IS_GRANTABLE FROM (SELECT TP.DATABASE_NAME AS DATABASE_NAME, TP.TABLE_NAME AS TABLE_NAME, U.USER_NAME AS USER_NAME, U.HOST AS HOST, CASE WHEN V1.C1 = 1 AND TP.PRIV_ALTER = 1 THEN 'ALTER' WHEN V1.C1 = 2 AND TP.PRIV_CREATE = 1 THEN 'CREATE' WHEN V1.C1 = 4 AND TP.PRIV_DELETE = 1 THEN 'DELETE' WHEN V1.C1 = 5 AND TP.PRIV_DROP = 1 THEN 'DROP' WHEN V1.C1 = 7 AND TP.PRIV_INSERT = 1 THEN 'INSERT' WHEN V1.C1 = 8 AND TP.PRIV_UPDATE = 1 THEN 'UPDATE' WHEN V1.C1 = 9 AND TP.PRIV_SELECT = 1 THEN 'SELECT' WHEN V1.C1 = 10 AND TP.PRIV_INDEX = 1 THEN 'INDEX' WHEN V1.C1 = 11 AND TP.PRIV_CREATE_VIEW = 1 THEN 'CREATE VIEW' WHEN V1.C1 = 12 AND TP.PRIV_SHOW_VIEW = 1 THEN 'SHOW VIEW' ELSE NULL END PRIVILEGE_TYPE , CASE WHEN TP.PRIV_GRANT_OPTION = 1 THEN 'YES' WHEN TP.PRIV_GRANT_OPTION = 0 THEN 'NO' END IS_GRANTABLE FROM TABLE_PRIV TP, oceanbase.__all_user U, (SELECT 1 AS C1 UNION ALL SELECT 2 AS C1 UNION ALL SELECT 4 AS C1 UNION ALL SELECT 5 AS C1 UNION ALL SELECT 7 AS C1 UNION ALL SELECT 8 AS C1 UNION ALL SELECT 9 AS C1 UNION ALL SELECT 10 AS C1 UNION ALL SELECT 11 AS C1 UNION ALL SELECT 12 AS C1) V1, (SELECT USER_ID FROM oceanbase.__all_user WHERE TENANT_ID = 0 AND CONCAT(USER_NAME, '@', HOST) = CURRENT_USER()) CURR LEFT JOIN (SELECT USER_ID FROM DB_PRIV WHERE TENANT_ID = 0 AND DATABASE_NAME = 'mysql' AND PRIV_SELECT = 1) DB ON CURR.USER_ID = DB.USER_ID WHERE TP.TENANT_ID = 0 AND TP.TENANT_ID = U.TENANT_ID AND TP.USER_ID = U.USER_ID AND (DB.USER_ID IS NOT NULL OR 512 & CURRENT_USER_PRIV() = 512 OR TP.USER_ID = CURR.USER_ID)) V WHERE V.PRIVILEGE_TYPE IS NOT NULL utf8mb4 utf8mb4_general_ci +TABLE_PRIVILEGES CREATE VIEW `TABLE_PRIVILEGES` AS WITH DB_PRIV AS ( select A.tenant_id TENANT_ID, A.user_id USER_ID, A.database_name DATABASE_NAME, A.priv_alter PRIV_ALTER, A.priv_create PRIV_CREATE, A.priv_delete PRIV_DELETE, A.priv_drop PRIV_DROP, A.priv_grant_option PRIV_GRANT_OPTION, A.priv_insert PRIV_INSERT, A.priv_update PRIV_UPDATE, A.priv_select PRIV_SELECT, A.priv_index PRIV_INDEX, A.priv_create_view PRIV_CREATE_VIEW, A.priv_show_view PRIV_SHOW_VIEW, A.GMT_CREATE GMT_CREATE, A.GMT_MODIFIED GMT_MODIFIED, A.PRIV_OTHERS PRIV_OTHERS from oceanbase.__all_database_privilege_history A, (select tenant_id, user_id, database_name, max(schema_version) schema_version from oceanbase.__all_database_privilege_history group by tenant_id, user_id, database_name, database_name collate utf8mb4_bin) B where A.tenant_id = B.tenant_id and A.user_id = B.user_id and A.database_name collate utf8mb4_bin = B.database_name collate utf8mb4_bin and A.schema_version = B.schema_version and A.is_deleted = 0 ), TABLE_PRIV AS ( select A.tenant_id TENANT_ID, A.user_id USER_ID, A.database_name DATABASE_NAME, A.table_name TABLE_NAME, A.priv_alter PRIV_ALTER, A.priv_create PRIV_CREATE, A.priv_delete PRIV_DELETE, A.priv_drop PRIV_DROP, A.priv_grant_option PRIV_GRANT_OPTION, A.priv_insert PRIV_INSERT, A.priv_update PRIV_UPDATE, A.priv_select PRIV_SELECT, A.priv_index PRIV_INDEX, A.priv_create_view PRIV_CREATE_VIEW, A.priv_show_view PRIV_SHOW_VIEW, A.PRIV_OTHERS PRIV_OTHERS from oceanbase.__all_table_privilege_history A, (select tenant_id, user_id, database_name, table_name, max(schema_version) schema_version from oceanbase.__all_table_privilege_history group by tenant_id, user_id, database_name, database_name collate utf8mb4_bin, table_name, table_name collate utf8mb4_bin) B where A.tenant_id = B.tenant_id and A.user_id = B.user_id and A.database_name collate utf8mb4_bin = B.database_name collate utf8mb4_bin and A.schema_version = B.schema_version and A.table_name collate utf8mb4_bin = B.table_name collate utf8mb4_bin and A.is_deleted = 0 ) SELECT CAST(CONCAT('''', V.USER_NAME, '''', '@', '''', V.HOST, '''') AS CHAR(81)) AS GRANTEE , CAST('def' AS CHAR(512)) AS TABLE_CATALOG , CAST(V.DATABASE_NAME AS CHAR(128)) collate utf8mb4_name_case AS TABLE_SCHEMA , CAST(V.TABLE_NAME AS CHAR(64)) collate utf8mb4_name_case AS TABLE_NAME, CAST(V.PRIVILEGE_TYPE AS CHAR(64)) AS PRIVILEGE_TYPE , CAST(V.IS_GRANTABLE AS CHAR(3)) AS IS_GRANTABLE FROM (SELECT TP.DATABASE_NAME AS DATABASE_NAME, TP.TABLE_NAME AS TABLE_NAME, U.USER_NAME AS USER_NAME, U.HOST AS HOST, CASE WHEN V1.C1 = 1 AND TP.PRIV_ALTER = 1 THEN 'ALTER' WHEN V1.C1 = 2 AND TP.PRIV_CREATE = 1 THEN 'CREATE' WHEN V1.C1 = 4 AND TP.PRIV_DELETE = 1 THEN 'DELETE' WHEN V1.C1 = 5 AND TP.PRIV_DROP = 1 THEN 'DROP' WHEN V1.C1 = 7 AND TP.PRIV_INSERT = 1 THEN 'INSERT' WHEN V1.C1 = 8 AND TP.PRIV_UPDATE = 1 THEN 'UPDATE' WHEN V1.C1 = 9 AND TP.PRIV_SELECT = 1 THEN 'SELECT' WHEN V1.C1 = 10 AND TP.PRIV_INDEX = 1 THEN 'INDEX' WHEN V1.C1 = 11 AND TP.PRIV_CREATE_VIEW = 1 THEN 'CREATE VIEW' WHEN V1.C1 = 12 AND TP.PRIV_SHOW_VIEW = 1 THEN 'SHOW VIEW' WHEN V1.C1 = 22 AND (TP.PRIV_OTHERS & (1 << 6)) != 0 THEN 'REFERENCES' WHEN V1.C1 = 44 AND (TP.PRIV_OTHERS & (1 << 9)) != 0 THEN 'TRIGGER' ELSE NULL END PRIVILEGE_TYPE , CASE WHEN TP.PRIV_GRANT_OPTION = 1 THEN 'YES' WHEN TP.PRIV_GRANT_OPTION = 0 THEN 'NO' END IS_GRANTABLE FROM TABLE_PRIV TP, oceanbase.__all_user U, (SELECT 1 AS C1 UNION ALL SELECT 2 AS C1 UNION ALL SELECT 4 AS C1 UNION ALL SELECT 5 AS C1 UNION ALL SELECT 7 AS C1 UNION ALL SELECT 8 AS C1 UNION ALL SELECT 9 AS C1 UNION ALL SELECT 10 AS C1 UNION ALL SELECT 11 AS C1 UNION ALL SELECT 12 AS C1 UNION ALL SELECT 22 AS C1 UNION ALL SELECT 44 AS C1) V1, (SELECT USER_ID FROM oceanbase.__all_user WHERE TENANT_ID = 0 AND CONCAT(USER_NAME, '@', HOST) = CURRENT_USER()) CURR LEFT JOIN (SELECT USER_ID FROM DB_PRIV WHERE TENANT_ID = 0 AND DATABASE_NAME = 'mysql' AND PRIV_SELECT = 1) DB ON CURR.USER_ID = DB.USER_ID WHERE TP.TENANT_ID = 0 AND TP.TENANT_ID = U.TENANT_ID AND TP.USER_ID = U.USER_ID AND (DB.USER_ID IS NOT NULL OR 512 & CURRENT_USER_PRIV() = 512 OR TP.USER_ID = CURR.USER_ID)) V WHERE V.PRIVILEGE_TYPE IS NOT NULL utf8mb4 utf8mb4_general_ci show create table tables; View Create View character_set_client collation_connection -TABLES CREATE VIEW `TABLES` AS select /*+ leading(a) no_use_nl(ts)*/ cast('def' as char(512)) as TABLE_CATALOG, cast(b.database_name as char(64)) collate utf8mb4_name_case as TABLE_SCHEMA, cast(a.table_name as char(64)) collate utf8mb4_name_case as TABLE_NAME, cast(case when (a.database_id = 201002 or a.table_type = 1) then 'SYSTEM VIEW' when a.table_type in (0, 2) then 'SYSTEM TABLE' when a.table_type = 4 then 'VIEW' when a.table_type = 14 then 'EXTERNAL TABLE' else 'BASE TABLE' end as char(64)) as TABLE_TYPE, cast(case when a.table_type in (0,3,5,6,7,11,12,13,15) then 'InnoDB' else 'MEMORY' end as char(64)) as ENGINE, cast(NULL as unsigned) as VERSION, cast(a.store_format as char(10)) as ROW_FORMAT, cast( coalesce(ts.row_cnt,0) as unsigned) as TABLE_ROWS, cast( coalesce(ts.avg_row_len,0) as unsigned) as AVG_ROW_LENGTH, cast( coalesce(ts.data_size,0) as unsigned) as DATA_LENGTH, cast(NULL as unsigned) as MAX_DATA_LENGTH, cast(NULL as unsigned) as INDEX_LENGTH, cast(NULL as unsigned) as DATA_FREE, cast(NULL as unsigned) as AUTO_INCREMENT, cast(a.gmt_create as datetime) as CREATE_TIME, cast(a.gmt_modified as datetime) as UPDATE_TIME, cast(NULL as datetime) as CHECK_TIME, cast(d.collation as char(32)) as TABLE_COLLATION, cast(NULL as unsigned) as CHECKSUM, cast(NULL as char(255)) as CREATE_OPTIONS, cast(case when a.table_type = 4 then 'VIEW' else a.comment end as char(2048)) as TABLE_COMMENT from ( select cast(0 as signed) as tenant_id, c.database_id, c.table_id, c.table_name, c.collation_type, c.table_type, usec_to_time(d.schema_version) as gmt_create, usec_to_time(c.schema_version) as gmt_modified, c.comment, c.store_format from oceanbase.__all_virtual_core_all_table c join oceanbase.__all_virtual_core_all_table d on c.tenant_id = d.tenant_id and d.table_name = '__all_core_table' where c.tenant_id = effective_tenant_id() union all select tenant_id, database_id, table_id, table_name, collation_type, table_type, gmt_create, gmt_modified, comment, store_format from oceanbase.__all_table where table_mode >> 12 & 15 in (0,1)) a join oceanbase.__all_database b on a.database_id = b.database_id and a.tenant_id = b.tenant_id join oceanbase.__tenant_virtual_collation d on a.collation_type = d.collation_type left join ( select tenant_id, table_id, row_cnt, avg_row_len, (macro_blk_cnt * 2 * 1024 * 1024) as data_size from oceanbase.__all_table_stat where partition_id = -1 or partition_id = table_id) ts on a.table_id = ts.table_id and a.tenant_id = ts.tenant_id where a.tenant_id = 0 and a.table_type in (0, 1, 2, 3, 4, 14, 15) and b.database_name != '__recyclebin' and b.in_recyclebin = 0 and 0 = sys_privilege_check('table_acc', effective_tenant_id(), b.database_name, a.table_name) utf8mb4 utf8mb4_general_ci +TABLES CREATE VIEW `TABLES` AS select /*+ leading(a) no_use_nl(ts)*/ cast('def' as char(512)) as TABLE_CATALOG, cast(b.database_name as char(64)) collate utf8mb4_name_case as TABLE_SCHEMA, cast(a.table_name as char(64)) collate utf8mb4_name_case as TABLE_NAME, cast(case when (a.database_id = 201002 or a.table_type = 1) then 'SYSTEM VIEW' when a.table_type in (0, 2) then 'SYSTEM TABLE' when a.table_type = 4 then 'VIEW' when a.table_type = 14 then 'EXTERNAL TABLE' else 'BASE TABLE' end as char(64)) as TABLE_TYPE, cast(case when a.table_type in (0,3,5,6,7,11,12,13,15) then 'InnoDB' else 'MEMORY' end as char(64)) as ENGINE, cast(NULL as unsigned) as VERSION, cast(a.store_format as char(10)) as ROW_FORMAT, cast( coalesce(ts.row_cnt,0) as unsigned) as TABLE_ROWS, cast( coalesce(ts.avg_row_len,0) as unsigned) as AVG_ROW_LENGTH, cast( coalesce(ts.data_size,0) as unsigned) as DATA_LENGTH, cast(NULL as unsigned) as MAX_DATA_LENGTH, cast( coalesce(idx_stat.index_length, 0) as unsigned) as INDEX_LENGTH, cast(NULL as unsigned) as DATA_FREE, cast(NULL as unsigned) as AUTO_INCREMENT, cast(a.gmt_create as datetime) as CREATE_TIME, cast(a.gmt_modified as datetime) as UPDATE_TIME, cast(NULL as datetime) as CHECK_TIME, cast(d.collation as char(32)) as TABLE_COLLATION, cast(NULL as unsigned) as CHECKSUM, cast(NULL as char(255)) as CREATE_OPTIONS, cast(case when a.table_type = 4 then 'VIEW' else a.comment end as char(2048)) as TABLE_COMMENT from ( select cast(0 as signed) as tenant_id, c.database_id, c.table_id, c.table_name, c.collation_type, c.table_type, usec_to_time(d.schema_version) as gmt_create, usec_to_time(c.schema_version) as gmt_modified, c.comment, c.store_format from oceanbase.__all_virtual_core_all_table c join oceanbase.__all_virtual_core_all_table d on c.tenant_id = d.tenant_id and d.table_name = '__all_core_table' where c.tenant_id = effective_tenant_id() union all select tenant_id, database_id, table_id, table_name, collation_type, table_type, gmt_create, gmt_modified, comment, store_format from oceanbase.__all_table where table_mode >> 12 & 15 in (0,1)) a join oceanbase.__all_database b on a.database_id = b.database_id and a.tenant_id = b.tenant_id join oceanbase.__tenant_virtual_collation d on a.collation_type = d.collation_type left join ( select tenant_id, table_id, row_cnt, avg_row_len, (macro_blk_cnt * 2 * 1024 * 1024) as data_size from oceanbase.__all_table_stat where partition_id = -1 or partition_id = table_id) ts on a.table_id = ts.table_id and a.tenant_id = ts.tenant_id left join ( select e.tenant_id as tenant_id, e.data_table_id as data_table_id, SUM(f.macro_blk_cnt * 2 * 1024 * 1024) AS index_length FROM oceanbase.__all_table e JOIN oceanbase.__all_table_stat f ON e.tenant_id = f.tenant_id and e.table_id = f.table_id and (f.partition_id = -1 or f.partition_id = e.table_id) WHERE e.index_type in (1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12) and e.table_type = 5 group by tenant_id, data_table_id ) idx_stat on idx_stat.tenant_id = a.tenant_id and idx_stat.data_table_id = a.table_id where a.tenant_id = 0 and a.table_type in (0, 1, 2, 3, 4, 14, 15) and b.database_name != '__recyclebin' and b.in_recyclebin = 0 and 0 = sys_privilege_check('table_acc', effective_tenant_id(), b.database_name, a.table_name) utf8mb4 utf8mb4_general_ci show create table user_privileges; View Create View character_set_client collation_connection -USER_PRIVILEGES CREATE VIEW `USER_PRIVILEGES` AS SELECT CAST(CONCAT('''', V.USER_NAME, '''', '@', '''', V.HOST, '''') AS CHAR(81)) AS GRANTEE , CAST('def' AS CHAR(512)) AS TABLE_CATALOG , CAST(V.PRIVILEGE_TYPE AS CHAR(64)) AS PRIVILEGE_TYPE , CAST(V.IS_GRANTABLE AS CHAR(3)) AS IS_GRANTABLE FROM (SELECT U.USER_NAME AS USER_NAME, U.HOST AS HOST, CASE WHEN V1.C1 = 1 AND U.PRIV_ALTER = 1 THEN 'ALTER' WHEN V1.C1 = 2 AND U.PRIV_CREATE = 1 THEN 'CREATE' WHEN V1.C1 = 3 AND U.PRIV_CREATE_USER = 1 THEN 'CREATE USER' WHEN V1.C1 = 4 AND U.PRIV_DELETE = 1 THEN 'DELETE' WHEN V1.C1 = 5 AND U.PRIV_DROP = 1 THEN 'DROP' WHEN V1.C1 = 7 AND U.PRIV_INSERT = 1 THEN 'INSERT' WHEN V1.C1 = 8 AND U.PRIV_UPDATE = 1 THEN 'UPDATE' WHEN V1.C1 = 9 AND U.PRIV_SELECT = 1 THEN 'SELECT' WHEN V1.C1 = 10 AND U.PRIV_INDEX = 1 THEN 'INDEX' WHEN V1.C1 = 11 AND U.PRIV_CREATE_VIEW = 1 THEN 'CREATE VIEW' WHEN V1.C1 = 12 AND U.PRIV_SHOW_VIEW = 1 THEN 'SHOW VIEW' WHEN V1.C1 = 13 AND U.PRIV_SHOW_DB = 1 THEN 'SHOW DATABASES' WHEN V1.C1 = 14 AND U.PRIV_SUPER = 1 THEN 'SUPER' WHEN V1.C1 = 15 AND U.PRIV_PROCESS = 1 THEN 'PROCESS' WHEN V1.C1 = 17 AND U.PRIV_CREATE_SYNONYM = 1 THEN 'CREATE SYNONYM' WHEN V1.C1 = 23 AND (U.PRIV_OTHERS & (1 << 0)) != 0 THEN 'EXECUTE' WHEN V1.C1 = 27 AND U.PRIV_FILE = 1 THEN 'FILE' WHEN V1.C1 = 28 AND U.PRIV_ALTER_TENANT = 1 THEN 'ALTER TENANT' WHEN V1.C1 = 29 AND U.PRIV_ALTER_SYSTEM = 1 THEN 'ALTER SYSTEM' WHEN V1.C1 = 30 AND U.PRIV_CREATE_RESOURCE_POOL = 1 THEN 'CREATE RESOURCE POOL' WHEN V1.C1 = 31 AND U.PRIV_CREATE_RESOURCE_UNIT = 1 THEN 'CREATE RESOURCE UNIT' WHEN V1.C1 = 33 AND U.PRIV_REPL_SLAVE = 1 THEN 'REPLICATION SLAVE' WHEN V1.C1 = 34 AND U.PRIV_REPL_CLIENT = 1 THEN 'REPLICATION CLIENT' WHEN V1.C1 = 35 AND U.PRIV_DROP_DATABASE_LINK = 1 THEN 'DROP DATABASE LINK' WHEN V1.C1 = 36 AND U.PRIV_CREATE_DATABASE_LINK = 1 THEN 'CREATE DATABASE LINK' WHEN V1.C1 = 37 AND (U.PRIV_OTHERS & (1 << 1)) != 0 THEN 'ALTER ROUTINE' WHEN V1.C1 = 38 AND (U.PRIV_OTHERS & (1 << 2)) != 0 THEN 'CREATE ROUTINE' WHEN V1.C1 = 39 AND (U.PRIV_OTHERS & (1 << 3)) != 0 THEN 'CREATE TABLESPACE' WHEN V1.C1 = 40 AND (U.PRIV_OTHERS & (1 << 4)) != 0 THEN 'SHUTDOWN' WHEN V1.C1 = 41 AND (U.PRIV_OTHERS & (1 << 5)) != 0 THEN 'RELOAD' WHEN V1.C1 = 0 AND U.PRIV_ALTER = 0 AND U.PRIV_CREATE = 0 AND U.PRIV_CREATE_USER = 0 AND U.PRIV_DELETE = 0 AND U.PRIV_DROP = 0 AND U.PRIV_INSERT = 0 AND U.PRIV_UPDATE = 0 AND U.PRIV_SELECT = 0 AND U.PRIV_INDEX = 0 AND U.PRIV_CREATE_VIEW = 0 AND U.PRIV_SHOW_VIEW = 0 AND U.PRIV_SHOW_DB = 0 AND U.PRIV_SUPER = 0 AND U.PRIV_PROCESS = 0 AND U.PRIV_CREATE_SYNONYM = 0 AND U.PRIV_FILE = 0 AND U.PRIV_ALTER_TENANT = 0 AND U.PRIV_ALTER_SYSTEM = 0 AND U.PRIV_CREATE_RESOURCE_POOL = 0 AND U.PRIV_CREATE_RESOURCE_UNIT = 0 AND U.PRIV_REPL_SLAVE = 0 AND U.PRIV_REPL_CLIENT = 0 AND U.PRIV_DROP_DATABASE_LINK = 0 AND U.PRIV_CREATE_DATABASE_LINK = 0 AND U.PRIV_OTHERS = 0 THEN 'USAGE' END PRIVILEGE_TYPE , CASE WHEN U.PRIV_GRANT_OPTION = 0 THEN 'NO' WHEN U.PRIV_ALTER = 0 AND U.PRIV_CREATE = 0 AND U.PRIV_CREATE_USER = 0 AND U.PRIV_DELETE = 0 AND U.PRIV_DROP = 0 AND U.PRIV_INSERT = 0 AND U.PRIV_UPDATE = 0 AND U.PRIV_SELECT = 0 AND U.PRIV_INDEX = 0 AND U.PRIV_CREATE_VIEW = 0 AND U.PRIV_SHOW_VIEW = 0 AND U.PRIV_SHOW_DB = 0 AND U.PRIV_SUPER = 0 AND U.PRIV_PROCESS = 0 AND U.PRIV_CREATE_SYNONYM = 0 AND U.PRIV_FILE = 0 AND U.PRIV_ALTER_TENANT = 0 AND U.PRIV_ALTER_SYSTEM = 0 AND U.PRIV_CREATE_RESOURCE_POOL = 0 AND U.PRIV_CREATE_RESOURCE_UNIT = 0 AND U.PRIV_REPL_SLAVE = 0 AND U.PRIV_REPL_CLIENT = 0 AND U.PRIV_DROP_DATABASE_LINK = 0 AND U.PRIV_CREATE_DATABASE_LINK = 0 AND U.PRIV_OTHERS = 0 THEN 'NO' WHEN U.PRIV_GRANT_OPTION = 1 THEN 'YES' END IS_GRANTABLE FROM oceanbase.__all_user U, (SELECT 0 AS C1 UNION ALL SELECT 1 AS C1 UNION ALL SELECT 2 AS C1 UNION ALL SELECT 3 AS C1 UNION ALL SELECT 4 AS C1 UNION ALL SELECT 5 AS C1 UNION ALL SELECT 7 AS C1 UNION ALL SELECT 8 AS C1 UNION ALL SELECT 9 AS C1 UNION ALL SELECT 10 AS C1 UNION ALL SELECT 11 AS C1 UNION ALL SELECT 12 AS C1 UNION ALL SELECT 13 AS C1 UNION ALL SELECT 14 AS C1 UNION ALL SELECT 15 AS C1 UNION ALL SELECT 17 AS C1 UNION ALL SELECT 23 AS C1 UNION ALL SELECT 27 AS C1 UNION ALL SELECT 28 AS C1 UNION ALL SELECT 29 AS C1 UNION ALL SELECT 30 AS C1 UNION ALL SELECT 31 AS C1 UNION ALL SELECT 33 AS C1 UNION ALL SELECT 34 AS C1 UNION ALL SELECT 35 AS C1 UNION ALL SELECT 36 AS C1 UNION ALL SELECT 37 AS C1 UNION ALL SELECT 38 AS C1 UNION ALL SELECT 39 AS C1 UNION ALL SELECT 40 AS C1 UNION ALL SELECT 41 AS C1) V1, (SELECT USER_ID FROM oceanbase.__all_user WHERE TENANT_ID = 0 AND CONCAT(USER_NAME, '@', HOST) = CURRENT_USER()) CURR LEFT JOIN (SELECT USER_ID FROM oceanbase.__all_database_privilege WHERE TENANT_ID = 0 AND DATABASE_NAME = 'mysql' AND PRIV_SELECT = 1) DB ON CURR.USER_ID = DB.USER_ID WHERE U.TENANT_ID = 0 AND (DB.USER_ID IS NOT NULL OR 512 & CURRENT_USER_PRIV() = 512 OR U.USER_ID = CURR.USER_ID)) V WHERE V.PRIVILEGE_TYPE IS NOT NULL utf8mb4 utf8mb4_general_ci +USER_PRIVILEGES CREATE VIEW `USER_PRIVILEGES` AS SELECT CAST(CONCAT('''', V.USER_NAME, '''', '@', '''', V.HOST, '''') AS CHAR(81)) AS GRANTEE , CAST('def' AS CHAR(512)) AS TABLE_CATALOG , CAST(V.PRIVILEGE_TYPE AS CHAR(64)) AS PRIVILEGE_TYPE , CAST(V.IS_GRANTABLE AS CHAR(3)) AS IS_GRANTABLE FROM (SELECT U.USER_NAME AS USER_NAME, U.HOST AS HOST, CASE WHEN V1.C1 = 1 AND U.PRIV_ALTER = 1 THEN 'ALTER' WHEN V1.C1 = 2 AND U.PRIV_CREATE = 1 THEN 'CREATE' WHEN V1.C1 = 3 AND U.PRIV_CREATE_USER = 1 THEN 'CREATE USER' WHEN V1.C1 = 4 AND U.PRIV_DELETE = 1 THEN 'DELETE' WHEN V1.C1 = 5 AND U.PRIV_DROP = 1 THEN 'DROP' WHEN V1.C1 = 7 AND U.PRIV_INSERT = 1 THEN 'INSERT' WHEN V1.C1 = 8 AND U.PRIV_UPDATE = 1 THEN 'UPDATE' WHEN V1.C1 = 9 AND U.PRIV_SELECT = 1 THEN 'SELECT' WHEN V1.C1 = 10 AND U.PRIV_INDEX = 1 THEN 'INDEX' WHEN V1.C1 = 11 AND U.PRIV_CREATE_VIEW = 1 THEN 'CREATE VIEW' WHEN V1.C1 = 12 AND U.PRIV_SHOW_VIEW = 1 THEN 'SHOW VIEW' WHEN V1.C1 = 13 AND U.PRIV_SHOW_DB = 1 THEN 'SHOW DATABASES' WHEN V1.C1 = 14 AND U.PRIV_SUPER = 1 THEN 'SUPER' WHEN V1.C1 = 15 AND U.PRIV_PROCESS = 1 THEN 'PROCESS' WHEN V1.C1 = 17 AND U.PRIV_CREATE_SYNONYM = 1 THEN 'CREATE SYNONYM' WHEN V1.C1 = 22 AND (U.PRIV_OTHERS & (1 << 6)) != 0 THEN 'REFERENCES' WHEN V1.C1 = 23 AND (U.PRIV_OTHERS & (1 << 0)) != 0 THEN 'EXECUTE' WHEN V1.C1 = 27 AND U.PRIV_FILE = 1 THEN 'FILE' WHEN V1.C1 = 28 AND U.PRIV_ALTER_TENANT = 1 THEN 'ALTER TENANT' WHEN V1.C1 = 29 AND U.PRIV_ALTER_SYSTEM = 1 THEN 'ALTER SYSTEM' WHEN V1.C1 = 30 AND U.PRIV_CREATE_RESOURCE_POOL = 1 THEN 'CREATE RESOURCE POOL' WHEN V1.C1 = 31 AND U.PRIV_CREATE_RESOURCE_UNIT = 1 THEN 'CREATE RESOURCE UNIT' WHEN V1.C1 = 33 AND U.PRIV_REPL_SLAVE = 1 THEN 'REPLICATION SLAVE' WHEN V1.C1 = 34 AND U.PRIV_REPL_CLIENT = 1 THEN 'REPLICATION CLIENT' WHEN V1.C1 = 35 AND U.PRIV_DROP_DATABASE_LINK = 1 THEN 'DROP DATABASE LINK' WHEN V1.C1 = 36 AND U.PRIV_CREATE_DATABASE_LINK = 1 THEN 'CREATE DATABASE LINK' WHEN V1.C1 = 37 AND (U.PRIV_OTHERS & (1 << 1)) != 0 THEN 'ALTER ROUTINE' WHEN V1.C1 = 38 AND (U.PRIV_OTHERS & (1 << 2)) != 0 THEN 'CREATE ROUTINE' WHEN V1.C1 = 39 AND (U.PRIV_OTHERS & (1 << 3)) != 0 THEN 'CREATE TABLESPACE' WHEN V1.C1 = 40 AND (U.PRIV_OTHERS & (1 << 4)) != 0 THEN 'SHUTDOWN' WHEN V1.C1 = 41 AND (U.PRIV_OTHERS & (1 << 5)) != 0 THEN 'RELOAD' WHEN V1.C1 = 42 AND (U.PRIV_OTHERS & (1 << 7)) != 0 THEN 'CREATE ROLE' WHEN V1.C1 = 43 AND (U.PRIV_OTHERS & (1 << 8)) != 0 THEN 'DROP ROLE' WHEN V1.C1 = 44 AND (U.PRIV_OTHERS & (1 << 9)) != 0 THEN 'TRIGGER' WHEN V1.C1 = 0 AND U.PRIV_ALTER = 0 AND U.PRIV_CREATE = 0 AND U.PRIV_CREATE_USER = 0 AND U.PRIV_DELETE = 0 AND U.PRIV_DROP = 0 AND U.PRIV_INSERT = 0 AND U.PRIV_UPDATE = 0 AND U.PRIV_SELECT = 0 AND U.PRIV_INDEX = 0 AND U.PRIV_CREATE_VIEW = 0 AND U.PRIV_SHOW_VIEW = 0 AND U.PRIV_SHOW_DB = 0 AND U.PRIV_SUPER = 0 AND U.PRIV_PROCESS = 0 AND U.PRIV_CREATE_SYNONYM = 0 AND U.PRIV_FILE = 0 AND U.PRIV_ALTER_TENANT = 0 AND U.PRIV_ALTER_SYSTEM = 0 AND U.PRIV_CREATE_RESOURCE_POOL = 0 AND U.PRIV_CREATE_RESOURCE_UNIT = 0 AND U.PRIV_REPL_SLAVE = 0 AND U.PRIV_REPL_CLIENT = 0 AND U.PRIV_DROP_DATABASE_LINK = 0 AND U.PRIV_CREATE_DATABASE_LINK = 0 AND U.PRIV_OTHERS = 0 THEN 'USAGE' END PRIVILEGE_TYPE , CASE WHEN U.PRIV_GRANT_OPTION = 0 THEN 'NO' WHEN U.PRIV_ALTER = 0 AND U.PRIV_CREATE = 0 AND U.PRIV_CREATE_USER = 0 AND U.PRIV_DELETE = 0 AND U.PRIV_DROP = 0 AND U.PRIV_INSERT = 0 AND U.PRIV_UPDATE = 0 AND U.PRIV_SELECT = 0 AND U.PRIV_INDEX = 0 AND U.PRIV_CREATE_VIEW = 0 AND U.PRIV_SHOW_VIEW = 0 AND U.PRIV_SHOW_DB = 0 AND U.PRIV_SUPER = 0 AND U.PRIV_PROCESS = 0 AND U.PRIV_CREATE_SYNONYM = 0 AND U.PRIV_FILE = 0 AND U.PRIV_ALTER_TENANT = 0 AND U.PRIV_ALTER_SYSTEM = 0 AND U.PRIV_CREATE_RESOURCE_POOL = 0 AND U.PRIV_CREATE_RESOURCE_UNIT = 0 AND U.PRIV_REPL_SLAVE = 0 AND U.PRIV_REPL_CLIENT = 0 AND U.PRIV_DROP_DATABASE_LINK = 0 AND U.PRIV_CREATE_DATABASE_LINK = 0 AND U.PRIV_OTHERS = 0 THEN 'NO' WHEN U.PRIV_GRANT_OPTION = 1 THEN 'YES' END IS_GRANTABLE FROM oceanbase.__all_user U, (SELECT 0 AS C1 UNION ALL SELECT 1 AS C1 UNION ALL SELECT 2 AS C1 UNION ALL SELECT 3 AS C1 UNION ALL SELECT 4 AS C1 UNION ALL SELECT 5 AS C1 UNION ALL SELECT 7 AS C1 UNION ALL SELECT 8 AS C1 UNION ALL SELECT 9 AS C1 UNION ALL SELECT 10 AS C1 UNION ALL SELECT 11 AS C1 UNION ALL SELECT 12 AS C1 UNION ALL SELECT 13 AS C1 UNION ALL SELECT 14 AS C1 UNION ALL SELECT 15 AS C1 UNION ALL SELECT 17 AS C1 UNION ALL SELECT 22 AS C1 UNION ALL SELECT 23 AS C1 UNION ALL SELECT 27 AS C1 UNION ALL SELECT 28 AS C1 UNION ALL SELECT 29 AS C1 UNION ALL SELECT 30 AS C1 UNION ALL SELECT 31 AS C1 UNION ALL SELECT 33 AS C1 UNION ALL SELECT 34 AS C1 UNION ALL SELECT 35 AS C1 UNION ALL SELECT 36 AS C1 UNION ALL SELECT 37 AS C1 UNION ALL SELECT 38 AS C1 UNION ALL SELECT 39 AS C1 UNION ALL SELECT 40 AS C1 UNION ALL SELECT 41 AS C1 UNION ALL SELECT 42 AS C1 UNION ALL SELECT 43 AS C1 UNION ALL SELECT 44 AS C1) V1, (SELECT USER_ID FROM oceanbase.__all_user WHERE TENANT_ID = 0 AND CONCAT(USER_NAME, '@', HOST) = CURRENT_USER()) CURR LEFT JOIN (SELECT USER_ID FROM oceanbase.__all_database_privilege WHERE TENANT_ID = 0 AND DATABASE_NAME = 'mysql' AND PRIV_SELECT = 1) DB ON CURR.USER_ID = DB.USER_ID WHERE U.TENANT_ID = 0 AND (DB.USER_ID IS NOT NULL OR 512 & CURRENT_USER_PRIV() = 512 OR U.USER_ID = CURR.USER_ID)) V WHERE V.PRIVILEGE_TYPE IS NOT NULL utf8mb4 utf8mb4_general_ci show create table views; View Create View character_set_client collation_connection VIEWS CREATE VIEW `VIEWS` AS select cast('def' as CHAR(64)) AS TABLE_CATALOG, d.database_name collate utf8mb4_name_case as TABLE_SCHEMA, t.table_name collate utf8mb4_name_case as TABLE_NAME, t.view_definition as VIEW_DEFINITION, case t.view_check_option when 1 then 'LOCAL' when 2 then 'CASCADED' else 'NONE' end as CHECK_OPTION, case t.view_is_updatable when 1 then 'YES' else 'NO' end as IS_UPDATABLE, cast((case t.define_user_id when -1 then 'NONE' else concat(u.user_name, '@', u.host) end) as CHAR(288)) as DEFINER, cast('NONE' as CHAR(7)) AS SECURITY_TYPE, cast((case t.collation_type when 45 then 'utf8mb4' else 'NONE' end) as CHAR(64)) AS CHARACTER_SET_CLIENT, cast((case t.collation_type when 45 then 'utf8mb4_general_ci' else 'NONE' end) as CHAR(64)) AS COLLATION_CONNECTION from oceanbase.__all_table as t join oceanbase.__all_database as d on t.tenant_id = d.tenant_id and t.database_id = d.database_id left join oceanbase.__all_user as u on t.tenant_id = u.tenant_id and t.define_user_id = u.user_id and t.define_user_id != -1 where t.tenant_id = 0 and t.table_type in (1, 4) and t.table_mode >> 12 & 15 in (0,1) and d.in_recyclebin = 0 and d.database_name != '__recyclebin' and d.database_name != 'information_schema' and d.database_name != 'oceanbase' and 0 = sys_privilege_check('table_acc', effective_tenant_id(), d.database_name, t.table_name) utf8mb4 utf8mb4_general_ci diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result index d09558c3c..91db95b37 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result @@ -27,6 +27,16 @@ alert_log_level all_server_list arbitration_timeout archive_lag_target +audit_log_buffer_size +audit_log_compression +audit_log_enable +audit_log_format +audit_log_max_size +audit_log_path +audit_log_prune_seconds +audit_log_query_sql +audit_log_rotate_on_size +audit_log_strategy audit_sys_operations audit_trail autoinc_cache_refresh_interval diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/character_sets.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/character_sets.result index 56c7680df..1106b2c69 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/character_sets.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/character_sets.result @@ -8,6 +8,8 @@ utf16 utf16_general_ci UTF-16 Unicode 2 gb18030 gb18030_chinese_ci GB18030 charset 4 latin1 latin1_swedish_ci cp1252 West European 1 gb18030_2022 gb18030_2022_chinese_ci GB18030-2022 charset 4 +ascii ascii_general_ci US ASCII 1 +tis620 tis620_thai_ci TIS620 Thai 1 select character_set_name, default_collate_name, description, maxlen from character_sets; character_set_name default_collate_name description maxlen binary binary Binary pseudo charset 1 @@ -17,6 +19,8 @@ utf16 utf16_general_ci UTF-16 Unicode 2 gb18030 gb18030_chinese_ci GB18030 charset 4 latin1 latin1_swedish_ci cp1252 West European 1 gb18030_2022 gb18030_2022_chinese_ci GB18030-2022 charset 4 +ascii ascii_general_ci US ASCII 1 +tis620 tis620_thai_ci TIS620 Thai 1 select maxlen from character_sets; maxlen 1 @@ -26,6 +30,8 @@ maxlen 4 1 4 +1 +1 select * from character_sets where character_set_name like '%binary%'; CHARACTER_SET_NAME DEFAULT_COLLATE_NAME DESCRIPTION MAXLEN binary binary Binary pseudo charset 1 diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/collations.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/collations.result index 87374e3ac..57e751614 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/collations.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/collations.result @@ -8,9 +8,9 @@ gbk_chinese_ci gbk 28 Yes Yes 1 gbk_bin gbk 87 Yes 1 utf16_general_ci utf16 54 Yes Yes 1 utf16_bin utf16 55 Yes 1 -utf8mb4_unicode_ci utf8mb4 224 Yes 1 -utf16_unicode_ci utf16 101 Yes 1 -gb18030_chinese_ci gb18030 248 Yes Yes 1 +utf8mb4_unicode_ci utf8mb4 224 Yes 8 +utf16_unicode_ci utf16 101 Yes 8 +gb18030_chinese_ci gb18030 248 Yes Yes 2 gb18030_bin gb18030 249 Yes 1 latin1_swedish_ci latin1 8 Yes Yes 1 latin1_bin latin1 47 Yes 1 @@ -21,6 +21,14 @@ gb18030_2022_radical_ci gb18030_2022 219 Yes 1 gb18030_2022_radical_cs gb18030_2022 220 Yes 1 gb18030_2022_stroke_ci gb18030_2022 221 Yes 1 gb18030_2022_stroke_cs gb18030_2022 222 Yes 1 +utf8mb4_croatian_ci utf8mb4 245 Yes 8 +utf8mb4_unicode_520_ci utf8mb4 246 Yes 8 +utf8mb4_czech_ci utf8mb4 234 Yes 8 +ascii_general_ci ascii 11 Yes Yes 1 +ascii_bin ascii 65 Yes 1 +tis620_thai_ci tis620 18 Yes Yes 1 +tis620_bin tis620 89 Yes 1 +utf8mb4_0900_ai_ci utf8mb4 255 Yes 1 select collation_name, character_set_name, id, is_default, is_compiled, sortlen from collations; collation_name character_set_name id is_default is_compiled sortlen utf8mb4_general_ci utf8mb4 45 Yes Yes 1 @@ -30,9 +38,9 @@ gbk_chinese_ci gbk 28 Yes Yes 1 gbk_bin gbk 87 Yes 1 utf16_general_ci utf16 54 Yes Yes 1 utf16_bin utf16 55 Yes 1 -utf8mb4_unicode_ci utf8mb4 224 Yes 1 -utf16_unicode_ci utf16 101 Yes 1 -gb18030_chinese_ci gb18030 248 Yes Yes 1 +utf8mb4_unicode_ci utf8mb4 224 Yes 8 +utf16_unicode_ci utf16 101 Yes 8 +gb18030_chinese_ci gb18030 248 Yes Yes 2 gb18030_bin gb18030 249 Yes 1 latin1_swedish_ci latin1 8 Yes Yes 1 latin1_bin latin1 47 Yes 1 @@ -43,11 +51,23 @@ gb18030_2022_radical_ci gb18030_2022 219 Yes 1 gb18030_2022_radical_cs gb18030_2022 220 Yes 1 gb18030_2022_stroke_ci gb18030_2022 221 Yes 1 gb18030_2022_stroke_cs gb18030_2022 222 Yes 1 +utf8mb4_croatian_ci utf8mb4 245 Yes 8 +utf8mb4_unicode_520_ci utf8mb4 246 Yes 8 +utf8mb4_czech_ci utf8mb4 234 Yes 8 +ascii_general_ci ascii 11 Yes Yes 1 +ascii_bin ascii 65 Yes 1 +tis620_thai_ci tis620 18 Yes Yes 1 +tis620_bin tis620 89 Yes 1 +utf8mb4_0900_ai_ci utf8mb4 255 Yes 1 select * from collations where collation_name like '%utf8%'; COLLATION_NAME CHARACTER_SET_NAME ID IS_DEFAULT IS_COMPILED SORTLEN utf8mb4_general_ci utf8mb4 45 Yes Yes 1 utf8mb4_bin utf8mb4 46 Yes 1 -utf8mb4_unicode_ci utf8mb4 224 Yes 1 +utf8mb4_unicode_ci utf8mb4 224 Yes 8 +utf8mb4_croatian_ci utf8mb4 245 Yes 8 +utf8mb4_unicode_520_ci utf8mb4 246 Yes 8 +utf8mb4_czech_ci utf8mb4 234 Yes 8 +utf8mb4_0900_ai_ci utf8mb4 255 Yes 1 show create table collations; View Create View character_set_client collation_connection COLLATIONS CREATE VIEW `COLLATIONS` AS select collation as COLLATION_NAME, charset as CHARACTER_SET_NAME, id as ID, `is_default` as IS_DEFAULT, is_compiled as IS_COMPILED, sortlen as SORTLEN from oceanbase.__tenant_virtual_collation utf8mb4 utf8mb4_general_ci diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result index 3ca3c318e..84961069d 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result @@ -164,7 +164,7 @@ TABLE_ROWS bigint(20) unsigned NO AVG_ROW_LENGTH bigint(21) unsigned NO DATA_LENGTH bigint(20) unsigned NO MAX_DATA_LENGTH bigint(0) unsigned NO -INDEX_LENGTH bigint(0) unsigned NO +INDEX_LENGTH bigint(21) unsigned NO DATA_FREE bigint(0) unsigned NO AUTO_INCREMENT bigint(0) unsigned NO CREATE_TIME datetime NO @@ -295,6 +295,42 @@ SOURCE_LINE bigint(20) NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from information_schema.PROFILING limit 1); cnt 1 +desc information_schema.OPTIMIZER_TRACE; +Field Type Null Key Default Extra +QUERY varchar(200) NO +TRACE varchar(200) NO +MISSING_BYTES_MAX_MEM_SIZE bigint(20) NO +INSUFFICIENT_PRIVILEGES bigint(1) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from information_schema.OPTIMIZER_TRACE limit 1); +cnt +1 +desc information_schema.PLUGINS; +Field Type Null Key Default Extra +PLUGIN_NAME varchar(64) NO +PLUGIN varchar(20) NO +PLUGIN_STATUS varchar(10) NO +PLUGIN_TYPE varchar(80) NO +PLUGIN_TYPE_VERSION varchar(20) NO +PLUGIN_LIBRARY varchar(64) NO +PLUGIN_LIBRARY_VERSION varchar(20) NO +PLUGIN_AUTHOR varchar(64) NO +PLUGIN_DESCRIPTION varchar(200) NO +PLUGIN_LICENSE varchar(80) NO +LOAD_OPTION varchar(64) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from information_schema.PLUGINS limit 1); +cnt +1 +desc information_schema.INNODB_SYS_COLUMNS; +Field Type Null Key Default Extra +TABLE_ID bigint(21) unsigned NO +NAME varchar(193) NO +POS bigint(21) unsigned NO +MTYPE bigint(11) NO +PRTYPE bigint(11) NO +LEN bigint(11) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from information_schema.INNODB_SYS_COLUMNS limit 1); +cnt +1 desc information_schema.INNODB_FT_BEING_DELETED; Field Type Null Key Default Extra DOC_ID bigint(21) unsigned NO @@ -4456,6 +4492,10 @@ PRIV_CREATE_ROUTINE varchar(3) NO PRIV_CREATE_TABLESPACE varchar(3) NO PRIV_SHUTDOWN varchar(3) NO PRIV_RELOAD varchar(3) NO +PRIV_REFERENCES varchar(3) NO +PRIV_CREATE_ROLE varchar(3) NO +PRIV_DROP_ROLE varchar(3) NO +PRIV_TRIGGER varchar(3) NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_OB_USERS limit 1); cnt 1 @@ -4480,6 +4520,8 @@ PRIV_SHOW_VIEW varchar(3) NO PRIV_EXECUTE varchar(3) NO PRIV_ALTER_ROUTINE varchar(3) NO PRIV_CREATE_ROUTINE varchar(3) NO +PRIV_REFERENCES varchar(3) NO +PRIV_TRIGGER varchar(3) NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_OB_DATABASE_PRIVILEGE limit 1); cnt 1 @@ -4761,9 +4803,9 @@ PARTITION_DESCRIPTION text NO SUBPARTITION_DESCRIPTION text NO TABLE_ROWS bigint(20) unsigned NO NULL AVG_ROW_LENGTH bigint(21) unsigned NO NULL -DATA_LENGTH bigint(0) unsigned NO +DATA_LENGTH bigint(20) unsigned NO MAX_DATA_LENGTH bigint(0) unsigned NO -INDEX_LENGTH bigint(0) unsigned NO +INDEX_LENGTH bigint(21) unsigned NO DATA_FREE bigint(0) unsigned NO CREATE_TIME timestamp(6) YES NULL UPDATE_TIME datetime NO @@ -5872,7 +5914,7 @@ Db varchar(64) NO NULL User varchar(32) NO NULL Routine_name varchar(64) NO NULL Routine_type varchar(9) NO -Grantor varchar(93) YES NULL +Grantor varchar(93) NO Proc_priv varchar(27) NO Timestamp date YES select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from mysql.procs_priv limit 1); @@ -5974,6 +6016,21 @@ DEFAULT_ROLE_USER varchar(128) NO NULL select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from mysql.default_roles limit 1); cnt 1 +desc mysql.audit_log_filter; +Field Type Null Key Default Extra +NAME varchar(64) NO +FILTER longtext NO NULL +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from mysql.audit_log_filter limit 1); +cnt +1 +desc mysql.audit_log_user; +Field Type Null Key Default Extra +USER varchar(128) NO +HOST varchar(128) NO NULL +FILTERNAME varchar(64) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from mysql.audit_log_user limit 1); +cnt +1 desc mysql.columns_priv; Field Type Null Key Default Extra Host varchar(255) NO @@ -6662,6 +6719,52 @@ SPEED_MBPS bigint(20) NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_NIC_INFO limit 1); cnt 1 +desc information_schema.ROLE_TABLE_GRANTS; +Field Type Null Key Default Extra +GRANTOR varchar(97) YES NULL +GRANTOR_HOST varchar(256) YES NULL +GRANTEE varchar(32) NO NULL +GRANTEE_HOST varchar(255) NO +TABLE_CATALOG varchar(3) NO +TABLE_SCHEMA varchar(64) NO NULL +TABLE_NAME varchar(64) NO NULL +PRIVILEGE_TYPE varchar(90) NO +IS_GRANTABLE varchar(3) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from information_schema.ROLE_TABLE_GRANTS limit 1); +cnt +1 +desc information_schema.ROLE_COLUMN_GRANTS; +Field Type Null Key Default Extra +GRANTOR null NO +GRANTOR_HOST null NO +GRANTEE varchar(32) NO NULL +GRANTEE_HOST varchar(255) NO +TABLE_CATALOG varchar(3) NO +TABLE_SCHEMA varchar(64) NO NULL +TABLE_NAME varchar(64) NO NULL +COLUMN_NAME varchar(64) NO NULL +PRIVILEGE_TYPE varchar(31) NO +IS_GRANTABLE varchar(3) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from information_schema.ROLE_COLUMN_GRANTS limit 1); +cnt +1 +desc information_schema.ROLE_ROUTINE_GRANTS; +Field Type Null Key Default Extra +GRANTOR varchar(97) YES NULL +GRANTOR_HOST varchar(256) YES NULL +GRANTEE varchar(32) NO NULL +GRANTEE_HOST varchar(255) NO +SPECIFIC_CATALOG varchar(3) NO +SPECIFIC_SCHEMA varchar(64) NO NULL +SPECIFIC_NAME varchar(64) NO NULL +ROUTINE_CATALOG varchar(3) NO +ROUTINE_SCHEMA varchar(64) NO NULL +ROUTINE_NAME varchar(64) NO NULL +PRIVILEGE_TYPE varchar(27) NO +IS_GRANTABLE varchar(3) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from information_schema.ROLE_ROUTINE_GRANTS limit 1); +cnt +1 desc mysql.func; Field Type Null Key Default Extra name varchar(64) NO NULL diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result index 320c60d4f..561477619 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result @@ -165,7 +165,7 @@ TABLE_ROWS bigint(20) unsigned NO AVG_ROW_LENGTH bigint(21) unsigned NO DATA_LENGTH bigint(20) unsigned NO MAX_DATA_LENGTH bigint(0) unsigned NO -INDEX_LENGTH bigint(0) unsigned NO +INDEX_LENGTH bigint(21) unsigned NO DATA_FREE bigint(0) unsigned NO AUTO_INCREMENT bigint(0) unsigned NO CREATE_TIME datetime NO @@ -296,6 +296,42 @@ SOURCE_LINE bigint(20) NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from information_schema.PROFILING limit 1); cnt 1 +desc information_schema.OPTIMIZER_TRACE; +Field Type Null Key Default Extra +QUERY varchar(200) NO +TRACE varchar(200) NO +MISSING_BYTES_MAX_MEM_SIZE bigint(20) NO +INSUFFICIENT_PRIVILEGES bigint(1) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from information_schema.OPTIMIZER_TRACE limit 1); +cnt +1 +desc information_schema.PLUGINS; +Field Type Null Key Default Extra +PLUGIN_NAME varchar(64) NO +PLUGIN varchar(20) NO +PLUGIN_STATUS varchar(10) NO +PLUGIN_TYPE varchar(80) NO +PLUGIN_TYPE_VERSION varchar(20) NO +PLUGIN_LIBRARY varchar(64) NO +PLUGIN_LIBRARY_VERSION varchar(20) NO +PLUGIN_AUTHOR varchar(64) NO +PLUGIN_DESCRIPTION varchar(200) NO +PLUGIN_LICENSE varchar(80) NO +LOAD_OPTION varchar(64) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from information_schema.PLUGINS limit 1); +cnt +1 +desc information_schema.INNODB_SYS_COLUMNS; +Field Type Null Key Default Extra +TABLE_ID bigint(21) unsigned NO +NAME varchar(193) NO +POS bigint(21) unsigned NO +MTYPE bigint(11) NO +PRTYPE bigint(11) NO +LEN bigint(11) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from information_schema.INNODB_SYS_COLUMNS limit 1); +cnt +1 desc information_schema.INNODB_FT_BEING_DELETED; Field Type Null Key Default Extra DOC_ID bigint(21) unsigned NO @@ -6203,6 +6239,10 @@ PRIV_CREATE_ROUTINE varchar(3) NO PRIV_CREATE_TABLESPACE varchar(3) NO PRIV_SHUTDOWN varchar(3) NO PRIV_RELOAD varchar(3) NO +PRIV_REFERENCES varchar(3) NO +PRIV_CREATE_ROLE varchar(3) NO +PRIV_DROP_ROLE varchar(3) NO +PRIV_TRIGGER varchar(3) NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_OB_USERS limit 1); cnt 1 @@ -6254,6 +6294,10 @@ PRIV_CREATE_ROUTINE varchar(3) NO PRIV_CREATE_TABLESPACE varchar(3) NO PRIV_SHUTDOWN varchar(3) NO PRIV_RELOAD varchar(3) NO +PRIV_REFERENCES varchar(3) NO +PRIV_CREATE_ROLE varchar(3) NO +PRIV_DROP_ROLE varchar(3) NO +PRIV_TRIGGER varchar(3) NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.CDB_OB_USERS limit 1); cnt 1 @@ -6278,6 +6322,8 @@ PRIV_SHOW_VIEW varchar(3) NO PRIV_EXECUTE varchar(3) NO PRIV_ALTER_ROUTINE varchar(3) NO PRIV_CREATE_ROUTINE varchar(3) NO +PRIV_REFERENCES varchar(3) NO +PRIV_TRIGGER varchar(3) NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_OB_DATABASE_PRIVILEGE limit 1); cnt 1 @@ -6303,6 +6349,8 @@ PRIV_SHOW_VIEW varchar(3) NO PRIV_EXECUTE varchar(3) NO PRIV_ALTER_ROUTINE varchar(3) NO PRIV_CREATE_ROUTINE varchar(3) NO +PRIV_REFERENCES varchar(3) NO +PRIV_TRIGGER varchar(3) NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.CDB_OB_DATABASE_PRIVILEGE limit 1); cnt 1 @@ -6605,9 +6653,9 @@ PARTITION_DESCRIPTION text NO SUBPARTITION_DESCRIPTION text NO TABLE_ROWS bigint(20) unsigned NO NULL AVG_ROW_LENGTH bigint(21) unsigned NO NULL -DATA_LENGTH bigint(0) unsigned NO +DATA_LENGTH bigint(20) unsigned NO MAX_DATA_LENGTH bigint(0) unsigned NO -INDEX_LENGTH bigint(0) unsigned NO +INDEX_LENGTH bigint(21) unsigned NO DATA_FREE bigint(0) unsigned NO CREATE_TIME timestamp(6) YES NULL UPDATE_TIME datetime NO @@ -8352,7 +8400,7 @@ Db varchar(64) NO NULL User varchar(32) NO NULL Routine_name varchar(64) NO NULL Routine_type varchar(9) NO -Grantor varchar(93) YES NULL +Grantor varchar(93) NO Proc_priv varchar(27) NO Timestamp date YES select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from mysql.procs_priv limit 1); @@ -8551,6 +8599,21 @@ LAST_USED varchar(128) NO NULL select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.CDB_INDEX_USAGE limit 1); cnt 1 +desc mysql.audit_log_filter; +Field Type Null Key Default Extra +NAME varchar(64) NO +FILTER longtext NO NULL +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from mysql.audit_log_filter limit 1); +cnt +1 +desc mysql.audit_log_user; +Field Type Null Key Default Extra +USER varchar(128) NO +HOST varchar(128) NO NULL +FILTERNAME varchar(64) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from mysql.audit_log_user limit 1); +cnt +1 desc mysql.columns_priv; Field Type Null Key Default Extra Host varchar(255) NO @@ -9469,6 +9532,52 @@ SPEED_MBPS bigint(20) NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_NIC_INFO limit 1); cnt 1 +desc information_schema.ROLE_TABLE_GRANTS; +Field Type Null Key Default Extra +GRANTOR varchar(97) YES NULL +GRANTOR_HOST varchar(256) YES NULL +GRANTEE varchar(32) NO NULL +GRANTEE_HOST varchar(255) NO +TABLE_CATALOG varchar(3) NO +TABLE_SCHEMA varchar(64) NO NULL +TABLE_NAME varchar(64) NO NULL +PRIVILEGE_TYPE varchar(90) NO +IS_GRANTABLE varchar(3) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from information_schema.ROLE_TABLE_GRANTS limit 1); +cnt +1 +desc information_schema.ROLE_COLUMN_GRANTS; +Field Type Null Key Default Extra +GRANTOR null NO +GRANTOR_HOST null NO +GRANTEE varchar(32) NO NULL +GRANTEE_HOST varchar(255) NO +TABLE_CATALOG varchar(3) NO +TABLE_SCHEMA varchar(64) NO NULL +TABLE_NAME varchar(64) NO NULL +COLUMN_NAME varchar(64) NO NULL +PRIVILEGE_TYPE varchar(31) NO +IS_GRANTABLE varchar(3) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from information_schema.ROLE_COLUMN_GRANTS limit 1); +cnt +1 +desc information_schema.ROLE_ROUTINE_GRANTS; +Field Type Null Key Default Extra +GRANTOR varchar(97) YES NULL +GRANTOR_HOST varchar(256) YES NULL +GRANTEE varchar(32) NO NULL +GRANTEE_HOST varchar(255) NO +SPECIFIC_CATALOG varchar(3) NO +SPECIFIC_SCHEMA varchar(64) NO NULL +SPECIFIC_NAME varchar(64) NO NULL +ROUTINE_CATALOG varchar(3) NO +ROUTINE_SCHEMA varchar(64) NO NULL +ROUTINE_NAME varchar(64) NO NULL +PRIVILEGE_TYPE varchar(27) NO +IS_GRANTABLE varchar(3) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from information_schema.ROLE_ROUTINE_GRANTS limit 1); +cnt +1 desc mysql.func; Field Type Null Key Default Extra name varchar(64) NO NULL diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result index 3b381fcf2..6bfce8b23 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result @@ -9431,6 +9431,29 @@ last_flush_time timestamp(6) NO NULL select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_index_usage_info; IF(count(*) >= 0, 1, 0) 1 +desc oceanbase.__all_virtual_audit_log_filter; +Field Type Null Key Default Extra +tenant_id bigint(20) NO PRI NULL +filter_name varbinary(256) NO PRI NULL +gmt_create timestamp(6) NO NULL +gmt_modified timestamp(6) NO NULL +definition longtext NO NULL +is_deleted bigint(20) NO NULL +select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_audit_log_filter; +IF(count(*) >= 0, 1, 0) +1 +desc oceanbase.__all_virtual_audit_log_user; +Field Type Null Key Default Extra +tenant_id bigint(20) NO PRI NULL +user_name varbinary(512) NO PRI NULL +host varchar(128) NO PRI NULL +gmt_create timestamp(6) NO NULL +gmt_modified timestamp(6) NO NULL +filter_name varbinary(256) NO NULL +is_deleted bigint(20) NO NULL +select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_audit_log_user; +IF(count(*) >= 0, 1, 0) +1 desc oceanbase.__all_virtual_column_privilege; Field Type Null Key Default Extra tenant_id bigint(20) NO PRI NULL diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result index 57c7f8ef6..be310cd82 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result @@ -290,6 +290,8 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr 499 __all_transfer_partition_task_history 0 201001 1 500 __all_tenant_snapshot_job 0 201001 1 502 __all_trusted_root_certificate 0 201001 1 +503 __all_audit_log_filter 0 201001 1 +504 __all_audit_log_user 0 201001 1 505 __all_column_privilege 0 201001 1 506 __all_column_privilege_history 0 201001 1 507 __all_tenant_snapshot_ls_replica_history 0 201001 1 @@ -740,6 +742,8 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr 12456 __all_virtual_dbms_lock_allocated 2 201001 1 12458 __all_virtual_ls_snapshot 2 201001 1 12459 __all_virtual_index_usage_info 2 201001 1 +12460 __all_virtual_audit_log_filter 2 201001 1 +12461 __all_virtual_audit_log_user 2 201001 1 12462 __all_virtual_column_privilege 2 201001 1 12463 __all_virtual_column_privilege_history 2 201001 1 12464 __all_virtual_tenant_snapshot_ls_replica_history 2 201001 1 @@ -773,6 +777,9 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr 20014 ENGINES 1 201002 1 20015 ROUTINES 1 201002 1 20016 PROFILING 1 201002 1 +20017 OPTIMIZER_TRACE 1 201002 1 +20018 PLUGINS 1 201002 1 +20019 INNODB_SYS_COLUMNS 1 201002 1 20020 INNODB_FT_BEING_DELETED 1 201002 1 20021 INNODB_FT_CONFIG 1 201002 1 20022 INNODB_FT_DELETED 1 201002 1 @@ -1150,6 +1157,8 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr 21511 role_edges 1 201003 1 21512 default_roles 1 201003 1 21513 CDB_INDEX_USAGE 1 201001 1 +21514 audit_log_filter 1 201003 1 +21515 audit_log_user 1 201003 1 21516 columns_priv 1 201003 1 21517 GV$OB_LS_SNAPSHOTS 1 201001 1 21518 V$OB_LS_SNAPSHOTS 1 201001 1 @@ -1203,6 +1212,9 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr 21579 INNODB_METRICS 1 201002 1 21580 EVENTS 1 201002 1 21581 V$OB_NIC_INFO 1 201001 1 +21582 ROLE_TABLE_GRANTS 1 201002 1 +21583 ROLE_COLUMN_GRANTS 1 201002 1 +21584 ROLE_ROUTINE_GRANTS 1 201002 1 21585 func 1 201003 1 21586 GV$OB_NIC_INFO 1 201001 1 21589 DBA_SCHEDULER_JOB_RUN_DETAILS 1 201001 1 diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/parameter_variable_conflict_check.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/parameter_variable_conflict_check.result index 6ecb4809d..6557c652f 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/parameter_variable_conflict_check.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/parameter_variable_conflict_check.result @@ -6,9 +6,17 @@ FROM __all_sys_variable a, __all_virtual_sys_parameter_stat b WHERE a.name = b.name GROUP BY a.name; para_name var_name +range_optimizer_max_mem_size range_optimizer_max_mem_size +connection_control_max_connection_delay connection_control_max_connection_delay +connection_control_min_connection_delay connection_control_min_connection_delay +connection_control_failed_connections_threshold connection_control_failed_connections_threshold ###### case2: under mysql tenant ###### SELECT a.name as para_name, b.name as var_name FROM V$OB_PARAMETERS a, __all_sys_variable b WHERE a.name = b.name; para_name var_name +range_optimizer_max_mem_size range_optimizer_max_mem_size +connection_control_max_connection_delay connection_control_max_connection_delay +connection_control_min_connection_delay connection_control_min_connection_delay +connection_control_failed_connections_threshold connection_control_failed_connections_threshold diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/partitions.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/partitions.result index 06a2376cb..d1ac37f09 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/partitions.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/partitions.result @@ -2,9 +2,9 @@ drop table if exists pt1; create table pt1 (c1 int primary key, c2 int) partition by key(c1) partitions 3; select * from information_schema.partitions where table_name = 'pt1' order by TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME, PARTITION_NAME; TABLE_CATALOG TABLE_SCHEMA TABLE_NAME PARTITION_NAME SUBPARTITION_NAME PARTITION_ORDINAL_POSITION SUBPARTITION_ORDINAL_POSITION PARTITION_METHOD SUBPARTITION_METHOD PARTITION_EXPRESSION SUBPARTITION_EXPRESSION PARTITION_DESCRIPTION SUBPARTITION_DESCRIPTION TABLE_ROWS AVG_ROW_LENGTH DATA_LENGTH MAX_DATA_LENGTH INDEX_LENGTH DATA_FREE CREATE_TIME UPDATE_TIME CHECK_TIME CHECKSUM PARTITION_COMMENT NODEGROUP TABLESPACE_NAME -def test pt1 p0 NULL 1 NULL KEY NULL c1 NULL NULL NULL NULL NULL NULL NULL NULL NULL YYYY-MM-DD hh:mm:ss NULL NULL NULL default NULL -def test pt1 p1 NULL 2 NULL KEY NULL c1 NULL NULL NULL NULL NULL NULL NULL NULL NULL YYYY-MM-DD hh:mm:ss NULL NULL NULL default NULL -def test pt1 p2 NULL 3 NULL KEY NULL c1 NULL NULL NULL NULL NULL NULL NULL NULL NULL YYYY-MM-DD hh:mm:ss NULL NULL NULL default NULL +def test pt1 p0 NULL 1 NULL KEY NULL c1 NULL NULL NULL NULL NULL 0 NULL 0 NULL YYYY-MM-DD hh:mm:ss NULL NULL NULL default NULL +def test pt1 p1 NULL 2 NULL KEY NULL c1 NULL NULL NULL NULL NULL 0 NULL 0 NULL YYYY-MM-DD hh:mm:ss NULL NULL NULL default NULL +def test pt1 p2 NULL 3 NULL KEY NULL c1 NULL NULL NULL NULL NULL 0 NULL 0 NULL YYYY-MM-DD hh:mm:ss NULL NULL NULL default NULL select table_schema, table_name from information_schema.partitions where table_name = 'pt1' order by TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME, PARTITION_NAME; table_schema table_name test pt1 @@ -27,9 +27,9 @@ PARTITION_DESCRIPTION text NO SUBPARTITION_DESCRIPTION text NO TABLE_ROWS bigint(20) unsigned NO NULL AVG_ROW_LENGTH bigint(21) unsigned NO NULL -DATA_LENGTH bigint(0) unsigned NO +DATA_LENGTH bigint(20) unsigned NO MAX_DATA_LENGTH bigint(0) unsigned NO -INDEX_LENGTH bigint(0) unsigned NO +INDEX_LENGTH bigint(21) unsigned NO DATA_FREE bigint(0) unsigned NO CREATE_TIME timestamp(6) YES NULL UPDATE_TIME datetime NO diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/table_privileges.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/table_privileges.result index 443ad9f82..ab030c9ab 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/table_privileges.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/table_privileges.result @@ -4,7 +4,7 @@ use test_user_priv_db; create table zhan_t1 (a int primary key, b int); show create table information_schema.table_privileges; View Create View character_set_client collation_connection -TABLE_PRIVILEGES CREATE VIEW `TABLE_PRIVILEGES` AS WITH DB_PRIV AS ( select A.tenant_id TENANT_ID, A.user_id USER_ID, A.database_name DATABASE_NAME, A.priv_alter PRIV_ALTER, A.priv_create PRIV_CREATE, A.priv_delete PRIV_DELETE, A.priv_drop PRIV_DROP, A.priv_grant_option PRIV_GRANT_OPTION, A.priv_insert PRIV_INSERT, A.priv_update PRIV_UPDATE, A.priv_select PRIV_SELECT, A.priv_index PRIV_INDEX, A.priv_create_view PRIV_CREATE_VIEW, A.priv_show_view PRIV_SHOW_VIEW, A.GMT_CREATE GMT_CREATE, A.GMT_MODIFIED GMT_MODIFIED, A.PRIV_OTHERS PRIV_OTHERS from oceanbase.__all_database_privilege_history A, (select tenant_id, user_id, database_name, max(schema_version) schema_version from oceanbase.__all_database_privilege_history group by tenant_id, user_id, database_name, database_name collate utf8mb4_bin) B where A.tenant_id = B.tenant_id and A.user_id = B.user_id and A.database_name collate utf8mb4_bin = B.database_name collate utf8mb4_bin and A.schema_version = B.schema_version and A.is_deleted = 0 ), TABLE_PRIV AS ( select A.tenant_id TENANT_ID, A.user_id USER_ID, A.database_name DATABASE_NAME, A.table_name TABLE_NAME, A.priv_alter PRIV_ALTER, A.priv_create PRIV_CREATE, A.priv_delete PRIV_DELETE, A.priv_drop PRIV_DROP, A.priv_grant_option PRIV_GRANT_OPTION, A.priv_insert PRIV_INSERT, A.priv_update PRIV_UPDATE, A.priv_select PRIV_SELECT, A.priv_index PRIV_INDEX, A.priv_create_view PRIV_CREATE_VIEW, A.priv_show_view PRIV_SHOW_VIEW, A.PRIV_OTHERS PRIV_OTHERS from oceanbase.__all_table_privilege_history A, (select tenant_id, user_id, database_name, table_name, max(schema_version) schema_version from oceanbase.__all_table_privilege_history group by tenant_id, user_id, database_name, database_name collate utf8mb4_bin, table_name, table_name collate utf8mb4_bin) B where A.tenant_id = B.tenant_id and A.user_id = B.user_id and A.database_name collate utf8mb4_bin = B.database_name collate utf8mb4_bin and A.schema_version = B.schema_version and A.table_name collate utf8mb4_bin = B.table_name collate utf8mb4_bin and A.is_deleted = 0 ) SELECT CAST(CONCAT('''', V.USER_NAME, '''', '@', '''', V.HOST, '''') AS CHAR(81)) AS GRANTEE , CAST('def' AS CHAR(512)) AS TABLE_CATALOG , CAST(V.DATABASE_NAME AS CHAR(128)) collate utf8mb4_name_case AS TABLE_SCHEMA , CAST(V.TABLE_NAME AS CHAR(64)) collate utf8mb4_name_case AS TABLE_NAME, CAST(V.PRIVILEGE_TYPE AS CHAR(64)) AS PRIVILEGE_TYPE , CAST(V.IS_GRANTABLE AS CHAR(3)) AS IS_GRANTABLE FROM (SELECT TP.DATABASE_NAME AS DATABASE_NAME, TP.TABLE_NAME AS TABLE_NAME, U.USER_NAME AS USER_NAME, U.HOST AS HOST, CASE WHEN V1.C1 = 1 AND TP.PRIV_ALTER = 1 THEN 'ALTER' WHEN V1.C1 = 2 AND TP.PRIV_CREATE = 1 THEN 'CREATE' WHEN V1.C1 = 4 AND TP.PRIV_DELETE = 1 THEN 'DELETE' WHEN V1.C1 = 5 AND TP.PRIV_DROP = 1 THEN 'DROP' WHEN V1.C1 = 7 AND TP.PRIV_INSERT = 1 THEN 'INSERT' WHEN V1.C1 = 8 AND TP.PRIV_UPDATE = 1 THEN 'UPDATE' WHEN V1.C1 = 9 AND TP.PRIV_SELECT = 1 THEN 'SELECT' WHEN V1.C1 = 10 AND TP.PRIV_INDEX = 1 THEN 'INDEX' WHEN V1.C1 = 11 AND TP.PRIV_CREATE_VIEW = 1 THEN 'CREATE VIEW' WHEN V1.C1 = 12 AND TP.PRIV_SHOW_VIEW = 1 THEN 'SHOW VIEW' ELSE NULL END PRIVILEGE_TYPE , CASE WHEN TP.PRIV_GRANT_OPTION = 1 THEN 'YES' WHEN TP.PRIV_GRANT_OPTION = 0 THEN 'NO' END IS_GRANTABLE FROM TABLE_PRIV TP, oceanbase.__all_user U, (SELECT 1 AS C1 UNION ALL SELECT 2 AS C1 UNION ALL SELECT 4 AS C1 UNION ALL SELECT 5 AS C1 UNION ALL SELECT 7 AS C1 UNION ALL SELECT 8 AS C1 UNION ALL SELECT 9 AS C1 UNION ALL SELECT 10 AS C1 UNION ALL SELECT 11 AS C1 UNION ALL SELECT 12 AS C1) V1, (SELECT USER_ID FROM oceanbase.__all_user WHERE TENANT_ID = 0 AND CONCAT(USER_NAME, '@', HOST) = CURRENT_USER()) CURR LEFT JOIN (SELECT USER_ID FROM DB_PRIV WHERE TENANT_ID = 0 AND DATABASE_NAME = 'mysql' AND PRIV_SELECT = 1) DB ON CURR.USER_ID = DB.USER_ID WHERE TP.TENANT_ID = 0 AND TP.TENANT_ID = U.TENANT_ID AND TP.USER_ID = U.USER_ID AND (DB.USER_ID IS NOT NULL OR 512 & CURRENT_USER_PRIV() = 512 OR TP.USER_ID = CURR.USER_ID)) V WHERE V.PRIVILEGE_TYPE IS NOT NULL utf8mb4 utf8mb4_general_ci +TABLE_PRIVILEGES CREATE VIEW `TABLE_PRIVILEGES` AS WITH DB_PRIV AS ( select A.tenant_id TENANT_ID, A.user_id USER_ID, A.database_name DATABASE_NAME, A.priv_alter PRIV_ALTER, A.priv_create PRIV_CREATE, A.priv_delete PRIV_DELETE, A.priv_drop PRIV_DROP, A.priv_grant_option PRIV_GRANT_OPTION, A.priv_insert PRIV_INSERT, A.priv_update PRIV_UPDATE, A.priv_select PRIV_SELECT, A.priv_index PRIV_INDEX, A.priv_create_view PRIV_CREATE_VIEW, A.priv_show_view PRIV_SHOW_VIEW, A.GMT_CREATE GMT_CREATE, A.GMT_MODIFIED GMT_MODIFIED, A.PRIV_OTHERS PRIV_OTHERS from oceanbase.__all_database_privilege_history A, (select tenant_id, user_id, database_name, max(schema_version) schema_version from oceanbase.__all_database_privilege_history group by tenant_id, user_id, database_name, database_name collate utf8mb4_bin) B where A.tenant_id = B.tenant_id and A.user_id = B.user_id and A.database_name collate utf8mb4_bin = B.database_name collate utf8mb4_bin and A.schema_version = B.schema_version and A.is_deleted = 0 ), TABLE_PRIV AS ( select A.tenant_id TENANT_ID, A.user_id USER_ID, A.database_name DATABASE_NAME, A.table_name TABLE_NAME, A.priv_alter PRIV_ALTER, A.priv_create PRIV_CREATE, A.priv_delete PRIV_DELETE, A.priv_drop PRIV_DROP, A.priv_grant_option PRIV_GRANT_OPTION, A.priv_insert PRIV_INSERT, A.priv_update PRIV_UPDATE, A.priv_select PRIV_SELECT, A.priv_index PRIV_INDEX, A.priv_create_view PRIV_CREATE_VIEW, A.priv_show_view PRIV_SHOW_VIEW, A.PRIV_OTHERS PRIV_OTHERS from oceanbase.__all_table_privilege_history A, (select tenant_id, user_id, database_name, table_name, max(schema_version) schema_version from oceanbase.__all_table_privilege_history group by tenant_id, user_id, database_name, database_name collate utf8mb4_bin, table_name, table_name collate utf8mb4_bin) B where A.tenant_id = B.tenant_id and A.user_id = B.user_id and A.database_name collate utf8mb4_bin = B.database_name collate utf8mb4_bin and A.schema_version = B.schema_version and A.table_name collate utf8mb4_bin = B.table_name collate utf8mb4_bin and A.is_deleted = 0 ) SELECT CAST(CONCAT('''', V.USER_NAME, '''', '@', '''', V.HOST, '''') AS CHAR(81)) AS GRANTEE , CAST('def' AS CHAR(512)) AS TABLE_CATALOG , CAST(V.DATABASE_NAME AS CHAR(128)) collate utf8mb4_name_case AS TABLE_SCHEMA , CAST(V.TABLE_NAME AS CHAR(64)) collate utf8mb4_name_case AS TABLE_NAME, CAST(V.PRIVILEGE_TYPE AS CHAR(64)) AS PRIVILEGE_TYPE , CAST(V.IS_GRANTABLE AS CHAR(3)) AS IS_GRANTABLE FROM (SELECT TP.DATABASE_NAME AS DATABASE_NAME, TP.TABLE_NAME AS TABLE_NAME, U.USER_NAME AS USER_NAME, U.HOST AS HOST, CASE WHEN V1.C1 = 1 AND TP.PRIV_ALTER = 1 THEN 'ALTER' WHEN V1.C1 = 2 AND TP.PRIV_CREATE = 1 THEN 'CREATE' WHEN V1.C1 = 4 AND TP.PRIV_DELETE = 1 THEN 'DELETE' WHEN V1.C1 = 5 AND TP.PRIV_DROP = 1 THEN 'DROP' WHEN V1.C1 = 7 AND TP.PRIV_INSERT = 1 THEN 'INSERT' WHEN V1.C1 = 8 AND TP.PRIV_UPDATE = 1 THEN 'UPDATE' WHEN V1.C1 = 9 AND TP.PRIV_SELECT = 1 THEN 'SELECT' WHEN V1.C1 = 10 AND TP.PRIV_INDEX = 1 THEN 'INDEX' WHEN V1.C1 = 11 AND TP.PRIV_CREATE_VIEW = 1 THEN 'CREATE VIEW' WHEN V1.C1 = 12 AND TP.PRIV_SHOW_VIEW = 1 THEN 'SHOW VIEW' WHEN V1.C1 = 22 AND (TP.PRIV_OTHERS & (1 << 6)) != 0 THEN 'REFERENCES' WHEN V1.C1 = 44 AND (TP.PRIV_OTHERS & (1 << 9)) != 0 THEN 'TRIGGER' ELSE NULL END PRIVILEGE_TYPE , CASE WHEN TP.PRIV_GRANT_OPTION = 1 THEN 'YES' WHEN TP.PRIV_GRANT_OPTION = 0 THEN 'NO' END IS_GRANTABLE FROM TABLE_PRIV TP, oceanbase.__all_user U, (SELECT 1 AS C1 UNION ALL SELECT 2 AS C1 UNION ALL SELECT 4 AS C1 UNION ALL SELECT 5 AS C1 UNION ALL SELECT 7 AS C1 UNION ALL SELECT 8 AS C1 UNION ALL SELECT 9 AS C1 UNION ALL SELECT 10 AS C1 UNION ALL SELECT 11 AS C1 UNION ALL SELECT 12 AS C1 UNION ALL SELECT 22 AS C1 UNION ALL SELECT 44 AS C1) V1, (SELECT USER_ID FROM oceanbase.__all_user WHERE TENANT_ID = 0 AND CONCAT(USER_NAME, '@', HOST) = CURRENT_USER()) CURR LEFT JOIN (SELECT USER_ID FROM DB_PRIV WHERE TENANT_ID = 0 AND DATABASE_NAME = 'mysql' AND PRIV_SELECT = 1) DB ON CURR.USER_ID = DB.USER_ID WHERE TP.TENANT_ID = 0 AND TP.TENANT_ID = U.TENANT_ID AND TP.USER_ID = U.USER_ID AND (DB.USER_ID IS NOT NULL OR 512 & CURRENT_USER_PRIV() = 512 OR TP.USER_ID = CURR.USER_ID)) V WHERE V.PRIVILEGE_TYPE IS NOT NULL utf8mb4 utf8mb4_general_ci desc information_schema.table_privileges; Field Type Null Key Default Extra GRANTEE varchar(81) NO diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/user_privileges.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/user_privileges.result index d953d0602..03a6d6e25 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/user_privileges.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/user_privileges.result @@ -1,6 +1,6 @@ show create table information_schema.user_privileges; View Create View character_set_client collation_connection -USER_PRIVILEGES CREATE VIEW `USER_PRIVILEGES` AS SELECT CAST(CONCAT('''', V.USER_NAME, '''', '@', '''', V.HOST, '''') AS CHAR(81)) AS GRANTEE , CAST('def' AS CHAR(512)) AS TABLE_CATALOG , CAST(V.PRIVILEGE_TYPE AS CHAR(64)) AS PRIVILEGE_TYPE , CAST(V.IS_GRANTABLE AS CHAR(3)) AS IS_GRANTABLE FROM (SELECT U.USER_NAME AS USER_NAME, U.HOST AS HOST, CASE WHEN V1.C1 = 1 AND U.PRIV_ALTER = 1 THEN 'ALTER' WHEN V1.C1 = 2 AND U.PRIV_CREATE = 1 THEN 'CREATE' WHEN V1.C1 = 3 AND U.PRIV_CREATE_USER = 1 THEN 'CREATE USER' WHEN V1.C1 = 4 AND U.PRIV_DELETE = 1 THEN 'DELETE' WHEN V1.C1 = 5 AND U.PRIV_DROP = 1 THEN 'DROP' WHEN V1.C1 = 7 AND U.PRIV_INSERT = 1 THEN 'INSERT' WHEN V1.C1 = 8 AND U.PRIV_UPDATE = 1 THEN 'UPDATE' WHEN V1.C1 = 9 AND U.PRIV_SELECT = 1 THEN 'SELECT' WHEN V1.C1 = 10 AND U.PRIV_INDEX = 1 THEN 'INDEX' WHEN V1.C1 = 11 AND U.PRIV_CREATE_VIEW = 1 THEN 'CREATE VIEW' WHEN V1.C1 = 12 AND U.PRIV_SHOW_VIEW = 1 THEN 'SHOW VIEW' WHEN V1.C1 = 13 AND U.PRIV_SHOW_DB = 1 THEN 'SHOW DATABASES' WHEN V1.C1 = 14 AND U.PRIV_SUPER = 1 THEN 'SUPER' WHEN V1.C1 = 15 AND U.PRIV_PROCESS = 1 THEN 'PROCESS' WHEN V1.C1 = 17 AND U.PRIV_CREATE_SYNONYM = 1 THEN 'CREATE SYNONYM' WHEN V1.C1 = 23 AND (U.PRIV_OTHERS & (1 << 0)) != 0 THEN 'EXECUTE' WHEN V1.C1 = 27 AND U.PRIV_FILE = 1 THEN 'FILE' WHEN V1.C1 = 28 AND U.PRIV_ALTER_TENANT = 1 THEN 'ALTER TENANT' WHEN V1.C1 = 29 AND U.PRIV_ALTER_SYSTEM = 1 THEN 'ALTER SYSTEM' WHEN V1.C1 = 30 AND U.PRIV_CREATE_RESOURCE_POOL = 1 THEN 'CREATE RESOURCE POOL' WHEN V1.C1 = 31 AND U.PRIV_CREATE_RESOURCE_UNIT = 1 THEN 'CREATE RESOURCE UNIT' WHEN V1.C1 = 33 AND U.PRIV_REPL_SLAVE = 1 THEN 'REPLICATION SLAVE' WHEN V1.C1 = 34 AND U.PRIV_REPL_CLIENT = 1 THEN 'REPLICATION CLIENT' WHEN V1.C1 = 35 AND U.PRIV_DROP_DATABASE_LINK = 1 THEN 'DROP DATABASE LINK' WHEN V1.C1 = 36 AND U.PRIV_CREATE_DATABASE_LINK = 1 THEN 'CREATE DATABASE LINK' WHEN V1.C1 = 37 AND (U.PRIV_OTHERS & (1 << 1)) != 0 THEN 'ALTER ROUTINE' WHEN V1.C1 = 38 AND (U.PRIV_OTHERS & (1 << 2)) != 0 THEN 'CREATE ROUTINE' WHEN V1.C1 = 39 AND (U.PRIV_OTHERS & (1 << 3)) != 0 THEN 'CREATE TABLESPACE' WHEN V1.C1 = 40 AND (U.PRIV_OTHERS & (1 << 4)) != 0 THEN 'SHUTDOWN' WHEN V1.C1 = 41 AND (U.PRIV_OTHERS & (1 << 5)) != 0 THEN 'RELOAD' WHEN V1.C1 = 0 AND U.PRIV_ALTER = 0 AND U.PRIV_CREATE = 0 AND U.PRIV_CREATE_USER = 0 AND U.PRIV_DELETE = 0 AND U.PRIV_DROP = 0 AND U.PRIV_INSERT = 0 AND U.PRIV_UPDATE = 0 AND U.PRIV_SELECT = 0 AND U.PRIV_INDEX = 0 AND U.PRIV_CREATE_VIEW = 0 AND U.PRIV_SHOW_VIEW = 0 AND U.PRIV_SHOW_DB = 0 AND U.PRIV_SUPER = 0 AND U.PRIV_PROCESS = 0 AND U.PRIV_CREATE_SYNONYM = 0 AND U.PRIV_FILE = 0 AND U.PRIV_ALTER_TENANT = 0 AND U.PRIV_ALTER_SYSTEM = 0 AND U.PRIV_CREATE_RESOURCE_POOL = 0 AND U.PRIV_CREATE_RESOURCE_UNIT = 0 AND U.PRIV_REPL_SLAVE = 0 AND U.PRIV_REPL_CLIENT = 0 AND U.PRIV_DROP_DATABASE_LINK = 0 AND U.PRIV_CREATE_DATABASE_LINK = 0 AND U.PRIV_OTHERS = 0 THEN 'USAGE' END PRIVILEGE_TYPE , CASE WHEN U.PRIV_GRANT_OPTION = 0 THEN 'NO' WHEN U.PRIV_ALTER = 0 AND U.PRIV_CREATE = 0 AND U.PRIV_CREATE_USER = 0 AND U.PRIV_DELETE = 0 AND U.PRIV_DROP = 0 AND U.PRIV_INSERT = 0 AND U.PRIV_UPDATE = 0 AND U.PRIV_SELECT = 0 AND U.PRIV_INDEX = 0 AND U.PRIV_CREATE_VIEW = 0 AND U.PRIV_SHOW_VIEW = 0 AND U.PRIV_SHOW_DB = 0 AND U.PRIV_SUPER = 0 AND U.PRIV_PROCESS = 0 AND U.PRIV_CREATE_SYNONYM = 0 AND U.PRIV_FILE = 0 AND U.PRIV_ALTER_TENANT = 0 AND U.PRIV_ALTER_SYSTEM = 0 AND U.PRIV_CREATE_RESOURCE_POOL = 0 AND U.PRIV_CREATE_RESOURCE_UNIT = 0 AND U.PRIV_REPL_SLAVE = 0 AND U.PRIV_REPL_CLIENT = 0 AND U.PRIV_DROP_DATABASE_LINK = 0 AND U.PRIV_CREATE_DATABASE_LINK = 0 AND U.PRIV_OTHERS = 0 THEN 'NO' WHEN U.PRIV_GRANT_OPTION = 1 THEN 'YES' END IS_GRANTABLE FROM oceanbase.__all_user U, (SELECT 0 AS C1 UNION ALL SELECT 1 AS C1 UNION ALL SELECT 2 AS C1 UNION ALL SELECT 3 AS C1 UNION ALL SELECT 4 AS C1 UNION ALL SELECT 5 AS C1 UNION ALL SELECT 7 AS C1 UNION ALL SELECT 8 AS C1 UNION ALL SELECT 9 AS C1 UNION ALL SELECT 10 AS C1 UNION ALL SELECT 11 AS C1 UNION ALL SELECT 12 AS C1 UNION ALL SELECT 13 AS C1 UNION ALL SELECT 14 AS C1 UNION ALL SELECT 15 AS C1 UNION ALL SELECT 17 AS C1 UNION ALL SELECT 23 AS C1 UNION ALL SELECT 27 AS C1 UNION ALL SELECT 28 AS C1 UNION ALL SELECT 29 AS C1 UNION ALL SELECT 30 AS C1 UNION ALL SELECT 31 AS C1 UNION ALL SELECT 33 AS C1 UNION ALL SELECT 34 AS C1 UNION ALL SELECT 35 AS C1 UNION ALL SELECT 36 AS C1 UNION ALL SELECT 37 AS C1 UNION ALL SELECT 38 AS C1 UNION ALL SELECT 39 AS C1 UNION ALL SELECT 40 AS C1 UNION ALL SELECT 41 AS C1) V1, (SELECT USER_ID FROM oceanbase.__all_user WHERE TENANT_ID = 0 AND CONCAT(USER_NAME, '@', HOST) = CURRENT_USER()) CURR LEFT JOIN (SELECT USER_ID FROM oceanbase.__all_database_privilege WHERE TENANT_ID = 0 AND DATABASE_NAME = 'mysql' AND PRIV_SELECT = 1) DB ON CURR.USER_ID = DB.USER_ID WHERE U.TENANT_ID = 0 AND (DB.USER_ID IS NOT NULL OR 512 & CURRENT_USER_PRIV() = 512 OR U.USER_ID = CURR.USER_ID)) V WHERE V.PRIVILEGE_TYPE IS NOT NULL utf8mb4 utf8mb4_general_ci +USER_PRIVILEGES CREATE VIEW `USER_PRIVILEGES` AS SELECT CAST(CONCAT('''', V.USER_NAME, '''', '@', '''', V.HOST, '''') AS CHAR(81)) AS GRANTEE , CAST('def' AS CHAR(512)) AS TABLE_CATALOG , CAST(V.PRIVILEGE_TYPE AS CHAR(64)) AS PRIVILEGE_TYPE , CAST(V.IS_GRANTABLE AS CHAR(3)) AS IS_GRANTABLE FROM (SELECT U.USER_NAME AS USER_NAME, U.HOST AS HOST, CASE WHEN V1.C1 = 1 AND U.PRIV_ALTER = 1 THEN 'ALTER' WHEN V1.C1 = 2 AND U.PRIV_CREATE = 1 THEN 'CREATE' WHEN V1.C1 = 3 AND U.PRIV_CREATE_USER = 1 THEN 'CREATE USER' WHEN V1.C1 = 4 AND U.PRIV_DELETE = 1 THEN 'DELETE' WHEN V1.C1 = 5 AND U.PRIV_DROP = 1 THEN 'DROP' WHEN V1.C1 = 7 AND U.PRIV_INSERT = 1 THEN 'INSERT' WHEN V1.C1 = 8 AND U.PRIV_UPDATE = 1 THEN 'UPDATE' WHEN V1.C1 = 9 AND U.PRIV_SELECT = 1 THEN 'SELECT' WHEN V1.C1 = 10 AND U.PRIV_INDEX = 1 THEN 'INDEX' WHEN V1.C1 = 11 AND U.PRIV_CREATE_VIEW = 1 THEN 'CREATE VIEW' WHEN V1.C1 = 12 AND U.PRIV_SHOW_VIEW = 1 THEN 'SHOW VIEW' WHEN V1.C1 = 13 AND U.PRIV_SHOW_DB = 1 THEN 'SHOW DATABASES' WHEN V1.C1 = 14 AND U.PRIV_SUPER = 1 THEN 'SUPER' WHEN V1.C1 = 15 AND U.PRIV_PROCESS = 1 THEN 'PROCESS' WHEN V1.C1 = 17 AND U.PRIV_CREATE_SYNONYM = 1 THEN 'CREATE SYNONYM' WHEN V1.C1 = 22 AND (U.PRIV_OTHERS & (1 << 6)) != 0 THEN 'REFERENCES' WHEN V1.C1 = 23 AND (U.PRIV_OTHERS & (1 << 0)) != 0 THEN 'EXECUTE' WHEN V1.C1 = 27 AND U.PRIV_FILE = 1 THEN 'FILE' WHEN V1.C1 = 28 AND U.PRIV_ALTER_TENANT = 1 THEN 'ALTER TENANT' WHEN V1.C1 = 29 AND U.PRIV_ALTER_SYSTEM = 1 THEN 'ALTER SYSTEM' WHEN V1.C1 = 30 AND U.PRIV_CREATE_RESOURCE_POOL = 1 THEN 'CREATE RESOURCE POOL' WHEN V1.C1 = 31 AND U.PRIV_CREATE_RESOURCE_UNIT = 1 THEN 'CREATE RESOURCE UNIT' WHEN V1.C1 = 33 AND U.PRIV_REPL_SLAVE = 1 THEN 'REPLICATION SLAVE' WHEN V1.C1 = 34 AND U.PRIV_REPL_CLIENT = 1 THEN 'REPLICATION CLIENT' WHEN V1.C1 = 35 AND U.PRIV_DROP_DATABASE_LINK = 1 THEN 'DROP DATABASE LINK' WHEN V1.C1 = 36 AND U.PRIV_CREATE_DATABASE_LINK = 1 THEN 'CREATE DATABASE LINK' WHEN V1.C1 = 37 AND (U.PRIV_OTHERS & (1 << 1)) != 0 THEN 'ALTER ROUTINE' WHEN V1.C1 = 38 AND (U.PRIV_OTHERS & (1 << 2)) != 0 THEN 'CREATE ROUTINE' WHEN V1.C1 = 39 AND (U.PRIV_OTHERS & (1 << 3)) != 0 THEN 'CREATE TABLESPACE' WHEN V1.C1 = 40 AND (U.PRIV_OTHERS & (1 << 4)) != 0 THEN 'SHUTDOWN' WHEN V1.C1 = 41 AND (U.PRIV_OTHERS & (1 << 5)) != 0 THEN 'RELOAD' WHEN V1.C1 = 42 AND (U.PRIV_OTHERS & (1 << 7)) != 0 THEN 'CREATE ROLE' WHEN V1.C1 = 43 AND (U.PRIV_OTHERS & (1 << 8)) != 0 THEN 'DROP ROLE' WHEN V1.C1 = 44 AND (U.PRIV_OTHERS & (1 << 9)) != 0 THEN 'TRIGGER' WHEN V1.C1 = 0 AND U.PRIV_ALTER = 0 AND U.PRIV_CREATE = 0 AND U.PRIV_CREATE_USER = 0 AND U.PRIV_DELETE = 0 AND U.PRIV_DROP = 0 AND U.PRIV_INSERT = 0 AND U.PRIV_UPDATE = 0 AND U.PRIV_SELECT = 0 AND U.PRIV_INDEX = 0 AND U.PRIV_CREATE_VIEW = 0 AND U.PRIV_SHOW_VIEW = 0 AND U.PRIV_SHOW_DB = 0 AND U.PRIV_SUPER = 0 AND U.PRIV_PROCESS = 0 AND U.PRIV_CREATE_SYNONYM = 0 AND U.PRIV_FILE = 0 AND U.PRIV_ALTER_TENANT = 0 AND U.PRIV_ALTER_SYSTEM = 0 AND U.PRIV_CREATE_RESOURCE_POOL = 0 AND U.PRIV_CREATE_RESOURCE_UNIT = 0 AND U.PRIV_REPL_SLAVE = 0 AND U.PRIV_REPL_CLIENT = 0 AND U.PRIV_DROP_DATABASE_LINK = 0 AND U.PRIV_CREATE_DATABASE_LINK = 0 AND U.PRIV_OTHERS = 0 THEN 'USAGE' END PRIVILEGE_TYPE , CASE WHEN U.PRIV_GRANT_OPTION = 0 THEN 'NO' WHEN U.PRIV_ALTER = 0 AND U.PRIV_CREATE = 0 AND U.PRIV_CREATE_USER = 0 AND U.PRIV_DELETE = 0 AND U.PRIV_DROP = 0 AND U.PRIV_INSERT = 0 AND U.PRIV_UPDATE = 0 AND U.PRIV_SELECT = 0 AND U.PRIV_INDEX = 0 AND U.PRIV_CREATE_VIEW = 0 AND U.PRIV_SHOW_VIEW = 0 AND U.PRIV_SHOW_DB = 0 AND U.PRIV_SUPER = 0 AND U.PRIV_PROCESS = 0 AND U.PRIV_CREATE_SYNONYM = 0 AND U.PRIV_FILE = 0 AND U.PRIV_ALTER_TENANT = 0 AND U.PRIV_ALTER_SYSTEM = 0 AND U.PRIV_CREATE_RESOURCE_POOL = 0 AND U.PRIV_CREATE_RESOURCE_UNIT = 0 AND U.PRIV_REPL_SLAVE = 0 AND U.PRIV_REPL_CLIENT = 0 AND U.PRIV_DROP_DATABASE_LINK = 0 AND U.PRIV_CREATE_DATABASE_LINK = 0 AND U.PRIV_OTHERS = 0 THEN 'NO' WHEN U.PRIV_GRANT_OPTION = 1 THEN 'YES' END IS_GRANTABLE FROM oceanbase.__all_user U, (SELECT 0 AS C1 UNION ALL SELECT 1 AS C1 UNION ALL SELECT 2 AS C1 UNION ALL SELECT 3 AS C1 UNION ALL SELECT 4 AS C1 UNION ALL SELECT 5 AS C1 UNION ALL SELECT 7 AS C1 UNION ALL SELECT 8 AS C1 UNION ALL SELECT 9 AS C1 UNION ALL SELECT 10 AS C1 UNION ALL SELECT 11 AS C1 UNION ALL SELECT 12 AS C1 UNION ALL SELECT 13 AS C1 UNION ALL SELECT 14 AS C1 UNION ALL SELECT 15 AS C1 UNION ALL SELECT 17 AS C1 UNION ALL SELECT 22 AS C1 UNION ALL SELECT 23 AS C1 UNION ALL SELECT 27 AS C1 UNION ALL SELECT 28 AS C1 UNION ALL SELECT 29 AS C1 UNION ALL SELECT 30 AS C1 UNION ALL SELECT 31 AS C1 UNION ALL SELECT 33 AS C1 UNION ALL SELECT 34 AS C1 UNION ALL SELECT 35 AS C1 UNION ALL SELECT 36 AS C1 UNION ALL SELECT 37 AS C1 UNION ALL SELECT 38 AS C1 UNION ALL SELECT 39 AS C1 UNION ALL SELECT 40 AS C1 UNION ALL SELECT 41 AS C1 UNION ALL SELECT 42 AS C1 UNION ALL SELECT 43 AS C1 UNION ALL SELECT 44 AS C1) V1, (SELECT USER_ID FROM oceanbase.__all_user WHERE TENANT_ID = 0 AND CONCAT(USER_NAME, '@', HOST) = CURRENT_USER()) CURR LEFT JOIN (SELECT USER_ID FROM oceanbase.__all_database_privilege WHERE TENANT_ID = 0 AND DATABASE_NAME = 'mysql' AND PRIV_SELECT = 1) DB ON CURR.USER_ID = DB.USER_ID WHERE U.TENANT_ID = 0 AND (DB.USER_ID IS NOT NULL OR 512 & CURRENT_USER_PRIV() = 512 OR U.USER_ID = CURR.USER_ID)) V WHERE V.PRIVILEGE_TYPE IS NOT NULL utf8mb4 utf8mb4_general_ci desc information_schema.user_privileges; Field Type Null Key Default Extra GRANTEE varchar(81) NO @@ -53,6 +53,7 @@ GRANTEE TABLE_CATALOG PRIVILEGE_TYPE IS_GRANTABLE 'root'@'%' def SUPER YES 'root'@'%' def PROCESS YES 'root'@'%' def CREATE SYNONYM YES +'root'@'%' def REFERENCES YES 'root'@'%' def EXECUTE YES 'root'@'%' def FILE YES 'root'@'%' def ALTER TENANT YES @@ -68,6 +69,9 @@ GRANTEE TABLE_CATALOG PRIVILEGE_TYPE IS_GRANTABLE 'root'@'%' def CREATE TABLESPACE YES 'root'@'%' def SHUTDOWN YES 'root'@'%' def RELOAD YES +'root'@'%' def CREATE ROLE YES +'root'@'%' def DROP ROLE YES +'root'@'%' def TRIGGER YES 'admin'@'%' def ALTER YES 'admin'@'%' def CREATE YES 'admin'@'%' def CREATE USER YES @@ -83,6 +87,7 @@ GRANTEE TABLE_CATALOG PRIVILEGE_TYPE IS_GRANTABLE 'admin'@'%' def SUPER YES 'admin'@'%' def PROCESS YES 'admin'@'%' def CREATE SYNONYM YES +'admin'@'%' def REFERENCES YES 'admin'@'%' def EXECUTE YES 'admin'@'%' def FILE YES 'admin'@'%' def ALTER TENANT YES @@ -98,3 +103,6 @@ GRANTEE TABLE_CATALOG PRIVILEGE_TYPE IS_GRANTABLE 'admin'@'%' def CREATE TABLESPACE YES 'admin'@'%' def SHUTDOWN YES 'admin'@'%' def RELOAD YES +'admin'@'%' def CREATE ROLE YES +'admin'@'%' def DROP ROLE YES +'admin'@'%' def TRIGGER YES diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/views.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/views.result index 08a24468d..79028718b 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/views.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/views.result @@ -6,10 +6,12 @@ def mysql time_zone_transition SELECT time_zone_id as Time_zone_id, def mysql time_zone_name SELECT name as Name, time_zone_id as Time_zone_id FROM oceanbase.__all_tenant_time_zone_name NONE NO NONE NONE utf8mb4 utf8mb4_general_ci def mysql time_zone SELECT time_zone_id as Time_zone_id, use_leap_seconds as Use_leap_seconds FROM oceanbase.__all_tenant_time_zone NONE NO NONE NONE utf8mb4 utf8mb4_general_ci def mysql role_edges SELECT cast(from_user.host AS char(255)) FROM_HOST, cast(from_user.user_name AS char(128)) FROM_USER, cast(to_user.host AS char(255)) TO_HOST, cast(to_user.user_name AS char(128)) TO_USER, cast(CASE role_map.admin_option WHEN 1 THEN 'Y' ELSE 'N' END AS char(1)) WITH_ADMIN_OPTION FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id; NONE NO NONE NONE utf8mb4 utf8mb4_general_ci -def mysql procs_priv SELECT cast(b.host as char(60)) as Host, cast(a.database_name as char(64)) as Db, cast(b.user_name as char(32)) as User, cast(a.routine_name as char(64)) as Routine_name, case when a.routine_type = 1 then 'PROCEDURE' else 'FUNCTION' end as Routine_type, cast(c.priv_user as char(93)) as Grantor, substr(concat(case when (a.all_priv & 1) > 0 then ',Execute' else '' end, case when (a.all_priv & 2) > 0 then ',Alter Routine' else '' end, case when (a.all_priv & 4) > 0 then ',Grant' else '' end), 2) as Proc_priv, cast(a.gmt_modified as date) as Timestamp FROM oceanbase.__all_routine_privilege a, oceanbase.__all_user b, oceanbase.__all_routine c, oceanbase.__all_database d WHERE a.tenant_id = b.tenant_id AND a.user_id = b.user_id AND a.tenant_id = d.tenant_id and a.database_name = d.database_name AND a.tenant_id = c.tenant_id AND a.routine_name = c.routine_name AND a.routine_type = c.routine_type AND c.database_id = d.database_id AND c.package_id = -1; NONE NO NONE NONE utf8mb4 utf8mb4_general_ci +def mysql procs_priv SELECT cast(b.host as char(60)) as Host, cast(a.database_name as char(64)) as Db, cast(b.user_name as char(32)) as User, cast(a.routine_name as char(64)) as Routine_name, case when a.routine_type = 1 then 'PROCEDURE' else 'FUNCTION' end as Routine_type, cast(concat(a.grantor, '@', a.grantor_host) as char(93)) as Grantor, substr(concat(case when (a.all_priv & 1) > 0 then ',Execute' else '' end, case when (a.all_priv & 2) > 0 then ',Alter Routine' else '' end, case when (a.all_priv & 4) > 0 then ',Grant' else '' end), 2) as Proc_priv, cast(a.gmt_modified as date) as Timestamp FROM oceanbase.__all_routine_privilege a, oceanbase.__all_user b WHERE a.tenant_id = 0 and a.tenant_id = b.tenant_id AND a.user_id = b.user_id; NONE NO NONE NONE utf8mb4 utf8mb4_general_ci def mysql func SELECT name, ret, dl, type FROM oceanbase.__all_func NONE NO NONE NONE utf8mb4 utf8mb4_general_ci def mysql default_roles SELECT cast(to_user.host AS char(255)) HOST, cast(to_user.user_name AS char(128)) USER, cast(from_user.host AS char(255)) DEFAULT_ROLE_HOST, cast(from_user.user_name AS char(128)) DEFAULT_ROLE_USER FROM oceanbase.__all_tenant_role_grantee_map role_map, oceanbase.__all_user from_user, oceanbase.__all_user to_user WHERE role_map.tenant_id = from_user.tenant_id AND role_map.tenant_id = to_user.tenant_id AND role_map.grantee_id = to_user.user_id AND role_map.role_id = from_user.user_id AND role_map.disable_flag = 0; NONE NO NONE NONE utf8mb4 utf8mb4_general_ci def mysql columns_priv SELECT cast(b.host as char(255)) as Host, cast(a.database_name as char(128)) as Db, cast(b.user_name as char(128)) as User, cast(a.table_name as char(128)) as Table_name, cast(a.column_name as char(128)) as Column_name, substr(concat(case when (a.all_priv & 1) > 0 then ',Select' else '' end, case when (a.all_priv & 2) > 0 then ',Insert' else '' end, case when (a.all_priv & 4) > 0 then ',Update' else '' end, case when (a.all_priv & 8) > 0 then ',References' else '' end), 2) as Column_priv, cast(a.gmt_modified as datetime) as Timestamp FROM oceanbase.__all_column_privilege a, oceanbase.__all_user b WHERE a.tenant_id = 0 and a.tenant_id = b.tenant_id AND a.user_id = b.user_id NONE NO NONE NONE utf8mb4 utf8mb4_general_ci +def mysql audit_log_user SELECT CAST(user_name AS CHAR(128)) collate utf8mb4_bin as USER, host as HOST, CAST(filter_name AS CHAR(64)) collate utf8mb4_bin as FILTERNAME FROM oceanbase.__all_audit_log_user WHERE tenant_id = 0 and is_deleted = 0; NONE NO NONE NONE utf8mb4 utf8mb4_general_ci +def mysql audit_log_filter SELECT CAST(filter_name AS CHAR(64)) collate utf8mb4_bin as NAME, definition as FILTER FROM oceanbase.__all_audit_log_filter WHERE tenant_id = 0 and is_deleted = 0; NONE NO NONE NONE utf8mb4 utf8mb4_general_ci show create view views; View Create View character_set_client collation_connection VIEWS CREATE VIEW `VIEWS` AS select cast('def' as CHAR(64)) AS TABLE_CATALOG, d.database_name collate utf8mb4_name_case as TABLE_SCHEMA, t.table_name collate utf8mb4_name_case as TABLE_NAME, t.view_definition as VIEW_DEFINITION, case t.view_check_option when 1 then 'LOCAL' when 2 then 'CASCADED' else 'NONE' end as CHECK_OPTION, case t.view_is_updatable when 1 then 'YES' else 'NO' end as IS_UPDATABLE, cast((case t.define_user_id when -1 then 'NONE' else concat(u.user_name, '@', u.host) end) as CHAR(288)) as DEFINER, cast('NONE' as CHAR(7)) AS SECURITY_TYPE, cast((case t.collation_type when 45 then 'utf8mb4' else 'NONE' end) as CHAR(64)) AS CHARACTER_SET_CLIENT, cast((case t.collation_type when 45 then 'utf8mb4_general_ci' else 'NONE' end) as CHAR(64)) AS COLLATION_CONNECTION from oceanbase.__all_table as t join oceanbase.__all_database as d on t.tenant_id = d.tenant_id and t.database_id = d.database_id left join oceanbase.__all_user as u on t.tenant_id = u.tenant_id and t.define_user_id = u.user_id and t.define_user_id != -1 where t.tenant_id = 0 and t.table_type in (1, 4) and t.table_mode >> 12 & 15 in (0,1) and d.in_recyclebin = 0 and d.database_name != '__recyclebin' and d.database_name != 'information_schema' and d.database_name != 'oceanbase' and 0 = sys_privilege_check('table_acc', effective_tenant_id(), d.database_name, t.table_name) utf8mb4 utf8mb4_general_ci diff --git a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_aes_encrypt.result b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_aes_encrypt.result index c9d196988..c6e238cfd 100644 --- a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_aes_encrypt.result +++ b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_aes_encrypt.result @@ -3,7 +3,7 @@ alter system flush plan cache global; set ob_enable_plan_cache = 0; set block_encryption_mode = 0; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; Variable_name Value block_encryption_mode aes-128-ecb select 1 as flag from dual where aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') = aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'); @@ -295,7 +295,7 @@ select aes_decrypt(aes_encrypt('没有什么好恐惧的, 除了恐惧本身', ' | 没有什么好恐惧的, 除了恐惧本身 | +----------------------------------------------------------------------------------------+ set block_encryption_mode = 1; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; Variable_name Value block_encryption_mode aes-192-ecb select 1 as flag from dual where aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') = aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'); @@ -587,7 +587,7 @@ select aes_decrypt(aes_encrypt('没有什么好恐惧的, 除了恐惧本身', ' | 没有什么好恐惧的, 除了恐惧本身 | +----------------------------------------------------------------------------------------+ set block_encryption_mode = 2; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; Variable_name Value block_encryption_mode aes-256-ecb select 1 as flag from dual where aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') = aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'); @@ -879,7 +879,7 @@ select aes_decrypt(aes_encrypt('没有什么好恐惧的, 除了恐惧本身', ' | 没有什么好恐惧的, 除了恐惧本身 | +----------------------------------------------------------------------------------------+ set block_encryption_mode = 3; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; Variable_name Value block_encryption_mode aes-128-cbc select 1 as flag from dual where aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') = aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'); @@ -985,7 +985,7 @@ select aes_decrypt(aes_encrypt('午后的天气还不错', '莎士比亚', 'asda | 午后的天气还不错 | +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ set block_encryption_mode = 4; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; Variable_name Value block_encryption_mode aes-192-cbc select 1 as flag from dual where aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') = aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'); @@ -1091,7 +1091,7 @@ select aes_decrypt(aes_encrypt('午后的天气还不错', '莎士比亚', 'asda | 午后的天气还不错 | +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ set block_encryption_mode = 5; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; Variable_name Value block_encryption_mode aes-256-cbc select 1 as flag from dual where aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') = aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'); @@ -1197,7 +1197,7 @@ select aes_decrypt(aes_encrypt('午后的天气还不错', '莎士比亚', 'asda | 午后的天气还不错 | +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ set block_encryption_mode = 6; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; Variable_name Value block_encryption_mode aes-128-cfb1 select 1 as flag from dual where aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') = aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'); @@ -1303,7 +1303,7 @@ select aes_decrypt(aes_encrypt('午后的天气还不错', '莎士比亚', 'asda | 午后的天气还不错 | +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ set block_encryption_mode = 7; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; Variable_name Value block_encryption_mode aes-192-cfb1 select 1 as flag from dual where aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') = aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'); @@ -1409,7 +1409,7 @@ select aes_decrypt(aes_encrypt('午后的天气还不错', '莎士比亚', 'asda | 午后的天气还不错 | +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ set block_encryption_mode = 8; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; Variable_name Value block_encryption_mode aes-256-cfb1 select 1 as flag from dual where aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') = aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'); @@ -1515,7 +1515,7 @@ select aes_decrypt(aes_encrypt('午后的天气还不错', '莎士比亚', 'asda | 午后的天气还不错 | +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ set block_encryption_mode = 9; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; Variable_name Value block_encryption_mode aes-128-cfb8 select 1 as flag from dual where aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') = aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'); @@ -1621,7 +1621,7 @@ select aes_decrypt(aes_encrypt('午后的天气还不错', '莎士比亚', 'asda | 午后的天气还不错 | +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ set block_encryption_mode = 10; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; Variable_name Value block_encryption_mode aes-192-cfb8 select 1 as flag from dual where aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') = aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'); @@ -1727,7 +1727,7 @@ select aes_decrypt(aes_encrypt('午后的天气还不错', '莎士比亚', 'asda | 午后的天气还不错 | +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ set block_encryption_mode = 11; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; Variable_name Value block_encryption_mode aes-256-cfb8 select 1 as flag from dual where aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') = aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'); @@ -1833,7 +1833,7 @@ select aes_decrypt(aes_encrypt('午后的天气还不错', '莎士比亚', 'asda | 午后的天气还不错 | +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ set block_encryption_mode = 12; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; Variable_name Value block_encryption_mode aes-128-cfb128 select 1 as flag from dual where aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') = aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'); @@ -1939,7 +1939,7 @@ select aes_decrypt(aes_encrypt('午后的天气还不错', '莎士比亚', 'asda | 午后的天气还不错 | +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ set block_encryption_mode = 13; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; Variable_name Value block_encryption_mode aes-192-cfb128 select 1 as flag from dual where aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') = aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'); @@ -2045,7 +2045,7 @@ select aes_decrypt(aes_encrypt('午后的天气还不错', '莎士比亚', 'asda | 午后的天气还不错 | +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ set block_encryption_mode = 14; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; Variable_name Value block_encryption_mode aes-256-cfb128 select 1 as flag from dual where aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') = aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'); @@ -2151,7 +2151,7 @@ select aes_decrypt(aes_encrypt('午后的天气还不错', '莎士比亚', 'asda | 午后的天气还不错 | +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ set block_encryption_mode = 15; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; Variable_name Value block_encryption_mode aes-128-ofb select 1 as flag from dual where aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') = aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'); @@ -2256,4 +2256,231 @@ select aes_decrypt(aes_encrypt('午后的天气还不错', '莎士比亚', 'asda +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | 午后的天气还不错 | +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +set block_encryption_mode = 16; +show variables like 'block_encryption_mode'; +Variable_name Value +block_encryption_mode aes-192-ofb +select 1 as flag from dual where aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') = aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'); ++------+ +| flag | ++------+ +| 1 | ++------+ +select hex(aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj')) from dual; ++--------------------------------------------------------------------------------------------------+ +| hex(aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj')) | ++--------------------------------------------------------------------------------------------------+ +| 923FA777D4A67CCFE6280ADC | ++--------------------------------------------------------------------------------------------------+ +select hex(aes_encrypt('asdas123123dasdasd', '12312313123', '$$&$(*&(%*&(*%&($&*%&(dkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj')) from dual; ++-------------------------------------------------------------------------------------------------------------------------+ +| hex(aes_encrypt('asdas123123dasdasd', '12312313123', '$$&$(*&(%*&(*%&($&*%&(dkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj')) | ++-------------------------------------------------------------------------------------------------------------------------+ +| 0EBF15EF11345391A512133D7D5EB76C5536 | ++-------------------------------------------------------------------------------------------------------------------------+ +select hex(aes_encrypt('asdasdasdasd', '', '1290380129038129038012938')) from dual; ++-------------------------------------------------------------------+ +| hex(aes_encrypt('asdasdasdasd', '', '1290380129038129038012938')) | ++-------------------------------------------------------------------+ +| 909F82502BCB6342EEE03CF7 | ++-------------------------------------------------------------------+ +select hex(aes_encrypt('', 'abc', '121903810293801289301298301982301928')) from dual; ++---------------------------------------------------------------------+ +| hex(aes_encrypt('', 'abc', '121903810293801289301298301982301928')) | ++---------------------------------------------------------------------+ +| | ++---------------------------------------------------------------------+ +select hex(aes_encrypt('asdasdasdasd', 'asdasdas', '1230121231230381209380913820912830')) from dual; ++------------------------------------------------------------------------------------+ +| hex(aes_encrypt('asdasdasdasd', 'asdasdas', '1230121231230381209380913820912830')) | ++------------------------------------------------------------------------------------+ +| 8FADC5A29B35D81339D014FB | ++------------------------------------------------------------------------------------+ +select aes_decrypt(aes_encrypt('asdasdasdasd', '', '1290380129038129038012938'), '','1290380129038129038012938') from dual; ++-----------------------------------------------------------------------------------------------------------+ +| aes_decrypt(aes_encrypt('asdasdasdasd', '', '1290380129038129038012938'), '','1290380129038129038012938') | ++-----------------------------------------------------------------------------------------------------------+ +| asdasdasdasd | ++-----------------------------------------------------------------------------------------------------------+ +select aes_decrypt(aes_encrypt('', 'abc', '121903810293801289301298301982301928'), 'abc', '121903810293801289301298301982301928') from dual; ++----------------------------------------------------------------------------------------------------------------------------+ +| aes_decrypt(aes_encrypt('', 'abc', '121903810293801289301298301982301928'), 'abc', '121903810293801289301298301982301928') | ++----------------------------------------------------------------------------------------------------------------------------+ +| | ++----------------------------------------------------------------------------------------------------------------------------+ +select aes_decrypt(aes_encrypt('asdasdasdasd', 'asdasdas', '1230121231230381209380913820912830'),'asdasdas', '1230121231230381209380913820912830') from dual; ++---------------------------------------------------------------------------------------------------------------------------------------------+ +| aes_decrypt(aes_encrypt('asdasdasdasd', 'asdasdas', '1230121231230381209380913820912830'),'asdasdas', '1230121231230381209380913820912830') | ++---------------------------------------------------------------------------------------------------------------------------------------------+ +| asdasdasdasd | ++---------------------------------------------------------------------------------------------------------------------------------------------+ +select hex(aes_encrypt('中文字符加一点', '你好', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj')) from dual; ++------------------------------------------------------------------------------------------------------+ +| hex(aes_encrypt('中文字符加一点', '你好', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj')) | ++------------------------------------------------------------------------------------------------------+ +| FF9FCD4F8E34B3066B313EA0CD085F39B498F6AA26 | ++------------------------------------------------------------------------------------------------------+ +select hex(aes_encrypt('测试', '是不是', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj')) from dual; ++------------------------------------------------------------------------------------------+ +| hex(aes_encrypt('测试', '是不是', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj')) | ++------------------------------------------------------------------------------------------+ +| 9AFB77C281C9 | ++------------------------------------------------------------------------------------------+ +select hex(aes_encrypt('意义', '周三', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj')) from dual; ++---------------------------------------------------------------------------------------+ +| hex(aes_encrypt('意义', '周三', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj')) | ++---------------------------------------------------------------------------------------+ +| D619E761CA78 | ++---------------------------------------------------------------------------------------+ +select hex(aes_encrypt('午后的天气还不错', '莎士比亚', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj')) from dual; ++---------------------------------------------------------------------------------------------------------------+ +| hex(aes_encrypt('午后的天气还不错', '莎士比亚', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj')) | ++---------------------------------------------------------------------------------------------------------------+ +| A9ECD837A6EFDBD963BD8485768F46B6C4C0643F3A50EEA6 | ++---------------------------------------------------------------------------------------------------------------+ +select aes_decrypt(aes_encrypt('中文字符加一点', '你好', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'), '你好', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') from dual; ++-------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| aes_decrypt(aes_encrypt('中文字符加一点', '你好', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'), '你好', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') | ++-------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| 中文字符加一点 | ++-------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +select aes_decrypt(aes_encrypt('测试', '是不是', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'),'是不是','asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') from dual; ++--------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| aes_decrypt(aes_encrypt('测试', '是不是', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'),'是不是','asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') | ++--------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| 测试 | ++--------------------------------------------------------------------------------------------------------------------------------------------------------------+ +select aes_decrypt(aes_encrypt('意义', '周三', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'),'周三','asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') from dual; ++--------------------------------------------------------------------------------------------------------------------------------------------------------+ +| aes_decrypt(aes_encrypt('意义', '周三', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'),'周三','asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') | ++--------------------------------------------------------------------------------------------------------------------------------------------------------+ +| 意义 | ++--------------------------------------------------------------------------------------------------------------------------------------------------------+ +select aes_decrypt(aes_encrypt('午后的天气还不错', '莎士比亚', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'),'莎士比亚','asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') from dual; ++--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| aes_decrypt(aes_encrypt('午后的天气还不错', '莎士比亚', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'),'莎士比亚','asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') | ++--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| 午后的天气还不错 | ++--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +set block_encryption_mode = 17; +show variables like 'block_encryption_mode'; +Variable_name Value +block_encryption_mode aes-256-ofb +select 1 as flag from dual where aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') = aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'); ++------+ +| flag | ++------+ +| 1 | ++------+ +select hex(aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj')) from dual; ++--------------------------------------------------------------------------------------------------+ +| hex(aes_encrypt('asdasdasdasd', '12312313123', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj')) | ++--------------------------------------------------------------------------------------------------+ +| AE03F339448F21267D753840 | ++--------------------------------------------------------------------------------------------------+ +select hex(aes_encrypt('asdas123123dasdasd', '12312313123', '$$&$(*&(%*&(*%&($&*%&(dkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj')) from dual; ++-------------------------------------------------------------------------------------------------------------------------+ +| hex(aes_encrypt('asdas123123dasdasd', '12312313123', '$$&$(*&(%*&(*%&($&*%&(dkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj')) | ++-------------------------------------------------------------------------------------------------------------------------+ +| 4AB0F361AA50F93A6525E8E2052806F06A7D | ++-------------------------------------------------------------------------------------------------------------------------+ +select hex(aes_encrypt('asdasdasdasd', '', '1290380129038129038012938')) from dual; ++-------------------------------------------------------------------+ +| hex(aes_encrypt('asdasdasdasd', '', '1290380129038129038012938')) | ++-------------------------------------------------------------------+ +| 89F0DDEB484970BAFF2B986D | ++-------------------------------------------------------------------+ +select hex(aes_encrypt('', 'abc', '121903810293801289301298301982301928')) from dual; ++---------------------------------------------------------------------+ +| hex(aes_encrypt('', 'abc', '121903810293801289301298301982301928')) | ++---------------------------------------------------------------------+ +| | ++---------------------------------------------------------------------+ +select hex(aes_encrypt('asdasdasdasd', 'asdasdas', '1230121231230381209380913820912830')) from dual; ++------------------------------------------------------------------------------------+ +| hex(aes_encrypt('asdasdasdasd', 'asdasdas', '1230121231230381209380913820912830')) | ++------------------------------------------------------------------------------------+ +| 66C922BF6A279AFEE1A68181 | ++------------------------------------------------------------------------------------+ +select aes_decrypt(aes_encrypt('asdasdasdasd', '', '1290380129038129038012938'), '','1290380129038129038012938') from dual; ++-----------------------------------------------------------------------------------------------------------+ +| aes_decrypt(aes_encrypt('asdasdasdasd', '', '1290380129038129038012938'), '','1290380129038129038012938') | ++-----------------------------------------------------------------------------------------------------------+ +| asdasdasdasd | ++-----------------------------------------------------------------------------------------------------------+ +select aes_decrypt(aes_encrypt('', 'abc', '121903810293801289301298301982301928'), 'abc', '121903810293801289301298301982301928') from dual; ++----------------------------------------------------------------------------------------------------------------------------+ +| aes_decrypt(aes_encrypt('', 'abc', '121903810293801289301298301982301928'), 'abc', '121903810293801289301298301982301928') | ++----------------------------------------------------------------------------------------------------------------------------+ +| | ++----------------------------------------------------------------------------------------------------------------------------+ +select aes_decrypt(aes_encrypt('asdasdasdasd', 'asdasdas', '1230121231230381209380913820912830'),'asdasdas', '1230121231230381209380913820912830') from dual; ++---------------------------------------------------------------------------------------------------------------------------------------------+ +| aes_decrypt(aes_encrypt('asdasdasdasd', 'asdasdas', '1230121231230381209380913820912830'),'asdasdas', '1230121231230381209380913820912830') | ++---------------------------------------------------------------------------------------------------------------------------------------------+ +| asdasdasdasd | ++---------------------------------------------------------------------------------------------------------------------------------------------+ +select hex(aes_encrypt('中文字符加一点', '你好', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj')) from dual; ++------------------------------------------------------------------------------------------------------+ +| hex(aes_encrypt('中文字符加一点', '你好', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj')) | ++------------------------------------------------------------------------------------------------------+ +| 69B7A8070388FFFDADF1AAE36B93E9FB4D3AF7DC20 | ++------------------------------------------------------------------------------------------------------+ +select hex(aes_encrypt('测试', '是不是', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj')) from dual; ++------------------------------------------------------------------------------------------+ +| hex(aes_encrypt('测试', '是不是', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj')) | ++------------------------------------------------------------------------------------------+ +| EBED2E7879EE | ++------------------------------------------------------------------------------------------+ +select hex(aes_encrypt('意义', '周三', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj')) from dual; ++---------------------------------------------------------------------------------------+ +| hex(aes_encrypt('意义', '周三', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj')) | ++---------------------------------------------------------------------------------------+ +| BDE73DC257C5 | ++---------------------------------------------------------------------------------------+ +select hex(aes_encrypt('午后的天气还不错', '莎士比亚', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj')) from dual; ++---------------------------------------------------------------------------------------------------------------+ +| hex(aes_encrypt('午后的天气还不错', '莎士比亚', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj')) | ++---------------------------------------------------------------------------------------------------------------+ +| 23FF28E7C58064F51F757F8C44C049652F70167B765EE07C | ++---------------------------------------------------------------------------------------------------------------+ +select aes_decrypt(aes_encrypt('中文字符加一点', '你好', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'), '你好', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') from dual; ++-------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| aes_decrypt(aes_encrypt('中文字符加一点', '你好', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'), '你好', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') | ++-------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| 中文字符加一点 | ++-------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +select aes_decrypt(aes_encrypt('测试', '是不是', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'),'是不是','asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') from dual; ++--------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| aes_decrypt(aes_encrypt('测试', '是不是', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'),'是不是','asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') | ++--------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| 测试 | ++--------------------------------------------------------------------------------------------------------------------------------------------------------------+ +select aes_decrypt(aes_encrypt('意义', '周三', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'),'周三','asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') from dual; ++--------------------------------------------------------------------------------------------------------------------------------------------------------+ +| aes_decrypt(aes_encrypt('意义', '周三', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'),'周三','asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') | ++--------------------------------------------------------------------------------------------------------------------------------------------------------+ +| 意义 | ++--------------------------------------------------------------------------------------------------------------------------------------------------------+ +select aes_decrypt(aes_encrypt('午后的天气还不错', '莎士比亚', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'),'莎士比亚','asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') from dual; ++--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| aes_decrypt(aes_encrypt('午后的天气还不错', '莎士比亚', 'asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj'),'莎士比亚','asdasdkljasdkljalskdjaklsdjaklsjdaklsdjlaksdj') | ++--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| 午后的天气还不错 | ++--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +set block_encryption_mode = 18; +show variables like 'block_encryption_mode'; +Variable_name Value +block_encryption_mode sm4-ecb +select hex(aes_encrypt('abcd', 'asd')) from dual; +ERROR 0A000: using aes_encrypt with not aes block_encryption_mode not supported + +set block_encryption_mode = 21; +show variables like 'block_encryption_mode'; +Variable_name Value +block_encryption_mode sm4-ofb +select hex(aes_encrypt('abcd', 'asd')) from dual; +ERROR 0A000: using aes_encrypt with not aes block_encryption_mode not supported + +set block_encryption_mode = 0; diff --git a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_and_or.result b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_and_or.result index 3a70858ea..12702dd2d 100644 --- a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_and_or.result +++ b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_and_or.result @@ -885,7 +885,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([1 AND 2 AND 3 AND t1.col_int]), filter(nil), rowset=16 + 0 - output([(T_OP_AND, 1, 2, 3, t1.col_int)]), filter(nil), rowset=16 access([t1.col_int]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -905,7 +905,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([1 AND t1.col_null AND 3 AND t1.col_int]), filter(nil), rowset=16 + 0 - output([(T_OP_AND, 1, t1.col_null, 3, t1.col_int)]), filter(nil), rowset=16 access([t1.col_null], [t1.col_int]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -925,7 +925,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([1 AND t1.col_null AND 3 AND t1.col_zero]), filter(nil), rowset=16 + 0 - output([(T_OP_AND, 1, t1.col_null, 3, t1.col_zero)]), filter(nil), rowset=16 access([t1.col_null], [t1.col_zero]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -945,7 +945,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([1 AND t1.col_null AND 3 AND cast(t1.col_empty_str, DOUBLE(-1, -1))]), filter(nil), rowset=16 + 0 - output([(T_OP_AND, 1, t1.col_null, 3, cast(t1.col_empty_str, DOUBLE(-1, -1)))]), filter(nil), rowset=16 access([t1.col_null], [t1.col_empty_str]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -967,7 +967,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([1 OR 2 OR t1.col_int]), filter(nil), rowset=16 + 0 - output([(T_OP_OR, 1, 2, t1.col_int)]), filter(nil), rowset=16 access([t1.col_int]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -987,7 +987,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([1 OR 2 OR t1.col_null]), filter(nil), rowset=16 + 0 - output([(T_OP_OR, 1, 2, t1.col_null)]), filter(nil), rowset=16 access([t1.col_null]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -1007,7 +1007,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast('', DOUBLE(-1, -1)) OR 0 OR t1.col_null]), filter(nil), rowset=16 + 0 - output([(T_OP_OR, cast('', DOUBLE(-1, -1)), 0, t1.col_null)]), filter(nil), rowset=16 access([t1.col_null]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true diff --git a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_collation.result b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_collation.result index 234655057..13b6f6d9f 100644 --- a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_collation.result +++ b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_collation.result @@ -208,7 +208,7 @@ select _utf8mb4'a' collate gbk_bin; ERROR 42000: COLLATION 'gbk_bin' is not valid for CHARACTER SET 'utf8mb4' // 下面的是等号表达式在类型推导时，进行aggregate collation发现collation不一致报错 select _utf8mb4'a' collate utf8mb4_general_ci = _utf8mb4'A' collate utf8mb4_bin; -ERROR HY000: Illegal mix of collations +ERROR HY000: Illegal mix of collations (utf8mb4_general_ci,EXPLICIT), (utf8mb4_bin,EXPLICIT) select _utf8mb4'a' collate utf8mb4_general_ci = _utf8mb4'A' collate utf8mb4_general_ci; +---------------------------------------------------------------------------------+ | _utf8mb4'a' collate utf8mb4_general_ci = _utf8mb4'A' collate utf8mb4_general_ci | diff --git a/tools/deploy/mysql_test/test_suite/static_engine/t/expr_aes_encrypt.test b/tools/deploy/mysql_test/test_suite/static_engine/t/expr_aes_encrypt.test index 65050421a..abd94622f 100644 --- a/tools/deploy/mysql_test/test_suite/static_engine/t/expr_aes_encrypt.test +++ b/tools/deploy/mysql_test/test_suite/static_engine/t/expr_aes_encrypt.test @@ -1,7 +1,6 @@ # owner: peihan.dph # owner group: sql2 ---disable_abort_on_error --result_format 4 connect (conn_admin, $OBMYSQL_MS0,admin,$OBMYSQL_PWD,oceanbase,$OBMYSQL_PORT); connection conn_admin; @@ -11,55 +10,72 @@ sleep 2; set ob_enable_plan_cache = 0; #basic test set block_encryption_mode = 0; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; --source mysql_test/test_suite/security/include/aes_three_param.inc --source mysql_test/test_suite/security/include/aes_two_param.inc set block_encryption_mode = 1; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; --source mysql_test/test_suite/security/include/aes_three_param.inc --source mysql_test/test_suite/security/include/aes_two_param.inc set block_encryption_mode = 2; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; --source mysql_test/test_suite/security/include/aes_three_param.inc --source mysql_test/test_suite/security/include/aes_two_param.inc set block_encryption_mode = 3; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; --source mysql_test/test_suite/security/include/aes_three_param.inc set block_encryption_mode = 4; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; --source mysql_test/test_suite/security/include/aes_three_param.inc set block_encryption_mode = 5; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; --source mysql_test/test_suite/security/include/aes_three_param.inc set block_encryption_mode = 6; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; --source mysql_test/test_suite/security/include/aes_three_param.inc set block_encryption_mode = 7; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; --source mysql_test/test_suite/security/include/aes_three_param.inc set block_encryption_mode = 8; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; --source mysql_test/test_suite/security/include/aes_three_param.inc set block_encryption_mode = 9; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; --source mysql_test/test_suite/security/include/aes_three_param.inc set block_encryption_mode = 10; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; --source mysql_test/test_suite/security/include/aes_three_param.inc set block_encryption_mode = 11; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; --source mysql_test/test_suite/security/include/aes_three_param.inc set block_encryption_mode = 12; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; --source mysql_test/test_suite/security/include/aes_three_param.inc set block_encryption_mode = 13; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; --source mysql_test/test_suite/security/include/aes_three_param.inc set block_encryption_mode = 14; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; --source mysql_test/test_suite/security/include/aes_three_param.inc set block_encryption_mode = 15; -show variables like '%encrypt%mode'; +show variables like 'block_encryption_mode'; +--source mysql_test/test_suite/security/include/aes_three_param.inc +set block_encryption_mode = 16; +show variables like 'block_encryption_mode'; +--source mysql_test/test_suite/security/include/aes_three_param.inc +set block_encryption_mode = 17; +show variables like 'block_encryption_mode'; --source mysql_test/test_suite/security/include/aes_three_param.inc +set block_encryption_mode = 18; +show variables like 'block_encryption_mode'; +--error 1235 +select hex(aes_encrypt('abcd', 'asd')) from dual; + +set block_encryption_mode = 21; +show variables like 'block_encryption_mode'; +--error 1235 +select hex(aes_encrypt('abcd', 'asd')) from dual; + +set block_encryption_mode = 0; sleep 2; diff --git a/unittest/observer/table/test_create_executor.cpp b/unittest/observer/table/test_create_executor.cpp index 71185c785..53be5ee33 100644 --- a/unittest/observer/table/test_create_executor.cpp +++ b/unittest/observer/table/test_create_executor.cpp @@ -98,7 +98,7 @@ void fill_column_schema(ObColumnSchemaV2 &column, uint64_t id, const char *name, value.set_int(100); column.set_orig_default_value(value); value.set_int(101); - column.set_cur_default_value(value); + column.set_cur_default_value(value, false); column.set_comment("black gives me black eyes"); } diff --git a/unittest/share/schema/test_schema_service.cpp b/unittest/share/schema/test_schema_service.cpp index 469ee41ae..70439ccda 100644 --- a/unittest/share/schema/test_schema_service.cpp +++ b/unittest/share/schema/test_schema_service.cpp @@ -60,7 +60,7 @@ TEST_F(TestSchemaService, ColumnSchema_test) default_value.set_int(100); column_schema.set_orig_default_value(default_value); default_value.set_int(101); - column_schema.set_cur_default_value(default_value); + column_schema.set_cur_default_value(default_value, false); column_schema.set_comment("black gives me black eyes"); char buff[BUFF_SIZE]; @@ -124,7 +124,7 @@ TEST_F(TestSchemaService, TableSchema_test) value.set_int(100); column_schema.set_orig_default_value(value); value.set_int(101); - column_schema.set_cur_default_value(value); + column_schema.set_cur_default_value(value, false); column_schema.set_comment("black gives me black eyes"); ASSERT_EQ(OB_SUCCESS, table_schema.add_column(column_schema)); @@ -190,7 +190,7 @@ TEST_F(TestSchemaService, other_test) obj.set_int(100); column_schema.set_orig_default_value(obj); obj.set_int(101); - column_schema.set_cur_default_value(obj); + column_schema.set_cur_default_value(obj, false); column_schema.set_comment("black gives me black eyes"); ASSERT_EQ(OB_SUCCESS, table_schema.add_column(column_schema)); diff --git a/unittest/share/schema/test_table_schema.cpp b/unittest/share/schema/test_table_schema.cpp index ae443a4c5..d3853dbf1 100644 --- a/unittest/share/schema/test_table_schema.cpp +++ b/unittest/share/schema/test_table_schema.cpp @@ -74,7 +74,7 @@ void fill_column_schema(ObColumnSchemaV2 &column, uint64_t id, const char *name, value.set_int(100); column.set_orig_default_value(value); value.set_int(101); - column.set_cur_default_value(value); + column.set_cur_default_value(value, false); column.set_comment("black gives me black eyes"); } diff --git a/unittest/sql/CMakeLists.txt b/unittest/sql/CMakeLists.txt index 7a249bd1b..47fef647d 100644 --- a/unittest/sql/CMakeLists.txt +++ b/unittest/sql/CMakeLists.txt @@ -24,3 +24,6 @@ add_subdirectory(session) add_subdirectory(module) add_subdirectory(monitor) add_subdirectory(dtl) +if(OB_BUILD_CLOSE_MODULES) + add_subdirectory(audit) +endif() diff --git a/unittest/sql/audit/CMakeLists.txt b/unittest/sql/audit/CMakeLists.txt new file mode 100644 index 000000000..8847f78dc --- /dev/null +++ b/unittest/sql/audit/CMakeLists.txt @@ -0,0 +1 @@ +sql_unittest(test_audit_log_utils) diff --git a/unittest/sql/audit/test_audit_log_utils.cpp b/unittest/sql/audit/test_audit_log_utils.cpp new file mode 100644 index 000000000..1d00d17d3 --- /dev/null +++ b/unittest/sql/audit/test_audit_log_utils.cpp @@ -0,0 +1,112 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL +#include +#define private public +#include "sql/audit/ob_audit_log_utils.h" +#include "sql/audit/ob_audit_logger.h" +#undef private + +namespace oceanbase +{ +namespace sql +{ + +TEST(TestAuditLogUtils, parse_filter_definition) +{ + const char *valid_definitions[] = { + "{}", + "{\"filter\": {\"log\":true}}", + "{\"filter\": {\"log\":false}}", + "{\"filter\": {\"log\":true,\"class\":[]}}", + "{\"filter\": {\"log\":true,\"class\":[{\"name\":\"connection\"}]}}", + "{\"filter\": {\"log\":true,\"class\":[{\"name\":\"connection\"},{\"name\":\"general\"},{\"name\":\"table_access\"}]}}", + "{\"filter\": {\"log\":false,\"class\":[{\"name\":\"connection\"},{\"name\":\"general\"},{\"name\":\"table_access\"}]}}" + }; + const int valid_class[] = {0, 7, 0, 0, 1, 7, 0}; + const char *invalid_definitions[] = { + "1", + "{\"filter\": 1}", + "{\"filter\": aaa}", + "{\"filter\": {\"log\":\"a\"}}", + "{\"filter\": {\"unknown\": true}}", + "{\"filter\": {\"log\":true,\"class\":[1,2]}}", + "{\"filter\": {\"log\":true,\"class\":[{\"name\":\"connection\"},{\"name\":\"unknown\"}]}}" + }; + int valid_num = sizeof(valid_definitions) / sizeof (valid_definitions[0]); + int valid_count_num = sizeof(valid_class) / sizeof (valid_class[0]); + int invalid_num = sizeof(invalid_definitions) / sizeof (invalid_definitions[0]); + bool is_valid = false; + uint64_t audit_class = 0; + ASSERT_EQ(valid_num, valid_count_num); + for (int i = 0; i < valid_num; ++i) { + is_valid = true; + audit_class = 0; + EXPECT_EQ(OB_SUCCESS, sql::ObAuditLogUtils::parse_filter_definition(valid_definitions[i], is_valid, audit_class)); + EXPECT_TRUE(is_valid); + EXPECT_EQ(valid_class[i], audit_class); + LOG_INFO("parse result", K(i), K(valid_definitions[i]), K(is_valid), K(audit_class)); + } + + for (int i = 0; i < invalid_num; ++i) { + is_valid = true; + audit_class = 0; + EXPECT_EQ(OB_SUCCESS, sql::ObAuditLogUtils::parse_filter_definition(invalid_definitions[i], is_valid, audit_class)); + EXPECT_FALSE(is_valid); + EXPECT_EQ(0, audit_class); + LOG_INFO("parse result", K(i), K(invalid_definitions[i]), K(is_valid), K(audit_class)); + } +} + +TEST(TestAuditLogUtils, compress_and_upload_log) +{ + sql::ObAuditLogger logger; + logger.log_cfg_.audit_log_path_ = "file://audit/1234/567"; + const char *dir_path = "1004/ip:port"; + system("rm -rf audit"); + databuff_printf(logger.log_file_.dir_path_, sizeof(logger.log_file_.dir_path_), "%s", dir_path); + EXPECT_EQ(OB_SUCCESS, logger.compress_and_upload_log("./test_audit_log_utils", "123", true)); + logger.log_cfg_.audit_log_compression_ = ObCompressorType::ZSTD_COMPRESSOR; + EXPECT_EQ(OB_SUCCESS, logger.compress_and_upload_log("./test_audit_log_utils", "456", true)); +} + +TEST(TestAuditLogUtils, prune_log) +{ + sql::ObAuditLogger logger; + system("rm -rf audit"); + system("fallocate -l 10K ./audit_file.test"); + logger.log_cfg_.audit_log_path_ = "file://audit/1234/567"; + const char *dir_path = "1004/ip:port"; + int64_t expire_time = 1711947601000000; + databuff_printf(logger.log_file_.dir_path_, sizeof(logger.log_file_.dir_path_), "%s", dir_path); + EXPECT_EQ(OB_SUCCESS, logger.compress_and_upload_log("./audit_file.test", "20240401120000000", true)); + EXPECT_EQ(OB_SUCCESS, logger.compress_and_upload_log("./audit_file.test", "20240401130000000", true)); + EXPECT_EQ(OB_SUCCESS, logger.compress_and_upload_log("./audit_file.test", "20240401140000000", true)); + EXPECT_EQ(OB_SUCCESS, logger.compress_and_upload_log("./audit_file.test", "20240401150000000", true)); + EXPECT_EQ(OB_SUCCESS, logger.compress_and_upload_log("./audit_file.test", "20240401160000000", true)); + EXPECT_EQ(OB_SUCCESS, logger.compress_and_upload_log("./audit_file.test", "20240401170000000", true)); + logger.log_cfg_.audit_log_max_size_ = 3 * 10 * 1024 * 1024 - 1; + EXPECT_EQ(OB_SUCCESS, logger.prune_log(expire_time)); +} + +} // end namespace sql +} // end namespace oceanbase + +int main(int argc, char **argv) +{ + system("rm -rf test_audit_log_utils.log*"); + oceanbase::common::ObLogger::get_logger().set_file_name("test_audit_log_utils.log", true); + oceanbase::common::ObLogger::get_logger().set_log_level("INFO"); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/unittest/sql/parser/print_parser_tree.result b/unittest/sql/parser/print_parser_tree.result index 65a69f1b5..320ee7c2b 100644 --- a/unittest/sql/parser/print_parser_tree.result +++ b/unittest/sql/parser/print_parser_tree.result @@ -325,7 +325,7 @@ question_mask_size: 0 |--[0],[T_IDENT], str_value_=[d], value=[9223372036854775807] |--[1],[T_IDENT], str_value_=[t1], value=[9223372036854775807] |--[5],[T_WHERE_CLAUSE], str_value_=[], value=[9223372036854775807] - |--[0],[T_OP_AND], str_value_=[], value=[9223372036854775807] + |--[0],[T_OP_AND], str_value_=[], value=[2] |--[0],[T_OP_GT], str_value_=[], value=[9223372036854775807] |--[0],[T_COLUMN_REF], str_value_=[c1], value=[9223372036854775807] |--[0],[T_IDENT], str_value_=[d], value=[9223372036854775807] @@ -382,7 +382,7 @@ question_mask_size: 0 |--[0],[T_RELATION_FACTOR], str_value_=[t1], value=[9223372036854775807] |--[1],[T_IDENT], str_value_=[t1], value=[9223372036854775807] |--[5],[T_WHERE_CLAUSE], str_value_=[], value=[9223372036854775807] - |--[0],[T_OP_AND], str_value_=[], value=[9223372036854775807] + |--[0],[T_OP_AND], str_value_=[], value=[2] |--[0],[T_OP_GT], str_value_=[], value=[9223372036854775807] |--[0],[T_COLUMN_REF], str_value_=[c1], value=[9223372036854775807] |--[1],[T_IDENT], str_value_=[t1], value=[9223372036854775807] @@ -434,7 +434,7 @@ question_mask_size: 0 |--[0],[T_RELATION_FACTOR], str_value_=[t1], value=[9223372036854775807] |--[1],[T_IDENT], str_value_=[t1], value=[9223372036854775807] |--[5],[T_WHERE_CLAUSE], str_value_=[], value=[9223372036854775807] - |--[0],[T_OP_AND], str_value_=[], value=[9223372036854775807] + |--[0],[T_OP_AND], str_value_=[], value=[2] |--[0],[T_OP_GT], str_value_=[], value=[9223372036854775807] |--[0],[T_COLUMN_REF], str_value_=[c1], value=[9223372036854775807] |--[2],[T_IDENT], str_value_=[c1], value=[9223372036854775807] @@ -2586,7 +2586,7 @@ question_mask_size: 0 |--[1],[T_IDENT], str_value_=[t1], value=[9223372036854775807] |--[5],[T_WHERE_CLAUSE], str_value_=[], value=[9223372036854775807] |--[0],[T_OP_IN], str_value_=[], value=[9223372036854775807] - |--[0],[T_EXPR_LIST], str_value_=[], value=[9223372036854775807] + |--[0],[T_EXPR_LIST], str_value_=[], value=[2] |--[0],[T_COLUMN_REF], str_value_=[c1], value=[9223372036854775807] |--[2],[T_IDENT], str_value_=[c1], value=[9223372036854775807] |--[1],[T_COLUMN_REF], str_value_=[c2], value=[9223372036854775807] @@ -2619,7 +2619,7 @@ question_mask_size: 0 |--[1],[T_IDENT], str_value_=[t1], value=[9223372036854775807] |--[5],[T_WHERE_CLAUSE], str_value_=[], value=[9223372036854775807] |--[0],[T_OP_IN], str_value_=[], value=[9223372036854775807] - |--[0],[T_EXPR_LIST], str_value_=[], value=[9223372036854775807] + |--[0],[T_EXPR_LIST], str_value_=[], value=[2] |--[0],[T_COLUMN_REF], str_value_=[c1], value=[9223372036854775807] |--[2],[T_IDENT], str_value_=[c1], value=[9223372036854775807] |--[1],[T_COLUMN_REF], str_value_=[c2], value=[9223372036854775807] diff --git a/unittest/sql/parser/test_parser.result b/unittest/sql/parser/test_parser.result index 569acc869..6a040c949 100644 --- a/unittest/sql/parser/test_parser.result +++ b/unittest/sql/parser/test_parser.result @@ -1498,7 +1498,7 @@ question_mask_size: 0 "children": [ { "type":"T_OP_AND", - "int_val":9223372036854775807, + "int_val":2, "str_len":0, "str_val":"", "children": [ @@ -1902,7 +1902,7 @@ question_mask_size: 0 "children": [ { "type":"T_OP_AND", - "int_val":9223372036854775807, + "int_val":2, "str_len":0, "str_val":"", "children": [ @@ -2281,7 +2281,7 @@ question_mask_size: 0 "children": [ { "type":"T_OP_AND", - "int_val":9223372036854775807, + "int_val":2, "str_len":0, "str_val":"", "children": [ @@ -17051,7 +17051,7 @@ question_mask_size: 0 "children": [ { "type":"T_EXPR_LIST", - "int_val":9223372036854775807, + "int_val":2, "str_len":0, "str_val":"", "children": [ @@ -17324,7 +17324,7 @@ question_mask_size: 0 "children": [ { "type":"T_EXPR_LIST", - "int_val":9223372036854775807, + "int_val":2, "str_len":0, "str_val":"", "children": [ diff --git a/unittest/sql/resolver/test_resolver.cpp b/unittest/sql/resolver/test_resolver.cpp index 69f4eab6b..f2fb98213 100644 --- a/unittest/sql/resolver/test_resolver.cpp +++ b/unittest/sql/resolver/test_resolver.cpp @@ -92,9 +92,11 @@ bool TestResolver::is_show_sql(const ParseNode &node) const case T_SHOW_PROCESSLIST: case T_SHOW_SERVER_STATUS: case T_SHOW_WARNINGS: + case T_SHOW_ERRORS: case T_SHOW_RESTORE_PREVIEW: case T_SHOW_SEQUENCES: - case T_SHOW_GRANTS:{ + case T_SHOW_GRANTS: + case T_SHOW_CREATE_USER:{ ret = true; break; } diff --git a/unittest/sql/rewrite/test_query_range.cpp b/unittest/sql/rewrite/test_query_range.cpp index cc47b19be..44378797b 100644 --- a/unittest/sql/rewrite/test_query_range.cpp +++ b/unittest/sql/rewrite/test_query_range.cpp @@ -224,7 +224,7 @@ public: int64_t pos = 0; int64_t data_len = 0; resolve_condition(range_columns, condition, expr); - OK(enc_query_range.preliminary_extract_query_range(range_columns, expr, NULL, &exec_ctx_)); + OK(enc_query_range.preliminary_extract_query_range(range_columns, expr, NULL, &exec_ctx_, &query_ctx_)); OK(enc_query_range.serialize(buf, sizeof(buf), pos)); data_len = pos; pos = 0; @@ -261,7 +261,7 @@ public: split_and_condition(expr, and_exprs); // OK(multi_query_range.preliminary_extract_query_range(range_columns, and_exprs, NULL)); // OK(multi_query_range.final_extract_query_range(params, NULL)); - OK(multi_query_range.preliminary_extract_query_range(range_columns, and_exprs, dtc_params, &exec_ctx_)); + OK(multi_query_range.preliminary_extract_query_range(range_columns, and_exprs, dtc_params, &exec_ctx_, &query_ctx_)); OK(multi_query_range.final_extract_query_range(exec_ctx_, dtc_params)); OK(multi_query_range.get_tablet_ranges(ranges, all_single_value_ranges, dtc_params)); _OB_LOG(DEBUG, "and_exprs_count: %ld, ranges: %s, except_range: %s", @@ -293,6 +293,7 @@ protected: ObArray extra_range_columns_; ObColumnRefRawExpr ref_col_; //a ObQueryRange query_range; + ObQueryCtx query_ctx_; void get_query_range(const char *expr, const char *&json_expr); void get_query_range_filter(const char *sql_expr, const char *&json_expr, char *buf, int64_t &pos, const int64_t cols_num); void get_query_range_collation(const char *expr, const char *&json_expr, char *buf, int64_t &pos); @@ -516,7 +517,7 @@ void ObQueryRangeTest::get_query_range(const char *sql_expr, const char *&json_e _OB_LOG(INFO, "expr: %s", CSJ(expr)); query_range.reset(); OB_LOG(INFO, "get query range sql", K(final_sql)); - OK(pre_query_range.preliminary_extract_query_range(range_columns, expr, dtc_params, &exec_ctx_)); + OK(pre_query_range.preliminary_extract_query_range(range_columns, expr, dtc_params, &exec_ctx_, &query_ctx_)); char *ser_buf = NULL; int64_t ser_len = pre_query_range.get_serialize_size(); int64_t pos = 0; @@ -583,7 +584,7 @@ void ObQueryRangeTest::get_query_range_filter(const char *sql_expr, const char * OB_ASSERT(expr); query_range.reset(); OB_LOG(INFO, "get query range sql", K(final_sql)); - OK(query_range.preliminary_extract_query_range(range_columns, exprs, dtc_params, &exec_ctx_)); + OK(query_range.preliminary_extract_query_range(range_columns, exprs, dtc_params, &exec_ctx_, &query_ctx_)); databuff_printf(buf, BUF_LEN, pos, "\n**rowkey num = %ld** \n", cols_num); databuff_printf(buf, BUF_LEN, pos, "**filter count = %ld**\n", query_range.get_range_exprs().count()); @@ -639,7 +640,7 @@ void ObQueryRangeTest::get_query_range_collation(const char *sql_expr, const cha // each result_type of expr is deduced by param_obj_type and column_type databuff_printf(buf, BUF_LEN, pos, "%s--------------connection_collation = %d col_type = %d\n",sql_expr, conn_type, col_type); OB_ASSERT(expr); - OK(query_range.preliminary_extract_query_range(range_columns, expr, dtc_params, &exec_ctx_)); + OK(query_range.preliminary_extract_query_range(range_columns, expr, dtc_params, &exec_ctx_, &query_ctx_)); OK(query_range.final_extract_query_range(exec_ctx_, NULL)); OK(query_range.get_tablet_ranges(ranges, all_single_value_ranges, dtc_params)); bool flag = is_min_to_max_range(ranges); @@ -820,7 +821,7 @@ TEST_F(ObQueryRangeTest, single_filed_key_whole_range1) ParamStore ¶ms = exec_ctx_.get_physical_plan_ctx()->get_param_store_for_update(); params.reset(); - OK(query_range.preliminary_extract_query_range(single_range_columns_, NULL, NULL, &exec_ctx_)); + OK(query_range.preliminary_extract_query_range(single_range_columns_, NULL, NULL, &exec_ctx_, &query_ctx_)); OK(query_range.final_extract_query_range(exec_ctx_, NULL)); OK(query_range.get_tablet_ranges(ranges, all_single_value_ranges, dtc_params)); ASSERT_EQ(0, strcmp(to_cstring(ranges), "[{\"range\":\"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)\"}]")); @@ -837,7 +838,7 @@ TEST_F(ObQueryRangeTest, single_filed_key_whole_range2) ParamStore ¶ms = exec_ctx_.get_physical_plan_ctx()->get_param_store_for_update(); params.reset(); - OK(query_range.preliminary_extract_query_range(single_range_columns_, exprs, dtc_params, &exec_ctx_)); + OK(query_range.preliminary_extract_query_range(single_range_columns_, exprs, dtc_params, &exec_ctx_, &query_ctx_)); OK(query_range.final_extract_query_range(exec_ctx_, dtc_params)); OK(query_range.get_tablet_ranges(ranges, all_single_value_ranges, dtc_params)); ASSERT_EQ(0, strcmp(to_cstring(ranges), "[{\"range\":\"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)\"}]")); @@ -853,7 +854,7 @@ TEST_F(ObQueryRangeTest, double_filed_key_whole_range1) ParamStore ¶ms = exec_ctx_.get_physical_plan_ctx()->get_param_store_for_update(); params.reset(); - OK(query_range.preliminary_extract_query_range(double_range_columns_, NULL, dtc_params, &exec_ctx_)); + OK(query_range.preliminary_extract_query_range(double_range_columns_, NULL, dtc_params, &exec_ctx_, &query_ctx_)); OK(query_range.final_extract_query_range(exec_ctx_, dtc_params)); OK(query_range.get_tablet_ranges(ranges, all_single_value_ranges, dtc_params)); ASSERT_EQ(0, strcmp(to_cstring(ranges), "[{\"range\":\"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)\"}]")); @@ -870,7 +871,7 @@ TEST_F(ObQueryRangeTest, double_filed_key_whole_range2) ParamStore ¶ms = exec_ctx_.get_physical_plan_ctx()->get_param_store_for_update(); params.reset(); - OK(query_range.preliminary_extract_query_range(double_range_columns_, exprs, dtc_params, &exec_ctx_)); + OK(query_range.preliminary_extract_query_range(double_range_columns_, exprs, dtc_params, &exec_ctx_, &query_ctx_)); OK(query_range.final_extract_query_range(exec_ctx_, dtc_params)); OK(query_range.get_tablet_ranges(ranges, all_single_value_ranges, dtc_params)); ASSERT_EQ(0, strcmp(to_cstring(ranges), "[{\"range\":\"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)\"}]")); @@ -889,9 +890,9 @@ TEST_F(ObQueryRangeTest, range_column_with_like) ParamStore ¶ms = exec_ctx_.get_physical_plan_ctx()->get_param_store_for_update(); params.reset(); - ASSERT_EQ(OB_INVALID_ARGUMENT, query_range.preliminary_extract_query_range(single_range_columns, expr, dtc_params, &exec_ctx_)); + ASSERT_EQ(OB_INVALID_ARGUMENT, query_range.preliminary_extract_query_range(single_range_columns, expr, dtc_params, &exec_ctx_, &query_ctx_)); resolve_expr("f like 'abc%'", expr, single_range_columns, params, CS_TYPE_UTF8MB4_GENERAL_CI, CS_TYPE_UTF8MB4_GENERAL_CI); - OK(query_range.preliminary_extract_query_range(single_range_columns, expr, dtc_params, &exec_ctx_)); + OK(query_range.preliminary_extract_query_range(single_range_columns, expr, dtc_params, &exec_ctx_, &query_ctx_)); OK(query_range.final_extract_query_range(exec_ctx_, dtc_params)); OK(query_range.get_tablet_ranges(ranges, all_single_value_ranges, dtc_params)); _OB_LOG(INFO, "range: %s", to_cstring(ranges)); @@ -901,14 +902,14 @@ TEST_F(ObQueryRangeTest, range_column_with_like) ObObj escape_obj; escape_obj.set_null(); escape_expr->set_value(escape_obj); - OK(query_range.preliminary_extract_query_range(single_range_columns, expr, dtc_params, &exec_ctx_)); + OK(query_range.preliminary_extract_query_range(single_range_columns, expr, dtc_params, &exec_ctx_, &query_ctx_)); OK(query_range.final_extract_query_range(exec_ctx_, dtc_params)); OK(query_range.get_tablet_ranges(ranges, all_single_value_ranges, dtc_params)); _OB_LOG(INFO, "range: %s", to_cstring(ranges)); query_range.reset(); resolve_condition(single_range_columns, "'a' like 'a'", expr); - OK(query_range.preliminary_extract_query_range(single_range_columns, expr, dtc_params, &exec_ctx_)); + OK(query_range.preliminary_extract_query_range(single_range_columns, expr, dtc_params, &exec_ctx_, &query_ctx_)); OK(query_range.final_extract_query_range(exec_ctx_, dtc_params)); OK(query_range.get_tablet_ranges(ranges, all_single_value_ranges, dtc_params)); _OB_LOG(INFO, "range: %s", to_cstring(ranges)); @@ -973,7 +974,7 @@ TEST_F(ObQueryRangeTest, range_column_with_triple_key) const ObDataTypeCastParams dtc_params; resolve_condition(triple_range_columns_, sql_str, expr, ¶ms); int64_t time1 = ObTimeUtility::current_time(); - OK(query_range.preliminary_extract_query_range(triple_range_columns_, expr, dtc_params, &exec_ctx_)); + OK(query_range.preliminary_extract_query_range(triple_range_columns_, expr, dtc_params, &exec_ctx_, &query_ctx_)); int64_t time2 = ObTimeUtility::current_time(); OK(query_range.final_extract_query_range(exec_ctx_, dtc_params)); int64_t time3 = ObTimeUtility::current_time(); @@ -1089,7 +1090,7 @@ TEST_F(ObQueryRangeTest, single_key_cost_time) const ObDataTypeCastParams dtc_params; ObQueryRange pre_query_range; - OK(pre_query_range.preliminary_extract_query_range(single_range_columns_, expr, dtc_params, &exec_ctx_)); + OK(pre_query_range.preliminary_extract_query_range(single_range_columns_, expr, dtc_params, &exec_ctx_, &query_ctx_)); int64_t deep_copy_cost = 0; int64_t extract_cost = 0; @@ -1113,7 +1114,7 @@ TEST_F(ObQueryRangeTest, single_key_cost_time) (float)deep_copy_cost / (float)1000, (float)extract_cost / (float)1000, (float)get_range_cost / (float)1000); ObQueryRange query_range2; - OK(query_range2.preliminary_extract_query_range(single_range_columns_, expr, dtc_params, &exec_ctx_)); + OK(query_range2.preliminary_extract_query_range(single_range_columns_, expr, dtc_params, &exec_ctx_, &query_ctx_)); get_range_cost = 0; for (int64_t i = 0; i < 1000; ++i) { ObQueryRangeArray ranges; From c7dcbee52b36896f5e4c78a83dfc5db5479e2a37 Mon Sep 17 00:00:00 2001 From: haohao022 Date: Fri, 23 Aug 2024 03:39:14 +0000 Subject: [PATCH 182/249] [CP] [to #2024080800104069136] fix: compile udf report parse error after upgrade observer from 410 to 425 --- src/pl/parser/pl_parser_mysql_mode.y | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/pl/parser/pl_parser_mysql_mode.y b/src/pl/parser/pl_parser_mysql_mode.y index 5ccd95be9..817384d0e 100644 --- a/src/pl/parser/pl_parser_mysql_mode.y +++ b/src/pl/parser/pl_parser_mysql_mode.y @@ -248,7 +248,7 @@ void obpl_mysql_wrap_get_user_var_into_subquery(ObParseCtx *parse_ctx, ParseNode %type sp_decl_idents sp_data_type opt_sp_decl_default opt_param_default %type sp_proc_stmt_if sp_if sp_proc_stmt_case sp_when_list sp_when sp_elseifs %type sp_proc_stmt_return -%type sql_stmt ident simple_ident +%type sql_stmt_prefix sql_stmt ident simple_ident %type into_clause %type sp_name sp_call_name opt_sp_param_list opt_sp_fparam_list sp_param_list sp_fparam_list %type sp_param sp_fparam sp_alter_chistics @@ -384,17 +384,21 @@ outer_stmt: * *****************************************************************************/ sql_keyword: - '(' sql_keyword { $$ = NULL; } - | SQL_KEYWORD { $$ = NULL; } + SQL_KEYWORD { $$ = NULL; } | TABLE { $$ = NULL; } | USER { $$ = NULL; } +; + +sql_stmt_prefix: + '(' sql_stmt_prefix { $$ = NULL; } + | SQL_KEYWORD { $$ = NULL; } | INSERT { $$ = NULL; } | DELETE { $$ = NULL; } | UPDATE { $$ = NULL; } ; sql_stmt: - sql_keyword /*sql stmt tail*/ + sql_stmt_prefix /*sql stmt tail*/ { //read sql query string直到读到token';'或者END_P ParseNode *sql_stmt = NULL; @@ -579,6 +583,14 @@ call_sp_stmt: { malloc_non_terminal_node($$, parse_ctx->mem_pool_, T_SP_CALL_STMT, 2, $2, $3); } + | CALL sp_proc_stmt + { + if (!parse_ctx->is_inner_parse_) { + obpl_mysql_yyerror(&@2, parse_ctx, "Syntax Error\n"); + YYERROR; //生成一个语法错误 + } + $$ = $2; + } | CALL PROCEDURE opt_if_not_exists sp_name '(' opt_sp_param_list ')' sp_create_chistics procedure_body { if (!parse_ctx->is_inner_parse_) { From 66063e6dce0e5d931073fa4f9276741fcd92c2a1 Mon Sep 17 00:00:00 2001 From: obdev Date: Fri, 23 Aug 2024 03:45:10 +0000 Subject: [PATCH 183/249] fix: int ObSharedNothingTmpFile::remove_useless_page_in_data_flush_infos_ --- src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp b/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp index fa83f64cd..e3a840f12 100644 --- a/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp +++ b/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp @@ -2042,7 +2042,7 @@ int ObSharedNothingTmpFile::remove_useless_page_in_data_flush_infos_(const int64 } else { // skip truncated flush infos for (int64_t i = start_pos; OB_SUCC(ret) && i < end_pos; i++) { - int64_t flushed_start_offset = get_page_begin_offset_by_virtual_id_(flush_infos_[start_pos].flush_virtual_page_id_); + int64_t flushed_start_offset = get_page_begin_offset_by_virtual_id_(flush_infos_[i].flush_virtual_page_id_); int64_t flushed_end_offset = flushed_start_offset + flush_infos_[i].flush_data_page_num_ * ObTmpFileGlobal::PAGE_SIZE; if (truncated_offset_ <= flushed_start_offset) { From ca63e309a3fd564586241f6c077dfc08e2f80f12 Mon Sep 17 00:00:00 2001 From: obdev Date: Fri, 23 Aug 2024 04:03:08 +0000 Subject: [PATCH 184/249] bugfix for tmp file flush --- .../tmp_file/ob_shared_nothing_tmp_file.cpp | 126 +++++++++++++----- .../tmp_file/ob_shared_nothing_tmp_file.h | 21 ++- .../tmp_file/ob_tmp_file_flush_ctx.cpp | 27 ++-- src/storage/tmp_file/ob_tmp_file_flush_ctx.h | 18 ++- .../tmp_file/ob_tmp_file_flush_manager.cpp | 45 +++++-- 5 files changed, 170 insertions(+), 67 deletions(-) diff --git a/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp b/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp index e3a840f12..6dbc8fea8 100644 --- a/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp +++ b/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp @@ -90,6 +90,8 @@ void ObSharedNothingTmpFile::InnerFlushContext::reset() meta_flush_infos_.reset(); data_flush_infos_.reset(); flush_seq_ = ObTmpFileGlobal::INVALID_FLUSH_SEQUENCE; + need_to_wait_for_the_previous_data_flush_req_to_complete_ = false; + need_to_wait_for_the_previous_meta_flush_req_to_complete_ = false; } int ObSharedNothingTmpFile::InnerFlushContext::update_finished_continuous_flush_info_num(const bool is_meta, const int64_t end_pos) @@ -1991,6 +1993,11 @@ int ObSharedNothingTmpFile::update_meta_after_flush(const int64_t info_idx, cons LOG_WARN("fail to update file meta", KR(ret), K(start_pos), K(end_pos), K(flushed_data_page_num), KPC(this)); } else { reset_ctx = true; + if (is_meta) { + inner_flush_ctx_.need_to_wait_for_the_previous_meta_flush_req_to_complete_ = true; + } else { + inner_flush_ctx_.need_to_wait_for_the_previous_data_flush_req_to_complete_ = true; + } } LOG_DEBUG("update meta based a set of flush info over", KR(ret), K(info_idx), K(start_pos), K(end_pos), K(inner_flush_ctx_), KPC(this)); @@ -2004,11 +2011,13 @@ int ObSharedNothingTmpFile::update_meta_after_flush(const int64_t info_idx, cons if (OB_ISNULL(data_flush_node_.get_next()) && OB_TMP_FAIL(reinsert_flush_node_(false /*is_meta*/))) { LOG_ERROR("fail to reinsert data flush node", KR(tmp_ret), K(is_meta), K(inner_flush_ctx_), KPC(this)); } + inner_flush_ctx_.need_to_wait_for_the_previous_data_flush_req_to_complete_ = false; } if (inner_flush_ctx_.is_meta_finished()) { if (OB_ISNULL(meta_flush_node_.get_next()) && OB_TMP_FAIL(reinsert_flush_node_(true /*is_meta*/))) { LOG_ERROR("fail to reinsert meta flush node", KR(tmp_ret), K(is_meta), K(inner_flush_ctx_), KPC(this)); } + inner_flush_ctx_.need_to_wait_for_the_previous_meta_flush_req_to_complete_ = false; } if (inner_flush_ctx_.is_all_finished()) { @@ -2255,6 +2264,54 @@ int ObSharedNothingTmpFile::update_meta_tree_after_flush_(const int64_t start_po return ret; } +int ObSharedNothingTmpFile::generate_data_flush_info_( + ObTmpFileFlushTask &flush_task, + ObTmpFileFlushInfo &info, + ObTmpFileDataFlushContext &data_flush_context, + const int64_t flush_sequence, + const bool need_flush_tail) +{ + int ret = OB_SUCCESS; + + uint32_t copy_begin_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + int64_t copy_begin_page_virtual_id = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; + + uint32_t copy_end_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; + if (OB_FAIL(cal_next_flush_page_id_from_flush_ctx_or_file_(data_flush_context, + copy_begin_page_id, + copy_begin_page_virtual_id))) { + LOG_WARN("fail to calculate next_flush_page_id", KR(ret), + K(flush_task), K(info), K(data_flush_context), KPC(this)); + } else if (ObTmpFileGlobal::INVALID_PAGE_ID == copy_begin_page_id) { + ret = OB_ITER_END; + LOG_DEBUG("no more data to flush", KR(ret), K(fd_), KPC(this)); + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == copy_begin_page_virtual_id)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("next_flush_page_virtual_id is invalid", KR(ret), K(fd_), K(copy_begin_page_id), + K(copy_begin_page_virtual_id), K(flush_task), K(info), K(data_flush_context), + K(flush_sequence), K(need_flush_tail), KPC(this)); + } else if (OB_FAIL(get_flush_end_page_id_(copy_end_page_id, need_flush_tail))) { + LOG_WARN("fail to get_flush_end_page_id_", KR(ret), + K(flush_task), K(info), K(data_flush_context), KPC(this)); + } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_FLUSH_SEQUENCE != inner_flush_ctx_.flush_seq_ + && flush_sequence != inner_flush_ctx_.flush_seq_ + && flush_sequence != flush_task.get_flush_seq())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("flush sequence not match", + KR(ret), K(inner_flush_ctx_.flush_seq_), K(flush_task), KPC(this)); + } else if (OB_FAIL(copy_flush_data_from_wbp_(flush_task, info, data_flush_context, + copy_begin_page_id, copy_begin_page_virtual_id, + copy_end_page_id, + flush_sequence, need_flush_tail))) { + LOG_WARN("fail to copy flush data from wbp", KR(ret), + K(flush_task), K(info), K(data_flush_context), KPC(this)); + } + + LOG_DEBUG("generate_data_flush_info_ end", + KR(ret), K(fd_), K(data_flush_context), K(info), K(flush_task), KPC(this)); + return ret; +} + int ObSharedNothingTmpFile::generate_data_flush_info( ObTmpFileFlushTask &flush_task, ObTmpFileFlushInfo &info, @@ -2270,45 +2327,19 @@ int ObSharedNothingTmpFile::generate_data_flush_info( LOG_WARN("fail to get truncate lock", KR(ret), K(fd_), KPC(this)); } else { common::TCRWLock::RLockGuard guard(meta_lock_); - uint32_t copy_begin_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; - int64_t copy_begin_page_virtual_id = ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID; - - uint32_t copy_end_page_id = ObTmpFileGlobal::INVALID_PAGE_ID; - if (OB_FAIL(cal_next_flush_page_id_from_flush_ctx_or_file_(data_flush_context, - copy_begin_page_id, - copy_begin_page_virtual_id))) { - LOG_WARN("fail to calculate next_flush_page_id", KR(ret), - K(flush_task), K(info), K(data_flush_context), KPC(this)); - } else if (ObTmpFileGlobal::INVALID_PAGE_ID == copy_begin_page_id) { + if (inner_flush_ctx_.need_to_wait_for_the_previous_data_flush_req_to_complete_) { ret = OB_ITER_END; - LOG_DEBUG("no more data to flush", KR(ret), K(fd_), KPC(this)); - } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_VIRTUAL_PAGE_ID == copy_begin_page_virtual_id)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("next_flush_page_virtual_id is invalid", KR(ret), K(fd_), K(copy_begin_page_id), - K(copy_begin_page_virtual_id), K(flush_task), K(info), K(data_flush_context), - K(flush_sequence), K(need_flush_tail), KPC(this)); - } else if (OB_FAIL(get_flush_end_page_id_(copy_end_page_id, need_flush_tail))) { - LOG_WARN("fail to get_flush_end_page_id_", KR(ret), - K(flush_task), K(info), K(data_flush_context), KPC(this)); - } else if (OB_UNLIKELY(ObTmpFileGlobal::INVALID_FLUSH_SEQUENCE != inner_flush_ctx_.flush_seq_ - && flush_sequence != inner_flush_ctx_.flush_seq_ - && flush_sequence != flush_task.get_flush_seq())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("flush sequence not match", - KR(ret), K(inner_flush_ctx_.flush_seq_), K(flush_task), KPC(this)); - } else if (OB_FAIL(copy_flush_data_from_wbp_(flush_task, info, data_flush_context, - copy_begin_page_id, copy_begin_page_virtual_id, - copy_end_page_id, - flush_sequence, need_flush_tail))) { - LOG_WARN("fail to copy flush data from wbp", KR(ret), - K(flush_task), K(info), K(data_flush_context), KPC(this)); + LOG_INFO("need_to_wait_for_the_previous_data_flush_req_to_complete_", KR(ret), K(fd_), KPC(this)); + } else if (OB_FAIL(generate_data_flush_info_(flush_task, info, + data_flush_context, flush_sequence, need_flush_tail))) { + STORAGE_LOG(WARN, "fail to generate_data_flush_info_", KR(ret), K(flush_task), + K(info), K(data_flush_context), K(flush_sequence), K(need_flush_tail), KPC(this)); } if (OB_FAIL(ret)) { truncate_lock_.unlock(); } - LOG_DEBUG("generate flush data info end", KR(ret), K(fd_), - K(data_flush_context), K(info), K(flush_task), KPC(this)); } + return ret; } @@ -2435,7 +2466,7 @@ int ObSharedNothingTmpFile::copy_flush_data_from_wbp_( return ret; } -int ObSharedNothingTmpFile::generate_meta_flush_info( +int ObSharedNothingTmpFile::generate_meta_flush_info_( ObTmpFileFlushTask &flush_task, ObTmpFileFlushInfo &info, ObTmpFileTreeFlushContext &meta_flush_context, @@ -2443,9 +2474,7 @@ int ObSharedNothingTmpFile::generate_meta_flush_info( const bool need_flush_tail) { int ret = OB_SUCCESS; - info.reset(); - common::TCRWLock::RLockGuard guard(meta_lock_); ObArray &flush_infos_ = inner_flush_ctx_.meta_flush_infos_; int64_t origin_info_num = flush_infos_.size(); @@ -2497,11 +2526,34 @@ int ObSharedNothingTmpFile::generate_meta_flush_info( } } - LOG_INFO("generate flush meta info end", KR(ret), K(fd_), K(need_flush_tail), + LOG_INFO("generate_meta_flush_info_ end", KR(ret), K(fd_), K(need_flush_tail), K(inner_flush_ctx_), K(meta_flush_context), K(info), K(flush_task), KPC(this)); return ret; } +int ObSharedNothingTmpFile::generate_meta_flush_info( + ObTmpFileFlushTask &flush_task, + ObTmpFileFlushInfo &info, + ObTmpFileTreeFlushContext &meta_flush_context, + const int64_t flush_sequence, + const bool need_flush_tail) +{ + int ret = OB_SUCCESS; + info.reset(); + + common::TCRWLock::RLockGuard guard(meta_lock_); + if (inner_flush_ctx_.need_to_wait_for_the_previous_meta_flush_req_to_complete_) { + ret = OB_ITER_END; + LOG_INFO("need_to_wait_for_the_previous_meta_flush_req_to_complete_", KR(ret), K(fd_), KPC(this)); + } else if (OB_FAIL(generate_meta_flush_info_(flush_task, info, + meta_flush_context, flush_sequence, need_flush_tail))) { + STORAGE_LOG(WARN, "fail to generate_meta_flush_info_", KR(ret), K(flush_task), + K(info), K(meta_flush_context), K(flush_sequence), K(need_flush_tail), KPC(this)); + } + + return ret; +} + int ObSharedNothingTmpFile::insert_meta_tree_item(const ObTmpFileFlushInfo &info, int64_t block_index) { int ret = OB_SUCCESS; diff --git a/src/storage/tmp_file/ob_shared_nothing_tmp_file.h b/src/storage/tmp_file/ob_shared_nothing_tmp_file.h index 441295dbb..6d6c3414d 100644 --- a/src/storage/tmp_file/ob_shared_nothing_tmp_file.h +++ b/src/storage/tmp_file/ob_shared_nothing_tmp_file.h @@ -78,7 +78,9 @@ public: data_finished_continuous_flush_info_num_(0), meta_finished_continuous_flush_info_num_(0), data_flush_infos_(), - meta_flush_infos_() + meta_flush_infos_(), + need_to_wait_for_the_previous_data_flush_req_to_complete_(false), + need_to_wait_for_the_previous_meta_flush_req_to_complete_(false) { data_flush_infos_.set_attr(ObMemAttr(MTL_ID(), "TmpFileFInfo")); meta_flush_infos_.set_attr(ObMemAttr(MTL_ID(), "TmpFileFInfo")); @@ -100,12 +102,16 @@ public: } TO_STRING_KV(K(flush_seq_), K(data_flush_infos_.size()), K(meta_flush_infos_.size()), K(data_finished_continuous_flush_info_num_), - K(meta_finished_continuous_flush_info_num_)); + K(meta_finished_continuous_flush_info_num_), + K(need_to_wait_for_the_previous_data_flush_req_to_complete_), + K(need_to_wait_for_the_previous_meta_flush_req_to_complete_)); int64_t flush_seq_; int64_t data_finished_continuous_flush_info_num_; int64_t meta_finished_continuous_flush_info_num_; ObArray data_flush_infos_; ObArray meta_flush_infos_; + bool need_to_wait_for_the_previous_data_flush_req_to_complete_; + bool need_to_wait_for_the_previous_meta_flush_req_to_complete_; }; public: ObSharedNothingTmpFile(); @@ -263,6 +269,17 @@ private: const int64_t end_pos, const int64_t flushed_data_page_num); int update_meta_tree_after_flush_(const int64_t start_pos, const int64_t end_pos); + + int generate_data_flush_info_(ObTmpFileFlushTask &flush_task, + ObTmpFileFlushInfo &info, + ObTmpFileDataFlushContext &data_flush_context, + const int64_t flush_sequence, + const bool need_flush_tail); + int generate_meta_flush_info_(ObTmpFileFlushTask &flush_task, + ObTmpFileFlushInfo &info, + ObTmpFileTreeFlushContext &meta_flush_context, + const int64_t flush_sequence, + const bool need_flush_tail); private: int reinsert_flush_node_(const bool is_meta); OB_INLINE bool has_unfinished_page_() const { return file_size_ % ObTmpFileGlobal::PAGE_SIZE != 0; } diff --git a/src/storage/tmp_file/ob_tmp_file_flush_ctx.cpp b/src/storage/tmp_file/ob_tmp_file_flush_ctx.cpp index 4aa258277..cfd2cac01 100644 --- a/src/storage/tmp_file/ob_tmp_file_flush_ctx.cpp +++ b/src/storage/tmp_file/ob_tmp_file_flush_ctx.cpp @@ -116,9 +116,9 @@ int ObTmpFileBatchFlushContext::clear_flush_ctx(ObTmpFileFlushPriorityManager &p } if (OB_FAIL(ret)) { - } else if (flush_seq_ctx_.send_io_succ_cnt_ == flush_seq_ctx_.create_flush_task_cnt_) { + } else if (flush_seq_ctx_.prepare_finished_cnt_ == flush_seq_ctx_.create_flush_task_cnt_) { LOG_DEBUG("reset flush_seq_ctx_", KPC(this)); - flush_seq_ctx_.send_io_succ_cnt_ = 0; + flush_seq_ctx_.prepare_finished_cnt_ = 0; flush_seq_ctx_.create_flush_task_cnt_ = 0; flush_seq_ctx_.flush_sequence_ += 1; // remove the files that have already been successfully flushed(recorded in file_ctx_hash_) @@ -130,10 +130,10 @@ int ObTmpFileBatchFlushContext::clear_flush_ctx(ObTmpFileFlushPriorityManager &p } else { file_ctx_hash_.clear(); } - } else if (flush_seq_ctx_.send_io_succ_cnt_ < flush_seq_ctx_.create_flush_task_cnt_) { + } else if (flush_seq_ctx_.prepare_finished_cnt_ < flush_seq_ctx_.create_flush_task_cnt_) { // likely to occur when write buffer pool is full and need to fast flush meta page LOG_WARN("flush_seq_ctx_ could not increase flush sequence", KPC(this)); - } else if (OB_UNLIKELY(flush_seq_ctx_.send_io_succ_cnt_ > flush_seq_ctx_.create_flush_task_cnt_)) { + } else if (OB_UNLIKELY(flush_seq_ctx_.prepare_finished_cnt_ > flush_seq_ctx_.create_flush_task_cnt_)) { LOG_ERROR("unexpected flush_seq_ctx_", KPC(this)); } @@ -184,13 +184,22 @@ void ObTmpFileBatchFlushContext::destroy() file_ctx_hash_.destroy(); } -void ObTmpFileBatchFlushContext::update_ctx_by_flush_task(const ObTmpFileFlushTask &flush_task) +void ObTmpFileBatchFlushContext::update_actual_flush_size(const ObTmpFileFlushTask &flush_task) { actual_flush_size_ += flush_task.get_data_length(); +} - if (ObTmpFileFlushTask::TFFT_WAIT == flush_task.get_state() || flush_task.get_data_length() == 0) { - // we will free task if its data_length == 0, therefore we count it as succ here - flush_seq_ctx_.send_io_succ_cnt_ += 1; +void ObTmpFileBatchFlushContext::try_update_prepare_finished_cnt(const ObTmpFileFlushTask &flush_task, bool& recorded) +{ + int ret = OB_SUCCESS; + + recorded = false; + if (OB_UNLIKELY(flush_seq_ctx_.prepare_finished_cnt_ >= flush_seq_ctx_.create_flush_task_cnt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("unexpected flush_seq_ctx_", KPC(this)); + } else if (ObTmpFileFlushTask::TFFT_WAIT == flush_task.get_state() || flush_task.get_data_length() == 0) { + ++flush_seq_ctx_.prepare_finished_cnt_; + recorded = true; } } @@ -229,6 +238,7 @@ ObTmpFileFlushTask::ObTmpFileFlushTask() create_ts_(-1), is_io_finished_(false), fast_flush_tree_page_(false), + recorded_as_prepare_finished_(false), task_state_(ObTmpFileFlushTaskState::TFFT_INITED), tmp_file_block_handle_(), handle_(), @@ -249,6 +259,7 @@ void ObTmpFileFlushTask::destroy() create_ts_ = -1; is_io_finished_ = false; fast_flush_tree_page_ = false; + recorded_as_prepare_finished_ = false; task_state_ = ObTmpFileFlushTaskState::TFFT_INITED; flush_infos_.reset(); } diff --git a/src/storage/tmp_file/ob_tmp_file_flush_ctx.h b/src/storage/tmp_file/ob_tmp_file_flush_ctx.h index 4c9726413..259379fac 100644 --- a/src/storage/tmp_file/ob_tmp_file_flush_ctx.h +++ b/src/storage/tmp_file/ob_tmp_file_flush_ctx.h @@ -122,7 +122,8 @@ public: void destroy(); void record_flush_stage(); void record_flush_task(const int64_t data_length); - void update_ctx_by_flush_task(const ObTmpFileFlushTask &flush_task); + void update_actual_flush_size(const ObTmpFileFlushTask &flush_task); + void try_update_prepare_finished_cnt(const ObTmpFileFlushTask &flush_task, bool& recorded); public: static const int64_t MAX_COPY_FAIL_COUNT = 512; typedef hash::ObHashMap 0; + return flush_seq_ctx_.prepare_finished_cnt_ == flush_seq_ctx_.create_flush_task_cnt_ && + flush_seq_ctx_.prepare_finished_cnt_ > 0; } OB_INLINE void set_fail_too_many(const bool fail_too_many) { fail_too_many_ = fail_too_many; } OB_INLINE bool is_fail_too_many() { return fail_too_many_; } @@ -266,6 +267,8 @@ public: OB_INLINE bool atomic_get_io_finished() const { return ATOMIC_LOAD(&is_io_finished_); } OB_INLINE void set_is_fast_flush_tree(const bool is_fast_flush_tree) { fast_flush_tree_page_ = is_fast_flush_tree; } OB_INLINE bool get_is_fast_flush_tree() const { return fast_flush_tree_page_; } + OB_INLINE void mark_recorded_as_prepare_finished() { recorded_as_prepare_finished_ = true; } + OB_INLINE bool get_recorded_as_prepare_finished() const { return recorded_as_prepare_finished_; } OB_INLINE void set_state(const ObTmpFileFlushTaskState state) { task_state_ = state; } OB_INLINE ObTmpFileFlushTaskState get_state() const { return task_state_; } OB_INLINE void set_tmp_file_block_handle(const ObTmpFileBlockHandle &tfb_handle) { tmp_file_block_handle_ = tfb_handle; } @@ -281,7 +284,7 @@ public: } TO_STRING_KV(KP(this), KP(kvpair_), K(ret_code_), K(data_length_), K(block_index_), K(flush_seq_), K(create_ts_), K(is_io_finished_), - K(fast_flush_tree_page_), K(task_state_), K(tmp_file_block_handle_), K(flush_infos_)); + K(fast_flush_tree_page_), K(recorded_as_prepare_finished_), K(task_state_), K(tmp_file_block_handle_), K(flush_infos_)); private: ObKVCacheInstHandle inst_handle_; ObKVCachePair *kvpair_; @@ -293,6 +296,7 @@ private: int64_t create_ts_; bool is_io_finished_; bool fast_flush_tree_page_; // indicate the task requires fast flush tree pages + bool recorded_as_prepare_finished_; ObTmpFileFlushTaskState task_state_; ObTmpFileBlockHandle tmp_file_block_handle_;// hold a reference to the corresponding tmp file block to prevent it from being released blocksstable::ObMacroBlockHandle handle_; diff --git a/src/storage/tmp_file/ob_tmp_file_flush_manager.cpp b/src/storage/tmp_file/ob_tmp_file_flush_manager.cpp index 2321df3ea..2ae360ecd 100644 --- a/src/storage/tmp_file/ob_tmp_file_flush_manager.cpp +++ b/src/storage/tmp_file/ob_tmp_file_flush_manager.cpp @@ -281,7 +281,12 @@ int ObTmpFileFlushManager::flush(ObSpLinkQueue &flushing_queue, } while (OB_SUCC(ret) && FlushState::TFFT_WAIT != next_state); STORAGE_LOG(DEBUG, "drive flush task finished", KR(ret), K(fast_flush_meta), KPC(flush_task), K(flush_ctx_)); - flush_ctx_.update_ctx_by_flush_task(*flush_task); + bool recorded_as_prepare_finished = false; + flush_ctx_.update_actual_flush_size(*flush_task); + flush_ctx_.try_update_prepare_finished_cnt(*flush_task, recorded_as_prepare_finished); + if (recorded_as_prepare_finished) { + flush_task->mark_recorded_as_prepare_finished(); + } if (ObTmpFileFlushTask::TFFT_FILL_BLOCK_BUF < flush_task->get_state()) { flush_ctx_.record_flush_task(flush_task->get_data_length()); // maintain statistics } @@ -707,9 +712,21 @@ int ObTmpFileFlushManager::retry(ObTmpFileFlushTask &flush_task) } } while (OB_SUCC(ret) && FlushState::TFFT_WAIT != next_state); - flush_ctx_.update_ctx_by_flush_task(flush_task); - if (flush_ctx_.can_clear_flush_ctx()) { - flush_ctx_.clear_flush_ctx(flush_priority_mgr_); + if (!flush_task.get_recorded_as_prepare_finished()) { + if (flush_task.get_flush_seq() != flush_ctx_.get_flush_sequence()) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(ERROR, "flush_seq in flush_task is not equal to flush_seq in flush_ctx", + KR(ret), K(flush_task), K(flush_ctx_)); + } else { + bool recorded_as_prepare_finished = false; + flush_ctx_.try_update_prepare_finished_cnt(flush_task, recorded_as_prepare_finished); + if (recorded_as_prepare_finished) { + flush_task.mark_recorded_as_prepare_finished(); + if (flush_ctx_.can_clear_flush_ctx()) { + flush_ctx_.clear_flush_ctx(flush_priority_mgr_); + } + } + } } return ret; } @@ -948,15 +965,17 @@ int ObTmpFileFlushManager::update_meta_data_after_flush_for_files_(ObTmpFileFlus } else if (OB_FAIL(file->update_meta_after_flush(flush_info.batch_flush_idx_, is_meta, reset_ctx))){ STORAGE_LOG(WARN, "fail to update meta data", KR(ret), K(is_meta), K(flush_info)); } else { - int tmp_ret = OB_SUCCESS; - // reset data/meta flush ctx after file update meta complete to prevent - // flush use stale flushed_page_id to copy data after the page is evicted. - // use empty ctx here because we have inserted ctx for every file when flushing begins. - ResetFlushCtxOp reset_op(is_meta); - ObTmpFileSingleFlushContext empty_ctx; - if (flush_task.get_flush_seq() == flush_ctx_.get_flush_sequence() && - OB_TMP_FAIL(flush_ctx_.get_file_ctx_hash().set_or_update(file->get_fd(), empty_ctx, reset_op))) { - STORAGE_LOG(WARN, "fail to clean file ctx from hash", KR(tmp_ret), K(file)); + if (reset_ctx) { + int tmp_ret = OB_SUCCESS; + // reset data/meta flush ctx after file update meta complete to prevent + // flush use stale flushed_page_id to copy data after the page is evicted. + // use empty ctx here because we have inserted ctx for every file when flushing begins. + ResetFlushCtxOp reset_op(is_meta); + ObTmpFileSingleFlushContext empty_ctx; + if (flush_task.get_flush_seq() == flush_ctx_.get_flush_sequence() && + OB_TMP_FAIL(flush_ctx_.get_file_ctx_hash().set_or_update(file->get_fd(), empty_ctx, reset_op))) { + STORAGE_LOG(ERROR, "fail to clean file ctx from hash", KR(tmp_ret), K(file)); + } } flush_info.update_meta_data_done_ = true; } From c3a9346bd59fbe96288494971472a3e01793b4e8 Mon Sep 17 00:00:00 2001 From: leftgeek <1094669802@qq.com> Date: Fri, 23 Aug 2024 06:38:15 +0000 Subject: [PATCH 185/249] [FEAT MERGE]Support insert overwrite partition --- .../ob_table_load_control_rpc_executor.cpp | 1 + .../table_load/ob_table_load_client_task.cpp | 2 + .../table_load/ob_table_load_instance.cpp | 53 +- .../table_load/ob_table_load_instance.h | 11 +- .../table_load/ob_table_load_redef_table.cpp | 1 + .../table_load/ob_table_load_redef_table.h | 7 +- .../table_load/ob_table_load_service.cpp | 127 +- .../table_load/ob_table_load_service.h | 8 + .../table_load/ob_table_load_struct.h | 10 +- src/rootserver/CMakeLists.txt | 4 + .../ob_direct_load_partition_exchange.cpp | 208 ++ .../ob_direct_load_partition_exchange.h | 51 + src/rootserver/ob_ddl_service.cpp | 167 +- src/rootserver/ob_ddl_service.h | 1 + src/rootserver/ob_partition_exchange.cpp | 1713 ++++++++++------- src/rootserver/ob_partition_exchange.h | 230 ++- src/rootserver/ob_root_service.cpp | 7 +- src/share/ob_rpc_struct.cpp | 5 +- src/share/ob_rpc_struct.h | 3 +- .../engine/cmd/ob_load_data_direct_impl.cpp | 2 + .../engine/cmd/ob_table_direct_insert_ctx.cpp | 82 +- .../engine/cmd/ob_table_direct_insert_ctx.h | 22 + src/sql/optimizer/ob_table_location.h | 5 + src/sql/resolver/dml/ob_insert_resolver.cpp | 13 - .../direct_load/ob_direct_load_struct.cpp | 12 + .../direct_load/ob_direct_load_struct.h | 13 + 26 files changed, 1957 insertions(+), 801 deletions(-) create mode 100644 src/rootserver/direct_load/ob_direct_load_partition_exchange.cpp create mode 100644 src/rootserver/direct_load/ob_direct_load_partition_exchange.h diff --git a/src/observer/table_load/control/ob_table_load_control_rpc_executor.cpp b/src/observer/table_load/control/ob_table_load_control_rpc_executor.cpp index 137240cd2..a4b2a0583 100644 --- a/src/observer/table_load/control/ob_table_load_control_rpc_executor.cpp +++ b/src/observer/table_load/control/ob_table_load_control_rpc_executor.cpp @@ -70,6 +70,7 @@ int ObDirectLoadControlPreBeginExecutor::process() param.load_mode_ = arg_.load_mode_; param.compressor_type_ = arg_.compressor_type_; param.online_sample_percent_ = arg_.online_sample_percent_; + param.load_level_ = ObDirectLoadLevel::TABLE; if (OB_FAIL(create_table_ctx(param, arg_.ddl_param_, table_ctx))) { LOG_WARN("fail to create table ctx", KR(ret)); } diff --git a/src/observer/table_load/ob_table_load_client_task.cpp b/src/observer/table_load/ob_table_load_client_task.cpp index b64c80bf7..7087f46e8 100644 --- a/src/observer/table_load/ob_table_load_client_task.cpp +++ b/src/observer/table_load/ob_table_load_client_task.cpp @@ -181,6 +181,7 @@ public: load_param.method_, load_param.insert_mode_, load_param.load_mode_, + load_param.load_level_, column_ids))) { LOG_WARN("fail to check support direct load", KR(ret)); } @@ -290,6 +291,7 @@ public: load_param.load_mode_ = ObDirectLoadMode::TABLE_LOAD; load_param.compressor_type_ = compressor_type; load_param.online_sample_percent_ = online_sample_percent; + load_param.load_level_ = ObDirectLoadLevel::TABLE; } return ret; } diff --git a/src/observer/table_load/ob_table_load_instance.cpp b/src/observer/table_load/ob_table_load_instance.cpp index b37a820b2..f451cacbf 100644 --- a/src/observer/table_load/ob_table_load_instance.cpp +++ b/src/observer/table_load/ob_table_load_instance.cpp @@ -65,6 +65,17 @@ void ObTableLoadInstance::destroy() int ObTableLoadInstance::init(ObTableLoadParam ¶m, const ObIArray &column_ids, ObTableLoadExecCtx *execute_ctx) +{ + int ret = OB_SUCCESS; + ObArray tablet_ids; + tablet_ids.reset(); + return init(param, column_ids, tablet_ids, execute_ctx); +} + +int ObTableLoadInstance::init(ObTableLoadParam ¶m, + const ObIArray &column_ids, + const ObIArray &tablet_ids, + ObTableLoadExecCtx *execute_ctx) { int ret = OB_SUCCESS; if (IS_INIT) { @@ -74,6 +85,10 @@ int ObTableLoadInstance::init(ObTableLoadParam ¶m, column_ids.count() != param.column_count_ || !execute_ctx->is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", KR(ret), K(param), K(column_ids), KPC(execute_ctx)); + } else if ((ObDirectLoadLevel::PARTITION == param.load_level_) + && OB_UNLIKELY(!is_valid_tablet_ids(tablet_ids))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tablet ids", KR(ret), K(param.load_level_), K(tablet_ids)); } else { DISABLE_SQL_MEMLEAK_GUARD; execute_ctx_ = execute_ctx; @@ -86,14 +101,15 @@ int ObTableLoadInstance::init(ObTableLoadParam ¶m, LOG_WARN("fail to check tenant", KR(ret), K(param.tenant_id_)); } // start stmt - else if (OB_FAIL(start_stmt(param))) { - LOG_WARN("fail to start stmt", KR(ret), K(param)); + else if (OB_FAIL(start_stmt(param, tablet_ids))) { + LOG_WARN("fail to start stmt", KR(ret), K(param), K(tablet_ids)); } // double check support for concurrency of direct load and ddl else if (OB_FAIL(ObTableLoadService::check_support_direct_load(param.table_id_, param.method_, param.insert_mode_, param.load_mode_, + param.load_level_, column_ids))) { LOG_WARN("fail to check support direct load", KR(ret), K(param)); } @@ -160,7 +176,9 @@ int ObTableLoadInstance::px_commit_ddl() return ret; } -int ObTableLoadInstance::start_stmt(const ObTableLoadParam ¶m) +int ObTableLoadInstance::start_stmt( + const ObTableLoadParam ¶m, + const ObIArray &tablet_ids) { int ret = OB_SUCCESS; stmt_ctx_.reset(); @@ -186,8 +204,8 @@ int ObTableLoadInstance::start_stmt(const ObTableLoadParam ¶m) } } } else { - if (OB_FAIL(start_redef_table(param))) { - LOG_WARN("fail to start redef table", KR(ret), K(param)); + if (OB_FAIL(start_redef_table(param, tablet_ids))) { + LOG_WARN("fail to start redef table", KR(ret), K(param), K(tablet_ids)); } } if (OB_SUCC(ret)) { @@ -404,7 +422,9 @@ int ObTableLoadInstance::init_ddl_param_for_inc_direct_load() return ret; } -int ObTableLoadInstance::start_redef_table(const ObTableLoadParam ¶m) +int ObTableLoadInstance::start_redef_table( + const ObTableLoadParam ¶m, + const ObIArray &tablet_ids) { int ret = OB_SUCCESS; ObTableLoadDDLParam &ddl_param = stmt_ctx_.ddl_param_; @@ -415,7 +435,10 @@ int ObTableLoadInstance::start_redef_table(const ObTableLoadParam ¶m) start_arg.parallelism_ = param.parallel_; start_arg.is_load_data_ = !param.px_mode_; start_arg.is_insert_overwrite_ = ObDirectLoadMode::is_insert_overwrite(param.load_mode_); - if (OB_FAIL(ObTableLoadRedefTable::start(start_arg, start_res, *stmt_ctx_.session_info_))) { + if ((ObDirectLoadLevel::PARTITION == param.load_level_) + && OB_FAIL(start_arg.tablet_ids_.assign(tablet_ids))) { + LOG_WARN("failed to assign tablet ids", KR(ret), K(param.load_level_), K(tablet_ids)); + } else if (OB_FAIL(ObTableLoadRedefTable::start(start_arg, start_res, *stmt_ctx_.session_info_))) { LOG_WARN("fail to start redef table", KR(ret), K(start_arg)); } else { ddl_param.dest_table_id_ = start_res.dest_table_id_; @@ -761,5 +784,21 @@ int ObTableLoadInstance::write_trans(TransCtx &trans_ctx, int32_t session_id, return ret; } +bool ObTableLoadInstance::is_valid_tablet_ids(const ObIArray &tablet_ids) +{ + bool is_valid = true; + if (tablet_ids.empty()) { + is_valid = false; + } else { + for (int64_t i = 0; is_valid && (i < tablet_ids.count()); ++i) { + const ObTabletID &tablet_id = tablet_ids.at(i); + if (OB_UNLIKELY(!tablet_id.is_valid())) { + is_valid = false; + } + } + } + return is_valid; +} + } // namespace observer } // namespace oceanbase diff --git a/src/observer/table_load/ob_table_load_instance.h b/src/observer/table_load/ob_table_load_instance.h index 40a9281c3..70fd70af6 100644 --- a/src/observer/table_load/ob_table_load_instance.h +++ b/src/observer/table_load/ob_table_load_instance.h @@ -41,6 +41,10 @@ public: int init(ObTableLoadParam ¶m, const common::ObIArray &column_ids, ObTableLoadExecCtx *execute_ctx); + int init(ObTableLoadParam ¶m, + const common::ObIArray &column_ids, + const common::ObIArray &tablet_ids, + ObTableLoadExecCtx *execute_ctx); int commit(); int px_commit_data(); int px_commit_ddl(); @@ -49,7 +53,8 @@ public: sql::ObLoadDataStat *get_job_stat() const { return job_stat_; } const table::ObTableLoadResultInfo &get_result_info() const { return result_info_; } private: - int start_stmt(const ObTableLoadParam ¶m); + int start_stmt(const ObTableLoadParam ¶m, + const common::ObIArray &tablet_ids); int end_stmt(const bool commit); // incremental static int64_t get_stmt_expire_ts(sql::ObSQLSessionInfo *session_info); @@ -59,7 +64,8 @@ private: int lock_table_in_tx(); int init_ddl_param_for_inc_direct_load(); // full - int start_redef_table(const ObTableLoadParam ¶m); + int start_redef_table(const ObTableLoadParam ¶m, + const common::ObIArray &tablet_ids); int commit_redef_table(); int abort_redef_table(); private: @@ -88,6 +94,7 @@ public: const table::ObTableLoadObjRowArray &obj_rows); private: int check_trans_committed(TransCtx &trans_ctx); + bool is_valid_tablet_ids(const common::ObIArray &tablet_ids); private: struct StmtCtx diff --git a/src/observer/table_load/ob_table_load_redef_table.cpp b/src/observer/table_load/ob_table_load_redef_table.cpp index d8f332cb6..7020fae94 100644 --- a/src/observer/table_load/ob_table_load_redef_table.cpp +++ b/src/observer/table_load/ob_table_load_redef_table.cpp @@ -72,6 +72,7 @@ int ObTableLoadRedefTable::start(const ObTableLoadRedefTableStartArg &arg, session_info.get_local_nls_timestamp_format(), session_info.get_local_nls_timestamp_tz_format(), session_info.get_tz_info_wrap(), + arg.tablet_ids_, need_reorder_column_id))) { LOG_WARN("fail to init create hidden table arg", KR(ret)); } else if (OB_FAIL(ObDDLServerClient::create_hidden_table(create_table_arg, create_table_res, diff --git a/src/observer/table_load/ob_table_load_redef_table.h b/src/observer/table_load/ob_table_load_redef_table.h index e45ac1f8b..0166b29f1 100644 --- a/src/observer/table_load/ob_table_load_redef_table.h +++ b/src/observer/table_load/ob_table_load_redef_table.h @@ -30,7 +30,7 @@ struct ObTableLoadRedefTableStartArg public: ObTableLoadRedefTableStartArg() : tenant_id_(common::OB_INVALID_ID), table_id_(common::OB_INVALID_ID), parallelism_(0), - is_load_data_(false), is_insert_overwrite_(false) + is_load_data_(false), is_insert_overwrite_(false), tablet_ids_() { } ~ObTableLoadRedefTableStartArg() = default; @@ -41,19 +41,22 @@ public: parallelism_ = 0; is_load_data_ = false; is_insert_overwrite_ = false; + tablet_ids_.reset(); } bool is_valid() const { return common::OB_INVALID_ID != tenant_id_ && common::OB_INVALID_ID != table_id_ && 0 != parallelism_; } - TO_STRING_KV(K_(tenant_id), K_(table_id), K_(parallelism), K_(is_load_data), K_(is_insert_overwrite)); + TO_STRING_KV(K_(tenant_id), K_(table_id), K_(parallelism), + K_(is_load_data), K_(is_insert_overwrite), K_(tablet_ids)); public: uint64_t tenant_id_; uint64_t table_id_; uint64_t parallelism_; bool is_load_data_; bool is_insert_overwrite_; + common::ObArray tablet_ids_; }; struct ObTableLoadRedefTableStartRes diff --git a/src/observer/table_load/ob_table_load_service.cpp b/src/observer/table_load/ob_table_load_service.cpp index 73f7b940c..c8b1fe168 100644 --- a/src/observer/table_load/ob_table_load_service.cpp +++ b/src/observer/table_load/ob_table_load_service.cpp @@ -434,6 +434,7 @@ int ObTableLoadService::check_support_direct_load( const ObDirectLoadMethod::Type method, const ObDirectLoadInsertMode::Type insert_mode, const ObDirectLoadMode::Type load_mode, + const ObDirectLoadLevel::Type load_level, const ObIArray &column_ids) { int ret = OB_SUCCESS; @@ -448,7 +449,7 @@ int ObTableLoadService::check_support_direct_load( ObTableLoadSchema::get_table_schema(tenant_id, table_id, schema_guard, table_schema))) { LOG_WARN("fail to get table schema", KR(ret), K(tenant_id), K(table_id)); } else { - ret = check_support_direct_load(schema_guard, table_schema, method, insert_mode, load_mode, column_ids); + ret = check_support_direct_load(schema_guard, table_schema, method, insert_mode, load_mode, load_level, column_ids); } } return ret; @@ -459,6 +460,7 @@ int ObTableLoadService::check_support_direct_load(ObSchemaGetterGuard &schema_gu const ObDirectLoadMethod::Type method, const ObDirectLoadInsertMode::Type insert_mode, const ObDirectLoadMode::Type load_mode, + const ObDirectLoadLevel::Type load_level, const ObIArray &column_ids) { int ret = OB_SUCCESS; @@ -474,7 +476,7 @@ int ObTableLoadService::check_support_direct_load(ObSchemaGetterGuard &schema_gu ret = OB_TABLE_NOT_EXIST; LOG_WARN("table schema is null", KR(ret)); } else { - ret = check_support_direct_load(schema_guard, table_schema, method, insert_mode, load_mode, column_ids); + ret = check_support_direct_load(schema_guard, table_schema, method, insert_mode, load_mode, load_level, column_ids); } } return ret; @@ -487,6 +489,7 @@ int ObTableLoadService::check_support_direct_load(ObSchemaGetterGuard &schema_gu const ObDirectLoadMethod::Type method, const ObDirectLoadInsertMode::Type insert_mode, const ObDirectLoadMode::Type load_mode, + const ObDirectLoadLevel::Type load_level, const ObIArray &column_ids) { int ret = OB_SUCCESS; @@ -499,6 +502,7 @@ int ObTableLoadService::check_support_direct_load(ObSchemaGetterGuard &schema_gu LOG_WARN("invalid args", KR(ret), KP(table_schema), K(method), K(insert_mode), K(column_ids)); } else { const uint64_t tenant_id = MTL_ID(); + uint64_t compat_version = 0; bool trigger_enabled = false; bool has_udt_column = false; bool has_fts_index = false; @@ -509,7 +513,9 @@ int ObTableLoadService::check_support_direct_load(ObSchemaGetterGuard &schema_gu // check if it is a user table const char *tmp_prefix = ObDirectLoadMode::is_insert_overwrite(load_mode) ? InsertOverwritePrefix : EmptyPrefix; - if (!table_schema->is_user_table()) { + if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) { + LOG_WARN("fail to get data version", KR(ret), K(tenant_id)); + } else if (!table_schema->is_user_table()) { ret = OB_NOT_SUPPORTED; if (lib::is_oracle_mode() && table_schema->is_tmp_table()) { LOG_WARN("direct-load does not support oracle temporary table", KR(ret)); @@ -593,13 +599,10 @@ int ObTableLoadService::check_support_direct_load(ObSchemaGetterGuard &schema_gu LOG_WARN("direct-load does not support table with materialized view log", KR(ret)); FORWARD_USER_ERROR_MSG(ret, "%sdirect-load does not support table with materialized view log", tmp_prefix); } else if (ObDirectLoadMethod::is_incremental(method)) { // incremental direct-load - uint64_t compat_version = 0; if (!ObDirectLoadInsertMode::is_valid_for_incremental_method(insert_mode)) { ret = OB_NOT_SUPPORTED; LOG_WARN("using incremental direct-load without inc_replace or normal is not supported", KR(ret)); FORWARD_USER_ERROR_MSG(ret, "using incremental direct-load without inc_replace or normal is not supported"); - } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) { - LOG_WARN("fail to get data version", KR(ret), K(tenant_id)); } else if (compat_version < DATA_VERSION_4_3_1_0) { ret = OB_NOT_SUPPORTED; LOG_WARN("version lower than 4.3.1.0 does not support incremental direct-load", KR(ret)); @@ -622,17 +625,9 @@ int ObTableLoadService::check_support_direct_load(ObSchemaGetterGuard &schema_gu ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected insert mode for full direct-load", KR(ret), K(method), K(insert_mode)); } else if (ObDirectLoadMode::is_insert_overwrite(load_mode)) { - uint64_t compat_version = 0; - if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) { - LOG_WARN("fail to get data version", KR(ret), K(tenant_id)); - } else if (compat_version < DATA_VERSION_4_3_2_0) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("version lower than 4.3.2.0 does not support insert overwrite", KR(ret)); - FORWARD_USER_ERROR_MSG(ret, "version lower than 4.3.2.0 does not support insert overwrite"); - } else if (table_schema->get_foreign_key_infos().count() > 0) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("insert overwrite with incremental direct-load does not support table with foreign keys", KR(ret)); - FORWARD_USER_ERROR_MSG(ret, "insert overwrite with direct-load does not support table with foreign keys"); + if (OB_FAIL(check_support_direct_load_for_insert_overwrite( + schema_guard, *table_schema, load_level, compat_version))) { + LOG_WARN("failed to check support direct load for insert overwrite", KR(ret)); } } } @@ -690,6 +685,104 @@ int ObTableLoadService::check_support_direct_load(ObSchemaGetterGuard &schema_gu return ret; } +int ObTableLoadService::check_support_direct_load_for_insert_overwrite( + ObSchemaGetterGuard &schema_guard, + const ObTableSchema &table_schema, + const ObDirectLoadLevel::Type load_level, + const uint64_t compat_version) +{ + int ret = OB_SUCCESS; + if (compat_version < DATA_VERSION_4_3_2_0) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("version lower than 4.3.2.0 does not support insert overwrite", KR(ret)); + FORWARD_USER_ERROR_MSG(ret, "version lower than 4.3.2.0 does not support insert overwrite"); + } else if (table_schema.get_foreign_key_infos().count() > 0) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("insert overwrite with incremental direct-load does not support table with foreign keys", KR(ret)); + FORWARD_USER_ERROR_MSG(ret, "insert overwrite with direct-load does not support table with foreign keys"); + } else if (ObDirectLoadLevel::PARTITION == load_level) { + if (compat_version < DATA_VERSION_4_3_3_0) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("version lower than 4.3.3.0 does not support insert overwrite partition", KR(ret)); + FORWARD_USER_ERROR_MSG(ret, "version lower than 4.3.3.0 does not support insert overwrite partition"); + } else if (table_schema.is_duplicate_table()) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("insert overwrite partition does not support duplicate table", KR(ret)); + FORWARD_USER_ERROR_MSG(ret, "insert overwrite partition does not support duplicate table"); + } else if (table_schema.get_index_tid_count() > 0) { + bool has_global_indexes = false; + ObSEArray simple_index_infos; + if (OB_FAIL(table_schema.get_simple_index_infos(simple_index_infos))) { + LOG_WARN("failed to get simple_index_infos", KR(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && !has_global_indexes && (i < simple_index_infos.count()); ++i) { + const ObTableSchema *index_schema = NULL; + if (OB_FAIL(schema_guard.get_table_schema( + table_schema.get_tenant_id(), simple_index_infos.at(i).table_id_, index_schema))) { + LOG_WARN("failed to get table schema", KR(ret)); + } else if (OB_ISNULL(index_schema)) { + ret = OB_TABLE_NOT_EXIST; + LOG_WARN("index schema from schema guard is NULL", KR(ret), KP(index_schema)); + } else if (index_schema->is_global_index_table()) { + has_global_indexes = true; + } + } + } + if (OB_SUCC(ret) && (has_global_indexes)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("insert overwrite partition does not support table with global indexes", KR(ret)); + FORWARD_USER_ERROR_MSG(ret, "insert overwrite partition does not support table with global indexes"); + } + } else if (0 != table_schema.get_autoinc_column_id()) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("insert overwrite partition does not support table with auto_increment columns", KR(ret)); + FORWARD_USER_ERROR_MSG(ret, "insert overwrite partition does not support table with auto_increment columns"); + } else if (lib::is_oracle_mode()) { + ObArray column_ids; + if (OB_FAIL(table_schema.get_column_ids(column_ids))) { + LOG_WARN("failed to get column ids", KR(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && (i < column_ids.count()); ++i) { + const ObColumnSchemaV2 *col_schema = table_schema.get_column_schema(column_ids.at(i)); + if (OB_ISNULL(col_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null column schema", KR(ret)); + } else if (col_schema->is_identity_column()) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("insert overwrite partition does not support table with identity columns", KR(ret)); + FORWARD_USER_ERROR_MSG(ret, "insert overwrite partition does not support table with identity columns"); + } + } + } + } + + if (OB_SUCC(ret)) { // check partition type + ObPartitionFuncType part_type = PARTITION_FUNC_TYPE_MAX; + switch (table_schema.get_part_level()) { + case ObPartitionLevel::PARTITION_LEVEL_ONE: + part_type = table_schema.get_part_option().get_part_func_type(); + break; + case ObPartitionLevel::PARTITION_LEVEL_TWO: + part_type = table_schema.get_sub_part_option().get_part_func_type(); + break; + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected partition level", KR(ret), K(table_schema.get_part_level())); + } + + if (OB_FAIL(ret)) { + } else if (OB_UNLIKELY(!((PARTITION_FUNC_TYPE_RANGE == part_type) + || (PARTITION_FUNC_TYPE_RANGE_COLUMNS == part_type) + || (PARTITION_FUNC_TYPE_LIST == part_type) + || (PARTITION_FUNC_TYPE_LIST_COLUMNS == part_type)))) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("insert overwrite partition does not support hash/key partitions", KR(ret), K(part_type)); + FORWARD_USER_ERROR_MSG(ret, "insert overwrite partition does not support hash/key partition type"); + } + } + } // end if + return ret; +} ObTableLoadTableCtx *ObTableLoadService::alloc_ctx() { return OB_NEW(ObTableLoadTableCtx, ObMemAttr(MTL_ID(), "TLD_TableCtxVal")); diff --git a/src/observer/table_load/ob_table_load_service.h b/src/observer/table_load/ob_table_load_service.h index aacc14db5..e2d8a0897 100644 --- a/src/observer/table_load/ob_table_load_service.h +++ b/src/observer/table_load/ob_table_load_service.h @@ -40,6 +40,7 @@ public: const storage::ObDirectLoadMethod::Type method, const storage::ObDirectLoadInsertMode::Type insert_mode, const storage::ObDirectLoadMode::Type load_mode, + const storage::ObDirectLoadLevel::Type load_level, const common::ObIArray &column_ids); // 业务层指定schema_guard进行检查 static int check_support_direct_load(share::schema::ObSchemaGetterGuard &schema_guard, @@ -47,13 +48,20 @@ public: const storage::ObDirectLoadMethod::Type method, const storage::ObDirectLoadInsertMode::Type insert_mode, const storage::ObDirectLoadMode::Type load_mode, + const storage::ObDirectLoadLevel::Type load_level, const common::ObIArray &column_ids); static int check_support_direct_load(share::schema::ObSchemaGetterGuard &schema_guard, const share::schema::ObTableSchema *table_schema, const storage::ObDirectLoadMethod::Type method, const storage::ObDirectLoadInsertMode::Type insert_mode, const storage::ObDirectLoadMode::Type load_mode, + const storage::ObDirectLoadLevel::Type load_level, const common::ObIArray &column_ids); + static int check_support_direct_load_for_insert_overwrite( + share::schema::ObSchemaGetterGuard &schema_guard, + const share::schema::ObTableSchema &table_schema, + const storage::ObDirectLoadLevel::Type load_level, + const uint64_t compat_version); static ObTableLoadTableCtx *alloc_ctx(); static void free_ctx(ObTableLoadTableCtx *table_ctx); static int add_ctx(ObTableLoadTableCtx *table_ctx); diff --git a/src/observer/table_load/ob_table_load_struct.h b/src/observer/table_load/ob_table_load_struct.h index 4783a4144..021bdbc83 100644 --- a/src/observer/table_load/ob_table_load_struct.h +++ b/src/observer/table_load/ob_table_load_struct.h @@ -115,7 +115,8 @@ public: insert_mode_(storage::ObDirectLoadInsertMode::INVALID_INSERT_MODE), load_mode_(storage::ObDirectLoadMode::INVALID_MODE), compressor_type_(ObCompressorType::INVALID_COMPRESSOR), - online_sample_percent_(1.) + online_sample_percent_(1.), + load_level_(storage::ObDirectLoadLevel::INVALID_LEVEL) { } @@ -151,7 +152,8 @@ public: (storage::ObDirectLoadInsertMode::INC_REPLACE == insert_mode_ ? sql::ObLoadDupActionType::LOAD_REPLACE == dup_action_ : true) && - ObCompressorType::INVALID_COMPRESSOR != compressor_type_; + ObCompressorType::INVALID_COMPRESSOR != compressor_type_ && + storage::ObDirectLoadLevel::is_type_valid(load_level_); } TO_STRING_KV(K_(tenant_id), @@ -173,7 +175,8 @@ public: "insert_mode", storage::ObDirectLoadInsertMode::get_type_string(insert_mode_), "direct_load_mode", storage::ObDirectLoadMode::get_type_string(load_mode_), K_(compressor_type), - K_(online_sample_percent)); + K_(online_sample_percent), + "direct_load_level", storage::ObDirectLoadLevel::get_type_string(load_level_)); public: uint64_t tenant_id_; @@ -196,6 +199,7 @@ public: storage::ObDirectLoadMode::Type load_mode_; ObCompressorType compressor_type_; double online_sample_percent_; + storage::ObDirectLoadLevel::Type load_level_; }; struct ObTableLoadDDLParam diff --git a/src/rootserver/CMakeLists.txt b/src/rootserver/CMakeLists.txt index cd45ae23f..46afd7cbf 100644 --- a/src/rootserver/CMakeLists.txt +++ b/src/rootserver/CMakeLists.txt @@ -195,6 +195,10 @@ ob_set_subtarget(ob_rootserver mview mview/ob_mview_dependency_service.cpp ) +ob_set_subtarget(ob_rootserver direct_load + direct_load/ob_direct_load_partition_exchange.cpp +) + ob_set_subtarget(ob_rootserver standby standby/ob_standby_service.cpp standby/ob_recovery_ls_service.cpp diff --git a/src/rootserver/direct_load/ob_direct_load_partition_exchange.cpp b/src/rootserver/direct_load/ob_direct_load_partition_exchange.cpp new file mode 100644 index 000000000..b9e3c1759 --- /dev/null +++ b/src/rootserver/direct_load/ob_direct_load_partition_exchange.cpp @@ -0,0 +1,208 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX RS + +#include "rootserver/direct_load/ob_direct_load_partition_exchange.h" +#include "rootserver/ob_ddl_service.h" +#include "rootserver/ob_ddl_operator.h" + +namespace oceanbase +{ +using namespace common; +using namespace obrpc; +using namespace share; +using namespace share::schema; +namespace rootserver +{ +ObDirectLoadPartitionExchange::ObDirectLoadPartitionExchange( + ObDDLService &ddl_service, const uint64_t data_version) + : ObPartitionExchange(ddl_service, data_version) +{ +} + +ObDirectLoadPartitionExchange::~ObDirectLoadPartitionExchange() +{ +} + +int ObDirectLoadPartitionExchange::exchange_multipart_table_partitions( + const uint64_t tenant_id, + ObDDLSQLTransaction &trans, + ObSchemaGetterGuard &schema_guard, + const ObTableSchema &base_table_schema, + const ObTableSchema &inc_table_schema, + const ObIArray &base_table_tablet_ids, + const ObIArray &inc_table_tablet_ids) +{ + int ret = OB_SUCCESS; + bool is_oracle_mode = false; + int64_t schema_version = OB_INVALID_VERSION; + const ObPartitionLevel exchange_partition_level = base_table_schema.get_part_level(); + bool is_subpartition = (PARTITION_LEVEL_TWO == exchange_partition_level); + ObDDLOperator ddl_operator(ddl_service_.get_schema_service(), ddl_service_.get_sql_proxy()); + + if (data_version_ < DATA_VERSION_4_3_3_0) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("version lower than 4.3.3.0 does not support insert overwrite partition", KR(ret)); + FORWARD_USER_ERROR_MSG(ret, "version lower than 4.3.2.0 does not support insert overwrite partition"); + } else if (OB_FAIL(schema_guard.get_schema_version(tenant_id, schema_version))) { + LOG_WARN("failed to get tenant schema version", KR(ret), K(tenant_id), K(schema_version)); + } else if (OB_FAIL(base_table_schema.check_if_oracle_compat_mode(is_oracle_mode))) { + LOG_WARN("check_if_oracle_compat_mode failed", KR(ret), K(is_oracle_mode)); + } else if (OB_FAIL(check_multipart_exchange_conditions(schema_guard, + base_table_schema, + inc_table_schema, + base_table_tablet_ids, + inc_table_tablet_ids, + is_oracle_mode))) { + LOG_WARN("failed to check multipart exchange conditions", KR(ret), K(base_table_schema), + K(inc_table_schema), K(base_table_tablet_ids), K(inc_table_tablet_ids), K(is_oracle_mode)); + } else if (OB_FAIL(inner_init(base_table_schema, + inc_table_schema, + exchange_partition_level, + is_oracle_mode, + schema_guard))) { + LOG_WARN("failed to inner init", KR(ret), K(base_table_schema), K(inc_table_schema), + K(exchange_partition_level), K(is_oracle_mode)); + } else if (OB_FAIL(exchange_data_table_partitions(tenant_id, + base_table_schema, + inc_table_schema, + base_table_tablet_ids, + inc_table_tablet_ids, + is_oracle_mode, + is_subpartition, + ddl_operator, + trans, + schema_guard))) { + LOG_WARN("failed to exchange data table partitions", + KR(ret), K(tenant_id), K(base_table_schema), K(inc_table_schema), + K(base_table_tablet_ids), K(inc_table_tablet_ids), K(is_oracle_mode), K(is_subpartition)); + } else if (OB_FAIL(exchange_auxiliary_table_partitions(tenant_id, + base_table_schema, + inc_table_schema, + base_table_tablet_ids, + inc_table_tablet_ids, + is_oracle_mode, + is_subpartition, + ddl_operator, + trans, + schema_guard))) { + LOG_WARN("failed to exchange auxiliary table partitions", + KR(ret), K(tenant_id), K(base_table_schema), K(inc_table_schema), + K(base_table_tablet_ids), K(inc_table_tablet_ids), K(is_oracle_mode), K(is_subpartition)); + } else { + int64_t new_inc_schema_version = OB_INVALID_VERSION; + int64_t new_base_schema_version = OB_INVALID_VERSION; + if (OB_FAIL(push_data_table_schema_version_(tenant_id, inc_table_schema, + nullptr/*ddl_stmt_str*/, base_table_schema.get_table_id(), new_inc_schema_version, trans))) { + LOG_WARN("failed to push data table schema version", + KR(ret), K(tenant_id), K(inc_table_schema), K(base_table_schema.get_table_id())); + } else if (OB_FAIL(push_data_table_schema_version_(tenant_id, base_table_schema, + nullptr/*ddl_stmt_str*/, inc_table_schema.get_table_id(), new_base_schema_version, trans))) { + LOG_WARN("failed to push data table schema version", + KR(ret), K(tenant_id), K(base_table_schema), K(inc_table_schema.get_table_id())); + } else if (OB_FAIL(adapting_cdc_changes_in_exchange_partition_(tenant_id, + base_table_schema.get_table_id(), inc_table_schema.get_table_id(), trans))) { + LOG_WARN("failed to adapting cdc changes in exchange_partition", + KR(ret), K(tenant_id), K(base_table_schema.get_table_id()), K(inc_table_schema.get_table_id())); + } else { + LOG_INFO("succeed to exchange direct load table partitions", + K(base_table_schema.get_table_name_str()), K(base_table_schema.get_table_id()), + K(inc_table_schema.get_table_name_str()), K(inc_table_schema.get_table_id())); + } + } + + return ret; +} + +int ObDirectLoadPartitionExchange::check_multipart_exchange_conditions( + ObSchemaGetterGuard &schema_guard, + const ObTableSchema &base_table_schema, + const ObTableSchema &inc_table_schema, + const ObIArray &base_tablet_ids, + const ObIArray &inc_tablet_ids, + const bool is_oracle_mode) +{ + int ret = OB_SUCCESS; + const ObPartitionLevel exchange_part_level = base_table_schema.get_part_level(); + const ObString &part_name = base_table_schema.get_table_name_str(); + if (OB_UNLIKELY(!base_table_schema.is_partitioned_table() || !inc_table_schema.is_partitioned_table())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("both base_table_schema and inc_table_schema should be partitioned tables", + KR(ret), K(base_table_schema.is_partitioned_table()), K(inc_table_schema.is_partitioned_table())); + } else if (OB_UNLIKELY(base_table_schema.get_part_level() != inc_table_schema.get_part_level())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("the partition level of base_table_schema and inc_table_schema should be the same", + KR(ret), K(base_table_schema.get_part_level()), K(inc_table_schema.get_part_level())); + } else if (OB_UNLIKELY(base_tablet_ids.count() != inc_tablet_ids.count())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("the count of base_tablet_ids and inc_tablet_ids should be equal", + KR(ret), K(base_tablet_ids.count()), K(inc_tablet_ids.count())); + } else if (OB_FAIL(check_data_table_partition_exchange_conditions_( + base_table_schema, inc_table_schema, base_tablet_ids, inc_tablet_ids, exchange_part_level, is_oracle_mode))) { + LOG_WARN("failed to check data table partition exchange conditions", + KR(ret), K(base_table_schema), K(inc_table_schema), K(part_name), K(exchange_part_level), K(is_oracle_mode)); + } else { + ObPartitionFuncType part_type = PARTITION_FUNC_TYPE_MAX; + if (ObPartitionLevel::PARTITION_LEVEL_ONE == exchange_part_level) { + part_type = base_table_schema.get_part_option().get_part_func_type(); + } else if (ObPartitionLevel::PARTITION_LEVEL_TWO == exchange_part_level) { + part_type = base_table_schema.get_sub_part_option().get_part_func_type(); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected partition level", KR(ret), K(exchange_part_level)); + } + + if (OB_FAIL(ret)) { + } else if (OB_UNLIKELY(!((PARTITION_FUNC_TYPE_RANGE == part_type) + || (PARTITION_FUNC_TYPE_RANGE_COLUMNS == part_type) + || (PARTITION_FUNC_TYPE_LIST == part_type) + || (PARTITION_FUNC_TYPE_LIST_COLUMNS == part_type)))) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("insert overwrite partition does not support hash/key partitions", KR(ret), K(part_type)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "insert overwrite partition to table with hash/key partition type is"); + } + } + return ret; +} + +int ObDirectLoadPartitionExchange::check_table_conditions_in_common_( + const ObTableSchema &base_table_schema, + const ObTableSchema &inc_table_schema, + const ObPartitionLevel exchange_partition_level, + const bool is_oracle_mode) +{ + int ret = OB_SUCCESS; + HEAP_VARS_2((ObTableSchema, new_base_table_schema), + (ObTableSchema, new_inc_table_schema)) { + if (OB_FAIL(new_base_table_schema.assign(base_table_schema))) { + LOG_WARN("failed to assign base table schema", KR(ret), K(base_table_schema)); + } else if (OB_FALSE_IT(new_base_table_schema.set_in_offline_ddl_white_list(true))) { + } else if (OB_FAIL(new_inc_table_schema.assign(inc_table_schema))) { + LOG_WARN("failed to assign inc table schema", KR(ret), K(inc_table_schema)); + } else if (OB_FALSE_IT(new_inc_table_schema.set_in_offline_ddl_white_list(true))) { + } else if (OB_FALSE_IT(new_inc_table_schema.set_table_mode(new_base_table_schema.get_table_mode()))) { + // hidden table has different table mode + } else if (OB_FAIL(ObPartitionExchange::check_table_conditions_in_common_( + new_base_table_schema, + new_inc_table_schema, + exchange_partition_level, + is_oracle_mode))) { + LOG_WARN("failed to check table conditions in common", KR(ret), + K(new_base_table_schema), K(new_inc_table_schema), K(exchange_partition_level), K(is_oracle_mode)); + } + } + return ret; +} + +} // end namespace rootserver +} // end namespace oceanbase \ No newline at end of file diff --git a/src/rootserver/direct_load/ob_direct_load_partition_exchange.h b/src/rootserver/direct_load/ob_direct_load_partition_exchange.h new file mode 100644 index 000000000..c8044f249 --- /dev/null +++ b/src/rootserver/direct_load/ob_direct_load_partition_exchange.h @@ -0,0 +1,51 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_ROOTSERVER_OB_DIRECT_LOAD_PARTITION_EXCHANGE_H_ +#define OCEANBASE_ROOTSERVER_OB_DIRECT_LOAD_PARTITION_EXCHANGE_H_ + +#include "rootserver/ob_partition_exchange.h" + +namespace oceanbase +{ +namespace rootserver +{ +class ObDirectLoadPartitionExchange : public ObPartitionExchange +{ +public: + explicit ObDirectLoadPartitionExchange(ObDDLService &ddl_service, const uint64_t data_version); + virtual ~ObDirectLoadPartitionExchange(); + int exchange_multipart_table_partitions(const uint64_t tenant_id, + ObDDLSQLTransaction &trans, + share::schema::ObSchemaGetterGuard &schema_guard, + const share::schema::ObTableSchema &base_table_schema, + const share::schema::ObTableSchema &inc_table_schema, + const common::ObIArray &base_table_tablet_ids, + const common::ObIArray &inc_table_tablet_ids); +private: + virtual int check_table_conditions_in_common_(const share::schema::ObTableSchema &base_table_schema, + const share::schema::ObTableSchema &inc_table_schema, + const share::schema::ObPartitionLevel exchange_partition_level, + const bool is_oracle_mode) override; + int check_multipart_exchange_conditions(share::schema::ObSchemaGetterGuard &schema_guard, + const share::schema::ObTableSchema &base_table_schema, + const share::schema::ObTableSchema &inc_table_schema, + const common::ObIArray &base_tablet_ids, + const common::ObIArray &inc_tablet_ids, + const bool is_oracle_mode); +private: + DISALLOW_COPY_AND_ASSIGN(ObDirectLoadPartitionExchange); +}; +} // end namespace rootserver +} // end namespace oceanbase + +#endif // OCEANBASE_ROOTSERVER_OB_DIRECT_LOAD_PARTITION_EXCHANGE_H_ \ No newline at end of file diff --git a/src/rootserver/ob_ddl_service.cpp b/src/rootserver/ob_ddl_service.cpp index 6c287d327..5022945bb 100755 --- a/src/rootserver/ob_ddl_service.cpp +++ b/src/rootserver/ob_ddl_service.cpp @@ -123,6 +123,7 @@ #include "storage/mview/ob_mview_sched_job_utils.h" #include "rootserver/restore/ob_tenant_clone_util.h" #include "rootserver/mview/ob_mview_dependency_service.h" +#include "rootserver/direct_load/ob_direct_load_partition_exchange.h" namespace oceanbase { @@ -14388,8 +14389,23 @@ int ObDDLService::create_hidden_table( LOG_WARN("param init failed", K(ret)); } else if (OB_FAIL(root_service->get_ddl_scheduler().prepare_alter_table_arg(param, &new_table_schema, alter_table_arg))) { LOG_WARN("prepare alter table arg fail", K(ret)); - } else { - LOG_DEBUG("alter table arg preparation complete!", K(ret), K(alter_table_arg)); + } else if (!create_hidden_table_arg.get_tablet_ids().empty()){ + const ObIArray &tablet_ids = create_hidden_table_arg.get_tablet_ids(); + alter_table_arg.alter_table_schema_.reset_partition_array(); + for (int64_t i = 0; OB_SUCC(ret) && (i < tablet_ids.count()); ++i) { + ObPartition part; + if (OB_FALSE_IT(part.set_tablet_id(tablet_ids.at(i)))) { + } else if (OB_FAIL(alter_table_arg.alter_table_schema_.add_partition(part))) { + LOG_WARN("failed to add partition", KR(ret), K(part)); + } + } + if (OB_SUCC(ret)) { + alter_table_arg.is_direct_load_partition_ = true; + } + } + + LOG_DEBUG("alter table arg preparation complete!", K(ret), K(alter_table_arg)); + if (OB_SUCC(ret)) { ObCreateDDLTaskParam param(tenant_id, create_hidden_table_arg.get_ddl_type(), orig_table_schema, @@ -19982,6 +19998,140 @@ int ObDDLService::swap_orig_and_hidden_table_state(obrpc::ObAlterTableArg &alter return ret; } +int ObDDLService::swap_orig_and_hidden_table_partitions(obrpc::ObAlterTableArg &alter_table_arg) +{ + int ret = OB_SUCCESS; + AlterTableSchema &alter_table_schema = alter_table_arg.alter_table_schema_; + const uint64_t tenant_id = alter_table_schema.get_tenant_id(); + ObPartition** part_array = alter_table_schema.get_part_array(); + ObArray tablet_ids; + ObArray inc_tablet_ids; + + if (OB_FAIL(check_inner_stat())) { + LOG_WARN("variable is not init", K(ret)); + } else if (OB_NOT_NULL(part_array)) { + for (int64_t i = 0; OB_SUCC(ret) && (i < alter_table_schema.get_partition_num()); ++i) { + ObPartition *part = part_array[i]; + if (OB_NOT_NULL(part)) { + if (OB_FAIL(tablet_ids.push_back(part->get_tablet_id()))) { + LOG_WARN("failed to add tablet id", KR(ret)); + } + } + } + } + + if (OB_SUCC(ret) && !tablet_ids.empty()) { + ObDDLSQLTransaction trans(schema_service_); + const ObTableSchema *orig_table_schema = nullptr; + const ObTableSchema *hidden_table_schema = nullptr; + HEAP_VARS_2((ObTableSchema, new_orig_table_schema), + (ObTableSchema, new_hidden_table_schema)) { + int64_t refreshed_schema_version = 0; + ObSchemaGetterGuard schema_guard; + schema_guard.set_session_id(alter_table_arg.session_id_); + if (OB_FAIL(get_tenant_schema_guard_with_version_in_inner_table(tenant_id, schema_guard))) { + LOG_WARN("failed to get schema guard with version in inner table", K(ret), K(tenant_id)); + } else if (OB_FAIL(schema_guard.get_schema_version(tenant_id, refreshed_schema_version))) { + LOG_WARN("failed to get tenant schema version", KR(ret), K(tenant_id)); + } else if (OB_FAIL(get_orig_and_hidden_table_schema(alter_table_arg, + schema_guard, + schema_guard, + alter_table_schema, + orig_table_schema, + hidden_table_schema))) { + LOG_WARN("failed to get orig and hidden table schema", KR(ret), K(alter_table_arg)); + } else if (OB_ISNULL(orig_table_schema)) { + ret = OB_TABLE_NOT_EXIST; + LOG_WARN("failed to get orig table schema", KR(ret), K(alter_table_arg), KP(orig_table_schema)); + } else if (OB_ISNULL(hidden_table_schema)) { + ret = OB_TABLE_NOT_EXIST; + LOG_WARN("failed to get hidden table schema", KR(ret), K(alter_table_arg), KP(hidden_table_schema)); + } else if (OB_FAIL(new_orig_table_schema.assign(*orig_table_schema)) + || OB_FAIL(new_hidden_table_schema.assign(*hidden_table_schema))) { + LOG_WARN("failed to assign schema", KR(ret), KPC(orig_table_schema), KPC(hidden_table_schema)); + } else if (OB_FAIL(trans.start(sql_proxy_, tenant_id, refreshed_schema_version))) { + LOG_WARN("failed to start trans, ", KR(ret), K(tenant_id), K(refreshed_schema_version)); + } else if (OB_FAIL(check_hidden_table_constraint_exist(hidden_table_schema, + orig_table_schema, + schema_guard))) { + LOG_WARN("failed to check hidden table constraint existence", KR(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && (i < tablet_ids.count()); ++i) { + const ObTabletID &orig_tablet_id = tablet_ids.at(i); + int64_t part_idx = OB_INVALID_INDEX; + int64_t subpart_idx = OB_INVALID_INDEX; + ObTabletID inc_tablet_id; + ObObjectID inc_object_id; + ObObjectID first_level_part_id; + if (OB_FAIL(orig_table_schema->get_part_idx_by_tablet(orig_tablet_id, part_idx, subpart_idx))) { + LOG_WARN("failed to get part idx by tablet", KR(ret), K(orig_tablet_id)); + } else if (OB_FAIL(hidden_table_schema->get_tablet_and_object_id_by_index( + part_idx, subpart_idx, inc_tablet_id, inc_object_id, first_level_part_id))) { + LOG_WARN("failed to get tablet and object id by index", KR(ret), K(part_idx), K(subpart_idx)); + } else if (OB_FAIL(inc_tablet_ids.push_back(inc_tablet_id))) { + LOG_WARN("failed to add inc tablet id", KR(ret), K(inc_tablet_id)); + } + } + } + + if (OB_SUCC(ret)) { + uint64_t tenant_data_version = 0; + if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, tenant_data_version))) { + LOG_WARN("get min data version failed", K(ret), K(tenant_id)); + } else { + ObDirectLoadPartitionExchange dl_part_exchange(*this, tenant_data_version); + new_orig_table_schema.set_in_offline_ddl_white_list(true); + new_hidden_table_schema.set_in_offline_ddl_white_list(true); + if (OB_FAIL(dl_part_exchange.exchange_multipart_table_partitions( + tenant_id, + trans, + schema_guard, + new_orig_table_schema, + new_hidden_table_schema, + tablet_ids, + inc_tablet_ids))) { + LOG_WARN("failed to exchange multipart table partitions", KR(ret)); + } + } + } + + if (OB_SUCC(ret)) { + ObTableLockOwnerID owner_id; + if (OB_FAIL(owner_id.convert_from_value(ObLockOwnerType::DEFAULT_OWNER_TYPE, alter_table_arg.task_id_))) { + LOG_WARN("failed to get owner id", K(ret), K(alter_table_arg.task_id_)); + } else if (OB_FAIL(ObDDLLock::unlock_for_offline_ddl(tenant_id, + orig_table_schema->get_table_id(), + owner_id, + trans))) { + LOG_WARN("failed to unlock ddl", K(ret)); + } + } + } + + if (trans.is_started()) { + int temp_ret = OB_SUCCESS; + if (OB_SUCCESS != (temp_ret = trans.end(OB_SUCC(ret)))) { + LOG_WARN("trans end failed", "is_commit", OB_SUCCESS == ret, K(temp_ret)); + ret = (OB_SUCC(ret)) ? temp_ret : ret; + } + } + } + + int tmp_ret = OB_SUCCESS; + if (OB_FAIL(ret)) { + } else if (OB_SUCCESS != (tmp_ret = publish_schema(tenant_id))) { + LOG_WARN("publish_schema failed", K(tmp_ret)); + } + if (OB_SUCC(ret)) { + ret = tmp_ret; + } + if (OB_NO_NEED_UPDATE == ret) { + ret = OB_SUCCESS; + } + + return ret; +} + int ObDDLService::check_and_replace_default_index_name_on_demand( const bool is_oracle_mode, common::ObIAllocator &allocator, @@ -20687,9 +20837,16 @@ int ObDDLService::cleanup_garbage(ObAlterTableArg &alter_table_arg) } if (OB_SUCC(ret)) { if (ddl_succ) { - if (OB_FAIL(new_orig_table_schema.assign(*hidden_table_schema)) - || OB_FAIL(new_hidden_table_schema.assign(*orig_table_schema))) { - LOG_WARN("fail to assign schema", K(ret)); + if (alter_table_arg.is_direct_load_partition_) { + if (OB_FAIL(new_orig_table_schema.assign(*orig_table_schema)) + || OB_FAIL(new_hidden_table_schema.assign(*hidden_table_schema))) { + LOG_WARN("fail to assign schema", K(ret)); + } + } else { + if (OB_FAIL(new_orig_table_schema.assign(*hidden_table_schema)) + || OB_FAIL(new_hidden_table_schema.assign(*orig_table_schema))) { + LOG_WARN("fail to assign schema", K(ret)); + } } } else { if (OB_FAIL(new_orig_table_schema.assign(*orig_table_schema)) diff --git a/src/rootserver/ob_ddl_service.h b/src/rootserver/ob_ddl_service.h index 8281af7b7..9299622aa 100644 --- a/src/rootserver/ob_ddl_service.h +++ b/src/rootserver/ob_ddl_service.h @@ -561,6 +561,7 @@ public: * step5: rename hidden table name to orig table name and modify the state to non-hidden */ int swap_orig_and_hidden_table_state(obrpc::ObAlterTableArg &alter_table_arg); + int swap_orig_and_hidden_table_partitions(obrpc::ObAlterTableArg &alter_table_arg); /** * The function is designed for the recover restore table ddl, which is to check whether the object diff --git a/src/rootserver/ob_partition_exchange.cpp b/src/rootserver/ob_partition_exchange.cpp index ca89f186d..17b51815c 100644 --- a/src/rootserver/ob_partition_exchange.cpp +++ b/src/rootserver/ob_partition_exchange.cpp @@ -25,6 +25,8 @@ #include "share/schema/ob_schema_service_sql_impl.h" #include "share/schema/ob_schema_struct.h" #include "share/tablet/ob_tablet_to_table_history_operator.h" // ObTabletToTableHistoryOperator +#include "share/tablet/ob_tablet_table_operator.h" +#include "share/tablet/ob_tablet_to_ls_operator.h" #include "sql/resolver/ddl/ob_ddl_resolver.h" #include "sql/resolver/ob_resolver_utils.h" @@ -37,7 +39,7 @@ using namespace share::schema; namespace rootserver { ObPartitionExchange::ObPartitionExchange(ObDDLService &ddl_service, const uint64_t data_version) - : ddl_service_(ddl_service), data_version_(data_version) + : ddl_service_(ddl_service), data_version_(data_version), is_inited_(false) { } @@ -72,12 +74,42 @@ int ObPartitionExchange::check_and_exchange_partition(const obrpc::ObExchangePar LOG_WARN("check_if_oracle_compat_mode failed", K(ret), K(is_oracle_mode)); } else if (OB_FAIL(check_partition_exchange_conditions_(arg, *base_table_schema, *inc_table_schema, is_oracle_mode, schema_guard))) { LOG_WARN("fail to check partition exchange conditions", K(ret), K(arg), KPC(base_table_schema), KPC(inc_table_schema), K(is_oracle_mode)); + } else if (OB_FAIL(inner_init(*base_table_schema, *inc_table_schema, arg.exchange_partition_level_, is_oracle_mode, schema_guard))) { + LOG_WARN("fail to inner init", K(ret), K(arg), KPC(base_table_schema), KPC(inc_table_schema), K(is_oracle_mode)); } else if (OB_FAIL(do_exchange_partition_(arg, res, *base_table_schema, *inc_table_schema, is_oracle_mode, schema_guard))) { LOG_WARN("fail to do exchange partition", K(ret), K(arg), K(res), KPC(base_table_schema), KPC(inc_table_schema), K(is_oracle_mode)); } return ret; } +int ObPartitionExchange::inner_init( + const ObTableSchema &base_table_schema, + const ObTableSchema &inc_table_schema, + const ObPartitionLevel exchange_partition_level, + const bool is_oracle_mode, + ObSchemaGetterGuard &schema_guard) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObPartitionExchange init twice", KR(ret), KP(this)); + } else if (OB_FAIL(used_pt_nt_id_map_.create(MAX_INDEXES, lib::ObLabel("ExchangePart")))) { + LOG_WARN("failed to create used pt nt id map", K(ret)); + } else if (OB_FAIL(used_table_to_tablet_ids_map_.create(MAX_INDEXES, lib::ObLabel("ExchangePart")))) { + LOG_WARN("failed to create used pt nt tablet id map", K(ret)); + } else if (OB_FAIL(generate_auxiliary_table_mapping_(base_table_schema, + inc_table_schema, + exchange_partition_level, + is_oracle_mode, + schema_guard))) { + LOG_WARN("fail to generate auxiliary table mapping", K(ret), + K(base_table_schema), K(inc_table_schema), K(exchange_partition_level), K(is_oracle_mode)); + } else { + is_inited_ = true; + } + return ret; +} + int ObPartitionExchange::check_partition_exchange_conditions_(const obrpc::ObExchangePartitionArg &arg, const ObTableSchema &base_table_schema, const ObTableSchema &inc_table_schema, const bool is_oracle_mode, ObSchemaGetterGuard &schema_guard) { int ret = OB_SUCCESS; @@ -89,19 +121,40 @@ int ObPartitionExchange::check_partition_exchange_conditions_(const obrpc::ObExc ret = OB_NOT_SUPPORTED; LOG_WARN("exchange partition table is not user table", K(ret), K(base_table_schema.is_user_table()), K(base_table_schema.is_ctas_tmp_table()), K(inc_table_schema.is_user_table()), K(inc_table_schema.is_ctas_tmp_table())); LOG_USER_ERROR(OB_NOT_SUPPORTED, "Partition exchange operations for non-user tables are"); - } else if (OB_FAIL(used_pt_nt_id_map_.create(MAX_INDEXES, lib::ObLabel("ExchangePart")))) { - LOG_WARN("failed to create used pt nt id map", K(ret)); - } else if (OB_FAIL(used_table_to_tablet_id_map_.create(MAX_INDEXES, lib::ObLabel("ExchangePart")))) { - LOG_WARN("failed to create used pt nt tablet id map", K(ret)); - } else if (OB_FAIL(check_data_table_partition_exchange_conditions_(base_table_schema, inc_table_schema, arg.base_table_part_name_, arg.exchange_partition_level_, is_oracle_mode))) { - LOG_WARN("failed to check data table partition exchange conditions", K(ret), K(base_table_schema), K(inc_table_schema), K(arg), K(is_oracle_mode)); - } else if (OB_FAIL(generate_auxiliary_table_mapping_(base_table_schema, - inc_table_schema, - arg.base_table_part_name_, - arg.exchange_partition_level_, - is_oracle_mode, - schema_guard))) { - LOG_WARN("fail to generate auxiliary table mapping", K(ret), K(base_table_schema), K(inc_table_schema), K(is_oracle_mode)); + } else if (OB_UNLIKELY(arg.base_table_part_name_.empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("base table part name should not be empty", K(ret), K(arg.base_table_part_name_)); + } else { + ObArray base_tablet_ids; + ObArray inc_tablet_ids; + ObTabletID base_tablet_id; + if (PARTITION_LEVEL_ONE == arg.exchange_partition_level_) { + const ObPartition *data_part = nullptr; + if (OB_FAIL(get_and_check_data_partition_by_name(base_table_schema, arg.base_table_part_name_, data_part))) { + LOG_WARN("fail to get and check data partition and index", K(ret), K(base_table_schema), K(arg.base_table_part_name_)); + } else { + base_tablet_id = data_part->get_tablet_id(); + } + } else if (PARTITION_LEVEL_TWO == arg.exchange_partition_level_) { + const ObPartition *data_part = nullptr; + const ObSubPartition *data_subpart = nullptr; + if (OB_FAIL(get_and_check_data_subpartition_by_name(base_table_schema, arg.base_table_part_name_, data_part, data_subpart))) { + LOG_WARN("fail to get and check data subpartition and index", K(ret), K(base_table_schema), K(arg.base_table_part_name_)); + } else { + base_tablet_id = data_subpart->get_tablet_id(); + } + } + + if (OB_SUCC(ret)) { + OZ (base_tablet_ids.push_back(base_tablet_id)); + OZ (inc_tablet_ids.push_back(inc_table_schema.get_tablet_id())); + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(check_data_table_partition_exchange_conditions_(base_table_schema, + inc_table_schema, base_tablet_ids, inc_tablet_ids, arg.exchange_partition_level_, is_oracle_mode))) { + LOG_WARN("failed to check data table partition exchange conditions", K(ret), K(base_table_schema), K(inc_table_schema), K(arg), K(is_oracle_mode)); + } + } } return ret; } @@ -120,43 +173,28 @@ int ObPartitionExchange::do_exchange_partition_(const obrpc::ObExchangePartition } else { if (PARTITION_LEVEL_ONE == arg.exchange_partition_level_) { const ObPartition *data_part = nullptr; - int64_t data_partition_index = OB_INVALID_INDEX; - if (OB_FAIL(get_data_partition_and_index_(base_table_schema, arg.base_table_part_name_, data_part, data_partition_index))) { - LOG_WARN("fail to get data partition and index", K(ret), K(base_table_schema), K(arg.base_table_part_name_)); - } else if (OB_ISNULL(data_part)) { - ret = OB_PARTITION_NOT_EXIST; - LOG_WARN("partition not found", K(ret), K(base_table_schema), K(arg.base_table_part_name_)); - } else if (OB_UNLIKELY(OB_INVALID_INDEX == data_partition_index)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid partition index", K(ret), K(base_table_schema), K(arg.base_table_part_name_)); + if (OB_FAIL(get_and_check_data_partition_by_name(base_table_schema, arg.base_table_part_name_, data_part))) { + LOG_WARN("fail to get and check data partition and index", K(ret), K(base_table_schema), K(arg.base_table_part_name_)); } else if (OB_FAIL(lock_exchange_data_table_and_partition_(tenant_id, base_table_schema, inc_table_schema, data_part->get_tablet_id(), trans))) { LOG_WARN("fail to exchange data table partition", K(ret), K(tenant_id), K(base_table_schema), K(inc_table_schema), K(data_part->get_tablet_id())); - } else if (OB_FAIL(exchange_data_table_partition_(tenant_id, base_table_schema, inc_table_schema, *data_part, is_oracle_mode, ddl_operator, trans, schema_guard))) { + } else if (OB_FAIL(exchange_data_table_partition_(tenant_id, base_table_schema, inc_table_schema, data_part->get_tablet_id(), inc_table_schema.get_tablet_id(), is_oracle_mode, false/*is_subpartition*/, ddl_operator, trans, schema_guard))) { LOG_WARN("fail to exchange data table partition", K(ret), K(tenant_id), K(base_table_schema), K(inc_table_schema), KPC(data_part), K(is_oracle_mode)); - } else if (OB_FAIL(exchange_auxiliary_table_partition_(tenant_id, data_partition_index, *data_part, is_oracle_mode, ddl_operator, trans, schema_guard))) { - LOG_WARN("fail to exchange auxiliary table partition", K(ret), K(tenant_id), K(data_partition_index), KPC(data_part), K(is_oracle_mode)); + } else if (OB_FAIL(exchange_auxiliary_table_partition_(tenant_id, base_table_schema, inc_table_schema, data_part->get_tablet_id(), inc_table_schema.get_tablet_id(), is_oracle_mode, false/*is_subpartition*/, ddl_operator, trans, schema_guard))) { + LOG_WARN("fail to exchange auxiliary table partition", K(ret), K(tenant_id), KPC(data_part), K(is_oracle_mode)); } else if (OB_FAIL(set_global_storage_index_unusable_(tenant_id, base_table_schema, inc_table_schema, ddl_operator, trans, schema_guard))) { LOG_WARN("fail to set global storage index unable", K(ret), K(tenant_id), K(base_table_schema), K(inc_table_schema)); } } else if (PARTITION_LEVEL_TWO == arg.exchange_partition_level_) { const ObPartition *data_part = nullptr; const ObSubPartition *data_subpart = nullptr; - int64_t data_partition_index = OB_INVALID_INDEX; - int64_t data_subpartition_index = OB_INVALID_INDEX; - if (OB_FAIL(get_data_subpartition_and_index_(base_table_schema, arg.base_table_part_name_, data_part, data_subpart, data_partition_index, data_subpartition_index))) { - LOG_WARN("fail to get data subpartition and index", K(ret), K(base_table_schema), K(arg.base_table_part_name_)); - } else if (OB_ISNULL(data_part) || OB_ISNULL(data_subpart)) { - ret = OB_PARTITION_NOT_EXIST; - LOG_WARN("subpartition not found", K(ret), K(base_table_schema), K(arg.base_table_part_name_), KPC(data_part), KPC(data_subpart)); - } else if (OB_UNLIKELY(OB_INVALID_INDEX == data_partition_index || OB_INVALID_INDEX == data_subpartition_index)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid subpartition index", K(ret), K(base_table_schema), K(arg.base_table_part_name_), K(data_partition_index), K(data_subpartition_index)); + if (OB_FAIL(get_and_check_data_subpartition_by_name(base_table_schema, arg.base_table_part_name_, data_part, data_subpart))) { + LOG_WARN("fail to get and check data subpartition and index", K(ret), K(base_table_schema), K(arg.base_table_part_name_)); } else if (OB_FAIL(lock_exchange_data_table_and_partition_(tenant_id, base_table_schema, inc_table_schema, data_subpart->get_tablet_id(), trans))) { LOG_WARN("fail to exchange data table partition", K(ret), K(tenant_id), K(base_table_schema), K(inc_table_schema), K(data_subpart->get_tablet_id())); - } else if (OB_FAIL(exchange_data_table_subpartition_(tenant_id, base_table_schema, inc_table_schema, *data_part, *data_subpart, is_oracle_mode, ddl_operator, trans, schema_guard))) { + } else if (OB_FAIL(exchange_data_table_partition_(tenant_id, base_table_schema, inc_table_schema, data_subpart->get_tablet_id(), inc_table_schema.get_tablet_id(), is_oracle_mode, true/*is_subpartition*/, ddl_operator, trans, schema_guard))) { LOG_WARN("fail to exchange data table subpartition", K(ret), K(tenant_id), K(base_table_schema), K(inc_table_schema), KPC(data_part), KPC(data_subpart), K(is_oracle_mode)); - } else if (OB_FAIL(exchange_auxiliary_table_subpartition_(tenant_id, data_partition_index, data_subpartition_index, *data_part, *data_subpart, is_oracle_mode, ddl_operator, trans, schema_guard))) { - LOG_WARN("fail to exchange auxiliary table subpartition", K(ret), K(tenant_id), K(data_partition_index), K(data_subpartition_index), KPC(data_part), KPC(data_subpart), K(is_oracle_mode)); + } else if (OB_FAIL(exchange_auxiliary_table_partition_(tenant_id, base_table_schema, inc_table_schema, data_subpart->get_tablet_id(), inc_table_schema.get_tablet_id(), is_oracle_mode, true/*is_subpartition*/, ddl_operator, trans, schema_guard))) { + LOG_WARN("fail to exchange auxiliary table partition", K(ret), K(tenant_id), K(base_table_schema), K(inc_table_schema), KPC(data_part), KPC(data_subpart), K(is_oracle_mode)); } else if (OB_FAIL(set_global_storage_index_unusable_(tenant_id, base_table_schema, inc_table_schema, ddl_operator, trans, schema_guard))) { LOG_WARN("fail to set global storage index unable", K(ret), K(tenant_id), K(base_table_schema), K(inc_table_schema)); } @@ -212,16 +250,27 @@ int ObPartitionExchange::lock_exchange_data_table_and_partition_(const uint64_t int ObPartitionExchange::check_data_table_partition_exchange_conditions_(const ObTableSchema &base_table_schema, const ObTableSchema &inc_table_schema, - const ObString &exchange_partition_name, + const ObIArray &base_tablet_ids, + const ObIArray &inc_tablet_ids, const ObPartitionLevel exchange_partition_level, const bool is_oracle_mode) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(exchange_partition_name.empty() || PARTITION_LEVEL_ZERO == exchange_partition_level)) { + if (OB_UNLIKELY((base_tablet_ids.count() != inc_tablet_ids.count()) || PARTITION_LEVEL_ZERO == exchange_partition_level)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(exchange_partition_name), K(exchange_partition_level)); - } else if (OB_FAIL(check_table_conditions_in_common_(base_table_schema, inc_table_schema, exchange_partition_name, exchange_partition_level, is_oracle_mode))) { - LOG_WARN("fail to check table conditions in common", K(ret), K(base_table_schema), K(inc_table_schema), K(exchange_partition_name), K(exchange_partition_level), K(is_oracle_mode)); + LOG_WARN("invalid argument", K(ret), K(base_tablet_ids.count()), K(inc_tablet_ids.count()), K(exchange_partition_level)); + } else if (OB_FAIL(check_data_table_partitions_and_tablespace_( + base_table_schema, base_tablet_ids, exchange_partition_level))) { + LOG_WARN("failed to check data table partitions and tablespace", + KR(ret), K(base_table_schema), K(base_tablet_ids), K(exchange_partition_level)); + } else if (inc_table_schema.is_partitioned_table() + && OB_FAIL(check_data_table_partitions_and_tablespace_( + inc_table_schema, inc_tablet_ids, exchange_partition_level))) { + LOG_WARN("failed to check data table partitions and tablespace", + KR(ret), K(inc_table_schema), K(inc_tablet_ids), K(exchange_partition_level)); + } else if (OB_FAIL(check_table_conditions_in_common_(base_table_schema, inc_table_schema, exchange_partition_level, is_oracle_mode))) { + LOG_WARN("fail to check table conditions in common", K(ret), + K(base_table_schema), K(inc_table_schema), K(exchange_partition_level), K(is_oracle_mode)); } else if (is_oracle_mode) { if (OB_FAIL(check_table_conditions_in_oracle_mode_(base_table_schema, inc_table_schema))) { LOG_WARN("fail to check table conditions in oracle mode", K(ret), K(base_table_schema), K(inc_table_schema)); @@ -239,18 +288,22 @@ int ObPartitionExchange::check_data_table_partition_exchange_conditions_(const O return ret; } -int ObPartitionExchange::check_table_conditions_in_common_(const ObTableSchema &base_table_schema, const ObTableSchema &inc_table_schema, const ObString &exchange_partition_name, const ObPartitionLevel exchange_partition_level, const bool is_oracle_mode) +int ObPartitionExchange::check_table_conditions_in_common_( + const ObTableSchema &base_table_schema, + const ObTableSchema &inc_table_schema, + const ObPartitionLevel exchange_partition_level, + const bool is_oracle_mode) { int ret = OB_SUCCESS; bool is_base_table_column_store = false; bool is_inc_table_column_store = false; bool is_equal = false; - if (OB_UNLIKELY(exchange_partition_name.empty() || PARTITION_LEVEL_ZERO == exchange_partition_level)) { + if (OB_UNLIKELY(PARTITION_LEVEL_ZERO == exchange_partition_level)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(exchange_partition_name), K(exchange_partition_level)); + LOG_WARN("invalid argument", K(ret), K(exchange_partition_level)); } else if (OB_UNLIKELY(!base_table_schema.check_can_do_ddl() || !inc_table_schema.check_can_do_ddl())) { ret = OB_NOT_SUPPORTED; - LOG_WARN("offline ddl is being executed, other ddl operations are not allowed", K(ret), K(base_table_schema.check_can_do_ddl()), K(inc_table_schema.check_can_do_ddl())); + LOG_WARN("offline ddl is being executed, other ddl operations are not allowed", K(ret), K(base_table_schema.check_can_do_ddl()), K(inc_table_schema.check_can_do_ddl()), K(base_table_schema), K(inc_table_schema)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "execute ddl while other ddl operations are executing long running ddl"); } else if (OB_UNLIKELY(base_table_schema.get_tenant_id() != inc_table_schema.get_tenant_id())) { ret = OB_OP_NOT_ALLOW; @@ -267,8 +320,8 @@ int ObPartitionExchange::check_table_conditions_in_common_(const ObTableSchema & LOG_USER_ERROR(OB_OP_NOT_ALLOW, "exchange partition in duplicate tables"); } else if (OB_UNLIKELY(base_table_schema.is_aux_table() != inc_table_schema.is_aux_table())) { LOG_WARN("aux table attribute of exchanging partition tables are not equal", K(ret), K(base_table_schema.is_aux_table()), K(inc_table_schema.is_aux_table())); - } else if (OB_FAIL(check_partition_and_table_tablespace_(base_table_schema, inc_table_schema, exchange_partition_name, exchange_partition_level, is_oracle_mode))) { - LOG_WARN("fail to check partition and table tablespace", K(ret), K(base_table_schema), K(inc_table_schema), K(exchange_partition_name), K(exchange_partition_level), K(is_oracle_mode)); + } else if (OB_FAIL(check_tablespace_(base_table_schema, inc_table_schema, is_oracle_mode))) { + LOG_WARN("fail to check tablespace ", K(ret), K(base_table_schema), K(inc_table_schema), K(is_oracle_mode)); } else if (OB_FAIL(check_table_rowkey_infos_(base_table_schema, inc_table_schema, is_oracle_mode))) { LOG_WARN("fail to check table rowkey infos", K(ret), K(base_table_schema), K(inc_table_schema), K(is_oracle_mode)); } else if (OB_FAIL(check_table_index_infos_(base_table_schema, inc_table_schema, is_oracle_mode))) { @@ -355,7 +408,7 @@ int ObPartitionExchange::check_table_conditions_in_common_(const ObTableSchema & } if (OB_SUCC(ret) && !is_equal) { ret = OB_TABLES_DIFFERENT_DEFINITIONS; - LOG_WARN("table conditions in common of exchange tables are not equal", K(ret), K(base_table_schema), K(inc_table_schema), K(exchange_partition_name), K(exchange_partition_level)); + LOG_WARN("table conditions in common of exchange tables are not equal", K(ret), K(base_table_schema), K(inc_table_schema), K(exchange_partition_level)); } return ret; } @@ -777,49 +830,13 @@ int ObPartitionExchange::compare_default_value_(ObObj &l_value, ObObj &r_value, return ret; } -int ObPartitionExchange::check_partition_and_table_tablespace_(const ObTableSchema &base_table_schema, const ObTableSchema &inc_table_schema, const ObString &exchange_partition_name, const ObPartitionLevel exchange_partition_level, const bool is_oracle_mode) +int ObPartitionExchange::check_tablespace_( + const ObTableSchema &base_table_schema, + const ObTableSchema &inc_table_schema, + const bool is_oracle_mode) { int ret = OB_SUCCESS; - const ObPartition *part = nullptr; - const ObSubPartition *subpart = nullptr; - bool is_equal = false; - if (OB_UNLIKELY(exchange_partition_name.empty() || (PARTITION_LEVEL_ZERO == exchange_partition_level))) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(exchange_partition_name), K(exchange_partition_level)); - } else if (OB_UNLIKELY(base_table_schema.get_tablespace_id() != inc_table_schema.get_tablespace_id())) { - LOG_WARN("tablespace id of exchanging tables are not equal", K(ret), K(base_table_schema.get_tablespace_id()), K(inc_table_schema.get_tablespace_id())); - } else if (!base_table_schema.is_aux_table() && !inc_table_schema.is_aux_table()) { - if (PARTITION_LEVEL_ONE == exchange_partition_level) { - if (OB_FAIL(base_table_schema.get_partition_by_name(exchange_partition_name, part))) { - LOG_WARN("get part by name failed", K(ret), K(exchange_partition_name), K(base_table_schema)); - } else if (OB_ISNULL(part)) { - ret = OB_PARTITION_NOT_EXIST; - LOG_WARN("partition not found", K(ret), K(exchange_partition_name), K(base_table_schema)); - } else if (OB_UNLIKELY(OB_INVALID_INDEX != part->get_tablespace_id())) { - ret = OB_NOT_SUPPORTED; - LOG_USER_ERROR(OB_NOT_SUPPORTED, "When the partition contains a tablespace, partition exchange is not"); - LOG_WARN("when the partition contains a tablespace, partition exchange is not supported", K(ret), K(part->get_tablespace_id()), K(inc_table_schema.get_tablespace_id())); - } else { - is_equal = true; - } - } else if (PARTITION_LEVEL_TWO == exchange_partition_level) { - if (OB_FAIL(base_table_schema.get_subpartition_by_name(exchange_partition_name, part, subpart))) { - LOG_WARN("get sub part by name failed", K(ret), K(exchange_partition_name), K(base_table_schema)); - } else if (OB_ISNULL(part) || OB_ISNULL(subpart)) { - ret = OB_PARTITION_NOT_EXIST; - LOG_WARN("partition not found", K(ret), K(exchange_partition_name), K(base_table_schema), K(part), K(subpart)); - } else if (OB_UNLIKELY(OB_INVALID_INDEX != subpart->get_tablespace_id())) { - ret = OB_NOT_SUPPORTED; - LOG_USER_ERROR(OB_NOT_SUPPORTED, "When the partition contains a tablespace, partition exchange is not"); - LOG_WARN("when the subpartition contains a tablespace, partition exchange is not supported", K(ret), K(subpart->get_tablespace_id()), K(inc_table_schema.get_tablespace_id())); - } else { - is_equal = true; - } - } - } else if (base_table_schema.is_aux_table() && inc_table_schema.is_aux_table()) { - is_equal = true; - } - if (OB_SUCC(ret) && !is_equal) { + if (OB_UNLIKELY(base_table_schema.get_tablespace_id() != inc_table_schema.get_tablespace_id())) { if (is_oracle_mode) { ret = OB_NOT_SUPPORTED; LOG_USER_ERROR(OB_NOT_SUPPORTED, "exchange partition in different tablespaces"); @@ -827,7 +844,39 @@ int ObPartitionExchange::check_partition_and_table_tablespace_(const ObTableSche ret = OB_ERR_PARTITION_EXCHANGE_DIFFERENT_OPTION; LOG_USER_ERROR(OB_ERR_PARTITION_EXCHANGE_DIFFERENT_OPTION, "TABLESPACE"); } - LOG_WARN("tablespace id are not equal between partition and table", K(ret), K(base_table_schema), K(inc_table_schema), K(exchange_partition_name), K(exchange_partition_level)); + LOG_WARN("tablespace id of exchanging tables are not equal", + KR(ret), K(base_table_schema.get_tablespace_id()), K(inc_table_schema.get_tablespace_id())); + } + return ret; +} + +int ObPartitionExchange::check_data_table_partitions_and_tablespace_( + const ObTableSchema &table_schema, + const ObIArray &tablet_ids, + const ObPartitionLevel exchange_partition_level) +{ + int ret = OB_SUCCESS; + const ObPartition *part = nullptr; + const ObSubPartition *subpart = nullptr; + if (OB_UNLIKELY(PARTITION_LEVEL_ZERO == exchange_partition_level)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(exchange_partition_level)); + } else if (!table_schema.is_aux_table()) { + const bool is_subpart = (PARTITION_LEVEL_TWO == exchange_partition_level); + for (int64_t i = 0; OB_SUCC(ret) && (i < tablet_ids.count()); ++i) { + int64_t part_tablespace_id = OB_INVALID_INDEX; + const ObTabletID &tablet_id = tablet_ids.at(i); + if (OB_FAIL(get_part_by_tablet_id(table_schema, tablet_id, part, subpart, is_subpart))) { + LOG_WARN("failed to get part by tablet id", KR(ret), K(table_schema), K(tablet_id), K(is_subpart)); + } else if (OB_FALSE_IT(part_tablespace_id = is_subpart ? + subpart->get_tablespace_id() : part->get_tablespace_id())) { + } else if (OB_UNLIKELY(OB_INVALID_INDEX != part->get_tablespace_id())) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "When the partition contains a tablespace, partition exchange is not"); + LOG_WARN("when the partition contains a tablespace, partition exchange is not supported", + KR(ret), K(part_tablespace_id)); + } + } } return ret; } @@ -1134,12 +1183,9 @@ int ObPartitionExchange::set_global_storage_index_unusable_(const uint64_t tenan return ret; } -int ObPartitionExchange::get_data_partition_and_index_(const ObTableSchema &partitioned_data_table_schema, const ObString &data_part_name, const ObPartition *&data_part, int64_t &data_partition_index) +int ObPartitionExchange::get_and_check_data_partition_by_name(const ObTableSchema &partitioned_data_table_schema, const ObString &data_part_name, const ObPartition *&data_part) { int ret = OB_SUCCESS; - int64_t part_id = 0; - data_partition_index = OB_INVALID_INDEX; - ObCheckPartitionMode check_partition_mode = CHECK_PARTITION_MODE_NORMAL; if (OB_UNLIKELY(!partitioned_data_table_schema.is_valid() || data_part_name.empty())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(partitioned_data_table_schema.is_valid()), K(data_part_name)); @@ -1148,26 +1194,19 @@ int ObPartitionExchange::get_data_partition_and_index_(const ObTableSchema &part } else if (OB_ISNULL(data_part)) { ret = OB_PARTITION_NOT_EXIST; LOG_WARN("partition not found", K(ret), K(data_part_name), K(partitioned_data_table_schema)); - } else if (FALSE_IT(part_id = data_part->get_part_id())) { - } else if (OB_FAIL(partitioned_data_table_schema.get_partition_index_by_id(part_id, check_partition_mode, data_partition_index))) { - LOG_WARN("fail to get partition index by id", K(ret), K(partitioned_data_table_schema), K(part_id)); + } else if (OB_INVALID_INDEX == data_part->get_part_idx()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid partition index", K(ret), KPC(data_part)); } return ret; } -int ObPartitionExchange::get_data_subpartition_and_index_(const ObTableSchema &partitioned_data_table_schema, +int ObPartitionExchange::get_and_check_data_subpartition_by_name(const ObTableSchema &partitioned_data_table_schema, const ObString &data_subpart_name, const ObPartition *&data_part, - const ObSubPartition *&data_subpart, - int64_t &data_partition_index, - int64_t &data_subpartition_index) + const ObSubPartition *&data_subpart) { int ret = OB_SUCCESS; - int64_t part_id = 0; - int64_t subpart_id = 0; - data_partition_index = OB_INVALID_INDEX; - data_subpartition_index = OB_INVALID_INDEX; - ObCheckPartitionMode check_partition_mode = CHECK_PARTITION_MODE_NORMAL; if (OB_UNLIKELY(!partitioned_data_table_schema.is_valid() || data_subpart_name.empty())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(partitioned_data_table_schema.is_valid()), K(data_subpart_name)); @@ -1176,505 +1215,592 @@ int ObPartitionExchange::get_data_subpartition_and_index_(const ObTableSchema &p } else if (OB_ISNULL(data_part) || OB_ISNULL(data_subpart)) { ret = OB_PARTITION_NOT_EXIST; LOG_WARN("partition not found", K(ret), K(partitioned_data_table_schema), K(data_subpart_name)); - } else if (FALSE_IT(part_id = data_part->get_part_id())) { - } else if (OB_FAIL(partitioned_data_table_schema.get_partition_index_by_id(part_id, check_partition_mode, data_partition_index))) { - LOG_WARN("fail to get partition index", K(ret), K(partitioned_data_table_schema), K(part_id)); - } else if (FALSE_IT(subpart_id = data_subpart->get_sub_part_id())) { - } else if (OB_FAIL(data_part->get_normal_subpartition_index_by_id(subpart_id, data_subpartition_index))) { - LOG_WARN("fail to get normal subpartition index by id", K(ret), KPC(data_part), K(subpart_id)); + } else if ((OB_INVALID_INDEX == data_part->get_part_idx()) || (OB_INVALID_INDEX == data_subpart->get_sub_part_idx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid subpartition index", K(ret), KPC(data_part), KPC(data_subpart)); } return ret; } int ObPartitionExchange::exchange_data_table_partition_(const uint64_t tenant_id, const ObTableSchema &partitioned_table_schema, - const ObTableSchema &non_partitioned_table_schema, - const ObPartition &part, + const ObTableSchema &inc_table_schema, + const common::ObTabletID &tablet_id, + const common::ObTabletID &inc_tablet_id, const bool is_oracle_mode, + const bool is_subpartition, ObDDLOperator &ddl_operator, ObDDLSQLTransaction &trans, ObSchemaGetterGuard &schema_guard) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || !partitioned_table_schema.is_valid() || !non_partitioned_table_schema.is_valid() || !part.is_valid() || !used_table_to_tablet_id_map_.created())) { + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObPartitionExchange not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || !partitioned_table_schema.is_valid() || !inc_table_schema.is_valid() || !tablet_id.is_valid() || !inc_tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(tenant_id), K(partitioned_table_schema.is_valid()), K(non_partitioned_table_schema.is_valid()), K(part.is_valid()), K(used_table_to_tablet_id_map_.created())); - } else if (OB_FAIL(used_table_to_tablet_id_map_.set_refactored(partitioned_table_schema.get_table_id(), part.get_tablet_id()))) { - LOG_WARN("fail to set refactored used table to tablet id map", K(ret), K(partitioned_table_schema.get_table_id()), K(part.get_tablet_id())); - } else if (OB_FAIL(used_table_to_tablet_id_map_.set_refactored(non_partitioned_table_schema.get_table_id(), non_partitioned_table_schema.get_tablet_id()))) { - LOG_WARN("fail to set refactored used table to tablet id map", K(ret), K(non_partitioned_table_schema.get_table_id()), K(non_partitioned_table_schema.get_tablet_id())); - } else if (OB_FAIL(exchange_partition_map_relationship_(tenant_id, - part, - partitioned_table_schema, - non_partitioned_table_schema, - is_oracle_mode, - ddl_operator, - trans, - schema_guard))) { - LOG_WARN("fail to exchange partition map relationship", K(ret), K(tenant_id), K(part), K(partitioned_table_schema), K(non_partitioned_table_schema), K(is_oracle_mode)); + LOG_WARN("invalid argument", K(ret), K(tenant_id), K(partitioned_table_schema.is_valid()), K(inc_table_schema.is_valid()), K(tablet_id.is_valid()), K(inc_tablet_id.is_valid())); + } else if (OB_FAIL(add_table_to_tablet_ids_map(partitioned_table_schema.get_table_id(), tablet_id))) { + LOG_WARN("failed to add table to tablet ids map", KR(ret), + K(partitioned_table_schema.get_table_id()), K(tablet_id)); + } else if (OB_FAIL(add_table_to_tablet_ids_map(inc_table_schema.get_table_id(), inc_tablet_id))) { + LOG_WARN("failed to add table to tablet ids map", KR(ret), + K(inc_table_schema.get_table_id()), K(inc_tablet_id)); + } else { + ObArray base_tablet_ids; + ObArray inc_tablet_ids; + OZ (base_tablet_ids.push_back(tablet_id)); + OZ (inc_tablet_ids.push_back(inc_tablet_id)); + if (OB_FAIL(ret)) { + } else if (OB_FAIL(exchange_partition_map_relationship_(tenant_id, + partitioned_table_schema, + inc_table_schema, + base_tablet_ids, + inc_tablet_ids, + is_oracle_mode, + is_subpartition, + ddl_operator, + trans, + schema_guard))) { + LOG_WARN("fail to exchange partition map relationship", K(ret), K(tenant_id), K(base_tablet_ids), K(inc_tablet_ids), K(partitioned_table_schema), K(inc_table_schema), K(is_oracle_mode)); + } } return ret; } -int ObPartitionExchange::exchange_data_table_subpartition_(const uint64_t tenant_id, - const ObTableSchema &partitioned_table_schema, - const ObTableSchema &non_partitioned_table_schema, - const ObPartition &part, - const ObSubPartition &subpart, - const bool is_oracle_mode, - ObDDLOperator &ddl_operator, - ObDDLSQLTransaction &trans, - ObSchemaGetterGuard &schema_guard) +int ObPartitionExchange::exchange_data_table_partitions(const uint64_t tenant_id, + const ObTableSchema &base_table_schema, + const ObTableSchema &inc_table_schema, + const ObIArray &base_tablet_ids, + const ObIArray &inc_tablet_ids, + const bool is_oracle_mode, + const bool is_subpartition, + ObDDLOperator &ddl_operator, + ObDDLSQLTransaction &trans, + ObSchemaGetterGuard &schema_guard) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || !partitioned_table_schema.is_valid() || !non_partitioned_table_schema.is_valid() || !part.is_valid() || !subpart.is_valid() || !used_table_to_tablet_id_map_.created())) { + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObPartitionExchange not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || !base_table_schema.is_valid() || !inc_table_schema.is_valid() || base_tablet_ids.count() != inc_tablet_ids.count())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(tenant_id), K(partitioned_table_schema.is_valid()), K(non_partitioned_table_schema.is_valid()), K(part.is_valid()), K(subpart.is_valid()), K(used_table_to_tablet_id_map_.created())); - } else if (OB_FAIL(used_table_to_tablet_id_map_.set_refactored(partitioned_table_schema.get_table_id(), subpart.get_tablet_id()))) { - LOG_WARN("fail to set refactored used table to tablet id map", K(ret), K(partitioned_table_schema.get_table_id()), K(subpart.get_tablet_id())); - } else if (OB_FAIL(used_table_to_tablet_id_map_.set_refactored(non_partitioned_table_schema.get_table_id(), non_partitioned_table_schema.get_tablet_id()))) { - LOG_WARN("fail to set refactored used table to tablet id map", K(ret), K(non_partitioned_table_schema.get_table_id()), K(non_partitioned_table_schema.get_tablet_id())); - } else if (OB_FAIL(exchange_subpartition_map_relationship_(tenant_id, - part, - subpart, - partitioned_table_schema, - non_partitioned_table_schema, - is_oracle_mode, - ddl_operator, - trans, - schema_guard))) { - LOG_WARN("fail to exchange subpartition map relationship", K(ret), K(tenant_id), K(part), K(subpart), K(partitioned_table_schema), K(non_partitioned_table_schema), K(is_oracle_mode)); + LOG_WARN("invalid argument", K(ret), K(tenant_id), K(base_table_schema.is_valid()), K(inc_table_schema.is_valid()), K(base_tablet_ids.count()), K(inc_tablet_ids.count())); + } else if (OB_FAIL(add_table_to_tablet_ids_map(base_table_schema.get_table_id(), base_tablet_ids))) { + LOG_WARN("failed to add table to tablet ids map", KR(ret), + K(base_table_schema.get_table_id()), K(base_tablet_ids)); + } else if (OB_FAIL(add_table_to_tablet_ids_map(inc_table_schema.get_table_id(), inc_tablet_ids))) { + LOG_WARN("failed to add table to tablet ids map", KR(ret), + K(inc_table_schema.get_table_id()), K(inc_tablet_ids)); + } else if (OB_FAIL(exchange_partition_map_relationship_(tenant_id, + base_table_schema, + inc_table_schema, + base_tablet_ids, + inc_tablet_ids, + is_oracle_mode, + is_subpartition, + ddl_operator, + trans, + schema_guard))) { + LOG_WARN("fail to exchange partition map relationship", K(ret), K(tenant_id), K(base_tablet_ids), K(inc_tablet_ids), K(base_table_schema), K(inc_table_schema), K(is_oracle_mode)); } return ret; } int ObPartitionExchange::exchange_auxiliary_table_partition_(const uint64_t tenant_id, - const int64_t ori_data_partition_index, - const ObPartition &ori_data_part, + const ObTableSchema &base_data_table_schema, + const ObTableSchema &inc_data_table_schema, + const ObTabletID &data_tablet_id, + const ObTabletID &inc_data_tablet_id, const bool is_oracle_mode, + const bool is_subpartition, ObDDLOperator &ddl_operator, ObDDLSQLTransaction &trans, ObSchemaGetterGuard &schema_guard) { int ret = OB_SUCCESS; - const ObPartition *part = nullptr; - const ObTableSchema *partitioned_table_schema = NULL; - const ObTableSchema *non_partitioned_table_schema = NULL; - bool is_matched = false; - ObCheckPartitionMode check_partition_mode = CHECK_PARTITION_MODE_NORMAL; - if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || OB_INVALID_INDEX == ori_data_partition_index || !ori_data_part.is_valid() || !used_pt_nt_id_map_.created() || !used_table_to_tablet_id_map_.created())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(tenant_id), K(ori_data_partition_index), K(ori_data_part.is_valid()), K(is_oracle_mode), K(used_pt_nt_id_map_.created()), K(used_table_to_tablet_id_map_.created())); - } else { - common::hash::ObHashMap::iterator iter; - for (iter = used_pt_nt_id_map_.begin(); OB_SUCC(ret) && iter != used_pt_nt_id_map_.end(); ++iter) { - uint64_t partitioned_table_id = iter->first; - uint64_t non_partitioned_table_id = iter->second; - if (OB_FAIL(schema_guard.get_table_schema(tenant_id, partitioned_table_id, partitioned_table_schema))) { - LOG_WARN("get table schema failed", K(ret), K(tenant_id), K(partitioned_table_id)); - } else if (OB_ISNULL(partitioned_table_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("table schema should not be null", K(ret), K(tenant_id), K(partitioned_table_id)); - } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, non_partitioned_table_id, non_partitioned_table_schema))) { - LOG_WARN("get table schema failed", K(ret), K(tenant_id), K(non_partitioned_table_id)); - } else if (OB_ISNULL(non_partitioned_table_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("table schema should not be null", K(ret), K(tenant_id), K(non_partitioned_table_id)); - } else { - const schema::ObPartitionOption &pt_part_option = partitioned_table_schema->get_part_option(); - schema::ObPartitionFuncType pt_part_func_type = pt_part_option.get_part_func_type(); - if (OB_FAIL(partitioned_table_schema->get_partition_by_partition_index(ori_data_partition_index, check_partition_mode, part))) { - LOG_WARN("get_partition_by_partition_index fail", K(ret), K(ori_data_partition_index), KPC(partitioned_table_schema)); - } else if (OB_ISNULL(part)) { - ret = OB_PARTITION_NOT_EXIST; - LOG_WARN("partition not found", K(ret), K(ori_data_partition_index), KPC(partitioned_table_schema)); - } else if (OB_FAIL(ddl_service_.check_same_partition(is_oracle_mode, ori_data_part, *part, pt_part_func_type, is_matched))) { - LOG_WARN("fail to check ori_table_part and ori_aux_part is the same", K(ret), K(is_oracle_mode), K(ori_data_part), KPC(part), K(pt_part_func_type)); - } else if (OB_UNLIKELY(!is_matched)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("part with the same offset not equal, maybe not the right index", K(ret), K(ori_data_part), KPC(part)); - } else if (OB_FAIL(used_table_to_tablet_id_map_.set_refactored(partitioned_table_id, part->get_tablet_id()))) { - LOG_WARN("fail to set refactored used table to tablet id map", K(ret), K(partitioned_table_id), K(part->get_tablet_id())); - } else if (OB_FAIL(used_table_to_tablet_id_map_.set_refactored(non_partitioned_table_id, non_partitioned_table_schema->get_tablet_id()))) { - LOG_WARN("fail to set refactored used table to tablet id map", K(ret), K(non_partitioned_table_id), K(non_partitioned_table_schema->get_tablet_id())); - } else if (OB_FAIL(exchange_partition_map_relationship_(tenant_id, - *part, - *partitioned_table_schema, - *non_partitioned_table_schema, - is_oracle_mode, - ddl_operator, - trans, - schema_guard))) { - LOG_WARN("fail to exchange partition map relationship", K(ret), K(tenant_id), KPC(part), KPC(partitioned_table_schema), KPC(non_partitioned_table_schema), K(is_oracle_mode)); - } - } - } + ObArray data_tablet_ids; + ObArray inc_data_tablet_ids; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObPartitionExchange not init", KR(ret), KP(this)); + } else if (OB_FAIL(data_tablet_ids.push_back(data_tablet_id))) { + LOG_WARN("failed push back data tablet id", KR(ret), K(data_tablet_id)); + } else if (OB_FAIL(inc_data_tablet_ids.push_back(inc_data_tablet_id))) { + LOG_WARN("failed push back inc data tablet id", KR(ret), K(inc_data_tablet_id)); + } else if OB_FAIL(exchange_auxiliary_table_partitions(tenant_id, + base_data_table_schema, + inc_data_table_schema, + data_tablet_ids, + inc_data_tablet_ids, + is_oracle_mode, + is_subpartition, + ddl_operator, + trans, + schema_guard)) { + LOG_WARN("failed to exchange auxiliary table partitions", KR(ret)); } return ret; } -int ObPartitionExchange::exchange_auxiliary_table_subpartition_(const uint64_t tenant_id, - const int64_t ori_data_partition_index, - const int64_t ori_data_subpartition_index, - const ObPartition &ori_data_part, - const ObSubPartition &ori_data_subpart, - const bool is_oracle_mode, - ObDDLOperator &ddl_operator, - ObDDLSQLTransaction &trans, - ObSchemaGetterGuard &schema_guard) +int ObPartitionExchange::exchange_auxiliary_table_partitions(const uint64_t tenant_id, + const ObTableSchema &base_data_table_schema, + const ObTableSchema &inc_data_table_schema, + const ObIArray &data_tablet_ids, + const ObIArray &inc_data_tablet_ids, + const bool is_oracle_mode, + const bool is_subpartition, + ObDDLOperator &ddl_operator, + ObDDLSQLTransaction &trans, + ObSchemaGetterGuard &schema_guard) { int ret = OB_SUCCESS; - const ObPartition *part = nullptr; - const ObSubPartition *subpart = nullptr; - const ObTableSchema *partitioned_table_schema = NULL; - const ObTableSchema *non_partitioned_table_schema = NULL; - bool is_matched = false; - ObCheckPartitionMode check_partition_mode = CHECK_PARTITION_MODE_NORMAL; - if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || OB_INVALID_INDEX == ori_data_partition_index || - OB_INVALID_INDEX == ori_data_subpartition_index || !ori_data_part.is_valid() || !ori_data_subpart.is_valid() || !used_table_to_tablet_id_map_.created())) { + bool is_inc_table_partitioned = inc_data_table_schema.is_partitioned_table(); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObPartitionExchange not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || data_tablet_ids.count() != inc_data_tablet_ids.count())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(tenant_id), K(ori_data_partition_index), K(ori_data_subpartition_index), K(ori_data_part.is_valid()), K(ori_data_subpart.is_valid()), K(used_table_to_tablet_id_map_.created())); + LOG_WARN("invalid argument", K(ret), K(tenant_id), K(data_tablet_ids.count()), K(inc_data_tablet_ids.count()), K(is_oracle_mode)); } else { common::hash::ObHashMap::iterator iter; for (iter = used_pt_nt_id_map_.begin(); OB_SUCC(ret) && iter != used_pt_nt_id_map_.end(); ++iter) { - uint64_t partitioned_table_id = iter->first; - uint64_t non_partitioned_table_id = iter->second; - if (OB_FAIL(schema_guard.get_table_schema(tenant_id, partitioned_table_id, partitioned_table_schema))) { - LOG_WARN("get table schema failed", K(ret), K(tenant_id), K(partitioned_table_id)); - } else if (OB_ISNULL(partitioned_table_schema)) { + uint64_t base_table_id = iter->first; + uint64_t inc_table_id = iter->second; + const ObTableSchema *base_table_schema = NULL; + const ObTableSchema *inc_table_schema = NULL; + if (OB_FAIL(schema_guard.get_table_schema(tenant_id, base_table_id, base_table_schema))) { + LOG_WARN("get table schema failed", K(ret), K(tenant_id), K(base_table_id)); + } else if (OB_ISNULL(base_table_schema)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("table schema should not be null", K(ret), K(tenant_id), K(partitioned_table_id)); - } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, non_partitioned_table_id, non_partitioned_table_schema))) { - LOG_WARN("get table schema failed", K(ret), K(tenant_id), K(non_partitioned_table_id)); - } else if (OB_ISNULL(non_partitioned_table_schema)) { + LOG_WARN("table schema should not be null", K(ret), K(tenant_id), K(base_table_id)); + } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, inc_table_id, inc_table_schema))) { + LOG_WARN("get table schema failed", K(ret), K(tenant_id), K(inc_table_id)); + } else if (OB_ISNULL(inc_table_schema)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("table schema should not be null", K(ret), K(tenant_id), K(non_partitioned_table_id)); + LOG_WARN("table schema should not be null", K(ret), K(tenant_id), K(inc_table_id)); } else { - const schema::ObPartitionOption &pt_part_option = partitioned_table_schema->get_part_option(); - schema::ObPartitionFuncType pt_part_func_type = pt_part_option.get_part_func_type(); - const schema::ObPartitionOption &pt_subpart_option = partitioned_table_schema->get_sub_part_option(); - schema::ObPartitionFuncType pt_subpart_func_type = pt_subpart_option.get_sub_part_func_type(); - if (OB_FAIL(partitioned_table_schema->get_partition_by_partition_index(ori_data_partition_index, check_partition_mode, part))) { - LOG_WARN("get_partition_by_partition_index fail", K(ret), K(ori_data_partition_index), KPC(partitioned_table_schema)); - } else if (OB_ISNULL(part)) { - ret = OB_PARTITION_NOT_EXIST; - LOG_WARN("partition not found", K(ret), K(ori_data_partition_index), KPC(partitioned_table_schema)); - } else if (OB_FAIL(ddl_service_.check_same_partition(is_oracle_mode, ori_data_part, *part, pt_part_func_type, is_matched))) { - LOG_WARN("fail to check ori_table_part and ori_aux_part is the same", K(ret), K(is_oracle_mode), K(ori_data_part), KPC(part), K(pt_part_func_type)); - } else if (!is_matched) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("part with the same offset not equal, maybe not the right index", K(ret), K(ori_data_part), KPC(part)); - } else if (OB_FAIL(part->get_normal_subpartition_by_subpartition_index(ori_data_subpartition_index, subpart))) { - LOG_WARN("fail to get src subpart by subpart index", K(ret), K(ori_data_subpartition_index)); - } else if (OB_ISNULL(subpart)) { - ret = OB_PARTITION_NOT_EXIST; - LOG_WARN("partition not found", K(ret), K(part), K(ori_data_subpartition_index), KPC(partitioned_table_schema)); - } else if (OB_FAIL(ddl_service_.check_same_subpartition(is_oracle_mode, ori_data_subpart, *subpart, pt_subpart_func_type, is_matched))) { - LOG_WARN("fail to check ori_table_subpart and ori_aux_subpart is the same", K(ret), K(is_oracle_mode), K(ori_data_subpart), KPC(subpart), K(pt_subpart_func_type)); - } else if (!is_matched) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("part with the same offset not equal, maybe not the right index", K(ret), K(ori_data_subpart), KPC(subpart)); - } else if (OB_FAIL(used_table_to_tablet_id_map_.set_refactored(partitioned_table_id, subpart->get_tablet_id()))) { - LOG_WARN("fail to set refactored used table to tablet id map", K(ret), K(partitioned_table_id), K(subpart->get_tablet_id())); - } else if (OB_FAIL(used_table_to_tablet_id_map_.set_refactored(non_partitioned_table_id, non_partitioned_table_schema->get_tablet_id()))) { - LOG_WARN("fail to set refactored used table to tablet id map", K(ret), K(non_partitioned_table_id), K(non_partitioned_table_schema->get_tablet_id())); - } else if (OB_FAIL(exchange_subpartition_map_relationship_(tenant_id, - *part, - *subpart, - *partitioned_table_schema, - *non_partitioned_table_schema, - is_oracle_mode, - ddl_operator, - trans, - schema_guard))) { - LOG_WARN("fail to exchange subpartition map relationship", K(ret), K(tenant_id), KPC(part), KPC(subpart), KPC(partitioned_table_schema), KPC(non_partitioned_table_schema), K(is_oracle_mode)); + ObArray base_tablet_ids; + ObArray inc_tablet_ids; + for (int64_t i = 0; OB_SUCC(ret) && (i < data_tablet_ids.count()); ++i) { + const ObTabletID &data_tablet_id = data_tablet_ids.at(i); + const ObTabletID &inc_data_tablet_id = inc_data_tablet_ids.at(i); + const ObPartition *base_data_part = nullptr; + const ObPartition *inc_data_part = nullptr; + const ObSubPartition *base_data_subpart = nullptr; + const ObSubPartition *inc_data_subpart = nullptr; + ObTabletID tablet_id; + ObTabletID inc_tablet_id; + + if (OB_FAIL(get_part_by_tablet_id(base_data_table_schema, data_tablet_id, base_data_part, base_data_subpart, is_subpartition))) { + LOG_WARN("failed to get part by tablet id", KR(ret), K(base_data_table_schema), K(data_tablet_id)); + } else if (is_inc_table_partitioned + && OB_FAIL(get_part_by_tablet_id(inc_data_table_schema, inc_data_tablet_id, inc_data_part, inc_data_subpart, is_subpartition))) { + LOG_WARN("failed to get part by tablet id", KR(ret), K(inc_data_table_schema), K(inc_data_tablet_id)); + } else if (OB_FAIL(get_and_check_aux_tablet_id(base_data_part, base_data_subpart, *base_table_schema, is_oracle_mode, is_subpartition, tablet_id))) { + LOG_WARN("failed to get and check aux tablet id", + KR(ret), K(is_oracle_mode), KPC(base_data_part), KPC(base_table_schema)); + } else if (is_inc_table_partitioned) { + if (OB_FAIL(get_and_check_aux_tablet_id(inc_data_part, inc_data_subpart, *inc_table_schema, is_oracle_mode, is_subpartition, inc_tablet_id))) { + LOG_WARN("failed to get and check aux tablet id", + KR(ret), K(is_oracle_mode), KPC(inc_data_part), KPC(inc_table_schema)); + } + } else { + inc_tablet_id = inc_table_schema->get_tablet_id(); + LOG_INFO("partition_exchange exchange_auxiliary_table_partition_", K(tablet_id), K(inc_tablet_id), + K(tablet_id.is_valid()), K(inc_tablet_id.is_valid())); + } + + if (OB_SUCC(ret)) { + OZ (base_tablet_ids.push_back(tablet_id)); + OZ (inc_tablet_ids.push_back(inc_tablet_id)); + } + } // end for + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(add_table_to_tablet_ids_map(base_table_id, base_tablet_ids))) { + LOG_WARN("failed to add table to tablet ids map", KR(ret), + K(base_table_id), K(base_tablet_ids)); + } else if (OB_FAIL(add_table_to_tablet_ids_map(inc_table_id, inc_tablet_ids))) { + LOG_WARN("failed to add table to tablet ids map", KR(ret), + K(inc_table_id), K(inc_tablet_ids)); + } else if (OB_FAIL(exchange_partition_map_relationship_(tenant_id, + *base_table_schema, + *inc_table_schema, + base_tablet_ids, + inc_tablet_ids, + is_oracle_mode, + is_subpartition, + ddl_operator, + trans, + schema_guard))) { + LOG_WARN("fail to exchange partition map relationship", + K(ret), K(tenant_id), KPC(base_table_schema), KPC(inc_table_schema), + K(base_tablet_ids), K(inc_tablet_ids), K(is_oracle_mode)); } - } - } - } + } // end if + } // end for + } // end if return ret; } int ObPartitionExchange::exchange_partition_map_relationship_(const uint64_t tenant_id, - const ObPartition &part, - const ObTableSchema &partitioned_table_schema, - const ObTableSchema &non_partitioned_table_schema, + const ObTableSchema &base_table_schema, + const ObTableSchema &inc_table_schema, + const ObIArray &base_tablet_ids, + const ObIArray &inc_tablet_ids, const bool is_oracle_mode, + const bool is_subpartition, ObDDLOperator &ddl_operator, ObDDLSQLTransaction &trans, ObSchemaGetterGuard &schema_guard) { int ret = OB_SUCCESS; - ObArray exchange_table_ids; - ObArray exchange_tablet_ids; - if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || !part.is_valid() || !partitioned_table_schema.is_valid() || !non_partitioned_table_schema.is_valid())) { + if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || base_tablet_ids.count() != inc_tablet_ids.count() || !base_table_schema.is_valid() || !inc_table_schema.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(tenant_id), K(part), K(partitioned_table_schema.is_valid()), K(non_partitioned_table_schema.is_valid())); - } else if (OB_FAIL(exchange_table_ids.push_back(non_partitioned_table_schema.get_table_id()))) { - LOG_WARN("fail to push back table id", K(ret), K(non_partitioned_table_schema.get_table_id()), K(exchange_table_ids), K(non_partitioned_table_schema)); - } else if (OB_FAIL(exchange_table_ids.push_back(partitioned_table_schema.get_table_id()))) { - LOG_WARN("fail to push back table id", K(ret), K(partitioned_table_schema.get_table_id()), K(exchange_table_ids), K(partitioned_table_schema)); - } else if (OB_FAIL(exchange_tablet_ids.push_back(part.get_tablet_id()))) { - LOG_WARN("fail to push back partitioned table tablet id", K(ret), K(part.get_tablet_id())); - } else if (OB_FAIL(exchange_tablet_ids.push_back(non_partitioned_table_schema.get_tablet_id()))) { - LOG_WARN("fail to push back non_partitioned table tablet id", K(ret), K(non_partitioned_table_schema.get_tablet_id())); - } else if (OB_UNLIKELY(2 != exchange_table_ids.count() || 2 != exchange_tablet_ids.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("the array length is incorrect", K(ret), K(exchange_table_ids.count()), K(exchange_tablet_ids.count())); + LOG_WARN("invalid argument", K(ret), K(tenant_id), K(base_tablet_ids.count()), K(inc_tablet_ids.count()), K(base_table_schema.is_valid()), K(inc_table_schema.is_valid())); } else { + const bool is_inc_table_partitioned = inc_table_schema.is_partitioned_table(); // drop exchange partition in partitioned table, and add new exchange partition in partitioned table - HEAP_VARS_3((ObPartition, new_part), - (ObTableSchema, new_nt_schema), - (AlterTableSchema, alter_pt_drop_part_schema)) { - if (OB_FAIL(new_part.assign(part))) { - LOG_WARN("fail to assign part", K(ret), K(part)); - } else if (OB_FAIL(alter_pt_drop_part_schema.assign(partitioned_table_schema))) { - LOG_WARN("fail to assign partitioned table schema", K(ret), K(partitioned_table_schema)); - } else if (FALSE_IT(alter_pt_drop_part_schema.reset_partition_schema())) { - } else if (OB_FAIL(alter_pt_drop_part_schema.add_partition(part))) { - LOG_WARN("fail to add partition", K(ret), K(part)); - } else if (FALSE_IT(alter_pt_drop_part_schema.set_part_level(partitioned_table_schema.get_part_level()))) { - } else if (OB_FAIL(alter_pt_drop_part_schema.get_part_option().assign(partitioned_table_schema.get_part_option()))) { - LOG_WARN("fail to assign part option", K(ret), K(partitioned_table_schema.get_part_option())); - } else if (FALSE_IT(alter_pt_drop_part_schema.get_part_option().set_part_num(alter_pt_drop_part_schema.get_partition_num()))) { - } else if (OB_FAIL(new_nt_schema.assign(non_partitioned_table_schema))) { - LOG_WARN("fail to assign non_partitioned table schema", K(ret), K(non_partitioned_table_schema)); - } else { - new_part.set_tablet_id(exchange_tablet_ids.at(1)); - new_nt_schema.set_tablet_id(exchange_tablet_ids.at(0)); + HEAP_VARS_4((AlterTableSchema, alter_pt_drop_part_schema), + (AlterTableSchema, alter_inc_drop_part_schema), + (AlterTableSchema, alter_pt_add_new_part_schema), + (AlterTableSchema, alter_inc_add_new_part_schema)) { + ObArray old_base_part_ids; + ObArray new_base_part_ids; + ObArray old_inc_part_ids; + ObArray new_inc_part_ids; + + if (OB_FAIL(init_alter_table_part_schema(base_table_schema, alter_pt_drop_part_schema))) { + LOG_WARN("failed to init alter table part schema", KR(ret)); + } else if (OB_FAIL(init_alter_table_part_schema(base_table_schema, alter_pt_add_new_part_schema))) { + LOG_WARN("failed to init alter table part schema", KR(ret)); + } else if (is_inc_table_partitioned) { + if (OB_FAIL(init_alter_table_part_schema(inc_table_schema, alter_inc_drop_part_schema))) { + LOG_WARN("failed to init alter table part schema", KR(ret)); + } else if (OB_FAIL(init_alter_table_part_schema(inc_table_schema, alter_inc_add_new_part_schema))) { + LOG_WARN("failed to init alter table part schema", KR(ret)); + } else if (OB_FAIL(generate_alter_table_part_schema_for_pt(base_table_schema, + inc_table_schema, + base_tablet_ids, + inc_tablet_ids, + is_subpartition, + alter_pt_drop_part_schema, + alter_pt_add_new_part_schema, + alter_inc_drop_part_schema, + alter_inc_add_new_part_schema, + old_base_part_ids, + new_base_part_ids, + old_inc_part_ids, + new_inc_part_ids))) { + LOG_WARN("failed to generate alter table part schema for pt", KR(ret), + K(base_table_schema), K(inc_table_schema), K(base_tablet_ids), K(inc_tablet_ids), K(is_subpartition)); + } + } else if (OB_FAIL(generate_alter_table_part_schema_for_npt(base_table_schema, + inc_table_schema, + base_tablet_ids.at(0), + inc_tablet_ids.at(0), + is_subpartition, + alter_pt_drop_part_schema, + alter_pt_add_new_part_schema, + old_base_part_ids, + new_base_part_ids, + old_inc_part_ids, + new_inc_part_ids))) { + LOG_WARN("failed to generate alter table part schema for npt", KR(ret), + K(base_table_schema), K(inc_table_schema), K(base_tablet_ids), K(inc_tablet_ids), K(is_subpartition)); + } + + if (OB_SUCC(ret)) { + StatLevel new_base_table_stat_level; + StatLevel new_inc_table_stat_level; + if (is_subpartition) { + new_base_table_stat_level = inc_table_schema.is_partitioned_table() ? + StatLevel::SUBPARTITION_LEVEL : StatLevel::TABLE_LEVEL; + new_inc_table_stat_level = StatLevel::SUBPARTITION_LEVEL; + } else { + new_base_table_stat_level = inc_table_schema.is_partitioned_table() ? + StatLevel::PARTITION_LEVEL : StatLevel::TABLE_LEVEL; + new_inc_table_stat_level = StatLevel::PARTITION_LEVEL; + } + HEAP_VARS_2((ObTableSchema, new_pt_schema), - (AlterTableSchema, alter_pt_add_new_part_schema)) { - int64_t new_partition_id = OB_INVALID_PARTITION_ID; - if (OB_FAIL(new_pt_schema.assign(partitioned_table_schema))) { - LOG_WARN("fail to assign partitioned table schema", K(ret), K(partitioned_table_schema)); - } else if (OB_FAIL(alter_pt_add_new_part_schema.assign(partitioned_table_schema))) { - LOG_WARN("fail to assign partitioned table schema", K(ret), K(partitioned_table_schema)); - } else if (FALSE_IT(alter_pt_add_new_part_schema.reset_partition_schema())) { - } else if (OB_FAIL(alter_pt_add_new_part_schema.add_partition(new_part))) { - LOG_WARN("fail to add partition", K(ret), K(new_part)); - } else if (FALSE_IT(alter_pt_add_new_part_schema.set_part_level(partitioned_table_schema.get_part_level()))) { - } else if (OB_FAIL(alter_pt_add_new_part_schema.get_part_option().assign(partitioned_table_schema.get_part_option()))) { - LOG_WARN("fail to assign part option", K(ret), K(partitioned_table_schema.get_part_option())); - } else if (FALSE_IT(alter_pt_add_new_part_schema.get_part_option().set_part_num(alter_pt_add_new_part_schema.get_partition_num()))) { - } else if (OB_FAIL(ddl_service_.generate_object_id_for_partition_schema(alter_pt_add_new_part_schema))) { - LOG_WARN("fail to generate object_id for partition schema", K(ret), K(alter_pt_add_new_part_schema)); - } else if (OB_FAIL(get_object_id_from_partition_schema_(alter_pt_add_new_part_schema, false/*get_subpart_only*/, new_partition_id))) { - LOG_WARN("fail get object id from partition schema", K(ret), K(alter_pt_add_new_part_schema), K(new_partition_id)); + (ObTableSchema, new_inc_schema)) { + if (OB_FAIL(new_pt_schema.assign(base_table_schema))) { + LOG_WARN("fail to assign base table schema", K(ret), K(base_table_schema)); + } else if (OB_FALSE_IT(new_pt_schema.set_in_offline_ddl_white_list(true))) { + } else if (OB_FAIL(new_inc_schema.assign(inc_table_schema))) { + LOG_WARN("fail to assign inc table schema", K(ret), K(inc_table_schema)); + } else if (!is_inc_table_partitioned && OB_FALSE_IT(new_inc_schema.set_tablet_id(base_tablet_ids.at(0)))) { + } else if (OB_FALSE_IT(new_inc_schema.set_in_offline_ddl_white_list(true))) { } else if (OB_FAIL(update_exchange_table_non_schema_attributes_(tenant_id, - part.get_part_id(), - new_partition_id, - false/*is_exchange_subpartition*/, - new_pt_schema, - new_nt_schema, - exchange_table_ids, - exchange_tablet_ids, + base_table_schema, + base_tablet_ids, + old_base_part_ids, + new_base_part_ids, + inc_table_schema.get_table_id(), + new_base_table_stat_level, is_oracle_mode, ddl_operator, trans, schema_guard))) { - LOG_WARN("fail to update exchange table non schema attributes", K(ret), K(part.get_part_id()), K(new_partition_id), K(new_pt_schema), K(new_nt_schema), K(exchange_table_ids), K(exchange_tablet_ids), K(is_oracle_mode)); - } else if (OB_FAIL(ddl_operator.exchange_table_partitions(partitioned_table_schema, - alter_pt_add_new_part_schema, - alter_pt_drop_part_schema, - trans))) { - LOG_WARN("failed to exchange partitions", K(ret), K(partitioned_table_schema), K(alter_pt_add_new_part_schema), K(alter_pt_drop_part_schema)); + LOG_WARN("fail to update exchange table non schema attributes", K(ret), K(old_base_part_ids), K(new_base_part_ids), K(base_table_schema), K(inc_table_schema), K(is_oracle_mode)); + } else if (OB_FAIL(update_exchange_table_non_schema_attributes_(tenant_id, + inc_table_schema, + inc_tablet_ids, + old_inc_part_ids, + new_inc_part_ids, + base_table_schema.get_table_id(), + new_inc_table_stat_level, + is_oracle_mode, + ddl_operator, + trans, + schema_guard))) { + LOG_WARN("fail to update exchange table non schema attributes", K(ret), K(old_inc_part_ids), K(new_inc_part_ids), K(base_table_schema), K(inc_table_schema), K(is_oracle_mode)); + } else if (OB_FAIL(ddl_exchange_table_partitions(new_pt_schema, + alter_pt_add_new_part_schema, + alter_pt_drop_part_schema, + ddl_operator, + trans, + is_subpartition))) { + LOG_WARN("failed to exchange partitions", K(ret), K(base_table_schema), K(alter_pt_add_new_part_schema), K(alter_pt_drop_part_schema)); + } else if (is_inc_table_partitioned + && OB_FAIL(ddl_exchange_table_partitions(new_inc_schema, + alter_inc_add_new_part_schema, + alter_inc_drop_part_schema, + ddl_operator, + trans, + is_subpartition))) { + LOG_WARN("failed to exchange partitions", K(ret), K(inc_table_schema), K(alter_inc_add_new_part_schema), K(alter_inc_drop_part_schema)); } else if (OB_FAIL(update_exchange_table_level_attributes_(tenant_id, - exchange_table_ids, - exchange_tablet_ids, - new_pt_schema, - new_nt_schema, - trans))) { - LOG_WARN("fail to update exchange table level attributes", K(ret), K(tenant_id), K(exchange_table_ids), K(exchange_tablet_ids), K(new_pt_schema), K(new_nt_schema)); + base_tablet_ids, + inc_tablet_ids, + new_pt_schema, + new_inc_schema, + trans))) { + LOG_WARN("fail to update exchange table level attributes", K(ret), K(tenant_id), K(base_tablet_ids), K(inc_tablet_ids), K(new_pt_schema), K(new_inc_schema)); } - } - } - } - } + } // end HEAP_VARS_2 + } // end if + } // end HEAP_VARS_4 + } // end if return ret; } -int ObPartitionExchange::exchange_subpartition_map_relationship_(const uint64_t tenant_id, - const ObPartition &part, - const ObSubPartition &subpart, - const ObTableSchema &partitioned_table_schema, - const ObTableSchema &non_partitioned_table_schema, - const bool is_oracle_mode, - ObDDLOperator &ddl_operator, - ObDDLSQLTransaction &trans, - ObSchemaGetterGuard &schema_guard) +// for partitioned table +int ObPartitionExchange::generate_alter_table_part_schema_for_pt( + const ObTableSchema &base_table_schema, + const ObTableSchema &inc_table_schema, + const ObIArray &base_tablet_ids, + const ObIArray &inc_tablet_ids, + const bool is_subpartition, + AlterTableSchema &alter_pt_drop_part_schema, + AlterTableSchema &alter_pt_add_new_part_schema, + AlterTableSchema &alter_inc_drop_part_schema, + AlterTableSchema &alter_inc_add_new_part_schema, + ObIArray &old_base_part_ids, + ObIArray &new_base_part_ids, + ObIArray &old_inc_part_ids, + ObIArray &new_inc_part_ids) { int ret = OB_SUCCESS; - ObArray exchange_table_ids; - ObArray exchange_tablet_ids; - if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || !part.is_valid() || !subpart.is_valid() || !partitioned_table_schema.is_valid() || !non_partitioned_table_schema.is_valid())) { + if (OB_UNLIKELY(!base_table_schema.is_valid() || !inc_table_schema.is_valid() || !base_table_schema.is_partitioned_table() || !inc_table_schema.is_partitioned_table() || base_tablet_ids.count() != inc_tablet_ids.count())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(tenant_id), K(part), K(subpart), K(partitioned_table_schema.is_valid()), K(non_partitioned_table_schema.is_valid())); - } else if (OB_FAIL(exchange_table_ids.push_back(non_partitioned_table_schema.get_table_id()))) { - LOG_WARN("fail to push back table id", K(ret), K(non_partitioned_table_schema.get_table_id()), K(exchange_table_ids), K(non_partitioned_table_schema)); - } else if (OB_FAIL(exchange_table_ids.push_back(partitioned_table_schema.get_table_id()))) { - LOG_WARN("fail to push back table id", K(ret), K(partitioned_table_schema.get_table_id()), K(exchange_table_ids), K(partitioned_table_schema)); - } else if (OB_FAIL(exchange_tablet_ids.push_back(subpart.get_tablet_id()))) { - LOG_WARN("fail to push back partitioned table tablet id", K(ret), K(subpart.get_tablet_id())); - } else if (OB_FAIL(exchange_tablet_ids.push_back(non_partitioned_table_schema.get_tablet_id()))) { - LOG_WARN("fail to push back non_partitioned table tablet id", K(ret), K(non_partitioned_table_schema.get_table_id())); - } else if (OB_UNLIKELY(2 != exchange_table_ids.count() || 2 != exchange_tablet_ids.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("the array length is incorrect", K(ret), K(exchange_table_ids.count()), K(exchange_tablet_ids.count())); + LOG_WARN("invalid argument", KR(ret), K(base_table_schema), K(inc_table_schema), K(base_tablet_ids.count()), K(inc_tablet_ids.count())); } else { - // drop exchange subpartition in partitioned table, and add new exchange subpartition in partitioned table - HEAP_VARS_4((ObSubPartition, new_subpart), - (ObPartition, dummy_part), - (ObTableSchema, new_nt_schema), - (AlterTableSchema, alter_pt_drop_subpart_schema)) { - dummy_part.set_sub_part_num(1); - dummy_part.set_part_id(part.get_part_id()); - if (OB_FAIL(new_subpart.assign(subpart))) { - LOG_WARN("fail to assign subpartition schema", K(ret), K(subpart)); - } else if (OB_FAIL(dummy_part.set_part_name(part.get_part_name()))) { - LOG_WARN("failed to set part name", K(ret), K(part.get_part_name())); - } else if (OB_FAIL(dummy_part.add_partition(subpart))){ - LOG_WARN("failed to add subpart", K(ret), K(subpart)); - } else if (OB_FAIL(alter_pt_drop_subpart_schema.assign(partitioned_table_schema))) { - LOG_WARN("fail to assign partitioned table schema", K(ret), K(partitioned_table_schema)); - } else if (FALSE_IT(alter_pt_drop_subpart_schema.reset_partition_schema())) { - } else if (OB_FAIL(alter_pt_drop_subpart_schema.add_partition(dummy_part))) { - LOG_WARN("failed to add partition", K(ret), K(dummy_part)); - } else if (FALSE_IT(alter_pt_drop_subpart_schema.set_part_level(partitioned_table_schema.get_part_level()))) { - } else if (OB_FAIL(alter_pt_drop_subpart_schema.get_part_option().assign(partitioned_table_schema.get_part_option()))) { - LOG_WARN("fail to assign part option", K(ret), K(partitioned_table_schema.get_part_option())); - } else if (OB_FAIL(alter_pt_drop_subpart_schema.get_sub_part_option().assign(partitioned_table_schema.get_sub_part_option()))) { - LOG_WARN("fail to assign subpart option", K(ret), K(partitioned_table_schema.get_sub_part_option())); - } else if (FALSE_IT(alter_pt_drop_subpart_schema.get_part_option().set_part_num(alter_pt_drop_subpart_schema.get_partition_num()))) { - } else if (OB_FAIL(new_nt_schema.assign(non_partitioned_table_schema))) { - LOG_WARN("fail to assign non_partitioned table schema", K(ret), K(non_partitioned_table_schema)); + HEAP_VARS_4((ObPartition, new_part), + (ObPartition, new_inc_part), + (ObSubPartition, new_subpart), + (ObSubPartition, new_inc_subpart)) { + for (int64_t i = 0; OB_SUCC(ret) && (i < base_tablet_ids.count()); ++i) { + // drop old part + const ObPartition *part = nullptr; + const ObPartition *inc_part = nullptr; + const ObSubPartition *subpart = nullptr; + const ObSubPartition *inc_subpart = nullptr; + int64_t old_part_id = OB_INVALID_PARTITION_ID; + int64_t new_part_id = OB_INVALID_PARTITION_ID; + int64_t old_inc_part_id = OB_INVALID_PARTITION_ID; + int64_t new_inc_part_id = OB_INVALID_PARTITION_ID; + new_part.reset(); + new_inc_part.reset(); + new_subpart.reset(); + new_inc_subpart.reset(); + + if (OB_FAIL(get_part_by_tablet_id(base_table_schema, base_tablet_ids.at(i), part, subpart, is_subpartition/*get_subpart*/))) { + LOG_WARN("failed to get part by tablet id", KR(ret)); + } else if (OB_FAIL(generate_alter_table_part_schema(base_table_schema, part, subpart, is_subpartition/*is_subpart*/, alter_pt_drop_part_schema))) { + LOG_WARN("failed to generate alter table part schema", KR(ret), K(base_table_schema), K(*part)); + } else if (OB_FAIL(get_part_by_tablet_id(inc_table_schema, inc_tablet_ids.at(i), inc_part, inc_subpart, is_subpartition/*get_subpart*/))) { + LOG_WARN("failed to get part by tablet id", KR(ret)); + } else if (OB_FAIL(generate_alter_table_part_schema( + inc_table_schema, inc_part, inc_subpart, is_subpartition/*is_subpart*/, alter_inc_drop_part_schema))) { + LOG_WARN("failed to generate alter table part schema", KR(ret), K(inc_table_schema), K(*inc_part)); + } else if (is_subpartition) { + if (OB_FAIL(new_subpart.assign(*inc_subpart))) { + LOG_WARN("fail to assign subpartition schema", K(ret), K(*subpart)); + } else if (OB_FAIL(new_inc_subpart.assign(*subpart))) { + LOG_WARN("fail to assign subpartition schema", K(ret), K(*subpart)); + } + } else { + if (OB_FAIL(new_part.assign(*inc_part))) { + LOG_WARN("fail to assign part", K(ret), K(*inc_part)); + } else if (OB_FAIL(new_inc_part.assign(*part))) { + LOG_WARN("fail to assign inc part", K(ret), K(*part)); + } + } + if (OB_SUCC(ret)) { + if (is_subpartition) { + old_inc_part_id = inc_subpart->get_sub_part_id(); + new_inc_part_id = subpart->get_sub_part_id(); + } else { + old_inc_part_id = inc_part->get_part_id(); + new_inc_part_id = part->get_part_id(); + } + } + + // add new part + if (OB_SUCC(ret)) { + const ObPartition *add_part = is_subpartition ? part : &new_part; + const ObSubPartition *add_subpart = is_subpartition ? &new_subpart : nullptr; + if (OB_FAIL(generate_alter_table_part_schema(base_table_schema, add_part, add_subpart, is_subpartition/*is_subpart*/, alter_pt_add_new_part_schema))) { + LOG_WARN("failed to generate alter table part schema", KR(ret), K(base_table_schema), K(new_part)); + } else { + const ObPartition *add_inc_part = is_subpartition ? inc_part : &new_inc_part; + const ObSubPartition *add_inc_subpart = is_subpartition ? &new_inc_subpart : nullptr; + if (OB_FAIL(generate_alter_table_part_schema(inc_table_schema, add_inc_part, add_inc_subpart, is_subpartition/*is_subpart*/, alter_inc_add_new_part_schema))) { + LOG_WARN("failed to generate alter table part schema", KR(ret), K(base_table_schema), K(new_part)); + } else { + new_part_id = old_inc_part_id; + } + } + } + if (OB_SUCC(ret)) { + old_part_id = is_subpartition ? subpart->get_sub_part_id() : part->get_part_id(); + OZ(old_base_part_ids.push_back(old_part_id)); // info to update part table + OZ(new_base_part_ids.push_back(new_inc_part_id)); + OZ(old_inc_part_ids.push_back(old_inc_part_id)); // info to update inc table + OZ(new_inc_part_ids.push_back(new_part_id)); + } + } // end for + } // end HEAP_VARS_4 + } // end if + return ret; +} + +// for non-partitioned table +int ObPartitionExchange::generate_alter_table_part_schema_for_npt( + const ObTableSchema &base_table_schema, + const ObTableSchema &inc_table_schema, + const ObTabletID &base_tablet_id, + const ObTabletID &inc_tablet_id, + const bool is_subpartition, + AlterTableSchema &alter_pt_drop_part_schema, + AlterTableSchema &alter_pt_add_new_part_schema, + ObIArray &old_base_part_ids, + ObIArray &new_base_part_ids, + ObIArray &old_inc_part_ids, + ObIArray &new_inc_part_ids) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!base_table_schema.is_valid() || !inc_table_schema.is_valid() || !base_table_schema.is_partitioned_table() || inc_table_schema.is_partitioned_table() || !base_tablet_id.is_valid() || !inc_tablet_id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(base_table_schema), K(inc_table_schema), K(base_tablet_id), K(inc_tablet_id)); + } else { + HEAP_VARS_2((ObPartition, new_part), + (ObSubPartition, new_subpart)) { + // drop old part + const ObPartition *part = nullptr; + const ObSubPartition *subpart = nullptr; + int64_t old_part_id = OB_INVALID_PARTITION_ID; + int64_t new_part_id = OB_INVALID_PARTITION_ID; + new_part.reset(); + new_subpart.reset(); + + if (OB_FAIL(get_part_by_tablet_id(base_table_schema, base_tablet_id, part, subpart, is_subpartition/*get_subpart*/))) { + LOG_WARN("failed to get part by tablet id", KR(ret)); + } else if (OB_FAIL(generate_alter_table_part_schema(base_table_schema, part, subpart, is_subpartition/*is_subpart*/, alter_pt_drop_part_schema))) { + LOG_WARN("failed to generate alter table part schema", KR(ret), K(base_table_schema), K(*part)); + } else if (is_subpartition) { + if (OB_FAIL(new_subpart.assign(*subpart))) { + LOG_WARN("fail to assign subpartition schema", K(ret), K(*subpart)); + } else if (OB_FALSE_IT(new_subpart.set_tablet_id(inc_tablet_id))) { + } } else { - new_subpart.set_tablet_id(exchange_tablet_ids.at(1)); - new_nt_schema.set_tablet_id(exchange_tablet_ids.at(0)); - HEAP_VARS_2((ObTableSchema, new_pt_schema), - (AlterTableSchema, alter_pt_add_new_subpart_schema)) { - dummy_part.reset(); - dummy_part.set_sub_part_num(1); - dummy_part.set_part_id(part.get_part_id()); - int64_t new_subpart_id = OB_INVALID_PARTITION_ID; - if (OB_FAIL(new_pt_schema.assign(partitioned_table_schema))) { - LOG_WARN("fail to assign partitioned table schema", K(ret), K(partitioned_table_schema)); - } else if (OB_FAIL(dummy_part.set_part_name(part.get_part_name()))) { - LOG_WARN("failed to set part name", K(ret), K(part.get_part_name())); - } else if (OB_FAIL(dummy_part.add_partition(new_subpart))){ - LOG_WARN("failed to add subpart", K(ret), K(new_subpart)); - } else if (OB_FAIL(alter_pt_add_new_subpart_schema.assign(partitioned_table_schema))) { - LOG_WARN("fail to assign partitioned table schema", K(ret), K(partitioned_table_schema)); - } else if (FALSE_IT(alter_pt_add_new_subpart_schema.reset_partition_schema())) { - } else if (FALSE_IT(alter_pt_add_new_subpart_schema.set_part_level(partitioned_table_schema.get_part_level()))) { - } else if (OB_FAIL(alter_pt_add_new_subpart_schema.get_sub_part_option().assign(partitioned_table_schema.get_sub_part_option()))) { - LOG_WARN("fail to assign subpart option", K(ret), K(partitioned_table_schema.get_sub_part_option())); - } else if (OB_FAIL(alter_pt_add_new_subpart_schema.add_partition(dummy_part))) { - LOG_WARN("fail to add subpartition", K(ret), K(dummy_part)); - } else if (OB_FAIL(alter_pt_add_new_subpart_schema.get_part_option().assign(partitioned_table_schema.get_part_option()))) { - LOG_WARN("fail to assign subpart option", K(ret), K(partitioned_table_schema.get_part_option())); - } else if (FALSE_IT(alter_pt_add_new_subpart_schema.set_part_num(alter_pt_add_new_subpart_schema.get_partition_num()))) { - } else if (OB_FAIL(ddl_service_.generate_object_id_for_partition_schema(alter_pt_add_new_subpart_schema, true/*gen_subpart_only*/))) { - LOG_WARN("fail to generate object_id for partition schema", K(ret), K(alter_pt_add_new_subpart_schema)); - } else if (OB_FAIL(get_object_id_from_partition_schema_(alter_pt_add_new_subpart_schema, true/*gen_subpart_only*/, new_subpart_id))) { - LOG_WARN("fail get object id from partition schema", K(ret), K(alter_pt_add_new_subpart_schema), K(new_subpart_id)); - } else if (OB_FAIL(update_exchange_table_non_schema_attributes_(tenant_id, - subpart.get_sub_part_id(), - new_subpart_id, - true/*is_exchange_subpartition*/, - new_pt_schema, - new_nt_schema, - exchange_table_ids, - exchange_tablet_ids, - is_oracle_mode, - ddl_operator, - trans, - schema_guard))) { - LOG_WARN("fail update exchange table non schema attributes", K(ret), K(subpart.get_sub_part_id()), K(new_subpart_id), K(new_pt_schema), K(new_nt_schema), K(exchange_table_ids), K(exchange_tablet_ids), K(is_oracle_mode)); - } else if (OB_FAIL(ddl_operator.exchange_table_subpartitions(partitioned_table_schema, - alter_pt_add_new_subpart_schema, - alter_pt_drop_subpart_schema, - trans))) { - LOG_WARN("failed to exchange subpartitions", K(ret), K(partitioned_table_schema), K(alter_pt_add_new_subpart_schema), K(alter_pt_drop_subpart_schema)); - } else if (OB_FAIL(update_exchange_table_level_attributes_(tenant_id, - exchange_table_ids, - exchange_tablet_ids, - new_pt_schema, - new_nt_schema, - trans))) { - LOG_WARN("fail to update exchange table level attributes", K(ret), K(tenant_id), K(exchange_table_ids), K(exchange_tablet_ids), K(new_pt_schema), K(new_nt_schema)); + if (OB_FAIL(new_part.assign(*part))) { + LOG_WARN("fail to assign part", K(ret), K(*part)); + } else if (OB_FALSE_IT(new_part.set_tablet_id(inc_tablet_id))) { + } + } + + // add new part + if (OB_SUCC(ret)) { + const ObPartition *add_part = is_subpartition ? part : &new_part; + const ObSubPartition *add_subpart = is_subpartition ? &new_subpart : nullptr; + if (OB_FAIL(generate_alter_table_part_schema(base_table_schema, add_part, add_subpart, is_subpartition/*is_subpart*/, alter_pt_add_new_part_schema))) { + LOG_WARN("failed to generate alter table part schema", KR(ret), K(base_table_schema), K(new_part)); + } else { + int64_t alloc_part_id = OB_INVALID_PARTITION_ID; + if (OB_FAIL(ddl_service_.generate_object_id_for_partition_schema(alter_pt_add_new_part_schema, is_subpartition/*gen_subpart_only*/))) { + LOG_WARN("fail to generate object_id for partition schema", K(ret), K(alter_pt_add_new_part_schema)); + } else if (OB_FAIL(get_object_id_from_partition_schema_(alter_pt_add_new_part_schema, is_subpartition/*get_subpart_only*/, alloc_part_id))) { + LOG_WARN("fail get object id from partition schema", K(ret), K(alter_pt_add_new_part_schema), K(alloc_part_id)); + } else { + new_part_id = alloc_part_id; } } } - } - } + if (OB_SUCC(ret)) { + old_part_id = is_subpartition ? subpart->get_sub_part_id() : part->get_part_id(); + OZ(old_base_part_ids.push_back(old_part_id)); // info to update part table + OZ(new_base_part_ids.push_back(inc_table_schema.get_table_id())); + OZ(old_inc_part_ids.push_back(inc_table_schema.get_table_id())); // info to update inc table + OZ(new_inc_part_ids.push_back(new_part_id)); + } + } // end HEAP_VARS_2 + } // end if return ret; } int ObPartitionExchange::update_exchange_table_non_schema_attributes_(const uint64_t tenant_id, - const int64_t old_partition_id, - const int64_t new_partition_id, - const bool is_exchange_subpartition, - const ObTableSchema &partitioned_table_schema, - const ObTableSchema &non_partitioned_table_schema, - const ObIArray &exchange_table_ids, - const ObIArray &exchange_tablet_ids, + const ObTableSchema &old_table_schema, + const ObIArray &old_tablet_ids, + const ObIArray &old_partition_ids, + const ObIArray &new_partition_ids, + const uint64_t new_table_id, + const StatLevel new_stat_level, const bool is_oracle_mode, ObDDLOperator &ddl_operator, ObDDLSQLTransaction &trans, ObSchemaGetterGuard &schema_guard) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || OB_INVALID_ID == old_partition_id || OB_INVALID_ID == new_partition_id || exchange_table_ids.count() != exchange_tablet_ids.count() || 2 != exchange_table_ids.count())) { + if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || old_partition_ids.count() != new_partition_ids.count())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(tenant_id), K(old_partition_id), K(new_partition_id), K(partitioned_table_schema), K(non_partitioned_table_schema), K(exchange_table_ids), K(exchange_tablet_ids)); + LOG_WARN("invalid argument", K(ret), K(tenant_id), K(old_partition_ids), K(new_partition_ids), K(old_table_schema), K(new_table_id), K(old_tablet_ids)); } else { // modify inner table __all_tablet_to_ls, __all_sequence_value or __all_sequence_value, __all_table_stat, __all_column_stat, __all_histogram_stat, __all_monitor_modified - if (OB_SUCC(ret)) { - if (OB_FAIL(update_table_to_tablet_id_mapping_(tenant_id, exchange_table_ids, exchange_tablet_ids, trans))) { - LOG_WARN("fail to update table to tablet id mapping", K(ret), K(tenant_id), K(exchange_table_ids), K(exchange_tablet_ids)); - } else if (!partitioned_table_schema.is_aux_table() && !non_partitioned_table_schema.is_aux_table()) { - // TODO: After confirming the specific behavior of self increasing columns in MySQL mode and identity in Oracle mode, supplement it. - // if (is_oracle_mode) { - // if (OB_FAIL(update_identity_column_information_(tenant_id, partitioned_table_schema, non_partitioned_table_schema, is_oracle_mode, ddl_operator, trans, schema_guard))) { - // LOG_WARN("failed to update identity column information", K(ret), K(tenant_id), K(partitioned_table_schema), K(non_partitioned_table_schema), K(is_oracle_mode)); - // } - // } else if (OB_FAIL(update_autoinc_column_information_(tenant_id, partitioned_table_schema, non_partitioned_table_schema, ddl_operator, trans))) { - // LOG_WARN("failed to update autoinc column information", K(ret), K(tenant_id), K(partitioned_table_schema), K(non_partitioned_table_schema)); - // } - if (OB_SUCC(ret)) { - if (!is_exchange_subpartition) { - if (OB_FAIL(sync_exchange_partition_stats_info_(tenant_id, - partitioned_table_schema.get_table_id(), - StatLevel::PARTITION_LEVEL, - non_partitioned_table_schema.get_table_id(), - new_partition_id, - exchange_tablet_ids.at(1), - non_partitioned_table_schema, - trans))) { - LOG_WARN("fail to sync exchange partition stats info", K(ret), K(partitioned_table_schema), K(non_partitioned_table_schema), K(new_partition_id), K(exchange_tablet_ids)); - } - } else if (OB_FAIL(sync_exchange_partition_stats_info_(tenant_id, - partitioned_table_schema.get_table_id(), - StatLevel::SUBPARTITION_LEVEL, - non_partitioned_table_schema.get_table_id(), - new_partition_id, - exchange_tablet_ids.at(1), - non_partitioned_table_schema, - trans))) { - LOG_WARN("fail to sync exchange subpartition stats info", K(ret), K(partitioned_table_schema), K(non_partitioned_table_schema), K(new_partition_id), K(exchange_tablet_ids)); - } - if (OB_SUCC(ret)) { - if (OB_FAIL(sync_exchange_partition_stats_info_(tenant_id, - non_partitioned_table_schema.get_table_id(), - StatLevel::TABLE_LEVEL, - old_partition_id, - non_partitioned_table_schema.get_table_id(), - exchange_tablet_ids.at(0), - partitioned_table_schema, - trans))) { - LOG_WARN("fail to sync exchange partition stats info", K(ret), K(partitioned_table_schema), K(non_partitioned_table_schema), K(old_partition_id), K(exchange_tablet_ids)); - } - } + if (OB_FAIL(update_table_to_tablet_ids_mapping_(tenant_id, new_table_id, old_tablet_ids, trans))) { + LOG_WARN("fail to update table to tablet id mapping", K(ret), K(tenant_id), K(new_table_id), K(old_tablet_ids)); + } else if (!old_table_schema.is_aux_table()) { + // TODO: After confirming the specific behavior of self increasing columns in MySQL mode and identity in Oracle mode, supplement it. + // if (is_oracle_mode) { + // if (OB_FAIL(update_identity_column_information_(tenant_id, base_table_schema, inc_table_schema, is_oracle_mode, ddl_operator, trans, schema_guard))) { + // LOG_WARN("failed to update identity column information", K(ret), K(tenant_id), K(base_table_schema), K(inc_table_schema), K(is_oracle_mode)); + // } + // } else if (OB_FAIL(update_autoinc_column_information_(tenant_id, base_table_schema, inc_table_schema, ddl_operator, trans))) { + // LOG_WARN("failed to update autoinc column information", K(ret), K(tenant_id), K(base_table_schema), K(inc_table_schema)); + // } + for (int64_t i = 0; OB_SUCC(ret) && (i < old_tablet_ids.count()); ++i) { + if (OB_FAIL(sync_exchange_partition_stats_info_(tenant_id, + new_table_id, + new_stat_level, + old_partition_ids.at(i), + new_partition_ids.at(i), + old_tablet_ids.at(i), + old_table_schema, + trans))) { + LOG_WARN("fail to sync exchange partition stats info", K(ret), K(old_table_schema), K(new_table_id), K(old_partition_ids.at(i)), K(new_partition_ids.at(i)), K(old_tablet_ids.at(i))); } } } @@ -1683,41 +1809,35 @@ int ObPartitionExchange::update_exchange_table_non_schema_attributes_(const uint } int ObPartitionExchange::update_exchange_table_level_attributes_(const uint64_t tenant_id, - const ObIArray &exchange_table_ids, - const ObIArray &exchange_tablet_ids, + const ObIArray &base_tablet_ids, + const ObIArray &inc_tablet_ids, ObTableSchema &partitioned_table_schema, ObTableSchema &non_partitioned_table_schema, ObDDLSQLTransaction &trans) { int ret = OB_SUCCESS; - ObArray pt_tablet_ids; - ObArray nt_tablet_ids; - if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || 2 != exchange_table_ids.count() || 2 != exchange_tablet_ids.count() || !partitioned_table_schema.is_valid() || !non_partitioned_table_schema.is_valid())) { + if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || base_tablet_ids.count() != inc_tablet_ids.count() || !partitioned_table_schema.is_valid() || !non_partitioned_table_schema.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(tenant_id), K(exchange_table_ids.count()), K(exchange_tablet_ids.count()), K(partitioned_table_schema.is_valid()), K(non_partitioned_table_schema.is_valid())); - } else if (OB_FAIL(nt_tablet_ids.push_back(exchange_tablet_ids.at(0)))) { - LOG_WARN("failed to push back tablet id", K(ret), K(exchange_tablet_ids.at(0))); - } else if (OB_FAIL(pt_tablet_ids.push_back(exchange_tablet_ids.at(1)))) { - LOG_WARN("failed to push back tablet id", K(ret), K(exchange_tablet_ids.at(1))); + LOG_WARN("invalid argument", K(ret), K(tenant_id), K(base_tablet_ids.count()), K(inc_tablet_ids.count()), K(partitioned_table_schema.is_valid()), K(non_partitioned_table_schema.is_valid())); } else { + const uint64_t base_table_id = partitioned_table_schema.get_table_id(); + const uint64_t inc_table_id = non_partitioned_table_schema.get_table_id(); // modify inner table __all_tablet_to_table_history and __all_table share::ObTabletTablePair pair; - ObArray pt_pairs; - ObArray nt_pairs; - for (int64_t i = 0; OB_SUCC(ret) && i < exchange_tablet_ids.count(); i++) { - if (OB_FAIL(pair.init(exchange_tablet_ids.at(i), exchange_table_ids.at(i)))) { - LOG_WARN("fail to init tablet to table pair", K(ret), K(exchange_tablet_ids.at(i)), K(exchange_table_ids.at(i))); - } else if (exchange_table_ids.at(i) == partitioned_table_schema.get_table_id()) { - if (OB_FAIL(pt_pairs.push_back(pair))) { - LOG_WARN("fail to push back tablet table pair", K(ret), K(pair), K(pt_pairs)); - } - } else if (exchange_table_ids.at(i) == non_partitioned_table_schema.get_table_id()) { - if (OB_FAIL(nt_pairs.push_back(pair))) { - LOG_WARN("fail to push back tablet table pair", K(ret), K(pair), K(nt_pairs)); - } - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("exchange table id is error", K(ret), K(exchange_table_ids.at(i)), K(partitioned_table_schema.get_table_id()), K(non_partitioned_table_schema.get_table_id())); + ObArray base_pairs; + ObArray inc_pairs; + for (int64_t i = 0; OB_SUCC(ret) && i < inc_tablet_ids.count(); i++) { + if (OB_FAIL(pair.init(inc_tablet_ids.at(i), base_table_id))) { + LOG_WARN("fail to init tablet to table pair", K(ret), K(inc_tablet_ids.at(i)), K(base_table_id)); + } else if (OB_FAIL(base_pairs.push_back(pair))) { + LOG_WARN("fail to push back tablet table pair", K(ret), K(pair), K(base_pairs)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < base_tablet_ids.count(); i++) { + if (OB_FAIL(pair.init(base_tablet_ids.at(i), inc_table_id))) { + LOG_WARN("fail to init tablet to table pair", K(ret), K(base_tablet_ids.at(i)), K(inc_table_id)); + } else if (OB_FAIL(inc_pairs.push_back(pair))) { + LOG_WARN("fail to push back tablet table pair", K(ret), K(pair), K(inc_pairs)); } } if (OB_SUCC(ret)) { @@ -1725,19 +1845,19 @@ int ObPartitionExchange::update_exchange_table_level_attributes_(const uint64_t LOG_WARN("fail to refresh table schema version", K(ret), K(non_partitioned_table_schema)); } else if (OB_FAIL(refresh_table_schema_version_(tenant_id, partitioned_table_schema))) { LOG_WARN("fail to refresh table schema version", K(ret), K(partitioned_table_schema)); - } else if (OB_FAIL(ObTabletToTableHistoryOperator::create_tablet_to_table_history(trans, tenant_id, partitioned_table_schema.get_schema_version(), pt_pairs))) { + } else if (OB_FAIL(ObTabletToTableHistoryOperator::create_tablet_to_table_history(trans, tenant_id, partitioned_table_schema.get_schema_version(), base_pairs))) { LOG_WARN("fail to create tablet to table history", K(ret), K(tenant_id), K(partitioned_table_schema)); - } else if (OB_FAIL(ObTabletToTableHistoryOperator::create_tablet_to_table_history(trans, tenant_id, non_partitioned_table_schema.get_schema_version(), nt_pairs))) { + } else if (OB_FAIL(ObTabletToTableHistoryOperator::create_tablet_to_table_history(trans, tenant_id, non_partitioned_table_schema.get_schema_version(), inc_pairs))) { LOG_WARN("fail to create tablet to table history", K(ret), K(tenant_id), K(non_partitioned_table_schema)); } else if (OB_FAIL(update_table_attribute_(non_partitioned_table_schema, trans))) { LOG_WARN("fail to update table attribute", K(ret), K(non_partitioned_table_schema)); } else if (OB_FAIL(update_table_attribute_(partitioned_table_schema, trans))) { LOG_WARN("fail to update table attribute", K(ret), K(partitioned_table_schema)); } else if (partitioned_table_schema.is_aux_table() && non_partitioned_table_schema.is_aux_table()) { - if (OB_FAIL(build_single_table_rw_defensive_(tenant_id, nt_tablet_ids, non_partitioned_table_schema.get_schema_version(), trans))) { - LOG_WARN("failed to build rw defensive", K(ret), K(tenant_id), K(nt_tablet_ids), K(non_partitioned_table_schema.get_schema_version())); - } else if (OB_FAIL(build_single_table_rw_defensive_(tenant_id, pt_tablet_ids, partitioned_table_schema.get_schema_version(), trans))) { - LOG_WARN("failed to build rw defensive", K(ret), K(tenant_id), K(pt_tablet_ids), K(partitioned_table_schema.get_schema_version())); + if (OB_FAIL(build_single_table_rw_defensive_(tenant_id, base_tablet_ids, non_partitioned_table_schema.get_schema_version(), trans))) { + LOG_WARN("failed to build rw defensive", K(ret), K(tenant_id), K(base_tablet_ids), K(non_partitioned_table_schema.get_schema_version())); + } else if (OB_FAIL(build_single_table_rw_defensive_(tenant_id, inc_tablet_ids, partitioned_table_schema.get_schema_version(), trans))) { + LOG_WARN("failed to build rw defensive", K(ret), K(tenant_id), K(inc_tablet_ids), K(partitioned_table_schema.get_schema_version())); } } } @@ -1745,19 +1865,19 @@ int ObPartitionExchange::update_exchange_table_level_attributes_(const uint64_t return ret; } -int ObPartitionExchange::update_table_to_tablet_id_mapping_(const uint64_t tenant_id, - const ObIArray &table_ids, +int ObPartitionExchange::update_table_to_tablet_ids_mapping_(const uint64_t tenant_id, + const uint64_t table_id, const ObIArray &tablet_ids, ObDDLSQLTransaction &trans) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id) || table_ids.count() != tablet_ids.count() || 2 != table_ids.count()) { + if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id) || tablet_ids.empty()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(tenant_id), K(table_ids.count()), K(tablet_ids.count())); + LOG_WARN("invalid argument", K(ret), K(tenant_id), K(table_id), K(tablet_ids.count())); } else { - for (int64_t i = 0; OB_SUCC(ret) && i < table_ids.count(); i++) { - if (OB_FAIL(ObTabletToLSTableOperator::update_table_to_tablet_id_mapping(trans, tenant_id, table_ids.at(i), tablet_ids.at(i)))) { - LOG_WARN("fail to update table to tablet id mapping", K(ret), K(tenant_id), K(table_ids.at(i)), K(tablet_ids.at(i))); + for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); i++) { + if (OB_FAIL(ObTabletToLSTableOperator::update_table_to_tablet_id_mapping(trans, tenant_id, table_id, tablet_ids.at(i)))) { + LOG_WARN("fail to update table to tablet id mapping", K(ret), K(tenant_id), K(table_id), K(tablet_ids.at(i))); } } } @@ -1808,19 +1928,19 @@ int ObPartitionExchange::push_data_table_schema_version_(const uint64_t tenant_i ObDDLSQLTransaction &trans) { int ret = OB_SUCCESS; - ObTabletID tablet_id; ObArray tablet_ids; ObSchemaOperationType operation_type = OB_DDL_EXCHANGE_PARTITION; ObSchemaService *schema_service = NULL; ObMultiVersionSchemaService &multi_schema_service = ddl_service_.get_schema_service(); new_schema_version = OB_INVALID_VERSION; - if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || OB_INVALID_ID == exchange_data_table_id)) { + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObPartitionExchange not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || OB_INVALID_ID == exchange_data_table_id)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(tenant_id), K(exchange_data_table_id)); - } else if (OB_FAIL(used_table_to_tablet_id_map_.get_refactored(exchange_data_table_id, tablet_id))) { - LOG_WARN("get_refactored tablet id from used_table_to_tablet_id_map failed", K(ret), K(exchange_data_table_id)); - } else if (OB_FAIL(tablet_ids.push_back(tablet_id))) { - LOG_WARN("failed to push back tablet id", K(ret), K(tablet_id)); + } else if (OB_FAIL(used_table_to_tablet_ids_map_.get_refactored(exchange_data_table_id, tablet_ids))) { + LOG_WARN("get_refactored tablet ids from used_table_to_tablet_ids_map failed", K(ret), K(exchange_data_table_id)); } else if (OB_ISNULL(schema_service = multi_schema_service.get_schema_service())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get schema_service is null", K(ret)); @@ -1830,6 +1950,7 @@ int ObPartitionExchange::push_data_table_schema_version_(const uint64_t tenant_i HEAP_VAR(ObTableSchema, new_table_schema) { if (OB_FAIL(schema_service->get_table_schema_from_inner_table(schema_status, table_schema.get_table_id(), trans, new_table_schema))) { LOG_WARN("get_table_schema failed", K(ret), K(schema_status), K(table_schema.get_table_id())); + } else if (OB_FALSE_IT(new_table_schema.set_in_offline_ddl_white_list(true))) { } else if (OB_FAIL(refresh_table_schema_version_(tenant_id, new_table_schema))) { LOG_WARN("fail to refresh table schema version", K(ret), K(new_table_schema)); } else if (OB_FAIL(schema_service->get_table_sql_service().update_table_schema_version(trans, @@ -1848,9 +1969,9 @@ int ObPartitionExchange::push_data_table_schema_version_(const uint64_t tenant_i } int ObPartitionExchange::get_local_storage_index_and_lob_table_schemas_(const ObTableSchema &table_schema, - const bool is_pt_schema, const bool is_oracle_mode, ObIArray &table_schemas, + ObIArray &unused_index_ids, ObSchemaGetterGuard &schema_guard) { int ret = OB_SUCCESS; @@ -1890,11 +2011,7 @@ int ObPartitionExchange::get_local_storage_index_and_lob_table_schemas_(const Ob } else if (OB_FAIL(check_auxiliary_schema_conditions_(aux_table_schema, is_oracle_mode))) { LOG_WARN("fail to check auxiliary schema conditions", K(ret), K(aux_table_schema), K(is_oracle_mode)); } else if (aux_table_schema->is_index_table() && aux_table_schema->is_global_index_table()) { - if (is_pt_schema) { - if (OB_FAIL(unused_pt_index_id_.push_back(aux_table_schema->get_table_id()))) { - LOG_WARN("failed to push back", K(ret), K(aux_table_schema->get_table_id())); - } - } else if (OB_FAIL(unused_nt_index_id_.push_back(aux_table_schema->get_table_id()))) { + if (OB_FAIL(unused_index_ids.push_back(aux_table_schema->get_table_id()))) { LOG_WARN("failed to push back", K(ret), K(aux_table_schema->get_table_id())); } } else if (OB_FAIL(table_schemas.push_back(aux_table_schema))) { @@ -1957,121 +2074,118 @@ bool ObPartitionExchange::in_find_same_aux_table_retry_white_list_(const int ret OB_TABLES_DIFFERENT_DEFINITIONS == ret_code; } -int ObPartitionExchange::generate_auxiliary_table_mapping_(const ObTableSchema &partitioned_data_table_schema, - const ObTableSchema &non_partitioned_data_table_schema, - const ObString &exchange_partition_name, +int ObPartitionExchange::generate_auxiliary_table_mapping_(const ObTableSchema &base_data_table_schema, + const ObTableSchema &inc_data_table_schema, const ObPartitionLevel exchange_partition_level, const bool is_oracle_mode, ObSchemaGetterGuard &schema_guard) { int ret = OB_SUCCESS; - ObArray partitioned_table_schemas; - ObArray non_partitioned_table_schemas; + ObArray base_table_schemas; + ObArray inc_table_schemas; ObArray used_nt_schema_flag; bool is_equal = false; - if (OB_UNLIKELY(!used_pt_nt_id_map_.created() || exchange_partition_name.empty() || PARTITION_LEVEL_ZERO == exchange_partition_level)) { + if (OB_UNLIKELY(!used_pt_nt_id_map_.created() || PARTITION_LEVEL_ZERO == exchange_partition_level)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(used_pt_nt_id_map_.created()), K(exchange_partition_name), K(exchange_partition_level)); - } else if (OB_FAIL(get_local_storage_index_and_lob_table_schemas_(partitioned_data_table_schema, true/*is_partitioned_table_schema*/, is_oracle_mode, partitioned_table_schemas, schema_guard))) { - LOG_WARN("fail to get local storage index and lob table schemas", K(ret), K(partitioned_data_table_schema), K(is_oracle_mode)); - } else if (OB_FAIL(get_local_storage_index_and_lob_table_schemas_(non_partitioned_data_table_schema, false/*is_partitioned_table_schema*/, is_oracle_mode, non_partitioned_table_schemas, schema_guard))) { - LOG_WARN("fail to get local storage index and lob table schemas", K(ret), K(non_partitioned_data_table_schema), K(is_oracle_mode)); - } else if (OB_UNLIKELY(partitioned_table_schemas.count() != non_partitioned_table_schemas.count())) { + LOG_WARN("invalid argument", K(ret), K(used_pt_nt_id_map_.created()), K(exchange_partition_level)); + } else if (OB_FAIL(get_local_storage_index_and_lob_table_schemas_(base_data_table_schema, is_oracle_mode, base_table_schemas, unused_pt_index_id_, schema_guard))) { + LOG_WARN("fail to get local storage index and lob table schemas", K(ret), K(base_data_table_schema), K(is_oracle_mode)); + } else if (OB_FAIL(get_local_storage_index_and_lob_table_schemas_(inc_data_table_schema, is_oracle_mode, inc_table_schemas, unused_nt_index_id_, schema_guard))) { + LOG_WARN("fail to get local storage index and lob table schemas", K(ret), K(inc_data_table_schema), K(is_oracle_mode)); + } else if (OB_UNLIKELY(base_table_schemas.count() != inc_table_schemas.count())) { if (is_oracle_mode) { ret = OB_ERR_INDEX_MISMATCH_ALTER_TABLE_EXCHANGE_PARTITION; } else { ret = OB_TABLES_DIFFERENT_DEFINITIONS; } - LOG_WARN("pt schemas count and nt schemas count are not equal", K(ret), K(partitioned_table_schemas.count()), K(non_partitioned_table_schemas.count())); + LOG_WARN("pt schemas count and nt schemas count are not equal", K(ret), K(base_table_schemas.count()), K(inc_table_schemas.count())); } else { - for (int64_t i = 0; OB_SUCC(ret) && i < non_partitioned_table_schemas.count(); i++) { + for (int64_t i = 0; OB_SUCC(ret) && i < inc_table_schemas.count(); i++) { if (OB_FAIL(used_nt_schema_flag.push_back(false))) { LOG_WARN("failed to push back", K(ret), K(i), K(used_nt_schema_flag)); } } - for (int64_t i = 0; OB_SUCC(ret) && i < partitioned_table_schemas.count(); i++) { - // for each partitioned table, find a one-to-one corresponding table in the non partitioned table - if (OB_FAIL(generate_local_storage_index_and_lob_table_mapping_(*partitioned_table_schemas.at(i), non_partitioned_table_schemas, exchange_partition_name, exchange_partition_level, is_oracle_mode, used_nt_schema_flag))) { - LOG_WARN("fail to generate used aux table id mapping", K(ret), KPC(partitioned_table_schemas.at(i)), K(non_partitioned_table_schemas.count()), K(exchange_partition_name), K(exchange_partition_level), K(is_oracle_mode), K(used_nt_schema_flag.count())); + for (int64_t i = 0; OB_SUCC(ret) && i < base_table_schemas.count(); i++) { + // for each base table, find a one-to-one corresponding table in the inc table + if (OB_FAIL(generate_local_storage_index_and_lob_table_mapping_(*base_table_schemas.at(i), inc_table_schemas, exchange_partition_level, is_oracle_mode, used_nt_schema_flag))) { + LOG_WARN("fail to generate used aux table id mapping", K(ret), KPC(base_table_schemas.at(i)), K(inc_table_schemas.count()), K(exchange_partition_level), K(is_oracle_mode), K(used_nt_schema_flag.count())); } } } return ret; } -int ObPartitionExchange::generate_local_storage_index_and_lob_table_mapping_(const ObTableSchema &partitioned_table_schema, - ObIArray &non_partitioned_table_schemas, - const ObString &exchange_partition_name, +int ObPartitionExchange::generate_local_storage_index_and_lob_table_mapping_(const ObTableSchema &base_table_schema, + ObIArray &inc_table_schemas, const ObPartitionLevel exchange_partition_level, const bool is_oracle_mode, ObIArray &used_nt_schema_flag) { int ret = OB_SUCCESS; bool find_related_nt_schema = false; - if (OB_UNLIKELY((!partitioned_table_schema.is_index_local_storage() && !partitioned_table_schema.is_aux_lob_table()) || non_partitioned_table_schemas.count() != used_nt_schema_flag.count() || exchange_partition_name.empty() || PARTITION_LEVEL_ZERO == exchange_partition_level)) { + if (OB_UNLIKELY((!base_table_schema.is_index_local_storage() && !base_table_schema.is_aux_lob_table()) || inc_table_schemas.count() != used_nt_schema_flag.count() || PARTITION_LEVEL_ZERO == exchange_partition_level)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(partitioned_table_schema), K(non_partitioned_table_schemas.count()), K(used_nt_schema_flag.count()), K(exchange_partition_name), K(exchange_partition_level)); - } else if (partitioned_table_schema.is_index_local_storage()) { + LOG_WARN("invalid argument", K(ret), K(base_table_schema), K(inc_table_schemas.count()), K(used_nt_schema_flag.count()), K(exchange_partition_level)); + } else if (base_table_schema.is_index_local_storage()) { if (!is_oracle_mode) { - if (OB_FAIL(generate_local_storage_index_table_mapping_in_mysql_mode_(partitioned_table_schema, non_partitioned_table_schemas, exchange_partition_name, exchange_partition_level, used_nt_schema_flag, find_related_nt_schema))) { - LOG_WARN("fail to generate local storage index table mapping in mysql mode", K(ret), K(partitioned_table_schema), K(non_partitioned_table_schemas.count()), K(exchange_partition_name), K(exchange_partition_level), K(used_nt_schema_flag.count()), K(find_related_nt_schema)); + if (OB_FAIL(generate_local_storage_index_table_mapping_in_mysql_mode_(base_table_schema, inc_table_schemas, exchange_partition_level, used_nt_schema_flag, find_related_nt_schema))) { + LOG_WARN("fail to generate local storage index table mapping in mysql mode", K(ret), K(base_table_schema), K(inc_table_schemas.count()), K(exchange_partition_level), K(used_nt_schema_flag.count()), K(find_related_nt_schema)); } else if (!find_related_nt_schema) { ret = OB_TABLES_DIFFERENT_DEFINITIONS; - LOG_WARN("can't find related nt schema in mysql mode", K(ret), K(partitioned_table_schema), K(non_partitioned_table_schemas.count()), K(used_nt_schema_flag.count()), K(find_related_nt_schema)); + LOG_WARN("can't find related nt schema in mysql mode", K(ret), K(base_table_schema), K(inc_table_schemas.count()), K(used_nt_schema_flag.count()), K(find_related_nt_schema)); } - } else if (OB_FAIL(generate_local_storage_index_table_mapping_in_oracle_mode_(partitioned_table_schema, non_partitioned_table_schemas, exchange_partition_name, exchange_partition_level, used_nt_schema_flag, find_related_nt_schema))) { - LOG_WARN("fail to generate local storage index table mapping in oracle mode", K(ret), K(partitioned_table_schema), K(non_partitioned_table_schemas.count()), K(exchange_partition_name), K(exchange_partition_level), K(used_nt_schema_flag.count()), K(find_related_nt_schema)); + } else if (OB_FAIL(generate_local_storage_index_table_mapping_in_oracle_mode_(base_table_schema, inc_table_schemas, exchange_partition_level, used_nt_schema_flag, find_related_nt_schema))) { + LOG_WARN("fail to generate local storage index table mapping in oracle mode", K(ret), K(base_table_schema), K(inc_table_schemas.count()), K(exchange_partition_level), K(used_nt_schema_flag.count()), K(find_related_nt_schema)); } else if (!find_related_nt_schema) { ret = OB_ERR_INDEX_MISMATCH_ALTER_TABLE_EXCHANGE_PARTITION; - LOG_WARN("can't find related nt schema in mysql mode", K(ret), K(partitioned_table_schema), K(non_partitioned_table_schemas.count()), K(used_nt_schema_flag.count()), K(find_related_nt_schema)); + LOG_WARN("can't find related nt schema in mysql mode", K(ret), K(base_table_schema), K(inc_table_schemas.count()), K(used_nt_schema_flag.count()), K(find_related_nt_schema)); } - } else if (OB_FAIL(generate_lob_table_mapping_(partitioned_table_schema, non_partitioned_table_schemas, exchange_partition_name, exchange_partition_level, is_oracle_mode, used_nt_schema_flag, find_related_nt_schema))){ - LOG_WARN("fail to generate lob table mapping", K(ret), K(partitioned_table_schema), K(non_partitioned_table_schemas.count()), K(exchange_partition_name), K(exchange_partition_level), K(is_oracle_mode), K(used_nt_schema_flag.count()), K(find_related_nt_schema)); + } else if (OB_FAIL(generate_lob_table_mapping_(base_table_schema, inc_table_schemas, exchange_partition_level, is_oracle_mode, used_nt_schema_flag, find_related_nt_schema))){ + LOG_WARN("fail to generate lob table mapping", K(ret), K(base_table_schema), K(inc_table_schemas.count()), K(exchange_partition_level), K(is_oracle_mode), K(used_nt_schema_flag.count()), K(find_related_nt_schema)); } else if (!find_related_nt_schema) { if (is_oracle_mode) { ret = OB_ERR_INDEX_MISMATCH_ALTER_TABLE_EXCHANGE_PARTITION; } else { ret = OB_TABLES_DIFFERENT_DEFINITIONS; } - LOG_WARN("can't find related nt_schema", K(ret), K(partitioned_table_schema), K(non_partitioned_table_schemas.count()), K(used_nt_schema_flag), K(is_oracle_mode)); + LOG_WARN("can't find related nt_schema", K(ret), K(base_table_schema), K(inc_table_schemas.count()), K(used_nt_schema_flag), K(is_oracle_mode)); } return ret; } -int ObPartitionExchange::generate_local_storage_index_table_mapping_in_mysql_mode_(const ObTableSchema &partitioned_table_schema, - ObIArray &non_partitioned_table_schemas, - const ObString &exchange_partition_name, +int ObPartitionExchange::generate_local_storage_index_table_mapping_in_mysql_mode_(const ObTableSchema &base_table_schema, + ObIArray &inc_table_schemas, const ObPartitionLevel exchange_partition_level, ObIArray &used_nt_schema_flag, bool &find_related_nt_schema) { int ret = OB_SUCCESS; find_related_nt_schema = false; - if (OB_UNLIKELY(!partitioned_table_schema.is_index_local_storage() || non_partitioned_table_schemas.count() != used_nt_schema_flag.count() || exchange_partition_name.empty() || PARTITION_LEVEL_ZERO == exchange_partition_level)) { + if (OB_UNLIKELY(!base_table_schema.is_index_local_storage() || inc_table_schemas.count() != used_nt_schema_flag.count() || PARTITION_LEVEL_ZERO == exchange_partition_level)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(partitioned_table_schema), K(partitioned_table_schema.is_index_local_storage()), K(non_partitioned_table_schemas.count()), K(used_nt_schema_flag.count()), K(exchange_partition_name), K(exchange_partition_level)); + LOG_WARN("invalid argument", K(ret), K(base_table_schema), K(base_table_schema.is_index_local_storage()), K(inc_table_schemas.count()), K(used_nt_schema_flag.count()), K(exchange_partition_level)); } else { - for (int64_t i = 0; OB_SUCC(ret) && !find_related_nt_schema && i < non_partitioned_table_schemas.count(); i++) { + for (int64_t i = 0; OB_SUCC(ret) && !find_related_nt_schema && i < inc_table_schemas.count(); i++) { ObString pt_index_name; ObString nt_index_name; - if (OB_ISNULL(non_partitioned_table_schemas.at(i))) { + if (OB_ISNULL(inc_table_schemas.at(i))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("table schema is null", K(ret)); - } else if (!non_partitioned_table_schemas.at(i)->is_index_local_storage() || used_nt_schema_flag.at(i)) { - } else if (OB_FAIL(partitioned_table_schema.get_index_name(pt_index_name))) { - LOG_WARN("fail to get index name", K(ret), K(partitioned_table_schema)); - } else if (OB_FAIL(non_partitioned_table_schemas.at(i)->get_index_name(nt_index_name))) { - LOG_WARN("fail to get index name", K(ret), KPC(non_partitioned_table_schemas.at(i))); + } else if (!inc_table_schemas.at(i)->is_index_local_storage() || used_nt_schema_flag.at(i)) { + } else if (OB_FAIL(base_table_schema.get_index_name(pt_index_name))) { + LOG_WARN("fail to get index name", K(ret), K(base_table_schema)); + } else if (OB_FAIL(inc_table_schemas.at(i)->get_index_name(nt_index_name))) { + LOG_WARN("fail to get index name", K(ret), KPC(inc_table_schemas.at(i))); } else if (0 == pt_index_name.compare(nt_index_name)) { - if (OB_FAIL(check_table_conditions_in_common_(partitioned_table_schema, *non_partitioned_table_schemas.at(i), exchange_partition_name, exchange_partition_level, false /*is mysql mode*/))) { - LOG_WARN("fail to check table conditions in common", K(ret), K(partitioned_table_schema), KPC(non_partitioned_table_schemas.at(i)), K(exchange_partition_name), K(exchange_partition_level)); - } else if (OB_FAIL(check_table_all_column_conditions_(partitioned_table_schema, *non_partitioned_table_schemas.at(i), false /*is mysql mode*/))) { - LOG_WARN("fail to check table all column conditions", K(ret), K(partitioned_table_schema.get_table_id()), K(non_partitioned_table_schemas.at(i)->get_table_id())); + if (OB_FAIL(check_table_conditions_in_common_(base_table_schema, *inc_table_schemas.at(i), exchange_partition_level, false /*is mysql mode*/))) { + LOG_WARN("fail to check table conditions in common", K(ret), K(base_table_schema), KPC(inc_table_schemas.at(i)), K(exchange_partition_level)); + } else if (OB_FAIL(check_table_all_column_conditions_(base_table_schema, *inc_table_schemas.at(i), false /*is mysql mode*/))) { + LOG_WARN("fail to check table all column conditions", K(ret), K(base_table_schema.get_table_id()), K(inc_table_schemas.at(i)->get_table_id())); } else { find_related_nt_schema = true; used_nt_schema_flag.at(i) = true; - if (OB_FAIL(used_pt_nt_id_map_.set_refactored(partitioned_table_schema.get_table_id(), non_partitioned_table_schemas.at(i)->get_table_id()))) { - LOG_WARN("fail to set refactored pt nt schema mapping", K(ret), K(partitioned_table_schema), K(non_partitioned_table_schemas.at(i)->get_table_id())); + if (OB_FAIL(used_pt_nt_id_map_.set_refactored(base_table_schema.get_table_id(), inc_table_schemas.at(i)->get_table_id()))) { + LOG_WARN("fail to set refactored pt nt schema mapping", K(ret), K(base_table_schema), K(inc_table_schemas.at(i)->get_table_id())); } } } else { @@ -2082,44 +2196,43 @@ int ObPartitionExchange::generate_local_storage_index_table_mapping_in_mysql_mod return ret; } -int ObPartitionExchange::generate_local_storage_index_table_mapping_in_oracle_mode_(const ObTableSchema &partitioned_table_schema, - ObIArray &non_partitioned_table_schemas, - const ObString &exchange_partition_name, +int ObPartitionExchange::generate_local_storage_index_table_mapping_in_oracle_mode_(const ObTableSchema &base_table_schema, + ObIArray &inc_table_schemas, const ObPartitionLevel exchange_partition_level, ObIArray &used_nt_schema_flag, bool &find_related_nt_schema) { int ret = OB_SUCCESS; find_related_nt_schema = false; - if (OB_UNLIKELY(!partitioned_table_schema.is_index_local_storage() || non_partitioned_table_schemas.count() != used_nt_schema_flag.count() || exchange_partition_name.empty() || PARTITION_LEVEL_ZERO == exchange_partition_level)) { + if (OB_UNLIKELY(!base_table_schema.is_index_local_storage() || inc_table_schemas.count() != used_nt_schema_flag.count() || PARTITION_LEVEL_ZERO == exchange_partition_level)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(partitioned_table_schema), K(partitioned_table_schema.is_index_local_storage()), K(non_partitioned_table_schemas.count()), K(used_nt_schema_flag.count()), K(exchange_partition_name), K(exchange_partition_level)); + LOG_WARN("invalid argument", K(ret), K(base_table_schema), K(base_table_schema.is_index_local_storage()), K(inc_table_schemas.count()), K(used_nt_schema_flag.count()), K(exchange_partition_level)); } else { - for (int64_t i = 0; OB_SUCC(ret) && !find_related_nt_schema && i < non_partitioned_table_schemas.count(); i++) { - if (OB_ISNULL(non_partitioned_table_schemas.at(i))) { + for (int64_t i = 0; OB_SUCC(ret) && !find_related_nt_schema && i < inc_table_schemas.count(); i++) { + if (OB_ISNULL(inc_table_schemas.at(i))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("table schema is null", K(ret)); - } else if (!non_partitioned_table_schemas.at(i)->is_index_local_storage() || used_nt_schema_flag.at(i)) { - } else if (OB_FAIL(check_table_conditions_in_common_(partitioned_table_schema, *non_partitioned_table_schemas.at(i), exchange_partition_name, exchange_partition_level, true /*is oracle mode*/))) { + } else if (!inc_table_schemas.at(i)->is_index_local_storage() || used_nt_schema_flag.at(i)) { + } else if (OB_FAIL(check_table_conditions_in_common_(base_table_schema, *inc_table_schemas.at(i), exchange_partition_level, true /*is oracle mode*/))) { if (in_find_same_aux_table_retry_white_list_(ret)) { - LOG_WARN("all column conditions of exchanging partition tables are not equal, and retry find the matched table", K(ret), K(partitioned_table_schema.get_table_id()), K(non_partitioned_table_schemas.at(i)->get_table_id()), K(exchange_partition_name), K(exchange_partition_level)); + LOG_WARN("all column conditions of exchanging partition tables are not equal, and retry find the matched table", K(ret), K(base_table_schema.get_table_id()), K(inc_table_schemas.at(i)->get_table_id()), K(exchange_partition_level)); ret = OB_SUCCESS; } else { - LOG_WARN("all column conditions of exchanging partition tables are not equal, and ret_code not in in_find_same_aux_table_retry_white_list", K(ret), K(partitioned_table_schema.get_table_id()), K(non_partitioned_table_schemas.at(i)->get_table_id()), K(exchange_partition_name), K(exchange_partition_level)); + LOG_WARN("all column conditions of exchanging partition tables are not equal, and ret_code not in in_find_same_aux_table_retry_white_list", K(ret), K(base_table_schema.get_table_id()), K(inc_table_schemas.at(i)->get_table_id()), K(exchange_partition_level)); } - } else if (OB_FAIL(check_table_all_column_conditions_(partitioned_table_schema, *non_partitioned_table_schemas.at(i), true /*is oracle mode*/))) { - // uncertain if other non_partitioned tables match the partitioned table, so try matching other non_partitioned tables + } else if (OB_FAIL(check_table_all_column_conditions_(base_table_schema, *inc_table_schemas.at(i), true /*is oracle mode*/))) { + // uncertain if other inc tables match the base table, so try matching other inc tables if (in_find_same_aux_table_retry_white_list_(ret)) { - LOG_WARN("all column conditions of exchanging partition tables are not equal, and retry find the matched table", K(ret), K(partitioned_table_schema.get_table_id()), K(non_partitioned_table_schemas.at(i)->get_table_id()), K(exchange_partition_name), K(exchange_partition_level)); + LOG_WARN("all column conditions of exchanging partition tables are not equal, and retry find the matched table", K(ret), K(base_table_schema.get_table_id()), K(inc_table_schemas.at(i)->get_table_id()), K(exchange_partition_level)); ret = OB_SUCCESS; } else { - LOG_WARN("all column conditions of exchanging partition tables are not equal, and ret_code not in in_find_same_aux_table_retry_white_list", K(ret), K(partitioned_table_schema.get_table_id()), K(non_partitioned_table_schemas.at(i)->get_table_id()), K(exchange_partition_name), K(exchange_partition_level)); + LOG_WARN("all column conditions of exchanging partition tables are not equal, and ret_code not in in_find_same_aux_table_retry_white_list", K(ret), K(base_table_schema.get_table_id()), K(inc_table_schemas.at(i)->get_table_id()), K(exchange_partition_level)); } } else { find_related_nt_schema = true; used_nt_schema_flag.at(i) = true; - if (OB_FAIL(used_pt_nt_id_map_.set_refactored(partitioned_table_schema.get_table_id(), non_partitioned_table_schemas.at(i)->get_table_id()))) { - LOG_WARN("fail to set refactored pt nt schema mapping", K(ret), K(partitioned_table_schema), K(non_partitioned_table_schemas.at(i)->get_table_id())); + if (OB_FAIL(used_pt_nt_id_map_.set_refactored(base_table_schema.get_table_id(), inc_table_schemas.at(i)->get_table_id()))) { + LOG_WARN("fail to set refactored pt nt schema mapping", K(ret), K(base_table_schema), K(inc_table_schemas.at(i)->get_table_id())); } } } @@ -2127,9 +2240,8 @@ int ObPartitionExchange::generate_local_storage_index_table_mapping_in_oracle_mo return ret; } -int ObPartitionExchange::generate_lob_table_mapping_(const ObTableSchema &partitioned_table_schema, - ObIArray &non_partitioned_table_schemas, - const ObString &exchange_partition_name, +int ObPartitionExchange::generate_lob_table_mapping_(const ObTableSchema &base_table_schema, + ObIArray &inc_table_schemas, const ObPartitionLevel exchange_partition_level, const bool is_oracle_mode, ObIArray &used_nt_schema_flag, @@ -2138,20 +2250,20 @@ int ObPartitionExchange::generate_lob_table_mapping_(const ObTableSchema &partit int ret = OB_SUCCESS; bool is_equal = false; find_related_nt_schema = false; - if (OB_UNLIKELY(!partitioned_table_schema.is_aux_lob_table() || non_partitioned_table_schemas.count() != used_nt_schema_flag.count() || exchange_partition_name.empty() || PARTITION_LEVEL_ZERO == exchange_partition_level)) { + if (OB_UNLIKELY(!base_table_schema.is_aux_lob_table() || inc_table_schemas.count() != used_nt_schema_flag.count() || PARTITION_LEVEL_ZERO == exchange_partition_level)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(partitioned_table_schema), K(partitioned_table_schema.is_aux_lob_table()), K(non_partitioned_table_schemas.count()), K(used_nt_schema_flag.count()), K(exchange_partition_name), K(exchange_partition_level), K(is_oracle_mode)); + LOG_WARN("invalid argument", K(ret), K(base_table_schema), K(base_table_schema.is_aux_lob_table()), K(inc_table_schemas.count()), K(used_nt_schema_flag.count()), K(exchange_partition_level), K(is_oracle_mode)); } else { - for (int64_t i = 0; OB_SUCC(ret) && !find_related_nt_schema && i < non_partitioned_table_schemas.count(); i++) { - if (OB_ISNULL(non_partitioned_table_schemas.at(i))) { + for (int64_t i = 0; OB_SUCC(ret) && !find_related_nt_schema && i < inc_table_schemas.count(); i++) { + if (OB_ISNULL(inc_table_schemas.at(i))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("table schema is null", K(ret)); - } else if (!non_partitioned_table_schemas.at(i)->is_aux_lob_table() || used_nt_schema_flag.at(i)) { - } else if ((partitioned_table_schema.is_aux_lob_meta_table() && non_partitioned_table_schemas.at(i)->is_aux_lob_meta_table()) || (partitioned_table_schema.is_aux_lob_piece_table() && non_partitioned_table_schemas.at(i)->is_aux_lob_piece_table())) { + } else if (!inc_table_schemas.at(i)->is_aux_lob_table() || used_nt_schema_flag.at(i)) { + } else if ((base_table_schema.is_aux_lob_meta_table() && inc_table_schemas.at(i)->is_aux_lob_meta_table()) || (base_table_schema.is_aux_lob_piece_table() && inc_table_schemas.at(i)->is_aux_lob_piece_table())) { find_related_nt_schema = true; used_nt_schema_flag.at(i) = true; - if (OB_FAIL(used_pt_nt_id_map_.set_refactored(partitioned_table_schema.get_table_id(), non_partitioned_table_schemas.at(i)->get_table_id()))) { - LOG_WARN("fail to set refactored pt nt schema mapping", K(ret), K(partitioned_table_schema), K(non_partitioned_table_schemas.at(i)->get_table_id())); + if (OB_FAIL(used_pt_nt_id_map_.set_refactored(base_table_schema.get_table_id(), inc_table_schemas.at(i)->get_table_id()))) { + LOG_WARN("fail to set refactored pt nt schema mapping", K(ret), K(base_table_schema), K(inc_table_schemas.at(i)->get_table_id())); } } } @@ -2194,7 +2306,7 @@ int ObPartitionExchange::update_index_status_(const uint64_t tenant_id, } int ObPartitionExchange::build_single_table_rw_defensive_(const uint64_t tenant_id, - const ObArray &tablet_ids, + const ObIArray &tablet_ids, const int64_t schema_version, ObDDLSQLTransaction &trans) { @@ -2237,7 +2349,7 @@ int ObPartitionExchange::build_single_table_rw_defensive_(const uint64_t tenant_ } int ObPartitionExchange::build_modify_tablet_binding_args_v1_(const uint64_t tenant_id, - const ObArray &tablet_ids, + const ObIArray &tablet_ids, const int64_t schema_version, ObIArray &modify_args, ObDDLSQLTransaction &trans) @@ -2282,7 +2394,7 @@ int ObPartitionExchange::build_modify_tablet_binding_args_v1_(const uint64_t ten } int ObPartitionExchange::get_tablets_(const uint64_t tenant_id, - const ObArray &tablet_ids, + const ObIArray &tablet_ids, ObIArray &tablets, ObDDLSQLTransaction &trans) { @@ -2299,7 +2411,7 @@ int ObPartitionExchange::get_tablets_(const uint64_t tenant_id, LOG_WARN("invalid tablet ids ls ids", K(ret)); } for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); i++) { - if (OB_FAIL(tablets.push_back({ls_ids[i], tablet_ids[i]}))) { + if (OB_FAIL(tablets.push_back({ls_ids.at(i), tablet_ids.at(i)}))) { LOG_WARN("failed to push back tablet id and ls id", K(ret)); } } @@ -2312,7 +2424,10 @@ int ObPartitionExchange::adapting_cdc_changes_in_exchange_partition_(const uint6 ObDDLSQLTransaction &trans) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || OB_INVALID_ID == partitioned_table_id || OB_INVALID_ID == non_partitioned_table_id)) { + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObPartitionExchange not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || OB_INVALID_ID == partitioned_table_id || OB_INVALID_ID == non_partitioned_table_id)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(tenant_id), K(partitioned_table_id), K(non_partitioned_table_id)); } else if (OB_FAIL(used_pt_nt_id_map_.set_refactored(partitioned_table_id, non_partitioned_table_id))) { @@ -2326,19 +2441,31 @@ int ObPartitionExchange::adapting_cdc_changes_in_exchange_partition_(const uint6 arg.inc_table_id_ = non_partitioned_table_id; common::hash::ObHashMap::iterator iter_table; for (iter_table = used_pt_nt_id_map_.begin(); OB_SUCC(ret) && iter_table != used_pt_nt_id_map_.end(); ++iter_table) { - ObTabletID tmp_tablet; - if (OB_FAIL(arg.table_ids_.push_back(iter_table->first))) { - LOG_WARN("failed to push back table id", K(ret), K(iter_table->first)); - } else if (OB_FAIL(used_table_to_tablet_id_map_.get_refactored(iter_table->second, tmp_tablet))) { - LOG_WARN("get_refactored tablet id from used_table_to_tablet_id_map failed", K(ret), K(iter_table->second), K(tmp_tablet)); - } else if (OB_FAIL(arg.tablet_ids_.push_back(tmp_tablet))) { - LOG_WARN("failed to push back tablet id", K(ret), K(tmp_tablet)); - } else if (OB_FAIL(arg.table_ids_.push_back(iter_table->second))) { - LOG_WARN("failed to push back table id", K(ret), K(iter_table->second)); - } else if (OB_FAIL(used_table_to_tablet_id_map_.get_refactored(iter_table->first, tmp_tablet))) { - LOG_WARN("get_refactored tablet id from used_table_to_tablet_id_map failed", K(ret), K(iter_table->first), K(tmp_tablet)); - } else if (OB_FAIL(arg.tablet_ids_.push_back(tmp_tablet))) { - LOG_WARN("failed to push back tablet id", K(ret), K(tmp_tablet)); + ObArray tmp_pt_tablet_ids; + ObArray tmp_npt_tablet_ids; + if (OB_FAIL(used_table_to_tablet_ids_map_.get_refactored(iter_table->second, tmp_npt_tablet_ids))) { + LOG_WARN("get_refactored tablet id from used_table_to_tablet_id_map failed", K(ret), K(iter_table->second)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && (i < tmp_npt_tablet_ids.count()); ++i) { + if (OB_FAIL(arg.table_ids_.push_back(iter_table->first))) { + LOG_WARN("failed to push back table id", K(ret), K(iter_table->first)); + } else if (OB_FAIL(arg.tablet_ids_.push_back(tmp_npt_tablet_ids.at(i)))) { + LOG_WARN("failed to push back tablet id", K(ret), K(tmp_npt_tablet_ids.at(i))); + } + } + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(used_table_to_tablet_ids_map_.get_refactored(iter_table->first, tmp_pt_tablet_ids))) { + LOG_WARN("get_refactored tablet id from used_table_to_tablet_id_map failed", K(ret), K(iter_table->first)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && (i < tmp_pt_tablet_ids.count()); ++i) { + if (OB_FAIL(arg.table_ids_.push_back(iter_table->second))) { + LOG_WARN("failed to push back table id", K(ret), K(iter_table->second)); + } else if (OB_FAIL(arg.tablet_ids_.push_back(tmp_pt_tablet_ids.at(i)))) { + LOG_WARN("failed to push back tablet id", K(ret), K(tmp_pt_tablet_ids.at(i))); + } + } } } if (OB_SUCC(ret)) { @@ -2655,6 +2782,260 @@ int ObPartitionExchange::get_object_id_from_partition_schema_(ObPartitionSchema return ret; } +int ObPartitionExchange::get_part_by_tablet_id( + const ObTableSchema &table_schema, + const ObTabletID &tablet_id, + const ObPartition *&part, + const ObSubPartition *&subpart, + const bool get_subpart) +{ + int ret = OB_SUCCESS; + int64_t part_id = OB_INVALID_INDEX; + int64_t part_idx = OB_INVALID_INDEX; + int64_t subpart_id = OB_INVALID_INDEX; + int64_t subpart_idx = OB_INVALID_INDEX; + const ObPartition *tmp_part = nullptr; + const ObSubPartition *tmp_subpart = nullptr; + if (OB_UNLIKELY(!tablet_id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tablet id", KR(ret), K(tablet_id)); + } else if (OB_FAIL(table_schema.get_part_id_by_tablet(tablet_id, part_id, subpart_id))) { + LOG_WARN("failed to get part id by tablet", KR(ret)); + } else if (OB_INVALID_INDEX == part_id) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected part id", KR(ret), K(part_id)); + } else if (OB_FAIL(table_schema.get_partition_by_part_id(part_id, CHECK_PARTITION_MODE_NORMAL, tmp_part))) { + LOG_WARN("failed to get partition by part id", KR(ret), K(part_id)); + } else if (OB_ISNULL(tmp_part)) { + ret = OB_PARTITION_NOT_EXIST; + LOG_WARN("partition not exist", KR(ret), KP(tmp_part)); + } else if (OB_FALSE_IT(part = tmp_part)) { + } else if (get_subpart) { + if (OB_INVALID_INDEX == subpart_id) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected subpart id", KR(ret), K(subpart_id)); + } else if (OB_ISNULL(tmp_part->get_subpart_array())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null subpart array", KR(ret), KP(tmp_part->get_subpart_array())); + } else if (OB_FAIL(table_schema.get_part_idx_by_tablet(tablet_id, part_idx, subpart_idx))) { + LOG_WARN("failed to get part idx by tablet", KR(ret), K(tablet_id)); + } else if ((OB_INVALID_INDEX == part_idx) || (OB_INVALID_INDEX == subpart_idx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected part idx", KR(ret), K(part_idx), K(subpart_idx)); + } else if (OB_ISNULL(tmp_subpart = tmp_part->get_subpart_array()[subpart_idx])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null subpart", KR(ret), KPC(tmp_part), K(subpart_idx), KP(tmp_subpart)); + } else { + subpart = tmp_subpart; + } + } + return ret; +} + +int ObPartitionExchange::init_alter_table_part_schema( + const ObTableSchema &table_schema, + AlterTableSchema &alter_table_schema) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(alter_table_schema.assign(table_schema))) { + LOG_WARN("fail to assign partitioned table schema", K(ret), K(table_schema)); + } else if (FALSE_IT(alter_table_schema.reset_partition_schema())) { + } else if (FALSE_IT(alter_table_schema.set_part_level(table_schema.get_part_level()))) { + } else if (OB_FAIL(alter_table_schema.get_sub_part_option().assign(table_schema.get_sub_part_option()))) { + LOG_WARN("fail to assign sub part option", K(ret), K(table_schema.get_sub_part_option())); + } else if (OB_FAIL(alter_table_schema.get_part_option().assign(table_schema.get_part_option()))) { + LOG_WARN("fail to assign part option", K(ret), K(table_schema.get_part_option())); + } + return ret; +} + +int ObPartitionExchange::generate_alter_table_part_schema( + const ObTableSchema &table_schema, + const ObPartition *part, + const ObSubPartition *subpart, + const bool is_subpart, + AlterTableSchema &alter_table_schema) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(part)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("part cannot be null", KR(ret), KP(part)); + } else if (is_subpart && OB_ISNULL(subpart)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("subpart cannot be null", KR(ret), KP(subpart), K(is_subpart)); + } else { + const ObPartition *target_part = nullptr; + HEAP_VAR(ObPartition, dummy_part) { + if (is_subpart) { + if (OB_FALSE_IT(dummy_part.set_part_id(part->get_part_id()))) { + } else if (OB_FAIL(dummy_part.set_part_name(part->get_part_name()))) { + LOG_WARN("failed to set part name", K(ret), K(part->get_part_name())); + } else if (OB_FAIL(dummy_part.add_partition(*subpart))){ + LOG_WARN("failed to add subpart", K(ret), K(*subpart)); + } else if (FALSE_IT(dummy_part.set_sub_part_num(dummy_part.get_subpartition_num()))) { + } else { + target_part = &dummy_part; + } + } else { + target_part = part; + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(alter_table_schema.add_partition(*target_part))) { + LOG_WARN("fail to add partition", K(ret), KPC(target_part)); + } else if (FALSE_IT(alter_table_schema.set_part_num(alter_table_schema.get_partition_num()))) { + } + } + } + return ret; +} + +int ObPartitionExchange::add_table_to_tablet_ids_map( + const uint64_t table_id, + const ObTabletID &tablet_id) +{ + int ret = OB_SUCCESS; + ObArray tablet_ids; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObPartitionExchange not init", KR(ret), KP(this)); + } else if (OB_FAIL(used_table_to_tablet_ids_map_.get_refactored(table_id, tablet_ids))) { + if (OB_HASH_NOT_EXIST == ret) { + ret = OB_SUCCESS; + } else { + LOG_WARN("failed to get refactored from used_table_to_tablet_ids_map", KR(ret), K(table_id)); + } + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(tablet_ids.push_back(tablet_id))) { + LOG_WARN("failed to add tablet id", KR(ret), K(tablet_id)); + } else if (OB_FAIL(used_table_to_tablet_ids_map_.set_refactored(table_id, tablet_ids, 1/*overwrite*/))) { + LOG_WARN("failed to set refactored to used_table_to_tablet_ids_map", KR(ret), K(table_id), K(tablet_ids)); + } + return ret; +} + +int ObPartitionExchange::add_table_to_tablet_ids_map( + const uint64_t table_id, + const ObIArray &inc_tablet_ids) +{ + int ret = OB_SUCCESS; + ObArray orig_tablet_ids; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObPartitionExchange not init", KR(ret), KP(this)); + } else if (OB_FAIL(used_table_to_tablet_ids_map_.get_refactored(table_id, orig_tablet_ids))) { + if (OB_HASH_NOT_EXIST == ret) { + ret = OB_SUCCESS; + } else { + LOG_WARN("failed to get refactored from used_table_to_tablet_ids_map", KR(ret), K(table_id)); + } + } + + if (OB_SUCC(ret)) { + for (int64_t i = 0; OB_SUCC(ret) && (i < inc_tablet_ids.count()); ++i) { + if (OB_FAIL(orig_tablet_ids.push_back(inc_tablet_ids.at(i)))) { + LOG_WARN("failed to add tablet id", KR(ret), K(inc_tablet_ids.at(i))); + } + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(used_table_to_tablet_ids_map_.set_refactored(table_id, orig_tablet_ids, 1/*overwrite*/))) { + LOG_WARN("failed to set refactored to used_table_to_tablet_ids_map", KR(ret), K(table_id), K(orig_tablet_ids)); + } + } + return ret; +} + +int ObPartitionExchange::get_and_check_aux_tablet_id( + const ObPartition *data_part, + const ObSubPartition *data_subpart, + const ObTableSchema &aux_table_schema, + const bool is_oracle_mode, + const bool is_subpart, + ObTabletID &aux_tablet_id) +{ + int ret = OB_SUCCESS; + const ObPartition *part = nullptr; + const ObSubPartition *subpart = nullptr; + if (!aux_table_schema.is_aux_table()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid table type", KR(ret), K(aux_table_schema.is_aux_table())); + } else if (OB_ISNULL(data_part) || !data_part->is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid data partition", KR(ret), KP(data_part)); + } else if (is_subpart && (OB_ISNULL(data_subpart) || !data_subpart->is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid data sub partition", KR(ret), KP(data_subpart)); + } else { + bool is_matched = false; + const int64_t data_part_idx = data_part->get_part_idx(); + const schema::ObPartitionOption &pt_part_option = aux_table_schema.get_part_option(); + schema::ObPartitionFuncType pt_part_func_type = pt_part_option.get_part_func_type(); + if (OB_FAIL(aux_table_schema.get_partition_by_partition_index(data_part_idx, CHECK_PARTITION_MODE_NORMAL, part))) { + LOG_WARN("failed to get partition by partition index", KR(ret), K(data_part_idx), K(aux_table_schema)); + } else if (OB_ISNULL(part)) { + ret = OB_PARTITION_NOT_EXIST; + LOG_WARN("partition not found", KR(ret), K(data_part_idx), K(aux_table_schema)); + } else if (OB_FAIL(ddl_service_.check_same_partition(is_oracle_mode, *data_part, *part, pt_part_func_type, is_matched))) { + LOG_WARN("fail to check ori_table_part and ori_aux_part is the same", + KR(ret), K(is_oracle_mode), KPC(data_part), KPC(part), K(pt_part_func_type)); + } else if (OB_UNLIKELY(!is_matched)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("part with the same offset not equal, maybe not the right index", KR(ret), K(data_part), KPC(part)); + } else if (is_subpart) { + const int64_t data_subpart_idx = data_subpart->get_sub_part_idx(); + const schema::ObPartitionOption &pt_subpart_option = aux_table_schema.get_sub_part_option(); + schema::ObPartitionFuncType pt_subpart_func_type = pt_subpart_option.get_sub_part_func_type(); + is_matched = false; + if (OB_FAIL(part->get_normal_subpartition_by_subpartition_index(data_subpart_idx, subpart))) { + LOG_WARN("fail to get src subpart by subpart index", K(ret), K(data_subpart_idx)); + } else if (OB_ISNULL(subpart)) { + ret = OB_PARTITION_NOT_EXIST; + LOG_WARN("partition not found", K(ret), K(part), K(data_subpart_idx), K(aux_table_schema)); + } else if (OB_FAIL(ddl_service_.check_same_subpartition(is_oracle_mode, *data_subpart, *subpart, pt_subpart_func_type, is_matched))) { + LOG_WARN("fail to check ori_table_subpart and ori_aux_subpart is the same", K(ret), K(is_oracle_mode), KPC(data_subpart), KPC(subpart), K(pt_subpart_func_type)); + } else if (!is_matched) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("part with the same offset not equal, maybe not the right index", K(ret), KPC(data_subpart), KPC(subpart)); + } else { + aux_tablet_id = subpart->get_tablet_id(); + } + } else { + aux_tablet_id = part->get_tablet_id(); + } + } + return ret; +} + +int ObPartitionExchange::ddl_exchange_table_partitions( + const ObTableSchema &orig_table_schema, + ObTableSchema &inc_table_schema, + ObTableSchema &del_table_schema, + ObDDLOperator &ddl_operator, + ObMySQLTransaction &trans, + const bool is_subpartition) +{ + int ret = OB_SUCCESS; + if (is_subpartition) { + if (OB_FAIL(ddl_operator.exchange_table_subpartitions(orig_table_schema, + inc_table_schema, + del_table_schema, + trans))) { + LOG_WARN("failed to exchange table subpartitions", KR(ret)); + } + } else { + if (OB_FAIL(ddl_operator.exchange_table_partitions(orig_table_schema, + inc_table_schema, + del_table_schema, + trans))) { + LOG_WARN("failed to exchange table partitions", KR(ret)); + } + } + return ret; +} + bool ObChangeTabletToTableArg::is_valid() const { return OB_INVALID_TENANT_ID != tenant_id_ && ls_id_.is_valid() && OB_INVALID_ID != base_table_id_ && OB_INVALID_ID != inc_table_id_ && tablet_ids_.count() > 0 && table_ids_.count() > 0 && tablet_ids_.count() == table_ids_.count(); } diff --git a/src/rootserver/ob_partition_exchange.h b/src/rootserver/ob_partition_exchange.h index 46655529b..4ed04f602 100644 --- a/src/rootserver/ob_partition_exchange.h +++ b/src/rootserver/ob_partition_exchange.h @@ -18,6 +18,7 @@ #include "share/ob_rpc_struct.h" #include "share/schema/ob_schema_struct.h" #include "storage/tablet/ob_tablet_binding_helper.h" +#include "share/stat/ob_stat_define.h" namespace oceanbase { @@ -33,25 +34,40 @@ namespace rootserver { class ObDDLService; class ObDDLSQLTransaction; -class ObPartitionExchange final +class ObPartitionExchange { public: typedef std::pair LSTabletID; explicit ObPartitionExchange(ObDDLService &ddl_service, const uint64_t data_version); - ~ObPartitionExchange(); + virtual ~ObPartitionExchange(); int check_and_exchange_partition(const obrpc::ObExchangePartitionArg &arg, obrpc::ObAlterTableRes &res, ObSchemaGetterGuard &schema_guard); -private: + +protected: int check_partition_exchange_conditions_(const obrpc::ObExchangePartitionArg &arg, const ObTableSchema &base_table_schema, const ObTableSchema &inc_table_schema, const bool is_oracle_mode, ObSchemaGetterGuard &schema_guard); int do_exchange_partition_(const obrpc::ObExchangePartitionArg &arg, obrpc::ObAlterTableRes &res, const ObTableSchema &base_table_schema, const ObTableSchema &inc_table_schema, const bool is_oracle_mode, ObSchemaGetterGuard &schema_guard); int lock_exchange_data_table_and_partition_(const uint64_t tenant_id, const ObTableSchema &partitioned_table_schema, const ObTableSchema &non_partitioned_table_schema, const common::ObTabletID &tablet_id, ObDDLSQLTransaction &trans); - int check_data_table_partition_exchange_conditions_(const ObTableSchema &base_table_schema, const ObTableSchema &inc_table_schema, const ObString &exchange_partition_name, const ObPartitionLevel exchange_partition_level,const bool is_oracle_mode); + int check_data_table_partition_exchange_conditions_(const ObTableSchema &base_table_schema, + const ObTableSchema &inc_table_schema, + const ObIArray &base_tablet_ids, + const ObIArray &inc_tablet_ids, + const ObPartitionLevel exchange_partition_level, + const bool is_oracle_mode); // table level conditions that need to be checked for partition exchange in mysql mode and oracle mode - int check_table_conditions_in_common_(const ObTableSchema &base_table_schema, const ObTableSchema &inc_table_schema, const ObString &exchange_partition_name, const ObPartitionLevel exchange_partition_level, const bool is_oracle_mode); + virtual int check_table_conditions_in_common_(const ObTableSchema &base_table_schema, + const ObTableSchema &inc_table_schema, + const ObPartitionLevel exchange_partition_level, + const bool is_oracle_mode); // table level conditions that need to be checked for partition exchange in mysql mode int check_table_conditions_in_mysql_mode_(const ObTableSchema &base_table_schema, const ObTableSchema &inc_table_schema); // table level conditions that need to be checked for partition exchange in oracle mode int check_table_conditions_in_oracle_mode_(const ObTableSchema &base_table_schema, const ObTableSchema &inc_table_schema); - int check_partition_and_table_tablespace_(const ObTableSchema &base_table_schema, const ObTableSchema &inc_table_schema, const ObString &exchange_partition_name, const ObPartitionLevel exchange_partition_level, const bool is_oracle_mode); + int check_tablespace_(const ObTableSchema &base_table_schema, + const ObTableSchema &inc_table_schema, + const bool is_oracle_mode); + int check_data_table_partitions_and_tablespace_(const ObTableSchema &table_schema, + const ObIArray &tablet_ids, + const ObPartitionLevel exchange_partition_level); + int check_table_index_infos_(const ObTableSchema &base_table_schema, const ObTableSchema &inc_table_schema, const bool is_oracle_mode); int check_table_lob_infos_(const ObTableSchema &base_table_schema, const ObTableSchema &inc_table_schema, const bool is_oracle_mode); int check_table_rowkey_infos_(const ObTableSchema &base_table_schema, const ObTableSchema &inc_table_schema, const bool is_oracle_mode); @@ -83,83 +99,62 @@ private: ObDDLOperator &ddl_operator, ObDDLSQLTransaction &trans, ObSchemaGetterGuard &schema_guard); - int get_data_partition_and_index_(const ObTableSchema &partitioned_data_table_schema, const ObString &data_part_name, const ObPartition *&data_part, int64_t &data_partition_index); - int get_data_subpartition_and_index_(const ObTableSchema &partitioned_data_table_schema, - const ObString &data_subpart_name, - const ObPartition *&data_part, - const ObSubPartition *&data_subpart, - int64_t &data_partition_index, - int64_t &data_subpartition_index); + int get_and_check_data_partition_by_name(const ObTableSchema &partitioned_data_table_schema, + const ObString &data_part_name, + const ObPartition *&data_part); + int get_and_check_data_subpartition_by_name(const ObTableSchema &partitioned_data_table_schema, + const ObString &data_subpart_name, + const ObPartition *&data_part, + const ObSubPartition *&data_subpart); int exchange_data_table_partition_(const uint64_t tenant_id, - const ObTableSchema &partitioned_table_schema, - const ObTableSchema &non_partitioned_table_schema, - const ObPartition &part, + const ObTableSchema &base_table_schema, + const ObTableSchema &inc_table_schema, + const common::ObTabletID &tablet_id, + const common::ObTabletID &inc_tablet_id, const bool is_oracle_mode, + const bool is_subpartition, ObDDLOperator &ddl_operator, ObDDLSQLTransaction &trans, ObSchemaGetterGuard &schema_guard); - int exchange_data_table_subpartition_(const uint64_t tenant_id, - const ObTableSchema &partitioned_table_schema, - const ObTableSchema &non_partitioned_table_schema, - const ObPartition &part, - const ObSubPartition &subpart, - const bool is_oracle_mode, - ObDDLOperator &ddl_operator, - ObDDLSQLTransaction &trans, - ObSchemaGetterGuard &schema_guard); int exchange_auxiliary_table_partition_(const uint64_t tenant_id, - const int64_t ori_data_partition_index, - const ObPartition &ori_data_part, + const ObTableSchema &base_data_table_schema, + const ObTableSchema &inc_data_table_schema, + const common::ObTabletID &data_tablet_id, + const common::ObTabletID &inc_data_tablet_id, const bool is_oracle_mode, + const bool is_subpartition, ObDDLOperator &ddl_operator, ObDDLSQLTransaction &trans, ObSchemaGetterGuard &schema_guard); - int exchange_auxiliary_table_subpartition_(const uint64_t tenant_id, - const int64_t ori_data_partition_index, - const int64_t ori_data_subpartition_index, - const ObPartition &ori_data_part, - const ObSubPartition &ori_data_subpart, - const bool is_oracle_mode, - ObDDLOperator &ddl_operator, - ObDDLSQLTransaction &trans, - ObSchemaGetterGuard &schema_guard); int exchange_partition_map_relationship_(const uint64_t tenant_id, - const ObPartition &part, - const ObTableSchema &partitioned_table_schema, - const ObTableSchema &non_partitioned_table_schema, + const ObTableSchema &base_table_schema, + const ObTableSchema &inc_table_schema, + const ObIArray &base_tablet_ids, + const ObIArray &inc_tablet_id, const bool is_oracle_mode, + const bool is_subpartition, ObDDLOperator &ddl_operator, ObDDLSQLTransaction &trans, ObSchemaGetterGuard &schema_guard); - int exchange_subpartition_map_relationship_(const uint64_t tenant_id, - const ObPartition &part, - const ObSubPartition &subpart, - const ObTableSchema &partitioned_table_schema, - const ObTableSchema &non_partitioned_table_schema, - const bool is_oracle_mode, - ObDDLOperator &ddl_operator, - ObDDLSQLTransaction &trans, - ObSchemaGetterGuard &schema_guard); int update_exchange_table_non_schema_attributes_(const uint64_t tenant_id, - const int64_t old_partition_id, - const int64_t new_partition_id, - const bool is_exchange_subpartition, - const ObTableSchema &partitioned_table_schema, - const ObTableSchema &non_partitioned_table_schema, - const ObIArray &exchange_table_ids, - const ObIArray &exchange_tablet_ids, + const ObTableSchema &old_table_schema, + const ObIArray &old_tablet_ids, + const ObIArray &old_partition_ids, + const ObIArray &new_partition_ids, + const uint64_t new_table_id, + const common::StatLevel new_stat_level, const bool is_oracle_mode, ObDDLOperator &ddl_operator, ObDDLSQLTransaction &trans, ObSchemaGetterGuard &schema_guard); int update_exchange_table_level_attributes_(const uint64_t tenant_id, - const ObIArray &exchange_table_ids, - const ObIArray &exchange_tablet_ids, + const ObIArray &base_tablet_ids, + const ObIArray &inc_tablet_ids, ObTableSchema &partitioned_table_schema, ObTableSchema &non_partitioned_table_schema, ObDDLSQLTransaction &trans); - int update_table_to_tablet_id_mapping_(const uint64_t tenant_id, - const ObIArray &table_ids, + int update_table_to_tablet_ids_mapping_(const uint64_t tenant_id, + const uint64_t table_id, const ObIArray &tablet_ids, ObDDLSQLTransaction &trans); int refresh_table_schema_version_(const uint64_t tenant_id, ObTableSchema &table_schema); @@ -172,9 +167,9 @@ private: int64_t &new_schema_version, ObDDLSQLTransaction &trans); int get_local_storage_index_and_lob_table_schemas_(const ObTableSchema &table_schema, - const bool is_pt_schema, const bool is_oracle_mode, ObIArray &table_schemas, + ObIArray &unused_index_ids, ObSchemaGetterGuard &schema_guard); int check_auxiliary_schema_conditions_(const ObTableSchema *table_schema, const bool is_oracle_mode); int compare_column_extended_type_info_(const common::ObIArray &l_extended_type_info, @@ -183,33 +178,28 @@ private: bool in_supported_table_type_white_list_(const ObTableSchema &table_schema); // generate corresponding auxiliary table mapping that need to exchange partitions bool in_find_same_aux_table_retry_white_list_(const int ret_code); - int generate_auxiliary_table_mapping_(const ObTableSchema &partitioned_data_table_schema, - const ObTableSchema &non_partitioned_data_table_schema, - const ObString &exchange_partition_name, + int generate_auxiliary_table_mapping_(const ObTableSchema &base_data_table_schema, + const ObTableSchema &inc_data_table_schema, const ObPartitionLevel exchange_partition_level, const bool is_oracle_mode, ObSchemaGetterGuard &schema_guard); - int generate_local_storage_index_and_lob_table_mapping_(const ObTableSchema &partitioned_table_schema, - ObIArray &non_partitioned_table_schemas, - const ObString &exchange_partition_name, + int generate_local_storage_index_and_lob_table_mapping_(const ObTableSchema &base_table_schema, + ObIArray &inc_table_schemas, const ObPartitionLevel exchange_partition_level, const bool is_oracle_mode, ObIArray &used_nt_schema_flag); - int generate_local_storage_index_table_mapping_in_mysql_mode_(const ObTableSchema &partitioned_table_schema, - ObIArray &non_partitioned_table_schemas, - const ObString &exchange_partition_name, + int generate_local_storage_index_table_mapping_in_mysql_mode_(const ObTableSchema &base_table_schema, + ObIArray &inc_table_schemas, const ObPartitionLevel exchange_partition_level, ObIArray &used_nt_schema_flag, bool &find_related_nt_schema); - int generate_local_storage_index_table_mapping_in_oracle_mode_(const ObTableSchema &partitioned_table_schema, - ObIArray &non_partitioned_table_schemas, - const ObString &exchange_partition_name, + int generate_local_storage_index_table_mapping_in_oracle_mode_(const ObTableSchema &base_table_schema, + ObIArray &inc_table_schemas, const ObPartitionLevel exchange_partition_level, ObIArray &used_nt_schema_flag, bool &find_related_nt_schema); - int generate_lob_table_mapping_(const ObTableSchema &partitioned_table_schema, - ObIArray &non_partitioned_table_schemas, - const ObString &exchange_partition_name, + int generate_lob_table_mapping_(const ObTableSchema &base_table_schema, + ObIArray &inc_table_schemas, const ObPartitionLevel exchange_partition_level, const bool is_oracle_mode, ObIArray &used_nt_schema_flag, @@ -223,16 +213,16 @@ private: ObSchemaGetterGuard &schema_guard); // Register MDS for read and write defense verification after single table ddl int build_single_table_rw_defensive_(const uint64_t tenant_id, - const ObArray &tablet_ids, + const ObIArray &tablet_ids, const int64_t schema_version, ObDDLSQLTransaction &trans); int build_modify_tablet_binding_args_v1_(const uint64_t tenant_id, - const ObArray &tablet_ids, + const ObIArray &tablet_ids, const int64_t schema_version, ObIArray &modify_args, ObDDLSQLTransaction &trans); int get_tablets_(const uint64_t tenant_id, - const ObArray &tablet_ids, + const ObIArray &tablet_ids, ObIArray &tablets, ObDDLSQLTransaction &trans); int adapting_cdc_changes_in_exchange_partition_(const uint64_t tenant_id, @@ -261,13 +251,93 @@ private: ObDDLSQLTransaction &trans); int update_table_all_monitor_modified_(const uint64_t tenant_id, const uint64_t new_table_id, const ObTabletID &tablet_id, const ObTableSchema &orig_table_schema, ObDDLSQLTransaction &trans); int get_object_id_from_partition_schema_(ObPartitionSchema &partition_schema, const bool get_subpart_only, int64_t &object_id); -private: + +protected: + int inner_init(const ObTableSchema &base_table_schema, + const ObTableSchema &inc_table_schema, + const ObPartitionLevel exchange_partition_level, + const bool is_oracle_mode, + ObSchemaGetterGuard &schema_guard); + int exchange_data_table_partitions(const uint64_t tenant_id, + const ObTableSchema &base_table_schema, + const ObTableSchema &inc_table_schema, + const ObIArray &base_tablet_ids, + const ObIArray &inc_tablet_ids, + const bool is_oracle_mode, + const bool is_subpartition, + ObDDLOperator &ddl_operator, + ObDDLSQLTransaction &trans, + ObSchemaGetterGuard &schema_guard); + int exchange_auxiliary_table_partitions(const uint64_t tenant_id, + const ObTableSchema &base_data_table_schema, + const ObTableSchema &inc_data_table_schema, + const ObIArray &data_tablet_ids, + const ObIArray &inc_data_tablet_ids, + const bool is_oracle_mode, + const bool is_subpartition, + ObDDLOperator &ddl_operator, + ObDDLSQLTransaction &trans, + ObSchemaGetterGuard &schema_guard); + int generate_alter_table_part_schema_for_pt(const ObTableSchema &base_table_schema, + const ObTableSchema &inc_table_schema, + const ObIArray &base_tablet_ids, + const ObIArray &inc_tablet_ids, + const bool is_subpartition, + AlterTableSchema &alter_pt_drop_part_schema, + AlterTableSchema &alter_pt_add_new_part_schema, + AlterTableSchema &alter_inc_drop_new_part_schema, + AlterTableSchema &alter_inc_add_part_schema, + ObIArray &old_base_part_ids, + ObIArray &new_base_part_ids, + ObIArray &old_inc_part_ids, + ObIArray &new_inc_part_ids); + int generate_alter_table_part_schema_for_npt(const ObTableSchema &base_table_schema, + const ObTableSchema &inc_table_schema, + const ObTabletID &base_tablet_id, + const ObTabletID &inc_tablet_id, + const bool is_subpartition, + AlterTableSchema &alter_pt_drop_part_schema, + AlterTableSchema &alter_pt_add_new_part_schema, + ObIArray &old_base_part_ids, + ObIArray &new_base_part_ids, + ObIArray &old_inc_part_ids, + ObIArray &new_inc_part_ids); + int get_part_by_tablet_id(const ObTableSchema &table_schema, + const ObTabletID &tablet_id, + const ObPartition *&part, + const ObSubPartition *&subpart, + const bool get_subpart); + int init_alter_table_part_schema(const ObTableSchema &table_schema, + AlterTableSchema &alter_table_schema); + int generate_alter_table_part_schema(const ObTableSchema &table_schema, + const ObPartition *part, + const ObSubPartition *subpart, + const bool is_subpart, + AlterTableSchema &alter_table_schema); + int add_table_to_tablet_ids_map(const uint64_t table_id, const ObTabletID &tablet_id); + int add_table_to_tablet_ids_map(const uint64_t table_id, const ObIArray &inc_tablet_ids); + int get_and_check_aux_tablet_id(const ObPartition *data_part, + const ObSubPartition *data_subpart, + const ObTableSchema &aux_table_schema, + const bool is_oracle_mode, + const bool is_subpart, + ObTabletID &aux_tablet_id); + int ddl_exchange_table_partitions(const share::schema::ObTableSchema &orig_table_schema, + share::schema::ObTableSchema &inc_table_schema, + share::schema::ObTableSchema &del_table_schema, + ObDDLOperator &ddl_operator, + common::ObMySQLTransaction &trans, + const bool is_subpartition); + +protected: ObDDLService &ddl_service_; uint64_t data_version_; +private: common::hash::ObHashMap used_pt_nt_id_map_; - common::hash::ObHashMap used_table_to_tablet_id_map_; + common::hash::ObHashMap> used_table_to_tablet_ids_map_; common::ObSArray unused_pt_index_id_; common::ObSArray unused_nt_index_id_; + bool is_inited_; private: DISALLOW_COPY_AND_ASSIGN(ObPartitionExchange); }; diff --git a/src/rootserver/ob_root_service.cpp b/src/rootserver/ob_root_service.cpp index ed38e6643..1291a72a2 100755 --- a/src/rootserver/ob_root_service.cpp +++ b/src/rootserver/ob_root_service.cpp @@ -4129,7 +4129,12 @@ int ObRootService::execute_ddl_task(const obrpc::ObAlterTableArg &arg, break; } case share::MAKE_DDL_TAKE_EFFECT_TASK: { - if (OB_FAIL(ddl_service_.swap_orig_and_hidden_table_state( + if (arg.is_direct_load_partition_) { + if (OB_FAIL(ddl_service_.swap_orig_and_hidden_table_partitions( + const_cast(arg)))) { + LOG_WARN("failed to swap orig and hidden table partitions", K(ret)); + } + } else if (OB_FAIL(ddl_service_.swap_orig_and_hidden_table_state( const_cast(arg)))) { LOG_WARN("failed to swap orig and hidden table state", K(ret)); } diff --git a/src/share/ob_rpc_struct.cpp b/src/share/ob_rpc_struct.cpp index 081ba3df2..77d183c6c 100644 --- a/src/share/ob_rpc_struct.cpp +++ b/src/share/ob_rpc_struct.cpp @@ -721,7 +721,8 @@ int ObCreateHiddenTableArg::init(const uint64_t tenant_id, const uint64_t dest_t const int64_t parallelism, const share::ObDDLType ddl_type, const ObSQLMode sql_mode, const ObTimeZoneInfo &tz_info, const common::ObString &local_nls_date, const common::ObString &local_nls_timestamp, const common::ObString &local_nls_timestamp_tz, - const ObTimeZoneInfoWrap &tz_info_wrap, const bool need_reorder_column_id) + const ObTimeZoneInfoWrap &tz_info_wrap, const ObIArray &tablet_ids, + const bool need_reorder_column_id) { int ret = OB_SUCCESS; reset(); @@ -733,6 +734,8 @@ int ObCreateHiddenTableArg::init(const uint64_t tenant_id, const uint64_t dest_t // do nothing } else if (FALSE_IT(nls_formats_[ObNLSFormatEnum::NLS_TIMESTAMP_TZ].assign_ptr(local_nls_timestamp_tz.ptr(), static_cast(local_nls_timestamp_tz.length())))) { // do nothing + } else if (OB_FAIL(tablet_ids_.assign(tablet_ids))) { + LOG_WARN("failed to assign tablet ids", KR(ret), K(tablet_ids)); } else { exec_tenant_id_ = exec_tenant_id; tenant_id_ = tenant_id; diff --git a/src/share/ob_rpc_struct.h b/src/share/ob_rpc_struct.h index 759c85e76..faf139180 100644 --- a/src/share/ob_rpc_struct.h +++ b/src/share/ob_rpc_struct.h @@ -1930,7 +1930,7 @@ public: const ObSQLMode sql_mode, const ObTimeZoneInfo &tz_info, const common::ObString &local_nls_date, const common::ObString &local_nls_timestamp, const common::ObString &local_nls_timestamp_tz, const ObTimeZoneInfoWrap &tz_info_wrap, - const bool need_reorder_column_id); + const common::ObIArray &tablet_ids, const bool need_reorder_column_id); uint64_t get_tenant_id() const { return tenant_id_; } int64_t get_table_id() const { return table_id_; } int64_t get_consumer_group_id() const { return consumer_group_id_; } @@ -1944,6 +1944,7 @@ public: common::ObTimeZoneInfo get_tz_info() const { return tz_info_; } const common::ObTimeZoneInfoWrap &get_tz_info_wrap() const { return tz_info_wrap_; } const common::ObString *get_nls_formats() const { return nls_formats_; } + const common::ObIArray &get_tablet_ids() const { return tablet_ids_; } bool get_need_reorder_column_id() const { return need_reorder_column_id_; } private: uint64_t tenant_id_; diff --git a/src/sql/engine/cmd/ob_load_data_direct_impl.cpp b/src/sql/engine/cmd/ob_load_data_direct_impl.cpp index 1d44e089a..f17dcfac8 100644 --- a/src/sql/engine/cmd/ob_load_data_direct_impl.cpp +++ b/src/sql/engine/cmd/ob_load_data_direct_impl.cpp @@ -2218,6 +2218,7 @@ int ObLoadDataDirectImpl::execute(ObExecContext &ctx, ObLoadDataStmt &load_stmt) execute_param_.method_, execute_param_.insert_mode_, ObDirectLoadMode::LOAD_DATA, + ObDirectLoadLevel::TABLE, execute_param_.column_ids_))) { LOG_WARN("fail to check support direct load", KR(ret)); } else if (OB_FAIL(init_execute_context())) { @@ -2504,6 +2505,7 @@ int ObLoadDataDirectImpl::init_execute_context() load_param.method_ = execute_param_.method_; load_param.insert_mode_ = execute_param_.insert_mode_; load_param.load_mode_ = ObDirectLoadMode::LOAD_DATA; + load_param.load_level_ = ObDirectLoadLevel::TABLE; double online_sample_percent = 100.; if (OB_SUCC(ret)) { diff --git a/src/sql/engine/cmd/ob_table_direct_insert_ctx.cpp b/src/sql/engine/cmd/ob_table_direct_insert_ctx.cpp index 0d10ceeb0..52900383f 100644 --- a/src/sql/engine/cmd/ob_table_direct_insert_ctx.cpp +++ b/src/sql/engine/cmd/ob_table_direct_insert_ctx.cpp @@ -93,7 +93,10 @@ int ObTableDirectInsertCtx::init( } ObDirectLoadMode::Type load_mode = is_insert_overwrite ? ObDirectLoadMode::INSERT_OVERWRITE : ObDirectLoadMode::INSERT_INTO; bool is_heap_table = false; - if (OB_FAIL(ObTableLoadSchema::get_table_schema(*schema_guard, tenant_id, table_id, table_schema))) { + ObDirectLoadLevel::Type load_level = ObDirectLoadLevel::INVALID_LEVEL; + if (OB_FAIL(get_direct_load_level(phy_plan, table_id, load_level))) { + LOG_WARN("failed to get direct load level", KR(ret), K(phy_plan), K(table_id)); + } else if (OB_FAIL(ObTableLoadSchema::get_table_schema(*schema_guard, tenant_id, table_id, table_schema))) { LOG_WARN("fail to get table schema", KR(ret)); } else if (OB_FAIL(ObDDLUtil::get_temp_store_compress_type(table_schema->get_compressor_type(), parallel, @@ -106,10 +109,13 @@ int ObTableDirectInsertCtx::init( method, insert_mode, load_mode, + load_level, column_ids))) { LOG_WARN("fail to check support direct load", KR(ret)); } else { ObTableLoadParam param; + ObArray tablet_ids; + tablet_ids.reset(); param.tenant_id_ = MTL_ID(); param.table_id_ = table_id; param.batch_size_ = 100; @@ -127,8 +133,12 @@ int ObTableDirectInsertCtx::init( param.load_mode_ = load_mode; param.compressor_type_ = compressor_type; param.online_sample_percent_ = online_sample_percent; - if (OB_FAIL(table_load_instance_->init(param, column_ids, load_exec_ctx_))) { - LOG_WARN("failed to init direct loader", KR(ret)); + param.load_level_ = load_level; + if ((ObDirectLoadLevel::PARTITION == param.load_level_) + && OB_FAIL(get_tablet_ids(*(exec_ctx->get_sql_ctx()), table_id, tablet_ids))) { + LOG_WARN("failed to get tablet ids", KR(ret), K(table_id)); + } else if (OB_FAIL(table_load_instance_->init(param, column_ids, tablet_ids, load_exec_ctx_))) { + LOG_WARN("failed to init direct loader", KR(ret), K(param), K(column_ids), K(tablet_ids)); } else { phy_plan.set_ddl_task_id(table_load_instance_->get_table_ctx()->ddl_param_.task_id_); is_inited_ = true; @@ -183,5 +193,71 @@ void ObTableDirectInsertCtx::destroy() is_online_gather_statistics_ = false; } +int ObTableDirectInsertCtx::get_direct_load_level( + const ObPhysicalPlan &phy_plan, + const uint64_t table_id, + ObDirectLoadLevel::Type &load_level) +{ + int ret = OB_SUCCESS; + const ObIArray &table_locs = phy_plan.get_table_locations(); + load_level = ObDirectLoadLevel::TABLE; + for (int64_t i = 0; OB_SUCC(ret) && (i < table_locs.count()); ++i) { + const ObTableLocation &table_loc = table_locs.at(i); + if (table_loc.get_ref_table_id() == table_id) { + if (table_loc.get_part_hint_ids().count() > 0) { + load_level = ObDirectLoadLevel::PARTITION; + } + break; + } + } + return ret; +} + +int ObTableDirectInsertCtx::get_tablet_ids( + const ObSqlCtx &sql_ctx, + const uint64_t table_id, + ObIArray &tablet_ids) +{ + int ret = OB_SUCCESS; + const ObTablePartitionInfo *table_part_info = nullptr; + const ObIArray &partition_infos = sql_ctx.partition_infos_; + for (int64_t i = 0; OB_SUCC(ret) && (i < partition_infos.count()); ++i) { + const ObTablePartitionInfo *part_info = partition_infos.at(i); + if (part_info->get_ref_table_id() == table_id) { + table_part_info = part_info; + break; + } + } + if (OB_SUCC(ret) && OB_NOT_NULL(table_part_info)) { + const ObCandiTableLoc &loc = table_part_info->get_phy_tbl_location_info(); + const ObCandiTabletLocIArray &tablet_locs = loc.get_phy_part_loc_info_list(); + for (int64_t i = 0; OB_SUCC(ret) && (i < tablet_locs.count()); ++i) { + const ObTabletID tablet_id = tablet_locs.at(i).get_partition_location().get_tablet_id(); + if (OB_FAIL(tablet_ids.push_back(tablet_id))) { + LOG_WARN("failed to add tablet id", KR(ret), K(tablet_id)); + } + } + } + return ret; +} + +int ObTableDirectInsertCtx::get_is_heap_table( + ObSchemaGetterGuard &schema_guard, + const uint64_t tenant_id, + const uint64_t table_id, + bool &is_heap_table) +{ + int ret = OB_SUCCESS; + const ObTableSchema *table_schema = nullptr; + if (OB_FAIL(schema_guard.get_table_schema(tenant_id, table_id, table_schema))) { + LOG_WARN("fail to get table schema", KR(ret), K(tenant_id), K(table_id)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_TABLE_NOT_EXIST; + LOG_WARN("table schema is null", KR(ret)); + } else { + is_heap_table = table_schema->is_heap_table(); + } + return ret; +} } // namespace sql } // namespace oceanbase diff --git a/src/sql/engine/cmd/ob_table_direct_insert_ctx.h b/src/sql/engine/cmd/ob_table_direct_insert_ctx.h index 269ff3628..bee21c46c 100644 --- a/src/sql/engine/cmd/ob_table_direct_insert_ctx.h +++ b/src/sql/engine/cmd/ob_table_direct_insert_ctx.h @@ -13,6 +13,7 @@ #pragma once #include "lib/container/ob_iarray.h" +#include "lib/compress/ob_compress_util.h" namespace oceanbase { @@ -21,10 +22,20 @@ namespace observer class ObTableLoadExecCtx; class ObTableLoadInstance; } +namespace common +{ +class ObTabletID; +} +namespace storage +{ +struct ObDirectLoadLevel; +} namespace sql { class ObExecContext; +class ObPhysicalPlan; +class ObSqlCtx; class ObTableDirectInsertCtx { @@ -70,6 +81,17 @@ public: return online_sample_percent_; } +private: + int get_direct_load_level(const sql::ObPhysicalPlan &phy_plan, + const uint64_t table_id, + storage::ObDirectLoadLevel::Type &load_level); + int get_tablet_ids(const sql::ObSqlCtx &sql_ctx, + const uint64_t table_id, + common::ObIArray &tablet_ids); + int get_is_heap_table(share::schema::ObSchemaGetterGuard &schema_guard, + const uint64_t tenant_id, + const uint64_t table_id, + bool &is_heap_table); private: observer::ObTableLoadExecCtx *load_exec_ctx_; observer::ObTableLoadInstance *table_load_instance_; diff --git a/src/sql/optimizer/ob_table_location.h b/src/sql/optimizer/ob_table_location.h index 4516f497d..49d235ad9 100644 --- a/src/sql/optimizer/ob_table_location.h +++ b/src/sql/optimizer/ob_table_location.h @@ -1131,6 +1131,11 @@ private: const uint64_t tenant_id, const ObTabletID src_tablet_id, const int64_t idx) const; +public: + inline const ObIArray &get_part_hint_ids() const + { + return part_hint_ids_; + } private: bool inited_; bool is_partitioned_; diff --git a/src/sql/resolver/dml/ob_insert_resolver.cpp b/src/sql/resolver/dml/ob_insert_resolver.cpp index 9327e0d50..a6ad48e3c 100644 --- a/src/sql/resolver/dml/ob_insert_resolver.cpp +++ b/src/sql/resolver/dml/ob_insert_resolver.cpp @@ -137,9 +137,6 @@ int ObInsertResolver::resolve(const ParseNode &parse_tree) } else if (!insert_stmt->value_from_select()) { ret = OB_NOT_SUPPORTED; LOG_USER_ERROR(OB_NOT_SUPPORTED, "insert overwrite with values"); - } else if (!tmp_table_item->access_all_part()) { - ret = OB_NOT_SUPPORTED; - LOG_USER_ERROR(OB_NOT_SUPPORTED, "insert overwrite stmt with partitions"); } } @@ -509,16 +506,6 @@ int ObInsertResolver::resolve_insert_field(const ParseNode &insert_into, TableIt } else { /*do nothing*/ } } - if (OB_SUCC(ret) && 2 == insert_into.num_child_) { - ParseNode *tmp_node = insert_into.children_[1]; - if (OB_NOT_NULL(tmp_node) && T_COLUMN_LIST == tmp_node->type_) { - if (insert_stmt->is_overwrite()) { - ret = OB_NOT_SUPPORTED; - LOG_USER_ERROR(OB_NOT_SUPPORTED, "insert overwrite stmt with column list"); - } - } - } - if (OB_SUCC(ret) && 2 == insert_into.num_child_ && OB_FAIL(resolve_insert_columns(insert_into.children_[1], insert_stmt->get_insert_table_info()))) { LOG_WARN("failed to resolve insert columns", K(ret)); diff --git a/src/storage/direct_load/ob_direct_load_struct.cpp b/src/storage/direct_load/ob_direct_load_struct.cpp index d9ab008d1..7772a1d18 100644 --- a/src/storage/direct_load/ob_direct_load_struct.cpp +++ b/src/storage/direct_load/ob_direct_load_struct.cpp @@ -51,5 +51,17 @@ bool ObDirectLoadInsertMode::is_type_valid(const Type type) return type > INVALID_INSERT_MODE && type < MAX_INSERT_MODE; } +/** + * ObDirectLoadLevel + */ + +DEFINE_ENUM_FUNC(ObDirectLoadLevel::Type, type, OB_DIRECT_LOAD_LEVEL_DEF, ObDirectLoadLevel::); + +bool ObDirectLoadLevel::is_type_valid(const Type type) +{ + return type > INVALID_LEVEL && type < MAX_LEVEL; +} + + } // namespace storage } // namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_struct.h b/src/storage/direct_load/ob_direct_load_struct.h index 8d0945a39..7404d68d6 100644 --- a/src/storage/direct_load/ob_direct_load_struct.h +++ b/src/storage/direct_load/ob_direct_load_struct.h @@ -68,5 +68,18 @@ struct ObDirectLoadInsertMode static bool is_valid_for_incremental_method(const Type type) { return NORMAL == type || INC_REPLACE == type; } }; +struct ObDirectLoadLevel +{ +#define OB_DIRECT_LOAD_LEVEL_DEF(DEF) \ + DEF(INVALID_LEVEL, = 0) \ + DEF(TABLE, = 1) \ + DEF(PARTITION, = 2) \ + DEF(MAX_LEVEL, ) + + DECLARE_ENUM(Type, type, OB_DIRECT_LOAD_LEVEL_DEF, static); + + static bool is_type_valid(const Type type); +}; + } // namespace storage } // namespace oceanbase From dc6978c8151d49e03b0fee9fc40329c2270dbfc0 Mon Sep 17 00:00:00 2001 From: yinyj17 Date: Fri, 23 Aug 2024 07:02:35 +0000 Subject: [PATCH 186/249] fix eval cost failed when merge into updatable view --- src/sql/optimizer/ob_log_plan.cpp | 3 ++- src/sql/optimizer/ob_merge_log_plan.cpp | 14 +++++------- src/sql/optimizer/ob_merge_log_plan.h | 2 ++ src/sql/resolver/dml/ob_dml_stmt.h | 5 ++-- src/sql/rewrite/ob_transform_pre_process.cpp | 24 ++++++++++++++++++++ src/sql/rewrite/ob_transform_pre_process.h | 1 + src/sql/rewrite/ob_transform_utils.cpp | 4 ---- 7 files changed, 38 insertions(+), 15 deletions(-) diff --git a/src/sql/optimizer/ob_log_plan.cpp b/src/sql/optimizer/ob_log_plan.cpp index c93687cf3..221a815de 100644 --- a/src/sql/optimizer/ob_log_plan.cpp +++ b/src/sql/optimizer/ob_log_plan.cpp @@ -12307,6 +12307,7 @@ int ObLogPlan::get_table_for_update_info(const uint64_t table_id, ObSEArray temp_rowkeys; if (OB_UNLIKELY(!table->is_basic_table()) || OB_UNLIKELY(is_virtual_table(table->ref_id_))) { // invalid usage + // bad case: select * from (select /*+no_merge*/ * from t1) for update ret = OB_ERR_FOR_UPDATE_SELECT_VIEW_CANNOT; LOG_USER_ERROR(OB_ERR_FOR_UPDATE_SELECT_VIEW_CANNOT); } else if (OB_FAIL(get_rowkey_exprs(table->table_id_, table->ref_id_, temp_rowkeys))) { @@ -12322,7 +12323,7 @@ int ObLogPlan::get_table_for_update_info(const uint64_t table_id, } else { index_dml_info = new (index_dml_info) IndexDMLInfo(); index_dml_info->table_id_ = table->table_id_; - index_dml_info->loc_table_id_ = table->get_base_table_item().table_id_; + index_dml_info->loc_table_id_ = table->table_id_; index_dml_info->ref_table_id_ = table->ref_id_; index_dml_info->distinct_algo_ = T_DISTINCT_NONE; index_dml_info->rowkey_cnt_ = temp_rowkeys.count(); diff --git a/src/sql/optimizer/ob_merge_log_plan.cpp b/src/sql/optimizer/ob_merge_log_plan.cpp index 64417112a..5f5b8bf95 100644 --- a/src/sql/optimizer/ob_merge_log_plan.cpp +++ b/src/sql/optimizer/ob_merge_log_plan.cpp @@ -443,6 +443,7 @@ int ObMergeLogPlan::check_merge_need_multi_partition_dml(ObLogicalOperator &top, } else if (!merge_stmt->has_insert_clause() || is_one_part_table) { is_multi_part_dml = false; } else if (OB_FAIL(check_update_insert_sharding_basic(top, + index_dml_infos_.at(0)->loc_table_id_, insert_sharding, update_sharding, is_basic, @@ -467,6 +468,7 @@ int ObMergeLogPlan::check_merge_need_multi_partition_dml(ObLogicalOperator &top, } int ObMergeLogPlan::check_update_insert_sharding_basic(ObLogicalOperator &top, + uint64_t table_id, ObShardingInfo *insert_sharding, ObShardingInfo *update_sharding, bool &is_basic, @@ -493,7 +495,7 @@ int ObMergeLogPlan::check_update_insert_sharding_basic(ObLogicalOperator &top, /* insert_sharding and update_sharding match basic, but the partition is different, need multi part merge */ is_basic = false; - } else if (OB_FAIL(generate_equal_constraint(top, *insert_sharding, can_gen_cons, equal_pairs))) { + } else if (OB_FAIL(generate_equal_constraint(top, table_id, *insert_sharding, can_gen_cons, equal_pairs))) { LOG_WARN("failed to generate equal constraint", K(ret)); } else if (can_gen_cons) { is_basic = true; @@ -526,6 +528,7 @@ bool ObMergeLogPlan::match_same_partition(const ObShardingInfo &l_sharding_info, // When insert_sharding and update_sharding match same partition, // need add equal constraints for const params in sharding conditions which equal to part key. int ObMergeLogPlan::generate_equal_constraint(ObLogicalOperator &top, + uint64_t table_id, ObShardingInfo &insert_sharding, bool &can_gen_cons, ObIArray> &equal_pairs) @@ -538,16 +541,11 @@ int ObMergeLogPlan::generate_equal_constraint(ObLogicalOperator &top, ObSEArray right_part_keys; ObSEArray right_conds; const ObMergeStmt *stmt = NULL; - const TableItem *target_table = NULL; if (OB_ISNULL(stmt = get_stmt())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret), K(stmt)); - } else if (OB_ISNULL(target_table = stmt->get_table_item_by_id(stmt->get_target_table_id()))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to get target table", K(ret)); - } else if (OB_FAIL(get_target_table_scan(target_table->get_base_table_item().table_id_, - &top, target_table_scan))) { - LOG_WARN("get target table scan failed", K(ret), K(*target_table)); + } else if (OB_FAIL(get_target_table_scan(table_id, &top, target_table_scan))) { + LOG_WARN("get target table scan failed", K(ret), K(table_id)); } else if (OB_ISNULL(target_table_scan) || OB_ISNULL(target_table_scan->get_sharding())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret), K(target_table_scan)); diff --git a/src/sql/optimizer/ob_merge_log_plan.h b/src/sql/optimizer/ob_merge_log_plan.h index d2ae4b153..55ea4dbc5 100644 --- a/src/sql/optimizer/ob_merge_log_plan.h +++ b/src/sql/optimizer/ob_merge_log_plan.h @@ -65,6 +65,7 @@ private: bool &is_multi_part_dml, ObIArray> &equal_pairs); int check_update_insert_sharding_basic(ObLogicalOperator &top, + uint64_t table_id, ObShardingInfo *insert_sharding, ObShardingInfo *update_sharding, bool &is_basic, @@ -72,6 +73,7 @@ private: bool match_same_partition(const ObShardingInfo &l_sharding_info, const ObShardingInfo &r_sharding_info); int generate_equal_constraint(ObLogicalOperator &top, + uint64_t table_id, ObShardingInfo &insert_sharding, bool &can_gen_cons, ObIArray> &equal_pairs); diff --git a/src/sql/resolver/dml/ob_dml_stmt.h b/src/sql/resolver/dml/ob_dml_stmt.h index b66faca6a..55d65b4e5 100644 --- a/src/sql/resolver/dml/ob_dml_stmt.h +++ b/src/sql/resolver/dml/ob_dml_stmt.h @@ -375,6 +375,7 @@ struct TableItem { return synonym_name_.empty() ? database_name_ : synonym_db_name_; } + // only can be used in resolve phase const TableItem &get_base_table_item() const { return (is_generated_table() || is_temp_table()) && view_base_item_ != NULL @@ -421,8 +422,8 @@ struct TableItem bool need_expand_rt_mv_; // for real-time materialized view uint64_t mview_id_; // for materialized view, ref_id_ is mv container table id, mview_id_ is the view id const ParseNode* node_; - // base table item for updatable view - const TableItem *view_base_item_; // seems to be useful only in the resolve phase + // base table item for updatable view, can not access after the resolve phase + const TableItem *view_base_item_; ObRawExpr *flashback_query_expr_; FlashBackQueryType flashback_query_type_; ObRawExpr *function_table_expr_; diff --git a/src/sql/rewrite/ob_transform_pre_process.cpp b/src/sql/rewrite/ob_transform_pre_process.cpp index 5a9382074..88c9d56b3 100644 --- a/src/sql/rewrite/ob_transform_pre_process.cpp +++ b/src/sql/rewrite/ob_transform_pre_process.cpp @@ -340,6 +340,9 @@ int ObTransformPreProcess::transform_one_stmt(common::ObIArray LOG_WARN("disable complex dml for fulltext index", K(ret)); // jinmao TODO: table scan 能吐出正确的 doc_id 后，可删除此限制 } + if (OB_SUCC(ret) && OB_FAIL(reset_view_base_item(stmt))) { + LOG_WARN("failed to reset view base item", K(ret)); + } if (OB_SUCC(ret)) { LOG_DEBUG("transform pre process succ", K(*stmt)); if (OB_FAIL(stmt->formalize_stmt(ctx_->session_info_))) { @@ -10893,5 +10896,26 @@ int ObTransformPreProcess::construct_leaf_leading_table(ObDMLStmt *stmt, return ret; } +int ObTransformPreProcess::reset_view_base_item(ObDMLStmt *stmt) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else { + ObIArray &tables = stmt->get_table_items(); + TableItem *table_item = NULL; + for (int64_t i = 0; OB_SUCC(ret) && i < tables.count(); ++i) { + if (OB_ISNULL(table_item = tables.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), KP(table_item)); + } else { + table_item->view_base_item_ = NULL; + } + } + } + return ret; +} + } // end namespace sql } // end namespace oceanbase diff --git a/src/sql/rewrite/ob_transform_pre_process.h b/src/sql/rewrite/ob_transform_pre_process.h index e7c15e76b..4219fe233 100644 --- a/src/sql/rewrite/ob_transform_pre_process.h +++ b/src/sql/rewrite/ob_transform_pre_process.h @@ -692,6 +692,7 @@ private: ObIArray &flattened_tables, ObLeadingTable &leading_table); int construct_leaf_leading_table(ObDMLStmt *stmt, TableItem *table, ObLeadingTable *&leading_table); + int reset_view_base_item(ObDMLStmt *stmt); private: DISALLOW_COPY_AND_ASSIGN(ObTransformPreProcess); diff --git a/src/sql/rewrite/ob_transform_utils.cpp b/src/sql/rewrite/ob_transform_utils.cpp index 9b602285a..6e0bda43f 100644 --- a/src/sql/rewrite/ob_transform_utils.cpp +++ b/src/sql/rewrite/ob_transform_utils.cpp @@ -7245,10 +7245,6 @@ int ObTransformUtils::adjust_updatable_view(ObRawExprFactory &expr_factory, merge_table_info->source_table_id_ = view_table_item.table_id_; } } - TableItem *base_item = view_stmt->get_table_item_by_id(table_info->table_id_); - if (NULL != base_item) { - view_table_item.view_base_item_ = base_item; - } table_info->table_id_ = view_table_item.table_id_; uint64_t loc_table_id = table_info->loc_table_id_; // create partition exprs for index dml infos From 06bbec1303bfe97e312b763505a4d1927a972081 Mon Sep 17 00:00:00 2001 From: 0xacc Date: Fri, 23 Aug 2024 07:09:33 +0000 Subject: [PATCH 187/249] [to #2024082100104212762] fix: fix core dump when ? in DBMS_SQL.PARSER --- src/pl/sys_package/ob_dbms_sql.cpp | 16 ++++++++++++---- src/share/ob_errno.cpp | 17 ++++++++++++++++- src/share/ob_errno.def | 1 + src/share/ob_errno.h | 6 +++++- 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/src/pl/sys_package/ob_dbms_sql.cpp b/src/pl/sys_package/ob_dbms_sql.cpp index 076d44377..7a9ef4792 100644 --- a/src/pl/sys_package/ob_dbms_sql.cpp +++ b/src/pl/sys_package/ob_dbms_sql.cpp @@ -596,13 +596,21 @@ int ObDbmsCursorInfo::parse(const ObString &sql_stmt, ObSQLSessionInfo &session) session.get_warnings_buffer().set_error_line_column(0, error_offset); } OZ (ObResolverUtils::resolve_stmt_type(parse_result, stmt_type_), sql_stmt); - // cann't execute multi select stmt - if (OB_SUCC(ret) - && !parser.is_pl_stmt(sql_stmt) - && !parser.is_single_stmt(sql_stmt)) { + + if (OB_FAIL(ret)) { + // do nothing + } else if (!parser.is_pl_stmt(sql_stmt) + && !parser.is_single_stmt(sql_stmt)) { + // cann't execute multi select stmt ret = OB_ERR_CMD_NOT_PROPERLY_ENDED; LOG_WARN("execute immdeidate only support one stmt", K(ret)); + } else if (0 < parse_result.question_mark_ctx_.count_ + &&!parse_result.question_mark_ctx_.by_name_) { + ret = OB_ERR_INVALID_CHARACTER; + LOG_WARN("use '?' in DBMS_SQL.PARSE is not supported", + K(ret), K(sql_stmt)); } + OX (param_names = parse_result.question_mark_ctx_.name_); OX (param_count = parse_result.question_mark_ctx_.count_); // 取出所有绑定变量名 diff --git a/src/share/ob_errno.cpp b/src/share/ob_errno.cpp index c5b7fefbf..3149f5b6e 100644 --- a/src/share/ob_errno.cpp +++ b/src/share/ob_errno.cpp @@ -31499,6 +31499,20 @@ static const _error _error_OB_NO_PARTITION_FOR_GIVEN_VALUE_SCHEMA_ERROR = { .ob_str_error = "OBE-14400: inserted partition key does not map to any partition", .ob_str_user_error = "OBE-14400: inserted partition key does not map to any partition" }; +static const _error _error_OB_ERR_INVALID_CHARACTER = { + .error_name = "OB_ERR_INVALID_CHARACTER", + .error_cause = "Internal Error", + .error_solution = "Contact OceanBase Support", + .mysql_errno = -1, + .sqlstate = "HY000", + .str_error = "invalid character", + .str_user_error = "invalid character", + .oracle_errno = 911, + .oracle_str_error = "ORA-00911: invalid character", + .oracle_str_user_error = "ORA-00911: invalid character", + .ob_str_error = "OBE-00911: invalid character", + .ob_str_user_error = "OBE-00911: invalid character" +}; static const _error _error_OB_ERR_KV_GLOBAL_INDEX_ROUTE = { .error_name = "OB_ERR_KV_GLOBAL_INDEX_ROUTE", .error_cause = "Internal Error", @@ -34846,6 +34860,7 @@ struct ObStrErrorInit _errors[-OB_ERR_EVENT_CANNOT_ALTER_IN_THE_PAST] = &_error_OB_ERR_EVENT_CANNOT_ALTER_IN_THE_PAST; _errors[-OB_ERR_EVENT_RECURSION_FORBIDDEN] = &_error_OB_ERR_EVENT_RECURSION_FORBIDDEN; _errors[-OB_NO_PARTITION_FOR_GIVEN_VALUE_SCHEMA_ERROR] = &_error_OB_NO_PARTITION_FOR_GIVEN_VALUE_SCHEMA_ERROR; + _errors[-OB_ERR_INVALID_CHARACTER] = &_error_OB_ERR_INVALID_CHARACTER; _errors[-OB_ERR_KV_GLOBAL_INDEX_ROUTE] = &_error_OB_ERR_KV_GLOBAL_INDEX_ROUTE; _errors[-OB_TTL_NOT_ENABLE] = &_error_OB_TTL_NOT_ENABLE; _errors[-OB_TTL_COLUMN_NOT_EXIST] = &_error_OB_TTL_COLUMN_NOT_EXIST; @@ -34958,7 +34973,7 @@ namespace oceanbase { namespace common { -int g_all_ob_errnos[2325] = {0, -4000, -4001, -4002, -4003, -4004, -4005, -4006, -4007, -4008, -4009, -4010, -4011, -4012, -4013, -4014, -4015, -4016, -4017, -4018, -4019, -4020, -4021, -4022, -4023, -4024, -4025, -4026, -4027, -4028, -4029, -4030, -4031, -4032, -4033, -4034, -4035, -4036, -4037, -4038, -4039, -4041, -4042, -4043, -4044, -4045, -4046, -4047, -4048, -4049, -4050, -4051, -4052, -4053, -4054, -4055, -4057, -4058, -4060, -4061, -4062, -4063, -4064, -4065, -4066, -4067, -4068, -4070, -4071, -4072, -4073, -4074, -4075, -4076, -4077, -4078, -4080, -4081, -4084, -4085, -4090, -4097, -4098, -4099, -4100, -4101, -4102, -4103, -4104, -4105, -4106, -4107, -4108, -4109, -4110, -4111, -4112, -4113, -4114, -4115, -4116, -4117, -4118, -4119, -4120, -4121, -4122, -4123, -4124, -4125, -4126, -4127, -4128, -4133, -4138, -4139, -4142, -4143, -4144, -4146, -4147, -4149, -4150, -4151, -4152, -4153, -4154, -4155, -4156, -4157, -4158, -4159, -4160, -4161, -4162, -4163, -4164, -4165, -4166, -4167, -4168, -4169, -4170, -4171, -4172, -4173, -4174, -4175, -4176, -4177, -4178, -4179, -4180, -4181, -4182, -4183, -4184, -4185, -4186, -4187, -4188, -4189, -4190, -4191, -4192, -4200, -4201, -4204, -4205, -4206, -4207, -4208, -4209, -4210, -4211, -4212, -4213, -4214, -4215, -4216, -4217, -4218, -4219, -4220, -4221, -4222, -4223, -4224, -4225, -4226, -4227, -4228, -4229, -4230, -4231, -4232, -4233, -4234, -4235, -4236, -4237, -4238, -4239, -4240, -4241, -4242, -4243, -4244, -4245, -4246, -4247, -4248, -4249, -4250, -4251, -4252, -4253, -4254, -4255, -4256, -4257, -4258, -4260, -4261, -4262, -4263, -4264, -4265, -4266, -4267, -4268, -4269, -4270, -4271, -4273, -4274, -4275, -4276, -4277, -4278, -4279, -4280, -4281, -4282, -4283, -4284, -4285, -4286, -4287, -4288, -4289, -4290, -4291, -4292, -4293, -4294, -4295, -4296, -4297, -4298, -4299, -4300, -4301, -4302, -4303, -4304, -4305, -4306, -4307, -4308, -4309, -4310, -4311, -4312, -4313, -4314, -4315, -4316, -4317, -4318, -4319, -4320, -4321, -4322, -4323, -4324, -4325, -4326, -4327, -4328, -4329, -4330, -4331, -4332, -4333, -4334, -4335, -4336, -4337, -4338, -4339, -4340, -4341, -4342, -4343, -4344, -4345, -4346, -4347, -4348, -4349, -4350, -4351, -4352, -4353, -4354, -4355, -4356, -4357, -4358, -4359, -4360, -4361, -4362, -4363, -4364, -4365, -4366, -4367, -4368, -4369, -4370, -4371, -4372, -4373, -4374, -4375, -4376, -4377, -4378, -4379, -4380, -4381, -4382, -4383, -4385, -4386, -4387, -4388, -4389, -4390, -4391, -4392, -4393, -4394, -4395, -4396, -4397, -4398, -4399, -4400, -4401, -4402, -4403, -4505, -4507, -4510, -4512, -4515, -4517, -4518, -4519, -4523, -4524, -4525, -4526, -4527, -4528, -4529, -4530, -4531, -4532, -4533, -4537, -4538, -4539, -4540, -4541, -4542, -4543, -4544, -4545, -4546, -4547, -4548, -4549, -4550, -4551, -4552, -4553, -4554, -4600, -4601, -4602, -4603, -4604, -4605, -4606, -4607, -4608, -4609, -4610, -4611, -4613, -4614, -4615, -4620, -4621, -4622, -4623, -4624, -4625, -4626, -4628, -4629, -4630, -4631, -4632, -4633, -4634, -4636, -4637, -4638, -4639, -4640, -4641, -4642, -4643, -4644, -4645, -4646, -4647, -4648, -4649, -4650, -4651, -4652, -4653, -4654, -4655, -4656, -4657, -4658, -4659, -4660, -4661, -4662, -4663, -4664, -4665, -4666, -4667, -4668, -4669, -4670, -4671, -4672, -4673, -4674, -4675, -4676, -4677, -4678, -4679, -4680, -4681, -4682, -4683, -4684, -4685, -4686, -4687, -4688, -4689, -4690, -4691, -4692, -4693, -4694, -4695, -4696, -4697, -4698, -4699, -4700, -4701, -4702, -4703, -4704, -4705, -4706, -4707, -4708, -4709, -4710, -4711, -4712, -4713, -4714, -4715, -4716, -4717, -4718, -4719, -4720, -4721, -4722, -4723, -4724, -4725, -4726, -4727, -4728, -4729, -4730, -4731, -4732, -4733, -4734, -4735, -4736, -4737, -4738, -4739, -4740, -4741, -4742, -4743, -4744, -4745, -4746, -4747, -4748, -4749, -4750, -4751, -4752, -4753, -4754, -4755, -4756, -4757, -4758, -4759, -4760, -4761, -4762, -4763, -4764, -4765, -4766, -4767, -4768, -4769, -4770, -4771, -4772, -4773, -4774, -4775, -4776, -4777, -4778, -4779, -4780, -4781, -4782, -4783, -4784, -5000, -5001, -5002, -5003, -5006, -5007, -5008, -5010, -5011, -5012, -5014, -5015, -5016, -5017, -5018, -5019, -5020, -5022, -5023, -5024, -5025, -5026, -5027, -5028, -5029, -5030, -5031, -5032, -5034, -5035, -5036, -5037, -5038, -5039, -5040, -5041, -5042, -5043, -5044, -5046, -5047, -5050, -5051, -5052, -5053, -5054, -5055, -5056, -5057, -5058, -5059, -5061, -5063, -5064, -5065, -5066, -5067, -5068, -5069, -5070, -5071, -5072, -5073, -5074, -5080, -5081, -5083, -5084, -5085, -5086, -5087, -5088, -5089, -5090, -5091, -5092, -5093, -5094, -5095, -5096, -5097, -5098, -5099, -5100, -5101, -5102, -5103, -5104, -5105, -5106, -5107, -5108, -5109, -5110, -5111, -5112, -5113, -5114, -5115, -5116, -5117, -5118, -5119, -5120, -5121, -5122, -5123, -5124, -5125, -5130, -5131, -5133, -5134, -5135, -5136, -5137, -5138, -5139, -5140, -5142, -5143, -5144, -5145, -5146, -5147, -5148, -5149, -5150, -5151, -5153, -5154, -5155, -5156, -5157, -5158, -5159, -5160, -5161, -5162, -5163, -5164, -5165, -5166, -5167, -5168, -5169, -5170, -5171, -5172, -5173, -5174, -5175, -5176, -5177, -5178, -5179, -5180, -5181, -5182, -5183, -5184, -5185, -5187, -5188, -5189, -5190, -5191, -5192, -5193, -5194, -5195, -5196, -5197, -5198, -5199, -5200, -5201, -5202, -5203, -5204, -5205, -5206, -5207, -5208, -5209, -5210, -5211, -5212, -5213, -5214, -5215, -5216, -5217, -5218, -5219, -5220, -5221, -5222, -5223, -5224, -5225, -5226, -5227, -5228, -5229, -5230, -5231, -5233, -5234, -5235, -5236, -5237, -5238, -5239, -5240, -5241, -5242, -5243, -5244, -5245, -5246, -5247, -5248, -5249, -5250, -5251, -5252, -5253, -5254, -5255, -5256, -5257, -5258, -5259, -5260, -5261, -5262, -5263, -5264, -5265, -5266, -5267, -5268, -5269, -5270, -5271, -5272, -5273, -5274, -5275, -5276, -5277, -5278, -5279, -5280, -5281, -5282, -5283, -5284, -5285, -5286, -5287, -5288, -5289, -5290, -5291, -5292, -5293, -5294, -5295, -5296, -5297, -5298, -5299, -5300, -5301, -5302, -5303, -5304, -5305, -5306, -5307, -5308, -5309, -5310, -5311, -5312, -5313, -5314, -5315, -5316, -5317, -5318, -5319, -5320, -5321, -5322, -5323, -5324, -5325, -5326, -5327, -5328, -5329, -5330, -5331, -5332, -5333, -5334, -5335, -5336, -5337, -5338, -5339, -5340, -5341, -5342, -5343, -5344, -5345, -5346, -5347, -5348, -5349, -5350, -5351, -5352, -5353, -5354, -5355, -5356, -5357, -5358, -5359, -5360, -5361, -5362, -5363, -5364, -5365, -5366, -5367, -5368, -5369, -5370, -5371, -5372, -5373, -5374, -5375, -5376, -5377, -5378, -5379, -5380, -5381, -5382, -5383, -5384, -5385, -5386, -5387, -5388, -5389, -5390, -5400, -5401, -5402, -5403, -5404, -5405, -5406, -5407, -5408, -5409, -5410, -5411, -5412, -5413, -5414, -5415, -5416, -5417, -5418, -5419, -5420, -5421, -5422, -5423, -5424, -5425, -5426, -5427, -5428, -5429, -5430, -5431, -5432, -5433, -5434, -5435, -5436, -5437, -5438, -5439, -5440, -5441, -5442, -5443, -5444, -5445, -5446, -5447, -5448, -5449, -5450, -5451, -5452, -5453, -5454, -5455, -5456, -5457, -5458, -5459, -5460, -5461, -5462, -5463, -5464, -5465, -5466, -5467, -5468, -5469, -5470, -5471, -5472, -5473, -5474, -5475, -5476, -5477, -5478, -5479, -5480, -5481, -5482, -5483, -5484, -5485, -5486, -5487, -5488, -5489, -5490, -5491, -5492, -5493, -5494, -5495, -5496, -5497, -5498, -5499, -5500, -5501, -5502, -5503, -5504, -5505, -5506, -5507, -5508, -5509, -5510, -5511, -5512, -5513, -5514, -5515, -5516, -5517, -5518, -5519, -5520, -5521, -5522, -5540, -5541, -5542, -5543, -5544, -5545, -5546, -5547, -5548, -5549, -5550, -5551, -5552, -5553, -5554, -5555, -5556, -5557, -5558, -5559, -5560, -5561, -5562, -5563, -5564, -5565, -5566, -5567, -5568, -5569, -5570, -5571, -5572, -5573, -5574, -5575, -5576, -5577, -5578, -5579, -5580, -5581, -5582, -5583, -5584, -5585, -5586, -5587, -5588, -5589, -5590, -5591, -5592, -5593, -5594, -5595, -5596, -5597, -5598, -5599, -5600, -5601, -5602, -5603, -5604, -5605, -5607, -5608, -5609, -5610, -5611, -5612, -5613, -5614, -5615, -5616, -5617, -5618, -5619, -5620, -5621, -5622, -5623, -5624, -5625, -5626, -5627, -5628, -5629, -5630, -5631, -5632, -5633, -5634, -5635, -5636, -5637, -5638, -5639, -5640, -5641, -5642, -5643, -5644, -5645, -5646, -5647, -5648, -5649, -5650, -5651, -5652, -5653, -5654, -5655, -5656, -5657, -5658, -5659, -5660, -5661, -5662, -5663, -5664, -5665, -5666, -5667, -5668, -5671, -5672, -5673, -5674, -5675, -5676, -5677, -5678, -5679, -5680, -5681, -5682, -5683, -5684, -5685, -5686, -5687, -5688, -5689, -5690, -5691, -5692, -5693, -5694, -5695, -5696, -5697, -5698, -5699, -5700, -5701, -5702, -5703, -5704, -5705, -5706, -5707, -5708, -5709, -5710, -5711, -5712, -5713, -5714, -5715, -5716, -5717, -5718, -5719, -5720, -5721, -5722, -5723, -5724, -5725, -5726, -5727, -5728, -5729, -5730, -5731, -5732, -5733, -5734, -5735, -5736, -5737, -5738, -5739, -5740, -5741, -5742, -5743, -5744, -5745, -5746, -5747, -5748, -5749, -5750, -5751, -5752, -5753, -5754, -5755, -5756, -5757, -5758, -5759, -5760, -5761, -5762, -5763, -5764, -5765, -5766, -5768, -5769, -5770, -5771, -5772, -5773, -5774, -5777, -5778, -5779, -5780, -5781, -5785, -5786, -5787, -5788, -5789, -5790, -5791, -5792, -5793, -5794, -5795, -5796, -5797, -5798, -5799, -5800, -5801, -5802, -5803, -5804, -5805, -5806, -5807, -5808, -5809, -5810, -5811, -5812, -5813, -5814, -5815, -5816, -5817, -5818, -5819, -5820, -5821, -5822, -5823, -5824, -5825, -5826, -5827, -5828, -5829, -5830, -5831, -5832, -5833, -5834, -5835, -5836, -5837, -5838, -5839, -5840, -5841, -5842, -5843, -5844, -5845, -5846, -5847, -5848, -5849, -5850, -5851, -5852, -5853, -5854, -5855, -5856, -5857, -5858, -5859, -5860, -5861, -5862, -5863, -5864, -5865, -5866, -5867, -5868, -5869, -5870, -5871, -5872, -5873, -5874, -5875, -5876, -5877, -5878, -5879, -5880, -5881, -5882, -5883, -5884, -5885, -5886, -5887, -5888, -5889, -5890, -5891, -5892, -5893, -5894, -5895, -5896, -5897, -5898, -5899, -5900, -5901, -5902, -5903, -5904, -5905, -5906, -5907, -5908, -5909, -5910, -5911, -5912, -5913, -5914, -5915, -5916, -5917, -5918, -5919, -5920, -5921, -5922, -5923, -5924, -5925, -5926, -5927, -5928, -5929, -5930, -5931, -5932, -5933, -5934, -5935, -5936, -5937, -5938, -5939, -5940, -5941, -5942, -5943, -5944, -5945, -5946, -5947, -5948, -5949, -5950, -5951, -5952, -5953, -5954, -5955, -5956, -5957, -5958, -5959, -5960, -5961, -5962, -5963, -5964, -5965, -5966, -5967, -5968, -5969, -5970, -5971, -5972, -5973, -5974, -5975, -5976, -5977, -5978, -5979, -5980, -5981, -5982, -5983, -5984, -5985, -5986, -5987, -5988, -5989, -5990, -5991, -5992, -5993, -5994, -5995, -5996, -5997, -5998, -5999, -6000, -6001, -6002, -6003, -6004, -6005, -6006, -6201, -6202, -6203, -6204, -6205, -6206, -6207, -6208, -6209, -6210, -6211, -6212, -6213, -6214, -6215, -6219, -6220, -6221, -6222, -6223, -6224, -6225, -6226, -6227, -6228, -6229, -6230, -6231, -6232, -6233, -6234, -6235, -6236, -6237, -6238, -6239, -6240, -6241, -6242, -6243, -6244, -6245, -6246, -6247, -6248, -6249, -6250, -6251, -6252, -6253, -6254, -6255, -6256, -6257, -6258, -6259, -6260, -6261, -6262, -6263, -6264, -6265, -6266, -6267, -6268, -6269, -6270, -6271, -6272, -6273, -6274, -6275, -6276, -6277, -6278, -6279, -6280, -6281, -6282, -6283, -6284, -6285, -6301, -6302, -6303, -6304, -6305, -6306, -6307, -6308, -6309, -6310, -6311, -6312, -6313, -6314, -6315, -6316, -6317, -6318, -6319, -6320, -6321, -6322, -6323, -6324, -6325, -6326, -6327, -6328, -6329, -6330, -6331, -6332, -7000, -7001, -7002, -7003, -7004, -7005, -7006, -7007, -7010, -7011, -7012, -7013, -7014, -7015, -7021, -7022, -7024, -7025, -7026, -7027, -7029, -7030, -7031, -7032, -7033, -7034, -7035, -7036, -7037, -7038, -7039, -7040, -7041, -7100, -7101, -7102, -7103, -7104, -7105, -7106, -7107, -7108, -7109, -7110, -7111, -7112, -7113, -7114, -7115, -7116, -7117, -7118, -7119, -7120, -7121, -7122, -7123, -7124, -7201, -7202, -7203, -7204, -7205, -7206, -7207, -7208, -7209, -7210, -7211, -7212, -7213, -7214, -7215, -7216, -7217, -7218, -7219, -7220, -7221, -7222, -7223, -7224, -7225, -7226, -7227, -7228, -7229, -7230, -7231, -7232, -7233, -7234, -7235, -7236, -7237, -7238, -7239, -7240, -7241, -7242, -7243, -7244, -7246, -7247, -7248, -7249, -7250, -7251, -7252, -7253, -7254, -7255, -7256, -7257, -7258, -7259, -7260, -7261, -7262, -7263, -7264, -7265, -7266, -7267, -7268, -7269, -7270, -7271, -7272, -7273, -7274, -7275, -7276, -7277, -7278, -7279, -7280, -7281, -7282, -7283, -7284, -7285, -7286, -7287, -7288, -7289, -7290, -7291, -7292, -7293, -7294, -7295, -7296, -7297, -7298, -7299, -7300, -7301, -7302, -7402, -7403, -7404, -7405, -7406, -7407, -7408, -7409, -7410, -7411, -7412, -7413, -7414, -7415, -7416, -7417, -7418, -7419, -7420, -7421, -7422, -7423, -7424, -7425, -7426, -7427, -7428, -7429, -7430, -7431, -7432, -7433, -7434, -7435, -7600, -7601, -7602, -7603, -7604, -8001, -8002, -8003, -8004, -8005, -9001, -9002, -9003, -9004, -9005, -9006, -9007, -9008, -9009, -9010, -9011, -9012, -9013, -9014, -9015, -9016, -9017, -9018, -9019, -9020, -9022, -9023, -9024, -9025, -9026, -9027, -9028, -9029, -9030, -9031, -9032, -9033, -9034, -9035, -9036, -9037, -9038, -9039, -9040, -9041, -9042, -9043, -9044, -9045, -9046, -9047, -9048, -9049, -9050, -9051, -9052, -9053, -9054, -9057, -9058, -9059, -9060, -9061, -9062, -9063, -9064, -9065, -9066, -9069, -9070, -9071, -9072, -9073, -9074, -9075, -9076, -9077, -9078, -9079, -9080, -9081, -9082, -9083, -9084, -9085, -9086, -9087, -9088, -9089, -9090, -9091, -9092, -9093, -9094, -9095, -9096, -9097, -9098, -9099, -9100, -9101, -9102, -9103, -9104, -9105, -9106, -9107, -9108, -9109, -9110, -9111, -9112, -9113, -9114, -9115, -9116, -9117, -9118, -9119, -9120, -9121, -9122, -9123, -9124, -9125, -9126, -9127, -9200, -9201, -9202, -9203, -9501, -9502, -9503, -9504, -9505, -9506, -9507, -9508, -9509, -9510, -9512, -9513, -9514, -9515, -9516, -9518, -9519, -9520, -9521, -9522, -9523, -9524, -9525, -9526, -9527, -9528, -9529, -9530, -9531, -9532, -9533, -9534, -9535, -9536, -9537, -9538, -9539, -9540, -9541, -9542, -9543, -9544, -9545, -9546, -9547, -9548, -9549, -9550, -9551, -9552, -9553, -9554, -9555, -9556, -9557, -9558, -9559, -9560, -9561, -9562, -9563, -9564, -9565, -9566, -9567, -9568, -9569, -9570, -9571, -9572, -9573, -9574, -9575, -9576, -9577, -9578, -9579, -9580, -9581, -9582, -9583, -9584, -9585, -9586, -9587, -9588, -9589, -9590, -9591, -9592, -9593, -9594, -9595, -9596, -9597, -9598, -9599, -9600, -9601, -9602, -9603, -9604, -9605, -9606, -9607, -9608, -9609, -9610, -9611, -9612, -9613, -9614, -9615, -9616, -9617, -9618, -9619, -9620, -9621, -9622, -9623, -9624, -9625, -9626, -9627, -9628, -9629, -9630, -9631, -9632, -9633, -9634, -9635, -9636, -9637, -9638, -9639, -9640, -9641, -9642, -9643, -9644, -9645, -9646, -9647, -9648, -9649, -9650, -9651, -9652, -9653, -9654, -9655, -9656, -9657, -9658, -9659, -9660, -9661, -9662, -9663, -9664, -9665, -9666, -9667, -9668, -9669, -9670, -9671, -9672, -9673, -9674, -9675, -9676, -9677, -9678, -9679, -9680, -9681, -9682, -9683, -9684, -9685, -9686, -9687, -9688, -9689, -9690, -9691, -9692, -9693, -9694, -9695, -9696, -9697, -9698, -9699, -9700, -9701, -9702, -9703, -9704, -9705, -9706, -9707, -9708, -9709, -9710, -9711, -9712, -9713, -9714, -9715, -9716, -9717, -9718, -9719, -9720, -9721, -9722, -9723, -9724, -9725, -9726, -9727, -9728, -9729, -9730, -9731, -9732, -9733, -9734, -9735, -9736, -9737, -9738, -9739, -9740, -9741, -9742, -9743, -9744, -9745, -9746, -9747, -9748, -9749, -9750, -9751, -9752, -9753, -9754, -9755, -9756, -9757, -9758, -9759, -9760, -9761, -9762, -9763, -9764, -9765, -9766, -9767, -9768, -9769, -9770, -9771, -9772, -9773, -9774, -9775, -9776, -9777, -9778, -9779, -9780, -9781, -10500, -10501, -10502, -10503, -10504, -10505, -10506, -10507, -10508, -10509, -10510, -10511, -10512, -10513, -10514, -10515, -10516, -10650, -11000, -11001, -11002, -11003, -11004, -11005, -11006, -11007, -11008, -11009, -11010, -11011, -11012, -11013, -11014, -11015, -11016, -11017, -11018, -11019, -11020, -11021, -11022, -11023, -11024, -11025, -11026, -11027, -11028, -11029, -11030, -11031, -11032, -11033, -11034, -11035, -11036, -11037, -11038, -11039, -11040, -11041, -11042, -11043, -11044, -11045, -11046, -11047, -11048, -11049, -11050, -11051, -11052, -20000, -21000, -22998, -30926, -32491, -38104, -38105}; +int g_all_ob_errnos[2326] = {0, -4000, -4001, -4002, -4003, -4004, -4005, -4006, -4007, -4008, -4009, -4010, -4011, -4012, -4013, -4014, -4015, -4016, -4017, -4018, -4019, -4020, -4021, -4022, -4023, -4024, -4025, -4026, -4027, -4028, -4029, -4030, -4031, -4032, -4033, -4034, -4035, -4036, -4037, -4038, -4039, -4041, -4042, -4043, -4044, -4045, -4046, -4047, -4048, -4049, -4050, -4051, -4052, -4053, -4054, -4055, -4057, -4058, -4060, -4061, -4062, -4063, -4064, -4065, -4066, -4067, -4068, -4070, -4071, -4072, -4073, -4074, -4075, -4076, -4077, -4078, -4080, -4081, -4084, -4085, -4090, -4097, -4098, -4099, -4100, -4101, -4102, -4103, -4104, -4105, -4106, -4107, -4108, -4109, -4110, -4111, -4112, -4113, -4114, -4115, -4116, -4117, -4118, -4119, -4120, -4121, -4122, -4123, -4124, -4125, -4126, -4127, -4128, -4133, -4138, -4139, -4142, -4143, -4144, -4146, -4147, -4149, -4150, -4151, -4152, -4153, -4154, -4155, -4156, -4157, -4158, -4159, -4160, -4161, -4162, -4163, -4164, -4165, -4166, -4167, -4168, -4169, -4170, -4171, -4172, -4173, -4174, -4175, -4176, -4177, -4178, -4179, -4180, -4181, -4182, -4183, -4184, -4185, -4186, -4187, -4188, -4189, -4190, -4191, -4192, -4200, -4201, -4204, -4205, -4206, -4207, -4208, -4209, -4210, -4211, -4212, -4213, -4214, -4215, -4216, -4217, -4218, -4219, -4220, -4221, -4222, -4223, -4224, -4225, -4226, -4227, -4228, -4229, -4230, -4231, -4232, -4233, -4234, -4235, -4236, -4237, -4238, -4239, -4240, -4241, -4242, -4243, -4244, -4245, -4246, -4247, -4248, -4249, -4250, -4251, -4252, -4253, -4254, -4255, -4256, -4257, -4258, -4260, -4261, -4262, -4263, -4264, -4265, -4266, -4267, -4268, -4269, -4270, -4271, -4273, -4274, -4275, -4276, -4277, -4278, -4279, -4280, -4281, -4282, -4283, -4284, -4285, -4286, -4287, -4288, -4289, -4290, -4291, -4292, -4293, -4294, -4295, -4296, -4297, -4298, -4299, -4300, -4301, -4302, -4303, -4304, -4305, -4306, -4307, -4308, -4309, -4310, -4311, -4312, -4313, -4314, -4315, -4316, -4317, -4318, -4319, -4320, -4321, -4322, -4323, -4324, -4325, -4326, -4327, -4328, -4329, -4330, -4331, -4332, -4333, -4334, -4335, -4336, -4337, -4338, -4339, -4340, -4341, -4342, -4343, -4344, -4345, -4346, -4347, -4348, -4349, -4350, -4351, -4352, -4353, -4354, -4355, -4356, -4357, -4358, -4359, -4360, -4361, -4362, -4363, -4364, -4365, -4366, -4367, -4368, -4369, -4370, -4371, -4372, -4373, -4374, -4375, -4376, -4377, -4378, -4379, -4380, -4381, -4382, -4383, -4385, -4386, -4387, -4388, -4389, -4390, -4391, -4392, -4393, -4394, -4395, -4396, -4397, -4398, -4399, -4400, -4401, -4402, -4403, -4505, -4507, -4510, -4512, -4515, -4517, -4518, -4519, -4523, -4524, -4525, -4526, -4527, -4528, -4529, -4530, -4531, -4532, -4533, -4537, -4538, -4539, -4540, -4541, -4542, -4543, -4544, -4545, -4546, -4547, -4548, -4549, -4550, -4551, -4552, -4553, -4554, -4600, -4601, -4602, -4603, -4604, -4605, -4606, -4607, -4608, -4609, -4610, -4611, -4613, -4614, -4615, -4620, -4621, -4622, -4623, -4624, -4625, -4626, -4628, -4629, -4630, -4631, -4632, -4633, -4634, -4636, -4637, -4638, -4639, -4640, -4641, -4642, -4643, -4644, -4645, -4646, -4647, -4648, -4649, -4650, -4651, -4652, -4653, -4654, -4655, -4656, -4657, -4658, -4659, -4660, -4661, -4662, -4663, -4664, -4665, -4666, -4667, -4668, -4669, -4670, -4671, -4672, -4673, -4674, -4675, -4676, -4677, -4678, -4679, -4680, -4681, -4682, -4683, -4684, -4685, -4686, -4687, -4688, -4689, -4690, -4691, -4692, -4693, -4694, -4695, -4696, -4697, -4698, -4699, -4700, -4701, -4702, -4703, -4704, -4705, -4706, -4707, -4708, -4709, -4710, -4711, -4712, -4713, -4714, -4715, -4716, -4717, -4718, -4719, -4720, -4721, -4722, -4723, -4724, -4725, -4726, -4727, -4728, -4729, -4730, -4731, -4732, -4733, -4734, -4735, -4736, -4737, -4738, -4739, -4740, -4741, -4742, -4743, -4744, -4745, -4746, -4747, -4748, -4749, -4750, -4751, -4752, -4753, -4754, -4755, -4756, -4757, -4758, -4759, -4760, -4761, -4762, -4763, -4764, -4765, -4766, -4767, -4768, -4769, -4770, -4771, -4772, -4773, -4774, -4775, -4776, -4777, -4778, -4779, -4780, -4781, -4782, -4783, -4784, -5000, -5001, -5002, -5003, -5006, -5007, -5008, -5010, -5011, -5012, -5014, -5015, -5016, -5017, -5018, -5019, -5020, -5022, -5023, -5024, -5025, -5026, -5027, -5028, -5029, -5030, -5031, -5032, -5034, -5035, -5036, -5037, -5038, -5039, -5040, -5041, -5042, -5043, -5044, -5046, -5047, -5050, -5051, -5052, -5053, -5054, -5055, -5056, -5057, -5058, -5059, -5061, -5063, -5064, -5065, -5066, -5067, -5068, -5069, -5070, -5071, -5072, -5073, -5074, -5080, -5081, -5083, -5084, -5085, -5086, -5087, -5088, -5089, -5090, -5091, -5092, -5093, -5094, -5095, -5096, -5097, -5098, -5099, -5100, -5101, -5102, -5103, -5104, -5105, -5106, -5107, -5108, -5109, -5110, -5111, -5112, -5113, -5114, -5115, -5116, -5117, -5118, -5119, -5120, -5121, -5122, -5123, -5124, -5125, -5130, -5131, -5133, -5134, -5135, -5136, -5137, -5138, -5139, -5140, -5142, -5143, -5144, -5145, -5146, -5147, -5148, -5149, -5150, -5151, -5153, -5154, -5155, -5156, -5157, -5158, -5159, -5160, -5161, -5162, -5163, -5164, -5165, -5166, -5167, -5168, -5169, -5170, -5171, -5172, -5173, -5174, -5175, -5176, -5177, -5178, -5179, -5180, -5181, -5182, -5183, -5184, -5185, -5187, -5188, -5189, -5190, -5191, -5192, -5193, -5194, -5195, -5196, -5197, -5198, -5199, -5200, -5201, -5202, -5203, -5204, -5205, -5206, -5207, -5208, -5209, -5210, -5211, -5212, -5213, -5214, -5215, -5216, -5217, -5218, -5219, -5220, -5221, -5222, -5223, -5224, -5225, -5226, -5227, -5228, -5229, -5230, -5231, -5233, -5234, -5235, -5236, -5237, -5238, -5239, -5240, -5241, -5242, -5243, -5244, -5245, -5246, -5247, -5248, -5249, -5250, -5251, -5252, -5253, -5254, -5255, -5256, -5257, -5258, -5259, -5260, -5261, -5262, -5263, -5264, -5265, -5266, -5267, -5268, -5269, -5270, -5271, -5272, -5273, -5274, -5275, -5276, -5277, -5278, -5279, -5280, -5281, -5282, -5283, -5284, -5285, -5286, -5287, -5288, -5289, -5290, -5291, -5292, -5293, -5294, -5295, -5296, -5297, -5298, -5299, -5300, -5301, -5302, -5303, -5304, -5305, -5306, -5307, -5308, -5309, -5310, -5311, -5312, -5313, -5314, -5315, -5316, -5317, -5318, -5319, -5320, -5321, -5322, -5323, -5324, -5325, -5326, -5327, -5328, -5329, -5330, -5331, -5332, -5333, -5334, -5335, -5336, -5337, -5338, -5339, -5340, -5341, -5342, -5343, -5344, -5345, -5346, -5347, -5348, -5349, -5350, -5351, -5352, -5353, -5354, -5355, -5356, -5357, -5358, -5359, -5360, -5361, -5362, -5363, -5364, -5365, -5366, -5367, -5368, -5369, -5370, -5371, -5372, -5373, -5374, -5375, -5376, -5377, -5378, -5379, -5380, -5381, -5382, -5383, -5384, -5385, -5386, -5387, -5388, -5389, -5390, -5400, -5401, -5402, -5403, -5404, -5405, -5406, -5407, -5408, -5409, -5410, -5411, -5412, -5413, -5414, -5415, -5416, -5417, -5418, -5419, -5420, -5421, -5422, -5423, -5424, -5425, -5426, -5427, -5428, -5429, -5430, -5431, -5432, -5433, -5434, -5435, -5436, -5437, -5438, -5439, -5440, -5441, -5442, -5443, -5444, -5445, -5446, -5447, -5448, -5449, -5450, -5451, -5452, -5453, -5454, -5455, -5456, -5457, -5458, -5459, -5460, -5461, -5462, -5463, -5464, -5465, -5466, -5467, -5468, -5469, -5470, -5471, -5472, -5473, -5474, -5475, -5476, -5477, -5478, -5479, -5480, -5481, -5482, -5483, -5484, -5485, -5486, -5487, -5488, -5489, -5490, -5491, -5492, -5493, -5494, -5495, -5496, -5497, -5498, -5499, -5500, -5501, -5502, -5503, -5504, -5505, -5506, -5507, -5508, -5509, -5510, -5511, -5512, -5513, -5514, -5515, -5516, -5517, -5518, -5519, -5520, -5521, -5522, -5540, -5541, -5542, -5543, -5544, -5545, -5546, -5547, -5548, -5549, -5550, -5551, -5552, -5553, -5554, -5555, -5556, -5557, -5558, -5559, -5560, -5561, -5562, -5563, -5564, -5565, -5566, -5567, -5568, -5569, -5570, -5571, -5572, -5573, -5574, -5575, -5576, -5577, -5578, -5579, -5580, -5581, -5582, -5583, -5584, -5585, -5586, -5587, -5588, -5589, -5590, -5591, -5592, -5593, -5594, -5595, -5596, -5597, -5598, -5599, -5600, -5601, -5602, -5603, -5604, -5605, -5607, -5608, -5609, -5610, -5611, -5612, -5613, -5614, -5615, -5616, -5617, -5618, -5619, -5620, -5621, -5622, -5623, -5624, -5625, -5626, -5627, -5628, -5629, -5630, -5631, -5632, -5633, -5634, -5635, -5636, -5637, -5638, -5639, -5640, -5641, -5642, -5643, -5644, -5645, -5646, -5647, -5648, -5649, -5650, -5651, -5652, -5653, -5654, -5655, -5656, -5657, -5658, -5659, -5660, -5661, -5662, -5663, -5664, -5665, -5666, -5667, -5668, -5671, -5672, -5673, -5674, -5675, -5676, -5677, -5678, -5679, -5680, -5681, -5682, -5683, -5684, -5685, -5686, -5687, -5688, -5689, -5690, -5691, -5692, -5693, -5694, -5695, -5696, -5697, -5698, -5699, -5700, -5701, -5702, -5703, -5704, -5705, -5706, -5707, -5708, -5709, -5710, -5711, -5712, -5713, -5714, -5715, -5716, -5717, -5718, -5719, -5720, -5721, -5722, -5723, -5724, -5725, -5726, -5727, -5728, -5729, -5730, -5731, -5732, -5733, -5734, -5735, -5736, -5737, -5738, -5739, -5740, -5741, -5742, -5743, -5744, -5745, -5746, -5747, -5748, -5749, -5750, -5751, -5752, -5753, -5754, -5755, -5756, -5757, -5758, -5759, -5760, -5761, -5762, -5763, -5764, -5765, -5766, -5768, -5769, -5770, -5771, -5772, -5773, -5774, -5777, -5778, -5779, -5780, -5781, -5785, -5786, -5787, -5788, -5789, -5790, -5791, -5792, -5793, -5794, -5795, -5796, -5797, -5798, -5799, -5800, -5801, -5802, -5803, -5804, -5805, -5806, -5807, -5808, -5809, -5810, -5811, -5812, -5813, -5814, -5815, -5816, -5817, -5818, -5819, -5820, -5821, -5822, -5823, -5824, -5825, -5826, -5827, -5828, -5829, -5830, -5831, -5832, -5833, -5834, -5835, -5836, -5837, -5838, -5839, -5840, -5841, -5842, -5843, -5844, -5845, -5846, -5847, -5848, -5849, -5850, -5851, -5852, -5853, -5854, -5855, -5856, -5857, -5858, -5859, -5860, -5861, -5862, -5863, -5864, -5865, -5866, -5867, -5868, -5869, -5870, -5871, -5872, -5873, -5874, -5875, -5876, -5877, -5878, -5879, -5880, -5881, -5882, -5883, -5884, -5885, -5886, -5887, -5888, -5889, -5890, -5891, -5892, -5893, -5894, -5895, -5896, -5897, -5898, -5899, -5900, -5901, -5902, -5903, -5904, -5905, -5906, -5907, -5908, -5909, -5910, -5911, -5912, -5913, -5914, -5915, -5916, -5917, -5918, -5919, -5920, -5921, -5922, -5923, -5924, -5925, -5926, -5927, -5928, -5929, -5930, -5931, -5932, -5933, -5934, -5935, -5936, -5937, -5938, -5939, -5940, -5941, -5942, -5943, -5944, -5945, -5946, -5947, -5948, -5949, -5950, -5951, -5952, -5953, -5954, -5955, -5956, -5957, -5958, -5959, -5960, -5961, -5962, -5963, -5964, -5965, -5966, -5967, -5968, -5969, -5970, -5971, -5972, -5973, -5974, -5975, -5976, -5977, -5978, -5979, -5980, -5981, -5982, -5983, -5984, -5985, -5986, -5987, -5988, -5989, -5990, -5991, -5992, -5993, -5994, -5995, -5996, -5997, -5998, -5999, -6000, -6001, -6002, -6003, -6004, -6005, -6006, -6201, -6202, -6203, -6204, -6205, -6206, -6207, -6208, -6209, -6210, -6211, -6212, -6213, -6214, -6215, -6219, -6220, -6221, -6222, -6223, -6224, -6225, -6226, -6227, -6228, -6229, -6230, -6231, -6232, -6233, -6234, -6235, -6236, -6237, -6238, -6239, -6240, -6241, -6242, -6243, -6244, -6245, -6246, -6247, -6248, -6249, -6250, -6251, -6252, -6253, -6254, -6255, -6256, -6257, -6258, -6259, -6260, -6261, -6262, -6263, -6264, -6265, -6266, -6267, -6268, -6269, -6270, -6271, -6272, -6273, -6274, -6275, -6276, -6277, -6278, -6279, -6280, -6281, -6282, -6283, -6284, -6285, -6301, -6302, -6303, -6304, -6305, -6306, -6307, -6308, -6309, -6310, -6311, -6312, -6313, -6314, -6315, -6316, -6317, -6318, -6319, -6320, -6321, -6322, -6323, -6324, -6325, -6326, -6327, -6328, -6329, -6330, -6331, -6332, -7000, -7001, -7002, -7003, -7004, -7005, -7006, -7007, -7010, -7011, -7012, -7013, -7014, -7015, -7021, -7022, -7024, -7025, -7026, -7027, -7029, -7030, -7031, -7032, -7033, -7034, -7035, -7036, -7037, -7038, -7039, -7040, -7041, -7100, -7101, -7102, -7103, -7104, -7105, -7106, -7107, -7108, -7109, -7110, -7111, -7112, -7113, -7114, -7115, -7116, -7117, -7118, -7119, -7120, -7121, -7122, -7123, -7124, -7201, -7202, -7203, -7204, -7205, -7206, -7207, -7208, -7209, -7210, -7211, -7212, -7213, -7214, -7215, -7216, -7217, -7218, -7219, -7220, -7221, -7222, -7223, -7224, -7225, -7226, -7227, -7228, -7229, -7230, -7231, -7232, -7233, -7234, -7235, -7236, -7237, -7238, -7239, -7240, -7241, -7242, -7243, -7244, -7246, -7247, -7248, -7249, -7250, -7251, -7252, -7253, -7254, -7255, -7256, -7257, -7258, -7259, -7260, -7261, -7262, -7263, -7264, -7265, -7266, -7267, -7268, -7269, -7270, -7271, -7272, -7273, -7274, -7275, -7276, -7277, -7278, -7279, -7280, -7281, -7282, -7283, -7284, -7285, -7286, -7287, -7288, -7289, -7290, -7291, -7292, -7293, -7294, -7295, -7296, -7297, -7298, -7299, -7300, -7301, -7302, -7402, -7403, -7404, -7405, -7406, -7407, -7408, -7409, -7410, -7411, -7412, -7413, -7414, -7415, -7416, -7417, -7418, -7419, -7420, -7421, -7422, -7423, -7424, -7425, -7426, -7427, -7428, -7429, -7430, -7431, -7432, -7433, -7434, -7435, -7600, -7601, -7602, -7603, -7604, -8001, -8002, -8003, -8004, -8005, -9001, -9002, -9003, -9004, -9005, -9006, -9007, -9008, -9009, -9010, -9011, -9012, -9013, -9014, -9015, -9016, -9017, -9018, -9019, -9020, -9022, -9023, -9024, -9025, -9026, -9027, -9028, -9029, -9030, -9031, -9032, -9033, -9034, -9035, -9036, -9037, -9038, -9039, -9040, -9041, -9042, -9043, -9044, -9045, -9046, -9047, -9048, -9049, -9050, -9051, -9052, -9053, -9054, -9057, -9058, -9059, -9060, -9061, -9062, -9063, -9064, -9065, -9066, -9069, -9070, -9071, -9072, -9073, -9074, -9075, -9076, -9077, -9078, -9079, -9080, -9081, -9082, -9083, -9084, -9085, -9086, -9087, -9088, -9089, -9090, -9091, -9092, -9093, -9094, -9095, -9096, -9097, -9098, -9099, -9100, -9101, -9102, -9103, -9104, -9105, -9106, -9107, -9108, -9109, -9110, -9111, -9112, -9113, -9114, -9115, -9116, -9117, -9118, -9119, -9120, -9121, -9122, -9123, -9124, -9125, -9126, -9127, -9200, -9201, -9202, -9203, -9501, -9502, -9503, -9504, -9505, -9506, -9507, -9508, -9509, -9510, -9512, -9513, -9514, -9515, -9516, -9518, -9519, -9520, -9521, -9522, -9523, -9524, -9525, -9526, -9527, -9528, -9529, -9530, -9531, -9532, -9533, -9534, -9535, -9536, -9537, -9538, -9539, -9540, -9541, -9542, -9543, -9544, -9545, -9546, -9547, -9548, -9549, -9550, -9551, -9552, -9553, -9554, -9555, -9556, -9557, -9558, -9559, -9560, -9561, -9562, -9563, -9564, -9565, -9566, -9567, -9568, -9569, -9570, -9571, -9572, -9573, -9574, -9575, -9576, -9577, -9578, -9579, -9580, -9581, -9582, -9583, -9584, -9585, -9586, -9587, -9588, -9589, -9590, -9591, -9592, -9593, -9594, -9595, -9596, -9597, -9598, -9599, -9600, -9601, -9602, -9603, -9604, -9605, -9606, -9607, -9608, -9609, -9610, -9611, -9612, -9613, -9614, -9615, -9616, -9617, -9618, -9619, -9620, -9621, -9622, -9623, -9624, -9625, -9626, -9627, -9628, -9629, -9630, -9631, -9632, -9633, -9634, -9635, -9636, -9637, -9638, -9639, -9640, -9641, -9642, -9643, -9644, -9645, -9646, -9647, -9648, -9649, -9650, -9651, -9652, -9653, -9654, -9655, -9656, -9657, -9658, -9659, -9660, -9661, -9662, -9663, -9664, -9665, -9666, -9667, -9668, -9669, -9670, -9671, -9672, -9673, -9674, -9675, -9676, -9677, -9678, -9679, -9680, -9681, -9682, -9683, -9684, -9685, -9686, -9687, -9688, -9689, -9690, -9691, -9692, -9693, -9694, -9695, -9696, -9697, -9698, -9699, -9700, -9701, -9702, -9703, -9704, -9705, -9706, -9707, -9708, -9709, -9710, -9711, -9712, -9713, -9714, -9715, -9716, -9717, -9718, -9719, -9720, -9721, -9722, -9723, -9724, -9725, -9726, -9727, -9728, -9729, -9730, -9731, -9732, -9733, -9734, -9735, -9736, -9737, -9738, -9739, -9740, -9741, -9742, -9743, -9744, -9745, -9746, -9747, -9748, -9749, -9750, -9751, -9752, -9753, -9754, -9755, -9756, -9757, -9758, -9759, -9760, -9761, -9762, -9763, -9764, -9765, -9766, -9767, -9768, -9769, -9770, -9771, -9772, -9773, -9774, -9775, -9776, -9777, -9778, -9779, -9780, -9781, -9782, -10500, -10501, -10502, -10503, -10504, -10505, -10506, -10507, -10508, -10509, -10510, -10511, -10512, -10513, -10514, -10515, -10516, -10650, -11000, -11001, -11002, -11003, -11004, -11005, -11006, -11007, -11008, -11009, -11010, -11011, -11012, -11013, -11014, -11015, -11016, -11017, -11018, -11019, -11020, -11021, -11022, -11023, -11024, -11025, -11026, -11027, -11028, -11029, -11030, -11031, -11032, -11033, -11034, -11035, -11036, -11037, -11038, -11039, -11040, -11041, -11042, -11043, -11044, -11045, -11046, -11047, -11048, -11049, -11050, -11051, -11052, -20000, -21000, -22998, -30926, -32491, -38104, -38105}; const char *ob_error_name(const int err) { const char *ret = "Unknown error"; diff --git a/src/share/ob_errno.def b/src/share/ob_errno.def index eee41eee9..667065f98 100755 --- a/src/share/ob_errno.def +++ b/src/share/ob_errno.def @@ -2643,6 +2643,7 @@ DEFINE_ERROR(OB_ERR_EVENT_CANNOT_CREATE_IN_THE_PAST, -9778, ER_EVENT_CANNOT_CREA DEFINE_ERROR(OB_ERR_EVENT_CANNOT_ALTER_IN_THE_PAST, -9779, ER_EVENT_CANNOT_ALTER_IN_THE_PAST, "HY000", "Event execution time is in the past and ON COMPLETION NOT PRESERVE is set. The event was not changed. Specify a time in the future."); DEFINE_ERROR(OB_ERR_EVENT_RECURSION_FORBIDDEN, -9780, ER_EVENT_RECURSION_FORBIDDEN, "HY000", "Recursion of EVENT DDL statements is forbidden when body is present"); DEFINE_ORACLE_ERROR(OB_NO_PARTITION_FOR_GIVEN_VALUE_SCHEMA_ERROR, -9781, ER_NO_PARTITION_FOR_GIVEN_VALUE, "HY000", "Table has no partition for value", 14400, "inserted partition key does not map to any partition"); +DEFINE_ORACLE_ERROR(OB_ERR_INVALID_CHARACTER, -9782, -1, "HY000", "invalid character", 911, "invalid character"); // 余留位置 //////////////////////////////////////////////////////////////// // PL/SQL错误码值域 [-9500, -10000) diff --git a/src/share/ob_errno.h b/src/share/ob_errno.h index 588f0adf7..17eebbd79 100755 --- a/src/share/ob_errno.h +++ b/src/share/ob_errno.h @@ -1867,6 +1867,7 @@ constexpr int OB_ERR_EVENT_CANNOT_CREATE_IN_THE_PAST = -9778; constexpr int OB_ERR_EVENT_CANNOT_ALTER_IN_THE_PAST = -9779; constexpr int OB_ERR_EVENT_RECURSION_FORBIDDEN = -9780; constexpr int OB_NO_PARTITION_FOR_GIVEN_VALUE_SCHEMA_ERROR = -9781; +constexpr int OB_ERR_INVALID_CHARACTER = -9782; constexpr int OB_ERR_KV_GLOBAL_INDEX_ROUTE = -10500; constexpr int OB_TTL_NOT_ENABLE = -10501; constexpr int OB_TTL_COLUMN_NOT_EXIST = -10502; @@ -4169,6 +4170,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_EVENT_CANNOT_ALTER_IN_THE_PAST__USER_ERROR_MSG "Event execution time is in the past and ON COMPLETION NOT PRESERVE is set. The event was not changed. Specify a time in the future." #define OB_ERR_EVENT_RECURSION_FORBIDDEN__USER_ERROR_MSG "Recursion of EVENT DDL statements is forbidden when body is present" #define OB_NO_PARTITION_FOR_GIVEN_VALUE_SCHEMA_ERROR__USER_ERROR_MSG "Table has no partition for value" +#define OB_ERR_INVALID_CHARACTER__USER_ERROR_MSG "invalid character" #define OB_ERR_KV_GLOBAL_INDEX_ROUTE__USER_ERROR_MSG "incorrect route for obkv global index, client router should refresh." #define OB_TTL_NOT_ENABLE__USER_ERROR_MSG "TTL feature is not enabled" #define OB_TTL_COLUMN_NOT_EXIST__USER_ERROR_MSG "TTL column '%.*s' not exists" @@ -8745,6 +8747,8 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_EVENT_RECURSION_FORBIDDEN__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -9780, Recursion of EVENT DDL statements is forbidden when body is present" #define OB_NO_PARTITION_FOR_GIVEN_VALUE_SCHEMA_ERROR__ORA_USER_ERROR_MSG "ORA-14400: inserted partition key does not map to any partition" #define OB_NO_PARTITION_FOR_GIVEN_VALUE_SCHEMA_ERROR__OBE_USER_ERROR_MSG "OBE-14400: inserted partition key does not map to any partition" +#define OB_ERR_INVALID_CHARACTER__ORA_USER_ERROR_MSG "ORA-00911: invalid character" +#define OB_ERR_INVALID_CHARACTER__OBE_USER_ERROR_MSG "OBE-00911: invalid character" #define OB_ERR_KV_GLOBAL_INDEX_ROUTE__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -10500, incorrect route for obkv global index, client router should refresh." #define OB_ERR_KV_GLOBAL_INDEX_ROUTE__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -10500, incorrect route for obkv global index, client router should refresh." #define OB_TTL_NOT_ENABLE__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -10501, TTL feature is not enabled" @@ -8906,7 +8910,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_INVALID_DATE_MSG_FMT_V2__ORA_USER_ERROR_MSG "ORA-01861: Incorrect datetime value for column '%.*s' at row %ld" #define OB_ERR_INVALID_DATE_MSG_FMT_V2__OBE_USER_ERROR_MSG "OBE-01861: Incorrect datetime value for column '%.*s' at row %ld" -extern int g_all_ob_errnos[2325]; +extern int g_all_ob_errnos[2326]; const char *ob_error_name(const int oberr); const char* ob_error_cause(const int oberr); From 9bb3af5e35dc12aefebe16b78b72596b42c9d216 Mon Sep 17 00:00:00 2001 From: obdev Date: Fri, 23 Aug 2024 07:15:18 +0000 Subject: [PATCH 188/249] recover compaction dag count limit configure --- .../scheduler/ob_tenant_dag_scheduler.cpp | 53 ++++++++++++++++--- src/share/scheduler/ob_tenant_dag_scheduler.h | 4 +- 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/src/share/scheduler/ob_tenant_dag_scheduler.cpp b/src/share/scheduler/ob_tenant_dag_scheduler.cpp index 225a2f6df..120b53d9a 100644 --- a/src/share/scheduler/ob_tenant_dag_scheduler.cpp +++ b/src/share/scheduler/ob_tenant_dag_scheduler.cpp @@ -2034,7 +2034,7 @@ int ObDagPrioScheduler::inner_add_dag_( COMMON_LOG(WARN, "unexpected value", K(ret), K(dag->get_priority()), K_(priority), KP_(scheduler)); } else if (check_size_overflow && scheduler_->dag_count_overflow(dag->get_type())) { ret = OB_SIZE_OVERFLOW; - COMMON_LOG(WARN, "ObTenantDagScheduler is full", K(ret), "dag_limit", scheduler_->get_dag_limit(), KPC(dag)); + COMMON_LOG(WARN, "ObTenantDagScheduler is full", K(ret), "dag_limit", scheduler_->get_dag_limit((ObDagPrio::ObDagPrioEnum)dag->get_priority()), KPC(dag)); } else if (OB_FAIL(add_dag_into_list_and_map_( is_waiting_dag_type(dag->get_type()) ? WAITING_DAG_LIST : is_rank_dag_type(dag->get_type()) ? RANK_DAG_LIST : READY_DAG_LIST, // compaction dag should add into RANK_LIST first. @@ -3895,6 +3895,7 @@ ObTenantDagScheduler::ObTenantDagScheduler() tg_id_(-1), dag_cnt_(0), dag_limit_(0), + compaction_dag_limit_(0), check_period_(0), loop_waiting_dag_list_period_(0), total_worker_cnt_(0), @@ -3935,6 +3936,7 @@ void ObTenantDagScheduler::reload_config() set_thread_score(ObDagPrio::DAG_PRIO_HA_LOW, tenant_config->ha_low_thread_score); set_thread_score(ObDagPrio::DAG_PRIO_DDL, tenant_config->ddl_thread_score); set_thread_score(ObDagPrio::DAG_PRIO_TTL, tenant_config->ttl_thread_score); + set_compaction_dag_limit(tenant_config->compaction_dag_cnt_limit); } } @@ -3986,6 +3988,7 @@ int ObTenantDagScheduler::init( check_period_ = check_period; loop_waiting_dag_list_period_ = loop_waiting_list_period; dag_limit_ = dag_limit; + compaction_dag_limit_ = dag_limit; work_thread_num_ = default_work_thread_num_ = 0; MEMSET(dag_cnts_, 0, sizeof(dag_cnts_)); MEMSET(running_dag_cnts_, 0, sizeof(running_dag_cnts_)); @@ -4071,6 +4074,7 @@ void ObTenantDagScheduler::reset() } dag_cnt_ = 0; dag_limit_ = 0; + compaction_dag_limit_ = 0; total_worker_cnt_ = 0; work_thread_num_ = 0; total_running_task_cnt_ = 0; @@ -4137,6 +4141,17 @@ int ObTenantDagScheduler::add_dag_net(ObIDagNet *dag_net) return ret; } +int64_t ObTenantDagScheduler::get_dag_limit(const ObDagPrio::ObDagPrioEnum dag_prio) +{ + int64_t dag_limit = dag_limit_; + if (ObDagPrio::DAG_PRIO_COMPACTION_HIGH == dag_prio + || ObDagPrio::DAG_PRIO_COMPACTION_MID == dag_prio + || ObDagPrio::DAG_PRIO_COMPACTION_LOW == dag_prio) { + dag_limit = compaction_dag_limit_; + } + return dag_limit; +} + int ObTenantDagScheduler::add_dag( ObIDag *dag, const bool emergency, @@ -4642,12 +4657,12 @@ int ObTenantDagScheduler::generate_dag_id(ObDagId &dag_id) bool ObTenantDagScheduler::dag_count_overflow(const ObDagType::ObDagTypeEnum type) { - return get_dag_count(type) >= get_dag_limit(); + return get_dag_count(type) >= get_dag_limit(OB_DAG_TYPES[type].init_dag_prio_); } int64_t ObTenantDagScheduler::allowed_schedule_dag_count(const ObDagType::ObDagTypeEnum type) { - int64_t count = get_dag_limit() - get_dag_count(type); + int64_t count = get_dag_limit(OB_DAG_TYPES[type].init_dag_prio_) - get_dag_count(type); return count < 0 ? 0 : count; } @@ -4818,6 +4833,30 @@ void ObTenantDagScheduler::destroy_all_workers() } } +int ObTenantDagScheduler::set_compaction_dag_limit(const int64_t new_val) +{ + int ret = OB_SUCCESS; + const int64_t old_val = compaction_dag_limit_; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + COMMON_LOG(WARN, "ObTenantDagScheduler is not inited", K(ret)); + } else if (new_val < 0) { + ret = OB_INVALID_ARGUMENT; + COMMON_LOG(WARN, "invalid argument", K(ret), K(new_val)); + } else if (old_val != new_val) { + ObThreadCondGuard guard(scheduler_sync_); + if (OB_SUCC(ret)) { + compaction_dag_limit_ = new_val; + if (OB_FAIL(scheduler_sync_.signal())) { + STORAGE_LOG(WARN, "Failed to signal", K(ret), K(compaction_dag_limit_)); + } else { + COMMON_LOG(INFO, "set compaction dag limit successfully", K(compaction_dag_limit_)); + } + } + } + return ret; +} + int ObTenantDagScheduler::set_thread_score(const int64_t priority, const int64_t score) { int ret = OB_SUCCESS; @@ -4831,13 +4870,11 @@ int ObTenantDagScheduler::set_thread_score(const int64_t priority, const int64_t COMMON_LOG(WARN, "invalid argument", K(ret), K(priority), K(score)); } else if (OB_FAIL(prio_sche_[priority].set_thread_score(score, old_val, new_val))){ COMMON_LOG(WARN, "fail to set thread score", K(ret)); - } else { + } else if (old_val != new_val) { ObThreadCondGuard guard(scheduler_sync_); if (OB_SUCC(ret)) { - if (old_val != new_val) { - work_thread_num_ -= old_val; - work_thread_num_ += new_val; - } + work_thread_num_ -= old_val; + work_thread_num_ += new_val; if (OB_FAIL(scheduler_sync_.signal())) { STORAGE_LOG(WARN, "Failed to signal", K(ret), K(priority), K(score)); } else { diff --git a/src/share/scheduler/ob_tenant_dag_scheduler.h b/src/share/scheduler/ob_tenant_dag_scheduler.h index 342df8cd0..1343f8c67 100644 --- a/src/share/scheduler/ob_tenant_dag_scheduler.h +++ b/src/share/scheduler/ob_tenant_dag_scheduler.h @@ -1087,7 +1087,7 @@ public: ObThreadCondGuard guard(scheduler_sync_); return work_thread_num_; } - int64_t get_dag_limit() const { return dag_limit_; } + int64_t get_dag_limit(const ObDagPrio::ObDagPrioEnum dag_prio); bool is_empty() { bool bret = true; @@ -1180,6 +1180,7 @@ private: int try_reclaim_threads(); void destroy_all_workers(); int set_thread_score(const int64_t priority, const int64_t concurrency); + int set_compaction_dag_limit(const int64_t new_val); void inner_get_suggestion_reason(const ObDagType::ObDagTypeEnum type, int64_t &reason); void dump_dag_status(const bool force_dump = false); void diagnose_for_suggestion(); @@ -1197,6 +1198,7 @@ private: int tg_id_; int64_t dag_cnt_; // atomic value int64_t dag_limit_; // only set in init/destroy + int64_t compaction_dag_limit_; int64_t check_period_; // only set in init/destroy int64_t loop_waiting_dag_list_period_; // only set in init/destroy int64_t total_worker_cnt_; // lock by scheduler_sync_ From 3c077a1a6d6899899f0b119ce796fbfa711f4449 Mon Sep 17 00:00:00 2001 From: helloamateur Date: Fri, 23 Aug 2024 09:57:37 +0000 Subject: [PATCH 189/249] [CP] [GIS] fix srs memory expand in exceptional cases --- src/observer/omt/ob_tenant_srs.cpp | 14 +++++++------- src/observer/omt/ob_tenant_srs.h | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/observer/omt/ob_tenant_srs.cpp b/src/observer/omt/ob_tenant_srs.cpp index d59f0822b..2d4e87c74 100644 --- a/src/observer/omt/ob_tenant_srs.cpp +++ b/src/observer/omt/ob_tenant_srs.cpp @@ -474,7 +474,7 @@ int ObTenantSrs::fetch_all_srs(ObSrsCacheSnapShot *&srs_snapshot, bool is_sys_sr const ObSrsItem *tmp = NULL; res_count++; if (OB_ISNULL(snapshot)) { - snapshot = OB_NEWx(ObSrsCacheSnapShot, &allocator_, &alloc_, snapshot_type); + snapshot = OB_NEWx(ObSrsCacheSnapShot, &allocator_, snapshot_type); if (OB_ISNULL(snapshot)) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to create ObSrsCacheSnapShot", K(ret)); @@ -584,16 +584,16 @@ int ObSrsCacheSnapShot::parse_srs_item(ObMySQLResult *result, const ObSrsItem *& LOG_WARN("failed to extract maxx value", K(ret)); } else if (OB_FAIL(extract_bounds_numberic(result, "maxY", max_y))) { LOG_WARN("failed to extract maxy value", K(ret)); - } else if (OB_FAIL(ObSrsWktParser::parse_srs_wkt(*allocator_, srs_id, definition, srs_info))) { + } else if (OB_FAIL(ObSrsWktParser::parse_srs_wkt(allocator_, srs_id, definition, srs_info))) { LOG_WARN("failed to parse srs wkt from definition", K(ret), K(definition)); } else { - ObSrsItem *new_srs_item = OB_NEWx(ObSrsItem, allocator_, srs_info); + ObSrsItem *new_srs_item = OB_NEWx(ObSrsItem, (&allocator_), srs_info); if (OB_ISNULL(new_srs_item)) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to alloc memory for srs item", K(ret)); } else if (!proj4text.empty()) { srs_info->set_bounds(min_x, min_y, max_x, max_y); - if (OB_FAIL(srs_info->set_proj4text(*allocator_, proj4text))) { + if (OB_FAIL(srs_info->set_proj4text(allocator_, proj4text))) { LOG_WARN("fail to set proj4text for srs item", K(ret), K(srs_id)); } } @@ -611,14 +611,14 @@ int ObSrsCacheSnapShot::add_pg_reserved_srs_item(const ObString &pg_wkt, const u ObSpatialReferenceSystemBase *srs_info = NULL; lib::ObMallocHookAttrGuard malloc_guard(lib::ObMemAttr(common::OB_SERVER_TENANT_ID, "SRSWKTParser")); - if (OB_FAIL(ObSrsWktParser::parse_srs_wkt(*allocator_, srs_id, pg_wkt, srs_info))) { + if (OB_FAIL(ObSrsWktParser::parse_srs_wkt(allocator_, srs_id, pg_wkt, srs_info))) { LOG_WARN("failed to parse pg reserved srs wkt", K(ret), K(srs_id), K(pg_wkt)); } else { - ObSrsItem *new_srs_item = OB_NEWx(ObSrsItem, allocator_, srs_info); + ObSrsItem *new_srs_item = OB_NEWx(ObSrsItem, (&allocator_), srs_info); if (OB_ISNULL(new_srs_item)) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to alloc memory for srs item", K(ret)); - } else if (OB_FAIL(ObGeoTypeUtil::get_pg_reserved_prj4text(allocator_, srs_id, proj4text))) { + } else if (OB_FAIL(ObGeoTypeUtil::get_pg_reserved_prj4text(&allocator_, srs_id, proj4text))) { LOG_WARN("fail to generate proj4text for pg srs item", K(ret)); } else if (OB_FAIL(add_srs_item(new_srs_item->get_srid(), new_srs_item))) { LOG_WARN("failed to add pg srs item to snapshot", K(ret), K(new_srs_item->get_srid())); diff --git a/src/observer/omt/ob_tenant_srs.h b/src/observer/omt/ob_tenant_srs.h index d1e21fe4a..336a6da7f 100644 --- a/src/observer/omt/ob_tenant_srs.h +++ b/src/observer/omt/ob_tenant_srs.h @@ -49,8 +49,8 @@ class ObSrsCacheSnapShot { public: static const uint32_t SRS_ITEM_BUCKET_NUM = 6144; - explicit ObSrsCacheSnapShot(common::ObIAllocator *allocator, ObSrsCacheType srs_type) - : allocator_(allocator), srs_type_(srs_type), srs_version_(0), ref_count_(0) {} + explicit ObSrsCacheSnapShot(ObSrsCacheType srs_type) + : allocator_("SrsSnapShot", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), srs_type_(srs_type), srs_version_(0), ref_count_(0) {} virtual ~ObSrsCacheSnapShot() { srs_item_map_.destroy(); } int init() { return srs_item_map_.create(SRS_ITEM_BUCKET_NUM, "SrsSnapShot", "SrsSnapShot", MTL_ID()); } int add_srs_item(uint64_t srid, const common::ObSrsItem* srs_item) { return srs_item_map_.set_refactored(srid, srs_item); } @@ -66,7 +66,7 @@ public: int add_pg_reserved_srs_item(const common::ObString &pg_wkt, const uint32_t srs_id); private: - common::ObIAllocator *allocator_; + common::ObArenaAllocator allocator_; ObSrsCacheType srs_type_; uint64_t srs_version_; volatile int64_t ref_count_; From 697cf80681fe721c11c59110f1aec75a24486774 Mon Sep 17 00:00:00 2001 From: zhjc1124 Date: Fri, 23 Aug 2024 10:16:15 +0000 Subject: [PATCH 190/249] fix cgroup issues --- src/observer/ob_srv_deliver.h | 16 +++++++++------- src/observer/omt/ob_multi_tenant.cpp | 3 ++- src/observer/omt/ob_tenant.cpp | 4 ++-- src/observer/omt/ob_th_worker.cpp | 1 - src/share/rc/ob_tenant_base.cpp | 7 +++++-- src/share/resource_manager/ob_cgroup_ctrl.cpp | 4 ++-- src/share/resource_manager/ob_cgroup_ctrl.h | 6 ++++-- src/sql/engine/px/ob_dfo.h | 7 +------ src/sql/engine/px/ob_px_worker.cpp | 2 -- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/observer/ob_srv_deliver.h b/src/observer/ob_srv_deliver.h index 316c61686..674c6020e 100644 --- a/src/observer/ob_srv_deliver.h +++ b/src/observer/ob_srv_deliver.h @@ -43,8 +43,9 @@ class QueueThread { public: QueueThread(const char *thread_name = nullptr, - uint64_t tenant_id = OB_SERVER_TENANT_ID) - : thread_(queue_, thread_name, tenant_id), tg_id_(0), + uint64_t tenant_id = OB_SERVER_TENANT_ID, + uint64_t group_id = share::OBCG_DEFAULT) + : thread_(queue_, thread_name, tenant_id, group_id), tg_id_(0), tenant_id_(tenant_id), n_thread_(0) {} ~QueueThread() { destroy(); } @@ -65,16 +66,16 @@ public: void destroy() { TG_DESTROY(tg_id_); } class Thread : public lib::TGRunnable { public: - Thread(ObReqQueue &queue, const char *thread_name, const uint64_t tenant_id) - : queue_(queue), thread_name_(thread_name), tenant_id_(tenant_id) {} + Thread(ObReqQueue &queue, const char *thread_name, + const uint64_t tenant_id, const uint64_t group_id) + : queue_(queue), thread_name_(thread_name), + tenant_id_(tenant_id), group_id_(group_id) {} void run1() { if (thread_name_ != nullptr) { lib::set_thread_name(thread_name_, get_thread_idx()); } - if (GCONF._enable_new_sql_nio && GCONF._enable_tenant_sql_net_thread) { - lib::SET_GROUP_ID(share::OBCG_MYSQL_LOGIN); - } + lib::SET_GROUP_ID(group_id_); queue_.loop(); } @@ -82,6 +83,7 @@ public: ObReqQueue &queue_; const char *thread_name_; const uint64_t tenant_id_; + const uint64_t group_id_; } thread_; ObReqQueue queue_; int tg_id_; diff --git a/src/observer/omt/ob_multi_tenant.cpp b/src/observer/omt/ob_multi_tenant.cpp index 1d7ad04e3..1109b35b6 100644 --- a/src/observer/omt/ob_multi_tenant.cpp +++ b/src/observer/omt/ob_multi_tenant.cpp @@ -388,7 +388,8 @@ static int start_mysql_queue(QueueThread *&qthread) int ret = OB_SUCCESS; const uint64_t tenant_id = MTL_ID(); if (is_sys_tenant(tenant_id) || is_user_tenant(tenant_id)) { - qthread = OB_NEW(QueueThread, ObMemAttr(tenant_id, ObModIds::OB_RPC), "MysqlQueueTh", tenant_id); + qthread = OB_NEW(QueueThread, ObMemAttr(tenant_id, ObModIds::OB_RPC), + "MysqlQueueTh", tenant_id, share::OBCG_MYSQL_LOGIN); if (OB_ISNULL(qthread)) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to new qthread", K(ret), K(tenant_id)); diff --git a/src/observer/omt/ob_tenant.cpp b/src/observer/omt/ob_tenant.cpp index 0ae8266d2..97804e31c 100644 --- a/src/observer/omt/ob_tenant.cpp +++ b/src/observer/omt/ob_tenant.cpp @@ -1854,7 +1854,7 @@ void ObTenant::lq_end(ObThWorker &w) { int ret = OB_SUCCESS; if (w.is_lq_yield()) { - if (OB_FAIL(cgroup_ctrl_.add_self_to_cgroup_(id_, w.get_group_id()))) { + if (OB_FAIL(SET_GROUP_ID(share::OBCG_DEFAULT))) { LOG_WARN("move thread from lq group failed", K(ret), K(id_)); } else { w.set_lq_yield(false); @@ -1889,7 +1889,7 @@ int ObTenant::lq_yield(ObThWorker &w) } } else if (w.is_lq_yield()) { // avoid duplicate change group - } else if (OB_FAIL(cgroup_ctrl_.add_self_to_cgroup_(id_, share::OBCG_LQ))) { + } else if (OB_FAIL(SET_GROUP_ID(share::OBCG_LQ))) { LOG_WARN("move thread to lq group failed", K(ret), K(id_)); } else { w.set_lq_yield(); diff --git a/src/observer/omt/ob_th_worker.cpp b/src/observer/omt/ob_th_worker.cpp index f8c5c4e56..00dd2b16b 100644 --- a/src/observer/omt/ob_th_worker.cpp +++ b/src/observer/omt/ob_th_worker.cpp @@ -428,7 +428,6 @@ void ObThWorker::run(int64_t idx) int64_t tenant_id = -1; int64_t req_recv_timestamp = -1; int32_t worker_level = -1; - SET_GROUP_ID(get_group_id()); this->worker(tenant_id, req_recv_timestamp, worker_level); } diff --git a/src/share/rc/ob_tenant_base.cpp b/src/share/rc/ob_tenant_base.cpp index e4afaf885..b28375bdf 100644 --- a/src/share/rc/ob_tenant_base.cpp +++ b/src/share/rc/ob_tenant_base.cpp @@ -298,7 +298,11 @@ int ObTenantBase::pre_run() { int ret = OB_SUCCESS; ObTenantEnv::set_tenant(this); - ret = lib::SET_GROUP_ID(OBCG_DEFAULT); + // register in tenant cgroup without modifying group_id + ObCgroupCtrl *cgroup_ctrl = get_cgroup(); + if (OB_NOT_NULL(cgroup_ctrl)) { + ret = cgroup_ctrl->add_self_to_cgroup_(id_, GET_GROUP_ID()); + } { ThreadListNode *node = lib::Thread::current().get_thread_list_node(); lib::ObMutexGuard guard(thread_list_lock_); @@ -316,7 +320,6 @@ int ObTenantBase::end_run() { int ret = OB_SUCCESS; ObTenantEnv::set_tenant(nullptr); - ObCgroupCtrl *cgroup_ctrl = get_cgroup(); { ThreadListNode *node = lib::Thread::current().get_thread_list_node(); lib::ObMutexGuard guard(thread_list_lock_); diff --git a/src/share/resource_manager/ob_cgroup_ctrl.cpp b/src/share/resource_manager/ob_cgroup_ctrl.cpp index 9582d2355..f44adcdaf 100644 --- a/src/share/resource_manager/ob_cgroup_ctrl.cpp +++ b/src/share/resource_manager/ob_cgroup_ctrl.cpp @@ -303,8 +303,8 @@ int ObCgroupCtrl::get_group_path( char meta_tenant_path[PATH_BUFSIZE] = ""; char group_name_path[PATH_BUFSIZE] = ""; // gen root_cgroup_path - if (is_server_tenant(tenant_id)) { - // if tenant_id is 500, return "other" + if (!is_valid_tenant_id(tenant_id) || is_server_tenant(tenant_id)) { + // if tenant_id is invalid or 500, return "other" snprintf(root_cgroup_path, path_bufsize, "%s", other_cgroup_); } else { snprintf(root_cgroup_path, path_bufsize, "%s", root_cgroup_); diff --git a/src/share/resource_manager/ob_cgroup_ctrl.h b/src/share/resource_manager/ob_cgroup_ctrl.h index 4c86ee910..df05b65a8 100644 --- a/src/share/resource_manager/ob_cgroup_ctrl.h +++ b/src/share/resource_manager/ob_cgroup_ctrl.h @@ -25,6 +25,7 @@ class ObString; namespace share { class ObGroupName; +class ObTenantBase; typedef enum : uint64_t { DEFAULT = 0, @@ -152,8 +153,6 @@ public: int remove_both_cgroup(const uint64_t tenant_id, const uint64_t group_id = OB_INVALID_GROUP_ID, const char *base_path = ""); static int remove_dir_(const char *curr_dir); - int add_self_to_cgroup_(const uint64_t tenant_id, const uint64_t group_id = OBCG_DEFAULT); - static int get_cgroup_config_(const char *group_path, const char *config_name, char *config_value); static int set_cgroup_config_(const char *group_path, const char *config_name, char *config_value); // 设定指定租户cgroup组的cpu.shares @@ -211,6 +210,9 @@ private: int64_t last_usage_check_time_; private: + friend class oceanbase::share::ObTenantBase; + friend int oceanbase::lib::SET_GROUP_ID(uint64_t group_id); + int add_self_to_cgroup_(const uint64_t tenant_id, const uint64_t group_id = OBCG_DEFAULT); int init_cgroup_root_dir_(const char *cgroup_path); static int init_dir_(const char *curr_dir); static int init_full_dir_(const char *curr_path); diff --git a/src/sql/engine/px/ob_dfo.h b/src/sql/engine/px/ob_dfo.h index 1c9e0146b..ae66b3cf1 100644 --- a/src/sql/engine/px/ob_dfo.h +++ b/src/sql/engine/px/ob_dfo.h @@ -1216,8 +1216,7 @@ class ObPxWorkerEnvArgs public : typedef common::ObCurTraceId::TraceId TraceId; ObPxWorkerEnvArgs () : trace_id_(), log_level_(OB_LOG_LEVEL_NONE), - is_oracle_mode_(false), enqueue_timestamp_(-1), gctx_(nullptr), - group_id_(0) { } + is_oracle_mode_(false), enqueue_timestamp_(-1), gctx_(nullptr){ } virtual ~ObPxWorkerEnvArgs() { } @@ -1227,7 +1226,6 @@ public : is_oracle_mode_ = other.is_oracle_mode_; enqueue_timestamp_ = other.enqueue_timestamp_; gctx_ = other.gctx_; - group_id_ = other.group_id_; return *this; } @@ -1241,8 +1239,6 @@ public : int64_t get_enqueue_timestamp() const { return enqueue_timestamp_; } void set_gctx(const observer::ObGlobalContext *ctx) { gctx_ = ctx; } const observer::ObGlobalContext *get_gctx() { return gctx_; } - void set_group_id(int32_t v) { group_id_ = v; } - int32_t get_group_id() const { return group_id_; } private: TraceId trace_id_; @@ -1250,7 +1246,6 @@ private: bool is_oracle_mode_; int64_t enqueue_timestamp_; const observer::ObGlobalContext *gctx_; - int32_t group_id_; }; } diff --git a/src/sql/engine/px/ob_px_worker.cpp b/src/sql/engine/px/ob_px_worker.cpp index 00e6050ea..af41fba9b 100644 --- a/src/sql/engine/px/ob_px_worker.cpp +++ b/src/sql/engine/px/ob_px_worker.cpp @@ -188,7 +188,6 @@ void PxWorkerFunctor::operator ()(bool need_exec) ObThreadLogLevelUtils::init(env_arg_.get_log_level()); } } - SET_GROUP_ID(env_arg_.get_group_id()); // When deserialize expr, sql mode will affect basic function of expr. CompatModeGuard mode_guard(env_arg_.is_oracle_mode() ? Worker::CompatMode::ORACLE : Worker::CompatMode::MYSQL); MTL_SWITCH(sqc_handler->get_tenant_id()) { @@ -298,7 +297,6 @@ int ObPxThreadWorker::run_at(ObPxRpcInitTaskArgs &task_arg, omt::ObPxPool &px_po env_args.set_trace_id(*ObCurTraceId::get_trace_id()); env_args.set_is_oracle_mode(lib::is_oracle_mode()); env_args.set_gctx(&gctx_); - env_args.set_group_id(THIS_WORKER.get_group_id()); if (OB_LOG_LEVEL_NONE != common::ObThreadLogLevelUtils::get_level()) { env_args.set_log_level(common::ObThreadLogLevelUtils::get_level()); } From d67262a45eeed65c8b0b3b747e9cb2ad58f5c055 Mon Sep 17 00:00:00 2001 From: Handora Date: Fri, 23 Aug 2024 10:22:23 +0000 Subject: [PATCH 191/249] [BUG] fix multi version garbage collector for tx_desc --- .../concurrency_control/ob_multi_version_garbage_collector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/concurrency_control/ob_multi_version_garbage_collector.cpp b/src/storage/concurrency_control/ob_multi_version_garbage_collector.cpp index 6fb5e7b54..385b33968 100644 --- a/src/storage/concurrency_control/ob_multi_version_garbage_collector.cpp +++ b/src/storage/concurrency_control/ob_multi_version_garbage_collector.cpp @@ -1355,7 +1355,7 @@ bool GetMinActiveSnapshotVersionFunctor::operator()(sql::ObSQLSessionMgr::Key ke sql::ObSQLSessionInfo::LockGuard data_lock_guard(sess_info->get_thread_data_lock()); share::SCN snapshot_version(share::SCN::max_scn()); - if (sess_info->is_in_transaction()) { + if (OB_NOT_NULL(sess_info->get_tx_desc())) { share::SCN desc_snapshot; transaction::ObTxDesc *tx_desc = nullptr; share::SCN sess_snapshot = sess_info->get_reserved_snapshot_version(); From 0ba68e5dc9092512b1f54f4ffc5761399e282ae2 Mon Sep 17 00:00:00 2001 From: haitaoyang Date: Fri, 23 Aug 2024 10:28:27 +0000 Subject: [PATCH 192/249] Fix padding for new added col --- src/storage/column_store/ob_virtual_cg_scanner.cpp | 3 ++- .../test_suite/column_store/r/mysql/add_column.result | 3 +++ .../mysql_test/test_suite/column_store/t/add_column.test | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/storage/column_store/ob_virtual_cg_scanner.cpp b/src/storage/column_store/ob_virtual_cg_scanner.cpp index b00c9b751..a3a88c56f 100644 --- a/src/storage/column_store/ob_virtual_cg_scanner.cpp +++ b/src/storage/column_store/ob_virtual_cg_scanner.cpp @@ -314,7 +314,8 @@ int ObDefaultCGScanner::init_datum_infos_and_default_row(const ObTableIterParam STORAGE_LOG(WARN, "unexpected null column_param", K(ret), K(iter_param)); } else if (OB_FAIL(default_row_.storage_datums_[0].from_obj_enhance(column_param->get_orig_default_value()))) { STORAGE_LOG(WARN, "Failed to transefer obj to datum", K(ret)); - } else if (OB_FAIL(pad_column(column_param->get_meta_type(), column_param->get_accuracy(), *access_ctx.stmt_allocator_, default_row_.storage_datums_[0]))) { + } else if (is_pad_char_to_full_length(access_ctx.sql_mode_) && + OB_FAIL(pad_column(column_param->get_meta_type(), column_param->get_accuracy(), *access_ctx.stmt_allocator_, default_row_.storage_datums_[0]))) { LOG_WARN("Failed to pad default column", K(ret), KPC(column_param), K_(default_row)); } else if (OB_FAIL(add_lob_header_if_need(*column_param, default_row_.local_allocator_, default_row_.storage_datums_[0]))) { STORAGE_LOG(WARN, "Failed to add lob header to default value", K(ret)); diff --git a/tools/deploy/mysql_test/test_suite/column_store/r/mysql/add_column.result b/tools/deploy/mysql_test/test_suite/column_store/r/mysql/add_column.result index 19223c805..77ed03c8c 100644 --- a/tools/deploy/mysql_test/test_suite/column_store/r/mysql/add_column.result +++ b/tools/deploy/mysql_test/test_suite/column_store/r/mysql/add_column.result @@ -189,6 +189,9 @@ alter system flush plan cache; select count(c2), min(c2), max(c2) from t3 where c1 < 256; count(c2) min(c2) max(c2) 16384 中文 中文 +select count(c2), min(c2), max(c2) from t3 where c1 < 256; +count(c2) min(c2) max(c2) +16384 中文 中文 alter system major freeze; alter table t2 add column c4 bigint default 10, add column c5 bigint unsigned default 20; alter table t2 add column c6 bigint default null; diff --git a/tools/deploy/mysql_test/test_suite/column_store/t/add_column.test b/tools/deploy/mysql_test/test_suite/column_store/t/add_column.test index 6d34bc3a9..a3a719073 100644 --- a/tools/deploy/mysql_test/test_suite/column_store/t/add_column.test +++ b/tools/deploy/mysql_test/test_suite/column_store/t/add_column.test @@ -149,6 +149,8 @@ alter system flush plan cache; sleep 1; --enable_query_log +select count(c2), min(c2), max(c2) from t3 where c1 < 256; + alter system major freeze; --source mysql_test/include/wait_daily_merge.inc From 36707d153e07e3c3ea19c7a25f4c2996d1134e50 Mon Sep 17 00:00:00 2001 From: bit-dance <2634358021@qq.com> Date: Fri, 23 Aug 2024 10:34:53 +0000 Subject: [PATCH 193/249] [CP] [to #2024081500104151462]Fix:pl cache no hit when different db have same name table --- src/pl/pl_cache/ob_pl_cache.cpp | 26 -------------------------- src/pl/pl_cache/ob_pl_cache_object.cpp | 2 +- 2 files changed, 1 insertion(+), 27 deletions(-) diff --git a/src/pl/pl_cache/ob_pl_cache.cpp b/src/pl/pl_cache/ob_pl_cache.cpp index d2b246726..6a083d1a5 100644 --- a/src/pl/pl_cache/ob_pl_cache.cpp +++ b/src/pl/pl_cache/ob_pl_cache.cpp @@ -656,36 +656,10 @@ int ObPLObjectValue::get_all_dep_schema(ObPLCacheCtx &pc_ctx, } } } else if (lib::is_oracle_mode()) { - if (pcv_schema->is_explicit_db_name_) { - //In oracle mode, if mark database name，use table id search schema directly. if (OB_FAIL(schema_guard.get_simple_table_schema(tenant_id, pcv_schema->schema_id_, table_schema))) { LOG_WARN("failed to get table schema", K(pcv_schema->schema_id_), K(ret)); } else { /* do nothing */ } - } else if (OB_FAIL(schema_guard.get_simple_table_schema(tenant_id, - database_id, - pcv_schema->table_name_, - false, - table_schema))) { - LOG_WARN("failed to get table schema", K(pcv_schema->schema_id_), K(ret)); - } else if (nullptr == table_schema && OB_FAIL(schema_guard.get_simple_table_schema(tenant_id, - pcv_schema->database_id_, - pcv_schema->table_name_, - false, - table_schema))) { - LOG_WARN("failed to get table schema", - K(ret), K(pcv_schema->tenant_id_), K(pcv_schema->database_id_), - K(pcv_schema->table_name_)); - } else if (nullptr == table_schema && OB_FAIL(schema_guard.get_simple_table_schema(tenant_id, - common::OB_ORA_SYS_DATABASE_ID, - pcv_schema->table_name_, - false, - table_schema))) { // finaly,find sys tenand - LOG_WARN("failed to get table schema", K(ret), K(tenant_id), - K(pcv_schema->table_name_)); - } else { - // do nothing - } } else if (OB_FAIL(schema_guard.get_simple_table_schema(tenant_id, pcv_schema->database_id_, pcv_schema->table_name_, diff --git a/src/pl/pl_cache/ob_pl_cache_object.cpp b/src/pl/pl_cache/ob_pl_cache_object.cpp index 0788d6683..68a299907 100644 --- a/src/pl/pl_cache/ob_pl_cache_object.cpp +++ b/src/pl/pl_cache/ob_pl_cache_object.cpp @@ -169,7 +169,7 @@ int ObPLCacheObject::update_cache_obj_stat(sql::ObILibCacheCtx &ctx) int ObPLCacheObject::update_execute_time(int64_t exec_time) { int ret = OB_SUCCESS; - ATOMIC_STORE(&(stat_.elapsed_time_), exec_time); + ATOMIC_AAF(&(stat_.elapsed_time_),exec_time); ATOMIC_INC(&(stat_.execute_times_)); int64_t slowest_usec = ATOMIC_LOAD(&stat_.slowest_exec_usec_); if (slowest_usec < exec_time) { From 8f7ce6d472f2bfce16f423f88f7a6a277decef1c Mon Sep 17 00:00:00 2001 From: Fengjingkun Date: Fri, 23 Aug 2024 10:41:09 +0000 Subject: [PATCH 194/249] fix cs replica checksum validation --- .../ob_major_merge_progress_checker.cpp | 1 + .../ob_compaction_locality_cache.cpp | 196 ++++++++++++------ .../compaction/ob_compaction_locality_cache.h | 13 +- .../compaction/ob_medium_compaction_func.cpp | 20 +- .../high_availability/ob_storage_ha_utils.cpp | 12 +- 5 files changed, 167 insertions(+), 75 deletions(-) diff --git a/src/rootserver/freeze/ob_major_merge_progress_checker.cpp b/src/rootserver/freeze/ob_major_merge_progress_checker.cpp index 46912199c..5658769cf 100644 --- a/src/rootserver/freeze/ob_major_merge_progress_checker.cpp +++ b/src/rootserver/freeze/ob_major_merge_progress_checker.cpp @@ -913,6 +913,7 @@ int ObMajorMergeProgressChecker::generate_tablet_status_map() if (OB_HASH_NOT_EXIST == ret) { ret = OB_SUCCESS; LOG_TRACE("can't find ls_info from ls_locality_cache", KR(ret), K(ls_id), K_(tenant_id)); + continue; } else { LOG_WARN("fail to get ls_info from ls_locality_cache", KR(ret), K(ls_id), K_(tenant_id)); } diff --git a/src/share/compaction/ob_compaction_locality_cache.cpp b/src/share/compaction/ob_compaction_locality_cache.cpp index a98370f7b..27a24a061 100644 --- a/src/share/compaction/ob_compaction_locality_cache.cpp +++ b/src/share/compaction/ob_compaction_locality_cache.cpp @@ -82,7 +82,8 @@ bool ObLSReplicaUniItem::operator != (const ObLSReplicaUniItem &other) const ObLSColumnReplicaCache::ObLSColumnReplicaCache() : is_inited_(false), ls_id_set_(), - ls_replica_set_() + ls_replica_set_(), + ls_infos_() {} ObLSColumnReplicaCache::~ObLSColumnReplicaCache() @@ -110,16 +111,13 @@ int ObLSColumnReplicaCache::init() void ObLSColumnReplicaCache::destroy() { int ret = OB_SUCCESS; // only for log - if (ls_replica_set_.created()) { - if (OB_FAIL(ls_replica_set_.destroy())) { - LOG_WARN("fail to destroy ls replica set", K(ret)); - } + if (ls_replica_set_.created() && OB_FAIL(ls_replica_set_.destroy())) { + LOG_WARN("fail to destroy ls replica set", K(ret)); } - if (ls_id_set_.created()) { - if (OB_FAIL(ls_id_set_.destroy())) { - LOG_WARN("fail to destroy ls replica set", K(ret)); - } + if (ls_id_set_.created() && OB_FAIL(ls_id_set_.destroy())) { + LOG_WARN("fail to destroy ls replica set", K(ret)); } + ls_infos_.reset(); is_inited_ = false; } @@ -127,6 +125,7 @@ void ObLSColumnReplicaCache::reuse() { ls_id_set_.reuse(); ls_replica_set_.reuse(); + ls_infos_.reuse(); } int ObLSColumnReplicaCache::check_contains_ls(const ObLSID &ls_id, bool &contained) const @@ -175,49 +174,111 @@ int ObLSColumnReplicaCache::add_cs_replica(const ObLSReplicaUniItem &ls_item) return ret; } -int ObLSColumnReplicaCache::update(const ObLSID &ls_id, const ObAddr &server) +int ObLSColumnReplicaCache::update(const ObLSID &ls_id) { int ret = OB_SUCCESS; + const int64_t cluster_id = GCONF.cluster_id; bool is_contained = false; + ObLSInfo ls_info; + if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("not inited", K(ret)); } else if (OB_FAIL(check_contains_ls(ls_id, is_contained))) { LOG_WARN("fail to check exist for ls", K(ls_id), KPC(this)); } else if (is_contained) { + // do nothing } else if (OB_ISNULL(GCTX.lst_operator_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("lst operator is null", K(ret)); - } else { - const int64_t tenant_id = MTL_ID(); - const int64_t cluster_id = GCONF.cluster_id; - ObLSInfo ls_info; - if (OB_FAIL(GCTX.lst_operator_->get(cluster_id, tenant_id, ls_id, ObLSTable::DEFAULT_MODE, ls_info))) { - LOG_WARN("fail to get ls info", K(ret), K(cluster_id), K(tenant_id), K(ls_id)); + } else if (OB_FAIL(GCTX.lst_operator_->get(cluster_id, MTL_ID(), ls_id, ObLSTable::DEFAULT_MODE, ls_info))) { + LOG_WARN("fail to get ls info", K(ret), K(ls_id), K(cluster_id), K(MTL_ID())); + } else if (OB_FAIL(update_with_ls_info(ls_info))) { + LOG_WARN("failed to try add cs replica info", K(ret), K(ls_id), K(ls_info)); + } + return ret; +} + +int ObLSColumnReplicaCache::update_with_ls_info(const ObLSInfo &ls_info) +{ + int ret = OB_SUCCESS; + const ObLSID &ls_id = ls_info.get_ls_id(); + bool replica_has_leader = false; + + for (int64_t idx = 0; OB_SUCC(ret) && idx < ls_info.get_replicas().count(); ++idx) { + const ObLSReplica &replica = ls_info.get_replicas().at(idx); + if (ObRole::LEADER == replica.get_role()) { + replica_has_leader = true; + } + + if (!replica.is_column_replica()) { + // do nothing + } else if (OB_FAIL(add_cs_replica(ObLSReplicaUniItem(ls_id, replica.get_server())))) { + LOG_WARN("failed to add cs replica", K(ret), K(ls_id), K(replica), K(ls_info)); } else { - const ObLSInfo::ReplicaArray &all_replicas = ls_info.get_replicas(); - ObLSReplicaUniItem ls_item(ls_id, server); - for (int64_t i = 0; i < all_replicas.count() && OB_SUCC(ret); ++i) { - const ObLSReplica &replica = all_replicas.at(i); - if (ObRole::LEADER == replica.get_role()) { - const common::GlobalLearnerList &learner_list = replica.get_learner_list(); - for (int64_t i = 0; OB_SUCC(ret) && i < learner_list.get_member_number(); ++i) { - ObMember learner; - if (OB_FAIL(learner_list.get_learner(i, learner))) { - LOG_WARN("fail to get learner", K(ret), K(i), K(learner_list)); - } else if (learner.is_columnstore()) { - ls_item.server_ = learner.get_server(); - if (OB_FAIL(add_cs_replica(ls_item))) { - LOG_WARN("fail to add ls item", K(ret), K(ls_item)); - } - } + LOG_INFO("success to add cs replica", K(ls_id), K(replica), K(ls_info)); + } + } + + if (OB_FAIL(ret)) { + } else if (!replica_has_leader) { + ret = OB_LEADER_NOT_EXIST; + LOG_WARN("ls has no leader, ls column replica cache cannot read", K(ret), K(ls_id)); + } else if (OB_FAIL(mark_ls_finished(ls_id))) { + LOG_WARN("fail to make ls finished", K(ret)); + } else if (OB_FAIL(ls_infos_.push_back(ls_info))) { + LOG_WARN("failed to add ls info", K(ret), K(ls_info)); + } else { + LOG_INFO("success to update with ls info", KPC(this)); // debug log, remove later + } + return ret; +} + +int ObLSColumnReplicaCache::check_can_skip( + const ObLSReplicaUniItem &ls_item, + bool &can_skip) const +{ + int ret = OB_SUCCESS; + can_skip = false; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", K(ret), KPC(this)); + } else if (OB_UNLIKELY(!ls_item.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(ls_item)); + } else { + const ObLSInfo *ls_info = nullptr; + for (int64_t idx = 0; OB_SUCC(ret) && idx < ls_infos_.count(); ++idx) { + if (ls_item.ls_id_ == ls_infos_.at(idx).get_ls_id()) { + ls_info = &ls_infos_.at(idx); + break; + } + } + if (OB_ISNULL(ls_info)) { + ret = OB_ENTRY_NOT_EXIST; + LOG_WARN("ls info not found", K(ret), K(ls_item), K(ls_infos_)); + } else { + const ObLSInfo::ReplicaArray &replicas = ls_info->get_replicas(); + for (int64_t idx = 0; OB_SUCC(ret) && idx < replicas.count(); ++idx) { + const ObLSReplica &curr_replica = replicas.at(idx); + ObMember learner; + + if (ObRole::LEADER != curr_replica.get_role()) { + continue; // follower replica, do nothing + } else if (curr_replica.server_is_in_member_list(curr_replica.get_member_list(), ls_item.server_)) { + // item is in member list, need to check + } else if (OB_FAIL(curr_replica.get_learner_list().get_learner_by_addr(ls_item.server_, learner))) { + if (OB_ENTRY_NOT_EXIST == ret) { + ret = OB_SUCCESS; + can_skip = true; // item not in member list && not in learner list, skip to check + } else { + LOG_WARN("faile to get learner", K(ret), K(ls_item), KPC(this)); } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(mark_ls_finished(ls_item.ls_id_))) { - LOG_WARN("fail to make ls finished", K(ret)); - } - LOG_TRACE("[CS-Replica] get learner list", K(ret), K(learner_list)); + } else { + // both R replica and CS replica need to check } + break; // found leader replica, stop checking another replica } } } @@ -228,9 +289,16 @@ int ObLSColumnReplicaCache::check_is_cs_replica(const ObLSReplicaUniItem &ls_ite { int ret = OB_SUCCESS; is_cs_replica = false; + bool is_contained = false; + if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("not inited", K(ret)); + } else if (OB_FAIL(check_contains_ls(ls_item.ls_id_, is_contained))) { + LOG_WARN("fail to check exist for ls", K(ls_item), KPC(this)); + } else if (OB_UNLIKELY(!is_contained)) { + ret = OB_LEADER_NOT_EXIST; + LOG_WARN("ls has no leader, ls column replica cache cannot read", K(ret)); } else if (OB_FAIL(ls_replica_set_.exist_refactored(ls_item))) { if (OB_HASH_EXIST == ret || OB_HASH_NOT_EXIST == ret) { is_cs_replica = (OB_HASH_EXIST == ret); @@ -317,28 +385,28 @@ int ObCompactionLocalityCache::inner_refresh_ls_locality() } } else if (OB_FAIL(get_zone_list_from_inner_table(zone_list))) { LOG_WARN("failed to get zone list", K(ret), K_(tenant_id)); - } else if (zone_list.empty()) { - LOG_INFO("zone list is empty, skip get ls locality", K(ret), K_(tenant_id)); - (void) MTL(compaction::ObDiagnoseTabletMgr *)->add_diagnose_tablet(UNKNOW_LS_ID, UNKNOW_TABLET_ID, - share::ObDiagnoseTabletType::TYPE_MEDIUM_MERGE); } - if (OB_SUCC(ret)) { + + if (OB_FAIL(ret)) { + } else if (OB_UNLIKELY(zone_list.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("zone list is empty, skip get ls locality", K(ret), K_(tenant_id)); + MTL(compaction::ObDiagnoseTabletMgr *)->add_diagnose_tablet(UNKNOW_LS_ID, UNKNOW_TABLET_ID, ObDiagnoseTabletType::TYPE_MEDIUM_MERGE); + } else { // 1. clear ls_infos cached in memory ls_infos_map_.reuse(); ls_cs_replica_cache_.reuse(); // 2. load ls_infos from __all_ls_meta_table ObArray ls_infos; ls_infos.set_attr(ObMemAttr(tenant_id_, "RefLSInfos")); - const bool inner_table_only = false; if (OB_ISNULL(GCTX.lst_operator_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("lst_operator is null", KR(ret), K_(tenant_id)); - } else if (OB_FAIL(GCTX.lst_operator_->get_by_tenant(tenant_id_, inner_table_only, ls_infos))) { + } else if (OB_FAIL(GCTX.lst_operator_->get_by_tenant(tenant_id_, false/*inner_table_only*/, ls_infos))) { LOG_WARN("fail to get ls infos", KR(ret), K_(tenant_id)); } else { // 3. update ls_infos cached in memory - const int64_t ls_infos_cnt = ls_infos.count(); - for (int64_t i = 0; (i < ls_infos_cnt) && OB_SUCC(ret); ++i) { + for (int64_t i = 0; OB_SUCC(ret) && i < ls_infos.count(); ++i) { const ObLSInfo &ls_info = ls_infos.at(i); if (OB_FAIL(refresh_by_zone(ls_info, zone_list))) { LOG_WARN("fail to refresh by zone", K(ret), K(ls_info), K_(tenant_id), K(zone_list)); @@ -346,8 +414,7 @@ int ObCompactionLocalityCache::inner_refresh_ls_locality() } } if (OB_SUCC(ret)) { - (void) MTL(compaction::ObDiagnoseTabletMgr *)->delete_diagnose_tablet(UNKNOW_LS_ID, UNKNOW_TABLET_ID, - share::ObDiagnoseTabletType::TYPE_MEDIUM_MERGE); + MTL(compaction::ObDiagnoseTabletMgr *)->delete_diagnose_tablet(UNKNOW_LS_ID, UNKNOW_TABLET_ID, ObDiagnoseTabletType::TYPE_MEDIUM_MERGE); } cost_ts = ObTimeUtility::fast_current_time() - cost_ts; LOG_INFO("finish to refresh ls locality cache", KR(ret), K_(tenant_id), K(cost_ts), K(zone_list)); @@ -385,8 +452,7 @@ struct ObMemberListInfo ObSEArray member_list_array_; // including member_list & learner_list }; -int ObMemberListInfo::build( - const ObLSReplica &tmp_replica) +int ObMemberListInfo::build(const ObLSReplica &tmp_replica) { int ret = OB_SUCCESS; const ObLSReplica::MemberList *member_list = &tmp_replica.get_member_list(); @@ -423,13 +489,11 @@ int ObCompactionLocalityCache::refresh_by_zone( const ObIArray &zone_list) { int ret = OB_SUCCESS; - if (!ls_info.is_valid()) { + if (OB_UNLIKELY(!ls_info.is_valid() || zone_list.empty())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid ls_info", K(ret), K(ls_info), K(zone_list)); } else { const ObLSID &ls_id = ls_info.get_ls_id(); - // create with tenant_id to set_attr for ObSArray in ObLSInfo - ObLSInfo tmp_ls_info(tenant_id_, ls_id); const ObLSInfo::ReplicaArray &all_replicas = ls_info.get_replicas(); ObMemberListInfo member_list_array(tenant_id_); // including member_list & learner_list for (int64_t i = 0; OB_SUCC(ret) && i < all_replicas.count(); ++i) { @@ -445,27 +509,27 @@ int ObCompactionLocalityCache::refresh_by_zone( ret = OB_ERR_UNEXPECTED; LOG_WARN("no leader in ls replica", KR(ret), K(all_replicas)); } + + ObLSInfo tmp_ls_info(tenant_id_, ls_id); for (int64_t i = 0; OB_SUCC(ret) && i < all_replicas.count(); ++i) { const ObLSReplica &tmp_replica = all_replicas.at(i); - if (replica_in_zone_list(tmp_replica, zone_list) // replica in zone list - && member_list_array.check_exist(tmp_replica.get_server())) { // replica in member list - if (tmp_ls_info.is_valid()) { - if (OB_FAIL(tmp_ls_info.add_replica(tmp_replica))) { - LOG_WARN("fail to add replica", KR(ret), K(tmp_replica)); - } - } else if (OB_FAIL(tmp_ls_info.init_by_replica(tmp_replica))) { - LOG_WARN("fail to init ls_info by replica", KR(ret), K(tmp_replica)); - } - if (OB_FAIL(ret)) { - } else if (tmp_replica.is_column_replica() && OB_FAIL(ls_cs_replica_cache_.add_cs_replica(ObLSReplicaUniItem(ls_id, tmp_replica.get_server())))) { - LOG_WARN("fail to add cs replica", K(ret), K(ls_id), K(tmp_replica), K_(ls_cs_replica_cache)); + if (!replica_in_zone_list(tmp_replica, zone_list) || + !member_list_array.check_exist(tmp_replica.get_server())) { + // do nothing + } else if (tmp_ls_info.is_valid()) { + if (OB_FAIL(tmp_ls_info.add_replica(tmp_replica))) { + LOG_WARN("fail to add replica", KR(ret), K(tmp_replica)); } + } else if (OB_FAIL(tmp_ls_info.init_by_replica(tmp_replica))) { + LOG_WARN("fail to init ls_info by replica", KR(ret), K(tmp_replica)); } } if (FAILEDx(ls_infos_map_.set_refactored(ls_id, tmp_ls_info, 1/*overwrite*/))) { LOG_WARN("fail to set refactored", KR(ret), K(ls_id), K(tmp_ls_info)); + } else if (OB_FAIL(ls_cs_replica_cache_.update_with_ls_info(tmp_ls_info))) { + LOG_WARN("failed to update with ls info", K(ret), K(tmp_ls_info)); } else { - FLOG_INFO("success to refresh cached ls_info", K(ret), K(tmp_ls_info), K(zone_list)); + FLOG_INFO("success to refresh cached ls_info", K(ret), K(tmp_ls_info), K(zone_list), K_(ls_cs_replica_cache)); } } return ret; diff --git a/src/share/compaction/ob_compaction_locality_cache.h b/src/share/compaction/ob_compaction_locality_cache.h index e0c290fee..363543df1 100644 --- a/src/share/compaction/ob_compaction_locality_cache.h +++ b/src/share/compaction/ob_compaction_locality_cache.h @@ -54,19 +54,23 @@ public: int init(); void destroy(); void reuse(); + int update(const ObLSID &ls_id); + int update_with_ls_info(const ObLSInfo &ls_info); + int check_is_cs_replica(const ObLSReplicaUniItem &ls_item, bool &is_cs_replica) const; + int check_can_skip(const ObLSReplicaUniItem &ls_item, bool &can_skip) const; + TO_STRING_KV(K_(is_inited), K_(ls_id_set), K_(ls_replica_set), K_(ls_infos)); +private: int check_contains_ls(const ObLSID &ls_id, bool &contained) const; int mark_ls_finished(const ObLSID &ls_id); int add_cs_replica(const ObLSReplicaUniItem &ls_item); - int update(const ObLSID &ls_id, const ObAddr &server); - int check_is_cs_replica(const ObLSReplicaUniItem &ls_item, bool &is_cs_replica) const; - TO_STRING_KV(K_(is_inited), K_(ls_id_set), K_(ls_replica_set)); private: const static int64_t BUCKET_NUM_OF_LS_ID_SET = 15; const static int64_t BUCKET_NUM_OF_LS_REPLICA_SET = 31; private: bool is_inited_; - hash::ObHashSet ls_id_set_; // pre-wamred ls id, unused + hash::ObHashSet ls_id_set_; // record looped ls id hash::ObHashSet ls_replica_set_; // cs-prelica ls + common::ObSEArray ls_infos_; // used for check member list and learner list }; class ObCompactionLocalityCache @@ -81,7 +85,6 @@ public: int get_ls_info(const share::ObLSID &ls_id, share::ObLSInfo &ls_info); const share::ObLSColumnReplicaCache& get_cs_replica_cache() const { return ls_cs_replica_cache_; } TO_STRING_KV(K_(is_inited), K_(tenant_id)); - private: const int64_t CHECK_LS_LOCALITY_INTERVAL = 5 * 60 * 1000 * 1000L; // 5 mins int get_zone_list_from_inner_table(ObIArray &zone_list); diff --git a/src/storage/compaction/ob_medium_compaction_func.cpp b/src/storage/compaction/ob_medium_compaction_func.cpp index e2a44486f..f17ebdb36 100644 --- a/src/storage/compaction/ob_medium_compaction_func.cpp +++ b/src/storage/compaction/ob_medium_compaction_func.cpp @@ -1247,7 +1247,15 @@ int ObMediumCompactionScheduleFunc::init_tablet_filters(share::ObTabletReplicaFi const ObTabletReplicaChecksumItem &curr_item = checksum_items.at(idx); bool is_cs_replica = false; ObLSReplicaUniItem ls_item(curr_item.ls_id_, curr_item.server_); - if (OB_FAIL(ls_cs_replica_cache.check_is_cs_replica(ls_item, is_cs_replica))) { + const ObLSReplica *replica = nullptr; + bool can_skip = false; + + if (OB_FAIL(ls_cs_replica_cache.check_can_skip(ls_item, can_skip))) { + LOG_WARN("failed to check item can skip", K(ret), K(ls_item), K(ls_cs_replica_cache)); + } else if (can_skip) { + LOG_INFO("curr ls item should skip check", K(ls_item), K(ls_cs_replica_cache)); + continue; + } else if (OB_FAIL(ls_cs_replica_cache.check_is_cs_replica(ls_item, is_cs_replica))) { LOG_WARN("fail to check is column replica", K(ret), K(ls_item), K(ls_cs_replica_cache)); } else if (OB_ISNULL(prev_item)) { } else if (!curr_item.is_same_tablet(*prev_item)) { @@ -1256,8 +1264,16 @@ int ObMediumCompactionScheduleFunc::init_tablet_filters(share::ObTabletReplicaFi } else if (OB_TMP_FAIL(data_checksum_checker.check_data_checksum(curr_item, is_cs_replica)) || OB_TMP_FAIL(curr_item.verify_column_checksum(*prev_item))) { if (OB_CHECKSUM_ERROR == tmp_ret) { + ObLSColumnReplicaCache dump_cache; + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(dump_cache.init())) { + LOG_WARN_RET(tmp_ret, "failed to init dump cache", K(MTL_ID())); + } else if (OB_TMP_FAIL(dump_cache.update(curr_item.ls_id_))) { + LOG_WARN_RET(tmp_ret, "failed to force refresh ls locality", K(curr_item)); + } + LOG_DBA_ERROR(OB_CHECKSUM_ERROR, "msg", "checksum error in tablet replica checksum", KR(tmp_ret), - K(curr_item), KPC(prev_item), K(is_cs_replica), K(data_checksum_checker)); + K(curr_item), KPC(prev_item), K(ls_cs_replica_cache), K(is_cs_replica), K(dump_cache), K(data_checksum_checker)); check_ret = OB_CHECKSUM_ERROR; if (curr_item.ls_id_ != prev_error_ls_id) { prev_error_ls_id = curr_item.ls_id_; diff --git a/src/storage/high_availability/ob_storage_ha_utils.cpp b/src/storage/high_availability/ob_storage_ha_utils.cpp index f16ae3408..c298fa38c 100644 --- a/src/storage/high_availability/ob_storage_ha_utils.cpp +++ b/src/storage/high_availability/ob_storage_ha_utils.cpp @@ -227,7 +227,7 @@ int ObStorageHAUtils::check_tablet_replica_checksum_(const uint64_t tenant_id, c for (int64_t i = 0; OB_SUCC(ret) && i < filter_items.count(); ++i) { const ObTabletReplicaChecksumItem &item = filter_items.at(i); - if (OB_FAIL(ls_cs_replica_cache.update(item.ls_id_, item.server_))) { + if (OB_FAIL(ls_cs_replica_cache.update(item.ls_id_))) { LOG_WARN("fail to update ls replica status", K(ret), K(item)); } } @@ -237,7 +237,15 @@ int ObStorageHAUtils::check_tablet_replica_checksum_(const uint64_t tenant_id, c const ObTabletReplicaChecksumItem &item = filter_items.at(i); const ObLSReplicaUniItem ls_item(item.ls_id_, item.server_); bool is_cs_replica = false; - if (OB_FAIL(ls_cs_replica_cache.check_is_cs_replica(ls_item, is_cs_replica))) { + bool can_skip = false; + const ObLSReplica *replica = nullptr; + + if (OB_FAIL(ls_cs_replica_cache.check_can_skip(ls_item, can_skip))) { + LOG_WARN("failed to get ls replica", K(ret), K(ls_item), K(ls_cs_replica_cache)); + } else if (can_skip) { + LOG_INFO("cur ls item can be skip", K(ret), K(ls_item), K(ls_cs_replica_cache)); + continue; + } else if (OB_FAIL(ls_cs_replica_cache.check_is_cs_replica(ls_item, is_cs_replica))) { LOG_WARN("fail to check is cs replica", K(ret), K(ls_item), K(ls_cs_replica_cache)); } else if (OB_FAIL(data_checksum_checker.check_data_checksum(item, is_cs_replica))) { LOG_ERROR("failed to verify data checksum", K(ret), K(tenant_id), K(tablet_id), From 058c8edbae8b34b26e183f66e466e31af946bc2d Mon Sep 17 00:00:00 2001 From: suz-yang Date: Fri, 23 Aug 2024 10:47:18 +0000 Subject: [PATCH 195/249] fix direct load coordinator write lock contention --- .../table_load/ob_table_load_coordinator.cpp | 12 +- .../table_load/ob_table_load_instance.cpp | 4 + .../table_load/ob_table_load_schema.cpp | 106 +++++++----------- .../table_load/ob_table_load_schema.h | 13 +-- .../engine/cmd/ob_load_data_direct_impl.cpp | 89 +++++++-------- src/sql/engine/cmd/ob_load_data_direct_impl.h | 6 +- .../engine/cmd/ob_table_direct_insert_ctx.cpp | 6 +- 7 files changed, 103 insertions(+), 133 deletions(-) diff --git a/src/observer/table_load/ob_table_load_coordinator.cpp b/src/observer/table_load/ob_table_load_coordinator.cpp index b1b16f090..26564f7ad 100644 --- a/src/observer/table_load/ob_table_load_coordinator.cpp +++ b/src/observer/table_load/ob_table_load_coordinator.cpp @@ -1753,12 +1753,12 @@ int ObTableLoadCoordinator::write(const ObTableLoadTransId &trans_id, int32_t se // 取出bucket_writer else if (OB_FAIL(trans->get_bucket_writer_for_write(bucket_writer))) { LOG_WARN("fail to get bucket writer", KR(ret)); - } else if (OB_FAIL(bucket_writer->advance_sequence_no(session_id, sequence_no, guard))) { - if (OB_UNLIKELY(OB_ENTRY_EXIST != ret)) { - LOG_WARN("fail to advance sequence no", KR(ret), K(session_id)); - } else { - ret = OB_SUCCESS; - } + // } else if (OB_FAIL(bucket_writer->advance_sequence_no(session_id, sequence_no, guard))) { + // if (OB_UNLIKELY(OB_ENTRY_EXIST != ret)) { + // LOG_WARN("fail to advance sequence no", KR(ret), K(session_id)); + // } else { + // ret = OB_SUCCESS; + // } } else { ObTableLoadTask *task = nullptr; WriteTaskProcessor *processor = nullptr; diff --git a/src/observer/table_load/ob_table_load_instance.cpp b/src/observer/table_load/ob_table_load_instance.cpp index f451cacbf..506c9b0fa 100644 --- a/src/observer/table_load/ob_table_load_instance.cpp +++ b/src/observer/table_load/ob_table_load_instance.cpp @@ -440,6 +440,10 @@ int ObTableLoadInstance::start_redef_table( LOG_WARN("failed to assign tablet ids", KR(ret), K(param.load_level_), K(tablet_ids)); } else if (OB_FAIL(ObTableLoadRedefTable::start(start_arg, start_res, *stmt_ctx_.session_info_))) { LOG_WARN("fail to start redef table", KR(ret), K(start_arg)); + // rewrite error code for concurrency of direct load and offline ddl + if (OB_TABLE_NOT_EXIST == ret) { + ret = OB_SCHEMA_NOT_UPTODATE; + } } else { ddl_param.dest_table_id_ = start_res.dest_table_id_; ddl_param.task_id_ = start_res.task_id_; diff --git a/src/observer/table_load/ob_table_load_schema.cpp b/src/observer/table_load/ob_table_load_schema.cpp index 21b3bb57a..77c040d93 100644 --- a/src/observer/table_load/ob_table_load_schema.cpp +++ b/src/observer/table_load/ob_table_load_schema.cpp @@ -240,22 +240,6 @@ int ObTableLoadSchema::get_user_column_id_and_names(const ObTableSchema *table_s return ret; } -int ObTableLoadSchema::get_user_column_count(ObSchemaGetterGuard &schema_guard, - uint64_t tenant_id, - uint64_t table_id, - int64_t &column_count) -{ - int ret = OB_SUCCESS; - column_count = 0; - ObArray column_schemas; - if (OB_FAIL(get_user_column_schemas(schema_guard, tenant_id, table_id, column_schemas))) { - LOG_WARN("fail to get user column schemas", KR(ret)); - } else { - column_count = column_schemas.count(); - } - return ret; -} - int ObTableLoadSchema::get_column_ids(const ObTableSchema *table_schema, ObIArray &column_ids, bool contain_hidden_pk_column) @@ -324,50 +308,6 @@ int ObTableLoadSchema::check_has_udt_column(const ObTableSchema *table_schema, b return ret; } -int ObTableLoadSchema::get_tenant_optimizer_gather_stats_on_load(const uint64_t tenant_id, - bool &value) -{ - int ret = OB_SUCCESS; - value = false; - ObSqlString sql; - SMART_VAR(ObMySQLProxy::MySQLResult, res) - { - sqlclient::ObMySQLResult *result = nullptr; - // TODO(suzhi.yt) 这里为啥是带zone纬度的? 如果查询结果中有多个zone的, 选哪个作为返回值呢? - if (OB_FAIL(sql.assign_fmt( - "SELECT value FROM %s WHERE tenant_id = %ld and (zone, name, schema_version) in (select " - "zone, name, max(schema_version) FROM %s group by zone, name) and name = '%s'", - OB_ALL_SYS_VARIABLE_HISTORY_TNAME, - ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id), - OB_ALL_SYS_VARIABLE_HISTORY_TNAME, OB_SV__OPTIMIZER_GATHER_STATS_ON_LOAD))) { - LOG_WARN("fail to append sql", KR(ret), K(tenant_id)); - } else if (OB_FAIL(GCTX.sql_proxy_->read(res, tenant_id, sql.ptr()))) { - LOG_WARN("fail to execute sql", KR(ret), K(sql), K(tenant_id)); - } else if (OB_ISNULL(result = res.get_result())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to get sql result", KR(ret), K(sql), K(tenant_id)); - } else { - while (OB_SUCC(ret)) { - if (OB_FAIL(result->next())) { - if (OB_ITER_END != ret) { - LOG_WARN("fail to get next row", KR(ret), K(tenant_id)); - } else { - ret = OB_SUCCESS; - break; - } - } else { - ObString data; - EXTRACT_VARCHAR_FIELD_MYSQL(*result, "value", data); - if (0 == strcmp(data.ptr(), "1")) { - value = true; - } - } - } - } - } - return ret; -} - int ObTableLoadSchema::check_has_invisible_column(const ObTableSchema *table_schema, bool &bret) { int ret = OB_SUCCESS; @@ -440,16 +380,46 @@ int ObTableLoadSchema::check_has_lob_column(const ObTableSchema *table_schema, b return ret; } -int ObTableLoadSchema::get_table_compressor_type(uint64_t tenant_id, uint64_t table_id, - ObCompressorType &compressor_type) +int ObTableLoadSchema::get_tenant_optimizer_gather_stats_on_load(const uint64_t tenant_id, + bool &value) { int ret = OB_SUCCESS; - ObSchemaGetterGuard schema_guard; - const share::schema::ObTableSchema *table_schema = nullptr; - if (OB_FAIL(get_table_schema(tenant_id, table_id, schema_guard, table_schema))) { - LOG_WARN("fail to get table schema", KR(ret), K(tenant_id), K(table_id)); - } else { - compressor_type = table_schema->get_compressor_type(); + value = false; + ObSqlString sql; + SMART_VAR(ObMySQLProxy::MySQLResult, res) + { + sqlclient::ObMySQLResult *result = nullptr; + // TODO(suzhi.yt) 这里为啥是带zone纬度的? 如果查询结果中有多个zone的, 选哪个作为返回值呢? + if (OB_FAIL(sql.assign_fmt( + "SELECT value FROM %s WHERE tenant_id = %ld and (zone, name, schema_version) in (select " + "zone, name, max(schema_version) FROM %s group by zone, name) and name = '%s'", + OB_ALL_SYS_VARIABLE_HISTORY_TNAME, + ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id), + OB_ALL_SYS_VARIABLE_HISTORY_TNAME, OB_SV__OPTIMIZER_GATHER_STATS_ON_LOAD))) { + LOG_WARN("fail to append sql", KR(ret), K(tenant_id)); + } else if (OB_FAIL(GCTX.sql_proxy_->read(res, tenant_id, sql.ptr()))) { + LOG_WARN("fail to execute sql", KR(ret), K(sql), K(tenant_id)); + } else if (OB_ISNULL(result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get sql result", KR(ret), K(sql), K(tenant_id)); + } else { + while (OB_SUCC(ret)) { + if (OB_FAIL(result->next())) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to get next row", KR(ret), K(tenant_id)); + } else { + ret = OB_SUCCESS; + break; + } + } else { + ObString data; + EXTRACT_VARCHAR_FIELD_MYSQL(*result, "value", data); + if (0 == strcmp(data.ptr(), "1")) { + value = true; + } + } + } + } } return ret; } diff --git a/src/observer/table_load/ob_table_load_schema.h b/src/observer/table_load/ob_table_load_schema.h index 6a0636079..fb9539bba 100644 --- a/src/observer/table_load/ob_table_load_schema.h +++ b/src/observer/table_load/ob_table_load_schema.h @@ -42,6 +42,7 @@ public: static int get_table_schema(share::schema::ObSchemaGetterGuard &schema_guard, uint64_t tenant_id, uint64_t table_id, const share::schema::ObTableSchema *&table_schema); + static int get_user_column_schemas(const share::schema::ObTableSchema *table_schema, ObIArray &column_schemas); static int get_user_column_schemas(share::schema::ObSchemaGetterGuard &schema_guard, @@ -59,10 +60,7 @@ public: static int get_user_column_id_and_names(const share::schema::ObTableSchema *table_schema, common::ObIArray &column_ids, common::ObIArray &column_names); - static int get_user_column_count(share::schema::ObSchemaGetterGuard &schema_guard, - uint64_t tenant_id, - uint64_t table_id, - int64_t &column_count); + static int get_column_ids(const share::schema::ObTableSchema *table_schema, common::ObIArray &column_ids, bool contain_hidden_pk_column = false); @@ -71,14 +69,15 @@ public: uint64_t table_id, common::ObIArray &column_ids, bool contain_hidden_pk_column = false); + static int check_has_udt_column(const share::schema::ObTableSchema *table_schema, bool &bret); - static int get_tenant_optimizer_gather_stats_on_load(const uint64_t tenant_id, bool &value); static int check_has_invisible_column(const share::schema::ObTableSchema *table_schema, bool &bret); static int check_has_unused_column(const share::schema::ObTableSchema *table_schema, bool &bret); static int check_has_roaringbitmap_column(const share::schema::ObTableSchema *table_schema, bool &bret); static int check_has_lob_column(const share::schema::ObTableSchema *table_schema, bool &bret); - static int get_table_compressor_type(uint64_t tenant_id, uint64_t table_id, - ObCompressorType &compressor_type); + + static int get_tenant_optimizer_gather_stats_on_load(const uint64_t tenant_id, bool &value); + public: ObTableLoadSchema(); ~ObTableLoadSchema(); diff --git a/src/sql/engine/cmd/ob_load_data_direct_impl.cpp b/src/sql/engine/cmd/ob_load_data_direct_impl.cpp index f17dcfac8..2340a48d2 100644 --- a/src/sql/engine/cmd/ob_load_data_direct_impl.cpp +++ b/src/sql/engine/cmd/ob_load_data_direct_impl.cpp @@ -69,7 +69,9 @@ ObLoadDataDirectImpl::LoadExecuteParam::LoadExecuteParam() ignore_row_num_(-1), dup_action_(ObLoadDupActionType::LOAD_INVALID_MODE), method_(ObDirectLoadMethod::INVALID_METHOD), - insert_mode_(ObDirectLoadInsertMode::INVALID_INSERT_MODE) + insert_mode_(ObDirectLoadInsertMode::INVALID_INSERT_MODE), + compressor_type_(ObCompressorType::INVALID_COMPRESSOR), + online_sample_percent_(100.) { column_ids_.set_tenant_id(MTL_ID()); } @@ -92,7 +94,8 @@ bool ObLoadDataDirectImpl::LoadExecuteParam::is_valid() const (storage::ObDirectLoadInsertMode::INC_REPLACE == insert_mode_ ? sql::ObLoadDupActionType::LOAD_REPLACE == dup_action_ : true) && - data_access_param_.is_valid() && !column_ids_.empty(); + data_access_param_.is_valid() && !column_ids_.empty() && + ObCompressorType::INVALID_COMPRESSOR != compressor_type_; } /** @@ -2326,6 +2329,8 @@ int ObLoadDataDirectImpl::init_execute_param() const ObLoadDataHint &hint = load_stmt_->get_hints(); const ObIArray &field_or_var_list = load_stmt_->get_field_or_var_list(); + ObSchemaGetterGuard *schema_guard = ctx_->get_sql_ctx()->schema_guard_; + const ObTableSchema *table_schema = nullptr; const bool is_backup = ObLoadDataFormat::OB_BACKUP_1_4 == load_args.access_info_.get_load_data_format(); execute_param_.tenant_id_ = load_args.tenant_id_; execute_param_.database_id_ = load_args.database_id_; @@ -2335,6 +2340,12 @@ int ObLoadDataDirectImpl::init_execute_param() execute_param_.combined_name_ = load_args.combined_name_; execute_param_.ignore_row_num_ = load_args.ignore_rows_; execute_param_.dup_action_ = load_args.dupl_action_; + if (OB_FAIL(ObTableLoadSchema::get_table_schema(*schema_guard, + execute_param_.tenant_id_, + execute_param_.table_id_, + table_schema))) { + LOG_WARN("fail to get table schema", KR(ret), K(execute_param_)); + } // parallel_ if (OB_SUCC(ret)) { ObTenant *tenant = nullptr; @@ -2423,33 +2434,19 @@ int ObLoadDataDirectImpl::init_execute_param() } // column_ids_ if (OB_SUCC(ret)) { - ObSchemaGetterGuard *schema_guard = ctx_->get_sql_ctx()->schema_guard_; - int64_t column_count = 0; execute_param_.column_ids_.reset(); if (is_backup) { // 备份数据导入 - if (OB_FAIL(ObTableLoadSchema::get_column_ids(*schema_guard, - execute_param_.tenant_id_, - execute_param_.table_id_, - execute_param_.column_ids_))) { + if (OB_FAIL(ObTableLoadSchema::get_column_ids(table_schema, execute_param_.column_ids_))) { LOG_WARN("fail to get column ids for backup", KR(ret)); } } else if (load_stmt_->get_default_table_columns()) { // 默认列导入 - if (OB_FAIL(ObTableLoadSchema::get_user_column_ids(*schema_guard, - execute_param_.tenant_id_, - execute_param_.table_id_, - execute_param_.column_ids_))) { + if (OB_FAIL(ObTableLoadSchema::get_user_column_ids(table_schema, execute_param_.column_ids_))) { LOG_WARN("fail to get user column ids", KR(ret)); } } else { // 指定列导入 const static uint64_t INVALID_COLUMN_ID = UINT64_MAX; - const ObTableSchema *table_schema = nullptr; ObArray user_column_ids; - if (OB_FAIL(ObTableLoadSchema::get_table_schema(*schema_guard, - execute_param_.tenant_id_, - execute_param_.table_id_, - table_schema))) { - LOG_WARN("fail to get table schema", KR(ret), K(execute_param_)); - } else if (OB_FAIL(ObTableLoadSchema::get_user_column_ids(table_schema, user_column_ids))) { + if (OB_FAIL(ObTableLoadSchema::get_user_column_ids(table_schema, user_column_ids))) { LOG_WARN("fail to get user column ids", KR(ret)); } for (int64_t i = 0; OB_SUCC(ret) && i < field_or_var_list.count(); ++i) { @@ -2479,6 +2476,23 @@ int ObLoadDataDirectImpl::init_execute_param() } } } + // compressor_type_ + if (OB_SUCC(ret)) { + if (OB_FAIL(ObDDLUtil::get_temp_store_compress_type( + table_schema->get_compressor_type(), execute_param_.parallel_, execute_param_.compressor_type_))) { + LOG_WARN("fail to get tmp store compressor type", KR(ret)); + } + } + // online_sample_percent_ + if (OB_SUCC(ret)) { + if (execute_param_.online_opt_stat_gather_ && + OB_FAIL(ObDbmsStatsUtils::get_sys_online_estimate_percent(*ctx_, + execute_param_.tenant_id_, + execute_param_.table_id_, + execute_param_.online_sample_percent_))) { + LOG_WARN("failed to get sys online sample percent", K(ret)); + } + } return ret; } @@ -2489,7 +2503,6 @@ int ObLoadDataDirectImpl::init_execute_context() const bool is_backup = ObLoadDataFormat::OB_BACKUP_1_4 == load_args.access_info_.get_load_data_format(); execute_ctx_.exec_ctx_.exec_ctx_ = ctx_; execute_ctx_.allocator_ = &ctx_->get_allocator(); - ObCompressorType table_compressor_type = ObCompressorType::NONE_COMPRESSOR; ObTableLoadParam load_param; load_param.tenant_id_ = execute_param_.tenant_id_; load_param.table_id_ = execute_param_.table_id_; @@ -2499,40 +2512,20 @@ int ObLoadDataDirectImpl::init_execute_context() load_param.max_error_row_count_ = execute_param_.max_error_rows_; load_param.column_count_ = execute_param_.column_ids_.count(); load_param.need_sort_ = is_backup ? false : execute_param_.need_sort_; - load_param.dup_action_ = execute_param_.dup_action_; load_param.px_mode_ = false; load_param.online_opt_stat_gather_ = execute_param_.online_opt_stat_gather_; + load_param.dup_action_ = execute_param_.dup_action_; load_param.method_ = execute_param_.method_; load_param.insert_mode_ = execute_param_.insert_mode_; load_param.load_mode_ = ObDirectLoadMode::LOAD_DATA; + load_param.compressor_type_ = execute_param_.compressor_type_; + load_param.online_sample_percent_ = execute_param_.online_sample_percent_; load_param.load_level_ = ObDirectLoadLevel::TABLE; - - double online_sample_percent = 100.; - if (OB_SUCC(ret)) { - if (execute_param_.online_opt_stat_gather_ && - OB_FAIL(ObDbmsStatsUtils::get_sys_online_estimate_percent(*ctx_, - execute_param_.tenant_id_, - execute_param_.table_id_, - online_sample_percent))) { - LOG_WARN("failed to get sys online sample percent", K(ret)); - } else { - load_param.online_sample_percent_ = online_sample_percent; - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(ObTableLoadSchema::get_table_compressor_type( - execute_param_.tenant_id_, execute_param_.table_id_, table_compressor_type))) { - LOG_WARN("fail to get table compressor type", KR(ret)); - } else if (OB_FAIL(ObDDLUtil::get_temp_store_compress_type( - table_compressor_type, execute_param_.parallel_, load_param.compressor_type_))) { - LOG_WARN("fail to get tmp store compressor type", KR(ret)); - } else if (OB_FAIL(direct_loader_.init(load_param, execute_param_.column_ids_, - &execute_ctx_.exec_ctx_))) { - LOG_WARN("fail to init direct loader", KR(ret)); - } else if (OB_FAIL(init_logger())) { - LOG_WARN("fail to init logger", KR(ret)); - } + if (OB_FAIL( + direct_loader_.init(load_param, execute_param_.column_ids_, &execute_ctx_.exec_ctx_))) { + LOG_WARN("fail to init direct loader", KR(ret)); + } else if (OB_FAIL(init_logger())) { + LOG_WARN("fail to init logger", KR(ret)); } if (OB_SUCC(ret)) { execute_ctx_.direct_loader_ = &direct_loader_; diff --git a/src/sql/engine/cmd/ob_load_data_direct_impl.h b/src/sql/engine/cmd/ob_load_data_direct_impl.h index 824769353..0c2d484f1 100644 --- a/src/sql/engine/cmd/ob_load_data_direct_impl.h +++ b/src/sql/engine/cmd/ob_load_data_direct_impl.h @@ -95,7 +95,9 @@ private: "method", storage::ObDirectLoadMethod::get_type_string(method_), "insert_mode", storage::ObDirectLoadInsertMode::get_type_string(insert_mode_), K_(data_access_param), - K_(column_ids)); + K_(column_ids), + K_(compressor_type), + K_(online_sample_percent)); public: uint64_t tenant_id_; uint64_t database_id_; @@ -116,6 +118,8 @@ private: storage::ObDirectLoadInsertMode::Type insert_mode_; DataAccessParam data_access_param_; ObArray column_ids_; + ObCompressorType compressor_type_; + double online_sample_percent_; }; struct LoadExecuteContext diff --git a/src/sql/engine/cmd/ob_table_direct_insert_ctx.cpp b/src/sql/engine/cmd/ob_table_direct_insert_ctx.cpp index 52900383f..8fad5983f 100644 --- a/src/sql/engine/cmd/ob_table_direct_insert_ctx.cpp +++ b/src/sql/engine/cmd/ob_table_direct_insert_ctx.cpp @@ -118,14 +118,14 @@ int ObTableDirectInsertCtx::init( tablet_ids.reset(); param.tenant_id_ = MTL_ID(); param.table_id_ = table_id; - param.batch_size_ = 100; param.parallel_ = parallel; param.session_count_ = parallel; + param.batch_size_ = 100; + param.max_error_row_count_ = 0; param.column_count_ = column_ids.count(); + param.need_sort_ = table_schema->is_heap_table() ? phy_plan.get_direct_load_need_sort() : true; param.px_mode_ = true; param.online_opt_stat_gather_ = is_online_gather_statistics_; - param.need_sort_ = table_schema->is_heap_table() ? phy_plan.get_direct_load_need_sort() : true; - param.max_error_row_count_ = 0; param.dup_action_ = (enable_inc_replace ? sql::ObLoadDupActionType::LOAD_REPLACE : sql::ObLoadDupActionType::LOAD_STOP_ON_DUP); param.method_ = method; From db0d5ef1b7bf8d4e0a2f16ca3557ddc1d128f7c1 Mon Sep 17 00:00:00 2001 From: suz-yang Date: Fri, 23 Aug 2024 10:54:10 +0000 Subject: [PATCH 196/249] [CP] fix direct load create tablet mgr twice --- .../ob_direct_load_insert_table_ctx.cpp | 100 ++++++++++++------ .../ob_direct_load_insert_table_ctx.h | 3 + 2 files changed, 70 insertions(+), 33 deletions(-) diff --git a/src/storage/direct_load/ob_direct_load_insert_table_ctx.cpp b/src/storage/direct_load/ob_direct_load_insert_table_ctx.cpp index 57cea772e..ca2a109c2 100644 --- a/src/storage/direct_load/ob_direct_load_insert_table_ctx.cpp +++ b/src/storage/direct_load/ob_direct_load_insert_table_ctx.cpp @@ -178,39 +178,72 @@ int ObDirectLoadInsertTabletContext::open() lib::ObMutexGuard guard(mutex_); if (OB_FAIL(open_err_)) { LOG_WARN("open has error", KR(ret), K(origin_tablet_id_), K(tablet_id_)); - } else if (!is_open_) { - ObTenantDirectLoadMgr *sstable_insert_mgr = MTL(ObTenantDirectLoadMgr *); - ObTabletDirectLoadInsertParam direct_load_param; - direct_load_param.is_replay_ = false; - direct_load_param.common_param_.direct_load_type_ = - param_->is_incremental_ ? ObDirectLoadType::DIRECT_LOAD_INCREMENTAL - : ObDirectLoadType::DIRECT_LOAD_LOAD_DATA; - direct_load_param.common_param_.data_format_version_ = param_->data_version_; - direct_load_param.common_param_.read_snapshot_ = param_->snapshot_version_; - direct_load_param.common_param_.ls_id_ = ls_id_; - direct_load_param.common_param_.tablet_id_ = tablet_id_; - direct_load_param.runtime_only_param_.exec_ctx_ = nullptr; - direct_load_param.runtime_only_param_.task_id_ = param_->ddl_task_id_; - direct_load_param.runtime_only_param_.table_id_ = param_->table_id_; - direct_load_param.runtime_only_param_.schema_version_ = param_->schema_version_; - direct_load_param.runtime_only_param_.task_cnt_ = 1; // default value. - direct_load_param.runtime_only_param_.parallel_ = param_->parallel_; - direct_load_param.runtime_only_param_.tx_desc_ = param_->trans_param_.tx_desc_; - direct_load_param.runtime_only_param_.trans_id_ = param_->trans_param_.tx_id_; - direct_load_param.runtime_only_param_.seq_no_ = param_->trans_param_.tx_seq_.cast_to_int(); - if (OB_FAIL(sstable_insert_mgr->create_tablet_direct_load( - context_id_, context_id_ /*execution_id*/, direct_load_param))) { - LOG_WARN("create tablet manager failed", KR(ret), K(direct_load_param)); - } else if (FALSE_IT(is_create_ = true)) { - } else if (OB_FAIL(sstable_insert_mgr->open_tablet_direct_load( - !param_->is_incremental_, ls_id_, tablet_id_, context_id_, start_scn_, handle_))) { - LOG_WARN("fail to open tablet direct load", KR(ret), K(tablet_id_)); - } else { - is_open_ = true; - } - if (OB_FAIL(ret) && !param_->is_incremental_) { - open_err_ = ret; // avoid open repeatedly when failed - } + } else if (is_open_) { + // do nothing + } else if (OB_FAIL(create_tablet_direct_load())) { + LOG_WARN("fail to create tablet direct load", KR(ret)); + } else if (OB_FAIL(open_tablet_direct_load())) { + LOG_WARN("fail to open tablet direct load", KR(ret)); + } + if (OB_FAIL(ret) && !param_->is_incremental_) { + open_err_ = ret; // avoid open repeatedly when failed + } + } + return ret; +} + +int ObDirectLoadInsertTabletContext::create_tablet_direct_load() +{ + int ret = OB_SUCCESS; + if (is_create_) { + // do nothing + } else { + ObTenantDirectLoadMgr *sstable_insert_mgr = MTL(ObTenantDirectLoadMgr *); + ObTabletDirectLoadInsertParam direct_load_param; + direct_load_param.is_replay_ = false; + direct_load_param.common_param_.direct_load_type_ = + param_->is_incremental_ ? ObDirectLoadType::DIRECT_LOAD_INCREMENTAL + : ObDirectLoadType::DIRECT_LOAD_LOAD_DATA; + direct_load_param.common_param_.data_format_version_ = param_->data_version_; + direct_load_param.common_param_.read_snapshot_ = param_->snapshot_version_; + direct_load_param.common_param_.ls_id_ = ls_id_; + direct_load_param.common_param_.tablet_id_ = tablet_id_; + direct_load_param.runtime_only_param_.exec_ctx_ = nullptr; + direct_load_param.runtime_only_param_.task_id_ = param_->ddl_task_id_; + direct_load_param.runtime_only_param_.table_id_ = param_->table_id_; + direct_load_param.runtime_only_param_.schema_version_ = param_->schema_version_; + direct_load_param.runtime_only_param_.task_cnt_ = 1; // default value. + direct_load_param.runtime_only_param_.parallel_ = param_->parallel_; + direct_load_param.runtime_only_param_.tx_desc_ = param_->trans_param_.tx_desc_; + direct_load_param.runtime_only_param_.trans_id_ = param_->trans_param_.tx_id_; + direct_load_param.runtime_only_param_.seq_no_ = param_->trans_param_.tx_seq_.cast_to_int(); + if (OB_FAIL(sstable_insert_mgr->create_tablet_direct_load(context_id_, + context_id_ /*execution_id*/, + direct_load_param))) { + LOG_WARN("create tablet manager failed", KR(ret), K(direct_load_param)); + } else { + is_create_ = true; + } + } + return ret; +} + +int ObDirectLoadInsertTabletContext::open_tablet_direct_load() +{ + int ret = OB_SUCCESS; + if (is_open_) { + // do nothing + } else { + ObTenantDirectLoadMgr *sstable_insert_mgr = MTL(ObTenantDirectLoadMgr *); + if (OB_FAIL(sstable_insert_mgr->open_tablet_direct_load(!param_->is_incremental_, + ls_id_, + tablet_id_, + context_id_, + start_scn_, + handle_))) { + LOG_WARN("fail to open tablet direct load", KR(ret), K(tablet_id_)); + } else { + is_open_ = true; } } return ret; @@ -233,6 +266,7 @@ int ObDirectLoadInsertTabletContext::close() LOG_WARN("fail to close tablet direct load", KR(ret), K(ls_id_), K(tablet_id_)); } else { is_open_ = false; + is_create_ = false; handle_.reset(); } } diff --git a/src/storage/direct_load/ob_direct_load_insert_table_ctx.h b/src/storage/direct_load/ob_direct_load_insert_table_ctx.h index d46573acb..af9c54f4e 100644 --- a/src/storage/direct_load/ob_direct_load_insert_table_ctx.h +++ b/src/storage/direct_load/ob_direct_load_insert_table_ctx.h @@ -189,6 +189,9 @@ public: K_(is_create), K_(is_cancel)); private: + int create_tablet_direct_load(); + int open_tablet_direct_load(); + int get_pk_interval(uint64_t count, share::ObTabletCacheInterval &pk_interval); int get_lob_pk_interval(uint64_t count, share::ObTabletCacheInterval &pk_interval); int refresh_pk_cache(const common::ObTabletID &tablet_id, share::ObTabletCacheInterval &pk_cache); From 1936fe670ff4adbc0c68adce7ad7a13bdeb848a6 Mon Sep 17 00:00:00 2001 From: obdev Date: Fri, 23 Aug 2024 11:39:00 +0000 Subject: [PATCH 197/249] parallel drop table rpc pcode placeholder --- deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h index e181d4891..7b82a5cb7 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h @@ -238,7 +238,7 @@ PCODE_DEF(OB_PARALLEL_CREATE_TABLE, 0x276) PCODE_DEF(OB_PARALLEL_SET_COMMENT, 0x277) PCODE_DEF(OB_PARALLEL_CREATE_INDEX, 0x278) PCODE_DEF(OB_PARALLEL_UPDATE_INDEX_STATUS, 0x279) - +PCODE_DEF(OB_PARALLEL_DROP_TABLE, 0x27A) // system admin command PCODE_DEF(OB_ADMIN_SWITCH_REPLICA_ROLE, 0x280) From afc4047103911cb554b1df22a0ac86cdfe056a9c Mon Sep 17 00:00:00 2001 From: AA-tuliwei-BB <1123152962@qq.com> Date: Fri, 23 Aug 2024 11:45:27 +0000 Subject: [PATCH 198/249] fix transform aggr subquery with window funtion bug --- .../rewrite/ob_transform_aggr_subquery.cpp | 33 +++++++++++++++++-- src/sql/rewrite/ob_transform_aggr_subquery.h | 2 ++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/sql/rewrite/ob_transform_aggr_subquery.cpp b/src/sql/rewrite/ob_transform_aggr_subquery.cpp index 3d07dc7b6..07fcc0277 100644 --- a/src/sql/rewrite/ob_transform_aggr_subquery.cpp +++ b/src/sql/rewrite/ob_transform_aggr_subquery.cpp @@ -358,12 +358,18 @@ int ObTransformAggrSubquery::gather_transform_params(ObDMLStmt &stmt, trans_param.ja_query_ref_ = static_cast(child_expr); ObSelectStmt *subquery = trans_param.ja_query_ref_->get_ref_stmt(); bool is_valid = false; + bool ja_query_ref_valid = false; bool hint_allowed = false; int64_t limit_value = 0; OPT_TRACE("try to pullup JA subquery", child_expr); - if (ObOptimizerUtil::find_item(no_rewrite_exprs_, child_expr)) { + // check ja query ref's exec param + if (OB_FAIL(check_ja_query_ref_param_validity(*trans_param.ja_query_ref_, ja_query_ref_valid))) { + LOG_WARN("failed to check ja query ref param validity", K(ret)); + } else if (!ja_query_ref_valid) { + OPT_TRACE("exec param expr of ja query ref is not valid, can not transform"); + } else if (ObOptimizerUtil::find_item(no_rewrite_exprs_, child_expr)) { LOG_TRACE("subquery in select expr and can use index"); - OPT_TRACE("subquery in select expr and can use index, no need transfrom"); + OPT_TRACE("subquery in select expr and can use index, no need transform"); } else if (OB_FAIL(check_hint_allowed_unnest(stmt, *subquery, ctx_->trans_list_loc_ + transform_params.count(), pullup_strategy, @@ -445,6 +451,29 @@ int ObTransformAggrSubquery::gather_transform_params(ObDMLStmt &stmt, return ret; } +int ObTransformAggrSubquery::check_ja_query_ref_param_validity(ObQueryRefRawExpr &ja_query_ref, bool &is_valid) { + int ret = OB_SUCCESS; + is_valid = true; + const ObIArray ¶ms = ja_query_ref.get_exec_params(); + for (int64_t i = 0; OB_SUCC(ret) && is_valid && i < params.count(); ++i) { + ObExecParamRawExpr *exec_param = params.at(i); + ObRawExpr *outer_expr = NULL; + if (OB_ISNULL(exec_param)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("exec param expr is null", K(ret)); + } else if (OB_ISNULL(outer_expr = exec_param->get_ref_expr())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("outer expr is null", K(ret)); + } else if (outer_expr->has_flag(CNT_WINDOW_FUNC) || + outer_expr->has_flag(CNT_ROWNUM) || + outer_expr->has_flag(CNT_RAND_FUNC) || + outer_expr->has_flag(CNT_STATE_FUNC)) { + is_valid = false; + } + } + return ret; +} + /** * @brief ObTransformAggrSubquery::check_aggr_first_validity * check the valiaidty of the subquery diff --git a/src/sql/rewrite/ob_transform_aggr_subquery.h b/src/sql/rewrite/ob_transform_aggr_subquery.h index dd0bfe4b1..48e6fb0a3 100644 --- a/src/sql/rewrite/ob_transform_aggr_subquery.h +++ b/src/sql/rewrite/ob_transform_aggr_subquery.h @@ -131,6 +131,8 @@ private: const bool is_select_item_expr, ObIArray ¶ms); + int check_ja_query_ref_param_validity(ObQueryRefRawExpr &ja_query_ref, bool &is_valid); + int check_aggr_first_validity(ObDMLStmt &stmt, ObQueryRefRawExpr &query_ref, const bool is_vector_assign, From 5a0b6f8129344541cbbc5816cb413ea51b8f32ca Mon Sep 17 00:00:00 2001 From: linqiucen Date: Fri, 23 Aug 2024 13:49:09 +0000 Subject: [PATCH 199/249] [CP] change join to right join --- src/share/ob_service_name_proxy.cpp | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/share/ob_service_name_proxy.cpp b/src/share/ob_service_name_proxy.cpp index e6e0368d4..9ffb3edd4 100644 --- a/src/share/ob_service_name_proxy.cpp +++ b/src/share/ob_service_name_proxy.cpp @@ -247,11 +247,12 @@ int ObServiceNameProxy::select_all_service_names_with_epoch( ret = OB_ERR_UNEXPECTED; LOG_WARN("GCTX.sql_proxy_ is null", KR(ret), KP(GCTX.sql_proxy_)); } else if (OB_FAIL(sql.assign_fmt("SELECT s.*, e.value as epoch FROM %s AS s " - "JOIN %s AS e ON s.tenant_id = e.tenant_id WHERE s.tenant_id = %lu and e.name='%s' ORDER BY s.gmt_create", + "RIGHT JOIN %s AS e ON s.tenant_id = e.tenant_id WHERE e.tenant_id = %lu and e.name='%s' ORDER BY s.gmt_create", OB_ALL_SERVICE_TNAME, OB_ALL_SERVICE_EPOCH_TNAME, tenant_id, ObServiceEpochProxy::SERVICE_NAME_EPOCH))) { // join the two tables to avoid add a row lock on __all_service_epoch // otherwise there might be conflicts and too many retries in tenant_info_loader thread // when the number of observers is large + // use right join instead of join, otherwise we cannot see service_name being deleted in the cache LOG_WARN("sql assign_fmt failed", KR(ret), K(sql)); } else if (OB_FAIL(select_service_name_sql_helper_(*GCTX.sql_proxy_, tenant_id, EXTRACT_EPOCH, sql, epoch, all_service_names))) { @@ -316,7 +317,7 @@ int ObServiceNameProxy::select_service_name_sql_helper_( } } else if (OB_FAIL(build_service_name_(*result, service_name))) { LOG_WARN("fail to build server status", KR(ret)); - } else if (OB_FAIL(all_service_names.push_back(service_name))) { + } else if (service_name.is_valid() && OB_FAIL(all_service_names.push_back(service_name))) { LOG_WARN("fail to build service_name", KR(ret)); } else if (extract_epoch) { // epoch can only be extracted when __all_service_epoch table is joined @@ -578,16 +579,20 @@ int ObServiceNameProxy::build_service_name_( ObString service_name_str; ObString service_status_str; service_name.reset(); - EXTRACT_VARCHAR_FIELD_MYSQL(res, "service_name", service_name_str); - EXTRACT_VARCHAR_FIELD_MYSQL(res, "service_status", service_status_str); - EXTRACT_INT_FIELD_MYSQL(res, "tenant_id", tenant_id, uint64_t); EXTRACT_INT_FIELD_MYSQL(res, "service_name_id", service_name_id, uint64_t); - if (FAILEDx(service_name.init(tenant_id, service_name_id, service_name_str, service_status_str))) { - LOG_WARN("fail to init service_name", KR(ret), K(tenant_id), K(service_name_id), K(service_name_str), - K(service_status_str)); - } else if (OB_UNLIKELY(!service_name.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("build invalid service_name", KR(ret), K(service_name)); + if (OB_ERR_NULL_VALUE == ret) { + ret = OB_SUCCESS; // __all_service table is empty, overwrite ret + } else { + EXTRACT_VARCHAR_FIELD_MYSQL(res, "service_name", service_name_str); + EXTRACT_VARCHAR_FIELD_MYSQL(res, "service_status", service_status_str); + EXTRACT_INT_FIELD_MYSQL(res, "tenant_id", tenant_id, uint64_t); + if (FAILEDx(service_name.init(tenant_id, service_name_id, service_name_str, service_status_str))) { + LOG_WARN("fail to init service_name", KR(ret), K(tenant_id), K(service_name_id), K(service_name_str), + K(service_status_str)); + } else if (OB_UNLIKELY(!service_name.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("build invalid service_name", KR(ret), K(service_name)); + } } return ret; } From 10e34896bc184d3d750aa881f864227b38a54849 Mon Sep 17 00:00:00 2001 From: chimyue Date: Fri, 23 Aug 2024 13:55:07 +0000 Subject: [PATCH 200/249] [FEAT MERGE] enhance mview rewrite, support join MAV fast refresh and real-time mview Co-authored-by: hy-guo --- src/rootserver/ob_ddl_service.cpp | 8 +- src/rootserver/ob_ddl_service.h | 6 +- src/share/ob_rpc_struct.h | 4 +- src/share/schema/ob_column_schema.h | 7 +- src/share/schema/ob_schema_retrieve_utils.ipp | 33 +- src/share/schema/ob_schema_struct.cpp | 381 -- src/share/schema/ob_schema_struct.h | 51 +- src/share/schema/ob_table_schema.h | 6 +- src/share/schema/ob_table_sql_service.cpp | 29 +- src/sql/CMakeLists.txt | 2 + .../code_generator/ob_static_engine_cg.cpp | 31 + src/sql/code_generator/ob_static_engine_cg.h | 3 + src/sql/engine/cmd/ob_outline_executor.cpp | 8 +- src/sql/engine/expr/ob_expr_operator.cpp | 6 +- src/sql/engine/expr/ob_expr_operator.h | 12 +- src/sql/engine/expr/ob_expr_util.h | 8 +- src/sql/ob_sql.cpp | 14 +- src/sql/ob_sql.h | 2 + src/sql/ob_sql_utils.cpp | 10 +- src/sql/ob_sql_utils.h | 12 +- src/sql/optimizer/ob_conflict_detector.cpp | 259 +- src/sql/optimizer/ob_conflict_detector.h | 43 +- src/sql/optimizer/ob_log_plan.cpp | 347 +- src/sql/optimizer/ob_log_plan.h | 32 +- .../resolver/ddl/ob_create_view_resolver.cpp | 61 + .../resolver/ddl/ob_create_view_resolver.h | 6 + src/sql/resolver/ddl/ob_ddl_resolver.cpp | 4 +- src/sql/resolver/ddl/ob_ddl_resolver.h | 4 +- src/sql/resolver/dml/ob_del_upd_stmt.cpp | 26 +- src/sql/resolver/dml/ob_del_upd_stmt.h | 1 + src/sql/resolver/dml/ob_dml_resolver.cpp | 21 +- src/sql/resolver/dml/ob_dml_resolver.h | 1 + src/sql/resolver/dml/ob_dml_stmt.cpp | 13 + src/sql/resolver/dml/ob_dml_stmt.h | 5 +- src/sql/resolver/dml/ob_select_stmt.cpp | 63 +- src/sql/resolver/dml/ob_select_stmt.h | 2 + src/sql/resolver/dml/ob_stmt_expr_visitor.h | 2 + src/sql/resolver/expr/ob_raw_expr.cpp | 88 +- src/sql/resolver/expr/ob_raw_expr.h | 27 +- .../resolver/expr/ob_raw_expr_replacer.cpp | 13 + src/sql/resolver/expr/ob_raw_expr_replacer.h | 1 + src/sql/resolver/expr/ob_raw_expr_util.cpp | 4 +- src/sql/resolver/mv/ob_mv_checker.cpp | 61 +- src/sql/resolver/mv/ob_mv_checker.h | 8 +- src/sql/resolver/mv/ob_mv_printer.cpp | 1207 +++-- src/sql/resolver/mv/ob_mv_printer.h | 88 +- src/sql/resolver/mv/ob_mv_provider.cpp | 75 + src/sql/resolver/mv/ob_mv_provider.h | 4 + src/sql/rewrite/ob_expand_aggregate_utils.cpp | 26 +- src/sql/rewrite/ob_expand_aggregate_utils.h | 6 +- .../rewrite/ob_transform_join_elimination.cpp | 7 +- src/sql/rewrite/ob_transform_min_max.cpp | 4 +- src/sql/rewrite/ob_transform_mv_rewrite.cpp | 4005 ++++++++++++----- src/sql/rewrite/ob_transform_mv_rewrite.h | 427 +- .../ob_transform_mv_rewrite_prepare.cpp | 489 ++ .../rewrite/ob_transform_mv_rewrite_prepare.h | 83 + src/sql/rewrite/ob_transform_or_expansion.cpp | 4 +- src/sql/rewrite/ob_transform_rule.cpp | 10 +- src/sql/rewrite/ob_transform_rule.h | 68 +- .../ob_transform_simplify_subquery.cpp | 4 +- src/sql/rewrite/ob_transform_utils.cpp | 82 +- src/sql/rewrite/ob_transform_utils.h | 14 +- src/sql/rewrite/ob_transform_win_magic.cpp | 40 +- src/sql/rewrite/ob_transform_win_magic.h | 4 - src/sql/rewrite/ob_transformer_impl.cpp | 21 + src/sql/rewrite/ob_transformer_impl.h | 1 + src/sql/session/ob_local_session_var.cpp | 556 +++ src/sql/session/ob_local_session_var.h | 103 + 68 files changed, 6438 insertions(+), 2615 deletions(-) create mode 100644 src/sql/rewrite/ob_transform_mv_rewrite_prepare.cpp create mode 100644 src/sql/rewrite/ob_transform_mv_rewrite_prepare.h create mode 100644 src/sql/session/ob_local_session_var.cpp create mode 100755 src/sql/session/ob_local_session_var.h diff --git a/src/rootserver/ob_ddl_service.cpp b/src/rootserver/ob_ddl_service.cpp index 5022945bb..dc889307b 100755 --- a/src/rootserver/ob_ddl_service.cpp +++ b/src/rootserver/ob_ddl_service.cpp @@ -8171,7 +8171,7 @@ int ObDDLService::update_generated_column_schema( const ObColumnSchemaV2 &orig_column_schema, const ObTableSchema &origin_table_schema, const ObTimeZoneInfoWrap &tz_info_wrap, - const share::schema::ObLocalSessionVar *local_session_var, + const sql::ObLocalSessionVar *local_session_var, ObTableSchema &new_table_schema, const bool need_update_default_value, const bool need_update_session_var, @@ -8305,7 +8305,7 @@ int ObDDLService::modify_generated_column_local_vars(ObColumnSchemaV2 &generated const ObObjType origin_type, const AlterColumnSchema &new_column_schema, const ObTableSchema &table_schema, - const share::schema::ObLocalSessionVar *local_session_var) { + const sql::ObLocalSessionVar *local_session_var) { int ret = OB_SUCCESS; if (generated_column.is_generated_column() && origin_type != new_column_schema.get_data_type()) { @@ -8353,7 +8353,7 @@ int ObDDLService::modify_generated_column_local_vars(ObColumnSchemaV2 &generated dst_type.set_collation_level(CS_LEVEL_IMPLICIT); ObSQLMode sql_mode = default_session.get_sql_mode(); if (NULL != local_session_var) { - share::schema::ObSessionSysVar *sys_var = NULL; + sql::ObSessionSysVar *sys_var = NULL; if (OB_FAIL(local_session_var->get_local_var(share::SYS_VAR_SQL_MODE, sys_var))) { LOG_WARN("fail to get sys var", K(ret)); } else if (NULL != sys_var) { @@ -9611,7 +9611,7 @@ int ObDDLService::add_new_column_to_table_schema( const AlterTableSchema &alter_table_schema, const common::ObTimeZoneInfoWrap &tz_info_wrap, const common::ObString &nls_formats, - share::schema::ObLocalSessionVar &local_session_var, + sql::ObLocalSessionVar &local_session_var, obrpc::ObSequenceDDLArg &sequence_ddl_arg, common::ObIAllocator &allocator, ObTableSchema &new_table_schema, diff --git a/src/rootserver/ob_ddl_service.h b/src/rootserver/ob_ddl_service.h index 9299622aa..9f0cadb02 100644 --- a/src/rootserver/ob_ddl_service.h +++ b/src/rootserver/ob_ddl_service.h @@ -346,7 +346,7 @@ public: const share::schema::AlterTableSchema &alter_table_schema, const common::ObTimeZoneInfoWrap &tz_info_wrap, const common::ObString &nls_formats, - share::schema::ObLocalSessionVar &local_session_var, + sql::ObLocalSessionVar &local_session_var, obrpc::ObSequenceDDLArg &sequence_ddl_arg, common::ObIAllocator &allocator, share::schema::ObTableSchema &new_table_schema, @@ -1871,7 +1871,7 @@ private: const share::schema::ObColumnSchemaV2 &orig_column_schema, const share::schema::ObTableSchema &origin_table_schema, const common::ObTimeZoneInfoWrap &tz_info_wrap, - const share::schema::ObLocalSessionVar *local_session_var, + const sql::ObLocalSessionVar *local_session_var, share::schema::ObTableSchema &new_table_schema, const bool need_update_default_value, const bool need_update_session_var, @@ -1887,7 +1887,7 @@ private: const ObObjType origin_type, const AlterColumnSchema &new_column_schema, const ObTableSchema &table_schema, - const share::schema::ObLocalSessionVar *local_session_var); + const sql::ObLocalSessionVar *local_session_var); int modify_depend_column_type(sql::ObRawExpr *expr, const ObString &column_name, const AlterColumnSchema &alter_column_schema, diff --git a/src/share/ob_rpc_struct.h b/src/share/ob_rpc_struct.h index faf139180..1a7b8bbdc 100644 --- a/src/share/ob_rpc_struct.h +++ b/src/share/ob_rpc_struct.h @@ -2325,7 +2325,7 @@ public: bool foreign_key_checks_; bool is_add_to_scheduler_; common::ObAddr inner_sql_exec_addr_; - ObLocalSessionVar local_session_var_; + sql::ObLocalSessionVar local_session_var_; ObMViewRefreshInfo mview_refresh_info_; AlterAlgorithm alter_algorithm_; bool alter_auto_partition_attr_; @@ -2773,7 +2773,7 @@ public: ObSQLMode sql_mode_; common::ObAddr inner_sql_exec_addr_; common::ObArenaAllocator allocator_; - ObLocalSessionVar local_session_var_; + sql::ObLocalSessionVar local_session_var_; bool exist_all_column_group_; common::ObSEArray index_cgs_; share::schema::ObVectorIndexRefreshInfo vidx_refresh_info_; diff --git a/src/share/schema/ob_column_schema.h b/src/share/schema/ob_column_schema.h index 2c542b4ec..92ad65cde 100644 --- a/src/share/schema/ob_column_schema.h +++ b/src/share/schema/ob_column_schema.h @@ -45,7 +45,6 @@ const char *const STR_COLUMN_TYPE_RAW = "raw"; const char *const STR_COLUMN_TYPE_UNKNOWN = "unknown"; class ObTableSchema; -class ObLocalSessionVar; class ObColumnSchemaV2 : public ObSchema { OB_UNIS_VERSION_V(1); @@ -339,8 +338,8 @@ int assign(const ObColumnSchemaV2 &src_schema); } int get_each_column_group_name(ObString &cg_name) const; - inline ObLocalSessionVar &get_local_session_var() { return local_session_vars_; } - inline const ObLocalSessionVar &get_local_session_var() const { return local_session_vars_; } + inline sql::ObLocalSessionVar &get_local_session_var() { return local_session_vars_; } + inline const sql::ObLocalSessionVar &get_local_session_var() const { return local_session_vars_; } DECLARE_VIRTUAL_TO_STRING; private: @@ -394,7 +393,7 @@ private: uint64_t sub_type_; ObSkipIndexColumnAttr skip_index_attr_; int64_t lob_chunk_size_; - ObLocalSessionVar local_session_vars_; + sql::ObLocalSessionVar local_session_vars_; }; inline int32_t ObColumnSchemaV2::get_data_length() const diff --git a/src/share/schema/ob_schema_retrieve_utils.ipp b/src/share/schema/ob_schema_retrieve_utils.ipp index 915b2d169..e58539ea6 100644 --- a/src/share/schema/ob_schema_retrieve_utils.ipp +++ b/src/share/schema/ob_schema_retrieve_utils.ipp @@ -1503,6 +1503,18 @@ int ObSchemaRetrieveUtils::fill_table_schema( bool, true, true/*ignore_column_error*/, false); EXTRACT_INT_FIELD_TO_CLASS_MYSQL_WITH_DEFAULT_VALUE(result, auto_increment_cache_size, table_schema, int64_t, true, true, 0); + if (OB_SUCC(ret) && table_schema.is_materialized_view()) { + bool skip_null_error = true; + bool skip_column_error = true; + ObString local_session_var; + ObString default_session_var(""); //default value is empty string + EXTRACT_VARCHAR_FIELD_MYSQL_WITH_DEFAULT_VALUE(result, "local_session_vars", local_session_var, + skip_null_error, skip_column_error, default_session_var); + if (OB_SUCC(ret) && !local_session_var.empty() + && OB_FAIL(table_schema.get_local_session_var().fill_local_session_var_from_str(local_session_var))) { + SHARE_SCHEMA_LOG(WARN, "fail to deserialize mview_session_var", K(ret)); + } + } } if (OB_SUCC(ret) && OB_FAIL(fill_sys_table_lob_tid(table_schema))) { SHARE_SCHEMA_LOG(WARN, "fail to fill lob table id for inner table", K(ret), K(table_schema.get_table_id())); @@ -1624,30 +1636,15 @@ int ObSchemaRetrieveUtils::fill_column_schema( bool skip_column_error = true; ObString local_session_var; ObString default_session_var(""); //default value is empty string - int64_t pos = 0; EXTRACT_VARCHAR_FIELD_MYSQL_WITH_DEFAULT_VALUE(result, "local_session_vars", local_session_var, skip_null_error, skip_column_error, default_session_var); - if (OB_SUCC(ret) && !local_session_var.empty()) { - ObArenaAllocator tmp_allocator(ObModIds::OB_TEMP_VARIABLES); - char *value_buf = NULL; - if (OB_ISNULL(value_buf = static_cast(tmp_allocator.alloc(local_session_var.length())))) { - ret = common::OB_ALLOCATE_MEMORY_FAILED; - SHARE_SCHEMA_LOG(WARN, "fail to alloc memory", K(ret)); - } else { - ObLength len = common::str_to_hex(local_session_var.ptr(), - local_session_var.length(), - value_buf, - local_session_var.length()); - if (OB_FAIL(column.get_local_session_var().deserialize_(value_buf, static_cast(len), pos))) { - SHARE_SCHEMA_LOG(WARN, "fail to deserialize local_session_var", K(ret)); - } else { - tmp_allocator.free(value_buf); - } - } + if (OB_SUCC(ret) && !local_session_var.empty() + && OB_FAIL(column.get_local_session_var().fill_local_session_var_from_str(local_session_var))) { + SHARE_SCHEMA_LOG(WARN, "fail to deserialize local_session_var", K(ret)); } } } diff --git a/src/share/schema/ob_schema_struct.cpp b/src/share/schema/ob_schema_struct.cpp index d8207c274..51f57af50 100644 --- a/src/share/schema/ob_schema_struct.cpp +++ b/src/share/schema/ob_schema_struct.cpp @@ -14582,387 +14582,6 @@ int ObIndexNameInfo::init( return ret; } -template -int ObLocalSessionVar::set_local_vars(T &var_array) -{ - int ret = OB_SUCCESS; - if (!local_session_vars_.empty()) { - local_session_vars_.reset(); - } - if (OB_FAIL(local_session_vars_.reserve(var_array.count()))) { - LOG_WARN("fail to reserve for local_session_vars", K(ret)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < var_array.count(); ++i) { - if (OB_FAIL(add_local_var(var_array.at(i)))) { - LOG_WARN("fail to add session var", K(ret)); - } - } - } - return ret; -} - -int ObLocalSessionVar::add_local_var(const ObSessionSysVar *var) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(var)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", K(ret), KP(var)); - } else if (OB_FAIL(add_local_var(var->type_, var->val_))) { - LOG_WARN("fail to add local session var", K(ret)); - } - return ret; -} - -int ObLocalSessionVar::add_local_var(ObSysVarClassType var_type, const ObObj &value) -{ - int ret = OB_SUCCESS; - ObSessionSysVar *cur_var = NULL; - if (OB_ISNULL(alloc_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", K(ret), KP(alloc_)); - } else if (OB_FAIL(get_local_var(var_type, cur_var))) { - LOG_WARN("get local var failed", K(ret)); - } else if (NULL == cur_var) { - ObSessionSysVar *new_var = OB_NEWx(ObSessionSysVar, alloc_); - if (OB_ISNULL(new_var)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("alloc new var failed.", K(ret)); - } else if (OB_FAIL(local_session_vars_.push_back(new_var))) { - LOG_WARN("push back new var failed", K(ret)); - } else if (OB_FAIL(deep_copy_obj(*alloc_, value, new_var->val_))) { - LOG_WARN("fail to deep copy obj", K(ret)); - } else { - new_var->type_ = var_type; - } - } else if (!cur_var->is_equal(value)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("local session var added before is not equal to the new var", K(ret), KPC(cur_var), K(value)); - } - return ret; -} - -int ObLocalSessionVar::get_local_var(ObSysVarClassType var_type, ObSessionSysVar *&sys_var) const -{ - int ret = OB_SUCCESS; - sys_var = NULL; - for (int64_t i = 0; OB_SUCC(ret) && NULL == sys_var && i < local_session_vars_.count(); ++i) { - if (OB_ISNULL(local_session_vars_.at(i))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", K(ret), K(local_session_vars_)); - } else if (local_session_vars_.at(i)->type_ == var_type) { - sys_var = local_session_vars_.at(i); - } - } - return ret; -} - -int ObLocalSessionVar::get_local_vars(ObIArray &var_array) const -{ - int ret = OB_SUCCESS; - var_array.reset(); - for (int64_t i = 0; OB_SUCC(ret) && i < local_session_vars_.count(); ++i){ - if (OB_FAIL(var_array.push_back(local_session_vars_.at(i)))) { - LOG_WARN("push back local session vars failed", K(ret)); - } - } - return ret; -} - -int ObLocalSessionVar::remove_vars_same_with_session(const sql::ObBasicSessionInfo *session) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(session)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", K(ret), KP(session)); - } else { - ObSEArray new_var_array; - for (int64_t i = 0; OB_SUCC(ret) && i < local_session_vars_.count(); ++i) { - ObSessionSysVar *local_var = local_session_vars_.at(i); - ObObj session_val; - if (OB_ISNULL(local_var)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", K(ret), KP(local_var)); - } else if (SYS_VAR_SQL_MODE == local_var->type_) { - if (local_var->val_.get_uint64() != session->get_sql_mode() - && OB_FAIL(new_var_array.push_back(local_var))) { - LOG_WARN("fail to push into new var array", K(ret)); - } - } else if (OB_FAIL(session->get_sys_variable(local_var->type_, session_val))) { - LOG_WARN("fail to get session variable", K(ret)); - } else if (!local_var->is_equal(session_val) && - OB_FAIL(new_var_array.push_back(local_var))) { - LOG_WARN("fail to push into new var array", K(ret)); - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(local_session_vars_.assign(new_var_array))) { - LOG_WARN("fail to set local session vars.", K(ret)); - } - } - } - return ret; -} - -int ObLocalSessionVar::deep_copy(const ObLocalSessionVar &other) -{ - int ret = OB_SUCCESS; - local_session_vars_.reset(); - if (this == &other) { - //do nothing - } else if (NULL != other.alloc_) { - if (NULL == alloc_) { - alloc_ = other.alloc_; - local_session_vars_.set_allocator(other.alloc_); - } - } - if (OB_FAIL(set_local_vars(other.local_session_vars_))) { - LOG_WARN("fail to add session var", K(ret)); - } - return ret; -} - -int ObLocalSessionVar::deep_copy_self() -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(alloc_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null allocator", K(ret)); - } else { - ObSEArray var_array; - if (OB_FAIL(get_local_vars(var_array))) { - LOG_WARN("get local vars failed", K(ret)); - } else if (OB_FAIL(set_local_vars(var_array))) { - LOG_WARN("set local vars failed", K(ret)); - } - } - return ret; -} - -int ObLocalSessionVar::assign(const ObLocalSessionVar &other) -{ - int ret = OB_SUCCESS; - local_session_vars_.reset(); - if (NULL != other.alloc_) { - if (NULL == alloc_) { - alloc_ = other.alloc_; - local_session_vars_.set_allocator(other.alloc_); - } - if (OB_FAIL(local_session_vars_.reserve(other.local_session_vars_.count()))) { - LOG_WARN("reserve failed", K(ret)); - } else if (OB_FAIL(local_session_vars_.assign(other.local_session_vars_))) { - LOG_WARN("fail to push back local var", K(ret)); - } - } else { - //do nothing, other is not inited - } - return ret; -} - -void ObLocalSessionVar::reset() -{ - local_session_vars_.reset(); -} - -int ObLocalSessionVar::set_local_var_capacity(int64_t sz) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(local_session_vars_.reserve(sz))) { - LOG_WARN("reserve failed", K(ret), K(sz)); - } - return ret; -} - -bool ObLocalSessionVar::operator == (const ObLocalSessionVar& other) const -{ - bool is_equal = local_session_vars_.count() == other.local_session_vars_.count(); - if (is_equal) { - int tmp_ret = OB_SUCCESS; - for (int64_t i = 0; is_equal && i < local_session_vars_.count(); ++i) { - ObSessionSysVar *var = local_session_vars_.at(i); - ObSessionSysVar *other_val = NULL; ; - if (OB_ISNULL(var)) { - is_equal = false; - } else if ((tmp_ret = other.get_local_var(var->type_, other_val)) != OB_SUCCESS) { - is_equal = false; - } else if (other_val == NULL) { - is_equal = false; - } else { - is_equal = var->is_equal(other_val->val_); - } - } - } - return is_equal; -} - -int64_t ObLocalSessionVar::get_deep_copy_size() const -{ - int64_t sz = sizeof(*this) + local_session_vars_.count() * sizeof(ObSessionSysVar *); - for (int64_t i = 0; i < local_session_vars_.count(); ++i) { - if (OB_NOT_NULL(local_session_vars_.at(i))) { - sz += local_session_vars_.at(i)->get_deep_copy_size(); - } - } - return sz; -} - -bool ObSessionSysVar::is_equal(const ObObj &other) const -{ - bool bool_ret = false; - if (val_.get_meta() != other.get_meta()) { - bool_ret = false; - if (ob_is_string_type(val_.get_type()) - && val_.get_type() == other.get_type() - && val_.get_collation_type() != other.get_collation_type()) { - //the collation type of string system variables will be set to the current connection collation type after updating values. - //return true if the string values are equal. - bool_ret = common::ObCharset::case_sensitive_equal(val_.get_string(), other.get_string()); - } - } else if (val_.is_equal(other, CS_TYPE_BINARY)) { - bool_ret = true; - } - return bool_ret; -} - -OB_DEF_SERIALIZE(ObSessionSysVar) -{ - int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_ENCODE, type_, val_); - return ret; -} - -OB_DEF_SERIALIZE_SIZE(ObSessionSysVar) -{ - int64_t len = 0; - LST_DO_CODE(OB_UNIS_ADD_LEN, type_, val_); - return len; -} - -OB_DEF_DESERIALIZE(ObSessionSysVar) -{ - int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_DECODE, type_, val_); - return ret; -} - -int64_t ObSessionSysVar::get_deep_copy_size() const { - int64_t sz = sizeof(*this) + val_.get_deep_copy_size(); - return sz; -} - -const ObSysVarClassType ObLocalSessionVar::ALL_LOCAL_VARS[] = { - SYS_VAR_TIME_ZONE, - SYS_VAR_SQL_MODE, - SYS_VAR_NLS_DATE_FORMAT, - SYS_VAR_NLS_TIMESTAMP_FORMAT, - SYS_VAR_NLS_TIMESTAMP_TZ_FORMAT, - SYS_VAR_COLLATION_CONNECTION, - SYS_VAR_MAX_ALLOWED_PACKET -}; - -//add all vars that can be solidified -int ObLocalSessionVar::load_session_vars(const sql::ObBasicSessionInfo *session) { - int ret = OB_SUCCESS; - int64_t var_num = sizeof(ALL_LOCAL_VARS) / sizeof(ObSysVarClassType); - if (OB_ISNULL(session)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null session", K(ret)); - } else if (!local_session_vars_.empty()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("local_session_vars can only be inited once", K(ret)); - } else if (OB_FAIL(local_session_vars_.reserve(var_num))) { - LOG_WARN("reserve failed", K(ret), K(var_num)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < var_num; ++i) { - ObObj var; - if (OB_FAIL(session->get_sys_variable(ALL_LOCAL_VARS[i], var))) { - LOG_WARN("fail to get session variable", K(ret)); - } else if (OB_FAIL(add_local_var(ALL_LOCAL_VARS[i], var))) { - LOG_WARN("fail to add session var", K(ret), K(var)); - } - } - } - return ret; -} - -int ObLocalSessionVar::update_session_vars_with_local(sql::ObBasicSessionInfo &session) const { - int ret = OB_SUCCESS; - for (int64_t i = 0; OB_SUCC(ret) && i < local_session_vars_.count(); ++i) { - if (OB_ISNULL(local_session_vars_.at(i))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", K(ret)); - } else if (OB_FAIL(session.update_sys_variable(local_session_vars_.at(i)->type_, local_session_vars_.at(i)->val_))) { - LOG_WARN("fail to update sys variable", K(ret)); - } - } - return ret; -} - -OB_DEF_SERIALIZE(ObLocalSessionVar) -{ - int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_ENCODE, local_session_vars_.count()); - for (int64_t i = 0; OB_SUCC(ret) && i < local_session_vars_.count(); ++i) { - if (OB_ISNULL(local_session_vars_.at(i))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", K(ret)); - } else { - LST_DO_CODE(OB_UNIS_ENCODE, *local_session_vars_.at(i)); - } - } - return ret; -} - -OB_DEF_SERIALIZE_SIZE(ObLocalSessionVar) -{ - int64_t len = 0; - LST_DO_CODE(OB_UNIS_ADD_LEN, local_session_vars_.count()); - for (int64_t i = 0; i < local_session_vars_.count(); ++i) { - if (OB_NOT_NULL(local_session_vars_.at(i))) { - LST_DO_CODE(OB_UNIS_ADD_LEN, *local_session_vars_.at(i)); - } - } - return len; -} - -OB_DEF_DESERIALIZE(ObLocalSessionVar) -{ - int ret = OB_SUCCESS; - int64_t cnt = 0; - OB_UNIS_DECODE(cnt); - if (OB_SUCC(ret)) { - if (OB_FAIL(local_session_vars_.reserve(cnt))) { - LOG_WARN("reserve failed", K(ret)); - } - } - for (int64_t i = 0; OB_SUCC(ret) && i < cnt; ++i) { - ObSessionSysVar var; - LST_DO_CODE(OB_UNIS_DECODE, var); - if (OB_SUCC(ret)) { - if (OB_FAIL(add_local_var(&var))) { - LOG_WARN("fail to add local session var", K(ret)); - } - } - } - return ret; -} - -DEF_TO_STRING(ObLocalSessionVar) -{ - int64_t pos = 0; - J_OBJ_START(); - for (int64_t i = 0; i < local_session_vars_.count(); ++i) { - if (i > 0) { - J_COMMA(); - } - if (OB_NOT_NULL(local_session_vars_.at(i))) { - J_KV("type", local_session_vars_.at(i)->type_, - "val", local_session_vars_.at(i)->val_); - } - } - J_OBJ_END(); - return pos; -} - // // } //namespace schema diff --git a/src/share/schema/ob_schema_struct.h b/src/share/schema/ob_schema_struct.h index ec720ce5d..0251b8b86 100755 --- a/src/share/schema/ob_schema_struct.h +++ b/src/share/schema/ob_schema_struct.h @@ -35,6 +35,7 @@ #include "share/cache/ob_kv_storecache.h" // ObKVCacheHandle #include "lib/hash/ob_pointer_hashmap.h" #include "lib/string/ob_sql_string.h" +#include "sql/session/ob_local_session_var.h" #ifdef __cplusplus extern "C" { @@ -59,6 +60,7 @@ namespace sql { class ObSQLSessionInfo; class ObPartitionExecutorUtils; +class ObLocalSessionVar; } namespace rootserver { @@ -9300,55 +9302,6 @@ struct GetIndexNameKey typedef common::hash::ObPointerHashMap ObIndexNameMap; -struct ObSessionSysVar { - OB_UNIS_VERSION(1); -public: - TO_STRING_KV(K_(type), K_(val)); - bool is_equal(const ObObj &other_val) const; - int64_t get_deep_copy_size() const; - ObSysVarClassType type_; - ObObj val_; -}; - -class ObLocalSessionVar { - OB_UNIS_VERSION(1); -public: - ObLocalSessionVar(ObIAllocator *alloc) - :alloc_(alloc), - local_session_vars_(alloc) { - } - ObLocalSessionVar () - :alloc_(NULL) { - } - ~ObLocalSessionVar() { reset(); } - void set_allocator(ObIAllocator *allocator) { - alloc_ = allocator; - local_session_vars_.set_allocator(allocator); - } - void reset(); - int set_local_var_capacity(int64_t sz); - template - int set_local_vars(T &var_array); - int add_local_var(ObSysVarClassType var_type, const ObObj &value); - int add_local_var(const ObSessionSysVar *var); - int get_local_var(ObSysVarClassType var_type, ObSessionSysVar *&sys_var) const; - int get_local_vars(ObIArray &var_array) const; - int load_session_vars(const sql::ObBasicSessionInfo *session); - int update_session_vars_with_local(sql::ObBasicSessionInfo &session) const; - int remove_vars_same_with_session(const sql::ObBasicSessionInfo *session); - int deep_copy(const ObLocalSessionVar &other); - int deep_copy_self(); - int assign(const ObLocalSessionVar &other); - bool operator == (const ObLocalSessionVar& other) const; - int64_t get_deep_copy_size() const ; - int64_t get_var_count() const { return local_session_vars_.count(); } - DECLARE_TO_STRING; -private: - const static ObSysVarClassType ALL_LOCAL_VARS[]; - common::ObIAllocator *alloc_; - ObFixedArray local_session_vars_; -}; - }//namespace schema }//namespace share }//namespace oceanbase diff --git a/src/share/schema/ob_table_schema.h b/src/share/schema/ob_table_schema.h index cb985c888..506809a90 100644 --- a/src/share/schema/ob_table_schema.h +++ b/src/share/schema/ob_table_schema.h @@ -1800,8 +1800,8 @@ public: } void set_mlog_tid(const uint64_t& table_id) { mlog_tid_ = table_id; } uint64_t get_mlog_tid() const { return mlog_tid_; } - inline ObLocalSessionVar &get_local_session_var() { return local_session_vars_; } - inline const ObLocalSessionVar &get_local_session_var() const { return local_session_vars_; } + inline sql::ObLocalSessionVar &get_local_session_var() { return local_session_vars_; } + inline const sql::ObLocalSessionVar &get_local_session_var() const { return local_session_vars_; } DECLARE_VIRTUAL_TO_STRING; protected: @@ -2004,7 +2004,7 @@ protected: CgIdHashArray *cg_id_hash_arr_; CgNameHashArray *cg_name_hash_arr_; uint64_t mlog_tid_; - ObLocalSessionVar local_session_vars_; + sql::ObLocalSessionVar local_session_vars_; // vector index common::ObString index_params_; }; diff --git a/src/share/schema/ob_table_sql_service.cpp b/src/share/schema/ob_table_sql_service.cpp index 71985cc42..b72351697 100644 --- a/src/share/schema/ob_table_sql_service.cpp +++ b/src/share/schema/ob_table_sql_service.cpp @@ -3030,6 +3030,8 @@ int ObTableSqlService::gen_table_dml( "" : table.get_ttl_definition().ptr(); const char *kv_attributes = table.get_kv_attributes().empty() ? "" : table.get_kv_attributes().ptr(); + ObString local_session_var; + ObArenaAllocator allocator(ObModIds::OB_SCHEMA_OB_SCHEMA_ARENA); if (OB_FAIL(check_table_options(table))) { LOG_WARN("fail to check table option", K(ret), K(table)); } else if (data_version < DATA_VERSION_4_1_0_0 && 0 != table.get_table_flags()) { @@ -3038,6 +3040,9 @@ int ObTableSqlService::gen_table_dml( K(table)); } else if (OB_FAIL(check_column_store_valid(table, data_version))) { LOG_WARN("fail to check column store valid", KR(ret), K(table), K(data_version)); + } else if (table.is_materialized_view() && data_version >= DATA_VERSION_4_3_3_0 + && OB_FAIL(table.get_local_session_var().gen_local_session_var_str(allocator, local_session_var))) { + LOG_WARN("fail to gen local session var str", K(ret)); } else if (OB_FAIL(dml.add_pk_column("tenant_id", ObSchemaUtils::get_extract_tenant_id( exec_tenant_id, table.get_tenant_id()))) || OB_FAIL(dml.add_pk_column("table_id", ObSchemaUtils::get_extract_schema_id( @@ -3147,6 +3152,8 @@ int ObTableSqlService::gen_table_dml( && OB_FAIL(dml.add_column("column_store", table.is_column_store_supported()))) || ((data_version >= DATA_VERSION_4_3_2_0 || (data_version < DATA_VERSION_4_3_0_0 && data_version >= MOCK_DATA_VERSION_4_2_3_0)) && OB_FAIL(dml.add_column("auto_increment_cache_size", table.get_auto_increment_cache_size()))) + || (data_version >= DATA_VERSION_4_3_3_0 + && OB_FAIL(dml.add_column("local_session_vars", ObHexEscapeSqlStr(local_session_var)))) ) { LOG_WARN("add column failed", K(ret)); } @@ -4335,25 +4342,9 @@ int ObTableSqlService::gen_column_dml( } } ObString local_session_var; - if (OB_SUCC(ret) && column.is_generated_column() && tenant_data_version >= DATA_VERSION_4_2_2_0) { - int64_t pos = 0; - int64_t buf_len = column.get_local_session_var().get_serialize_size(); - char *binary_str = NULL;; - char *hex_str = NULL; - int64_t hex_pos = 0; - ObArenaAllocator tmp_allocator(ObModIds::OB_TEMP_VARIABLES); - if (OB_ISNULL(binary_str = static_cast(tmp_allocator.alloc(buf_len))) - || OB_ISNULL(hex_str = static_cast(allocator.alloc(buf_len * 2)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("allocate memory for local_session_var failed", K(ret), KP(binary_str), KP(hex_str)); - } else if (OB_FAIL(column.get_local_session_var().serialize_(binary_str, buf_len, pos))) { - LOG_WARN("fail to serialize local_session_var", K(ret)); - } else if (OB_FAIL(common::hex_print(binary_str, pos, hex_str, buf_len * 2, hex_pos))) { - LOG_WARN("print hex string failed", K(ret)); - } else { - local_session_var.assign(hex_str, hex_pos); - tmp_allocator.free(binary_str); - } + if (OB_SUCC(ret) && column.is_generated_column() && tenant_data_version >= DATA_VERSION_4_2_2_0 + && OB_FAIL(column.get_local_session_var().gen_local_session_var_str(allocator, local_session_var))) { + LOG_WARN("fail to gen local session var str", K(ret)); } if (OB_SUCC(ret) && (OB_FAIL(dml.add_pk_column("tenant_id", ObSchemaUtils::get_extract_tenant_id( exec_tenant_id, column.get_tenant_id()))) diff --git a/src/sql/CMakeLists.txt b/src/sql/CMakeLists.txt index dd0c596b2..dcccf6446 100644 --- a/src/sql/CMakeLists.txt +++ b/src/sql/CMakeLists.txt @@ -1347,6 +1347,7 @@ ob_set_subtarget(ob_sql rewrite rewrite/ob_union_find.cpp rewrite/ob_transform_decorrelate.cpp rewrite/ob_transform_late_materialization.cpp + rewrite/ob_transform_mv_rewrite_prepare.cpp ) ob_set_subtarget(ob_sql session @@ -1357,6 +1358,7 @@ ob_set_subtarget(ob_sql session session/ob_sys_params_mgr.cpp session/ob_user_resource_mgr.cpp session/ob_sess_info_verify.cpp + session/ob_local_session_var.cpp ) ob_set_subtarget(ob_sql udr diff --git a/src/sql/code_generator/ob_static_engine_cg.cpp b/src/sql/code_generator/ob_static_engine_cg.cpp index fd129c36b..5fe678e2c 100644 --- a/src/sql/code_generator/ob_static_engine_cg.cpp +++ b/src/sql/code_generator/ob_static_engine_cg.cpp @@ -160,6 +160,7 @@ #include "sql/engine/set/ob_hash_union_vec_op.h" #include "sql/engine/set/ob_hash_intersect_vec_op.h" #include "sql/engine/set/ob_hash_except_vec_op.h" +#include "sql/resolver/mv/ob_mv_provider.h" #ifdef OB_BUILD_TDE_SECURITY #include "share/ob_master_key_getter.h" #endif @@ -8353,6 +8354,9 @@ int ObStaticEngineCG::set_other_properties(const ObLogPlan &log_plan, ObPhysical || OB_ISNULL(exec_ctx->get_stmt_factory()->get_query_ctx())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid query_ctx", K(ret)); + } else if (my_session->get_ddl_info().is_refreshing_mview() + && OB_FAIL(check_refreshing_mview_session_var(*schema_guard, *my_session, log_plan.get_stmt()))) { + LOG_WARN("failed to check refreshing mview session var", K(ret)); } else { ret = phy_plan.set_params_info(*(log_plan.get_optimizer_context().get_params())); } @@ -8727,6 +8731,33 @@ int ObStaticEngineCG::set_other_properties(const ObLogPlan &log_plan, ObPhysical return ret; } +int ObStaticEngineCG::check_refreshing_mview_session_var(ObSchemaGetterGuard &schema_guard, + ObSQLSessionInfo &session, + const ObDMLStmt *dml_stmt) +{ + int ret = OB_SUCCESS; + bool is_same = false; + uint64_t mview_id = OB_INVALID_ID; + const share::schema::ObTableSchema *mview_schema = NULL; + const ObDelUpdStmt *del_up_stmt = dynamic_cast(dml_stmt); + bool is_vars_matched = false; + if (!session.get_ddl_info().is_refreshing_mview() || OB_ISNULL(del_up_stmt)) { + /* do nothing */ + } else if (OB_FAIL(del_up_stmt->get_modified_materialized_view_id(mview_id))) { + LOG_WARN("fail to get modified mview_id", K(ret), K(mview_id)); + } else if (OB_INVALID_ID == mview_id) { + /* do nothing */ + } else if (OB_FAIL(schema_guard.get_table_schema(session.get_effective_tenant_id(), mview_id, mview_schema))) { + LOG_WARN("fail to get mview schema", K(ret), K(mview_id)); + } else if (OB_ISNULL(mview_schema)) { + ret = OB_TABLE_NOT_EXIST; + LOG_WARN("fail to get mview schema", K(ret), K(mview_id)); + } else if (OB_FAIL(ObMVProvider::check_mview_dep_session_vars(*mview_schema, session, true, is_vars_matched))) { + LOG_WARN("failed to check mview dep session vars", K(ret)); + } + return ret; +} + // FIXME bin.lb: We should split the big switch case into logical operator class. int ObStaticEngineCG::get_phy_op_type(ObLogicalOperator &log_op, ObPhyOperatorType &type, diff --git a/src/sql/code_generator/ob_static_engine_cg.h b/src/sql/code_generator/ob_static_engine_cg.h index f029b4cb6..b5815f7f3 100644 --- a/src/sql/code_generator/ob_static_engine_cg.h +++ b/src/sql/code_generator/ob_static_engine_cg.h @@ -582,6 +582,9 @@ private: int extract_all_mview_ids(const ObIArray &exprs); int extract_all_mview_ids(const ObRawExpr *expr); int check_is_insert_overwrite_stmt(const ObLogPlan *plan, bool &is_insert_overwrite); + int check_refreshing_mview_session_var(ObSchemaGetterGuard &schema_guard, + ObSQLSessionInfo &session, + const ObDMLStmt *dml_stmt); private: struct BatchExecParamCache { BatchExecParamCache(ObExecParamRawExpr* expr, ObOpSpec* spec, bool is_left) diff --git a/src/sql/engine/cmd/ob_outline_executor.cpp b/src/sql/engine/cmd/ob_outline_executor.cpp index 6d9ece9a4..971a6e018 100644 --- a/src/sql/engine/cmd/ob_outline_executor.cpp +++ b/src/sql/engine/cmd/ob_outline_executor.cpp @@ -190,6 +190,10 @@ int ObOutlineExecutor::generate_logical_plan(ObExecContext &ctx, ObPhysicalPlan *phy_plan = NULL; ObOptimizer optimizer(opt_ctx); ObCacheObjGuard guard(OUTLINE_EXEC_HANDLE); + ObStmtNeedPrivs mock_stmt_need_privs; + ObStmtOraNeedPrivs mock_stmt_ora_need_privs; + mock_stmt_need_privs.need_privs_.set_allocator(&opt_ctx.get_allocator()); + mock_stmt_ora_need_privs.need_privs_.set_allocator(&opt_ctx.get_allocator()); if (OB_ISNULL(session_info) || OB_ISNULL(outline_stmt)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid parameter", K(session_info), K(outline_stmt)); @@ -211,7 +215,9 @@ int ObOutlineExecutor::generate_logical_plan(ObExecContext &ctx, &opt_ctx.get_local_server_addr(), phy_plan, ctx, - outline_stmt))) { + outline_stmt, + mock_stmt_need_privs, + mock_stmt_ora_need_privs))) { LOG_WARN("fail to transform outline stmt", K(ret)); } else if (FALSE_IT(opt_ctx.set_root_stmt(outline_stmt))) { /*do nothing*/ diff --git a/src/sql/engine/expr/ob_expr_operator.cpp b/src/sql/engine/expr/ob_expr_operator.cpp index 7543f9c53..ef46ef882 100644 --- a/src/sql/engine/expr/ob_expr_operator.cpp +++ b/src/sql/engine/expr/ob_expr_operator.cpp @@ -2480,12 +2480,12 @@ DEF_SET_LOCAL_SESSION_VARS(ObExprOperator, raw_expr) { } int ObExprOperator::add_local_var_to_expr(ObSysVarClassType var_type, - const share::schema::ObLocalSessionVar *local_session_var, + const ObLocalSessionVar *local_session_var, const ObBasicSessionInfo *session, - share::schema::ObLocalSessionVar &local_vars) + ObLocalSessionVar &local_vars) { int ret = OB_SUCCESS; - share::schema::ObSessionSysVar *sys_var = NULL; + ObSessionSysVar *sys_var = NULL; if (NULL != local_session_var) { if (OB_FAIL(local_session_var->get_local_var(var_type, sys_var))) { LOG_WARN("fail to get sys var", K(ret), K(var_type)); diff --git a/src/sql/engine/expr/ob_expr_operator.h b/src/sql/engine/expr/ob_expr_operator.h index 8df436ba3..618b47f80 100644 --- a/src/sql/engine/expr/ob_expr_operator.h +++ b/src/sql/engine/expr/ob_expr_operator.h @@ -40,7 +40,7 @@ #include "sql/engine/expr/ob_expr_extra_info_factory.h" #include "sql/engine/expr/ob_i_expr_extra_info.h" #include "lib/hash/ob_hashset.h" -#include "share/schema/ob_schema_struct.h" +#include "sql/session/ob_local_session_var.h" #define GET_EXPR_CTX(ClassType, ctx, id) static_cast((ctx).get_expr_op_ctx(id)) @@ -290,15 +290,15 @@ protected: #define DECLARE_SET_LOCAL_SESSION_VARS \ virtual int set_local_session_vars(ObRawExpr *raw_expr, \ - const share::schema::ObLocalSessionVar *local_session_vars, \ + const ObLocalSessionVar *local_session_vars, \ const ObBasicSessionInfo *session, \ - share::schema::ObLocalSessionVar &local_vars) + ObLocalSessionVar &local_vars) #define DEF_SET_LOCAL_SESSION_VARS(TypeName, raw_expr) \ int TypeName::set_local_session_vars(ObRawExpr *raw_expr, \ - const share::schema::ObLocalSessionVar *local_session_vars, \ + const ObLocalSessionVar *local_session_vars, \ const ObBasicSessionInfo *session, \ - share::schema::ObLocalSessionVar &local_vars) + ObLocalSessionVar &local_vars) class ObExprOperator : public common::ObDLinkBase { @@ -683,7 +683,7 @@ public: static int check_first_param_not_time(const common::ObIArray &exprs, bool ¬_time); //Extract the info of sys vars which need to be used in resolving or excuting into local_sys_vars. DECLARE_SET_LOCAL_SESSION_VARS; - static int add_local_var_to_expr(share::ObSysVarClassType var_type, const share::schema::ObLocalSessionVar *local_session_var, const ObBasicSessionInfo *session, share::schema::ObLocalSessionVar &local_vars); + static int add_local_var_to_expr(share::ObSysVarClassType var_type, const ObLocalSessionVar *local_session_var, const ObBasicSessionInfo *session, ObLocalSessionVar &local_vars); protected: ObExpr *get_rt_expr(const ObRawExpr &raw_expr) const; diff --git a/src/sql/engine/expr/ob_expr_util.h b/src/sql/engine/expr/ob_expr_util.h index 7656561c5..1b1b11c36 100644 --- a/src/sql/engine/expr/ob_expr_util.h +++ b/src/sql/engine/expr/ob_expr_util.h @@ -235,7 +235,7 @@ public: local_tz_wrap_(NULL) { } - ObSolidifiedVarsContext(share::schema::ObLocalSessionVar *local_var, common::ObIAllocator *alloc) : + ObSolidifiedVarsContext(ObLocalSessionVar *local_var, common::ObIAllocator *alloc) : local_session_var_(local_var), alloc_(alloc), local_tz_wrap_(NULL) @@ -250,10 +250,10 @@ public: } } int get_local_tz_info(const sql::ObBasicSessionInfo *session, const common::ObTimeZoneInfo *&tz_info); - share::schema::ObLocalSessionVar *get_local_vars() const { return local_session_var_; } + ObLocalSessionVar *get_local_vars() const { return local_session_var_; } DECLARE_TO_STRING; private: - share::schema::ObLocalSessionVar *local_session_var_; + ObLocalSessionVar *local_session_var_; common::ObIAllocator *alloc_; //cached vars ObTimeZoneInfoWrap *local_tz_wrap_; @@ -279,7 +279,7 @@ public: int get_max_allowed_packet(int64_t &max_size); int get_compat_version(uint64_t &compat_version); //get the specified solidified var - int get_local_var(share::ObSysVarClassType var_type, share::schema::ObSessionSysVar *&sys_var); + int get_local_var(share::ObSysVarClassType var_type, ObSessionSysVar *&sys_var); private: const ObSolidifiedVarsContext *local_session_var_; const ObBasicSessionInfo *session_; diff --git a/src/sql/ob_sql.cpp b/src/sql/ob_sql.cpp index a5da5af48..5dc3003b4 100644 --- a/src/sql/ob_sql.cpp +++ b/src/sql/ob_sql.cpp @@ -3446,7 +3446,9 @@ int ObSql::generate_plan(ParseResult &parse_result, &self_addr_, phy_plan, result.get_exec_context(), - stmt))) { //rewrite stmt + stmt, + stmt_need_privs, + stmt_ora_need_privs))) { //rewrite stmt LOG_WARN("Failed to transform stmt", K(ret)); } else if (OB_FAIL(generate_stmt_with_reconstruct_sql(stmt, pc_ctx, @@ -3571,6 +3573,10 @@ int ObSql::generate_stmt_with_reconstruct_sql(ObDMLStmt* &stmt, session->get_charsets4parser(), pc_ctx->def_name_ctx_); stmt->get_query_ctx()->global_dependency_tables_.reuse(); + ObStmtNeedPrivs mock_stmt_need_privs; + ObStmtOraNeedPrivs mock_stmt_ora_need_privs; + mock_stmt_need_privs.need_privs_.set_allocator(&pc_ctx->allocator_); + mock_stmt_ora_need_privs.need_privs_.set_allocator(&pc_ctx->allocator_); if (OB_FAIL(parser.parse(sql, parse_result))) { LOG_WARN("failed to parser sql", K(ret)); } else if (OB_FAIL(generate_stmt(parse_result, @@ -3591,6 +3597,8 @@ int ObSql::generate_stmt_with_reconstruct_sql(ObDMLStmt* &stmt, phy_plan, result.get_exec_context(), stmt, + mock_stmt_need_privs, + mock_stmt_ora_need_privs, true))) { //rewrite stmt LOG_WARN("Failed to transform stmt", K(ret)); } @@ -3719,6 +3727,8 @@ int ObSql::transform_stmt(ObSqlSchemaGuard *sql_schema_guard, ObPhysicalPlan *phy_plan, ObExecContext &exec_ctx, ObDMLStmt *&stmt, + ObStmtNeedPrivs &stmt_need_privs, + ObStmtOraNeedPrivs &stmt_ora_need_privs, bool ignore_trace_event) { int ret = OB_SUCCESS; @@ -3759,6 +3769,8 @@ int ObSql::transform_stmt(ObSqlSchemaGuard *sql_schema_guard, trans_ctx.opt_stat_mgr_ = opt_stat_mgr; trans_ctx.sql_schema_guard_ = sql_schema_guard; trans_ctx.self_addr_ = self_addr; + trans_ctx.stmt_need_privs_ = &stmt_need_privs; + trans_ctx.stmt_ora_need_privs_ = &stmt_ora_need_privs; // trans_ctx.merged_version_ = merged_version; trans_ctx.phy_plan_ = phy_plan; diff --git a/src/sql/ob_sql.h b/src/sql/ob_sql.h index 487d5652f..8078f5b2a 100644 --- a/src/sql/ob_sql.h +++ b/src/sql/ob_sql.h @@ -177,6 +177,8 @@ public: ObPhysicalPlan *phy_plan, ObExecContext &exec_ctx, ObDMLStmt *&stmt, + ObStmtNeedPrivs &stmt_need_privs, + ObStmtOraNeedPrivs &stmt_ora_need_privs, bool ignore_trace_event=false); //optimize stmt, generate logical_plan diff --git a/src/sql/ob_sql_utils.cpp b/src/sql/ob_sql_utils.cpp index 16229f1a4..8f9b784bc 100644 --- a/src/sql/ob_sql_utils.cpp +++ b/src/sql/ob_sql_utils.cpp @@ -3295,7 +3295,7 @@ int ObSQLUtils::wrap_column_convert_ctx(const ObExprCtx &expr_ctx, ObCastCtx &co return ret; } -int ObSQLUtils::merge_solidified_var_into_collation(const share::schema::ObLocalSessionVar &session_vars_snapshot, +int ObSQLUtils::merge_solidified_var_into_collation(const ObLocalSessionVar &session_vars_snapshot, ObCollationType &cs_type) { int ret = OB_SUCCESS; if (OB_SUCC(ret) && lib::is_mysql_mode()) { @@ -3369,7 +3369,7 @@ int ObSQLUtils::merge_solidified_vars_into_type_ctx(ObExprTypeCtx &type_ctx, return ret; } -int ObSQLUtils::merge_solidified_var_into_dtc_params(const share::schema::ObLocalSessionVar *local_vars, +int ObSQLUtils::merge_solidified_var_into_dtc_params(const ObLocalSessionVar *local_vars, const ObTimeZoneInfo *local_timezone, ObDataTypeCastParams &dtc_param) { @@ -3405,7 +3405,7 @@ int ObSQLUtils::merge_solidified_var_into_dtc_params(const share::schema::ObLoca return ret; } -int ObSQLUtils::merge_solidified_var_into_sql_mode(const share::schema::ObLocalSessionVar *local_vars, +int ObSQLUtils::merge_solidified_var_into_sql_mode(const ObLocalSessionVar *local_vars, ObSQLMode &sql_mode) { int ret = OB_SUCCESS; ObSessionSysVar *local_var = NULL; @@ -3426,7 +3426,7 @@ int ObSQLUtils::merge_solidified_var_into_sql_mode(const share::schema::ObLocalS return ret; } -int ObSQLUtils::merge_solidified_var_into_max_allowed_packet(const share::schema::ObLocalSessionVar *local_vars, +int ObSQLUtils::merge_solidified_var_into_max_allowed_packet(const ObLocalSessionVar *local_vars, int64_t &max_allowed_packet) { int ret = OB_SUCCESS; @@ -3446,7 +3446,7 @@ int ObSQLUtils::merge_solidified_var_into_max_allowed_packet(const share::schema return ret; } -int ObSQLUtils::merge_solidified_var_into_compat_version(const share::schema::ObLocalSessionVar *local_vars, +int ObSQLUtils::merge_solidified_var_into_compat_version(const ObLocalSessionVar *local_vars, uint64_t &compat_version) { int ret = OB_SUCCESS; diff --git a/src/sql/ob_sql_utils.h b/src/sql/ob_sql_utils.h index 78fee24a4..52618c7cf 100644 --- a/src/sql/ob_sql_utils.h +++ b/src/sql/ob_sql_utils.h @@ -520,17 +520,17 @@ public: static void init_type_ctx(const ObSQLSessionInfo *session, ObExprTypeCtx &type_ctx); static int merge_solidified_vars_into_type_ctx(ObExprTypeCtx &type_ctx, - const share::schema::ObLocalSessionVar &session_vars_snapshot); - static int merge_solidified_var_into_dtc_params(const share::schema::ObLocalSessionVar *local_vars, + const ObLocalSessionVar &session_vars_snapshot); + static int merge_solidified_var_into_dtc_params(const ObLocalSessionVar *local_vars, const ObTimeZoneInfo *local_timezone, ObDataTypeCastParams &dtc_param); - static int merge_solidified_var_into_sql_mode(const share::schema::ObLocalSessionVar *local_vars, + static int merge_solidified_var_into_sql_mode(const ObLocalSessionVar *local_vars, ObSQLMode &sql_mode); - static int merge_solidified_var_into_collation(const share::schema::ObLocalSessionVar &session_vars_snapshot, + static int merge_solidified_var_into_collation(const ObLocalSessionVar &session_vars_snapshot, ObCollationType &cs_type); - static int merge_solidified_var_into_max_allowed_packet(const share::schema::ObLocalSessionVar *local_vars, + static int merge_solidified_var_into_max_allowed_packet(const ObLocalSessionVar *local_vars, int64_t &max_allowed_packet); - static int merge_solidified_var_into_compat_version(const share::schema::ObLocalSessionVar *local_vars, + static int merge_solidified_var_into_compat_version(const ObLocalSessionVar *local_vars, uint64_t &compat_version); static bool is_oracle_sys_view(const ObString &table_name); diff --git a/src/sql/optimizer/ob_conflict_detector.cpp b/src/sql/optimizer/ob_conflict_detector.cpp index 6b108f861..e7b344db0 100644 --- a/src/sql/optimizer/ob_conflict_detector.cpp +++ b/src/sql/optimizer/ob_conflict_detector.cpp @@ -274,28 +274,187 @@ int ObConflictDetector::check_join_legal(const ObRelIds &left_set, return ret; } +/** + * 为左右连接树选择合法的连接条件 + * is_strict_order： + * true：left join right是合法的，right join left是非法的 + * false：left join right是非法的，right join left是合法的 + */ +int ObConflictDetector::choose_detectors(ObRelIds &left_tables, + ObRelIds &right_tables, + ObIArray &left_used_detectors, + ObIArray &right_used_detectors, + ObIArray &table_depend_infos, + ObIArray &all_detectors, + ObIArray &valid_detectors, + bool delay_cross_product, + bool &is_strict_order) +{ + int ret = OB_SUCCESS; + bool is_legal = false; + ObRelIds combined_relids; + if (OB_FAIL(combined_relids.add_members(left_tables))) { + LOG_WARN("failed to add left relids into combined relids", K(ret)); + } else if (OB_FAIL(combined_relids.add_members(right_tables))) { + LOG_WARN("failed to add right relids into combined relids", K(ret)); + } + is_strict_order = true; + for (int64_t i = 0; OB_SUCC(ret) && i < all_detectors.count(); ++i) { + ObConflictDetector *detector = all_detectors.at(i); + bool is_used = false; + if (OB_ISNULL(detector)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("conflict detector is null", K(ret)); + } else if (INNER_JOIN == detector->get_join_info().join_type_ && + detector->get_join_info().where_conditions_.empty()) { + // 笛卡尔积的冲突检测器可以重复使用 + } else if (ObOptimizerUtil::find_item(left_used_detectors, detector)) { + is_used = true; + } else if (ObOptimizerUtil::find_item(right_used_detectors, detector)) { + is_used = true; + } + if (OB_FAIL(ret) || is_used) { + // do nothing + } else if (OB_FAIL(detector->check_join_legal(left_tables, + right_tables, + combined_relids, + delay_cross_product, + table_depend_infos, + is_legal))) { + LOG_WARN("failed to check join legal", K(ret)); + } else if (!is_legal) { + //对于可交换的join如inner join，既left join right合法 + //也right join left合法，但是我们只需要保存left inner join right + //因为在generate join path的时候会生成right inner join left的path + //并且不可能存在一种join，既有left join1 right合法，又有right join2 left合法 + LOG_TRACE("left tree join right tree is not legal", K(left_tables), + K(right_tables), K(*detector)); + //尝试right tree join left tree + if (OB_FAIL(detector->check_join_legal(right_tables, + left_tables, + combined_relids, + delay_cross_product, + table_depend_infos, + is_legal))) { + LOG_WARN("failed to check join legal", K(ret)); + } else if (!is_legal) { + LOG_TRACE("right tree join left tree is not legal", K(right_tables), + K(left_tables), K(*detector)); + } else if (OB_FAIL(valid_detectors.push_back(detector))) { + LOG_WARN("failed to push back detector", K(ret)); + } else { + is_strict_order = false; + LOG_TRACE("succeed to find join info for ", K(left_tables), + K(right_tables), KPC(detector)); + } + } else if (OB_FAIL(valid_detectors.push_back(detector))) { + LOG_WARN("failed to push back detector", K(ret)); + } else { + LOG_TRACE("succeed to find join info for ", K(left_tables), + K(right_tables), KPC(detector)); + } + } + return ret; +} + +/** + * 只允许多个inner join叠加成一个join info + * 例如：select * from A, B where A.c1 = B.c1 and A.c2 = B.c2 + * 或者单个OUTER JOIN加上多个inner join info，inner join info作为join qual + * 例如：select * from A left join B on A.c1 = B.c1 where A.c2 = B.c2 + */ +int ObConflictDetector::check_join_info(const ObIArray &valid_detectors, + ObJoinType &join_type, + bool &is_valid) +{ + int ret = OB_SUCCESS; + ObConflictDetector *detector = NULL; + bool has_non_inner_join = false; + is_valid = true; + join_type = INNER_JOIN; + if (valid_detectors.empty()) { + is_valid = false; + } + for (int64_t i = 0; OB_SUCC(ret) && is_valid && i < valid_detectors.count(); ++i) { + detector = valid_detectors.at(i); + if (OB_ISNULL(detector)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect null detectors", K(ret)); + } else if (INNER_JOIN == detector->get_join_info().join_type_) { + //do nothing + } else if (has_non_inner_join) { + //不允许出现多个非inner join的join info + is_valid = false; + } else { + has_non_inner_join = true; + join_type = detector->get_join_info().join_type_; + } + } + return ret; +} + +int ObConflictDetector::merge_join_info(const ObIArray &valid_detectors, + JoinInfo &join_info) +{ + int ret = OB_SUCCESS; + bool is_valid = false; + if (OB_FAIL(check_join_info(valid_detectors, + join_info.join_type_, + is_valid))) { + LOG_WARN("failed to check join info", K(ret)); + } else if (!is_valid) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect different join info", K(valid_detectors), K(ret)); + } else { + ObConflictDetector *detector = NULL; + for (int64_t i = 0; OB_SUCC(ret) && i < valid_detectors.count(); ++i) { + detector = valid_detectors.at(i); + if (OB_ISNULL(detector)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect null detectors", K(ret)); + } else if (OB_FAIL(join_info.table_set_.add_members(detector->get_join_info().table_set_))) { + LOG_WARN("failed to add members", K(ret)); + } else if (OB_FAIL(append_array_no_dup(join_info.where_conditions_, detector->get_join_info().where_conditions_))) { + LOG_WARN("failed to append exprs", K(ret)); + } else if (INNER_JOIN == join_info.join_type_ && + OB_FAIL(append_array_no_dup(join_info.where_conditions_, detector->get_join_info().on_conditions_))) { + LOG_WARN("failed to append exprs", K(ret)); + } else if (INNER_JOIN != join_info.join_type_ && + OB_FAIL(append_array_no_dup(join_info.on_conditions_, detector->get_join_info().on_conditions_))) { + LOG_WARN("failed to append exprs", K(ret)); + } + } + } + return ret; +} int ObConflictDetectorGenerator::generate_conflict_detectors(const ObDMLStmt *stmt, const ObIArray &table_items, const ObIArray &semi_infos, - ObIArray &quals, - ObIArray &baserels, + const ObIArray &quals, + ObIArray> &baserel_filters, ObIArray &conflict_detectors) { int ret = OB_SUCCESS; ObRelIds table_ids; ObSEArray semi_join_detectors; ObSEArray inner_join_detectors; + ObSEArray new_quals; LOG_TRACE("start to generate conflict detector", K(table_items), K(semi_infos), K(quals)); if (OB_ISNULL(stmt)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpect null stmt", K(ret)); } else if (OB_FAIL(stmt->get_table_rel_ids(table_items, table_ids))) { LOG_WARN("failed to get table ids", K(ret)); + } else if (OB_FALSE_IT(baserel_filters.reuse())) { + } else if (OB_FAIL(baserel_filters.prepare_allocate(stmt->get_table_size()))) { + LOG_WARN("failed to prepare allocate base rel filters", K(ret)); + } else if (OB_FAIL(new_quals.assign(quals))) { + LOG_WARN("failed to assign quals", K(ret)); } else if (OB_FAIL(generate_inner_join_detectors(stmt, table_items, - quals, - baserels, + new_quals, + baserel_filters, inner_join_detectors))) { LOG_WARN("failed to generate inner join detectors", K(ret)); } else if (OB_FAIL(generate_semi_join_detectors(stmt, @@ -566,7 +725,7 @@ int ObConflictDetectorGenerator::generate_semi_join_detectors(const ObDMLStmt *s int ObConflictDetectorGenerator::generate_inner_join_detectors(const ObDMLStmt *stmt, const ObIArray &table_items, ObIArray &quals, - ObIArray &baserels, + ObIArray> &baserel_filters, ObIArray &inner_join_detectors) { int ret = OB_SUCCESS; @@ -589,17 +748,15 @@ int ObConflictDetectorGenerator::generate_inner_join_detectors(const ObDMLStmt * LOG_WARN("failed to split or quals", K(ret)); } else if (OB_FAIL(stmt->get_table_rel_ids(table_items, all_table_ids))) { LOG_WARN("failed to get table ids", K(ret)); - } else if (OB_FAIL(deduce_redundant_join_conds(stmt, - quals, - table_items, - redundant_quals))) { + } else if (should_deduce_conds_ && OB_FAIL(deduce_redundant_join_conds(stmt, + quals, + table_items, + redundant_quals))) { LOG_WARN("failed to deduce redundancy quals", K(ret)); } else if (OB_FAIL(all_quals.assign(quals))) { LOG_WARN("failed to assign array", K(ret)); } else if (OB_FAIL(append(all_quals, redundant_quals))) { LOG_WARN("failed to append array", K(ret)); - } else { - OPT_TRACE("deduce redundant qual:", redundant_quals); } //1. 生成单个table item内部的冲突规则 for (int64_t i = 0; OB_SUCC(ret) && i < table_items.count(); ++i) { @@ -634,7 +791,7 @@ int ObConflictDetectorGenerator::generate_inner_join_detectors(const ObDMLStmt * if (OB_FAIL(generate_outer_join_detectors(stmt, table_items.at(i), table_filters, - baserels, + baserel_filters, outer_join_detectors))) { LOG_WARN("failed to generate outer join detectors", K(ret)); } @@ -769,7 +926,7 @@ int ObConflictDetectorGenerator::generate_inner_join_detectors(const ObDMLStmt * int ObConflictDetectorGenerator::generate_outer_join_detectors(const ObDMLStmt *stmt, TableItem *table_item, ObIArray &table_filter, - ObIArray &baserels, + ObIArray> &baserel_filters, ObIArray &outer_join_detectors) { int ret = OB_SUCCESS; @@ -779,7 +936,7 @@ int ObConflictDetectorGenerator::generate_outer_join_detectors(const ObDMLStmt * LOG_WARN("unexpect null table item", K(ret)); } else if (!table_item->is_joined_table()) { //如果是基表，直接把过程谓词分发到join order里 - if (OB_FAIL(distribute_quals(stmt, table_item, table_filter, baserels))) { + if (OB_FAIL(distribute_quals(stmt, table_item, table_filter, baserel_filters))) { LOG_WARN("failed to distribute table filter", K(ret)); } } else if (INNER_JOIN == joined_table->joined_type_) { @@ -789,10 +946,10 @@ int ObConflictDetectorGenerator::generate_outer_join_detectors(const ObDMLStmt * if (OB_FAIL(flatten_inner_join(table_item, table_filter, table_items))) { LOG_WARN("failed to flatten inner join", K(ret)); } else if (OB_FAIL(SMART_CALL(generate_inner_join_detectors(stmt, - table_items, - table_filter, - baserels, - detectors)))) { + table_items, + table_filter, + baserel_filters, + detectors)))) { LOG_WARN("failed to generate inner join detectors", K(ret)); } else if (OB_FAIL(append(outer_join_detectors, detectors))) { LOG_WARN("failed to append detectors", K(ret)); @@ -800,7 +957,7 @@ int ObConflictDetectorGenerator::generate_outer_join_detectors(const ObDMLStmt * } else if (OB_FAIL(inner_generate_outer_join_detectors(stmt, joined_table, table_filter, - baserels, + baserel_filters, outer_join_detectors))) { LOG_WARN("failed to generate outer join detectors", K(ret)); } @@ -810,11 +967,10 @@ int ObConflictDetectorGenerator::generate_outer_join_detectors(const ObDMLStmt * int ObConflictDetectorGenerator::distribute_quals(const ObDMLStmt *stmt, TableItem *table_item, const ObIArray &table_filter, - ObIArray &baserels) + ObIArray> &baserel_filters) { int ret = OB_SUCCESS; - ObJoinOrder *cur_rel = NULL; - ObSEArray relids; + ObSEArray relids; ObRelIds table_ids; if (OB_ISNULL(stmt) || OB_ISNULL(table_item)) { ret = OB_ERR_UNEXPECTED; @@ -823,16 +979,14 @@ int ObConflictDetectorGenerator::distribute_quals(const ObDMLStmt *stmt, LOG_WARN("failed to get table ids", K(ret)); } else if (OB_FAIL(table_ids.to_array(relids))) { LOG_WARN("to_array error", K(ret)); - } else if (1 != relids.count()) { + } else if (OB_UNLIKELY(1 != relids.count())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("expect basic table item", K(ret)); - } else if (OB_FAIL(find_base_rel(baserels, relids.at(0), cur_rel))) { - LOG_WARN("find_base_rel fails", K(ret)); - } else if (OB_ISNULL(cur_rel)) { - ret = OB_SQL_OPT_ERROR; - LOG_WARN("failed to distribute qual to rel", K(baserels), K(relids), K(ret)); - } else if (OB_FAIL(append(cur_rel->get_restrict_infos(), table_filter))) { - LOG_WARN("failed to distribute qual to rel", K(ret)); + } else if (OB_UNLIKELY(relids.at(0) < 1 || relids.at(0) > baserel_filters.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected rel id", K(ret), K(relids.at(0)), K(baserel_filters.count())); + } else if (OB_FAIL(append(baserel_filters.at(relids.at(0) - 1), table_filter))) { + LOG_WARN("failed to append table filters", K(ret)); } return ret; } @@ -864,6 +1018,9 @@ int ObConflictDetectorGenerator::flatten_inner_join(TableItem *table_item, onetime_copier_, session_info_))) { LOG_WARN("failed to adjust join conditions with onetime", K(ret)); + } else if (NULL == onetime_copier_ + && OB_FAIL(new_conditions.assign(joined_table->join_conditions_))) { + LOG_WARN("failed to assign join conditions", K(ret)); } else if (OB_FAIL(append(table_filter, new_conditions))) { LOG_WARN("failed to append exprs", K(ret)); } @@ -873,7 +1030,7 @@ int ObConflictDetectorGenerator::flatten_inner_join(TableItem *table_item, int ObConflictDetectorGenerator::inner_generate_outer_join_detectors(const ObDMLStmt *stmt, JoinedTable *joined_table, ObIArray &table_filter, - ObIArray &baserels, + ObIArray> &baserel_filters, ObIArray &outer_join_detectors) { int ret = OB_SUCCESS; @@ -908,10 +1065,10 @@ int ObConflictDetectorGenerator::inner_generate_outer_join_detectors(const ObDML LOG_WARN("failed to pushdown on conditions", K(ret)); //3. generate left child detectors } else if (OB_FAIL(SMART_CALL(generate_outer_join_detectors(stmt, - joined_table->left_table_, - left_quals, - baserels, - left_detectors)))) { + joined_table->left_table_, + left_quals, + baserel_filters, + left_detectors)))) { LOG_WARN("failed to generate outer join detectors", K(ret)); } else if (OB_FAIL(append(outer_join_detectors, left_detectors))) { LOG_WARN("failed to append detectors", K(ret)); @@ -919,7 +1076,7 @@ int ObConflictDetectorGenerator::inner_generate_outer_join_detectors(const ObDML } else if (OB_FAIL(SMART_CALL(generate_outer_join_detectors(stmt, joined_table->right_table_, right_quals, - baserels, + baserel_filters, right_detectors)))) { LOG_WARN("failed to generate outer join detectors", K(ret)); } else if (OB_FAIL(append(outer_join_detectors, right_detectors))) { @@ -1143,6 +1300,9 @@ int ObConflictDetectorGenerator::pushdown_on_conditions(const ObDMLStmt *stmt, onetime_copier_, session_info_))) { LOG_WARN("failed to adjust join conditions with onetime", K(ret)); + } else if (NULL == onetime_copier_ + && OB_FAIL(new_conditions.assign(joined_table->join_conditions_))) { + LOG_WARN("failed to assign join conditions", K(ret)); } else if (OB_UNLIKELY(new_conditions.count() != N)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("expr count mismatch", K(ret)); @@ -1490,6 +1650,9 @@ int ObConflictDetectorGenerator::deduce_redundant_join_conds(const ObDMLStmt *st LOG_WARN("failed to deduce redundancy quals with equal set", K(ret)); } } + if (OB_SUCC(ret)) { + OPT_TRACE("deduce redundant qual:", redundant_quals); + } return ret; } @@ -1575,32 +1738,10 @@ bool ObConflictDetectorGenerator::has_depend_table(const ObRelIds& table_ids) { bool b_ret = false; for (int64_t i = 0; !b_ret && i < table_depend_infos_.count(); ++i) { - TableDependInfo &info = table_depend_infos_.at(i); + const TableDependInfo &info = table_depend_infos_.at(i); if (table_ids.has_member(info.table_idx_)) { b_ret = true; } } return b_ret; -} - -int ObConflictDetectorGenerator::find_base_rel(ObIArray &base_level, - int64_t table_idx, - ObJoinOrder *&base_rel) -{ - int ret = OB_SUCCESS; - ObJoinOrder *cur_rel = NULL; - bool find = false; - base_rel = NULL; - for (int64_t i = 0; OB_SUCC(ret) && !find && i < base_level.count(); ++i) { - //如果是OJ，这里table_set_可能不止一项，所以不能认为cur_rel->table_set_.num_members() == 1 - if (OB_ISNULL(cur_rel = base_level.at(i))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(cur_rel)); - } else if (cur_rel->get_tables().has_member(table_idx)){ - find = true; - base_rel = cur_rel; - LOG_TRACE("succeed to find base rel", K(cur_rel->get_tables()), K(table_idx)); - } else { /* do nothing */ } - } - return ret; } \ No newline at end of file diff --git a/src/sql/optimizer/ob_conflict_detector.h b/src/sql/optimizer/ob_conflict_detector.h index 667d67e98..5df0f61d7 100644 --- a/src/sql/optimizer/ob_conflict_detector.h +++ b/src/sql/optimizer/ob_conflict_detector.h @@ -56,7 +56,7 @@ struct JoinInfo ObRelIds table_set_; //要连接的表集合（即包含在join_qual_中的，除自己之外的所有表） common::ObSEArray on_conditions_; //来自on的条件，如果是outer join common::ObSEArray where_conditions_; //来自where的条件，如果是outer join，则是join filter，如果是inner join，则是join condition - common::ObSEArray equal_join_conditions_;//是连接条件（outer的on condition，inner join的where condition）的子集，仅简单等值，在预测未来的mergejoin所需的序的时候使用 + common::ObSEArray equal_join_conditions_;//是连接条件（outer的on condition，inner join的where condition）的子集，仅简单等值，在预测未来的mergejoin所需的序的时候使用 ObJoinType join_type_; }; @@ -111,6 +111,23 @@ public: const ObConflictDetector &right, bool &is_satisfy); + static int choose_detectors(ObRelIds &left_tables, + ObRelIds &right_tables, + ObIArray &left_used_detectors, + ObIArray &right_used_detectors, + ObIArray &table_depend_infos, + ObIArray &all_detectors, + ObIArray &valid_detectors, + bool delay_cross_product, + bool &is_strict_order); + + static int check_join_info(const ObIArray &valid_detectors, + ObJoinType &join_type, + bool &is_valid); + + static int merge_join_info(const ObIArray &valid_detectors, + JoinInfo &join_info); + int check_join_legal(const ObRelIds &left_set, const ObRelIds &right_set, const ObRelIds &combined_set, @@ -148,13 +165,15 @@ public: ObRawExprFactory &expr_factory, ObSQLSessionInfo *session_info, ObRawExprCopier *onetime_copier, - common::ObIArray &table_depend_infos, + bool should_deduce_conds, + const common::ObIArray &table_depend_infos, common::ObIArray &bushy_tree_infos, common::ObIArray &new_or_quals) : allocator_(allocator), expr_factory_(expr_factory), session_info_(session_info), onetime_copier_(onetime_copier), + should_deduce_conds_(should_deduce_conds), table_depend_infos_(table_depend_infos), bushy_tree_infos_(bushy_tree_infos), new_or_quals_(new_or_quals) @@ -166,13 +185,10 @@ public: int generate_conflict_detectors(const ObDMLStmt *stmt, const ObIArray &table_items, const ObIArray &semi_infos, - ObIArray &quals, - ObIArray &baserels, + const ObIArray &quals, + ObIArray> &baserel_filters, ObIArray &conflict_detectors); - inline common::ObIArray &get_new_or_quals() {return new_or_quals_;} - inline common::ObIArray &get_bushy_tree_infos() {return bushy_tree_infos_;} - private: int add_conflict_rule(const ObRelIds &left, const ObRelIds &right, @@ -192,19 +208,19 @@ private: int generate_inner_join_detectors(const ObDMLStmt *stmt, const ObIArray &table_items, ObIArray &quals, - ObIArray &baserels, + ObIArray> &baserel_filters, ObIArray &inner_join_detectors); int generate_outer_join_detectors(const ObDMLStmt *stmt, TableItem *table_item, ObIArray &table_filter, - ObIArray &baserels, + ObIArray> &baserel_filters, ObIArray &outer_join_detectors); int distribute_quals(const ObDMLStmt *stmt, TableItem *table_item, const ObIArray &table_filter, - ObIArray &baserels); + ObIArray> &baserel_filters); int flatten_inner_join(TableItem *table_item, ObIArray &table_filter, @@ -213,7 +229,7 @@ private: int inner_generate_outer_join_detectors(const ObDMLStmt *stmt, JoinedTable *joined_table, ObIArray &table_filter, - ObIArray &baserels, + ObIArray> &baserel_filters, ObIArray &outer_join_detectors); int pushdown_where_filters(const ObDMLStmt *stmt, @@ -259,14 +275,13 @@ private: bool has_depend_table(const ObRelIds& table_ids); - int find_base_rel(ObIArray &base_level, int64_t table_idx, ObJoinOrder *&base_rel); - private: common::ObIAllocator &allocator_; ObRawExprFactory &expr_factory_; ObSQLSessionInfo *session_info_; ObRawExprCopier *onetime_copier_; - common::ObIArray &table_depend_infos_; + bool should_deduce_conds_; + const common::ObIArray &table_depend_infos_; common::ObIArray &bushy_tree_infos_; common::ObIArray &new_or_quals_; }; diff --git a/src/sql/optimizer/ob_log_plan.cpp b/src/sql/optimizer/ob_log_plan.cpp index 221a815de..ee136441e 100644 --- a/src/sql/optimizer/ob_log_plan.cpp +++ b/src/sql/optimizer/ob_log_plan.cpp @@ -211,36 +211,6 @@ int64_t ObLogPlan::to_string(char *buf, return 0; } -/* - * 找出from items中对应的table items - */ -int ObLogPlan::get_from_table_items(const ObIArray &from_items, - ObIArray &table_items) -{ - int ret = OB_SUCCESS; - const ObDMLStmt *stmt = NULL; - if (OB_ISNULL(stmt = get_stmt())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("null stmt", K(stmt), K(ret)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < from_items.count(); i++) { - const FromItem &from_item = from_items.at(i); - TableItem *temp_table_item = NULL; - if (!from_item.is_joined_) { - //如果是基表或SubQuery - temp_table_item = stmt->get_table_item_by_id(from_item.table_id_); - } else { - //如果是Joined table - temp_table_item = stmt->get_joined_table(from_item.table_id_); - } - if (OB_FAIL(table_items.push_back(temp_table_item))) { - LOG_WARN("failed to push back table item", K(ret)); - } else { /*do nothing*/ } - } - } - return ret; -} - int ObLogPlan::get_base_table_items(const ObDMLStmt &stmt, const ObIArray &table_items, const ObIArray &semi_infos, @@ -290,14 +260,15 @@ int ObLogPlan::generate_join_orders() ObSEArray quals; ObSEArray from_table_items; ObSEArray base_table_items; + ObSEArray, 8> baserel_filters; JoinOrderArray base_level; int64_t join_level = 0; common::ObArray join_rels; if (OB_ISNULL(stmt = get_stmt())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected NULL", K(stmt), K(ret)); - } else if (OB_FAIL(get_from_table_items(stmt->get_from_items(), from_table_items))) { - LOG_WARN("failed to get table items", K(ret)); + } else if (OB_FAIL(stmt->get_from_tables(from_table_items))) { + LOG_WARN("failed to get table items", K(ret)); } else if (OB_FAIL(get_base_table_items(*stmt, from_table_items, stmt->get_semi_infos(), @@ -314,6 +285,7 @@ int ObLogPlan::generate_join_orders() get_optimizer_context().get_expr_factory(), get_optimizer_context().get_session_info(), onetime_copier_, + true, table_depend_infos_, bushy_tree_infos_, new_or_quals_); @@ -337,9 +309,11 @@ int ObLogPlan::generate_join_orders() from_table_items, stmt->get_semi_infos(), quals, - base_level, + baserel_filters, conflict_detectors_))) { LOG_WARN("failed to generate conflict detectors", K(ret)); + } else if (OB_FAIL(distribute_filters_to_baserels(base_level, baserel_filters))) { + LOG_WARN("failed to distribute filters to baserels", K(ret)); } else { //初始化动规数据结构 join_level = base_level.count(); //需要连接的层次数 @@ -413,6 +387,31 @@ int ObLogPlan::generate_join_orders() return ret; } +int ObLogPlan::distribute_filters_to_baserels(ObIArray &base_level, + ObIArray> &baserel_filters) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < base_level.count(); ++i) { + ObJoinOrder *cur_rel= base_level.at(i); + ObSEArray rel_id; + if (OB_ISNULL(cur_rel)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(i)); + } else if (OB_FAIL(cur_rel->get_tables().to_array(rel_id))) { + LOG_WARN("failed to get rel id", K(ret)); + } else if (OB_UNLIKELY(1 != rel_id.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected rel id count", K(ret), K(rel_id.count())); + } else if (OB_UNLIKELY(rel_id.at(0) < 1 || rel_id.at(0) > baserel_filters.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected rel id", K(ret), K(rel_id.at(0)), K(baserel_filters.count())); + } else if (OB_FAIL(append(cur_rel->get_restrict_infos(), baserel_filters.at(rel_id.at(0) - 1)))) { + LOG_WARN("failed to append restrict infos", K(ret)); + } + } + return ret; +} + int ObLogPlan::prepare_ordermap_pathset(const JoinOrderArray base_level) { int ret = OB_SUCCESS; @@ -1488,7 +1487,7 @@ int ObLogPlan::generate_join_levels_with_orgleading(common::ObIArrayget_from_items(), table_items))) { + } else if (OB_FAIL(stmt->get_from_tables(table_items))) { LOG_WARN("failed to get table items", K(ret)); } for (int64_t i = 0; OB_SUCC(ret) && i < stmt->get_semi_infos().count(); ++i) { @@ -2021,11 +2020,15 @@ int ObLogPlan::inner_generate_join_order(ObIArray &join_rels, is_valid_join = true; OPT_TRACE(left_tree, right_tree, " is legal, and has join path cache, no need to generate join path"); - } else if (OB_FAIL(choose_join_info(left_tree, - right_tree, - valid_detectors, - delay_cross_product, - is_strict_order))) { + } else if (OB_FAIL(ObConflictDetector::choose_detectors(left_tree->get_tables(), + right_tree->get_tables(), + left_tree->get_conflict_detectors(), + right_tree->get_conflict_detectors(), + table_depend_infos_, + conflict_detectors_, + valid_detectors, + delay_cross_product, + is_strict_order))) { LOG_WARN("failed to choose join info", K(ret)); } else if (valid_detectors.empty()) { OPT_TRACE("there is no valid join condition for ", left_tree, "join", right_tree); @@ -2037,14 +2040,14 @@ int ObLogPlan::inner_generate_join_order(ObIArray &join_rels, valid_detectors, join_tree, is_detector_valid))) { - LOG_WARN("failed to check detector valid", K(ret)); - } else if (!is_detector_valid) { - OPT_TRACE("join tree will be remove: ", left_tree, "join", right_tree); - } else if (OB_FAIL(merge_join_info(left_tree, - right_tree, - valid_detectors, - join_info))) { + LOG_WARN("failed to check detector valid", K(ret)); + } else if (!is_detector_valid) { + OPT_TRACE("join tree will be remove: ", left_tree, "join", right_tree); + } else if (OB_FAIL(ObConflictDetector::merge_join_info(valid_detectors, + join_info))) { LOG_WARN("failed to merge join info", K(ret)); + } else if (OB_FAIL(process_join_pred(left_tree, right_tree, join_info))) { + LOG_WARN("failed to preocess join pred", K(ret)); } else if (NULL != join_tree && level <= 1 && !hint_force_order) { //level==1的时候，左右树都是单表，如果已经生成过AB的话，BA的path也已经生成了，没必要再次生成一遍BA is_valid_join = true; @@ -2103,215 +2106,6 @@ int ObLogPlan::inner_generate_join_order(ObIArray &join_rels, return ret; } -int ObLogPlan::is_detector_used(ObJoinOrder *left_tree, - ObJoinOrder *right_tree, - ObConflictDetector *detector, - bool &is_used) -{ - int ret = OB_SUCCESS; - is_used = false; - if (OB_ISNULL(left_tree) || OB_ISNULL(right_tree) || OB_ISNULL(detector)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect null param", K(ret)); - } else if (INNER_JOIN == detector->get_join_info().join_type_ && - detector->get_join_info().where_conditions_.empty()) { - //笛卡尔积的冲突检测器可以重复使用 - }else if (ObOptimizerUtil::find_item(left_tree->get_conflict_detectors(), detector)) { - is_used = true; - } else if (ObOptimizerUtil::find_item(right_tree->get_conflict_detectors(), detector)) { - is_used = true; - } - return ret; -} - -/** - * 为左右连接树选择合法的连接条件 - * is_strict_order： - * true：left join right是合法的，right join left是非法的 - * false：left join right是非法的，right join left是合法的 - */ -int ObLogPlan::choose_join_info(ObJoinOrder *left_tree, - ObJoinOrder *right_tree, - ObIArray &valid_detectors, - bool delay_cross_product, - bool &is_strict_order) -{ - int ret = OB_SUCCESS; - bool is_legal = false; - ObRelIds combined_relids; - if (OB_ISNULL(left_tree) || OB_ISNULL(right_tree)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect empty conflict detectors", K(ret)); - } else if (OB_FAIL(combined_relids.add_members(left_tree->get_tables()))) { - LOG_WARN("failed to add left relids into combined relids", K(ret)); - } else if (OB_FAIL(combined_relids.add_members(right_tree->get_tables()))) { - LOG_WARN("failed to add right relids into combined relids", K(ret)); - } - is_strict_order = true; - for (int64_t i = 0; OB_SUCC(ret) && i < conflict_detectors_.count(); ++i) { - ObConflictDetector *detector = conflict_detectors_.at(i); - bool is_used = false; - if (OB_ISNULL(detector)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("conflict detector is null", K(ret)); - } else if (OB_FAIL(is_detector_used(left_tree, - right_tree, - detector, - is_used))) { - LOG_WARN("failed to check detector is used", K(ret)); - } else if (is_used) { - //do nothing - } else if (OB_FAIL(detector->check_join_legal(left_tree->get_tables(), - right_tree->get_tables(), - combined_relids, - delay_cross_product, - table_depend_infos_, - is_legal))) { - LOG_WARN("failed to check join legal", K(ret)); - } else if (!is_legal) { - //对于可交换的join如inner join，既left join right合法 - //也right join left合法，但是我们只需要保存left inner join right - //因为在generate join path的时候会生成right inner join left的path - //并且不可能存在一种join，既有left join1 right合法，又有right join2 left合法 - LOG_TRACE("left tree join right tree is not legal", K(left_tree->get_tables()), - K(right_tree->get_tables()), K(*detector)); - //尝试right tree join left tree - if (OB_FAIL(detector->check_join_legal(right_tree->get_tables(), - left_tree->get_tables(), - combined_relids, - delay_cross_product, - table_depend_infos_, - is_legal))) { - LOG_WARN("failed to check join legal", K(ret)); - } else if (!is_legal) { - LOG_TRACE("right tree join left tree is not legal", K(right_tree->get_tables()), - K(left_tree->get_tables()), K(*detector)); - } else if (OB_FAIL(valid_detectors.push_back(detector))) { - LOG_WARN("failed to push back detector", K(ret)); - } else { - is_strict_order = false; - LOG_TRACE("succeed to find join info for ", K(left_tree->get_tables()), - K(right_tree->get_tables()), K(left_tree->get_conflict_detectors()), - K(right_tree->get_conflict_detectors()), K(*detector)); - } - } else if (OB_FAIL(valid_detectors.push_back(detector))) { - LOG_WARN("failed to push back detector", K(ret)); - } else { - LOG_TRACE("succeed to find join info for ", K(left_tree->get_tables()), - K(right_tree->get_tables()), K(left_tree->get_conflict_detectors()), - K(right_tree->get_conflict_detectors()), K(*detector)); - } - } - return ret; -} - -/** - * 只允许多个inner join叠加成一个join info - * 例如：select * from A, B where A.c1 = B.c1 and A.c2 = B.c2 - * 或者单个OUTER JOIN加上多个inner join info，inner join info作为join qual - * 例如：select * from A left join B on A.c1 = B.c1 where A.c2 = B.c2 - */ -int ObLogPlan::check_join_info(const ObIArray &valid_detectors, - ObJoinType &join_type, - bool &is_valid) -{ - int ret = OB_SUCCESS; - ObConflictDetector *detector = NULL; - bool has_non_inner_join = false; - is_valid = true; - join_type = INNER_JOIN; - if (valid_detectors.empty()) { - is_valid = false; - } - for (int64_t i = 0; OB_SUCC(ret) && is_valid && i < valid_detectors.count(); ++i) { - detector = valid_detectors.at(i); - if (OB_ISNULL(detector)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect null detectors", K(ret)); - } else if (INNER_JOIN == detector->get_join_info().join_type_) { - //do nothing - } else if (has_non_inner_join) { - //不允许出现多个非inner join的join info - is_valid = false; - } else { - has_non_inner_join = true; - join_type = detector->get_join_info().join_type_; - } - } - return ret; -} - -int ObLogPlan::merge_join_info(ObJoinOrder *left_tree, - ObJoinOrder *right_tree, - const ObIArray &valid_detectors, - JoinInfo &join_info) -{ - int ret = OB_SUCCESS; - bool is_valid = false; - if (OB_FAIL(check_join_info(valid_detectors, - join_info.join_type_, - is_valid))) { - LOG_WARN("failed to check join info", K(ret)); - } else if (!is_valid) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect different join info", K(valid_detectors), K(ret)); - } else { - ObConflictDetector *detector = NULL; - for (int64_t i = 0; OB_SUCC(ret) && i < valid_detectors.count(); ++i) { - detector = valid_detectors.at(i); - if (OB_ISNULL(detector)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect null detectors", K(ret)); - } else if (OB_FAIL(join_info.table_set_.add_members(detector->get_join_info().table_set_))) { - LOG_WARN("failed to add members", K(ret)); - } else if (OB_FAIL(append_array_no_dup(join_info.where_conditions_, detector->get_join_info().where_conditions_))) { - LOG_WARN("failed to append exprs", K(ret)); - } else if (INNER_JOIN == join_info.join_type_ && - OB_FAIL(append_array_no_dup(join_info.where_conditions_, detector->get_join_info().on_conditions_))) { - LOG_WARN("failed to append exprs", K(ret)); - } else if (INNER_JOIN != join_info.join_type_ && - OB_FAIL(append_array_no_dup(join_info.on_conditions_, detector->get_join_info().on_conditions_))) { - LOG_WARN("failed to append exprs", K(ret)); - } - } - if (OB_SUCC(ret)) { - // TODO: need to optimize this hotspot - if (OB_FAIL(remove_redundancy_pred(left_tree, right_tree, join_info))) { - LOG_WARN("failed to remove redundancy pred", K(ret)); - } - } - //抽取简单join condition - if (OB_SUCC(ret)) { - if (INNER_JOIN == join_info.join_type_) { - for (int64_t i = 0; OB_SUCC(ret) && i < join_info.where_conditions_.count(); ++i) { - ObRawExpr *qual = join_info.where_conditions_.at(i); - if (OB_ISNULL(qual)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect null join qual", K(ret)); - } else if (!qual->has_flag(IS_JOIN_COND)) { - //do nothing - } else if (OB_FAIL(join_info.equal_join_conditions_.push_back(qual))) { - LOG_WARN("failed to push back join qual", K(ret)); - } - } - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < join_info.on_conditions_.count(); ++i) { - ObRawExpr *qual = join_info.on_conditions_.at(i); - if (OB_ISNULL(qual)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect null join qual", K(ret)); - } else if (!qual->has_flag(IS_JOIN_COND)) { - //do nothing - } else if (OB_FAIL(join_info.equal_join_conditions_.push_back(qual))) { - LOG_WARN("failed to push back join qual", K(ret)); - } - } - } - } - } - return ret; -} - int ObLogPlan::check_detector_valid(ObJoinOrder *left_tree, ObJoinOrder *right_tree, const ObIArray &valid_detectors, @@ -2341,6 +2135,8 @@ int ObLogPlan::check_detector_valid(ObJoinOrder *left_tree, } /** + * Remove redundant join conditions and extract equal join conditions + * * Try keep EQ preds that join same two tables. * For example, we will keep the two preds which join t1 and t3 in the below case. * (t1 join t2 on t1.c1 = t2.c1) @@ -2357,11 +2153,12 @@ int ObLogPlan::check_detector_valid(ObJoinOrder *left_tree, * (t1 where c1 = 1) join (t2 where c2 = 1) on t1.c1 = t2.c1 * => (t1 where c1 = 1) join (t2 where c2 = 1) on true * */ -int ObLogPlan::remove_redundancy_pred(ObJoinOrder *left_tree, - ObJoinOrder *right_tree, - JoinInfo &join_info) +int ObLogPlan::process_join_pred(ObJoinOrder *left_tree, + ObJoinOrder *right_tree, + JoinInfo &join_info) { int ret = OB_SUCCESS; + // remove redundancy pred if (INNER_JOIN == join_info.join_type_ || LEFT_SEMI_JOIN == join_info.join_type_ || LEFT_ANTI_JOIN == join_info.join_type_ || @@ -2374,9 +2171,9 @@ int ObLogPlan::remove_redundancy_pred(ObJoinOrder *left_tree, join_info.where_conditions_; EqualSets input_equal_sets; if (OB_FAIL(ObEqualAnalysis::merge_equal_set(&allocator_, - left_tree->get_output_equal_sets(), - right_tree->get_output_equal_sets(), - input_equal_sets))) { + left_tree->get_output_equal_sets(), + right_tree->get_output_equal_sets(), + input_equal_sets))) { LOG_WARN("failed to compute equal sets for inner join", K(ret)); } else if (OB_FAIL(inner_remove_redundancy_pred(join_pred, input_equal_sets, @@ -2385,6 +2182,34 @@ int ObLogPlan::remove_redundancy_pred(ObJoinOrder *left_tree, LOG_WARN("failed to inner remove redundancy pred", K(ret)); } } + // extract equal join conditions + if (OB_FAIL(ret)) { + // do nothing + } else if (INNER_JOIN == join_info.join_type_) { + for (int64_t i = 0; OB_SUCC(ret) && i < join_info.where_conditions_.count(); ++i) { + ObRawExpr *qual = join_info.where_conditions_.at(i); + if (OB_ISNULL(qual)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect null join qual", K(ret)); + } else if (!qual->has_flag(IS_JOIN_COND)) { + //do nothing + } else if (OB_FAIL(join_info.equal_join_conditions_.push_back(qual))) { + LOG_WARN("failed to push back join qual", K(ret)); + } + } + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < join_info.on_conditions_.count(); ++i) { + ObRawExpr *qual = join_info.on_conditions_.at(i); + if (OB_ISNULL(qual)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect null join qual", K(ret)); + } else if (!qual->has_flag(IS_JOIN_COND)) { + //do nothing + } else if (OB_FAIL(join_info.equal_join_conditions_.push_back(qual))) { + LOG_WARN("failed to push back join qual", K(ret)); + } + } + } return ret; } diff --git a/src/sql/optimizer/ob_log_plan.h b/src/sql/optimizer/ob_log_plan.h index 98c0ad62a..8a8aabdaa 100644 --- a/src/sql/optimizer/ob_log_plan.h +++ b/src/sql/optimizer/ob_log_plan.h @@ -320,9 +320,6 @@ public: int sort_pwj_constraint(ObLocationConstraintContext &location_constraint) const; int resolve_dup_tab_constraint(ObLocationConstraintContext &location_constraint) const; - int get_from_table_items(const ObIArray &from_items, - ObIArray &table_items); - int get_current_semi_infos(const ObIArray &semi_infos, const ObIArray &table_items, ObIArray ¤t_semi_infos); @@ -1533,35 +1530,16 @@ protected: bool &is_valid_join, ObJoinOrder *&join_tree); - int is_detector_used(ObJoinOrder *left_tree, - ObJoinOrder *right_tree, - ObConflictDetector *detector, - bool &is_used); - - int choose_join_info(ObJoinOrder *left_tree, - ObJoinOrder *right_tree, - ObIArray &valid_detectors, - bool delay_cross_product, - bool &is_strict_order); - - int check_join_info(const ObIArray &valid_detectors, - ObJoinType &join_type, - bool &is_valid); - - int merge_join_info(ObJoinOrder *left_tree, - ObJoinOrder *right_tree, - const ObIArray &valid_detectors, - JoinInfo &join_info); - int check_detector_valid(ObJoinOrder *left_tree, ObJoinOrder *right_tree, const ObIArray &valid_detectors, ObJoinOrder *cur_tree, bool &is_valid); - int remove_redundancy_pred(ObJoinOrder *left_tree, - ObJoinOrder *right_tree, - JoinInfo &join_info); + int process_join_pred(ObJoinOrder *left_tree, + ObJoinOrder *right_tree, + JoinInfo &join_info); + int try_keep_pred_join_same_tables(ObJoinOrder *left_tree, ObJoinOrder *right_tree, @@ -1700,6 +1678,8 @@ protected: int init_lateral_table_depend_info(const ObIArray &table_items); private: // member functions + static int distribute_filters_to_baserels(ObIArray &base_level, + ObIArray> &baserel_filters); static int strong_select_replicas(const common::ObAddr &local_server, common::ObIArray &phy_tbl_loc_info_list, bool &is_hit_partition, diff --git a/src/sql/resolver/ddl/ob_create_view_resolver.cpp b/src/sql/resolver/ddl/ob_create_view_resolver.cpp index 1af72b392..e959bd746 100644 --- a/src/sql/resolver/ddl/ob_create_view_resolver.cpp +++ b/src/sql/resolver/ddl/ob_create_view_resolver.cpp @@ -423,6 +423,8 @@ int ObCreateViewResolver::resolve(const ParseNode &parse_tree) mv_ainfo->mv_refresh_info_, table_schema))) { LOG_WARN("fail to resolve mv options", K(ret)); + } else if (OB_FAIL(load_mview_dep_session_vars(*session_info_, select_stmt, table_schema.get_local_session_var()))) { + LOG_WARN("fail to load mview dep session variables", K(ret)); } else if (!tenant_config.is_valid()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tenant config is invalid", KR(ret)); @@ -1831,5 +1833,64 @@ int ObCreateViewResolver::resolve_columns_nullable_value(const sql::ObSelectStmt return ret; } +int ObCreateViewResolver::load_mview_dep_session_vars(ObSQLSessionInfo &session_info, + ObSelectStmt *stmt, + ObLocalSessionVar &dep_vars) +{ + int ret = OB_SUCCESS; + uint64_t data_version = 0; + if (OB_FAIL(GET_MIN_DATA_VERSION(session_info.get_effective_tenant_id(), data_version))) { + LOG_WARN("failed to get min data version", K(ret)); + } else if (data_version < DATA_VERSION_4_3_3_0) { + // when use data version before DATA_VERSION_4_3_3_0, do not extract local var + } else if (OB_FAIL(dep_vars.reserve_max_local_vars_capacity())) { + LOG_WARN("fail to reserve max local vars capacity", K(ret)); + } else if (OB_FAIL(get_dep_session_vars_from_stmt(session_info, stmt, dep_vars))) { + LOG_WARN("fail to get dep session vars from stmt", K(ret)); + } else if (OB_FAIL(dep_vars.remove_local_var(SYS_VAR_SQL_MODE))) { + LOG_WARN("fail to remove sql_mode from vars", K(ret)); + } else { + LOG_TRACE("finish load mview dep session vars", K(session_info.get_sql_mode()), K(dep_vars)); + } + return ret; +} + +int ObCreateViewResolver::get_dep_session_vars_from_stmt(ObSQLSessionInfo &session_info, + ObSelectStmt *stmt, + ObLocalSessionVar &dep_vars) +{ + int ret = OB_SUCCESS; + ObSEArray childs; + if (OB_ISNULL(stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null stmt", K(ret)); + } else if (OB_FAIL(stmt->get_child_stmts(childs))) { + LOG_WARN("get sel exprs failed", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < childs.count(); i++) { + if (OB_FAIL(SMART_CALL(get_dep_session_vars_from_stmt(session_info, childs.at(i), dep_vars)))) { + LOG_WARN("fail to get dep session vars from stmt", K(ret)); + } + } + } + + if (OB_SUCC(ret)) { + ObSEArray exprs; + if (OB_FAIL(stmt->get_relation_exprs(exprs))) { + LOG_WARN("failed to get relation exprs", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < exprs.count(); i++) { + if (OB_ISNULL(exprs.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_FAIL(exprs.at(i)->get_expr_dep_session_vars_recursively(&session_info, dep_vars))) { + LOG_WARN("fail to get expr dep session vars recursively", K(ret)); + } + } + } + } + return ret; +} + } // namespace sql } // namespace oceanbase diff --git a/src/sql/resolver/ddl/ob_create_view_resolver.h b/src/sql/resolver/ddl/ob_create_view_resolver.h index 1b6c003eb..927b8d1fd 100644 --- a/src/sql/resolver/ddl/ob_create_view_resolver.h +++ b/src/sql/resolver/ddl/ob_create_view_resolver.h @@ -146,6 +146,12 @@ private: ObSEArray& csts); int resolve_primary_key_node(ParseNode &pk_node, ObTableSchema &table_schema); int check_on_query_computation_supported(const ObSelectStmt *stmt); + int load_mview_dep_session_vars(ObSQLSessionInfo &session_info, + ObSelectStmt *stmt, + ObLocalSessionVar &dep_vars); + int get_dep_session_vars_from_stmt(ObSQLSessionInfo &session_info, + ObSelectStmt *stmt, + ObLocalSessionVar &dep_vars); private: DISALLOW_COPY_AND_ASSIGN(ObCreateViewResolver); }; diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.cpp b/src/sql/resolver/ddl/ob_ddl_resolver.cpp index 2f9f683fd..a6e495ab2 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.cpp +++ b/src/sql/resolver/ddl/ob_ddl_resolver.cpp @@ -6319,7 +6319,7 @@ int ObDDLResolver::check_dup_gen_col(const ObString &expr, // construct an empty session int ObDDLResolver::init_empty_session(const common::ObTimeZoneInfoWrap &tz_info_wrap, const common::ObString *nls_formats, - const share::schema::ObLocalSessionVar *local_session_var, + const ObLocalSessionVar *local_session_var, ObIAllocator &allocator, ObTableSchema &table_schema, const ObSQLMode sql_mode, @@ -6375,7 +6375,7 @@ int ObDDLResolver::init_empty_session(const common::ObTimeZoneInfoWrap &tz_info_ int ObDDLResolver::reformat_generated_column_expr(ObObj &default_value, const common::ObTimeZoneInfoWrap &tz_info_wrap, const common::ObString *nls_formats, - const share::schema::ObLocalSessionVar &local_session_var, + const ObLocalSessionVar &local_session_var, ObIAllocator &allocator, ObTableSchema &table_schema, ObColumnSchemaV2 &column, diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.h b/src/sql/resolver/ddl/ob_ddl_resolver.h index 83232c508..09abb31eb 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.h +++ b/src/sql/resolver/ddl/ob_ddl_resolver.h @@ -260,7 +260,7 @@ public: ObIArray &gen_col_expr_arr); static int init_empty_session(const common::ObTimeZoneInfoWrap &tz_info_wrap, const ObString *nls_formats, - const share::schema::ObLocalSessionVar *local_session_var, + const ObLocalSessionVar *local_session_var, common::ObIAllocator &allocator, share::schema::ObTableSchema &table_schema, const ObSQLMode sql_mode, @@ -269,7 +269,7 @@ public: static int reformat_generated_column_expr(ObObj &default_value, const common::ObTimeZoneInfoWrap &tz_info_wrap, const common::ObString *nls_formats, - const share::schema::ObLocalSessionVar &local_session_var, + const ObLocalSessionVar &local_session_var, common::ObIAllocator &allocator, share::schema::ObTableSchema &table_schema, share::schema::ObColumnSchemaV2 &column, diff --git a/src/sql/resolver/dml/ob_del_upd_stmt.cpp b/src/sql/resolver/dml/ob_del_upd_stmt.cpp index a6e3f84c5..5831cb6b6 100644 --- a/src/sql/resolver/dml/ob_del_upd_stmt.cpp +++ b/src/sql/resolver/dml/ob_del_upd_stmt.cpp @@ -759,4 +759,28 @@ int ObDelUpdStmt::check_dml_source_from_join() set_dml_source_from_join(true); } return ret; -} \ No newline at end of file +} + +int ObDelUpdStmt::get_modified_materialized_view_id(uint64_t &mview_id) const +{ + int ret = OB_SUCCESS; + mview_id = OB_INVALID_ID; + const ObIArray &tables = get_table_items(); + const TableItem *table_item = NULL; + bool is_modified = false; + for (int64_t i = 0; OB_INVALID_ID == mview_id && OB_SUCC(ret) && i < tables.count(); ++i) { + if (OB_ISNULL(table_item = tables.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect null", K(ret), K(table_item)); + } else if (MATERIALIZED_VIEW != table_item->table_type_) { + /* do nothing */ + } else if (OB_FAIL(check_table_be_modified(table_item->ref_id_, is_modified))) { + LOG_WARN("fail to check table be modified", K(ret)); + } else if (!is_modified) { + /* do nothing */ + } else { + mview_id = table_item->mview_id_; + } + } + return ret; +} diff --git a/src/sql/resolver/dml/ob_del_upd_stmt.h b/src/sql/resolver/dml/ob_del_upd_stmt.h index bdf20d36f..e9b46eebf 100644 --- a/src/sql/resolver/dml/ob_del_upd_stmt.h +++ b/src/sql/resolver/dml/ob_del_upd_stmt.h @@ -480,6 +480,7 @@ public: int check_dml_source_from_join(); bool is_pdml_disabled() const { return pdml_disabled_; } void set_pdml_disabled() { pdml_disabled_ = true; } + int get_modified_materialized_view_id(uint64_t &mview_id) const; protected: common::ObSEArray returning_exprs_; common::ObSEArray returning_into_exprs_; diff --git a/src/sql/resolver/dml/ob_dml_resolver.cpp b/src/sql/resolver/dml/ob_dml_resolver.cpp index 537f6c646..d9c5dcafa 100755 --- a/src/sql/resolver/dml/ob_dml_resolver.cpp +++ b/src/sql/resolver/dml/ob_dml_resolver.cpp @@ -4119,7 +4119,7 @@ int ObDMLResolver::resolve_table(const ParseNode &parse_tree, } if (OB_FAIL(ret)) { - } else if (!stmt->is_select_stmt() && + } else if (!stmt->is_select_stmt() && !is_update_for_mv_fast_refresh(*stmt) && OB_FAIL(check_stmt_has_flashback_query(table_item->ref_query_, false, has_flashback_query))) { LOG_WARN("failed to find stmt refer to flashback query", K(ret)); } else if (has_flashback_query) { @@ -4201,6 +4201,23 @@ int ObDMLResolver::resolve_table(const ParseNode &parse_tree, return ret; } +// allowed flashback query in mview fast refresh update/insert operator in mysql mode. +// specific update stmt is used as below: +// update mv1, (... inline view use flashback query, mv1 is not used in this view ...) v1 +// set mv1.cnt = mv1.cnt + v.cnt +// where mv1.c1 = v.c1; +bool ObDMLResolver::is_update_for_mv_fast_refresh(const ObDMLStmt &stmt) +{ + bool is_refresh_stmt = false; + if (lib::is_mysql_mode() && stmt.is_update_stmt() && 2 == stmt.get_table_size()) { + const TableItem *table1 = stmt.get_table_item(0); + const TableItem *table2 = stmt.get_table_item(1); + is_refresh_stmt = (NULL != table1 && NULL != table2 && MATERIALIZED_VIEW == table1->table_type_ + && table2->is_generated_table()); + } + return is_refresh_stmt; +} + int ObDMLResolver::check_table_item_with_gen_col_using_udf(const TableItem *table_item, bool &ans) { int ret = OB_SUCCESS; @@ -8461,7 +8478,7 @@ int ObDMLResolver::resolve_generated_column_expr(const ObString &expr_str, LOG_WARN("add local session var failed", K(ret)); } else if (!session_info->is_inner() && lib::is_mysql_mode()) { //print user warnings - ObSEArray var_array; + ObSEArray var_array; if (OB_FAIL(local_vars.get_local_vars(var_array))) { LOG_WARN("extract sysvars failed", K(ret)); } else { diff --git a/src/sql/resolver/dml/ob_dml_resolver.h b/src/sql/resolver/dml/ob_dml_resolver.h index 4b77219fd..718cba9bd 100644 --- a/src/sql/resolver/dml/ob_dml_resolver.h +++ b/src/sql/resolver/dml/ob_dml_resolver.h @@ -1036,6 +1036,7 @@ private: int add_udt_dependency(const pl::ObUserDefinedType &udt_type); int add_obj_to_llc_bitmap(const ObObj &obj, char *llc_bitmap, double &num_null); int compute_values_table_row_count(ObValuesTableDef &table_def); + bool is_update_for_mv_fast_refresh(const ObDMLStmt &stmt); protected: struct GenColumnExprInfo { GenColumnExprInfo(): diff --git a/src/sql/resolver/dml/ob_dml_stmt.cpp b/src/sql/resolver/dml/ob_dml_stmt.cpp index 113306de0..27ddf8255 100644 --- a/src/sql/resolver/dml/ob_dml_stmt.cpp +++ b/src/sql/resolver/dml/ob_dml_stmt.cpp @@ -2517,6 +2517,19 @@ TableItem *ObDMLStmt::create_table_item(ObIAllocator &allocator) return table_item; } +JoinedTable *ObDMLStmt::create_joined_table(ObIAllocator &allocator) +{ + JoinedTable *table_item = NULL; + void *ptr = NULL; + if (NULL == (ptr = allocator.alloc(sizeof(JoinedTable)))) { + LOG_WARN_RET(OB_ALLOCATE_MEMORY_FAILED, "alloc joined table failed"); + } else { + table_item = new(ptr) JoinedTable(); + table_item->type_ = TableItem::JOINED_TABLE; + } + return table_item; +} + int ObDMLStmt::add_table_item(const ObSQLSessionInfo *session_info, TableItem *table_item) { int ret = OB_SUCCESS; diff --git a/src/sql/resolver/dml/ob_dml_stmt.h b/src/sql/resolver/dml/ob_dml_stmt.h index 55d65b4e5..bbe640f41 100644 --- a/src/sql/resolver/dml/ob_dml_stmt.h +++ b/src/sql/resolver/dml/ob_dml_stmt.h @@ -770,7 +770,7 @@ public: int get_stmt_by_stmt_id(int64_t stmt_id, ObDMLStmt *&stmt); int64_t get_from_item_size() const { return from_items_.count(); } - void clear_from_items() { from_items_.reset(); } + void clear_from_items() { from_items_.reuse(); } int add_from_item(uint64_t tid, bool is_joined = false) { int ret = common::OB_SUCCESS; @@ -790,7 +790,8 @@ public: int remove_joined_table_item(const JoinedTable *joined_table); int remove_joined_table_item(uint64_t tid, bool *remove_happened = NULL); - TableItem *create_table_item(common::ObIAllocator &allocator); + static TableItem *create_table_item(common::ObIAllocator &allocator); + static JoinedTable *create_joined_table(common::ObIAllocator &allocator); int merge_from_items(const ObDMLStmt &stmt); const FromItem &get_from_item(int64_t index) const { return from_items_[index]; } FromItem &get_from_item(int64_t index) { return from_items_[index]; } diff --git a/src/sql/resolver/dml/ob_select_stmt.cpp b/src/sql/resolver/dml/ob_select_stmt.cpp index 470945d2e..375a627c6 100644 --- a/src/sql/resolver/dml/ob_select_stmt.cpp +++ b/src/sql/resolver/dml/ob_select_stmt.cpp @@ -896,26 +896,57 @@ int ObSelectStmt::remove_useless_sharable_expr(ObRawExprFactory *expr_factory, bool ObSelectStmt::is_spj() const { + int ret = OB_SUCCESS; + bool bret = false; bool has_rownum_expr = false; - bool ret = !(has_distinct() - || has_group_by() - || is_set_stmt() - || has_rollup() - || has_order_by() - || has_limit() - || get_aggr_item_size() != 0 - || get_from_item_size() == 0 - || is_contains_assignment() - || has_window_function() - || has_sequence() - || is_hierarchical_query()); - if (!ret) { + bret = !(has_distinct() + || has_group_by() + || is_set_stmt() + || has_rollup() + || has_order_by() + || has_limit() + || get_aggr_item_size() != 0 + || get_from_item_size() == 0 + || is_contains_assignment() + || has_window_function() + || has_sequence() + || is_hierarchical_query()); + if (!bret) { + // do nothing } else if (OB_FAIL(has_rownum(has_rownum_expr))) { - ret = false; + LOG_WARN("failed to check has rownum", K(ret)); + bret = false; } else { - ret = !has_rownum_expr; + bret = !has_rownum_expr; } - return ret; + return bret; +} + +bool ObSelectStmt::is_spjg() const +{ + int ret = OB_SUCCESS; + bool bret = false; + bool has_rownum_expr = false; + bret = !(has_distinct() + || has_having() + || is_set_stmt() + || has_rollup() + || has_order_by() + || has_limit() + || get_from_item_size() == 0 + || is_contains_assignment() + || has_window_function() + || has_sequence() + || is_hierarchical_query()); + if (!bret) { + // do nothing + } else if (OB_FAIL(has_rownum(has_rownum_expr))) { + LOG_WARN("failed to check has rownum", K(ret)); + bret = false; + } else { + bret = !has_rownum_expr; + } + return bret; } int ObSelectStmt::get_select_exprs(ObIArray &select_exprs, diff --git a/src/sql/resolver/dml/ob_select_stmt.h b/src/sql/resolver/dml/ob_select_stmt.h index 985389371..88b28e588 100644 --- a/src/sql/resolver/dml/ob_select_stmt.h +++ b/src/sql/resolver/dml/ob_select_stmt.h @@ -438,6 +438,8 @@ public: into_item_->into_type_ == T_INTO_OUTFILE; } // check if the stmt is a Select-Project-Join(SPJ) query bool is_spj() const; + // is_spj + normal group by or scalar group by + bool is_spjg() const; ObRawExpr *get_expr(uint64_t expr_id); inline bool is_single_table_stmt() const { return (1 == get_table_size() diff --git a/src/sql/resolver/dml/ob_stmt_expr_visitor.h b/src/sql/resolver/dml/ob_stmt_expr_visitor.h index 23a187858..799fcbd75 100644 --- a/src/sql/resolver/dml/ob_stmt_expr_visitor.h +++ b/src/sql/resolver/dml/ob_stmt_expr_visitor.h @@ -187,8 +187,10 @@ public: int add_replace_exprs(const ObIArray &from_exprs, const ObIArray &to_exprs, const ObIArray *skip_exprs = NULL); + inline int add_replace_expr(ObRawExpr *from_expr, ObRawExpr *to_expr) { return replacer_.add_replace_expr(from_expr, to_expr); } void set_skip_bool_param_mysql(bool skip) { replacer_.set_skip_bool_param_mysql(skip); } bool is_skip_bool_param_mysql() { return replacer_.is_skip_bool_param_mysql(); } + inline bool is_existed(const ObRawExpr *target) const { return replacer_.is_existed(target); } private: int add_skip_expr(const ObRawExpr *skip_expr); int check_expr_need_skip(const ObRawExpr *skip_expr, bool &need_skip); diff --git a/src/sql/resolver/expr/ob_raw_expr.cpp b/src/sql/resolver/expr/ob_raw_expr.cpp index 267f3da70..4e22f1e57 100644 --- a/src/sql/resolver/expr/ob_raw_expr.cpp +++ b/src/sql/resolver/expr/ob_raw_expr.cpp @@ -1089,11 +1089,11 @@ bool ObRawExpr::is_specified_pseudocolumn_expr() const return false; } -int ObRawExpr::extract_local_session_vars_recursively(ObIArray &var_array) +int ObRawExpr::extract_local_session_vars_recursively(ObIArray &var_array) { int ret = OB_SUCCESS; if (get_local_session_var().get_var_count() > 0) { - ObSEArray local_vars; + ObSEArray local_vars; if (OB_FAIL(get_local_session_var().get_local_vars(local_vars))) { LOG_WARN("fail to append session var array", K(ret)); } else if (OB_FAIL(append(var_array, local_vars))) { @@ -1113,6 +1113,25 @@ int ObRawExpr::extract_local_session_vars_recursively(ObIArrayget_expr_dep_session_vars_recursively(session, dep_vars)))) { + LOG_WARN("fail to get expr dep session vars recursively", K(ret)); + } + } + } + return ret; +} + int ObRawExpr::has_exec_param(bool &bool_ret) const { int ret = OB_SUCCESS; @@ -1319,7 +1338,7 @@ bool ObConstRawExpr::inner_same_as( return bool_ret; } -int ObConstRawExpr::set_local_session_vars(const share::schema::ObLocalSessionVar *local_sys_vars, +int ObConstRawExpr::set_local_session_vars(const ObLocalSessionVar *local_sys_vars, const ObBasicSessionInfo *session, int64_t ctx_array_idx) { @@ -1340,6 +1359,21 @@ int ObConstRawExpr::set_local_session_vars(const share::schema::ObLocalSessionVa return ret; } +int ObConstRawExpr::get_expr_dep_session_vars(const ObBasicSessionInfo *session, + ObLocalSessionVar &dep_vars) { + int ret = OB_SUCCESS; + if (ob_is_string_type(get_result_type().get_type())) { + //solidify vars for parser + if (OB_FAIL(ObExprOperator::add_local_var_to_expr(SYS_VAR_SQL_MODE, NULL, session, dep_vars))) { + LOG_WARN("fail to add sql mode", K(ret)); + } else if (OB_FAIL(ObExprOperator::add_local_var_to_expr(SYS_VAR_COLLATION_CONNECTION, NULL, + session, dep_vars))) { + LOG_WARN("fail to add collation connection", K(ret)); + } + } + return ret; +} + int ObConstRawExpr::set_dynamic_eval_questionmark(const ObExprResType &dst_type) { int ret = OB_SUCCESS; @@ -2374,6 +2408,38 @@ int ObNonTerminalRawExpr::replace_expr(const ObIArray &other_exprs, return ret; } +int ObNonTerminalRawExpr::set_local_session_vars(const ObLocalSessionVar *local_var_info, + const ObBasicSessionInfo *session, + int64_t ctx_array_idx) { + int ret = OB_SUCCESS; + ObExprOperator * op = get_op(); + local_session_var_id_ = ctx_array_idx; + local_session_var_.reset(); + if (T_OP_ROW == get_expr_type()) { + /* do nothing */ + } else if (OB_ISNULL(op = get_op())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(op)); + } else if (OB_FAIL(op->set_local_session_vars(this, local_var_info, session, local_session_var_))) { + LOG_WARN("fail to set local session info for expr operators", K(ret)); + } + return ret; +} + +int ObNonTerminalRawExpr::get_expr_dep_session_vars(const ObBasicSessionInfo *session, + ObLocalSessionVar &dep_vars) { + int ret = OB_SUCCESS; + ObExprOperator *op = NULL; + if (T_OP_ROW == get_expr_type()) { + /* do nothing */ + } else if (OB_ISNULL(op = get_op())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(op)); + } else if (OB_FAIL(op->set_local_session_vars(this, NULL, session, dep_vars))) { + LOG_WARN("fail to set local session info for expr operators", K(ret)); + } + return ret; +} //////////////////////////////////////////////////////////////// ObOpRawExpr::ObOpRawExpr(ObRawExpr *first_expr, @@ -4715,22 +4781,6 @@ int ObSysFunRawExpr::get_autoinc_nextval_name(char *buf, int64_t buf_len, int64_ return ret; } -int ObSysFunRawExpr::set_local_session_vars(const share::schema::ObLocalSessionVar *local_var_info, - const ObBasicSessionInfo *session, - int64_t ctx_array_idx) { - int ret = OB_SUCCESS; - ObExprOperator * op = get_op(); - local_session_var_id_ = ctx_array_idx; - local_session_var_.reset(); - if (OB_ISNULL(op)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", K(ret), K(op)); - } else if (OB_FAIL(op->set_local_session_vars(this, local_var_info, session, local_session_var_))) { - LOG_WARN("fail to set local session info for expr operators", K(ret)); - } - return ret; -} - int ObSequenceRawExpr::assign(const ObRawExpr &other) { int ret = OB_SUCCESS; diff --git a/src/sql/resolver/expr/ob_raw_expr.h b/src/sql/resolver/expr/ob_raw_expr.h index 8efe637f5..d04cfd54c 100644 --- a/src/sql/resolver/expr/ob_raw_expr.h +++ b/src/sql/resolver/expr/ob_raw_expr.h @@ -2025,15 +2025,20 @@ public: K_(is_deterministic), K_(partition_id_calc_type), K_(may_add_interval_part)); - virtual int set_local_session_vars(const share::schema::ObLocalSessionVar *local_sys_vars, + virtual int set_local_session_vars(const ObLocalSessionVar *local_sys_vars, const ObBasicSessionInfo *session, int64_t ctx_array_idx) { return OB_SUCCESS; } - share::schema::ObLocalSessionVar& get_local_session_var() { return local_session_var_; } - const share::schema::ObLocalSessionVar& get_local_session_var() const { return local_session_var_; } - int extract_local_session_vars_recursively(ObIArray &var_array); + virtual int get_expr_dep_session_vars(const ObBasicSessionInfo *session, + ObLocalSessionVar &dep_vars) + { return OB_SUCCESS; } + ObLocalSessionVar& get_local_session_var() { return local_session_var_; } + const ObLocalSessionVar& get_local_session_var() const { return local_session_var_; } + int extract_local_session_vars_recursively(ObIArray &var_array); void set_local_session_var_id(int64_t idx) { local_session_var_id_ = idx; } int64_t get_local_session_var_id() { return local_session_var_id_; } + int get_expr_dep_session_vars_recursively(const ObBasicSessionInfo *session, + ObLocalSessionVar &dep_vars); int has_exec_param(bool &bool_ret) const; @@ -2079,7 +2084,7 @@ protected: // when join with '<=>' in mysql mode, mark runtime filter with null equal condition, // and can not be pushed down as storege white filter bool with_null_equal_cond_; - share::schema::ObLocalSessionVar local_session_var_; + ObLocalSessionVar local_session_var_; int64_t local_session_var_id_; private: DISALLOW_COPY_AND_ASSIGN(ObRawExpr); @@ -2298,9 +2303,11 @@ public: bool is_batch_stmt_parameter() { return is_batch_stmt_parameter_; } void set_array_param_group_id(int64_t id) { array_param_group_id_ = id; } int64_t get_array_param_group_id() const { return array_param_group_id_; } - virtual int set_local_session_vars(const share::schema::ObLocalSessionVar *local_sys_vars, + virtual int set_local_session_vars(const ObLocalSessionVar *local_sys_vars, const ObBasicSessionInfo *session, int64_t ctx_array_idx); + virtual int get_expr_dep_session_vars(const ObBasicSessionInfo *session, + ObLocalSessionVar &dep_vars); int set_dynamic_eval_questionmark(const ObExprResType &dst_type); bool is_dynamic_eval_questionmark() const { return is_dynamic_eval_questionmark_; } @@ -2981,6 +2988,11 @@ public: virtual void reset() { free_op(); input_types_.reset(); } virtual ObExprOperator *get_op(); + virtual int set_local_session_vars(const ObLocalSessionVar *local_sys_vars, + const ObBasicSessionInfo *session, + int64_t ctx_array_idx); + virtual int get_expr_dep_session_vars(const ObBasicSessionInfo *session, + ObLocalSessionVar &dep_vars); void free_op(); /* * 为了在Resolve阶段记录下函数操作数的目标类型，引入input_types_。 @@ -3663,9 +3675,6 @@ public: K_(local_session_var), K_(local_session_var_id), K_(mview_id)); - virtual int set_local_session_vars(const share::schema::ObLocalSessionVar *local_sys_vars, - const ObBasicSessionInfo *session, - int64_t ctx_array_idx); private: int check_param_num_internal(int32_t param_num, int32_t param_count, ObExprOperatorType type); DISALLOW_COPY_AND_ASSIGN(ObSysFunRawExpr); diff --git a/src/sql/resolver/expr/ob_raw_expr_replacer.cpp b/src/sql/resolver/expr/ob_raw_expr_replacer.cpp index 46f52e340..2428ceada 100644 --- a/src/sql/resolver/expr/ob_raw_expr_replacer.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_replacer.cpp @@ -429,6 +429,19 @@ int ObRawExprReplacer::try_init_expr_map(int64_t bucket_size) return ret; } +bool ObRawExprReplacer::is_existed(const ObRawExpr *target) const +{ + bool bret = false; + if (OB_LIKELY(expr_replace_map_.created())) { + uint64_t key = reinterpret_cast(target); + uint64_t val = 0; + if (OB_SUCCESS == expr_replace_map_.get_refactored(key, val)) { + bret = true; + } + } + return bret; +} + int ObRawExprReplacer::check_from_expr_existed(const ObRawExpr *from_expr, const ObRawExpr *to_expr, const bool overwrite, diff --git a/src/sql/resolver/expr/ob_raw_expr_replacer.h b/src/sql/resolver/expr/ob_raw_expr_replacer.h index 70b75a974..76e5cc54c 100644 --- a/src/sql/resolver/expr/ob_raw_expr_replacer.h +++ b/src/sql/resolver/expr/ob_raw_expr_replacer.h @@ -67,6 +67,7 @@ public: int add_replace_exprs(const ObIArray> &to_replace_exprs); int append_replace_exprs(const ObRawExprReplacer &other); int check_need_replace(const ObRawExpr *old_expr, ObRawExpr *&new_expr, bool &need_replace); + bool is_existed(const ObRawExpr *target) const; private: // types and constants diff --git a/src/sql/resolver/expr/ob_raw_expr_util.cpp b/src/sql/resolver/expr/ob_raw_expr_util.cpp index 3efddd811..4dc3a5b3b 100644 --- a/src/sql/resolver/expr/ob_raw_expr_util.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_util.cpp @@ -6769,6 +6769,8 @@ int ObRawExprUtils::create_equal_expr(ObRawExprFactory &expr_factory, LOG_WARN("set param expr failed", K(ret)); } else if (OB_FAIL(equal_expr->formalize(session_info))) { LOG_WARN("formalize equal expr failed", K(ret)); + } else if (OB_FAIL(equal_expr->pull_relation_id())) { + LOG_WARN("pull expr relation ids failed", K(ret)); } else {} return ret; } @@ -9721,7 +9723,7 @@ int ObRawExprUtils::extract_local_vars_for_gencol(ObRawExpr *expr, { int ret = OB_SUCCESS; ObSEArray dep_columns; - ObSEArray var_array; + ObSEArray var_array; if (OB_FAIL(expr->extract_local_session_vars_recursively(var_array))) { LOG_WARN("extract sysvars failed", K(ret)); } else if (OB_FAIL(ObRawExprUtils::extract_column_exprs(expr, dep_columns))) { diff --git a/src/sql/resolver/mv/ob_mv_checker.cpp b/src/sql/resolver/mv/ob_mv_checker.cpp index 955e4a8e9..f2be6833c 100644 --- a/src/sql/resolver/mv/ob_mv_checker.cpp +++ b/src/sql/resolver/mv/ob_mv_checker.cpp @@ -71,6 +71,7 @@ int ObMVChecker::check_mv_refresh_type() } else if (OB_FAIL(check_mjv_refresh_type(stmt_, refresh_type_))) { LOG_WARN("failed to check mjv refresh type", K(ret)); } + LOG_TRACE("finish check mv refresh type", K_(refresh_type)); return ret; } @@ -231,6 +232,9 @@ int ObMVChecker::check_mv_dependency_mlog_tables(const ObSelectStmt &stmt, bool } else if (OB_UNLIKELY(!table->is_basic_table())) { is_valid = false; append_fast_refreshable_note("basic table allowed only"); + } else if (OB_NOT_NULL(table->flashback_query_expr_)) { + is_valid = false; + append_fast_refreshable_note("flashback query not support"); } else if (OB_FAIL(sql_schema_guard->get_table_schema(table->ref_id_, table_schema))) { LOG_WARN("failed to get table schema", K(ret)); } else if (OB_ISNULL(table_schema)) { @@ -324,8 +328,12 @@ int ObMVChecker::check_mav_refresh_type(const ObSelectStmt &stmt, ObMVRefreshabl LOG_WARN("failed to check mav aggr valid", K(ret)); } else if (!is_valid) { refresh_type = OB_MV_COMPLETE_REFRESH; - } else if (stmt.is_single_table_stmt()) { // only support single table MAV + } else if (stmt.is_single_table_stmt()) { // single table MAV refresh_type = OB_MV_FAST_REFRESH_SIMPLE_MAV; + } else if (OB_FAIL(check_join_mv_fast_refresh_valid(stmt, true, is_valid))) { + LOG_WARN("failed to check join mv fast refresh valid", K(ret)); + } else if (is_valid) { // join MAV + refresh_type = OB_MV_FAST_REFRESH_SIMPLE_JOIN_MAV; } else { append_fast_refreshable_note("group by with multi table not support now", OB_MV_FAST_REFRESH_SIMPLE_MAV); refresh_type = OB_MV_COMPLETE_REFRESH; @@ -653,8 +661,26 @@ int ObMVChecker::check_mjv_refresh_type(const ObSelectStmt &stmt, ObMVRefreshabl { int ret = OB_SUCCESS; refresh_type = OB_MV_COMPLETE_REFRESH; - bool join_type_valid = true; - bool select_valid = true; + bool is_valid = false; + if (OB_FAIL(check_join_mv_fast_refresh_valid(stmt, false, is_valid))) { + LOG_WARN("failed to check join mv fast refresh valid", K(ret)); + } else if (!is_valid) { + /* do nothing */ + } else { + refresh_type = OB_MV_FAST_REFRESH_SIMPLE_MJV; + } + return ret; +} + +int ObMVChecker::check_join_mv_fast_refresh_valid(const ObSelectStmt &stmt, + const bool for_join_mav, + bool &is_valid) +{ + int ret = OB_SUCCESS; + is_valid = false; + bool join_type_valid = false; + bool all_table_exists_rowkey = false; + bool select_valid = false; if (stmt.get_table_size() <= 1) { append_fast_refreshable_note("table size not support"); // } else if (stmt.get_table_size() > 5) { @@ -663,12 +689,18 @@ int ObMVChecker::check_mjv_refresh_type(const ObSelectStmt &stmt, ObMVRefreshabl LOG_WARN("failed to check mv join type", K(ret)); } else if (!join_type_valid) { append_fast_refreshable_note("outer join not support"); - } else if (OB_FAIL(check_select_contains_all_tables_primary_key(stmt, select_valid))) { - LOG_WARN("failed to check check select contains all tables primary key", K(ret)); - } else if (!select_valid) { - append_fast_refreshable_note("base table primary key need in select for MJV"); + } else if (OB_FAIL(check_select_contains_all_tables_primary_key(stmt, all_table_exists_rowkey, select_valid))) { + LOG_WARN("failed to check select contains all tables primary key", K(ret)); + } else if (for_join_mav) { + if (all_table_exists_rowkey) { + is_valid = true; + } else { + append_fast_refreshable_note("base table need primary key for join MAV"); + } + } else if (select_valid) { + is_valid = true; } else { - refresh_type = OB_MV_FAST_REFRESH_SIMPLE_MJV; + append_fast_refreshable_note("base table primary key need in select for MJV"); } return ret; } @@ -703,19 +735,22 @@ bool ObMVChecker::is_mv_join_type_valid(const TableItem *table) } int ObMVChecker::check_select_contains_all_tables_primary_key(const ObSelectStmt &stmt, + bool &all_table_exists_rowkey, bool &contain_all_rowkey) { int ret = OB_SUCCESS; contain_all_rowkey = false; + all_table_exists_rowkey = false; if (OB_ISNULL(stmt.get_query_ctx())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret), K(stmt.get_query_ctx())); } else { contain_all_rowkey = true; + all_table_exists_rowkey = true; int64_t all_rowkey_size = 0; ObSEArray rowkeys; ObSqlSchemaGuard &sql_schema_guard = stmt.get_query_ctx()->sql_schema_guard_; - for (int64_t i = 0; contain_all_rowkey && OB_SUCC(ret) && i < stmt.get_table_items().count(); ++i) { + for (int64_t i = 0; all_table_exists_rowkey && OB_SUCC(ret) && i < stmt.get_table_items().count(); ++i) { TableItem *table_item = NULL; const ObTableSchema *table_schema = NULL; if (OB_ISNULL(table_item = stmt.get_table_items().at(i)) @@ -728,13 +763,13 @@ int ObMVChecker::check_select_contains_all_tables_primary_key(const ObSelectStmt ret = OB_ERR_UNEXPECTED; LOG_WARN("get invalid table schema", K(ret), K(table_schema)); } else if (table_schema->is_heap_table()) { - contain_all_rowkey = false; + all_table_exists_rowkey = false; } else { all_rowkey_size += table_schema->get_rowkey_info().get_size(); } } - for (int64_t i = 0; OB_SUCC(ret) && i < stmt.get_select_items().count(); ++i) { + for (int64_t i = 0; all_table_exists_rowkey &&OB_SUCC(ret) && i < stmt.get_select_items().count(); ++i) { const ObRawExpr *expr = NULL; int64_t idx = OB_INVALID_INDEX; if (OB_ISNULL(expr = stmt.get_select_items().at(i).expr_)) { @@ -747,8 +782,8 @@ int ObMVChecker::check_select_contains_all_tables_primary_key(const ObSelectStmt } } - if (OB_FAIL(ret) || !contain_all_rowkey) { - /* do nothing */ + if (OB_FAIL(ret) || !all_table_exists_rowkey) { + contain_all_rowkey = false; } else if (all_rowkey_size > rowkeys.count()) { contain_all_rowkey = false; } else if (OB_UNLIKELY(rowkeys.count() != all_rowkey_size)) { diff --git a/src/sql/resolver/mv/ob_mv_checker.h b/src/sql/resolver/mv/ob_mv_checker.h index e31fd12fc..b9cab946b 100644 --- a/src/sql/resolver/mv/ob_mv_checker.h +++ b/src/sql/resolver/mv/ob_mv_checker.h @@ -29,6 +29,7 @@ enum ObMVRefreshableType OB_MV_COMPLETE_REFRESH, // can not fast refresh OB_MV_FAST_REFRESH_SIMPLE_MAV, // fast refresh for single table MAV OB_MV_FAST_REFRESH_SIMPLE_MJV, // fast refresh for inner join MJV + OB_MV_FAST_REFRESH_SIMPLE_JOIN_MAV, // fast refresh for inner join MAV }; struct FastRefreshableNotes @@ -78,7 +79,9 @@ private: int check_mv_stmt_refresh_type_basic(const ObSelectStmt &stmt, bool &is_valid); int check_mv_join_type(const ObSelectStmt &stmt, bool &join_type_valid); bool is_mv_join_type_valid(const TableItem *table); - int check_select_contains_all_tables_primary_key(const ObSelectStmt &stmt, bool &contain_all_rowkey); + int check_select_contains_all_tables_primary_key(const ObSelectStmt &stmt, + bool &all_table_exists_rowkey, + bool &contain_all_rowkey); int check_mv_stmt_use_special_expr(const ObSelectStmt &stmt, bool &has_special_expr); int check_mv_dependency_mlog_tables(const ObSelectStmt &stmt, bool &is_valid); int check_mv_duplicated_exprs(const ObSelectStmt &stmt, bool &has_dup_exprs); @@ -111,6 +114,9 @@ private: const ObIArray &aggrs, const ObAggFunRawExpr *&target_aggr); int check_mjv_refresh_type(const ObSelectStmt &stmt, ObMVRefreshableType &refresh_type); + int check_join_mv_fast_refresh_valid(const ObSelectStmt &stmt, + const bool for_join_mav, + bool &is_valid); void append_fast_refreshable_note(const char *str, const ObMVRefreshableType type = OB_MV_COMPLETE_REFRESH); const ObSelectStmt &stmt_; diff --git a/src/sql/resolver/mv/ob_mv_printer.cpp b/src/sql/resolver/mv/ob_mv_printer.cpp index 01b0e08e6..50719217b 100644 --- a/src/sql/resolver/mv/ob_mv_printer.cpp +++ b/src/sql/resolver/mv/ob_mv_printer.cpp @@ -109,10 +109,11 @@ int ObMVPrinter::gen_refresh_dmls_for_mv(ObIArray &dml_stmts) { int ret = OB_SUCCESS; dml_stmts.reuse(); + const bool mysql_mode = lib::is_mysql_mode(); switch (mv_checker_.get_refersh_type()) { case OB_MV_FAST_REFRESH_SIMPLE_MAV: { ObMergeStmt *merge_stmt = NULL; - if (lib::is_mysql_mode()) { + if (mysql_mode) { if (OB_FAIL(gen_update_insert_delete_for_simple_mav(dml_stmts))) { LOG_WARN("failed to gen update insert delete for simple mav", K(ret)); } @@ -130,6 +131,14 @@ int ObMVPrinter::gen_refresh_dmls_for_mv(ObIArray &dml_stmts) } break; } + case OB_MV_FAST_REFRESH_SIMPLE_JOIN_MAV: { + if (mysql_mode && OB_FAIL(gen_update_insert_delete_for_simple_join_mav(dml_stmts))) { + LOG_WARN("failed to gen update insert delete for simple mav", K(ret)); + } else if (!mysql_mode && OB_FAIL(gen_merge_for_simple_join_mav(dml_stmts))) { + LOG_WARN("failed to gen merge into for simple mav", K(ret)); + } + break; + } default: { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected refresh type", K(ret), K(mv_checker_.get_refersh_type())); @@ -143,15 +152,15 @@ int ObMVPrinter::gen_real_time_view_for_mv(ObSelectStmt *&sel_stmt) { int ret = OB_SUCCESS; switch (mv_checker_.get_refersh_type()) { - case OB_MV_FAST_REFRESH_SIMPLE_MAV: { - if (OB_FAIL(gen_real_time_view_for_simple_mav(sel_stmt))) { - LOG_WARN("failed to gen real time view for simple mav", K(ret)); + case OB_MV_FAST_REFRESH_SIMPLE_MAV: + case OB_MV_FAST_REFRESH_SIMPLE_JOIN_MAV: { + if (OB_FAIL(gen_real_time_view_for_mav(sel_stmt))) { + LOG_WARN("failed to gen real time view for mav", K(ret)); } break; } case OB_MV_FAST_REFRESH_SIMPLE_MJV: { if (OB_FAIL(gen_real_time_view_for_simple_mjv(sel_stmt))) { - ret = OB_ERR_UNEXPECTED; LOG_WARN("fail to gen real time view for simple mjv", K(ret)); } break; @@ -186,9 +195,9 @@ int ObMVPrinter::gen_real_time_view_for_simple_mjv(ObSelectStmt *&sel_stmt) if (OB_FAIL(create_simple_stmt(sel_stmt))) { LOG_WARN("failed to create simple stmt", K(ret)); } else if (OB_FAIL(gen_access_mv_data_for_simple_mjv(access_mv_stmt))) { - LOG_WARN("failed to generate ", K(ret)); + LOG_WARN("failed to generate access mv data for simple mjv", K(ret)); } else if (OB_FAIL(gen_access_delta_data_for_simple_mjv(access_delta_stmts))) { - LOG_WARN("failed to generate ", K(ret)); + LOG_WARN("failed to generate access delta data for simple mjv", K(ret)); } else if (OB_FAIL(sel_stmt->get_set_query().push_back(access_mv_stmt) || OB_FAIL(append(sel_stmt->get_set_query(), access_delta_stmts)))) { LOG_WARN("failed to set set query", K(ret)); @@ -448,30 +457,32 @@ int ObMVPrinter::gen_delete_conds(const ObIArray &mv_column } /* - select inner_rt_mv.*, sum_c3/cnt_c3 as avg_c3 - from (select nvl(mv.c1, d_mv.c1) as c1, - nvl(mv.c2, d_mv.c2) as c2, - (case when mv.cnt is null then d_mv.cnt, - when d_mv.cnt is null then mv.cnt, - else mv.cnt + d_mv.cnt) as cnt - (case when mv.cnt_c3 is null then d_mv.cnt_c3, - when d_mv.cnt_c3 is null then mv.cnt_c3, - else mv.cnt_c3 + d_mv.cnt) as cnt_c3 - (case when mv.sum_c3 is null then d_mv.sum_c3, - when d_mv.sum_c3 is null then mv.sum_c3, - else mv.sum_c3 + d_mv.sum_c3) as sum_c3 - from mv full join d_mv on mv.c1 <=> d_mv.c1 and mv.c2 <=> d_mv.c2) inner_rt_mv - where cnt > 0; +for real-time mv: + select t1.c1 c1, count(*) cnt, count(t2.c2) cnt_c2, sum(t2.c2) sum_c2, avg(t2.c2) avg_c2 + from t1, t2 + where t1.c1 = t2.c1 + group by t1.c1; +generate real time view as: + select c1, + cnt, + nvl(cnt_c2, 0) cnt_c2, + case when cnt_c2 <> 0 then sum_c2 else null end sum_c2 + (case when cnt_c2 <> 0 then sum_c2 else null end)/nvl(cnt_c2, 0) avg_c2 + from inner_rt_view + where cnt > 0; */ -int ObMVPrinter::gen_real_time_view_for_simple_mav(ObSelectStmt *&sel_stmt) +int ObMVPrinter::gen_real_time_view_for_mav(ObSelectStmt *&sel_stmt) { int ret = OB_SUCCESS; sel_stmt = NULL; TableItem *view_table = NULL; + ObSelectStmt *view_stmt = NULL; if (OB_FAIL(create_simple_stmt(sel_stmt))) { LOG_WARN("failed to create simple stmt", K(ret)); - } else if (OB_FAIL(gen_inner_real_time_view_for_mav(*sel_stmt, view_table))) { - LOG_WARN("failed to generate real time view filter", K(ret)); + } else if (OB_FAIL(gen_inner_real_time_view_for_mav(view_stmt))) { + LOG_WARN("failed to generate inner real time view for simple mav", K(ret)); + } else if (OB_FAIL(create_simple_table_item(sel_stmt, INNER_RT_MV_VIEW_NAME, view_table, view_stmt))) { + LOG_WARN("failed to create simple table item", K(ret)); } else if (OB_FAIL(gen_select_items_for_mav(view_table->get_table_name(), view_table->table_id_, sel_stmt->get_select_items()))) { @@ -514,152 +525,86 @@ int ObMVPrinter::gen_real_time_view_filter_for_mav(ObSelectStmt &sel_stmt) return ret; } -int ObMVPrinter::gen_inner_real_time_view_for_mav(ObSelectStmt &sel_stmt, TableItem *&view_table) +/* +for real-time mv: + select t1.c1 c1, count(*) cnt, count(t2.c2) cnt_c2, sum(t2.c2) sum_c2, avg(t2.c2) avg_c2 + from t1, t2 + where t1.c1 = t2.c1 + group by t1.c1; +generate inner real time view as: + select c1, sum(cnt) cnt, sum(cnt_c2) cnt_c2, sum(sum_c2) sum_c2 + from ( select c1, cnt, cnt_c2, sum_c2 from rt_mv + union all + inner_delta_mav_1 + union all + inner_delta_mav_2) + group by c1; +only group by and basic aggregate functions added to select list +*/ +int ObMVPrinter::gen_inner_real_time_view_for_mav(ObSelectStmt *&inner_rt_view) { int ret = OB_SUCCESS; - ObSelectStmt *view_stmt = NULL; - if (OB_FAIL(create_simple_stmt(view_stmt))) { - LOG_WARN("failed to create simple stmt", K(ret)); - } else if (OB_FAIL(gen_inner_real_time_view_tables_for_mav(*view_stmt))) { - LOG_WARN("failed to generate real_time_view tables for mav", K(ret)); - } else if (OB_FAIL(gen_inner_real_time_view_select_list_for_mav(*view_stmt))) { - LOG_WARN("failed to generate select list", K(ret)); - } else if (OB_FAIL(create_simple_table_item(&sel_stmt, INNER_RT_MV_VIEW_NAME, view_table, view_stmt))) { - LOG_WARN("failed to create simple table item", K(ret)); - } - return ret; -} - -int ObMVPrinter::gen_inner_real_time_view_tables_for_mav(ObSelectStmt &sel_stmt) -{ - int ret = OB_SUCCESS; - ObSelectStmt *delta_view_stmt = NULL; + inner_rt_view = NULL; + ObSelectStmt *access_mv_stmt = NULL; + ObSelectStmt *set_stmt = NULL; + ObSEArray inner_delta_mavs; TableItem *mv_table = NULL; - TableItem *delta_mv_table = NULL; - JoinedTable *joined_table = NULL; - void *ptr = NULL; - if (OB_FAIL(create_simple_table_item(&sel_stmt, mv_schema_.get_table_name(), mv_table, NULL, false))) { + TableItem *view_table = NULL; + if (OB_FAIL(create_simple_stmt(inner_rt_view)) + || OB_FAIL(create_simple_stmt(access_mv_stmt)) + || OB_FAIL(create_simple_stmt(set_stmt))) { + LOG_WARN("failed to create simple stmt", K(ret)); + } else if (OB_FAIL(create_simple_table_item(access_mv_stmt, mv_schema_.get_table_name(), mv_table))) { LOG_WARN("failed to create simple table item", K(ret)); - } else if (OB_FAIL(gen_simple_mav_delta_mv_view(delta_view_stmt))) { // for complex mav, just replace this function - LOG_WARN("failed gen simple source stmt", K(ret)); - } else if (OB_FAIL(create_simple_table_item(&sel_stmt, DELTA_BASIC_MAV_VIEW_NAME, delta_mv_table, delta_view_stmt, false))) { + } else if (OB_FAIL(gen_simple_join_mav_basic_select_list(*mv_table, access_mv_stmt->get_select_items(), NULL))) { + LOG_WARN("failed to generate simple join mav basic select list", K(ret)); + } else if (OB_FAIL(gen_inner_delta_mav_for_mav(inner_delta_mavs))) { + LOG_WARN("failed to gen inner delta mav for mav", K(ret)); + } else if (OB_FAIL(set_stmt->get_set_query().push_back(access_mv_stmt) + || OB_FAIL(append(set_stmt->get_set_query(), inner_delta_mavs)))) { + LOG_WARN("failed to set set query", K(ret)); + } else if (OB_FAIL(create_simple_table_item(inner_rt_view, INNER_RT_MV_VIEW_NAME, view_table, set_stmt))) { LOG_WARN("failed to create simple table item", K(ret)); - } else if (OB_ISNULL(ptr = (alloc_.alloc(sizeof(JoinedTable))))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to allocate memory", K(ret)); + } else if (OB_FAIL(gen_simple_join_mav_basic_select_list(*view_table, + inner_rt_view->get_select_items(), + &inner_rt_view->get_group_exprs()))) { + LOG_WARN("failed to generate simple join mav basic select list", K(ret)); } else { - joined_table = new (ptr) JoinedTable(); - joined_table->type_ = TableItem::JOINED_TABLE; - joined_table->table_id_ = sel_stmt.get_query_ctx()->available_tb_id_--; - joined_table->joined_type_ = ObJoinType::FULL_OUTER_JOIN; - joined_table->left_table_ = delta_mv_table; - joined_table->right_table_ = mv_table; mv_table->database_name_ = mv_db_name_; - if (OB_FAIL(sel_stmt.add_from_item(joined_table->table_id_, true))) { - LOG_WARN("failed to add from item", K(ret)); - } else if (OB_FAIL(sel_stmt.add_joined_table(joined_table))) { - LOG_WARN("failed to add joined table into stmt", K(ret)); - } else if (OB_FAIL(joined_table->single_table_ids_.push_back(delta_mv_table->table_id_)) - || OB_FAIL(joined_table->single_table_ids_.push_back(mv_table->table_id_))) { - LOG_WARN("push back single table id failed", K(ret)); - } else { - const ObIArray &group_by_exprs = mv_checker_.get_stmt().get_group_exprs(); - const ObIArray &select_items = mv_checker_.get_stmt().get_select_items(); - ObRawExpr *col1 = NULL; - ObRawExpr *col2 = NULL; - ObRawExpr *match_cond = NULL; - // add on condition - for (int64_t i = 0; OB_SUCC(ret) && i < select_items.count(); ++i) { - const SelectItem &select_item = select_items.at(i); - if (OB_ISNULL(select_item.expr_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected select item", K(ret), K(i), K(select_items)); - } else if (!ObOptimizerUtil::find_item(group_by_exprs, select_item.expr_)) { - /* do nothing */ - } else if (OB_FAIL(create_simple_column_expr(mv_table->get_table_name(), select_item.alias_name_, mv_table->table_id_, col1)) - || OB_FAIL(create_simple_column_expr(delta_mv_table->get_table_name(), select_item.alias_name_, delta_mv_table->table_id_, col2)) - || OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, T_OP_NSEQ, col1, col2, match_cond))) { - LOG_WARN("failed to build null safe equal expr", K(ret)); - } else if (OB_FAIL(joined_table->join_conditions_.push_back(match_cond))) { - LOG_WARN("failed to push back join conditions", K(ret)); - } - } + set_stmt->assign_set_all(); + set_stmt->assign_set_op(ObSelectStmt::UNION); + } + return ret; +} + +int ObMVPrinter::gen_inner_delta_mav_for_mav(ObIArray &inner_delta_mavs) +{ + int ret = OB_SUCCESS; + inner_delta_mavs.reuse(); + if (OB_MV_FAST_REFRESH_SIMPLE_MAV == mv_checker_.get_refersh_type()) { + ObSelectStmt *inner_delta_mav = NULL; + if (OB_FAIL(gen_simple_mav_delta_mv_view(inner_delta_mav))) { + LOG_WARN("failed gen simple mav delta mv view", K(ret)); + } else if (OB_FAIL(inner_delta_mavs.push_back(inner_delta_mav))) { + LOG_WARN("failed to push back", K(ret)); + } + } else if (OB_MV_FAST_REFRESH_SIMPLE_JOIN_MAV == mv_checker_.get_refersh_type()) { + if (OB_FAIL(gen_inner_delta_mav_for_simple_join_mav(inner_delta_mavs))) { + LOG_WARN("failed to gen inner delta mav for simple join mav", K(ret)); } } return ret; } -int ObMVPrinter::gen_inner_real_time_view_select_list_for_mav(ObSelectStmt &sel_stmt) +int ObMVPrinter::gen_merge_for_simple_mav(ObMergeStmt *&merge_stmt) { int ret = OB_SUCCESS; - ObIArray &select_items = sel_stmt.get_select_items(); - select_items.reuse(); - const ObIArray &orig_group_by_exprs = mv_checker_.get_stmt().get_group_exprs(); - const ObIArray &orig_aggr_exprs = mv_checker_.get_stmt().get_aggr_items(); - TableItem *table1 = NULL; - TableItem *table2 = NULL; - ObRawExpr *col1 = NULL; - ObRawExpr *col2 = NULL; - ObRawExpr *is_null = NULL; - ObCaseOpRawExpr *case_when_expr = NULL; - const ObRawExpr *const_expr = NULL; - ObOpRawExpr *add_expr = NULL; - if (OB_UNLIKELY(2 != sel_stmt.get_table_size()) - || OB_ISNULL(table1 = sel_stmt.get_table_item(0)) - || OB_ISNULL(table2 = sel_stmt.get_table_item(1))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected inner real time view for mav", K(ret), K(sel_stmt.get_table_size()), K(table1), K(table2)); - } - - // add select list for group by - for (int64_t i = 0; OB_SUCC(ret) && i < orig_group_by_exprs.count(); ++i) { - SelectItem sel_item; - sel_item.is_real_alias_ = true; - if (OB_FAIL(get_mv_select_item_name(orig_group_by_exprs.at(i), sel_item.alias_name_))) { - LOG_WARN("failed to get mv select item name", K(ret)); - } else if (OB_FAIL(create_simple_column_expr(table1->get_table_name(), sel_item.alias_name_, table1->table_id_, col1)) - || OB_FAIL(create_simple_column_expr(table2->get_table_name(), sel_item.alias_name_, table2->table_id_, col2))) { - LOG_WARN("failed to create simple column exprs", K(ret)); - } else if (OB_FAIL(add_nvl_above_exprs(col1, col2, sel_item.expr_))) { - LOG_WARN("failed to add nvl above exprs", K(ret)); - } else if (OB_FAIL(select_items.push_back(sel_item))) { - LOG_WARN("failed to pushback", K(ret)); - } - } - - // add select list for basic aggr - for (int64_t i = 0; OB_SUCC(ret) && i < orig_aggr_exprs.count(); ++i) { - SelectItem sel_item; - sel_item.is_real_alias_ = true; - if (OB_ISNULL(const_expr = orig_aggr_exprs.at(i))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", K(ret), K(const_expr)); - } else if (!ObMVChecker::is_basic_aggr(const_expr->get_expr_type())) { - /* do nothing */ - } else if (OB_FAIL(get_mv_select_item_name(const_expr, sel_item.alias_name_))) { - LOG_WARN("failed to get mv select item name", K(ret)); - } else if (OB_FAIL(create_simple_column_expr(table1->get_table_name(), sel_item.alias_name_, table1->table_id_, col1)) - || OB_FAIL(create_simple_column_expr(table2->get_table_name(), sel_item.alias_name_, table2->table_id_, col2))) { - LOG_WARN("failed to create simple column exprs", K(ret)); - } else if (OB_FAIL(expr_factory_.create_raw_expr(T_OP_CASE, case_when_expr))) { - LOG_WARN("create add expr failed", K(ret)); - } else if (OB_FAIL(ObRawExprUtils::build_is_not_null_expr(expr_factory_, col1, false, is_null)) - || OB_FAIL(case_when_expr->add_when_param_expr(is_null)) - || OB_FAIL(case_when_expr->add_then_param_expr(col2))) { - LOG_WARN("failed to build and add is null / then exprs", K(ret)); - } else if (OB_FAIL(ObRawExprUtils::build_is_not_null_expr(expr_factory_, col2, false, is_null)) - || OB_FAIL(case_when_expr->add_when_param_expr(is_null)) - || OB_FAIL(case_when_expr->add_then_param_expr(col1))) { - LOG_WARN("failed to build and add is null / then exprs", K(ret)); - } else if (OB_FAIL(ObRawExprUtils::build_add_expr(expr_factory_, col1, col2, add_expr))) { - LOG_WARN("failed to build add expr", K(ret)); - } else { - case_when_expr->set_default_param_expr(add_expr); - sel_item.expr_ = case_when_expr; - if (OB_FAIL(select_items.push_back(sel_item))) { - LOG_WARN("failed to pushback", K(ret)); - } - } + merge_stmt = NULL; + ObSelectStmt *source_stmt = NULL; + if (OB_FAIL(gen_simple_mav_delta_mv_view(source_stmt))) { + LOG_WARN("failed to gen simple source stmt", K(ret)); + } else if (OB_FAIL(gen_merge_for_simple_mav_use_delta_view(source_stmt, merge_stmt))) { + LOG_WARN("failed to gen merge for simple mav", K(ret)); } return ret; } @@ -692,16 +637,21 @@ int ObMVPrinter::gen_inner_real_time_view_select_list_for_mav(ObSelectStmt &sel_ where d_mv.d_cnt <> 0; */ // generate merge into to fast refresh simple mav in oracle mode -int ObMVPrinter::gen_merge_for_simple_mav(ObMergeStmt *&merge_stmt) +int ObMVPrinter::gen_merge_for_simple_mav_use_delta_view(ObSelectStmt *delta_mav, ObMergeStmt *&merge_stmt) { int ret = OB_SUCCESS; merge_stmt = NULL; TableItem *target_table = NULL; TableItem *source_table = NULL; - if (OB_FAIL(create_simple_stmt(merge_stmt))) { + if (OB_ISNULL(delta_mav)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("target table or source table is null", K(ret), K(delta_mav)); + } else if (OB_FAIL(create_simple_stmt(merge_stmt))) { LOG_WARN("failed to create simple stmt", K(ret)); - } else if (OB_FAIL(gen_merge_tables(*merge_stmt, target_table, source_table))) { - LOG_WARN("failed to gen merge tables", K(ret)); + } else if (OB_FAIL(create_simple_table_item(merge_stmt, mv_schema_.get_table_name(), target_table, NULL, false))) { + LOG_WARN("failed to create simple table item", K(ret)); + } else if (OB_FAIL(create_simple_table_item(merge_stmt, DELTA_BASIC_MAV_VIEW_NAME, source_table, delta_mav, false))) { + LOG_WARN("failed to create simple table item", K(ret)); } else if (OB_FAIL(gen_insert_values_and_desc(target_table, source_table, merge_stmt->get_values_desc(), @@ -714,28 +664,10 @@ int ObMVPrinter::gen_merge_for_simple_mav(ObMergeStmt *&merge_stmt) LOG_WARN("failed to gen update assignments", K(ret)); } else if (OB_FAIL(gen_merge_conds(*merge_stmt))) { LOG_WARN("failed to gen merge conds", K(ret)); - } - return ret; -} - -int ObMVPrinter::gen_merge_tables(ObMergeStmt &merge_stmt, - TableItem *&target_table, - TableItem *&source_table) -{ - int ret = OB_SUCCESS; - target_table = NULL; - source_table = NULL; - ObSelectStmt *source_stmt = NULL; - if (OB_FAIL(create_simple_table_item(&merge_stmt, mv_schema_.get_table_name(), target_table, NULL, false))) { - LOG_WARN("failed to create simple table item", K(ret)); - } else if (OB_FAIL(gen_simple_mav_delta_mv_view(source_stmt))) { - LOG_WARN("failed gen simple source stmt", K(ret)); - } else if (OB_FAIL(create_simple_table_item(&merge_stmt, DELTA_BASIC_MAV_VIEW_NAME, source_table, source_stmt, false))) { - LOG_WARN("failed to create simple table item", K(ret)); } else { target_table->database_name_ = mv_db_name_; - merge_stmt.set_target_table_id(target_table->table_id_); - merge_stmt.set_source_table_id(source_table->table_id_); + merge_stmt->set_target_table_id(target_table->table_id_); + merge_stmt->set_source_table_id(source_table->table_id_); } return ret; } @@ -1152,7 +1084,7 @@ int ObMVPrinter::gen_simple_mav_delta_mv_view(ObSelectStmt *&view_stmt) LOG_WARN("failed to gen delta table view", K(ret)); } else if (OB_FAIL(create_simple_table_item(view_stmt, DELTA_TABLE_VIEW_NAME, table_item, delta_view))) { LOG_WARN("failed to create simple table item", K(ret)); - } else if (OB_FAIL(init_expr_copier_for_delta_mv_view(*table_item, copier))) { + } else if (OB_FAIL(init_expr_copier_for_stmt(*view_stmt, copier))) { LOG_WARN("failed to init expr copier", K(ret)); } else if (OB_FAIL(copier.copy_on_replace(mv_checker_.get_stmt().get_group_exprs(), view_stmt->get_group_exprs()))) { @@ -1161,26 +1093,10 @@ int ObMVPrinter::gen_simple_mav_delta_mv_view(ObSelectStmt *&view_stmt) view_stmt->get_group_exprs(), view_stmt->get_select_items()))) { LOG_WARN("failed to gen select list ", K(ret)); - } else if (OB_FAIL(gen_simple_mav_delta_mv_filter(copier, *table_item, view_stmt->get_condition_exprs()))) { - LOG_WARN("failed to gen filter ", K(ret)); - } - return ret; -} - -int ObMVPrinter::init_expr_copier_for_delta_mv_view(const TableItem &table, ObRawExprCopier &copier) -{ - int ret = OB_SUCCESS; - const ObIArray &column_items = mv_checker_.get_stmt().get_column_items(); - ObRawExpr *old_col = NULL; - ObRawExpr *new_col = NULL; - for (int64_t i = 0; OB_SUCC(ret) && i < column_items.count(); ++i) { - const ColumnItem &col_item = column_items.at(i); - old_col = col_item.expr_; - if (OB_FAIL(create_simple_column_expr(table.get_table_name(), col_item.column_name_, table.table_id_, new_col))) { - LOG_WARN("failed to create simple column expr", K(ret)); - } else if (OB_FAIL(copier.add_replaced_expr(old_col, new_col))) { - LOG_WARN("failed to add replace pair", K(ret)); - } + } else if (OB_FAIL(copier.copy_on_replace(mv_checker_.get_stmt().get_condition_exprs(), view_stmt->get_condition_exprs()))) { + LOG_WARN("failed to convert conds exprs", K(ret)); + } else if (OB_FAIL(append_old_new_row_filter(*table_item, view_stmt->get_condition_exprs()))) { + LOG_WARN("failed to append old new row filter ", K(ret)); } return ret; } @@ -1233,6 +1149,75 @@ int ObMVPrinter::gen_simple_mav_delta_mv_select_list(ObRawExprCopier &copier, return ret; } +// keep the select items ordering with gen_simple_mav_delta_mv_select_list +int ObMVPrinter::gen_simple_join_mav_basic_select_list(const TableItem &table, + ObIArray &select_items, + ObIArray *group_by_exprs) +{ + int ret = OB_SUCCESS; + select_items.reuse(); + const bool stmt_need_aggr = NULL != group_by_exprs; + if (NULL != group_by_exprs) { + group_by_exprs->reuse(); + } + const ObIArray &orig_select_items = mv_checker_.get_stmt().get_select_items(); + const ObIArray &orig_group_by_exprs = mv_checker_.get_stmt().get_group_exprs(); + const ObIArray &orig_aggr_exprs = mv_checker_.get_stmt().get_aggr_items(); + ObAggFunRawExpr *aggr_expr = NULL; + ObRawExpr *col_expr = NULL; + // 1. add select list for group by + for (int64_t i = 0; OB_SUCC(ret) && i < orig_group_by_exprs.count(); ++i) { + SelectItem sel_item; + sel_item.is_real_alias_ = true; + if (OB_FAIL(get_mv_select_item_name(orig_group_by_exprs.at(i), sel_item.alias_name_))) { + LOG_WARN("failed to get mv select item name", K(ret)); + } else if (OB_FAIL(create_simple_column_expr(table.get_table_name(), + sel_item.alias_name_, + table.table_id_, + sel_item.expr_))) { + LOG_WARN("failed to create simple column exprs", K(ret)); + } else if (OB_FAIL(select_items.push_back(sel_item))) { + LOG_WARN("failed to pushback", K(ret)); + } else if (stmt_need_aggr && OB_FAIL(group_by_exprs->push_back(sel_item.expr_))) { + LOG_WARN("failed to pushback", K(ret)); + } + } + // 2. add select list for basic aggr + for (int64_t i = 0; OB_SUCC(ret) && i < orig_aggr_exprs.count(); ++i) { + SelectItem sel_item; + sel_item.is_real_alias_ = true; + sel_item.expr_ = NULL; + if (OB_ISNULL(aggr_expr = orig_aggr_exprs.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(aggr_expr)); + } else if (!ObMVChecker::is_basic_aggr(aggr_expr->get_expr_type())) { + /* do nothing */ + } else if (OB_FAIL(get_mv_select_item_name(aggr_expr, sel_item.alias_name_))) { + LOG_WARN("failed to get mv select item name", K(ret)); + } else if (OB_FAIL(create_simple_column_expr(table.get_table_name(), + sel_item.alias_name_, + table.table_id_, col_expr))) { + LOG_WARN("failed to create simple column exprs", K(ret)); + } else if (!stmt_need_aggr) { + sel_item.expr_ = col_expr; + } else if (OB_FAIL(expr_factory_.create_raw_expr(T_FUN_SUM, aggr_expr))) { + LOG_WARN("create ObAggFunRawExpr failed", K(ret)); + } else if (OB_ISNULL(aggr_expr) || OB_ISNULL(col_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(aggr_expr), K(col_expr)); + } else if (OB_FAIL(aggr_expr->add_real_param_expr(col_expr))) { + LOG_WARN("failed to add param expr to agg expr", K(ret)); + } else { + sel_item.expr_ = aggr_expr; + } + + if (OB_SUCC(ret) && NULL != sel_item.expr_ && OB_FAIL(select_items.push_back(sel_item))) { + LOG_WARN("failed to pushback", K(ret)); + } + } + return ret; +} + int ObMVPrinter::get_mv_select_item_name(const ObRawExpr *expr, ObString &select_name) { int ret = OB_SUCCESS; @@ -1337,12 +1322,12 @@ int ObMVPrinter::add_nvl_above_exprs(ObRawExpr *expr, ObRawExpr *default_expr, O } //(old_new = 'N' and seq_no = "MAXSEQ$$") or (old_new = 'O' and seq_no = "MINSEQ$$") -int ObMVPrinter::gen_simple_mav_delta_mv_filter(ObRawExprCopier &copier, - const TableItem &table_item, - ObIArray &filters) +int ObMVPrinter::append_old_new_row_filter(const TableItem &table_item, + ObIArray &filters, + const bool get_old_row, /* default true */ + const bool get_new_row /* default true */) { int ret = OB_SUCCESS; - filters.reuse(); ObRawExpr *old_new = NULL; ObRawExpr *seq_no = NULL; ObRawExpr *win_seq = NULL; @@ -1353,21 +1338,39 @@ int ObMVPrinter::gen_simple_mav_delta_mv_filter(ObRawExprCopier &copier, ObRawExpr *filter = NULL; const ObString &table_name = table_item.get_table_name(); const uint64_t table_id = table_item.table_id_; - if (OB_FAIL(copier.copy_on_replace(mv_checker_.get_stmt().get_condition_exprs(), filters))) { - LOG_WARN("failed to generate conds exprs", K(ret)); + if (OB_UNLIKELY(!get_old_row && !get_new_row)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected params", K(ret), K(get_old_row), K(get_new_row)); } else if (OB_FAIL(create_simple_column_expr(table_name, OLD_NEW_COL_NAME, table_id, old_new)) ||OB_FAIL(create_simple_column_expr(table_name, SEQUENCE_COL_NAME, table_id, seq_no))) { LOG_WARN("failed to create simple column exprs", K(ret)); + } + + if (OB_FAIL(ret) || !get_new_row) { } else if (OB_FAIL(create_simple_column_expr(table_name, WIN_MAX_SEQ_COL_NAME , table_id, win_seq)) || OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, T_OP_EQ, old_new, exprs_.str_n_, equal_old_new)) || OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, T_OP_EQ, seq_no, win_seq, equal_seq_no)) || OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, T_OP_AND, equal_old_new, equal_seq_no, new_row_filter))) { LOG_WARN("failed to build new row filter expr", K(ret)); + } + + if (OB_FAIL(ret) || !get_old_row) { } else if (OB_FAIL(create_simple_column_expr(table_name, WIN_MIN_SEQ_COL_NAME, table_id, win_seq)) || OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, T_OP_EQ, old_new, exprs_.str_o_, equal_old_new)) || OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, T_OP_EQ, seq_no, win_seq, equal_seq_no)) || OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, T_OP_AND, equal_old_new, equal_seq_no, old_row_filter))) { LOG_WARN("failed to build old row filter expr", K(ret)); + } + + if (OB_FAIL(ret)) { + } else if (get_old_row && !get_new_row) { + if (OB_FAIL(filters.push_back(old_row_filter))) { + LOG_WARN("failed to pushback", K(ret)); + } + } else if (!get_old_row && get_new_row) { + if (OB_FAIL(filters.push_back(new_row_filter))) { + LOG_WARN("failed to pushback", K(ret)); + } } else if (OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, T_OP_OR, new_row_filter, old_row_filter, filter))) { LOG_WARN("failed to build or op expr", K(ret)); } else if (OB_FAIL(filters.push_back(filter))) { @@ -1382,7 +1385,9 @@ int ObMVPrinter::gen_simple_mav_delta_mv_filter(ObRawExprCopier &copier, // (case when "OLD_NEW$$" = 'N' then 1 else -1 end) dml_factor // from mlog_tbale // where scn > xxx and scn <= scn; -int ObMVPrinter::gen_delta_table_view(const TableItem &source_table, ObSelectStmt *&view_stmt) +int ObMVPrinter::gen_delta_table_view(const TableItem &source_table, + ObSelectStmt *&view_stmt, + const uint64_t ext_sel_flags) { int ret = OB_SUCCESS; view_stmt = NULL; @@ -1400,13 +1405,13 @@ int ObMVPrinter::gen_delta_table_view(const TableItem &source_table, ObSelectStm LOG_WARN("failed to create simple table item", K(ret)); } else if (OB_FAIL(gen_delta_table_view_conds(*table_item, view_stmt->get_condition_exprs()))) { LOG_WARN("failed to generate delta table view conds", K(ret)); - } else if (OB_FAIL(gen_delta_table_view_select_list(*table_item, source_table, *view_stmt))) { + } else if (OB_FAIL(gen_delta_table_view_select_list(*table_item, source_table, *view_stmt, ext_sel_flags))) { LOG_WARN("failed to generate delta table view select lists", K(ret)); } else { table_item->database_name_ = source_table.database_name_; - // zhanyuetodo: mlog need not flashback query - // table_item->flashback_query_expr_ = exprs_.refresh_scn_; - // table_item->flashback_query_type_ = TableItem::USING_SCN; + // mlog need not flashback query + table_item->flashback_query_expr_ = NULL; + table_item->flashback_query_type_ = TableItem::NOT_USING; } return ret; } @@ -1441,57 +1446,160 @@ int ObMVPrinter::gen_delta_table_view_conds(const TableItem &table, // 4. min/max window function: min(sequence$$) over (partition by unique_keys), max(sequence$$) over (partition by unique_keys) int ObMVPrinter::gen_delta_table_view_select_list(const TableItem &table, const TableItem &source_table, - ObSelectStmt &stmt) + ObSelectStmt &stmt, + const uint64_t ext_sel_flags) { int ret = OB_SUCCESS; ObIArray &select_items = stmt.get_select_items(); select_items.reuse(); - const ObIArray &column_items = mv_checker_.get_stmt().get_column_items(); - ObRawExpr *equal_expr = NULL; - if (OB_ISNULL(stmt_factory_.get_query_ctx())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", K(ret), K(stmt_factory_.get_query_ctx())); - } else if (OB_FAIL(select_items.prepare_allocate(column_items.count() + 5))) { - LOG_WARN("failed to prepare allocate select items", K(ret)); - } else if (OB_FAIL(create_simple_column_expr(table.get_table_name(), OLD_NEW_COL_NAME, table.table_id_, select_items.at(0).expr_))) { - LOG_WARN("failed to create simple column expr", K(ret)); - } else if (OB_FAIL(create_simple_column_expr(table.get_table_name(), SEQUENCE_COL_NAME, table.table_id_, select_items.at(1).expr_))) { - LOG_WARN("failed to create simple column expr", K(ret)); - } else if (OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, T_OP_EQ, select_items.at(0).expr_, exprs_.str_n_, equal_expr))) { - LOG_WARN("failed to build mul expr", K(ret)); - } else if (OB_FAIL(ObRawExprUtils::build_case_when_expr(expr_factory_, equal_expr, exprs_.int_one_, exprs_.int_neg_one_, select_items.at(2).expr_))) { - LOG_WARN("failed to build case when expr", K(ret)); - } else if (OB_FAIL(gen_max_min_seq_window_func_exprs(stmt_factory_.get_query_ctx()->sql_schema_guard_, - table, source_table, select_items.at(1).expr_, - select_items.at(3).expr_, select_items.at(4).expr_))) { - LOG_WARN("failed to gen max min seq window func exprs", K(ret)); + ObRawExpr *old_new_col = NULL; + ObRawExpr *sequence_expr = NULL; + const bool need_old_new_col = (ext_sel_flags & MLOG_EXT_COL_OLD_NEW) + || (ext_sel_flags & MLOG_EXT_COL_DML_FACTOR); + const bool need_sql_col = (ext_sel_flags & MLOG_EXT_COL_SEQ) + || (ext_sel_flags & MLOG_EXT_COL_WIN_MIN_SEQ) + || (ext_sel_flags & MLOG_EXT_COL_WIN_MAX_SEQ); + const bool need_dml_factor_col = ext_sel_flags & MLOG_EXT_COL_DML_FACTOR; + const bool need_win_min_col = ext_sel_flags & MLOG_EXT_COL_WIN_MIN_SEQ; + const bool need_win_max_col = ext_sel_flags & MLOG_EXT_COL_WIN_MAX_SEQ; + if (OB_FAIL(ret) || !need_old_new_col) { + } else if (OB_FAIL(add_normal_column_to_select_list(table, OLD_NEW_COL_NAME, select_items))) { + LOG_WARN("failed to add normal column to select list", K(ret)); } else { - select_items.at(0).is_real_alias_ = true; - select_items.at(0).alias_name_ = OLD_NEW_COL_NAME; - select_items.at(1).is_real_alias_ = true; - select_items.at(1).alias_name_ = SEQUENCE_COL_NAME; - select_items.at(2).is_real_alias_ = true; - select_items.at(2).alias_name_ = DML_FACTOR_COL_NAME; - select_items.at(3).is_real_alias_ = true; - select_items.at(3).alias_name_ = WIN_MAX_SEQ_COL_NAME; - select_items.at(4).is_real_alias_ = true; - select_items.at(4).alias_name_ = WIN_MIN_SEQ_COL_NAME; + old_new_col = select_items.at(select_items.count() - 1).expr_; + } + + if (OB_FAIL(ret) || !need_sql_col) { + } else if (OB_FAIL(add_normal_column_to_select_list(table, SEQUENCE_COL_NAME, select_items))) { + LOG_WARN("failed to add normal column to select list", K(ret)); + } else { + sequence_expr = select_items.at(select_items.count() - 1).expr_; + } + + if (OB_FAIL(ret)) { + } else if (need_dml_factor_col + && OB_FAIL(add_dml_factor_to_select_list(old_new_col, select_items))) { + LOG_WARN("failed to add dml factor to select list", K(ret)); + } else if ((need_win_min_col || need_win_max_col) + && OB_FAIL(add_max_min_seq_window_to_select_list(table, source_table, + sequence_expr, + select_items, + need_win_max_col, + need_win_min_col))) { + LOG_WARN("failed to add max min seq window func to select list", K(ret)); + } else if (OB_FAIL(add_normal_column_to_select_list(table, source_table, select_items))) { + LOG_WARN("failed to add normal column to select list", K(ret)); + } + return ret; +} + +int ObMVPrinter::add_dml_factor_to_select_list(ObRawExpr *old_new_col, + ObIArray &select_items) +{ + int ret = OB_SUCCESS; + ObRawExpr *equal_expr = NULL; + SelectItem *sel_item = NULL; + if (OB_ISNULL(sel_item = select_items.alloc_place_holder())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("Allocate select item from array error", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, T_OP_EQ, old_new_col, exprs_.str_n_, equal_expr))) { + LOG_WARN("failed to build mul expr", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::build_case_when_expr(expr_factory_, equal_expr, exprs_.int_one_, exprs_.int_neg_one_, sel_item->expr_))) { + LOG_WARN("failed to build case when expr", K(ret)); + } else { + sel_item->is_real_alias_ = true; + sel_item->alias_name_ = DML_FACTOR_COL_NAME; + } + return ret; +} + +int ObMVPrinter::add_normal_column_to_select_list(const TableItem &table, + const ObString &col_name, + ObIArray &select_items) +{ + int ret = OB_SUCCESS; + ObRawExpr *equal_expr = NULL; + SelectItem *sel_item = NULL; + if (OB_ISNULL(sel_item = select_items.alloc_place_holder())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("Allocate select item from array error", K(ret)); + } else if (OB_FAIL(create_simple_column_expr(table.get_table_name(), col_name, table.table_id_, sel_item->expr_))) { + LOG_WARN("failed to create simple column expr", K(ret)); + } else { + sel_item->is_real_alias_ = true; + sel_item->alias_name_ = col_name; + } + return ret; +} + +int ObMVPrinter::add_normal_column_to_select_list(const TableItem &table, + const TableItem &source_table, + ObIArray &select_items) +{ + int ret = OB_SUCCESS; + ObSEArray column_items; + if (OB_FAIL(mv_checker_.get_stmt().get_column_items(source_table.table_id_, column_items))) { + LOG_WARN("failed to get table_id columns items", K(ret)); + } else { + SelectItem *sel_item = NULL; for (int64_t i = 0; OB_SUCC(ret) && i < column_items.count(); ++i) { - SelectItem &sel_item = select_items.at(i + 5); - const ColumnItem &col_item = column_items.at(i); - if (OB_FAIL(create_simple_column_expr(table.get_table_name(), col_item.column_name_, table.table_id_, sel_item.expr_))) { + const ObString &column_name = column_items.at(i).column_name_; + if (OB_ISNULL(sel_item = select_items.alloc_place_holder())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("Allocate select item from array error", K(ret)); + } else if (OB_FAIL(create_simple_column_expr(table.get_table_name(), column_name, table.table_id_, sel_item->expr_))) { LOG_WARN("failed to create simple column expr", K(ret)); } else { - sel_item.is_real_alias_ = true; - sel_item.alias_name_ = col_item.column_name_; + sel_item->is_real_alias_ = true; + sel_item->alias_name_ = column_name; } } } return ret; } -int ObMVPrinter::gen_max_min_seq_window_func_exprs(const ObSqlSchemaGuard &sql_schema_guard, - const TableItem &table, +int ObMVPrinter::add_max_min_seq_window_to_select_list(const TableItem &table, + const TableItem &source_table, + ObRawExpr *sequence_expr, + ObIArray &select_items, + bool need_win_max_col, + bool need_win_min_col) +{ + int ret = OB_SUCCESS; + SelectItem *sel_item = NULL; + ObRawExpr *win_max_expr = NULL; + ObRawExpr *win_min_expr = NULL; + if (!need_win_max_col && !need_win_min_col) { + /* do nothing */ + } else if (OB_FAIL(gen_max_min_seq_window_func_exprs(table, source_table, sequence_expr, win_max_expr, win_min_expr))) { + LOG_WARN("failed to gen max min seq window func exprs", K(ret)); + } + + if (OB_SUCC(ret) && need_win_max_col) { + if (OB_ISNULL(sel_item = select_items.alloc_place_holder())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("Allocate select item from array error", K(ret)); + } else { + sel_item->expr_ = win_max_expr; + sel_item->is_real_alias_ = true; + sel_item->alias_name_ = WIN_MAX_SEQ_COL_NAME; + } + } + + if (OB_SUCC(ret) && need_win_min_col) { + if (OB_ISNULL(sel_item = select_items.alloc_place_holder())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("Allocate select item from array error", K(ret)); + } else { + sel_item->expr_ = win_min_expr; + sel_item->is_real_alias_ = true; + sel_item->alias_name_ = WIN_MIN_SEQ_COL_NAME; + } + } + return ret; +} + +int ObMVPrinter::gen_max_min_seq_window_func_exprs(const TableItem &table, const TableItem &source_table, ObRawExpr *sequence_expr, ObRawExpr *&win_max_expr, @@ -1509,7 +1617,10 @@ int ObMVPrinter::gen_max_min_seq_window_func_exprs(const ObSqlSchemaGuard &sql_s ObAggFunRawExpr *aggr_max = NULL; ObAggFunRawExpr *aggr_min = NULL; ObRawExpr *col_expr = NULL; - if (OB_FAIL(sql_schema_guard.get_table_schema(source_table.ref_id_, source_data_schema))) { + if (OB_ISNULL(stmt_factory_.get_query_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(stmt_factory_.get_query_ctx())); + } else if (OB_FAIL(stmt_factory_.get_query_ctx()->sql_schema_guard_.get_table_schema(source_table.ref_id_, source_data_schema))) { LOG_WARN("failed to get table schema", K(ret)); } else if (OB_ISNULL(source_data_schema)) { ret = OB_ERR_UNEXPECTED; @@ -1675,26 +1786,17 @@ int ObMVPrinter::gen_delete_for_simple_mjv(ObIArray &dml_stmts) ObDeleteStmt *base_del_stmt = NULL; ObDeleteStmt *del_stmt = NULL; TableItem *mv_table = NULL; - ObSEArray fake_sel_exprs; const ObIArray &orig_select_items = mv_checker_.get_stmt().get_select_items(); if (OB_FAIL(create_simple_stmt(base_del_stmt))) { LOG_WARN("failed to create simple stmt", K(ret)); } else if (OB_FAIL(create_simple_table_item(base_del_stmt, mv_schema_.get_table_name(), mv_table))) { LOG_WARN("failed to create simple table item", K(ret)); - } else if (OB_FAIL(fake_sel_exprs.prepare_allocate(orig_select_items.count()))) { - LOG_WARN("failed to prepare allocate arrays", K(ret)); } else { mv_table->database_name_ = mv_db_name_; const ObIArray &orig_table_items = mv_checker_.get_stmt().get_table_items(); ObRawExpr *semi_filter = NULL; - for (int64_t i = 0; OB_SUCC(ret) && i < orig_select_items.count(); ++i) { - if (OB_FAIL(create_simple_column_expr(mv_table->get_table_name(), orig_select_items.at(i).alias_name_, - mv_table->table_id_, fake_sel_exprs.at(i)))) { - LOG_WARN("failed to create simple column exprs", K(ret)); - } - } for (int64_t i = 0; OB_SUCC(ret) && i < orig_table_items.count(); ++i) { - if (OB_FAIL(gen_exists_cond_for_mjv(fake_sel_exprs, orig_table_items.at(i), true, semi_filter))) { + if (OB_FAIL(gen_exists_cond_for_table(orig_table_items.at(i), mv_table, true, true, semi_filter))) { LOG_WARN("failed to create simple column exprs", K(ret)); } else if (orig_table_items.count() - 1 == i) { if (OB_FAIL(base_del_stmt->get_condition_exprs().push_back(semi_filter))) { @@ -1727,7 +1829,7 @@ int ObMVPrinter::gen_insert_into_select_for_simple_mjv(ObIArray &dml const ObIArray &orig_select_items = mv_checker_.get_stmt().get_select_items(); const ObIArray &orig_table_items = mv_checker_.get_stmt().get_table_items(); if (OB_FAIL(gen_access_delta_data_for_simple_mjv(access_delta_stmts))) { - LOG_WARN("failed to generate ", K(ret)); + LOG_WARN("failed to generate access delta data for simple mjv", K(ret)); } else if (OB_UNLIKELY(orig_table_items.count() != access_delta_stmts.count())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected params", K(ret), K(orig_table_items.count()), K(access_delta_stmts.count())); @@ -1785,17 +1887,16 @@ int ObMVPrinter::gen_access_mv_data_for_simple_mjv(ObSelectStmt *&sel_stmt) mv_table->database_name_ = mv_db_name_; ObIArray &select_items = sel_stmt->get_select_items(); ObIArray &conds = sel_stmt->get_condition_exprs(); - ObSEArray select_exprs; for (int64_t i = 0; OB_SUCC(ret) && i < select_items.count(); ++i) { if (OB_FAIL(create_simple_column_expr(mv_table->get_table_name(), orig_select_items.at(i).alias_name_, mv_table->table_id_, select_items.at(i).expr_))) { LOG_WARN("failed to create simple column exprs", K(ret)); - } else if (OB_FAIL(select_exprs.push_back(select_items.at(i).expr_))) { - LOG_WARN("failed to push back", K(ret)); + } else { + select_items.at(i).alias_name_ = orig_select_items.at(i).alias_name_; } } for (int64_t i = 0; OB_SUCC(ret) && i < orig_table_items.count(); ++i) { - if (OB_FAIL(gen_exists_cond_for_mjv(select_exprs, orig_table_items.at(i), false, conds.at(i)))) { + if (OB_FAIL(gen_exists_cond_for_table(orig_table_items.at(i), mv_table, false, true, conds.at(i)))) { LOG_WARN("failed to create simple column exprs", K(ret)); } } @@ -1803,11 +1904,13 @@ int ObMVPrinter::gen_access_mv_data_for_simple_mjv(ObSelectStmt *&sel_stmt) return ret; } +// rowkeys must exist on source_table. // generate: not exists (select 1 from mlog$_t1 t1 where mv.t1_pk = t1.pk and ora_rowscn > last_refresh_scn(mv_id)) -int ObMVPrinter::gen_exists_cond_for_mjv(const ObIArray &upper_sel_exprs, - const TableItem *source_table, - bool is_exists, - ObRawExpr *&exists_expr) +int ObMVPrinter::gen_exists_cond_for_table(const TableItem *source_table, + const TableItem *outer_table, + const bool is_exists, + const bool use_orig_sel_alias, + ObRawExpr *&exists_expr) { int ret = OB_SUCCESS; exists_expr = NULL; @@ -1817,62 +1920,102 @@ int ObMVPrinter::gen_exists_cond_for_mjv(const ObIArray &upper_sel_e ObSelectStmt *subquery = NULL; TableItem *mlog_table = NULL; SelectItem sel_item; + ObSEArray rowkey_column_ids; + const ObTableSchema *source_table_schema = NULL; sel_item.expr_ = exprs_.int_one_; - const ObIArray &select_items = mv_checker_.get_stmt().get_select_items(); - if (OB_UNLIKELY(upper_sel_exprs.count() != select_items.count()) || OB_ISNULL(source_table)) { + if (OB_ISNULL(outer_table) || OB_ISNULL(source_table) || OB_ISNULL(stmt_factory_.get_query_ctx())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected params", K(ret), K(upper_sel_exprs.count()), K(select_items.count()), K(source_table)); - } else if (OB_FAIL(create_simple_stmt(subquery))) { - LOG_WARN("failed to create simple stmt", K(ret)); + LOG_WARN("unexpected null", K(ret), K(outer_table), K(source_table), K(stmt_factory_.get_query_ctx())); } else if (OB_FAIL(mv_checker_.get_mlog_table_schema(source_table, mlog_schema))) { LOG_WARN("failed to get mlog schema", K(ret), KPC(source_table)); + } else if (OB_FAIL(stmt_factory_.get_query_ctx()->sql_schema_guard_.get_table_schema(source_table->ref_id_, source_table_schema))) { + LOG_WARN("failed to get table schema", K(ret)); } else if (OB_FAIL(expr_factory_.create_raw_expr(T_REF_QUERY, query_ref_expr)) || OB_FAIL(expr_factory_.create_raw_expr(is_exists ? T_OP_EXISTS : T_OP_NOT_EXISTS, exists_op_expr))) { LOG_WARN("failed to create raw expr", K(ret)); - } else if (OB_ISNULL(query_ref_expr) || OB_ISNULL(exists_op_expr) || OB_ISNULL(mlog_schema)) { + } else if (OB_ISNULL(query_ref_expr) || OB_ISNULL(exists_op_expr) || OB_ISNULL(mlog_schema) + || OB_ISNULL(source_table_schema)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected NULL", K(ret), K(query_ref_expr), K(exists_op_expr), K(mlog_schema)); + LOG_WARN("unexpected NULL", K(ret), K(query_ref_expr), K(exists_op_expr), K(mlog_schema), K(source_table_schema)); + } else if (OB_UNLIKELY(source_table_schema->is_heap_table())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(source_table_schema)); } else if (OB_FAIL(exists_op_expr->add_param_expr(query_ref_expr))) { LOG_WARN("failed to add param expr", K(ret)); + } else if (OB_FAIL(create_simple_stmt(subquery))) { + LOG_WARN("failed to create simple stmt", K(ret)); } else if (OB_FAIL(create_simple_table_item(subquery, mlog_schema->get_table_name_str(), mlog_table))) { LOG_WARN("failed to create simple table item", K(ret)); } else if (OB_FAIL(subquery->get_select_items().push_back(sel_item))) { LOG_WARN("failed to push back not exists expr", K(ret)); } else if (OB_FAIL(gen_delta_table_view_conds(*mlog_table, subquery->get_condition_exprs()))) { LOG_WARN("failed to generate delta table view conds", K(ret)); + } else if (OB_FAIL(source_table_schema->get_rowkey_column_ids(rowkey_column_ids))) { + LOG_WARN("failed to get rowkey column ids", KR(ret)); } else { exists_expr = exists_op_expr; mlog_table->database_name_ = source_table->database_name_; query_ref_expr->set_ref_stmt(subquery); - ObRawExpr *expr = NULL; - ObColumnRefRawExpr *col_expr = NULL; - ObSEArray rowkeys; - for (int64_t i = 0; OB_SUCC(ret) && i < select_items.count(); ++i) { - int64_t idx = OB_INVALID_INDEX; - if (OB_ISNULL(expr = select_items.at(i).expr_)) { + const ObColumnSchemaV2 *rowkey_column = NULL; + ObRawExpr *inner_expr = NULL; + ObRawExpr *outer_expr = NULL; + ObRawExpr *filter = NULL; + const ObString *orig_sel_alias = NULL; + for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_column_ids.count(); ++i) { + orig_sel_alias = NULL; + if (OB_ISNULL(rowkey_column = source_table_schema->get_column_schema(rowkey_column_ids.at(i)))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", K(ret), K(i), K(select_items)); - } else if (!expr->is_column_ref_expr()) { - /* do nothing */ - } else if (OB_FALSE_IT(col_expr = static_cast(expr))) { - } else if (!col_expr->is_rowkey_column() || col_expr->get_table_id() != source_table->table_id_) { - /* do nothing */ - } else if (OB_FAIL(add_var_to_array_no_dup(rowkeys, expr, &idx))) { - LOG_WARN("failed to add_var to array no dup", K(ret)); - } else if (OB_INVALID_INDEX == idx) { - /* do nothing */ - } else if (OB_FAIL(create_simple_column_expr(mlog_table->get_table_name(), col_expr->get_column_name(), mlog_table->table_id_, expr))) { - LOG_WARN("failed to create simple column exprs", K(ret)); - } else if (OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, T_OP_EQ, upper_sel_exprs.at(i), expr, expr))) { + LOG_WARN("unexpected null", K(ret), K(rowkey_column)); + } else if (use_orig_sel_alias && + OB_FAIL(get_column_name_from_origin_select_items(source_table->table_id_, + rowkey_column_ids.at(i), + orig_sel_alias))) { + LOG_WARN("failed to get column_name from origin select_items", K(ret)); + } else if (OB_FAIL(create_simple_column_expr(mlog_table->get_table_name(), rowkey_column->get_column_name_str(), mlog_table->table_id_, inner_expr)) + || OB_FAIL(create_simple_column_expr(outer_table->get_table_name(), use_orig_sel_alias ? *orig_sel_alias : rowkey_column->get_column_name_str(), outer_table->table_id_, outer_expr))) { + LOG_WARN("failed to create simple column expr", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, T_OP_EQ, outer_expr, inner_expr, filter))) { LOG_WARN("failed to build equal expr", K(ret)); - } else if (OB_FAIL(subquery->get_condition_exprs().push_back(expr))) { - LOG_WARN("failed to push back null safe equal expr", K(ret)); + } else if (OB_FAIL(subquery->get_condition_exprs().push_back(filter))) { + LOG_WARN("failed to push back equal expr", K(ret)); } } } return ret; } +int ObMVPrinter::get_column_name_from_origin_select_items(const uint64_t table_id, + const uint64_t column_id, + const ObString *&col_name) +{ + int ret = OB_SUCCESS; + col_name = NULL; + const ObIArray &select_items = mv_checker_.get_stmt().get_select_items(); + ObColumnRefRawExpr *col_expr = NULL; + for (int64_t i = 0; OB_SUCC(ret) && NULL == col_name && i < select_items.count(); ++i) { + int64_t idx = OB_INVALID_INDEX; + if (OB_ISNULL(select_items.at(i).expr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(i), K(select_items.at(i))); + } else if (NULL == (col_expr = dynamic_cast(select_items.at(i).expr_))) { + /* do nothing */ + } else if (col_expr->get_table_id() != table_id || col_expr->get_column_id() != column_id) { + /* do nothing */ + } else { + col_name = &select_items.at(i).alias_name_; + if (col_name->empty()) { + col_name = &col_expr->get_column_name(); + } + } + } + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(col_name) || OB_UNLIKELY(col_name->empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get column name", K(ret), KPC(col_name)); + } + return ret; +} + int ObMVPrinter::gen_access_delta_data_for_simple_mjv(ObIArray &access_delta_stmts) { int ret = OB_SUCCESS; @@ -1913,100 +2056,156 @@ int ObMVPrinter::prepare_gen_access_delta_data_for_simple_mjv(ObSelectStmt *&bas { int ret = OB_SUCCESS; base_delta_stmt = NULL; + ObRawExprCopier copier(expr_factory_); + ObSemiToInnerHint *semi_to_inner_hint = NULL; + ObSEArray conflict_hints; const ObIArray &orig_table_items = mv_checker_.get_stmt().get_table_items(); if (OB_FAIL(semi_filters.prepare_allocate(orig_table_items.count())) || OB_FAIL(anti_filters.prepare_allocate(orig_table_items.count()))) { LOG_WARN("failed to prepare allocate arrays", K(ret), K(orig_table_items.count())); } else if (OB_FAIL(create_simple_stmt(base_delta_stmt))) { LOG_WARN("failed to create simple stmt", K(ret)); + } else if (OB_FAIL(construct_table_items_for_simple_mjv_delta_data(base_delta_stmt))) { + LOG_WARN("failed to construct table items for simple mjv delta data", K(ret)); + } else if (OB_FAIL(init_expr_copier_for_stmt(*base_delta_stmt, copier))) { + LOG_WARN("failed to init expr copier for stmt", K(ret)); + } else if (OB_FAIL(ObQueryHint::create_hint(&alloc_, T_SEMI_TO_INNER, semi_to_inner_hint))) { + LOG_WARN("failed to create hint", K(ret)); + } else if (OB_FAIL(base_delta_stmt->get_stmt_hint().merge_hint(*semi_to_inner_hint, + ObHintMergePolicy::HINT_DOMINATED_EQUAL, + conflict_hints))) { + LOG_WARN("failed to merge hint", K(ret)); + } else if (OB_FAIL(construct_from_items_for_simple_mjv_delta_data(copier, *base_delta_stmt))) { + LOG_WARN("failed to construct from items for simple mjv delta data", K(ret)); + } else if (OB_FAIL(copier.copy_on_replace(mv_checker_.get_stmt().get_condition_exprs(), base_delta_stmt->get_condition_exprs()))) { + LOG_WARN("failed to deep copy where conditions", K(ret)); } else { + SelectItem sel_item; + const ObIArray &cur_table_items = base_delta_stmt->get_table_items(); const ObIArray &orig_select_items = mv_checker_.get_stmt().get_select_items(); ObIArray &select_items = base_delta_stmt->get_select_items(); - ObIArray &cur_table_items = base_delta_stmt->get_table_items(); - ObSEArray column_items; - SelectItem sel_item; - ObRawExpr *old_col = NULL; - ObRawExpr *new_col = NULL; - const TableItem *orig_table = NULL; - TableItem *table = NULL; - ObRawExprCopier copier(expr_factory_); - ObSemiToInnerHint *semi_to_inner_hint = NULL; - ObSEArray conflict_hints; - ObSEArray select_exprs; - for (int64_t i = 0; OB_SUCC(ret) && i < orig_table_items.count(); ++i) { - column_items.reuse(); - if (OB_ISNULL(orig_table = orig_table_items.at(i))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", K(ret), K(i), K(orig_table_items)); - } else if (OB_FAIL(mv_checker_.get_stmt().get_column_items(orig_table->table_id_, column_items))) { - LOG_WARN("failed to get column items", K(ret)); - } else if (OB_FAIL(create_simple_table_item(base_delta_stmt, orig_table->table_name_, table))) { - LOG_WARN("failed to create simple table item", K(ret)); - } else { - table->alias_name_ = orig_table->alias_name_; - table->synonym_name_ = orig_table->synonym_name_; - table->database_name_ = orig_table->database_name_; - table->synonym_db_name_ = orig_table->synonym_db_name_; - if (!for_rt_expand_) { - table->flashback_query_expr_ = exprs_.refresh_scn_; - table->flashback_query_type_ = TableItem::USING_SCN; - } - } - for (int64_t j = 0; OB_SUCC(ret) && j < column_items.count(); ++j) { - if (OB_FAIL(create_simple_column_expr(table->get_table_name(), column_items.at(j).column_name_, - table->table_id_, new_col))) { - LOG_WARN("failed to create simple column expr", K(ret)); - } else if (OB_FAIL(copier.add_replaced_expr(column_items.at(j).expr_, new_col))) { - LOG_WARN("failed to add replace pair", K(ret)); - } - } + if (OB_UNLIKELY(cur_table_items.count() != orig_table_items.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected table items count", K(ret), K(cur_table_items.count()), K(orig_table_items.count())); } for (int64_t i = 0; OB_SUCC(ret) && i < orig_select_items.count(); ++i) { sel_item.is_real_alias_ = true; sel_item.alias_name_ = orig_select_items.at(i).alias_name_; if (OB_FAIL(copier.copy_on_replace(orig_select_items.at(i).expr_, sel_item.expr_))) { LOG_WARN("failed to generate group by exprs", K(ret)); - } else if (OB_FAIL(select_items.push_back(sel_item)) - || OB_FAIL(select_exprs.push_back(sel_item.expr_))) { + } else if (OB_FAIL(select_items.push_back(sel_item))) { LOG_WARN("failed to pushback", K(ret)); } } for (int64_t i = 0; OB_SUCC(ret) && i < orig_table_items.count(); ++i) { - if (OB_FAIL(gen_exists_cond_for_mjv(select_exprs, orig_table_items.at(i), true, semi_filters.at(i)))) { + if (OB_FAIL(gen_exists_cond_for_table(orig_table_items.at(i), + cur_table_items.at(i), + true, false, + semi_filters.at(i)))) { LOG_WARN("failed to generate exists filter", K(ret)); - } else if (OB_FAIL(gen_exists_cond_for_mjv(select_exprs, orig_table_items.at(i), false, anti_filters.at(i)))) { + } else if (OB_FAIL(gen_exists_cond_for_table(orig_table_items.at(i), + cur_table_items.at(i), + false, false, + anti_filters.at(i)))) { LOG_WARN("failed to generate not exists filter", K(ret)); } } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(ObQueryHint::create_hint(&alloc_, T_SEMI_TO_INNER, semi_to_inner_hint))) { - LOG_WARN("failed to create hint", K(ret)); - } else if (OB_FAIL(base_delta_stmt->get_stmt_hint().merge_hint(*semi_to_inner_hint, - ObHintMergePolicy::HINT_DOMINATED_EQUAL, - conflict_hints))) { - LOG_WARN("failed to merge hint", K(ret)); - } else if (OB_FAIL(base_delta_stmt->deep_copy_join_tables(alloc_, copier, mv_checker_.get_stmt()))) { - LOG_WARN("failed to deep copy join tables", K(ret)); - } else if (OB_FAIL(copier.copy_on_replace(mv_checker_.get_stmt().get_condition_exprs(), base_delta_stmt->get_condition_exprs()))) { - LOG_WARN("failed to deep copy where conditions", K(ret)); - } else if (OB_FAIL(base_delta_stmt->get_from_items().assign(mv_checker_.get_stmt().get_from_items()))) { - LOG_WARN("failed to assign from items", K(ret)); + } + return ret; +} + +int ObMVPrinter::construct_table_items_for_simple_mjv_delta_data(ObSelectStmt *stmt) +{ + int ret = OB_SUCCESS; + const TableItem *orig_table = NULL; + TableItem *table = NULL; + const ObIArray &orig_table_items = mv_checker_.get_stmt().get_table_items(); + for (int64_t i = 0; OB_SUCC(ret) && i < orig_table_items.count(); ++i) { + if (OB_ISNULL(orig_table = orig_table_items.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(i), K(orig_table_items)); + } else if (OB_FAIL(create_simple_table_item(stmt, orig_table->table_name_, table, NULL, false))) { + LOG_WARN("failed to create simple table item", K(ret)); } else { - // for non joined table, adjust table id in from item - ObIArray &from_items = base_delta_stmt->get_from_items(); - int64_t idx = OB_INVALID_INDEX; - for (int64_t i = 0; OB_SUCC(ret) && i < from_items.count(); ++i) { - if (from_items.at(i).is_joined_) { - /* do nothing */ - } else if (OB_FAIL(mv_checker_.get_stmt().get_table_item_idx(from_items.at(i).table_id_, idx))) { - LOG_WARN("failed to get table item", K(ret)); - } else if (OB_UNLIKELY(idx < 0 || idx >= base_delta_stmt->get_table_size()) - || OB_ISNULL(base_delta_stmt->get_table_item(idx))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected idx", K(ret), K(idx), K(base_delta_stmt->get_table_size())); - } else { - from_items.at(i).table_id_ = base_delta_stmt->get_table_item(idx)->table_id_; - } + set_info_for_simple_table_item(*table, *orig_table); + } + } + return ret; +} + +void ObMVPrinter::set_info_for_simple_table_item(TableItem &table, const TableItem &source_table) +{ + table.alias_name_ = source_table.alias_name_; + table.synonym_name_ = source_table.synonym_name_; + table.database_name_ = source_table.database_name_; + table.synonym_db_name_ = source_table.synonym_db_name_; + if (!for_rt_expand_) { + table.flashback_query_expr_ = exprs_.refresh_scn_; + table.flashback_query_type_ = TableItem::USING_SCN; + } +} + +int ObMVPrinter::init_expr_copier_for_stmt(ObSelectStmt &target_stmt, ObRawExprCopier &copier) +{ + int ret = OB_SUCCESS; + const ObIArray &orig_table_items = mv_checker_.get_stmt().get_table_items(); + const ObIArray &target_table_items = target_stmt.get_table_items(); + ObSEArray column_items; + const TableItem *orig_table = NULL; + const TableItem *target_table = NULL; + ObRawExpr *new_col = NULL; + if (OB_UNLIKELY(orig_table_items.count() != target_table_items.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected table items", K(ret), K(orig_table_items.count()), K(target_table_items.count())); + } + + for (int64_t i = 0; OB_SUCC(ret) && i < orig_table_items.count(); ++i) { + column_items.reuse(); + if (OB_ISNULL(orig_table = orig_table_items.at(i)) + || OB_ISNULL(target_table = target_table_items.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(i), K(orig_table), K(target_table)); + } else if (OB_FAIL(mv_checker_.get_stmt().get_column_items(orig_table->table_id_, column_items))) { + LOG_WARN("failed to get column items", K(ret)); + } + + for (int64_t j = 0; OB_SUCC(ret) && j < column_items.count(); ++j) { + if (OB_FAIL(create_simple_column_expr(target_table->get_table_name(), + column_items.at(j).column_name_, + target_table->table_id_, new_col))) { + LOG_WARN("failed to create simple column expr", K(ret)); + } else if (OB_FAIL(copier.add_replaced_expr(column_items.at(j).expr_, new_col))) { + LOG_WARN("failed to add replace pair", K(ret)); + } + } + } + return ret; +} + +// generate joined table and from item +int ObMVPrinter::construct_from_items_for_simple_mjv_delta_data(ObRawExprCopier &copier, + ObSelectStmt &target_stmt) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(target_stmt.deep_copy_join_tables(alloc_, copier, mv_checker_.get_stmt()))) { + LOG_WARN("failed to deep copy join tables", K(ret)); + } else if (OB_FAIL(target_stmt.get_from_items().assign(mv_checker_.get_stmt().get_from_items()))) { + LOG_WARN("failed to assign from items", K(ret)); + } else { + // for non joined table, adjust table id in from item + ObIArray &from_items = target_stmt.get_from_items(); + int64_t idx = OB_INVALID_INDEX; + for (int64_t i = 0; OB_SUCC(ret) && i < from_items.count(); ++i) { + if (from_items.at(i).is_joined_) { + /* do nothing */ + } else if (OB_FAIL(mv_checker_.get_stmt().get_table_item_idx(from_items.at(i).table_id_, idx))) { + LOG_WARN("failed to get table item", K(ret)); + } else if (OB_UNLIKELY(idx < 0 || idx >= target_stmt.get_table_size()) + || OB_ISNULL(target_stmt.get_table_item(idx))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected idx", K(ret), K(idx), K(target_stmt.get_table_size())); + } else { + from_items.at(i).table_id_ = target_stmt.get_table_item(idx)->table_id_; } } } @@ -2051,5 +2250,279 @@ int ObMVPrinter::gen_one_access_delta_data_for_simple_mjv(const ObSelectStmt &ba return ret; } +int ObMVPrinter::gen_update_insert_delete_for_simple_join_mav(ObIArray &dml_stmts) +{ + int ret = OB_SUCCESS; + dml_stmts.reuse(); + ObSEArray inner_delta_mavs; + if (OB_FAIL(gen_inner_delta_mav_for_simple_join_mav(inner_delta_mavs))) { + LOG_WARN("failed to gen inner delta mav for simple join mav", K(ret)); + } else { + // zhanyue todo: call gen_merge_for_simple_mav once, and assign stmt, adjust inner_delta_mavs + ObUpdateStmt *update_stmt = NULL; + ObInsertStmt *insert_stmt = NULL; + ObSEArray values; + for (int64_t i = 0; OB_SUCC(ret) && i < inner_delta_mavs.count(); ++i) { + // zhanyue todo: call gen_merge_for_simple_mav once, and assign stmt, adjust inner_delta_mavs + values.reuse(); + if (OB_FAIL(gen_insert_for_mav(inner_delta_mavs.at(i), values, insert_stmt))) { + LOG_WARN("failed to gen insert for mav", K(ret)); + } else if (OB_FAIL(gen_update_for_mav(inner_delta_mavs.at(i), insert_stmt->get_values_desc(), + values, update_stmt))) { + LOG_WARN("failed to gen update for mav", K(ret)); + } else if (OB_FAIL(dml_stmts.push_back(update_stmt)) // pushback and execute in this ordering + || OB_FAIL(dml_stmts.push_back(insert_stmt))) { + LOG_WARN("failed to push back", K(ret)); + } + } + + if (OB_SUCC(ret) && !mv_checker_.get_stmt().is_scala_group_by()) { + ObDeleteStmt *delete_stmt = NULL; + if (OB_ISNULL(insert_stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(insert_stmt)); + } else if (OB_FAIL(gen_delete_for_mav(insert_stmt->get_values_desc(), delete_stmt))) { + LOG_WARN("failed gen delete for mav", K(ret)); + } else if (OB_FAIL(dml_stmts.push_back(delete_stmt))) { // pushback and execute in this ordering + LOG_WARN("failed to push back", K(ret)); + } + } + } + return ret; +} + +int ObMVPrinter::gen_merge_for_simple_join_mav(ObIArray &dml_stmts) +{ + int ret = OB_SUCCESS; + dml_stmts.reuse(); + ObSEArray inner_delta_mavs; + if (OB_FAIL(gen_inner_delta_mav_for_simple_join_mav(inner_delta_mavs))) { + LOG_WARN("failed to gen inner delta mav for simple join mav", K(ret)); + } else { + ObMergeStmt *merge_stmt = NULL; + for (int64_t i = 0; OB_SUCC(ret) && i < inner_delta_mavs.count(); ++i) { + // zhanyue todo: call gen_merge_for_simple_mav once, and assign stmt, adjust inner_delta_mavs + if (OB_FAIL(gen_merge_for_simple_mav_use_delta_view(inner_delta_mavs.at(i), merge_stmt))) { + LOG_WARN("failed to gen merge for simple mav", K(ret)); + } else if (OB_FAIL(dml_stmts.push_back(merge_stmt))) { + LOG_WARN("failed to push back stmt", K(ret)); + } + } + } + return ret; +} + +int ObMVPrinter::gen_inner_delta_mav_for_simple_join_mav(ObIArray &inner_delta_mavs) +{ + int ret = OB_SUCCESS; + inner_delta_mavs.reuse(); + ObSEArray delta_datas; + ObSEArray pre_datas; + const ObIArray &source_tables = mv_checker_.get_stmt().get_table_items(); + const TableItem *source_table = NULL; + if (OB_FAIL(inner_delta_mavs.prepare_allocate(source_tables.count())) + || OB_FAIL(pre_datas.prepare_allocate(source_tables.count())) + || OB_FAIL(delta_datas.prepare_allocate(source_tables.count()))) { + LOG_WARN("failed to prepare allocate ObSelectStmt pointer arrays", K(ret), K(source_tables.count())); + } + + for (int64_t i = 0; OB_SUCC(ret) && i < source_tables.count(); ++i) { + pre_datas.at(i) = NULL; + delta_datas.at(i) = NULL; + if (OB_ISNULL(source_table = source_tables.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(i), K(source_table)); + } else if (OB_FAIL(gen_delta_data_access_stmt(*source_tables.at(i), delta_datas.at(i)))) { + LOG_WARN("failed to gen delta data access stmt", K(ret)); + } else if (0 < i && OB_FAIL(gen_pre_data_access_stmt(*source_tables.at(i), pre_datas.at(i)))) { + LOG_WARN("failed to gen pre data access stmt", K(ret)); + } + } + + for (int64_t i = 0; OB_SUCC(ret) && i < source_tables.count(); ++i) { + if (OB_FAIL(gen_inner_delta_mav_for_simple_join_mav(i, delta_datas, pre_datas, inner_delta_mavs.at(i)))) { + LOG_WARN("failed to gen inner delta mav for simple join mav", K(ret)); + } + } + return ret; +} + +int ObMVPrinter::gen_inner_delta_mav_for_simple_join_mav(const int64_t inner_delta_no, + const ObIArray &all_delta_datas, + const ObIArray &all_pre_datas, + ObSelectStmt *&inner_delta_mav) +{ + int ret = OB_SUCCESS; + inner_delta_mav = NULL; + ObRawExprCopier copier(expr_factory_); + const TableItem *delta_table = NULL; + if (OB_FAIL(create_simple_stmt(inner_delta_mav))) { + LOG_WARN("failed to create simple stmt", K(ret)); + } else if (OB_FAIL(construct_table_items_for_simple_join_mav_delta_data(inner_delta_no, + all_delta_datas, + all_pre_datas, + inner_delta_mav))) { + LOG_WARN("failed to construct table items for simple join mav delta data", K(ret)); + } else if (OB_UNLIKELY(inner_delta_mav->get_table_size() <= inner_delta_no) + || OB_ISNULL(delta_table = inner_delta_mav->get_table_item(inner_delta_no))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected table items", K(ret), K(inner_delta_no), K(inner_delta_mav->get_table_items())); + } else if (OB_FAIL(init_expr_copier_for_stmt(*inner_delta_mav, copier))) { + LOG_WARN("failed to init expr copier for stmt", K(ret)); + } else if (OB_FAIL(construct_from_items_for_simple_mjv_delta_data(copier, *inner_delta_mav))) { + LOG_WARN("failed to construct from items for simple mjv delta data", K(ret)); + } else if (OB_FAIL(copier.copy_on_replace(mv_checker_.get_stmt().get_condition_exprs(), + inner_delta_mav->get_condition_exprs()))) { + LOG_WARN("failed to deep copy where conditions", K(ret)); + } else if (OB_FAIL(copier.copy_on_replace(mv_checker_.get_stmt().get_group_exprs(), + inner_delta_mav->get_group_exprs()))) { + LOG_WARN("failed to generate group by exprs", K(ret)); + } else if (OB_FAIL(gen_simple_mav_delta_mv_select_list(copier, *delta_table, + inner_delta_mav->get_group_exprs(), + inner_delta_mav->get_select_items()))) { + LOG_WARN("failed to gen select list ", K(ret)); + } + return ret; +} + +// mjv: t1 join t2 join t3 +// delta_mjv = delta_t1 join pre_t2 join pre_t3 +// union all t1 join delta_t2 join pre_t3 +// union all t1 join t2 join delta_t3 +// this function generate table items for one branch of union all +int ObMVPrinter::construct_table_items_for_simple_join_mav_delta_data(const int64_t inner_delta_no, + const ObIArray &all_delta_datas, + const ObIArray &all_pre_datas, + ObSelectStmt *&stmt) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(0 > inner_delta_no || inner_delta_no >= all_pre_datas.count() + || all_pre_datas.count() != all_delta_datas.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected params", K(ret), K(inner_delta_no), K(all_pre_datas.count()), K(all_delta_datas.count())); + } else { + const TableItem *orig_table = NULL; + TableItem *table = NULL; + const ObIArray &orig_table_items = mv_checker_.get_stmt().get_table_items(); + ObSelectStmt *view_stmt = NULL; + const uint64_t OB_MAX_SUBQUERY_NAME_LENGTH = 64; + int64_t pos = 0; + char buf[OB_MAX_SUBQUERY_NAME_LENGTH]; + int64_t buf_len = OB_MAX_SUBQUERY_NAME_LENGTH; + for (int64_t i = 0; OB_SUCC(ret) && i < orig_table_items.count(); ++i) { + pos = 0; + view_stmt = inner_delta_no == i + ? all_delta_datas.at(i) + : (inner_delta_no < i ? all_pre_datas.at(i) : NULL); + if (OB_ISNULL(orig_table = orig_table_items.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(i), K(orig_table_items)); + } else if (OB_FAIL(create_simple_table_item(stmt, orig_table->table_name_, table, view_stmt ,false))) { + LOG_WARN("failed to create simple table item", K(ret)); + } else if (inner_delta_no > i) { // access current data + set_info_for_simple_table_item(*table, *orig_table); + } else if (OB_FAIL(BUF_PRINTF(inner_delta_no == i ? "DLT_%.*s" : "PRE_%.*s", + orig_table->get_object_name().length(), + orig_table->get_object_name().ptr()))) { + LOG_WARN("failed to buf print for delta/pre view name", K(ret)); + } else if (OB_FAIL(ob_write_string(alloc_, ObString(pos, buf), table->alias_name_))) { + LOG_WARN("failed to write string", K(ret)); + } + } + } + return ret; +} + +// todo: for multi dml operators on the same rowkey, we need access at most two rows actually. +int ObMVPrinter::gen_delta_data_access_stmt(const TableItem &source_table, + ObSelectStmt *&access_sel) +{ + int ret = OB_SUCCESS; + access_sel = NULL; + const uint64_t mlog_sel_flags = MLOG_EXT_COL_DML_FACTOR; + if (OB_FAIL(gen_delta_table_view(source_table, access_sel, mlog_sel_flags))) { + LOG_WARN("failed to gen delta table view", K(ret)); + } + return ret; +} + +// select * from t where not exists (select 1 from mlog_t +// where old_new = 'I' and seq_no = "MAXSEQ$$" +// and mlog_t.pk <=> t.pk); +// union all +// select ... +// from (select ... from mlog_t where ora_rowscn > xxx and ora_rowscn < xxx) +// where (old_new = 'O' and seq_no = "MINSEQ$$") +// ; +int ObMVPrinter::gen_pre_data_access_stmt(const TableItem &source_table, + ObSelectStmt *&access_sel) +{ + int ret = OB_SUCCESS; + ObSelectStmt *union_stmt = NULL; + ObSelectStmt *unchanged_data_stmt = NULL; + ObSelectStmt *deleted_data_stmt = NULL; + if (OB_FAIL(create_simple_stmt(union_stmt))) { + LOG_WARN("failed to create simple stmt", K(ret)); + } else if (OB_FAIL(gen_unchanged_data_access_stmt(source_table, unchanged_data_stmt))) { + LOG_WARN("failed to unchanged deleted data access stmt ", K(ret)); + } else if (OB_FAIL(gen_deleted_data_access_stmt(source_table, deleted_data_stmt))) { + LOG_WARN("failed to generate deleted data access stmt ", K(ret)); + } else if (OB_FAIL(union_stmt->get_set_query().push_back(unchanged_data_stmt) + || OB_FAIL(union_stmt->get_set_query().push_back(deleted_data_stmt)))) { + LOG_WARN("failed to set set query", K(ret)); + } else { + union_stmt->assign_set_all(); + union_stmt->assign_set_op(ObSelectStmt::UNION); + access_sel = union_stmt; + } + return ret; +} + +int ObMVPrinter::gen_unchanged_data_access_stmt(const TableItem &source_table, + ObSelectStmt *&access_sel) +{ + int ret = OB_SUCCESS; + access_sel = NULL; + TableItem *table = NULL; + ObRawExpr *anti_filter = NULL; + const uint64_t access_sel_flags = 0; + if (OB_FAIL(create_simple_stmt(access_sel))) { + LOG_WARN("failed to create simple stmt", K(ret)); + } else if (OB_FAIL(create_simple_table_item(access_sel, source_table.table_name_, table))) { + LOG_WARN("failed to create simple table item", K(ret)); + } else if (OB_FALSE_IT(set_info_for_simple_table_item(*table, source_table))) { + } else if (OB_FAIL(gen_delta_table_view_select_list(*table, source_table, *access_sel, access_sel_flags))) { + LOG_WARN("failed to generate delta table view select lists", K(ret)); + } else if (OB_FAIL(gen_exists_cond_for_table(&source_table, table, false, false, anti_filter))) { + LOG_WARN("failed to create simple column exprs", K(ret)); + } else if (OB_FAIL(access_sel->get_condition_exprs().push_back(anti_filter))) { + LOG_WARN("failed to push back anti filter", K(ret)); + } + return ret; +} + +int ObMVPrinter::gen_deleted_data_access_stmt(const TableItem &source_table, + ObSelectStmt *&access_sel) +{ + int ret = OB_SUCCESS; + access_sel = NULL; + const uint64_t mlog_sel_flags = MLOG_EXT_COL_OLD_NEW | MLOG_EXT_COL_SEQ | MLOG_EXT_COL_WIN_MIN_SEQ; + const uint64_t access_sel_flags = 0; + ObSelectStmt *mlog_delta_sel = NULL; + TableItem *cur_table = NULL; + if (OB_FAIL(gen_delta_table_view(source_table, mlog_delta_sel, mlog_sel_flags))) { + LOG_WARN("failed to gen delta table view", K(ret)); + } else if (OB_FAIL(create_simple_stmt(access_sel))) { + LOG_WARN("failed to create simple stmt", K(ret)); + } else if (OB_FAIL(create_simple_table_item(access_sel, DELTA_TABLE_VIEW_NAME, cur_table, mlog_delta_sel))) { + LOG_WARN("failed to create simple table item", K(ret)); + } else if (OB_FAIL(gen_delta_table_view_select_list(*cur_table, source_table, *access_sel, access_sel_flags))) { + LOG_WARN("failed to generate delta table view select lists", K(ret)); + } else if (OB_FAIL(append_old_new_row_filter(*cur_table, access_sel->get_condition_exprs(), true, false))) { + LOG_WARN("failed to append old new row filter ", K(ret)); + } + return ret; +} + }//end of namespace sql }//end of namespace oceanbase diff --git a/src/sql/resolver/mv/ob_mv_printer.h b/src/sql/resolver/mv/ob_mv_printer.h index 5a5655553..2de91012e 100644 --- a/src/sql/resolver/mv/ob_mv_printer.h +++ b/src/sql/resolver/mv/ob_mv_printer.h @@ -98,38 +98,70 @@ public: ObMVRefreshableType &refreshable_type); private: + + enum MlogExtColFlag { + MLOG_EXT_COL_OLD_NEW = 0x1, + MLOG_EXT_COL_SEQ = 0x1 << 1, + MLOG_EXT_COL_DML_FACTOR = 0x1 << 2, + MLOG_EXT_COL_WIN_MIN_SEQ = 0x1 << 3, + MLOG_EXT_COL_WIN_MAX_SEQ = 0x1 << 4, + }; + int init(const share::SCN &last_refresh_scn, const share::SCN &refresh_scn); int gen_mv_operator_stmts(ObIArray &dml_stmts); int gen_refresh_dmls_for_mv(ObIArray &dml_stmts); int gen_real_time_view_for_mv(ObSelectStmt *&sel_stmt); int gen_delete_insert_for_simple_mjv(ObIArray &dml_stmts); int gen_insert_into_select_for_simple_mjv(ObIArray &dml_stmts); + int gen_update_insert_delete_for_simple_join_mav(ObIArray &dml_stmts); + int gen_merge_for_simple_join_mav(ObIArray &dml_stmts); int gen_real_time_view_for_simple_mjv(ObSelectStmt *&sel_stmt); + int gen_inner_delta_mav_for_simple_join_mav(ObIArray &inner_delta_mavs); + int gen_inner_delta_mav_for_simple_join_mav(const int64_t inner_delta_no, + const ObIArray &all_delta_datas, + const ObIArray &all_pre_datas, + ObSelectStmt *&inner_delta_mav); + int construct_table_items_for_simple_join_mav_delta_data(const int64_t inner_delta_no, + const ObIArray &all_delta_datas, + const ObIArray &all_pre_datas, + ObSelectStmt *&stmt); + int gen_delta_data_access_stmt(const TableItem &source_table, ObSelectStmt *&access_sel); + int gen_pre_data_access_stmt(const TableItem &source_table, ObSelectStmt *&access_sel); + int gen_unchanged_data_access_stmt(const TableItem &source_table, ObSelectStmt *&access_sel); + int gen_deleted_data_access_stmt(const TableItem &source_table, ObSelectStmt *&access_sel); int gen_delete_for_simple_mjv(ObIArray &dml_stmts); int gen_access_mv_data_for_simple_mjv(ObSelectStmt *&sel_stmt); - int gen_exists_cond_for_mjv(const ObIArray &upper_sel_exprs, - const TableItem *source_table, - bool is_exists, - ObRawExpr *&exists_expr); + int gen_exists_cond_for_table(const TableItem *source_table, + const TableItem *outer_table, + const bool is_exists, + const bool use_orig_sel_alias, + ObRawExpr *&exists_expr); + int get_column_name_from_origin_select_items(const uint64_t table_id, + const uint64_t column_id, + const ObString *&col_name); int gen_access_delta_data_for_simple_mjv(ObIArray &access_delta_stmts); int prepare_gen_access_delta_data_for_simple_mjv(ObSelectStmt *&base_delta_stmt, ObIArray &semi_filters, ObIArray &anti_filters); + int construct_table_items_for_simple_mjv_delta_data(ObSelectStmt *stmt); + void set_info_for_simple_table_item(TableItem &table, const TableItem &source_table); + int init_expr_copier_for_stmt(ObSelectStmt &target_stmt, ObRawExprCopier &copier); + int construct_from_items_for_simple_mjv_delta_data(ObRawExprCopier &copier, + ObSelectStmt &target_stmt); int gen_one_access_delta_data_for_simple_mjv(const ObSelectStmt &base_delta_stmt, const int64_t table_idx, const ObIArray &semi_filters, const ObIArray &anti_filters, ObSelectStmt *&sel_stmt); - int gen_real_time_view_for_simple_mav(ObSelectStmt *&sel_stmt); - int gen_real_time_view_scn_cte(ObSelectStmt &root_stmt); + int gen_real_time_view_for_mav(ObSelectStmt *&sel_stmt); int gen_real_time_view_filter_for_mav(ObSelectStmt &sel_stmt); - int gen_inner_real_time_view_for_mav(ObSelectStmt &sel_stmt, TableItem *&view_table); - int gen_inner_real_time_view_tables_for_mav(ObSelectStmt &sel_stmt); - int gen_inner_real_time_view_select_list_for_mav(ObSelectStmt &sel_stmt); + int gen_inner_real_time_view_for_mav(ObSelectStmt *&inner_rt_view); + int gen_inner_delta_mav_for_mav(ObIArray &inner_delta_mavs); int gen_select_items_for_mav(const ObString &table_name, const uint64_t table_id, ObIArray &select_items); int gen_merge_for_simple_mav(ObMergeStmt *&merge_stmt); + int gen_merge_for_simple_mav_use_delta_view(ObSelectStmt *delta_mav, ObMergeStmt *&merge_stmt); int gen_update_insert_delete_for_simple_mav(ObIArray &dml_stmts); int gen_insert_for_mav(ObSelectStmt *delta_mv_stmt, ObIArray &values, @@ -149,9 +181,6 @@ private: ObDeleteStmt *&delete_stmt); int gen_delete_conds(const ObIArray &mv_columns, ObIArray &conds); - int gen_merge_tables(ObMergeStmt &merge_stmt, - TableItem *&target_table, - TableItem *&source_table); int gen_insert_values_and_desc(const TableItem *target_table, const TableItem *source_table, ObIArray &target_columns, @@ -174,27 +203,46 @@ private: const bool for_mysql_update = false); int gen_merge_conds(ObMergeStmt &merge_stmt); int gen_simple_mav_delta_mv_view(ObSelectStmt *&view_stmt); - int init_expr_copier_for_delta_mv_view(const TableItem &table, ObRawExprCopier &copier); int gen_simple_mav_delta_mv_select_list(ObRawExprCopier &copier, const TableItem &table, const ObIArray &group_by_exprs, ObIArray &select_items); + int gen_simple_join_mav_basic_select_list(const TableItem &table, + ObIArray &select_items, + ObIArray *group_by_exprs); int get_mv_select_item_name(const ObRawExpr *expr, ObString &select_name); int gen_basic_aggr_expr(ObRawExprCopier &copier, ObRawExpr *dml_factor, ObAggFunRawExpr &aggr_expr, ObRawExpr *&aggr_print_expr); int add_nvl_above_exprs(ObRawExpr *expr, ObRawExpr *default_expr, ObRawExpr *&res_expr); - int gen_simple_mav_delta_mv_filter(ObRawExprCopier &copier, - const TableItem &table_item, - ObIArray &filters); - int gen_delta_table_view(const TableItem &source_table, ObSelectStmt *&view_stmt); + int append_old_new_row_filter(const TableItem &table_item, + ObIArray &filters, + const bool get_old_row = true, + const bool get_new_row = true); + int gen_delta_table_view(const TableItem &source_table, + ObSelectStmt *&view_stmt, + const uint64_t ext_sel_flags = UINT64_MAX); int gen_delta_table_view_conds(const TableItem &table, ObIArray &conds); int gen_delta_table_view_select_list(const TableItem &table, const TableItem &source_table, - ObSelectStmt &stmt); - int gen_max_min_seq_window_func_exprs(const ObSqlSchemaGuard &sql_schema_guard, - const TableItem &table, + ObSelectStmt &stmt, + const uint64_t ext_sel_flags = UINT64_MAX); + int add_dml_factor_to_select_list(ObRawExpr *old_new_col, + ObIArray &select_items); + int add_normal_column_to_select_list(const TableItem &table, + const ObString &col_name, + ObIArray &select_items); + int add_normal_column_to_select_list(const TableItem &table, + const TableItem &source_table, + ObIArray &select_items); + int add_max_min_seq_window_to_select_list(const TableItem &table, + const TableItem &source_table, + ObRawExpr *sequence_expr, + ObIArray &select_items, + bool need_win_max_col, + bool need_win_min_col); + int gen_max_min_seq_window_func_exprs(const TableItem &table, const TableItem &source_table, ObRawExpr *sequence_expr, ObRawExpr *&win_max_expr, diff --git a/src/sql/resolver/mv/ob_mv_provider.cpp b/src/sql/resolver/mv/ob_mv_provider.cpp index 681fb09fa..3e55a136b 100644 --- a/src/sql/resolver/mv/ob_mv_provider.cpp +++ b/src/sql/resolver/mv/ob_mv_provider.cpp @@ -303,6 +303,7 @@ int ObMVProvider::generate_mv_stmt(ObIAllocator &alloc, resolver_ctx.query_ctx_ = stmt_factory.get_query_ctx(); ObSelectStmt *sel_stmt = NULL; ObSelectResolver select_resolver(resolver_ctx); + bool is_vars_matched = false; if (OB_ISNULL(resolver_ctx.query_ctx_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret), K(resolver_ctx.query_ctx_)); @@ -312,6 +313,8 @@ int ObMVProvider::generate_mv_stmt(ObIAllocator &alloc, ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected mv schema", K(ret), KPC(mv_schema)); mv_schema = NULL; + } else if (OB_FAIL(check_mview_dep_session_vars(*mv_schema, session_info, true, is_vars_matched))) { + LOG_WARN("failed to check mview dep session vars", K(ret)); } else if (OB_FAIL(ObSQLUtils::generate_view_definition_for_resolve(alloc, session_info.get_local_collation_connection(), mv_schema->get_view_schema(), @@ -337,5 +340,77 @@ int ObMVProvider::generate_mv_stmt(ObIAllocator &alloc, return ret; } +int ObMVProvider::check_mview_dep_session_vars(const ObTableSchema &mv_schema, + const ObSQLSessionInfo &session, + const bool gen_error, + bool &is_vars_matched) +{ + int ret = OB_SUCCESS; + is_vars_matched = false; + ObSEArray local_diff_vars; + ObSEArray cur_var_vals; + if (OB_UNLIKELY(!mv_schema.is_materialized_view())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected table schema", K(ret), K(mv_schema)); + } else if (OB_FAIL(mv_schema.get_local_session_var().get_different_vars_from_session(&session, + local_diff_vars, + cur_var_vals))) { + LOG_WARN("failed to check vars same with session ", K(ret), K(mv_schema.get_local_session_var())); + } else if (local_diff_vars.empty()) { + is_vars_matched = true; + } else if (OB_UNLIKELY(local_diff_vars.count() != cur_var_vals.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected array size", K(ret), K(local_diff_vars.count()), K(cur_var_vals.count())); + } else { + is_vars_matched = false; + ObArenaAllocator alloc; + ObString var_name; + ObString local_var_val; + ObString cur_var_val; + const ObSessionSysVar *sys_var = NULL; + const ObString &mview_name = mv_schema.get_table_name(); + OPT_TRACE_BEGIN_SECTION; + OPT_TRACE_TITLE("some session variables differ from values used when the mview was created: ", mview_name); + if (gen_error) { + LOG_WARN("some session variables differ from values used when the mview was created. ", K(mview_name)); + } else { + LOG_TRACE("some session variables differ from values used when the mview was created. ", K(mview_name)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < local_diff_vars.count(); ++i) { + if (OB_ISNULL(sys_var = local_diff_vars.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(sys_var)); + } else if (OB_FAIL(ObSysVarFactory::get_sys_var_name_by_id(sys_var->type_, var_name))) { + LOG_WARN("get sysvar name failed", K(ret)); + } else if (OB_FAIL(ObSessionSysVar::get_sys_var_val_str(sys_var->type_, sys_var->val_, alloc, local_var_val))) { + LOG_WARN("failed to get sys var str", K(ret)); + } else if (OB_FAIL(ObSessionSysVar::get_sys_var_val_str(sys_var->type_, cur_var_vals.at(i), alloc, cur_var_val))) { + LOG_WARN("failed to get sys var str", K(ret)); + } else { + OPT_TRACE(i, ".", var_name, ", old value:", local_var_val, ", current value:", cur_var_val); + if (gen_error) { + LOG_WARN("session variable changed", K(i), K(var_name), K(local_var_val), K(cur_var_val)); + } else { + LOG_TRACE("session variable changed", K(i), K(var_name), K(local_var_val), K(cur_var_val)); + } + } + } + + OPT_TRACE_END_SECTION; + + if (!gen_error) { + ret = OB_SUCCESS; + } else { + // show user error use the last different sys variables + ret = OB_ERR_SESSION_VAR_CHANGED; + LOG_USER_ERROR(OB_ERR_SESSION_VAR_CHANGED, + var_name.length(), var_name.ptr(), + mv_schema.get_table_name_str().length(), mv_schema.get_table_name_str().ptr(), + local_var_val.length(), local_var_val.ptr()); + } + } + return ret; +} + }//end of namespace sql }//end of namespace oceanbase diff --git a/src/sql/resolver/mv/ob_mv_provider.h b/src/sql/resolver/mv/ob_mv_provider.h index 784015e2e..b21a454dc 100644 --- a/src/sql/resolver/mv/ob_mv_provider.h +++ b/src/sql/resolver/mv/ob_mv_provider.h @@ -49,6 +49,10 @@ public: int get_fast_refresh_operators(const ObIArray *&operators) const; int get_real_time_mv_expand_view(ObIAllocator &alloc, ObString &expand_view) const; int get_mv_dependency_infos(ObIArray &dep_infos) const; + static int check_mview_dep_session_vars(const ObTableSchema &mv_schema, + const ObSQLSessionInfo &session, + const bool gen_error, + bool &is_vars_matched); private: int check_mv_column_type(const ObTableSchema *mv_schema, const ObSelectStmt *view_stmt); int check_mv_column_type(const ObColumnSchemaV2 &org_column, const ObColumnSchemaV2 &cur_column); diff --git a/src/sql/rewrite/ob_expand_aggregate_utils.cpp b/src/sql/rewrite/ob_expand_aggregate_utils.cpp index 60507bf43..7d8b25597 100644 --- a/src/sql/rewrite/ob_expand_aggregate_utils.cpp +++ b/src/sql/rewrite/ob_expand_aggregate_utils.cpp @@ -525,7 +525,8 @@ int ObExpandAggregateUtils::extract_candi_window_aggr(ObSelectStmt *select_stmt, } int ObExpandAggregateUtils::add_aggr_item(ObIArray &new_aggr_items, - ObAggFunRawExpr *&aggr_expr) + ObAggFunRawExpr *&aggr_expr, + const bool need_strict_check /* = true */) { int ret = OB_SUCCESS; if (OB_ISNULL(aggr_expr)) { @@ -537,10 +538,14 @@ int ObExpandAggregateUtils::add_aggr_item(ObIArray &new_aggr_i if (OB_ISNULL(new_aggr_items.at(i))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret), K(new_aggr_items.at(i))); - } else if (aggr_expr->same_as(*new_aggr_items.at(i))) { - aggr_expr = new_aggr_items.at(i); + } else if (need_strict_check) { + if (aggr_expr->same_as(*new_aggr_items.at(i))) { + aggr_expr = new_aggr_items.at(i); + break; + } + } else if (aggr_expr == new_aggr_items.at(i)) { break; - } else {/*do nothing*/} + } } if (OB_SUCC(ret) && i == new_aggr_items.count()) { if (OB_FAIL(new_aggr_items.push_back(aggr_expr))) { @@ -552,7 +557,8 @@ int ObExpandAggregateUtils::add_aggr_item(ObIArray &new_aggr_i } int ObExpandAggregateUtils::add_win_expr(common::ObIArray &new_win_exprs, - ObWinFunRawExpr *&win_expr) + ObWinFunRawExpr *&win_expr, + const bool need_strict_check /* = true */) { int ret = OB_SUCCESS; if (OB_ISNULL(win_expr)) { @@ -564,10 +570,14 @@ int ObExpandAggregateUtils::add_win_expr(common::ObIArray &new if (OB_ISNULL(new_win_exprs.at(i))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret), K(new_win_exprs.at(i))); - } else if (win_expr->same_as(*new_win_exprs.at(i))) { - win_expr = new_win_exprs.at(i); + } else if (need_strict_check) { + if (win_expr->same_as(*new_win_exprs.at(i))) { + win_expr = new_win_exprs.at(i); + break; + } + } else if (win_expr == new_win_exprs.at(i)) { break; - } else {/*do nothing*/} + } } if (OB_SUCC(ret) && i == new_win_exprs.count()) { if (OB_FAIL(new_win_exprs.push_back(win_expr))) { diff --git a/src/sql/rewrite/ob_expand_aggregate_utils.h b/src/sql/rewrite/ob_expand_aggregate_utils.h index 367558765..211a68621 100644 --- a/src/sql/rewrite/ob_expand_aggregate_utils.h +++ b/src/sql/rewrite/ob_expand_aggregate_utils.h @@ -43,10 +43,12 @@ public: ObIArray &new_aggr_items); static int add_aggr_item(common::ObIArray &new_aggr_items, - ObAggFunRawExpr *&aggr_expr); + ObAggFunRawExpr *&aggr_expr, + const bool need_strict_check = true); static int add_win_expr(common::ObIArray &new_win_exprs, - ObWinFunRawExpr *&win_expr); + ObWinFunRawExpr *&win_expr, + const bool need_strict_check = true); private: int extract_candi_aggr(ObDMLStmt *select_stmt, diff --git a/src/sql/rewrite/ob_transform_join_elimination.cpp b/src/sql/rewrite/ob_transform_join_elimination.cpp index 2ea98d9d3..a031c0540 100644 --- a/src/sql/rewrite/ob_transform_join_elimination.cpp +++ b/src/sql/rewrite/ob_transform_join_elimination.cpp @@ -3843,7 +3843,7 @@ int ObTransformJoinElimination::add_is_not_null_if_needed(ObDMLStmt *stmt, } for (int64_t i = 0; OB_SUCC(ret) && i < col_exprs.count(); ++i) { bool is_not_null = true; - ObOpRawExpr *is_not_expr = NULL; + ObRawExpr *is_not_expr = NULL; if (OB_ISNULL(col_exprs.at(i))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret)); @@ -3857,9 +3857,8 @@ int ObTransformJoinElimination::add_is_not_null_if_needed(ObDMLStmt *stmt, LOG_WARN("failed to add param not null constraint", K(ret)); } } else if (OB_FAIL(ObTransformUtils::add_is_not_null(ctx_, - stmt, - col_exprs.at(i), - is_not_expr))) { + col_exprs.at(i), + is_not_expr))) { LOG_WARN("failed to add is not null for col", K(ret)); } else if (OB_FAIL(cond_exprs.push_back(is_not_expr))) { LOG_WARN("failed to add is_not_expr into condition", K(ret)); diff --git a/src/sql/rewrite/ob_transform_min_max.cpp b/src/sql/rewrite/ob_transform_min_max.cpp index a8c3a2c52..30bb5ab0c 100644 --- a/src/sql/rewrite/ob_transform_min_max.cpp +++ b/src/sql/rewrite/ob_transform_min_max.cpp @@ -567,7 +567,7 @@ int ObTransformMinMax::set_child_order_item(ObSelectStmt *stmt, int ObTransformMinMax::set_child_condition(ObSelectStmt *stmt, ObRawExpr *aggr_param) { int ret = OB_SUCCESS; - ObOpRawExpr *not_null_expr = NULL; + ObRawExpr *not_null_expr = NULL; bool is_not_null = false; ObArray constraints; if (OB_ISNULL(stmt) || OB_ISNULL(aggr_param) || OB_ISNULL(ctx_)) { @@ -580,7 +580,7 @@ int ObTransformMinMax::set_child_condition(ObSelectStmt *stmt, ObRawExpr *aggr_p if (OB_FAIL(ObTransformUtils::add_param_not_null_constraint(*ctx_, constraints))) { LOG_WARN("failed to add param not null constraints", K(ret)); } - } else if (OB_FAIL(ObTransformUtils::add_is_not_null(ctx_, stmt, aggr_param, not_null_expr))) { + } else if (OB_FAIL(ObTransformUtils::add_is_not_null(ctx_, aggr_param, not_null_expr))) { LOG_WARN("failed to add is not null", K(ret)); } else if (OB_FAIL(stmt->add_condition_expr(not_null_expr))) { LOG_WARN("failed to add condition expr", K(ret)); diff --git a/src/sql/rewrite/ob_transform_mv_rewrite.cpp b/src/sql/rewrite/ob_transform_mv_rewrite.cpp index 16b576682..65f3c7e6c 100644 --- a/src/sql/rewrite/ob_transform_mv_rewrite.cpp +++ b/src/sql/rewrite/ob_transform_mv_rewrite.cpp @@ -11,10 +11,15 @@ */ #define USING_LOG_PREFIX SQL_REWRITE +#include #include "sql/rewrite/ob_transform_mv_rewrite.h" #include "sql/rewrite/ob_transform_pre_process.h" +#include "sql/rewrite/ob_transform_mv_rewrite_prepare.h" +#include "sql/rewrite/ob_equal_analysis.h" +#include "sql/rewrite/ob_expand_aggregate_utils.h" #include "sql/resolver/dml/ob_select_resolver.h" #include "sql/resolver/expr/ob_raw_expr_wrap_enum_set.h" +#include "sql/privilege_check/ob_privilege_check.h" #include "lib/mysqlclient/ob_mysql_result.h" #include "share/schema/ob_table_schema.h" @@ -28,43 +33,57 @@ int ObTransformMVRewrite::transform_one_stmt(common::ObIArray & bool &trans_happened) { int ret = OB_SUCCESS; + const int64_t MAX_MV_STMT_GEN = 10; trans_happened = false; - const ObDMLStmt *root_stmt = parent_stmts.empty() ? stmt : parent_stmts.at(0).stmt_; - if (OB_ISNULL(stmt)) { + parent_stmts_ = &parent_stmts; + if (OB_ISNULL(ctx_) || OB_ISNULL(stmt) || OB_ISNULL(stmt->get_query_ctx())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(stmt)); + LOG_WARN("get unexpected null", K(ret), K(ctx_), K(stmt)); } else if (!stmt->is_select_stmt()) { // do nothing - } else if (OB_FAIL(prepare_mv_info(root_stmt, stmt))) { - LOG_WARN("failed to prepate mv info", K(ret)); - } else if (OB_FAIL(do_transform(stmt, trans_happened))) { - LOG_WARN("failed to do transform", K(ret)); - } - return ret; -} - -ObTransformMVRewrite::~ObTransformMVRewrite() { - int ret = OB_SUCCESS; - if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->stmt_factory_)) { - // do nothing } else { - for (int64_t i = 0; OB_SUCC(ret) && i < mv_infos_.count(); ++i) { - if (OB_ISNULL(mv_infos_.at(i).view_stmt_)) { + ObSelectStmt *ori_stmt = static_cast(stmt); + ObSelectStmt *new_stmt = NULL; + const ObCollationType cs_type = stmt->get_query_ctx()->get_query_hint().cs_type_; + const ObMVRewriteHint *myhint = static_cast(get_hint(stmt->get_stmt_hint())); + for (int64_t i = 0; OB_SUCC(ret) && i < ctx_->mv_infos_.count(); ++i) { + bool is_happened = false; + bool is_match_hint = true; + MvInfo &mv_info = ctx_->mv_infos_.at(i); + if (NULL != myhint && OB_FAIL(myhint->check_mv_match_hint(cs_type, + mv_info.mv_schema_, + mv_info.db_schema_, + is_match_hint))) { + LOG_WARN("failed to check mv match hint", K(ret)); + } else if (!is_match_hint) { // do nothing - } else if (OB_FAIL(ObTransformUtils::free_stmt(*ctx_->stmt_factory_, mv_infos_.at(i).view_stmt_))) { - LOG_WARN("failed to free stmt", K(ret)); - } else { - mv_infos_.at(i).view_stmt_ = NULL; + } else if (NULL == mv_info.view_stmt_) { + if (++ctx_->mv_stmt_gen_count_ > MAX_MV_STMT_GEN) { + OPT_TRACE("reach the mv stmt generate count limit!"); + } else if (OB_FAIL(ObTransformMVRewritePrepare::generate_mv_stmt(mv_info, ctx_, &mv_temp_query_ctx_))) { + LOG_WARN("failed to generate mv stmt", K(ret), K(mv_info)); + } else if (OB_FAIL(mv_privilege_check(mv_info, mv_info.has_select_priv_))) { + LOG_WARN("failed to check mv privilege", K(ret)); + } else if (!mv_info.has_select_priv_) { + OPT_TRACE("SELECT command is denied to current user for", mv_info.mv_schema_->get_table_name()); + } } - if (OB_FAIL(ret) || OB_ISNULL(mv_infos_.at(i).select_mv_stmt_)) { + + if (OB_FAIL(ret) || !mv_info.has_select_priv_) { // do nothing - } else if (OB_FAIL(ObTransformUtils::free_stmt(*ctx_->stmt_factory_, mv_infos_.at(i).select_mv_stmt_))) { - LOG_WARN("failed to free stmt", K(ret)); - } else { - mv_infos_.at(i).select_mv_stmt_ = NULL; + } else if (OB_FAIL(try_transform_with_one_mv(ori_stmt, mv_info, new_stmt, is_happened))) { + LOG_WARN("failed to try one mv", K(ret)); + } else if (is_happened) { + ori_stmt = new_stmt; + trans_happened = true; + break; } } + if (OB_SUCC(ret) && trans_happened) { + stmt = ori_stmt; + } } + return ret; } int ObTransformMVRewrite::need_transform(const common::ObIArray &parent_stmts, @@ -74,11 +93,9 @@ int ObTransformMVRewrite::need_transform(const common::ObIArray { int ret = OB_SUCCESS; int64_t query_rewrite_enabled = QueryRewriteEnabledType::REWRITE_ENABLED_FALSE; - bool has_mv = false; bool hint_rewrite = false; bool hint_no_rewrite = false; bool is_stmt_valid = false; - uint64_t data_version; need_trans = false; if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->session_info_) || OB_ISNULL(stmt.get_query_ctx())) { ret = OB_ERR_UNEXPECTED; @@ -95,18 +112,14 @@ int ObTransformMVRewrite::need_transform(const common::ObIArray OPT_TRACE("only do mv rewrite in the first iteration"); } else if (stmt.is_select_stmt() && static_cast(&stmt)->is_expanded_mview()) { need_trans = false; - } else if (ctx_->session_info_->get_ddl_info().is_refreshing_mview()) { + } else if (OB_FAIL(check_basic_validity(stmt, is_stmt_valid))) { + LOG_WARN("failed to check basic validity", K(ret)); + } else if (!is_stmt_valid) { need_trans = false; - OPT_TRACE("not a user SQL, skip mv rewrite"); - } else if (stmt.get_query_ctx()->optimizer_features_enable_version_ < COMPAT_VERSION_4_3_1) { + OPT_TRACE("stmt can not do mv rewrite"); + } else if (ctx_->mv_infos_.empty()) { need_trans = false; - OPT_TRACE("optimizer features enable version is lower than 4.3.1, skip mv rewrite"); - } else if (OB_FAIL(GET_MIN_DATA_VERSION(ctx_->session_info_->get_effective_tenant_id(), data_version))) { - LOG_WARN("failed to get data version", K(ret), K(ctx_->session_info_->get_effective_tenant_id())); - } else if (OB_UNLIKELY(data_version < DATA_VERSION_4_3_1_0)) { - // data version lower than 4.3.1 does not have the inner table used to get mv list - need_trans = false; - OPT_TRACE("min data version is lower than 4.3.1, skip mv rewrite"); + OPT_TRACE("there is no mv to perform rewrite"); } else if (OB_FAIL(check_hint_valid(stmt, hint_rewrite, hint_no_rewrite))) { LOG_WARN("failed to check mv rewrite hint", K(ret)); } else if (hint_rewrite) { @@ -120,16 +133,6 @@ int ObTransformMVRewrite::need_transform(const common::ObIArray } else if (QueryRewriteEnabledType::REWRITE_ENABLED_FALSE == query_rewrite_enabled) { need_trans = false; OPT_TRACE("system variable reject mv rewrite"); - } else if (OB_FAIL(check_table_has_mv(stmt, has_mv))) { - LOG_WARN("failed to check table has mv", K(ret)); - } else if (!has_mv) { - need_trans = false; - OPT_TRACE("table does not have mv, no need to rewrite"); - } else if (OB_FAIL(check_basic_validity(stmt, is_stmt_valid))) { - LOG_WARN("failed to check basic validity", K(ret)); - } else if (!is_stmt_valid) { - need_trans = false; - OPT_TRACE("stmt can not do mv rewrite"); } else { need_trans = true; } @@ -157,38 +160,6 @@ int ObTransformMVRewrite::check_hint_valid(const ObDMLStmt &stmt, return ret; } -int ObTransformMVRewrite::check_table_has_mv(const ObDMLStmt &stmt, - bool &has_mv) -{ - int ret = OB_SUCCESS; - has_mv = false; - if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->schema_checker_) || OB_ISNULL(ctx_->session_info_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(ctx_)); - } - for (int64_t i = 0; OB_SUCC(ret) && !has_mv && i < stmt.get_table_size(); ++i) { - const TableItem *table = NULL; - const ObTableSchema *table_schema = NULL; - if (OB_ISNULL(table = stmt.get_table_item(i))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("table item is null", K(ret), K(i), K(stmt.get_table_size())); - } else if (!table->is_basic_table()) { - // do nothing - } else if (OB_FAIL(ctx_->schema_checker_->get_table_schema(ctx_->session_info_->get_effective_tenant_id(), - table->ref_id_, - table_schema))) { - LOG_WARN("failed to get table schema", K(ret), K(i), K(table->ref_id_)); - } else if (OB_ISNULL(table_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("table schema is null", K(ret), K(table->ref_id_)); - } else if (ObTableReferencedByMVFlag::IS_REFERENCED_BY_MV == - ObTableMode::get_table_referenced_by_mv_flag(table_schema->get_table_mode())) { - has_mv = true; - } - } - return ret; -} - int ObTransformMVRewrite::check_basic_validity(const ObDMLStmt &stmt, bool &is_valid) { int ret = OB_SUCCESS; @@ -198,36 +169,33 @@ int ObTransformMVRewrite::check_basic_validity(const ObDMLStmt &stmt, bool &is_v OPT_TRACE("stmt can not do mv rewrite, not a select stmt"); is_valid = false; } else if (stmt.is_set_stmt()) { - OPT_TRACE("stmt can not do mv rewrite, is a set stmt"); + OPT_TRACE("stmt can not do mv rewrite, it is a set stmt"); is_valid = false; } else if (stmt.is_hierarchical_query()) { - OPT_TRACE("stmt can not do mv rewrite, is a hierarchical query"); + OPT_TRACE("stmt can not do mv rewrite, it is a hierarchical query"); is_valid = false; } else if (stmt.is_contains_assignment()) { - OPT_TRACE("stmt can not do mv rewrite, is a contains assignment"); + OPT_TRACE("stmt can not do mv rewrite, it contains assignment"); is_valid = false; } else if (stmt.has_for_update()) { - OPT_TRACE("stmt can not do mv rewrite, is a contains for update"); + OPT_TRACE("stmt can not do mv rewrite, it contains for update"); is_valid = false; } else if (stmt.has_ora_rowscn()) { - OPT_TRACE("stmt can not do mv rewrite, is a contains ora rowscn"); + OPT_TRACE("stmt can not do mv rewrite, it contains ora rowscn"); is_valid = false; } else if (stmt.is_unpivot_select()) { - OPT_TRACE("stmt can not do mv rewrite, is a contains unpivot"); + OPT_TRACE("stmt can not do mv rewrite, it contains unpivot"); is_valid = false; } else if (!stmt.get_pseudo_column_like_exprs().empty()) { - OPT_TRACE("stmt can not do mv rewrite, is a contains pseudo column"); + OPT_TRACE("stmt can not do mv rewrite, it contains pseudo column"); is_valid = false; - } else if (static_cast(&stmt)->has_rollup()) { - OPT_TRACE("stmt can not do mv rewrite, is a contains roll up"); - is_valid = false; - } else if (static_cast(&stmt)->has_select_into()) { - OPT_TRACE("stmt can not do mv rewrite, is a contains select into"); + } else if (!stmt.get_semi_infos().empty()) { + OPT_TRACE("stmt can not do mv rewrite, it contains semi join"); is_valid = false; } else if (OB_FAIL(stmt.get_sequence_exprs(seq_exprs))) { LOG_WARN("failed to get sequence exprs", K(ret)); } else if (!seq_exprs.empty()) { - OPT_TRACE("stmt can not do mv rewrite, is a contains sequence"); + OPT_TRACE("stmt can not do mv rewrite, it contains sequence"); is_valid = false; } return ret; @@ -258,411 +226,214 @@ int ObTransformMVRewrite::construct_transform_hint(ObDMLStmt &stmt, void *trans_ return ret; } -int ObTransformMVRewrite::prepare_mv_info(const ObDMLStmt *root_stmt, - const ObDMLStmt *stmt) +/** + * @brief ObTransformMVRewrite::gen_base_table_map + * + * Generate the table map between from_tables and to_tables based on ref_id. + * + * e.g. INPUT from_tables: t1, t1, t1, t2, t3 + * to_tables: t1, t1, t2, t4 + * + * OUTPUT table_maps: [0, 1, -1, 2, -1], + * [0, -1, 1, 2, -1], + * [1, 0, -1, 2, -1], + * [1, -1, 0, 2, -1], + * [-1, 0, 1, 2, -1], + * [-1, 1, 0, 2, -1] + */ +int ObTransformMVRewrite::gen_base_table_map(const ObIArray &from_tables, + const ObIArray &to_tables, + int64_t max_map_num, + ObIArray> &table_maps) { int ret = OB_SUCCESS; - ObSEArray mv_list; - if (is_mv_info_generated_) { + if (max_map_num <= 0 || from_tables.count() <= 0 || to_tables.count() <= 0) { // do nothing - } else if (OB_FAIL(get_mv_list(root_stmt, mv_list))) { - LOG_WARN("failed to get mv list", K(ret)); - } else if (OB_FAIL(generate_mv_info(stmt, mv_list))) { - LOG_WARN("failed to generate mv info", K(ret)); + } else if (OB_FAIL(table_maps.reserve(max_map_num))) { + LOG_WARN("failed to reserve array", K(ret)); } else { - is_mv_info_generated_ = true; - OPT_TRACE("use", mv_infos_.count(), "materialized view(s) to perform rewrite"); - } - return ret; -} - -int ObTransformMVRewrite::get_mv_list(const ObDMLStmt *root_stmt, - ObIArray &mv_list) -{ - int ret = OB_SUCCESS; - ObMySQLProxy *sql_proxy = NULL; - ObSqlString sql; - sqlclient::ObMySQLResult *mysql_result = NULL; - ObSqlString table_ids; - if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->exec_ctx_) || OB_ISNULL(ctx_->session_info_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(ctx_)); - } else if (OB_FAIL(get_base_table_id_string(root_stmt, table_ids))) { - LOG_WARN("failed to get base table id string", K(ret)); - } else if (OB_ISNULL(sql_proxy = ctx_->exec_ctx_->get_sql_proxy())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is null", K(ret)); - } else if (OB_FAIL(sql.assign_fmt("SELECT DISTINCT MVIEW_ID FROM `%s`.`%s` WHERE TENANT_ID = 0 AND P_OBJ IN (%.*s)", - OB_SYS_DATABASE_NAME, OB_ALL_MVIEW_DEP_TNAME, - static_cast(table_ids.length()), table_ids.ptr()))) { - LOG_WARN("failed to assign sql", K(ret)); - } else { - SMART_VAR(ObMySQLProxy::MySQLResult, res) { - if (OB_FAIL(sql_proxy->read(res, ctx_->session_info_->get_effective_tenant_id(), sql.ptr()))) { - LOG_WARN("execute sql failed", K(ret), K(sql)); - } else if (OB_ISNULL(mysql_result = res.get_result())) { + // prepare the auxiliary data + hash::ObHashMap from_table_num; // map ref_id to the number of tables that have same ref_id in from_tables + hash::ObHashMap to_table_map; // map ref_id to index of to_table_ids array in which table's ref_id is same as hash key + ObSEArray, 4> to_table_ids; // store index of to_tables with the same ref_id + if (OB_FAIL(from_table_num.create(from_tables.count(), "MvRewrite"))) { + LOG_WARN("failed to init stmt map", K(ret)); + } else if (OB_FAIL(to_table_map.create(to_tables.count(), "MvRewrite"))) { + LOG_WARN("failed to init stmt map", K(ret)); + } + // collect the number of from_tables that have same ref_id + for (int64_t i = 0; OB_SUCC(ret) && i < from_tables.count(); ++i) { + const TableItem *from_table = from_tables.at(i); + const int64_t *num = NULL; + if (OB_ISNULL(from_table)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to get result", K(ret)); - } else { - while (OB_SUCC(ret) && OB_SUCC(mysql_result->next())) { - int64_t mv_id = OB_INVALID_ID; - if (OB_FAIL(mysql_result->get_int(0L, mv_id))) { - LOG_WARN("failed to get mv id", K(ret)); - } else if (OB_FAIL(mv_list.push_back(mv_id))) { - LOG_WARN("failed to push back mv id", K(ret), K(mv_id)); - } - } - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - OPT_TRACE("get", mv_list.count(), "materialized view(s)"); - } else if (OB_FAIL(ret)) { - LOG_WARN("failed to get inner sql result", K(ret)); + LOG_WARN("get unexpected null table", K(ret), K(i)); + } else if (!(from_table->is_basic_table() || from_table->is_generated_table()) + || OB_INVALID_ID == from_table->ref_id_) { + // do nothing + } else if (NULL == (num = from_table_num.get(from_table->ref_id_))) { + if (OB_FAIL(from_table_num.set_refactored(from_table->ref_id_, 1))) { + LOG_WARN("failed to set refactored", K(ret), K(from_table->ref_id_)); } + } else if (OB_FAIL(from_table_num.set_refactored(from_table->ref_id_, (*num) + 1, 1))) { + LOG_WARN("failed to push back table rel id", K(ret)); + } + } + // collect to_table index, group by ref_id + for (int64_t i = 0; OB_SUCC(ret) && i < to_tables.count(); ++i) { + const TableItem *to_table = to_tables.at(i); + const int64_t *idx = NULL; + if (OB_ISNULL(to_table)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null table", K(ret), K(i)); + } else if (!(to_table->is_basic_table() || to_table->is_generated_table()) + || OB_INVALID_ID == to_table->ref_id_) { + // do nothing + } else if (NULL == (idx = to_table_map.get(to_table->ref_id_))) { + ObSEArray temp_array; + if (OB_FAIL(temp_array.push_back(i))) { + LOG_WARN("failed to push back rel id", K(ret)); + } else if (OB_FAIL(to_table_ids.push_back(temp_array))) { + LOG_WARN("failed to push back array", K(ret), K(temp_array)); + } else if (OB_FAIL(to_table_map.set_refactored(to_table->ref_id_, to_table_ids.count() - 1))) { + LOG_WARN("failed to set refactored", K(ret), K(to_table->ref_id_), K(to_table_ids)); + } + } else if (OB_FAIL(to_table_ids.at(*idx).push_back(i))) { + LOG_WARN("failed to push back table rel id", K(ret)); } } - } - return ret; -} -int ObTransformMVRewrite::get_base_table_id_string(const ObDMLStmt *stmt, - ObSqlString &table_ids) -{ - int ret = OB_SUCCESS; - ObSEArray table_id_list; - ObSEArray visited_id; - if (OB_ISNULL(stmt)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(stmt)); - } else if (OB_FAIL(get_all_base_table_id(stmt, table_id_list))) { - LOG_WARN("failed to get table ids", K(ret)); - } - for (int64_t i = 0; OB_SUCC(ret) && i < table_id_list.count(); ++i) { - if (is_contain(visited_id, table_id_list.at(i))) { + // recursive generate base table map + ObSqlBitSet<> used_to_table; + ObSEArray empty_map; + if (OB_FAIL(ret)) { // do nothing - } else if (OB_FAIL(visited_id.push_back(table_id_list.at(i)))) { - LOG_WARN("failed to push back table ref id", K(ret)); - } else if (i > 0 && OB_FAIL(table_ids.append(","))) { - LOG_WARN("failed to append comma", K(ret)); - } else if (OB_FAIL(table_ids.append(to_cstring(table_id_list.at(i))))) { - LOG_WARN("failed to append table id", K(ret)); + } else if (OB_FAIL(empty_map.prepare_allocate(from_tables.count()))) { + LOG_WARN("failed to prepare allocate map array", K(ret), K(from_tables.count())); + } else if (OB_FAIL(inner_gen_base_table_map(0, + from_tables, + from_table_num, + to_table_ids, + to_table_map, + used_to_table, + max_map_num, + empty_map, + table_maps))) { + LOG_WARN("failed to generate base table map", K(ret)); + } else if (OB_FAIL(from_table_num.destroy())) { + LOG_WARN("failed to destroy from table num hash map", K(ret)); + } else if (OB_FAIL(to_table_map.destroy())) { + LOG_WARN("failed to destroy to table map hash map", K(ret)); } } return ret; } -int ObTransformMVRewrite::get_all_base_table_id(const ObDMLStmt *stmt, - ObIArray &table_ids) +// generate a base table map for from_table_idx-th from table +int ObTransformMVRewrite::inner_gen_base_table_map(int64_t from_table_idx, + const ObIArray &from_tables, + hash::ObHashMap &from_table_num, + const ObIArray> &to_table_ids, + const hash::ObHashMap &to_table_map, + ObSqlBitSet<> &used_to_table, + int64_t max_map_num, + ObIArray ¤t_map, + ObIArray> &table_maps) { int ret = OB_SUCCESS; - ObSEArray child_stmts; - if (OB_ISNULL(stmt)) { + if (OB_UNLIKELY(from_table_idx < 0)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(stmt)); - } else if (OB_FAIL(stmt->get_child_stmts(child_stmts))) { - LOG_WARN("failed to get child stmts", K(ret)); - } - for (int64_t i = 0; OB_SUCC(ret) && i < stmt->get_table_size(); ++i) { - const TableItem *table = NULL; - if (OB_ISNULL(table = stmt->get_table_item(i))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(i)); - } else if (!table->is_basic_table()) { - // do nothing - } else if (OB_FAIL(table_ids.push_back(table->ref_id_))) { - LOG_WARN("failed to push back table id", K(ret)); - } - } - for (int64_t i = 0; OB_SUCC(ret) && i < child_stmts.count(); ++i) { - if (OB_FAIL(SMART_CALL(get_all_base_table_id(child_stmts.at(i), table_ids)))) { - LOG_WARN("failed to get all base table id", K(ret)); - } - } - return ret; -} - -int ObTransformMVRewrite::generate_mv_info(const ObDMLStmt *stmt, - ObIArray &mv_list) -{ - int ret = OB_SUCCESS; - int64_t query_rewrite_integrity = QueryRewriteIntegrityType::REWRITE_INTEGRITY_ENFORCED; - if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->schema_checker_) - || OB_ISNULL(ctx_->session_info_) || OB_ISNULL(stmt)) { + LOG_WARN("unexpected from table idx", K(ret), K(from_table_idx), K(from_tables)); + } else if (OB_UNLIKELY(current_map.count() != from_tables.count())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(ctx_), K(stmt)); - } else if (OB_FAIL(ctx_->session_info_->get_query_rewrite_integrity(query_rewrite_integrity))) { - LOG_WARN("failed to get query rewrite integrity", K(ret)); - } - for (int64_t i = 0; OB_SUCC(ret) && i < mv_list.count(); ++i) { - const ObTableSchema *mv_schema = NULL; - const ObDatabaseSchema *db_schema = NULL; - bool is_valid = true; - if (OB_FAIL(ctx_->schema_checker_->get_table_schema(ctx_->session_info_->get_effective_tenant_id(), - mv_list.at(i), - mv_schema))) { - LOG_WARN("failed to get mv schema", K(ret), K(mv_list.at(i))); - } else if (OB_ISNULL(mv_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("mv schema is null", K(ret), K(mv_list.at(i))); - } else if (OB_FAIL(ctx_->schema_checker_->get_database_schema(ctx_->session_info_->get_effective_tenant_id(), - mv_schema->get_database_id(), - db_schema))) { - LOG_WARN("failed to get data base schema", K(ret), K(mv_schema->get_database_id())); - } else if (OB_ISNULL(db_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("database schema is NULL", K(ret), K(mv_schema->get_database_id())); - } else if (OB_FAIL(quick_rewrite_check(mv_schema, - query_rewrite_integrity == QueryRewriteIntegrityType::REWRITE_INTEGRITY_STALE_TOLERATED, - is_valid))) { - LOG_WARN("failed to do quick check", K(ret)); - } else if (!is_valid) { - // do nothing + LOG_WARN("unexpected map array count", K(ret), K(current_map.count()), K(from_tables.count())); + } else if (from_table_idx >= from_tables.count()) { + // add current map into table_maps + ObSEArray new_table_map; + if (OB_FAIL(new_table_map.assign(current_map))) { + LOG_WARN("failed to assign table map", K(ret), K(current_map)); + } else if (OB_FAIL(table_maps.push_back(new_table_map))) { + LOG_WARN("failed to push back current_map", K(ret), K(new_table_map)); } else { - uint64_t data_table_id = mv_schema->get_data_table_id(); - const ObTableSchema *data_table_schema = NULL; - if (OB_FAIL(ctx_->schema_checker_->get_table_schema(ctx_->session_info_->get_effective_tenant_id(), - data_table_id, - data_table_schema))) { - LOG_WARN("failed to get data table schema", K(ret), K(data_table_id)); - } else if (OB_ISNULL(data_table_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("data table schema is null", K(ret), K(mv_list.at(i))); - } else if (OB_FAIL(mv_infos_.push_back(MvInfo(mv_list.at(i), - data_table_id, - mv_schema, - data_table_schema, - db_schema, - NULL)))) { - LOG_WARN("failed to push back mv info", K(ret)); - } + LOG_DEBUG("generated one table map", K(new_table_map)); } - } - return ret; -} - -int ObTransformMVRewrite::quick_rewrite_check(const ObTableSchema *mv_schema, - bool allow_stale, - bool &is_valid) -{ - int ret = OB_SUCCESS; - is_valid = false; - if (OB_ISNULL(mv_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("mv schema is null", K(ret)); - } else if (!mv_schema->mv_enable_query_rewrite()) { - is_valid = false; - } else if (!allow_stale && !mv_schema->mv_on_query_computation()) { - is_valid = false; } else { - is_valid = true; - } - return false; -} - -int ObTransformMVRewrite::check_mv_stmt_basic_validity(const MvInfo &mv_info, - bool &is_valid) -{ - int ret = OB_SUCCESS; - const ObSelectStmt *stmt = NULL; - is_valid = false; - if (OB_ISNULL(stmt = mv_info.view_stmt_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(mv_info)); - } else if (!stmt->is_spj()) { - is_valid = false; - OPT_TRACE("mv can not be used for rewrite"); - } else { - is_valid = true; - } - return ret; -} - -int ObTransformMVRewrite::generate_mv_stmt(MvInfo &mv_info) -{ - int ret = OB_SUCCESS; - void* qctx_ptr = NULL; - if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->allocator_) || OB_ISNULL(ctx_->stmt_factory_) - || OB_ISNULL(ctx_->session_info_) || OB_ISNULL(mv_info.mv_schema_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("mv_info is null", K(ret), K(ctx_), KPC(mv_info.mv_schema_)); - } else { - SMART_VARS_2((ObTransformerCtx, trans_ctx), (ObResolverParams, resolver_ctx)) { - ObParser parser(*ctx_->allocator_, ctx_->session_info_->get_sql_mode(), ctx_->session_info_->get_charsets4parser()); - ParseResult parse_result; - ParseNode *node = NULL; - ObString view_definition; - ObDMLStmt *view_stmt = NULL; - uint64_t dummy_value = 0; - ObIAllocator &alloc = *ctx_->allocator_; - resolver_ctx.allocator_ = ctx_->allocator_; - resolver_ctx.schema_checker_ = ctx_->schema_checker_; - resolver_ctx.session_info_ = ctx_->session_info_; - resolver_ctx.expr_factory_ = ctx_->expr_factory_; - resolver_ctx.stmt_factory_ = ctx_->stmt_factory_; - resolver_ctx.sql_proxy_ = GCTX.sql_proxy_; - resolver_ctx.query_ctx_ = &mv_temp_query_ctx_; - // resolver_ctx.is_for_rt_mv_ = true; - trans_ctx = *ctx_; - trans_ctx.reset(); - ObSelectResolver select_resolver(resolver_ctx); - ObTransformPreProcess transform_pre_process(&trans_ctx); - transform_pre_process.set_transformer_type(PRE_PROCESS); - STOP_OPT_TRACE; - if (OB_ISNULL(resolver_ctx.query_ctx_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("query ctx is null", K(ret)); - } else if (OB_FAIL(ObSQLUtils::generate_view_definition_for_resolve(*ctx_->allocator_, - ctx_->session_info_->get_local_collation_connection(), - mv_info.mv_schema_->get_view_schema(), - view_definition))) { - LOG_WARN("fail to generate view definition for resolve", K(ret)); - } else if (OB_FAIL(parser.parse(view_definition, parse_result))) { - LOG_WARN("parse view definition failed", K(view_definition), K(ret)); - } else if (OB_ISNULL(node = parse_result.result_tree_->children_[0]) || OB_UNLIKELY(T_SELECT != node->type_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid mv select node", K(ret), K(node), K(node->type_)); - } else if (OB_FALSE_IT(resolver_ctx.query_ctx_->question_marks_count_ - = static_cast(parse_result.question_mark_ctx_.count_))) { - } else if (OB_FAIL(select_resolver.resolve(*node))) { - LOG_WARN("resolve view definition failed", K(ret)); - } else if (OB_ISNULL(view_stmt = static_cast(select_resolver.get_basic_stmt()))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid mv stmt", K(ret), K(view_stmt)); - } else if (OB_FAIL(resolver_ctx.query_ctx_->query_hint_.init_query_hint(resolver_ctx.allocator_, - resolver_ctx.session_info_, - view_stmt))) { - LOG_WARN("failed to init query hint.", K(ret)); - } else if (OB_FAIL(resolver_ctx.query_ctx_->query_hint_.check_and_set_params_from_hint(resolver_ctx, - *view_stmt))) { - LOG_WARN("failed to check and set params from hint", K(ret)); - } else if (OB_FAIL(transform_pre_process.transform(view_stmt, dummy_value))) { - LOG_WARN("failed to do transform pre process", K(ret), KPC(view_stmt)); - } else if (OB_ISNULL(mv_info.view_stmt_ = static_cast(view_stmt))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid mv stmt", K(ret), K(view_stmt)); - } else { - LOG_DEBUG("generate mv stmt", KPC(mv_info.view_stmt_)); + const TableItem *from_table = from_tables.at(from_table_idx); + const int64_t *to_idx = NULL; + if (OB_ISNULL(from_table)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("from table item is NULL", K(ret), K(from_table_idx)); + } else if (!(from_table->is_basic_table() || from_table->is_generated_table()) + || OB_INVALID_ID == from_table->ref_id_ + || NULL == (to_idx = to_table_map.get(from_table->ref_id_))) { + // table does not exists in to_tables + current_map.at(from_table_idx) = -1; + if (OB_FAIL(SMART_CALL(inner_gen_base_table_map(from_table_idx + 1, + from_tables, + from_table_num, + to_table_ids, + to_table_map, + used_to_table, + max_map_num, + current_map, + table_maps)))) { + LOG_WARN("failed to inner gen base table map", K(ret), K(from_table_idx)); } - - // resolve "select 1 from mv" for mv with multi partitions - bool has_multi_part = false; - if (OB_FAIL(ret)) { - // do nothing - } else if (OB_FAIL(check_mv_has_multi_part(mv_info, has_multi_part))) { - LOG_WARN("failed to check mv has multi part", K(ret)); - } else if (has_multi_part) { - ObParser mv_sql_parser(*ctx_->allocator_, ctx_->session_info_->get_sql_mode(), ctx_->session_info_->get_charsets4parser()); - ObSelectResolver mv_sql_select_resolver(resolver_ctx); - ObSqlString mv_sql; // select 1 from mv; - if (OB_FAIL(ret)) { - // do nothing - } else if (OB_FAIL(mv_sql.assign_fmt(lib::is_oracle_mode() ? - "SELECT /*+no_mv_rewrite*/ 1 FROM \"%.*s\".\"%.*s\"" : - "SELECT /*+no_mv_rewrite*/ 1 FROM `%.*s`.`%.*s`", - mv_info.db_schema_->get_database_name_str().length(), mv_info.db_schema_->get_database_name_str().ptr(), - mv_info.mv_schema_->get_table_name_str().length(), mv_info.mv_schema_->get_table_name_str().ptr()))) { - LOG_WARN("failed to assign sql", K(ret)); - } else if (OB_FAIL(mv_sql_parser.parse(ObString::make_string(mv_sql.ptr()), parse_result))) { - LOG_WARN("parse view definition failed", K(mv_sql), K(ret)); - } else if (OB_ISNULL(node = parse_result.result_tree_->children_[0]) || OB_UNLIKELY(T_SELECT != node->type_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid mv select node", K(ret), K(node), K(node->type_)); - } else if (OB_FALSE_IT(resolver_ctx.query_ctx_->question_marks_count_ - = static_cast(parse_result.question_mark_ctx_.count_))) { - } else if (OB_FAIL(mv_sql_select_resolver.resolve(*node))) { - LOG_WARN("resolve view definition failed", K(ret)); - } else if (OB_ISNULL(mv_info.select_mv_stmt_ = static_cast(mv_sql_select_resolver.get_basic_stmt()))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid mv stmt", K(ret), K(mv_info.select_mv_stmt_)); + } else if (OB_UNLIKELY(*to_idx < 0 || *to_idx >= to_table_ids.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected to_idx", K(ret), K(*to_idx)); + } else { + // try to map from_table to to_table which has same ref_id and not be used + for (int64_t i = 0; OB_SUCC(ret) && max_map_num > table_maps.count() + && i < to_table_ids.at(*to_idx).count(); ++i) { + int64_t to_table_idx = to_table_ids.at(*to_idx).at(i); + if (used_to_table.has_member(to_table_idx)) { + // do nothing, to_table has been used + } else if (OB_FAIL(used_to_table.add_member(to_table_idx))) { + LOG_WARN("failed to add member", K(ret)); + } else if (OB_FALSE_IT(current_map.at(from_table_idx) = to_table_idx)) { + } else if (OB_FAIL(SMART_CALL(inner_gen_base_table_map(from_table_idx + 1, + from_tables, + from_table_num, + to_table_ids, + to_table_map, + used_to_table, + max_map_num, + current_map, + table_maps)))) { + LOG_WARN("failed to inner gen base table map", K(ret), K(from_table_idx)); + } else if (OB_FAIL(used_to_table.del_member(to_table_idx))) { + LOG_WARN("failed to del member", K(ret)); } - } else { - mv_info.select_mv_stmt_ = NULL; } - - RESUME_OPT_TRACE; - OPT_TRACE(mv_info.mv_schema_->get_table_name(), ":", mv_info.view_stmt_); - } - } - return ret; -} - -int ObTransformMVRewrite::check_mv_has_multi_part(const MvInfo &mv_info, - bool &has_multi_part) -{ - int ret = OB_SUCCESS; - has_multi_part = false; - ObSEArray simple_index_infos; - if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->schema_checker_) - || OB_ISNULL(ctx_->session_info_) || OB_ISNULL(mv_info.data_table_schema_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(ctx_), K(mv_info)); - } else if (mv_info.data_table_schema_->get_part_level() != PARTITION_LEVEL_ZERO) { - has_multi_part = true; - } else if (OB_FAIL(mv_info.data_table_schema_->get_simple_index_infos(simple_index_infos))) { - LOG_WARN("failed to get simple index infos", K(ret)); - } - for (int64_t i = 0; OB_SUCC(ret) && !has_multi_part && i < simple_index_infos.count(); ++i) { - const ObTableSchema *index_schema = NULL; - if (OB_FAIL(ctx_->schema_checker_->get_table_schema(ctx_->session_info_->get_effective_tenant_id(), simple_index_infos.at(i).table_id_, index_schema))) { - LOG_WARN("get index schema from schema checker failed", K(ret), K(simple_index_infos.at(i).table_id_)); - } else if (OB_ISNULL(index_schema)) { - ret = OB_TABLE_NOT_EXIST; - LOG_WARN("index table not exists", K(simple_index_infos.at(i).table_id_)); - } else if (index_schema->is_final_invalid_index() || !index_schema->is_global_index_table()) { - //do nothing - } else if (index_schema->get_part_level() != PARTITION_LEVEL_ZERO) { - has_multi_part = true; - } - } - return ret; -} - -int ObTransformMVRewrite::do_transform(ObDMLStmt *&stmt, - bool &trans_happened) -{ - int ret = OB_SUCCESS; - const int64_t max_mv_stmt_gen = 10; - trans_happened = false; - if (OB_ISNULL(ctx_) || OB_ISNULL(stmt) || OB_ISNULL(stmt->get_query_ctx())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(ctx_), K(stmt)); - } else if (!stmt->is_select_stmt()) { - // do nothing - } else { - ObSelectStmt *ori_stmt = static_cast(stmt); - ObSelectStmt *new_stmt = NULL; - const ObCollationType cs_type = stmt->get_query_ctx()->get_query_hint().cs_type_; - const ObMVRewriteHint *myhint = static_cast(get_hint(stmt->get_stmt_hint())); - for (int64_t i = 0; OB_SUCC(ret) && i < mv_infos_.count(); ++i) { - bool is_happened = false; - bool is_match_hint = true; - bool is_mv_valid = false; - MvInfo &mv_info = mv_infos_.at(i); - if (NULL != myhint && OB_FAIL(myhint->check_mv_match_hint(cs_type, - mv_info.mv_schema_, - mv_info.db_schema_, - is_match_hint))) { - LOG_WARN("failed to check mv match hint", K(ret)); - } else if (!is_match_hint) { + // try to map from_table to nothing + int64_t from_num; // number of from tables with the same ref_id minus table has been mapped to -1 + if (OB_FAIL(ret) || max_map_num <= table_maps.count() + || !(from_table->is_basic_table() || from_table->is_generated_table()) + || OB_INVALID_ID == from_table->ref_id_) { // do nothing - } else if (NULL == mv_info.view_stmt_ && ++mv_stmt_gen_count_ > max_mv_stmt_gen) { - // do nothing - } else if (NULL == mv_info.view_stmt_ && OB_FAIL(generate_mv_stmt(mv_info))) { - LOG_WARN("failed to generate mv stmt", K(ret), K(mv_info)); - } else if (OB_FAIL(check_mv_stmt_basic_validity(mv_info, is_mv_valid))) { - LOG_WARN("failed to check mv basic validity", K(ret), K(mv_info)); - } else if (!is_mv_valid) { - // do nothing - } else if (OB_FAIL(try_transform_with_one_mv(ori_stmt, mv_info, new_stmt, is_happened))) { - LOG_WARN("failed to try one mv", K(ret)); - } else if (is_happened) { - ori_stmt = new_stmt; - trans_happened = true; - break; + } else if (OB_FAIL(from_table_num.get_refactored(from_table->ref_id_, from_num))) { + LOG_WARN("failed to get from table num", K(ret), KPC(from_table)); + } else if (from_num <= to_table_ids.at(*to_idx).count()) { + // do nothing, the number of remaining unmapped from tables is less than or equal to + // the number of remaining to tables, can not map from_table to nothing. + } else if (OB_FAIL(from_table_num.set_refactored(from_table->ref_id_, from_num - 1, 1))) { + LOG_WARN("failed to set from table num", K(ret), KPC(from_table)); + } else if (OB_FALSE_IT(current_map.at(from_table_idx) = -1)) { + } else if (OB_FAIL(SMART_CALL(inner_gen_base_table_map(from_table_idx + 1, + from_tables, + from_table_num, + to_table_ids, + to_table_map, + used_to_table, + max_map_num, + current_map, + table_maps)))) { + LOG_WARN("failed to inner gen base table map", K(ret), K(from_table_idx)); + } else if (OB_FAIL(from_table_num.set_refactored(from_table->ref_id_, from_num, 1))) { + LOG_WARN("failed to set from table num", K(ret), KPC(from_table)); } } - if (OB_SUCC(ret) && trans_happened) { - stmt = ori_stmt; - } } return ret; } @@ -673,175 +444,2196 @@ int ObTransformMVRewrite::try_transform_with_one_mv(ObSelectStmt *origin_stmt, bool &transform_happened) { int ret = OB_SUCCESS; - ObStmtMapInfo map_info; - QueryRelation relation; - ObTryTransHelper try_trans_helper; - bool is_valid = false; transform_happened = false; - new_stmt = NULL; if (OB_ISNULL(origin_stmt) || OB_ISNULL(mv_info.mv_schema_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret), K(origin_stmt), K(mv_info.mv_schema_)); - } else if (OB_FAIL(try_trans_helper.fill_helper(origin_stmt->get_query_ctx()))) { - LOG_WARN("failed to fill try trans helper", K(ret)); - } else if (OB_FAIL(ObStmtComparer::check_stmt_containment(origin_stmt, - mv_info.view_stmt_, - map_info, - relation, - false, - false, - false))) { - LOG_WARN("failed to check stmt containment", K(ret)); - } else if ((QueryRelation::QUERY_EQUAL != relation - && QueryRelation::QUERY_LEFT_SUBSET != relation) - || map_info.left_can_be_replaced_ == false) { - OPT_TRACE(mv_info.mv_schema_->get_table_name(), ": stmt is not matched"); - } else if (OB_FAIL(do_transform_use_one_mv(origin_stmt, - mv_info, - map_info, - new_stmt, - is_valid))) { - LOG_WARN("failed to do transform use one mv", K(ret)); - } else if (!is_valid) { - OPT_TRACE(mv_info.mv_schema_->get_table_name(), ": stmt is not matched"); - } else if (OB_FAIL(check_rewrite_expected(origin_stmt, - new_stmt, - mv_info, - is_valid))) { - LOG_WARN("failed to check rewrite expected", K(ret)); - } else if (!is_valid) { + } else if (!mv_info.view_stmt_->is_spjg()) { // do nothing - } else if (OB_FAIL(add_param_constraint(map_info))) { - LOG_WARN("failed to add param constraint", K(ret)); + // TODO: try_transform_complete_mode + OPT_TRACE(mv_info.mv_schema_->get_table_name(), "is not spjg"); + } else if (OB_FAIL(try_transform_contain_mode(origin_stmt, mv_info, new_stmt, transform_happened))) { + LOG_WARN("failed to try transform contain mode", K(ret)); + } else if (!transform_happened) { + // do nothing + } else if (OB_FAIL(push_back_stmt_privilege(mv_info))) { + LOG_WARN("failed to push back stmt privilege", K(ret)); } else if (OB_FAIL(add_transform_hint(*new_stmt, &mv_info))) { LOG_WARN("failed to add transform hint", K(ret)); } else { - transform_happened = true; LOG_DEBUG("succeed to transform with one mv", KPC(new_stmt)); } - if (OB_SUCC(ret) && !transform_happened - && OB_FAIL(try_trans_helper.recover(origin_stmt->get_query_ctx()))) { - LOG_WARN("failed to recover params", K(ret)); + return ret; +} + +int ObTransformMVRewrite::try_transform_contain_mode(ObSelectStmt *origin_stmt, + MvInfo &mv_info, + ObSelectStmt *&new_stmt, + bool &transform_happened) +{ + int ret = OB_SUCCESS; + const int MAX_MAP_NUM = 10; + ObSEArray,10> base_table_maps; + ObTryTransHelper try_trans_helper; + ObDMLStmt *query_stmt = NULL; + new_stmt = NULL; + transform_happened = false; + if (OB_ISNULL(origin_stmt) || OB_ISNULL(mv_info.mv_schema_) || OB_ISNULL(mv_info.view_stmt_) + || OB_ISNULL(ctx_) || OB_ISNULL(ctx_->stmt_factory_) || OB_ISNULL(ctx_->expr_factory_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(origin_stmt), K(mv_info.mv_schema_), K(ctx_)); + } else if (OB_FAIL(gen_base_table_map( + origin_stmt->get_table_items(), mv_info.view_stmt_->get_table_items(), MAX_MAP_NUM, base_table_maps))) { + LOG_WARN("failed to gen base table map", K(ret)); + } else if (OB_FAIL(try_trans_helper.fill_helper(origin_stmt->get_query_ctx()))) { + LOG_WARN("failed to fill try trans helper", K(ret)); + } else { + // try to rewrite use every base table map + OPT_TRACE("try rewrite use", mv_info.mv_schema_->get_table_name()); + LOG_TRACE("begin try rewrite contain mode", K(mv_info.mv_schema_->get_table_name_str()), K(base_table_maps.count())); + for (int64_t i = 0; OB_SUCC(ret) && !transform_happened && i < base_table_maps.count(); ++i) { + bool is_valid = true; + SMART_VAR(MvRewriteHelper, + helper, + *origin_stmt, + mv_info, + static_cast(query_stmt), + base_table_maps.at(i)) { + if (NULL == query_stmt) { + if (OB_FAIL(ObTransformUtils::deep_copy_stmt(*ctx_->stmt_factory_, + *ctx_->expr_factory_, + origin_stmt, + query_stmt))) { + LOG_WARN("failed to deep copy stmt", K(ret)); + } else if (OB_ISNULL(helper.query_stmt_ = static_cast(query_stmt))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("deep copy stmt is null", K(ret)); + } + } + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(check_mv_rewrite_validity(helper, is_valid))) { + LOG_WARN("failed to check mv rewrite validity", K(ret), K(base_table_maps.at(i))); + } else if (!is_valid) { + LOG_TRACE("does not pass the rewrite validity check", K(base_table_maps.at(i))); + } else if (OB_FAIL(generate_rewrite_stmt_contain_mode(helper))) { + LOG_WARN("failed to generate rewrite stmt contain mode", K(ret), K(base_table_maps.at(i))); + } else if (OB_FAIL(check_rewrite_expected(helper, is_valid))) { + LOG_WARN("failed to check rewrite expected", K(ret), K(base_table_maps.at(i))); + } else if (!is_valid) { + if (OB_FAIL(try_trans_helper.recover(origin_stmt->get_query_ctx()))) { + LOG_WARN("failed to recover params", K(ret)); + } else if (OB_FAIL(ObTransformUtils::free_stmt(*ctx_->stmt_factory_, query_stmt))) { + LOG_WARN("failed to free stmt", K(ret)); + } else { + // query_stmt has been rewrited in generate_rewrite_stmt_contain_mode, need to re-copy stmt + query_stmt = NULL; + LOG_TRACE("does not pass the rewrite expected check", K(base_table_maps.at(i))); + } + } else if (OB_FAIL(add_param_constraint(helper))) { + LOG_WARN("failed to add param constraint", K(ret)); + } else { + new_stmt = helper.query_stmt_; + transform_happened = true; + } + } + } + if (OB_FAIL(ret) || transform_happened) { + // do nothing + } else if (NULL != query_stmt + && OB_FAIL(ObTransformUtils::free_stmt(*ctx_->stmt_factory_, query_stmt))) { + LOG_WARN("failed to free stmt", K(ret)); + } else if (MAX_MAP_NUM == base_table_maps.count()) { + OPT_TRACE(mv_info.mv_schema_->get_table_name(), ": reach the base table map try count limit!"); + } else { + OPT_TRACE(mv_info.mv_schema_->get_table_name(), ": stmt is not matched or cost raised"); + } } return ret; } -int ObTransformMVRewrite::do_transform_use_one_mv(ObSelectStmt *origin_stmt, - const MvInfo &mv_info, - ObStmtMapInfo &map_info, - ObSelectStmt *&new_stmt, - bool &is_valid_transform) +int ObTransformMVRewrite::check_mv_rewrite_validity(MvRewriteHelper &helper, + bool &is_valid) { int ret = OB_SUCCESS; - is_valid_transform = false; - new_stmt = NULL; - if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->allocator_) - || OB_ISNULL(ctx_->stmt_factory_) || OB_ISNULL(ctx_->expr_factory_) - || OB_ISNULL(origin_stmt) || OB_ISNULL(mv_info.mv_schema_)) { + if (OB_FAIL(check_delta_table(helper, is_valid))) { + LOG_WARN("failed to check delta table", K(ret)); + } else if (!is_valid) { + LOG_TRACE("does not pass the mv delta table check"); + } else if (OB_FAIL(check_join_compatibility(helper, is_valid))) { + LOG_WARN("failed to check join compatibility", K(ret)); + } else if (!is_valid) { + LOG_TRACE("does not pass the join compatibility check"); + } else if (OB_FAIL(check_predicate_compatibility(helper, is_valid))) { + LOG_WARN("failed to check predicate compatibility", K(ret)); + } else if (!is_valid) { + LOG_TRACE("does not pass the predicate compatibility check"); + } else if (!helper.query_compensation_preds_.empty()) { + // not support now + // TODO: try union rewrite + is_valid = false; + LOG_TRACE("does not pass the predicate compatibility check, has query compensation predicate"); + } else if (OB_FAIL(check_group_by_col(helper, is_valid))) { + LOG_WARN("failed to check group by column", K(ret)); + } else if (!is_valid) { + LOG_TRACE("does not pass the group by column check"); + } else if (OB_FAIL(compute_stmt_expr_map(helper, is_valid))) { + LOG_WARN("failed to compute expr map", K(ret)); + } else if (!is_valid) { + LOG_TRACE("does not pass the expr map check"); + } else if (OB_FAIL(check_opt_feat_ctrl(helper, is_valid))) { + LOG_WARN("failed to check optimizer feature control", K(ret)); + } else if (!is_valid) { + LOG_TRACE("does not pass the optimizer feature control check"); + } + return ret; +} + +int ObTransformMVRewrite::check_delta_table(MvRewriteHelper &helper, + bool &is_valid) +{ + int ret = OB_SUCCESS; + ObSelectStmt *mv_stmt = helper.mv_info_.view_stmt_; + ObSqlBitSet<> mv_common_table; // table exists in both query and mv, used to collect mv delta tables + is_valid = true; + if (OB_ISNULL(mv_stmt)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(ctx_), K(origin_stmt), K(mv_info.mv_schema_)); + LOG_WARN("get unexpected null", K(ret), K(helper.mv_info_)); + } + // collect query delta tables, but do not need to check. query delta table + // validity will be checked in build_query_join_tree(). + for (int64_t i = 0; OB_SUCC(ret) && i < helper.base_table_map_.count(); ++i) { + if (helper.base_table_map_.at(i) == -1) { + if (OB_FAIL(helper.query_delta_table_.add_member(i + 1))) { + LOG_WARN("failed to add member to query delta table", K(ret)); + } + } else if (OB_FAIL(mv_common_table.add_member(helper.base_table_map_.at(i) + 1))) { + LOG_WARN("failed to add member to mv common table", K(ret)); + } + } + if (OB_SUCC(ret) && !helper.query_delta_table_.is_empty() && mv_stmt->has_group_by()) { + // not allow query delta table when mv has group by + is_valid = false; + } + // collect and check the mv delta tables. + for (int64_t i = 0; OB_SUCC(ret) && is_valid && i < mv_stmt->get_table_size(); ++i) { + if (mv_common_table.has_member(i + 1)) { + // do nothing + } else if (OB_FAIL(helper.mv_delta_table_.add_member(i + 1))) { + LOG_WARN("failed to add member to mv delta table", K(ret)); + } else { + is_valid = false; + // TODO: need to check whether mv delta table is legal here + // we do not allow mv delta table now + } + } + return ret; +} + +int ObTransformMVRewrite::check_join_compatibility(MvRewriteHelper &helper, + bool &is_valid) +{ + int ret = OB_SUCCESS; + is_valid = true; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->allocator_) || OB_ISNULL(ctx_->expr_factory_) + || OB_ISNULL(helper.query_stmt_) || OB_ISNULL(helper.mv_info_.view_stmt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_)); } else { - bool is_valid = true; - GenerateStmtHelper helper(*ctx_->expr_factory_); - if (OB_FAIL(ctx_->stmt_factory_->create_stmt(helper.new_stmt_))) { - LOG_WARN("failed to create stmt", K(ret)); - } else if (OB_ISNULL(helper.new_stmt_)) { + // generate conflict detectors + ObSEArray mv_from_tables; + ObSEArray query_from_tables; + ObSEArray mv_conditions; + ObSEArray query_conditions; + ObSEArray mv_conflict_detectors; + ObSEArray, 8> mv_baserel_filters; + ObSEArray query_conflict_detectors; + ObSEArray, 8> query_baserel_filters; + ObSEArray bushy_tree_infos; + ObSEArray new_or_quals; + ObSEArray fake_depend_info; // WARNING query should not have depend info + ObConflictDetectorGenerator generator(*ctx_->allocator_, + *ctx_->expr_factory_, + ctx_->session_info_, + NULL, /* onetime_copier */ + true, /* should_deduce_conds */ + fake_depend_info, + bushy_tree_infos, + new_or_quals); + STOP_OPT_TRACE; + if (OB_FAIL(helper.mv_info_.view_stmt_->get_from_tables(mv_from_tables))) { + LOG_WARN("failed to get mv from table items", K(ret)); + } else if (OB_FAIL(helper.query_stmt_->get_from_tables(query_from_tables))) { + LOG_WARN("failed to get query from table items", K(ret)); + } else if (OB_FAIL(pre_process_quals(helper.mv_info_.view_stmt_->get_condition_exprs(), + mv_conditions, + helper.mv_conds_))) { + LOG_WARN("failed to pre process mv quals", K(ret)); + } else if (OB_FAIL(pre_process_quals(helper.query_stmt_->get_condition_exprs(), + query_conditions, + helper.query_delta_table_.is_empty() ? + helper.query_conds_ : + helper.query_other_conds_))) { + LOG_WARN("failed to pre process query quals", K(ret)); + } else if (OB_FAIL(generator.generate_conflict_detectors(helper.mv_info_.view_stmt_, + mv_from_tables, + helper.mv_info_.view_stmt_->get_semi_infos(), + mv_conditions, + mv_baserel_filters, + mv_conflict_detectors))) { + LOG_WARN("failed to generate mv conflict detectors", K(ret)); + } else if (OB_FAIL(generator.generate_conflict_detectors(helper.query_stmt_, + query_from_tables, + helper.query_stmt_->get_semi_infos(), + query_conditions, + query_baserel_filters, + query_conflict_detectors))) { + LOG_WARN("failed to generate query conflict detectors", K(ret)); + } + RESUME_OPT_TRACE; + + // build and check join tree + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(build_mv_join_tree(helper, mv_baserel_filters, mv_conflict_detectors))) { + LOG_WARN("failed to build mv join tree", K(ret)); + } else if (OB_FAIL(build_query_join_tree(helper, query_baserel_filters, query_conflict_detectors, is_valid))) { + LOG_WARN("failed to build query join tree", K(ret)); + } else if (!is_valid) { + LOG_DEBUG("does not pass the query join tree build"); + } else if (OB_FAIL(compare_join_tree(helper, + helper.mv_tree_, + helper.query_tree_mv_part_, + true, /* can_compensate */ + is_valid))) { + LOG_WARN("failed to compare join tree", K(ret)); + } else if (!is_valid) { + LOG_DEBUG("does not pass the join tree compare"); + } + } + return ret; +} + +int ObTransformMVRewrite::pre_process_quals(const ObIArray &input_conds, + ObIArray &normal_conds, + ObIArray &other_conds) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < input_conds.count(); ++i) { + ObRawExpr *cond = input_conds.at(i); + if (OB_ISNULL(cond)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("stmt is null", K(ret)); - } else if (OB_FALSE_IT(helper.new_stmt_->set_query_ctx(origin_stmt->get_query_ctx()))) { - } else if (OB_FAIL(helper.new_stmt_->get_stmt_hint().assign(origin_stmt->get_stmt_hint()))) { - LOG_WARN("failed to assign stmt hint", K(ret)); - } else if (OB_FAIL(helper.new_stmt_->adjust_statement_id(ctx_->allocator_, - ctx_->src_qb_name_, - ctx_->src_hash_val_))) { - LOG_WARN("failed to adjust statement id", K(ret)); - } else if (OB_FALSE_IT(helper.map_info_ = &map_info)) { - } else if (OB_FAIL(create_mv_table_item(mv_info, helper))) { - LOG_WARN("failed to create mv table item", K(ret), K(mv_info)); - } else if (OB_ISNULL(helper.mv_item_)) { + LOG_WARN("condition expr is null", K(ret)); + } else if (cond->has_flag(CNT_ROWNUM) || cond->has_flag(CNT_SUB_QUERY) || cond->has_flag(CNT_RAND_FUNC) + || cond->has_flag(CNT_DYNAMIC_USER_VARIABLE) || cond->is_const_expr()) { + if (OB_FAIL(other_conds.push_back(cond))) { + LOG_WARN("failed to push back other conds", K(ret)); + } + } else if (OB_FAIL(normal_conds.push_back(cond))) { + LOG_WARN("failed to push back other conds", K(ret)); + } + } + return ret; +} + +int ObTransformMVRewrite::build_mv_join_tree(MvRewriteHelper &helper, + ObIArray> &baserel_filters, + ObIArray &conflict_detectors) +{ + int ret = OB_SUCCESS; + const ObSelectStmt *mv_stmt = helper.mv_info_.view_stmt_; + // here we only need one array to store used conflict detectors + ObSEArray used_detectors; + JoinTreeNode *root_node = NULL; + if (OB_ISNULL(mv_stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(mv_stmt)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < mv_stmt->get_from_item_size(); ++i) { + JoinTreeNode *current_node = NULL; + JoinTreeNode *new_joined_node = NULL; + bool is_valid = false; + if (OB_FAIL(inner_build_mv_join_tree(helper, + mv_stmt->get_table_item(mv_stmt->get_from_item(i)), + baserel_filters, + conflict_detectors, + used_detectors, + current_node))) { + LOG_WARN("failed to build mv join tree", K(ret), KPC(mv_stmt->get_table_item(mv_stmt->get_from_item(i)))); + } else if (OB_ISNULL(current_node)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("mv table item is null", K(ret)); - } else if (OB_FAIL(create_mv_column_item(mv_info, helper))) { - LOG_WARN("failed to create mv column item", K(ret)); - } else if (OB_FAIL(fill_from_item(origin_stmt, mv_info, helper, is_valid))) { - LOG_WARN("failed to add from item", K(ret)); + LOG_WARN("join tree node is null", K(ret), K(current_node)); + } else if (i == 0) { + root_node = current_node; + } else if (OB_FAIL(build_join_tree_node(root_node, /* left child node */ + current_node, /* right child node */ + conflict_detectors, + used_detectors, + new_joined_node, + is_valid))) { + LOG_WARN("failed to build tree node", K(ret)); + } else if (OB_UNLIKELY(!is_valid)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("no valid detector when generating mv join tree", K(ret)); + } else { + root_node = new_joined_node; + } + } + if (OB_SUCC(ret)) { + helper.mv_tree_ = root_node; + LOG_DEBUG("build mv join tree", KPC(helper.mv_tree_)); + } + return ret; +} + +int ObTransformMVRewrite::inner_build_mv_join_tree(MvRewriteHelper &helper, + const TableItem *table, + ObIArray> &baserel_filters, + ObIArray &conflict_detectors, + ObIArray &used_detectors, + JoinTreeNode *&node) +{ + int ret = OB_SUCCESS; + const ObSelectStmt *mv_stmt = helper.mv_info_.view_stmt_; + int64_t rel_id; + if (OB_ISNULL(table) || OB_ISNULL(mv_stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(table), K(mv_stmt)); + } else if (table->is_joined_table()) { + const JoinedTable *join_table = static_cast(table); + JoinTreeNode *left_node = NULL; + JoinTreeNode *right_node = NULL; + bool is_valid = false; + if (OB_FAIL(SMART_CALL(inner_build_mv_join_tree(helper, + join_table->left_table_, + baserel_filters, + conflict_detectors, + used_detectors, + left_node)))) { + LOG_WARN("failed to build left mv join tree", K(ret), KPC(join_table->left_table_)); + } else if (OB_FAIL(SMART_CALL(inner_build_mv_join_tree(helper, + join_table->right_table_, + baserel_filters, + conflict_detectors, + used_detectors, + right_node)))) { + LOG_WARN("failed to build right mv join tree", K(ret), KPC(join_table->right_table_)); + } else if (OB_FAIL(build_join_tree_node(left_node, + right_node, + conflict_detectors, + used_detectors, + node, + is_valid))) { + LOG_WARN("failed to build tree node", K(ret)); + } else if (OB_UNLIKELY(!is_valid)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("no valid detector when generating mv join tree", K(ret)); + } + } else if (OB_FALSE_IT(rel_id = mv_stmt->get_table_bit_index(table->table_id_))) { + } else if (OB_FAIL(build_leaf_tree_node(rel_id, table->table_id_, baserel_filters, node))) { + LOG_WARN("failed to build leaf tree node", K(ret), K(rel_id)); + } + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_ISNULL(node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mv join tree node is null", K(ret), KPC(table)); + } else { + node->ori_table_ = table; + } + return ret; +} + +/** + * For MV: select ... from t1 left join t2 on t1.c1 = t2.c2; + * Query: select ... from t1 left join t2 on t1.c1 = t2.c2, t3 where t3.c1 = t1.c2; + * + * The join trees of MV and Query are: + * MV: Query: + * Inner Join ─┐ + * Left Join / \ ├─ (delta part) + * / \ ┌─ Left Join t3 ─┘ + * t1 t2 (mv part) ─┤ / \ + * └─ t1 t2 + * + * The Left Join node in Query join tree is called query_tree_mv_part_, which has same + * structure as the MV join tree. + */ +int ObTransformMVRewrite::build_query_join_tree(MvRewriteHelper &helper, + ObIArray> &baserel_filters, + ObIArray &conflict_detectors, + bool &is_valid) +{ + int ret = OB_SUCCESS; + // here we only need one array to store used conflict detectors + ObSEArray used_detectors; + // build mv part + // post order traversal the mv join tree, and build join tree with the same structure + if (OB_ISNULL(helper.query_stmt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("query stmt in null", K(ret)); + } else if (OB_FAIL(inner_build_query_tree_mv_part(helper, + helper.mv_tree_, + baserel_filters, + conflict_detectors, + used_detectors, + helper.query_tree_mv_part_, + is_valid))) { + LOG_WARN("failed to build query tree mv part", K(ret)); + } else if (!is_valid) { + LOG_TRACE("query tree mv part is invalid"); + } else if (OB_ISNULL(helper.query_tree_ = helper.query_tree_mv_part_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("query tree in null", K(ret)); + } else { + // build query delta part + // post order traversal each from table, and build join tree node for query delta table + for (int64_t i = 0; OB_SUCC(ret) && is_valid && i < helper.query_stmt_->get_from_item_size(); ++i) { + JoinTreeNode *current_node = NULL; + JoinTreeNode *new_joined_node = NULL; + bool is_delta_table; + if (OB_FAIL(inner_build_query_tree_delta_part(helper, + helper.query_stmt_->get_table_item(helper.query_stmt_->get_from_item(i)), + baserel_filters, + conflict_detectors, + used_detectors, + current_node, + is_delta_table, + is_valid))) { + LOG_WARN("failed to build query tree delta part", K(ret), KPC(helper.query_stmt_->get_table_item(helper.query_stmt_->get_from_item(i)))); + } else if (!is_valid) { + LOG_TRACE("query tree delta part is invalid"); + } else if (!is_delta_table) { + // do nothing, delta table has been merged in query tree + } else if (OB_FAIL(build_join_tree_node(helper.query_tree_, /* left child node */ + current_node, /* right child node */ + conflict_detectors, + used_detectors, + new_joined_node, + is_valid))) { + LOG_WARN("failed to build query tree node, delta part", K(ret)); + } else if (!is_valid) { + LOG_TRACE("no valid detector when generating query join tree, delta part"); + } else { + helper.query_tree_ = new_joined_node; + } + } + if (OB_SUCC(ret) && is_valid) { + LOG_DEBUG("build query join tree", KPC(helper.query_tree_)); + } + } + return ret; +} + +int ObTransformMVRewrite::inner_build_query_tree_mv_part(MvRewriteHelper &helper, + JoinTreeNode *mv_node, + ObIArray> &baserel_filters, + ObIArray &conflict_detectors, + ObIArray &used_detectors, + JoinTreeNode *&node, + bool &is_valid) +{ + int ret = OB_SUCCESS; + ObSEArray mv_relids; + int64_t query_relid; + TableItem *base_table; + is_valid = true; + if (OB_ISNULL(helper.query_stmt_) || OB_ISNULL(mv_node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(helper.query_stmt_), K(mv_node)); + } else if (mv_node->join_info_.join_type_ != UNKNOWN_JOIN) { + // not a leaf node + JoinTreeNode *left_node = NULL; + JoinTreeNode *right_node = NULL; + if (OB_FAIL(SMART_CALL(inner_build_query_tree_mv_part(helper, + mv_node->left_child_, + baserel_filters, + conflict_detectors, + used_detectors, + left_node, + is_valid)))) { + LOG_WARN("failed to build left child", K(ret)); + } else if (!is_valid) { + LOG_DEBUG("left child is invalid"); + } else if (OB_FAIL(SMART_CALL(inner_build_query_tree_mv_part(helper, + mv_node->right_child_, + baserel_filters, + conflict_detectors, + used_detectors, + right_node, + is_valid)))) { + LOG_WARN("failed to build right child", K(ret)); + } else if (!is_valid) { + LOG_DEBUG("right child is invalid"); + } else if (OB_FAIL(build_join_tree_node(left_node, + right_node, + conflict_detectors, + used_detectors, + node, + is_valid))) { + LOG_WARN("failed to build tree node", K(ret)); + } else if (!is_valid) { + LOG_DEBUG("no valid detector when generating query join tree, mv part"); + } + } else if (OB_FAIL(mv_node->table_set_.to_array(mv_relids))) { + LOG_WARN("failed to convert table set to array", K(ret)); + } else if (OB_UNLIKELY(1 != mv_relids.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mv leaf node has multi tables", K(ret), K(mv_relids.count())); + } else if (OB_FALSE_IT(query_relid = find_query_rel_id(helper, mv_relids.at(0)))) { + // mv delta table has been handled in check_delta_table(), here query_relid should not be -1 + } else if (OB_UNLIKELY(query_relid < 1 || query_relid > helper.query_stmt_->get_table_size())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected query table rel id", K(ret), K(query_relid), K(helper.query_stmt_->get_table_size()), K(mv_relids.at(0))); + } else if (OB_ISNULL(base_table = helper.query_stmt_->get_table_item(query_relid - 1))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table item is null", K(ret), K(query_relid)); + } else if (OB_FAIL(build_leaf_tree_node(query_relid, + base_table->table_id_, + baserel_filters, + node))) { + LOG_WARN("failed to build query tree leaf node, mv part", K(ret), K(query_relid), K(mv_relids.at(0))); + } + return ret; +} + +/** + * @param is_delta_table: + * TRUE: all child tables are query delta table, generate a new join node, + * and node has not been merged into helper.query_tree_ + * FALSE: table is not a delta table or some of child tables are not delta table, + * all child nodes has been merged into helper.query_tree_ + */ +int ObTransformMVRewrite::inner_build_query_tree_delta_part(MvRewriteHelper &helper, + const TableItem *table, + ObIArray> &baserel_filters, + ObIArray &conflict_detectors, + ObIArray &used_detectors, + JoinTreeNode *&node, + bool &is_delta_table, + bool &is_valid) +{ + int ret = OB_SUCCESS; + int64_t query_relid; + is_delta_table = false; + is_valid = true; + if (OB_ISNULL(helper.query_stmt_) || OB_ISNULL(table)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(table)); + } else if (table->is_joined_table()) { + // if both left and right child are delta tables, generate a new join node + // if only one child is delta table, merge the child tree node into query tree + const JoinedTable *join_table = static_cast(table); + JoinTreeNode *left_node = NULL; + JoinTreeNode *right_node = NULL; + bool left_is_delta_table = false; + bool right_is_delta_table = false; + if (OB_FAIL(SMART_CALL(inner_build_query_tree_delta_part(helper, + join_table->left_table_, + baserel_filters, + conflict_detectors, + used_detectors, + left_node, + left_is_delta_table, + is_valid)))) { + LOG_WARN("failed to build left child", K(ret)); + } else if (!is_valid) { + LOG_DEBUG("left child is invalid"); + } else if (OB_FAIL(SMART_CALL(inner_build_query_tree_delta_part(helper, + join_table->right_table_, + baserel_filters, + conflict_detectors, + used_detectors, + right_node, + right_is_delta_table, + is_valid)))) { + LOG_WARN("failed to build right child", K(ret)); + } else if (!is_valid) { + LOG_DEBUG("right child is invalid"); + } else if (left_is_delta_table && right_is_delta_table) { + if (OB_FAIL(build_join_tree_node(left_node, + right_node, + conflict_detectors, + used_detectors, + node, + is_valid))) { + LOG_WARN("failed to build query tree node, delta part", K(ret)); + } else if (!is_valid) { + LOG_DEBUG("no valid detector when generating query join tree, delta part"); + } else { + is_delta_table = true; + } + } else if (left_is_delta_table || right_is_delta_table) { + if (OB_FAIL(build_join_tree_node(helper.query_tree_, + left_is_delta_table ? left_node : right_node, + conflict_detectors, + used_detectors, + node, + is_valid))) { + LOG_WARN("failed to build query tree node, delta part", K(ret)); + } else if (!is_valid) { + LOG_DEBUG("no valid detector when generating query join tree, delta part"); + } else { + helper.query_tree_ = node; + is_delta_table = false; + } + } else { + // not a query delta table + is_delta_table = false; + } + } else if (OB_FALSE_IT(query_relid = helper.query_stmt_->get_table_bit_index(table->table_id_))) { + } else if (!helper.query_delta_table_.has_member(query_relid)) { + // not a query delta table + is_delta_table = false; + } else if (OB_FAIL(build_leaf_tree_node(query_relid, table->table_id_, baserel_filters, node))) { + LOG_WARN("failed to build query tree leaf node, delta part", K(ret), K(query_relid)); + } else { + is_delta_table = true; + } + return ret; +} + +int ObTransformMVRewrite::build_join_tree_node(JoinTreeNode *left_node, + JoinTreeNode *right_node, + ObIArray &conflict_detectors, + ObIArray &used_detectors, + JoinTreeNode *&new_node, + bool &is_valid) +{ + int ret = OB_SUCCESS; + void *ptr = NULL; + is_valid = true; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->allocator_) || OB_ISNULL(left_node) || OB_ISNULL(right_node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_), K(left_node), K(right_node)); + } else if (OB_ISNULL(ptr = ctx_->allocator_->alloc(sizeof(JoinTreeNode)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate join tree node", K(ret)); + } else if (OB_FALSE_IT(new_node = new(ptr) JoinTreeNode())) { + } else if (OB_FAIL(new_node->table_set_.add_members(left_node->table_set_))) { + LOG_WARN("failed to add left table rel ids", K(ret)); + } else if (OB_FAIL(new_node->table_set_.add_members(right_node->table_set_))) { + LOG_WARN("failed to add right table rel ids", K(ret)); + } else { + new_node->left_child_ = left_node; + new_node->right_child_ = right_node; + // generate join info for new node + bool is_strict_order; + ObSEArray valid_detectors; + // fake conflict detectors is a empty array to imitate right_used_detectors + ObSEArray fake_conflict_detectors; + // WARNING query should not have depend info + ObSEArray fake_depend_info; + if (OB_FAIL(ObConflictDetector::choose_detectors(left_node->table_set_, + right_node->table_set_, + used_detectors, + fake_conflict_detectors, + fake_depend_info, + conflict_detectors, + valid_detectors, + false, /* delay_cross_product */ + is_strict_order))) { + LOG_WARN("failed to choose join info", K(ret)); + } else if (valid_detectors.empty()) { + // can not generate valid join tree + is_valid = false; + LOG_TRACE("no valid detector when generating join tree"); + } else if (OB_FAIL(append(used_detectors, valid_detectors))) { + LOG_WARN("failed to append used conflict detectors", K(ret)); + } else if (OB_FAIL(ObConflictDetector::merge_join_info(valid_detectors, new_node->join_info_))) { + LOG_WARN("failed to merge join info", K(ret)); + } else if (!is_strict_order) { + new_node->join_info_.join_type_ = get_opposite_join_type(new_node->join_info_.join_type_); + } + } + return ret; +} + +int ObTransformMVRewrite::build_leaf_tree_node(int64_t rel_id, + uint64_t table_id, + ObIArray> &baserel_filters, + JoinTreeNode *&leaf_node) +{ + int ret = OB_SUCCESS; + void *ptr = NULL; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->allocator_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_)); + } else if (OB_ISNULL(ptr = ctx_->allocator_->alloc(sizeof(JoinTreeNode)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate join tree node", K(ret)); + } else if (OB_FALSE_IT(leaf_node = new(ptr) JoinTreeNode())) { + } else if (OB_FAIL(leaf_node->table_set_.add_member(rel_id))) { + LOG_WARN("failed to add table rel id", K(ret)); + } else if (OB_UNLIKELY(rel_id < 1 || rel_id > baserel_filters.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected query table rel id", K(ret), K(rel_id), K(baserel_filters.count())); + } else if (OB_FAIL(append(leaf_node->join_info_.where_conditions_, + baserel_filters.at(rel_id - 1)))) { + LOG_WARN("failed to append where conditions", K(ret)); + } else { + leaf_node->table_id_ = table_id; + leaf_node->join_info_.join_type_ = UNKNOWN_JOIN; + } + return ret; +} + +int ObTransformMVRewrite::find_query_rel_id(MvRewriteHelper &helper, int64_t mv_relid) +{ + int query_rel_id = -1; + if (mv_relid > 0) { + for (int64_t i = 0; i < helper.base_table_map_.count(); ++i) { + if (helper.base_table_map_.at(i) == mv_relid - 1) { + query_rel_id = i + 1; + break; + } + } + } + return query_rel_id; +} + +int ObTransformMVRewrite::compare_join_tree(MvRewriteHelper &helper, + JoinTreeNode *mv_node, + JoinTreeNode *query_node, + bool can_compensate, + bool &is_valid) +{ + int ret = OB_SUCCESS; + is_valid = true; + if (OB_ISNULL(mv_node) || OB_ISNULL(query_node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(mv_node), K(query_node)); + } else if (mv_node->join_info_.join_type_ == UNKNOWN_JOIN) { + // is leaf node + ObSEArray mv_relids; + ObSEArray query_relids; + if (OB_UNLIKELY(query_node->join_info_.join_type_ != UNKNOWN_JOIN)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("query trees are not matched", K(ret), KPC(mv_node), KPC(query_node)); + } else if (OB_FAIL(mv_node->table_set_.to_array(mv_relids))) { + LOG_WARN("failed to convert mv table set to array", K(ret)); + } else if (OB_UNLIKELY(1 != mv_relids.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mv leaf node has multi tables", K(ret), K(mv_relids.count())); + } else if (OB_FAIL(query_node->table_set_.to_array(query_relids))) { + LOG_WARN("failed to convert query table set to array", K(ret)); + } else if (OB_UNLIKELY(1 != query_relids.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("query leaf node has multi tables", K(ret), K(query_relids.count())); + } else if (OB_UNLIKELY(query_relids.at(0) != find_query_rel_id(helper, mv_relids.at(0)))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("query trees are not matched", K(ret), KPC(mv_node), KPC(query_node)); + } else if (can_compensate) { + if (OB_FAIL(append(helper.mv_conds_, mv_node->join_info_.where_conditions_))) { + LOG_WARN("failed to append mv conditions", K(ret)); + } else if (OB_FAIL(append(helper.query_conds_, query_node->join_info_.where_conditions_))) { + LOG_WARN("failed to append query conditions", K(ret)); + } + } else if (OB_FAIL(compare_join_conds(helper, + mv_node->join_info_.where_conditions_, + query_node->join_info_.where_conditions_, + is_valid))) { + LOG_WARN("failed to compare join conditions", K(ret)); + } + } else if (OB_FAIL(SMART_CALL(compare_join_tree(helper, + mv_node->left_child_, + query_node->left_child_, + can_compensate && + (INNER_JOIN == query_node->join_info_.join_type_ + || LEFT_OUTER_JOIN == query_node->join_info_.join_type_ + || LEFT_SEMI_JOIN == query_node->join_info_.join_type_ + || LEFT_ANTI_JOIN == query_node->join_info_.join_type_), + is_valid)))) { + LOG_WARN("failed to compare left join tree", K(ret)); + } else if (!is_valid) { + // do nothing + } else if (OB_FAIL(SMART_CALL(compare_join_tree(helper, + mv_node->right_child_, + query_node->right_child_, + can_compensate && + (INNER_JOIN == query_node->join_info_.join_type_ + || RIGHT_OUTER_JOIN == query_node->join_info_.join_type_ + || RIGHT_SEMI_JOIN == query_node->join_info_.join_type_ + || RIGHT_ANTI_JOIN == query_node->join_info_.join_type_), + is_valid)))) { + LOG_WARN("failed to compare right join tree", K(ret)); + } else if (!is_valid) { + // do nothing + } else if (can_compensate) { + // check join type is compatible, compare join conditions and pull up join filter + if (OB_FAIL(compare_join_type(helper, mv_node, query_node, is_valid))) { + LOG_WARN("failed to compare join type", K(ret)); } else if (!is_valid) { // do nothing - } else if (OB_FAIL(fill_condition_exprs(origin_stmt, mv_info, helper, is_valid))) { - LOG_WARN("failed to add condition exprs", K(ret)); + } else if (INNER_JOIN == query_node->join_info_.join_type_) { + // inner join does not need to compare join conditions + // just pull up join filter simply + // mv_node may no be inner join, we also need to pull up on_conditions_ + if (OB_FAIL(append(helper.mv_conds_, mv_node->join_info_.where_conditions_))) { + LOG_WARN("failed to append mv where conditions", K(ret)); + } else if (OB_FAIL(append(helper.mv_conds_, mv_node->join_info_.on_conditions_))) { + LOG_WARN("failed to append mv on conditions", K(ret)); + } else if (OB_FAIL(append(helper.query_conds_, query_node->join_info_.where_conditions_))) { + LOG_WARN("failed to append query conditions", K(ret)); + } + } else if (OB_FAIL(compare_join_conds(helper, + mv_node->join_info_.on_conditions_, + query_node->join_info_.on_conditions_, + is_valid))) { + LOG_WARN("failed to compare join conditions", K(ret)); } else if (!is_valid) { // do nothing - } else if (OB_FAIL(fill_groupby_exprs(origin_stmt, mv_info, helper, is_valid))) { - LOG_WARN("failed to add group by exprs", K(ret)); + } else if (OB_FAIL(append(helper.mv_conds_, mv_node->join_info_.where_conditions_))) { + LOG_WARN("failed to append mv conditions", K(ret)); + } else if (OB_FAIL(append(helper.query_conds_, query_node->join_info_.where_conditions_))) { + LOG_WARN("failed to append query conditions", K(ret)); + } + } else if (mv_node->join_info_.join_type_ != query_node->join_info_.join_type_) { + is_valid = false; + LOG_TRACE("join type not match", K(mv_node->join_info_), K(query_node->join_info_)); + } else if (OB_FAIL(compare_join_conds(helper, + mv_node->join_info_.where_conditions_, + query_node->join_info_.where_conditions_, + is_valid))) { + LOG_WARN("failed to compare join conditions", K(ret)); + } else if (!is_valid) { + // do nothing + } else if (OB_FAIL(compare_join_conds(helper, + mv_node->join_info_.on_conditions_, + query_node->join_info_.on_conditions_, + is_valid))) { + LOG_WARN("failed to compare join conditions", K(ret)); + } else if (!is_valid) { + // do nothing + } + return ret; +} + +// check whether mv_conds and query_conds are same +int ObTransformMVRewrite::compare_join_conds(MvRewriteHelper &helper, + const ObIArray &mv_conds, + const ObIArray &query_conds, + bool &is_valid) +{ + int ret = OB_SUCCESS; + is_valid = true; + ObSqlBitSet<> mv_checked_conds; + for (int64_t i = 0; OB_SUCC(ret) && is_valid && i < query_conds.count(); ++i) { + bool has_found = false; + for (int64_t j = 0; OB_SUCC(ret) && !has_found && j < mv_conds.count(); ++j) { + if (OB_FAIL(is_same_condition(helper, query_conds.at(i), mv_conds.at(j), has_found))) { + LOG_WARN("failed to compare join conditions", K(ret)); + } else if (!has_found) { + // do nothing + } else if (OB_FAIL(mv_checked_conds.add_member(j))) { + LOG_WARN("failed to add member", K(ret)); + } + } + if (OB_SUCC(ret) && !has_found) { + is_valid = false; + } + } + for (int64_t i = 0; OB_SUCC(ret) && is_valid && i < mv_conds.count(); ++i) { + bool has_found = false; + if (mv_checked_conds.has_member(i)) { + has_found = true; + } + for (int64_t j = 0; OB_SUCC(ret) && !has_found && j < query_conds.count(); ++j) { + if (OB_FAIL(is_same_condition(helper, query_conds.at(j), mv_conds.at(i), has_found))) { + LOG_WARN("failed to compare join conditions", K(ret)); + } + } + if (OB_SUCC(ret) && !has_found) { + is_valid = false; + } + } + return ret; +} + +// check whether the join type is compatible and add a not null compensation predicate +int ObTransformMVRewrite::compare_join_type(MvRewriteHelper &helper, + JoinTreeNode *mv_node, + JoinTreeNode *query_node, + bool &is_valid) +{ + int ret = OB_SUCCESS; + is_valid = true; + if (OB_ISNULL(mv_node) || OB_ISNULL(query_node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(mv_node), K(query_node)); + } else if (mv_node->join_info_.join_type_ == query_node->join_info_.join_type_) { + // do nothing + } else { + switch(query_node->join_info_.join_type_) { + case INNER_JOIN: + if (LEFT_OUTER_JOIN == mv_node->join_info_.join_type_) { + if (OB_FAIL(add_not_null_compensate(helper, mv_node, false, is_valid))) { + LOG_WARN("failed to add not null compensate", K(ret)); + } + } else if (RIGHT_OUTER_JOIN == mv_node->join_info_.join_type_) { + if (OB_FAIL(add_not_null_compensate(helper, mv_node, true, is_valid))) { + LOG_WARN("failed to add not null compensate", K(ret)); + } + } else if (FULL_OUTER_JOIN == mv_node->join_info_.join_type_) { + if (OB_FAIL(add_not_null_compensate(helper, mv_node, true, is_valid))) { + LOG_WARN("failed to add not null compensate", K(ret)); + } else if (!is_valid) { + // do nothing + } else if (OB_FAIL(add_not_null_compensate(helper, mv_node, false, is_valid))) { + LOG_WARN("failed to add not null compensate", K(ret)); + } + } else { + is_valid = false; + } + break; + case LEFT_OUTER_JOIN: + case RIGHT_OUTER_JOIN: + if (FULL_OUTER_JOIN == mv_node->join_info_.join_type_) { + if (OB_FAIL(add_not_null_compensate(helper, + mv_node, + LEFT_OUTER_JOIN == query_node->join_info_.join_type_, + is_valid))) { + LOG_WARN("failed to add not null compensate", K(ret)); + } + } else { + is_valid = false; + } + break; + case LEFT_SEMI_JOIN: + case RIGHT_SEMI_JOIN: + // not support now + is_valid = false; + break; + case LEFT_ANTI_JOIN: + case RIGHT_ANTI_JOIN: + // not support now + is_valid = false; + break; + default: + is_valid = false; + break; + } + } + return ret; +} + +int ObTransformMVRewrite::add_not_null_compensate(MvRewriteHelper &helper, + JoinTreeNode *mv_upper_node, + bool comp_left_child, + bool &is_valid) +{ + int ret = OB_SUCCESS; + ObRawExpr *not_null_expr = NULL; + ObRawExpr *compensate_expr = NULL; + is_valid = true; + if (OB_ISNULL(mv_upper_node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(mv_upper_node)); + } else if (OB_FAIL(find_mv_not_null_expr(helper, + mv_upper_node, + comp_left_child, + not_null_expr))) { + LOG_WARN("failed to find mv not null expr", K(ret)); + } else if (NULL == not_null_expr) { + is_valid = false; + LOG_TRACE("can not find mv not null expr to compensate"); + } else if (OB_FAIL(ObTransformUtils::add_is_not_null(ctx_, + not_null_expr, + compensate_expr))) { + LOG_WARN("failed to build is not null expr", K(ret)); + } else if (OB_ISNULL(compensate_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("build null compensate expr", K(ret)); + } else if (OB_FAIL(helper.mv_compensation_preds_.push_back(compensate_expr))) { + LOG_WARN("failed to push back compensate expr", K(ret)); + } + return ret; +} + +/** + * @brief ObTransformMVRewrite::find_mv_not_null_expr + * + * Find an MV select item, which is not null before mv_upper_node's left/right + * (depends on comp_left_child) child, and is null in the outer join fill NULL + * rows. + * + * @param comp_left_child True: find not null expr in mv_upper_node's left child, + * False: find not null expr in mv_upper_node's right child + */ +int ObTransformMVRewrite::find_mv_not_null_expr(MvRewriteHelper &helper, + JoinTreeNode *mv_upper_node, + bool comp_left_child, + ObRawExpr *¬_null_expr) +{ + int ret = OB_SUCCESS; + ObSelectStmt *mv_stmt = helper.mv_info_.view_stmt_; + JoinTreeNode *mv_comp_node = NULL; + not_null_expr = NULL; + if (OB_ISNULL(ctx_) || OB_ISNULL(mv_stmt) || OB_ISNULL(mv_upper_node) + || OB_ISNULL(mv_comp_node = comp_left_child ? mv_upper_node->left_child_ : mv_upper_node->right_child_) + || OB_ISNULL(mv_comp_node->ori_table_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_), K(mv_stmt), KPC(mv_upper_node), KPC(mv_comp_node)); + } else { + ObNotNullContext not_null_context(*ctx_, mv_stmt); + if (mv_comp_node->ori_table_->is_joined_table() && + OB_FAIL(not_null_context.add_joined_table(static_cast(mv_comp_node->ori_table_)))) { + LOG_WARN("failed to add joined table into context", K(ret), KPC(mv_comp_node->ori_table_)); + } else if (OB_FAIL(not_null_context.add_filter(mv_upper_node->join_info_.on_conditions_))) { + LOG_WARN("failed to add null reject conditions", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && NULL == not_null_expr && i < mv_stmt->get_select_item_size(); ++i) { + ObRawExpr *select_expr = mv_stmt->get_select_item(i).expr_; + ObSEArray col_exprs; + bool is_null_propagate = false; + bool is_not_null = false; + if (OB_ISNULL(select_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (select_expr->is_const_expr()) { + // do nothing + } else if (!select_expr->get_relation_ids().is_subset(mv_comp_node->table_set_)) { + // do nothing + } else if (OB_FAIL(ObRawExprUtils::extract_column_exprs(select_expr, col_exprs))) { + LOG_WARN("failed to extract column exprs", K(ret), KPC(select_expr)); + } else if (OB_FAIL(ObTransformUtils::is_null_propagate_expr(select_expr, + col_exprs, + is_null_propagate))) { + LOG_WARN("failed to check is null propagate expr", K(ret), KPC(select_expr)); + } else if (!is_null_propagate) { + // do nothing + } else if (OB_FAIL(ObTransformUtils::is_expr_not_null(not_null_context, + select_expr, + is_not_null, + NULL /* constraints */ ))) { + LOG_WARN("failed to check expr not null", K(ret)); + } else if (is_not_null) { + not_null_expr = select_expr; + break; + } + } + } + return ret; +} + +/** + * @brief ObTransformMVRewrite::check_predicate_compatibility + * Check whether mv predicate and query predicate are same, and generate + * compensation predicates. + */ +int ObTransformMVRewrite::check_predicate_compatibility(MvRewriteHelper &helper, + bool &is_valid) +{ + int ret = OB_SUCCESS; + is_valid = true; + ObSEArray mv_equal_conds; + ObSEArray mv_other_conds; + ObSEArray query_equal_conds; + ObSEArray query_other_conds; + if (OB_FAIL(classify_predicates(helper.mv_conds_, + mv_equal_conds, + mv_other_conds))) { + LOG_WARN("failed to split predicate", K(ret)); + } else if (OB_FAIL(classify_predicates(helper.query_conds_, + query_equal_conds, + query_other_conds))) { + LOG_WARN("failed to split predicate", K(ret)); + } else if (OB_FAIL(check_equal_predicate(helper, mv_equal_conds, query_equal_conds, is_valid))) { + LOG_WARN("failed to check equal predicate", K(ret)); + } else if (!is_valid) { + // do nothing + } else if (OB_FAIL(check_other_predicate(helper, mv_other_conds, query_other_conds))) { + LOG_WARN("failed to check other predicate", K(ret)); + } else if (OB_FAIL(check_compensation_preds_validity(helper, is_valid))) { + LOG_WARN("failed to check compensation preds validity", K(ret)); + } + return ret; +} + +int ObTransformMVRewrite::check_equal_predicate(MvRewriteHelper &helper, + const ObIArray &mv_conds, + const ObIArray &query_conds, + bool &is_valid) +{ + int ret = OB_SUCCESS; + ObSEArray mv_column_exprs; + ObSEArray query_column_exprs; + is_valid = true; + if (OB_ISNULL(helper.query_stmt_) || OB_ISNULL(helper.mv_info_.view_stmt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(helper.query_stmt_), K(helper.mv_info_.view_stmt_)); + } else if (OB_FAIL(helper.mv_info_.view_stmt_->get_column_exprs(mv_column_exprs))) { + LOG_WARN("failed to get mv column exprs", K(ret)); + } else if (OB_FAIL(helper.query_stmt_->get_column_exprs(query_column_exprs))) { + LOG_WARN("failed to get query column exprs", K(ret)); + } else if (OB_FAIL(build_equal_sets(mv_conds, + mv_column_exprs, + helper.mv_equal_sets_, + helper.mv_es_map_))) { + LOG_WARN("failed to build equal class", K(ret)); + } else if (OB_FAIL(build_equal_sets(query_conds, + query_column_exprs, + helper.query_equal_sets_, + helper.query_es_map_))) { + LOG_WARN("failed to build equal class", K(ret)); + } else if (OB_FAIL(generate_equal_compensation_preds(helper, is_valid))) { + LOG_WARN("failed to compare equal sets", K(ret)); + } + return ret; +} + +int ObTransformMVRewrite::build_equal_sets(const ObIArray &input_conds, + const ObIArray &all_columns, + EqualSets &equal_sets, + hash::ObHashMap &equal_set_map) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->allocator_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_)); + } else if (OB_FAIL(ObEqualAnalysis::compute_equal_set(ctx_->allocator_, + input_conds, + equal_sets))) { + LOG_WARN("failed to compute equal set", K(ret)); + } else if (OB_FAIL(equal_set_map.create(all_columns.count(), "MvRewrite"))) { + LOG_WARN("failed to init equal set map", K(ret)); + } + // generate equal set map + for (int64_t i = 0; OB_SUCC(ret) && i < equal_sets.count(); ++i) { + ObIArray *es = equal_sets.at(i); + if (OB_ISNULL(es)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("equal set is null", K(ret)); + } + for (int64_t j = 0; OB_SUCC(ret) && j < es->count(); ++j) { + const ObRawExpr *expr = es->at(j); + if (OB_ISNULL(expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expr is null", K(ret)); + } else if (OB_FAIL(equal_set_map.set_refactored(expr, i))) { + LOG_WARN("failed to set equal set map", K(ret), K(expr), K(i)); + } + } + } + // map columns that are not in equal sets into -1 + for (int64_t i = 0; OB_SUCC(ret) && i < all_columns.count(); ++i) { + ObRawExpr *col_expr = all_columns.at(i); + if (OB_ISNULL(col_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expr is null", K(ret)); + } else if (NULL != equal_set_map.get(col_expr)) { + // do nothing, col_expr already exists in equal set + } else if (OB_FAIL(equal_set_map.set_refactored(col_expr, -1))) { + LOG_WARN("failed to set equal set map", K(ret), K(col_expr)); + } + } + return ret; +} + +int ObTransformMVRewrite::generate_equal_compensation_preds(MvRewriteHelper &helper, + bool &is_valid) +{ + int ret = OB_SUCCESS; + ObSelectStmt *mv_stmt = helper.mv_info_.view_stmt_; + ObSEArray,16> mv_es_to_query_es; // use to check mv equal set map to multi query equal set + ObSEArray mv_column_exprs; + hash::ObHashSet visited_mv_column_exprs; + is_valid = true; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->expr_factory_) || OB_ISNULL(mv_stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_), K(mv_stmt)); + } else if (OB_FAIL(helper.equal_sets_map_.prepare_allocate(helper.query_equal_sets_.count()))) { + LOG_WARN("failed to prepare allocate array", K(ret), K(helper.query_equal_sets_.count())); + } else if (OB_FAIL(mv_es_to_query_es.prepare_allocate(helper.mv_equal_sets_.count()))) { + LOG_WARN("failed to prepare allocate array", K(ret), K(helper.mv_equal_sets_.count())); + } else if (OB_FAIL(mv_stmt->get_column_exprs(mv_column_exprs))) { + LOG_WARN("failed to get mv column exprs", K(ret)); + } else if (OB_FAIL(visited_mv_column_exprs.create(mv_column_exprs.count(), "MvRewrite", "HashNode"))) { + LOG_WARN("failed to init visited mv expr hash set", K(ret)); + } + // check each query equal set, make sure every single expr in the set map to + // the same mv equal set, otherwise add mv compensation predicate + for (int64_t i = 0; OB_SUCC(ret) && is_valid && i < helper.query_equal_sets_.count(); ++i) { + ObIArray *query_es = helper.query_equal_sets_.at(i); + ObSqlBitSet<> &mv_es_ids = helper.equal_sets_map_.at(i); + ObColumnRefRawExpr *first_mv_expr = NULL; + if (OB_ISNULL(query_es)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("query equal set is null", K(ret)); + } + for (int64_t j = 0; OB_SUCC(ret) && is_valid && j < query_es->count(); ++j) { + ObColumnRefRawExpr *query_expr = NULL; + ObColumnRefRawExpr *mv_expr = NULL; + ObRawExpr *compensate_expr = NULL; + int64_t mv_es_id; + if (OB_ISNULL(query_es->at(j))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("query expr is null", K(ret)); + } else if (OB_UNLIKELY(!query_es->at(j)->is_column_ref_expr())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("query expr is not a column ref expr", K(ret), KPC(query_es->at(j))); + } else if (OB_FALSE_IT(query_expr = static_cast(query_es->at(j)))) { + } else if (OB_FAIL(map_to_mv_column(helper, query_expr, mv_expr))) { + LOG_WARN("failed to get mv column expr", K(ret), KPC(query_expr)); + } else if (NULL == mv_expr) { + // column does not exists in MV + // we can not build the compensation predicate + is_valid = false; + LOG_TRACE("query column does not exists in mv", KPC(query_expr)); + } else if (OB_FAIL(visited_mv_column_exprs.set_refactored(mv_expr))) { + LOG_WARN("failed to set visited mv expr hash set", K(ret), KPC(mv_expr)); + } else if (OB_FAIL(helper.mv_es_map_.get_refactored(mv_expr, mv_es_id))) { + LOG_WARN("failed to get mv equal set index", K(ret), KPC(mv_expr)); + } else if (-1 == mv_es_id) { + // MV column does not appear in equal predicates, + // For the first expr, add not null compensation, + // For the other expr, add equal compensation. + int tmp_ret; + if (NULL == first_mv_expr) { + if (OB_FAIL(ObTransformUtils::add_is_not_null(ctx_, + mv_expr, + compensate_expr))) { + LOG_WARN("failed to build is not null expr", K(ret)); + } else if (OB_FAIL(helper.mv_compensation_preds_.push_back(compensate_expr))) { + LOG_WARN("failed to push back compensate expr", K(ret)); + } else { + first_mv_expr = mv_expr; + } + } else if (OB_FAIL(ObRawExprUtils::create_equal_expr(*ctx_->expr_factory_, + ctx_->session_info_, + first_mv_expr, + mv_expr, + compensate_expr))) { + LOG_WARN("failed to create equal expr", K(ret), KPC(first_mv_expr), KPC(mv_expr)); + } else if (OB_FAIL(helper.mv_compensation_preds_.push_back(compensate_expr))) { + LOG_WARN("failed to push back compensate expr", K(ret)); + } + } else if (OB_UNLIKELY(mv_es_id < 0 || mv_es_id >= mv_es_to_query_es.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mv equal set index is unexpected", K(ret), K(mv_es_id), K(mv_es_to_query_es.count()), K(helper.mv_equal_sets_.count())); + } else if (OB_FAIL(check_mv_equal_set_used(helper, + mv_es_to_query_es.at(mv_es_id), + i))) { + LOG_WARN("failed to check mv equal set used", K(ret)); + } else if (NULL == first_mv_expr) { + if (OB_FAIL(mv_es_ids.add_member(mv_es_id))) { + LOG_WARN("failed to add members", K(ret)); + } else { + first_mv_expr = mv_expr; + } + } else if (mv_es_ids.has_member(mv_es_id)) { + // do nothing + } else if (OB_FAIL(mv_es_ids.add_member(mv_es_id))) { + LOG_WARN("failed to add members", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::create_equal_expr(*ctx_->expr_factory_, + ctx_->session_info_, + first_mv_expr, + mv_expr, + compensate_expr))) { + LOG_WARN("failed to create equal expr", K(ret), KPC(first_mv_expr), KPC(mv_expr)); + } else if (OB_FAIL(helper.mv_compensation_preds_.push_back(compensate_expr))) { + LOG_WARN("failed to push back compensate expr", K(ret)); + } + } + } + // Because the stmt may no contain all the columns, we need to check each mv + // column expr, make sure every single expr has been visited, and add query + // compensation predicate if needed. + for (int64_t i = 0; OB_SUCC(ret) && is_valid && i < mv_column_exprs.count(); ++i) { + ObColumnRefRawExpr *mv_expr = mv_column_exprs.at(i); + int64_t mv_es_id; + int tmp_ret; + if (OB_ISNULL(mv_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mv column expr is null", K(ret), K(i)); + } else if (mv_expr->get_relation_ids().is_subset(helper.mv_delta_table_)) { + // do nothing, it is a mv delta table expr + } else if (OB_FALSE_IT(tmp_ret = visited_mv_column_exprs.exist_refactored(mv_expr))) { + } else if (OB_HASH_EXIST == tmp_ret) { + // do nothing, expr has been visited + } else if (OB_UNLIKELY(OB_HASH_NOT_EXIST != tmp_ret)) { + ret = tmp_ret; + LOG_WARN("failed to check mv expr hash set", K(ret), KPC(mv_expr)); + } else if (OB_FAIL(helper.mv_es_map_.get_refactored(mv_expr, mv_es_id))) { + LOG_WARN("failed to get mv equal set index", K(ret), KPC(mv_expr)); + } else if (-1 == mv_es_id) { + // do nothing, mv column does not appear in equal predicates + } else if (OB_UNLIKELY(mv_es_id < 0 || mv_es_id >= helper.mv_equal_sets_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mv equal set index is unexpected", K(ret), K(mv_es_id), K(helper.mv_equal_sets_.count())); + } else if (OB_ISNULL(helper.mv_equal_sets_.at(mv_es_id))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mv equal set is null", K(ret), K(mv_es_id)); + } else { + // TODO: need to generate query compensation predicate, but the corresponding + // query expr does not exist. Now just simply set to invalid. + is_valid = false; + } + } + if (OB_SUCC(ret) && OB_FAIL(visited_mv_column_exprs.destroy())) { + LOG_WARN("failed to destroy visited mv expr hash set", K(ret)); + } + return ret; +} + +/** + * @brief ObTransformMVRewrite::check_mv_equal_set_used + * Check whether mv equal set map to multi query equal set, if so, generate + * a query compensation predicate. + */ +int ObTransformMVRewrite::check_mv_equal_set_used(MvRewriteHelper &helper, + ObIArray &mv_map_query_ids, + int64_t current_query_es_id) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->expr_factory_) || OB_ISNULL(ctx_->session_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_)); + } else if (mv_map_query_ids.empty()) { + if (OB_FAIL(mv_map_query_ids.push_back(current_query_es_id))) { + LOG_WARN("failed to push back query es id", K(ret)); + } + } else if (is_contain(mv_map_query_ids, current_query_es_id)) { + // do nothing + } else { + // mv equal set has already mapped to the other query equal set + // generate query compensation predicate + ObIArray *current_es = helper.query_equal_sets_.at(current_query_es_id); + if (OB_ISNULL(current_es) || OB_UNLIKELY(current_es->empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("query equal set is unexpected", K(ret), K(current_es)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < mv_map_query_ids.count(); ++i) { + ObIArray *query_es = helper.query_equal_sets_.at(mv_map_query_ids.at(i)); + ObRawExpr *equal_expr = NULL; + ObRawExpr *compensate_expr = NULL; + if (OB_ISNULL(query_es)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("query equal set is unexpected", K(ret), K(query_es)); + } else if (OB_FAIL(ObRawExprUtils::create_equal_expr(*ctx_->expr_factory_, + ctx_->session_info_, + query_es->at(0), + current_es->at(0), + equal_expr))) { + LOG_WARN("failed to create equal expr", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::build_lnnvl_expr(*ctx_->expr_factory_, + equal_expr, + compensate_expr))) { + LOG_WARN("failed to build lnnvl expr", K(ret)); + } else if (OB_ISNULL(compensate_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("built compensate expr is null", K(ret), KPC(equal_expr)); + } else if (OB_FAIL(compensate_expr->formalize(ctx_->session_info_))) { + LOG_WARN("failed to formalize expr", K(ret)); + } else if (OB_FAIL(helper.query_compensation_preds_.push_back(compensate_expr))) { + LOG_WARN("failed to push back compensate expr", K(ret)); + } + } + if (OB_SUCC(ret) && OB_FAIL(mv_map_query_ids.push_back(current_query_es_id))) { + LOG_WARN("failed to push back query id", K(ret)); + } + } + return ret; +} + +int ObTransformMVRewrite::map_to_mv_column(MvRewriteHelper &helper, + const ObColumnRefRawExpr *query_expr, + ObColumnRefRawExpr *&mv_expr) +{ + int ret = OB_SUCCESS; + ObSelectStmt *mv_stmt = helper.mv_info_.view_stmt_; + int64_t query_rel_id; + int64_t mv_tbl_idx; // = rel id - 1 + TableItem *mv_table_item; + ColumnItem *mv_column_item; + if (OB_ISNULL(query_expr) || OB_ISNULL(helper.query_stmt_) || OB_ISNULL(mv_stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(query_expr), K(helper.query_stmt_), K(mv_stmt)); + } else if (OB_FALSE_IT(query_rel_id = helper.query_stmt_->get_table_bit_index(query_expr->get_table_id()))) { + } else if (OB_UNLIKELY(query_rel_id < 1 || query_rel_id > helper.base_table_map_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected query rel id", K(ret), K(query_rel_id)); + } else if (OB_FALSE_IT(mv_tbl_idx = helper.base_table_map_.at(query_rel_id - 1))) { + } else if (-1 == mv_tbl_idx) { + // it is query delta table + mv_expr = NULL; + } else if (OB_UNLIKELY(mv_tbl_idx < 0 || mv_tbl_idx >= mv_stmt->get_table_size())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected mv table idx", K(ret), K(mv_tbl_idx), K(mv_stmt->get_table_size()), K(query_rel_id)); + } else if (OB_ISNULL(mv_table_item = mv_stmt->get_table_item(mv_tbl_idx))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null table item", K(ret), K(mv_tbl_idx)); + } else if (NULL == (mv_column_item + = mv_stmt->get_column_item(mv_table_item->table_id_, query_expr->get_column_id()))) { + mv_expr = NULL; + } else { + mv_expr = mv_column_item->expr_; + } + return ret; +} + +int ObTransformMVRewrite::check_other_predicate(MvRewriteHelper &helper, + const ObIArray &mv_conds, + const ObIArray &query_conds) +{ + int ret = OB_SUCCESS; + ObSqlBitSet<> mv_common_conds; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->expr_factory_) || OB_ISNULL(ctx_->session_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_)); + } + // check query conditions and generate mv compensation predicate + for (int64_t i = 0; OB_SUCC(ret) && i < query_conds.count(); ++i) { + bool has_found = false; + for (int64_t j = 0; OB_SUCC(ret) && j < mv_conds.count(); ++j) { + bool is_same = false; + if (has_found && mv_common_conds.has_member(j)) { + // do nothing + } else if (OB_FAIL(is_same_condition(helper, query_conds.at(i), mv_conds.at(j), is_same))) { + LOG_WARN("failed to compare condition", K(ret)); + } else if (!is_same) { + // do nothing + } else if (OB_FAIL(mv_common_conds.add_member(j))) { + LOG_WARN("failed to add members", K(ret)); + } else { + has_found = true; + // continue to check the rest of mv exprs to find all common conds + } + } + if (OB_SUCC(ret) && !has_found) { + if (OB_FAIL(helper.mv_compensation_preds_.push_back(query_conds.at(i)))) { + LOG_WARN("failed to push back compensate expr", K(ret)); + } + } + } + // check mv conditions and generate query compensation predicate + for (int64_t i = 0; OB_SUCC(ret) && i < mv_conds.count(); ++i) { + ObRawExpr *compensate_expr; + if (mv_common_conds.has_member(i)) { + // do nothing + } else if (OB_FAIL(ObRawExprUtils::build_lnnvl_expr(*ctx_->expr_factory_, + mv_conds.at(i), + compensate_expr))) { + LOG_WARN("failed to build lnnvl expr", K(ret)); + } else if (OB_FAIL(compensate_expr->formalize(ctx_->session_info_))) { + LOG_WARN("failed to formalize expr", K(ret)); + } else if (OB_FAIL(helper.query_compensation_preds_.push_back(compensate_expr))) { + LOG_WARN("failed to push back compensate expr", K(ret)); + } + } + return ret; +} + +/** + * @brief ObTransformMVRewrite::check_compensation_preds_validity + * + * If mv has group by, check all columns in mv compensation predicates are mv's + * group by column. + */ +int ObTransformMVRewrite::check_compensation_preds_validity(MvRewriteHelper &helper, + bool &is_valid) +{ + int ret = OB_SUCCESS; + ObSEArray comp_col_exprs; + ObSelectStmt *mv_stmt = helper.mv_info_.view_stmt_; + is_valid = true; + if (OB_ISNULL(mv_stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(mv_stmt)); + } else if (helper.mv_compensation_preds_.empty() + || !mv_stmt->has_group_by()) { + // do nothing + } else if (OB_FAIL(ObRawExprUtils::extract_column_exprs(helper.mv_compensation_preds_, comp_col_exprs))) { + LOG_WARN("failed to extract column exprs", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && is_valid && i < comp_col_exprs.count(); ++i) { + bool has_found = false; + for (int64_t j = 0; OB_SUCC(ret) && !has_found && j < mv_stmt->get_group_expr_size(); ++j) { + if (OB_FAIL(is_same_condition(helper, + comp_col_exprs.at(i), + mv_stmt->get_group_exprs().at(j), + has_found))) { + LOG_WARN("failed to compare group by expr", K(ret)); + } + } + if (OB_SUCC(ret) && !has_found) { + is_valid = false; + } + } + } + return ret; +} + +int ObTransformMVRewrite::classify_predicates(const ObIArray &input_conds, + ObIArray &equal_conds, + ObIArray &other_conds) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < input_conds.count(); ++i) { + ObRawExpr *expr = input_conds.at(i); + if (is_equal_cond(expr)) { + if (OB_FAIL(equal_conds.push_back(expr))) { + LOG_WARN("failed to push back equal expr", K(ret)); + } + } else { + if (OB_FAIL(other_conds.push_back(expr))) { + LOG_WARN("failed to push back other expr", K(ret)); + } + } + } + return ret; +} + +// t1.c1 = t2.c2 -> true +// t1.c1 = t1.c1 -> false +// t1.c1 > t2.c2 -> false +// t1.c1 = t2.c2 +1 -> false +bool ObTransformMVRewrite::is_equal_cond(ObRawExpr *expr) +{ + bool bret = true; + if (OB_ISNULL(expr)) { + bret = false; + } else if (T_OP_EQ != expr->get_expr_type()) { + bret = false; + } else if (OB_UNLIKELY(2 != expr->get_param_count())) { + bret = false; + } else if (OB_ISNULL(expr->get_param_expr(0))) { + bret = false; + } else if (!expr->get_param_expr(0)->is_column_ref_expr()) { + bret = false; + } else if (OB_ISNULL(expr->get_param_expr(1))) { + bret = false; + } else if (!expr->get_param_expr(1)->is_column_ref_expr()) { + bret = false; + } else if (expr->get_param_expr(0) == expr->get_param_expr(1)) { + // ObEqualAnalysis::compute_equal_set will discard such expr + bret = false; + } + return bret; +} + +// t1.c1 > 20 -> true +// t1.c1 > t2.c2 -> false +bool ObTransformMVRewrite::is_range_cond(ObRawExpr *expr) +{ + bool bret = true; + bool has_colref = false; + bool has_const = false; + if (OB_ISNULL(expr)) { + bret = false; + } else if (expr->get_expr_type() < T_OP_LE + || expr->get_expr_type() > T_OP_GT) { + bret = false; + } else if (OB_UNLIKELY(2 != expr->get_param_count())) { + bret = false; + } else { + if (OB_ISNULL(expr->get_param_expr(0))) { + bret = false; + } else if (expr->get_param_expr(0)->is_column_ref_expr()) { + has_colref = true; + } else if (expr->get_param_expr(0)->is_const_expr()) { + if (expr->get_type_class() >= ObIntTC + && expr->get_type_class() <= ObYearTC) { + has_const = true; + } + } + if (OB_ISNULL(expr->get_param_expr(1))) { + bret = false; + } else if (expr->get_param_expr(1)->is_column_ref_expr()) { + has_colref = true; + } else if (expr->get_param_expr(1)->is_const_expr()) { + if (expr->get_type_class() >= ObIntTC + && expr->get_type_class() <= ObYearTC) { + has_const = true; + } + } + } + if (bret && has_colref && has_const) { + bret = true; + } else { + bret = false; + } + return bret; +} + +int ObTransformMVRewrite::is_same_condition(MvRewriteHelper &helper, + const ObRawExpr *left, + const ObRawExpr *right, + bool &is_same) +{ + int ret = OB_SUCCESS; + MvRewriteCheckCtx context(helper); + is_same = false; + if (OB_ISNULL(left) || OB_ISNULL(right)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expr is null", K(ret)); + } else if (OB_FAIL(compare_expr(context, left, right, is_same))) { + LOG_WARN("failed to compare expr", K(ret)); + } else if (!is_same) { + // do nothing + } else if (OB_FAIL(append_constraint_info(helper, context))) { + LOG_WARN("failed to append constraint info", K(ret)); + } + return ret; +} + +int ObTransformMVRewrite::append_constraint_info(MvRewriteHelper &helper, + const MvRewriteCheckCtx &context) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(append(helper.equal_param_info_, context.equal_param_info_))) { + LOG_WARN("failed to append equal param", K(ret)); + } else if (OB_FAIL(append(helper.const_param_info_, context.const_param_info_))) { + LOG_WARN("failed to append const param", K(ret)); + } else if (OB_FAIL(append(helper.expr_cons_info_, context.expr_cons_info_))) { + LOG_WARN("failed to append expr param", K(ret)); + } + return ret; +} + +int ObTransformMVRewrite::compare_expr(MvRewriteCheckCtx &context, + const ObRawExpr *left, + const ObRawExpr *right, + bool &is_same) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(left) || OB_ISNULL(right)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expr is null", K(ret)); + } else if (!(is_same = left->same_as(*right, &context))) { + context.equal_param_info_.reset(); + context.const_param_info_.reset(); + context.expr_cons_info_.reset(); + if (!IS_COMMON_COMPARISON_OP(left->get_expr_type()) || + get_opposite_compare_type(left->get_expr_type()) != right->get_expr_type()) { + // do nothing + } else if (OB_ISNULL(left->get_param_expr(0)) || + OB_ISNULL(left->get_param_expr(1)) || + OB_ISNULL(right->get_param_expr(0)) || + OB_ISNULL(right->get_param_expr(1))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("param exprs are null", K(ret)); + } else if (!left->get_param_expr(0)->same_as(*right->get_param_expr(1), &context)) { + // do nothing + } else if (!left->get_param_expr(1)->same_as(*right->get_param_expr(0), &context)) { + // do nothing + } else { + is_same = true; + } + } + return ret; +} + +int ObTransformMVRewrite::check_group_by_col(MvRewriteHelper &helper, + bool &is_valid) +{ + int ret = OB_SUCCESS; + ObSelectStmt *mv_stmt = helper.mv_info_.view_stmt_; + is_valid = true; + if (OB_ISNULL(helper.query_stmt_) || OB_ISNULL(mv_stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(helper.query_stmt_), K(mv_stmt)); + } else if (!mv_stmt->has_group_by()) { + if (helper.query_stmt_->has_group_by()) { + helper.need_group_by_roll_up_ = true; + } else { + helper.need_group_by_roll_up_ = false; + } + } else if (!helper.query_stmt_->has_group_by()) { + is_valid = false; + } else { + ObSqlBitSet<> found_mv_gby_col; + for (int64_t i = 0; OB_SUCC(ret) && is_valid && i < helper.query_stmt_->get_group_expr_size(); ++i) { + bool has_found = false; + for (int64_t j = 0; OB_SUCC(ret) && !has_found && j < mv_stmt->get_group_expr_size(); ++j) { + if (OB_FAIL(is_same_condition(helper, + helper.query_stmt_->get_group_exprs().at(i), + mv_stmt->get_group_exprs().at(j), + has_found))) { + LOG_WARN("failed to compare group by expr", K(ret)); + } else if (!has_found) { + // do nothing + } else if (OB_FAIL(found_mv_gby_col.add_member(j))) { + LOG_WARN("failed to add member", K(ret), K(j)); + } + } + if (OB_SUCC(ret) && !has_found) { + is_valid = false; + } + } + if (OB_SUCC(ret) && is_valid) { + helper.need_group_by_roll_up_ = (found_mv_gby_col.num_members() != mv_stmt->get_group_expr_size()); + if (helper.need_group_by_roll_up_ && mv_stmt->has_having()) { + is_valid = false; + } + } + } + return ret; +} + +int ObTransformMVRewrite::check_opt_feat_ctrl(MvRewriteHelper &helper, + bool &is_valid) +{ + int ret = OB_SUCCESS; + uint64_t opt_version = LASTED_COMPAT_VERSION; + is_valid = true; + if (OB_ISNULL(helper.ori_stmt_.get_query_ctx()) || OB_ISNULL(helper.mv_info_.view_stmt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(helper.ori_stmt_.get_query_ctx()), K(helper.mv_info_.view_stmt_)); + } else if (OB_FALSE_IT(opt_version = helper.ori_stmt_.get_query_ctx()->optimizer_features_enable_version_)) { + } else if (opt_version >= LASTED_COMPAT_VERSION) { + // do nothing + } else if (opt_version < COMPAT_VERSION_4_3_3 + && (!helper.query_delta_table_.is_empty() + || helper.mv_info_.view_stmt_->has_group_by())) { + is_valid = false; + LOG_TRACE("optimizer feature is lower than 4.3.3", K(opt_version)); + } + return ret; +} + +int ObTransformMVRewrite::compute_stmt_expr_map(MvRewriteHelper &helper, + bool &is_valid) +{ + int ret = OB_SUCCESS; + is_valid = true; + if (OB_ISNULL(helper.query_stmt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("query stmt is null", K(ret)); + } else if (OB_FAIL(compute_join_expr_map(helper, is_valid))) { + LOG_WARN("failed to compute join expr", K(ret)); + } else if (!is_valid) { + LOG_TRACE("compute join expr does not pass"); + } else if (OB_FAIL(compute_select_expr_map(helper, is_valid))) { + LOG_WARN("failed to compute select expr", K(ret)); + } else if (!is_valid) { + LOG_TRACE("compute select expr does not pass"); + } else if (OB_FAIL(inner_compute_exprs_map(helper, helper.query_other_conds_, is_valid))) { + LOG_WARN("failed to compute query other conditions expr", K(ret)); + } else if (!is_valid) { + LOG_TRACE("compute query other conditions expr does not pass"); + } else if (OB_FAIL(inner_compute_exprs_map(helper, helper.mv_compensation_preds_, is_valid))) { + LOG_WARN("failed to compute compensation expr", K(ret)); + } else if (!is_valid) { + LOG_TRACE("compute compensation expr does not pass"); + } else if (OB_FAIL(inner_compute_exprs_map(helper, helper.query_stmt_->get_having_exprs(), is_valid))) { + LOG_WARN("failed to compute having expr", K(ret)); + } else if (!is_valid) { + LOG_TRACE("compute having expr does not pass"); + } else if (OB_FAIL(compute_group_by_expr_map(helper, is_valid))) { + LOG_WARN("failed to compute group by expr", K(ret)); + } else if (!is_valid) { + LOG_TRACE("compute group by expr does not pass"); + } else if (OB_FAIL(compute_order_by_expr_map(helper, is_valid))) { + LOG_WARN("failed to compute order by expr", K(ret)); + } else if (!is_valid) { + LOG_TRACE("compute order by expr does not pass"); + } + return ret; +} + +int ObTransformMVRewrite::compute_join_expr_map(MvRewriteHelper &helper, + bool &is_valid) +{ + int ret = OB_SUCCESS; + ObSEArray nodes_stack; + is_valid = true; + if (OB_FAIL(nodes_stack.push_back(helper.query_tree_))) { + LOG_WARN("failed to push back node", K(ret)); + } + while (OB_SUCC(ret) && is_valid && !nodes_stack.empty()) { + JoinTreeNode *current_node = NULL; + if (OB_FAIL(nodes_stack.pop_back(current_node))) { + LOG_WARN("failed to pop back node", K(ret)); + } else if (OB_ISNULL(current_node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("current node unexpected null", K(ret)); + } else if (current_node == helper.query_tree_mv_part_) { + // do nothing, do not need check mv part + } else if (OB_FAIL(inner_compute_exprs_map(helper, current_node->join_info_.on_conditions_, is_valid))) { + LOG_WARN("failed to compute on exprs map", K(ret)); } else if (!is_valid) { // do nothing - } else if (OB_FAIL(fill_select_item(origin_stmt, mv_info, helper, is_valid))) { - LOG_WARN("failed to add select item", K(ret)); + } else if (OB_FAIL(inner_compute_exprs_map(helper, current_node->join_info_.where_conditions_, is_valid))) { + LOG_WARN("failed to compute where exprs map", K(ret)); } else if (!is_valid) { // do nothing - } else if (OB_FAIL(fill_having_exprs(origin_stmt, mv_info, helper, is_valid))) { - LOG_WARN("failed to add having exprs", K(ret)); - } else if (!is_valid) { + } else if (UNKNOWN_JOIN == current_node->join_info_.join_type_) { + // do nothing, is leaf node + } else if (OB_FAIL(nodes_stack.push_back(current_node->left_child_))) { + LOG_WARN("failed to push back left node", K(ret)); + } else if (OB_FAIL(nodes_stack.push_back(current_node->right_child_))) { + LOG_WARN("failed to push back right node", K(ret)); + } + } + return ret; +} + +int ObTransformMVRewrite::compute_select_expr_map(MvRewriteHelper &helper, + bool &is_valid) +{ + int ret = OB_SUCCESS; + ObSEArray select_exprs; + if (OB_ISNULL(helper.query_stmt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (OB_FAIL(helper.query_stmt_->get_select_exprs(select_exprs))) { + LOG_WARN("failed to get select exprs", K(ret)); + } else if (OB_FAIL(inner_compute_exprs_map(helper, select_exprs, is_valid))) { + LOG_WARN("failed to compute exprs map", K(ret)); + } + return ret; +} + +int ObTransformMVRewrite::compute_group_by_expr_map(MvRewriteHelper &helper, + bool &is_valid) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(helper.query_stmt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (!helper.need_group_by_roll_up_) { + // do nothing + } else if (OB_FAIL(inner_compute_exprs_map(helper, helper.query_stmt_->get_group_exprs(), is_valid))) { + LOG_WARN("failed to compute group exprs map", K(ret)); + } else if (!is_valid) { + // do nothing + } else if (OB_FAIL(inner_compute_exprs_map(helper, helper.query_stmt_->get_rollup_exprs(), is_valid))) { + LOG_WARN("failed to compute rollup exprs map", K(ret)); + } + return ret; +} + +int ObTransformMVRewrite::compute_order_by_expr_map(MvRewriteHelper &helper, + bool &is_valid) +{ + int ret = OB_SUCCESS; + ObSEArray orderby_exprs; + if (OB_ISNULL(helper.query_stmt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (OB_FAIL(helper.query_stmt_->get_order_exprs(orderby_exprs))) { + LOG_WARN("failed to get order by exprs", K(ret)); + } else if (OB_FAIL(inner_compute_exprs_map(helper, orderby_exprs, is_valid))) { + LOG_WARN("failed to compute group exprs map", K(ret)); + } + return ret; +} + +int ObTransformMVRewrite::inner_compute_exprs_map(MvRewriteHelper &helper, + ObIArray &exprs, + bool &is_valid) +{ + int ret = OB_SUCCESS; + MvRewriteCheckCtx context(helper); + is_valid = true; + for (int64_t i = 0; OB_SUCC(ret) && is_valid && i < exprs.count(); ++i) { + if (OB_FAIL(inner_compute_expr_map(helper, + exprs.at(i), + context, + is_valid))) { + LOG_WARN("failed to compute join condition", K(ret)); + } + } + if (OB_FAIL(ret) || !is_valid) { + // do nothing + } else if (OB_FAIL(append_constraint_info(helper, context))) { + LOG_WARN("failed to append constraint info", K(ret)); + } + return ret; +} + +int ObTransformMVRewrite::inner_compute_expr_map(MvRewriteHelper &helper, + ObRawExpr *query_expr, + MvRewriteCheckCtx &context, + bool &is_valid) +{ + int ret = OB_SUCCESS; + ObSelectStmt *mv_stmt = helper.mv_info_.view_stmt_; + ObRawExpr *mv_select_expr = NULL; + is_valid = false; + if (OB_ISNULL(query_expr) || OB_ISNULL(mv_stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(query_expr), K(mv_stmt)); + } else if (query_expr->is_const_expr()) { + // const expr does not need to compute + is_valid = true; + } else if (helper.query_expr_replacer_.is_existed(query_expr)) { + // the input expr has been computed + is_valid = true; + } else if (mv_stmt->has_group_by() && query_expr->is_aggr_expr()) { + if (OB_FAIL(compute_agg_expr_map(helper, static_cast(query_expr), context, is_valid))) { + LOG_WARN("failed to compute agg expr map", K(ret)); + } + } else if (query_expr->get_relation_ids().is_subset(helper.query_delta_table_)) { + // column only appear in query, does not need to compute + is_valid = true; + } else if (OB_FAIL(find_mv_select_expr(helper, query_expr, context, mv_select_expr))) { + LOG_WARN("failed to find mv select expr", K(ret)); + } else if (NULL != mv_select_expr) { + if (OB_FAIL(helper.query_expr_replacer_.add_replace_expr(query_expr, mv_select_expr))) { + LOG_WARN("failed to add replaceed expr", K(ret), KPC(query_expr), KPC(mv_select_expr)); + } else { + is_valid = true; + } + } else if (query_expr->get_param_count() > 0) { + // Try to match each param expr + bool param_match = true; + MvRewriteCheckCtx new_ctx(helper); + for (int64_t i = 0; OB_SUCC(ret) && param_match && i < query_expr->get_param_count(); ++i) { + if (OB_FAIL(SMART_CALL(inner_compute_expr_map(helper, + query_expr->get_param_expr(i), + new_ctx, + param_match)))) { + LOG_WARN("failed to check param expr", K(ret)); + } + } + if (OB_SUCC(ret) && param_match) { + is_valid = true; + if (OB_FAIL(context.append_constraint_info(new_ctx))) { + LOG_WARN("failed to append constraint info", K(ret)); + } + } + } + return ret; +} + + +/** + * @brief ObTransformMVRewrite::compute_agg_expr_map + * + * If MV has group by, we can not just simply replace param expr for aggregate + * functions. Hence we need to handle the agg expr separately. + * + * 1. If need group by roll up, check whether query_expr can roll up. + * 2. Find the query_expr in MV select exprs. + * 3. If need group by roll up, wrap the roll up agg func. + */ +int ObTransformMVRewrite::compute_agg_expr_map(MvRewriteHelper &helper, + ObAggFunRawExpr *query_expr, + MvRewriteCheckCtx &context, + bool &is_valid) +{ + int ret = OB_SUCCESS; + ObRawExpr *mv_select_expr = NULL; + is_valid = true; + if (OB_ISNULL(query_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(query_expr)); + } else if (helper.need_group_by_roll_up_ && OB_FAIL(check_agg_roll_up_expr(query_expr, is_valid))) { + LOG_WARN("failed to check agg roll up expr", K(ret)); + } else if (!is_valid) { + // do nothing + } else if (OB_FAIL(find_mv_select_expr(helper, query_expr, context, mv_select_expr))) { + LOG_WARN("failed to find mv select expr", K(ret)); + } else if (NULL == mv_select_expr) { + is_valid = false; + } else if (helper.need_group_by_roll_up_ && OB_FAIL(wrap_agg_roll_up_expr(query_expr, mv_select_expr))) { + LOG_WARN("failed to wrap agg roll up expr", K(ret)); + } else if (OB_FAIL(helper.query_expr_replacer_.add_replace_expr(query_expr, mv_select_expr))) { + LOG_WARN("failed to add replaceed expr", K(ret), KPC(query_expr), KPC(mv_select_expr)); + } + return ret; +} + +int ObTransformMVRewrite::find_mv_select_expr(MvRewriteHelper &helper, + const ObRawExpr *query_expr, + MvRewriteCheckCtx &context, + ObRawExpr *&select_expr) +{ + int ret = OB_SUCCESS; + ObSelectStmt *mv_stmt = helper.mv_info_.view_stmt_; + ObSEArray mv_select_exprs; + bool is_match = false; + select_expr = NULL; + if (OB_ISNULL(query_expr) || OB_ISNULL(mv_stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(query_expr), K(mv_stmt)); + } else if (OB_FAIL(mv_stmt->get_select_exprs(mv_select_exprs))) { + LOG_WARN("failed to get select exprs", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && !is_match && i < mv_select_exprs.count(); ++i) { + MvRewriteCheckCtx new_ctx(helper); + if (OB_FAIL(compare_expr(new_ctx, query_expr, mv_select_exprs.at(i), is_match))) { + LOG_WARN("failed to check is condition equal", K(ret)); + } else if (!is_match) { // do nothing - } else if (OB_FAIL(fill_distinct(origin_stmt, mv_info, helper, is_valid))) { - LOG_WARN("failed to add distinct", K(ret)); - } else if (!is_valid) { - // do nothing - } else if (OB_FAIL(fill_orderby_exprs(origin_stmt, mv_info, helper, is_valid))) { - LOG_WARN("failed to add order by exprs", K(ret)); - } else if (!is_valid) { - // do nothing - } else if (OB_FAIL(fill_limit(origin_stmt, mv_info, helper, is_valid))) { - LOG_WARN("failed to add limit exprs", K(ret)); - } else if (!is_valid) { - // do nothing - } else if (OB_FAIL(expand_rt_mv_table(helper))) { - LOG_WARN("failed to expand real time mv", K(ret)); - } else if (OB_FAIL(helper.new_stmt_->formalize_stmt(ctx_->session_info_))) { - LOG_WARN("failed to formalize stmt info", K(ret)); + } else if (OB_FAIL(context.append_constraint_info(new_ctx))) { + LOG_WARN("failed to append constraint info", K(ret)); + } else { + select_expr = mv_select_exprs.at(i); + } + } + return ret; +} + +int ObTransformMVRewrite::check_agg_roll_up_expr(ObRawExpr *agg_expr, + bool &is_valid) +{ + int ret = OB_SUCCESS; + is_valid = false; + if (OB_ISNULL(agg_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(agg_expr)); + } else { + switch (agg_expr->get_expr_type()) { + case T_FUN_SUM: + case T_FUN_COUNT: + case T_FUN_COUNT_SUM: + case T_FUN_SYS_BIT_AND: + case T_FUN_SYS_BIT_OR: + case T_FUN_SYS_BIT_XOR: + is_valid = !static_cast(agg_expr)->is_param_distinct(); + break; + case T_FUN_MIN: + case T_FUN_MAX: + is_valid = true; + break; + default: + is_valid = false; + break; + } + } + return ret; +} + +/** + * @brief ObTransformMVRewrite::wrap_agg_roll_up_expr + * + * @param ori_expr the origin aggregate expr, used to determine the roll up func type + * @param output_expr the expr found in mv stmt, roll up will be performed on this expr + */ +int ObTransformMVRewrite::wrap_agg_roll_up_expr(ObRawExpr *ori_expr, + ObRawExpr *&output_expr) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(ori_expr) || OB_ISNULL(output_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ori_expr), K(output_expr)); + } else { + ObAggFunRawExpr *tmp_expr = NULL; + ObItemType wrap_type = T_INVALID; + switch (ori_expr->get_expr_type()) { + case T_FUN_COUNT: + wrap_type = T_FUN_COUNT_SUM; + break; + case T_FUN_SUM: + case T_FUN_COUNT_SUM: + case T_FUN_SYS_BIT_AND: + case T_FUN_SYS_BIT_OR: + case T_FUN_SYS_BIT_XOR: + case T_FUN_MIN: + case T_FUN_MAX: + wrap_type = ori_expr->get_expr_type(); + break; + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("input expr does not support aggregate roll up", K(ret), KPC(ori_expr), KPC(output_expr)); + break; } if (OB_FAIL(ret)) { - } else if (is_valid) { - new_stmt = helper.new_stmt_; - is_valid_transform = true; - OPT_TRACE("generate rewrite stmt use", mv_info.mv_schema_->get_table_name(), ":", new_stmt); - } else if (OB_FAIL(ObTransformUtils::free_stmt(*ctx_->stmt_factory_, helper.new_stmt_))) { - LOG_WARN("failed to free stmt", K(ret)); + // do nothing + } else if (OB_FAIL(ObTransformUtils::create_aggr_expr(ctx_, wrap_type, tmp_expr, output_expr))) { + LOG_WARN("failed to create sum expr", K(ret)); } else { - helper.new_stmt_ = NULL; + output_expr = tmp_expr; } } return ret; } -int ObTransformMVRewrite::create_mv_table_item(const MvInfo &mv_info, - GenerateStmtHelper &helper) +int ObTransformMVRewrite::generate_rewrite_stmt_contain_mode(MvRewriteHelper &helper) { int ret = OB_SUCCESS; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->session_info_) + || OB_ISNULL(helper.mv_info_.mv_schema_) || OB_ISNULL(helper.query_stmt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_), K(helper.mv_info_), K(helper.query_stmt_)); + } else if (OB_FAIL(clear_mv_common_tables(helper))) { + LOG_WARN("failed to clear rewrite stmt", K(ret)); + } else if (OB_FAIL(create_mv_table_item(helper))) { + LOG_WARN("failed to create mv table item", K(ret)); + } else if (OB_FAIL(create_mv_column_item(helper))) { + LOG_WARN("failed to create mv column item", K(ret)); + } else if (OB_FAIL(adjust_rewrite_stmt(helper))) { + LOG_WARN("failed to adjust rewrite stmt", K(ret)); + } else if (OB_FAIL(adjust_aggr_winfun_expr(helper.query_stmt_))) { + LOG_WARN("failed to adjust aggr winfun expr", K(ret)); + } else if (OB_FAIL(adjust_mv_item(helper))) { + LOG_WARN("failed to adjust mv item", K(ret)); + } else if (OB_FAIL(helper.query_stmt_->update_column_item_rel_id())) { + LOG_WARN("failed to update column item rel id", K(ret)); + } else if (OB_FAIL(helper.query_stmt_->formalize_stmt(ctx_->session_info_))) { + LOG_WARN("failed to formalize stmt", K(ret)); + } else { + OPT_TRACE("generate rewrite stmt use", helper.mv_info_.mv_schema_->get_table_name(), ":", helper.query_stmt_); + } + return ret; +} + +/** + * @brief ObTransformMVRewrite::clear_mv_common_tables + * + * Clear the tables that appear in both mv and origin query with the redundant + * column items, part exprs, etc. + */ +int ObTransformMVRewrite::clear_mv_common_tables(MvRewriteHelper &helper) { + int ret = OB_SUCCESS; + ObSelectStmt *rewrite_stmt = helper.query_stmt_; + ObSEArray mv_common_tables; + if (OB_ISNULL(rewrite_stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(rewrite_stmt)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < rewrite_stmt->get_table_size(); ++i) { + TableItem *table = rewrite_stmt->get_table_item(i); + if (OB_ISNULL(table)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table item is null", K(ret), K(i)); + } else if (helper.query_delta_table_.has_member(i + 1)) { + // do nothing + } else if (OB_FAIL(mv_common_tables.push_back(table))) { + LOG_WARN("push back table item failed", K(ret)); + } + } + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(rewrite_stmt->remove_table_info(mv_common_tables))) { + LOG_WARN("failed to remove query delta table info", K(ret)); + } else { + rewrite_stmt->get_joined_tables().reuse(); + rewrite_stmt->clear_from_items(); + } + return ret; +} + +int ObTransformMVRewrite::create_mv_table_item(MvRewriteHelper &helper) +{ + int ret = OB_SUCCESS; + ObSelectStmt *rewrite_stmt = helper.query_stmt_; + MvInfo &mv_info = helper.mv_info_; + const TableItem *mv_ori_item = NULL; TableItem *mv_item = NULL; ObString qb_name; - if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->allocator_) || OB_ISNULL(ctx_->schema_checker_) || - OB_ISNULL(ctx_->session_info_) || OB_ISNULL(helper.new_stmt_) || OB_ISNULL(helper.new_stmt_->get_query_ctx()) || - OB_ISNULL(mv_info.mv_schema_) || OB_ISNULL(mv_info.data_table_schema_) || OB_ISNULL(mv_info.db_schema_)) { + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->allocator_) || OB_ISNULL(ctx_->expr_factory_) + || OB_ISNULL(rewrite_stmt) || OB_ISNULL(rewrite_stmt->get_query_ctx()) + || OB_ISNULL(mv_info.mv_schema_) || OB_ISNULL(mv_info.data_table_schema_) + || OB_ISNULL(mv_info.select_mv_stmt_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(ctx_), K(helper.new_stmt_), K(mv_info)); - } else if (OB_FAIL(helper.new_stmt_->get_qb_name(qb_name))) { - } else if (OB_UNLIKELY(NULL == (mv_item = helper.new_stmt_->create_table_item(*ctx_->allocator_)))) { + LOG_WARN("get unexpected null", K(ret), K(ctx_), KPC(rewrite_stmt), K(mv_info)); + } else if (OB_FAIL(rewrite_stmt->get_qb_name(qb_name))) { + LOG_WARN("failed to get qb name", K(ret)); + } else if (OB_UNLIKELY(NULL == (mv_item = rewrite_stmt->create_table_item(*ctx_->allocator_)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("create table item failed", K(ret)); + } else if (OB_UNLIKELY(!mv_info.select_mv_stmt_->is_single_table_stmt())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("select mv stmt is not single table stmt", K(ret), KPC(mv_info.select_mv_stmt_)); + } else if (OB_ISNULL(mv_ori_item = mv_info.select_mv_stmt_->get_table_item(0))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mv ori item is null", K(ret)); } else { - mv_item->type_ = TableItem::ALIAS_TABLE; - // mv_item->synonym_name_ = ; - // mv_item->synonym_db_name_ = ; - mv_item->database_name_ = mv_info.db_schema_->get_database_name_str(); - mv_item->ref_id_ = mv_info.mv_schema_->get_data_table_id(); - mv_item->mview_id_ = mv_info.mv_schema_->get_table_id(); - mv_item->table_type_ = mv_info.mv_schema_->get_table_type(); - mv_item->table_id_ = helper.new_stmt_->get_query_ctx()->available_tb_id_--; - mv_item->is_index_table_ = mv_info.mv_schema_->is_index_table(); - mv_item->table_name_ = mv_info.data_table_schema_->get_table_name_str(); - mv_item->alias_name_ = mv_info.mv_schema_->get_table_name_str(); - mv_item->qb_name_ = qb_name; + ObRawExprCopier copier(*ctx_->expr_factory_); ObSchemaObjVersion table_version; table_version.object_id_ = mv_info.mv_schema_->get_table_id(); @@ -857,504 +2649,375 @@ int ObTransformMVRewrite::create_mv_table_item(const MvInfo &mv_info, data_table_version.is_db_explicit_ = true; uint64_t data_dep_db_id = mv_info.data_table_schema_->get_database_id(); - if (OB_FAIL(helper.new_stmt_->add_global_dependency_table(table_version))) { - LOG_WARN("add global dependency table failed", K(ret)); - } else if (OB_FAIL(helper.new_stmt_->add_ref_obj_version(mv_info.mv_schema_->get_table_id(), - dep_db_id, - ObObjectType::VIEW, - table_version, - *ctx_->allocator_))) { - LOG_WARN("failed to add ref obj version", K(ret)); - } else if (OB_FAIL(helper.new_stmt_->add_global_dependency_table(data_table_version))) { - LOG_WARN("add global dependency table failed", K(ret)); - } else if (OB_FAIL(helper.new_stmt_->add_ref_obj_version(mv_info.mv_schema_->get_table_id(), - data_dep_db_id, - ObObjectType::VIEW, - data_table_version, - *ctx_->allocator_))) { - LOG_WARN("failed to add ref obj version", K(ret)); - } else if (OB_FAIL(helper.new_stmt_->add_table_item(ctx_->session_info_, mv_item))) { - LOG_WARN("push back table item failed", K(ret), KPC(mv_item)); - } else if (OB_FAIL(helper.new_stmt_->rebuild_tables_hash())) { + if (OB_FAIL(mv_item->deep_copy(copier, *mv_ori_item, ctx_->allocator_))) { + LOG_WARN("failed to deep copy table item", K(ret)); + } else if (OB_FALSE_IT(mv_item->table_id_ = rewrite_stmt->get_query_ctx()->available_tb_id_--)) { + } else if (OB_FALSE_IT(mv_item->qb_name_ = qb_name)) { + } else if (OB_FAIL(ObTransformUtils::add_table_item(rewrite_stmt, mv_item))) { + LOG_WARN("failed to add mv table item", K(ret)); + } else if (OB_FAIL(rewrite_stmt->rebuild_tables_hash())) { LOG_WARN("failed to rebuid table hash", K(ret)); + } else if (OB_FAIL(rewrite_stmt->add_global_dependency_table(table_version))) { + LOG_WARN("add global dependency table failed", K(ret)); + } else if (OB_FAIL(rewrite_stmt->add_ref_obj_version(mv_info.mv_schema_->get_table_id(), + dep_db_id, + ObObjectType::VIEW, + table_version, + *ctx_->allocator_))) { + LOG_WARN("failed to add ref obj version", K(ret)); + } else if (OB_FAIL(rewrite_stmt->add_global_dependency_table(data_table_version))) { + LOG_WARN("add global dependency table failed", K(ret)); + } else if (OB_FAIL(rewrite_stmt->add_ref_obj_version(mv_info.mv_schema_->get_table_id(), + data_dep_db_id, + ObObjectType::VIEW, + data_table_version, + *ctx_->allocator_))) { + LOG_WARN("failed to add ref obj version", K(ret)); + } else { + LOG_DEBUG("succ to fill table_item", K(mv_item)); } } if (OB_SUCC(ret)) { + helper.mv_upper_stmt_ = rewrite_stmt; helper.mv_item_ = mv_item; } return ret; } -int ObTransformMVRewrite::create_mv_column_item(const MvInfo &mv_info, - GenerateStmtHelper &helper) +int ObTransformMVRewrite::create_mv_column_item(MvRewriteHelper &helper) { int ret = OB_SUCCESS; - if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->expr_factory_) || OB_ISNULL(ctx_->sql_schema_guard_) - || OB_ISNULL(ctx_->sql_schema_guard_->get_schema_guard()) || OB_ISNULL(ctx_->session_info_) - || OB_ISNULL(helper.new_stmt_) || OB_ISNULL(helper.mv_item_) - || OB_ISNULL(mv_info.view_stmt_) || OB_ISNULL(mv_info.data_table_schema_)) { + MvInfo &mv_info = helper.mv_info_; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->expr_factory_) + || OB_ISNULL(helper.query_stmt_) || OB_ISNULL(helper.mv_item_) + || OB_ISNULL(mv_info.view_stmt_) || OB_ISNULL(mv_info.select_mv_stmt_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(helper.new_stmt_), K(helper.mv_item_), K(mv_info)); - } - // create mv column item - for (int64_t i = 0; OB_SUCC(ret) && i < mv_info.data_table_schema_->get_column_count(); ++i) { - const ObColumnSchemaV2 *col_schema = NULL; - ObColumnRefRawExpr *col_expr = NULL; - bool is_uni = false; - bool is_mul = false; - ColumnItem column_item; - if (OB_ISNULL(col_schema = mv_info.data_table_schema_->get_column_schema_by_idx(i))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("column schema is null", K(ret)); - } else if (OB_FAIL(ObRawExprUtils::build_column_expr(*ctx_->expr_factory_, *col_schema, col_expr))) { - LOG_WARN("build column expr failed", K(ret)); - } else if (OB_FAIL(mv_info.data_table_schema_->is_unique_key_column(*ctx_->sql_schema_guard_->get_schema_guard(), - col_schema->get_column_id(), - is_uni))) { - LOG_WARN("fail to check is unique key column", K(ret), KPC(mv_info.data_table_schema_), K(col_schema->get_column_id())); - } else if (OB_FAIL(mv_info.data_table_schema_->is_multiple_key_column(*ctx_->sql_schema_guard_->get_schema_guard(), - col_schema->get_column_id(), - is_mul))) { - LOG_WARN("fail to check is multiple key column", K(ret), KPC(mv_info.data_table_schema_), K(col_schema->get_column_id())); - } else { - if (!ob_enable_lob_locator_v2()) { - // Notice: clob will not convert to ObLobType if locator v2 enabled - if (is_oracle_mode() && ObLongTextType == col_expr->get_data_type() - && ! is_virtual_table(col_schema->get_table_id())) { - col_expr->set_data_type(ObLobType); + LOG_WARN("get unexpected null", K(ret), K(helper.query_stmt_), K(helper.mv_item_), K(mv_info)); + } else { + ObRawExprCopier copier(*ctx_->expr_factory_); + // 1. create mv column item + for (int64_t i = 0; OB_SUCC(ret) && i < mv_info.select_mv_stmt_->get_column_items().count(); ++i) { + ColumnItem column_item; + if (OB_FAIL(column_item.deep_copy(copier, mv_info.select_mv_stmt_->get_column_items().at(i)))) { + LOG_WARN("build column expr failed", K(ret)); + } else { + column_item.expr_->set_ref_id(helper.mv_item_->table_id_, column_item.expr_->get_column_id()); + column_item.table_id_ = column_item.expr_->get_table_id(); + if (OB_FAIL(helper.query_stmt_->add_column_item(column_item))) { + LOG_WARN("add column item to stmt failed", K(ret)); + } else if (OB_FAIL(column_item.expr_->pull_relation_id())) { + LOG_WARN("failed to pullup relation ids", K(ret)); + } else { + LOG_DEBUG("succ to fill column_item", K(column_item)); } } - if (helper.mv_item_->alias_name_.empty()) { - col_expr->set_synonym_db_name(helper.mv_item_->synonym_db_name_); - col_expr->set_synonym_name(helper.mv_item_->synonym_name_); - } - col_expr->set_column_attr(helper.mv_item_->get_table_name(), col_schema->get_column_name_str()); - col_expr->set_from_alias_table(!helper.mv_item_->alias_name_.empty()); - col_expr->set_database_name(helper.mv_item_->database_name_); - //column maybe from alias table, so must reset ref id by table id from table_item - col_expr->set_ref_id(helper.mv_item_->table_id_, col_schema->get_column_id()); - col_expr->set_unique_key_column(is_uni); - col_expr->set_mul_key_column(is_mul); - if (!helper.mv_item_->alias_name_.empty()) { - col_expr->set_table_alias_name(); - } - col_expr->set_lob_column(is_lob_storage(col_schema->get_data_type())); - if (ctx_->session_info_->get_ddl_info().is_ddl()) { - column_item.set_default_value(col_schema->get_orig_default_value()); - } else { - column_item.set_default_value(col_schema->get_cur_default_value()); - } - column_item.expr_ = col_expr; - column_item.table_id_ = col_expr->get_table_id(); - column_item.column_id_ = col_expr->get_column_id(); - column_item.column_name_ = col_expr->get_column_name(); - column_item.base_tid_ = helper.mv_item_->ref_id_; - column_item.base_cid_ = column_item.column_id_; - column_item.is_geo_ = col_schema->is_geometry(); - LOG_DEBUG("succ to fill column_item", K(column_item), KPC(col_schema)); - if (OB_FAIL(helper.new_stmt_->add_column_item(column_item))) { - LOG_WARN("add column item to stmt failed", K(ret)); - } else if (OB_FAIL(col_expr->pull_relation_id())) { - LOG_WARN("failed to pullup relation ids", K(ret)); + } + // 2. add column expr to copier + for (int64_t i = 0; OB_SUCC(ret) && i < mv_info.view_stmt_->get_select_item_size(); ++i) { + ObColumnRefRawExpr *col_expr = NULL; + if (OB_ISNULL(col_expr = helper.query_stmt_->get_column_expr_by_id(helper.mv_item_->table_id_, OB_APP_MIN_COLUMN_ID + i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get column id", K(ret), K(helper.mv_item_->table_id_), K(OB_APP_MIN_COLUMN_ID + i)); + } else if (OB_FAIL(helper.mv_select_replacer_.add_replace_expr(mv_info.view_stmt_->get_select_item(i).expr_, col_expr))) { + LOG_WARN("failed to add replaced expr", K(ret), KPC(mv_info.view_stmt_->get_select_item(i).expr_), KPC(col_expr)); } } - } - // add column expr to copier - for (int64_t i = 0; OB_SUCC(ret) && i < mv_info.view_stmt_->get_select_item_size(); ++i) { - ObColumnRefRawExpr *col_expr = NULL; - if (OB_ISNULL(col_expr = helper.new_stmt_->get_column_expr_by_id(helper.mv_item_->table_id_, OB_APP_MIN_COLUMN_ID + i))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to get column id", K(ret), K(helper.mv_item_->table_id_), K(OB_APP_MIN_COLUMN_ID + i)); - } else if (OB_FAIL(helper.col_copier_.add_replaced_expr(mv_info.view_stmt_->get_select_item(i).expr_, col_expr))) { - LOG_WARN("failed to add replaced expr", K(ret), KPC(mv_info.view_stmt_->get_select_item(i).expr_), KPC(col_expr)); - } - } - // fill part expr - if (OB_SUCC(ret) && NULL != mv_info.select_mv_stmt_ - && mv_info.select_mv_stmt_->get_part_exprs().count() > 0) { - ObRawExprCopier part_expr_copier(*ctx_->expr_factory_); // FROM select_mv_stmt_ TO new_stmt_ - ObSEArray mv_col_exprs; // column expr in select_mv_stmt_ - // make part_expr_copier which maps select_mv_stmt_ columns to new_stmt_ columns - if (OB_FAIL(mv_info.select_mv_stmt_->get_column_exprs(mv_col_exprs))) { - LOG_WARN("failed to get column exprs", K(ret)); - } - for (int64_t i = 0; OB_SUCC(ret) && i < mv_col_exprs.count(); ++i) { - ObColumnRefRawExpr *col_expr = NULL; // column expr in new_stmt_ - if (OB_ISNULL(mv_col_exprs.at(i))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("column expr is null", K(ret), K(i)); - } else if (OB_ISNULL(col_expr = helper.new_stmt_->get_column_expr_by_id(helper.mv_item_->table_id_, mv_col_exprs.at(i)->get_column_id()))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to get column id", K(ret), K(helper.mv_item_->table_id_), K(i)); - } else if (OB_FAIL(part_expr_copier.add_replaced_expr(mv_col_exprs.at(i), col_expr))) { - LOG_WARN("failed to add replaced expr", K(ret), KPC(mv_col_exprs.at(i)), KPC(col_expr)); + // 3. fill part expr + // 3.1. fill expr copier, from select_mv_stmt_ column to query_stmt_ column + if (OB_SUCC(ret) && mv_info.select_mv_stmt_->get_part_exprs().count() > 0) { + ObRawExprCopier part_expr_copier(*ctx_->expr_factory_); // FROM select_mv_stmt_ TO query_stmt_ + ObSEArray mv_col_exprs; // column expr in select_mv_stmt_ + // make part_expr_copier which maps select_mv_stmt_ columns to query_stmt_ columns + if (OB_FAIL(mv_info.select_mv_stmt_->get_column_exprs(mv_col_exprs))) { + LOG_WARN("failed to get column exprs", K(ret)); } - } - // do fill part expr for new_stmt_ - for (int64_t i = 0; OB_SUCC(ret) && i < mv_info.select_mv_stmt_->get_part_exprs().count(); ++i) { - ObDMLStmt::PartExprItem &part_item = mv_info.select_mv_stmt_->get_part_exprs().at(i); - ObRawExpr *part_expr = NULL; - ObRawExpr *subpart_expr = NULL; - if (OB_FAIL(part_expr_copier.copy_on_replace(part_item.part_expr_, part_expr))) { - LOG_WARN("failed to copy part expr", K(ret), K(part_item)); - } else if (NULL != part_item.subpart_expr_ - && OB_FAIL(part_expr_copier.copy_on_replace(part_item.subpart_expr_, subpart_expr))) { - LOG_WARN("failed to copy subpart expr", K(ret), K(part_item)); - } else if (OB_FAIL(helper.new_stmt_->set_part_expr(helper.mv_item_->table_id_, - part_item.index_tid_, - part_expr, - subpart_expr))) { - LOG_WARN("set part expr to new stmt failed", K(ret)); + for (int64_t i = 0; OB_SUCC(ret) && i < mv_col_exprs.count(); ++i) { + ObColumnRefRawExpr *col_expr = NULL; // column expr in query_stmt_ + if (OB_ISNULL(mv_col_exprs.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column expr is null", K(ret), K(i)); + } else if (OB_ISNULL(col_expr = helper.query_stmt_->get_column_expr_by_id(helper.mv_item_->table_id_, + mv_col_exprs.at(i)->get_column_id()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get column id", K(ret), K(helper.mv_item_->table_id_), K(i)); + } else if (OB_FAIL(part_expr_copier.add_replaced_expr(mv_col_exprs.at(i), col_expr))) { + LOG_WARN("failed to add replaced expr", K(ret), KPC(mv_col_exprs.at(i)), KPC(col_expr)); + } + } + // 3.2. do fill part expr for query_stmt_ + for (int64_t i = 0; OB_SUCC(ret) && i < mv_info.select_mv_stmt_->get_part_exprs().count(); ++i) { + ObDMLStmt::PartExprItem &part_item = mv_info.select_mv_stmt_->get_part_exprs().at(i); + ObRawExpr *part_expr = NULL; + ObRawExpr *subpart_expr = NULL; + if (OB_FAIL(part_expr_copier.copy_on_replace(part_item.part_expr_, part_expr))) { + LOG_WARN("failed to copy part expr", K(ret), K(part_item)); + } else if (NULL != part_item.subpart_expr_ + && OB_FAIL(part_expr_copier.copy_on_replace(part_item.subpart_expr_, subpart_expr))) { + LOG_WARN("failed to copy subpart expr", K(ret), K(part_item)); + } else if (OB_FAIL(helper.query_stmt_->set_part_expr(helper.mv_item_->table_id_, + part_item.index_tid_, + part_expr, + subpart_expr))) { + LOG_WARN("set part expr to new stmt failed", K(ret)); + } } } } return ret; } -int ObTransformMVRewrite::fill_from_item(ObSelectStmt *origin_stmt, - const MvInfo &mv_info, - GenerateStmtHelper &helper, - bool &is_valid) +/** + * @brief ObTransformMVRewrite::adjust_rewrite_stmt + * + * Generate the rewrite stmt, and replace column expr of tables in mv into mv column. + * + * 1. Generate new from items and joined tables for rewrite stmt based on query + * join tree. + * 2. Reset the where conditions based on query join tree. + * 3. Reset group by exprs if do not need_group_by_roll_up. + * 4. Replace all mv relation exprs into mv column. + */ +int ObTransformMVRewrite::adjust_rewrite_stmt(MvRewriteHelper &helper) { int ret = OB_SUCCESS; - TableItem *mv_item = NULL; - is_valid = false; - if (OB_ISNULL(origin_stmt) || OB_ISNULL(helper.new_stmt_) || OB_ISNULL(helper.mv_item_)) { + ObSelectStmt *rewrite_stmt = helper.query_stmt_; + TableItem *from_table = NULL; + ObSEArray where_conditions; + helper.query_expr_replacer_.set_relation_scope(); + helper.query_expr_replacer_.set_recursive(false); + helper.mv_select_replacer_.set_relation_scope(); + helper.mv_select_replacer_.set_recursive(false); + if (OB_ISNULL(rewrite_stmt)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(origin_stmt), K(helper.new_stmt_), K(helper.mv_item_)); - } else if (OB_FAIL(helper.new_stmt_->add_from_item(helper.mv_item_->table_id_, false))) { - LOG_WARN("failed to add from item", K(ret), KPC(helper.new_stmt_), KPC(helper.mv_item_)); - } else { - is_valid = true; - } - return ret; -} - -int ObTransformMVRewrite::fill_select_item(ObSelectStmt *origin_stmt, - const MvInfo &mv_info, - GenerateStmtHelper &helper, - bool &is_valid) -{ - int ret = OB_SUCCESS; - ObSEArray ori_select_exprs; - ObSEArray mv_select_exprs; - ObSEArray new_select_exprs; - bool is_sub_valid = false; - is_valid = false; - if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->allocator_) || OB_ISNULL(origin_stmt) - || OB_ISNULL(mv_info.view_stmt_) || OB_ISNULL(helper.new_stmt_) - || OB_ISNULL(helper.map_info_)) { + LOG_WARN("get unexpected null", K(ret), K(rewrite_stmt)); + } else if (OB_FAIL(recursive_build_from_table(helper, + helper.query_tree_, + from_table, + where_conditions))) { + LOG_WARN("failed to build from table", K(ret)); + } else if (OB_ISNULL(from_table)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(ctx_), K(origin_stmt), K(helper.new_stmt_), K(helper.map_info_)); - } else if (OB_FAIL(origin_stmt->get_select_exprs(ori_select_exprs))) { - LOG_WARN("failed to get origin select exprs", K(ret)); - } else if (OB_FAIL(mv_info.view_stmt_->get_select_exprs(mv_select_exprs))) { - LOG_WARN("failed to get mv select exprs", K(ret)); - } else if (OB_FAIL(ObStmtComparer::compute_new_expr(ori_select_exprs, - origin_stmt, - mv_select_exprs, - mv_info.view_stmt_, - *helper.map_info_, - helper.compute_expr_copier_, - new_select_exprs, - is_sub_valid))) { - LOG_WARN("failed to compute select expr", K(ret)); - } else if (!is_sub_valid) { - LOG_TRACE("select exprs can not be computed", K(new_select_exprs)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < new_select_exprs.count(); ++i) { - ObRawExpr *select_expr = NULL; - if (OB_FAIL(helper.col_copier_.copy_on_replace(new_select_exprs.at(i), select_expr))) { - LOG_WARN("failed to copy expr", K(ret), KPC(new_select_exprs.at(i))); - } else if (OB_FAIL(ObTransformUtils::create_select_item(*ctx_->allocator_, - select_expr, - helper.new_stmt_))) { - LOG_WARN("failed to create select item", K(ret)); - } else if (OB_FAIL(ObTransformUtils::add_aggr_winfun_expr(helper.new_stmt_, select_expr))) { - LOG_WARN("failed to add aggr winfun expr", K(ret), KPC(select_expr)); - } - } - is_valid = true; - } - return ret; -} - -int ObTransformMVRewrite::fill_condition_exprs(ObSelectStmt *origin_stmt, - const MvInfo &mv_info, - GenerateStmtHelper &helper, - bool &is_valid) -{ - int ret = OB_SUCCESS; - ObSEArray origin_unmatched_conds; - ObSEArray mv_unmatched_conds; - ObSEArray mv_select_exprs; - ObSEArray cond_exprs; - ObSEArray new_cond_exprs; - bool is_sub_valid = false; - is_valid = false; - if (OB_ISNULL(origin_stmt) || OB_ISNULL(mv_info.view_stmt_) - || OB_ISNULL(helper.map_info_) || OB_ISNULL(helper.new_stmt_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(origin_stmt), K(mv_info), K(helper.map_info_)); - } else if (OB_FAIL(ObStmtComparer::compute_unmatched_item(helper.map_info_->cond_map_, - origin_stmt->get_condition_size(), - mv_info.view_stmt_->get_condition_size(), - origin_unmatched_conds, - mv_unmatched_conds))) { - LOG_WARN("failed to compute unmatched item", K(ret)); - } else if (OB_FAIL(cond_exprs.prepare_allocate(origin_unmatched_conds.count()))) { - LOG_WARN("failed to pre-allocate table map", K(ret)); - } - for (int64_t i = 0; OB_SUCC(ret) && i < origin_unmatched_conds.count(); ++i) { - cond_exprs.at(i) = origin_stmt->get_condition_expr(origin_unmatched_conds.at(i)); + LOG_WARN("from table is null", K(ret)); + } else if (OB_FAIL(rewrite_stmt->add_from_item(from_table->table_id_, + from_table->is_joined_table()))) { + LOG_WARN("failed to add from item", K(ret)); + } else if (OB_FALSE_IT(rewrite_stmt->get_condition_exprs().reuse())) { + } else if (OB_FAIL(append(rewrite_stmt->get_condition_exprs(), where_conditions))) { + LOG_WARN("failed to append where conditions", K(ret)); + } else if (OB_FAIL(append(rewrite_stmt->get_condition_exprs(), helper.query_other_conds_))) { + LOG_WARN("failed to append query other conditions", K(ret)); + } else if (!helper.need_group_by_roll_up_) { + rewrite_stmt->get_group_exprs().reset(); } + // replace relation exprs if (OB_FAIL(ret)) { - } else if (OB_FAIL(mv_info.view_stmt_->get_select_exprs(mv_select_exprs))) { - LOG_WARN("failed to get mv select exprs", K(ret)); - } else if (OB_FAIL(ObStmtComparer::compute_new_expr(cond_exprs, - origin_stmt, - mv_select_exprs, - mv_info.view_stmt_, - *helper.map_info_, - helper.compute_expr_copier_, - new_cond_exprs, - is_sub_valid))) { - LOG_WARN("failed to compute select expr", K(ret)); - } else if (!is_sub_valid) { - LOG_TRACE("condition exprs can not be computed", K(new_cond_exprs)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < new_cond_exprs.count(); ++i) { - ObRawExpr *cond_expr = NULL; - if (OB_FAIL(helper.col_copier_.copy_on_replace(new_cond_exprs.at(i), cond_expr))) { - LOG_WARN("failed to copy expr", K(ret), KPC(new_cond_exprs.at(i))); - } else if (OB_FAIL(helper.new_stmt_->add_condition_expr(cond_expr))) { - LOG_WARN("failed to add condition expr", K(ret), KPC(cond_expr)); - } - } - is_valid = true; - } - return ret; -} - -int ObTransformMVRewrite::fill_having_exprs(ObSelectStmt *origin_stmt, - const MvInfo &mv_info, - GenerateStmtHelper &helper, - bool &is_valid) -{ - int ret = OB_SUCCESS; - ObSEArray origin_unmatched_conds; - ObSEArray mv_unmatched_conds; - ObSEArray mv_select_exprs; - ObSEArray new_having_exprs; - bool is_sub_valid = false; - is_valid = false; - if (OB_ISNULL(origin_stmt) || OB_ISNULL(mv_info.view_stmt_) - || OB_ISNULL(helper.map_info_) || OB_ISNULL(helper.new_stmt_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(origin_stmt), K(mv_info), K(helper.map_info_)); - } else if (OB_FAIL(mv_info.view_stmt_->get_select_exprs(mv_select_exprs))) { - LOG_WARN("failed to get mv select exprs", K(ret)); - } else if (OB_FAIL(ObStmtComparer::compute_new_expr(origin_stmt->get_having_exprs(), - origin_stmt, - mv_select_exprs, - mv_info.view_stmt_, - *helper.map_info_, - helper.compute_expr_copier_, - new_having_exprs, - is_sub_valid))) { - LOG_WARN("failed to compute select expr", K(ret)); - } else if (!is_sub_valid) { - LOG_TRACE("having exprs can not be computed", K(new_having_exprs)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < new_having_exprs.count(); ++i) { - ObRawExpr *having_expr = NULL; - if (OB_FAIL(helper.col_copier_.copy_on_replace(new_having_exprs.at(i), having_expr))) { - LOG_WARN("failed to copy expr", K(ret), KPC(new_having_exprs.at(i))); - } else if (OB_FAIL(helper.new_stmt_->add_having_expr(having_expr))) { - LOG_WARN("failed to add having expr", K(ret), KPC(having_expr)); - } else if (OB_FAIL(ObTransformUtils::add_aggr_winfun_expr(helper.new_stmt_, having_expr))) { - LOG_WARN("failed to add aggr winfun expr", K(ret), KPC(having_expr)); - } - } - is_valid = true; - } - return ret; -} - -int ObTransformMVRewrite::fill_groupby_exprs(ObSelectStmt *origin_stmt, - const MvInfo &mv_info, - GenerateStmtHelper &helper, - bool &is_valid) -{ - // TODO only mv has no group by - int ret = OB_SUCCESS; - ObSEArray mv_select_exprs; - ObSEArray new_groupby_exprs; - ObSEArray new_rollup_exprs; - bool is_sub_valid = false; - is_valid = false; - if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->allocator_) || OB_ISNULL(origin_stmt) - || OB_ISNULL(mv_info.view_stmt_) || OB_ISNULL(helper.new_stmt_) - || OB_ISNULL(helper.map_info_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(ctx_), K(origin_stmt), K(mv_info), K(helper.new_stmt_), K(helper.map_info_)); - } else if (OB_FAIL(mv_info.view_stmt_->get_select_exprs(mv_select_exprs))) { - LOG_WARN("failed to get mv select exprs", K(ret)); - } else if (OB_FAIL(ObStmtComparer::compute_new_expr(origin_stmt->get_group_exprs(), - origin_stmt, - mv_select_exprs, - mv_info.view_stmt_, - *helper.map_info_, - helper.compute_expr_copier_, - new_groupby_exprs, - is_sub_valid))) { - LOG_WARN("failed to compute select expr", K(ret)); - } else if (!is_sub_valid) { - LOG_TRACE("group by exprs can not be computed", K(new_groupby_exprs)); - } else if (OB_FAIL(ObStmtComparer::compute_new_expr(origin_stmt->get_rollup_exprs(), - origin_stmt, - mv_select_exprs, - mv_info.view_stmt_, - *helper.map_info_, - helper.compute_expr_copier_, - new_rollup_exprs, - is_sub_valid))) { - LOG_WARN("failed to compute select expr", K(ret)); - } else if (!is_sub_valid) { // do nothing - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < new_groupby_exprs.count(); ++i) { - ObRawExpr *groupby_expr = NULL; - if (OB_FAIL(helper.col_copier_.copy_on_replace(new_groupby_exprs.at(i), groupby_expr))) { - LOG_WARN("failed to copy expr", K(ret), KPC(new_groupby_exprs.at(i))); - } else if (OB_FAIL(helper.new_stmt_->get_group_exprs().push_back(groupby_expr))) { - LOG_WARN("failed to push back group by expr", K(ret), KPC(groupby_expr)); - } - } - for (int64_t i = 0; OB_SUCC(ret) && i < new_rollup_exprs.count(); ++i) { - ObRawExpr *rollup_expr = NULL; - if (OB_FAIL(helper.col_copier_.copy_on_replace(new_rollup_exprs.at(i), rollup_expr))) { - LOG_WARN("failed to copy expr", K(ret), KPC(new_rollup_exprs.at(i))); - } else if (OB_FAIL(helper.new_stmt_->get_rollup_exprs().push_back(rollup_expr))) { - LOG_WARN("failed to push back roll up expr", K(ret), KPC(rollup_expr)); - } - } - is_valid = true; + } else if (OB_FAIL(rewrite_stmt->iterate_stmt_expr(helper.query_expr_replacer_))) { + LOG_WARN("failed to iterate stmt expr, query expr replacer", K(ret)); + } else if (OB_FAIL(rewrite_stmt->iterate_stmt_expr(helper.mv_select_replacer_))) { + LOG_WARN("failed to iterate stmt expr, mv select replacer", K(ret)); } return ret; } -int ObTransformMVRewrite::fill_distinct(ObSelectStmt *origin_stmt, - const MvInfo &mv_info, - GenerateStmtHelper &helper, - bool &is_valid) +int ObTransformMVRewrite::recursive_build_from_table(MvRewriteHelper &helper, + JoinTreeNode *node, + TableItem *&output_table, + ObIArray &filters) { int ret = OB_SUCCESS; - is_valid = false; - if (OB_ISNULL(origin_stmt) || OB_ISNULL(mv_info.view_stmt_) - || OB_ISNULL(helper.new_stmt_) || OB_ISNULL(helper.map_info_)) { + ObSelectStmt *rewrite_stmt = helper.query_stmt_; + TableItem *left_table = NULL; + TableItem *right_table = NULL; + ObSEArray left_filters; + ObSEArray right_filters; + JoinedTable *new_joined_table; + output_table = NULL; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->allocator_) + || OB_ISNULL(node) || OB_ISNULL(rewrite_stmt)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(origin_stmt), K(mv_info), K(helper.new_stmt_), K(helper.map_info_)); - } else if (!origin_stmt->is_distinct()) { // here we assume that mv will not contain DISTINCT - is_valid = true; - } else if (origin_stmt->is_distinct()) { - helper.new_stmt_->assign_distinct(); - is_valid = true; - } else { - is_valid = false; - } - return ret; -} - -int ObTransformMVRewrite::fill_orderby_exprs(ObSelectStmt *origin_stmt, - const MvInfo &mv_info, - GenerateStmtHelper &helper, - bool &is_valid) -{ - int ret = OB_SUCCESS; - ObSEArray mv_select_exprs; - ObSEArray ori_orderby_exprs; - ObSEArray new_orderby_exprs; - bool is_sub_valid = false; - is_valid = false; - if (OB_ISNULL(origin_stmt) || OB_ISNULL(mv_info.view_stmt_) - || OB_ISNULL(helper.new_stmt_) || OB_ISNULL(helper.map_info_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(origin_stmt), K(mv_info), K(helper.new_stmt_), K(helper.map_info_)); - } else if (OB_FAIL(origin_stmt->get_order_exprs(ori_orderby_exprs))) { - LOG_WARN("failed to get order by exprs", K(ret)); - } else if (OB_FAIL(mv_info.view_stmt_->get_select_exprs(mv_select_exprs))) { - LOG_WARN("failed to get mv select exprs", K(ret)); - } else if (OB_FAIL(ObStmtComparer::compute_new_expr(ori_orderby_exprs, - origin_stmt, - mv_select_exprs, - mv_info.view_stmt_, - *helper.map_info_, - helper.compute_expr_copier_, - new_orderby_exprs, - is_sub_valid))) { - LOG_WARN("failed to compute select expr", K(ret)); - } else if (!is_sub_valid) { - LOG_TRACE("order by exprs can not be computed", K(new_orderby_exprs)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < new_orderby_exprs.count(); ++i) { - ObRawExpr *orderby_expr = NULL; - OrderItem order_item; - if (OB_FAIL(helper.col_copier_.copy_on_replace(new_orderby_exprs.at(i), orderby_expr))) { - LOG_WARN("failed to copy expr", K(ret), KPC(new_orderby_exprs.at(i))); - } else if (OB_FALSE_IT(order_item.expr_ = orderby_expr)) { - } else if (OB_FALSE_IT(order_item.order_type_ = origin_stmt->get_order_item(i).order_type_)) { - } else if (OB_FAIL(helper.new_stmt_->add_order_item(order_item))) { - LOG_WARN("fail to add order item", K(ret), K(order_item)); - } else if (OB_FAIL(ObTransformUtils::add_aggr_winfun_expr(helper.new_stmt_, orderby_expr))) { - LOG_WARN("failed to add aggr winfun expr", K(ret), KPC(orderby_expr)); - } + LOG_WARN("get unexpected null", K(ret), K(ctx_), K(node), K(rewrite_stmt)); + } else if (node == helper.query_tree_mv_part_) { + output_table = helper.mv_item_; + } else if (UNKNOWN_JOIN == node->join_info_.join_type_) { + // is leaf node + if (OB_ISNULL(output_table = rewrite_stmt->get_table_item_by_id(node->table_id_))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get table item", K(ret), K(node->table_id_)); + } else if (OB_FAIL(append(filters, node->join_info_.where_conditions_))) { + LOG_WARN("failed to append base table filters", K(ret)); } - is_valid = true; - } - return ret; -} - -int ObTransformMVRewrite::fill_limit(ObSelectStmt *origin_stmt, - const MvInfo &mv_info, - GenerateStmtHelper &helper, - bool &is_valid) -{ - // TODO we assumed that the materialized view has no LIMIT - int ret = OB_SUCCESS; - is_valid = false; - if (OB_ISNULL(origin_stmt) || OB_ISNULL(mv_info.view_stmt_) - || OB_ISNULL(helper.new_stmt_) || OB_ISNULL(helper.map_info_)) { + } else if (OB_FAIL(SMART_CALL(recursive_build_from_table(helper, + node->left_child_, + left_table, + left_filters)))) { + LOG_WARN("failed to recursive build left from table", K(ret)); + } else if (OB_ISNULL(left_table)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(origin_stmt), K(mv_info), K(helper.new_stmt_), K(helper.map_info_)); - } else if (!origin_stmt->has_limit()) { - is_valid = true; + LOG_WARN("left table is null", K(ret)); + } else if (OB_FAIL(SMART_CALL(recursive_build_from_table(helper, + node->right_child_, + right_table, + right_filters)))) { + LOG_WARN("failed to recursive build right from table", K(ret)); + } else if (OB_ISNULL(right_table)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("right table is null", K(ret)); + } else if (OB_ISNULL(new_joined_table = ObDMLStmt::create_joined_table(*ctx_->allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to create joined table", K(ret)); + } else if (OB_FAIL(rewrite_stmt->add_joined_table(new_joined_table))) { + LOG_WARN("failed to add joined table", K(ret)); + } else if (OB_FALSE_IT(new_joined_table->table_id_ = rewrite_stmt->get_query_ctx()->available_tb_id_--)) { + } else if (OB_FAIL(ObTransformUtils::add_joined_table_single_table_ids(*new_joined_table, *left_table))) { + LOG_WARN("failed to add joined table left single table ids", K(ret)); + } else if (OB_FAIL(ObTransformUtils::add_joined_table_single_table_ids(*new_joined_table, *right_table))) { + LOG_WARN("failed to add joined table right single table ids", K(ret)); + } else if (OB_FAIL(append(new_joined_table->join_conditions_, node->join_info_.on_conditions_))) { + LOG_WARN("failed to append join conditions", K(ret)); } else { - helper.new_stmt_->get_limit_expr() = origin_stmt->get_limit_expr(); - helper.new_stmt_->get_offset_expr() = origin_stmt->get_offset_expr(); - helper.new_stmt_->get_limit_percent_expr() = origin_stmt->get_limit_percent_expr(); - helper.new_stmt_->set_has_fetch(origin_stmt->has_fetch()); - helper.new_stmt_->set_fetch_with_ties(origin_stmt->is_fetch_with_ties()); - is_valid = true; + new_joined_table->joined_type_ = node->join_info_.join_type_; + new_joined_table->left_table_ = left_table; + new_joined_table->right_table_ = right_table; + output_table = new_joined_table; + switch (new_joined_table->joined_type_) { + case INNER_JOIN: + if (OB_FAIL(append(new_joined_table->join_conditions_, node->join_info_.where_conditions_))) { + LOG_WARN("failed to append join where conditions", K(ret)); + } else if (OB_FAIL(append(new_joined_table->join_conditions_, left_filters))) { + LOG_WARN("failed to append join left filters", K(ret)); + } else if (OB_FAIL(append(new_joined_table->join_conditions_, right_filters))) { + LOG_WARN("failed to append join right filters", K(ret)); + } + break; + case LEFT_OUTER_JOIN: + case RIGHT_OUTER_JOIN: { + ObIArray &upper_filters = (LEFT_OUTER_JOIN == new_joined_table->joined_type_ ? left_filters : right_filters); + ObIArray &join_on_conds = (LEFT_OUTER_JOIN == new_joined_table->joined_type_ ? right_filters : left_filters); + if (OB_FAIL(append(filters, node->join_info_.where_conditions_))) { + LOG_WARN("failed to append join where conditions", K(ret)); + } else if (OB_FAIL(append(filters, upper_filters))) { + LOG_WARN("failed to append join filters", K(ret)); + } else if (OB_FAIL(append(new_joined_table->join_conditions_, join_on_conds))) { + LOG_WARN("failed to append join conditions", K(ret)); + } + break; + } + case FULL_OUTER_JOIN: + if (OB_UNLIKELY(!left_filters.empty() || !right_filters.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("outer join filters is not empty", K(ret), K(left_filters), K(right_filters)); + } else if (OB_FAIL(append(filters, node->join_info_.where_conditions_))) { + LOG_WARN("failed to append join where conditions", K(ret)); + } + break; + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected join type", K(ret), K(new_joined_table->joined_type_)); + break; + } } return ret; } -int ObTransformMVRewrite::expand_rt_mv_table(GenerateStmtHelper &helper) +int ObTransformMVRewrite::append_on_replace(MvRewriteHelper &helper, + ObIArray &dst, + const ObIArray &src) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < src.count(); ++i) { + ObRawExpr *tmp_expr = src.at(i); + if (OB_FAIL(helper.query_expr_replacer_.do_visit(tmp_expr))) { + LOG_WARN("failed to replace query expr", K(ret)); + } else if (OB_FAIL(helper.mv_select_replacer_.do_visit(tmp_expr))) { + LOG_WARN("failed to replace mv select expr", K(ret)); + } else if (OB_FAIL(dst.push_back(tmp_expr))) { + LOG_WARN("failed to push back dst expr", K(ret)); + } + } + return ret; +} + +int ObTransformMVRewrite::adjust_aggr_winfun_expr(ObSelectStmt *rewrite_stmt) +{ + int ret = OB_SUCCESS; + ObSEArray exprs; + if (OB_ISNULL(rewrite_stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(rewrite_stmt)); + } else if (OB_FAIL(rewrite_stmt->get_select_exprs(exprs))) { + LOG_WARN("failed to get select exprs", K(ret)); + } else if (OB_FAIL(append(exprs, rewrite_stmt->get_having_exprs()))) { + LOG_WARN("failed to append having exprs", K(ret)); + } else if (OB_FAIL(rewrite_stmt->get_order_exprs(exprs))) { + LOG_WARN("failed to get order exprs", K(ret)); + } else { + rewrite_stmt->clear_aggr_item(); + rewrite_stmt->get_window_func_exprs().reuse(); + } + /* For MV: SELECT count(1) a0 FROM t1 GROUP BY c1, c2; + * Query: SELECT count(1) a1, count(1) a2 FROM t1 GROUP BY c1; + * Rewrite: SELECT count_sum(mv.a0) ra1, count_sum(mv.a0) ra2 FROM mv GROUP BY c1; + * Because of the parameterization, "a1" and "a2" will have different pointers. + * Which makes "ra1" and "ra2" have different pointers, although they are the same. + */ + for (int64_t i = 0; OB_SUCC(ret) && i < exprs.count(); ++i) { + if (OB_FAIL(ObTransformUtils::add_aggr_winfun_expr(rewrite_stmt, exprs.at(i), false))) { + LOG_WARN("failed to add aggr winfun expr", K(ret), KPC(exprs.at(i))); + } + } + return ret; +} + +/** + * @brief ObTransformMVRewrite::adjust_mv_item + * + * 1. If there is mv compensation predicates, create an inline view and add + * compensation predicates into view. + * 2. Expand real-time mv if needed. + */ +int ObTransformMVRewrite::adjust_mv_item(MvRewriteHelper &helper) { int ret = OB_SUCCESS; ObString src_qb_name; - int64_t query_rewrite_integrity = QueryRewriteIntegrityType::REWRITE_INTEGRITY_ENFORCED; - if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->session_info_) || OB_ISNULL(helper.mv_item_)) { + int64_t rewrite_integrity = QueryRewriteIntegrityType::REWRITE_INTEGRITY_ENFORCED; + // wrap mv table into inline view if needed + if (!helper.mv_compensation_preds_.empty()) { + TableItem *view_table = NULL; + ObSEArray view_conds; + if (OB_FAIL(append_on_replace(helper, view_conds, helper.mv_compensation_preds_))) { + LOG_WARN("failed to append on replace query expr", K(ret)); + } else if (OB_FAIL(ObTransformUtils::replace_with_empty_view(ctx_, + helper.query_stmt_, + view_table, + helper.mv_item_))) { + LOG_WARN("failed to replace with empty view", K(ret)); + } else if (OB_FAIL(ObTransformUtils::create_inline_view(ctx_, + helper.query_stmt_, + view_table, + helper.mv_item_, + &view_conds))) { + LOG_WARN("failed to create inline view", K(ret)); + } else if (OB_ISNULL(view_table)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("new view table is null", K(ret)); + } else { + helper.mv_upper_stmt_ = view_table->ref_query_; + } + } + // expand real-time mv if needed + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->session_info_) || OB_ISNULL(helper.mv_item_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret), K(ctx_), K(helper.mv_item_)); - } else if (OB_FAIL(ctx_->session_info_->get_query_rewrite_integrity(query_rewrite_integrity))) { + } else if (OB_FAIL(ctx_->session_info_->get_query_rewrite_integrity(rewrite_integrity))) { LOG_WARN("failed to get query rewrite integrity", K(ret)); - } else if (QueryRewriteIntegrityType::REWRITE_INTEGRITY_ENFORCED != query_rewrite_integrity) { - // do nothing + } else if (QueryRewriteIntegrityType::REWRITE_INTEGRITY_ENFORCED != rewrite_integrity) { + // do nothing, do not need to expand } else if (OB_FALSE_IT(src_qb_name = ctx_->src_qb_name_)) { } else if (OB_FALSE_IT(helper.mv_item_->need_expand_rt_mv_ = true)) { - } else if (OB_FAIL(ObTransformUtils::expand_mview_table(ctx_, helper.new_stmt_, helper.mv_item_))) { + } else if (OB_FAIL(ObTransformUtils::expand_mview_table(ctx_, helper.mv_upper_stmt_, helper.mv_item_))) { LOG_WARN("failed to expand mv table", K(ret)); } else { ctx_->src_qb_name_ = src_qb_name; @@ -1362,22 +3025,24 @@ int ObTransformMVRewrite::expand_rt_mv_table(GenerateStmtHelper &helper) return ret; } -int ObTransformMVRewrite::check_rewrite_expected(const ObSelectStmt *origin_stmt, - const ObSelectStmt *new_stmt, - const MvInfo &mv_info, +int ObTransformMVRewrite::check_rewrite_expected(MvRewriteHelper &helper, bool &is_expected) { int ret = OB_SUCCESS; bool hint_rewrite = false; bool hint_no_rewrite = false; - int64_t query_rewrite_enabled = QueryRewriteEnabledType::REWRITE_ENABLED_TRUE; bool is_match_index = false; + bool accepted = false; + bool need_check_cost = !helper.query_delta_table_.is_empty(); + int64_t query_rewrite_enabled = QueryRewriteEnabledType::REWRITE_ENABLED_TRUE; + ObDMLStmt *ori_stmt = &helper.ori_stmt_; + ObSelectStmt *rewrite_stmt = helper.query_stmt_; is_expected = false; if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->session_info_) - || OB_ISNULL(origin_stmt) || OB_ISNULL(new_stmt)) { + || OB_ISNULL(parent_stmts_) || OB_ISNULL(ori_stmt) || OB_ISNULL(rewrite_stmt)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(ctx_), K(origin_stmt), K(new_stmt)); - } else if (OB_FAIL(check_hint_valid(*origin_stmt, hint_rewrite, hint_no_rewrite))) { + LOG_WARN("get unexpected null", K(ret), K(ctx_), K(parent_stmts_), K(ori_stmt), K(rewrite_stmt)); + } else if (OB_FAIL(check_hint_valid(helper.ori_stmt_, hint_rewrite, hint_no_rewrite))) { LOG_WARN("failed to check mv rewrite hint", K(ret)); } else if (hint_rewrite) { is_expected = true; @@ -1387,32 +3052,38 @@ int ObTransformMVRewrite::check_rewrite_expected(const ObSelectStmt *origin_stmt } else if (QueryRewriteEnabledType::REWRITE_ENABLED_FORCE == query_rewrite_enabled) { is_expected = true; OPT_TRACE("system variable force mv rewrite, skip cost check"); - } else if (OB_FAIL(check_condition_match_index(new_stmt, is_match_index))) { + } else if (OB_FAIL(check_condition_match_index(helper, is_match_index))) { LOG_WARN("failed to check condition match index", K(ret)); } else if (!is_match_index) { is_expected = false; OPT_TRACE("condition does not match index, can not rewrite"); + } else if (OB_FAIL(accept_transform(*parent_stmts_, ori_stmt, rewrite_stmt, + !need_check_cost, false, accepted))) { + LOG_WARN("failed to accept transform", K(ret)); + } else if (!accepted) { + is_expected = false; + OPT_TRACE("does not pass the cost check, can not rewrite"); } else { is_expected = true; } - // TODO for the first version, we do not use the cost check return ret; } -int ObTransformMVRewrite::check_condition_match_index(const ObSelectStmt *new_stmt, +int ObTransformMVRewrite::check_condition_match_index(MvRewriteHelper &helper, bool &is_match_index) { int ret = OB_SUCCESS; is_match_index = false; - if (OB_ISNULL(ctx_) || OB_ISNULL(new_stmt)) { + if (OB_ISNULL(ctx_) || OB_ISNULL(helper.query_stmt_) || OB_ISNULL(helper.mv_upper_stmt_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(ctx_), K(new_stmt)); - } else if (0 == new_stmt->get_condition_size()) { + LOG_WARN("get unexpected null", K(ret), K(ctx_), K(helper.query_stmt_), K(helper.mv_upper_stmt_)); + } else if (helper.query_stmt_ == helper.mv_upper_stmt_) { + // there is no mv compensation predicates is_match_index = true; } - for (int64_t i = 0; OB_SUCC(ret) && !is_match_index && i < new_stmt->get_condition_size(); ++i) { + for (int64_t i = 0; OB_SUCC(ret) && !is_match_index && i < helper.mv_upper_stmt_->get_condition_size(); ++i) { ObSEArray column_exprs; - if (OB_FAIL(ObRawExprUtils::extract_column_exprs(new_stmt->get_condition_expr(i), column_exprs))) { + if (OB_FAIL(ObRawExprUtils::extract_column_exprs(helper.mv_upper_stmt_->get_condition_expr(i), column_exprs))) { LOG_WARN("failed to extract column exprs", K(ret)); } for (int64_t j = 0; OB_SUCC(ret) && !is_match_index && j < column_exprs.count(); ++j) { @@ -1422,8 +3093,8 @@ int ObTransformMVRewrite::check_condition_match_index(const ObSelectStmt *new_st ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpect null expr", K(ret)); } else if (OB_FALSE_IT(col_expr = static_cast(e))) { - } else if (OB_FAIL(ObTransformUtils::check_column_match_index(new_stmt, - new_stmt, + } else if (OB_FAIL(ObTransformUtils::check_column_match_index(helper.query_stmt_, + helper.mv_upper_stmt_, ctx_->sql_schema_guard_, col_expr, is_match_index))) { @@ -1434,22 +3105,228 @@ int ObTransformMVRewrite::check_condition_match_index(const ObSelectStmt *new_st return ret; } -int ObTransformMVRewrite::add_param_constraint(const ObStmtMapInfo &map_info) +int ObTransformMVRewrite::add_param_constraint(MvRewriteHelper &helper) { int ret = OB_SUCCESS; if (OB_ISNULL(ctx_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret), K(ctx_)); - } else if (OB_FAIL(append_array_no_dup(ctx_->expr_constraints_, map_info.expr_cons_map_))) { - LOG_WARN("failed to add param constraint", K(ret)); - } else if (OB_FAIL(append_array_no_dup(ctx_->plan_const_param_constraints_, map_info.const_param_map_))) { - LOG_WARN("failed to add param constraint", K(ret)); - } else if (OB_FAIL(append_array_no_dup(ctx_->equal_param_constraints_, map_info.equal_param_map_))) { - LOG_WARN("failed to add param constraint", K(ret)); + } else if (OB_FAIL(append_array_no_dup(ctx_->equal_param_constraints_, helper.equal_param_info_))) { + LOG_WARN("failed to add equal param constraint", K(ret)); + } else if (OB_FAIL(append_array_no_dup(ctx_->plan_const_param_constraints_, helper.const_param_info_))) { + LOG_WARN("failed to add const param constraint", K(ret)); + } else if (OB_FAIL(append_array_no_dup(ctx_->expr_constraints_, helper.expr_cons_info_))) { + LOG_WARN("failed to add expr param constraint", K(ret)); } return ret; } +int ObTransformMVRewrite::mv_privilege_check(MvInfo &mv_info, + bool &is_valid) +{ + int ret = OB_SUCCESS; + is_valid = false; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->allocator_) || OB_ISNULL(ctx_->exec_ctx_) + || OB_ISNULL(ctx_->exec_ctx_->get_sql_ctx()) || OB_ISNULL(mv_info.select_mv_stmt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_), K(mv_info.select_mv_stmt_)); + } else { + int tmp_ret; + mv_info.mv_need_privs_.need_privs_.set_allocator(ctx_->allocator_); + mv_info.mv_ora_need_privs_.need_privs_.set_allocator(ctx_->allocator_); + tmp_ret = ObPrivilegeCheck::check_privilege_new(*ctx_->exec_ctx_->get_sql_ctx(), + mv_info.select_mv_stmt_, + mv_info.mv_need_privs_, + mv_info.mv_ora_need_privs_); + if (OB_SUCCESS == tmp_ret) { + is_valid = true; + } else if (OB_UNLIKELY(OB_ERR_NO_TABLE_PRIVILEGE != tmp_ret + && OB_TABLE_NOT_EXIST != tmp_ret)) { + ret = tmp_ret; + LOG_WARN("failed to check mv privilege", K(ret)); + } + } + + return ret; +} + +int ObTransformMVRewrite::push_back_stmt_privilege(MvInfo &mv_info) { + int ret = OB_SUCCESS; + ObSEArray tmp_need_privs; + ObSEArray tmp_ora_need_privs; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->stmt_need_privs_) + || OB_ISNULL(ctx_->stmt_ora_need_privs_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_)); + } else if (OB_FAIL(append(tmp_need_privs, ctx_->stmt_need_privs_->need_privs_))) { + LOG_WARN("failed to append stmt need privs", K(ret)); + } else if (OB_FAIL(append(tmp_need_privs, mv_info.mv_need_privs_.need_privs_))) { + LOG_WARN("failed to append mv need privs", K(ret)); + } else if (OB_FAIL(append(tmp_ora_need_privs, ctx_->stmt_ora_need_privs_->need_privs_))) { + LOG_WARN("failed to append stmt ora need privs", K(ret)); + } else if (OB_FAIL(append(tmp_ora_need_privs, mv_info.mv_ora_need_privs_.need_privs_))) { + LOG_WARN("failed to append mv ora need privs", K(ret)); + } else if (OB_FALSE_IT(ctx_->stmt_need_privs_->need_privs_.reset())) { + } else if (OB_FAIL(ctx_->stmt_need_privs_->need_privs_.assign(tmp_need_privs))) { + LOG_WARN("failed to assign stmt need privs", K(ret)); + } else if (OB_FALSE_IT(ctx_->stmt_ora_need_privs_->need_privs_.reset())) { + } else if (OB_FAIL(ctx_->stmt_ora_need_privs_->need_privs_.assign(tmp_ora_need_privs))) { + LOG_WARN("failed to assign stmt ora need privs", K(ret)); + } + return ret; +} + +bool ObTransformMVRewrite::MvRewriteCheckCtx::compare_column(const ObColumnRefRawExpr &inner, + const ObColumnRefRawExpr &outer) +{ + int &ret = err_code_; + bool bret = false; + bool is_inner_from_query; + bool is_outer_from_query; + if (OB_ISNULL(helper_.query_stmt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(helper_.query_stmt_)); + } else { + const ColumnItem *col_item_inner = helper_.query_stmt_->get_column_item(inner.get_table_id(), + inner.get_column_id()); + const ColumnItem *col_item_outer = helper_.query_stmt_->get_column_item(outer.get_table_id(), + outer.get_column_id()); + is_inner_from_query = NULL != col_item_inner && col_item_inner->get_expr() == &inner; + is_outer_from_query = NULL != col_item_outer && col_item_outer->get_expr() == &outer; + } + if (OB_FAIL(ret)) { + // do nothing + } else if ((is_inner_from_query && is_outer_from_query) + || (!is_inner_from_query && !is_outer_from_query)) { + // inner and outer are from the same stmt + bret = &inner == &outer; + } else { + int64_t query_rel_id; + int64_t mv_tbl_idx; // = rel id - 1 + const TableItem *mv_table_item; + const ObColumnRefRawExpr *mv_col = is_inner_from_query ? &outer : &inner; + const ObColumnRefRawExpr *query_col = is_inner_from_query ? &inner : &outer; + if (mv_col->get_column_id() != query_col->get_column_id()) { + // do nothig, not match + } else if (OB_ISNULL(helper_.mv_info_.view_stmt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mv stmt is null", K(ret)); + } else if (OB_FALSE_IT(query_rel_id = helper_.query_stmt_->get_table_bit_index(query_col->get_table_id()))) { + } else if (OB_UNLIKELY(query_rel_id < 1 || query_rel_id > helper_.base_table_map_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected query rel id", K(ret), K(query_rel_id)); + } else if (OB_FALSE_IT(mv_tbl_idx = helper_.base_table_map_.at(query_rel_id - 1))) { + } else if (OB_UNLIKELY(mv_tbl_idx < 0 || mv_tbl_idx >= helper_.mv_info_.view_stmt_->get_table_size())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected mv table idx", K(ret), K(mv_tbl_idx), K(query_rel_id)); + } else if (OB_ISNULL(mv_table_item = + helper_.mv_info_.view_stmt_->get_table_item(mv_tbl_idx))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null table item", K(ret), K(mv_tbl_idx), K(query_rel_id)); + } else if (mv_table_item->table_id_ == mv_col->get_table_id()) { + // column matched + bret = true; + } else { + // do nothing, not match + } + } + return bret; +} + +bool ObTransformMVRewrite::MvRewriteCheckCtx::compare_const(const ObConstRawExpr &left, + const ObConstRawExpr &right) +{ + int &ret = err_code_; + bool bret = false; + if (OB_FAIL(ret) || left.get_result_type() != right.get_result_type()) { + // do nothing + } else if (&left == &right) { + bret = true; + } else if (left.is_param_expr() && right.is_param_expr()) { + if (!left.has_flag(IS_DYNAMIC_PARAM) || !right.has_flag(IS_DYNAMIC_PARAM)) { + // one of left or right is mv expr, would not be unknown_expr + bret = false; + } else { + const ObRawExpr *l_expr = static_cast(&left)->get_ref_expr(); + const ObRawExpr *r_expr = static_cast(&right)->get_ref_expr(); + bret = l_expr->same_as(*r_expr, this); + } + } else if (left.is_param_expr() || right.is_param_expr()) { + bret = ObExprEqualCheckContext::compare_const(left, right); + if (bret && (left.get_value().is_unknown() || right.get_value().is_unknown())) { + const ObConstRawExpr &unkonwn_expr = left.get_value().is_unknown() ? left : right; + ObPCConstParamInfo const_param_info; + if (OB_FAIL(const_param_info.const_idx_.push_back(unkonwn_expr.get_value().get_unknown()))) { + LOG_WARN("failed to push back element", K(ret)); + } else if (OB_FAIL(const_param_info.const_params_.push_back(unkonwn_expr.get_result_type().get_param()))) { + LOG_WARN("failed to psuh back param const value", K(ret)); + } else if (OB_FAIL(const_param_info_.push_back(const_param_info))) { + LOG_WARN("failed to push back const param info", K(ret)); + } + } + } else { + bret = left.get_value().is_equal(right.get_value(), CS_TYPE_BINARY); + } + return bret; +} + +bool ObTransformMVRewrite::MvRewriteCheckCtx::compare_query(const ObQueryRefRawExpr &first, + const ObQueryRefRawExpr &second) +{ + int &ret = err_code_; + bool bret = false; + ObStmtMapInfo stmt_map_info; + QueryRelation relation = QueryRelation::QUERY_UNCOMPARABLE; + const ObSelectStmt *first_sel = NULL; + const ObSelectStmt *second_sel = NULL; + if (&first == &second) { + bret = true; + } else if (first.is_set() != second.is_set() || first.is_multiset() != second.is_multiset() || + OB_ISNULL(first_sel = first.get_ref_stmt()) || + OB_ISNULL(second_sel = second.get_ref_stmt())) { + bret = false; + } else if (OB_FAIL(ObStmtComparer::check_stmt_containment(first_sel, + second_sel, + stmt_map_info, + relation, + true, /* is_strict_select_list */ + true, /* need_check_select_items */ + false /* is_in_same_stmt */ ))) { + LOG_WARN("failed to compute stmt relationship", K(ret)); + } else if (stmt_map_info.is_select_item_equal_ && QueryRelation::QUERY_EQUAL == relation) { + bret = true; + if (OB_FAIL(append(equal_param_info_, stmt_map_info.equal_param_map_))) { + LOG_WARN("failed to append equal param", K(ret)); + } else if (OB_FAIL(append(const_param_info_, stmt_map_info.const_param_map_))) { + LOG_WARN("failed to append const param", K(ret)); + } else if (OB_FAIL(append(expr_cons_info_, stmt_map_info.expr_cons_map_))) { + LOG_WARN("failed to append expr param", K(ret)); + } + } + return bret; +} + +int ObTransformMVRewrite::MvRewriteCheckCtx::append_constraint_info(const MvRewriteCheckCtx &other) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(append(equal_param_info_, other.equal_param_info_))) { + LOG_WARN("failed to append equal param", K(ret)); + } else if (OB_FAIL(append(const_param_info_, other.const_param_info_))) { + LOG_WARN("failed to append const param", K(ret)); + } else if (OB_FAIL(append(expr_cons_info_, other.expr_cons_info_))) { + LOG_WARN("failed to append expr param", K(ret)); + } + return ret; +} + +ObTransformMVRewrite::MvRewriteHelper::~MvRewriteHelper() { + int ret = OB_SUCCESS; + if (mv_es_map_.created() && OB_FAIL(mv_es_map_.destroy())) { + LOG_WARN("failed to destroy mv equal set map", K(ret)); + } else if (query_es_map_.created() && OB_FAIL(query_es_map_.destroy())) { + LOG_WARN("failed to destroy query equal set map", K(ret)); + } +} } //namespace sql } //namespace oceanbase \ No newline at end of file diff --git a/src/sql/rewrite/ob_transform_mv_rewrite.h b/src/sql/rewrite/ob_transform_mv_rewrite.h index f4c26743b..91d06b387 100644 --- a/src/sql/rewrite/ob_transform_mv_rewrite.h +++ b/src/sql/rewrite/ob_transform_mv_rewrite.h @@ -15,9 +15,11 @@ #include "sql/rewrite/ob_transform_rule.h" #include "sql/rewrite/ob_stmt_comparer.h" +#include "sql/optimizer/ob_conflict_detector.h" #include "sql/resolver/dml/ob_select_stmt.h" #include "objit/common/ob_item_type.h" + namespace oceanbase { @@ -31,15 +33,24 @@ class ObIArray; namespace sql { -enum QueryRewriteEnabledType { - REWRITE_ENABLED_FALSE = 0, - REWRITE_ENABLED_TRUE , - REWRITE_ENABLED_FORCE -}; - -enum QueryRewriteIntegrityType { - REWRITE_INTEGRITY_ENFORCED = 0, - REWRITE_INTEGRITY_STALE_TOLERATED +struct PtrKey +{ + PtrKey () : ptr_(NULL) {} + PtrKey (const void *ptr) : ptr_(ptr) {} + const void *ptr_; + uint64_t hash() const + { + return murmurhash(&ptr_, sizeof(ptr_), 0); + } + int hash(uint64_t &res) const + { + res = hash(); + return OB_SUCCESS; + } + bool operator==(const PtrKey &other) const + { + return (other.ptr_ == this->ptr_); + } }; /** @@ -60,9 +71,7 @@ class ObTransformMVRewrite : public ObTransformRule public: ObTransformMVRewrite(ObTransformerCtx *ctx) : ObTransformRule(ctx, TransMethod::PRE_ORDER, T_MV_REWRITE), - is_mv_info_generated_(false), - mv_stmt_gen_count_(0) {} - virtual ~ObTransformMVRewrite(); + parent_stmts_(NULL) {} virtual int transform_one_stmt(common::ObIArray &parent_stmts, ObDMLStmt *&stmt, bool &trans_happened) override; @@ -71,61 +80,104 @@ public: protected: private: - struct MvInfo { - MvInfo() : mv_id_(common::OB_INVALID_ID), - data_table_id_(common::OB_INVALID_ID), - mv_schema_(NULL), - data_table_schema_(NULL), - db_schema_(NULL), - view_stmt_(NULL), - select_mv_stmt_(NULL) {} - - MvInfo(uint64_t mv_id, - uint64_t data_table_id, - const ObTableSchema *mv_schema, - const ObTableSchema *data_table_schema, - const ObDatabaseSchema *db_schema, - ObSelectStmt *view_stmt) - : mv_id_(mv_id), - data_table_id_(data_table_id), - mv_schema_(mv_schema), - data_table_schema_(data_table_schema), - db_schema_(db_schema), - view_stmt_(view_stmt), - select_mv_stmt_(NULL) {} - + struct JoinTreeNode { + JoinTreeNode() : table_set_(), + table_id_(common::OB_INVALID_ID), + ori_table_(NULL), + left_child_(NULL), + right_child_(NULL), + join_info_() {} TO_STRING_KV( - K_(mv_id), - K_(data_table_id), - K_(mv_schema), - K_(data_table_schema), - K_(db_schema), - K_(view_stmt), - K_(select_mv_stmt) + K_(table_set), + K_(table_id), + KPC_(ori_table), + KPC_(left_child), + KPC_(right_child), + K_(join_info) ); - - uint64_t mv_id_; - uint64_t data_table_id_; - const ObTableSchema *mv_schema_; - const ObTableSchema *data_table_schema_; - const ObDatabaseSchema *db_schema_; - ObSelectStmt *view_stmt_; - ObSelectStmt *select_mv_stmt_; + ObRelIds table_set_; // all tables included in this tree + uint64_t table_id_; // only for leaf node, base table id + const TableItem* ori_table_; // the original table item, only for mv join tree + JoinTreeNode *left_child_; // left child tree + JoinTreeNode *right_child_; // right child tree + JoinInfo join_info_; // if tree node is leaf, join_info_.join_type_ is UNKNOWN_JOIN, and filters are stored in join_info_.where_conditions_ }; - struct GenerateStmtHelper { - GenerateStmtHelper(ObRawExprFactory &expr_factory) : new_stmt_(NULL), - mv_item_(NULL), - map_info_(NULL), - relation_(QUERY_UNCOMPARABLE), - compute_expr_copier_(expr_factory), - col_copier_(expr_factory) {} - ObSelectStmt *new_stmt_; - TableItem *mv_item_; - ObStmtMapInfo *map_info_; - QueryRelation relation_; - ObRawExprCopier compute_expr_copier_; - ObRawExprCopier col_copier_; + struct MvRewriteHelper { + MvRewriteHelper (ObSelectStmt &ori_stmt, + MvInfo &mv_info, + ObSelectStmt *query_stmt, + ObIArray &base_table_map) + : ori_stmt_(ori_stmt), + mv_info_(mv_info), + query_stmt_(query_stmt), + base_table_map_(base_table_map) {} + ~MvRewriteHelper(); + + ObSelectStmt &ori_stmt_; + MvInfo &mv_info_; + // the following is used to do rewrite checks + ObSelectStmt *query_stmt_; // the deep copy of ori_stmt_, and the rewrite will be done in this stmt + ObIArray &base_table_map_; // ori_stmt table -> mv table e.g. base_table_map_[0] = 1 means ori_stmt table 0 is map to mv table 1 + ObRelIds mv_delta_table_; // rel_ids of tables that only appear in mv + ObRelIds query_delta_table_; // rel_ids of tables that only appear in query + JoinTreeNode *mv_tree_; // mv join tree + JoinTreeNode *query_tree_; // origin query join tree + JoinTreeNode *query_tree_mv_part_; // mv part of origin query join tree + ObSEArray mv_conds_; // where/on conditions pulled up from mv join tree + ObSEArray query_conds_; // where/on conditions pulled up from query join tree (only mv part) + ObSEArray query_other_conds_; // where conditions which are not participating in join tree build/compare, will be added into rewrite stmt directly + EqualSets mv_equal_sets_; // mv equal sets + EqualSets query_equal_sets_; // origin query equal sets + hash::ObHashMap mv_es_map_; // mv equal sets map, expr -> equal set idx + hash::ObHashMap query_es_map_; // origin query equal sets map, expr -> equal set idx + ObSEArray, 16> equal_sets_map_; // map origin query equal set to mv equal sets + // the following is used to generate rewrite stmt + ObSEArray mv_compensation_preds_; // mv compensation predicates + ObSEArray query_compensation_preds_; // query compensation predicates, used to generate union rewrite + bool need_group_by_roll_up_; // need group by roll up in rewrite stmt + ObStmtExprReplacer query_expr_replacer_; // replace query expr into mv select expr + ObStmtExprReplacer mv_select_replacer_; // replace mv select expr into mv table item column expr + ObSelectStmt *mv_upper_stmt_; // the upper stmt which contain mv_item + TableItem *mv_item_; // mv table item in rewrite stmt + // the following is plan cache constraint info + ObSEArray equal_param_info_; + ObSEArray const_param_info_; + ObSEArray expr_cons_info_; + }; + + struct MvRewriteCheckCtx : ObExprEqualCheckContext { + MvRewriteCheckCtx(MvRewriteHelper &helper) + : ObExprEqualCheckContext(), + helper_(helper) + { + init_override_params(); + } + + MvRewriteCheckCtx(MvRewriteHelper &helper, bool can_use_equal_sets) + : ObExprEqualCheckContext(), + helper_(helper) + { + init_override_params(); + } + + inline void init_override_params() + { + override_column_compare_ = true; + override_const_compare_ = true; + override_query_compare_ = true; + } + + bool compare_column(const ObColumnRefRawExpr &inner, const ObColumnRefRawExpr &outer) override; + bool compare_const(const ObConstRawExpr &left, const ObConstRawExpr &right) override; + bool compare_query(const ObQueryRefRawExpr &first, const ObQueryRefRawExpr &second) override; + + int append_constraint_info(const MvRewriteCheckCtx &other); + + MvRewriteHelper &helper_; + ObSEArray equal_param_info_; + ObSEArray const_param_info_; + ObSEArray expr_cons_info_; }; private: @@ -136,91 +188,200 @@ private: int check_hint_valid(const ObDMLStmt &stmt, bool &force_rewrite, bool &force_no_rewrite); - int check_table_has_mv(const ObDMLStmt &stmt, - bool &has_mv); int check_basic_validity(const ObDMLStmt &stmt, bool &is_valid); - int prepare_mv_info(const ObDMLStmt *root_stmt, - const ObDMLStmt *stmt); - int get_mv_list(const ObDMLStmt *root_stmt, - ObIArray &mv_list); - int get_base_table_id_string(const ObDMLStmt *stmt, - ObSqlString &table_ids); - int get_all_base_table_id(const ObDMLStmt *stmt, - ObIArray &table_ids); - int generate_mv_info(const ObDMLStmt *stmt, - ObIArray &mv_list); - int generate_mv_stmt(MvInfo &mv_info); - int quick_rewrite_check(const ObTableSchema *mv_schema, - bool allow_stale, - bool &is_valid); - int check_mv_stmt_basic_validity(const MvInfo &mv_info, - bool &is_valid); int try_transform_with_one_mv(ObSelectStmt *origin_stmt, MvInfo &mv_info, ObSelectStmt *&new_stmt, bool &transform_happened); - int do_transform(ObDMLStmt *&stmt, - bool &trans_happened); - int do_transform_use_one_mv(ObSelectStmt *origin_stmt, - const MvInfo &mv_info, - ObStmtMapInfo &map_info, - ObSelectStmt *&new_stmt, - bool &is_valid_transform); - int create_mv_table_item(const MvInfo &mv_info, - GenerateStmtHelper &helper); - int create_mv_column_item(const MvInfo &mv_info, - GenerateStmtHelper &helper); - int fill_from_item(ObSelectStmt *origin_stmt, - const MvInfo &mv_info, - GenerateStmtHelper &helper, - bool &is_valid); - int fill_select_item(ObSelectStmt *origin_stmt, - const MvInfo &mv_info, - GenerateStmtHelper &helper, - bool &is_valid); - int fill_condition_exprs(ObSelectStmt *origin_stmt, - const MvInfo &mv_info, - GenerateStmtHelper &helper, - bool &is_valid); - int fill_having_exprs(ObSelectStmt *origin_stmt, - const MvInfo &mv_info, - GenerateStmtHelper &helper, + int gen_base_table_map(const ObIArray &from_tables, + const ObIArray &to_tables, + int64_t max_map_num, + ObIArray> &table_maps); + int inner_gen_base_table_map(int64_t from_rel_id, + const ObIArray &from_tables, + hash::ObHashMap &from_table_num, + const ObIArray> &to_table_ids, + const hash::ObHashMap &to_table_map, + ObSqlBitSet<> &used_to_table, + int64_t max_map_num, + ObIArray ¤t_map, + ObIArray> &table_maps); + int try_transform_contain_mode(ObSelectStmt *origin_stmt, + MvInfo &mv_info, + ObSelectStmt *&new_stmt, + bool &transform_happened); + int check_mv_rewrite_validity(MvRewriteHelper &helper, + bool &is_valid); + int check_delta_table(MvRewriteHelper &helper, bool &is_valid); - int fill_groupby_exprs(ObSelectStmt *origin_stmt, - const MvInfo &mv_info, - GenerateStmtHelper &helper, + int check_join_compatibility(MvRewriteHelper &helper, + bool &is_valid); + int pre_process_quals(const ObIArray &input_conds, + ObIArray &normal_conds, + ObIArray &other_conds); + int build_mv_join_tree(MvRewriteHelper &helper, + ObIArray> &baserel_filters, + ObIArray &conflict_detectors); + int inner_build_mv_join_tree(MvRewriteHelper &helper, + const TableItem *table, + ObIArray> &baserel_filters, + ObIArray &conflict_detectors, + ObIArray &used_detectors, + JoinTreeNode *&node); + int build_query_join_tree(MvRewriteHelper &helper, + ObIArray> &baserel_filters, + ObIArray &conflict_detectors, + bool &is_valid); + int inner_build_query_tree_mv_part(MvRewriteHelper &helper, + JoinTreeNode *mv_node, + ObIArray> &baserel_filters, + ObIArray &conflict_detectors, + ObIArray &used_detectors, + JoinTreeNode *&node, + bool &is_valid); + int inner_build_query_tree_delta_part(MvRewriteHelper &helper, + const TableItem *table, + ObIArray> &baserel_filters, + ObIArray &conflict_detectors, + ObIArray &used_detectors, + JoinTreeNode *&node, + bool &is_delta_table, + bool &is_valid); + int build_join_tree_node(JoinTreeNode *left_node, + JoinTreeNode *right_node, + ObIArray &conflict_detectors, + ObIArray &used_detectors, + JoinTreeNode *&new_node, + bool &is_valid); + int build_leaf_tree_node(int64_t rel_id, + uint64_t table_id, + ObIArray> &baserel_filters, + JoinTreeNode *&leaf_node); + int compare_join_tree(MvRewriteHelper &helper, + JoinTreeNode *mv_node, + JoinTreeNode *query_node, + bool can_compensate, + bool &is_valid); + int compare_join_conds(MvRewriteHelper &helper, + const ObIArray &mv_conds, + const ObIArray &query_conds, bool &is_valid); - int fill_distinct(ObSelectStmt *origin_stmt, - const MvInfo &mv_info, - GenerateStmtHelper &helper, - bool &is_valid); - int fill_orderby_exprs(ObSelectStmt *origin_stmt, - const MvInfo &mv_info, - GenerateStmtHelper &helper, + int compare_join_type(MvRewriteHelper &helper, + JoinTreeNode *mv_node, + JoinTreeNode *query_node, + bool &is_valid); + int add_not_null_compensate(MvRewriteHelper &helper, + JoinTreeNode *mv_upper_node, + bool comp_left_child, + bool &is_valid); + int find_mv_not_null_expr(MvRewriteHelper &helper, + JoinTreeNode *mv_upper_node, + bool comp_left_child, + ObRawExpr *¬_null_expr); + int check_predicate_compatibility(MvRewriteHelper &helper, + bool &is_valid); + int check_equal_predicate(MvRewriteHelper &helper, + const ObIArray &mv_conds, + const ObIArray &query_conds, + bool &is_valid); + int build_equal_sets(const ObIArray &input_conds, + const ObIArray &all_columns, + EqualSets &equal_sets, + hash::ObHashMap &equal_set_map); + int generate_equal_compensation_preds(MvRewriteHelper &helper, + bool &is_valid); + int map_to_mv_column(MvRewriteHelper &helper, + const ObColumnRefRawExpr *query_expr, + ObColumnRefRawExpr *&mv_expr); + int check_mv_equal_set_used(MvRewriteHelper &helper, + ObIArray &mv_map_query_ids, + int64_t current_query_es_id); + int check_other_predicate(MvRewriteHelper &helper, + const ObIArray &mv_conds, + const ObIArray &query_conds); + int check_compensation_preds_validity(MvRewriteHelper &helper, + bool &is_valid); + int is_same_condition(MvRewriteHelper &helper, + const ObRawExpr *left, + const ObRawExpr *right, + bool &is_same); + int append_constraint_info(MvRewriteHelper &helper, + const MvRewriteCheckCtx &context); + int compare_expr(MvRewriteCheckCtx &context, + const ObRawExpr *left, + const ObRawExpr *right, + bool &is_same); + int check_group_by_col(MvRewriteHelper &helper, bool &is_valid); - int fill_limit(ObSelectStmt *origin_stmt, - const MvInfo &mv_info, - GenerateStmtHelper &helper, - bool &is_valid); - int expand_rt_mv_table(GenerateStmtHelper &helper); - int check_rewrite_expected(const ObSelectStmt *origin_stmt, - const ObSelectStmt *new_stmt, - const MvInfo &mv_info, + int check_opt_feat_ctrl(MvRewriteHelper &helper, + bool &is_valid); + int compute_stmt_expr_map(MvRewriteHelper &helper, + bool &is_valid); + int compute_join_expr_map(MvRewriteHelper &helper, + bool &is_valid); + int compute_select_expr_map(MvRewriteHelper &helper, + bool &is_valid); + int compute_group_by_expr_map(MvRewriteHelper &helper, + bool &is_valid); + int compute_order_by_expr_map(MvRewriteHelper &helper, + bool &is_valid); + int inner_compute_exprs_map(MvRewriteHelper &helper, + ObIArray &exprs, + bool &is_valid); + int inner_compute_expr_map(MvRewriteHelper &helper, + ObRawExpr *query_expr, + MvRewriteCheckCtx &context, + bool &is_valid); + int compute_agg_expr_map(MvRewriteHelper &helper, + ObAggFunRawExpr *query_expr, + MvRewriteCheckCtx &context, + bool &is_valid); + int find_mv_select_expr(MvRewriteHelper &helper, + const ObRawExpr *query_expr, + MvRewriteCheckCtx &context, + ObRawExpr *&select_expr); + int trans_split_sum_expr(MvRewriteHelper &helper, + ObAggFunRawExpr *query_expr, + MvRewriteCheckCtx &context, + ObRawExpr *&output_expr); + int check_agg_roll_up_expr(ObRawExpr *agg_expr, + bool &is_valid); + int wrap_agg_roll_up_expr(ObRawExpr *ori_expr, + ObRawExpr *&output_expr); + int generate_rewrite_stmt_contain_mode(MvRewriteHelper &helper); + int clear_mv_common_tables(MvRewriteHelper &helper); + int create_mv_table_item(MvRewriteHelper &helper); + int create_mv_column_item(MvRewriteHelper &helper); + int adjust_rewrite_stmt(MvRewriteHelper &helper); + int recursive_build_from_table(MvRewriteHelper &helper, + JoinTreeNode *node, + TableItem *&output_table, + ObIArray &filters); + int append_on_replace(MvRewriteHelper &helper, + ObIArray &dst, + const ObIArray &src); + int adjust_aggr_winfun_expr(ObSelectStmt *rewrite_stmt); + int adjust_mv_item(MvRewriteHelper &helper); + int check_rewrite_expected(MvRewriteHelper &helper, bool &is_expected); - int check_condition_match_index(const ObSelectStmt *new_stmt, + int check_condition_match_index(MvRewriteHelper &helper, bool &is_match_index); - int add_param_constraint(const ObStmtMapInfo &map_info); + int add_param_constraint(MvRewriteHelper &helper); + int mv_privilege_check(MvInfo &mv_info, + bool &is_valid); + int push_back_stmt_privilege(MvInfo &mv_info); - int check_mv_has_multi_part(const MvInfo &mv_info, - bool &has_multi_part); + static int find_query_rel_id(MvRewriteHelper &helper, int64_t mv_relid); + static int classify_predicates(const ObIArray &input_conds, + ObIArray &equal_conds, + ObIArray &other_conds); + static bool is_equal_cond(ObRawExpr *expr); + static bool is_range_cond(ObRawExpr *expr); private: - ObSEArray mv_infos_; - bool is_mv_info_generated_; - int64_t mv_stmt_gen_count_; + ObIArray *parent_stmts_; ObQueryCtx mv_temp_query_ctx_; // used for generating mv stmt - private: DISALLOW_COPY_AND_ASSIGN(ObTransformMVRewrite); diff --git a/src/sql/rewrite/ob_transform_mv_rewrite_prepare.cpp b/src/sql/rewrite/ob_transform_mv_rewrite_prepare.cpp new file mode 100644 index 000000000..98277a037 --- /dev/null +++ b/src/sql/rewrite/ob_transform_mv_rewrite_prepare.cpp @@ -0,0 +1,489 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_REWRITE +#include "sql/rewrite/ob_transform_mv_rewrite_prepare.h" + +namespace oceanbase +{ +namespace sql +{ + +int ObTransformMVRewritePrepare::prepare_mv_rewrite_info(const ObDMLStmt *stmt) +{ + int ret = OB_SUCCESS; + bool need_prepare; + if (OB_FAIL(need_do_prepare(stmt, need_prepare))) { + LOG_WARN("failed to check need do prepare", K(ret)); + } else if (!need_prepare) { + // do nothing + } else if (OB_FAIL(prepare_mv_info(stmt))) { + LOG_WARN("failed to prepare mv info", K(ret)); + } + return ret; +} + +int ObTransformMVRewritePrepare::need_do_prepare(const ObDMLStmt *stmt, + bool &need_prepare) +{ + int ret = OB_SUCCESS; + bool has_mv = false; + uint64_t data_version; + need_prepare = false; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->session_info_) + || OB_ISNULL(stmt) || OB_ISNULL(stmt->get_query_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexprcted null", K(ret), K(ctx_), K(stmt)); + } else if (ctx_->session_info_->get_ddl_info().is_refreshing_mview()) { + need_prepare = false; + OPT_TRACE("not a user SQL, skip mv rewrite"); + } else if (stmt->get_query_ctx()->optimizer_features_enable_version_ < COMPAT_VERSION_4_3_1) { + need_prepare = false; + OPT_TRACE("optimizer features enable version is lower than 4.3.1, skip mv rewrite"); + } else if (OB_FAIL(GET_MIN_DATA_VERSION(ctx_->session_info_->get_effective_tenant_id(), data_version))) { + LOG_WARN("failed to get data version", K(ret), K(ctx_->session_info_->get_effective_tenant_id())); + } else if (OB_UNLIKELY(data_version < DATA_VERSION_4_3_1_0)) { + // data version lower than 4.3.1 does not have the inner table used to get mv list + need_prepare = false; + OPT_TRACE("min data version is lower than 4.3.1, skip mv rewrite"); + } else if (OB_FAIL(check_table_has_mv(stmt, has_mv))) { + LOG_WARN("failed to check table has mv", K(ret)); + } else if (!has_mv) { + need_prepare = false; + OPT_TRACE("table does not have mv, no need to rewrite"); + } else if (OB_FAIL(check_sys_var_and_hint(stmt, need_prepare))) { + LOG_WARN("failed to check sys var and hint", K(ret)); + } else if (!need_prepare) { + OPT_TRACE("system variable reject mv rewrite, no need to rewrite"); + } else { + need_prepare = true; + } + return ret; +} + +int ObTransformMVRewritePrepare::check_table_has_mv(const ObDMLStmt *stmt, + bool &has_mv) +{ + int ret = OB_SUCCESS; + ObSEArray child_stmts; + has_mv = false; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->schema_checker_) + || OB_ISNULL(ctx_->session_info_) || OB_ISNULL(stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_), K(stmt)); + } else if (OB_FAIL(stmt->get_child_stmts(child_stmts))) { + LOG_WARN("failed to get child stmts", K(ret), KPC(stmt)); + } + for (int64_t i = 0; OB_SUCC(ret) && !has_mv && i < stmt->get_table_size(); ++i) { + const TableItem *table = NULL; + const ObTableSchema *table_schema = NULL; + if (OB_ISNULL(table = stmt->get_table_item(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table item is null", K(ret), K(i), K(stmt->get_table_size())); + } else if (!table->is_basic_table()) { + // do nothing + } else if (OB_FAIL(ctx_->schema_checker_->get_table_schema(ctx_->session_info_->get_effective_tenant_id(), + table->ref_id_, + table_schema))) { + LOG_WARN("failed to get table schema", K(ret), K(i), K(table->ref_id_)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table schema is null", K(ret), K(table->ref_id_)); + } else if (ObTableReferencedByMVFlag::IS_REFERENCED_BY_MV == + ObTableMode::get_table_referenced_by_mv_flag(table_schema->get_table_mode())) { + has_mv = true; + } + } + for (int64_t i = 0; OB_SUCC(ret) && !has_mv && i < child_stmts.count(); ++i) { + if (OB_FAIL(SMART_CALL(check_table_has_mv(child_stmts.at(i), has_mv)))) { + LOG_WARN("failed to check child stmt has mv", K(ret), K(i)); + } + } + return ret; +} + +int ObTransformMVRewritePrepare::check_sys_var_and_hint(const ObDMLStmt *stmt, + bool &need_prepare) +{ + int ret = OB_SUCCESS; + int64_t query_rewrite_enabled = QueryRewriteEnabledType::REWRITE_ENABLED_FALSE; + need_prepare = false; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->session_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_)); + } else if (OB_FAIL(ctx_->session_info_->get_query_rewrite_enabled(query_rewrite_enabled))) { + LOG_WARN("failed to get query rewrite enabled", K(ret)); + } else if (QueryRewriteEnabledType::REWRITE_ENABLED_FALSE != query_rewrite_enabled) { + need_prepare = true; + } else if (OB_FAIL(recursive_check_hint(stmt, need_prepare))) { + LOG_WARN("failed to check hint enabled", K(ret)); + } + return ret; +} + +int ObTransformMVRewritePrepare::recursive_check_hint(const ObDMLStmt *stmt, + bool &need_prepare) +{ + int ret = OB_SUCCESS; + need_prepare = false; + if (OB_ISNULL(stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("stmt is null", K(ret), K(stmt)); + } else { + const ObMVRewriteHint *myhint = static_cast(stmt->get_stmt_hint().get_normal_hint(T_MV_REWRITE)); + need_prepare = NULL != myhint && myhint->is_enable_hint(); // has mv_rewrite hint + ObSEArray child_stmts; + if (!need_prepare && OB_FAIL(stmt->get_child_stmts(child_stmts))) { + LOG_WARN("failed to get child stmts", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && !need_prepare && i < child_stmts.count(); ++i) { + if (OB_FAIL(SMART_CALL(recursive_check_hint(child_stmts.at(i), need_prepare)))) { + LOG_WARN("failed to recursive check hint", K(ret)); + } + } + } + return ret; +} + +int ObTransformMVRewritePrepare::prepare_mv_info(const ObDMLStmt *root_stmt) +{ + int ret = OB_SUCCESS; + ObSEArray mv_list; + ObSEArray intersect_tbl_num; + if (OB_ISNULL(ctx_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_)); + } else if (OB_FAIL(get_mv_list(root_stmt, mv_list, intersect_tbl_num))) { + LOG_WARN("failed to get mv list", K(ret)); + } else if (OB_FAIL(generate_mv_info(mv_list, intersect_tbl_num))) { + LOG_WARN("failed to generate mv info", K(ret)); + } else if (OB_FAIL(sort_mv_infos())) { + LOG_WARN("failed to sort mv infos", K(ret)); + } else { + OPT_TRACE("prepare", ctx_->mv_infos_.count(), "materialized view(s) to perform rewrite"); + } + return ret; +} + +int ObTransformMVRewritePrepare::get_mv_list(const ObDMLStmt *root_stmt, + ObIArray &mv_list, + ObIArray &intersect_tbl_num) +{ + int ret = OB_SUCCESS; + ObMySQLProxy *sql_proxy = NULL; + ObSqlString sql; + sqlclient::ObMySQLResult *mysql_result = NULL; + ObSqlString table_ids; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->exec_ctx_) || OB_ISNULL(ctx_->session_info_) + || OB_ISNULL(sql_proxy = ctx_->exec_ctx_->get_sql_proxy())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_), K(sql_proxy)); + } else if (OB_FAIL(get_base_table_id_string(root_stmt, table_ids))) { + LOG_WARN("failed to get base table id string", K(ret)); + } else if (OB_FAIL(sql.assign_fmt("SELECT MVIEW_ID ID, COUNT(P_OBJ) CNT FROM `%s`.`%s` WHERE TENANT_ID = 0 AND P_OBJ IN (%.*s) GROUP BY MVIEW_ID" + " HAVING NOT EXISTS(SELECT 1 FROM `%s`.`%s` WHERE TENANT_ID = 0 AND MVIEW_ID = ID AND P_OBJ NOT IN (%.*s))", // remove mv which contains table not in query + OB_SYS_DATABASE_NAME, OB_ALL_MVIEW_DEP_TNAME, + static_cast(table_ids.length()), table_ids.ptr(), + OB_SYS_DATABASE_NAME, OB_ALL_MVIEW_DEP_TNAME, + static_cast(table_ids.length()), table_ids.ptr()))) { + LOG_WARN("failed to assign sql", K(ret)); + } else { + SMART_VAR(ObMySQLProxy::MySQLResult, res) { + if (OB_FAIL(sql_proxy->read(res, ctx_->session_info_->get_effective_tenant_id(), sql.ptr()))) { + LOG_WARN("execute sql failed", K(ret), K(sql)); + } else if (OB_ISNULL(mysql_result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get result", K(ret)); + } else { + while (OB_SUCC(ret) && OB_SUCC(mysql_result->next())) { + int64_t mv_id = OB_INVALID_ID; + int64_t tbl_num = 0; + if (OB_FAIL(mysql_result->get_int(0L, mv_id))) { + LOG_WARN("failed to get mv id", K(ret)); + } else if (OB_FAIL(mysql_result->get_int(1L, tbl_num))) { + LOG_WARN("failed to get intersect table num", K(ret)); + } else if (OB_FAIL(mv_list.push_back(mv_id))) { + LOG_WARN("failed to push back mv id", K(ret), K(mv_id)); + } else if (OB_FAIL(intersect_tbl_num.push_back(tbl_num))) { + LOG_WARN("failed to push back intersect table num", K(ret), K(tbl_num)); + } + } + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + OPT_TRACE("find", mv_list.count(), "materialized view(s)"); + } else if (OB_FAIL(ret)) { + LOG_WARN("failed to get inner sql result", K(ret)); + } + } + } + } + return ret; +} + +int ObTransformMVRewritePrepare::get_base_table_id_string(const ObDMLStmt *stmt, + ObSqlString &table_ids) +{ + int ret = OB_SUCCESS; + ObSEArray table_id_list; + hash::ObHashSet visited_id; + if (OB_ISNULL(stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(stmt)); + } else if (OB_FAIL(get_all_base_table_id(stmt, table_id_list))) { + LOG_WARN("failed to get table ids", K(ret)); + } else if (OB_FAIL(visited_id.create(table_id_list.count(), "MvRewrite", "HashNode"))) { + LOG_WARN("failed to init visited id hash set", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < table_id_list.count(); ++i) { + int tmp_ret = visited_id.exist_refactored(table_id_list.at(i)); + if (OB_HASH_EXIST == tmp_ret) { + // do nothing + } else if (OB_UNLIKELY(OB_HASH_NOT_EXIST != tmp_ret)) { + ret = tmp_ret; + LOG_WARN("failed to check visited id", K(ret), K(table_id_list.at(i))); + } else if (OB_FAIL(visited_id.set_refactored(table_id_list.at(i)))) { + LOG_WARN("failed to push back table ref id", K(ret)); + } else if (i > 0 && OB_FAIL(table_ids.append(","))) { + LOG_WARN("failed to append comma", K(ret)); + } else if (OB_FAIL(table_ids.append(to_cstring(table_id_list.at(i))))) { + LOG_WARN("failed to append table id", K(ret)); + } + } + if (OB_SUCC(ret) && OB_FAIL(visited_id.destroy())) { + LOG_WARN("failed to destroy visited id hash set", K(ret)); + } + return ret; +} + +int ObTransformMVRewritePrepare::get_all_base_table_id(const ObDMLStmt *stmt, + ObIArray &table_ids) +{ + int ret = OB_SUCCESS; + ObSEArray child_stmts; + if (OB_ISNULL(stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(stmt)); + } else if (OB_FAIL(stmt->get_child_stmts(child_stmts))) { + LOG_WARN("failed to get child stmts", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < stmt->get_table_size(); ++i) { + const TableItem *table = NULL; + if (OB_ISNULL(table = stmt->get_table_item(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(i)); + } else if (!table->is_basic_table()) { + // do nothing + } else if (OB_FAIL(table_ids.push_back(table->ref_id_))) { + LOG_WARN("failed to push back table id", K(ret)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < child_stmts.count(); ++i) { + if (OB_FAIL(SMART_CALL(get_all_base_table_id(child_stmts.at(i), table_ids)))) { + LOG_WARN("failed to get all base table id", K(ret)); + } + } + return ret; +} + +int ObTransformMVRewritePrepare::generate_mv_info(ObIArray &mv_list, + ObIArray &intersect_tbl_num) +{ + int ret = OB_SUCCESS; + int64_t query_rewrite_integrity = QueryRewriteIntegrityType::REWRITE_INTEGRITY_ENFORCED; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->schema_checker_) || OB_ISNULL(ctx_->session_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_)); + } else if (OB_FAIL(ctx_->session_info_->get_query_rewrite_integrity(query_rewrite_integrity))) { + LOG_WARN("failed to get query rewrite integrity", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < mv_list.count(); ++i) { + const ObTableSchema *mv_schema = NULL; + const ObDatabaseSchema *db_schema = NULL; + bool is_valid = true; + if (OB_FAIL(ctx_->schema_checker_->get_table_schema(ctx_->session_info_->get_effective_tenant_id(), + mv_list.at(i), + mv_schema))) { + LOG_WARN("failed to get mv schema", K(ret), K(mv_list.at(i))); + } else if (OB_ISNULL(mv_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mv schema is null", K(ret), K(mv_list.at(i))); + } else if (OB_FAIL(ctx_->schema_checker_->get_database_schema(ctx_->session_info_->get_effective_tenant_id(), + mv_schema->get_database_id(), + db_schema))) { + LOG_WARN("failed to get data base schema", K(ret), K(mv_schema->get_database_id())); + } else if (OB_ISNULL(db_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("database schema is NULL", K(ret), K(mv_schema->get_database_id())); + } else if (OB_FAIL(quick_rewrite_check(*ctx_->session_info_, + *mv_schema, + query_rewrite_integrity == QueryRewriteIntegrityType::REWRITE_INTEGRITY_STALE_TOLERATED, + is_valid))) { + LOG_WARN("failed to do quick check", K(ret)); + } else if (!is_valid) { + // do nothing + } else { + uint64_t data_table_id = mv_schema->get_data_table_id(); + const ObTableSchema *data_table_schema = NULL; + if (OB_FAIL(ctx_->schema_checker_->get_table_schema(ctx_->session_info_->get_effective_tenant_id(), + data_table_id, + data_table_schema))) { + LOG_WARN("failed to get data table schema", K(ret), K(data_table_id)); + } else if (OB_ISNULL(data_table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("data table schema is null", K(ret), K(mv_list.at(i))); + } else if (OB_FAIL(ctx_->mv_infos_.push_back(MvInfo(mv_list.at(i), + data_table_id, + mv_schema, + data_table_schema, + db_schema, + NULL, + NULL, + intersect_tbl_num.at(i))))) { + LOG_WARN("failed to push back mv info", K(ret)); + } + } + } + return ret; +} + +int ObTransformMVRewritePrepare::sort_mv_infos() +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(ctx_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx_)); + } else { + lib::ob_sort(ctx_->mv_infos_.begin(), ctx_->mv_infos_.end()); + } + return ret; +} + +int ObTransformMVRewritePrepare::quick_rewrite_check(const ObSQLSessionInfo &session_info, + const ObTableSchema &mv_schema, + bool allow_stale, + bool &is_valid) +{ + int ret = OB_SUCCESS; + is_valid = false; + if (OB_FAIL(ObMVProvider::check_mview_dep_session_vars(mv_schema, session_info, false, is_valid))) { + LOG_WARN("failed to check mview dep session vars", K(ret)); + } else if (!is_valid) { + /* do nothing */ + } else if (!mv_schema.mv_enable_query_rewrite()) { + is_valid = false; + } else if (!allow_stale && !mv_schema.mv_on_query_computation()) { + is_valid = false; + } else { + is_valid = true; + } + return ret; +} + +int ObTransformMVRewritePrepare::generate_mv_stmt(MvInfo &mv_info, + ObTransformerCtx *ctx, + ObQueryCtx *temp_query_ctx) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(ctx) || OB_ISNULL(ctx->allocator_) + || OB_ISNULL(ctx->session_info_) || OB_ISNULL(mv_info.mv_schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mv_info is null", K(ret), K(ctx), KPC(mv_info.mv_schema_)); + } else { + ObString view_definition; + ObSqlString mv_sql; // select * from mv; + if (OB_FAIL(ObSQLUtils::generate_view_definition_for_resolve(*ctx->allocator_, + ctx->session_info_->get_local_collation_connection(), + mv_info.mv_schema_->get_view_schema(), + view_definition))) { + LOG_WARN("fail to generate view definition for resolve", K(ret)); + } else if (OB_FAIL(resolve_temp_stmt(view_definition, ctx, temp_query_ctx, mv_info.view_stmt_))) { + LOG_WARN("failed to resolve mv define stmt", K(ret), K(view_definition)); + } else if (OB_FAIL(mv_sql.assign_fmt(lib::is_oracle_mode() ? + "SELECT * FROM \"%.*s\".\"%.*s\"" : + "SELECT * FROM `%.*s`.`%.*s`", + mv_info.db_schema_->get_database_name_str().length(), mv_info.db_schema_->get_database_name_str().ptr(), + mv_info.mv_schema_->get_table_name_str().length(), mv_info.mv_schema_->get_table_name_str().ptr()))) { + LOG_WARN("failed to assign sql", K(ret)); + } else if (OB_FAIL(resolve_temp_stmt(ObString::make_string(mv_sql.ptr()), ctx, temp_query_ctx, mv_info.select_mv_stmt_))) { + LOG_WARN("failed to resolve mv define stmt", K(ret), K(mv_sql)); + } else { + LOG_DEBUG("generate mv stmt", KPC(mv_info.view_stmt_), KPC(mv_info.select_mv_stmt_)); + } + OPT_TRACE("generate stmt for", mv_info.mv_schema_->get_table_name(), ":", mv_info.view_stmt_); + } + return ret; +} + +int ObTransformMVRewritePrepare::resolve_temp_stmt(const ObString &sql_string, + ObTransformerCtx *ctx, + ObQueryCtx *query_ctx, + ObSelectStmt *&output_stmt) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(ctx) || OB_ISNULL(query_ctx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx), K(query_ctx)); + } else { + SMART_VARS_2((ObTransformerCtx, trans_ctx), (ObResolverParams, resolver_ctx)) { + ObParser parser(*ctx->allocator_, ctx->session_info_->get_sql_mode(), ctx->session_info_->get_charsets4parser()); + ParseResult parse_result; + ParseNode *node = NULL; + ObDMLStmt *dml_stmt = NULL; + uint64_t dummy_value = 0; + ObIAllocator &alloc = *ctx->allocator_; + resolver_ctx.allocator_ = ctx->allocator_; + resolver_ctx.schema_checker_ = ctx->schema_checker_; + resolver_ctx.session_info_ = ctx->session_info_; + resolver_ctx.expr_factory_ = ctx->expr_factory_; + resolver_ctx.stmt_factory_ = ctx->stmt_factory_; + resolver_ctx.sql_proxy_ = GCTX.sql_proxy_; + resolver_ctx.query_ctx_ = query_ctx; + resolver_ctx.is_for_rt_mv_ = true; + trans_ctx = *ctx; + trans_ctx.reset(); + ObSelectResolver select_resolver(resolver_ctx); + ObTransformPreProcess transform_pre_process(&trans_ctx); + transform_pre_process.set_transformer_type(PRE_PROCESS); + STOP_OPT_TRACE; + if (OB_FAIL(parser.parse(sql_string, parse_result))) { + LOG_WARN("parse view definition failed", K(sql_string), K(ret)); + } else if (OB_ISNULL(node = parse_result.result_tree_->children_[0]) || OB_UNLIKELY(T_SELECT != node->type_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid mv select node", K(ret), K(node)); + } else if (OB_FALSE_IT(resolver_ctx.query_ctx_->question_marks_count_ + = static_cast(parse_result.question_mark_ctx_.count_))) { + } else if (OB_FAIL(select_resolver.resolve(*node))) { + LOG_WARN("resolve view definition failed", K(ret)); + } else if (OB_ISNULL(dml_stmt = static_cast(select_resolver.get_basic_stmt()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid mv stmt", K(ret), K(dml_stmt)); + } else if (OB_FAIL(resolver_ctx.query_ctx_->query_hint_.init_query_hint(resolver_ctx.allocator_, + resolver_ctx.session_info_, + dml_stmt))) { + LOG_WARN("failed to init query hint.", K(ret)); + } else if (OB_FAIL(resolver_ctx.query_ctx_->query_hint_.check_and_set_params_from_hint(resolver_ctx, + *dml_stmt))) { + LOG_WARN("failed to check and set params from hint", K(ret)); + } else if (OB_FAIL(transform_pre_process.transform(dml_stmt, dummy_value))) { + LOG_WARN("failed to do transform pre process", K(ret), KPC(dml_stmt)); + } else if (OB_ISNULL(output_stmt = static_cast(dml_stmt))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid mv stmt", K(ret), K(dml_stmt)); + } + RESUME_OPT_TRACE; + } + } + return ret; +} + +} //namespace sql +} //namespace oceanbase \ No newline at end of file diff --git a/src/sql/rewrite/ob_transform_mv_rewrite_prepare.h b/src/sql/rewrite/ob_transform_mv_rewrite_prepare.h new file mode 100644 index 000000000..e5fae1443 --- /dev/null +++ b/src/sql/rewrite/ob_transform_mv_rewrite_prepare.h @@ -0,0 +1,83 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef _OB_TRANSFORM_MV_REWRITE_PREPARE_H +#define _OB_TRANSFORM_MV_REWRITE_PREPARE_H + +#include "sql/rewrite/ob_transform_rule.h" +#include "sql/resolver/dml/ob_select_resolver.h" + +namespace oceanbase +{ +namespace sql +{ + +enum QueryRewriteEnabledType { + REWRITE_ENABLED_FALSE = 0, + REWRITE_ENABLED_TRUE , + REWRITE_ENABLED_FORCE +}; + +enum QueryRewriteIntegrityType { + REWRITE_INTEGRITY_ENFORCED = 0, + REWRITE_INTEGRITY_STALE_TOLERATED +}; + +class ObTransformMVRewritePrepare +{ + public: + explicit ObTransformMVRewritePrepare(ObTransformerCtx *ctx) + : ctx_(ctx) {} + ~ObTransformMVRewritePrepare() {} + + int prepare_mv_rewrite_info(const ObDMLStmt *stmt); + static int generate_mv_stmt(MvInfo &mv_info, + ObTransformerCtx *ctx, + ObQueryCtx *temp_query_ctx); + static int resolve_temp_stmt(const ObString &sql_string, + ObTransformerCtx *ctx, + ObQueryCtx *query_ctx, + ObSelectStmt *&output_stmt); + + private: + int need_do_prepare(const ObDMLStmt *stmt, + bool &need_prepare); + int check_table_has_mv(const ObDMLStmt *stmt, + bool &has_mv); + int check_sys_var_and_hint(const ObDMLStmt *stmt, + bool &need_prepare); + int recursive_check_hint(const ObDMLStmt *stmt, + bool &need_prepare); + int prepare_mv_info(const ObDMLStmt *root_stmt); + int get_mv_list(const ObDMLStmt *root_stmt, + ObIArray &mv_list, + ObIArray &intersect_tbl_num); + int get_base_table_id_string(const ObDMLStmt *stmt, + ObSqlString &table_ids); + int get_all_base_table_id(const ObDMLStmt *stmt, + ObIArray &table_ids); + int generate_mv_info(ObIArray &mv_list, + ObIArray &intersect_tbl_num); + int sort_mv_infos(); + int quick_rewrite_check(const ObSQLSessionInfo &session_info, + const ObTableSchema &mv_schema, + bool allow_stale, + bool &is_valid); + + private: + ObTransformerCtx *ctx_; +}; + + +} //namespace sql +} //namespace oceanbase +#endif \ No newline at end of file diff --git a/src/sql/rewrite/ob_transform_or_expansion.cpp b/src/sql/rewrite/ob_transform_or_expansion.cpp index c4dc8e852..eb60fa072 100644 --- a/src/sql/rewrite/ob_transform_or_expansion.cpp +++ b/src/sql/rewrite/ob_transform_or_expansion.cpp @@ -998,7 +998,7 @@ int ObTransformOrExpansion::add_filter_to_stmt(ObSelectStmt *stmt, int ret = OB_SUCCESS; ObConstRawExpr *const_one = NULL; ObRawExpr *equal_expr = NULL; - ObOpRawExpr *is_not_null = NULL; + ObRawExpr *is_not_null = NULL; ObSEArray conds; ObRawExpr *or_expr = NULL; if (OB_ISNULL(stmt) || OB_ISNULL(ctx_) || OB_ISNULL(ctx_->expr_factory_) @@ -1018,7 +1018,7 @@ int ObTransformOrExpansion::add_filter_to_stmt(ObSelectStmt *stmt, LOG_WARN("failed to build common binary op expr", K(ret)); } else if (OB_FAIL(conds.push_back(equal_expr))) { LOG_WARN("failed to push back expr", K(ret)); - } else if (OB_FAIL(ObTransformUtils::add_is_not_null(ctx_, stmt, flag_exprs.at(0), + } else if (OB_FAIL(ObTransformUtils::add_is_not_null(ctx_, flag_exprs.at(0), is_not_null))) { LOG_WARN("failed to add is not null", K(ret)); } else if (OB_FAIL(conds.push_back(is_not_null))) { diff --git a/src/sql/rewrite/ob_transform_rule.cpp b/src/sql/rewrite/ob_transform_rule.cpp index 5df4bdd3b..25fcf53e0 100644 --- a/src/sql/rewrite/ob_transform_rule.cpp +++ b/src/sql/rewrite/ob_transform_rule.cpp @@ -49,7 +49,9 @@ bool ObTransformerCtx::is_valid() NULL != stmt_factory_ && NULL != sql_schema_guard_ && NULL != sql_schema_guard_->get_schema_guard() && - NULL != self_addr_; + NULL != self_addr_ && + NULL != stmt_need_privs_ && + NULL != stmt_ora_need_privs_; } void ObTransformerCtx::reset() @@ -71,6 +73,8 @@ void ObTransformerCtx::reset() is_spm_outline_ = false; push_down_filters_.reset(); iteration_level_ = 0; + mv_infos_.reset(); + mv_stmt_gen_count_ = 0; } int ObTransformerCtx::add_src_hash_val(const ObString &src_str) @@ -931,8 +935,10 @@ int ObTryTransHelper::fill_helper(const ObQueryCtx *query_ctx) LOG_WARN("failed to get qb name info", K(ret)); } else { available_tb_id_ = query_ctx->available_tb_id_; + stmt_count_ = query_ctx->stmt_count_; subquery_count_ = query_ctx->subquery_count_; temp_table_count_ = query_ctx->temp_table_count_; + anonymous_view_count_ = query_ctx->anonymous_view_count_; } return ret; } @@ -955,8 +961,10 @@ int ObTryTransHelper::recover(ObQueryCtx *query_ctx) } else { unique_key_provider_ = NULL; query_ctx->available_tb_id_ = available_tb_id_; + query_ctx->stmt_count_ = stmt_count_; query_ctx->subquery_count_ = subquery_count_; query_ctx->temp_table_count_ = temp_table_count_; + query_ctx->anonymous_view_count_ = anonymous_view_count_; } return ret; } diff --git a/src/sql/rewrite/ob_transform_rule.h b/src/sql/rewrite/ob_transform_rule.h index 63b131f9f..a1c8015d6 100644 --- a/src/sql/rewrite/ob_transform_rule.h +++ b/src/sql/rewrite/ob_transform_rule.h @@ -39,6 +39,63 @@ class ObCodeGeneratorImpl; class ObLogPlan; class StmtUniqueKeyProvider; +struct MvInfo { + MvInfo() : mv_id_(common::OB_INVALID_ID), + data_table_id_(common::OB_INVALID_ID), + mv_schema_(NULL), + data_table_schema_(NULL), + db_schema_(NULL), + view_stmt_(NULL), + select_mv_stmt_(NULL), + mv_intersect_tbl_num_(0) {} + + MvInfo(uint64_t mv_id, + uint64_t data_table_id, + const ObTableSchema *mv_schema, + const ObTableSchema *data_table_schema, + const ObDatabaseSchema *db_schema, + ObSelectStmt *view_stmt, + ObSelectStmt *select_mv_stmt, + uint64_t mv_intersect_tbl_num) + : mv_id_(mv_id), + data_table_id_(data_table_id), + mv_schema_(mv_schema), + data_table_schema_(data_table_schema), + db_schema_(db_schema), + view_stmt_(view_stmt), + select_mv_stmt_(select_mv_stmt), + mv_intersect_tbl_num_(mv_intersect_tbl_num), + has_select_priv_(false) {} + + TO_STRING_KV( + K_(mv_id), + K_(data_table_id), + K_(mv_schema), + K_(data_table_schema), + K_(db_schema), + K_(view_stmt), + K_(select_mv_stmt), + K_(mv_intersect_tbl_num), + K_(has_select_priv) + ); + + bool operator<(const MvInfo &other) { + return mv_intersect_tbl_num_ > other.mv_intersect_tbl_num_; + } + + uint64_t mv_id_; + uint64_t data_table_id_; + const ObTableSchema *mv_schema_; // schema of mv table + const ObTableSchema *data_table_schema_; // schema of mv container table + const ObDatabaseSchema *db_schema_; + ObSelectStmt *view_stmt_; // stmt of mv's definition + ObSelectStmt *select_mv_stmt_; // stmt of "SELECT * FROM mv;" + uint64_t mv_intersect_tbl_num_; // number of tables that appear in both mv and origin query, used for sort mv info + ObStmtNeedPrivs mv_need_privs_; // privilege needed of mv + ObStmtOraNeedPrivs mv_ora_need_privs_; // ora privilege needed of mv + bool has_select_priv_; +}; + struct ObTransformerCtx { ObTransformerCtx() @@ -74,7 +131,10 @@ struct ObTransformerCtx is_spm_outline_(false), push_down_filters_(), in_accept_transform_(false), - iteration_level_(0) + iteration_level_(0), + mv_stmt_gen_count_(0), + stmt_need_privs_(NULL), + stmt_ora_need_privs_(NULL) { } virtual ~ObTransformerCtx() {} @@ -143,6 +203,10 @@ struct ObTransformerCtx ObSEArray push_down_filters_; bool in_accept_transform_; uint64_t iteration_level_; + ObSEArray mv_infos_; // used to perform mv rewrite + int64_t mv_stmt_gen_count_; + ObStmtNeedPrivs *stmt_need_privs_; + ObStmtOraNeedPrivs *stmt_ora_need_privs_; }; enum TransMethod @@ -226,8 +290,10 @@ struct ObTryTransHelper int is_filled() const { return !qb_name_counts_.empty(); } uint64_t available_tb_id_; + int64_t stmt_count_; int64_t subquery_count_; int64_t temp_table_count_; + int64_t anonymous_view_count_; int64_t qb_name_sel_start_id_; int64_t qb_name_set_start_id_; int64_t qb_name_other_start_id_; diff --git a/src/sql/rewrite/ob_transform_simplify_subquery.cpp b/src/sql/rewrite/ob_transform_simplify_subquery.cpp index 49c1cb5d9..896670d5c 100644 --- a/src/sql/rewrite/ob_transform_simplify_subquery.cpp +++ b/src/sql/rewrite/ob_transform_simplify_subquery.cpp @@ -977,8 +977,8 @@ int ObTransformSimplifySubquery::do_transform_any_all_as_min_max(ObSelectStmt *s if (OB_FAIL(stmt->add_agg_item(*aggr_expr))) { LOG_WARN("fail to add agg_item", K(ret)); } else if (is_with_all) { - ObOpRawExpr *is_not_expr = NULL; - if (OB_FAIL(ObTransformUtils::add_is_not_null(ctx_, stmt, aggr_expr, is_not_expr))) { + ObRawExpr *is_not_expr = NULL; + if (OB_FAIL(ObTransformUtils::add_is_not_null(ctx_, aggr_expr, is_not_expr))) { LOG_WARN("failed to add is not null", K(ret)); } else if (OB_FAIL(stmt->add_having_expr(is_not_expr))) { LOG_WARN("failed to add having expr", K(ret)); diff --git a/src/sql/rewrite/ob_transform_utils.cpp b/src/sql/rewrite/ob_transform_utils.cpp index 6e0bda43f..c57891ad6 100644 --- a/src/sql/rewrite/ob_transform_utils.cpp +++ b/src/sql/rewrite/ob_transform_utils.cpp @@ -1624,17 +1624,16 @@ int ObTransformUtils::is_aggr_query(const ObSelectStmt *stmt, } int ObTransformUtils::add_is_not_null(ObTransformerCtx *ctx, - const ObDMLStmt *stmt, ObRawExpr *child_expr, - ObOpRawExpr *&is_not_expr) + ObRawExpr *&is_not_expr) { int ret = OB_SUCCESS; ObRawExprFactory *expr_factory = NULL; is_not_expr = NULL; ObRawExpr* tmp_is_not_expr = NULL; - if (OB_ISNULL(stmt) || OB_ISNULL(ctx)) { + if (OB_ISNULL(ctx)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("parameters have null", K(ret), K(stmt), K(ctx)); + LOG_WARN("parameters have null", K(ret), K(ctx)); } else if (OB_ISNULL(ctx->session_info_) || OB_ISNULL(expr_factory = ctx->expr_factory_)) { ret = OB_ERR_UNEXPECTED; @@ -1649,7 +1648,7 @@ int ObTransformUtils::add_is_not_null(ObTransformerCtx *ctx, } else if (OB_FAIL(tmp_is_not_expr->pull_relation_id())) { LOG_WARN("pull expr relation ids failed", K(ret)); } else { - is_not_expr = static_cast(tmp_is_not_expr); + is_not_expr = tmp_is_not_expr; } return ret; } @@ -2378,7 +2377,7 @@ int ObTransformUtils::is_column_expr_not_null(ObNotNullContext &ctx, if (OB_ISNULL(stmt = ctx.stmt_) || OB_ISNULL(expr) || OB_ISNULL(table = stmt->get_table_item_by_id(expr->get_table_id()))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("table item is null", K(ret), K(expr), K(stmt)); + LOG_WARN("table item is null", K(ret), KPC(expr), KPC(stmt), KPC(table)); } else if (is_virtual_table(table->ref_id_)) { // 'NOT NULL' of the virtual table is unreliable } else if (ObOptimizerUtil::find_item(ctx.right_table_ids_, table->table_id_)) { @@ -7835,6 +7834,33 @@ int ObTransformUtils::pushdown_qualify_filters(ObSelectStmt *stmt) return ret; } +int ObTransformUtils::create_aggr_expr(ObTransformerCtx *ctx, + ObItemType type, + ObAggFunRawExpr *&agg_expr, + ObRawExpr *child_expr) +{ + int ret = OB_SUCCESS; + ObRawExprFactory *expr_factory = NULL; + if (OB_ISNULL(ctx) || OB_ISNULL(expr_factory = ctx->expr_factory_) || + OB_ISNULL(ctx->session_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid argument", K(ret), K(ctx), K(expr_factory), K(agg_expr)); + } else if (OB_FAIL(expr_factory->create_raw_expr(type, + agg_expr))) { + LOG_WARN("create raw expr failed", K(ret)); + } else if (OB_ISNULL(agg_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid argument", K(ret), K(agg_expr)); + } else if (OB_FAIL(agg_expr->add_real_param_expr(child_expr))) { + LOG_WARN("fail to set partition exprs", K(ret)); + } else if (OB_FAIL(agg_expr->formalize(ctx->session_info_))) { + LOG_WARN("failed to formalize windown function", K(ret)); + } else if (OB_FAIL(agg_expr->pull_relation_id())) { + LOG_WARN("failed to pull relation id and levels", K(ret)); + } + return ret; +} + // stmt should call formalize_stmt after generate select_list, // and ref_query inside ths table has no need to call formalize again int ObTransformUtils::generate_select_list(ObTransformerCtx *ctx, @@ -8517,13 +8543,13 @@ int ObTransformUtils::build_case_when_expr(ObDMLStmt &stmt, { int ret = OB_SUCCESS; ObCaseOpRawExpr *case_expr = NULL; - ObOpRawExpr *is_not_expr = NULL; + ObRawExpr *is_not_expr = NULL; if (OB_ISNULL(ctx) || OB_ISNULL(ctx->expr_factory_) || OB_ISNULL(ctx->session_info_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("param expr is null", K(ret), K(ctx)); } else if (OB_FAIL(ctx->expr_factory_->create_raw_expr(T_OP_CASE, case_expr))) { LOG_WARN("failed to create case expr", K(ret)); - } else if (OB_FAIL(ObTransformUtils::add_is_not_null(ctx, &stmt, expr, is_not_expr))) { + } else if (OB_FAIL(ObTransformUtils::add_is_not_null(ctx, expr, is_not_expr))) { LOG_WARN("failed to build is not null expr", K(ret)); } else if (OB_FAIL(case_expr->add_when_param_expr(is_not_expr))) { LOG_WARN("failed to add when param expr", K(ret)); @@ -14497,13 +14523,7 @@ int ObTransformUtils::expand_mview_table(ObTransformerCtx *ctx, ObDMLStmt *upper ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpect null", K(ret), K(ctx), KPC(rt_mv_table)); } else { - const uint64_t OB_MAX_SUBQUERY_NAME_LENGTH = 64; - const char *EXPAND_VIEW_PREFIX = "RT_"; ObString expand_view; - ObString expand_view_name; - int64_t pos = 0; - char buf[OB_MAX_SUBQUERY_NAME_LENGTH]; - int64_t buf_len = OB_MAX_SUBQUERY_NAME_LENGTH; ObSelectStmt *view_stmt = NULL; OPT_TRACE("expand real time materialized view: ", rt_mv_table->get_object_name()); OPT_TRACE_BEGIN_SECTION; @@ -14519,15 +14539,9 @@ int ObTransformUtils::expand_mview_table(ObTransformerCtx *ctx, ObDMLStmt *upper LOG_WARN("failed to push back", K(ret)); } else if (OB_FAIL(set_expand_mview_flag(view_stmt))) { LOG_WARN("fail to set expand mview flag", K(ret)); - } else if (OB_FAIL(BUF_PRINTF("%s", EXPAND_VIEW_PREFIX))) { - LOG_WARN("append expand view prefix to buf error", K(ret)); - } else if (OB_FAIL(BUF_PRINTF("%.*s", rt_mv_table->get_object_name().length(), rt_mv_table->get_object_name().ptr()))) { - LOG_WARN("append mv table name to buf error", K(ret)); - } else if (OB_FALSE_IT(expand_view_name = common::ObString::make_string(buf))) { - } else if (OB_FAIL(ob_write_string(*ctx->allocator_, - expand_view_name, - rt_mv_table->table_name_))) { - LOG_WARN("failed to write string", K(ret)); + } else if (OB_FAIL(upper_stmt->generate_view_name(*ctx->allocator_, + rt_mv_table->table_name_))) { + LOG_WARN("failed to generate view name", K(ret)); } else if (OB_FAIL(adjust_col_and_sel_for_expand_mview(ctx, upper_stmt->get_column_items(), view_stmt->get_select_items(), @@ -15249,7 +15263,8 @@ int ObTransformUtils::check_fulltext_index_match_column(const ColumnReferenceSet } int ObTransformUtils::add_aggr_winfun_expr(ObSelectStmt *stmt, - ObRawExpr *expr) + ObRawExpr *expr, + bool need_strict_check /* = true */) { int ret = OB_SUCCESS; if (OB_ISNULL(expr) || OB_ISNULL(stmt)) { @@ -15257,17 +15272,22 @@ int ObTransformUtils::add_aggr_winfun_expr(ObSelectStmt *stmt, LOG_WARN("get unexpected null", K(ret), K(expr), K(stmt)); } else if (expr->is_aggr_expr()) { ObAggFunRawExpr *agg_expr = static_cast(expr); - if (OB_FAIL(ObExpandAggregateUtils::add_aggr_item(stmt->get_aggr_items(), agg_expr))) { + if (OB_FAIL(ObExpandAggregateUtils::add_aggr_item(stmt->get_aggr_items(), + agg_expr, + need_strict_check))) { LOG_WARN("failed to add agg item", K(ret), KPC(agg_expr)); } - } else if (expr->is_win_func_expr()) { - ObWinFunRawExpr *win_expr = static_cast(expr); - if (OB_FAIL(ObExpandAggregateUtils::add_win_expr(stmt->get_window_func_exprs(), win_expr))) { - LOG_WARN("failed to add win func expr", K(ret), KPC(win_expr)); - } } else { + if (expr->is_win_func_expr()) { + ObWinFunRawExpr *win_expr = static_cast(expr); + if (OB_FAIL(ObExpandAggregateUtils::add_win_expr(stmt->get_window_func_exprs(), + win_expr, + need_strict_check))) { + LOG_WARN("failed to add win func expr", K(ret), KPC(win_expr)); + } + } for (int64_t i = 0; OB_SUCC(ret) && i < expr->get_param_count(); ++i) { - if (OB_FAIL(SMART_CALL(add_aggr_winfun_expr(stmt, expr->get_param_expr(i))))) { + if (OB_FAIL(SMART_CALL(add_aggr_winfun_expr(stmt, expr->get_param_expr(i), need_strict_check)))) { LOG_WARN("failed to add aggr winfun expr", K(ret), KPC(expr->get_param_expr(i))); } } diff --git a/src/sql/rewrite/ob_transform_utils.h b/src/sql/rewrite/ob_transform_utils.h index dcc5dc7a0..add80efa5 100644 --- a/src/sql/rewrite/ob_transform_utils.h +++ b/src/sql/rewrite/ob_transform_utils.h @@ -417,12 +417,12 @@ public: /** * @brief add_is_not_null * 增加对 child_expr 结果的 not null 判断 - * @param stmt * @param child_expr * @return */ - static int add_is_not_null(ObTransformerCtx *ctx, const ObDMLStmt *stmt, - ObRawExpr *child_expr, ObOpRawExpr *&is_not_expr); + static int add_is_not_null(ObTransformerCtx *ctx, + ObRawExpr *child_expr, + ObRawExpr *&is_not_expr); static int is_column_nullable(const ObDMLStmt *stmt, ObSchemaChecker *schema_checker, @@ -1778,6 +1778,11 @@ public: ObIArray *having_exprs = NULL, ObIArray *order_items = NULL); + static int create_aggr_expr(ObTransformerCtx *ctx, + ObItemType type, + ObAggFunRawExpr *&agg_expr, + ObRawExpr *child_expr); + /* Push all content of the parent stmt into an inline view, and keep the ptr of the parent stmt */ static int pack_stmt(ObTransformerCtx *ctx, @@ -1918,7 +1923,8 @@ public: ObSQLSessionInfo *session_info, bool &has_fts_or_multivalue_index); static int add_aggr_winfun_expr(ObSelectStmt *stmt, - ObRawExpr *expr); + ObRawExpr *expr, + bool need_strict_check = true); static int expand_mview_table(ObTransformerCtx *ctx, ObDMLStmt *upper_stmt, TableItem *rt_mv_table); static int adjust_col_and_sel_for_expand_mview(ObTransformerCtx *ctx, ObIArray &uppper_col_items, diff --git a/src/sql/rewrite/ob_transform_win_magic.cpp b/src/sql/rewrite/ob_transform_win_magic.cpp index 97752ad1e..e6c0bd6bf 100644 --- a/src/sql/rewrite/ob_transform_win_magic.cpp +++ b/src/sql/rewrite/ob_transform_win_magic.cpp @@ -938,8 +938,8 @@ int ObTransformWinMagic::remove_dup_condition(ObDMLStmt *stmt) { LOG_WARN("push back failed", K(ret)); } } else { - ObOpRawExpr *is_not_null = NULL; - if (OB_FAIL(ObTransformUtils::add_is_not_null(ctx_, stmt, expr->get_param_expr(0), is_not_null))) { + ObRawExpr *is_not_null = NULL; + if (OB_FAIL(ObTransformUtils::add_is_not_null(ctx_, expr->get_param_expr(0), is_not_null))) { LOG_WARN("failed to add is not null expr", K(ret)); } else if (OB_FAIL(new_conditions.push_back(is_not_null))) { LOG_WARN("failed to append is not null exprs to where conditions", K(ret)); @@ -1104,32 +1104,6 @@ int ObTransformWinMagic::check_select_expr_validity(ObSelectStmt &subquery, bool return ret; } -int ObTransformWinMagic::create_aggr_expr(ObItemType type, - ObAggFunRawExpr *&agg_expr, - ObRawExpr *child_expr) -{ - int ret = OB_SUCCESS; - ObRawExprFactory *expr_factory = NULL; - if (OB_ISNULL(ctx_) || OB_ISNULL(expr_factory = ctx_->expr_factory_) || - OB_ISNULL(ctx_->session_info_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid argument", K(ret), K(ctx_), K(expr_factory), K(agg_expr)); - } else if (OB_FAIL(expr_factory->create_raw_expr(type, - agg_expr))) { - LOG_WARN("create window function expr failed", K(ret)); - } else if (OB_ISNULL(agg_expr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid argument", K(ret), K(agg_expr)); - } else if (OB_FAIL(agg_expr->add_real_param_expr(child_expr))) { - LOG_WARN("fail to set partition exprs", K(ret)); - } else if (OB_FAIL(agg_expr->formalize(ctx_->session_info_))) { - LOG_WARN("failed to formalize windown function", K(ret)); - } else if (OB_FAIL(agg_expr->pull_relation_id())) { - LOG_WARN("failed to pull relation id and levels", K(ret)); - } - return ret; -} - int ObTransformWinMagic::adjust_column_and_table(ObDMLStmt *main_stmt, TableItem *view, ObStmtMapInfo &map_info) @@ -1318,11 +1292,11 @@ int ObTransformWinMagic::adjust_column_and_table(ObDMLStmt *main_stmt, for (int64_t i = 0; OB_SUCC(ret) && i < view_stmt->get_group_expr_size(); i++) { ObRawExpr *expr = view_stmt->get_group_exprs().at(i); ObSEArray is_not_null_exprs; - ObOpRawExpr *is_not_null = NULL; + ObRawExpr *is_not_null = NULL; if (OB_ISNULL(expr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("gourp by expr is null", K(ret)); - } else if (OB_FAIL(ObTransformUtils::add_is_not_null(ctx_, view_stmt, expr, is_not_null))) { + } else if (OB_FAIL(ObTransformUtils::add_is_not_null(ctx_, expr, is_not_null))) { LOG_WARN("failed to add is not null expr", K(ret)); } else if (OB_FAIL(is_not_null_exprs.push_back(is_not_null))) { LOG_WARN("failed to push is not null expr into array", K(ret)); @@ -1634,11 +1608,11 @@ int ObTransformWinMagic::change_agg_to_win_func(ObDMLStmt *main_stmt, for (int64_t i = 0; OB_SUCC(ret) && i < partition_exprs.count(); i++) { ObRawExpr *expr = partition_exprs.at(i); ObSEArray is_not_null_exprs; - ObOpRawExpr *is_not_null = NULL; + ObRawExpr *is_not_null = NULL; if (OB_ISNULL(expr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("gourp by expr is null", K(ret)); - } else if (OB_FAIL(ObTransformUtils::add_is_not_null(ctx_, transed_stmt, expr, is_not_null))) { + } else if (OB_FAIL(ObTransformUtils::add_is_not_null(ctx_, expr, is_not_null))) { LOG_WARN("failed to add is not null expr", K(ret)); } else if (OB_FAIL(is_not_null_exprs.push_back(is_not_null))) { LOG_WARN("failed to push is not null expr into array", K(ret)); @@ -1684,7 +1658,7 @@ int ObTransformWinMagic::change_agg_to_win_func(ObDMLStmt *main_stmt, (agg_expr->get_expr_type() == T_FUN_COUNT ? T_FUN_COUNT_SUM : agg_expr->get_expr_type())))) { //never reach - } else if (OB_FAIL(create_aggr_expr(type, new_agg_expr, col_expr))) { + } else if (OB_FAIL(ObTransformUtils::create_aggr_expr(ctx_, type, new_agg_expr, col_expr))) { LOG_WARN("creat aggr expr failed", K(ret)); } else if (OB_ISNULL(new_agg_expr)) { ret = OB_ERR_UNEXPECTED; diff --git a/src/sql/rewrite/ob_transform_win_magic.h b/src/sql/rewrite/ob_transform_win_magic.h index 7271d0896..1c8ba2ce2 100644 --- a/src/sql/rewrite/ob_transform_win_magic.h +++ b/src/sql/rewrite/ob_transform_win_magic.h @@ -109,10 +109,6 @@ private: ObIArray &map, bool &is_valid); - int create_aggr_expr(ObItemType type, - ObAggFunRawExpr *&agg_expr, - ObRawExpr *child_expr); - int adjust_agg_to_win(ObSelectStmt *view_stmt); int adjust_view_for_trans(ObDMLStmt *main_stmt, diff --git a/src/sql/rewrite/ob_transformer_impl.cpp b/src/sql/rewrite/ob_transformer_impl.cpp index b0a4a6a8b..0e87825af 100644 --- a/src/sql/rewrite/ob_transformer_impl.cpp +++ b/src/sql/rewrite/ob_transformer_impl.cpp @@ -50,6 +50,7 @@ #include "sql/rewrite/ob_transform_conditional_aggr_coalesce.h" #include "sql/rewrite/ob_transform_mv_rewrite.h" #include "sql/rewrite/ob_transform_decorrelate.h" +#include "sql/rewrite/ob_transform_mv_rewrite_prepare.h" #include "sql/rewrite/ob_transform_late_materialization.h" #include "common/ob_smart_call.h" #include "sql/engine/ob_exec_context.h" @@ -82,6 +83,8 @@ int ObTransformerImpl::transform(ObDMLStmt *&stmt) LOG_WARN("failed to formalize query ref exprs"); } else if (OB_FAIL(stmt->formalize_stmt_expr_reference(ctx_->expr_factory_, ctx_->session_info_))) { LOG_WARN("failed to formalize stmt reference", K(ret)); + } else if (OB_FAIL(do_prepare_mv_rewrite(stmt))) { + LOG_WARN("failed to do prepare mv rewrite", K(ret)); } else if (OB_FAIL(do_transform(stmt))) { LOG_WARN("failed to do transform", K(ret)); } else if (OB_FAIL(do_transform_dblink_read(stmt))) { @@ -344,6 +347,24 @@ int ObTransformerImpl::do_transform_pre_precessing(ObDMLStmt *&stmt) return ret; } +int ObTransformerImpl::do_prepare_mv_rewrite(const ObDMLStmt *stmt) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(stmt) || OB_ISNULL(ctx_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(stmt), K(ret)); + } else { + ObTransformMVRewritePrepare trans(ctx_); + OPT_TRACE_TITLE("start prepare mv rewrite info"); + if (OB_FAIL(trans.prepare_mv_rewrite_info(stmt))) { + LOG_WARN("failed to do transform mv rewrite prepare", K(ret)); + } else { + LOG_TRACE("succeed to do transform mv rewrite prepare"); + } + } + return ret; +} + int ObTransformerImpl::do_transform_dblink_write(ObDMLStmt *&stmt, bool &trans_happened) { int ret = OB_SUCCESS; diff --git a/src/sql/rewrite/ob_transformer_impl.h b/src/sql/rewrite/ob_transformer_impl.h index ac25688f9..f20892e88 100644 --- a/src/sql/rewrite/ob_transformer_impl.h +++ b/src/sql/rewrite/ob_transformer_impl.h @@ -56,6 +56,7 @@ public: int transform(ObDMLStmt *&stmt); int do_transform(ObDMLStmt *&stmt); int do_transform_pre_precessing(ObDMLStmt *&stmt); + int do_prepare_mv_rewrite(const ObDMLStmt *stmt); int do_transform_dblink_write(ObDMLStmt *&stmt, bool &trans_happened); int do_transform_dblink_read(ObDMLStmt *&stmt); int transform_heuristic_rule(ObDMLStmt *&stmt); diff --git a/src/sql/session/ob_local_session_var.cpp b/src/sql/session/ob_local_session_var.cpp new file mode 100644 index 000000000..8b0378bce --- /dev/null +++ b/src/sql/session/ob_local_session_var.cpp @@ -0,0 +1,556 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_SESSION + +#include "sql/session/ob_local_session_var.h" +#include "sql/session/ob_basic_session_info.h" + +using namespace oceanbase::common; +using namespace oceanbase::share; + +namespace oceanbase +{ +namespace sql +{ + +template +int ObLocalSessionVar::set_local_vars(T &var_array) +{ + int ret = OB_SUCCESS; + if (!local_session_vars_.empty()) { + local_session_vars_.reset(); + } + if (OB_FAIL(local_session_vars_.reserve(var_array.count()))) { + LOG_WARN("fail to reserve for local_session_vars", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < var_array.count(); ++i) { + if (OB_FAIL(add_local_var(var_array.at(i)))) { + LOG_WARN("fail to add session var", K(ret)); + } + } + } + return ret; +} + +int ObLocalSessionVar::add_local_var(const ObSessionSysVar *var) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(var)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), KP(var)); + } else if (OB_FAIL(add_local_var(var->type_, var->val_))) { + LOG_WARN("fail to add local session var", K(ret)); + } + return ret; +} + +int ObLocalSessionVar::add_local_var(ObSysVarClassType var_type, const ObObj &value) +{ + int ret = OB_SUCCESS; + ObSessionSysVar *cur_var = NULL; + if (OB_ISNULL(alloc_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), KP(alloc_)); + } else if (OB_FAIL(get_local_var(var_type, cur_var))) { + LOG_WARN("get local var failed", K(ret)); + } else if (NULL == cur_var) { + ObSessionSysVar *new_var = OB_NEWx(ObSessionSysVar, alloc_); + if (OB_ISNULL(new_var)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc new var failed.", K(ret)); + } else if (OB_FAIL(local_session_vars_.push_back(new_var))) { + LOG_WARN("push back new var failed", K(ret)); + } else if (OB_FAIL(deep_copy_obj(*alloc_, value, new_var->val_))) { + LOG_WARN("fail to deep copy obj", K(ret)); + } else { + new_var->type_ = var_type; + } + } else if (!cur_var->is_equal(value)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("local session var added before is not equal to the new var", K(ret), KPC(cur_var), K(value)); + } + return ret; +} + +int ObLocalSessionVar::get_local_var(ObSysVarClassType var_type, ObSessionSysVar *&sys_var) const +{ + int ret = OB_SUCCESS; + sys_var = NULL; + for (int64_t i = 0; OB_SUCC(ret) && NULL == sys_var && i < local_session_vars_.count(); ++i) { + if (OB_ISNULL(local_session_vars_.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(local_session_vars_)); + } else if (local_session_vars_.at(i)->type_ == var_type) { + sys_var = local_session_vars_.at(i); + } + } + return ret; +} + +int ObLocalSessionVar::remove_local_var(ObSysVarClassType var_type) +{ + int ret = OB_SUCCESS; + bool find = false; + for (int64_t i = 0; OB_SUCC(ret) && !find && i < local_session_vars_.count(); ++i) { + if (OB_ISNULL(local_session_vars_.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(local_session_vars_)); + } else if (local_session_vars_.at(i)->type_ == var_type) { + local_session_vars_.at(i) = local_session_vars_.at(local_session_vars_.count() - 1); + find = true; + } + } + if (OB_SUCC(ret) && find) { + local_session_vars_.pop_back(); + } + return ret; +} + +int ObLocalSessionVar::get_local_vars(ObIArray &var_array) const +{ + int ret = OB_SUCCESS; + var_array.reset(); + for (int64_t i = 0; OB_SUCC(ret) && i < local_session_vars_.count(); ++i){ + if (OB_FAIL(var_array.push_back(local_session_vars_.at(i)))) { + LOG_WARN("push back local session vars failed", K(ret)); + } + } + return ret; +} + +int ObLocalSessionVar::remove_vars_same_with_session(const sql::ObBasicSessionInfo *session) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(session)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), KP(session)); + } else { + bool is_same = false; + ObSEArray new_var_array; + for (int64_t i = 0; OB_SUCC(ret) && i < local_session_vars_.count(); ++i) { + if (OB_FAIL(check_var_same_with_session(*session, local_session_vars_.at(i), is_same))) { + LOG_WARN("fail to check var same with session", K(ret)); + } else if (is_same) { + /* do nothing */ + } else if (OB_FAIL(new_var_array.push_back(local_session_vars_.at(i)))) { + LOG_WARN("fail to push into new var array", K(ret)); + } + } + if (OB_SUCC(ret) && new_var_array.count() != local_session_vars_.count()) { + if (OB_FAIL(local_session_vars_.assign(new_var_array))) { + LOG_WARN("fail to set local session vars.", K(ret)); + } + } + } + return ret; +} + +int ObLocalSessionVar::get_different_vars_from_session(const sql::ObBasicSessionInfo *session, + ObIArray &local_diff_vars, + ObIArray &session_vals) const +{ + int ret = OB_SUCCESS; + local_diff_vars.reuse(); + if (OB_ISNULL(session)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), KP(session)); + } else { + bool is_same = false; + ObObj session_val; + for (int64_t i = 0; OB_SUCC(ret) && i < local_session_vars_.count(); ++i) { + if (OB_FAIL(check_var_same_with_session(*session, local_session_vars_.at(i), is_same, &session_val))) { + LOG_WARN("fail to check var same with session", K(ret)); + } else if (is_same) { + /* do nothing */ + } else if (OB_FAIL(local_diff_vars.push_back(local_session_vars_.at(i)))) { + LOG_WARN("fail to push back sys var", K(ret)); + } else if (OB_FAIL(session_vals.push_back(session_val))) { + LOG_WARN("fail to push back obj", K(ret)); + } + } + } + return ret; +} + +int ObLocalSessionVar::check_var_same_with_session(const sql::ObBasicSessionInfo &session, + const ObSessionSysVar *local_var, + bool &is_same, + ObObj *diff_val /* default null */ ) const +{ + int ret = OB_SUCCESS; + is_same = false; + ObObj session_val; + if (OB_ISNULL(local_var)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), KP(local_var)); + } else if (SYS_VAR_SQL_MODE == local_var->type_) { + is_same = local_var->val_.get_uint64() == session.get_sql_mode(); + if (!is_same && NULL != diff_val) { + diff_val->set_uint64(session.get_sql_mode()); + } + } else if (OB_FAIL(session.get_sys_variable(local_var->type_, session_val))) { + LOG_WARN("fail to get session variable", K(ret)); + } else { + is_same = local_var->is_equal(session_val); + if (!is_same && NULL != diff_val) { + *diff_val = session_val; + } + } + return ret; +} + +int ObLocalSessionVar::gen_local_session_var_str(ObIAllocator &allocator, + ObString &local_session_var_str) const +{ + int ret = OB_SUCCESS; + int64_t pos = 0; + int64_t buf_len = get_serialize_size(); + char *binary_str = NULL; + char *hex_str = NULL; + int64_t hex_pos = 0; + ObArenaAllocator tmp_allocator(ObModIds::OB_TEMP_VARIABLES); + if (OB_ISNULL(binary_str = static_cast(tmp_allocator.alloc(buf_len))) + || OB_ISNULL(hex_str = static_cast(allocator.alloc(buf_len * 2)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate memory for local_session_var failed", K(ret), KP(binary_str), KP(hex_str)); + } else if (OB_FAIL(serialize_(binary_str, buf_len, pos))) { + LOG_WARN("fail to serialize local_session_var", K(ret)); + } else if (OB_FAIL(common::hex_print(binary_str, pos, hex_str, buf_len * 2, hex_pos))) { + LOG_WARN("print hex string failed", K(ret)); + } else { + local_session_var_str.assign(hex_str, hex_pos); + } + return ret; +} + +int ObLocalSessionVar::fill_local_session_var_from_str(const ObString &local_session_var_str) +{ + int ret = OB_SUCCESS; + ObArenaAllocator tmp_allocator(ObModIds::OB_TEMP_VARIABLES); + char *value_buf = NULL; + ObLength len = 0; + int64_t pos = 0; + if (OB_UNLIKELY(local_session_var_str.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected empty str", K(ret), K(local_session_var_str)); + } else if (OB_ISNULL(value_buf = static_cast(tmp_allocator.alloc(local_session_var_str.length())))) { + ret = common::OB_ALLOCATE_MEMORY_FAILED; + SHARE_SCHEMA_LOG(WARN, "fail to alloc memory", K(ret)); + LOG_WARN("fail to alloc memory", K(ret)); + } else if (OB_FALSE_IT(len = common::str_to_hex(local_session_var_str.ptr(), local_session_var_str.length(), + value_buf, local_session_var_str.length()))) { + } else if (OB_FAIL(deserialize_(value_buf, static_cast(len), pos))) { + LOG_WARN("fail to deserialize local_session_var", K(ret)); + } + return ret; +} + +int ObLocalSessionVar::deep_copy(const ObLocalSessionVar &other) +{ + int ret = OB_SUCCESS; + local_session_vars_.reset(); + if (this == &other) { + //do nothing + } else if (NULL != other.alloc_) { + if (NULL == alloc_) { + alloc_ = other.alloc_; + local_session_vars_.set_allocator(other.alloc_); + } + } + if (OB_FAIL(set_local_vars(other.local_session_vars_))) { + LOG_WARN("fail to add session var", K(ret)); + } + return ret; +} + +int ObLocalSessionVar::deep_copy_self() +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(alloc_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null allocator", K(ret)); + } else { + ObSEArray var_array; + if (OB_FAIL(get_local_vars(var_array))) { + LOG_WARN("get local vars failed", K(ret)); + } else if (OB_FAIL(set_local_vars(var_array))) { + LOG_WARN("set local vars failed", K(ret)); + } + } + return ret; +} + +int ObLocalSessionVar::assign(const ObLocalSessionVar &other) +{ + int ret = OB_SUCCESS; + local_session_vars_.reset(); + if (NULL != other.alloc_) { + if (NULL == alloc_) { + alloc_ = other.alloc_; + local_session_vars_.set_allocator(other.alloc_); + } + if (OB_FAIL(local_session_vars_.reserve(other.local_session_vars_.count()))) { + LOG_WARN("reserve failed", K(ret)); + } else if (OB_FAIL(local_session_vars_.assign(other.local_session_vars_))) { + LOG_WARN("fail to push back local var", K(ret)); + } + } else { + //do nothing, other is not inited + } + return ret; +} + +void ObLocalSessionVar::reset() +{ + local_session_vars_.reset(); +} + +int ObLocalSessionVar::set_local_var_capacity(int64_t sz) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(local_session_vars_.reserve(sz))) { + LOG_WARN("reserve failed", K(ret), K(sz)); + } + return ret; +} + +bool ObLocalSessionVar::operator == (const ObLocalSessionVar& other) const +{ + bool is_equal = local_session_vars_.count() == other.local_session_vars_.count(); + if (is_equal) { + int tmp_ret = OB_SUCCESS; + for (int64_t i = 0; is_equal && i < local_session_vars_.count(); ++i) { + ObSessionSysVar *var = local_session_vars_.at(i); + ObSessionSysVar *other_val = NULL; ; + if (OB_ISNULL(var)) { + is_equal = false; + } else if ((tmp_ret = other.get_local_var(var->type_, other_val)) != OB_SUCCESS) { + is_equal = false; + } else if (other_val == NULL) { + is_equal = false; + } else { + is_equal = var->is_equal(other_val->val_); + } + } + } + return is_equal; +} + +int64_t ObLocalSessionVar::get_deep_copy_size() const +{ + int64_t sz = sizeof(*this) + local_session_vars_.count() * sizeof(ObSessionSysVar *); + for (int64_t i = 0; i < local_session_vars_.count(); ++i) { + if (OB_NOT_NULL(local_session_vars_.at(i))) { + sz += local_session_vars_.at(i)->get_deep_copy_size(); + } + } + return sz; +} + +bool ObSessionSysVar::is_equal(const ObObj &other) const +{ + bool bool_ret = false; + if (val_.get_meta() != other.get_meta()) { + bool_ret = false; + if (ob_is_string_type(val_.get_type()) + && val_.get_type() == other.get_type() + && val_.get_collation_type() != other.get_collation_type()) { + //the collation type of string system variables will be set to the current connection collation type after updating values. + //return true if the string values are equal. + bool_ret = common::ObCharset::case_sensitive_equal(val_.get_string(), other.get_string()); + } + } else if (val_.is_equal(other, CS_TYPE_BINARY)) { + bool_ret = true; + } + return bool_ret; +} + +int ObSessionSysVar::get_sys_var_val_str(const ObSysVarClassType var_type, + const ObObj &var_val, + ObIAllocator &allocator, + ObString &val_str) +{ + int ret = OB_SUCCESS; + val_str.reset(); + char *buffer = NULL; + int64_t length = 0; + int64_t pos = 0; + if (SYS_VAR_SQL_MODE == var_type) { + ObObj res_obj; + if (OB_FAIL(ob_sql_mode_to_str(var_val, res_obj, &allocator))) { + LOG_WARN("fail to convert sql mode to str", K(ret), K(var_val)); + } else if (OB_FAIL(res_obj.get_string(val_str))) { + LOG_WARN("fail to get string form obj", K(ret), K(res_obj)); + } + } else if (OB_FAIL(var_val.print_sql_literal(buffer, length, pos, allocator))) { + LOG_WARN("print value failed", K(ret)); + } else { + val_str.assign(buffer, pos); + } + return ret; +} + +OB_DEF_SERIALIZE(ObSessionSysVar) +{ + int ret = OB_SUCCESS; + LST_DO_CODE(OB_UNIS_ENCODE, type_, val_); + return ret; +} + +OB_DEF_SERIALIZE_SIZE(ObSessionSysVar) +{ + int64_t len = 0; + LST_DO_CODE(OB_UNIS_ADD_LEN, type_, val_); + return len; +} + +OB_DEF_DESERIALIZE(ObSessionSysVar) +{ + int ret = OB_SUCCESS; + LST_DO_CODE(OB_UNIS_DECODE, type_, val_); + return ret; +} + +int64_t ObSessionSysVar::get_deep_copy_size() const { + int64_t sz = sizeof(*this) + val_.get_deep_copy_size(); + return sz; +} + +const ObSysVarClassType ObLocalSessionVar::ALL_LOCAL_VARS[] = { + SYS_VAR_TIME_ZONE, + SYS_VAR_SQL_MODE, + SYS_VAR_NLS_DATE_FORMAT, + SYS_VAR_NLS_TIMESTAMP_FORMAT, + SYS_VAR_NLS_TIMESTAMP_TZ_FORMAT, + SYS_VAR_COLLATION_CONNECTION, + SYS_VAR_MAX_ALLOWED_PACKET +}; + +//add all vars that can be solidified +int ObLocalSessionVar::load_session_vars(const sql::ObBasicSessionInfo *session) { + int ret = OB_SUCCESS; + int64_t var_num = sizeof(ALL_LOCAL_VARS) / sizeof(ObSysVarClassType); + if (OB_ISNULL(session)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null session", K(ret)); + } else if (!local_session_vars_.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("local_session_vars can only be inited once", K(ret)); + } else if (OB_FAIL(local_session_vars_.reserve(var_num))) { + LOG_WARN("reserve failed", K(ret), K(var_num)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < var_num; ++i) { + ObObj var; + if (OB_FAIL(session->get_sys_variable(ALL_LOCAL_VARS[i], var))) { + LOG_WARN("fail to get session variable", K(ret)); + } else if (OB_FAIL(add_local_var(ALL_LOCAL_VARS[i], var))) { + LOG_WARN("fail to add session var", K(ret), K(var)); + } + } + } + return ret; +} + +int ObLocalSessionVar::reserve_max_local_vars_capacity() { + int ret = OB_SUCCESS; + int64_t var_num = sizeof(ALL_LOCAL_VARS) / sizeof(ObSysVarClassType); + if (!local_session_vars_.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("local_session_vars can only be inited once", K(ret)); + } else if (OB_FAIL(local_session_vars_.reserve(var_num))) { + LOG_WARN("reserve failed", K(ret), K(var_num)); + } + return ret; +} + +int ObLocalSessionVar::update_session_vars_with_local(sql::ObBasicSessionInfo &session) const { + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < local_session_vars_.count(); ++i) { + if (OB_ISNULL(local_session_vars_.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_FAIL(session.update_sys_variable(local_session_vars_.at(i)->type_, local_session_vars_.at(i)->val_))) { + LOG_WARN("fail to update sys variable", K(ret)); + } + } + return ret; +} + +OB_DEF_SERIALIZE(ObLocalSessionVar) +{ + int ret = OB_SUCCESS; + LST_DO_CODE(OB_UNIS_ENCODE, local_session_vars_.count()); + for (int64_t i = 0; OB_SUCC(ret) && i < local_session_vars_.count(); ++i) { + if (OB_ISNULL(local_session_vars_.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else { + LST_DO_CODE(OB_UNIS_ENCODE, *local_session_vars_.at(i)); + } + } + return ret; +} + +OB_DEF_SERIALIZE_SIZE(ObLocalSessionVar) +{ + int64_t len = 0; + LST_DO_CODE(OB_UNIS_ADD_LEN, local_session_vars_.count()); + for (int64_t i = 0; i < local_session_vars_.count(); ++i) { + if (OB_NOT_NULL(local_session_vars_.at(i))) { + LST_DO_CODE(OB_UNIS_ADD_LEN, *local_session_vars_.at(i)); + } + } + return len; +} + +OB_DEF_DESERIALIZE(ObLocalSessionVar) +{ + int ret = OB_SUCCESS; + int64_t cnt = 0; + OB_UNIS_DECODE(cnt); + if (OB_SUCC(ret)) { + if (OB_FAIL(local_session_vars_.reserve(cnt))) { + LOG_WARN("reserve failed", K(ret)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < cnt; ++i) { + ObSessionSysVar var; + LST_DO_CODE(OB_UNIS_DECODE, var); + if (OB_SUCC(ret)) { + if (OB_FAIL(add_local_var(&var))) { + LOG_WARN("fail to add local session var", K(ret)); + } + } + } + return ret; +} + +DEF_TO_STRING(ObLocalSessionVar) +{ + int64_t pos = 0; + J_OBJ_START(); + for (int64_t i = 0; i < local_session_vars_.count(); ++i) { + if (i > 0) { + J_COMMA(); + } + if (OB_NOT_NULL(local_session_vars_.at(i))) { + J_KV("type", local_session_vars_.at(i)->type_, + "val", local_session_vars_.at(i)->val_); + } + } + J_OBJ_END(); + return pos; +} + +}//end of namespace sql +}//end of namespace oceanbase diff --git a/src/sql/session/ob_local_session_var.h b/src/sql/session/ob_local_session_var.h new file mode 100755 index 000000000..7f12da132 --- /dev/null +++ b/src/sql/session/ob_local_session_var.h @@ -0,0 +1,103 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + + +#ifndef OCEANBASE_SQL_LOCAL_SESSION_VAR_H_ +#define OCEANBASE_SQL_LOCAL_SESSION_VAR_H_ + +#include "share/system_variable/ob_sys_var_class_type.h" +#include "lib/container/ob_se_array.h" +#include "common/object/ob_object.h" +#include "lib/container/ob_fixed_array.h" +#include "lib/allocator/page_arena.h" + +namespace oceanbase +{ + +namespace share +{ + enum ObSysVarClassType; +} + +namespace sql +{ +class ObBasicSessionInfo; + +struct ObSessionSysVar { + OB_UNIS_VERSION(1); +public: + TO_STRING_KV(K_(type), K_(val)); + bool is_equal(const ObObj &other_val) const; + int64_t get_deep_copy_size() const; + static int get_sys_var_val_str(const share::ObSysVarClassType var_type, + const ObObj &var_val, + common::ObIAllocator &allocator, + ObString &val_str); + + share::ObSysVarClassType type_; + ObObj val_; +}; + +class ObLocalSessionVar { + OB_UNIS_VERSION(1); +public: + ObLocalSessionVar(common::ObIAllocator *alloc) + :alloc_(alloc), + local_session_vars_(alloc) { + } + ObLocalSessionVar () + :alloc_(NULL) { + } + ~ObLocalSessionVar() { reset(); } + void set_allocator(common::ObIAllocator *allocator) { + alloc_ = allocator; + local_session_vars_.set_allocator(allocator); + } + void reset(); + int set_local_var_capacity(int64_t sz); + template + int set_local_vars(T &var_array); + int add_local_var(share::ObSysVarClassType var_type, const ObObj &value); + int add_local_var(const ObSessionSysVar *var); + int get_local_var(share::ObSysVarClassType var_type, ObSessionSysVar *&sys_var) const; + int remove_local_var(share::ObSysVarClassType var_type); + int get_local_vars(common::ObIArray &var_array) const; + int load_session_vars(const sql::ObBasicSessionInfo *session); + int reserve_max_local_vars_capacity(); + int update_session_vars_with_local(sql::ObBasicSessionInfo &session) const; + int remove_vars_same_with_session(const ObBasicSessionInfo *session); + int get_different_vars_from_session(const ObBasicSessionInfo *session, + common::ObIArray &local_diff_vars, + common::ObIArray &session_vals) const; + int check_var_same_with_session(const ObBasicSessionInfo &session, + const ObSessionSysVar *local_var, + bool &is_same, + ObObj *diff_val = NULL) const; + int deep_copy(const ObLocalSessionVar &other); + int deep_copy_self(); + int assign(const ObLocalSessionVar &other); + bool operator == (const ObLocalSessionVar& other) const; + int64_t get_deep_copy_size() const ; + int64_t get_var_count() const { return local_session_vars_.count(); } + int gen_local_session_var_str(common::ObIAllocator &allocator, ObString &local_session_var_str) const; + int fill_local_session_var_from_str(const ObString &local_session_var_str); + DECLARE_TO_STRING; +private: + const static share::ObSysVarClassType ALL_LOCAL_VARS[]; + common::ObIAllocator *alloc_; + ObFixedArray local_session_vars_; +}; + +} // namespace sql +}//namespace oceanbase + +#endif /* _OB_OCEANBASE_SCHEMA_SCHEMA_STRUCT_H */ From d52d52c555442e2799eec0488a72cfadc4e71aac Mon Sep 17 00:00:00 2001 From: SanmuWangZJU Date: Fri, 23 Aug 2024 14:01:09 +0000 Subject: [PATCH 201/249] [CP] [OBCDC] Support update ls_svr_list refresh interval --- src/logservice/libobcdc/src/ob_log_config.h | 2 +- .../libobcdc/src/ob_log_ls_fetch_ctx.cpp | 20 +++--- .../logrouteservice/ob_log_route_service.cpp | 68 ++++++++++++------- .../logrouteservice/ob_log_route_service.h | 5 +- 4 files changed, 57 insertions(+), 38 deletions(-) diff --git a/src/logservice/libobcdc/src/ob_log_config.h b/src/logservice/libobcdc/src/ob_log_config.h index 2d235d952..523d41c56 100644 --- a/src/logservice/libobcdc/src/ob_log_config.h +++ b/src/logservice/libobcdc/src/ob_log_config.h @@ -343,7 +343,7 @@ public: // If the logs are not fetched after a certain period of time, the stream will be cut T_DEF_INT_INFT(ls_fetch_progress_update_timeout_sec, OB_CLUSTER_PARAMETER, 15, 1, "logstream fetch progress update timeout in seconds"); - T_DEF_INT_INFT(log_router_background_refresh_interval_sec, OB_CLUSTER_PARAMETER, 10, 1, + T_DEF_INT_INFT(log_router_background_refresh_interval_sec, OB_CLUSTER_PARAMETER, 1200, 1, "log_route_service background_refresh_time in seconds"); // cache update interval of sys table __all_server T_DEF_INT_INFT(all_server_cache_update_interval_sec, OB_CLUSTER_PARAMETER, 5, 1, diff --git a/src/logservice/libobcdc/src/ob_log_ls_fetch_ctx.cpp b/src/logservice/libobcdc/src/ob_log_ls_fetch_ctx.cpp index 3674ca49b..42a38faa9 100644 --- a/src/logservice/libobcdc/src/ob_log_ls_fetch_ctx.cpp +++ b/src/logservice/libobcdc/src/ob_log_ls_fetch_ctx.cpp @@ -124,9 +124,7 @@ void LSFetchCtx::reset() progress_.reset(); start_parameters_.reset(); fetch_info_.reset(); - //svr_list_need_update_ = true; - //TODO tmp test - svr_list_need_update_ = false; + svr_list_need_update_ = true; start_lsn_locate_req_.reset(); end_lsn_locate_req_.reset(); FetchTaskListNode::reset(); @@ -802,7 +800,9 @@ bool LSFetchCtx::need_update_svr_list() if (is_direct_fetching_mode(fetching_mode_)) { bool_ret = false; } else if(is_integrated_fetching_mode(fetching_mode_)) { - if (OB_FAIL(get_log_route_service_(log_route_service))) { + if (svr_list_need_update_) { + bool_ret = true; + } else if (OB_FAIL(get_log_route_service_(log_route_service))) { LOG_ERROR("get_log_route_service_ failed", KR(ret)); } else if (OB_FAIL(log_route_service->get_server_count(tls_id_.get_tenant_id(), tls_id_.get_ls_id(), avail_svr_count))) { @@ -811,12 +811,13 @@ bool LSFetchCtx::need_update_svr_list() } else { bool_ret = true; } - } else { // If no server is available, or if a proactive update is requested, an update is required - // if (avail_svr_count <= 0 || svr_list_need_update_) { - if (avail_svr_count <= 0) { - bool_ret = true; - } + } else if (avail_svr_count <= 0) { + bool_ret = true; + } + + if (bool_ret) { + mark_svr_list_update_flag(false); // will request svr_list, mark svr_list_need_update_ to false } } else { ret = OB_ERR_UNEXPECTED; @@ -868,6 +869,7 @@ int LSFetchCtx::update_svr_list(const bool need_print_info) LOG_ERROR("ObLogRouteService async_server_query_req failed", KR(ret), K(tls_id_)); } } else { + mark_svr_list_update_flag(false); LOG_DEBUG("async_server_query_req succ", K_(tls_id)); } diff --git a/src/logservice/logrouteservice/ob_log_route_service.cpp b/src/logservice/logrouteservice/ob_log_route_service.cpp index 192119dfd..625abb921 100755 --- a/src/logservice/logrouteservice/ob_log_route_service.cpp +++ b/src/logservice/logrouteservice/ob_log_route_service.cpp @@ -50,7 +50,8 @@ ObLogRouteService::ObLogRouteService() : blacklist_survival_time_upper_limit_min_(0), blacklist_survival_time_penalty_period_min_(0), blacklist_history_overdue_time_min_(0), - blacklist_history_clear_interval_min_(0) + blacklist_history_clear_interval_min_(0), + ls_svr_list_last_update_time_(OB_INVALID_TIMESTAMP) { } @@ -165,14 +166,15 @@ int ObLogRouteService::init(ObISQLClient *proxy, int ObLogRouteService::start() { int ret = OB_SUCCESS; + const static int64_t timer_task_interval = 5 * _SEC_; if (OB_FAIL(ls_route_timer_task_.init(lib::TGDefIDs::LogRouterTimer))) { LOG_WARN("ObLSRouteTimerTask init failed", KR(ret)); - } else if (OB_FAIL(timer_.schedule_repeate_task_immediately(ls_route_timer_task_, ObLSRouteTimerTask::REFRESH_INTERVAL))) { - LOG_WARN("fail to schedule min minor sstable gc task", K(ret)); + } else if (OB_FAIL(timer_.schedule_repeate_task_immediately(ls_route_timer_task_, timer_task_interval))) { + LOG_WARN("fail to schedule min minor sstable gc task", K(ret), K(timer_task_interval)); } else { is_stopped_ = false; - LOG_INFO("ObLogRouteService start succ", K(timer_id_), K(tg_id_)); + LOG_INFO("ObLogRouteService start succ", K(timer_id_), K(tg_id_), K(timer_task_interval)); } return ret; @@ -238,6 +240,7 @@ void ObLogRouteService::destroy() blacklist_survival_time_penalty_period_min_ = 0; blacklist_history_overdue_time_min_ = 0; blacklist_history_clear_interval_min_ = 0; + ls_svr_list_last_update_time_ = OB_INVALID_TIMESTAMP; is_tenant_mode_ = false; is_inited_ = false; @@ -322,9 +325,16 @@ int ObLogRouteService::update_background_refresh_time(const int64_t background_r if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_ERROR("ObLogRouteService has not been inited", KR(ret)); + } else if (OB_UNLIKELY(background_refresh_time_sec <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid background_refresh_time", KR(ret), K(background_refresh_time_sec)); } else { - const int64_t background_refresh_time = background_refresh_time_sec * _SEC_; - ATOMIC_SET(&background_refresh_time_sec_, background_refresh_time); + const int64_t prev_refresh_time = ATOMIC_LOAD(&background_refresh_time_sec_); + const int64_t new_background_refresh_time = background_refresh_time_sec * _SEC_; + if (OB_UNLIKELY((prev_refresh_time != new_background_refresh_time))) { + ATOMIC_SET(&background_refresh_time_sec_, new_background_refresh_time); + LOG_INFO("[CONFIG] update background_refresh_time", K(prev_refresh_time), K(new_background_refresh_time)); + } } return ret; @@ -1021,10 +1031,17 @@ bool ObLogRouteService::ObLSRouterValueGetter::operator()(const ObLSRouterKey &k int ObLogRouteService::update_all_ls_server_list_() { int ret = OB_SUCCESS; + const int64_t background_refresh_time = ATOMIC_LOAD(&background_refresh_time_sec_); + const int64_t current_timestamp = get_timestamp(); if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_ERROR("ObLogRouteService has not been inited", KR(ret)); + } else if (OB_UNLIKELY(background_refresh_time < 0)) { + // ignore + } else if (OB_LIKELY(ls_svr_list_last_update_time_ != OB_INVALID_TIMESTAMP + && ls_svr_list_last_update_time_ + background_refresh_time_sec_ > current_timestamp)) { + // not touch update interval, ignore } else { ObAllLSRouterKeyGetter all_ls_routerkey_getter; if (OB_FAIL(ls_router_map_.for_each(all_ls_routerkey_getter))) { @@ -1045,8 +1062,25 @@ int ObLogRouteService::update_all_ls_server_list_() LOG_WARN("failed to update router_value for key", K(key)); } } + + if (OB_FAIL(ret) && OB_ENTRY_NOT_EXIST != ret) { + // registe a task to update ls_server_list for failed task + if (OB_FAIL(registered(key.get_tenant_id(), key.get_ls_id()))) { + if (OB_HASH_EXIST != ret) { + LOG_WARN("registe async ls_update task failed", KR(ret), K(key)); + } + } else { + LOG_INFO("registe async ls_update task", KR(ret), K(key)); + } + } } + + // set update_time to launch ls_update on next interval + // update failed route_value should update on next interval or launch asnc_task by invoker + ls_svr_list_last_update_time_ = current_timestamp; } + + LOG_INFO("update_all_ls_server_list", KR(ret), K_(ls_svr_list_last_update_time)); } return ret; @@ -1141,9 +1175,10 @@ int ObLogRouteService::ObLSRouteTimerTask::init(int tg_id) if (IS_INIT) { ret = OB_INIT_TWICE; - LOG_ERROR("ObLSRouteTimerTask has already been inited", KR(ret)); + LOG_ERROR("ObLSRouteTimerTask has already been inited", KR(ret), K(tg_id)); } else { is_inited_ = true; + LOG_INFO("ls_route_timer_task inited", K(tg_id)); } return ret; @@ -1165,26 +1200,9 @@ void ObLogRouteService::ObLSRouteTimerTask::runTimerTask() LOG_WARN("ObLogRouteService update_all_server_and_zone_cache_ failed", KR(ret)); } else if (OB_FAIL(log_route_service_.update_all_ls_server_list_())) { LOG_WARN("ObLogRouteService update_all_ls_server_list_ failed", KR(ret)); - } else {} - - // ignore ret - if (OB_FAIL(log_route_service_.schedule_ls_timer_task_())) { - LOG_WARN("schedule_ls_timer_task_ failed", KR(ret)); - } -} - -int ObLogRouteService::schedule_ls_timer_task_() -{ - int ret = OB_SUCCESS; - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_ERROR("ObLSRouteTimerTask has not been inited", KR(ret)); } else { - // do nothing + LOG_TRACE("ls_route_timer_task task done"); } - - return ret; } } // namespace logservice diff --git a/src/logservice/logrouteservice/ob_log_route_service.h b/src/logservice/logrouteservice/ob_log_route_service.h index 432754955..ed2a01362 100755 --- a/src/logservice/logrouteservice/ob_log_route_service.h +++ b/src/logservice/logrouteservice/ob_log_route_service.h @@ -77,7 +77,7 @@ public: const bool is_across_cluster, logfetcher::IObLogErrHandler *err_handler, const char *external_server_blacklist = "|", - const int64_t background_refresh_time_sec = 10, + const int64_t background_refresh_time_sec = 1200, const int64_t all_server_cache_update_interval_sec = 5, const int64_t all_zone_cache_update_interval_sec = 5, const int64_t blacklist_survival_time_sec = 30, @@ -364,12 +364,10 @@ private: int init(int tg_id); void destroy(); virtual void runTimerTask() override; - static const int64_t REFRESH_INTERVAL = 5 * _SEC_; private: bool is_inited_; ObLogRouteService &log_route_service_; }; - int schedule_ls_timer_task_(); private: bool is_inited_; @@ -396,6 +394,7 @@ private: int64_t blacklist_survival_time_penalty_period_min_; int64_t blacklist_history_overdue_time_min_; int64_t blacklist_history_clear_interval_min_; + int64_t ls_svr_list_last_update_time_; DISALLOW_COPY_AND_ASSIGN(ObLogRouteService); }; From 3e2733ca2877472b9a56ae7a744f50b1776eca57 Mon Sep 17 00:00:00 2001 From: jingtaoye35 <1255153887@qq.com> Date: Fri, 23 Aug 2024 14:19:15 +0000 Subject: [PATCH 202/249] [CP] support set system variable using ':=' --- src/sql/parser/sql_parser_mysql_mode.y | 37 ++++++++++---------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/src/sql/parser/sql_parser_mysql_mode.y b/src/sql/parser/sql_parser_mysql_mode.y index 14a79e512..fec1ea60a 100644 --- a/src/sql/parser/sql_parser_mysql_mode.y +++ b/src/sql/parser/sql_parser_mysql_mode.y @@ -16489,11 +16489,6 @@ USER_VARIABLE to_or_eq expr malloc_non_terminal_node($$, result->malloc_pool_, T_VAR_VAL, 2, $1, $3); $$->value_ = 2; } -| USER_VARIABLE SET_VAR expr -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_VAR_VAL, 2, $1, $3); - $$->value_ = 2; -} | sys_var_and_val { $$ = $1; @@ -16541,11 +16536,6 @@ var_name to_or_eq set_expr_or_default malloc_non_terminal_node($$, result->malloc_pool_, T_VAR_VAL, 2, $1, $3); $$->value_ = 2; } -| var_name SET_VAR set_expr_or_default -{ - malloc_non_terminal_node($$, result->malloc_pool_, T_VAR_VAL, 2, $1, $3); - $$->value_ = 2; -} ; scope_or_scope_alias: @@ -16556,8 +16546,9 @@ GLOBAL %prec LOWER_PARENS { $$[0] = 1; } ; to_or_eq: -TO { $$ = NULL; } +TO { $$ = NULL; } | COMP_EQ { $$ = NULL; } +| SET_VAR { $$ = NULL; } ; set_role_stmt: @@ -21788,18 +21779,18 @@ part_info *===========================================================*/ var_name: - NAME_OB - { - $$ = $1; - } - | unreserved_keyword_normal - { - get_non_reserved_node($$, result->malloc_pool_, @1.first_column, @1.last_column); - } - | new_or_old_column_ref - { - $$ = $1; - } +NAME_OB +{ + $$ = $1; +} +| unreserved_keyword_normal +{ + get_non_reserved_node($$, result->malloc_pool_, @1.first_column, @1.last_column); +} +| new_or_old_column_ref +{ + $$ = $1; +} ; new_or_old: From d01c1a98849c53a3c08c479fd18f2ccad6f8101d Mon Sep 17 00:00:00 2001 From: jingtaoye35 <1255153887@qq.com> Date: Fri, 23 Aug 2024 14:25:19 +0000 Subject: [PATCH 203/249] [CP] need deep copy stmt for evaluate stmt cost --- .../ob_transform_late_materialization.cpp | 33 +++++++++---------- .../ob_transform_late_materialization.h | 3 +- src/sql/rewrite/ob_transform_rule.h | 10 +++--- 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/src/sql/rewrite/ob_transform_late_materialization.cpp b/src/sql/rewrite/ob_transform_late_materialization.cpp index 3cb3e6db7..0801363a1 100644 --- a/src/sql/rewrite/ob_transform_late_materialization.cpp +++ b/src/sql/rewrite/ob_transform_late_materialization.cpp @@ -449,7 +449,8 @@ int ObTransformLateMaterialization::check_index_match_late_materialization( return ret; } -int ObTransformLateMaterialization::evaluate_stmt_cost(ObDMLStmt *&stmt, +int ObTransformLateMaterialization::evaluate_stmt_cost(ObIArray &parent_stmts, + ObDMLStmt *&stmt, bool is_trans_stmt, double &plan_cost, bool &is_expected, @@ -457,6 +458,7 @@ int ObTransformLateMaterialization::evaluate_stmt_cost(ObDMLStmt *&stmt, { int ret = OB_SUCCESS; ObEvalCostHelper eval_cost_helper; + ObDMLStmt *root_stmt = NULL; is_expected = true; if (OB_ISNULL(stmt) || OB_ISNULL(ctx_) || OB_UNLIKELY(!ctx_->is_valid()) || OB_ISNULL(ctx_->exec_ctx_->get_physical_plan_ctx()) || @@ -467,16 +469,9 @@ int ObTransformLateMaterialization::evaluate_stmt_cost(ObDMLStmt *&stmt, } else if (OB_FAIL(eval_cost_helper.fill_helper(*ctx_->exec_ctx_->get_physical_plan_ctx(), *stmt->get_query_ctx(), *ctx_))) { LOG_WARN("failed to fill eval cost helper", K(ret)); - } else if (!is_trans_stmt) { - /* do nothing */ - } else if (OB_FAIL(construct_transform_hint(*stmt, NULL))) { - LOG_WARN("failed to construct transform hint", K(ret)); - } else if (OB_FAIL(stmt->adjust_qb_name(ctx_->allocator_, - ctx_->src_qb_name_, - ctx_->src_hash_val_))) { - LOG_WARN("failed to adjust qb name", K(ret)); - } - if (OB_SUCC(ret)) { + } else if (OB_FAIL(prepare_eval_cost_stmt(parent_stmts, *stmt, root_stmt, is_trans_stmt))) { + LOG_WARN("failed to prepare eval cost stmt", K(ret)); + } else { ctx_->eval_cost_ = true; lib::ContextParam param; param.set_mem_attr(ctx_->session_info_->get_effective_tenant_id(), @@ -496,14 +491,14 @@ int ObTransformLateMaterialization::evaluate_stmt_cost(ObDMLStmt *&stmt, &ctx_->exec_ctx_->get_physical_plan_ctx()->get_param_store(), *ctx_->self_addr_, GCTX.srv_rpc_proxy_, - stmt->get_query_ctx()->get_global_hint(), + root_stmt->get_query_ctx()->get_global_hint(), tmp_expr_factory, - stmt, + root_stmt, false, ctx_->exec_ctx_->get_stmt_factory()->get_query_ctx()) { ObOptimizer optimizer(optctx); ObLogPlan *plan = NULL; - if (OB_FAIL(optimizer.get_optimization_cost(*stmt, plan, plan_cost))) { + if (OB_FAIL(optimizer.get_optimization_cost(*root_stmt, plan, plan_cost))) { LOG_WARN("failed to get optimization cost", K(ret)); } else if (OB_FAIL(is_expected_plan(plan, &check_ctx, is_trans_stmt, is_expected))) { LOG_WARN("failed to check transformed plan", K(ret)); @@ -512,6 +507,8 @@ int ObTransformLateMaterialization::evaluate_stmt_cost(ObDMLStmt *&stmt, *ctx_->exec_ctx_->get_stmt_factory()->get_query_ctx(), *ctx_))) { LOG_WARN("failed to recover context", K(ret)); + } else if (OB_FAIL(ObTransformUtils::free_stmt(*ctx_->stmt_factory_, root_stmt))) { + LOG_WARN("failed to free stmt", K(ret)); } } } @@ -553,12 +550,14 @@ int ObTransformLateMaterialization::inner_accept_transform(ObIArrayget_stmt_hint().query_hint_->has_outline_data()) { trans_happened = true; LOG_TRACE("force accept to use late materialization"); - } else if (OB_FAIL(evaluate_stmt_cost(trans_stmt, true, trans_stmt_cost, is_expected, check_ctx))) { + } else if (OB_FAIL(evaluate_stmt_cost(parent_stmts, trans_stmt, true, trans_stmt_cost, + is_expected, check_ctx))) { LOG_WARN("failed to evaluate cost for the transformed stmt", K(ret)); } else if (!is_expected) { trans_happened = false; - } else if (!force_accept && OB_FAIL(evaluate_stmt_cost(stmt, false, base_stmt_cost, - is_base_expected, check_ctx))) { + } else if (!force_accept && OB_FAIL(evaluate_stmt_cost(parent_stmts, stmt, false, + base_stmt_cost, is_base_expected, + check_ctx))) { LOG_WARN("failed to evaluate cost of select_stmt"); } else { trans_happened = force_accept || diff --git a/src/sql/rewrite/ob_transform_late_materialization.h b/src/sql/rewrite/ob_transform_late_materialization.h index 452b1fb28..b42606869 100644 --- a/src/sql/rewrite/ob_transform_late_materialization.h +++ b/src/sql/rewrite/ob_transform_late_materialization.h @@ -132,7 +132,8 @@ private: ObCostBasedLateMaterializationCtx &ctx, bool &is_valid); int get_index_of_base_stmt_path(ObLogicalOperator* top, ObCostBasedLateMaterializationCtx &ctx); - int evaluate_stmt_cost(ObDMLStmt *&stmt, + int evaluate_stmt_cost(ObIArray &parent_stmts, + ObDMLStmt *&stmt, bool is_trans_stmt, double &plan_cost, bool &is_expected, diff --git a/src/sql/rewrite/ob_transform_rule.h b/src/sql/rewrite/ob_transform_rule.h index a1c8015d6..6e9c1ab79 100644 --- a/src/sql/rewrite/ob_transform_rule.h +++ b/src/sql/rewrite/ob_transform_rule.h @@ -487,6 +487,10 @@ protected: ObDMLStmt *&orgin_stmt, ObDMLStmt *&root_stmt); void reset_stmt_cost() { stmt_cost_ = -1; } + int prepare_eval_cost_stmt(common::ObIArray &parent_stmts, + ObDMLStmt &stmt, + ObDMLStmt *&copied_stmt, + bool is_trans_stmt); private: // pre-order transformation int transform_pre_order(common::ObIArray &parent_stmts, @@ -516,12 +520,6 @@ private: double &plan_cost, bool &is_expected, void *check_ctx = NULL); - - int prepare_eval_cost_stmt(common::ObIArray &parent_stmts, - ObDMLStmt &stmt, - ObDMLStmt *&copied_stmt, - bool is_trans_stmt); - int prepare_root_stmt_with_temp_table_filter(ObDMLStmt &root_stmt, ObDMLStmt *&root_stmt_with_filter); virtual int is_expected_plan(ObLogPlan *plan, From 30097a942fa512a9a7e4f4ab1a2e272971b46477 Mon Sep 17 00:00:00 2001 From: lmjhh <576788582@qq.com> Date: Fri, 23 Aug 2024 14:31:51 +0000 Subject: [PATCH 204/249] [CP] dbms sched executor delete conn.check_priv --- src/observer/dbms_scheduler/ob_dbms_sched_job_executor.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/observer/dbms_scheduler/ob_dbms_sched_job_executor.cpp b/src/observer/dbms_scheduler/ob_dbms_sched_job_executor.cpp index 1acfe1c87..1b31417b8 100644 --- a/src/observer/dbms_scheduler/ob_dbms_sched_job_executor.cpp +++ b/src/observer/dbms_scheduler/ob_dbms_sched_job_executor.cpp @@ -350,9 +350,6 @@ int ObDBMSSchedJobExecutor::run_dbms_sched_job( CK (OB_NOT_NULL(pool = static_cast(sql_proxy_->get_pool()))); OX (session_info->set_job_info(&job_info)); OZ (pool->acquire_spi_conn(session_info, conn)); - if (OB_NOT_NULL(conn)) { - conn->set_check_priv(true); - } OZ (conn->execute_write(tenant_id, what.string().ptr(), affected_rows)); if (OB_NOT_NULL(conn)) { sql_proxy_->close(conn, ret); From 230369e51b9236093a856f3234f16478737e6881 Mon Sep 17 00:00:00 2001 From: obdev Date: Fri, 23 Aug 2024 14:38:32 +0000 Subject: [PATCH 205/249] occupy for max_failures of __all_tenant_scheduler_job --- .../ob_inner_table_schema.12251_12300.cpp | 19 +++++++++++++++++++ .../ob_inner_table_schema.15201_15250.cpp | 15 +++++++++++++++ .../ob_inner_table_schema.351_400.cpp | 19 +++++++++++++++++++ .../inner_table/ob_inner_table_schema_def.py | 3 ++- .../r/mysql/desc_virtual_table_in_sys.result | 1 + 5 files changed, 56 insertions(+), 1 deletion(-) diff --git a/src/share/inner_table/ob_inner_table_schema.12251_12300.cpp b/src/share/inner_table/ob_inner_table_schema.12251_12300.cpp index 261c9cb69..e063293ae 100644 --- a/src/share/inner_table/ob_inner_table_schema.12251_12300.cpp +++ b/src/share/inner_table/ob_inner_table_schema.12251_12300.cpp @@ -4189,6 +4189,25 @@ int ObInnerTableSchema::all_virtual_tenant_scheduler_job_schema(ObTableSchema &t database_id_default, database_id_default); //default_value } + + if (OB_SUCC(ret)) { + ObObj max_failures_default; + max_failures_default.set_int(0); + ADD_COLUMN_SCHEMA_T("max_failures", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false, //is_autoincrement + max_failures_default, + max_failures_default); //default_value + } table_schema.set_index_using_type(USING_BTREE); table_schema.set_row_store_type(ENCODING_ROW_STORE); table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); diff --git a/src/share/inner_table/ob_inner_table_schema.15201_15250.cpp b/src/share/inner_table/ob_inner_table_schema.15201_15250.cpp index 53fc21184..7e094bde0 100644 --- a/src/share/inner_table/ob_inner_table_schema.15201_15250.cpp +++ b/src/share/inner_table/ob_inner_table_schema.15201_15250.cpp @@ -1002,6 +1002,21 @@ int ObInnerTableSchema::all_virtual_tenant_scheduler_job_real_agent_ora_schema(O false); //is_autoincrement } + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("MAX_FAILURES", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + true, //is_nullable + false); //is_autoincrement + } + if (OB_SUCC(ret)) { ADD_COLUMN_SCHEMA("GMT_CREATE", //column_name ++column_id, //column_id diff --git a/src/share/inner_table/ob_inner_table_schema.351_400.cpp b/src/share/inner_table/ob_inner_table_schema.351_400.cpp index 65341399f..ad42f9091 100644 --- a/src/share/inner_table/ob_inner_table_schema.351_400.cpp +++ b/src/share/inner_table/ob_inner_table_schema.351_400.cpp @@ -8969,6 +8969,25 @@ int ObInnerTableSchema::all_tenant_scheduler_job_schema(ObTableSchema &table_sch database_id_default, database_id_default); //default_value } + + if (OB_SUCC(ret)) { + ObObj max_failures_default; + max_failures_default.set_int(0); + ADD_COLUMN_SCHEMA_T("max_failures", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false, //is_autoincrement + max_failures_default, + max_failures_default); //default_value + } table_schema.set_index_using_type(USING_BTREE); table_schema.set_row_store_type(ENCODING_ROW_STORE); table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index cdcb373cb..d4cc29c5a 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -4435,7 +4435,8 @@ def_table_schema( ('destination_name', 'varchar:128', 'true'), ('interval_ts', 'int', 'true'), ('user_id', 'int', 'true', 'OB_INVALID_ID'), - ('database_id', 'int', 'true', 'OB_INVALID_ID') + ('database_id', 'int', 'true', 'OB_INVALID_ID'), + ('max_failures', 'int', 'true', '0') ], ) diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result index 6bfce8b23..509911307 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result @@ -6073,6 +6073,7 @@ destination_name varchar(128) YES NULL interval_ts bigint(20) YES NULL user_id bigint(20) YES -1 database_id bigint(20) YES -1 +max_failures bigint(20) YES 0 select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_tenant_scheduler_job; IF(count(*) >= 0, 1, 0) 1 From e48a0f2d220e481e977a718d4d9ed966c6dc45ed Mon Sep 17 00:00:00 2001 From: JinmaoLi Date: Fri, 23 Aug 2024 14:44:41 +0000 Subject: [PATCH 206/249] bugfix: always add column convert for dependant expr of gen col --- src/observer/table/ob_table_cg_service.cpp | 2 +- .../ob_external_table_file_mgr.cpp | 2 +- src/sql/ob_sql_utils.cpp | 2 +- src/sql/resolver/dml/ob_dml_resolver.cpp | 2 +- src/sql/resolver/dml/ob_insert_stmt.cpp | 2 +- src/sql/resolver/dml/ob_merge_stmt.cpp | 2 +- src/sql/resolver/expr/ob_raw_expr_util.cpp | 47 ++-- src/sql/resolver/expr/ob_raw_expr_util.h | 5 +- .../r/mysql/hash_distinct.result | 203 +++++++++--------- .../test_suite/update/r/mysql/update2.result | 10 +- 10 files changed, 136 insertions(+), 141 deletions(-) diff --git a/src/observer/table/ob_table_cg_service.cpp b/src/observer/table/ob_table_cg_service.cpp index e797165aa..0aeb9d894 100644 --- a/src/observer/table/ob_table_cg_service.cpp +++ b/src/observer/table/ob_table_cg_service.cpp @@ -340,7 +340,7 @@ int ObTableExprCgService::build_generated_column_expr(ObTableCtx &ctx, if (OB_SUCC(ret)) { if (OB_FAIL(gen_expr->formalize(&sess_info))) { LOG_WARN("fail to formailize column reference expr", K(ret)); - } else if (ObRawExprUtils::need_column_conv(item.expr_->get_result_type(), *gen_expr) + } else if (ObRawExprUtils::need_column_conv(item.expr_->get_result_type(), *gen_expr, true) && OB_FAIL(ObRawExprUtils::build_column_conv_expr(expr_factory, ctx.get_allocator(), *item.expr_, diff --git a/src/share/external_table/ob_external_table_file_mgr.cpp b/src/share/external_table/ob_external_table_file_mgr.cpp index bb3989c7d..be1f59f51 100644 --- a/src/share/external_table/ob_external_table_file_mgr.cpp +++ b/src/share/external_table/ob_external_table_file_mgr.cpp @@ -326,7 +326,7 @@ int ObExternalTableFileManager::get_genarated_expr_from_partition_column(const O expected_type.set_meta(column_schema->get_meta_type()); expected_type.set_accuracy(column_schema->get_accuracy()); expected_type.set_result_flag(ObRawExprUtils::calc_column_result_flag(*column_schema)); - if (ObRawExprUtils::need_column_conv(expected_type, *gen_expr)) { + if (ObRawExprUtils::need_column_conv(expected_type, *gen_expr, true)) { if (OB_FAIL(ObRawExprUtils::build_column_conv_expr(*expr_factory, column_schema, gen_expr, session_info))) { LOG_WARN("create cast expr failed", K(ret)); } diff --git a/src/sql/ob_sql_utils.cpp b/src/sql/ob_sql_utils.cpp index 8f9b784bc..2a11cc596 100644 --- a/src/sql/ob_sql_utils.cpp +++ b/src/sql/ob_sql_utils.cpp @@ -979,7 +979,7 @@ int ObSQLUtils::make_generated_expression_from_str(const common::ObString &expr_ ObExprResType dest_type; dest_type.set_meta(gen_col.get_meta_type()); dest_type.set_accuracy(gen_col.get_accuracy()); - if (ObRawExprUtils::need_column_conv(dest_type, *expr)) { + if (ObRawExprUtils::need_column_conv(dest_type, *expr, true)) { if (OB_FAIL(ObRawExprUtils::build_column_conv_expr(expr_factory, &gen_col, expr, &session))) { LOG_WARN("create column convert expr failed", K(ret)); } diff --git a/src/sql/resolver/dml/ob_dml_resolver.cpp b/src/sql/resolver/dml/ob_dml_resolver.cpp index d9c5dcafa..c7aaa4234 100755 --- a/src/sql/resolver/dml/ob_dml_resolver.cpp +++ b/src/sql/resolver/dml/ob_dml_resolver.cpp @@ -8516,7 +8516,7 @@ int ObDMLResolver::resolve_generated_column_expr(const ObString &expr_str, LOG_WARN("build padding expr for self failed", K(ret)); } else if (OB_FAIL(ref_expr->formalize_with_local_vars(session_info, &local_vars, var_array_idx))) { LOG_WARN("formailize column reference expr failed", K(ret)); - } else if (ObRawExprUtils::need_column_conv(column.get_result_type(), *ref_expr)) { + } else if (ObRawExprUtils::need_column_conv(column.get_result_type(), *ref_expr, true)) { if (OB_FAIL(ObRawExprUtils::build_column_conv_expr(*expr_factory, *allocator_, column, ref_expr, session_info, used_for_generated_column, diff --git a/src/sql/resolver/dml/ob_insert_stmt.cpp b/src/sql/resolver/dml/ob_insert_stmt.cpp index b8365b4aa..a0d613588 100644 --- a/src/sql/resolver/dml/ob_insert_stmt.cpp +++ b/src/sql/resolver/dml/ob_insert_stmt.cpp @@ -254,7 +254,7 @@ int ObInsertStmt::get_value_exprs(ObIArray &value_exprs) const if (OB_ISNULL(param)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("param expr is null", K(ret)); - } else if (ObRawExprUtils::need_column_conv(column_expr->get_result_type(), *param)) { + } else if (ObRawExprUtils::need_column_conv(column_expr->get_result_type(), *param, false)) { param = column_conv_expr; } } diff --git a/src/sql/resolver/dml/ob_merge_stmt.cpp b/src/sql/resolver/dml/ob_merge_stmt.cpp index a7389b706..3bf1bc013 100644 --- a/src/sql/resolver/dml/ob_merge_stmt.cpp +++ b/src/sql/resolver/dml/ob_merge_stmt.cpp @@ -145,7 +145,7 @@ int ObMergeStmt::get_value_exprs(ObIArray &value_exprs) const if (OB_ISNULL(param)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("param expr is null", K(ret)); - } else if (ObRawExprUtils::need_column_conv(column_expr->get_result_type(), *param)) { + } else if (ObRawExprUtils::need_column_conv(column_expr->get_result_type(), *param, false)) { param = column_conv_expr; } } diff --git a/src/sql/resolver/expr/ob_raw_expr_util.cpp b/src/sql/resolver/expr/ob_raw_expr_util.cpp index 4dc3a5b3b..795010c4f 100644 --- a/src/sql/resolver/expr/ob_raw_expr_util.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_util.cpp @@ -3165,7 +3165,7 @@ int ObRawExprUtils::build_generated_column_expr(const ObString &expr_str, expected_type.set_meta(gen_col_schema.get_meta_type()); expected_type.set_accuracy(gen_col_schema.get_accuracy()); expected_type.set_result_flag(calc_column_result_flag(gen_col_schema)); - if (ObRawExprUtils::need_column_conv(expected_type, *expr)) { + if (ObRawExprUtils::need_column_conv(expected_type, *expr, true)) { if (OB_FAIL(build_column_conv_expr(expr_factory, &gen_col_schema, expr, &session_info, &gen_col_schema.get_local_session_var()))) { LOG_WARN("create cast expr failed", K(ret)); @@ -5156,7 +5156,9 @@ int ObRawExprUtils::create_param_expr(ObRawExprFactory &expr_factory, int64_t pa return ret; } -bool ObRawExprUtils::need_column_conv(const ObExprResType &expected_type, const ObRawExpr &expr) +bool ObRawExprUtils::need_column_conv(const ObExprResType &expected_type, + const ObRawExpr &expr, + bool strict_type_check) { int bret = true; if (expected_type.get_type() == expr.get_data_type()) { @@ -5173,35 +5175,18 @@ bool ObRawExprUtils::need_column_conv(const ObExprResType &expected_type, const bret = false; } } - return bret; -} - -bool ObRawExprUtils::need_column_conv(const ColumnItem &column, ObRawExpr &expr) -{ - int bret = true; - if (column.get_expr() != NULL - && (column.get_expr()->is_fulltext_column() - || column.get_expr()->is_spatial_generated_column() - || column.get_expr()->is_multivalue_generated_column() - || column.get_expr()->is_multivalue_generated_array_column())) { - //全文索引的生成列是内部生成的隐藏列，不需要做column convert - bret = false; - } else if (column.get_column_type() != NULL) { - const ObExprResType &column_type = *column.get_column_type(); - if (column_type.get_type() == expr.get_data_type() - && column_type.get_collation_type() == expr.get_collation_type() - && column_type.get_accuracy().get_accuracy() == expr.get_accuracy().get_accuracy()) { - //类型相同，满足不做类型转换的条件 - if (column.is_not_null_for_write() && expr.is_not_null_for_read()) { - //从表达式可以判断两个类型都为not null - //类型相同，并且唯一性约束满足，不需要加column convert检查 - bret = false; - } else if (!column.is_not_null_for_write()) { - //column没有唯一性约束限制 - bret = false; - } else { /*do nothing*/ } - } else { /*do nothing*/ } - } else { /*do nothing*/ } + // the precision of the data stored in datum may exceed that of inferenced type, so column_convert must be added. + // e.g. 1/3 requires storing data with precision beyond inference to ensure that 1/3 * 3 equals 1 and not 0.9999 + if (!bret && strict_type_check) { + if ((expected_type.get_type() == ObNumberType || + expected_type.get_type() == ObNumberFloatType) && + expected_type.get_scale() == NUMBER_SCALE_UNKNOWN_YET && + expected_type.get_precision() == PRECISION_UNKNOWN_YET) { + // do nothing + } else { + bret = true; + } + } return bret; } diff --git a/src/sql/resolver/expr/ob_raw_expr_util.h b/src/sql/resolver/expr/ob_raw_expr_util.h index 5401bd317..1da9a8885 100644 --- a/src/sql/resolver/expr/ob_raw_expr_util.h +++ b/src/sql/resolver/expr/ob_raw_expr_util.h @@ -602,7 +602,6 @@ public: static int replace_qual_param_if_need(ObRawExpr* qual, int64_t qual_idx, ObColumnRefRawExpr *col_expr); - static bool need_column_conv(const ColumnItem &column, ObRawExpr &expr); static int build_pad_expr(ObRawExprFactory &expr_factory, bool is_char, const share::schema::ObColumnSchemaV2 *column_schema, @@ -610,7 +609,9 @@ public: const sql::ObSQLSessionInfo *session_info, const ObLocalSessionVar *local_vars = NULL, int64_t local_var_id = OB_INVALID_INDEX_INT64); - static bool need_column_conv(const ObExprResType &expected_type, const ObRawExpr &expr); + static bool need_column_conv(const ObExprResType &expected_type, + const ObRawExpr &expr, + bool strict_type_check); static bool check_exprs_type_collation_accuracy_equal(const ObRawExpr *expr1, const ObRawExpr *expr2); // 此方法请谨慎使用,会丢失enum类型的 enum_set_values static int build_column_conv_expr(ObRawExprFactory &expr_factory, diff --git a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/hash_distinct.result b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/hash_distinct.result index b7a5a0823..50f836030 100644 --- a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/hash_distinct.result +++ b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/hash_distinct.result @@ -2356,9 +2356,9 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]), + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]), update([t1.c0=column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'06')], [t1.__substr1_16=column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL, - substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'06'), 1, 1))], [t1.__pk_increment=T_HIDDEN_PK]) + column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'06'), 1, 1)))], [t1.__pk_increment=T_HIDDEN_PK]) 1 - output([__values.c0]), filter(nil) values({'W'}, {'u|Qxg6*bV 8Xcx!HQo*VO'}) INSERT INTO t1(c0) VALUES("W"), ('u|Qxg6*bV 8Xcx!HQo*VO') ON DUPLICATE KEY UPDATE c0='06'; @@ -2375,7 +2375,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'5jr'}) INSERT INTO t1(c0) VALUES("5jr"); @@ -2441,7 +2441,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({''}) INSERT INTO t1(c0) VALUES(""); @@ -2474,7 +2474,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'b.6,RL-c[<.ScG<'}) INSERT /*+parallel(5) enable_parallel_dml*/ INTO t1(c0) VALUES('b.6,RL-c[<.ScG<'); @@ -2491,7 +2491,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'-1773558993'}) INSERT INTO t1(c0) VALUES('-1773558993'); @@ -2525,9 +2525,9 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]), + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]), update([t1.c0=column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'u>|!T4e/?W')], [t1.__substr1_16=column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'u>|!T4e/?W'), 1, 1))], [t1.__pk_increment=T_HIDDEN_PK]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'u>|!T4e/?W'), 1, 1)))], [t1.__pk_increment=T_HIDDEN_PK]) 1 - output([__values.c0]), filter(nil) values({'-j&7LN&bUm*drF?'}) INSERT INTO t1(c0) VALUES('-j&7LN&bUm*drF?') ON DUPLICATE KEY UPDATE c0="u>|!T4e/?W"; @@ -2595,9 +2595,9 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]), + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]), update([t1.c0=column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'AS')], [t1.__substr1_16=column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL, - substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'AS'), 1, 1))], [t1.__pk_increment=T_HIDDEN_PK]) + column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'AS'), 1, 1)))], [t1.__pk_increment=T_HIDDEN_PK]) 1 - output([__values.c0]), filter(nil) values({'38814790'}) INSERT INTO t1(c0) VALUES("38814790") ON DUPLICATE KEY UPDATE c0='AS'; @@ -2631,7 +2631,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'1623564365'}) REPLACE INTO t1(c0) VALUES("1623564365"); @@ -2682,9 +2682,9 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]), + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]), update([t1.c0=column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'4W[^L2o')], [t1.__substr1_16=column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'4W[^L2o'), 1, 1))], [t1.__pk_increment=T_HIDDEN_PK]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'4W[^L2o'), 1, 1)))], [t1.__pk_increment=T_HIDDEN_PK]) 1 - output([__values.c0]), filter(nil) values({'#'}, {'-2025228192'}) INSERT INTO t1(c0) VALUES("#"), ('-2025228192') ON DUPLICATE KEY UPDATE c0='4W[^L2o'; @@ -2769,7 +2769,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'W'}) REPLACE INTO t1(c0) VALUES("W"); @@ -2786,9 +2786,9 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]), + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]), update([t1.c0=column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'79355437')], [t1.__substr1_16=column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'79355437'), 1, 1))], [t1.__pk_increment=T_HIDDEN_PK]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'79355437'), 1, 1)))], [t1.__pk_increment=T_HIDDEN_PK]) 1 - output([__values.c0]), filter(nil) values({'6Hk'}) INSERT INTO t1(c0) VALUES('6Hk') ON DUPLICATE KEY UPDATE c0='79355437'; @@ -2805,7 +2805,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'-532544134'}, {'666893151'}) INSERT INTO t1(c0) VALUES("-532544134"), ("666893151"); @@ -2822,9 +2822,10 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]), + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]), update([t1.c0=column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'~qpBxh2{~O4VOlX]>}')], [t1.__substr1_16=column_conv(VARCHAR,utf8mb4_general_ci, - length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'~qpBxh2{~O4VOlX]>}'), 1, 1))], [t1.__pk_increment=T_HIDDEN_PK]) + length:1,NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'~qpBxh2{~O4VOlX]>}'), + 1, 1)))], [t1.__pk_increment=T_HIDDEN_PK]) 1 - output([__values.c0]), filter(nil) values({'eX ad2g'}) INSERT /*+parallel(10) enable_parallel_dml*/ INTO t1(c0) VALUES("eX ad2g") ON DUPLICATE KEY UPDATE c0='~qpBxh2{~O4VOlX]>}'; @@ -2841,7 +2842,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'1030190317'}, {'KU^)'}) REPLACE INTO t1(c0) VALUES("1030190317"), ("KU^)"); @@ -2907,7 +2908,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'\''}, {'-1490343411'}, {'[4/dbA*1XnY~7'}) INSERT /*+parallel(3) enable_parallel_dml*/ INTO t1(c0) VALUES('_W>nY~7') ON DUPLICATE KEY UPDATE c0='ZNTH|'; @@ -3012,9 +3014,9 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]), + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]), update([t1.c0=column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'1mdf')], [t1.__substr1_16=column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL, - substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'1mdf'), 1, 1))], [t1.__pk_increment=T_HIDDEN_PK]) + column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'1mdf'), 1, 1)))], [t1.__pk_increment=T_HIDDEN_PK]) 1 - output([__values.c0]), filter(nil) values({'wv^wo}]0Ye]0._'}, {'B}F?kb10lgWz'}, {'TRUE'}, {'lrzyLtiPj*IrdBu9%-O*Ih<>3'}) INSERT INTO t1(c0) VALUES('wv^wo}]0Ye]0._'), ("B}F?kb10lgWz"), ('TRUE'), ('lrzyLtiPj*IrdBu9%-O*Ih<>3') ON DUPLICATE KEY UPDATE c0="1mdf"; @@ -3031,7 +3033,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'06'}, {'i.3'}) REPLACE INTO t1(c0) VALUES('06'), ("i.3"); @@ -3100,7 +3102,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'Fny~7'}) INSERT INTO t1(c0) VALUES('_w>ny~7'); @@ -3679,7 +3682,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'PO'}) REPLACE INTO t1(c0) VALUES('PO'); @@ -3696,7 +3699,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'-j&7ln&bum*drf?'}, {''}, {'1090711076'}, {'4*1xlr38daa9lgb1-2q'}) INSERT INTO t1(c0) VALUES('-j&7ln&bum*drf?'), (""), ('1090711076'), ('4*1xlr38daa9lgb1-2q'); @@ -3749,9 +3752,9 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]), + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]), update([t1.c0=column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'5JR')], [t1.__substr1_16=column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL, - substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'5JR'), 1, 1))], [t1.__pk_increment=T_HIDDEN_PK]) + column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'5JR'), 1, 1)))], [t1.__pk_increment=T_HIDDEN_PK]) 1 - output([__values.c0]), filter(nil) values({''}) INSERT INTO t1(c0) VALUES("") ON DUPLICATE KEY UPDATE c0='5JR'; @@ -3784,9 +3787,10 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]), + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]), update([t1.c0=column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'-rv\\!dh1m\'gtk+cs-t#xli')], [t1.__substr1_16=column_conv(VARCHAR,utf8mb4_general_ci, - length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'-rv\\!dh1m\'gtk+cs-t#xli'), 1, 1))], [t1.__pk_increment=T_HIDDEN_PK]) + length:1,NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'-rv\\!dh1m\'gtk+cs-t#xli'), + 1, 1)))], [t1.__pk_increment=T_HIDDEN_PK]) 1 - output([__values.c0]), filter(nil) values({'xkwDL3io6,T!'}) INSERT INTO t1(c0) VALUES('xkwDL3io6,T!') ON DUPLICATE KEY UPDATE c0='-rv\\!dh1m''gtk+cs-t#xli'; @@ -3838,7 +3842,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'398204275'}) REPLACE INTO t1(c0) VALUES('398204275'); @@ -3855,7 +3859,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({']a'}) REPLACE INTO t1(c0) VALUES(']a'); @@ -3924,9 +3928,10 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]), + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]), update([t1.c0=column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'w,}kpz)vwu[*.8az38ag4ajqy')], [t1.__substr1_16=column_conv(VARCHAR,utf8mb4_general_ci, - length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'w,}kpz)vwu[*.8az38ag4ajqy'), 1, 1))], [t1.__pk_increment=T_HIDDEN_PK]) + length:1,NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'w,}kpz)vwu[*.8az38ag4ajqy'), + 1, 1)))], [t1.__pk_increment=T_HIDDEN_PK]) 1 - output([__values.c0]), filter(nil) values({'Hg 4QF9^yYv(|n0aC'}) INSERT /*+parallel(8) enable_parallel_dml*/ INTO t1(c0) VALUES("Hg 4QF9^yYv(|n0aC") ON DUPLICATE KEY UPDATE c0="w,}kpz)vwu[*.8az38ag4ajqy"; @@ -3943,7 +3948,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'h*9X7!SX95R?Xh'}, {'xog)!uy#&7e-5vdyqt1}cv'}, {'e\\ DN?Y'}) REPLACE INTO t1(c0) VALUES('h*9X7!SX95R?Xh'), ('xog)!uy#&7e-5vdyqt1}cv'), ('e\\ DN?Y'); @@ -3994,7 +3999,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'ZxJ|lh\\%UKIy]/x&C?Q-+vweYo&'}) REPLACE INTO t1(c0) VALUES("ZxJ|lh\\%UKIy]/x&C?Q-+vweYo&"); @@ -4011,7 +4016,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'o'}) REPLACE INTO t1(c0) VALUES("o"); @@ -4028,7 +4033,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'6MQXB)Er'}) REPLACE INTO t1(c0) VALUES('6MQXB)Er'); @@ -4045,7 +4050,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'[4/DBA*1X')], [t1.__substr1_16=column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL, - substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'v>'), 1, 1))], [t1.__pk_increment=T_HIDDEN_PK]) + column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'v>'), 1, 1)))], [t1.__pk_increment=T_HIDDEN_PK]) 1 - output([__values.c0]), filter(nil) values({''}) INSERT INTO t1(c0) VALUES("") ON DUPLICATE KEY UPDATE c0="v>"; @@ -5000,7 +5005,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'q6h]zjLt)|[?S*C'}, {''}) REPLACE INTO t1(c0) VALUES('q6h]zjLt)|[?S*C'), (""); @@ -5065,7 +5070,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'t'}) REPLACE INTO t1(c0) VALUES('t'); @@ -5132,9 +5137,10 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]), + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]), update([t1.c0=column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'%*5?JXe5x\'--N*w|NM#+~\'yj}w')], [t1.__substr1_16=column_conv(VARCHAR,utf8mb4_general_ci, - length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'%*5?JXe5x\'--N*w|NM#+~\'yj}w'), 1, 1))], [t1.__pk_increment=T_HIDDEN_PK]) + length:1,NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'%*5?JXe5x\'--N*w|NM#+~\'yj}w'), + 1, 1)))], [t1.__pk_increment=T_HIDDEN_PK]) 1 - output([__values.c0]), filter(nil) values({'&a>a'}, {'}8Z'}, {'d|ze^fJh(Y|tof0-w#FVF[{pTf 1Z'}) INSERT /*+parallel(5) enable_parallel_dml*/ INTO t1(c0) VALUES("&a>a"), ('}8Z'), ('d|ze^fJh(Y|tof0-w#FVF[{pTf 1Z') ON DUPLICATE KEY UPDATE c0='%*5?JXe5x''--N*w|NM#+~''yj}w'; @@ -5184,7 +5190,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'1796091423'}) INSERT /*+parallel(1) enable_parallel_dml*/ INTO t1(c0) VALUES('1796091423'); @@ -5235,7 +5241,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({''}, {'u['}) REPLACE INTO t1(c0) VALUES(""), ("u["); @@ -5269,9 +5275,10 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]), + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]), update([t1.c0=column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'upb,KpUrE8ZdvnzOe')], [t1.__substr1_16=column_conv(VARCHAR,utf8mb4_general_ci, - length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'upb,KpUrE8ZdvnzOe'), 1, 1))], [t1.__pk_increment=T_HIDDEN_PK]) + length:1,NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'upb,KpUrE8ZdvnzOe'), 1, + 1)))], [t1.__pk_increment=T_HIDDEN_PK]) 1 - output([__values.c0]), filter(nil) values({'\''}, {'ehOqju3G72*M'}, {'E.p1qkG'}, {'%P%_5\\+1J}\\cjXWYbcD\\Y(]qX8+z]'}, {''}) INSERT /*+parallel(8) enable_parallel_dml*/ INTO t1(c0) VALUES(''''), ('ehOqju3G72*M'), ('E.p1qkG'), ("%P%_5\\+1J}\\cjXWYbcD\\Y(]qX8+z]"), ("") ON DUPLICATE KEY UPDATE c0="upb,KpUrE8ZdvnzOe"; @@ -5507,7 +5514,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'L0'}) REPLACE INTO t1(c0) VALUES('L0'); @@ -5559,7 +5566,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'417065122'}) REPLACE INTO t1(c0) VALUES('417065122'); @@ -5576,7 +5583,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'+'}, {'V5Bd'}, {'mo)!qHyHCC5l\'K'}) INSERT INTO t1(c0) VALUES("+"), ("V5Bd"), ('mo)!qHyHCC5l''K'); @@ -5593,7 +5600,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'G%Dy&j6yjSU{Kk#Szx0Z#S_i4pkLq'}, {''}) REPLACE INTO t1(c0) VALUES('G%Dy&j6yjSU{Kk#Szx0Z#S_i4pkLq'), (""); @@ -5610,7 +5617,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'gNy.{w^bliF#ob4w8Bgg)'}) REPLACE INTO t1(c0) VALUES("gNy.{w^bliF#ob4w8Bgg)"); @@ -5627,7 +5634,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'N'}) REPLACE INTO t1(c0) VALUES("N"); @@ -5744,7 +5751,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'C]'}) REPLACE INTO t1(c0) VALUES("C]"); @@ -5761,9 +5768,9 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]), - update([t1.c0=column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'')], [t1.__substr1_16=column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR, - utf8mb4_general_ci,length:500,NULL,''), 1, 1))], [t1.__pk_increment=T_HIDDEN_PK]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]), + update([t1.c0=column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,'')], [t1.__substr1_16=column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,column_conv(VARCHAR, + utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,''), 1, 1)))], [t1.__pk_increment=T_HIDDEN_PK]) 1 - output([__values.c0]), filter(nil) values({'-1971421495'}) INSERT /*+parallel(6) enable_parallel_dml*/ INTO t1(c0) VALUES("-1971421495") ON DUPLICATE KEY UPDATE c0=''; @@ -5797,7 +5804,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'FtK<%7j6kT2|206T4OaoRlx,]mNN/'}, {'eOtb0c_zm\'a*U%{6g0/%l7BpQmY '}, {''}, {''}, {'xa'}) REPLACE INTO t1(c0) VALUES("FtK<%7j6kT2|206T4OaoRlx,]mNN/"), ("eOtb0c_zm'a*U%{6g0/%l7BpQmY "), (""), (""), ("xa"); @@ -5814,7 +5821,7 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'^.MC2C(*5BCTF^_QUVE+GIH)E'}) REPLACE INTO t1(c0) VALUES('^.MC2C(*5BCTF^_QUVE+GIH)E'); @@ -5882,7 +5889,7 @@ Outputs & filters: columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)}, {i260: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'lXpUQ'}, {'xbo'}, {'W'}, {'y1p3qByYNx.bym)/q4c'}, {'-1750125326'}, {'y'}) REPLACE INTO t1(c0) VALUES('lXpUQ'), ('xbo'), ('W'), ('y1p3qByYNx.bym)/q4c'), ("-1750125326"), ('y'); @@ -5900,7 +5907,7 @@ Outputs & filters: columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)}, {i260: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'s'}) REPLACE INTO t1(c0) VALUES("s"); @@ -5935,7 +5942,7 @@ Outputs & filters: columns([{t1: ({t1: (t1.__pk_increment, t1.c0, t1.__substr1_16)}, {i430: (t1.__substr1_16, t1.__pk_increment, t1.c0)}, {i260: (t1.__substr1_16, t1.__pk_increment, t1.c0)})}]), column_values([T_HIDDEN_PK], [column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0)], [column_conv(VARCHAR,utf8mb4_general_ci,length:1, - NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1))]) + NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:1,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:500,NULL,__values.c0), 1, 1)))]) 1 - output([__values.c0]), filter(nil) values({'rF0Lll5'}) REPLACE INTO t1(c0) VALUES('rF0Lll5'); diff --git a/tools/deploy/mysql_test/test_suite/update/r/mysql/update2.result b/tools/deploy/mysql_test/test_suite/update/r/mysql/update2.result index aae0a93e0..5c6c37a47 100644 --- a/tools/deploy/mysql_test/test_suite/update/r/mysql/update2.result +++ b/tools/deploy/mysql_test/test_suite/update/r/mysql/update2.result @@ -1533,7 +1533,8 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.a, t1.b, t1.__substr5_17)})}]), column_values([T_HIDDEN_PK], [column_conv(INT,PS:(11,0),NULL,__values.a)], [column_conv(VARCHAR,utf8mb4_general_ci,length:10,NULL,__values.b)], [column_conv(VARCHAR, - utf8mb4_general_ci,length:5,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:10,NULL,__values.b), 1, 5))]) + utf8mb4_general_ci,length:5,NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:5,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:10,NULL,__values.b), + 1, 5)))]) 1 - output([__values.a], [__values.b]), filter(nil) values({1, 'abcd1e'}) insert into t1 values ( 1, 'abcd1e'); @@ -1550,7 +1551,8 @@ Outputs & filters: 0 - output(nil), filter(nil) columns([{t1: ({t1: (t1.__pk_increment, t1.a, t1.b, t1.__substr5_17)})}]), column_values([T_HIDDEN_PK], [column_conv(INT,PS:(11,0),NULL,__values.a)], [column_conv(VARCHAR,utf8mb4_general_ci,length:10,NULL,__values.b)], [column_conv(VARCHAR, - utf8mb4_general_ci,length:5,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:10,NULL,__values.b), 1, 5))]) + utf8mb4_general_ci,length:5,NULL,column_conv(VARCHAR,utf8mb4_general_ci,length:5,NULL,substr(column_conv(VARCHAR,utf8mb4_general_ci,length:10,NULL,__values.b), + 1, 5)))]) 1 - output([__values.a], [__values.b]), filter(nil) values({2, 'abcd2e'}) insert into t1 values ( 2, 'abcd2e'); @@ -1601,9 +1603,9 @@ Outputs & filters: 0 - output(nil), filter(nil) table_columns([{t1: ({t1: (t1.__pk_increment, t1.a, t1.b, t1.__substr5_17)}), hash_distinct}]), update([t1.a=column_conv(INT,PS:(11,0),NULL,t2.a)]) - 1 - output([t1.__pk_increment], [t1.a], [t1.b], [substr(t1.b, 1, 5)], [t2.a]), filter(nil), rowset=16 + 1 - output([t1.__pk_increment], [t1.a], [t1.b], [column_conv(VARCHAR,utf8mb4_general_ci,length:5,NULL,substr(t1.b, 1, 5))], [t2.a]), filter(nil), rowset=16 equal_conds([t2.b = t1.b]), other_conds(nil) - 2 - output([t1.__pk_increment], [t1.b], [substr(t1.b, 1, 5)], [t1.a]), filter(nil), rowset=16 + 2 - output([t1.__pk_increment], [t1.b], [column_conv(VARCHAR,utf8mb4_general_ci,length:5,NULL,substr(t1.b, 1, 5))], [t1.a]), filter(nil), rowset=16 access([t1.__pk_increment], [t1.b], [t1.a]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true From 8074d8588a4f5934c8f2695cb70b6d03ec0d6a48 Mon Sep 17 00:00:00 2001 From: haohao022 Date: Fri, 23 Aug 2024 14:56:32 +0000 Subject: [PATCH 207/249] [CP] [to #2024080100104015178] fix: move sysvar AUTOCOMMIT check for arraybinding after sessoin info synchronization --- src/observer/mysql/obmp_stmt_execute.cpp | 22 ++++++++++++++-------- src/observer/mysql/obmp_stmt_execute.h | 4 ++-- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/observer/mysql/obmp_stmt_execute.cpp b/src/observer/mysql/obmp_stmt_execute.cpp index b12a92859..c050944e6 100644 --- a/src/observer/mysql/obmp_stmt_execute.cpp +++ b/src/observer/mysql/obmp_stmt_execute.cpp @@ -281,9 +281,7 @@ int ObMPStmtExecute::init_for_arraybinding(ObIAllocator &alloc) return ret; } -int ObMPStmtExecute::check_param_type_for_arraybinding( - ObSQLSessionInfo *session_info, - ParamTypeInfoArray ¶m_type_infos) +int ObMPStmtExecute::check_precondition_for_arraybinding(const ObSQLSessionInfo &session_info) { int ret = OB_SUCCESS; if (!ObStmt::is_dml_write_stmt(stmt_type_) @@ -292,11 +290,18 @@ int ObMPStmtExecute::check_param_type_for_arraybinding( ret = OB_NOT_SUPPORTED; LOG_WARN("arraybinding only support write dml", K(ret), K(stmt_type_)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "arraybinding got no write dml"); - } else if (session_info->get_local_autocommit()) { + } else if (session_info.get_local_autocommit()) { // read system variable after session info synchronized ret = OB_NOT_SUPPORTED; LOG_WARN("arraybinding must in autocommit off", K(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "arraybinding has autocommit = on"); - } else if (OB_UNLIKELY(param_type_infos.count() <= 0)) { + } + return ret; +} + +int ObMPStmtExecute::check_param_type_for_arraybinding(ParamTypeInfoArray ¶m_type_infos) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(param_type_infos.count() <= 0)) { ret = OB_NOT_SUPPORTED; LOG_WARN("arraybinding must has parameters", K(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "arraybinding has no parameter"); @@ -306,8 +311,7 @@ int ObMPStmtExecute::check_param_type_for_arraybinding( if (type_info.is_basic_type_ || !type_info.is_elem_type_) { ret = OB_NOT_SUPPORTED; LOG_WARN("arraybinding parameter must be anonymous array", K(ret)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, - "arraybinding parameter is not anonymous array"); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "arraybinding parameter is not anonymous array"); } } } @@ -1000,7 +1004,7 @@ int ObMPStmtExecute::request_params(ObSQLSessionInfo *session, } if (OB_SUCC(ret) && is_arraybinding_) { - OZ (check_param_type_for_arraybinding(session, param_type_infos)); + OZ (check_param_type_for_arraybinding(param_type_infos)); } if (OB_SUCC(ret) && (stmt::T_CALL_PROCEDURE == ps_session_info->get_stmt_type() @@ -1964,6 +1968,8 @@ int ObMPStmtExecute::process() LOG_WARN("failed to init flt extra info", K(ret)); } else if (OB_FAIL(session.gen_configs_in_pc_str())) { LOG_WARN("fail to generate configuration string that can influence execution plan", K(ret)); + } else if (is_arraybinding_ && OB_FAIL(check_precondition_for_arraybinding(session))) { + LOG_WARN("precondition for arraybinding is not satisfied", K(ret)); } else { FLTSpanGuard(ps_execute); FLT_SET_TAG(log_trace_id, ObCurTraceId::get_trace_id_str(), diff --git a/src/observer/mysql/obmp_stmt_execute.h b/src/observer/mysql/obmp_stmt_execute.h index 5931a6b6a..1ebe12c54 100644 --- a/src/observer/mysql/obmp_stmt_execute.h +++ b/src/observer/mysql/obmp_stmt_execute.h @@ -264,8 +264,8 @@ private: int init_arraybinding_field(int64_t column_field_cnt, const ColumnsFieldIArray *column_fields); int init_row_for_arraybinding(ObIAllocator &alloc, int64_t array_binding_row_num); - int check_param_type_for_arraybinding( - sql::ObSQLSessionInfo *session_info, sql::ParamTypeInfoArray ¶m_type_infos); + int check_precondition_for_arraybinding(const ObSQLSessionInfo &session_info); + int check_param_type_for_arraybinding(sql::ParamTypeInfoArray ¶m_type_infos); int check_param_value_for_arraybinding(ObObjParam ¶m); int construct_execute_param_for_arraybinding(int64_t pos); void reset_complex_param_memory(ParamStore *params, sql::ObSQLSessionInfo *session_info = nullptr); From 0cc45414b002cd650e73e4f3f39fa61215547e76 Mon Sep 17 00:00:00 2001 From: hanr881 <1741282579@qq.com> Date: Fri, 23 Aug 2024 15:02:39 +0000 Subject: [PATCH 208/249] [CP] to issue<2024080100104012294>:fix sql in pl without questionmark cannot hit plan issue Co-authored-by: yfzcsc --- src/sql/executor/ob_task.cpp | 16 ++++++++++++++-- src/sql/ob_sql.cpp | 19 ++++++++++--------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/sql/executor/ob_task.cpp b/src/sql/executor/ob_task.cpp index 4e3bb62d7..76e5af9a0 100644 --- a/src/sql/executor/ob_task.cpp +++ b/src/sql/executor/ob_task.cpp @@ -307,15 +307,21 @@ OB_DEF_SERIALIZE(ObRemoteTask) int ret = OB_SUCCESS; int64_t tenant_id = OB_INVALID_ID; ParamStore *ps_params = nullptr; + ParamStore empty_param_store; //for serialize ObObjParam' param_meta_ int64_t param_meta_count = 0; if (OB_ISNULL(remote_sql_info_) || OB_ISNULL(session_info_) - || OB_ISNULL(ps_params = remote_sql_info_->ps_params_)) { + || OB_ISNULL(remote_sql_info_->ps_params_)) { ret = OB_NOT_INIT; LOG_WARN("remote task not init", K(ret), K_(remote_sql_info), K_(session_info), K(ps_params)); } else { tenant_id = session_info_->get_effective_tenant_id(); + if (!remote_sql_info_->use_ps_) { + ps_params = &empty_param_store; + } else { + ps_params = remote_sql_info_->ps_params_; + } param_meta_count = ps_params->count(); } LST_DO_CODE(OB_UNIS_ENCODE, @@ -348,13 +354,19 @@ OB_DEF_SERIALIZE_SIZE(ObRemoteTask) { int64_t len = 0; ParamStore *ps_params = nullptr; + ParamStore empty_param_store; int64_t param_meta_count = 0; if (OB_ISNULL(remote_sql_info_) || OB_ISNULL(session_info_) - || OB_ISNULL(ps_params = remote_sql_info_->ps_params_)) { + || OB_ISNULL(remote_sql_info_->ps_params_)) { LOG_WARN_RET(OB_NOT_INIT, "remote task not init", K_(remote_sql_info), K_(session_info), K(ps_params)); } else { int64_t tenant_id = session_info_->get_effective_tenant_id(); + if (!remote_sql_info_->use_ps_) { + ps_params = &empty_param_store; + } else { + ps_params = remote_sql_info_->ps_params_; + } LST_DO_CODE(OB_UNIS_ADD_LEN, tenant_schema_version_, sys_schema_version_, diff --git a/src/sql/ob_sql.cpp b/src/sql/ob_sql.cpp index 5dc3003b4..cacd8fcc3 100644 --- a/src/sql/ob_sql.cpp +++ b/src/sql/ob_sql.cpp @@ -775,8 +775,7 @@ int ObSql::fill_select_result_set(ObResultSet &result_set, ObSqlCtx *context, co } else if (lib::is_oracle_mode() && expr->is_column_ref_expr() && static_cast(expr)->is_xml_column()) { // xmltype is supported, do nothing - } else if (NULL == context->secondary_namespace_ // pl resolve - && NULL == context->session_info_->get_pl_context()) { // pl execute + } else { is_ext_field = true; field.type_.set_collation_type(CS_TYPE_BINARY); field.type_.set_collation_level(CS_LEVEL_IMPLICIT); @@ -787,10 +786,13 @@ int ObSql::fill_select_result_set(ObResultSet &result_set, ObSqlCtx *context, co } else { field.charsetnr_ = static_cast(expr->get_collation_type()); } - if (OB_FAIL(get_composite_type_field_name(*context->schema_guard_, - expr->get_result_type().get_udt_id(), - composite_field_name))) { - LOG_WARN("get record member name fail.", K(ret), K(composite_field_name)); + if (NULL == context->secondary_namespace_ // pl resolve + && NULL == context->session_info_->get_pl_context()) { + if (OB_FAIL(get_composite_type_field_name(*context->schema_guard_, + expr->get_result_type().get_udt_id(), + composite_field_name))) { + LOG_WARN("get record member name fail.", K(ret), K(composite_field_name)); + } } } } @@ -1690,7 +1692,7 @@ int ObSql::handle_sql_execute(const ObString &sql, if (OB_SUCC(ret) && !context.is_text_ps_mode_) { if (OB_FAIL(after_get_plan(pc_ctx, *session, result.get_physical_plan(), - result.get_is_from_plan_cache(), ¶ms, pc_ctx.exec_ctx_.get_min_cluster_version()))) { + result.get_is_from_plan_cache(), (mode == PC_PS_MODE || mode == PC_PL_MODE) ? ¶ms : nullptr, pc_ctx.exec_ctx_.get_min_cluster_version()))) { LOG_WARN("fail to handle after get plan", K(ret)); } } @@ -1767,7 +1769,7 @@ int ObSql::handle_pl_execute(const ObString &sql, LOG_WARN("failed to set timeout for pl", K(ret)); } else if (OB_FAIL(session.store_query_string(sql))) { LOG_WARN("store query string fail", K(ret)); - } else if (OB_FAIL(handle_sql_execute(sql, context, result, params, PC_PL_MODE))) { + } else if (OB_FAIL(handle_sql_execute(sql, context, result, params, is_prepare_protocol ? PC_PL_MODE : PC_TEXT_MODE))) { LOG_WARN("failed to handle sql execute", K(ret)); } else { result.get_session().set_exec_min_cluster_version(); @@ -4848,7 +4850,6 @@ int ObSql::after_get_plan(ObPlanCacheCtx &pc_ctx, pctx->get_remote_sql_info().ps_param_cnt_ = static_cast(param_store.count()); } else { //没有进plan cache，并且是文本协议，在远端再走一次文本解析 - param_store.reset(); //走文本协议不需要携带参数，清空掉 pctx->get_remote_sql_info().use_ps_ = false; pctx->get_remote_sql_info().is_batched_stmt_ = pc_ctx.sql_ctx_.multi_stmt_item_.is_batched_multi_stmt(); From 9f2fa5d3bcdd357004631b1dd37c71798912d9f8 Mon Sep 17 00:00:00 2001 From: chinaxing Date: Fri, 23 Aug 2024 15:09:00 +0000 Subject: [PATCH 209/249] [CP] allow pl implicit savepoint create on txn-route temp node --- src/sql/ob_sql_trans_control.cpp | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/sql/ob_sql_trans_control.cpp b/src/sql/ob_sql_trans_control.cpp index 05b6fe573..f1624a1b6 100644 --- a/src/sql/ob_sql_trans_control.cpp +++ b/src/sql/ob_sql_trans_control.cpp @@ -942,6 +942,16 @@ int ObSqlTransControl::stmt_setup_savepoint_(ObSQLSessionInfo *session, K(session->get_txn_free_route_ctx()), KPC(session)); \ } +#define CHECK_EXPLICIT_SAVEPOINT_TXN_FREE_ROUTE_ALLOWED(savepoint) \ + if (OB_SUCC(ret) && !session->is_inner() && session->is_txn_free_route_temp()) { \ + if (pl::PL_IMPLICIT_SAVEPOINT != savepoint) { \ + ret = OB_TRANS_FREE_ROUTE_NOT_SUPPORTED; \ + LOG_WARN("create savepoint is not allowed executed on txn tmp node", K(ret), \ + K(savepoint), K(session->get_txn_free_route_ctx()), KPC(session)); \ + } \ + } + + #define CHECK_DEFAULT_SAVEPOINTNAME_ALLOWED(sp_name) \ if (OB_SUCC(ret) && (sp_name == DBLINK_DEFAULT_SAVEPOINT || sp_name == PL_DBLINK_DEFAULT_SAVEPOINT)) { \ ret = OB_ERR_INVALID_CHARACTER_STRING; \ @@ -957,7 +967,7 @@ int ObSqlTransControl::create_savepoint(ObExecContext &exec_ctx, ObTransService *txs = NULL; CK (OB_NOT_NULL(session)); CHECK_SESSION (session); - CHECK_TXN_FREE_ROUTE_ALLOWED(); + CHECK_EXPLICIT_SAVEPOINT_TXN_FREE_ROUTE_ALLOWED(sp_name); OZ (get_tx_service(session, txs)); OZ (acquire_tx_if_need_(txs, *session)); // for dblink trans @@ -1106,7 +1116,7 @@ int ObSqlTransControl::rollback_savepoint(ObExecContext &exec_ctx, CK (OB_NOT_NULL(session), OB_NOT_NULL(plan_ctx)); CHECK_SESSION (session); - CHECK_TXN_FREE_ROUTE_ALLOWED(); + CHECK_EXPLICIT_SAVEPOINT_TXN_FREE_ROUTE_ALLOWED(sp_name); OZ (get_tx_service(session, txs)); OZ (acquire_tx_if_need_(txs, *session)); OX (stmt_expire_ts = get_stmt_expire_ts(plan_ctx, *session)); @@ -1169,7 +1179,7 @@ int ObSqlTransControl::release_savepoint(ObExecContext &exec_ctx, ObTransService *txs = NULL; CK (OB_NOT_NULL(session)); CHECK_SESSION (session); - CHECK_TXN_FREE_ROUTE_ALLOWED(); + CHECK_EXPLICIT_SAVEPOINT_TXN_FREE_ROUTE_ALLOWED(sp_name); OZ (get_tx_service(session, txs), *session); OZ (acquire_tx_if_need_(txs, *session)); bool start_hook = false; From a39c058fbc821d7ef407e7e883c9beef99a7eaf2 Mon Sep 17 00:00:00 2001 From: obdev Date: Fri, 23 Aug 2024 17:18:39 +0000 Subject: [PATCH 210/249] Fix copy major of rebuild when restore --- .../high_availability/ob_ls_migration.cpp | 2 +- .../high_availability/ob_storage_ha_dag.cpp | 9 +++- .../high_availability/ob_storage_ha_dag.h | 2 + .../ob_storage_ha_tablet_builder.cpp | 29 ++++++++++++- .../ob_storage_ha_tablet_builder.h | 5 +++ .../ob_tablet_group_restore.cpp | 1 + src/storage/ls/ob_ls_tablet_service.cpp | 4 +- src/storage/tablet/ob_tablet.cpp | 43 ++++++++++++++++++- src/storage/tablet/ob_tablet.h | 1 + src/storage/tablet/ob_tablet_table_store.cpp | 7 +++ 10 files changed, 98 insertions(+), 5 deletions(-) diff --git a/src/storage/high_availability/ob_ls_migration.cpp b/src/storage/high_availability/ob_ls_migration.cpp index 5bac1cb7e..32f39e176 100644 --- a/src/storage/high_availability/ob_ls_migration.cpp +++ b/src/storage/high_availability/ob_ls_migration.cpp @@ -3084,7 +3084,7 @@ int ObTabletMigrationTask::check_need_copy_sstable_( LOG_WARN("check need copy sstable get invlaid argument", K(ret), K(table_key)); } else if (OB_FAIL(ctx_->ha_table_info_mgr_.get_table_info(copy_tablet_ctx_->tablet_id_, table_key, copy_table_info))) { LOG_WARN("failed to get table info", K(ret), KPC(copy_tablet_ctx_), K(table_key)); - } else if (OB_FAIL(ObStorageHATaskUtils::check_need_copy_sstable(*copy_table_info, copy_tablet_ctx_->tablet_handle_, need_copy))) { + } else if (OB_FAIL(ObStorageHATaskUtils::check_need_copy_sstable(*copy_table_info, false /* is_restore */, copy_tablet_ctx_->tablet_handle_, need_copy))) { LOG_WARN("failed to check need copy sstable", K(ret), KPC(copy_tablet_ctx_), K(table_key)); } return ret; diff --git a/src/storage/high_availability/ob_storage_ha_dag.cpp b/src/storage/high_availability/ob_storage_ha_dag.cpp index 9a86eb1b0..cf5f4ce98 100644 --- a/src/storage/high_availability/ob_storage_ha_dag.cpp +++ b/src/storage/high_availability/ob_storage_ha_dag.cpp @@ -733,6 +733,7 @@ int ObHATabletGroupMgr::get_tablet_group_ctx( /******************ObStorageHATaskUtils*********************/ int ObStorageHATaskUtils::check_need_copy_sstable( const ObMigrationSSTableParam ¶m, + const bool &is_restore, ObTabletHandle &tablet_handle, bool &need_copy) { @@ -742,7 +743,7 @@ int ObStorageHATaskUtils::check_need_copy_sstable( ret = OB_INVALID_ARGUMENT; LOG_WARN("check need copy sstable get invalid argument", K(ret), K(param)); } else if (param.table_key_.is_major_sstable()) { - if (OB_FAIL(check_major_sstable_need_copy_(param, tablet_handle, need_copy))) { + if (OB_FAIL(check_major_sstable_need_copy_(param, is_restore, tablet_handle, need_copy))) { LOG_WARN("failed to check major sstable need copy", K(ret), K(param), K(tablet_handle)); } } else if (param.table_key_.is_minor_sstable()) { @@ -764,6 +765,7 @@ int ObStorageHATaskUtils::check_need_copy_sstable( int ObStorageHATaskUtils::check_major_sstable_need_copy_( const ObMigrationSSTableParam ¶m, + const bool &is_restore, ObTabletHandle &tablet_handle, bool &need_copy) { @@ -793,6 +795,11 @@ int ObStorageHATaskUtils::check_major_sstable_need_copy_( LOG_WARN("failed to get sstable meta handle", K(ret)); } else if (OB_FAIL(ObSSTableMetaChecker::check_sstable_meta(param, sst_meta_hdl.get_sstable_meta()))) { LOG_WARN("failed to check sstable meta", K(ret), K(param), K(sstable_wrapper)); + } else if (!is_restore && sst_meta_hdl.get_sstable_meta().get_basic_meta().table_backup_flag_.has_backup() + && param.basic_meta_.table_backup_flag_.has_no_backup()) { + // when migration or rebuild, if dst sstable has backup macro and the corresponding src sstable has no backup macro, need copy + need_copy = true; + FLOG_INFO("dst sstable has backup macro and src sstable has no backup macro, need copy", K(need_copy), K(sst_meta_hdl), K(param.basic_meta_)); } else { need_copy = false; } diff --git a/src/storage/high_availability/ob_storage_ha_dag.h b/src/storage/high_availability/ob_storage_ha_dag.h index 5af011892..f2564c2a3 100644 --- a/src/storage/high_availability/ob_storage_ha_dag.h +++ b/src/storage/high_availability/ob_storage_ha_dag.h @@ -193,12 +193,14 @@ class ObStorageHATaskUtils public: static int check_need_copy_sstable( const ObMigrationSSTableParam ¶m, + const bool &is_restore, ObTabletHandle &tablet_handle, bool &need_copy); private: static int check_major_sstable_need_copy_( const ObMigrationSSTableParam ¶m, + const bool &is_restore, ObTabletHandle &tablet_handle, bool &need_copy); diff --git a/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp b/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp index 578e84222..a4d7b95af 100644 --- a/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp +++ b/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp @@ -937,8 +937,17 @@ int ObStorageHATabletsBuilder::get_major_sstable_max_snapshot_( } else if (major_sstable_array.count() > 0 && OB_FAIL(major_sstable_array.get_all_table_wrappers(sstables))) { LOG_WARN("failed to get all tables", K(ret), K(param_)); } else { + ObSSTableMetaHandle sst_meta_hdl; + MajorSSTableSnapshotVersionCmp cmp; + + // step 1: sort major sstables by snapshot version + lib::ob_sort(sstables.begin(), sstables.end(), cmp); + + // step 2: get the major that has maximun snapshot version + // and all major snapshot version lower than it has no backup for (int64_t i = 0; OB_SUCC(ret) && i < sstables.count(); ++i) { - const ObITable *table = sstables.at(i).get_sstable(); + ObITable *table = sstables.at(i).get_sstable(); + sst_meta_hdl.reset(); if (OB_ISNULL(table)) { ret = OB_ERR_UNEXPECTED; @@ -946,6 +955,11 @@ int ObStorageHATabletsBuilder::get_major_sstable_max_snapshot_( } else if (!table->is_major_sstable()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("sstable type is unexpected", K(ret), KP(table), K(param_)); + } else if (OB_FAIL(static_cast(table)->get_meta(sst_meta_hdl))) { + LOG_WARN("failed to get sstable meta handle", K(ret), K(table)); + } else if (sst_meta_hdl.get_sstable_meta().get_basic_meta().table_backup_flag_.has_backup()) { + // stop at first major sstable that has backup + break; } else { max_snapshot_version = std::max(max_snapshot_version, table->get_key().get_snapshot_version()); } @@ -1313,6 +1327,19 @@ int ObStorageHATabletsBuilder::hold_local_tablet_( return ret; } +bool ObStorageHATabletsBuilder::MajorSSTableSnapshotVersionCmp::operator()(const ObSSTableWrapper &lhs, const ObSSTableWrapper &rhs) const +{ + int ret = OB_SUCCESS; + int result = false; + if (!lhs.is_valid() || !rhs.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("invalid sstable wrapper", K(ret), K(lhs), K(rhs)); + } else { + result = lhs.get_sstable()->get_snapshot_version() < rhs.get_sstable()->get_snapshot_version(); + } + return result; +} + /******************ObStorageHATabletTableInfoMgr*********************/ ObStorageHATableInfoMgr::ObStorageHATabletTableInfoMgr::ObStorageHATabletTableInfoMgr() : is_inited_(false), diff --git a/src/storage/high_availability/ob_storage_ha_tablet_builder.h b/src/storage/high_availability/ob_storage_ha_tablet_builder.h index 74b03739c..3569221cc 100644 --- a/src/storage/high_availability/ob_storage_ha_tablet_builder.h +++ b/src/storage/high_availability/ob_storage_ha_tablet_builder.h @@ -154,6 +154,11 @@ private: int hold_local_tablet_( common::ObIArray &tablet_handle_array); private: + struct MajorSSTableSnapshotVersionCmp + { + bool operator()(const ObSSTableWrapper &lhs, const ObSSTableWrapper &rhs) const; + }; + bool is_inited_; ObStorageHATabletsBuilderParam param_; DISALLOW_COPY_AND_ASSIGN(ObStorageHATabletsBuilder); diff --git a/src/storage/high_availability/ob_tablet_group_restore.cpp b/src/storage/high_availability/ob_tablet_group_restore.cpp index dbc8d40dd..4af4116ae 100644 --- a/src/storage/high_availability/ob_tablet_group_restore.cpp +++ b/src/storage/high_availability/ob_tablet_group_restore.cpp @@ -2643,6 +2643,7 @@ int ObTabletRestoreTask::check_need_copy_sstable_( LOG_WARN("failed to get table info", K(ret), KPC(tablet_restore_ctx_), K(table_key)); } else if (!ObTabletRestoreAction::is_restore_replace_remote_sstable(restore_action)) { if (OB_FAIL(ObStorageHATaskUtils::check_need_copy_sstable(*copy_table_info, + tablet_restore_ctx_->is_leader_ /* is_restore */, tablet_restore_ctx_->tablet_handle_, need_copy))) { LOG_WARN("failed to check need copy sstable", K(ret), KPC(tablet_restore_ctx_), K(table_key)); diff --git a/src/storage/ls/ob_ls_tablet_service.cpp b/src/storage/ls/ob_ls_tablet_service.cpp index 25f0fffb1..ec7da2599 100644 --- a/src/storage/ls/ob_ls_tablet_service.cpp +++ b/src/storage/ls/ob_ls_tablet_service.cpp @@ -1682,7 +1682,9 @@ int ObLSTabletService::update_tablet_ha_data_status( } else if (OB_FAIL(tablet->tablet_meta_.ha_status_.set_data_status(data_status))) { LOG_WARN("failed to set data status", K(ret), KPC(tablet), K(data_status)); } else { - if (OB_FAIL(ObTabletPersister::persist_and_transform_tablet(*tablet, new_tablet_handle))) { + if (OB_FAIL(tablet->check_valid())) { + LOG_WARN("failed to check tablet valid", K(ret), K(data_status), KPC(tablet)); + } else if (OB_FAIL(ObTabletPersister::persist_and_transform_tablet(*tablet, new_tablet_handle))) { LOG_WARN("fail to persist and transform tablet", K(ret), KPC(tablet), K(new_tablet_handle)); } else if (FALSE_IT(time_guard.click("Persist"))) { } else if (FALSE_IT(disk_addr = new_tablet_handle.get_obj()->tablet_addr_)) { diff --git a/src/storage/tablet/ob_tablet.cpp b/src/storage/tablet/ob_tablet.cpp index b8b3d9686..e7d885ce0 100644 --- a/src/storage/tablet/ob_tablet.cpp +++ b/src/storage/tablet/ob_tablet.cpp @@ -1652,7 +1652,7 @@ int ObTablet::check_sstable_column_checksum() const ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected error, table is nullptr", K(ret), KPC(table)); } else { - ObSSTable *cur = reinterpret_cast(table); + ObSSTable *cur = static_cast(table); ObSSTableMetaHandle meta_handle; if (OB_FAIL(cur->get_meta(meta_handle))) { LOG_WARN("fail to get sstable meta", K(ret), KPC(cur), KPC(this)); @@ -1670,6 +1670,45 @@ int ObTablet::check_sstable_column_checksum() const return ret; } +int ObTablet::check_no_backup_data() const +{ + int ret = OB_SUCCESS; + ObTableStoreIterator iter; + + if (!tablet_meta_.ha_status_.is_none()) { + // skip + } else if (OB_FAIL(inner_get_all_sstables(iter, true /* need_unpack */))) { + LOG_WARN("fail to get all sstables", K(ret)); + } else { + ObITable *table = nullptr; + while (OB_SUCC(ret)) { + if (OB_FAIL(iter.get_next(table))) { + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + break; + } else { + LOG_WARN("failed to get next table", K(ret), KPC(this)); + } + } else if (OB_ISNULL(table) || OB_UNLIKELY(!table->is_sstable())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error, table is nullptr", K(ret), KPC(table)); + } else { + ObSSTable *cur = static_cast(table); + ObSSTableMetaHandle meta_handle; + if (OB_FAIL(cur->get_meta(meta_handle))) { + LOG_WARN("fail to get sstable meta", K(ret), KPC(cur), KPC(this)); + } else if (OB_UNLIKELY(meta_handle.get_sstable_meta().get_basic_meta().table_backup_flag_.has_backup())) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("The sstable has backup data, but restore status is full.", + K(ret), KPC(cur), KPC(this), K(tablet_meta_.ha_status_)); + } + } + } + } + + return ret; +} + int ObTablet::serialize(char *buf, const int64_t len, int64_t &pos, const ObSArray &meta_arr) const { int ret = OB_SUCCESS; @@ -7680,6 +7719,8 @@ int ObTablet::inner_check_valid(const bool ignore_ha_status) const LOG_WARN("failed to check medium list", K(ret), KPC(this)); } else if (OB_FAIL(check_sstable_column_checksum())) { LOG_WARN("failed to check sstable column checksum", K(ret), KPC(this)); + } else if (OB_FAIL(check_no_backup_data())) { + LOG_WARN("failed to check no backup data", K(ret), KPC(this)); } return ret; } diff --git a/src/storage/tablet/ob_tablet.h b/src/storage/tablet/ob_tablet.h index b15a72e95..2418aa34c 100644 --- a/src/storage/tablet/ob_tablet.h +++ b/src/storage/tablet/ob_tablet.h @@ -714,6 +714,7 @@ private: int check_medium_list() const; int check_sstable_column_checksum() const; + int check_no_backup_data() const; int get_finish_medium_scn(int64_t &finish_medium_scn) const; int inner_get_mds_sstables(ObTableStoreIterator &table_store_iter) const; diff --git a/src/storage/tablet/ob_tablet_table_store.cpp b/src/storage/tablet/ob_tablet_table_store.cpp index ef56e9395..abcafde98 100644 --- a/src/storage/tablet/ob_tablet_table_store.cpp +++ b/src/storage/tablet/ob_tablet_table_store.cpp @@ -1402,6 +1402,13 @@ int ObTabletTableStore::check_and_build_new_major_tables( new_sst_meta_hdl.get_sstable_meta(), old_sst_meta_hdl.get_sstable_meta()))) { LOG_WARN("failed to check sstable meta", K(ret), KPC(new_sstable), KPC(old_sstable)); } else { + if (new_sst_meta_hdl.get_sstable_meta().get_basic_meta().table_backup_flag_.has_no_backup() + && old_sst_meta_hdl.get_sstable_meta().get_basic_meta().table_backup_flag_.has_backup()) { + // if new sstable has no backup macro block and old sstable has backup macro block + // replace the old sstable with the new one + FLOG_INFO("replace major sstable with new one", KPC(new_sstable), KPC(old_sstable)); + major_tables.at(j) = new_table; + } need_add = false; } } From 26c0e1f1611ce6f9ebf8d0cb6a6112518dbccf7b Mon Sep 17 00:00:00 2001 From: chinaxing Date: Fri, 23 Aug 2024 17:24:42 +0000 Subject: [PATCH 211/249] [CP] support parallel dml modify same row concurrently Co-authored-by: Handora --- mittest/mtlenv/storage/test_memtable_v2.cpp | 7 +- src/sql/code_generator/ob_dml_cg_service.cpp | 96 +++++++++++++++++++ src/sql/code_generator/ob_dml_cg_service.h | 10 ++ src/sql/das/ob_das_dml_ctx_define.h | 4 +- src/sql/engine/dml/ob_dml_service.cpp | 3 + src/storage/CMakeLists.txt | 1 + src/storage/memtable/mvcc/ob_mvcc_acc_ctx.cpp | 39 ++++++++ src/storage/memtable/mvcc/ob_mvcc_acc_ctx.h | 3 +- src/storage/memtable/mvcc/ob_mvcc_define.h | 7 ++ src/storage/memtable/mvcc/ob_mvcc_engine.cpp | 8 +- src/storage/memtable/mvcc/ob_mvcc_row.cpp | 50 ++++++++-- src/storage/memtable/mvcc/ob_mvcc_row.h | 13 ++- src/storage/memtable/mvcc/ob_mvcc_trans_ctx.h | 1 + src/storage/memtable/ob_concurrent_control.h | 9 +- src/storage/memtable/ob_memtable.cpp | 26 ++++- src/storage/memtable/ob_memtable_context.h | 2 +- src/storage/tx/ob_trans_define_v4.h | 3 +- .../storage/memtable/test_memtable_basic.cpp | 7 +- 18 files changed, 252 insertions(+), 37 deletions(-) create mode 100644 src/storage/memtable/mvcc/ob_mvcc_acc_ctx.cpp diff --git a/mittest/mtlenv/storage/test_memtable_v2.cpp b/mittest/mtlenv/storage/test_memtable_v2.cpp index 083236b8d..1c64311f1 100644 --- a/mittest/mtlenv/storage/test_memtable_v2.cpp +++ b/mittest/mtlenv/storage/test_memtable_v2.cpp @@ -70,9 +70,10 @@ void ObMemtableCtx::free_mvcc_row_callback(ObITransCallback *cb) } } -int ObMvccRow::check_double_insert_(const share::SCN , - ObMvccTransNode &, - ObMvccTransNode *) +int ObMvccRow::mvcc_sanity_check_(const share::SCN , + const concurrent_control::ObWriteFlag , + ObMvccTransNode &, + ObMvccTransNode *) { return OB_SUCCESS; } diff --git a/src/sql/code_generator/ob_dml_cg_service.cpp b/src/sql/code_generator/ob_dml_cg_service.cpp index 0c0a05b17..26955a6e9 100644 --- a/src/sql/code_generator/ob_dml_cg_service.cpp +++ b/src/sql/code_generator/ob_dml_cg_service.cpp @@ -351,6 +351,91 @@ int ObDmlCgService::generate_update_ctdef(ObLogDelUpd &op, return ret; } +int ObDmlCgService::check_is_update_uk(ObLogDelUpd &op, + const IndexDMLInfo &index_dml_info, + ObIArray &update_cids, + ObDASUpdCtDef &das_upd_ctdef) +{ + int ret = OB_SUCCESS; + ObSchemaGetterGuard *schema_guard = NULL; + const ObTableSchema *table_schema = NULL; + ObSEArray rowkey_cids; + bool is_update_uk = false; + + ObLogPlan *log_plan = op.get_plan(); + if (OB_ISNULL(log_plan) || + OB_ISNULL(schema_guard = log_plan->get_optimizer_context().get_schema_guard())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected status", K(ret)); + } else if (OB_FAIL(schema_guard->get_table_schema(MTL_ID(), index_dml_info.ref_table_id_, table_schema))) { + LOG_WARN("fail to get unique index schema", K(ret), K(index_dml_info)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (index_dml_info.is_primary_index_) { + // 主表 + if (OB_FAIL(table_schema->get_rowkey_column_ids(rowkey_cids))) { + LOG_WARN("fail to get rowkey cids", K(ret)); + } + } else if (!table_schema->is_global_unique_index_table()) { + // 非global unique index，不需要检查 + } else if (OB_FAIL(table_schema->get_rowkey_column_ids(rowkey_cids))) { + LOG_WARN("fail to get rowkey cids", K(ret)); + } + + if (OB_SUCC(ret)) { + for (int64_t i = 0; OB_SUCC(ret) && !is_update_uk && i < update_cids.count(); i++) { + if (has_exist_in_array(rowkey_cids, update_cids.at(i))) { + is_update_uk = true; + } + } + } + + if (OB_SUCC(ret)) { + das_upd_ctdef.is_update_uk_ = is_update_uk; + } + + return ret; +} + +int ObDmlCgService::check_is_update_local_unique_index(ObLogDelUpd &op, + uint64_t index_tid, + ObIArray &update_cids, + ObDASUpdCtDef &das_upd_ctdef) +{ + int ret = OB_SUCCESS; + ObSchemaGetterGuard *schema_guard = NULL; + const ObTableSchema *unique_index_schema = NULL; + ObSEArray rowkey_cids; + ObLogPlan *log_plan = op.get_plan(); + + bool is_update_uk = false; + if (OB_ISNULL(log_plan) || + OB_ISNULL(schema_guard = log_plan->get_optimizer_context().get_schema_guard())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected status", K(ret)); + } else if (OB_FAIL(schema_guard->get_table_schema(MTL_ID(), index_tid, unique_index_schema))) { + LOG_WARN("fail to get unique index schema", K(ret), K(index_tid)); + } else if (OB_ISNULL(unique_index_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (!unique_index_schema->is_local_unique_index_table()) { + // not need check it + } else if (OB_FAIL(unique_index_schema->get_rowkey_column_ids(rowkey_cids))) { + LOG_WARN("fail to get rowkey column_ids", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && !is_update_uk && i < update_cids.count(); i++) { + if (has_exist_in_array(rowkey_cids, update_cids.at(i))) { + is_update_uk = true; + } + } + } + + if (OB_SUCC(ret)) { + das_upd_ctdef.is_update_uk_ = is_update_uk; + } + return ret; +} int ObDmlCgService::generate_update_ctdef(ObLogDelUpd &op, const IndexDMLInfo &index_dml_info, ObUpdCtDef &upd_ctdef) @@ -359,6 +444,7 @@ int ObDmlCgService::generate_update_ctdef(ObLogDelUpd &op, ObSEArray old_row; ObSEArray new_row; ObSEArray full_row; + bool is_update_uk = false; const ObAssignments &assigns = index_dml_info.assignments_; bool gen_expand_ctdef = false; LOG_TRACE("begin to generate update ctdef", K(index_dml_info)); @@ -402,6 +488,11 @@ int ObDmlCgService::generate_update_ctdef(ObLogDelUpd &op, new_row, full_row))) { LOG_WARN("generate das update ctdef failed", K(ret)); + } else if (OB_FAIL(check_is_update_uk(op, + index_dml_info, + upd_ctdef.dupd_ctdef_.updated_column_ids_, + upd_ctdef.dupd_ctdef_))) { + LOG_WARN("fail to check is update uk", K(ret), K(upd_ctdef.dupd_ctdef_)); } else if (OB_FAIL(generate_related_upd_ctdef(op, index_dml_info.related_index_ids_, index_dml_info, @@ -2189,6 +2280,11 @@ int ObDmlCgService::generate_related_upd_ctdef(ObLogDelUpd &op, new_row, full_row))) { LOG_WARN("generate das ins ctdef failed", K(ret)); + } else if (OB_FAIL(check_is_update_local_unique_index(op, + related_tid, + related_ctdef->updated_column_ids_, + *related_ctdef))) { + LOG_WARN("fail to check is update uk", K(ret), K(related_tid)); } else if (related_ctdef->updated_column_ids_.empty()) { //ignore invalid update ctdef } else if (OB_FAIL(upd_ctdefs.push_back(related_ctdef))) { diff --git a/src/sql/code_generator/ob_dml_cg_service.h b/src/sql/code_generator/ob_dml_cg_service.h index 151dd3e4d..40286391b 100644 --- a/src/sql/code_generator/ob_dml_cg_service.h +++ b/src/sql/code_generator/ob_dml_cg_service.h @@ -51,6 +51,16 @@ public: const IndexDMLInfo &index_dml_info, ObUpdCtDef &upd_ctdef); + int check_is_update_local_unique_index(ObLogDelUpd &op, + uint64_t index_tid, + ObIArray &update_cids, + ObDASUpdCtDef &das_upd_ctdef); + + int check_is_update_uk(ObLogDelUpd &op, + const IndexDMLInfo &index_dml_info, + ObIArray &update_cids, + ObDASUpdCtDef &das_upd_ctdef); + int generate_lock_ctdef(ObLogForUpdate &op, const IndexDMLInfo &index_dml_info, ObLockCtDef *&lock_ctdef); diff --git a/src/sql/das/ob_das_dml_ctx_define.h b/src/sql/das/ob_das_dml_ctx_define.h index e57604555..17e3a5448 100644 --- a/src/sql/das/ob_das_dml_ctx_define.h +++ b/src/sql/das/ob_das_dml_ctx_define.h @@ -83,7 +83,9 @@ public: uint64_t is_insert_up_ : 1; uint64_t is_table_api_ : 1; uint64_t is_access_mlog_as_master_table_ : 1; - uint64_t reserved_ : 58; + uint64_t is_update_partition_key_ : 1; + uint64_t is_update_uk_ : 1; + uint64_t reserved_ : 56; }; }; protected: diff --git a/src/sql/engine/dml/ob_dml_service.cpp b/src/sql/engine/dml/ob_dml_service.cpp index 85019b46d..466c970c0 100644 --- a/src/sql/engine/dml/ob_dml_service.cpp +++ b/src/sql/engine/dml/ob_dml_service.cpp @@ -1223,6 +1223,9 @@ int ObDMLService::init_dml_param(const ObDASDMLBaseCtDef &base_ctdef, if (base_rtdef.is_for_foreign_key_check_) { dml_param.write_flag_.set_check_row_locked(); } + if (base_ctdef.is_update_uk_) { + dml_param.write_flag_.set_update_uk(); + } return ret; } diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 6d89f6a7e..f54ea23ef 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -711,6 +711,7 @@ ob_set_subtarget(ob_storage memtable ob_set_subtarget(ob_storage memtable_mvcc memtable/mvcc/ob_multi_version_iterator.cpp + memtable/mvcc/ob_mvcc_acc_ctx.cpp memtable/mvcc/ob_mvcc_ctx.cpp memtable/mvcc/ob_mvcc_engine.cpp memtable/mvcc/ob_mvcc_iterator.cpp diff --git a/src/storage/memtable/mvcc/ob_mvcc_acc_ctx.cpp b/src/storage/memtable/mvcc/ob_mvcc_acc_ctx.cpp new file mode 100644 index 000000000..26bf4f0d3 --- /dev/null +++ b/src/storage/memtable/mvcc/ob_mvcc_acc_ctx.cpp @@ -0,0 +1,39 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "storage/memtable/mvcc/ob_mvcc_acc_ctx.h" +#include "storage/memtable/ob_memtable_context.h" +namespace oceanbase +{ +namespace memtable +{ +int ObMvccAccessCtx::get_write_seq(transaction::ObTxSEQ &seq) const +{ + int ret = OB_SUCCESS; + // for update uk or pk, set branch part to 0, in orer to let tx-callback fall into single list + if (tx_scn_.support_branch() && write_flag_.is_update_uk()) { + const int branch = tx_scn_.get_branch(); + if (branch == 0) { + seq = tx_scn_; + } else if (OB_UNLIKELY(ObTxDesc::is_alloced_branch_id(branch))) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "external branch not support concurrent update uk / pk", K(ret), KPC(this)); + } else { + seq = ObTxSEQ(tx_scn_.get_seq(), 0); + } + } else { + seq = tx_scn_; + } + return ret; +} +} // memtable +} // oceanbase diff --git a/src/storage/memtable/mvcc/ob_mvcc_acc_ctx.h b/src/storage/memtable/mvcc/ob_mvcc_acc_ctx.h index 97f81a169..18e8ac9fc 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_acc_ctx.h +++ b/src/storage/memtable/mvcc/ob_mvcc_acc_ctx.h @@ -233,6 +233,7 @@ public: } return expire_ts; } + int get_write_seq(transaction::ObTxSEQ &seq) const; TO_STRING_KV(K_(type), K_(abs_lock_timeout_ts), K_(tx_lock_timeout_us), @@ -272,7 +273,7 @@ public: // NOTE: those field should only be accessed by txn relative routine transaction::ObTxDesc *tx_desc_; // the txn descriptor transaction::ObPartTransCtx *tx_ctx_; // the txn context ObMemtableCtx *mem_ctx_; // memtable-ctx - transaction::ObTxSEQ tx_scn_; // the change's number of this modify + transaction::ObTxSEQ tx_scn_; // the change's number of this modify concurrent_control::ObWriteFlag write_flag_; // the write flag of the write process // this was used for runtime metric diff --git a/src/storage/memtable/mvcc/ob_mvcc_define.h b/src/storage/memtable/mvcc/ob_mvcc_define.h index f0f42aa5c..a42e83bc3 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_define.h +++ b/src/storage/memtable/mvcc/ob_mvcc_define.h @@ -44,6 +44,8 @@ struct ObTxNodeArg int64_t memstore_version_; // seq_no_ is the sequence no of the executing sql transaction::ObTxSEQ seq_no_; + // parallel write epoch no + int64_t write_epoch_; // scn_ is thee log ts of the redo log share::SCN scn_; int64_t column_cnt_; @@ -55,6 +57,7 @@ struct ObTxNodeArg K_(acc_checksum), K_(memstore_version), K_(seq_no), + K_(write_epoch), K_(scn), K_(column_cnt)); @@ -64,6 +67,7 @@ struct ObTxNodeArg const ObRowData *old_row, const int64_t memstore_version, const transaction::ObTxSEQ seq_no, + const int64_t write_epoch, const int64_t column_cnt) : tx_id_(tx_id), data_(data), @@ -72,6 +76,7 @@ struct ObTxNodeArg acc_checksum_(0), memstore_version_(memstore_version), seq_no_(seq_no), + write_epoch_(write_epoch), scn_(share::SCN::max_scn()), column_cnt_(column_cnt) {} @@ -92,6 +97,7 @@ struct ObTxNodeArg acc_checksum_(acc_checksum), memstore_version_(memstore_version), seq_no_(seq_no), + write_epoch_(0), scn_(scn), column_cnt_(column_cnt) {} @@ -103,6 +109,7 @@ struct ObTxNodeArg acc_checksum_ = 0; memstore_version_ = 0; seq_no_.reset(); + write_epoch_ = 0; scn_ = share::SCN::min_scn(); column_cnt_ = 0; } diff --git a/src/storage/memtable/mvcc/ob_mvcc_engine.cpp b/src/storage/memtable/mvcc/ob_mvcc_engine.cpp index c8ef8ff29..4748f2937 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_engine.cpp +++ b/src/storage/memtable/mvcc/ob_mvcc_engine.cpp @@ -326,8 +326,6 @@ int ObMvccEngine::mvcc_write(storage::ObStoreCtx &ctx, { int ret = OB_SUCCESS; ObMvccTransNode *node = NULL; - ObMemtableCtx *mem_ctx = ctx.mvcc_acc_ctx_.get_mem_ctx(); - if (OB_FAIL(build_tx_node_(arg, node))) { TRANS_LOG(WARN, "build tx node failed", K(ret), K(ctx), K(arg)); } else if (OB_FAIL(value.mvcc_write(ctx, @@ -336,10 +334,10 @@ int ObMvccEngine::mvcc_write(storage::ObStoreCtx &ctx, res))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) { - TRANS_LOG(WARN, "mvcc write failed", K(ret), KPC(mem_ctx), K(arg)); + TRANS_LOG(WARN, "mvcc write failed", K(ret), K(arg)); } } else { - TRANS_LOG(DEBUG, "mvcc write succeed", K(ret), KPC(mem_ctx), K(arg), K(*node)); + TRANS_LOG(DEBUG, "mvcc write succeed", K(ret), K(arg), K(*node)); } return ret; @@ -350,7 +348,6 @@ int ObMvccEngine::mvcc_replay(const ObTxNodeArg &arg, { int ret = OB_SUCCESS; ObMvccTransNode *node = NULL; - if (OB_FAIL(build_tx_node_(arg, node))) { TRANS_LOG(WARN, "build tx node failed", K(ret), K(arg)); } else { @@ -374,6 +371,7 @@ int ObMvccEngine::build_tx_node_(const ObTxNodeArg &arg, node->version_ = arg.memstore_version_; node->scn_ = arg.scn_; node->seq_no_ = arg.seq_no_; + node->write_epoch_ = arg.write_epoch_; node->prev_ = NULL; node->next_ = NULL; diff --git a/src/storage/memtable/mvcc/ob_mvcc_row.cpp b/src/storage/memtable/mvcc/ob_mvcc_row.cpp index cae8e1db4..e5675b3d9 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_row.cpp +++ b/src/storage/memtable/mvcc/ob_mvcc_row.cpp @@ -223,7 +223,8 @@ int64_t ObMvccTransNode::to_string(char *buf, const int64_t buf_len) const "snapshot_barrier=%ld " "snapshot_barrier_flag=%ld " "mtd=%s " - "seq_no=%s", + "seq_no=%s " + "write_epoch=%ld", this, to_cstring(trans_version_), to_cstring(scn_), @@ -239,7 +240,8 @@ int64_t ObMvccTransNode::to_string(char *buf, const int64_t buf_len) const & (~SNAPSHOT_VERSION_BARRIER_BIT), snapshot_version_barrier_ >> 62, to_cstring(*mtd), - to_cstring(seq_no_)); + to_cstring(seq_no_), + write_epoch_); return pos; } @@ -514,7 +516,10 @@ int ObMvccRow::insert_trans_node(ObIMvccCtx &ctx, ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "meet unexpected index_node", KR(ret), K(*prev), K(node), K(*index_node), K(*this)); abort_unless(0); - } else if (prev->tx_id_ == node.tx_id_ && OB_UNLIKELY(prev->seq_no_ > node.seq_no_)) { + } else if (prev->tx_id_ == node.tx_id_ + && OB_UNLIKELY(prev->seq_no_ > node.seq_no_) + // exclude the concurrently update uk case, which always in branch 0 + && !(prev->seq_no_.get_branch() == 0 && node.seq_no_.get_branch() == 0)) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "prev node seq_no > this node", KR(ret), KPC(prev), K(node), KPC(this)); usleep(1000); @@ -825,9 +830,9 @@ int ObMvccRow::mvcc_write_(ObStoreCtx &ctx, iter = iter->prev_; need_retry = true; } else if (data_tx_id == writer_tx_id) { + bool is_lock_node = false; // Case 4: the newest node is not decided and locked by itself, so we // can insert into it - bool is_lock_node = false; if (OB_FAIL(writer_node.is_lock_node(is_lock_node))) { TRANS_LOG(ERROR, "get is lock node failed", K(ret), K(writer_node)); } else if (is_lock_node) { @@ -881,9 +886,10 @@ int ObMvccRow::mvcc_write_(ObStoreCtx &ctx, list_head_->get_seq_no()))) { TRANS_LOG(WARN, "check sequence set violation failed", K(ret), KPC(this)); } else if (nullptr != list_head_ && FALSE_IT(res.is_checked_ = true)) { - } else if (OB_SUCC(check_double_insert_(snapshot_version, - writer_node, - list_head_))) { + } else if (OB_SUCC(mvcc_sanity_check_(snapshot_version, + ctx.mvcc_acc_ctx_.write_flag_, + writer_node, + list_head_))) { ATOMIC_STORE(&(writer_node.prev_), list_head_); ATOMIC_STORE(&(writer_node.next_), NULL); if (NULL != list_head_) { @@ -914,19 +920,43 @@ int ObMvccRow::mvcc_write_(ObStoreCtx &ctx, } __attribute__((noinline)) -int ObMvccRow::check_double_insert_(const SCN snapshot_version, - ObMvccTransNode &node, - ObMvccTransNode *prev) +int ObMvccRow::mvcc_sanity_check_(const SCN snapshot_version, + const concurrent_control::ObWriteFlag write_flag, + ObMvccTransNode &node, + ObMvccTransNode *prev) { int ret = OB_SUCCESS; + const bool compliant_with_sql_semantic = !write_flag.is_table_api(); + if (NULL != prev) { if (blocksstable::ObDmlFlag::DF_INSERT == node.get_dml_flag() && blocksstable::ObDmlFlag::DF_DELETE != prev->get_dml_flag() && prev->is_committed() && snapshot_version >= prev->trans_version_) { + // Case 1: Check double insert case ret = OB_ERR_PRIMARY_KEY_DUPLICATE; TRANS_LOG(WARN, "find double insert node", K(ret), K(node), KPC(prev), K(snapshot_version), K(*this)); + } else if (prev->get_tx_id() == node.get_tx_id() + && prev->is_incomplete() + && compliant_with_sql_semantic) { + // TODO(handora.qc): remove after kaizhan.dkz finish the concureent + // insert/delete feature + // + // Case 2: The current implementation allows the same rowkey to perform + // insert and delete within the same statement concurrently. So to prevent + // disorder between insert and callback registeration, we return an error + // in this scenario, hoping that the sql layer will retry later. + // Otherwise, it may lead to an out-of-order actions from logs between + // leader and follower. + ret = OB_SEQ_NO_REORDER_UNDER_PDML; + TRANS_LOG(INFO, "mvcc_write meet current write by self", K(ret), KPC(prev), K(node)); + } else if (prev->get_tx_id() == node.get_tx_id() + && prev->get_write_epoch() == node.get_write_epoch() + && prev->get_seq_no().get_branch() != node.get_seq_no().get_branch()) { + // Case 3: Check concurrent modify to the same row + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(ERROR, "concurrent modify to the same row", K(ret), KPC(prev), K(node)); } } diff --git a/src/storage/memtable/mvcc/ob_mvcc_row.h b/src/storage/memtable/mvcc/ob_mvcc_row.h index 6f59c2088..b751cea0b 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_row.h +++ b/src/storage/memtable/mvcc/ob_mvcc_row.h @@ -69,6 +69,7 @@ public: trans_version_(share::SCN::min_scn()), scn_(share::SCN::max_scn()), seq_no_(), + write_epoch_(0), tx_end_scn_(share::SCN::max_scn()), prev_(NULL), next_(NULL), @@ -85,6 +86,7 @@ public: share::SCN trans_version_; share::SCN scn_; transaction::ObTxSEQ seq_no_; + int64_t write_epoch_; share::SCN tx_end_scn_; ObMvccTransNode *prev_; ObMvccTransNode *next_; @@ -201,7 +203,7 @@ public: share::SCN get_tx_end_scn() const { return tx_end_scn_.atomic_load(); } share::SCN get_tx_version() const { return trans_version_.atomic_load(); } share::SCN get_scn() const { return scn_.atomic_load(); } - + int64_t get_write_epoch() const { return write_epoch_; } private: // the row flag of the mvcc tx node static const uint8_t F_INIT; @@ -420,10 +422,11 @@ struct ObMvccRow ObMvccWriteResult &res); // ===================== ObMvccRow Protection Code ===================== - // check double insert - int check_double_insert_(const share::SCN snapshot_version, - ObMvccTransNode &node, - ObMvccTransNode *prev); + // sanity check during mvcc_write + int mvcc_sanity_check_(const share::SCN snapshot_version, + const concurrent_control::ObWriteFlag write_flag, + ObMvccTransNode &node, + ObMvccTransNode *prev); }; } diff --git a/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.h b/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.h index 1e102e331..d0584e104 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.h +++ b/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.h @@ -253,6 +253,7 @@ public: int acquire_callback_list(const bool new_epoch); void revert_callback_list(); int get_tx_seq_replay_idx(const transaction::ObTxSEQ seq) const; + int64_t get_write_epoch() const { return write_epoch_; } common::SpinRWLock& get_rwlock() { return rwlock_; } private: void wakeup_waiting_txns_(); diff --git a/src/storage/memtable/ob_concurrent_control.h b/src/storage/memtable/ob_concurrent_control.h index 092999201..3d1a5ebfd 100644 --- a/src/storage/memtable/ob_concurrent_control.h +++ b/src/storage/memtable/ob_concurrent_control.h @@ -33,7 +33,8 @@ struct ObWriteFlag #define OBWF_BIT_CHECK_ROW_LOCKED 1 #define OBWF_BIT_LOB_AUX 1 #define OBWF_BIT_SKIP_FLUSH_REDO 1 - #define OBWF_BIT_RESERVED 55 + #define OBWF_BIT_UPDATE_UK 1 + #define OBWF_BIT_RESERVED 54 static const uint64_t OBWF_MASK_TABLE_API = (0x1UL << OBWF_BIT_TABLE_API) - 1; static const uint64_t OBWF_MASK_TABLE_LOCK = (0x1UL << OBWF_BIT_TABLE_LOCK) - 1; @@ -57,6 +58,7 @@ struct ObWriteFlag uint64_t is_check_row_locked_ : OBWF_BIT_CHECK_ROW_LOCKED; // 0: false(default), 1: true uint64_t is_lob_aux_ : OBWF_BIT_LOB_AUX; // 0: false(default), 1: true uint64_t is_skip_flush_redo_ : OBWF_BIT_SKIP_FLUSH_REDO; // 0: false(default), 1: true + uint64_t is_update_uk_ : OBWF_BIT_UPDATE_UK; // 0: false(default), 1: true uint64_t reserved_ : OBWF_BIT_RESERVED; }; }; @@ -82,6 +84,8 @@ struct ObWriteFlag inline bool is_skip_flush_redo() const { return is_skip_flush_redo_; } inline void set_skip_flush_redo() { is_skip_flush_redo_ = true; } inline void unset_skip_flush_redo() { is_skip_flush_redo_ = false; } + inline void set_update_uk() { is_update_uk_ = true; } + inline bool is_update_uk() const { return is_update_uk_; } TO_STRING_KV("is_table_api", is_table_api_, "is_table_lock", is_table_lock_, @@ -91,7 +95,8 @@ struct ObWriteFlag "is_write_only_index", is_write_only_index_, "is_check_row_locked", is_check_row_locked_, "is_lob_aux", is_lob_aux_, - "is_skip_flush_redo", is_skip_flush_redo_); + "is_skip_flush_redo", is_skip_flush_redo_, + "is_update_uk", is_update_uk_); OB_UNIS_VERSION(1); }; diff --git a/src/storage/memtable/ob_memtable.cpp b/src/storage/memtable/ob_memtable.cpp index a11fd6fd7..837cd1d8b 100644 --- a/src/storage/memtable/ob_memtable.cpp +++ b/src/storage/memtable/ob_memtable.cpp @@ -2635,9 +2635,14 @@ int ObMemtable::set_( } if (OB_SUCC(ret)) { bool is_new_locked = false; + ObTxSEQ write_seq; + int64_t write_epoch = 0; row_writer.reset(); if (OB_FAIL(ret)) { //do nothing + } else if (FALSE_IT(write_epoch = mem_ctx->get_write_epoch())) { + } else if (OB_FAIL(ctx.mvcc_acc_ctx_.get_write_seq(write_seq))) { + TRANS_LOG(WARN, "get write seq failed", K(ret)); } else if (OB_FAIL(row_writer.write(param.get_schema_rowkey_count(), new_row, update_idx, buf, len))) { TRANS_LOG(WARN, "Failed to write new row", K(ret), K(new_row)); } else if (OB_UNLIKELY(new_row.flag_.is_not_exist())) { @@ -2650,7 +2655,8 @@ int ObMemtable::set_( &mtd, /*memtable_data*/ NULL == old_row ? NULL : &old_row_data, init_timestamp_, /*memstore_version*/ - ctx.mvcc_acc_ctx_.tx_scn_, /*seq_no*/ + write_seq, /*seq_no*/ + write_epoch, /*write_epoch*/ new_row.row_val_.count_ /*column_cnt*/); if (OB_FAIL(mvcc_write_(param, context, @@ -2722,8 +2728,17 @@ int ObMemtable::lock_( blocksstable::ObRowWriter row_writer; char *buf = NULL; int64_t len = 0; - - if (OB_FAIL(row_writer.write_rowkey(rowkey, buf, len))) { + ObTxSEQ lock_seq; + ObMvccAccessCtx &acc_ctx = context.store_ctx_->mvcc_acc_ctx_; + ObMemtableCtx *mem_ctx = acc_ctx.get_mem_ctx(); + int64_t write_epoch = 0; + if (OB_ISNULL(mem_ctx)) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(ERROR, "mem_ctx is null", K(ret), KPC(context.store_ctx_)); + } else if (FALSE_IT(write_epoch = mem_ctx->get_write_epoch())) { + } else if (OB_FAIL(acc_ctx.get_write_seq(lock_seq))) { + TRANS_LOG(WARN, "get write seq failed", K(ret)); + } else if (OB_FAIL(row_writer.write_rowkey(rowkey, buf, len))) { TRANS_LOG(WARN, "Failed to writer rowkey", K(ret), K(rowkey)); } else { // for elr optimization @@ -2733,8 +2748,9 @@ int ObMemtable::lock_( ObTxNodeArg arg(acc_ctx.tx_id_, /*trans id*/ &mtd, /*memtable_data*/ NULL, /*old_data*/ - init_timestamp_, /*memstore_version*/ - acc_ctx.tx_scn_, /*seq_no*/ + init_timestamp_, /*memstore_version*/ + lock_seq, /*seq_no*/ + write_epoch, /*write_epoch*/ rowkey.get_obj_cnt()); /*column_cnt*/ if (OB_FAIL(mvcc_write_(param, context, diff --git a/src/storage/memtable/ob_memtable_context.h b/src/storage/memtable/ob_memtable_context.h index 4dcc4c8f0..73cf230cb 100644 --- a/src/storage/memtable/ob_memtable_context.h +++ b/src/storage/memtable/ob_memtable_context.h @@ -475,7 +475,7 @@ public: // callback void set_for_replay(const bool for_replay) { trans_mgr_.set_for_replay(for_replay); } void inc_pending_log_size(const int64_t size) { trans_mgr_.inc_pending_log_size(size); } void inc_flushed_log_size(const int64_t size) { trans_mgr_.inc_flushed_log_size(size); } - + int64_t get_write_epoch() const { return trans_mgr_.get_write_epoch(); } public: // tx_status enum ObTxStatus { diff --git a/src/storage/tx/ob_trans_define_v4.h b/src/storage/tx/ob_trans_define_v4.h index e35e4f2e1..a52cc3470 100644 --- a/src/storage/tx/ob_trans_define_v4.h +++ b/src/storage/tx/ob_trans_define_v4.h @@ -693,7 +693,8 @@ public: bool support_branch() const { return seq_base_ > 0; } // used by SQL alloc branch_id refer the min branch_id allowed // because branch_id bellow this is reserved for internal use - int branch_id_offset() const { return MAX_CALLBACK_LIST_COUNT; } + static int branch_id_offset() { return MAX_CALLBACK_LIST_COUNT; } + static bool is_alloced_branch_id(int branch_id) { return branch_id >= branch_id_offset(); } int alloc_branch_id(const int64_t count, int16_t &branch_id); int fetch_conflict_txs(ObIArray &array); void reset_conflict_txs() diff --git a/unittest/storage/memtable/test_memtable_basic.cpp b/unittest/storage/memtable/test_memtable_basic.cpp index 659a92dff..3838115da 100644 --- a/unittest/storage/memtable/test_memtable_basic.cpp +++ b/unittest/storage/memtable/test_memtable_basic.cpp @@ -113,9 +113,10 @@ int ObReadInfoStruct::init_compat_version() namespace memtable { -int ObMvccRow::check_double_insert_(const share::SCN , - ObMvccTransNode &, - ObMvccTransNode *) +int ObMvccRow::mvcc_sanity_check_(const share::SCN , + const concurrent_control::ObWriteFlag , + ObMvccTransNode &, + ObMvccTransNode *) { return OB_SUCCESS; } From 42f1e479c98a2af7642891087997710244f23189 Mon Sep 17 00:00:00 2001 From: godyangfight Date: Mon, 26 Aug 2024 03:20:05 +0000 Subject: [PATCH 212/249] Fix change member list get tablet status using wrong timeout --- src/storage/high_availability/ob_ls_member_list_service.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/storage/high_availability/ob_ls_member_list_service.cpp b/src/storage/high_availability/ob_ls_member_list_service.cpp index 63508e2af..0a76e95ba 100644 --- a/src/storage/high_availability/ob_ls_member_list_service.cpp +++ b/src/storage/high_availability/ob_ls_member_list_service.cpp @@ -179,6 +179,7 @@ int ObLSMemberListService::get_max_tablet_transfer_scn(share::SCN &transfer_scn) ObHALSTabletIDIterator iter(ls_->get_ls_id(), need_initial_state, need_sorted_tablet_id); share::SCN max_transfer_scn = share::SCN::min_scn(); static const int64_t LOCK_TIMEOUT = 100_ms; // 100ms + const int64_t timeout = 0; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; STORAGE_LOG(WARN, "not inited", K(ret), K_(is_inited)); @@ -206,7 +207,7 @@ int ObLSMemberListService::get_max_tablet_transfer_scn(share::SCN &transfer_scn) } else if (OB_FALSE_IT(key.tablet_id_ = tablet_id)) { } else if (OB_FAIL(t3m->get_tablet(priority, key, tablet_handle))) { STORAGE_LOG(WARN, "failed to get tablet", K(ret), K(key)); - } else if (OB_FAIL(tablet_handle.get_obj()->get_tablet_status(share::SCN::max_scn(), mds_data))) { + } else if (OB_FAIL(tablet_handle.get_obj()->get_tablet_status(share::SCN::max_scn(), mds_data, timeout))) { if (OB_EMPTY_RESULT == ret) { STORAGE_LOG(INFO, "committed tablet_status does not exist", K(ret), K(key)); ret = OB_SUCCESS; From e410ec98ee58fa42e54e97260f6fa7fec28d2b7a Mon Sep 17 00:00:00 2001 From: 0xacc Date: Mon, 26 Aug 2024 03:48:14 +0000 Subject: [PATCH 213/249] [CP] [to #2024082000104192277] fix: fix routine DDL printing when SQL_MODE='ORACLE' --- src/observer/virtual_table/ob_mysql_proc_table.cpp | 4 +++- src/share/schema/ob_schema_printer.cpp | 9 ++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/observer/virtual_table/ob_mysql_proc_table.cpp b/src/observer/virtual_table/ob_mysql_proc_table.cpp index 52ac77e42..209c4b302 100644 --- a/src/observer/virtual_table/ob_mysql_proc_table.cpp +++ b/src/observer/virtual_table/ob_mysql_proc_table.cpp @@ -416,7 +416,9 @@ int ObMySQLProcTable::extract_create_node_from_routine_info(ObIAllocator &alloc, ParseResult parse_result; ObString routine_stmt; - pl::ObPLParser parser(alloc, sql::ObCharsets4Parser(), exec_env.get_sql_mode()); + ObSQLMode sql_mode = exec_env.get_sql_mode(); + sql_mode &= ~SMO_ORACLE; + pl::ObPLParser parser(alloc, sql::ObCharsets4Parser(), sql_mode); const ObString &routine_body = routine_info.get_routine_body(); const char prefix[] = "CREATE\n"; int64_t prefix_len = STRLEN(prefix); diff --git a/src/share/schema/ob_schema_printer.cpp b/src/share/schema/ob_schema_printer.cpp index 8f2fddebd..6fdd8343b 100644 --- a/src/share/schema/ob_schema_printer.cpp +++ b/src/share/schema/ob_schema_printer.cpp @@ -4459,8 +4459,7 @@ int ObSchemaPrinter::print_routine_definition( } else if (routine_info->get_routine_body().prefix_match_ci("procedure") || routine_info->get_routine_body().prefix_match_ci("function")) { use_v1 = false; - - pl::ObPLParser parser(allocator, ObCharsets4Parser(), exec_env.get_sql_mode()); + ObSQLMode sql_mode = exec_env.get_sql_mode(); if (lib::is_mysql_mode()) { const char prefix[] = "CREATE\n"; @@ -4475,13 +4474,17 @@ int ObSchemaPrinter::print_routine_definition( MEMCPY(stmt_buf + prefix_len, routine_body.ptr(), routine_body.length()); routine_stmt.assign_ptr(stmt_buf, buf_sz); + sql_mode &= ~SMO_ORACLE; } } else { // oracle mode routine_stmt = routine_body; } CK(!routine_stmt.empty()); - OZ (parser.parse(routine_stmt, routine_stmt, parse_result, true)); + if (OB_SUCC(ret)) { + pl::ObPLParser parser(allocator, ObCharsets4Parser(), sql_mode); + OZ (parser.parse(routine_stmt, routine_stmt, parse_result, true)); + } } if (OB_SUCC(ret) && !use_v1) { From 6d520e25a9291ff8c02a32c45d81deb7025ff6b0 Mon Sep 17 00:00:00 2001 From: Handora Date: Mon, 26 Aug 2024 06:08:06 +0000 Subject: [PATCH 214/249] [BUG] fix transfer during rollback to --- .../test_transfer_between_rollback_to.cpp | 18 +++- src/storage/tx/ob_trans_define.cpp | 9 +- src/storage/tx/ob_trans_define_v4.h | 1 + src/storage/tx/ob_trans_part_ctx.cpp | 83 +++++++++---------- src/storage/tx/ob_trans_part_ctx.h | 15 ++-- src/storage/tx/ob_trans_rpc.cpp | 13 ++- src/storage/tx/ob_trans_rpc.h | 5 +- src/storage/tx/ob_trans_service_v4.cpp | 16 +++- src/storage/tx/ob_trans_service_v4.h | 7 +- src/storage/tx/ob_tx_api.cpp | 29 +++++-- src/storage/tx/ob_tx_msg.cpp | 5 +- src/storage/tx/ob_tx_msg.h | 21 +++-- 12 files changed, 138 insertions(+), 84 deletions(-) diff --git a/mittest/simple_server/test_transfer_between_rollback_to.cpp b/mittest/simple_server/test_transfer_between_rollback_to.cpp index 272742837..548c91624 100644 --- a/mittest/simple_server/test_transfer_between_rollback_to.cpp +++ b/mittest/simple_server/test_transfer_between_rollback_to.cpp @@ -131,7 +131,6 @@ public: WRITE_SQL_BY_CONN(connection, "set GLOBAL ob_query_timeout = 10000000000"); WRITE_SQL_BY_CONN(connection, "alter system set enable_early_lock_release = False;"); WRITE_SQL_BY_CONN(connection, "alter system set undo_retention = 1800;"); - WRITE_SQL_BY_CONN(connection, "alter system set partition_balance_schedule_interval = '10s';"); sleep(5); } @@ -213,6 +212,7 @@ public: MTL_SWITCH(tenant_id) { TRANS_LOG(INFO, "worker to do partition_balance"); auto b_svr = MTL(rootserver::ObTenantBalanceService*); + b_svr->stop(); b_svr->reset(); int64_t job_cnt = 0; int64_t start_time = OB_INVALID_TIMESTAMP, finish_time = OB_INVALID_TIMESTAMP; @@ -410,6 +410,22 @@ TEST_F(ObTransferBetweenRollbackTo, transfer_between_rollback_to) } ASSERT_EQ(loc1, loc2); + while (true) { + int64_t transfer_task_count = 0; + if (OB_FAIL(SSH::g_select_int64(tenant_id, "select count(*) as val from __all_transfer_task", transfer_task_count))) { + TRANS_LOG(WARN, "fail to wait for transfer task"); + } else if (transfer_task_count == 0) { + fprintf(stdout, "succeed wait for balancer v2\n"); + break; + } else if (ObTimeUtility::current_time() - begin_time > 300 * 1000 * 1000) { + fprintf(stdout, "ERROR: fail to wait for balancer\n"); + break; + } else { + fprintf(stdout, "wait for balancer v2\n"); + ob_usleep(200 * 1000); + } + } + usleep(1000 * 1000); global_ls_id.reset(); diff --git a/src/storage/tx/ob_trans_define.cpp b/src/storage/tx/ob_trans_define.cpp index 5ed129c77..667416e83 100644 --- a/src/storage/tx/ob_trans_define.cpp +++ b/src/storage/tx/ob_trans_define.cpp @@ -1007,7 +1007,9 @@ int RollbackMaskSet::merge_part(const share::ObLSID add_ls_id, const int64_t exe break; } } - if (!is_exist && OB_FAIL(rollback_parts_->push_back(ObTxExecPart(add_ls_id, exec_epoch, transfer_epoch)))) { + if (!is_exist && OB_FAIL(rollback_parts_->push_back(ObTxExecPart(add_ls_id, + exec_epoch, + transfer_epoch)))) { TRANS_LOG(WARN, "push part to array failed", KR(ret), K(add_ls_id)); } } @@ -1016,6 +1018,7 @@ int RollbackMaskSet::merge_part(const share::ObLSID add_ls_id, const int64_t exe int RollbackMaskSet::find_part(const share::ObLSID ls_id, const int64_t orig_epoch, + const int64_t transfer_epoch, ObTxExecPart &part) { int ret = OB_SUCCESS; @@ -1031,6 +1034,8 @@ int RollbackMaskSet::find_part(const share::ObLSID ls_id, ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "check rollback part failed", K(ret), K(rollback_parts_), K(orig_epoch)); } else { + rollback_parts_->at(idx).transfer_epoch_ = + MAX(transfer_epoch, rollback_parts_->at(idx).transfer_epoch_); part = rollback_parts_->at(idx); is_exist = true; } @@ -1042,7 +1047,7 @@ int RollbackMaskSet::find_part(const share::ObLSID ls_id, ret = OB_ENTRY_NOT_EXIST; } if (OB_FAIL(ret)) { - TRANS_LOG(WARN, "find part", K(ret), K(ls_id), K(orig_epoch), K(rollback_parts_)); + TRANS_LOG(WARN, "find part", K(ret), K(ls_id), K(orig_epoch), K(rollback_parts_), K(transfer_epoch)); } return ret; } diff --git a/src/storage/tx/ob_trans_define_v4.h b/src/storage/tx/ob_trans_define_v4.h index a52cc3470..3d07a4cc2 100644 --- a/src/storage/tx/ob_trans_define_v4.h +++ b/src/storage/tx/ob_trans_define_v4.h @@ -432,6 +432,7 @@ public: const int64_t transfer_epoch); int find_part(const share::ObLSID ls_id, const int64_t orig_epoch, + const int64_t transfer_epoch, ObTxExecPart &part); private: ObSpinLock lock_; diff --git a/src/storage/tx/ob_trans_part_ctx.cpp b/src/storage/tx/ob_trans_part_ctx.cpp index 4f2415588..0a44d3137 100644 --- a/src/storage/tx/ob_trans_part_ctx.cpp +++ b/src/storage/tx/ob_trans_part_ctx.cpp @@ -318,10 +318,6 @@ void ObPartTransCtx::destroy() trace_info_.reset(); block_frozen_memtable_ = nullptr; - last_rollback_to_request_id_ = 0; - last_rollback_to_timestamp_ = 0; - last_transfer_in_timestamp_ = 0; - is_inited_ = false; } } @@ -386,9 +382,6 @@ void ObPartTransCtx::default_init_() standby_part_collected_.reset(); trace_log_.reset(); transfer_deleted_ = false; - last_rollback_to_request_id_ = 0; - last_rollback_to_timestamp_ = 0; - last_transfer_in_timestamp_ = 0; } int ObPartTransCtx::init_log_cbs_(const ObLSID &ls_id, const ObTransID &tx_id) @@ -8627,6 +8620,18 @@ int ObPartTransCtx::end_access() return ret; } +int64_t ObPartTransCtx::get_max_transfer_epoch_() +{ + int64_t max_transfer_epoch = 0; + + for (int64_t i = 0; i < exec_info_.transfer_parts_.count(); i++) { + max_transfer_epoch = MAX(max_transfer_epoch, + exec_info_.transfer_parts_[i].transfer_epoch_); + } + + return max_transfer_epoch; +} + /* * rollback_to_savepoint - rollback to savepoint * @@ -8657,7 +8662,8 @@ int ObPartTransCtx::rollback_to_savepoint(const int64_t op_sn, ObTxSEQ from_scn, const ObTxSEQ to_scn, const int64_t seq_base, - const int64_t request_id, + const int64_t input_transfer_epoch, + int64_t &output_transfer_epoch, ObIArray &downstream_parts) { int ret = OB_SUCCESS; @@ -8697,25 +8703,16 @@ int ObPartTransCtx::rollback_to_savepoint(const int64_t op_sn, if (OB_SUCC(ret)) { bool need_downstream = true; - int64_t current_rollback_to_timestamp = ObTimeUtility::current_time(); - if (request_id != 0 && // come from rollback to request - request_id == last_rollback_to_request_id_) { // the same rollback to with the last one - if (last_transfer_in_timestamp_ != 0 && - last_rollback_to_timestamp_ != 0 && - // encounter transfer between two same rollback to - last_transfer_in_timestamp_ > last_rollback_to_timestamp_) { - need_downstream = true; - TRANS_LOG(INFO, "transfer between rollback to happened", K(ret), K(request_id), - K(last_rollback_to_timestamp_), K(last_transfer_in_timestamp_), - K(last_rollback_to_request_id_), KPC(this)); - } else { - need_downstream = false; - TRANS_LOG(INFO, "no transfer between rollback to happened", K(ret), K(request_id), - K(last_rollback_to_timestamp_), K(last_transfer_in_timestamp_), - K(last_rollback_to_request_id_), KPC(this)); - } - } else { + output_transfer_epoch = get_max_transfer_epoch_(); + + if (input_transfer_epoch != output_transfer_epoch) { need_downstream = true; + TRANS_LOG(INFO, "transfer between rollback to happened", K(ret), + K(input_transfer_epoch), K(output_transfer_epoch), KPC(this)); + } else { + need_downstream = false; + TRANS_LOG(INFO, "no transfer between rollback to happened", K(ret), + K(input_transfer_epoch), K(output_transfer_epoch), KPC(this)); } // must add downstream parts when return success @@ -8730,11 +8727,6 @@ int ObPartTransCtx::rollback_to_savepoint(const int64_t op_sn, TRANS_LOG(WARN, "push parts to array failed", K(ret), KPC(this)); } } - - if (OB_SUCC(ret)) { - last_rollback_to_request_id_ = request_id; - last_rollback_to_timestamp_ = current_rollback_to_timestamp; - } } REC_TRANS_TRACE_EXT(tlog_, rollback_savepoint, @@ -10266,21 +10258,22 @@ int ObPartTransCtx::move_tx_op(const ObTransferMoveTxParam &move_tx_param, ctx_tx_data_.set_state(ObTxData::ABORT); } - if (!arg.happened_before_) { - bool epoch_exist = false; - for (int64_t idx = 0; idx < exec_info_.transfer_parts_.count(); idx++) { - if (exec_info_.transfer_parts_.at(idx).ls_id_ == move_tx_param.src_ls_id_ && - exec_info_.transfer_parts_.at(idx).transfer_epoch_ == move_tx_param.transfer_epoch_) { - epoch_exist = true; - break; - } - } - if (!epoch_exist) { - if (OB_FAIL(exec_info_.transfer_parts_.push_back(ObTxExecPart(move_tx_param.src_ls_id_, -1, move_tx_param.transfer_epoch_)))) { - TRANS_LOG(WARN, "epochs push failed", K(ret)); - } + bool epoch_exist = false; + for (int64_t idx = 0; idx < exec_info_.transfer_parts_.count(); idx++) { + if (exec_info_.transfer_parts_.at(idx).ls_id_ == move_tx_param.src_ls_id_ && + exec_info_.transfer_parts_.at(idx).transfer_epoch_ == move_tx_param.transfer_epoch_) { + epoch_exist = true; + break; } } + if (!epoch_exist) { + if (OB_FAIL(exec_info_.transfer_parts_.push_back(ObTxExecPart(move_tx_param.src_ls_id_, + -1, /*exec_epoch*/ + move_tx_param.transfer_epoch_)))) { + TRANS_LOG(WARN, "epochs push failed", K(ret)); + } + } + if (OB_FAIL(ret)) { } else if (exec_info_.state_ < ObTxState::COMMIT && OB_FAIL(mt_ctx_.recover_from_table_lock_durable_info(arg.table_lock_info_, true))) { @@ -10296,8 +10289,6 @@ int ObPartTransCtx::move_tx_op(const ObTransferMoveTxParam &move_tx_param, exec_info_.is_transfer_blocking_ = false; if (OB_FAIL(transfer_op_log_cb_(move_tx_param.op_scn_, move_tx_param.op_type_))) { TRANS_LOG(WARN, "transfer op loc_cb failed", KR(ret), K(move_tx_param)); - } else { - last_transfer_in_timestamp_ = ObTimeUtility::current_time(); } } } diff --git a/src/storage/tx/ob_trans_part_ctx.h b/src/storage/tx/ob_trans_part_ctx.h index 3af85819a..96516d9c5 100644 --- a/src/storage/tx/ob_trans_part_ctx.h +++ b/src/storage/tx/ob_trans_part_ctx.h @@ -173,10 +173,7 @@ public: coord_prepare_info_arr_(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator(reserve_allocator_, "PREPARE_INFO")), standby_part_collected_(), ask_state_info_interval_(100 * 1000), refresh_state_info_interval_(100 * 1000), - transfer_deleted_(false), - last_rollback_to_request_id_(0), - last_rollback_to_timestamp_(0), - last_transfer_in_timestamp_(0) + transfer_deleted_(false) { /*reset();*/ } ~ObPartTransCtx() { destroy(); } void destroy(); @@ -739,6 +736,8 @@ private: int check_is_aborted_in_tx_data_(const ObTransID tx_id, bool &is_aborted); + int64_t get_max_transfer_epoch_(); + // ======================================================== // ======================== C2PC MSG HANDLER BEGIN ======================== @@ -921,7 +920,8 @@ public: ObTxSEQ from_seq, const ObTxSEQ to_seq, const int64_t seq_base, - const int64_t request_id, + const int64_t input_transfer_epoch, + int64_t &output_transfer_epoch, ObIArray &downstream_parts); bool is_xa_trans() const { return !exec_info_.xid_.empty(); } bool is_transfer_deleted() const { return transfer_deleted_; } @@ -1120,11 +1120,6 @@ private: // for transfer move tx ctx to clean for abort bool transfer_deleted_; - - // TODO(handora.qc): remove after fix the transfer bwteen rollback_to bug - int64_t last_rollback_to_request_id_; - int64_t last_rollback_to_timestamp_; - int64_t last_transfer_in_timestamp_; // ======================================================== }; diff --git a/src/storage/tx/ob_trans_rpc.cpp b/src/storage/tx/ob_trans_rpc.cpp index 9692cf67e..8ecc5f005 100644 --- a/src/storage/tx/ob_trans_rpc.cpp +++ b/src/storage/tx/ob_trans_rpc.cpp @@ -30,7 +30,8 @@ using namespace share; namespace obrpc { OB_SERIALIZE_MEMBER(ObTransRpcResult, status_, send_timestamp_, private_data_); -OB_SERIALIZE_MEMBER(ObTxRpcRollbackSPResult, status_, send_timestamp_, addr_, born_epoch_, ignore_, downstream_parts_); +OB_SERIALIZE_MEMBER(ObTxRpcRollbackSPResult, status_, send_timestamp_, addr_, + born_epoch_, ignore_, downstream_parts_, output_transfer_epoch_); bool need_refresh_location_cache_(const int ret) { @@ -79,8 +80,14 @@ int handle_sp_rollback_resp(const share::ObLSID &receiver_ls_id, return OB_SUCCESS; } return MTL(ObTransService *)->handle_sp_rollback_resp(receiver_ls_id, - epoch, tx_id, status, request_id, result.born_epoch_, result.addr_, - result.downstream_parts_); + epoch, + tx_id, + status, + request_id, + result.born_epoch_, + result.addr_, + result.output_transfer_epoch_, + result.downstream_parts_); } void ObTransRpcResult::reset() diff --git a/src/storage/tx/ob_trans_rpc.h b/src/storage/tx/ob_trans_rpc.h index 425500390..0d332ddfb 100644 --- a/src/storage/tx/ob_trans_rpc.h +++ b/src/storage/tx/ob_trans_rpc.h @@ -79,9 +79,12 @@ public: // use this field to indicate handler ignore handle by this msg bool ignore_; ObSEArray downstream_parts_; + // used for transfer info during rollback + int64_t output_transfer_epoch_; public: int get_status() const { return status_; } - TO_STRING_KV(K_(status), K_(send_timestamp), K_(born_epoch), K_(addr), K_(ignore), K_(downstream_parts)); + TO_STRING_KV(K_(status), K_(send_timestamp), K_(born_epoch), K_(addr), + K_(ignore), K_(output_transfer_epoch), K_(downstream_parts)); }; class ObTransRpcProxy : public obrpc::ObRpcProxy diff --git a/src/storage/tx/ob_trans_service_v4.cpp b/src/storage/tx/ob_trans_service_v4.cpp index b74f9d445..4e0736072 100644 --- a/src/storage/tx/ob_trans_service_v4.cpp +++ b/src/storage/tx/ob_trans_service_v4.cpp @@ -1867,6 +1867,7 @@ int ObTransService::batch_post_rollback_savepoint_msg_(ObTxDesc &tx, if (p.exec_epoch_ <= 0 && p.transfer_epoch_ > 0) { msg.set_for_transfer(); } + msg.input_transfer_epoch_ = p.transfer_epoch_; if (OB_FAIL(rpc_->post_msg(msg.receiver_, msg))) { if (OB_LS_IS_DELETED == ret) { ObSpinLockGuard lock(tx.lock_); @@ -2109,7 +2110,8 @@ int ObTransService::handle_sp_rollback_request(ObTxRollbackSPMsg &msg, msg.tx_ptr_, msg.for_transfer(), msg.specified_from_scn_, - msg.request_id_, + msg.input_transfer_epoch_, + result.output_transfer_epoch_, result.downstream_parts_); if (msg.use_async_resp()) { ObTxRollbackSPRespMsg resp; @@ -2124,6 +2126,7 @@ int ObTransService::handle_sp_rollback_request(ObTxRollbackSPMsg &msg, resp.ret_ = ret; resp.orig_epoch_ = msg.epoch_, resp.epoch_ = ctx_born_epoch; + resp.output_transfer_epoch_ = result.output_transfer_epoch_; int tmp_ret = OB_SUCCESS; if (OB_TMP_FAIL(resp.downstream_parts_.assign(result.downstream_parts_))) { TRANS_LOG(WARN, "parts assign failed", K(tmp_ret), K(resp)); @@ -2157,6 +2160,7 @@ int ObTransService::handle_sp_rollback_response(ObTxRollbackSPRespMsg &msg, msg.request_id_, msg.epoch_, msg.sender_addr_, + msg.output_transfer_epoch_, msg.downstream_parts_); result.reset(); result.init(ret, msg.get_timestamp()); @@ -2319,7 +2323,9 @@ int ObTransService::merge_rollback_downstream_parts_(ObTxDesc &tx, const ObIArra int ret = OB_SUCCESS; for (int64_t idx = 0; OB_SUCC(ret) && idx < downstream_parts.count(); idx++) { ObLSID add_ls_id = downstream_parts.at(idx).left_; - if (OB_FAIL(tx.brpc_mask_set_.merge_part(add_ls_id, 0, downstream_parts.at(idx).right_))) { + if (OB_FAIL(tx.brpc_mask_set_.merge_part(add_ls_id, + 0/*exec_epoch*/, + -1/*transfer_epoch*/))) { TRANS_LOG(WARN, "merge part failed", KR(ret), K(tx.tx_id_), K(add_ls_id)); } else { TRANS_LOG(INFO, "merge rollback parts", K(tx.tx_id_), K(add_ls_id)); @@ -2335,10 +2341,12 @@ int ObTransService::handle_sp_rollback_resp(const share::ObLSID &ls_id, const int64_t request_id, const int64_t ret_epoch, const ObAddr &ret_addr, + const int64_t transfer_epoch, const ObIArray &downstream_parts) { int ret = OB_SUCCESS; - TRANS_LOG(INFO, "handle_sp_rollback_resp", K(tx_id), K(ls_id), K(status), K(downstream_parts)); + TRANS_LOG(INFO, "handle_sp_rollback_resp", K(tx_id), K(ls_id), K(status), + K(transfer_epoch), K(downstream_parts)); ObRollbackSPMsgGuard *rollback_sp_msg_guard = NULL; ObTxDesc *tx = NULL; // find tx_msg by request_id @@ -2368,7 +2376,7 @@ int ObTransService::handle_sp_rollback_resp(const share::ObLSID &ls_id, ObTxExecPart p; if (downstream_parts.count() > 0 && OB_FAIL(merge_rollback_downstream_parts_(*tx, downstream_parts))) { TRANS_LOG(WARN, "merge rollback downstream parts failed", K(ret), K(tx_id), K(downstream_parts)); - } else if (OB_FAIL(tx->brpc_mask_set_.find_part(ls_id, orig_epoch, p))) { + } else if (OB_FAIL(tx->brpc_mask_set_.find_part(ls_id, orig_epoch, transfer_epoch, p))) { TRANS_LOG(WARN, "find part failed", K(ret), K(ls_id), K(tx_id)); } else { // find rollback part by ls_id diff --git a/src/storage/tx/ob_trans_service_v4.h b/src/storage/tx/ob_trans_service_v4.h index 59c5d6a1c..f6c6f5332 100644 --- a/src/storage/tx/ob_trans_service_v4.h +++ b/src/storage/tx/ob_trans_service_v4.h @@ -107,6 +107,7 @@ int handle_sp_rollback_resp(const share::ObLSID &ls_id, const int64_t request_id, const int64_t ret_epoch, const ObAddr &ret_addr, + const int64_t transfer_epoch, const ObIArray &downstream_parts); int handle_trans_msg_callback(const share::ObLSID &sender_ls_id, const share::ObLSID &receiver_ls_id, @@ -369,7 +370,8 @@ int ls_rollback_to_savepoint_(const ObTransID &tx_id, const ObTxDesc *tx, const bool for_transfer, const ObTxSEQ from_scn, - const int64_t request_id, + const int64_t input_transfer_epoch, + int64_t &output_transfer_epoch, ObIArray &downstream_parts, int64_t expire_ts = -1); int sync_rollback_savepoint__(ObTxDesc &tx, @@ -397,7 +399,8 @@ int ls_sync_rollback_savepoint__(ObPartTransCtx *part_ctx, const int64_t tx_seq_base, const int64_t expire_ts, const ObTxSEQ specified_from_scn, - const int64_t request_id, + const int64_t input_transfer_epoch, + int64_t &output_transfer_epoch, ObIArray &downstream_parts); void tx_post_terminate_(ObTxDesc &tx); int start_epoch_(ObTxDesc &tx); diff --git a/src/storage/tx/ob_tx_api.cpp b/src/storage/tx/ob_tx_api.cpp index 9537d53fa..14bc7c861 100644 --- a/src/storage/tx/ob_tx_api.cpp +++ b/src/storage/tx/ob_tx_api.cpp @@ -1132,6 +1132,7 @@ int ObTransService::rollback_to_local_implicit_savepoint_(ObTxDesc &tx, ARRAY_FOREACH(parts, i) { ObPartTransCtx *ctx = NULL; ObTxPart &p = parts[i]; + int64_t output_transfer_epoch = 0; ObSEArray downstream_parts; if (OB_FAIL(get_tx_ctx_(p.id_, tx.tx_id_, ctx))) { TRANS_LOG(WARN, "get tx ctx fail", K(ret), K_(p.id), K(tx)); @@ -1143,13 +1144,16 @@ int ObTransService::rollback_to_local_implicit_savepoint_(ObTxDesc &tx, tx.seq_base_, expire_ts, from_scn, - 0, /*request_id, only used for request*/ + -1, /*input_transfer_epoch*/ + output_transfer_epoch, downstream_parts))) { TRANS_LOG(WARN, "LS rollback savepoint fail", K(ret), K(savepoint), K(tx)); } else { // merge find new downstream to tx.rollback parts for (int64_t idx = 0; OB_SUCC(ret) && idx < downstream_parts.count(); idx++) { - if (OB_FAIL(rollback_parts.push_back(ObTxExecPart(downstream_parts.at(idx).left_, 0, downstream_parts.at(idx).right_)))) { + if (OB_FAIL(rollback_parts.push_back(ObTxExecPart(downstream_parts.at(idx).left_, + 0, /*exec_epoch*/ + -1 /*transfer_epoch*/)))) { TRANS_LOG(WARN, "push part to array failed", K(ret), K(tx)); } } @@ -1308,7 +1312,8 @@ int ObTransService::ls_sync_rollback_savepoint__(ObPartTransCtx *part_ctx, const int64_t tx_seq_base, const int64_t expire_ts, const ObTxSEQ specified_from_scn, - const int64_t request_id, + const int64_t input_transfer_epoch, + int64_t &output_transfer_epoch, ObIArray &downstream_parts) { int ret = OB_SUCCESS; @@ -1319,7 +1324,8 @@ int ObTransService::ls_sync_rollback_savepoint__(ObPartTransCtx *part_ctx, specified_from_scn, savepoint, tx_seq_base, - request_id, + input_transfer_epoch, + output_transfer_epoch, downstream_parts); if ((OB_NEED_RETRY == ret || OB_EAGAIN == ret) && blockable) { if (ObTimeUtility::current_time() >= expire_ts) { @@ -1537,6 +1543,7 @@ int ObTransService::rollback_savepoint_(ObTxDesc &tx, ObTxPart &p = parts[0]; int64_t born_epoch = 0; ObSEArray downstream_parts; + int64_t output_transfer_epoch = 0; if (OB_FAIL(ls_rollback_to_savepoint_(tx.tx_id_, p.id_, p.epoch_, @@ -1547,7 +1554,8 @@ int ObTransService::rollback_savepoint_(ObTxDesc &tx, &tx, false,/*for transfer*/ ObTxSEQ::INVL(), - 0, /*request_id, only for rollback_to request*/ + -1,/*input_transfer_epoch*/ + output_transfer_epoch, downstream_parts, -1/*non-blocking*/))) { if (common_retryable_error_(ret)) { @@ -1571,7 +1579,9 @@ int ObTransService::rollback_savepoint_(ObTxDesc &tx, TRANS_LOG(WARN, "reserve space fail", K(ret), K(parts), K(tx)); } else { ARRAY_FOREACH(parts, i) { - rollback_parts.push_back(ObTxExecPart(parts[i].id_, parts[i].epoch_, 0)); + rollback_parts.push_back(ObTxExecPart(parts[i].id_, + parts[i].epoch_, + -1 /*transfer_epoch*/)); } } if (FAILEDx(rollback_savepoint_slowpath_(tx, @@ -1618,7 +1628,8 @@ int ObTransService::ls_rollback_to_savepoint_(const ObTransID &tx_id, const ObTxDesc *tx, const bool for_transfer, const ObTxSEQ from_scn, - const int64_t request_id, + const int64_t input_transfer_epoch, + int64_t &output_transfer_epoch, ObIArray &downstream_parts, int64_t expire_ts) { @@ -1682,7 +1693,8 @@ int ObTransService::ls_rollback_to_savepoint_(const ObTransID &tx_id, tx_seq_base, expire_ts, from_scn, - request_id, + input_transfer_epoch, + output_transfer_epoch, downstream_parts))) { TRANS_LOG(WARN, "LS rollback to savepoint fail", K(ret), K(tx_id), K(ls), K(op_sn), K(savepoint), KPC(ctx)); } @@ -1724,6 +1736,7 @@ inline int ObTransService::rollback_savepoint_slowpath_(ObTxDesc &tx, msg.epoch_ = -1; msg.request_id_ = tx_msg_id; msg.specified_from_scn_ = specified_from_scn; + msg.input_transfer_epoch_ = -1; // prepare msg.tx_ptr_ if required // TODO(yunxing.cyx) : in 4.1 rework here, won't serialize txDesc ObTxDesc *tmp_tx_desc = NULL; diff --git a/src/storage/tx/ob_tx_msg.cpp b/src/storage/tx/ob_tx_msg.cpp index f208e1bbf..2d3c2d3f7 100644 --- a/src/storage/tx/ob_tx_msg.cpp +++ b/src/storage/tx/ob_tx_msg.cpp @@ -127,7 +127,7 @@ OB_TX_MSG_SERDE(ObAskStateMsg, ObTxMsg, snapshot_, ori_ls_id_, ori_addr_); OB_TX_MSG_SERDE(ObAskStateRespMsg, ObTxMsg, state_info_array_); OB_TX_MSG_SERDE(ObCollectStateMsg, ObTxMsg, snapshot_, check_info_); OB_TX_MSG_SERDE(ObCollectStateRespMsg, ObTxMsg, state_info_, transfer_parts_); -OB_SERIALIZE_MEMBER((ObTxRollbackSPRespMsg, ObTxMsg), ret_, orig_epoch_, downstream_parts_); +OB_SERIALIZE_MEMBER((ObTxRollbackSPRespMsg, ObTxMsg), ret_, orig_epoch_, downstream_parts_, output_transfer_epoch_); OB_DEF_SERIALIZE_SIZE(ObTxRollbackSPMsg) { @@ -142,6 +142,7 @@ OB_DEF_SERIALIZE_SIZE(ObTxRollbackSPMsg) } OB_UNIS_ADD_LEN(flag_); OB_UNIS_ADD_LEN(specified_from_scn_); + OB_UNIS_ADD_LEN(input_transfer_epoch_); return len; } @@ -158,6 +159,7 @@ OB_DEF_SERIALIZE(ObTxRollbackSPMsg) } OB_UNIS_ENCODE(flag_); OB_UNIS_ENCODE(specified_from_scn_); + OB_UNIS_ENCODE(input_transfer_epoch_); } return ret; } @@ -181,6 +183,7 @@ OB_DEF_DESERIALIZE(ObTxRollbackSPMsg) } OB_UNIS_DECODE(flag_); OB_UNIS_DECODE(specified_from_scn_); + OB_UNIS_DECODE(input_transfer_epoch_); } return ret; } diff --git a/src/storage/tx/ob_tx_msg.h b/src/storage/tx/ob_tx_msg.h index 64cdacfea..b00942b7b 100644 --- a/src/storage/tx/ob_tx_msg.h +++ b/src/storage/tx/ob_tx_msg.h @@ -249,7 +249,8 @@ namespace transaction tx_seq_base_(0), tx_ptr_(NULL), flag_(USE_ASYNC_RESP), - specified_from_scn_() + specified_from_scn_(), + input_transfer_epoch_(-1) {} ~ObTxRollbackSPMsg() { if (OB_NOT_NULL(tx_ptr_)) { @@ -258,6 +259,7 @@ namespace transaction tx_ptr_ = NULL; } specified_from_scn_.reset(); + input_transfer_epoch_ = -1; } ObTxSEQ savepoint_; int64_t op_sn_; @@ -265,6 +267,7 @@ namespace transaction const ObTxDesc *tx_ptr_; uint8_t flag_; ObTxSEQ specified_from_scn_; + int64_t input_transfer_epoch_; bool use_async_resp() const { return (flag_ & USE_ASYNC_RESP) !=0; } void set_for_transfer() { flag_ |= ROLLBACK_FOR_TRANSFER; } bool for_transfer() const { return (flag_ & ROLLBACK_FOR_TRANSFER) !=0; } @@ -273,24 +276,30 @@ namespace transaction bool is_valid() const; INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(savepoint), K_(op_sn), K_(tx_seq_base), K_(flag), - K_(specified_from_scn), KP_(tx_ptr)); + K_(specified_from_scn), KP_(tx_ptr), K_(input_transfer_epoch)); OB_UNIS_VERSION(1); }; struct ObTxRollbackSPRespMsg : public ObTxMsg { ObTxRollbackSPRespMsg() : - ObTxMsg(ROLLBACK_SAVEPOINT_RESP), - ret_(-1), - orig_epoch_(0) + ObTxMsg(ROLLBACK_SAVEPOINT_RESP), + ret_(-1), + orig_epoch_(0), + downstream_parts_(), + output_transfer_epoch_(-1) {} ~ObTxRollbackSPRespMsg() { ret_ = -1; orig_epoch_ = 0; + output_transfer_epoch_ = -1; } int ret_; int64_t orig_epoch_; ObSEArray downstream_parts_; - INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(ret), K_(orig_epoch), K_(downstream_parts)); + int64_t output_transfer_epoch_; + + INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(ret), K_(orig_epoch), + K_(output_transfer_epoch), K_(downstream_parts)); OB_UNIS_VERSION(1); }; From 695a8e0b549f57e1d05688b76070f5b099b05ca7 Mon Sep 17 00:00:00 2001 From: lalalafeier Date: Mon, 26 Aug 2024 06:26:38 +0000 Subject: [PATCH 215/249] add virtual table| view about tmp file --- src/observer/CMakeLists.txt | 1 + .../virtual_table/ob_all_virtual_tmp_file.cpp | 221 + .../virtual_table/ob_all_virtual_tmp_file.h | 81 + .../ob_virtual_table_iterator_factory.cpp | 13 + .../ob_inner_table_schema.12501_12550.cpp | 402 + .../ob_inner_table_schema.15451_15500.cpp | 369 + .../ob_inner_table_schema.21601_21650.cpp | 100 + .../ob_inner_table_schema.28201_28250.cpp | 19750 --------------- .../ob_inner_table_schema.28251_28300.cpp | 19830 ++++++++++++++++ src/share/inner_table/ob_inner_table_schema.h | 34 +- .../ob_inner_table_schema_constants.h | 10 + .../inner_table/ob_inner_table_schema_def.py | 138 +- src/share/inner_table/table_id_to_name | 6 + .../tmp_file/ob_shared_nothing_tmp_file.cpp | 143 +- .../tmp_file/ob_shared_nothing_tmp_file.h | 91 +- .../tmp_file/ob_tmp_file_block_manager.cpp | 6 +- .../tmp_file/ob_tmp_file_block_manager.h | 1 - src/storage/tmp_file/ob_tmp_file_io_ctx.cpp | 2 + src/storage/tmp_file/ob_tmp_file_io_ctx.h | 3 + src/storage/tmp_file/ob_tmp_file_manager.cpp | 66 +- src/storage/tmp_file/ob_tmp_file_manager.h | 16 + .../r/mysql/information_schema.result | 4 + .../r/mysql/desc_sys_views_in_mysql.result | 20 + .../r/mysql/desc_sys_views_in_sys.result | 41 + .../mysql/desc_virtual_table_in_mysql.result | 29 + .../r/mysql/desc_virtual_table_in_sys.result | 29 + .../r/mysql/inner_table_overall.result | 3 + 27 files changed, 21639 insertions(+), 19770 deletions(-) create mode 100644 src/observer/virtual_table/ob_all_virtual_tmp_file.cpp create mode 100644 src/observer/virtual_table/ob_all_virtual_tmp_file.h create mode 100644 src/share/inner_table/ob_inner_table_schema.12501_12550.cpp create mode 100644 src/share/inner_table/ob_inner_table_schema.28251_28300.cpp diff --git a/src/observer/CMakeLists.txt b/src/observer/CMakeLists.txt index 77d4ff66d..addb3f1b8 100644 --- a/src/observer/CMakeLists.txt +++ b/src/observer/CMakeLists.txt @@ -448,6 +448,7 @@ ob_set_subtarget(ob_server virtual_table virtual_table/ob_all_virtual_tenant_scheduler_running_job.cpp virtual_table/ob_all_virtual_compatibility_control.cpp virtual_table/ob_all_virtual_session_ps_info.cpp + virtual_table/ob_all_virtual_tmp_file.cpp ) ob_server_add_target(ob_server) diff --git a/src/observer/virtual_table/ob_all_virtual_tmp_file.cpp b/src/observer/virtual_table/ob_all_virtual_tmp_file.cpp new file mode 100644 index 000000000..5f2500bb7 --- /dev/null +++ b/src/observer/virtual_table/ob_all_virtual_tmp_file.cpp @@ -0,0 +1,221 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "observer/virtual_table/ob_all_virtual_tmp_file.h" +#include "observer/ob_server.h" +#include "storage/tmp_file/ob_shared_nothing_tmp_file.h" + +using namespace oceanbase::common; +using namespace oceanbase::transaction; + +namespace oceanbase +{ +namespace observer +{ + +ObAllVirtualTmpFileInfo::ObAllVirtualTmpFileInfo() + : ObVirtualTableScannerIterator(), + fd_arr_(), + is_ready_(false), + fd_idx_(-1) +{ +} + +ObAllVirtualTmpFileInfo::~ObAllVirtualTmpFileInfo() +{ + reset(); +} + +void ObAllVirtualTmpFileInfo::reset() +{ + // release tenant resources first + omt::ObMultiTenantOperator::reset(); + ip_buffer_[0] = '\0'; + trace_id_buffer_[0] = '\0'; + fd_arr_.reset(); + is_ready_ = false; + fd_idx_ = -1; + ObVirtualTableScannerIterator::reset(); +} + +void ObAllVirtualTmpFileInfo::release_last_tenant() +{ + // resources related with tenant must be released by this function + fd_arr_.reset(); + is_ready_ = false; + fd_idx_ = -1; +} + +bool ObAllVirtualTmpFileInfo::is_need_process(uint64_t tenant_id) +{ + bool bool_ret = false; + if (is_sys_tenant(effective_tenant_id_) || tenant_id == effective_tenant_id_) { + bool_ret = true; + } + + return bool_ret; +} + +int ObAllVirtualTmpFileInfo::get_next_tmp_file_info_(tmp_file::ObSNTmpFileInfo &tmp_file_info) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(0 > fd_idx_)) { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "unexpected fd_idx_", KR(ret), K(fd_idx_)); + } else { + bool has_get = false; + while (OB_SUCC(ret) + && !has_get) { + if (fd_idx_ >= fd_arr_.count()) { + ret = OB_ITER_END; + SERVER_LOG(INFO, "iterate current tenant reach end", K(fd_idx_), K(fd_arr_.count())); + } else if (OB_FAIL(FILE_MANAGER_INSTANCE_V2.get_tmp_file_info(fd_arr_.at(fd_idx_), tmp_file_info))) { + if (OB_ENTRY_NOT_EXIST == ret || OB_TIMEOUT == ret) { + SERVER_LOG(INFO, "tmp file does not exist or is locked by others", KR(ret), K(fd_arr_.at(fd_idx_))); + ret = OB_SUCCESS; + } else { + SERVER_LOG(WARN, "fail to get tmp file info", KR(ret), K(fd_idx_), K(fd_arr_), K(fd_arr_.at(fd_idx_))); + } + } else { + has_get = true; + } + if (OB_SUCC(ret)) { + fd_idx_++; + } + } + } + return ret; +} + +int ObAllVirtualTmpFileInfo::process_curr_tenant(common::ObNewRow *&row) +{ + int ret = OB_SUCCESS; + + if (nullptr == allocator_) { + ret = OB_NOT_INIT; + SERVER_LOG(WARN, "allocator_ shouldn't be nullptr", K(allocator_), KR(ret)); + } else if (FALSE_IT(start_to_read_ = true)) { + } else if (!is_ready_) { + if (OB_UNLIKELY(!fd_arr_.empty())) { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "unexpected fd_arr_", KR(ret), K(fd_arr_)); + } else if (OB_FAIL(FILE_MANAGER_INSTANCE_V2.get_tmp_file_fds(fd_arr_))) { + SERVER_LOG(WARN, "fail to get tmp file fd arr", KR(ret)); + if (OB_NOT_INIT == ret) { + ret = OB_SUCCESS; + } + } + if (OB_SUCC(ret)) { + is_ready_ = true; + fd_idx_ = 0; + } + } + + if (OB_SUCC(ret)) { + tmp_file::ObSNTmpFileInfo tmp_file_info; + if (OB_FAIL(get_next_tmp_file_info_(tmp_file_info))) { + if (OB_ITER_END != ret) { + SERVER_LOG(WARN, "fail to get next tmp file info", KR(ret)); + } + } else { + const int64_t col_count = output_column_ids_.count(); + ObAddr self_addr = GCONF.self_addr_; + for (int64_t i = 0; OB_SUCC(ret) && i < col_count; ++i) { + uint64_t col_id = output_column_ids_.at(i); + switch (col_id) { + case TENANT_ID: + cur_row_.cells_[i].set_int(tmp_file_info.tenant_id_); + break; + case SVR_IP: + MEMSET(ip_buffer_, '\0', OB_IP_STR_BUFF); + (void)self_addr.ip_to_string(ip_buffer_, common::OB_IP_STR_BUFF); + cur_row_.cells_[i].set_varchar(ip_buffer_); + cur_row_.cells_[i].set_default_collation_type(); + break; + case SVR_PORT: + cur_row_.cells_[i].set_int(self_addr.get_port()); + break; + case FILE_ID: + cur_row_.cells_[i].set_int(tmp_file_info.fd_); + break; + case TRACE_ID: + MEMSET(trace_id_buffer_, '\0', OB_MAX_TRACE_ID_BUFFER_SIZE); + if (!tmp_file_info.trace_id_.is_invalid()) { + tmp_file_info.trace_id_.to_string(trace_id_buffer_, OB_MAX_TRACE_ID_BUFFER_SIZE); + } + cur_row_.cells_[i].set_varchar(trace_id_buffer_); + cur_row_.cells_[i].set_default_collation_type(); + break; + case DIR_ID: + cur_row_.cells_[i].set_int(tmp_file_info.dir_id_); + break; + case BYTES: + cur_row_.cells_[i].set_int(tmp_file_info.file_size_); + break; + case START_OFFSET: + cur_row_.cells_[i].set_int(tmp_file_info.truncated_offset_); + break; + case IS_DELETING: + cur_row_.cells_[i].set_bool(tmp_file_info.is_deleting_); + break; + case CACHED_PAGE_NUM: + cur_row_.cells_[i].set_int(tmp_file_info.cached_page_num_); + break; + case WRITE_BACK_PAGE_NUM: + cur_row_.cells_[i].set_int(tmp_file_info.write_back_data_page_num_); + break; + case FLUSHED_PAGE_NUM: + cur_row_.cells_[i].set_int(tmp_file_info.flushed_data_page_num_); + break; + case REF_CNT: + cur_row_.cells_[i].set_int(tmp_file_info.ref_cnt_); + break; + case TOTAL_WRITES: + cur_row_.cells_[i].set_int(tmp_file_info.write_req_cnt_); + break; + case UNALIGNED_WRITES: + cur_row_.cells_[i].set_int(tmp_file_info.unaligned_write_req_cnt_); + break; + case TOTAL_READS: + cur_row_.cells_[i].set_int(tmp_file_info.read_req_cnt_); + break; + case UNALIGNED_READS: + cur_row_.cells_[i].set_int(tmp_file_info.unaligned_read_req_cnt_); + break; + case TOTAL_READ_BYTES: + cur_row_.cells_[i].set_int(tmp_file_info.total_read_size_); + break; + case LAST_ACCESS_TIME: + cur_row_.cells_[i].set_timestamp(tmp_file_info.last_access_ts_); + break; + case LAST_MODIFY_TIME: + cur_row_.cells_[i].set_timestamp(tmp_file_info.last_modify_ts_); + break; + case BIRTH_TIME: + cur_row_.cells_[i].set_timestamp(tmp_file_info.birth_ts_); + break; + default: + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "invalid coloum_id", KR(ret), K(col_id)); + break; + } + } + } + } + if (OB_SUCC(ret)) { + row = &cur_row_; + } + return ret; +} + +} +} diff --git a/src/observer/virtual_table/ob_all_virtual_tmp_file.h b/src/observer/virtual_table/ob_all_virtual_tmp_file.h new file mode 100644 index 000000000..8481774dc --- /dev/null +++ b/src/observer/virtual_table/ob_all_virtual_tmp_file.h @@ -0,0 +1,81 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OB_ALL_VIRTUAL_TMP_FILE_H_ +#define OB_ALL_VIRTUAL_TMP_FILE_H_ + +#include "share/ob_virtual_table_scanner_iterator.h" +#include "common/ob_clock_generator.h" +#include "observer/omt/ob_multi_tenant_operator.h" + +namespace oceanbase +{ +namespace tmp_file +{ +class ObSNTmpFileInfo; +} +namespace observer +{ + +class ObAllVirtualTmpFileInfo: public common::ObVirtualTableScannerIterator, + public omt::ObMultiTenantOperator +{ +public: + ObAllVirtualTmpFileInfo(); + ~ObAllVirtualTmpFileInfo(); + +public: + virtual int inner_get_next_row(common::ObNewRow *&row) { return execute(row);} + virtual void reset(); + +private: + virtual bool is_need_process(uint64_t tenant_id) override; + virtual int process_curr_tenant(common::ObNewRow *&row) override; + virtual void release_last_tenant() override; + int get_next_tmp_file_info_(tmp_file::ObSNTmpFileInfo &tmp_file_info); + +private: + enum + { + TENANT_ID = common::OB_APP_MIN_COLUMN_ID, + SVR_IP, + SVR_PORT, + FILE_ID, + TRACE_ID, + DIR_ID, + BYTES, + START_OFFSET, + IS_DELETING, + CACHED_PAGE_NUM, + WRITE_BACK_PAGE_NUM, + FLUSHED_PAGE_NUM, + REF_CNT, + TOTAL_WRITES, + UNALIGNED_WRITES, + TOTAL_READS, + UNALIGNED_READS, + TOTAL_READ_BYTES, + LAST_ACCESS_TIME, + LAST_MODIFY_TIME, + BIRTH_TIME + }; + char ip_buffer_[common::OB_IP_STR_BUFF]; + char trace_id_buffer_[common::OB_MAX_TRACE_ID_BUFFER_SIZE]; + ObArray fd_arr_; + bool is_ready_; + int64_t fd_idx_; + DISALLOW_COPY_AND_ASSIGN(ObAllVirtualTmpFileInfo); +}; + +} +} +#endif /* OB_ALL_VIRTUAL_TMP_FILE_H_ */ diff --git a/src/observer/virtual_table/ob_virtual_table_iterator_factory.cpp b/src/observer/virtual_table/ob_virtual_table_iterator_factory.cpp index 04c198807..daf90b5f3 100644 --- a/src/observer/virtual_table/ob_virtual_table_iterator_factory.cpp +++ b/src/observer/virtual_table/ob_virtual_table_iterator_factory.cpp @@ -233,6 +233,7 @@ #include "observer/virtual_table/ob_information_schema_enable_roles_table.h" #include "observer/virtual_table/ob_all_virtual_tenant_scheduler_running_job.h" #include "observer/virtual_table/ob_all_virtual_compatibility_control.h" +#include "observer/virtual_table/ob_all_virtual_tmp_file.h" namespace oceanbase { @@ -2779,6 +2780,18 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, } break; } + case OB_ALL_VIRTUAL_TEMP_FILE_TID: + { + ObAllVirtualTmpFileInfo *all_tmp_file_info = NULL; + if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualTmpFileInfo, all_tmp_file_info))) { + SERVER_LOG(ERROR, "ObAllVirtualTmpFileInfo construct failed", K(ret)); + } else if (OB_FAIL(all_tmp_file_info->init())) { + SERVER_LOG(WARN, "fail to init all_tmp_file_info", K(ret)); + } else { + vt_iter = static_cast(all_tmp_file_info); + } + break; + } END_CREATE_VT_ITER_SWITCH_LAMBDA #define AGENT_VIRTUAL_TABLE_CREATE_ITER diff --git a/src/share/inner_table/ob_inner_table_schema.12501_12550.cpp b/src/share/inner_table/ob_inner_table_schema.12501_12550.cpp new file mode 100644 index 000000000..67f5b51f2 --- /dev/null +++ b/src/share/inner_table/ob_inner_table_schema.12501_12550.cpp @@ -0,0 +1,402 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SHARE_SCHEMA +#include "ob_inner_table_schema.h" + +#include "share/schema/ob_schema_macro_define.h" +#include "share/schema/ob_schema_service_sql_impl.h" +#include "share/schema/ob_table_schema.h" +#include "share/scn.h" + +namespace oceanbase +{ +using namespace share::schema; +using namespace common; +namespace share +{ + +int ObInnerTableSchema::all_virtual_temp_file_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TEMP_FILE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(VIRTUAL_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TEMP_FILE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("svr_ip", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 1, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_IP_ADDR_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("svr_port", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 2, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("file_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("trace_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_TRACE_ID_BUFFER_SIZE, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("dir_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("bytes", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("start_offset", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("is_deleting", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTinyIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + 1, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("cached_page_num", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("write_back_page_num", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("flushed_page_num", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ref_cnt", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("total_writes", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("unaligned_writes", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("total_reads", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("unaligned_reads", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("total_read_bytes", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("last_access_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("last_modify_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("birth_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + if (OB_SUCC(ret)) { + table_schema.get_part_option().set_part_num(1); + table_schema.set_part_level(PARTITION_LEVEL_ONE); + table_schema.get_part_option().set_part_func_type(PARTITION_FUNC_TYPE_LIST_COLUMNS); + if (OB_FAIL(table_schema.get_part_option().set_part_expr("svr_ip, svr_port"))) { + LOG_WARN("set_part_expr failed", K(ret)); + } else if (OB_FAIL(table_schema.mock_list_partition_array())) { + LOG_WARN("mock list partition array failed", K(ret)); + } + } + table_schema.set_index_using_type(USING_HASH); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + + +} // end namespace share +} // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema.15451_15500.cpp b/src/share/inner_table/ob_inner_table_schema.15451_15500.cpp index 7a24d085e..c00587808 100644 --- a/src/share/inner_table/ob_inner_table_schema.15451_15500.cpp +++ b/src/share/inner_table/ob_inner_table_schema.15451_15500.cpp @@ -1220,6 +1220,375 @@ int ObInnerTableSchema::all_virtual_spatial_reference_systems_real_agent_ora_sch return ret; } +int ObInnerTableSchema::all_virtual_temp_file_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TEMP_FILE_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(VIRTUAL_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TEMP_FILE_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SVR_IP", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 1, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + MAX_IP_ADDR_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SVR_PORT", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 2, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("FILE_ID", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TRACE_ID", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_TRACE_ID_BUFFER_SIZE, //column_length + 2, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DIR_ID", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("BYTES", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("START_OFFSET", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("IS_DELETING", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("CACHED_PAGE_NUM", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("WRITE_BACK_PAGE_NUM", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("FLUSHED_PAGE_NUM", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("REF_CNT", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TOTAL_WRITES", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("UNALIGNED_WRITES", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TOTAL_READS", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("UNALIGNED_READS", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TOTAL_READ_BYTES", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("LAST_ACCESS_TIME", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampLTZType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("LAST_MODIFY_TIME", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampLTZType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("BIRTH_TIME", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampLTZType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + if (OB_SUCC(ret)) { + table_schema.get_part_option().set_part_num(1); + table_schema.set_part_level(PARTITION_LEVEL_ONE); + table_schema.get_part_option().set_part_func_type(PARTITION_FUNC_TYPE_LIST); + if (OB_FAIL(table_schema.get_part_option().set_part_expr("SVR_IP, SVR_PORT"))) { + LOG_WARN("set_part_expr failed", K(ret)); + } else if (OB_FAIL(table_schema.mock_list_partition_array())) { + LOG_WARN("mock list partition array failed", K(ret)); + } + } + table_schema.set_index_using_type(USING_HASH); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + } // end namespace share } // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema.21601_21650.cpp b/src/share/inner_table/ob_inner_table_schema.21601_21650.cpp index 39deefdd4..458b98ea1 100644 --- a/src/share/inner_table/ob_inner_table_schema.21601_21650.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21601_21650.cpp @@ -175,6 +175,106 @@ int ObInnerTableSchema::innodb_sys_foreign_cols_schema(ObTableSchema &table_sche return ret; } +int ObInnerTableSchema::dba_ob_temp_files_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_OB_TEMP_FILES_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_OB_TEMP_FILES_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT SVR_IP, SVR_PORT, FILE_ID, TRACE_ID, DIR_ID, BYTES, START_OFFSET, TOTAL_WRITES, UNALIGNED_WRITES, TOTAL_READS, UNALIGNED_READS, TOTAL_READ_BYTES, LAST_ACCESS_TIME, LAST_MODIFY_TIME, BIRTH_TIME FROM oceanbase.__all_virtual_temp_file WHERE TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::cdb_ob_temp_files_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_CDB_OB_TEMP_FILES_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_CDB_OB_TEMP_FILES_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT TENANT_ID, SVR_IP, SVR_PORT, FILE_ID, TRACE_ID, DIR_ID, BYTES, START_OFFSET, TOTAL_WRITES, UNALIGNED_WRITES, TOTAL_READS, UNALIGNED_READS, TOTAL_READ_BYTES, LAST_ACCESS_TIME, LAST_MODIFY_TIME, BIRTH_TIME FROM oceanbase.__all_virtual_temp_file )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + } // end namespace share } // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema.28201_28250.cpp b/src/share/inner_table/ob_inner_table_schema.28201_28250.cpp index 981297c68..720771e35 100644 --- a/src/share/inner_table/ob_inner_table_schema.28201_28250.cpp +++ b/src/share/inner_table/ob_inner_table_schema.28201_28250.cpp @@ -925,19756 +925,6 @@ int ObInnerTableSchema::dba_ob_spatial_columns_ora_schema(ObTableSchema &table_s return ret; } -int ObInnerTableSchema::all_table_idx_data_table_id_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TABLE_IDX_DATA_TABLE_ID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TABLE_IDX_DATA_TABLE_ID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TABLE_IDX_DATA_TABLE_ID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("data_table_id", //column_name - column_id + 21, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("table_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TABLE_TID); - - table_schema.set_max_used_column_id(column_id + 21); - return ret; -} - -int ObInnerTableSchema::all_table_idx_db_tb_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TABLE_IDX_DB_TB_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TABLE_IDX_DB_TB_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TABLE_IDX_DB_TB_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("database_id", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ObObj table_name_default; - table_name_default.set_varchar(ObString::make_string("")); - ADD_COLUMN_SCHEMA_T("table_name", //column_name - column_id + 3, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_TABLE_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false, //is_nullable - false, //is_autoincrement - table_name_default, - table_name_default); //default_value - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("table_id", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TABLE_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_table_idx_tb_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TABLE_IDX_TB_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TABLE_IDX_TB_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TABLE_IDX_TB_NAME_TID); - - if (OB_SUCC(ret)) { - ObObj table_name_default; - table_name_default.set_varchar(ObString::make_string("")); - ADD_COLUMN_SCHEMA_T("table_name", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_TABLE_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false, //is_nullable - false, //is_autoincrement - table_name_default, - table_name_default); //default_value - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("table_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TABLE_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_column_idx_tb_column_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_COLUMN_IDX_TB_COLUMN_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_COLUMN_IDX_TB_COLUMN_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_COLUMN_IDX_TB_COLUMN_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("table_id", //column_name - column_id + 2, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ObObj column_name_default; - column_name_default.set_varchar(ObString::make_string("")); - ADD_COLUMN_SCHEMA_T("column_name", //column_name - column_id + 4, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_COLUMN_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false, //is_nullable - false, //is_autoincrement - column_name_default, - column_name_default); //default_value - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("column_id", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_COLUMN_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_column_idx_column_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_COLUMN_IDX_COLUMN_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_COLUMN_IDX_COLUMN_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_COLUMN_IDX_COLUMN_NAME_TID); - - if (OB_SUCC(ret)) { - ObObj column_name_default; - column_name_default.set_varchar(ObString::make_string("")); - ADD_COLUMN_SCHEMA_T("column_name", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_COLUMN_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false, //is_nullable - false, //is_autoincrement - column_name_default, - column_name_default); //default_value - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("table_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("column_id", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_COLUMN_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_ddl_operation_idx_ddl_type_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_DDL_OPERATION_IDX_DDL_TYPE_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(1); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_DDL_OPERATION_IDX_DDL_TYPE_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_DDL_OPERATION_IDX_DDL_TYPE_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("operation_type", //column_name - column_id + 9, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("schema_version", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_DDL_OPERATION_TID); - - table_schema.set_max_used_column_id(column_id + 9); - return ret; -} - -int ObInnerTableSchema::all_table_history_idx_data_table_id_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TABLE_HISTORY_IDX_DATA_TABLE_ID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TABLE_HISTORY_IDX_DATA_TABLE_ID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TABLE_HISTORY_IDX_DATA_TABLE_ID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("data_table_id", //column_name - column_id + 23, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("table_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("schema_version", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TABLE_HISTORY_TID); - - table_schema.set_max_used_column_id(column_id + 23); - return ret; -} - -int ObInnerTableSchema::all_log_archive_piece_files_idx_status_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_LOG_ARCHIVE_PIECE_FILES_IDX_STATUS_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(4); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_LOG_ARCHIVE_PIECE_FILES_IDX_STATUS_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_LOG_ARCHIVE_PIECE_FILES_IDX_STATUS_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("dest_id", //column_name - column_id + 2, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ObObj file_status_default; - file_status_default.set_varchar(ObString::make_string("INVALID")); - ADD_COLUMN_SCHEMA_T("file_status", //column_name - column_id + 18, //column_id - 3, //rowkey_id - 3, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_DEFAULT_STATUS_LENTH, //column_length - -1, //column_precision - -1, //column_scale - false, //is_nullable - false, //is_autoincrement - file_status_default, - file_status_default); //default_value - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("round_id", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("piece_id", //column_name - column_id + 4, //column_id - 5, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_LOG_ARCHIVE_PIECE_FILES_TID); - - table_schema.set_max_used_column_id(column_id + 18); - return ret; -} - -int ObInnerTableSchema::all_backup_set_files_idx_status_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_BACKUP_SET_FILES_IDX_STATUS_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_BACKUP_SET_FILES_IDX_STATUS_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_BACKUP_SET_FILES_IDX_STATUS_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("dest_id", //column_name - column_id + 3, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("file_status", //column_name - column_id + 11, //column_id - 3, //rowkey_id - 3, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_DEFAULT_STATUS_LENTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("backup_set_id", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_BACKUP_SET_FILES_TID); - - table_schema.set_max_used_column_id(column_id + 11); - return ret; -} - -int ObInnerTableSchema::all_ddl_task_status_idx_task_key_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_DDL_TASK_STATUS_IDX_TASK_KEY_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(1); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_UNIQUE_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_DDL_TASK_STATUS_IDX_TASK_KEY_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_DDL_TASK_STATUS_IDX_TASK_KEY_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("target_object_id", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("object_id", //column_name - column_id + 3, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("schema_version", //column_name - column_id + 6, //column_id - 3, //rowkey_id - 3, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA_WITH_COLUMN_FLAGS("shadow_pk_0", //column_name - column_id + 32768, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - true,//is_nullable - false,//is_autoincrement - true,//is_hidden - false);//is_storing_column - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("task_id", //column_name - column_id + 1, //column_id - 0, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_UNIQUE_LOCAL); - table_schema.set_data_table_id(OB_ALL_DDL_TASK_STATUS_TID); - - table_schema.set_max_used_column_id(column_id + 32768); - return ret; -} - -int ObInnerTableSchema::all_user_idx_ur_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_USER_IDX_UR_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_USER_IDX_UR_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_USER_IDX_UR_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("user_name", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_USER_NAME_LENGTH_STORE, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("user_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_USER_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_database_idx_db_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_DATABASE_IDX_DB_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_DATABASE_IDX_DB_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_DATABASE_IDX_DB_NAME_TID); - - if (OB_SUCC(ret)) { - ObObj database_name_default; - database_name_default.set_varchar(ObString::make_string("")); - ADD_COLUMN_SCHEMA_T("database_name", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_DATABASE_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false, //is_nullable - false, //is_autoincrement - database_name_default, - database_name_default); //default_value - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("database_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_DATABASE_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_tablegroup_idx_tg_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TABLEGROUP_IDX_TG_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TABLEGROUP_IDX_TG_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TABLEGROUP_IDX_TG_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tablegroup_name", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_TABLEGROUP_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tablegroup_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TABLEGROUP_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_tenant_history_idx_tenant_deleted_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TENANT_HISTORY_IDX_TENANT_DELETED_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_HISTORY_IDX_TENANT_DELETED_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TENANT_HISTORY_IDX_TENANT_DELETED_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("is_deleted", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("schema_version", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TENANT_HISTORY_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_rootservice_event_history_idx_rs_module_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_ROOTSERVICE_EVENT_HISTORY_IDX_RS_MODULE_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(1); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_ROOTSERVICE_EVENT_HISTORY_IDX_RS_MODULE_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_ROOTSERVICE_EVENT_HISTORY_IDX_RS_MODULE_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("module", //column_name - column_id + 2, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - MAX_ROOTSERVICE_EVENT_DESC_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ObObj gmt_default; - ObObj gmt_default_null; - - gmt_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); - gmt_default_null.set_null(); - ADD_COLUMN_SCHEMA_TS_T("gmt_create", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObTimestampType, //column_type - CS_TYPE_INVALID, //column_collation_type - 0, //column_length - -1, //column_precision - 6, //column_scale - false, //is_nullable - false, //is_autoincrement - false, //is_on_update_for_timestamp - gmt_default_null, - gmt_default); - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_ROOTSERVICE_EVENT_HISTORY_TID); - - table_schema.set_max_used_column_id(column_id + 2); - return ret; -} - -int ObInnerTableSchema::all_rootservice_event_history_idx_rs_event_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_ROOTSERVICE_EVENT_HISTORY_IDX_RS_EVENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(1); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_ROOTSERVICE_EVENT_HISTORY_IDX_RS_EVENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_ROOTSERVICE_EVENT_HISTORY_IDX_RS_EVENT_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("event", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - MAX_ROOTSERVICE_EVENT_DESC_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ObObj gmt_default; - ObObj gmt_default_null; - - gmt_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); - gmt_default_null.set_null(); - ADD_COLUMN_SCHEMA_TS_T("gmt_create", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObTimestampType, //column_type - CS_TYPE_INVALID, //column_collation_type - 0, //column_length - -1, //column_precision - 6, //column_scale - false, //is_nullable - false, //is_autoincrement - false, //is_on_update_for_timestamp - gmt_default_null, - gmt_default); - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_ROOTSERVICE_EVENT_HISTORY_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_recyclebin_idx_recyclebin_db_type_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_RECYCLEBIN_IDX_RECYCLEBIN_DB_TYPE_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_RECYCLEBIN_IDX_RECYCLEBIN_DB_TYPE_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_RECYCLEBIN_IDX_RECYCLEBIN_DB_TYPE_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("database_id", //column_name - column_id + 4, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("type", //column_name - column_id + 3, //column_id - 3, //rowkey_id - 3, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("object_name", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_OBJECT_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_RECYCLEBIN_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_part_idx_part_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_PART_IDX_PART_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_PART_IDX_PART_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_PART_IDX_PART_NAME_TID); - - if (OB_SUCC(ret)) { - ObObj part_name_default; - part_name_default.set_varchar(ObString::make_string("")); - ADD_COLUMN_SCHEMA_T("part_name", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_PARTITION_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false, //is_nullable - false, //is_autoincrement - part_name_default, - part_name_default); //default_value - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("table_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("part_id", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_PART_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_sub_part_idx_sub_part_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_SUB_PART_IDX_SUB_PART_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(4); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_SUB_PART_IDX_SUB_PART_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_SUB_PART_IDX_SUB_PART_NAME_TID); - - if (OB_SUCC(ret)) { - ObObj sub_part_name_default; - sub_part_name_default.set_varchar(ObString::make_string("")); - ADD_COLUMN_SCHEMA_T("sub_part_name", //column_name - column_id + 5, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_PARTITION_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false, //is_nullable - false, //is_autoincrement - sub_part_name_default, - sub_part_name_default); //default_value - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("table_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("part_id", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("sub_part_id", //column_name - column_id + 4, //column_id - 5, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_SUB_PART_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_def_sub_part_idx_def_sub_part_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_DEF_SUB_PART_IDX_DEF_SUB_PART_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_DEF_SUB_PART_IDX_DEF_SUB_PART_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_DEF_SUB_PART_IDX_DEF_SUB_PART_NAME_TID); - - if (OB_SUCC(ret)) { - ObObj sub_part_name_default; - sub_part_name_default.set_varchar(ObString::make_string("")); - ADD_COLUMN_SCHEMA_T("sub_part_name", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_PARTITION_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false, //is_nullable - false, //is_autoincrement - sub_part_name_default, - sub_part_name_default); //default_value - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("table_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("sub_part_id", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_DEF_SUB_PART_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_server_event_history_idx_server_module_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_SERVER_EVENT_HISTORY_IDX_SERVER_MODULE_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_SERVER_EVENT_HISTORY_IDX_SERVER_MODULE_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_SERVER_EVENT_HISTORY_IDX_SERVER_MODULE_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("module", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - MAX_ROOTSERVICE_EVENT_DESC_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ObObj gmt_default; - ObObj gmt_default_null; - - gmt_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); - gmt_default_null.set_null(); - ADD_COLUMN_SCHEMA_TS_T("gmt_create", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObTimestampType, //column_type - CS_TYPE_INVALID, //column_collation_type - 0, //column_length - -1, //column_precision - 6, //column_scale - false, //is_nullable - false, //is_autoincrement - false, //is_on_update_for_timestamp - gmt_default_null, - gmt_default); - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("svr_ip", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - MAX_IP_ADDR_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("svr_port", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_SERVER_EVENT_HISTORY_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_server_event_history_idx_server_event_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_SERVER_EVENT_HISTORY_IDX_SERVER_EVENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_SERVER_EVENT_HISTORY_IDX_SERVER_EVENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_SERVER_EVENT_HISTORY_IDX_SERVER_EVENT_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("event", //column_name - column_id + 5, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - MAX_ROOTSERVICE_EVENT_DESC_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ObObj gmt_default; - ObObj gmt_default_null; - - gmt_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); - gmt_default_null.set_null(); - ADD_COLUMN_SCHEMA_TS_T("gmt_create", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObTimestampType, //column_type - CS_TYPE_INVALID, //column_collation_type - 0, //column_length - -1, //column_precision - 6, //column_scale - false, //is_nullable - false, //is_autoincrement - false, //is_on_update_for_timestamp - gmt_default_null, - gmt_default); - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("svr_ip", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - MAX_IP_ADDR_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("svr_port", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_SERVER_EVENT_HISTORY_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_rootservice_job_idx_rs_job_type_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_ROOTSERVICE_JOB_IDX_RS_JOB_TYPE_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(1); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_ROOTSERVICE_JOB_IDX_RS_JOB_TYPE_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_ROOTSERVICE_JOB_IDX_RS_JOB_TYPE_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("job_type", //column_name - column_id + 2, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - 128, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("job_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_ROOTSERVICE_JOB_TID); - - table_schema.set_max_used_column_id(column_id + 2); - return ret; -} - -int ObInnerTableSchema::all_foreign_key_idx_fk_child_tid_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_FOREIGN_KEY_IDX_FK_CHILD_TID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_FOREIGN_KEY_IDX_FK_CHILD_TID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_FOREIGN_KEY_IDX_FK_CHILD_TID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("child_table_id", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("foreign_key_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_FOREIGN_KEY_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_foreign_key_idx_fk_parent_tid_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_FOREIGN_KEY_IDX_FK_PARENT_TID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_FOREIGN_KEY_IDX_FK_PARENT_TID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_FOREIGN_KEY_IDX_FK_PARENT_TID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("parent_table_id", //column_name - column_id + 5, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("foreign_key_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_FOREIGN_KEY_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_foreign_key_idx_fk_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_FOREIGN_KEY_IDX_FK_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_FOREIGN_KEY_IDX_FK_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_FOREIGN_KEY_IDX_FK_NAME_TID); - - if (OB_SUCC(ret)) { - ObObj foreign_key_name_default; - foreign_key_name_default.set_varchar(ObString::make_string("")); - ADD_COLUMN_SCHEMA_T("foreign_key_name", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_CONSTRAINT_NAME_LENGTH_ORACLE, //column_length - -1, //column_precision - -1, //column_scale - false, //is_nullable - false, //is_autoincrement - foreign_key_name_default, - foreign_key_name_default); //default_value - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("foreign_key_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_FOREIGN_KEY_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_foreign_key_history_idx_fk_his_child_tid_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_FOREIGN_KEY_HISTORY_IDX_FK_HIS_CHILD_TID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_FOREIGN_KEY_HISTORY_IDX_FK_HIS_CHILD_TID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_FOREIGN_KEY_HISTORY_IDX_FK_HIS_CHILD_TID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("child_table_id", //column_name - column_id + 6, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("schema_version", //column_name - column_id + 3, //column_id - 3, //rowkey_id - 3, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("foreign_key_id", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_FOREIGN_KEY_HISTORY_TID); - - table_schema.set_max_used_column_id(column_id + 6); - return ret; -} - -int ObInnerTableSchema::all_foreign_key_history_idx_fk_his_parent_tid_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_FOREIGN_KEY_HISTORY_IDX_FK_HIS_PARENT_TID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_FOREIGN_KEY_HISTORY_IDX_FK_HIS_PARENT_TID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_FOREIGN_KEY_HISTORY_IDX_FK_HIS_PARENT_TID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("parent_table_id", //column_name - column_id + 7, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("schema_version", //column_name - column_id + 3, //column_id - 3, //rowkey_id - 3, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("foreign_key_id", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_FOREIGN_KEY_HISTORY_TID); - - table_schema.set_max_used_column_id(column_id + 7); - return ret; -} - -int ObInnerTableSchema::all_synonym_idx_db_synonym_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_SYNONYM_IDX_DB_SYNONYM_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_SYNONYM_IDX_DB_SYNONYM_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_SYNONYM_IDX_DB_SYNONYM_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("database_id", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ObObj synonym_name_default; - synonym_name_default.set_varchar(ObString::make_string("")); - ADD_COLUMN_SCHEMA_T("synonym_name", //column_name - column_id + 5, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_SYNONYM_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false, //is_nullable - false, //is_autoincrement - synonym_name_default, - synonym_name_default); //default_value - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("synonym_id", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_SYNONYM_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_synonym_idx_synonym_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_SYNONYM_IDX_SYNONYM_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_SYNONYM_IDX_SYNONYM_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_SYNONYM_IDX_SYNONYM_NAME_TID); - - if (OB_SUCC(ret)) { - ObObj synonym_name_default; - synonym_name_default.set_varchar(ObString::make_string("")); - ADD_COLUMN_SCHEMA_T("synonym_name", //column_name - column_id + 5, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_SYNONYM_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false, //is_nullable - false, //is_autoincrement - synonym_name_default, - synonym_name_default); //default_value - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("synonym_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_SYNONYM_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_ddl_checksum_idx_ddl_checksum_task_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_DDL_CHECKSUM_IDX_DDL_CHECKSUM_TASK_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(6); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_DDL_CHECKSUM_IDX_DDL_CHECKSUM_TASK_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_DDL_CHECKSUM_IDX_DDL_CHECKSUM_TASK_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("ddl_task_id", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("table_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("execution_id", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("column_id", //column_name - column_id + 5, //column_id - 5, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("task_id", //column_name - column_id + 6, //column_id - 6, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_DDL_CHECKSUM_TID); - - table_schema.set_max_used_column_id(column_id + 6); - return ret; -} - -int ObInnerTableSchema::all_routine_idx_db_routine_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_ROUTINE_IDX_DB_ROUTINE_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_ROUTINE_IDX_DB_ROUTINE_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_ROUTINE_IDX_DB_ROUTINE_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("database_id", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("routine_name", //column_name - column_id + 5, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_ROUTINE_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("routine_id", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_ROUTINE_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_routine_idx_routine_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_ROUTINE_IDX_ROUTINE_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_ROUTINE_IDX_ROUTINE_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_ROUTINE_IDX_ROUTINE_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("routine_name", //column_name - column_id + 5, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_ROUTINE_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("routine_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_ROUTINE_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_routine_idx_routine_pkg_id_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_ROUTINE_IDX_ROUTINE_PKG_ID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_ROUTINE_IDX_ROUTINE_PKG_ID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_ROUTINE_IDX_ROUTINE_PKG_ID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("package_id", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("routine_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_ROUTINE_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_routine_param_idx_routine_param_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_ROUTINE_PARAM_IDX_ROUTINE_PARAM_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_ROUTINE_PARAM_IDX_ROUTINE_PARAM_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_ROUTINE_PARAM_IDX_ROUTINE_PARAM_NAME_TID); - - if (OB_SUCC(ret)) { - ObObj param_name_default; - param_name_default.set_varchar(ObString::make_string("")); - ADD_COLUMN_SCHEMA_T("param_name", //column_name - column_id + 7, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_COLUMN_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - true, //is_nullable - false, //is_autoincrement - param_name_default, - param_name_default); //default_value - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("routine_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("sequence", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_ROUTINE_PARAM_TID); - - table_schema.set_max_used_column_id(column_id + 7); - return ret; -} - -int ObInnerTableSchema::all_package_idx_db_pkg_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_PACKAGE_IDX_DB_PKG_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_PACKAGE_IDX_DB_PKG_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_PACKAGE_IDX_DB_PKG_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("database_id", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ObObj package_name_default; - package_name_default.set_varchar(ObString::make_string("")); - ADD_COLUMN_SCHEMA_T("package_name", //column_name - column_id + 4, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_PACKAGE_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false, //is_nullable - false, //is_autoincrement - package_name_default, - package_name_default); //default_value - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("package_id", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_PACKAGE_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_package_idx_pkg_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_PACKAGE_IDX_PKG_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_PACKAGE_IDX_PKG_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_PACKAGE_IDX_PKG_NAME_TID); - - if (OB_SUCC(ret)) { - ObObj package_name_default; - package_name_default.set_varchar(ObString::make_string("")); - ADD_COLUMN_SCHEMA_T("package_name", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_PACKAGE_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false, //is_nullable - false, //is_autoincrement - package_name_default, - package_name_default); //default_value - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("package_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_PACKAGE_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_acquired_snapshot_idx_snapshot_tablet_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_ACQUIRED_SNAPSHOT_IDX_SNAPSHOT_TABLET_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_ACQUIRED_SNAPSHOT_IDX_SNAPSHOT_TABLET_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_ACQUIRED_SNAPSHOT_IDX_SNAPSHOT_TABLET_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tablet_id", //column_name - column_id + 6, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ObObj gmt_default; - ObObj gmt_default_null; - - gmt_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); - gmt_default_null.set_null(); - ADD_COLUMN_SCHEMA_TS_T("gmt_create", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObTimestampType, //column_type - CS_TYPE_INVALID, //column_collation_type - 0, //column_length - -1, //column_precision - 6, //column_scale - false, //is_nullable - false, //is_autoincrement - false, //is_on_update_for_timestamp - gmt_default_null, - gmt_default); - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_ACQUIRED_SNAPSHOT_TID); - - table_schema.set_max_used_column_id(column_id + 6); - return ret; -} - -int ObInnerTableSchema::all_constraint_idx_cst_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_CONSTRAINT_IDX_CST_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_CONSTRAINT_IDX_CST_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_CONSTRAINT_IDX_CST_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("constraint_name", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_CONSTRAINT_NAME_LENGTH_ORACLE, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("table_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("constraint_id", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_CONSTRAINT_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_type_idx_db_type_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TYPE_IDX_DB_TYPE_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TYPE_IDX_DB_TYPE_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TYPE_IDX_DB_TYPE_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("database_id", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("type_name", //column_name - column_id + 18, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_TABLE_TYPE_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("type_id", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TYPE_TID); - - table_schema.set_max_used_column_id(column_id + 18); - return ret; -} - -int ObInnerTableSchema::all_type_idx_type_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TYPE_IDX_TYPE_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TYPE_IDX_TYPE_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TYPE_IDX_TYPE_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("type_name", //column_name - column_id + 18, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_TABLE_TYPE_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("type_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TYPE_TID); - - table_schema.set_max_used_column_id(column_id + 18); - return ret; -} - -int ObInnerTableSchema::all_type_attr_idx_type_attr_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TYPE_ATTR_IDX_TYPE_ATTR_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TYPE_ATTR_IDX_TYPE_ATTR_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TYPE_ATTR_IDX_TYPE_ATTR_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("name", //column_name - column_id + 6, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_TABLE_TYPE_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("type_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("attribute", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TYPE_ATTR_TID); - - table_schema.set_max_used_column_id(column_id + 6); - return ret; -} - -int ObInnerTableSchema::all_coll_type_idx_coll_name_type_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_COLL_TYPE_IDX_COLL_NAME_TYPE_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_COLL_TYPE_IDX_COLL_NAME_TYPE_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_COLL_TYPE_IDX_COLL_NAME_TYPE_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("coll_name", //column_name - column_id + 16, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_TABLE_TYPE_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("coll_type", //column_name - column_id + 13, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("coll_type_id", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_COLL_TYPE_TID); - - table_schema.set_max_used_column_id(column_id + 16); - return ret; -} - -int ObInnerTableSchema::all_dblink_idx_owner_dblink_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_DBLINK_IDX_OWNER_DBLINK_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_DBLINK_IDX_OWNER_DBLINK_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_DBLINK_IDX_OWNER_DBLINK_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("owner_id", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("dblink_name", //column_name - column_id + 3, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_DBLINK_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("dblink_id", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_DBLINK_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_dblink_idx_dblink_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_DBLINK_IDX_DBLINK_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_DBLINK_IDX_DBLINK_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_DBLINK_IDX_DBLINK_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("dblink_name", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_DBLINK_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("dblink_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_DBLINK_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_tenant_role_grantee_map_idx_grantee_role_id_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TENANT_ROLE_GRANTEE_MAP_IDX_GRANTEE_ROLE_ID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_ROLE_GRANTEE_MAP_IDX_GRANTEE_ROLE_ID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TENANT_ROLE_GRANTEE_MAP_IDX_GRANTEE_ROLE_ID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("role_id", //column_name - column_id + 3, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("grantee_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TENANT_ROLE_GRANTEE_MAP_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_tenant_role_grantee_map_history_idx_grantee_his_role_id_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TENANT_ROLE_GRANTEE_MAP_HISTORY_IDX_GRANTEE_HIS_ROLE_ID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(4); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_ROLE_GRANTEE_MAP_HISTORY_IDX_GRANTEE_HIS_ROLE_ID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TENANT_ROLE_GRANTEE_MAP_HISTORY_IDX_GRANTEE_HIS_ROLE_ID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("role_id", //column_name - column_id + 3, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("schema_version", //column_name - column_id + 4, //column_id - 3, //rowkey_id - 3, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("grantee_id", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TENANT_ROLE_GRANTEE_MAP_HISTORY_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_tenant_keystore_idx_keystore_master_key_id_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TENANT_KEYSTORE_IDX_KEYSTORE_MASTER_KEY_ID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_KEYSTORE_IDX_KEYSTORE_MASTER_KEY_ID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TENANT_KEYSTORE_IDX_KEYSTORE_MASTER_KEY_ID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("master_key_id", //column_name - column_id + 6, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("keystore_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TENANT_KEYSTORE_TID); - - table_schema.set_max_used_column_id(column_id + 6); - return ret; -} - -int ObInnerTableSchema::all_tenant_keystore_history_idx_keystore_his_master_key_id_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TENANT_KEYSTORE_HISTORY_IDX_KEYSTORE_HIS_MASTER_KEY_ID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_KEYSTORE_HISTORY_IDX_KEYSTORE_HIS_MASTER_KEY_ID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TENANT_KEYSTORE_HISTORY_IDX_KEYSTORE_HIS_MASTER_KEY_ID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("master_key_id", //column_name - column_id + 8, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("keystore_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("schema_version", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TENANT_KEYSTORE_HISTORY_TID); - - table_schema.set_max_used_column_id(column_id + 8); - return ret; -} - -int ObInnerTableSchema::all_tenant_ols_policy_idx_ols_policy_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TENANT_OLS_POLICY_IDX_OLS_POLICY_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_OLS_POLICY_IDX_OLS_POLICY_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TENANT_OLS_POLICY_IDX_OLS_POLICY_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("policy_name", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_COLUMN_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("label_se_policy_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TENANT_OLS_POLICY_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_tenant_ols_policy_idx_ols_policy_col_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TENANT_OLS_POLICY_IDX_OLS_POLICY_COL_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_OLS_POLICY_IDX_OLS_POLICY_COL_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TENANT_OLS_POLICY_IDX_OLS_POLICY_COL_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("column_name", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_COLUMN_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("label_se_policy_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TENANT_OLS_POLICY_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_tenant_ols_component_idx_ols_com_policy_id_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TENANT_OLS_COMPONENT_IDX_OLS_COM_POLICY_ID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_OLS_COMPONENT_IDX_OLS_COM_POLICY_ID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TENANT_OLS_COMPONENT_IDX_OLS_COM_POLICY_ID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("label_se_policy_id", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("comp_type", //column_name - column_id + 4, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("label_se_component_id", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TENANT_OLS_COMPONENT_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_tenant_ols_label_idx_ols_lab_policy_id_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TENANT_OLS_LABEL_IDX_OLS_LAB_POLICY_ID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_OLS_LABEL_IDX_OLS_LAB_POLICY_ID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TENANT_OLS_LABEL_IDX_OLS_LAB_POLICY_ID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("label_se_policy_id", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("label_se_label_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TENANT_OLS_LABEL_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_tenant_ols_label_idx_ols_lab_tag_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TENANT_OLS_LABEL_IDX_OLS_LAB_TAG_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_OLS_LABEL_IDX_OLS_LAB_TAG_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TENANT_OLS_LABEL_IDX_OLS_LAB_TAG_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("label_tag", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("label_se_label_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TENANT_OLS_LABEL_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_tenant_ols_label_idx_ols_lab_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TENANT_OLS_LABEL_IDX_OLS_LAB_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_OLS_LABEL_IDX_OLS_LAB_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TENANT_OLS_LABEL_IDX_OLS_LAB_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("label", //column_name - column_id + 5, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_COLUMN_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("label_se_label_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TENANT_OLS_LABEL_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_tenant_ols_user_level_idx_ols_level_uid_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TENANT_OLS_USER_LEVEL_IDX_OLS_LEVEL_UID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_OLS_USER_LEVEL_IDX_OLS_LEVEL_UID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TENANT_OLS_USER_LEVEL_IDX_OLS_LEVEL_UID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("user_id", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("label_se_user_level_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TENANT_OLS_USER_LEVEL_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_tenant_ols_user_level_idx_ols_level_policy_id_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TENANT_OLS_USER_LEVEL_IDX_OLS_LEVEL_POLICY_ID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_OLS_USER_LEVEL_IDX_OLS_LEVEL_POLICY_ID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TENANT_OLS_USER_LEVEL_IDX_OLS_LEVEL_POLICY_ID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("label_se_policy_id", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("label_se_user_level_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TENANT_OLS_USER_LEVEL_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_tenant_profile_idx_profile_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TENANT_PROFILE_IDX_PROFILE_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_PROFILE_IDX_PROFILE_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TENANT_PROFILE_IDX_PROFILE_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("profile_name", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - MAX_ORACLE_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("profile_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TENANT_PROFILE_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_tenant_security_audit_idx_audit_type_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TENANT_SECURITY_AUDIT_IDX_AUDIT_TYPE_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_SECURITY_AUDIT_IDX_AUDIT_TYPE_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TENANT_SECURITY_AUDIT_IDX_AUDIT_TYPE_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("audit_type", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObUInt64Type, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(uint64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("audit_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TENANT_SECURITY_AUDIT_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_tenant_trigger_idx_trigger_base_obj_id_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TENANT_TRIGGER_IDX_TRIGGER_BASE_OBJ_ID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_TRIGGER_IDX_TRIGGER_BASE_OBJ_ID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TENANT_TRIGGER_IDX_TRIGGER_BASE_OBJ_ID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("base_object_id", //column_name - column_id + 11, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("trigger_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TENANT_TRIGGER_TID); - - table_schema.set_max_used_column_id(column_id + 11); - return ret; -} - -int ObInnerTableSchema::all_tenant_trigger_idx_db_trigger_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TENANT_TRIGGER_IDX_DB_TRIGGER_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_TRIGGER_IDX_DB_TRIGGER_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TENANT_TRIGGER_IDX_DB_TRIGGER_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("database_id", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("trigger_name", //column_name - column_id + 3, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_TRIGGER_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("trigger_id", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TENANT_TRIGGER_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_tenant_trigger_idx_trigger_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TENANT_TRIGGER_IDX_TRIGGER_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_TRIGGER_IDX_TRIGGER_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TENANT_TRIGGER_IDX_TRIGGER_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("trigger_name", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_TRIGGER_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("trigger_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TENANT_TRIGGER_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_tenant_trigger_history_idx_trigger_his_base_obj_id_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TENANT_TRIGGER_HISTORY_IDX_TRIGGER_HIS_BASE_OBJ_ID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_TRIGGER_HISTORY_IDX_TRIGGER_HIS_BASE_OBJ_ID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TENANT_TRIGGER_HISTORY_IDX_TRIGGER_HIS_BASE_OBJ_ID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("base_object_id", //column_name - column_id + 12, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("schema_version", //column_name - column_id + 3, //column_id - 3, //rowkey_id - 3, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("trigger_id", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TENANT_TRIGGER_HISTORY_TID); - - table_schema.set_max_used_column_id(column_id + 12); - return ret; -} - -int ObInnerTableSchema::all_tenant_objauth_idx_objauth_grantor_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TENANT_OBJAUTH_IDX_OBJAUTH_GRANTOR_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(7); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_OBJAUTH_IDX_OBJAUTH_GRANTOR_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TENANT_OBJAUTH_IDX_OBJAUTH_GRANTOR_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("grantor_id", //column_name - column_id + 5, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("obj_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("objtype", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("col_id", //column_name - column_id + 4, //column_id - 5, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("grantee_id", //column_name - column_id + 6, //column_id - 6, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("priv_id", //column_name - column_id + 7, //column_id - 7, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TENANT_OBJAUTH_TID); - - table_schema.set_max_used_column_id(column_id + 7); - return ret; -} - -int ObInnerTableSchema::all_tenant_objauth_idx_objauth_grantee_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TENANT_OBJAUTH_IDX_OBJAUTH_GRANTEE_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(7); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_OBJAUTH_IDX_OBJAUTH_GRANTEE_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TENANT_OBJAUTH_IDX_OBJAUTH_GRANTEE_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("grantee_id", //column_name - column_id + 6, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("obj_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("objtype", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("col_id", //column_name - column_id + 4, //column_id - 5, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("grantor_id", //column_name - column_id + 5, //column_id - 6, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("priv_id", //column_name - column_id + 7, //column_id - 7, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TENANT_OBJAUTH_TID); - - table_schema.set_max_used_column_id(column_id + 7); - return ret; -} - -int ObInnerTableSchema::all_tenant_object_type_idx_obj_type_db_obj_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TENANT_OBJECT_TYPE_IDX_OBJ_TYPE_DB_OBJ_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_OBJECT_TYPE_IDX_OBJ_TYPE_DB_OBJ_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TENANT_OBJECT_TYPE_IDX_OBJ_TYPE_DB_OBJ_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("database_id", //column_name - column_id + 13, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("object_name", //column_name - column_id + 17, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_TABLE_TYPE_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("object_type_id", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("type", //column_name - column_id + 3, //column_id - 5, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TENANT_OBJECT_TYPE_TID); - - table_schema.set_max_used_column_id(column_id + 17); - return ret; -} - -int ObInnerTableSchema::all_tenant_object_type_idx_obj_type_obj_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TENANT_OBJECT_TYPE_IDX_OBJ_TYPE_OBJ_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_OBJECT_TYPE_IDX_OBJ_TYPE_OBJ_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TENANT_OBJECT_TYPE_IDX_OBJ_TYPE_OBJ_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("object_name", //column_name - column_id + 17, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_TABLE_TYPE_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("object_type_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("type", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TENANT_OBJECT_TYPE_TID); - - table_schema.set_max_used_column_id(column_id + 17); - return ret; -} - -int ObInnerTableSchema::all_tenant_global_transaction_idx_xa_trans_id_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TENANT_GLOBAL_TRANSACTION_IDX_XA_TRANS_ID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(4); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_GLOBAL_TRANSACTION_IDX_XA_TRANS_ID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TENANT_GLOBAL_TRANSACTION_IDX_XA_TRANS_ID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("trans_id", //column_name - column_id + 5, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("gtrid", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_BINARY, //column_collation_type - 128, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("bqual", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_BINARY, //column_collation_type - 128, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ObObj format_id_default; - format_id_default.set_int(1); - ADD_COLUMN_SCHEMA_T("format_id", //column_name - column_id + 4, //column_id - 5, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false, //is_nullable - false, //is_autoincrement - format_id_default, - format_id_default); //default_value - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TENANT_GLOBAL_TRANSACTION_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_tenant_dependency_idx_dependency_ref_obj_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TENANT_DEPENDENCY_IDX_DEPENDENCY_REF_OBJ_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(4); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_DEPENDENCY_IDX_DEPENDENCY_REF_OBJ_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TENANT_DEPENDENCY_IDX_DEPENDENCY_REF_OBJ_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("ref_obj_id", //column_name - column_id + 8, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("ref_obj_type", //column_name - column_id + 7, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("dep_obj_type", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("dep_obj_id", //column_name - column_id + 3, //column_id - 5, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("dep_order", //column_name - column_id + 4, //column_id - 6, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TENANT_DEPENDENCY_TID); - - table_schema.set_max_used_column_id(column_id + 8); - return ret; -} - -int ObInnerTableSchema::all_ddl_error_message_idx_ddl_error_object_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_DDL_ERROR_MESSAGE_IDX_DDL_ERROR_OBJECT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(7); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_DDL_ERROR_MESSAGE_IDX_DDL_ERROR_OBJECT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_DDL_ERROR_MESSAGE_IDX_DDL_ERROR_OBJECT_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("object_id", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("target_object_id", //column_name - column_id + 3, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("task_id", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("schema_version", //column_name - column_id + 5, //column_id - 5, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("svr_ip", //column_name - column_id + 6, //column_id - 6, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - MAX_IP_ADDR_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("svr_port", //column_name - column_id + 7, //column_id - 7, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_DDL_ERROR_MESSAGE_TID); - - table_schema.set_max_used_column_id(column_id + 7); - return ret; -} - -int ObInnerTableSchema::all_table_stat_history_idx_table_stat_his_savtime_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TABLE_STAT_HISTORY_IDX_TABLE_STAT_HIS_SAVTIME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(4); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TABLE_STAT_HISTORY_IDX_TABLE_STAT_HIS_SAVTIME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TABLE_STAT_HISTORY_IDX_TABLE_STAT_HIS_SAVTIME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA_TS("savtime", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObTimestampType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(ObPreciseDateTime), //column_length - -1, //column_precision - -1, //column_scale - false, //is_nullable - false, //is_autoincrement - false); //is_on_update_for_timestamp - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("table_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("partition_id", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TABLE_STAT_HISTORY_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_column_stat_history_idx_column_stat_his_savtime_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_COLUMN_STAT_HISTORY_IDX_COLUMN_STAT_HIS_SAVTIME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(5); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_COLUMN_STAT_HISTORY_IDX_COLUMN_STAT_HIS_SAVTIME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_COLUMN_STAT_HISTORY_IDX_COLUMN_STAT_HIS_SAVTIME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA_TS("savtime", //column_name - column_id + 5, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObTimestampType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(ObPreciseDateTime), //column_length - -1, //column_precision - -1, //column_scale - false, //is_nullable - false, //is_autoincrement - false); //is_on_update_for_timestamp - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("table_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("partition_id", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("column_id", //column_name - column_id + 4, //column_id - 5, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_COLUMN_STAT_HISTORY_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_histogram_stat_history_idx_histogram_stat_his_savtime_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_HISTOGRAM_STAT_HISTORY_IDX_HISTOGRAM_STAT_HIS_SAVTIME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(6); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_HISTOGRAM_STAT_HISTORY_IDX_HISTOGRAM_STAT_HIS_SAVTIME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_HISTOGRAM_STAT_HISTORY_IDX_HISTOGRAM_STAT_HIS_SAVTIME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA_TS("savtime", //column_name - column_id + 6, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObTimestampType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(ObPreciseDateTime), //column_length - -1, //column_precision - -1, //column_scale - false, //is_nullable - false, //is_autoincrement - false); //is_on_update_for_timestamp - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("table_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("partition_id", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("column_id", //column_name - column_id + 4, //column_id - 5, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("endpoint_num", //column_name - column_id + 5, //column_id - 6, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_HISTOGRAM_STAT_HISTORY_TID); - - table_schema.set_max_used_column_id(column_id + 6); - return ret; -} - -int ObInnerTableSchema::all_tablet_to_ls_idx_tablet_to_ls_id_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TABLET_TO_LS_IDX_TABLET_TO_LS_ID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(1); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TABLET_TO_LS_IDX_TABLET_TO_LS_ID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TABLET_TO_LS_IDX_TABLET_TO_LS_ID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("ls_id", //column_name - column_id + 2, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tablet_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TABLET_TO_LS_TID); - - table_schema.set_max_used_column_id(column_id + 2); - return ret; -} - -int ObInnerTableSchema::all_tablet_to_ls_idx_tablet_to_table_id_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TABLET_TO_LS_IDX_TABLET_TO_TABLE_ID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(1); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TABLET_TO_LS_IDX_TABLET_TO_TABLE_ID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TABLET_TO_LS_IDX_TABLET_TO_TABLE_ID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("table_id", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tablet_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TABLET_TO_LS_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_pending_transaction_idx_pending_tx_id_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_PENDING_TRANSACTION_IDX_PENDING_TX_ID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_PENDING_TRANSACTION_IDX_PENDING_TX_ID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_PENDING_TRANSACTION_IDX_PENDING_TX_ID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("gtrid", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_BINARY, //column_collation_type - 128, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("bqual", //column_name - column_id + 4, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_BINARY, //column_collation_type - 128, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ObObj format_id_default; - format_id_default.set_int(1); - ADD_COLUMN_SCHEMA_T("format_id", //column_name - column_id + 5, //column_id - 3, //rowkey_id - 3, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false, //is_nullable - false, //is_autoincrement - format_id_default, - format_id_default); //default_value - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("trans_id", //column_name - column_id + 2, //column_id - 5, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_PENDING_TRANSACTION_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_context_idx_ctx_namespace_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_CONTEXT_IDX_CTX_NAMESPACE_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_CONTEXT_IDX_CTX_NAMESPACE_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_CONTEXT_IDX_CTX_NAMESPACE_TID); - - if (OB_SUCC(ret)) { - ObObj namespace_default; - namespace_default.set_varchar(ObString::make_string("")); - ADD_COLUMN_SCHEMA_T("namespace", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_CONTEXT_STRING_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false, //is_nullable - false, //is_autoincrement - namespace_default, - namespace_default); //default_value - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("context_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_CONTEXT_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_plan_baseline_item_idx_spm_item_sql_id_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_PLAN_BASELINE_ITEM_IDX_SPM_ITEM_SQL_ID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(4); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_PLAN_BASELINE_ITEM_IDX_SPM_ITEM_SQL_ID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_PLAN_BASELINE_ITEM_IDX_SPM_ITEM_SQL_ID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("sql_id", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_SQL_ID_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("database_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObUInt64Type, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(uint64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("plan_hash_value", //column_name - column_id + 4, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObUInt64Type, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(uint64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_PLAN_BASELINE_ITEM_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_plan_baseline_item_idx_spm_item_value_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_PLAN_BASELINE_ITEM_IDX_SPM_ITEM_VALUE_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(4); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_PLAN_BASELINE_ITEM_IDX_SPM_ITEM_VALUE_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_PLAN_BASELINE_ITEM_IDX_SPM_ITEM_VALUE_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("plan_hash_value", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObUInt64Type, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(uint64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("database_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObUInt64Type, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(uint64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("sql_id", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_SQL_ID_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_PLAN_BASELINE_ITEM_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_tenant_directory_idx_directory_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TENANT_DIRECTORY_IDX_DIRECTORY_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_DIRECTORY_IDX_DIRECTORY_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TENANT_DIRECTORY_IDX_DIRECTORY_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("directory_name", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - 128, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("directory_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TENANT_DIRECTORY_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_job_idx_job_powner_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_JOB_IDX_JOB_POWNER_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_JOB_IDX_JOB_POWNER_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_JOB_IDX_JOB_POWNER_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("powner", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_DATABASE_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("job", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_JOB_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_sequence_object_idx_seq_obj_db_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_SEQUENCE_OBJECT_IDX_SEQ_OBJ_DB_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_SEQUENCE_OBJECT_IDX_SEQ_OBJ_DB_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_SEQUENCE_OBJECT_IDX_SEQ_OBJ_DB_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("database_id", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("sequence_name", //column_name - column_id + 5, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_SEQUENCE_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("sequence_id", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_SEQUENCE_OBJECT_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_sequence_object_idx_seq_obj_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_SEQUENCE_OBJECT_IDX_SEQ_OBJ_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_SEQUENCE_OBJECT_IDX_SEQ_OBJ_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_SEQUENCE_OBJECT_IDX_SEQ_OBJ_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("sequence_name", //column_name - column_id + 5, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_SEQUENCE_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("sequence_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_SEQUENCE_OBJECT_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_recyclebin_idx_recyclebin_ori_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_RECYCLEBIN_IDX_RECYCLEBIN_ORI_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_RECYCLEBIN_IDX_RECYCLEBIN_ORI_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_RECYCLEBIN_IDX_RECYCLEBIN_ORI_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("original_name", //column_name - column_id + 7, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_ORIGINAL_NANE_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("object_name", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_OBJECT_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("type", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_RECYCLEBIN_TID); - - table_schema.set_max_used_column_id(column_id + 7); - return ret; -} - -int ObInnerTableSchema::all_table_privilege_idx_tb_priv_db_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TABLE_PRIVILEGE_IDX_TB_PRIV_DB_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(4); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TABLE_PRIVILEGE_IDX_TB_PRIV_DB_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TABLE_PRIVILEGE_IDX_TB_PRIV_DB_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("database_name", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_DATABASE_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("user_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("table_name", //column_name - column_id + 4, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_CORE_TALBE_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TABLE_PRIVILEGE_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_table_privilege_idx_tb_priv_tb_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TABLE_PRIVILEGE_IDX_TB_PRIV_TB_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(4); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TABLE_PRIVILEGE_IDX_TB_PRIV_TB_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TABLE_PRIVILEGE_IDX_TB_PRIV_TB_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("table_name", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_CORE_TALBE_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("user_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("database_name", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_DATABASE_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_TABLE_PRIVILEGE_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_database_privilege_idx_db_priv_db_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_DATABASE_PRIVILEGE_IDX_DB_PRIV_DB_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_DATABASE_PRIVILEGE_IDX_DB_PRIV_DB_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_DATABASE_PRIVILEGE_IDX_DB_PRIV_DB_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("database_name", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_DATABASE_NAME_LENGTH, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("user_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_DATABASE_PRIVILEGE_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_rls_policy_idx_rls_policy_table_id_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_RLS_POLICY_IDX_RLS_POLICY_TABLE_ID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_RLS_POLICY_IDX_RLS_POLICY_TABLE_ID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_RLS_POLICY_IDX_RLS_POLICY_TABLE_ID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("table_id", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("rls_policy_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_RLS_POLICY_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_rls_policy_idx_rls_policy_group_id_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_RLS_POLICY_IDX_RLS_POLICY_GROUP_ID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_RLS_POLICY_IDX_RLS_POLICY_GROUP_ID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_RLS_POLICY_IDX_RLS_POLICY_GROUP_ID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("rls_group_id", //column_name - column_id + 5, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("rls_policy_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_RLS_POLICY_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_rls_policy_history_idx_rls_policy_his_table_id_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_RLS_POLICY_HISTORY_IDX_RLS_POLICY_HIS_TABLE_ID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_RLS_POLICY_HISTORY_IDX_RLS_POLICY_HIS_TABLE_ID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_RLS_POLICY_HISTORY_IDX_RLS_POLICY_HIS_TABLE_ID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("table_id", //column_name - column_id + 6, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("rls_policy_id", //column_name - column_id + 2, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("schema_version", //column_name - column_id + 3, //column_id - 3, //rowkey_id - 3, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_RLS_POLICY_HISTORY_TID); - - table_schema.set_max_used_column_id(column_id + 6); - return ret; -} - -int ObInnerTableSchema::all_rls_group_idx_rls_group_table_id_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_RLS_GROUP_IDX_RLS_GROUP_TABLE_ID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_RLS_GROUP_IDX_RLS_GROUP_TABLE_ID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_RLS_GROUP_IDX_RLS_GROUP_TABLE_ID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("table_id", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("rls_group_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_RLS_GROUP_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_rls_group_history_idx_rls_group_his_table_id_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_RLS_GROUP_HISTORY_IDX_RLS_GROUP_HIS_TABLE_ID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_RLS_GROUP_HISTORY_IDX_RLS_GROUP_HIS_TABLE_ID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_RLS_GROUP_HISTORY_IDX_RLS_GROUP_HIS_TABLE_ID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("table_id", //column_name - column_id + 6, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("rls_group_id", //column_name - column_id + 2, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("schema_version", //column_name - column_id + 3, //column_id - 3, //rowkey_id - 3, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_RLS_GROUP_HISTORY_TID); - - table_schema.set_max_used_column_id(column_id + 6); - return ret; -} - -int ObInnerTableSchema::all_rls_context_idx_rls_context_table_id_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_RLS_CONTEXT_IDX_RLS_CONTEXT_TABLE_ID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_RLS_CONTEXT_IDX_RLS_CONTEXT_TABLE_ID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_RLS_CONTEXT_IDX_RLS_CONTEXT_TABLE_ID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("table_id", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("rls_context_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_RLS_CONTEXT_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_rls_context_history_idx_rls_context_his_table_id_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_RLS_CONTEXT_HISTORY_IDX_RLS_CONTEXT_HIS_TABLE_ID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_RLS_CONTEXT_HISTORY_IDX_RLS_CONTEXT_HIS_TABLE_ID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_RLS_CONTEXT_HISTORY_IDX_RLS_CONTEXT_HIS_TABLE_ID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("table_id", //column_name - column_id + 5, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("rls_context_id", //column_name - column_id + 2, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("schema_version", //column_name - column_id + 3, //column_id - 3, //rowkey_id - 3, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_RLS_CONTEXT_HISTORY_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_tenant_snapshot_idx_tenant_snapshot_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TENANT_SNAPSHOT_IDX_TENANT_SNAPSHOT_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_UNIQUE_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_SNAPSHOT_IDX_TENANT_SNAPSHOT_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TENANT_SNAPSHOT_IDX_TENANT_SNAPSHOT_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("snapshot_name", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - OB_MAX_TENANT_SNAPSHOT_NAME_LENGTH_STORE, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA_WITH_COLUMN_FLAGS("shadow_pk_0", //column_name - column_id + 32768, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - true,//is_nullable - false,//is_autoincrement - true,//is_hidden - false);//is_storing_column - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA_WITH_COLUMN_FLAGS("shadow_pk_1", //column_name - column_id + 32769, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - true,//is_nullable - false,//is_autoincrement - true,//is_hidden - false);//is_storing_column - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 0, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("snapshot_id", //column_name - column_id + 2, //column_id - 0, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_UNIQUE_LOCAL); - table_schema.set_data_table_id(OB_ALL_TENANT_SNAPSHOT_TID); - - table_schema.set_max_used_column_id(column_id + 32769); - return ret; -} - -int ObInnerTableSchema::all_dbms_lock_allocated_idx_dbms_lock_allocated_lockhandle_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_DBMS_LOCK_ALLOCATED_IDX_DBMS_LOCK_ALLOCATED_LOCKHANDLE_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(1); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_DBMS_LOCK_ALLOCATED_IDX_DBMS_LOCK_ALLOCATED_LOCKHANDLE_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_DBMS_LOCK_ALLOCATED_IDX_DBMS_LOCK_ALLOCATED_LOCKHANDLE_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("lockhandle", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - 128, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("name", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - 128, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_DBMS_LOCK_ALLOCATED_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_dbms_lock_allocated_idx_dbms_lock_allocated_expiration_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_DBMS_LOCK_ALLOCATED_IDX_DBMS_LOCK_ALLOCATED_EXPIRATION_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(1); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_DBMS_LOCK_ALLOCATED_IDX_DBMS_LOCK_ALLOCATED_EXPIRATION_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_DBMS_LOCK_ALLOCATED_IDX_DBMS_LOCK_ALLOCATED_EXPIRATION_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA_TS("expiration", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObTimestampType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(ObPreciseDateTime), //column_length - -1, //column_precision - -1, //column_scale - false, //is_nullable - false, //is_autoincrement - false); //is_on_update_for_timestamp - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("name", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - 128, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_DBMS_LOCK_ALLOCATED_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_kv_ttl_task_idx_kv_ttl_task_table_id_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_KV_TTL_TASK_IDX_KV_TTL_TASK_TABLE_ID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(4); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_KV_TTL_TASK_IDX_KV_TTL_TASK_TABLE_ID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_KV_TTL_TASK_IDX_KV_TTL_TASK_TABLE_ID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("table_id", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("task_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tablet_id", //column_name - column_id + 4, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_KV_TTL_TASK_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_kv_ttl_task_history_idx_kv_ttl_task_history_upd_time_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_KV_TTL_TASK_HISTORY_IDX_KV_TTL_TASK_HISTORY_UPD_TIME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(4); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_KV_TTL_TASK_HISTORY_IDX_KV_TTL_TASK_HISTORY_UPD_TIME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_KV_TTL_TASK_HISTORY_IDX_KV_TTL_TASK_HISTORY_UPD_TIME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("task_update_time", //column_name - column_id + 6, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("task_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("table_id", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tablet_id", //column_name - column_id + 4, //column_id - 5, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_KV_TTL_TASK_HISTORY_TID); - - table_schema.set_max_used_column_id(column_id + 6); - return ret; -} - -int ObInnerTableSchema::all_mview_refresh_run_stats_idx_mview_refresh_run_stats_num_mvs_current_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_MVIEW_REFRESH_RUN_STATS_IDX_MVIEW_REFRESH_RUN_STATS_NUM_MVS_CURRENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_MVIEW_REFRESH_RUN_STATS_IDX_MVIEW_REFRESH_RUN_STATS_NUM_MVS_CURRENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_MVIEW_REFRESH_RUN_STATS_IDX_MVIEW_REFRESH_RUN_STATS_NUM_MVS_CURRENT_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("num_mvs_current", //column_name - column_id + 5, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("refresh_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_MVIEW_REFRESH_RUN_STATS_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_mview_refresh_stats_idx_mview_refresh_stats_end_time_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_MVIEW_REFRESH_STATS_IDX_MVIEW_REFRESH_STATS_END_TIME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(4); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_MVIEW_REFRESH_STATS_IDX_MVIEW_REFRESH_STATS_END_TIME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_MVIEW_REFRESH_STATS_IDX_MVIEW_REFRESH_STATS_END_TIME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA_TS("end_time", //column_name - column_id + 7, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObTimestampType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(ObPreciseDateTime), //column_length - -1, //column_precision - -1, //column_scale - false, //is_nullable - false, //is_autoincrement - false); //is_on_update_for_timestamp - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("refresh_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("mview_id", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("retry_id", //column_name - column_id + 4, //column_id - 5, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_MVIEW_REFRESH_STATS_TID); - - table_schema.set_max_used_column_id(column_id + 7); - return ret; -} - -int ObInnerTableSchema::all_mview_refresh_stats_idx_mview_refresh_stats_mview_end_time_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_MVIEW_REFRESH_STATS_IDX_MVIEW_REFRESH_STATS_MVIEW_END_TIME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(4); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_MVIEW_REFRESH_STATS_IDX_MVIEW_REFRESH_STATS_MVIEW_END_TIME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_MVIEW_REFRESH_STATS_IDX_MVIEW_REFRESH_STATS_MVIEW_END_TIME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("mview_id", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA_TS("end_time", //column_name - column_id + 7, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObTimestampType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(ObPreciseDateTime), //column_length - -1, //column_precision - -1, //column_scale - false, //is_nullable - false, //is_autoincrement - false); //is_on_update_for_timestamp - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("refresh_id", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("retry_id", //column_name - column_id + 4, //column_id - 5, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_MVIEW_REFRESH_STATS_TID); - - table_schema.set_max_used_column_id(column_id + 7); - return ret; -} - -int ObInnerTableSchema::all_transfer_partition_task_idx_transfer_partition_key_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_TRANSFER_PARTITION_TASK_IDX_TRANSFER_PARTITION_KEY_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(1); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_UNIQUE_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_TRANSFER_PARTITION_TASK_IDX_TRANSFER_PARTITION_KEY_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_TRANSFER_PARTITION_TASK_IDX_TRANSFER_PARTITION_KEY_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("table_id", //column_name - column_id + 2, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("object_id", //column_name - column_id + 3, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA_WITH_COLUMN_FLAGS("shadow_pk_0", //column_name - column_id + 32768, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - true,//is_nullable - false,//is_autoincrement - true,//is_hidden - false);//is_storing_column - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("task_id", //column_name - column_id + 1, //column_id - 0, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_UNIQUE_LOCAL); - table_schema.set_data_table_id(OB_ALL_TRANSFER_PARTITION_TASK_TID); - - table_schema.set_max_used_column_id(column_id + 32768); - return ret; -} - -int ObInnerTableSchema::all_client_to_server_session_info_idx_client_to_server_session_info_client_session_id_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_CLIENT_TO_SERVER_SESSION_INFO_IDX_CLIENT_TO_SERVER_SESSION_INFO_CLIENT_SESSION_ID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(1); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_CLIENT_TO_SERVER_SESSION_INFO_IDX_CLIENT_TO_SERVER_SESSION_INFO_CLIENT_SESSION_ID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_CLIENT_TO_SERVER_SESSION_INFO_IDX_CLIENT_TO_SERVER_SESSION_INFO_CLIENT_SESSION_ID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("client_session_id", //column_name - column_id + 2, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("server_session_id", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_CLIENT_TO_SERVER_SESSION_INFO_TID); - - table_schema.set_max_used_column_id(column_id + 2); - return ret; -} - -int ObInnerTableSchema::all_column_privilege_idx_column_privilege_name_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_COLUMN_PRIVILEGE_IDX_COLUMN_PRIVILEGE_NAME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_COLUMN_PRIVILEGE_IDX_COLUMN_PRIVILEGE_NAME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_COLUMN_PRIVILEGE_IDX_COLUMN_PRIVILEGE_NAME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("user_id", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("database_name", //column_name - column_id + 4, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_BINARY, //column_collation_type - 1024, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("table_name", //column_name - column_id + 5, //column_id - 3, //rowkey_id - 3, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_BINARY, //column_collation_type - 1024, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("column_name", //column_name - column_id + 6, //column_id - 4, //rowkey_id - 4, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_BINARY, //column_collation_type - 1024, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 5, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("priv_id", //column_name - column_id + 2, //column_id - 6, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_COLUMN_PRIVILEGE_TID); - - table_schema.set_max_used_column_id(column_id + 6); - return ret; -} - -int ObInnerTableSchema::all_user_proxy_info_idx_user_proxy_info_proxy_user_id_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_USER_PROXY_INFO_IDX_USER_PROXY_INFO_PROXY_USER_ID_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_USER_PROXY_INFO_IDX_USER_PROXY_INFO_PROXY_USER_ID_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_USER_PROXY_INFO_IDX_USER_PROXY_INFO_PROXY_USER_ID_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("proxy_user_id", //column_name - column_id + 3, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("client_user_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_USER_PROXY_INFO_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_user_proxy_info_history_idx_user_proxy_info_proxy_user_id_history_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_USER_PROXY_INFO_HISTORY_IDX_USER_PROXY_INFO_PROXY_USER_ID_HISTORY_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(4); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_USER_PROXY_INFO_HISTORY_IDX_USER_PROXY_INFO_PROXY_USER_ID_HISTORY_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_USER_PROXY_INFO_HISTORY_IDX_USER_PROXY_INFO_PROXY_USER_ID_HISTORY_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("proxy_user_id", //column_name - column_id + 3, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("client_user_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("schema_version", //column_name - column_id + 4, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_USER_PROXY_INFO_HISTORY_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_user_proxy_role_info_history_idx_user_proxy_role_info_proxy_user_id_history_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_USER_PROXY_ROLE_INFO_HISTORY_IDX_USER_PROXY_ROLE_INFO_PROXY_USER_ID_HISTORY_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(5); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_USER_PROXY_ROLE_INFO_HISTORY_IDX_USER_PROXY_ROLE_INFO_PROXY_USER_ID_HISTORY_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_USER_PROXY_ROLE_INFO_HISTORY_IDX_USER_PROXY_ROLE_INFO_PROXY_USER_ID_HISTORY_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("tenant_id", //column_name - column_id + 1, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("proxy_user_id", //column_name - column_id + 3, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("client_user_id", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("role_id", //column_name - column_id + 4, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("schema_version", //column_name - column_id + 5, //column_id - 5, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObIntType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(int64_t), //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_USER_PROXY_ROLE_INFO_HISTORY_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_scheduler_job_run_detail_v2_idx_scheduler_job_run_detail_v2_time_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_IDX_SCHEDULER_JOB_RUN_DETAIL_V2_TIME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_IDX_SCHEDULER_JOB_RUN_DETAIL_V2_TIME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_IDX_SCHEDULER_JOB_RUN_DETAIL_V2_TIME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA_TS("time", //column_name - column_id + 2, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObTimestampType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(ObPreciseDateTime), //column_length - -1, //column_precision - -1, //column_scale - false, //is_nullable - false, //is_autoincrement - false); //is_on_update_for_timestamp - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("job_name", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - 128, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_TID); - - table_schema.set_max_used_column_id(column_id + 2); - return ret; -} - -int ObInnerTableSchema::all_scheduler_job_run_detail_v2_idx_scheduler_job_run_detail_v2_job_class_time_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); - table_schema.set_database_id(OB_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_IDX_SCHEDULER_JOB_RUN_DETAIL_V2_JOB_CLASS_TIME_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_IDX_SCHEDULER_JOB_RUN_DETAIL_V2_JOB_CLASS_TIME_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_create - } - - if (OB_SUCC(ret)) { - ++column_id; // for gmt_modified - } - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_IDX_SCHEDULER_JOB_RUN_DETAIL_V2_JOB_CLASS_TIME_TID); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("job_class", //column_name - column_id + 8, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - 128, //column_length - -1, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA_TS("time", //column_name - column_id + 2, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObTimestampType, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(ObPreciseDateTime), //column_length - -1, //column_precision - -1, //column_scale - false, //is_nullable - false, //is_autoincrement - false); //is_on_update_for_timestamp - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("job_name", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_INVALID, //column_collation_type - 128, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_TID); - - table_schema.set_max_used_column_id(column_id + 8); - return ret; -} - -int ObInnerTableSchema::all_virtual_table_real_agent_ora_idx_data_table_id_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_IDX_DATA_TABLE_ID_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_IDX_DATA_TABLE_ID_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("DATA_TABLE_ID", //column_name - column_id + 21, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TABLE_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 21); - return ret; -} - -int ObInnerTableSchema::all_virtual_table_real_agent_ora_idx_db_tb_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_IDX_DB_TB_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_IDX_DB_TB_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("DATABASE_ID", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TABLE_NAME", //column_name - column_id + 3, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_TABLE_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TABLE_ID", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_virtual_table_real_agent_ora_idx_tb_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_IDX_TB_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_IDX_TB_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TABLE_NAME", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_TABLE_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TABLE_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_virtual_column_real_agent_ora_idx_tb_column_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_COLUMN_REAL_AGENT_ORA_IDX_TB_COLUMN_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_COLUMN_REAL_AGENT_ORA_IDX_TB_COLUMN_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TABLE_ID", //column_name - column_id + 2, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("COLUMN_NAME", //column_name - column_id + 4, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_COLUMN_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("COLUMN_ID", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_COLUMN_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_virtual_column_real_agent_ora_idx_column_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_COLUMN_REAL_AGENT_ORA_IDX_COLUMN_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_COLUMN_REAL_AGENT_ORA_IDX_COLUMN_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("COLUMN_NAME", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_COLUMN_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TABLE_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("COLUMN_ID", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_COLUMN_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_virtual_user_real_agent_ora_idx_ur_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_USER_REAL_AGENT_ORA_IDX_UR_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_USER_REAL_AGENT_ORA_IDX_UR_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("USER_NAME", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_USER_NAME_LENGTH_STORE, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("USER_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_USER_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_virtual_database_real_agent_ora_idx_db_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_DATABASE_REAL_AGENT_ORA_IDX_DB_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_DATABASE_REAL_AGENT_ORA_IDX_DB_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("DATABASE_NAME", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_DATABASE_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("DATABASE_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_DATABASE_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_virtual_tablegroup_real_agent_ora_idx_tg_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TABLEGROUP_REAL_AGENT_ORA_IDX_TG_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TABLEGROUP_REAL_AGENT_ORA_IDX_TG_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TABLEGROUP_NAME", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_TABLEGROUP_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TABLEGROUP_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TABLEGROUP_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_virtual_recyclebin_real_agent_ora_idx_recyclebin_db_type_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_RECYCLEBIN_REAL_AGENT_ORA_IDX_RECYCLEBIN_DB_TYPE_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_RECYCLEBIN_REAL_AGENT_ORA_IDX_RECYCLEBIN_DB_TYPE_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("DATABASE_ID", //column_name - column_id + 4, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TYPE", //column_name - column_id + 3, //column_id - 3, //rowkey_id - 3, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("OBJECT_NAME", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_OBJECT_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_RECYCLEBIN_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_virtual_part_real_agent_ora_idx_part_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_PART_REAL_AGENT_ORA_IDX_PART_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_PART_REAL_AGENT_ORA_IDX_PART_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("PART_NAME", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_PARTITION_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TABLE_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("PART_ID", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_PART_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_virtual_sub_part_real_agent_ora_idx_sub_part_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_SUB_PART_REAL_AGENT_ORA_IDX_SUB_PART_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(4); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_SUB_PART_REAL_AGENT_ORA_IDX_SUB_PART_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("SUB_PART_NAME", //column_name - column_id + 5, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_PARTITION_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TABLE_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("PART_ID", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("SUB_PART_ID", //column_name - column_id + 4, //column_id - 5, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_SUB_PART_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_virtual_def_sub_part_real_agent_ora_idx_def_sub_part_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_DEF_SUB_PART_REAL_AGENT_ORA_IDX_DEF_SUB_PART_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_DEF_SUB_PART_REAL_AGENT_ORA_IDX_DEF_SUB_PART_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("SUB_PART_NAME", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_PARTITION_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TABLE_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("SUB_PART_ID", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_DEF_SUB_PART_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_virtual_foreign_key_real_agent_ora_idx_fk_child_tid_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_FOREIGN_KEY_REAL_AGENT_ORA_IDX_FK_CHILD_TID_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_FOREIGN_KEY_REAL_AGENT_ORA_IDX_FK_CHILD_TID_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("CHILD_TABLE_ID", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("FOREIGN_KEY_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_FOREIGN_KEY_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_virtual_foreign_key_real_agent_ora_idx_fk_parent_tid_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_FOREIGN_KEY_REAL_AGENT_ORA_IDX_FK_PARENT_TID_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_FOREIGN_KEY_REAL_AGENT_ORA_IDX_FK_PARENT_TID_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("PARENT_TABLE_ID", //column_name - column_id + 5, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("FOREIGN_KEY_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_FOREIGN_KEY_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_virtual_foreign_key_real_agent_ora_idx_fk_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_FOREIGN_KEY_REAL_AGENT_ORA_IDX_FK_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_FOREIGN_KEY_REAL_AGENT_ORA_IDX_FK_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("FOREIGN_KEY_NAME", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_CONSTRAINT_NAME_LENGTH_ORACLE, //column_length - 2, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("FOREIGN_KEY_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_FOREIGN_KEY_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_virtual_synonym_real_agent_ora_idx_db_synonym_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_SYNONYM_REAL_AGENT_ORA_IDX_DB_SYNONYM_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_SYNONYM_REAL_AGENT_ORA_IDX_DB_SYNONYM_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("DATABASE_ID", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("SYNONYM_NAME", //column_name - column_id + 5, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_SYNONYM_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("SYNONYM_ID", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_SYNONYM_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_virtual_synonym_real_agent_ora_idx_synonym_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_SYNONYM_REAL_AGENT_ORA_IDX_SYNONYM_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_SYNONYM_REAL_AGENT_ORA_IDX_SYNONYM_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("SYNONYM_NAME", //column_name - column_id + 5, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_SYNONYM_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("SYNONYM_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_SYNONYM_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_virtual_routine_real_agent_ora_idx_db_routine_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_ROUTINE_REAL_AGENT_ORA_IDX_DB_ROUTINE_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_ROUTINE_REAL_AGENT_ORA_IDX_DB_ROUTINE_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("DATABASE_ID", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("ROUTINE_NAME", //column_name - column_id + 5, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_ROUTINE_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("ROUTINE_ID", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_ROUTINE_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_virtual_routine_real_agent_ora_idx_routine_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_ROUTINE_REAL_AGENT_ORA_IDX_ROUTINE_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_ROUTINE_REAL_AGENT_ORA_IDX_ROUTINE_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("ROUTINE_NAME", //column_name - column_id + 5, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_ROUTINE_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("ROUTINE_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_ROUTINE_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_virtual_routine_real_agent_ora_idx_routine_pkg_id_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_ROUTINE_REAL_AGENT_ORA_IDX_ROUTINE_PKG_ID_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_ROUTINE_REAL_AGENT_ORA_IDX_ROUTINE_PKG_ID_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("PACKAGE_ID", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("ROUTINE_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_ROUTINE_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_virtual_routine_param_real_agent_ora_idx_routine_param_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_ROUTINE_PARAM_REAL_AGENT_ORA_IDX_ROUTINE_PARAM_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_ROUTINE_PARAM_REAL_AGENT_ORA_IDX_ROUTINE_PARAM_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("PARAM_NAME", //column_name - column_id + 7, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_COLUMN_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("ROUTINE_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("SEQUENCE", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_ROUTINE_PARAM_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 7); - return ret; -} - -int ObInnerTableSchema::all_virtual_package_real_agent_ora_idx_db_pkg_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_PACKAGE_REAL_AGENT_ORA_IDX_DB_PKG_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_PACKAGE_REAL_AGENT_ORA_IDX_DB_PKG_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("DATABASE_ID", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("PACKAGE_NAME", //column_name - column_id + 4, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_PACKAGE_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("PACKAGE_ID", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_PACKAGE_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_virtual_package_real_agent_ora_idx_pkg_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_PACKAGE_REAL_AGENT_ORA_IDX_PKG_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_PACKAGE_REAL_AGENT_ORA_IDX_PKG_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("PACKAGE_NAME", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_PACKAGE_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("PACKAGE_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_PACKAGE_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_virtual_constraint_real_agent_ora_idx_cst_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_CONSTRAINT_REAL_AGENT_ORA_IDX_CST_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_CONSTRAINT_REAL_AGENT_ORA_IDX_CST_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("CONSTRAINT_NAME", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_CONSTRAINT_NAME_LENGTH_ORACLE, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TABLE_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("CONSTRAINT_ID", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_CONSTRAINT_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_virtual_type_real_agent_ora_idx_db_type_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TYPE_REAL_AGENT_ORA_IDX_DB_TYPE_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TYPE_REAL_AGENT_ORA_IDX_DB_TYPE_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("DATABASE_ID", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TYPE_NAME", //column_name - column_id + 18, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_TABLE_TYPE_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TYPE_ID", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TYPE_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 18); - return ret; -} - -int ObInnerTableSchema::all_virtual_type_real_agent_ora_idx_type_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TYPE_REAL_AGENT_ORA_IDX_TYPE_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TYPE_REAL_AGENT_ORA_IDX_TYPE_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TYPE_NAME", //column_name - column_id + 18, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_TABLE_TYPE_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TYPE_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TYPE_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 18); - return ret; -} - -int ObInnerTableSchema::all_virtual_type_attr_real_agent_ora_idx_type_attr_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TYPE_ATTR_REAL_AGENT_ORA_IDX_TYPE_ATTR_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TYPE_ATTR_REAL_AGENT_ORA_IDX_TYPE_ATTR_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("NAME", //column_name - column_id + 6, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_TABLE_TYPE_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TYPE_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("ATTRIBUTE", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TYPE_ATTR_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 6); - return ret; -} - -int ObInnerTableSchema::all_virtual_coll_type_real_agent_ora_idx_coll_name_type_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_COLL_TYPE_REAL_AGENT_ORA_IDX_COLL_NAME_TYPE_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_COLL_TYPE_REAL_AGENT_ORA_IDX_COLL_NAME_TYPE_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("COLL_NAME", //column_name - column_id + 16, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_TABLE_TYPE_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("COLL_TYPE", //column_name - column_id + 13, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("COLL_TYPE_ID", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_COLL_TYPE_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 16); - return ret; -} - -int ObInnerTableSchema::all_virtual_dblink_real_agent_ora_idx_owner_dblink_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_DBLINK_REAL_AGENT_ORA_IDX_OWNER_DBLINK_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_DBLINK_REAL_AGENT_ORA_IDX_OWNER_DBLINK_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("OWNER_ID", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("DBLINK_NAME", //column_name - column_id + 3, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_DBLINK_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("DBLINK_ID", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_DBLINK_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_virtual_dblink_real_agent_ora_idx_dblink_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_DBLINK_REAL_AGENT_ORA_IDX_DBLINK_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_DBLINK_REAL_AGENT_ORA_IDX_DBLINK_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("DBLINK_NAME", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_DBLINK_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("DBLINK_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_DBLINK_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_virtual_tenant_role_grantee_map_real_agent_ora_idx_grantee_role_id_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_ROLE_GRANTEE_MAP_REAL_AGENT_ORA_IDX_GRANTEE_ROLE_ID_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_ROLE_GRANTEE_MAP_REAL_AGENT_ORA_IDX_GRANTEE_ROLE_ID_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("ROLE_ID", //column_name - column_id + 3, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("GRANTEE_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_ROLE_GRANTEE_MAP_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_virtual_tenant_keystore_real_agent_ora_idx_keystore_master_key_id_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_KEYSTORE_REAL_AGENT_ORA_IDX_KEYSTORE_MASTER_KEY_ID_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_KEYSTORE_REAL_AGENT_ORA_IDX_KEYSTORE_MASTER_KEY_ID_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("MASTER_KEY_ID", //column_name - column_id + 6, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("KEYSTORE_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_KEYSTORE_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 6); - return ret; -} - -int ObInnerTableSchema::all_virtual_tenant_ols_policy_real_agent_ora_idx_ols_policy_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_OLS_POLICY_REAL_AGENT_ORA_IDX_OLS_POLICY_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_OLS_POLICY_REAL_AGENT_ORA_IDX_OLS_POLICY_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("POLICY_NAME", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_COLUMN_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("LABEL_SE_POLICY_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_OLS_POLICY_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_virtual_tenant_ols_policy_real_agent_ora_idx_ols_policy_col_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_OLS_POLICY_REAL_AGENT_ORA_IDX_OLS_POLICY_COL_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_OLS_POLICY_REAL_AGENT_ORA_IDX_OLS_POLICY_COL_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("COLUMN_NAME", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_COLUMN_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("LABEL_SE_POLICY_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_OLS_POLICY_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_virtual_tenant_ols_component_real_agent_ora_idx_ols_com_policy_id_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_OLS_COMPONENT_REAL_AGENT_ORA_IDX_OLS_COM_POLICY_ID_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_OLS_COMPONENT_REAL_AGENT_ORA_IDX_OLS_COM_POLICY_ID_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("LABEL_SE_POLICY_ID", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("COMP_TYPE", //column_name - column_id + 4, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("LABEL_SE_COMPONENT_ID", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_OLS_COMPONENT_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_virtual_tenant_ols_label_real_agent_ora_idx_ols_lab_policy_id_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_OLS_LABEL_REAL_AGENT_ORA_IDX_OLS_LAB_POLICY_ID_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_OLS_LABEL_REAL_AGENT_ORA_IDX_OLS_LAB_POLICY_ID_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("LABEL_SE_POLICY_ID", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("LABEL_SE_LABEL_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_OLS_LABEL_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_virtual_tenant_ols_label_real_agent_ora_idx_ols_lab_tag_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_OLS_LABEL_REAL_AGENT_ORA_IDX_OLS_LAB_TAG_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_OLS_LABEL_REAL_AGENT_ORA_IDX_OLS_LAB_TAG_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("LABEL_TAG", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("LABEL_SE_LABEL_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_OLS_LABEL_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_virtual_tenant_ols_label_real_agent_ora_idx_ols_lab_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_OLS_LABEL_REAL_AGENT_ORA_IDX_OLS_LAB_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_OLS_LABEL_REAL_AGENT_ORA_IDX_OLS_LAB_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("LABEL", //column_name - column_id + 5, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_COLUMN_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("LABEL_SE_LABEL_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_OLS_LABEL_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_virtual_tenant_ols_user_level_real_agent_ora_idx_ols_level_uid_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_OLS_USER_LEVEL_REAL_AGENT_ORA_IDX_OLS_LEVEL_UID_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_OLS_USER_LEVEL_REAL_AGENT_ORA_IDX_OLS_LEVEL_UID_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("USER_ID", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("LABEL_SE_USER_LEVEL_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_OLS_USER_LEVEL_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_virtual_tenant_ols_user_level_real_agent_ora_idx_ols_level_policy_id_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_OLS_USER_LEVEL_REAL_AGENT_ORA_IDX_OLS_LEVEL_POLICY_ID_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_OLS_USER_LEVEL_REAL_AGENT_ORA_IDX_OLS_LEVEL_POLICY_ID_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("LABEL_SE_POLICY_ID", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("LABEL_SE_USER_LEVEL_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_OLS_USER_LEVEL_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_virtual_tenant_profile_real_agent_ora_idx_profile_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_PROFILE_REAL_AGENT_ORA_IDX_PROFILE_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_PROFILE_REAL_AGENT_ORA_IDX_PROFILE_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("PROFILE_NAME", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - MAX_ORACLE_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("PROFILE_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_PROFILE_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_virtual_tenant_security_audit_real_agent_ora_idx_audit_type_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_SECURITY_AUDIT_REAL_AGENT_ORA_IDX_AUDIT_TYPE_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_SECURITY_AUDIT_REAL_AGENT_ORA_IDX_AUDIT_TYPE_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("AUDIT_TYPE", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("AUDIT_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_SECURITY_AUDIT_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_virtual_tenant_trigger_real_agent_ora_idx_trigger_base_obj_id_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT_ORA_IDX_TRIGGER_BASE_OBJ_ID_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT_ORA_IDX_TRIGGER_BASE_OBJ_ID_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("BASE_OBJECT_ID", //column_name - column_id + 11, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TRIGGER_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 11); - return ret; -} - -int ObInnerTableSchema::all_virtual_tenant_trigger_real_agent_ora_idx_db_trigger_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT_ORA_IDX_DB_TRIGGER_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT_ORA_IDX_DB_TRIGGER_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("DATABASE_ID", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TRIGGER_NAME", //column_name - column_id + 3, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_TRIGGER_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TRIGGER_ID", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_virtual_tenant_trigger_real_agent_ora_idx_trigger_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT_ORA_IDX_TRIGGER_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT_ORA_IDX_TRIGGER_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TRIGGER_NAME", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_TRIGGER_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TRIGGER_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_virtual_tenant_objauth_real_agent_ora_idx_objauth_grantor_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_OBJAUTH_REAL_AGENT_ORA_IDX_OBJAUTH_GRANTOR_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(7); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_OBJAUTH_REAL_AGENT_ORA_IDX_OBJAUTH_GRANTOR_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("GRANTOR_ID", //column_name - column_id + 5, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("OBJ_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("OBJTYPE", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("COL_ID", //column_name - column_id + 4, //column_id - 5, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("GRANTEE_ID", //column_name - column_id + 6, //column_id - 6, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("PRIV_ID", //column_name - column_id + 7, //column_id - 7, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_OBJAUTH_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 7); - return ret; -} - -int ObInnerTableSchema::all_virtual_tenant_objauth_real_agent_ora_idx_objauth_grantee_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_OBJAUTH_REAL_AGENT_ORA_IDX_OBJAUTH_GRANTEE_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(7); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_OBJAUTH_REAL_AGENT_ORA_IDX_OBJAUTH_GRANTEE_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("GRANTEE_ID", //column_name - column_id + 6, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("OBJ_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("OBJTYPE", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("COL_ID", //column_name - column_id + 4, //column_id - 5, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("GRANTOR_ID", //column_name - column_id + 5, //column_id - 6, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("PRIV_ID", //column_name - column_id + 7, //column_id - 7, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_OBJAUTH_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 7); - return ret; -} - -int ObInnerTableSchema::all_virtual_tenant_object_type_real_agent_ora_idx_obj_type_db_obj_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_OBJECT_TYPE_REAL_AGENT_ORA_IDX_OBJ_TYPE_DB_OBJ_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_OBJECT_TYPE_REAL_AGENT_ORA_IDX_OBJ_TYPE_DB_OBJ_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("DATABASE_ID", //column_name - column_id + 13, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("OBJECT_NAME", //column_name - column_id + 17, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_TABLE_TYPE_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("OBJECT_TYPE_ID", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TYPE", //column_name - column_id + 3, //column_id - 5, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_OBJECT_TYPE_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 17); - return ret; -} - -int ObInnerTableSchema::all_virtual_tenant_object_type_real_agent_ora_idx_obj_type_obj_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_OBJECT_TYPE_REAL_AGENT_ORA_IDX_OBJ_TYPE_OBJ_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_OBJECT_TYPE_REAL_AGENT_ORA_IDX_OBJ_TYPE_OBJ_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("OBJECT_NAME", //column_name - column_id + 17, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_TABLE_TYPE_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("OBJECT_TYPE_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TYPE", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_OBJECT_TYPE_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 17); - return ret; -} - -int ObInnerTableSchema::all_virtual_tenant_dependency_real_agent_ora_idx_dependency_ref_obj_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_DEPENDENCY_REAL_AGENT_ORA_IDX_DEPENDENCY_REF_OBJ_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(4); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_DEPENDENCY_REAL_AGENT_ORA_IDX_DEPENDENCY_REF_OBJ_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("REF_OBJ_ID", //column_name - column_id + 8, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("REF_OBJ_TYPE", //column_name - column_id + 7, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("DEP_OBJ_TYPE", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("DEP_OBJ_ID", //column_name - column_id + 3, //column_id - 5, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("DEP_ORDER", //column_name - column_id + 4, //column_id - 6, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_DEPENDENCY_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 8); - return ret; -} - -int ObInnerTableSchema::all_virtual_table_stat_history_real_agent_ora_idx_table_stat_his_savtime_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TABLE_STAT_HISTORY_REAL_AGENT_ORA_IDX_TABLE_STAT_HIS_SAVTIME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(4); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TABLE_STAT_HISTORY_REAL_AGENT_ORA_IDX_TABLE_STAT_HIS_SAVTIME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("SAVTIME", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObTimestampLTZType, //column_type - CS_TYPE_INVALID, //column_collation_type - 0, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TABLE_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("PARTITION_ID", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TABLE_STAT_HISTORY_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_virtual_column_stat_history_real_agent_ora_idx_column_stat_his_savtime_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_COLUMN_STAT_HISTORY_REAL_AGENT_ORA_IDX_COLUMN_STAT_HIS_SAVTIME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(5); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_COLUMN_STAT_HISTORY_REAL_AGENT_ORA_IDX_COLUMN_STAT_HIS_SAVTIME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("SAVTIME", //column_name - column_id + 5, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObTimestampLTZType, //column_type - CS_TYPE_INVALID, //column_collation_type - 0, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TABLE_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("PARTITION_ID", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("COLUMN_ID", //column_name - column_id + 4, //column_id - 5, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_COLUMN_STAT_HISTORY_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_virtual_histogram_stat_history_real_agent_ora_idx_histogram_stat_his_savtime_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_HISTOGRAM_STAT_HISTORY_REAL_AGENT_ORA_IDX_HISTOGRAM_STAT_HIS_SAVTIME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(6); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_HISTOGRAM_STAT_HISTORY_REAL_AGENT_ORA_IDX_HISTOGRAM_STAT_HIS_SAVTIME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("SAVTIME", //column_name - column_id + 6, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObTimestampLTZType, //column_type - CS_TYPE_INVALID, //column_collation_type - 0, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TABLE_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("PARTITION_ID", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("COLUMN_ID", //column_name - column_id + 4, //column_id - 5, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("ENDPOINT_NUM", //column_name - column_id + 5, //column_id - 6, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_HISTOGRAM_STAT_HISTORY_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 6); - return ret; -} - -int ObInnerTableSchema::all_virtual_tablet_to_ls_real_agent_ora_idx_tablet_to_ls_id_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TABLET_TO_LS_REAL_AGENT_ORA_IDX_TABLET_TO_LS_ID_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(1); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TABLET_TO_LS_REAL_AGENT_ORA_IDX_TABLET_TO_LS_ID_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("LS_ID", //column_name - column_id + 2, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TABLET_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TABLET_TO_LS_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 2); - return ret; -} - -int ObInnerTableSchema::all_virtual_tablet_to_ls_real_agent_ora_idx_tablet_to_table_id_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TABLET_TO_LS_REAL_AGENT_ORA_IDX_TABLET_TO_TABLE_ID_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(1); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TABLET_TO_LS_REAL_AGENT_ORA_IDX_TABLET_TO_TABLE_ID_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TABLE_ID", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TABLET_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TABLET_TO_LS_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_virtual_context_real_agent_ora_idx_ctx_namespace_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_CONTEXT_REAL_AGENT_ORA_IDX_CTX_NAMESPACE_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_CONTEXT_REAL_AGENT_ORA_IDX_CTX_NAMESPACE_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("NAMESPACE", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_CONTEXT_STRING_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("CONTEXT_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_CONTEXT_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_virtual_plan_baseline_item_real_agent_ora_idx_spm_item_sql_id_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_PLAN_BASELINE_ITEM_REAL_AGENT_ORA_IDX_SPM_ITEM_SQL_ID_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(4); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_PLAN_BASELINE_ITEM_REAL_AGENT_ORA_IDX_SPM_ITEM_SQL_ID_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("SQL_ID", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_SQL_ID_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("DATABASE_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("PLAN_HASH_VALUE", //column_name - column_id + 4, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_PLAN_BASELINE_ITEM_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_virtual_plan_baseline_item_real_agent_ora_idx_spm_item_value_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_PLAN_BASELINE_ITEM_REAL_AGENT_ORA_IDX_SPM_ITEM_VALUE_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(4); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_PLAN_BASELINE_ITEM_REAL_AGENT_ORA_IDX_SPM_ITEM_VALUE_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("PLAN_HASH_VALUE", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("DATABASE_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("SQL_ID", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_SQL_ID_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_PLAN_BASELINE_ITEM_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_virtual_tenant_directory_real_agent_ora_idx_directory_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_DIRECTORY_REAL_AGENT_ORA_IDX_DIRECTORY_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_DIRECTORY_REAL_AGENT_ORA_IDX_DIRECTORY_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("DIRECTORY_NAME", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - 128, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("DIRECTORY_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_DIRECTORY_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_virtual_job_real_agent_ora_idx_job_powner_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_JOB_REAL_AGENT_ORA_IDX_JOB_POWNER_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_JOB_REAL_AGENT_ORA_IDX_JOB_POWNER_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("POWNER", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_DATABASE_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("JOB", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_JOB_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_virtual_sequence_object_real_agent_ora_idx_seq_obj_db_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_SEQUENCE_OBJECT_REAL_AGENT_ORA_IDX_SEQ_OBJ_DB_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_SEQUENCE_OBJECT_REAL_AGENT_ORA_IDX_SEQ_OBJ_DB_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("DATABASE_ID", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("SEQUENCE_NAME", //column_name - column_id + 5, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_SEQUENCE_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("SEQUENCE_ID", //column_name - column_id + 2, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_SEQUENCE_OBJECT_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_virtual_sequence_object_real_agent_ora_idx_seq_obj_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_SEQUENCE_OBJECT_REAL_AGENT_ORA_IDX_SEQ_OBJ_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_SEQUENCE_OBJECT_REAL_AGENT_ORA_IDX_SEQ_OBJ_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("SEQUENCE_NAME", //column_name - column_id + 5, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_SEQUENCE_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("SEQUENCE_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_SEQUENCE_OBJECT_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_virtual_recyclebin_real_agent_ora_idx_recyclebin_ori_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_RECYCLEBIN_REAL_AGENT_ORA_IDX_RECYCLEBIN_ORI_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_RECYCLEBIN_REAL_AGENT_ORA_IDX_RECYCLEBIN_ORI_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("ORIGINAL_NAME", //column_name - column_id + 7, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_ORIGINAL_NANE_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("OBJECT_NAME", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_OBJECT_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TYPE", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_RECYCLEBIN_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 7); - return ret; -} - -int ObInnerTableSchema::all_virtual_table_privilege_real_agent_ora_idx_tb_priv_db_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TABLE_PRIVILEGE_REAL_AGENT_ORA_IDX_TB_PRIV_DB_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(4); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TABLE_PRIVILEGE_REAL_AGENT_ORA_IDX_TB_PRIV_DB_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("DATABASE_NAME", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_DATABASE_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("USER_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TABLE_NAME", //column_name - column_id + 4, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_CORE_TALBE_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TABLE_PRIVILEGE_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_virtual_table_privilege_real_agent_ora_idx_tb_priv_tb_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_TABLE_PRIVILEGE_REAL_AGENT_ORA_IDX_TB_PRIV_TB_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(4); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TABLE_PRIVILEGE_REAL_AGENT_ORA_IDX_TB_PRIV_TB_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TABLE_NAME", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_CORE_TALBE_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("USER_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("DATABASE_NAME", //column_name - column_id + 3, //column_id - 4, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_DATABASE_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_TABLE_PRIVILEGE_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_virtual_database_privilege_real_agent_ora_idx_db_priv_db_name_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_DATABASE_PRIVILEGE_REAL_AGENT_ORA_IDX_DB_PRIV_DB_NAME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_DATABASE_PRIVILEGE_REAL_AGENT_ORA_IDX_DB_PRIV_DB_NAME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("DATABASE_NAME", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_DATABASE_NAME_LENGTH, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("USER_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_DATABASE_PRIVILEGE_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_virtual_rls_policy_real_agent_ora_idx_rls_policy_table_id_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_RLS_POLICY_REAL_AGENT_ORA_IDX_RLS_POLICY_TABLE_ID_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_RLS_POLICY_REAL_AGENT_ORA_IDX_RLS_POLICY_TABLE_ID_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TABLE_ID", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("RLS_POLICY_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_RLS_POLICY_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_virtual_rls_policy_real_agent_ora_idx_rls_policy_group_id_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_RLS_POLICY_REAL_AGENT_ORA_IDX_RLS_POLICY_GROUP_ID_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_RLS_POLICY_REAL_AGENT_ORA_IDX_RLS_POLICY_GROUP_ID_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("RLS_GROUP_ID", //column_name - column_id + 5, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("RLS_POLICY_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_RLS_POLICY_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 5); - return ret; -} - -int ObInnerTableSchema::all_virtual_rls_group_real_agent_ora_idx_rls_group_table_id_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_RLS_GROUP_REAL_AGENT_ORA_IDX_RLS_GROUP_TABLE_ID_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_RLS_GROUP_REAL_AGENT_ORA_IDX_RLS_GROUP_TABLE_ID_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TABLE_ID", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("RLS_GROUP_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_RLS_GROUP_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_virtual_rls_context_real_agent_ora_idx_rls_context_table_id_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_RLS_CONTEXT_REAL_AGENT_ORA_IDX_RLS_CONTEXT_TABLE_ID_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_RLS_CONTEXT_REAL_AGENT_ORA_IDX_RLS_CONTEXT_TABLE_ID_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TABLE_ID", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("RLS_CONTEXT_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_RLS_CONTEXT_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_virtual_dbms_lock_allocated_real_agent_ora_idx_dbms_lock_allocated_lockhandle_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_DBMS_LOCK_ALLOCATED_REAL_AGENT_ORA_IDX_DBMS_LOCK_ALLOCATED_LOCKHANDLE_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(1); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_DBMS_LOCK_ALLOCATED_REAL_AGENT_ORA_IDX_DBMS_LOCK_ALLOCATED_LOCKHANDLE_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("LOCKHANDLE", //column_name - column_id + 3, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - 128, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("NAME", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - 128, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_DBMS_LOCK_ALLOCATED_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_virtual_dbms_lock_allocated_real_agent_ora_idx_dbms_lock_allocated_expiration_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_DBMS_LOCK_ALLOCATED_REAL_AGENT_ORA_IDX_DBMS_LOCK_ALLOCATED_EXPIRATION_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(1); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_DBMS_LOCK_ALLOCATED_REAL_AGENT_ORA_IDX_DBMS_LOCK_ALLOCATED_EXPIRATION_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("EXPIRATION", //column_name - column_id + 4, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObTimestampLTZType, //column_type - CS_TYPE_INVALID, //column_collation_type - 0, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("NAME", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - 128, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_DBMS_LOCK_ALLOCATED_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 4); - return ret; -} - -int ObInnerTableSchema::all_virtual_user_proxy_info_real_agent_ora_idx_user_proxy_info_proxy_user_id_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_USER_PROXY_INFO_REAL_AGENT_ORA_IDX_USER_PROXY_INFO_PROXY_USER_ID_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(3); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_USER_PROXY_INFO_REAL_AGENT_ORA_IDX_USER_PROXY_INFO_PROXY_USER_ID_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TENANT_ID", //column_name - column_id + 1, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("PROXY_USER_ID", //column_name - column_id + 3, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("CLIENT_USER_ID", //column_name - column_id + 2, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_USER_PROXY_INFO_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 3); - return ret; -} - -int ObInnerTableSchema::all_virtual_scheduler_job_run_detail_v2_real_agent_ora_idx_scheduler_job_run_detail_v2_time_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_SCHEDULER_JOB_RUN_DETAIL_V2_REAL_AGENT_ORA_IDX_SCHEDULER_JOB_RUN_DETAIL_V2_TIME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_SCHEDULER_JOB_RUN_DETAIL_V2_REAL_AGENT_ORA_IDX_SCHEDULER_JOB_RUN_DETAIL_V2_TIME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TIME", //column_name - column_id + 2, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObTimestampLTZType, //column_type - CS_TYPE_INVALID, //column_collation_type - 0, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("JOB_NAME", //column_name - column_id + 1, //column_id - 2, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - 128, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_SCHEDULER_JOB_RUN_DETAIL_V2_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 2); - return ret; -} - -int ObInnerTableSchema::all_virtual_scheduler_job_run_detail_v2_real_agent_ora_idx_scheduler_job_run_detail_v2_job_class_time_real_agent_schema(ObTableSchema &table_schema) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; - - //generated fields: - table_schema.set_tenant_id(OB_SYS_TENANT_ID); - table_schema.set_tablegroup_id(OB_INVALID_ID); - table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); - table_schema.set_table_id(OB_ALL_VIRTUAL_SCHEDULER_JOB_RUN_DETAIL_V2_REAL_AGENT_ORA_IDX_SCHEDULER_JOB_RUN_DETAIL_V2_JOB_CLASS_TIME_REAL_AGENT_TID); - table_schema.set_rowkey_split_pos(0); - table_schema.set_is_use_bloomfilter(false); - table_schema.set_progressive_merge_num(0); - table_schema.set_rowkey_column_num(2); - table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); - table_schema.set_table_type(USER_INDEX); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_SCHEDULER_JOB_RUN_DETAIL_V2_REAL_AGENT_ORA_IDX_SCHEDULER_JOB_RUN_DETAIL_V2_JOB_CLASS_TIME_REAL_AGENT_TNAME))) { - LOG_ERROR("fail to set table_name", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { - LOG_ERROR("fail to set compress_func_name", K(ret)); - } - } - table_schema.set_part_level(PARTITION_LEVEL_ZERO); - table_schema.set_charset_type(ObCharset::get_default_charset()); - table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); - table_schema.set_index_using_type(USING_BTREE); - table_schema.set_row_store_type(ENCODING_ROW_STORE); - table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); - table_schema.set_progressive_merge_round(1); - table_schema.set_storage_format_version(3); - table_schema.set_tablet_id(0); - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("JOB_CLASS", //column_name - column_id + 8, //column_id - 1, //rowkey_id - 1, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - 128, //column_length - 2, //column_precision - -1, //column_scale - true,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("TIME", //column_name - column_id + 2, //column_id - 2, //rowkey_id - 2, //index_id - 0, //part_key_pos - ObTimestampLTZType, //column_type - CS_TYPE_INVALID, //column_collation_type - 0, //column_length - -1, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("JOB_NAME", //column_name - column_id + 1, //column_id - 3, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - 128, //column_length - 2, //column_precision - -1, //column_scale - false,//is_nullable - false); //is_autoincrement - } - table_schema.set_index_status(INDEX_STATUS_AVAILABLE); - table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); - table_schema.set_data_table_id(OB_ALL_VIRTUAL_SCHEDULER_JOB_RUN_DETAIL_V2_REAL_AGENT_ORA_TID); - - table_schema.set_max_used_column_id(column_id + 8); - return ret; -} - } // end namespace share } // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema.28251_28300.cpp b/src/share/inner_table/ob_inner_table_schema.28251_28300.cpp new file mode 100644 index 000000000..4d535519e --- /dev/null +++ b/src/share/inner_table/ob_inner_table_schema.28251_28300.cpp @@ -0,0 +1,19830 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SHARE_SCHEMA +#include "ob_inner_table_schema.h" + +#include "share/schema/ob_schema_macro_define.h" +#include "share/schema/ob_schema_service_sql_impl.h" +#include "share/schema/ob_table_schema.h" +#include "share/scn.h" + +namespace oceanbase +{ +using namespace share::schema; +using namespace common; +namespace share +{ + +int ObInnerTableSchema::dba_ob_temp_files_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_OB_TEMP_FILES_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_OB_TEMP_FILES_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT SVR_IP, SVR_PORT, FILE_ID, TRACE_ID, DIR_ID, BYTES, START_OFFSET, TOTAL_WRITES, UNALIGNED_WRITES, TOTAL_READS, UNALIGNED_READS, TOTAL_READ_BYTES, LAST_ACCESS_TIME, LAST_MODIFY_TIME, BIRTH_TIME FROM SYS.ALL_VIRTUAL_TEMP_FILE WHERE TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::all_table_idx_data_table_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TABLE_IDX_DATA_TABLE_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TABLE_IDX_DATA_TABLE_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TABLE_IDX_DATA_TABLE_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("data_table_id", //column_name + column_id + 21, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TABLE_TID); + + table_schema.set_max_used_column_id(column_id + 21); + return ret; +} + +int ObInnerTableSchema::all_table_idx_db_tb_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TABLE_IDX_DB_TB_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TABLE_IDX_DB_TB_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TABLE_IDX_DB_TB_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("database_id", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ObObj table_name_default; + table_name_default.set_varchar(ObString::make_string("")); + ADD_COLUMN_SCHEMA_T("table_name", //column_name + column_id + 3, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_TABLE_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + table_name_default, + table_name_default); //default_value + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TABLE_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_table_idx_tb_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TABLE_IDX_TB_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TABLE_IDX_TB_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TABLE_IDX_TB_NAME_TID); + + if (OB_SUCC(ret)) { + ObObj table_name_default; + table_name_default.set_varchar(ObString::make_string("")); + ADD_COLUMN_SCHEMA_T("table_name", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_TABLE_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + table_name_default, + table_name_default); //default_value + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TABLE_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_column_idx_tb_column_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_COLUMN_IDX_TB_COLUMN_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_COLUMN_IDX_TB_COLUMN_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_COLUMN_IDX_TB_COLUMN_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + column_id + 2, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ObObj column_name_default; + column_name_default.set_varchar(ObString::make_string("")); + ADD_COLUMN_SCHEMA_T("column_name", //column_name + column_id + 4, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_COLUMN_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + column_name_default, + column_name_default); //default_value + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("column_id", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_COLUMN_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_column_idx_column_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_COLUMN_IDX_COLUMN_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_COLUMN_IDX_COLUMN_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_COLUMN_IDX_COLUMN_NAME_TID); + + if (OB_SUCC(ret)) { + ObObj column_name_default; + column_name_default.set_varchar(ObString::make_string("")); + ADD_COLUMN_SCHEMA_T("column_name", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_COLUMN_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + column_name_default, + column_name_default); //default_value + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("column_id", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_COLUMN_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_ddl_operation_idx_ddl_type_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_DDL_OPERATION_IDX_DDL_TYPE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(1); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_DDL_OPERATION_IDX_DDL_TYPE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_DDL_OPERATION_IDX_DDL_TYPE_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("operation_type", //column_name + column_id + 9, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("schema_version", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_DDL_OPERATION_TID); + + table_schema.set_max_used_column_id(column_id + 9); + return ret; +} + +int ObInnerTableSchema::all_table_history_idx_data_table_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TABLE_HISTORY_IDX_DATA_TABLE_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TABLE_HISTORY_IDX_DATA_TABLE_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TABLE_HISTORY_IDX_DATA_TABLE_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("data_table_id", //column_name + column_id + 23, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("schema_version", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TABLE_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id + 23); + return ret; +} + +int ObInnerTableSchema::all_log_archive_piece_files_idx_status_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_LOG_ARCHIVE_PIECE_FILES_IDX_STATUS_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_LOG_ARCHIVE_PIECE_FILES_IDX_STATUS_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_LOG_ARCHIVE_PIECE_FILES_IDX_STATUS_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("dest_id", //column_name + column_id + 2, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ObObj file_status_default; + file_status_default.set_varchar(ObString::make_string("INVALID")); + ADD_COLUMN_SCHEMA_T("file_status", //column_name + column_id + 18, //column_id + 3, //rowkey_id + 3, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_DEFAULT_STATUS_LENTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + file_status_default, + file_status_default); //default_value + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("round_id", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("piece_id", //column_name + column_id + 4, //column_id + 5, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_LOG_ARCHIVE_PIECE_FILES_TID); + + table_schema.set_max_used_column_id(column_id + 18); + return ret; +} + +int ObInnerTableSchema::all_backup_set_files_idx_status_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_BACKUP_SET_FILES_IDX_STATUS_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_BACKUP_SET_FILES_IDX_STATUS_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_BACKUP_SET_FILES_IDX_STATUS_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("dest_id", //column_name + column_id + 3, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("file_status", //column_name + column_id + 11, //column_id + 3, //rowkey_id + 3, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_DEFAULT_STATUS_LENTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("backup_set_id", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_BACKUP_SET_FILES_TID); + + table_schema.set_max_used_column_id(column_id + 11); + return ret; +} + +int ObInnerTableSchema::all_ddl_task_status_idx_task_key_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_DDL_TASK_STATUS_IDX_TASK_KEY_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(1); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_UNIQUE_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_DDL_TASK_STATUS_IDX_TASK_KEY_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_DDL_TASK_STATUS_IDX_TASK_KEY_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("target_object_id", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("object_id", //column_name + column_id + 3, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("schema_version", //column_name + column_id + 6, //column_id + 3, //rowkey_id + 3, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_WITH_COLUMN_FLAGS("shadow_pk_0", //column_name + column_id + 32768, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true,//is_nullable + false,//is_autoincrement + true,//is_hidden + false);//is_storing_column + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_id", //column_name + column_id + 1, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_UNIQUE_LOCAL); + table_schema.set_data_table_id(OB_ALL_DDL_TASK_STATUS_TID); + + table_schema.set_max_used_column_id(column_id + 32768); + return ret; +} + +int ObInnerTableSchema::all_user_idx_ur_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_USER_IDX_UR_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_USER_IDX_UR_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_USER_IDX_UR_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("user_name", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_USER_NAME_LENGTH_STORE, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("user_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_USER_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_database_idx_db_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_DATABASE_IDX_DB_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_DATABASE_IDX_DB_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_DATABASE_IDX_DB_NAME_TID); + + if (OB_SUCC(ret)) { + ObObj database_name_default; + database_name_default.set_varchar(ObString::make_string("")); + ADD_COLUMN_SCHEMA_T("database_name", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_DATABASE_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + database_name_default, + database_name_default); //default_value + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("database_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_DATABASE_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_tablegroup_idx_tg_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TABLEGROUP_IDX_TG_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TABLEGROUP_IDX_TG_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TABLEGROUP_IDX_TG_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tablegroup_name", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_TABLEGROUP_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tablegroup_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TABLEGROUP_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_tenant_history_idx_tenant_deleted_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TENANT_HISTORY_IDX_TENANT_DELETED_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_HISTORY_IDX_TENANT_DELETED_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TENANT_HISTORY_IDX_TENANT_DELETED_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("is_deleted", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("schema_version", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TENANT_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_rootservice_event_history_idx_rs_module_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_ROOTSERVICE_EVENT_HISTORY_IDX_RS_MODULE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(1); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_ROOTSERVICE_EVENT_HISTORY_IDX_RS_MODULE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_ROOTSERVICE_EVENT_HISTORY_IDX_RS_MODULE_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("module", //column_name + column_id + 2, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_ROOTSERVICE_EVENT_DESC_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ObObj gmt_default; + ObObj gmt_default_null; + + gmt_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("gmt_create", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + 6, //column_scale + false, //is_nullable + false, //is_autoincrement + false, //is_on_update_for_timestamp + gmt_default_null, + gmt_default); + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_ROOTSERVICE_EVENT_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id + 2); + return ret; +} + +int ObInnerTableSchema::all_rootservice_event_history_idx_rs_event_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_ROOTSERVICE_EVENT_HISTORY_IDX_RS_EVENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(1); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_ROOTSERVICE_EVENT_HISTORY_IDX_RS_EVENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_ROOTSERVICE_EVENT_HISTORY_IDX_RS_EVENT_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("event", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_ROOTSERVICE_EVENT_DESC_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ObObj gmt_default; + ObObj gmt_default_null; + + gmt_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("gmt_create", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + 6, //column_scale + false, //is_nullable + false, //is_autoincrement + false, //is_on_update_for_timestamp + gmt_default_null, + gmt_default); + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_ROOTSERVICE_EVENT_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_recyclebin_idx_recyclebin_db_type_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_RECYCLEBIN_IDX_RECYCLEBIN_DB_TYPE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_RECYCLEBIN_IDX_RECYCLEBIN_DB_TYPE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_RECYCLEBIN_IDX_RECYCLEBIN_DB_TYPE_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("database_id", //column_name + column_id + 4, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("type", //column_name + column_id + 3, //column_id + 3, //rowkey_id + 3, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("object_name", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_OBJECT_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_RECYCLEBIN_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_part_idx_part_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_PART_IDX_PART_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_PART_IDX_PART_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_PART_IDX_PART_NAME_TID); + + if (OB_SUCC(ret)) { + ObObj part_name_default; + part_name_default.set_varchar(ObString::make_string("")); + ADD_COLUMN_SCHEMA_T("part_name", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_PARTITION_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + part_name_default, + part_name_default); //default_value + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("part_id", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_PART_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_sub_part_idx_sub_part_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_SUB_PART_IDX_SUB_PART_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_SUB_PART_IDX_SUB_PART_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_SUB_PART_IDX_SUB_PART_NAME_TID); + + if (OB_SUCC(ret)) { + ObObj sub_part_name_default; + sub_part_name_default.set_varchar(ObString::make_string("")); + ADD_COLUMN_SCHEMA_T("sub_part_name", //column_name + column_id + 5, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_PARTITION_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + sub_part_name_default, + sub_part_name_default); //default_value + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("part_id", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("sub_part_id", //column_name + column_id + 4, //column_id + 5, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_SUB_PART_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_def_sub_part_idx_def_sub_part_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_DEF_SUB_PART_IDX_DEF_SUB_PART_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_DEF_SUB_PART_IDX_DEF_SUB_PART_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_DEF_SUB_PART_IDX_DEF_SUB_PART_NAME_TID); + + if (OB_SUCC(ret)) { + ObObj sub_part_name_default; + sub_part_name_default.set_varchar(ObString::make_string("")); + ADD_COLUMN_SCHEMA_T("sub_part_name", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_PARTITION_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + sub_part_name_default, + sub_part_name_default); //default_value + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("sub_part_id", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_DEF_SUB_PART_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_server_event_history_idx_server_module_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_SERVER_EVENT_HISTORY_IDX_SERVER_MODULE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_SERVER_EVENT_HISTORY_IDX_SERVER_MODULE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_SERVER_EVENT_HISTORY_IDX_SERVER_MODULE_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("module", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_ROOTSERVICE_EVENT_DESC_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ObObj gmt_default; + ObObj gmt_default_null; + + gmt_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("gmt_create", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + 6, //column_scale + false, //is_nullable + false, //is_autoincrement + false, //is_on_update_for_timestamp + gmt_default_null, + gmt_default); + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("svr_ip", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_IP_ADDR_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("svr_port", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_SERVER_EVENT_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_server_event_history_idx_server_event_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_SERVER_EVENT_HISTORY_IDX_SERVER_EVENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_SERVER_EVENT_HISTORY_IDX_SERVER_EVENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_SERVER_EVENT_HISTORY_IDX_SERVER_EVENT_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("event", //column_name + column_id + 5, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_ROOTSERVICE_EVENT_DESC_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ObObj gmt_default; + ObObj gmt_default_null; + + gmt_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("gmt_create", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + 6, //column_scale + false, //is_nullable + false, //is_autoincrement + false, //is_on_update_for_timestamp + gmt_default_null, + gmt_default); + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("svr_ip", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_IP_ADDR_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("svr_port", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_SERVER_EVENT_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_rootservice_job_idx_rs_job_type_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_ROOTSERVICE_JOB_IDX_RS_JOB_TYPE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(1); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_ROOTSERVICE_JOB_IDX_RS_JOB_TYPE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_ROOTSERVICE_JOB_IDX_RS_JOB_TYPE_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("job_type", //column_name + column_id + 2, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + 128, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("job_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_ROOTSERVICE_JOB_TID); + + table_schema.set_max_used_column_id(column_id + 2); + return ret; +} + +int ObInnerTableSchema::all_foreign_key_idx_fk_child_tid_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_FOREIGN_KEY_IDX_FK_CHILD_TID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_FOREIGN_KEY_IDX_FK_CHILD_TID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_FOREIGN_KEY_IDX_FK_CHILD_TID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("child_table_id", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("foreign_key_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_FOREIGN_KEY_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_foreign_key_idx_fk_parent_tid_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_FOREIGN_KEY_IDX_FK_PARENT_TID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_FOREIGN_KEY_IDX_FK_PARENT_TID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_FOREIGN_KEY_IDX_FK_PARENT_TID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("parent_table_id", //column_name + column_id + 5, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("foreign_key_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_FOREIGN_KEY_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_foreign_key_idx_fk_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_FOREIGN_KEY_IDX_FK_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_FOREIGN_KEY_IDX_FK_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_FOREIGN_KEY_IDX_FK_NAME_TID); + + if (OB_SUCC(ret)) { + ObObj foreign_key_name_default; + foreign_key_name_default.set_varchar(ObString::make_string("")); + ADD_COLUMN_SCHEMA_T("foreign_key_name", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_CONSTRAINT_NAME_LENGTH_ORACLE, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + foreign_key_name_default, + foreign_key_name_default); //default_value + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("foreign_key_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_FOREIGN_KEY_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_foreign_key_history_idx_fk_his_child_tid_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_FOREIGN_KEY_HISTORY_IDX_FK_HIS_CHILD_TID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_FOREIGN_KEY_HISTORY_IDX_FK_HIS_CHILD_TID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_FOREIGN_KEY_HISTORY_IDX_FK_HIS_CHILD_TID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("child_table_id", //column_name + column_id + 6, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("schema_version", //column_name + column_id + 3, //column_id + 3, //rowkey_id + 3, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("foreign_key_id", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_FOREIGN_KEY_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id + 6); + return ret; +} + +int ObInnerTableSchema::all_foreign_key_history_idx_fk_his_parent_tid_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_FOREIGN_KEY_HISTORY_IDX_FK_HIS_PARENT_TID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_FOREIGN_KEY_HISTORY_IDX_FK_HIS_PARENT_TID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_FOREIGN_KEY_HISTORY_IDX_FK_HIS_PARENT_TID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("parent_table_id", //column_name + column_id + 7, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("schema_version", //column_name + column_id + 3, //column_id + 3, //rowkey_id + 3, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("foreign_key_id", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_FOREIGN_KEY_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id + 7); + return ret; +} + +int ObInnerTableSchema::all_synonym_idx_db_synonym_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_SYNONYM_IDX_DB_SYNONYM_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_SYNONYM_IDX_DB_SYNONYM_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_SYNONYM_IDX_DB_SYNONYM_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("database_id", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ObObj synonym_name_default; + synonym_name_default.set_varchar(ObString::make_string("")); + ADD_COLUMN_SCHEMA_T("synonym_name", //column_name + column_id + 5, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_SYNONYM_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + synonym_name_default, + synonym_name_default); //default_value + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("synonym_id", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_SYNONYM_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_synonym_idx_synonym_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_SYNONYM_IDX_SYNONYM_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_SYNONYM_IDX_SYNONYM_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_SYNONYM_IDX_SYNONYM_NAME_TID); + + if (OB_SUCC(ret)) { + ObObj synonym_name_default; + synonym_name_default.set_varchar(ObString::make_string("")); + ADD_COLUMN_SCHEMA_T("synonym_name", //column_name + column_id + 5, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_SYNONYM_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + synonym_name_default, + synonym_name_default); //default_value + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("synonym_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_SYNONYM_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_ddl_checksum_idx_ddl_checksum_task_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_DDL_CHECKSUM_IDX_DDL_CHECKSUM_TASK_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(6); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_DDL_CHECKSUM_IDX_DDL_CHECKSUM_TASK_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_DDL_CHECKSUM_IDX_DDL_CHECKSUM_TASK_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ddl_task_id", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("execution_id", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("column_id", //column_name + column_id + 5, //column_id + 5, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_id", //column_name + column_id + 6, //column_id + 6, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_DDL_CHECKSUM_TID); + + table_schema.set_max_used_column_id(column_id + 6); + return ret; +} + +int ObInnerTableSchema::all_routine_idx_db_routine_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_ROUTINE_IDX_DB_ROUTINE_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_ROUTINE_IDX_DB_ROUTINE_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_ROUTINE_IDX_DB_ROUTINE_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("database_id", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("routine_name", //column_name + column_id + 5, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_ROUTINE_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("routine_id", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_ROUTINE_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_routine_idx_routine_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_ROUTINE_IDX_ROUTINE_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_ROUTINE_IDX_ROUTINE_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_ROUTINE_IDX_ROUTINE_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("routine_name", //column_name + column_id + 5, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_ROUTINE_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("routine_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_ROUTINE_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_routine_idx_routine_pkg_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_ROUTINE_IDX_ROUTINE_PKG_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_ROUTINE_IDX_ROUTINE_PKG_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_ROUTINE_IDX_ROUTINE_PKG_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("package_id", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("routine_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_ROUTINE_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_routine_param_idx_routine_param_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_ROUTINE_PARAM_IDX_ROUTINE_PARAM_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_ROUTINE_PARAM_IDX_ROUTINE_PARAM_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_ROUTINE_PARAM_IDX_ROUTINE_PARAM_NAME_TID); + + if (OB_SUCC(ret)) { + ObObj param_name_default; + param_name_default.set_varchar(ObString::make_string("")); + ADD_COLUMN_SCHEMA_T("param_name", //column_name + column_id + 7, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_COLUMN_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false, //is_autoincrement + param_name_default, + param_name_default); //default_value + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("routine_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("sequence", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_ROUTINE_PARAM_TID); + + table_schema.set_max_used_column_id(column_id + 7); + return ret; +} + +int ObInnerTableSchema::all_package_idx_db_pkg_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_PACKAGE_IDX_DB_PKG_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_PACKAGE_IDX_DB_PKG_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_PACKAGE_IDX_DB_PKG_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("database_id", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ObObj package_name_default; + package_name_default.set_varchar(ObString::make_string("")); + ADD_COLUMN_SCHEMA_T("package_name", //column_name + column_id + 4, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_PACKAGE_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + package_name_default, + package_name_default); //default_value + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("package_id", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_PACKAGE_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_package_idx_pkg_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_PACKAGE_IDX_PKG_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_PACKAGE_IDX_PKG_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_PACKAGE_IDX_PKG_NAME_TID); + + if (OB_SUCC(ret)) { + ObObj package_name_default; + package_name_default.set_varchar(ObString::make_string("")); + ADD_COLUMN_SCHEMA_T("package_name", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_PACKAGE_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + package_name_default, + package_name_default); //default_value + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("package_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_PACKAGE_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_acquired_snapshot_idx_snapshot_tablet_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_ACQUIRED_SNAPSHOT_IDX_SNAPSHOT_TABLET_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_ACQUIRED_SNAPSHOT_IDX_SNAPSHOT_TABLET_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_ACQUIRED_SNAPSHOT_IDX_SNAPSHOT_TABLET_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tablet_id", //column_name + column_id + 6, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ObObj gmt_default; + ObObj gmt_default_null; + + gmt_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("gmt_create", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + 6, //column_scale + false, //is_nullable + false, //is_autoincrement + false, //is_on_update_for_timestamp + gmt_default_null, + gmt_default); + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_ACQUIRED_SNAPSHOT_TID); + + table_schema.set_max_used_column_id(column_id + 6); + return ret; +} + +int ObInnerTableSchema::all_constraint_idx_cst_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_CONSTRAINT_IDX_CST_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_CONSTRAINT_IDX_CST_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_CONSTRAINT_IDX_CST_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("constraint_name", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_CONSTRAINT_NAME_LENGTH_ORACLE, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("constraint_id", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_CONSTRAINT_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_type_idx_db_type_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TYPE_IDX_DB_TYPE_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TYPE_IDX_DB_TYPE_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TYPE_IDX_DB_TYPE_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("database_id", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("type_name", //column_name + column_id + 18, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_TABLE_TYPE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("type_id", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TYPE_TID); + + table_schema.set_max_used_column_id(column_id + 18); + return ret; +} + +int ObInnerTableSchema::all_type_idx_type_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TYPE_IDX_TYPE_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TYPE_IDX_TYPE_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TYPE_IDX_TYPE_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("type_name", //column_name + column_id + 18, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_TABLE_TYPE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("type_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TYPE_TID); + + table_schema.set_max_used_column_id(column_id + 18); + return ret; +} + +int ObInnerTableSchema::all_type_attr_idx_type_attr_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TYPE_ATTR_IDX_TYPE_ATTR_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TYPE_ATTR_IDX_TYPE_ATTR_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TYPE_ATTR_IDX_TYPE_ATTR_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("name", //column_name + column_id + 6, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_TABLE_TYPE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("type_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("attribute", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TYPE_ATTR_TID); + + table_schema.set_max_used_column_id(column_id + 6); + return ret; +} + +int ObInnerTableSchema::all_coll_type_idx_coll_name_type_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_COLL_TYPE_IDX_COLL_NAME_TYPE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_COLL_TYPE_IDX_COLL_NAME_TYPE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_COLL_TYPE_IDX_COLL_NAME_TYPE_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("coll_name", //column_name + column_id + 16, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_TABLE_TYPE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("coll_type", //column_name + column_id + 13, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("coll_type_id", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_COLL_TYPE_TID); + + table_schema.set_max_used_column_id(column_id + 16); + return ret; +} + +int ObInnerTableSchema::all_dblink_idx_owner_dblink_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_DBLINK_IDX_OWNER_DBLINK_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_DBLINK_IDX_OWNER_DBLINK_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_DBLINK_IDX_OWNER_DBLINK_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("owner_id", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("dblink_name", //column_name + column_id + 3, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_DBLINK_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("dblink_id", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_DBLINK_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_dblink_idx_dblink_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_DBLINK_IDX_DBLINK_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_DBLINK_IDX_DBLINK_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_DBLINK_IDX_DBLINK_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("dblink_name", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_DBLINK_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("dblink_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_DBLINK_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_tenant_role_grantee_map_idx_grantee_role_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TENANT_ROLE_GRANTEE_MAP_IDX_GRANTEE_ROLE_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_ROLE_GRANTEE_MAP_IDX_GRANTEE_ROLE_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TENANT_ROLE_GRANTEE_MAP_IDX_GRANTEE_ROLE_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("role_id", //column_name + column_id + 3, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("grantee_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TENANT_ROLE_GRANTEE_MAP_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_tenant_role_grantee_map_history_idx_grantee_his_role_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TENANT_ROLE_GRANTEE_MAP_HISTORY_IDX_GRANTEE_HIS_ROLE_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_ROLE_GRANTEE_MAP_HISTORY_IDX_GRANTEE_HIS_ROLE_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TENANT_ROLE_GRANTEE_MAP_HISTORY_IDX_GRANTEE_HIS_ROLE_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("role_id", //column_name + column_id + 3, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("schema_version", //column_name + column_id + 4, //column_id + 3, //rowkey_id + 3, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("grantee_id", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TENANT_ROLE_GRANTEE_MAP_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_tenant_keystore_idx_keystore_master_key_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TENANT_KEYSTORE_IDX_KEYSTORE_MASTER_KEY_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_KEYSTORE_IDX_KEYSTORE_MASTER_KEY_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TENANT_KEYSTORE_IDX_KEYSTORE_MASTER_KEY_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("master_key_id", //column_name + column_id + 6, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("keystore_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TENANT_KEYSTORE_TID); + + table_schema.set_max_used_column_id(column_id + 6); + return ret; +} + +int ObInnerTableSchema::all_tenant_keystore_history_idx_keystore_his_master_key_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TENANT_KEYSTORE_HISTORY_IDX_KEYSTORE_HIS_MASTER_KEY_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_KEYSTORE_HISTORY_IDX_KEYSTORE_HIS_MASTER_KEY_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TENANT_KEYSTORE_HISTORY_IDX_KEYSTORE_HIS_MASTER_KEY_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("master_key_id", //column_name + column_id + 8, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("keystore_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("schema_version", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TENANT_KEYSTORE_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id + 8); + return ret; +} + +int ObInnerTableSchema::all_tenant_ols_policy_idx_ols_policy_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TENANT_OLS_POLICY_IDX_OLS_POLICY_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_OLS_POLICY_IDX_OLS_POLICY_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TENANT_OLS_POLICY_IDX_OLS_POLICY_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("policy_name", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_COLUMN_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("label_se_policy_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TENANT_OLS_POLICY_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_tenant_ols_policy_idx_ols_policy_col_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TENANT_OLS_POLICY_IDX_OLS_POLICY_COL_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_OLS_POLICY_IDX_OLS_POLICY_COL_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TENANT_OLS_POLICY_IDX_OLS_POLICY_COL_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("column_name", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_COLUMN_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("label_se_policy_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TENANT_OLS_POLICY_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_tenant_ols_component_idx_ols_com_policy_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TENANT_OLS_COMPONENT_IDX_OLS_COM_POLICY_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_OLS_COMPONENT_IDX_OLS_COM_POLICY_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TENANT_OLS_COMPONENT_IDX_OLS_COM_POLICY_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("label_se_policy_id", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("comp_type", //column_name + column_id + 4, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("label_se_component_id", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TENANT_OLS_COMPONENT_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_tenant_ols_label_idx_ols_lab_policy_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TENANT_OLS_LABEL_IDX_OLS_LAB_POLICY_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_OLS_LABEL_IDX_OLS_LAB_POLICY_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TENANT_OLS_LABEL_IDX_OLS_LAB_POLICY_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("label_se_policy_id", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("label_se_label_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TENANT_OLS_LABEL_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_tenant_ols_label_idx_ols_lab_tag_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TENANT_OLS_LABEL_IDX_OLS_LAB_TAG_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_OLS_LABEL_IDX_OLS_LAB_TAG_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TENANT_OLS_LABEL_IDX_OLS_LAB_TAG_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("label_tag", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("label_se_label_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TENANT_OLS_LABEL_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_tenant_ols_label_idx_ols_lab_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TENANT_OLS_LABEL_IDX_OLS_LAB_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_OLS_LABEL_IDX_OLS_LAB_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TENANT_OLS_LABEL_IDX_OLS_LAB_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("label", //column_name + column_id + 5, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_COLUMN_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("label_se_label_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TENANT_OLS_LABEL_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_tenant_ols_user_level_idx_ols_level_uid_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TENANT_OLS_USER_LEVEL_IDX_OLS_LEVEL_UID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_OLS_USER_LEVEL_IDX_OLS_LEVEL_UID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TENANT_OLS_USER_LEVEL_IDX_OLS_LEVEL_UID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("user_id", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("label_se_user_level_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TENANT_OLS_USER_LEVEL_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_tenant_ols_user_level_idx_ols_level_policy_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TENANT_OLS_USER_LEVEL_IDX_OLS_LEVEL_POLICY_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_OLS_USER_LEVEL_IDX_OLS_LEVEL_POLICY_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TENANT_OLS_USER_LEVEL_IDX_OLS_LEVEL_POLICY_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("label_se_policy_id", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("label_se_user_level_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TENANT_OLS_USER_LEVEL_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_tenant_profile_idx_profile_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TENANT_PROFILE_IDX_PROFILE_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_PROFILE_IDX_PROFILE_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TENANT_PROFILE_IDX_PROFILE_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("profile_name", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_ORACLE_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("profile_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TENANT_PROFILE_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_tenant_security_audit_idx_audit_type_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TENANT_SECURITY_AUDIT_IDX_AUDIT_TYPE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_SECURITY_AUDIT_IDX_AUDIT_TYPE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TENANT_SECURITY_AUDIT_IDX_AUDIT_TYPE_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("audit_type", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("audit_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TENANT_SECURITY_AUDIT_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_tenant_trigger_idx_trigger_base_obj_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TENANT_TRIGGER_IDX_TRIGGER_BASE_OBJ_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_TRIGGER_IDX_TRIGGER_BASE_OBJ_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TENANT_TRIGGER_IDX_TRIGGER_BASE_OBJ_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("base_object_id", //column_name + column_id + 11, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("trigger_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TENANT_TRIGGER_TID); + + table_schema.set_max_used_column_id(column_id + 11); + return ret; +} + +int ObInnerTableSchema::all_tenant_trigger_idx_db_trigger_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TENANT_TRIGGER_IDX_DB_TRIGGER_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_TRIGGER_IDX_DB_TRIGGER_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TENANT_TRIGGER_IDX_DB_TRIGGER_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("database_id", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("trigger_name", //column_name + column_id + 3, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_TRIGGER_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("trigger_id", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TENANT_TRIGGER_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_tenant_trigger_idx_trigger_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TENANT_TRIGGER_IDX_TRIGGER_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_TRIGGER_IDX_TRIGGER_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TENANT_TRIGGER_IDX_TRIGGER_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("trigger_name", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_TRIGGER_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("trigger_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TENANT_TRIGGER_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_tenant_trigger_history_idx_trigger_his_base_obj_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TENANT_TRIGGER_HISTORY_IDX_TRIGGER_HIS_BASE_OBJ_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_TRIGGER_HISTORY_IDX_TRIGGER_HIS_BASE_OBJ_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TENANT_TRIGGER_HISTORY_IDX_TRIGGER_HIS_BASE_OBJ_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("base_object_id", //column_name + column_id + 12, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("schema_version", //column_name + column_id + 3, //column_id + 3, //rowkey_id + 3, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("trigger_id", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TENANT_TRIGGER_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id + 12); + return ret; +} + +int ObInnerTableSchema::all_tenant_objauth_idx_objauth_grantor_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TENANT_OBJAUTH_IDX_OBJAUTH_GRANTOR_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(7); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_OBJAUTH_IDX_OBJAUTH_GRANTOR_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TENANT_OBJAUTH_IDX_OBJAUTH_GRANTOR_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("grantor_id", //column_name + column_id + 5, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("obj_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("objtype", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("col_id", //column_name + column_id + 4, //column_id + 5, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("grantee_id", //column_name + column_id + 6, //column_id + 6, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("priv_id", //column_name + column_id + 7, //column_id + 7, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TENANT_OBJAUTH_TID); + + table_schema.set_max_used_column_id(column_id + 7); + return ret; +} + +int ObInnerTableSchema::all_tenant_objauth_idx_objauth_grantee_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TENANT_OBJAUTH_IDX_OBJAUTH_GRANTEE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(7); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_OBJAUTH_IDX_OBJAUTH_GRANTEE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TENANT_OBJAUTH_IDX_OBJAUTH_GRANTEE_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("grantee_id", //column_name + column_id + 6, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("obj_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("objtype", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("col_id", //column_name + column_id + 4, //column_id + 5, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("grantor_id", //column_name + column_id + 5, //column_id + 6, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("priv_id", //column_name + column_id + 7, //column_id + 7, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TENANT_OBJAUTH_TID); + + table_schema.set_max_used_column_id(column_id + 7); + return ret; +} + +int ObInnerTableSchema::all_tenant_object_type_idx_obj_type_db_obj_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TENANT_OBJECT_TYPE_IDX_OBJ_TYPE_DB_OBJ_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_OBJECT_TYPE_IDX_OBJ_TYPE_DB_OBJ_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TENANT_OBJECT_TYPE_IDX_OBJ_TYPE_DB_OBJ_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("database_id", //column_name + column_id + 13, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("object_name", //column_name + column_id + 17, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_TABLE_TYPE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("object_type_id", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("type", //column_name + column_id + 3, //column_id + 5, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TENANT_OBJECT_TYPE_TID); + + table_schema.set_max_used_column_id(column_id + 17); + return ret; +} + +int ObInnerTableSchema::all_tenant_object_type_idx_obj_type_obj_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TENANT_OBJECT_TYPE_IDX_OBJ_TYPE_OBJ_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_OBJECT_TYPE_IDX_OBJ_TYPE_OBJ_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TENANT_OBJECT_TYPE_IDX_OBJ_TYPE_OBJ_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("object_name", //column_name + column_id + 17, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_TABLE_TYPE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("object_type_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("type", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TENANT_OBJECT_TYPE_TID); + + table_schema.set_max_used_column_id(column_id + 17); + return ret; +} + +int ObInnerTableSchema::all_tenant_global_transaction_idx_xa_trans_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TENANT_GLOBAL_TRANSACTION_IDX_XA_TRANS_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_GLOBAL_TRANSACTION_IDX_XA_TRANS_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TENANT_GLOBAL_TRANSACTION_IDX_XA_TRANS_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("trans_id", //column_name + column_id + 5, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("gtrid", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 128, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("bqual", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 128, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ObObj format_id_default; + format_id_default.set_int(1); + ADD_COLUMN_SCHEMA_T("format_id", //column_name + column_id + 4, //column_id + 5, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + format_id_default, + format_id_default); //default_value + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TENANT_GLOBAL_TRANSACTION_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_tenant_dependency_idx_dependency_ref_obj_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TENANT_DEPENDENCY_IDX_DEPENDENCY_REF_OBJ_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_DEPENDENCY_IDX_DEPENDENCY_REF_OBJ_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TENANT_DEPENDENCY_IDX_DEPENDENCY_REF_OBJ_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ref_obj_id", //column_name + column_id + 8, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ref_obj_type", //column_name + column_id + 7, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("dep_obj_type", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("dep_obj_id", //column_name + column_id + 3, //column_id + 5, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("dep_order", //column_name + column_id + 4, //column_id + 6, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TENANT_DEPENDENCY_TID); + + table_schema.set_max_used_column_id(column_id + 8); + return ret; +} + +int ObInnerTableSchema::all_ddl_error_message_idx_ddl_error_object_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_DDL_ERROR_MESSAGE_IDX_DDL_ERROR_OBJECT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(7); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_DDL_ERROR_MESSAGE_IDX_DDL_ERROR_OBJECT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_DDL_ERROR_MESSAGE_IDX_DDL_ERROR_OBJECT_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("object_id", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("target_object_id", //column_name + column_id + 3, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_id", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("schema_version", //column_name + column_id + 5, //column_id + 5, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("svr_ip", //column_name + column_id + 6, //column_id + 6, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_IP_ADDR_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("svr_port", //column_name + column_id + 7, //column_id + 7, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_DDL_ERROR_MESSAGE_TID); + + table_schema.set_max_used_column_id(column_id + 7); + return ret; +} + +int ObInnerTableSchema::all_table_stat_history_idx_table_stat_his_savtime_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TABLE_STAT_HISTORY_IDX_TABLE_STAT_HIS_SAVTIME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TABLE_STAT_HISTORY_IDX_TABLE_STAT_HIS_SAVTIME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TABLE_STAT_HISTORY_IDX_TABLE_STAT_HIS_SAVTIME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("savtime", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("partition_id", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TABLE_STAT_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_column_stat_history_idx_column_stat_his_savtime_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_COLUMN_STAT_HISTORY_IDX_COLUMN_STAT_HIS_SAVTIME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(5); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_COLUMN_STAT_HISTORY_IDX_COLUMN_STAT_HIS_SAVTIME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_COLUMN_STAT_HISTORY_IDX_COLUMN_STAT_HIS_SAVTIME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("savtime", //column_name + column_id + 5, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("partition_id", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("column_id", //column_name + column_id + 4, //column_id + 5, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_COLUMN_STAT_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_histogram_stat_history_idx_histogram_stat_his_savtime_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_HISTOGRAM_STAT_HISTORY_IDX_HISTOGRAM_STAT_HIS_SAVTIME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(6); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_HISTOGRAM_STAT_HISTORY_IDX_HISTOGRAM_STAT_HIS_SAVTIME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_HISTOGRAM_STAT_HISTORY_IDX_HISTOGRAM_STAT_HIS_SAVTIME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("savtime", //column_name + column_id + 6, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("partition_id", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("column_id", //column_name + column_id + 4, //column_id + 5, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("endpoint_num", //column_name + column_id + 5, //column_id + 6, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_HISTOGRAM_STAT_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id + 6); + return ret; +} + +int ObInnerTableSchema::all_tablet_to_ls_idx_tablet_to_ls_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TABLET_TO_LS_IDX_TABLET_TO_LS_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(1); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TABLET_TO_LS_IDX_TABLET_TO_LS_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TABLET_TO_LS_IDX_TABLET_TO_LS_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ls_id", //column_name + column_id + 2, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tablet_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TABLET_TO_LS_TID); + + table_schema.set_max_used_column_id(column_id + 2); + return ret; +} + +int ObInnerTableSchema::all_tablet_to_ls_idx_tablet_to_table_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TABLET_TO_LS_IDX_TABLET_TO_TABLE_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(1); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TABLET_TO_LS_IDX_TABLET_TO_TABLE_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TABLET_TO_LS_IDX_TABLET_TO_TABLE_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tablet_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TABLET_TO_LS_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_pending_transaction_idx_pending_tx_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_PENDING_TRANSACTION_IDX_PENDING_TX_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_PENDING_TRANSACTION_IDX_PENDING_TX_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_PENDING_TRANSACTION_IDX_PENDING_TX_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("gtrid", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 128, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("bqual", //column_name + column_id + 4, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 128, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ObObj format_id_default; + format_id_default.set_int(1); + ADD_COLUMN_SCHEMA_T("format_id", //column_name + column_id + 5, //column_id + 3, //rowkey_id + 3, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + format_id_default, + format_id_default); //default_value + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("trans_id", //column_name + column_id + 2, //column_id + 5, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_PENDING_TRANSACTION_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_context_idx_ctx_namespace_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_CONTEXT_IDX_CTX_NAMESPACE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_CONTEXT_IDX_CTX_NAMESPACE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_CONTEXT_IDX_CTX_NAMESPACE_TID); + + if (OB_SUCC(ret)) { + ObObj namespace_default; + namespace_default.set_varchar(ObString::make_string("")); + ADD_COLUMN_SCHEMA_T("namespace", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_CONTEXT_STRING_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + namespace_default, + namespace_default); //default_value + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("context_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_CONTEXT_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_plan_baseline_item_idx_spm_item_sql_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_PLAN_BASELINE_ITEM_IDX_SPM_ITEM_SQL_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_PLAN_BASELINE_ITEM_IDX_SPM_ITEM_SQL_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_PLAN_BASELINE_ITEM_IDX_SPM_ITEM_SQL_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("sql_id", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_SQL_ID_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("database_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("plan_hash_value", //column_name + column_id + 4, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_PLAN_BASELINE_ITEM_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_plan_baseline_item_idx_spm_item_value_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_PLAN_BASELINE_ITEM_IDX_SPM_ITEM_VALUE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_PLAN_BASELINE_ITEM_IDX_SPM_ITEM_VALUE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_PLAN_BASELINE_ITEM_IDX_SPM_ITEM_VALUE_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("plan_hash_value", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("database_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("sql_id", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_SQL_ID_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_PLAN_BASELINE_ITEM_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_tenant_directory_idx_directory_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TENANT_DIRECTORY_IDX_DIRECTORY_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_DIRECTORY_IDX_DIRECTORY_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TENANT_DIRECTORY_IDX_DIRECTORY_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("directory_name", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + 128, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("directory_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TENANT_DIRECTORY_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_job_idx_job_powner_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_JOB_IDX_JOB_POWNER_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_JOB_IDX_JOB_POWNER_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_JOB_IDX_JOB_POWNER_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("powner", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_DATABASE_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("job", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_JOB_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_sequence_object_idx_seq_obj_db_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_SEQUENCE_OBJECT_IDX_SEQ_OBJ_DB_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_SEQUENCE_OBJECT_IDX_SEQ_OBJ_DB_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_SEQUENCE_OBJECT_IDX_SEQ_OBJ_DB_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("database_id", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("sequence_name", //column_name + column_id + 5, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_SEQUENCE_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("sequence_id", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_SEQUENCE_OBJECT_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_sequence_object_idx_seq_obj_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_SEQUENCE_OBJECT_IDX_SEQ_OBJ_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_SEQUENCE_OBJECT_IDX_SEQ_OBJ_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_SEQUENCE_OBJECT_IDX_SEQ_OBJ_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("sequence_name", //column_name + column_id + 5, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_SEQUENCE_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("sequence_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_SEQUENCE_OBJECT_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_recyclebin_idx_recyclebin_ori_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_RECYCLEBIN_IDX_RECYCLEBIN_ORI_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_RECYCLEBIN_IDX_RECYCLEBIN_ORI_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_RECYCLEBIN_IDX_RECYCLEBIN_ORI_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("original_name", //column_name + column_id + 7, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_ORIGINAL_NANE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("object_name", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_OBJECT_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("type", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_RECYCLEBIN_TID); + + table_schema.set_max_used_column_id(column_id + 7); + return ret; +} + +int ObInnerTableSchema::all_table_privilege_idx_tb_priv_db_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TABLE_PRIVILEGE_IDX_TB_PRIV_DB_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TABLE_PRIVILEGE_IDX_TB_PRIV_DB_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TABLE_PRIVILEGE_IDX_TB_PRIV_DB_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("database_name", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_DATABASE_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("user_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_name", //column_name + column_id + 4, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_CORE_TALBE_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TABLE_PRIVILEGE_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_table_privilege_idx_tb_priv_tb_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TABLE_PRIVILEGE_IDX_TB_PRIV_TB_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TABLE_PRIVILEGE_IDX_TB_PRIV_TB_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TABLE_PRIVILEGE_IDX_TB_PRIV_TB_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_name", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_CORE_TALBE_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("user_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("database_name", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_DATABASE_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_TABLE_PRIVILEGE_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_database_privilege_idx_db_priv_db_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_DATABASE_PRIVILEGE_IDX_DB_PRIV_DB_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_DATABASE_PRIVILEGE_IDX_DB_PRIV_DB_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_DATABASE_PRIVILEGE_IDX_DB_PRIV_DB_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("database_name", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_DATABASE_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("user_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_DATABASE_PRIVILEGE_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_rls_policy_idx_rls_policy_table_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_RLS_POLICY_IDX_RLS_POLICY_TABLE_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_RLS_POLICY_IDX_RLS_POLICY_TABLE_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_RLS_POLICY_IDX_RLS_POLICY_TABLE_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("rls_policy_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_RLS_POLICY_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_rls_policy_idx_rls_policy_group_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_RLS_POLICY_IDX_RLS_POLICY_GROUP_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_RLS_POLICY_IDX_RLS_POLICY_GROUP_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_RLS_POLICY_IDX_RLS_POLICY_GROUP_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("rls_group_id", //column_name + column_id + 5, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("rls_policy_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_RLS_POLICY_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_rls_policy_history_idx_rls_policy_his_table_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_RLS_POLICY_HISTORY_IDX_RLS_POLICY_HIS_TABLE_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_RLS_POLICY_HISTORY_IDX_RLS_POLICY_HIS_TABLE_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_RLS_POLICY_HISTORY_IDX_RLS_POLICY_HIS_TABLE_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + column_id + 6, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("rls_policy_id", //column_name + column_id + 2, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("schema_version", //column_name + column_id + 3, //column_id + 3, //rowkey_id + 3, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_RLS_POLICY_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id + 6); + return ret; +} + +int ObInnerTableSchema::all_rls_group_idx_rls_group_table_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_RLS_GROUP_IDX_RLS_GROUP_TABLE_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_RLS_GROUP_IDX_RLS_GROUP_TABLE_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_RLS_GROUP_IDX_RLS_GROUP_TABLE_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("rls_group_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_RLS_GROUP_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_rls_group_history_idx_rls_group_his_table_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_RLS_GROUP_HISTORY_IDX_RLS_GROUP_HIS_TABLE_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_RLS_GROUP_HISTORY_IDX_RLS_GROUP_HIS_TABLE_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_RLS_GROUP_HISTORY_IDX_RLS_GROUP_HIS_TABLE_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + column_id + 6, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("rls_group_id", //column_name + column_id + 2, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("schema_version", //column_name + column_id + 3, //column_id + 3, //rowkey_id + 3, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_RLS_GROUP_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id + 6); + return ret; +} + +int ObInnerTableSchema::all_rls_context_idx_rls_context_table_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_RLS_CONTEXT_IDX_RLS_CONTEXT_TABLE_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_RLS_CONTEXT_IDX_RLS_CONTEXT_TABLE_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_RLS_CONTEXT_IDX_RLS_CONTEXT_TABLE_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("rls_context_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_RLS_CONTEXT_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_rls_context_history_idx_rls_context_his_table_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_RLS_CONTEXT_HISTORY_IDX_RLS_CONTEXT_HIS_TABLE_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_RLS_CONTEXT_HISTORY_IDX_RLS_CONTEXT_HIS_TABLE_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_RLS_CONTEXT_HISTORY_IDX_RLS_CONTEXT_HIS_TABLE_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + column_id + 5, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("rls_context_id", //column_name + column_id + 2, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("schema_version", //column_name + column_id + 3, //column_id + 3, //rowkey_id + 3, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_RLS_CONTEXT_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_tenant_snapshot_idx_tenant_snapshot_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TENANT_SNAPSHOT_IDX_TENANT_SNAPSHOT_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_UNIQUE_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TENANT_SNAPSHOT_IDX_TENANT_SNAPSHOT_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TENANT_SNAPSHOT_IDX_TENANT_SNAPSHOT_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("snapshot_name", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_TENANT_SNAPSHOT_NAME_LENGTH_STORE, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_WITH_COLUMN_FLAGS("shadow_pk_0", //column_name + column_id + 32768, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true,//is_nullable + false,//is_autoincrement + true,//is_hidden + false);//is_storing_column + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_WITH_COLUMN_FLAGS("shadow_pk_1", //column_name + column_id + 32769, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true,//is_nullable + false,//is_autoincrement + true,//is_hidden + false);//is_storing_column + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("snapshot_id", //column_name + column_id + 2, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_UNIQUE_LOCAL); + table_schema.set_data_table_id(OB_ALL_TENANT_SNAPSHOT_TID); + + table_schema.set_max_used_column_id(column_id + 32769); + return ret; +} + +int ObInnerTableSchema::all_dbms_lock_allocated_idx_dbms_lock_allocated_lockhandle_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_DBMS_LOCK_ALLOCATED_IDX_DBMS_LOCK_ALLOCATED_LOCKHANDLE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(1); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_DBMS_LOCK_ALLOCATED_IDX_DBMS_LOCK_ALLOCATED_LOCKHANDLE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_DBMS_LOCK_ALLOCATED_IDX_DBMS_LOCK_ALLOCATED_LOCKHANDLE_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lockhandle", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + 128, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("name", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + 128, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_DBMS_LOCK_ALLOCATED_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_dbms_lock_allocated_idx_dbms_lock_allocated_expiration_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_DBMS_LOCK_ALLOCATED_IDX_DBMS_LOCK_ALLOCATED_EXPIRATION_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(1); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_DBMS_LOCK_ALLOCATED_IDX_DBMS_LOCK_ALLOCATED_EXPIRATION_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_DBMS_LOCK_ALLOCATED_IDX_DBMS_LOCK_ALLOCATED_EXPIRATION_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("expiration", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("name", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + 128, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_DBMS_LOCK_ALLOCATED_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_kv_ttl_task_idx_kv_ttl_task_table_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_KV_TTL_TASK_IDX_KV_TTL_TASK_TABLE_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_KV_TTL_TASK_IDX_KV_TTL_TASK_TABLE_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_KV_TTL_TASK_IDX_KV_TTL_TASK_TABLE_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tablet_id", //column_name + column_id + 4, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_KV_TTL_TASK_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_kv_ttl_task_history_idx_kv_ttl_task_history_upd_time_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_KV_TTL_TASK_HISTORY_IDX_KV_TTL_TASK_HISTORY_UPD_TIME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_KV_TTL_TASK_HISTORY_IDX_KV_TTL_TASK_HISTORY_UPD_TIME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_KV_TTL_TASK_HISTORY_IDX_KV_TTL_TASK_HISTORY_UPD_TIME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_update_time", //column_name + column_id + 6, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tablet_id", //column_name + column_id + 4, //column_id + 5, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_KV_TTL_TASK_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id + 6); + return ret; +} + +int ObInnerTableSchema::all_mview_refresh_run_stats_idx_mview_refresh_run_stats_num_mvs_current_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_MVIEW_REFRESH_RUN_STATS_IDX_MVIEW_REFRESH_RUN_STATS_NUM_MVS_CURRENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_MVIEW_REFRESH_RUN_STATS_IDX_MVIEW_REFRESH_RUN_STATS_NUM_MVS_CURRENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_MVIEW_REFRESH_RUN_STATS_IDX_MVIEW_REFRESH_RUN_STATS_NUM_MVS_CURRENT_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("num_mvs_current", //column_name + column_id + 5, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("refresh_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_MVIEW_REFRESH_RUN_STATS_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_mview_refresh_stats_idx_mview_refresh_stats_end_time_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_MVIEW_REFRESH_STATS_IDX_MVIEW_REFRESH_STATS_END_TIME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_MVIEW_REFRESH_STATS_IDX_MVIEW_REFRESH_STATS_END_TIME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_MVIEW_REFRESH_STATS_IDX_MVIEW_REFRESH_STATS_END_TIME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("end_time", //column_name + column_id + 7, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("refresh_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("mview_id", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("retry_id", //column_name + column_id + 4, //column_id + 5, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_MVIEW_REFRESH_STATS_TID); + + table_schema.set_max_used_column_id(column_id + 7); + return ret; +} + +int ObInnerTableSchema::all_mview_refresh_stats_idx_mview_refresh_stats_mview_end_time_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_MVIEW_REFRESH_STATS_IDX_MVIEW_REFRESH_STATS_MVIEW_END_TIME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_MVIEW_REFRESH_STATS_IDX_MVIEW_REFRESH_STATS_MVIEW_END_TIME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_MVIEW_REFRESH_STATS_IDX_MVIEW_REFRESH_STATS_MVIEW_END_TIME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("mview_id", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("end_time", //column_name + column_id + 7, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("refresh_id", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("retry_id", //column_name + column_id + 4, //column_id + 5, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_MVIEW_REFRESH_STATS_TID); + + table_schema.set_max_used_column_id(column_id + 7); + return ret; +} + +int ObInnerTableSchema::all_transfer_partition_task_idx_transfer_partition_key_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TRANSFER_PARTITION_TASK_IDX_TRANSFER_PARTITION_KEY_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(1); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_UNIQUE_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TRANSFER_PARTITION_TASK_IDX_TRANSFER_PARTITION_KEY_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TRANSFER_PARTITION_TASK_IDX_TRANSFER_PARTITION_KEY_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + column_id + 2, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("object_id", //column_name + column_id + 3, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_WITH_COLUMN_FLAGS("shadow_pk_0", //column_name + column_id + 32768, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true,//is_nullable + false,//is_autoincrement + true,//is_hidden + false);//is_storing_column + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_id", //column_name + column_id + 1, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_UNIQUE_LOCAL); + table_schema.set_data_table_id(OB_ALL_TRANSFER_PARTITION_TASK_TID); + + table_schema.set_max_used_column_id(column_id + 32768); + return ret; +} + +int ObInnerTableSchema::all_client_to_server_session_info_idx_client_to_server_session_info_client_session_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_CLIENT_TO_SERVER_SESSION_INFO_IDX_CLIENT_TO_SERVER_SESSION_INFO_CLIENT_SESSION_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(1); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_CLIENT_TO_SERVER_SESSION_INFO_IDX_CLIENT_TO_SERVER_SESSION_INFO_CLIENT_SESSION_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_CLIENT_TO_SERVER_SESSION_INFO_IDX_CLIENT_TO_SERVER_SESSION_INFO_CLIENT_SESSION_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("client_session_id", //column_name + column_id + 2, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("server_session_id", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_CLIENT_TO_SERVER_SESSION_INFO_TID); + + table_schema.set_max_used_column_id(column_id + 2); + return ret; +} + +int ObInnerTableSchema::all_column_privilege_idx_column_privilege_name_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_COLUMN_PRIVILEGE_IDX_COLUMN_PRIVILEGE_NAME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_COLUMN_PRIVILEGE_IDX_COLUMN_PRIVILEGE_NAME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_COLUMN_PRIVILEGE_IDX_COLUMN_PRIVILEGE_NAME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("user_id", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("database_name", //column_name + column_id + 4, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 1024, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_name", //column_name + column_id + 5, //column_id + 3, //rowkey_id + 3, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 1024, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("column_name", //column_name + column_id + 6, //column_id + 4, //rowkey_id + 4, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 1024, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 5, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("priv_id", //column_name + column_id + 2, //column_id + 6, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_COLUMN_PRIVILEGE_TID); + + table_schema.set_max_used_column_id(column_id + 6); + return ret; +} + +int ObInnerTableSchema::all_user_proxy_info_idx_user_proxy_info_proxy_user_id_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_USER_PROXY_INFO_IDX_USER_PROXY_INFO_PROXY_USER_ID_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_USER_PROXY_INFO_IDX_USER_PROXY_INFO_PROXY_USER_ID_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_USER_PROXY_INFO_IDX_USER_PROXY_INFO_PROXY_USER_ID_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("proxy_user_id", //column_name + column_id + 3, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("client_user_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_USER_PROXY_INFO_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_user_proxy_info_history_idx_user_proxy_info_proxy_user_id_history_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_USER_PROXY_INFO_HISTORY_IDX_USER_PROXY_INFO_PROXY_USER_ID_HISTORY_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_USER_PROXY_INFO_HISTORY_IDX_USER_PROXY_INFO_PROXY_USER_ID_HISTORY_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_USER_PROXY_INFO_HISTORY_IDX_USER_PROXY_INFO_PROXY_USER_ID_HISTORY_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("proxy_user_id", //column_name + column_id + 3, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("client_user_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("schema_version", //column_name + column_id + 4, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_USER_PROXY_INFO_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_user_proxy_role_info_history_idx_user_proxy_role_info_proxy_user_id_history_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_USER_PROXY_ROLE_INFO_HISTORY_IDX_USER_PROXY_ROLE_INFO_PROXY_USER_ID_HISTORY_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(5); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_USER_PROXY_ROLE_INFO_HISTORY_IDX_USER_PROXY_ROLE_INFO_PROXY_USER_ID_HISTORY_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_USER_PROXY_ROLE_INFO_HISTORY_IDX_USER_PROXY_ROLE_INFO_PROXY_USER_ID_HISTORY_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + column_id + 1, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("proxy_user_id", //column_name + column_id + 3, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("client_user_id", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("role_id", //column_name + column_id + 4, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("schema_version", //column_name + column_id + 5, //column_id + 5, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_USER_PROXY_ROLE_INFO_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_scheduler_job_run_detail_v2_idx_scheduler_job_run_detail_v2_time_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_IDX_SCHEDULER_JOB_RUN_DETAIL_V2_TIME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_IDX_SCHEDULER_JOB_RUN_DETAIL_V2_TIME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_IDX_SCHEDULER_JOB_RUN_DETAIL_V2_TIME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("time", //column_name + column_id + 2, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("job_name", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + 128, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_TID); + + table_schema.set_max_used_column_id(column_id + 2); + return ret; +} + +int ObInnerTableSchema::all_scheduler_job_run_detail_v2_idx_scheduler_job_run_detail_v2_job_class_time_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_IDX_SCHEDULER_JOB_RUN_DETAIL_V2_JOB_CLASS_TIME_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_IDX_SCHEDULER_JOB_RUN_DETAIL_V2_JOB_CLASS_TIME_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_create + } + + if (OB_SUCC(ret)) { + ++column_id; // for gmt_modified + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_IDX_SCHEDULER_JOB_RUN_DETAIL_V2_JOB_CLASS_TIME_TID); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("job_class", //column_name + column_id + 8, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + 128, //column_length + -1, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("time", //column_name + column_id + 2, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("job_name", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + 128, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_TID); + + table_schema.set_max_used_column_id(column_id + 8); + return ret; +} + +int ObInnerTableSchema::all_virtual_table_real_agent_ora_idx_data_table_id_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_IDX_DATA_TABLE_ID_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_IDX_DATA_TABLE_ID_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DATA_TABLE_ID", //column_name + column_id + 21, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TABLE_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 21); + return ret; +} + +int ObInnerTableSchema::all_virtual_table_real_agent_ora_idx_db_tb_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_IDX_DB_TB_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_IDX_DB_TB_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DATABASE_ID", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TABLE_NAME", //column_name + column_id + 3, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_TABLE_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TABLE_ID", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_virtual_table_real_agent_ora_idx_tb_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_IDX_TB_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_IDX_TB_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TABLE_NAME", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_TABLE_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TABLE_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TABLE_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_virtual_column_real_agent_ora_idx_tb_column_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_COLUMN_REAL_AGENT_ORA_IDX_TB_COLUMN_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_COLUMN_REAL_AGENT_ORA_IDX_TB_COLUMN_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TABLE_ID", //column_name + column_id + 2, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("COLUMN_NAME", //column_name + column_id + 4, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_COLUMN_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("COLUMN_ID", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_COLUMN_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_virtual_column_real_agent_ora_idx_column_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_COLUMN_REAL_AGENT_ORA_IDX_COLUMN_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_COLUMN_REAL_AGENT_ORA_IDX_COLUMN_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("COLUMN_NAME", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_COLUMN_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TABLE_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("COLUMN_ID", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_COLUMN_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_virtual_user_real_agent_ora_idx_ur_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_USER_REAL_AGENT_ORA_IDX_UR_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_USER_REAL_AGENT_ORA_IDX_UR_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("USER_NAME", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_USER_NAME_LENGTH_STORE, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("USER_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_USER_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_virtual_database_real_agent_ora_idx_db_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_DATABASE_REAL_AGENT_ORA_IDX_DB_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_DATABASE_REAL_AGENT_ORA_IDX_DB_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DATABASE_NAME", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_DATABASE_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DATABASE_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_DATABASE_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_virtual_tablegroup_real_agent_ora_idx_tg_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TABLEGROUP_REAL_AGENT_ORA_IDX_TG_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TABLEGROUP_REAL_AGENT_ORA_IDX_TG_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TABLEGROUP_NAME", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_TABLEGROUP_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TABLEGROUP_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TABLEGROUP_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_virtual_recyclebin_real_agent_ora_idx_recyclebin_db_type_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_RECYCLEBIN_REAL_AGENT_ORA_IDX_RECYCLEBIN_DB_TYPE_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_RECYCLEBIN_REAL_AGENT_ORA_IDX_RECYCLEBIN_DB_TYPE_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DATABASE_ID", //column_name + column_id + 4, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TYPE", //column_name + column_id + 3, //column_id + 3, //rowkey_id + 3, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("OBJECT_NAME", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_OBJECT_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_RECYCLEBIN_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_virtual_part_real_agent_ora_idx_part_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_PART_REAL_AGENT_ORA_IDX_PART_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_PART_REAL_AGENT_ORA_IDX_PART_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("PART_NAME", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_PARTITION_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TABLE_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("PART_ID", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_PART_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_virtual_sub_part_real_agent_ora_idx_sub_part_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_SUB_PART_REAL_AGENT_ORA_IDX_SUB_PART_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_SUB_PART_REAL_AGENT_ORA_IDX_SUB_PART_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SUB_PART_NAME", //column_name + column_id + 5, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_PARTITION_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TABLE_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("PART_ID", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SUB_PART_ID", //column_name + column_id + 4, //column_id + 5, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_SUB_PART_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_virtual_def_sub_part_real_agent_ora_idx_def_sub_part_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_DEF_SUB_PART_REAL_AGENT_ORA_IDX_DEF_SUB_PART_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_DEF_SUB_PART_REAL_AGENT_ORA_IDX_DEF_SUB_PART_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SUB_PART_NAME", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_PARTITION_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TABLE_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SUB_PART_ID", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_DEF_SUB_PART_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_virtual_foreign_key_real_agent_ora_idx_fk_child_tid_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_FOREIGN_KEY_REAL_AGENT_ORA_IDX_FK_CHILD_TID_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_FOREIGN_KEY_REAL_AGENT_ORA_IDX_FK_CHILD_TID_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("CHILD_TABLE_ID", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("FOREIGN_KEY_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_FOREIGN_KEY_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_virtual_foreign_key_real_agent_ora_idx_fk_parent_tid_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_FOREIGN_KEY_REAL_AGENT_ORA_IDX_FK_PARENT_TID_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_FOREIGN_KEY_REAL_AGENT_ORA_IDX_FK_PARENT_TID_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("PARENT_TABLE_ID", //column_name + column_id + 5, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("FOREIGN_KEY_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_FOREIGN_KEY_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_virtual_foreign_key_real_agent_ora_idx_fk_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_FOREIGN_KEY_REAL_AGENT_ORA_IDX_FK_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_FOREIGN_KEY_REAL_AGENT_ORA_IDX_FK_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("FOREIGN_KEY_NAME", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_CONSTRAINT_NAME_LENGTH_ORACLE, //column_length + 2, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("FOREIGN_KEY_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_FOREIGN_KEY_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_virtual_synonym_real_agent_ora_idx_db_synonym_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_SYNONYM_REAL_AGENT_ORA_IDX_DB_SYNONYM_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_SYNONYM_REAL_AGENT_ORA_IDX_DB_SYNONYM_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DATABASE_ID", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SYNONYM_NAME", //column_name + column_id + 5, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_SYNONYM_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SYNONYM_ID", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_SYNONYM_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_virtual_synonym_real_agent_ora_idx_synonym_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_SYNONYM_REAL_AGENT_ORA_IDX_SYNONYM_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_SYNONYM_REAL_AGENT_ORA_IDX_SYNONYM_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SYNONYM_NAME", //column_name + column_id + 5, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_SYNONYM_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SYNONYM_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_SYNONYM_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_virtual_routine_real_agent_ora_idx_db_routine_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_ROUTINE_REAL_AGENT_ORA_IDX_DB_ROUTINE_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_ROUTINE_REAL_AGENT_ORA_IDX_DB_ROUTINE_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DATABASE_ID", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ROUTINE_NAME", //column_name + column_id + 5, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_ROUTINE_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ROUTINE_ID", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_ROUTINE_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_virtual_routine_real_agent_ora_idx_routine_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_ROUTINE_REAL_AGENT_ORA_IDX_ROUTINE_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_ROUTINE_REAL_AGENT_ORA_IDX_ROUTINE_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ROUTINE_NAME", //column_name + column_id + 5, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_ROUTINE_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ROUTINE_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_ROUTINE_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_virtual_routine_real_agent_ora_idx_routine_pkg_id_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_ROUTINE_REAL_AGENT_ORA_IDX_ROUTINE_PKG_ID_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_ROUTINE_REAL_AGENT_ORA_IDX_ROUTINE_PKG_ID_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("PACKAGE_ID", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ROUTINE_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_ROUTINE_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_virtual_routine_param_real_agent_ora_idx_routine_param_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_ROUTINE_PARAM_REAL_AGENT_ORA_IDX_ROUTINE_PARAM_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_ROUTINE_PARAM_REAL_AGENT_ORA_IDX_ROUTINE_PARAM_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("PARAM_NAME", //column_name + column_id + 7, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_COLUMN_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ROUTINE_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SEQUENCE", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_ROUTINE_PARAM_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 7); + return ret; +} + +int ObInnerTableSchema::all_virtual_package_real_agent_ora_idx_db_pkg_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_PACKAGE_REAL_AGENT_ORA_IDX_DB_PKG_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_PACKAGE_REAL_AGENT_ORA_IDX_DB_PKG_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DATABASE_ID", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("PACKAGE_NAME", //column_name + column_id + 4, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_PACKAGE_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("PACKAGE_ID", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_PACKAGE_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_virtual_package_real_agent_ora_idx_pkg_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_PACKAGE_REAL_AGENT_ORA_IDX_PKG_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_PACKAGE_REAL_AGENT_ORA_IDX_PKG_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("PACKAGE_NAME", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_PACKAGE_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("PACKAGE_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_PACKAGE_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_virtual_constraint_real_agent_ora_idx_cst_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_CONSTRAINT_REAL_AGENT_ORA_IDX_CST_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_CONSTRAINT_REAL_AGENT_ORA_IDX_CST_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("CONSTRAINT_NAME", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_CONSTRAINT_NAME_LENGTH_ORACLE, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TABLE_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("CONSTRAINT_ID", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_CONSTRAINT_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_virtual_type_real_agent_ora_idx_db_type_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TYPE_REAL_AGENT_ORA_IDX_DB_TYPE_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TYPE_REAL_AGENT_ORA_IDX_DB_TYPE_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DATABASE_ID", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TYPE_NAME", //column_name + column_id + 18, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_TABLE_TYPE_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TYPE_ID", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TYPE_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 18); + return ret; +} + +int ObInnerTableSchema::all_virtual_type_real_agent_ora_idx_type_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TYPE_REAL_AGENT_ORA_IDX_TYPE_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TYPE_REAL_AGENT_ORA_IDX_TYPE_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TYPE_NAME", //column_name + column_id + 18, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_TABLE_TYPE_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TYPE_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TYPE_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 18); + return ret; +} + +int ObInnerTableSchema::all_virtual_type_attr_real_agent_ora_idx_type_attr_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TYPE_ATTR_REAL_AGENT_ORA_IDX_TYPE_ATTR_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TYPE_ATTR_REAL_AGENT_ORA_IDX_TYPE_ATTR_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("NAME", //column_name + column_id + 6, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_TABLE_TYPE_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TYPE_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ATTRIBUTE", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TYPE_ATTR_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 6); + return ret; +} + +int ObInnerTableSchema::all_virtual_coll_type_real_agent_ora_idx_coll_name_type_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_COLL_TYPE_REAL_AGENT_ORA_IDX_COLL_NAME_TYPE_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_COLL_TYPE_REAL_AGENT_ORA_IDX_COLL_NAME_TYPE_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("COLL_NAME", //column_name + column_id + 16, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_TABLE_TYPE_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("COLL_TYPE", //column_name + column_id + 13, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("COLL_TYPE_ID", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_COLL_TYPE_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 16); + return ret; +} + +int ObInnerTableSchema::all_virtual_dblink_real_agent_ora_idx_owner_dblink_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_DBLINK_REAL_AGENT_ORA_IDX_OWNER_DBLINK_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_DBLINK_REAL_AGENT_ORA_IDX_OWNER_DBLINK_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("OWNER_ID", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DBLINK_NAME", //column_name + column_id + 3, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_DBLINK_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DBLINK_ID", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_DBLINK_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_virtual_dblink_real_agent_ora_idx_dblink_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_DBLINK_REAL_AGENT_ORA_IDX_DBLINK_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_DBLINK_REAL_AGENT_ORA_IDX_DBLINK_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DBLINK_NAME", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_DBLINK_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DBLINK_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_DBLINK_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_virtual_tenant_role_grantee_map_real_agent_ora_idx_grantee_role_id_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_ROLE_GRANTEE_MAP_REAL_AGENT_ORA_IDX_GRANTEE_ROLE_ID_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_ROLE_GRANTEE_MAP_REAL_AGENT_ORA_IDX_GRANTEE_ROLE_ID_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ROLE_ID", //column_name + column_id + 3, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("GRANTEE_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_ROLE_GRANTEE_MAP_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_virtual_tenant_keystore_real_agent_ora_idx_keystore_master_key_id_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_KEYSTORE_REAL_AGENT_ORA_IDX_KEYSTORE_MASTER_KEY_ID_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_KEYSTORE_REAL_AGENT_ORA_IDX_KEYSTORE_MASTER_KEY_ID_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("MASTER_KEY_ID", //column_name + column_id + 6, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("KEYSTORE_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_KEYSTORE_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 6); + return ret; +} + +int ObInnerTableSchema::all_virtual_tenant_ols_policy_real_agent_ora_idx_ols_policy_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_OLS_POLICY_REAL_AGENT_ORA_IDX_OLS_POLICY_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_OLS_POLICY_REAL_AGENT_ORA_IDX_OLS_POLICY_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("POLICY_NAME", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_COLUMN_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("LABEL_SE_POLICY_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_OLS_POLICY_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_virtual_tenant_ols_policy_real_agent_ora_idx_ols_policy_col_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_OLS_POLICY_REAL_AGENT_ORA_IDX_OLS_POLICY_COL_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_OLS_POLICY_REAL_AGENT_ORA_IDX_OLS_POLICY_COL_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("COLUMN_NAME", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_COLUMN_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("LABEL_SE_POLICY_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_OLS_POLICY_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_virtual_tenant_ols_component_real_agent_ora_idx_ols_com_policy_id_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_OLS_COMPONENT_REAL_AGENT_ORA_IDX_OLS_COM_POLICY_ID_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_OLS_COMPONENT_REAL_AGENT_ORA_IDX_OLS_COM_POLICY_ID_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("LABEL_SE_POLICY_ID", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("COMP_TYPE", //column_name + column_id + 4, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("LABEL_SE_COMPONENT_ID", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_OLS_COMPONENT_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_virtual_tenant_ols_label_real_agent_ora_idx_ols_lab_policy_id_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_OLS_LABEL_REAL_AGENT_ORA_IDX_OLS_LAB_POLICY_ID_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_OLS_LABEL_REAL_AGENT_ORA_IDX_OLS_LAB_POLICY_ID_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("LABEL_SE_POLICY_ID", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("LABEL_SE_LABEL_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_OLS_LABEL_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_virtual_tenant_ols_label_real_agent_ora_idx_ols_lab_tag_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_OLS_LABEL_REAL_AGENT_ORA_IDX_OLS_LAB_TAG_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_OLS_LABEL_REAL_AGENT_ORA_IDX_OLS_LAB_TAG_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("LABEL_TAG", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("LABEL_SE_LABEL_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_OLS_LABEL_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_virtual_tenant_ols_label_real_agent_ora_idx_ols_lab_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_OLS_LABEL_REAL_AGENT_ORA_IDX_OLS_LAB_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_OLS_LABEL_REAL_AGENT_ORA_IDX_OLS_LAB_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("LABEL", //column_name + column_id + 5, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_COLUMN_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("LABEL_SE_LABEL_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_OLS_LABEL_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_virtual_tenant_ols_user_level_real_agent_ora_idx_ols_level_uid_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_OLS_USER_LEVEL_REAL_AGENT_ORA_IDX_OLS_LEVEL_UID_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_OLS_USER_LEVEL_REAL_AGENT_ORA_IDX_OLS_LEVEL_UID_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("USER_ID", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("LABEL_SE_USER_LEVEL_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_OLS_USER_LEVEL_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_virtual_tenant_ols_user_level_real_agent_ora_idx_ols_level_policy_id_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_OLS_USER_LEVEL_REAL_AGENT_ORA_IDX_OLS_LEVEL_POLICY_ID_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_OLS_USER_LEVEL_REAL_AGENT_ORA_IDX_OLS_LEVEL_POLICY_ID_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("LABEL_SE_POLICY_ID", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("LABEL_SE_USER_LEVEL_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_OLS_USER_LEVEL_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_virtual_tenant_profile_real_agent_ora_idx_profile_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_PROFILE_REAL_AGENT_ORA_IDX_PROFILE_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_PROFILE_REAL_AGENT_ORA_IDX_PROFILE_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("PROFILE_NAME", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + MAX_ORACLE_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("PROFILE_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_PROFILE_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_virtual_tenant_security_audit_real_agent_ora_idx_audit_type_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_SECURITY_AUDIT_REAL_AGENT_ORA_IDX_AUDIT_TYPE_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_SECURITY_AUDIT_REAL_AGENT_ORA_IDX_AUDIT_TYPE_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("AUDIT_TYPE", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("AUDIT_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_SECURITY_AUDIT_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_virtual_tenant_trigger_real_agent_ora_idx_trigger_base_obj_id_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT_ORA_IDX_TRIGGER_BASE_OBJ_ID_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT_ORA_IDX_TRIGGER_BASE_OBJ_ID_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("BASE_OBJECT_ID", //column_name + column_id + 11, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TRIGGER_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 11); + return ret; +} + +int ObInnerTableSchema::all_virtual_tenant_trigger_real_agent_ora_idx_db_trigger_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT_ORA_IDX_DB_TRIGGER_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT_ORA_IDX_DB_TRIGGER_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DATABASE_ID", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TRIGGER_NAME", //column_name + column_id + 3, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_TRIGGER_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TRIGGER_ID", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_virtual_tenant_trigger_real_agent_ora_idx_trigger_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT_ORA_IDX_TRIGGER_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT_ORA_IDX_TRIGGER_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TRIGGER_NAME", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_TRIGGER_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TRIGGER_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_TRIGGER_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_virtual_tenant_objauth_real_agent_ora_idx_objauth_grantor_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_OBJAUTH_REAL_AGENT_ORA_IDX_OBJAUTH_GRANTOR_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(7); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_OBJAUTH_REAL_AGENT_ORA_IDX_OBJAUTH_GRANTOR_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("GRANTOR_ID", //column_name + column_id + 5, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("OBJ_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("OBJTYPE", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("COL_ID", //column_name + column_id + 4, //column_id + 5, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("GRANTEE_ID", //column_name + column_id + 6, //column_id + 6, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("PRIV_ID", //column_name + column_id + 7, //column_id + 7, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_OBJAUTH_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 7); + return ret; +} + +int ObInnerTableSchema::all_virtual_tenant_objauth_real_agent_ora_idx_objauth_grantee_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_OBJAUTH_REAL_AGENT_ORA_IDX_OBJAUTH_GRANTEE_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(7); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_OBJAUTH_REAL_AGENT_ORA_IDX_OBJAUTH_GRANTEE_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("GRANTEE_ID", //column_name + column_id + 6, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("OBJ_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("OBJTYPE", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("COL_ID", //column_name + column_id + 4, //column_id + 5, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("GRANTOR_ID", //column_name + column_id + 5, //column_id + 6, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("PRIV_ID", //column_name + column_id + 7, //column_id + 7, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_OBJAUTH_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 7); + return ret; +} + +int ObInnerTableSchema::all_virtual_tenant_object_type_real_agent_ora_idx_obj_type_db_obj_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_OBJECT_TYPE_REAL_AGENT_ORA_IDX_OBJ_TYPE_DB_OBJ_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_OBJECT_TYPE_REAL_AGENT_ORA_IDX_OBJ_TYPE_DB_OBJ_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DATABASE_ID", //column_name + column_id + 13, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("OBJECT_NAME", //column_name + column_id + 17, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_TABLE_TYPE_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("OBJECT_TYPE_ID", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TYPE", //column_name + column_id + 3, //column_id + 5, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_OBJECT_TYPE_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 17); + return ret; +} + +int ObInnerTableSchema::all_virtual_tenant_object_type_real_agent_ora_idx_obj_type_obj_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_OBJECT_TYPE_REAL_AGENT_ORA_IDX_OBJ_TYPE_OBJ_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_OBJECT_TYPE_REAL_AGENT_ORA_IDX_OBJ_TYPE_OBJ_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("OBJECT_NAME", //column_name + column_id + 17, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_TABLE_TYPE_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("OBJECT_TYPE_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TYPE", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_OBJECT_TYPE_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 17); + return ret; +} + +int ObInnerTableSchema::all_virtual_tenant_dependency_real_agent_ora_idx_dependency_ref_obj_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_DEPENDENCY_REAL_AGENT_ORA_IDX_DEPENDENCY_REF_OBJ_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_DEPENDENCY_REAL_AGENT_ORA_IDX_DEPENDENCY_REF_OBJ_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("REF_OBJ_ID", //column_name + column_id + 8, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("REF_OBJ_TYPE", //column_name + column_id + 7, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DEP_OBJ_TYPE", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DEP_OBJ_ID", //column_name + column_id + 3, //column_id + 5, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DEP_ORDER", //column_name + column_id + 4, //column_id + 6, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_DEPENDENCY_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 8); + return ret; +} + +int ObInnerTableSchema::all_virtual_table_stat_history_real_agent_ora_idx_table_stat_his_savtime_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TABLE_STAT_HISTORY_REAL_AGENT_ORA_IDX_TABLE_STAT_HIS_SAVTIME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TABLE_STAT_HISTORY_REAL_AGENT_ORA_IDX_TABLE_STAT_HIS_SAVTIME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SAVTIME", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObTimestampLTZType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TABLE_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("PARTITION_ID", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TABLE_STAT_HISTORY_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_virtual_column_stat_history_real_agent_ora_idx_column_stat_his_savtime_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_COLUMN_STAT_HISTORY_REAL_AGENT_ORA_IDX_COLUMN_STAT_HIS_SAVTIME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(5); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_COLUMN_STAT_HISTORY_REAL_AGENT_ORA_IDX_COLUMN_STAT_HIS_SAVTIME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SAVTIME", //column_name + column_id + 5, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObTimestampLTZType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TABLE_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("PARTITION_ID", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("COLUMN_ID", //column_name + column_id + 4, //column_id + 5, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_COLUMN_STAT_HISTORY_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_virtual_histogram_stat_history_real_agent_ora_idx_histogram_stat_his_savtime_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_HISTOGRAM_STAT_HISTORY_REAL_AGENT_ORA_IDX_HISTOGRAM_STAT_HIS_SAVTIME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(6); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_HISTOGRAM_STAT_HISTORY_REAL_AGENT_ORA_IDX_HISTOGRAM_STAT_HIS_SAVTIME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SAVTIME", //column_name + column_id + 6, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObTimestampLTZType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TABLE_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("PARTITION_ID", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("COLUMN_ID", //column_name + column_id + 4, //column_id + 5, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ENDPOINT_NUM", //column_name + column_id + 5, //column_id + 6, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_HISTOGRAM_STAT_HISTORY_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 6); + return ret; +} + +int ObInnerTableSchema::all_virtual_tablet_to_ls_real_agent_ora_idx_tablet_to_ls_id_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TABLET_TO_LS_REAL_AGENT_ORA_IDX_TABLET_TO_LS_ID_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(1); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TABLET_TO_LS_REAL_AGENT_ORA_IDX_TABLET_TO_LS_ID_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("LS_ID", //column_name + column_id + 2, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TABLET_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TABLET_TO_LS_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 2); + return ret; +} + +int ObInnerTableSchema::all_virtual_tablet_to_ls_real_agent_ora_idx_tablet_to_table_id_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TABLET_TO_LS_REAL_AGENT_ORA_IDX_TABLET_TO_TABLE_ID_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(1); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TABLET_TO_LS_REAL_AGENT_ORA_IDX_TABLET_TO_TABLE_ID_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TABLE_ID", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TABLET_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TABLET_TO_LS_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_virtual_context_real_agent_ora_idx_ctx_namespace_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_CONTEXT_REAL_AGENT_ORA_IDX_CTX_NAMESPACE_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_CONTEXT_REAL_AGENT_ORA_IDX_CTX_NAMESPACE_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("NAMESPACE", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_CONTEXT_STRING_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("CONTEXT_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_CONTEXT_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_virtual_plan_baseline_item_real_agent_ora_idx_spm_item_sql_id_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_PLAN_BASELINE_ITEM_REAL_AGENT_ORA_IDX_SPM_ITEM_SQL_ID_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_PLAN_BASELINE_ITEM_REAL_AGENT_ORA_IDX_SPM_ITEM_SQL_ID_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SQL_ID", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_SQL_ID_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DATABASE_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("PLAN_HASH_VALUE", //column_name + column_id + 4, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_PLAN_BASELINE_ITEM_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_virtual_plan_baseline_item_real_agent_ora_idx_spm_item_value_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_PLAN_BASELINE_ITEM_REAL_AGENT_ORA_IDX_SPM_ITEM_VALUE_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_PLAN_BASELINE_ITEM_REAL_AGENT_ORA_IDX_SPM_ITEM_VALUE_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("PLAN_HASH_VALUE", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DATABASE_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SQL_ID", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_SQL_ID_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_PLAN_BASELINE_ITEM_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_virtual_tenant_directory_real_agent_ora_idx_directory_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TENANT_DIRECTORY_REAL_AGENT_ORA_IDX_DIRECTORY_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TENANT_DIRECTORY_REAL_AGENT_ORA_IDX_DIRECTORY_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DIRECTORY_NAME", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + 128, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DIRECTORY_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TENANT_DIRECTORY_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_virtual_job_real_agent_ora_idx_job_powner_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_JOB_REAL_AGENT_ORA_IDX_JOB_POWNER_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_JOB_REAL_AGENT_ORA_IDX_JOB_POWNER_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("POWNER", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_DATABASE_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("JOB", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_JOB_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_virtual_sequence_object_real_agent_ora_idx_seq_obj_db_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_SEQUENCE_OBJECT_REAL_AGENT_ORA_IDX_SEQ_OBJ_DB_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_SEQUENCE_OBJECT_REAL_AGENT_ORA_IDX_SEQ_OBJ_DB_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DATABASE_ID", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SEQUENCE_NAME", //column_name + column_id + 5, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_SEQUENCE_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SEQUENCE_ID", //column_name + column_id + 2, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_SEQUENCE_OBJECT_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_virtual_sequence_object_real_agent_ora_idx_seq_obj_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_SEQUENCE_OBJECT_REAL_AGENT_ORA_IDX_SEQ_OBJ_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_SEQUENCE_OBJECT_REAL_AGENT_ORA_IDX_SEQ_OBJ_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SEQUENCE_NAME", //column_name + column_id + 5, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_SEQUENCE_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SEQUENCE_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_SEQUENCE_OBJECT_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_virtual_recyclebin_real_agent_ora_idx_recyclebin_ori_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_RECYCLEBIN_REAL_AGENT_ORA_IDX_RECYCLEBIN_ORI_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_RECYCLEBIN_REAL_AGENT_ORA_IDX_RECYCLEBIN_ORI_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ORIGINAL_NAME", //column_name + column_id + 7, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_ORIGINAL_NANE_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("OBJECT_NAME", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_OBJECT_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TYPE", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_RECYCLEBIN_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 7); + return ret; +} + +int ObInnerTableSchema::all_virtual_table_privilege_real_agent_ora_idx_tb_priv_db_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TABLE_PRIVILEGE_REAL_AGENT_ORA_IDX_TB_PRIV_DB_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TABLE_PRIVILEGE_REAL_AGENT_ORA_IDX_TB_PRIV_DB_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DATABASE_NAME", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_DATABASE_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("USER_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TABLE_NAME", //column_name + column_id + 4, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_CORE_TALBE_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TABLE_PRIVILEGE_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_virtual_table_privilege_real_agent_ora_idx_tb_priv_tb_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TABLE_PRIVILEGE_REAL_AGENT_ORA_IDX_TB_PRIV_TB_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(4); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TABLE_PRIVILEGE_REAL_AGENT_ORA_IDX_TB_PRIV_TB_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TABLE_NAME", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_CORE_TALBE_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("USER_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DATABASE_NAME", //column_name + column_id + 3, //column_id + 4, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_DATABASE_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_TABLE_PRIVILEGE_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_virtual_database_privilege_real_agent_ora_idx_db_priv_db_name_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_DATABASE_PRIVILEGE_REAL_AGENT_ORA_IDX_DB_PRIV_DB_NAME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_DATABASE_PRIVILEGE_REAL_AGENT_ORA_IDX_DB_PRIV_DB_NAME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("DATABASE_NAME", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_DATABASE_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("USER_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_DATABASE_PRIVILEGE_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_virtual_rls_policy_real_agent_ora_idx_rls_policy_table_id_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_RLS_POLICY_REAL_AGENT_ORA_IDX_RLS_POLICY_TABLE_ID_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_RLS_POLICY_REAL_AGENT_ORA_IDX_RLS_POLICY_TABLE_ID_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TABLE_ID", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("RLS_POLICY_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_RLS_POLICY_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_virtual_rls_policy_real_agent_ora_idx_rls_policy_group_id_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_RLS_POLICY_REAL_AGENT_ORA_IDX_RLS_POLICY_GROUP_ID_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_RLS_POLICY_REAL_AGENT_ORA_IDX_RLS_POLICY_GROUP_ID_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("RLS_GROUP_ID", //column_name + column_id + 5, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("RLS_POLICY_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_RLS_POLICY_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 5); + return ret; +} + +int ObInnerTableSchema::all_virtual_rls_group_real_agent_ora_idx_rls_group_table_id_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_RLS_GROUP_REAL_AGENT_ORA_IDX_RLS_GROUP_TABLE_ID_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_RLS_GROUP_REAL_AGENT_ORA_IDX_RLS_GROUP_TABLE_ID_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TABLE_ID", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("RLS_GROUP_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_RLS_GROUP_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_virtual_rls_context_real_agent_ora_idx_rls_context_table_id_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_RLS_CONTEXT_REAL_AGENT_ORA_IDX_RLS_CONTEXT_TABLE_ID_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_RLS_CONTEXT_REAL_AGENT_ORA_IDX_RLS_CONTEXT_TABLE_ID_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TABLE_ID", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("RLS_CONTEXT_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_RLS_CONTEXT_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_virtual_dbms_lock_allocated_real_agent_ora_idx_dbms_lock_allocated_lockhandle_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_DBMS_LOCK_ALLOCATED_REAL_AGENT_ORA_IDX_DBMS_LOCK_ALLOCATED_LOCKHANDLE_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(1); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_DBMS_LOCK_ALLOCATED_REAL_AGENT_ORA_IDX_DBMS_LOCK_ALLOCATED_LOCKHANDLE_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("LOCKHANDLE", //column_name + column_id + 3, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + 128, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("NAME", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + 128, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_DBMS_LOCK_ALLOCATED_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_virtual_dbms_lock_allocated_real_agent_ora_idx_dbms_lock_allocated_expiration_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_DBMS_LOCK_ALLOCATED_REAL_AGENT_ORA_IDX_DBMS_LOCK_ALLOCATED_EXPIRATION_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(1); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_DBMS_LOCK_ALLOCATED_REAL_AGENT_ORA_IDX_DBMS_LOCK_ALLOCATED_EXPIRATION_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("EXPIRATION", //column_name + column_id + 4, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObTimestampLTZType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("NAME", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + 128, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_DBMS_LOCK_ALLOCATED_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 4); + return ret; +} + +int ObInnerTableSchema::all_virtual_user_proxy_info_real_agent_ora_idx_user_proxy_info_proxy_user_id_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_USER_PROXY_INFO_REAL_AGENT_ORA_IDX_USER_PROXY_INFO_PROXY_USER_ID_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_USER_PROXY_INFO_REAL_AGENT_ORA_IDX_USER_PROXY_INFO_PROXY_USER_ID_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + column_id + 1, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("PROXY_USER_ID", //column_name + column_id + 3, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("CLIENT_USER_ID", //column_name + column_id + 2, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_USER_PROXY_INFO_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 3); + return ret; +} + +int ObInnerTableSchema::all_virtual_scheduler_job_run_detail_v2_real_agent_ora_idx_scheduler_job_run_detail_v2_time_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_SCHEDULER_JOB_RUN_DETAIL_V2_REAL_AGENT_ORA_IDX_SCHEDULER_JOB_RUN_DETAIL_V2_TIME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_SCHEDULER_JOB_RUN_DETAIL_V2_REAL_AGENT_ORA_IDX_SCHEDULER_JOB_RUN_DETAIL_V2_TIME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TIME", //column_name + column_id + 2, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObTimestampLTZType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("JOB_NAME", //column_name + column_id + 1, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + 128, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_SCHEDULER_JOB_RUN_DETAIL_V2_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 2); + return ret; +} + +int ObInnerTableSchema::all_virtual_scheduler_job_run_detail_v2_real_agent_ora_idx_scheduler_job_run_detail_v2_job_class_time_real_agent_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_SCHEDULER_JOB_RUN_DETAIL_V2_REAL_AGENT_ORA_IDX_SCHEDULER_JOB_RUN_DETAIL_V2_JOB_CLASS_TIME_REAL_AGENT_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(USER_INDEX); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_SCHEDULER_JOB_RUN_DETAIL_V2_REAL_AGENT_ORA_IDX_SCHEDULER_JOB_RUN_DETAIL_V2_JOB_CLASS_TIME_REAL_AGENT_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("JOB_CLASS", //column_name + column_id + 8, //column_id + 1, //rowkey_id + 1, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + 128, //column_length + 2, //column_precision + -1, //column_scale + true,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TIME", //column_name + column_id + 2, //column_id + 2, //rowkey_id + 2, //index_id + 0, //part_key_pos + ObTimestampLTZType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("JOB_NAME", //column_name + column_id + 1, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + 128, //column_length + 2, //column_precision + -1, //column_scale + false,//is_nullable + false); //is_autoincrement + } + table_schema.set_index_status(INDEX_STATUS_AVAILABLE); + table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); + table_schema.set_data_table_id(OB_ALL_VIRTUAL_SCHEDULER_JOB_RUN_DETAIL_V2_REAL_AGENT_ORA_TID); + + table_schema.set_max_used_column_id(column_id + 8); + return ret; +} + + +} // end namespace share +} // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema.h b/src/share/inner_table/ob_inner_table_schema.h index 60f945002..8570c5367 100644 --- a/src/share/inner_table/ob_inner_table_schema.h +++ b/src/share/inner_table/ob_inner_table_schema.h @@ -1090,6 +1090,7 @@ public: static int all_virtual_nic_info_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_scheduler_job_run_detail_v2_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_spatial_reference_systems_schema(share::schema::ObTableSchema &table_schema); + static int all_virtual_temp_file_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_sql_audit_ora_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_plan_stat_ora_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_plan_cache_plan_explain_ora_schema(share::schema::ObTableSchema &table_schema); @@ -1366,6 +1367,7 @@ public: static int all_virtual_nic_info_ora_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_scheduler_job_run_detail_v2_real_agent_ora_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_spatial_reference_systems_real_agent_ora_schema(share::schema::ObTableSchema &table_schema); + static int all_virtual_temp_file_ora_schema(share::schema::ObTableSchema &table_schema); static int gv_ob_plan_cache_stat_schema(share::schema::ObTableSchema &table_schema); static int gv_ob_plan_cache_plan_stat_schema(share::schema::ObTableSchema &table_schema); static int schemata_schema(share::schema::ObTableSchema &table_schema); @@ -1826,6 +1828,8 @@ public: static int innodb_sys_fields_schema(share::schema::ObTableSchema &table_schema); static int innodb_sys_foreign_schema(share::schema::ObTableSchema &table_schema); static int innodb_sys_foreign_cols_schema(share::schema::ObTableSchema &table_schema); + static int dba_ob_temp_files_schema(share::schema::ObTableSchema &table_schema); + static int cdb_ob_temp_files_schema(share::schema::ObTableSchema &table_schema); static int dba_synonyms_schema(share::schema::ObTableSchema &table_schema); static int dba_objects_ora_schema(share::schema::ObTableSchema &table_schema); static int all_objects_schema(share::schema::ObTableSchema &table_schema); @@ -2293,6 +2297,7 @@ public: static int gv_ob_nic_info_ora_schema(share::schema::ObTableSchema &table_schema); static int v_ob_nic_info_ora_schema(share::schema::ObTableSchema &table_schema); static int dba_ob_spatial_columns_ora_schema(share::schema::ObTableSchema &table_schema); + static int dba_ob_temp_files_ora_schema(share::schema::ObTableSchema &table_schema); static int all_table_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_column_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_ddl_operation_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); @@ -3883,6 +3888,7 @@ const schema_create_func virtual_table_schema_creators [] = { ObInnerTableSchema::all_virtual_nic_info_schema, ObInnerTableSchema::all_virtual_scheduler_job_run_detail_v2_schema, ObInnerTableSchema::all_virtual_spatial_reference_systems_schema, + ObInnerTableSchema::all_virtual_temp_file_schema, ObInnerTableSchema::all_virtual_ash_all_virtual_ash_i1_schema, ObInnerTableSchema::all_virtual_sql_plan_monitor_all_virtual_sql_plan_monitor_i1_schema, ObInnerTableSchema::all_virtual_sql_audit_all_virtual_sql_audit_i1_schema, @@ -4169,6 +4175,7 @@ const schema_create_func virtual_table_schema_creators [] = { ObInnerTableSchema::all_virtual_nic_info_ora_schema, ObInnerTableSchema::all_virtual_scheduler_job_run_detail_v2_real_agent_ora_schema, ObInnerTableSchema::all_virtual_spatial_reference_systems_real_agent_ora_schema, + ObInnerTableSchema::all_virtual_temp_file_ora_schema, ObInnerTableSchema::all_virtual_table_real_agent_ora_idx_data_table_id_real_agent_schema, ObInnerTableSchema::all_virtual_table_real_agent_ora_idx_db_tb_name_real_agent_schema, ObInnerTableSchema::all_virtual_table_real_agent_ora_idx_tb_name_real_agent_schema, @@ -4716,6 +4723,8 @@ const schema_create_func sys_view_schema_creators [] = { ObInnerTableSchema::innodb_sys_fields_schema, ObInnerTableSchema::innodb_sys_foreign_schema, ObInnerTableSchema::innodb_sys_foreign_cols_schema, + ObInnerTableSchema::dba_ob_temp_files_schema, + ObInnerTableSchema::cdb_ob_temp_files_schema, ObInnerTableSchema::dba_synonyms_schema, ObInnerTableSchema::dba_objects_ora_schema, ObInnerTableSchema::all_objects_schema, @@ -5183,6 +5192,7 @@ const schema_create_func sys_view_schema_creators [] = { ObInnerTableSchema::gv_ob_nic_info_ora_schema, ObInnerTableSchema::v_ob_nic_info_ora_schema, ObInnerTableSchema::dba_ob_spatial_columns_ora_schema, + ObInnerTableSchema::dba_ob_temp_files_ora_schema, NULL,}; const schema_create_func core_index_table_schema_creators [] = { @@ -5823,6 +5833,7 @@ const uint64_t tenant_space_tables [] = { OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_TID, OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_DETAIL_TID, OB_ALL_VIRTUAL_NIC_INFO_TID, + OB_ALL_VIRTUAL_TEMP_FILE_TID, OB_ALL_VIRTUAL_SQL_AUDIT_ORA_TID, OB_ALL_VIRTUAL_SQL_AUDIT_ORA_ALL_VIRTUAL_SQL_AUDIT_I1_TID, OB_ALL_VIRTUAL_PLAN_STAT_ORA_TID, @@ -6108,6 +6119,7 @@ const uint64_t tenant_space_tables [] = { OB_ALL_VIRTUAL_NIC_INFO_ORA_TID, OB_ALL_VIRTUAL_SCHEDULER_JOB_RUN_DETAIL_V2_REAL_AGENT_ORA_TID, OB_ALL_VIRTUAL_SPATIAL_REFERENCE_SYSTEMS_REAL_AGENT_ORA_TID, + OB_ALL_VIRTUAL_TEMP_FILE_ORA_TID, OB_GV_OB_PLAN_CACHE_STAT_TID, OB_GV_OB_PLAN_CACHE_PLAN_STAT_TID, OB_SCHEMATA_TID, @@ -6446,6 +6458,7 @@ const uint64_t tenant_space_tables [] = { OB_INNODB_SYS_FIELDS_TID, OB_INNODB_SYS_FOREIGN_TID, OB_INNODB_SYS_FOREIGN_COLS_TID, + OB_DBA_OB_TEMP_FILES_TID, OB_DBA_SYNONYMS_TID, OB_DBA_OBJECTS_ORA_TID, OB_ALL_OBJECTS_TID, @@ -6913,6 +6926,7 @@ const uint64_t tenant_space_tables [] = { OB_GV_OB_NIC_INFO_ORA_TID, OB_V_OB_NIC_INFO_ORA_TID, OB_DBA_OB_SPATIAL_COLUMNS_ORA_TID, + OB_DBA_OB_TEMP_FILES_ORA_TID, OB_ALL_TABLE_IDX_DATA_TABLE_ID_TID, OB_ALL_TABLE_IDX_DB_TB_NAME_TID, OB_ALL_TABLE_IDX_TB_NAME_TID, @@ -7799,7 +7813,8 @@ const uint64_t all_ora_mapping_virtual_table_org_tables [] = { OB_ALL_VIRTUAL_SERVICE_TID, OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_TID, OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_DETAIL_TID, - OB_ALL_VIRTUAL_NIC_INFO_TID, }; + OB_ALL_VIRTUAL_NIC_INFO_TID, + OB_ALL_VIRTUAL_TEMP_FILE_TID, }; const uint64_t all_ora_mapping_virtual_tables [] = { OB_ALL_VIRTUAL_SQL_AUDIT_ORA_TID , OB_ALL_VIRTUAL_PLAN_STAT_ORA_TID @@ -7949,6 +7964,7 @@ const uint64_t all_ora_mapping_virtual_tables [] = { OB_ALL_VIRTUAL_SQL_AUDIT_O , OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_ORA_TID , OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_DETAIL_ORA_TID , OB_ALL_VIRTUAL_NIC_INFO_ORA_TID +, OB_ALL_VIRTUAL_TEMP_FILE_ORA_TID , }; /* start/end_pos is start/end postition for column with tenant id */ @@ -8478,6 +8494,7 @@ const char* const tenant_space_table_names [] = { OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_TNAME, OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_DETAIL_TNAME, OB_ALL_VIRTUAL_NIC_INFO_TNAME, + OB_ALL_VIRTUAL_TEMP_FILE_TNAME, OB_ALL_VIRTUAL_SQL_AUDIT_ORA_TNAME, OB_ALL_VIRTUAL_SQL_AUDIT_ORA_ALL_VIRTUAL_SQL_AUDIT_I1_TNAME, OB_ALL_VIRTUAL_PLAN_STAT_ORA_TNAME, @@ -8763,6 +8780,7 @@ const char* const tenant_space_table_names [] = { OB_ALL_VIRTUAL_NIC_INFO_ORA_TNAME, OB_ALL_VIRTUAL_SCHEDULER_JOB_RUN_DETAIL_V2_REAL_AGENT_ORA_TNAME, OB_ALL_VIRTUAL_SPATIAL_REFERENCE_SYSTEMS_REAL_AGENT_ORA_TNAME, + OB_ALL_VIRTUAL_TEMP_FILE_ORA_TNAME, OB_GV_OB_PLAN_CACHE_STAT_TNAME, OB_GV_OB_PLAN_CACHE_PLAN_STAT_TNAME, OB_SCHEMATA_TNAME, @@ -9101,6 +9119,7 @@ const char* const tenant_space_table_names [] = { OB_INNODB_SYS_FIELDS_TNAME, OB_INNODB_SYS_FOREIGN_TNAME, OB_INNODB_SYS_FOREIGN_COLS_TNAME, + OB_DBA_OB_TEMP_FILES_TNAME, OB_DBA_SYNONYMS_TNAME, OB_DBA_OBJECTS_ORA_TNAME, OB_ALL_OBJECTS_TNAME, @@ -9568,6 +9587,7 @@ const char* const tenant_space_table_names [] = { OB_GV_OB_NIC_INFO_ORA_TNAME, OB_V_OB_NIC_INFO_ORA_TNAME, OB_DBA_OB_SPATIAL_COLUMNS_ORA_TNAME, + OB_DBA_OB_TEMP_FILES_ORA_TNAME, OB_ALL_TABLE_IDX_DATA_TABLE_ID_TNAME, OB_ALL_TABLE_IDX_DB_TB_NAME_TNAME, OB_ALL_TABLE_IDX_TB_NAME_TNAME, @@ -10463,6 +10483,7 @@ const uint64_t tenant_distributed_vtables [] = { OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_TID, OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_DETAIL_TID, OB_ALL_VIRTUAL_NIC_INFO_TID, + OB_ALL_VIRTUAL_TEMP_FILE_TID, OB_ALL_VIRTUAL_SQL_AUDIT_ORA_TID, OB_ALL_VIRTUAL_SQL_AUDIT_ORA_ALL_VIRTUAL_SQL_AUDIT_I1_TID, OB_ALL_VIRTUAL_PLAN_STAT_ORA_TID, @@ -10537,7 +10558,8 @@ const uint64_t tenant_distributed_vtables [] = { OB_ALL_VIRTUAL_TRACEPOINT_INFO_ORA_TID, OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_ORA_TID, OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_DETAIL_ORA_TID, - OB_ALL_VIRTUAL_NIC_INFO_ORA_TID, }; + OB_ALL_VIRTUAL_NIC_INFO_ORA_TID, + OB_ALL_VIRTUAL_TEMP_FILE_ORA_TID, }; const uint64_t restrict_access_virtual_tables[] = { OB_ALL_VIRTUAL_SQL_AUDIT_ORA_TID, @@ -13280,11 +13302,11 @@ static inline int get_sys_table_lob_aux_schema(const uint64_t tid, const int64_t OB_CORE_TABLE_COUNT = 4; const int64_t OB_SYS_TABLE_COUNT = 300; -const int64_t OB_VIRTUAL_TABLE_COUNT = 829; -const int64_t OB_SYS_VIEW_COUNT = 927; -const int64_t OB_SYS_TENANT_TABLE_COUNT = 2061; +const int64_t OB_VIRTUAL_TABLE_COUNT = 831; +const int64_t OB_SYS_VIEW_COUNT = 930; +const int64_t OB_SYS_TENANT_TABLE_COUNT = 2066; const int64_t OB_CORE_SCHEMA_VERSION = 1; -const int64_t OB_BOOTSTRAP_SCHEMA_VERSION = 2064; +const int64_t OB_BOOTSTRAP_SCHEMA_VERSION = 2069; } // end namespace share } // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema_constants.h b/src/share/inner_table/ob_inner_table_schema_constants.h index 04e062152..f6b56de05 100644 --- a/src/share/inner_table/ob_inner_table_schema_constants.h +++ b/src/share/inner_table/ob_inner_table_schema_constants.h @@ -790,6 +790,7 @@ const uint64_t OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_DETAIL_TID = 12482; // "__al const uint64_t OB_ALL_VIRTUAL_NIC_INFO_TID = 12487; // "__all_virtual_nic_info" const uint64_t OB_ALL_VIRTUAL_SCHEDULER_JOB_RUN_DETAIL_V2_TID = 12488; // "__all_virtual_scheduler_job_run_detail_v2" const uint64_t OB_ALL_VIRTUAL_SPATIAL_REFERENCE_SYSTEMS_TID = 12490; // "__all_virtual_spatial_reference_systems" +const uint64_t OB_ALL_VIRTUAL_TEMP_FILE_TID = 12505; // "__all_virtual_temp_file" const uint64_t OB_ALL_VIRTUAL_SQL_AUDIT_ORA_TID = 15009; // "ALL_VIRTUAL_SQL_AUDIT_ORA" const uint64_t OB_ALL_VIRTUAL_PLAN_STAT_ORA_TID = 15010; // "ALL_VIRTUAL_PLAN_STAT_ORA" const uint64_t OB_ALL_VIRTUAL_PLAN_CACHE_PLAN_EXPLAIN_ORA_TID = 15012; // "ALL_VIRTUAL_PLAN_CACHE_PLAN_EXPLAIN_ORA" @@ -1066,6 +1067,7 @@ const uint64_t OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_DETAIL_ORA_TID = 15451; // " const uint64_t OB_ALL_VIRTUAL_NIC_INFO_ORA_TID = 15456; // "ALL_VIRTUAL_NIC_INFO_ORA" const uint64_t OB_ALL_VIRTUAL_SCHEDULER_JOB_RUN_DETAIL_V2_REAL_AGENT_ORA_TID = 15458; // "ALL_VIRTUAL_SCHEDULER_JOB_RUN_DETAIL_V2_REAL_AGENT_ORA" const uint64_t OB_ALL_VIRTUAL_SPATIAL_REFERENCE_SYSTEMS_REAL_AGENT_ORA_TID = 15459; // "ALL_VIRTUAL_SPATIAL_REFERENCE_SYSTEMS_REAL_AGENT_ORA" +const uint64_t OB_ALL_VIRTUAL_TEMP_FILE_ORA_TID = 15485; // "ALL_VIRTUAL_TEMP_FILE_ORA" const uint64_t OB_GV_OB_PLAN_CACHE_STAT_TID = 20001; // "GV$OB_PLAN_CACHE_STAT" const uint64_t OB_GV_OB_PLAN_CACHE_PLAN_STAT_TID = 20002; // "GV$OB_PLAN_CACHE_PLAN_STAT" const uint64_t OB_SCHEMATA_TID = 20003; // "SCHEMATA" @@ -1526,6 +1528,8 @@ const uint64_t OB_CDB_SCHEDULER_JOB_RUN_DETAILS_TID = 21590; // "CDB_SCHEDULER_J const uint64_t OB_INNODB_SYS_FIELDS_TID = 21603; // "INNODB_SYS_FIELDS" const uint64_t OB_INNODB_SYS_FOREIGN_TID = 21604; // "INNODB_SYS_FOREIGN" const uint64_t OB_INNODB_SYS_FOREIGN_COLS_TID = 21605; // "INNODB_SYS_FOREIGN_COLS" +const uint64_t OB_DBA_OB_TEMP_FILES_TID = 21622; // "DBA_OB_TEMP_FILES" +const uint64_t OB_CDB_OB_TEMP_FILES_TID = 21623; // "CDB_OB_TEMP_FILES" const uint64_t OB_DBA_SYNONYMS_TID = 25001; // "DBA_SYNONYMS" const uint64_t OB_DBA_OBJECTS_ORA_TID = 25002; // "DBA_OBJECTS_ORA" const uint64_t OB_ALL_OBJECTS_TID = 25003; // "ALL_OBJECTS" @@ -1993,6 +1997,7 @@ const uint64_t OB_V_OB_TENANT_RESOURCE_LIMIT_DETAIL_ORA_TID = 28227; // "V$OB_TE const uint64_t OB_GV_OB_NIC_INFO_ORA_TID = 28230; // "GV$OB_NIC_INFO_ORA" const uint64_t OB_V_OB_NIC_INFO_ORA_TID = 28231; // "V$OB_NIC_INFO_ORA" const uint64_t OB_DBA_OB_SPATIAL_COLUMNS_ORA_TID = 28234; // "DBA_OB_SPATIAL_COLUMNS_ORA" +const uint64_t OB_DBA_OB_TEMP_FILES_ORA_TID = 28264; // "DBA_OB_TEMP_FILES_ORA" const uint64_t OB_ALL_TABLE_AUX_LOB_META_TID = 50003; // "__all_table_aux_lob_meta" const uint64_t OB_ALL_COLUMN_AUX_LOB_META_TID = 50004; // "__all_column_aux_lob_meta" const uint64_t OB_ALL_DDL_OPERATION_AUX_LOB_META_TID = 50005; // "__all_ddl_operation_aux_lob_meta" @@ -3567,6 +3572,7 @@ const char *const OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_DETAIL_TNAME = "__all_vir const char *const OB_ALL_VIRTUAL_NIC_INFO_TNAME = "__all_virtual_nic_info"; const char *const OB_ALL_VIRTUAL_SCHEDULER_JOB_RUN_DETAIL_V2_TNAME = "__all_virtual_scheduler_job_run_detail_v2"; const char *const OB_ALL_VIRTUAL_SPATIAL_REFERENCE_SYSTEMS_TNAME = "__all_virtual_spatial_reference_systems"; +const char *const OB_ALL_VIRTUAL_TEMP_FILE_TNAME = "__all_virtual_temp_file"; const char *const OB_ALL_VIRTUAL_SQL_AUDIT_ORA_TNAME = "ALL_VIRTUAL_SQL_AUDIT"; const char *const OB_ALL_VIRTUAL_PLAN_STAT_ORA_TNAME = "ALL_VIRTUAL_PLAN_STAT"; const char *const OB_ALL_VIRTUAL_PLAN_CACHE_PLAN_EXPLAIN_ORA_TNAME = "ALL_VIRTUAL_PLAN_CACHE_PLAN_EXPLAIN"; @@ -3843,6 +3849,7 @@ const char *const OB_ALL_VIRTUAL_TENANT_RESOURCE_LIMIT_DETAIL_ORA_TNAME = "ALL_V const char *const OB_ALL_VIRTUAL_NIC_INFO_ORA_TNAME = "ALL_VIRTUAL_NIC_INFO"; const char *const OB_ALL_VIRTUAL_SCHEDULER_JOB_RUN_DETAIL_V2_REAL_AGENT_ORA_TNAME = "ALL_VIRTUAL_SCHEDULER_JOB_RUN_DETAIL_V2_REAL_AGENT"; const char *const OB_ALL_VIRTUAL_SPATIAL_REFERENCE_SYSTEMS_REAL_AGENT_ORA_TNAME = "ALL_VIRTUAL_SPATIAL_REFERENCE_SYSTEMS_REAL_AGENT"; +const char *const OB_ALL_VIRTUAL_TEMP_FILE_ORA_TNAME = "ALL_VIRTUAL_TEMP_FILE"; const char *const OB_GV_OB_PLAN_CACHE_STAT_TNAME = "GV$OB_PLAN_CACHE_STAT"; const char *const OB_GV_OB_PLAN_CACHE_PLAN_STAT_TNAME = "GV$OB_PLAN_CACHE_PLAN_STAT"; const char *const OB_SCHEMATA_TNAME = "SCHEMATA"; @@ -4303,6 +4310,8 @@ const char *const OB_CDB_SCHEDULER_JOB_RUN_DETAILS_TNAME = "CDB_SCHEDULER_JOB_RU const char *const OB_INNODB_SYS_FIELDS_TNAME = "INNODB_SYS_FIELDS"; const char *const OB_INNODB_SYS_FOREIGN_TNAME = "INNODB_SYS_FOREIGN"; const char *const OB_INNODB_SYS_FOREIGN_COLS_TNAME = "INNODB_SYS_FOREIGN_COLS"; +const char *const OB_DBA_OB_TEMP_FILES_TNAME = "DBA_OB_TEMP_FILES"; +const char *const OB_CDB_OB_TEMP_FILES_TNAME = "CDB_OB_TEMP_FILES"; const char *const OB_DBA_SYNONYMS_TNAME = "DBA_SYNONYMS"; const char *const OB_DBA_OBJECTS_ORA_TNAME = "DBA_OBJECTS"; const char *const OB_ALL_OBJECTS_TNAME = "ALL_OBJECTS"; @@ -4770,6 +4779,7 @@ const char *const OB_V_OB_TENANT_RESOURCE_LIMIT_DETAIL_ORA_TNAME = "V$OB_TENANT_ const char *const OB_GV_OB_NIC_INFO_ORA_TNAME = "GV$OB_NIC_INFO"; const char *const OB_V_OB_NIC_INFO_ORA_TNAME = "V$OB_NIC_INFO"; const char *const OB_DBA_OB_SPATIAL_COLUMNS_ORA_TNAME = "DBA_OB_SPATIAL_COLUMNS"; +const char *const OB_DBA_OB_TEMP_FILES_ORA_TNAME = "DBA_OB_TEMP_FILES"; const char *const OB_ALL_TABLE_AUX_LOB_META_TNAME = "__all_table_aux_lob_meta"; const char *const OB_ALL_COLUMN_AUX_LOB_META_TNAME = "__all_column_aux_lob_meta"; const char *const OB_ALL_DDL_OPERATION_AUX_LOB_META_TNAME = "__all_ddl_operation_aux_lob_meta"; diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index d4cc29c5a..f79f2b9f6 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -14730,7 +14730,43 @@ def_table_schema(**gen_iterate_virtual_table_def( # 12502: __all_virtual_wr_res_mgr_sysstat # 12503: __all_virtual_kv_redis_table # 12504: __all_virtual_function_io_stat -# 12505: __all_virtual_temp_file + +def_table_schema( + owner = 'wuyuefei.wyf', + table_name = '__all_virtual_temp_file', + table_id = '12505', + table_type = 'VIRTUAL_TABLE', + gm_columns = [], + rowkey_columns = [], + in_tenant_space = True, + + normal_columns = [ + ('tenant_id', 'int'), + ('svr_ip', 'varchar:MAX_IP_ADDR_LENGTH'), + ('svr_port', 'int'), + ('file_id', 'int'), + ('trace_id', 'varchar:OB_MAX_TRACE_ID_BUFFER_SIZE', 'false'), + ('dir_id', 'int'), + ('bytes', 'int'), + ('start_offset', 'int'), + ('is_deleting', 'bool'), + ('cached_page_num', 'int'), + ('write_back_page_num', 'int'), + ('flushed_page_num', 'int'), + ('ref_cnt', 'int'), + ('total_writes', 'int'), + ('unaligned_writes', 'int'), + ('total_reads', 'int'), + ('unaligned_reads', 'int'), + ('total_read_bytes', 'int'), + ('last_access_time', 'timestamp'), + ('last_modify_time', 'timestamp'), + ('birth_time', 'timestamp'), + ], + partition_columns = ['svr_ip', 'svr_port'], + vtable_route_policy = 'distributed', +) + # 余留位置（此行之前占位） # 本区域占位建议：采用真实表名进行占位 ################################################################################ @@ -15238,7 +15274,9 @@ def_table_schema(**no_direct_access(gen_oracle_mapping_real_virtual_table_def('1 # 15482: __all_virtual_res_mgr_sysstat # 15483: __all_virtual_wr_res_mgr_sysstat # 15484: __all_virtual_function_io_stat -# 15485: __all_virtual_temp_file + +def_table_schema(**gen_oracle_mapping_virtual_table_def('15485', all_def_keywords['__all_virtual_temp_file'])) + # 余留位置（此行之前占位） # 本区域定义的Oracle表名比较复杂，一般都采用gen_xxx_table_def()方式定义，占位建议采用基表表名占位 # - 示例：def_table_schema(**no_direct_access(gen_oracle_mapping_virtual_table_def('15009', all_def_keywords['__all_virtual_sql_audit']))) @@ -36427,8 +36465,66 @@ def_table_schema( # 21619: CDB_OB_KV_REDIS_TABLE # 21620: GV$OB_FUNCTION_IO_STAT # 21621: V$OB_FUNCTION_IO_STAT -# 21622: DBA_OB_TEMP_FILES -# 21623: CDB_OB_TEMP_FILES + +def_table_schema( + owner = 'wuyuefei.wyf', + table_name = 'DBA_OB_TEMP_FILES', + table_id = '21622', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """SELECT + SVR_IP, + SVR_PORT, + FILE_ID, + TRACE_ID, + DIR_ID, + BYTES, + START_OFFSET, + TOTAL_WRITES, + UNALIGNED_WRITES, + TOTAL_READS, + UNALIGNED_READS, + TOTAL_READ_BYTES, + LAST_ACCESS_TIME, + LAST_MODIFY_TIME, + BIRTH_TIME + FROM oceanbase.__all_virtual_temp_file + WHERE TENANT_ID = EFFECTIVE_TENANT_ID() +""".replace("\n", " "), +) + +def_table_schema( + owner = 'wuyuefei.wyf', + table_name = 'CDB_OB_TEMP_FILES', + table_id = '21623', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + view_definition = """SELECT + TENANT_ID, + SVR_IP, + SVR_PORT, + FILE_ID, + TRACE_ID, + DIR_ID, + BYTES, + START_OFFSET, + TOTAL_WRITES, + UNALIGNED_WRITES, + TOTAL_READS, + UNALIGNED_READS, + TOTAL_READ_BYTES, + LAST_ACCESS_TIME, + LAST_MODIFY_TIME, + BIRTH_TIME + FROM oceanbase.__all_virtual_temp_file +""".replace("\n", " ") +) + # 余留位置（此行之前占位） # 本区域占位建议：采用真实视图名进行占位 ################################################################################ @@ -64566,7 +64662,39 @@ left join # 28261: DBA_OB_SPM_EVO_RESULT # 28262: GV$OB_FUNCTION_IO_STAT # 28263: V$OB_FUNCTION_IO_STAT -# 28264: DBA_OB_TEMP_FILES + +def_table_schema( + owner = 'wuyuefei.wyf', + table_name = 'DBA_OB_TEMP_FILES', + name_postfix = '_ORA', + database_id = 'OB_ORA_SYS_DATABASE_ID', + table_id = '28264', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """SELECT + SVR_IP, + SVR_PORT, + FILE_ID, + TRACE_ID, + DIR_ID, + BYTES, + START_OFFSET, + TOTAL_WRITES, + UNALIGNED_WRITES, + TOTAL_READS, + UNALIGNED_READS, + TOTAL_READ_BYTES, + LAST_ACCESS_TIME, + LAST_MODIFY_TIME, + BIRTH_TIME + FROM SYS.ALL_VIRTUAL_TEMP_FILE + WHERE TENANT_ID = EFFECTIVE_TENANT_ID() +""".replace("\n", " "), +) + # 余留位置（此行之前占位） # 本区域占位建议：采用真实视图名进行占位 ################################################################################ diff --git a/src/share/inner_table/table_id_to_name b/src/share/inner_table/table_id_to_name index f558bfc9d..af88010ca 100644 --- a/src/share/inner_table/table_id_to_name +++ b/src/share/inner_table/table_id_to_name @@ -1137,6 +1137,7 @@ # 12488: __all_scheduler_job_run_detail_v2 # BASE_TABLE_NAME # 12490: __all_virtual_spatial_reference_systems # 12490: __all_spatial_reference_systems # BASE_TABLE_NAME +# 12505: __all_virtual_temp_file # 15009: ALL_VIRTUAL_SQL_AUDIT # 15009: __all_virtual_sql_audit # BASE_TABLE_NAME # 15010: ALL_VIRTUAL_PLAN_STAT @@ -1748,6 +1749,8 @@ # 15458: __all_scheduler_job_run_detail_v2 # BASE_TABLE_NAME # 15459: ALL_VIRTUAL_SPATIAL_REFERENCE_SYSTEMS_REAL_AGENT # 15459: __all_spatial_reference_systems # BASE_TABLE_NAME +# 15485: ALL_VIRTUAL_TEMP_FILE +# 15485: __all_virtual_temp_file # BASE_TABLE_NAME # 20001: GV$OB_PLAN_CACHE_STAT # 20002: GV$OB_PLAN_CACHE_PLAN_STAT # 20003: SCHEMATA @@ -2208,6 +2211,8 @@ # 21603: INNODB_SYS_FIELDS # 21604: INNODB_SYS_FOREIGN # 21605: INNODB_SYS_FOREIGN_COLS +# 21622: DBA_OB_TEMP_FILES +# 21623: CDB_OB_TEMP_FILES # 25001: DBA_SYNONYMS # 25002: DBA_OBJECTS # 25003: ALL_OBJECTS @@ -2675,6 +2680,7 @@ # 28230: GV$OB_NIC_INFO # 28231: V$OB_NIC_INFO # 28234: DBA_OB_SPATIAL_COLUMNS +# 28264: DBA_OB_TEMP_FILES # 14999: __idx_11003_all_virtual_plan_cache_stat_i1 # 14999: all_virtual_plan_cache_stat_i1 # INDEX_NAME # 14999: __all_virtual_plan_cache_stat # DATA_BASE_TABLE_NAME diff --git a/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp b/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp index 6dbc8fea8..8bd564507 100644 --- a/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp +++ b/src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp @@ -28,6 +28,74 @@ namespace oceanbase { namespace tmp_file { + +int ObSNTmpFileInfo::init( + const ObCurTraceId::TraceId &trace_id, + const uint64_t tenant_id, + const int64_t dir_id, + const int64_t fd, + const int64_t file_size, + const int64_t truncated_offset, + const bool is_deleting, + const int64_t cached_page_num, + const int64_t write_back_data_page_num, + const int64_t flushed_data_page_num, + const int64_t ref_cnt, + const int64_t write_req_cnt, + const int64_t unaligned_write_req_cnt, + const int64_t read_req_cnt, + const int64_t unaligned_read_req_cnt, + const int64_t total_read_size, + const int64_t last_access_ts, + const int64_t last_modify_ts, + const int64_t birth_ts) +{ + int ret = OB_SUCCESS; + trace_id_ = trace_id; + tenant_id_ = tenant_id; + dir_id_ = dir_id; + fd_ = fd; + file_size_ = file_size; + truncated_offset_ = truncated_offset; + is_deleting_ = is_deleting; + cached_page_num_ = cached_page_num; + write_back_data_page_num_ = write_back_data_page_num; + flushed_data_page_num_ = flushed_data_page_num; + ref_cnt_ = ref_cnt; + write_req_cnt_ = write_req_cnt; + unaligned_write_req_cnt_ = unaligned_write_req_cnt; + read_req_cnt_ = read_req_cnt; + unaligned_read_req_cnt_ = unaligned_read_req_cnt; + total_read_size_ = total_read_size; + last_access_ts_ = last_access_ts; + last_modify_ts_ = last_modify_ts; + birth_ts_ = birth_ts; + return ret; +} + +void ObSNTmpFileInfo::reset() +{ + trace_id_.reset(); + tenant_id_ = OB_INVALID_TENANT_ID; + dir_id_ = ObTmpFileGlobal::INVALID_TMP_FILE_DIR_ID; + fd_ = ObTmpFileGlobal::INVALID_TMP_FILE_FD; + file_size_ = 0; + truncated_offset_ = 0; + is_deleting_ = false; + cached_page_num_ = 0; + write_back_data_page_num_ = 0; + flushed_data_page_num_ = 0; + ref_cnt_ = 0; + write_req_cnt_ = 0; + unaligned_write_req_cnt_ = 0; + read_req_cnt_ = 0; + unaligned_read_req_cnt_ = 0; + total_read_size_ = 0; + last_access_ts_ = -1; + last_modify_ts_ = -1; + birth_ts_ = -1; +} + ObTmpFileHandle::ObTmpFileHandle(ObSharedNothingTmpFile *tmp_file) : ptr_(tmp_file) { @@ -188,7 +256,16 @@ ObSharedNothingTmpFile::ObSharedNothingTmpFile() last_page_lock_(common::ObLatchIds::TMP_FILE_LOCK), multi_write_lock_(common::ObLatchIds::TMP_FILE_LOCK), truncate_lock_(common::ObLatchIds::TMP_FILE_LOCK), - inner_flush_ctx_() + inner_flush_ctx_(), + trace_id_(), + write_req_cnt_(0), + unaligned_write_req_cnt_(0), + read_req_cnt_(0), + unaligned_read_req_cnt_(0), + total_read_size_(0), + last_access_ts_(-1), + last_modify_ts_(-1), + birth_ts_(-1) { } @@ -233,6 +310,15 @@ int ObSharedNothingTmpFile::init(const uint64_t tenant_id, const int64_t fd, con tenant_id_ = tenant_id; dir_id_ = dir_id; fd_ = fd; + ObCurTraceId::TraceId *cur_trace_id = ObCurTraceId::get_trace_id(); + if (nullptr != cur_trace_id) { + trace_id_ = *cur_trace_id; + } else { + trace_id_.init(GCONF.self_addr_); + } + last_access_ts_ = ObTimeUtility::current_time(); + last_modify_ts_ = ObTimeUtility::current_time(); + birth_ts_ = ObTimeUtility::current_time(); } LOG_INFO("tmp file init over", KR(ret), K(fd), K(dir_id)); @@ -314,6 +400,17 @@ void ObSharedNothingTmpFile::reset() data_eviction_node_.unlink(); meta_eviction_node_.unlink(); inner_flush_ctx_.reset(); + /******for virtual table begin******/ + trace_id_.reset(); + write_req_cnt_ = 0; + unaligned_write_req_cnt_ = 0; + read_req_cnt_ = 0; + unaligned_read_req_cnt_ = 0; + total_read_size_ = 0; + last_access_ts_ = -1; + last_modify_ts_ = -1; + birth_ts_ = -1; + /******for virtual table end******/ } bool ObSharedNothingTmpFile::is_deleting() @@ -360,6 +457,10 @@ int ObSharedNothingTmpFile::aio_pread(ObTmpFileIOCtx &io_ctx) if (io_ctx.get_read_offset_in_file() < 0) { io_ctx.set_read_offset_in_file(read_offset_); } + if (0 != io_ctx.get_read_offset_in_file() % ObTmpFileGlobal::PAGE_SIZE + || 0 != io_ctx.get_todo_size() % ObTmpFileGlobal::PAGE_SIZE) { + io_ctx.set_is_unaligned_read(true); + } LOG_DEBUG("start to inner read tmp file", K(fd_), K(io_ctx.get_read_offset_in_file()), K(io_ctx.get_todo_size()), K(io_ctx.get_done_size()), KPC(this)); @@ -898,6 +999,8 @@ int ObSharedNothingTmpFile::aio_write(ObTmpFileIOCtx &io_ctx) ret = OB_ERR_UNEXPECTED; LOG_WARN("attempt to write a deleting file", KR(ret), K(fd_)); } else { + bool is_unaligned_write = 0 != file_size_ % ObTmpFileGlobal::PAGE_SIZE + || 0 != io_ctx.get_todo_size() % ObTmpFileGlobal::PAGE_SIZE; while (OB_SUCC(ret) && io_ctx.get_todo_size() > 0) { if (OB_FAIL(inner_write_(io_ctx))) { if (OB_ALLOCATE_TMP_FILE_PAGE_FAILED == ret) { @@ -915,6 +1018,13 @@ int ObSharedNothingTmpFile::aio_write(ObTmpFileIOCtx &io_ctx) } } } // end while + if (OB_SUCC(ret)) { + write_req_cnt_++; + if (is_unaligned_write) { + unaligned_write_req_cnt_++; + } + last_modify_ts_ = ObTimeUtility::current_time(); + } } if (OB_SUCC(ret)) { @@ -1594,6 +1704,7 @@ int ObSharedNothingTmpFile::truncate(const int64_t truncate_offset) LOG_WARN("fail to truncate data page", KR(ret), K(fd_), K(truncate_offset), KPC(this)); } else { truncated_offset_ = truncate_offset; + last_modify_ts_ = ObTimeUtility::current_time(); } } @@ -1816,6 +1927,36 @@ int64_t ObSharedNothingTmpFile::cal_wbp_begin_offset_() const return res; } +void ObSharedNothingTmpFile::set_read_stats_vars(const bool is_unaligned_read, const int64_t read_size) +{ + common::TCRWLock::WLockGuard guard(meta_lock_); + read_req_cnt_++; + if (is_unaligned_read) { + unaligned_read_req_cnt_++; + } + total_read_size_ += read_size; + last_access_ts_ = ObTimeUtility::current_time(); +} + +int ObSharedNothingTmpFile::copy_info_for_virtual_table(ObSNTmpFileInfo &tmp_file_info) +{ + int ret = OB_SUCCESS; + common::TCRWLock::RLockGuardWithTimeout lock_guard(meta_lock_, 100 * 1000L, ret); + if (OB_FAIL(ret)) { + } else if (OB_FAIL(tmp_file_info.init(trace_id_, tenant_id_, + dir_id_, fd_, file_size_, + truncated_offset_, is_deleting_, + cached_page_nums_, write_back_data_page_num_, + flushed_data_page_num_, ref_cnt_, + write_req_cnt_, unaligned_write_req_cnt_, + read_req_cnt_, unaligned_read_req_cnt_, + total_read_size_, last_access_ts_, + last_modify_ts_, birth_ts_))) { + LOG_WARN("fail to init tmp_file_info", KR(ret), KPC(this)); + } + return ret; +}; + int ObSharedNothingTmpFile::remove_flush_node(const bool is_meta) { int ret = OB_SUCCESS; diff --git a/src/storage/tmp_file/ob_shared_nothing_tmp_file.h b/src/storage/tmp_file/ob_shared_nothing_tmp_file.h index 6d6c3414d..dd3ca8bb4 100644 --- a/src/storage/tmp_file/ob_shared_nothing_tmp_file.h +++ b/src/storage/tmp_file/ob_shared_nothing_tmp_file.h @@ -40,6 +40,79 @@ class ObTmpFileFlushManager; class ObTmpFileTreeFlushContext; class ObTmpFileDataFlushContext; +//for virtual table show +class ObSNTmpFileInfo +{ +public: + ObSNTmpFileInfo() : + trace_id_(), + tenant_id_(OB_INVALID_TENANT_ID), + dir_id_(ObTmpFileGlobal::INVALID_TMP_FILE_DIR_ID), + fd_(ObTmpFileGlobal::INVALID_TMP_FILE_FD), + file_size_(0), + truncated_offset_(0), + is_deleting_(false), + cached_page_num_(0), + write_back_data_page_num_(0), + flushed_data_page_num_(0), + ref_cnt_(0), + write_req_cnt_(0), + unaligned_write_req_cnt_(0), + read_req_cnt_(0), + unaligned_read_req_cnt_(0), + total_read_size_(0), + last_access_ts_(-1), + last_modify_ts_(-1), + birth_ts_(-1) {} + int init(const ObCurTraceId::TraceId &trace_id, + const uint64_t tenant_id, + const int64_t dir_id, + const int64_t fd, + const int64_t file_size, + const int64_t truncated_offset, + const bool is_deleting, + const int64_t cached_page_num, + const int64_t write_back_data_page_num, + const int64_t flushed_data_page_num, + const int64_t ref_cnt, + const int64_t write_req_cnt, + const int64_t unaligned_write_req_cnt, + const int64_t read_req_cnt, + const int64_t unaligned_read_req_cnt, + const int64_t total_read_size, + const int64_t last_access_ts, + const int64_t last_modify_ts, + const int64_t birth_ts); + void reset(); +public: + common::ObCurTraceId::TraceId trace_id_; + uint64_t tenant_id_; + int64_t dir_id_; + int64_t fd_; + int64_t file_size_; + int64_t truncated_offset_; + bool is_deleting_; + int64_t cached_page_num_; + int64_t write_back_data_page_num_; + int64_t flushed_data_page_num_; + int64_t ref_cnt_; + int64_t write_req_cnt_; + int64_t unaligned_write_req_cnt_; + int64_t read_req_cnt_; + int64_t unaligned_read_req_cnt_; + int64_t total_read_size_; + int64_t last_access_ts_; + int64_t last_modify_ts_; + int64_t birth_ts_; + + TO_STRING_KV(K(trace_id_), K(tenant_id_), K(dir_id_), K(fd_), K(file_size_), + K(truncated_offset_), K(is_deleting_), K(cached_page_num_), + K(write_back_data_page_num_), K(flushed_data_page_num_), + K(ref_cnt_), K(write_req_cnt_), K(unaligned_write_req_cnt_), + K(read_req_cnt_), K(unaligned_read_req_cnt_), K(total_read_size_), + K(last_access_ts_), K(last_modify_ts_), K(birth_ts_)); +}; + class ObSharedNothingTmpFile final { public: @@ -143,7 +216,10 @@ public: KP(meta_flush_node_.get_next()), K(is_in_data_eviction_list_), K(is_in_meta_eviction_list_), KP(data_eviction_node_.get_next()), - KP(meta_eviction_node_.get_next())); + KP(meta_eviction_node_.get_next()), K(trace_id_), + K(write_req_cnt_), K(unaligned_write_req_cnt_), K(read_req_cnt_), + K(unaligned_read_req_cnt_), K(total_read_size_), + K(last_access_ts_), K(last_modify_ts_), K(birth_ts_)); // XXX Currently, K(tmp_file) is used to print the ObSharedNothingTmpFile structure without holding // the file lock. Before adding the print field, make sure it is thread-safe. @@ -209,6 +285,8 @@ public: void get_dirty_meta_page_num(int64_t &non_rightmost_dirty_page_num, int64_t &rightmost_dirty_page_num) const; void get_dirty_meta_page_num_with_lock(int64_t &non_rightmost_dirty_page_num, int64_t &rightmost_dirty_page_num); int64_t cal_wbp_begin_offset(); + void set_read_stats_vars(const bool is_unaligned_read, const int64_t read_size); + int copy_info_for_virtual_table(ObSNTmpFileInfo &tmp_file_info); private: int inner_read_truncated_part_(ObTmpFileIOCtx &io_ctx); @@ -360,6 +438,17 @@ private: ObSpinLock multi_write_lock_; // handle conflicts between multiple writes SpinRWLock truncate_lock_; // handle conflicts between truncate and flushing InnerFlushContext inner_flush_ctx_; // file-level flush context + /********for virtual table begin********/ + common::ObCurTraceId::TraceId trace_id_; + int64_t write_req_cnt_; + int64_t unaligned_write_req_cnt_; + int64_t read_req_cnt_; + int64_t unaligned_read_req_cnt_; + int64_t total_read_size_; + int64_t last_access_ts_; + int64_t last_modify_ts_; + int64_t birth_ts_; + /********for virtual table end********/ }; class ObTmpFileHandle final diff --git a/src/storage/tmp_file/ob_tmp_file_block_manager.cpp b/src/storage/tmp_file/ob_tmp_file_block_manager.cpp index 7fe4d4ada..9c49c617b 100644 --- a/src/storage/tmp_file/ob_tmp_file_block_manager.cpp +++ b/src/storage/tmp_file/ob_tmp_file_block_manager.cpp @@ -546,8 +546,7 @@ ObTmpFileBlockManager::ObTmpFileBlockManager() : block_index_generator_(0), block_map_(), block_allocator_(), - stat_lock_(), - map_lock_() + stat_lock_() {} ObTmpFileBlockManager::~ObTmpFileBlockManager() @@ -613,7 +612,6 @@ int ObTmpFileBlockManager::create_tmp_file_block(const int64_t begin_page_id, co } else if (OB_FAIL(handle.init(blk, this))) { LOG_WARN("fail to init tmp file block handle", KR(ret), K(block_index)); } else { - SharedLockGuard guard(map_lock_); if (OB_FAIL(block_map_.insert(ObTmpFileBlockKey(block_index), handle))) { LOG_WARN("fail to insert tmp file block into map", KR(ret), K(block_index)); } @@ -765,7 +763,6 @@ int ObTmpFileBlockManager::remove_tmp_file_block_(const int64_t block_index) ret = OB_NOT_INIT; LOG_WARN("ObTmpFileBlockManager has not been inited", KR(ret), K(tenant_id_)); } else { - SharedLockGuard guard(map_lock_); if (OB_FAIL(block_map_.erase(ObTmpFileBlockKey(block_index), handle))) { if (ret != OB_ENTRY_NOT_EXIST) { LOG_WARN("fail to erase tmp file block", KR(ret), K(block_index)); @@ -847,7 +844,6 @@ int ObTmpFileBlockManager::get_macro_block_count(int64_t ¯o_block_count) int ObTmpFileBlockManager::get_macro_block_list(common::ObIArray ¯o_id_list) { int ret = OB_SUCCESS; - ExclusiveLockGuard guard(map_lock_); CollectMacroBlockIdFunctor func(macro_id_list); if (IS_NOT_INIT) { ret = OB_NOT_INIT; diff --git a/src/storage/tmp_file/ob_tmp_file_block_manager.h b/src/storage/tmp_file/ob_tmp_file_block_manager.h index 593ec8f9a..927b15b12 100644 --- a/src/storage/tmp_file/ob_tmp_file_block_manager.h +++ b/src/storage/tmp_file/ob_tmp_file_block_manager.h @@ -212,7 +212,6 @@ private: ObTmpFileBlockMap block_map_; common::ObConcurrentFIFOAllocator block_allocator_; common::SpinRWLock stat_lock_; // to protect the consistency of used_page_num_ and physical_block_num_ - common::SpinRWLock map_lock_; // to protect the for_each operation of block_map_ DISALLOW_COPY_AND_ASSIGN(ObTmpFileBlockManager); }; diff --git a/src/storage/tmp_file/ob_tmp_file_io_ctx.cpp b/src/storage/tmp_file/ob_tmp_file_io_ctx.cpp index 0930b1dd3..072dfe0cf 100644 --- a/src/storage/tmp_file/ob_tmp_file_io_ctx.cpp +++ b/src/storage/tmp_file/ob_tmp_file_io_ctx.cpp @@ -33,6 +33,7 @@ ObTmpFileIOCtx::ObTmpFileIOCtx(): read_offset_in_file_(-1), disable_page_cache_(false), disable_block_cache_(false), + is_unaligned_read_(false), io_flag_(), io_timeout_ms_(DEFAULT_IO_WAIT_TIME_MS), io_handles_(), @@ -111,6 +112,7 @@ void ObTmpFileIOCtx::reset() dir_id_ = ObTmpFileGlobal::INVALID_TMP_FILE_DIR_ID; disable_page_cache_ = false; disable_block_cache_ = false; + is_unaligned_read_ = false; io_flag_.reset(); io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS; } diff --git a/src/storage/tmp_file/ob_tmp_file_io_ctx.h b/src/storage/tmp_file/ob_tmp_file_io_ctx.h index 6ea971511..d4f17d000 100644 --- a/src/storage/tmp_file/ob_tmp_file_io_ctx.h +++ b/src/storage/tmp_file/ob_tmp_file_io_ctx.h @@ -62,6 +62,8 @@ public: OB_INLINE bool is_disable_block_cache() const { return disable_block_cache_; } OB_INLINE common::ObIOFlag get_io_flag() const { return io_flag_; } OB_INLINE int64_t get_io_timeout_ms() const { return io_timeout_ms_; } + OB_INLINE void set_is_unaligned_read(const bool is_unaligned_read) { is_unaligned_read_ = is_unaligned_read; } + OB_INLINE bool is_unaligned_read() { return is_unaligned_read_; } TO_STRING_KV(K(is_inited_), K(is_read_), K(fd_), K(dir_id_), KP(buf_), @@ -161,6 +163,7 @@ private: int64_t read_offset_in_file_; bool disable_page_cache_; bool disable_block_cache_; // only used in ut, to control whether read data from block cache + bool is_unaligned_read_; //for statistics common::ObIOFlag io_flag_; int64_t io_timeout_ms_; common::ObSEArray io_handles_; diff --git a/src/storage/tmp_file/ob_tmp_file_manager.cpp b/src/storage/tmp_file/ob_tmp_file_manager.cpp index bed07322c..2340346e3 100644 --- a/src/storage/tmp_file/ob_tmp_file_manager.cpp +++ b/src/storage/tmp_file/ob_tmp_file_manager.cpp @@ -307,6 +307,9 @@ int ObTenantTmpFileManager::aio_read(const ObTmpFileIOInfo &io_info, ObTmpFileIO LOG_WARN("fail to init io handle", KR(ret), K(io_info)); } else if (OB_FAIL(tmp_file_handle.get()->aio_pread(io_handle.get_io_ctx()))) { LOG_WARN("fail to aio pread", KR(ret), K(io_info)); + } else { + tmp_file_handle.get()->set_read_stats_vars(io_handle.get_io_ctx().is_unaligned_read(), + io_info.size_); } LOG_DEBUG("aio_read a tmp file over", KR(ret), K(io_info), K(io_handle), KPC(tmp_file_handle.get())); @@ -336,6 +339,9 @@ int ObTenantTmpFileManager::aio_pread(const ObTmpFileIOInfo &io_info, LOG_WARN("fail to init io handle", KR(ret), K(io_info)); } else if (OB_FAIL(tmp_file_handle.get()->aio_pread(io_handle.get_io_ctx()))) { LOG_WARN("fail to aio pread", KR(ret), K(io_info)); + } else { + tmp_file_handle.get()->set_read_stats_vars(io_handle.get_io_ctx().is_unaligned_read(), + io_info.size_); } LOG_DEBUG("aio_pread a tmp file over", KR(ret), K(io_info), K(offset), K(io_handle), KPC(tmp_file_handle.get())); @@ -363,6 +369,9 @@ int ObTenantTmpFileManager::read(const ObTmpFileIOInfo &io_info, ObTmpFileIOHand LOG_WARN("fail to init io handle", KR(ret), K(io_info)); } else if (OB_FAIL(tmp_file_handle.get()->aio_pread(io_handle.get_io_ctx()))) { LOG_WARN("fail to aio pread", KR(ret), K(io_info)); + } else { + tmp_file_handle.get()->set_read_stats_vars(io_handle.get_io_ctx().is_unaligned_read(), + io_info.size_); } if (OB_SUCC(ret) || OB_ITER_END == ret) { @@ -398,6 +407,9 @@ int ObTenantTmpFileManager::pread(const ObTmpFileIOInfo &io_info, const int64_t LOG_WARN("fail to init io handle", KR(ret), K(io_info)); } else if (OB_FAIL(tmp_file_handle.get()->aio_pread(io_handle.get_io_ctx()))) { LOG_WARN("fail to aio pread", KR(ret), K(io_info)); + } else { + tmp_file_handle.get()->set_read_stats_vars(io_handle.get_io_ctx().is_unaligned_read(), + io_info.size_); } if (OB_SUCC(ret) || OB_ITER_END == ret) { @@ -480,7 +492,7 @@ int ObTenantTmpFileManager::get_tmp_file(const int64_t fd, ObTmpFileHandle &file int ret = OB_SUCCESS; if (OB_FAIL(files_.get(ObTmpFileKey(fd), file_handle))) { - if (OB_HASH_NOT_EXIST == ret) { + if (OB_ENTRY_NOT_EXIST == ret) { LOG_WARN("tmp file does not exist", KR(ret), K(fd)); } else { LOG_WARN("fail to get tmp file", KR(ret), K(fd)); @@ -538,5 +550,57 @@ int ObTenantTmpFileManager::get_macro_block_count(int64_t ¯o_block_count) return ret; } +bool ObTenantTmpFileManager::CollectTmpFileKeyFunctor::operator()( + const ObTmpFileKey &key, const ObTmpFileHandle &tmp_file_handle) +{ + int ret = OB_SUCCESS; + ObSharedNothingTmpFile *tmp_file_ptr = NULL; + + if (OB_ISNULL(tmp_file_handle.get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get invalid tmp file pointer", KR(ret), K(key), KP(tmp_file_handle.get())); + } else if (OB_FAIL(fds_.push_back(key.fd_))) { + LOG_WARN("failed to push back", KR(ret), K(key)); + } + return OB_SUCCESS == ret; +} + +int ObTenantTmpFileManager::get_tmp_file_fds(ObIArray &fd_arr) +{ + int ret = OB_SUCCESS; + CollectTmpFileKeyFunctor func(fd_arr); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTenantTmpFileManager has not been inited", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(files_.for_each(func))) { + LOG_WARN("fail to collect tmp file fds", KR(ret)); + } + + return ret; +} + +int ObTenantTmpFileManager::get_tmp_file_info(const int64_t fd, ObSNTmpFileInfo &tmp_file_info) +{ + int ret = OB_SUCCESS; + ObTmpFileHandle file_handle; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTenantTmpFileManager has not been inited", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(get_tmp_file(fd, file_handle))) { + if (OB_ENTRY_NOT_EXIST == ret) { + LOG_INFO("tmp file not exist", KR(ret), K(fd)); + } else { + LOG_WARN("fail to get tmp file", KR(ret), K(fd)); + } + } else if (OB_ISNULL(file_handle.get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get invalid tmp file pointer", KR(ret), K(fd), KP(file_handle.get())); + } else if (OB_FAIL(file_handle.get()->copy_info_for_virtual_table(tmp_file_info))) { + LOG_WARN("failed to copy info for virtual table", KR(ret), K(fd), KPC(file_handle.get())); + } + + return ret; +} + } // end namespace tmp_file } // end namespace oceanbase diff --git a/src/storage/tmp_file/ob_tmp_file_manager.h b/src/storage/tmp_file/ob_tmp_file_manager.h index 2d9353288..3c794b8e1 100644 --- a/src/storage/tmp_file/ob_tmp_file_manager.h +++ b/src/storage/tmp_file/ob_tmp_file_manager.h @@ -83,6 +83,22 @@ public: OB_INLINE ObTmpFileBlockManager &get_tmp_file_block_manager() { return tmp_file_block_manager_; } OB_INLINE ObTmpFilePageCacheController &get_page_cache_controller() { return page_cache_controller_; } +public: + //for virtual table to show + int get_tmp_file_fds(ObIArray &fd_arr); + int get_tmp_file_info(const int64_t fd, ObSNTmpFileInfo &tmp_file_info); +private: + class CollectTmpFileKeyFunctor final + { + public: + CollectTmpFileKeyFunctor(ObIArray &fds) + : fds_(fds) {} + bool operator()(const ObTmpFileKey &key, const ObTmpFileHandle &tmp_file_handle); + + private: + ObIArray &fds_; + }; + private: static const int64_t REFRESH_CONFIG_INTERVAL = 5 * 60 * 1000 * 1000L; // 5min static const int64_t META_DEFAULT_LIMIT = 15 * 1024L * 1024L * 1024L; diff --git a/tools/deploy/mysql_test/r/mysql/information_schema.result b/tools/deploy/mysql_test/r/mysql/information_schema.result index a712732cd..6ef41f4cf 100644 --- a/tools/deploy/mysql_test/r/mysql/information_schema.result +++ b/tools/deploy/mysql_test/r/mysql/information_schema.result @@ -351,6 +351,7 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_TABLE_STAT_STALE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_TASK_OPT_STAT_GATHER_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TEMP_FILES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_TENANTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_TENANT_EVENT_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_TRANSFER_PARTITION_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | @@ -888,6 +889,7 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | __all_virtual_tablet_meta_table | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_tablet_stat | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_task_opt_stat_gather_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_temp_file | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_tenant_event_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_tenant_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_tenant_memory_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | @@ -1882,6 +1884,7 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_TABLE_STAT_STALE_INFO | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_TASK_OPT_STAT_GATHER_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_OB_TEMP_FILES | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_TENANTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_TENANT_EVENT_HISTORY | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_TRANSFER_PARTITION_TASKS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | @@ -2419,6 +2422,7 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | __all_virtual_table_mgr | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_table_opt_stat_gather_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_task_opt_stat_gather_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | __all_virtual_temp_file | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_tenant_event_history | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_tenant_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | __all_virtual_tenant_memory_info | SYSTEM TABLE | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result index 84961069d..a935ea56a 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result @@ -6839,6 +6839,26 @@ POS bigint(0) unsigned NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from information_schema.INNODB_SYS_FOREIGN_COLS limit 1); cnt 1 +desc oceanbase.DBA_OB_TEMP_FILES; +Field Type Null Key Default Extra +SVR_IP varchar(46) NO NULL +SVR_PORT bigint(20) NO NULL +FILE_ID bigint(20) NO NULL +TRACE_ID varchar(64) NO NULL +DIR_ID bigint(20) NO NULL +BYTES bigint(20) NO NULL +START_OFFSET bigint(20) NO NULL +TOTAL_WRITES bigint(20) NO NULL +UNALIGNED_WRITES bigint(20) NO NULL +TOTAL_READS bigint(20) NO NULL +UNALIGNED_READS bigint(20) NO NULL +TOTAL_READ_BYTES bigint(20) NO NULL +LAST_ACCESS_TIME timestamp(6) NO NULL +LAST_MODIFY_TIME timestamp(6) NO NULL +BIRTH_TIME timestamp(6) NO NULL +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_OB_TEMP_FILES limit 1); +cnt +1 select case cnt when 0 then NULL else 'UNEXPECTED ERROR: It is expected to be an empty set, which means that all GV$ and V$ view column names are defined consistently' end ERROR_INFO from (select /*+no_rewrite*/ count(*) cnt from (SELECT t.table_name, group_concat(c.column_name) as column_name_list diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result index 561477619..0d840b986 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result @@ -9682,6 +9682,47 @@ POS bigint(0) unsigned NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from information_schema.INNODB_SYS_FOREIGN_COLS limit 1); cnt 1 +desc oceanbase.DBA_OB_TEMP_FILES; +Field Type Null Key Default Extra +SVR_IP varchar(46) NO NULL +SVR_PORT bigint(20) NO NULL +FILE_ID bigint(20) NO NULL +TRACE_ID varchar(64) NO NULL +DIR_ID bigint(20) NO NULL +BYTES bigint(20) NO NULL +START_OFFSET bigint(20) NO NULL +TOTAL_WRITES bigint(20) NO NULL +UNALIGNED_WRITES bigint(20) NO NULL +TOTAL_READS bigint(20) NO NULL +UNALIGNED_READS bigint(20) NO NULL +TOTAL_READ_BYTES bigint(20) NO NULL +LAST_ACCESS_TIME timestamp(6) NO NULL +LAST_MODIFY_TIME timestamp(6) NO NULL +BIRTH_TIME timestamp(6) NO NULL +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_OB_TEMP_FILES limit 1); +cnt +1 +desc oceanbase.CDB_OB_TEMP_FILES; +Field Type Null Key Default Extra +TENANT_ID bigint(20) NO NULL +SVR_IP varchar(46) NO NULL +SVR_PORT bigint(20) NO NULL +FILE_ID bigint(20) NO NULL +TRACE_ID varchar(64) NO NULL +DIR_ID bigint(20) NO NULL +BYTES bigint(20) NO NULL +START_OFFSET bigint(20) NO NULL +TOTAL_WRITES bigint(20) NO NULL +UNALIGNED_WRITES bigint(20) NO NULL +TOTAL_READS bigint(20) NO NULL +UNALIGNED_READS bigint(20) NO NULL +TOTAL_READ_BYTES bigint(20) NO NULL +LAST_ACCESS_TIME timestamp(6) NO NULL +LAST_MODIFY_TIME timestamp(6) NO NULL +BIRTH_TIME timestamp(6) NO NULL +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.CDB_OB_TEMP_FILES limit 1); +cnt +1 select case cnt when 0 then NULL else 'UNEXPECTED ERROR: It is expected to be an empty set, which means that all GV$ and V$ view column names are defined consistently' end ERROR_INFO from (select /*+no_rewrite*/ count(*) cnt from (SELECT t.table_name, group_concat(c.column_name) as column_name_list diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result index 354f437c9..714f6df88 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result @@ -5039,3 +5039,32 @@ IF(count(*) >= 0, 1, 0) "oceanbase.__all_virtual_nic_info runs in single server" IF(count(*) >= 0, 1, 0) 1 +desc oceanbase.__all_virtual_temp_file; +Field Type Null Key Default Extra +tenant_id bigint(20) NO NULL +svr_ip varchar(46) NO NULL +svr_port bigint(20) NO NULL +file_id bigint(20) NO NULL +trace_id varchar(64) NO NULL +dir_id bigint(20) NO NULL +bytes bigint(20) NO NULL +start_offset bigint(20) NO NULL +is_deleting tinyint(4) NO NULL +cached_page_num bigint(20) NO NULL +write_back_page_num bigint(20) NO NULL +flushed_page_num bigint(20) NO NULL +ref_cnt bigint(20) NO NULL +total_writes bigint(20) NO NULL +unaligned_writes bigint(20) NO NULL +total_reads bigint(20) NO NULL +unaligned_reads bigint(20) NO NULL +total_read_bytes bigint(20) NO NULL +last_access_time timestamp(6) NO NULL +last_modify_time timestamp(6) NO NULL +birth_time timestamp(6) NO NULL +select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_temp_file; +IF(count(*) >= 0, 1, 0) +1 +"oceanbase.__all_virtual_temp_file runs in single server" +IF(count(*) >= 0, 1, 0) +1 diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result index 509911307..d24a8e582 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result @@ -9750,3 +9750,32 @@ description varchar(2048) YES NULL select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_spatial_reference_systems; IF(count(*) >= 0, 1, 0) 1 +desc oceanbase.__all_virtual_temp_file; +Field Type Null Key Default Extra +tenant_id bigint(20) NO NULL +svr_ip varchar(46) NO NULL +svr_port bigint(20) NO NULL +file_id bigint(20) NO NULL +trace_id varchar(64) NO NULL +dir_id bigint(20) NO NULL +bytes bigint(20) NO NULL +start_offset bigint(20) NO NULL +is_deleting tinyint(4) NO NULL +cached_page_num bigint(20) NO NULL +write_back_page_num bigint(20) NO NULL +flushed_page_num bigint(20) NO NULL +ref_cnt bigint(20) NO NULL +total_writes bigint(20) NO NULL +unaligned_writes bigint(20) NO NULL +total_reads bigint(20) NO NULL +unaligned_reads bigint(20) NO NULL +total_read_bytes bigint(20) NO NULL +last_access_time timestamp(6) NO NULL +last_modify_time timestamp(6) NO NULL +birth_time timestamp(6) NO NULL +select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_temp_file; +IF(count(*) >= 0, 1, 0) +1 +"oceanbase.__all_virtual_temp_file runs in single server" +IF(count(*) >= 0, 1, 0) +1 diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result index be310cd82..a2b3714ec 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result @@ -762,6 +762,7 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr 12487 __all_virtual_nic_info 2 201001 1 12488 __all_virtual_scheduler_job_run_detail_v2 2 201001 1 12490 __all_virtual_spatial_reference_systems 2 201001 1 +12505 __all_virtual_temp_file 2 201001 1 20001 GV$OB_PLAN_CACHE_STAT 1 201001 1 20002 GV$OB_PLAN_CACHE_PLAN_STAT 1 201001 1 20003 SCHEMATA 1 201002 1 @@ -1222,6 +1223,8 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr 21603 INNODB_SYS_FIELDS 1 201002 1 21604 INNODB_SYS_FOREIGN 1 201002 1 21605 INNODB_SYS_FOREIGN_COLS 1 201002 1 +21622 DBA_OB_TEMP_FILES 1 201001 1 +21623 CDB_OB_TEMP_FILES 1 201001 1 check sys table count and table_id range success check count and table_id range for virtual table success select * from information_schema.CHARACTER_SETS limit 1; From ae6a932613d00b0616889c74af40b9d70b44966b Mon Sep 17 00:00:00 2001 From: SevenJ-swj Date: Mon, 26 Aug 2024 06:32:54 +0000 Subject: [PATCH 216/249] [FEAT MERGE] merge external table orc read feature --- deps/init/oceanbase.el7.aarch64.deps | 1 + deps/init/oceanbase.el7.x86_64.deps | 1 + deps/init/oceanbase.el8.aarch64.deps | 1 + deps/init/oceanbase.el8.x86_64.deps | 1 + deps/init/oceanbase.el9.aarch64.deps | 1 + deps/init/oceanbase.el9.x86_64.deps | 1 + deps/oblib/src/CMakeLists.txt | 13 + deps/oblib/unittest/lib/CMakeLists.txt | 1 + deps/oblib/unittest/lib/orc/test_orc.cpp | 603 +++++++ src/sql/CMakeLists.txt | 1 + src/sql/engine/cmd/ob_load_data_parser.cpp | 12 +- src/sql/engine/cmd/ob_load_data_parser.h | 2 +- .../ob_external_table_access_service.cpp | 9 +- .../engine/table/ob_orc_table_row_iter.cpp | 1382 +++++++++++++++++ src/sql/engine/table/ob_orc_table_row_iter.h | 235 +++ src/sql/resolver/dml/ob_dml_resolver.cpp | 18 +- 16 files changed, 2271 insertions(+), 11 deletions(-) create mode 100644 deps/oblib/unittest/lib/orc/test_orc.cpp create mode 100644 src/sql/engine/table/ob_orc_table_row_iter.cpp create mode 100644 src/sql/engine/table/ob_orc_table_row_iter.h diff --git a/deps/init/oceanbase.el7.aarch64.deps b/deps/init/oceanbase.el7.aarch64.deps index 0d7a5fa21..01601d065 100644 --- a/deps/init/oceanbase.el7.aarch64.deps +++ b/deps/init/oceanbase.el7.aarch64.deps @@ -33,6 +33,7 @@ devdeps-s3-cpp-sdk-1.11.156-102023122011.el7.aarch64.rpm devdeps-protobuf-c-1.4.1-100000072023102410.el7.aarch64.rpm devdeps-roaringbitmap-croaring-3.0.0-42024042816.el7.aarch64.rpm devdeps-apache-arrow-9.0.0-302024052920.el7.aarch64.rpm +devdeps-apache-orc-1.8.3-202024072510.el7.aarch64.rpm [tools] obdevtools-binutils-2.30-12022100413.el7.aarch64.rpm diff --git a/deps/init/oceanbase.el7.x86_64.deps b/deps/init/oceanbase.el7.x86_64.deps index 168f4725f..1f3f350de 100644 --- a/deps/init/oceanbase.el7.x86_64.deps +++ b/deps/init/oceanbase.el7.x86_64.deps @@ -36,6 +36,7 @@ devdeps-s3-cpp-sdk-1.11.156-102023122011.el7.x86_64.rpm devdeps-protobuf-c-1.4.1-100000062023102016.el7.x86_64.rpm devdeps-roaringbitmap-croaring-3.0.0-42024042816.el7.x86_64.rpm devdeps-apache-arrow-9.0.0-222024052223.el7.x86_64.rpm +devdeps-apache-orc-1.8.3-202024072510.el7.x86_64.rpm [tools] obdevtools-binutils-2.30-12022100413.el7.x86_64.rpm diff --git a/deps/init/oceanbase.el8.aarch64.deps b/deps/init/oceanbase.el8.aarch64.deps index 25f5f5b46..7f32f9b7f 100644 --- a/deps/init/oceanbase.el8.aarch64.deps +++ b/deps/init/oceanbase.el8.aarch64.deps @@ -33,6 +33,7 @@ devdeps-s3-cpp-sdk-1.11.156-102023122011.el8.aarch64.rpm devdeps-protobuf-c-1.4.1-100000072023102410.el8.aarch64.rpm devdeps-roaringbitmap-croaring-3.0.0-42024042816.el8.aarch64.rpm devdeps-apache-arrow-9.0.0-322024052923.el8.aarch64.rpm +devdeps-apache-orc-1.8.3-202024072510.el8.aarch64.rpm [tools] obdevtools-binutils-2.30-12022100413.el8.aarch64.rpm diff --git a/deps/init/oceanbase.el8.x86_64.deps b/deps/init/oceanbase.el8.x86_64.deps index 78c1eeb55..0373426cd 100644 --- a/deps/init/oceanbase.el8.x86_64.deps +++ b/deps/init/oceanbase.el8.x86_64.deps @@ -35,6 +35,7 @@ devdeps-s3-cpp-sdk-1.11.156-102023122011.el8.x86_64.rpm devdeps-protobuf-c-1.4.1-100000062023102016.el8.x86_64.rpm devdeps-roaringbitmap-croaring-3.0.0-42024042816.el8.x86_64.rpm devdeps-apache-arrow-9.0.0-172024052218.el8.x86_64.rpm +devdeps-apache-orc-1.8.3-202024072510.el8.x86_64.rpm [tools] obdevtools-binutils-2.30-12022100413.el8.x86_64.rpm diff --git a/deps/init/oceanbase.el9.aarch64.deps b/deps/init/oceanbase.el9.aarch64.deps index f7616be6e..dc3f98ec1 100644 --- a/deps/init/oceanbase.el9.aarch64.deps +++ b/deps/init/oceanbase.el9.aarch64.deps @@ -37,6 +37,7 @@ devdeps-s3-cpp-sdk-1.11.156-102023122011.el8.aarch64.rpm devdeps-protobuf-c-1.4.1-100000072023102410.el8.aarch64.rpm devdeps-roaringbitmap-croaring-3.0.0-42024042816.el8.aarch64.rpm devdeps-apache-arrow-9.0.0-322024052923.el8.aarch64.rpm +devdeps-apache-orc-1.8.3-202024072510.el8.aarch64.rpm [deps-el9] devdeps-apr-1.6.5-232023090616.el9.aarch64.rpm target=el9 diff --git a/deps/init/oceanbase.el9.x86_64.deps b/deps/init/oceanbase.el9.x86_64.deps index c4dcc9b53..e049cb055 100644 --- a/deps/init/oceanbase.el9.x86_64.deps +++ b/deps/init/oceanbase.el9.x86_64.deps @@ -39,6 +39,7 @@ devdeps-s3-cpp-sdk-1.11.156-102023122011.el8.x86_64.rpm devdeps-protobuf-c-1.4.1-100000062023102016.el8.x86_64.rpm devdeps-apache-arrow-9.0.0-172024052218.el8.x86_64.rpm devdeps-roaringbitmap-croaring-3.0.0-42024042816.el8.x86_64.rpm +devdeps-apache-orc-1.8.3-202024072510.el8.x86_64.rpm [deps-el9] devdeps-apr-1.6.5-232023090616.el9.x86_64.rpm target=el9 diff --git a/deps/oblib/src/CMakeLists.txt b/deps/oblib/src/CMakeLists.txt index 540df4b68..93e6171aa 100644 --- a/deps/oblib/src/CMakeLists.txt +++ b/deps/oblib/src/CMakeLists.txt @@ -21,6 +21,7 @@ target_include_directories( ${DEP_DIR}/include/apr-1/ ${DEP_DIR}/include/icu/common ${DEP_DIR}/include/apache-arrow + ${DEP_DIR}/include/apache-orc ${USSL_INCLUDE_DIRS} ) @@ -208,6 +209,12 @@ target_link_libraries(oblib_base_base_base ${DEP_DIR}/lib64/libarrow.a ${DEP_DIR}/lib64/libparquet.a ${DEP_DIR}/lib64/libarrow_bundled_dependencies.a + ${DEP_DIR}/lib64/liborc.a + ${DEP_DIR}/lib64/libsnappy.a + ${DEP_DIR}/lib64/libprotoc.a + ${DEP_DIR}/lib64/libprotobuf.a + ${DEP_DIR}/lib64/liblz4.a + ${DEP_DIR}/lib64/libzstd.a -L${DEP_DIR}/var/usr/lib64 -L${DEP_DIR}/var/usr/lib -L${DEP_3RD_DIR}/usr/lib @@ -237,6 +244,12 @@ target_link_libraries(oblib_base_base_base ${DEP_DIR}/lib64/libarrow.a ${DEP_DIR}/lib64/libparquet.a ${DEP_DIR}/lib64/libarrow_bundled_dependencies.a + ${DEP_DIR}/lib64/liborc.a + ${DEP_DIR}/lib64/libsnappy.a + ${DEP_DIR}/lib64/libprotoc.a + ${DEP_DIR}/lib64/libprotobuf.a + ${DEP_DIR}/lib64/liblz4.a + ${DEP_DIR}/lib64/libzstd.a -L${DEP_DIR}/var/usr/lib64 -L${DEP_DIR}/var/usr/lib -L${DEP_3RD_DIR}/usr/lib diff --git a/deps/oblib/unittest/lib/CMakeLists.txt b/deps/oblib/unittest/lib/CMakeLists.txt index 601238a7c..a7bb13cba 100644 --- a/deps/oblib/unittest/lib/CMakeLists.txt +++ b/deps/oblib/unittest/lib/CMakeLists.txt @@ -6,6 +6,7 @@ # oblib_addtest(time/test_ob_time_utility.cpp) # oblib_addtest(timezone/test_ob_timezone_utils.cpp) oblib_addtest(parquet/test_parquet.cpp) +oblib_addtest(orc/test_orc.cpp) oblib_addtest(alloc/test_alloc_struct.cpp) oblib_addtest(alloc/test_block_set.cpp) oblib_addtest(alloc/test_chunk_mgr.cpp) diff --git a/deps/oblib/unittest/lib/orc/test_orc.cpp b/deps/oblib/unittest/lib/orc/test_orc.cpp new file mode 100644 index 000000000..39ae5fb16 --- /dev/null +++ b/deps/oblib/unittest/lib/orc/test_orc.cpp @@ -0,0 +1,603 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#define USING_LOG_PREFIX SQL + +#include "gtest/gtest.h" +#include "lib/oblog/ob_log.h" +#include "lib/oblog/ob_log_module.h" + +#include +#include +#include +#include +#include + +#include +#include + + +#include "lib/allocator/page_arena.h" +#include "lib/file/ob_file.h" +#include "lib/file/file_directory_utils.h" +#include "lib/charset/ob_template_helper.h" +#include "lib/net/ob_net_util.h" + +#define USING_LOG_PREFIX SQL + +using namespace oceanbase::common; + +class TestOrc: public ::testing::Test +{ +public: + TestOrc() {}; + virtual ~TestOrc() {}; + virtual void SetUp(); + virtual void TearDown(); +}; + + + +void TestOrc::SetUp() +{ +} + +void TestOrc::TearDown() +{ +} + +class OrcMemoryPool : public ::orc::MemoryPool +{ + public: + virtual char* malloc(uint64_t size) override { + return (char* )alloc_.alloc(size); + } + virtual void free(char* p) override { + //do nothing + } + private: + oceanbase::common::ObArenaAllocator alloc_; +}; + + +// Result> GetOrcType(const DataType& type) { +// Type::type kind = type.id(); +// switch (kind) { +// case Type::type::BOOL: +// return liborc::createPrimitiveType(liborc::TypeKind::BOOLEAN); +// case Type::type::INT8: +// return liborc::createPrimitiveType(liborc::TypeKind::BYTE); +// case Type::type::INT16: +// return liborc::createPrimitiveType(liborc::TypeKind::SHORT); +// case Type::type::INT32: +// return liborc::createPrimitiveType(liborc::TypeKind::INT); +// case Type::type::INT64: +// return liborc::createPrimitiveType(liborc::TypeKind::LONG); +// case Type::type::FLOAT: +// return liborc::createPrimitiveType(liborc::TypeKind::FLOAT); +// case Type::type::DOUBLE: +// return liborc::createPrimitiveType(liborc::TypeKind::DOUBLE); +// // Use STRING instead of VARCHAR for now, both use UTF-8 +// case Type::type::STRING: +// case Type::type::LARGE_STRING: +// return liborc::createPrimitiveType(liborc::TypeKind::STRING); +// case Type::type::BINARY: +// case Type::type::LARGE_BINARY: +// case Type::type::FIXED_SIZE_BINARY: +// return liborc::createPrimitiveType(liborc::TypeKind::BINARY); +// case Type::type::DATE32: +// return liborc::createPrimitiveType(liborc::TypeKind::DATE); +// case Type::type::DATE64: +// return liborc::createPrimitiveType(liborc::TypeKind::TIMESTAMP); +// case Type::type::TIMESTAMP: { +// const auto& timestamp_type = checked_cast(type); +// if (!timestamp_type.timezone().empty()) { +// // The timestamp values stored in the arrow array are normalized to UTC. +// // TIMESTAMP_INSTANT type is always preferred over TIMESTAMP type. +// return liborc::createPrimitiveType(liborc::TypeKind::TIMESTAMP_INSTANT); +// } +// // The timestamp values stored in the arrow array can be in any timezone. +// return liborc::createPrimitiveType(liborc::TypeKind::TIMESTAMP); +// } +// case Type::type::DECIMAL128: { +// const uint64_t precision = +// static_cast(checked_cast(type).precision()); +// const uint64_t scale = +// static_cast(checked_cast(type).scale()); +// return liborc::createDecimalType(precision, scale); +// } +// case Type::type::LIST: +// case Type::type::FIXED_SIZE_LIST: +// case Type::type::LARGE_LIST: { +// const auto& value_field = checked_cast(type).value_field(); +// ARROW_ASSIGN_OR_RAISE(auto orc_subtype, GetOrcType(*value_field->type())); +// SetAttributes(value_field, orc_subtype.get()); +// return liborc::createListType(std::move(orc_subtype)); +// } +// case Type::type::STRUCT: { +// std::unique_ptr out_type = liborc::createStructType(); +// std::vector> arrow_fields = +// checked_cast(type).fields(); +// for (auto it = arrow_fields.begin(); it != arrow_fields.end(); ++it) { +// std::string field_name = (*it)->name(); +// ARROW_ASSIGN_OR_RAISE(auto orc_subtype, GetOrcType(*(*it)->type())); +// SetAttributes(*it, orc_subtype.get()); +// out_type->addStructField(field_name, std::move(orc_subtype)); +// } +// return out_type; +// } +// case Type::type::MAP: { +// const auto& key_field = checked_cast(type).key_field(); +// const auto& item_field = checked_cast(type).item_field(); +// ARROW_ASSIGN_OR_RAISE(auto key_orc_type, GetOrcType(*key_field->type())); +// ARROW_ASSIGN_OR_RAISE(auto item_orc_type, GetOrcType(*item_field->type())); +// SetAttributes(key_field, key_orc_type.get()); +// SetAttributes(item_field, item_orc_type.get()); +// return liborc::createMapType(std::move(key_orc_type), std::move(item_orc_type)); +// } +// case Type::type::DENSE_UNION: +// case Type::type::SPARSE_UNION: { +// std::unique_ptr out_type = liborc::createUnionType(); +// std::vector> arrow_fields = +// checked_cast(type).fields(); +// for (const auto& arrow_field : arrow_fields) { +// std::shared_ptr arrow_child_type = arrow_field->type(); +// ARROW_ASSIGN_OR_RAISE(auto orc_subtype, GetOrcType(*arrow_child_type)); +// SetAttributes(arrow_field, orc_subtype.get()); +// out_type->addUnionChild(std::move(orc_subtype)); +// } +// return out_type; +// } +// default: { +// return Status::NotImplemented("Unknown or unsupported Arrow type: ", +// type.ToString()); +// } +// } +// } + + // class ObOrcRandomAccess : public :orc::InputStream { + // public: + // ObOrcRandomAccess(oceanbase::sql::ObExternalDataAccessDriver &file_reader, const char* file_name, orc::MemoryPool *pool) + // : file_reader_(file_reader), file_name_(file_name), pool_(pool) { + + // } + + // uint64_t getLength() const override { + // return totalLength; + // } + + // uint64_t getNaturalReadSize() const override { + // return 128 * 1024; + // } + + // void read(void* buf, + // uint64_t length, + // uint64_t offset) override { + // int64_t bytesRead = 0; + // int ret = file_reader_.pread(buf, length, offset, bytesRead); + // totalLength += bytesRead; + // if (ret != OB_SUCCESS) { + // throw orc::ParseError("Bad read of " + std::string(file_name_)); + // } + // } + + // const std::string& getName() const override { + // return file_name_; + // } + // private: + // oceanbase::sql::ObExternalDataAccessDriver &file_reader_; + // const std::string file_name_; + // orc::MemoryPool *pool_; + // uint64_t totalLength; + // }; + + +void wirte_orc_file() { + std::unique_ptr outStream = orc::writeLocalFile("my-file.orc"); + //std::unique_ptr schema(orc::Type::buildTypeFromString("struct")); + std::unique_ptr schema = orc::createStructType(); + schema->addStructField("x", orc::createPrimitiveType(orc::TypeKind::INT)); + schema->addStructField("y", orc::createPrimitiveType(orc::TypeKind::BOOLEAN)); + std::unique_ptr sub_schema = orc::createStructType(); + sub_schema->addStructField("z", orc::createPrimitiveType(orc::TypeKind::FLOAT)); + sub_schema->addStructField("d", orc::createPrimitiveType(orc::TypeKind::DATE)); + schema->addStructField("S2", std::move(sub_schema)); + orc::WriterOptions options; + OrcMemoryPool pool; + options.setMemoryPool(&pool); + options.setCompression(orc::CompressionKind::CompressionKind_ZLIB); + std::unique_ptr writer = orc::createWriter(*schema, outStream.get(), options); + + uint64_t batchSize = 8, rowCount = 100; + std::unique_ptr batch = + writer->createRowBatch(batchSize); + orc::StructVectorBatch *root = + dynamic_cast(batch.get()); + orc::LongVectorBatch *x = + dynamic_cast(root->fields[0]); + orc::LongVectorBatch *y = + dynamic_cast(root->fields[1]); + orc::DoubleVectorBatch *z = + dynamic_cast(dynamic_cast(root->fields[2])->fields[0]); + + uint64_t rows = 0; + for (uint64_t i = 0; i < rowCount; ++i) { + if (i % 5 == 0) { + x->notNull[rows] = 0; + y->notNull[rows] = 0; + z->notNull[rows] = 0; + x->hasNulls = true; + y->hasNulls = true; + z->hasNulls = true; + x->data[rows] = 0; + y->data[rows] = 0; + z->data[rows] = i * 1.1 + 0.01; + rows++; + } else { + x->notNull[rows] = true; + y->notNull[rows] = true; + z->notNull[rows] = true; + x->data[rows] = i + 1; + y->data[rows] = i * 3 + 1; + z->data[rows] = i * 1.1 + 0.01; + rows++; + } + + + if (rows == batchSize) { + root->numElements = rows; + x->numElements = rows; + y->numElements = rows; + z->numElements = rows; + + writer->add(*batch); + rows = 0; + } + } + + if (rows != 0) { + root->numElements = rows; + x->numElements = rows; + y->numElements = rows; + //z->numElements = rows; + + writer->add(*batch); + rows = 0; + } + + writer->close(); +} + +void read_orc_file() { + std::unique_ptr inStream = orc::readLocalFile("my-file.orc"); + orc::ReaderOptions options; + OrcMemoryPool pool; + options.setMemoryPool(pool); + std::unique_ptr reader = orc::createReader(std::move(inStream), options); + + orc::RowReaderOptions rowReaderOptions; + std::unique_ptr rowReader = reader->createRowReader(rowReaderOptions); + std::unique_ptr batch = rowReader->createRowBatch(2); + + //std::cout <<"root field size: " << root->fields.size() << std::endl; + while (rowReader->next(*batch)) { + std::cout<<"column batch:" << batch->toString() <<"\n"; + for (uint64_t r = 0; r < batch->numElements; ++r) { + orc::StructVectorBatch *root = + dynamic_cast(batch.get()); + orc::ColumnVectorBatch *col[10] = {NULL}; + int k = 0; + std::cout<< "row:" << r ; + for (int i = 0; i < reader->getType().getSubtypeCount(); i++) { + const uint8_t* valid_bytes = NULL; + switch (reader->getType().getSubtype(i)->getKind()) { + case orc::TypeKind::BOOLEAN: + case orc::TypeKind::BYTE: + case orc::TypeKind::SHORT: + case orc::TypeKind::INT: + case orc::TypeKind::LONG: + case orc::TypeKind::DATE: + //valid_bytes = reinterpret_cast( dynamic_cast(root->fields[i])->notNull.data()) + r; + std::cout<<" col" << i <<":" << dynamic_cast(root->fields[i])->data[r] << " is not null:"<< bool(root->fields[i]->notNull[r]); + break; + case orc::TypeKind::FLOAT: + case orc::TypeKind::DOUBLE: + std::cout<< " col" << i <<":" << dynamic_cast(root->fields[i])->data[r] ; + std::cout<<" has NULL:"<<(root->fields[i])->hasNulls<<" is not null:"<< bool(root->fields[i]->notNull[r]); + break; + case orc::TypeKind::STRING: + case orc::TypeKind::VARCHAR: + case orc::TypeKind::CHAR: + case orc::TypeKind::BINARY: + std::cout<< " col" << i <<":" << dynamic_cast(root->fields[i])->data[r] <<" is not null:"<< bool(root->fields[i]->notNull[r]); + break; + case orc::TypeKind::LIST: + case orc::TypeKind::MAP: + case orc::TypeKind::UNION: + //not supported + break; + case orc::TypeKind::DECIMAL: + //std::cout<< "row:" << r << " col" << i <<":" << dynamic_cast(root->fields[i])->data[r] <<" is not null:"<< (bool)dynamic_cast(root->fields[i])->notNull[r]; + break; + case orc::TypeKind::TIMESTAMP: + case orc::TypeKind::TIMESTAMP_INSTANT: + //std::cout<< "row:" << r << " col" << i <<":" << dynamic_cast(root->fields[i])->data[r] <<" is not null:"<< (bool)dynamic_cast(root->fields[i])->notNull[r]; + break; + case orc::TypeKind::STRUCT: + std::cout<< " col" << i <<":" << dynamic_cast(dynamic_cast(root->fields[i])->fields[0])->data[r]; + break; + default: + //error + break; + } + // if (reader->getType().getColumnId() == reader->getType().getMaximumColumnId()) {//is primitive + // col[k++] = x; + // } + } + std::cout<<"\n"; + + } + } +} + +void printType(const orc::Type &type) { + std::cout << " type:" << type.toString() <<" type subTypeCount:" << type.getSubtypeCount()<< " typeKind: " << type.getKind() << " ColumnId:" << type.getColumnId() << " maxColumnId:" << type.getMaximumColumnId() << "\n"; + for (int i = 0; i < type.getSubtypeCount(); i++) { + printType(*type.getSubtype(i)); + } +} + + +bool is_primitive_Type(const orc::Type &type) { + if (type.getColumnId() == type.getMaximumColumnId()) { + return true; + } + return false; +} +void read_file_footer() { + std::unique_ptr inStream = orc::readLocalFile("my-file.orc"); + orc::ReaderOptions options; + OrcMemoryPool pool; + //options.setMemoryPool(pool); + std::unique_ptr reader = orc::createReader(std::move(inStream), options); + const orc::Type& type = reader->getType(); + printType(type); + for (int i = 0; i < reader->getType().getSubtypeCount(); i++) { + std::cout << "Subfield" << i << ": " << type.getFieldName(i); + printType(*type.getSubtype(i)); + } + for (int i = 0; i <= reader->getType().getMaximumColumnId(); i++) { + std::cout<<"column" <getColumnStatistics(i)->toString() <<"\n"; + } +} + +// Result> GetArrowSchema(const liborc::Type& type) { +// if (type.getKind() != liborc::STRUCT) { +// return Status::NotImplemented( +// "Only ORC files with a top-level struct " +// "can be handled"); +// } +// int size = static_cast(type.getSubtypeCount()); +// std::vector> fields; +// fields.reserve(size); +// for (int child = 0; child < size; ++child) { +// const std::string& name = type.getFieldName(child); +// ARROW_ASSIGN_OR_RAISE(auto elem_field, GetArrowField(name, type.getSubtype(child))); +// fields.push_back(std::move(elem_field)); +// } +// ARROW_ASSIGN_OR_RAISE(auto metadata, ReadMetadata()); +// return std::make_shared(std::move(fields), std::move(metadata)); +// } + +// Result> ReadMetadata() { +// const std::list keys = reader_->getMetadataKeys(); +// auto metadata = std::make_shared(); +// for (const auto& key : keys) { +// metadata->Append(key, reader_->getMetadataValue(key)); +// } +// return std::const_pointer_cast(metadata); +// } + +void read_column() { + std::cout<<"=================== test read column ===================\n"; + std::unique_ptr inStream = orc::readLocalFile("my-file.orc"); + orc::ReaderOptions options; + // OrcMemoryPool pool; + // options.setMemoryPool(pool); + std::list include_names_list; + include_names_list.push_front(std::string("y")); + include_names_list.push_front(std::string("S2.z")); + include_names_list.push_front(std::string("x")); + std::unique_ptr reader = orc::createReader(std::move(inStream), options); + orc::RowReaderOptions rowReaderOptions; + rowReaderOptions.include(include_names_list); + std::unique_ptr rowReader = reader->createRowReader(rowReaderOptions); + std::unique_ptr batch = rowReader->createRowBatch(1024); + std::cout<<"column batch:" << batch->toString() <<"\n"; + printType(rowReader->getSelectedType()); + const orc::Type &type = rowReader->getSelectedType(); + int size = static_cast(type.getSubtypeCount()); + for (int child = 0; child < size; ++child) { + const std::string& name = type.getFieldName(child); + std::cout<<"field "<< child << " name:" << name < keys = reader->getMetadataKeys(); + // auto metadata = std::make_shared(); + // for (const auto& key : keys) { + // metadata->Append(key, reader_->getMetadataValue(key)); + // } + // const orc::Type& type = reader->getType(); + // printType(type); + // for (int i = 0; i < reader->getType().getSubtypeCount(); i++) { + // std::cout << "Subfield" << i << ": " << type.getFieldName(i); + // printType(*type.getSubtype(i)); + // } + // for (int i = 0; i <= reader->getType().getMaximumColumnId(); i++) { + // std::cout<<"column" <getColumnStatistics(i)->toString() <<"\n"; + // } + +} + +void read_schema(std::unique_ptr &rowReader) { + const orc::Type &type = rowReader->getSelectedType(); + if (type.getKind() != orc::STRUCT) { + throw std::runtime_error("Only ORC files with a top-level struct can be handled"); + } else { + + } +} + +void read_stripe() { + // int64_t nstripes = reader_->getNumberOfStripes(); + // stripes_.resize(static_cast(nstripes)); + // std::unique_ptr stripe; + // uint64_t first_row_of_stripe = 0; + // for (int i = 0; i < nstripes; ++i) { + // stripe = reader_->getStripe(i); + // stripes_[i] = StripeInformation({static_cast(stripe->getOffset()), + // static_cast(stripe->getLength()), + // static_cast(stripe->getNumberOfRows()), + // static_cast(first_row_of_stripe)}); + // first_row_of_stripe += stripe->getNumberOfRows(); + // } +} + +void generate_orc_file(const bool output_null) { + std::unique_ptr outStream = orc::writeLocalFile("test1.orc"); + //std::unique_ptr schema(orc::Type::buildTypeFromString("struct")); + //std::unique_ptr schema = orc::createStructType(); + // schema->addStructField("age", orc::createPrimitiveType(orc::TypeKind::INT)) + // ->addStructField("is_male", orc::createPrimitiveType(orc::TypeKind::BOOLEAN)) + // ->addStructField("name", orc::createPrimitiveType(orc::TypeKind::Varchar)) + // ->addStructField("comment", orc::createPrimitiveType(orc::TypeKind::String)); + // std::unique_ptr schema(Type::buildTypeFromString(" + // struct")); + // std::unique_ptr sub_schema = orc::createStructType(); + // sub_schema->addStructField("z", orc::createPrimitiveType(orc::TypeKind::FLOAT)); + // schema->addStructField("S2", std::move(sub_schema)); + // orc::WriterOptions options; + // OrcMemoryPool pool; + // options.setCompression(orc::CompressionKind::CompressionKind_SNAPPY); + // //options.setStripeSize(1024); + // options.setCompressionBlockSize(1024); + // options.setMemoryPool(&pool); + // std::unique_ptr writer = orc::createWriter(*schema, outStream.get(), options); + + // uint64_t batchSize = 128, rowCount = 12800; + // std::unique_ptr batch = writer->createRowBatch(batchSize); + // orc::StructVectorBatch *root = dynamic_cast(batch.get()); + + // uint64_t rows = 0; + // for (uint64_t i = 0; i < rowCount; ++i) { + // if (output_null) { + // int put_null = std::rand() % 10; + // if (put_null == 1) { + + // } + // } + + // if (i % 5 == 0) { + // x->notNull[rows] = 0; + // y->notNull[rows] = 0; + // z->notNull[rows] = 0; + // x->hasNulls = true; + // y->hasNulls = true; + // z->hasNulls = true; + // x->data[rows] = 0; + // y->data[rows] = 0; + // z->data[rows] = i * 1.1 + 0.01; + // rows++; + // } else { + // x->notNull[rows] = true; + // y->notNull[rows] = true; + // z->notNull[rows] = true; + // x->data[rows] = i + 1; + // y->data[rows] = i * 3 + 1; + // z->data[rows] = i * 1.1 + 0.01; + // rows++; + // } + + + // if (rows == batchSize) { + // root->numElements = rows; + // x->numElements = rows; + // y->numElements = rows; + // z->numElements = rows; + + // writer->add(*batch); + // rows = 0; + // } + // } + + // if (rows != 0) { + // root->numElements = rows; + // x->numElements = rows; + // y->numElements = rows; + // //z->numElements = rows; + + // writer->add(*batch); + // rows = 0; + // } + + // writer->close(); +} + +void read_test_orc_file() +{ + std::cout<<"=================== test read orc file column ===================\n"; + std::unique_ptr inStream = orc::readLocalFile("/data/1/mingye.swj/work/support_master_orc/data/t.orc"); + orc::ReaderOptions options; + // OrcMemoryPool pool; + // options.setMemoryPool(pool); + // std::list include_names_list; + // include_names_list.push_front(std::string("S2.z")); + // include_names_list.push_front(std::string("x")); + std::unique_ptr reader = orc::createReader(std::move(inStream), options); + orc::RowReaderOptions rowReaderOptions; + //rowReaderOptions.include(include_names_list); + std::unique_ptr rowReader = reader->createRowReader(rowReaderOptions); + std::unique_ptr batch = rowReader->createRowBatch(256); + while (rowReader->next(*batch)) { + std::cout<<"column batch:" << batch->toString() <<"\n"; + for (uint64_t r = 0; r < batch->numElements; ++r) { + orc::StructVectorBatch *root = + dynamic_cast(batch.get()); + std::cout<<" row" << r <<":" << dynamic_cast(root->fields[0])->data[r] << " is not null:"<< bool(root->fields[0]->notNull[r]); + } + } + printType(rowReader->getSelectedType()); + const orc::Type &type = rowReader->getSelectedType(); + int size = static_cast(type.getSubtypeCount()); + for (int child = 0; child < size; ++child) { + const std::string& name = type.getFieldName(child); + printType(*type.getSubtype(child)); + } +} + +TEST_F(TestOrc, read_write_orc_file_test) +{ + wirte_orc_file(); + read_orc_file(); + read_file_footer(); + read_column(); + //read_test_orc_file(); +} + +int main(int argc, char **argv) +{ + OB_LOGGER.set_log_level("INFO"); + testing::InitGoogleTest(&argc,argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/src/sql/CMakeLists.txt b/src/sql/CMakeLists.txt index dcccf6446..f5fee5e8d 100644 --- a/src/sql/CMakeLists.txt +++ b/src/sql/CMakeLists.txt @@ -889,6 +889,7 @@ ob_set_subtarget(ob_sql engine_table engine/table/ob_index_lookup_op_impl.cpp engine/table/ob_table_scan_with_index_back_op.cpp engine/table/ob_external_table_access_service.cpp + engine/table/ob_orc_table_row_iter.cpp engine/table/ob_parquet_table_row_iter.cpp ) diff --git a/src/sql/engine/cmd/ob_load_data_parser.cpp b/src/sql/engine/cmd/ob_load_data_parser.cpp index 7a2f6b54c..32038712a 100644 --- a/src/sql/engine/cmd/ob_load_data_parser.cpp +++ b/src/sql/engine/cmd/ob_load_data_parser.cpp @@ -32,10 +32,9 @@ const char INVALID_TERM_CHAR = '\xff'; const char * ObExternalFileFormat::FORMAT_TYPE_STR[] = { "CSV", "PARQUET", + "ORC", }; - -static_assert(array_elements(ObExternalFileFormat::FORMAT_TYPE_STR) == ObExternalFileFormat::MAX_FORMAT, - "Not enough initializer for ObExternalFileFormat"); +static_assert(array_elements(ObExternalFileFormat::FORMAT_TYPE_STR) == ObExternalFileFormat::MAX_FORMAT, "Not enough initializer for ObExternalFileFormat"); int ObCSVGeneralFormat::init_format(const ObDataInFileStruct &format, int64_t file_column_nums, @@ -391,7 +390,7 @@ int64_t ObExternalFileFormat::to_string(char *buf, const int64_t buf_len) const J_OBJ_START(); - databuff_print_kv(buf, buf_len, pos, "\"TYPE\"", is_valid_format ? FORMAT_TYPE_STR[format_type_] : "INVALID"); + databuff_print_kv(buf, buf_len, pos, "\"TYPE\"", is_valid_format ? ObExternalFileFormat::FORMAT_TYPE_STR[format_type_] : "INVALID"); switch (format_type_) { case CSV_FORMAT: @@ -434,8 +433,8 @@ int ObExternalFileFormat::load_from_string(const ObString &str, ObIAllocator &al LOG_WARN("unexpected json format", K(ret), K(str)); } else { ObString format_type_str = format_type_node->value_->get_string(); - for (int i = 0; i < array_elements(FORMAT_TYPE_STR); ++i) { - if (format_type_str.case_compare(FORMAT_TYPE_STR[i]) == 0) { + for (int i = 0; i < array_elements(ObExternalFileFormat::FORMAT_TYPE_STR); ++i) { + if (format_type_str.case_compare(ObExternalFileFormat::FORMAT_TYPE_STR[i]) == 0) { format_type_ = static_cast(i); break; } @@ -447,6 +446,7 @@ int ObExternalFileFormat::load_from_string(const ObString &str, ObIAllocator &al OZ (origin_file_format_str_.load_from_json_data(format_type_node, allocator)); break; case PARQUET_FORMAT: + case ORC_FORMAT: break; default: ret = OB_ERR_UNEXPECTED; diff --git a/src/sql/engine/cmd/ob_load_data_parser.h b/src/sql/engine/cmd/ob_load_data_parser.h index bf422b472..d8ef54a21 100644 --- a/src/sql/engine/cmd/ob_load_data_parser.h +++ b/src/sql/engine/cmd/ob_load_data_parser.h @@ -524,6 +524,7 @@ struct ObExternalFileFormat INVALID_FORMAT = -1, CSV_FORMAT, PARQUET_FORMAT, + ORC_FORMAT, MAX_FORMAT }; @@ -543,7 +544,6 @@ struct ObExternalFileFormat sql::ObCSVGeneralFormat csv_format_; ObLoadCompressionFormat compression_format_; uint64_t options_; - static const char *FORMAT_TYPE_STR[]; }; diff --git a/src/sql/engine/table/ob_external_table_access_service.cpp b/src/sql/engine/table/ob_external_table_access_service.cpp index 673b3b707..937c24eeb 100644 --- a/src/sql/engine/table/ob_external_table_access_service.cpp +++ b/src/sql/engine/table/ob_external_table_access_service.cpp @@ -23,6 +23,7 @@ #include "lib/utility/ob_macro_utils.h" #include "sql/engine/table/ob_parquet_table_row_iter.h" #include "sql/engine/cmd/ob_load_data_file_reader.h" +#include "sql/engine/table/ob_orc_table_row_iter.h" namespace oceanbase { @@ -558,13 +559,18 @@ int ObExternalTableAccessService::table_scan( LOG_WARN("alloc memory failed", K(ret)); } break; - case ObExternalFileFormat::PARQUET_FORMAT: if (OB_ISNULL(row_iter = OB_NEWx(ObParquetTableRowIterator, (scan_param.allocator_)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("alloc memory failed", K(ret)); } break; + case ObExternalFileFormat::ORC_FORMAT: + if (OB_ISNULL(row_iter = OB_NEWx(ObOrcTableRowIterator, (scan_param.allocator_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc memory failed", K(ret)); + } + break; default: ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected format", K(ret), "format", param.external_file_format_.format_type_); @@ -594,6 +600,7 @@ int ObExternalTableAccessService::table_rescan(ObVTableScanParam ¶m, ObNewRo switch (param.external_file_format_.format_type_) { case ObExternalFileFormat::CSV_FORMAT: case ObExternalFileFormat::PARQUET_FORMAT: + case ObExternalFileFormat::ORC_FORMAT: result->reset(); break; default: diff --git a/src/sql/engine/table/ob_orc_table_row_iter.cpp b/src/sql/engine/table/ob_orc_table_row_iter.cpp new file mode 100644 index 000000000..c1b4996a6 --- /dev/null +++ b/src/sql/engine/table/ob_orc_table_row_iter.cpp @@ -0,0 +1,1382 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + #define USING_LOG_PREFIX SQL_ENG +#include "ob_orc_table_row_iter.h" +#include "sql/engine/expr/ob_expr_get_path.h" +#include "share/external_table/ob_external_table_utils.h" +#include "sql/engine/expr/ob_datum_cast.h" +#include "sql/engine/ob_exec_context.h" + + + +namespace oceanbase +{ +using namespace share::schema; +using namespace common; +using namespace share; +namespace sql { + +int ObOrcTableRowIterator::to_dot_column_path(ObIArray &col_names, ObString &path) +{ + int ret = OB_SUCCESS; + ObSqlString tmp_string; + for (int64_t i = 0; OB_SUCC(ret) && i < col_names.count(); i++) { + if (i > 0) { + OZ (tmp_string.append(".")); + } + OZ (tmp_string.append(col_names.at(i))); + } + OZ (ob_write_string(allocator_, tmp_string.string(), path)); + return ret; +} + + +/** + * Recurses over a type tree and build two maps + * map, map + */ +int ObOrcTableRowIterator::build_type_name_id_map(const orc::Type* type, ObIArray &col_names) +{ + int ret = OB_SUCCESS; + CK (type != nullptr); + OZ (id_to_type_.set_refactored(type->getColumnId(), type, 0)); + if (OB_FAIL(ret)) { + } else if (orc::TypeKind::STRUCT == type->getKind()) { + for (size_t i = 0; OB_SUCC(ret) && i < type->getSubtypeCount(); ++i) { + const std::string& cpp_field_name = type->getFieldName(i); + ObString field_name; + OZ (ob_write_string(allocator_, ObString(cpp_field_name.c_str()), field_name)); + OZ (col_names.push_back(field_name)); + ObString path; + OZ (to_dot_column_path(col_names, path)); + OZ (name_to_id_.set_refactored(path, type->getSubtype(i)->getColumnId(), 0)); + OZ (build_type_name_id_map(type->getSubtype(i), col_names)); + if (OB_FAIL(ret)) { + } else if (col_names.count() > 0) { + col_names.pop_back(); + } + } + } else { + // other non-primitive type + for (size_t j = 0; OB_SUCC(ret) && j < type->getSubtypeCount(); ++j) { + OZ (build_type_name_id_map(type->getSubtype(j), col_names)); + } + } + return ret; +} + +int ObOrcTableRowIterator::init(const storage::ObTableScanParam *scan_param) +{ + int ret = OB_SUCCESS; + + CK (scan_param != nullptr); + CK (scan_param->op_ != nullptr); + CK (scan_param->ext_column_convert_exprs_ != nullptr); + if (OB_SUCC(ret)) { + ObEvalCtx &eval_ctx = scan_param->op_->get_eval_ctx(); + int64_t column_cnt = scan_param->ext_column_convert_exprs_->count(); + mem_attr_ = ObMemAttr(MTL_ID(), "OrcRowIter"); + allocator_.set_attr(mem_attr_); + orc_alloc_.init(MTL_ID()); + OZ (id_to_type_.create(512, mem_attr_)); + OZ (name_to_id_.create(512, mem_attr_)); + OZ (ObExternalTableRowIterator::init(scan_param)); + OZ (data_access_driver_.init(scan_param->external_file_location_, + scan_param->external_file_access_info_)); + + if (OB_SUCC(ret) && OB_ISNULL(bit_vector_cache_)) { + void *mem = nullptr; + if (OB_ISNULL(mem = allocator_.alloc(ObBitVector::memory_size(eval_ctx.max_batch_size_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory for skip", K(ret), K(eval_ctx.max_batch_size_)); + } else { + bit_vector_cache_ = to_bit_vector(mem); + bit_vector_cache_->reset(eval_ctx.max_batch_size_); + } + } + + ObArray file_column_exprs; + ObArray file_meta_column_exprs; + for (int i = 0; OB_SUCC(ret) && i < scan_param->ext_file_column_exprs_->count(); i++) { + ObExpr* ext_file_column_expr = scan_param->ext_file_column_exprs_->at(i); + if (OB_ISNULL(ext_file_column_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected ptr", K(ret)); + } else if (ext_file_column_expr->type_ == T_PSEUDO_EXTERNAL_FILE_URL + || ext_file_column_expr->type_ == T_PSEUDO_PARTITION_LIST_COL) { + OZ (file_meta_column_exprs.push_back(ext_file_column_expr)); + } else if (ext_file_column_expr->type_ == T_PSEUDO_EXTERNAL_FILE_COL) { + OZ (file_column_exprs.push_back(ext_file_column_expr)); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr", KPC(ext_file_column_expr)); + } + } + OZ (file_column_exprs_.assign(file_column_exprs)); + OZ (file_meta_column_exprs_.assign(file_meta_column_exprs)); + + if (OB_SUCC(ret) && file_column_exprs_.count() > 0) { + OZ (column_indexs_.allocate_array(allocator_, file_column_exprs_.count())); + //OZ (column_readers_.allocate_array(allocator_, file_column_exprs_.count())); + OZ (load_funcs_.allocate_array(allocator_, file_column_exprs_.count())); + } + + if (OB_SUCC(ret)) { + OZ (file_url_ptrs_.allocate_array(allocator_, eval_ctx.max_batch_size_)); + OZ (file_url_lens_.allocate_array(allocator_, eval_ctx.max_batch_size_)); + } + } + return ret; +} + + +int ObOrcTableRowIterator::next_stripe() +{ + int ret = OB_SUCCESS; + //init all meta + if (state_.cur_stripe_idx_ > state_.end_stripe_idx_) { + if (OB_FAIL(next_file())) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to get next srtipe", K(ret)); + } + } + } + if (OB_SUCC(ret)) { + int64_t cur_stripe = (state_.cur_stripe_idx_++) - 1; + CK (cur_stripe < stripes_.count()); + if (OB_SUCC(ret)) { + LOG_TRACE("show current stripe info", K(stripes_.at(cur_stripe))); + try { + // for (int i = 0; OB_SUCC(ret) && i < column_readers_.count(); i++) { + // if (column_readers_.at(i)) { + // column_readers_.at(i)->seekToRow(stripes_.at(cur_stripe).first_row_id); + // } else { + // ret = OB_ERR_UNEXPECTED; + // LOG_WARN("column reader is null", K(ret)); + // } + // } + if (row_reader_) { + row_reader_->seekToRow(stripes_.at(cur_stripe).first_row_id); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column reader is null", K(ret)); + } + state_.cur_stripe_read_row_count_ = 0; + state_.cur_stripe_row_count_ = stripes_.at(cur_stripe).num_rows; + } catch(const std::exception& e) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected index", K(ret), "Info", e.what(), K(cur_stripe), K(column_indexs_)); + } catch(...) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected index", K(ret), K(cur_stripe), K(column_indexs_)); + } + } + } + return ret; +} + +int ObOrcTableRowIterator::next_file() +{ + int ret = OB_SUCCESS; + CK (scan_param_ != nullptr); + CK (scan_param_->op_ != nullptr); + if (OB_SUCC(ret)) { + ObEvalCtx &eval_ctx = scan_param_->op_->get_eval_ctx(); + ObString location = scan_param_->external_file_location_; + int64_t task_idx = 0; + int64_t file_size = 0; + if (data_access_driver_.is_opened()) { + data_access_driver_.close(); + } + + do { + if ((task_idx = state_.file_idx_++) >= scan_param_->key_ranges_.count()) { + ret = OB_ITER_END; + } else { + state_.cur_file_url_ = scan_param_->key_ranges_.at(task_idx).get_start_key().get_obj_ptr()[ObExternalTableUtils::FILE_URL].get_string(); + url_.reuse(); + const char *split_char = "/"; + OZ (url_.append_fmt("%.*s%s%.*s", location.length(), location.ptr(), + (location.empty() || location[location.length() - 1] == '/') ? "" : split_char, + state_.cur_file_url_.length(), state_.cur_file_url_.ptr())); + OZ (data_access_driver_.get_file_size(url_.string(), file_size)); + + if (OB_SUCC(ret)) { + ObString expr_file_url; + if (data_access_driver_.get_storage_type() == OB_STORAGE_FILE) { + ObSqlString full_name; + if (ip_port_.empty()) { + OZ(gen_ip_port(allocator_)); + } + OZ (full_name.append_fmt("%.*s%%%.*s", ip_port_.length(), ip_port_.ptr(), + state_.cur_file_url_.length(), state_.cur_file_url_.ptr())); + OZ (ob_write_string(allocator_, full_name.string(), expr_file_url)); + } else { + expr_file_url = state_.cur_file_url_; + } + for (int i = 0; OB_SUCC(ret) && i < eval_ctx.max_batch_size_; i++) { + file_url_ptrs_.at(i) = expr_file_url.ptr(); + file_url_lens_.at(i) = expr_file_url.length(); + } + } + LOG_DEBUG("current external file", K(url_), K(file_size)); + } + } while (OB_SUCC(ret) && OB_UNLIKELY(0 >= file_size)); //skip not exist or empty file + + if (OB_SUCC(ret)) { + int64_t part_id = scan_param_->key_ranges_.at(task_idx).get_start_key().get_obj_ptr()[ObExternalTableUtils::PARTITION_ID].get_int(); + if (part_id != 0 && state_.part_id_ != part_id) { + state_.part_id_ = part_id; + OZ (calc_file_partition_list_value(part_id, allocator_, state_.part_list_val_)); + } + + state_.cur_file_id_ = scan_param_->key_ranges_.at(task_idx).get_start_key().get_obj_ptr()[ObExternalTableUtils::FILE_ID].get_int(); + OZ (ObExternalTableUtils::resolve_line_number_range(scan_param_->key_ranges_.at(task_idx), + ObExternalTableUtils::ROW_GROUP_NUMBER, + state_.cur_stripe_idx_, + state_.end_stripe_idx_)); + + try { + OZ (data_access_driver_.open(url_.ptr()), url_); + std::unique_ptr inStream(new ObOrcFileAccess(data_access_driver_, + url_.ptr(), file_size)); + orc::ReaderOptions options; + options.setMemoryPool(orc_alloc_); + std::unique_ptr reader = orc::createReader(std::move(inStream), options); + if (!reader) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("orc create reader failed", K(ret)); + throw std::bad_exception(); + } + int64_t nstripes = reader->getNumberOfStripes(); + LOG_TRACE("read file access: number of stipes", K(nstripes), K(url_)); + OZ (stripes_.allocate_array(allocator_, static_cast(nstripes))); + state_.end_stripe_idx_ = std::min(nstripes, state_.end_stripe_idx_); + std::unique_ptr stripe; + uint64_t first_row_of_stripe = 0; + for (int i = 0; OB_SUCC(ret) && i < nstripes; i++) { + stripe = reader->getStripe(i); + if (!stripe) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get orc file stripe failed", K(ret)); + throw std::bad_exception(); + } + stripes_.at(i) = StripeInformation({static_cast(stripe->getOffset()), + static_cast(stripe->getLength()), + static_cast(stripe->getNumberOfRows()), + static_cast(first_row_of_stripe)}); + first_row_of_stripe += stripe->getNumberOfRows(); + } + + std::list include_names_list; + for (int i = 0; OB_SUCC(ret) && i < file_column_exprs_.count(); i++) { + ObDataAccessPathExtraInfo *data_access_info = + static_cast(file_column_exprs_.at(i)->extra_info_); + CK (data_access_info != nullptr); + CK (data_access_info->data_access_path_.ptr() != nullptr); + CK (data_access_info->data_access_path_.length() != 0); + if (OB_SUCC(ret)) { + include_names_list.push_front(std::string(data_access_info->data_access_path_.ptr(), + data_access_info->data_access_path_.length())); //x.y.z -> column_id + } + } + orc::RowReaderOptions rowReaderOptions; + rowReaderOptions.include(include_names_list); + row_reader_ = reader->createRowReader(rowReaderOptions); + if (OB_FAIL(ret)) { + } else if (!row_reader_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("create row reader failed", K(ret)); + } else { + //column_indexs_.at(i) = rowReader->getSelectedType().getColumnId(); + //column_readers_.at(i) = std::move(rowReader); + ObArray col_names; + id_to_type_.reuse(); + name_to_id_.reuse(); + OZ (SMART_CALL(build_type_name_id_map(&row_reader_->getSelectedType(), col_names))); + for (int i = 0; OB_SUCC(ret) && i < file_column_exprs_.count(); i++) { + ObDataAccessPathExtraInfo *data_access_info = + static_cast(file_column_exprs_.at(i)->extra_info_); + CK (data_access_info != nullptr); + CK (data_access_info->data_access_path_.ptr() != nullptr); + CK (data_access_info->data_access_path_.length() != 0); + int64_t col_id = -1; + OZ (name_to_id_.get_refactored(ObString(data_access_info->data_access_path_.length(), data_access_info->data_access_path_.ptr()), col_id)); + CK (col_id != -1); + const orc::Type *type = nullptr; + OZ (id_to_type_.get_refactored(col_id, type)); + CK (type != nullptr); + if (OB_SUCC(ret)) { + load_funcs_.at(i) = DataLoader::select_load_function(file_column_exprs_.at(i)->datum_meta_, + *type); + } + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(load_funcs_.at(i))) { + ret = OB_ERR_INVALID_TYPE_FOR_OP; + if (i >= row_reader_->getSelectedType().getSubtypeCount()) { + //error for report + LOG_WARN("not supported type", K(ret), K(file_column_exprs_.at(i)->datum_meta_)); + const char *ob_type = ob_obj_type_str(file_column_exprs_.at(i)->datum_meta_.type_); + LOG_USER_ERROR(OB_EXTERNAL_FILE_COLUMN_TYPE_MISMATCH, "", ob_type); + } else { + std::string p_type = row_reader_->getSelectedType().getSubtype(i) == nullptr ? + "INVALID ORC TYPE" : row_reader_->getSelectedType().getSubtype(i)->toString(); + int64_t pos = 0; + ObArrayWrap buf; + ObDatumMeta &meta = file_column_exprs_.at(i)->datum_meta_; + const char *ob_type = ob_obj_type_str(file_column_exprs_.at(i)->datum_meta_.type_); + if (OB_SUCCESS == buf.allocate_array(allocator_, 100)) { + ob_sql_type_str(buf.get_data(), buf.count(), pos, meta.type_, + OB_MAX_VARCHAR_LENGTH, meta.precision_, meta.scale_, meta.cs_type_); + if (pos < buf.count()) { + buf.at(pos++) = '\0'; + ob_type = buf.get_data(); + } + } + LOG_WARN("not supported type", K(ret), K(file_column_exprs_.at(i)->datum_meta_), + K(ObString(p_type.length(), p_type.data()))); + LOG_USER_ERROR(OB_EXTERNAL_FILE_COLUMN_TYPE_MISMATCH, p_type.c_str(), ob_type); + } + } + } + } + } catch(const std::exception& e) { + if (OB_SUCC(ret)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error", K(ret), "Info", e.what()); + } + } catch(...) { + if (OB_SUCC(ret)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error", K(ret)); + } + } + } + } + return ret; +} + +bool ObOrcTableRowIterator::DataLoader::is_orc_read_utc() +{ + return true; +} + +bool ObOrcTableRowIterator::DataLoader::is_ob_type_store_utc(const ObDatumMeta &meta) +{ + return ObTimestampType == meta.type_ || ObDateType == meta.type_ + || (lib::is_mysql_mode() && ObDateTimeType == meta.type_) + || (lib::is_mysql_mode() && ObTimeType == meta.type_); +} + +int64_t ObOrcTableRowIterator::DataLoader::calc_tz_adjust_us() +{ + int64_t res = 0; + int ret = OB_SUCCESS; + bool is_utc_src = is_orc_read_utc(); + bool is_utc_dst = is_ob_type_store_utc(file_col_expr_->datum_meta_); + if (is_utc_src != is_utc_dst) { + int32_t tmp_offset = 0; + if (OB_NOT_NULL(eval_ctx_.exec_ctx_.get_my_session()) + && OB_NOT_NULL(eval_ctx_.exec_ctx_.get_my_session()->get_timezone_info()) + && OB_SUCCESS == eval_ctx_.exec_ctx_.get_my_session()->get_timezone_info()->get_timezone_offset(0, tmp_offset)) { + res = SEC_TO_USEC(tmp_offset) * (is_utc_dst ? 1 : -1); + } + } + LOG_DEBUG("tz adjust", K(is_utc_src), K(is_utc_dst), K(res), K(file_col_expr_->datum_meta_)); + return res; +} + +int ObOrcTableRowIterator::DataLoader::load_data_for_col(LOAD_FUNC &func) +{ + return (this->*func)(); +} + +ObOrcTableRowIterator::DataLoader::LOAD_FUNC ObOrcTableRowIterator::DataLoader::select_load_function( + const ObDatumMeta &datum_type, const orc::Type &type) +{ + LOAD_FUNC func = NULL; + + // int size = static_cast(type.getSubtypeCount()); + // if (size > 1) { + // LOG_USER_ERROR(OB_NOT_SUPPORTED, "Non-primitive type now are"); + // throw std::invalid_argument(type.toString()); + // } + const orc::Type* col_desc = &type; + orc::TypeKind type_kind = col_desc->getKind(); + if (ob_is_integer_type(datum_type.type_)) { + switch (type_kind) { + case orc::TypeKind::BOOLEAN: + case orc::TypeKind::BYTE: + case orc::TypeKind::SHORT: + case orc::TypeKind::INT: + case orc::TypeKind::LONG: + func = &DataLoader::load_int64_vec; + break; + case orc::TypeKind::DATE: + func = &DataLoader::load_int32_vec; + break; + default: + func = NULL; + } + } else if (ob_is_string_type(datum_type.type_) || ob_is_enum_or_set_type(datum_type.type_)) { + //convert parquet enum/string to enum/string vector + switch (type_kind) { + case orc::TypeKind::STRING: + case orc::TypeKind::VARCHAR: + case orc::TypeKind::BINARY: + case orc::TypeKind::CHAR: + func = &DataLoader::load_string_col; + break; + default: + func = NULL; + } + } else if (ob_is_number_or_decimal_int_tc(datum_type.type_)) { + //convert parquet int storing as int32/int64 to number/decimal vector + if (type_kind == orc::TypeKind::DECIMAL) { + if (col_desc->getPrecision() != datum_type.precision_ || col_desc->getScale() != datum_type.scale_) { + func = NULL; + } else if (col_desc->getPrecision() == 0 || col_desc->getPrecision() > 18) { + func = &DataLoader::load_dec128_vec; + } else { + func = &DataLoader::load_dec64_vec; + } + } else { + func = NULL; + } + } else if (ob_is_date_tc(datum_type.type_) || + ob_is_datetime(datum_type.type_) || + ob_is_time_tc(datum_type.type_) || + ob_is_otimestamp_type(datum_type.type_) || + ObTimestampType == datum_type.type_) { + switch (type_kind) { + // Values of TIMESTAMP type are stored in the writer timezone in the Orc file. + // Values are read back in the reader timezone. However, the writer timezone + // information in the Orc stripe footer is optional and may be missing. What is + // more, stripes in the same Orc file may have different writer timezones (though + // unlikely). So we cannot tell the exact timezone of values read back. In the adapter + // implementations, we set both writer and + // reader timezone to UTC to avoid any conversion so users can get the same values + // as written. To get rid of this burden, TIMESTAMP_INSTANT type is always preferred + // over TIMESTAMP type. + case orc::TypeKind::TIMESTAMP: + case orc::TypeKind::TIMESTAMP_INSTANT: + LOG_DEBUG("show type kind", K(type_kind), K(orc::TypeKind::TIMESTAMP_INSTANT)); + if (ob_is_date_tc(datum_type.type_) || + ob_is_datetime(datum_type.type_) || + ob_is_time_tc(datum_type.type_) || + ObTimestampType == datum_type.type_ || + ObTimestampLTZType == datum_type.type_) { + func = &DataLoader::load_timestamp_vec; + } + break; + case orc::TypeKind::DATE: + if (ob_is_date_tc(datum_type.type_)) { + func = &DataLoader::load_int32_vec; + } else if (ob_is_datetime(datum_type.type_) || + ob_is_time_tc(datum_type.type_) || + ObTimestampType == datum_type.type_ || + ObTimestampLTZType == datum_type.type_) { + func = &DataLoader::load_date_to_time_or_stamp; + } + break; + default: + func = NULL; + } + } else if (orc::TypeKind::FLOAT == type_kind && ObFloatType == datum_type.type_) { + func = &DataLoader::load_float; + } else if (orc::TypeKind::DOUBLE == type_kind && ObDoubleType == datum_type.type_) { + func = &DataLoader::load_double; + } + return func; +} + +int ObOrcTableRowIterator::get_data_column_batch_idxs(const orc::Type *type, const int col_id, ObIArray &idxs) +{ + int ret = OB_SUCCESS; + CK (type != NULL); + bool found = false; + while (OB_SUCC(ret) && !found) { + const orc::Type *cur = type; + for (int i = 0; OB_SUCC(ret) && !found && i < type->getSubtypeCount(); i++) { + if (type->getSubtype(i)->getColumnId() == col_id) { + OZ (idxs.push_back(i)); + found = true; + } else if (type->getSubtype(i)->getColumnId() < col_id && col_id < type->getSubtype(i)->getMaximumColumnId()) { + OZ (idxs.push_back(i)); + type = type->getSubtype(i); + } else { + //do nothing + } + } + if (OB_FAIL(ret)) { + } else if (cur == type && !found) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get data colum batch failed", K(ret)); + } + } + return ret; +} + +int ObOrcTableRowIterator::get_next_rows(int64_t &count, int64_t capacity) +{ + int ret = OB_SUCCESS; + ObEvalCtx &eval_ctx = scan_param_->op_->get_eval_ctx(); + const ExprFixedArray &column_conv_exprs = *(scan_param_->ext_column_convert_exprs_); + int64_t read_count = 0; + ObMallocHookAttrGuard guard(mem_attr_); + + if (OB_SUCC(ret) && state_.cur_stripe_read_row_count_ >= state_.cur_stripe_row_count_) { + if (OB_FAIL(next_stripe())) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to next row group", K(ret)); + } + } + } + if (OB_FAIL(ret)) { + } else if (!file_column_exprs_.count()) { + read_count = std::min(capacity, state_.cur_stripe_row_count_ - state_.cur_stripe_read_row_count_); + } else if (!row_reader_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("row reader is null", K(ret)); + } else { + std::unique_ptr batch = row_reader_->createRowBatch(capacity); + if (!batch) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("create orc row batch failed", K(ret)); + } else if (row_reader_->next(*batch)) { + //ok + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("read next batch failed", K(ret), K(state_.cur_stripe_read_row_count_), K(state_.cur_stripe_row_count_)); + } + //load vec data from parquet file to file column expr + for (int i = 0; OB_SUCC(ret) && i < file_column_exprs_.count(); ++i) { + if (OB_ISNULL(file_column_exprs_.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("file column expr is null", K(ret)); + } else { + int idx = -1; + int64_t col_id = -1; + ObDataAccessPathExtraInfo *data_access_info = + static_cast(file_column_exprs_.at(i)->extra_info_); + CK (data_access_info != nullptr); + CK (data_access_info->data_access_path_.ptr() != nullptr); + CK (data_access_info->data_access_path_.length() != 0); + OZ (name_to_id_.get_refactored(ObString(data_access_info->data_access_path_.length(), data_access_info->data_access_path_.ptr()), col_id)); + ObArray idxs; + OZ (get_data_column_batch_idxs(&row_reader_->getSelectedType(), col_id, idxs)); + DataLoader loader(eval_ctx, file_column_exprs_.at(i), batch, capacity, idxs, read_count); + OZ (file_column_exprs_.at(i)->init_vector_for_write( + eval_ctx, file_column_exprs_.at(i)->get_default_res_format(), eval_ctx.max_batch_size_)); + OZ (loader.load_data_for_col(load_funcs_.at(i))); + if (OB_SUCC(ret)) { + file_column_exprs_.at(i)->set_evaluated_projected(eval_ctx); + } + if (OB_SUCC(ret) && read_count == 0) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("read result count is zero", K(ret)); + } + } + } + } + if (OB_SUCC(ret) && read_count > 0) { + //fill expr results from metadata + for (int i = 0; OB_SUCC(ret) && i < file_meta_column_exprs_.count(); i++) { + ObExpr *meta_expr = file_meta_column_exprs_.at(i); + CK (OB_NOT_NULL(meta_expr)); + if (OB_FAIL(ret)) { + } else if (meta_expr->type_ == T_PSEUDO_EXTERNAL_FILE_URL) { + StrDiscVec *text_vec = static_cast(meta_expr->get_vector(eval_ctx)); + CK (OB_NOT_NULL(text_vec)); + OZ (meta_expr->init_vector_for_write(eval_ctx, VEC_DISCRETE, read_count)); + if (OB_SUCC(ret)) { + text_vec->set_ptrs(file_url_ptrs_.get_data()); + text_vec->set_lens(file_url_lens_.get_data()); + } + } else if (meta_expr->type_ == T_PSEUDO_PARTITION_LIST_COL) { + OZ (meta_expr->init_vector_for_write(eval_ctx, VEC_UNIFORM, read_count)); + OZ (fill_file_partition_expr(meta_expr, state_.part_list_val_, read_count)); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr", KPC(meta_expr)); + } + meta_expr->set_evaluated_projected(eval_ctx); + } + + for (int i = 0; OB_SUCC(ret) && i < column_exprs_.count(); i++) { + //column_conv_exprs is 1-1 mapped to column_exprs + //calc gen column exprs + CK (OB_NOT_NULL(column_conv_exprs.at(i))); + if (OB_FAIL(ret)) { + } else if (!column_conv_exprs.at(i)->get_eval_info(eval_ctx).evaluated_) { + OZ (column_conv_exprs.at(i)->init_vector_default(eval_ctx, read_count)); + CK (OB_NOT_NULL(bit_vector_cache_)); + OZ (column_conv_exprs.at(i)->eval_vector(eval_ctx, *bit_vector_cache_, read_count, true)); + OX (column_conv_exprs.at(i)->set_evaluated_projected(eval_ctx)); + } + //assign gen column exprs value to column exprs(output exprs) + if (OB_SUCC(ret)) { + ObExpr *to = column_exprs_.at(i); + ObExpr *from = column_conv_exprs.at(i); + CK (OB_NOT_NULL(to)); + CK (OB_NOT_NULL(from)); + VectorHeader &to_vec_header = to->get_vector_header(eval_ctx); + VectorHeader &from_vec_header = from->get_vector_header(eval_ctx); + if (OB_FAIL(ret)) { + } else if (from_vec_header.format_ == VEC_UNIFORM_CONST) { + ObDatum *from_datum = + static_cast(from->get_vector(eval_ctx))->get_datums(); + CK (OB_NOT_NULL(from_datum)); + OZ(to->init_vector(eval_ctx, VEC_UNIFORM, read_count)); + ObUniformBase *to_vec = static_cast(to->get_vector(eval_ctx)); + CK (OB_NOT_NULL(to_vec)); + if (OB_SUCC(ret)) { + ObDatum *to_datums = to_vec->get_datums(); + CK (OB_NOT_NULL(to_datums)); + for (int64_t j = 0; j < read_count && OB_SUCC(ret); j++) { + to_datums[j] = *from_datum; + } + } + } else if (from_vec_header.format_ == VEC_UNIFORM) { + ObUniformBase *uni_vec = static_cast(from->get_vector(eval_ctx)); + CK (OB_NOT_NULL(uni_vec)); + if (OB_SUCC(ret)) { + ObDatum *src = uni_vec->get_datums(); + ObDatum *dst = to->locate_batch_datums(eval_ctx); + CK (OB_NOT_NULL(src)); + CK (OB_NOT_NULL(dst)); + if (OB_SUCC(ret) && src != dst) { + MEMCPY(dst, src, read_count * sizeof(ObDatum)); + } + OZ(to->init_vector(eval_ctx, VEC_UNIFORM, read_count)); + } + } else { + to_vec_header = from_vec_header; + } + column_exprs_.at(i)->set_evaluated_projected(eval_ctx); + } + } + } + if (OB_SUCC(ret)) { + state_.cur_stripe_read_row_count_ += read_count; + count = read_count; + } + return ret; +} + +int ObOrcTableRowIterator::DataLoader::load_int64_vec() +{ + int ret = OB_SUCCESS; + int64_t values_cnt = 0; + row_count_ = 0; + ObEvalCtx::TempAllocGuard tmp_alloc_g(eval_ctx_); + CK (OB_NOT_NULL(file_col_expr_)); + if (OB_SUCC(ret)) { + ObFixedLengthBase *int64_vec = static_cast(file_col_expr_->get_vector(eval_ctx_)); + CK (OB_NOT_NULL(int64_vec)); + CK (VEC_FIXED == int64_vec->get_format()); + if (OB_SUCC(ret)) { + if (batch_) { + row_count_ = batch_->numElements; + orc::StructVectorBatch *root = dynamic_cast(batch_.get()); + if (OB_ISNULL(root)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic cast orc column vector batch failed", K(ret)); + } + CK (root->fields.size() > 0); + CK (idxs_.count() > 0); + if (OB_SUCC(ret)) { + orc::StructVectorBatch *cb = root; + for (int64_t i = 0; OB_SUCC(ret) && i < idxs_.count() - 1; i++) { + CK (root->fields.size() > idxs_.at(i)); + if (OB_SUCC(ret)) { + cb = dynamic_cast(cb->fields[idxs_.at(i)]); + CK (cb != nullptr); + } + } + } + if (OB_SUCC(ret)) { + orc::LongVectorBatch *long_batch = dynamic_cast(root->fields[idxs_.at(idxs_.count() - 1)]); + if (!long_batch) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic cast orc type failed", K(ret)); + } else if (!long_batch->hasNulls) { + CK (OB_NOT_NULL(long_batch->data.data())); + CK (OB_NOT_NULL(int64_vec->get_data())); + if (OB_SUCC(ret)) { + MEMCPY(pointer_cast(int64_vec->get_data()), long_batch->data.data(), sizeof(int64_t) * row_count_); + } + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < row_count_; i++) { + CK (OB_NOT_NULL(long_batch->notNull.data())); + if (OB_SUCC(ret)) { + const uint8_t* valid_bytes = reinterpret_cast(long_batch->notNull.data()) + i; + if (OB_ISNULL(valid_bytes)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("orc not null batch valid bytes is null", K(ret)); + } else if (*valid_bytes == 1) { + int64_vec->set_int(i, long_batch->data[i]); + } else { + int64_vec->set_null(i); + } + } + } + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("read orc next batch failed", K(ret)); + } + } + } + LOG_DEBUG("load int64 vec", K(ret), K(row_count_)); + return ret; +} + +int ObOrcTableRowIterator::DataLoader::load_int32_vec() +{ + int ret = OB_SUCCESS; + int64_t values_cnt = 0; + ObEvalCtx::TempAllocGuard tmp_alloc_g(eval_ctx_); + CK (OB_NOT_NULL(file_col_expr_)); + if (OB_SUCC(ret)) { + ObFixedLengthBase *int32_vec = static_cast(file_col_expr_->get_vector(eval_ctx_)); + CK (OB_NOT_NULL(int32_vec)); + CK (VEC_FIXED == int32_vec->get_format()); + if (OB_SUCC(ret)) { + if (batch_) { + row_count_ = batch_->numElements; + orc::StructVectorBatch *root = dynamic_cast(batch_.get()); + if (OB_ISNULL(root)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic cast orc column vector batch failed", K(ret)); + } + CK (root->fields.size() > 0); + CK (idxs_.count() > 0); + if (OB_SUCC(ret)) { + orc::StructVectorBatch *cb = root; + for (int64_t i = 0; OB_SUCC(ret) && i < idxs_.count() - 1; i++) { + CK (root->fields.size() > idxs_.at(i)); + if (OB_SUCC(ret)) { + cb = dynamic_cast(cb->fields[idxs_.at(i)]); + CK (cb != nullptr); + } + } + } + if (OB_SUCC(ret)) { + orc::LongVectorBatch *long_batch = dynamic_cast(root->fields[idxs_.at(idxs_.count() - 1)]); + if (!long_batch) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic cast orc type failed", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < row_count_; i++) { + if (long_batch->hasNulls) { + CK (OB_NOT_NULL(long_batch->notNull.data())); + if (OB_SUCC(ret)) { + const uint8_t* valid_bytes = reinterpret_cast(long_batch->notNull.data()) + i; + if (OB_ISNULL(valid_bytes)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("orc not null batch valid bytes is null", K(ret)); + } else if (*valid_bytes == 1) { + int32_vec->set_int32(i, (int32_t)long_batch->data[i]); + } else { + int32_vec->set_null(i); + } + } + } else { + int32_vec->set_int32(i, (int32_t)long_batch->data[i]); + } + } + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("read orc next batch failed", K(ret)); + } + } + } + LOG_DEBUG("load int32 vec", K(ret), K(row_count_)); + return ret; +} + +int ObOrcTableRowIterator::DataLoader::load_string_col() +{ + int ret = OB_SUCCESS; + int64_t values_cnt = 0; + ObEvalCtx::TempAllocGuard tmp_alloc_g(eval_ctx_); + CK (OB_NOT_NULL(file_col_expr_)); + if (OB_SUCC(ret)) { + StrDiscVec *text_vec = static_cast(file_col_expr_->get_vector(eval_ctx_)); + ObArrayWrap values; + CK (OB_NOT_NULL(text_vec)); + CK (VEC_DISCRETE == text_vec->get_format()); + OZ (values.allocate_array(tmp_alloc_g.get_allocator(), batch_size_)); + if (OB_SUCC(ret)) { + if (batch_) { + row_count_ = batch_->numElements; + orc::StructVectorBatch *root = dynamic_cast(batch_.get()); + if (OB_ISNULL(root)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic cast orc column vector batch failed", K(ret)); + } + CK (root->fields.size() > 0); + CK (idxs_.count() > 0); + if (OB_SUCC(ret)) { + orc::StructVectorBatch *cb = root; + for (int64_t i = 0; OB_SUCC(ret) && i < idxs_.count() - 1; i++) { + CK (root->fields.size() > idxs_.at(i)); + if (OB_SUCC(ret)) { + cb = dynamic_cast(cb->fields[idxs_.at(i)]); + CK (cb != nullptr); + } + } + } + if (OB_SUCC(ret)) { + bool is_oracle_mode = lib::is_oracle_mode(); + bool is_byte_length = is_oracle_byte_length( + is_oracle_mode, file_col_expr_->datum_meta_.length_semantics_); + orc::StringVectorBatch *string_batch = dynamic_cast(root->fields[idxs_.at(idxs_.count() - 1)]); + if (!string_batch) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic cast orc type failed", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < row_count_; i++) { + if (string_batch->hasNulls) { + CK (OB_NOT_NULL(string_batch->data.data())); + CK (OB_NOT_NULL(string_batch->notNull.data())); + if (OB_SUCC(ret)) { + const uint8_t* valid_bytes = reinterpret_cast(string_batch->notNull.data()) + i; + if (OB_ISNULL(valid_bytes)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("orc not null batch valid bytes is null", K(ret)); + } else if (*valid_bytes == 1) { + if (string_batch->length[i] == 0 && is_oracle_mode) { + text_vec->set_null(i); + } else { + if (OB_UNLIKELY(string_batch->length[i] > file_col_expr_->max_length_ + && (is_byte_length || ObCharset::strlen_char(CS_TYPE_UTF8MB4_BIN, + pointer_cast(string_batch->data[i]), + string_batch->length[i]) > file_col_expr_->max_length_))) { + ret = OB_ERR_DATA_TOO_LONG; + LOG_WARN("data too long", K(ret)); + } else { + values.at(i) = std::string(string_batch->data[i], string_batch->data[i] + string_batch->length[i]); + text_vec->set_string(i, values.at(i).data(), values.at(i).length()); + } + } + } else { + text_vec->set_null(i); + } + } + } else { + CK (OB_NOT_NULL(string_batch->length.data())); + if (OB_FAIL(ret)) { + } else if (string_batch->length[i] == 0 && is_oracle_mode) { + text_vec->set_null(i); + } else { + CK (OB_NOT_NULL(string_batch->data.data())); + if (OB_FAIL(ret)) { + } else if (OB_UNLIKELY(string_batch->length[i] > file_col_expr_->max_length_ + && (is_byte_length || ObCharset::strlen_char(CS_TYPE_UTF8MB4_BIN, + pointer_cast(string_batch->data[i]), + string_batch->length[i]) > file_col_expr_->max_length_))) { + ret = OB_ERR_DATA_TOO_LONG; + LOG_WARN("data too long", K(ret)); + } else { + values.at(i) = std::string(string_batch->data[i], string_batch->data[i] + string_batch->length[i]); + text_vec->set_string(i, values.at(i).data(), values.at(i).length()); + } + } + } + } + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("read orc next batch failed", K(ret)); + } + } + } + return ret; +} + +int ObOrcTableRowIterator::DataLoader::load_timestamp_vec() +{ + int ret = OB_SUCCESS; + int64_t values_cnt = 0; + ObEvalCtx::TempAllocGuard tmp_alloc_g(eval_ctx_); + CK (OB_NOT_NULL(file_col_expr_)); + if (OB_SUCC(ret)) { + ObFixedLengthBase *dec_vec = static_cast(file_col_expr_->get_vector(eval_ctx_)); + int64_t adjust_us = calc_tz_adjust_us(); + LOG_DEBUG("adjust value", K(adjust_us)); + CK (OB_NOT_NULL(dec_vec)); + if (OB_SUCC(ret)) { + if (batch_) { + row_count_ = batch_->numElements; + orc::StructVectorBatch *root = dynamic_cast(batch_.get()); + if (OB_ISNULL(root)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic cast orc column vector batch failed", K(ret)); + } + CK (root->fields.size() > 0); + CK (idxs_.count() > 0); + if (OB_SUCC(ret)) { + orc::StructVectorBatch *cb = root; + for (int64_t i = 0; OB_SUCC(ret) && i < idxs_.count() - 1; i++) { + CK (root->fields.size() > idxs_.at(i)); + if (OB_SUCC(ret)) { + cb = dynamic_cast(cb->fields[idxs_.at(i)]); + CK (cb != nullptr); + } + } + } + if (OB_SUCC(ret)) { + row_count_ = batch_->numElements; + orc::TimestampVectorBatch *timestamp_batch = dynamic_cast(root->fields[idxs_.at(idxs_.count() - 1)]); + if (!timestamp_batch) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic cast orc type failed", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < row_count_; i++) { + const uint8_t* valid_bytes = nullptr; + CK (OB_NOT_NULL(timestamp_batch->data.data())); + CK (OB_NOT_NULL(timestamp_batch->nanoseconds.data())); + if (OB_SUCC(ret) && timestamp_batch->hasNulls) { + CK (OB_NOT_NULL(timestamp_batch->notNull.data())); + if (OB_SUCC(ret)) { + valid_bytes = reinterpret_cast(timestamp_batch->notNull.data()) + i; + } + } + if (OB_FAIL(ret)) { + } else if (!timestamp_batch->hasNulls || *valid_bytes == 1) { + int64_t adjusted_value = timestamp_batch->data[i] * USECS_PER_SEC + timestamp_batch->nanoseconds[i] / NSECS_PER_USEC + adjust_us; + if (ObTimestampType == file_col_expr_->datum_meta_.type_) { + dec_vec->set_timestamp(i, adjusted_value); + } else if (ObDateTimeType == file_col_expr_->datum_meta_.type_) { + dec_vec->set_datetime(i, adjusted_value); + } else if (ObDateType == file_col_expr_->datum_meta_.type_) { + dec_vec->set_date(i, adjusted_value / USECS_PER_DAY); + } else if (ObTimeType == file_col_expr_->datum_meta_.type_) { + dec_vec->set_time(i, adjusted_value); + } else { + ObOTimestampData data; + data.time_us_ = adjusted_value; + data.time_ctx_.set_tail_nsec(timestamp_batch->nanoseconds[i] % NSECS_PER_USEC); + dec_vec->set_otimestamp_tiny(i, ObOTimestampTinyData().from_timestamp_data(data)); + } + } else { + dec_vec->set_null(i); + } + } + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("read orc next batch failed", K(ret)); + } + } + } + return ret; +} + + +int ObOrcTableRowIterator::DataLoader::load_date_to_time_or_stamp() +{ + int ret = OB_SUCCESS; + int64_t values_cnt = 0; + ObEvalCtx::TempAllocGuard tmp_alloc_g(eval_ctx_); + CK (OB_NOT_NULL(file_col_expr_)); + if (OB_SUCC(ret)) { + ObFixedLengthBase *dec_vec = static_cast(file_col_expr_->get_vector(eval_ctx_)); + int64_t adjust_us = calc_tz_adjust_us(); + LOG_DEBUG("show adjust value in date to ts", K(adjust_us)); + if (OB_SUCC(ret)) { + if (batch_) { + row_count_ = batch_->numElements; + orc::StructVectorBatch *root = dynamic_cast(batch_.get()); + CK (root->fields.size() > 0); + CK (idxs_.count() > 0); + if (OB_SUCC(ret)) { + orc::StructVectorBatch *cb = root; + for (int64_t i = 0; OB_SUCC(ret) && i < idxs_.count() - 1; i++) { + CK (root->fields.size() > idxs_.at(i)); + if (OB_SUCC(ret)) { + cb = dynamic_cast(cb->fields[idxs_.at(i)]); + CK (cb != nullptr); + } + } + } + if (OB_SUCC(ret)) { + row_count_ = batch_->numElements; + orc::LongVectorBatch *date_batch = dynamic_cast(root->fields[idxs_.at(idxs_.count() - 1)]); + if (OB_ISNULL(date_batch)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic dec64 batch cast failed", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < row_count_; i++) { + const uint8_t* valid_bytes = nullptr; + CK (OB_NOT_NULL(date_batch->data.data())); + if (OB_SUCC(ret)) { + if (date_batch->hasNulls) { + CK (OB_NOT_NULL(date_batch->notNull.data())); + if (OB_SUCC(ret)) { + valid_bytes = reinterpret_cast(date_batch->notNull.data()) + i; + } + } + if (OB_FAIL(ret)) { + } else if (!date_batch->hasNulls || *valid_bytes == 1) { + int64_t adjusted_value = date_batch->data[i] * USECS_PER_DAY + adjust_us; + if (ObTimestampType == file_col_expr_->datum_meta_.type_) { + dec_vec->set_timestamp(i, adjusted_value); + } else if (ObDateTimeType == file_col_expr_->datum_meta_.type_) { + dec_vec->set_datetime(i, date_batch->data[i] * USECS_PER_DAY); + } else if (ObDateType == file_col_expr_->datum_meta_.type_) { + dec_vec->set_date(i, adjusted_value / USECS_PER_DAY); + } else if (ObTimeType == file_col_expr_->datum_meta_.type_) { + dec_vec->set_time(i, adjusted_value); + } else { + ObOTimestampData data; + data.time_us_ = adjusted_value; + dec_vec->set_otimestamp_tiny(i, ObOTimestampTinyData().from_timestamp_data(data)); + } + } else { + dec_vec->set_null(i); + } + } + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("read orc next batch failed", K(ret)); + } + } + } + return ret; +} + +int ObOrcTableRowIterator::DataLoader::load_dec64_vec() +{ + int ret = OB_SUCCESS; + int64_t values_cnt = 0; + ObEvalCtx::TempAllocGuard tmp_alloc_g(eval_ctx_); + CK (OB_NOT_NULL(file_col_expr_)); + if (OB_SUCC(ret)) { + ObFixedLengthBase *vec = static_cast(file_col_expr_->get_vector(eval_ctx_)); + CK (OB_NOT_NULL(vec)); + CK (VEC_FIXED == vec->get_format()); + if (OB_SUCC(ret)) { + if (batch_) { + row_count_ = batch_->numElements; + orc::StructVectorBatch *root = dynamic_cast(batch_.get()); + if (OB_ISNULL(root)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic cast orc column vector batch failed", K(ret)); + } + CK (root->fields.size() > 0); + CK (idxs_.count() > 0); + if (OB_SUCC(ret)) { + orc::StructVectorBatch *cb = root; + for (int64_t i = 0; OB_SUCC(ret) && i < idxs_.count() - 1; i++) { + CK (root->fields.size() > idxs_.at(i)); + if (OB_SUCC(ret)) { + cb = dynamic_cast(cb->fields[idxs_.at(i)]); + CK (cb != nullptr); + } + } + } + if (OB_SUCC(ret)) { + orc::Decimal64VectorBatch *dec64_batch = dynamic_cast(root->fields[idxs_.at(idxs_.count() - 1)]); + if (OB_ISNULL(dec64_batch)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic dec64 batch cast failed", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < row_count_; i++) { + CK (OB_NOT_NULL(dec64_batch->values.data())); + if (OB_SUCC(ret)) { + const uint8_t* valid_bytes = nullptr; + if (dec64_batch->hasNulls) { + CK (OB_NOT_NULL(dec64_batch->notNull.data())); + if (OB_SUCC(ret)) { + valid_bytes = reinterpret_cast(dec64_batch->notNull.data()) + i; + } + } + if (OB_FAIL(ret)) { + } else if (!dec64_batch->hasNulls || *valid_bytes == 1) { + if (ObDecimalIntType == file_col_expr_->datum_meta_.type_) { + ObDecimalInt *decint = NULL; + int32_t int_bytes = 0; + if (OB_FAIL(wide::from_integer(dec64_batch->values[i], tmp_alloc_g.get_allocator(), decint, + int_bytes, file_col_expr_->datum_meta_.precision_))) { + LOG_WARN("fail to from integer", K(ret)); + } else if (OB_ISNULL(decint)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("int to dec failed", K(ret)); + } else { + vec->set_decimal_int(i, decint, int_bytes); + } + } else if (ObNumberType == file_col_expr_->datum_meta_.type_) { + ObDiscreteBase *vec = static_cast(file_col_expr_->get_vector(eval_ctx_)); + number::ObNumber res_nmb; + if (OB_FAIL(res_nmb.from(dec64_batch->values[i], tmp_alloc_g.get_allocator()))) { + LOG_WARN("fail to from number", K(ret)); + } else { + vec->set_number(i, res_nmb); + } + } + } else { + file_col_expr_->get_vector(eval_ctx_)->set_null(i); + } + } + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("read orc next batch failed", K(ret)); + } + } + } + return ret; +} + +int ObOrcTableRowIterator::DataLoader::load_dec128_vec() +{ + int ret = OB_SUCCESS; + int64_t values_cnt = 0; + ObEvalCtx::TempAllocGuard tmp_alloc_g(eval_ctx_); + if (OB_SUCC(ret)) { + if (batch_) { + row_count_ = batch_->numElements; + orc::StructVectorBatch *root = dynamic_cast(batch_.get()); + if (OB_ISNULL(root)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic cast orc column vector batch failed", K(ret)); + } + CK (root->fields.size() > 0); + CK (idxs_.count() > 0); + if (OB_SUCC(ret)) { + orc::StructVectorBatch *cb = root; + for (int64_t i = 0; OB_SUCC(ret) && i < idxs_.count() - 1; i++) { + CK (root->fields.size() > idxs_.at(i)); + if (OB_SUCC(ret)) { + cb = dynamic_cast(cb->fields[idxs_.at(i)]); + CK (cb != nullptr); + } + } + } + if (OB_SUCC(ret)) { + orc::Decimal128VectorBatch *dec128_batch = dynamic_cast(root->fields[idxs_.at(idxs_.count() - 1)]); + if (OB_ISNULL(dec128_batch)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic dec128 batch cast failed", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < row_count_; i++) { + CK (OB_NOT_NULL(dec128_batch->values.data())); + if (OB_SUCC(ret)) { + const uint8_t* valid_bytes = nullptr; + if (dec128_batch->hasNulls) { + CK (OB_NOT_NULL(dec128_batch->notNull.data())); + if (OB_SUCC(ret)) { + valid_bytes = reinterpret_cast(dec128_batch->notNull.data()) + i; + } + } + if (OB_FAIL(ret)) { + } else if (!dec128_batch->hasNulls || *valid_bytes == 1) { + ObDecimalInt *decint = NULL; + int32_t int_bytes = sizeof(int128_t); + int128_t val = dec128_batch->values[i].getHighBits(); + val = val << 64; + val = val | dec128_batch->values[i].getLowBits(); + void *data = nullptr; + + if (OB_ISNULL(data = tmp_alloc_g.get_allocator().alloc(int_bytes))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + COMMON_LOG(WARN, "allocate memory failed", K(ret), K(int_bytes)); + } else { + decint = reinterpret_cast(data); + *decint->int128_v_ = val; + } + if (OB_FAIL(ret)) { + } else if (ObDecimalIntType == file_col_expr_->datum_meta_.type_) { + ObFixedLengthBase *vec = static_cast(file_col_expr_->get_vector(eval_ctx_)); + CK (OB_NOT_NULL(vec)); + CK (VEC_FIXED == vec->get_format()); + if (OB_SUCC(ret)) { + vec->set_decimal_int(i, decint, int_bytes); + } + } else if (ObNumberType == file_col_expr_->datum_meta_.type_) { + ObDiscreteBase *vec = static_cast(file_col_expr_->get_vector(eval_ctx_)); + CK (OB_NOT_NULL(vec)); + if (OB_SUCC(ret)) { + number::ObNumber res_nmb; + if (OB_FAIL(wide::to_number(decint, int_bytes, file_col_expr_->datum_meta_.scale_, + tmp_alloc_g.get_allocator(), res_nmb))) { + LOG_WARN("fail to from", K(ret)); + } else { + vec->set_number(i, res_nmb); + } + } + } + } else { + file_col_expr_->get_vector(eval_ctx_)->set_null(i); + } + } + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("read orc next batch failed", K(ret)); + } + } + return ret; +} + +int ObOrcTableRowIterator::DataLoader::load_float() +{ + int ret = OB_SUCCESS; + int64_t values_cnt = 0; + ObEvalCtx::TempAllocGuard tmp_alloc_g(eval_ctx_); + ObFixedLengthBase *float_vec = static_cast(file_col_expr_->get_vector(eval_ctx_)); + CK (OB_NOT_NULL(float_vec)); + CK (VEC_FIXED == float_vec->get_format()); + if (OB_SUCC(ret)) { + if (batch_) { + row_count_ = batch_->numElements; + orc::StructVectorBatch *root = dynamic_cast(batch_.get()); + if (OB_ISNULL(root)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic cast orc column vector batch failed", K(ret)); + } + CK (root->fields.size() > 0); + CK (idxs_.count() > 0); + if (OB_SUCC(ret)) { + orc::StructVectorBatch *cb = root; + for (int64_t i = 0; OB_SUCC(ret) && i < idxs_.count() - 1; i++) { + CK (root->fields.size() > idxs_.at(i)); + if (OB_SUCC(ret)) { + cb = dynamic_cast(cb->fields[idxs_.at(i)]); + CK (cb != nullptr); + } + } + } + if (OB_SUCC(ret)) { + orc::DoubleVectorBatch *double_batch = dynamic_cast(root->fields[idxs_.at(idxs_.count() - 1)]); + if (OB_ISNULL(double_batch)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic double batch cast failed", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < row_count_; i++) { + CK (OB_NOT_NULL(double_batch->data.data())); + if (OB_FAIL(ret)) { + } else if (double_batch->hasNulls) { + CK (OB_NOT_NULL(double_batch->notNull.data())); + if (OB_SUCC(ret)) { + const uint8_t* valid_bytes = reinterpret_cast(double_batch->notNull.data()) + i; + if (*valid_bytes == 1) { + float_vec->set_float(i, (float)double_batch->data[i]); + } else { + float_vec->set_null(i); + } + } + } else { + float_vec->set_float(i, (float)double_batch->data[i]); + } + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("read orc next batch failed", K(ret)); + } + } + return ret; +} + +int ObOrcTableRowIterator::DataLoader::load_double() +{ + int ret = OB_SUCCESS; + int64_t values_cnt = 0; + ObEvalCtx::TempAllocGuard tmp_alloc_g(eval_ctx_); + ObFixedLengthBase *double_vec = static_cast(file_col_expr_->get_vector(eval_ctx_)); + CK (OB_NOT_NULL(double_vec)); + CK (VEC_FIXED == double_vec->get_format()); + if (OB_SUCC(ret)) { + if (batch_) { + row_count_ = batch_->numElements; + orc::StructVectorBatch *root = dynamic_cast(batch_.get()); + if (OB_ISNULL(root)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic cast orc column vector batch failed", K(ret)); + } + CK (root->fields.size() > 0); + CK (idxs_.count() > 0); + if (OB_SUCC(ret)) { + orc::StructVectorBatch *cb = root; + for (int64_t i = 0; OB_SUCC(ret) && i < idxs_.count() - 1; i++) { + CK (root->fields.size() > idxs_.at(i)); + if (OB_SUCC(ret)) { + cb = dynamic_cast(cb->fields[idxs_.at(i)]); + CK (cb != nullptr); + } + } + } + if (OB_SUCC(ret)) { + orc::DoubleVectorBatch *double_batch = dynamic_cast(root->fields[idxs_.at(idxs_.count() - 1)]); + if (OB_ISNULL(double_batch)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic double batch cast failed", K(ret)); + } + if (OB_FAIL(ret)) { + } else if (!double_batch->hasNulls) { + CK (OB_NOT_NULL(double_batch->data.data())); + CK (OB_NOT_NULL(double_vec->get_data())); + if (OB_SUCC(ret)) { + MEMCPY(pointer_cast(double_vec->get_data()), double_batch->data.data(), sizeof(double) * row_count_); + } + } else { + for (int64_t i = 0; i < row_count_; i++) { + CK (OB_NOT_NULL(double_batch->notNull.data())); + CK (OB_NOT_NULL(double_batch->data.data())); + if (OB_SUCC(ret)) { + const uint8_t* valid_bytes = reinterpret_cast(double_batch->notNull.data()) + i; + CK (OB_NOT_NULL(valid_bytes)); + if (OB_FAIL(ret)) { + } else if (*valid_bytes == 1) { + double_vec->set_double(i, double_batch->data[i]); + } else { + double_vec->set_null(i); + } + } + } + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("read orc next batch failed", K(ret)); + } + } + return ret; +} + + +int ObOrcTableRowIterator::get_next_row() +{ + int ret = OB_NOT_SUPPORTED; + return ret; +} + +void ObOrcTableRowIterator::reset() { + // reset state_ to initial values for rescan + state_.reuse(); +} + +} +} \ No newline at end of file diff --git a/src/sql/engine/table/ob_orc_table_row_iter.h b/src/sql/engine/table/ob_orc_table_row_iter.h new file mode 100644 index 000000000..2b75531e5 --- /dev/null +++ b/src/sql/engine/table/ob_orc_table_row_iter.h @@ -0,0 +1,235 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OB_ORC_TABLE_ROW_ITER_H +#define OB_ORC_TABLE_ROW_ITER_H + +#include "share/ob_i_tablet_scan.h" +#include "lib/file/ob_file.h" +#include "common/row/ob_row_iterator.h" +#include "storage/access/ob_dml_param.h" +#include "common/storage/ob_io_device.h" +#include "share/backup/ob_backup_struct.h" +#include "sql/engine/table/ob_external_table_access_service.h" +#include +#include +#include +#include +#include +#include + +namespace oceanbase { +namespace sql { + class ObOrcMemPool : public orc::MemoryPool { + public: + void init(uint64_t tenant_id) { + mem_attr_ = ObMemAttr(tenant_id, "OrcMemPool"); + } + + virtual char* malloc(uint64_t size) override { + int ret = OB_SUCCESS; + void *buf = ob_malloc_align(64, size, mem_attr_); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate memory", K(size), K(lbt())); + throw std::bad_alloc(); + } + return (char*)buf; + } + + virtual void free(char* p) override { + if (OB_ISNULL(p)) { + throw std::bad_exception(); + } + ob_free_align(p); + } + + private: + common::ObMemAttr mem_attr_; + }; + + class ObOrcFileAccess : public orc::InputStream { + public: + ObOrcFileAccess(ObExternalDataAccessDriver &file_reader, const char* file_name, int64_t len) + : file_reader_(file_reader), file_name_(file_name), total_length_(len), fileName_(file_name) { + } + + uint64_t getLength() const { + return total_length_; + } + + uint64_t getNaturalReadSize() const { + return 128 * 1024; + } + + void read(void* buf, + uint64_t length, + uint64_t offset) override { + int64_t bytesRead = 0; + int ret = file_reader_.pread(buf, length, offset, bytesRead); + if (ret != OB_SUCCESS) { + throw std::bad_exception(); + } + LOG_TRACE("read file access", K(file_name_), K(bytesRead)); + } + + const std::string& getName() const override { + return fileName_; + } + + private: + ObExternalDataAccessDriver &file_reader_; + const char* file_name_; + uint64_t total_length_; + std::string fileName_; + }; + + struct StripeInformation { + /// \brief Offset of the stripe from the start of the file, in bytes + int64_t offset; + /// \brief Length of the stripe, in bytes + int64_t length; + /// \brief Number of rows in the stripe + int64_t num_rows; + /// \brief Index of the first row of the stripe + int64_t first_row_id; + + TO_STRING_KV(K(offset), K(length), K(num_rows), K(first_row_id)); + }; + + class ObOrcTableRowIterator : public ObExternalTableRowIterator { + public: + struct StateValues { + StateValues() : + file_idx_(0), + part_id_(0), + cur_file_id_(0), + cur_file_url_(), + cur_stripe_idx_(0), + end_stripe_idx_(-1), + cur_stripe_read_row_count_(0), + cur_stripe_row_count_(0), + batch_size_(128), + part_list_val_() {} + void reuse() { + file_idx_ = 0; + part_id_ = 0; + cur_file_id_ = 0; + cur_stripe_idx_ = 0; + end_stripe_idx_ = -1; + cur_stripe_read_row_count_ = 0; + cur_stripe_row_count_ = 0; + cur_file_url_.reset(); + part_list_val_.reset(); + } + int64_t file_idx_; + int64_t part_id_; + int64_t cur_file_id_; + ObString cur_file_url_; + int64_t cur_stripe_idx_; + int64_t end_stripe_idx_; + int64_t cur_stripe_read_row_count_; + int64_t cur_stripe_row_count_; + int64_t batch_size_; + ObNewRow part_list_val_; + }; + public: + ObOrcTableRowIterator() : file_column_exprs_(allocator_), file_meta_column_exprs_(allocator_), bit_vector_cache_(NULL) {} + virtual ~ObOrcTableRowIterator() { + + } + + int init(const storage::ObTableScanParam *scan_param) override; + + virtual int get_next_row(ObNewRow *&row) override { + UNUSED(row); + return common::OB_ERR_UNEXPECTED; + } + + int get_next_row() override; + int get_next_rows(int64_t &count, int64_t capacity) override; + + virtual void reset() override; +private: + // load vec data from orc file to expr mem + struct DataLoader { + DataLoader(ObEvalCtx &eval_ctx, + ObExpr *file_col_expr, + std::unique_ptr &batch, + const int64_t batch_size, + const ObIArray &idxs, + int64_t &row_count): + eval_ctx_(eval_ctx), + file_col_expr_(file_col_expr), + batch_(batch), + batch_size_(batch_size), + idxs_(idxs), + row_count_(row_count) + {} + typedef int (DataLoader::*LOAD_FUNC)(); + static LOAD_FUNC select_load_function(const ObDatumMeta &datum_type, + const orc::Type &type); + int load_data_for_col(LOAD_FUNC &func); + int load_string_col(); + int load_int32_vec(); + int load_int64_vec(); + int load_timestamp_vec(); + int load_date_to_time_or_stamp(); + int load_float(); + int load_double(); + int load_dec128_vec(); + int load_dec64_vec(); + + bool is_orc_read_utc(); + bool is_ob_type_store_utc(const ObDatumMeta &meta); + + int64_t calc_tz_adjust_us(); + ObEvalCtx &eval_ctx_; + ObExpr *file_col_expr_; + std::unique_ptr &batch_; + const int64_t batch_size_; + const ObIArray &idxs_; + int64_t &row_count_; + }; + private: + int next_file(); + int next_stripe(); + int build_type_name_id_map(const orc::Type* type, ObIArray &col_names); + int to_dot_column_path(ObIArray &col_names, ObString &path); + int get_data_column_batch_idxs(const orc::Type *type, const int col_id, ObIArray &idxs); + private: + + StateValues state_; + lib::ObMemAttr mem_attr_; + ObArenaAllocator allocator_; + ObOrcMemPool orc_alloc_; + std::unique_ptr reader_; + std::unique_ptr row_reader_; + common::ObArrayWrap stripes_; + ObExternalDataAccessDriver data_access_driver_; + common::ObArrayWrap column_indexs_; //for getting statistics, may useless now. + ExprFixedArray file_column_exprs_; //column value from parquet file + ExprFixedArray file_meta_column_exprs_; //column value from file meta + common::ObArrayWrap load_funcs_; + ObSqlString url_; + ObBitVector *bit_vector_cache_; + common::ObArrayWrap file_url_ptrs_; //for file url expr + common::ObArrayWrap file_url_lens_; //for file url expr + hash::ObHashMap id_to_type_; + hash::ObHashMap name_to_id_; + +}; + +} +} + +#endif \ No newline at end of file diff --git a/src/sql/resolver/dml/ob_dml_resolver.cpp b/src/sql/resolver/dml/ob_dml_resolver.cpp index c7aaa4234..618b9852f 100755 --- a/src/sql/resolver/dml/ob_dml_resolver.cpp +++ b/src/sql/resolver/dml/ob_dml_resolver.cpp @@ -8247,9 +8247,20 @@ int ObDMLResolver::resolve_external_table_generated_column( ObExternalFileFormat format; if (OB_FAIL(format.load_from_string(table_schema->get_external_file_format(), *params_.allocator_))) { LOG_WARN("load from string failed", K(ret)); + } else if (format.format_type_ == ObExternalFileFormat::ORC_FORMAT && lib::is_oracle_mode()) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not support orc in oracle mode", K(ret)); + LOG_USER_WARN(OB_NOT_SUPPORTED, "orc in oracle mode"); } else if (format.format_type_ != ObResolverUtils::resolve_external_file_column_type(col.col_name_)) { - ret = OB_WRONG_COLUMN_NAME; - LOG_USER_ERROR(OB_WRONG_COLUMN_NAME, col.col_name_.length(), col.col_name_.ptr()); + if (format.format_type_ == ObExternalFileFormat::ORC_FORMAT && + ObExternalFileFormat::PARQUET_FORMAT != ObResolverUtils::resolve_external_file_column_type(col.col_name_)) { + ret = OB_WRONG_COLUMN_NAME; + LOG_WARN("wrong column name", K(format.format_type_)); + LOG_USER_ERROR(OB_WRONG_COLUMN_NAME, col.col_name_.length(), col.col_name_.ptr()); + } + } + + if (OB_FAIL(ret)) { } else if (ObExternalFileFormat::CSV_FORMAT == format.format_type_) { if (OB_FAIL(ObResolverUtils::calc_file_column_idx(col.col_name_, file_column_idx))) { LOG_WARN("fail to calc file column idx", K(ret)); @@ -8263,7 +8274,8 @@ int ObDMLResolver::resolve_external_table_generated_column( LOG_WARN("fail to build external table file column expr", K(ret)); } } - } else if (ObExternalFileFormat::PARQUET_FORMAT == format.format_type_) { + } else if (ObExternalFileFormat::PARQUET_FORMAT == format.format_type_ || + ObExternalFileFormat::ORC_FORMAT == format.format_type_ ) { ObRawExpr *cast_expr = NULL; ObRawExpr *get_path_expr = NULL; ObRawExpr *cast_type_expr = NULL; From 56af8cfa43d5d33ee9cfefa53a5d38eadbf2293d Mon Sep 17 00:00:00 2001 From: Handora Date: Mon, 26 Aug 2024 06:39:16 +0000 Subject: [PATCH 217/249] fix create memtable during replay corner case --- .../test_callbacks_with_reverse_order.cpp | 2 +- src/storage/ls/ob_ls_tablet_service.cpp | 2 +- src/storage/ob_i_tablet_memtable.cpp | 20 +--- src/storage/ob_i_tablet_memtable.h | 7 -- src/storage/ob_storage_table_guard.cpp | 91 ++++++++++++++----- src/storage/ob_storage_table_guard.h | 15 +-- src/storage/ob_value_row_iterator.cpp | 2 +- src/storage/tablet/ob_tablet.cpp | 46 ++++++---- src/storage/tablet/ob_tablet_memtable_mgr.cpp | 36 ++++---- src/storage/tx/ob_tx_replay_executor.cpp | 2 +- 10 files changed, 127 insertions(+), 96 deletions(-) diff --git a/mittest/simple_server/test_callbacks_with_reverse_order.cpp b/mittest/simple_server/test_callbacks_with_reverse_order.cpp index d7309353b..8babff814 100644 --- a/mittest/simple_server/test_callbacks_with_reverse_order.cpp +++ b/mittest/simple_server/test_callbacks_with_reverse_order.cpp @@ -109,7 +109,7 @@ int ObLSTabletService::insert_tablet_rows( return ret; } -int ObStorageTableGuard::refresh_and_protect_table(ObRelativeTable &relative_table) +int ObStorageTableGuard::refresh_and_protect_memtable_for_write(ObRelativeTable &relative_table) { int ret = OB_SUCCESS; ObTabletTableIterator &iter = relative_table.tablet_iter_; diff --git a/src/storage/ls/ob_ls_tablet_service.cpp b/src/storage/ls/ob_ls_tablet_service.cpp index ec7da2599..2612b1c53 100644 --- a/src/storage/ls/ob_ls_tablet_service.cpp +++ b/src/storage/ls/ob_ls_tablet_service.cpp @@ -5148,7 +5148,7 @@ int ObLSTabletService::check_row_locked_by_myself( K(relative_table), K(store_ctx), K(rowkey)); } else { ObStorageTableGuard guard(tablet, store_ctx, true); - if (OB_FAIL(guard.refresh_and_protect_table(relative_table))) { + if (OB_FAIL(guard.refresh_and_protect_memtable_for_write(relative_table))) { LOG_WARN("fail to protect table", K(ret), K(tablet_handle)); } else if (OB_FAIL(tablet->check_row_locked_by_myself(relative_table, store_ctx, rowkey, locked))) { LOG_WARN("fail to check row locked, ", K(ret), K(rowkey)); diff --git a/src/storage/ob_i_tablet_memtable.cpp b/src/storage/ob_i_tablet_memtable.cpp index 9e55f582e..acfa06fc3 100644 --- a/src/storage/ob_i_tablet_memtable.cpp +++ b/src/storage/ob_i_tablet_memtable.cpp @@ -95,10 +95,9 @@ int ObITabletMemtable::resolve_left_boundary_for_active_memtable_() { int ret = OB_SUCCESS; storage::ObTabletMemtableMgr *memtable_mgr = get_memtable_mgr(); - const SCN new_start_scn = MAX(get_end_scn(), get_migration_clog_checkpoint_scn()); if (OB_NOT_NULL(memtable_mgr)) { - if (OB_FAIL(memtable_mgr->resolve_left_boundary_for_active_memtable(this, new_start_scn))) { + if (OB_FAIL(memtable_mgr->resolve_left_boundary_for_active_memtable(this, get_end_scn()))) { TRANS_LOG(WARN, "fail to resolve left boundary for active memtable", K(ret), K(ls_id_), KPC(this)); } } @@ -142,23 +141,6 @@ int ObITabletMemtable::set_freezer(ObFreezer *handler) return ret; } -int ObITabletMemtable::set_migration_clog_checkpoint_scn(const SCN &clog_checkpoint_scn) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!is_inited())) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not inited", K(ret)); - } else if (clog_checkpoint_scn <= ObScnRange::MIN_SCN) { - ret = OB_SCN_OUT_OF_BOUND; - TRANS_LOG(WARN, "invalid clog_checkpoint_ts", K(ret)); - } else { - (void)migration_clog_checkpoint_scn_.atomic_store(clog_checkpoint_scn); - } - - return ret; -} - int ObITabletMemtable::set_rec_scn(const SCN rec_scn) { int ret = OB_SUCCESS; diff --git a/src/storage/ob_i_tablet_memtable.h b/src/storage/ob_i_tablet_memtable.h index 06c4a7f80..07c89a2c8 100644 --- a/src/storage/ob_i_tablet_memtable.h +++ b/src/storage/ob_i_tablet_memtable.h @@ -209,14 +209,12 @@ public: unsubmitted_cnt_(0), logging_blocked_start_time_(0), write_ref_cnt_(0), - migration_clog_checkpoint_scn_(), freeze_state_(TabletMemtableFreezeState::INVALID), memtable_mgr_handle_() { max_end_scn_.set_min(); rec_scn_.set_max(); freeze_scn_.set_max(); - migration_clog_checkpoint_scn_.set_min(); } void reset() @@ -241,7 +239,6 @@ public: max_end_scn_.set_min(); rec_scn_.set_max(); freeze_scn_.set_max(); - migration_clog_checkpoint_scn_.set_min(); freezer_ = nullptr; memtable_mgr_handle_.reset(); mt_stat_.reset(); @@ -257,7 +254,6 @@ public: int dec_unsubmitted_cnt(); int set_freezer(ObFreezer *handler); int set_rec_scn(const share::SCN rec_scn); - int set_migration_clog_checkpoint_scn(const share::SCN &clog_checkpoint_scn); int resolve_left_boundary(share::SCN end_scn) { return set_start_scn(end_scn); } int resolve_right_boundary(); int replay_schema_version_change_log(const int64_t schema_version); @@ -374,7 +370,6 @@ public: const ObMtStat &get_mt_stat() const { return mt_stat_; } share::SCN get_max_end_scn() const { return max_end_scn_.atomic_get(); } share::SCN get_rec_scn() { return rec_scn_.atomic_get(); } - share::SCN get_migration_clog_checkpoint_scn() { return migration_clog_checkpoint_scn_.atomic_get(); } ObTabletMemtableMgr *get_memtable_mgr(); // *************** getter ***************** @@ -399,7 +394,6 @@ public: K(max_end_scn_), K(rec_scn_), K(freeze_scn_), - K(migration_clog_checkpoint_scn_), KP(freezer_), K(memtable_mgr_handle_), K(mt_stat_.frozen_time_), @@ -477,7 +471,6 @@ private: int64_t unsubmitted_cnt_; int64_t logging_blocked_start_time_; // record the start time of logging blocked int64_t write_ref_cnt_ CACHE_ALIGNED; - share::SCN migration_clog_checkpoint_scn_; TabletMemtableFreezeState freeze_state_; ObMemtableMgrHandle memtable_mgr_handle_; }; diff --git a/src/storage/ob_storage_table_guard.cpp b/src/storage/ob_storage_table_guard.cpp index dd739f63d..20ea22139 100644 --- a/src/storage/ob_storage_table_guard.cpp +++ b/src/storage/ob_storage_table_guard.cpp @@ -37,8 +37,7 @@ ObStorageTableGuard::ObStorageTableGuard( ObStoreCtx &store_ctx, const bool need_control_mem, const bool for_replay, - const SCN replay_scn, - const bool for_multi_source_data) + const SCN replay_scn) : tablet_(tablet), store_ctx_(store_ctx), need_control_mem_(need_control_mem), @@ -46,8 +45,7 @@ ObStorageTableGuard::ObStorageTableGuard( retry_count_(0), last_ts_(0), for_replay_(for_replay), - replay_scn_(replay_scn), - for_multi_source_data_(for_multi_source_data) + replay_scn_(replay_scn) { init_ts_ = ObClockGenerator::getClock(); share::memstore_throttled_alloc() = 0; @@ -105,7 +103,7 @@ void ObStorageTableGuard::throttle_if_needed_() } } -int ObStorageTableGuard::refresh_and_protect_table(ObRelativeTable &relative_table) +int ObStorageTableGuard::refresh_and_protect_memtable_for_write(ObRelativeTable &relative_table) { int ret = OB_SUCCESS; ObTabletTableIterator &iter = relative_table.tablet_iter_; @@ -142,7 +140,7 @@ int ObStorageTableGuard::refresh_and_protect_table(ObRelativeTable &relative_tab return ret; } -int ObStorageTableGuard::refresh_and_protect_memtable() +int ObStorageTableGuard::refresh_and_protect_memtable_for_replay() { const int64_t DEFAULT_REFRESH_WARN_INTERVAL = 10LL * 1000LL; // 10 ms const int64_t FIND_DIRECT_LOAD_MT_WARN_INTERVAL = 10LL * 1000LL * 1000LL; // 10 seconds @@ -163,7 +161,7 @@ int ObStorageTableGuard::refresh_and_protect_memtable() if (OB_FAIL(tablet_->get_boundary_memtable_from_memtable_mgr(handle))) { // if there is no memtable, create a new one if (OB_ENTRY_NOT_EXIST == ret) { - ret = create_data_memtable_(ls_id, tablet_id, need_retry); + ret = create_data_memtable_for_replay_(ls_id, tablet_id, need_retry); } else { // OB_ENTRY_NOT_EXIST != ret LOG_WARN("fail to get boundary memtable", K(ret), K(ls_id), K(tablet_id)); } @@ -172,7 +170,7 @@ int ObStorageTableGuard::refresh_and_protect_memtable() } else if (tablet_memtable->is_direct_load_memtable()) { // set warn interval to 1 second because freeze direct load memtable is an async task warn_interval = FIND_DIRECT_LOAD_MT_WARN_INTERVAL; - ret = create_data_memtable_(ls_id, tablet_id, need_retry); + ret = create_data_memtable_for_replay_(ls_id, tablet_id, need_retry); } else if (OB_FAIL(check_freeze_to_inc_write_ref(static_cast(tablet_memtable), need_retry))) { if (OB_EAGAIN == ret) { } else if (OB_MINOR_FREEZE_NOT_ALLOW != ret) { @@ -201,9 +199,9 @@ int ObStorageTableGuard::refresh_and_protect_memtable() return ret; } -int ObStorageTableGuard::create_data_memtable_(const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, - bool &need_retry) +int ObStorageTableGuard::create_data_memtable_for_replay_(const share::ObLSID &ls_id, + const common::ObTabletID &tablet_id, + bool &need_retry) { int ret = OB_SUCCESS; LOG_DEBUG("there is no boundary memtable", K(ret), K(ls_id), K(tablet_id)); @@ -229,13 +227,43 @@ int ObStorageTableGuard::create_data_memtable_(const share::ObLSID &ls_id, tablet_id, 0 /* schema version */, false /* for_direct_load */, for_replay_, clog_checkpoint_scn))) { LOG_WARN("fail to create a boundary memtable", K(ret), K(ls_id), K(tablet_id)); } - } else { - // replay_log_scn_ <= clog_checkpoint_scn. no need to create a boundary memtable - need_retry = false; + // In situation that replay_log_scn_ <= clog_checkpoint_scn, we have no need + // to create the memtable. While we need double check to decide whether + // another thread has created the memtable that we need replay. And if it + // does, we must replay on the memtable. + } else if (OB_FAIL(double_check_get_memtable_for_replay_(replay_scn_, need_retry))) { + LOG_WARN("fail to double check replay memtable", K(ret), K(ls_id), K(tablet_id), + K(replay_scn_), K(clog_checkpoint_scn)); } return ret; } +int ObStorageTableGuard::double_check_get_memtable_for_replay_(const share::SCN replay_scn, + bool &need_retry) +{ + int ret = OB_SUCCESS; + ObTableHandleV2 handle; + ObProtectedMemtableMgrHandle *protected_handle = NULL; + + if (OB_FAIL(tablet_->get_protected_memtable_mgr_handle(protected_handle))) { + LOG_WARN("failed to get_protected_memtable_mgr_handle", K(ret), KPC(tablet_)); + } else if (OB_FAIL(protected_handle->get_memtable_for_replay(replay_scn_, handle))) { + if (OB_NO_NEED_UPDATE == ret) { + // no need to replay the log + need_retry = false; + ret = OB_SUCCESS; + } else if (OB_ENTRY_NOT_EXIST == ret) { + // memtable_mgr not exist, it means nothing need replay + need_retry = false; + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to get memtable for replay", K(ret), K(need_retry), K(replay_scn)); + } + } + + return ret; +} + void ObStorageTableGuard::reset() { if (NULL != memtable_) { @@ -302,7 +330,7 @@ int ObStorageTableGuard::check_freeze_to_inc_write_ref(ObMemtable *memtable, boo } else if (memtable->is_active_memtable()) { // the most recent memtable is active // no need to create a new memtable - if (for_replay_ || for_multi_source_data_) { + if (for_replay_) { // filter memtables for replay or multi_source_data according to scn ObTableHandleV2 handle; if (OB_FAIL(tablet_->get_protected_memtable_mgr_handle(protected_handle))) { @@ -334,7 +362,6 @@ int ObStorageTableGuard::check_freeze_to_inc_write_ref(ObMemtable *memtable, boo if (0 == write_ref) { SCN clog_checkpoint_scn; bool need_create_memtable = true; - SCN migration_clog_checkpoint_scn; ObTabletHandle tmp_handle; ObLSHandle ls_handle; if (OB_FAIL(MTL(ObLSService *)->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { @@ -343,15 +370,13 @@ int ObStorageTableGuard::check_freeze_to_inc_write_ref(ObMemtable *memtable, boo ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected error, invalid ls handle", K(ret), K(need_retry), K(ls_handle), K(ls_id), K(tablet_id)); } else if (OB_FAIL(ls_handle.get_ls()->get_tablet_svr()->get_tablet(tablet_id, - tmp_handle, 0, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { + tmp_handle, + 0, + ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { LOG_WARN("fail to get tablet", K(ret), K(ls_id), K(tablet_id)); } else if (FALSE_IT(clog_checkpoint_scn = tmp_handle.get_obj()->get_tablet_meta().clog_checkpoint_scn_)) { - } else if (FALSE_IT(migration_clog_checkpoint_scn = memtable->get_migration_clog_checkpoint_scn())) { - } else if (for_replay_ && !migration_clog_checkpoint_scn.is_min()) { - memtable->resolve_right_boundary(); - if (replay_scn_ <= clog_checkpoint_scn) { - need_create_memtable = false; - } + } else if (for_replay_ && replay_scn_ <= clog_checkpoint_scn) { + need_create_memtable = false; } // create a new memtable if no write in the old memtable @@ -368,6 +393,26 @@ int ObStorageTableGuard::check_freeze_to_inc_write_ref(ObMemtable *memtable, boo LOG_WARN("fail to create new memtable for freeze", K(ret), K(need_retry), K(ls_id), K(tablet_id)); } } + } else if (for_replay_) { + ObTableHandleV2 handle; + if (OB_FAIL(tablet_->get_protected_memtable_mgr_handle(protected_handle))) { + LOG_WARN("failed to get_protected_memtable_mgr_handle", K(ret), KPC(tablet_)); + } else if (OB_FAIL(protected_handle->get_memtable_for_replay(replay_scn_, handle))) { + if (OB_NO_NEED_UPDATE == ret) { + // no need to replay the log + need_retry = false; + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to get memtable for replay", K(ret), K(need_retry), K(ls_id), K(tablet_id)); + } + } else if (OB_FAIL(handle.get_data_memtable(memtable))) { + LOG_WARN("fail to get memtable from ObTableHandle", K(ret), K(need_retry), K(ls_id), K(tablet_id)); + } else { + if (memtable != old_memtable) { + is_tablet_freeze = memtable->get_is_tablet_freeze(); + } + double_check_inc_write_ref(old_freeze_flag, is_tablet_freeze, memtable, need_retry); + } } } } diff --git a/src/storage/ob_storage_table_guard.h b/src/storage/ob_storage_table_guard.h index 957bc4b75..ee64f133d 100644 --- a/src/storage/ob_storage_table_guard.h +++ b/src/storage/ob_storage_table_guard.h @@ -49,21 +49,21 @@ public: ObStoreCtx &store_ctx, const bool need_control_mem, const bool for_replay = false, - const share::SCN replay_scn = share::SCN(), - const bool for_multi_source_data = false); + const share::SCN replay_scn = share::SCN()); ~ObStorageTableGuard(); ObStorageTableGuard(const ObStorageTableGuard&) = delete; ObStorageTableGuard &operator=(const ObStorageTableGuard&) = delete; public: - int refresh_and_protect_table(ObRelativeTable &relative_table); - int refresh_and_protect_memtable(); + // refresh and get/create the memtable for write + int refresh_and_protect_memtable_for_write(ObRelativeTable &relative_table); + // refresh and get/create the memtable for replay + int refresh_and_protect_memtable_for_replay(); int get_memtable_for_replay(ObIMemtable *&memtable); TO_STRING_KV(KP(tablet_), K(need_control_mem_), K(for_replay_), - K(for_multi_source_data_), K(replay_scn_), KP(memtable_), K(retry_count_), @@ -78,10 +78,12 @@ private: memtable::ObMemtable *memtable, bool &bool_ret); int check_freeze_to_inc_write_ref(memtable::ObMemtable *table, bool &bool_ret); - int create_data_memtable_(const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, bool &no_need_create); + int create_data_memtable_for_replay_(const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, bool &no_need_create); bool need_to_refresh_table(ObTableStoreIterator &iter); void check_if_need_log_(bool &need_log, bool &need_log_error); void throttle_if_needed_(); + int double_check_get_memtable_for_replay_(const share::SCN replay_scn, + bool &need_retry); private: static const int64_t LOG_INTERVAL_US = 10 * 1000 * 1000; // 10s @@ -99,7 +101,6 @@ private: int64_t init_ts_; bool for_replay_; share::SCN replay_scn_; - bool for_multi_source_data_; }; } // namespace storage } // namespace oceanbase diff --git a/src/storage/ob_value_row_iterator.cpp b/src/storage/ob_value_row_iterator.cpp index aa99478ee..124f2d377 100644 --- a/src/storage/ob_value_row_iterator.cpp +++ b/src/storage/ob_value_row_iterator.cpp @@ -261,7 +261,7 @@ int ObSingleRowGetter::open(const ObDatumRowkey &rowkey, bool use_fuse_row_cache } else { { ObStorageTableGuard guard(tablet_, *store_ctx_, false); - if (OB_FAIL(guard.refresh_and_protect_table(*relative_table_))) { + if (OB_FAIL(guard.refresh_and_protect_memtable_for_write(*relative_table_))) { STORAGE_LOG(WARN, "fail to protect table", K(ret)); } } diff --git a/src/storage/tablet/ob_tablet.cpp b/src/storage/tablet/ob_tablet.cpp index e7d885ce0..7fbcbf133 100644 --- a/src/storage/tablet/ob_tablet.cpp +++ b/src/storage/tablet/ob_tablet.cpp @@ -3656,7 +3656,7 @@ int ObTablet::lock_row( } else if (OB_UNLIKELY(relative_table.get_tablet_id() != tablet_meta_.tablet_id_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet id doesn't match", K(ret), K(relative_table.get_tablet_id()), K(tablet_meta_.tablet_id_)); - } else if (OB_FAIL(guard.refresh_and_protect_table(relative_table))) { + } else if (OB_FAIL(guard.refresh_and_protect_memtable_for_write(relative_table))) { LOG_WARN("fail to protect table", K(ret), "tablet_id", tablet_meta_.tablet_id_); } if (OB_SUCC(ret)) { @@ -3696,7 +3696,7 @@ int ObTablet::lock_row( } else if (OB_UNLIKELY(relative_table.get_tablet_id() != tablet_meta_.tablet_id_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet id doesn't match", K(ret), K(relative_table.get_tablet_id()), K(tablet_meta_.tablet_id_)); - } else if (OB_FAIL(guard.refresh_and_protect_table(relative_table))) { + } else if (OB_FAIL(guard.refresh_and_protect_memtable_for_write(relative_table))) { LOG_WARN("fail to protect table", K(ret)); } else { ObArenaAllocator allocator(common::ObMemAttr(MTL_ID(), ObModIds::OB_STORE_ROW_LOCK_CHECKER)); @@ -4127,7 +4127,7 @@ int ObTablet::update_row( } else if (OB_UNLIKELY(relative_table.get_tablet_id() != tablet_meta_.tablet_id_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet id doesn't match", K(ret), K(relative_table.get_tablet_id()), K(tablet_meta_.tablet_id_)); - } else if (OB_FAIL(guard.refresh_and_protect_table(relative_table))) { + } else if (OB_FAIL(guard.refresh_and_protect_memtable_for_write(relative_table))) { LOG_WARN("fail to protect table", K(ret)); } else if (OB_FAIL(prepare_memtable(relative_table, store_ctx, write_memtable))) { LOG_WARN("prepare write memtable fail", K(ret), K(relative_table)); @@ -4182,7 +4182,7 @@ int ObTablet::insert_rows( } else if (OB_UNLIKELY(relative_table.get_tablet_id() != tablet_meta_.tablet_id_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("Tablet id doesn't match", K(ret), K(relative_table.get_tablet_id()), K(tablet_meta_.tablet_id_)); - } else if (OB_FAIL(guard.refresh_and_protect_table(relative_table))) { + } else if (OB_FAIL(guard.refresh_and_protect_memtable_for_write(relative_table))) { LOG_WARN("Failed to protect table", K(ret)); } else if (OB_FAIL(prepare_memtable(relative_table, store_ctx, write_memtable))) { LOG_WARN("Failed to prepare write memtable", K(ret), K(relative_table)); @@ -4231,7 +4231,7 @@ int ObTablet::insert_row_without_rowkey_check( } else if (OB_UNLIKELY(relative_table.get_tablet_id() != tablet_meta_.tablet_id_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet id doesn't match", K(ret), K(relative_table.get_tablet_id()), K(tablet_meta_.tablet_id_)); - } else if (OB_FAIL(guard.refresh_and_protect_table(relative_table))) { + } else if (OB_FAIL(guard.refresh_and_protect_memtable_for_write(relative_table))) { LOG_WARN("fail to protect table", K(ret)); } else if (OB_FAIL(prepare_memtable(relative_table, store_ctx, write_memtable))) { LOG_WARN("prepare write memtable fail", K(ret), K(relative_table)); @@ -4393,7 +4393,7 @@ int ObTablet::rowkey_exists( } else { { ObStorageTableGuard guard(this, store_ctx, false); - if (OB_FAIL(guard.refresh_and_protect_table(relative_table))) { + if (OB_FAIL(guard.refresh_and_protect_memtable_for_write(relative_table))) { LOG_WARN("fail to protect table", K(ret)); } } @@ -4445,7 +4445,7 @@ int ObTablet::rowkeys_exists( } else { { ObStorageTableGuard guard(this, store_ctx, false); - if (OB_FAIL(guard.refresh_and_protect_table(relative_table))) { + if (OB_FAIL(guard.refresh_and_protect_memtable_for_write(relative_table))) { LOG_WARN("fail to protect table", K(ret)); } } @@ -4683,7 +4683,11 @@ int ObTablet::create_memtable( ObTimeGuard time_guard("ObTablet::create_memtable", 10 * 1000); common::SpinWLockGuard guard(memtables_lock_); time_guard.click("lock"); - const SCN new_clog_checkpoint_scn = clog_checkpoint_scn.is_min() ? tablet_meta_.clog_checkpoint_scn_ : clog_checkpoint_scn; + // we use the parameter clog_checkpoint_scn to double check whether the + // clog_checkpoint_scn has been changed during memtable replay check. + // So we complement the input_clog_checkpoint_scn for other scenario. + const SCN input_clog_checkpoint_scn = clog_checkpoint_scn.is_min() ? + tablet_meta_.clog_checkpoint_scn_ : clog_checkpoint_scn; if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -4692,11 +4696,14 @@ int ObTablet::create_memtable( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid schema version", K(ret), K(schema_version)); } else if (FALSE_IT(time_guard.click("prepare_memtables"))) { - } else if (OB_FAIL(inner_create_memtable(new_clog_checkpoint_scn, schema_version, for_direct_load, for_replay))) { + } else if (OB_FAIL(inner_create_memtable(input_clog_checkpoint_scn, + schema_version, + for_direct_load, + for_replay))) { if (OB_ENTRY_EXIST == ret) { ret = OB_SUCCESS; } else if (OB_MINOR_FREEZE_NOT_ALLOW != ret) { - LOG_WARN("failed to create memtable", K(ret), K(clog_checkpoint_scn), + LOG_WARN("failed to create memtable", K(ret), K(input_clog_checkpoint_scn), K(schema_version), K(for_replay)); } } else { @@ -4729,14 +4736,14 @@ int ObTablet::create_memtable( STORAGE_LOG(DEBUG, "Tablet finish create memtable", K(schema_version), - K(clog_checkpoint_scn), + K(input_clog_checkpoint_scn), K(for_replay), K(lbt())); return ret; } int ObTablet::inner_create_memtable( - const SCN clog_checkpoint_scn, + const SCN input_clog_checkpoint_scn, const int64_t schema_version, const bool for_direct_load, const bool for_replay) @@ -4744,12 +4751,12 @@ int ObTablet::inner_create_memtable( int ret = OB_SUCCESS; const share::ObLSID &ls_id = tablet_meta_.ls_id_; const common::ObTabletID &tablet_id = tablet_meta_.tablet_id_; - const SCN new_clog_checkpoint_scn = tablet_meta_.clog_checkpoint_scn_; + const SCN latest_clog_checkpoint_scn = tablet_meta_.clog_checkpoint_scn_; ObProtectedMemtableMgrHandle *protected_handle = NULL; - if (OB_UNLIKELY(!clog_checkpoint_scn.is_valid_and_not_min()) || OB_UNLIKELY(schema_version < 0)) { + if (OB_UNLIKELY(!input_clog_checkpoint_scn.is_valid_and_not_min()) || OB_UNLIKELY(schema_version < 0)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(clog_checkpoint_scn), K(schema_version)); + LOG_WARN("invalid args", K(ret), K(input_clog_checkpoint_scn), K(schema_version)); } else if (OB_UNLIKELY(MAX_MEMSTORE_CNT == memtable_count_)) { ret = OB_MINOR_FREEZE_NOT_ALLOW; if (TC_REACH_TIME_INTERVAL(1_s)) { @@ -4760,16 +4767,17 @@ int ObTablet::inner_create_memtable( LOG_WARN("failed to get_protected_memtable_mgr_handle", K(ret), KPC(this)); } else if (OB_FAIL(protected_handle->create_memtable(tablet_meta_, CreateMemtableArg(schema_version, - clog_checkpoint_scn, - new_clog_checkpoint_scn, + input_clog_checkpoint_scn, + latest_clog_checkpoint_scn, for_replay, for_direct_load)))) { if (OB_ENTRY_EXIST != ret && OB_MINOR_FREEZE_NOT_ALLOW != ret) { LOG_WARN("failed to create memtable", K(ret), K(ls_id), K(tablet_id), KPC(this)); } } else { - LOG_INFO("succeeded to create memtable for tablet", K(ret), K(ls_id), K(tablet_id), - K(clog_checkpoint_scn), K(schema_version), K(for_replay)); + LOG_INFO("succeeded to create memtable for tablet", K(ret), K(tablet_id), + K(ls_id), K(schema_version), K(for_replay), K(for_direct_load), + K(input_clog_checkpoint_scn), K(latest_clog_checkpoint_scn)); } return ret; diff --git a/src/storage/tablet/ob_tablet_memtable_mgr.cpp b/src/storage/tablet/ob_tablet_memtable_mgr.cpp index 6041e6d3e..1fd1dafd2 100644 --- a/src/storage/tablet/ob_tablet_memtable_mgr.cpp +++ b/src/storage/tablet/ob_tablet_memtable_mgr.cpp @@ -189,12 +189,13 @@ int ObTabletMemtableMgr::try_resolve_boundary_on_create_memtable_for_leader_( } else if (can_resolve) { SCN new_start_scn; last_frozen_tablet_memtable->resolve_right_boundary(); - last_frozen_tablet_memtable->set_resolved_active_memtable_left_boundary(); if (new_tablet_memtable != last_frozen_tablet_memtable) { new_start_scn = MAX(last_frozen_tablet_memtable->get_end_scn(), - last_frozen_tablet_memtable->get_migration_clog_checkpoint_scn()); + new_tablet_memtable->get_start_scn()); if (OB_FAIL(new_tablet_memtable->resolve_left_boundary(new_start_scn))) { TRANS_LOG(ERROR, "resolve left boundary failed", KR(ret), K(new_start_scn), KPC(new_tablet_memtable)); + } else { + last_frozen_tablet_memtable->set_resolved_active_memtable_left_boundary(); } } TRANS_LOG(INFO, "[resolve_right_boundary] in create_memtable on leader", K(ret), @@ -271,9 +272,10 @@ int ObTabletMemtableMgr::check_boundary_memtable_(const uint32_t logstream_freez } else if (tablet_memtable->is_direct_load_memtable()) { if (tablet_memtable->get_end_scn().is_max()) { PAUSE(); - // if end_scn of direct load memtable has not decided, return OB_ENTRY_EXIST and Tablet will reset it to - // OB_SUCCESS. Then refresh_and_protect_table(refresh_and_protect_memtable) in StorageTableGuard will retry create - // memtable + // if end_scn of direct load memtable has not decided, return + // OB_ENTRY_EXIST and Tablet will reset it to OB_SUCCESS. Then + // refresh_and_protect_memtable_for_write(refresh_and_protect_memtable_for_replay) + // in StorageTableGuard will retry create memtable ret = OB_ENTRY_EXIST; } } else if (tablet_memtable->is_data_memtable()) { @@ -389,19 +391,19 @@ int ObTabletMemtableMgr::resolve_data_memtable_boundary_(ObITabletMemtable *froz { int ret = OB_SUCCESS; if (arg.for_replay_) { - SCN new_memtable_start_scn; frozen_tablet_memtable->resolve_right_boundary(); if (new_tablet_memtable != frozen_tablet_memtable) { - new_memtable_start_scn = - MAX(frozen_tablet_memtable->get_end_scn(), frozen_tablet_memtable->get_migration_clog_checkpoint_scn()); - if (OB_FAIL(new_tablet_memtable->resolve_left_boundary(new_memtable_start_scn))) { - TRANS_LOG(WARN, "resolve left boundary fail", K(ret), K(new_tablet_memtable)); + if (OB_FAIL(new_tablet_memtable->resolve_left_boundary( + MAX(frozen_tablet_memtable->get_end_scn(), + new_tablet_memtable->get_start_scn())))) { + TRANS_LOG(WARN, "resolve left boundary fail", K(ret), K(arg), + KPC(new_tablet_memtable), KPC(frozen_tablet_memtable)); } } - TRANS_LOG(INFO, "[resolve_right_boundary] in create_memtable on replay", KPC(frozen_tablet_memtable)); - } - // for leader, decide the right boundary of frozen memtable - else if (OB_FAIL(try_resolve_boundary_on_create_memtable_for_leader_(frozen_tablet_memtable, new_tablet_memtable))) { + TRANS_LOG(INFO, "[resolve_right_boundary] in create_memtable on replay", + KPC(frozen_tablet_memtable), KPC(new_tablet_memtable), K(arg)); + } else if (OB_FAIL(try_resolve_boundary_on_create_memtable_for_leader_( + frozen_tablet_memtable, new_tablet_memtable))) { TRANS_LOG(WARN, "try resolve boundary fail", K(ret)); } return ret; @@ -420,7 +422,7 @@ int ObTabletMemtableMgr::resolve_direct_load_memtable_boundary_(ObITabletMemtabl STORAGE_LOG(ERROR, "frozen direct load memtable must have a valid end_scn", KPC(frozen_tablet_memtable)); } else if (active_tablet_memtable != frozen_tablet_memtable) { new_memtable_start_scn = - MAX(frozen_tablet_memtable->get_end_scn(), frozen_tablet_memtable->get_migration_clog_checkpoint_scn()); + MAX(frozen_tablet_memtable->get_end_scn(), active_tablet_memtable->get_start_scn()); if (OB_FAIL(active_tablet_memtable->resolve_left_boundary(new_memtable_start_scn))) { STORAGE_LOG(ERROR, "fail to resolve left boundary", KPC(active_tablet_memtable)); } else { @@ -622,8 +624,8 @@ int ObTabletMemtableMgr::resolve_left_boundary_for_active_memtable(ObITabletMemt } else if (OB_FAIL(handle.get_tablet_memtable(active_tablet_memtable))) { LOG_WARN("fail to get active memtable", K(ret)); } else { - // set the start_scn of the new memtable - int tmp_ret = active_tablet_memtable->resolve_left_boundary(start_scn); + int tmp_ret = active_tablet_memtable->resolve_left_boundary( + MAX(start_scn, active_tablet_memtable->get_start_scn())); if (OB_SUCCESS != tmp_ret) { TRANS_LOG(ERROR, "resolve left boundary failed", K(start_scn), KPC(active_tablet_memtable), KPC(tablet_memtable)); } diff --git a/src/storage/tx/ob_tx_replay_executor.cpp b/src/storage/tx/ob_tx_replay_executor.cpp index bdf79986c..91f0b0ede 100644 --- a/src/storage/tx/ob_tx_replay_executor.cpp +++ b/src/storage/tx/ob_tx_replay_executor.cpp @@ -783,7 +783,7 @@ int ObTxReplayExecutor::prepare_memtable_replay_(ObStorageTableGuard &w_guard, ObIMemtable *&mem_ptr) { int ret = OB_SUCCESS; - if (OB_FAIL(w_guard.refresh_and_protect_memtable())) { + if (OB_FAIL(w_guard.refresh_and_protect_memtable_for_replay())) { TRANS_LOG(WARN, "[Replay Tx] refresh and protect memtable error", K(ret)); } else if (OB_FAIL(w_guard.get_memtable_for_replay(mem_ptr))) { // OB_NO_NEED_UPDATE => don't need to replay From 2509b30374749f613f2f8791547857c3b73f450d Mon Sep 17 00:00:00 2001 From: hy-guo Date: Mon, 26 Aug 2024 06:50:57 +0000 Subject: [PATCH 218/249] add materialized view on query computation err code --- src/share/ob_errno.cpp | 17 ++++++++++++++++- src/share/ob_errno.def | 1 + src/share/ob_errno.h | 6 +++++- .../resolver/ddl/ob_create_view_resolver.cpp | 5 +++++ 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/share/ob_errno.cpp b/src/share/ob_errno.cpp index 3149f5b6e..209327789 100644 --- a/src/share/ob_errno.cpp +++ b/src/share/ob_errno.cpp @@ -31513,6 +31513,20 @@ static const _error _error_OB_ERR_INVALID_CHARACTER = { .ob_str_error = "OBE-00911: invalid character", .ob_str_user_error = "OBE-00911: invalid character" }; +static const _error _error_OB_ERR_MVIEW_CAN_NOT_ON_QUERY_COMPUTE = { + .error_name = "OB_ERR_MVIEW_CAN_NOT_ON_QUERY_COMPUTE", + .error_cause = "Internal Error", + .error_solution = "Contact OceanBase Support", + .mysql_errno = -1, + .sqlstate = "HY000", + .str_error = "cannot ENABLE ON QUERY COMPUTATION for the materialized view", + .str_user_error = "cannot ENABLE ON QUERY COMPUTATION for the materialized view `%s`.`%s`", + .oracle_errno = 32361, + .oracle_str_error = "ORA-32361: cannot ENABLE ON QUERY COMPUTATION for the materialized view", + .oracle_str_user_error = "ORA-32361: cannot ENABLE ON QUERY COMPUTATION for the materialized view %s.%s", + .ob_str_error = "OBE-32361: cannot ENABLE ON QUERY COMPUTATION for the materialized view", + .ob_str_user_error = "OBE-32361: cannot ENABLE ON QUERY COMPUTATION for the materialized view %s.%s" +}; static const _error _error_OB_ERR_KV_GLOBAL_INDEX_ROUTE = { .error_name = "OB_ERR_KV_GLOBAL_INDEX_ROUTE", .error_cause = "Internal Error", @@ -34861,6 +34875,7 @@ struct ObStrErrorInit _errors[-OB_ERR_EVENT_RECURSION_FORBIDDEN] = &_error_OB_ERR_EVENT_RECURSION_FORBIDDEN; _errors[-OB_NO_PARTITION_FOR_GIVEN_VALUE_SCHEMA_ERROR] = &_error_OB_NO_PARTITION_FOR_GIVEN_VALUE_SCHEMA_ERROR; _errors[-OB_ERR_INVALID_CHARACTER] = &_error_OB_ERR_INVALID_CHARACTER; + _errors[-OB_ERR_MVIEW_CAN_NOT_ON_QUERY_COMPUTE] = &_error_OB_ERR_MVIEW_CAN_NOT_ON_QUERY_COMPUTE; _errors[-OB_ERR_KV_GLOBAL_INDEX_ROUTE] = &_error_OB_ERR_KV_GLOBAL_INDEX_ROUTE; _errors[-OB_TTL_NOT_ENABLE] = &_error_OB_TTL_NOT_ENABLE; _errors[-OB_TTL_COLUMN_NOT_EXIST] = &_error_OB_TTL_COLUMN_NOT_EXIST; @@ -34973,7 +34988,7 @@ namespace oceanbase { namespace common { -int g_all_ob_errnos[2326] = {0, -4000, -4001, -4002, -4003, -4004, -4005, -4006, -4007, -4008, -4009, -4010, -4011, -4012, -4013, -4014, -4015, -4016, -4017, -4018, -4019, -4020, -4021, -4022, -4023, -4024, -4025, -4026, -4027, -4028, -4029, -4030, -4031, -4032, -4033, -4034, -4035, -4036, -4037, -4038, -4039, -4041, -4042, -4043, -4044, -4045, -4046, -4047, -4048, -4049, -4050, -4051, -4052, -4053, -4054, -4055, -4057, -4058, -4060, -4061, -4062, -4063, -4064, -4065, -4066, -4067, -4068, -4070, -4071, -4072, -4073, -4074, -4075, -4076, -4077, -4078, -4080, -4081, -4084, -4085, -4090, -4097, -4098, -4099, -4100, -4101, -4102, -4103, -4104, -4105, -4106, -4107, -4108, -4109, -4110, -4111, -4112, -4113, -4114, -4115, -4116, -4117, -4118, -4119, -4120, -4121, -4122, -4123, -4124, -4125, -4126, -4127, -4128, -4133, -4138, -4139, -4142, -4143, -4144, -4146, -4147, -4149, -4150, -4151, -4152, -4153, -4154, -4155, -4156, -4157, -4158, -4159, -4160, -4161, -4162, -4163, -4164, -4165, -4166, -4167, -4168, -4169, -4170, -4171, -4172, -4173, -4174, -4175, -4176, -4177, -4178, -4179, -4180, -4181, -4182, -4183, -4184, -4185, -4186, -4187, -4188, -4189, -4190, -4191, -4192, -4200, -4201, -4204, -4205, -4206, -4207, -4208, -4209, -4210, -4211, -4212, -4213, -4214, -4215, -4216, -4217, -4218, -4219, -4220, -4221, -4222, -4223, -4224, -4225, -4226, -4227, -4228, -4229, -4230, -4231, -4232, -4233, -4234, -4235, -4236, -4237, -4238, -4239, -4240, -4241, -4242, -4243, -4244, -4245, -4246, -4247, -4248, -4249, -4250, -4251, -4252, -4253, -4254, -4255, -4256, -4257, -4258, -4260, -4261, -4262, -4263, -4264, -4265, -4266, -4267, -4268, -4269, -4270, -4271, -4273, -4274, -4275, -4276, -4277, -4278, -4279, -4280, -4281, -4282, -4283, -4284, -4285, -4286, -4287, -4288, -4289, -4290, -4291, -4292, -4293, -4294, -4295, -4296, -4297, -4298, -4299, -4300, -4301, -4302, -4303, -4304, -4305, -4306, -4307, -4308, -4309, -4310, -4311, -4312, -4313, -4314, -4315, -4316, -4317, -4318, -4319, -4320, -4321, -4322, -4323, -4324, -4325, -4326, -4327, -4328, -4329, -4330, -4331, -4332, -4333, -4334, -4335, -4336, -4337, -4338, -4339, -4340, -4341, -4342, -4343, -4344, -4345, -4346, -4347, -4348, -4349, -4350, -4351, -4352, -4353, -4354, -4355, -4356, -4357, -4358, -4359, -4360, -4361, -4362, -4363, -4364, -4365, -4366, -4367, -4368, -4369, -4370, -4371, -4372, -4373, -4374, -4375, -4376, -4377, -4378, -4379, -4380, -4381, -4382, -4383, -4385, -4386, -4387, -4388, -4389, -4390, -4391, -4392, -4393, -4394, -4395, -4396, -4397, -4398, -4399, -4400, -4401, -4402, -4403, -4505, -4507, -4510, -4512, -4515, -4517, -4518, -4519, -4523, -4524, -4525, -4526, -4527, -4528, -4529, -4530, -4531, -4532, -4533, -4537, -4538, -4539, -4540, -4541, -4542, -4543, -4544, -4545, -4546, -4547, -4548, -4549, -4550, -4551, -4552, -4553, -4554, -4600, -4601, -4602, -4603, -4604, -4605, -4606, -4607, -4608, -4609, -4610, -4611, -4613, -4614, -4615, -4620, -4621, -4622, -4623, -4624, -4625, -4626, -4628, -4629, -4630, -4631, -4632, -4633, -4634, -4636, -4637, -4638, -4639, -4640, -4641, -4642, -4643, -4644, -4645, -4646, -4647, -4648, -4649, -4650, -4651, -4652, -4653, -4654, -4655, -4656, -4657, -4658, -4659, -4660, -4661, -4662, -4663, -4664, -4665, -4666, -4667, -4668, -4669, -4670, -4671, -4672, -4673, -4674, -4675, -4676, -4677, -4678, -4679, -4680, -4681, -4682, -4683, -4684, -4685, -4686, -4687, -4688, -4689, -4690, -4691, -4692, -4693, -4694, -4695, -4696, -4697, -4698, -4699, -4700, -4701, -4702, -4703, -4704, -4705, -4706, -4707, -4708, -4709, -4710, -4711, -4712, -4713, -4714, -4715, -4716, -4717, -4718, -4719, -4720, -4721, -4722, -4723, -4724, -4725, -4726, -4727, -4728, -4729, -4730, -4731, -4732, -4733, -4734, -4735, -4736, -4737, -4738, -4739, -4740, -4741, -4742, -4743, -4744, -4745, -4746, -4747, -4748, -4749, -4750, -4751, -4752, -4753, -4754, -4755, -4756, -4757, -4758, -4759, -4760, -4761, -4762, -4763, -4764, -4765, -4766, -4767, -4768, -4769, -4770, -4771, -4772, -4773, -4774, -4775, -4776, -4777, -4778, -4779, -4780, -4781, -4782, -4783, -4784, -5000, -5001, -5002, -5003, -5006, -5007, -5008, -5010, -5011, -5012, -5014, -5015, -5016, -5017, -5018, -5019, -5020, -5022, -5023, -5024, -5025, -5026, -5027, -5028, -5029, -5030, -5031, -5032, -5034, -5035, -5036, -5037, -5038, -5039, -5040, -5041, -5042, -5043, -5044, -5046, -5047, -5050, -5051, -5052, -5053, -5054, -5055, -5056, -5057, -5058, -5059, -5061, -5063, -5064, -5065, -5066, -5067, -5068, -5069, -5070, -5071, -5072, -5073, -5074, -5080, -5081, -5083, -5084, -5085, -5086, -5087, -5088, -5089, -5090, -5091, -5092, -5093, -5094, -5095, -5096, -5097, -5098, -5099, -5100, -5101, -5102, -5103, -5104, -5105, -5106, -5107, -5108, -5109, -5110, -5111, -5112, -5113, -5114, -5115, -5116, -5117, -5118, -5119, -5120, -5121, -5122, -5123, -5124, -5125, -5130, -5131, -5133, -5134, -5135, -5136, -5137, -5138, -5139, -5140, -5142, -5143, -5144, -5145, -5146, -5147, -5148, -5149, -5150, -5151, -5153, -5154, -5155, -5156, -5157, -5158, -5159, -5160, -5161, -5162, -5163, -5164, -5165, -5166, -5167, -5168, -5169, -5170, -5171, -5172, -5173, -5174, -5175, -5176, -5177, -5178, -5179, -5180, -5181, -5182, -5183, -5184, -5185, -5187, -5188, -5189, -5190, -5191, -5192, -5193, -5194, -5195, -5196, -5197, -5198, -5199, -5200, -5201, -5202, -5203, -5204, -5205, -5206, -5207, -5208, -5209, -5210, -5211, -5212, -5213, -5214, -5215, -5216, -5217, -5218, -5219, -5220, -5221, -5222, -5223, -5224, -5225, -5226, -5227, -5228, -5229, -5230, -5231, -5233, -5234, -5235, -5236, -5237, -5238, -5239, -5240, -5241, -5242, -5243, -5244, -5245, -5246, -5247, -5248, -5249, -5250, -5251, -5252, -5253, -5254, -5255, -5256, -5257, -5258, -5259, -5260, -5261, -5262, -5263, -5264, -5265, -5266, -5267, -5268, -5269, -5270, -5271, -5272, -5273, -5274, -5275, -5276, -5277, -5278, -5279, -5280, -5281, -5282, -5283, -5284, -5285, -5286, -5287, -5288, -5289, -5290, -5291, -5292, -5293, -5294, -5295, -5296, -5297, -5298, -5299, -5300, -5301, -5302, -5303, -5304, -5305, -5306, -5307, -5308, -5309, -5310, -5311, -5312, -5313, -5314, -5315, -5316, -5317, -5318, -5319, -5320, -5321, -5322, -5323, -5324, -5325, -5326, -5327, -5328, -5329, -5330, -5331, -5332, -5333, -5334, -5335, -5336, -5337, -5338, -5339, -5340, -5341, -5342, -5343, -5344, -5345, -5346, -5347, -5348, -5349, -5350, -5351, -5352, -5353, -5354, -5355, -5356, -5357, -5358, -5359, -5360, -5361, -5362, -5363, -5364, -5365, -5366, -5367, -5368, -5369, -5370, -5371, -5372, -5373, -5374, -5375, -5376, -5377, -5378, -5379, -5380, -5381, -5382, -5383, -5384, -5385, -5386, -5387, -5388, -5389, -5390, -5400, -5401, -5402, -5403, -5404, -5405, -5406, -5407, -5408, -5409, -5410, -5411, -5412, -5413, -5414, -5415, -5416, -5417, -5418, -5419, -5420, -5421, -5422, -5423, -5424, -5425, -5426, -5427, -5428, -5429, -5430, -5431, -5432, -5433, -5434, -5435, -5436, -5437, -5438, -5439, -5440, -5441, -5442, -5443, -5444, -5445, -5446, -5447, -5448, -5449, -5450, -5451, -5452, -5453, -5454, -5455, -5456, -5457, -5458, -5459, -5460, -5461, -5462, -5463, -5464, -5465, -5466, -5467, -5468, -5469, -5470, -5471, -5472, -5473, -5474, -5475, -5476, -5477, -5478, -5479, -5480, -5481, -5482, -5483, -5484, -5485, -5486, -5487, -5488, -5489, -5490, -5491, -5492, -5493, -5494, -5495, -5496, -5497, -5498, -5499, -5500, -5501, -5502, -5503, -5504, -5505, -5506, -5507, -5508, -5509, -5510, -5511, -5512, -5513, -5514, -5515, -5516, -5517, -5518, -5519, -5520, -5521, -5522, -5540, -5541, -5542, -5543, -5544, -5545, -5546, -5547, -5548, -5549, -5550, -5551, -5552, -5553, -5554, -5555, -5556, -5557, -5558, -5559, -5560, -5561, -5562, -5563, -5564, -5565, -5566, -5567, -5568, -5569, -5570, -5571, -5572, -5573, -5574, -5575, -5576, -5577, -5578, -5579, -5580, -5581, -5582, -5583, -5584, -5585, -5586, -5587, -5588, -5589, -5590, -5591, -5592, -5593, -5594, -5595, -5596, -5597, -5598, -5599, -5600, -5601, -5602, -5603, -5604, -5605, -5607, -5608, -5609, -5610, -5611, -5612, -5613, -5614, -5615, -5616, -5617, -5618, -5619, -5620, -5621, -5622, -5623, -5624, -5625, -5626, -5627, -5628, -5629, -5630, -5631, -5632, -5633, -5634, -5635, -5636, -5637, -5638, -5639, -5640, -5641, -5642, -5643, -5644, -5645, -5646, -5647, -5648, -5649, -5650, -5651, -5652, -5653, -5654, -5655, -5656, -5657, -5658, -5659, -5660, -5661, -5662, -5663, -5664, -5665, -5666, -5667, -5668, -5671, -5672, -5673, -5674, -5675, -5676, -5677, -5678, -5679, -5680, -5681, -5682, -5683, -5684, -5685, -5686, -5687, -5688, -5689, -5690, -5691, -5692, -5693, -5694, -5695, -5696, -5697, -5698, -5699, -5700, -5701, -5702, -5703, -5704, -5705, -5706, -5707, -5708, -5709, -5710, -5711, -5712, -5713, -5714, -5715, -5716, -5717, -5718, -5719, -5720, -5721, -5722, -5723, -5724, -5725, -5726, -5727, -5728, -5729, -5730, -5731, -5732, -5733, -5734, -5735, -5736, -5737, -5738, -5739, -5740, -5741, -5742, -5743, -5744, -5745, -5746, -5747, -5748, -5749, -5750, -5751, -5752, -5753, -5754, -5755, -5756, -5757, -5758, -5759, -5760, -5761, -5762, -5763, -5764, -5765, -5766, -5768, -5769, -5770, -5771, -5772, -5773, -5774, -5777, -5778, -5779, -5780, -5781, -5785, -5786, -5787, -5788, -5789, -5790, -5791, -5792, -5793, -5794, -5795, -5796, -5797, -5798, -5799, -5800, -5801, -5802, -5803, -5804, -5805, -5806, -5807, -5808, -5809, -5810, -5811, -5812, -5813, -5814, -5815, -5816, -5817, -5818, -5819, -5820, -5821, -5822, -5823, -5824, -5825, -5826, -5827, -5828, -5829, -5830, -5831, -5832, -5833, -5834, -5835, -5836, -5837, -5838, -5839, -5840, -5841, -5842, -5843, -5844, -5845, -5846, -5847, -5848, -5849, -5850, -5851, -5852, -5853, -5854, -5855, -5856, -5857, -5858, -5859, -5860, -5861, -5862, -5863, -5864, -5865, -5866, -5867, -5868, -5869, -5870, -5871, -5872, -5873, -5874, -5875, -5876, -5877, -5878, -5879, -5880, -5881, -5882, -5883, -5884, -5885, -5886, -5887, -5888, -5889, -5890, -5891, -5892, -5893, -5894, -5895, -5896, -5897, -5898, -5899, -5900, -5901, -5902, -5903, -5904, -5905, -5906, -5907, -5908, -5909, -5910, -5911, -5912, -5913, -5914, -5915, -5916, -5917, -5918, -5919, -5920, -5921, -5922, -5923, -5924, -5925, -5926, -5927, -5928, -5929, -5930, -5931, -5932, -5933, -5934, -5935, -5936, -5937, -5938, -5939, -5940, -5941, -5942, -5943, -5944, -5945, -5946, -5947, -5948, -5949, -5950, -5951, -5952, -5953, -5954, -5955, -5956, -5957, -5958, -5959, -5960, -5961, -5962, -5963, -5964, -5965, -5966, -5967, -5968, -5969, -5970, -5971, -5972, -5973, -5974, -5975, -5976, -5977, -5978, -5979, -5980, -5981, -5982, -5983, -5984, -5985, -5986, -5987, -5988, -5989, -5990, -5991, -5992, -5993, -5994, -5995, -5996, -5997, -5998, -5999, -6000, -6001, -6002, -6003, -6004, -6005, -6006, -6201, -6202, -6203, -6204, -6205, -6206, -6207, -6208, -6209, -6210, -6211, -6212, -6213, -6214, -6215, -6219, -6220, -6221, -6222, -6223, -6224, -6225, -6226, -6227, -6228, -6229, -6230, -6231, -6232, -6233, -6234, -6235, -6236, -6237, -6238, -6239, -6240, -6241, -6242, -6243, -6244, -6245, -6246, -6247, -6248, -6249, -6250, -6251, -6252, -6253, -6254, -6255, -6256, -6257, -6258, -6259, -6260, -6261, -6262, -6263, -6264, -6265, -6266, -6267, -6268, -6269, -6270, -6271, -6272, -6273, -6274, -6275, -6276, -6277, -6278, -6279, -6280, -6281, -6282, -6283, -6284, -6285, -6301, -6302, -6303, -6304, -6305, -6306, -6307, -6308, -6309, -6310, -6311, -6312, -6313, -6314, -6315, -6316, -6317, -6318, -6319, -6320, -6321, -6322, -6323, -6324, -6325, -6326, -6327, -6328, -6329, -6330, -6331, -6332, -7000, -7001, -7002, -7003, -7004, -7005, -7006, -7007, -7010, -7011, -7012, -7013, -7014, -7015, -7021, -7022, -7024, -7025, -7026, -7027, -7029, -7030, -7031, -7032, -7033, -7034, -7035, -7036, -7037, -7038, -7039, -7040, -7041, -7100, -7101, -7102, -7103, -7104, -7105, -7106, -7107, -7108, -7109, -7110, -7111, -7112, -7113, -7114, -7115, -7116, -7117, -7118, -7119, -7120, -7121, -7122, -7123, -7124, -7201, -7202, -7203, -7204, -7205, -7206, -7207, -7208, -7209, -7210, -7211, -7212, -7213, -7214, -7215, -7216, -7217, -7218, -7219, -7220, -7221, -7222, -7223, -7224, -7225, -7226, -7227, -7228, -7229, -7230, -7231, -7232, -7233, -7234, -7235, -7236, -7237, -7238, -7239, -7240, -7241, -7242, -7243, -7244, -7246, -7247, -7248, -7249, -7250, -7251, -7252, -7253, -7254, -7255, -7256, -7257, -7258, -7259, -7260, -7261, -7262, -7263, -7264, -7265, -7266, -7267, -7268, -7269, -7270, -7271, -7272, -7273, -7274, -7275, -7276, -7277, -7278, -7279, -7280, -7281, -7282, -7283, -7284, -7285, -7286, -7287, -7288, -7289, -7290, -7291, -7292, -7293, -7294, -7295, -7296, -7297, -7298, -7299, -7300, -7301, -7302, -7402, -7403, -7404, -7405, -7406, -7407, -7408, -7409, -7410, -7411, -7412, -7413, -7414, -7415, -7416, -7417, -7418, -7419, -7420, -7421, -7422, -7423, -7424, -7425, -7426, -7427, -7428, -7429, -7430, -7431, -7432, -7433, -7434, -7435, -7600, -7601, -7602, -7603, -7604, -8001, -8002, -8003, -8004, -8005, -9001, -9002, -9003, -9004, -9005, -9006, -9007, -9008, -9009, -9010, -9011, -9012, -9013, -9014, -9015, -9016, -9017, -9018, -9019, -9020, -9022, -9023, -9024, -9025, -9026, -9027, -9028, -9029, -9030, -9031, -9032, -9033, -9034, -9035, -9036, -9037, -9038, -9039, -9040, -9041, -9042, -9043, -9044, -9045, -9046, -9047, -9048, -9049, -9050, -9051, -9052, -9053, -9054, -9057, -9058, -9059, -9060, -9061, -9062, -9063, -9064, -9065, -9066, -9069, -9070, -9071, -9072, -9073, -9074, -9075, -9076, -9077, -9078, -9079, -9080, -9081, -9082, -9083, -9084, -9085, -9086, -9087, -9088, -9089, -9090, -9091, -9092, -9093, -9094, -9095, -9096, -9097, -9098, -9099, -9100, -9101, -9102, -9103, -9104, -9105, -9106, -9107, -9108, -9109, -9110, -9111, -9112, -9113, -9114, -9115, -9116, -9117, -9118, -9119, -9120, -9121, -9122, -9123, -9124, -9125, -9126, -9127, -9200, -9201, -9202, -9203, -9501, -9502, -9503, -9504, -9505, -9506, -9507, -9508, -9509, -9510, -9512, -9513, -9514, -9515, -9516, -9518, -9519, -9520, -9521, -9522, -9523, -9524, -9525, -9526, -9527, -9528, -9529, -9530, -9531, -9532, -9533, -9534, -9535, -9536, -9537, -9538, -9539, -9540, -9541, -9542, -9543, -9544, -9545, -9546, -9547, -9548, -9549, -9550, -9551, -9552, -9553, -9554, -9555, -9556, -9557, -9558, -9559, -9560, -9561, -9562, -9563, -9564, -9565, -9566, -9567, -9568, -9569, -9570, -9571, -9572, -9573, -9574, -9575, -9576, -9577, -9578, -9579, -9580, -9581, -9582, -9583, -9584, -9585, -9586, -9587, -9588, -9589, -9590, -9591, -9592, -9593, -9594, -9595, -9596, -9597, -9598, -9599, -9600, -9601, -9602, -9603, -9604, -9605, -9606, -9607, -9608, -9609, -9610, -9611, -9612, -9613, -9614, -9615, -9616, -9617, -9618, -9619, -9620, -9621, -9622, -9623, -9624, -9625, -9626, -9627, -9628, -9629, -9630, -9631, -9632, -9633, -9634, -9635, -9636, -9637, -9638, -9639, -9640, -9641, -9642, -9643, -9644, -9645, -9646, -9647, -9648, -9649, -9650, -9651, -9652, -9653, -9654, -9655, -9656, -9657, -9658, -9659, -9660, -9661, -9662, -9663, -9664, -9665, -9666, -9667, -9668, -9669, -9670, -9671, -9672, -9673, -9674, -9675, -9676, -9677, -9678, -9679, -9680, -9681, -9682, -9683, -9684, -9685, -9686, -9687, -9688, -9689, -9690, -9691, -9692, -9693, -9694, -9695, -9696, -9697, -9698, -9699, -9700, -9701, -9702, -9703, -9704, -9705, -9706, -9707, -9708, -9709, -9710, -9711, -9712, -9713, -9714, -9715, -9716, -9717, -9718, -9719, -9720, -9721, -9722, -9723, -9724, -9725, -9726, -9727, -9728, -9729, -9730, -9731, -9732, -9733, -9734, -9735, -9736, -9737, -9738, -9739, -9740, -9741, -9742, -9743, -9744, -9745, -9746, -9747, -9748, -9749, -9750, -9751, -9752, -9753, -9754, -9755, -9756, -9757, -9758, -9759, -9760, -9761, -9762, -9763, -9764, -9765, -9766, -9767, -9768, -9769, -9770, -9771, -9772, -9773, -9774, -9775, -9776, -9777, -9778, -9779, -9780, -9781, -9782, -10500, -10501, -10502, -10503, -10504, -10505, -10506, -10507, -10508, -10509, -10510, -10511, -10512, -10513, -10514, -10515, -10516, -10650, -11000, -11001, -11002, -11003, -11004, -11005, -11006, -11007, -11008, -11009, -11010, -11011, -11012, -11013, -11014, -11015, -11016, -11017, -11018, -11019, -11020, -11021, -11022, -11023, -11024, -11025, -11026, -11027, -11028, -11029, -11030, -11031, -11032, -11033, -11034, -11035, -11036, -11037, -11038, -11039, -11040, -11041, -11042, -11043, -11044, -11045, -11046, -11047, -11048, -11049, -11050, -11051, -11052, -20000, -21000, -22998, -30926, -32491, -38104, -38105}; +int g_all_ob_errnos[2327] = {0, -4000, -4001, -4002, -4003, -4004, -4005, -4006, -4007, -4008, -4009, -4010, -4011, -4012, -4013, -4014, -4015, -4016, -4017, -4018, -4019, -4020, -4021, -4022, -4023, -4024, -4025, -4026, -4027, -4028, -4029, -4030, -4031, -4032, -4033, -4034, -4035, -4036, -4037, -4038, -4039, -4041, -4042, -4043, -4044, -4045, -4046, -4047, -4048, -4049, -4050, -4051, -4052, -4053, -4054, -4055, -4057, -4058, -4060, -4061, -4062, -4063, -4064, -4065, -4066, -4067, -4068, -4070, -4071, -4072, -4073, -4074, -4075, -4076, -4077, -4078, -4080, -4081, -4084, -4085, -4090, -4097, -4098, -4099, -4100, -4101, -4102, -4103, -4104, -4105, -4106, -4107, -4108, -4109, -4110, -4111, -4112, -4113, -4114, -4115, -4116, -4117, -4118, -4119, -4120, -4121, -4122, -4123, -4124, -4125, -4126, -4127, -4128, -4133, -4138, -4139, -4142, -4143, -4144, -4146, -4147, -4149, -4150, -4151, -4152, -4153, -4154, -4155, -4156, -4157, -4158, -4159, -4160, -4161, -4162, -4163, -4164, -4165, -4166, -4167, -4168, -4169, -4170, -4171, -4172, -4173, -4174, -4175, -4176, -4177, -4178, -4179, -4180, -4181, -4182, -4183, -4184, -4185, -4186, -4187, -4188, -4189, -4190, -4191, -4192, -4200, -4201, -4204, -4205, -4206, -4207, -4208, -4209, -4210, -4211, -4212, -4213, -4214, -4215, -4216, -4217, -4218, -4219, -4220, -4221, -4222, -4223, -4224, -4225, -4226, -4227, -4228, -4229, -4230, -4231, -4232, -4233, -4234, -4235, -4236, -4237, -4238, -4239, -4240, -4241, -4242, -4243, -4244, -4245, -4246, -4247, -4248, -4249, -4250, -4251, -4252, -4253, -4254, -4255, -4256, -4257, -4258, -4260, -4261, -4262, -4263, -4264, -4265, -4266, -4267, -4268, -4269, -4270, -4271, -4273, -4274, -4275, -4276, -4277, -4278, -4279, -4280, -4281, -4282, -4283, -4284, -4285, -4286, -4287, -4288, -4289, -4290, -4291, -4292, -4293, -4294, -4295, -4296, -4297, -4298, -4299, -4300, -4301, -4302, -4303, -4304, -4305, -4306, -4307, -4308, -4309, -4310, -4311, -4312, -4313, -4314, -4315, -4316, -4317, -4318, -4319, -4320, -4321, -4322, -4323, -4324, -4325, -4326, -4327, -4328, -4329, -4330, -4331, -4332, -4333, -4334, -4335, -4336, -4337, -4338, -4339, -4340, -4341, -4342, -4343, -4344, -4345, -4346, -4347, -4348, -4349, -4350, -4351, -4352, -4353, -4354, -4355, -4356, -4357, -4358, -4359, -4360, -4361, -4362, -4363, -4364, -4365, -4366, -4367, -4368, -4369, -4370, -4371, -4372, -4373, -4374, -4375, -4376, -4377, -4378, -4379, -4380, -4381, -4382, -4383, -4385, -4386, -4387, -4388, -4389, -4390, -4391, -4392, -4393, -4394, -4395, -4396, -4397, -4398, -4399, -4400, -4401, -4402, -4403, -4505, -4507, -4510, -4512, -4515, -4517, -4518, -4519, -4523, -4524, -4525, -4526, -4527, -4528, -4529, -4530, -4531, -4532, -4533, -4537, -4538, -4539, -4540, -4541, -4542, -4543, -4544, -4545, -4546, -4547, -4548, -4549, -4550, -4551, -4552, -4553, -4554, -4600, -4601, -4602, -4603, -4604, -4605, -4606, -4607, -4608, -4609, -4610, -4611, -4613, -4614, -4615, -4620, -4621, -4622, -4623, -4624, -4625, -4626, -4628, -4629, -4630, -4631, -4632, -4633, -4634, -4636, -4637, -4638, -4639, -4640, -4641, -4642, -4643, -4644, -4645, -4646, -4647, -4648, -4649, -4650, -4651, -4652, -4653, -4654, -4655, -4656, -4657, -4658, -4659, -4660, -4661, -4662, -4663, -4664, -4665, -4666, -4667, -4668, -4669, -4670, -4671, -4672, -4673, -4674, -4675, -4676, -4677, -4678, -4679, -4680, -4681, -4682, -4683, -4684, -4685, -4686, -4687, -4688, -4689, -4690, -4691, -4692, -4693, -4694, -4695, -4696, -4697, -4698, -4699, -4700, -4701, -4702, -4703, -4704, -4705, -4706, -4707, -4708, -4709, -4710, -4711, -4712, -4713, -4714, -4715, -4716, -4717, -4718, -4719, -4720, -4721, -4722, -4723, -4724, -4725, -4726, -4727, -4728, -4729, -4730, -4731, -4732, -4733, -4734, -4735, -4736, -4737, -4738, -4739, -4740, -4741, -4742, -4743, -4744, -4745, -4746, -4747, -4748, -4749, -4750, -4751, -4752, -4753, -4754, -4755, -4756, -4757, -4758, -4759, -4760, -4761, -4762, -4763, -4764, -4765, -4766, -4767, -4768, -4769, -4770, -4771, -4772, -4773, -4774, -4775, -4776, -4777, -4778, -4779, -4780, -4781, -4782, -4783, -4784, -5000, -5001, -5002, -5003, -5006, -5007, -5008, -5010, -5011, -5012, -5014, -5015, -5016, -5017, -5018, -5019, -5020, -5022, -5023, -5024, -5025, -5026, -5027, -5028, -5029, -5030, -5031, -5032, -5034, -5035, -5036, -5037, -5038, -5039, -5040, -5041, -5042, -5043, -5044, -5046, -5047, -5050, -5051, -5052, -5053, -5054, -5055, -5056, -5057, -5058, -5059, -5061, -5063, -5064, -5065, -5066, -5067, -5068, -5069, -5070, -5071, -5072, -5073, -5074, -5080, -5081, -5083, -5084, -5085, -5086, -5087, -5088, -5089, -5090, -5091, -5092, -5093, -5094, -5095, -5096, -5097, -5098, -5099, -5100, -5101, -5102, -5103, -5104, -5105, -5106, -5107, -5108, -5109, -5110, -5111, -5112, -5113, -5114, -5115, -5116, -5117, -5118, -5119, -5120, -5121, -5122, -5123, -5124, -5125, -5130, -5131, -5133, -5134, -5135, -5136, -5137, -5138, -5139, -5140, -5142, -5143, -5144, -5145, -5146, -5147, -5148, -5149, -5150, -5151, -5153, -5154, -5155, -5156, -5157, -5158, -5159, -5160, -5161, -5162, -5163, -5164, -5165, -5166, -5167, -5168, -5169, -5170, -5171, -5172, -5173, -5174, -5175, -5176, -5177, -5178, -5179, -5180, -5181, -5182, -5183, -5184, -5185, -5187, -5188, -5189, -5190, -5191, -5192, -5193, -5194, -5195, -5196, -5197, -5198, -5199, -5200, -5201, -5202, -5203, -5204, -5205, -5206, -5207, -5208, -5209, -5210, -5211, -5212, -5213, -5214, -5215, -5216, -5217, -5218, -5219, -5220, -5221, -5222, -5223, -5224, -5225, -5226, -5227, -5228, -5229, -5230, -5231, -5233, -5234, -5235, -5236, -5237, -5238, -5239, -5240, -5241, -5242, -5243, -5244, -5245, -5246, -5247, -5248, -5249, -5250, -5251, -5252, -5253, -5254, -5255, -5256, -5257, -5258, -5259, -5260, -5261, -5262, -5263, -5264, -5265, -5266, -5267, -5268, -5269, -5270, -5271, -5272, -5273, -5274, -5275, -5276, -5277, -5278, -5279, -5280, -5281, -5282, -5283, -5284, -5285, -5286, -5287, -5288, -5289, -5290, -5291, -5292, -5293, -5294, -5295, -5296, -5297, -5298, -5299, -5300, -5301, -5302, -5303, -5304, -5305, -5306, -5307, -5308, -5309, -5310, -5311, -5312, -5313, -5314, -5315, -5316, -5317, -5318, -5319, -5320, -5321, -5322, -5323, -5324, -5325, -5326, -5327, -5328, -5329, -5330, -5331, -5332, -5333, -5334, -5335, -5336, -5337, -5338, -5339, -5340, -5341, -5342, -5343, -5344, -5345, -5346, -5347, -5348, -5349, -5350, -5351, -5352, -5353, -5354, -5355, -5356, -5357, -5358, -5359, -5360, -5361, -5362, -5363, -5364, -5365, -5366, -5367, -5368, -5369, -5370, -5371, -5372, -5373, -5374, -5375, -5376, -5377, -5378, -5379, -5380, -5381, -5382, -5383, -5384, -5385, -5386, -5387, -5388, -5389, -5390, -5400, -5401, -5402, -5403, -5404, -5405, -5406, -5407, -5408, -5409, -5410, -5411, -5412, -5413, -5414, -5415, -5416, -5417, -5418, -5419, -5420, -5421, -5422, -5423, -5424, -5425, -5426, -5427, -5428, -5429, -5430, -5431, -5432, -5433, -5434, -5435, -5436, -5437, -5438, -5439, -5440, -5441, -5442, -5443, -5444, -5445, -5446, -5447, -5448, -5449, -5450, -5451, -5452, -5453, -5454, -5455, -5456, -5457, -5458, -5459, -5460, -5461, -5462, -5463, -5464, -5465, -5466, -5467, -5468, -5469, -5470, -5471, -5472, -5473, -5474, -5475, -5476, -5477, -5478, -5479, -5480, -5481, -5482, -5483, -5484, -5485, -5486, -5487, -5488, -5489, -5490, -5491, -5492, -5493, -5494, -5495, -5496, -5497, -5498, -5499, -5500, -5501, -5502, -5503, -5504, -5505, -5506, -5507, -5508, -5509, -5510, -5511, -5512, -5513, -5514, -5515, -5516, -5517, -5518, -5519, -5520, -5521, -5522, -5540, -5541, -5542, -5543, -5544, -5545, -5546, -5547, -5548, -5549, -5550, -5551, -5552, -5553, -5554, -5555, -5556, -5557, -5558, -5559, -5560, -5561, -5562, -5563, -5564, -5565, -5566, -5567, -5568, -5569, -5570, -5571, -5572, -5573, -5574, -5575, -5576, -5577, -5578, -5579, -5580, -5581, -5582, -5583, -5584, -5585, -5586, -5587, -5588, -5589, -5590, -5591, -5592, -5593, -5594, -5595, -5596, -5597, -5598, -5599, -5600, -5601, -5602, -5603, -5604, -5605, -5607, -5608, -5609, -5610, -5611, -5612, -5613, -5614, -5615, -5616, -5617, -5618, -5619, -5620, -5621, -5622, -5623, -5624, -5625, -5626, -5627, -5628, -5629, -5630, -5631, -5632, -5633, -5634, -5635, -5636, -5637, -5638, -5639, -5640, -5641, -5642, -5643, -5644, -5645, -5646, -5647, -5648, -5649, -5650, -5651, -5652, -5653, -5654, -5655, -5656, -5657, -5658, -5659, -5660, -5661, -5662, -5663, -5664, -5665, -5666, -5667, -5668, -5671, -5672, -5673, -5674, -5675, -5676, -5677, -5678, -5679, -5680, -5681, -5682, -5683, -5684, -5685, -5686, -5687, -5688, -5689, -5690, -5691, -5692, -5693, -5694, -5695, -5696, -5697, -5698, -5699, -5700, -5701, -5702, -5703, -5704, -5705, -5706, -5707, -5708, -5709, -5710, -5711, -5712, -5713, -5714, -5715, -5716, -5717, -5718, -5719, -5720, -5721, -5722, -5723, -5724, -5725, -5726, -5727, -5728, -5729, -5730, -5731, -5732, -5733, -5734, -5735, -5736, -5737, -5738, -5739, -5740, -5741, -5742, -5743, -5744, -5745, -5746, -5747, -5748, -5749, -5750, -5751, -5752, -5753, -5754, -5755, -5756, -5757, -5758, -5759, -5760, -5761, -5762, -5763, -5764, -5765, -5766, -5768, -5769, -5770, -5771, -5772, -5773, -5774, -5777, -5778, -5779, -5780, -5781, -5785, -5786, -5787, -5788, -5789, -5790, -5791, -5792, -5793, -5794, -5795, -5796, -5797, -5798, -5799, -5800, -5801, -5802, -5803, -5804, -5805, -5806, -5807, -5808, -5809, -5810, -5811, -5812, -5813, -5814, -5815, -5816, -5817, -5818, -5819, -5820, -5821, -5822, -5823, -5824, -5825, -5826, -5827, -5828, -5829, -5830, -5831, -5832, -5833, -5834, -5835, -5836, -5837, -5838, -5839, -5840, -5841, -5842, -5843, -5844, -5845, -5846, -5847, -5848, -5849, -5850, -5851, -5852, -5853, -5854, -5855, -5856, -5857, -5858, -5859, -5860, -5861, -5862, -5863, -5864, -5865, -5866, -5867, -5868, -5869, -5870, -5871, -5872, -5873, -5874, -5875, -5876, -5877, -5878, -5879, -5880, -5881, -5882, -5883, -5884, -5885, -5886, -5887, -5888, -5889, -5890, -5891, -5892, -5893, -5894, -5895, -5896, -5897, -5898, -5899, -5900, -5901, -5902, -5903, -5904, -5905, -5906, -5907, -5908, -5909, -5910, -5911, -5912, -5913, -5914, -5915, -5916, -5917, -5918, -5919, -5920, -5921, -5922, -5923, -5924, -5925, -5926, -5927, -5928, -5929, -5930, -5931, -5932, -5933, -5934, -5935, -5936, -5937, -5938, -5939, -5940, -5941, -5942, -5943, -5944, -5945, -5946, -5947, -5948, -5949, -5950, -5951, -5952, -5953, -5954, -5955, -5956, -5957, -5958, -5959, -5960, -5961, -5962, -5963, -5964, -5965, -5966, -5967, -5968, -5969, -5970, -5971, -5972, -5973, -5974, -5975, -5976, -5977, -5978, -5979, -5980, -5981, -5982, -5983, -5984, -5985, -5986, -5987, -5988, -5989, -5990, -5991, -5992, -5993, -5994, -5995, -5996, -5997, -5998, -5999, -6000, -6001, -6002, -6003, -6004, -6005, -6006, -6201, -6202, -6203, -6204, -6205, -6206, -6207, -6208, -6209, -6210, -6211, -6212, -6213, -6214, -6215, -6219, -6220, -6221, -6222, -6223, -6224, -6225, -6226, -6227, -6228, -6229, -6230, -6231, -6232, -6233, -6234, -6235, -6236, -6237, -6238, -6239, -6240, -6241, -6242, -6243, -6244, -6245, -6246, -6247, -6248, -6249, -6250, -6251, -6252, -6253, -6254, -6255, -6256, -6257, -6258, -6259, -6260, -6261, -6262, -6263, -6264, -6265, -6266, -6267, -6268, -6269, -6270, -6271, -6272, -6273, -6274, -6275, -6276, -6277, -6278, -6279, -6280, -6281, -6282, -6283, -6284, -6285, -6301, -6302, -6303, -6304, -6305, -6306, -6307, -6308, -6309, -6310, -6311, -6312, -6313, -6314, -6315, -6316, -6317, -6318, -6319, -6320, -6321, -6322, -6323, -6324, -6325, -6326, -6327, -6328, -6329, -6330, -6331, -6332, -7000, -7001, -7002, -7003, -7004, -7005, -7006, -7007, -7010, -7011, -7012, -7013, -7014, -7015, -7021, -7022, -7024, -7025, -7026, -7027, -7029, -7030, -7031, -7032, -7033, -7034, -7035, -7036, -7037, -7038, -7039, -7040, -7041, -7100, -7101, -7102, -7103, -7104, -7105, -7106, -7107, -7108, -7109, -7110, -7111, -7112, -7113, -7114, -7115, -7116, -7117, -7118, -7119, -7120, -7121, -7122, -7123, -7124, -7201, -7202, -7203, -7204, -7205, -7206, -7207, -7208, -7209, -7210, -7211, -7212, -7213, -7214, -7215, -7216, -7217, -7218, -7219, -7220, -7221, -7222, -7223, -7224, -7225, -7226, -7227, -7228, -7229, -7230, -7231, -7232, -7233, -7234, -7235, -7236, -7237, -7238, -7239, -7240, -7241, -7242, -7243, -7244, -7246, -7247, -7248, -7249, -7250, -7251, -7252, -7253, -7254, -7255, -7256, -7257, -7258, -7259, -7260, -7261, -7262, -7263, -7264, -7265, -7266, -7267, -7268, -7269, -7270, -7271, -7272, -7273, -7274, -7275, -7276, -7277, -7278, -7279, -7280, -7281, -7282, -7283, -7284, -7285, -7286, -7287, -7288, -7289, -7290, -7291, -7292, -7293, -7294, -7295, -7296, -7297, -7298, -7299, -7300, -7301, -7302, -7402, -7403, -7404, -7405, -7406, -7407, -7408, -7409, -7410, -7411, -7412, -7413, -7414, -7415, -7416, -7417, -7418, -7419, -7420, -7421, -7422, -7423, -7424, -7425, -7426, -7427, -7428, -7429, -7430, -7431, -7432, -7433, -7434, -7435, -7600, -7601, -7602, -7603, -7604, -8001, -8002, -8003, -8004, -8005, -9001, -9002, -9003, -9004, -9005, -9006, -9007, -9008, -9009, -9010, -9011, -9012, -9013, -9014, -9015, -9016, -9017, -9018, -9019, -9020, -9022, -9023, -9024, -9025, -9026, -9027, -9028, -9029, -9030, -9031, -9032, -9033, -9034, -9035, -9036, -9037, -9038, -9039, -9040, -9041, -9042, -9043, -9044, -9045, -9046, -9047, -9048, -9049, -9050, -9051, -9052, -9053, -9054, -9057, -9058, -9059, -9060, -9061, -9062, -9063, -9064, -9065, -9066, -9069, -9070, -9071, -9072, -9073, -9074, -9075, -9076, -9077, -9078, -9079, -9080, -9081, -9082, -9083, -9084, -9085, -9086, -9087, -9088, -9089, -9090, -9091, -9092, -9093, -9094, -9095, -9096, -9097, -9098, -9099, -9100, -9101, -9102, -9103, -9104, -9105, -9106, -9107, -9108, -9109, -9110, -9111, -9112, -9113, -9114, -9115, -9116, -9117, -9118, -9119, -9120, -9121, -9122, -9123, -9124, -9125, -9126, -9127, -9200, -9201, -9202, -9203, -9501, -9502, -9503, -9504, -9505, -9506, -9507, -9508, -9509, -9510, -9512, -9513, -9514, -9515, -9516, -9518, -9519, -9520, -9521, -9522, -9523, -9524, -9525, -9526, -9527, -9528, -9529, -9530, -9531, -9532, -9533, -9534, -9535, -9536, -9537, -9538, -9539, -9540, -9541, -9542, -9543, -9544, -9545, -9546, -9547, -9548, -9549, -9550, -9551, -9552, -9553, -9554, -9555, -9556, -9557, -9558, -9559, -9560, -9561, -9562, -9563, -9564, -9565, -9566, -9567, -9568, -9569, -9570, -9571, -9572, -9573, -9574, -9575, -9576, -9577, -9578, -9579, -9580, -9581, -9582, -9583, -9584, -9585, -9586, -9587, -9588, -9589, -9590, -9591, -9592, -9593, -9594, -9595, -9596, -9597, -9598, -9599, -9600, -9601, -9602, -9603, -9604, -9605, -9606, -9607, -9608, -9609, -9610, -9611, -9612, -9613, -9614, -9615, -9616, -9617, -9618, -9619, -9620, -9621, -9622, -9623, -9624, -9625, -9626, -9627, -9628, -9629, -9630, -9631, -9632, -9633, -9634, -9635, -9636, -9637, -9638, -9639, -9640, -9641, -9642, -9643, -9644, -9645, -9646, -9647, -9648, -9649, -9650, -9651, -9652, -9653, -9654, -9655, -9656, -9657, -9658, -9659, -9660, -9661, -9662, -9663, -9664, -9665, -9666, -9667, -9668, -9669, -9670, -9671, -9672, -9673, -9674, -9675, -9676, -9677, -9678, -9679, -9680, -9681, -9682, -9683, -9684, -9685, -9686, -9687, -9688, -9689, -9690, -9691, -9692, -9693, -9694, -9695, -9696, -9697, -9698, -9699, -9700, -9701, -9702, -9703, -9704, -9705, -9706, -9707, -9708, -9709, -9710, -9711, -9712, -9713, -9714, -9715, -9716, -9717, -9718, -9719, -9720, -9721, -9722, -9723, -9724, -9725, -9726, -9727, -9728, -9729, -9730, -9731, -9732, -9733, -9734, -9735, -9736, -9737, -9738, -9739, -9740, -9741, -9742, -9743, -9744, -9745, -9746, -9747, -9748, -9749, -9750, -9751, -9752, -9753, -9754, -9755, -9756, -9757, -9758, -9759, -9760, -9761, -9762, -9763, -9764, -9765, -9766, -9767, -9768, -9769, -9770, -9771, -9772, -9773, -9774, -9775, -9776, -9777, -9778, -9779, -9780, -9781, -9782, -9783, -10500, -10501, -10502, -10503, -10504, -10505, -10506, -10507, -10508, -10509, -10510, -10511, -10512, -10513, -10514, -10515, -10516, -10650, -11000, -11001, -11002, -11003, -11004, -11005, -11006, -11007, -11008, -11009, -11010, -11011, -11012, -11013, -11014, -11015, -11016, -11017, -11018, -11019, -11020, -11021, -11022, -11023, -11024, -11025, -11026, -11027, -11028, -11029, -11030, -11031, -11032, -11033, -11034, -11035, -11036, -11037, -11038, -11039, -11040, -11041, -11042, -11043, -11044, -11045, -11046, -11047, -11048, -11049, -11050, -11051, -11052, -20000, -21000, -22998, -30926, -32491, -38104, -38105}; const char *ob_error_name(const int err) { const char *ret = "Unknown error"; diff --git a/src/share/ob_errno.def b/src/share/ob_errno.def index 667065f98..8d49e6fec 100755 --- a/src/share/ob_errno.def +++ b/src/share/ob_errno.def @@ -2644,6 +2644,7 @@ DEFINE_ERROR(OB_ERR_EVENT_CANNOT_ALTER_IN_THE_PAST, -9779, ER_EVENT_CANNOT_ALTER DEFINE_ERROR(OB_ERR_EVENT_RECURSION_FORBIDDEN, -9780, ER_EVENT_RECURSION_FORBIDDEN, "HY000", "Recursion of EVENT DDL statements is forbidden when body is present"); DEFINE_ORACLE_ERROR(OB_NO_PARTITION_FOR_GIVEN_VALUE_SCHEMA_ERROR, -9781, ER_NO_PARTITION_FOR_GIVEN_VALUE, "HY000", "Table has no partition for value", 14400, "inserted partition key does not map to any partition"); DEFINE_ORACLE_ERROR(OB_ERR_INVALID_CHARACTER, -9782, -1, "HY000", "invalid character", 911, "invalid character"); +DEFINE_ORACLE_ERROR_EXT(OB_ERR_MVIEW_CAN_NOT_ON_QUERY_COMPUTE, -9783, -1, "HY000", "cannot ENABLE ON QUERY COMPUTATION for the materialized view", "cannot ENABLE ON QUERY COMPUTATION for the materialized view `%s`.`%s`", 32361, "cannot ENABLE ON QUERY COMPUTATION for the materialized view", "cannot ENABLE ON QUERY COMPUTATION for the materialized view %s.%s"); // 余留位置 //////////////////////////////////////////////////////////////// // PL/SQL错误码值域 [-9500, -10000) diff --git a/src/share/ob_errno.h b/src/share/ob_errno.h index 17eebbd79..c618a49a6 100755 --- a/src/share/ob_errno.h +++ b/src/share/ob_errno.h @@ -1868,6 +1868,7 @@ constexpr int OB_ERR_EVENT_CANNOT_ALTER_IN_THE_PAST = -9779; constexpr int OB_ERR_EVENT_RECURSION_FORBIDDEN = -9780; constexpr int OB_NO_PARTITION_FOR_GIVEN_VALUE_SCHEMA_ERROR = -9781; constexpr int OB_ERR_INVALID_CHARACTER = -9782; +constexpr int OB_ERR_MVIEW_CAN_NOT_ON_QUERY_COMPUTE = -9783; constexpr int OB_ERR_KV_GLOBAL_INDEX_ROUTE = -10500; constexpr int OB_TTL_NOT_ENABLE = -10501; constexpr int OB_TTL_COLUMN_NOT_EXIST = -10502; @@ -4171,6 +4172,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_EVENT_RECURSION_FORBIDDEN__USER_ERROR_MSG "Recursion of EVENT DDL statements is forbidden when body is present" #define OB_NO_PARTITION_FOR_GIVEN_VALUE_SCHEMA_ERROR__USER_ERROR_MSG "Table has no partition for value" #define OB_ERR_INVALID_CHARACTER__USER_ERROR_MSG "invalid character" +#define OB_ERR_MVIEW_CAN_NOT_ON_QUERY_COMPUTE__USER_ERROR_MSG "cannot ENABLE ON QUERY COMPUTATION for the materialized view `%s`.`%s`" #define OB_ERR_KV_GLOBAL_INDEX_ROUTE__USER_ERROR_MSG "incorrect route for obkv global index, client router should refresh." #define OB_TTL_NOT_ENABLE__USER_ERROR_MSG "TTL feature is not enabled" #define OB_TTL_COLUMN_NOT_EXIST__USER_ERROR_MSG "TTL column '%.*s' not exists" @@ -8749,6 +8751,8 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_NO_PARTITION_FOR_GIVEN_VALUE_SCHEMA_ERROR__OBE_USER_ERROR_MSG "OBE-14400: inserted partition key does not map to any partition" #define OB_ERR_INVALID_CHARACTER__ORA_USER_ERROR_MSG "ORA-00911: invalid character" #define OB_ERR_INVALID_CHARACTER__OBE_USER_ERROR_MSG "OBE-00911: invalid character" +#define OB_ERR_MVIEW_CAN_NOT_ON_QUERY_COMPUTE__ORA_USER_ERROR_MSG "ORA-32361: cannot ENABLE ON QUERY COMPUTATION for the materialized view %s.%s" +#define OB_ERR_MVIEW_CAN_NOT_ON_QUERY_COMPUTE__OBE_USER_ERROR_MSG "OBE-32361: cannot ENABLE ON QUERY COMPUTATION for the materialized view %s.%s" #define OB_ERR_KV_GLOBAL_INDEX_ROUTE__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -10500, incorrect route for obkv global index, client router should refresh." #define OB_ERR_KV_GLOBAL_INDEX_ROUTE__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -10500, incorrect route for obkv global index, client router should refresh." #define OB_TTL_NOT_ENABLE__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -10501, TTL feature is not enabled" @@ -8910,7 +8914,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_INVALID_DATE_MSG_FMT_V2__ORA_USER_ERROR_MSG "ORA-01861: Incorrect datetime value for column '%.*s' at row %ld" #define OB_ERR_INVALID_DATE_MSG_FMT_V2__OBE_USER_ERROR_MSG "OBE-01861: Incorrect datetime value for column '%.*s' at row %ld" -extern int g_all_ob_errnos[2326]; +extern int g_all_ob_errnos[2327]; const char *ob_error_name(const int oberr); const char* ob_error_cause(const int oberr); diff --git a/src/sql/resolver/ddl/ob_create_view_resolver.cpp b/src/sql/resolver/ddl/ob_create_view_resolver.cpp index e959bd746..8b6ac16b2 100644 --- a/src/sql/resolver/ddl/ob_create_view_resolver.cpp +++ b/src/sql/resolver/ddl/ob_create_view_resolver.cpp @@ -1321,6 +1321,11 @@ int ObCreateViewResolver::resolve_mv_options(const ObSelectStmt *stmt, && OB_FAIL(ObMVChecker::check_mv_fast_refresh_valid(stmt, params_.stmt_factory_, params_.expr_factory_, params_.session_info_))) { + // When creating an MV, which can not be fast refreshed, with both fast refresh + // and on query computation, we should return CAN_NOT_ON_QUERY_COMPUTE + if (table_schema.mv_on_query_computation() && OB_ERR_MVIEW_CAN_NOT_FAST_REFRESH == ret) { + ret = OB_ERR_MVIEW_CAN_NOT_ON_QUERY_COMPUTE; + } LOG_WARN("fail to check fast refresh valid", K(ret)); } else if (table_schema.mv_on_query_computation() && OB_FAIL(check_on_query_computation_supported(stmt))) { From 9b2deedb61a7aeac643827d3dcac48844b43f512 Mon Sep 17 00:00:00 2001 From: obdev Date: Mon, 26 Aug 2024 07:26:48 +0000 Subject: [PATCH 219/249] If the tmp file is deleted, the flush task will not be retried; delete the tmp file as soon as possible; --- .../tmp_file/ob_tmp_file_flush_ctx.cpp | 4 ++- src/storage/tmp_file/ob_tmp_file_flush_ctx.h | 1 + .../tmp_file/ob_tmp_file_flush_manager.cpp | 29 +++++++++++++-- .../tmp_file/ob_tmp_file_flush_manager.h | 1 + .../tmp_file/ob_tmp_file_thread_wrapper.cpp | 35 ++++++++++++------- .../tmp_file/ob_tmp_file_thread_wrapper.h | 1 + 6 files changed, 56 insertions(+), 15 deletions(-) diff --git a/src/storage/tmp_file/ob_tmp_file_flush_ctx.cpp b/src/storage/tmp_file/ob_tmp_file_flush_ctx.cpp index cfd2cac01..1586d5371 100644 --- a/src/storage/tmp_file/ob_tmp_file_flush_ctx.cpp +++ b/src/storage/tmp_file/ob_tmp_file_flush_ctx.cpp @@ -197,7 +197,9 @@ void ObTmpFileBatchFlushContext::try_update_prepare_finished_cnt(const ObTmpFile if (OB_UNLIKELY(flush_seq_ctx_.prepare_finished_cnt_ >= flush_seq_ctx_.create_flush_task_cnt_)) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("unexpected flush_seq_ctx_", KPC(this)); - } else if (ObTmpFileFlushTask::TFFT_WAIT == flush_task.get_state() || flush_task.get_data_length() == 0) { + } else if (ObTmpFileFlushTask::TFFT_WAIT == flush_task.get_state() || + ObTmpFileFlushTask::TFFT_ABORT == flush_task.get_state() || + flush_task.get_data_length() == 0) { ++flush_seq_ctx_.prepare_finished_cnt_; recorded = true; } diff --git a/src/storage/tmp_file/ob_tmp_file_flush_ctx.h b/src/storage/tmp_file/ob_tmp_file_flush_ctx.h index 259379fac..13041fb46 100644 --- a/src/storage/tmp_file/ob_tmp_file_flush_ctx.h +++ b/src/storage/tmp_file/ob_tmp_file_flush_ctx.h @@ -240,6 +240,7 @@ public: TFFT_ASYNC_WRITE = 5, TFFT_WAIT = 6, TFFT_FINISH = 7, + TFFT_ABORT = 8, }; public: void destroy(); diff --git a/src/storage/tmp_file/ob_tmp_file_flush_manager.cpp b/src/storage/tmp_file/ob_tmp_file_flush_manager.cpp index 2ae360ecd..e8f21780d 100644 --- a/src/storage/tmp_file/ob_tmp_file_flush_manager.cpp +++ b/src/storage/tmp_file/ob_tmp_file_flush_manager.cpp @@ -650,6 +650,27 @@ int ObTmpFileFlushManager::drive_flush_task_prepare_(ObTmpFileFlushTask &flush_t return ret; } +void ObTmpFileFlushManager::try_remove_unused_flush_info_(ObTmpFileFlushTask &flush_task) +{ + int ret = OB_SUCCESS; + + ObArray &flush_infos = flush_task.get_flush_infos(); + for (int64_t i = 0; OB_SUCC(ret) && i >= 0 && i < flush_infos.count(); ++i) { + ObTmpFileFlushInfo &flush_info = flush_infos.at(i); + ObSharedNothingTmpFile *file = flush_info.file_handle_.get(); + if (OB_ISNULL(file)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "file is nullptr", KR(ret), K(flush_info)); + } else if (file->is_deleting()) { + STORAGE_LOG(INFO, "the file is deleting, abort this flush info", + KR(ret), K(flush_info), K(flush_task)); + flush_info.reset(); + flush_infos.remove(i); + --i; + } + } +} + int ObTmpFileFlushManager::drive_flush_task_retry_( ObTmpFileFlushTask &flush_task, const FlushState state, @@ -664,7 +685,11 @@ int ObTmpFileFlushManager::drive_flush_task_retry_( } break; case FlushState::TFFT_ASYNC_WRITE: - if (OB_FAIL(handle_async_write_(flush_task, next_state))) { + try_remove_unused_flush_info_(flush_task); + if (0 == flush_task.get_flush_infos().count()) { + STORAGE_LOG(INFO, "all flush info is aborted", KR(ret), K(flush_task)); + next_state = FlushState::TFFT_ABORT; + } else if (OB_FAIL(handle_async_write_(flush_task, next_state))) { STORAGE_LOG(WARN, "fail to handle flush task async write", KR(ret), K(flush_task)); } break; @@ -710,7 +735,7 @@ int ObTmpFileFlushManager::retry(ObTmpFileFlushTask &flush_task) } else if (OB_FAIL(advance_status_(flush_task, next_state))) { STORAGE_LOG(WARN, "fail to advance status", KR(ret), K(state), K(next_state), K(flush_task)); } - } while (OB_SUCC(ret) && FlushState::TFFT_WAIT != next_state); + } while (OB_SUCC(ret) && FlushState::TFFT_WAIT != next_state && FlushState::TFFT_ABORT != next_state); if (!flush_task.get_recorded_as_prepare_finished()) { if (flush_task.get_flush_seq() != flush_ctx_.get_flush_sequence()) { diff --git a/src/storage/tmp_file/ob_tmp_file_flush_manager.h b/src/storage/tmp_file/ob_tmp_file_flush_manager.h index 87e1383eb..d52b0407a 100644 --- a/src/storage/tmp_file/ob_tmp_file_flush_manager.h +++ b/src/storage/tmp_file/ob_tmp_file_flush_manager.h @@ -110,6 +110,7 @@ private: int evict_pages_and_retry_insert_(ObTmpFileFlushTask &flush_task, ObTmpFileFlushInfo &flush_info, const int64_t logic_block_index); + void try_remove_unused_flush_info_(ObTmpFileFlushTask &flush_task); DISALLOW_COPY_AND_ASSIGN(ObTmpFileFlushManager); private: bool is_inited_; diff --git a/src/storage/tmp_file/ob_tmp_file_thread_wrapper.cpp b/src/storage/tmp_file/ob_tmp_file_thread_wrapper.cpp index d08c739e9..e082a6f29 100644 --- a/src/storage/tmp_file/ob_tmp_file_thread_wrapper.cpp +++ b/src/storage/tmp_file/ob_tmp_file_thread_wrapper.cpp @@ -382,7 +382,10 @@ int ObTmpFileFlushTG::retry_task_() } // push task into wait_list_/retry_list_ according to task state, ignore error code FlushState state = flush_task->get_state(); - if (FlushState::TFFT_WAIT == state) { + if (FlushState::TFFT_ABORT == state) { + STORAGE_LOG(INFO, "free abort flush task", KPC(flush_task)); + flush_task_finished_(flush_task); + } else if (FlushState::TFFT_WAIT == state) { push_wait_list_(flush_task); } else if (FlushState::TFFT_FILL_BLOCK_BUF < state) { push_retry_list_(flush_task); @@ -455,29 +458,37 @@ int ObTmpFileFlushTG::check_flush_task_io_finished_() ret = OB_ERR_UNEXPECTED; STORAGE_LOG(WARN, "flush task is nullptr", KR(ret)); } else { - bool is_flush_tree_page = flush_task->get_is_fast_flush_tree(); STORAGE_LOG(DEBUG, "flush task io complete", K(flushing_block_num_), KPC(flush_task)); // if the update fails, it will be retried during the next wakeup if (OB_FAIL(flush_mgr_.update_file_meta_after_flush(*flush_task))) { STORAGE_LOG(WARN, "fail to drive flush state machine", KR(ret), KPC(flush_task)); push_finished_list_(flush_task); } else { - flush_mgr_.free_flush_task(flush_task); - ATOMIC_DEC(&flushing_block_num_); - fast_flush_meta_task_cnt_ -= is_flush_tree_page ? 1 : 0; - if (fast_flush_meta_task_cnt_ == 0) { - // reset is_fast_flush_meta_ flag to resume retry task and flush - is_fast_flush_meta_ = false; - } else if (OB_UNLIKELY(fast_flush_meta_task_cnt_ < 0)) { - STORAGE_LOG(ERROR, "fast_flush_meta_task_cnt_ is negative", KPC(this)); - } - signal_io_finish(); + flush_task_finished_(flush_task); } } } return ret; } +void ObTmpFileFlushTG::flush_task_finished_(ObTmpFileFlushTask *flush_task) +{ + int ret = OB_SUCCESS; + + bool is_flush_tree_page = flush_task->get_is_fast_flush_tree(); + flush_mgr_.free_flush_task(flush_task); + ATOMIC_DEC(&flushing_block_num_); + fast_flush_meta_task_cnt_ -= is_flush_tree_page ? 1 : 0; + if (fast_flush_meta_task_cnt_ == 0) { + // reset is_fast_flush_meta_ flag to resume retry task and flush + is_fast_flush_meta_ = false; + } else if (OB_UNLIKELY(fast_flush_meta_task_cnt_ < 0)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(ERROR, "fast_flush_meta_task_cnt_ is negative", KR(ret), KPC(this)); + } + signal_io_finish(); +} + int ObTmpFileFlushTG::push_wait_list_(ObTmpFileFlushTask *flush_task) { int ret = OB_SUCCESS; diff --git a/src/storage/tmp_file/ob_tmp_file_thread_wrapper.h b/src/storage/tmp_file/ob_tmp_file_thread_wrapper.h index 79fe3fbae..1afedd17c 100644 --- a/src/storage/tmp_file/ob_tmp_file_thread_wrapper.h +++ b/src/storage/tmp_file/ob_tmp_file_thread_wrapper.h @@ -74,6 +74,7 @@ private: int pop_retry_list_(ObTmpFileFlushTask *&flush_task); int push_finished_list_(ObTmpFileFlushTask *flush_task); int pop_finished_list_(ObTmpFileFlushTask *&flush_task); + void flush_task_finished_(ObTmpFileFlushTask *flush_task); private: bool is_inited_; RUNNING_MODE mode_; From e96f717f75769b9af637f8e4ec87505d57f2b201 Mon Sep 17 00:00:00 2001 From: zhjc1124 Date: Mon, 26 Aug 2024 08:18:08 +0000 Subject: [PATCH 220/249] resume group_id when group guard destruct --- deps/oblib/src/lib/worker.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deps/oblib/src/lib/worker.h b/deps/oblib/src/lib/worker.h index 600f7592f..730bc8600 100644 --- a/deps/oblib/src/lib/worker.h +++ b/deps/oblib/src/lib/worker.h @@ -220,7 +220,7 @@ public: ~ConsumerGroupIdGuard() { if (group_changed_) { - // SET_GROUP_ID(thread_group_id_); + SET_GROUP_ID(thread_group_id_); } } int get_ret() @@ -241,7 +241,7 @@ public: ConsumerGroupFuncGuard(uint8_t func_type) : thread_group_id_(GET_GROUP_ID()), thread_func_type_(GET_FUNC_TYPE()), group_changed_(false), ret_(OB_SUCCESS) { - // THIS_WORKER.set_func_type_(func_type); + THIS_WORKER.set_func_type_(func_type); uint64_t group_id = 0; ret_ = CONVERT_FUNCTION_TYPE_TO_GROUP_ID(func_type, group_id); if (OB_SUCCESS == ret_ && is_user_group(group_id) && group_id != thread_group_id_) { @@ -252,8 +252,8 @@ public: ~ConsumerGroupFuncGuard() { if (group_changed_) { - // SET_GROUP_ID(thread_group_id_); - // THIS_WORKER.set_func_type_(thread_func_type_); + SET_GROUP_ID(thread_group_id_); + THIS_WORKER.set_func_type_(thread_func_type_); } } int get_ret() From 83350fabe2991e4a4d0b08323346b1b419924e0a Mon Sep 17 00:00:00 2001 From: zzg19950727 <1071026277@qq.com> Date: Mon, 26 Aug 2024 08:23:55 +0000 Subject: [PATCH 221/249] fix log_exchange compute op_ordering bug --- src/sql/optimizer/ob_log_exchange.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/sql/optimizer/ob_log_exchange.cpp b/src/sql/optimizer/ob_log_exchange.cpp index 72473e196..3fdb48bd7 100644 --- a/src/sql/optimizer/ob_log_exchange.cpp +++ b/src/sql/optimizer/ob_log_exchange.cpp @@ -363,7 +363,9 @@ int ObLogExchange::compute_op_ordering() } else if (is_producer()) { // for FULL_INPUT_SAMPLE, we cache all rows in transmit and send in random range // to avoid send to one worker at one time if input order is the same with %sort_keys_ - is_local_order_ = FULL_INPUT_SAMPLE == sample_type_; + if (FULL_INPUT_SAMPLE == sample_type_) { + is_local_order_ = true; + } } else if (is_consumer()) { if (is_merge_sort_) { if (OB_UNLIKELY(sort_keys_.empty())) { From b8adb830533f7db648dd921f2a8763e31e5d2be9 Mon Sep 17 00:00:00 2001 From: godyangfight Date: Mon, 26 Aug 2024 08:48:45 +0000 Subject: [PATCH 222/249] Fix migration src 0.0.0.0 question. --- src/storage/high_availability/ob_ls_migration_handler.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/storage/high_availability/ob_ls_migration_handler.cpp b/src/storage/high_availability/ob_ls_migration_handler.cpp index ec3dd6a64..3abf966af 100644 --- a/src/storage/high_availability/ob_ls_migration_handler.cpp +++ b/src/storage/high_availability/ob_ls_migration_handler.cpp @@ -608,7 +608,7 @@ int ObLSMigrationHandler::do_init_status_() SERVER_EVENT_ADD("storage_ha", "ls_ha_start", "tenant_id", ls_->get_tenant_id(), "ls_id", ls_->get_ls_id().id(), - "src", task.arg_.data_src_.get_server(), + "src", task.arg_.src_.get_server(), "dst", task.arg_.dst_.get_server(), "task_id", task.task_id_, "is_failed", OB_SUCCESS, @@ -715,7 +715,7 @@ int ObLSMigrationHandler::do_finish_status_() SERVER_EVENT_ADD("storage_ha", "ls_ha_finish", "tenant_id", ls_->get_tenant_id(), "ls_id", ls_->get_ls_id().id(), - "src", task.arg_.data_src_.get_server(), + "src", task.arg_.src_.get_server(), "dst", task.arg_.dst_.get_server(), "task_id", task.task_id_, "is_failed", result, From 9895d573eb6aa2a879f7c4c076da2199d0b66e3a Mon Sep 17 00:00:00 2001 From: 0xacc Date: Mon, 26 Aug 2024 08:54:55 +0000 Subject: [PATCH 223/249] [to #2024082100104215842] fix: fix get_sys_var sanity core dump --- src/sql/engine/expr/ob_expr_get_sys_var.cpp | 51 +++++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/src/sql/engine/expr/ob_expr_get_sys_var.cpp b/src/sql/engine/expr/ob_expr_get_sys_var.cpp index 2c59ae359..144d4047d 100644 --- a/src/sql/engine/expr/ob_expr_get_sys_var.cpp +++ b/src/sql/engine/expr/ob_expr_get_sys_var.cpp @@ -276,14 +276,59 @@ int ObExprGetSysVar::calc_get_sys_val_expr(const ObExpr &expr, ObEvalCtx &ctx, LOG_WARN("invalid arg cnt", K(ret), K(expr.arg_cnt_)); } else if (OB_FAIL(expr.eval_param_value(ctx, name, scope))) { LOG_WARN("eval param failed", K(ret)); + } else if (OB_ISNULL(name) || OB_ISNULL(scope)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected NULL name or scope", K(ret), KPC(name), KPC(scope), K(expr)); } else { const ObString &var_name = name->get_string(); - int64_t var_scope = scope->get_int(); + int64_t var_scope = OB_INVALID_COUNT; + ObObjType scope_type = expr.args_[1]->datum_meta_.get_type(); ObObj result; ObEvalCtx::TempAllocGuard alloc_guard(ctx); ObIAllocator &calc_alloc = alloc_guard.get_allocator(); - if (OB_FAIL(calc_(result, var_name, var_scope, ctx.exec_ctx_.get_my_session(), - &ctx.exec_ctx_, calc_alloc))) { + + if (ObIntType == scope_type) { + var_scope = scope->get_int(); + } else if (ObDecimalIntType == scope_type) { + int32_t int_bytes = scope->get_int_bytes(); + const ObDecimalInt *dec_int = scope->get_decimal_int(); + switch (int_bytes) { + case sizeof(int32_t): { + var_scope = *dec_int->int32_v_; + break; + } + case sizeof(int64_t): { + var_scope = *dec_int->int64_v_; + break; + } + case sizeof(int128_t): { + var_scope = *dec_int->int128_v_; + break; + } + case sizeof(int256_t): { + var_scope = *dec_int->int256_v_; + break; + } + case sizeof(int512_t): { + var_scope = *dec_int->int512_v_; + break; + } + default: { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected decimalint bytes", + K(ret), K(int_bytes), KPC(scope)); + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected scope type", + K(ret), K(scope_type), KPC(scope), K(expr), K(lbt())); + } + + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(calc_(result, var_name, var_scope, ctx.exec_ctx_.get_my_session(), + &ctx.exec_ctx_, calc_alloc))) { LOG_WARN("calc_ failed", K(ret), K(name), K(scope)); } else { const ObObjType &obj_type = result.get_type(); From bde6581619f583f3e47756951dfa9702612d8401 Mon Sep 17 00:00:00 2001 From: obdev Date: Mon, 26 Aug 2024 09:48:20 +0000 Subject: [PATCH 224/249] fix non strict constraints partition wise join reorder partition bug --- src/sql/engine/ob_exec_context.cpp | 126 +++++++++++++-- src/sql/engine/ob_exec_context.h | 11 +- src/sql/engine/px/ob_dfo.cpp | 15 ++ src/sql/engine/px/ob_px_util.cpp | 163 ++++++++++++++------ src/sql/engine/px/ob_px_util.h | 47 ++++-- src/sql/optimizer/ob_log_plan.cpp | 47 +++--- src/sql/optimizer/ob_pwj_comparer.cpp | 2 + src/sql/optimizer/ob_pwj_comparer.h | 24 ++- src/sql/plan_cache/ob_plan_match_helper.cpp | 39 +++-- 9 files changed, 359 insertions(+), 115 deletions(-) diff --git a/src/sql/engine/ob_exec_context.cpp b/src/sql/engine/ob_exec_context.cpp index 7f5099122..54aa7030d 100644 --- a/src/sql/engine/ob_exec_context.cpp +++ b/src/sql/engine/ob_exec_context.cpp @@ -108,7 +108,7 @@ ObExecContext::ObExecContext(ObIAllocator &allocator) frame_cnt_(0), op_kit_store_(), convert_allocator_(nullptr), - pwj_map_(nullptr), + group_pwj_map_(nullptr), calc_type_(CALC_NORMAL), fixed_id_(OB_INVALID_ID), check_status_times_(0), @@ -169,9 +169,9 @@ ObExecContext::~ObExecContext() package_guard_->~ObPLPackageGuard(); package_guard_ = NULL; } - if (OB_NOT_NULL(pwj_map_)) { - pwj_map_->destroy(); - pwj_map_ = NULL; + if (OB_NOT_NULL(group_pwj_map_)) { + group_pwj_map_->destroy(); + group_pwj_map_ = nullptr; } if (OB_NOT_NULL(vt_ift_)) { vt_ift_->~ObIVirtualTableIteratorFactory(); @@ -840,24 +840,50 @@ int ObExecContext::add_row_id_list(const common::ObIArray *row_id_list) return ret; } -int ObExecContext::get_pwj_map(PWJTabletIdMap *&pwj_map) +int ObExecContext::get_group_pwj_map(GroupPWJTabletIdMap *&group_pwj_map) { int ret = OB_SUCCESS; - pwj_map = nullptr; - if (nullptr == pwj_map_) { - void *buf = allocator_.alloc(sizeof(PWJTabletIdMap)); + group_pwj_map = nullptr; + if (nullptr == group_pwj_map_) { + void *buf = allocator_.alloc(sizeof(GroupPWJTabletIdMap)); if (nullptr == buf) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("Failed to allocate memories", K(ret)); - } else if (FALSE_IT(pwj_map_ = new(buf) PWJTabletIdMap())) { - } else if (OB_FAIL(pwj_map_->create(PARTITION_WISE_JOIN_TSC_HASH_BUCKET_NUM, /* assume no more than 8 table scan in a plan */ - ObModIds::OB_SQL_PX))) { - LOG_WARN("Failed to create gi task map", K(ret)); } else { - pwj_map = pwj_map_; + group_pwj_map_ = new (buf) GroupPWJTabletIdMap(); + /* assume no more than 8table scan in a plan */ + if (OB_FAIL(group_pwj_map_->create(PARTITION_WISE_JOIN_TSC_HASH_BUCKET_NUM, ObModIds::OB_SQL_PX))) { + LOG_WARN("Failed to create group_pwj_map_", K(ret)); + } else { + group_pwj_map = group_pwj_map_; + } } } else { - pwj_map = pwj_map_; + group_pwj_map = group_pwj_map_; + } + return ret; +} + +int ObExecContext::deep_copy_group_pwj_map(const GroupPWJTabletIdMap *src) +{ + int ret = OB_SUCCESS; + GroupPWJTabletIdMap *des = nullptr; + if (OB_ISNULL(src)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null"); + } else if (OB_FAIL(get_group_pwj_map(des))) { + LOG_WARN("failed to get_group_pwj_map"); + } else if (des->size() > 0) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("size should be 0", K(des->size()), K(src->size())); + } else { + FOREACH_X(iter, *src, OB_SUCC(ret)) { + const uint64_t table_id = iter->first; + const GroupPWJTabletIdInfo &group_pwj_tablet_id_info = iter->second; + if (OB_FAIL(des->set_refactored(table_id, group_pwj_tablet_id_info))) { + LOG_WARN("failed to set refactored", K(table_id)); + } + } } return ret; } @@ -1040,6 +1066,78 @@ DEFINE_GET_SERIALIZE_SIZE(ObExecContext) return len; } +int64_t ObExecContext::get_group_pwj_map_serialize_size() const +{ + int64_t len = 0; + // add serialize size for group_pwj_map_ + int64_t pwj_map_element_count = 0; + if (group_pwj_map_ != nullptr) { + pwj_map_element_count = group_pwj_map_->size(); + OB_UNIS_ADD_LEN(pwj_map_element_count); + FOREACH(iter, *group_pwj_map_) { + const uint64_t table_id = iter->first; + const GroupPWJTabletIdInfo &group_pwj_tablet_id_info = iter->second; + OB_UNIS_ADD_LEN(table_id); + OB_UNIS_ADD_LEN(group_pwj_tablet_id_info); + } + } else { + OB_UNIS_ADD_LEN(pwj_map_element_count); + } + return len; +} + +int ObExecContext::serialize_group_pwj_map(char *buf, const int64_t buf_len, int64_t &pos) const +{ + int ret = OB_SUCCESS; + // serialize group_pwj_map_ + int64_t pwj_map_element_count = 0; + if (OB_SUCC(ret)) { + if (group_pwj_map_ != nullptr) { + pwj_map_element_count = group_pwj_map_->size(); + OB_UNIS_ENCODE(pwj_map_element_count); + FOREACH_X(iter, *group_pwj_map_, OB_SUCC(ret)) { + const uint64_t table_id = iter->first; + const GroupPWJTabletIdInfo &group_pwj_tablet_id_info = iter->second; + OB_UNIS_ENCODE(table_id); + OB_UNIS_ENCODE(group_pwj_tablet_id_info); + } + } else { + OB_UNIS_ENCODE(pwj_map_element_count); + } + } + return ret; +} + +int ObExecContext::deserialize_group_pwj_map(const char *buf, const int64_t data_len, int64_t &pos) +{ + int ret = OB_SUCCESS; + // deserialize size for group_pwj_map_ + int64_t pwj_map_element_count = 0; + OB_UNIS_DECODE(pwj_map_element_count); + if (OB_SUCC(ret) && pwj_map_element_count > 0) { + GroupPWJTabletIdMap *group_pwj_map = nullptr; + uint64_t table_id; + GroupPWJTabletIdInfo group_pwj_tablet_id_info; + if (OB_FAIL(get_group_pwj_map(group_pwj_map))) { + LOG_WARN("failed to get_group_pwj_map"); + } else { + for (int64_t i = 0; i < pwj_map_element_count && OB_SUCC(ret); ++i) { + OB_UNIS_DECODE(table_id); + OB_UNIS_DECODE(group_pwj_tablet_id_info); + if (OB_FAIL(ret)) { + } else if (OB_FAIL(group_pwj_map->set_refactored(table_id, group_pwj_tablet_id_info))) { + LOG_WARN("failed to set refactored", K(table_id), K(pwj_map_element_count), + K(group_pwj_map->size())); + } + } + } + if (OB_SUCC(ret)) { + group_pwj_map_ = group_pwj_map; + } + } + return ret; +} + int ObExecContext::get_sqludt_meta_by_subschema_id(uint16_t subschema_id, ObSqlUDTMeta &udt_meta) { int ret = OB_SUCCESS; diff --git a/src/sql/engine/ob_exec_context.h b/src/sql/engine/ob_exec_context.h index cdf862ea6..e143e00d0 100644 --- a/src/sql/engine/ob_exec_context.h +++ b/src/sql/engine/ob_exec_context.h @@ -452,8 +452,13 @@ public: ObIArray& get_temp_table_ctx() { return temp_ctx_; } - int get_pwj_map(PWJTabletIdMap *&pwj_map); - PWJTabletIdMap *get_pwj_map() { return pwj_map_; } + int get_group_pwj_map(GroupPWJTabletIdMap *&group_pwj_map); + inline GroupPWJTabletIdMap *get_group_pwj_map() { return group_pwj_map_; } + int deep_copy_group_pwj_map(const GroupPWJTabletIdMap *src); + int64_t get_group_pwj_map_serialize_size() const; + int serialize_group_pwj_map(char *buf, const int64_t buf_len, int64_t &pos) const; + int deserialize_group_pwj_map(const char *buf, const int64_t data_len, int64_t &pos); + void set_partition_id_calc_type(PartitionIdCalcType calc_type) { calc_type_ = calc_type; } PartitionIdCalcType get_partition_id_calc_type() { return calc_type_; } void set_fixed_id(ObObjectID fixed_id) { fixed_id_ = fixed_id; } @@ -648,7 +653,7 @@ protected: // just for convert charset in query response result lib::MemoryContext convert_allocator_; - PWJTabletIdMap* pwj_map_; + GroupPWJTabletIdMap *group_pwj_map_; // the following two parameters only used in calc_partition_id expr PartitionIdCalcType calc_type_; ObObjectID fixed_id_; // fixed part id or fixed subpart ids diff --git a/src/sql/engine/px/ob_dfo.cpp b/src/sql/engine/px/ob_dfo.cpp index 29c8da631..8092b23ee 100644 --- a/src/sql/engine/px/ob_dfo.cpp +++ b/src/sql/engine/px/ob_dfo.cpp @@ -653,6 +653,9 @@ OB_DEF_SERIALIZE(ObPxRpcInitSqcArgs) // can reuse cache from now on (const_cast(ser_cache_)).cache_serialized_ = ser_cache_.enable_serialize_cache_; LST_DO_CODE(OB_UNIS_ENCODE, qc_order_gi_tasks_); + if (OB_SUCC(ret) && sqc_.is_fulltree()) { + ret = exec_ctx_->serialize_group_pwj_map(buf, buf_len, pos); + } LOG_TRACE("serialize sqc", K_(sqc)); LOG_DEBUG("end trace sqc args", K(pos), K(buf_len), K(this->get_serialize_size())); return ret; @@ -705,6 +708,9 @@ OB_DEF_SERIALIZE_SIZE(ObPxRpcInitSqcArgs) LST_DO_CODE(OB_UNIS_ADD_LEN, sqc_); LST_DO_CODE(OB_UNIS_ADD_LEN, qc_order_gi_tasks_); } + if (OB_SUCC(ret) && sqc_.is_fulltree()) { + len += exec_ctx_->get_group_pwj_map_serialize_size(); + } return len; } @@ -789,6 +795,9 @@ int ObPxRpcInitSqcArgs::do_deserialize(int64_t &pos, const char *net_buf, int64_ // if version of qc is old, qc_order_gi_tasks_ will not be serialized and the value will be false. qc_order_gi_tasks_ = false; LST_DO_CODE(OB_UNIS_DECODE, qc_order_gi_tasks_); + if (OB_SUCC(ret) && sqc_.is_fulltree() && pos < data_len) { + ret = exec_ctx_->deserialize_group_pwj_map(buf, data_len, pos); + } LOG_TRACE("deserialize qc order gi tasks", K(qc_order_gi_tasks_), K(sqc_), K(this)); } return ret; @@ -1008,6 +1017,12 @@ int ObPxRpcInitTaskArgs::deep_copy_assign(ObPxRpcInitTaskArgs &src, ret = OB_DESERIALIZE_ERROR; LOG_WARN("data_len and pos mismatch", K(ser_arg_len), K(ser_pos), K(des_pos), K(ret)); } + if (OB_SUCC(ret)) { + if (sqc_handler_->get_sqc_init_arg().sqc_.is_fulltree() + && nullptr != src.exec_ctx_->get_group_pwj_map()) { + exec_ctx_->deep_copy_group_pwj_map(src.exec_ctx_->get_group_pwj_map()); + } + } return ret; } diff --git a/src/sql/engine/px/ob_px_util.cpp b/src/sql/engine/px/ob_px_util.cpp index 0e0f5fe70..0c0e45784 100644 --- a/src/sql/engine/px/ob_px_util.cpp +++ b/src/sql/engine/px/ob_px_util.cpp @@ -50,6 +50,83 @@ case ERR_CODE: { \ OB_SERIALIZE_MEMBER(ObExprExtraSerializeInfo, *current_time_, *last_trace_id_, *mview_ids_, *last_refresh_scns_); +ObBaseOrderMap::~ObBaseOrderMap() +{ + int ret = OB_SUCCESS; + ClearMapFunc clear_func; + if (OB_FAIL(map_.foreach_refactored(clear_func))) { + LOG_WARN("failed to clear"); + } + map_.destroy(); + allocator_.reset(); +} + +int ObBaseOrderMap::init(int64_t count) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(map_.create(count, ObModIds::OB_SQL_PX))) { + SQL_LOG(WARN, "Failed to create hash table", K(count)); + } + return ret; +} + +int ObBaseOrderMap::add_base_partition_order(int64_t pwj_group_id, + const TabletIdArray &tablet_id_array, + const DASTabletLocIArray &dst_locations) +{ + int ret = OB_SUCCESS; + void *buf = nullptr; + ObTMArray *base_order = nullptr; + if (OB_ISNULL(buf = reinterpret_cast *>( + allocator_.alloc(sizeof(ObTMArray))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory"); + } else if (FALSE_IT(base_order = new(buf) ObTMArray())) { + } else if (OB_FAIL(base_order->reserve(dst_locations.count()))) { + LOG_WARN("fail reserve base order", K(ret), K(dst_locations.count())); + } else if (OB_FAIL(map_.set_refactored(pwj_group_id, base_order))) { + base_order->destroy(); + LOG_WARN("failed to set", K(pwj_group_id)); + } else { + for (int i = 0; i < dst_locations.count() && OB_SUCC(ret); ++i) { + for (int j = 0; j < tablet_id_array.count() && OB_SUCC(ret); ++j) { + if (dst_locations.at(i)->tablet_id_.id() == tablet_id_array.at(j)) { + if (OB_FAIL(base_order->push_back(j))) { + LOG_WARN("fail to push idx into base order", K(ret)); + } + break; + } + } + } + } + return ret; +} + +int ObBaseOrderMap::reorder_partition_as_base_order(int64_t pwj_group_id, + const TabletIdArray &tablet_id_array, + DASTabletLocIArray &dst_locations) +{ + int ret = OB_SUCCESS; + ObIArray *base_order = nullptr; + if (OB_FAIL(map_.get_refactored(pwj_group_id, base_order))) { + LOG_WARN("hash not found", K(pwj_group_id)); + } else if (base_order->count() != dst_locations.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("not match count", K(base_order->count()), K(dst_locations.count())); + } else { + int index = 0; + for (int i = 0; i < base_order->count() && OB_SUCC(ret); ++i) { + for (int j = 0; j < dst_locations.count() && OB_SUCC(ret); ++j) { + if (dst_locations.at(j)->tablet_id_.id() == tablet_id_array.at(base_order->at(i))) { + std::swap(dst_locations.at(j), dst_locations.at(index++)); + break; + } + } + } + } + return ret; +} + // 物理分布策略：对于叶子节点，dfo 分布一般直接按照数据分布来 // Note：如果 dfo 中有两个及以上的 scan，仅仅考虑第一个。并且，要求其余 scan // 的副本分布和第一个 scan 完全一致，否则报错。 @@ -997,7 +1074,10 @@ int ObPXServerAddrUtil::set_dfo_accessed_location(ObExecContext &ctx, ObDASTableLoc *dml_table_loc = nullptr; ObTableID dml_table_location_key = OB_INVALID_ID; ObTableID dml_ref_table_id = OB_INVALID_ID; - ObSEArraybase_order; + ObBaseOrderMap base_order_map; + if (OB_FAIL(base_order_map.init(max(1, scan_ops.count())))) { + LOG_WARN("Failed to init base_order_map"); + } // 处理insert op 对应的partition location信息 if (OB_FAIL(ret) || OB_ISNULL(dml_op)) { // pass @@ -1030,7 +1110,7 @@ int ObPXServerAddrUtil::set_dfo_accessed_location(ObExecContext &ctx, ret = OB_ERR_UNEXPECTED; LOG_WARN("table loc is null", K(ret)); } else if (OB_FAIL(set_sqcs_accessed_location(ctx, base_table_location_key, - dfo, base_order, table_loc, dml_op))) { + dfo, base_order_map, table_loc, dml_op))) { LOG_WARN("failed to set sqc accessed location", K(ret)); } dml_table_loc = table_loc; @@ -1060,7 +1140,7 @@ int ObPXServerAddrUtil::set_dfo_accessed_location(ObExecContext &ctx, // dml op has already set sqc.get_location information, // table scan does not need to be set again OB_ISNULL(dml_op) ? base_table_location_key : OB_INVALID_ID, - dfo, base_order, table_loc, scan_op))) { + dfo, base_order_map, table_loc, scan_op))) { LOG_WARN("failed to set sqc accessed location", K(ret), K(table_location_key), K(ref_table_id), KPC(table_loc)); } @@ -1077,13 +1157,10 @@ int ObPXServerAddrUtil::set_dfo_accessed_location(ObExecContext &ctx, return ret; } - -int ObPXServerAddrUtil::set_sqcs_accessed_location(ObExecContext &ctx, - int64_t base_table_location_key, - ObDfo &dfo, - ObIArray &base_order, - const ObDASTableLoc *table_loc, - const ObOpSpec *phy_op) +int ObPXServerAddrUtil::set_sqcs_accessed_location( + ObExecContext &ctx, int64_t base_table_location_key, ObDfo &dfo, + ObBaseOrderMap &base_order_map, + const ObDASTableLoc *table_loc, const ObOpSpec *phy_op) { int ret = OB_SUCCESS; common::ObArray sqcs; @@ -1111,7 +1188,7 @@ int ObPXServerAddrUtil::set_sqcs_accessed_location(ObExecContext &ctx, LOG_WARN("fail to get table scan partition order", K(ret)); } else if (OB_FAIL(ObPXServerAddrUtil::reorder_all_partitions(table_location_key, table_loc->get_ref_table_id(), locations, - temp_locations, asc_order, ctx, base_order))) { + temp_locations, asc_order, ctx, base_order_map))) { // 按照GI要求的访问顺序对当前SQC涉及到的分区进行排序 // 如果是partition wise join场景, 需要根据partition_wise_join要求结合GI要求做asc/desc排序 LOG_WARN("fail to reorder all partitions", K(ret)); @@ -1225,11 +1302,10 @@ private: ObTabletIdxMap *map_; }; -int ObPXServerAddrUtil::reorder_all_partitions(int64_t table_location_key, - int64_t ref_table_id, - const DASTabletLocList &src_locations, - DASTabletLocIArray &dst_locations, - bool asc, ObExecContext &exec_ctx, ObIArray &base_order) +int ObPXServerAddrUtil::reorder_all_partitions( + int64_t table_location_key, int64_t ref_table_id, const DASTabletLocList &src_locations, + DASTabletLocIArray &dst_locations, bool asc, ObExecContext &exec_ctx, + ObBaseOrderMap &base_order_map) { int ret = OB_SUCCESS; dst_locations.reset(); @@ -1237,7 +1313,7 @@ int ObPXServerAddrUtil::reorder_all_partitions(int64_t table_location_key, ObTabletIdxMap tablet_order_map; if (OB_FAIL(dst_locations.reserve(src_locations.size()))) { LOG_WARN("fail reserve locations", K(ret), K(src_locations.size())); - // virtual table is list parition now, + // virtual table is list partition now, // no actual partition define, can't traverse // table schema for partition info } else if (!is_virtual_table(ref_table_id) && @@ -1264,41 +1340,40 @@ int ObPXServerAddrUtil::reorder_all_partitions(int64_t table_location_key, ret = OB_SCHEMA_ERROR; } } - PWJTabletIdMap *pwj_map = NULL; + GroupPWJTabletIdMap *group_pwj_map = nullptr; if (OB_FAIL(ret)) { LOG_WARN("fail to sort locations", K(ret)); - } else if (OB_NOT_NULL(pwj_map = exec_ctx.get_pwj_map())) { - TabletIdArray tablet_id_array; - if (OB_FAIL(pwj_map->get_refactored(table_location_key, tablet_id_array))) { + } else if (OB_NOT_NULL(group_pwj_map = exec_ctx.get_group_pwj_map())) { + GroupPWJTabletIdInfo group_pwj_tablet_id_info; + TabletIdArray &tablet_id_array = group_pwj_tablet_id_info.tablet_id_array_; + if (OB_FAIL(group_pwj_map->get_refactored(table_location_key, group_pwj_tablet_id_info))) { if (OB_HASH_NOT_EXIST == ret) { - // map中没有意味着不需要pwj调序 + // means this is not a partition wise join table, do not need to reorder partition ret = OB_SUCCESS; - } - } else if (0 == base_order.count()) { - //TODO @yishen 在partition数量较多的情况, 使用hash map优化. - if (OB_FAIL(base_order.reserve(dst_locations.count()))) { - LOG_WARN("fail reserve base order", K(ret), K(dst_locations.count())); - } - for (int i = 0; i < dst_locations.count() && OB_SUCC(ret); ++i) { - for (int j = 0; j < tablet_id_array.count() && OB_SUCC(ret); ++j) { - if (dst_locations.at(i)->tablet_id_.id() == tablet_id_array.at(j)) { - if (OB_FAIL(base_order.push_back(j))) { - LOG_WARN("fail to push idx into base order", K(ret)); - } - break; - } - } + } else { + LOG_WARN("failed to get_refactored", K(table_location_key)); } } else { - //TODO @yishen 在partition数量较多的情况, 使用hash map优化. - int index = 0; - for (int i = 0; i < base_order.count() && OB_SUCC(ret); ++i) { - for (int j = 0; j < dst_locations.count() && OB_SUCC(ret); ++j) { - if (dst_locations.at(j)->tablet_id_.id() == tablet_id_array.at(base_order.at(i))) { - std::swap(dst_locations.at(j), dst_locations.at(index++)); - break; + // set base order or reorder partition as base order + uint64_t pwj_group_id = group_pwj_tablet_id_info.group_id_; + ObIArray *base_order = nullptr; + if (OB_FAIL(base_order_map.get_map().get_refactored(pwj_group_id, base_order))) { + if (ret == OB_HASH_NOT_EXIST) { + ret = base_order_map.add_base_partition_order(pwj_group_id, tablet_id_array, + dst_locations); + if (ret != OB_SUCCESS) { + LOG_WARN("failed to add_base_partition_order"); + } else { + LOG_TRACE("succ to add_base_partition_order", K(pwj_group_id), K(table_location_key)); } + } else { + LOG_WARN("failed to get_refactored"); } + } else if (OB_FAIL(base_order_map.reorder_partition_as_base_order( + pwj_group_id, tablet_id_array, dst_locations))) { + LOG_WARN("failed to reorder_partition_as_base_order"); + } else { + LOG_TRACE("succ to reorder_partition_as_base_order", K(pwj_group_id), K(table_location_key)); } } } diff --git a/src/sql/engine/px/ob_px_util.h b/src/sql/engine/px/ob_px_util.h index 0c6faf08c..870f0e25c 100644 --- a/src/sql/engine/px/ob_px_util.h +++ b/src/sql/engine/px/ob_px_util.h @@ -60,6 +60,34 @@ public: common::ObFixedArray *last_refresh_scns_; }; +class ObBaseOrderMap +{ +public: + struct ClearMapFunc + { + int operator()(const hash::HashMapPair *> &entry) { + entry.second->destroy(); + return OB_SUCCESS; + } + }; + ObBaseOrderMap() { + } + ~ObBaseOrderMap(); + int init(int64_t count); + inline hash::ObHashMap *, hash::NoPthreadDefendMode> &get_map() + { + return map_; + } + int add_base_partition_order(int64_t pwj_group_id, const TabletIdArray &tablet_id_array, + const DASTabletLocIArray &dst_locations); + int reorder_partition_as_base_order(int64_t pwj_group_id, + const TabletIdArray &tablet_id_array, + DASTabletLocIArray &dst_locations); +private: + ObArenaAllocator allocator_; + hash::ObHashMap *, hash::NoPthreadDefendMode> map_; +}; + class ObPxSqcUtil { public: @@ -182,11 +210,10 @@ private: int64_t tenant_id, uint64_t ref_table_id, ObTabletIdxMap &idx_map); - static int reorder_all_partitions(int64_t location_key, - int64_t ref_table_id, - const DASTabletLocList &src_locations, - DASTabletLocIArray &tsc_locations, - bool asc, ObExecContext &exec_ctx, ObIArray &base_order); + static int reorder_all_partitions( + int64_t location_key, int64_t ref_table_id, const DASTabletLocList &src_locations, + DASTabletLocIArray &tsc_locations, bool asc, ObExecContext &exec_ctx, + ObBaseOrderMap &base_order_map); static int build_dynamic_partition_table_location(common::ObIArray &scan_ops, const ObIArray *table_locations, ObDfo &dfo); @@ -215,12 +242,10 @@ private: * Add the partition information (table_loc) involved in the * current phy_op to the corresponding SQC access location */ - static int set_sqcs_accessed_location(ObExecContext &ctx, - int64_t base_table_location_key, - ObDfo &dfo, - ObIArray &base_order, - const ObDASTableLoc *table_loc, - const ObOpSpec *phy_op); + static int set_sqcs_accessed_location( + ObExecContext &ctx, int64_t base_table_location_key, ObDfo &dfo, + ObBaseOrderMap &base_order_map, + const ObDASTableLoc *table_loc, const ObOpSpec *phy_op); /** * Get the access sequence of the partition of the current phy_op, * the access sequence of the phy_op partition is determined by diff --git a/src/sql/optimizer/ob_log_plan.cpp b/src/sql/optimizer/ob_log_plan.cpp index ee136441e..11723a8f3 100644 --- a/src/sql/optimizer/ob_log_plan.cpp +++ b/src/sql/optimizer/ob_log_plan.cpp @@ -11494,15 +11494,11 @@ int ObLogPlan::calc_and_set_exec_pwj_map(ObLocationConstraintContext &location_c ObIArray &base_location_cons = location_constraint.base_table_constraints_; ObIArray &strict_cons = location_constraint.strict_constraints_; const int64_t tbl_count = location_constraint.base_table_constraints_.count(); - ObSEArray pwj_tables; SMART_VAR(ObStrictPwjComparer, strict_pwj_comparer) { PWJTabletIdMap pwj_map; - if (OB_FAIL(pwj_tables.prepare_allocate(tbl_count))) { - LOG_WARN("failed to prepare allocate pwj tables", K(ret)); - } else if (OB_FAIL(pwj_map.create(8, ObModIds::OB_PLAN_EXECUTE))) { + if (OB_FAIL(pwj_map.create(8, ObModIds::OB_PLAN_EXECUTE))) { LOG_WARN("create pwj map failed", K(ret)); } - for (int64_t i = 0; OB_SUCC(ret) && i < strict_cons.count(); ++i) { const ObPwjConstraint *pwj_cons = strict_cons.at(i); if (OB_ISNULL(pwj_cons) || OB_UNLIKELY(pwj_cons->count() <= 1)) { @@ -11515,25 +11511,32 @@ int ObLogPlan::calc_and_set_exec_pwj_map(ObLocationConstraintContext &location_c } if (OB_SUCC(ret)) { - PWJTabletIdMap *exec_pwj_map = NULL; - if (OB_FAIL(exec_ctx->get_pwj_map(exec_pwj_map))) { - LOG_WARN("failed to get exec pwj map", K(ret)); - } else if (OB_FAIL(exec_pwj_map->reuse())) { - LOG_WARN("failed to reuse pwj map", K(ret)); + GroupPWJTabletIdMap *group_pwj_map = nullptr; + if (OB_FAIL(exec_ctx->get_group_pwj_map(group_pwj_map))) { + LOG_WARN("failed to get exec group pwj map", K(ret)); + } else if (OB_FAIL(group_pwj_map->reuse())) { + LOG_WARN("failed to reuse group pwj map", K(ret)); } - for (int64_t i = 0; OB_SUCC(ret) && i < base_location_cons.count(); ++i) { - if (!base_location_cons.at(i).is_multi_part_insert()) { - TabletIdArray tablet_id_array; - if (OB_FAIL(pwj_map.get_refactored(i, tablet_id_array))) { - if (OB_HASH_NOT_EXIST == ret) { - // 没找到说明当前表不需要做partition wise join - ret = OB_SUCCESS; - } else { - LOG_WARN("failed to get refactored", K(ret)); + GroupPWJTabletIdInfo group_pwj_tablet_id_info; + TabletIdArray &tablet_id_array = group_pwj_tablet_id_info.tablet_id_array_; + for (int64_t group_id = 0; OB_SUCC(ret) && group_id < strict_cons.count(); ++group_id) { + group_pwj_tablet_id_info.group_id_ = group_id; + const ObPwjConstraint *pwj_cons = strict_cons.at(group_id); + for (int64_t i = 0; OB_SUCC(ret) && i < pwj_cons->count(); ++i) { + const int64_t table_idx = pwj_cons->at(i); + uint64_t table_id = base_location_cons.at(table_idx).key_.table_id_; + tablet_id_array.reset(); + if (!base_location_cons.at(table_idx).is_multi_part_insert()) { + if (OB_FAIL(pwj_map.get_refactored(table_idx, tablet_id_array))) { + if (OB_HASH_NOT_EXIST == ret) { + // means this is not a partition wise join table + ret = OB_SUCCESS; + } else { + LOG_WARN("failed to get refactored", K(ret)); + } + } else if (OB_FAIL(group_pwj_map->set_refactored(table_id, group_pwj_tablet_id_info))) { + LOG_WARN("failed to set refactored", K(ret)); } - } else if (OB_FAIL(exec_pwj_map->set_refactored(base_location_cons.at(i).key_.table_id_, - tablet_id_array))) { - LOG_WARN("failed to set refactored", K(ret)); } } } diff --git a/src/sql/optimizer/ob_pwj_comparer.cpp b/src/sql/optimizer/ob_pwj_comparer.cpp index 2cd7d26a0..060cd1f03 100644 --- a/src/sql/optimizer/ob_pwj_comparer.cpp +++ b/src/sql/optimizer/ob_pwj_comparer.cpp @@ -110,6 +110,8 @@ int PwjTable::assign(const PwjTable &other) return ret; } +OB_SERIALIZE_MEMBER(GroupPWJTabletIdInfo, group_id_, tablet_id_array_); + void ObPwjComparer::reset() { pwj_tables_.reset(); diff --git a/src/sql/optimizer/ob_pwj_comparer.h b/src/sql/optimizer/ob_pwj_comparer.h index 69ec3099c..8b7c76f75 100644 --- a/src/sql/optimizer/ob_pwj_comparer.h +++ b/src/sql/optimizer/ob_pwj_comparer.h @@ -83,9 +83,31 @@ struct PwjTable { common::ObSEArray all_subpartition_indexes_; }; -// TODO yibo 用PartitionIdArray的指针作为value, 否则每次get都要拷贝一次array typedef common::ObSEArray TabletIdArray; + +struct GroupPWJTabletIdInfo { + OB_UNIS_VERSION(1); +public: + TO_STRING_KV(K_(group_id), K_(tablet_id_array)); + /* + for union all non strict partition wise join, there may be several partition wise join groups, + for example: + union all + | | + join join + | | | | + t1(p0-p15) t2(p0-p15) t3(p0-p8) t4(p0-p8) + + t1 and t2 are in group 0 + t3 and t4 are in group 1 + */ + int64_t group_id_{0}; + TabletIdArray tablet_id_array_; +}; + +// TODO yibo 用PartitionIdArray的指针作为value, 否则每次get都要拷贝一次array typedef common::hash::ObHashMap PWJTabletIdMap; +typedef common::hash::ObHashMap GroupPWJTabletIdMap; typedef common::hash::ObHashMap, common::hash::equal_to, diff --git a/src/sql/plan_cache/ob_plan_match_helper.cpp b/src/sql/plan_cache/ob_plan_match_helper.cpp index 4bab7d9f3..65813672c 100644 --- a/src/sql/plan_cache/ob_plan_match_helper.cpp +++ b/src/sql/plan_cache/ob_plan_match_helper.cpp @@ -82,32 +82,31 @@ int ObPlanMatchHelper::match_plan(const ObPlanCacheCtx &pc_ctx, use_pwj_map = true; } - if (OB_SUCC(ret) && is_matched) { - PWJTabletIdMap *exec_pwj_map = NULL; - ObDASCtx &das_ctx = DAS_CTX(pc_ctx.exec_ctx_); - if (use_pwj_map) { - if (OB_FAIL(pc_ctx.exec_ctx_.get_pwj_map(exec_pwj_map))) { - LOG_WARN("failed to get exec pwj map", K(ret)); - } else if (OB_FAIL(exec_pwj_map->reuse())) { - LOG_WARN("failed to reuse pwj map", K(ret)); - } + if (OB_SUCC(ret) && is_matched && use_pwj_map) { + GroupPWJTabletIdMap *exec_group_pwj_map = nullptr; + if (OB_FAIL(pc_ctx.exec_ctx_.get_group_pwj_map(exec_group_pwj_map))) { + LOG_WARN("failed to get exec group pwj map", K(ret)); + } else if (OB_FAIL(exec_group_pwj_map->reuse())) { + LOG_WARN("failed to reuse pwj map", K(ret)); } - for (int64_t i = 0; OB_SUCC(ret) && i < base_cons.count(); ++i) { - // in the case of multi part insert, only the location constraint is matched, but the - // corresponding phy table location information does not need to be added to table_locs - if (!base_cons.at(i).is_multi_part_insert()) { - ObCandiTableLoc &src_location = phy_tbl_infos.at(i); - if (use_pwj_map) { - TabletIdArray tablet_id_array; - if (OB_FAIL(pwj_map.get_refactored(i, tablet_id_array))) { + GroupPWJTabletIdInfo group_pwj_tablet_id_info; + TabletIdArray &tablet_id_array = group_pwj_tablet_id_info.tablet_id_array_; + for (int64_t group_id = 0; OB_SUCC(ret) && group_id < strict_cons.count(); ++group_id) { + group_pwj_tablet_id_info.group_id_ = group_id; + const ObPlanPwjConstraint &pwj_cons = strict_cons.at(group_id); + for (int64_t i = 0; OB_SUCC(ret) && i < pwj_cons.count(); ++i) { + const int64_t table_idx = pwj_cons.at(i); + uint64_t table_id = base_cons.at(table_idx).key_.table_id_; + tablet_id_array.reset(); + if (!base_cons.at(table_idx).is_multi_part_insert()) { + if (OB_FAIL(pwj_map.get_refactored(table_idx, tablet_id_array))) { if (OB_HASH_NOT_EXIST == ret) { - // 没找到说明当前表不需要做partition wise join + // means this is not a partition wise join table ret = OB_SUCCESS; } else { LOG_WARN("failed to get refactored", K(ret)); } - } else if (OB_FAIL(exec_pwj_map->set_refactored(base_cons.at(i).key_.table_id_, - tablet_id_array))) { + } else if (OB_FAIL(exec_group_pwj_map->set_refactored(table_id, group_pwj_tablet_id_info))) { LOG_WARN("failed to set refactored", K(ret)); } } From 0c96e4258dae6b74fef0a1c07db1ff6af7708692 Mon Sep 17 00:00:00 2001 From: HaHaJeff Date: Mon, 26 Aug 2024 10:23:39 +0000 Subject: [PATCH 225/249] [CP] fixed partial read failure return OB_SUCCESS --- src/logservice/ob_log_external_storage_io_task.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/logservice/ob_log_external_storage_io_task.cpp b/src/logservice/ob_log_external_storage_io_task.cpp index dbdd0f704..17fdcd055 100644 --- a/src/logservice/ob_log_external_storage_io_task.cpp +++ b/src/logservice/ob_log_external_storage_io_task.cpp @@ -129,7 +129,7 @@ int ObLogExternalStorageIOTaskCtx::get_ret_code() const { int ret = OB_SUCCESS; for (int64_t i = 0; i < total_task_count_; i++) { - if (0 == i && OB_SUCCESS != running_status_[i].ret_) { + if (OB_SUCCESS != running_status_[i].ret_) { ret = running_status_[i].ret_; CLOG_LOG(WARN, "asyn task execute failed", KPC(this)); break; From 726029ea3c353cbafc39dc1edacd8ef1fb56f796 Mon Sep 17 00:00:00 2001 From: yinyj17 Date: Mon, 26 Aug 2024 10:29:29 +0000 Subject: [PATCH 226/249] [CP] fix privilege check for dml stmt is to strict --- .../privilege_check/ob_privilege_check.cpp | 73 +++++++++++-------- 1 file changed, 44 insertions(+), 29 deletions(-) diff --git a/src/sql/privilege_check/ob_privilege_check.cpp b/src/sql/privilege_check/ob_privilege_check.cpp index d1d36fb19..62fdf31e7 100644 --- a/src/sql/privilege_check/ob_privilege_check.cpp +++ b/src/sql/privilege_check/ob_privilege_check.cpp @@ -385,10 +385,15 @@ int add_col_priv_to_need_priv( visitor.remove_scope(SCOPE_DML_CONSTRAINT); visitor.remove_scope(SCOPE_DMLINFOS); ObSEArray col_exprs; + bool has_dml_info = false; if (OB_ISNULL(basic_stmt)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("basic_stmt is NULL", K(ret)); - } else { + } else if (basic_stmt->is_dml_write_stmt() && + OB_FAIL(static_cast(basic_stmt)->has_dml_table_info( + table_item.table_id_, has_dml_info))) { + LOG_WARN("failed to check has dml table info", K(ret)); + } else if (has_dml_info) { stmt::StmtType stmt_type = basic_stmt->get_stmt_type(); switch (stmt_type) { case stmt::T_DELETE: { @@ -504,39 +509,39 @@ int add_col_priv_to_need_priv( break; } } - if (OB_SUCC(ret)) { - ObSEArray rel_exprs; - need_priv.priv_set_ = OB_PRIV_SELECT; - if (OB_FAIL(static_cast(basic_stmt)->get_relation_exprs(rel_exprs, visitor))) { - LOG_WARN("get rel exprs failed", K(ret)); - } else if (OB_FAIL(ObRawExprUtils::extract_column_exprs(rel_exprs, col_exprs))) { - LOG_WARN("extract column exprs failed", K(ret)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < col_exprs.count(); i++) { - if (OB_ISNULL(col_exprs.at(i)) || OB_UNLIKELY(!col_exprs.at(i)->is_column_ref_expr())) { + } + if (OB_SUCC(ret)) { + ObSEArray rel_exprs; + need_priv.priv_set_ = OB_PRIV_SELECT; + if (OB_FAIL(static_cast(basic_stmt)->get_relation_exprs(rel_exprs, visitor))) { + LOG_WARN("get rel exprs failed", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::extract_column_exprs(rel_exprs, col_exprs))) { + LOG_WARN("extract column exprs failed", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < col_exprs.count(); i++) { + if (OB_ISNULL(col_exprs.at(i)) || OB_UNLIKELY(!col_exprs.at(i)->is_column_ref_expr())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error", K(ret)); + } else { + ObColumnRefRawExpr *col_expr = static_cast(col_exprs.at(i)); + if (OB_ISNULL(col_expr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected error", K(ret)); - } else { - ObColumnRefRawExpr *col_expr = static_cast(col_exprs.at(i)); - if (OB_ISNULL(col_expr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error", K(ret)); - } else if (col_expr->get_table_id() == table_id && col_expr->get_column_id() >= OB_APP_MIN_COLUMN_ID) { - OZ (need_priv.columns_.push_back(col_expr->get_column_name())); - } + } else if (col_expr->get_table_id() == table_id && col_expr->get_column_id() >= OB_APP_MIN_COLUMN_ID) { + OZ (need_priv.columns_.push_back(col_expr->get_column_name())); } } - if (OB_SUCC(ret)) { - if (need_priv.columns_.empty()) { - if (basic_stmt->is_select_stmt()) { - need_priv.check_any_column_priv_ = true; - ADD_NEED_PRIV(need_priv); - need_priv.check_any_column_priv_ = false; - } - } else { + } + if (OB_SUCC(ret)) { + if (need_priv.columns_.empty()) { + if (basic_stmt->is_select_stmt()) { + need_priv.check_any_column_priv_ = true; ADD_NEED_PRIV(need_priv); - need_priv.columns_.reuse(); + need_priv.check_any_column_priv_ = false; } + } else { + ADD_NEED_PRIV(need_priv); + need_priv.columns_.reuse(); } } } @@ -1170,7 +1175,6 @@ int get_dml_stmt_need_privs( || table_item->is_view_table_) { need_priv.db_ = table_item->database_name_; need_priv.table_ = table_item->table_name_; - need_priv.priv_set_ = priv_set; need_priv.is_sys_table_ = table_item->is_system_table_; need_priv.is_for_update_ = table_item->for_update_; need_priv.priv_level_ = OB_PRIV_TABLE_LEVEL; @@ -1187,6 +1191,17 @@ int get_dml_stmt_need_privs( table_item->table_name_.length(), table_item->table_name_.ptr()); } } + if (OB_SUCC(ret)) { + bool has = false; + if (stmt::T_SELECT == dml_stmt->get_stmt_type()) { + need_priv.priv_set_ = priv_set; + } else if (OB_FAIL(static_cast(dml_stmt)->has_dml_table_info( + table_item->table_id_, has))) { + LOG_WARN("failed to check has dml table info", K(ret)); + } else { + need_priv.priv_set_ = has ? priv_set : OB_PRIV_SELECT; + } + } if (OB_SUCC(ret)) { if (session_priv.is_tenant_changed() && 0 != table_item->database_name_.case_compare(OB_SYS_DATABASE_NAME)) { From bfba7eb3fd5f0e4740722d1933e8d49a3f9a58f9 Mon Sep 17 00:00:00 2001 From: dontknow9179 <545187809@qq.com> Date: Mon, 26 Aug 2024 10:48:14 +0000 Subject: [PATCH 227/249] [CP] bugfix: partitioned external table not add pwj constraints --- src/sql/engine/basic/ob_select_into_op.cpp | 2 +- src/sql/engine/basic/ob_select_into_op.h | 4 ---- src/sql/optimizer/ob_logical_operator.cpp | 2 ++ 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/sql/engine/basic/ob_select_into_op.cpp b/src/sql/engine/basic/ob_select_into_op.cpp index efb13d6c0..b16d1d091 100644 --- a/src/sql/engine/basic/ob_select_into_op.cpp +++ b/src/sql/engine/basic/ob_select_into_op.cpp @@ -31,7 +31,7 @@ OB_SERIALIZE_MEMBER((ObSelectIntoSpec, ObOpSpec), into_type_, user_vars_, outfil field_str_, // FARM COMPAT WHITELIST FOR filed_str_: renamed line_str_, closed_cht_, is_optional_, select_exprs_, is_single_, max_file_size_, escaped_cht_, cs_type_, parallel_, file_partition_expr_, buffer_size_, is_overwrite_, - external_properties_, external_partition_, per_row_group_size_, compression_algorithm_); + external_properties_, external_partition_); int ObSelectIntoOp::inner_open() diff --git a/src/sql/engine/basic/ob_select_into_op.h b/src/sql/engine/basic/ob_select_into_op.h index 0fc024a67..f6e9a4bff 100644 --- a/src/sql/engine/basic/ob_select_into_op.h +++ b/src/sql/engine/basic/ob_select_into_op.h @@ -73,8 +73,6 @@ public: external_properties_(alloc), external_partition_(alloc) { - compression_algorithm_ = 0; /* parquet::Compression::UNCOMPRESSED */ - per_row_group_size_ = DEFAULT_MAX_FILE_SIZE; cs_type_ = ObCharset::get_system_collation(); } @@ -97,8 +95,6 @@ public: bool is_overwrite_; ObExternalFileFormat::StringData external_properties_; ObExternalFileFormat::StringData external_partition_; - int64_t per_row_group_size_; - int64_t compression_algorithm_; static const int64_t DEFAULT_MAX_FILE_SIZE = 256LL * 1024 * 1024; static const int64_t DEFAULT_BUFFER_SIZE = 1LL * 1024 * 1024; }; diff --git a/src/sql/optimizer/ob_logical_operator.cpp b/src/sql/optimizer/ob_logical_operator.cpp index 4546a45c6..a272a159b 100644 --- a/src/sql/optimizer/ob_logical_operator.cpp +++ b/src/sql/optimizer/ob_logical_operator.cpp @@ -2678,6 +2678,8 @@ int ObLogicalOperator::gen_location_constraint(void *ctx) LOG_WARN("failed to push back location constraint", K(ret)); } else if (OB_FAIL(loc_cons_ctx->base_table_constraints_.push_back(loc_cons))) { LOG_WARN("failed to push back location constraint", K(ret)); + } else if (EXTERNAL_TABLE == log_scan_op->get_table_type()) { + // do not add pwj constraints for external table } else if (OB_FAIL(strict_pwj_constraint_.push_back( loc_cons_ctx->base_table_constraints_.count() - 1))) { LOG_WARN("failed to push back location constraint offset", K(ret)); From 4713cafd8515663309986e2b0c1abe065943f7e6 Mon Sep 17 00:00:00 2001 From: akaError Date: Mon, 26 Aug 2024 11:18:58 +0000 Subject: [PATCH 228/249] fix latin leng_sp bug --- deps/oblib/src/lib/charset/ob_ctype_latin1.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/oblib/src/lib/charset/ob_ctype_latin1.cc b/deps/oblib/src/lib/charset/ob_ctype_latin1.cc index 173d67f1a..be9936c2c 100644 --- a/deps/oblib/src/lib/charset/ob_ctype_latin1.cc +++ b/deps/oblib/src/lib/charset/ob_ctype_latin1.cc @@ -47,7 +47,7 @@ static ObCharsetHandler ob_charset_latin1_handler= ob_charpos_8bit, ob_max_bytes_charpos_8bit, ob_well_formed_len_8bit, - ob_lengthsp_binary, + ob_lengthsp_8bit, //ob_numcells_8bit, ob_mb_wc_latin1, ob_wc_mb_latin1, From 24f761a6da2940473376af30a63cdc98c341ff3a Mon Sep 17 00:00:00 2001 From: dongb0 <708848999@qq.com> Date: Tue, 27 Aug 2024 08:12:59 +0000 Subject: [PATCH 229/249] fix tmp file flush manager insert empty flush context into file_ctx_hash unexpectedly --- .../tmp_file/ob_tmp_file_flush_manager.cpp | 44 +++++++++++++++---- .../tmp_file/ob_tmp_file_flush_manager.h | 1 + 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/src/storage/tmp_file/ob_tmp_file_flush_manager.cpp b/src/storage/tmp_file/ob_tmp_file_flush_manager.cpp index e8f21780d..ddb7575ba 100644 --- a/src/storage/tmp_file/ob_tmp_file_flush_manager.cpp +++ b/src/storage/tmp_file/ob_tmp_file_flush_manager.cpp @@ -990,16 +990,10 @@ int ObTmpFileFlushManager::update_meta_data_after_flush_for_files_(ObTmpFileFlus } else if (OB_FAIL(file->update_meta_after_flush(flush_info.batch_flush_idx_, is_meta, reset_ctx))){ STORAGE_LOG(WARN, "fail to update meta data", KR(ret), K(is_meta), K(flush_info)); } else { - if (reset_ctx) { + if (reset_ctx && flush_task.get_flush_seq() == flush_ctx_.get_flush_sequence()) { int tmp_ret = OB_SUCCESS; - // reset data/meta flush ctx after file update meta complete to prevent - // flush use stale flushed_page_id to copy data after the page is evicted. - // use empty ctx here because we have inserted ctx for every file when flushing begins. - ResetFlushCtxOp reset_op(is_meta); - ObTmpFileSingleFlushContext empty_ctx; - if (flush_task.get_flush_seq() == flush_ctx_.get_flush_sequence() && - OB_TMP_FAIL(flush_ctx_.get_file_ctx_hash().set_or_update(file->get_fd(), empty_ctx, reset_op))) { - STORAGE_LOG(ERROR, "fail to clean file ctx from hash", KR(tmp_ret), K(file)); + if (OB_TMP_FAIL(reset_flush_ctx_for_file_(file, is_meta))) { + STORAGE_LOG(WARN, "fail to reset flush ctx", KR(tmp_ret), K(file)); } } flush_info.update_meta_data_done_ = true; @@ -1009,5 +1003,37 @@ int ObTmpFileFlushManager::update_meta_data_after_flush_for_files_(ObTmpFileFlus return ret; } +// reset data/meta flush ctx after file update meta complete to prevent +// flush use stale flushed_page_id to copy data after the page is evicted +int ObTmpFileFlushManager::reset_flush_ctx_for_file_(const ObSharedNothingTmpFile *file, const bool is_meta) +{ + int ret = OB_SUCCESS; + ObTmpFileSingleFlushContext update_ctx; + if (OB_ISNULL(file)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(ERROR, "tmp file ptr is null", KR(ret), K(file)); + } else if (OB_FAIL(flush_ctx_.get_file_ctx_hash().get_refactored(file->get_fd(), update_ctx))) { + if (OB_HASH_NOT_EXIST == ret) { + // do nothing + } else { + STORAGE_LOG(WARN, "fail to get file ctx from hash", KR(ret), K(file)); + } + } else if (OB_ISNULL(update_ctx.file_handle_.get())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(ERROR, "file handle in flush context is null", KR(ret), K(file)); + } else { + if (is_meta) { + update_ctx.meta_ctx_.reset(); + } else { + update_ctx.data_ctx_.reset(); + } + + if (OB_FAIL(flush_ctx_.get_file_ctx_hash().set_refactored(file->get_fd(), update_ctx, 1/*cover_object*/))) { + STORAGE_LOG(ERROR, "fail to set file ctx into hash", KR(ret), K(file)); + } + } + return ret; +} + } // end namespace tmp_file } // end namespace oceanbase diff --git a/src/storage/tmp_file/ob_tmp_file_flush_manager.h b/src/storage/tmp_file/ob_tmp_file_flush_manager.h index d52b0407a..6f9501470 100644 --- a/src/storage/tmp_file/ob_tmp_file_flush_manager.h +++ b/src/storage/tmp_file/ob_tmp_file_flush_manager.h @@ -106,6 +106,7 @@ private: int handle_finish_(ObTmpFileFlushTask &flush_task); private: int update_meta_data_after_flush_for_files_(ObTmpFileFlushTask &flush_task); + int reset_flush_ctx_for_file_(const ObSharedNothingTmpFile *file, const bool is_meta); int get_or_create_file_in_ctx_(const int64_t fd, ObTmpFileSingleFlushContext &file_flush_ctx); int evict_pages_and_retry_insert_(ObTmpFileFlushTask &flush_task, ObTmpFileFlushInfo &flush_info, From 4edb7b4b37ad607a664ff07d5dd5b989849a8309 Mon Sep 17 00:00:00 2001 From: mjhmllover Date: Tue, 27 Aug 2024 08:31:08 +0000 Subject: [PATCH 230/249] [FEAT MERGE] DML stong type refactoring and optimization --- .../test_sstable_row_multi_scanner.cpp | 5 - mittest/mtlenv/storage/test_co_merge.cpp | 39 +- mittest/mtlenv/storage/test_lob_manager.cpp | 4 +- mittest/mtlenv/storage/test_memtable_v2.cpp | 101 +- .../test_multi_version_merge_recycle.cpp | 24 +- .../test_multi_version_sstable_merge.cpp | 81 +- .../test_table_scan_pure_data_table.cpp | 4 +- mittest/mtlenv/storage/test_trans.cpp | 4 +- .../test_callbacks_with_reverse_order.cpp | 2 +- .../table/ob_table_modify_executor.cpp | 4 +- .../ob_table_load_store_trans_px_writer.cpp | 10 +- .../ob_table_load_store_trans_px_writer.h | 2 +- .../table_load/ob_table_load_trans_store.cpp | 25 +- .../table_load/ob_table_load_trans_store.h | 2 +- src/share/datum/ob_datum.h | 10 +- src/sql/das/ob_das_dml_ctx_define.cpp | 90 +- src/sql/das/ob_das_dml_ctx_define.h | 40 +- src/sql/das/ob_das_domain_utils.cpp | 199 ++- src/sql/das/ob_das_domain_utils.h | 6 +- src/sql/das/ob_das_insert_op.cpp | 39 +- src/sql/das/ob_das_insert_op.h | 23 +- src/sql/das/ob_das_update_op.cpp | 21 +- src/sql/das/ob_das_utils.cpp | 47 +- src/sql/das/ob_das_utils.h | 4 +- src/sql/engine/dml/ob_table_insert_up_op.cpp | 4 +- src/sql/engine/dml/ob_table_replace_op.cpp | 4 +- .../expr/ob_expr_pl_get_cursor_attr.cpp | 2 +- src/sql/engine/table/ob_table_scan_op.cpp | 55 +- src/sql/engine/table/ob_table_scan_op.h | 6 +- src/storage/CMakeLists.txt | 3 + src/storage/access/ob_rows_info.cpp | 14 +- src/storage/access/ob_rows_info.h | 6 +- .../access/ob_table_access_context.cpp | 5 +- src/storage/access/ob_table_access_context.h | 3 +- src/storage/blocksstable/ob_datum_row.cpp | 390 +---- src/storage/blocksstable/ob_datum_row.h | 389 +---- .../blocksstable/ob_datum_row_iterator.h | 98 ++ .../blocksstable/ob_datum_row_store.cpp | 242 ++++ src/storage/blocksstable/ob_datum_row_store.h | 92 ++ .../blocksstable/ob_datum_row_utils.cpp | 97 ++ src/storage/blocksstable/ob_datum_row_utils.h | 38 + src/storage/blocksstable/ob_datum_rowkey.cpp | 19 + src/storage/blocksstable/ob_datum_rowkey.h | 7 +- src/storage/blocksstable/ob_row_writer.cpp | 37 +- src/storage/blocksstable/ob_row_writer.h | 6 +- src/storage/blocksstable/ob_storage_datum.cpp | 386 +++++ src/storage/blocksstable/ob_storage_datum.h | 427 ++++++ src/storage/lob/ob_ext_info_callback.cpp | 19 +- src/storage/lob/ob_ext_info_callback.h | 11 +- src/storage/lob/ob_lob_meta.cpp | 4 +- src/storage/lob/ob_lob_meta.h | 4 +- src/storage/lob/ob_lob_persistent_adaptor.cpp | 160 +- src/storage/lob/ob_lob_persistent_adaptor.h | 29 +- .../lob/ob_lob_persistent_iterator.cpp | 24 +- src/storage/lob/ob_lob_persistent_iterator.h | 41 +- src/storage/ls/ob_ls_tablet_service.cpp | 1101 ++++++-------- src/storage/ls/ob_ls_tablet_service.h | 147 +- src/storage/memtable/mvcc/ob_mvcc_ctx.cpp | 5 +- src/storage/memtable/mvcc/ob_mvcc_ctx.h | 3 +- .../memtable/ob_concurrent_control.cpp | 2 +- src/storage/memtable/ob_memtable.cpp | 200 +-- src/storage/memtable/ob_memtable.h | 26 +- src/storage/memtable/ob_memtable_key.cpp | 128 +- src/storage/memtable/ob_memtable_key.h | 53 +- src/storage/ob_dml_running_ctx.cpp | 112 +- src/storage/ob_dml_running_ctx.h | 13 +- src/storage/ob_query_iterator_factory.cpp | 2 +- src/storage/ob_query_iterator_factory.h | 6 +- src/storage/ob_value_row_iterator.cpp | 181 ++- src/storage/ob_value_row_iterator.h | 20 +- src/storage/tablet/ob_tablet.cpp | 31 +- src/storage/tablet/ob_tablet.h | 14 +- .../tablet/ob_tablet_table_store_iterator.cpp | 1 + src/storage/tx_storage/ob_access_service.cpp | 18 +- src/storage/tx_storage/ob_access_service.h | 18 +- .../storage/blocksstable/test_row_reader.cpp | 84 +- unittest/storage/mock_access_service.cpp | 2 +- unittest/storage/mock_access_service.h | 2 +- .../mockcontainer/mock_ob_iterator.cpp | 1283 +++++++++++++---- .../storage/mockcontainer/mock_ob_iterator.h | 253 +++- unittest/storage/test_ob_mock_iterator.cpp | 8 +- unittest/storage/tx/it/tx_node.cpp | 25 +- 82 files changed, 4340 insertions(+), 2806 deletions(-) create mode 100644 src/storage/blocksstable/ob_datum_row_iterator.h create mode 100644 src/storage/blocksstable/ob_datum_row_store.cpp create mode 100644 src/storage/blocksstable/ob_datum_row_store.h create mode 100644 src/storage/blocksstable/ob_datum_row_utils.cpp create mode 100644 src/storage/blocksstable/ob_datum_row_utils.h create mode 100644 src/storage/blocksstable/ob_storage_datum.cpp create mode 100644 src/storage/blocksstable/ob_storage_datum.h diff --git a/mittest/mtlenv/storage/blocksstable/test_sstable_row_multi_scanner.cpp b/mittest/mtlenv/storage/blocksstable/test_sstable_row_multi_scanner.cpp index b7b803bac..fa958f5d0 100644 --- a/mittest/mtlenv/storage/blocksstable/test_sstable_row_multi_scanner.cpp +++ b/mittest/mtlenv/storage/blocksstable/test_sstable_row_multi_scanner.cpp @@ -769,7 +769,6 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan( // first half multi scan, second half multi get ranges.reuse(); - test_allocator.reuse(); for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) { ObDatumRowkey tmp_rowkey; mget_ranges[i].border_flag_.set_inclusive_start(); @@ -824,7 +823,6 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan( // first half multi get, second half multi scan ranges.reuse(); - test_allocator.reuse(); for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) { ObDatumRowkey tmp_rowkey; mget_ranges[i].border_flag_.set_inclusive_start(); @@ -883,7 +881,6 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan( // first one multi get, others multi scan ranges.reuse(); - test_allocator.reuse(); for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) { ObDatumRowkey tmp_rowkey; mget_ranges[i].border_flag_.set_inclusive_start(); @@ -938,7 +935,6 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan( // first one multi scan, others multi get ranges.reuse(); - test_allocator.reuse(); for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) { ObDatumRowkey tmp_rowkey; mget_ranges[i].border_flag_.set_inclusive_start(); @@ -994,7 +990,6 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan( // multi scan not exist row STORAGE_LOG(DEBUG, "multi_scan_not_exist_row"); ranges.reuse(); - test_allocator.reuse(); for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) { ObDatumRowkey tmp_rowkey; mget_ranges[i].border_flag_.set_inclusive_start(); diff --git a/mittest/mtlenv/storage/test_co_merge.cpp b/mittest/mtlenv/storage/test_co_merge.cpp index f49df085d..2b50074dd 100644 --- a/mittest/mtlenv/storage/test_co_merge.cpp +++ b/mittest/mtlenv/storage/test_co_merge.cpp @@ -620,7 +620,8 @@ TEST_F(TestCOMerge, test_merge_default_row_store_with_empty_major) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, false/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -722,7 +723,8 @@ TEST_F(TestCOMerge, test_merge_default_row_store_with_empty_major) // ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); // ObMockDirectReadIterator sstable_iter; // ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); -// ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/)); +// bool is_equal = res_iter.equals(sstable_iter, false/*cmp multi version row flag*/); +// ASSERT_TRUE(is_equal); // scanner->~ObStoreRowIterator(); // handle1.reset(); // handle2.reset(); @@ -819,7 +821,8 @@ TEST_F(TestCOMerge, test_column_store_merge_with_empty_co_table) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, *cg_read_info)); - ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, false/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); merger.reset(); @@ -946,7 +949,8 @@ TEST_F(TestCOMerge, test_co_merge_with_twice_major) ObMockIterator res_iter; res_iter.reset(); ASSERT_EQ(OB_SUCCESS, res_iter.from(result[i])); - ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, false/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); } handle1.reset(); @@ -1049,7 +1053,8 @@ TEST_F(TestCOMerge, test_co_merge_with_twice_major) ObMockIterator res_iter; res_iter.reset(); ASSERT_EQ(OB_SUCCESS, res_iter.from(new_result[i])); - ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, false/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); } } @@ -1205,7 +1210,8 @@ TEST_F(TestCOMerge, test_merge_range) ObMockIterator res_iter; res_iter.reset(); ASSERT_EQ(OB_SUCCESS, res_iter.from(result[i])); - ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, false/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); } } @@ -1355,7 +1361,8 @@ TEST_F(TestCOMerge, test_merge_range_with_open) ObMockIterator res_iter; res_iter.reset(); ASSERT_EQ(OB_SUCCESS, res_iter.from(result[i])); - ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, false/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); } } @@ -1508,7 +1515,8 @@ TEST_F(TestCOMerge, test_merge_range_with_left_open) ObMockIterator res_iter; res_iter.reset(); ASSERT_EQ(OB_SUCCESS, res_iter.from(result[i])); - ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, false/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); } } @@ -1661,7 +1669,8 @@ TEST_F(TestCOMerge, test_merge_range_with_right_open) ObMockIterator res_iter; res_iter.reset(); ASSERT_EQ(OB_SUCCESS, res_iter.from(result[i])); - ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, false/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); } } @@ -1823,7 +1832,8 @@ TEST_F(TestCOMerge, test_merge_range_left_is_min) ObMockIterator res_iter; res_iter.reset(); ASSERT_EQ(OB_SUCCESS, res_iter.from(result[i])); - ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, false/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); } } @@ -1988,7 +1998,8 @@ TEST_F(TestCOMerge, test_merge_range_with_right_max) ObMockIterator res_iter; res_iter.reset(); ASSERT_EQ(OB_SUCCESS, res_iter.from(result[i])); - ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, false/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); } } @@ -2268,7 +2279,8 @@ TEST_F(TestCOMerge, test_merge_range_is_whole_range) ObMockIterator res_iter; res_iter.reset(); ASSERT_EQ(OB_SUCCESS, res_iter.from(result[i])); - ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, false/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); } } @@ -2568,7 +2580,8 @@ TEST_F(TestCOMerge, test_rebuild_sstable) ObMockIterator res_iter; res_iter.reset(); ASSERT_EQ(OB_SUCCESS, res_iter.from(result[i])); - ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, false/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); } } diff --git a/mittest/mtlenv/storage/test_lob_manager.cpp b/mittest/mtlenv/storage/test_lob_manager.cpp index d37acf4fa..f8c4b5edf 100644 --- a/mittest/mtlenv/storage/test_lob_manager.cpp +++ b/mittest/mtlenv/storage/test_lob_manager.cpp @@ -328,7 +328,7 @@ void TestLobManager::insert_lob_piece( ASSERT_NE(nullptr, tablet); // insert rows - ObMockNewRowIterator mock_iter; + ObMockDatumRowIterator mock_iter; ObSEArray column_ids; column_ids.push_back(OB_APP_MIN_COLUMN_ID + 0); // pk column_ids.push_back(OB_APP_MIN_COLUMN_ID + 1); // c1 @@ -452,7 +452,7 @@ void TestLobManager::insert_lob_meta( ASSERT_NE(nullptr, tablet); // insert rows - ObMockNewRowIterator mock_iter; + ObMockDatumRowIterator mock_iter; ObSEArray column_ids; for (int i = 0; i < ObLobMetaUtil::LOB_META_COLUMN_CNT; i++) { column_ids.push_back(OB_APP_MIN_COLUMN_ID + i); diff --git a/mittest/mtlenv/storage/test_memtable_v2.cpp b/mittest/mtlenv/storage/test_memtable_v2.cpp index 1c64311f1..3937a4d6d 100644 --- a/mittest/mtlenv/storage/test_memtable_v2.cpp +++ b/mittest/mtlenv/storage/test_memtable_v2.cpp @@ -467,7 +467,7 @@ public: void write_tx(ObStoreCtx *wtx, ObMemtable *memtable, const int64_t snapshot, - const ObStoreRow &write_row, + ObDatumRow &write_row, const int expect_ret = OB_SUCCESS, const int64_t expire_time = 10000000000) { @@ -737,20 +737,23 @@ public: int mock_row(const int64_t key, const int64_t value, ObDatumRowkey &rowkey, - ObStoreRow &row) + ObDatumRow &row) { rowkey_datums_[0].set_int(key); rowkey_datums_[1].set_int(value); rowkey.assign(rowkey_datums_, 1); + ObStorageDatum *datum = new ObStorageDatum[2]; + datum[0].set_int(key); + datum[1].set_int(value); + ObObj *obj = new ObObj[2]; obj[0].set_int(key); obj[1].set_int(value); - row.row_val_.cells_ = obj; - row.row_val_.count_ = 2; - row.row_val_.projector_ = NULL; - row.flag_.set_flag(ObDmlFlag::DF_INSERT); + row.storage_datums_ = datum; + row.count_ = 2; + row.row_flag_.set_flag(ObDmlFlag::DF_INSERT); rowkey.store_rowkey_.assign(obj, 1); return OB_SUCCESS; @@ -758,18 +761,20 @@ public: int mock_delete(const int64_t key, ObDatumRowkey &rowkey, - ObStoreRow &row) + ObDatumRow &row) { rowkey_datums_[0].set_int(key); rowkey.assign(rowkey_datums_, 1); + ObStorageDatum *datum = new ObStorageDatum[1]; + datum[0].set_int(key); + ObObj *obj = new ObObj[1]; obj[0].set_int(key); - row.row_val_.cells_ = obj; - row.row_val_.count_ = 2; - row.row_val_.projector_ = NULL; - row.flag_.set_flag(ObDmlFlag::DF_DELETE); + row.storage_datums_ = datum; + row.count_ = 2; + row.row_flag_.set_flag(ObDmlFlag::DF_DELETE); rowkey.store_rowkey_.assign(obj, 1); return OB_SUCCESS; @@ -1240,7 +1245,7 @@ TEST_F(TestMemtableV2, test_write_read_conflict) TRANS_LOG(INFO, "######## CASE1: write row into memtable"); ObDatumRowkey rowkey; - ObStoreRow write_row; + ObDatumRow write_row; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ rowkey, @@ -1384,7 +1389,7 @@ TEST_F(TestMemtableV2, test_tx_abort) TRANS_LOG(INFO, "######## CASE1: write row into memtable"); ObDatumRowkey rowkey; - ObStoreRow write_row; + ObDatumRow write_row; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ rowkey, @@ -1497,7 +1502,7 @@ TEST_F(TestMemtableV2, test_write_write_conflict) TRANS_LOG(INFO, "######## CASE1: txn1 write row into memtable"); ObDatumRowkey rowkey; - ObStoreRow write_row; + ObDatumRow write_row; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ rowkey, @@ -1542,7 +1547,7 @@ TEST_F(TestMemtableV2, test_write_write_conflict) TRANS_LOG(INFO, "######## CASE2: txn2 write row into memtable, lock for write failed"); ObDatumRowkey rowkey2; - ObStoreRow write_row2; + ObDatumRow write_row2; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 3, /*value*/ rowkey2, @@ -1564,7 +1569,7 @@ TEST_F(TestMemtableV2, test_write_write_conflict) TRANS_LOG(INFO, "######## CASE3: txn1 write row into memtable, lock for write succeed"); ObDatumRowkey rowkey3; - ObStoreRow write_row3; + ObDatumRow write_row3; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 4, /*value*/ rowkey3, @@ -1713,7 +1718,7 @@ TEST_F(TestMemtableV2, test_lock) TRANS_LOG(INFO, "######## CASE1: txn1 lock row in memtable"); ObDatumRowkey rowkey; - ObStoreRow tmp_row; + ObDatumRow tmp_row; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ rowkey, @@ -1768,7 +1773,7 @@ TEST_F(TestMemtableV2, test_lock) TRANS_LOG(INFO, "######## CASE3: txn2 write row in memtable with lock for write failed"); ObDatumRowkey rowkey2; - ObStoreRow write_row2; + ObDatumRow write_row2; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 3, /*value*/ rowkey2, @@ -1889,7 +1894,7 @@ TEST_F(TestMemtableV2, test_lock) ObTransID write_tx_id3 = ObTransID(3); ObStoreCtx *wtx3 = start_tx(write_tx_id3); ObDatumRowkey rowkey3; - ObStoreRow write_row3; + ObDatumRow write_row3; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 4, /*value*/ rowkey3, @@ -1975,7 +1980,7 @@ TEST_F(TestMemtableV2, test_sstable_lock) is_sstable_contains_lock_ = true; ObDatumRowkey rowkey; - ObStoreRow write_row; + ObDatumRow write_row; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ rowkey, @@ -2007,8 +2012,8 @@ TEST_F(TestMemtableV2, test_rollback_to) TRANS_LOG(INFO, "######## CASE1: write row into memtable"); ObDatumRowkey rowkey; - ObStoreRow write_row; - ObStoreRow write_row2; + ObDatumRow write_row; + ObDatumRow write_row2; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ rowkey, @@ -2070,7 +2075,7 @@ TEST_F(TestMemtableV2, test_rollback_to) wtx_seq_no1 + 1 /*to*/); ObDatumRowkey rowkey3; - ObStoreRow write_row3; + ObDatumRow write_row3; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 4, /*value*/ rowkey3, @@ -2099,11 +2104,11 @@ TEST_F(TestMemtableV2, test_replay) TRANS_LOG(INFO, "######## CASE1: txn1 and txn3 write row in lmemtable"); ObDatumRowkey rowkey; - ObStoreRow write_row; + ObDatumRow write_row; ObDatumRowkey rowkey2; - ObStoreRow write_row2; + ObDatumRow write_row2; ObDatumRowkey rowkey3; - ObStoreRow write_row3; + ObDatumRow write_row3; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ @@ -2286,9 +2291,9 @@ TEST_F(TestMemtableV2, test_replay) // TRANS_LOG(INFO, "######## CASE1: txn1 write row in lmemtable"); // ObDatumRowkey rowkey; -// ObStoreRow write_row; +// ObDatumRow write_row; // ObDatumRowkey rowkey2; -// ObStoreRow write_row2; +// ObDatumRow write_row2; // EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ // 2, /*value*/ @@ -2421,11 +2426,11 @@ TEST_F(TestMemtableV2, test_compact) TRANS_LOG(INFO, "######## CASE1: txn1 and txn3 write row in lmemtable"); ObDatumRowkey rowkey; - ObStoreRow write_row; + ObDatumRow write_row; ObDatumRowkey rowkey2; - ObStoreRow write_row2; + ObDatumRow write_row2; ObDatumRowkey rowkey3; - ObStoreRow write_row3; + ObDatumRow write_row3; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ @@ -2653,11 +2658,11 @@ TEST_F(TestMemtableV2, test_compact_v2) TRANS_LOG(INFO, "######## CASE1: txn1 write two rows and txn3 write a row in memtable"); ObDatumRowkey rowkey; - ObStoreRow write_row; + ObDatumRow write_row; ObDatumRowkey rowkey2; - ObStoreRow write_row2; + ObDatumRow write_row2; ObDatumRowkey rowkey3; - ObStoreRow write_row3; + ObDatumRow write_row3; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ @@ -2820,13 +2825,13 @@ TEST_F(TestMemtableV2, test_compact_v3) TRANS_LOG(INFO, "######## CASE1: txn1 write two row and txn2 write row in lmemtable"); ObDatumRowkey rowkey; - ObStoreRow write_row; + ObDatumRow write_row; ObDatumRowkey rowkey2; - ObStoreRow write_row2; + ObDatumRow write_row2; ObDatumRowkey rowkey3; - ObStoreRow write_row3; + ObDatumRow write_row3; ObDatumRowkey rowkey4; - ObStoreRow write_row4; + ObDatumRow write_row4; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ @@ -2958,11 +2963,11 @@ TEST_F(TestMemtableV2, test_dml_flag) TRANS_LOG(INFO, "######## CASE1: txns write and row in lmemtable, test its dml flag"); ObDatumRowkey rowkey; - ObStoreRow write_row1; + ObDatumRow write_row1; ObDatumRowkey rowkey2; - ObStoreRow write_row2; + ObDatumRow write_row2; ObDatumRowkey rowkey3; - ObStoreRow write_row3; + ObDatumRow write_row3; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ @@ -3121,7 +3126,7 @@ TEST_F(TestMemtableV2, test_fast_commit) TRANS_LOG(INFO, "######## CASE1: write row into memtable and fast commit, then check result is ok"); ObDatumRowkey rowkey; - ObStoreRow write_row; + ObDatumRow write_row; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ rowkey, @@ -3241,7 +3246,7 @@ TEST_F(TestMemtableV2, test_fast_commit_with_no_delay_cleanout) TRANS_LOG(INFO, "######## CASE1: write row into memtable and not fast commit, then check result is ok"); ObDatumRowkey rowkey; - ObStoreRow write_row; + ObDatumRow write_row; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ rowkey, @@ -3369,8 +3374,8 @@ TEST_F(TestMemtableV2, test_seq_set_violation) TRANS_LOG(INFO, "######## CASE1: write row into memtable"); ObDatumRowkey rowkey; - ObStoreRow write_row; - ObStoreRow write_row2; + ObDatumRow write_row; + ObDatumRow write_row2; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ rowkey, @@ -3427,7 +3432,7 @@ TEST_F(TestMemtableV2, test_parallel_lock_with_same_txn) TRANS_LOG(INFO, "######## CASE1: lock row into memtable parallelly"); ObDatumRowkey rowkey; - ObStoreRow write_row; + ObDatumRow write_row; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ rowkey, @@ -3490,14 +3495,14 @@ TEST_F(TestMemtableV2, test_sync_log_fail_on_frozen_memtable) ObStoreCtx *tx_1 = start_tx(txid_1); int i = 1; for (; i <= 10; i++) { - ObStoreRow row1; + ObDatumRow row1; EXPECT_EQ(OB_SUCCESS, mock_row(i, i*2, rowkey, row1)); write_tx(tx_1, memtable, 10000, row1); } ObTransID txid_2 = ObTransID(2); ObStoreCtx *tx_2 = start_tx(txid_2); for (; i <= 20; i++) { - ObStoreRow row1; + ObDatumRow row1; EXPECT_EQ(OB_SUCCESS, mock_row(i, i*4, rowkey, row1)); write_tx(tx_2, memtable, 10000, row1); } diff --git a/mittest/mtlenv/storage/test_multi_version_merge_recycle.cpp b/mittest/mtlenv/storage/test_multi_version_merge_recycle.cpp index d531d5bc4..8304b48f3 100644 --- a/mittest/mtlenv/storage/test_multi_version_merge_recycle.cpp +++ b/mittest/mtlenv/storage/test_multi_version_merge_recycle.cpp @@ -305,7 +305,8 @@ TEST_F(TestMultiVersionMergeRecycle, recycle_macro) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -403,7 +404,8 @@ TEST_F(TestMultiVersionMergeRecycle, recycle_after_reuse) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -499,7 +501,8 @@ TEST_F(TestMultiVersionMergeRecycle, reuse_after_recycle) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -599,7 +602,8 @@ TEST_F(TestMultiVersionMergeRecycle, recycled_micros_after_reuse) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -708,7 +712,8 @@ TEST_F(TestMultiVersionMergeRecycle, rowkeys_across_micros) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -828,7 +833,8 @@ TEST_F(TestMultiVersionMergeRecycle, rowkeys_across_macro) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -925,7 +931,8 @@ TEST_F(TestMultiVersionMergeRecycle, recycle_macro_with_last_row) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -1025,7 +1032,8 @@ TEST_F(TestMultiVersionMergeRecycle, reuse_after_recycle_with_last) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); diff --git a/mittest/mtlenv/storage/test_multi_version_sstable_merge.cpp b/mittest/mtlenv/storage/test_multi_version_sstable_merge.cpp index 659a45e9e..3a1b47e6f 100644 --- a/mittest/mtlenv/storage/test_multi_version_sstable_merge.cpp +++ b/mittest/mtlenv/storage/test_multi_version_sstable_merge.cpp @@ -345,7 +345,8 @@ TEST_F(TestMultiVersionMerge, rowkey_cross_two_macro_and_second_macro_is_filtere ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -452,7 +453,8 @@ TEST_F(TestMultiVersionMerge, rowkey_cross_three_macro_inc_merge) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -575,7 +577,8 @@ TEST_F(TestMultiVersionMerge, uncommit_rowkey_committed_in_minor) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -667,7 +670,8 @@ TEST_F(TestMultiVersionMerge, uncommit_rowkey_in_one_macro_committed_is_last) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -787,7 +791,8 @@ TEST_F(TestMultiVersionMerge, uncommit_rowkey_in_one_macro_committed_following_l ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -913,7 +918,8 @@ TEST_F(TestMultiVersionMerge, uncommit_rowkey_in_one_macro_committed_following_s ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -1021,7 +1027,8 @@ TEST_F(TestMultiVersionMerge, rowkey_cross_three_macro_full_merge) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -1175,7 +1182,8 @@ TEST_F(TestMultiVersionMerge, test_merge_with_multi_trans) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -1337,7 +1345,8 @@ TEST_F(TestMultiVersionMerge, test_merge_with_multi_trans_can_compact) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -1504,7 +1513,8 @@ TEST_F(TestMultiVersionMerge, test_merge_with_multi_trans_can_not_compact) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -1610,7 +1620,8 @@ TEST_F(TestMultiVersionMerge, test_merge_with_macro_reused_with_shadow) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -1736,7 +1747,8 @@ TEST_F(TestMultiVersionMerge, test_merge_with_macro_reused_without_shadow) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -1820,7 +1832,8 @@ TEST_F(TestMultiVersionMerge, test_merge_with_greater_multi_version) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -1931,7 +1944,8 @@ TEST_F(TestMultiVersionMerge, test_merge_with_greater_multi_version_and_uncommit ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -2066,7 +2080,8 @@ TEST_F(TestMultiVersionMerge, test_merge_with_ghost_row) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -2151,7 +2166,8 @@ TEST_F(TestMultiVersionMerge, compare_dml_flag) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -2250,7 +2266,8 @@ TEST_F(TestMultiVersionMerge, get_last_after_reuse) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -2340,7 +2357,8 @@ TEST_F(TestMultiVersionMerge, rowkey_cross_two_macro_with_commit_scn_less_multi_ ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -2461,7 +2479,8 @@ TEST_F(TestMultiVersionMerge, rowkey_cross_macro_with_last_shadow_version_less_t ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -2562,7 +2581,8 @@ TEST_F(TestMultiVersionMerge, shadow_row_is_last_in_macro) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -2684,7 +2704,8 @@ TEST_F(TestMultiVersionMerge, rowkey_cross_macro_without_open_next_macro) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -2792,7 +2813,8 @@ TEST_F(TestMultiVersionMerge, range_cross_macro) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -2933,7 +2955,8 @@ TEST_F(TestMultiVersionMerge, test_merge_base_iter_have_ghost_row) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -3033,7 +3056,8 @@ TEST_F(TestMultiVersionMerge, test_major_range_cross_macro) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -3150,7 +3174,8 @@ TEST_F(TestMultiVersionMerge, test_trans_cross_macro_with_ghost_row) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -3272,7 +3297,8 @@ TEST_F(TestMultiVersionMerge, test_trans_cross_macro_with_ghost_row2) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -3387,7 +3413,8 @@ TEST_F(TestMultiVersionMerge, test_running_trans_cross_macro_with_abort_sql_seq) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); diff --git a/mittest/mtlenv/storage/test_table_scan_pure_data_table.cpp b/mittest/mtlenv/storage/test_table_scan_pure_data_table.cpp index 91cb6d6b1..c03e9d012 100644 --- a/mittest/mtlenv/storage/test_table_scan_pure_data_table.cpp +++ b/mittest/mtlenv/storage/test_table_scan_pure_data_table.cpp @@ -97,7 +97,7 @@ void TestTableScanPureDataTable::insert_data_to_tablet(MockObAccessService *acce ASSERT_NE(nullptr, tablet); // insert rows - ObMockNewRowIterator mock_iter; + ObMockDatumRowIterator mock_iter; ObSEArray column_ids; column_ids.push_back(OB_APP_MIN_COLUMN_ID + 0); // pk column_ids.push_back(OB_APP_MIN_COLUMN_ID + 1); // c1 @@ -105,7 +105,7 @@ void TestTableScanPureDataTable::insert_data_to_tablet(MockObAccessService *acce column_ids.push_back(OB_APP_MIN_COLUMN_ID + 3); // c3 column_ids.push_back(OB_APP_MIN_COLUMN_ID + 4); // c4 - ASSERT_EQ(OB_SUCCESS, mock_iter.from(TestDmlCommon::data_row_str)); + ASSERT_EQ(OB_SUCCESS, mock_iter.from_for_datum(TestDmlCommon::data_row_str)); transaction::ObTransService *tx_service = MTL(transaction::ObTransService*); // 1. get tx desc diff --git a/mittest/mtlenv/storage/test_trans.cpp b/mittest/mtlenv/storage/test_trans.cpp index daa2c0142..3b327c3ca 100644 --- a/mittest/mtlenv/storage/test_trans.cpp +++ b/mittest/mtlenv/storage/test_trans.cpp @@ -164,8 +164,8 @@ void TestTrans::create_ls(uint64_t tenant_id, ObLSID &ls_id, ObLS *&ls) void TestTrans::insert_rows(ObLSID &ls_id, ObTabletID &tablet_id, ObTxDesc &tx_desc, ObTxReadSnapshot snapshot, const char* ins_str) { int64_t affected_rows = 0; - ObMockNewRowIterator ins_iter; - ASSERT_EQ(OB_SUCCESS, ins_iter.from(ins_str)); + ObMockDatumRowIterator ins_iter; + ASSERT_EQ(OB_SUCCESS, ins_iter.from_for_datum(ins_str)); ObArenaAllocator allocator; share::schema::ObTableDMLParam table_dml_param(allocator); diff --git a/mittest/simple_server/test_callbacks_with_reverse_order.cpp b/mittest/simple_server/test_callbacks_with_reverse_order.cpp index 8babff814..aef6a21d8 100644 --- a/mittest/simple_server/test_callbacks_with_reverse_order.cpp +++ b/mittest/simple_server/test_callbacks_with_reverse_order.cpp @@ -36,7 +36,7 @@ int ObLSTabletService::insert_tablet_rows( const int64_t row_count, ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, - ObStoreRow *rows, + ObDatumRow *rows, ObRowsInfo &rows_info) { int ret = OB_SUCCESS; diff --git a/src/observer/table/ob_table_modify_executor.cpp b/src/observer/table/ob_table_modify_executor.cpp index b303f5adc..50610b1cd 100644 --- a/src/observer/table/ob_table_modify_executor.cpp +++ b/src/observer/table/ob_table_modify_executor.cpp @@ -321,11 +321,11 @@ int ObTableApiModifyExecutor::get_next_conflict_rowkey(DASTaskIter &task_iter, bool got_row = false; while (OB_SUCC(ret) && !got_row) { - ObNewRow *dup_row = nullptr; + ObDatumRow *dup_row = nullptr; ObChunkDatumStore::StoredRow *stored_row = nullptr; ObDASWriteBuffer::DmlShadowRow ssr; ObDASInsertOp *ins_op = static_cast(*task_iter); - ObNewRowIterator *conflict_result = ins_op->get_duplicated_result(); + ObDatumRowIterator *conflict_result = ins_op->get_duplicated_result(); const ObDASInsCtDef *ins_ctdef = static_cast(ins_op->get_ctdef()); if (OB_ISNULL(conflict_result)) { ret = OB_ERR_UNEXPECTED; diff --git a/src/observer/table_load/ob_table_load_store_trans_px_writer.cpp b/src/observer/table_load/ob_table_load_store_trans_px_writer.cpp index 97e83f2eb..a50d1d674 100644 --- a/src/observer/table_load/ob_table_load_store_trans_px_writer.cpp +++ b/src/observer/table_load/ob_table_load_store_trans_px_writer.cpp @@ -186,7 +186,7 @@ int ObTableLoadStoreTransPXWriter::check_status() return ret; } -int ObTableLoadStoreTransPXWriter::write(const ObNewRow &row) +int ObTableLoadStoreTransPXWriter::write(const blocksstable::ObDatumRow &row) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { @@ -199,11 +199,13 @@ int ObTableLoadStoreTransPXWriter::write(const ObNewRow &row) ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", KR(ret), K(row), K(column_count_)); } else { - ObNewRow new_row; + ObDatumRow new_row; if (is_heap_table_) { - new_row.assign(row.cells_ + 1, row.count_ - 1); + new_row.storage_datums_ = row.storage_datums_ + 1; + new_row.count_ = row.count_ - 1; } else { - new_row.assign(row.cells_, row.count_); + new_row.storage_datums_ = row.storage_datums_; + new_row.count_ = row.count_; } if (OB_FAIL(writer_->px_write(tablet_id_, new_row))) { LOG_WARN("fail to px write", KR(ret), K(row), K(new_row)); diff --git a/src/observer/table_load/ob_table_load_store_trans_px_writer.h b/src/observer/table_load/ob_table_load_store_trans_px_writer.h index 8743d61b2..6590e7075 100644 --- a/src/observer/table_load/ob_table_load_store_trans_px_writer.h +++ b/src/observer/table_load/ob_table_load_store_trans_px_writer.h @@ -38,7 +38,7 @@ public: ObTableLoadTransStoreWriter *writer); int prepare_write(const common::ObTabletID &tablet_id, const common::ObIArray &column_ids); - int write(const common::ObNewRow &row); + int write(const blocksstable::ObDatumRow &row); TO_STRING_KV(KP_(store_ctx), KP_(trans), diff --git a/src/observer/table_load/ob_table_load_trans_store.cpp b/src/observer/table_load/ob_table_load_trans_store.cpp index d41fb2ff8..8fe9f1dde 100644 --- a/src/observer/table_load/ob_table_load_trans_store.cpp +++ b/src/observer/table_load/ob_table_load_trans_store.cpp @@ -317,7 +317,7 @@ int ObTableLoadTransStoreWriter::write(int32_t session_id, return ret; } -int ObTableLoadTransStoreWriter::px_write(const ObTabletID &tablet_id, const ObNewRow &row) +int ObTableLoadTransStoreWriter::px_write(const ObTabletID &tablet_id, const blocksstable::ObDatumRow &row) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { @@ -330,22 +330,13 @@ int ObTableLoadTransStoreWriter::px_write(const ObTabletID &tablet_id, const ObN } else { ObTableLoadSequenceNo seq_no(0); // pdml导入的行目前不存在主键冲突，先都用一个默认的seq_no SessionContext &session_ctx = session_ctx_array_[0]; - for (int64_t i = 0; OB_SUCC(ret) && i < table_data_desc_->column_count_; ++i) { - ObStorageDatum &datum = session_ctx.datum_row_.storage_datums_[i]; - const ObObj &obj = row.cells_[i]; - if (OB_FAIL(datum.from_obj_enhance(obj))) { - LOG_WARN("fail to from obj enhance", KR(ret), K(obj)); - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(write_row_to_table_store(session_ctx.table_store_, - tablet_id, - seq_no, - session_ctx.datum_row_))) { - LOG_WARN("fail to write row", KR(ret), K(tablet_id)); - } else { - ATOMIC_AAF(&trans_ctx_->ctx_->job_stat_->store_.processed_rows_, 1); - } + if (OB_FAIL(write_row_to_table_store(session_ctx.table_store_, + tablet_id, + seq_no, + row))) { + LOG_WARN("fail to write row", KR(ret), K(tablet_id), K(row)); + } else { + ATOMIC_AAF(&trans_ctx_->ctx_->job_stat_->store_.processed_rows_, 1); } } return ret; diff --git a/src/observer/table_load/ob_table_load_trans_store.h b/src/observer/table_load/ob_table_load_trans_store.h index 338df857b..314920502 100644 --- a/src/observer/table_load/ob_table_load_trans_store.h +++ b/src/observer/table_load/ob_table_load_trans_store.h @@ -74,7 +74,7 @@ public: public: // 只在对应工作线程中调用, 串行执行 int write(int32_t session_id, const table::ObTableLoadTabletObjRowArray &row_array); - int px_write(const ObTabletID &tablet_id, const common::ObNewRow &row); + int px_write(const ObTabletID &tablet_id, const blocksstable::ObDatumRow &row); int flush(int32_t session_id); int clean_up(int32_t session_id); public: diff --git a/src/share/datum/ob_datum.h b/src/share/datum/ob_datum.h index 056c706f2..a1f90f43b 100644 --- a/src/share/datum/ob_datum.h +++ b/src/share/datum/ob_datum.h @@ -128,6 +128,7 @@ struct ObDatumPtr { const char *inner_enumset_; const ObLobCommon *lob_data_; const ObLobLocator *lob_locator_; + const ObMemLobCommon *mem_lob_; const ObObj *extend_obj_; // for extend type const ObDecimalInt *decimal_int_; }; @@ -283,6 +284,10 @@ public: return ObURowIDData(pack_, (const uint8_t *)ptr_); } inline const ObLobLocator &get_lob_locator() const { return *lob_locator_; } + inline void get_mem_lob(ObLobLocatorV2 &lob_locator) const + { + lob_locator.assign_ptr(mem_lob_, len_, has_lob_header()); + } inline const ObLobCommon &get_lob_data() const { return *lob_data_; } inline const ObDecimalInt *get_decimal_int() const { return decimal_int_; } @@ -389,11 +394,6 @@ public: ptr_ = reinterpret_cast(urowid_data.rowid_content_); pack_ = static_cast(urowid_data.rowid_len_); } - inline void set_urowid(const char *ptr, const int64_t size) - { - ptr_ = ptr; - pack_ = static_cast(size); - } inline void set_lob_locator(const ObLobLocator &value) { lob_locator_ = &value; diff --git a/src/sql/das/ob_das_dml_ctx_define.cpp b/src/sql/das/ob_das_dml_ctx_define.cpp index eb7b818de..bd841970c 100644 --- a/src/sql/das/ob_das_dml_ctx_define.cpp +++ b/src/sql/das/ob_das_dml_ctx_define.cpp @@ -19,6 +19,7 @@ #include "sql/engine/dml/ob_dml_service.h" #include "sql/engine/expr/ob_expr_lob_utils.h" #include "storage/access/ob_dml_param.h" +#include "storage/blocksstable/ob_datum_row_utils.h" namespace oceanbase { namespace sql @@ -107,7 +108,7 @@ ObDASDMLIterator::~ObDASDMLIterator() } } -int ObDASDMLIterator::get_next_domain_index_row(ObNewRow *&row) +int ObDASDMLIterator::get_next_domain_index_row(ObDatumRow *&row) { int ret = OB_SUCCESS; if (OB_ISNULL(domain_iter_) && OB_FAIL(ObDomainDMLIterator::create_domain_dml_iterator( @@ -121,7 +122,7 @@ int ObDASDMLIterator::get_next_domain_index_row(ObNewRow *&row) return ret; } -int ObDASDMLIterator::get_next_domain_index_rows(ObNewRow *&rows, int64_t &row_count) +int ObDASDMLIterator::get_next_domain_index_rows(ObDatumRow *&rows, int64_t &row_count) { int ret = OB_SUCCESS; if (OB_ISNULL(domain_iter_) && OB_FAIL(ObDomainDMLIterator::create_domain_dml_iterator( @@ -135,25 +136,25 @@ int ObDASDMLIterator::get_next_domain_index_rows(ObNewRow *&rows, int64_t &row_c return ret; } -int ObDASDMLIterator::get_next_row(ObNewRow *&row) +int ObDASDMLIterator::get_next_row(blocksstable::ObDatumRow *&datum_row) { int ret = OB_SUCCESS; - if (OB_ISNULL(cur_row_)) { - if (OB_FAIL(ob_create_row(allocator_, row_projector_->count(), cur_row_))) { - LOG_WARN("create current row failed", K(ret), K(row_projector_)); + if (OB_ISNULL(cur_datum_row_)) { + if (OB_FAIL(blocksstable::ObDatumRowUtils::ob_create_row(allocator_, row_projector_->count(), cur_datum_row_))) { + LOG_WARN("create current datum row failed", K(ret), K(row_projector_)); } else if (OB_FAIL(write_buffer_.begin(write_iter_))) { LOG_WARN("begin write iterator failed", K(ret)); } } - if (OB_SUCC(ret) && das_ctdef_->table_param_.get_data_table().is_domain_index()) { - if (OB_FAIL(get_next_domain_index_row(row))) { - if (OB_ITER_END != ret) { - LOG_WARN("get next domain index row", K(ret), K(das_ctdef_->table_param_.get_data_table())); + if (OB_SUCC(ret)) { + if (das_ctdef_->table_param_.get_data_table().is_domain_index()) { + if (OB_FAIL(get_next_domain_index_row(datum_row))) { + if (OB_ITER_END != ret) { + LOG_WARN("get next domain index row", K(ret), K(das_ctdef_->table_param_.get_data_table())); + } } - } - } else { - if (OB_SUCC(ret)) { + } else { const ObChunkDatumStore::StoredRow *sr = nullptr; if (OB_FAIL(write_iter_.get_next_row(sr))) { if (OB_ITER_END != ret) { @@ -163,23 +164,18 @@ int ObDASDMLIterator::get_next_row(ObNewRow *&row) *sr, *row_projector_, allocator_, - *cur_row_))) { + *cur_datum_row_))) { LOG_WARN("project storage row failed", K(ret)); } else { - row = cur_row_; - LOG_TRACE("get next row from dml das iterator", KPC(sr), KPC(row), K(das_ctdef_)); + datum_row = cur_datum_row_; + LOG_TRACE("get next row from dml das iterator", KPC(sr), KPC(datum_row), K(das_ctdef_)); } } } return ret; } -int ObDASDMLIterator::get_next_row() -{ - return OB_NOT_IMPLEMENT; -} - -int ObDASDMLIterator::get_next_rows(ObNewRow *&rows, int64_t &row_count) +int ObDASDMLIterator::get_next_rows(blocksstable::ObDatumRow *&rows, int64_t &row_count) { int ret = OB_SUCCESS; const bool is_domain_index = das_ctdef_->table_param_.get_data_table().is_domain_index(); @@ -193,8 +189,8 @@ int ObDASDMLIterator::get_next_rows(ObNewRow *&rows, int64_t &row_count) row_count = 1; } } else { - if (OB_ISNULL(cur_rows_)) { - if (OB_FAIL(ob_create_rows(allocator_, batch_size_, row_projector_->count(), cur_rows_))) { + if (OB_ISNULL(cur_datum_rows_)) { + if (OB_FAIL(blocksstable::ObDatumRowUtils::ob_create_rows(allocator_, batch_size_, row_projector_->count(), cur_datum_rows_))) { LOG_WARN("Failed to create rows", K(ret), K_(row_projector)); } else if (OB_FAIL(write_buffer_.begin(write_iter_))) { LOG_WARN("Failed to begin write iterator", K(ret)); @@ -215,18 +211,18 @@ int ObDASDMLIterator::get_next_rows(ObNewRow *&rows, int64_t &row_count) *sr, *row_projector_, allocator_, - cur_rows_[row_count]))) { + cur_datum_rows_[row_count]))) { LOG_WARN("Failed to project storage row", K(ret)); } else { ++row_count; - LOG_TRACE("Get next rows from dml das iterator", KPC(sr), K(cur_rows_[row_count - 1]), K_(das_ctdef)); + LOG_TRACE("Get next rows from dml das iterator", KPC(sr), K(cur_datum_rows_[row_count - 1]), K_(das_ctdef)); } } if (OB_SUCC(ret) || OB_LIKELY(OB_ITER_END == ret)) { if (0 == row_count) { ret = OB_ITER_END; } else { - rows = cur_rows_; + rows = cur_datum_rows_; ret = OB_SUCCESS; } } @@ -238,8 +234,8 @@ int ObDASDMLIterator::get_next_rows(ObNewRow *&rows, int64_t &row_count) int ObDASDMLIterator::rewind(const ObDASDMLBaseCtDef *das_ctdef) { int ret = common::OB_SUCCESS; - cur_row_ = nullptr; - cur_rows_ = nullptr; + cur_datum_row_ = nullptr; + cur_datum_rows_ = nullptr; set_ctdef(das_ctdef); if (OB_NOT_NULL(domain_iter_)) { if (OB_FAIL(domain_iter_->rewind())) { @@ -269,7 +265,7 @@ void ObDASDMLIterator::set_ctdef(const ObDASDMLBaseCtDef *das_ctdef) } } -int ObDASMLogDMLIterator::get_next_row(ObNewRow *&row) +int ObDASMLogDMLIterator::get_next_row(blocksstable::ObDatumRow *&row) { int ret = OB_SUCCESS; if (OB_ISNULL(row_iter_)) { @@ -368,27 +364,29 @@ int ObDASWriteBuffer::DmlShadowRow::shadow_copy(const ObIArray &exprs, return ret; } -int ObDASWriteBuffer::DmlShadowRow::shadow_copy(const ObNewRow &row) +int ObDASWriteBuffer::DmlShadowRow::shadow_copy(const blocksstable::ObDatumRow &row) { int ret = OB_SUCCESS; - if (OB_ISNULL(store_row_) || OB_UNLIKELY(store_row_->cnt_ != row.get_count())) { + if (OB_ISNULL(store_row_) || OB_UNLIKELY(store_row_->cnt_ != row.get_column_count())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("NULL datums or count mismatch", K(ret), KPC(store_row_), K(row)); } else { ObDatum *cells = store_row_->cells(); - for (int64_t i = 0; OB_SUCC(ret) && i < row.get_count(); ++i) { - if (lib::is_oracle_mode() && row.get_cell(i).is_lob_locator() && strip_lob_locator_) { + ObObjDatumMapType map_type = OBJ_DATUM_MAPPING_MAX; + for (int64_t i = 0; OB_SUCC(ret) && i < row.get_column_count(); ++i) { + if (lib::is_oracle_mode() && column_types_ != nullptr && column_types_->at(i).is_lob_locator() && strip_lob_locator_) { ObString payload; - if (column_types_ != nullptr && !column_types_->at(i).is_lob()) { + if (!column_types_->at(i).is_lob()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid column type", K(ret), K(column_types_->at(i))); - } else if (OB_FAIL(row.get_cell(i).get_lob_locator()->get_payload(payload))) { + } else if (OB_FAIL(row.storage_datums_[i].get_lob_locator().get_payload(payload))) { LOG_WARN("get lob payload failed", K(ret)); } else { cells[i].set_string(payload); } - } else if (OB_FAIL(cells[i].from_obj(row.get_cell(i)))) { - LOG_WARN("shadow copy obj failed", K(ret), K(i), K(row)); + } else if (FALSE_IT(map_type = ObDatum::get_obj_datum_map_type(column_types_->at(i).get_type()))) { + } else if (OB_FAIL(cells[i].from_storage_datum(row.storage_datums_[i], map_type))) { + LOG_WARN("shadow copy storage datum failed", K(ret), K(i), K(row)); } if (OB_SUCC(ret)) { //add the data length of datum @@ -671,7 +669,7 @@ int ObDASWriteBuffer::begin(NewRowIterator &it, const ObIArray &col_t } } if (OB_SUCC(ret)) { - if (OB_FAIL(ob_create_row(*das_alloc_, col_types.count(), it.cur_new_row_))) { + if (OB_FAIL(blocksstable::ObDatumRowUtils::ob_create_row(*das_alloc_, col_types.count(), it.cur_new_row_))) { LOG_WARN("create new row failed", K(ret)); } else { it.col_types_ = &col_types; @@ -683,8 +681,8 @@ int ObDASWriteBuffer::begin(NewRowIterator &it, const ObIArray &col_t int ObDASWriteBuffer::dump_data(const ObDASDMLBaseCtDef &das_base_ctdef) const { int ret = OB_SUCCESS; - ObNewRow *new_row = NULL; - ObNewRow *old_row = NULL; + blocksstable::ObDatumRow *new_row = NULL; + blocksstable::ObDatumRow *old_row = NULL; const ObChunkDatumStore::StoredRow *store_row = NULL; int64_t rownum = 0; ObArenaAllocator tmp_alloc; @@ -705,7 +703,7 @@ int ObDASWriteBuffer::dump_data(const ObDASDMLBaseCtDef &das_base_ctdef) const if (!das_base_ctdef.old_row_projector_.empty()) { // create old row if (OB_ISNULL(old_row) - && OB_FAIL(ob_create_row(tmp_alloc, das_base_ctdef.old_row_projector_.count(), old_row))) { + && OB_FAIL(blocksstable::ObDatumRowUtils::ob_create_row(tmp_alloc, das_base_ctdef.old_row_projector_.count(), old_row))) { LOG_WARN("create old row buffer failed", K(ret), K(das_base_ctdef.old_row_projector_.count())); } else if (OB_FAIL(ObDASUtils::project_storage_row(das_base_ctdef, *store_row, @@ -720,7 +718,7 @@ int ObDASWriteBuffer::dump_data(const ObDASDMLBaseCtDef &das_base_ctdef) const if (OB_SUCC(ret)) { if (!das_base_ctdef.new_row_projector_.empty()) { if (OB_ISNULL(new_row) - && OB_FAIL(ob_create_row(tmp_alloc, das_base_ctdef.new_row_projector_.count(), new_row))) { + && OB_FAIL(blocksstable::ObDatumRowUtils::ob_create_row(tmp_alloc, das_base_ctdef.new_row_projector_.count(), new_row))) { LOG_WARN("create new row buffer failed", K(ret), K(das_base_ctdef.new_row_projector_.count())); } else if (OB_FAIL(ObDASUtils::project_storage_row(das_base_ctdef, *store_row, @@ -905,7 +903,7 @@ int ObDASWriteBuffer::Iterator::get_next_row_skip_const(ObEvalCtx &ctx, const Ob return ret; } -int ObDASWriteBuffer::NewRowIterator::get_next_row(ObNewRow *&row) +int ObDASWriteBuffer::NewRowIterator::get_next_row(blocksstable::ObDatumRow *&row) { int ret = OB_SUCCESS; const DmlRow *sr = nullptr; @@ -924,9 +922,7 @@ int ObDASWriteBuffer::NewRowIterator::get_next_row(ObNewRow *&row) if (OB_SUCC(ret) && sr != nullptr) { for (int64_t i = 0; OB_SUCC(ret) && i < sr->cnt_; ++i) { - if (OB_FAIL(sr->cells()[i].to_obj(cur_new_row_->cells_[i], col_types_->at(i)))) { - LOG_WARN("convert datum to ObNewRow failed", K(ret)); - } + cur_new_row_->storage_datums_[i].shallow_copy_from_datum(sr->cells()[i]); } if (OB_SUCC(ret)) { row = cur_new_row_; diff --git a/src/sql/das/ob_das_dml_ctx_define.h b/src/sql/das/ob_das_dml_ctx_define.h index 17e3a5448..d38d79710 100644 --- a/src/sql/das/ob_das_dml_ctx_define.h +++ b/src/sql/das/ob_das_dml_ctx_define.h @@ -22,6 +22,7 @@ #include "sql/engine/ob_operator.h" #include "sql/resolver/dml/ob_hint.h" #include "storage/fts/ob_fts_plugin_helper.h" +#include "storage/blocksstable/ob_datum_row_iterator.h" namespace oceanbase { namespace storage @@ -33,7 +34,8 @@ namespace sql typedef common::ObFixedArray ObjMetaFixedArray; typedef common::ObFixedArray AccuracyFixedArray; static const int64_t SAPTIAL_INDEX_DEFAULT_ROW_COUNT = 32; // 一个wkb生成的cellid数量（设定值） -typedef common::ObSEArray ObDomainIndexRow; +static const int64_t SAPTIAL_INDEX_DEFAULT_COL_COUNT = 2; +typedef common::ObSEArray ObDomainIndexRow; class ObDomainDMLIterator; struct ObDASDMLBaseRtDef; @@ -286,7 +288,7 @@ public: const common::ObIArray &col_types, bool strip_lob_locator); virtual int shadow_copy(const common::ObIArray &exprs, ObEvalCtx &ctx) override; - int shadow_copy(const common::ObNewRow &row); + int shadow_copy(const blocksstable::ObDatumRow &row); // reset && release referenced memory virtual void reset() { @@ -355,13 +357,13 @@ public: { } ~NewRowIterator() { } - int get_next_row(ObNewRow *&row); + int get_next_row(blocksstable::ObDatumRow *&row); inline bool is_inited() { return nullptr != col_types_; } private: DmlRow *cur_row_; ObChunkDatumStore::Iterator *datum_iter_; const common::ObIArray *col_types_; - ObNewRow *cur_new_row_; + blocksstable::ObDatumRow *cur_new_row_; }; OB_UNIS_VERSION(1); @@ -459,7 +461,7 @@ private: uint32_t row_extend_size_; }; -class ObDASDMLIterator : public common::ObNewRowIterator +class ObDASDMLIterator : public blocksstable::ObDatumRowIterator { public: static const int64_t DEFAULT_BATCH_SIZE = 256; @@ -471,8 +473,8 @@ public: das_ctdef_(das_ctdef), row_projector_(nullptr), allocator_(alloc), - cur_row_(nullptr), - cur_rows_(nullptr), + cur_datum_row_(nullptr), + cur_datum_rows_(nullptr), main_ctdef_(das_ctdef), domain_iter_(nullptr) { @@ -480,37 +482,37 @@ public: batch_size_ = MIN(write_buffer_.get_row_cnt(), DEFAULT_BATCH_SIZE); } virtual ~ObDASDMLIterator(); - virtual int get_next_row(common::ObNewRow *&row) override; - virtual int get_next_row() override; - virtual int get_next_rows(ObNewRow *&rows, int64_t &row_count); + virtual int get_next_row(blocksstable::ObDatumRow *&datum_row) override; + virtual int get_next_rows(blocksstable::ObDatumRow *&rows, int64_t &row_count); ObDASWriteBuffer &get_write_buffer() { return write_buffer_; } virtual void reset() override { } int rewind(const ObDASDMLBaseCtDef *das_ctdef); private: void set_ctdef(const ObDASDMLBaseCtDef *das_ctdef); - int get_next_domain_index_row(ObNewRow *&row); - int get_next_domain_index_rows(ObNewRow *&rows, int64_t &row_count); + int get_next_domain_index_row(blocksstable::ObDatumRow *&row); + int get_next_domain_index_rows(blocksstable::ObDatumRow *&rows, int64_t &row_count); private: ObDASWriteBuffer &write_buffer_; const ObDASDMLBaseCtDef *das_ctdef_; const IntFixedArray *row_projector_; ObDASWriteBuffer::Iterator write_iter_; common::ObIAllocator &allocator_; - common::ObNewRow *cur_row_; - common::ObNewRow *cur_rows_; + blocksstable::ObDatumRow *cur_datum_row_; + blocksstable::ObDatumRow *cur_datum_rows_; const ObDASDMLBaseCtDef *main_ctdef_; ObDomainDMLIterator *domain_iter_; int64_t batch_size_; }; -class ObDASMLogDMLIterator : public ObNewRowIterator +class ObDASMLogDMLIterator : public blocksstable::ObDatumRowIterator { public: + // support get next datum row ObDASMLogDMLIterator( const ObTabletID &tablet_id, const storage::ObDMLBaseParam &dml_param, - ObNewRowIterator *iter, + ObDatumRowIterator *iter, ObDASOpType op_type) : tablet_id_(tablet_id), dml_param_(dml_param), @@ -524,14 +526,12 @@ public: } } virtual ~ObDASMLogDMLIterator() {} - virtual int get_next_row(ObNewRow *&row) override; - virtual int get_next_row() override { return OB_NOT_IMPLEMENT; } - virtual void reset() override {} + virtual int get_next_row(blocksstable::ObDatumRow *&datum_row) override; private: const ObTabletID &tablet_id_; const storage::ObDMLBaseParam &dml_param_; - ObNewRowIterator *row_iter_; + ObDatumRowIterator *row_iter_; ObDASOpType op_type_; bool is_old_row_; }; diff --git a/src/sql/das/ob_das_domain_utils.cpp b/src/sql/das/ob_das_domain_utils.cpp index 80af2f483..a62781e6d 100644 --- a/src/sql/das/ob_das_domain_utils.cpp +++ b/src/sql/das/ob_das_domain_utils.cpp @@ -19,6 +19,7 @@ #include "sql/das/ob_das_utils.h" #include "sql/engine/expr/ob_expr_lob_utils.h" #include "observer/omt/ob_tenant_srs.h" +#include "storage/blocksstable/ob_datum_row_utils.h" using namespace oceanbase::common; @@ -59,10 +60,11 @@ int ObDASDomainUtils::generate_spatial_index_rows( } else { ObS2Adapter s2object(&allocator, srid != 0 ? srs_item->is_geographical_srs() : false); ObSpatialMBR spa_mbr; - ObObj *obj_arr = NULL; ObS2Cellids cellids; - char *mbr = NULL; + char *mbr = nullptr; int64_t mbr_len = 0; + void *rows_buf = nullptr; + blocksstable::ObDatumRow *rows = nullptr; if (OB_FAIL(s2object.init(wkb_str, srs_bound))) { LOG_WARN("Init s2object failed", K(ret)); } else if (OB_FAIL(s2object.get_cellids(cellids, false))) { @@ -81,38 +83,35 @@ int ObDASDomainUtils::generate_spatial_index_rows( LOG_WARN("failed to alloc memory for spatial index row mbr", K(ret)); } else if (OB_FAIL(spa_mbr.to_char(mbr, mbr_len))) { LOG_WARN("failed transform ObSpatialMBR to string", K(ret)); + } else if (OB_ISNULL(rows_buf = reinterpret_cast(allocator.alloc(cellids.size() * sizeof(blocksstable::ObDatumRow))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory for spatial index rows buffer", K(ret)); } else { + rows = new (rows_buf) blocksstable::ObDatumRow[cellids.size()]; + int64_t cellid_col_idx = 0; + int64_t mbr_col_idx = 1; for (uint64_t i = 0; OB_SUCC(ret) && i < cellids.size(); i++) { - if (OB_ISNULL(obj_arr = reinterpret_cast(allocator.alloc(sizeof(ObObj) * rowkey_num)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc memory for spatial index row cells", K(ret)); + if (OB_FAIL(rows[i].init(allocator, rowkey_num))) { + LOG_WARN("init datum row failed", K(ret), K(rowkey_num)); } else { // 索引行[cellid_obj][mbr_obj][rowkey_obj] for(uint64_t j = 0; OB_SUCC(ret) && j < rowkey_num; j++) { - obj_arr[j].set_nop_value(); const ObObjMeta &col_type = das_ctdef.column_types_.at(j); const ObAccuracy &col_accuracy = das_ctdef.column_accuracys_.at(j); int64_t projector_idx = row_projector.at(j); - if (OB_FAIL(dml_row.cells()[projector_idx].to_obj(obj_arr[j], col_type))) { - LOG_WARN("stored row to new row obj failed", K(ret), - K(dml_row.cells()[projector_idx]), K(col_type), K(projector_idx), K(j)); - } else if (OB_FAIL(ObDASUtils::reshape_storage_value(col_type, col_accuracy, allocator, obj_arr[j]))) { + if (FALSE_IT(rows[i].storage_datums_[j].shallow_copy_from_datum(dml_row.cells()[projector_idx]))) { + } else if (rows[i].storage_datums_[j].is_null()) { + } else if (OB_FAIL(ObDASUtils::reshape_datum_value(col_type, col_accuracy, true, allocator, rows[i].storage_datums_[j]))) { LOG_WARN("reshape storage value failed", K(ret), K(col_type), K(projector_idx), K(j)); } } if (OB_SUCC(ret)) { - int64_t cellid_col_idx = 0; - int64_t mbr_col_idx = 1; - obj_arr[cellid_col_idx].set_uint64(cellids.at(i)); + rows[i].storage_datums_[cellid_col_idx].set_uint(cellids.at(i)); ObString mbr_val(mbr_len, mbr); - obj_arr[mbr_col_idx].set_varchar(mbr_val); - obj_arr[mbr_col_idx].set_collation_type(CS_TYPE_BINARY); - obj_arr[mbr_col_idx].set_collation_level(CS_LEVEL_IMPLICIT); - ObNewRow row; - row.cells_ = obj_arr; - row.count_ = rowkey_num; - if (OB_FAIL(spat_rows.push_back(row))) { - LOG_WARN("failed to push back spatial index row", K(ret), K(row)); + rows[i].storage_datums_[mbr_col_idx].set_string(mbr_val); + // not set_collation_type(CS_TYPE_BINARY) and set_collation_level(CS_LEVEL_IMPLICIT) + if (OB_FAIL(spat_rows.push_back(&rows[i]))) { + LOG_WARN("failed to push back spatial index row", K(ret), K(rows[i])); } } } @@ -137,7 +136,8 @@ int ObDASDomainUtils::generate_spatial_index_rows( const int64_t ft_word_bkt_cnt = MAX(fulltext.length() / 10, 2); int64_t doc_length = 0; ObFTWordMap ft_word_map; - ObObj *obj_arr = nullptr; + void *rows_buf = nullptr; + blocksstable::ObDatumRow *rows = nullptr; if (OB_ISNULL(helper) || OB_UNLIKELY(!ft_obj_meta.is_valid()) || OB_UNLIKELY(doc_id.length() != sizeof(ObDocId) || doc_id.empty()) ) { @@ -153,47 +153,41 @@ int ObDASDomainUtils::generate_spatial_index_rows( K(ft_obj_meta.get_collation_type()), K(fulltext)); } else if (0 == ft_word_map.size()) { ret = OB_ITER_END; - } else { - const int64_t obj_cnt = ft_word_map.size() * FT_WORD_DOC_COL_CNT; - const int64_t obj_arr_size = sizeof(ObObj) * obj_cnt; - if (OB_ISNULL(obj_arr = reinterpret_cast(allocator.alloc(obj_arr_size)))) { + } else if (OB_ISNULL(rows_buf = reinterpret_cast(allocator.alloc(ft_word_map.size() * sizeof(blocksstable::ObDatumRow))))) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to allocate obj array", K(ret), K(obj_arr_size)); - } else { - for (int64_t i = 0; i < obj_cnt; ++i) { - new (obj_arr + i) ObObj(); - } - } + LOG_WARN("failed to alloc memory for full text index rows buffer", K(ret)); + } else { int64_t i = 0; + rows = new (rows_buf) blocksstable::ObDatumRow[ft_word_map.size()]; for (ObFTWordMap::const_iterator iter = ft_word_map.begin(); OB_SUCC(ret) && iter != ft_word_map.end(); ++iter) { - const ObFTWord &ft_word = iter->first; - const int64_t word_cnt = iter->second; - // index row format - // - FTS_INDEX: [WORD], [DOC_ID], [WORD_COUNT] - // - FTS_DOC_WORD: [DOC_ID], [WORD], [WORD_COUNT] - const int64_t word_idx = is_fts_index_aux ? 0 : 1; - const int64_t doc_id_idx = is_fts_index_aux ? 1 : 0; - const int64_t word_cnt_idx = 2; - const int64_t doc_len_idx = 3; - obj_arr[i * FT_WORD_DOC_COL_CNT + word_idx].set_varchar(ft_word.get_word()); - obj_arr[i * FT_WORD_DOC_COL_CNT + word_idx].set_meta_type(ft_obj_meta); - obj_arr[i * FT_WORD_DOC_COL_CNT + doc_id_idx].set_varbinary(doc_id); - obj_arr[i * FT_WORD_DOC_COL_CNT + word_cnt_idx].set_uint64(word_cnt); - obj_arr[i * FT_WORD_DOC_COL_CNT + doc_len_idx].set_uint64(doc_length); - ObNewRow row; - row.cells_ = &(obj_arr[i * FT_WORD_DOC_COL_CNT]); - row.count_ = FT_WORD_DOC_COL_CNT; - if (OB_FAIL(word_rows.push_back(row))) { - LOG_WARN("fail to push back row", K(ret), K(row)); + if (OB_FAIL(rows[i].init(allocator, FT_WORD_DOC_COL_CNT))) { + LOG_WARN("init datum row failed", K(ret), K(FT_WORD_DOC_COL_CNT)); } else { - ObDocId tmp_doc_id; - tmp_doc_id.tablet_id_ = ((const uint64_t *)doc_id.ptr())[0]; - tmp_doc_id.seq_id_ = ((const uint64_t *)doc_id.ptr())[1]; - STORAGE_FTS_LOG(DEBUG, "succeed to add word row", K(ret), K(is_fts_index_aux), "doc_id", tmp_doc_id, - K(ft_word), K(word_cnt), K(i), K(row)); - ++i; + const ObFTWord &ft_word = iter->first; + const int64_t word_cnt = iter->second; + // index row format + // - FTS_INDEX: [WORD], [DOC_ID], [WORD_COUNT] + // - FTS_DOC_WORD: [DOC_ID], [WORD], [WORD_COUNT] + const int64_t word_idx = is_fts_index_aux ? 0 : 1; + const int64_t doc_id_idx = is_fts_index_aux ? 1 : 0; + const int64_t word_cnt_idx = 2; + const int64_t doc_len_idx = 3; + rows[i].storage_datums_[word_idx].set_string(ft_word.get_word()); + rows[i].storage_datums_[doc_id_idx].set_string(doc_id); + rows[i].storage_datums_[word_cnt_idx].set_uint(word_cnt); + rows[i].storage_datums_[doc_len_idx].set_uint(doc_length); + if (OB_FAIL(word_rows.push_back(&rows[i]))) { + LOG_WARN("fail to push back row", K(ret), K(rows[i])); + } else { + ObDocId tmp_doc_id; + tmp_doc_id.tablet_id_ = ((const uint64_t *)doc_id.ptr())[0]; + tmp_doc_id.seq_id_ = ((const uint64_t *)doc_id.ptr())[1]; + STORAGE_FTS_LOG(DEBUG, "succeed to add word row", K(ret), K(is_fts_index_aux), "doc_id", tmp_doc_id, + K(ft_word), K(word_cnt), K(i), K(rows[i])); + ++i; + } } } } @@ -332,65 +326,66 @@ int ObDASDomainUtils::generate_multivalue_index_rows(ObIAllocator &allocator, uint64_t rowkey_column_start = column_num - 1 - data_table_rowkey_cnt; uint64_t rowkey_column_end = column_num - 1; + void *rows_buf = nullptr; + if (OB_FAIL(get_pure_mutivalue_data(json_str, data, data_len, record_num))) { LOG_WARN("failed to parse binary.", K(ret), K(json_str)); } else if (record_num == 0 && (is_unique_index && rowkey_column_start == 1)) { } else if (OB_FAIL(calc_save_rowkey_policy(allocator, das_ctdef, row_projector, dml_row, record_num, is_save_rowkey))) { LOG_WARN("failed to calc store policy.", K(ret), K(data_table_rowkey_cnt)); + } else if (OB_ISNULL(rows_buf = reinterpret_cast(allocator.alloc(record_num * sizeof(blocksstable::ObDatumRow))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory for multi value index rows buffer", K(ret)); } else { - ObObj *obj_arr = nullptr; int64_t pos = sizeof(uint32_t); + blocksstable::ObDatumRow *rows = new (rows_buf) blocksstable::ObDatumRow[record_num]; + ObObj obj; for (int i = 0; OB_SUCC(ret) && (i < record_num || !is_none_unique_done) ; ++i) { - if (OB_ISNULL(obj_arr = reinterpret_cast(allocator.alloc(sizeof(ObObj) * column_num)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc memory for multivalue index row cells", K(ret)); - } - for(uint64_t j = 0; OB_SUCC(ret) && j < column_num; j++) { - obj_arr[j].set_nop_value(); - ObObjMeta col_type = das_ctdef.column_types_.at(j); - const ObAccuracy &col_accuracy = das_ctdef.column_accuracys_.at(j); - int64_t projector_idx = row_projector.at(j); + if (OB_FAIL(rows[i].init(allocator, column_num))) { + LOG_WARN("init datum row failed", K(ret), K(column_num)); + } else { + for(uint64_t j = 0; OB_SUCC(ret) && j < column_num; j++) { + ObObjMeta col_type = das_ctdef.column_types_.at(j); + const ObAccuracy &col_accuracy = das_ctdef.column_accuracys_.at(j); + int64_t projector_idx = row_projector.at(j); - if (multivalue_idx == projector_idx) { - if (OB_FAIL(obj_arr[j].deserialize(data, data_len, pos))) { - LOG_WARN("failed to deserialize datum.", K(ret), K(json_str)); - } else { - if (ob_is_number_or_decimal_int_tc(col_type.get_type()) || ob_is_temporal_type(col_type.get_type())) { - col_type.set_collation_level(CS_LEVEL_NUMERIC); + if (multivalue_idx == projector_idx) { + // TODO: change obj to datum when do deserialize@xuanxi + obj.set_nop_value(); + if (OB_FAIL(obj.deserialize(data, data_len, pos))) { + LOG_WARN("failed to deserialize datum", K(ret), K(json_str)); + } else if (OB_FAIL(rows[i].storage_datums_[j].from_obj_enhance(obj))) { + LOG_WARN("failed to convert datum from obj", K(ret), K(obj)); } else { - col_type.set_collation_level(CS_LEVEL_IMPLICIT); + if (ob_is_number_or_decimal_int_tc(col_type.get_type()) || ob_is_temporal_type(col_type.get_type())) { + col_type.set_collation_level(CS_LEVEL_NUMERIC); + } else { + col_type.set_collation_level(CS_LEVEL_IMPLICIT); + } + is_none_unique_done = true; } - - obj_arr[j].set_collation_level(col_type.get_collation_level()); - obj_arr[j].set_collation_type(col_type.get_collation_type()); - obj_arr[j].set_type(col_type.get_type()); - is_none_unique_done = true; + } else if (!is_save_rowkey && (rowkey_column_start >= j && j < rowkey_column_end)) { + rows[i].storage_datums_[j].set_null(); + } else if (multivalue_arr_idx == projector_idx) { + rows[i].storage_datums_[j].set_null(); + } else { + rows[i].storage_datums_[j].shallow_copy_from_datum(dml_row.cells()[projector_idx]); + } + + if (OB_SUCC(ret) && OB_FAIL(ObDASUtils::reshape_datum_value(col_type, col_accuracy, true, allocator, rows[i].storage_datums_[j]))) { + LOG_WARN("reshape storage value failed", K(ret), K(col_type), K(projector_idx), K(j)); } - } else if (!is_save_rowkey && (rowkey_column_start >= j && j < rowkey_column_end)) { - obj_arr[j].set_null(); - } else if (multivalue_arr_idx == projector_idx) { - obj_arr[j].set_null(); - } else if (OB_FAIL(dml_row.cells()[projector_idx].to_obj(obj_arr[j], col_type))) { - LOG_WARN("stored row to new row obj failed", K(ret), - K(dml_row.cells()[projector_idx]), K(col_type), K(projector_idx), K(j)); } - if (OB_SUCC(ret) && OB_FAIL(ObDASUtils::reshape_storage_value(col_type, col_accuracy, allocator, obj_arr[j]))) { - LOG_WARN("reshape storage value failed", K(ret), K(col_type), K(projector_idx), K(j)); - } + if (OB_SUCC(ret)) { + if (OB_FAIL(mvi_rows.push_back(&rows[i]))) { + LOG_WARN("failed to push back spatial index row", K(ret), K(rows[i])); + } + } // end if (OB_SUCC(ret)) } - - if (OB_SUCC(ret)) { - ObNewRow row; - row.cells_ = obj_arr; - row.count_ = column_num; - if (OB_FAIL(mvi_rows.push_back(row))) { - LOG_WARN("failed to push back spatial index row", K(ret), K(row)); - } - } // end if (OB_SUCC(ret)) } } @@ -501,7 +496,7 @@ bool ObDomainDMLIterator::is_same_domain_type(const ObDASDMLBaseCtDef *das_ctdef return is_same_domain_type; } -int ObDomainDMLIterator::get_next_domain_row(ObNewRow *&row) +int ObDomainDMLIterator::get_next_domain_row(blocksstable::ObDatumRow *&row) { int ret = OB_SUCCESS; const ObChunkDatumStore::StoredRow *sr = nullptr; @@ -529,7 +524,7 @@ int ObDomainDMLIterator::get_next_domain_row(ObNewRow *&row) } } if (OB_SUCC(ret) && row_idx_ < rows_.count()) { - row = &(rows_[row_idx_]); + row = rows_[row_idx_]; ++row_idx_; got_row = true; } @@ -538,7 +533,7 @@ int ObDomainDMLIterator::get_next_domain_row(ObNewRow *&row) return ret; } -int ObDomainDMLIterator::get_next_domain_rows(ObNewRow *&row, int64_t &row_count) +int ObDomainDMLIterator::get_next_domain_rows(blocksstable::ObDatumRow *&row, int64_t &row_count) { int ret = OB_SUCCESS; const ObChunkDatumStore::StoredRow *sr = nullptr; @@ -572,7 +567,7 @@ int ObDomainDMLIterator::get_next_domain_rows(ObNewRow *&row, int64_t &row_count } } if (OB_SUCC(ret) && row_idx_ < rows_.count()) { - row = &(rows_[row_idx_]); + row = rows_[row_idx_]; row_count = rows_.count() - row_idx_; row_idx_ = rows_.count(); got_row = true; diff --git a/src/sql/das/ob_das_domain_utils.h b/src/sql/das/ob_das_domain_utils.h index 63568f2bc..2ca8c2547 100644 --- a/src/sql/das/ob_das_domain_utils.h +++ b/src/sql/das/ob_das_domain_utils.h @@ -84,7 +84,7 @@ class ObDomainDMLIterator { public: static const int64_t DEFAULT_DOMAIN_ROW_COUNT = 32; - typedef common::ObSEArray ObDomainIndexRow; + typedef common::ObSEArray ObDomainIndexRow; static int create_domain_dml_iterator( common::ObIAllocator &allocator, @@ -104,8 +104,8 @@ public: } void set_ctdef(const ObDASDMLBaseCtDef *das_ctdef, const IntFixedArray *row_projector); void set_row_projector(const IntFixedArray *row_projector) { row_projector_ = row_projector; } - int get_next_domain_row(ObNewRow *&row); - int get_next_domain_rows(ObNewRow *&row, int64_t &row_count); + int get_next_domain_row(blocksstable::ObDatumRow *&row); + int get_next_domain_rows(blocksstable::ObDatumRow *&row, int64_t &row_count); bool is_same_domain_type(const ObDASDMLBaseCtDef *das_ctdef) const; TO_STRING_KV(K_(row_idx), K_(rows), KPC_(row_projector), KPC_(das_ctdef), K_(main_ctdef)); diff --git a/src/sql/das/ob_das_insert_op.cpp b/src/sql/das/ob_das_insert_op.cpp index 688f05b5e..fb7770041 100644 --- a/src/sql/das/ob_das_insert_op.cpp +++ b/src/sql/das/ob_das_insert_op.cpp @@ -149,7 +149,7 @@ int ObDASInsertOp::insert_rows() } int ObDASInsertOp::insert_index_with_fetch(ObDMLBaseParam &dml_param, ObAccessService *as, - ObNewRowIterator &dml_iter, + ObDatumRowIterator &dml_iter, ObDASConflictIterator *result_iter, const ObDASInsCtDef *ins_ctdef, ObDASInsRtDef *ins_rtdef, @@ -158,7 +158,7 @@ int ObDASInsertOp::insert_index_with_fetch(ObDMLBaseParam &dml_param, common::ObTabletID tablet_id) { int ret = OB_SUCCESS; - ObNewRow *insert_row = NULL; + blocksstable::ObDatumRow *insert_row = NULL; int64_t affected_rows = 0; if (OB_FAIL(ObDMLService::init_dml_param(*ins_ctdef, *ins_rtdef, @@ -170,7 +170,7 @@ int ObDASInsertOp::insert_index_with_fetch(ObDMLBaseParam &dml_param, LOG_WARN("init index dml param failed", K(ret), KPC(ins_ctdef), KPC(ins_rtdef)); } while (OB_SUCC(ret) && OB_SUCC(dml_iter.get_next_row(insert_row))) { - ObNewRowIterator *duplicated_rows = NULL; + blocksstable::ObDatumRowIterator *duplicated_rows = NULL; if (OB_ISNULL(insert_row)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("insert_row is null", K(ret)); @@ -297,7 +297,7 @@ int ObDASInsertOp::insert_row_with_fetch() LOG_WARN("rewind dml iter failed", K(ret)); } else { ObDASMLogDMLIterator mlog_iter(index_tablet_id, dml_param, &dml_iter, DAS_OP_TABLE_INSERT); - ObNewRowIterator *new_iter = nullptr; + ObDatumRowIterator *new_iter = nullptr; if (index_ins_ctdef->table_param_.get_data_table().is_mlog_table() && !index_ins_ctdef->is_access_mlog_as_master_table_) { new_iter = &mlog_iter; @@ -331,7 +331,7 @@ int ObDASInsertOp::store_conflict_row(ObDASInsertResult &ins_result) { int ret = OB_SUCCESS; bool added = false; - ObNewRow *dup_row = nullptr; + ObDatumRow *dup_row = nullptr; ObDASWriteBuffer &result_buffer = ins_result.get_result_buffer(); ObDASWriteBuffer::DmlShadowRow ssr; if (OB_ISNULL(result_)) { @@ -483,10 +483,10 @@ ObDASInsertResult::~ObDASInsertResult() { } -int ObDASInsertResult::get_next_row(ObNewRow *&row) +int ObDASInsertResult::get_next_row(ObDatumRow *&row) { int ret = OB_SUCCESS; - ObNewRow *result_row = NULL; + ObDatumRow *result_row = NULL; if (OB_FAIL(result_newrow_iter_.get_next_row(result_row))) { if (OB_ITER_END != ret) { LOG_WARN("get next row from result iter failed", K(ret)); @@ -497,19 +497,6 @@ int ObDASInsertResult::get_next_row(ObNewRow *&row) return ret; } -int ObDASInsertResult::get_next_rows(int64_t &count, int64_t capacity) -{ - UNUSED(count); - UNUSED(capacity); - return OB_NOT_IMPLEMENT; -} - -int ObDASInsertResult::get_next_row() -{ -int ret = OB_NOT_IMPLEMENT; -return ret; -} - int ObDASInsertResult::link_extra_result(ObDASExtraData &extra_result) { UNUSED(extra_result); @@ -585,11 +572,11 @@ void ObDASConflictIterator::reset() duplicated_iter_list_.reset(); } -int ObDASConflictIterator::get_next_row(common::ObNewRow *&row) +int ObDASConflictIterator::get_next_row(ObDatumRow *&row) { int ret = OB_SUCCESS; bool find_next_iter = false; - ObNewRow *dup_row = NULL; + ObDatumRow *dup_row = NULL; do { if (find_next_iter) { ++curr_iter_; @@ -599,7 +586,7 @@ int ObDASConflictIterator::get_next_row(common::ObNewRow *&row) ret = OB_ITER_END; LOG_DEBUG("fetch conflict row iterator end"); } else { - ObNewRowIterator *dup_row_iter = *curr_iter_; + blocksstable::ObDatumRowIterator *dup_row_iter = *curr_iter_; if (OB_ISNULL(dup_row_iter)) { find_next_iter = true; } else if (OB_FAIL(dup_row_iter->get_next_row(dup_row))) { @@ -624,11 +611,5 @@ int ObDASConflictIterator::get_next_row(common::ObNewRow *&row) return ret; } -int ObDASConflictIterator::get_next_row() -{ - int ret = OB_NOT_IMPLEMENT; - return ret; -} - } // namespace sql } // namespace oceanbase diff --git a/src/sql/das/ob_das_insert_op.h b/src/sql/das/ob_das_insert_op.h index 88a43105a..ab2a30f02 100644 --- a/src/sql/das/ob_das_insert_op.h +++ b/src/sql/das/ob_das_insert_op.h @@ -22,8 +22,8 @@ namespace sql { class ObDASInsertResult; -typedef common::ObList ObDuplicatedIterList; -class ObDASConflictIterator : public common::ObNewRowIterator +typedef common::ObList ObDuplicatedIterList; +class ObDASConflictIterator : public blocksstable::ObDatumRowIterator { public: ObDASConflictIterator(const ObjMetaFixedArray &output_types, @@ -36,9 +36,8 @@ public: ~ObDASConflictIterator() {}; - virtual int get_next_row(common::ObNewRow *&row) override; - virtual int get_next_row() override; - virtual void reset() override; + void reset(); + virtual int get_next_row(blocksstable::ObDatumRow *&row) override; void init_curr_iter() { curr_iter_ = duplicated_iter_list_.begin(); } @@ -78,7 +77,7 @@ public: return insert_buffer_.dump_data(*ins_ctdef_); } - common::ObNewRowIterator *get_duplicated_result() + blocksstable::ObDatumRowIterator *get_duplicated_result() { return result_; } INHERIT_TO_STRING_KV("parent", ObIDASTaskOp, @@ -93,7 +92,7 @@ private: int insert_index_with_fetch(ObDMLBaseParam &dml_param, ObAccessService *as, - common::ObNewRowIterator &dml_iter, + blocksstable::ObDatumRowIterator &dml_iter, ObDASConflictIterator *result_iter, const ObDASInsCtDef *ins_ctdef, ObDASInsRtDef *ins_rtdef, @@ -105,12 +104,12 @@ private: const ObDASInsCtDef *ins_ctdef_; ObDASInsRtDef *ins_rtdef_; ObDASWriteBuffer insert_buffer_; - common::ObNewRowIterator *result_; + blocksstable::ObDatumRowIterator *result_; int64_t affected_rows_; // local execute result, no need to serialize bool is_duplicated_; // local execute result, no need to serialize }; -class ObDASInsertResult : public ObIDASTaskResult, public common::ObNewRowIterator +class ObDASInsertResult : public ObIDASTaskResult, public blocksstable::ObDatumRowIterator { OB_UNIS_VERSION_V(1); public: @@ -118,10 +117,8 @@ public: virtual ~ObDASInsertResult(); virtual int init(const ObIDASTaskOp &op, common::ObIAllocator &alloc) override; virtual int reuse() override; - virtual int get_next_row(ObNewRow *&row) override; - virtual int get_next_row() override; - virtual int get_next_rows(int64_t &count, int64_t capacity) override; - virtual void reset() override; + virtual int get_next_row(blocksstable::ObDatumRow *&row) override; + virtual void reset(); virtual int link_extra_result(ObDASExtraData &extra_result) override; int init_result_newrow_iter(const ObjMetaFixedArray *output_types); ObDASWriteBuffer &get_result_buffer() { return result_buffer_; } diff --git a/src/sql/das/ob_das_update_op.cpp b/src/sql/das/ob_das_update_op.cpp index 80028ffa6..fdc89e44b 100644 --- a/src/sql/das/ob_das_update_op.cpp +++ b/src/sql/das/ob_das_update_op.cpp @@ -18,6 +18,7 @@ #include "sql/das/ob_das_utils.h" #include "sql/das/ob_das_domain_utils.h" #include "storage/tx_storage/ob_access_service.h" +#include "storage/blocksstable/ob_datum_row_utils.h" #include "sql/engine/expr/ob_expr_lob_utils.h" namespace oceanbase { @@ -43,7 +44,7 @@ using namespace share; namespace sql { -class ObDASUpdIterator : public ObNewRowIterator +class ObDASUpdIterator : public blocksstable::ObDatumRowIterator { public: ObDASUpdIterator(const ObDASUpdCtDef *das_ctdef, @@ -60,10 +61,8 @@ public: { } virtual ~ObDASUpdIterator(); - virtual int get_next_row(ObNewRow *&row) override; - virtual int get_next_row() override { return OB_NOT_IMPLEMENT; } + virtual int get_next_row(blocksstable::ObDatumRow *&row) override; ObDASWriteBuffer &get_write_buffer() { return write_buffer_; } - virtual void reset() override { } int rewind(const ObDASDMLBaseCtDef *das_ctdef) { int ret = common::OB_SUCCESS; @@ -92,12 +91,12 @@ public: } private: // domain index - int get_next_domain_index_row(ObNewRow *&row); + int get_next_domain_index_row(blocksstable::ObDatumRow *&row); private: const ObDASUpdCtDef *das_ctdef_; ObDASWriteBuffer &write_buffer_; - ObNewRow *old_row_; - ObNewRow *new_row_; + blocksstable::ObDatumRow *old_row_; + blocksstable::ObDatumRow *new_row_; ObDASWriteBuffer::Iterator result_iter_; ObDomainDMLIterator *domain_iter_; bool got_old_row_; @@ -105,7 +104,7 @@ private: common::ObIAllocator &allocator_; }; -int ObDASUpdIterator::get_next_row(ObNewRow *&row) +int ObDASUpdIterator::get_next_row(blocksstable::ObDatumRow *&row) { int ret = OB_SUCCESS; const ObChunkDatumStore::StoredRow *sr = NULL; @@ -119,7 +118,7 @@ int ObDASUpdIterator::get_next_row(ObNewRow *&row) } else if (!got_old_row_) { got_old_row_ = true; if (OB_ISNULL(old_row_)) { - if (OB_FAIL(ob_create_row(allocator_, das_ctdef_->old_row_projector_.count(), old_row_))) { + if (OB_FAIL(blocksstable::ObDatumRowUtils::ob_create_row(allocator_, das_ctdef_->old_row_projector_.count(), old_row_))) { LOG_WARN("create row buffer failed", K(ret), K(das_ctdef_->old_row_projector_.count())); } else if (OB_FAIL(write_buffer_.begin(result_iter_))) { LOG_WARN("begin datum result iterator failed", K(ret)); @@ -127,7 +126,7 @@ int ObDASUpdIterator::get_next_row(ObNewRow *&row) } if (OB_SUCC(ret) && OB_ISNULL(new_row_)) { - if (OB_FAIL(ob_create_row(allocator_, das_ctdef_->new_row_projector_.count(), new_row_))) { + if (OB_FAIL(blocksstable::ObDatumRowUtils::ob_create_row(allocator_, das_ctdef_->new_row_projector_.count(), new_row_))) { LOG_WARN("create row buffer failed", K(ret), K(das_ctdef_->new_row_projector_.count())); } } @@ -182,7 +181,7 @@ ObDASUpdIterator::~ObDASUpdIterator() } } -int ObDASUpdIterator::get_next_domain_index_row(ObNewRow *&row) +int ObDASUpdIterator::get_next_domain_index_row(ObDatumRow *&row) { int ret = OB_SUCCESS; if (!iter_has_built_) { diff --git a/src/sql/das/ob_das_utils.cpp b/src/sql/das/ob_das_utils.cpp index b59394bab..66260fd4a 100644 --- a/src/sql/das/ob_das_utils.cpp +++ b/src/sql/das/ob_das_utils.cpp @@ -23,6 +23,7 @@ #include "observer/omt/ob_tenant_srs.h" #include "share/ob_tablet_autoincrement_service.h" #include "storage/access/ob_dml_param.h" +#include "storage/blocksstable/ob_datum_row_utils.h" namespace oceanbase { using namespace common; @@ -203,7 +204,7 @@ int ObDASUtils::project_storage_row(const ObDASDMLBaseCtDef &dml_ctdef, const ObDASWriteBuffer::DmlRow &dml_row, const IntFixedArray &row_projector, ObIAllocator &allocator, - ObNewRow &storage_row) + blocksstable::ObDatumRow &storage_row) { int ret = OB_SUCCESS; for (int64_t i = 0; OB_SUCC(ret) && i < row_projector.count(); ++i) { @@ -212,14 +213,17 @@ int ObDASUtils::project_storage_row(const ObDASDMLBaseCtDef &dml_ctdef, const ObAccuracy &col_accuracy = dml_ctdef.column_accuracys_.at(i); if (projector_idx < 0) { //this column is not touched by query, only need to be marked as nop - storage_row.cells_[i].set_nop_value(); - } else if (OB_FAIL(dml_row.cells()[projector_idx].to_obj(storage_row.cells_[i], col_type))) { - LOG_WARN("stored row to new row obj failed", K(ret), - K(dml_row.cells()[projector_idx]), K(col_type), K(projector_idx), K(i)); - } else if (OB_FAIL(reshape_storage_value(col_type, col_accuracy, allocator, storage_row.cells_[i]))) { + storage_row.storage_datums_[i].set_nop(); + } else if (FALSE_IT(storage_row.storage_datums_[i].shallow_copy_from_datum(dml_row.cells()[projector_idx]))) { + } else if (storage_row.storage_datums_[i].is_null()) { + //nothing to do + } else if (OB_FAIL(reshape_datum_value(col_type, col_accuracy, true, allocator, storage_row.storage_datums_[i]))) { LOG_WARN("reshape storage value failed", K(ret)); + } else if (col_type.is_lob_storage() && col_type.has_lob_header()) { + storage_row.storage_datums_[i].set_has_lob_header(); } } + //to project shadow rowkey with unique index if (OB_SUCC(ret) && dml_ctdef.spk_cnt_) { bool need_shadow_columns = false; @@ -229,7 +233,7 @@ int ObDASUtils::project_storage_row(const ObDASDMLBaseCtDef &dml_ctdef, //need to fill shadow pk with the real pk value bool rowkey_has_null = false; for (int64_t i = 0; !rowkey_has_null && i < index_key_cnt; i++) { - rowkey_has_null = storage_row.cells_[i].is_null(); + rowkey_has_null = storage_row.storage_datums_[i].is_null(); } need_shadow_columns = rowkey_has_null; } else { @@ -237,14 +241,14 @@ int ObDASUtils::project_storage_row(const ObDASDMLBaseCtDef &dml_ctdef, //need to fill shadow pk with the real pk value bool is_rowkey_all_null = true; for (int64_t i = 0; is_rowkey_all_null && i < index_key_cnt; i++) { - is_rowkey_all_null = storage_row.cells_[i].is_null(); + is_rowkey_all_null = storage_row.storage_datums_[i].is_null(); } need_shadow_columns = is_rowkey_all_null; } if (!need_shadow_columns) { for (int64_t i = 0; i < dml_ctdef.spk_cnt_; ++i) { int64_t spk_idx = index_key_cnt + i; - storage_row.cells_[spk_idx].set_null(); + storage_row.storage_datums_[spk_idx].set_null(); } } } @@ -398,7 +402,7 @@ int ObDASUtils::find_child_das_def(const ObDASBaseCtDef *root_ctdef, int ObDASUtils::generate_mlog_row(const ObTabletID &tablet_id, const storage::ObDMLBaseParam &dml_param, - ObNewRow &row, + blocksstable::ObDatumRow &row, ObDASOpType op_type, bool is_old_row) { @@ -432,23 +436,26 @@ int ObDASUtils::generate_mlog_row(const ObTabletID &tablet_id, } } - row.cells_[sequence_col].set_int(ObObjType::ObIntType, static_cast(autoinc_seq)); + row.storage_datums_[sequence_col].reuse(); + row.storage_datums_[dmltype_col].reuse(); + row.storage_datums_[old_new_col].reuse(); + + row.storage_datums_[sequence_col].set_int(static_cast(autoinc_seq)); if (sql::DAS_OP_TABLE_DELETE == op_type) { - row.cells_[dmltype_col].set_varchar("D"); - row.cells_[old_new_col].set_varchar("O"); + row.storage_datums_[dmltype_col].set_string(ObString("D")); + row.storage_datums_[old_new_col].set_string(ObString("O")); } else if (sql::DAS_OP_TABLE_UPDATE == op_type) { - row.cells_[dmltype_col].set_varchar("U"); + row.storage_datums_[dmltype_col].set_string(ObString("U")); if (is_old_row) { - row.cells_[old_new_col].set_varchar("O"); + row.storage_datums_[old_new_col].set_string(ObString("O")); } else { - row.cells_[old_new_col].set_varchar("N"); + row.storage_datums_[old_new_col].set_string(ObString("N")); } } else { - row.cells_[dmltype_col].set_varchar("I"); - row.cells_[old_new_col].set_varchar("N"); + row.storage_datums_[dmltype_col].set_string(ObString("I")); + row.storage_datums_[old_new_col].set_string(ObString("N")); } - row.cells_[dmltype_col].set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_GENERAL_CI); - row.cells_[old_new_col].set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_GENERAL_CI); + // TODO: if we need to update col_type_ in col_descs with ObCollationType::CS_TYPE_UTF8MB4_GENERAL_CI ?@xuanxi } return ret; } diff --git a/src/sql/das/ob_das_utils.h b/src/sql/das/ob_das_utils.h index da50e6a3f..7c76cf4e1 100644 --- a/src/sql/das/ob_das_utils.h +++ b/src/sql/das/ob_das_utils.h @@ -53,7 +53,7 @@ public: const ObDASWriteBuffer::DmlRow &dml_row, const IntFixedArray &row_projector, common::ObIAllocator &allocator, - common::ObNewRow &storage_row); + blocksstable::ObDatumRow &storage_row); static int reshape_storage_value(const common::ObObjMeta &col_type, const common::ObAccuracy &col_accuracy, common::ObIAllocator &allocator, @@ -93,7 +93,7 @@ public: } static int generate_mlog_row(const common::ObTabletID &tablet_id, const storage::ObDMLBaseParam &dml_param, - common::ObNewRow &row, + blocksstable::ObDatumRow &row, ObDASOpType op_type, bool is_old_row); }; diff --git a/src/sql/engine/dml/ob_table_insert_up_op.cpp b/src/sql/engine/dml/ob_table_insert_up_op.cpp index 3cf464d4b..e2ff8d5e5 100644 --- a/src/sql/engine/dml/ob_table_insert_up_op.cpp +++ b/src/sql/engine/dml/ob_table_insert_up_op.cpp @@ -981,11 +981,11 @@ int ObTableInsertUpOp::get_next_conflict_rowkey(DASTaskIter &task_iter) int ret = OB_SUCCESS; bool got_row = false; while (OB_SUCC(ret) && !got_row) { - ObNewRow *dup_row = nullptr; + ObDatumRow *dup_row = nullptr; ObChunkDatumStore::StoredRow *stored_row = nullptr; ObDASWriteBuffer::DmlShadowRow ssr; ObDASInsertOp *ins_op = static_cast(*task_iter); - ObNewRowIterator *conflict_result = ins_op->get_duplicated_result(); + ObDatumRowIterator *conflict_result = ins_op->get_duplicated_result(); const ObDASInsCtDef *ins_ctdef = static_cast(ins_op->get_ctdef()); // 因为返回的都是主表的主键，主表的主键一定是在存储层有储存的，是不需要再收起来层再做运算的， // 所以这里不需要clear eval flag diff --git a/src/sql/engine/dml/ob_table_replace_op.cpp b/src/sql/engine/dml/ob_table_replace_op.cpp index b0fd727d2..fe824f855 100644 --- a/src/sql/engine/dml/ob_table_replace_op.cpp +++ b/src/sql/engine/dml/ob_table_replace_op.cpp @@ -413,11 +413,11 @@ int ObTableReplaceOp::get_next_conflict_rowkey(DASTaskIter &task_iter) int ret = OB_SUCCESS; bool got_row = false; while (OB_SUCC(ret) && !got_row) { - ObNewRow *dup_row = nullptr; + ObDatumRow *dup_row = nullptr; ObChunkDatumStore::StoredRow *stored_row = nullptr; ObDASWriteBuffer::DmlShadowRow ssr; ObDASInsertOp *ins_op = static_cast(*task_iter); - ObNewRowIterator *conflict_result = ins_op->get_duplicated_result(); + ObDatumRowIterator *conflict_result = ins_op->get_duplicated_result(); const ObDASInsCtDef *ins_ctdef = static_cast(ins_op->get_ctdef()); // 因为返回的都是主表的主键，主表的主键一定是在存储层有储存的，是不需要再收起来层再做运算的， // 所以这里不需要clear eval flag diff --git a/src/sql/engine/expr/ob_expr_pl_get_cursor_attr.cpp b/src/sql/engine/expr/ob_expr_pl_get_cursor_attr.cpp index e5fec97ea..1f7db7f64 100644 --- a/src/sql/engine/expr/ob_expr_pl_get_cursor_attr.cpp +++ b/src/sql/engine/expr/ob_expr_pl_get_cursor_attr.cpp @@ -292,7 +292,7 @@ int ObExprPLGetCursorAttr::calc_pl_get_cursor_attr( } else if (OB_UNLIKELY(rowid.empty())) { expr_datum.set_null(); } else { - expr_datum.set_urowid(rowid.ptr(), rowid.length()); + expr_datum.set_string(rowid.ptr(), rowid.length()); } } break; diff --git a/src/sql/engine/table/ob_table_scan_op.cpp b/src/sql/engine/table/ob_table_scan_op.cpp index b8f0eb48c..8d2c5a90f 100644 --- a/src/sql/engine/table/ob_table_scan_op.cpp +++ b/src/sql/engine/table/ob_table_scan_op.cpp @@ -3135,25 +3135,17 @@ int ObTableScanOp::inner_get_next_spatial_index_row() } else if (cellids.size() > SAPTIAL_INDEX_DEFAULT_ROW_COUNT) { ret = OB_ERR_UNEXPECTED; LOG_WARN("cellid over size", K(ret), K(cellids.size())); - } else if (OB_ISNULL(spat_index_.obj_buffer_)) { + } else if (OB_ISNULL(spat_index_.rows_)) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc memory for spatial index row cells", K(ret)); + LOG_WARN("failed to alloc memory for spatial index datum row", K(ret)); } else { - ObObj *obj_arr = reinterpret_cast(spat_index_.obj_buffer_); - uint64_t obj_idx = 0; - for (uint64_t i = 0; OB_SUCC(ret) && i < cellids.size(); i++) { - obj_arr[obj_idx].set_nop_value(); - obj_arr[obj_idx].set_uint64(cellids.at(i)); - obj_arr[obj_idx + 1].set_nop_value(); - obj_arr[obj_idx + 1].set_varchar(mbr_val); - obj_arr[obj_idx + 1].set_collation_type(CS_TYPE_BINARY); - obj_arr[obj_idx + 1].set_collation_level(CS_LEVEL_IMPLICIT); - ObNewRow row; - row.cells_ = &obj_arr[obj_idx]; - row.count_ = 2; - obj_idx += 2; - if (OB_FAIL(spat_index_.spat_rows_->push_back(row))) { - LOG_WARN("failed to push back spatial index row", K(ret), K(row)); + for (uint64_t i = 0, datum_idx = 0; OB_SUCC(ret) && i < cellids.size(); i++) { + spat_index_.rows_[i].reuse(); + spat_index_.rows_[i].storage_datums_[datum_idx].set_uint(cellids.at(i)); + spat_index_.rows_[i].storage_datums_[datum_idx + 1].set_string(mbr_val); + // not set_collation_type(CS_TYPE_BINARY) and set_collation_level(CS_LEVEL_IMPLICIT) + if (OB_FAIL(spat_index_.spat_rows_->push_back(spat_index_.rows_ + i))) { + LOG_WARN("failed to push back spatial index row", K(ret), K(spat_index_.rows_[i])); } } } @@ -3162,10 +3154,10 @@ int ObTableScanOp::inner_get_next_spatial_index_row() } } } - if (OB_SUCC(ret) && !need_ignore_null) { - ObNewRow &row = (*(spat_index_.spat_rows_))[spat_index_.spat_row_index_++]; - ObObj &cellid= row.get_cell(0); - ObObj &mbr = row.get_cell(1); + if (OB_SUCC(ret)) { + ObDatumRow *row = (*(spat_index_.spat_rows_))[spat_index_.spat_row_index_++]; + ObStorageDatum &cellid = row->storage_datums_[0]; + ObStorageDatum &mbr = row->storage_datums_[1]; if (OB_FAIL(fill_generated_cellid_mbr(cellid, mbr))) { LOG_WARN("fill cellid mbr failed", K(ret), K(cellid), K(mbr)); } @@ -3178,19 +3170,24 @@ int ObTableScanOp::init_spatial_index_rows() { int ret = OB_SUCCESS; void *buf = ctx_.get_allocator().alloc(sizeof(ObDomainIndexRow)); + void *row_buf = ctx_.get_allocator().alloc(sizeof(blocksstable::ObDatumRow) * SAPTIAL_INDEX_DEFAULT_ROW_COUNT); void *mbr_buffer = ctx_.get_allocator().alloc(OB_DEFAULT_MBR_SIZE); - void *obj_buf = ctx_.get_allocator().alloc(sizeof(ObObj) * 2 * SAPTIAL_INDEX_DEFAULT_ROW_COUNT); - if (OB_ISNULL(buf) || OB_ISNULL(mbr_buffer) || OB_ISNULL(obj_buf)) { + if (OB_ISNULL(buf) || OB_ISNULL(mbr_buffer) || OB_ISNULL(row_buf)) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("allocate spatial row store failed", K(ret), K(buf), K(mbr_buffer)); } else { spat_index_.spat_rows_ = new(buf) ObDomainIndexRow(); + spat_index_.rows_ = new(row_buf) blocksstable::ObDatumRow[SAPTIAL_INDEX_DEFAULT_ROW_COUNT]; spat_index_.mbr_buffer_ = mbr_buffer; - spat_index_.obj_buffer_ = obj_buf; const ObExprPtrIArray &exprs = MY_SPEC.output_; const uint8_t spatial_expr_cnt = 3; uint8_t cnt = 0; - for (uint32_t i = 0; i < exprs.count() && cnt < spatial_expr_cnt; i++) { + for (uint32_t i = 0; OB_SUCC(ret) && i < SAPTIAL_INDEX_DEFAULT_ROW_COUNT; i++) { + if (OB_FAIL(spat_index_.rows_[i].init(SAPTIAL_INDEX_DEFAULT_COL_COUNT))) { + LOG_WARN("init datum row failed", K(ret)); + } + } + for (uint32_t i = 0; OB_SUCC(ret) && i < exprs.count() && cnt < spatial_expr_cnt; i++) { if (exprs.at(i)->type_ == T_FUN_SYS_SPATIAL_CELLID) { spat_index_.cell_idx_ = i; cnt++; @@ -3202,7 +3199,7 @@ int ObTableScanOp::init_spatial_index_rows() cnt++; } } - if (cnt != spatial_expr_cnt) { + if (OB_FAIL(ret) || cnt != spatial_expr_cnt) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid spatial index exprs", K(ret), K(cnt)); } @@ -3210,7 +3207,7 @@ int ObTableScanOp::init_spatial_index_rows() return ret; } -int ObTableScanOp::fill_generated_cellid_mbr(const ObObj &cellid, const ObObj &mbr) +int ObTableScanOp::fill_generated_cellid_mbr(const ObStorageDatum &cellid, const ObStorageDatum &mbr) { int ret = OB_SUCCESS; const ObExprPtrIArray &exprs = MY_SPEC.output_; @@ -3220,12 +3217,12 @@ int ObTableScanOp::fill_generated_cellid_mbr(const ObObj &cellid, const ObObj &m } else { for (uint8_t i = 0; i < 2 && OB_SUCC(ret); i++) { ObObjDatumMapType type = i == 0 ? OBJ_DATUM_8BYTE_DATA : OBJ_DATUM_STRING; - const ObObj &value = i == 0 ? cellid : mbr; + const ObStorageDatum &value = i == 0 ? cellid : mbr; uint32_t idx = i == 0 ? spat_index_.cell_idx_ : spat_index_.mbr_idx_; ObExpr *expr = exprs.at(idx); ObDatum *datum = &expr->locate_datum_for_write(get_eval_ctx()); ObEvalInfo *eval_info = &expr->get_eval_info(get_eval_ctx()); - if (OB_FAIL(datum->from_obj(value, type))) { + if (OB_FAIL(datum->from_storage_datum(value, type))) { LOG_WARN("fill spatial index row failed", K(ret)); } else { eval_info->evaluated_ = true; diff --git a/src/sql/engine/table/ob_table_scan_op.h b/src/sql/engine/table/ob_table_scan_op.h index a1d814cd3..5ea75621a 100644 --- a/src/sql/engine/table/ob_table_scan_op.h +++ b/src/sql/engine/table/ob_table_scan_op.h @@ -71,18 +71,18 @@ struct ObSpatialIndexCache public: ObSpatialIndexCache() : spat_rows_(nullptr), + rows_(nullptr), spat_row_index_(0), mbr_buffer_(nullptr), - obj_buffer_(nullptr), geo_idx_(0), cell_idx_(0), mbr_idx_(0) {} ~ObSpatialIndexCache() {}; ObDomainIndexRow *spat_rows_; + blocksstable::ObDatumRow *rows_; uint8_t spat_row_index_; void *mbr_buffer_; - void *obj_buffer_; uint32_t geo_idx_; uint32_t cell_idx_; uint32_t mbr_idx_; @@ -509,7 +509,7 @@ protected: ObTableScanStat &scan_stat) const; void set_cache_stat(const ObPlanStat &plan_stat); int inner_get_next_row_implement(); - int fill_generated_cellid_mbr(const ObObj &cellid, const ObObj &mbr); + int fill_generated_cellid_mbr(const ObStorageDatum &cellid, const ObStorageDatum &mbr); int inner_get_next_spatial_index_row(); int init_spatial_index_rows(); void set_real_rescan_cnt(int64_t real_rescan_cnt) { group_rescan_cnt_ = real_rescan_cnt; } diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index f54ea23ef..44ae7d35d 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -43,8 +43,11 @@ ob_set_subtarget(ob_storage blocksstable blocksstable/ob_sstable_printer.cpp blocksstable/ob_storage_cache_suite.cpp blocksstable/ob_super_block_buffer_holder.cpp + blocksstable/ob_storage_datum.cpp blocksstable/ob_datum_row.cpp blocksstable/ob_datum_rowkey.cpp + blocksstable/ob_datum_row_store.cpp + blocksstable/ob_datum_row_utils.cpp blocksstable/ob_data_store_desc.cpp blocksstable/ob_table_flag.cpp blocksstable/ob_datum_rowkey_vector.cpp diff --git a/src/storage/access/ob_rows_info.cpp b/src/storage/access/ob_rows_info.cpp index 576a14888..9bab938d4 100644 --- a/src/storage/access/ob_rows_info.cpp +++ b/src/storage/access/ob_rows_info.cpp @@ -14,6 +14,7 @@ #include "storage/ob_storage_struct.h" #include "storage/ob_relative_table.h" #include "storage/ob_storage_schema.h" +#include "storage/blocksstable/ob_datum_row_utils.h" #include "ob_store_row_iterator.h" namespace oceanbase @@ -93,6 +94,7 @@ ObRowsInfo::ObRowsInfo() tablet_id_(), datum_utils_(nullptr), min_key_(), + col_descs_(nullptr), conflict_rowkey_idx_(-1), error_code_(0), delete_count_(0), @@ -124,9 +126,11 @@ void ObRowsInfo::reset() error_code_ = 0; conflict_rowkey_idx_ = -1; is_inited_ = false; + col_descs_ = nullptr; } int ObRowsInfo::init( + const ObColDescIArray &column_descs, const ObRelativeTable &table, ObStoreCtx &store_ctx, const ObITableReadInfo &rowkey_read_info) @@ -139,6 +143,7 @@ int ObRowsInfo::init( } else if (OB_FAIL(exist_helper_.init(table, store_ctx, rowkey_read_info, exist_allocator_, scan_mem_allocator_))) { STORAGE_LOG(WARN, "Failed to init exist helper", K(ret)); } else { + col_descs_ = &column_descs; datum_utils_ = &rowkey_read_info.get_datum_utils(); tablet_id_ = table.get_tablet_id(); rowkey_column_num_ = table.get_rowkey_column_num(); @@ -160,7 +165,7 @@ void ObRowsInfo::reuse() } //not only checking duplicate, but also assign rowkeys -int ObRowsInfo::check_duplicate(ObStoreRow *rows, const int64_t row_count, ObRelativeTable &table) +int ObRowsInfo::check_duplicate(ObDatumRow *rows, const int64_t row_count, ObRelativeTable &table) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -192,10 +197,9 @@ int ObRowsInfo::check_duplicate(ObStoreRow *rows, const int64_t row_count, ObRel } else { ObMarkedRowkeyAndLockState marked_rowkey_and_lock_state; marked_rowkey_and_lock_state.row_idx_ = i; - ObRowkey rowkey(rows_[i].row_val_.cells_, rowkey_column_num_); - if (OB_FAIL(marked_rowkey_and_lock_state.marked_rowkey_.get_rowkey().from_rowkey(rowkey, - key_allocator_))) { - STORAGE_LOG(WARN, "Failed to transfer rowkey", K(ret), K(rowkey)); + ObDatumRowkey &datum_rowkey = marked_rowkey_and_lock_state.marked_rowkey_.get_rowkey(); + if (OB_FAIL(blocksstable::ObDatumRowUtils::prepare_rowkey(rows[i], rowkey_column_num_, *col_descs_, key_allocator_, datum_rowkey))) { + STORAGE_LOG(WARN, "Failed to prepare rowkey", K(ret), K(rowkey_column_num_), K(rows_[i])); } else if (OB_FAIL(rowkeys_.push_back(marked_rowkey_and_lock_state))) { STORAGE_LOG(WARN, "Failed to push back rowkey", K(ret), K(marked_rowkey_and_lock_state)); } diff --git a/src/storage/access/ob_rows_info.h b/src/storage/access/ob_rows_info.h index 604f1fde8..05caf2fc0 100644 --- a/src/storage/access/ob_rows_info.h +++ b/src/storage/access/ob_rows_info.h @@ -90,10 +90,11 @@ public: return delete_count_ == rowkeys_.count(); } int init( + const ObColDescIArray &column_descs, const ObRelativeTable &table, ObStoreCtx &store_ctx, const ObITableReadInfo &rowkey_read_info); - int check_duplicate(ObStoreRow *rows, const int64_t row_count, ObRelativeTable &table); + int check_duplicate(blocksstable::ObDatumRow *rows, const int64_t row_count, ObRelativeTable &table); blocksstable::ObDatumRowkey& get_duplicate_rowkey() { return min_key_; @@ -265,12 +266,13 @@ private: public: ObRowkeyAndLockStates rowkeys_; ObPermutation permutation_; - ObStoreRow *rows_; + blocksstable::ObDatumRow *rows_; ExistHelper exist_helper_; ObTabletID tablet_id_; private: const blocksstable::ObStorageDatumUtils *datum_utils_; blocksstable::ObDatumRowkey min_key_; + const ObColDescIArray *col_descs_; int64_t conflict_rowkey_idx_; int error_code_; int64_t delete_count_; diff --git a/src/storage/access/ob_table_access_context.cpp b/src/storage/access/ob_table_access_context.cpp index eea822cb2..4e68b9c90 100644 --- a/src/storage/access/ob_table_access_context.cpp +++ b/src/storage/access/ob_table_access_context.cpp @@ -266,10 +266,12 @@ int ObTableAccessContext::init(const common::ObQueryFlag &query_flag, } return ret; } + int ObTableAccessContext::init(const common::ObQueryFlag &query_flag, ObStoreCtx &ctx, common::ObIAllocator &allocator, - const common::ObVersionRange &trans_version_range) + const common::ObVersionRange &trans_version_range, + CachedIteratorNode *cached_iter_node) { int ret = OB_SUCCESS; if (is_inited_) { @@ -288,6 +290,7 @@ int ObTableAccessContext::init(const common::ObQueryFlag &query_flag, ls_id_ = ctx.ls_id_; tablet_id_ = ctx.tablet_id_; lob_locator_helper_ = nullptr; + cached_iter_node_ = cached_iter_node; if (!micro_block_handle_mgr_.is_valid() && OB_FAIL(micro_block_handle_mgr_.init(enable_limit, table_store_stat_, query_flag_))) { LOG_WARN("Fail to init micro block handle mgr", K(ret)); diff --git a/src/storage/access/ob_table_access_context.h b/src/storage/access/ob_table_access_context.h index 987145afb..93db844a6 100644 --- a/src/storage/access/ob_table_access_context.h +++ b/src/storage/access/ob_table_access_context.h @@ -199,7 +199,8 @@ struct ObTableAccessContext int init(const common::ObQueryFlag &query_flag, ObStoreCtx &ctx, common::ObIAllocator &allocator, - const common::ObVersionRange &trans_version_range); + const common::ObVersionRange &trans_version_range, + CachedIteratorNode *cached_iter_node = nullptr); int alloc_iter_pool(const bool use_column_store); void inc_micro_access_cnt(); int init_scan_allocator(ObTableScanParam &scan_param); diff --git a/src/storage/blocksstable/ob_datum_row.cpp b/src/storage/blocksstable/ob_datum_row.cpp index a71b34464..cab794458 100644 --- a/src/storage/blocksstable/ob_datum_row.cpp +++ b/src/storage/blocksstable/ob_datum_row.cpp @@ -22,73 +22,6 @@ namespace oceanbase using namespace common; namespace blocksstable { -static int nonext_nonext_compare(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret) -{ - int ret = cmp_func.cmp_func_(left, right, cmp_ret); - STORAGE_LOG(DEBUG, "chaser debug compare datum", K(ret), K(left), K(right), K(cmp_ret)); - return ret; -} - -static int nonext_ext_compare(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret) -{ - int ret = OB_SUCCESS; - UNUSEDx(left, cmp_func); - if (right.is_max()) { - cmp_ret = -1; - } else if (right.is_min()) { - cmp_ret = 1; - } else { - ret = OB_ERR_SYS; - STORAGE_LOG(ERROR, "Unexpected datum in rowkey to compare", K(ret), K(right)); - } - STORAGE_LOG(DEBUG, "chaser debug compare datum", K(ret), K(left), K(right), K(cmp_ret)); - return ret; -} - -static int ext_nonext_compare(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret) -{ - int ret = OB_SUCCESS; - UNUSEDx(right, cmp_func); - if (left.is_max()) { - cmp_ret = 1; - } else if (left.is_min()) { - cmp_ret = -1; - } else { - ret = OB_ERR_SYS; - STORAGE_LOG(ERROR, "Unexpected datum in rowkey to compare", K(ret), K(left)); - } - STORAGE_LOG(DEBUG, "chaser debug compare datum", K(ret), K(left), K(right), K(cmp_ret)); - return ret; -} - -static int ext_ext_compare(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret) -{ - int ret = OB_SUCCESS; - UNUSEDx(cmp_func); - int64_t lv = left.is_max() - left.is_min(); - int64_t rv = right.is_max() - right.is_min(); - if (OB_UNLIKELY(0 == lv || 0 == rv)) { - ret = OB_ERR_SYS; - STORAGE_LOG(ERROR, "Unexpected datum in rowkey to compare", K(ret), K(left), K(right)); - } else { - cmp_ret = lv - rv; - } - STORAGE_LOG(DEBUG, "chaser debug compare datum", K(ret), K(left), K(right), K(cmp_ret)); - - return ret; -} - -typedef int (*ExtSafeCompareFunc)(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret); -static ExtSafeCompareFunc ext_safe_cmp_funcs[2][2] = { - {nonext_nonext_compare, nonext_ext_compare}, - {ext_nonext_compare, ext_ext_compare} -}; - -int ObStorageDatumCmpFunc::compare(const ObStorageDatum &left, const ObStorageDatum &right, int &cmp_ret) const -{ - return ext_safe_cmp_funcs[left.is_ext()][right.is_ext()](left, right, cmp_func_, cmp_ret); -} - const char *get_dml_str(ObDmlFlag dml_flag) { @@ -106,86 +39,6 @@ OB_SERIALIZE_MEMBER(ObDmlRowFlag, whole_flag_); OB_SERIALIZE_MEMBER(ObMultiVersionRowFlag, flag_); -/* - *ObStorageDatumBuffer - */ -ObStorageDatumBuffer::ObStorageDatumBuffer(common::ObIAllocator *allocator) - : capacity_(LOCAL_BUFFER_ARRAY), - local_datums_(), - datums_(local_datums_), - allocator_(allocator), - is_inited_(nullptr != allocator) -{} - -ObStorageDatumBuffer::~ObStorageDatumBuffer() -{ - if (datums_ != local_datums_ && nullptr != allocator_) { - allocator_->free(datums_); - } -} - -void ObStorageDatumBuffer::reset() -{ - if (datums_ != local_datums_ && nullptr != allocator_) { - allocator_->free(datums_); - } - allocator_ = nullptr; - datums_ = local_datums_; - capacity_ = LOCAL_BUFFER_ARRAY; - for (int64_t i = 0; i < capacity_; i++) { - datums_[i].reuse(); - } - is_inited_ = false; -} - -int ObStorageDatumBuffer::init(common::ObIAllocator &allocator) -{ - int ret = OB_SUCCESS; - - if (IS_INIT) { - ret = OB_INIT_TWICE; - STORAGE_LOG(WARN, "ObStorageDatumBuffer init twice", K(ret), K(*this)); - } else { - OB_ASSERT(datums_ == local_datums_); - allocator_ = &allocator; - is_inited_ = true; - } - - return ret; -} - -int ObStorageDatumBuffer::reserve(const int64_t count, const bool keep_data) -{ - int ret = OB_SUCCESS; - void *buf = nullptr; - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObStorageDatumBuffer is not inited", K(ret), K(*this)); - } else if (OB_UNLIKELY(count <= 0)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "Invalid argument to reserve datum buffer", K(ret), K(count)); - } else if (count <= capacity_){ - } else if (OB_ISNULL(buf = allocator_->alloc(sizeof(ObStorageDatum) * count))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - STORAGE_LOG(WARN, "Failed to alloc memory", K(ret), K(count)); - } else { - ObStorageDatum *new_datums = new (buf) ObStorageDatum [count]; - if (keep_data) { - for (int64_t i = 0; i < capacity_; i++) { - new_datums[i] = datums_[i]; - } - } - if (nullptr != datums_ && datums_ != local_datums_) { - allocator_->free(datums_); - } - datums_ = new_datums; - capacity_ = count; - } - - return ret; -} - /* *ObConstDatumRow */ @@ -232,8 +85,7 @@ ObDatumRow::ObDatumRow(const uint64_t tenant_id) } ObDatumRow::~ObDatumRow() -{ -} +{} int ObDatumRow::init(ObIAllocator &allocator, const int64_t capacity, char *trans_info_ptr) { @@ -256,7 +108,9 @@ int ObDatumRow::init(ObIAllocator &allocator, const int64_t capacity, char *tran } return ret; + } + int ObDatumRow::init(const int64_t capacity) { int ret = OB_SUCCESS; @@ -442,6 +296,27 @@ int ObDatumRow::is_datums_changed(const ObDatumRow &other, bool &is_changed) con return ret; } +int ObDatumRow::shallow_copy(const ObDatumRow &other) +{ + int ret = OB_SUCCESS; + if (!other.is_valid()) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "Unexpected error for shallow copy invalid datum row", K(ret), K(*this), K(other)); + } else { + trans_info_ = nullptr; + fast_filter_skipped_ = other.fast_filter_skipped_; + storage_datums_ = other.storage_datums_; + snapshot_version_ = other.snapshot_version_; + group_idx_ = other.group_idx_; + scan_index_ = other.scan_index_; + trans_id_ = other.trans_id_; + mvcc_row_flag_ = other.mvcc_row_flag_; + row_flag_ = other.row_flag_; + count_ = other.count_; + } + return ret; +} + OB_DEF_SERIALIZE(ObDatumRow) { int ret = OB_SUCCESS; @@ -503,8 +378,8 @@ DEF_TO_STRING(ObDatumRow) { int64_t pos = 0; J_OBJ_START(); - J_KV(K_(row_flag), K_(trans_id), K_(scan_index), K_(mvcc_row_flag), - K_(snapshot_version), K_(fast_filter_skipped), K_(have_uncommited_row), K_(group_idx), K_(count), K_(datum_buffer)); + J_KV(K_(row_flag), K_(trans_id), K_(scan_index), K_(mvcc_row_flag), K_(snapshot_version), K_(fast_filter_skipped), + K_(have_uncommited_row), K_(group_idx), K_(count), K_(datum_buffer)); if (NULL != buf && buf_len >= 0) { if (NULL != storage_datums_) { J_COMMA(); @@ -615,219 +490,6 @@ int ObNewRowBuilder::build_store_row( return ret; } -/* - *ObStorageDatumUtils - */ -ObStorageDatumUtils::ObStorageDatumUtils() - : rowkey_cnt_(0), - cmp_funcs_(), - hash_funcs_(), - ext_hash_func_(), - is_oracle_mode_(false), - is_inited_(false) -{} - -ObStorageDatumUtils::~ObStorageDatumUtils() -{} - -int ObStorageDatumUtils::transform_multi_version_col_desc(const ObIArray &col_descs, - const int64_t schema_rowkey_cnt, - ObIArray &mv_col_descs) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(schema_rowkey_cnt > col_descs.count())) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "Invalid argument to transform mv col descs", K(ret), K(schema_rowkey_cnt), K(col_descs)); - } else { - mv_col_descs.reuse(); - for (int64_t i = 0; OB_SUCC(ret) && i < schema_rowkey_cnt; i++) { - if (OB_FAIL(mv_col_descs.push_back(col_descs.at(i)))) { - STORAGE_LOG(WARN, "Failed to push back col desc", K(ret), K(i)); - } - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(storage::ObMultiVersionRowkeyHelpper::add_extra_rowkey_cols(mv_col_descs))) { - STORAGE_LOG(WARN, "Fail to add extra_rowkey_cols", K(ret), K(schema_rowkey_cnt)); - } else { - for (int64_t i = schema_rowkey_cnt; OB_SUCC(ret) && i < col_descs.count(); i++) { - const share::schema::ObColDesc &col_desc = col_descs.at(i); - if (col_desc.col_id_ == common::OB_HIDDEN_TRANS_VERSION_COLUMN_ID - || col_desc.col_id_ == common::OB_HIDDEN_SQL_SEQUENCE_COLUMN_ID) { - continue; - } else if (OB_FAIL(mv_col_descs.push_back(col_desc))) { - STORAGE_LOG(WARN, "Failed to push back col desc", K(ret), K(col_desc)); - } - } - } - } - - return ret; -} - -int ObStorageDatumUtils::init(const ObIArray &col_descs, - const int64_t schema_rowkey_cnt, - const bool is_oracle_mode, - ObIAllocator &allocator, - const bool is_column_store) -{ - int ret = OB_SUCCESS; - ObSEArray mv_col_descs; - int64_t mv_rowkey_cnt = 0; - int64_t mv_column_cnt = 0; - if (IS_INIT) { - ret = OB_INIT_TWICE; - STORAGE_LOG(WARN, "ObStorageDatumUtils init twice", K(ret), K(*this)); - } else if (OB_UNLIKELY(schema_rowkey_cnt < 0 || schema_rowkey_cnt > OB_MAX_ROWKEY_COLUMN_NUMBER - || schema_rowkey_cnt > col_descs.count())) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "Invalid argument to init storage datum utils", K(ret), K(col_descs), K(schema_rowkey_cnt)); - } else if (OB_FAIL(transform_multi_version_col_desc(col_descs, schema_rowkey_cnt, mv_col_descs))) { - STORAGE_LOG(WARN, "Failed to transform multi version col descs", K(ret)); - } else if (FALSE_IT(mv_column_cnt = is_column_store ? 0 : storage::ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt())) { - } else if (FALSE_IT(mv_rowkey_cnt = schema_rowkey_cnt + mv_column_cnt)) { - } else if (OB_FAIL(cmp_funcs_.init(mv_rowkey_cnt, allocator))) { - STORAGE_LOG(WARN, "Failed to reserve cmp func array", K(ret)); - } else if (OB_FAIL(hash_funcs_.init(mv_rowkey_cnt, allocator))) { - STORAGE_LOG(WARN, "Failed to reserve hash func array", K(ret)); - } else if (OB_FAIL(inner_init(mv_col_descs, mv_rowkey_cnt, is_oracle_mode))) { - STORAGE_LOG(WARN, "Failed to inner init datum utils", K(ret), K(mv_col_descs), K(mv_rowkey_cnt)); - } - - return ret; -} - -int ObStorageDatumUtils::init(const common::ObIArray &col_descs, - const int64_t schema_rowkey_cnt, - const bool is_oracle_mode, - const int64_t arr_buf_len, - char *arr_buf) -{ - int ret = OB_SUCCESS; - ObSEArray mv_col_descs; - int64_t pos = 0; - int64_t mv_rowkey_cnt = 0; - - if (IS_INIT) { - ret = OB_INIT_TWICE; - STORAGE_LOG(WARN, "ObStorageDatumUtils init twice", K(ret), K(*this)); - } else if (OB_UNLIKELY(schema_rowkey_cnt < 0 || schema_rowkey_cnt > OB_MAX_ROWKEY_COLUMN_NUMBER - || schema_rowkey_cnt > col_descs.count())) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "Invalid argument to init storage datum utils", K(ret), K(col_descs), K(schema_rowkey_cnt)); - } else if (OB_FAIL(transform_multi_version_col_desc(col_descs, schema_rowkey_cnt, mv_col_descs))) { - STORAGE_LOG(WARN, "Failed to transform multi version col descs", K(ret)); - } else if (FALSE_IT(mv_rowkey_cnt = schema_rowkey_cnt + storage::ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt())) { - } else if (OB_FAIL(cmp_funcs_.init(mv_rowkey_cnt, arr_buf_len, arr_buf, pos))) { - STORAGE_LOG(WARN, "Failed to init compare function array", K(ret)); - } else if (OB_FAIL(hash_funcs_.init(mv_rowkey_cnt, arr_buf_len, arr_buf, pos))) { - STORAGE_LOG(WARN, "Failed to init hash function array", K(ret)); - } else if (OB_FAIL(inner_init(mv_col_descs, mv_rowkey_cnt, is_oracle_mode))) { - STORAGE_LOG(WARN, "Failed to inner init datum utils", K(ret), K(mv_col_descs), K(mv_rowkey_cnt)); - } - return ret; -} - -int ObStorageDatumUtils::inner_init( - const common::ObIArray &mv_col_descs, - const int64_t mv_rowkey_col_cnt, - const bool is_oracle_mode) -{ - int ret = OB_SUCCESS; - is_oracle_mode_ = is_oracle_mode; - // support column order index until next task done - // - // we could use the cmp funcs in the basic funcs directlly - bool is_null_last = is_oracle_mode_; - ObCmpFunc cmp_func; - ObHashFunc hash_func; - for (int64_t i = 0; OB_SUCC(ret) && i < mv_rowkey_col_cnt; i++) { - const share::schema::ObColDesc &col_desc = mv_col_descs.at(i); - //TODO @hanhui support desc rowkey - bool is_ascending = true || col_desc.col_order_ == ObOrderType::ASC; - bool has_lob_header = is_lob_storage(col_desc.col_type_.get_type()); - ObPrecision precision = PRECISION_UNKNOWN_YET; - if (col_desc.col_type_.is_decimal_int()) { - precision = col_desc.col_type_.get_stored_precision(); - OB_ASSERT(precision != PRECISION_UNKNOWN_YET); - } - sql::ObExprBasicFuncs *basic_funcs = ObDatumFuncs::get_basic_func(col_desc.col_type_.get_type(), - col_desc.col_type_.get_collation_type(), - col_desc.col_type_.get_scale(), - is_oracle_mode, - has_lob_header, - precision); - if (OB_UNLIKELY(nullptr == basic_funcs - || nullptr == basic_funcs->null_last_cmp_ - || nullptr == basic_funcs->murmur_hash_)) { - ret = OB_ERR_SYS; - STORAGE_LOG(ERROR, "Unexpected null basic funcs", K(ret), K(col_desc)); - } else { - cmp_func.cmp_func_ = is_null_last ? basic_funcs->null_last_cmp_ : basic_funcs->null_first_cmp_; - hash_func.hash_func_ = basic_funcs->murmur_hash_; - if (OB_FAIL(hash_funcs_.push_back(hash_func))) { - STORAGE_LOG(WARN, "Failed to push back hash func", K(ret), K(i), K(col_desc)); - } else if (is_ascending) { - if (OB_FAIL(cmp_funcs_.push_back(ObStorageDatumCmpFunc(cmp_func)))) { - STORAGE_LOG(WARN, "Failed to push back cmp func", K(ret), K(i), K(col_desc)); - } - } else { - ret = OB_ERR_SYS; - STORAGE_LOG(WARN, "Unsupported desc column order", K(ret), K(col_desc), K(i)); - } - } - } - if (OB_SUCC(ret)) { - sql::ObExprBasicFuncs *basic_funcs = ObDatumFuncs::get_basic_func(ObExtendType, CS_TYPE_BINARY); - if (OB_UNLIKELY(nullptr == basic_funcs || nullptr == basic_funcs->murmur_hash_)) { - ret = OB_ERR_SYS; - STORAGE_LOG(ERROR, "Unexpected null basic funcs for extend type", K(ret)); - } else { - ext_hash_func_.hash_func_ = basic_funcs->murmur_hash_; - rowkey_cnt_ = mv_rowkey_col_cnt; - is_inited_ = true; - } - } - - return ret; -} - -int ObStorageDatumUtils::assign(const ObStorageDatumUtils &other_utils, ObIAllocator &allocator) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!other_utils.is_valid())) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "Invalid argument to assign datum utils", K(ret), K(other_utils)); - } else { - rowkey_cnt_ = other_utils.get_rowkey_count(); - is_oracle_mode_ = other_utils.is_oracle_mode(); - ext_hash_func_ = other_utils.get_ext_hash_funcs(); - if (OB_FAIL(cmp_funcs_.init_and_assign(other_utils.get_cmp_funcs(), allocator))) { - STORAGE_LOG(WARN, "Failed to assign cmp func array", K(ret)); - } else if (OB_FAIL(hash_funcs_.init_and_assign(other_utils.get_hash_funcs(), allocator))) { - STORAGE_LOG(WARN, "Failed to assign hash func array", K(ret)); - } else { - is_inited_ = true; - } - } - - return ret; -} - -void ObStorageDatumUtils::reset() -{ - rowkey_cnt_ = 0; - cmp_funcs_.reset(); - hash_funcs_.reset(); - ext_hash_func_.hash_func_ = nullptr; - is_inited_ = false; -} - -int64_t ObStorageDatumUtils::get_deep_copy_size() const -{ - return cmp_funcs_.get_deep_copy_size() + hash_funcs_.get_deep_copy_size(); -} - int ObGhostRowUtil::is_ghost_row( const blocksstable::ObMultiVersionRowFlag &flag, bool &is_ghost_row) diff --git a/src/storage/blocksstable/ob_datum_row.h b/src/storage/blocksstable/ob_datum_row.h index 7bb5deda9..23309fa66 100644 --- a/src/storage/blocksstable/ob_datum_row.h +++ b/src/storage/blocksstable/ob_datum_row.h @@ -21,6 +21,7 @@ #include "storage/tx/ob_trans_define.h" #include "common/row/ob_row.h" #include "storage/ob_storage_util.h" +#include "storage/blocksstable/ob_datum_rowkey.h" namespace oceanbase { @@ -38,6 +39,7 @@ namespace blocksstable { struct ObDmlRowFlag; +struct ObDatumRowkey; enum ObDmlFlag { @@ -307,65 +309,6 @@ public: K_(flag)); }; -//TODO optimize number buffer -struct ObStorageDatum : public common::ObDatum -{ - ObStorageDatum() { set_nop(); } - ObStorageDatum(const ObStorageDatum &datum) { reuse(); *this = datum; } - - ~ObStorageDatum() = default; - // ext value section - OB_INLINE void reuse() { ptr_ = buf_; reserved_ = 0; pack_ = 0; } - OB_INLINE void set_ext_value(const int64_t ext_value) - { reuse(); set_ext(); no_cv(extend_obj_)->set_ext(ext_value); } - OB_INLINE void set_nop() { set_ext_value(ObActionFlag::OP_NOP); } - OB_INLINE void set_min() { set_ext_value(common::ObObj::MIN_OBJECT_VALUE); } - OB_INLINE void set_max() { set_ext_value(common::ObObj::MAX_OBJECT_VALUE); } - OB_INLINE bool is_nop_value() const { return is_nop(); } // temp solution - // transfer section - OB_INLINE bool is_local_buf() const { return ptr_ == buf_; } - OB_INLINE int from_buf_enhance(const char *buf, const int64_t buf_len); - OB_INLINE int from_obj_enhance(const common::ObObj &obj); - OB_INLINE int to_obj_enhance(common::ObObj &obj, const common::ObObjMeta &meta) const; - OB_INLINE int deep_copy(const ObStorageDatum &src, common::ObIAllocator &allocator); - OB_INLINE int deep_copy(const ObStorageDatum &src, char * buf, const int64_t buf_len, int64_t &pos); - OB_INLINE void shallow_copy_from_datum(const ObDatum &src); - OB_INLINE int64_t get_deep_copy_size() const; - OB_INLINE ObStorageDatum& operator=(const ObStorageDatum &other); - OB_INLINE int64_t storage_to_string(char *buf, int64_t buf_len, const bool for_dump = false) const; - OB_INLINE bool need_copy_for_encoding_column_with_flat_format(const ObObjDatumMapType map_type) const; - OB_INLINE const char *to_cstring(const bool for_dump = false) const; - //only for unittest - OB_INLINE bool operator==(const ObStorageDatum &other) const; - OB_INLINE bool operator==(const ObObj &other) const; - - //datum 12 byte - int32_t reserved_; - // buf 16 byte - char buf_[common::OBJ_DATUM_NUMBER_RES_SIZE]; -}; - -struct ObStorageDatumBuffer -{ -public: - ObStorageDatumBuffer(common::ObIAllocator *allocator = nullptr); - ~ObStorageDatumBuffer(); - void reset(); - int init(common::ObIAllocator &allocator); - int reserve(const int64_t count, const bool keep_data = false); - OB_INLINE bool is_valid() const { return is_inited_; } - OB_INLINE ObStorageDatum *get_datums() { return datums_; } - OB_INLINE int64_t get_capacity() const { return capacity_; } - TO_STRING_KV(K_(capacity), KP_(datums), KP_(local_datums)); -private: - static const int64_t LOCAL_BUFFER_ARRAY = common::OB_ROW_DEFAULT_COLUMNS_COUNT; - int64_t capacity_; - ObStorageDatum local_datums_[LOCAL_BUFFER_ARRAY]; - ObStorageDatum *datums_; - common::ObIAllocator *allocator_; - bool is_inited_; -}; - struct ObDatumRow { OB_UNIS_VERSION(1); @@ -379,6 +322,7 @@ public: int reserve(const int64_t capacity, const bool keep_data = false); int deep_copy(const ObDatumRow &src, common::ObIAllocator &allocator); int from_store_row(const storage::ObStoreRow &store_row); + int shallow_copy(const ObDatumRow &other); //only for unittest bool operator==(const ObDatumRow &other) const; bool operator==(const common::ObNewRow &other) const; @@ -487,333 +431,6 @@ public: int64_t datum_row_offset_; }; -struct ObStorageDatumCmpFunc -{ -public: - ObStorageDatumCmpFunc(common::ObCmpFunc &cmp_func) : cmp_func_(cmp_func) {} - ObStorageDatumCmpFunc() = default; - ~ObStorageDatumCmpFunc() = default; - int compare(const ObStorageDatum &left, const ObStorageDatum &right, int &cmp_ret) const; - OB_INLINE const common::ObCmpFunc &get_cmp_func() const { return cmp_func_; } - TO_STRING_KV(K_(cmp_func)); -private: - common::ObCmpFunc cmp_func_; -}; -typedef storage::ObFixedMetaObjArray ObStoreCmpFuncs; -typedef storage::ObFixedMetaObjArray ObStoreHashFuncs; -struct ObStorageDatumUtils -{ -public: - ObStorageDatumUtils(); - ~ObStorageDatumUtils(); - // init with array memory from allocator - int init(const common::ObIArray &col_descs, - const int64_t schema_rowkey_cnt, - const bool is_oracle_mode, - common::ObIAllocator &allocator, - const bool is_column_store = false); - // init with array memory on fixed size memory buffer - int init(const common::ObIArray &col_descs, - const int64_t schema_rowkey_cnt, - const bool is_oracle_mode, - const int64_t arr_buf_len, - char *arr_buf); - int assign(const ObStorageDatumUtils &other_utils, common::ObIAllocator &allocator); - void reset(); - OB_INLINE bool is_valid() const - { - return is_inited_ && cmp_funcs_.count() >= rowkey_cnt_ && hash_funcs_.count() >= rowkey_cnt_; - } - OB_INLINE bool is_oracle_mode() const { return is_oracle_mode_; } - OB_INLINE int64_t get_rowkey_count() const { return rowkey_cnt_; } - OB_INLINE const ObStoreCmpFuncs &get_cmp_funcs() const { return cmp_funcs_; } - OB_INLINE const ObStoreHashFuncs &get_hash_funcs() const { return hash_funcs_; } - OB_INLINE const common::ObHashFunc &get_ext_hash_funcs() const { return ext_hash_func_; } - int64_t get_deep_copy_size() const; - TO_STRING_KV(K_(is_oracle_mode), K_(rowkey_cnt), K_(is_inited), K_(is_oracle_mode)); -private: - //TODO to be removed by @hanhui - int transform_multi_version_col_desc(const common::ObIArray &col_descs, - const int64_t schema_rowkey_cnt, - common::ObIArray &mv_col_descs); - int inner_init( - const common::ObIArray &mv_col_descs, - const int64_t mv_rowkey_col_cnt, - const bool is_oracle_mode); -private: - int32_t rowkey_cnt_; // multi version rowkey - ObStoreCmpFuncs cmp_funcs_; // multi version rowkey cmp funcs - ObStoreHashFuncs hash_funcs_; // multi version rowkey cmp funcs - common::ObHashFunc ext_hash_func_; - bool is_oracle_mode_; - bool is_inited_; - DISALLOW_COPY_AND_ASSIGN(ObStorageDatumUtils); -}; - - -OB_INLINE int ObStorageDatum::deep_copy(const ObStorageDatum &src, common::ObIAllocator &allocator) -{ - int ret = common::OB_SUCCESS; - - reuse(); - pack_ = src.pack_; - if (is_null()) { - } else if (src.len_ == 0) { - } else if (src.is_local_buf()) { - OB_ASSERT(src.len_ <= common::OBJ_DATUM_NUMBER_RES_SIZE); - MEMCPY(buf_, src.ptr_, src.len_); - ptr_ = buf_; - } else { - char * buf = static_cast(allocator.alloc(src.len_)); - if (OB_ISNULL(buf)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - COMMON_LOG(WARN, "allocate memory failed", K(ret), K(src)); - pack_ = 0; - } else { - MEMCPY(buf, src.ptr_, src.len_); - // need set ptr_ after memory copy, if this == &src - ptr_ = buf; - } - } - return ret; -} - -OB_INLINE int ObStorageDatum::deep_copy(const ObStorageDatum &src, char * buf, const int64_t buf_len, int64_t &pos) -{ - int ret = common::OB_SUCCESS; - - reuse(); - pack_ = src.pack_; - if (is_null()) { - } else if (src.len_ == 0) { - } else if (src.is_local_buf()) { - OB_ASSERT(src.len_ <= common::OBJ_DATUM_NUMBER_RES_SIZE); - MEMCPY(buf_, src.ptr_, src.len_); - ptr_ = buf_; - } else if (OB_UNLIKELY(nullptr == buf || buf_len < pos + src.len_)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "Invalid argument to deep copy datum", K(ret), K(src), KP(buf), K(buf_len), K(pos)); - pack_ = 0; - } else { - MEMCPY(buf + pos, src.ptr_, src.len_); - // need set ptr_ after memory copy, if this == &src - ptr_ = buf + pos; - pos += src.len_; - } - - return ret; -} - -OB_INLINE void ObStorageDatum::shallow_copy_from_datum(const ObDatum &src) -{ - if (this != &src) { - reuse(); - pack_ = src.pack_; - if (is_null()) { - } else if (src.len_ == 0) { - } else { - ptr_ = src.ptr_; - } - } -} - -OB_INLINE int64_t ObStorageDatum::get_deep_copy_size() const -{ - int64_t deep_copy_len = 0; - if (is_null()) { - } else if (is_local_buf()) { - OB_ASSERT(len_ <= common::OBJ_DATUM_NUMBER_RES_SIZE); - } else { - deep_copy_len = len_; - } - return deep_copy_len; -} - -OB_INLINE int ObStorageDatum::from_buf_enhance(const char *buf, const int64_t buf_len) -{ - int ret = common::OB_SUCCESS; - - if (OB_UNLIKELY(nullptr == buf || buf_len < 0 || buf_len > UINT32_MAX)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "Invalid argument to transfer from buf", K(ret), KP(buf), K(buf_len)); - } else { - reuse(); - len_ = static_cast(buf_len); - if (buf_len > 0) { - ptr_ = buf; - } - } - - - return ret; -} - -OB_INLINE int ObStorageDatum::from_obj_enhance(const common::ObObj &obj) -{ - int ret = common::OB_SUCCESS; - - reuse(); - if (obj.is_ext()) { - set_ext_value(obj.get_ext()); - } else if (OB_FAIL(from_obj(obj))) { - STORAGE_LOG(WARN, "Failed to transfer obj to datum", K(ret), K(obj)); - } - STORAGE_LOG(DEBUG, "chaser debug from obj", K(obj), K(*this)); - - return ret; -} - - -OB_INLINE int ObStorageDatum::to_obj_enhance(common::ObObj &obj, const common::ObObjMeta &meta) const -{ - int ret = common::OB_SUCCESS; - if (is_outrow()) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "lob should not set outrow in datum", K(ret), K(*this), K(obj), K(meta)); - } else if (is_ext()) { - obj.set_ext(get_ext()); - } else if (OB_FAIL(to_obj(obj, meta))) { - STORAGE_LOG(WARN, "Failed to transfer datum to obj", K(ret), K(*this), K(obj), K(meta)); - } - - return ret; -} - -OB_INLINE ObStorageDatum& ObStorageDatum::operator=(const ObStorageDatum &other) -{ - if (&other != this) { - reuse(); - pack_ = other.pack_; - if (is_null()) { - } else if (len_ == 0) { - } else if (other.is_local_buf()) { - OB_ASSERT(other.len_ <= common::OBJ_DATUM_NUMBER_RES_SIZE); - MEMCPY(buf_, other.ptr_, other.len_); - ptr_ = buf_; - } else { - ptr_ = other.ptr_; - } - } - return *this; -} - -OB_INLINE bool ObStorageDatum::operator==(const ObStorageDatum &other) const -{ - bool bret = true; - if (is_null()) { - bret = other.is_null(); - } else if (is_ext()) { - bret = other.is_ext() && extend_obj_->get_ext() == other.extend_obj_->get_ext(); - } else { - bret = ObDatum::binary_equal(*this, other); - } - if (!bret) { - STORAGE_LOG(DEBUG, "obj and datum no equal", K(other), K(*this)); - } - return bret; - -} - -OB_INLINE bool ObStorageDatum::operator==(const common::ObObj &other) const -{ - - int ret = OB_SUCCESS; - bool bret = true; - ObStorageDatum datum; - if (OB_FAIL(datum.from_obj_enhance(other))) { - STORAGE_LOG(WARN, "Failed to transfer obj to datum", K(ret), K(other), K(datum)); - } else { - bret = *this == datum; - } - if (!bret) { - STORAGE_LOG(DEBUG, "obj and datum no equal", K(other), K(datum), KPC(this)); - } - return bret; -} - -OB_INLINE int64_t ObStorageDatum::storage_to_string(char *buf, int64_t buf_len, const bool for_dump) const -{ - int64_t pos = 0; - if (is_ext()) { - if (is_nop()) { - J_NOP(); - } else if (is_max()) { - BUF_PRINTF("MAX_OBJ"); - } else if (is_min()) { - BUF_PRINTF("MIN_OBJ"); - } - } else if(!for_dump) { - pos = to_string(buf, buf_len); - } else { - int ret = OB_SUCCESS; - const static int64_t STR_MAX_PRINT_LEN = 128L; - if (null_) { - J_NULL(); - } else { - J_OBJ_START(); - BUF_PRINTF("len: %d, flag: %d, null: %d", len_, flag_, null_); - if (len_ > 0) { - OB_ASSERT(NULL != ptr_); - const int64_t plen = std::min(static_cast(len_), - static_cast(STR_MAX_PRINT_LEN)); - // print hex value - BUF_PRINTF(", hex: "); - if (OB_FAIL(hex_print(ptr_, plen, buf, buf_len, pos))) { - // no logging in to_string function. - } else { - // maybe ObIntTC - if (sizeof(int64_t) == len_) { - BUF_PRINTF(", int: %ld", *int_); - // maybe number with one digit - if (1 == num_->desc_.len_) { - BUF_PRINTF(", num_digit0: %u", num_->digits_[0]); - } - } - // maybe printable C string - int64_t idx = 0; - while (idx < plen && isprint(ptr_[idx])) { - idx++; - } - if (idx >= plen) { - BUF_PRINTF(", cstr: %.*s", static_cast(plen), ptr_); - } - } - } - J_OBJ_END(); - } - } - - return pos; -} - -OB_INLINE const char *ObStorageDatum::to_cstring(const bool for_dump) const -{ - char *buffer = NULL; - int64_t str_len = 0; - CStringBufMgr &mgr = CStringBufMgr::get_thread_local_instance(); - mgr.inc_level(); - const int64_t buf_len = mgr.acquire(buffer); - if (OB_ISNULL(buffer)) { - LIB_LOG_RET(ERROR, OB_ALLOCATE_MEMORY_FAILED, "buffer is NULL"); - } else { - str_len = storage_to_string(buffer, buf_len -1, for_dump); - if (str_len >= 0 && str_len < buf_len) { - buffer[str_len] = '\0'; - } else { - buffer[0] = '\0'; - } - mgr.update_position(str_len + 1); - } - mgr.try_clear_list(); - mgr.dec_level(); - return buffer; -} - -OB_INLINE bool ObStorageDatum::need_copy_for_encoding_column_with_flat_format(const ObObjDatumMapType map_type) const -{ - return OBJ_DATUM_STRING == map_type && sizeof(uint64_t) == len_ && is_local_buf(); -} - struct ObGhostRowUtil { public: ObGhostRowUtil() = delete; diff --git a/src/storage/blocksstable/ob_datum_row_iterator.h b/src/storage/blocksstable/ob_datum_row_iterator.h new file mode 100644 index 000000000..95c2ad351 --- /dev/null +++ b/src/storage/blocksstable/ob_datum_row_iterator.h @@ -0,0 +1,98 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_DATUM_ROW_ITERATOR_ +#define OCEANBASE_DATUM_ROW_ITERATOR_ + +#include "src/storage/blocksstable/ob_datum_row.h" + +namespace oceanbase +{ +namespace blocksstable +{ + +class ObDatumRowIterator +{ +public: + typedef common::ObReserveArenaAllocator<1024> ObStorageReserveAllocator; +public: + ObDatumRowIterator() {} + virtual ~ObDatumRowIterator() {} + /** + * get the next datum row and move the cursor + * + * @param row [out] + * + * @return OB_ITER_END if end of iteration + */ + virtual int get_next_row(ObDatumRow *&row) = 0; + virtual int get_next_rows(ObDatumRow *&rows, int64_t &row_count) + { + int ret = OB_SUCCESS; + if (OB_FAIL(get_next_row(rows))) { + } else { + row_count = 1; + } + return ret; + } + virtual void reset() {} + TO_STRING_EMPTY(); +}; + +/// wrap one datum row as an iterator +class ObSingleDatumRowIteratorWrapper: public ObDatumRowIterator +{ +public: + ObSingleDatumRowIteratorWrapper(); + ObSingleDatumRowIteratorWrapper(ObDatumRow *row); + virtual ~ObSingleDatumRowIteratorWrapper() {} + + void set_row(ObDatumRow *row) { row_ = row; } + virtual int get_next_row(ObDatumRow *&row); + virtual void reset() { iter_end_ = false; } +private: + // disallow copy + DISALLOW_COPY_AND_ASSIGN(ObSingleDatumRowIteratorWrapper); +private: + // data members + ObDatumRow *row_; + bool iter_end_; +}; + +inline ObSingleDatumRowIteratorWrapper::ObSingleDatumRowIteratorWrapper() + :row_(NULL), + iter_end_(false) +{} + +inline ObSingleDatumRowIteratorWrapper::ObSingleDatumRowIteratorWrapper(ObDatumRow *row) + :row_(row), + iter_end_(false) +{} + +inline int ObSingleDatumRowIteratorWrapper::get_next_row(ObDatumRow *&row) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(row_)) { + ret = OB_NOT_INIT; + } else if (iter_end_) { + ret = OB_ITER_END; + } else { + row = row_; + iter_end_ = true; + } + return ret; +} + +} +} + +#endif //OCEANBASE_DATUM_ROW_ITERATOR_ diff --git a/src/storage/blocksstable/ob_datum_row_store.cpp b/src/storage/blocksstable/ob_datum_row_store.cpp new file mode 100644 index 000000000..c0d2aa45b --- /dev/null +++ b/src/storage/blocksstable/ob_datum_row_store.cpp @@ -0,0 +1,242 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX COMMON +#include "ob_datum_row_store.h" +#include "lib/utility/utility.h" +namespace oceanbase +{ +namespace blocksstable +{ + +//////////////////////////////////////////////////////////////// +struct ObDatumRowStore::BlockInfo +{ + explicit BlockInfo(int64_t block_size) + :magic_(0xabcd4444abcd4444), + next_(NULL), + curr_data_pos_(0), + block_size_(block_size) + { + } + OB_INLINE int64_t get_remain_size() const { return block_size_ - curr_data_pos_; } + OB_INLINE int64_t get_remain_size_for_read(int64_t pos) const { return curr_data_pos_ - pos; } + OB_INLINE char *get_buffer() { return data_ + curr_data_pos_; } + OB_INLINE const char *get_buffer_head() const { return data_; } + OB_INLINE void advance(const int64_t length) { curr_data_pos_ += length; } + OB_INLINE BlockInfo *get_next_block() { return next_; } + OB_INLINE const BlockInfo *get_next_block() const { return next_; } + OB_INLINE int64_t get_block_size() const { return block_size_; }; + + int append_row(const ObDatumRow &row, const int64_t length); + friend class ObDatumRowStore::BlockList; +private: +#ifdef __clang__ + int64_t magic_ [[gnu::unused]]; +#else + int64_t magic_; +#endif + BlockInfo *next_; + /** + * cur_data_pos_ must be set when BlockInfo deserialized + */ + int64_t curr_data_pos_; + int64_t block_size_; + char data_[0]; +}; + +int ObDatumRowStore::BlockInfo::append_row(const ObDatumRow &row, const int64_t length) +{ + int ret = OB_SUCCESS; + int64_t pos = 0; + if (OB_UNLIKELY(0 > curr_data_pos_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "out of memory range", + K(ret), K_(block_size), K_(curr_data_pos), K(length)); + } else if (OB_FAIL(row.serialize(get_buffer(), get_remain_size(), pos))) { + STORAGE_LOG(WARN, "fail to serialize datum row", K(ret), K(row)); + } else { + advance(pos); + } + return ret; +} + +//////////////////////////////////////////////////////////////// +ObDatumRowStore::BlockList::BlockList() + :first_(NULL), + last_(NULL), + count_(0), + used_mem_size_(0) +{ +}; + +void ObDatumRowStore::BlockList::reset() +{ + first_ = NULL; + last_ = NULL; + count_ = 0; + used_mem_size_ = 0; +} + +int ObDatumRowStore::BlockList::add_last(BlockInfo *block) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(block)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", K(ret), K(block)); + } else { + block->next_ = NULL; + if (OB_ISNULL(last_)) { + if (OB_ISNULL(first_)) { + first_ = block; + last_ = block; + } else { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "invalid block list", K(ret), K_(last), K_(first)); + } + } else { + last_->next_ = block; + last_ = block; + } + } + if (OB_SUCC(ret)) { + used_mem_size_ += block->get_block_size(); + ++count_; + } + return ret; +} + +//////////////////////////////////////////////////////////////// +ObDatumRowStore::Iterator::Iterator(const ObDatumRowStore &row_store) + : row_store_(row_store), + cur_iter_block_(row_store_.blocks_.get_first()), + cur_iter_pos_(0) +{ +} + +int ObDatumRowStore::Iterator::get_next_row(ObDatumRow &row) +{ + int ret = OB_SUCCESS; + int64_t pos = 0; + if (OB_UNLIKELY(row.count_ < row_store_.get_col_count())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "column buffer count is not enough", K(ret), K_(row.count), K(row_store_.get_col_count())); + } else if (OB_ISNULL(cur_iter_block_)) { + // the last block + ret = OB_ITER_END; + } else if (OB_FAIL(row.deserialize(cur_iter_block_->get_buffer_head() + cur_iter_pos_, cur_iter_block_->get_remain_size_for_read(cur_iter_pos_), pos))) { + STORAGE_LOG(WARN, "failed to deserialize datum row", K(ret), K(cur_iter_block_->get_block_size()), K(cur_iter_pos_), + K(row.get_serialize_size()), K(pos), K(row)); + } else { + // next + cur_iter_pos_ += pos; + // update current block when current block reach its end + if (cur_iter_block_->get_remain_size_for_read(cur_iter_pos_) <= 0) { + // next block + cur_iter_block_ = cur_iter_block_->get_next_block(); + cur_iter_pos_ = 0; + } + } // end else + return ret; +} + +//////////////////////////////////////////////////////////////// +ObDatumRowStore::ObDatumRowStore() +: inner_alloc_("DatumRowStore", MTL_ID()), + blocks_(), + row_count_(0), + col_count_(0) +{ +} + + +ObDatumRowStore::~ObDatumRowStore() +{ + clear_rows(); +} + +// method for ObAggregateFunction::prepare() +// prepare need to reuse ObDatumRowStore for WRITE, +// it needs to reuse reserved_columns_ which should not be cleared +void ObDatumRowStore::clear_rows() +{ + row_count_ = 0; + col_count_ = 0; + + // free all blocks + BlockInfo *block = blocks_.get_first(); + while (NULL != block) { + BlockInfo *next = block->get_next_block(); + block->~BlockInfo(); + inner_alloc_.free(block); + block = next; + } + blocks_.reset(); +} + +int ObDatumRowStore::new_block(int64_t block_size, ObDatumRowStore::BlockInfo *&block) +{ + int ret = OB_SUCCESS; + // normalize block size + if (block_size > BIG_BLOCK_SIZE) { + block_size = OB_MAX_ROW_LENGTH_IN_MEMTABLE; + } else if (block_size > NORMAL_BLOCK_SIZE) { + block_size = BIG_BLOCK_SIZE; + } else { + block_size = NORMAL_BLOCK_SIZE; + } + // make sure all memory allocated under the right tenant + block = static_cast(inner_alloc_.alloc(block_size)); + if (OB_ISNULL(block)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + STORAGE_LOG(WARN, "failed to alloc block memory", K(ret), K(block_size + sizeof(BlockInfo))); + } else { + block = new(block) BlockInfo(block_size - sizeof(BlockInfo)); + if (OB_FAIL(blocks_.add_last(block))) { + STORAGE_LOG(WARN, "failed to add a new block to block list", K(ret)); + } + } + return ret; +} + +int ObDatumRowStore::add_row(const ObDatumRow &row) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(0 < col_count_) && OB_UNLIKELY(row.count_ != col_count_)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "all rows should have the same columns", K(col_count_), K(row.count_)); + } else { + int64_t length = row.get_serialize_size(); + BlockInfo *block = blocks_.get_last(); + if (OB_ISNULL(block) || block->get_remain_size() < length) { + if (OB_FAIL(new_block(length, block))) { + STORAGE_LOG(WARN, "failed to new block", K(ret), K(length)); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(block->append_row(row, length))) { + STORAGE_LOG(WARN, "failed to append row", K(ret), K(row)); + } else { + ++row_count_; + } + } + } + return ret; +} + +ObDatumRowStore::Iterator ObDatumRowStore::begin() const +{ + return Iterator(*this); +} + +} //end common +} //end oceanbase diff --git a/src/storage/blocksstable/ob_datum_row_store.h b/src/storage/blocksstable/ob_datum_row_store.h new file mode 100644 index 000000000..4353301f5 --- /dev/null +++ b/src/storage/blocksstable/ob_datum_row_store.h @@ -0,0 +1,92 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_COMMON_OB_DATUM_ROW_STORE_H +#define OCEANBASE_COMMON_OB_DATUM_ROW_STORE_H +#include +#include +#include "common/row/ob_row.h" +#include "common/row/ob_row_iterator.h" +#include "lib/container/ob_fixed_array.h" +#include "lib/string/ob_string.h" +#include "ob_datum_row.h" +#include "storage/ob_i_store.h" + +namespace oceanbase +{ +namespace blocksstable +{ +class ObDatumRowStore +{ +public: + struct BlockInfo; + class Iterator + { + public: + friend class ObDatumRowStore; + int get_next_row(ObDatumRow &row); + private: + explicit Iterator(const ObDatumRowStore &row_store); + protected: + const ObDatumRowStore &row_store_; + const BlockInfo *cur_iter_block_; + int64_t cur_iter_pos_; + }; +public: + ObDatumRowStore(); + ~ObDatumRowStore(); + void clear_rows(); + int add_row(const ObDatumRow &row); + inline int64_t get_row_count() const { return row_count_; } + inline int64_t get_col_count() const { return col_count_; } + Iterator begin() const; + TO_STRING_KV(N_BLOCK_NUM, blocks_.get_block_count(), + N_ROW_COUNT, row_count_, + N_COLUMN_COUNT, col_count_); +private: + DISALLOW_COPY_AND_ASSIGN(ObDatumRowStore); + + static const int64_t BIG_BLOCK_SIZE = OB_MALLOC_BIG_BLOCK_SIZE; + static const int64_t NORMAL_BLOCK_SIZE = OB_MALLOC_NORMAL_BLOCK_SIZE; + + // non-circular doubly linked list + class BlockList + { + public: + BlockList(); + void reset(); + int add_last(BlockInfo *block); + BlockInfo *get_first() { return first_; } + BlockInfo *get_last() { return last_; } + const BlockInfo *get_first() const { return first_; } + const BlockInfo *get_last() const { return last_; } + int64_t get_block_count() const { return count_; } + int64_t get_used_mem_size() const { return used_mem_size_; } + private: + BlockInfo *first_; + BlockInfo *last_; + int64_t count_; + int64_t used_mem_size_; // bytes of all blocks + }; +private: + int new_block(int64_t block_size, BlockInfo *&block); +private: + DefaultPageAllocator inner_alloc_; + BlockList blocks_; + int64_t row_count_; + int64_t col_count_; +}; + +} // end namespace common +} // end namespace oceanbase + +#endif /* OCEANBASE_COMMON_OB_DATUM_ROW_STORE_H */ diff --git a/src/storage/blocksstable/ob_datum_row_utils.cpp b/src/storage/blocksstable/ob_datum_row_utils.cpp new file mode 100644 index 000000000..6d39756a9 --- /dev/null +++ b/src/storage/blocksstable/ob_datum_row_utils.cpp @@ -0,0 +1,97 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE +#include "ob_datum_row_utils.h" +namespace oceanbase +{ +namespace blocksstable +{ + +int ObDatumRowUtils::ob_create_row(ObIAllocator &allocator, int64_t col_count, ObDatumRow *&datum_row) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(col_count <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("row_count is invalid", K(ret), K(col_count)); + } else { + void *row_buf = NULL; + if (OB_ISNULL(row_buf = allocator.alloc(sizeof(blocksstable::ObDatumRow)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate row buffer failed", K(ret), K(sizeof(blocksstable::ObDatumRow))); + } else if (FALSE_IT(datum_row = new(row_buf) blocksstable::ObDatumRow())) { + } else if (OB_FAIL(datum_row->init(allocator, col_count))) { + LOG_WARN("fail to init datum row", K(ret), K(col_count)); + } + if (OB_FAIL(ret) && nullptr != datum_row) { + datum_row->~ObDatumRow(); + datum_row = nullptr; + allocator.free(row_buf); + } + } + return ret; +} + +int ObDatumRowUtils::ob_create_rows(ObIAllocator &allocator, int64_t row_count, int64_t col_count, ObDatumRow *&datum_rows) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(col_count <= 0 || row_count <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("col count or row count is invalid", K(ret), K(col_count), K(row_count)); + } else { + void *rows_buf = nullptr; + const size_t rows_buf_len = sizeof(blocksstable::ObDatumRow) * row_count; + if (OB_ISNULL(rows_buf = allocator.alloc(rows_buf_len))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("Failed to allocate row buffer", K(ret), K(rows_buf_len)); + } else { + char *row_buf = static_cast(rows_buf); + datum_rows = new(row_buf) blocksstable::ObDatumRow[row_count](); + int64_t i = 0; + for (; OB_SUCC(ret) && i < row_count; ++i) { + if (OB_FAIL(datum_rows[i].init(allocator, col_count))) { + LOG_WARN("fail to init datum row", K(ret), K(col_count), K(datum_rows[i])); + } + } + if (OB_FAIL(ret)) { + // release storage_datums + for (int64_t j = i; j >= 0; --j) { + datum_rows[j].~ObDatumRow(); + } + allocator.free(rows_buf); + } + } + } + return ret; +} + +int ObDatumRowUtils::prepare_rowkey( + const ObDatumRow &datum_row, + const int key_datum_cnt, + const ObColDescIArray &col_descs, + common::ObIAllocator &allocator, + ObDatumRowkey &rowkey) +{ + int ret = OB_SUCCESS; + if (!datum_row.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get invalid datum row", K(ret), K(datum_row)); + } else if (OB_FAIL(rowkey.assign(datum_row.storage_datums_, key_datum_cnt))) { + LOG_WARN("failed to assign datum rowkey", K(ret), K(datum_row), K(key_datum_cnt)); + } else if (OB_FAIL(rowkey.prepare_memtable_readable(col_descs, allocator))) { + LOG_WARN("failed to prepare store rowkey to read memtable", K(ret), K(datum_row), K(rowkey)); + } + return ret; +} + +} // namespace sql +} // namespace oceanbase diff --git a/src/storage/blocksstable/ob_datum_row_utils.h b/src/storage/blocksstable/ob_datum_row_utils.h new file mode 100644 index 000000000..e746602eb --- /dev/null +++ b/src/storage/blocksstable/ob_datum_row_utils.h @@ -0,0 +1,38 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef DEV_SRC_STORAGE_BLOCKSSTABLE_OB_DATUM_ROW_UTILS_H_ +#define DEV_SRC_STORAGE_BLOCKSSTABLE_OB_DATUM_ROW_UTILS_H_ +#include "share/ob_define.h" +#include "ob_datum_row.h" +namespace oceanbase +{ +namespace blocksstable +{ +class ObDatumRowUtils +{ + typedef common::ObIArray ObColDescIArray; +public: + static int ob_create_row(ObIAllocator &allocator, int64_t col_count, ObDatumRow *&datum_row); + static int ob_create_rows(ObIAllocator &allocator, int64_t row_count, int64_t col_count, ObDatumRow *&datum_rows); + // TODO@xuanxi: rewrite it when store rowkey is no longer needed + static int prepare_rowkey( + const ObDatumRow &datum_row, + const int key_datum_cnt, + const ObColDescIArray &col_descs, + common::ObIAllocator &allocator, + ObDatumRowkey &rowkey); +}; + +} // namespace sql +} // namespace oceanbase +#endif /* DEV_SRC_STORAGE_BLOCKSSTABLE_OB_DATUM_ROW_UTILS_H_ */ diff --git a/src/storage/blocksstable/ob_datum_rowkey.cpp b/src/storage/blocksstable/ob_datum_rowkey.cpp index 2b8cfbf60..b66150fbb 100644 --- a/src/storage/blocksstable/ob_datum_rowkey.cpp +++ b/src/storage/blocksstable/ob_datum_rowkey.cpp @@ -616,6 +616,25 @@ int ObDatumRowkeyHelper::convert_store_rowkey(const ObDatumRowkey &datum_rowkey, return ret; } +int ObDatumRowkeyHelper::prepare_datum_rowkey(const ObDatumRow &datum_row, + const int key_datum_cnt, + const ObIArray &col_descs, + ObDatumRowkey &datum_rowkey) +{ + int ret = OB_SUCCESS; + + if (!datum_row.is_valid() || col_descs.count() < datum_row.get_column_count()) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Get invalid datum row", K(ret), K(datum_row), K(col_descs)); + } else if (OB_FAIL(datum_rowkey.assign(datum_row.storage_datums_, key_datum_cnt))) { + STORAGE_LOG(WARN, "Failed to assign datum rowkey", K(ret), K(datum_row), K(key_datum_cnt)); + } else if (OB_FAIL(convert_store_rowkey(datum_rowkey, col_descs, datum_rowkey.store_rowkey_))) { + STORAGE_LOG(WARN, "Failed to convert store rowkeyy", K(ret), K(datum_rowkey)); + } + + return ret; +} + int ObDatumRowkeyHelper::reserve(const int64_t rowkey_cnt) { diff --git a/src/storage/blocksstable/ob_datum_rowkey.h b/src/storage/blocksstable/ob_datum_rowkey.h index b6108c5d1..3164fefdc 100644 --- a/src/storage/blocksstable/ob_datum_rowkey.h +++ b/src/storage/blocksstable/ob_datum_rowkey.h @@ -13,7 +13,7 @@ #ifndef OB_STORAGE_BLOCKSSTABLE_DATUM_ROWKEY_H #define OB_STORAGE_BLOCKSSTABLE_DATUM_ROWKEY_H -#include "ob_datum_row.h" +#include "ob_storage_datum.h" #include "lib/utility/ob_print_kv.h" //to be removed #include "common/rowkey/ob_store_rowkey.h" @@ -27,6 +27,7 @@ struct ObDatumRange; class ObRowkeyVector; struct ObDiscreteDatumRowkey; struct ObCommonDatumRowkey; +struct ObDatumRow; struct ObDatumRowkey { @@ -123,6 +124,10 @@ public: int convert_store_rowkey(const ObDatumRowkey &datum_rowkey, const common::ObIArray &col_descs, common::ObStoreRowkey &rowkey); + int prepare_datum_rowkey(const ObDatumRow &datum_row, + const int key_datum_cnt, + const ObIArray &col_descs, + ObDatumRowkey &datum_rowkey); int reserve(const int64_t rowkey_cnt); OB_INLINE ObStorageDatum *get_datums() { return datum_buffer_.get_datums(); } OB_INLINE int64_t get_capacity() const { return datum_buffer_.get_capacity(); } diff --git a/src/storage/blocksstable/ob_row_writer.cpp b/src/storage/blocksstable/ob_row_writer.cpp index a46a81b4f..e8d96330e 100644 --- a/src/storage/blocksstable/ob_row_writer.cpp +++ b/src/storage/blocksstable/ob_row_writer.cpp @@ -76,17 +76,16 @@ int ObRowWriter::init_common(char *buf, const int64_t buf_size, const int64_t po } int ObRowWriter::check_row_valid( - const ObStoreRow &row, + const ObDatumRow &row, const int64_t rowkey_column_count) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!row.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid row writer input argument", K(row), K(ret)); - } else if (OB_UNLIKELY(rowkey_column_count <= 0 || rowkey_column_count > row.row_val_.count_)) { + LOG_WARN("invalid row writer input argument", K(ret), K(row)); + } else if (OB_UNLIKELY(rowkey_column_count <= 0 || rowkey_column_count > row.count_)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid row writer input argument", - K(rowkey_column_count), K(row.row_val_.count_), K(ret)); + LOG_WARN("invalid row writer input argument", K(ret), K(rowkey_column_count), K(row.count_)); } return ret; } @@ -230,24 +229,24 @@ int ObRowWriter::write(const int64_t rowkey_column_cnt, const ObDatumRow &datum_ return ret; } -// when update_idx == nullptr, write full row; else only write rowkey + update cells +// when update_idx == nullptr, write full row; else only write rowkey + update storage_datums int ObRowWriter::write( const int64_t rowkey_column_count, - const storage::ObStoreRow &row, + const ObDatumRow &datum_row, const ObIArray *update_idx, char *&buf, int64_t &len) { int ret = OB_SUCCESS; len = 0; - if (OB_UNLIKELY(nullptr != update_idx && update_idx->count() > row.row_val_.count_)) { + if (OB_UNLIKELY(nullptr != update_idx && update_idx->count() > datum_row.count_)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("update idx is invalid", K(ret), KPC(update_idx), K_(row.row_val_.count), K(rowkey_column_count)); + LOG_WARN("update idx is invalid", K(ret), KPC(update_idx), K_(datum_row.count), K(rowkey_column_count)); } else { do { if (OB_FAIL(alloc_buf_and_init(OB_BUF_NOT_ENOUGH == ret))) { LOG_WARN("row writer fail to alloc and init", K(ret)); - } else if (OB_FAIL(inner_write_row(rowkey_column_count, row, update_idx))) { + } else if (OB_FAIL(inner_write_row(rowkey_column_count, datum_row, update_idx))) { if (OB_BUF_NOT_ENOUGH != ret) { LOG_WARN("row writer fail to append row header", K(ret), K(row_buffer_), K(pos_)); } @@ -289,29 +288,29 @@ int ObRowWriter::check_update_idx_array_valid( int ObRowWriter::inner_write_row( const int64_t rowkey_column_count, - const ObStoreRow &row, + const ObDatumRow &datum_row, const ObIArray *update_idx) { int ret = OB_SUCCESS; - if (OB_FAIL(check_row_valid(row, rowkey_column_count))) { + if (OB_FAIL(check_row_valid(datum_row, rowkey_column_count))) { LOG_WARN("row writer fail to init store row", K(ret), K(rowkey_column_count)); } else if (nullptr != update_idx && OB_FAIL(check_update_idx_array_valid(rowkey_column_count, update_idx))) { LOG_WARN("invalid update idx array", K(ret)); } else if (OB_FAIL(append_row_header( - row.flag_.get_serialize_flag(), - row.row_type_flag_.flag_, - row.trans_id_.get_id(), - row.row_val_.count_, + datum_row.row_flag_.get_serialize_flag(), + datum_row.mvcc_row_flag_.flag_, + datum_row.trans_id_.get_id(), + datum_row.count_, rowkey_column_count))) { if (OB_BUF_NOT_ENOUGH != ret) { - LOG_WARN("row writer fail to append row header", K(ret), K(row)); + LOG_WARN("row writer fail to append row header", K(ret), K(datum_row)); } } else { update_idx_array_ = update_idx; rowkey_column_cnt_ = rowkey_column_count; - if (OB_FAIL(inner_write_cells(row.row_val_.cells_, row.row_val_.count_))) { + if (OB_FAIL(inner_write_cells(datum_row.storage_datums_, datum_row.count_))) { if (OB_BUF_NOT_ENOUGH != ret) { - LOG_WARN("failed to write cells", K(ret), K(row)); + LOG_WARN("failed to write cells", K(ret), K(datum_row)); } } } diff --git a/src/storage/blocksstable/ob_row_writer.h b/src/storage/blocksstable/ob_row_writer.h index b3cc98192..773e3f4cb 100644 --- a/src/storage/blocksstable/ob_row_writer.h +++ b/src/storage/blocksstable/ob_row_writer.h @@ -63,7 +63,7 @@ public: int write_rowkey(const common::ObStoreRowkey &rowkey, char *&buf, int64_t &len); int write( const int64_t rowkey_cnt, - const storage::ObStoreRow &row, + const ObDatumRow &datum_row, const ObIArray *update_idx, char *&buf, int64_t &len); @@ -102,7 +102,7 @@ private: }; int inner_write_row( const int64_t rowkey_column_count, - const storage::ObStoreRow &row, + const ObDatumRow &row, const ObIArray *update_idx); OB_INLINE int write_oracle_timestamp(const common::ObOTimestampData &ot_data, const common::ObOTimestampMetaAttrType otmat); int append_column(const common::ObObj &obj); @@ -110,7 +110,7 @@ private: int append_8_bytes_column(const ObStorageDatum &datum); int init_common(char *buf, const int64_t buf_size, const int64_t pos); int check_row_valid( - const storage::ObStoreRow &row, + const ObDatumRow &row, const int64_t rowkey_column_count); int append_row_header( const uint8_t row_flag, diff --git a/src/storage/blocksstable/ob_storage_datum.cpp b/src/storage/blocksstable/ob_storage_datum.cpp new file mode 100644 index 000000000..52e3c513d --- /dev/null +++ b/src/storage/blocksstable/ob_storage_datum.cpp @@ -0,0 +1,386 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE +#include "ob_storage_datum.h" +#include "share/schema/ob_table_param.h" +#include "share/ob_force_print_log.h" +#include "storage/ob_i_store.h" +#include "share/scheduler/ob_tenant_dag_scheduler.h" + +namespace oceanbase +{ +using namespace common; +namespace blocksstable +{ +static int nonext_nonext_compare(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret) +{ + int ret = cmp_func.cmp_func_(left, right, cmp_ret); + STORAGE_LOG(DEBUG, "chaser debug compare datum", K(ret), K(left), K(right), K(cmp_ret)); + return ret; +} + +static int nonext_ext_compare(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret) +{ + int ret = OB_SUCCESS; + UNUSEDx(left, cmp_func); + if (right.is_max()) { + cmp_ret = -1; + } else if (right.is_min()) { + cmp_ret = 1; + } else { + ret = OB_ERR_SYS; + STORAGE_LOG(ERROR, "Unexpected datum in rowkey to compare", K(ret), K(right)); + } + STORAGE_LOG(DEBUG, "chaser debug compare datum", K(ret), K(left), K(right), K(cmp_ret)); + return ret; +} + +static int ext_nonext_compare(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret) +{ + int ret = OB_SUCCESS; + UNUSEDx(right, cmp_func); + if (left.is_max()) { + cmp_ret = 1; + } else if (left.is_min()) { + cmp_ret = -1; + } else { + ret = OB_ERR_SYS; + STORAGE_LOG(ERROR, "Unexpected datum in rowkey to compare", K(ret), K(left)); + } + STORAGE_LOG(DEBUG, "chaser debug compare datum", K(ret), K(left), K(right), K(cmp_ret)); + return ret; +} + +static int ext_ext_compare(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret) +{ + int ret = OB_SUCCESS; + UNUSEDx(cmp_func); + int64_t lv = left.is_max() - left.is_min(); + int64_t rv = right.is_max() - right.is_min(); + if (OB_UNLIKELY(0 == lv || 0 == rv)) { + ret = OB_ERR_SYS; + STORAGE_LOG(ERROR, "Unexpected datum in rowkey to compare", K(ret), K(left), K(right)); + } else { + cmp_ret = lv - rv; + } + STORAGE_LOG(DEBUG, "chaser debug compare datum", K(ret), K(left), K(right), K(cmp_ret)); + + return ret; +} + +typedef int (*ExtSafeCompareFunc)(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret); +static ExtSafeCompareFunc ext_safe_cmp_funcs[2][2] = { + {nonext_nonext_compare, nonext_ext_compare}, + {ext_nonext_compare, ext_ext_compare} +}; + +int ObStorageDatumCmpFunc::compare(const ObStorageDatum &left, const ObStorageDatum &right, int &cmp_ret) const +{ + return ext_safe_cmp_funcs[left.is_ext()][right.is_ext()](left, right, cmp_func_, cmp_ret); +} + +/* + *ObStorageDatumUtils + */ +ObStorageDatumUtils::ObStorageDatumUtils() + : rowkey_cnt_(0), + cmp_funcs_(), + hash_funcs_(), + ext_hash_func_(), + is_oracle_mode_(false), + is_inited_(false) +{} + +ObStorageDatumUtils::~ObStorageDatumUtils() +{} + +int ObStorageDatumUtils::transform_multi_version_col_desc(const ObIArray &col_descs, + const int64_t schema_rowkey_cnt, + ObIArray &mv_col_descs) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(schema_rowkey_cnt > col_descs.count())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Invalid argument to transform mv col descs", K(ret), K(schema_rowkey_cnt), K(col_descs)); + } else { + mv_col_descs.reuse(); + for (int64_t i = 0; OB_SUCC(ret) && i < schema_rowkey_cnt; i++) { + if (OB_FAIL(mv_col_descs.push_back(col_descs.at(i)))) { + STORAGE_LOG(WARN, "Failed to push back col desc", K(ret), K(i)); + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(storage::ObMultiVersionRowkeyHelpper::add_extra_rowkey_cols(mv_col_descs))) { + STORAGE_LOG(WARN, "Fail to add extra_rowkey_cols", K(ret), K(schema_rowkey_cnt)); + } else { + for (int64_t i = schema_rowkey_cnt; OB_SUCC(ret) && i < col_descs.count(); i++) { + const share::schema::ObColDesc &col_desc = col_descs.at(i); + if (col_desc.col_id_ == common::OB_HIDDEN_TRANS_VERSION_COLUMN_ID + || col_desc.col_id_ == common::OB_HIDDEN_SQL_SEQUENCE_COLUMN_ID) { + continue; + } else if (OB_FAIL(mv_col_descs.push_back(col_desc))) { + STORAGE_LOG(WARN, "Failed to push back col desc", K(ret), K(col_desc)); + } + } + } + } + + return ret; +} + +int ObStorageDatumUtils::init(const ObIArray &col_descs, + const int64_t schema_rowkey_cnt, + const bool is_oracle_mode, + ObIAllocator &allocator, + const bool is_column_store) +{ + int ret = OB_SUCCESS; + ObSEArray mv_col_descs; + int64_t mv_rowkey_cnt = 0; + int64_t mv_extra_rowkey_cnt = 0; + if (IS_INIT) { + ret = OB_INIT_TWICE; + STORAGE_LOG(WARN, "ObStorageDatumUtils init twice", K(ret), K(*this)); + } else if (OB_UNLIKELY(schema_rowkey_cnt < 0 || schema_rowkey_cnt > OB_MAX_ROWKEY_COLUMN_NUMBER + || schema_rowkey_cnt > col_descs.count())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Invalid argument to init storage datum utils", K(ret), K(schema_rowkey_cnt), K(col_descs)); + } else if (OB_FAIL(transform_multi_version_col_desc(col_descs, schema_rowkey_cnt, mv_col_descs))) { + STORAGE_LOG(WARN, "Failed to transform multi version col descs", K(ret)); + } else if (FALSE_IT(mv_extra_rowkey_cnt = is_column_store ? 0 : storage::ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt())) { + } else if (FALSE_IT(mv_rowkey_cnt = schema_rowkey_cnt + mv_extra_rowkey_cnt)) { + } else if (OB_FAIL(cmp_funcs_.init(mv_rowkey_cnt, allocator))) { + STORAGE_LOG(WARN, "Failed to reserve cmp func array", K(ret)); + } else if (OB_FAIL(hash_funcs_.init(mv_rowkey_cnt, allocator))) { + STORAGE_LOG(WARN, "Failed to reserve hash func array", K(ret)); + } else if (OB_FAIL(inner_init(mv_col_descs, mv_rowkey_cnt, is_oracle_mode))) { + STORAGE_LOG(WARN, "Failed to inner init datum utils", K(ret), K(mv_col_descs), K(mv_rowkey_cnt)); + } + + return ret; +} + +int ObStorageDatumUtils::init(const common::ObIArray &col_descs, + const int64_t schema_rowkey_cnt, + const bool is_oracle_mode, + const int64_t arr_buf_len, + char *arr_buf) +{ + int ret = OB_SUCCESS; + ObSEArray mv_col_descs; + int64_t pos = 0; + int64_t mv_rowkey_cnt = 0; + + if (IS_INIT) { + ret = OB_INIT_TWICE; + STORAGE_LOG(WARN, "ObStorageDatumUtils init twice", K(ret), K(*this)); + } else if (OB_UNLIKELY(schema_rowkey_cnt < 0 || schema_rowkey_cnt > OB_MAX_ROWKEY_COLUMN_NUMBER + || schema_rowkey_cnt > col_descs.count())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Invalid argument to init storage datum utils", K(ret), K(col_descs), K(schema_rowkey_cnt)); + } else if (OB_FAIL(transform_multi_version_col_desc(col_descs, schema_rowkey_cnt, mv_col_descs))) { + STORAGE_LOG(WARN, "Failed to transform multi version col descs", K(ret)); + } else if (FALSE_IT(mv_rowkey_cnt = schema_rowkey_cnt + storage::ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt())) { + } else if (OB_FAIL(cmp_funcs_.init(mv_rowkey_cnt, arr_buf_len, arr_buf, pos))) { + STORAGE_LOG(WARN, "Failed to init compare function array", K(ret)); + } else if (OB_FAIL(hash_funcs_.init(mv_rowkey_cnt, arr_buf_len, arr_buf, pos))) { + STORAGE_LOG(WARN, "Failed to init hash function array", K(ret)); + } else if (OB_FAIL(inner_init(mv_col_descs, mv_rowkey_cnt, is_oracle_mode))) { + STORAGE_LOG(WARN, "Failed to inner init datum utils", K(ret), K(mv_col_descs), K(mv_rowkey_cnt)); + } + return ret; +} + +int ObStorageDatumUtils::inner_init( + const common::ObIArray &mv_col_descs, + const int64_t mv_rowkey_col_cnt, + const bool is_oracle_mode) +{ + int ret = OB_SUCCESS; + is_oracle_mode_ = is_oracle_mode; + // support column order index until next task done + // + // we could use the cmp funcs in the basic funcs directlly + bool is_null_last = is_oracle_mode_; + ObCmpFunc cmp_func; + ObHashFunc hash_func; + for (int64_t i = 0; OB_SUCC(ret) && i < mv_rowkey_col_cnt; i++) { + const share::schema::ObColDesc &col_desc = mv_col_descs.at(i); + //TODO @hanhui support desc rowkey + bool is_ascending = true || col_desc.col_order_ == ObOrderType::ASC; + bool has_lob_header = is_lob_storage(col_desc.col_type_.get_type()); + ObPrecision precision = PRECISION_UNKNOWN_YET; + if (col_desc.col_type_.is_decimal_int()) { + precision = col_desc.col_type_.get_stored_precision(); + OB_ASSERT(precision != PRECISION_UNKNOWN_YET); + } + sql::ObExprBasicFuncs *basic_funcs = ObDatumFuncs::get_basic_func(col_desc.col_type_.get_type(), + col_desc.col_type_.get_collation_type(), + col_desc.col_type_.get_scale(), + is_oracle_mode, + has_lob_header, + precision); + if (OB_UNLIKELY(nullptr == basic_funcs + || nullptr == basic_funcs->null_last_cmp_ + || nullptr == basic_funcs->murmur_hash_)) { + ret = OB_ERR_SYS; + STORAGE_LOG(ERROR, "Unexpected null basic funcs", K(ret), K(col_desc)); + } else { + cmp_func.cmp_func_ = is_null_last ? basic_funcs->null_last_cmp_ : basic_funcs->null_first_cmp_; + hash_func.hash_func_ = basic_funcs->murmur_hash_; + if (OB_FAIL(hash_funcs_.push_back(hash_func))) { + STORAGE_LOG(WARN, "Failed to push back hash func", K(ret), K(i), K(col_desc)); + } else if (is_ascending) { + if (OB_FAIL(cmp_funcs_.push_back(ObStorageDatumCmpFunc(cmp_func)))) { + STORAGE_LOG(WARN, "Failed to push back cmp func", K(ret), K(i), K(col_desc)); + } + } else { + ret = OB_ERR_SYS; + STORAGE_LOG(WARN, "Unsupported desc column order", K(ret), K(col_desc), K(i)); + } + } + } + if (OB_SUCC(ret)) { + sql::ObExprBasicFuncs *basic_funcs = ObDatumFuncs::get_basic_func(ObExtendType, CS_TYPE_BINARY); + if (OB_UNLIKELY(nullptr == basic_funcs || nullptr == basic_funcs->murmur_hash_)) { + ret = OB_ERR_SYS; + STORAGE_LOG(ERROR, "Unexpected null basic funcs for extend type", K(ret)); + } else { + ext_hash_func_.hash_func_ = basic_funcs->murmur_hash_; + rowkey_cnt_ = mv_rowkey_col_cnt; + is_inited_ = true; + } + } + + return ret; +} + +int ObStorageDatumUtils::assign(const ObStorageDatumUtils &other_utils, ObIAllocator &allocator) +{ + int ret = OB_SUCCESS; + + if (OB_UNLIKELY(!other_utils.is_valid())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Invalid argument to assign datum utils", K(ret), K(other_utils)); + } else { + rowkey_cnt_ = other_utils.get_rowkey_count(); + is_oracle_mode_ = other_utils.is_oracle_mode(); + ext_hash_func_ = other_utils.get_ext_hash_funcs(); + if (OB_FAIL(cmp_funcs_.init_and_assign(other_utils.get_cmp_funcs(), allocator))) { + STORAGE_LOG(WARN, "Failed to assign cmp func array", K(ret)); + } else if (OB_FAIL(hash_funcs_.init_and_assign(other_utils.get_hash_funcs(), allocator))) { + STORAGE_LOG(WARN, "Failed to assign hash func array", K(ret)); + } else { + is_inited_ = true; + } + } + + return ret; +} + +void ObStorageDatumUtils::reset() +{ + rowkey_cnt_ = 0; + cmp_funcs_.reset(); + hash_funcs_.reset(); + ext_hash_func_.hash_func_ = nullptr; + is_inited_ = false; +} + +int64_t ObStorageDatumUtils::get_deep_copy_size() const +{ + return cmp_funcs_.get_deep_copy_size() + hash_funcs_.get_deep_copy_size(); +} + +/* + *ObStorageDatumBuffer + */ +ObStorageDatumBuffer::ObStorageDatumBuffer(common::ObIAllocator *allocator) + : capacity_(LOCAL_BUFFER_ARRAY), + local_datums_(), + datums_(local_datums_), + allocator_(allocator), + is_inited_(nullptr != allocator) +{} + +ObStorageDatumBuffer::~ObStorageDatumBuffer() +{ + if (datums_ != local_datums_ && nullptr != allocator_) { + allocator_->free(datums_); + } +} + +void ObStorageDatumBuffer::reset() +{ + if (datums_ != local_datums_ && nullptr != allocator_) { + allocator_->free(datums_); + } + allocator_ = nullptr; + datums_ = local_datums_; + capacity_ = LOCAL_BUFFER_ARRAY; + for (int64_t i = 0; i < capacity_; i++) { + datums_[i].reuse(); + } + is_inited_ = false; +} + +int ObStorageDatumBuffer::init(common::ObIAllocator &allocator) +{ + int ret = OB_SUCCESS; + + if (IS_INIT) { + ret = OB_INIT_TWICE; + STORAGE_LOG(WARN, "ObStorageDatumBuffer init twice", K(ret), K(*this)); + } else { + OB_ASSERT(datums_ == local_datums_); + allocator_ = &allocator; + is_inited_ = true; + } + + return ret; +} + +int ObStorageDatumBuffer::reserve(const int64_t count, const bool keep_data) +{ + int ret = OB_SUCCESS; + void *buf = nullptr; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + STORAGE_LOG(WARN, "ObStorageDatumBuffer is not inited", K(ret), K(*this)); + } else if (OB_UNLIKELY(count <= 0)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Invalid argument to reserve datum buffer", K(ret), K(count)); + } else if (count <= capacity_){ + } else if (OB_ISNULL(buf = allocator_->alloc(sizeof(ObStorageDatum) * count))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + STORAGE_LOG(WARN, "Failed to alloc memory", K(ret), K(count)); + } else { + ObStorageDatum *new_datums = new (buf) ObStorageDatum [count]; + if (keep_data) { + for (int64_t i = 0; i < capacity_; i++) { + new_datums[i] = datums_[i]; + } + } + if (nullptr != datums_ && datums_ != local_datums_) { + allocator_->free(datums_); + } + datums_ = new_datums; + capacity_ = count; + } + + return ret; +} + +} // namespace blocksstable +} // namespace oceanbase diff --git a/src/storage/blocksstable/ob_storage_datum.h b/src/storage/blocksstable/ob_storage_datum.h new file mode 100644 index 000000000..8dd1477f4 --- /dev/null +++ b/src/storage/blocksstable/ob_storage_datum.h @@ -0,0 +1,427 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OB_STORAGE_BLOCKSSTABLE_STORAGE_DATUM_H +#define OB_STORAGE_BLOCKSSTABLE_STORAGE_DATUM_H + +#include "common/ob_common_types.h" +#include "common/ob_tablet_id.h" +#include "share/datum/ob_datum.h" +#include "share/datum/ob_datum_funcs.h" +#include "storage/meta_mem/ob_fixed_meta_obj_array.h" +#include "storage/tx/ob_trans_define.h" +#include "common/row/ob_row.h" +#include "storage/ob_storage_util.h" + +namespace oceanbase +{ +namespace share{ +namespace schema +{ +struct ObColDesc; +} +} +namespace storage +{ +struct ObStoreRow; +} +namespace blocksstable +{ + +//TODO optimize number buffer +struct ObStorageDatum : public common::ObDatum +{ + ObStorageDatum() { set_nop(); } + ObStorageDatum(const ObStorageDatum &datum) { reuse(); *this = datum; } + + ~ObStorageDatum() = default; + // ext value section + OB_INLINE void reuse() { ptr_ = buf_; reserved_ = 0; pack_ = 0; } + OB_INLINE void set_ext_value(const int64_t ext_value) + { reuse(); set_ext(); no_cv(extend_obj_)->set_ext(ext_value); } + OB_INLINE void set_nop() { set_ext_value(ObActionFlag::OP_NOP); } + OB_INLINE void set_min() { set_ext_value(common::ObObj::MIN_OBJECT_VALUE); } + OB_INLINE void set_max() { set_ext_value(common::ObObj::MAX_OBJECT_VALUE); } + OB_INLINE bool is_nop_value() const { return is_nop(); } // temp solution + // transfer section + OB_INLINE bool is_local_buf() const { return ptr_ == buf_; } + OB_INLINE int from_buf_enhance(const char *buf, const int64_t buf_len); + OB_INLINE int from_obj_enhance(const common::ObObj &obj); + OB_INLINE int to_obj_enhance(common::ObObj &obj, const common::ObObjMeta &meta) const; + OB_INLINE int deep_copy(const ObStorageDatum &src, common::ObIAllocator &allocator); + OB_INLINE int deep_copy(const ObStorageDatum &src, char * buf, const int64_t buf_len, int64_t &pos); + OB_INLINE void shallow_copy_from_datum(const ObDatum &src); + OB_INLINE int64_t get_deep_copy_size() const; + OB_INLINE ObStorageDatum& operator=(const ObStorageDatum &other); + OB_INLINE int64_t storage_to_string(char *buf, int64_t buf_len, const bool for_dump = false) const; + OB_INLINE bool need_copy_for_encoding_column_with_flat_format(const ObObjDatumMapType map_type) const; + OB_INLINE const char *to_cstring(const bool for_dump = false) const; + //only for unittest + OB_INLINE bool operator==(const ObStorageDatum &other) const; + OB_INLINE bool operator==(const ObObj &other) const; + + //datum 12 byte + int32_t reserved_; + // buf 16 byte + char buf_[common::OBJ_DATUM_NUMBER_RES_SIZE]; +}; + +struct ObStorageDatumCmpFunc +{ +public: + ObStorageDatumCmpFunc(common::ObCmpFunc &cmp_func) : cmp_func_(cmp_func) {} + ObStorageDatumCmpFunc() = default; + ~ObStorageDatumCmpFunc() = default; + int compare(const ObStorageDatum &left, const ObStorageDatum &right, int &cmp_ret) const; + OB_INLINE const common::ObCmpFunc &get_cmp_func() const { return cmp_func_; } + TO_STRING_KV(K_(cmp_func)); +private: + common::ObCmpFunc cmp_func_; +}; +typedef storage::ObFixedMetaObjArray ObStoreCmpFuncs; +typedef storage::ObFixedMetaObjArray ObStoreHashFuncs; +struct ObStorageDatumUtils +{ +public: + ObStorageDatumUtils(); + ~ObStorageDatumUtils(); + // init with array memory from allocator + int init(const common::ObIArray &col_descs, + const int64_t schema_rowkey_cnt, + const bool is_oracle_mode, + common::ObIAllocator &allocator, + const bool is_column_store = false); + // init with array memory on fixed size memory buffer + int init(const common::ObIArray &col_descs, + const int64_t schema_rowkey_cnt, + const bool is_oracle_mode, + const int64_t arr_buf_len, + char *arr_buf); + int assign(const ObStorageDatumUtils &other_utils, common::ObIAllocator &allocator); + void reset(); + OB_INLINE bool is_valid() const + { + return is_inited_ && cmp_funcs_.count() >= rowkey_cnt_ && hash_funcs_.count() >= rowkey_cnt_; + } + OB_INLINE bool is_oracle_mode() const { return is_oracle_mode_; } + OB_INLINE int64_t get_rowkey_count() const { return rowkey_cnt_; } + OB_INLINE const ObStoreCmpFuncs &get_cmp_funcs() const { return cmp_funcs_; } + OB_INLINE const ObStoreHashFuncs &get_hash_funcs() const { return hash_funcs_; } + OB_INLINE const common::ObHashFunc &get_ext_hash_funcs() const { return ext_hash_func_; } + int64_t get_deep_copy_size() const; + TO_STRING_KV(K_(is_oracle_mode), K_(rowkey_cnt), K_(is_inited), K_(is_oracle_mode)); +private: + //TODO to be removed by @hanhui + int transform_multi_version_col_desc(const common::ObIArray &col_descs, + const int64_t schema_rowkey_cnt, + common::ObIArray &mv_col_descs); + int inner_init( + const common::ObIArray &mv_col_descs, + const int64_t mv_rowkey_col_cnt, + const bool is_oracle_mode); +private: + int32_t rowkey_cnt_; // multi version rowkey + ObStoreCmpFuncs cmp_funcs_; // multi version rowkey cmp funcs + ObStoreHashFuncs hash_funcs_; // multi version rowkey cmp funcs + common::ObHashFunc ext_hash_func_; + bool is_oracle_mode_; + bool is_inited_; + DISALLOW_COPY_AND_ASSIGN(ObStorageDatumUtils); +}; + +struct ObStorageDatumBuffer +{ +public: + ObStorageDatumBuffer(common::ObIAllocator *allocator = nullptr); + ~ObStorageDatumBuffer(); + void reset(); + int init(common::ObIAllocator &allocator); + int reserve(const int64_t count, const bool keep_data = false); + OB_INLINE bool is_valid() const { return is_inited_; } + OB_INLINE ObStorageDatum *get_datums() { return datums_; } + OB_INLINE int64_t get_capacity() const { return capacity_; } + TO_STRING_KV(K_(capacity), KP_(datums), KP_(local_datums)); +private: + static const int64_t LOCAL_BUFFER_ARRAY = common::OB_ROW_DEFAULT_COLUMNS_COUNT >> 1; + int64_t capacity_; + ObStorageDatum local_datums_[LOCAL_BUFFER_ARRAY]; + ObStorageDatum *datums_; + common::ObIAllocator *allocator_; + bool is_inited_; +}; + + +OB_INLINE int ObStorageDatum::deep_copy(const ObStorageDatum &src, common::ObIAllocator &allocator) +{ + int ret = common::OB_SUCCESS; + + reuse(); + pack_ = src.pack_; + if (is_null()) { + } else if (src.len_ == 0) { + } else if (src.is_local_buf()) { + OB_ASSERT(src.len_ <= common::OBJ_DATUM_NUMBER_RES_SIZE); + MEMCPY(buf_, src.ptr_, src.len_); + ptr_ = buf_; + } else { + char * buf = static_cast(allocator.alloc(src.len_)); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + COMMON_LOG(WARN, "allocate memory failed", K(ret), K(src)); + pack_ = 0; + } else { + MEMCPY(buf, src.ptr_, src.len_); + // need set ptr_ after memory copy, if this == &src + ptr_ = buf; + } + } + return ret; +} + +OB_INLINE int ObStorageDatum::deep_copy(const ObStorageDatum &src, char * buf, const int64_t buf_len, int64_t &pos) +{ + int ret = common::OB_SUCCESS; + + reuse(); + pack_ = src.pack_; + if (is_null()) { + } else if (src.len_ == 0) { + } else if (src.is_local_buf()) { + OB_ASSERT(src.len_ <= common::OBJ_DATUM_NUMBER_RES_SIZE); + MEMCPY(buf_, src.ptr_, src.len_); + ptr_ = buf_; + } else if (OB_UNLIKELY(nullptr == buf || buf_len < pos + src.len_)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Invalid argument to deep copy datum", K(ret), K(src), KP(buf), K(buf_len), K(pos)); + pack_ = 0; + } else { + MEMCPY(buf + pos, src.ptr_, src.len_); + // need set ptr_ after memory copy, if this == &src + ptr_ = buf + pos; + pos += src.len_; + } + + return ret; +} + +OB_INLINE void ObStorageDatum::shallow_copy_from_datum(const ObDatum &src) +{ + if (this != &src) { + reuse(); + pack_ = src.pack_; + if (is_null()) { + } else if (src.len_ == 0) { + } else { + ptr_ = src.ptr_; + } + } +} + +OB_INLINE int64_t ObStorageDatum::get_deep_copy_size() const +{ + int64_t deep_copy_len = 0; + if (is_null()) { + } else if (is_local_buf()) { + OB_ASSERT(len_ <= common::OBJ_DATUM_NUMBER_RES_SIZE); + } else { + deep_copy_len = len_; + } + return deep_copy_len; +} + +OB_INLINE int ObStorageDatum::from_buf_enhance(const char *buf, const int64_t buf_len) +{ + int ret = common::OB_SUCCESS; + + if (OB_UNLIKELY(nullptr == buf || buf_len < 0 || buf_len > UINT32_MAX)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Invalid argument to transfer from buf", K(ret), KP(buf), K(buf_len)); + } else { + reuse(); + len_ = static_cast(buf_len); + if (buf_len > 0) { + ptr_ = buf; + } + } + + + return ret; +} + +OB_INLINE int ObStorageDatum::from_obj_enhance(const common::ObObj &obj) +{ + int ret = common::OB_SUCCESS; + + reuse(); + if (obj.is_ext()) { + set_ext_value(obj.get_ext()); + } else if (OB_FAIL(from_obj(obj))) { + STORAGE_LOG(WARN, "Failed to transfer obj to datum", K(ret), K(obj)); + } + STORAGE_LOG(DEBUG, "chaser debug from obj", K(obj), K(*this)); + + return ret; +} + + +OB_INLINE int ObStorageDatum::to_obj_enhance(common::ObObj &obj, const common::ObObjMeta &meta) const +{ + int ret = common::OB_SUCCESS; + if (is_outrow()) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "lob should not set outrow in datum", K(ret), K(*this), K(obj), K(meta)); + } else if (is_ext()) { + obj.set_ext(get_ext()); + } else if (OB_FAIL(to_obj(obj, meta))) { + STORAGE_LOG(WARN, "Failed to transfer datum to obj", K(ret), K(*this), K(obj), K(meta)); + } + + return ret; +} + +OB_INLINE ObStorageDatum& ObStorageDatum::operator=(const ObStorageDatum &other) +{ + if (&other != this) { + reuse(); + pack_ = other.pack_; + if (is_null()) { + } else if (len_ == 0) { + } else if (other.is_local_buf()) { + OB_ASSERT(other.len_ <= common::OBJ_DATUM_NUMBER_RES_SIZE); + MEMCPY(buf_, other.ptr_, other.len_); + ptr_ = buf_; + } else { + ptr_ = other.ptr_; + } + } + return *this; +} + +OB_INLINE bool ObStorageDatum::operator==(const ObStorageDatum &other) const +{ + bool bret = true; + if (is_null()) { + bret = other.is_null(); + } else if (is_ext()) { + bret = other.is_ext() && extend_obj_->get_ext() == other.extend_obj_->get_ext(); + } else { + bret = ObDatum::binary_equal(*this, other); + } + if (!bret) { + STORAGE_LOG(DEBUG, "datum and datum no equal", K(other), K(*this)); + } + return bret; + +} + +OB_INLINE bool ObStorageDatum::operator==(const common::ObObj &other) const +{ + + int ret = OB_SUCCESS; + bool bret = true; + ObStorageDatum datum; + if (OB_FAIL(datum.from_obj_enhance(other))) { + STORAGE_LOG(WARN, "Failed to transfer obj to datum", K(ret), K(other), K(datum)); + } else { + bret = *this == datum; + } + if (!bret) { + STORAGE_LOG(DEBUG, "obj and datum no equal", K(other), K(datum), KPC(this)); + } + return bret; +} + +OB_INLINE int64_t ObStorageDatum::storage_to_string(char *buf, int64_t buf_len, const bool for_dump) const +{ + int64_t pos = 0; + if (is_ext()) { + if (is_nop()) { + J_NOP(); + } else if (is_max()) { + BUF_PRINTF("MAX_OBJ"); + } else if (is_min()) { + BUF_PRINTF("MIN_OBJ"); + } + } else if(!for_dump) { + pos = to_string(buf, buf_len); + } else { + int ret = OB_SUCCESS; + const static int64_t STR_MAX_PRINT_LEN = 128L; + if (null_) { + J_NULL(); + } else { + J_OBJ_START(); + BUF_PRINTF("len: %d, flag: %d, null: %d", len_, flag_, null_); + if (len_ > 0) { + OB_ASSERT(NULL != ptr_); + const int64_t plen = std::min(static_cast(len_), + static_cast(STR_MAX_PRINT_LEN)); + // print hex value + BUF_PRINTF(", hex: "); + if (OB_FAIL(hex_print(ptr_, plen, buf, buf_len, pos))) { + // no logging in to_string function. + } else { + // maybe ObIntTC + if (sizeof(int64_t) == len_) { + BUF_PRINTF(", int: %ld", *int_); + // maybe number with one digit + if (1 == num_->desc_.len_) { + BUF_PRINTF(", num_digit0: %u", num_->digits_[0]); + } + } + // maybe printable C string + int64_t idx = 0; + while (idx < plen && isprint(ptr_[idx])) { + idx++; + } + if (idx >= plen) { + BUF_PRINTF(", cstr: %.*s", static_cast(plen), ptr_); + } + } + } + J_OBJ_END(); + } + } + + return pos; +} + +OB_INLINE const char *ObStorageDatum::to_cstring(const bool for_dump) const +{ + char *buffer = NULL; + int64_t str_len = 0; + CStringBufMgr &mgr = CStringBufMgr::get_thread_local_instance(); + mgr.inc_level(); + const int64_t buf_len = mgr.acquire(buffer); + if (OB_ISNULL(buffer)) { + LIB_LOG_RET(ERROR, OB_ALLOCATE_MEMORY_FAILED, "buffer is NULL"); + } else { + str_len = storage_to_string(buffer, buf_len -1, for_dump); + if (str_len >= 0 && str_len < buf_len) { + buffer[str_len] = '\0'; + } else { + buffer[0] = '\0'; + } + mgr.update_position(str_len + 1); + } + mgr.try_clear_list(); + mgr.dec_level(); + return buffer; +} + +OB_INLINE bool ObStorageDatum::need_copy_for_encoding_column_with_flat_format(const ObObjDatumMapType map_type) const +{ + return OBJ_DATUM_STRING == map_type && sizeof(uint64_t) == len_ && is_local_buf(); +} +} // namespace blocksstable +} // namespace oceanbase +#endif diff --git a/src/storage/lob/ob_ext_info_callback.cpp b/src/storage/lob/ob_ext_info_callback.cpp index bdd4b12c6..5db3148cb 100644 --- a/src/storage/lob/ob_ext_info_callback.cpp +++ b/src/storage/lob/ob_ext_info_callback.cpp @@ -186,7 +186,7 @@ int ObExtInfoCallback::set( { int ret = OB_SUCCESS; ObLobManager *lob_mngr = MTL(ObLobManager*); - ObDatumRow datum_row; + blocksstable::ObDatumRow datum_row; char *buf = nullptr; int64_t len = 0; @@ -250,7 +250,8 @@ int ObExtInfoCbRegister::register_cb( const blocksstable::ObDmlFlag dml_flag, transaction::ObTxDesc *tx_desc, transaction::ObTxSEQ &parent_seq_no, - ObObj &index_data, + blocksstable::ObStorageDatum &index_data, + ObObjType &type, ObObj &ext_info_data) { int ret = OB_SUCCESS; @@ -269,7 +270,7 @@ int ObExtInfoCbRegister::register_cb( } else if (OB_ISNULL(mvcc_ctx_ = ctx)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("data is empty", K(ret), K(ext_info_data)); - } else if (OB_FAIL(init_header(index_data, ext_info_data))) { + } else if (OB_FAIL(init_header(type))) { LOG_WARN("init_header_ fail", K(ret), K(ext_info_data)); } else if (OB_FAIL(build_data_iter(ext_info_data))) { LOG_WARN("build data iter fail", K(ret)); @@ -306,24 +307,24 @@ int ObExtInfoCbRegister::register_cb( } if (OB_FAIL(ret)) { } else if (OB_FALSE_IT(seq_no_cnt_ = cb_cnt)) { - } else if (OB_FAIL(set_index_data(index_data))) { + } else if (OB_FAIL(set_index_data(index_data, type))) { LOG_WARN("set_index_data fail", K(ret)); } } return ret; } -int ObExtInfoCbRegister::init_header(ObObj& index_data, ObObj &ext_info_data) +int ObExtInfoCbRegister::init_header(ObObjType &type) { int ret = OB_SUCCESS; - header_.type_ = get_type(index_data.get_type()); + header_.type_ = get_type(type); return ret; } -int ObExtInfoCbRegister::set_index_data(ObObj &index_data) +int ObExtInfoCbRegister::set_index_data(blocksstable::ObStorageDatum &index_data, ObObjType &type) { int ret = OB_SUCCESS; - if (is_lob_storage(index_data.get_type())) { + if (is_lob_storage(type)) { if (OB_FAIL(set_outrow_ctx_seq_no(index_data))) { LOG_WARN("set_outrow_ctx_seq_no fail", K(ret)); } @@ -334,7 +335,7 @@ int ObExtInfoCbRegister::set_index_data(ObObj &index_data) return ret; } -int ObExtInfoCbRegister::set_outrow_ctx_seq_no(ObObj& index_data) +int ObExtInfoCbRegister::set_outrow_ctx_seq_no(blocksstable::ObStorageDatum& index_data) { int ret = OB_SUCCESS; ObLobLocatorV2 locator; diff --git a/src/storage/lob/ob_ext_info_callback.h b/src/storage/lob/ob_ext_info_callback.h index b6c32be34..c0857c298 100644 --- a/src/storage/lob/ob_ext_info_callback.h +++ b/src/storage/lob/ob_ext_info_callback.h @@ -153,12 +153,13 @@ public: const blocksstable::ObDmlFlag dml_flag, transaction::ObTxDesc *tx_desc, transaction::ObTxSEQ &parent_seq_no, - ObObj &index_data, + blocksstable::ObStorageDatum &index_data, + ObObjType &type, ObObj &ext_info_data); private: - static ObExtInfoLogType get_type(ObObjType obj_type) + static ObExtInfoLogType get_type(ObObjType &obj_type) { ObExtInfoLogType res = OB_INVALID_EXT_INFO_LOG; switch (obj_type) @@ -174,12 +175,12 @@ private: int build_data_iter(ObObj &ext_info_data); - int set_index_data(ObObj &index_data); - int set_outrow_ctx_seq_no(ObObj& index_data); + int set_index_data(blocksstable::ObStorageDatum &index_data, ObObjType &type); + int set_outrow_ctx_seq_no(blocksstable::ObStorageDatum& index_data); int get_data(ObString &data); - int init_header(ObObj& index_data, ObObj &ext_info_data); + int init_header(ObObjType &type); public: TO_STRING_KV(K(timeout_), K(data_size_), K(seq_no_st_), K(seq_no_cnt_), K(header_writed_)); diff --git a/src/storage/lob/ob_lob_meta.cpp b/src/storage/lob/ob_lob_meta.cpp index 93382f6ba..546f256bd 100644 --- a/src/storage/lob/ob_lob_meta.cpp +++ b/src/storage/lob/ob_lob_meta.cpp @@ -972,7 +972,7 @@ int ObLobMetaManager::write(ObLobAccessParam& param, ObLobMetaInfo& in_row) return ret; } -int ObLobMetaManager::batch_insert(ObLobAccessParam& param, ObNewRowIterator &iter) +int ObLobMetaManager::batch_insert(ObLobAccessParam& param, blocksstable::ObDatumRowIterator &iter) { int ret = OB_SUCCESS; if (OB_FAIL(persistent_lob_adapter_.write_lob_meta(param, iter))) { @@ -981,7 +981,7 @@ int ObLobMetaManager::batch_insert(ObLobAccessParam& param, ObNewRowIterator &it return ret; } -int ObLobMetaManager::batch_delete(ObLobAccessParam& param, ObNewRowIterator &iter) +int ObLobMetaManager::batch_delete(ObLobAccessParam& param, blocksstable::ObDatumRowIterator &iter) { int ret = OB_SUCCESS; if (OB_FAIL(persistent_lob_adapter_.erase_lob_meta(param, iter))) { diff --git a/src/storage/lob/ob_lob_meta.h b/src/storage/lob/ob_lob_meta.h index 3d840d07d..707dfd844 100644 --- a/src/storage/lob/ob_lob_meta.h +++ b/src/storage/lob/ob_lob_meta.h @@ -230,8 +230,8 @@ public: ~ObLobMetaManager() {} // write one lob meta row int write(ObLobAccessParam& param, ObLobMetaInfo& in_row); - int batch_insert(ObLobAccessParam& param, ObNewRowIterator &iter); - int batch_delete(ObLobAccessParam& param, ObNewRowIterator &iter); + int batch_insert(ObLobAccessParam& param, blocksstable::ObDatumRowIterator &iter); + int batch_delete(ObLobAccessParam& param, blocksstable::ObDatumRowIterator &iter); // append int append(ObLobAccessParam& param, ObLobMetaWriteIter& iter); // return ObLobMetaWriteResult diff --git a/src/storage/lob/ob_lob_persistent_adaptor.cpp b/src/storage/lob/ob_lob_persistent_adaptor.cpp index 0fee1d240..e6b8ab4a0 100644 --- a/src/storage/lob/ob_lob_persistent_adaptor.cpp +++ b/src/storage/lob/ob_lob_persistent_adaptor.cpp @@ -418,15 +418,14 @@ int ObPersistentLobApator::erase_lob_piece_tablet(ObLobAccessParam& param, ObLob // construct insert data int64_t affected_rows = 0; - ObObj cell[ObLobPieceUtil::LOB_PIECE_COLUMN_CNT]; char serialize_buf[32] = {0}; // make insert iterator - ObNewRow new_row; + ObDatumRow new_row; + blocksstable::ObSingleDatumRowIteratorWrapper single_iter; - common::ObSingleRowIteratorWrapper single_iter; - single_iter.set_row(&new_row); - - if (OB_FAIL(set_lob_piece_row(serialize_buf, 32, cell, new_row, &single_iter, in_row))) { + if (OB_FAIL(new_row.init(ObLobPieceUtil::LOB_PIECE_COLUMN_CNT))) { + LOG_WARN("failed to init new datum row", K(ret)); + } else if (OB_FAIL(set_lob_piece_row(serialize_buf, 32, new_row, &single_iter, in_row))) { LOG_WARN("failed to set insert piece row.", K(ret), K(in_row)); } else if (OB_FAIL(oas->delete_rows(param.ls_id_, lob_piece_tablet.get_obj()->get_tablet_meta().tablet_id_, @@ -534,15 +533,14 @@ int ObPersistentLobApator::write_lob_piece_tablet(ObLobAccessParam& param, ObLob // construct insert data int64_t affected_rows = 0; - ObObj cell[ObLobPieceUtil::LOB_PIECE_COLUMN_CNT]; char serialize_buf[32] = {0}; // make insert iterator - ObNewRow new_row; + ObDatumRow new_row; + blocksstable::ObSingleDatumRowIteratorWrapper single_iter; - common::ObSingleRowIteratorWrapper single_iter; - single_iter.set_row(&new_row); - - if (OB_FAIL(set_lob_piece_row(serialize_buf, 32, cell, new_row, &single_iter, in_row))) { + if (OB_FAIL(new_row.init(ObLobPieceUtil::LOB_PIECE_COLUMN_CNT))) { + LOG_WARN("failed to init new datum row", K(ret)); + } else if (OB_FAIL(set_lob_piece_row(serialize_buf, 32, new_row, &single_iter, in_row))) { LOG_WARN("failed to set insert piece row.", K(ret), K(in_row)); } else if (OB_FAIL(oas->insert_rows(param.ls_id_, lob_piece_tablet.get_obj()->get_tablet_meta().tablet_id_, @@ -604,15 +602,14 @@ int ObPersistentLobApator::update_lob_piece_tablet(ObLobAccessParam& param, ObLo // construct insert data int64_t affected_rows = 0; - ObObj cell[ObLobPieceUtil::LOB_PIECE_COLUMN_CNT]; char serialize_buf[32] = {0}; // make insert iterator - ObNewRow new_row; + ObDatumRow new_row; + blocksstable::ObSingleDatumRowIteratorWrapper single_iter; - common::ObSingleRowIteratorWrapper single_iter; - single_iter.set_row(&new_row); - - if (OB_FAIL(set_lob_piece_row(serialize_buf, 32, cell, new_row, &single_iter, in_row))) { + if (OB_FAIL(new_row.init(ObLobPieceUtil::LOB_PIECE_COLUMN_CNT))) { + LOG_WARN("failed to init new datum row", K(ret)); + } else if (OB_FAIL(set_lob_piece_row(serialize_buf, 32, new_row, &single_iter, in_row))) { LOG_WARN("failed to set insert piece row.", K(ret), K(in_row)); } else if (OB_FAIL(oas->update_rows(param.ls_id_, lob_piece_tablet.get_obj()->get_tablet_meta().tablet_id_, @@ -824,43 +821,32 @@ int ObPersistentLobApator::inner_get_tablet( } void ObPersistentLobApator::set_lob_meta_row( - ObObj* cell, - ObNewRow& new_row, + blocksstable::ObDatumRow& datum_row, ObLobMetaInfo& in_row) { - for (int64_t i = 0; i < ObLobMetaUtil::LOB_META_COLUMN_CNT; ++i) { - cell[i].reset(); - cell[i].set_nop_value(); - } - cell[ObLobMetaUtil::LOB_ID_COL_ID].set_varchar(reinterpret_cast(&in_row.lob_id_), sizeof(ObLobId)); - cell[ObLobMetaUtil::LOB_ID_COL_ID].set_collation_type(common::ObCollationType::CS_TYPE_BINARY); - cell[ObLobMetaUtil::SEQ_ID_COL_ID].set_varchar(in_row.seq_id_); - cell[ObLobMetaUtil::SEQ_ID_COL_ID].set_collation_type(common::ObCollationType::CS_TYPE_BINARY); - cell[ObLobMetaUtil::BYTE_LEN_COL_ID].set_uint32(in_row.byte_len_); - cell[ObLobMetaUtil::CHAR_LEN_COL_ID].set_uint32(in_row.char_len_); - cell[ObLobMetaUtil::PIECE_ID_COL_ID].set_uint64(in_row.piece_id_); - - cell[ObLobMetaUtil::LOB_DATA_COL_ID].set_varchar(in_row.lob_data_); - cell[ObLobMetaUtil::LOB_DATA_COL_ID].set_collation_type(common::ObCollationType::CS_TYPE_BINARY); - - new_row.assign(cell, ObLobMetaUtil::LOB_META_COLUMN_CNT); + datum_row.reuse(); + datum_row.storage_datums_[ObLobMetaUtil::LOB_ID_COL_ID].set_string(reinterpret_cast(&in_row.lob_id_), sizeof(ObLobId)); + // TODO: if we need set collation type to be common::ObCollationType::CS_TYPE_BINARY@xuanxi + datum_row.storage_datums_[ObLobMetaUtil::SEQ_ID_COL_ID].set_string(in_row.seq_id_); + // TODO: if we need set collation type to be common::ObCollationType::CS_TYPE_BINARY@xuanxi + datum_row.storage_datums_[ObLobMetaUtil::BYTE_LEN_COL_ID].set_uint32(in_row.byte_len_); + datum_row.storage_datums_[ObLobMetaUtil::CHAR_LEN_COL_ID].set_uint32(in_row.char_len_); + datum_row.storage_datums_[ObLobMetaUtil::PIECE_ID_COL_ID].set_uint(in_row.piece_id_); + datum_row.storage_datums_[ObLobMetaUtil::LOB_DATA_COL_ID].set_string(in_row.lob_data_); + // TODO: if we need set collation type to be common::ObCollationType::CS_TYPE_BINARY@xuanxi } int ObPersistentLobApator::set_lob_piece_row( char* buf, size_t buf_len, - ObObj* cell, - ObNewRow& new_row, - common::ObSingleRowIteratorWrapper* new_row_iter, + ObDatumRow& datum_row, + blocksstable::ObSingleDatumRowIteratorWrapper* new_row_iter, ObLobPieceInfo& in_row) { int ret = OB_SUCCESS; - for (int64_t i = 0; i < ObLobPieceUtil::LOB_PIECE_COLUMN_CNT; ++i) { - cell[i].reset(); - cell[i].set_nop_value(); - } - cell[0].set_uint64(in_row.piece_id_); - cell[1].set_uint32(in_row.len_); + datum_row.reuse(); + datum_row.storage_datums_[0].set_uint(in_row.piece_id_); + datum_row.storage_datums_[1].set_uint32(in_row.len_); int64_t pos = 0; if (!in_row.macro_id_.is_valid()) { @@ -868,11 +854,9 @@ int ObPersistentLobApator::set_lob_piece_row( } else if (OB_FAIL(in_row.macro_id_.serialize(buf, buf_len, pos))) { LOG_WARN("failed to serialize macro id", K(ret), K(buf_len), K(pos)); } else { - cell[2].set_varchar(buf, pos); - cell[2].set_collation_type(common::ObCollationType::CS_TYPE_BINARY); - - new_row.assign(cell, ObLobPieceUtil::LOB_PIECE_COLUMN_CNT); - new_row_iter->set_row(&new_row); + datum_row.storage_datums_[2].set_string(buf, pos); + // TODO: if we need set collation type to be common::ObCollationType::CS_TYPE_BINARY@xuanxi + new_row_iter->set_row(&datum_row); } return ret; @@ -928,20 +912,20 @@ int ObPersistentLobApator::do_scan_lob_meta( ObTabletHandle lob_meta_tablet; ObTabletHandle lob_piece_tablet; if (OB_FAIL(get_lob_tablets(param, data_tablet, lob_meta_tablet, lob_piece_tablet))) { - LOG_WARN("failed to get tablets.", K(ret), K(param)); + LOG_WARN("failed to get tablets", K(ret), K(param)); } else { ObAccessService *oas = MTL(ObAccessService*); scan_param.tablet_id_ = param.lob_meta_tablet_id_; scan_param.schema_version_ = lob_meta_tablet.get_obj()->get_tablet_meta().max_sync_storage_schema_version_; if (OB_ISNULL(oas)) { ret = OB_ERR_INTERVAL_INVALID; - LOG_WARN("get access service failed.", K(ret)); + LOG_WARN("get access service failed", K(ret)); } else if (OB_FAIL(build_common_scan_param(param, param.has_single_chunk(), ObLobMetaUtil::LOB_META_COLUMN_CNT, scan_param))) { - LOG_WARN("build common scan param failed.", K(ret), K(param)); + LOG_WARN("build common scan param failed", K(ret), K(param)); } else if (OB_FAIL(prepare_table_param(param, scan_param, true/*is_meta*/))) { - LOG_WARN("prepare lob meta table param failed.", K(ret), K(param)); + LOG_WARN("prepare lob meta table param failed", K(ret), K(param)); } else if (OB_FAIL(oas->table_scan(scan_param, meta_iter))) { - LOG_WARN("do table scan falied.", K(ret), K(scan_param), K(param)); + LOG_WARN("do table scan falied", K(ret), K(scan_param), K(param)); } } return ret; @@ -967,7 +951,7 @@ int ObPersistentLobApator::scan_lob_meta( } else if (OB_FAIL(param.get_rowkey_range(rowkey_objs, range))) { LOG_WARN("get_rowkey_range fail", K(ret)); } else if (OB_FAIL(scan_param.key_ranges_.push_back(range))) { - LOG_WARN("failed to push key range.", K(ret), K(scan_param), K(range)); + LOG_WARN("failed to push key range", K(ret), K(scan_param), K(range)); } else if (OB_FAIL(do_scan_lob_meta(param, scan_param, meta_iter))) { LOG_WARN("do_scan_lob_meta fail", K(ret)); } @@ -1004,28 +988,28 @@ int ObPersistentLobApator::set_dml_seq_no(ObLobAccessParam ¶m) LOG_DEBUG("dml lob meta with seq no", K(param.dml_base_param_->spec_seq_no_)); } else { ret = OB_INVALID_ARGUMENT; - LOG_WARN("failed to get seq no from param.", K(ret), K(param)); + LOG_WARN("failed to get seq no from param", K(ret), K(param)); } } else { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid seq no from param.", K(ret), K(param)); + LOG_WARN("invalid seq no from param", K(ret), K(param)); } return ret; } -int ObPersistentLobApator::erase_lob_meta(ObLobAccessParam ¶m, ObNewRowIterator& row_iter) +int ObPersistentLobApator::erase_lob_meta(ObLobAccessParam ¶m, ObDatumRowIterator& row_iter) { int ret = OB_SUCCESS; int64_t affected_rows = 0; ObAccessService *oas = MTL(ObAccessService*); if (OB_ISNULL(oas)) { ret = OB_ERR_INTERVAL_INVALID; - LOG_WARN("get access service failed.", K(ret), KP(oas)); + LOG_WARN("get access service failed", K(ret), KP(oas)); } else if (OB_ISNULL(param.tx_desc_)) { ret = OB_ERR_NULL_VALUE; - LOG_WARN("get tx desc null.", K(ret), K(param)); + LOG_WARN("get tx desc null", K(ret), K(param)); } else if (OB_FAIL(prepare_lob_tablet_id(param))) { - LOG_WARN("failed to get tablets.", K(ret), K(param)); + LOG_WARN("failed to get tablets", K(ret), K(param)); } else if (OB_FAIL(prepare_lob_meta_dml(param))) { LOG_WARN("failed to prepare lob meta dml", K(ret)); } else if (OB_FAIL(oas->delete_rows( @@ -1041,19 +1025,19 @@ int ObPersistentLobApator::erase_lob_meta(ObLobAccessParam ¶m, ObNewRowItera return ret; } -int ObPersistentLobApator::write_lob_meta(ObLobAccessParam& param, ObNewRowIterator& row_iter) +int ObPersistentLobApator::write_lob_meta(ObLobAccessParam& param, ObDatumRowIterator& row_iter) { int ret = OB_SUCCESS; int64_t affected_rows = 0; ObAccessService *oas = MTL(ObAccessService*); if (OB_ISNULL(oas)) { ret = OB_ERR_INTERVAL_INVALID; - LOG_WARN("get access service failed.", K(ret), KP(oas)); + LOG_WARN("get access service failed", K(ret), KP(oas)); } else if (OB_ISNULL(param.tx_desc_)) { ret = OB_ERR_NULL_VALUE; - LOG_WARN("get tx desc null.", K(ret), K(param)); + LOG_WARN("get tx desc null", K(ret), K(param)); } else if (OB_FAIL(prepare_lob_tablet_id(param))) { - LOG_WARN("failed to get tablets.", K(ret), K(param)); + LOG_WARN("failed to get tablets", K(ret), K(param)); } else if (OB_FAIL(prepare_lob_meta_dml(param))) { LOG_WARN("failed to prepare lob meta dml", K(ret)); } else if (OB_FAIL(oas->insert_rows( @@ -1069,26 +1053,26 @@ int ObPersistentLobApator::write_lob_meta(ObLobAccessParam& param, ObNewRowItera return ret; } -int ObPersistentLobApator::update_lob_meta(ObLobAccessParam& param, ObNewRowIterator &row_iter) +int ObPersistentLobApator::update_lob_meta(ObLobAccessParam& param, ObDatumRowIterator &row_iter) { int ret = OB_SUCCESS; int64_t affected_rows = 0; ObAccessService *oas = MTL(ObAccessService*); if (OB_ISNULL(oas)) { ret = OB_ERR_INTERVAL_INVALID; - LOG_WARN("get access service failed.", K(ret), KP(oas)); + LOG_WARN("get access service failed", K(ret), KP(oas)); } else if (OB_ISNULL(param.tx_desc_)) { ret = OB_ERR_NULL_VALUE; - LOG_WARN("get tx desc null.", K(ret), K(param)); + LOG_WARN("get tx desc null", K(ret), K(param)); } else if (OB_FAIL(prepare_lob_tablet_id(param))) { - LOG_WARN("failed to get tablets.", K(ret), K(param)); + LOG_WARN("failed to get tablets", K(ret), K(param)); } else if (OB_FAIL(prepare_lob_meta_dml(param))) { LOG_WARN("failed to prepare lob meta dml", K(ret)); } else { ObSEArray update_column_ids; for (int i = 2; OB_SUCC(ret) && i < ObLobMetaUtil::LOB_META_COLUMN_CNT; ++i) { if (OB_FAIL(update_column_ids.push_back(OB_APP_MIN_COLUMN_ID + i))) { - LOG_WARN("push column ids failed.", K(ret), K(i)); + LOG_WARN("push column ids failed", K(ret), K(i)); } } if (OB_FAIL(ret)) { @@ -1110,11 +1094,12 @@ int ObPersistentLobApator::update_lob_meta(ObLobAccessParam& param, ObNewRowIter int ObPersistentLobApator::write_lob_meta(ObLobAccessParam ¶m, ObLobMetaInfo& row_info) { int ret = OB_SUCCESS; - ObObj cell[ObLobMetaUtil::LOB_META_COLUMN_CNT]; - ObNewRow new_row; + ObDatumRow new_row; ObLobPersistInsertSingleRowIter single_iter; - set_lob_meta_row(cell, new_row, row_info); - if (OB_FAIL(single_iter.init(¶m, &new_row))) { + if (OB_FAIL(new_row.init(ObLobMetaUtil::LOB_META_COLUMN_CNT))) { + LOG_WARN("failed to init datum row", K(ret)); + } else if (FALSE_IT(set_lob_meta_row(new_row, row_info))) { + } else if (OB_FAIL(single_iter.init(¶m, &new_row))) { LOG_WARN("single_iter init fail", K(ret)); } else if (OB_FAIL(write_lob_meta(param, single_iter))) { LOG_WARN("write_lob_meta fail", K(ret)); @@ -1125,11 +1110,12 @@ int ObPersistentLobApator::write_lob_meta(ObLobAccessParam ¶m, ObLobMetaInfo int ObPersistentLobApator::erase_lob_meta(ObLobAccessParam ¶m, ObLobMetaInfo& row_info) { int ret = OB_SUCCESS; - ObObj cell[ObLobMetaUtil::LOB_META_COLUMN_CNT]; - ObNewRow new_row; + ObDatumRow new_row; ObLobPersistDeleteSingleRowIter single_iter; - set_lob_meta_row(cell, new_row, row_info); - if (OB_FAIL(single_iter.init(¶m, &new_row))) { + if (OB_FAIL(new_row.init(ObLobMetaUtil::LOB_META_COLUMN_CNT))) { + LOG_WARN("failed to init datum row", K(ret)); + } else if (FALSE_IT(set_lob_meta_row(new_row, row_info))) { + } else if (OB_FAIL(single_iter.init(¶m, &new_row))) { LOG_WARN("single_iter init fail", K(ret)); } else if (OB_FAIL(erase_lob_meta(param, single_iter))) { LOG_WARN("erase_lob_meta fail", K(ret)); @@ -1140,14 +1126,16 @@ int ObPersistentLobApator::erase_lob_meta(ObLobAccessParam ¶m, ObLobMetaInfo int ObPersistentLobApator::update_lob_meta(ObLobAccessParam& param, ObLobMetaInfo& old_row, ObLobMetaInfo& new_row) { int ret = OB_SUCCESS; - ObObj new_row_cell[ObLobMetaUtil::LOB_META_COLUMN_CNT]; - ObNewRow new_tbl_row; - set_lob_meta_row(new_row_cell, new_tbl_row, new_row); - ObObj old_row_cell[ObLobMetaUtil::LOB_META_COLUMN_CNT]; - ObNewRow old_tbl_row; - set_lob_meta_row(old_row_cell, old_tbl_row, old_row); + ObDatumRow new_datum_row; + ObDatumRow old_datum_row; ObLobPersistUpdateSingleRowIter upd_iter; - if (OB_FAIL(upd_iter.init(¶m, &old_tbl_row, &new_tbl_row))) { + if (OB_FAIL(new_datum_row.init(ObLobMetaUtil::LOB_META_COLUMN_CNT))) { + LOG_WARN("failed to init new datum row", K(ret)); + } else if (OB_FAIL(old_datum_row.init(ObLobMetaUtil::LOB_META_COLUMN_CNT))) { + LOG_WARN("failed to init old datum row", K(ret)); + } else if (FALSE_IT(set_lob_meta_row(new_datum_row, new_row))) { + } else if (FALSE_IT(set_lob_meta_row(old_datum_row, old_row))) { + } else if (OB_FAIL(upd_iter.init(¶m, &old_datum_row, &new_datum_row))) { LOG_WARN("upd_iter init fail", K(ret)); } else if (OB_FAIL(update_lob_meta(param, upd_iter))) { LOG_WARN("update_lob_meta fail", K(ret)); diff --git a/src/storage/lob/ob_lob_persistent_adaptor.h b/src/storage/lob/ob_lob_persistent_adaptor.h index fb41e92bb..1fc4d719d 100644 --- a/src/storage/lob/ob_lob_persistent_adaptor.h +++ b/src/storage/lob/ob_lob_persistent_adaptor.h @@ -17,6 +17,7 @@ #include "storage/blocksstable/ob_macro_block_id.h" #include "ob_i_lob_adaptor.h" #include "common/row/ob_row_iterator.h" +#include "storage/blocksstable/ob_datum_row_iterator.h" namespace oceanbase { @@ -25,22 +26,20 @@ namespace storage class ObStoreCtxGuard; -class ObLobUpdIterator : public ObNewRowIterator +class ObLobUpdIterator : public blocksstable::ObDatumRowIterator { public: - ObLobUpdIterator(ObNewRow *old_row, - ObNewRow *new_row) + ObLobUpdIterator(blocksstable::ObDatumRow *old_row, + blocksstable::ObDatumRow *new_row) : old_row_(old_row), new_row_(new_row), got_old_row_(false), is_iter_end_(false) {} - virtual int get_next_row(ObNewRow *&row) override; - virtual int get_next_row() override { return OB_NOT_IMPLEMENT; } - virtual void reset() override {} + virtual int get_next_row(blocksstable::ObDatumRow *&row) override; private: - ObNewRow *old_row_; - ObNewRow *new_row_; + blocksstable::ObDatumRow *old_row_; + blocksstable::ObDatumRow *new_row_; bool got_old_row_; bool is_iter_end_; }; @@ -94,13 +93,12 @@ public: virtual int update_lob_meta(ObLobAccessParam& param, ObLobMetaInfo& old_row, ObLobMetaInfo& new_row) override; static void set_lob_meta_row( - ObObj* cell, - ObNewRow& new_row, + blocksstable::ObDatumRow& datum_row, ObLobMetaInfo& in_row); - int write_lob_meta(ObLobAccessParam ¶m, ObNewRowIterator& row_iter); - int erase_lob_meta(ObLobAccessParam ¶m, ObNewRowIterator& row_iter); - int update_lob_meta(ObLobAccessParam& param, ObNewRowIterator &row_iter); + int write_lob_meta(ObLobAccessParam ¶m, blocksstable::ObDatumRowIterator& row_iter); + int erase_lob_meta(ObLobAccessParam ¶m, blocksstable::ObDatumRowIterator& row_iter); + int update_lob_meta(ObLobAccessParam& param, blocksstable::ObDatumRowIterator &row_iter); int build_common_scan_param( const ObLobAccessParam ¶m, @@ -159,9 +157,8 @@ private: int set_lob_piece_row( char* buf, size_t buf_len, - ObObj* cell, - ObNewRow& new_row, - common::ObSingleRowIteratorWrapper* new_row_iter, + blocksstable::ObDatumRow& datum_row, + blocksstable::ObSingleDatumRowIteratorWrapper* new_row_iter, ObLobPieceInfo& in_row); int init_table_param(); diff --git a/src/storage/lob/ob_lob_persistent_iterator.cpp b/src/storage/lob/ob_lob_persistent_iterator.cpp index 03fcf147d..bdfc804e5 100644 --- a/src/storage/lob/ob_lob_persistent_iterator.cpp +++ b/src/storage/lob/ob_lob_persistent_iterator.cpp @@ -65,7 +65,7 @@ int ObLobPersistWriteIter::dec_lob_size(ObLobMetaInfo &info) return ret; } -int ObLobPersistUpdateSingleRowIter::init(ObLobAccessParam *param, ObNewRow *old_row, ObNewRow *new_row) +int ObLobPersistUpdateSingleRowIter::init(ObLobAccessParam *param, blocksstable::ObDatumRow *old_row, blocksstable::ObDatumRow *new_row) { int ret = OB_SUCCESS; if (OB_ISNULL(param) || OB_ISNULL(old_row) || OB_ISNULL(new_row)) { @@ -79,7 +79,7 @@ int ObLobPersistUpdateSingleRowIter::init(ObLobAccessParam *param, ObNewRow *old return ret; } -int ObLobPersistUpdateSingleRowIter::get_next_row(ObNewRow *&row) +int ObLobPersistUpdateSingleRowIter::get_next_row(blocksstable::ObDatumRow *&row) { int ret = OB_SUCCESS; if (OB_ISNULL(old_row_) || OB_ISNULL(new_row_)) { @@ -100,7 +100,7 @@ int ObLobPersistUpdateSingleRowIter::get_next_row(ObNewRow *&row) return ret; } -int ObLobPersistInsertSingleRowIter::init(ObLobAccessParam *param, ObNewRow *row) +int ObLobPersistInsertSingleRowIter::init(ObLobAccessParam *param, blocksstable::ObDatumRow *row) { int ret = OB_SUCCESS; if (OB_ISNULL(param) || OB_ISNULL(row)) { @@ -113,7 +113,7 @@ int ObLobPersistInsertSingleRowIter::init(ObLobAccessParam *param, ObNewRow *row return ret; } -int ObLobPersistInsertSingleRowIter::get_next_row(ObNewRow *&row) +int ObLobPersistInsertSingleRowIter::get_next_row(blocksstable::ObDatumRow *&row) { int ret = OB_SUCCESS; if (OB_ISNULL(param_) || OB_ISNULL(row_)) { @@ -130,7 +130,7 @@ int ObLobPersistInsertSingleRowIter::get_next_row(ObNewRow *&row) return ret; } -int ObLobPersistDeleteSingleRowIter::init(ObLobAccessParam *param, ObNewRow *row) +int ObLobPersistDeleteSingleRowIter::init(ObLobAccessParam *param, blocksstable::ObDatumRow *row) { int ret = OB_SUCCESS; if (OB_ISNULL(param) || OB_ISNULL(row)) { @@ -143,7 +143,7 @@ int ObLobPersistDeleteSingleRowIter::init(ObLobAccessParam *param, ObNewRow *row return ret; } -int ObLobPersistDeleteSingleRowIter::get_next_row(ObNewRow *&row) +int ObLobPersistDeleteSingleRowIter::get_next_row(blocksstable::ObDatumRow *&row) { int ret = OB_SUCCESS; if (OB_ISNULL(param_) || OB_ISNULL(row_)) { @@ -166,6 +166,8 @@ int ObLobPersistInsertIter::init(ObLobAccessParam *param, ObLobMetaWriteIter *me if (OB_ISNULL(param) || OB_ISNULL(meta_iter)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("param or meta_iter is null", K(ret), KP(param), KP(meta_iter)); + } else if (OB_FAIL(new_row_.init(ObLobMetaUtil::LOB_META_COLUMN_CNT))) { + LOG_WARN("init new datum row failed", K(ret)); } else { param_ = param; meta_iter_ = meta_iter; @@ -173,7 +175,7 @@ int ObLobPersistInsertIter::init(ObLobAccessParam *param, ObLobMetaWriteIter *me return ret; } -int ObLobPersistInsertIter::get_next_row(ObNewRow *&row) +int ObLobPersistInsertIter::get_next_row(blocksstable::ObDatumRow *&row) { int ret = OB_SUCCESS; if (OB_FAIL(meta_iter_->get_next_row(result_))) { @@ -188,7 +190,7 @@ int ObLobPersistInsertIter::get_next_row(ObNewRow *&row) } else if (OB_FAIL(inc_lob_size(result_.info_))) { LOG_WARN("inc_lob_size fail", K(ret)); } else { - ObPersistentLobApator::set_lob_meta_row(row_cell_, new_row_, result_.info_); + ObPersistentLobApator::set_lob_meta_row(new_row_, result_.info_); row = &new_row_; } return ret; @@ -200,6 +202,8 @@ int ObLobPersistDeleteIter::init(ObLobAccessParam *param, ObLobMetaScanIter *met if (OB_ISNULL(param) || OB_ISNULL(meta_iter)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("param or meta_iter is null", K(ret), KP(param), KP(meta_iter)); + } else if (OB_FAIL(new_row_.init(ObLobMetaUtil::LOB_META_COLUMN_CNT))) { + LOG_WARN("init new datum row failed", K(ret)); } else { param_ = param; meta_iter_ = meta_iter; @@ -207,7 +211,7 @@ int ObLobPersistDeleteIter::init(ObLobAccessParam *param, ObLobMetaScanIter *met return ret; } -int ObLobPersistDeleteIter::get_next_row(ObNewRow *&row) +int ObLobPersistDeleteIter::get_next_row(blocksstable::ObDatumRow *&row) { int ret = OB_SUCCESS; if (OB_FAIL(meta_iter_->get_next_row(result_))) { @@ -221,7 +225,7 @@ int ObLobPersistDeleteIter::get_next_row(ObNewRow *&row) } else if (OB_FAIL(dec_lob_size(result_.info_))) { LOG_WARN("dec_lob_size fail", K(ret)); } else { - ObPersistentLobApator::set_lob_meta_row(row_cell_, new_row_, result_.info_); + ObPersistentLobApator::set_lob_meta_row(new_row_, result_.info_); row = &new_row_; } return ret; diff --git a/src/storage/lob/ob_lob_persistent_iterator.h b/src/storage/lob/ob_lob_persistent_iterator.h index d3be04c16..19598dfdf 100644 --- a/src/storage/lob/ob_lob_persistent_iterator.h +++ b/src/storage/lob/ob_lob_persistent_iterator.h @@ -12,19 +12,22 @@ #ifndef OCEABASE_STORAGE_OB_LOB_PERSISTENT_ITERATOR_ #define OCEABASE_STORAGE_OB_LOB_PERSISTENT_ITERATOR_ +#include "storage/blocksstable/ob_datum_row_iterator.h" +#include "storage/lob/ob_lob_util.h" +#include "storage/lob/ob_lob_meta.h" namespace oceanbase { namespace storage { -class ObLobPersistWriteIter : public ObNewRowIterator +class ObLobPersistWriteIter : public blocksstable::ObDatumRowIterator { public: ObLobPersistWriteIter(): param_(nullptr) {} virtual ~ObLobPersistWriteIter() {} - virtual int get_next_row() override { return OB_NOT_IMPLEMENT; } + virtual int get_next_row(blocksstable::ObDatumRow *&row) override { return OB_NOT_IMPLEMENT; } protected: int update_seq_no(); @@ -44,10 +47,10 @@ public: row_(nullptr), iter_end_(false) {} - int init(ObLobAccessParam *param, ObNewRow *row); + int init(ObLobAccessParam *param, blocksstable::ObDatumRow *row); virtual ~ObLobPersistInsertSingleRowIter() {} - virtual int get_next_row(ObNewRow *&row); + virtual int get_next_row(blocksstable::ObDatumRow *&row); virtual void reset() { iter_end_ = false; } private: @@ -55,7 +58,7 @@ private: DISALLOW_COPY_AND_ASSIGN(ObLobPersistInsertSingleRowIter); private: // data members - ObNewRow *row_; + blocksstable::ObDatumRow *row_; bool iter_end_; }; @@ -67,10 +70,10 @@ public: row_(nullptr), iter_end_(false) {} - int init(ObLobAccessParam *param, ObNewRow *row); + int init(ObLobAccessParam *param, blocksstable::ObDatumRow *row); virtual ~ObLobPersistDeleteSingleRowIter() {} - virtual int get_next_row(ObNewRow *&row); + virtual int get_next_row(blocksstable::ObDatumRow *&row); virtual void reset() { iter_end_ = false; } private: @@ -78,7 +81,7 @@ private: DISALLOW_COPY_AND_ASSIGN(ObLobPersistDeleteSingleRowIter); private: // data members - ObNewRow *row_; + blocksstable::ObDatumRow *row_; bool iter_end_; }; @@ -94,9 +97,9 @@ public: virtual ~ObLobPersistUpdateSingleRowIter() {} - int init(ObLobAccessParam *param, ObNewRow *old_row, ObNewRow *new_row); + int init(ObLobAccessParam *param, blocksstable::ObDatumRow *old_row, blocksstable::ObDatumRow *new_row); - virtual int get_next_row(ObNewRow *&row) override; + virtual int get_next_row(blocksstable::ObDatumRow *&row) override; virtual void reset() override {} private: @@ -104,8 +107,8 @@ private: DISALLOW_COPY_AND_ASSIGN(ObLobPersistUpdateSingleRowIter); private: - ObNewRow *old_row_; - ObNewRow *new_row_; + blocksstable::ObDatumRow *old_row_; + blocksstable::ObDatumRow *new_row_; bool got_old_row_; bool is_iter_end_; }; @@ -114,10 +117,10 @@ private: class ObLobPersistInsertIter: public ObLobPersistWriteIter { public: - ObLobPersistInsertIter() : meta_iter_(nullptr), new_row_(), row_cell_(), result_() {} + ObLobPersistInsertIter() : meta_iter_(nullptr), new_row_(), result_() {} int init(ObLobAccessParam *param, ObLobMetaWriteIter *meta_iter); virtual ~ObLobPersistInsertIter() {} - virtual int get_next_row(ObNewRow *&row); + virtual int get_next_row(blocksstable::ObDatumRow *&row); virtual void reset() { new_row_.reset(); } private: @@ -126,8 +129,7 @@ private: private: // data members ObLobMetaWriteIter *meta_iter_; - ObNewRow new_row_; - ObObj row_cell_[ObLobMetaUtil::LOB_META_COLUMN_CNT]; + blocksstable::ObDatumRow new_row_; ObLobMetaWriteResult result_; }; @@ -135,10 +137,10 @@ private: class ObLobPersistDeleteIter: public ObLobPersistWriteIter { public: - ObLobPersistDeleteIter() : meta_iter_(nullptr), new_row_(), row_cell_(), result_() {} + ObLobPersistDeleteIter() : meta_iter_(nullptr), new_row_(), result_() {} int init(ObLobAccessParam *param, ObLobMetaScanIter *meta_iter); virtual ~ObLobPersistDeleteIter() {} - virtual int get_next_row(ObNewRow *&row); + virtual int get_next_row(blocksstable::ObDatumRow *&row); virtual void reset() { new_row_.reset(); } @@ -148,8 +150,7 @@ private: private: // data members ObLobMetaScanIter *meta_iter_; - ObNewRow new_row_; - ObObj row_cell_[ObLobMetaUtil::LOB_META_COLUMN_CNT]; + blocksstable::ObDatumRow new_row_; ObLobMetaScanResult result_; }; diff --git a/src/storage/ls/ob_ls_tablet_service.cpp b/src/storage/ls/ob_ls_tablet_service.cpp index 2612b1c53..46f9083b8 100644 --- a/src/storage/ls/ob_ls_tablet_service.cpp +++ b/src/storage/ls/ob_ls_tablet_service.cpp @@ -29,8 +29,11 @@ #include "share/schema/ob_table_dml_param.h" #include "share/schema/ob_tenant_schema_service.h" #include "share/ob_ddl_common.h" +#include "sql/das/ob_das_update_op.h" #include "storage/blocksstable/index_block/ob_index_block_builder.h" #include "storage/blocksstable/ob_sstable_meta.h" +#include "storage/blocksstable/ob_datum_row_store.h" +#include "storage/blocksstable/ob_datum_row_utils.h" #include "storage/ob_dml_running_ctx.h" #include "storage/ob_partition_range_spliter.h" #include "storage/ob_query_iterator_factory.h" @@ -2609,14 +2612,14 @@ int ObLSTabletService::insert_rows( ObStoreCtx &ctx, const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows) { int ret = OB_SUCCESS; + NG_TRACE(S_insert_rows_begin); int64_t afct_num = 0; int64_t dup_num = 0; - if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not inited", K(ret), K_(is_inited)); @@ -2647,20 +2650,16 @@ int ObLSTabletService::insert_rows( dml_param, ctx.mvcc_acc_ctx_.mem_ctx_->get_query_allocator(), ObDmlFlag::DF_INSERT); - ObIAllocator &work_allocator = run_ctx.allocator_; - void *ptr = nullptr; - ObStoreRow *tbl_rows = nullptr; int64_t row_count = 0; - int64_t row_buf_cnt = 0; - ObNewRow *rows = nullptr; + ObDatumRow *rows = nullptr; if (OB_FAIL(prepare_dml_running_ctx(&column_ids, nullptr, tablet_handle, run_ctx))) { LOG_WARN("failed to prepare dml running ctx", K(ret)); } else { ObTabletHandle tmp_handle; SMART_VAR(ObRowsInfo, rows_info) { const ObRelativeTable &data_table = run_ctx.relative_table_; + const ObColDescIArray &col_descs = *(run_ctx.col_descs_); while (OB_SUCC(ret) && OB_SUCC(get_next_rows(row_iter, rows, row_count))) { - ObStoreRow reserved_row; // need to be called just after get_next_row to ensure that previous row's LOB memoroy is valid if get_next_row accesses it dml_param.lob_allocator_.reuse(); // Let ObStorageTableGuard refresh retired memtable, should not hold origin tablet handle @@ -2668,7 +2667,7 @@ int ObLSTabletService::insert_rows( if (tmp_handle.get_obj() != run_ctx.relative_table_.tablet_iter_.get_tablet_handle().get_obj()) { tmp_handle = run_ctx.relative_table_.tablet_iter_.get_tablet_handle(); rows_info.reset(); - if (OB_FAIL(rows_info.init(data_table, ctx, tmp_handle.get_obj()->get_rowkey_read_info()))) { + if (OB_FAIL(rows_info.init(col_descs, data_table, ctx, tmp_handle.get_obj()->get_rowkey_read_info()))) { LOG_WARN("Failed to init rows info", K(ret), K(data_table)); } } @@ -2676,34 +2675,15 @@ int ObLSTabletService::insert_rows( } else if (row_count <= 0) { ret = OB_ERR_UNEXPECTED; LOG_WARN("row_count should be greater than 0", K(ret)); - } else if (1 == row_count) { - tbl_rows = &reserved_row; - tbl_rows[0].flag_.set_flag(ObDmlFlag::DF_INSERT); - } else if (row_buf_cnt < row_count) { - if (nullptr != ptr) { - work_allocator.free(ptr); - ptr = nullptr; + } else { + for (int64_t i = 0; i < row_count; i++) { + rows[i].row_flag_.set_flag(ObDmlFlag::DF_INSERT); } - if (OB_ISNULL(ptr = work_allocator.alloc(row_count * sizeof(ObStoreRow)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_ERROR("fail to allocate memory", K(ret), K(row_count)); - } else { - row_buf_cnt = row_count; - tbl_rows = new (ptr) ObStoreRow[row_count]; - for (int64_t i = 0; i < row_count; i++) { - tbl_rows[i].flag_.set_flag(ObDmlFlag::DF_INSERT); - } - } - } else if (tbl_rows != static_cast(ptr)) { - tbl_rows = static_cast(ptr); } if (OB_FAIL(ret)) { - } else if (OB_ISNULL(tbl_rows)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, tbl_rows is NULL", K(ret), KP(tbl_rows)); } else if (OB_FAIL(insert_rows_to_tablet(tablet_handle, run_ctx, rows, - row_count, rows_info, tbl_rows, afct_num, dup_num))) { + row_count, rows_info, afct_num, dup_num))) { LOG_WARN("insert to each tablets fail", K(ret)); } } @@ -2715,12 +2695,9 @@ int ObLSTabletService::insert_rows( if (OB_ITER_END == ret) { ret = OB_SUCCESS; } - if (nullptr != ptr) { - work_allocator.free(ptr); - } } if (OB_SUCC(ret)) { - LOG_DEBUG("succeeded to insert rows", K(ret)); + LOG_DEBUG("succeeded to insert rows", K(ret), K(afct_num)); affected_rows = afct_num; EVENT_ADD(STORAGE_INSERT_ROW_COUNT, afct_num); } @@ -2735,7 +2712,7 @@ int ObLSTabletService::direct_insert_rows( const int64_t ddl_task_id, const ObTabletID &tablet_id, const ObIArray &column_ids, - ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows) { int ret = OB_SUCCESS; @@ -2745,7 +2722,7 @@ int ObLSTabletService::direct_insert_rows( LOG_WARN("fail to get table ctx", KR(ret), K(key)); } else { int64_t row_count = 0; - ObNewRow *rows = nullptr; + ObDatumRow *rows = nullptr; table::ObTableLoadTransId trans_id; trans_id.segment_id_ = px_task_id; trans_id.trans_gid_ = 1; @@ -2765,7 +2742,7 @@ int ObLSTabletService::direct_insert_rows( LOG_WARN("row_count should be greater than 0", K(ret)); } else { for (int64_t i = 0; OB_SUCC(ret) && (i < row_count); ++i) { - const ObNewRow &row = rows[i]; + const ObDatumRow &row = rows[i]; if (OB_FAIL(writer.write(row))) { LOG_WARN("fail to write", KR(ret), K(i), K(row)); } else { @@ -2786,7 +2763,34 @@ int ObLSTabletService::direct_insert_rows( return ret; } -int ObLSTabletService::mock_duplicated_rows_(common::ObNewRowIterator *&duplicated_rows) +int ObLSTabletService::get_storage_row( + const ObDatumRow &sql_row, + const ObIArray &column_ids, + ObSingleRowGetter &row_getter, + ObDMLRunningCtx &run_ctx, + ObDatumRow *&out_row, + bool use_fuse_row_cache) +{ + int ret = OB_SUCCESS; + ObDatumRowkey datum_rowkey; + ObDatumRowkeyHelper rowkey_helper; + ObRelativeTable &data_table = run_ctx.relative_table_; + const ObColDescIArray &column_descs = *run_ctx.col_descs_; + if (OB_FAIL(rowkey_helper.prepare_datum_rowkey(sql_row, data_table.get_rowkey_column_num(), column_descs, datum_rowkey))) { + LOG_WARN("failed to prepare rowkey", K(ret), K(sql_row), K(column_descs)); + } else if (OB_FAIL(init_single_row_getter(row_getter, run_ctx, column_ids, data_table, true))) { + LOG_WARN("failed to init single row getter", K(ret), K(column_ids)); + } else if (OB_FAIL(row_getter.open(datum_rowkey, use_fuse_row_cache))) { + LOG_WARN("failed to open storage row", K(ret), K(datum_rowkey)); + } else if (OB_FAIL(row_getter.get_next_row(out_row))) { + if (OB_ITER_END != ret) { + LOG_WARN("failed to get single storage row", K(ret), K(sql_row)); + } + } + return ret; +} + +int ObLSTabletService::mock_duplicated_rows_(blocksstable::ObDatumRowIterator *&duplicated_rows) { int ret = OB_SUCCESS; ObValueRowIterator *dup_iter = NULL; @@ -2812,10 +2816,10 @@ int ObLSTabletService::insert_row( const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, const common::ObIArray &duplicated_column_ids, - const common::ObNewRow &row, + blocksstable::ObDatumRow &row, const ObInsertFlag flag, int64_t &affected_rows, - common::ObNewRowIterator *&duplicated_rows) + blocksstable::ObDatumRowIterator *&duplicated_rows) { int ret = OB_SUCCESS; const ObTabletID &data_tablet_id = ctx.tablet_id_; @@ -2840,18 +2844,17 @@ int ObLSTabletService::insert_row( ctx.mvcc_acc_ctx_.mem_ctx_->get_query_allocator(), ObDmlFlag::DF_INSERT); ObIAllocator &work_allocator = run_ctx.allocator_; - ObStoreRow &tbl_row = run_ctx.tbl_row_; + duplicated_rows = nullptr; const ObRelativeTable &data_table = run_ctx.relative_table_; if (OB_FAIL(prepare_dml_running_ctx(&column_ids, nullptr, tablet_handle, run_ctx))) { LOG_WARN("failed to prepare dml running ctx", K(ret)); } else { - tbl_row.flag_.set_flag(ObDmlFlag::DF_INSERT); - tbl_row.row_val_ = row; + row.row_flag_.set_flag(ObDmlFlag::DF_INSERT); const bool check_exist = !data_table.is_storage_index_table() || data_table.is_unique_index(); if (OB_FAIL(insert_row_to_tablet(check_exist, tablet_handle, run_ctx, - tbl_row))) { + row))) { if (OB_ERR_PRIMARY_KEY_DUPLICATE == ret) { int tmp_ret = OB_SUCCESS; // For primary key conflicts caused by concurrent insertions within @@ -2863,7 +2866,7 @@ int ObLSTabletService::insert_row( run_ctx, flag, duplicated_column_ids, - tbl_row.row_val_, + row, duplicated_rows))) { LOG_WARN("failed to get conflict row(s)", K(ret), K(duplicated_column_ids), K(row)); ret = tmp_ret; @@ -2906,7 +2909,7 @@ int ObLSTabletService::update_rows( const ObDMLBaseParam &dml_param, const ObIArray &column_ids, const ObIArray< uint64_t> &updated_column_ids, - ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows) { int ret = OB_SUCCESS; @@ -2934,14 +2937,13 @@ int ObLSTabletService::update_rows( ObDMLRunningCtx run_ctx(ctx, dml_param, ctx.mvcc_acc_ctx_.mem_ctx_->get_query_allocator(), - ObDmlFlag::DF_UPDATE); - ObIAllocator &work_allocator = run_ctx.allocator_; - ObStoreRow old_tbl_row; - void *old_row_cells = nullptr; - ObStoreRow &new_tbl_row = run_ctx.tbl_row_; + ObDmlFlag::DF_UPDATE, + true /* is_need_row_datum_utils */); + ObDatumRow old_datum_row; + ObDatumRow &new_datum_row = run_ctx.datum_row_; bool rowkey_change = false; UpdateIndexArray update_idx; - ObRowStore row_store; + ObDatumRowStore row_store; bool delay_new = false; bool lob_update = false; const ObRelativeTable &relative_table = run_ctx.relative_table_; @@ -2961,16 +2963,10 @@ int ObLSTabletService::update_rows( LOG_WARN("failed to check rowkey changes", K(ret)); } else { timeguard.click("Check"); - const int64_t num = relative_table.get_column_count(); - if (OB_ISNULL(old_row_cells = work_allocator.alloc( - static_cast(sizeof(common::ObObj) * num)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_ERROR("failed to malloc temp row cells", K(ret)); + if (OB_FAIL(old_datum_row.init(run_ctx.allocator_, relative_table.get_column_count()))) { + LOG_ERROR("failed to init old datum row", K(ret), K(relative_table.get_column_count())); } else { timeguard.click("AllocOld"); - old_tbl_row.row_val_.cells_ = new (old_row_cells) ObObj[num](); - old_tbl_row.flag_.set_flag(ObDmlFlag::DF_UPDATE); - new_tbl_row.flag_.set_flag(ObDmlFlag::DF_UPDATE); } lob_update = is_lob_update(run_ctx, update_idx); } @@ -2978,9 +2974,9 @@ int ObLSTabletService::update_rows( int64_t cur_time = 0; ObTabletHandle tmp_handle; while (OB_SUCC(ret) - && OB_SUCC(get_next_row_from_iter(row_iter, old_tbl_row, true)) - && OB_SUCC(get_next_row_from_iter(row_iter, new_tbl_row, false))) { - LOG_DEBUG("get_dml_update_row", KP(row_iter), K(old_tbl_row), K(new_tbl_row)); + && OB_SUCC(get_next_row_from_iter(row_iter, old_datum_row, true)) + && OB_SUCC(get_next_row_from_iter(row_iter, new_datum_row, false))) { + LOG_DEBUG("get_dml_update_row", KP(row_iter), K(old_datum_row), K(new_datum_row)); // need to be called just after get_next_row to ensure that previous row's LOB memoroy is valid if get_next_row accesses it dml_param.lob_allocator_.reuse(); // Let ObStorageTableGuard refresh retired memtable, should not hold origin tablet handle @@ -3001,11 +2997,11 @@ int ObLSTabletService::update_rows( update_idx, delay_new, lob_update, - old_tbl_row, - new_tbl_row, + old_datum_row, + new_datum_row, &row_store, duplicate))) { - LOG_WARN("failed to update row to tablets", K(ret), K(old_tbl_row), K(new_tbl_row)); + LOG_WARN("failed to update row to tablets", K(ret), K(old_datum_row), K(new_datum_row)); } else if (duplicate) { dup_num++; } else { @@ -3018,20 +3014,18 @@ int ObLSTabletService::update_rows( } if (OB_SUCC(ret) && row_store.get_row_count() > 0) { - void *ptr1 = nullptr; - const int64_t num = relative_table.get_column_count(); - if (OB_ISNULL(ptr1 = work_allocator.alloc(sizeof(common::ObObj) * num))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_ERROR("failed to malloc temp row cells", K(ret)); + new_datum_row.storage_datums_ = nullptr; + if (OB_FAIL(new_datum_row.init(run_ctx.allocator_, relative_table.get_column_count()))) { + LOG_ERROR("failed to init new datum row", K(ret), K(relative_table.get_column_count())); } else { timeguard.click("AllocNew"); - new_tbl_row.row_val_.cells_ = new(ptr1) ObObj[num](); - ObRowStore::Iterator row_iter2 = row_store.begin(); + ObDatumRowStore::Iterator row_iter2 = row_store.begin(); ObTabletHandle tmp_handle; - // when total_quantity_log is true, we should iterate new_tbl_row and old_tbl_row, and + const share::schema::ObTableDMLParam *table_param = dml_param.table_param_; + // when total_quantity_log is true, we should iterate new_datum_row and old_datum_row, and // dispose these two rows together, otherwise, when total_quantity_log is false, - // row_iter2 doesn't contain old rows, and old_tbl_row is a dummy param in process_new_row - while (OB_SUCC(ret) && OB_SUCC(row_iter2.get_next_row(new_tbl_row.row_val_))) { + // row_iter2 doesn't contain old rows, and old_datum_row is a dummy param in process_new_row + while (OB_SUCC(ret) && OB_SUCC(row_iter2.get_next_row(new_datum_row))) { // Let ObStorageTableGuard refresh retired memtable, should not hold origin tablet handle // outside the while loop. if (tmp_handle.get_obj() != run_ctx.relative_table_.tablet_iter_.get_tablet_handle().get_obj()) { @@ -3039,38 +3033,37 @@ int ObLSTabletService::update_rows( } int64_t data_tbl_rowkey_len = relative_table.get_rowkey_column_num(); bool tbl_rowkey_change = false; - if (OB_FAIL(row_iter2.get_next_row(old_tbl_row.row_val_))) { - LOG_WARN("fail to get row from row stores", K(ret)); + if (OB_FAIL(row_iter2.get_next_row(old_datum_row))) { + LOG_WARN("fail to get old row from row stores", K(ret)); } else if (FALSE_IT(timeguard.click("GetNext"))) { - } else if (rowkey_change && OB_FAIL(check_rowkey_value_change(old_tbl_row.row_val_, - new_tbl_row.row_val_, + } else if (OB_ISNULL(table_param)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get invalid table param", K(ret)); + } else if (rowkey_change && OB_FAIL(check_rowkey_value_change(old_datum_row, + new_datum_row, data_tbl_rowkey_len, + table_param->get_data_table().get_read_info().get_datum_utils(), tbl_rowkey_change))) { - LOG_WARN("check data table rowkey change failed", K(ret), K(old_tbl_row), - K(new_tbl_row), K(data_tbl_rowkey_len)); + LOG_WARN("check data table rowkey change failed", K(ret), K(old_datum_row), + K(new_datum_row), K(data_tbl_rowkey_len)); } else if (OB_FAIL(process_new_row(tmp_handle, run_ctx, update_idx, - old_tbl_row, - new_tbl_row, + old_datum_row, + new_datum_row, tbl_rowkey_change))) { - LOG_WARN("fail to process new row", K(ret), K(old_tbl_row), K(new_tbl_row)); + LOG_WARN("fail to process new row", K(ret), K(old_datum_row), K(new_datum_row)); } timeguard.click("Process"); } - work_allocator.free(ptr1); - ptr1 = nullptr; - new_tbl_row.row_val_.cells_ = nullptr; + new_datum_row.reset(); } } if (OB_ITER_END == ret) { ret = OB_SUCCESS; } - if (nullptr != old_row_cells) { - work_allocator.free(old_row_cells); - } - old_tbl_row.row_val_.cells_ = nullptr; + old_datum_row.reset(); if (OB_SUCC(ret)) { affected_rows = afct_num; EVENT_ADD(STORAGE_UPDATE_ROW_COUNT, afct_num); @@ -3089,7 +3082,7 @@ int ObLSTabletService::put_rows( ObStoreCtx &ctx, const ObDMLBaseParam &dml_param, const ObIArray &column_ids, - ObNewRowIterator *row_iter, + ObDatumRowIterator *row_iter, int64_t &affected_rows) { int ret = OB_SUCCESS; @@ -3112,16 +3105,12 @@ int ObLSTabletService::put_rows( dml_param, ctx.mvcc_acc_ctx_.mem_ctx_->get_query_allocator(), ObDmlFlag::DF_UPDATE); - ObIAllocator &work_allocator = run_ctx.allocator_; - ObNewRow *row = nullptr; - ObStoreRow &tbl_row = run_ctx.tbl_row_; - const ObRelativeTable &data_table = run_ctx.relative_table_; + ObDatumRow *row = nullptr; if (OB_FAIL(prepare_dml_running_ctx(&column_ids, nullptr, tablet_handle, run_ctx))) { LOG_WARN("failed to prepare dml running ctx", K(ret)); } else { tablet_handle.reset(); - tbl_row.flag_.set_flag(ObDmlFlag::DF_UPDATE); } int64_t cur_time = 0; @@ -3135,14 +3124,14 @@ int ObLSTabletService::put_rows( tmp_handle = run_ctx.relative_table_.tablet_iter_.get_tablet_handle(); } cur_time = ObClockGenerator::getClock(); - tbl_row.row_val_ = *row; + row->row_flag_ = ObDmlFlag::DF_UPDATE; if (cur_time > dml_param.timeout_) { ret = OB_TIMEOUT; LOG_WARN("query timeout", K(ret), K(cur_time), K(dml_param)); } else if (OB_FAIL(insert_row_to_tablet(false/*check_exist*/, tmp_handle, run_ctx, - tbl_row))) { + *row))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) { LOG_WARN("failed to write row", K(ret)); } @@ -3167,7 +3156,7 @@ int ObLSTabletService::delete_rows( ObStoreCtx &ctx, const ObDMLBaseParam &dml_param, const ObIArray &column_ids, - ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows) { int ret = OB_SUCCESS; @@ -3189,8 +3178,9 @@ int ObLSTabletService::delete_rows( ObDMLRunningCtx run_ctx(ctx, dml_param, ctx.mvcc_acc_ctx_.mem_ctx_->get_query_allocator(), - ObDmlFlag::DF_DELETE); - ObNewRow *row = nullptr; + ObDmlFlag::DF_DELETE, + true /* is_need_row_datum_utils */); + ObDatumRow *row = nullptr; if (OB_FAIL(prepare_dml_running_ctx(&column_ids, nullptr, tablet_handle, run_ctx))) { LOG_WARN("failed to prepare dml running ctx", K(ret)); } else { @@ -3239,7 +3229,7 @@ int ObLSTabletService::lock_rows( const ObDMLBaseParam &dml_param, const ObLockFlag lock_flag, const bool is_sfu, - ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows) { UNUSEDx(lock_flag, is_sfu); @@ -3266,7 +3256,7 @@ int ObLSTabletService::lock_rows( dml_param, ctx.mvcc_acc_ctx_.mem_ctx_->get_query_allocator(), ObDmlFlag::DF_LOCK); - ObNewRow *row = nullptr; + ObDatumRow *row = nullptr; if (OB_FAIL(prepare_dml_running_ctx(nullptr, nullptr, tablet_handle, run_ctx))) { LOG_WARN("failed to prepare dml running ctx", K(ret)); } else if (FALSE_IT(tablet_handle.reset())) { @@ -3278,6 +3268,7 @@ int ObLSTabletService::lock_rows( } else { timeguard.click("GetIds"); run_ctx.column_ids_ = &column_ids; + run_ctx.col_descs_ = &col_desc; ObTabletHandle tmp_handle; while (OB_SUCCESS == ret && OB_SUCC(row_iter->get_next_row(row))) { // Let ObStorageTableGuard refresh retired memtable, should not hold origin tablet handle @@ -3286,19 +3277,20 @@ int ObLSTabletService::lock_rows( tmp_handle = run_ctx.relative_table_.tablet_iter_.get_tablet_handle(); } ObRelativeTable &relative_table = run_ctx.relative_table_; + const ObStorageDatumUtils &datum_utils = dml_param.table_param_->get_data_table().get_read_info().get_datum_utils(); bool is_exists = true; if (ObTimeUtility::current_time() > dml_param.timeout_) { ret = OB_TIMEOUT; int64_t cur_time = ObClockGenerator::getClock(); LOG_WARN("query timeout", K(cur_time), K(dml_param), K(ret)); } else if (GCONF.enable_defensive_check() - && OB_FAIL(check_old_row_legitimacy(tmp_handle, run_ctx, *row))) { + && OB_FAIL(check_old_row_legitimacy(datum_utils.get_cmp_funcs(), tmp_handle, run_ctx, *row))) { LOG_WARN("check row legitimacy failed", K(ret), KPC(row)); } else if (GCONF.enable_defensive_check() - && OB_FAIL(check_new_row_nullable_value(col_desc, relative_table, *row))) { + && OB_FAIL(check_datum_row_nullable_value(col_desc, relative_table, *row))) { LOG_WARN("check lock row nullable failed", K(ret)); } else if (FALSE_IT(timeguard.click("Check"))) { - } else if (OB_FAIL(tmp_handle.get_obj()->lock_row(run_ctx.relative_table_, ctx, *row))) { + } else if (OB_FAIL(tmp_handle.get_obj()->lock_row(run_ctx.relative_table_, ctx, col_desc, *row))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret) { LOG_WARN("failed to lock row", K(*row), K(ret)); } @@ -3321,7 +3313,7 @@ int ObLSTabletService::lock_row( ObTabletHandle &tablet_handle, ObStoreCtx &ctx, const ObDMLBaseParam &dml_param, - const ObNewRow &row, + blocksstable::ObDatumRow &row, const ObLockFlag lock_flag, const bool is_sfu) { @@ -3352,7 +3344,7 @@ int ObLSTabletService::lock_row( ret = OB_TIMEOUT; int64_t cur_time = ObClockGenerator::getClock(); LOG_WARN("query timeout", K(cur_time), K(dml_param), K(ret)); - } else if (OB_FAIL(tablet_handle.get_obj()->lock_row(run_ctx.relative_table_, ctx, row))) { + } else if (OB_FAIL(tablet_handle.get_obj()->lock_row(run_ctx.relative_table_, ctx, col_desc, row))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret) { LOG_WARN("failed to lock row", K(row), K(ret)); } @@ -3655,80 +3647,11 @@ int ObLSTabletService::safe_create_cas_empty_shell( return ret; } -int ObLSTabletService::need_check_old_row_legitimacy(ObDMLRunningCtx &run_ctx, - bool &need_check, - bool &is_udf) -{ - int ret = OB_SUCCESS; - ObRelativeTable &data_table = run_ctx.relative_table_; - need_check = false; - const ObDMLBaseParam &dml_param = run_ctx.dml_param_; - is_udf = false; - // TODO(jingxing): setting this to true - if (OB_FAIL(data_table.has_udf_column(need_check))) { - LOG_WARN("check has udf column failed", K(ret)); - } else if (need_check) { - is_udf = true; - ObTableStoreIterator &table_iter = *data_table.tablet_iter_.table_iter(); - while (OB_SUCC(ret) && !need_check) { - ObITable *table_ptr = nullptr; - if (OB_FAIL(table_iter.get_next(table_ptr))) { - if (OB_ITER_END != ret) { - LOG_WARN("get next table failed", K(ret)); - } - } else if (OB_ISNULL(table_ptr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("error unexpected, table ptr must not be nullptr", K(ret)); - } else { - need_check = table_ptr->is_major_sstable(); - } - } - } else if (dml_param.is_batch_stmt_ && !data_table.is_index_table()) { - //batch stmt execution dependency defensive check to check - //if the same row was modified multiple times - need_check = true; - ret = OB_E(EventTable::EN_INS_MULTI_VALUES_BATCH_OPT) OB_SUCCESS; - // no need to check old row, just for bmsql performance optimization - // TODO yuchen.ywc - if (OB_SUCCESS != ret) { - LOG_INFO("error sim when current statement is batch update", K(ret), K(is_udf)); - need_check = false; - ret = OB_SUCCESS; - } - } else if (GCONF.enable_defensive_check()) { - need_check = true; - if (data_table.is_index_table() && !data_table.can_read_index()) { - //index can not be read during building index, so does not check old index row - need_check = false; - } - if (ObDmlFlag::DF_LOCK == run_ctx.dml_flag_) { - need_check = false; - } - } - return ret; -} - -int ObLSTabletService::construct_table_rows( - const ObNewRow *rows, - ObStoreRow *tbl_rows, - int64_t row_count) -{ - int ret = OB_SUCCESS; - if (row_count <= 0) { - ret = OB_ERR_WRONG_VALUE_COUNT_ON_ROW; - LOG_WARN("row count should be bigger than 0", K(row_count), K(ret)); - } else { - for (int64_t i = 0; i < row_count; i++) { - tbl_rows[i].row_val_ = rows[i]; - } - } - return ret; -} - int ObLSTabletService::check_old_row_legitimacy( + const ObStoreCmpFuncs &cmp_funcs, ObTabletHandle &data_tablet_handle, ObDMLRunningCtx &run_ctx, - const common::ObNewRow &old_row) + blocksstable::ObDatumRow &old_row) { int ret = OB_SUCCESS; // usage: @@ -3739,56 +3662,41 @@ int ObLSTabletService::check_old_row_legitimacy( ret = OB_ERR_DEFENSIVE_CHECK; } ObRelativeTable &data_table = run_ctx.relative_table_; - ObStoreRowkey rowkey; - bool need_check = false; - bool is_udf = false; - rowkey.assign(old_row.cells_, data_table.get_rowkey_column_num()); if (OB_FAIL(ret)) { - } else if (OB_UNLIKELY(rowkey.get_obj_cnt() > old_row.count_) || OB_ISNULL(run_ctx.column_ids_)) { + } else if (OB_UNLIKELY(data_table.get_rowkey_column_num() > old_row.count_) || OB_ISNULL(run_ctx.column_ids_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("old row is invalid", K(ret), K(old_row), K(rowkey.get_obj_cnt()), KP(run_ctx.column_ids_)); - } else if (OB_FAIL(need_check_old_row_legitimacy(run_ctx, need_check, is_udf))) { - LOG_WARN("identify need check old row legitimacy", K(ret)); - } else if (need_check) { + LOG_WARN("old row is invalid", K(ret), K(old_row), K(data_table.get_rowkey_column_num()), KP(run_ctx.column_ids_)); + } else if (run_ctx.is_need_check_old_row_) { //the vertical partition is no longer maintained, //and the defense check skips the vertical partition function - const ObDMLBaseParam &dml_param = run_ctx.dml_param_; ObArenaAllocator scan_allocator(common::ObMemAttr(MTL_ID(), ObModIds::OB_TABLE_SCAN_ITER)); - ObIAllocator *allocator = &scan_allocator; - ObSingleRowGetter old_row_getter(*allocator, *data_tablet_handle.get_obj()); - ObNewRow *storage_old_row = nullptr; - //check old row whether different with SQL.old_row, - ObDatumRowkey datum_rowkey; - ObDatumRowkeyHelper rowkey_helper; + ObSingleRowGetter storage_row_getter(scan_allocator, *data_tablet_handle.get_obj()); + ObDatumRow *storage_old_row = nullptr; const ObIArray &column_ids = *run_ctx.column_ids_; + const ObColDescIArray &column_descs = *run_ctx.col_descs_; uint64_t err_col_id = OB_INVALID_ID; - if (OB_FAIL(rowkey_helper.convert_datum_rowkey(rowkey.get_rowkey(), datum_rowkey))) { - STORAGE_LOG(WARN, "Failed to transfer datum rowkey", K(ret), K(rowkey)); - } else if (OB_FAIL(init_single_row_getter(old_row_getter, run_ctx, column_ids, data_table, true))) { - LOG_WARN("failed to init single row getter", K(ret)); - } else if (OB_FAIL(old_row_getter.open(datum_rowkey, true))) { - LOG_WARN("open old row getter failed", K(ret), K(rowkey)); - } else if (OB_FAIL(old_row_getter.get_next_row(storage_old_row))) { + if (OB_FAIL(get_storage_row(old_row, column_ids, storage_row_getter, run_ctx, storage_old_row, true))) { if (OB_ITER_END == ret) { ret = OB_ERR_DEFENSIVE_CHECK; - FLOG_WARN("old row in storage is not exists", K(ret)); + FLOG_WARN("old row in storage is not exists", K(ret), K(old_row)); } else { - LOG_WARN("get next row from old_row_iter failed", K(ret), KPC(run_ctx.column_ids_), K(old_row)); + LOG_WARN("get next row from old_row_getter failed", K(ret), K(column_ids), K(old_row)); } } else if (OB_ISNULL(storage_old_row)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected error, storage old row is NULL", K(ret)); - } else if (storage_old_row->get_count() != old_row.get_count()) { + } else if (storage_old_row->count_ != old_row.count_) { ret = OB_ERR_DEFENSIVE_CHECK; FLOG_WARN("storage old row is not matched with sql old row", K(ret)); } else { - for (int64_t i = 0; OB_SUCC(ret) && i < old_row.get_count(); ++i) { - const ObObj &storage_val = storage_old_row->get_cell(i); - const ObObj &sql_val = old_row.get_cell(i); - int cmp = 0; - if (sql_val.is_lob_storage()) { + for (int64_t i = 0; OB_SUCC(ret) && i < old_row.count_; ++i) { + const ObStorageDatum &storage_val = storage_old_row->storage_datums_[i]; + const ObStorageDatum &sql_val = old_row.storage_datums_[i]; + const ObObjMeta &sql_meta = column_descs.at(i).col_type_; + int cmp_ret = 0; + if (sql_meta.is_lob_storage()) { // skip all text and lob - } else if (OB_UNLIKELY(ObLongTextType == storage_val.get_type() && sql_val.is_lob_locator())) { + } else if (sql_meta.is_lob_locator()) { //skip check lob column type when do the normal sql write check } else if (OB_UNLIKELY(storage_val.is_nop_value())) { bool is_nop = false; @@ -3803,16 +3711,16 @@ int ObLSTabletService::check_old_row_legitimacy( } else if (sql_val.is_nop_value()) { //this column is nop val, means that this column does not be touched by DML //just ignore it - } else if (OB_FAIL(storage_val.compare(sql_val, cmp)) || 0 != cmp) { + } else if (OB_FAIL(cmp_funcs.at(i).compare(storage_val, sql_val, cmp_ret)) || 0 != cmp_ret) { ret = OB_ERR_DEFENSIVE_CHECK; err_col_id = column_ids.at(i); FLOG_WARN("storage_val is not equal with sql_val, maybe catch a bug", K(ret), - K(storage_val), K(sql_val), K(cmp), K(column_ids.at(i))); + K(storage_val), K(sql_val), K(column_ids.at(i)), K(cmp_ret)); } } } - if (OB_ERR_DEFENSIVE_CHECK == ret && dml_param.is_batch_stmt_) { + if (OB_ERR_DEFENSIVE_CHECK == ret && run_ctx.dml_param_.is_batch_stmt_) { // 批量删除的时候可能索引表的删除在主表前边，所以所有的表在batch删除的时候出现4377，都可能是重复删导致的 ret = OB_BATCHED_MULTI_STMT_ROLLBACK; LOG_TRACE("batch stmt execution has a correctness error, needs rollback", K(ret), @@ -3836,7 +3744,7 @@ int ObLSTabletService::check_old_row_legitimacy( } else if (OB_TMP_FAIL(check_parts_tx_state_in_transfer_for_4377_(run_ctx.store_ctx_.mvcc_acc_ctx_.tx_desc_))) { ret = tmp_ret; LOG_WARN("check need rollback in transfer for 4377 found exception", K(ret), K(old_row), K(data_table)); - } else if (is_udf) { + } else if (run_ctx.is_udf_) { ret = OB_ERR_INDEX_KEY_NOT_FOUND; LOG_WARN("index key not found on udf column", K(ret), K(old_row)); } else if (data_table.is_index_table() && OB_TMP_FAIL(check_is_gencol_check_failed(data_table, err_col_id, is_virtual_gen_col))) { @@ -3868,6 +3776,7 @@ int ObLSTabletService::check_old_row_legitimacy( } } } + return ret; } @@ -3934,7 +3843,7 @@ int ObLSTabletService::check_is_gencol_check_failed(const ObRelativeTable &data_ int ObLSTabletService::check_new_row_legitimacy( ObDMLRunningCtx &run_ctx, - const common::ObNewRow &new_row) + const blocksstable::ObDatumRow &datum_row) { int ret = OB_SUCCESS; ObRelativeTable &data_table = run_ctx.relative_table_; @@ -3942,12 +3851,13 @@ int ObLSTabletService::check_new_row_legitimacy( if (OB_ISNULL(run_ctx.column_ids_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("column ids is nullptr", K(ret)); - } else if (OB_FAIL(check_new_row_nullable_value(*run_ctx.column_ids_, data_table, new_row))) { - LOG_WARN("check new row nullable value failed", K(ret), + } else if (OB_FAIL(check_datum_row_nullable_value(*run_ctx.col_descs_, data_table, datum_row))) { + LOG_WARN("check datum row nullable value failed", K(ret), "dml_param", run_ctx.dml_param_, "dml_type", run_ctx.dml_flag_); - } else if (OB_FAIL(check_new_row_shadow_pk(*run_ctx.column_ids_, data_table, new_row))) { - LOG_WARN("check new row nullable value failed", K(ret), + } else if (OB_FAIL(check_datum_row_shadow_pk(*run_ctx.column_ids_, data_table, datum_row, + run_ctx.dml_param_.table_param_->get_data_table().get_read_info().get_datum_utils()))) { + LOG_WARN("check datum row nullable value failed", K(ret), "dml_param", run_ctx.dml_param_, "dml_type", run_ctx.dml_flag_); } @@ -3957,10 +3867,9 @@ int ObLSTabletService::check_new_row_legitimacy( int ObLSTabletService::insert_rows_to_tablet( ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, - const ObNewRow * const rows, + blocksstable::ObDatumRow *rows, const int64_t row_count, ObRowsInfo &rows_info, - ObStoreRow *tbl_rows, int64_t &afct_num, int64_t &dup_num) { @@ -3973,9 +3882,7 @@ int ObLSTabletService::insert_rows_to_tablet( ret = OB_TIMEOUT; int64_t cur_time = ObClockGenerator::getClock(); LOG_WARN("query timeout", K(cur_time), K(dml_param), K(ret)); - } else if (OB_FAIL(construct_table_rows(rows, tbl_rows, row_count))) { - LOG_WARN("fail to construct table rows", K(ret)); - } else if (OB_FAIL(rows_info.check_duplicate(tbl_rows, row_count, data_table))) { + } else if (OB_FAIL(rows_info.check_duplicate(rows, row_count, data_table))) { if (OB_ERR_PRIMARY_KEY_DUPLICATE == ret) { char rowkey_buffer[OB_TMP_BUF_SIZE_256]; if (OB_SUCCESS != extract_rowkey(data_table, @@ -3998,9 +3905,9 @@ int ObLSTabletService::insert_rows_to_tablet( } else { LOG_WARN("fail to check duplicate", K(ret)); } - } else if (OB_FAIL(insert_lob_tablet_rows(tablet_handle, run_ctx, tbl_rows, row_count))) { + } else if (OB_FAIL(insert_lob_tablet_rows(tablet_handle, run_ctx, rows, row_count))) { LOG_WARN("failed to insert rows to lob tablet", K(ret)); - } else if (OB_FAIL(insert_tablet_rows(row_count, tablet_handle, run_ctx, tbl_rows, rows_info))) { + } else if (OB_FAIL(insert_tablet_rows(row_count, tablet_handle, run_ctx, rows, rows_info))) { LOG_WARN("failed to insert rows to data tablet", K(ret)); } else { afct_num = afct_num + row_count; @@ -4012,7 +3919,7 @@ int ObLSTabletService::insert_tablet_rows( const int64_t row_count, ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, - ObStoreRow *rows, + ObDatumRow *rows, ObRowsInfo &rows_info) { int ret = OB_SUCCESS; @@ -4022,9 +3929,9 @@ int ObLSTabletService::insert_tablet_rows( // 1. Defensive checking of new rows. if (GCONF.enable_defensive_check()) { for (int64_t i = 0; OB_SUCC(ret) && i < row_count; i++) { - ObStoreRow &tbl_row = rows[i]; - if (OB_FAIL(check_new_row_legitimacy(run_ctx, tbl_row.row_val_))) { - LOG_WARN("Failed to check new row legitimacy", K(ret), K_(tbl_row.row_val)); + ObDatumRow &datum_row = rows[i]; + if (OB_FAIL(check_new_row_legitimacy(run_ctx, datum_row))) { + LOG_WARN("Failed to check new row legitimacy", K(ret), K(datum_row)); } } } @@ -4082,7 +3989,7 @@ int ObLSTabletService::insert_tablet_rows( int ObLSTabletService::insert_lob_col( ObDMLRunningCtx &run_ctx, const ObColDesc &column, - ObObj &obj, + blocksstable::ObStorageDatum &datum, ObLobAccessParam *del_param, ObLobCommon *lob_common) { @@ -4092,7 +3999,7 @@ int ObLSTabletService::insert_lob_col( if (OB_ISNULL(lob_mngr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("[STORAGE_LOB]failed to get lob manager handle.", K(ret)); - } else if (!column.col_type_.is_lob_storage() || obj.is_nop_value() || obj.is_null()) { + } else if (!column.col_type_.is_lob_storage() || datum.is_nop_value() || datum.is_null()) { // do nothing } else { // init lob access param @@ -4126,10 +4033,10 @@ int ObLSTabletService::insert_lob_col( lob_param.scan_backward_ = false; lob_param.offset_ = 0; // Notice: currently only inrow data - ObString raw_data = obj.get_string(); + ObString raw_data = datum.get_string(); ObString data; // for not strict sql mode, will insert empty string without lob header - bool has_lob_header = obj.has_lob_header() && raw_data.length() > 0; + bool has_lob_header = datum.has_lob_header() && raw_data.length() > 0; ObLobLocatorV2 loc(raw_data, has_lob_header); if (OB_FAIL(set_lob_storage_params(run_ctx, column, lob_param))) { LOG_WARN("set_lob_storage_params fail", K(ret), K(column)); @@ -4137,7 +4044,7 @@ int ObLSTabletService::insert_lob_col( LOG_WARN("[STORAGE_LOB]lob append failed.", K(ret)); } else { ObLobCommon *res_lob_common = lob_param.lob_common_; - obj.set_lob_value(obj.get_type(), res_lob_common, lob_param.handle_size_); + datum.set_lob_data(*res_lob_common, lob_param.handle_size_); LOG_DEBUG("[STORAGE_LOB]write ob lob data.", K(lob_param), KPC(res_lob_common), K(lob_param.handle_size_), K(column.col_type_.get_collation_type())); } @@ -4148,7 +4055,7 @@ int ObLSTabletService::insert_lob_col( int ObLSTabletService::insert_lob_tablet_row( ObTabletHandle &data_tablet, ObDMLRunningCtx &run_ctx, - ObStoreRow &row) + blocksstable::ObDatumRow &datum_row) { int ret = OB_SUCCESS; int64_t col_cnt = run_ctx.col_descs_->count(); @@ -4156,9 +4063,9 @@ int ObLSTabletService::insert_lob_tablet_row( if (OB_ISNULL(lob_mngr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("[STORAGE_LOB]failed to get lob manager handle.", K(ret)); - } else if (row.row_val_.count_ != col_cnt) { + } else if (datum_row.count_ != col_cnt) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("[STORAGE_LOB]column count invalid", K(ret), K(col_cnt), K(row.row_val_.count_), KPC(run_ctx.col_descs_)); + LOG_WARN("[STORAGE_LOB]column count invalid", K(ret), K(col_cnt), K(datum_row.count_), KPC(run_ctx.col_descs_)); } else { const int64_t cur_time = ObClockGenerator::getClock(); const int64_t relative_timeout = run_ctx.dml_param_.timeout_ - cur_time; @@ -4169,12 +4076,12 @@ int ObLSTabletService::insert_lob_tablet_row( for (int64_t i = 0; OB_SUCC(ret) && i < col_cnt; ++i) { const ObColDesc &column = run_ctx.col_descs_->at(i); - ObObj &obj = row.row_val_.get_cell(i); - if (obj.is_null() || obj.is_nop_value()) { + ObStorageDatum &datum = datum_row.storage_datums_[i]; + if (datum.is_null() || datum.is_nop_value()) { // do nothing } else if (column.col_type_.is_lob_storage()) { - if (OB_FAIL(insert_lob_col(run_ctx, column, obj, nullptr, nullptr))) { - LOG_WARN("[STORAGE_LOB]failed to insert lob col.", K(ret), K(row), K(i)); + if (OB_FAIL(insert_lob_col(run_ctx, column, datum, nullptr, nullptr))) { + LOG_WARN("[STORAGE_LOB]failed to insert lob col.", K(ret), K(datum_row), K(i)); } } } @@ -4212,7 +4119,7 @@ int update_lob_meta_table_seq_no(ObDMLRunningCtx &run_ctx, int64_t row_count) int ObLSTabletService::insert_lob_tablet_rows( ObTabletHandle &data_tablet, ObDMLRunningCtx &run_ctx, - ObStoreRow *rows, + blocksstable::ObDatumRow *rows, int64_t row_count) { int ret = OB_SUCCESS; @@ -4300,8 +4207,8 @@ int ObLSTabletService::extract_rowkey( } int ObLSTabletService::get_next_rows( - ObNewRowIterator *row_iter, - ObNewRow *&rows, + blocksstable::ObDatumRowIterator *row_iter, + blocksstable::ObDatumRow *&rows, int64_t &row_count) { return row_iter->get_next_rows(rows, row_count); @@ -4405,9 +4312,10 @@ int ObLSTabletService::check_rowkey_change( } int ObLSTabletService::check_rowkey_value_change( - const common::ObNewRow &old_row, - const common::ObNewRow &new_row, + const ObDatumRow &old_row, + const ObDatumRow &new_row, const int64_t rowkey_len, + const blocksstable::ObStorageDatumUtils &rowkey_datum_utils, bool &rowkey_change) { int ret = OB_SUCCESS; @@ -4419,8 +4327,11 @@ int ObLSTabletService::check_rowkey_value_change( K(rowkey_len), K(ret)); } else { rowkey_change = false; - for (int64_t i = 0; !rowkey_change && i < rowkey_len; ++i) { - if (old_row.cells_[i] != new_row.cells_[i]) { + for (int i = 0; OB_SUCC(ret) && !rowkey_change && i < rowkey_len; ++i) { + int cmp_ret = 0; + if (OB_FAIL(rowkey_datum_utils.get_cmp_funcs().at(i).compare(old_row.storage_datums_[i], new_row.storage_datums_[i], cmp_ret))) { + LOG_WARN("compare old datum and new datum failed", K(old_row.storage_datums_[i]), K(new_row.storage_datums_[i])); + } else if (0 != cmp_ret) { rowkey_change = true; } } @@ -4432,9 +4343,9 @@ int ObLSTabletService::check_rowkey_value_change( int ObLSTabletService::process_delta_lob( ObDMLRunningCtx &run_ctx, const ObColDesc &column, - ObObj &old_obj, + ObStorageDatum &old_datum, ObLobLocatorV2 &delta_lob, - ObObj &obj) + ObStorageDatum &datum) { int ret = OB_SUCCESS; ObLobManager *lob_mngr = MTL(ObLobManager*); @@ -4468,8 +4379,7 @@ int ObLSTabletService::process_delta_lob( ObString old_disk_lob; if (OB_FAIL(set_lob_storage_params(run_ctx, column, lob_param))) { LOG_WARN("set_lob_storage_params fail", K(ret), K(column)); - } else if (OB_FAIL(old_obj.get_lob_locatorv2(old_lob))) { - LOG_WARN("get old lob locator failed.", K(ret), K(old_obj)); + } else if (FALSE_IT(old_datum.get_mem_lob(old_lob))) { } else if (!old_lob.is_valid()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("old lob locator is invalid.", K(ret)); @@ -4487,9 +4397,9 @@ int ObLSTabletService::process_delta_lob( LOG_WARN("failed to process delta lob.", K(ret), K(lob_param), K(delta_lob)); } else { // update obj with new disk locator - obj.set_lob_value(obj.get_type(), lob_param.lob_common_, lob_param.handle_size_); + datum.set_lob_data(*(lob_param.lob_common_), lob_param.handle_size_); if (! lob_param.ext_info_log_.is_null() - && OB_FAIL(register_ext_info_commit_cb(run_ctx, obj, lob_param.ext_info_log_))) { + && OB_FAIL(register_ext_info_commit_cb(run_ctx, datum, column.col_type_.get_type(), lob_param.ext_info_log_))) { LOG_WARN("register_ext_info_commit_cb fail", K(ret), K(lob_param)); } } @@ -4500,7 +4410,8 @@ int ObLSTabletService::process_delta_lob( int ObLSTabletService::register_ext_info_commit_cb( ObDMLRunningCtx &run_ctx, - ObObj &col_data, + ObStorageDatum &col_data, + ObObjType type, ObObj &ext_info_data) { int ret = OB_SUCCESS; @@ -4516,8 +4427,9 @@ int ObLSTabletService::register_ext_info_commit_cb( run_ctx.store_ctx_.mvcc_acc_ctx_.tx_desc_, run_ctx.store_ctx_.mvcc_acc_ctx_.tx_scn_, col_data, + type, ext_info_data))) { - LOG_WARN("register_ext_info_commit_cb fail", K(ret), K(run_ctx.store_ctx_), K(col_data), K(ext_info_data)); + LOG_WARN("register_ext_info_commit_cb fail", K(ret), K(run_ctx.store_ctx_), K(col_data), K(type), K(ext_info_data)); } return ret; } @@ -4548,15 +4460,14 @@ int ObLSTabletService::process_lob_row( ObDMLRunningCtx &run_ctx, const ObIArray &update_idx, bool data_tbl_rowkey_change, - ObStoreRow &old_sql_row, - ObStoreRow &old_row, - ObStoreRow &new_row) + ObDatumRow &old_row, + ObDatumRow &new_row) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(old_row.row_val_.get_count() != new_row.row_val_.get_count())) { + if (OB_UNLIKELY(old_row.count_ != new_row.count_)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("[STORAGE_LOB]invalid args", K(old_row), K(new_row), K(ret)); - } else if (OB_UNLIKELY(old_row.row_val_.get_count() != run_ctx.col_descs_->count())) { + } else if (OB_UNLIKELY(old_row.count_ != run_ctx.col_descs_->count())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("[STORAGE_LOB]invalid args", K(old_row), K(new_row), KPC(run_ctx.col_descs_)); } else if (OB_FAIL(update_lob_meta_table_seq_no(run_ctx, 1/*row_count*/))) { @@ -4569,11 +4480,10 @@ int ObLSTabletService::process_lob_row( LOG_WARN("timeout has reached", K(ret), "timeout", run_ctx.dml_param_.timeout_, K(cur_time)); } - for (int64_t i = 0; OB_SUCC(ret) && i < old_row.row_val_.get_count(); ++i) { + for (int64_t i = 0; OB_SUCC(ret) && i < old_row.count_; ++i) { if (run_ctx.col_descs_->at(i).col_type_.is_lob_storage()) { - ObObj &old_obj = old_row.row_val_.get_cell(i); - ObObj &old_sql_obj = old_sql_row.row_val_.get_cell(i); - ObObj &new_obj = new_row.row_val_.get_cell(i); + ObStorageDatum &old_datum = old_row.storage_datums_[i]; + ObStorageDatum &new_datum = new_row.storage_datums_[i]; bool is_update = false; for (int64_t j = 0; !is_update && j < update_idx.count(); ++j) { if (update_idx.at(j) == i) { @@ -4582,14 +4492,14 @@ int ObLSTabletService::process_lob_row( } if (is_update) { // get new lob locator - ObString new_lob_str = (new_obj.is_null() || new_obj.is_nop_value()) - ? ObString(0, nullptr) : new_obj.get_string(); + ObString new_lob_str = (new_datum.is_null() || new_datum.is_nop_value()) + ? ObString(0, nullptr) : new_datum.get_string(); // for not strict sql mode, will insert empty string without lob header - bool has_lob_header = new_obj.has_lob_header() && new_lob_str.length() > 0; + bool has_lob_header = new_datum.has_lob_header() && new_lob_str.length() > 0; ObLobLocatorV2 new_lob(new_lob_str, has_lob_header); if (OB_FAIL(ret)) { - } else if (new_obj.is_null() || - new_obj.is_nop_value() || + } else if (new_datum.is_null() || + new_datum.is_nop_value() || new_lob.is_full_temp_lob() || new_lob.is_persist_lob() || (new_lob.is_lob_disk_locator() && new_lob.has_inrow_data())) { @@ -4597,47 +4507,42 @@ int ObLSTabletService::process_lob_row( ObLobAccessParam lob_param; if (OB_FAIL(new_lob.get_lob_data_byte_len(lob_param.update_len_))) { LOG_WARN("fail to get new lob byte len", K(ret), K(new_lob), K(i)); - } else if (OB_FAIL(delete_lob_col(run_ctx, run_ctx.col_descs_->at(i), old_obj, old_sql_obj, lob_common, lob_param))) { - LOG_WARN("[STORAGE_LOB]failed to erase old lob col", K(ret), K(old_sql_row), K(old_row), K(i)); - } else if (OB_FAIL(insert_lob_col(run_ctx, run_ctx.col_descs_->at(i), new_obj, &lob_param, lob_common))) { + } else if (OB_FAIL(delete_lob_col(run_ctx, run_ctx.col_descs_->at(i), old_datum, lob_common, lob_param))) { + LOG_WARN("[STORAGE_LOB]failed to erase old lob col", K(ret), K(old_row), K(i)); + } else if (OB_FAIL(insert_lob_col(run_ctx, run_ctx.col_descs_->at(i), new_datum, &lob_param, lob_common))) { LOG_WARN("[STORAGE_LOB]failed to insert new lob col.", K(ret), K(new_row), K(i)); } } else if (new_lob.is_delta_temp_lob()) { - if (OB_FAIL(process_delta_lob(run_ctx, run_ctx.col_descs_->at(i), old_sql_obj, new_lob, new_obj))) { + if (OB_FAIL(process_delta_lob(run_ctx, run_ctx.col_descs_->at(i), old_datum, new_lob, new_datum))) { LOG_WARN("failed to process delta lob.", K(ret), K(i)); } } else { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected obj for new lob", K(ret), K(i), K(new_obj), K(new_lob)); + LOG_WARN("unexpected obj for new lob", K(ret), K(i), K(new_datum), K(new_lob)); } } else { - if (old_obj.is_null()) { - new_obj.set_null(); - } else if (old_obj.is_nop_value()) { - new_obj.set_nop_value(); - } else if (new_obj.is_nop_value() || new_obj.is_null()) { + if (old_datum.is_null()) { + new_datum.set_null(); + } else if (old_datum.is_nop_value()) { + new_datum.set_nop(); + } else if (new_datum.is_nop_value() || new_datum.is_null()) { // do nothing } else { - if (old_obj.is_null()) { - new_obj.set_null(); - } else if (old_obj.is_nop_value()) { - new_obj.set_nop_value(); - } else { - ObString val_str = old_obj.get_string(); - ObLobCommon *lob_common = reinterpret_cast(val_str.ptr()); - if (!lob_common->in_row_ && data_tbl_rowkey_change) { - ObLobAccessParam lob_param; - if (val_str.length() < ObLobManager::LOB_WITH_OUTROW_CTX_SIZE) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("not enough space for lob header", K(ret), K(val_str), K(i)); - } else if (OB_FAIL(delete_lob_col(run_ctx, run_ctx.col_descs_->at(i), old_obj, old_sql_obj, lob_common, lob_param))) { - LOG_WARN("[STORAGE_LOB]failed to erase old lob col", K(ret), K(old_sql_row), K(old_row), K(i)); - } else if (OB_FAIL(insert_lob_col(run_ctx, run_ctx.col_descs_->at(i), new_obj, nullptr, nullptr))) { // no need del_param - LOG_WARN("[STORAGE_LOB]failed to insert new lob col.", K(ret), K(new_row), K(i)); - } - } else { - new_obj.set_lob_value(new_obj.get_type(), val_str.ptr(), val_str.length()); // remove has lob header flag + ObString val_str = old_datum.get_string(); + ObLobCommon *lob_common = reinterpret_cast(val_str.ptr()); + if (!lob_common->in_row_ && data_tbl_rowkey_change) { + ObLobAccessParam lob_param; + if (val_str.length() < ObLobManager::LOB_WITH_OUTROW_CTX_SIZE) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("not enough space for lob header", K(ret), K(val_str), K(i)); + } else if (OB_FAIL(delete_lob_col(run_ctx, run_ctx.col_descs_->at(i), old_datum, lob_common, lob_param))) { + LOG_WARN("[STORAGE_LOB]failed to erase old lob col", K(ret), K(old_row), K(i)); + } else if (OB_FAIL(insert_lob_col(run_ctx, run_ctx.col_descs_->at(i), new_datum, nullptr, nullptr))) { // no need del_param + LOG_WARN("[STORAGE_LOB]failed to insert new lob col.", K(ret), K(new_row), K(i)); } + } else { + new_datum.reuse(); + new_datum.set_string(val_str.ptr(), val_str.length()); // remove has lob header flag } } } @@ -4654,9 +4559,9 @@ int ObLSTabletService::update_row_to_tablet( const ObIArray &update_idx, const bool delay_new, const bool lob_update, - ObStoreRow &old_tbl_row, - ObStoreRow &new_tbl_row, - ObRowStore *row_store, + ObDatumRow &old_datum_row, + ObDatumRow &new_datum_row, + ObDatumRowStore *row_store, bool &duplicate) { int ret = OB_SUCCESS; @@ -4665,38 +4570,37 @@ int ObLSTabletService::update_row_to_tablet( bool data_tbl_rowkey_change = false; int64_t data_tbl_rowkey_len = run_ctx.relative_table_.get_rowkey_column_num(); ObSQLMode sql_mode = dml_param.sql_mode_; + const share::schema::ObTableDMLParam *table_param = dml_param.table_param_; duplicate = false; - ObStoreRow old_sql_tbl_row; - old_sql_tbl_row.row_val_ = old_tbl_row.row_val_; - if (OB_UNLIKELY(col_descs.count() != old_tbl_row.row_val_.get_count() - || col_descs.count() != new_tbl_row.row_val_.get_count())) { + if (OB_UNLIKELY(col_descs.count() != old_datum_row.count_ + || col_descs.count() != new_datum_row.count_ || OB_ISNULL(table_param))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", K(ret), K(col_descs.count()), - K(old_tbl_row.row_val_), K(new_tbl_row.row_val_)); - } else if (rowkey_change && OB_FAIL(check_rowkey_value_change(old_tbl_row.row_val_, - new_tbl_row.row_val_, + K(old_datum_row), K(new_datum_row)); + } else if (rowkey_change && OB_FAIL(check_rowkey_value_change(old_datum_row, + new_datum_row, data_tbl_rowkey_len, + table_param->get_data_table().get_read_info().get_datum_utils(), data_tbl_rowkey_change))) { LOG_WARN("failed to check data table rowkey change", K(ret), - K(old_tbl_row), K(new_tbl_row), K(data_tbl_rowkey_len)); + K(old_datum_row), K(new_datum_row), K(data_tbl_rowkey_len)); } else if (OB_FAIL(process_old_row(tablet_handle, run_ctx, data_tbl_rowkey_change, lob_update, - old_tbl_row))) { + old_datum_row))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) { - LOG_WARN("fail to process old row", K(ret), K(*run_ctx.col_descs_), - K(old_tbl_row), K(data_tbl_rowkey_change)); + LOG_WARN("fail to process old row", K(ret), K(col_descs), + K(old_datum_row), K(data_tbl_rowkey_change)); } } else if (OB_FAIL(process_lob_row(tablet_handle, run_ctx, update_idx, data_tbl_rowkey_change, - old_sql_tbl_row, - old_tbl_row, - new_tbl_row))) { - LOG_WARN("failed to process lob col change", K(ret), K(old_tbl_row), K(new_tbl_row)); + old_datum_row, + new_datum_row))) { + LOG_WARN("failed to process lob col change", K(ret), K(old_datum_row), K(new_datum_row)); } else if (delay_new && lib::is_oracle_mode()) { // if total quantity log is needed, we should cache both new row and old row, // and the sequence is new_row1, old_row1, new_row2, old_row2...., @@ -4704,21 +4608,21 @@ int ObLSTabletService::update_row_to_tablet( if (OB_ISNULL(row_store)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("row_store is NULL", K(ret)); - } else if (OB_FAIL(row_store->add_row(new_tbl_row.row_val_))) { - LOG_WARN("failed to store new row", K(new_tbl_row), K(ret)); - } else if (OB_FAIL(row_store->add_row(old_tbl_row.row_val_))) { - LOG_WARN("failed to store old row", K(old_tbl_row), K(ret)); + } else if (OB_FAIL(row_store->add_row(new_datum_row))) { + LOG_WARN("failed to store new row", K(new_datum_row), K(ret)); + } else if (OB_FAIL(row_store->add_row(old_datum_row))) { + LOG_WARN("failed to store old row", K(old_datum_row), K(ret)); } else { - LOG_DEBUG("add row store for delay new", K(old_tbl_row), K(new_tbl_row)); + LOG_DEBUG("add row store for delay new", K(old_datum_row), K(new_datum_row)); } } else if (OB_FAIL(process_new_row(tablet_handle, run_ctx, update_idx, - old_tbl_row, - new_tbl_row, + old_datum_row, + new_datum_row, data_tbl_rowkey_change))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) { - LOG_WARN("fail to process new row", K(new_tbl_row), K(ret)); + LOG_WARN("fail to process new row", K(new_datum_row), K(ret)); } } @@ -4730,7 +4634,7 @@ int ObLSTabletService::process_old_row( ObDMLRunningCtx &run_ctx, const bool data_tbl_rowkey_change, const bool lob_update, - ObStoreRow &tbl_row) + ObDatumRow &datum_row) { int ret = OB_SUCCESS; ObStoreCtx &store_ctx = run_ctx.store_ctx_; @@ -4742,68 +4646,66 @@ int ObLSTabletService::process_old_row( } else if (OB_UNLIKELY(!store_ctx.is_valid() || nullptr == run_ctx.col_descs_ || run_ctx.col_descs_->count() <= 0 - || !tbl_row.is_valid())) { + || !datum_row.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(store_ctx), KP(run_ctx.col_descs_), K(tbl_row), K(is_delete_total_quantity_log)); - } else if (OB_FAIL(check_old_row_legitimacy(tablet_handle, run_ctx, tbl_row.row_val_))) { + LOG_WARN("invalid args", K(ret), K(store_ctx), KP(run_ctx.col_descs_), K(datum_row), K(is_delete_total_quantity_log)); + } else if (OB_FAIL(check_old_row_legitimacy(run_ctx.cmp_funcs_, tablet_handle, run_ctx, datum_row))) { if (OB_ERR_DEFENSIVE_CHECK == ret) { - dump_diag_info_for_old_row_loss(relative_table, store_ctx, tbl_row); + dump_diag_info_for_old_row_loss(run_ctx, datum_row); } - LOG_WARN("check old row legitimacy failed", K(tbl_row.row_val_)); - } else if (OB_FAIL(process_old_row_lob_col(tablet_handle, run_ctx, tbl_row))){ - LOG_WARN("failed to process old row lob col", K(ret), K(tbl_row)); + LOG_WARN("check old row legitimacy failed", K(ret), K(datum_row)); + } else if (OB_FAIL(process_old_row_lob_col(tablet_handle, run_ctx, datum_row))){ + LOG_WARN("failed to process old row lob col", K(ret), K(datum_row)); } else { ObColDescIArray &col_descs = const_cast(*run_ctx.col_descs_); const uint64_t &table_id = relative_table.get_table_id(); int64_t rowkey_size = relative_table.get_rowkey_column_num(); - ObStoreRowkey store_rowkey; ObDatumRowkey datum_rowkey; ObDatumRowkeyHelper rowkey_helper(run_ctx.allocator_); - if (OB_UNLIKELY(run_ctx.dml_param_.prelock_)) { bool locked = false; - if (OB_FAIL(store_rowkey.assign(tbl_row.row_val_.cells_, rowkey_size))) { - LOG_WARN("Failed to assign rowkey", K(ret), K(tbl_row), K(rowkey_size)); - } else if (OB_FAIL(rowkey_helper.convert_datum_rowkey(store_rowkey.get_rowkey(), datum_rowkey))) { - STORAGE_LOG(WARN, "Failed to transfer datum rowkey", K(ret), K(store_rowkey), K(rowkey_size), K(tbl_row)); + if (OB_FAIL(rowkey_helper.prepare_datum_rowkey(datum_row, rowkey_size, col_descs, datum_rowkey))) { + LOG_WARN("Failed to prepare rowkey", K(ret), K(datum_row), K(rowkey_size), K(datum_rowkey)); } else if (OB_FAIL(check_row_locked_by_myself(tablet_handle, relative_table, store_ctx, datum_rowkey, locked))) { - LOG_WARN("fail to check row locked", K(ret), K(tbl_row)); + LOG_WARN("fail to check row locked", K(ret), K(datum_row), K(datum_rowkey)); } else if (!locked) { ret = OB_ERR_ROW_NOT_LOCKED; - LOG_DEBUG("row has not been locked", K(ret), K(tbl_row)); + LOG_DEBUG("row has not been locked", K(ret), K(datum_row), K(datum_rowkey)); } } if (OB_FAIL(ret)) { } else if (data_tbl_rowkey_change) { - ObStoreRow del_row(tbl_row); - ObStoreRow new_tbl_row; - new_tbl_row.flag_.set_flag(ObDmlFlag::DF_DELETE); - new_tbl_row.row_val_ = tbl_row.row_val_; - del_row.flag_.set_flag(ObDmlFlag::DF_UPDATE); + ObDatumRow del_row; + ObDatumRow new_row; + ObSEArray update_idx; - if (OB_FAIL(tablet_handle.get_obj()->update_row(relative_table, + if (OB_FAIL(del_row.shallow_copy(datum_row))) { + LOG_WARN("failed to shallow copy datum row", K(ret), K(datum_row), K(del_row)); + } else if (FALSE_IT(del_row.row_flag_.set_flag(ObDmlFlag::DF_UPDATE))) { + } else if (OB_FAIL(new_row.shallow_copy(datum_row))) { + LOG_WARN("failed to shallow copy datum row", K(ret), K(datum_row), K(new_row)); + } else if (FALSE_IT(new_row.row_flag_.set_flag(ObDmlFlag::DF_DELETE))) { + } else if (OB_FAIL(tablet_handle.get_obj()->update_row(relative_table, run_ctx.store_ctx_, col_descs, update_idx, del_row, - new_tbl_row, + new_row, run_ctx.dml_param_.encrypt_meta_))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) { - LOG_WARN("failed to write data tablet row", K(ret), K(del_row), K(new_tbl_row)); + LOG_WARN("failed to write data tablet row", K(ret), K(del_row), K(new_row)); } } } else if (lob_update) { // need to lock main table rows that don't need to be deleted - if (OB_FAIL(store_rowkey.assign(tbl_row.row_val_.cells_, rowkey_size))) { - LOG_WARN("Failed to assign rowkey", K(ret), K(tbl_row), K(rowkey_size)); - } else if (OB_FAIL(rowkey_helper.convert_datum_rowkey(store_rowkey.get_rowkey(), datum_rowkey))) { - STORAGE_LOG(WARN, "Failed to transfer datum rowkey", K(ret), K(store_rowkey)); + if (OB_FAIL(rowkey_helper.prepare_datum_rowkey(datum_row, rowkey_size, col_descs, datum_rowkey))) { + LOG_WARN("Failed to prepare rowkey", K(ret), K(datum_row), K(rowkey_size)); } else if (OB_FAIL(tablet_handle.get_obj()->lock_row(relative_table, store_ctx, datum_rowkey))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) { - LOG_WARN("lock row failed", K(ret), K(table_id), K(tbl_row), K(rowkey_size)); + LOG_WARN("lock row failed", K(ret), K(table_id), K(datum_row), K(rowkey_size), K(datum_rowkey)); } } - LOG_DEBUG("generate lock node before update lob columns", K(ret), K(table_id), K(tbl_row)); + LOG_DEBUG("generate lock node before update lob columns", K(ret), K(table_id), K(datum_row), K(datum_rowkey)); } } return ret; @@ -4813,28 +4715,28 @@ int ObLSTabletService::process_new_row( ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, const common::ObIArray &update_idx, - const ObStoreRow &old_tbl_row, - const ObStoreRow &new_tbl_row, + const ObDatumRow &old_datum_row, + ObDatumRow &new_datum_row, const bool rowkey_change) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(update_idx.count() < 0 || !new_tbl_row.is_valid())) { + if (OB_UNLIKELY(update_idx.count() < 0 || !old_datum_row.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(update_idx), K(new_tbl_row), K(rowkey_change)); + LOG_WARN("invalid args", K(ret), K(update_idx), K(old_datum_row), K(rowkey_change)); } else if (GCONF.enable_defensive_check() - && OB_FAIL(check_new_row_legitimacy(run_ctx, new_tbl_row.row_val_))) { - LOG_WARN("check new row legitimacy failed", K(ret), K(new_tbl_row.row_val_)); + && OB_FAIL(check_new_row_legitimacy(run_ctx, old_datum_row))) { + LOG_WARN("check datum row legitimacy failed", K(ret), K(old_datum_row)); } else { // write full column clog needs to construct update_idx and pass to memtable if (OB_FAIL(process_data_table_row(tablet_handle, run_ctx, update_idx, - old_tbl_row, - new_tbl_row, + old_datum_row, + new_datum_row, rowkey_change))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) { LOG_WARN("fail to process data table row", K(ret), - K(update_idx), K(old_tbl_row), K(new_tbl_row), K(rowkey_change)); + K(update_idx), K(old_datum_row), K(new_datum_row), K(rowkey_change)); } } } @@ -4845,8 +4747,8 @@ int ObLSTabletService::process_data_table_row( ObTabletHandle &data_tablet, ObDMLRunningCtx &run_ctx, const ObIArray &update_idx, - const ObStoreRow &old_tbl_row, - const ObStoreRow &new_tbl_row, + const ObDatumRow &old_datum_row, + ObDatumRow &new_datum_row, const bool rowkey_change) { int ret = OB_SUCCESS; @@ -4859,34 +4761,34 @@ int ObLSTabletService::process_data_table_row( || nullptr == run_ctx.col_descs_ || run_ctx.col_descs_->count() <= 0 || update_idx.count() < 0 - || (is_update_total_quantity_log && !old_tbl_row.is_valid()) - || !new_tbl_row.is_valid())) { + || (is_update_total_quantity_log && !old_datum_row.is_valid()) + || !new_datum_row.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", K(ret), K(ctx), - KP(run_ctx.col_descs_), K(update_idx), K(old_tbl_row), K(new_tbl_row), + KP(run_ctx.col_descs_), K(update_idx), K(old_datum_row), K(new_datum_row), K(is_update_total_quantity_log), K(rowkey_change)); } else { const ObColDescIArray &col_descs = *run_ctx.col_descs_; - ObStoreRow new_row; - new_row.flag_.set_flag(rowkey_change ? ObDmlFlag::DF_INSERT : ObDmlFlag::DF_UPDATE); - new_row.row_val_ = new_tbl_row.row_val_; - + new_datum_row.row_flag_.set_flag(rowkey_change ? ObDmlFlag::DF_INSERT : ObDmlFlag::DF_UPDATE); if (!rowkey_change) { - ObStoreRow old_row; - old_row.flag_.set_flag(ObDmlFlag::DF_UPDATE); - old_row.row_val_ = old_tbl_row.row_val_; - if (!is_update_total_quantity_log) { - // For minimal mode, set pk columns of old_row to nop value, because - // they are already stored in new_row. - const int64_t rowkey_col_cnt = relative_table.get_rowkey_column_num(); - for (int64_t i = 0; i < rowkey_col_cnt; ++i) { - (old_row.row_val_.cells_[i]).set_nop_value(); + ObDatumRow old_row; + if (OB_FAIL(old_row.shallow_copy(old_datum_row))) { + LOG_WARN("failed to shallow copy datum row", K(ret), K(old_datum_row), K(old_row)); + } else { + old_row.row_flag_.set_flag(ObDmlFlag::DF_UPDATE); + if (!is_update_total_quantity_log) { + // For minimal mode, set pk columns of old_row to nop value, because + // they are already stored in new_row. + const int64_t rowkey_col_cnt = relative_table.get_rowkey_column_num(); + for (int64_t i = 0; i < rowkey_col_cnt; ++i) { + (old_row.storage_datums_[i]).set_nop(); + } } - } - if (OB_FAIL(data_tablet.get_obj()->update_row(relative_table, - ctx, col_descs, update_idx, old_row, new_row, run_ctx.dml_param_.encrypt_meta_))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) { - LOG_WARN("failed to update to row", K(ret), K(old_row), K(new_row)); + if (OB_FAIL(data_tablet.get_obj()->update_row(relative_table, + ctx, col_descs, update_idx, old_row, new_datum_row, run_ctx.dml_param_.encrypt_meta_))) { + if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) { + LOG_WARN("failed to update to row", K(ret), K(old_row), K(new_datum_row)); + } } } } else { @@ -4895,13 +4797,13 @@ int ObLSTabletService::process_data_table_row( ctx, check_exist, col_descs, - new_row, + new_datum_row, run_ctx.dml_param_.encrypt_meta_))) { if (OB_ERR_PRIMARY_KEY_DUPLICATE == ret) { char buffer[OB_TMP_BUF_SIZE_256]; - ObStoreRowkey rowkey; - if (OB_SUCCESS != rowkey.assign(new_tbl_row.row_val_.cells_, relative_table.get_rowkey_column_num())) { - LOG_WARN("Failed to assign rowkey", K(new_tbl_row)); + ObDatumRowkey rowkey; + if (OB_SUCCESS != rowkey.assign(new_datum_row.storage_datums_, relative_table.get_rowkey_column_num())) { + LOG_WARN("Failed to assign rowkey", K(new_datum_row)); } else if (OB_SUCCESS != extract_rowkey(relative_table, rowkey, buffer, OB_TMP_BUF_SIZE_256, tz_info)) { LOG_WARN("extract rowkey failed", K(rowkey)); } else { @@ -4911,9 +4813,9 @@ int ObLSTabletService::process_data_table_row( } LOG_USER_ERROR(OB_ERR_PRIMARY_KEY_DUPLICATE, buffer, index_name.length(), index_name.ptr()); } - LOG_WARN("rowkey already exists", K(ret), K(new_tbl_row)); + LOG_WARN("rowkey already exists", K(ret), K(new_datum_row)); } else if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) { - LOG_WARN("failed to update to row", K(ret), K(new_row)); + LOG_WARN("failed to update to row", K(ret), K(new_datum_row)); } } } @@ -4921,94 +4823,26 @@ int ObLSTabletService::process_data_table_row( return ret; } -int ObLSTabletService::check_new_row_nullable_value( - const ObIArray &column_ids, - ObRelativeTable &data_table, - const ObNewRow &new_row) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(column_ids.count() != new_row.get_count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("new row is invalid", K(ret), K(new_row.get_count()), K(column_ids.count())); - } - for (int64_t i = 0; OB_SUCC(ret) && i < column_ids.count(); ++i) { - uint64_t column_id = column_ids.at(i); - bool is_nullable = false; - if (OB_UNLIKELY(is_shadow_column(column_id))) { - //the shadow pk is generated internally, - //and the nullable attribute check for it is skipped - } else if (OB_FAIL(data_table.is_column_nullable_for_write(column_id, is_nullable))) { - LOG_WARN("check is_column_nullable_for_write failed", K(ret), K(column_id)); - } else if (new_row.get_cell(i).is_null() && !is_nullable) { - bool is_hidden = false; - bool is_gen_col = false; - bool is_nullable_for_read = false; - if (OB_FAIL(data_table.is_column_nullable_for_read(column_id, is_nullable_for_read))) { - LOG_WARN("check is nullable for read failed", K(ret)); - } else if (is_nullable_for_read) { - LOG_TRACE("Catch a defensive nullable error, but this column is not null novalidate", - K(column_id), K(column_ids), K(new_row), K(data_table)); - } else if (OB_FAIL(data_table.is_hidden_column(column_id, is_hidden))) { - LOG_WARN("get is hidden column failed", K(ret), K(column_id)); - } else if (OB_FAIL(data_table.is_gen_column(column_id, is_gen_col))) { - LOG_WARN("get is gen column failed", K(ret), K(column_id)); - } else if (is_hidden && !is_gen_col) { - ret = OB_BAD_NULL_ERROR; - LOG_WARN("Catch a defensive nullable error, " - "maybe cause by add column not null default null ONLINE", K(ret), - K(column_id), K(column_ids), K(new_row), K(data_table)); - } else { - ret = OB_ERR_DEFENSIVE_CHECK; - ObString func_name = ObString::make_string("check_new_row_nullable_value"); - LOG_USER_ERROR(OB_ERR_DEFENSIVE_CHECK, func_name.length(), func_name.ptr()); - LOG_ERROR_RET(OB_ERR_DEFENSIVE_CHECK, - "Fatal Error!!! Catch a defensive error!", K(ret), - K(column_id), K(column_ids), K(new_row), K(data_table)); - LOG_DBA_ERROR_V2(OB_STORAGE_DEFENSIVE_CHECK_FAIL, - OB_ERR_DEFENSIVE_CHECK, - "Fatal Error!!! Catch a defensive error!"); - } - } else if (new_row.get_cell(i).is_number()) { - number::ObNumber num; - if (OB_FAIL(new_row.get_cell(i).get_number(num))) { - LOG_WARN("get number value from object fail", K(ret), K(new_row.get_cell(i))); - } else if (OB_FAIL(num.sanity_check())) { - LOG_WARN("sanity check number failed", K(ret), K(new_row.get_cell(i))); - } - if (OB_SUCCESS != ret) { - ret = OB_ERR_DEFENSIVE_CHECK; - ObString func_name = ObString::make_string("check_new_row_nullable_value"); - LOG_USER_ERROR(OB_ERR_DEFENSIVE_CHECK, func_name.length(), func_name.ptr()); - LOG_ERROR_RET(OB_ERR_DEFENSIVE_CHECK, - "Fatal Error!!! Catch a defensive error!", K(ret), - K(column_id), K(column_ids), K(new_row), K(data_table)); - LOG_DBA_ERROR_V2(OB_STORAGE_DEFENSIVE_CHECK_FAIL, - OB_ERR_DEFENSIVE_CHECK, - "Fatal Error!!! Catch a defensive error!"); - } - } - } - return ret; -} - -int ObLSTabletService::check_new_row_nullable_value(const ObIArray &col_descs, +int ObLSTabletService::check_datum_row_nullable_value(const ObIArray &col_descs, ObRelativeTable &relative_table, - const ObNewRow &new_row) + const blocksstable::ObDatumRow &datum_row) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(col_descs.count() > new_row.get_count())) { + if (OB_UNLIKELY(col_descs.count() > datum_row.get_column_count())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("new row is invalid", K(ret), K(new_row.get_count()), K(col_descs.count())); + LOG_WARN("new row is invalid", K(ret), K(datum_row.get_column_count()), K(col_descs.count())); } for (int64_t i = 0; OB_SUCC(ret) && i < col_descs.count(); ++i) { uint64_t column_id = col_descs.at(i).col_id_; bool is_nullable = false; - if (OB_UNLIKELY(is_shadow_column(column_id))) { + if (datum_row.storage_datums_[i].is_nop()) { + //nothing + } else if (OB_UNLIKELY(is_shadow_column(column_id))) { //the shadow pk is generated internally, //and the nullable attribute check for it is skipped } else if (OB_FAIL(relative_table.is_column_nullable_for_write(column_id, is_nullable))) { LOG_WARN("check is_column_nullable_for_write failed", K(ret), K(column_id)); - } else if (new_row.get_cell(i).is_null() && !is_nullable) { + } else if (datum_row.storage_datums_[i].is_null() && !is_nullable) { bool is_hidden = false; bool is_gen_col = false; bool is_nullable_for_read = false; @@ -5018,7 +4852,7 @@ int ObLSTabletService::check_new_row_nullable_value(const ObIArray &c //this column is not null novalidate, maybe the null column come from the old data //so output trace log and ignore it LOG_TRACE("Catch a defensive nullable error, but this column is not null novalidate", - K(column_id), K(col_descs), K(new_row), K(relative_table)); + K(column_id), K(col_descs), K(datum_row), K(relative_table)); } else if (OB_FAIL(relative_table.is_hidden_column(column_id, is_hidden))) { LOG_WARN("get is hidden column failed", K(ret), K(column_id)); } else if (OB_FAIL(relative_table.is_gen_column(column_id, is_gen_col))) { @@ -5027,32 +4861,30 @@ int ObLSTabletService::check_new_row_nullable_value(const ObIArray &c ret = OB_BAD_NULL_ERROR; LOG_WARN("Catch a defensive nullable error, " "maybe cause by add column not null default null ONLINE", K(ret), - K(column_id), K(col_descs), K(new_row), K(relative_table)); + K(column_id), K(col_descs), K(datum_row), K(relative_table)); } else { ret = OB_ERR_DEFENSIVE_CHECK; - ObString func_name = ObString::make_string("check_new_row_nullable_value"); + ObString func_name = ObString::make_string("check_datum_row_nullable_value"); LOG_USER_ERROR(OB_ERR_DEFENSIVE_CHECK, func_name.length(), func_name.ptr()); LOG_ERROR_RET(OB_ERR_DEFENSIVE_CHECK, "Fatal Error!!! Catch a defensive error!", K(ret), - K(column_id), K(col_descs), K(new_row), K(relative_table)); + K(column_id), K(col_descs), K(datum_row), K(relative_table)); LOG_DBA_ERROR_V2(OB_STORAGE_DEFENSIVE_CHECK_FAIL, OB_ERR_DEFENSIVE_CHECK, "Fatal Error!!! Catch a defensive error!"); } - } else if (new_row.get_cell(i).is_number()) { - number::ObNumber num; - if (OB_FAIL(new_row.get_cell(i).get_number(num))) { - LOG_WARN("get number value from object fail", K(ret), K(new_row.get_cell(i))); - } else if (OB_FAIL(num.sanity_check())) { - LOG_WARN("sanity check number failed", K(ret), K(new_row.get_cell(i))); + } else if (!datum_row.storage_datums_[i].is_null() && col_descs.at(i).col_type_.is_number()) { + number::ObNumber num(datum_row.storage_datums_[i].get_number()); + if (OB_FAIL(num.sanity_check())) { + LOG_WARN("sanity check number failed", K(ret), K(i), K(datum_row.storage_datums_[i])); } if (OB_SUCCESS != ret) { ret = OB_ERR_DEFENSIVE_CHECK; - ObString func_name = ObString::make_string("check_new_row_nullable_value"); + ObString func_name = ObString::make_string("check_datum_row_nullable_value"); LOG_USER_ERROR(OB_ERR_DEFENSIVE_CHECK, func_name.length(), func_name.ptr()); LOG_ERROR_RET(OB_ERR_DEFENSIVE_CHECK, "Fatal Error!!! Catch a defensive error!", K(ret), - K(column_id), K(col_descs), K(new_row), K(relative_table)); + K(column_id), K(col_descs), K(datum_row), K(relative_table)); LOG_DBA_ERROR_V2(OB_STORAGE_DEFENSIVE_CHECK_FAIL, OB_ERR_DEFENSIVE_CHECK, "Fatal Error!!! Catch a defensive error!"); @@ -5062,10 +4894,11 @@ int ObLSTabletService::check_new_row_nullable_value(const ObIArray &c return ret; } -int ObLSTabletService::check_new_row_shadow_pk( +int ObLSTabletService::check_datum_row_shadow_pk( const ObIArray &column_ids, ObRelativeTable &data_table, - const ObNewRow &new_row) + const blocksstable::ObDatumRow &datum_row, + const blocksstable::ObStorageDatumUtils &rowkey_datum_utils) { int ret = OB_SUCCESS; if (data_table.get_shadow_rowkey_column_num() > 0) { @@ -5082,43 +4915,43 @@ int ObLSTabletService::check_new_row_shadow_pk( // mysql兼容：只要unique index key中有null列，则需要填充shadow列 bool rowkey_has_null = false; for (int64_t i = 0; !rowkey_has_null && i < index_col_cnt; i++) { - rowkey_has_null = new_row.get_cell(i).is_null(); + rowkey_has_null = datum_row.storage_datums_[i].is_null(); } need_spk = rowkey_has_null; } else { // oracle兼容：只有unique index key全为null列时，才需要填充shadow列 bool is_rowkey_all_null = true; for (int64_t i = 0; is_rowkey_all_null && i < index_col_cnt; i++) { - is_rowkey_all_null = new_row.get_cell(i).is_null(); + is_rowkey_all_null = datum_row.storage_datums_[i].is_null(); } need_spk = is_rowkey_all_null; } for (int64_t i = index_col_cnt; OB_SUCC(ret) && i < rowkey_cnt; ++i) { uint64_t spk_column_id = column_ids.at(i); uint64_t real_pk_id = spk_column_id - OB_MIN_SHADOW_COLUMN_ID; - const ObObj &spk_value = new_row.get_cell(i); + const ObStorageDatum &spk_value = datum_row.storage_datums_[i]; int64_t pk_idx = OB_INVALID_INDEX; int cmp = 0; if (OB_LIKELY(!need_spk)) { if (!spk_value.is_null()) { ret = OB_ERR_DEFENSIVE_CHECK; - ObString func_name = ObString::make_string("check_new_row_shadow_pk"); + ObString func_name = ObString::make_string("check_datum_row_shadow_pk"); LOG_USER_ERROR(OB_ERR_DEFENSIVE_CHECK, func_name.length(), func_name.ptr()); LOG_ERROR("Fatal Error!!! Catch a defensive error!", K(ret), - "column_id", column_ids, K(new_row), K(data_table), + "column_id", column_ids, K(datum_row), K(data_table), K(spk_value), K(i), K(spk_column_id), K(real_pk_id)); } } else if (OB_UNLIKELY(!has_exist_in_array(column_ids, real_pk_id, &pk_idx))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("real pk column not exists in column_ids", K(ret), K(column_ids), K(real_pk_id)); - } else if (OB_FAIL(new_row.get_cell(pk_idx).compare(spk_value, cmp)) || 0 != cmp) { + } else if (OB_FAIL(rowkey_datum_utils.get_cmp_funcs().at(i).compare(datum_row.storage_datums_[pk_idx], spk_value, cmp)) || 0 != cmp) { ret = OB_ERR_DEFENSIVE_CHECK; - ObString func_name = ObString::make_string("check_new_row_shadow_pk"); + ObString func_name = ObString::make_string("check_datum_row_shadow_pk"); LOG_USER_ERROR(OB_ERR_DEFENSIVE_CHECK, func_name.length(), func_name.ptr()); LOG_ERROR_RET(OB_ERR_DEFENSIVE_CHECK, "Fatal Error!!! Catch a defensive error!", K(ret), - "column_id", column_ids, K(new_row), K(data_table), - K(spk_value), "pk_value", new_row.get_cell(pk_idx), + "column_id", column_ids, K(datum_row), K(data_table), + K(spk_value), "pk_value", datum_row.storage_datums_[pk_idx], K(pk_idx), K(i), K(spk_column_id), K(real_pk_id)); LOG_DBA_ERROR_V2(OB_STORAGE_DEFENSIVE_CHECK_FAIL, OB_ERR_DEFENSIVE_CHECK, @@ -5163,32 +4996,27 @@ int ObLSTabletService::get_conflict_rows( ObDMLRunningCtx &run_ctx, const ObInsertFlag flag, const common::ObIArray &out_col_ids, - const common::ObNewRow &row, - common::ObNewRowIterator *&duplicated_rows) + blocksstable::ObDatumRow &row, + blocksstable::ObDatumRowIterator *&duplicated_rows) { TRANS_LOG(DEBUG, "get conflict rows", K(flag), K(row), K(lbt())); int ret = OB_SUCCESS; - ObRelativeTable &data_table = run_ctx.relative_table_; ObArenaAllocator scan_allocator(common::ObMemAttr(MTL_ID(), ObModIds::OB_TABLE_SCAN_ITER)); - ObIAllocator *allocator = &scan_allocator; ObTablet *data_tablet = tablet_handle.get_obj(); - ObDatumRowkeyHelper rowkey_helper(scan_allocator); - ObDatumRowkey datum_rowkey; - ObStoreRowkey rowkey; - rowkey.assign(row.cells_, data_table.get_rowkey_column_num()); + ObSingleRowGetter storage_row_getter(scan_allocator, *data_tablet); + ObDatumRow *out_row = nullptr; if (OB_ISNULL(data_tablet)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet is null", K(ret), K(tablet_handle)); - } else { - ObSingleRowGetter single_row_getter(*allocator, *data_tablet); - if (OB_FAIL(init_single_row_getter(single_row_getter, run_ctx, out_col_ids, data_table, true))) { - LOG_WARN("failed to init single row getter", K(ret)); - } else if (OB_FAIL(rowkey_helper.convert_datum_rowkey(rowkey.get_rowkey(), datum_rowkey))) { - STORAGE_LOG(WARN, "Failed to transfer datum rowkey", K(ret), K(rowkey)); - } else if (OB_FAIL(single_get_row(single_row_getter, datum_rowkey, duplicated_rows))) { - LOG_WARN("failed to single get row", K(ret)); + } else if (OB_FAIL(get_storage_row(row, out_col_ids, storage_row_getter, run_ctx, out_row))) { + if (OB_ITER_END != ret) { + LOG_WARN("get next row from single row getter failed", K(ret)); + } else { + ret = OB_SUCCESS; } + } else if (OB_FAIL(add_duplicate_row(out_row, data_tablet->get_rowkey_read_info().get_datum_utils(), duplicated_rows))) { + LOG_WARN("failed to single get row", K(ret), K(row), K(out_row)); } if (OB_FAIL(ret)) { @@ -5210,33 +5038,25 @@ int ObLSTabletService::init_single_row_getter( { int ret = OB_SUCCESS; - if (OB_FAIL(row_getter.init_dml_access_ctx(run_ctx.store_ctx_, run_ctx.dml_param_, skip_read_lob))) { - LOG_WARN("init dml access ctx failed", K(ret)); - } else if (OB_FAIL(row_getter.init_dml_access_param(relative_table, + if (OB_FAIL(row_getter.init_dml_access_param(relative_table, run_ctx.dml_param_, out_col_ids, skip_read_lob))) { LOG_WARN("init dml access param failed", K(ret)); + } else if (OB_FAIL(row_getter.prepare_cached_iter_node())) { + LOG_WARN("prepare cached iter node failed", K(ret)); + } else if (OB_FAIL(row_getter.init_dml_access_ctx(run_ctx.store_ctx_, run_ctx.dml_param_, skip_read_lob))) { + LOG_WARN("init dml access ctx failed", K(ret)); } return ret; } -int ObLSTabletService::single_get_row( - ObSingleRowGetter &row_getter, - const ObDatumRowkey &rowkey, - ObNewRowIterator *&duplicated_rows) +int ObLSTabletService::add_duplicate_row( + ObDatumRow *storage_row, + const blocksstable::ObStorageDatumUtils &rowkey_datum_utils, + blocksstable::ObDatumRowIterator *&duplicated_rows) { int ret = OB_SUCCESS; - ObNewRow *row = nullptr; - - if (OB_FAIL(row_getter.open(rowkey))) { - LOG_WARN("init single row getter failed", K(ret)); - } else if (OB_FAIL(row_getter.get_next_row(row))) { - if (OB_ITER_END != ret) { - LOG_WARN("get next row from single row getter failed", K(ret)); - } else { - ret = OB_SUCCESS; - } - } else if (NULL == duplicated_rows) { + if (NULL == duplicated_rows) { ObValueRowIterator *dup_iter = NULL; if (OB_ISNULL(dup_iter = ObQueryIteratorFactory::get_insert_dup_iter())) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -5248,35 +5068,18 @@ int ObLSTabletService::single_get_row( } } } - if (OB_SUCC(ret) && row != nullptr) { + if (OB_SUCC(ret) && storage_row != nullptr) { ObValueRowIterator *dup_iter = static_cast(duplicated_rows); - if (OB_FAIL(dup_iter->add_row(*row))) { - LOG_WARN("failed to store conflict row", K(ret), K(*row)); + if (OB_FAIL(dup_iter->add_row(*storage_row, rowkey_datum_utils))) { + LOG_WARN("failed to store conflict row", K(ret), K(*storage_row)); } else { - LOG_DEBUG("get conflict row", KPC(row)); + LOG_DEBUG("get conflict row", KPC(storage_row)); } } return ret; } -int ObLSTabletService::convert_row_to_rowkey( - ObSingleRowGetter &index_row_getter, - ObStoreRowkey &rowkey) -{ - int ret = OB_SUCCESS; - ObNewRow *row = nullptr; - if (OB_FAIL(index_row_getter.get_next_row(row))) { - if (OB_ITER_END != ret) { - LOG_WARN("get next row from index row getter failed", K(ret)); - } - } else { - rowkey.assign(row->cells_, row->count_); - } - - return ret; -} - /* this func is an encapsulation of ObNewRowIterator->get_next_row. * 1. need_copy_cells is true, perform a cells copy, but not a deep copy. * memory for store_row.row_val.cells_ has already allocated before @@ -5284,12 +5087,12 @@ int ObLSTabletService::convert_row_to_rowkey( * 2. need_copy_cells is false, just perform an assignment, no any copy behavior, */ int ObLSTabletService::get_next_row_from_iter( - ObNewRowIterator *row_iter, - ObStoreRow &store_row, + blocksstable::ObDatumRowIterator *row_iter, + ObDatumRow &datum_row, const bool need_copy_cells) { int ret = OB_SUCCESS; - ObNewRow *row = nullptr; + ObDatumRow *row = nullptr; if (OB_ISNULL(row_iter)) { ret = OB_INVALID_ARGUMENT; @@ -5300,15 +5103,16 @@ int ObLSTabletService::get_next_row_from_iter( } } else { if (need_copy_cells) { - // in this situation, store_row.row_val has already hold mem for cells_, - // no need to alloc mem here, we copy cells only. - store_row.row_val_.count_ = row->count_; + // in this situation, datum_row has already hold mem for storage_datums_, + // no need to alloc mem here, we copy storage_datums_ only. + datum_row.count_ = row->count_; for (int64_t i = 0; i < row->count_; ++i) { - store_row.row_val_.cells_[i] = row->cells_[i]; + datum_row.storage_datums_[i] = row->storage_datums_[i]; } - } else { - store_row.row_val_ = *row; + } else if (OB_FAIL(datum_row.shallow_copy(*row))) { + LOG_WARN("fail to shallow copy from iterator", K(ret), KPC(row), K(datum_row)); } + datum_row.row_flag_.set_flag(ObDmlFlag::DF_UPDATE); } return ret; @@ -5318,7 +5122,7 @@ int ObLSTabletService::insert_row_to_tablet( const bool check_exist, ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, - ObStoreRow &tbl_row) + blocksstable::ObDatumRow &datum_row) { int ret = OB_SUCCESS; ObStoreCtx &store_ctx = run_ctx.store_ctx_; @@ -5328,13 +5132,13 @@ int ObLSTabletService::insert_row_to_tablet( || !relative_table.is_valid() || nullptr == run_ctx.col_descs_ || run_ctx.col_descs_->count() <= 0 - || !tbl_row.is_valid())) { + || !datum_row.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(store_ctx), KP(run_ctx.col_descs_), K(tbl_row), K(ret)); + LOG_WARN("invalid args", K(store_ctx), KP(run_ctx.col_descs_), K(datum_row), K(ret)); } else if (GCONF.enable_defensive_check() - && OB_FAIL(check_new_row_legitimacy(run_ctx, tbl_row.row_val_))) { - LOG_WARN("check new row legitimacy failed", K(ret), K(tbl_row.row_val_)); - } else if (OB_FAIL(insert_lob_tablet_row(tablet_handle, run_ctx, tbl_row))) { + && OB_FAIL(check_new_row_legitimacy(run_ctx, datum_row))) { + LOG_WARN("check new row legitimacy failed", K(ret), K(datum_row)); + } else if (OB_FAIL(insert_lob_tablet_row(tablet_handle, run_ctx, datum_row))) { LOG_WARN("failed to write lob tablets rows", K(ret)); } else { const ObColDescIArray &col_descs = *run_ctx.col_descs_; @@ -5343,12 +5147,12 @@ int ObLSTabletService::insert_row_to_tablet( store_ctx, check_exist /*check_exist*/, col_descs, - tbl_row, + datum_row, run_ctx.dml_param_.encrypt_meta_))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret) { LOG_WARN("failed to write table row", K(ret), "table id", relative_table.get_table_id(), - K(col_descs), K(tbl_row)); + K(col_descs), K(datum_row)); } } } @@ -5356,26 +5160,28 @@ int ObLSTabletService::insert_row_to_tablet( return ret; } +// revert start int ObLSTabletService::process_old_row_lob_col( ObTabletHandle &data_tablet_handle, ObDMLRunningCtx &run_ctx, - ObStoreRow &tbl_row) + blocksstable::ObDatumRow &datum_row) { int ret = OB_SUCCESS; + run_ctx.is_old_row_valid_for_lob_ = false; bool has_lob_col = false; bool need_reread = is_sys_table(run_ctx.relative_table_.get_table_id()); int64_t col_cnt = run_ctx.col_descs_->count(); - if (tbl_row.row_val_.count_ != col_cnt) { + if (datum_row.count_ != col_cnt) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("[STORAGE_LOB]Invliad row col cnt", K(ret), K(col_cnt), K(tbl_row)); + LOG_WARN("[STORAGE_LOB]Invliad row col cnt", K(ret), K(col_cnt), K(datum_row)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < col_cnt; ++i) { const ObColDesc &column = run_ctx.col_descs_->at(i); if (is_lob_storage(column.col_type_.get_type())) { has_lob_col = true; - ObObj &obj = tbl_row.row_val_.cells_[i]; - need_reread = need_reread || (!obj.is_null() && !obj.is_nop_value() && !obj.has_lob_header()); + ObStorageDatum &datum = datum_row.storage_datums_[i]; + need_reread = need_reread || (!datum.is_null() && !datum.is_nop_value() && !datum.has_lob_header()); break; } } @@ -5385,15 +5191,15 @@ int ObLSTabletService::process_old_row_lob_col( for (int64_t i = 0; OB_SUCC(ret) && i < col_cnt; ++i) { const ObColDesc &column = run_ctx.col_descs_->at(i); if (is_lob_storage(column.col_type_.get_type())) { - ObObj &obj = tbl_row.row_val_.cells_[i]; - bool has_lob_header = obj.has_lob_header(); - if (obj.is_null() || obj.is_nop_value()) { + ObStorageDatum &datum = datum_row.storage_datums_[i]; + bool has_lob_header = datum.has_lob_header(); + if (datum.is_null() || datum.is_nop_value()) { // do nothing } else if (!has_lob_header) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("lob should have lob locator here.", K(ret), K(i), K(tbl_row.row_val_.cells_[i])); + LOG_WARN("lob should have lob locator here.", K(ret), K(i), K(datum)); } else { - ObLobLocatorV2 lob(obj.get_string(), has_lob_header); + ObLobLocatorV2 lob(datum.get_string(), has_lob_header); ObString disk_loc; if (!lob.is_valid()) { ret = OB_ERR_UNEXPECTED; @@ -5403,9 +5209,9 @@ int ObLSTabletService::process_old_row_lob_col( } else if (OB_FAIL(lob.get_disk_locator(disk_loc))) { LOG_WARN("failed to get disk lob locator.", K(ret), K(lob)); } else { - obj.set_lob_value(obj.get_type(), disk_loc.ptr(), disk_loc.length()); + datum.set_string(disk_loc.ptr(), disk_loc.length()); if (has_lob_header) { - obj.set_has_lob_header(); + datum.set_has_lob_header(); } run_ctx.is_old_row_valid_for_lob_ = true; } @@ -5413,7 +5219,7 @@ int ObLSTabletService::process_old_row_lob_col( } } } else { - if (OB_FAIL(table_refresh_row(data_tablet_handle, run_ctx, tbl_row.row_val_))) { + if (OB_FAIL(table_refresh_row(data_tablet_handle, run_ctx, datum_row))) { LOG_WARN("[STORAGE_LOB]re-read lob col failed", K(ret)); } } @@ -5424,16 +5230,12 @@ int ObLSTabletService::process_old_row_lob_col( int ObLSTabletService::table_refresh_row( ObTabletHandle &data_tablet_handle, ObDMLRunningCtx &run_ctx, - ObNewRow &row) + blocksstable::ObDatumRow &datum_row) { int ret = OB_SUCCESS; ObArenaAllocator scan_allocator(common::ObMemAttr(MTL_ID(), ObModIds::OB_LOB_ACCESS_BUFFER)); - ObTablet *data_tablet = data_tablet_handle.get_obj(); + ObSingleRowGetter storage_row_getter(scan_allocator, *data_tablet_handle.get_obj()); ObRelativeTable &data_table = run_ctx.relative_table_; - ObStoreRowkey rowkey; - if (OB_FAIL(rowkey.assign(row.cells_, data_table.get_rowkey_column_num()))) { - LOG_WARN("get rowkey col num failed", K(ret)); - } int64_t col_cnt = run_ctx.col_descs_->count(); ObSEArray out_col_ids; @@ -5442,22 +5244,12 @@ int ObLSTabletService::table_refresh_row( LOG_WARN("push col id failed.", K(ret), K(i)); } } - ObDatumRowkey datum_rowkey; - ObDatumRowkeyHelper rowkey_helper(scan_allocator); - ObSingleRowGetter single_row_getter(scan_allocator, *data_tablet); if (OB_FAIL(ret)) { - } else if (OB_FAIL(init_single_row_getter(single_row_getter, - run_ctx, out_col_ids, data_table, true))) { - LOG_WARN("failed to init single row getter", K(ret)); - } else if (OB_FAIL(rowkey_helper.convert_datum_rowkey(rowkey.get_rowkey(), datum_rowkey))) { - LOG_WARN("Failed to transfer datum rowkey", K(ret), K(rowkey)); } else { - ObNewRow *new_row = nullptr; - if (OB_FAIL(single_row_getter.open(datum_rowkey))) { - LOG_WARN("init single row getter failed", K(ret)); - } else if (OB_FAIL(single_row_getter.get_next_row(new_row))) { + ObDatumRow *new_row = nullptr; + if (OB_FAIL(get_storage_row(datum_row, out_col_ids, storage_row_getter, run_ctx, new_row))) { if (ret == OB_ITER_END) { - LOG_DEBUG("re-read old row not exist", K(ret), K(row)); + LOG_DEBUG("re-read old row not exist", K(ret), K(datum_row)); ret = OB_SUCCESS; } else { LOG_WARN("get next row from single row getter failed", K(ret)); @@ -5465,18 +5257,16 @@ int ObLSTabletService::table_refresh_row( } else if (OB_ISNULL(new_row)) { ret = OB_ERR_NULL_VALUE; LOG_WARN("get next row from single row null", K(ret)); - } else if (new_row->get_count() != row.get_count()) { + } else if (new_row->count_ != datum_row.count_) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get row from single row col count not equal.", K(ret), K(row.get_count()), K(new_row->get_count())); + LOG_WARN("get row from single row col count not equal.", K(ret), K(datum_row.count_), K(new_row->count_)); } else { - LOG_DEBUG("get new row success.", K(row), KPC(new_row)); + LOG_DEBUG("get new row success.", K(datum_row), KPC(new_row)); // only write cells, not write row - // passing fixed double scale from row to new_row - for (int64_t i = 0; OB_SUCC(ret) && i < new_row->get_count(); ++i) { - if (row.cells_[i].is_fixed_double()) { - new_row->cells_[i].set_scale(row.cells_[i].get_scale()); - } else if (OB_FAIL(ob_write_obj(run_ctx.dml_param_.lob_allocator_, new_row->cells_[i], row.cells_[i]))) { - LOG_WARN("copy ObObj error", K(ret), K(i), K(new_row->cells_[i])); + // TODO：concern about fixed double scale from row to new_row？@xuanxi + for (int64_t i = 0; OB_SUCC(ret) && i < new_row->count_; ++i) { + if (OB_FAIL(datum_row.storage_datums_[i].deep_copy(new_row->storage_datums_[i], run_ctx.dml_param_.lob_allocator_))) { + LOG_WARN("copy storage datum error", K(ret), K(i), K(new_row->storage_datums_[i])); } } if (OB_SUCC(ret)) { @@ -5490,40 +5280,38 @@ int ObLSTabletService::table_refresh_row( int ObLSTabletService::delete_row_in_tablet( ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, - const ObNewRow &row) + blocksstable::ObDatumRow &datum_row) { int ret = OB_SUCCESS; const ObDMLBaseParam &dml_param = run_ctx.dml_param_; ObStoreCtx &ctx = run_ctx.store_ctx_; ObRelativeTable &relative_table = run_ctx.relative_table_; - ObStoreRow &tbl_row = run_ctx.tbl_row_; - ObStoreRow new_tbl_row; ObSEArray update_idx; // update_idx is a dummy param here - tbl_row.flag_.set_flag(ObDmlFlag::DF_DELETE); - tbl_row.row_val_ = row; + datum_row.row_flag_.set_flag(ObDmlFlag::DF_DELETE); - if (OB_FAIL(check_old_row_legitimacy(tablet_handle, run_ctx, row))) { + if (OB_FAIL(check_old_row_legitimacy(run_ctx.cmp_funcs_, tablet_handle, run_ctx, datum_row))) { if (OB_ERR_DEFENSIVE_CHECK == ret) { - dump_diag_info_for_old_row_loss(relative_table, ctx, tbl_row); + dump_diag_info_for_old_row_loss(run_ctx, datum_row); } - LOG_WARN("check old row legitimacy failed", K(row)); - } else if (OB_FAIL(process_old_row_lob_col(tablet_handle, run_ctx, tbl_row))) { - LOG_WARN("failed to process old row lob col", K(ret), K(tbl_row)); - } else if (OB_FAIL(delete_lob_tablet_rows(run_ctx, tablet_handle, tbl_row, row))) { - LOG_WARN("failed to delete lob rows.", K(ret), K(tbl_row), K(row)); + LOG_WARN("check old row legitimacy failed", K(ret), K(datum_row)); + } else if (OB_FAIL(process_old_row_lob_col(tablet_handle, run_ctx, datum_row))) { + LOG_WARN("failed to process old row lob col", K(ret), K(datum_row)); + } else if (OB_FAIL(delete_lob_tablet_rows(run_ctx, tablet_handle, datum_row))) { + LOG_WARN("failed to delete lob rows.", K(ret), K(datum_row)); } else { update_idx.reset(); // update_idx is a dummy param here - new_tbl_row.reset(); - new_tbl_row.flag_.set_flag(ObDmlFlag::DF_DELETE); - new_tbl_row.row_val_ = tbl_row.row_val_; - tbl_row.flag_.set_flag(ObDmlFlag::DF_UPDATE); - if (OB_FAIL(tablet_handle.get_obj()->update_row(relative_table, ctx, - *run_ctx.col_descs_, update_idx, tbl_row, new_tbl_row, dml_param.encrypt_meta_))) { + ObDatumRow new_datum_row; + datum_row.row_flag_.set_flag(ObDmlFlag::DF_UPDATE); + if (OB_FAIL(new_datum_row.shallow_copy(datum_row))) { + LOG_WARN("failed to shallow copy datum row", K(ret), K(datum_row), K(new_datum_row)); + } else if (FALSE_IT(new_datum_row.row_flag_.set_flag(ObDmlFlag::DF_DELETE))) { + } else if (OB_FAIL(tablet_handle.get_obj()->update_row(relative_table, ctx, + *run_ctx.col_descs_, update_idx, datum_row, new_datum_row, dml_param.encrypt_meta_))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) { - LOG_WARN("failed to set row", K(ret), K(*run_ctx.col_descs_), K(tbl_row), K(new_tbl_row)); + LOG_WARN("failed to set row", K(ret), K(*run_ctx.col_descs_), K(datum_row), K(new_datum_row)); } } else { - LOG_DEBUG("succeeded to del main table row", K(tbl_row), K(new_tbl_row)); + LOG_DEBUG("succeeded to del main table row", K(datum_row), K(new_datum_row)); } } @@ -5533,8 +5321,7 @@ int ObLSTabletService::delete_row_in_tablet( int ObLSTabletService::delete_lob_col( ObDMLRunningCtx &run_ctx, const ObColDesc &column, - ObObj &obj, - const ObObj &sql_obj, + blocksstable::ObStorageDatum &datum, ObLobCommon *&lob_common, ObLobAccessParam &lob_param) { @@ -5543,17 +5330,15 @@ int ObLSTabletService::delete_lob_col( if (OB_ISNULL(lob_mngr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("[STORAGE_LOB]get lob manager instance failed.", K(ret)); - } else if (!column.col_type_.is_lob_storage() || obj.is_nop_value() || obj.is_null() || + } else if (!column.col_type_.is_lob_storage() || datum.is_nop_value() || datum.is_null() || !run_ctx.is_old_row_valid_for_lob_) { // do nothing } else { - ObString data = obj.get_string(); - // Notice: Only disk locator here! - ObString sql_data = sql_obj.get_string(); - ObLobLocatorV2 locator(data, obj.has_lob_header()); + ObString data = datum.get_string(); + ObLobLocatorV2 locator(data, datum.has_lob_header()); if (data.length() < sizeof(ObLobCommon)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("[STORAGE_LOB]Invalid Lob data.", K(ret), K(obj), K(data)); + LOG_WARN("[STORAGE_LOB]Invalid Lob data.", K(ret), K(datum), K(data)); } else if (locator.is_inrow()) { // deelete inrow lob no need to use the lob manager } else if (OB_FAIL(set_lob_storage_params(run_ctx, column, lob_param))) { @@ -5599,14 +5384,13 @@ int ObLSTabletService::delete_lob_col( int ObLSTabletService::delete_lob_tablet_rows( ObDMLRunningCtx &run_ctx, ObTabletHandle &data_tablet, - ObStoreRow &tbl_row, - const ObNewRow &row) + blocksstable::ObDatumRow &datum_row) { int ret = OB_SUCCESS; int64_t col_cnt = run_ctx.col_descs_->count(); - if (tbl_row.row_val_.count_ != col_cnt) { + if (datum_row.count_ != col_cnt) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("[STORAGE_LOB]Invliad row col cnt", K(col_cnt), K(tbl_row)); + LOG_WARN("[STORAGE_LOB]Invliad row col cnt", K(col_cnt), K(datum_row)); } else if (OB_FAIL(update_lob_meta_table_seq_no(run_ctx, 1/*row_count*/))) { LOG_WARN("update_lob_meta_table_seq_no fail", K(ret), K(run_ctx.dml_param_)); } else { @@ -5614,17 +5398,17 @@ int ObLSTabletService::delete_lob_tablet_rows( for (int64_t i = 0; OB_SUCC(ret) && i < col_cnt; ++i) { const ObColDesc &column = run_ctx.col_descs_->at(i); if (column.col_type_.is_lob_storage()) { - ObObj &obj = tbl_row.row_val_.get_cell(i); - const ObObj &sql_obj = row.get_cell(i); + blocksstable::ObStorageDatum &datum = datum_row.storage_datums_[i]; ObLobAccessParam lob_param; - if (OB_FAIL(delete_lob_col(run_ctx, column, obj, sql_obj, lob_common, lob_param))) { - LOG_WARN("[STORAGE_LOB]failed to erase lob col.", K(ret), K(i), K(tbl_row)); + if (OB_FAIL(delete_lob_col(run_ctx, column, datum, lob_common, lob_param))) { + LOG_WARN("[STORAGE_LOB]failed to erase lob col.", K(ret), K(i), K(datum_row)); } } } } return ret; } +// revert end int ObLSTabletService::prepare_scan_table_param( ObTableScanParam ¶m, @@ -5668,16 +5452,17 @@ int ObLSTabletService::prepare_scan_table_param( } void ObLSTabletService::dump_diag_info_for_old_row_loss( - ObRelativeTable &data_table, - ObStoreCtx &store_ctx, - const ObStoreRow &tbl_row) + ObDMLRunningCtx &run_ctx, + blocksstable::ObDatumRow &datum_row) { int ret = OB_SUCCESS; + ObStoreCtx &store_ctx = run_ctx.store_ctx_; + ObRelativeTable &data_table = run_ctx.relative_table_; + ObColDescIArray &col_descs = const_cast(*run_ctx.col_descs_); ObArenaAllocator allocator(common::ObMemAttr(MTL_ID(), "DumpDIAGInfo")); ObTableAccessParam access_param; ObTableAccessContext access_ctx; ObSEArray out_col_pros; - ObStoreRowkey rowkey; ObDatumRowkey datum_rowkey; ObDatumRowkeyHelper rowkey_helper(allocator); const int64_t schema_rowkey_cnt = data_table.get_rowkey_column_num(); @@ -5704,10 +5489,8 @@ void ObLSTabletService::dump_diag_info_for_old_row_loss( } if (OB_FAIL(ret)) { - } else if (OB_FAIL(rowkey.assign(tbl_row.row_val_.cells_, schema_rowkey_cnt))) { - LOG_WARN("Failed to assign rowkey", K(ret)); - } else if (OB_FAIL(rowkey_helper.convert_datum_rowkey(rowkey.get_rowkey(), datum_rowkey))) { - STORAGE_LOG(WARN, "Failed to transfer datum rowkey", K(ret), K(rowkey)); + } else if (OB_FAIL(rowkey_helper.prepare_datum_rowkey(datum_row, schema_rowkey_cnt, col_descs, datum_rowkey))) { + LOG_WARN("Failed to prepare rowkey", K(ret), K(datum_row), K(datum_rowkey)); } else if (OB_FAIL(access_ctx.init(query_flag, store_ctx, allocator, trans_version_rang))) { LOG_WARN("Fail to init access ctx", K(ret)); } else { @@ -5724,8 +5507,8 @@ void ObLSTabletService::dump_diag_info_for_old_row_loss( ObITable *table = nullptr; const ObDatumRow *row = nullptr; - FLOG_INFO("Try to find the specified rowkey within all the sstable", K(tbl_row), K(table_iter)); - FLOG_INFO("Prepare the diag env to dump the rows", K(store_ctx), K(rowkey), K(datum_rowkey), + FLOG_INFO("Try to find the specified rowkey within all the sstable", K(datum_row), K(table_iter)); + FLOG_INFO("Prepare the diag env to dump the rows", K(store_ctx), K(datum_rowkey), K(access_ctx.trans_version_range_)); table_iter.resume(); @@ -5784,7 +5567,7 @@ void ObLSTabletService::dump_diag_info_for_old_row_loss( } if (OB_SUCC(ret)) { - FLOG_INFO("prepare to use single merge to find row", K(rowkey), K(datum_rowkey), K(access_param)); + FLOG_INFO("prepare to use single merge to find row", K(datum_rowkey), K(access_param)); ObSingleMerge *get_merge = nullptr; ObGetTableParam get_table_param; ObDatumRow *row = nullptr; @@ -5804,7 +5587,7 @@ void ObLSTabletService::dump_diag_info_for_old_row_loss( while (OB_SUCC(get_merge->get_next_row(row))) { FLOG_INFO("Found one row for the rowkey", KPC(row)); } - FLOG_INFO("Finish to find rowkey with single merge", K(ret), K(rowkey), K(datum_rowkey)); + FLOG_INFO("Finish to find rowkey with single merge", K(ret), K(datum_rowkey)); } if (OB_NOT_NULL(get_merge)) { get_merge->~ObSingleMerge(); diff --git a/src/storage/ls/ob_ls_tablet_service.h b/src/storage/ls/ob_ls_tablet_service.h index 65ca56856..3bbc572c8 100644 --- a/src/storage/ls/ob_ls_tablet_service.h +++ b/src/storage/ls/ob_ls_tablet_service.h @@ -33,6 +33,7 @@ #include "storage/tablet/ob_tablet_persister.h" #include "storage/lob/ob_lob_manager.h" #include "storage/multi_data_source/mds_table_mgr.h" +#include "storage/blocksstable/ob_datum_row_iterator.h" namespace oceanbase { @@ -57,12 +58,18 @@ namespace blocksstable { class ObMigrationSSTableParam; struct ObDatumRowkey; +class ObDatumRowStore; } namespace compaction { class ObTabletMergeCtx; } +namespace sql +{ +class ObDASDMLIterator; +class ObDASUpdIterator; +} namespace storage { @@ -317,7 +324,7 @@ public: ObStoreCtx &ctx, const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows); int insert_row( ObTabletHandle &tablet_handle, @@ -325,31 +332,31 @@ public: const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, const common::ObIArray &duplicated_column_ids, - const common::ObNewRow &row, + blocksstable::ObDatumRow &row, const ObInsertFlag flag, int64_t &affected_rows, - common::ObNewRowIterator *&duplicated_rows); + blocksstable::ObDatumRowIterator *&duplicated_rows); int update_rows( ObTabletHandle &tablet_handle, ObStoreCtx &ctx, const ObDMLBaseParam &dml_param, const ObIArray &column_ids, const ObIArray< uint64_t> &updated_column_ids, - ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows); int put_rows( ObTabletHandle &tablet_handle, ObStoreCtx &ctx, const ObDMLBaseParam &dml_param, const ObIArray &column_ids, - ObNewRowIterator *row_iter, + ObDatumRowIterator *row_iter, int64_t &affected_rows); // for htable, insert or update int delete_rows( ObTabletHandle &tablet_handle, ObStoreCtx &ctx, const ObDMLBaseParam &dml_param, const ObIArray &column_ids, - ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows); int lock_rows( ObTabletHandle &tablet_handle, @@ -357,13 +364,13 @@ public: const ObDMLBaseParam &dml_param, const ObLockFlag lock_flag, const bool is_sfu, - ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows); int lock_row( ObTabletHandle &tablet_handle, ObStoreCtx &ctx, const ObDMLBaseParam &dml_param, - const ObNewRow &row, + blocksstable::ObDatumRow &row, const ObLockFlag lock_flag, const bool is_sfu); int get_multi_ranges_cost( @@ -553,7 +560,7 @@ private: int offline_build_tablet_without_memtable_(); int offline_gc_tablet_for_create_or_transfer_in_abort_(); int offline_destroy_memtable_and_mds_table_(); - int mock_duplicated_rows_(common::ObNewRowIterator *&duplicated_rows); + int mock_duplicated_rows_(blocksstable::ObDatumRowIterator *&duplicated_rows); private: static int check_real_leader_for_4377_(const ObLSID ls_id); static int check_need_rollback_in_transfer_for_4377_(const transaction::ObTxDesc *tx_desc, @@ -562,50 +569,42 @@ private: static int build_create_sstable_param_for_migration( const blocksstable::ObMigrationSSTableParam &migrate_sstable_param, ObTabletCreateSSTableParam &create_sstable_param); - static int need_check_old_row_legitimacy( - ObDMLRunningCtx &run_ctx, - bool &need_check, - bool &is_udf); - static int construct_table_rows( - const ObNewRow *rows, - ObStoreRow *tbl_rows, - int64_t row_count); static int check_old_row_legitimacy( + const blocksstable::ObStoreCmpFuncs &cmp_funcs, ObTabletHandle &data_tablet_handle, ObDMLRunningCtx &run_ctx, - const common::ObNewRow &old_row); + blocksstable::ObDatumRow &old_row); static int check_new_row_legitimacy( ObDMLRunningCtx &run_ctx, - const common::ObNewRow &new_row); + const blocksstable::ObDatumRow &datum_row); static int insert_rows_to_tablet( ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, - const common::ObNewRow *const rows, + blocksstable::ObDatumRow *rows, const int64_t row_count, ObRowsInfo &rows_info, - storage::ObStoreRow *tbl_rows, int64_t &afct_num, int64_t &dup_num); static int insert_tablet_rows( const int64_t row_count, ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, - ObStoreRow *rows, + blocksstable::ObDatumRow *rows, ObRowsInfo &rows_info); static int insert_lob_col( ObDMLRunningCtx &run_ctx, const ObColDesc &column, - ObObj &obj, + blocksstable::ObStorageDatum &datum, ObLobAccessParam *del_param, ObLobCommon *lob_common); static int insert_lob_tablet_row( ObTabletHandle &data_tablet, ObDMLRunningCtx &run_ctx, - ObStoreRow &row); + blocksstable::ObDatumRow &datum_row); static int insert_lob_tablet_rows( ObTabletHandle &data_tablet, ObDMLRunningCtx &run_ctx, - ObStoreRow *rows, + blocksstable::ObDatumRow *rows, int64_t row_count); static int extract_rowkey( const ObRelativeTable &table, @@ -620,8 +619,8 @@ private: const int64_t buffer_len, const common::ObTimeZoneInfo *tz_info = nullptr); static int get_next_rows( - ObNewRowIterator *row_iter, - ObNewRow *&rows, + blocksstable::ObDatumRowIterator *row_iter, + blocksstable::ObDatumRow *&rows, int64_t &row_count); static int construct_update_idx( const int64_t schema_rowkey_cnt, @@ -634,19 +633,21 @@ private: bool &rowkey_change, bool &delay_new); static int check_rowkey_value_change( - const common::ObNewRow &old_row, - const common::ObNewRow &new_row, + const ObDatumRow &old_row, + const ObDatumRow &new_row, const int64_t rowkey_len, + const blocksstable::ObStorageDatumUtils &rowkey_datum_utils, bool &rowkey_change); static int process_delta_lob( ObDMLRunningCtx &run_ctx, const ObColDesc &column, - ObObj &old_obj, + ObStorageDatum &old_datum, ObLobLocatorV2 &delta_lob, - ObObj &obj); + ObStorageDatum &datum); static int register_ext_info_commit_cb( ObDMLRunningCtx &run_ctx, - ObObj &col_data, + ObStorageDatum &col_data, + ObObjType type, ObObj &ext_info_data); static int set_lob_storage_params( ObDMLRunningCtx &run_ctx, @@ -658,9 +659,8 @@ private: ObDMLRunningCtx &run_ctx, const ObIArray &update_idx, bool data_tbl_rowkey_change, - ObStoreRow &old_sql_row, - ObStoreRow &old_row, - ObStoreRow &new_row); + ObDatumRow &old_row, + ObDatumRow &new_row); static int update_row_to_tablet( ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, @@ -668,42 +668,39 @@ private: const ObIArray &update_idx, const bool delay_new, const bool lob_update, - ObStoreRow &old_tbl_row, - ObStoreRow &new_tbl_row, - ObRowStore *row_store, + ObDatumRow &old_row, + ObDatumRow &new_row, + ObDatumRowStore *row_store, bool &duplicate); static int process_old_row( ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, const bool data_tbl_rowkey_change, const bool lob_update, - ObStoreRow &tbl_row); + ObDatumRow &datum_row); static int process_new_row( ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, const common::ObIArray &update_idx, - const ObStoreRow &old_tbl_row, - const ObStoreRow &new_tbl_row, + const ObDatumRow &old_datum_row, + ObDatumRow &new_datum_row, const bool rowkey_change); static int process_data_table_row( ObTabletHandle &data_tablet, ObDMLRunningCtx &run_ctx, const ObIArray &update_idx, - const ObStoreRow &old_tbl_row, - const ObStoreRow &new_tbl_row, + const ObDatumRow &old_datum_row, + ObDatumRow &new_datum_row, const bool rowkey_change); - static int check_new_row_nullable_value( - const ObIArray &column_ids, - ObRelativeTable &data_table, - const ObNewRow &new_row); - static int check_new_row_nullable_value( + static int check_datum_row_nullable_value( const common::ObIArray &col_descs, ObRelativeTable &relative_table, - const common::ObNewRow &new_row); - static int check_new_row_shadow_pk( + const blocksstable::ObDatumRow &datum_row); + static int check_datum_row_shadow_pk( const ObIArray &column_ids, ObRelativeTable &data_table, - const ObNewRow &new_row); + const blocksstable::ObDatumRow &datum_row, + const blocksstable::ObStorageDatumUtils &rowkey_datum_utils); static int check_row_locked_by_myself( ObTabletHandle &tablet_handle, ObRelativeTable &relative_table, @@ -715,61 +712,55 @@ private: ObDMLRunningCtx &run_ctx, const ObInsertFlag flag, const common::ObIArray &out_col_ids, - const common::ObNewRow &row, - common::ObNewRowIterator *&duplicated_rows); + blocksstable::ObDatumRow &row, + blocksstable::ObDatumRowIterator *&duplicated_rows); static int init_single_row_getter( ObSingleRowGetter &row_getter, ObDMLRunningCtx &run_ctx, const ObIArray &out_col_ids, ObRelativeTable &relative_table, bool skip_read_lob = false); - static int single_get_row( - ObSingleRowGetter &row_getter, - const blocksstable::ObDatumRowkey &rowkey, - ObNewRowIterator *&duplicated_rows); - static int convert_row_to_rowkey( - ObSingleRowGetter &index_row_getter, - ObStoreRowkey &rowkey); + static int add_duplicate_row( + ObDatumRow *storage_row, + const blocksstable::ObStorageDatumUtils &rowkey_datum_utils, + blocksstable::ObDatumRowIterator *&duplicated_rows); static int get_next_row_from_iter( - ObNewRowIterator *row_iter, - ObStoreRow &store_row, + blocksstable::ObDatumRowIterator *row_iter, + ObDatumRow &datum_row, const bool need_copy_cells); static int insert_row_to_tablet( const bool check_exist, ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, - ObStoreRow &tbl_row); - static int process_old_row_lob_col( + blocksstable::ObDatumRow &datum_row); + static int process_old_row_lob_col( ObTabletHandle &data_tablet_handle, ObDMLRunningCtx &run_ctx, - ObStoreRow &tbl_row); + blocksstable::ObDatumRow &datum_row); static int table_refresh_row( ObTabletHandle &data_tablet_handle, ObDMLRunningCtx &run_ctx, - ObNewRow &row); + blocksstable::ObDatumRow &datum_row); static int delete_row_in_tablet( ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, - const ObNewRow &row); + blocksstable::ObDatumRow &datum_row); static int delete_lob_col( ObDMLRunningCtx &run_ctx, const ObColDesc &column, - ObObj &obj, - const ObObj &sql_obj, + blocksstable::ObStorageDatum &datum, ObLobCommon *&lob_common, ObLobAccessParam &lob_param); static int delete_lob_tablet_rows( ObDMLRunningCtx &run_ctx, ObTabletHandle &data_tablet, - ObStoreRow &tbl_row, - const ObNewRow &row); + blocksstable::ObDatumRow &datum_row); static int prepare_scan_table_param( ObTableScanParam ¶m, share::schema::ObMultiVersionSchemaService &schema_service); static void dump_diag_info_for_old_row_loss( - ObRelativeTable &data_table, - ObStoreCtx &store_ctx, - const ObStoreRow &tbl_row); + ObDMLRunningCtx &run_ctx, + blocksstable::ObDatumRow &datum_row); int set_allow_to_read_(ObLS *ls); // TODO(chenqingxiang.cqx): remove this int create_empty_shell_tablet( @@ -786,8 +777,14 @@ private: const int64_t ddl_task_id, const common::ObTabletID &tablet_id, const common::ObIArray &column_ids, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows); + static int get_storage_row(const ObDatumRow &sql_row, + const ObIArray &column_ids, + ObSingleRowGetter &row_getter, + ObDMLRunningCtx &run_ctx, + ObDatumRow *&out_row, + bool use_fuse_row_cache = false); static int check_is_gencol_check_failed(const ObRelativeTable &data_table, uint64_t error_col_id, bool &is_virtual_gencol); diff --git a/src/storage/memtable/mvcc/ob_mvcc_ctx.cpp b/src/storage/memtable/mvcc/ob_mvcc_ctx.cpp index ff35ce2bd..33b297f5e 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_ctx.cpp +++ b/src/storage/memtable/mvcc/ob_mvcc_ctx.cpp @@ -269,12 +269,13 @@ int ObIMvccCtx::register_ext_info_commit_cb( const blocksstable::ObDmlFlag dml_flag, transaction::ObTxDesc *tx_desc, transaction::ObTxSEQ &parent_seq_no, - ObObj &index_data, + blocksstable::ObStorageDatum &index_data, + ObObjType &type, ObObj &ext_info_data) { int ret = OB_SUCCESS; storage::ObExtInfoCbRegister cb_register; - if (OB_FAIL(cb_register.register_cb(this, timeout, dml_flag, tx_desc, parent_seq_no, index_data, ext_info_data))) { + if (OB_FAIL(cb_register.register_cb(this, timeout, dml_flag, tx_desc, parent_seq_no, index_data, type, ext_info_data))) { TRANS_LOG(ERROR, "register ext info callback failed", K(ret), K(cb_register), K(*this)); } return ret; diff --git a/src/storage/memtable/mvcc/ob_mvcc_ctx.h b/src/storage/memtable/mvcc/ob_mvcc_ctx.h index b5486a246..c52ce42fe 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_ctx.h +++ b/src/storage/memtable/mvcc/ob_mvcc_ctx.h @@ -178,7 +178,8 @@ public: const blocksstable::ObDmlFlag dml_flag, transaction::ObTxDesc *tx_desc, transaction::ObTxSEQ &parent_seq_no, - ObObj &index_data, + blocksstable::ObStorageDatum &index_data, + ObObjType &type, ObObj &ext_info_data); public: virtual void reset() diff --git a/src/storage/memtable/ob_concurrent_control.cpp b/src/storage/memtable/ob_concurrent_control.cpp index 2012f0609..1ddd00b5b 100644 --- a/src/storage/memtable/ob_concurrent_control.cpp +++ b/src/storage/memtable/ob_concurrent_control.cpp @@ -68,7 +68,7 @@ int check_sequence_set_violation(const concurrent_control::ObWriteFlag write_fla ret = OB_ERR_PRIMARY_KEY_DUPLICATE; TRANS_LOG(WARN, "pdml duplicate primary key found", K(ret), K(writer_tx_id), K(writer_dml_flag), K(writer_seq_no), - K(locker_tx_id), K(locker_dml_flag), K(locker_seq_no)); + K(locker_tx_id), K(locker_dml_flag), K(locker_seq_no), K(reader_seq_no)); // Case 2.1: For the case of the update in the storage layer, it may be // split into lock and update in a single statement and fail the check, so // we need bypass this case(Currently only the update of the lob will cause diff --git a/src/storage/memtable/ob_memtable.cpp b/src/storage/memtable/ob_memtable.cpp index 837cd1d8b..668857797 100644 --- a/src/storage/memtable/ob_memtable.cpp +++ b/src/storage/memtable/ob_memtable.cpp @@ -373,7 +373,7 @@ int ObMemtable::multi_set( const storage::ObTableIterParam ¶m, storage::ObTableAccessContext &context, const common::ObIArray &columns, - const storage::ObStoreRow *rows, + blocksstable::ObDatumRow *rows, const int64_t row_count, const bool check_exist, const share::ObEncryptMeta *encrypt_meta, @@ -381,7 +381,6 @@ int ObMemtable::multi_set( { int ret = OB_SUCCESS; ObMvccWriteGuard guard(ret); - ObMemtableKeyGenerator mtk_generator; if (IS_NOT_INIT) { TRANS_LOG(WARN, "Not inited", K(*this)); ret = OB_NOT_INIT; @@ -389,7 +388,7 @@ int ObMemtable::multi_set( ret = OB_INVALID_ARGUMENT; TRANS_LOG(WARN, "Invalid argument", K(ret), K(param), K(context)); } else if (OB_UNLIKELY(nullptr == context.store_ctx_->mvcc_acc_ctx_.get_mem_ctx() - || param.get_schema_rowkey_count() > columns.count())) { + || param.get_schema_rowkey_count() > columns.count())) { ret = OB_INVALID_ARGUMENT; TRANS_LOG(WARN, "Invalid param", K(ret), K(param), K(columns.count())); #ifdef OB_BUILD_TDE_SECURITY @@ -400,8 +399,6 @@ int ObMemtable::multi_set( } else if (need_for_save(encrypt_meta) && OB_FAIL(save_encrypt_meta(param.table_id_, encrypt_meta))) { TRANS_LOG(WARN, "store encrypt meta to memtable failed", KPC(encrypt_meta), KR(ret)); #endif - } else if (OB_FAIL(mtk_generator.init(rows, row_count, param.get_schema_rowkey_count(), columns))) { - TRANS_LOG(WARN, "fail to generate memtable keys", KPC(encrypt_meta), K(*context.store_ctx_), KR(ret)); } if (OB_FAIL(ret)) { @@ -410,17 +407,22 @@ int ObMemtable::multi_set( } else { lib::CompatModeGuard compat_guard(mode_); if (row_count > 1) { - ret = multi_set_(param, columns, rows, row_count, check_exist, mtk_generator, context, rows_info); + ret = multi_set_(param, columns, rows, row_count, check_exist, context, rows_info); } else { - ret = set_(param, - columns, - rows[0], - nullptr, /*old_row*/ - nullptr, /*update_idx*/ - mtk_generator[0], - check_exist, - context, - nullptr /*mvcc_row*/); + ObMemtableKeyGenerator memtable_key_generator(param.get_schema_rowkey_count(), columns); + if (OB_FAIL(memtable_key_generator.init())) { + TRANS_LOG(WARN, "fail to init memtable key generator", K(ret)); + } else { + ret = set_(param, + columns, + rows[0], + nullptr, /*old_row*/ + nullptr, /*update_idx*/ + check_exist, + context, + memtable_key_generator, + nullptr /*mvcc_row*/); + } } guard.set_memtable(this); } @@ -529,24 +531,23 @@ int ObMemtable::set( const storage::ObTableIterParam ¶m, storage::ObTableAccessContext &context, const common::ObIArray &columns, - const storage::ObStoreRow &row, + blocksstable::ObDatumRow &row, const share::ObEncryptMeta *encrypt_meta, const bool check_exist) { int ret = OB_SUCCESS; ObMvccWriteGuard guard(ret); - ObMemtableKeyGenerator mtk_generator; if (IS_NOT_INIT) { TRANS_LOG(WARN, "not init", K(*this)); ret = OB_NOT_INIT; } else if (!param.is_valid() || !context.is_valid()) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument, ", K(ret), K(param), K(context)); + ret = OB_INVALID_ARGUMENT; + TRANS_LOG(WARN, "invalid argument, ", K(ret), K(param), K(context)); } else if (NULL == context.store_ctx_->mvcc_acc_ctx_.get_mem_ctx() - || param.get_schema_rowkey_count() > columns.count() - || row.row_val_.count_ < columns.count()) { + || param.get_schema_rowkey_count() > columns.count() + || row.count_ < columns.count()) { TRANS_LOG(WARN, "invalid param", K(param), - K(columns.count()), K(row.row_val_.count_)); + K(columns.count()), K(row.count_)); ret = OB_INVALID_ARGUMENT; #ifdef OB_BUILD_TDE_SECURITY //TODO: table_id is just used as encrypt_index, we may rename it in the future. @@ -556,32 +557,28 @@ int ObMemtable::set( } else if (need_for_save(encrypt_meta) && OB_FAIL(save_encrypt_meta(param.table_id_, encrypt_meta))) { TRANS_LOG(WARN, "store encrypt meta to memtable failed", KPC(encrypt_meta), KR(ret)); #endif - } else if (OB_FAIL(mtk_generator.init(&row, 1, param.get_schema_rowkey_count(), columns))) { - TRANS_LOG(WARN, "fail to generate memtable keys", KPC(encrypt_meta), K(*context.store_ctx_), KR(ret)); } if (OB_FAIL(ret)) { } else if (OB_FAIL(guard.write_auth(*context.store_ctx_))) { TRANS_LOG(WARN, "not allow to write", K(*context.store_ctx_)); } else { - lib::CompatModeGuard compat_guard(mode_); - - ret = set_(param, - columns, - row, - NULL, /*old_row*/ - NULL, /*update_idx*/ - mtk_generator[0], - check_exist, - context, - nullptr /*mvcc_row*/); - guard.set_memtable(this); - } - - if (OB_SUCC(ret)) { - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(try_report_dml_stat_(param.table_id_))) { - TRANS_LOG_RET(WARN, tmp_ret, "fail to report dml stat", K_(reported_dml_stat)); + ObMemtableKeyGenerator memtable_key_generator(param.get_schema_rowkey_count(), columns); + if (OB_FAIL(memtable_key_generator.init())) { + TRANS_LOG(WARN, "fail to init memtable key generator", K(ret)); + } else { + lib::CompatModeGuard compat_guard(mode_); + ret = set_(param, + columns, + row, + NULL, /*old_row*/ + NULL, /*update_idx*/ + check_exist, + context, + memtable_key_generator, + nullptr /*mvcc_row*/); + TRANS_LOG(WARN, "[xuanxi] set row", K(ret), K(row), K(check_exist), K(memtable_key_generator.get_memtable_key())); + guard.set_memtable(this); } } return ret; @@ -592,13 +589,12 @@ int ObMemtable::set( storage::ObTableAccessContext &context, const ObIArray &columns, const ObIArray &update_idx, - const storage::ObStoreRow &old_row, - const storage::ObStoreRow &new_row, + const blocksstable::ObDatumRow &old_row, + blocksstable::ObDatumRow &new_row, const share::ObEncryptMeta *encrypt_meta) { int ret = OB_SUCCESS; ObMvccWriteGuard guard(ret); - ObMemtableKeyGenerator mtk_generator; if (IS_NOT_INIT) { TRANS_LOG(WARN, "not init", K(*this)); ret = OB_NOT_INIT; @@ -606,7 +602,7 @@ int ObMemtable::set( ret = OB_INVALID_ARGUMENT; TRANS_LOG(WARN, "invalid argument, ", K(ret), K(param), K(context)); } else if (NULL == context.store_ctx_->mvcc_acc_ctx_.get_mem_ctx() - || param.get_schema_rowkey_count() > columns.count()) { + || param.get_schema_rowkey_count() > columns.count()) { ret = OB_INVALID_ARGUMENT; TRANS_LOG(ERROR, "invalid param", K(ret), K(param)); #ifdef OB_BUILD_TDE_SECURITY @@ -617,32 +613,28 @@ int ObMemtable::set( } else if (need_for_save(encrypt_meta) && OB_FAIL(save_encrypt_meta(param.table_id_, encrypt_meta))) { TRANS_LOG(WARN, "store encrypt meta to memtable failed", KPC(encrypt_meta), KR(ret)); #endif - } else if (OB_FAIL(mtk_generator.init(&new_row, 1, param.get_schema_rowkey_count(), columns))) { - TRANS_LOG(WARN, "fail to generate memtable keys", KPC(encrypt_meta), K(*context.store_ctx_), KR(ret)); } if (OB_FAIL(ret)){ } else if (OB_FAIL(guard.write_auth(*context.store_ctx_))) { TRANS_LOG(WARN, "not allow to write", K(*context.store_ctx_)); } else { - lib::CompatModeGuard compat_guard(mode_); + ObMemtableKeyGenerator memtable_key_generator(param.get_schema_rowkey_count(), columns); + if (OB_FAIL(memtable_key_generator.init())) { + TRANS_LOG(WARN, "fail to init memtable key generator", K(ret)); + } else { + lib::CompatModeGuard compat_guard(mode_); - ret = set_(param, - columns, - new_row, - &old_row, - &update_idx, - mtk_generator[0], - false/*check_exist*/, - context, - nullptr /*mvcc_row*/); - guard.set_memtable(this); - } - - if (OB_SUCC(ret)) { - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(try_report_dml_stat_(param.table_id_))) { - TRANS_LOG_RET(WARN, tmp_ret, "fail to report dml stat", K_(reported_dml_stat)); + ret = set_(param, + columns, + new_row, + &old_row, + &update_idx, + false/*check_exist*/, + context, + memtable_key_generator, + nullptr /*mvcc_row*/); + guard.set_memtable(this); } } return ret; @@ -651,10 +643,13 @@ int ObMemtable::set( int ObMemtable::lock( const storage::ObTableIterParam ¶m, storage::ObTableAccessContext &context, - const common::ObNewRow &row) + ObColDescArray &col_desc, + blocksstable::ObDatumRow &row) { int ret = OB_SUCCESS; ObMvccWriteGuard guard(ret); + ObDatumRowkeyHelper rowkey_helper; + ObDatumRowkey datum_rowkey(row.storage_datums_, param.get_schema_rowkey_count()); ObStoreRowkey tmp_key; ObMemtableKey mtk; ObMvccAccessCtx &acc_ctx = context.store_ctx_->mvcc_acc_ctx_; @@ -670,9 +665,9 @@ int ObMemtable::lock( // actually, there is no circumstance in where locking the index table is need. ret = OB_NOT_SUPPORTED; TRANS_LOG(WARN, "locking the non-unique local index is not supported", K(ret), K(row), K(param)); - } else if (OB_FAIL(tmp_key.assign(row.cells_, param.get_schema_rowkey_count()))) { - TRANS_LOG(WARN, "Failed to assign rowkey", K(row), K(param)); - } else if (OB_FAIL(mtk.encode(param.get_read_info()->get_columns_desc(), &tmp_key))) { + } else if (OB_FAIL(rowkey_helper.convert_store_rowkey(datum_rowkey, col_desc, tmp_key))) { + LOG_WARN("Failed to convert store rowkey from datum rowkey", K(ret), K(row), K(datum_rowkey)); + } else if (OB_FAIL(mtk.encode(col_desc, &tmp_key))) { TRANS_LOG(WARN, "encode mtk failed", K(ret), K(param)); } else if (acc_ctx.write_flag_.is_check_row_locked()) { if (OB_FAIL(ObRowConflictHandler::check_foreign_key_constraint(param, context, tmp_key))) { @@ -1589,7 +1584,6 @@ void ObMemtable::lock_row_on_frozen_stores_on_failure( int ObMemtable::lock_rows_on_frozen_stores_( const bool check_exist, const storage::ObTableIterParam ¶m, - const ObMemtableKeyGenerator &memtable_keys, storage::ObTableAccessContext &context, ObMvccRowAndWriteResults &mvcc_rows, ObRowsInfo &rows_info) @@ -2499,10 +2493,9 @@ bool ObMemtable::need_for_save(const share::ObEncryptMeta *encrypt_meta) int ObMemtable::multi_set_( const storage::ObTableIterParam ¶m, const common::ObIArray &columns, - const storage::ObStoreRow *rows, + const blocksstable::ObDatumRow *rows, const int64_t row_count, const bool check_exist, - const ObMemtableKeyGenerator &memtable_keys, storage::ObTableAccessContext &context, storage::ObRowsInfo &rows_info) { @@ -2511,8 +2504,13 @@ int ObMemtable::multi_set_( int64_t conflict_idx = -1; int64_t row_size_stat = 0; ObMvccRowAndWriteResults mvcc_rows; + // TODO(xuanxi): remove it later + ObMemtableKeyGenerator::ObMemtableKeyBuffer memtable_key_buffer; + ObMemtableKeyGenerator memtable_key_generator(param.get_schema_rowkey_count(), columns, &memtable_key_buffer); if (OB_FAIL(mvcc_rows.prepare_allocate(row_count))) { TRANS_LOG(WARN, "Failed to prepare allocate mvcc rows", K(ret), K(row_count)); + } else if (OB_FAIL(memtable_key_generator.init())) { + TRANS_LOG(WARN, "fail to init memtable key generator", K(ret)); } // 1. Check write conflict in memtables. @@ -2524,9 +2522,9 @@ int ObMemtable::multi_set_( rows[i], nullptr, /*old_row*/ nullptr, /*update_idx*/ - memtable_keys[i], check_exist, context, + memtable_key_generator, &(mvcc_rows[permutation_idx])))) { if (OB_UNLIKELY(OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret)) { TRANS_LOG(WARN, "Failed to insert new row", K(ret), K(i), K(permutation_idx), K(rows[i])); @@ -2553,7 +2551,7 @@ int ObMemtable::multi_set_( // 2. Check uniqueness constraint and write conflict in sstables. if (OB_FAIL(ret)) { } else if (rows_info.all_rows_found()) { - } else if (OB_FAIL(lock_rows_on_frozen_stores_(check_exist, param, memtable_keys, context, mvcc_rows, rows_info))) { + } else if (OB_FAIL(lock_rows_on_frozen_stores_(check_exist, param, context, mvcc_rows, rows_info))) { TRANS_LOG(WARN, "Failed to lock rows on frozen stores", K(ret)); } else if (rows_info.have_conflict()) { conflict_idx = rows_info.get_conflict_idx(); @@ -2579,9 +2577,9 @@ int ObMemtable::multi_set_( if (param.is_non_unique_local_index_) { // no need to detect deadlock for non-unique local index table } else { - for (int64_t idx = 0; idx < memtable_keys.count(); ++idx) { + for (int64_t idx = 0; idx < memtable_key_buffer.count(); ++idx) { MTL(ObLockWaitMgr*)->set_hash_holder(key_.get_tablet_id(), - memtable_keys[idx], + memtable_key_buffer.at(idx), context.store_ctx_->mvcc_acc_ctx_.get_mem_ctx()->get_tx_id()); } } @@ -2601,12 +2599,12 @@ int ObMemtable::multi_set_( int ObMemtable::set_( const storage::ObTableIterParam ¶m, const common::ObIArray &columns, - const storage::ObStoreRow &new_row, - const storage::ObStoreRow *old_row, + const blocksstable::ObDatumRow &new_row, + const blocksstable::ObDatumRow *old_row, const common::ObIArray *update_idx, - const ObMemtableKey &mtk, const bool check_exist, storage::ObTableAccessContext &context, + ObMemtableKeyGenerator &memtable_key_generator, ObMvccRowAndWriteResult *mvcc_row) { int ret = OB_SUCCESS; @@ -2645,11 +2643,11 @@ int ObMemtable::set_( TRANS_LOG(WARN, "get write seq failed", K(ret)); } else if (OB_FAIL(row_writer.write(param.get_schema_rowkey_count(), new_row, update_idx, buf, len))) { TRANS_LOG(WARN, "Failed to write new row", K(ret), K(new_row)); - } else if (OB_UNLIKELY(new_row.flag_.is_not_exist())) { + } else if (OB_UNLIKELY(new_row.row_flag_.is_not_exist())) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "Unexpected not exist trans node", K(ret), K(new_row)); } else { - ObMemtableData mtd(new_row.flag_.get_dml_flag(), len, buf); + ObMemtableData mtd(new_row.row_flag_.get_dml_flag(), len, buf); ObTxNodeArg arg( ctx.mvcc_acc_ctx_.tx_id_, /*trans id*/ &mtd, /*memtable_data*/ @@ -2657,24 +2655,27 @@ int ObMemtable::set_( init_timestamp_, /*memstore_version*/ write_seq, /*seq_no*/ write_epoch, /*write_epoch*/ - new_row.row_val_.count_ /*column_cnt*/); - if (OB_FAIL(mvcc_write_(param, + new_row.count_ /*column_cnt*/); + if (OB_FAIL(memtable_key_generator.generate_memtable_key(new_row))) { + TRANS_LOG(WARN, "generate memtable key fail", K(ret), K(new_row)); + } else if (OB_FAIL(mvcc_write_(param, context, - &mtk, + memtable_key_generator.get_memtable_key(), arg, check_exist, is_new_locked, + memtable_key_generator.get_key_buffer(), mvcc_row))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret && OB_ERR_PRIMARY_KEY_DUPLICATE != ret) { - TRANS_LOG(WARN, "mvcc write fail", K(mtk), K(ret)); + TRANS_LOG(WARN, "mvcc write fail", K(memtable_key_generator.get_memtable_key()), K(ret)); } } else { TRANS_LOG(DEBUG, "set end, success", "ret", ret, "tablet_id_", key_.tablet_id_, - "dml_flag", new_row.flag_.get_dml_flag(), + "dml_flag", new_row.row_flag_.get_dml_flag(), "columns", strarray(columns), "old_row", to_cstring(old_row), "new_row", to_cstring(new_row), @@ -2711,9 +2712,8 @@ int ObMemtable::set_( //set_end(ctx.mvcc_acc_ctx_, ret); if (OB_SUCC(ret)) { set_max_data_schema_version(ctx.table_version_); - set_max_column_cnt(new_row.row_val_.count_); + set_max_column_cnt(new_row.count_); } - return ret; } @@ -2754,10 +2754,11 @@ int ObMemtable::lock_( rowkey.get_obj_cnt()); /*column_cnt*/ if (OB_FAIL(mvcc_write_(param, context, - &mtk, + mtk, arg, false, /*check_exist*/ is_new_locked, + nullptr, /*memtable_key_buffer*/ nullptr /*mvcc_row*/))) { } else if (OB_UNLIKELY(!is_new_locked)) { TRANS_LOG(DEBUG, "lock twice, no need to store lock trans node"); @@ -2810,10 +2811,11 @@ int ObMemtable::mvcc_replay_(storage::ObStoreCtx &ctx, int ObMemtable::mvcc_write_( const storage::ObTableIterParam ¶m, storage::ObTableAccessContext &context, - const ObMemtableKey *key, + const ObMemtableKey &key, const ObTxNodeArg &arg, const bool check_exist, bool &is_new_locked, + ObMemtableKeyGenerator::ObMemtableKeyBuffer *memtable_key_buffer, ObMvccRowAndWriteResult *mvcc_row) { int ret = OB_SUCCESS; @@ -2826,13 +2828,13 @@ int ObMemtable::mvcc_write_( SCN snapshot_version = ctx.mvcc_acc_ctx_.get_snapshot_version(); transaction::ObTxSnapshot &snapshot = ctx.mvcc_acc_ctx_.snapshot_; - if (OB_FAIL(mvcc_engine_.create_kv(key, + if (OB_FAIL(mvcc_engine_.create_kv(&key, // is_insert blocksstable::ObDmlFlag::DF_INSERT == arg.data_->dml_flag_, &stored_key, value, is_new_add))) { - TRANS_LOG(WARN, "create kv failed", K(ret), K(arg), K(*key)); + TRANS_LOG(WARN, "create kv failed", K(ret), K(arg), K(key)); } else if (OB_FAIL(mvcc_engine_.mvcc_write(ctx, snapshot, *value, @@ -2840,29 +2842,29 @@ int ObMemtable::mvcc_write_( res))) { if (OB_TRY_LOCK_ROW_CONFLICT == ret) { ret = post_row_write_conflict_(ctx.mvcc_acc_ctx_, - *key, + key, res.lock_state_, value->get_last_compact_cnt(), value->get_total_trans_node_cnt()); } else if (OB_TRANSACTION_SET_VIOLATION == ret) { - mem_ctx->on_tsc_retry(*key, + mem_ctx->on_tsc_retry(key, snapshot_version, value->get_max_trans_version(), value->get_max_trans_id()); } else if (OB_ERR_PRIMARY_KEY_DUPLICATE == ret) { - mem_ctx->on_key_duplication_retry(*key); + mem_ctx->on_key_duplication_retry(key); } else { TRANS_LOG(WARN, "mvcc write fail", K(ret)); } } else if (nullptr == mvcc_row && OB_FAIL(lock_row_on_frozen_stores_(param, arg, - key, + &key, check_exist, context, value, res))) { lock_row_on_frozen_stores_on_failure(arg.data_->dml_flag_, - *key, + key, ret, value, context, @@ -2890,6 +2892,8 @@ int ObMemtable::mvcc_write_( (void)mvcc_engine_.mvcc_undo(value); res.is_mvcc_undo_ = true; TRANS_LOG(WARN, "register row commit failed", K(ret)); + } else if (nullptr != memtable_key_buffer && OB_FAIL(memtable_key_buffer->push_back(stored_key))) { + TRANS_LOG(WARN, "push back stored memtable key into buffer failed", K(ret)); } else if (nullptr == mvcc_row && res.has_insert()) { (void)mvcc_engine_.finish_kv(res); /*****[for deadlock]*****/ @@ -2898,7 +2902,7 @@ int ObMemtable::mvcc_write_( // no need to detect deadlock for non-unique local index table } else { MTL(ObLockWaitMgr*)->set_hash_holder(key_.get_tablet_id(), - *key, + key, context.store_ctx_->mvcc_acc_ctx_.get_mem_ctx()->get_tx_id()); } /***********************/ diff --git a/src/storage/memtable/ob_memtable.h b/src/storage/memtable/ob_memtable.h index dd05d9922..27668e588 100644 --- a/src/storage/memtable/ob_memtable.h +++ b/src/storage/memtable/ob_memtable.h @@ -245,22 +245,22 @@ public: // derived from ObITable const storage::ObTableIterParam ¶m, storage::ObTableAccessContext &context, const common::ObIArray &columns, // TODO: remove columns - const storage::ObStoreRow &row, + blocksstable::ObDatumRow &row, const share::ObEncryptMeta *encrypt_meta, const bool check_exist); virtual int set( const storage::ObTableIterParam ¶m, storage::ObTableAccessContext &context, - const common::ObIArray &columns, // TODO: remove columns + const common::ObIArray &columns, const ObIArray &update_idx, - const storage::ObStoreRow &old_row, - const storage::ObStoreRow &new_row, + const blocksstable::ObDatumRow &old_row, + blocksstable::ObDatumRow &new_row, const share::ObEncryptMeta *encrypt_meta); int multi_set( const storage::ObTableIterParam ¶m, storage::ObTableAccessContext &context, const common::ObIArray &columns, - const storage::ObStoreRow *rows, + blocksstable::ObDatumRow *rows, const int64_t row_count, const bool check_exist, const share::ObEncryptMeta *encrypt_meta, @@ -287,7 +287,8 @@ public: // derived from ObITable virtual int lock( const storage::ObTableIterParam ¶m, storage::ObTableAccessContext &context, - const common::ObNewRow &row); + ObColDescArray &col_desc, + blocksstable::ObDatumRow &row); virtual int lock( const storage::ObTableIterParam ¶m, storage::ObTableAccessContext &context, @@ -476,10 +477,11 @@ private: int mvcc_write_( const storage::ObTableIterParam ¶m, storage::ObTableAccessContext &context, - const ObMemtableKey *key, + const ObMemtableKey &key, const ObTxNodeArg &arg, const bool check_exist, bool &is_new_locked, + ObMemtableKeyGenerator::ObMemtableKeyBuffer *memtable_key_buffer, ObMvccRowAndWriteResult *mvcc_row = nullptr); int mvcc_replay_(storage::ObStoreCtx &ctx, @@ -513,7 +515,6 @@ private: int lock_rows_on_frozen_stores_( const bool check_exist, const storage::ObTableIterParam ¶m, - const ObMemtableKeyGenerator &memtable_keys, storage::ObTableAccessContext &context, ObMvccRowAndWriteResults &mvcc_rows, ObRowsInfo &rows_info); @@ -544,20 +545,19 @@ private: int set_( const storage::ObTableIterParam ¶m, const common::ObIArray &columns, - const storage::ObStoreRow &new_row, - const storage::ObStoreRow *old_row, + const blocksstable::ObDatumRow &new_row, + const blocksstable::ObDatumRow *old_row, const common::ObIArray *update_idx, - const ObMemtableKey &mtk, const bool check_exist, storage::ObTableAccessContext &context, + ObMemtableKeyGenerator &memtable_key_generator, ObMvccRowAndWriteResult *mvcc_row = nullptr); int multi_set_( const storage::ObTableIterParam ¶m, const common::ObIArray &columns, - const storage::ObStoreRow *rows, + const blocksstable::ObDatumRow *rows, const int64_t row_count, const bool check_exist, - const ObMemtableKeyGenerator &memtable_keys, storage::ObTableAccessContext &context, storage::ObRowsInfo &rows_info); int lock_( diff --git a/src/storage/memtable/ob_memtable_key.cpp b/src/storage/memtable/ob_memtable_key.cpp index 847d70e77..f80bede5c 100644 --- a/src/storage/memtable/ob_memtable_key.cpp +++ b/src/storage/memtable/ob_memtable_key.cpp @@ -1,5 +1,5 @@ /** - * Copyright (c) 2021 OceanBase + * Copyright (c) 2024 OceanBase * OceanBase CE is licensed under Mulan PubL v2. * You can use this software according to the terms and conditions of the Mulan PubL v2. * You may obtain a copy of Mulan PubL v2 at: @@ -11,110 +11,56 @@ */ #include "ob_memtable_key.h" -#include "lib/ob_errno.h" -#include "rowkey/ob_rowkey.h" -#include "share/rc/ob_tenant_base.h" -#include "storage/ob_i_store.h" namespace oceanbase { namespace memtable { - -constexpr int64_t ObMemtableKeyGenerator::STACK_BUFFER_SIZE; - -int ObMemtableKeyGenerator::init(const storage::ObStoreRow *rows, - const int64_t row_count, - const int64_t schema_rowkey_count, - const common::ObIArray &columns) +int ObMemtableKeyGenerator::init() { int ret = OB_SUCCESS; - if (size_ != 0) { + if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; + TRANS_LOG(WARN, "init ObMemtableKeyGenerator twice", K(ret)); + } else if (columns_.count() < rowkey_cnt_) { + ret = OB_INVALID_ARGUMENT; + TRANS_LOG(WARN, "rowkey number mismatched", K(columns_.count()), K(rowkey_cnt_)); + } else if (OB_FAIL(obj_buf_.init(&allocator_))) { + TRANS_LOG(WARN, "init obj buffer fail", K(ret)); + } else if (OB_FAIL(obj_buf_.reserve(rowkey_cnt_))) { + TRANS_LOG(WARN, "reserve obj buffer fail", K(ret), K(rowkey_cnt_)); } else { - int64_t extra_size = row_count - STACK_BUFFER_SIZE; - if (extra_size > 0) { - if (FALSE_IT(p_extra_store_row_keys_ = - (ObStoreRowkey *)share::mtl_malloc(extra_size * (sizeof(ObStoreRowkey)), "MemTableKey"))) { - } else if (OB_ISNULL(p_extra_store_row_keys_)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - TRANS_LOG(WARN, "Failed to alloc ObStoreRowkey memory", K(ret), K(row_count), K(schema_rowkey_count), K(columns), KP(MTL_CTX()), - KP(lib::ObMallocAllocator::get_instance()), KP(p_extra_store_row_keys_)); - } else if (FALSE_IT(p_extra_memtable_keys_ = - (ObMemtableKey *)share::mtl_malloc(extra_size * (sizeof(ObMemtableKey)), "MemTableKey"))) { - } else if (OB_ISNULL(p_extra_memtable_keys_)) { - share::mtl_free(p_extra_store_row_keys_); - p_extra_store_row_keys_ = nullptr; - ret = OB_ALLOCATE_MEMORY_FAILED; - TRANS_LOG(WARN, "Failed to alloc ObMemtableKey memory", K(ret), K(row_count), K(schema_rowkey_count), K(columns), KP(MTL_CTX()), - KP(lib::ObMallocAllocator::get_instance()), KP(p_extra_store_row_keys_)); + is_inited_ = true; + } + return ret; +} + +int ObMemtableKeyGenerator::generate_memtable_key(const blocksstable::ObDatumRow &row) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + TRANS_LOG(WARN, "ObMemtableKeyGenerator not inited", K(ret)); + } else if (OB_UNLIKELY(!row.is_valid())) { + ret = OB_INVALID_ARGUMENT; + TRANS_LOG(WARN, "invalid argument", K(ret), K(row)); + } else { + ObObj *objs = obj_buf_.get_data(); + for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_cnt_; i++) { + if (OB_FAIL(row.storage_datums_[i].to_obj_enhance(objs[i], columns_.at(i).col_type_))) { + TRANS_LOG(WARN, "failed to transfer datum to obj", K(ret), K(i), K(row)); } } - for (int i = 0; i < row_count && OB_SUCC(ret); ++i) { - ObStoreRowkey *p_store_row_key = i < STACK_BUFFER_SIZE ? &store_row_key_buffer_[i] : &p_extra_store_row_keys_[i - STACK_BUFFER_SIZE]; - ObMemtableKey *p_memtable_key = i < STACK_BUFFER_SIZE ? &memtable_key_buffer_[i] : &p_extra_memtable_keys_[i - STACK_BUFFER_SIZE]; - new (p_store_row_key) ObStoreRowkey(); - new (p_memtable_key) ObMemtableKey(); - if (OB_FAIL(p_store_row_key->assign(rows[i].row_val_.cells_, schema_rowkey_count))) { - p_store_row_key->~ObStoreRowkey(); - TRANS_LOG(WARN, "Failed to assign tmp rowkey", K(ret), K(rows[i]), K(row_count), K(schema_rowkey_count)); - } else if (OB_FAIL(p_memtable_key->encode(columns, p_store_row_key))) { - p_memtable_key->~ObMemtableKey(); - p_store_row_key->~ObStoreRowkey(); - TRANS_LOG(WARN, "mtk encode fail", K(ret), K(rows[i]), K(row_count), K(schema_rowkey_count)); - } else { - size_++; + if (OB_SUCC(ret)) { + if (OB_FAIL(store_rowkey_.assign(objs, rowkey_cnt_))) { + TRANS_LOG(WARN, "failed to assign rowkey", K(ret), K(row), K(objs), K(rowkey_cnt_)); + } else if (OB_FAIL(memtable_key_.encode(columns_, &store_rowkey_))) { + TRANS_LOG(WARN, "memtable key encode failed", K(ret), K(row), K(columns_), K(store_rowkey_)); } } - if (OB_FAIL(ret)) { - reset(); - } } return ret; } -ObMemtableKey &ObMemtableKeyGenerator::operator[](int64_t idx) { - ObMemtableKey *element = nullptr; - if (OB_UNLIKELY(idx < 0 || idx >= size_)) { - ob_abort(); - } - if (idx < STACK_BUFFER_SIZE) { - element = &memtable_key_buffer_[idx]; - } else { - element = &p_extra_memtable_keys_[idx - STACK_BUFFER_SIZE]; - } - return *element; -} - -const ObMemtableKey &ObMemtableKeyGenerator::operator[](int64_t idx) const -{ - return const_cast(this)->operator[](idx); -} - -void ObMemtableKeyGenerator::reset() -{ - int64_t idx = size_ - 1; - for(; idx >= STACK_BUFFER_SIZE; --idx) { - p_extra_memtable_keys_[idx - STACK_BUFFER_SIZE].~ObMemtableKey(); - p_extra_store_row_keys_[idx - STACK_BUFFER_SIZE].~ObStoreRowkey(); - } - for(; idx >= 0; --idx) { - memtable_key_buffer_[idx].~ObMemtableKey(); - store_row_key_buffer_[idx].~ObStoreRowkey(); - } - if (OB_UNLIKELY(nullptr != p_extra_memtable_keys_)) { - share::mtl_free(p_extra_memtable_keys_); - } - if (OB_UNLIKELY(nullptr != p_extra_store_row_keys_)) { - share::mtl_free(p_extra_store_row_keys_); - } - new (this) ObMemtableKeyGenerator(); -} - -ObMemtableKeyGenerator::~ObMemtableKeyGenerator() -{ - reset(); -} - -} -} \ No newline at end of file +} // namespace memtable +} // namespace oceanbase \ No newline at end of file diff --git a/src/storage/memtable/ob_memtable_key.h b/src/storage/memtable/ob_memtable_key.h index 22386ee40..882487245 100644 --- a/src/storage/memtable/ob_memtable_key.h +++ b/src/storage/memtable/ob_memtable_key.h @@ -283,7 +283,7 @@ public: private: common::ObStoreRowkey *rowkey_; mutable uint64_t hash_val_; // Perf optimization. - DISALLOW_COPY_AND_ASSIGN(ObMemtableKey); + //DISALLOW_COPY_AND_ASSIGN(ObMemtableKey); }; class ObStoreRowkeyWrapper @@ -308,31 +308,38 @@ public: const common::ObStoreRowkey *rowkey_; }; - -// this is for multi_set pre alloc memory to generate memtable key -class ObMemtableKeyGenerator {// RAII - static constexpr int64_t STACK_BUFFER_SIZE = 32; +// TODO(xuanxi): remove it later +class ObMemtableKeyGenerator { public: - ObMemtableKeyGenerator() : p_extra_store_row_keys_(nullptr), p_extra_memtable_keys_(nullptr), size_(0) {} - ~ObMemtableKeyGenerator(); - int init(const storage::ObStoreRow *rows, - const int64_t row_count, - const int64_t schema_rowkey_count, - const common::ObIArray &columns); - void reset(); - int64_t count() const { return size_; } - ObMemtableKey &operator[](int64_t idx); - const ObMemtableKey &operator[](int64_t idx) const; + using ObMemtableKeyBuffer = common::ObSEArray; +public: + ObMemtableKeyGenerator( + const int64_t rowkey_cnt, + const common::ObIArray &columns, + ObMemtableKeyBuffer *memtable_key_buffer = nullptr) + : allocator_(common::ObMemAttr(MTL_ID(), "ObMemtableKey")), + rowkey_cnt_(rowkey_cnt), + columns_(columns), + memtable_key_buffer_(memtable_key_buffer), + is_inited_(false) + {} + ~ObMemtableKeyGenerator() = default; + int init(); + int generate_memtable_key(const blocksstable::ObDatumRow &datum_row); + ObMemtableKey &get_memtable_key() { return memtable_key_; } + ObMemtableKeyBuffer *get_key_buffer() { return memtable_key_buffer_; } private: - // this is for avoid memory allocation when rows not so much - ObStoreRowkey store_row_key_buffer_[STACK_BUFFER_SIZE]; - ObMemtableKey memtable_key_buffer_[STACK_BUFFER_SIZE]; - ObStoreRowkey *p_extra_store_row_keys_; - ObMemtableKey *p_extra_memtable_keys_; - int64_t size_; + ObArenaAllocator allocator_; + int64_t rowkey_cnt_; + const common::ObIArray &columns_; + storage::ObObjBufArray obj_buf_; + ObStoreRowkey store_rowkey_; + ObMemtableKey memtable_key_; + ObMemtableKeyBuffer *memtable_key_buffer_; + bool is_inited_; }; -} -} +} // namespace memtable +} // namespace oceanbase #endif // OCEANBASE_MEMTABLE_OB_MEMTABLE_KEY2_ diff --git a/src/storage/ob_dml_running_ctx.cpp b/src/storage/ob_dml_running_ctx.cpp index 7e802090a..41b3f56cc 100644 --- a/src/storage/ob_dml_running_ctx.cpp +++ b/src/storage/ob_dml_running_ctx.cpp @@ -35,7 +35,8 @@ ObDMLRunningCtx::ObDMLRunningCtx( ObStoreCtx &store_ctx, const ObDMLBaseParam &dml_param, common::ObIAllocator &allocator, - const blocksstable::ObDmlFlag dml_flag) + const blocksstable::ObDmlFlag dml_flag, + bool is_need_row_datum_utils) : store_ctx_(store_ctx), dml_param_(dml_param), allocator_(allocator), @@ -44,13 +45,21 @@ ObDMLRunningCtx::ObDMLRunningCtx( col_map_(nullptr), col_descs_(nullptr), column_ids_(nullptr), - tbl_row_(), + datum_row_(), + cmp_funcs_(), is_old_row_valid_for_lob_(false), + is_need_check_old_row_(is_need_row_datum_utils), + is_udf_(false), schema_guard_(share::schema::ObSchemaMgrItem::MOD_RELATIVE_TABLE), + is_need_row_datum_utils_(is_need_row_datum_utils), is_inited_(false) { } +ObDMLRunningCtx::~ObDMLRunningCtx() +{ +} + int ObDMLRunningCtx::init( const common::ObIArray *column_ids, const common::ObIArray *upd_col_ids, @@ -89,6 +98,10 @@ int ObDMLRunningCtx::init( LOG_WARN("failed to get relative table", K(ret), K(dml_param_)); } else if (NULL != column_ids && OB_FAIL(prepare_column_info(*column_ids))) { LOG_WARN("fail to get column descriptions and column map", K(ret), K(*column_ids)); + } else if (is_need_check_old_row_ && OB_FAIL(check_need_old_row_legitimacy())) { + LOG_WARN("fail to get flag of checking old row legitimacy", K(ret)); + } else if (is_need_check_old_row_ && OB_FAIL(init_cmp_funcs())) { + LOG_WARN("fail to init compare functions", K(ret)); } else { store_ctx_.mvcc_acc_ctx_.mem_ctx_->set_table_version(dml_param_.schema_version_); store_ctx_.table_version_ = dml_param_.schema_version_; @@ -161,6 +174,101 @@ int ObDMLRunningCtx::prepare_column_info(const common::ObIArray &colum return ret; } +int ObDMLRunningCtx::check_need_old_row_legitimacy() +{ + int ret = OB_SUCCESS; + // TODO(jingxing): setting this to true + if (OB_FAIL(relative_table_.has_udf_column(is_need_check_old_row_))) { + LOG_WARN("check has udf column failed", K(ret)); + } else if (is_need_check_old_row_) { + is_udf_ = true; + ObTableStoreIterator &table_iter = *relative_table_.tablet_iter_.table_iter(); + while (OB_SUCC(ret) && !is_need_check_old_row_) { + ObITable *table_ptr = nullptr; + if (OB_FAIL(table_iter.get_next(table_ptr))) { + if (OB_ITER_END != ret) { + LOG_WARN("get next table failed", K(ret)); + } + } else if (OB_ISNULL(table_ptr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("error unexpected, table ptr must not be nullptr", K(ret)); + } else { + is_need_check_old_row_ = table_ptr->is_major_sstable(); + } + } + } else if (dml_param_.is_batch_stmt_ && !relative_table_.is_index_table()) { + //batch stmt execution dependency defensive check to check + //if the same row was modified multiple times + is_need_check_old_row_ = true; + ret = OB_E(EventTable::EN_INS_MULTI_VALUES_BATCH_OPT) OB_SUCCESS; + // no need to check old row, just for bmsql performance optimization + // TODO yuchen.ywc + if (OB_SUCCESS != ret) { + LOG_INFO("error sim when current statement is batch update", K(ret), K_(is_udf)); + is_need_check_old_row_ = false; + ret = OB_SUCCESS; + } + } else if (GCONF.enable_defensive_check()) { + is_need_check_old_row_ = true; + if (relative_table_.is_index_table() && !relative_table_.can_read_index()) { + //index can not be read during building index, so does not check old index row + is_need_check_old_row_ = false; + } + if (ObDmlFlag::DF_LOCK == dml_flag_) { + is_need_check_old_row_ = false; + } + } + return ret; +} + +int ObDMLRunningCtx::init_cmp_funcs() +{ + int ret = OB_SUCCESS; + const common::ObIArray &col_descs = dml_param_.table_param_->get_col_descs(); + int64_t column_cnt = col_descs.count(); + if (OB_UNLIKELY(column_cnt < 0 || column_cnt > OB_ROW_MAX_COLUMNS_COUNT)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Invalid argument to init compare functions", K(ret), K(column_cnt), K(col_descs)); + } else if (OB_FAIL(cmp_funcs_.init(column_cnt, allocator_))) { + STORAGE_LOG(WARN, "Failed to reserve cmp func array", K(ret)); + } else { + bool is_oracle_mode = lib::is_oracle_mode(); + ObCmpFunc cmp_func; + for (int64_t i = 0; OB_SUCC(ret) && i < col_descs.count(); i++) { + const share::schema::ObColDesc &col_desc = col_descs.at(i); + //TODO @hanhui support desc rowkey + bool is_ascending = true || col_desc.col_order_ == ObOrderType::ASC; + bool has_lob_header = is_lob_storage(col_desc.col_type_.get_type()); + ObPrecision precision = PRECISION_UNKNOWN_YET; + if (col_desc.col_type_.is_decimal_int()) { + precision = col_desc.col_type_.get_stored_precision(); + OB_ASSERT(precision != PRECISION_UNKNOWN_YET); + } + sql::ObExprBasicFuncs *basic_funcs = ObDatumFuncs::get_basic_func(col_desc.col_type_.get_type(), + col_desc.col_type_.get_collation_type(), + col_desc.col_type_.get_scale(), + is_oracle_mode, + has_lob_header, + precision); + if (OB_UNLIKELY(nullptr == basic_funcs || nullptr == basic_funcs->null_last_cmp_)) { + ret = OB_ERR_SYS; + STORAGE_LOG(ERROR, "Unexpected null basic funcs", K(ret), K(col_desc)); + } else { + if (is_ascending) { + cmp_func.cmp_func_ = is_oracle_mode ? basic_funcs->null_last_cmp_ : basic_funcs->null_first_cmp_; + if (OB_FAIL(cmp_funcs_.push_back(ObStorageDatumCmpFunc(cmp_func)))) { + STORAGE_LOG(WARN, "Failed to push back cmp func", K(ret), K(i), K(col_desc)); + } + } else { + ret = OB_ERR_SYS; + STORAGE_LOG(WARN, "Unsupported desc column order", K(ret), K(col_desc), K(i)); + } + } + } + } + return ret; +} + int ObDMLRunningCtx::check_schema_version( share::schema::ObMultiVersionSchemaService &schema_service, const uint64_t tenant_id, diff --git a/src/storage/ob_dml_running_ctx.h b/src/storage/ob_dml_running_ctx.h index bac8cf0d2..8af46f265 100644 --- a/src/storage/ob_dml_running_ctx.h +++ b/src/storage/ob_dml_running_ctx.h @@ -47,8 +47,9 @@ public: ObStoreCtx &store_ctx, const ObDMLBaseParam &dml_param, common::ObIAllocator &allocator, - const blocksstable::ObDmlFlag dml_flag); - ~ObDMLRunningCtx() {} + const blocksstable::ObDmlFlag dml_flag, + bool is_need_row_datum_utils = false); + ~ObDMLRunningCtx(); int init( const common::ObIArray *column_ids, @@ -65,6 +66,8 @@ private: const share::schema::ObTableSchemaParam &schema, ObTabletHandle &tablet_handle, const share::SCN &read_snapshot); + int check_need_old_row_legitimacy(); + int init_cmp_funcs(); int check_schema_version(share::schema::ObMultiVersionSchemaService &schema_service, const uint64_t tenant_id, const uint64_t table_id, @@ -86,11 +89,15 @@ public: const share::schema::ColumnMap *col_map_; const ObColDescIArray *col_descs_; const common::ObIArray *column_ids_; - ObStoreRow tbl_row_; + blocksstable::ObDatumRow datum_row_; + blocksstable::ObStoreCmpFuncs cmp_funcs_; bool is_old_row_valid_for_lob_; + bool is_need_check_old_row_; + bool is_udf_; private: share::schema::ObSchemaGetterGuard schema_guard_; + bool is_need_row_datum_utils_; bool is_inited_; }; } // namespace storage diff --git a/src/storage/ob_query_iterator_factory.cpp b/src/storage/ob_query_iterator_factory.cpp index 3aa650133..17f642aff 100644 --- a/src/storage/ob_query_iterator_factory.cpp +++ b/src/storage/ob_query_iterator_factory.cpp @@ -113,7 +113,7 @@ void ObQueryIteratorFactory::free_table_scan_iter(common::ObNewRowIterator *iter } } -void ObQueryIteratorFactory::free_insert_dup_iter(common::ObNewRowIterator *iter) +void ObQueryIteratorFactory::free_insert_dup_iter(blocksstable::ObDatumRowIterator *iter) { if (OB_LIKELY(NULL != iter)) { (void)ATOMIC_FAA(&insert_dup_release_count_, 1); diff --git a/src/storage/ob_query_iterator_factory.h b/src/storage/ob_query_iterator_factory.h index a86302775..708f60f1f 100644 --- a/src/storage/ob_query_iterator_factory.h +++ b/src/storage/ob_query_iterator_factory.h @@ -18,6 +18,10 @@ namespace oceanbase { +namespace blocksstable +{ +class ObDatumRowIterator; +} namespace storage { class ObMultipleScanMerge; @@ -37,7 +41,7 @@ public: static ObColMap *get_col_map(); static void free_table_scan_iter(common::ObNewRowIterator *iter); - static void free_insert_dup_iter(common::ObNewRowIterator *iter); + static void free_insert_dup_iter(blocksstable::ObDatumRowIterator *iter); static void free_merge_iter(ObQueryRowIterator *iter); static void free_col_map(ObColMap *col_map); static void free_work_row(ObStoreRow *row); diff --git a/src/storage/ob_value_row_iterator.cpp b/src/storage/ob_value_row_iterator.cpp index 124f2d377..9cebccf06 100644 --- a/src/storage/ob_value_row_iterator.cpp +++ b/src/storage/ob_value_row_iterator.cpp @@ -25,7 +25,7 @@ using namespace blocksstable; namespace storage { ObValueRowIterator::ObValueRowIterator() - : ObNewRowIterator(), + : ObDatumRowIterator(), is_inited_(false), unique_(false), allocator_("ObValueRowAlloc"), @@ -53,10 +53,9 @@ int ObValueRowIterator::init(bool unique) return ret; } -int ObValueRowIterator::add_row(common::ObNewRow &row) +int ObValueRowIterator::add_row(ObDatumRow &row, const ObStorageDatumUtils &rowkey_datum_utils) { int ret = OB_SUCCESS; - ObNewRow *cur_row = NULL; if (!row.is_valid()) { ret = OB_INVALID_ARGUMENT; STORAGE_LOG(WARN, "invalid row", K(ret), K(row)); @@ -71,29 +70,34 @@ int ObValueRowIterator::add_row(common::ObNewRow &row) //on multiple unique index is small, so there is usually only one row in the value row iterator //so using list traversal to deduplicate unique index is more efficiently //and also saves the CPU overhead that constructs the hash map - ObStoreRowkey rowkey; - ObStoreRowkey tmp_rowkey; - if (OB_FAIL(rowkey.assign(row.cells_, row.count_))) { + ObDatumRowkey rowkey; + ObDatumRowkey tmp_rowkey; + if (OB_FAIL(rowkey.assign(row.storage_datums_, row.count_))) { STORAGE_LOG(WARN, "Failed to assign rowkey", K(ret), K(row)); } for (int64_t i = 0; OB_SUCC(ret) && !exist && i < rows_.count(); ++i) { - if (OB_FAIL(tmp_rowkey.assign(rows_.at(i).cells_, rows_.at(i).count_))) { - STORAGE_LOG(WARN, "Failed to assign rowkey", K(ret), K(i), K(rows_.at(i))); - } else if (OB_UNLIKELY(tmp_rowkey == rowkey)) { - exist = true; + if (OB_FAIL(tmp_rowkey.assign(rows_.at(i)->storage_datums_, rows_.at(i)->count_))) { + STORAGE_LOG(WARN, "Failed to assign rowkey", K(ret), K(i), KPC(rows_.at(i))); + } else if (OB_FAIL(tmp_rowkey.equal(rowkey, rowkey_datum_utils, exist))) { + STORAGE_LOG(WARN, "Failed to compare rowkey", K(ret), K(i), K(tmp_rowkey), K(rowkey)); } } } // store non-exist row if (OB_SUCC(ret)) { if (!exist) { - if (NULL == (cur_row = rows_.alloc_place_holder())) { + ObDatumRow *cur_row = nullptr; + void *buff = nullptr; + if (OB_ISNULL(buff = allocator_.alloc(sizeof(ObDatumRow)))) { ret = OB_ALLOCATE_MEMORY_FAILED; - STORAGE_LOG(ERROR, "add row error", K(ret)); - } else if (OB_SUCCESS != (ret = ob_write_row(allocator_, - row, - *cur_row))) { + STORAGE_LOG(WARN, "alloc memory for datum row error", K(ret)); + } else if (FALSE_IT(cur_row = new (buff) ObDatumRow())) { + } else if (OB_FAIL(cur_row->init(allocator_, row.count_))) { + STORAGE_LOG(WARN, "init datum row error", K(ret), K(row), KPC(cur_row)); + } else if (OB_FAIL(cur_row->deep_copy(row, allocator_))) { STORAGE_LOG(WARN, "copy row error", K(ret), K(row)); + } else if (OB_FAIL(rows_.push_back(cur_row))) { + STORAGE_LOG(WARN, "fail to push datum row to iterator array", K(ret), K(cur_row)); } } } @@ -101,27 +105,14 @@ int ObValueRowIterator::add_row(common::ObNewRow &row) return ret; } -int ObValueRowIterator::get_next_row(common::ObNewRow *&row) +int ObValueRowIterator::get_next_row(ObDatumRow *&row) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; STORAGE_LOG(WARN, "ObValueRowIterator is not initialized", K(ret)); } else if (cur_idx_ < rows_.count()) { - row = &rows_.at(cur_idx_++); - } else { - ret = OB_ITER_END; - } - return ret; -} - -int ObValueRowIterator::get_next_rows(ObNewRow *&rows, int64_t &row_count) -{ - int ret = OB_SUCCESS; - if (cur_idx_ < rows_.count()) { - rows = &(rows_.at(cur_idx_)); - row_count = rows_.count() - cur_idx_; - cur_idx_ = rows_.count(); + row = rows_.at(cur_idx_++); } else { ret = OB_ITER_END; } @@ -143,23 +134,23 @@ ObSingleRowGetter::ObSingleRowGetter(ObIAllocator &allocator, ObTablet &tablet) store_ctx_(nullptr), output_projector_(allocator), relative_table_(nullptr), - table_param_(nullptr), allocator_(allocator), - new_row_builder_() + cached_iter_node_(nullptr) { } ObSingleRowGetter::~ObSingleRowGetter() { - if (single_merge_ != nullptr) { - single_merge_->~ObSingleMerge(); - allocator_.free(single_merge_); + if (nullptr != single_merge_) { + if (nullptr == cached_iter_node_) { + single_merge_->~ObSingleMerge(); + allocator_.free(single_merge_); + } single_merge_ = nullptr; } - if (table_param_ != nullptr) { - table_param_->~ObTableParam(); - allocator_.free(table_param_); - table_param_ = nullptr; + if (nullptr != cached_iter_node_) { + ObGlobalIteratorPool *iter_pool = MTL(ObGlobalIteratorPool*); + iter_pool->release(cached_iter_node_); } } @@ -171,6 +162,7 @@ int ObSingleRowGetter::init_dml_access_ctx( int ret = OB_SUCCESS; common::ObQueryFlag query_flag; common::ObVersionRange trans_version_range; + query_flag.set_not_use_bloomfilter_cache(); query_flag.read_latest_ = ObQueryFlag::OBSF_MASK_READ_LATEST; if (skip_read_lob) { query_flag.skip_read_lob_ = ObQueryFlag::OBSF_MASK_SKIP_READ_LOB; @@ -180,7 +172,7 @@ int ObSingleRowGetter::init_dml_access_ctx( trans_version_range.multi_version_start_ = 0; store_ctx_ = &store_ctx; - if (OB_FAIL(access_ctx_.init(query_flag, store_ctx, allocator_, trans_version_range))) { + if (OB_FAIL(access_ctx_.init(query_flag, store_ctx, allocator_, trans_version_range, cached_iter_node_))) { LOG_WARN("failed to init table access ctx", K(ret)); } return ret; @@ -233,18 +225,21 @@ int ObSingleRowGetter::init_dml_access_param(ObRelativeTable &relative_table, return ret; } -int ObSingleRowGetter::create_table_param() +int ObSingleRowGetter::prepare_cached_iter_node() { int ret = OB_SUCCESS; - void *buf = nullptr; - if (table_param_ != nullptr) { - ret = OB_INIT_TWICE; - LOG_WARN("init table param twice", K(ret)); - } else if (OB_ISNULL(buf = allocator_.alloc(sizeof(share::schema::ObTableParam)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("allocate table param failed", K(ret), K(sizeof(share::schema::ObTableParam))); - } else { - table_param_ = new(buf) share::schema::ObTableParam(allocator_); + if (OB_UNLIKELY(nullptr != cached_iter_node_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "Unexpected not null cached iter node", K(ret), KP(cached_iter_node_)); + } else if (can_use_global_iter_pool()) { + ObGlobalIteratorPool *iter_pool = MTL(ObGlobalIteratorPool*); + if (OB_FAIL(iter_pool->get(ITER_TYPE, cached_iter_node_))) { + STORAGE_LOG(WARN, "Failed to get from iter pool", K(ret)); + } else if (nullptr != cached_iter_node_) { + access_param_.set_use_global_iter_pool(); + access_param_.iter_param_.set_use_stmt_iter_pool(); + STORAGE_LOG(TRACE, "use global iter pool", K(access_param_)); + } } return ret; } @@ -252,28 +247,18 @@ int ObSingleRowGetter::create_table_param() int ObSingleRowGetter::open(const ObDatumRowkey &rowkey, bool use_fuse_row_cache) { int ret = OB_SUCCESS; - void *buf = nullptr; - const common::ObIArray * col_descs = nullptr; - - if (OB_ISNULL(buf = allocator_.alloc(sizeof(ObSingleMerge)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("Fail to allocate memory for multi get merge ", K(ret)); - } else { - { - ObStorageTableGuard guard(tablet_, *store_ctx_, false); - if (OB_FAIL(guard.refresh_and_protect_memtable_for_write(*relative_table_))) { - STORAGE_LOG(WARN, "fail to protect table", K(ret)); - } + { + ObStorageTableGuard guard(tablet_, *store_ctx_, false); + if (OB_FAIL(guard.refresh_and_protect_memtable_for_write(*relative_table_))) { + STORAGE_LOG(WARN, "fail to protect table", K(ret)); } - single_merge_ = new(buf) ObSingleMerge(); } if (OB_SUCC(ret)) { - if (OB_FAIL(single_merge_->init(access_param_, access_ctx_, get_table_param_))) { - STORAGE_LOG(WARN, "Fail to init ObSingleMerge, ", K(ret)); + ACTIVE_GLOBAL_ITERATOR_GUARD(ret, cached_iter_node_); + if (OB_FAIL(init_single_merge())) { + STORAGE_LOG(WARN, "Fail to init ObSingleMerge", K(ret)); } else if (OB_FAIL(single_merge_->open(rowkey))) { - STORAGE_LOG(WARN, "Fail to open iter, ", K(ret)); - } else if (OB_FAIL(new_row_builder_.init(single_merge_->get_out_project_cells(), allocator_))) { - LOG_WARN("Failed to init ObNewRowBuilder", K(ret)); + STORAGE_LOG(WARN, "Fail to open iter", K(ret)); } if (use_fuse_row_cache) { access_ctx_.use_fuse_row_cache_ = true; @@ -282,9 +267,10 @@ int ObSingleRowGetter::open(const ObDatumRowkey &rowkey, bool use_fuse_row_cache return ret; } -int ObSingleRowGetter::get_next_row(ObNewRow *&row) +int ObSingleRowGetter::get_next_row(blocksstable::ObDatumRow *&row) { int ret = OB_SUCCESS; + ACTIVE_GLOBAL_ITERATOR_GUARD(ret, cached_iter_node_); row = nullptr; while (OB_SUCC(ret)) { blocksstable::ObDatumRow *store_row = NULL; @@ -293,9 +279,7 @@ int ObSingleRowGetter::get_next_row(ObNewRow *&row) STORAGE_LOG(WARN, "failed to get next row", K(ret)); } } else if (store_row->row_flag_.is_exist_without_delete()) { - if (OB_FAIL(new_row_builder_.build(*store_row, row))) { - STORAGE_LOG(WARN, "Failed to build new row", K(ret), KPC(store_row)); - } + row = store_row; break; } } @@ -319,5 +303,58 @@ int ObSingleRowGetter::get_next_row(ObNewRow *&row) } return ret; } + +bool ObSingleRowGetter::can_use_global_iter_pool() const +{ + bool use_pool = false; + if (access_param_.iter_param_.tablet_id_.is_inner_tablet()) { + } else if (access_param_.iter_param_.has_lob_column_out_) { + } else { + const int64_t table_cnt = get_table_param_.tablet_iter_.table_iter()->count(); + const int64_t col_cnt = get_table_param_.tablet_iter_.get_tablet()->get_rowkey_read_info().get_schema_column_count(); + ObGlobalIteratorPool *iter_pool = MTL(ObGlobalIteratorPool*); + if (OB_NOT_NULL(iter_pool)) { + use_pool = iter_pool->can_use_iter_pool(table_cnt, col_cnt, ITER_TYPE); + } + } + return use_pool; +} + +int ObSingleRowGetter::init_single_merge() +{ + int ret = OB_SUCCESS; + ObQueryRowIterator *cached_iter = nullptr == cached_iter_node_ ? nullptr : cached_iter_node_->get_iter(); + if (OB_NOT_NULL(cached_iter)) { + if (OB_UNLIKELY(cached_iter->get_type() != ITER_TYPE)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "Unexpected cached iter type", K(ret), K(cached_iter->get_type())); + } else { + single_merge_ = static_cast(cached_iter); + } + } + if (OB_FAIL(ret)) { + } else if (nullptr == single_merge_) { + void *buf = nullptr; + if (OB_ISNULL(buf = access_ctx_.get_long_life_allocator()->alloc(sizeof(ObSingleMerge)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + STORAGE_LOG(WARN, "Fail to allocate memory", K(ret)); + } else { + single_merge_ = new (buf) ObSingleMerge(); + if (OB_FAIL(single_merge_->init(access_param_, access_ctx_, get_table_param_))) { + STORAGE_LOG(WARN, "Failed to init multiple merge", K(ret)); + } + if (OB_FAIL(ret)) { + single_merge_->~ObSingleMerge(); + access_ctx_.get_long_life_allocator()->free(single_merge_); + single_merge_ = nullptr; + } else if (nullptr != cached_iter_node_) { + cached_iter_node_->set_iter(single_merge_); + } + } + } else if (OB_FAIL(single_merge_->switch_table(access_param_, access_ctx_, get_table_param_))) { + STORAGE_LOG(WARN, "Failed to switch table", K(ret), K(access_param_)); + } + return ret; +} } // end namespace storage } // end namespace oceanbase diff --git a/src/storage/ob_value_row_iterator.h b/src/storage/ob_value_row_iterator.h index 64ff40253..903db50f3 100644 --- a/src/storage/ob_value_row_iterator.h +++ b/src/storage/ob_value_row_iterator.h @@ -22,6 +22,7 @@ #include "storage/ob_i_store.h" #include "storage/access/ob_dml_param.h" #include "blocksstable/ob_datum_rowkey.h" +#include "blocksstable/ob_datum_row_iterator.h" namespace oceanbase { @@ -29,17 +30,16 @@ namespace storage { class ObTablet; -class ObValueRowIterator : public common::ObNewRowIterator +class ObValueRowIterator : public blocksstable::ObDatumRowIterator { static const int64_t DEFAULT_ROW_NUM = 2; - typedef common::ObSEArray RowArray; + typedef common::ObSEArray RowArray; public: ObValueRowIterator(); virtual ~ObValueRowIterator(); virtual int init(bool unique); - virtual int get_next_row(common::ObNewRow *&row); - virtual int get_next_rows(common::ObNewRow *&rows, int64_t &row_count); - virtual int add_row(common::ObNewRow &row); + virtual int get_next_row(blocksstable::ObDatumRow *&row); + virtual int add_row(blocksstable::ObDatumRow &row, const blocksstable::ObStorageDatumUtils &rowkey_datum_utils); virtual void reset(); private: bool is_inited_; @@ -55,6 +55,7 @@ class ObSingleMerge; class ObSingleRowGetter { typedef common::ObFixedArray Projector; + const ObQRIterType ITER_TYPE = T_SINGLE_GET; public: ObSingleRowGetter(common::ObIAllocator &allocator, ObTablet &tablet); ~ObSingleRowGetter(); @@ -68,13 +69,15 @@ public: const ObDMLBaseParam &dml_param, const common::ObIArray &out_col_ids, const bool skip_read_lob = false); + int prepare_cached_iter_node(); ObTableAccessParam &get_access_param() { return access_param_; } ObTableAccessContext &get_access_ctx() { return access_ctx_; } void set_relative_table(ObRelativeTable *relative_table) { relative_table_ = relative_table; } int open(const blocksstable::ObDatumRowkey &rowkey, bool use_fuse_row_cache = false); - int get_next_row(common::ObNewRow *&row); + int get_next_row(blocksstable::ObDatumRow *&row); private: - int create_table_param(); + bool can_use_global_iter_pool() const; + int init_single_merge(); private: ObTablet *tablet_; ObSingleMerge *single_merge_; @@ -84,9 +87,8 @@ private: ObTableAccessContext access_ctx_; ObGetTableParam get_table_param_; ObRelativeTable *relative_table_; - share::schema::ObTableParam *table_param_; common::ObIAllocator &allocator_; - blocksstable::ObNewRowBuilder new_row_builder_; + CachedIteratorNode *cached_iter_node_; }; } // end namespace storage } // end namespace oceanbase diff --git a/src/storage/tablet/ob_tablet.cpp b/src/storage/tablet/ob_tablet.cpp index 7fbcbf133..7ffa156c8 100644 --- a/src/storage/tablet/ob_tablet.cpp +++ b/src/storage/tablet/ob_tablet.cpp @@ -91,6 +91,7 @@ #include "storage/tablet/ob_tablet_mds_table_mini_merger.h" #include "storage/blocksstable/ob_shared_macro_block_manager.h" #include "storage/ob_direct_load_table_guard.h" +#include "storage/blocksstable/ob_datum_row_utils.h" namespace oceanbase { @@ -3639,7 +3640,8 @@ int ObTablet::check_schema_version_for_bounded_staleness_read( int ObTablet::lock_row( ObRelativeTable &relative_table, ObStoreCtx &store_ctx, - const common::ObNewRow &row) + ObColDescArray &col_desc, + blocksstable::ObDatumRow &row) { int ret = OB_SUCCESS; ObStorageTableGuard guard(this, store_ctx, true); @@ -3669,7 +3671,7 @@ int ObTablet::lock_row( LOG_WARN("prepare write memtable fail", K(ret), K(relative_table)); } else if (OB_FAIL(prepare_param_ctx(allocator, relative_table, store_ctx, param, context))) { LOG_WARN("prepare param ctx fail, ", K(ret)); - } else if (OB_FAIL(write_memtable->lock(param, context, row))) { + } else if (OB_FAIL(write_memtable->lock(param, context, col_desc, row))) { LOG_WARN("failed to lock write_memtable", K(ret), K(row)); } } @@ -4101,8 +4103,8 @@ int ObTablet::update_row( storage::ObStoreCtx &store_ctx, const common::ObIArray &col_descs, const ObIArray &update_idx, - const storage::ObStoreRow &old_row, - const storage::ObStoreRow &new_row, + const blocksstable::ObDatumRow &old_row, + blocksstable::ObDatumRow &new_row, const common::ObIArray *encrypt_meta_arr) { int ret = OB_SUCCESS; @@ -4159,7 +4161,7 @@ int ObTablet::update_row( int ObTablet::insert_rows( ObRelativeTable &relative_table, ObStoreCtx &store_ctx, - ObStoreRow *rows, + ObDatumRow *rows, ObRowsInfo &rows_info, const bool check_exist, const ObColDescIArray &col_descs, @@ -4209,7 +4211,7 @@ int ObTablet::insert_row_without_rowkey_check( ObStoreCtx &store_ctx, const bool check_exist, const common::ObIArray &col_descs, - const storage::ObStoreRow &row, + blocksstable::ObDatumRow &row, const common::ObIArray *encrypt_meta_arr) { int ret = OB_SUCCESS; @@ -4373,7 +4375,8 @@ int ObTablet::do_rowkeys_exist( int ObTablet::rowkey_exists( ObRelativeTable &relative_table, ObStoreCtx &store_ctx, - const common::ObNewRow &row, + const ObColDescIArray &col_descs, + ObDatumRow &row, bool &exists) { int ret = OB_SUCCESS; @@ -4399,24 +4402,22 @@ int ObTablet::rowkey_exists( } if (OB_SUCC(ret)) { - ObStoreRowkey rowkey; ObDatumRowkey datum_rowkey; ObDatumRowkeyHelper rowkey_helper; ObArenaAllocator allocator(common::ObMemAttr(MTL_ID(), "rowkey_acc_ctx")); ObTableIterParam param; ObTableAccessContext context; - if (OB_FAIL(rowkey.assign(row.cells_, relative_table.get_rowkey_column_num()))) { - LOG_WARN("Failed to assign rowkey", K(ret), K(row)); - } else if (OB_FAIL(rowkey_helper.convert_datum_rowkey(rowkey.get_rowkey(), datum_rowkey))) { - LOG_WARN("Failed to transfer datum rowkey", K(ret), K(rowkey)); + if (OB_FAIL(rowkey_helper.prepare_datum_rowkey(row, relative_table.get_rowkey_column_num(), + col_descs, datum_rowkey))) { + LOG_WARN("Failed to prepare rowkey", K(ret), K(row)); } else if (OB_FAIL(prepare_param_ctx(allocator, relative_table, store_ctx, param, context))) { - LOG_WARN("Failed to prepare param ctx, ", K(ret), K(rowkey)); + LOG_WARN("Failed to prepare param ctx, ", K(ret), K(row)); } else if (OB_FAIL(relative_table.tablet_iter_.get_tablet()->do_rowkey_exists( param, context, datum_rowkey, exists))) { - LOG_WARN("do rowkey exist fail", K(ret), K(rowkey)); + LOG_WARN("do rowkey exist fail", K(ret), K(row), K(datum_rowkey)); } - LOG_DEBUG("chaser debug row", K(ret), K(row), K(rowkey)); + LOG_DEBUG("chaser debug row", K(ret), K(row)); } } return ret; diff --git a/src/storage/tablet/ob_tablet.h b/src/storage/tablet/ob_tablet.h index 2418aa34c..50e2abad8 100644 --- a/src/storage/tablet/ob_tablet.h +++ b/src/storage/tablet/ob_tablet.h @@ -314,7 +314,7 @@ public: int insert_rows( ObRelativeTable &relative_table, ObStoreCtx &store_ctx, - ObStoreRow *rows, + blocksstable::ObDatumRow *rows, ObRowsInfo &rows_info, const bool check_exist, const ObColDescIArray &col_descs, @@ -325,20 +325,21 @@ public: ObStoreCtx &store_ctx, const bool check_exist, const ObColDescIArray &col_descs, - const storage::ObStoreRow &row, + blocksstable::ObDatumRow &row, const common::ObIArray *encrypt_meta_arr); int update_row( ObRelativeTable &relative_table, ObStoreCtx &store_ctx, const ObColDescIArray &col_descs, const ObIArray &update_idx, - const storage::ObStoreRow &old_row, - const storage::ObStoreRow &new_row, + const blocksstable::ObDatumRow &old_row, + blocksstable::ObDatumRow &new_row, const common::ObIArray *encrypt_meta_arr); int lock_row( ObRelativeTable &relative_table, ObStoreCtx &store_ctx, - const common::ObNewRow &row); + ObColDescArray &col_desc, + blocksstable::ObDatumRow &row); int lock_row( ObRelativeTable &relative_table, ObStoreCtx &store_ctx, @@ -399,7 +400,8 @@ public: int rowkey_exists( ObRelativeTable &relative_table, ObStoreCtx &store_ctx, - const common::ObNewRow &row, + const ObColDescIArray &col_descs, + blocksstable::ObDatumRow &row, bool &exists); int rowkeys_exists( ObStoreCtx &store_ctx, diff --git a/src/storage/tablet/ob_tablet_table_store_iterator.cpp b/src/storage/tablet/ob_tablet_table_store_iterator.cpp index fa64bcff8..4f43b26e2 100644 --- a/src/storage/tablet/ob_tablet_table_store_iterator.cpp +++ b/src/storage/tablet/ob_tablet_table_store_iterator.cpp @@ -96,6 +96,7 @@ void ObTableStoreIterator::reset() table_ptr_array_.reset(); sstable_handle_array_.reset(); table_store_handle_.reset(); + if (nullptr != transfer_src_table_store_handle_) { transfer_src_table_store_handle_->~ObStorageMetaHandle(); ob_free(transfer_src_table_store_handle_); diff --git a/src/storage/tx_storage/ob_access_service.cpp b/src/storage/tx_storage/ob_access_service.cpp index be855159c..9fcf90351 100644 --- a/src/storage/tx_storage/ob_access_service.cpp +++ b/src/storage/tx_storage/ob_access_service.cpp @@ -754,7 +754,7 @@ int ObAccessService::delete_rows( transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows) { ACTIVE_SESSION_FLAG_SETTER_GUARD(in_storage_write); @@ -808,7 +808,7 @@ int ObAccessService::put_rows( transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows) { ACTIVE_SESSION_FLAG_SETTER_GUARD(in_storage_write); @@ -861,7 +861,7 @@ int ObAccessService::insert_rows( transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows) { ACTIVE_SESSION_FLAG_SETTER_GUARD(in_storage_write); @@ -916,10 +916,10 @@ int ObAccessService::insert_row( const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, const common::ObIArray &duplicated_column_ids, - const common::ObNewRow &row, + blocksstable::ObDatumRow &row, const ObInsertFlag flag, int64_t &affected_rows, - common::ObNewRowIterator *&duplicated_rows) + blocksstable::ObDatumRowIterator *&duplicated_rows) { ACTIVE_SESSION_FLAG_SETTER_GUARD(in_storage_write); int ret = OB_SUCCESS; @@ -970,7 +970,7 @@ int ObAccessService::insert_row( return ret; } -int ObAccessService::revert_insert_iter(common::ObNewRowIterator *iter) +int ObAccessService::revert_insert_iter(blocksstable::ObDatumRowIterator *iter) { ACTIVE_SESSION_FLAG_SETTER_GUARD(in_storage_write); int ret = OB_SUCCESS; @@ -987,7 +987,7 @@ int ObAccessService::update_rows( const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, const common::ObIArray< uint64_t> &updated_column_ids, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows) { ACTIVE_SESSION_FLAG_SETTER_GUARD(in_storage_write); @@ -1043,7 +1043,7 @@ int ObAccessService::lock_rows( const ObDMLBaseParam &dml_param, const int64_t abs_lock_timeout, const ObLockFlag lock_flag, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows) { ACTIVE_SESSION_FLAG_SETTER_GUARD(in_storage_write); @@ -1098,7 +1098,7 @@ int ObAccessService::lock_row( transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, const int64_t abs_lock_timeout, - const common::ObNewRow &row, + blocksstable::ObDatumRow &row, const ObLockFlag lock_flag) { ACTIVE_SESSION_FLAG_SETTER_GUARD(in_storage_write); diff --git a/src/storage/tx_storage/ob_access_service.h b/src/storage/tx_storage/ob_access_service.h index c7d2f2dd8..541054a44 100644 --- a/src/storage/tx_storage/ob_access_service.h +++ b/src/storage/tx_storage/ob_access_service.h @@ -137,7 +137,7 @@ public: transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows); int put_rows( const share::ObLSID &ls_id, @@ -145,7 +145,7 @@ public: transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows); int insert_rows( const share::ObLSID &ls_id, @@ -153,7 +153,7 @@ public: transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows); int insert_row( const share::ObLSID &ls_id, @@ -162,11 +162,11 @@ public: const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, const common::ObIArray &duplicated_column_ids, - const common::ObNewRow &row, + blocksstable::ObDatumRow &row, const ObInsertFlag flag, int64_t &affected_rows, - common::ObNewRowIterator *&duplicated_rows); - int revert_insert_iter(common::ObNewRowIterator *iter); + blocksstable::ObDatumRowIterator *&duplicated_rows); + int revert_insert_iter(blocksstable::ObDatumRowIterator *iter); int update_rows( const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, @@ -174,7 +174,7 @@ public: const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, const common::ObIArray< uint64_t> &updated_column_ids, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows); int lock_rows( const share::ObLSID &ls_id, @@ -183,7 +183,7 @@ public: const ObDMLBaseParam &dml_param, const int64_t abs_lock_timeout, /* -1: undefined, 0: nowait */ const ObLockFlag lock_flag, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows); int lock_row( const share::ObLSID &ls_id, @@ -191,7 +191,7 @@ public: transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, const int64_t abs_lock_timeout, - const common::ObNewRow &row, + blocksstable::ObDatumRow &row, const ObLockFlag lock_flag); int estimate_row_count( const ObTableScanParam ¶m, diff --git a/unittest/storage/blocksstable/test_row_reader.cpp b/unittest/storage/blocksstable/test_row_reader.cpp index d47aaa240..6005174a6 100644 --- a/unittest/storage/blocksstable/test_row_reader.cpp +++ b/unittest/storage/blocksstable/test_row_reader.cpp @@ -1326,30 +1326,30 @@ TEST_F(TestNewRowReader, test_write_update_row) const int64_t num = 200; const int64_t write_col_cnt = 80; const int64_t rowkey_cnt = 5; - oceanbase::common::ObObj objs[num]; - ObStoreRow writer_row; - writer_row.row_val_.cells_ = objs; - writer_row.row_val_.count_ = column_num; + blocksstable::ObStorageDatum objs[num]; + ObDatumRow writer_row; + writer_row.storage_datums_ = objs; + writer_row.count_ = column_num; ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(writer_row)); - for (int i = writer_row.row_val_.count_; i < write_col_cnt; ++i) { - writer_row.row_val_.cells_[i] = writer_row.row_val_.cells_[i - writer_row.row_val_.count_]; + for (int i = writer_row.count_; i < write_col_cnt; ++i) { + writer_row.storage_datums_[i] = writer_row.storage_datums_[i - writer_row.count_]; } - writer_row.row_val_.count_ = write_col_cnt; + writer_row.count_ = write_col_cnt; memtable::ObNopBitMap nop_bitmap; bool read_finished = false; - ret = nop_bitmap.init(writer_row.row_val_.count_, rowkey_cnt); + ret = nop_bitmap.init(writer_row.count_, rowkey_cnt); ObDatumRow reader_row; - ASSERT_EQ(OB_SUCCESS, reader_row.init(allocator_, writer_row.row_val_.count_)); - for (int i = 0; i < writer_row.row_val_.count_; ++i) { + ASSERT_EQ(OB_SUCCESS, reader_row.init(allocator_, writer_row.count_)); + for (int i = 0; i < writer_row.count_; ++i) { reader_row.storage_datums_[i].set_nop(); } int64_t array[] = {5, 18, 29, 45, 75, 78}; - writer_row.row_val_.cells_[array[2]].set_int(0); - writer_row.row_val_.cells_[array[4]].set_int(0); + writer_row.storage_datums_[array[2]].set_int(0); + writer_row.storage_datums_[array[4]].set_int(0); build_column_read_info(rowkey_cnt, writer_row); STORAGE_LOG(INFO, "write_row", K(writer_row)); @@ -1363,11 +1363,11 @@ TEST_F(TestNewRowReader, test_write_update_row) int64_t len = 0; char *buf = nullptr; - writer_row.flag_.set_flag(ObDmlFlag::DF_UPDATE); - writer_row.row_val_.cells_[array[2]].set_int(100 * i); - writer_row.row_val_.cells_[array[4]].set_int(100 * i); + writer_row.row_flag_.set_flag(ObDmlFlag::DF_UPDATE); + writer_row.storage_datums_[array[2]].set_int(100 * i); + writer_row.storage_datums_[array[4]].set_int(100 * i); if (i == 4) { - writer_row.flag_.set_flag(ObDmlFlag::DF_INSERT); + writer_row.row_flag_.set_flag(ObDmlFlag::DF_INSERT); ret = row_writer[i].write(rowkey_cnt, writer_row, nullptr, buf, len); } else { ret = row_writer[i].write(rowkey_cnt, writer_row, &update_idx, buf, len); @@ -1387,7 +1387,7 @@ TEST_F(TestNewRowReader, test_write_update_row) STORAGE_LOG(INFO, "chaser check writer row", K(read_info_)); int update_pos = 0; - for (int i = 0; i < writer_row.row_val_.count_; ++i) { + for (int i = 0; i < writer_row.count_; ++i) { bool check_cell_flag = false; if (i < rowkey_cnt) { check_cell_flag = true; @@ -1397,12 +1397,10 @@ TEST_F(TestNewRowReader, test_write_update_row) } if (check_cell_flag) { STORAGE_LOG(INFO, "check", K(i), K(update_pos), K(reader_row.storage_datums_[i])); - if (ObNumberFloatType != writer_row.row_val_.cells_[i].get_type()) { - if (i == array[2] || i == array[4]) { - ASSERT_TRUE(reader_row.storage_datums_[i].get_int() == 0); - } else { - ASSERT_TRUE(reader_row.storage_datums_[i] == writer_row.row_val_.cells_[i]); - } + if (i == array[2] || i == array[4]) { + ASSERT_TRUE(reader_row.storage_datums_[i].get_int() == 0); + } else { + ASSERT_TRUE(reader_row.storage_datums_[i] == writer_row.storage_datums_[i]); } } else { ASSERT_TRUE(!reader_row.storage_datums_[i].is_nop()); @@ -1416,28 +1414,28 @@ TEST_F(TestNewRowReader, test_write_write_nop_val) const int64_t num = 400; const int64_t write_col_cnt = 302; const int64_t rowkey_cnt = 3; - oceanbase::common::ObObj objs[num]; - ObStoreRow writer_row; - writer_row.row_val_.cells_ = objs; - writer_row.row_val_.count_ = column_num; + blocksstable::ObStorageDatum objs[num]; + ObDatumRow writer_row; + writer_row.storage_datums_ = objs; + writer_row.count_ = column_num; ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(writer_row)); int64_t idx = 0; - writer_row.row_val_.cells_[idx++].set_int(14); - writer_row.row_val_.cells_[idx++].set_int(-1658240586131896801); - writer_row.row_val_.cells_[idx++].set_int(-INT64_MAX); + writer_row.storage_datums_[idx++].set_int(14); + writer_row.storage_datums_[idx++].set_int(-1658240586131896801); + writer_row.storage_datums_[idx++].set_int(-INT64_MAX); for (int i = idx; i < write_col_cnt; ++i) { - writer_row.row_val_.cells_[i].set_nop_value(); + writer_row.storage_datums_[i].set_nop(); } - writer_row.row_val_.count_ = write_col_cnt; + writer_row.count_ = write_col_cnt; memtable::ObNopBitMap nop_bitmap; bool read_finished = false; - ret = nop_bitmap.init(writer_row.row_val_.count_, rowkey_cnt); + ret = nop_bitmap.init(writer_row.count_, rowkey_cnt); ObDatumRow reader_row; ASSERT_EQ(OB_SUCCESS, reader_row.init(allocator_, num)); - reader_row.count_ = writer_row.row_val_.count_; + reader_row.count_ = writer_row.count_; build_column_read_info(rowkey_cnt, writer_row); int64_t len = 0; @@ -1449,11 +1447,9 @@ TEST_F(TestNewRowReader, test_write_write_nop_val) ret = row_reader.read_row(buf, len, &read_info_, reader_row); STORAGE_LOG(INFO, "chaser check writer row", K(read_info_), K(reader_row)); - for (int i = 0; i < writer_row.row_val_.count_; ++i) { + for (int i = 0; i < writer_row.count_; ++i) { STORAGE_LOG(INFO, "check", K(i), K(reader_row.storage_datums_[i])); - if (ObNumberFloatType != writer_row.row_val_.cells_[i].get_type()) { - ASSERT_TRUE(reader_row.storage_datums_[i] == writer_row.row_val_.cells_[i]); - } + ASSERT_TRUE(reader_row.storage_datums_[i] == writer_row.storage_datums_[i]); } } @@ -1504,10 +1500,10 @@ TEST_F(TestNewRowReader, test_update_idx_with_rowkey) const int64_t write_col_cnt = 80; const int64_t rowkey_cnt = 3; - oceanbase::common::ObObj objs[num]; - ObStoreRow writer_row; - writer_row.row_val_.cells_ = objs; - writer_row.row_val_.count_ = column_num; + blocksstable::ObStorageDatum objs[num]; + ObDatumRow writer_row; + writer_row.storage_datums_ = objs; + writer_row.count_ = column_num; ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(writer_row)); int64_t update_array[] = {0, 1, 2, 5, 18}; @@ -1515,9 +1511,9 @@ TEST_F(TestNewRowReader, test_update_idx_with_rowkey) build_column_read_info(rowkey_cnt, writer_row); int64_t array_idx = 0; - for (int i = 0; i < writer_row.row_val_.count_; ++i) { + for (int i = 0; i < writer_row.count_; ++i) { if (i != update_array[array_idx]) { - writer_row.row_val_.cells_[i].set_nop_value(); + writer_row.storage_datums_[i].set_nop(); } else { update_idx.push_back(update_array[array_idx]); ++array_idx; diff --git a/unittest/storage/mock_access_service.cpp b/unittest/storage/mock_access_service.cpp index c24caba41..59e2f2875 100644 --- a/unittest/storage/mock_access_service.cpp +++ b/unittest/storage/mock_access_service.cpp @@ -32,7 +32,7 @@ int MockObAccessService::insert_rows( transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows) { int ret = OB_SUCCESS; diff --git a/unittest/storage/mock_access_service.h b/unittest/storage/mock_access_service.h index 741f43030..fd8d750fb 100644 --- a/unittest/storage/mock_access_service.h +++ b/unittest/storage/mock_access_service.h @@ -33,7 +33,7 @@ public: transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows); public: ObLSTabletService *tablet_service_; // different kinds of mock ls tablet service diff --git a/unittest/storage/mockcontainer/mock_ob_iterator.cpp b/unittest/storage/mockcontainer/mock_ob_iterator.cpp index 9893d1109..7c9051203 100644 --- a/unittest/storage/mockcontainer/mock_ob_iterator.cpp +++ b/unittest/storage/mockcontainer/mock_ob_iterator.cpp @@ -24,8 +24,48 @@ using namespace oceanbase::blocksstable; using namespace oceanbase::common::hash; using namespace oceanbase::share; +int malloc_datum_row( + ObIAllocator &allocator, + const int64_t cell_count, + ObDatumRow *&row) +{ + int ret = common::OB_SUCCESS; + row = NULL; + if (cell_count <= 0) { + ret = common::OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid args", K(cell_count)); + } else { + void *ptr = NULL; + int64_t size = sizeof(ObDatumRow) + sizeof(blocksstable::ObStorageDatum) * cell_count; + if (NULL == (ptr = allocator.alloc(size))) { + ret = common::OB_ALLOCATE_MEMORY_FAILED; + STORAGE_LOG(ERROR, "fail to alloc ObDatumRow", K(size), K(cell_count)); + } else { + void *cell_ptr = static_cast(ptr) + sizeof(ObDatumRow); + row = new (ptr) ObDatumRow; + row->storage_datums_ = new (cell_ptr) blocksstable::ObStorageDatum[cell_count](); + row->row_flag_.set_flag(blocksstable::ObDmlFlag::DF_NOT_EXIST); + row->count_ = cell_count; + } + } + + if (OB_FAIL(ret)) { + row = NULL; + } + return ret; +} + +void free_datum_row(ObIAllocator &allocator, ObDatumRow *&row) +{ + if (nullptr != row) { + allocator.free(row); + row = nullptr; + } +} + ObMockIterator::ObMockIterator(bool reverse) - : rows_(), + : datum_rows_(), + rows_(), cursor_(0), reverse_(reverse), trans_id_(888), @@ -41,7 +81,7 @@ ObMockIterator::~ObMockIterator() void ObMockIterator::setup_start_cursor() { if (reverse_) { - cursor_ = rows_.count() - 1; + cursor_ = rows_.empty() ? (datum_rows_.count() - 1) : (rows_.count() - 1); } else { cursor_ = 0; } @@ -61,10 +101,28 @@ bool ObMockIterator::end_of_row() const if (reverse_) { return cursor_ < 0; } else { - return cursor_ >= rows_.count(); + return cursor_ >= (rows_.empty() ? (datum_rows_.count()) : (rows_.count())); } } +int ObMockIterator::get_next_row(const ObDatumRow *&row) +{ + return get_next_row(const_cast(row)); +} + +int ObMockIterator::get_next_row(ObDatumRow *&row) +{ + int ret = OB_SUCCESS; + if (end_of_row()) { + row = NULL; + ret = OB_ITER_END; + } else { + row = datum_rows_[cursor_]; + advance(); + } + return ret; +} + int ObMockIterator::get_next_row(ObStoreRow *&row) { int ret = OB_SUCCESS; @@ -96,6 +154,32 @@ int ObMockIterator::get_next_row(ObNewRow *&row) return ret; } +int ObMockIterator::get_row(const int64_t idx, const ObDatumRow *&row) const +{ + int ret = OB_SUCCESS; + if (idx < 0 || idx >= rows_.count()) { + STORAGE_LOG(WARN, "failed to idx", K(idx), K(rows_.count())); + row = NULL; + ret = OB_ITER_END; + } else { + row = datum_rows_[idx]; + } + return ret; +} + +int ObMockIterator::get_row(const int64_t idx, ObDatumRow *&row) const +{ + int ret = OB_SUCCESS; + if (idx < 0 || idx >= rows_.count()) { + STORAGE_LOG(WARN, "failed to idx", K(idx), K(rows_.count())); + row = NULL; + ret = OB_ITER_END; + } else { + row = datum_rows_[idx]; + } + return ret; +} + int ObMockIterator::get_row(const int64_t idx, const ObStoreRow *&row) const { int ret = OB_SUCCESS; @@ -135,6 +219,44 @@ int ObMockIterator::get_row(const int64_t idx, ObNewRow *&row) const return ret; } +int ObMockIterator::add_row(blocksstable::ObDatumRow *row) +{ + int ret = OB_SUCCESS; + ObDatumRow *row_clone = NULL; + if (NULL == row) { + ret = OB_ERR_NULL_VALUE; + } else { + ObRowStoreType row_type = FLAT_ROW_STORE; + if (OB_FAIL(malloc_datum_row(allocator_, row->count_, row_clone))) { + STORAGE_LOG(WARN, "failed to malloc datum row", K(ret)); + } else { + row_clone->row_flag_ = row->row_flag_; + if (row->mvcc_row_flag_.is_uncommitted_row()) { // make all row point to the same trans_id + row_clone->trans_id_ = row->trans_id_.is_valid() ? row->trans_id_ : trans_id_; + } + row_clone->count_ = row->count_; + for (int64_t i = 0; OB_SUCC(ret) && i < row->count_; ++i) { + if (OB_SUCCESS != (ret = row_clone->storage_datums_[i].deep_copy(row->storage_datums_[i], allocator_))) { + STORAGE_LOG(WARN, "ob write datum error.", K(ret) ,K(row)); + } + } + + if (OB_SUCC(ret)) { + if (OB_SUCCESS != (ret = datum_rows_.push_back(row_clone))) { + STORAGE_LOG(WARN, "add datum row failed", K(ret), K(row_clone)); + } else { + setup_start_cursor(); + } + } + } + + if (OB_SUCCESS != ret && row_clone != NULL) { + free_datum_row(allocator_, row_clone); + } + } + return ret; +} + // deep copy int ObMockIterator::add_row(ObStoreRow *row) { @@ -187,6 +309,11 @@ void ObMockIterator::reset_iter() void ObMockIterator::reset() { cursor_ = 0; + for (int64_t i = 0; i < datum_rows_.count(); ++i) { + ObDatumRow *row = datum_rows_[i]; + free_datum_row(allocator_, row); + } + datum_rows_.reset(); for (int64_t i = 0; i < rows_.count(); ++i) { ObStoreRow *row = rows_[i]; free_store_row(allocator_, row); @@ -219,6 +346,102 @@ int ObMockIterator::from(const ObString &str, char escape, uint16_t *col_id_arra return ret; } +int ObMockIterator::from_for_datum(const char *cstr, char escape, uint16_t* col_id_array, int64_t *result_col_id_array) +{ + return from_for_datum(ObString::make_string(cstr), escape, col_id_array, result_col_id_array); +} + +int ObMockIterator::from_for_datum(const ObString &str, char escape, uint16_t *col_id_array, int64_t *result_col_id_array) +{ + int ret = OB_SUCCESS; + ObMockIteratorBuilder builder; + ObArenaAllocator buffer("StTemp"); + if (OB_SUCCESS != (ret != builder.init(&buffer, escape))) { + STORAGE_LOG(WARN, "init builder failed"); + } else { + if (OB_ISNULL(result_col_id_array)) { + ret = builder.parse_for_datum(str, *this, col_id_array); + } else { + ret = builder.parse_datum_with_specified_col_ids(str, *this, col_id_array, result_col_id_array); + } + + } + buffer.clear(); + return ret; +} + +bool ObMockIterator::inner_equals(const blocksstable::ObDatumRow &r1, const blocksstable::ObDatumRow &r2) +{ + bool bool_ret = true; + int64_t idx = 0; + if (!r1.storage_datums_ || !r2.storage_datums_) { + bool_ret = false; + } else if (r1.count_ != r2.count_) { + bool_ret = false; + } else { + for (idx = 0; idx < r1.count_ && bool_ret; idx++) { + if (r1.storage_datums_[idx].is_null() && r2.storage_datums_[idx].is_null()) { + continue; + } else if (r1.storage_datums_[idx].is_null() || r2.storage_datums_[idx].is_null()) { + bool_ret = false; + } else if (r1.storage_datums_[idx] == r2.storage_datums_[idx]) { + continue; + } else { + bool_ret = false; + } + } + } + return bool_ret; +} + +bool ObMockIterator::equals(const blocksstable::ObDatumRow &r1, const blocksstable::ObDatumRow &r2, + const bool cmp_multi_version_row_flag, const bool cmp_is_get_and_scan_index) +{ + int ret = OB_SUCCESS; + bool bool_ret = false; + STORAGE_LOG(INFO, "compare two rows", K(r1), K(r2)); + if (r1.row_flag_ != r2.row_flag_) { + STORAGE_LOG(WARN, "flag not equals", K(r1), K(r2)); + } else if (!cmp_multi_version_row_flag && (r1.row_flag_.is_not_exist() || r1.row_flag_.is_delete())) { + STORAGE_LOG(DEBUG, "flag is not row exist, return true", K(r1), K(r2)); + bool_ret = true; + } else if (cmp_multi_version_row_flag && r1.mvcc_row_flag_.flag_ != r2.mvcc_row_flag_.flag_) { + STORAGE_LOG(WARN, "row_type_flag not equals", K(r1), K(r2)); + bool_ret = false; + } else if (cmp_is_get_and_scan_index + && (r1.scan_index_ != r2.scan_index_)) { + STORAGE_LOG(WARN, "scan_index not equals", K(r1), K(r2)); + bool_ret = false; + } else { + bool_ret = inner_equals(r1, r2); + STORAGE_LOG(INFO, "compare two rows", K(bool_ret), K(r1), K(r2)); + } + return bool_ret; +} + +bool ObMockIterator::equals(int64_t idx, blocksstable::ObDatumRow &other_row) const +{ + bool bool_ret = false; + int ret = OB_SUCCESS; + const blocksstable::ObDatumRow *this_row = NULL; + if (OB_SUCCESS != get_row(idx, this_row)) { + STORAGE_LOG(WARN, "invalid idx"); + bool_ret = false; + } else { + bool_ret = inner_equals(*this_row, other_row); + if (!bool_ret) { + STORAGE_LOG(WARN, "store row is not equal", + K(idx), KPC(this_row), K(other_row)); + } + } + return bool_ret; +} + +bool ObMockIterator::equals(int64_t idx, const blocksstable::ObDatumRow &other_row) const +{ + return equals(idx, const_cast(other_row)); +} + bool ObMockIterator::equals(const ObNewRow &r1, const ObNewRow &r2) { bool bool_ret = true; @@ -334,6 +557,10 @@ ObHashMap ObMockIteratorBuilder::str_to_obj_parse_func_; ObHashMap ObMockIteratorBuilder::str_to_info_parse_func_; +ObHashMap +ObMockIteratorBuilder::str_to_datum_parse_func_; +ObHashMap +ObMockIteratorBuilder::str_to_datum_info_parse_func_; ObHashMap ObMockIteratorBuilder::str_to_obj_type_; ObHashMap ObMockIteratorBuilder::str_to_flag_; @@ -356,18 +583,46 @@ ObObjMeta ObMockIteratorBuilder::LOB_TYPE; ObObjMeta ObMockIteratorBuilder::TS_TYPE; ObObjMeta ObMockIteratorBuilder::NU_TYPE; -int ObMockIteratorBuilder::parse_varchar(ObIAllocator *allocator, - const ObString &word, - ObStoreRow &row, - int64_t &idx) +int ObMockIteratorBuilder::prepare_parse_varchar(ObIAllocator *allocator, + const ObString &word, + const uint16_t count, + int64_t &idx) { OB_ASSERT(allocator); int ret = OB_SUCCESS; char *buf = NULL; - if (idx >= row.row_val_.count_) { + if (idx >= count) { ret = OB_ARRAY_OUT_OF_RANGE; } else if (NULL == (buf = static_cast(allocator->alloc(word.length())))) { ret = OB_ALLOCATE_MEMORY_FAILED; + } + return ret; +} + +int ObMockIteratorBuilder::parse_datum_varchar(ObIAllocator *allocator, + const ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(prepare_parse_varchar(allocator, word, row.count_, idx))) { + STORAGE_LOG(WARN, "failed to parse datum varchar"); + } else { + ObStorageDatum datum_tmp; + datum_tmp.set_string(word); + ret = row.storage_datums_[idx++].deep_copy(datum_tmp, *allocator); + } + return ret; +} + +int ObMockIteratorBuilder::parse_obj_varchar(ObIAllocator *allocator, + const ObString &word, + storage::ObStoreRow &row, + int64_t &idx) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(prepare_parse_varchar(allocator, word, row.row_val_.count_, idx))) { + STORAGE_LOG(WARN, "failed to parse obj varchar"); } else { ObObj obj_tmp; obj_tmp.set_varchar(word); @@ -378,21 +633,22 @@ int ObMockIteratorBuilder::parse_varchar(ObIAllocator *allocator, return ret; } -int ObMockIteratorBuilder::parse_lob(ObIAllocator *allocator, - const ObString &word, - ObStoreRow &row, - int64_t &idx) +int ObMockIteratorBuilder::prepare_parse_lob(ObIAllocator *allocator, + const ObString &word, + const uint16_t count, + int64_t &idx, + ObLobCommon *&lob_data, + int64_t &val) { OB_ASSERT(allocator); int ret = OB_SUCCESS; char *buf = NULL; - if (idx >= row.row_val_.count_) { + if (idx >= count) { ret = OB_ARRAY_OUT_OF_RANGE; } else if (NULL == (buf = static_cast(allocator->alloc(sizeof(ObLobCommon))))) { ret = OB_ALLOCATE_MEMORY_FAILED; } else { char str[MAX_DATA_LENGTH]; - int64_t val = 0; char *end_ptr = NULL; snprintf(str, MAX_DATA_LENGTH, "%.*s", word.length(), word.ptr()); @@ -416,18 +672,51 @@ int ObMockIteratorBuilder::parse_lob(ObIAllocator *allocator, ret = OB_NUMERIC_OVERFLOW; STORAGE_LOG(WARN, "too big lob index cnt", K(val), K(word)); } else { - ObLobCommon *lob_data = nullptr; - const int64_t block_size = 1 << 20; lob_data = new (buf) ObLobCommon(); - ObObj obj_tmp; - obj_tmp.set_lob_value(ObLongTextType, lob_data, lob_data->get_handle_size(val * block_size)); - obj_tmp.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); - ret = ob_write_obj(*allocator, obj_tmp, row.row_val_.cells_[idx++]); } } return ret; } +int ObMockIteratorBuilder::parse_datum_lob(ObIAllocator *allocator, + const ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx) +{ + int ret = OB_SUCCESS; + int64_t val = 0; + ObLobCommon *lob_data = nullptr; + if (OB_FAIL(prepare_parse_lob(allocator, word, row.count_, idx, lob_data, val))) { + STORAGE_LOG(WARN, "failed to parse datum lob"); + } else { + const int64_t block_size = 1 << 20; + ObStorageDatum datum_tmp; + datum_tmp.set_lob_data(*lob_data, lob_data->get_handle_size(val * block_size)); + ret = row.storage_datums_[idx++].deep_copy(datum_tmp, *allocator); + } + return ret; +} + +int ObMockIteratorBuilder::parse_obj_lob(ObIAllocator *allocator, + const ObString &word, + storage::ObStoreRow &row, + int64_t &idx) +{ + int ret = OB_SUCCESS; + int64_t val = 0; + ObLobCommon *lob_data = nullptr; + if (OB_FAIL(prepare_parse_lob(allocator, word, row.row_val_.count_, idx, lob_data, val))) { + STORAGE_LOG(WARN, "failed to parse obj lob"); + } else { + const int64_t block_size = 1 << 20; + ObObj obj_tmp; + obj_tmp.set_lob_value(ObLongTextType, lob_data, lob_data->get_handle_size(val * block_size)); + obj_tmp.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ret = ob_write_obj(*allocator, obj_tmp, row.row_val_.cells_[idx++]); + } + return ret; +} + /* int ObMockIteratorBuilder::parse_bool(ObIAllocator *allocator, const ObString &word, ObStoreRow &row, int64_t &idx) @@ -460,39 +749,67 @@ int ObMockIteratorBuilder::parse_bool(ObIAllocator *allocator, } */ -int ObMockIteratorBuilder::parse_timestamp(ObIAllocator *allocator, - const ObString &word, - ObStoreRow &row, - int64_t &idx) +int ObMockIteratorBuilder::prepare_parse_timestamp(ObIAllocator *allocator, + const ObString &word, + const uint16_t count, + int64_t &idx, + int64_t &usec) { UNUSED(allocator); int ret = OB_SUCCESS; - int64_t usec = 0; - if (idx >= row.row_val_.count_) { + if (idx >= count) { ret = OB_ARRAY_OUT_OF_RANGE; } else if (OB_SUCCESS != (ret = ObTimeUtility2::str_to_usec(word, usec))) { STORAGE_LOG(WARN, "str to microsecond failed", K(word), K(usec), K(ret)); + } + return ret; +} + +int ObMockIteratorBuilder::parse_datum_timestamp(ObIAllocator *allocator, + const ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx) +{ + int ret = OB_SUCCESS; + int64_t usec = 0; + if (OB_FAIL(prepare_parse_timestamp(allocator, word, row.count_, idx, usec))) { + STORAGE_LOG(WARN, "failed to parse datum timestamp"); + } else { + row.storage_datums_[idx++].set_timestamp(usec); + } + return ret; +} + +int ObMockIteratorBuilder::parse_obj_timestamp(ObIAllocator *allocator, + const ObString &word, + storage::ObStoreRow &row, + int64_t &idx) +{ + int ret = OB_SUCCESS; + int64_t usec = 0; + if (OB_FAIL(prepare_parse_timestamp(allocator, word, row.row_val_.count_, idx, usec))) { + STORAGE_LOG(WARN, "failed to parse obj timestamp"); } else { row.row_val_.cells_[idx++].set_timestamp(usec); } return ret; } -int ObMockIteratorBuilder::parse_int(ObIAllocator *allocator, - const ObString &word, - ObStoreRow &row, - int64_t &idx) +int ObMockIteratorBuilder::prepare_parse_int(ObIAllocator *allocator, + const ObString &word, + const uint16_t count, + int64_t &idx, + int64_t &val) { UNUSED(allocator); OB_ASSERT(NULL != word.ptr() && 0 <= word.length()); int ret = OB_SUCCESS; - if (idx >= row.row_val_.count_) { + if (idx >= count) { ret = OB_ARRAY_OUT_OF_RANGE; } else { char str[MAX_DATA_LENGTH]; - int64_t val = 0; char *end_ptr = NULL; snprintf(str, MAX_DATA_LENGTH, "%.*s", word.length(), word.ptr()); @@ -510,33 +827,59 @@ int ObMockIteratorBuilder::parse_int(ObIAllocator *allocator, || (INT_MAX < val || INT_MIN > val)) { STORAGE_LOG(WARN, "strtol fail", K(val), K(word), K(errno)); ret = OB_NUMERIC_OVERFLOW; - } else { - if (str == end_ptr) { - STORAGE_LOG(WARN, "no digits found"); - ret = OB_ERR_CAST_VARCHAR_TO_NUMBER; - } else { - row.row_val_.cells_[idx++].set_int32(static_cast(val)); - } + } else if (str == end_ptr) { + STORAGE_LOG(WARN, "no digits found"); + ret = OB_ERR_CAST_VARCHAR_TO_NUMBER; } } return ret; } -int ObMockIteratorBuilder::parse_bigint(ObIAllocator *allocator, - const ObString &word, - ObStoreRow &row, - int64_t &idx) +int ObMockIteratorBuilder::parse_datum_int(ObIAllocator *allocator, + const ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx) +{ + int ret = OB_SUCCESS; + int64_t val = 0; + if (OB_FAIL(prepare_parse_int(allocator, word, row.count_, idx, val))) { + STORAGE_LOG(WARN, "failed to parse datum int"); + } else { + row.storage_datums_[idx++].set_int32(static_cast(val)); + } + return ret; +} + +int ObMockIteratorBuilder::parse_obj_int(ObIAllocator *allocator, + const ObString &word, + storage::ObStoreRow &row, + int64_t &idx) +{ + int ret = OB_SUCCESS; + int64_t val = 0; + if (OB_FAIL(prepare_parse_int(allocator, word, row.row_val_.count_, idx, val))) { + STORAGE_LOG(WARN, "failed to parse obj int"); + } else { + row.row_val_.cells_[idx++].set_int32(static_cast(val)); + } + return ret; +} + +int ObMockIteratorBuilder::prepare_parse_bigint(ObIAllocator *allocator, + const ObString &word, + const uint16_t count, + int64_t &idx, + int64_t &val) { UNUSED(allocator); OB_ASSERT(NULL != word.ptr() && 0 <= word.length()); int ret = OB_SUCCESS; - if (idx >= row.row_val_.count_) { + if (idx >= count) { ret = OB_ARRAY_OUT_OF_RANGE; } else { char str[MAX_DATA_LENGTH]; - int64_t val = 0; char *end_ptr = NULL; snprintf(str, MAX_DATA_LENGTH, "%.*s", word.length(), word.ptr()); @@ -553,32 +896,86 @@ int ObMockIteratorBuilder::parse_bigint(ObIAllocator *allocator, || (errno != 0 && val == 0)) { STORAGE_LOG(WARN, "strtoll fail", K(val), K(word), K(errno)); ret = OB_NUMERIC_OVERFLOW; - } else { - if (str == end_ptr) { - STORAGE_LOG(WARN, "no digits found"); - ret = OB_ERR_CAST_VARCHAR_TO_NUMBER; - } else { - row.row_val_.cells_[idx++].set_int(val); - } + } else if (str == end_ptr) { + STORAGE_LOG(WARN, "no digits found"); + ret = OB_ERR_CAST_VARCHAR_TO_NUMBER; } } return ret; } -int ObMockIteratorBuilder::parse_number(ObIAllocator *allocator, - const ObString &word, - ObStoreRow &row, - int64_t &idx) +int ObMockIteratorBuilder::parse_datum_bigint(ObIAllocator *allocator, + const ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx) +{ + int ret = OB_SUCCESS; + int64_t val = 0; + if (OB_FAIL(prepare_parse_bigint(allocator, word, row.count_, idx, val))) { + STORAGE_LOG(WARN, "failed to parse datum bigint"); + } else { + row.storage_datums_[idx++].set_int(val); + } + return ret; +} + +int ObMockIteratorBuilder::parse_obj_bigint(ObIAllocator *allocator, + const ObString &word, + storage::ObStoreRow &row, + int64_t &idx) +{ + int ret = OB_SUCCESS; + int64_t val = 0; + if (OB_FAIL(prepare_parse_bigint(allocator, word, row.row_val_.count_, idx, val))) { + STORAGE_LOG(WARN, "failed to parse obj bigint"); + } else { + row.row_val_.cells_[idx++].set_int(val); + } + return ret; +} + +int ObMockIteratorBuilder::prepare_parse_number(ObIAllocator *allocator, + const ObString &word, + const uint16_t count, + int64_t &idx, + number::ObNumber &nmb) { OB_ASSERT(NULL != word.ptr() && 0 <= word.length()); int ret = OB_SUCCESS; - number::ObNumber nmb; - if (idx >= row.row_val_.count_) { + if (idx >= count) { ret = OB_ARRAY_OUT_OF_RANGE; } else if (OB_SUCCESS != (ret = nmb.from(word.ptr(), word.length(), *allocator))) { STORAGE_LOG(WARN, "parse number failed", K(ret), K(word)); + } + return ret; +} + +int ObMockIteratorBuilder::parse_datum_number(ObIAllocator *allocator, + const ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx) +{ + int ret = OB_SUCCESS; + number::ObNumber nmb; + if (OB_FAIL(prepare_parse_number(allocator, word, row.count_, idx, nmb))) { + STORAGE_LOG(WARN, "failed to parse datum int"); + } else { + row.storage_datums_[idx++].set_number(nmb); + } + return ret; +} + +int ObMockIteratorBuilder::parse_obj_number(ObIAllocator *allocator, + const ObString &word, + storage::ObStoreRow &row, + int64_t &idx) +{ + int ret = OB_SUCCESS; + number::ObNumber nmb; + if (OB_FAIL(prepare_parse_number(allocator, word, row.row_val_.count_, idx, nmb))) { + STORAGE_LOG(WARN, "failed to parse obj int"); } else { row.row_val_.cells_[idx++].set_number(nmb); } @@ -587,7 +984,16 @@ int ObMockIteratorBuilder::parse_number(ObIAllocator *allocator, int ObMockIteratorBuilder::parse_dml(ObIAllocator *allocator, const ObString &word, - ObStoreRow &row, + storage::ObStoreRow &row, + int64_t &idx) +{ + UNUSEDx(allocator, word, row, idx); + return OB_SUCCESS; +} + +int ObMockIteratorBuilder::parse_datum_dml(ObIAllocator *allocator, + const ObString &word, + blocksstable::ObDatumRow &row, int64_t &idx) { UNUSEDx(allocator, word, row, idx); @@ -596,29 +1002,58 @@ int ObMockIteratorBuilder::parse_dml(ObIAllocator *allocator, int ObMockIteratorBuilder::parse_first_dml(ObIAllocator *allocator, const ObString &word, - ObStoreRow &row, + storage::ObStoreRow &row, int64_t &idx) { UNUSEDx(allocator, word, row, idx); return OB_SUCCESS; } -int ObMockIteratorBuilder::parse_flag(ObIAllocator *allocator, - const ObString &word, - ObStoreRow &row, - int64_t &idx) +int ObMockIteratorBuilder::prepare_parse_flag(ObIAllocator *allocator, + const ObString &word, + int64_t &idx, + int64_t &flag) { - UNUSED(allocator); - UNUSED(idx); OB_ASSERT(NULL != word.ptr() && 0 <= word.length()); int ret = OB_SUCCESS; - int64_t flag = 0; if (OB_SUCCESS != str_to_flag_.get_refactored(word, flag)) { STORAGE_LOG(WARN, "failed to parse flag", K(word)); ret = OB_HASH_NOT_EXIST; + } + return ret; +} + +int ObMockIteratorBuilder::parse_datum_flag(ObIAllocator *allocator, + const ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx) +{ + UNUSED(allocator); + UNUSED(idx); + int ret = OB_SUCCESS; + int64_t flag = 0; + if (OB_FAIL(prepare_parse_flag(allocator, word, idx, flag))) { + STORAGE_LOG(WARN, "failed to parse datum row dml flag"); } else { - row.flag_ = (ObDmlFlag)flag; + row.row_flag_.set_flag((ObDmlFlag)flag); + } + return ret; +} + +int ObMockIteratorBuilder::parse_obj_flag(ObIAllocator *allocator, + const ObString &word, + storage::ObStoreRow &row, + int64_t &idx) +{ + UNUSED(allocator); + UNUSED(idx); + int ret = OB_SUCCESS; + int64_t flag = 0; + if (OB_FAIL(prepare_parse_flag(allocator, word, idx, flag))) { + STORAGE_LOG(WARN, "failed to parse obj row dml flag"); + } else { + row.flag_.set_flag((ObDmlFlag)flag); } return ret; } @@ -659,10 +1094,10 @@ int ObMockIteratorBuilder::parse_is_get(ObIAllocator *allocator, return ret; } -int ObMockIteratorBuilder::parse_trans_id(ObIAllocator *allocator, - const ObString &word, - ObStoreRow &row, - int64_t &idx) +int ObMockIteratorBuilder::parse_datum_trans_id(ObIAllocator *allocator, + const ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx) { UNUSED(allocator); UNUSED(idx); @@ -670,7 +1105,7 @@ int ObMockIteratorBuilder::parse_trans_id(ObIAllocator *allocator, int ret = OB_SUCCESS; if (OB_SUCCESS != str_to_trans_id_.get_refactored(word, row.trans_id_)) { - STORAGE_LOG(WARN, "failed to parse is_get", K(word)); + STORAGE_LOG(WARN, "failed to parse trans_id", K(word)); ret = OB_HASH_NOT_EXIST; } else { STORAGE_LOG(DEBUG, "parse str_to_trans_id_", K(row), K(word), K(row.trans_id_)); @@ -678,17 +1113,35 @@ int ObMockIteratorBuilder::parse_trans_id(ObIAllocator *allocator, return ret; } -int ObMockIteratorBuilder::parse_scan_index(ObIAllocator *allocator, - const ObString &word, - ObStoreRow &row, - int64_t &idx) +int ObMockIteratorBuilder::parse_obj_trans_id(ObIAllocator *allocator, + const ObString &word, + storage::ObStoreRow &row, + int64_t &idx) +{ + UNUSED(allocator); + UNUSED(idx); + OB_ASSERT(NULL != word.ptr() && 0 <= word.length()); + + int ret = OB_SUCCESS; + if (OB_SUCCESS != str_to_trans_id_.get_refactored(word, row.trans_id_)) { + STORAGE_LOG(WARN, "failed to parse trans_id", K(word)); + ret = OB_HASH_NOT_EXIST; + } else { + STORAGE_LOG(DEBUG, "parse str_to_trans_id_", K(row), K(word), K(row.trans_id_)); + } + return ret; +} + +int ObMockIteratorBuilder::prepare_parse_scan_index(ObIAllocator *allocator, + const ObString &word, + int64_t &idx, + int64_t &val) { UNUSED(allocator); UNUSED(idx); OB_ASSERT(NULL != word.ptr() && 0 <= word.length()); int ret = OB_SUCCESS; char str[MAX_DATA_LENGTH]; - int64_t val = 0; char *end_ptr = NULL; snprintf(str, MAX_DATA_LENGTH, "%.*s", word.length(), word.ptr()); @@ -705,22 +1158,70 @@ int ObMockIteratorBuilder::parse_scan_index(ObIAllocator *allocator, || (errno != 0 && val == 0)) { STORAGE_LOG(WARN, "strtoll fail", K(val), K(word), K(errno)); ret = OB_NUMERIC_OVERFLOW; - } else { - if (str == end_ptr) { - STORAGE_LOG(WARN, "no digits found"); - ret = OB_ERR_CAST_VARCHAR_TO_NUMBER; - } else { - row.scan_index_ = val; - STORAGE_LOG(DEBUG, "parse scan_index", K(row)); - } + } else if (str == end_ptr) { + STORAGE_LOG(WARN, "no digits found"); + ret = OB_ERR_CAST_VARCHAR_TO_NUMBER; } return ret; } -int ObMockIteratorBuilder::parse_multi_version_row_flag(ObIAllocator *allocator, - const ObString &word, - ObStoreRow &row, - int64_t &idx) +int ObMockIteratorBuilder::parse_datum_scan_index(ObIAllocator *allocator, + const ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx) +{ + UNUSED(allocator); + UNUSED(idx); + int ret = OB_SUCCESS; + int64_t val = 0; + if (OB_FAIL(prepare_parse_scan_index(allocator, word, idx, val))) { + STORAGE_LOG(WARN, "failed to parse datum row scan_index"); + } else { + row.scan_index_ = val; + STORAGE_LOG(DEBUG, "parse scan_index", K(row)); + } + return ret; +} + +int ObMockIteratorBuilder::parse_obj_scan_index(ObIAllocator *allocator, + const ObString &word, + storage::ObStoreRow &row, + int64_t &idx) +{ + UNUSED(allocator); + UNUSED(idx); + int ret = OB_SUCCESS; + int64_t val = 0; + if (OB_FAIL(prepare_parse_scan_index(allocator, word, idx, val))) { + STORAGE_LOG(WARN, "failed to parse obj row scan_index"); + } else { + row.scan_index_ = val; + STORAGE_LOG(DEBUG, "parse scan_index", K(row)); + } + return ret; +} + +int ObMockIteratorBuilder::parse_datum_multi_version_row_flag(ObIAllocator *allocator, + const ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx) +{ + UNUSED(allocator); + UNUSED(idx); + OB_ASSERT(NULL != word.ptr() && 0 <= word.length()); + + int ret = OB_SUCCESS; + if (OB_SUCCESS != str_to_multi_version_row_flag_.get_refactored(word, row.mvcc_row_flag_.flag_)) { + STORAGE_LOG(WARN, "failed to parse multi version row flag for datum row", K(word)); + ret = OB_HASH_NOT_EXIST; + } + return ret; +} + +int ObMockIteratorBuilder::parse_obj_multi_version_row_flag(ObIAllocator *allocator, + const ObString &word, + storage::ObStoreRow &row, + int64_t &idx) { UNUSED(allocator); UNUSED(idx); @@ -728,7 +1229,7 @@ int ObMockIteratorBuilder::parse_multi_version_row_flag(ObIAllocator *allocator, int ret = OB_SUCCESS; if (OB_SUCCESS != str_to_multi_version_row_flag_.get_refactored(word, row.row_type_flag_.flag_)) { - STORAGE_LOG(WARN, "failed to parse multi version row flag", K(word)); + STORAGE_LOG(WARN, "failed to parse multi version row flag for obj row", K(word)); ret = OB_HASH_NOT_EXIST; } return ret; @@ -754,37 +1255,74 @@ int ObMockIteratorBuilder::static_init() trans_id_list_[i - 1] = transaction::ObTransID(i); } + if (OB_SUCCESS != (ret = str_to_datum_parse_func_.create( + cal_next_prime(TYPE_NUM * 10), + ObModIds::OB_HASH_BUCKET))) { + STORAGE_LOG(WARN, "out of memory"); + } else if (OB_SUCCESS != str_to_datum_parse_func_.set_refactored( + ObString::make_string("int"), + ObMockIteratorBuilder::parse_datum_int) + || OB_SUCCESS != str_to_datum_parse_func_.set_refactored( + ObString::make_string("bigint"), + ObMockIteratorBuilder::parse_datum_bigint) + || OB_SUCCESS != str_to_datum_parse_func_.set_refactored( + ObString::make_string("varchar"), + ObMockIteratorBuilder::parse_datum_varchar) + || OB_SUCCESS != str_to_datum_parse_func_.set_refactored( + ObString::make_string("lob"), + ObMockIteratorBuilder::parse_datum_lob) + || OB_SUCCESS != str_to_datum_parse_func_.set_refactored( + ObString::make_string("var"), + ObMockIteratorBuilder::parse_datum_varchar) + || OB_SUCCESS != str_to_datum_parse_func_.set_refactored( + ObString::make_string("ts"), + ObMockIteratorBuilder::parse_datum_timestamp) + || OB_SUCCESS != str_to_datum_parse_func_.set_refactored( + ObString::make_string("timestamp"), + ObMockIteratorBuilder::parse_datum_timestamp) + || OB_SUCCESS != str_to_datum_parse_func_.set_refactored( + ObString::make_string("num"), + ObMockIteratorBuilder::parse_datum_number) + || OB_SUCCESS != str_to_datum_parse_func_.set_refactored( + ObString::make_string("number"), + ObMockIteratorBuilder::parse_datum_number)) { + ret = OB_INIT_FAIL; + STORAGE_LOG(WARN, "obj parse func hashtable insert failed"); + } else { + STORAGE_LOG(DEBUG, "init obj parse func hashtable"); + } + if (OB_SUCCESS != (ret = str_to_obj_parse_func_.create( cal_next_prime(TYPE_NUM * 10), ObModIds::OB_HASH_BUCKET))) { STORAGE_LOG(WARN, "out of memory"); } else if (OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("int"), - ObMockIteratorBuilder::parse_int) + ObMockIteratorBuilder::parse_obj_int) || OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("bigint"), - ObMockIteratorBuilder::parse_bigint) + ObMockIteratorBuilder::parse_obj_bigint) || OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("varchar"), - ObMockIteratorBuilder::parse_varchar) + ObMockIteratorBuilder::parse_obj_varchar) || OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("lob"), - ObMockIteratorBuilder::parse_lob) + ObMockIteratorBuilder::parse_obj_lob) || OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("var"), - ObMockIteratorBuilder::parse_varchar) + ObMockIteratorBuilder::parse_obj_varchar) || OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("ts"), - ObMockIteratorBuilder::parse_timestamp) + ObMockIteratorBuilder::parse_obj_timestamp) || OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("timestamp"), - ObMockIteratorBuilder::parse_timestamp) + ObMockIteratorBuilder::parse_obj_timestamp) || OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("num"), - ObMockIteratorBuilder::parse_number) + ObMockIteratorBuilder::parse_obj_number) || OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("number"), - ObMockIteratorBuilder::parse_number)) { + ObMockIteratorBuilder::parse_obj_number)) { ret = OB_INIT_FAIL; STORAGE_LOG(WARN, "obj parse func hashtable insert failed"); } else { @@ -819,13 +1357,38 @@ int ObMockIteratorBuilder::static_init() STORAGE_LOG(DEBUG, "init obj type hashtable"); } + if (OB_SUCCESS != (ret = str_to_datum_info_parse_func_.create( + cal_next_prime(INFO_NUM), + ObModIds::OB_HASH_BUCKET))) { + STORAGE_LOG(DEBUG, "out of memory"); + } else if (OB_SUCCESS != str_to_datum_info_parse_func_.set_refactored( + ObString::make_string("flag"), + ObMockIteratorBuilder::parse_datum_flag) + || OB_SUCCESS != str_to_datum_info_parse_func_.set_refactored( + ObString::make_string("dml"), + ObMockIteratorBuilder::parse_datum_dml) + || OB_SUCCESS != str_to_datum_info_parse_func_.set_refactored( + ObString::make_string("multi_version_row_flag"), + ObMockIteratorBuilder::parse_datum_multi_version_row_flag) + || OB_SUCCESS != str_to_datum_info_parse_func_.set_refactored( + ObString::make_string("scan_index"), + ObMockIteratorBuilder::parse_datum_scan_index) + || OB_SUCCESS != str_to_datum_info_parse_func_.set_refactored( + ObString::make_string("trans_id"), + ObMockIteratorBuilder::parse_datum_trans_id)) { + ret = OB_INIT_FAIL; + STORAGE_LOG(WARN, "info parse func hashtable insert failed"); + } else { + STORAGE_LOG(DEBUG, "init info parse func hashtable"); + } + if (OB_SUCCESS != (ret = str_to_info_parse_func_.create( cal_next_prime(INFO_NUM), ObModIds::OB_HASH_BUCKET))) { STORAGE_LOG(DEBUG, "out of memory"); } else if (OB_SUCCESS != str_to_info_parse_func_.set_refactored( ObString::make_string("flag"), - ObMockIteratorBuilder::parse_flag) + ObMockIteratorBuilder::parse_obj_flag) || OB_SUCCESS != str_to_info_parse_func_.set_refactored( ObString::make_string("dml"), ObMockIteratorBuilder::parse_dml) @@ -834,19 +1397,13 @@ int ObMockIteratorBuilder::static_init() ObMockIteratorBuilder::parse_first_dml) || OB_SUCCESS != str_to_info_parse_func_.set_refactored( ObString::make_string("multi_version_row_flag"), - ObMockIteratorBuilder::parse_multi_version_row_flag) - || OB_SUCCESS != str_to_info_parse_func_.set_refactored( - ObString::make_string("is_get"), - ObMockIteratorBuilder::parse_is_get) + ObMockIteratorBuilder::parse_obj_multi_version_row_flag) || OB_SUCCESS != str_to_info_parse_func_.set_refactored( ObString::make_string("scan_index"), - ObMockIteratorBuilder::parse_scan_index) - || OB_SUCCESS != str_to_info_parse_func_.set_refactored( - ObString::make_string("from_base"), - ObMockIteratorBuilder::parse_base) + ObMockIteratorBuilder::parse_obj_scan_index) || OB_SUCCESS != str_to_info_parse_func_.set_refactored( ObString::make_string("trans_id"), - ObMockIteratorBuilder::parse_trans_id)) { + ObMockIteratorBuilder::parse_obj_trans_id)) { ret = OB_INIT_FAIL; STORAGE_LOG(WARN, "info parse func hashtable insert failed"); } else { @@ -907,36 +1464,6 @@ int ObMockIteratorBuilder::static_init() STORAGE_LOG(DEBUG, "init flag hashtable"); } - if (ret == OB_SUCCESS - && OB_SUCCESS != (ret = str_to_base_.create( - cal_next_prime(BASE_NUM * 2), - ObModIds::OB_HASH_BUCKET))) { - STORAGE_LOG(WARN, "out of memory"); - } else if (OB_SUCCESS != str_to_base_.set_refactored( - ObString::make_string("TRUE"), true) - || OB_SUCCESS != str_to_base_.set_refactored( - ObString::make_string("FALSE"), false)) { - ret = OB_INIT_FAIL; - STORAGE_LOG(WARN, "from_base hashtable insert failed"); - } else { - STORAGE_LOG(DEBUG, "init from_base hashtable"); - } - - if (ret == OB_SUCCESS - && OB_SUCCESS != (ret = str_to_is_get_.create( - cal_next_prime(BASE_NUM * 2), - ObModIds::OB_HASH_BUCKET))) { - STORAGE_LOG(WARN, "out of memory"); - } else if (OB_SUCCESS != str_to_is_get_.set_refactored( - ObString::make_string("TRUE"), true) - || OB_SUCCESS != str_to_is_get_.set_refactored( - ObString::make_string("FALSE"), false)) { - ret = OB_INIT_FAIL; - STORAGE_LOG(WARN, "is_get hashtable insert failed"); - } else { - STORAGE_LOG(DEBUG, "init is_get hashtable"); - } - if (ret == OB_SUCCESS && OB_SUCCESS != (ret = str_to_trans_id_.create( cal_next_prime(BASE_NUM * 3), @@ -1006,22 +1533,22 @@ int ObMockIteratorBuilder::static_init() // for sstable row if (OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("table_type"), - ObMockIteratorBuilder::parse_bigint) + ObMockIteratorBuilder::parse_obj_bigint) || OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("start_scn"), - ObMockIteratorBuilder::parse_bigint) + ObMockIteratorBuilder::parse_obj_bigint) || OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("end_scn"), - ObMockIteratorBuilder::parse_bigint) + ObMockIteratorBuilder::parse_obj_bigint) || OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("max_ver"), - ObMockIteratorBuilder::parse_bigint) + ObMockIteratorBuilder::parse_obj_bigint) || OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("upper_ver"), - ObMockIteratorBuilder::parse_bigint) + ObMockIteratorBuilder::parse_obj_bigint) || OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("row_cnt"), - ObMockIteratorBuilder::parse_bigint)) { + ObMockIteratorBuilder::parse_obj_bigint)) { ret = OB_INIT_FAIL; STORAGE_LOG(WARN, "obj parse func hashtable insert failed"); } else { @@ -1070,6 +1597,65 @@ int ObMockIteratorBuilder::init(ObIAllocator *allocator, char escape) return ret; } +int ObMockIteratorBuilder::parse_for_datum( + const ObString &str, + ObMockIterator &iter, + uint16_t *col_id_array_list) +{ + int ret = OB_SUCCESS; + if (!is_inited_) { + ret = OB_NOT_INIT; + } else { + int64_t pos = 0; + int64_t obj_num = 0; + ObSEArray header; + iter.reset(); + ret = parse_datum_header(str, pos, header, obj_num, iter); + if (OB_ITER_END == ret) { + ret = OB_EMPTY_RESULT; + STORAGE_LOG(WARN, "no data input"); + } else if (OB_FAIL(ret)) { + //OB_ASSERT(false); + // will not happen + STORAGE_LOG(WARN, "parse header error", K(ret)); + } else if (0 == obj_num) { + ret = OB_EMPTY_RESULT; + STORAGE_LOG(WARN, "no data(such as int, var) col", K(str)); + } else if (OB_FAIL(iter.set_column_cnt(obj_num))) { + STORAGE_LOG(WARN, "set column cnt failed", K(ret), K(obj_num)); + } else { + STORAGE_LOG(TRACE, "get header success", K(header.count()), K(obj_num)); + int idx = 0; + while (OB_SUCC(ret)) { + ObDatumRow *row = NULL; + if (OB_FAIL(malloc_datum_row(*allocator_, obj_num, row))) { + STORAGE_LOG(WARN, "failed to malloc store row", K(ret)); + } else { + // set default value + row->row_flag_.set_flag(ObDmlFlag::DF_INSERT); + for (int64_t i = 0; i < obj_num; ++i) { + row->storage_datums_[i].set_nop(); + } + // parse one row + ret = parse_datum_row(str, pos, header, col_id_array_list, *row); + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + STORAGE_LOG(INFO, "parse successfully"); + break; + } else if (OB_FAIL(ret)) { + STORAGE_LOG(WARN, "failed to get row", K(ret), K(pos)); + } else { + STORAGE_LOG(WARN, "add row", K(ret), KPC(row)); + iter.add_row(row); + } + } + + } // end of while + } // end of else + } // end of is_inited_'s else + return ret; +} + int ObMockIteratorBuilder::parse( const ObString &str, ObMockIterator &iter, @@ -1128,6 +1714,68 @@ int ObMockIteratorBuilder::parse( return ret; } +int ObMockIteratorBuilder::parse_datum_header(const ObString &str, + int64_t &pos, + ObIArray &header, + int64_t &obj_num, + ObMockIterator &iter) +{ + OB_ASSERT(is_inited_); + + int ret = OB_SUCCESS; + int64_t ext = NOT_EXT; + obj_num = 0; + while (OB_SUCCESS == ret && pos < str.length()) { + char buf[MAX_DATA_LENGTH]; + ObString word; + word.assign_buffer(buf, MAX_DATA_LENGTH); + ret = get_next_word(str, pos, word, ext); + if (OB_FAIL(ret)) { + // will be OB_ITER_END only + STORAGE_LOG(WARN, "fail", K(ret), K(str), K(pos), K(word), K(obj_num)); + } else if (obj_num >= OB_ROW_MAX_COLUMNS_COUNT) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "too many columns", K(ret), K(obj_num)); + } else if (EXT_END == ext) { + // header parse end + break; + } else { + ObParseDatumFunc fp = NULL; + ObObjMeta *type = NULL; + // tolower, as case must be ignore in following hash get function + for (int64_t i = 0; i < word.length(); i++) { + /* + if (word.ptr()[i] >= 'A' && word.ptr()[i] <= 'Z') { + word.ptr()[i] = static_cast(word.ptr()[i] - 'A' + 'a'); + } + */ + word.ptr()[i] = static_cast(tolower(word.ptr()[i])); + } + //STORAGE_LOG(WARN, "chaser debug", K(word), K(ret)); + // match data col, such as var, int, num... + if (OB_SUCCESS == str_to_datum_parse_func_.get_refactored(word, fp) && NULL != fp) { + if (OB_SUCCESS != str_to_obj_type_.get_refactored(word, type)) { + STORAGE_LOG(WARN, "get obj type from hashmap failed", K(ret)); + } else if (NULL == type) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "type is null", K(ret)); + } else { + iter.set_column_type(obj_num, *type); + ret = header.push_back(fp); + ++obj_num; + } + // match row info, such as dml, flag and from_base + } else if (OB_SUCCESS == str_to_datum_info_parse_func_.get_refactored(word, fp) + && NULL != fp) { + ret = header.push_back(fp); + } else { + ret = OB_OBJ_TYPE_ERROR; + } + } + } + return ret; +} + int ObMockIteratorBuilder::parse_header(const ObString &str, int64_t &pos, ObIArray &header, @@ -1190,6 +1838,76 @@ int ObMockIteratorBuilder::parse_header(const ObString &str, return ret; } +int ObMockIteratorBuilder::parse_datum_with_specified_col_ids( + const ObString &str, + ObMockIterator &iter, + uint16_t *col_id_array_list/* = nullptr*/, + int64_t *result_col_id_array/* = nullptr*/) +{ + int ret = OB_SUCCESS; + if (!is_inited_) { + ret = OB_NOT_INIT; + } else { + int64_t pos = 0; + int64_t obj_num = 0; + ObSEArray header; + iter.reset(); + ret = parse_datum_header(str, pos, header, obj_num, iter); + if (OB_ITER_END == ret) { + ret = OB_EMPTY_RESULT; + STORAGE_LOG(WARN, "no data input"); + } else if (OB_FAIL(ret)) { + //OB_ASSERT(false); + // will not happen + STORAGE_LOG(WARN, "parse header error", K(ret)); + } else if (0 == obj_num) { + ret = OB_EMPTY_RESULT; + STORAGE_LOG(WARN, "no data(such as int, var) col"); + } else if (OB_FAIL(iter.set_column_cnt(obj_num))) { + STORAGE_LOG(WARN, "set column cnt failed", K(ret), K(obj_num)); + } else { + STORAGE_LOG(TRACE, "get header success", K(header.count()), K(obj_num)); + bool init_col_id_flag = false; + if (OB_NOT_NULL(col_id_array_list) && OB_NOT_NULL(result_col_id_array)) { + init_col_id_flag = true; + } + int idx = 0; + int col_id_pos = 0; + while (OB_SUCC(ret)) { + ObDatumRow *row = NULL; + if (OB_FAIL(malloc_datum_row(*allocator_, obj_num, row))) { + STORAGE_LOG(WARN, "failed to malloc store row", K(ret)); + } else { + // set default value + row->row_flag_.set_flag(ObDmlFlag::DF_INSERT); + for (int64_t i = 0; i < obj_num; ++i) { + row->storage_datums_[i].set_nop(); + } + // parse one row + ret = parse_datum_row(str, pos, header, nullptr, *row); + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + STORAGE_LOG(INFO, "parse successfully"); + break; + } else if (OB_FAIL(ret)) { + STORAGE_LOG(WARN, "failed to get row", K(ret), K(pos)); + } else { + if (init_col_id_flag) { + row->count_ = result_col_id_array[idx]; + col_id_pos += result_col_id_array[idx]; // move forward + STORAGE_LOG(TRACE, "get row successfully", K(idx), K(*row), K(row->trans_id_)); + ++idx; + } + iter.add_row(row); + } + } + + } // end of while + } // end of else + } // end of is_inited_'s else + return ret; +} + int ObMockIteratorBuilder::parse_with_specified_col_ids( const ObString &str, ObMockIterator &iter, @@ -1262,6 +1980,85 @@ int ObMockIteratorBuilder::parse_with_specified_col_ids( return ret; } +int ObMockIteratorBuilder::parse_datum_row(const ObString &str, + int64_t &pos, + const ObIArray &header, + const uint16_t *col_id_array_list, + blocksstable::ObDatumRow &row) +{ + OB_ASSERT(is_inited_); + + int ret = OB_SUCCESS; + int64_t idx = 0; + int64_t ext = NOT_EXT; + int64_t col_id_idx = 0; + // i <= header.count() so row end is included + for (int64_t i = 0; OB_SUCC(ret) && i <= header.count(); ++i) { + char buf[MAX_DATA_LENGTH]; + ObString word; + ObParseDatumFunc fp = NULL; + word.assign_buffer(buf, MAX_DATA_LENGTH); + if (OB_SUCCESS != (ret = get_next_word(str, pos, word, ext))) { + // must be OB_ITER_END only + break; + } else if (EXT_END == ext) { + // check col num if flag is RF_ROW_EXIST + break; + } else if (OB_SUCCESS != (ret = header.at(i, fp))) { + STORAGE_LOG(WARN, "invalid array index"); + } else { + //STORAGE_LOG(DEBUG, "parsing word", K(word)); + switch (ext) { + case EXT_NOP: + row.storage_datums_[idx++].set_nop(); + break; + case EXT_NULL: + row.storage_datums_[idx++].set_null(); + break; + case EXT_MAX: + if (ObMockIteratorBuilder::parse_datum_int != fp && ObMockIteratorBuilder::parse_datum_bigint != fp) { + row.storage_datums_[idx++].set_max(); + } else { + row.storage_datums_[idx++].set_int(INT64_MAX); + } + break; + case EXT_MIN: + if (ObMockIteratorBuilder::parse_datum_int != fp && ObMockIteratorBuilder::parse_datum_bigint != fp) { + row.storage_datums_[idx++].set_min(); + } else { + row.storage_datums_[idx++].set_int(-INT64_MAX); + } + break; + case EXT_MIN_2_TRANS: + if (ObMockIteratorBuilder::parse_datum_int != fp && ObMockIteratorBuilder::parse_datum_bigint != fp) { + row.storage_datums_[idx++].set_min(); + } else { + row.storage_datums_[idx++].set_int(-(INT64_MAX - 7)); + } + break; + case EXT_GHOST: + row.storage_datums_[idx++].set_int(ObGhostRowUtil::GHOST_NUM); + break; + case NOT_EXT: + // use parse func to parse a word + ret = (*fp)(allocator_, word, row, idx); + if (OB_ARRAY_OUT_OF_RANGE == ret) { + STORAGE_LOG(WARN, "data col out of range, you may missing '\\n'?"); + } else if (OB_FAIL(ret)) { + STORAGE_LOG(WARN, "failed to parse word", K(ret), K(word)); + } + break; + default: + OB_ASSERT(false); + } // end of switch + } // end of else + } // end of for + if (OB_SUCC(ret)) { + row.count_ = idx; + } + return ret; +} + int ObMockIteratorBuilder::parse_row(const ObString &str, int64_t &pos, const ObIArray &header, @@ -1302,21 +2099,21 @@ int ObMockIteratorBuilder::parse_row(const ObString &str, row.row_val_.cells_[idx++].set_null(); break; case EXT_MAX: - if (ObMockIteratorBuilder::parse_int != fp && ObMockIteratorBuilder::parse_bigint != fp) { + if (ObMockIteratorBuilder::parse_obj_int != fp && ObMockIteratorBuilder::parse_obj_bigint != fp) { row.row_val_.cells_[idx++].set_max_value(); } else { row.row_val_.cells_[idx++].set_int(INT64_MAX); } break; case EXT_MIN: - if (ObMockIteratorBuilder::parse_int != fp && ObMockIteratorBuilder::parse_bigint != fp) { + if (ObMockIteratorBuilder::parse_obj_int != fp && ObMockIteratorBuilder::parse_obj_bigint != fp) { row.row_val_.cells_[idx++].set_min_value(); } else { row.row_val_.cells_[idx++].set_int(-INT64_MAX); } break; case EXT_MIN_2_TRANS: - if (ObMockIteratorBuilder::parse_int != fp && ObMockIteratorBuilder::parse_bigint != fp) { + if (ObMockIteratorBuilder::parse_obj_int != fp && ObMockIteratorBuilder::parse_obj_bigint != fp) { row.row_val_.cells_[idx++].set_min_value(); } else { row.row_val_.cells_[idx++].set_int(-(INT64_MAX - 7)); @@ -1338,7 +2135,7 @@ int ObMockIteratorBuilder::parse_row(const ObString &str, OB_ASSERT(false); } // end of switch if (OB_SUCC(ret) && row.is_sparse_row_) { - if (EXT_NOP != ext && parse_flag != fp && parse_multi_version_row_flag != fp && parse_trans_id != fp) { + if (EXT_NOP != ext && parse_obj_flag != fp && parse_obj_multi_version_row_flag != fp && parse_obj_trans_id != fp) { row.column_ids_[idx - 1] = col_id_array_list[col_id_idx++]; } } @@ -1487,111 +2284,111 @@ int ObMockIteratorBuilder::handle_escape(const ObString &str, int64_t &pos, char return ret; } -MockObNewRowIterator::MockObNewRowIterator() -{ - allocator_.set_label(ObModIds::OB_TRANS_CHECK); -} +// MockObNewRowIterator::MockObNewRowIterator() +// { +// allocator_.set_label(ObModIds::OB_TRANS_CHECK); +// } -MockObNewRowIterator::~MockObNewRowIterator() -{ - allocator_.free(); -} +// MockObNewRowIterator::~MockObNewRowIterator() +// { +// allocator_.free(); +// } -void MockObNewRowIterator::reset() -{ - allocator_.free(); - return iter_.reset(); -} +// void MockObNewRowIterator::reset() +// { +// allocator_.free(); +// return iter_.reset(); +// } -bool MockObNewRowIterator::equals(const ObNewRow &r1, const ObNewRow &r2) -{ - bool bool_ret = true; - int64_t idx = 0; - if (!r1.cells_ || !r2.cells_) { - bool_ret = false; - } else if (r1.count_ != r2.count_) { - bool_ret = false; - } else { - for (idx = 0; idx < r1.count_ && bool_ret; idx++) { - bool_ret = (r1.cells_[idx] == r2.cells_[idx]); - } - } - return bool_ret; -} +// bool MockObNewRowIterator::equals(const ObNewRow &r1, const ObNewRow &r2) +// { +// bool bool_ret = true; +// int64_t idx = 0; +// if (!r1.cells_ || !r2.cells_) { +// bool_ret = false; +// } else if (r1.count_ != r2.count_) { +// bool_ret = false; +// } else { +// for (idx = 0; idx < r1.count_ && bool_ret; idx++) { +// bool_ret = (r1.cells_[idx] == r2.cells_[idx]); +// } +// } +// return bool_ret; +// } -OB_DEF_SERIALIZE(MockObNewRowIterator) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(serialization::encode_vi64(buf, buf_len, pos, iter_.count()))) { - TRANS_LOG(WARN, "MockObNewRowIterator serialize error", K(ret), K(iter_.count())); - } else { - ObStoreRow *row = NULL; - for (int64_t i = 0; OB_SUCC(ret) && i < iter_.count(); i++) { - if (OB_SUCCESS != (ret = get_row(i, row))) { - TRANS_LOG(WARN, "MockObNewRowIterator get row error", K(ret), K(i)); - } else if (OB_SUCCESS != (ret = row->serialize(buf, buf_len, pos))) { - TRANS_LOG(WARN, "MockObNewRowIterator serialize ObStoreRow error", K(ret), K(row->row_val_)); - } else { - TRANS_LOG(DEBUG, "MockObNewRowIterator serialize ObStoreRow success", K(row->row_val_)); - } - } - } +// OB_DEF_SERIALIZE(MockObNewRowIterator) +// { +// int ret = OB_SUCCESS; +// if (OB_FAIL(serialization::encode_vi64(buf, buf_len, pos, iter_.count()))) { +// TRANS_LOG(WARN, "MockObNewRowIterator serialize error", K(ret), K(iter_.count())); +// } else { +// ObStoreRow *row = NULL; +// for (int64_t i = 0; OB_SUCC(ret) && i < iter_.count(); i++) { +// if (OB_SUCCESS != (ret = get_row(i, row))) { +// TRANS_LOG(WARN, "MockObNewRowIterator get row error", K(ret), K(i)); +// } else if (OB_SUCCESS != (ret = row->serialize(buf, buf_len, pos))) { +// TRANS_LOG(WARN, "MockObNewRowIterator serialize ObStoreRow error", K(ret), K(row->row_val_)); +// } else { +// TRANS_LOG(DEBUG, "MockObNewRowIterator serialize ObStoreRow success", K(row->row_val_)); +// } +// } +// } - return ret; -} +// return ret; +// } -OB_DEF_DESERIALIZE(MockObNewRowIterator) -{ - int ret = OB_SUCCESS; - int64_t count = 0; +// OB_DEF_DESERIALIZE(MockObNewRowIterator) +// { +// int ret = OB_SUCCESS; +// int64_t count = 0; - reset(); +// reset(); - if (OB_FAIL(serialization::decode_vi64(buf, data_len, pos, &count))) { - TRANS_LOG(WARN, "MockObNewRowIterator deserialize row size error", K(ret), K(count)); - } else { - TRANS_LOG(INFO, "MockObNewRowIterator deserialize iter count", K(count)); - for (int64_t i = 0; OB_SUCC(ret) && i < count; i++) { - ObStoreRow row; - ObObj *ptr = (ObObj*)allocator_.alloc(sizeof(ObObj) * OB_ROW_MAX_COLUMNS_COUNT); - if (NULL == ptr) { - TRANS_LOG(WARN, "MockObNewRowIterator alloc ObObj error", K(ret)); - ret = OB_ALLOCATE_MEMORY_FAILED; - } else { - row.row_val_.assign(ptr, OB_ROW_MAX_COLUMNS_COUNT); - if (OB_SUCCESS != (ret = row.deserialize(buf, data_len, pos))) { - TRANS_LOG(WARN, "MockObNewRowIterator deserialize ObNewRow error", K(ret)); - } else { - if (OB_SUCCESS != (ret = add_row(&row))) { - TRANS_LOG(WARN, "MockObNewRowIterator deserialize add row error", K(ret), K(row.row_val_)); - } else { - TRANS_LOG(DEBUG, "MockObNewRowIterator deserialize add row success", K(row.row_val_)); - } - } - } - } - } +// if (OB_FAIL(serialization::decode_vi64(buf, data_len, pos, &count))) { +// TRANS_LOG(WARN, "MockObNewRowIterator deserialize row size error", K(ret), K(count)); +// } else { +// TRANS_LOG(INFO, "MockObNewRowIterator deserialize iter count", K(count)); +// for (int64_t i = 0; OB_SUCC(ret) && i < count; i++) { +// ObStoreRow row; +// ObObj *ptr = (ObObj*)allocator_.alloc(sizeof(ObObj) * OB_ROW_MAX_COLUMNS_COUNT); +// if (NULL == ptr) { +// TRANS_LOG(WARN, "MockObNewRowIterator alloc ObObj error", K(ret)); +// ret = OB_ALLOCATE_MEMORY_FAILED; +// } else { +// row.row_val_.assign(ptr, OB_ROW_MAX_COLUMNS_COUNT); +// if (OB_SUCCESS != (ret = row.deserialize(buf, data_len, pos))) { +// TRANS_LOG(WARN, "MockObNewRowIterator deserialize ObNewRow error", K(ret)); +// } else { +// if (OB_SUCCESS != (ret = add_row(&row))) { +// TRANS_LOG(WARN, "MockObNewRowIterator deserialize add row error", K(ret), K(row.row_val_)); +// } else { +// TRANS_LOG(DEBUG, "MockObNewRowIterator deserialize add row success", K(row.row_val_)); +// } +// } +// } +// } +// } - return ret; -} +// return ret; +// } -OB_DEF_SERIALIZE_SIZE(MockObNewRowIterator) -{ - int ret = OB_SUCCESS; - int64_t len = 0; +// OB_DEF_SERIALIZE_SIZE(MockObNewRowIterator) +// { +// int ret = OB_SUCCESS; +// int64_t len = 0; - len += serialization::encoded_length_vi64(iter_.count()); - for (int64_t i = 0; OB_SUCC(ret) && i < iter_.count(); i++) { - ObStoreRow *row = NULL; - if (OB_SUCCESS != (ret = get_row(i, row))) { - TRANS_LOG(WARN, "MockObNewRowIterator get_serialize size get row error", K(ret), K(i)); - } else { - len += row->get_serialize_size(); - } - } +// len += serialization::encoded_length_vi64(iter_.count()); +// for (int64_t i = 0; OB_SUCC(ret) && i < iter_.count(); i++) { +// ObStoreRow *row = NULL; +// if (OB_SUCCESS != (ret = get_row(i, row))) { +// TRANS_LOG(WARN, "MockObNewRowIterator get_serialize size get row error", K(ret), K(i)); +// } else { +// len += row->get_serialize_size(); +// } +// } - return len; -} +// return len; +// } int ObMockDirectReadIterator::init(ObStoreRowIterator *iter, common::ObIAllocator &alloc, diff --git a/unittest/storage/mockcontainer/mock_ob_iterator.h b/unittest/storage/mockcontainer/mock_ob_iterator.h index 00dd49a8a..8f1806ccb 100644 --- a/unittest/storage/mockcontainer/mock_ob_iterator.h +++ b/unittest/storage/mockcontainer/mock_ob_iterator.h @@ -23,6 +23,7 @@ #include "storage/access/ob_store_row_iterator.h" #include "storage/access/ob_table_read_info.h" #include "storage/access/ob_sstable_row_whole_scanner.h" +#include "storage/blocksstable/ob_datum_row_iterator.h" namespace oceanbase { @@ -39,10 +40,16 @@ public: int64_t count() const { return rows_.count(); } // get row at idx, do not move iter + int get_row(const int64_t idx, const blocksstable::ObDatumRow *&row) const; + int get_row(const int64_t idx, blocksstable::ObDatumRow *&row) const; + // get row at idx, do not move iter int get_row(const int64_t idx, const storage::ObStoreRow *&row) const; int get_row(const int64_t idx, storage::ObStoreRow *&row) const; int get_row(const int64_t idx, common::ObNewRow *&row) const; // get row and move iter to the next + int get_next_row(const blocksstable::ObDatumRow *&row); + int get_next_row(blocksstable::ObDatumRow *&row); + // get row and move iter to the next int get_next_row(const storage::ObStoreRow *&row); int get_next_row(storage::ObStoreRow *&row); int get_next_row(common::ObNewRow *&row); @@ -52,6 +59,7 @@ public: // rewind iter void reset_iter(); + int add_row(blocksstable::ObDatumRow *row); int add_row(storage::ObStoreRow *row); int from( const char *cstr, @@ -63,6 +71,24 @@ public: char escape = '\\', uint16_t* col_id_array = nullptr, int64_t *result_col_id_array = nullptr); + int from_for_datum( + const char *cstr, + char escape = '\\', + uint16_t* col_id_array = nullptr, + int64_t *result_col_id_array = nullptr); + int from_for_datum( + const common::ObString &str, + char escape = '\\', + uint16_t* col_id_array = nullptr, + int64_t *result_col_id_array = nullptr); + + static bool inner_equals(const blocksstable::ObDatumRow &r1, const blocksstable::ObDatumRow &r2); + static bool equals(const blocksstable::ObDatumRow &r1, const blocksstable::ObDatumRow &r2, + const bool cmp_multi_version_row_flag = false, const bool cmp_is_get_and_scan_index = false); + bool equals(int64_t idx, blocksstable::ObDatumRow &row) const; + bool equals(int64_t idx, const blocksstable::ObDatumRow &row) const; + bool equals(blocksstable::ObDatumRow &other_row) const { return equals(0, other_row); } + bool equals(const blocksstable::ObDatumRow &other_row) const { return equals(0, other_row); } static bool equals(const common::ObNewRow &r1, const common::ObNewRow &r2); static bool equals(const storage::ObStoreRow &r1, const storage::ObStoreRow &r2, @@ -97,15 +123,15 @@ public: return ret; } - template + template bool equals(T &other_iter, const bool cmp_multi_version_row_flag = false, const bool cmp_is_get_and_scan_index = false) { bool bool_ret = true; int ret1 = common::OB_SUCCESS; int ret2 = common::OB_SUCCESS; - const storage::ObStoreRow *other_row = NULL; - storage::ObStoreRow *this_row = NULL; + const T_ROW *other_row = NULL; + T_ROW *this_row = NULL; int64_t idx = 0; while (bool_ret) { @@ -155,7 +181,7 @@ private: void setup_start_cursor(); void advance(); bool end_of_row() const; - + common::ObSEArray datum_rows_; common::ObSEArray rows_; int64_t column_cnt_; common::ObObjMeta metas_[common::OB_ROW_MAX_COLUMNS_COUNT]; @@ -177,6 +203,8 @@ public: void reset_iter() { return iter_.reset_iter(); } int from(const char *cstr, char escape = '\\') { return iter_.from(cstr, escape); } int from(const common::ObString &str, char escape = '\\') { return iter_.from(str, escape); } + int from_for_datum(const char *cstr, char escape = '\\') { return iter_.from_for_datum(cstr, escape); } + int from_for_datum(const common::ObString &str, char escape = '\\') { return iter_.from_for_datum(str, escape); } bool equals(ROW_TYPE &row) const { return iter_.equals(row); } bool equals(int64_t idx, ROW_TYPE &row) const { return iter_.equals(idx, row); } // compare to an iterator, rewind the iter before call this function @@ -219,10 +247,10 @@ public: private: ObMockIterator iter_; }; -typedef ObMockRowIterator -ObMockStoreRowIterator; +typedef ObMockRowIterator ObMockStoreRowIterator; typedef ObMockRowIterator ObMockQueryRowIterator; typedef ObMockRowIterator ObMockNewRowIterator; +typedef ObMockRowIterator ObMockDatumRowIterator; // init // parse -> parse->header @@ -270,6 +298,10 @@ public: static common::ObObjMeta TS_TYPE; static common::ObObjMeta NU_TYPE; + typedef int (*ObParseDatumFunc)(common::ObIAllocator *, + const common::ObString &, + blocksstable::ObDatumRow &, + int64_t &); typedef int (*ObParseFunc)(common::ObIAllocator *, const common::ObString &, storage::ObStoreRow &, @@ -281,10 +313,19 @@ public: ~ObMockIteratorBuilder() {} int init(common::ObIAllocator *allocator = NULL, char escape = '\\'); + int parse_for_datum( + const common::ObString &str, + ObMockIterator &iter, + uint16_t *col_id_array_list = nullptr); int parse( const common::ObString &str, ObMockIterator &iter, uint16_t *col_id_array_list = nullptr); + int parse_datum_with_specified_col_ids( + const ObString &str, + ObMockIterator &iter, + uint16_t *col_id_array_list = nullptr, + int64_t *result_col_id_array = nullptr); int parse_with_specified_col_ids( const ObString &str, ObMockIterator &iter, @@ -292,72 +333,164 @@ public: int64_t *result_col_id_array = nullptr); private: static int static_init(); - static int parse_varchar(common::ObIAllocator *allocator, + static int prepare_parse_varchar(common::ObIAllocator *allocator, + const common::ObString &word, + const uint16_t count, + int64_t &idx); + static int parse_datum_varchar(common::ObIAllocator *allocator, + const common::ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx); + static int parse_obj_varchar(common::ObIAllocator *allocator, + const common::ObString &word, + storage::ObStoreRow &row, + int64_t &idx); + static int prepare_parse_lob(common::ObIAllocator *allocator, + const common::ObString &word, + const uint16_t count, + int64_t &idx, + ObLobCommon *&lob_data, + int64_t &val); + static int parse_datum_lob(common::ObIAllocator *allocator, + const common::ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx); + static int parse_obj_lob(common::ObIAllocator *allocator, const common::ObString &word, storage::ObStoreRow &row, int64_t &idx); - static int parse_lob(common::ObIAllocator *allocator, - const common::ObString &word, - storage::ObStoreRow &row, - int64_t &idx); /* static int parse_bool(common::ObIAllocator *allocator, - const common::ObString &word, storage::ObStoreRow &row, int64_t &idx); + const common::ObString &word, blocksstable::ObDatumRow &row, int64_t &idx); */ - static int parse_timestamp(common::ObIAllocator *allocator, + static int prepare_parse_timestamp(common::ObIAllocator *allocator, + const common::ObString &word, + const uint16_t count, + int64_t &idx, + int64_t &usec); + static int parse_datum_timestamp(common::ObIAllocator *allocator, + const common::ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx); + static int parse_obj_timestamp(common::ObIAllocator *allocator, + const common::ObString &word, + storage::ObStoreRow &row, + int64_t &idx); + static int prepare_parse_int(common::ObIAllocator *allocator, + const common::ObString &word, + const uint16_t count, + int64_t &idx, + int64_t &val); + static int parse_datum_int(common::ObIAllocator *allocator, const common::ObString &word, - storage::ObStoreRow &row, + blocksstable::ObDatumRow &row, int64_t &idx); - static int parse_int(common::ObIAllocator *allocator, - const common::ObString &word, - storage::ObStoreRow &row, - int64_t &idx); - static int parse_bigint(common::ObIAllocator *allocator, - const common::ObString &word, - storage::ObStoreRow &row, - int64_t &idx); - static int parse_number(common::ObIAllocator *allocator, - const common::ObString &word, - storage::ObStoreRow &row, - int64_t &idx); + static int parse_obj_int(common::ObIAllocator *allocator, + const common::ObString &word, + storage::ObStoreRow &row, + int64_t &idx); + static int prepare_parse_bigint(common::ObIAllocator *allocator, + const common::ObString &word, + const uint16_t count, + int64_t &idx, + int64_t &val); + static int parse_datum_bigint(common::ObIAllocator *allocator, + const common::ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx); + static int parse_obj_bigint(common::ObIAllocator *allocator, + const common::ObString &word, + storage::ObStoreRow &row, + int64_t &idx); + static int prepare_parse_number(common::ObIAllocator *allocator, + const common::ObString &word, + const uint16_t count, + int64_t &idx, + number::ObNumber &nmb); + static int parse_datum_number(common::ObIAllocator *allocator, + const common::ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx); + static int parse_obj_number(common::ObIAllocator *allocator, + const common::ObString &word, + storage::ObStoreRow &row, + int64_t &idx); static int parse_dml(common::ObIAllocator *allocator, const common::ObString &word, storage::ObStoreRow &row, int64_t &idx); + static int parse_datum_dml(common::ObIAllocator *allocator, + const common::ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx); static int parse_first_dml(common::ObIAllocator *allocator, const common::ObString &word, storage::ObStoreRow &row, int64_t &idx); - static int parse_flag(common::ObIAllocator *allocator, - const common::ObString &word, - storage::ObStoreRow &row, - int64_t &idx); + static int prepare_parse_flag(common::ObIAllocator *allocator, + const common::ObString &word, + int64_t &idx, + int64_t &flag); + static int parse_datum_flag(common::ObIAllocator *allocator, + const common::ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx); + static int parse_obj_flag(common::ObIAllocator *allocator, + const common::ObString &word, + storage::ObStoreRow &row, + int64_t &idx); static int parse_base(common::ObIAllocator *allocator, const common::ObString &word, storage::ObStoreRow &row, int64_t &idx); - static int parse_multi_version_row_flag(common::ObIAllocator *allocator, - const common::ObString &word, - storage::ObStoreRow &row, - int64_t &idx); + static int parse_datum_multi_version_row_flag(common::ObIAllocator *allocator, + const common::ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx); + static int parse_obj_multi_version_row_flag(common::ObIAllocator *allocator, + const common::ObString &word, + storage::ObStoreRow &row, + int64_t &idx); static int parse_is_get(common::ObIAllocator *allocator, const common::ObString &word, storage::ObStoreRow &row, int64_t &idx); - static int parse_scan_index(common::ObIAllocator *allocator, + static int prepare_parse_scan_index(common::ObIAllocator *allocator, + const common::ObString &word, + int64_t &idx, + int64_t &val); + static int parse_datum_scan_index(common::ObIAllocator *allocator, + const common::ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx); + static int parse_obj_scan_index(common::ObIAllocator *allocator, + const common::ObString &word, + storage::ObStoreRow &row, + int64_t &idx); + static int parse_datum_trans_id(common::ObIAllocator *allocator, const common::ObString &word, - storage::ObStoreRow &row, - int64_t &idx); - static int parse_trans_id(common::ObIAllocator *allocator, - const common::ObString &word, - storage::ObStoreRow &row, + blocksstable::ObDatumRow &row, int64_t &idx); + static int parse_obj_trans_id(common::ObIAllocator *allocator, + const common::ObString &word, + storage::ObStoreRow &row, + int64_t &idx); + int parse_datum_header(const ObString &str, + int64_t &pos, + ObIArray &header, + int64_t &obj_num, + ObMockIterator &iter); int parse_header(const common::ObString &str, int64_t &pos, common::ObIArray &header, int64_t &obj_num, ObMockIterator &iter); + int parse_datum_row(const common::ObString &str, + int64_t &pos, + const common::ObIArray &header, + const uint16_t *col_id_array_list, + blocksstable::ObDatumRow &row); int parse_row(const common::ObString &str, int64_t &pos, const common::ObIArray &header, @@ -388,6 +521,10 @@ private: static common::hash::ObHashMap str_to_obj_parse_func_; // hash ObString to row info parse func , such as parse_dml ... static common::hash::ObHashMap str_to_info_parse_func_; + // hash ObString to obj parse func , such as parse_int ... + static common::hash::ObHashMap str_to_datum_parse_func_; + // hash ObString to row info parse func , such as parse_dml ... + static common::hash::ObHashMap str_to_datum_info_parse_func_; static common::hash::ObHashMap str_to_obj_type_; static common::hash::ObHashMap str_to_flag_; static common::hash::ObHashMap str_to_dml_; @@ -402,25 +539,25 @@ private: char escape_; }; -class MockObNewRowIterator: public ObNewRowIterator -{ - OB_UNIS_VERSION(1); -public: - MockObNewRowIterator(); - ~MockObNewRowIterator(); - static bool equals(const common::ObNewRow &r1, const common::ObNewRow &r2); - void reset(); - int from(const char *cstr, char escape = '\\') { return iter_.from(cstr, escape); } - int from(const common::ObString &str, char escape = '\\') { return iter_.from(str, escape); } - int get_next_row(ObNewRow *&row) { return iter_.get_next_row(row); } - int get_row(const int64_t idex, storage::ObStoreRow *&row) const { return iter_.get_row(idex, row); } - int64_t count() const { return iter_.count(); } - int add_row(storage::ObStoreRow *row) { return iter_.add_row(row); } +// class MockObNewRowIterator: public ObNewRowIterator +// { +// OB_UNIS_VERSION(1); +// public: +// MockObNewRowIterator(); +// ~MockObNewRowIterator(); +// static bool equals(const common::ObNewRow &r1, const common::ObNewRow &r2); +// void reset(); +// int from(const char *cstr, char escape = '\\') { return iter_.from(cstr, escape); } +// int from(const common::ObString &str, char escape = '\\') { return iter_.from(str, escape); } +// int get_next_row(ObNewRow *&row) { return iter_.get_next_row(row); } +// int get_row(const int64_t idex, storage::ObStoreRow *&row) const { return iter_.get_row(idex, row); } +// int64_t count() const { return iter_.count(); } +// int add_row(storage::ObStoreRow *row) { return iter_.add_row(row); } -private: - ObMockIterator iter_; - common::PageArena allocator_; -}; +// private: +// ObMockIterator iter_; +// common::PageArena allocator_; +// }; class ObMockDirectReadIterator : public storage::ObStoreRowIterator { diff --git a/unittest/storage/test_ob_mock_iterator.cpp b/unittest/storage/test_ob_mock_iterator.cpp index b5a0e1663..f48cc71ff 100644 --- a/unittest/storage/test_ob_mock_iterator.cpp +++ b/unittest/storage/test_ob_mock_iterator.cpp @@ -429,17 +429,17 @@ TEST_F(TestObMockIteratorBuilder, equals) TEST_F(TestObMockIteratorBuilder, test_new_row) { - ObMockNewRowIterator iter1; + ObMockDatumRowIterator iter1; const char *input1 = "int var num\n" "1 2 3\n"; - ASSERT_EQ(OB_SUCCESS, iter1.from(input1)); + ASSERT_EQ(OB_SUCCESS, iter1.from_for_datum(input1)); - ObMockNewRowIterator iter2; + ObMockDatumRowIterator iter2; const char *input2 = "int var num\n" "1 2 4\n"; - EXPECT_EQ(OB_SUCCESS, iter2.from(input2)); + EXPECT_EQ(OB_SUCCESS, iter2.from_for_datum(input2)); EXPECT_FALSE(iter2.equals(iter1)); } diff --git a/unittest/storage/tx/it/tx_node.cpp b/unittest/storage/tx/it/tx_node.cpp index 72c7cb10d..ce128208c 100644 --- a/unittest/storage/tx/it/tx_node.cpp +++ b/unittest/storage/tx/it/tx_node.cpp @@ -658,12 +658,13 @@ int ObTxNode::write(ObTxDesc &tx, write_store_ctx)); write_store_ctx.mvcc_acc_ctx_.tx_table_guards_.tx_table_guard_.init(&fake_tx_table_); ObArenaAllocator allocator; - ObStoreRow row; - ObObj cols[2] = {ObObj(key), ObObj(value)}; - row.capacity_ = 2; - row.row_val_.cells_ = cols; - row.row_val_.count_ = 2; - row.flag_ = blocksstable::ObDmlFlag::DF_UPDATE; + ObDatumRow row; + ObStorageDatum cols[2] = {ObStorageDatum(), ObStorageDatum()}; + cols[0].set_int(key); + cols[1].set_int(value); + row.count_ = 2; + row.storage_datums_ = cols; + row.row_flag_ = blocksstable::ObDmlFlag::DF_UPDATE; row.trans_id_.reset(); ObTableIterParam param; @@ -723,11 +724,13 @@ int ObTxNode::write_one_row(ObStoreCtx& write_store_ctx, const int64_t key, cons const transaction::ObSerializeEncryptMeta *encrypt_meta = NULL; const int64_t schema_version = 100; read_info.init(allocator, 2, 1, false, columns_, nullptr/*storage_cols_index*/); - ObStoreRow row; - ObObj cols[2] = {ObObj(key), ObObj(value)}; - row.flag_ = blocksstable::ObDmlFlag::DF_UPDATE; - row.row_val_.cells_ = cols; - row.row_val_.count_ = 2; + ObDatumRow row; + ObStorageDatum cols[2] = {ObStorageDatum(), ObStorageDatum()}; + cols[0].set_int(key); + cols[1].set_int(value); + row.row_flag_ = blocksstable::ObDmlFlag::DF_UPDATE; + row.storage_datums_ = cols; + row.count_ = 2; ObTableIterParam param; ObTableAccessContext context; From b81b5f9d1f4b6d6eb4903cb27db10486791a33d3 Mon Sep 17 00:00:00 2001 From: obdev Date: Tue, 27 Aug 2024 09:18:23 +0000 Subject: [PATCH 231/249] [CP] [to #2024080700104062851] fix dup condition resolve --- src/pl/ob_pl_resolver.cpp | 41 ++++++++++++++++++++++++++------------- src/pl/ob_pl_resolver.h | 5 ++++- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/src/pl/ob_pl_resolver.cpp b/src/pl/ob_pl_resolver.cpp index 3f80f892c..75c355d1c 100644 --- a/src/pl/ob_pl_resolver.cpp +++ b/src/pl/ob_pl_resolver.cpp @@ -6724,12 +6724,22 @@ int ObPLResolver::resolve_declare_handler(const ObStmtNodeTree *parse_tree, ObPL ObPLConditionType actual_type = INVALID_TYPE; if (OB_FAIL(resolve_handler_condition(handler_list->children_[i], value, func))) { LOG_WARN("failed to resolve condition value", K(handler_list->children_[i]), K(ret)); - } else if (OB_FAIL(check_duplicate_condition(*stmt, value, dup, desc))) { + } else if (OB_FAIL(check_duplicate_condition(*stmt, value, dup))) { LOG_WARN("failed to check duplication", K(value), K(ret)); } else if (dup) { ret = OB_ERR_SP_DUP_HANDLER; LOG_USER_ERROR(OB_ERR_SP_DUP_HANDLER); LOG_WARN("Duplicate handler declared in the same block", K(value), K(dup), K(ret)); + } else if (OB_FAIL(check_duplicate_condition(value, *desc, dup))) { + LOG_WARN("failed to check duplication", K(ret), K(value), KPC(desc)); + } else if (dup) { + if (lib::is_mysql_mode()) { + ret = OB_ERR_SP_DUP_HANDLER; + LOG_USER_ERROR(OB_ERR_SP_DUP_HANDLER); + LOG_WARN("Duplicate handler declared in the same block", K(value), K(dup), K(ret)); + } else { + // continue, Oracle same Condition on same handle is legal. such as WHEN NO_DATA_FOUND or NO_DATA_FOUND + } } else if (OB_FAIL(ObPLResolver::analyze_actual_condition_type(value, actual_type))) { LOG_WARN("failed to analyze actual condition type", K(value), K(ret)); } else if (lib::is_oracle_mode() @@ -15589,10 +15599,25 @@ int ObPLResolver::resolve_condition_value(const ObStmtNodeTree *parse_tree, return ret; } +int ObPLResolver::check_duplicate_condition(const ObPLConditionValue &value, + ObPLDeclareHandlerStmt::DeclareHandler::HandlerDesc& cur_desc, + bool &dup) +{ + int ret = OB_SUCCESS; + for (int64_t j = 0; !dup && j < cur_desc.get_conditions().count(); ++j) { + if (value.type_ == cur_desc.get_condition(j).type_ && + value.error_code_ == cur_desc.get_condition(j).error_code_ && + value.str_len_ == cur_desc.get_condition(j).str_len_ && + 0 == STRNCMP(value.sql_state_, cur_desc.get_condition(j).sql_state_, value.str_len_)) { + dup = true; + } + } + return ret; +} + int ObPLResolver::check_duplicate_condition(const ObPLDeclareHandlerStmt &stmt, const ObPLConditionValue &value, - bool &dup, - ObPLDeclareHandlerStmt::DeclareHandler::HandlerDesc* cur_desc) + bool &dup) { int ret = OB_SUCCESS; dup = false; @@ -15635,16 +15660,6 @@ int ObPLResolver::check_duplicate_condition(const ObPLDeclareHandlerStmt &stmt, break; } } - if (OB_NOT_NULL(cur_desc) && lib::is_mysql_mode()) { - for (int64_t j = 0; !dup && j < cur_desc->get_conditions().count(); ++j) { - if (value.type_ == cur_desc->get_condition(j).type_ && - value.error_code_ == cur_desc->get_condition(j).error_code_ && - value.str_len_ == cur_desc->get_condition(j).str_len_ && - 0 == STRNCMP(value.sql_state_, cur_desc->get_condition(j).sql_state_, value.str_len_)) { - dup = true; - } - } - } return ret; } diff --git a/src/pl/ob_pl_resolver.h b/src/pl/ob_pl_resolver.h index 75467cb96..c165ff539 100644 --- a/src/pl/ob_pl_resolver.h +++ b/src/pl/ob_pl_resolver.h @@ -949,8 +949,11 @@ private: ObPLFunctionAST &func, ObIArray &access_idxs); private: + int check_duplicate_condition(const ObPLConditionValue &value, + ObPLDeclareHandlerStmt::DeclareHandler::HandlerDesc &cur_desc, + bool &dup); int check_duplicate_condition(const ObPLDeclareHandlerStmt &stmt, const ObPLConditionValue &value, - bool &dup, ObPLDeclareHandlerStmt::DeclareHandler::HandlerDesc* cur_desc); + bool &dup); int analyze_actual_condition_type(const ObPLConditionValue &value, ObPLConditionType &type); #ifdef OB_BUILD_ORACLE_PL int check_collection_constructor(const ParseNode *node, const common::ObString &type_name, bool &is_constructor); From ad224ebdb6edb12b7a815ceef7a70e5b891e645b Mon Sep 17 00:00:00 2001 From: obdev Date: Tue, 27 Aug 2024 11:43:42 +0000 Subject: [PATCH 232/249] patch [FEAT MERGE] support olap async job Co-authored-by: lmjhh <576788582@qq.com> --- .../ob_dbms_sched_job_executor.cpp | 86 +++-- .../ob_dbms_sched_job_executor.h | 1 + .../ob_dbms_sched_job_master.cpp | 21 +- .../ob_dbms_sched_job_rpc_proxy.cpp | 4 +- .../ob_dbms_sched_job_rpc_proxy.h | 2 +- .../ob_dbms_sched_job_utils.cpp | 293 +++++++++++++++-- .../dbms_scheduler/ob_dbms_sched_job_utils.h | 97 +++++- .../ob_dbms_sched_table_operator.cpp | 93 +++++- .../ob_dbms_sched_table_operator.h | 4 +- src/observer/omt/ob_tenant.cpp | 4 +- src/observer/omt/ob_tenant.h | 2 + ...l_virtual_tenant_scheduler_running_job.cpp | 9 + ...all_virtual_tenant_scheduler_running_job.h | 3 +- .../pl_non_reserved_keywords_mysql_mode.c | 3 + src/pl/parser/pl_parser_mysql_mode.l | 1 + src/pl/parser/pl_parser_mysql_mode.y | 35 +- .../ob_inner_table_schema.12401_12450.cpp | 15 + .../inner_table/ob_inner_table_schema_def.py | 1 + src/sql/CMakeLists.txt | 3 + .../engine/cmd/ob_olap_async_job_executor.cpp | 66 ++++ .../engine/cmd/ob_olap_async_job_executor.h | 37 +++ src/sql/executor/ob_cmd_executor.cpp | 6 + src/sql/parser/ob_parser.cpp | 20 +- src/sql/parser/ob_parser.h | 3 + src/sql/parser/sql_parser_mysql_mode.y | 8 + .../ob_values_table_compression.cpp | 6 +- .../cmd/ob_olap_async_job_resolver.cpp | 305 ++++++++++++++++++ .../resolver/cmd/ob_olap_async_job_resolver.h | 42 +++ .../resolver/cmd/ob_olap_async_job_stmt.cpp | 66 ++++ src/sql/resolver/cmd/ob_olap_async_job_stmt.h | 94 ++++++ src/sql/resolver/cmd/ob_show_resolver.cpp | 60 +++- src/sql/resolver/cmd/ob_show_resolver.h | 1 + src/sql/resolver/ob_resolver.cpp | 10 + .../r/mysql/desc_virtual_table_in_sys.result | 1 + .../sql/parser/test_mysql_pl_keyword.result | 4 +- 35 files changed, 1333 insertions(+), 73 deletions(-) create mode 100644 src/sql/engine/cmd/ob_olap_async_job_executor.cpp create mode 100644 src/sql/engine/cmd/ob_olap_async_job_executor.h create mode 100644 src/sql/resolver/cmd/ob_olap_async_job_resolver.cpp create mode 100644 src/sql/resolver/cmd/ob_olap_async_job_resolver.h create mode 100644 src/sql/resolver/cmd/ob_olap_async_job_stmt.cpp create mode 100644 src/sql/resolver/cmd/ob_olap_async_job_stmt.h diff --git a/src/observer/dbms_scheduler/ob_dbms_sched_job_executor.cpp b/src/observer/dbms_scheduler/ob_dbms_sched_job_executor.cpp index 1b31417b8..2f98c91e6 100644 --- a/src/observer/dbms_scheduler/ob_dbms_sched_job_executor.cpp +++ b/src/observer/dbms_scheduler/ob_dbms_sched_job_executor.cpp @@ -24,6 +24,7 @@ #include "observer/ob_inner_sql_connection_pool.h" #include "sql/session/ob_sql_session_info.h" #include "sql/ob_sql.h" +#include "sql/executor/ob_executor_rpc_processor.h" namespace oceanbase { @@ -65,6 +66,7 @@ int ObDBMSSchedJobExecutor::init_session( ObDBMSSchedJobInfo &job_info) { int ret = OB_SUCCESS; + ObPrivSet db_priv_set = OB_PRIV_SET_EMPTY; ObArenaAllocator *allocator = NULL; const bool print_info_log = true; const bool is_sys_tenant = true; @@ -97,19 +99,34 @@ int ObDBMSSchedJobExecutor::init_session( OX (session.set_database_id(database_id)); OZ (session.set_user( user_info->get_user_name(), user_info->get_host_name_str(), user_info->get_user_id())); + OX (session.set_priv_user_id(user_info->get_user_id())); OX (session.set_user_priv_set(user_info->get_priv_set())); + OZ (schema_guard.get_db_priv_set(tenant_id, user_info->get_user_id(), database_name, db_priv_set)); + OX (session.set_db_priv_set(db_priv_set)); OX (session.set_shadow(true)); - if (OB_SUCC(ret) && job_info.is_date_expression_job_class()) { - // set larger timeout for mview scheduler jobs - const int64_t QUERY_TIMEOUT_US = (24 * 60 * 60 * 1000000L); // 24hours - const int64_t TRX_TIMEOUT_US = (24 * 60 * 60 * 1000000L); // 24hours - ObObj query_timeout_obj; - ObObj trx_timeout_obj; - query_timeout_obj.set_int(QUERY_TIMEOUT_US); - trx_timeout_obj.set_int(TRX_TIMEOUT_US); - OZ (session.update_sys_variable(SYS_VAR_OB_QUERY_TIMEOUT, query_timeout_obj)); - OZ (session.update_sys_variable(SYS_VAR_OB_TRX_TIMEOUT, trx_timeout_obj)); + if (OB_SUCC(ret)) { + if (job_info.is_date_expression_job_class()) { + // set larger timeout for mview scheduler jobs + const int64_t QUERY_TIMEOUT_US = (24 * 60 * 60 * 1000000L); // 24hours + const int64_t TRX_TIMEOUT_US = (24 * 60 * 60 * 1000000L); // 24hours + ObObj query_timeout_obj; + ObObj trx_timeout_obj; + query_timeout_obj.set_int(QUERY_TIMEOUT_US); + trx_timeout_obj.set_int(TRX_TIMEOUT_US); + OZ (session.update_sys_variable(SYS_VAR_OB_QUERY_TIMEOUT, query_timeout_obj)); + OZ (session.update_sys_variable(SYS_VAR_OB_TRX_TIMEOUT, trx_timeout_obj)); + } else if (job_info.is_olap_async_job_class()) { + const int64_t QUERY_TIMEOUT_US = ((job_info.get_max_run_duration() - OLAP_ASYNC_JOB_DEVIATION_SECOND) * 1000000L); + const int64_t TRX_TIMEOUT_US = ((job_info.get_max_run_duration() - OLAP_ASYNC_JOB_DEVIATION_SECOND) * 1000000L); + ObObj query_timeout_obj; + ObObj trx_timeout_obj; + query_timeout_obj.set_int(QUERY_TIMEOUT_US); + trx_timeout_obj.set_int(TRX_TIMEOUT_US); + OZ (session.update_sys_variable(SYS_VAR_OB_QUERY_TIMEOUT, query_timeout_obj)); + OZ (session.update_sys_variable(SYS_VAR_OB_TRX_TIMEOUT, trx_timeout_obj)); + } } + return ret; } @@ -136,6 +153,9 @@ int ObDBMSSchedJobExecutor::init_env(ObDBMSSchedJobInfo &job_info, ObSQLSessionI job_info.get_tenant_id(), job_info.get_powner(), user_infos)); OV (1 == user_infos.count(), OB_ERR_UNEXPECTED, K(job_info), K(user_infos)); CK (OB_NOT_NULL(user_info = user_infos.at(0))); + } else if (job_info.get_user_id() != OB_INVALID_ID) { + OZ (schema_guard.get_user_info( + job_info.get_tenant_id(), job_info.get_user_id(), user_info)); } else { ObString user = job_info.get_powner(); if (OB_SUCC(ret)) { @@ -247,6 +267,9 @@ int ObDBMSSchedJobExecutor::run_dbms_sched_job( if (job_info.is_oracle_tenant_) { OZ (what.append_fmt("BEGIN %.*s; END;", job_info.get_what().length(), job_info.get_what().ptr())); + } else if (job_info.is_olap_async_job_class()){ + OZ (what.append_fmt("%.*s", + job_info.get_what().length(), job_info.get_what().ptr())); } else { //mysql mode not support anonymous block OZ (what.append_fmt("CALL %.*s;", @@ -346,6 +369,7 @@ int ObDBMSSchedJobExecutor::run_dbms_sched_job( } } if (OB_SUCC(ret)) { + ObWorkerSessionGuard worker_session_guard(session_info); OZ (ObDBMSSchedJobExecutor::init_env(job_info, *session_info)); CK (OB_NOT_NULL(pool = static_cast(sql_proxy_->get_pool()))); OX (session_info->set_job_info(&job_info)); @@ -383,19 +407,37 @@ int ObDBMSSchedJobExecutor::run_dbms_sched_job(uint64_t tenant_id, bool is_oracl OZ (table_operator_.get_dbms_sched_job_info(tenant_id, is_oracle_tenant, job_id, job_name, allocator, job_info)); if (OB_SUCC(ret)) { - - OZ (run_dbms_sched_job(tenant_id, job_info)); - - int tmp_ret = OB_SUCCESS; - ObString errmsg = common::ob_get_tsi_err_msg(ret); - if (errmsg.empty() && ret != OB_SUCCESS) { - errmsg = ObString(strlen(ob_errpkt_strerror(ret, lib::is_oracle_mode())), - ob_errpkt_strerror(ret, lib::is_oracle_mode())); + if (job_info.is_killed()) { //Intercept user cancellation requests before the actual execution of the job + OZ(table_operator_.update_for_kill(job_info)); + } else { + OZ (run_dbms_sched_job(tenant_id, job_info)); + bool job_is_user_stop = false; + if (OB_ERR_SESSION_INTERRUPTED == ret) { //It may have been the user interrupted, need to check. + int tmp_user_stop_ret = OB_SUCCESS; + bool job_is_killed = false; + if ((tmp_user_stop_ret = table_operator_.get_dbms_sched_job_is_killed(job_info, job_is_killed)) != OB_SUCCESS) { + LOG_WARN("double check get dbms sched job failed", K(tmp_user_stop_ret), K(ret)); + } else if (job_is_killed) { + job_is_user_stop = true; + } + } + int tmp_ret = OB_SUCCESS; + if (job_is_user_stop) { + if ((OB_TMP_FAIL(table_operator_.update_for_kill(job_info)))) { + LOG_WARN("update user stop dbms sched job failed", K(tmp_ret), K(ret)); + } + } else { + ObString errmsg = common::ob_get_tsi_err_msg(ret); + if (errmsg.empty() && ret != OB_SUCCESS) { + errmsg = ObString(strlen(ob_errpkt_strerror(ret, lib::is_oracle_mode())), + ob_errpkt_strerror(ret, lib::is_oracle_mode())); + } + if ((OB_TMP_FAIL(table_operator_.update_for_end(job_info, ret, errmsg)))) { + LOG_WARN("update dbms sched job failed", K(tmp_ret), K(ret)); + } + } + ret = OB_SUCCESS == ret ? tmp_ret : ret; } - if ((tmp_ret = table_operator_.update_for_end(job_info, ret, errmsg)) != OB_SUCCESS) { - LOG_WARN("update dbms sched job failed", K(tmp_ret), K(ret)); - } - ret = OB_SUCCESS == ret ? tmp_ret : ret; } return ret; } diff --git a/src/observer/dbms_scheduler/ob_dbms_sched_job_executor.h b/src/observer/dbms_scheduler/ob_dbms_sched_job_executor.h index a05f63c81..f0ddc9026 100644 --- a/src/observer/dbms_scheduler/ob_dbms_sched_job_executor.h +++ b/src/observer/dbms_scheduler/ob_dbms_sched_job_executor.h @@ -36,6 +36,7 @@ public: int init_env(ObDBMSSchedJobInfo &job_info, sql::ObSQLSessionInfo &session); private: + const static int OLAP_ASYNC_JOB_DEVIATION_SECOND = 60; static int init_session( sql::ObSQLSessionInfo &session, share::schema::ObSchemaGetterGuard &schema_guard, diff --git a/src/observer/dbms_scheduler/ob_dbms_sched_job_master.cpp b/src/observer/dbms_scheduler/ob_dbms_sched_job_master.cpp index 23d050a1e..57d9b2016 100644 --- a/src/observer/dbms_scheduler/ob_dbms_sched_job_master.cpp +++ b/src/observer/dbms_scheduler/ob_dbms_sched_job_master.cpp @@ -294,7 +294,8 @@ int64_t ObDBMSSchedJobMaster::run_job(ObDBMSSchedJobInfo &job_info, ObDBMSSchedJ job_key->get_job_id(), job_key->get_job_name(), execute_addr, - self_addr_))) { + self_addr_, + job_info.is_olap_async_job_class() ? share::OBCG_OLAP_ASYNC_JOB : 0))) { LOG_WARN("failed to run dbms sched job", K(ret), K(job_info), KPC(job_key)); if (is_server_down_error(ret)) { int tmp = OB_SUCCESS; @@ -340,6 +341,9 @@ int ObDBMSSchedJobMaster::scheduler() ob_usleep(MIN_SCHEDULER_INTERVAL); } else { ob_usleep(max(0, delay)); + common::ObCurTraceId::TraceId job_trace_id; + job_trace_id.init(GCONF.self_addr_); + ObTraceIdGuard trace_id_guard(job_trace_id); if (OB_SUCCESS != (tmp_ret = scheduler_task_.wait_vector().remove(scheduler_task_.wait_vector().begin()))) { LOG_WARN("fail to remove job_id from sorted vector", K(ret)); } else if (OB_SUCCESS != (tmp_ret = scheduler_job(job_key))) { @@ -389,6 +393,15 @@ int ObDBMSSchedJobMaster::scheduler_job(ObDBMSSchedJobKey *job_key) LOG_WARN("job is timeout, force update for end", K(job_info), K(now)); } } + } else if (job_info.is_killed()) { + free_job_key(job_key); + job_key = NULL; + int tmp = OB_SUCCESS; + if (OB_SUCCESS != (tmp = table_operator_.update_for_kill(job_info))) { + LOG_WARN("update for stop failed", K(tmp), K(job_info)); + } else { + LOG_WARN("update for stop job", K(job_info)); + } } else if (job_info.is_disabled()) { free_job_key(job_key); job_key = NULL; @@ -658,6 +671,10 @@ int ObDBMSSchedJobMaster::check_all_tenants() OZ (table_operator_.purge_run_detail_histroy(tenant_ids.at(i))); } */ // not open + if (OB_FAIL(table_operator_.purge_olap_async_job_run_detail(tenant_ids.at(i)))) { + LOG_WARN("purge olap async job run detail failed", K(ret), K(tenant_ids.at(i))); + ret = OB_SUCCESS; // not affect subsequent operations + } OZ (check_new_jobs(tenant_ids.at(i), tenant_schema->is_oracle_tenant())); } ret = OB_SUCCESS; // one tenant failed should not affect other @@ -670,7 +687,7 @@ int ObDBMSSchedJobMaster::check_all_tenants() int ObDBMSSchedJobMaster::check_new_jobs(uint64_t tenant_id, bool is_oracle_tenant) { int ret = OB_SUCCESS; - ObSEArray job_infos; + ObSEArray job_infos; ObArenaAllocator allocator("DBMSSchedTmp"); OZ (table_operator_.get_dbms_sched_job_infos_in_tenant(tenant_id, is_oracle_tenant, allocator, job_infos)); OZ (register_new_jobs(tenant_id, is_oracle_tenant, job_infos)); diff --git a/src/observer/dbms_scheduler/ob_dbms_sched_job_rpc_proxy.cpp b/src/observer/dbms_scheduler/ob_dbms_sched_job_rpc_proxy.cpp index 37013c634..c5bc9ac7a 100644 --- a/src/observer/dbms_scheduler/ob_dbms_sched_job_rpc_proxy.cpp +++ b/src/observer/dbms_scheduler/ob_dbms_sched_job_rpc_proxy.cpp @@ -33,13 +33,13 @@ OB_SERIALIZE_MEMBER(ObDBMSSchedJobResult, tenant_id_, job_id_, server_addr_, sta OB_SERIALIZE_MEMBER(ObDBMSSchedStopJobArg, tenant_id_, job_name_, session_id_, rpc_send_time_); int ObDBMSSchedJobRpcProxy::run_dbms_sched_job( - uint64_t tenant_id, bool is_oracle_tenant, uint64_t job_id, ObString &job_name, ObAddr server_addr, ObAddr master_addr) + uint64_t tenant_id, bool is_oracle_tenant, uint64_t job_id, ObString &job_name, ObAddr server_addr, ObAddr master_addr, int64_t group_id) { int ret = OB_SUCCESS; ObDBMSSchedJobArg arg(tenant_id, job_id, server_addr, master_addr, is_oracle_tenant, job_name); ObRpcAPDBMSSchedJobCB cb; CK (arg.is_valid()); - OZ (this->to(arg.server_addr_).by(arg.tenant_id_).run_dbms_sched_job(arg, &cb), arg); + OZ (this->to(arg.server_addr_).by(arg.tenant_id_).group_id(group_id).run_dbms_sched_job(arg, &cb), arg); return ret; } diff --git a/src/observer/dbms_scheduler/ob_dbms_sched_job_rpc_proxy.h b/src/observer/dbms_scheduler/ob_dbms_sched_job_rpc_proxy.h index bca773f3d..463610f82 100644 --- a/src/observer/dbms_scheduler/ob_dbms_sched_job_rpc_proxy.h +++ b/src/observer/dbms_scheduler/ob_dbms_sched_job_rpc_proxy.h @@ -173,7 +173,7 @@ public: public: int run_dbms_sched_job( uint64_t tenant_id, bool is_oracle_tenant, uint64_t job_id, common::ObString &job_name, - common::ObAddr server_addr, common::ObAddr master_addr); + common::ObAddr server_addr, common::ObAddr master_addr, int64_t group_id); int stop_dbms_sched_job( uint64_t tenant_id, common::ObString &job_name, common::ObAddr server_addr, uint64_t session_id); diff --git a/src/observer/dbms_scheduler/ob_dbms_sched_job_utils.cpp b/src/observer/dbms_scheduler/ob_dbms_sched_job_utils.cpp index 3e958c3ec..db76291b0 100644 --- a/src/observer/dbms_scheduler/ob_dbms_sched_job_utils.cpp +++ b/src/observer/dbms_scheduler/ob_dbms_sched_job_utils.cpp @@ -33,6 +33,9 @@ #include "sql/session/ob_sql_session_mgr.h" #include "share/schema/ob_schema_getter_guard.h" #include "share/stat/ob_dbms_stats_maintenance_window.h" +#include "observer/dbms_scheduler/ob_dbms_sched_table_operator.h" +#include "storage/ob_common_id_utils.h" +#include "observer/dbms_scheduler/ob_dbms_sched_job_rpc_proxy.h" namespace oceanbase { @@ -50,6 +53,8 @@ int ObDBMSSchedJobInfo::deep_copy(ObIAllocator &allocator, const ObDBMSSchedJobI { int ret = OB_SUCCESS; tenant_id_ = other.tenant_id_; + user_id_ = other.user_id_; + database_id_ = other.database_id_; job_ = other.job_; last_modify_ = other.last_modify_; last_date_ = other.last_date_; @@ -82,6 +87,7 @@ int ObDBMSSchedJobInfo::deep_copy(ObIAllocator &allocator, const ObDBMSSchedJobI OZ (ob_write_string(allocator, other.job_class_, job_class_)); OZ (ob_write_string(allocator, other.program_name_, program_name_)); OZ (ob_write_string(allocator, other.state_, state_)); + OZ (ob_write_string(allocator, other.job_action_, job_action_)); return ret; } @@ -99,6 +105,18 @@ int ObDBMSSchedJobClassInfo::deep_copy(common::ObIAllocator &allocator, const Ob return ret; } +int ObDBMSSchedJobUtils::generate_job_id(int64_t tenant_id, int64_t &max_job_id) +{ + int ret = OB_SUCCESS; + ObCommonID raw_id; + if (OB_FAIL(storage::ObCommonIDUtils::gen_unique_id_by_rpc(tenant_id, raw_id))) { + LOG_WARN("gen unique id failed", K(ret), K(tenant_id)); + } else { + max_job_id = raw_id.id() + ObDBMSSchedTableOperator::JOB_ID_OFFSET; + } + return ret; +} + int ObDBMSSchedJobUtils::disable_dbms_sched_job( ObISQLClient &sql_client, const uint64_t tenant_id, @@ -131,6 +149,94 @@ int ObDBMSSchedJobUtils::disable_dbms_sched_job( return ret; } +int ObDBMSSchedJobUtils::stop_dbms_sched_job( + common::ObISQLClient &sql_client, + const ObDBMSSchedJobInfo &job_info, + const bool is_delete_after_stop) +{ + int ret = OB_SUCCESS; + obrpc::ObDBMSSchedJobRpcProxy *rpc_proxy = GCTX.dbms_sched_job_rpc_proxy_; + uint64_t tenant_id = job_info.tenant_id_; + ObSqlString sql; + CK (OB_NOT_NULL(rpc_proxy)); + uint64_t data_version = 0; + if (OB_SUCC(ret)) { + if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, data_version))) { + LOG_WARN("fail to get tenant data version", KR(ret), K(data_version)); + } else if (data_version < MOCK_DATA_VERSION_4_2_1_5 + || (data_version >= DATA_VERSION_4_2_2_0 && data_version < MOCK_DATA_VERSION_4_2_4_0) + || (data_version >= DATA_VERSION_4_3_0_0 && data_version < DATA_VERSION_4_3_2_0)) { + ret = OB_NOT_SUPPORTED; + } + } + if (OB_SUCC(ret)) { + dbms_scheduler::ObDBMSSchedJobInfo update_job_info; + update_job_info.tenant_id_ = job_info.tenant_id_; + update_job_info.is_oracle_tenant_ = job_info.is_oracle_tenant_; + update_job_info.job_name_ = job_info.job_name_; + if (is_delete_after_stop) { + update_job_info.state_ = "KILLED"; + } + if(OB_FAIL(update_dbms_sched_job_info(sql_client, update_job_info))) { + LOG_WARN("update job info failed", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(sql.append_fmt("select svr_ip, svr_port, session_id from %s where tenant_id = %lu and job_name = \'%.*s\'", + OB_ALL_VIRTUAL_TENANT_SCHEDULER_RUNNING_JOB_TNAME, tenant_id, job_info.job_name_.length(),job_info.job_name_.ptr()))) { + LOG_WARN("append sql failed", KR(ret)); + } else { + SMART_VAR(ObMySQLProxy::MySQLResult, result) { + if (OB_FAIL(sql_client.read(result, sql.ptr()))) { + LOG_WARN("execute query failed", K(ret), K(sql), K(tenant_id), K(job_info.job_name_)); + } else if (OB_ISNULL(result.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result is null", K(ret), K(sql), K(tenant_id), K(job_info.job_name_)); + } else { + bool result_empty = true; + do { + if (OB_FAIL(result.get_result()->next())) { + if (ret == OB_ITER_END) { + //do nothing + } else { + LOG_WARN("fail to get result", K(ret)); + } + } else { + result_empty = false; + uint64_t session_id = OB_INVALID_ID; + ObAddr svr; + ObString svr_ip; + int64_t svr_port = OB_INVALID_INDEX; + EXTRACT_VARCHAR_FIELD_MYSQL(*(result.get_result()), "svr_ip", svr_ip); + EXTRACT_UINT_FIELD_MYSQL(*(result.get_result()), "session_id", session_id, uint64_t); + EXTRACT_INT_FIELD_MYSQL(*(result.get_result()), "svr_port", svr_port, int64_t); + if (OB_SUCC(ret)) { + if (!svr.set_ip_addr(svr_ip, svr_port)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("set addr failed", K(svr_ip), K(svr_port)); + } else { + LOG_INFO("send rpc", K(tenant_id), K(job_info.job_name_), K(svr), K(session_id)); + ObString stop_job_name = ObString(job_info.job_name_); + OZ (rpc_proxy->stop_dbms_sched_job(tenant_id, + stop_job_name, + svr, + session_id)); + } + } + } + } while (OB_SUCC(ret)); + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + } + } + } + } + } + LOG_INFO("stop job", K(ret), K(job_info.job_name_), K(tenant_id)); + return ret; +} + int ObDBMSSchedJobUtils::remove_dbms_sched_job( ObISQLClient &sql_client, const uint64_t tenant_id, @@ -153,6 +259,9 @@ int ObDBMSSchedJobUtils::remove_dbms_sched_job( int64_t affected_rows = 0; if (OB_FAIL(exec.exec_delete(OB_ALL_TENANT_SCHEDULER_JOB_TNAME, dml, affected_rows))) { LOG_WARN("execute delete failed", KR(ret)); + } else if (is_zero_row(affected_rows) && !if_exists) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("execute delete failed", KR(ret), K(if_exists)); } else if (!if_exists && !is_double_row(affected_rows)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("affected_rows unexpected to be two", KR(ret), K(affected_rows)); @@ -198,35 +307,45 @@ int ObDBMSSchedJobUtils::add_dbms_sched_job( ObDMLExecHelper exec(sql_client, exec_tenant_id); int64_t affected_rows = 0; const int64_t now = ObTimeUtility::current_time(); - - OZ (dml.add_gmt_create(now)); - OZ (dml.add_gmt_modified(now)); - OZ (dml.add_pk_column("tenant_id", - ObSchemaUtils::get_extract_tenant_id(job_info.tenant_id_, job_info.tenant_id_))); - OZ (dml.add_pk_column("job", job_id)); - OZ (dml.add_column("lowner", ObHexEscapeSqlStr(job_info.lowner_))); - OZ (dml.add_column("powner", ObHexEscapeSqlStr(job_info.powner_))); - OZ (dml.add_column("cowner", ObHexEscapeSqlStr(job_info.cowner_))); - OZ (dml.add_raw_time_column("next_date", job_info.start_date_)); - OZ (dml.add_column("total", 0)); - OZ (dml.add_column("`interval#`", ObHexEscapeSqlStr( - job_info.repeat_interval_.empty() ? ObString("null") : job_info.repeat_interval_))); - OZ (dml.add_column("flag", job_info.flag_)); - OZ (dml.add_column("job_name", ObHexEscapeSqlStr(job_info.job_name_))); - OZ (dml.add_column("job_style", ObHexEscapeSqlStr(job_info.job_style_))); - OZ (dml.add_column("job_type", ObHexEscapeSqlStr(job_info.job_type_))); - OZ (dml.add_column("job_class", ObHexEscapeSqlStr(job_info.job_class_))); - OZ (dml.add_column("job_action", ObHexEscapeSqlStr(job_info.job_action_))); - OZ (dml.add_column("what", ObHexEscapeSqlStr(job_info.job_action_))); - OZ (dml.add_raw_time_column("start_date", job_info.start_date_)); - OZ (dml.add_raw_time_column("end_date", job_info.end_date_)); - OZ (dml.add_column("repeat_interval", ObHexEscapeSqlStr(job_info.repeat_interval_))); - OZ (dml.add_column("enabled", job_info.enabled_)); - OZ (dml.add_column("auto_drop", job_info.auto_drop_)); - OZ (dml.add_column("max_run_duration", job_info.max_run_duration_)); - OZ (dml.add_column("interval_ts", job_info.interval_ts_)); - OZ (dml.add_column("scheduler_flags", job_info.scheduler_flags_)); - OZ (dml.add_column("exec_env", job_info.exec_env_)); + uint64_t data_version = 0; + if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, data_version))) { + LOG_WARN("fail to get tenant data version", KR(ret), K(data_version)); + } else { + OZ (dml.add_gmt_create(now)); + OZ (dml.add_gmt_modified(now)); + OZ (dml.add_pk_column("tenant_id", + ObSchemaUtils::get_extract_tenant_id(job_info.tenant_id_, job_info.tenant_id_))); + if ((MOCK_DATA_VERSION_4_2_4_0 <= data_version && DATA_VERSION_4_3_0_0 > data_version) || + data_version > DATA_VERSION_4_3_2_0) { + OZ (dml.add_column("user_id", job_info.user_id_)); + OZ (dml.add_column("database_id", job_info.database_id_)); + } + OZ (dml.add_pk_column("job", job_id)); + OZ (dml.add_column("lowner", ObHexEscapeSqlStr(job_info.lowner_))); + OZ (dml.add_column("powner", ObHexEscapeSqlStr(job_info.powner_))); + OZ (dml.add_column("cowner", ObHexEscapeSqlStr(job_info.cowner_))); + OZ (dml.add_raw_time_column("next_date", job_info.start_date_)); + OZ (dml.add_column("total", 0)); + OZ (dml.add_column("`interval#`", ObHexEscapeSqlStr( + job_info.repeat_interval_.empty() ? ObString("null") : job_info.repeat_interval_))); + OZ (dml.add_column("flag", job_info.flag_)); + OZ (dml.add_column("job_name", ObHexEscapeSqlStr(job_info.job_name_))); + OZ (dml.add_column("job_style", ObHexEscapeSqlStr(job_info.job_style_))); + OZ (dml.add_column("job_type", ObHexEscapeSqlStr(job_info.job_type_))); + OZ (dml.add_column("job_class", ObHexEscapeSqlStr(job_info.job_class_))); + OZ (dml.add_column("job_action", ObHexEscapeSqlStr(job_info.job_action_))); + OZ (dml.add_column("what", ObHexEscapeSqlStr(job_info.job_action_))); + OZ (dml.add_raw_time_column("start_date", job_info.start_date_)); + OZ (dml.add_raw_time_column("end_date", job_info.end_date_)); + OZ (dml.add_column("repeat_interval", ObHexEscapeSqlStr(job_info.repeat_interval_))); + OZ (dml.add_column("enabled", job_info.enabled_)); + OZ (dml.add_column("auto_drop", job_info.auto_drop_)); + OZ (dml.add_column("max_run_duration", job_info.max_run_duration_)); + OZ (dml.add_column("interval_ts", job_info.interval_ts_)); + OZ (dml.add_column("scheduler_flags", job_info.scheduler_flags_)); + OZ (dml.add_column("exec_env", job_info.exec_env_)); + OZ (dml.add_column("comments", ObHexEscapeSqlStr(job_info.comments_))); + } if (OB_SUCC(ret) && OB_FAIL(exec.exec_insert( OB_ALL_TENANT_SCHEDULER_JOB_TNAME, dml, affected_rows))) { @@ -239,6 +358,41 @@ int ObDBMSSchedJobUtils::add_dbms_sched_job( return ret; } +int ObDBMSSchedJobUtils::update_dbms_sched_job_info( + common::ObISQLClient &sql_client, + const ObDBMSSchedJobInfo &update_job_info) +{ + int ret = OB_SUCCESS; + uint64_t tenant_id = update_job_info.tenant_id_; + if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || update_job_info.job_name_.empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(tenant_id), K(update_job_info)); + } else { + const uint64_t exec_tenant_id = ObSchemaUtils::get_exec_tenant_id(tenant_id); + ObDMLSqlSplicer dml; + if (OB_FAIL(dml.add_pk_column( + "tenant_id", ObSchemaUtils::get_extract_tenant_id(exec_tenant_id, tenant_id))) + || OB_FAIL(dml.add_pk_column("job_name", update_job_info.job_name_))) { + LOG_WARN("add column failed", KR(ret)); + } else if (!update_job_info.state_.empty() && OB_FAIL(dml.add_column("state", update_job_info.state_))) { + LOG_WARN("add column failed", KR(ret), K(update_job_info.state_)); + } else { + ObDMLExecHelper exec(sql_client, exec_tenant_id); + int64_t affected_rows = 0; + if (OB_FAIL(exec.exec_update(OB_ALL_TENANT_SCHEDULER_JOB_TNAME, dml, affected_rows))) { + LOG_WARN("execute update failed", KR(ret)); + } else if (is_zero_row(affected_rows)) { + ret = OB_ENTRY_NOT_EXIST; + LOG_WARN("not change", KR(ret), K(affected_rows)); + } else if (!is_double_row(affected_rows)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("affected_rows unexpected to be two", KR(ret), K(affected_rows)); + } + } + } + return ret; +} + int ObDBMSSchedJobUtils::reserve_user_with_minimun_id(ObIArray &user_infos) { int ret = OB_SUCCESS; @@ -264,5 +418,86 @@ int ObDBMSSchedJobUtils::reserve_user_with_minimun_id(ObIArray 0", + OB_ALL_TENANT_SCHEDULER_JOB_TNAME, + ObSchemaUtils::get_extract_tenant_id(exec_tenant_id, tenant_id), + job_name.length(), job_name.ptr()))) { + LOG_WARN("failed to assign sql", K(ret)); + } else { + SMART_VAR(ObMySQLProxy::MySQLResult, res) { + if (OB_FAIL(sql_client.read(res, tenant_id, sql.ptr()))) { + LOG_WARN("execute query failed", K(ret), K(sql)); + } else { + if (res.get_result() != NULL && OB_SUCCESS == (ret = res.get_result()->next())) { + ObDBMSSchedTableOperator table_operator; + OZ (table_operator.extract_info(*(res.get_result()), tenant_id, is_oracle_tenant, allocator, job_info)); + } + if (OB_FAIL(ret)) { + if (OB_ITER_END == ret) { + ret = OB_ENTRY_NOT_EXIST; + } else { + LOG_WARN("next failed", K(ret)); + } + } + } + } + } + } + return ret; +} + +int ObDBMSSchedJobUtils::check_dbms_sched_job_priv(const ObUserInfo *user_info, + const ObDBMSSchedJobInfo &job_info) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(user_info)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("user info is NULL", KR(ret)); + } else if (is_root_user(user_info->get_user_id())) { + // do nothing + } else if (job_info.is_oracle_tenant_) { + //TODO 连雨 + } else { + if (job_info.user_id_ != OB_INVALID_ID) { //如果 job 有 user_id 优先使用 + if (job_info.user_id_ != user_info->get_user_id()) { + ret = OB_ERR_NO_PRIVILEGE; + LOG_WARN("job user id check failed", KR(ret), K(user_info), K(job_info.user_id_)); + } + } else if (0 != job_info.powner_.case_compare(user_info->get_user_name())) { // job 保存的 owner 可能是 root@% or root (旧) + const char *c = job_info.powner_.reverse_find('@'); + if (OB_ISNULL(c)) { + ret = OB_ERR_NO_PRIVILEGE; + LOG_WARN("job user id check failed", KR(ret), K(user_info), K(job_info.user_id_)); + } else { + ObString user = job_info.powner_; + ObString user_name; + ObString host_name; + user_name = user.split_on(c); + host_name = user; + if (0 != user_name.case_compare(user_info->get_user_name()) || 0 != host_name.case_compare(user_info->get_host_name())) { + ret = OB_ERR_NO_PRIVILEGE; + LOG_WARN("job user id check failed", KR(ret), K(user_info), K(job_info.user_id_)); + } + } + } + } + + return ret; +} + } // end for namespace dbms_scheduler } // end for namespace oceanbase diff --git a/src/observer/dbms_scheduler/ob_dbms_sched_job_utils.h b/src/observer/dbms_scheduler/ob_dbms_sched_job_utils.h index 83f055ed0..056789a90 100644 --- a/src/observer/dbms_scheduler/ob_dbms_sched_job_utils.h +++ b/src/observer/dbms_scheduler/ob_dbms_sched_job_utils.h @@ -22,6 +22,7 @@ #define DATA_VERSION_SUPPORT_JOB_CLASS(data_version) (data_version >= DATA_VERSION_4_3_2_0) #define DATA_VERSION_SUPPORT_RUN_DETAIL_V2(data_version) ((MOCK_DATA_VERSION_4_2_4_0 <= data_version && DATA_VERSION_4_3_0_0 > data_version) || DATA_VERSION_4_3_2_0 <= data_version) +#define DATA_VERSION_SUPPORT_RUN_DETAIL_V2_DATABASE_NAME_AND_RUNNING_JOB_JOB_CLASS(data_version) (DATA_VERSION_4_3_2_1 <= data_version) namespace oceanbase { @@ -62,6 +63,8 @@ class ObDBMSSchedJobInfo public: ObDBMSSchedJobInfo() : tenant_id_(common::OB_INVALID_ID), + user_id_(common::OB_INVALID_ID), + database_id_(common::OB_INVALID_ID), job_(common::OB_INVALID_ID), lowner_(), powner_(), @@ -104,6 +107,8 @@ public: is_oracle_tenant_(true) {} TO_STRING_KV(K(tenant_id_), + K(user_id_), + K(database_id_), K(job_), K(job_name_), K(lowner_), @@ -138,6 +143,8 @@ public: } uint64_t get_tenant_id() { return tenant_id_; } + uint64_t get_user_id() { return user_id_; } + uint64_t get_database_id() { return database_id_; } uint64_t get_job_id() { return job_; } uint64_t get_job_id_with_tenant() { return common::combine_two_ids(tenant_id_, job_); } int64_t get_this_date() { return this_date_; } @@ -153,6 +160,7 @@ public: bool is_broken() { return 0x1 == (flag_ & 0x1); } bool is_running(){ return this_date_ != 0; } bool is_disabled() { return 0x0 == (enabled_ & 0x1); } + bool is_killed() { return 0 == state_.case_compare("KILLED"); } common::ObString &get_what() { return what_; } common::ObString &get_exec_env() { return exec_env_; } @@ -164,14 +172,19 @@ public: common::ObString &get_program_name() { return program_name_; } common::ObString &get_job_name() { return job_name_; } common::ObString &get_job_class() { return job_class_; } + common::ObString &get_job_action() { return job_action_; } bool is_oracle_tenant() { return is_oracle_tenant_; } bool is_date_expression_job_class() const { return !!(scheduler_flags_ & JOB_SCHEDULER_FLAG_DATE_EXPRESSION_JOB_CLASS); } + bool is_mysql_event_job_class() const { return (0 == job_class_.case_compare("MYSQL_EVENT_JOB_CLASS")); } + bool is_olap_async_job_class() const { return (0 == job_class_.case_compare("OLAP_ASYNC_JOB_CLASS")); } int deep_copy(common::ObIAllocator &allocator, const ObDBMSSchedJobInfo &other); public: uint64_t tenant_id_; + uint64_t user_id_; + uint64_t database_id_; uint64_t job_; common::ObString lowner_; common::ObString powner_; @@ -264,18 +277,94 @@ public: class ObDBMSSchedJobUtils { public: + //TO DO DELETE 连雨 + static int generate_job_id(int64_t tenant_id, int64_t &max_job_id); static int disable_dbms_sched_job(common::ObISQLClient &sql_client, const uint64_t tenant_id, const common::ObString &job_name, const bool if_exists = false); - static int remove_dbms_sched_job(common::ObISQLClient &sql_client, - const uint64_t tenant_id, - const common::ObString &job_name, - const bool if_exists = false); + /** + * @brief 创建一个 job + * @param [in] sql_client - 创建 JOB 会执行两次 insert, 应该传入一个 trans + * @param [in] tenant_id - 租户id + * @param [in] job_id - 唯一id + * @retval OB_SUCCESS execute success + * @retval OB_ERR_UNEXPECTED 未知错误 + * @retval OB_INVALID_ARGUMENT 当前 JOB 不存在 + * @retval OB_ERR_NO_PRIVILEGE 当前 JOB 传入的用户没有权限修改 + */ static int create_dbms_sched_job(common::ObISQLClient &sql_client, const uint64_t tenant_id, const int64_t job_id, const ObDBMSSchedJobInfo &job_info); + /** + * @brief 直接删除一个 job + * @param [in] sql_client + * @param [in] tenant_id - 租户id + * @param [in] job_name - job名 + * @param [in] if_exists - 为 true (删除一个不存在 JOB 会报错) + * @retval OB_SUCCESS execute success + * @retval OB_ERR_UNEXPECTED 未知错误 + * @retval OB_INVALID_ARGUMENT 当前 JOB 不存在 + */ + static int remove_dbms_sched_job(common::ObISQLClient &sql_client, + const uint64_t tenant_id, + const common::ObString &job_name, + const bool if_exists = false); + /** + * @brief 停止一个 未开始/正在运行的 job + * @param [in] sql_client + * @param [in] is_delete_after_stop - 停止后是否删除 + * @param [in] job_info - 要停止的 job 信息 + * @retval OB_SUCCESS execute success + * @retval OB_NOT_SUPPORTED 当前版本不支持 + * @retval OB_ERR_UNEXPECTED 未知错误 + * @retval OB_ENTRY_NOT_EXIST 当前 JOB 不在运行 + * @retval OB_ERR_NO_PRIVILEGE 当前 JOB 传入的用户没有权限修改 + */ + static int stop_dbms_sched_job(common::ObISQLClient &sql_client, + const ObDBMSSchedJobInfo &job_info, + const bool is_delete_after_stop); + /** + * @brief 更新 JOB 信息 + * @param [in] update_job_info - 要更新的 job 信息 (必须要填 tenant_id, job_name) + * @retval OB_SUCCESS execute success + * @retval OB_ERR_UNEXPECTED 未知错误 + * @retval OB_INVALID_ARGUMENT 无效参数 + * @retval OB_ENTRY_NOT_EXIST JOB 不存在/没做更改 + */ + static int update_dbms_sched_job_info(common::ObISQLClient &sql_client, + const ObDBMSSchedJobInfo &update_job_info); + /** + * @brief 获取 JOB 信息 + * @param [in] tenant_id - 租户id + * @param [in] is_oracle_tenant - 是否 oracle 租户, 内部表没有的值, 由调用者确定 + * @param [in] job_name - job名 + * @param [in] allocator - 合理的 allocator, 防止 job_info 失效 + * @param [in] job_info - job 信息 + * @retval OB_SUCCESS execute success + * @retval OB_ERR_UNEXPECTED 未知错误 + * @retval OB_ENTRY_NOT_EXIST JOB 不存在 + */ + static int get_dbms_sched_job_info(common::ObISQLClient &sql_client, + const uint64_t tenant_id, + const bool is_oracle_tenant, + const ObString &job_name, + common::ObIAllocator &allocator, + ObDBMSSchedJobInfo &job_info); + /** + * @brief 检查用户是否有修改 JOB 的权限（策略, root 用户能修改所有 job, 普通用户只能修改自己的 job) + * @param [in] user_info - 用户信息 + * @param [in] is_oracle_tenant - 是否 oracle 租户 + * @param [in] job_info - job 信息 + * @retval OB_SUCCESS execute success + * @retval OB_ERR_UNEXPECTED 未知错误 + * @retval OB_INVALID_ARGUMENT 无效参数/ user_info 为 NULL + * @retval OB_ERR_NO_PRIVILEGE 传入用户没有权限 + */ + static int check_dbms_sched_job_priv(const ObUserInfo *user_info, + const ObDBMSSchedJobInfo &job_info); + //TO DO DELETE 连雨 static int add_dbms_sched_job(common::ObISQLClient &sql_client, const uint64_t tenant_id, const int64_t job_id, diff --git a/src/observer/dbms_scheduler/ob_dbms_sched_table_operator.cpp b/src/observer/dbms_scheduler/ob_dbms_sched_table_operator.cpp index d540e0ddd..07553df73 100644 --- a/src/observer/dbms_scheduler/ob_dbms_sched_table_operator.cpp +++ b/src/observer/dbms_scheduler/ob_dbms_sched_table_operator.cpp @@ -174,6 +174,13 @@ int ObDBMSSchedTableOperator::_build_job_log_dml( } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, data_version))) { LOG_WARN("fail to get tenant data version", KR(ret), K(data_version)); } else if (DATA_VERSION_SUPPORT_RUN_DETAIL_V2(data_version)) { + if (DATA_VERSION_SUPPORT_RUN_DETAIL_V2_DATABASE_NAME_AND_RUNNING_JOB_JOB_CLASS(data_version)) { + OZ (dml.add_column("database_name", ObHexEscapeSqlStr(job_info.cowner_))); + OZ (dml.add_column("owner", job_info.powner_)); + OZ (dml.add_column("operation", ObHexEscapeSqlStr(job_info.job_action_))); + OZ (dml.add_column("status", job_info.state_)); + OZ (dml.add_time_column("req_start_date", job_info.start_date_)); + } OZ (dml.add_pk_column("job_name", job_info.job_name_)); OZ (dml.splice_insert_sql(OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_TNAME, sql)); } else { @@ -317,6 +324,7 @@ int ObDBMSSchedTableOperator::update_for_end(ObDBMSSchedJobInfo &job_info, int e } else if (job_info.is_date_expression_job_class() && now >= job_info.end_date_ && true == job_info.auto_drop_) { OZ (_build_job_drop_dml(now, job_info, sql1)); } else if ((now >= job_info.end_date_ || job_info.get_interval_ts() == 0) && (true == job_info.auto_drop_)) { + job_info.state_ = ObString("COMPLETED"); OZ (_build_job_drop_dml(now, job_info, sql1)); } else { OX (job_info.failures_ = (err == 0) ? 0 : (job_info.failures_ + 1)); @@ -357,6 +365,40 @@ int ObDBMSSchedTableOperator::update_for_end(ObDBMSSchedJobInfo &job_info, int e return ret; } +int ObDBMSSchedTableOperator::update_for_kill(ObDBMSSchedJobInfo &job_info) +{ + int ret = OB_SUCCESS; + ObMySQLTransaction trans; + ObSqlString sql1; + ObSqlString sql2; + int64_t affected_rows = 0; + const int64_t now = ObTimeUtility::current_time(); + bool need_record = true; + int64_t tenant_id = job_info.tenant_id_; + CK (OB_NOT_NULL(sql_proxy_)); + CK (OB_LIKELY(tenant_id != OB_INVALID_ID)); + CK (OB_LIKELY(job_info.job_ != OB_INVALID_ID)); + OZ (_check_need_record(job_info, need_record)); + OZ (_build_job_drop_dml(now, job_info, sql1)); + if (OB_SUCC(ret) && need_record) { + job_info.state_ = ObString("KILLED"); + OZ (_build_job_log_dml(now, job_info, OB_ERR_SESSION_INTERRUPTED, "user stop job", sql2)); + } + OZ (trans.start(sql_proxy_, tenant_id, true)); + OZ (trans.write(tenant_id, sql1.ptr(), affected_rows)); + if (OB_SUCC(ret) && need_record) { + OZ (trans.write(tenant_id, sql2.ptr(), affected_rows)); + } + if (trans.is_started()) { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { + LOG_WARN("failed to commit trans", KR(ret), KR(tmp_ret)); + ret = OB_SUCC(ret) ? tmp_ret : ret; + } + } + return ret; +} + int ObDBMSSchedTableOperator::check_job_can_running(int64_t tenant_id, int64_t alive_job_count, bool &can_running) { int ret = OB_SUCCESS; @@ -418,6 +460,16 @@ int ObDBMSSchedTableOperator::extract_info( job_info_local.tenant_id_ = tenant_id; job_info_local.is_oracle_tenant_ = is_oracle_tenant; EXTRACT_INT_FIELD_MYSQL(result, "job", job_info_local.job_, uint64_t); + EXTRACT_INT_FIELD_MYSQL(result, "user_id", job_info_local.user_id_, uint64_t); + if (OB_ERR_NULL_VALUE == ret || OB_ERR_COLUMN_NOT_FOUND == ret) { + ret = OB_SUCCESS; + job_info_local.user_id_ = OB_INVALID_ID; + } + EXTRACT_INT_FIELD_MYSQL(result, "database_id", job_info_local.database_id_, uint64_t); + if (OB_ERR_NULL_VALUE == ret || OB_ERR_COLUMN_NOT_FOUND == ret) { + ret = OB_SUCCESS; + job_info_local.database_id_ = OB_INVALID_ID; + } EXTRACT_VARCHAR_FIELD_MYSQL_SKIP_RET(result, "lowner", job_info_local.lowner_); EXTRACT_VARCHAR_FIELD_MYSQL_SKIP_RET(result, "powner", job_info_local.powner_); EXTRACT_VARCHAR_FIELD_MYSQL_SKIP_RET(result, "cowner", job_info_local.cowner_); @@ -479,7 +531,7 @@ do { \ EXTRACT_VARCHAR_FIELD_MYSQL_SKIP_RET(result, "job_class", job_info_local.job_class_); EXTRACT_VARCHAR_FIELD_MYSQL_SKIP_RET(result, "program_name", job_info_local.program_name_); //job_type not used - //job_action not used + EXTRACT_VARCHAR_FIELD_MYSQL_SKIP_RET(result, "job_action", job_info_local.job_action_); //number_of_argument not used //repeat_interval not used EXTRACT_BOOL_FIELD_MYSQL_SKIP_RET(result, "enabled", job_info_local.enabled_); @@ -499,6 +551,19 @@ do { \ return ret; } +int ObDBMSSchedTableOperator::get_dbms_sched_job_is_killed(const ObDBMSSchedJobInfo &job_info, bool &is_killed) +{ + int ret = OB_SUCCESS; + is_killed = false; + ObArenaAllocator allocator("SchedStateTmp"); + ObDBMSSchedJobInfo update_job_info; + OZ(get_dbms_sched_job_info(job_info.tenant_id_, job_info.is_oracle_tenant_, job_info.job_, job_info.job_name_, allocator, update_job_info)); + if (OB_SUCC(ret) && update_job_info.is_killed()) { + is_killed = true; + } + return ret; +} + int ObDBMSSchedTableOperator::get_dbms_sched_job_info( uint64_t tenant_id, bool is_oracle_tenant, uint64_t job_id, const common::ObString &job_name, ObIAllocator &allocator, ObDBMSSchedJobInfo &job_info) @@ -724,5 +789,31 @@ int ObDBMSSchedTableOperator::purge_run_detail_histroy(uint64_t tenant_id) return ret; } +int ObDBMSSchedTableOperator::purge_olap_async_job_run_detail(uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + ObSqlString sql; + int64_t affected_rows = 0; + uint64_t data_version = 0; + int64_t log_history = 30; + + CK (OB_NOT_NULL(sql_proxy_)); + CK (OB_LIKELY(tenant_id != OB_INVALID_ID)); + + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, data_version))) { + LOG_WARN("fail to get tenant data version", KR(ret), K(data_version)); + } else if (DATA_VERSION_SUPPORT_RUN_DETAIL_V2(data_version)) { + OZ (sql.assign_fmt("delete from %s where job_class=\'OLAP_ASYNC_JOB_CLASS\' and timewrite(tenant_id, sql.ptr(), affected_rows)); + if (affected_rows > 0) { + LOG_INFO("purge olap async job run detail finish", K(ret), K(tenant_id), K(sql), K(affected_rows)); + } + } + return ret; +} + } // end for namespace dbms_scheduler } // end for namespace oceanbase diff --git a/src/observer/dbms_scheduler/ob_dbms_sched_table_operator.h b/src/observer/dbms_scheduler/ob_dbms_sched_table_operator.h index f33b10bf2..bef0e234f 100644 --- a/src/observer/dbms_scheduler/ob_dbms_sched_table_operator.h +++ b/src/observer/dbms_scheduler/ob_dbms_sched_table_operator.h @@ -67,7 +67,8 @@ public: int update_for_rollback(ObDBMSSchedJobInfo &job_info); int update_for_timeout(ObDBMSSchedJobInfo &job_info); int update_for_end(ObDBMSSchedJobInfo &job_info, int err, const common::ObString &errmsg); - + int update_for_kill(ObDBMSSchedJobInfo &job_info); + int get_dbms_sched_job_is_killed(const ObDBMSSchedJobInfo &job_info, bool &is_killed); int get_dbms_sched_job_info( uint64_t tenant_id, bool is_oracle_tenant, uint64_t job_id, const common::ObString &job_name, common::ObIAllocator &allocator, ObDBMSSchedJobInfo &job_info); @@ -90,6 +91,7 @@ public: int purge_run_detail_histroy(uint64_t tenant_id); + int purge_olap_async_job_run_detail(uint64_t tenant_id); private: DISALLOW_COPY_AND_ASSIGN(ObDBMSSchedTableOperator); diff --git a/src/observer/omt/ob_tenant.cpp b/src/observer/omt/ob_tenant.cpp index 97804e31c..c18a3d8e6 100644 --- a/src/observer/omt/ob_tenant.cpp +++ b/src/observer/omt/ob_tenant.cpp @@ -445,7 +445,7 @@ void ObResourceGroup::check_worker_count() { int ret = OB_SUCCESS; if (OB_SUCC(workers_lock_.trylock())) { - if (is_user_group(group_id_) + if ((is_user_group(group_id_) || is_job_group(group_id_)) && nesting_worker_cnt_ < (MAX_REQUEST_LEVEL - GROUP_MULTI_LEVEL_THRESHOLD)) { for (int level = GROUP_MULTI_LEVEL_THRESHOLD + nesting_worker_cnt_; OB_SUCC(ret) && level < MAX_REQUEST_LEVEL; level++) { if (OB_SUCC(acquire_level_worker(level))) { @@ -1390,7 +1390,7 @@ int ObTenant::recv_group_request(ObRequest &req, int64_t group_id) if (req_level < 0) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("unexpected level", K(req_level), K(id_), K(group_id)); - } else if (is_user_group(group_id) && req_level >= GROUP_MULTI_LEVEL_THRESHOLD) { + } else if ((is_user_group(group_id) || is_job_group(group_id)) && req_level >= GROUP_MULTI_LEVEL_THRESHOLD) { group->recv_level_rpc_cnt_.atomic_inc(req_level); if (OB_FAIL(group->multi_level_queue_.push(req, req_level, 0))) { LOG_WARN("push request to queue fail", K(req_level), K(id_), K(group_id)); diff --git a/src/observer/omt/ob_tenant.h b/src/observer/omt/ob_tenant.h index bee1bee4f..ad1ebdb77 100644 --- a/src/observer/omt/ob_tenant.h +++ b/src/observer/omt/ob_tenant.h @@ -282,6 +282,7 @@ public: WList &get_workers() { return workers_; } lib::ObMutex &get_workers_lock() { return workers_lock_; } share::ObCgroupCtrl *get_cgroup_ctrl() { return cgroup_ctrl_; } + bool is_job_group(int64_t group_id) { return share::OBCG_OLAP_ASYNC_JOB == group_id; } int init(); void update_queue_size(); @@ -542,6 +543,7 @@ private: int construct_mtl_init_ctx(const ObTenantMeta &meta, share::ObTenantModuleInitCtx *&ctx); int recv_group_request(rpc::ObRequest &req, int64_t group_id); + bool is_job_group(int64_t group_id) { return share::OBCG_OLAP_ASYNC_JOB == group_id; } protected: mutable common::TCRWLock meta_lock_; diff --git a/src/observer/virtual_table/ob_all_virtual_tenant_scheduler_running_job.cpp b/src/observer/virtual_table/ob_all_virtual_tenant_scheduler_running_job.cpp index d1abc108d..ef1172b42 100644 --- a/src/observer/virtual_table/ob_all_virtual_tenant_scheduler_running_job.cpp +++ b/src/observer/virtual_table/ob_all_virtual_tenant_scheduler_running_job.cpp @@ -151,6 +151,15 @@ int ObAllVirtualTenantSchedulerRunningJob::FillScanner::operator()( cur_row_->cells_[cell_idx].set_null(); break; } + case JOB_CLASS: { + if (OB_NOT_NULL(sess_info->get_job_info())) { + cur_row_->cells_[cell_idx].set_varchar(sess_info->get_job_info()->get_job_class()); + cur_row_->cells_[cell_idx].set_collation_type(default_collation); + } else { + cur_row_->cells_[cell_idx].set_null(); + } + break; + } case JOB_STYLE: { cur_row_->cells_[cell_idx].set_null(); break; diff --git a/src/observer/virtual_table/ob_all_virtual_tenant_scheduler_running_job.h b/src/observer/virtual_table/ob_all_virtual_tenant_scheduler_running_job.h index ba6b71c9f..bcc35991a 100644 --- a/src/observer/virtual_table/ob_all_virtual_tenant_scheduler_running_job.h +++ b/src/observer/virtual_table/ob_all_virtual_tenant_scheduler_running_job.h @@ -56,7 +56,8 @@ private: DESTINATION_OWNER, DESTINATION, CREDENTIAL_OWNER, - CREDENTIAL_NAME + CREDENTIAL_NAME, + JOB_CLASS }; class FillScanner { diff --git a/src/pl/parser/pl_non_reserved_keywords_mysql_mode.c b/src/pl/parser/pl_non_reserved_keywords_mysql_mode.c index 2cf722171..f2ed1f239 100644 --- a/src/pl/parser/pl_non_reserved_keywords_mysql_mode.c +++ b/src/pl/parser/pl_non_reserved_keywords_mysql_mode.c @@ -177,6 +177,9 @@ static const NonReservedKeyword Mysql_pl_none_reserved_keywords[] = {"pragma", PRAGMA}, {"interface", INTERFACE}, {"c", C}, + {"submit", SUBMIT}, + {"job", JOB}, + {"cancel", CANCEL}, }; const NonReservedKeyword *mysql_pl_non_reserved_keyword_lookup(const char *word) diff --git a/src/pl/parser/pl_parser_mysql_mode.l b/src/pl/parser/pl_parser_mysql_mode.l index 943b64ff4..be5de55fa 100644 --- a/src/pl/parser/pl_parser_mysql_mode.l +++ b/src/pl/parser/pl_parser_mysql_mode.l @@ -97,6 +97,7 @@ TABLE { return TABLE; } UPDATE { return UPDATE; } JSON { return JSON; } REPLACE { return REPLACE; } +LOAD { return LOAD; } TRIGGER { ObParseCtx *parse_ctx = (ObParseCtx *)yyextra; parse_ctx->is_for_trigger_ = 1; diff --git a/src/pl/parser/pl_parser_mysql_mode.y b/src/pl/parser/pl_parser_mysql_mode.y index 817384d0e..ae5385b04 100644 --- a/src/pl/parser/pl_parser_mysql_mode.y +++ b/src/pl/parser/pl_parser_mysql_mode.y @@ -210,7 +210,7 @@ void obpl_mysql_wrap_get_user_var_into_subquery(ObParseCtx *parse_ctx, ParseNode TINYINT SMALLINT MEDIUMINT INTEGER BIGINT FLOAT DOUBLE PRECISION DEC DECIMAL NUMERIC CHARACTER VARCHAR BINARY VARBINARY UNSIGNED ZEROFILL COLLATE SET BLOB TINYTEXT MEDIUMTEXT LONGTEXT TINYBLOB - MEDIUMBLOB LONGBLOB VARYING + MEDIUMBLOB LONGBLOB VARYING LOAD /* reserved key words only used in ob, in mysql these keywords are non reserved*/ CHARSET COMMIT ROLLBACK DO UNTIL //-----------------------------reserved keyword end------------------------------------------------- @@ -223,7 +223,7 @@ void obpl_mysql_wrap_get_user_var_into_subquery(ObParseCtx *parse_ctx, ParseNode DATA DEFINER END_KEY EXTEND FOLLOWS FOUND FUNCTION HANDLER INTERFACE INVOKER JSON LANGUAGE MESSAGE_TEXT MYSQL_ERRNO NATIONAL NEXT NO OF OPEN PACKAGE PRAGMA PRECEDES RECORD RETURNS ROW ROWTYPE SCHEMA_NAME SECURITY SUBCLASS_ORIGIN TABLE_NAME USER TYPE VALUE DATETIME TIMESTAMP TIME DATE YEAR - TEXT NCHAR NVARCHAR BOOL BOOLEAN ENUM BIT FIXED SIGNED ROLE + TEXT NCHAR NVARCHAR BOOL BOOLEAN ENUM BIT FIXED SIGNED ROLE SUBMIT CANCEL JOB //-----------------------------non_reserved keyword end--------------------------------------------- %right END_KEY %left ELSE IF ELSEIF @@ -276,6 +276,7 @@ void obpl_mysql_wrap_get_user_var_into_subquery(ObParseCtx *parse_ctx, ParseNode %type create_trigger_stmt drop_trigger_stmt plsql_trigger_source %type trigger_definition trigger_event trigger_body pl_obj_access_ref %type trigger_time +%type submit_job_stmt cancel_job_stmt /*SQL data type*/ %type scalar_data_type opt_charset collation opt_collation charset_name collation_name %type number_literal literal charset_key opt_float_precision opt_number_precision opt_binary @@ -359,6 +360,8 @@ outer_stmt: | create_package_stmt { $$ = $1; } | create_package_body_stmt { $$ = $1; } | drop_package_stmt { $$ = $1; } + | submit_job_stmt { $$ = $1; } + | cancel_job_stmt { $$ = $1; } | sql_stmt { $$ = $1; } | call_sp_stmt { $$ = $1; } | do_sp_stmt { $$ = $1; } @@ -395,6 +398,7 @@ sql_stmt_prefix: | INSERT { $$ = NULL; } | DELETE { $$ = NULL; } | UPDATE { $$ = NULL; } + | LOAD { $$ = NULL; } ; sql_stmt: @@ -734,6 +738,7 @@ unreserved_keyword: | BODY %prec LOWER_PARENS | C | CATALOG_NAME + | CANCEL | CLASS_ORIGIN | CLOSE | COLUMN_NAME @@ -754,6 +759,7 @@ unreserved_keyword: | HANDLER | INTERFACE | INVOKER + | JOB | JSON | LANGUAGE | MESSAGE_TEXT @@ -773,6 +779,7 @@ unreserved_keyword: | SCHEMA_NAME | SECURITY | SUBCLASS_ORIGIN + | SUBMIT | TABLE_NAME | TYPE | VALUE @@ -2734,6 +2741,30 @@ scond_info_item_name: | MYSQL_ERRNO { $$ = DIAG_MYSQL_ERRNO; } ; +/***************************************************************************** + * + * OLAP ASYNC JOB grammar + * + *****************************************************************************/ +submit_job_stmt: + SUBMIT JOB sql_stmt + { + malloc_terminal_node($$, parse_ctx->mem_pool_, T_OLAP_ASYNC_JOB_SUBMIT); + const char *stmt_str = parse_ctx->stmt_str_ + @3.first_column; + int32_t str_len = @3.last_column - @3.first_column + 1; + $$->str_value_ = parse_strndup(stmt_str, str_len, parse_ctx->mem_pool_); + check_ptr($$->str_value_); + $$->str_len_ = str_len; + } +; + +cancel_job_stmt: + CANCEL JOB STRING + { + malloc_non_terminal_node($$, parse_ctx->mem_pool_, T_OLAP_ASYNC_JOB_CANCEL, 1, $3); + } +; + %% /** * parser function diff --git a/src/share/inner_table/ob_inner_table_schema.12401_12450.cpp b/src/share/inner_table/ob_inner_table_schema.12401_12450.cpp index e1f28ef5e..2f26d221b 100644 --- a/src/share/inner_table/ob_inner_table_schema.12401_12450.cpp +++ b/src/share/inner_table/ob_inner_table_schema.12401_12450.cpp @@ -12079,6 +12079,21 @@ int ObInnerTableSchema::all_virtual_tenant_scheduler_running_job_schema(ObTableS true, //is_nullable false); //is_autoincrement } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("job_class", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + 128, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } if (OB_SUCC(ret)) { table_schema.get_part_option().set_part_num(1); table_schema.set_part_level(PARTITION_LEVEL_ONE); diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index f79f2b9f6..f001a70b3 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -14370,6 +14370,7 @@ def_table_schema( ('destination', 'varchar:128', 'true'), ('credential_owner', 'varchar:30', 'true'), ('credential_name', 'varchar:30', 'true'), + ('job_class', 'varchar:128', 'true'), ], partition_columns = ['svr_ip', 'svr_port'], vtable_route_policy = 'distributed', diff --git a/src/sql/CMakeLists.txt b/src/sql/CMakeLists.txt index f5fee5e8d..8371ef900 100644 --- a/src/sql/CMakeLists.txt +++ b/src/sql/CMakeLists.txt @@ -272,6 +272,7 @@ ob_set_subtarget(ob_sql engine_cmd engine/cmd/ob_table_direct_insert_service.cpp engine/cmd/ob_tenant_snapshot_executor.cpp engine/cmd/ob_mock_executor.cpp + engine/cmd/ob_olap_async_job_executor.cpp ) ob_set_subtarget(ob_sql engine_dml @@ -1119,6 +1120,8 @@ ob_set_subtarget(ob_sql resolver_cmd resolver/cmd/ob_alter_system_resolver.cpp resolver/cmd/ob_tenant_snapshot_resolver.cpp resolver/cmd/ob_tenant_clone_resolver.cpp + resolver/cmd/ob_olap_async_job_resolver.cpp + resolver/cmd/ob_olap_async_job_stmt.cpp ) ob_set_subtarget(ob_sql resolver_dcl diff --git a/src/sql/engine/cmd/ob_olap_async_job_executor.cpp b/src/sql/engine/cmd/ob_olap_async_job_executor.cpp new file mode 100644 index 000000000..a8dc68196 --- /dev/null +++ b/src/sql/engine/cmd/ob_olap_async_job_executor.cpp @@ -0,0 +1,66 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_ENG +#include "sql/engine/cmd/ob_olap_async_job_executor.h" +#include +#include +#include +#include "lib/mysqlclient/ob_mysql_transaction.h" +#include "lib/string/ob_sql_string.h" +#include "sql/resolver/cmd/ob_olap_async_job_stmt.h" +#include "sql/engine/ob_exec_context.h" +#include "observer/dbms_scheduler/ob_dbms_sched_table_operator.h" +namespace oceanbase +{ +using namespace common; +using namespace obrpc; +using namespace share::schema; +namespace sql +{ +int ObOLAPAsyncCancelJobExecutor::execute(ObExecContext &ctx, ObOLAPAsyncCancelJobStmt &stmt) +{ + int ret = OB_SUCCESS; + uint64_t tenant_id = stmt.get_tenant_id(); + uint64_t user_id = stmt.get_user_id(); + const ObUserInfo *user_info = nullptr; + ObArenaAllocator allocator("ASYNC_JOB_TMP"); + dbms_scheduler::ObDBMSSchedJobInfo job_info; + schema::ObSchemaGetterGuard schema_guard; + if (OB_FAIL(ObMultiVersionSchemaService::get_instance().get_tenant_schema_guard(tenant_id, schema_guard))) { + LOG_WARN("fail to get schema guard", K(ret), K(tenant_id)); + } else if(OB_FAIL(schema_guard.get_user_info(tenant_id, user_id, user_info))) { + LOG_WARN("fail to get user id", KR(ret), K(tenant_id), K(user_id)); + } else if (OB_ISNULL(user_info)) { + ret = OB_USER_NOT_EXIST; + LOG_WARN("user not exist", KR(ret), K(tenant_id), K(user_id)); + } else if (OB_FAIL(dbms_scheduler::ObDBMSSchedJobUtils::get_dbms_sched_job_info( + *GCTX.sql_proxy_, + tenant_id, + false, // is_oracle_tenant + stmt.get_job_name(), + allocator, + job_info))) { + LOG_WARN("get job info failed", KR(ret), K(tenant_id), K(stmt.get_job_name())); + } else if (!job_info.is_olap_async_job_class()) { + ret = OB_ENTRY_NOT_EXIST; + LOG_WARN("cancel not olap async job", KR(ret), K(tenant_id), K(job_info)); + } else if (OB_FAIL(dbms_scheduler::ObDBMSSchedJobUtils::check_dbms_sched_job_priv(user_info, job_info))) { + LOG_WARN("check user priv failed", KR(ret), K(tenant_id), K(job_info)); + } else if (OB_FAIL(dbms_scheduler::ObDBMSSchedJobUtils::stop_dbms_sched_job(*GCTX.sql_proxy_, job_info, true /* delete after stop */))) { + LOG_WARN("failed to stop dbms scheduler job", KR(ret)); + } + return ret; +} + +} +} \ No newline at end of file diff --git a/src/sql/engine/cmd/ob_olap_async_job_executor.h b/src/sql/engine/cmd/ob_olap_async_job_executor.h new file mode 100644 index 000000000..6a973252b --- /dev/null +++ b/src/sql/engine/cmd/ob_olap_async_job_executor.h @@ -0,0 +1,37 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_SQL_ENGINE_CMD_OLAP_ASYNC_JOB_CMD_EXECUTOR_ +#define OCEANBASE_SQL_ENGINE_CMD_OLAP_ASYNC_JOB_CMD_EXECUTOR_ +#include "lib/string/ob_string.h" +#include "lib/container/ob_array_serialization.h" +#include "share/schema/ob_schema_struct.h" + +namespace oceanbase +{ +namespace sql +{ +class ObExecContext; +class ObOLAPAsyncCancelJobStmt; + +class ObOLAPAsyncCancelJobExecutor +{ +public: + ObOLAPAsyncCancelJobExecutor() {} + virtual ~ObOLAPAsyncCancelJobExecutor() {} + int execute(ObExecContext &ctx, ObOLAPAsyncCancelJobStmt &stmt); +private: + DISALLOW_COPY_AND_ASSIGN(ObOLAPAsyncCancelJobExecutor); +}; +} +} +#endif //OCEANBASE_SQL_ENGINE_CMD_OLAP_ASYNC_JOB_CMD_EXECUTOR_ diff --git a/src/sql/executor/ob_cmd_executor.cpp b/src/sql/executor/ob_cmd_executor.cpp index 3fcedd28d..bbc0e6d82 100644 --- a/src/sql/executor/ob_cmd_executor.cpp +++ b/src/sql/executor/ob_cmd_executor.cpp @@ -152,6 +152,8 @@ #include "sql/engine/cmd/ob_tenant_snapshot_executor.h" #include "sql/resolver/cmd/ob_tenant_clone_stmt.h" #include "sql/engine/cmd/ob_clone_executor.h" +#include "sql/resolver/cmd/ob_olap_async_job_stmt.h" +#include "sql/engine/cmd/ob_olap_async_job_executor.h" #ifdef OB_BUILD_TDE_SECURITY #include "sql/resolver/ddl/ob_create_keystore_stmt.h" #include "sql/resolver/ddl/ob_alter_keystore_stmt.h" @@ -1084,6 +1086,10 @@ int ObCmdExecutor::execute(ObExecContext &ctx, ObICmd &cmd) DEFINE_EXECUTE_CMD(ObTransferPartitionStmt, ObTransferPartitionExecutor); break; } + case stmt::T_OLAP_ASYNC_JOB_CANCEL: { + DEFINE_EXECUTE_CMD(ObOLAPAsyncCancelJobStmt, ObOLAPAsyncCancelJobExecutor); + break; + } case stmt::T_CS_DISKMAINTAIN: case stmt::T_TABLET_CMD: case stmt::T_SWITCH_ROOTSERVER: diff --git a/src/sql/parser/ob_parser.cpp b/src/sql/parser/ob_parser.cpp index 690d33de8..2b9bfbd9f 100644 --- a/src/sql/parser/ob_parser.cpp +++ b/src/sql/parser/ob_parser.cpp @@ -61,7 +61,9 @@ bool ObParser::is_pl_stmt(const ObString &stmt, bool *is_create_func, bool *is_c case S_BEGIN: case S_DROP: case S_ALTER: - case S_UPDATE: { + case S_UPDATE: + case S_SUBMIT: + case S_CANCEL: { if (ISSPACE(*p)) { p++; } else { @@ -381,6 +383,9 @@ ObParser::State ObParser::transform_normal(ObString &normal) ELSIF(6, S_SIGNAL, "signal") ELSIF(8, S_RESIGNAL, "resignal") ELSIF(5, S_FORCE, "force") + ELSIF(6, S_SUBMIT, "submit") + ELSIF(6, S_CANCEL, "cancel") + ELSIF(3, S_JOB, "job") ELSE() if (S_INVALID == state @@ -429,7 +434,9 @@ ObParser::State ObParser::transform_normal( case S_BEGIN: case S_DROP: case S_ALTER: - case S_UPDATE: { + case S_UPDATE: + case S_SUBMIT: + case S_CANCEL: { state = token; } break; case S_INVALID: @@ -497,6 +504,15 @@ ObParser::State ObParser::transform_normal( is_not_pl = true; } } break; + case S_SUBMIT: + case S_CANCEL: { + State token = transform_normal(normal); + if (S_JOB == token) { + is_pl = true; + } else { + is_not_pl = true; + } + } break; default: { is_not_pl = true; LOG_WARN_RET(common::OB_ERR_UNEXPECTED, "unexpected state", K(state)); diff --git a/src/sql/parser/ob_parser.h b/src/sql/parser/ob_parser.h index 6444e08e8..d8828ab6f 100644 --- a/src/sql/parser/ob_parser.h +++ b/src/sql/parser/ob_parser.h @@ -153,6 +153,9 @@ enum State { S_VALUES, S_TABLE, S_INTO, + S_SUBMIT, + S_CANCEL, + S_JOB, // add new states above me S_MAX }; diff --git a/src/sql/parser/sql_parser_mysql_mode.y b/src/sql/parser/sql_parser_mysql_mode.y index fec1ea60a..4f2035dd4 100644 --- a/src/sql/parser/sql_parser_mysql_mode.y +++ b/src/sql/parser/sql_parser_mysql_mode.y @@ -14440,6 +14440,14 @@ SHOW opt_extended_or_full TABLES opt_from_or_in_database_clause opt_show_conditi (void)($4); malloc_non_terminal_node($$, result->malloc_pool_, T_SHOW_OPEN_TABLES, 1, $5); } +| SHOW JOB STATUS +{ + malloc_terminal_node($$, result->malloc_pool_, T_SHOW_OLAP_ASYNC_JOB_STATUS); +} +| SHOW JOB STATUS WHERE JOB COMP_EQ STRING_VALUE +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_SHOW_OLAP_ASYNC_JOB_STATUS, 1, $7); +} | CHECK TABLE table_list check_table_options { (void) ($4); diff --git a/src/sql/plan_cache/ob_values_table_compression.cpp b/src/sql/plan_cache/ob_values_table_compression.cpp index 0f0d138b4..e065a6a09 100644 --- a/src/sql/plan_cache/ob_values_table_compression.cpp +++ b/src/sql/plan_cache/ob_values_table_compression.cpp @@ -28,14 +28,16 @@ const char *ObValuesTableCompression::lower_[ObParser::S_MAX] = { "", "", "", "", "", "", "", "", "", "", /* 0 ~9 */ "", "", "", "update", "", "", "", "", "", "", /* 10 ~19 */ "", "", "", "", "", "", "", "", "", "", /* 20 ~29 */ - "", "", "", "select", "insert", "delete", "values", "table", "into" /* 30 ~38 */ + "", "", "", "select", "insert", "delete", "values", "table", "into", "", /* 30 ~39 */ + "", "" /* 40 ~41 */ }; const char *ObValuesTableCompression::upper_[ObParser::S_MAX] = { "", "", "", "", "", "", "", "", "", "", /* 0 ~9 */ "", "", "", "UPDATE", "", "", "", "", "", "", /* 10 ~19 */ "", "", "", "", "", "", "", "", "", "", /* 20 ~29 */ - "", "", "", "SELECT", "INSERT", "DELETE", "VALUES", "TABLE", "INTO" /* 30 ~38 */ + "", "", "", "SELECT", "INSERT", "DELETE", "VALUES", "TABLE", "INTO", "", /* 30 ~39 */ + "", "" /* 40 ~41 */ }; #define ISSPACE(c) ((c) == ' ' || (c) == '\n' || (c) == '\r' || (c) == '\t' || (c) == '\f' || (c) == '\v') diff --git a/src/sql/resolver/cmd/ob_olap_async_job_resolver.cpp b/src/sql/resolver/cmd/ob_olap_async_job_resolver.cpp new file mode 100644 index 000000000..cb8d12864 --- /dev/null +++ b/src/sql/resolver/cmd/ob_olap_async_job_resolver.cpp @@ -0,0 +1,305 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_RESV +#include "sql/resolver/cmd/ob_olap_async_job_resolver.h" +#include "sql/parser/ob_parser.h" +#include "sql/resolver/ob_resolver_utils.h" +#include + +namespace oceanbase +{ +using namespace common; +using namespace share; +namespace sql +{ +ObOLAPAsyncJobResolver::ObOLAPAsyncJobResolver(ObResolverParams ¶ms) + : ObSelectResolver(params) +{ +} + +int ObOLAPAsyncJobResolver::resolve(const ParseNode &parse_tree) +{ + + int ret = OB_SUCCESS; + ObItemType stmt_type = parse_tree.type_; + switch (stmt_type) { + case T_OLAP_ASYNC_JOB_SUBMIT: { + ObOLAPAsyncSubmitJobStmt job_stmt; + OZ (resolve_submit_job_stmt(parse_tree, job_stmt)); + OZ (execute_submit_job(job_stmt)); + OZ (init_select_stmt(job_stmt)); + break; + } + case T_OLAP_ASYNC_JOB_CANCEL: { + ObOLAPAsyncCancelJobStmt *stmt = create_stmt(); + OV (OB_NOT_NULL(stmt), OB_ALLOCATE_MEMORY_FAILED); + OZ (resolve_cancel_job_stmt(parse_tree, stmt)); + break; + } + default: + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid stmt type", K(ret), K(stmt_type)); + } + return ret; +} + +int ObOLAPAsyncJobResolver::resolve_submit_job_stmt(const ParseNode &parse_tree, ObOLAPAsyncSubmitJobStmt &stmt) +{ + int ret = OB_SUCCESS; + int64_t session_query_time_out_ts = 0; + if (OB_JOB_SQL_MAX_LENGTH - 1 < parse_tree.str_len_) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("sql too long", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "sql length"); + } else if (OB_INVALID_ID == session_info_->get_database_id()) { + ret = OB_ERR_NO_DB_SELECTED; + LOG_WARN("not select database", KR(ret)); + } else if (OB_FAIL(session_info_->get_query_timeout(session_query_time_out_ts))){ + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get session query timeout failed", KR(ret)); + } else { + + const int definer_buf_size = OB_MAX_USER_NAME_LENGTH + OB_MAX_HOST_NAME_LENGTH + 2; // @ + \0 + char *definer_buf = static_cast(allocator_->alloc(definer_buf_size)); + if (OB_ISNULL(definer_buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory", K(ret)); + } else { + memset(definer_buf, 0, definer_buf_size); + snprintf(definer_buf, definer_buf_size, "%.*s@%.*s", session_info_->get_user_name().length(), session_info_->get_user_name().ptr(), + session_info_->get_host_name().length(), session_info_->get_host_name().ptr()); + stmt.set_job_definer(definer_buf); + } + + if (OB_SUCC(ret)) { + const uint64_t tenant_id = params_.session_info_->get_effective_tenant_id(); + stmt.set_tenant_id(tenant_id); + stmt.set_user_id(session_info_->get_user_id()); + stmt.set_job_database(session_info_->get_database_name()); + stmt.set_database_id(session_info_->get_database_id()); + stmt.set_query_time_out_second(session_query_time_out_ts / 1000 / 1000); + } + + if (OB_SUCC(ret)) { + char *sql_buf = static_cast(allocator_->alloc(OB_JOB_SQL_MAX_LENGTH)); + if (OB_ISNULL(sql_buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory", K(ret)); + } else { + memset(sql_buf, 0, OB_JOB_SQL_MAX_LENGTH); + snprintf(sql_buf, OB_JOB_SQL_MAX_LENGTH, "%.*s;", (int)parse_tree.str_len_, parse_tree.str_value_); + stmt.set_job_action(sql_buf); + } + } + + if (OB_SUCC(ret)) { + char *job_exec_buf = static_cast(allocator_->alloc(OB_MAX_PROC_ENV_LENGTH)); + if (OB_ISNULL(job_exec_buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory", K(ret)); + } else { + memset(job_exec_buf, 0, OB_MAX_PROC_ENV_LENGTH); + int64_t pos = 0; + if (OB_FAIL(ObExecEnv::gen_exec_env(*session_info_, job_exec_buf, OB_MAX_PROC_ENV_LENGTH, pos))){ + ret = OB_ERR_UNEXPECTED; + LOG_WARN("generate exec env failed", K(ret), K(session_info_)); + } else { + stmt.set_exec_env((ObString(pos, job_exec_buf))); + } + } + } + + if (OB_SUCC(ret)) { + int64_t job_id = OB_INVALID_ID; + if (OB_FAIL(dbms_scheduler::ObDBMSSchedJobUtils::generate_job_id(stmt.get_tenant_id(), job_id))) { + LOG_WARN("generate_job_id failed", KR(ret), K(stmt.get_tenant_id())); + } else { + stmt.set_job_id(job_id); + char *job_name_buf = static_cast(allocator_->alloc(OB_JOB_NAME_MAX_LENGTH)); + if (OB_ISNULL(job_name_buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory", K(ret)); + } else { + memset(job_name_buf, 0, OB_JOB_NAME_MAX_LENGTH); + snprintf(job_name_buf, OB_JOB_NAME_MAX_LENGTH, "%"PRIu64"%"PRIu64,stmt.get_database_id(), stmt.get_job_id()); + stmt.set_job_name(job_name_buf); + } + } + } + } + return ret; +} +int ObOLAPAsyncJobResolver::resolve_cancel_job_stmt(const ParseNode &parse_tree, ObOLAPAsyncCancelJobStmt *stmt) +{ + int ret = OB_SUCCESS; + const uint64_t tenant_id = params_.session_info_->get_effective_tenant_id(); + stmt->set_tenant_id(tenant_id); + stmt->set_user_id(session_info_->get_user_id()); + if (parse_tree.num_child_ != 1) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid job name", K(ret)); + } else { + const ParseNode *job_name_node = parse_tree.children_[0]; + if (OB_ISNULL(job_name_node)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid job name", K(ret)); + } else { + char *job_name_buf = static_cast(allocator_->alloc(OB_JOB_NAME_MAX_LENGTH)); + if (OB_ISNULL(job_name_buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory", K(ret)); + } else { + memset(job_name_buf, 0, OB_JOB_NAME_MAX_LENGTH); + snprintf(job_name_buf, OB_JOB_NAME_MAX_LENGTH, "%.*s", (int)job_name_node->str_len_, job_name_node->str_value_); + stmt->set_job_name(job_name_buf); + } + } + } + return ret; +} + +#ifdef ERRSIM +ERRSIM_POINT_DEF(ERRSIM_SUBMIT_ERR_JOB_NAME); +ERRSIM_POINT_DEF(ERRSIM_SUBMIT_ERR_JOB_START_TIME); +#endif + +int ObOLAPAsyncJobResolver::execute_submit_job(ObOLAPAsyncSubmitJobStmt &stmt) +{ + int ret = OB_SUCCESS; + if (OB_INVALID_TENANT_ID == stmt.get_tenant_id()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tenant id", KR(ret), K(stmt.get_tenant_id())); + } else { + int64_t start_date_us = ObTimeUtility::current_time(); + int64_t end_date_us = 64060560000000000; // 4000-01-01 + HEAP_VAR(dbms_scheduler::ObDBMSSchedJobInfo, job_info) { + job_info.tenant_id_ = stmt.get_tenant_id(); + job_info.user_id_ = stmt.get_user_id(); + job_info.database_id_ = stmt.get_database_id(); + job_info.job_ = stmt.get_job_id(); + job_info.job_name_ = stmt.get_job_name(); + job_info.job_action_ = stmt.get_job_action(); + job_info.lowner_ = stmt.get_job_definer(); + job_info.powner_ = stmt.get_job_definer(); + job_info.cowner_ = stmt.get_job_database(); + job_info.job_style_ = ObString("regular"); + job_info.job_type_ = ObString("PLSQL_BLOCK"); + job_info.job_class_ = ObString("OLAP_ASYNC_JOB_CLASS"); + job_info.what_ = stmt.get_job_action(); + job_info.start_date_ = start_date_us; + job_info.end_date_ = end_date_us; + job_info.interval_ = job_info.repeat_interval_; + job_info.repeat_interval_ = job_info.repeat_interval_; + job_info.enabled_ = true; + job_info.auto_drop_ = true; + job_info.max_run_duration_ = stmt.get_query_time_out_second() + 60; + job_info.interval_ts_ = 0; + job_info.exec_env_ = stmt.get_exec_env(); + job_info.comments_ = ObString("olap async job"); + + #ifdef ERRSIM + if (OB_SUCCESS != ERRSIM_SUBMIT_ERR_JOB_NAME) { //注入一个错误的JOB NAME + job_info.job_name_ = "ERRSIM_JOB"; + } + if (OB_SUCCESS != ERRSIM_SUBMIT_ERR_JOB_START_TIME) { //注入一个错误的开始时间 + job_info.start_date_ = 64060560000000000; + } + #endif + + ObMySQLTransaction trans; + if (OB_FAIL(trans.start(GCTX.sql_proxy_, stmt.get_tenant_id()))) { + LOG_WARN("failed to start trans", KR(ret), K(stmt.get_tenant_id())); + } else if (OB_FAIL(dbms_scheduler::ObDBMSSchedJobUtils::create_dbms_sched_job( + trans, stmt.get_tenant_id(), stmt.get_job_id(), job_info))) { + LOG_WARN("failed to create dbms scheduler job", KR(ret)); + } + if (trans.is_started()) { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { + LOG_WARN("failed to commit trans", KR(ret), KR(tmp_ret)); + ret = OB_SUCC(ret) ? tmp_ret : ret; + } + } + if (ret == OB_ERR_PRIMARY_KEY_DUPLICATE) { + LOG_WARN("job is exist", KR(ret)); + } + } + } + return ret; +} + +int ObOLAPAsyncJobResolver::init_select_stmt(ObOLAPAsyncSubmitJobStmt &stmt) +{ + int ret = OB_SUCCESS; + ObSelectStmt *select_stmt = create_stmt(); + OV (OB_NOT_NULL(select_stmt), OB_ALLOCATE_MEMORY_FAILED); + if (OB_LIKELY(OB_SUCC(ret))) { + ObSqlString select_sql; + if (OB_FAIL(select_sql.assign_fmt( + "SELECT '%.*s' as job_id",stmt.get_job_name().length(), stmt.get_job_name().ptr()))) { + LOG_WARN("assign sql string failed", KR(ret), K(stmt.get_job_name())); + } else if (OB_FAIL(parse_and_resolve_select_sql(select_sql.string()))) { + LOG_WARN("fail to parse and resolve select sql", K(ret), K(select_sql)); + } else if (OB_UNLIKELY(stmt::T_SELECT != stmt_->get_stmt_type())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected stmt type", K(stmt_->get_stmt_type())); + } else { + select_stmt->set_select_type(NOT_AFFECT_FOUND_ROWS); + if(OB_FAIL(select_stmt->formalize_stmt(session_info_))) { + LOG_WARN("pull select stmt all expr relation ids failed", K(ret)); + } + } + } + return ret; +} + +int ObOLAPAsyncJobResolver::parse_and_resolve_select_sql(const ObString &select_sql) +{ + int ret = OB_SUCCESS; + // 1. parse and resolve view defination + if (OB_ISNULL(session_info_) || OB_ISNULL(params_.allocator_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("data member is not init", K(ret), K(session_info_), K(params_.allocator_)); + } else { + ParseResult select_result; + ObParser parser(*params_.allocator_, session_info_->get_sql_mode()); + if (OB_FAIL(parser.parse(select_sql, select_result))) { + LOG_WARN("parse select sql failed", K(select_sql), K(ret)); + } else { + // use alias to make all columns number continued + if (OB_ISNULL(select_result.result_tree_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result tree is NULL", K(ret)); + } else if (OB_UNLIKELY(select_result.result_tree_->num_child_ != 1 + || NULL == select_result.result_tree_->children_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result tree is invalid", + K(ret), K(select_result.result_tree_->num_child_), K(select_result.result_tree_->children_)); + } else if (OB_UNLIKELY(NULL == select_result.result_tree_->children_[0])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result tree is invalid", K(ret), "child ptr", select_result.result_tree_->children_[0]); + } else { + ParseNode *select_stmt_node = select_result.result_tree_->children_[0]; + if (OB_FAIL(ObSelectResolver::resolve(*select_stmt_node))) { + LOG_WARN("resolve select in view definition failed", K(ret), K(select_stmt_node)); + } + } + } + } + return ret; +} + + + +} //namespace sql +} //namespace oceanbase \ No newline at end of file diff --git a/src/sql/resolver/cmd/ob_olap_async_job_resolver.h b/src/sql/resolver/cmd/ob_olap_async_job_resolver.h new file mode 100644 index 000000000..8f9a15ea6 --- /dev/null +++ b/src/sql/resolver/cmd/ob_olap_async_job_resolver.h @@ -0,0 +1,42 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef _OB_OLAP_ASYNC_JOB_RESOLVER_H +#define _OB_OLAP_ASYNC_JOB_RESOLVER_H 1 + +#include "sql/resolver/dml/ob_select_resolver.h" +#include "sql/resolver/cmd/ob_olap_async_job_stmt.h" +namespace oceanbase +{ +namespace sql +{ +class ObOLAPAsyncJobResolver: public ObSelectResolver +{ +public: + explicit ObOLAPAsyncJobResolver(ObResolverParams ¶ms); + virtual ~ObOLAPAsyncJobResolver() = default; + + virtual int resolve(const ParseNode &parse_tree); +private: + static const int OB_JOB_NAME_MAX_LENGTH = 128; + static const int OB_JOB_SQL_MAX_LENGTH = 4096; + int resolve_submit_job_stmt(const ParseNode &parse_tree, ObOLAPAsyncSubmitJobStmt &stmt); + int resolve_cancel_job_stmt(const ParseNode &parse_tree, ObOLAPAsyncCancelJobStmt *stmt); + int execute_submit_job(ObOLAPAsyncSubmitJobStmt &stmt); + int init_select_stmt(ObOLAPAsyncSubmitJobStmt &stmt); + int parse_and_resolve_select_sql(const common::ObString &select_sql); + // disallow copy + DISALLOW_COPY_AND_ASSIGN(ObOLAPAsyncJobResolver); +}; +}//namespace sql +}//namespace oceanbase +#endif // _OB_OLAP_ASYNC_JOB_RESOLVER_H \ No newline at end of file diff --git a/src/sql/resolver/cmd/ob_olap_async_job_stmt.cpp b/src/sql/resolver/cmd/ob_olap_async_job_stmt.cpp new file mode 100644 index 000000000..c5c05fcc8 --- /dev/null +++ b/src/sql/resolver/cmd/ob_olap_async_job_stmt.cpp @@ -0,0 +1,66 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_RESV +#include "sql/resolver/cmd/ob_olap_async_job_stmt.h" + +#include "share/ob_define.h" +#include "lib/string/ob_string.h" +#include "lib/string/ob_strings.h" +#include "lib/utility/ob_print_utils.h" + +using namespace oceanbase; +using namespace oceanbase::common; +using namespace oceanbase::sql; + +ObOLAPAsyncSubmitJobStmt::ObOLAPAsyncSubmitJobStmt(ObIAllocator *name_pool) + : ObCMDStmt(name_pool, stmt::T_OLAP_ASYNC_JOB_SUBMIT) +{ +} + +ObOLAPAsyncSubmitJobStmt::ObOLAPAsyncSubmitJobStmt() + : ObCMDStmt(NULL, stmt::T_OLAP_ASYNC_JOB_SUBMIT) +{ +} + +int64_t ObOLAPAsyncSubmitJobStmt::to_string(char *buf, const int64_t buf_len) const +{ + int64_t pos = 0; + if (NULL != buf) { + J_OBJ_START(); + J_KV(N_STMT_TYPE, ((int)stmt_type_)); + J_OBJ_END(); + } + return pos; +} + + +ObOLAPAsyncCancelJobStmt::ObOLAPAsyncCancelJobStmt(ObIAllocator *name_pool) + : ObCMDStmt(name_pool, stmt::T_OLAP_ASYNC_JOB_CANCEL) +{ +} + +ObOLAPAsyncCancelJobStmt::ObOLAPAsyncCancelJobStmt() + : ObCMDStmt(NULL, stmt::T_OLAP_ASYNC_JOB_CANCEL) +{ +} + +int64_t ObOLAPAsyncCancelJobStmt::to_string(char *buf, const int64_t buf_len) const +{ + int64_t pos = 0; + if (NULL != buf) { + J_OBJ_START(); + J_KV(N_STMT_TYPE, ((int)stmt_type_)); + J_OBJ_END(); + } + return pos; +} \ No newline at end of file diff --git a/src/sql/resolver/cmd/ob_olap_async_job_stmt.h b/src/sql/resolver/cmd/ob_olap_async_job_stmt.h new file mode 100644 index 000000000..89ffc00e7 --- /dev/null +++ b/src/sql/resolver/cmd/ob_olap_async_job_stmt.h @@ -0,0 +1,94 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_SQL_RESOLVER_CMD_OB_OLAP_ASYNC_JOB_STMT_ +#define OCEANBASE_SQL_RESOLVER_CMD_OB_OLAP_ASYNC_JOB_STMT_ + +#include "sql/resolver/cmd/ob_cmd_stmt.h" +#include "lib/string/ob_strings.h" +#include "share/ob_define.h" + +namespace oceanbase +{ +namespace sql +{ +class ObOLAPAsyncSubmitJobStmt: public ObCMDStmt +{ +public: + explicit ObOLAPAsyncSubmitJobStmt(common::ObIAllocator *name_pool); + ObOLAPAsyncSubmitJobStmt(); + virtual ~ObOLAPAsyncSubmitJobStmt() = default; + + inline void set_tenant_id(const uint64_t id) { tenant_id_ = id; } + inline void set_database_id(const uint64_t id) { database_id_ = id; } + inline void set_user_id(const uint64_t id) { user_id_ = id; } + inline void set_job_id(const int64_t id) { job_id_ = id; } + inline void set_query_time_out_second(const uint64_t time) { query_time_out_second_ = time; } + inline void set_job_name(const common::ObString &job_name) { job_name_ = job_name; } + inline void set_job_definer(const common::ObString &job_definer) { job_definer_ = job_definer; } + inline void set_job_database(const common::ObString &job_database) { job_database_ = job_database; } + inline void set_job_action(const common::ObString &job_action) { job_action_ = job_action; } + inline void set_exec_env(const common::ObString &exec_env) { exec_env_ = exec_env; } + inline uint64_t get_tenant_id() const { return tenant_id_; } + inline uint64_t get_database_id() const { return database_id_; } + inline uint64_t get_user_id() const { return user_id_; } + inline int64_t get_job_id() const { return job_id_; } + inline uint64_t get_query_time_out_second() const { return query_time_out_second_; } + inline const common::ObString &get_job_name() const { return job_name_; } + inline const common::ObString &get_job_definer() const { return job_definer_; } + inline const common::ObString &get_job_action() const { return job_action_; } + inline const common::ObString &get_exec_env() const { return exec_env_; } + inline const common::ObString &get_job_database() const { return job_database_; } + DECLARE_VIRTUAL_TO_STRING; +private: + // data members + uint64_t tenant_id_; + uint64_t database_id_; + uint64_t user_id_; + int64_t job_id_; + uint64_t query_time_out_second_; + common::ObString job_name_; + common::ObString job_definer_; + common::ObString job_database_; + common::ObString job_action_; + common::ObString exec_env_; + +private: + DISALLOW_COPY_AND_ASSIGN(ObOLAPAsyncSubmitJobStmt); +}; + +class ObOLAPAsyncCancelJobStmt: public ObCMDStmt +{ +public: + explicit ObOLAPAsyncCancelJobStmt(common::ObIAllocator *name_pool); + ObOLAPAsyncCancelJobStmt(); + virtual ~ObOLAPAsyncCancelJobStmt() = default; + + inline void set_tenant_id(const uint64_t id) { tenant_id_ = id; } + inline void set_user_id(const uint64_t id) { user_id_ = id; } + inline void set_job_name(const common::ObString &job_name) { job_name_ = job_name; } + inline uint64_t get_tenant_id() const { return tenant_id_; } + inline uint64_t get_user_id() const { return user_id_; } + inline const common::ObString &get_job_name() const { return job_name_; } + DECLARE_VIRTUAL_TO_STRING; +private: + // data members + uint64_t tenant_id_; + uint64_t user_id_; + common::ObString job_name_; +private: + DISALLOW_COPY_AND_ASSIGN(ObOLAPAsyncCancelJobStmt); +}; + +} // end namespace sql +} // end namespace oceanbase +#endif //OCEANBASE_SQL_RESOLVER_CMD_OB_OLAP_ASYNC_JOB_STMT_ \ No newline at end of file diff --git a/src/sql/resolver/cmd/ob_show_resolver.cpp b/src/sql/resolver/cmd/ob_show_resolver.cpp index a487674b0..99b157490 100644 --- a/src/sql/resolver/cmd/ob_show_resolver.cpp +++ b/src/sql/resolver/cmd/ob_show_resolver.cpp @@ -161,7 +161,8 @@ int ObShowResolver::resolve(const ParseNode &parse_tree) && OB_UNLIKELY(parse_tree.type_ != T_SHOW_ENGINE) && OB_UNLIKELY(parse_tree.type_ != T_SHOW_OPEN_TABLES) && OB_UNLIKELY(parse_tree.type_ != T_SHOW_CREATE_USER) - && OB_UNLIKELY(parse_tree.type_ != T_SHOW_CHECK_TABLE)) { + && OB_UNLIKELY(parse_tree.type_ != T_SHOW_CHECK_TABLE) + && OB_UNLIKELY(parse_tree.type_ != T_SHOW_OLAP_ASYNC_JOB_STATUS)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected parse tree type", K(ret), K(parse_tree.type_)); } else { @@ -1729,6 +1730,56 @@ int ObShowResolver::resolve(const ParseNode &parse_tree) } break; } + case T_SHOW_OLAP_ASYNC_JOB_STATUS: { + [&] { + const int WHERE_JOB_NAME_LENGTH = 128 + 20; + const int LIMIT_LENGTH = 40; + char where_job_name[WHERE_JOB_NAME_LENGTH] = {}; + char limit_count [LIMIT_LENGTH] = {}; + uint64_t min_version = OB_INVALID_VERSION; + + if (OB_FAIL(GET_MIN_DATA_VERSION(real_tenant_id, min_version))) { + LOG_WARN("get min data_version failed", K(ret), K(real_tenant_id)); + } else if (min_version < DATA_VERSION_4_3_2_1) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not support show async job", K(ret), K(real_tenant_id)); + } else if (parse_tree.num_child_ == 1 && OB_NOT_NULL(parse_tree.children_)) { + snprintf(where_job_name, WHERE_JOB_NAME_LENGTH, " AND JOB_NAME = '%.*s' ", (int)parse_tree.children_[0]->str_len_, parse_tree.children_[0]->str_value_); + snprintf(limit_count, LIMIT_LENGTH, "order by update_time desc limit 1"); + } else if (parse_tree.num_child_ == 0) { + //nothing to do + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("parse tree is wrong", K(ret), K(parse_tree.num_child_), + K(parse_tree.children_)); + } + if (OB_SUCC(ret)) { + const int WHERE_USR_NAME_LENGTH = OB_MAX_USER_NAME_LENGTH + OB_MAX_HOST_NAME_LENGTH + 20; + char where_user_from_jobs[WHERE_USR_NAME_LENGTH] = {}; + char where_user_from_logs[WHERE_USR_NAME_LENGTH] = {}; + if (0 != session_info_->get_user_name().case_compare("root")) { + snprintf(where_user_from_jobs, WHERE_USR_NAME_LENGTH, " AND T.POWNER = '%.*s@%.*s' ", session_info_->get_user_name().length(), session_info_->get_user_name().ptr(), + session_info_->get_host_name().length(), session_info_->get_host_name().ptr()); + snprintf(where_user_from_logs, WHERE_USR_NAME_LENGTH, " AND R.OWNER = '%.*s@%.*s' ", session_info_->get_user_name().length(), session_info_->get_user_name().ptr(), + session_info_->get_host_name().length(), session_info_->get_host_name().ptr()); + } + show_resv_ctx.stmt_type_ = stmt::T_SHOW_OLAP_ASYNC_JOB_STATUS; + GEN_SQL_STEP_1(ObShowSqlSet::SHOW_OLAP_ASYNC_JOB_STATUS); + GEN_SQL_STEP_2(ObShowSqlSet::SHOW_OLAP_ASYNC_JOB_STATUS, + OB_SYS_DATABASE_NAME, + OB_ALL_SCHEDULER_JOB_RUN_DETAIL_V2_TNAME, + where_job_name, + where_user_from_logs, + OB_SYS_DATABASE_NAME, + OB_ALL_TENANT_SCHEDULER_JOB_TNAME, + sql_tenant_id, + where_job_name, + where_user_from_jobs, + limit_count); + } + }(); + break; + } default: /* won't be here */ ret = OB_NOT_IMPLEMENT; @@ -3725,6 +3776,13 @@ DEFINE_SHOW_CLAUSE_SET(SHOW_SEQUENCES_LIKE, "SELECT sequence_name FROM %s.%s WHERE database_id = %ld ORDER BY sequence_name COLLATE utf8mb4_bin ASC", NULL, "sequence_name"); +DEFINE_SHOW_CLAUSE_SET(SHOW_OLAP_ASYNC_JOB_STATUS, + NULL, + "(SELECT R.job_name AS 'job_id', R.database_name AS 'schema_name', CASE WHEN R.status = 'COMPLETED' AND R.message = 'SUCCESS' THEN 'FINISH' WHEN R.status = 'COMPLETED' AND R.message <> 'SUCCESS' THEN 'FAILED' WHEN R.status = 'KILLED' THEN 'CANCELLED' ELSE R.status END as 'status', R.message as 'fail_msg', R.req_start_date as 'create_time', R.time as 'update_time', R.operation AS 'definition' FROM %s.%s R WHERE R.JOB_CLASS = 'OLAP_ASYNC_JOB_CLASS' %s %s\ + UNION ALL\ + SELECT T.job_name AS 'job_id', T.cowner AS 'schema_name', CASE WHEN T.state IS NULL THEN 'SUBMITTED' WHEN T.state = 'SCHEDULED' THEN 'RUNNING' WHEN T.state = 'KILLED' THEN 'CANCELLED' ELSE T.state END as 'status', NULL as fail_msg, T.start_date as 'create_time', T.gmt_modified as 'update_time', T.job_action as 'definition' FROM %s.%s T WHERE T.JOB_CLASS = 'OLAP_ASYNC_JOB_CLASS' AND T.TENANT_ID = %d AND T.JOB > 0 %s %s) %s", + NULL, + NULL); DEFINE_SHOW_CLAUSE_SET(SHOW_CREATE_USER, NULL, "SELECT \"%.*s\" AS `CREATE USER for %.*s@%.*s` FROM DUAL", diff --git a/src/sql/resolver/cmd/ob_show_resolver.h b/src/sql/resolver/cmd/ob_show_resolver.h index 78a0f1dd1..d6b8b01fa 100644 --- a/src/sql/resolver/cmd/ob_show_resolver.h +++ b/src/sql/resolver/cmd/ob_show_resolver.h @@ -209,6 +209,7 @@ struct ObShowResolver::ObShowSqlSet DECLARE_SHOW_CLAUSE_SET(SHOW_SEQUENCES_LIKE); DECLARE_SHOW_CLAUSE_SET(SHOW_ENGINE); DECLARE_SHOW_CLAUSE_SET(SHOW_OPEN_TABLES); + DECLARE_SHOW_CLAUSE_SET(SHOW_OLAP_ASYNC_JOB_STATUS); DECLARE_SHOW_CLAUSE_SET(SHOW_CREATE_USER); };// ObShowSqlSet diff --git a/src/sql/resolver/ob_resolver.cpp b/src/sql/resolver/ob_resolver.cpp index 347635c63..049d8fff8 100644 --- a/src/sql/resolver/ob_resolver.cpp +++ b/src/sql/resolver/ob_resolver.cpp @@ -137,6 +137,7 @@ #include "sql/resolver/ddl/ob_drop_context_resolver.h" #include "sql/resolver/cmd/ob_tenant_snapshot_resolver.h" #include "sql/resolver/cmd/ob_tenant_clone_resolver.h" +#include "sql/resolver/cmd/ob_olap_async_job_resolver.h" #ifdef OB_BUILD_TDE_SECURITY #include "sql/resolver/ddl/ob_create_tablespace_resolver.h" #include "sql/resolver/ddl/ob_alter_tablespace_resolver.h" @@ -761,6 +762,7 @@ int ObResolver::resolve(IsPrepared if_prepared, const ParseNode &parse_tree, ObS case T_SHOW_ENGINE: case T_SHOW_OPEN_TABLES: case T_SHOW_SEQUENCES: + case T_SHOW_OLAP_ASYNC_JOB_STATUS: case T_SHOW_CHECK_TABLE: case T_SHOW_CREATE_USER: { REGISTER_STMT_RESOLVER(Show); @@ -1272,6 +1274,14 @@ int ObResolver::resolve(IsPrepared if_prepared, const ParseNode &parse_tree, ObS REGISTER_STMT_RESOLVER(Mock); break; } + case T_OLAP_ASYNC_JOB_SUBMIT: { + REGISTER_STMT_RESOLVER(OLAPAsyncJob); + break; + } + case T_OLAP_ASYNC_JOB_CANCEL: { + REGISTER_STMT_RESOLVER(OLAPAsyncJob); + break; + } default: { ret = OB_NOT_SUPPORTED; const char *type_name = get_type_name(parse_tree.type_); diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result index d24a8e582..04697e0de 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result @@ -9236,6 +9236,7 @@ destination_owner varchar(128) YES NULL destination varchar(128) YES NULL credential_owner varchar(30) YES NULL credential_name varchar(30) YES NULL +job_class varchar(128) YES NULL select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_tenant_scheduler_running_job; IF(count(*) >= 0, 1, 0) 1 diff --git a/unittest/sql/parser/test_mysql_pl_keyword.result b/unittest/sql/parser/test_mysql_pl_keyword.result index 32f38d07e..753f95405 100644 --- a/unittest/sql/parser/test_mysql_pl_keyword.result +++ b/unittest/sql/parser/test_mysql_pl_keyword.result @@ -171,6 +171,8 @@ Reserved Keyword: LONGBLOB Test PL procedure name Sql:create procedure LONGBLOB () select 1; Reserved Keyword: VARYING Test PL procedure name Sql:create procedure VARYING () select 1; +Reserved Keyword: LOAD +Test PL procedure name Sql:create procedure LOAD () select 1; Reserved Keyword: CHARSET Test PL procedure name Sql:create procedure CHARSET () select 1; Reserved Keyword: COMMIT @@ -181,7 +183,7 @@ Reserved Keyword: DO Test PL procedure name Sql:create procedure DO () select 1; Reserved Keyword: UNTIL Test PL procedure name Sql:create procedure UNTIL () select 1; -************** Total Count of Reserved Keyword:91 *************** +************** Total Count of Reserved Keyword:92 *************** ************** End Test Reserved Keyword *************** ************** Begin Test Non-Reserved Keyword *************** ************** End Test Non-Reserved Keyword *************** From 2dca1d44377732f4e912eae8191b3f5dd03748df Mon Sep 17 00:00:00 2001 From: obdev Date: Tue, 27 Aug 2024 11:49:44 +0000 Subject: [PATCH 233/249] Writing to the tmp file may quickly exit with an error code when OB_SERVER_OUTOF_DISK_SPACE reported; --- .../tmp_file/ob_tmp_file_flush_ctx.cpp | 4 ++ src/storage/tmp_file/ob_tmp_file_flush_ctx.h | 9 +++- .../ob_tmp_file_page_cache_controller.cpp | 8 ++- .../tmp_file/ob_tmp_file_thread_job.cpp | 5 +- src/storage/tmp_file/ob_tmp_file_thread_job.h | 5 +- .../tmp_file/ob_tmp_file_thread_wrapper.cpp | 52 ++++++++++++++++--- .../tmp_file/ob_tmp_file_thread_wrapper.h | 5 +- 7 files changed, 75 insertions(+), 13 deletions(-) diff --git a/src/storage/tmp_file/ob_tmp_file_flush_ctx.cpp b/src/storage/tmp_file/ob_tmp_file_flush_ctx.cpp index 1586d5371..864562376 100644 --- a/src/storage/tmp_file/ob_tmp_file_flush_ctx.cpp +++ b/src/storage/tmp_file/ob_tmp_file_flush_ctx.cpp @@ -233,6 +233,7 @@ ObTmpFileFlushTask::ObTmpFileFlushTask() : inst_handle_(), kvpair_(nullptr), block_handle_(), + write_block_ret_code_(OB_SUCCESS), ret_code_(OB_SUCCESS), data_length_(0), block_index_(-1), @@ -254,6 +255,7 @@ void ObTmpFileFlushTask::destroy() block_handle_.reset(); inst_handle_.reset(); kvpair_ = nullptr; + write_block_ret_code_ = OB_SUCCESS; ret_code_ = OB_SUCCESS; data_length_ = 0; block_index_ = -1; @@ -298,6 +300,8 @@ int ObTmpFileFlushTask::write_one_block() true/*update_to_max_time)*/))){ // update to max time to skip bad block inspect LOG_WARN("failed to update write time", KR(ret), K(handle_)); } + atomic_set_write_block_ret_code(ret); + return ret; } diff --git a/src/storage/tmp_file/ob_tmp_file_flush_ctx.h b/src/storage/tmp_file/ob_tmp_file_flush_ctx.h index 13041fb46..c48c02b9c 100644 --- a/src/storage/tmp_file/ob_tmp_file_flush_ctx.h +++ b/src/storage/tmp_file/ob_tmp_file_flush_ctx.h @@ -256,6 +256,12 @@ public: OB_INLINE char *get_data_buf() const { return block_handle_.value_ == nullptr ? nullptr : block_handle_.value_->get_buffer(); } OB_INLINE void atomic_set_ret_code(int ret_code) { ATOMIC_SET(&ret_code_, ret_code); } OB_INLINE int atomic_get_ret_code() const { return ATOMIC_LOAD(&ret_code_); } + OB_INLINE void atomic_set_write_block_ret_code(int write_block_ret_code) { + ATOMIC_SET(&write_block_ret_code_, write_block_ret_code); + } + OB_INLINE int atomic_get_write_block_ret_code() const { + return ATOMIC_LOAD(&write_block_ret_code_); + } OB_INLINE void set_data_length(const int64_t len) { data_length_ = len; } OB_INLINE int64_t get_data_length() const { return data_length_; } OB_INLINE void set_block_index(const int64_t block_index) { block_index_ = block_index; } @@ -283,13 +289,14 @@ public: return buffer != nullptr && get_data_buf() != nullptr && buffer >= get_data_buf() && buffer + length <= get_data_buf() + OB_SERVER_BLOCK_MGR.get_macro_block_size(); } - TO_STRING_KV(KP(this), KP(kvpair_), K(ret_code_), K(data_length_), + TO_STRING_KV(KP(this), KP(kvpair_), K(write_block_ret_code_), K(ret_code_), K(data_length_), K(block_index_), K(flush_seq_), K(create_ts_), K(is_io_finished_), K(fast_flush_tree_page_), K(recorded_as_prepare_finished_), K(task_state_), K(tmp_file_block_handle_), K(flush_infos_)); private: ObKVCacheInstHandle inst_handle_; ObKVCachePair *kvpair_; ObTmpBlockValueHandle block_handle_; + int write_block_ret_code_; int ret_code_; int64_t data_length_; // data length (including padding to make length upper align to page size) int64_t block_index_; // tmp file block logical index in ObTmpFileBlockManager diff --git a/src/storage/tmp_file/ob_tmp_file_page_cache_controller.cpp b/src/storage/tmp_file/ob_tmp_file_page_cache_controller.cpp index 802c1633e..1f09c4ea1 100644 --- a/src/storage/tmp_file/ob_tmp_file_page_cache_controller.cpp +++ b/src/storage/tmp_file/ob_tmp_file_page_cache_controller.cpp @@ -147,11 +147,15 @@ int ObTmpFilePageCacheController::invoke_swap_and_wait(int64_t expect_swap_size, } if (OB_NOT_NULL(swap_job)) { + if (OB_SUCCESS != swap_job->get_ret_code()) { + ret = swap_job->get_ret_code(); + } // reset swap job to set is_finished to false in case of failure to push into queue: // otherwise job is not finished, but it will not be executed, so it will never become finished. swap_job->reset(); - if (OB_FAIL(free_swap_job_(swap_job))) { - STORAGE_LOG(ERROR, "fail to free swap job", KR(ret)); + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(free_swap_job_(swap_job))) { + STORAGE_LOG(ERROR, "fail to free swap job", KR(ret), KR(tmp_ret)); } } return ret; diff --git a/src/storage/tmp_file/ob_tmp_file_thread_job.cpp b/src/storage/tmp_file/ob_tmp_file_thread_job.cpp index 7c7d3c291..1a961daac 100644 --- a/src/storage/tmp_file/ob_tmp_file_thread_job.cpp +++ b/src/storage/tmp_file/ob_tmp_file_thread_job.cpp @@ -33,6 +33,7 @@ int ObTmpFileSwapJob::init(int64_t expect_swap_size, uint32_t timeout_ms) STORAGE_LOG(WARN, "ObTmpFileSwapJob init cond failed", KR(ret)); } else { is_inited_ = true; + ret_code_ = OB_SUCCESS; is_finished_ = false; expect_swap_size_ = expect_swap_size; timeout_ms_ = timeout_ms; @@ -45,6 +46,7 @@ int ObTmpFileSwapJob::init(int64_t expect_swap_size, uint32_t timeout_ms) void ObTmpFileSwapJob::reset() { is_inited_ = false; + ret_code_ = OB_SUCCESS; is_finished_ = false; expect_swap_size_ = 0; timeout_ms_ = DEFAULT_TIMEOUT_MS; @@ -78,7 +80,7 @@ int ObTmpFileSwapJob::wait_swap_complete() } // set swap job is_finished, wake up threads that invoke swap job -int ObTmpFileSwapJob::signal_swap_complete() +int ObTmpFileSwapJob::signal_swap_complete(int ret_code) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { @@ -86,6 +88,7 @@ int ObTmpFileSwapJob::signal_swap_complete() STORAGE_LOG(WARN, "ObTmpFileSwapJob not init", KR(ret)); } else { ObThreadCondGuard guard(swap_cond_); + ATOMIC_SET(&ret_code_, ret_code); ATOMIC_SET(&is_finished_, true); if (OB_FAIL(swap_cond_.signal())) { STORAGE_LOG(WARN, "ObTmpFileSwapJob signal swap complete failed", KR(ret)); diff --git a/src/storage/tmp_file/ob_tmp_file_thread_job.h b/src/storage/tmp_file/ob_tmp_file_thread_job.h index 24a92914b..0145898be 100644 --- a/src/storage/tmp_file/ob_tmp_file_thread_job.h +++ b/src/storage/tmp_file/ob_tmp_file_thread_job.h @@ -29,6 +29,7 @@ public: static const uint32_t DEFAULT_TIMEOUT_MS = 10 * 1000; ObTmpFileSwapJob() : is_inited_(false), + ret_code_(OB_SUCCESS), is_finished_(false), timeout_ms_(DEFAULT_TIMEOUT_MS), create_ts_(0), @@ -39,16 +40,18 @@ public: int init(int64_t expect_swap_size, uint32_t timeout_ms = DEFAULT_TIMEOUT_MS); void reset(); int wait_swap_complete(); - int signal_swap_complete(); + int signal_swap_complete(int ret_code); OB_INLINE int64_t get_create_ts() const { return create_ts_; } OB_INLINE int64_t get_abs_timeout_ts() const { return abs_timeout_ts_; } OB_INLINE int64_t get_expect_swap_size() const { return expect_swap_size_; } OB_INLINE bool is_valid() { return ATOMIC_LOAD(&is_inited_) && swap_cond_.is_inited(); } OB_INLINE bool is_finished() const { return ATOMIC_LOAD(&is_finished_); } OB_INLINE bool is_inited() const { return ATOMIC_LOAD(&is_inited_); } + OB_INLINE int get_ret_code() const { return ATOMIC_LOAD(&ret_code_); } TO_STRING_KV(KP(this), K(is_inited_), K(is_finished_), K(create_ts_), K(timeout_ms_), K(abs_timeout_ts_), K(expect_swap_size_)); private: bool is_inited_; + int ret_code_; bool is_finished_; uint32_t timeout_ms_; int64_t create_ts_; diff --git a/src/storage/tmp_file/ob_tmp_file_thread_wrapper.cpp b/src/storage/tmp_file/ob_tmp_file_thread_wrapper.cpp index e082a6f29..28ea22fb9 100644 --- a/src/storage/tmp_file/ob_tmp_file_thread_wrapper.cpp +++ b/src/storage/tmp_file/ob_tmp_file_thread_wrapper.cpp @@ -31,6 +31,7 @@ ObTmpFileFlushTG::ObTmpFileFlushTG( : is_inited_(false), mode_(RUNNING_MODE::INVALID), last_flush_timestamp_(0), + flush_io_finished_ret_(OB_SUCCESS), flush_io_finished_round_(0), flushing_block_num_(0), is_fast_flush_meta_(false), @@ -62,6 +63,7 @@ int ObTmpFileFlushTG::init() is_inited_ = true; mode_ = RUNNING_MODE::NORMAL; last_flush_timestamp_ = 0; + flush_io_finished_ret_ = OB_SUCCESS; flush_io_finished_round_ = 0; flushing_block_num_ = 0; @@ -84,6 +86,7 @@ void ObTmpFileFlushTG::destroy() clean_up_lists(); mode_ = RUNNING_MODE::INVALID; last_flush_timestamp_ = 0; + flush_io_finished_ret_ = OB_SUCCESS; flush_io_finished_round_ = 0; flushing_block_num_ = 0; @@ -168,8 +171,9 @@ void ObTmpFileFlushTG::notify_doing_flush() last_flush_timestamp_ = 0; } -void ObTmpFileFlushTG::signal_io_finish() +void ObTmpFileFlushTG::signal_io_finish(int flush_io_finished_ret) { + flush_io_finished_ret_ = flush_io_finished_ret; ++flush_io_finished_round_; } @@ -178,6 +182,11 @@ int64_t ObTmpFileFlushTG::get_flush_io_finished_round() return flush_io_finished_round_; } +int64_t ObTmpFileFlushTG::get_flush_io_finished_ret() +{ + return flush_io_finished_ret_; +} + int64_t ObTmpFileFlushTG::cal_idle_time() { int64_t idle_time = 0; @@ -320,6 +329,10 @@ int ObTmpFileFlushTG::handle_generated_flush_tasks_(ObSpLinkQueue &flushing_list ret = OB_ERR_UNEXPECTED; LOG_ERROR("fail to switch state after data copied from tmp file", KR(ret), KPC(flush_task)); } else if (FlushState::TFFT_FILL_BLOCK_BUF < state) { + if (FlushState::TFFT_ASYNC_WRITE == state && + flush_task->atomic_get_write_block_ret_code() == OB_SERVER_OUTOF_DISK_SPACE) { + signal_io_finish(OB_SERVER_OUTOF_DISK_SPACE); + } push_retry_list_(flush_task); ATOMIC_INC(&flushing_block_num_); STORAGE_LOG(DEBUG, "push flush task to retry list", KPC(flush_task)); @@ -349,7 +362,7 @@ int ObTmpFileFlushTG::wash_(const int64_t expect_flush_size, const RUNNING_MODE bool idle_loop = flushing_task_cnt == 0; if (idle_loop && wbp_.get_dirty_page_percentage() < ObTmpFileFlushManager::FLUSH_WATERMARK_F5) { - signal_io_finish(); + signal_io_finish(OB_SUCCESS); } if (RUNNING_MODE::NORMAL == mode) { @@ -388,6 +401,10 @@ int ObTmpFileFlushTG::retry_task_() } else if (FlushState::TFFT_WAIT == state) { push_wait_list_(flush_task); } else if (FlushState::TFFT_FILL_BLOCK_BUF < state) { + if (FlushState::TFFT_ASYNC_WRITE == state && + flush_task->atomic_get_write_block_ret_code() == OB_SERVER_OUTOF_DISK_SPACE) { + signal_io_finish(OB_SERVER_OUTOF_DISK_SPACE); + } push_retry_list_(flush_task); if (FlushState::TFFT_INSERT_META_TREE == state && OB_ALLOCATE_TMP_FILE_PAGE_FAILED == ret) { STORAGE_LOG(WARN, "fail to retry insert meta item in TFFT_INSERT_META_TREE", KPC(flush_task)); @@ -486,7 +503,7 @@ void ObTmpFileFlushTG::flush_task_finished_(ObTmpFileFlushTask *flush_task) ret = OB_ERR_UNEXPECTED; STORAGE_LOG(ERROR, "fast_flush_meta_task_cnt_ is negative", KR(ret), KPC(this)); } - signal_io_finish(); + signal_io_finish(OB_SUCCESS); } int ObTmpFileFlushTG::push_wait_list_(ObTmpFileFlushTask *flush_task) @@ -786,7 +803,7 @@ void ObTmpFileSwapTG::clean_up_lists_() ObTmpFileSwapJob *swap_job = nullptr; if (OB_FAIL(swap_job_dequeue(swap_job))) { STORAGE_LOG(WARN, "fail dequeue swap job or swap job is nullptr", KR(ret), KP(swap_job)); - } else if (OB_FAIL(swap_job->signal_swap_complete())){ + } else if (OB_FAIL(swap_job->signal_swap_complete(OB_SUCCESS))){ STORAGE_LOG(WARN, "fail to signal swap complete", KR(ret)); } } @@ -796,7 +813,7 @@ void ObTmpFileSwapTG::clean_up_lists_() ObTmpFileSwapJob *swap_job = nullptr; if (OB_FAIL(pop_working_job_(swap_job))) { STORAGE_LOG(WARN, "fail to pop working job or ptr is null", KR(ret), KP(swap_job)); - } else if (OB_FAIL(swap_job->signal_swap_complete())){ + } else if (OB_FAIL(swap_job->signal_swap_complete(OB_SUCCESS))){ STORAGE_LOG(WARN, "fail to signal swap complete", KR(ret)); } } @@ -898,6 +915,10 @@ int ObTmpFileSwapTG::swap_fast_() int64_t wakeup_job_cnt = 0; wakeup_satisfied_jobs_(wakeup_job_cnt); wakeup_timeout_jobs_(); + int io_finished_ret = flush_tg_ref_.get_flush_io_finished_ret(); + if (OB_SERVER_OUTOF_DISK_SPACE == io_finished_ret) { + wakeup_all_jobs_(OB_SERVER_OUTOF_DISK_SPACE); + } // do flush if could not evict enough pages if (OB_SUCC(ret) && !working_list_.is_empty() && wakeup_job_cnt < PROCCESS_JOB_NUM_PER_BATCH) { @@ -953,7 +974,7 @@ int ObTmpFileSwapTG::wakeup_satisfied_jobs_(int64_t& wakeup_job_cnt) wbp_free_page_cnt -= min(wbp_free_page_cnt, single_job_swap_page_cnt); int64_t response_time = ObTimeUtility::current_time() - swap_job->get_create_ts(); swap_monitor_.record_swap_response_time(response_time); - if (OB_FAIL(swap_job->signal_swap_complete())) { + if (OB_FAIL(swap_job->signal_swap_complete(OB_SUCCESS))) { STORAGE_LOG(WARN, "fail to signal swap complete", KR(ret), KPC(swap_job)); } else { ++wakeup_job_cnt; @@ -974,7 +995,7 @@ int ObTmpFileSwapTG::wakeup_timeout_jobs_() // timeout, wake it up int64_t response_time = ObTimeUtility::current_time() - swap_job->get_create_ts(); swap_monitor_.record_swap_response_time(response_time); - if (OB_FAIL(swap_job->signal_swap_complete())) { + if (OB_FAIL(swap_job->signal_swap_complete(OB_TIMEOUT))) { STORAGE_LOG(WARN, "fail to signal swap complete", KR(ret), KP(swap_job)); } } else { @@ -986,6 +1007,23 @@ int ObTmpFileSwapTG::wakeup_timeout_jobs_() return ret; } +void ObTmpFileSwapTG::wakeup_all_jobs_(int ret_code) +{ + int ret = OB_SUCCESS; + for (int64_t i = working_list_size_; OB_SUCC(ret) && i > 0 && !working_list_.is_empty(); --i) { + ObTmpFileSwapJob *swap_job = nullptr; + if (OB_FAIL(pop_working_job_(swap_job))) { + STORAGE_LOG(WARN, "fail to pop working job or ptr is null", KR(ret), KP(swap_job)); + } else { + int64_t response_time = ObTimeUtility::current_time() - swap_job->get_create_ts(); + swap_monitor_.record_swap_response_time(response_time); + if (OB_FAIL(swap_job->signal_swap_complete(ret_code))) { + STORAGE_LOG(WARN, "fail to signal swap complete", KR(ret), KP(swap_job)); + } + } + } +} + int ObTmpFileSwapTG::push_working_job_(ObTmpFileSwapJob *swap_job) { int ret = OB_SUCCESS; diff --git a/src/storage/tmp_file/ob_tmp_file_thread_wrapper.h b/src/storage/tmp_file/ob_tmp_file_thread_wrapper.h index 1afedd17c..123de50d9 100644 --- a/src/storage/tmp_file/ob_tmp_file_thread_wrapper.h +++ b/src/storage/tmp_file/ob_tmp_file_thread_wrapper.h @@ -50,7 +50,8 @@ public: int try_work(); void set_running_mode(const RUNNING_MODE mode); void notify_doing_flush(); - void signal_io_finish(); + void signal_io_finish(int flush_io_finished_ret); + int64_t get_flush_io_finished_ret(); int64_t get_flush_io_finished_round(); int64_t cal_idle_time(); void clean_up_lists(); @@ -79,6 +80,7 @@ private: bool is_inited_; RUNNING_MODE mode_; int64_t last_flush_timestamp_; + int flush_io_finished_ret_; int64_t flush_io_finished_round_; int64_t flushing_block_num_; // maintain it when ObTmpFileFlushTask is created and freed @@ -133,6 +135,7 @@ private: int calculate_swap_page_num_(const int64_t batch_size, int64_t &expect_swap_cnt); int wakeup_satisfied_jobs_(int64_t& wakeup_job_cnt); int wakeup_timeout_jobs_(); + void wakeup_all_jobs_(int ret_code); private: bool is_inited_; int tg_id_; From 4920198e75d82fd8205788eeba7f5a3bba9baa62 Mon Sep 17 00:00:00 2001 From: zhjc1124 Date: Tue, 27 Aug 2024 11:55:43 +0000 Subject: [PATCH 234/249] move threads of deleted group to OBCG_DEFAULT --- src/share/resource_manager/ob_cgroup_ctrl.cpp | 29 ++++++++++++------- src/share/resource_manager/ob_cgroup_ctrl.h | 2 +- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/share/resource_manager/ob_cgroup_ctrl.cpp b/src/share/resource_manager/ob_cgroup_ctrl.cpp index f44adcdaf..4aa41b877 100644 --- a/src/share/resource_manager/ob_cgroup_ctrl.cpp +++ b/src/share/resource_manager/ob_cgroup_ctrl.cpp @@ -176,13 +176,17 @@ int ObCgroupCtrl::which_type_dir_(const char *curr_path, int &type) return ret; } -int ObCgroupCtrl::remove_dir_(const char *curr_dir) +int ObCgroupCtrl::remove_dir_(const char *curr_dir, bool is_delete_group) { int ret = OB_SUCCESS; char group_task_path[PATH_BUFSIZE]; - char parent_task_path[PATH_BUFSIZE]; + char target_task_path[PATH_BUFSIZE]; snprintf(group_task_path, PATH_BUFSIZE, "%s/tasks", curr_dir); - snprintf(parent_task_path, PATH_BUFSIZE, "%s/../tasks", curr_dir); + if (is_delete_group) { + snprintf(target_task_path, PATH_BUFSIZE, "%s/../tasks", curr_dir); + } else { + snprintf(target_task_path, PATH_BUFSIZE, "%s/../OBCG_DEFAULT/tasks", curr_dir); + } FILE* group_task_file = nullptr; if (OB_ISNULL(group_task_file = fopen(group_task_path, "r"))) { ret = OB_IO_ERROR; @@ -191,8 +195,8 @@ int ObCgroupCtrl::remove_dir_(const char *curr_dir) char tid_buf[VALUE_BUFSIZE]; int tmp_ret = OB_SUCCESS; while (fgets(tid_buf, VALUE_BUFSIZE, group_task_file)) { - if (OB_TMP_FAIL(ObCgroupCtrl::write_string_to_file_(parent_task_path, tid_buf))) { - LOG_WARN("remove tenant task failed", K(tmp_ret), K(parent_task_path)); + if (OB_TMP_FAIL(ObCgroupCtrl::write_string_to_file_(target_task_path, tid_buf))) { + LOG_WARN("remove tenant task failed", K(tmp_ret), K(target_task_path)); } } fclose(group_task_file); @@ -266,13 +270,18 @@ int ObCgroupCtrl::recursion_process_group_(const char *curr_path, DirProcessor * int ObCgroupCtrl::remove_cgroup(const uint64_t tenant_id, uint64_t group_id, const char *base_path) { int ret = OB_SUCCESS; - char tenant_path[PATH_BUFSIZE]; - if (OB_FAIL(get_group_path(tenant_path, PATH_BUFSIZE, tenant_id, group_id, base_path))) { + char group_path[PATH_BUFSIZE]; + if (OB_FAIL(get_group_path(group_path, PATH_BUFSIZE, tenant_id, group_id, base_path))) { LOG_WARN("fail get group path", K(tenant_id), K(ret)); - } else if (OB_FAIL(recursion_remove_group_(tenant_path))) { - LOG_WARN("remove cgroup directory failed", K(ret), K(tenant_path), K(tenant_id)); + } else if (is_valid_group(group_id)) { + ret = remove_dir_(group_path, true /* is_delete_group */); } else { - LOG_INFO("remove cgroup directory success", K(tenant_path), K(tenant_id)); + ret = recursion_remove_group_(group_path); + } + if (OB_FAIL(ret)) { + LOG_WARN("remove cgroup directory failed", K(ret), K(group_path), K(tenant_id)); + } else { + LOG_INFO("remove cgroup directory success", K(group_path), K(tenant_id)); } return ret; } diff --git a/src/share/resource_manager/ob_cgroup_ctrl.h b/src/share/resource_manager/ob_cgroup_ctrl.h index df05b65a8..8a945fcad 100644 --- a/src/share/resource_manager/ob_cgroup_ctrl.h +++ b/src/share/resource_manager/ob_cgroup_ctrl.h @@ -151,7 +151,7 @@ public: // 删除租户cgroup规则 int remove_cgroup(const uint64_t tenant_id, const uint64_t group_id = OB_INVALID_GROUP_ID, const char *base_path = ""); int remove_both_cgroup(const uint64_t tenant_id, const uint64_t group_id = OB_INVALID_GROUP_ID, const char *base_path = ""); - static int remove_dir_(const char *curr_dir); + static int remove_dir_(const char *curr_dir, bool is_delete_group = false); static int get_cgroup_config_(const char *group_path, const char *config_name, char *config_value); static int set_cgroup_config_(const char *group_path, const char *config_name, char *config_value); From 0de8b3356d88b0ecfff84b691eec8023b072b60f Mon Sep 17 00:00:00 2001 From: obdev Date: Tue, 27 Aug 2024 12:01:46 +0000 Subject: [PATCH 235/249] Fix the error code overwriting that causes crash --- deps/oblib/src/common/ob_field.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/deps/oblib/src/common/ob_field.cpp b/deps/oblib/src/common/ob_field.cpp index 263319382..944553e1b 100644 --- a/deps/oblib/src/common/ob_field.cpp +++ b/deps/oblib/src/common/ob_field.cpp @@ -31,10 +31,13 @@ int ObParamedSelectItemCtx::deep_copy(const ObParamedSelectItemCtx &other, ObIAl LOG_WARN("invalid null allocator", K(ret)); } else if (OB_FAIL(ob_write_string(*allocator, other.paramed_cname_, paramed_cname_))) { LOG_WARN("failed to write stirng", K(ret)); + } else if (OB_FAIL(param_str_offsets_.assign(other.param_str_offsets_))) { + LOG_WARN("failed to deep copy param string offsets", K(ret)); + } else if (OB_FAIL(param_idxs_.assign(other.param_idxs_))) { + LOG_WARN("failed to deep copy param idxs", K(ret)); + } else if (OB_FAIL(neg_param_idxs_.assign(other.neg_param_idxs_))) { + LOG_WARN("failed to deep copy neg param idxs", K(ret)); } else { - param_str_offsets_ = other.param_str_offsets_; - param_idxs_ = other.param_idxs_; - neg_param_idxs_ = other.neg_param_idxs_; esc_str_flag_ = other.esc_str_flag_; need_check_dup_name_ = other.need_check_dup_name_; is_column_field_ = other.is_column_field_; From 502c2b251dd58d48c5db43797fef5ed08b4656da Mon Sep 17 00:00:00 2001 From: PatZhuang Date: Tue, 27 Aug 2024 12:08:24 +0000 Subject: [PATCH 236/249] fix: do not merge view if contained correlated join/semi join condition expr --- .../rewrite/ob_transform_aggr_subquery.cpp | 45 ++++++---------- src/sql/rewrite/ob_transform_decorrelate.cpp | 15 ++---- src/sql/rewrite/ob_transform_utils.cpp | 51 +++++++------------ src/sql/rewrite/ob_transform_utils.h | 10 ++-- src/sql/rewrite/ob_transform_view_merge.cpp | 8 +-- .../ob_transform_where_subquery_pullup.cpp | 26 +++------- 6 files changed, 50 insertions(+), 105 deletions(-) diff --git a/src/sql/rewrite/ob_transform_aggr_subquery.cpp b/src/sql/rewrite/ob_transform_aggr_subquery.cpp index 07fcc0277..62041aba5 100644 --- a/src/sql/rewrite/ob_transform_aggr_subquery.cpp +++ b/src/sql/rewrite/ob_transform_aggr_subquery.cpp @@ -591,23 +591,16 @@ int ObTransformAggrSubquery::check_aggr_first_validity(ObDMLStmt &stmt, LOG_TRACE("select list is invalid", K(is_valid)); OPT_TRACE("subquery select item contain subquery"); // 3. check from list is not correlated - } else if (OB_FAIL(ObTransformUtils::is_table_item_correlated(query_ref.get_exec_params(), - *subquery, - is_correlated))) { - LOG_WARN("failed to check table item correlated or not", K(ret)); + // 4. check correlated join on conditions + // 5. check correlated semi conditions + } else if (OB_FAIL(ObTransformUtils::is_from_item_correlated(query_ref.get_exec_params(), + *subquery, + is_correlated))) { + LOG_WARN("failed to check from item correlated or not", K(ret)); } else if (is_correlated) { is_valid = false; - OPT_TRACE("subquery`s table item is correlated"); - // 4. check correlated join on contiditons - // 5. check correlated semi contiditons - } else if (OB_FAIL(ObTransformUtils::is_join_conditions_correlated(query_ref.get_exec_params(), - subquery, - is_correlated))) { - LOG_WARN("failed to check is join condition correlated", K(ret)); - } else if (is_correlated) { - is_valid = false; - OPT_TRACE("subquery`s outer/semi join condition is correlated"); - // 6. check correlated join contiditons + OPT_TRACE("subquery's table item is correlated"); + // 6. check correlated subquery conditions } else if (OB_FALSE_IT(hint_allowed_transform = (subquery->get_stmt_hint().has_enable_hint(T_UNNEST) || subquery->get_stmt_hint().has_enable_hint(T_AGGR_FIRST_UNNEST)))) { } else if (OB_FALSE_IT(check_match_index = hint_allowed_transform ? false : @@ -1583,24 +1576,18 @@ int ObTransformAggrSubquery::check_join_first_validity(ObQueryRefRawExpr &query_ LOG_TRACE("aggr item is invalid", K(is_valid)); OPT_TRACE("exists COUNT(NULL)"); // never reach - // 3. check from list is not correlated - } else if (OB_FAIL(ObTransformUtils::is_table_item_correlated( + // 3. check from list is not correlated + // 4. check correlated join on conditions + // 5. check correlated semi conditions + } else if (OB_FAIL(ObTransformUtils::is_from_item_correlated( query_ref.get_exec_params(), *subquery, is_correlated))) { - LOG_WARN("failed to check subquery table item is correlated", K(ret)); + LOG_WARN("failed to check if from item is correlated", K(ret)); } else if (is_correlated) { is_valid = false; - OPT_TRACE("subquery`s table item is correlated"); - // 4. check correlated join on contiditons - // 5. check correlated semi contiditons - } else if (OB_FAIL(ObTransformUtils::is_join_conditions_correlated(query_ref.get_exec_params(), - subquery, - is_correlated))) { - LOG_WARN("failed to check join condition correlated", K(ret)); - } else if (is_correlated) { - is_valid = false; - OPT_TRACE("subquery`s outer/semi join on condition is correlated"); + OPT_TRACE("subquery's table item is correlated"); + } - // 5. check correlated join contiditons + // 5. check correlated subquery conditions for (int64_t i = 0; OB_SUCC(ret) && is_valid && i < subquery->get_condition_size(); ++i) { ObRawExpr *cond = subquery->get_condition_expr(i); if (OB_ISNULL(cond)) { diff --git a/src/sql/rewrite/ob_transform_decorrelate.cpp b/src/sql/rewrite/ob_transform_decorrelate.cpp index 4b5dba53e..376f354eb 100644 --- a/src/sql/rewrite/ob_transform_decorrelate.cpp +++ b/src/sql/rewrite/ob_transform_decorrelate.cpp @@ -491,17 +491,10 @@ int ObTransformDecorrelate::check_lateral_inline_view_validity(TableItem *table_ } else if (check_status) { is_valid = false; OPT_TRACE("lateral inline view select expr contain subquery"); - } else if (OB_FAIL(ObTransformUtils::is_join_conditions_correlated(table_item->exec_params_, - ref_query, - check_status))) { - LOG_WARN("failed to is joined table conditions correlated", K(ret)); - } else if (check_status) { - is_valid = false; - OPT_TRACE("lateral inline view contain correlated on condition"); - } else if (OB_FAIL(ObTransformUtils::is_table_item_correlated(table_item->exec_params_, - *ref_query, - check_status))) { - LOG_WARN("failed to check if subquery contain correlated subquery", K(ret)); + } else if (OB_FAIL(ObTransformUtils::is_from_item_correlated(table_item->exec_params_, + *ref_query, + check_status))) { + LOG_WARN("failed to check if from items contains correlated subquery", K(ret)); } else if (check_status) { is_valid = false; OPT_TRACE("lateral inline view contain correlated table item"); diff --git a/src/sql/rewrite/ob_transform_utils.cpp b/src/sql/rewrite/ob_transform_utils.cpp index c57891ad6..b893fad0c 100644 --- a/src/sql/rewrite/ob_transform_utils.cpp +++ b/src/sql/rewrite/ob_transform_utils.cpp @@ -11240,15 +11240,10 @@ int ObTransformUtils::check_correlated_exprs_can_pullup(const ObIArray &exec_params, const ObSelectStmt &subquery, bool &contains) +int ObTransformUtils::is_from_item_correlated( + const ObIArray &exec_params, const ObSelectStmt &subquery, bool &is_correlated) { int ret = OB_SUCCESS; int64_t N = subquery.get_table_size(); - contains = false; - for (int64_t i = 0; OB_SUCC(ret) && !contains && i < N; ++i) { + is_correlated = false; + for (int64_t i = 0; OB_SUCC(ret) && !is_correlated && i < N; ++i) { const TableItem *table = subquery.get_table_item(i); if (OB_ISNULL(table)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("table should not be NULL", K(ret), K(table)); } else if (table->is_generated_table() || table->is_lateral_table()) { - if (OB_FAIL(is_correlated_subquery(exec_params, table->ref_query_, contains))) { + if (OB_FAIL(is_correlated_subquery(exec_params, table->ref_query_, is_correlated))) { LOG_WARN("check if subquery correlated failed", K(ret)); } } else if (table->is_function_table()) { - if (OB_FAIL(is_correlated_expr(exec_params, table->function_table_expr_, contains))) { + if (OB_FAIL(is_correlated_expr(exec_params, table->function_table_expr_, is_correlated))) { LOG_WARN("failed to check function table expr correlated", K(ret)); } } else if (table->is_json_table()) { if (OB_ISNULL(table->json_table_def_->doc_expr_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpect null expr", K(ret)); - } else if (OB_FAIL(is_correlated_expr(exec_params, table->json_table_def_->doc_expr_, contains))) { + } else if (OB_FAIL(is_correlated_expr(exec_params, table->json_table_def_->doc_expr_, is_correlated))) { LOG_WARN("failed to check function table expr correlated", K(ret)); } } else if (table->is_values_table()) { @@ -11649,30 +11644,18 @@ int ObTransformUtils::is_table_item_correlated( LOG_WARN("unexpect null expr", K(ret)); } else { ObIArray &access_exprs = table->values_table_def_->access_exprs_; - for (int64_t j = 0; OB_SUCC(ret) && !contains && j < access_exprs.count(); ++j) { - if (OB_FAIL(is_correlated_expr(exec_params, access_exprs.at(j), contains))) { + for (int64_t j = 0; OB_SUCC(ret) && !is_correlated && j < access_exprs.count(); ++j) { + if (OB_FAIL(is_correlated_expr(exec_params, access_exprs.at(j), is_correlated))) { LOG_WARN("failed to check values table expr correlated", K(ret)); } } } } } - return ret; -} - -// check joined table on conditions and semi info semi condition correlated -int ObTransformUtils::is_join_conditions_correlated(const ObIArray &exec_params, - const ObSelectStmt *subquery, - bool &is_correlated) -{ - int ret = OB_SUCCESS; - is_correlated = false; - if (OB_ISNULL(subquery)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(subquery), K(ret)); - } else { - const ObIArray &joined_tables = subquery->get_joined_tables(); - const ObIArray &semi_infos = subquery->get_semi_infos(); + // check joined table on conditions and semi info semi condition correlated + if (OB_SUCC(ret) && !is_correlated) { + const ObIArray &joined_tables = subquery.get_joined_tables(); + const ObIArray &semi_infos = subquery.get_semi_infos(); for (int64_t i = 0; OB_SUCC(ret) && !is_correlated && i < joined_tables.count(); ++i) { if (OB_FAIL(check_joined_conditions_correlated(exec_params, joined_tables.at(i), diff --git a/src/sql/rewrite/ob_transform_utils.h b/src/sql/rewrite/ob_transform_utils.h index add80efa5..8b170fd86 100644 --- a/src/sql/rewrite/ob_transform_utils.h +++ b/src/sql/rewrite/ob_transform_utils.h @@ -1590,13 +1590,9 @@ public: static int check_can_pullup_conds(const ObSelectStmt &subquery, bool &has_special_expr); - static int is_table_item_correlated(const ObIArray &exec_params, - const ObSelectStmt &subquery, - bool &contains); - - static int is_join_conditions_correlated(const ObIArray &exec_params, - const ObSelectStmt *subquery, - bool &is_correlated); + static int is_from_item_correlated(const ObIArray &exec_params, + const ObSelectStmt &subquery, + bool &is_correlated); static int check_semi_conditions_correlated(const ObIArray &exec_params, const SemiInfo *semi_info, diff --git a/src/sql/rewrite/ob_transform_view_merge.cpp b/src/sql/rewrite/ob_transform_view_merge.cpp index 2e575edb7..e6270d454 100644 --- a/src/sql/rewrite/ob_transform_view_merge.cpp +++ b/src/sql/rewrite/ob_transform_view_merge.cpp @@ -639,10 +639,10 @@ int ObTransformViewMerge::check_can_be_merged(ObDMLStmt *parent_stmt, LOG_WARN("unexpect null table item", K(ret)); } else if (!helper.trans_table->is_lateral_table()) { // do nothing - } else if (OB_FAIL(ObTransformUtils::is_table_item_correlated(helper.trans_table->exec_params_, - *child_stmt, - contain))) { - LOG_WARN("failed to check is table item correlated", K(ret)); + } else if (OB_FAIL(ObTransformUtils::is_from_item_correlated(helper.trans_table->exec_params_, + *child_stmt, + contain))) { + LOG_WARN("failed to check is from item correlated", K(ret)); } else if (contain) { can_be = false; OPT_TRACE("lateral inline view has correlated table item, can not merge"); diff --git a/src/sql/rewrite/ob_transform_where_subquery_pullup.cpp b/src/sql/rewrite/ob_transform_where_subquery_pullup.cpp index f9c8b7093..afb30359f 100644 --- a/src/sql/rewrite/ob_transform_where_subquery_pullup.cpp +++ b/src/sql/rewrite/ob_transform_where_subquery_pullup.cpp @@ -419,21 +419,14 @@ int ObWhereSubQueryPullup::check_subquery_validity(ObQueryRefRawExpr *query_ref, OPT_TRACE("subquery`s select expr contain subquery"); } else if (!is_correlated) { // do nothing - } else if (OB_FAIL(ObTransformUtils::is_join_conditions_correlated(query_ref->get_exec_params(), - subquery, - check_status))) { - LOG_WARN("failed to is joined table conditions correlated", K(ret)); - } else if (check_status) { - is_valid = false; - OPT_TRACE("subquery`s on condition is correlated"); } else if (OB_FAIL(is_where_having_subquery_correlated(query_ref->get_exec_params(), *subquery, check_status))) { LOG_WARN("failed to check select item contain subquery", K(subquery), K(ret)); } else if (check_status) { is_valid = false; OPT_TRACE("subquery`s where condition contain correlated subquery"); - } else if (OB_FAIL(ObTransformUtils::is_table_item_correlated( + } else if (OB_FAIL(ObTransformUtils::is_from_item_correlated( query_ref->get_exec_params(), *subquery, check_status))) { - LOG_WARN("failed to check if subquery contain correlated subquery", K(ret)); + LOG_WARN("failed to check if from item contains correlated subquery", K(ret)); } else if (check_status) { is_valid = false; OPT_TRACE("subquery`s table item is correlated"); @@ -1370,17 +1363,10 @@ int ObWhereSubQueryPullup::check_subquery_validity(ObDMLStmt &stmt, //2.check from item correlated if (OB_SUCC(ret) && is_valid) { bool is_correlated = false; - if (OB_FAIL(ObTransformUtils::is_join_conditions_correlated(query_ref->get_exec_params(), - subquery, - is_correlated))) { - LOG_WARN("failed to is joined table conditions correlated", K(ret)); - } else if (is_correlated) { - is_valid = false; - OPT_TRACE("subquery contain correlated on condition"); - } else if (OB_FAIL(ObTransformUtils::is_table_item_correlated(query_ref->get_exec_params(), - *subquery, - is_correlated))) { - LOG_WARN("failed to check if subquery contain correlated subquery", K(ret)); + if (OB_FAIL(ObTransformUtils::is_from_item_correlated(query_ref->get_exec_params(), + *subquery, + is_correlated))) { + LOG_WARN("failed to check if from item contains correlated subquery", K(ret)); } else if (is_correlated) { is_valid = false; OPT_TRACE("subquery contain correlated table item"); From ee76b4deb804e34c1528f7264c7900ae435e60e0 Mon Sep 17 00:00:00 2001 From: obdev Date: Tue, 27 Aug 2024 12:14:26 +0000 Subject: [PATCH 237/249] [to #2024082000104197520] fix memory leak of session cursor & fix var%notfound --- src/share/ob_errno.cpp | 17 ++++++- src/share/ob_errno.def | 2 + src/share/ob_errno.h | 6 ++- .../expr/ob_expr_pl_get_cursor_attr.cpp | 6 +++ src/sql/session/ob_sql_session_info.cpp | 10 +++-- src/sql/session/ob_sql_session_info.h | 44 ++++++++++++------- 6 files changed, 62 insertions(+), 23 deletions(-) diff --git a/src/share/ob_errno.cpp b/src/share/ob_errno.cpp index 209327789..d87853fd5 100644 --- a/src/share/ob_errno.cpp +++ b/src/share/ob_errno.cpp @@ -31527,6 +31527,20 @@ static const _error _error_OB_ERR_MVIEW_CAN_NOT_ON_QUERY_COMPUTE = { .ob_str_error = "OBE-32361: cannot ENABLE ON QUERY COMPUTATION for the materialized view", .ob_str_user_error = "OBE-32361: cannot ENABLE ON QUERY COMPUTATION for the materialized view %s.%s" }; +static const _error _error_OB_ERR_CURSOR_ATTR_APPLY = { + .error_name = "OB_ERR_CURSOR_ATTR_APPLY", + .error_cause = "Internal Error", + .error_solution = "Contact OceanBase Support", + .mysql_errno = -1, + .sqlstate = "HY000", + .str_error = "cursor attribute may not be applied to non-cursor", + .str_user_error = "cursor attribute may not be applied to non-cursor %.*s", + .oracle_errno = 324, + .oracle_str_error = "PLS-00324: cursor attribute may not be applied to non-cursor", + .oracle_str_user_error = "PLS-00324: cursor attribute may not be applied to non-cursor %.*s", + .ob_str_error = "PLS-00324: cursor attribute may not be applied to non-cursor", + .ob_str_user_error = "PLS-00324: cursor attribute may not be applied to non-cursor %.*s" +}; static const _error _error_OB_ERR_KV_GLOBAL_INDEX_ROUTE = { .error_name = "OB_ERR_KV_GLOBAL_INDEX_ROUTE", .error_cause = "Internal Error", @@ -34876,6 +34890,7 @@ struct ObStrErrorInit _errors[-OB_NO_PARTITION_FOR_GIVEN_VALUE_SCHEMA_ERROR] = &_error_OB_NO_PARTITION_FOR_GIVEN_VALUE_SCHEMA_ERROR; _errors[-OB_ERR_INVALID_CHARACTER] = &_error_OB_ERR_INVALID_CHARACTER; _errors[-OB_ERR_MVIEW_CAN_NOT_ON_QUERY_COMPUTE] = &_error_OB_ERR_MVIEW_CAN_NOT_ON_QUERY_COMPUTE; + _errors[-OB_ERR_CURSOR_ATTR_APPLY] = &_error_OB_ERR_CURSOR_ATTR_APPLY; _errors[-OB_ERR_KV_GLOBAL_INDEX_ROUTE] = &_error_OB_ERR_KV_GLOBAL_INDEX_ROUTE; _errors[-OB_TTL_NOT_ENABLE] = &_error_OB_TTL_NOT_ENABLE; _errors[-OB_TTL_COLUMN_NOT_EXIST] = &_error_OB_TTL_COLUMN_NOT_EXIST; @@ -34988,7 +35003,7 @@ namespace oceanbase { namespace common { -int g_all_ob_errnos[2327] = {0, -4000, -4001, -4002, -4003, -4004, -4005, -4006, -4007, -4008, -4009, -4010, -4011, -4012, -4013, -4014, -4015, -4016, -4017, -4018, -4019, -4020, -4021, -4022, -4023, -4024, -4025, -4026, -4027, -4028, -4029, -4030, -4031, -4032, -4033, -4034, -4035, -4036, -4037, -4038, -4039, -4041, -4042, -4043, -4044, -4045, -4046, -4047, -4048, -4049, -4050, -4051, -4052, -4053, -4054, -4055, -4057, -4058, -4060, -4061, -4062, -4063, -4064, -4065, -4066, -4067, -4068, -4070, -4071, -4072, -4073, -4074, -4075, -4076, -4077, -4078, -4080, -4081, -4084, -4085, -4090, -4097, -4098, -4099, -4100, -4101, -4102, -4103, -4104, -4105, -4106, -4107, -4108, -4109, -4110, -4111, -4112, -4113, -4114, -4115, -4116, -4117, -4118, -4119, -4120, -4121, -4122, -4123, -4124, -4125, -4126, -4127, -4128, -4133, -4138, -4139, -4142, -4143, -4144, -4146, -4147, -4149, -4150, -4151, -4152, -4153, -4154, -4155, -4156, -4157, -4158, -4159, -4160, -4161, -4162, -4163, -4164, -4165, -4166, -4167, -4168, -4169, -4170, -4171, -4172, -4173, -4174, -4175, -4176, -4177, -4178, -4179, -4180, -4181, -4182, -4183, -4184, -4185, -4186, -4187, -4188, -4189, -4190, -4191, -4192, -4200, -4201, -4204, -4205, -4206, -4207, -4208, -4209, -4210, -4211, -4212, -4213, -4214, -4215, -4216, -4217, -4218, -4219, -4220, -4221, -4222, -4223, -4224, -4225, -4226, -4227, -4228, -4229, -4230, -4231, -4232, -4233, -4234, -4235, -4236, -4237, -4238, -4239, -4240, -4241, -4242, -4243, -4244, -4245, -4246, -4247, -4248, -4249, -4250, -4251, -4252, -4253, -4254, -4255, -4256, -4257, -4258, -4260, -4261, -4262, -4263, -4264, -4265, -4266, -4267, -4268, -4269, -4270, -4271, -4273, -4274, -4275, -4276, -4277, -4278, -4279, -4280, -4281, -4282, -4283, -4284, -4285, -4286, -4287, -4288, -4289, -4290, -4291, -4292, -4293, -4294, -4295, -4296, -4297, -4298, -4299, -4300, -4301, -4302, -4303, -4304, -4305, -4306, -4307, -4308, -4309, -4310, -4311, -4312, -4313, -4314, -4315, -4316, -4317, -4318, -4319, -4320, -4321, -4322, -4323, -4324, -4325, -4326, -4327, -4328, -4329, -4330, -4331, -4332, -4333, -4334, -4335, -4336, -4337, -4338, -4339, -4340, -4341, -4342, -4343, -4344, -4345, -4346, -4347, -4348, -4349, -4350, -4351, -4352, -4353, -4354, -4355, -4356, -4357, -4358, -4359, -4360, -4361, -4362, -4363, -4364, -4365, -4366, -4367, -4368, -4369, -4370, -4371, -4372, -4373, -4374, -4375, -4376, -4377, -4378, -4379, -4380, -4381, -4382, -4383, -4385, -4386, -4387, -4388, -4389, -4390, -4391, -4392, -4393, -4394, -4395, -4396, -4397, -4398, -4399, -4400, -4401, -4402, -4403, -4505, -4507, -4510, -4512, -4515, -4517, -4518, -4519, -4523, -4524, -4525, -4526, -4527, -4528, -4529, -4530, -4531, -4532, -4533, -4537, -4538, -4539, -4540, -4541, -4542, -4543, -4544, -4545, -4546, -4547, -4548, -4549, -4550, -4551, -4552, -4553, -4554, -4600, -4601, -4602, -4603, -4604, -4605, -4606, -4607, -4608, -4609, -4610, -4611, -4613, -4614, -4615, -4620, -4621, -4622, -4623, -4624, -4625, -4626, -4628, -4629, -4630, -4631, -4632, -4633, -4634, -4636, -4637, -4638, -4639, -4640, -4641, -4642, -4643, -4644, -4645, -4646, -4647, -4648, -4649, -4650, -4651, -4652, -4653, -4654, -4655, -4656, -4657, -4658, -4659, -4660, -4661, -4662, -4663, -4664, -4665, -4666, -4667, -4668, -4669, -4670, -4671, -4672, -4673, -4674, -4675, -4676, -4677, -4678, -4679, -4680, -4681, -4682, -4683, -4684, -4685, -4686, -4687, -4688, -4689, -4690, -4691, -4692, -4693, -4694, -4695, -4696, -4697, -4698, -4699, -4700, -4701, -4702, -4703, -4704, -4705, -4706, -4707, -4708, -4709, -4710, -4711, -4712, -4713, -4714, -4715, -4716, -4717, -4718, -4719, -4720, -4721, -4722, -4723, -4724, -4725, -4726, -4727, -4728, -4729, -4730, -4731, -4732, -4733, -4734, -4735, -4736, -4737, -4738, -4739, -4740, -4741, -4742, -4743, -4744, -4745, -4746, -4747, -4748, -4749, -4750, -4751, -4752, -4753, -4754, -4755, -4756, -4757, -4758, -4759, -4760, -4761, -4762, -4763, -4764, -4765, -4766, -4767, -4768, -4769, -4770, -4771, -4772, -4773, -4774, -4775, -4776, -4777, -4778, -4779, -4780, -4781, -4782, -4783, -4784, -5000, -5001, -5002, -5003, -5006, -5007, -5008, -5010, -5011, -5012, -5014, -5015, -5016, -5017, -5018, -5019, -5020, -5022, -5023, -5024, -5025, -5026, -5027, -5028, -5029, -5030, -5031, -5032, -5034, -5035, -5036, -5037, -5038, -5039, -5040, -5041, -5042, -5043, -5044, -5046, -5047, -5050, -5051, -5052, -5053, -5054, -5055, -5056, -5057, -5058, -5059, -5061, -5063, -5064, -5065, -5066, -5067, -5068, -5069, -5070, -5071, -5072, -5073, -5074, -5080, -5081, -5083, -5084, -5085, -5086, -5087, -5088, -5089, -5090, -5091, -5092, -5093, -5094, -5095, -5096, -5097, -5098, -5099, -5100, -5101, -5102, -5103, -5104, -5105, -5106, -5107, -5108, -5109, -5110, -5111, -5112, -5113, -5114, -5115, -5116, -5117, -5118, -5119, -5120, -5121, -5122, -5123, -5124, -5125, -5130, -5131, -5133, -5134, -5135, -5136, -5137, -5138, -5139, -5140, -5142, -5143, -5144, -5145, -5146, -5147, -5148, -5149, -5150, -5151, -5153, -5154, -5155, -5156, -5157, -5158, -5159, -5160, -5161, -5162, -5163, -5164, -5165, -5166, -5167, -5168, -5169, -5170, -5171, -5172, -5173, -5174, -5175, -5176, -5177, -5178, -5179, -5180, -5181, -5182, -5183, -5184, -5185, -5187, -5188, -5189, -5190, -5191, -5192, -5193, -5194, -5195, -5196, -5197, -5198, -5199, -5200, -5201, -5202, -5203, -5204, -5205, -5206, -5207, -5208, -5209, -5210, -5211, -5212, -5213, -5214, -5215, -5216, -5217, -5218, -5219, -5220, -5221, -5222, -5223, -5224, -5225, -5226, -5227, -5228, -5229, -5230, -5231, -5233, -5234, -5235, -5236, -5237, -5238, -5239, -5240, -5241, -5242, -5243, -5244, -5245, -5246, -5247, -5248, -5249, -5250, -5251, -5252, -5253, -5254, -5255, -5256, -5257, -5258, -5259, -5260, -5261, -5262, -5263, -5264, -5265, -5266, -5267, -5268, -5269, -5270, -5271, -5272, -5273, -5274, -5275, -5276, -5277, -5278, -5279, -5280, -5281, -5282, -5283, -5284, -5285, -5286, -5287, -5288, -5289, -5290, -5291, -5292, -5293, -5294, -5295, -5296, -5297, -5298, -5299, -5300, -5301, -5302, -5303, -5304, -5305, -5306, -5307, -5308, -5309, -5310, -5311, -5312, -5313, -5314, -5315, -5316, -5317, -5318, -5319, -5320, -5321, -5322, -5323, -5324, -5325, -5326, -5327, -5328, -5329, -5330, -5331, -5332, -5333, -5334, -5335, -5336, -5337, -5338, -5339, -5340, -5341, -5342, -5343, -5344, -5345, -5346, -5347, -5348, -5349, -5350, -5351, -5352, -5353, -5354, -5355, -5356, -5357, -5358, -5359, -5360, -5361, -5362, -5363, -5364, -5365, -5366, -5367, -5368, -5369, -5370, -5371, -5372, -5373, -5374, -5375, -5376, -5377, -5378, -5379, -5380, -5381, -5382, -5383, -5384, -5385, -5386, -5387, -5388, -5389, -5390, -5400, -5401, -5402, -5403, -5404, -5405, -5406, -5407, -5408, -5409, -5410, -5411, -5412, -5413, -5414, -5415, -5416, -5417, -5418, -5419, -5420, -5421, -5422, -5423, -5424, -5425, -5426, -5427, -5428, -5429, -5430, -5431, -5432, -5433, -5434, -5435, -5436, -5437, -5438, -5439, -5440, -5441, -5442, -5443, -5444, -5445, -5446, -5447, -5448, -5449, -5450, -5451, -5452, -5453, -5454, -5455, -5456, -5457, -5458, -5459, -5460, -5461, -5462, -5463, -5464, -5465, -5466, -5467, -5468, -5469, -5470, -5471, -5472, -5473, -5474, -5475, -5476, -5477, -5478, -5479, -5480, -5481, -5482, -5483, -5484, -5485, -5486, -5487, -5488, -5489, -5490, -5491, -5492, -5493, -5494, -5495, -5496, -5497, -5498, -5499, -5500, -5501, -5502, -5503, -5504, -5505, -5506, -5507, -5508, -5509, -5510, -5511, -5512, -5513, -5514, -5515, -5516, -5517, -5518, -5519, -5520, -5521, -5522, -5540, -5541, -5542, -5543, -5544, -5545, -5546, -5547, -5548, -5549, -5550, -5551, -5552, -5553, -5554, -5555, -5556, -5557, -5558, -5559, -5560, -5561, -5562, -5563, -5564, -5565, -5566, -5567, -5568, -5569, -5570, -5571, -5572, -5573, -5574, -5575, -5576, -5577, -5578, -5579, -5580, -5581, -5582, -5583, -5584, -5585, -5586, -5587, -5588, -5589, -5590, -5591, -5592, -5593, -5594, -5595, -5596, -5597, -5598, -5599, -5600, -5601, -5602, -5603, -5604, -5605, -5607, -5608, -5609, -5610, -5611, -5612, -5613, -5614, -5615, -5616, -5617, -5618, -5619, -5620, -5621, -5622, -5623, -5624, -5625, -5626, -5627, -5628, -5629, -5630, -5631, -5632, -5633, -5634, -5635, -5636, -5637, -5638, -5639, -5640, -5641, -5642, -5643, -5644, -5645, -5646, -5647, -5648, -5649, -5650, -5651, -5652, -5653, -5654, -5655, -5656, -5657, -5658, -5659, -5660, -5661, -5662, -5663, -5664, -5665, -5666, -5667, -5668, -5671, -5672, -5673, -5674, -5675, -5676, -5677, -5678, -5679, -5680, -5681, -5682, -5683, -5684, -5685, -5686, -5687, -5688, -5689, -5690, -5691, -5692, -5693, -5694, -5695, -5696, -5697, -5698, -5699, -5700, -5701, -5702, -5703, -5704, -5705, -5706, -5707, -5708, -5709, -5710, -5711, -5712, -5713, -5714, -5715, -5716, -5717, -5718, -5719, -5720, -5721, -5722, -5723, -5724, -5725, -5726, -5727, -5728, -5729, -5730, -5731, -5732, -5733, -5734, -5735, -5736, -5737, -5738, -5739, -5740, -5741, -5742, -5743, -5744, -5745, -5746, -5747, -5748, -5749, -5750, -5751, -5752, -5753, -5754, -5755, -5756, -5757, -5758, -5759, -5760, -5761, -5762, -5763, -5764, -5765, -5766, -5768, -5769, -5770, -5771, -5772, -5773, -5774, -5777, -5778, -5779, -5780, -5781, -5785, -5786, -5787, -5788, -5789, -5790, -5791, -5792, -5793, -5794, -5795, -5796, -5797, -5798, -5799, -5800, -5801, -5802, -5803, -5804, -5805, -5806, -5807, -5808, -5809, -5810, -5811, -5812, -5813, -5814, -5815, -5816, -5817, -5818, -5819, -5820, -5821, -5822, -5823, -5824, -5825, -5826, -5827, -5828, -5829, -5830, -5831, -5832, -5833, -5834, -5835, -5836, -5837, -5838, -5839, -5840, -5841, -5842, -5843, -5844, -5845, -5846, -5847, -5848, -5849, -5850, -5851, -5852, -5853, -5854, -5855, -5856, -5857, -5858, -5859, -5860, -5861, -5862, -5863, -5864, -5865, -5866, -5867, -5868, -5869, -5870, -5871, -5872, -5873, -5874, -5875, -5876, -5877, -5878, -5879, -5880, -5881, -5882, -5883, -5884, -5885, -5886, -5887, -5888, -5889, -5890, -5891, -5892, -5893, -5894, -5895, -5896, -5897, -5898, -5899, -5900, -5901, -5902, -5903, -5904, -5905, -5906, -5907, -5908, -5909, -5910, -5911, -5912, -5913, -5914, -5915, -5916, -5917, -5918, -5919, -5920, -5921, -5922, -5923, -5924, -5925, -5926, -5927, -5928, -5929, -5930, -5931, -5932, -5933, -5934, -5935, -5936, -5937, -5938, -5939, -5940, -5941, -5942, -5943, -5944, -5945, -5946, -5947, -5948, -5949, -5950, -5951, -5952, -5953, -5954, -5955, -5956, -5957, -5958, -5959, -5960, -5961, -5962, -5963, -5964, -5965, -5966, -5967, -5968, -5969, -5970, -5971, -5972, -5973, -5974, -5975, -5976, -5977, -5978, -5979, -5980, -5981, -5982, -5983, -5984, -5985, -5986, -5987, -5988, -5989, -5990, -5991, -5992, -5993, -5994, -5995, -5996, -5997, -5998, -5999, -6000, -6001, -6002, -6003, -6004, -6005, -6006, -6201, -6202, -6203, -6204, -6205, -6206, -6207, -6208, -6209, -6210, -6211, -6212, -6213, -6214, -6215, -6219, -6220, -6221, -6222, -6223, -6224, -6225, -6226, -6227, -6228, -6229, -6230, -6231, -6232, -6233, -6234, -6235, -6236, -6237, -6238, -6239, -6240, -6241, -6242, -6243, -6244, -6245, -6246, -6247, -6248, -6249, -6250, -6251, -6252, -6253, -6254, -6255, -6256, -6257, -6258, -6259, -6260, -6261, -6262, -6263, -6264, -6265, -6266, -6267, -6268, -6269, -6270, -6271, -6272, -6273, -6274, -6275, -6276, -6277, -6278, -6279, -6280, -6281, -6282, -6283, -6284, -6285, -6301, -6302, -6303, -6304, -6305, -6306, -6307, -6308, -6309, -6310, -6311, -6312, -6313, -6314, -6315, -6316, -6317, -6318, -6319, -6320, -6321, -6322, -6323, -6324, -6325, -6326, -6327, -6328, -6329, -6330, -6331, -6332, -7000, -7001, -7002, -7003, -7004, -7005, -7006, -7007, -7010, -7011, -7012, -7013, -7014, -7015, -7021, -7022, -7024, -7025, -7026, -7027, -7029, -7030, -7031, -7032, -7033, -7034, -7035, -7036, -7037, -7038, -7039, -7040, -7041, -7100, -7101, -7102, -7103, -7104, -7105, -7106, -7107, -7108, -7109, -7110, -7111, -7112, -7113, -7114, -7115, -7116, -7117, -7118, -7119, -7120, -7121, -7122, -7123, -7124, -7201, -7202, -7203, -7204, -7205, -7206, -7207, -7208, -7209, -7210, -7211, -7212, -7213, -7214, -7215, -7216, -7217, -7218, -7219, -7220, -7221, -7222, -7223, -7224, -7225, -7226, -7227, -7228, -7229, -7230, -7231, -7232, -7233, -7234, -7235, -7236, -7237, -7238, -7239, -7240, -7241, -7242, -7243, -7244, -7246, -7247, -7248, -7249, -7250, -7251, -7252, -7253, -7254, -7255, -7256, -7257, -7258, -7259, -7260, -7261, -7262, -7263, -7264, -7265, -7266, -7267, -7268, -7269, -7270, -7271, -7272, -7273, -7274, -7275, -7276, -7277, -7278, -7279, -7280, -7281, -7282, -7283, -7284, -7285, -7286, -7287, -7288, -7289, -7290, -7291, -7292, -7293, -7294, -7295, -7296, -7297, -7298, -7299, -7300, -7301, -7302, -7402, -7403, -7404, -7405, -7406, -7407, -7408, -7409, -7410, -7411, -7412, -7413, -7414, -7415, -7416, -7417, -7418, -7419, -7420, -7421, -7422, -7423, -7424, -7425, -7426, -7427, -7428, -7429, -7430, -7431, -7432, -7433, -7434, -7435, -7600, -7601, -7602, -7603, -7604, -8001, -8002, -8003, -8004, -8005, -9001, -9002, -9003, -9004, -9005, -9006, -9007, -9008, -9009, -9010, -9011, -9012, -9013, -9014, -9015, -9016, -9017, -9018, -9019, -9020, -9022, -9023, -9024, -9025, -9026, -9027, -9028, -9029, -9030, -9031, -9032, -9033, -9034, -9035, -9036, -9037, -9038, -9039, -9040, -9041, -9042, -9043, -9044, -9045, -9046, -9047, -9048, -9049, -9050, -9051, -9052, -9053, -9054, -9057, -9058, -9059, -9060, -9061, -9062, -9063, -9064, -9065, -9066, -9069, -9070, -9071, -9072, -9073, -9074, -9075, -9076, -9077, -9078, -9079, -9080, -9081, -9082, -9083, -9084, -9085, -9086, -9087, -9088, -9089, -9090, -9091, -9092, -9093, -9094, -9095, -9096, -9097, -9098, -9099, -9100, -9101, -9102, -9103, -9104, -9105, -9106, -9107, -9108, -9109, -9110, -9111, -9112, -9113, -9114, -9115, -9116, -9117, -9118, -9119, -9120, -9121, -9122, -9123, -9124, -9125, -9126, -9127, -9200, -9201, -9202, -9203, -9501, -9502, -9503, -9504, -9505, -9506, -9507, -9508, -9509, -9510, -9512, -9513, -9514, -9515, -9516, -9518, -9519, -9520, -9521, -9522, -9523, -9524, -9525, -9526, -9527, -9528, -9529, -9530, -9531, -9532, -9533, -9534, -9535, -9536, -9537, -9538, -9539, -9540, -9541, -9542, -9543, -9544, -9545, -9546, -9547, -9548, -9549, -9550, -9551, -9552, -9553, -9554, -9555, -9556, -9557, -9558, -9559, -9560, -9561, -9562, -9563, -9564, -9565, -9566, -9567, -9568, -9569, -9570, -9571, -9572, -9573, -9574, -9575, -9576, -9577, -9578, -9579, -9580, -9581, -9582, -9583, -9584, -9585, -9586, -9587, -9588, -9589, -9590, -9591, -9592, -9593, -9594, -9595, -9596, -9597, -9598, -9599, -9600, -9601, -9602, -9603, -9604, -9605, -9606, -9607, -9608, -9609, -9610, -9611, -9612, -9613, -9614, -9615, -9616, -9617, -9618, -9619, -9620, -9621, -9622, -9623, -9624, -9625, -9626, -9627, -9628, -9629, -9630, -9631, -9632, -9633, -9634, -9635, -9636, -9637, -9638, -9639, -9640, -9641, -9642, -9643, -9644, -9645, -9646, -9647, -9648, -9649, -9650, -9651, -9652, -9653, -9654, -9655, -9656, -9657, -9658, -9659, -9660, -9661, -9662, -9663, -9664, -9665, -9666, -9667, -9668, -9669, -9670, -9671, -9672, -9673, -9674, -9675, -9676, -9677, -9678, -9679, -9680, -9681, -9682, -9683, -9684, -9685, -9686, -9687, -9688, -9689, -9690, -9691, -9692, -9693, -9694, -9695, -9696, -9697, -9698, -9699, -9700, -9701, -9702, -9703, -9704, -9705, -9706, -9707, -9708, -9709, -9710, -9711, -9712, -9713, -9714, -9715, -9716, -9717, -9718, -9719, -9720, -9721, -9722, -9723, -9724, -9725, -9726, -9727, -9728, -9729, -9730, -9731, -9732, -9733, -9734, -9735, -9736, -9737, -9738, -9739, -9740, -9741, -9742, -9743, -9744, -9745, -9746, -9747, -9748, -9749, -9750, -9751, -9752, -9753, -9754, -9755, -9756, -9757, -9758, -9759, -9760, -9761, -9762, -9763, -9764, -9765, -9766, -9767, -9768, -9769, -9770, -9771, -9772, -9773, -9774, -9775, -9776, -9777, -9778, -9779, -9780, -9781, -9782, -9783, -10500, -10501, -10502, -10503, -10504, -10505, -10506, -10507, -10508, -10509, -10510, -10511, -10512, -10513, -10514, -10515, -10516, -10650, -11000, -11001, -11002, -11003, -11004, -11005, -11006, -11007, -11008, -11009, -11010, -11011, -11012, -11013, -11014, -11015, -11016, -11017, -11018, -11019, -11020, -11021, -11022, -11023, -11024, -11025, -11026, -11027, -11028, -11029, -11030, -11031, -11032, -11033, -11034, -11035, -11036, -11037, -11038, -11039, -11040, -11041, -11042, -11043, -11044, -11045, -11046, -11047, -11048, -11049, -11050, -11051, -11052, -20000, -21000, -22998, -30926, -32491, -38104, -38105}; +int g_all_ob_errnos[2328] = {0, -4000, -4001, -4002, -4003, -4004, -4005, -4006, -4007, -4008, -4009, -4010, -4011, -4012, -4013, -4014, -4015, -4016, -4017, -4018, -4019, -4020, -4021, -4022, -4023, -4024, -4025, -4026, -4027, -4028, -4029, -4030, -4031, -4032, -4033, -4034, -4035, -4036, -4037, -4038, -4039, -4041, -4042, -4043, -4044, -4045, -4046, -4047, -4048, -4049, -4050, -4051, -4052, -4053, -4054, -4055, -4057, -4058, -4060, -4061, -4062, -4063, -4064, -4065, -4066, -4067, -4068, -4070, -4071, -4072, -4073, -4074, -4075, -4076, -4077, -4078, -4080, -4081, -4084, -4085, -4090, -4097, -4098, -4099, -4100, -4101, -4102, -4103, -4104, -4105, -4106, -4107, -4108, -4109, -4110, -4111, -4112, -4113, -4114, -4115, -4116, -4117, -4118, -4119, -4120, -4121, -4122, -4123, -4124, -4125, -4126, -4127, -4128, -4133, -4138, -4139, -4142, -4143, -4144, -4146, -4147, -4149, -4150, -4151, -4152, -4153, -4154, -4155, -4156, -4157, -4158, -4159, -4160, -4161, -4162, -4163, -4164, -4165, -4166, -4167, -4168, -4169, -4170, -4171, -4172, -4173, -4174, -4175, -4176, -4177, -4178, -4179, -4180, -4181, -4182, -4183, -4184, -4185, -4186, -4187, -4188, -4189, -4190, -4191, -4192, -4200, -4201, -4204, -4205, -4206, -4207, -4208, -4209, -4210, -4211, -4212, -4213, -4214, -4215, -4216, -4217, -4218, -4219, -4220, -4221, -4222, -4223, -4224, -4225, -4226, -4227, -4228, -4229, -4230, -4231, -4232, -4233, -4234, -4235, -4236, -4237, -4238, -4239, -4240, -4241, -4242, -4243, -4244, -4245, -4246, -4247, -4248, -4249, -4250, -4251, -4252, -4253, -4254, -4255, -4256, -4257, -4258, -4260, -4261, -4262, -4263, -4264, -4265, -4266, -4267, -4268, -4269, -4270, -4271, -4273, -4274, -4275, -4276, -4277, -4278, -4279, -4280, -4281, -4282, -4283, -4284, -4285, -4286, -4287, -4288, -4289, -4290, -4291, -4292, -4293, -4294, -4295, -4296, -4297, -4298, -4299, -4300, -4301, -4302, -4303, -4304, -4305, -4306, -4307, -4308, -4309, -4310, -4311, -4312, -4313, -4314, -4315, -4316, -4317, -4318, -4319, -4320, -4321, -4322, -4323, -4324, -4325, -4326, -4327, -4328, -4329, -4330, -4331, -4332, -4333, -4334, -4335, -4336, -4337, -4338, -4339, -4340, -4341, -4342, -4343, -4344, -4345, -4346, -4347, -4348, -4349, -4350, -4351, -4352, -4353, -4354, -4355, -4356, -4357, -4358, -4359, -4360, -4361, -4362, -4363, -4364, -4365, -4366, -4367, -4368, -4369, -4370, -4371, -4372, -4373, -4374, -4375, -4376, -4377, -4378, -4379, -4380, -4381, -4382, -4383, -4385, -4386, -4387, -4388, -4389, -4390, -4391, -4392, -4393, -4394, -4395, -4396, -4397, -4398, -4399, -4400, -4401, -4402, -4403, -4505, -4507, -4510, -4512, -4515, -4517, -4518, -4519, -4523, -4524, -4525, -4526, -4527, -4528, -4529, -4530, -4531, -4532, -4533, -4537, -4538, -4539, -4540, -4541, -4542, -4543, -4544, -4545, -4546, -4547, -4548, -4549, -4550, -4551, -4552, -4553, -4554, -4600, -4601, -4602, -4603, -4604, -4605, -4606, -4607, -4608, -4609, -4610, -4611, -4613, -4614, -4615, -4620, -4621, -4622, -4623, -4624, -4625, -4626, -4628, -4629, -4630, -4631, -4632, -4633, -4634, -4636, -4637, -4638, -4639, -4640, -4641, -4642, -4643, -4644, -4645, -4646, -4647, -4648, -4649, -4650, -4651, -4652, -4653, -4654, -4655, -4656, -4657, -4658, -4659, -4660, -4661, -4662, -4663, -4664, -4665, -4666, -4667, -4668, -4669, -4670, -4671, -4672, -4673, -4674, -4675, -4676, -4677, -4678, -4679, -4680, -4681, -4682, -4683, -4684, -4685, -4686, -4687, -4688, -4689, -4690, -4691, -4692, -4693, -4694, -4695, -4696, -4697, -4698, -4699, -4700, -4701, -4702, -4703, -4704, -4705, -4706, -4707, -4708, -4709, -4710, -4711, -4712, -4713, -4714, -4715, -4716, -4717, -4718, -4719, -4720, -4721, -4722, -4723, -4724, -4725, -4726, -4727, -4728, -4729, -4730, -4731, -4732, -4733, -4734, -4735, -4736, -4737, -4738, -4739, -4740, -4741, -4742, -4743, -4744, -4745, -4746, -4747, -4748, -4749, -4750, -4751, -4752, -4753, -4754, -4755, -4756, -4757, -4758, -4759, -4760, -4761, -4762, -4763, -4764, -4765, -4766, -4767, -4768, -4769, -4770, -4771, -4772, -4773, -4774, -4775, -4776, -4777, -4778, -4779, -4780, -4781, -4782, -4783, -4784, -5000, -5001, -5002, -5003, -5006, -5007, -5008, -5010, -5011, -5012, -5014, -5015, -5016, -5017, -5018, -5019, -5020, -5022, -5023, -5024, -5025, -5026, -5027, -5028, -5029, -5030, -5031, -5032, -5034, -5035, -5036, -5037, -5038, -5039, -5040, -5041, -5042, -5043, -5044, -5046, -5047, -5050, -5051, -5052, -5053, -5054, -5055, -5056, -5057, -5058, -5059, -5061, -5063, -5064, -5065, -5066, -5067, -5068, -5069, -5070, -5071, -5072, -5073, -5074, -5080, -5081, -5083, -5084, -5085, -5086, -5087, -5088, -5089, -5090, -5091, -5092, -5093, -5094, -5095, -5096, -5097, -5098, -5099, -5100, -5101, -5102, -5103, -5104, -5105, -5106, -5107, -5108, -5109, -5110, -5111, -5112, -5113, -5114, -5115, -5116, -5117, -5118, -5119, -5120, -5121, -5122, -5123, -5124, -5125, -5130, -5131, -5133, -5134, -5135, -5136, -5137, -5138, -5139, -5140, -5142, -5143, -5144, -5145, -5146, -5147, -5148, -5149, -5150, -5151, -5153, -5154, -5155, -5156, -5157, -5158, -5159, -5160, -5161, -5162, -5163, -5164, -5165, -5166, -5167, -5168, -5169, -5170, -5171, -5172, -5173, -5174, -5175, -5176, -5177, -5178, -5179, -5180, -5181, -5182, -5183, -5184, -5185, -5187, -5188, -5189, -5190, -5191, -5192, -5193, -5194, -5195, -5196, -5197, -5198, -5199, -5200, -5201, -5202, -5203, -5204, -5205, -5206, -5207, -5208, -5209, -5210, -5211, -5212, -5213, -5214, -5215, -5216, -5217, -5218, -5219, -5220, -5221, -5222, -5223, -5224, -5225, -5226, -5227, -5228, -5229, -5230, -5231, -5233, -5234, -5235, -5236, -5237, -5238, -5239, -5240, -5241, -5242, -5243, -5244, -5245, -5246, -5247, -5248, -5249, -5250, -5251, -5252, -5253, -5254, -5255, -5256, -5257, -5258, -5259, -5260, -5261, -5262, -5263, -5264, -5265, -5266, -5267, -5268, -5269, -5270, -5271, -5272, -5273, -5274, -5275, -5276, -5277, -5278, -5279, -5280, -5281, -5282, -5283, -5284, -5285, -5286, -5287, -5288, -5289, -5290, -5291, -5292, -5293, -5294, -5295, -5296, -5297, -5298, -5299, -5300, -5301, -5302, -5303, -5304, -5305, -5306, -5307, -5308, -5309, -5310, -5311, -5312, -5313, -5314, -5315, -5316, -5317, -5318, -5319, -5320, -5321, -5322, -5323, -5324, -5325, -5326, -5327, -5328, -5329, -5330, -5331, -5332, -5333, -5334, -5335, -5336, -5337, -5338, -5339, -5340, -5341, -5342, -5343, -5344, -5345, -5346, -5347, -5348, -5349, -5350, -5351, -5352, -5353, -5354, -5355, -5356, -5357, -5358, -5359, -5360, -5361, -5362, -5363, -5364, -5365, -5366, -5367, -5368, -5369, -5370, -5371, -5372, -5373, -5374, -5375, -5376, -5377, -5378, -5379, -5380, -5381, -5382, -5383, -5384, -5385, -5386, -5387, -5388, -5389, -5390, -5400, -5401, -5402, -5403, -5404, -5405, -5406, -5407, -5408, -5409, -5410, -5411, -5412, -5413, -5414, -5415, -5416, -5417, -5418, -5419, -5420, -5421, -5422, -5423, -5424, -5425, -5426, -5427, -5428, -5429, -5430, -5431, -5432, -5433, -5434, -5435, -5436, -5437, -5438, -5439, -5440, -5441, -5442, -5443, -5444, -5445, -5446, -5447, -5448, -5449, -5450, -5451, -5452, -5453, -5454, -5455, -5456, -5457, -5458, -5459, -5460, -5461, -5462, -5463, -5464, -5465, -5466, -5467, -5468, -5469, -5470, -5471, -5472, -5473, -5474, -5475, -5476, -5477, -5478, -5479, -5480, -5481, -5482, -5483, -5484, -5485, -5486, -5487, -5488, -5489, -5490, -5491, -5492, -5493, -5494, -5495, -5496, -5497, -5498, -5499, -5500, -5501, -5502, -5503, -5504, -5505, -5506, -5507, -5508, -5509, -5510, -5511, -5512, -5513, -5514, -5515, -5516, -5517, -5518, -5519, -5520, -5521, -5522, -5540, -5541, -5542, -5543, -5544, -5545, -5546, -5547, -5548, -5549, -5550, -5551, -5552, -5553, -5554, -5555, -5556, -5557, -5558, -5559, -5560, -5561, -5562, -5563, -5564, -5565, -5566, -5567, -5568, -5569, -5570, -5571, -5572, -5573, -5574, -5575, -5576, -5577, -5578, -5579, -5580, -5581, -5582, -5583, -5584, -5585, -5586, -5587, -5588, -5589, -5590, -5591, -5592, -5593, -5594, -5595, -5596, -5597, -5598, -5599, -5600, -5601, -5602, -5603, -5604, -5605, -5607, -5608, -5609, -5610, -5611, -5612, -5613, -5614, -5615, -5616, -5617, -5618, -5619, -5620, -5621, -5622, -5623, -5624, -5625, -5626, -5627, -5628, -5629, -5630, -5631, -5632, -5633, -5634, -5635, -5636, -5637, -5638, -5639, -5640, -5641, -5642, -5643, -5644, -5645, -5646, -5647, -5648, -5649, -5650, -5651, -5652, -5653, -5654, -5655, -5656, -5657, -5658, -5659, -5660, -5661, -5662, -5663, -5664, -5665, -5666, -5667, -5668, -5671, -5672, -5673, -5674, -5675, -5676, -5677, -5678, -5679, -5680, -5681, -5682, -5683, -5684, -5685, -5686, -5687, -5688, -5689, -5690, -5691, -5692, -5693, -5694, -5695, -5696, -5697, -5698, -5699, -5700, -5701, -5702, -5703, -5704, -5705, -5706, -5707, -5708, -5709, -5710, -5711, -5712, -5713, -5714, -5715, -5716, -5717, -5718, -5719, -5720, -5721, -5722, -5723, -5724, -5725, -5726, -5727, -5728, -5729, -5730, -5731, -5732, -5733, -5734, -5735, -5736, -5737, -5738, -5739, -5740, -5741, -5742, -5743, -5744, -5745, -5746, -5747, -5748, -5749, -5750, -5751, -5752, -5753, -5754, -5755, -5756, -5757, -5758, -5759, -5760, -5761, -5762, -5763, -5764, -5765, -5766, -5768, -5769, -5770, -5771, -5772, -5773, -5774, -5777, -5778, -5779, -5780, -5781, -5785, -5786, -5787, -5788, -5789, -5790, -5791, -5792, -5793, -5794, -5795, -5796, -5797, -5798, -5799, -5800, -5801, -5802, -5803, -5804, -5805, -5806, -5807, -5808, -5809, -5810, -5811, -5812, -5813, -5814, -5815, -5816, -5817, -5818, -5819, -5820, -5821, -5822, -5823, -5824, -5825, -5826, -5827, -5828, -5829, -5830, -5831, -5832, -5833, -5834, -5835, -5836, -5837, -5838, -5839, -5840, -5841, -5842, -5843, -5844, -5845, -5846, -5847, -5848, -5849, -5850, -5851, -5852, -5853, -5854, -5855, -5856, -5857, -5858, -5859, -5860, -5861, -5862, -5863, -5864, -5865, -5866, -5867, -5868, -5869, -5870, -5871, -5872, -5873, -5874, -5875, -5876, -5877, -5878, -5879, -5880, -5881, -5882, -5883, -5884, -5885, -5886, -5887, -5888, -5889, -5890, -5891, -5892, -5893, -5894, -5895, -5896, -5897, -5898, -5899, -5900, -5901, -5902, -5903, -5904, -5905, -5906, -5907, -5908, -5909, -5910, -5911, -5912, -5913, -5914, -5915, -5916, -5917, -5918, -5919, -5920, -5921, -5922, -5923, -5924, -5925, -5926, -5927, -5928, -5929, -5930, -5931, -5932, -5933, -5934, -5935, -5936, -5937, -5938, -5939, -5940, -5941, -5942, -5943, -5944, -5945, -5946, -5947, -5948, -5949, -5950, -5951, -5952, -5953, -5954, -5955, -5956, -5957, -5958, -5959, -5960, -5961, -5962, -5963, -5964, -5965, -5966, -5967, -5968, -5969, -5970, -5971, -5972, -5973, -5974, -5975, -5976, -5977, -5978, -5979, -5980, -5981, -5982, -5983, -5984, -5985, -5986, -5987, -5988, -5989, -5990, -5991, -5992, -5993, -5994, -5995, -5996, -5997, -5998, -5999, -6000, -6001, -6002, -6003, -6004, -6005, -6006, -6201, -6202, -6203, -6204, -6205, -6206, -6207, -6208, -6209, -6210, -6211, -6212, -6213, -6214, -6215, -6219, -6220, -6221, -6222, -6223, -6224, -6225, -6226, -6227, -6228, -6229, -6230, -6231, -6232, -6233, -6234, -6235, -6236, -6237, -6238, -6239, -6240, -6241, -6242, -6243, -6244, -6245, -6246, -6247, -6248, -6249, -6250, -6251, -6252, -6253, -6254, -6255, -6256, -6257, -6258, -6259, -6260, -6261, -6262, -6263, -6264, -6265, -6266, -6267, -6268, -6269, -6270, -6271, -6272, -6273, -6274, -6275, -6276, -6277, -6278, -6279, -6280, -6281, -6282, -6283, -6284, -6285, -6301, -6302, -6303, -6304, -6305, -6306, -6307, -6308, -6309, -6310, -6311, -6312, -6313, -6314, -6315, -6316, -6317, -6318, -6319, -6320, -6321, -6322, -6323, -6324, -6325, -6326, -6327, -6328, -6329, -6330, -6331, -6332, -7000, -7001, -7002, -7003, -7004, -7005, -7006, -7007, -7010, -7011, -7012, -7013, -7014, -7015, -7021, -7022, -7024, -7025, -7026, -7027, -7029, -7030, -7031, -7032, -7033, -7034, -7035, -7036, -7037, -7038, -7039, -7040, -7041, -7100, -7101, -7102, -7103, -7104, -7105, -7106, -7107, -7108, -7109, -7110, -7111, -7112, -7113, -7114, -7115, -7116, -7117, -7118, -7119, -7120, -7121, -7122, -7123, -7124, -7201, -7202, -7203, -7204, -7205, -7206, -7207, -7208, -7209, -7210, -7211, -7212, -7213, -7214, -7215, -7216, -7217, -7218, -7219, -7220, -7221, -7222, -7223, -7224, -7225, -7226, -7227, -7228, -7229, -7230, -7231, -7232, -7233, -7234, -7235, -7236, -7237, -7238, -7239, -7240, -7241, -7242, -7243, -7244, -7246, -7247, -7248, -7249, -7250, -7251, -7252, -7253, -7254, -7255, -7256, -7257, -7258, -7259, -7260, -7261, -7262, -7263, -7264, -7265, -7266, -7267, -7268, -7269, -7270, -7271, -7272, -7273, -7274, -7275, -7276, -7277, -7278, -7279, -7280, -7281, -7282, -7283, -7284, -7285, -7286, -7287, -7288, -7289, -7290, -7291, -7292, -7293, -7294, -7295, -7296, -7297, -7298, -7299, -7300, -7301, -7302, -7402, -7403, -7404, -7405, -7406, -7407, -7408, -7409, -7410, -7411, -7412, -7413, -7414, -7415, -7416, -7417, -7418, -7419, -7420, -7421, -7422, -7423, -7424, -7425, -7426, -7427, -7428, -7429, -7430, -7431, -7432, -7433, -7434, -7435, -7600, -7601, -7602, -7603, -7604, -8001, -8002, -8003, -8004, -8005, -9001, -9002, -9003, -9004, -9005, -9006, -9007, -9008, -9009, -9010, -9011, -9012, -9013, -9014, -9015, -9016, -9017, -9018, -9019, -9020, -9022, -9023, -9024, -9025, -9026, -9027, -9028, -9029, -9030, -9031, -9032, -9033, -9034, -9035, -9036, -9037, -9038, -9039, -9040, -9041, -9042, -9043, -9044, -9045, -9046, -9047, -9048, -9049, -9050, -9051, -9052, -9053, -9054, -9057, -9058, -9059, -9060, -9061, -9062, -9063, -9064, -9065, -9066, -9069, -9070, -9071, -9072, -9073, -9074, -9075, -9076, -9077, -9078, -9079, -9080, -9081, -9082, -9083, -9084, -9085, -9086, -9087, -9088, -9089, -9090, -9091, -9092, -9093, -9094, -9095, -9096, -9097, -9098, -9099, -9100, -9101, -9102, -9103, -9104, -9105, -9106, -9107, -9108, -9109, -9110, -9111, -9112, -9113, -9114, -9115, -9116, -9117, -9118, -9119, -9120, -9121, -9122, -9123, -9124, -9125, -9126, -9127, -9200, -9201, -9202, -9203, -9501, -9502, -9503, -9504, -9505, -9506, -9507, -9508, -9509, -9510, -9512, -9513, -9514, -9515, -9516, -9518, -9519, -9520, -9521, -9522, -9523, -9524, -9525, -9526, -9527, -9528, -9529, -9530, -9531, -9532, -9533, -9534, -9535, -9536, -9537, -9538, -9539, -9540, -9541, -9542, -9543, -9544, -9545, -9546, -9547, -9548, -9549, -9550, -9551, -9552, -9553, -9554, -9555, -9556, -9557, -9558, -9559, -9560, -9561, -9562, -9563, -9564, -9565, -9566, -9567, -9568, -9569, -9570, -9571, -9572, -9573, -9574, -9575, -9576, -9577, -9578, -9579, -9580, -9581, -9582, -9583, -9584, -9585, -9586, -9587, -9588, -9589, -9590, -9591, -9592, -9593, -9594, -9595, -9596, -9597, -9598, -9599, -9600, -9601, -9602, -9603, -9604, -9605, -9606, -9607, -9608, -9609, -9610, -9611, -9612, -9613, -9614, -9615, -9616, -9617, -9618, -9619, -9620, -9621, -9622, -9623, -9624, -9625, -9626, -9627, -9628, -9629, -9630, -9631, -9632, -9633, -9634, -9635, -9636, -9637, -9638, -9639, -9640, -9641, -9642, -9643, -9644, -9645, -9646, -9647, -9648, -9649, -9650, -9651, -9652, -9653, -9654, -9655, -9656, -9657, -9658, -9659, -9660, -9661, -9662, -9663, -9664, -9665, -9666, -9667, -9668, -9669, -9670, -9671, -9672, -9673, -9674, -9675, -9676, -9677, -9678, -9679, -9680, -9681, -9682, -9683, -9684, -9685, -9686, -9687, -9688, -9689, -9690, -9691, -9692, -9693, -9694, -9695, -9696, -9697, -9698, -9699, -9700, -9701, -9702, -9703, -9704, -9705, -9706, -9707, -9708, -9709, -9710, -9711, -9712, -9713, -9714, -9715, -9716, -9717, -9718, -9719, -9720, -9721, -9722, -9723, -9724, -9725, -9726, -9727, -9728, -9729, -9730, -9731, -9732, -9733, -9734, -9735, -9736, -9737, -9738, -9739, -9740, -9741, -9742, -9743, -9744, -9745, -9746, -9747, -9748, -9749, -9750, -9751, -9752, -9753, -9754, -9755, -9756, -9757, -9758, -9759, -9760, -9761, -9762, -9763, -9764, -9765, -9766, -9767, -9768, -9769, -9770, -9771, -9772, -9773, -9774, -9775, -9776, -9777, -9778, -9779, -9780, -9781, -9782, -9783, -9784, -10500, -10501, -10502, -10503, -10504, -10505, -10506, -10507, -10508, -10509, -10510, -10511, -10512, -10513, -10514, -10515, -10516, -10650, -11000, -11001, -11002, -11003, -11004, -11005, -11006, -11007, -11008, -11009, -11010, -11011, -11012, -11013, -11014, -11015, -11016, -11017, -11018, -11019, -11020, -11021, -11022, -11023, -11024, -11025, -11026, -11027, -11028, -11029, -11030, -11031, -11032, -11033, -11034, -11035, -11036, -11037, -11038, -11039, -11040, -11041, -11042, -11043, -11044, -11045, -11046, -11047, -11048, -11049, -11050, -11051, -11052, -20000, -21000, -22998, -30926, -32491, -38104, -38105}; const char *ob_error_name(const int err) { const char *ret = "Unknown error"; diff --git a/src/share/ob_errno.def b/src/share/ob_errno.def index 8d49e6fec..e7f14b4ac 100755 --- a/src/share/ob_errno.def +++ b/src/share/ob_errno.def @@ -2645,6 +2645,8 @@ DEFINE_ERROR(OB_ERR_EVENT_RECURSION_FORBIDDEN, -9780, ER_EVENT_RECURSION_FORBIDD DEFINE_ORACLE_ERROR(OB_NO_PARTITION_FOR_GIVEN_VALUE_SCHEMA_ERROR, -9781, ER_NO_PARTITION_FOR_GIVEN_VALUE, "HY000", "Table has no partition for value", 14400, "inserted partition key does not map to any partition"); DEFINE_ORACLE_ERROR(OB_ERR_INVALID_CHARACTER, -9782, -1, "HY000", "invalid character", 911, "invalid character"); DEFINE_ORACLE_ERROR_EXT(OB_ERR_MVIEW_CAN_NOT_ON_QUERY_COMPUTE, -9783, -1, "HY000", "cannot ENABLE ON QUERY COMPUTATION for the materialized view", "cannot ENABLE ON QUERY COMPUTATION for the materialized view `%s`.`%s`", 32361, "cannot ENABLE ON QUERY COMPUTATION for the materialized view", "cannot ENABLE ON QUERY COMPUTATION for the materialized view %s.%s"); +DEFINE_PLS_ERROR_EXT(OB_ERR_CURSOR_ATTR_APPLY, -9784, -1, "HY000", "cursor attribute may not be applied to non-cursor", "cursor attribute may not be applied to non-cursor %.*s", 324, "cursor attribute may not be applied to non-cursor", "cursor attribute may not be applied to non-cursor %.*s"); + // 余留位置 //////////////////////////////////////////////////////////////// // PL/SQL错误码值域 [-9500, -10000) diff --git a/src/share/ob_errno.h b/src/share/ob_errno.h index c618a49a6..f295b0e7a 100755 --- a/src/share/ob_errno.h +++ b/src/share/ob_errno.h @@ -1869,6 +1869,7 @@ constexpr int OB_ERR_EVENT_RECURSION_FORBIDDEN = -9780; constexpr int OB_NO_PARTITION_FOR_GIVEN_VALUE_SCHEMA_ERROR = -9781; constexpr int OB_ERR_INVALID_CHARACTER = -9782; constexpr int OB_ERR_MVIEW_CAN_NOT_ON_QUERY_COMPUTE = -9783; +constexpr int OB_ERR_CURSOR_ATTR_APPLY = -9784; constexpr int OB_ERR_KV_GLOBAL_INDEX_ROUTE = -10500; constexpr int OB_TTL_NOT_ENABLE = -10501; constexpr int OB_TTL_COLUMN_NOT_EXIST = -10502; @@ -4173,6 +4174,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_NO_PARTITION_FOR_GIVEN_VALUE_SCHEMA_ERROR__USER_ERROR_MSG "Table has no partition for value" #define OB_ERR_INVALID_CHARACTER__USER_ERROR_MSG "invalid character" #define OB_ERR_MVIEW_CAN_NOT_ON_QUERY_COMPUTE__USER_ERROR_MSG "cannot ENABLE ON QUERY COMPUTATION for the materialized view `%s`.`%s`" +#define OB_ERR_CURSOR_ATTR_APPLY__USER_ERROR_MSG "cursor attribute may not be applied to non-cursor %.*s" #define OB_ERR_KV_GLOBAL_INDEX_ROUTE__USER_ERROR_MSG "incorrect route for obkv global index, client router should refresh." #define OB_TTL_NOT_ENABLE__USER_ERROR_MSG "TTL feature is not enabled" #define OB_TTL_COLUMN_NOT_EXIST__USER_ERROR_MSG "TTL column '%.*s' not exists" @@ -8753,6 +8755,8 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_INVALID_CHARACTER__OBE_USER_ERROR_MSG "OBE-00911: invalid character" #define OB_ERR_MVIEW_CAN_NOT_ON_QUERY_COMPUTE__ORA_USER_ERROR_MSG "ORA-32361: cannot ENABLE ON QUERY COMPUTATION for the materialized view %s.%s" #define OB_ERR_MVIEW_CAN_NOT_ON_QUERY_COMPUTE__OBE_USER_ERROR_MSG "OBE-32361: cannot ENABLE ON QUERY COMPUTATION for the materialized view %s.%s" +#define OB_ERR_CURSOR_ATTR_APPLY__ORA_USER_ERROR_MSG "PLS-00324: cursor attribute may not be applied to non-cursor %.*s" +#define OB_ERR_CURSOR_ATTR_APPLY__OBE_USER_ERROR_MSG "PLS-00324: cursor attribute may not be applied to non-cursor %.*s" #define OB_ERR_KV_GLOBAL_INDEX_ROUTE__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -10500, incorrect route for obkv global index, client router should refresh." #define OB_ERR_KV_GLOBAL_INDEX_ROUTE__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -10500, incorrect route for obkv global index, client router should refresh." #define OB_TTL_NOT_ENABLE__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -10501, TTL feature is not enabled" @@ -8914,7 +8918,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_INVALID_DATE_MSG_FMT_V2__ORA_USER_ERROR_MSG "ORA-01861: Incorrect datetime value for column '%.*s' at row %ld" #define OB_ERR_INVALID_DATE_MSG_FMT_V2__OBE_USER_ERROR_MSG "OBE-01861: Incorrect datetime value for column '%.*s' at row %ld" -extern int g_all_ob_errnos[2327]; +extern int g_all_ob_errnos[2328]; const char *ob_error_name(const int oberr); const char* ob_error_cause(const int oberr); diff --git a/src/sql/engine/expr/ob_expr_pl_get_cursor_attr.cpp b/src/sql/engine/expr/ob_expr_pl_get_cursor_attr.cpp index 1f7db7f64..b963b839a 100644 --- a/src/sql/engine/expr/ob_expr_pl_get_cursor_attr.cpp +++ b/src/sql/engine/expr/ob_expr_pl_get_cursor_attr.cpp @@ -211,6 +211,12 @@ int ObExprPLGetCursorAttr::calc_pl_get_cursor_attr( if (info->pl_cursor_info_.is_explicit_cursor()) { CK (ObExtendType == datum_meta.type_); OX (datum->to_obj(obj, obj_meta)); + if (OB_SUCC(ret) + && obj.get_meta().get_extend_type() != pl::PL_CURSOR_TYPE + && obj.get_meta().get_extend_type() != pl::PL_REF_CURSOR_TYPE) { + ret = OB_ERR_CURSOR_ATTR_APPLY; + LOG_WARN("cursor attribute may not applied to non-cursor", K(ret), K(obj.get_meta())); + } OX (cursor = reinterpret_cast(obj.get_ext())); } else { if (OB_ISNULL(cursor = session->get_pl_implicit_cursor())) { diff --git a/src/sql/session/ob_sql_session_info.cpp b/src/sql/session/ob_sql_session_info.cpp index a9d167a11..9ce80b3e8 100644 --- a/src/sql/session/ob_sql_session_info.cpp +++ b/src/sql/session/ob_sql_session_info.cpp @@ -705,14 +705,16 @@ void ObSQLSessionInfo::destroy(bool skip_sys_var) } //close all cursor - if (OB_SUCC(ret) && pl_cursor_cache_.is_inited()) { - if (OB_FAIL(pl_cursor_cache_.close_all(*this))) { + if (pl_cursor_cache_.is_inited()) { + int temp_ret = pl_cursor_cache_.close_all(*this); + if (temp_ret != OB_SUCCESS) { LOG_WARN("failed to close all cursor", K(ret)); } } - if (OB_SUCC(ret) && NULL != piece_cache_) { - if (OB_FAIL(piece_cache_->close_all(*this))) { + if (NULL != piece_cache_) { + int temp_ret = piece_cache_->close_all(*this); + if (temp_ret != OB_SUCCESS) { LOG_WARN("failed to close all piece", K(ret)); } piece_cache_->~ObPieceCache(); diff --git a/src/sql/session/ob_sql_session_info.h b/src/sql/session/ob_sql_session_info.h index 3bb79ed50..2ae47887c 100644 --- a/src/sql/session/ob_sql_session_info.h +++ b/src/sql/session/ob_sql_session_info.h @@ -639,37 +639,47 @@ public: } else { /*do nothing*/ } return ret; } + int close_all(sql::ObSQLSessionInfo &session) { int ret = OB_SUCCESS; - common::ObSEArray cursor_ids; ObSessionStatEstGuard guard(session.get_effective_tenant_id(), session.get_sessid()); - for (CursorMap::iterator iter = pl_cursor_map_.begin(); //ignore ret - iter != pl_cursor_map_.end(); - ++iter) { - pl::ObPLCursorInfo *cursor_info = iter->second; - if (OB_ISNULL(cursor_info)) { + while (pl_cursor_map_.size() > 0) { // ignore error, just log, try to close all cursor in this loop. + int ret = OB_SUCCESS; + CursorMap::iterator iter = pl_cursor_map_.begin(); + pl::ObPLCursorInfo *cursor = NULL; + if (iter == pl_cursor_map_.end()) { ret = OB_ERR_UNEXPECTED; - SQL_ENG_LOG(WARN, "cursor info is NULL", K(cursor_info), K(ret)); + SQL_ENG_LOG(ERROR, "unexpected hashmap iter", K(ret)); + break; + } else if (OB_ISNULL(cursor = iter->second)) { + ret = OB_ERR_UNEXPECTED; + SQL_ENG_LOG(WARN, "unexpected nullptr cursor info", K(ret), K(iter->first)); + if (OB_FAIL(pl_cursor_map_.erase_refactored(iter->first))) { + SQL_ENG_LOG(ERROR, "failed to erase hash map", K(ret), K(iter->first)); + break; + } + } else if (OB_FAIL(session.close_cursor(cursor->get_id()))) { + SQL_ENG_LOG(WARN, "failed to close session cursor", K(ret), K(cursor->get_id())); } else { - cursor_ids.push_back(cursor_info->get_id()); + SQL_ENG_LOG(INFO, "clsoe session cursor implicit successed!", K(cursor->get_id())); } } - for (int64_t i = 0; i < cursor_ids.count() && OB_SUCC(ret); i++) { - uint64_t cursor_id = cursor_ids.at(i); - if (OB_FAIL(session.close_cursor(cursor_id))) { - SQL_ENG_LOG(WARN, "failed to close cursor", - K(cursor_id), K(session.get_sessid()), K(ret)); - } else { - SQL_ENG_LOG(INFO, "NOTICE: cursor is closed unexpectedly", - K(cursor_id), K(session.get_sessid()), K(ret)); - } + if (pl_cursor_map_.size() > 0) { + ret = OB_ERR_UNEXPECTED; + SQL_ENG_LOG(ERROR, "failed to close all cursor", K(ret), K(pl_cursor_map_.size())); } return ret; } + inline bool is_inited() const { return NULL != mem_context_; } void reset() { + int ret = OB_SUCCESS; + if (pl_cursor_map_.size() != 0) { + ret = OB_ERR_UNEXPECTED; + SQL_ENG_LOG(ERROR, "session cursor map not empty, cursor leaked", K(pl_cursor_map_.size())); + } pl_cursor_map_.reuse(); next_cursor_id_ = 1LL << 31; if (NULL != mem_context_) { From c11ef76f19b0e18455039cced7814f0e20b5b8e3 Mon Sep 17 00:00:00 2001 From: cqliang1995 Date: Tue, 27 Aug 2024 14:09:55 +0000 Subject: [PATCH 238/249] [FEAT MERGE]ODPS patch to master Co-authored-by: dontknow9179 <545187809@qq.com> --- cmake/Env.cmake | 6 + deps/init/oceanbase.el7.aarch64.deps | 2 +- deps/init/oceanbase.el7.x86_64.deps | 2 +- deps/init/oceanbase.el8.aarch64.deps | 2 +- deps/init/oceanbase.el8.x86_64.deps | 3 +- deps/init/oceanbase.el9.aarch64.deps | 2 +- deps/init/oceanbase.el9.x86_64.deps | 2 +- deps/oblib/src/CMakeLists.txt | 26 +- deps/oblib/src/lib/charset/ob_charset.cpp | 2 +- deps/oblib/src/lib/ob_define.h | 2 + deps/oblib/src/lib/ob_name_def.h | 1 + deps/oblib/unittest/lib/CMakeLists.txt | 2 +- .../ob_external_table_file_mgr.cpp | 137 +- .../ob_external_table_file_mgr.h | 9 +- .../ob_external_table_utils.cpp | 214 +- .../external_table/ob_external_table_utils.h | 17 +- src/share/schema/ob_schema_printer.cpp | 43 +- src/share/schema/ob_schema_retrieve_utils.ipp | 2 + src/share/schema/ob_schema_service.cpp | 2 + src/share/schema/ob_table_schema.cpp | 1 - src/share/schema/ob_table_sql_service.cpp | 3 + src/sql/CMakeLists.txt | 21 +- .../code_generator/ob_static_engine_cg.cpp | 16 + src/sql/code_generator/ob_tsc_cg_service.cpp | 8 +- src/sql/engine/basic/ob_select_into_op.cpp | 1986 ++++++++++--- src/sql/engine/basic/ob_select_into_op.h | 238 +- src/sql/engine/cmd/ob_load_data_parser.cpp | 314 ++- src/sql/engine/cmd/ob_load_data_parser.h | 51 + src/sql/engine/cmd/ob_table_executor.cpp | 10 +- src/sql/engine/expr/ob_datum_cast.cpp | 291 ++ src/sql/engine/expr/ob_datum_cast.h | 39 + src/sql/engine/px/ob_dfo.cpp | 1 + src/sql/engine/px/ob_dfo.h | 6 +- src/sql/engine/px/ob_dfo_mgr.cpp | 13 + src/sql/engine/px/ob_dfo_scheduler.cpp | 2 +- src/sql/engine/px/ob_granule_pump.cpp | 36 +- src/sql/engine/px/ob_granule_pump.h | 14 + src/sql/engine/px/ob_granule_util.cpp | 57 +- src/sql/engine/px/ob_px_sub_coord.cpp | 14 +- src/sql/engine/px/ob_px_util.cpp | 1 - .../ob_external_table_access_service.cpp | 42 +- .../engine/table/ob_odps_table_row_iter.cpp | 2454 +++++++++++++++++ src/sql/engine/table/ob_odps_table_row_iter.h | 286 ++ src/sql/ob_sql_utils.cpp | 47 +- src/sql/ob_sql_utils.h | 2 + src/sql/optimizer/ob_insert_log_plan.cpp | 128 +- src/sql/optimizer/ob_insert_log_plan.h | 4 + src/sql/optimizer/ob_log_plan.cpp | 51 +- src/sql/optimizer/ob_log_plan.h | 45 +- src/sql/optimizer/ob_log_select_into.cpp | 2 + src/sql/optimizer/ob_log_select_into.h | 60 +- .../parser/non_reserved_keywords_mysql_mode.c | 10 + src/sql/parser/sql_parser_mysql_mode.y | 150 +- .../plan_cache/ob_sql_parameterization.cpp | 2 + .../resolver/ddl/ob_alter_table_resolver.cpp | 1 + .../resolver/ddl/ob_create_table_resolver.cpp | 17 +- .../ddl/ob_create_table_resolver_base.cpp | 8 +- src/sql/resolver/ddl/ob_ddl_resolver.cpp | 159 +- src/sql/resolver/ddl/ob_ddl_resolver.h | 1 + .../resolver/dml/ob_default_value_utils.cpp | 15 +- src/sql/resolver/dml/ob_del_upd_resolver.cpp | 51 +- src/sql/resolver/dml/ob_del_upd_resolver.h | 3 + src/sql/resolver/dml/ob_dml_resolver.cpp | 95 +- src/sql/resolver/dml/ob_dml_stmt.cpp | 2 + src/sql/resolver/dml/ob_dml_stmt.h | 9 +- src/sql/resolver/dml/ob_insert_resolver.cpp | 36 +- src/sql/resolver/dml/ob_insert_resolver.h | 1 + src/sql/resolver/dml/ob_select_resolver.cpp | 94 +- src/sql/resolver/dml/ob_select_resolver.h | 3 +- src/sql/resolver/dml/ob_select_stmt.cpp | 37 +- src/sql/resolver/dml/ob_select_stmt.h | 15 +- src/sql/resolver/ob_resolver.cpp | 3 +- src/sql/resolver/ob_resolver_utils.cpp | 86 +- src/sql/resolver/ob_resolver_utils.h | 11 + unittest/sql/engine/dml/CMakeLists.txt | 1 + unittest/sql/engine/dml/test_odps.cpp | 98 + unittest/sql/test_sql_utils.h | 2 +- 77 files changed, 6933 insertions(+), 696 deletions(-) create mode 100644 src/sql/engine/table/ob_odps_table_row_iter.cpp create mode 100644 src/sql/engine/table/ob_odps_table_row_iter.h create mode 100644 unittest/sql/engine/dml/test_odps.cpp diff --git a/cmake/Env.cmake b/cmake/Env.cmake index 1fc0c0197..8a8ebc36a 100644 --- a/cmake/Env.cmake +++ b/cmake/Env.cmake @@ -117,6 +117,8 @@ if(OB_BUILD_CLOSE_MODULES) ob_define(OB_BUILD_ORACLE_PL ON) # dblink ob_define(OB_BUILD_DBLINK ON) + # odps + ob_define(OB_BUILD_CPP_ODPS ON) # 仲裁功能 ob_define(OB_BUILD_ARBITRATION ON) @@ -172,6 +174,10 @@ if(OB_BUILD_DBLINK) add_definitions(-DOB_BUILD_DBLINK) endif() +if(OB_BUILD_CPP_ODPS) + add_definitions(-DOB_BUILD_CPP_ODPS) +endif() + # should not use initial-exec for tls-model if building OBCDC. if(NOT OB_BUILD_CDC) add_definitions(-DENABLE_INITIAL_EXEC_TLS_MODEL) diff --git a/deps/init/oceanbase.el7.aarch64.deps b/deps/init/oceanbase.el7.aarch64.deps index 01601d065..51f5062ed 100644 --- a/deps/init/oceanbase.el7.aarch64.deps +++ b/deps/init/oceanbase.el7.aarch64.deps @@ -33,7 +33,7 @@ devdeps-s3-cpp-sdk-1.11.156-102023122011.el7.aarch64.rpm devdeps-protobuf-c-1.4.1-100000072023102410.el7.aarch64.rpm devdeps-roaringbitmap-croaring-3.0.0-42024042816.el7.aarch64.rpm devdeps-apache-arrow-9.0.0-302024052920.el7.aarch64.rpm -devdeps-apache-orc-1.8.3-202024072510.el7.aarch64.rpm +# devdeps-apache-orc-1.8.3-202024072510.el7.aarch64.rpm [tools] obdevtools-binutils-2.30-12022100413.el7.aarch64.rpm diff --git a/deps/init/oceanbase.el7.x86_64.deps b/deps/init/oceanbase.el7.x86_64.deps index 1f3f350de..7d3496ff8 100644 --- a/deps/init/oceanbase.el7.x86_64.deps +++ b/deps/init/oceanbase.el7.x86_64.deps @@ -36,7 +36,7 @@ devdeps-s3-cpp-sdk-1.11.156-102023122011.el7.x86_64.rpm devdeps-protobuf-c-1.4.1-100000062023102016.el7.x86_64.rpm devdeps-roaringbitmap-croaring-3.0.0-42024042816.el7.x86_64.rpm devdeps-apache-arrow-9.0.0-222024052223.el7.x86_64.rpm -devdeps-apache-orc-1.8.3-202024072510.el7.x86_64.rpm +# devdeps-apache-orc-1.8.3-202024072510.el7.x86_64.rpm [tools] obdevtools-binutils-2.30-12022100413.el7.x86_64.rpm diff --git a/deps/init/oceanbase.el8.aarch64.deps b/deps/init/oceanbase.el8.aarch64.deps index 7f32f9b7f..5943a48eb 100644 --- a/deps/init/oceanbase.el8.aarch64.deps +++ b/deps/init/oceanbase.el8.aarch64.deps @@ -33,7 +33,7 @@ devdeps-s3-cpp-sdk-1.11.156-102023122011.el8.aarch64.rpm devdeps-protobuf-c-1.4.1-100000072023102410.el8.aarch64.rpm devdeps-roaringbitmap-croaring-3.0.0-42024042816.el8.aarch64.rpm devdeps-apache-arrow-9.0.0-322024052923.el8.aarch64.rpm -devdeps-apache-orc-1.8.3-202024072510.el8.aarch64.rpm +# devdeps-apache-orc-1.8.3-202024072510.el8.aarch64.rpm [tools] obdevtools-binutils-2.30-12022100413.el8.aarch64.rpm diff --git a/deps/init/oceanbase.el8.x86_64.deps b/deps/init/oceanbase.el8.x86_64.deps index 0373426cd..54692f896 100644 --- a/deps/init/oceanbase.el8.x86_64.deps +++ b/deps/init/oceanbase.el8.x86_64.deps @@ -35,7 +35,8 @@ devdeps-s3-cpp-sdk-1.11.156-102023122011.el8.x86_64.rpm devdeps-protobuf-c-1.4.1-100000062023102016.el8.x86_64.rpm devdeps-roaringbitmap-croaring-3.0.0-42024042816.el8.x86_64.rpm devdeps-apache-arrow-9.0.0-172024052218.el8.x86_64.rpm -devdeps-apache-orc-1.8.3-202024072510.el8.x86_64.rpm +devdeps-odps-cpp-sdk-1.0.0-482024080517.el8.x86_64.rpm +# devdeps-apache-orc-1.8.3-202024072510.el8.x86_64.rpm [tools] obdevtools-binutils-2.30-12022100413.el8.x86_64.rpm diff --git a/deps/init/oceanbase.el9.aarch64.deps b/deps/init/oceanbase.el9.aarch64.deps index dc3f98ec1..7ab33f270 100644 --- a/deps/init/oceanbase.el9.aarch64.deps +++ b/deps/init/oceanbase.el9.aarch64.deps @@ -37,7 +37,7 @@ devdeps-s3-cpp-sdk-1.11.156-102023122011.el8.aarch64.rpm devdeps-protobuf-c-1.4.1-100000072023102410.el8.aarch64.rpm devdeps-roaringbitmap-croaring-3.0.0-42024042816.el8.aarch64.rpm devdeps-apache-arrow-9.0.0-322024052923.el8.aarch64.rpm -devdeps-apache-orc-1.8.3-202024072510.el8.aarch64.rpm +# devdeps-apache-orc-1.8.3-202024072510.el8.aarch64.rpm [deps-el9] devdeps-apr-1.6.5-232023090616.el9.aarch64.rpm target=el9 diff --git a/deps/init/oceanbase.el9.x86_64.deps b/deps/init/oceanbase.el9.x86_64.deps index e049cb055..b3a986afd 100644 --- a/deps/init/oceanbase.el9.x86_64.deps +++ b/deps/init/oceanbase.el9.x86_64.deps @@ -39,7 +39,7 @@ devdeps-s3-cpp-sdk-1.11.156-102023122011.el8.x86_64.rpm devdeps-protobuf-c-1.4.1-100000062023102016.el8.x86_64.rpm devdeps-apache-arrow-9.0.0-172024052218.el8.x86_64.rpm devdeps-roaringbitmap-croaring-3.0.0-42024042816.el8.x86_64.rpm -devdeps-apache-orc-1.8.3-202024072510.el8.x86_64.rpm +# devdeps-apache-orc-1.8.3-202024072510.el8.x86_64.rpm [deps-el9] devdeps-apr-1.6.5-232023090616.el9.x86_64.rpm target=el9 diff --git a/deps/oblib/src/CMakeLists.txt b/deps/oblib/src/CMakeLists.txt index 93e6171aa..fc85c7200 100644 --- a/deps/oblib/src/CMakeLists.txt +++ b/deps/oblib/src/CMakeLists.txt @@ -21,7 +21,7 @@ target_include_directories( ${DEP_DIR}/include/apr-1/ ${DEP_DIR}/include/icu/common ${DEP_DIR}/include/apache-arrow - ${DEP_DIR}/include/apache-orc + # ${DEP_DIR}/include/apache-orc ${USSL_INCLUDE_DIRS} ) @@ -209,12 +209,12 @@ target_link_libraries(oblib_base_base_base ${DEP_DIR}/lib64/libarrow.a ${DEP_DIR}/lib64/libparquet.a ${DEP_DIR}/lib64/libarrow_bundled_dependencies.a - ${DEP_DIR}/lib64/liborc.a - ${DEP_DIR}/lib64/libsnappy.a - ${DEP_DIR}/lib64/libprotoc.a - ${DEP_DIR}/lib64/libprotobuf.a - ${DEP_DIR}/lib64/liblz4.a - ${DEP_DIR}/lib64/libzstd.a + # ${DEP_DIR}/lib64/liborc.a + # ${DEP_DIR}/lib64/libsnappy.a + # ${DEP_DIR}/lib64/libprotoc.a + # ${DEP_DIR}/lib64/libprotobuf.a + # ${DEP_DIR}/lib64/liblz4.a + # ${DEP_DIR}/lib64/libzstd.a -L${DEP_DIR}/var/usr/lib64 -L${DEP_DIR}/var/usr/lib -L${DEP_3RD_DIR}/usr/lib @@ -244,12 +244,12 @@ target_link_libraries(oblib_base_base_base ${DEP_DIR}/lib64/libarrow.a ${DEP_DIR}/lib64/libparquet.a ${DEP_DIR}/lib64/libarrow_bundled_dependencies.a - ${DEP_DIR}/lib64/liborc.a - ${DEP_DIR}/lib64/libsnappy.a - ${DEP_DIR}/lib64/libprotoc.a - ${DEP_DIR}/lib64/libprotobuf.a - ${DEP_DIR}/lib64/liblz4.a - ${DEP_DIR}/lib64/libzstd.a + # ${DEP_DIR}/lib64/liborc.a + # ${DEP_DIR}/lib64/libsnappy.a + # ${DEP_DIR}/lib64/libprotoc.a + # ${DEP_DIR}/lib64/libprotobuf.a + # ${DEP_DIR}/lib64/liblz4.a + # ${DEP_DIR}/lib64/libzstd.a -L${DEP_DIR}/var/usr/lib64 -L${DEP_DIR}/var/usr/lib -L${DEP_3RD_DIR}/usr/lib diff --git a/deps/oblib/src/lib/charset/ob_charset.cpp b/deps/oblib/src/lib/charset/ob_charset.cpp index fbe4b2cf8..261bb0d7e 100644 --- a/deps/oblib/src/lib/charset/ob_charset.cpp +++ b/deps/oblib/src/lib/charset/ob_charset.cpp @@ -3337,7 +3337,7 @@ int ObCharset::charset_convert(ObIAllocator &alloc, char *res_buf = static_cast(alloc.alloc(res_buf_len)); if (OB_ISNULL(res_buf)) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("alloc memory failed", K(ret)); + LOG_WARN("alloc memory failed", K(ret), K(lbt())); } else { if (OB_SUCC(charset_convert(src_cs_type, in.ptr(), in.length(), dst_cs_type, res_buf, res_buf_len, res_len))) { diff --git a/deps/oblib/src/lib/ob_define.h b/deps/oblib/src/lib/ob_define.h index cb5388f0b..6e16d74b7 100644 --- a/deps/oblib/src/lib/ob_define.h +++ b/deps/oblib/src/lib/ob_define.h @@ -372,6 +372,8 @@ const int64_t OB_MAX_PASSWORD_LENGTH = 128; const int64_t OB_MAX_PASSWORD_BUF_LENGTH = OB_MAX_PASSWORD_LENGTH + 1; // After each sha1 is 41 characters, the incremental backup is up to 64 times, and the maximum password required for recovery is 64*(41+1)=2,688 const int64_t OB_MAX_ENCRYPTED_PASSWORD_LENGTH = OB_MAX_PASSWORD_LENGTH * 4; +const int64_t OB_MAX_EXTERNAL_TABLE_PROPERTIES_ITEM_LENGTH = 128; +const int64_t OB_MAX_ENCRYPTED_EXTERNAL_TABLE_PROPERTIES_ITEM_LENGTH = OB_MAX_EXTERNAL_TABLE_PROPERTIES_ITEM_LENGTH * 4; const int64_t OB_MAX_PASSWORD_ARRAY_LENGTH = 4096; const int64_t OB_MAX_ERROR_MSG_LEN = 512; const int64_t OB_MAX_RESULT_MESSAGE_LENGTH = 1024; diff --git a/deps/oblib/src/lib/ob_name_def.h b/deps/oblib/src/lib/ob_name_def.h index 1ca13f8e7..1b07db2a9 100644 --- a/deps/oblib/src/lib/ob_name_def.h +++ b/deps/oblib/src/lib/ob_name_def.h @@ -1067,6 +1067,7 @@ #define N_ST_WITHIN "st_within" #define N_SQL_MODE_CONVERT "sql_mode_convert" #define N_EXTERNAL_FILE_COLUMN_PREFIX "metadata$filecol" +#define N_EXTERNAL_TABLE_COLUMN_PREFIX "external$tablecol" #define N_PARTITION_LIST_COL "metadata$partition_list_col" #define N_EXTERNAL_FILE_URL "metadata$fileurl" #define N_EXTERNAL_FILE_ROW "external$filerow" diff --git a/deps/oblib/unittest/lib/CMakeLists.txt b/deps/oblib/unittest/lib/CMakeLists.txt index a7bb13cba..2547f00e9 100644 --- a/deps/oblib/unittest/lib/CMakeLists.txt +++ b/deps/oblib/unittest/lib/CMakeLists.txt @@ -6,7 +6,7 @@ # oblib_addtest(time/test_ob_time_utility.cpp) # oblib_addtest(timezone/test_ob_timezone_utils.cpp) oblib_addtest(parquet/test_parquet.cpp) -oblib_addtest(orc/test_orc.cpp) +# oblib_addtest(orc/test_orc.cpp) oblib_addtest(alloc/test_alloc_struct.cpp) oblib_addtest(alloc/test_block_set.cpp) oblib_addtest(alloc/test_chunk_mgr.cpp) diff --git a/src/share/external_table/ob_external_table_file_mgr.cpp b/src/share/external_table/ob_external_table_file_mgr.cpp index be1f59f51..b52daaedc 100644 --- a/src/share/external_table/ob_external_table_file_mgr.cpp +++ b/src/share/external_table/ob_external_table_file_mgr.cpp @@ -246,7 +246,6 @@ int ObExternalTableFileManager::get_external_files_by_part_id( LOG_WARN("fail to fill cache from inner table", K(ret)); } } - for (int i = 0; OB_SUCC(ret) && i < ext_files->file_urls_.count(); ++i) { bool in_ranges = false; if (range_filter != NULL && OB_FAIL(ObExternalTableUtils::is_file_id_in_ranges(*range_filter, @@ -584,6 +583,83 @@ int ObExternalTableFileManager::get_all_partition_list_val(const ObTableSchema * return ret; } +int ObExternalTableFileManager::calculate_odps_part_val_by_part_spec(const ObTableSchema *table_schema, + const ObIArray &file_infos, + ObIArray &part_vals, + share::schema::ObSchemaGetterGuard &schema_guard, + ObExecContext &exec_ctx) +{ + int ret = OB_SUCCESS; + ObIAllocator &allocator = exec_ctx.get_allocator(); + CK (OB_NOT_NULL(table_schema) && OB_LIKELY(table_schema->is_odps_external_table())); + if (OB_SUCC(ret)) { + const common::ObPartitionKeyInfo &part_key_info = table_schema->get_partition_key_info(); + const int part_key_size = part_key_info.get_size(); + for (int64_t i = 0; OB_SUCC(ret) && i < file_infos.count(); i++) { + const ObString &all_part_spec = file_infos.at(i).file_url_; + ObSEArray part_spec_list; + if (OB_FAIL(ObSQLUtils::extract_odps_part_spec(all_part_spec, part_spec_list))) { + LOG_WARN("failed to extract odps part spec", K(ret), K(all_part_spec)); + } else if (part_spec_list.count() != part_key_size) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected count find part spec of odps", K(ret), K(file_infos), K(file_infos.count()), K(i), K(all_part_spec), K(part_spec_list.count()), K(part_key_size)); + } else { + ObNewRow odps_part_row; + ObObj *obj_array = nullptr; + if (OB_ISNULL(obj_array = static_cast(allocator.alloc(sizeof(ObObj) * part_key_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate memory", K(ret)); + } else { + for (ObObj *ptr = obj_array; OB_SUCC(ret) && ptr < obj_array + part_key_size; ++ptr) { + new(ptr)ObObj(); + } + odps_part_row.assign(obj_array, part_key_size); + } + for (int64_t j = 0; OB_SUCC(ret) && j < part_spec_list.count(); ++j) { + const ObRowkeyColumn *part_col = part_key_info.get_column(j); + ObObjType part_key_type = ObUnknownType; + ObString& part_spec = part_spec_list.at(j); + if (OB_ISNULL(part_col)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null ptr", K(ret), K(j)); + } else if (FALSE_IT(part_key_type = part_col->get_meta_type().get_type())) { + } else if (part_key_type == ObVarcharType || + part_key_type == ObCharType) { + odps_part_row.get_cell(j).set_meta_type(part_col->get_meta_type()); + odps_part_row.get_cell(j).set_collation_type(ObCharset::get_system_collation()); + odps_part_row.get_cell(j).set_varchar_value(part_spec.ptr(), part_spec.length()); + } else if (part_key_type == ObTinyIntType || + part_key_type == ObSmallIntType || + part_key_type == ObMediumIntType || + part_key_type == ObInt32Type || + part_key_type == ObIntType) { + int64_t val = 0; + for (int64_t k = 0; OB_SUCC(ret) && k < part_spec.length(); ++k) { + if (part_spec.ptr()[k] >= '0' && part_spec.ptr()[k] <= '9') { + val = val * 10 + part_spec.ptr()[k] - '0'; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected part_spec", K(ret), K(part_spec)); + } + } + if (OB_SUCC(ret)) { + odps_part_row.get_cell(j).set_meta_type(part_col->get_meta_type()); + odps_part_row.get_cell(j).set_int(val); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected part_key_type", K(part_key_type), K(j), K(ret)); + } + } + if (OB_SUCC(ret) && OB_FAIL(part_vals.push_back(odps_part_row))) { + LOG_WARN("failed push back odps_part_row", K(ret)); + } + } + } + } + return ret; +} + int ObExternalTableFileManager::calculate_file_part_val_by_file_name(const ObTableSchema *table_schema, const ObIArray &file_infos, ObIArray &part_vals, @@ -603,8 +679,11 @@ int ObExternalTableFileManager::calculate_file_part_val_by_file_name(const ObTab ObNewRow list_val; ObObj *obj_array = nullptr; if (file_name_row.get_count() > 0) { - file_name_row.get_cell(0).set_string(ObVarcharType, is_local_storage ? - file_infos.at(i).file_url_.after(ip_delimiter) : file_infos.at(i).file_url_); + file_name_row.get_cell(0).set_string(ObVarcharType, table_schema->is_odps_external_table() ? + file_infos.at(i).file_url_.after(equals_delimiter) : + (is_local_storage ? + file_infos.at(i).file_url_.after(ip_delimiter) : + file_infos.at(i).file_url_)); } else { ret = OB_ERR_UNEXPECTED; LOG_WARN("row cell count not expected", K(ret)); @@ -675,9 +754,16 @@ int ObExternalTableFileManager::calculate_all_files_partitions(share::schema::Ob ObArray existed_part_vals; ObArray existed_part_ids; ObArray file_part_vals; + bool is_odps_external_table = false; CK (OB_NOT_NULL(table_schema) && OB_LIKELY(table_schema->is_external_table())); OZ (get_all_partition_list_val(table_schema, existed_part_vals, existed_part_ids)); - OZ (calculate_file_part_val_by_file_name(table_schema, file_infos, file_part_vals, schema_guard, exec_ctx)); + OX(is_odps_external_table = table_schema->is_odps_external_table()); + + if (is_odps_external_table) { + OZ(calculate_odps_part_val_by_part_spec(table_schema, file_infos, file_part_vals, schema_guard, exec_ctx)); + } else { + OZ (calculate_file_part_val_by_file_name(table_schema, file_infos, file_part_vals, schema_guard, exec_ctx)); + } for (int64_t i = 0; OB_SUCC(ret) && i < file_part_vals.count(); i++) { int64_t idx = -1; OZ (find_partition_existed(existed_part_vals, file_part_vals.at(i), idx)); @@ -805,7 +891,7 @@ int ObExternalTableFileManager::update_inner_table_files_list_by_table( if (OB_SUCC(ret) && partitions_to_del.count() > 0) { ObAlterTableStmt *alter_table_stmt = NULL; - OZ (create_alter_table_stmt(exec_ctx, table_schema, database_schema, partitions_to_add.count(), ObAlterTableArg::DROP_PARTITION, alter_table_stmt)); + OZ (create_alter_table_stmt(exec_ctx, table_schema, database_schema, partitions_to_del.count(), ObAlterTableArg::DROP_PARTITION, alter_table_stmt)); if (OB_SUCC(ret)) { for (int64_t i = 0; OB_SUCC(ret) && i < partitions_to_del.count(); i++) { if (OB_ISNULL(partitions_to_del.at(i))) { @@ -858,11 +944,7 @@ int ObExternalTableFileManager::update_inner_table_files_list_by_table( int64_t mock_part_id = GET_EXT_MOCK_PART_ID(part_ids.at(i)); CHECK_EXT_MOCK_PART_ID_VALID(file_part_ids_added, mock_part_id); part_id = MAP_EXT_MOCK_PART_ID_TO_REAL_PART_ID(file_part_ids_added, mock_part_id); - } else { - part_id = part_ids.at(i); - } - if (OB_SUCC(ret)) { - OZ (add_item_to_map(exec_ctx.get_allocator(), part_id_to_file_urls, part_id, file_infos.at(i))); + part_ids.at(i) = part_id; } } #undef GET_EXT_MOCK_PART_ID @@ -870,6 +952,9 @@ int ObExternalTableFileManager::update_inner_table_files_list_by_table( } } + for (int64_t i = 0; OB_SUCC(ret) && i < part_ids.count(); i++) { + OZ (add_item_to_map(exec_ctx.get_allocator(), part_id_to_file_urls, part_ids.at(i), file_infos.at(i))); + } //OZ (get_part_id_to_file_urls_map(table_schema, database_schema, is_local_storage, file_urls, file_sizes, schema_guard, exec_ctx, trans, part_id_to_file_urls)); for (common::hash::ObHashMap *>::iterator it = part_id_to_file_urls.begin(); OB_SUCC(ret) && it != part_id_to_file_urls.end(); it++) { @@ -939,6 +1024,16 @@ int ObExternalTableFileManager::update_inner_table_files_list_by_part( int64_t insert_rows = 0; int64_t max_file_id = 0;// ObCSVTableRowIterator::MIN_EXTERNAL_TABLE_FILE_ID - 1 common::hash::ObHashMap hash_map; + share::schema::ObSchemaGetterGuard schema_guard; + const ObTableSchema *table_schema = NULL; + bool is_odps_external_table = false; + char file_url_buf[256] = { 0 }; + OZ (GCTX.schema_service_->get_tenant_schema_guard(tenant_id, schema_guard)); + OZ (schema_guard.get_table_schema(tenant_id, table_id, table_schema)); + CK (OB_NOT_NULL(table_schema)); + if (OB_SUCC(ret)) { + is_odps_external_table = table_schema->is_odps_external_table(); + } OZ(get_all_records_from_inner_table(allocator, tenant_id, table_id, partition_id, old_file_infos, old_file_ids)); OZ(hash_map.create(std::max(file_infos.count(), old_file_infos.count()) + 1, "ExternalFile")); for (int64_t i = 0; OB_SUCC(ret) && i < old_file_infos.count(); i++) { @@ -952,7 +1047,7 @@ int ObExternalTableFileManager::update_inner_table_files_list_by_part( if (ret == OB_HASH_NOT_EXIST) { ret = OB_SUCCESS; OZ(insert_file_infos.push_back(file_infos.at(i))); - OZ(insert_file_ids.push_back(++max_file_id)); + OZ(insert_file_ids.push_back(is_odps_external_table ? 0 : ++max_file_id)); // odps table's file_id is 0 } else if (ret == OB_SUCCESS) { OZ(update_file_infos.push_back(file_infos.at(i))); OZ(update_file_ids.push_back(file_id)); @@ -960,7 +1055,7 @@ int ObExternalTableFileManager::update_inner_table_files_list_by_part( } OZ(hash_map.reuse()); for (int64_t i = 0; OB_SUCC(ret) && i < file_infos.count(); i++) { - OZ(hash_map.set_refactored(file_infos.at(i).file_url_, 1)); + OZ(hash_map.set_refactored(file_infos.at(i).file_url_, is_odps_external_table ? 0 : 1)); // odps table's file_id is 0 } for (int64_t i = 0; OB_SUCC(ret) && i < old_file_infos.count(); i++) { int64_t existed = 0; @@ -997,10 +1092,13 @@ int ObExternalTableFileManager::update_inner_table_files_list_by_part( OZ(insert_sql.assign_fmt("INSERT INTO %s(TABLE_ID,PART_ID,FILE_ID,FILE_URL,CREATE_VERSION,DELETE_VERSION,FILE_SIZE) VALUES", OB_ALL_EXTERNAL_TABLE_FILE_TNAME)); for (int64_t i = 0; OB_SUCC(ret) && i < insert_file_infos.count(); i++) { + ObString file_url = insert_file_infos.at(i).file_url_; + int new_url_len = ObHexEscapeSqlStr(insert_file_infos.at(i).file_url_).to_string(file_url_buf, 256); + file_url.assign(file_url_buf, new_url_len); OZ(insert_sql.append_fmt("%c(%lu,%lu,%ld,'%.*s',%ld,%ld,%ld)", (0 == i) ? ' ' : ',', table_id, partition_id, insert_file_ids.at(i), - insert_file_infos.at(i).file_url_.length(), insert_file_infos.at(i).file_url_.ptr(), + file_url.length(), file_url.ptr(), cur_time, MAX_VERSION, insert_file_infos.at(i).file_size_)); } OZ(trans.write(tenant_id, insert_sql.ptr(), insert_rows)); @@ -1020,7 +1118,7 @@ int ObExternalTableFileManager::get_all_records_from_inner_table(ObIAllocator &a SMART_VAR(ObMySQLProxy::MySQLResult, res) { sqlclient::ObMySQLResult *result = NULL; ObSqlString sql; - OZ (sql.append_fmt("SELECT file_url, file_id FROM %s" + OZ (sql.append_fmt("SELECT file_url, file_id, file_size FROM %s" " WHERE table_id = %lu AND part_id = %lu", OB_ALL_EXTERNAL_TABLE_FILE_TNAME, table_id, partition_id)); OZ (GCTX.sql_proxy_->read(res, tenant_id, sql.ptr())); @@ -1031,13 +1129,17 @@ int ObExternalTableFileManager::get_all_records_from_inner_table(ObIAllocator &a } else { while (OB_SUCC(result->next())) { ObString file_url; - int64_t file_id; + int64_t file_id = 0; + int64_t file_size = 0; EXTRACT_VARCHAR_FIELD_MYSQL(*result, "file_url", file_url); EXTRACT_INT_FIELD_MYSQL(*result, "file_id", file_id, int64_t); + EXTRACT_INT_FIELD_MYSQL(*result, "file_size", file_size, int64_t); ObString tmp_url; OZ (ob_write_string(allocator, file_url, tmp_url)); ObExternalFileInfoTmp file_info; + file_info.part_id_ = partition_id; file_info.file_url_ = tmp_url; + file_info.file_size_ = file_size; OZ (file_urls.push_back(file_info)); OZ (file_ids.push_back(file_id)); } @@ -1272,7 +1374,10 @@ int ObExternalTableFileManager::refresh_external_table(const uint64_t tenant_id, table_schema->get_table_id(), table_schema->get_external_file_location(), table_schema->get_external_file_location_access_info(), - table_schema->get_external_file_pattern(), regexp_vars, exec_ctx.get_allocator(), + table_schema->get_external_file_pattern(), + table_schema->get_external_properties(), + table_schema->is_partitioned_table(), + regexp_vars, exec_ctx.get_allocator(), full_path, file_urls, file_sizes)); //TODO [External Table] opt performance diff --git a/src/share/external_table/ob_external_table_file_mgr.h b/src/share/external_table/ob_external_table_file_mgr.h index 12b492378..9c0e23f22 100644 --- a/src/share/external_table/ob_external_table_file_mgr.h +++ b/src/share/external_table/ob_external_table_file_mgr.h @@ -21,6 +21,7 @@ #include "sql/resolver/ob_resolver_utils.h" #include "sql/resolver/expr/ob_raw_expr_util.h" #include "src/sql/resolver/ob_stmt_resolver.h" +#include "sql/engine/table/ob_odps_table_row_iter.h" namespace oceanbase { namespace sql { @@ -104,6 +105,7 @@ public: const char* auto_refresh_job_name = "auto_refresh_external_table_job"; const char ip_delimiter = '%'; + const char equals_delimiter = '='; ObExternalTableFileManager() {} @@ -200,7 +202,7 @@ private: const uint64_t partition_id, const ObIArray &file_infos); - int update_inner_table_files_list_by_table( + int update_inner_table_files_list_by_table( sql::ObExecContext &exec_ctx, ObMySQLTransaction &trans, const uint64_t tenant_id, @@ -255,6 +257,11 @@ private: ObIArray &part_vals, share::schema::ObSchemaGetterGuard &schema_guard, ObExecContext &exec_ctx); + int calculate_odps_part_val_by_part_spec(const ObTableSchema *table_schema, + const ObIArray &file_infos, + ObIArray &part_vals, + share::schema::ObSchemaGetterGuard &schema_guard, + ObExecContext &exec_ctx); int find_partition_existed(ObIArray &existed_part, ObNewRow &file_part_val, int64_t &found); diff --git a/src/share/external_table/ob_external_table_utils.cpp b/src/share/external_table/ob_external_table_utils.cpp index 611cb71e7..f5a10a18e 100644 --- a/src/share/external_table/ob_external_table_utils.cpp +++ b/src/share/external_table/ob_external_table_utils.cpp @@ -74,6 +74,9 @@ int ObExternalTableUtils::is_file_id_in_ranges(const ObIArray &ran in_ranges = true; } } + if (0 == file_id) { + in_ranges = true; + } return ret; } @@ -100,7 +103,8 @@ int ObExternalTableUtils::resolve_file_id_range(const ObNewRange &range, start_file = ObCSVTableRowIterator::MIN_EXTERNAL_TABLE_FILE_ID; end_file = INT64_MAX; if (column_idx >= range.get_start_key().get_obj_cnt() || - column_idx >= range.get_end_key().get_obj_cnt() ) { + column_idx >= range.get_end_key().get_obj_cnt() || + column_idx < 0) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed. input column idx invalid", K(ret), K(range), K(column_idx)); } else { @@ -139,6 +143,25 @@ int ObExternalTableUtils::resolve_line_number_range(const ObNewRange &range, return ret; } +int ObExternalTableUtils::resolve_odps_start_step(const ObNewRange &range, + const int64_t &column_idx, + int64_t &start, + int64_t &step) +{ + int ret = OB_SUCCESS; + if (column_idx >= range.get_start_key().get_obj_cnt() || + column_idx >= range.get_end_key().get_obj_cnt() ) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed. input column idx invalid", K(ret), K(range), K(column_idx)); + } else { + const ObObj &start_obj = range.get_start_key().get_obj_ptr()[column_idx]; + const ObObj &end_obj = range.get_end_key().get_obj_ptr()[column_idx]; + start = start_obj.get_int(); + step = end_obj.get_int(); + } + return ret; +} + int ObExternalTableUtils::convert_external_table_new_range(const ObString &file_url, const int64_t file_id, const uint64_t part_id, @@ -237,7 +260,7 @@ int ObExternalTableUtils::make_external_table_scan_range(const common::ObString int ret = OB_SUCCESS; ObObj *obj_start = NULL; ObObj *obj_end = NULL; - if (OB_UNLIKELY(first_lineno > last_lineno)) { + if (OB_UNLIKELY(first_lineno > last_lineno && file_id != 0)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed. get invalid params", K(ret), K(first_lineno), K(last_lineno)); } else if (OB_ISNULL(obj_start = static_cast(allocator.alloc(sizeof(ObObj) * @@ -307,29 +330,54 @@ int ObExternalTableUtils::prepare_single_scan_range(const uint64_t tenant_id, } else { new_range.reset(); } - for (int64_t i = 0; OB_SUCC(ret) && i < tmp_ranges.count(); ++i) { - if (OB_ISNULL(tmp_ranges.at(i))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected NULL ptr", K(ret)); - } else { - for (int64_t j = 0; OB_SUCC(ret) && j < file_urls.count(); ++j) { - ObNewRange *range = NULL; - bool is_valid = false; - if (OB_ISNULL(range = OB_NEWx(ObNewRange, (&range_allocator)))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to new a ptr", K(ret)); - } else if (OB_FAIL(ObExternalTableUtils::convert_external_table_new_range( - file_urls.at(j).file_url_, - file_urls.at(j).file_id_, - file_urls.at(j).part_id_, - *tmp_ranges.at(i), - range_allocator, - *range, - is_valid))) { - LOG_WARN("failed to convert external table new range", K(ret), K(file_urls.at(j)), - K(ranges.at(i))); - } else if (is_valid) { - OZ (new_range.push_back(range)); + if (!file_urls.empty() && file_urls.at(0).file_id_ == 0) {// if file_id_ == 0 means, it's odps table + for (int64_t i = 0; OB_SUCC(ret) && i < file_urls.count(); ++i) { + const ObExternalFileInfo& external_info = file_urls.at(i); + ObNewRange *range = NULL; + if (0 != external_info.file_id_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected file id", K(ret), K(i), K(external_info.file_id_)); + } else if (OB_ISNULL(range = OB_NEWx(ObNewRange, (&range_allocator)))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to new a ptr", K(ret)); + } else if (OB_FAIL(ObExternalTableUtils::make_external_table_scan_range(external_info.file_url_, + external_info.file_id_, + external_info.part_id_, + 0, + INT64_MAX, + range_allocator, + *range))) { + LOG_WARN("failed to make external table scan range", K(ret)); + } else { + OZ (new_range.push_back(range)); + } + } + + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < tmp_ranges.count(); ++i) { + if (OB_ISNULL(tmp_ranges.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected NULL ptr", K(ret)); + } else { + for (int64_t j = 0; OB_SUCC(ret) && j < file_urls.count(); ++j) { + ObNewRange *range = NULL; + bool is_valid = false; + if (OB_ISNULL(range = OB_NEWx(ObNewRange, (&range_allocator)))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to new a ptr", K(ret)); + } else if (OB_FAIL(ObExternalTableUtils::convert_external_table_new_range( + file_urls.at(j).file_url_, + file_urls.at(j).file_id_, + file_urls.at(j).part_id_, + *tmp_ranges.at(i), + range_allocator, + *range, + is_valid))) { + LOG_WARN("failed to convert external table new range", K(ret), K(file_urls.at(j)), + K(ranges.at(i))); + } else if (is_valid) { + OZ (new_range.push_back(range)); + } } } } @@ -634,6 +682,8 @@ int ObExternalTableUtils::collect_external_file_list( const ObString &location, const ObString &access_info, const ObString &pattern, + const ObString &properties, + const bool &is_partitioned_table, const sql::ObExprRegexpSessionVariables ®exp_vars, ObIAllocator &allocator, ObSqlString &full_path, @@ -642,32 +692,102 @@ int ObExternalTableUtils::collect_external_file_list( { int ret = OB_SUCCESS; - ObSEArray all_servers; - share::schema::ObSchemaGetterGuard schema_guard; - OZ (GCTX.location_service_->external_table_get(tenant_id, table_id, all_servers)); - const bool is_local_storage = ObSQLUtils::is_external_files_on_local_disk(location); - if (OB_SUCC(ret) && full_path.length() > 0 - && *(full_path.ptr() + full_path.length() - 1) != '/' ) { - OZ (full_path.append("/")); - } - if (OB_FAIL(ret)) { - } else if (is_local_storage) { - OZ (collect_local_files_on_servers(tenant_id, location, pattern, regexp_vars, all_servers, file_urls, file_sizes, full_path, allocator)); - } else { - OZ (ObExternalTableFileManager::get_instance().get_external_file_list_on_device( - location, pattern, regexp_vars, file_urls, file_sizes, access_info, allocator)); - for (int64_t i = 0; OB_SUCC(ret) && i < file_urls.count(); i++) { - ObSqlString tmp_file_url; - ObString &file_url = file_urls.at(i); - OZ (tmp_file_url.append(full_path.string())); - OZ (tmp_file_url.append(file_urls.at(i))); - OZ (ob_write_string(allocator, tmp_file_url.string(), file_url)); + if (!properties.empty()) { +#ifdef OB_BUILD_CPP_ODPS + // Since each partition information of an ODPS table obtained by the ODPS driver is a string, + // OceanBase treat partition string as an external table filename, one file corresponds to one odps partition, + // the number of files corresponds to the number of partitions. + sql::ObODPSTableRowIterator odps_driver; + sql::ObExternalFileFormat ex_format; + ex_format.format_type_ = sql::ObExternalFileFormat::ODPS_FORMAT; + if (OB_FAIL(ex_format.load_from_string(properties, allocator))) { + LOG_WARN("failed to load from string", K(ret)); + } else if (OB_FAIL(odps_driver.init_tunnel(ex_format.odps_format_))) { + LOG_WARN("failed to init tunnel", K(ret)); + } else if (OB_FAIL(odps_driver.pull_partition_info())) { + LOG_WARN("failed to pull partition info", K(ret)); + } else if (odps_driver.is_part_table()) { + if (!is_partitioned_table) { + ret = OB_EXTERNAL_ODPS_UNEXPECTED_ERROR; + LOG_WARN("remote odps table is partitioned table, but local odps external table is not partitioned table", K(ret)); + LOG_USER_ERROR(OB_EXTERNAL_ODPS_UNEXPECTED_ERROR, "remote odps table is partitioned table, but local odps external table is not partitioned table"); + } + ObIArray& part_list_info = odps_driver.get_partition_info(); + for (int64_t i = 0; OB_SUCC(ret) && i < part_list_info.count(); ++i) { + const char *part_spec_src = part_list_info.at(i).name_.c_str(); + int64_t part_spec_src_len = STRLEN(part_spec_src); + char *part_spec = NULL; + if (OB_ISNULL(part_spec = reinterpret_cast(allocator.alloc(part_spec_src_len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc mem", K(part_spec_src_len), K(ret)); + } else { + MEMCPY(part_spec, part_spec_src, part_spec_src_len); + OZ(file_sizes.push_back(part_list_info.at(i).record_count_)); + OZ (file_urls.push_back(ObString(part_spec_src_len, part_spec))); + } + } + } else { + ObIArray& part_list_info = odps_driver.get_partition_info(); + if (is_partitioned_table) { + ret = OB_EXTERNAL_ODPS_UNEXPECTED_ERROR; + LOG_WARN("remote odps table is not partitioned table, but local odps external table is partitioned table", K(ret)); + LOG_USER_ERROR(OB_EXTERNAL_ODPS_UNEXPECTED_ERROR, "remote odps table is not partitioned table, but local odps external table is partitioned table"); + } else if (1 != part_list_info.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected count of partition info", K(ret), K(part_list_info.count())); + } + OZ(file_sizes.push_back(part_list_info.at(0).record_count_)); + OZ (file_urls.push_back(ObString(""))); + } +#else + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "external odps table"); + LOG_WARN("not support odps table in opensource", K(ret)); +#endif + } else if (location.empty()) { + //do nothing + } else { + ObSEArray all_servers; + share::schema::ObSchemaGetterGuard schema_guard; + OZ (GCTX.location_service_->external_table_get(tenant_id, table_id, all_servers)); + const bool is_local_storage = ObSQLUtils::is_external_files_on_local_disk(location); + if (OB_SUCC(ret) && full_path.length() > 0 + && *(full_path.ptr() + full_path.length() - 1) != '/' ) { + OZ (full_path.append("/")); + } + if (OB_FAIL(ret)) { + } else if (is_local_storage) { + OZ (collect_local_files_on_servers(tenant_id, location, pattern, regexp_vars, all_servers, file_urls, file_sizes, full_path, allocator)); + } else { + OZ (ObExternalTableFileManager::get_instance().get_external_file_list_on_device( + location, pattern, regexp_vars, file_urls, file_sizes, access_info, allocator)); + for (int64_t i = 0; OB_SUCC(ret) && i < file_urls.count(); i++) { + ObSqlString tmp_file_url; + ObString &file_url = file_urls.at(i); + OZ (tmp_file_url.append(full_path.string())); + OZ (tmp_file_url.append(file_urls.at(i))); + OZ (ob_write_string(allocator, tmp_file_url.string(), file_url)); + } } - } - OZ (ObExternalTableUtils::sort_external_files(file_urls, file_sizes)); + OZ (ObExternalTableUtils::sort_external_files(file_urls, file_sizes)); + } return ret; } +bool ObExternalTableUtils::is_skipped_insert_column(const schema::ObColumnSchemaV2& column) +{ + bool is_skip = false; + if (OB_HIDDEN_FILE_ID_COLUMN_ID == column.get_column_id() + || OB_HIDDEN_LINE_NUMBER_COLUMN_ID == column.get_column_id()) { + // 外表插入时不写隐藏列 + is_skip = true; + } else if (column.is_tbl_part_key_column()) { + // 外表插入时不写分区键的列 + is_skip = true; + } + return is_skip; +} + } // namespace share } // namespace oceanbase diff --git a/src/share/external_table/ob_external_table_utils.h b/src/share/external_table/ob_external_table_utils.h index 715973479..e80f17d7c 100644 --- a/src/share/external_table/ob_external_table_utils.h +++ b/src/share/external_table/ob_external_table_utils.h @@ -16,6 +16,7 @@ #include "lib/container/ob_iarray.h" #include "lib/string/ob_string.h" #include "lib/allocator/page_arena.h" +#include "src/share/schema/ob_column_schema.h" namespace oceanbase { @@ -76,6 +77,10 @@ class ObExternalTableUtils { const int64_t &column_idx, int64_t &start_lineno, int64_t &end_lineno); + static int resolve_odps_start_step(const common::ObNewRange &range, + const int64_t &column_idx, + int64_t &start, + int64_t &step); static int convert_external_table_new_range(const common::ObString &file_url, const int64_t file_id, const uint64_t ref_table_id, @@ -111,6 +116,8 @@ class ObExternalTableUtils { const ObString &location, const ObString &access_info, const ObString &pattern, + const ObString &properties, + const bool &is_partitioned_table, const sql::ObExprRegexpSessionVariables ®exp_vars, ObIAllocator &allocator, common::ObSqlString &full_path, @@ -127,11 +134,6 @@ class ObExternalTableUtils { ObIArray &file_sizes, common::ObSqlString &partition_path, ObIAllocator &allocator); - - private: - static bool is_left_edge(const common::ObObj &value); - static bool is_right_edge(const common::ObObj &value); - static int64_t get_edge_value(const common::ObObj &edge); static int make_external_table_scan_range(const common::ObString &file_url, const int64_t file_id, const uint64_t ref_table_id, @@ -139,7 +141,12 @@ class ObExternalTableUtils { const int64_t last_lineno, common::ObIAllocator &allocator, common::ObNewRange &new_range); + static bool is_skipped_insert_column(const schema::ObColumnSchemaV2& column); + private: + static bool is_left_edge(const common::ObObj &value); + static bool is_right_edge(const common::ObObj &value); + static int64_t get_edge_value(const common::ObObj &edge); static int sort_external_files(ObIArray &file_urls, ObIArray &file_sizes); diff --git a/src/share/schema/ob_schema_printer.cpp b/src/share/schema/ob_schema_printer.cpp index 6fdd8343b..4b9c7b886 100644 --- a/src/share/schema/ob_schema_printer.cpp +++ b/src/share/schema/ob_schema_printer.cpp @@ -5559,10 +5559,15 @@ int ObSchemaPrinter::print_external_table_file_info(const ObTableSchema &table_s // 1. print file location, pattern const ObString &location = table_schema.get_external_file_location(); const ObString &pattern = table_schema.get_external_file_pattern(); + const ObString &format_string = table_schema.get_external_file_format(); + const ObString &properties_string = table_schema.get_external_properties(); const bool user_specified = table_schema.is_user_specified_partition_for_external_table(); - if (OB_FAIL(databuff_printf(buf, buf_len, pos, "\nLOCATION='%.*s'", location.length(), location.ptr()))) { + bool is_odps_table = false; + if (OB_FAIL(ObSQLUtils::is_external_odps_table(properties_string, allocator, is_odps_table))) { + LOG_WARN("failed check is odps table or not", K(ret)); + } else if (!is_odps_table && OB_FAIL(databuff_printf(buf, buf_len, pos, "\nLOCATION='%.*s'", location.length(), location.ptr()))) { SHARE_SCHEMA_LOG(WARN, "fail to print LOCATION", K(ret)); - } else if (!pattern.empty() && OB_FAIL(databuff_printf(buf, buf_len, pos, "\nPATTERN='%.*s'", pattern.length(), pattern.ptr()))) { + } else if (!is_odps_table && !pattern.empty() && OB_FAIL(databuff_printf(buf, buf_len, pos, "\nPATTERN='%.*s'", pattern.length(), pattern.ptr()))) { SHARE_SCHEMA_LOG(WARN, "fail to print PATTERN", K(ret)); } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "\nAUTO_REFRESH = %s", table_schema.get_external_table_auto_refresh() == 0 ? "OFF" : table_schema.get_external_table_auto_refresh() == 1 ? "IMMEDIATE" : "INTERVAL"))) { @@ -5576,13 +5581,19 @@ int ObSchemaPrinter::print_external_table_file_info(const ObTableSchema &table_s // 2. print file format if (OB_SUCC(ret)) { ObExternalFileFormat format; - if (OB_FAIL(format.load_from_string(table_schema.get_external_file_format(), allocator))) { + const ObString &format_or_properties = is_odps_table ? properties_string : format_string; + if (format_or_properties.empty()) { + ret = OB_ERR_UNEXPECTED; + SHARE_SCHEMA_LOG(WARN, "format_or_properties is empty", K(ret)); + } else if (OB_FAIL(format.load_from_string(format_or_properties, allocator))) { SHARE_SCHEMA_LOG(WARN, "fail to load from json string", K(ret)); } else if (!(format.format_type_ > ObExternalFileFormat::INVALID_FORMAT && format.format_type_ < ObExternalFileFormat::MAX_FORMAT)) { ret = OB_NOT_SUPPORTED; SHARE_SCHEMA_LOG(WARN, "unsupported to print file format", K(ret), K(format.format_type_)); - } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "\nFORMAT (\n"))) { + } else if (!is_odps_table && OB_FAIL(databuff_printf(buf, buf_len, pos, "\nFORMAT (\n"))) { + SHARE_SCHEMA_LOG(WARN, "fail to print FORMAT (", K(ret)); + } else if (is_odps_table && OB_FAIL(databuff_printf(buf, buf_len, pos, "\nPROPERTIES (\n"))) { SHARE_SCHEMA_LOG(WARN, "fail to print FORMAT (", K(ret)); } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, " TYPE = '%s',", ObExternalFileFormat::FORMAT_TYPE_STR[format.format_type_]))) { SHARE_SCHEMA_LOG(WARN, "fail to print TYPE", K(ret)); @@ -5620,6 +5631,30 @@ int ObSchemaPrinter::print_external_table_file_info(const ObTableSchema &table_s databuff_printf(buf, buf_len, pos, "\n NULL_IF = (%.*s),", origin_format.origin_null_if_str_.length(), origin_format.origin_null_if_str_.ptr()))) { SHARE_SCHEMA_LOG(WARN, "fail to print NULL_IF", K(ret)); } + } else if (OB_SUCC(ret) && ObExternalFileFormat::ODPS_FORMAT == format.format_type_) { + const ObODPSGeneralFormat &odps = format.odps_format_; + ObString scret_str("********"); + if (OB_FAIL(databuff_printf(buf, buf_len, pos, "\n %s = '%.*s',", ObODPSGeneralFormat::OPTION_NAMES[0], odps.access_type_.length(), odps.access_type_.ptr()))) { + SHARE_SCHEMA_LOG(WARN, "fail to print ODPS_INFO", K(ret)); + } else if (!odps.access_id_.empty() && OB_FAIL(databuff_printf(buf, buf_len, pos, "\n %s = '%.*s',", ObODPSGeneralFormat::OPTION_NAMES[1], scret_str.length(), scret_str.ptr()))) { + SHARE_SCHEMA_LOG(WARN, "fail to print ODPS_INFO", K(ret)); + } else if (!odps.access_key_.empty() && OB_FAIL(databuff_printf(buf, buf_len, pos, "\n %s = '%.*s',", ObODPSGeneralFormat::OPTION_NAMES[2], scret_str.length(), scret_str.ptr()))) { + SHARE_SCHEMA_LOG(WARN, "fail to print ODPS_INFO", K(ret)); + } else if (!odps.sts_token_.empty() && OB_FAIL(databuff_printf(buf, buf_len, pos, "\n %s = '%.*s',", ObODPSGeneralFormat::OPTION_NAMES[3], scret_str.length(), scret_str.ptr()))) { + SHARE_SCHEMA_LOG(WARN, "fail to print ODPS_INFO", K(ret)); + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "\n %s = '%.*s',", ObODPSGeneralFormat::OPTION_NAMES[4], odps.endpoint_.length(), odps.endpoint_.ptr()))) { + SHARE_SCHEMA_LOG(WARN, "fail to print ODPS_INFO", K(ret)); + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "\n %s = '%.*s',", ObODPSGeneralFormat::OPTION_NAMES[5], odps.project_.length(), odps.project_.ptr()))) { + SHARE_SCHEMA_LOG(WARN, "fail to print ODPS_INFO", K(ret)); + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "\n %s = '%.*s',", ObODPSGeneralFormat::OPTION_NAMES[6], odps.schema_.length(), odps.schema_.ptr()))) { + SHARE_SCHEMA_LOG(WARN, "fail to print ODPS_INFO", K(ret)); + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "\n %s = '%.*s',", ObODPSGeneralFormat::OPTION_NAMES[7], odps.table_.length(), odps.table_.ptr()))) { + SHARE_SCHEMA_LOG(WARN, "fail to print ODPS_INFO", K(ret)); + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "\n %s = '%.*s',", ObODPSGeneralFormat::OPTION_NAMES[8], odps.quota_.length(), odps.quota_.ptr()))) { + SHARE_SCHEMA_LOG(WARN, "fail to print ODPS_INFO", K(ret)); + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "\n %s = '%.*s',", ObODPSGeneralFormat::OPTION_NAMES[9], odps.compression_code_.length(), odps.compression_code_.ptr()))) { + SHARE_SCHEMA_LOG(WARN, "fail to print ODPS_INFO", K(ret)); + } } if (OB_SUCC(ret)) { --pos; diff --git a/src/share/schema/ob_schema_retrieve_utils.ipp b/src/share/schema/ob_schema_retrieve_utils.ipp index e58539ea6..2e1a60af3 100644 --- a/src/share/schema/ob_schema_retrieve_utils.ipp +++ b/src/share/schema/ob_schema_retrieve_utils.ipp @@ -1503,6 +1503,8 @@ int ObSchemaRetrieveUtils::fill_table_schema( bool, true, true/*ignore_column_error*/, false); EXTRACT_INT_FIELD_TO_CLASS_MYSQL_WITH_DEFAULT_VALUE(result, auto_increment_cache_size, table_schema, int64_t, true, true, 0); + EXTRACT_VARCHAR_FIELD_TO_CLASS_MYSQL_WITH_DEFAULT_VALUE( + result, external_properties, table_schema, true/*skip null*/, true/*ignore column error*/, empty_str); if (OB_SUCC(ret) && table_schema.is_materialized_view()) { bool skip_null_error = true; bool skip_column_error = true; diff --git a/src/share/schema/ob_schema_service.cpp b/src/share/schema/ob_schema_service.cpp index 756159501..7ea59b53f 100644 --- a/src/share/schema/ob_schema_service.cpp +++ b/src/share/schema/ob_schema_service.cpp @@ -424,6 +424,8 @@ int AlterTableSchema::assign(const ObTableSchema &src_schema) LOG_WARN("deep copy external_file_format failed", K(ret)); } else if (OB_FAIL(deep_copy_str(src_schema.external_file_pattern_, external_file_pattern_))) { LOG_WARN("deep copy external_file_pattern failed", K(ret)); + } else if (OB_FAIL(deep_copy_str(src_schema.external_properties_, external_properties_))) { + LOG_WARN("deep copy external_properties failed", K(ret)); } //view schema diff --git a/src/share/schema/ob_table_schema.cpp b/src/share/schema/ob_table_schema.cpp index cbb8eb68c..6e61286a6 100644 --- a/src/share/schema/ob_table_schema.cpp +++ b/src/share/schema/ob_table_schema.cpp @@ -1529,7 +1529,6 @@ int ObTableSchema::assign(const ObTableSchema &src_schema) } else if (OB_FAIL(deep_copy_str(src_schema.external_properties_, external_properties_))) { LOG_WARN("deep copy external_properties failed", K(ret)); } - //view schema if (OB_SUCC(ret)) { view_schema_ = src_schema.view_schema_; diff --git a/src/share/schema/ob_table_sql_service.cpp b/src/share/schema/ob_table_sql_service.cpp index b72351697..ea60525aa 100644 --- a/src/share/schema/ob_table_sql_service.cpp +++ b/src/share/schema/ob_table_sql_service.cpp @@ -2972,6 +2972,7 @@ int ObTableSqlService::gen_table_dml( && (table.is_external_table() || !table.get_external_file_location().empty() || !table.get_external_file_format().empty() + || !table.get_external_properties().empty() || !table.get_external_file_location_access_info().empty() || !table.get_external_file_pattern().empty()))) { ret = OB_NOT_SUPPORTED; @@ -3152,6 +3153,8 @@ int ObTableSqlService::gen_table_dml( && OB_FAIL(dml.add_column("column_store", table.is_column_store_supported()))) || ((data_version >= DATA_VERSION_4_3_2_0 || (data_version < DATA_VERSION_4_3_0_0 && data_version >= MOCK_DATA_VERSION_4_2_3_0)) && OB_FAIL(dml.add_column("auto_increment_cache_size", table.get_auto_increment_cache_size()))) + || (data_version >= DATA_VERSION_4_3_2_1 && + OB_FAIL(dml.add_column("external_properties", ObHexEscapeSqlStr(table.get_external_properties())))) || (data_version >= DATA_VERSION_4_3_3_0 && OB_FAIL(dml.add_column("local_session_vars", ObHexEscapeSqlStr(local_session_var)))) ) { diff --git a/src/sql/CMakeLists.txt b/src/sql/CMakeLists.txt index 8371ef900..80f9a1096 100644 --- a/src/sql/CMakeLists.txt +++ b/src/sql/CMakeLists.txt @@ -890,8 +890,9 @@ ob_set_subtarget(ob_sql engine_table engine/table/ob_index_lookup_op_impl.cpp engine/table/ob_table_scan_with_index_back_op.cpp engine/table/ob_external_table_access_service.cpp - engine/table/ob_orc_table_row_iter.cpp + # engine/table/ob_orc_table_row_iter.cpp engine/table/ob_parquet_table_row_iter.cpp + engine/table/ob_odps_table_row_iter.cpp ) ob_set_subtarget(ob_sql executor @@ -1399,8 +1400,24 @@ add_library(ob_sql_static STATIC EXCLUDE_FROM_ALL) +if(OB_BUILD_OPENSOURCE) target_link_libraries(ob_sql_static - PUBLIC ob_sql ob_sql_server_parser_static) + PUBLIC ob_sql + ob_sql_server_parser_static) +else() +target_link_libraries(ob_sql_static + PUBLIC ob_sql + ob_sql_server_parser_static + ${DEP_DIR}/lib64/libprotobuf.a + ${DEP_DIR}/lib64/libzstd.a + ${DEP_DIR}/lib64/libbrotlidec.a + ${DEP_DIR}/lib64/libbrotlienc.a + ${DEP_DIR}/lib64/libbrotlicommon.a + ${DEP_DIR}/lib64/libbz2.a + ${DEP_DIR}/lib64/libodps_sdk_tunnel_static.a + ${DEP_DIR}/lib64/libodps_sdk_common_static.a + ${DEP_DIR}/lib64/libodps_sdk_core_static.a) +endif() execute_process( COMMAND bash gen_parser.sh diff --git a/src/sql/code_generator/ob_static_engine_cg.cpp b/src/sql/code_generator/ob_static_engine_cg.cpp index 5fe678e2c..95c1eaffc 100644 --- a/src/sql/code_generator/ob_static_engine_cg.cpp +++ b/src/sql/code_generator/ob_static_engine_cg.cpp @@ -7999,6 +7999,10 @@ int ObStaticEngineCG::generate_spec(ObLogSelectInto &op, ObSelectIntoSpec &spec, LOG_WARN("fail to set closed cht", K(op.get_closed_cht()), K(ret)); } else if (OB_FAIL(deep_copy_obj(alloc, op.get_escaped_cht(), spec.escaped_cht_))) { LOG_WARN("fail to set escaped cht", K(op.get_escaped_cht()), K(ret)); + } else if (OB_FAIL(spec.external_properties_.store_str(op.get_external_properties()))) { + LOG_WARN("fail to set external properties", K(op.get_external_properties()), K(ret)); + } else if (OB_FAIL(spec.external_partition_.store_str(op.get_external_partition()))) { + LOG_WARN("fail to set external partition", K(op.get_external_partition()), K(ret)); } else if (OB_FAIL(spec.user_vars_.init(op.get_user_vars().count()))) { LOG_WARN("init fixed array failed", K(ret), K(op.get_user_vars().count())); } else if (OB_FAIL(spec.select_exprs_.init(op.get_select_exprs().count()))) { @@ -8025,13 +8029,25 @@ int ObStaticEngineCG::generate_spec(ObLogSelectInto &op, ObSelectIntoSpec &spec, LOG_WARN("failed to push back expr", K(ret)); } } + if (OB_SUCC(ret)) { + ObExpr *rt_expr = nullptr; + const ObRawExpr* file_partition_expr = op.get_file_partition_expr(); + if (file_partition_expr == NULL) { + } else if (OB_FAIL(generate_rt_expr(*file_partition_expr, rt_expr))) { + LOG_WARN("failed to generate rt expr", K(ret)); + } else { + spec.file_partition_expr_ = rt_expr; + } + } if (OB_SUCC(ret)) { spec.into_type_ = op.get_into_type(); spec.is_optional_ = op.get_is_optional(); spec.is_single_ = op.get_is_single(); spec.max_file_size_ = op.get_max_file_size(); + spec.buffer_size_ = op.get_buffer_size(); spec.cs_type_ = op.get_cs_type(); spec.parallel_ = op.get_parallel(); + spec.is_overwrite_ = op.get_is_overwrite(); spec.plan_->need_drive_dml_query_ = true; } } diff --git a/src/sql/code_generator/ob_tsc_cg_service.cpp b/src/sql/code_generator/ob_tsc_cg_service.cpp index 42bb8d0c9..83e36724f 100644 --- a/src/sql/code_generator/ob_tsc_cg_service.cpp +++ b/src/sql/code_generator/ob_tsc_cg_service.cpp @@ -65,7 +65,13 @@ int ObTscCgService::generate_tsc_ctdef(ObLogTableScan &op, ObTableScanCtDef &tsc LOG_WARN("fail to check location access priv", K(ret)); } else { scan_ctdef.is_external_table_ = true; - if (OB_FAIL(scan_ctdef.external_file_format_str_.store_str(table_schema->get_external_file_format()))) { + const ObString &format_or_properties = table_schema->get_external_file_format().empty() ? + table_schema->get_external_properties() : + table_schema->get_external_file_format(); + if (format_or_properties.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("format_or_properties is empty", K(ret)); + } else if (OB_FAIL(scan_ctdef.external_file_format_str_.store_str(format_or_properties))) { LOG_WARN("fail to set string", K(ret)); } else if (OB_FAIL(scan_ctdef.external_file_location_.store_str(table_schema->get_external_file_location()))) { LOG_WARN("fail to set string", K(ret)); diff --git a/src/sql/engine/basic/ob_select_into_op.cpp b/src/sql/engine/basic/ob_select_into_op.cpp index b16d1d091..5b02c4096 100644 --- a/src/sql/engine/basic/ob_select_into_op.cpp +++ b/src/sql/engine/basic/ob_select_into_op.cpp @@ -20,6 +20,7 @@ #include "share/ob_device_manager.h" #include "sql/resolver/ob_resolver_utils.h" #include "lib/charset/ob_charset_string_helper.h" +#include "sql/engine/px/ob_px_sqc_handler.h" namespace oceanbase { @@ -35,6 +36,65 @@ OB_SERIALIZE_MEMBER((ObSelectIntoSpec, ObOpSpec), into_type_, user_vars_, outfil int ObSelectIntoOp::inner_open() +{ + int ret = OB_SUCCESS; + bool need_check = false; + ObPhysicalPlanCtx *phy_plan_ctx = NULL; + ObSQLSessionInfo *session = NULL; + if (OB_ISNULL(session = ctx_.get_my_session())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get session failed", K(ret)); + } else { + // since we call get_next_row in inner_open, we have to set opened_ first in avoid to a infinite loop. + opened_ = true; + if (!lib::is_oracle_mode()) { + if (OB_FAIL(session->get_sql_select_limit(top_limit_cnt_))) { + LOG_WARN("fail tp get sql select limit", K(ret)); + } + } + } + if (OB_SUCC(ret) && !MY_SPEC.external_properties_.str_.empty()) { + if (OB_FAIL(external_properties_.load_from_string(MY_SPEC.external_properties_.str_, + ctx_.get_allocator()))) { + LOG_WARN("failed to load external properties", K(ret)); + } else { + format_type_ = external_properties_.format_type_; + } + } + if (OB_SUCC(ret)) { + switch (format_type_) + { + case ObExternalFileFormat::FormatType::CSV_FORMAT: + { + if (OB_FAIL(init_csv_env())) { + LOG_WARN("failed to init csv env", K(ret)); + } + break; + } + case ObExternalFileFormat::FormatType::ODPS_FORMAT: + { +#ifdef OB_BUILD_CPP_ODPS + if (OB_FAIL(init_odps_tunnel())) { + LOG_WARN("failed to init odps tunnel", K(ret)); + } +#else + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "external odps table"); + LOG_WARN("not support to write odps in opensource", K(ret)); +#endif + break; + } + default: + { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not support select into type", K(format_type_)); + } + } + } + return ret; +} + +int ObSelectIntoOp::init_csv_env() { int ret = OB_SUCCESS; file_name_ = MY_SPEC.outfile_name_; @@ -44,18 +104,19 @@ int ObSelectIntoOp::inner_open() char_enclose_ = has_enclose_ ? MY_SPEC.closed_cht_.get_char().ptr()[0] : 0; has_escape_ = MY_SPEC.escaped_cht_.get_val_len() > 0; char_escape_ = has_escape_ ? MY_SPEC.escaped_cht_.get_char().ptr()[0] : 0; - ObSelectIntoOpInput *input = static_cast(input_); - int64_t row_count = 0; + do_partition_ = MY_SPEC.file_partition_expr_ == NULL ? false : true; bool need_check = false; ObPhysicalPlanCtx *phy_plan_ctx = NULL; ObSQLSessionInfo *session = NULL; const ObItemType into_type = MY_SPEC.into_type_; - if (OB_ISNULL(phy_plan_ctx = ctx_.get_physical_plan_ctx())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get phy_plan_ctx failed", K(ret)); - } else if (OB_ISNULL(session = ctx_.get_my_session())) { + if (OB_ISNULL(session = ctx_.get_my_session())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get session failed", K(ret)); + } else if (OB_FAIL(check_has_lob_or_json())) { + LOG_WARN("failed to check has lob", K(ret)); + } else if (OB_ISNULL(phy_plan_ctx = ctx_.get_physical_plan_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get phy_plan_ctx failed", K(ret)); } else if (OB_FAIL(ObSQLUtils::get_param_value(MY_SPEC.outfile_name_, phy_plan_ctx->get_param_store(), file_name_, @@ -73,124 +134,135 @@ int ObSelectIntoOp::inner_open() LOG_WARN("get param value failed", K(ret)); } else if (OB_FAIL(prepare_escape_printer())) { LOG_WARN("failed to calc escape info", K(ret)); - } else if (OB_FAIL(check_has_lob_or_json())) { - LOG_WARN("failed to check has lob", K(ret)); } else { print_params_.tz_info_ = session->get_timezone_info(); print_params_.use_memcpy_ = true; print_params_.binary_string_print_hex_ = lib::is_oracle_mode(); print_params_.cs_type_ = MY_SPEC.cs_type_; - // since we call get_next_row in inner_open, we have to set opened_ first in avoid to a infinite loop. - opened_ = true; - if (!lib::is_oracle_mode()) { - if (OB_FAIL(session->get_sql_select_limit(top_limit_cnt_))) { - LOG_WARN("fail tp get sql select limit", K(ret)); - } - } } //create buffer - if (OB_SUCC(ret)) { - const int64_t buf_len = has_lob_ ? (5 * OB_MALLOC_BIG_BLOCK_SIZE) : OB_MALLOC_BIG_BLOCK_SIZE; - char *buf = NULL; - if (OB_ISNULL(buf = static_cast(ctx_.get_allocator().alloc(buf_len)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to allocate buffer", K(ret), K(buf_len)); - } else { - data_writer_.init(buf, buf_len); - } - if (has_json_ && has_escape_) { - const int64_t json_buf_len = OB_MALLOC_MIDDLE_BLOCK_SIZE; - char *json_buf = NULL; - if (OB_ISNULL(json_buf = static_cast(ctx_.get_allocator().alloc(json_buf_len)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to allocate buffer", K(ret), K(json_buf_len)); - } else { - data_writer_.init_json_buf(json_buf, json_buf_len); - } - } + if (OB_SUCC(ret) && T_INTO_OUTFILE == into_type && OB_FAIL(create_shared_buffer_for_data_writer())) { + LOG_WARN("failed to create buffer for data writer", K(ret)); } + //calc first data_writer.url_ and basic_url_ if (OB_SUCC(ret)) { ObString path = file_name_.get_varchar().trim(); - ObSqlString file_name_with_suffix; - ObString input_file_name; file_location_ = path.prefix_match_ci(OB_OSS_PREFIX) ? IntoFileLocation::REMOTE_OSS : IntoFileLocation::SERVER_DISK; - if (T_INTO_OUTFILE == into_type && !MY_SPEC.is_single_ && OB_FAIL(calc_first_file_path(path))) { + if (file_location_ == IntoFileLocation::SERVER_DISK && do_partition_) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not support partition option on server disk", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "partition option on server disk"); + } else if (T_INTO_OUTFILE == into_type && !MY_SPEC.is_single_ && OB_FAIL(calc_first_file_path(path))) { LOG_WARN("failed to calc first file path", K(ret)); } else if (file_location_ == IntoFileLocation::REMOTE_OSS) { ObString temp_url = path.split_on('?'); temp_url.trim(); ObString storage_info; - if (OB_FAIL(ob_write_string(ctx_.get_allocator(), temp_url, url_, true))) { - LOG_WARN("fail to append string", K(ret)); + if (OB_FAIL(ob_write_string(ctx_.get_allocator(), temp_url, basic_url_, true))) { + LOG_WARN("failed to append string", K(ret)); } else if (OB_FAIL(ob_write_string(ctx_.get_allocator(), path, storage_info, true))) { - LOG_WARN("fail to append string", K(ret)); - } else if (OB_FAIL(access_info_.set(url_.ptr(), storage_info.ptr()))) { - LOG_WARN("fail to set access info", K(ret), K(path)); + LOG_WARN("failed to append string", K(ret)); + } else if (OB_FAIL(access_info_.set(basic_url_.ptr(), storage_info.ptr()))) { + LOG_WARN("failed to set access info", K(ret), K(path)); } //init device handle if (OB_SUCC(ret)) { ObBackupIoAdapter util; - if (url_.empty() || !access_info_.is_valid()) { + if (basic_url_.empty() || !access_info_.is_valid()) { ret = OB_FILE_NOT_EXIST; - LOG_WARN("file path not exist", K(ret), K(url_), K(access_info_)); - } else if (OB_FAIL(util.get_and_init_device(device_handle_, &access_info_, url_))) { - LOG_WARN("fail to init device", K(ret), K(url_), K(access_info_)); + LOG_WARN("file path not exist", K(ret), K(basic_url_), K(access_info_)); + } else if (OB_FAIL(util.get_and_init_device(device_handle_, &access_info_, basic_url_))) { + LOG_WARN("failed to init device", K(ret), K(basic_url_), K(access_info_)); } } } else { // IntoFileLocation::SERVER_DISK - if (OB_FAIL(ob_write_string(ctx_.get_allocator(), path, url_, true))) { - LOG_WARN("fail to write string", K(ret)); + if (OB_FAIL(ob_write_string(ctx_.get_allocator(), path, basic_url_, true))) { + LOG_WARN("failed to write string", K(ret)); } } } - if (OB_SUCC(ret) - && (T_INTO_OUTFILE == into_type || T_INTO_DUMPFILE == into_type) - && IntoFileLocation::SERVER_DISK == file_location_) { - ObString file_name = url_; - ObString file_path = file_name.split_on(file_name.reverse_find('/')); - char full_path_buf[PATH_MAX+1]; - char *actual_path = nullptr; - ObSqlString sql_str; - if (OB_FAIL(sql_str.append(file_path.empty() ? "." : file_path))) { - LOG_WARN("fail to append string", K(ret)); - } else if (OB_ISNULL(actual_path = realpath(sql_str.ptr(), full_path_buf))) { - ret = OB_FILE_NOT_EXIST; - LOG_WARN("file not exist", K(ret), K(sql_str)); - } - if (OB_SUCC(ret)) { - ObString secure_file_priv; - int64_t tenant_id = MTL_ID(); - if (OB_FAIL(ObSchemaUtils::get_tenant_varchar_variable( - tenant_id, - SYS_VAR_SECURE_FILE_PRIV, - ctx_.get_allocator(), - secure_file_priv))) { - LOG_WARN("fail get tenant variable", K(tenant_id), K(secure_file_priv), K(ret)); - } else if (OB_FAIL(ObResolverUtils::check_secure_path(secure_file_priv, actual_path))) { - LOG_WARN("failed to check secure path", K(ret), K(secure_file_priv)); - if (OB_ERR_NO_PRIVILEGE == ret) { - ret = OB_ERR_NO_PRIV_DIRECT_PATH_ACCESS; - LOG_ERROR("fail to check secure path", K(ret), K(secure_file_priv), K(session->get_is_deserialized())); - } - } - } + if (OB_SUCC(ret) && (T_INTO_OUTFILE == into_type || T_INTO_DUMPFILE == into_type) + && IntoFileLocation::SERVER_DISK == file_location_ && OB_FAIL(check_secure_file_path(basic_url_))) { + LOG_WARN("failed to check secure file path", K(ret)); + } + if (OB_SUCC(ret) && do_partition_) { + partition_map_.create(128, ObLabel("SelectInto"), ObLabel("SelectInto"), MTL_ID()); } return ret; } +#ifdef OB_BUILD_CPP_ODPS +int ObSelectIntoOp::init_odps_tunnel() +{ + int ret = OB_SUCCESS; + bool is_in_px = (NULL != ctx_.get_sqc_handler()); + ObSelectIntoOpInput *input = static_cast(input_); + if (OB_ISNULL(input)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("input is unexpected null", K(ret)); + } else if (is_in_px) { + ObOdpsPartitionDownloaderMgr &odps_mgr = ctx_.get_sqc_handler()->get_sqc_ctx().gi_pump_.get_odps_mgr(); + if (OB_FAIL(odps_mgr.get_odps_uploader(input->task_id_, upload_, record_writer_))) { + LOG_WARN("failed to get odps uploader", K(ret)); + } + } else if (OB_FAIL(external_properties_.odps_format_.decrypt())) { + LOG_WARN("failed to decrypt odps format", K(ret)); + } else { + ObMallocHookAttrGuard guard(ObMemAttr(MTL_ID(), "IntoOdps")); + try { + if (OB_FAIL(ObOdpsPartitionDownloaderMgr::create_upload_session(external_properties_.odps_format_, + MY_SPEC.external_partition_.str_, + MY_SPEC.is_overwrite_, + upload_))) { + LOG_WARN("failed to create upload session", K(ret)); + } else if (OB_UNLIKELY(!(record_writer_ = upload_->OpenWriter(block_id_, true)))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted null ptr", K(ret)); + } + } catch (apsara::odps::sdk::OdpsException& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when init odps tunnel", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (const std::exception& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when init odps tunnel", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (...) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when init odps tunnel", K(ret)); + } + } + } + if (OB_FAIL(ret)) { + need_commit_ = false; + } + return ret; +} +#endif + int ObSelectIntoOp::inner_get_next_row() { int ret = 0 == top_limit_cnt_ ? OB_ITER_END : OB_SUCCESS; int64_t row_count = 0; const ObItemType into_type = MY_SPEC.into_type_; ObPhysicalPlanCtx *phy_plan_ctx = NULL; + ObIOBufferWriter *data_writer = NULL; if (OB_ISNULL(phy_plan_ctx = ctx_.get_physical_plan_ctx())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get phy_plan_ctx failed", K(ret)); - } else if (T_INTO_OUTFILE == into_type && MY_SPEC.is_single_ && OB_FAIL(open_file())) { - LOG_WARN("failed to open file", K(ret)); + } + //when do_partition is false, create the only data_writer here + if (OB_SUCC(ret) && ObExternalFileFormat::FormatType::CSV_FORMAT == format_type_ + && T_INTO_VARIABLES != into_type && !do_partition_ + && OB_FAIL(create_the_only_data_writer(data_writer))) { + LOG_WARN("failed to create the only data writer", K(ret)); } while (OB_SUCC(ret) && row_count < top_limit_cnt_) { clear_evaluated_flag(); @@ -201,22 +273,33 @@ int ObSelectIntoOp::inner_get_next_row() } } else { ++row_count; - if (T_INTO_VARIABLES == into_type) { + if (ObExternalFileFormat::FormatType::ODPS_FORMAT == format_type_) { +#ifdef OB_BUILD_CPP_ODPS + if (OB_FAIL(into_odps())) { + LOG_WARN("into odps failed", K(ret)); + } +#else + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "external odps table"); + LOG_WARN("not support to write odps in opensource", K(ret)); +#endif + } else if (T_INTO_VARIABLES == into_type) { if (OB_FAIL(into_varlist())) { LOG_WARN("into varlist failed", K(ret)); } } else if (T_INTO_OUTFILE == into_type) { - if (OB_FAIL(into_outfile())) { + if (OB_FAIL(into_outfile(data_writer))) { LOG_WARN("into outfile failed", K(ret)); } } else { - if (OB_FAIL(into_dumpfile())) { + if (OB_FAIL(into_dumpfile(data_writer))) { LOG_WARN("into dumpfile failed", K(ret)); } } } if (OB_SUCC(ret) || OB_ITER_END == ret) { // if into user variables or into dumpfile, must be one row - if ((T_INTO_VARIABLES == into_type || T_INTO_DUMPFILE == into_type) && row_count > 1) { + if (ObExternalFileFormat::FormatType::CSV_FORMAT == format_type_ + && (T_INTO_VARIABLES == into_type || T_INTO_DUMPFILE == into_type) && row_count > 1) { ret = OB_ERR_TOO_MANY_ROWS; LOG_WARN("more than one row for into variables or into dumpfile", K(ret), K(row_count)); } @@ -225,6 +308,9 @@ int ObSelectIntoOp::inner_get_next_row() if (OB_ITER_END == ret || OB_SUCC(ret)) { // set affected rows phy_plan_ctx->set_affected_rows(row_count); } + if (OB_FAIL(ret) && OB_ITER_END != ret) { + need_commit_ = false; + } return ret; } @@ -236,14 +322,20 @@ int ObSelectIntoOp::inner_get_next_batch(const int64_t max_row_cnt) int64_t row_count = 0; const ObItemType into_type = MY_SPEC.into_type_; ObPhysicalPlanCtx *phy_plan_ctx = NULL; + ObIOBufferWriter *data_writer = NULL; + bool stop_loop = false; + bool is_iter_end = false; + LOG_TRACE("debug select into get next batch begin"); if (OB_ISNULL(phy_plan_ctx = ctx_.get_physical_plan_ctx())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get phy_plan_ctx failed", K(ret)); - } else if (T_INTO_OUTFILE == into_type && MY_SPEC.is_single_ && OB_FAIL(open_file())) { - LOG_WARN("failed to open file", K(ret)); } - bool stop_loop = false; - bool is_iter_end = false; + //when do_partition is false, create the only data_writer here + if (OB_SUCC(ret) && ObExternalFileFormat::FormatType::CSV_FORMAT == format_type_ + && T_INTO_VARIABLES != into_type && !do_partition_ + && OB_FAIL(create_the_only_data_writer(data_writer))) { + LOG_WARN("failed to create the only data writer", K(ret)); + } if (0 == top_limit_cnt_) { brs_.size_ = 0; brs_.end_ = true; @@ -261,8 +353,18 @@ int ObSelectIntoOp::inner_get_next_batch(const int64_t max_row_cnt) if (brs_.size_ > 0) { brs_.skip_->deep_copy(*(child_brs->skip_), brs_.size_); row_count += brs_.size_ - brs_.skip_->accumulate_bit_cnt(brs_.size_); - if (T_INTO_OUTFILE == into_type) { - if (OB_FAIL(into_outfile_batch(brs_))) { + if (ObExternalFileFormat::FormatType::ODPS_FORMAT == format_type_) { +#ifdef OB_BUILD_CPP_ODPS + if (OB_FAIL(into_odps_batch(brs_))) { + LOG_WARN("into odps batch failed", K(ret)); + } +#else + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "external odps table"); + LOG_WARN("not support to write odps in opensource", K(ret)); +#endif + } else if (T_INTO_OUTFILE == into_type) { + if (OB_FAIL(into_outfile_batch(brs_, data_writer))) { LOG_WARN("into outfile batch failed", K(ret)); } } else { @@ -278,7 +380,7 @@ int ObSelectIntoOp::inner_get_next_batch(const int64_t max_row_cnt) LOG_WARN("into varlist failed", K(ret)); } } else { - if (OB_FAIL(into_dumpfile())) { + if (OB_FAIL(into_dumpfile(data_writer))) { LOG_WARN("into dumpfile failed", K(ret)); } } @@ -299,6 +401,10 @@ int ObSelectIntoOp::inner_get_next_batch(const int64_t max_row_cnt) if (OB_SUCC(ret)) { // set affected rows phy_plan_ctx->set_affected_rows(row_count); } + if (OB_FAIL(ret)) { + need_commit_ = false; + } + LOG_TRACE("debug select into get next batch end"); return ret; } @@ -311,10 +417,29 @@ int ObSelectIntoOp::inner_rescan() int ObSelectIntoOp::inner_close() { int ret = OB_SUCCESS; - if (!has_lob_ && OB_FAIL(data_writer_.flush(get_flush_function()))) { + ObIOBufferWriter *data_writer = NULL; + if (ObExternalFileFormat::FormatType::ODPS_FORMAT == format_type_) { +#ifdef OB_BUILD_CPP_ODPS + if (OB_FAIL(odps_commit_upload())) { + LOG_WARN("failed to commit upload", K(ret)); + } +#else + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "external odps table"); + LOG_WARN("not support to write odps in opensource", K(ret)); +#endif + } else if (do_partition_) { + for (ObPartitionWriterMap::iterator iter = partition_map_.begin(); + OB_SUCC(ret) && iter != partition_map_.end(); iter++) { + if (OB_ISNULL(data_writer = iter->second)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("data writer is unexpected null", K(ret)); + } else if (OB_FAIL(flush_buf(*data_writer))) { + LOG_WARN("failed to flush buffer", K(ret)); + } + } + } else if (OB_NOT_NULL(data_writer_) && OB_FAIL(flush_buf(*data_writer_))) { LOG_WARN("failed to flush buffer", K(ret)); - } else if (has_lob_ && OB_FAIL(data_writer_.flush_all_for_lob(get_flush_function()))) { - LOG_WARN("failed to flush buffer for lob", K(ret)); } return ret; } @@ -371,7 +496,7 @@ int ObSelectIntoOp::get_row_str(const int64_t buf_len, return ret; } -int ObSelectIntoOp::open_file() +int ObSelectIntoOp::open_file(ObIOBufferWriter &data_writer) { int ret = OB_SUCCESS; if (IntoFileLocation::REMOTE_OSS == file_location_) { @@ -381,21 +506,21 @@ int ObSelectIntoOp::open_file() iod_opts.opts_ = &opt; iod_opts.opt_cnt_ = 1; bool is_exist = false; - if (OB_FAIL(device_handle_->exist(url_.ptr(), is_exist))) { - LOG_WARN("failed to check file exist", K(ret), K(url_)); + if (OB_FAIL(device_handle_->exist(data_writer.url_.ptr(), is_exist))) { + LOG_WARN("failed to check file exist", K(ret), K(data_writer.url_)); } else if (is_exist) { ret = OB_FILE_ALREADY_EXIST; - LOG_WARN("file already exist", K(ret), K(url_)); - } else if (OB_FAIL(device_handle_->open(url_.ptr(), -1, 0, fd_, &iod_opts))) { + LOG_WARN("file already exist", K(ret), K(data_writer.url_)); + } else if (OB_FAIL(device_handle_->open(data_writer.url_.ptr(), -1, 0, data_writer.fd_, &iod_opts))) { LOG_WARN("failed to open file", K(ret)); } else { - is_file_opened_ = true; + data_writer.is_file_opened_ = true; } } else if (IntoFileLocation::SERVER_DISK == file_location_) { - if (OB_FAIL(file_appender_.create(url_, true))) { - LOG_WARN("failed to create file", K(ret), K(url_)); + if (OB_FAIL(data_writer.file_appender_.create(data_writer.url_, true))) { + LOG_WARN("failed to create file", K(ret), K(data_writer.url_)); } else { - is_file_opened_ = true; + data_writer.is_file_opened_ = true; } } else { ret = OB_ERR_UNEXPECTED; @@ -417,17 +542,17 @@ int ObSelectIntoOp::calc_first_file_path(ObString &path) LOG_WARN("get unexpected path or input is null", K(ret)); } else { if (input_file_name.ptr()[input_file_name.length() - 1] == '/'){ - file_name_with_suffix.append_fmt("%sdata", to_cstring(input_file_name)); + file_name_with_suffix.append_fmt("%.*sdata", input_file_name.length(), input_file_name.ptr()); } else { - file_name_with_suffix.append_fmt("%s", to_cstring(input_file_name)); + file_name_with_suffix.append_fmt("%.*s", input_file_name.length(), input_file_name.ptr()); } if (MY_SPEC.parallel_ > 1) { - file_name_with_suffix.append_fmt("_%ld_%ld_%ld", input->sqc_id_, input->task_id_, split_file_id_); + file_name_with_suffix.append_fmt("_%ld_%ld_%d", input->sqc_id_, input->task_id_, 0); } else { - file_name_with_suffix.append_fmt("_%ld", split_file_id_); + file_name_with_suffix.append_fmt("_%d", 0); } if (file_location_ == IntoFileLocation::REMOTE_OSS) { - file_name_with_suffix.append_fmt("?%s", to_cstring(path)); + file_name_with_suffix.append_fmt("?%.*s", path.length(), path.ptr()); } if (OB_FAIL(ob_write_string(ctx_.get_allocator(), file_name_with_suffix.string(), path))) { LOG_WARN("failed to write string", K(ret)); @@ -436,63 +561,111 @@ int ObSelectIntoOp::calc_first_file_path(ObString &path) return ret; } -int ObSelectIntoOp::calc_next_file_path() +int ObSelectIntoOp::calc_next_file_path(ObIOBufferWriter &data_writer) { int ret = OB_SUCCESS; ObSqlString url_with_suffix; ObString file_path; - if (split_file_id_ > 0) { + data_writer.split_file_id_++; + if (data_writer.split_file_id_ > 0) { if (MY_SPEC.is_single_ && IntoFileLocation::REMOTE_OSS == file_location_) { - file_path = (split_file_id_ > 1) ? url_.split_on(url_.reverse_find('.')) : url_; + file_path = (data_writer.split_file_id_ > 1) + ? data_writer.url_.split_on(data_writer.url_.reverse_find('.')) + : data_writer.url_; if (OB_FAIL(url_with_suffix.assign(file_path))) { - LOG_WARN("fail to assign string", K(ret)); - } else if (OB_FAIL(url_with_suffix.append_fmt(".extend%ld", split_file_id_))) { - LOG_WARN("fail to append string", K(ret)); + LOG_WARN("failed to assign string", K(ret)); + } else if (OB_FAIL(url_with_suffix.append_fmt(".extend%ld", data_writer.split_file_id_))) { + LOG_WARN("failed to append string", K(ret)); + } + } else if (!MY_SPEC.is_single_) { + file_path = data_writer.url_.split_on(data_writer.url_.reverse_find('_')); + if (OB_FAIL(url_with_suffix.assign(file_path))) { + LOG_WARN("failed to assign string", K(ret)); + } else if (OB_FAIL(url_with_suffix.append_fmt("_%ld", data_writer.split_file_id_))) { + LOG_WARN("failed to append string", K(ret)); } } else { - file_path = url_.split_on(url_.reverse_find('_')); - if (OB_FAIL(url_with_suffix.assign(file_path))) { - LOG_WARN("fail to assign string", K(ret)); - } else if (OB_FAIL(url_with_suffix.append_fmt("_%ld", split_file_id_))) { - LOG_WARN("fail to append string", K(ret)); - } + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected single value", K(ret)); } - if (OB_SUCC(ret) - && OB_FAIL(ob_write_string(ctx_.get_allocator(), url_with_suffix.string(), url_, true))) { - LOG_WARN("fail to write string", K(ret)); + if (OB_SUCC(ret) && OB_FAIL(ob_write_string(ctx_.get_allocator(), + url_with_suffix.string(), + data_writer.url_, true))) { + LOG_WARN("failed to write string", K(ret)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected split file id", K(ret)); + } + return ret; +} + +// 根据传入的partition和basic_url_设置当前data_writer的url_, 每个分区只需要计算一次, 后续只要改split id +int ObSelectIntoOp::calc_file_path_with_partition(ObString partition, ObIOBufferWriter &data_writer) +{ + int ret = OB_SUCCESS; + ObSqlString url_with_partition; + ObString dir_path; + if (OB_FAIL(ob_write_string(ctx_.get_allocator(), basic_url_, data_writer.url_))) { + LOG_WARN("failed to write string", K(ret)); + } else { + dir_path = data_writer.url_.split_on(data_writer.url_.reverse_find('/')); + if (OB_FAIL(url_with_partition.assign(dir_path))) { + LOG_WARN("failed to assign string", K(ret)); + } else if (url_with_partition.length() != 0 && OB_FAIL(url_with_partition.append("/"))) { + LOG_WARN("failed to append string", K(ret)); + } else if (partition.length() != 0 && OB_FAIL(url_with_partition.append_fmt("%.*s/", + partition.length(), + partition.ptr()))) { + LOG_WARN("failed to append string", K(ret)); + } else if (partition.length() == 0 && OB_FAIL(url_with_partition.append("__NULL__/"))) { + LOG_WARN("failed to append string", K(ret)); + } else if (OB_FAIL(url_with_partition.append_fmt("%.*s", + data_writer.url_.length(), + data_writer.url_.ptr()))) { + LOG_WARN("failed to append string", K(ret)); + } else if (OB_FAIL(ob_write_string(ctx_.get_allocator(), + url_with_partition.string(), + data_writer.url_, + true))) { + LOG_WARN("failed to write string", K(ret)); } } return ret; } -void ObSelectIntoOp::close_file() +void ObSelectIntoOp::close_file(ObIOBufferWriter &data_writer) { if (IntoFileLocation::SERVER_DISK == file_location_) { - file_appender_.close(); + data_writer.file_appender_.close(); } else { - if (fd_.is_valid()) { - device_handle_->close(fd_); - fd_.reset(); + if (data_writer.fd_.is_valid()) { + device_handle_->close(data_writer.fd_); + data_writer.fd_.reset(); } } - is_file_opened_ = false; + data_writer.is_file_opened_ = false; } -std::function ObSelectIntoOp::get_flush_function() +std::function ObSelectIntoOp::get_flush_function() { - return [this](const char *data, int64_t data_len) -> int + return [this](const char *data, int64_t data_len, ObSelectIntoOp::ObIOBufferWriter *data_writer) -> int { int ret = OB_SUCCESS; - if (!is_file_opened_ && OB_FAIL(open_file())) { - LOG_WARN("failed to open file", K(ret), K(url_)); + if (data == NULL || data_len == 0) { + } else if (OB_ISNULL(data_writer)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null data writer", K(ret)); + } else if (!data_writer->is_file_opened_ && OB_FAIL(open_file(*data_writer))) { + LOG_WARN("failed to open file", K(ret), K(data_writer->url_)); } else if (file_location_ == IntoFileLocation::SERVER_DISK) { - if (OB_FAIL(file_appender_.append(data, data_len, false))) { + if (OB_FAIL(data_writer->file_appender_.append(data, data_len, false))) { LOG_WARN("failed to append file", K(ret), K(data_len)); } } else if (file_location_ == IntoFileLocation::REMOTE_OSS) { int64_t write_size = 0; int64_t begin_ts = ObTimeUtility::current_time(); - if (OB_FAIL(device_handle_->write(fd_, data, data_len, write_size))) { + if (OB_FAIL(device_handle_->write(data_writer->fd_, data, data_len, write_size))) { LOG_WARN("failed to write device", K(ret)); } else if (OB_UNLIKELY(write_size != data_len)) { ret = OB_IO_ERROR; @@ -515,129 +688,192 @@ std::function ObSelectIntoOp::get_flush_function() }; } -int ObSelectIntoOp::split_file() +int ObSelectIntoOp::split_file(ObIOBufferWriter &data_writer) { int ret = OB_SUCCESS; - int64_t dummy_pos = 0; - if (OB_FAIL(flush_buf(dummy_pos))) { - LOG_WARN("fail to flush buffer", K(ret)); + if (!use_shared_buf_ && OB_FAIL(flush_buf(data_writer))) { + LOG_WARN("failed to flush buffer", K(ret)); + } else if (has_lob_ && use_shared_buf_ && OB_FAIL(flush_shared_buf(data_writer, get_flush_function()))) { + // 要保证文件中每一行的完整性, 有lob的时候shared buffer里不一定是完整的一行 + // 因此剩下的shared buffer里的内容也要刷到当前文件里, 这种情况下无法严格满足max_file_size的限制 + LOG_WARN("failed to flush shared buffer", K(ret)); } else { - close_file(); + close_file(data_writer); } - - //rename the first file name - /* rename not support for current version - if (OB_SUCC(ret) && 0 == split_file_id_) { - ObSqlString url_old; - ObSqlString url_new; - - if (OB_FAIL(url_old.assign(url_))) { - LOG_WARN("fail to assign string", K(ret)); - } else if (OB_FAIL(url_new.assign(url_))) { - LOG_WARN("fail to assign string", K(ret)); - } else if (OB_FAIL(url_new.append_fmt(".part%ld", split_file_id_))) { - LOG_WARN("fail to append format", K(ret)); - } else if (OB_FAIL(device_handle_->rename(url_old.ptr(), url_new.ptr()))) { - LOG_WARN("fail to rename", K(ret)); - } - } - */ - - //create new file - if (OB_SUCC(ret)) { - split_file_id_++; - if (OB_FAIL(calc_next_file_path())) { - LOG_WARN("failed to calculate new file path", K(ret)); - } + if (OB_SUCC(ret) && OB_FAIL(calc_next_file_path(data_writer))) { + LOG_WARN("failed to calculate new file path", K(ret)); } return ret; } -int ObSelectIntoOp::try_split_file() +int ObSelectIntoOp::try_split_file(ObIOBufferWriter &data_writer) { int ret = OB_SUCCESS; - const int64_t MAX_OSS_FILE_SIZE = 5LL * 1024 * 1024 * 1024; //5G int64_t curr_line_len = 0; int64_t curr_bytes = 0; bool has_split = false; - if (!has_lob_ || data_writer_.get_curr_line_len() == 0) { - curr_line_len = data_writer_.get_curr_pos() - data_writer_.get_last_line_pos(); + bool has_use_shared_buf = use_shared_buf_; + if (!has_lob_ || data_writer.get_curr_line_len() == 0) { + curr_line_len = data_writer.get_curr_pos() - data_writer.get_last_line_pos(); } else { - curr_line_len = data_writer_.get_curr_pos() + data_writer_.get_curr_line_len(); + curr_line_len = data_writer.get_curr_pos() + data_writer.get_curr_line_len(); } - curr_bytes = write_bytes_ + curr_line_len; - if (!has_lob_ && data_writer_.get_last_line_pos() == 0) { + curr_bytes = data_writer.get_write_bytes() + curr_line_len; + if (!(has_lob_ && has_use_shared_buf) && data_writer.get_write_bytes() == 0) { } else if ((file_location_ == IntoFileLocation::SERVER_DISK && !MY_SPEC.is_single_ && curr_bytes > MY_SPEC.max_file_size_) || (file_location_ == IntoFileLocation::REMOTE_OSS && ((!MY_SPEC.is_single_ && curr_bytes > min(MY_SPEC.max_file_size_, MAX_OSS_FILE_SIZE)) || (MY_SPEC.is_single_ && curr_bytes > MAX_OSS_FILE_SIZE)))) { - if (OB_FAIL(split_file())) { + if (OB_FAIL(split_file(data_writer))) { LOG_WARN("failed to split file", K(ret)); } else { has_split = true; } } if (OB_SUCC(ret)) { - if (!has_lob_) { - write_bytes_ = has_split ? curr_line_len : curr_bytes; + if (has_lob_ && has_use_shared_buf) { + data_writer.set_write_bytes(has_split ? 0 : curr_bytes); + data_writer.reset_curr_line_len(); } else { - write_bytes_ = has_split ? 0 : curr_bytes; - data_writer_.reset_curr_line_len(); + data_writer.set_write_bytes(has_split ? curr_line_len : curr_bytes); } - data_writer_.update_last_line_pos(); + data_writer.update_last_line_pos(); } return ret; } -void ObSelectIntoOp::get_buf(char* &buf, int64_t &buf_len, int64_t &pos, bool is_json) -{ - buf = is_json ? data_writer_.get_json_buf() : data_writer_.get_buf(); - buf_len = is_json ? data_writer_.get_json_buf_len() : data_writer_.get_buf_len(); - pos = is_json ? 0 : data_writer_.get_curr_pos(); -} - -int ObSelectIntoOp::flush_buf(int64_t &pos) +int ObSelectIntoOp::get_buf(char* &buf, int64_t &buf_len, int64_t &pos, ObIOBufferWriter &data_writer) { int ret = OB_SUCCESS; - if (!has_lob_ && OB_FAIL(data_writer_.flush(get_flush_function()))) { - LOG_WARN("failed to flush buffer", K(ret)); - } else if (has_lob_ && OB_FAIL(data_writer_.flush_all_for_lob(get_flush_function()))) { - LOG_WARN("failed to flush buffer for lob", K(ret)); - } else { - pos = data_writer_.get_curr_pos(); + buf = use_shared_buf_ ? get_shared_buf() : data_writer.get_buf(); + buf_len = use_shared_buf_ ? get_shared_buf_len() : data_writer.get_buf_len(); + pos = data_writer.get_curr_pos(); + if (OB_ISNULL(buf) && !use_shared_buf_ && OB_FAIL(use_shared_buf(data_writer, buf, buf_len, pos))) { + LOG_WARN("failed to use shared buffer", K(ret)); + } else if (OB_ISNULL(buf)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("buf should not be null", K(ret)); } return ret; } -int ObSelectIntoOp::resize_buf(char* &buf, int64_t &buf_len, int64_t &pos, bool is_json) +int ObSelectIntoOp::flush_buf(ObIOBufferWriter &data_writer) +{ + int ret = OB_SUCCESS; + if (use_shared_buf_) { + // do nothing + } else if (OB_FAIL(data_writer.flush(get_flush_function()))) { + LOG_WARN("failed to flush buffer", K(ret)); + } + return ret; +} + +int ObSelectIntoOp::use_shared_buf(ObIOBufferWriter &data_writer, + char* &buf, + int64_t &buf_len, + int64_t &pos) { + int ret = OB_SUCCESS; + int64_t curr_pos = data_writer.get_curr_pos(); + if (!use_shared_buf_ && data_writer.get_last_line_pos() == 0) { + if (OB_NOT_NULL(data_writer.get_buf()) && curr_pos > 0) { + MEMCPY(shared_buf_, data_writer.get_buf(), curr_pos); + } + use_shared_buf_ = true; + buf = shared_buf_; + buf_len = shared_buf_len_; + pos = curr_pos; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("last line should be flushed before this line copied", K(ret)); + } + return ret; +} + +int ObSelectIntoOp::resize_buf(char* &buf, + int64_t &buf_len, + int64_t &pos, + int64_t curr_pos, + bool is_json) { int ret = OB_SUCCESS; int64_t new_buf_len = buf_len * 2; char* new_buf = NULL; - int curr_pos = data_writer_.get_curr_pos(); if (OB_ISNULL(new_buf = static_cast(ctx_.get_allocator().alloc(new_buf_len)))) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to allocate buffer", K(ret), K(new_buf_len)); + LOG_WARN("failed to allocate buffer", K(ret), K(new_buf_len)); } else if (!is_json) { - data_writer_.init(new_buf, new_buf_len); if (curr_pos > 0) { - MEMCPY(new_buf, buf, curr_pos); + MEMCPY(new_buf, shared_buf_, curr_pos); } + shared_buf_ = new_buf; + shared_buf_len_ = new_buf_len; } else { - data_writer_.init_json_buf(new_buf, new_buf_len); + json_buf_ = new_buf; + json_buf_len_ = new_buf_len; } if (OB_SUCC(ret)) { - get_buf(buf, buf_len, pos, is_json); + buf = new_buf; + buf_len = new_buf_len; + pos = is_json ? 0 : curr_pos; } return ret; } -int ObSelectIntoOp::write_obj_to_file(const ObObj &obj, bool need_escape) +int ObSelectIntoOp::resize_or_flush_shared_buf(ObIOBufferWriter &data_writer, + char* &buf, + int64_t &buf_len, + int64_t &pos) +{ + int ret = OB_SUCCESS; + if (!use_shared_buf_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get invalid argument", K(use_shared_buf_), K(ret)); + } else if (has_lob_ && data_writer.get_curr_pos() > 0) { + if (OB_FAIL(flush_shared_buf(data_writer, get_flush_function(), true))) { + LOG_WARN("failed to flush shared buffer", K(ret)); + } else { + pos = 0; + } + } else if (OB_FAIL(resize_buf(buf, buf_len, pos, data_writer.get_curr_pos()))) { + LOG_WARN("failed to resize shared buffer", K(ret)); + } + return ret; +} + +int ObSelectIntoOp::check_buf_sufficient(ObIOBufferWriter &data_writer, + char* &buf, + int64_t &buf_len, + int64_t &pos, + int64_t str_len) +{ + int ret = OB_SUCCESS; + if (buf_len < str_len * 1.1) { + if (OB_FAIL(flush_buf(data_writer))) { + LOG_WARN("failed to flush buffer", K(ret)); + } else if (OB_FAIL(use_shared_buf(data_writer, buf, buf_len, pos))) { + LOG_WARN("failed to use shared buffer", K(ret)); + } + } + return ret; +} + +int ObSelectIntoOp::write_obj_to_file(const ObObj &obj, ObIOBufferWriter &data_writer, bool need_escape) +{ + int ret = OB_SUCCESS; + if ((obj.is_string_type() || obj.is_json()) && need_escape) { + if (OB_FAIL(print_str_or_json_with_escape(obj, data_writer))) { + LOG_WARN("failed to print str or json with escape", K(ret)); + } + } else if (OB_FAIL(print_normal_obj_without_escape(obj, data_writer))) { + LOG_WARN("failed to print normal obj without escape", K(ret)); + } + return ret; +} + +int ObSelectIntoOp::print_str_or_json_with_escape(const ObObj &obj, ObIOBufferWriter &data_writer) { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - bool print_succ = false; char* buf = NULL; int64_t buf_len = 0; int64_t pos = 0; @@ -647,103 +883,154 @@ int ObSelectIntoOp::write_obj_to_file(const ObObj &obj, bool need_escape) || src_type == CHARSET_INVALID); escape_printer_.need_enclose_ = has_enclose_ && !obj.is_null() && (!MY_SPEC.is_optional_ || obj.is_string_type()); - escape_printer_.do_escape_ = need_escape; + escape_printer_.do_escape_ = true; escape_printer_.print_hex_ = obj.get_collation_type() == CS_TYPE_BINARY && print_params_.binary_string_print_hex_; ObString str_to_escape; - - if ((obj.is_string_type() || obj.is_json()) && need_escape) { - if (obj.is_json()) { - ObObj inrow_obj = obj; - if (obj.is_lob_storage()) { - ObEvalCtx::TempAllocGuard tmp_alloc_g(eval_ctx_); - common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); - if (OB_FAIL(ObTextStringIter::convert_outrow_lob_to_inrow_templob(obj, inrow_obj, NULL, &temp_allocator))) { - LOG_WARN("failed to convert outrow lobs", K(ret), K(obj)); + ObEvalCtx::TempAllocGuard tmp_alloc_g(eval_ctx_); + common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); + if (OB_FAIL(get_buf(escape_printer_.buf_, escape_printer_.buf_len_, escape_printer_.pos_, data_writer))) { + LOG_WARN("failed to get buffer", K(ret)); + } else if (obj.is_json()) { + ObObj inrow_obj = obj; + if (obj.is_lob_storage() + && OB_FAIL(ObTextStringIter::convert_outrow_lob_to_inrow_templob(obj, inrow_obj, NULL, &temp_allocator))) { + LOG_WARN("failed to convert outrow lobs", K(ret), K(obj)); + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(print_json_to_json_buf(inrow_obj, buf, buf_len, pos, data_writer))) { + LOG_WARN("failed to print normal obj without escape", K(ret)); + } else { + str_to_escape.assign_ptr(buf, pos); + escape_printer_.do_encode_ = false; + } + } else { + str_to_escape = obj.get_varchar(); + } + if (OB_SUCC(ret) && !use_shared_buf_ && OB_FAIL(check_buf_sufficient(data_writer, + escape_printer_.buf_, + escape_printer_.buf_len_, + escape_printer_.pos_, + str_to_escape.length()))) { + LOG_WARN("failed to check if buf is sufficient", K(ret)); + } + if (OB_SUCC(ret) && !use_shared_buf_) { + if (OB_FAIL(ObFastStringScanner::foreach_char(str_to_escape, + src_type, + escape_printer_, + escape_printer_.do_encode_, + escape_printer_.ignore_convert_failed_))) { + if (OB_SIZE_OVERFLOW != ret) { + LOG_WARN("failed to print plain str", K(ret), K(src_type), K(escape_printer_.do_encode_)); + } else if (OB_FAIL(flush_buf(data_writer))) { + LOG_WARN("failed to flush buffer", K(ret)); + } else if (OB_FALSE_IT(escape_printer_.pos_ = data_writer.get_curr_pos())) { + } else if (OB_FAIL(ObFastStringScanner::foreach_char(str_to_escape, + src_type, + escape_printer_, + escape_printer_.do_encode_, + escape_printer_.ignore_convert_failed_))) { + if (OB_SIZE_OVERFLOW != ret) { + LOG_WARN("failed to print plain str", K(ret), K(src_type), K(escape_printer_.do_encode_)); + } else if (OB_FAIL(use_shared_buf(data_writer, + escape_printer_.buf_, + escape_printer_.buf_len_, + escape_printer_.pos_))) { + LOG_WARN("failed to use shared buffer", K(ret)); } } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(print_normal_obj_without_escape(inrow_obj, buf, buf_len, pos, true))) { - LOG_WARN("failed to print normal obj without escape", K(ret)); - } else { - str_to_escape.assign_ptr(buf, pos); - escape_printer_.do_encode_ = false; - } - } else { - str_to_escape = obj.get_varchar(); } - if (OB_SUCC(ret)) { - get_buf(escape_printer_.buf_, escape_printer_.buf_len_, escape_printer_.pos_); - } - for (int i = 0; OB_SUCC(ret) && !print_succ; ++i) { + } + if (OB_SUCC(ret) && use_shared_buf_) { + do { if (OB_FAIL(ObFastStringScanner::foreach_char(str_to_escape, src_type, escape_printer_, escape_printer_.do_encode_, escape_printer_.ignore_convert_failed_))) { - if (OB_SIZE_OVERFLOW == ret) { - if (i == 0 && OB_UNLIKELY(OB_SUCCESS != (tmp_ret = flush_buf(escape_printer_.pos_)))) { - LOG_WARN("failed to flush buffer", K(tmp_ret), K(ret)); - } else if (i > 0 && OB_UNLIKELY(OB_SUCCESS != (tmp_ret = resize_buf( - escape_printer_.buf_, - escape_printer_.buf_len_, - escape_printer_.pos_)))) { - LOG_WARN("failed to resize buffer", K(tmp_ret), K(ret)); - } else { - ret = OB_SUCCESS; - } - } else { - LOG_WARN("failed to print plain str", K(ret), K(src_type), K(escape_printer_.do_encode_)); - } - } else { - print_succ = true; + LOG_WARN("failed to print plain str", K(ret), K(src_type), K(escape_printer_.do_encode_)); } - } - if (OB_SUCC(ret)) { - data_writer_.set_curr_pos(escape_printer_.pos_); - } - } else { - if (OB_FAIL(print_normal_obj_without_escape(obj, buf, buf_len, pos))) { - LOG_WARN("failed to print normal obj without escape", K(ret)); - } else { - data_writer_.set_curr_pos(pos); + } while (OB_SIZE_OVERFLOW == ret && OB_SUCC(resize_or_flush_shared_buf(data_writer, + escape_printer_.buf_, + escape_printer_.buf_len_, + escape_printer_.pos_))); + if (OB_FAIL(ret)) { + LOG_WARN("failed to print plain str", K(ret)); } } + if (OB_SUCC(ret)) { + data_writer.set_curr_pos(escape_printer_.pos_); + } + return ret; } -int ObSelectIntoOp::print_normal_obj_without_escape(const ObObj &obj, - char* &buf, - int64_t &buf_len, - int64_t &pos, - bool is_json) +int ObSelectIntoOp::print_normal_obj_without_escape(const ObObj &obj, ObIOBufferWriter &data_writer) { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - bool print_succ = false; - get_buf(buf, buf_len, pos, is_json); - for (int i = 0; OB_SUCC(ret) && !print_succ; ++i) { + char* buf = NULL; + int64_t buf_len = 0; + int64_t pos = 0; + OZ(get_buf(buf, buf_len, pos, data_writer)); + if (OB_SUCC(ret) && !use_shared_buf_) { if (OB_FAIL(obj.print_plain_str_literal(buf, buf_len, pos, print_params_))) { - if (OB_SIZE_OVERFLOW == ret) { - if (i == 0 && !is_json && OB_UNLIKELY(OB_SUCCESS != (tmp_ret = flush_buf(pos)))) { - LOG_WARN("failed to flush buffer", K(tmp_ret), K(ret)); - } else if ((i > 0 || is_json) - && OB_UNLIKELY(OB_SUCCESS != (tmp_ret = resize_buf(buf, buf_len, pos, is_json)))) { - LOG_WARN("failed to resize buffer", K(tmp_ret), K(ret)); - } else { - ret = OB_SUCCESS; + if (OB_SIZE_OVERFLOW != ret) { + LOG_WARN("failed to print obj", K(ret)); + } else if (OB_FAIL(flush_buf(data_writer))) { + LOG_WARN("failed to flush buffer", K(ret)); + } else if (OB_FALSE_IT(pos = data_writer.get_curr_pos())) { + } else if (OB_FAIL(obj.print_plain_str_literal(buf, buf_len, pos, print_params_))) { + if (OB_SIZE_OVERFLOW != ret) { + LOG_WARN("failed to print obj", K(ret)); + } else if (OB_FAIL(use_shared_buf(data_writer, buf, buf_len, pos))) { + LOG_WARN("failed to use shared buffer", K(ret)); } - } else { - LOG_WARN("failed to print plain str", K(ret)); } - } else { - print_succ = true; } } + if (OB_SUCC(ret) && use_shared_buf_) { + do { + if (OB_FAIL(obj.print_plain_str_literal(buf, buf_len, pos, print_params_))) { + LOG_WARN("failed to print obj", K(ret)); + } + } while (OB_SIZE_OVERFLOW == ret + && OB_SUCC(resize_or_flush_shared_buf(data_writer, buf, buf_len, pos))); + if (OB_FAIL(ret)) { + LOG_WARN("failed to print obj", K(ret)); + } + } + if (OB_SUCC(ret)) { + data_writer.set_curr_pos(pos); + } return ret; } -int ObSelectIntoOp::write_lob_to_file(const ObObj &obj, const ObExpr &expr, const ObDatum &datum) +int ObSelectIntoOp::print_json_to_json_buf(const ObObj &obj, + char* &buf, + int64_t &buf_len, + int64_t &pos, + ObIOBufferWriter &data_writer) +{ + int ret = OB_SUCCESS; + buf = get_json_buf(); + buf_len = get_json_buf_len(); + pos = 0; + do { + if (OB_FAIL(obj.print_plain_str_literal(buf, buf_len, pos, print_params_))) { + LOG_WARN("failed to print obj", K(ret)); + } + } while (OB_SIZE_OVERFLOW == ret + && OB_SUCC(resize_buf(buf, buf_len, pos, data_writer.get_curr_pos(), true))); + if (OB_FAIL(ret)) { + LOG_WARN("failed to print json to json buffer", K(ret)); + } + return ret; +} + +int ObSelectIntoOp::write_lob_to_file(const ObObj &obj, + const ObExpr &expr, + const ObDatum &datum, + ObIOBufferWriter &data_writer) { int ret = OB_SUCCESS; ObCharsetType src_type = ObCharset::charset_type_by_coll(obj.get_collation_type()); @@ -754,8 +1041,6 @@ int ObSelectIntoOp::write_lob_to_file(const ObObj &obj, const ObExpr &expr, cons escape_printer_.do_escape_ = has_escape_; escape_printer_.print_hex_ = obj.get_collation_type() == CS_TYPE_BINARY && print_params_.binary_string_print_hex_; - get_buf(escape_printer_.buf_, escape_printer_.buf_len_, escape_printer_.pos_); - ObDatumMeta input_meta = expr.datum_meta_; ObTextStringIterState state; ObString src_block_data; @@ -765,36 +1050,93 @@ int ObSelectIntoOp::write_lob_to_file(const ObObj &obj, const ObExpr &expr, cons common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); int64_t truncated_len = 0; bool stop_when_truncated = false; + OZ(lob_iter.init(0, NULL, &temp_allocator)); + OZ(get_buf(escape_printer_.buf_, escape_printer_.buf_len_, escape_printer_.pos_, data_writer)); - if (OB_FAIL(lob_iter.init(0, NULL, &temp_allocator))) { - LOG_WARN("init lob_iter failed ", K(ret), K(lob_iter)); - } // 当truncated_len == src_block_data.length()时 // 表明当前foreach_char处理的仅为lob末尾的无效的数据, 即上一轮的truncated data, 要避免死循环 while (OB_SUCC(ret) && (state = lob_iter.get_next_block(src_block_data)) == TEXTSTRING_ITER_NEXT) { // outrow lob最后一次才有可能为false, inrow lob只迭代一次, 为false stop_when_truncated = (truncated_len != src_block_data.length()) && lob_iter.is_outrow_lob(); - if ((escape_printer_.buf_len_ - escape_printer_.pos_) < (src_block_data.length() * 5) - && OB_FAIL(flush_buf(escape_printer_.pos_))) { - LOG_WARN("failed to flush buf", K(ret)); - } else if (OB_FAIL(ObFastStringScanner::foreach_char(src_block_data, - src_type, - escape_printer_, - escape_printer_.do_encode_, - escape_printer_.ignore_convert_failed_, - stop_when_truncated, - &truncated_len))) { - if (OB_ERR_DATA_TRUNCATED == ret && stop_when_truncated) { - lob_iter.set_reserved_byte_len(truncated_len); - ret = OB_SUCCESS; - } else { - LOG_WARN("failed to print lob", K(ret)); + if (!use_shared_buf_ && OB_FAIL(check_buf_sufficient(data_writer, + escape_printer_.buf_, + escape_printer_.buf_len_, + escape_printer_.pos_, + src_block_data.length()))) { + LOG_WARN("failed to check if buf is sufficient", K(ret)); + } + if (OB_SUCC(ret) && !use_shared_buf_) { + if (OB_FAIL(ObFastStringScanner::foreach_char(src_block_data, + src_type, + escape_printer_, + escape_printer_.do_encode_, + escape_printer_.ignore_convert_failed_, + stop_when_truncated, + &truncated_len))) { + if (OB_ERR_DATA_TRUNCATED == ret && stop_when_truncated) { + lob_iter.set_reserved_byte_len(truncated_len); + ret = OB_SUCCESS; + } else if (OB_SIZE_OVERFLOW != ret) { + LOG_WARN("failed to print lob", K(ret)); + } else if (OB_FAIL(flush_buf(data_writer))) { + LOG_WARN("failed to flush buffer", K(ret)); + } else if (OB_FALSE_IT(escape_printer_.pos_ = data_writer.get_curr_pos())) { + } else if (OB_FAIL(ObFastStringScanner::foreach_char(src_block_data, + src_type, + escape_printer_, + escape_printer_.do_encode_, + escape_printer_.ignore_convert_failed_, + stop_when_truncated, + &truncated_len))) { + if (OB_ERR_DATA_TRUNCATED == ret && stop_when_truncated) { + lob_iter.set_reserved_byte_len(truncated_len); + ret = OB_SUCCESS; + } else if (OB_SIZE_OVERFLOW != ret) { + LOG_WARN("failed to print lob", K(ret)); + } else if (OB_FAIL(use_shared_buf(data_writer, + escape_printer_.buf_, + escape_printer_.buf_len_, + escape_printer_.pos_))) { + LOG_WARN("failed to use shared buffer", K(ret)); + } + } } } - if (OB_SUCC(ret)) { - data_writer_.set_curr_pos(escape_printer_.pos_); + if (OB_SUCC(ret) && use_shared_buf_) { + if (OB_FAIL(ObFastStringScanner::foreach_char(src_block_data, + src_type, + escape_printer_, + escape_printer_.do_encode_, + escape_printer_.ignore_convert_failed_, + stop_when_truncated, + &truncated_len))) { + if (OB_ERR_DATA_TRUNCATED == ret && stop_when_truncated) { + lob_iter.set_reserved_byte_len(truncated_len); + ret = OB_SUCCESS; + } else if (OB_SIZE_OVERFLOW != ret) { + LOG_WARN("failed to print lob", K(ret)); + } else if (OB_FAIL(flush_shared_buf(data_writer, get_flush_function(), true))) { + LOG_WARN("failed to flush shared buffer", K(ret)); + } else if (OB_FALSE_IT(escape_printer_.pos_ = 0)) { + } else if (OB_FAIL(ObFastStringScanner::foreach_char(src_block_data, + src_type, + escape_printer_, + escape_printer_.do_encode_, + escape_printer_.ignore_convert_failed_, + stop_when_truncated, + &truncated_len))) { + if (OB_ERR_DATA_TRUNCATED == ret && stop_when_truncated) { + lob_iter.set_reserved_byte_len(truncated_len); + ret = OB_SUCCESS; + } else { + LOG_WARN("failed to print lob", K(ret), K(src_block_data.length()), K(shared_buf_len_), + K(data_writer.get_curr_pos()), K(escape_printer_.buf_len_), K(escape_printer_.pos_)); + } + } + } } + data_writer.set_curr_pos(escape_printer_.pos_); } if (OB_FAIL(ret)) { } else if (state != TEXTSTRING_ITER_NEXT && state != TEXTSTRING_ITER_END) { @@ -805,72 +1147,101 @@ int ObSelectIntoOp::write_lob_to_file(const ObObj &obj, const ObExpr &expr, cons return ret; } -int ObSelectIntoOp::write_single_char_to_file(const char *wchar) +int ObSelectIntoOp::write_single_char_to_file(const char *wchar, ObIOBufferWriter &data_writer) { int ret = OB_SUCCESS; char* buf = NULL; int64_t buf_len = 0; int64_t pos = 0; - get_buf(buf, buf_len, pos); - if (pos == buf_len && OB_FAIL(flush_buf(pos))) { - LOG_WARN("failed to flush buffer", K(ret)); - } else if (pos < buf_len) { - MEMCPY(buf + pos, wchar, 1); - data_writer_.set_curr_pos(pos + 1); - } else if (OB_FAIL(resize_buf(buf, buf_len, pos))) { - LOG_WARN("failed to resize buffer", K(ret)); - } else if (pos < buf_len) { - MEMCPY(buf + pos, wchar, 1); - data_writer_.set_curr_pos(pos + 1); - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret)); + OZ(get_buf(buf, buf_len, pos, data_writer)); + if (OB_SUCC(ret) && !use_shared_buf_) { + if (pos < buf_len) { + MEMCPY(buf + pos, wchar, 1); + data_writer.set_curr_pos(pos + 1); + } else if (OB_FAIL(flush_buf(data_writer))) { + LOG_WARN("failed to flush buffer", K(ret)); + } else if (OB_FALSE_IT(pos = data_writer.get_curr_pos())) { + } else if (pos < buf_len) { + MEMCPY(buf + pos, wchar, 1); + data_writer.set_curr_pos(pos + 1); + } else if (OB_FAIL(use_shared_buf(data_writer, buf, buf_len, pos))) { + LOG_WARN("failed to use shared buffer", K(ret)); + } + } + if (OB_SUCC(ret) && use_shared_buf_) { + if (pos < buf_len) { + MEMCPY(buf + pos, wchar, 1); + data_writer.set_curr_pos(pos + 1); + } else if (OB_FAIL(resize_or_flush_shared_buf(data_writer, buf, buf_len, pos))) { + LOG_WARN("failed to resize or flush shared buffer", K(ret)); + } else if (pos < buf_len) { + MEMCPY(buf + pos, wchar, 1); + data_writer.set_curr_pos(pos + 1); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret)); + } } return ret; } -int ObSelectIntoOp::print_lob_field(const ObObj &obj, const ObExpr &expr, const ObDatum &datum) +int ObSelectIntoOp::print_lob_field(const ObObj &obj, + const ObExpr &expr, + const ObDatum &datum, + ObIOBufferWriter &data_writer) { int ret = OB_SUCCESS; if (has_enclose_) { - OZ(write_single_char_to_file(&char_enclose_)); + OZ(write_single_char_to_file(&char_enclose_, data_writer)); } - OZ(write_lob_to_file(obj, expr, datum)); + OZ(write_lob_to_file(obj, expr, datum, data_writer)); if (has_enclose_) { - OZ(write_single_char_to_file(&char_enclose_)); + OZ(write_single_char_to_file(&char_enclose_, data_writer)); } return ret; } -int ObSelectIntoOp::print_field(const ObObj &obj) +int ObSelectIntoOp::print_field(const ObObj &obj, ObIOBufferWriter &data_writer) { int ret = OB_SUCCESS; char char_n = 'N'; const bool need_enclose = has_enclose_ && !obj.is_null() && (!MY_SPEC.is_optional_ || obj.is_string_type()); if (need_enclose) { - OZ(write_single_char_to_file(&char_enclose_)); + OZ(write_single_char_to_file(&char_enclose_, data_writer)); } if (!has_escape_) { - OZ(write_obj_to_file(obj, false)); + OZ(write_obj_to_file(obj, data_writer, false)); } else if (obj.is_null()) { - OZ(write_single_char_to_file(&char_escape_)); - OZ(write_single_char_to_file(&char_n)); + OZ(write_single_char_to_file(&char_escape_, data_writer)); + OZ(write_single_char_to_file(&char_n, data_writer)); } else { - OZ(write_obj_to_file(obj, true)); + OZ(write_obj_to_file(obj, data_writer, true)); } if (need_enclose) { - OZ(write_single_char_to_file(&char_enclose_)); + OZ(write_single_char_to_file(&char_enclose_, data_writer)); } return ret; } -int ObSelectIntoOp::into_outfile() +int ObSelectIntoOp::into_outfile(ObIOBufferWriter *data_writer) { int ret = OB_SUCCESS; const ObIArray &select_exprs = MY_SPEC.select_exprs_; ObDatum *datum = NULL; ObObj obj; + ObDatum *partition_datum = NULL; + if (do_partition_) { + if (OB_FAIL(MY_SPEC.file_partition_expr_->eval(eval_ctx_, partition_datum))) { + LOG_WARN("eval expr failed", K(ret)); + } else if (OB_FAIL(get_data_writer_for_partition(partition_datum, data_writer))) { + LOG_WARN("failed to set data writer for partition", K(ret)); + } + } + if (OB_SUCC(ret) && OB_ISNULL(data_writer)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null data writer", K(ret)); + } for (int64_t i = 0; OB_SUCC(ret) && i < select_exprs.count(); ++i) { if (OB_ISNULL(select_exprs.at(i))) { ret = OB_ERR_UNEXPECTED; @@ -885,29 +1256,728 @@ int ObSelectIntoOp::into_outfile() select_exprs.at(i)->obj_datum_map_))) { LOG_WARN("failed to get obj from datum", K(ret)); } else if (!ob_is_text_tc(select_exprs.at(i)->obj_meta_.get_type())) { - OZ(print_field(obj)); + OZ(print_field(obj, *data_writer)); } else { // text tc - OZ(print_lob_field(obj, *select_exprs.at(i), *datum)); + OZ(print_lob_field(obj, *select_exprs.at(i), *datum, *data_writer)); } // print field terminator if (OB_SUCC(ret) && i != select_exprs.count() - 1) { - OZ(write_obj_to_file(MY_SPEC.field_str_)); + OZ(write_obj_to_file(MY_SPEC.field_str_, *data_writer)); } } // print line terminator - OZ(write_obj_to_file(MY_SPEC.line_str_)); + OZ(write_obj_to_file(MY_SPEC.line_str_, *data_writer)); // check if need split file - OZ(try_split_file()); + OZ(try_split_file(*data_writer)); + // clear shared buffer + OZ(flush_shared_buf(*data_writer, get_flush_function())); return ret; } -int ObSelectIntoOp::into_outfile_batch(const ObBatchRows &brs) +#ifdef OB_BUILD_CPP_ODPS +int ObSelectIntoOp::into_odps() +{ + int ret = OB_SUCCESS; + const ObIArray &select_exprs = MY_SPEC.select_exprs_; + apsara::odps::sdk::ODPSTableRecordPtr table_record; + ObDatum *datum = NULL; + try { + if (OB_UNLIKELY(!upload_ || !record_writer_ || !(table_record = upload_->CreateBufferRecord()) + || !(table_record->GetSchema()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted null ptr", K(ret)); + } else if (table_record->GetSchema()->GetColumnCount() != select_exprs.count()) { + ret = OB_NOT_SUPPORTED; + LOG_USER_WARN(OB_NOT_SUPPORTED, "insert into partial column in external table"); + LOG_WARN("column count of odps record is not equal to count of select exprs", + K(table_record->GetSchema()->GetColumnCount()), K(select_exprs.count())); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < select_exprs.count(); ++i) { + if (OB_ISNULL(select_exprs.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("select expr is unexpected null", K(ret)); + } else if (OB_FAIL(select_exprs.at(i)->eval(eval_ctx_, datum))) { + LOG_WARN("eval expr failed", K(ret)); + } else if (OB_ISNULL(datum)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("datum is unexpected null", K(ret)); + } else if (lib::is_mysql_mode() + && OB_FAIL(set_odps_column_value_mysql(*table_record, *datum, + select_exprs.at(i)->datum_meta_, + select_exprs.at(i)->obj_meta_, + i))) { + LOG_WARN("failed to set odps column value", K(ret)); + } else if (lib::is_oracle_mode() + && OB_FAIL(set_odps_column_value_oracle(*table_record, *datum, + select_exprs.at(i)->datum_meta_, + select_exprs.at(i)->obj_meta_, + i))) { + LOG_WARN("failed to set odps column value", K(ret)); + } + } + record_writer_->Write(*table_record); + } + } catch (apsara::odps::sdk::OdpsException& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when write one row to odps", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (const std::exception& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when write one row to odps", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (...) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when write one row to odps", K(ret)); + } + } + return ret; +} + +int ObSelectIntoOp::into_odps_batch(const ObBatchRows &brs) +{ + int ret = OB_SUCCESS; + const ObIArray &select_exprs = MY_SPEC.select_exprs_; + ObArray datum_vectors; + ObDatum *datum = NULL; + apsara::odps::sdk::ODPSTableRecordPtr table_record; + for (int64_t i = 0; OB_SUCC(ret) && i < select_exprs.count(); ++i) { + if (OB_ISNULL(select_exprs.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("select expr is unexpected null", K(ret)); + } else if (OB_FAIL(select_exprs.at(i)->eval_batch(eval_ctx_, *brs.skip_, brs.size_))) { + LOG_WARN("failed to eval batch", K(ret), KPC(select_exprs.at(i))); + } else if (OB_FAIL(datum_vectors.push_back(select_exprs.at(i)->locate_expr_datumvector(eval_ctx_)))) { + LOG_WARN("failed to push back datum vector", K(ret)); + } + } + try { + if (OB_FAIL(ret)) { + } else if (OB_UNLIKELY(!upload_ || !record_writer_ + || !(table_record = upload_->CreateBufferRecord()) + || !(table_record->GetSchema()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted null ptr", K(ret)); + } else if (table_record->GetSchema()->GetColumnCount() != select_exprs.count()) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "insert into partial column in external table"); + LOG_WARN("column count of odps record is not equal to count of select exprs", + K(table_record->GetSchema()->GetColumnCount()), K(select_exprs.count())); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < brs.size_; ++i) { + if (brs.skip_->contain(i)) { + // do nothing + } else { + for (int64_t j = 0; OB_SUCC(ret) && j < select_exprs.count(); ++j) { + if (OB_ISNULL(datum = datum_vectors.at(j).at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("datum is unexpected null", K(ret)); + } else if (lib::is_mysql_mode() + && OB_FAIL(set_odps_column_value_mysql(*table_record, *datum, + select_exprs.at(j)->datum_meta_, + select_exprs.at(j)->obj_meta_, + j))) { + LOG_WARN("failed to set odps column value", K(ret)); + } else if (lib::is_oracle_mode() + && OB_FAIL(set_odps_column_value_oracle(*table_record, *datum, + select_exprs.at(j)->datum_meta_, + select_exprs.at(j)->obj_meta_, + j))) { + LOG_WARN("failed to set odps column value", K(ret)); + } + } + record_writer_->Write(*table_record); + } + } + } + } catch (apsara::odps::sdk::OdpsException& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when write one batch to odps", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (const std::exception& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when write one batch to odps", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (...) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when write one batch to odps", K(ret)); + } + } + return ret; +} + +int ObSelectIntoOp::set_odps_column_value_mysql(apsara::odps::sdk::ODPSTableRecord &table_record, + const ObDatum &datum, + const ObDatumMeta &datum_meta, + const ObObjMeta &obj_meta, + uint32_t col_idx) +{ + int ret = OB_SUCCESS; + ObObjType ob_type = datum_meta.get_type(); + apsara::odps::sdk::ODPSColumnType odps_type; + uint32_t res_len = 0; + char *buf = NULL; + int64_t buf_size = 0; + ObArenaAllocator allocator("IntoOdps", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); + ObMallocHookAttrGuard guard(ObMemAttr(MTL_ID(), "IntoOdps")); + try { + if (OB_UNLIKELY(!(table_record.GetSchema()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted null ptr", K(ret)); + } else if (datum.is_null()) { + table_record.SetNullValue(col_idx); + } else { + odps_type = table_record.GetSchema()->GetTableColumn(col_idx).GetType(); + switch (odps_type) + { + case apsara::odps::sdk::ODPS_BOOLEAN: + { + if (ObTinyIntType == ob_type) { + table_record.SetBoolValue(col_idx, datum.get_tinyint() != 0); + } else if (ObSmallIntType == ob_type) { + table_record.SetBoolValue(col_idx, datum.get_smallint() != 0); + } else if (ObMediumIntType == ob_type || ObInt32Type == ob_type) { + table_record.SetBoolValue(col_idx, datum.get_int32() != 0); + } else if (ObIntType == ob_type) { + table_record.SetBoolValue(col_idx, datum.get_int() != 0); + } + break; + } + case apsara::odps::sdk::ODPS_TINYINT: + { + table_record.SetTinyIntValue(col_idx, datum.get_tinyint()); + break; + } + case apsara::odps::sdk::ODPS_SMALLINT: + { + table_record.SetSmallIntValue(col_idx, datum.get_smallint()); + break; + } + case apsara::odps::sdk::ODPS_INTEGER: + { + table_record.SetIntegerValue(col_idx, datum.get_int32()); + break; + } + case apsara::odps::sdk::ODPS_BIGINT: + { + table_record.SetBigIntValue(col_idx, datum.get_int()); + break; + } + case apsara::odps::sdk::ODPS_FLOAT: + { + table_record.SetFloatValue(col_idx, datum.get_float()); + break; + } + case apsara::odps::sdk::ODPS_DOUBLE: + { + table_record.SetDoubleValue(col_idx, datum.get_double()); + break; + } + case apsara::odps::sdk::ODPS_DECIMAL: + { + std::string dec; + if (OB_FAIL(decimal_to_string(datum, datum_meta, dec, allocator))) { + LOG_WARN("failed to get string", K(ret)); + } else { + table_record.SetDecimalValue(col_idx, dec); + } + break; + } + case apsara::odps::sdk::ODPS_CHAR: + case apsara::odps::sdk::ODPS_VARCHAR: + { + buf_size = datum.get_string().length() * ObCharset::MAX_MB_LEN; + if (CHARSET_UTF8MB4 == ObCharset::charset_type_by_coll(datum_meta.cs_type_)) { + res_len = static_cast(datum.get_string().length()); + buf = const_cast(datum.get_string().ptr()); + } else if (OB_ISNULL(buf = static_cast(allocator.alloc(buf_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory", K(ret)); + } else if (OB_FAIL(ObCharset::charset_convert(datum_meta.cs_type_, + datum.get_string().ptr(), + datum.get_string().length(), + CS_TYPE_UTF8MB4_BIN, + buf, + buf_size, + res_len, + false, + false))) { + LOG_WARN("failed to convert charset", K(ret)); + } + if (OB_FAIL(ret)) { + } else if ((apsara::odps::sdk::ODPS_CHAR == odps_type && res_len > 255) + || (apsara::odps::sdk::ODPS_VARCHAR == odps_type && res_len > 65535)) { + ret = OB_DATA_OUT_OF_RANGE; + LOG_WARN("string length out of range", K(res_len)); + } else if (buf == NULL && res_len == 0) { + table_record.SetStringValue(col_idx, "", res_len, odps_type); + } else { + table_record.SetStringValue(col_idx, buf, res_len, odps_type); + } + break; + } + case apsara::odps::sdk::ODPS_STRING: + case apsara::odps::sdk::ODPS_BINARY: + { + ObString lob_str; + if (OB_FAIL(ObTextStringHelper::read_real_string_data(allocator, + datum, + datum_meta, + obj_meta.has_lob_header(), + lob_str, + &ctx_))) { + LOG_WARN("failed to read string", K(ret)); + } else if (apsara::odps::sdk::ODPS_BINARY == odps_type + || CHARSET_UTF8MB4 == ObCharset::charset_type_by_coll(datum_meta.cs_type_) + || CS_TYPE_BINARY == datum_meta.cs_type_) { + res_len = static_cast(lob_str.length()); + buf = const_cast(lob_str.ptr()); + } else if (OB_FALSE_IT(buf_size = lob_str.length() * ObCharset::MAX_MB_LEN)) { + } else if (OB_ISNULL(buf = static_cast(allocator.alloc(buf_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory", K(ret)); + } else if (OB_FAIL(ObCharset::charset_convert(datum_meta.cs_type_, + lob_str.ptr(), + lob_str.length(), + CS_TYPE_UTF8MB4_BIN, + buf, + buf_size, + res_len, + false, + false))) { + LOG_WARN("failed to convert charset", K(ret)); + } + if (OB_FAIL(ret)) { + } else if (res_len > 8 * 1024 * 1024) { + ret = OB_DATA_OUT_OF_RANGE; + LOG_WARN("string length out of range", K(res_len)); + } else if (buf == NULL && res_len == 0) { + table_record.SetStringValue(col_idx, "", res_len, odps_type); + } else { + LOG_DEBUG("debug select into lob", K(datum_meta.cs_type_), K(ObString(res_len, buf))); + table_record.SetStringValue(col_idx, buf, res_len, odps_type); + } + break; + } + case apsara::odps::sdk::ODPS_JSON: + { + ObString json_str; + ObIJsonBase *j_base = NULL; + ObJsonBuffer jbuf(&allocator); + ObJsonInType in_type = ObJsonInType::JSON_BIN; + uint32_t parse_flag = lib::is_mysql_mode() ? 0 : ObJsonParser::JSN_RELAXED_FLAG; + if (OB_FAIL(ObTextStringHelper::read_real_string_data(allocator, + datum, + datum_meta, + obj_meta.has_lob_header(), + json_str, + &ctx_))) { + LOG_WARN("failed to read string", K(ret)); + } else if (OB_FAIL(ObJsonBaseFactory::get_json_base(&allocator, json_str, in_type, + in_type, j_base, parse_flag))) { + COMMON_LOG(WARN, "fail to get json base", K(ret), K(in_type)); + } else if (OB_FAIL(j_base->print(jbuf, false))) { // json binary to string + COMMON_LOG(WARN, "fail to convert json to string", K(ret)); + } else if (jbuf.length() > UINT32_MAX) { + ret = OB_DATA_OUT_OF_RANGE; + LOG_WARN("data out of range", K(odps_type), K(jbuf.length()), K(ret)); + } else { + LOG_DEBUG("debug select into json", K(datum_meta.cs_type_), K(ObString(jbuf.length(), jbuf.ptr()))); + table_record.SetJsonValue(col_idx, jbuf.ptr(), static_cast(jbuf.length())); + } + break; + } + case apsara::odps::sdk::ODPS_TIMESTAMP: + case apsara::odps::sdk::ODPS_TIMESTAMP_NTZ: + { + int64_t us = apsara::odps::sdk::ODPS_TIMESTAMP == odps_type + ? datum.get_timestamp() + : datum.get_datetime(); + int64_t sec = us / 1000000; + int32_t ns = (us % 1000000) * 1000; + if (us < ORACLE_DATETIME_MIN_VAL) { + ret = OB_DATETIME_FUNCTION_OVERFLOW; + LOG_WARN("odps timestamp min value is 0001-01-01 00:00:00", K(ret), K(us)); + } else { + table_record.SetTimeValue(col_idx, sec, ns, odps_type); + } + break; + } + case apsara::odps::sdk::ODPS_DATE: + { + if (datum.get_date() < ODPS_DATE_MIN_VAL) { + ret = OB_DATETIME_FUNCTION_OVERFLOW; + LOG_WARN("odps date min value is 0001-01-01", K(ret)); + } else { + table_record.SetDateValue(col_idx, datum.get_date()); + } + break; + } + case apsara::odps::sdk::ODPS_DATETIME: + { + int32_t tmp_offset = 0; + if (OB_ISNULL(ctx_.get_my_session()) || OB_ISNULL(ctx_.get_my_session()->get_timezone_info())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (OB_FAIL(ctx_.get_my_session()->get_timezone_info()->get_timezone_offset(0, tmp_offset))) { + LOG_WARN("failed to get timezone offset", K(ret)); + } else if (datum.get_datetime() < ORACLE_DATETIME_MIN_VAL + SEC_TO_USEC(tmp_offset)) { + ret = OB_DATETIME_FUNCTION_OVERFLOW; + LOG_WARN("odps datetime min value is 0001-01-01 00:00:00", K(ret)); + } else { + table_record.SetDatetimeValue(col_idx, (datum.get_datetime() - SEC_TO_USEC(tmp_offset)) / 1000); + } + break; + } + default: + { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected type", K(ob_type), K(odps_type), K(ret)); + } + } + } + } catch (apsara::odps::sdk::OdpsException& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when set odps column value mysql", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (const std::exception& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when set odps column value mysql", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (...) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when set odps column value mysql", K(ret)); + } + } + return ret; +} + +int ObSelectIntoOp::set_odps_column_value_oracle(apsara::odps::sdk::ODPSTableRecord &table_record, + const ObDatum &datum, + const ObDatumMeta &datum_meta, + const ObObjMeta &obj_meta, + uint32_t col_idx) +{ + int ret = OB_SUCCESS; + ObObjType ob_type = datum_meta.get_type(); + apsara::odps::sdk::ODPSColumnType odps_type; + int64_t int_value = 0; + uint32_t res_len = 0; + char *buf = NULL; + int64_t buf_size = 0; + ObArenaAllocator allocator("IntoOdps", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); + ObMallocHookAttrGuard guard(ObMemAttr(MTL_ID(), "IntoOdps")); + try { + if (OB_UNLIKELY(!(table_record.GetSchema()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted null ptr", K(ret)); + } else if (datum.is_null()) { + table_record.SetNullValue(col_idx); + } else { + odps_type = table_record.GetSchema()->GetTableColumn(col_idx).GetType(); + switch (odps_type) + { + case apsara::odps::sdk::ODPS_BOOLEAN: + { + if (OB_FAIL(decimal_or_number_to_int64(datum, datum_meta, int_value))) { + LOG_WARN("failed to get int64", K(ret)); + } else { + table_record.SetBoolValue(col_idx, int_value != 0); + } + break; + } + case apsara::odps::sdk::ODPS_TINYINT: + { + if (OB_FAIL(decimal_or_number_to_int64(datum, datum_meta, int_value))) { + LOG_WARN("failed to get int64", K(ret)); + } else if (int_value < INT8_MIN || int_value > INT8_MAX) { + ret = OB_DATA_OUT_OF_RANGE; + LOG_WARN("data out of range", K(odps_type), K(ret)); + } else { + table_record.SetTinyIntValue(col_idx, static_cast(int_value)); + } + break; + } + case apsara::odps::sdk::ODPS_SMALLINT: + { + if (OB_FAIL(decimal_or_number_to_int64(datum, datum_meta, int_value))) { + LOG_WARN("failed to get int64", K(ret)); + } else if (int_value < INT16_MIN || int_value > INT16_MAX) { + ret = OB_DATA_OUT_OF_RANGE; + LOG_WARN("data out of range", K(odps_type), K(ret)); + } else { + table_record.SetSmallIntValue(col_idx, static_cast(int_value)); + } + break; + } + case apsara::odps::sdk::ODPS_INTEGER: + { + if (OB_FAIL(decimal_or_number_to_int64(datum, datum_meta, int_value))) { + LOG_WARN("failed to get int64", K(ret)); + } else if (int_value < INT32_MIN || int_value > INT32_MAX) { + ret = OB_DATA_OUT_OF_RANGE; + LOG_WARN("data out of range", K(odps_type), K(ret)); + } else { + table_record.SetIntegerValue(col_idx, static_cast(int_value)); + } + break; + } + case apsara::odps::sdk::ODPS_BIGINT: + { + if (OB_FAIL(decimal_or_number_to_int64(datum, datum_meta, int_value))) { + LOG_WARN("failed to get int64", K(ret)); + } else { + table_record.SetBigIntValue(col_idx, int_value); + } + break; + } + case apsara::odps::sdk::ODPS_FLOAT: + { + table_record.SetFloatValue(col_idx, datum.get_float()); + break; + } + case apsara::odps::sdk::ODPS_DOUBLE: + { + table_record.SetDoubleValue(col_idx, datum.get_double()); + break; + } + case apsara::odps::sdk::ODPS_DECIMAL: + { + std::string dec; + if (OB_FAIL(decimal_to_string(datum, datum_meta, dec, allocator))) { + LOG_WARN("failed to get string", K(ret)); + } else { + table_record.SetDecimalValue(col_idx, dec); + } + break; + } + case apsara::odps::sdk::ODPS_CHAR: + case apsara::odps::sdk::ODPS_VARCHAR: + { + buf_size = datum.get_string().length() * ObCharset::MAX_MB_LEN; + if (CHARSET_UTF8MB4 == ObCharset::charset_type_by_coll(datum_meta.cs_type_)) { + res_len = static_cast(datum.get_string().length()); + buf = const_cast(datum.get_string().ptr()); + } else if (OB_ISNULL(buf = static_cast(allocator.alloc(buf_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory", K(ret)); + } else if (OB_FAIL(ObCharset::charset_convert(datum_meta.cs_type_, + datum.get_string().ptr(), + datum.get_string().length(), + CS_TYPE_UTF8MB4_BIN, + buf, + buf_size, + res_len, + false, + false))) { + LOG_WARN("failed to convert charset", K(ret)); + } + if (OB_FAIL(ret)) { + } else if ((apsara::odps::sdk::ODPS_CHAR == odps_type && res_len > 255) + || (apsara::odps::sdk::ODPS_VARCHAR == odps_type && res_len > 65535)) { + ret = OB_DATA_OUT_OF_RANGE; + LOG_WARN("string length out of range", K(res_len)); + } else if (buf == NULL && res_len == 0) { + table_record.SetStringValue(col_idx, "", res_len, odps_type); + } else { + table_record.SetStringValue(col_idx, buf, res_len, odps_type); + } + break; + } + case apsara::odps::sdk::ODPS_STRING: + case apsara::odps::sdk::ODPS_BINARY: + { + ObString lob_str; + if (OB_FAIL(ObTextStringHelper::read_real_string_data(allocator, + datum, + datum_meta, + obj_meta.has_lob_header(), + lob_str, + &ctx_))) { + LOG_WARN("failed to read string", K(ret)); + } else if (apsara::odps::sdk::ODPS_BINARY == odps_type + || CHARSET_UTF8MB4 == ObCharset::charset_type_by_coll(datum_meta.cs_type_) + || CS_TYPE_BINARY == datum_meta.cs_type_) { + res_len = static_cast(lob_str.length()); + buf = const_cast(lob_str.ptr()); + } else if (OB_FALSE_IT(buf_size = lob_str.length() * ObCharset::MAX_MB_LEN)) { + } else if (OB_ISNULL(buf = static_cast(allocator.alloc(buf_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory", K(ret)); + } else if (OB_FAIL(ObCharset::charset_convert(datum_meta.cs_type_, + lob_str.ptr(), + lob_str.length(), + CS_TYPE_UTF8MB4_BIN, + buf, + buf_size, + res_len, + false, + false))) { + LOG_WARN("failed to convert charset", K(ret)); + } + if (OB_FAIL(ret)) { + } else if (res_len > 8 * 1024 * 1024) { + ret = OB_DATA_OUT_OF_RANGE; + LOG_WARN("string length out of range", K(res_len)); + } else if (buf == NULL && res_len == 0) { + table_record.SetStringValue(col_idx, "", res_len, odps_type); + } else { + table_record.SetStringValue(col_idx, buf, res_len, odps_type); + } + break; + } + case apsara::odps::sdk::ODPS_JSON: + { + ObString json_str; + ObIJsonBase *j_base = NULL; + ObJsonBuffer jbuf(&allocator); + ObJsonInType in_type = ObJsonInType::JSON_BIN; + uint32_t parse_flag = lib::is_mysql_mode() ? 0 : ObJsonParser::JSN_RELAXED_FLAG; + if (OB_FAIL(ObTextStringHelper::read_real_string_data(allocator, + datum, + datum_meta, + obj_meta.has_lob_header(), + json_str, + &ctx_))) { + LOG_WARN("failed to read string", K(ret)); + } else if (OB_FAIL(ObJsonBaseFactory::get_json_base(&allocator, json_str, in_type, + in_type, j_base, parse_flag))) { + COMMON_LOG(WARN, "fail to get json base", K(ret), K(in_type)); + } else if (OB_FAIL(j_base->print(jbuf, false))) { // json binary to string + COMMON_LOG(WARN, "fail to convert json to string", K(ret)); + } else if (jbuf.length() > UINT32_MAX) { + ret = OB_DATA_OUT_OF_RANGE; + LOG_WARN("data out of range", K(odps_type), K(jbuf.length()), K(ret)); + } else { + table_record.SetJsonValue(col_idx, jbuf.ptr(), static_cast(jbuf.length())); + } + break; + } + case apsara::odps::sdk::ODPS_TIMESTAMP: + case apsara::odps::sdk::ODPS_TIMESTAMP_NTZ: + { + ObOTimestampData timestamp = datum.get_otimestamp_tiny(); + table_record.SetTimeValue(col_idx, timestamp.time_us_ / 1000000, timestamp.time_ctx_.tail_nsec_, odps_type); + break; + } + case apsara::odps::sdk::ODPS_DATE: + { + table_record.SetDateValue(col_idx, datum.get_datetime() / 1000000 / 3600 / 24); + break; + } + case apsara::odps::sdk::ODPS_DATETIME: + { + ObOTimestampData timestamp = datum.get_otimestamp_tiny(); + int32_t tmp_offset = 0; + if (OB_ISNULL(ctx_.get_my_session()) || OB_ISNULL(ctx_.get_my_session()->get_timezone_info())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (OB_FAIL(ctx_.get_my_session()->get_timezone_info()->get_timezone_offset(0, tmp_offset))) { + LOG_WARN("failed to get timezone offset", K(ret)); + } else { + table_record.SetDatetimeValue(col_idx, (timestamp.time_us_ - SEC_TO_USEC(tmp_offset)) / 1000); + } + break; + } + default: + { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected type", K(ob_type), K(odps_type), K(ret)); + } + } + } + } catch (apsara::odps::sdk::OdpsException& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when set odps column value oracle", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (const std::exception& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when set odps column value oracle", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (...) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when set odps column value oracle", K(ret)); + } + } + return ret; +} +#endif + +int ObSelectIntoOp::decimal_to_string(const ObDatum &datum, + const ObDatumMeta &datum_meta, + std::string &res, + ObIAllocator &allocator) +{ + int ret = OB_SUCCESS; + char *buf = NULL; + int64_t pos = 0; + int32_t int_bytes = wide::ObDecimalIntConstValue::get_int_bytes_by_precision(datum_meta.precision_); + if (OB_ISNULL(buf = static_cast(allocator.alloc(OB_CAST_TO_VARCHAR_MAX_LENGTH)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory", K(ret)); + } else if (OB_FAIL(wide::to_string(datum.get_decimal_int(), datum.get_int_bytes(), datum_meta.scale_, + buf, OB_CAST_TO_VARCHAR_MAX_LENGTH, pos))) { + LOG_WARN("failed to get string", K(ret)); + } else { + res.assign(buf, pos); + } + return ret; +} + +int ObSelectIntoOp::decimal_or_number_to_int64(const ObDatum &datum, + const ObDatumMeta &datum_meta, + int64_t &res) +{ + int ret = OB_SUCCESS; + ObObjType ob_type = datum_meta.get_type(); + if (ObNumberType == ob_type) { + const number::ObNumber nmb(datum.get_number()); + if (OB_FAIL(nmb.extract_valid_int64_with_trunc(res))) { + LOG_WARN("failed to cast number to int64", K(ret)); + } + } else if (ObDecimalIntType == ob_type) { + int32_t int_bytes = wide::ObDecimalIntConstValue::get_int_bytes_by_precision(datum_meta.precision_); + bool is_valid; + if (OB_FAIL(wide::check_range_valid_int64(datum.get_decimal_int(), int_bytes, is_valid, res))) { + LOG_WARN("failed to check decimal int", K(int_bytes), K(ret)); + } else if (!is_valid) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("decimal int is not valid int64", K(ret)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected type", K(ob_type), K(ret)); + } + return ret; +} + +int ObSelectIntoOp::into_outfile_batch(const ObBatchRows &brs, ObIOBufferWriter *data_writer) { int ret = OB_SUCCESS; const ObIArray &select_exprs = MY_SPEC.select_exprs_; ObArray datum_vectors; ObDatum *datum = NULL; ObObj obj; + ObDatumVector partition_datum_vector; for (int64_t i = 0; OB_SUCC(ret) && i < select_exprs.count(); ++i) { if (OB_FAIL(select_exprs.at(i)->eval_batch(eval_ctx_, *brs.skip_, brs.size_))) { LOG_WARN("failed to eval batch", K(ret)); @@ -915,9 +1985,22 @@ int ObSelectIntoOp::into_outfile_batch(const ObBatchRows &brs) LOG_WARN("failed to push back datum vector", K(ret)); } } + if (OB_SUCC(ret) && do_partition_) { + if (OB_FAIL(MY_SPEC.file_partition_expr_->eval_batch(eval_ctx_, *brs.skip_, brs.size_))) { + LOG_WARN("failed to eval batch", K(ret)); + } else { + partition_datum_vector = MY_SPEC.file_partition_expr_->locate_expr_datumvector(eval_ctx_); + } + } for (int64_t i = 0; OB_SUCC(ret) && i < brs.size_; ++i) { if (brs.skip_->contain(i)) { // do nothing + } else if (do_partition_ && OB_FAIL(get_data_writer_for_partition(partition_datum_vector.at(i), + data_writer))) { + LOG_WARN("failed to set data writer for partition", K(ret)); + } else if (OB_ISNULL(data_writer)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null data writer", K(ret)); } else { for (int64_t j = 0; OB_SUCC(ret) && j < select_exprs.count(); ++j) { if (OB_ISNULL(datum = datum_vectors.at(j).at(i))) { @@ -928,41 +2011,46 @@ int ObSelectIntoOp::into_outfile_batch(const ObBatchRows &brs) select_exprs.at(j)->obj_datum_map_))) { LOG_WARN("failed to get obj from datum", K(ret)); } else if (!ob_is_text_tc(select_exprs.at(j)->obj_meta_.get_type())) { - OZ(print_field(obj)); + OZ(print_field(obj, *data_writer)); } else { // text tc - OZ(print_lob_field(obj, *select_exprs.at(j), *datum)); + OZ(print_lob_field(obj, *select_exprs.at(j), *datum, *data_writer)); } // print field terminator if (OB_SUCC(ret) && j != select_exprs.count() - 1) { - OZ(write_obj_to_file(MY_SPEC.field_str_)); + OZ(write_obj_to_file(MY_SPEC.field_str_, *data_writer)); } } // print line terminator - OZ(write_obj_to_file(MY_SPEC.line_str_)); + OZ(write_obj_to_file(MY_SPEC.line_str_, *data_writer)); // check if need split file - OZ(try_split_file()); + OZ(try_split_file(*data_writer)); + // clear shared buffer + OZ(flush_shared_buf(*data_writer, get_flush_function())); } } return ret; } -int ObSelectIntoOp::into_dumpfile() +int ObSelectIntoOp::into_dumpfile(ObIOBufferWriter *data_writer) { int ret = OB_SUCCESS; char buf[MAX_VALUE_LENGTH]; int64_t buf_len = MAX_VALUE_LENGTH; int64_t pos = 0; - if (OB_FAIL(get_row_str(buf_len, is_first_, buf, pos))) { + if (OB_ISNULL(data_writer)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (OB_FAIL(get_row_str(buf_len, is_first_, buf, pos))) { LOG_WARN("get str failed", K(ret)); } else if (is_first_) { // create file - if (OB_FAIL(file_appender_.create(file_name_.get_varchar(), true))) { + if (OB_FAIL(data_writer->file_appender_.create(file_name_.get_varchar(), true))) { LOG_WARN("create dumpfile failed", K(ret), K(file_name_)); } else { is_first_ = false; } } if (OB_SUCC(ret)) { - if (OB_FAIL(file_appender_.append(buf, pos, false))) { + if (OB_FAIL(data_writer->file_appender_.append(buf, pos, false))) { LOG_WARN("failed to append file"); } else { //do nothing @@ -1057,10 +2145,10 @@ int ObSelectIntoOp::prepare_escape_printer() // wc->mb if (OB_ISNULL(buf = static_cast(ctx_.get_allocator().alloc(buf_len)))) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to allocate buffer", K(ret), K(buf_len)); + LOG_WARN("failed to allocate buffer", K(ret), K(buf_len)); } - OZ(print_wchar_to_buf(buf, buf_len, pos, wchar_enclose, escape_printer_.enclose_, MY_SPEC.cs_type_)); - OZ(print_wchar_to_buf(buf, buf_len, pos, wchar_escape, escape_printer_.escape_, MY_SPEC.cs_type_)); + OZ(print_wchar_to_buf(buf, buf_len, pos, wchar_enclose, escape_printer_.enclose_, MY_SPEC.cs_type_)); //todo@linyi if has_enclose_ + OZ(print_wchar_to_buf(buf, buf_len, pos, wchar_escape, escape_printer_.escape_, MY_SPEC.cs_type_)); //todo@linyi OZ(print_wchar_to_buf(buf, buf_len, pos, wchar_zero, escape_printer_.zero_, MY_SPEC.cs_type_)); OZ(print_wchar_to_buf(buf, buf_len, pos, wchar_field, escape_printer_.field_terminator_, MY_SPEC.cs_type_)); OZ(print_wchar_to_buf(buf, buf_len, pos, wchar_line, escape_printer_.line_terminator_, MY_SPEC.cs_type_)); @@ -1087,14 +2175,228 @@ int ObSelectIntoOp::check_has_lob_or_json() return ret; } +int ObSelectIntoOp::create_shared_buffer_for_data_writer() +{ + int ret = OB_SUCCESS; + shared_buf_len_ = has_lob_ ? (5 * SHARED_BUFFER_SIZE) : SHARED_BUFFER_SIZE; + if (OB_ISNULL(shared_buf_ = static_cast(ctx_.get_allocator().alloc(shared_buf_len_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate buffer", K(ret), K(shared_buf_len_)); + } + if (OB_SUCC(ret) && has_json_ && has_escape_) { + json_buf_len_ = OB_MALLOC_MIDDLE_BLOCK_SIZE; + if (OB_ISNULL(json_buf_ = static_cast(ctx_.get_allocator().alloc(json_buf_len_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate buffer", K(ret), K(json_buf_len_)); + } + } + return ret; +} + +int ObSelectIntoOp::check_secure_file_path(ObString file_name) +{ + int ret = OB_SUCCESS; + ObString file_path = file_name.split_on(file_name.reverse_find('/')); + char full_path_buf[PATH_MAX+1]; + char *actual_path = nullptr; + ObSqlString sql_str; + ObString secure_file_priv; + int64_t tenant_id = MTL_ID(); + if (OB_FAIL(sql_str.append(file_path.empty() ? "." : file_path))) { + LOG_WARN("failed to append string", K(ret)); + } else if (OB_ISNULL(actual_path = realpath(sql_str.ptr(), full_path_buf))) { + ret = OB_FILE_NOT_EXIST; + LOG_WARN("file not exist", K(ret), K(sql_str)); + } else if (OB_FAIL(ObSchemaUtils::get_tenant_varchar_variable(tenant_id, + SYS_VAR_SECURE_FILE_PRIV, + ctx_.get_allocator(), + secure_file_priv))) { + LOG_WARN("fail get tenant variable", K(tenant_id), K(secure_file_priv), K(ret)); + } else if (OB_FAIL(ObResolverUtils::check_secure_path(secure_file_priv, actual_path))) { + LOG_WARN("failed to check secure path", K(ret), K(secure_file_priv)); + if (OB_ERR_NO_PRIVILEGE == ret) { + ret = OB_ERR_NO_PRIV_DIRECT_PATH_ACCESS; + LOG_ERROR("failed to check secure path", K(ret), K(secure_file_priv)); + } + } + return ret; +} + +int ObSelectIntoOp::get_data_writer_for_partition(ObDatum *partition_datum, + ObIOBufferWriter *&data_writer) +{ + int ret = OB_SUCCESS; + ObString partition; + void *ptr = NULL; + ObIOBufferWriter *value = NULL; + const int64_t buf_len = MY_SPEC.buffer_size_; + char *buf = NULL; + if (OB_ISNULL(partition_datum)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (OB_SUCC(partition_map_.get_refactored(partition_datum->get_string(), value))) { + if (OB_ISNULL(value)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else { + data_writer = value; + } + } else if (OB_UNLIKELY(OB_HASH_NOT_EXIST != ret)){ + LOG_WARN("get unexpected error", K(ret)); + } else if (curr_partition_num_ >= OB_MAX_PARTITION_NUM_ORACLE) { + ret = OB_TOO_MANY_PARTITIONS_ERROR; + LOG_WARN("too many partitions", K(ret)); + } else { + ret = OB_SUCCESS; + //new data_writer + if (OB_ISNULL(ptr = ctx_.get_allocator().alloc(sizeof(ObIOBufferWriter)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate data writer", K(ret), K(sizeof(ObIOBufferWriter))); + } else { + data_writer = new(ptr) ObIOBufferWriter(); + } + //init buffer + if (OB_FAIL(ret) || buf_len <= 0) { + } else if (OB_ISNULL(buf = static_cast(ctx_.get_allocator().alloc(buf_len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate buffer", K(ret), K(buf_len)); + } else { + data_writer->init(buf, buf_len); + } + //add to hashmap + if (OB_FAIL(ret)) { + } else if (OB_FAIL(ob_write_string(ctx_.get_allocator(), + partition_datum->get_string(), + partition))) { + LOG_WARN("failed to write string", K(ret)); + } else if (OB_FAIL(partition_map_.set_refactored(partition, data_writer))) { + LOG_WARN("failed to add data writer to map", K(ret)); + } else { + curr_partition_num_++; + } + if (OB_FAIL(ret) && NULL != data_writer) { + data_writer->~ObIOBufferWriter(); + } + //calc file path + if (OB_SUCC(ret) && OB_FAIL(calc_file_path_with_partition(partition, *data_writer))) { + LOG_WARN("failed to calc file path with partition", K(ret)); + } + } + return ret; +} + +int ObSelectIntoOp::create_the_only_data_writer(ObIOBufferWriter *&data_writer) +{ + int ret = OB_SUCCESS; + void *ptr = NULL; + const int64_t buf_len = MY_SPEC.buffer_size_; + char *buf = NULL; + if (OB_ISNULL(ptr = ctx_.get_allocator().alloc(sizeof(ObIOBufferWriter)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate data writer", K(ret), K(sizeof(ObIOBufferWriter))); + } else { + data_writer = new(ptr) ObIOBufferWriter(); + data_writer->url_ = basic_url_; + data_writer_ = data_writer; + } + if (OB_FAIL(ret)) { + } else if (T_INTO_OUTFILE == MY_SPEC.into_type_ && MY_SPEC.is_single_ + && OB_FAIL(open_file(*data_writer))) { + LOG_WARN("failed to open file", K(ret)); + } else if (buf_len <= 0) { + } else if (OB_ISNULL(buf = static_cast(ctx_.get_allocator().alloc(buf_len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate buffer", K(ret), K(buf_len)); + } else { + data_writer->init(buf, buf_len); + } + if (OB_FAIL(ret) && NULL != data_writer) { + data_writer->~ObIOBufferWriter(); + } + return ret; +} + +#ifdef OB_BUILD_CPP_ODPS +int ObSelectIntoOp::odps_commit_upload() +{ + int ret = OB_SUCCESS; + bool is_in_px = (NULL != ctx_.get_sqc_handler()); + if (is_in_px) { + ObOdpsPartitionDownloaderMgr &odps_mgr = ctx_.get_sqc_handler()->get_sqc_ctx().gi_pump_.get_odps_mgr(); + if (!need_commit_) { + odps_mgr.set_fail(); + } + __sync_synchronize(); + int64_t ref = odps_mgr.dec_ref(); + if (0 > ref) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected ref", K(ref), K(ret)); + } else if (0 == ref && OB_FAIL(odps_mgr.commit_upload())) { + LOG_WARN("failed to commit upload", K(ret)); + } + } else { + std::vector blocks; + try { + if (OB_UNLIKELY(!record_writer_ || !upload_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted null ptr", K(ret)); + } else { + record_writer_->Close(); + blocks.push_back(block_id_); + if (need_commit_) { + upload_->Commit(blocks); + } + } + } catch (apsara::odps::sdk::OdpsException& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when commit", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (const std::exception& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when commit", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (...) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when commit", K(ret)); + } + } + } + return ret; +} +#endif + void ObSelectIntoOp::destroy() { - file_appender_.~ObFileAppender(); - close_file(); + ObIOBufferWriter *data_writer = NULL; + if (ObExternalFileFormat::FormatType::ODPS_FORMAT == format_type_) { +#ifdef OB_BUILD_CPP_ODPS + upload_.reset(); + record_writer_.reset(); +#endif + } else if (do_partition_) { + for (ObPartitionWriterMap::iterator iter = partition_map_.begin(); + iter != partition_map_.end(); iter++) { + if (OB_ISNULL(data_writer = iter->second)) { + } else { + close_file(*data_writer); + data_writer->~ObIOBufferWriter(); + } + } + } else if (OB_NOT_NULL(data_writer_)) { + close_file(*data_writer_); + data_writer_->~ObIOBufferWriter(); + } if (NULL != device_handle_) { common::ObDeviceManager::get_instance().release_device(device_handle_); device_handle_ = NULL; } + external_properties_.~ObExternalFileFormat(); + partition_map_.destroy(); ObOperator::destroy(); } diff --git a/src/sql/engine/basic/ob_select_into_op.h b/src/sql/engine/basic/ob_select_into_op.h index f6e9a4bff..cd8961b3b 100644 --- a/src/sql/engine/basic/ob_select_into_op.h +++ b/src/sql/engine/basic/ob_select_into_op.h @@ -17,6 +17,11 @@ #include "lib/file/ob_file.h" #include "common/storage/ob_io_device.h" #include "share/backup/ob_backup_struct.h" +#include "sql/engine/cmd/ob_load_data_parser.h" +#ifdef OB_BUILD_CPP_ODPS +#include +#include +#endif namespace oceanbase { @@ -109,22 +114,36 @@ public: ObSelectIntoOp(ObExecContext &exec_ctx, const ObOpSpec &spec, ObOpInput *input) : ObOperator(exec_ctx, spec, input), top_limit_cnt_(INT64_MAX), - file_appender_(), is_first_(true), + basic_url_(), device_handle_(NULL), file_location_(IntoFileLocation::SERVER_DISK), write_offset_(0), - write_bytes_(0), - split_file_id_(0), + data_writer_(NULL), char_enclose_(0), char_escape_(0), has_enclose_(false), has_escape_(false), has_lob_(false), has_json_(false), - is_file_opened_(false), print_params_(), - escape_printer_() + escape_printer_(), + do_partition_(false), + json_buf_(NULL), + json_buf_len_(0), + shared_buf_(NULL), + shared_buf_len_(0), + use_shared_buf_(false), + partition_map_(), + curr_partition_num_(0), + external_properties_(), + format_type_(ObExternalFileFormat::FormatType::CSV_FORMAT), +#ifdef OB_BUILD_CPP_ODPS + upload_(NULL), + record_writer_(NULL), +#endif + block_id_(0), + need_commit_(true) { } @@ -187,20 +206,30 @@ public: { public: ObIOBufferWriter(): - buf_(NULL), curr_pos_(0), last_line_pos_(0), buf_len_(0), curr_line_len_(0) {} + buf_(NULL), + buf_len_(0), + curr_pos_(0), + last_line_pos_(0), + curr_line_len_(0), + write_bytes_(0), + is_file_opened_(false), + file_appender_(), + fd_(), + split_file_id_(0), + url_() + {} + ~ObIOBufferWriter() { + file_appender_.~ObFileAppender(); + } void init(char *buf, int64_t buf_len) { buf_ = buf; buf_len_ = buf_len; } - void init_json_buf(char *buf, int64_t buf_len) { - json_buf_ = buf; - json_buf_len_ = buf_len; - } template int flush(flush_func flush_data) { int ret = common::OB_SUCCESS; - if (last_line_pos_ > 0) { - if (OB_FAIL(flush_data(buf_, last_line_pos_))) { + if (last_line_pos_ > 0 && OB_NOT_NULL(buf_)) { + if (OB_FAIL(flush_data(buf_, last_line_pos_, this))) { } else { MEMCPY(buf_, buf_ + last_line_pos_, curr_pos_ - last_line_pos_); curr_pos_ = curr_pos_ - last_line_pos_; @@ -209,37 +238,30 @@ public: } return ret; } - template - int flush_all_for_lob(flush_func flush_data) { - int ret = common::OB_SUCCESS; - if (curr_pos_ > 0) { - if (OB_FAIL(flush_data(buf_, curr_pos_))) { - } else { - curr_line_len_ += (curr_pos_ - last_line_pos_); - curr_pos_ = 0; - last_line_pos_ = 0; - } - } - return ret; - } char *get_buf() { return buf_; } int64_t get_buf_len() { return buf_len_; } - char *get_json_buf() { return json_buf_; } - int64_t get_json_buf_len() { return json_buf_len_; } int64_t get_curr_pos() { return curr_pos_; } int64_t get_last_line_pos() { return last_line_pos_; } int64_t get_curr_line_len() {return curr_line_len_; } + int64_t get_write_bytes() { return write_bytes_; } void set_curr_pos(int64_t curr_pos) { curr_pos_ = curr_pos; } void update_last_line_pos() { last_line_pos_ = curr_pos_; } - void reset_curr_line_len() {curr_line_len_ = 0; } + void reset_curr_line_len() { curr_line_len_ = 0; } + void increase_curr_line_len() { curr_line_len_ += (curr_pos_ - last_line_pos_); } + void set_write_bytes(int64_t write_bytes) { write_bytes_ = write_bytes; } private: char *buf_; + int64_t buf_len_; int64_t curr_pos_; int64_t last_line_pos_; - int64_t buf_len_; int64_t curr_line_len_; - char *json_buf_; //json需要多一个buffer用来放转义前的string - int64_t json_buf_len_; + int64_t write_bytes_; + public: + bool is_file_opened_; + ObFileAppender file_appender_; + ObIOFd fd_; + int64_t split_file_id_; + ObString url_; }; virtual int inner_open() override; @@ -248,24 +270,34 @@ public: virtual int inner_get_next_row() override; virtual int inner_get_next_batch(const int64_t max_row_cnt) override; virtual void destroy() override; - void reset() - { - is_first_ = true; - file_appender_.close(); - device_handle_ = NULL; - file_location_ = IntoFileLocation::SERVER_DISK; - write_offset_ = 0; - write_bytes_ = 0; - split_file_id_ = 0; - data_writer_.init(NULL, 0); - is_file_opened_ = false; - } private: + int init_csv_env(); +#ifdef OB_BUILD_CPP_ODPS + int init_odps_tunnel(); + int into_odps(); + int into_odps_batch(const ObBatchRows &brs); + int odps_commit_upload(); + int set_odps_column_value_mysql(apsara::odps::sdk::ODPSTableRecord &table_record, + const ObDatum &datum, + const ObDatumMeta &datum_meta, + const ObObjMeta &obj_meta, + uint32_t col_idx); + int set_odps_column_value_oracle(apsara::odps::sdk::ODPSTableRecord &table_record, + const ObDatum &datum, + const ObDatumMeta &datum_meta, + const ObObjMeta &obj_meta, + uint32_t col_idx); +#endif + int decimal_or_number_to_int64(const ObDatum &datum, const ObDatumMeta &datum_meta, int64_t &res); + int decimal_to_string(const ObDatum &datum, + const ObDatumMeta &datum_meta, + std::string &res, + ObIAllocator &allocator); int get_row_str(const int64_t buf_len, bool is_first_row, char *buf, int64_t &pos); - int into_dumpfile(); - int into_outfile(); - int into_outfile_batch(const ObBatchRows &brs); + int into_dumpfile(ObIOBufferWriter *data_writer); + int into_outfile(ObIOBufferWriter *data_writer); + int into_outfile_batch(const ObBatchRows &brs, ObIOBufferWriter *data_writer); int extract_fisrt_wchar_from_varhcar(const ObObj &obj, int32_t &wchar); int print_wchar_to_buf(char *buf, const int64_t buf_len, @@ -273,55 +305,117 @@ private: int32_t wchar, ObString &str, ObCollationType coll_type); - int print_field(const ObObj &obj); - int print_lob_field(const ObObj &obj, const ObExpr &expr, const ObDatum &datum); - void get_buf(char* &buf, int64_t &buf_len, int64_t &pos, bool is_json = false); - int flush_buf(int64_t &pos); - int resize_buf(char* &buf, int64_t &buf_len, int64_t &pos, bool is_json = false); - int write_obj_to_file(const ObObj &obj, bool need_escape = false); - int print_normal_obj_without_escape(const ObObj &obj, - char* &buf, - int64_t &buf_len, - int64_t &pos, - bool is_json = false); - int write_single_char_to_file(const char *wchar); - int write_lob_to_file(const ObObj &obj, const ObExpr &expr, const ObDatum &datum); - int try_split_file(); + int print_field(const ObObj &obj, ObIOBufferWriter &data_writer); + int print_lob_field(const ObObj &obj, + const ObExpr &expr, + const ObDatum &datum, + ObIOBufferWriter &data_writer); + int get_buf(char* &buf, int64_t &buf_len, int64_t &pos, ObIOBufferWriter &data_writer); + int flush_buf(ObIOBufferWriter &data_writer); + int use_shared_buf(ObIOBufferWriter &data_writer, char* &buf, int64_t &buf_len, int64_t &pos); + template + int flush_shared_buf(ObIOBufferWriter &data_writer, flush_func flush_data, bool continue_use_shared_buf = false) { + int ret = common::OB_SUCCESS; + if (data_writer.get_curr_pos() > 0 && use_shared_buf_) { + if (OB_FAIL(flush_data(shared_buf_, data_writer.get_curr_pos(), &data_writer))) { + } else { + if (has_lob_) { + data_writer.increase_curr_line_len(); + } + data_writer.set_curr_pos(0); + data_writer.update_last_line_pos(); + use_shared_buf_ = continue_use_shared_buf; + } + } + return ret; + } + int resize_buf(char* &buf, + int64_t &buf_len, + int64_t &pos, + int64_t curr_pos, + bool is_json = false); + int resize_or_flush_shared_buf(ObIOBufferWriter &data_writer, + char* &buf, + int64_t &buf_len, + int64_t &pos); + int check_buf_sufficient(ObIOBufferWriter &data_writer, + char* &buf, + int64_t &buf_len, + int64_t &pos, + int64_t str_len); + int write_obj_to_file(const ObObj &obj, ObIOBufferWriter &data_writer, bool need_escape = false); + int print_str_or_json_with_escape(const ObObj &obj, ObIOBufferWriter &data_writer); + int print_normal_obj_without_escape(const ObObj &obj, ObIOBufferWriter &data_writer); + int print_json_to_json_buf(const ObObj &obj, + char* &buf, + int64_t &buf_len, + int64_t &pos, + ObIOBufferWriter &data_writer); + int write_single_char_to_file(const char *wchar, ObIOBufferWriter &data_writer); + int write_lob_to_file(const ObObj &obj, + const ObExpr &expr, + const ObDatum &datum, + ObIOBufferWriter &data_writer); int into_varlist(); - int open_file(); - int calc_next_file_path(); + int open_file(ObIOBufferWriter &data_writer); + int calc_next_file_path(ObIOBufferWriter &data_writer); int calc_first_file_path(ObString &path); - int split_file(); - void close_file(); - std::function get_flush_function(); + int calc_file_path_with_partition(ObString partition, ObIOBufferWriter &data_writer); + int try_split_file(ObIOBufferWriter &data_writer); + int split_file(ObIOBufferWriter &data_writer); + void close_file(ObIOBufferWriter &data_writer); + std::function get_flush_function(); int prepare_escape_printer(); int check_has_lob_or_json(); + int create_shared_buffer_for_data_writer(); + int create_the_only_data_writer(ObIOBufferWriter *&data_writer); + int check_secure_file_path(ObString file_name); + int get_data_writer_for_partition(ObDatum *partition_datum, ObIOBufferWriter *&data_writer); + char *get_json_buf() { return json_buf_; } + int64_t get_json_buf_len() { return json_buf_len_; } + char *get_shared_buf() { return shared_buf_; } + int64_t get_shared_buf_len() { return shared_buf_len_; } private: int64_t top_limit_cnt_; - ObFileAppender file_appender_; bool is_first_; ObObj field_str_; ObObj line_str_; ObObj file_name_; - ObString url_; + ObString basic_url_; // url without partition expr share::ObBackupStorageInfo access_info_; ObIODevice* device_handle_; - ObIOFd fd_; IntoFileLocation file_location_; int64_t write_offset_; - int64_t write_bytes_; - int64_t split_file_id_; - ObIOBufferWriter data_writer_; + ObIOBufferWriter* data_writer_; char char_enclose_; char char_escape_; bool has_enclose_; bool has_escape_; bool has_lob_; bool has_json_; - bool is_file_opened_; common::ObObjPrintParams print_params_; ObEscapePrinter escape_printer_; + bool do_partition_; + char *json_buf_; //json需要多一个buffer用来放转义前的string + int64_t json_buf_len_; + char *shared_buf_; + int64_t shared_buf_len_; + bool use_shared_buf_; + typedef common::hash::ObHashMap ObPartitionWriterMap; + ObPartitionWriterMap partition_map_; + int curr_partition_num_; + ObExternalFileFormat external_properties_; + ObExternalFileFormat::FormatType format_type_; +#ifdef OB_BUILD_CPP_ODPS + apsara::odps::sdk::IUploadPtr upload_; + apsara::odps::sdk::IRecordWriterPtr record_writer_; +#endif + uint32_t block_id_; + bool need_commit_; + static const int64_t SHARED_BUFFER_SIZE = 2LL * 1024 * 1024; + static const int64_t MAX_OSS_FILE_SIZE = 5LL * 1024 * 1024 * 1024; + static const int32_t ODPS_DATE_MIN_VAL = -719162; // '0001-1-1' }; diff --git a/src/sql/engine/cmd/ob_load_data_parser.cpp b/src/sql/engine/cmd/ob_load_data_parser.cpp index 32038712a..a9c642e8b 100644 --- a/src/sql/engine/cmd/ob_load_data_parser.cpp +++ b/src/sql/engine/cmd/ob_load_data_parser.cpp @@ -19,6 +19,10 @@ #include "lib/string/ob_hex_utils_base.h" #include "deps/oblib/src/lib/list/ob_dlist.h" #include "share/schema/ob_column_schema.h" +#ifdef OB_BUILD_CPP_ODPS +#include "share/ob_encryption_util.h" +#endif +#include "lib/utility/ob_print_utils.h" using namespace oceanbase::sql; using namespace oceanbase::common; @@ -32,10 +36,302 @@ const char INVALID_TERM_CHAR = '\xff'; const char * ObExternalFileFormat::FORMAT_TYPE_STR[] = { "CSV", "PARQUET", + "ODPS", "ORC", }; static_assert(array_elements(ObExternalFileFormat::FORMAT_TYPE_STR) == ObExternalFileFormat::MAX_FORMAT, "Not enough initializer for ObExternalFileFormat"); +int64_t ObODPSGeneralFormat::to_json_kv_string(char *buf, const int64_t buf_len) const +{ + int64_t pos = 0; + int64_t idx = 0; + J_COMMA(); + databuff_printf(buf, buf_len, pos, "\"%s\":\"%s\"", OPTION_NAMES[idx++], to_cstring(ObHexStringWrap(access_type_))); + J_COMMA(); + databuff_printf(buf, buf_len, pos, "\"%s\":\"%s\"", OPTION_NAMES[idx++], to_cstring(ObHexStringWrap(access_id_))); + J_COMMA(); + databuff_printf(buf, buf_len, pos, "\"%s\":\"%s\"", OPTION_NAMES[idx++], to_cstring(ObHexStringWrap(access_key_))); + J_COMMA(); + databuff_printf(buf, buf_len, pos, "\"%s\":\"%s\"", OPTION_NAMES[idx++], to_cstring(ObHexStringWrap(sts_token_))); + J_COMMA(); + databuff_printf(buf, buf_len, pos, "\"%s\":\"%s\"", OPTION_NAMES[idx++], to_cstring(ObHexStringWrap(endpoint_))); + J_COMMA(); + databuff_printf(buf, buf_len, pos, "\"%s\":\"%s\"", OPTION_NAMES[idx++], to_cstring(ObHexStringWrap(project_))); + J_COMMA(); + databuff_printf(buf, buf_len, pos, "\"%s\":\"%s\"", OPTION_NAMES[idx++], to_cstring(ObHexStringWrap(schema_))); + J_COMMA(); + databuff_printf(buf, buf_len, pos, "\"%s\":\"%s\"", OPTION_NAMES[idx++], to_cstring(ObHexStringWrap(table_))); + J_COMMA(); + databuff_printf(buf, buf_len, pos, "\"%s\":\"%s\"", OPTION_NAMES[idx++], to_cstring(ObHexStringWrap(quota_))); + J_COMMA(); + databuff_printf(buf, buf_len, pos, "\"%s\":\"%s\"", OPTION_NAMES[idx++], to_cstring(ObHexStringWrap(compression_code_))); + return pos; +} + +int ObODPSGeneralFormat::encrypt_str(common::ObString &src, common::ObString &dst) +{ + int ret = OB_SUCCESS; +#ifdef OB_BUILD_CPP_ODPS + const uint64_t tenant_id = MTL_ID(); + if (src.empty()) { + //do nothing + dst = src; + } else { + char encrypted_string[common::OB_MAX_ENCRYPTED_EXTERNAL_TABLE_PROPERTIES_ITEM_LENGTH] = {0}; + + char hex_buff[common::OB_MAX_ENCRYPTED_EXTERNAL_TABLE_PROPERTIES_ITEM_LENGTH + 1] = {0}; // +1 to reserve space for \0 + int64_t encrypt_len = -1; + if (OB_FAIL(oceanbase::share::ObEncryptionUtil::encrypt_sys_data(tenant_id, + src.ptr(), + src.length(), + encrypted_string, + common::OB_MAX_ENCRYPTED_EXTERNAL_TABLE_PROPERTIES_ITEM_LENGTH, + encrypt_len))) { + + LOG_WARN("fail to encrypt_sys_data", KR(ret), K(src)); + } else if (0 >= encrypt_len || common::OB_MAX_ENCRYPTED_EXTERNAL_TABLE_PROPERTIES_ITEM_LENGTH < encrypt_len * 2) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("encrypt_len is invalid", K(ret), K(encrypt_len), K(common::OB_MAX_ENCRYPTED_EXTERNAL_TABLE_PROPERTIES_ITEM_LENGTH)); + } else if (OB_FAIL(to_hex_cstr(encrypted_string, encrypt_len, hex_buff, common::OB_MAX_ENCRYPTED_EXTERNAL_TABLE_PROPERTIES_ITEM_LENGTH + 1))) { + LOG_WARN("fail to print to hex str", K(ret)); + } else if (OB_FAIL(deep_copy_str(ObString(hex_buff), dst))) { + LOG_WARN("failed to deep copy encrypted_string", K(ret)); + } else { + LOG_TRACE("succ to encrypt src", K(ret)); + } + } +#endif + return ret; +} + +int ObODPSGeneralFormat::decrypt_str(common::ObString &src, common::ObString &dst) +{ + int ret = OB_SUCCESS; +#ifdef OB_BUILD_CPP_ODPS + const uint64_t tenant_id = MTL_ID(); + if (src.empty()) { + // do nothing + dst = src; + } else if (0 != src.length() % 2) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid src", K(src.length()), K(ret)); + } else { + char encrypted_password_not_hex[common::OB_MAX_ENCRYPTED_EXTERNAL_TABLE_PROPERTIES_ITEM_LENGTH] = {0}; + char plain_string[common::OB_MAX_EXTERNAL_TABLE_PROPERTIES_ITEM_LENGTH + 1] = { 0 }; // need +1 to reserve space for \0 + int64_t plain_string_len = -1; + if (OB_FAIL(hex_to_cstr(src.ptr(), + src.length(), + encrypted_password_not_hex, + common::OB_MAX_ENCRYPTED_EXTERNAL_TABLE_PROPERTIES_ITEM_LENGTH))) { + LOG_WARN("failed to hex to cstr", K(src.length()), K(ret)); + } else if (OB_FAIL(ObEncryptionUtil::decrypt_sys_data(tenant_id, + encrypted_password_not_hex, + + src.length() / 2, + plain_string, + common::OB_MAX_EXTERNAL_TABLE_PROPERTIES_ITEM_LENGTH + 1, + plain_string_len))) { + LOG_WARN("failed to decrypt_sys_data", K(ret), K(src.length())); + } else if (0 >= plain_string_len) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("decrypt dblink password failed", K(ret), K(plain_string_len)); + } else if (OB_FAIL(deep_copy_str(ObString(plain_string_len, plain_string), dst))) { + LOG_WARN("failed to deep copy plain_string", K(ret)); + } else { + LOG_TRACE("succ to decrypt src", K(ret)); + } + } +#endif + return ret; +} + +int ObODPSGeneralFormat::encrypt() +{ + int ret = OB_SUCCESS; + #ifdef OB_BUILD_CPP_ODPS + ObString encrypted_access_id; + ObString encrypted_access_key; + ObString encrypted_sts_token; + if (OB_FAIL(encrypt_str(access_id_, encrypted_access_id))) { + LOG_WARN("failed to encrypt", K(ret)); + } else if (OB_FAIL(encrypt_str(access_key_, encrypted_access_key))) { + LOG_WARN("failed to encrypt", K(ret)); + } else if (OB_FAIL(encrypt_str(sts_token_, encrypted_sts_token))) { + LOG_WARN("failed to encrypt", K(ret)); + } else { + access_id_ = encrypted_access_id; + access_key_ = encrypted_access_key; + sts_token_ = encrypted_sts_token; + } + #endif + return ret; +} + +int ObODPSGeneralFormat::decrypt() +{ + int ret = OB_SUCCESS; + #ifdef OB_BUILD_CPP_ODPS + ObString decrypted_access_id; + ObString decrypted_access_key; + ObString decrypted_sts_token; + if (OB_FAIL(decrypt_str(access_id_, decrypted_access_id))) { + LOG_WARN("failed to encrypt", K(ret)); + } else if (OB_FAIL(decrypt_str(access_key_, decrypted_access_key))) { + LOG_WARN("failed to encrypt", K(ret)); + } else if (OB_FAIL(decrypt_str(sts_token_, decrypted_sts_token))) { + LOG_WARN("failed to encrypt", K(ret)); + } else { + access_id_ = decrypted_access_id; + access_key_ = decrypted_access_key; + sts_token_ = decrypted_sts_token; + } + #endif + return ret; +} + +int ObODPSGeneralFormat::deep_copy_str(const ObString &src, ObString &dest) +{ + int ret = OB_SUCCESS; + char *buf = NULL; + if (src.length() > 0) { + int64_t len = src.length() + 1; + if (OB_ISNULL(buf = static_cast(arena_alloc_.alloc(len)))) { + LOG_ERROR("allocate memory fail", K(len)); + ret = OB_ALLOCATE_MEMORY_FAILED; + } else { + MEMCPY(buf, src.ptr(), len - 1); + buf[len - 1] = '\0'; + dest.assign_ptr(buf, static_cast(len - 1)); + } + } else { + dest.reset(); + } + return ret; +} + +int ObODPSGeneralFormat::deep_copy(const ObODPSGeneralFormat &src) { + int ret = OB_SUCCESS; + if (OB_FAIL(deep_copy_str(src.access_type_, access_type_))) { + LOG_WARN("failed to deep copy", K(ret)); + } else if (OB_FAIL(deep_copy_str(src.access_id_, access_id_))) { + LOG_WARN("failed to deep copy", K(ret)); + } else if (OB_FAIL(deep_copy_str(src.access_key_, access_key_))) { + LOG_WARN("failed to deep copy", K(ret)); + } else if (OB_FAIL(deep_copy_str(src.sts_token_, sts_token_))) { + LOG_WARN("failed to deep copy", K(ret)); + } else if (OB_FAIL(deep_copy_str(src.endpoint_, endpoint_))) { + LOG_WARN("failed to deep copy", K(ret)); + } else if (OB_FAIL(deep_copy_str(src.project_, project_))) { + LOG_WARN("failed to deep copy", K(ret)); + } else if (OB_FAIL(deep_copy_str(src.schema_, schema_))) { + LOG_WARN("failed to deep copy", K(ret)); + } else if (OB_FAIL(deep_copy_str(src.table_, table_))) { + LOG_WARN("failed to deep copy", K(ret)); + } else if (OB_FAIL(deep_copy_str(src.quota_, quota_))) { + LOG_WARN("failed to deep copy", K(ret)); + } else if (OB_FAIL(deep_copy_str(src.compression_code_, compression_code_))) { + LOG_WARN("failed to deep copy", K(ret)); + } + return ret; +} + +int ObODPSGeneralFormat::load_from_json_data(json::Pair *&node, ObIAllocator &allocator) +{ + int ret = OB_SUCCESS; + int64_t idx = 0; + if (OB_NOT_NULL(node) && 0 == node->name_.case_compare(OPTION_NAMES[idx++]) + && json::JT_STRING == node->value_->get_type()) { + ObObj obj; + OZ (ObHexUtilsBase::unhex(node->value_->get_string(), allocator, obj)); + if (OB_SUCC(ret) && !obj.is_null()) { + access_type_ = obj.get_string(); + } + node = node->get_next(); + } + if (OB_SUCC(ret) && OB_NOT_NULL(node) && 0 == node->name_.case_compare(OPTION_NAMES[idx++]) + && json::JT_STRING == node->value_->get_type()) { + ObObj obj; + OZ (ObHexUtilsBase::unhex(node->value_->get_string(), allocator, obj)); + if (OB_SUCC(ret) && !obj.is_null()) { + access_id_ = obj.get_string(); + } + node = node->get_next(); + } + if (OB_NOT_NULL(node) && 0 == node->name_.case_compare(OPTION_NAMES[idx++]) + && json::JT_STRING == node->value_->get_type()) { + ObObj obj; + OZ (ObHexUtilsBase::unhex(node->value_->get_string(), allocator, obj)); + if (OB_SUCC(ret) && !obj.is_null()) { + access_key_ = obj.get_string(); + } + node = node->get_next(); + } + if (OB_NOT_NULL(node) && 0 == node->name_.case_compare(OPTION_NAMES[idx++]) + && json::JT_STRING == node->value_->get_type()) { + ObObj obj; + OZ (ObHexUtilsBase::unhex(node->value_->get_string(), allocator, obj)); + if (OB_SUCC(ret) && !obj.is_null()) { + sts_token_ = obj.get_string(); + } + node = node->get_next(); + } + if (OB_NOT_NULL(node) && 0 == node->name_.case_compare(OPTION_NAMES[idx++]) + && json::JT_STRING == node->value_->get_type()) { + ObObj obj; + OZ (ObHexUtilsBase::unhex(node->value_->get_string(), allocator, obj)); + if (OB_SUCC(ret) && !obj.is_null()) { + endpoint_ = obj.get_string(); + } + node = node->get_next(); + } + if (OB_NOT_NULL(node) && 0 == node->name_.case_compare(OPTION_NAMES[idx++]) + && json::JT_STRING == node->value_->get_type()) { + ObObj obj; + OZ (ObHexUtilsBase::unhex(node->value_->get_string(), allocator, obj)); + if (OB_SUCC(ret) && !obj.is_null()) { + project_ = obj.get_string(); + } + node = node->get_next(); + } + if (OB_NOT_NULL(node) && 0 == node->name_.case_compare(OPTION_NAMES[idx++]) + && json::JT_STRING == node->value_->get_type()) { + ObObj obj; + OZ (ObHexUtilsBase::unhex(node->value_->get_string(), allocator, obj)); + if (OB_SUCC(ret) && !obj.is_null()) { + schema_ = obj.get_string(); + } + node = node->get_next(); + } + if (OB_NOT_NULL(node) && 0 == node->name_.case_compare(OPTION_NAMES[idx++]) + && json::JT_STRING == node->value_->get_type()) { + ObObj obj; + OZ (ObHexUtilsBase::unhex(node->value_->get_string(), allocator, obj)); + if (OB_SUCC(ret) && !obj.is_null()) { + table_ = obj.get_string(); + } + node = node->get_next(); + } + if (OB_NOT_NULL(node) && 0 == node->name_.case_compare(OPTION_NAMES[idx++]) + && json::JT_STRING == node->value_->get_type()) { + ObObj obj; + OZ (ObHexUtilsBase::unhex(node->value_->get_string(), allocator, obj)); + if (OB_SUCC(ret) && !obj.is_null()) { + quota_ = obj.get_string(); + } + node = node->get_next(); + } + if (OB_NOT_NULL(node) && 0 == node->name_.case_compare(OPTION_NAMES[idx++]) + && json::JT_STRING == node->value_->get_type()) { + ObObj obj; + OZ (ObHexUtilsBase::unhex(node->value_->get_string(), allocator, obj)); + if (OB_SUCC(ret) && !obj.is_null()) { + compression_code_ = obj.get_string(); + } + node = node->get_next(); + } + return ret; +} + int ObCSVGeneralFormat::init_format(const ObDataInFileStruct &format, int64_t file_column_nums, ObCollationType file_cs_type) @@ -397,6 +693,9 @@ int64_t ObExternalFileFormat::to_string(char *buf, const int64_t buf_len) const pos += csv_format_.to_json_kv_string(buf + pos, buf_len - pos); pos += origin_file_format_str_.to_json_kv_string(buf + pos, buf_len - pos); break; + case ODPS_FORMAT: + pos += odps_format_.to_json_kv_string(buf + pos, buf_len - pos); + break; default: pos += 0; } @@ -445,6 +744,9 @@ int ObExternalFileFormat::load_from_string(const ObString &str, ObIAllocator &al OZ (csv_format_.load_from_json_data(format_type_node, allocator)); OZ (origin_file_format_str_.load_from_json_data(format_type_node, allocator)); break; + case ODPS_FORMAT: + OZ (odps_format_.load_from_json_data(format_type_node, allocator)); + break; case PARQUET_FORMAT: case ORC_FORMAT: break; @@ -480,6 +782,14 @@ int ObExternalFileFormat::mock_gen_column_def( } break; } + case ODPS_FORMAT: { + uint64_t odps_column_idx = column.get_column_id() - OB_APP_MIN_COLUMN_ID + 1; + if (OB_FAIL(temp_str.append_fmt("%s%lu", N_EXTERNAL_TABLE_COLUMN_PREFIX, odps_column_idx))) { + LOG_WARN("fail to append sql str", K(ret)); + } else { + } + break; + } case PARQUET_FORMAT: { if (OB_FAIL(temp_str.append_fmt("get_path(%s, '%.*s')", N_EXTERNAL_FILE_ROW, @@ -497,8 +807,8 @@ int ObExternalFileFormat::mock_gen_column_def( } if (OB_SUCC(ret)) { if (OB_FAIL(ob_write_string(allocator, temp_str.string(), def))) { - LOG_WARN("fail to write string", K(ret)); - } + LOG_WARN("fail to write string", K(ret)); + } } return ret; diff --git a/src/sql/engine/cmd/ob_load_data_parser.h b/src/sql/engine/cmd/ob_load_data_parser.h index d8ef54a21..8683f169d 100644 --- a/src/sql/engine/cmd/ob_load_data_parser.h +++ b/src/sql/engine/cmd/ob_load_data_parser.h @@ -30,6 +30,55 @@ namespace sql { class ObDataInFileStruct; +struct ObODPSGeneralFormat { + ObODPSGeneralFormat() : + access_type_(), + access_id_(), + access_key_(), + sts_token_(), + endpoint_(), + project_(), + schema_(), + table_(), + quota_(), + compression_code_() + {} + int deep_copy_str(const ObString &src, + ObString &dest); + int deep_copy(const ObODPSGeneralFormat &src); + int encrypt_str(common::ObString &src, common::ObString &dst); + int decrypt_str(common::ObString &src, common::ObString &dst); + int encrypt(); + int decrypt(); + static constexpr const char *OPTION_NAMES[] = { + "ACCESSTYPE", + "ACCESSID", + "ACCESSKEY", + "STSTOKEN", + "ENDPOINT", + "PROJECT_NAME", + "SCHEMA_NAME", + "TABLE_NAME", + "QUOTA_NAME", + "COMPRESSION_CODE", + }; + common::ObString access_type_; + common::ObString access_id_; + common::ObString access_key_; + common::ObString sts_token_; + common::ObString endpoint_; + common::ObString project_; + common::ObString schema_; + common::ObString table_; + common::ObString quota_; + common::ObString compression_code_; + common::ObArenaAllocator arena_alloc_; + int64_t to_json_kv_string(char* buf, const int64_t buf_len) const; + int load_from_json_data(json::Pair *&node, common::ObIAllocator &allocator); + TO_STRING_KV(K_(access_type), K_(access_id), K_(access_key), K_(sts_token), K_(endpoint), K_(project), K_(schema), K_(table), K_(quota), K_(compression_code)); + OB_UNIS_VERSION(1); +}; + struct ObCSVGeneralFormat { ObCSVGeneralFormat () : line_start_str_(), @@ -524,6 +573,7 @@ struct ObExternalFileFormat INVALID_FORMAT = -1, CSV_FORMAT, PARQUET_FORMAT, + ODPS_FORMAT, ORC_FORMAT, MAX_FORMAT }; @@ -542,6 +592,7 @@ struct ObExternalFileFormat ObOriginFileFormat origin_file_format_str_; FormatType format_type_; sql::ObCSVGeneralFormat csv_format_; + sql::ObODPSGeneralFormat odps_format_; ObLoadCompressionFormat compression_format_; uint64_t options_; static const char *FORMAT_TYPE_STR[]; diff --git a/src/sql/engine/cmd/ob_table_executor.cpp b/src/sql/engine/cmd/ob_table_executor.cpp index 13bcd126f..22e3f2cdc 100644 --- a/src/sql/engine/cmd/ob_table_executor.cpp +++ b/src/sql/engine/cmd/ob_table_executor.cpp @@ -621,13 +621,14 @@ int ObCreateTableExecutor::execute(ObExecContext &ctx, ObCreateTableStmt &stmt) table_schema.get_external_file_location(), table_schema.get_external_file_location_access_info(), table_schema.get_external_file_pattern(), + table_schema.get_external_properties(), + table_schema.is_partitioned_table(), regexp_vars, ctx.get_allocator(), tmp, file_urls, file_sizes)); } - if (OB_FAIL(ret)) { } else { create_table_arg.is_inner_ = my_session->is_inner(); @@ -1025,7 +1026,10 @@ int ObAlterTableExecutor::execute_alter_external_table(ObExecContext &ctx, ObAlt arg.alter_table_schema_.get_table_id(), arg.alter_table_schema_.get_external_file_location(), arg.alter_table_schema_.get_external_file_location_access_info(), - arg.alter_table_schema_.get_external_file_pattern(), regexp_vars, ctx.get_allocator(), + arg.alter_table_schema_.get_external_file_pattern(), + arg.alter_table_schema_.get_external_properties(), + arg.alter_table_schema_.is_partitioned_table(), + regexp_vars, ctx.get_allocator(), full_path, file_urls, file_sizes)); @@ -1157,6 +1161,8 @@ int ObAlterTableExecutor::execute(ObExecContext &ctx, ObAlterTableStmt &stmt) alter_table_arg.alter_table_schema_.get_external_file_location(), alter_table_arg.alter_table_schema_.get_external_file_location_access_info(), alter_table_arg.alter_table_schema_.get_external_file_pattern(), + alter_table_arg.alter_table_schema_.get_external_properties(), + alter_table_arg.alter_table_schema_.is_partitioned_table(), regexp_vars, ctx.get_allocator(), full_path, diff --git a/src/sql/engine/expr/ob_datum_cast.cpp b/src/sql/engine/expr/ob_datum_cast.cpp index 0e06e812e..b0a921276 100644 --- a/src/sql/engine/expr/ob_datum_cast.cpp +++ b/src/sql/engine/expr/ob_datum_cast.cpp @@ -898,6 +898,14 @@ static OB_INLINE int common_int_number(const ObExpr &expr, return ret; } +int ObOdpsDataTypeCastUtil::common_int_number_wrap(const ObExpr &expr, + int64_t in_val, + ObIAllocator &alloc, + number::ObNumber &nmb) +{ + return common_int_number(expr, in_val, alloc, nmb); +} + static OB_INLINE int common_int_date(const ObExpr &expr, const int64_t in_val, ObDatum &res_datum) @@ -1265,6 +1273,14 @@ static OB_INLINE int common_string_number(const ObExpr &expr, return ret; } +int ObOdpsDataTypeCastUtil::common_string_number_wrap(const ObExpr &expr, + const ObString &in_str, + ObIAllocator &alloc, + number::ObNumber &nmb) +{ + return common_string_number(expr, in_str, alloc, nmb); +} + static int common_string_decimalint(const ObExpr &expr, const ObString &in_str, const ObUserLoggingCtx *user_logging_ctx, ObDecimalIntBuilder &res_val) @@ -1395,6 +1411,128 @@ static int common_string_decimalint(const ObExpr &expr, const ObString &in_str, #undef SET_ZERO } +int ObOdpsDataTypeCastUtil::common_string_decimalint_wrap(const ObExpr &expr, const ObString &in_str, + const ObUserLoggingCtx *user_logging_ctx, + ObDecimalIntBuilder &res_val) +{// TODO: add cases +#define SET_ZERO(int_type) \ + int_type v = 0; \ + res_val.from(v); \ + break + + int ret = OB_SUCCESS; + ObObjType in_type = ObVarcharType; + int16_t in_scale = 0, in_precision = 0; + ObScale out_scale = expr.datum_meta_.scale_; + ObPrecision out_prec = expr.datum_meta_.precision_; + ObDecimalIntBuilder tmp_alloc; + ObDecimalInt *decint = nullptr; + int32_t int_bytes = 0; + // set default value + switch (get_decimalint_type(out_prec)) { + case common::DECIMAL_INT_32: { + SET_ZERO(int32_t); + } + case common::DECIMAL_INT_64: { + SET_ZERO(int64_t); + } + case common::DECIMAL_INT_128: { + SET_ZERO(int128_t); + } + case common::DECIMAL_INT_256: { + SET_ZERO(int256_t); + } + case common::DECIMAL_INT_512: { + SET_ZERO(int512_t); + } + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected precision", K(out_prec)); + } + if (OB_FAIL(ret)) { + } else { + if (ObHexStringType == in_type) { + uint64_t in_val = hex_to_uint64(in_str); + in_precision = ob_fast_digits10(in_val); + if (OB_FAIL(wide::from_integer(in_val, tmp_alloc, decint, int_bytes, in_precision))) { + LOG_WARN("from integer failed", K(in_val), K(ret)); + } else { + in_scale = 0; + } + } else if (0 == in_str.length()) { + ret = OB_ERR_TRUNCATED_WRONG_VALUE_FOR_FIELD; + } else if (OB_FAIL(wide::from_string(in_str.ptr(), in_str.length(), tmp_alloc, in_scale, + in_precision, int_bytes, decint))) { + LOG_WARN("failed to parse string", K(ret)); + if (OB_NUMERIC_OVERFLOW == ret && lib::is_mysql_mode()) { + // bug: 4263211. compatible with mysql behavior when value overflows type range. + // select cast('1e500' as decimal); -> max_val + // select cast('-1e500' as decimal); -> min_val + int64_t i = 0; + while (i < in_str.length() && isspace(in_str[i])) { ++i; } + bool is_neg = (in_str[i] == '-'); + const ObDecimalInt *limit_decint = nullptr; + if (is_neg) { + limit_decint = wide::ObDecimalIntConstValue::get_min_value(out_prec); + int_bytes = wide::ObDecimalIntConstValue::get_int_bytes_by_precision(out_prec); + } else { + limit_decint = wide::ObDecimalIntConstValue::get_max_value(out_prec); + int_bytes = wide::ObDecimalIntConstValue::get_int_bytes_by_precision(out_prec); + } + in_scale = out_scale; + in_precision = out_prec; + if (OB_ISNULL(limit_decint)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null decimal int", K(ret)); + } else if (OB_ISNULL(decint = (ObDecimalInt *)tmp_alloc.alloc(int_bytes))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory", K(ret)); + } else { + MEMCPY(decint, limit_decint, int_bytes); + } + } + } + int warning = ret; + ret = OB_SUCCESS; + if (decint != nullptr && int_bytes != 0) { + // Decimal int not null means a valid decimal int was parsed regardless of wether there's + // error or not.We then do scale and calculate res_datum as normal in order to be compatible + // with mysql. + // e.g. + // OceanBase(root@test)>set sql_mode = ''; + // Query OK, 0 rows affected (0.00 sec) + // + // OceanBase(root@test)>insert into t2 values ('1ab'); + // Query OK, 1 row affected (0.00 sec) + // + // OceanBase(root@test)>select * from t2; + // +-------+ + // | a | + // +-------+ + // | 1.000 | + // +-------+ + // 1 row in set (0.01 sec) + if (ObDatumCast::need_scale_decimalint(in_scale, in_precision, out_scale, out_prec)) { + if (OB_FAIL(ObDatumCast::common_scale_decimalint(decint, int_bytes, in_scale, out_scale, + out_prec, expr.extra_, res_val, + user_logging_ctx))) { + LOG_WARN("scale decimal int failed", K(ret)); + } + } else { + res_val.from(decint, int_bytes); + } + } + if (OB_SUCC(ret)) { + const ObCastMode cast_mode = expr.extra_; + if (CAST_FAIL(warning)) { + LOG_WARN("string_decimalint failed", K(ret), K(in_type), K(cast_mode), K(in_str)); + } + } + } + return ret; +#undef SET_ZERO +} + static OB_INLINE int common_string_datetime(const ObExpr &expr, const ObString &in_str, ObEvalCtx &ctx, @@ -1644,6 +1782,111 @@ static int common_string_string(const ObExpr &expr, return ret; } +int ObOdpsDataTypeCastUtil::common_check_convert_string(const ObExpr &expr, + ObEvalCtx &ctx, + const ObString &in_str, + ObObjType in_type, + ObCollationType in_cs_type, + ObDatum &res_datum, + bool &has_set_res) +{ + int ret = OB_SUCCESS; + ObObjType out_type = expr.datum_meta_.type_; + ObCollationType out_cs_type = expr.datum_meta_.cs_type_; + if (lib::is_oracle_mode() && + (ob_is_blob(out_type, out_cs_type) || ob_is_blob_locator(out_type, out_cs_type)) && + !(ob_is_blob(in_type, in_cs_type) || ob_is_blob_locator(in_type, in_cs_type) + || ob_is_raw(in_type))) { + // !blob -> blob + if (ObCharType == in_type || ObVarcharType == in_type) { + if (OB_FAIL(ObDatumHexUtils::hextoraw_string(expr, in_str, ctx, res_datum, has_set_res))) { + LOG_WARN("fail to hextoraw_string for blob", K(ret), K(in_str)); + } + } else { + ret = OB_NOT_SUPPORTED; + LOG_ERROR("invalid use of blob type", K(ret), K(in_str), K(out_type)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "cast to blob type"); + } + } else { + // When convert blob/binary/varbinary to other charset, need to align to mbminlen of destination charset + // by add '\0' prefix in mysql mode. (see mysql String::copy) + const ObCharsetInfo *cs = NULL; + int64_t align_offset = 0; + if (CS_TYPE_BINARY == in_cs_type && lib::is_mysql_mode() + && (NULL != (cs = ObCharset::get_charset(out_cs_type)))) { + if (cs->mbminlen > 0 && in_str.length() % cs->mbminlen != 0) { + align_offset = cs->mbminlen - in_str.length() % cs->mbminlen; + } + } + if (OB_FAIL(common_copy_string_zf(expr, in_str, ctx, res_datum, align_offset))) { + LOG_WARN("common_copy_string_zf failed", K(ret), K(in_str)); + } + } + return ret; +} + +int ObOdpsDataTypeCastUtil::common_string_string_wrap(const ObExpr &expr, + const ObObjType in_type, + const ObCollationType in_cs_type, + const ObObjType out_type, + const ObCollationType out_cs_type, + const ObString &in_str, + ObEvalCtx &ctx, + ObDatum &res_datum, + bool& has_set_res) +{ + int ret = OB_SUCCESS; + if (lib::is_oracle_mode() + && ob_is_clob(in_type, in_cs_type) + && (0 == in_str.length()) + && !ob_is_clob(out_type, out_cs_type)) { + // oracle 模式下的 empty_clob 被 cast 成其他类型时结果是 NULL + res_datum.set_null(); + } else if (CS_TYPE_BINARY != in_cs_type && + CS_TYPE_BINARY != out_cs_type && + (ObCharset::charset_type_by_coll(in_cs_type) != + ObCharset::charset_type_by_coll(out_cs_type))) { + // handle !blob->!blob + char *buf = NULL; + //latin1 1bytes,utf8mb4 4bytes,the factor should be 4 + int64_t buf_len = in_str.length() * ObCharset::CharConvertFactorNum; + uint32_t result_len = 0; + buf = expr.get_str_res_mem(ctx, buf_len); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc memory failed", K(ret)); + } else if (OB_FAIL(ObCharset::charset_convert(in_cs_type, in_str.ptr(), + in_str.length(), out_cs_type, buf, + buf_len, result_len, lib::is_mysql_mode(), + !CM_IS_IGNORE_CHARSET_CONVERT_ERR(expr.extra_) && CM_IS_IMPLICIT_CAST(expr.extra_), + ObCharset::is_cs_unicode(out_cs_type) ? 0xFFFD : '?'))) { + LOG_WARN("charset convert failed", K(ret)); + } else { + res_datum.set_string(buf, result_len); + } + } else { + if (CS_TYPE_BINARY == in_cs_type || CS_TYPE_BINARY == out_cs_type) { + // just copy string when in_cs_type or out_cs_type is binary + if (OB_FAIL(ObOdpsDataTypeCastUtil::common_check_convert_string(expr, ctx, in_str, in_type, in_cs_type, res_datum, has_set_res))) { + LOG_WARN("fail to common_check_convert_string", K(ret), K(in_str)); + } + } else if (lib::is_oracle_mode() + && ob_is_clob(in_type, in_cs_type)) { + res_datum.set_string(in_str.ptr(), in_str.length()); + } else if (lib::is_oracle_mode() + && ob_is_clob(out_type, out_cs_type)) { + res_datum.set_string(in_str.ptr(), in_str.length()); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("same charset should not be here, just use cast_eval_arg", K(ret), + K(in_type), K(out_type), K(in_cs_type), K(out_cs_type)); + } + } + LOG_DEBUG("string_string cast", K(ret), K(in_str), + K(ObString(res_datum.len_, res_datum.ptr_))); + return ret; +} + static int common_string_otimestamp(const ObExpr &expr, const ObString &in_str, ObEvalCtx &ctx, @@ -1956,6 +2199,54 @@ static int common_string_text(const ObExpr &expr, return ret; } +int ObOdpsDataTypeCastUtil::common_string_text_wrap(const ObExpr &expr, + const ObString &in_str, + ObEvalCtx &ctx, + const ObLobLocatorV2 *lob_locator, + ObDatum &res_datum, + ObObjType &in_type, + ObCollationType &in_cs_type) +{ + int ret = OB_SUCCESS; + ObObjType out_type = expr.datum_meta_.type_; // ObLongTextType + ObCollationType out_cs_type = expr.datum_meta_.cs_type_; + ObString res_str = in_str; + bool is_final_res = false; + bool is_different_charset_type = (ObCharset::charset_type_by_coll(in_cs_type) + != ObCharset::charset_type_by_coll(out_cs_type)); + OB_ASSERT(ob_is_text_tc(out_type)); + if (is_different_charset_type) { + if (OB_FAIL(ObOdpsDataTypeCastUtil::common_string_string_wrap(expr, in_type, in_cs_type, out_type, + out_cs_type, in_str, ctx, res_datum, is_final_res))) { + LOG_WARN("Lob: fail to cast string to longtext", K(ret), K(in_str), K(expr)); + } else if (res_datum.is_null()) { + // only for blob cast to other types in oracle mode, in/out type/collation type must be different. + is_final_res = true; + } else if (is_final_res) { + // is_final_res = true; // hex to text + } else if (OB_FAIL(copy_datum_str_with_tmp_alloc(ctx, res_datum, res_str))) { + LOG_WARN("Lob: copy datum str with tmp alloc", K(ret)); + } else { /* do nothing */ } + } + + if (OB_SUCC(ret) && !is_final_res) { + ObTextStringDatumResult str_result(expr.datum_meta_.type_, &expr, &ctx, &res_datum); + if (lob_locator == NULL) { + if (OB_FAIL(str_result.init(res_str.length()))) { + LOG_WARN("Lob: init lob result failed"); + } else if (OB_FAIL(str_result.append(res_str.ptr(), res_str.length()))) { + LOG_WARN("Lob: append lob result failed"); + } else { /* do nothing */ } + } else if (OB_FAIL(str_result.copy(lob_locator))) { + LOG_WARN("Lob: copy lob result failed"); + } else { /* do nothing*/ } + str_result.set_result(); + } + + string_lob_debug(in_type, in_cs_type, expr.obj_meta_.has_lob_header(), out_type, out_cs_type, res_str, res_datum, ret); + return ret; +} + static int common_uint_bit(const ObExpr &expr, const uint64_t &in_value, ObEvalCtx &ctx, diff --git a/src/sql/engine/expr/ob_datum_cast.h b/src/sql/engine/expr/ob_datum_cast.h index 7be19a7b8..6cd3d2c5c 100644 --- a/src/sql/engine/expr/ob_datum_cast.h +++ b/src/sql/engine/expr/ob_datum_cast.h @@ -32,6 +32,45 @@ namespace sql class ObPhysicalPlanCtx; struct ObUserLoggingCtx; +class ObOdpsDataTypeCastUtil +{ +public: + static int common_int_number_wrap(const ObExpr &expr, + int64_t in_val, + ObIAllocator &alloc, + number::ObNumber &nmb); + static int common_string_decimalint_wrap(const ObExpr &expr, const ObString &in_str, + const ObUserLoggingCtx *user_logging_ctx, + ObDecimalIntBuilder &res_val); + static int common_string_number_wrap(const ObExpr &expr, + const ObString &in_str, + ObIAllocator &alloc, + number::ObNumber &nmb); + static int common_string_string_wrap(const ObExpr &expr, + const ObObjType in_type, + const ObCollationType in_cs_type, + const ObObjType out_type, + const ObCollationType out_cs_type, + const ObString &in_str, + ObEvalCtx &ctx, + ObDatum &res_datum, + bool& has_set_res); + static int common_string_text_wrap(const ObExpr &expr, + const ObString &in_str, + ObEvalCtx &ctx, + const ObLobLocatorV2 *lob_locator, + ObDatum &res_datum, + ObObjType &in_type, + ObCollationType &in_cs_type); + static int common_check_convert_string(const ObExpr &expr, + ObEvalCtx &ctx, + const ObString &in_str, + ObObjType in_type, + ObCollationType in_cs_type, + ObDatum &res_datum, + bool &has_set_res); +}; + // extract accuracy info from %expr and call datum_accuracy_check() below. int datum_accuracy_check(const ObExpr &expr, const uint64_t cast_mode, diff --git a/src/sql/engine/px/ob_dfo.cpp b/src/sql/engine/px/ob_dfo.cpp index 8092b23ee..60926c6ae 100644 --- a/src/sql/engine/px/ob_dfo.cpp +++ b/src/sql/engine/px/ob_dfo.cpp @@ -194,6 +194,7 @@ int ObPxSqcMeta::assign(const ObPxSqcMeta &other) temp_file.file_id_ = other_file.file_id_; temp_file.part_id_ = other_file.part_id_; temp_file.file_addr_ = other_file.file_addr_; + temp_file.file_size_ = other_file.file_size_; if (OB_FAIL(ob_write_string(allocator_, other_file.file_url_, temp_file.file_url_))) { LOG_WARN("fail to write string", K(ret)); } else if (OB_FAIL(access_external_table_files_.push_back(temp_file))) { diff --git a/src/sql/engine/px/ob_dfo.h b/src/sql/engine/px/ob_dfo.h index ae66b3cf1..8ff38a640 100644 --- a/src/sql/engine/px/ob_dfo.h +++ b/src/sql/engine/px/ob_dfo.h @@ -520,7 +520,8 @@ public: p2p_dh_map_info_(), coord_info_ptr_(nullptr), force_bushy_(false), - query_sql_() + query_sql_(), + has_into_odps_(false) { } @@ -561,6 +562,8 @@ public: inline bool has_need_branch_id_op() const { return has_need_branch_id_op_; } inline void set_temp_table_scan(bool has_scan) { has_temp_scan_ = has_scan; } inline bool has_temp_table_scan() const { return has_temp_scan_; } + inline void set_into_odps(bool has_into_odps) { has_into_odps_ = has_into_odps; } + inline bool has_into_odps() const { return has_into_odps_; } inline bool is_fast_dfo() const { return is_prealloc_receive_channel() || is_prealloc_transmit_channel(); } inline void set_slave_mapping_type(SlaveMappingType v) { slave_mapping_type_ = v; } inline SlaveMappingType get_slave_mapping_type() { return slave_mapping_type_; } @@ -813,6 +816,7 @@ private: bool force_bushy_; bool partition_random_affinitize_{true}; // whether do partition random in gi task split ObString query_sql_; + bool has_into_odps_; }; diff --git a/src/sql/engine/px/ob_dfo_mgr.cpp b/src/sql/engine/px/ob_dfo_mgr.cpp index 8ead8434a..39c12c7b4 100644 --- a/src/sql/engine/px/ob_dfo_mgr.cpp +++ b/src/sql/engine/px/ob_dfo_mgr.cpp @@ -28,6 +28,7 @@ #include "share/detect/ob_detect_manager_utils.h" #include "sql/engine/px/ob_px_coord_op.h" #include "sql/engine/basic/ob_material_vec_op.h" +#include "sql/engine/basic/ob_select_into_op.h" using namespace oceanbase::common; using namespace oceanbase::sql; @@ -541,6 +542,18 @@ int ObDfoMgr::do_split(ObExecContext &exec_ctx, OZ(px_coord_info.p2p_temp_table_info_.temp_access_ops_.push_back(phy_op)); OZ(px_coord_info.p2p_temp_table_info_.dfos_.push_back(parent_dfo)); } + } else if (phy_op->get_type() == PHY_SELECT_INTO && NULL != parent_dfo) { + // odps只支持一台机器上的并行 只能有一个sqc + const ObSelectIntoSpec *select_into_spec = static_cast(phy_op); + ObExternalFileFormat external_properties; + if (!select_into_spec->external_properties_.str_.empty()) { + if (OB_FAIL(external_properties.load_from_string(select_into_spec->external_properties_.str_, + allocator))) { + LOG_WARN("failed to load external properties", K(ret)); + } else if (ObExternalFileFormat::FormatType::ODPS_FORMAT == external_properties.format_type_) { + parent_dfo->set_into_odps(true); + } + } } else if (IS_PX_GI(phy_op->get_type()) && NULL != parent_dfo) { const ObGranuleIteratorSpec *gi_spec = static_cast(phy_op); diff --git a/src/sql/engine/px/ob_dfo_scheduler.cpp b/src/sql/engine/px/ob_dfo_scheduler.cpp index b8b7f681c..44ff81dfa 100644 --- a/src/sql/engine/px/ob_dfo_scheduler.cpp +++ b/src/sql/engine/px/ob_dfo_scheduler.cpp @@ -1426,7 +1426,7 @@ int ObParallelDfoScheduler::schedule_pair(ObExecContext &exec_ctx, parent))) { LOG_WARN("fail alloc addr by data distribution", K(parent), K(ret)); } else { /*do nohting.*/ } - } else if (parent.is_root_dfo()) { + } else if (parent.is_root_dfo() || parent.has_into_odps()) { // QC/local dfo，直接在本机本线程执行，无需计算执行位置 if (OB_FAIL(ObPXServerAddrUtil::alloc_by_local_distribution(exec_ctx, parent))) { diff --git a/src/sql/engine/px/ob_granule_pump.cpp b/src/sql/engine/px/ob_granule_pump.cpp index 233cb7620..24c8395d0 100644 --- a/src/sql/engine/px/ob_granule_pump.cpp +++ b/src/sql/engine/px/ob_granule_pump.cpp @@ -643,6 +643,34 @@ int ObGranulePump::add_new_gi_task(ObGranulePumpArgs &args) random_type, partition_granule))) { LOG_WARN("failed to prepare random gi task", K(ret), K(partition_granule)); + } else if (OB_FAIL(init_external_odps_table_downloader(args))) { + LOG_WARN("failed to init external odps table downloader", K(ret)); + } + } + return ret; +} + +int ObGranulePump::init_external_odps_table_downloader(ObGranulePumpArgs &args) +{ + int ret = OB_SUCCESS; + const ObTableScanSpec *tsc = NULL; + sql::ObExternalFileFormat external_odps_format; + if (!args.external_table_files_.empty() && + 0 == args.external_table_files_.at(0).file_id_) { //file_id_ == 0 means it's a external odps table + ObIArray &scan_ops = args.op_info_.get_scan_ops(); + if (scan_ops.empty() || scan_ops.count() != gi_task_array_map_.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid scan ops and gi task array result", K(ret), K(scan_ops.count()), K(gi_task_array_map_.count())); + } else if (OB_ISNULL(tsc = scan_ops.at(0))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null ptr", K(ret)); +#ifdef OB_BUILD_CPP_ODPS + } else if (OB_FAIL(odps_partition_downloader_mgr_.init_downloader(args.external_table_files_, + tsc->tsc_ctdef_.scan_ctdef_.external_file_format_str_.str_))) { + LOG_WARN("init odps_partition_downloader_mgr_ failed", K(ret), K(args.external_table_files_.count())); +#endif + } else { + LOG_TRACE("succ to init odps table partition downloader", K(ret)); } } return ret; @@ -693,6 +721,9 @@ void ObGranulePump::destroy() { gi_task_array_map_.reset(); pump_args_.reset(); +#ifdef OB_BUILD_CPP_ODPS + odps_partition_downloader_mgr_.reset(); +#endif } void ObGranulePump::reset_task_array() @@ -805,6 +836,8 @@ int ObGranuleSplitter::split_gi_task(ObGranulePumpArgs &args, K(ss_ranges), K(taskset_idxs), K(random_type)); + } else { + } } return ret; @@ -997,8 +1030,6 @@ int ObRandomGranuleSplitter::split_granule(ObGranulePumpArgs &args, } else { gi_task_array_result.at(idx).tsc_op_id_ = op_id; } - LOG_TRACE("random granule split a task_array", - K(op_id), K(scan_key_id), K(taskset_array), K(ret), K(scan_ops.count())); } } return ret; @@ -1592,7 +1623,6 @@ int ObGranulePump::init_arg( for (int i = 0; OB_SUCC(ret) && i < partitions_info.count(); ++i) { OZ(arg.partitions_info_.push_back(partitions_info.at(i))); } - OZ(arg.external_table_files_.assign(external_table_files)); if (OB_SUCC(ret)) { diff --git a/src/sql/engine/px/ob_granule_pump.h b/src/sql/engine/px/ob_granule_pump.h index 0d75c523f..5c5f895ac 100644 --- a/src/sql/engine/px/ob_granule_pump.h +++ b/src/sql/engine/px/ob_granule_pump.h @@ -556,7 +556,18 @@ public: common::ObIArray *get_pruning_table_location() { return &pruning_table_locations_; } int get_first_tsc_range_cnt(int64_t &cnt); const GITaskArrayMap &get_task_array_map() const { return gi_task_array_map_; } +#ifdef OB_BUILD_CPP_ODPS + inline int get_odps_downloader(int64_t part_id, apsara::odps::sdk::IDownloadPtr &downloader) { + int ret = OB_SUCCESS; + downloader = NULL; + ret = odps_partition_downloader_mgr_.get_odps_downloader(part_id, downloader); + return ret; + } + inline bool is_odps_downloader_inited() { return odps_partition_downloader_mgr_.is_download_mgr_inited(); } + ObOdpsPartitionDownloaderMgr &get_odps_mgr() { return odps_partition_downloader_mgr_; } +#endif private: + int init_external_odps_table_downloader(ObGranulePumpArgs &args); int fetch_granule_by_worker_id(const ObGITaskSet *&task_set, int64_t &pos, int64_t thread_id, @@ -597,6 +608,9 @@ private: bool partition_wise_join_; volatile bool no_more_task_from_shared_pool_; // try notify worker exit earlier GITaskArrayMap gi_task_array_map_; +#ifdef OB_BUILD_CPP_ODPS + ObOdpsPartitionDownloaderMgr odps_partition_downloader_mgr_; +#endif ObGranuleSplitterType splitter_type_; common::ObArray pump_args_; bool need_partition_pruning_; diff --git a/src/sql/engine/px/ob_granule_util.cpp b/src/sql/engine/px/ob_granule_util.cpp index 5a4d020e5..f47568df9 100644 --- a/src/sql/engine/px/ob_granule_util.cpp +++ b/src/sql/engine/px/ob_granule_util.cpp @@ -28,6 +28,9 @@ #include "share/external_table/ob_external_table_utils.h" #include "sql/engine/table/ob_external_table_access_service.h" #include "sql/das/ob_das_simple_op.h" +#ifdef OB_BUILD_CPP_ODPS +#include "sql/engine/table/ob_odps_table_row_iter.h" +#endif using namespace oceanbase::common; using namespace oceanbase::share; @@ -85,12 +88,14 @@ int ObGranuleUtil::split_granule_for_external_table(ObIAllocator &allocator, ObIArray &granule_ranges, ObIArray &granule_idx) { - UNUSED(parallelism); UNUSED(tsc); int ret = OB_SUCCESS; + sql::ObExternalFileFormat external_file_format; if (ranges.count() < 1 || tablets.count() < 1 || OB_ISNULL(tsc)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("the invalid argument", K(ret), K(ranges.count()), K(tablets.count())); + } else if (OB_FAIL(external_file_format.load_from_string(tsc->tsc_ctdef_.scan_ctdef_.external_file_format_str_.str_, allocator))) { + LOG_WARN("failed to load from string", K(ret), K(tsc->tsc_ctdef_.scan_ctdef_.external_file_format_str_.str_)); } else if (external_table_files.count() == 1 && external_table_files.at(0).file_id_ == INT64_MAX) { // dealing dummy file @@ -107,6 +112,48 @@ int ObGranuleUtil::split_granule_for_external_table(ObIAllocator &allocator, OB_FAIL(granule_tablets.push_back(tablets.at(0)))) { LOG_WARN("fail to push back", K(ret)); } + } else if (!external_table_files.empty() && + ObExternalFileFormat::ODPS_FORMAT == external_file_format.format_type_) { +#ifdef OB_BUILD_CPP_ODPS + int64_t task_idx = 0; + for (int64_t i = 0; OB_SUCC(ret) && i < external_table_files.count(); ++i) { + const ObExternalFileInfo& external_info = external_table_files.at(i); + if (0 != external_info.file_id_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected file id", K(ret), K(i), K(external_info.file_id_)); + } else { + // file_size_ is the total row cnt of odps table partition + uint64_t block_cnt = (external_info.file_size_ + sql::ObODPSTableRowIterator::ODPS_BLOCK_DOWNLOAD_SIZE - 1) + / sql::ObODPSTableRowIterator::ODPS_BLOCK_DOWNLOAD_SIZE; + uint64_t start_idx = 0; + block_cnt = (0 == block_cnt ? 1 : block_cnt); // one odps table partition should have at least one task, even it's empty + for (int64_t j = 0; OB_SUCC(ret) && j < block_cnt; ++j) { + ObNewRange new_range; + int64_t start = start_idx + (sql::ObODPSTableRowIterator::ODPS_BLOCK_DOWNLOAD_SIZE * j); + int64_t end = sql::ObODPSTableRowIterator::ODPS_BLOCK_DOWNLOAD_SIZE; + if (OB_FAIL(ObExternalTableUtils::make_external_table_scan_range(external_info.file_url_, + external_info.file_id_, + external_info.part_id_, + start_idx + (sql::ObODPSTableRowIterator::ODPS_BLOCK_DOWNLOAD_SIZE * j), + j == block_cnt -1 ? + INT64_MAX : + sql::ObODPSTableRowIterator::ODPS_BLOCK_DOWNLOAD_SIZE, + allocator, + new_range))) { + LOG_WARN("failed to make external table scan range", K(ret)); + } else if ((OB_FAIL(granule_ranges.push_back(new_range)) || + OB_FAIL(granule_idx.push_back(task_idx++)) || + OB_FAIL(granule_tablets.push_back(tablets.at(0))))) { + LOG_WARN("fail to push back", K(ret)); + } + } + } + } +#else + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "external odps table"); + LOG_WARN("not support odps table in opensource", K(ret)); +#endif } else { for (int64_t i = 0; OB_SUCC(ret) && i < ranges.count(); ++i) { for (int64_t j = 0; OB_SUCC(ret) && j < external_table_files.count(); ++j) { @@ -202,10 +249,6 @@ int ObGranuleUtil::split_block_ranges(ObExecContext &exec_ctx, LOG_TRACE("get the splited results through the new gi split method", K(ret), K(granule_tablets.count()), K(granule_ranges.count()), K(granule_idx)); } - LOG_TRACE("split ranges to granule", K(ret), K(total_task_count), K(parallelism), - K(total_macros_count), K(macros_count_by_partition), K(macros_count_per_task), - K(granule_tablets.count()), K(granule_tablets), K(granule_ranges.count()), K(granule_ranges), - K(granule_idx.count()), K(granule_idx), K(tablets), K(task_count_by_partition)); return ret; } @@ -363,8 +406,8 @@ int ObGranuleUtil::split_block_granule(ObExecContext &exec_ctx, granule_tablets.count() != granule_ranges.count() || granule_tablets.count() != granule_idx.count()) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("the ranges or offsets are empty", - K(ret), K(granule_tablets.count()), K(granule_ranges.count()), K(granule_idx.count())); + LOG_WARN("the ranges or offsets are empty", K(ret), K(granule_tablets.count()), K(granule_ranges.count()), + K(granule_idx.count()), K(granule_tablets), K(granule_ranges), K(granule_idx)); } } } diff --git a/src/sql/engine/px/ob_px_sub_coord.cpp b/src/sql/engine/px/ob_px_sub_coord.cpp index 8f4caf29e..2e1586897 100644 --- a/src/sql/engine/px/ob_px_sub_coord.cpp +++ b/src/sql/engine/px/ob_px_sub_coord.cpp @@ -47,6 +47,7 @@ #include "sql/das/ob_das_utils.h" #include "sql/engine/px/p2p_datahub/ob_p2p_dh_mgr.h" #include "sql/engine/window_function/ob_window_function_vec_op.h" +#include "sql/engine/basic/ob_select_into_op.h" using namespace oceanbase::common; using namespace oceanbase::sql; @@ -101,7 +102,6 @@ int ObPxSubCoord::pre_process() LOG_WARN("fail to setup receive/transmit op input", K(ret)); } } - if (OB_SUCC(ret) && !sqc_arg_.sqc_.get_pruning_table_locations().empty()) { sqc_ctx_.gi_pump_.set_need_partition_pruning(true); OZ(sqc_ctx_.gi_pump_.set_pruning_table_location(sqc_arg_.sqc_.get_pruning_table_locations())); @@ -557,6 +557,18 @@ int ObPxSubCoord::setup_op_input(ObExecContext &ctx, LOG_DEBUG("debug wf input", K(wf_spec->role_type_), K(sqc.get_task_count()), K(sqc.get_total_task_count())); } + } else if (root.get_type() == PHY_SELECT_INTO) { + ObPxSqcMeta &sqc = sqc_arg_.sqc_; + ObSelectIntoSpec *select_into_spec = reinterpret_cast(&root); +#ifdef OB_BUILD_CPP_ODPS + ObOdpsPartitionDownloaderMgr &odps_mgr = sqc_ctx.gi_pump_.get_odps_mgr(); + if (OB_FAIL(odps_mgr.init_uploader(select_into_spec->external_properties_.str_, + select_into_spec->external_partition_.str_, + select_into_spec->is_overwrite_, + sqc.get_task_count()))) { + LOG_WARN("failed to init odps uploader", K(ret)); + } +#endif } if (OB_SUCC(ret)) { if (OB_FAIL(root.register_to_datahub(ctx))) { diff --git a/src/sql/engine/px/ob_px_util.cpp b/src/sql/engine/px/ob_px_util.cpp index 0c0e45784..a6292c82b 100644 --- a/src/sql/engine/px/ob_px_util.cpp +++ b/src/sql/engine/px/ob_px_util.cpp @@ -257,7 +257,6 @@ int ObPXServerAddrUtil::get_external_table_loc( //For recovered cluster, the file addr may not in the cluster. Then igore it. LOG_WARN("filter files in location failed", K(ret)); } - if (OB_FAIL(ret)) { } else if (ext_file_urls.empty()) { const char* dummy_file_name = "#######DUMMY_FILE#######"; diff --git a/src/sql/engine/table/ob_external_table_access_service.cpp b/src/sql/engine/table/ob_external_table_access_service.cpp index 937c24eeb..6aab47fc2 100644 --- a/src/sql/engine/table/ob_external_table_access_service.cpp +++ b/src/sql/engine/table/ob_external_table_access_service.cpp @@ -22,8 +22,11 @@ #include "share/ob_device_manager.h" #include "lib/utility/ob_macro_utils.h" #include "sql/engine/table/ob_parquet_table_row_iter.h" +#ifdef OB_BUILD_CPP_ODPS +#include "sql/engine/table/ob_odps_table_row_iter.h" +#endif #include "sql/engine/cmd/ob_load_data_file_reader.h" -#include "sql/engine/table/ob_orc_table_row_iter.h" +//#include "sql/engine/table/ob_orc_table_row_iter.h" namespace oceanbase { @@ -565,11 +568,24 @@ int ObExternalTableAccessService::table_scan( LOG_WARN("alloc memory failed", K(ret)); } break; - case ObExternalFileFormat::ORC_FORMAT: - if (OB_ISNULL(row_iter = OB_NEWx(ObOrcTableRowIterator, (scan_param.allocator_)))) { + case ObExternalFileFormat::ODPS_FORMAT: +#ifdef OB_BUILD_CPP_ODPS + if (OB_ISNULL(row_iter = OB_NEWx(ObODPSTableRowIterator, (scan_param.allocator_)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("alloc memory failed", K(ret)); } +#else + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "external odps table"); + LOG_WARN("not support to read odps in opensource", K(ret)); +#endif + case ObExternalFileFormat::ORC_FORMAT: + // if (OB_ISNULL(row_iter = OB_NEWx(ObOrcTableRowIterator, (scan_param.allocator_)))) { + // ret = OB_ALLOCATE_MEMORY_FAILED; + // LOG_WARN("alloc memory failed", K(ret)); + // } + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected format", K(ret), "format", param.external_file_format_.format_type_); break; default: ret = OB_ERR_UNEXPECTED; @@ -603,6 +619,15 @@ int ObExternalTableAccessService::table_rescan(ObVTableScanParam ¶m, ObNewRo case ObExternalFileFormat::ORC_FORMAT: result->reset(); break; + case ObExternalFileFormat::ODPS_FORMAT: +#ifdef OB_BUILD_CPP_ODPS + result->reset(); +#else + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "external odps table"); + LOG_WARN("not support to read odps in opensource", K(ret)); +#endif + break; default: ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected format", K(ret), "format", param.external_file_format_.format_type_); @@ -778,6 +803,13 @@ int ObCSVTableRowIterator::init(const storage::ObTableScanParam *scan_param) } } } + for (int i = 0; i < scan_param_->key_ranges_.count(); ++i) { + int64_t start = 0; + int64_t step = 0; + int64_t part_id = scan_param_->key_ranges_.at(i).get_start_key().get_obj_ptr()[ObExternalTableUtils::PARTITION_ID].get_int(); + const ObString &file_url = scan_param_->key_ranges_.at(i).get_start_key().get_obj_ptr()[ObExternalTableUtils::FILE_URL].get_string(); + int64_t file_id = scan_param_->key_ranges_.at(i).get_start_key().get_obj_ptr()[ObExternalTableUtils::FILE_ID].get_int(); + } return ret; } @@ -847,7 +879,7 @@ int ObExternalTableRowIterator::calc_file_partition_list_value(const int64_t par } else if (OB_ISNULL(table_schema)) { ret = OB_TABLE_NOT_EXIST; LOG_WARN("table not exist", K(scan_param_->index_id_), K(scan_param_->tenant_id_)); - } else if (table_schema->is_partitioned_table() && table_schema->is_user_specified_partition_for_external_table()) { + } else if (table_schema->is_partitioned_table() && (table_schema->is_user_specified_partition_for_external_table() || table_schema->is_odps_external_table())) { if (OB_FAIL(table_schema->get_partition_by_part_id(part_id, CHECK_PARTITION_MODE_NORMAL, partition))) { LOG_WARN("get partition failed", K(ret), K(part_id)); } else if (OB_ISNULL(partition) || OB_UNLIKELY(partition->get_list_row_values().count() != 1) @@ -1126,7 +1158,6 @@ int ObCSVTableRowIterator::get_next_row() column_expr->set_evaluated_flag(eval_ctx); } } - return ret; } @@ -1267,7 +1298,6 @@ int ObCSVTableRowIterator::get_next_rows(int64_t &count, int64_t capacity) } count = returned_row_cnt; - return ret; } diff --git a/src/sql/engine/table/ob_odps_table_row_iter.cpp b/src/sql/engine/table/ob_odps_table_row_iter.cpp new file mode 100644 index 000000000..5888515e3 --- /dev/null +++ b/src/sql/engine/table/ob_odps_table_row_iter.cpp @@ -0,0 +1,2454 @@ +#ifdef OB_BUILD_CPP_ODPS +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_ENG +#include "ob_parquet_table_row_iter.h" +#include "lib/charset/ob_charset.h" +#include "sql/engine/px/ob_px_sqc_handler.h" + +namespace oceanbase { +namespace sql { + +int ObODPSTableRowIterator::OdpsPartition::reset() +{ + int ret = OB_SUCCESS; + try { + download_handle_->Complete(); + download_handle_ = NULL; + record_count_ = -1; + name_ = ""; + download_id_ = ""; + } catch (apsara::odps::sdk::OdpsTunnelException& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("odps exception occured when calling Complete method", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (const std::exception &ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("odps exception occured when calling Complete method", K(ret), K(ex.what()), KP(this)); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (...) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("odps exception occured when calling Complete method", K(ret)); + } + } + return ret; +} + +int ObODPSTableRowIterator::init_tunnel(const sql::ObODPSGeneralFormat &odps_format) +{ + int ret = OB_SUCCESS; + try { + if (OB_FAIL(odps_format_.deep_copy(odps_format))) { + LOG_WARN("failed to deep copy odps format", K(ret)); + } else if (OB_FAIL(odps_format_.decrypt())) { + LOG_WARN("failed to decrypt odps format", K(ret)); + } else { + LOG_TRACE("init tunnel format", K(ret)); + if (0 == odps_format_.access_type_.case_compare("aliyun") || + odps_format_.access_type_.empty()) { + account_ = apsara::odps::sdk::Account(std::string(apsara::odps::sdk::ACCOUNT_ALIYUN), + std::string(odps_format_.access_id_.ptr(), odps_format_.access_id_.length()), + std::string(odps_format_.access_key_.ptr(), odps_format_.access_key_.length())); + } else if (0 == odps_format_.access_type_.case_compare("sts")) { + account_ = apsara::odps::sdk::Account(std::string(apsara::odps::sdk::ACCOUNT_STS), + std::string(odps_format_.sts_token_.ptr(), odps_format_.sts_token_.length())); + } else if (0 == odps_format_.access_type_.case_compare("token")) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("unsupported access type", K(ret), K(odps_format_.access_type_)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "ODPS access type: token"); + } else if (0 == odps_format_.access_type_.case_compare("domain")) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("unsupported access type", K(ret), K(odps_format_.access_type_)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "ODPS access type: domain"); + } else if (0 == odps_format_.access_type_.case_compare("taobao")) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("unsupported access type", K(ret), K(odps_format_.access_type_)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "ODPS access type: taobao"); + } else if (0 == odps_format_.access_type_.case_compare("app")) { + account_ = apsara::odps::sdk::Account(std::string(apsara::odps::sdk::ACCOUNT_APPLICATION), + std::string(odps_format_.access_id_.ptr(), odps_format_.access_id_.length()), + std::string(odps_format_.access_key_.ptr(), odps_format_.access_key_.length())); + } else { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "ODPS access type"); + } + conf_.SetAccount(account_); + conf_.SetEndpoint(std::string(odps_format_.endpoint_.ptr(), odps_format_.endpoint_.length())); + conf_.SetUserAgent("OB_ACCESS_ODPS"); + conf_.SetTunnelQuotaName(std::string(odps_format_.quota_.ptr(), odps_format_.quota_.length())); + if (0 == odps_format_.compression_code_.case_compare("zlib")) { + conf_.SetCompressOption(apsara::odps::sdk::CompressOption::ZLIB_COMPRESS); + } else if (0 == odps_format_.compression_code_.case_compare("zstd")) { + conf_.SetCompressOption(apsara::odps::sdk::CompressOption::ZSTD_COMPRESS); + } else if (0 == odps_format_.compression_code_.case_compare("lz4")) { + conf_.SetCompressOption(apsara::odps::sdk::CompressOption::LZ4_COMPRESS); + } else if (0 == odps_format_.compression_code_.case_compare("odps_lz4")) { + conf_.SetCompressOption(apsara::odps::sdk::CompressOption::ODPS_LZ4_COMPRESS); + } else { + conf_.SetCompressOption(apsara::odps::sdk::CompressOption::NO_COMPRESS); + } + tunnel_.Init(conf_); // do not need try catch + if (OB_ISNULL((odps_ = apsara::odps::sdk::IODPS::Create(conf_, // do not need try catch + std::string(odps_format_.project_.ptr(), + odps_format_.project_.length()))).get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted null ptr", K(ret)); + } else if (OB_ISNULL((odps_->GetTables()).get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted null ptr", K(ret)); + } else if (OB_ISNULL((table_handle_ = odps_->GetTables()->Get(std::string(odps_format_.project_.ptr(), odps_format_.project_.length()), // do not need try catch + std::string(odps_format_.schema_.ptr(), odps_format_.schema_.length()), + std::string(odps_format_.table_.ptr(), odps_format_.table_.length()))).get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted null ptr", K(ret)); + } + } + } catch (apsara::odps::sdk::OdpsTunnelException& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when call external driver api", K(ret), K(ex.what()), KP(this)); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (const std::exception &ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when call external driver api", K(ret), K(ex.what()), KP(this)); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (...) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when call external driver api", K(ret), KP(this)); + } + } + + return ret; +} + +int ObODPSTableRowIterator::create_downloader(ObString &part_spec, apsara::odps::sdk::IDownloadPtr &downloader) +{ + int ret = OB_SUCCESS; + try { + apsara::odps::sdk::IDownloadPtr download_handle = NULL; + downloader = NULL; + std::string project(odps_format_.project_.ptr(), odps_format_.project_.length()); + std::string table(odps_format_.table_.ptr(), odps_format_.table_.length()); + std::string std_part_spec(part_spec.ptr(), part_spec.length()); + std::string download_id(""); + std::string schema(odps_format_.schema_.ptr(), odps_format_.schema_.length()); + download_handle = tunnel_.CreateDownload(project, + table, + std_part_spec, + download_id, + schema); + if (OB_ISNULL(download_handle.get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted null ptr", K(ret)); + } else { + downloader = download_handle; + } + } catch (apsara::odps::sdk::OdpsTunnelException& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when call CreateDownload", K(ret), K(ex.what()), KP(this)); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (const std::exception &ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when call CreateDownload", K(ret), K(ex.what()), KP(this)); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (...) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when call CreateDownload", K(ret), KP(this)); + } + } + return ret; +} + +int ObODPSTableRowIterator::init(const storage::ObTableScanParam *scan_param) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(scan_param)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("scan param is null", K(ret)); + } else if (OB_FAIL(ObExternalTableRowIterator::init(scan_param))) { + LOG_WARN("failed to call ObExternalTableRowIterator::init", K(ret)); + } else { + if (OB_FAIL(init_tunnel(scan_param_->external_file_format_.odps_format_))) { + LOG_WARN("failed to init odps tunnel", K(ret)); + } else if (OB_FAIL(pull_column())) { + LOG_WARN("failed to pull column info", K(ret)); + } else if (OB_FAIL(prepare_expr())) { + LOG_WARN("failed to prepare expr", K(ret)); + } + } + return ret; +} + +int ObODPSTableRowIterator::next_task() +{ + int ret = OB_SUCCESS; + ObEvalCtx &eval_ctx = scan_param_->op_->get_eval_ctx(); + int64_t task_idx = state_.task_idx_; + int64_t start = 0; + int64_t step = 0; + if (++task_idx >= scan_param_->key_ranges_.count()) { + ret = OB_ITER_END; + LOG_WARN("odps table iter end", K(total_count_), K(state_), K(task_idx), K(ret)); + } else { + ObEvalCtx &ctx = scan_param_->op_->get_eval_ctx(); + ObPxSqcHandler *sqc = ctx.exec_ctx_.get_sqc_handler();// if sqc is not NULL, odps read is in px plan + if (OB_FAIL(ObExternalTableUtils::resolve_odps_start_step(scan_param_->key_ranges_.at(task_idx), + ObExternalTableUtils::LINE_NUMBER, + start, + step))) { + LOG_WARN("failed to resolve range in external table", K(ret)); + } else { + try { + const ObString &part_spec = scan_param_->key_ranges_.at(task_idx).get_start_key().get_obj_ptr()[ObExternalTableUtils::FILE_URL].get_string(); + int64_t part_id = scan_param_->key_ranges_.at(task_idx).get_start_key().get_obj_ptr()[ObExternalTableUtils::PARTITION_ID].get_int(); + std::string project(odps_format_.project_.ptr(), odps_format_.project_.length()); + std::string table(odps_format_.table_.ptr(), odps_format_.table_.length()); + std::string std_part_spec(part_spec.ptr(), part_spec.length()); + std::string download_id(""); + std::string schema(odps_format_.schema_.ptr(), odps_format_.schema_.length()); + std::vector column_names; + if (OB_ISNULL(sqc) && + OB_ISNULL((state_.download_handle_ = tunnel_.CreateDownload(project, + table, + std_part_spec, + download_id, + schema)).get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted null ptr", K(ret)); + } else if (OB_NOT_NULL(sqc) && + !sqc->get_sqc_ctx().gi_pump_.is_odps_downloader_inited() && + OB_ISNULL((state_.download_handle_ = tunnel_.CreateDownload(project, + table, + std_part_spec, + download_id, + schema)).get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted null ptr", K(ret)); + } else if (OB_NOT_NULL(sqc) && + sqc->get_sqc_ctx().gi_pump_.is_odps_downloader_inited() && + OB_FAIL(sqc->get_sqc_ctx().gi_pump_.get_odps_downloader(part_id, state_.download_handle_))) { + LOG_WARN("failed to get odps downloader", K(ret), K(part_id)); + } else if (OB_ISNULL(state_.download_handle_.get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted null ptr", K(ret)); + } else if (OB_ISNULL((state_.record_reader_handle_ = state_.download_handle_->OpenReader(start, + step, + column_names, + true)).get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted null ptr", K(ret)); + } else if (OB_FAIL(calc_file_partition_list_value(part_id, arena_alloc_, state_.part_list_val_))) { + LOG_WARN("failed to calc parttion list value", K(part_id), K(ret)); + } else { + state_.task_idx_ = task_idx; + state_.part_id_ = part_id; + state_.start_ = start; + state_.step_ = step; + state_.count_ = 0; + state_.is_from_gi_pump_ = OB_NOT_NULL(sqc) && sqc->get_sqc_ctx().gi_pump_.is_odps_downloader_inited(); + state_.download_id_ = state_.download_handle_->GetDownloadId(); + state_.part_spec_ = std_part_spec; + // what if error occur after this line, how to close state_.record_reader_handle_? + LOG_TRACE("get a new task", K(ret), K(batch_size_), K(state_)); + if (OB_SUCC(ret) && -1 == batch_size_) { // exec once only + batch_size_ = eval_ctx.max_batch_size_; + if (0 == batch_size_) { + // even state_.record_reader_handle_ was destroyed, record_/records_ is still valid. + // see class RecordReader : public IRecordReader to check it. + if (OB_ISNULL((record_ = state_.record_reader_handle_->CreateBufferRecord()).get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null ptr", K(ret)); + } else { + LOG_TRACE("odps record_ inited", K(ret), K(batch_size_)); + } + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < batch_size_; ++i) { + if (OB_ISNULL((records_[i] = state_.record_reader_handle_->CreateBufferRecord()).get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null ptr", K(ret), K(i)); + } + } + LOG_TRACE("odps records_ inited", K(ret), K(batch_size_)); + } + } + } + } catch (apsara::odps::sdk::OdpsException& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("odps exception occured when calling odps api", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (const std::exception &ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("odps exception occured when calling odps api", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (...) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("odps exception occured when calling odps api", K(ret)); + } + } + } + } + return ret; +} + +int ObODPSTableRowIterator::print_type_map_user_info(apsara::odps::sdk::ODPSColumnTypeInfo odps_type_info, + const ObExpr *ob_type_expr) +{ + int ret = OB_SUCCESS; + try { + std::string odps_type_str = odps_type_info.ToTypeString();//need try catch + const char* odps_type_cstr = odps_type_str.c_str(); + const char* ob_type_cstr = ""; + if (OB_NOT_NULL(ob_type_expr)) { + ObArrayWrap buf; + int64_t pos = 0; + ob_type_cstr = ob_obj_type_str(ob_type_expr->datum_meta_.type_); + if (OB_SUCCESS == buf.allocate_array(arena_alloc_, 128)) { // 128 is enough to hold user info str + ob_sql_type_str(buf.get_data(), buf.count(), pos, + ob_type_expr->datum_meta_.type_, + ob_type_expr->max_length_, + ob_type_expr->datum_meta_.precision_, + ob_type_expr->datum_meta_.scale_, + ob_type_expr->datum_meta_.cs_type_); + if (pos < buf.count()) { + buf.at(pos++) = '\0'; + ob_type_cstr = buf.get_data(); + } + } + } + LOG_USER_ERROR(OB_EXTERNAL_ODPS_COLUMN_TYPE_MISMATCH, odps_type_cstr, ob_type_cstr); + } catch (apsara::odps::sdk::OdpsTunnelException& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when call ToTypeString", K(ret), K(ex.what()), KP(this)); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (const std::exception& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when call ToTypeString", K(ret), K(ex.what()), KP(this)); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (...) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when call ToTypeString", K(ret), KP(this)); + } + } + return ret; +} + +int ObODPSTableRowIterator::check_type_static(apsara::odps::sdk::ODPSColumnTypeInfo odps_type_info, + const ObExpr *ob_type_expr) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(ob_type_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null ptr", K(ret)); + } else { + const apsara::odps::sdk::ODPSColumnType odps_type = odps_type_info.mType; + const int32_t odps_type_length = odps_type_info.mSpecifiedLength; + const int32_t odps_type_precision = odps_type_info.mPrecision; + const int32_t odps_type_scale = odps_type_info.mScale; + const ObObjType ob_type = ob_type_expr->obj_meta_.get_type(); + const int32_t ob_type_length = ob_type_expr->max_length_; + const int32_t ob_type_precision = ob_type_expr->datum_meta_.precision_; + const int32_t ob_type_scale = ob_type_expr->datum_meta_.scale_; + switch(odps_type) + { + case apsara::odps::sdk::ODPS_TINYINT: + case apsara::odps::sdk::ODPS_BOOLEAN: + { + if (ObTinyIntType == ob_type) { + // odps_type to ob_type is valid + } else { + ret = OB_EXTERNAL_ODPS_COLUMN_TYPE_MISMATCH; + LOG_WARN("invalid odps type map to ob type", K(ret), K(odps_type), K(ob_type)); + print_type_map_user_info(odps_type_info, ob_type_expr); + } + break; + } + case apsara::odps::sdk::ODPS_SMALLINT: + { + if (ObSmallIntType == ob_type) { + // odps_type to ob_type is valid + } else { + ret = OB_EXTERNAL_ODPS_COLUMN_TYPE_MISMATCH; + LOG_WARN("invalid odps type map to ob type", K(ret), K(odps_type), K(ob_type)); + print_type_map_user_info(odps_type_info, ob_type_expr); + } + break; + } + case apsara::odps::sdk::ODPS_INTEGER: + { + if (ObInt32Type == ob_type) { + // odps_type to ob_type is valid + } else { + ret = OB_EXTERNAL_ODPS_COLUMN_TYPE_MISMATCH; + LOG_WARN("invalid odps type map to ob type", K(ret), K(odps_type), K(ob_type)); + print_type_map_user_info(odps_type_info, ob_type_expr); + } + break; + } + case apsara::odps::sdk::ODPS_BIGINT: + { + if (ObIntType == ob_type) { + // odps_type to ob_type is valid + } else { + ret = OB_EXTERNAL_ODPS_COLUMN_TYPE_MISMATCH; + LOG_WARN("invalid odps type map to ob type", K(ret), K(odps_type), K(ob_type)); + print_type_map_user_info(odps_type_info, ob_type_expr); + } + break; + } + case apsara::odps::sdk::ODPS_FLOAT: + { + if (ObFloatType == ob_type && ob_type_length == 12 && ob_type_scale == -1) { + // odps_type to ob_type is valid + } else { + ret = OB_EXTERNAL_ODPS_COLUMN_TYPE_MISMATCH; + LOG_WARN("invalid odps type map to ob type", K(ret), K(odps_type), K(ob_type)); + print_type_map_user_info(odps_type_info, ob_type_expr); + } + break; + } + case apsara::odps::sdk::ODPS_DOUBLE: + { + if (ObDoubleType == ob_type && ob_type_length == 23 && ob_type_scale == -1) { + // odps_type to ob_type is valid + } else { + ret = OB_EXTERNAL_ODPS_COLUMN_TYPE_MISMATCH; + LOG_WARN("invalid odps type map to ob type", K(ret), K(odps_type), K(ob_type)); + print_type_map_user_info(odps_type_info, ob_type_expr); + } + break; + } + case apsara::odps::sdk::ODPS_DECIMAL: + { + if (ObDecimalIntType == ob_type || + ObNumberType == ob_type) { + // odps_type to ob_type is valid + if (ob_type_precision != odps_type_precision || // in ObExpr, max_length_ is decimal type's precision + ob_type_scale != odps_type_scale) { + ret = OB_EXTERNAL_ODPS_COLUMN_TYPE_MISMATCH; + LOG_WARN("invalid precision or scale or length", K(ret), K(odps_type), K(ob_type), + K(ob_type_length), + K(ob_type_precision), + K(ob_type_scale), + K(odps_type_length), + K(odps_type_precision), + K(odps_type_scale)); + print_type_map_user_info(odps_type_info, ob_type_expr); + } + } else { + ret = OB_EXTERNAL_ODPS_COLUMN_TYPE_MISMATCH; + print_type_map_user_info(odps_type_info, ob_type_expr); + } + break; + } + case apsara::odps::sdk::ODPS_CHAR: + { + if (ObCharType == ob_type) { + // odps_type to ob_type is valid + if (ob_type_length != odps_type_length) { + ret = OB_EXTERNAL_ODPS_COLUMN_TYPE_MISMATCH; + LOG_WARN("invalid precision or scale or length", K(ret), K(odps_type), K(ob_type), + K(ob_type_length), + K(ob_type_precision), + K(ob_type_scale), + K(odps_type_length), + K(odps_type_precision), + K(odps_type_scale)); + print_type_map_user_info(odps_type_info, ob_type_expr); + } + } else { + ret = OB_EXTERNAL_ODPS_COLUMN_TYPE_MISMATCH; + LOG_WARN("invalid odps type map to ob type", K(ret), K(odps_type), K(ob_type)); + print_type_map_user_info(odps_type_info, ob_type_expr); + } + break; + } + case apsara::odps::sdk::ODPS_VARCHAR: + { + if (ObVarcharType == ob_type) { + // odps_type to ob_type is valid + if (ob_type_length != odps_type_length) { + ret = OB_EXTERNAL_ODPS_COLUMN_TYPE_MISMATCH; + LOG_WARN("invalid precision or scale or length", K(ret), K(odps_type), K(ob_type), + K(ob_type_length), + K(ob_type_precision), + K(ob_type_scale), + K(odps_type_length), + K(odps_type_precision), + K(odps_type_scale)); + print_type_map_user_info(odps_type_info, ob_type_expr); + } + } else { + ret = OB_EXTERNAL_ODPS_COLUMN_TYPE_MISMATCH; + LOG_WARN("invalid odps type map to ob type", K(ret), K(odps_type), K(ob_type)); + print_type_map_user_info(odps_type_info, ob_type_expr); + } + break; + } + case apsara::odps::sdk::ODPS_STRING: + case apsara::odps::sdk::ODPS_BINARY://check length at runtime + { + if (ObVarcharType == ob_type || + ObTinyTextType == ob_type || + ObTextType == ob_type || + ObLongTextType == ob_type || + ObMediumTextType == ob_type) { + // odps_type to ob_type is valid + } else { + ret = OB_EXTERNAL_ODPS_COLUMN_TYPE_MISMATCH; + LOG_WARN("invalid odps type map to ob type", K(ret), K(odps_type), K(ob_type)); + print_type_map_user_info(odps_type_info, ob_type_expr); + } + break; + } + case apsara::odps::sdk::ODPS_TIMESTAMP: + { + if (ObTimestampType == ob_type) { + // odps_type to ob_type is valid + if (ob_type_scale < 6) { + ret = OB_EXTERNAL_ODPS_COLUMN_TYPE_MISMATCH; + LOG_WARN("invalid precision or scale or length", K(ret), K(odps_type), K(ob_type), + K(ob_type_length), + K(ob_type_precision), + K(ob_type_scale), + K(odps_type_length), + K(odps_type_precision), + K(odps_type_scale)); + print_type_map_user_info(odps_type_info, ob_type_expr); + } + } else { + ret = OB_EXTERNAL_ODPS_COLUMN_TYPE_MISMATCH; + LOG_WARN("invalid odps type map to ob type", K(ret), K(odps_type), K(ob_type)); + print_type_map_user_info(odps_type_info, ob_type_expr); + } + break; + } + case apsara::odps::sdk::ODPS_TIMESTAMP_NTZ: + { + if (ObDateTimeType == ob_type) { + // odps_type to ob_type is valid + if (ob_type_scale < 6) { + ret = OB_EXTERNAL_ODPS_COLUMN_TYPE_MISMATCH; + LOG_WARN("invalid precision or scale or length", K(ret), K(odps_type), K(ob_type), + K(ob_type_length), + K(ob_type_precision), + K(ob_type_scale), + K(odps_type_length), + K(odps_type_precision), + K(odps_type_scale)); + print_type_map_user_info(odps_type_info, ob_type_expr); + } + } else { + ret = OB_EXTERNAL_ODPS_COLUMN_TYPE_MISMATCH; + LOG_WARN("invalid odps type map to ob type", K(ret), K(odps_type), K(ob_type)); + print_type_map_user_info(odps_type_info, ob_type_expr); + } + break; + } + case apsara::odps::sdk::ODPS_DATE: + { + if (ObDateType == ob_type) { + // odps_type to ob_type is valid + } else { + ret = OB_EXTERNAL_ODPS_COLUMN_TYPE_MISMATCH; + LOG_WARN("invalid odps type map to ob type", K(ret), K(odps_type), K(ob_type)); + print_type_map_user_info(odps_type_info, ob_type_expr); + } + break; + } + case apsara::odps::sdk::ODPS_DATETIME: + { + if (ObDateTimeType == ob_type) { + // odps_type to ob_type is valid + if (ob_type_scale < 3) { + ret = OB_EXTERNAL_ODPS_COLUMN_TYPE_MISMATCH; + LOG_WARN("invalid precision or scale or length", K(ret), K(odps_type), K(ob_type), + K(ob_type_length), + K(ob_type_precision), + K(ob_type_scale), + K(odps_type_length), + K(odps_type_precision), + K(odps_type_scale)); + print_type_map_user_info(odps_type_info, ob_type_expr); + } + } else { + ret = OB_EXTERNAL_ODPS_COLUMN_TYPE_MISMATCH; + LOG_WARN("invalid odps type map to ob type", K(ret), K(odps_type), K(ob_type)); + print_type_map_user_info(odps_type_info, ob_type_expr); + } + break; + } + default: + { + ret = OB_NOT_SUPPORTED; + LOG_WARN("unsupported odps type", K(ret)); + } + } + } + return ret; +} + +int ObODPSTableRowIterator::prepare_expr() +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(scan_param_) || OB_ISNULL(scan_param_->ext_file_column_exprs_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted null ptr", KP(scan_param_), K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < scan_param_->ext_file_column_exprs_->count(); ++i) { + const ObExpr *cur_expr = scan_param_->ext_file_column_exprs_->at(i); // do no check is NULL or not + int target_idx = cur_expr->extra_ - 1; + if (OB_UNLIKELY(cur_expr->type_ == T_PSEUDO_EXTERNAL_FILE_COL && + (target_idx < 0 || target_idx >= column_list_.count()))) { + ret = OB_EXTERNAL_ODPS_UNEXPECTED_ERROR; + LOG_WARN("unexcepted target_idx", K(ret), K(target_idx), K(column_list_.count())); + LOG_USER_ERROR(OB_EXTERNAL_ODPS_UNEXPECTED_ERROR, "wrong column index point to odps, please check the index of external$tablecol[index] and metadata$partition_list_col[index]"); + } else if (OB_FAIL(target_column_id_list_.push_back(target_idx))) { + LOG_WARN("failed to keep target_idx", K(ret)); + } else if (cur_expr->type_ == T_PSEUDO_EXTERNAL_FILE_COL && + OB_FAIL(check_type_static(column_list_.at(target_idx).type_info_, cur_expr))) { + LOG_WARN("odps type map ob type not support", K(ret), K(target_idx)); + } + } + ObEvalCtx &eval_ctx = scan_param_->op_->get_eval_ctx(); + void *vec_mem = NULL; + void *records_mem = NULL; + malloc_alloc_.set_attr(lib::ObMemAttr(scan_param_->tenant_id_, "ODPSRowIter")); + if (OB_FAIL(ret)) { + // do nothing + } else if (0 > eval_ctx.max_batch_size_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected max_batch_size_", K(ret), K(eval_ctx.max_batch_size_)); + } else { + if (0 == eval_ctx.max_batch_size_) { + // do nothing + } else if (OB_ISNULL(vec_mem = malloc_alloc_.alloc(ObBitVector::memory_size(eval_ctx.max_batch_size_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory for skip", K(ret), K(eval_ctx.max_batch_size_)); + } else if (OB_ISNULL(records_mem = malloc_alloc_.alloc(eval_ctx.max_batch_size_ * sizeof(apsara::odps::sdk::ODPSTableRecordPtr)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory for skip", K(ret), K(eval_ctx.max_batch_size_)); + } else { + bit_vector_cache_ = to_bit_vector(vec_mem); + bit_vector_cache_->reset(eval_ctx.max_batch_size_); + records_ = static_cast(records_mem); + for (int64_t i = 0; i < eval_ctx.max_batch_size_; ++i) { + new (&records_[i]) apsara::odps::sdk::ODPSTableRecordPtr; + } + } + } + } + return ret; +} + +int ObODPSTableRowIterator::pull_partition_info() +{ + int ret = OB_SUCCESS; + partition_list_.reset(); + std::vector part_specs; + try { + if (OB_ISNULL(table_handle_.get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted null ptr", K(ret)); + } else { + table_handle_->GetPartitionNames(part_specs); + is_part_table_ = true; + } + } catch (apsara::odps::sdk::OdpsException& ex) { + std::string ex_msg = ex.what(); + if (std::string::npos != ex_msg.find("ODPS-0110031")) { // ODPS-0110031 means table is not a partitional table + is_part_table_ = false; + } else if (OB_FAIL(ret)) { + //do nothing + } else { + ret = OB_ODPS_ERROR; + LOG_WARN("failed to call GetPartitionNames method in ODPS sdk", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (const std::exception& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("failed to call GetPartitionNames method in ODPS sdk", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (...) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("odps exception occured when calling GetPartitionNames method", K(ret)); + } + } + if (OB_SUCC(ret) && !is_part_table_) { + part_specs.push_back(""); + } + try { + std::string project(odps_format_.project_.ptr(), odps_format_.project_.length()); + std::string table(odps_format_.table_.ptr(), odps_format_.table_.length()); + std::string schema(odps_format_.schema_.ptr(), odps_format_.schema_.length()); + for (std::vector::iterator part_spec = part_specs.begin(); OB_SUCC(ret) && part_spec != part_specs.end(); part_spec++) { + std::string download_id(""); + apsara::odps::sdk::IDownloadPtr download_handle = NULL; + int64_t record_count = -1; + if (OB_ISNULL((download_handle = tunnel_.CreateDownload(project, + table, + *part_spec, + download_id, + schema)).get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted null ptr", K(ret)); + } else if (FALSE_IT(download_id = download_handle->GetDownloadId())) { + } else if (FALSE_IT(record_count = download_handle->GetRecordCount())) { + } else if (OB_FAIL(partition_list_.push_back(OdpsPartition(*part_spec, + download_handle, + download_id, + record_count)))){ + LOG_WARN("failed to push back partition_list_", K(ret)); + } + } + } catch (apsara::odps::sdk::OdpsTunnelException& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when call external driver api", K(ret), K(ex.what()), KP(this)); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (const std::exception &ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when call external driver api", K(ret), K(ex.what()), KP(this)); + } + } catch (...) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when call external driver api", K(ret), KP(this)); + } + } + return ret; +} + +int ObODPSTableRowIterator::pull_column() { + int ret = OB_SUCCESS; + if (OB_ISNULL(table_handle_.get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted null ptr", K(ret)); + } else { + try { + apsara::odps::sdk::IODPSTableSchemaPtr schema_handle = table_handle_->GetSchema(); + if (OB_ISNULL(schema_handle.get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted null ptr", K(ret)); + } else { + for (uint32_t i = 0; OB_SUCC(ret) && i < schema_handle->GetColumnCount(); i++) { + if (OB_FAIL(column_list_.push_back(OdpsColumn(schema_handle->GetTableColumn(i).GetName(), + schema_handle->GetTableColumn(i).GetTypeInfo())))) { + LOG_WARN("failed to push back column_list_", K(ret)); + } + } + } + } catch (apsara::odps::sdk::OdpsTunnelException& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("failed to call GetSchema method in ODPS sdk", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (const std::exception &ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("failed to call GetSchema method in ODPS sdk", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (...) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("odps exception occured when calling GetSchema method", K(ret)); + } + } + } + return ret; +} + +void ObODPSTableRowIterator::reset() +{ + state_.reuse(); // reset state_ to initial values for rescan +} + +int ObODPSTableRowIterator::StateValues::reuse() +{ + int ret = OB_SUCCESS; + try { + if (-1 == task_idx_) { + // do nothing + } else if (OB_ISNULL(record_reader_handle_.get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null ptr", K(ret), K(lbt())); + } else if (OB_ISNULL(download_handle_.get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null ptr", K(ret), K(lbt())); + } else { + record_reader_handle_->Close(); + record_reader_handle_.reset(); + if (!is_from_gi_pump_) { + download_handle_->Complete(); + } + download_handle_.reset(); + } + } catch (const apsara::odps::sdk::OdpsTunnelException& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("odps exception occured when calling Complete method, ignore it", K(ret), K(ex.what())); + //LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (const std::exception& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("odps exception occured when calling Complete method, ignore it", K(ret), K(ex.what())); + //LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (...) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("odps exception occured when calling Complete method, ignore it", K(ret)); + } + } + task_idx_ = -1; + part_id_ = 0; + start_ = 0; + step_ = 0; + count_ = 0; + part_spec_.clear(); + download_id_.clear(); + part_list_val_.reset(); + is_from_gi_pump_ = false; + return ret; +} + +int ObODPSTableRowIterator::get_next_rows(int64_t &count, int64_t capacity) +{ + int ret = 0; + ObMallocHookAttrGuard guard(mem_attr_); + int64_t returned_row_cnt = 0; + ObEvalCtx &ctx = scan_param_->op_->get_eval_ctx(); + const ExprFixedArray &file_column_exprs = *(scan_param_->ext_file_column_exprs_); + if (state_.count_ >= state_.step_ && OB_FAIL(next_task())) { + if (OB_ITER_END != ret) { + LOG_WARN("get next task failed", K(ret)); + } + } else { + int64_t returned_row_cnt = 0; + try { + while(returned_row_cnt < capacity && OB_SUCC(ret)) { + if (!(state_.record_reader_handle_->Read(*records_[returned_row_cnt]))) { + break; + } else { + ++state_.count_; + ++total_count_; + ++returned_row_cnt; + } + } + } catch (apsara::odps::sdk::OdpsTunnelException& ex) { + if (OB_SUCC(ret)) { + std::string ex_msg = ex.what(); + if (std::string::npos != ex_msg.find("EOF")) { //EOF + LOG_TRACE("odps eof", K(ret), K(total_count_), K(returned_row_cnt), K(ex.what())); + if (0 == returned_row_cnt && (INT64_MAX == state_.step_ || state_.count_ == state_.step_)) { + state_.step_ = state_.count_; // goto get next task + count = 0; + } else if (0 == returned_row_cnt) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected returned_row_cnt", K(total_count_), K(returned_row_cnt), K(state_), K(ret)); + } + } else { + ret = OB_ODPS_ERROR; + LOG_WARN("odps exception occured when calling Read method", K(ret), K(total_count_), K(returned_row_cnt), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } + } catch (const std::exception& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("odps exception occured when calling Read method", K(ret), K(total_count_), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (...) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("odps exception occured when calling Read method", K(ret), K(total_count_)); + } + } + if (OB_FAIL(ret)) { + // do nothing + } else if (0 == returned_row_cnt && (INT64_MAX == state_.step_ || state_.count_ == state_.step_)) { + state_.step_ = state_.count_; // goto get next task + count = 0; + } else if (0 == returned_row_cnt) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected returned_row_cnt", K(total_count_), K(returned_row_cnt), K(state_), K(ret)); + } else { + for (int64_t column_idx = 0; OB_SUCC(ret) && column_idx < target_column_id_list_.count(); ++column_idx) { + uint32_t target_idx = target_column_id_list_.at(column_idx); + ObExpr &expr = *file_column_exprs.at(column_idx); + ObDatum *datums = expr.locate_batch_datums(ctx); + ObObjType type = expr.obj_meta_.get_type(); + if (expr.type_ == T_PSEUDO_PARTITION_LIST_COL) { + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < returned_row_cnt; ++row_idx) { + int64_t loc_idx = file_column_exprs.at(column_idx)->extra_ - 1; + if (OB_UNLIKELY(loc_idx < 0 || loc_idx >= state_.part_list_val_.get_count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted loc_idx", K(ret), K(loc_idx), K(state_.part_list_val_.get_count()), KP(&state_.part_list_val_)); + } else if (state_.part_list_val_.get_cell(loc_idx).is_null()) { + datums[row_idx].set_null(); + } else { + CK (OB_NOT_NULL(datums[row_idx].ptr_)); + OZ (datums[row_idx].from_obj(state_.part_list_val_.get_cell(loc_idx))); + } + } + } else { + apsara::odps::sdk::ODPSColumnType odps_type = column_list_.at(target_idx).type_info_.mType; + try { + switch(odps_type) + { + case apsara::odps::sdk::ODPS_BOOLEAN: + { + if ((ObTinyIntType == type || + ObSmallIntType == type || + ObMediumIntType == type || + ObInt32Type == type || + ObIntType == type) && !is_oracle_mode()) { + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < returned_row_cnt; ++row_idx) { + const bool* v = records_[row_idx]->GetBoolValue(target_idx); + if (v == NULL) { + datums[row_idx].set_null(); + } else { + datums[row_idx].set_int(*v); + } + } + } else if (ObNumberType == type && is_oracle_mode()) { + ObEvalCtx::BatchInfoScopeGuard batch_info_guard(ctx); + batch_info_guard.set_batch_idx(0); + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < returned_row_cnt; ++row_idx) { + batch_info_guard.set_batch_idx(row_idx); + const bool* v = records_[row_idx]->GetBoolValue(target_idx); + if (v == NULL) { + datums[row_idx].set_null(); + } else { + int64_t in_val = *v; + ObNumStackOnceAlloc tmp_alloc; + number::ObNumber nmb; + OZ(ObOdpsDataTypeCastUtil::common_int_number_wrap(expr, in_val, tmp_alloc, nmb), in_val); + if (OB_FAIL(ret)) { + LOG_WARN("failed to cast int to number", K(ret), K(row_idx), K(column_idx)); + } + } + } + } else if (ObDecimalIntType == type && is_oracle_mode()) { + ObEvalCtx::BatchInfoScopeGuard batch_info_guard(ctx); + batch_info_guard.set_batch_idx(0); + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < returned_row_cnt; ++row_idx) { + batch_info_guard.set_batch_idx(row_idx); + const bool* v = records_[row_idx]->GetBoolValue(target_idx); + if (v == NULL) { + datums[row_idx].set_null(); + } else { + int64_t in_val = *v; + ObDecimalInt *decint = nullptr; + int32_t int_bytes = 0; + ObDecimalIntBuilder tmp_alloc; + ObScale out_scale = expr.datum_meta_.scale_; + ObScale in_scale = 0; + ObPrecision out_prec = expr.datum_meta_.precision_; + ObPrecision in_prec = + ObAccuracy::MAX_ACCURACY2[lib::is_oracle_mode()][type] + .get_precision(); + const static int64_t DECINT64_MAX = get_scale_factor(MAX_PRECISION_DECIMAL_INT_64); + if (in_prec > MAX_PRECISION_DECIMAL_INT_64 && in_val < DECINT64_MAX) { + in_prec = MAX_PRECISION_DECIMAL_INT_64; + } + if (OB_FAIL(wide::from_integer(in_val, tmp_alloc, decint, int_bytes, in_prec))) { + LOG_WARN("from_integer failed", K(ret), K(in_val), K(row_idx), K(column_idx)); + } else if (ObDatumCast::need_scale_decimalint(in_scale, in_prec, out_scale, out_prec)) { + ObDecimalIntBuilder res_val; + if (OB_FAIL(ObDatumCast::common_scale_decimalint(decint, int_bytes, in_scale, out_scale, + out_prec, expr.extra_, res_val))) { + LOG_WARN("scale decimal int failed", K(ret), K(row_idx), K(column_idx)); + } else { + datums[row_idx].set_decimal_int(res_val.get_decimal_int(), res_val.get_int_bytes()); + } + } else { + datums[row_idx].set_decimal_int(decint, int_bytes); + } + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(type), K(column_idx)); + } + break; + } + case apsara::odps::sdk::ODPS_TINYINT: + case apsara::odps::sdk::ODPS_SMALLINT: + case apsara::odps::sdk::ODPS_INTEGER: + case apsara::odps::sdk::ODPS_BIGINT: + { + if ((ObTinyIntType == type || + ObSmallIntType == type || + ObMediumIntType == type || + ObInt32Type == type || + ObIntType == type) && !is_oracle_mode()) { + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < returned_row_cnt; ++row_idx) { + const int64_t* v = records_[row_idx]->GetIntValue(target_idx, odps_type); + if (v == NULL) { + datums[row_idx].set_null(); + } else { + datums[row_idx].set_int(*v); + } + } + } else if (ObNumberType == type && is_oracle_mode()) { + ObEvalCtx::BatchInfoScopeGuard batch_info_guard(ctx); + batch_info_guard.set_batch_idx(0); + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < returned_row_cnt; ++row_idx) { + batch_info_guard.set_batch_idx(row_idx); + const int64_t* v = records_[row_idx]->GetIntValue(target_idx, odps_type); + if (v == NULL) { + datums[row_idx].set_null(); + } else { + int64_t in_val = *v; + ObNumStackOnceAlloc tmp_alloc; + number::ObNumber nmb; + OZ(ObOdpsDataTypeCastUtil::common_int_number_wrap(expr, in_val, tmp_alloc, nmb), in_val); + if (OB_FAIL(ret)) { + LOG_WARN("failed to cast int to number", K(ret), K(row_idx), K(column_idx)); + } + } + } + } else if (ObDecimalIntType == type && is_oracle_mode()) { + ObEvalCtx::BatchInfoScopeGuard batch_info_guard(ctx); + batch_info_guard.set_batch_idx(0); + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < returned_row_cnt; ++row_idx) { + batch_info_guard.set_batch_idx(row_idx); + const int64_t* v = records_[row_idx]->GetIntValue(target_idx, odps_type); + if (v == NULL) { + datums[row_idx].set_null(); + } else { + int64_t in_val = *v; + ObDecimalInt *decint = nullptr; + int32_t int_bytes = 0; + ObDecimalIntBuilder tmp_alloc; + ObScale out_scale = expr.datum_meta_.scale_; + ObScale in_scale = 0; + ObPrecision out_prec = expr.datum_meta_.precision_; + ObPrecision in_prec = + ObAccuracy::MAX_ACCURACY2[lib::is_oracle_mode()][type] + .get_precision(); + const static int64_t DECINT64_MAX = get_scale_factor(MAX_PRECISION_DECIMAL_INT_64); + if (in_prec > MAX_PRECISION_DECIMAL_INT_64 && in_val < DECINT64_MAX) { + in_prec = MAX_PRECISION_DECIMAL_INT_64; + } + if (OB_FAIL(wide::from_integer(in_val, tmp_alloc, decint, int_bytes, in_prec))) { + LOG_WARN("from_integer failed", K(ret), K(in_val), K(row_idx), K(column_idx)); + } else if (ObDatumCast::need_scale_decimalint(in_scale, in_prec, out_scale, out_prec)) { + ObDecimalIntBuilder res_val; + if (OB_FAIL(ObDatumCast::common_scale_decimalint(decint, int_bytes, in_scale, out_scale, + out_prec, expr.extra_, res_val))) { + LOG_WARN("scale decimal int failed", K(ret), K(row_idx), K(column_idx)); + } else { + datums[row_idx].set_decimal_int(res_val.get_decimal_int(), res_val.get_int_bytes()); + } + } else { + datums[row_idx].set_decimal_int(decint, int_bytes); + } + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(type), K(column_idx)); + } + break; + } + case apsara::odps::sdk::ODPS_FLOAT: + { + if (ObFloatType == type) { + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < returned_row_cnt; ++row_idx) { + const float* v = records_[row_idx]->GetFloatValue(target_idx); + if (v == NULL) { + datums[row_idx].set_null(); + } else { + datums[row_idx].set_float(*v); + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(type), K(column_idx)); + } + break; + } + case apsara::odps::sdk::ODPS_DOUBLE: + { + if (ObDoubleType == type) { + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < returned_row_cnt; ++row_idx) { + const double* v = records_[row_idx]->GetDoubleValue(target_idx); + if (v == NULL) { + datums[row_idx].set_null(); + } else { + datums[row_idx].set_double(*v); + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(type), K(column_idx)); + } + break; + } + case apsara::odps::sdk::ODPS_DECIMAL: + { + if (ObDecimalIntType == type) { + ObEvalCtx::BatchInfoScopeGuard batch_info_guard(ctx); + batch_info_guard.set_batch_idx(0); + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < returned_row_cnt; ++row_idx) { + batch_info_guard.set_batch_idx(row_idx); + uint32_t len; + const char* v = records_[row_idx]->GetDecimalValue(target_idx, len); + if (v == NULL || len == 0) { + datums[row_idx].set_null(); + } else { + ObString in_str(len, v); + ObDecimalIntBuilder res_val; + if (OB_FAIL(ObOdpsDataTypeCastUtil::common_string_decimalint_wrap(expr, in_str, ctx.exec_ctx_.get_user_logging_ctx(), + res_val))) { + LOG_WARN("cast string to decimal int failed", K(ret), K(row_idx), K(column_idx)); + } else { + datums[row_idx].set_decimal_int(res_val.get_decimal_int(), res_val.get_int_bytes()); + } + } + } + } else if (ObNumberType == type) { + ObEvalCtx::BatchInfoScopeGuard batch_info_guard(ctx); + batch_info_guard.set_batch_idx(0); + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < returned_row_cnt; ++row_idx) { + batch_info_guard.set_batch_idx(row_idx); + uint32_t len; + const char* v = records_[row_idx]->GetDecimalValue(target_idx, len); + if (v == NULL || len == 0) { + datums[row_idx].set_null(); + } else { + ObString in_str(len, v); + number::ObNumber nmb; + ObNumStackOnceAlloc tmp_alloc; + if (OB_FAIL(ObOdpsDataTypeCastUtil::common_string_number_wrap(expr, in_str, tmp_alloc, nmb))) { + LOG_WARN("cast string to number failed", K(ret), K(row_idx), K(column_idx)); + } else { + datums[row_idx].set_number(nmb); + } + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(type), K(column_idx)); + } + break; + } + case apsara::odps::sdk::ODPS_CHAR: + { + if (ObCharType == type) { + ObEvalCtx::BatchInfoScopeGuard batch_info_guard(ctx); + batch_info_guard.set_batch_idx(0); + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < returned_row_cnt; ++row_idx) { + batch_info_guard.set_batch_idx(row_idx); + uint32_t len; + const char* v = records_[row_idx]->GetStringValue(target_idx, len, apsara::odps::sdk::ODPS_CHAR); + if (v == NULL || (0 == len && lib::is_oracle_mode())) { + datums[row_idx].set_null(); + } else { + ObObjType in_type = ObCharType; + ObObjType out_type = ObCharType; + ObCollationType in_cs_type = CS_TYPE_UTF8MB4_BIN; // odps's collation + ObCollationType out_cs_type = expr.datum_meta_.cs_type_; + ObString in_str(len, v); + bool has_set_res = false; + ObCharsetType out_charset = common::ObCharset::charset_type_by_coll(out_cs_type); + if (CHARSET_UTF8MB4 == out_charset || CHARSET_BINARY == out_charset) { + datums[row_idx].set_string(in_str); + } else if (OB_FAIL(oceanbase::sql::ObOdpsDataTypeCastUtil::common_string_string_wrap(expr, in_type, in_cs_type, out_type, + out_cs_type, in_str, ctx, datums[row_idx], has_set_res))) { + LOG_WARN("cast string to string failed", K(ret), K(row_idx), K(column_idx)); + } + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(type), K(column_idx)); + } + break; + } + case apsara::odps::sdk::ODPS_VARCHAR: + { + if (ObVarcharType == type) { + ObEvalCtx::BatchInfoScopeGuard batch_info_guard(ctx); + batch_info_guard.set_batch_idx(0); + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < returned_row_cnt; ++row_idx) { + batch_info_guard.set_batch_idx(row_idx); + uint32_t len; + const char* v = records_[row_idx]->GetStringValue(target_idx, len, apsara::odps::sdk::ODPS_VARCHAR); + if (v == NULL || (0 == len && lib::is_oracle_mode())) { + datums[row_idx].set_null(); + } else { + ObObjType in_type = ObVarcharType; + ObObjType out_type = ObVarcharType; + ObCollationType in_cs_type = CS_TYPE_UTF8MB4_BIN; // odps's collation + ObCollationType out_cs_type = expr.datum_meta_.cs_type_; + ObString in_str(len, v); + bool has_set_res = false; + ObCharsetType out_charset = common::ObCharset::charset_type_by_coll(out_cs_type); + if (CHARSET_UTF8MB4 == out_charset || CHARSET_BINARY == out_charset) { + datums[row_idx].set_string(in_str); + } else if (OB_FAIL(ObOdpsDataTypeCastUtil::common_string_string_wrap(expr, in_type, in_cs_type, out_type, + out_cs_type, in_str, ctx, datums[row_idx], has_set_res))) { + LOG_WARN("cast string to string failed", K(ret), K(row_idx), K(column_idx)); + } + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(type), K(column_idx)); + } + break; + } + case apsara::odps::sdk::ODPS_STRING: + case apsara::odps::sdk::ODPS_BINARY: + { + if (ObVarcharType == type) { + ObEvalCtx::BatchInfoScopeGuard batch_info_guard(ctx); + batch_info_guard.set_batch_idx(0); + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < returned_row_cnt; ++row_idx) { + batch_info_guard.set_batch_idx(row_idx); + uint32_t len; + const char* v = records_[row_idx]->GetStringValue(target_idx, len, odps_type); + if (v == NULL || (0 == len && lib::is_oracle_mode())) { + datums[row_idx].set_null(); + } else if (len > expr.max_length_) { + ret = OB_EXTERNAL_ODPS_COLUMN_TYPE_MISMATCH; + LOG_WARN("unexpected data length", K(ret), + K(len), + K(expr.max_length_), + K(column_list_.at(target_idx)), + K(type)); + print_type_map_user_info(column_list_.at(target_idx).type_info_, &expr); + } else { + ObObjType in_type = ObVarcharType; + ObObjType out_type = ObVarcharType; + ObCollationType in_cs_type = apsara::odps::sdk::ODPS_STRING == odps_type ? CS_TYPE_UTF8MB4_BIN : CS_TYPE_BINARY; + ObCollationType out_cs_type = expr.datum_meta_.cs_type_; + ObString in_str(len, v); + bool has_set_res = false; + ObCharsetType out_charset = common::ObCharset::charset_type_by_coll(out_cs_type); + if (CHARSET_UTF8MB4 == out_charset || CHARSET_BINARY == out_charset) { + datums[row_idx].set_string(in_str); + } else if (OB_FAIL(ObOdpsDataTypeCastUtil::common_string_string_wrap(expr, in_type, in_cs_type, out_type, + out_cs_type, in_str, ctx, datums[row_idx], has_set_res))) { + LOG_WARN("cast string to string failed", K(ret), K(row_idx), K(column_idx)); + } + } + } + } else if (ObTinyTextType == type || + ObTextType == type || + ObLongTextType == type || + ObMediumTextType == type) { + ObEvalCtx::BatchInfoScopeGuard batch_info_guard(ctx); + batch_info_guard.set_batch_idx(0); + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < returned_row_cnt; ++row_idx) { + batch_info_guard.set_batch_idx(row_idx); + uint32_t len; + const char* v = records_[row_idx]->GetStringValue(target_idx, len, odps_type); + if (v == NULL || (0 == len && lib::is_oracle_mode())) { + datums[row_idx].set_null(); + } else if (!text_type_length_is_valid_at_runtime(type, len)) { + ret = OB_EXTERNAL_ODPS_COLUMN_TYPE_MISMATCH; + LOG_WARN("unexpected data length", K(ret), + K(len), + K(expr.max_length_), + K(column_list_.at(target_idx)), + K(type)); + print_type_map_user_info(column_list_.at(target_idx).type_info_, &expr); + } else { + ObString in_str(len, v); + ObObjType in_type = ObVarcharType; + ObCollationType in_cs_type = apsara::odps::sdk::ODPS_STRING == odps_type ? CS_TYPE_UTF8MB4_BIN : CS_TYPE_BINARY; + if (OB_FAIL(ObOdpsDataTypeCastUtil::common_string_text_wrap(expr, in_str, ctx, NULL, datums[row_idx], in_type, in_cs_type))) { + LOG_WARN("cast string to text failed", K(ret), K(row_idx), K(column_idx)); + } + } + } + } else if (ObRawType == type) { + ObEvalCtx::BatchInfoScopeGuard batch_info_guard(ctx); + batch_info_guard.set_batch_idx(0); + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < returned_row_cnt; ++row_idx) { + batch_info_guard.set_batch_idx(row_idx); + uint32_t len; + const char* v = records_[row_idx]->GetStringValue(target_idx, len, odps_type); + if (v == NULL || (0 == len && lib::is_oracle_mode())) { + datums[row_idx].set_null(); + } else { + ObString in_str(len, v); + bool has_set_res = false; + if (OB_FAIL(ObDatumHexUtils::hextoraw_string(expr, in_str, ctx, datums[row_idx], has_set_res))) { + LOG_WARN("cast string to raw failed", K(ret), K(row_idx), K(column_idx)); + } + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(type), K(column_idx)); + } + break; + } + case apsara::odps::sdk::ODPS_TIMESTAMP: + { + if (ObTimestampType == type && !is_oracle_mode()) { + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < returned_row_cnt; ++row_idx) { + const apsara::odps::sdk::TimeStamp* v = records_[row_idx]->GetTimestampValue(target_idx); + if (v == NULL) { + datums[row_idx].set_null(); + } else { + int64_t datetime = v->GetSecond() * USECS_PER_SEC + (v->GetNano() + 500) / 1000; // suppose odps's timezone is same to oceanbase + datums[row_idx].set_datetime(datetime); + } + } + } else if (false && ObTimestampLTZType == type && is_oracle_mode()) { + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < returned_row_cnt; ++row_idx) { + const apsara::odps::sdk::TimeStamp* v = records_[row_idx]->GetTimestampValue(target_idx); + if (v == NULL) { + datums[row_idx].set_null(); + } else { + + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(type), K(column_idx)); + } + break; + } + case apsara::odps::sdk::ODPS_TIMESTAMP_NTZ: + { + if (ObDateTimeType == type && !is_oracle_mode()) { + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < returned_row_cnt; ++row_idx) { + const apsara::odps::sdk::TimeStamp* v = records_[row_idx]->GetTimestampNTZValue(target_idx); + if (v == NULL) { + datums[row_idx].set_null(); + } else { + int64_t datetime = v->GetSecond() * USECS_PER_SEC + (v->GetNano() + 500) / 1000; + datums[row_idx].set_datetime(datetime); + } + } + } else if (false && ObTimestampNanoType == type && is_oracle_mode()) { + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < returned_row_cnt; ++row_idx) { + const apsara::odps::sdk::TimeStamp* v = records_[row_idx]->GetTimestampNTZValue(target_idx); + if (v == NULL) { + datums[row_idx].set_null(); + } else { + + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(type), K(column_idx)); + } + break; + } + case apsara::odps::sdk::ODPS_DATE: + { + if (ObDateType == type && !is_oracle_mode()) { + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < returned_row_cnt; ++row_idx) { + const int64_t* v = records_[row_idx]->GetDateValue(target_idx); + if (v == NULL) { + datums[row_idx].set_null(); + } else { + int32_t date = *v; + datums[row_idx].set_date(date); + } + } + } else if (false && ObDateTimeType == type && is_oracle_mode()) { + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < returned_row_cnt; ++row_idx) { + const int64_t* v = records_[row_idx]->GetDateValue(target_idx); + if (v == NULL) { + datums[row_idx].set_null(); + } else { + + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(type), K(column_idx)); + } + break; + } + case apsara::odps::sdk::ODPS_DATETIME: + { + if (ObDateTimeType == type && !is_oracle_mode()) { + int32_t tmp_offset = 0; + int64_t res_offset = 0; + if (OB_FAIL(ctx.exec_ctx_.get_my_session()->get_timezone_info()->get_timezone_offset(0, tmp_offset))) { + LOG_WARN("failed to get timezone offset", K(ret)); + } else { + res_offset = SEC_TO_USEC(tmp_offset); + } + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < returned_row_cnt; ++row_idx) { + const int64_t* v = records_[row_idx]->GetDatetimeValue(target_idx); + if (v == NULL) { + datums[row_idx].set_null(); + } else { + int64_t datetime = *v * 1000 + res_offset; + datums[row_idx].set_datetime(datetime); + } + } + } else if (false && ObTimestampNanoType == type && is_oracle_mode()) { + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < returned_row_cnt; ++row_idx) { + const int64_t* v = records_[row_idx]->GetDatetimeValue(target_idx); + if (v == NULL) { + datums[row_idx].set_null(); + } else { + + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(type), K(column_idx)); + } + break; + } + case apsara::odps::sdk::ODPS_JSON: + { + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < returned_row_cnt; ++row_idx) { + uint32_t len; + const char* v = records_[row_idx]->GetJsonValue(target_idx, len); + if (v == NULL || (0 == len && lib::is_oracle_mode())) { + datums[row_idx].set_null(); + } else { + datums[row_idx].set_string(v, len); + } + } + break; + } + default: + { + ret = OB_NOT_SUPPORTED; + LOG_WARN("odps not support type", K(ret)); + } + } + } catch (apsara::odps::sdk::OdpsTunnelException& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("odps exception occured when calling OpenReader method", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (const std::exception& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("odps exception occured when calling OpenReader method", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (...) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("odps exception occured when calling OpenReader method", K(ret)); + } + } + } + } + ObEvalCtx::BatchInfoScopeGuard batch_info_guard(ctx); + batch_info_guard.set_batch_idx(0); + for (int i = 0; OB_SUCC(ret) && i < file_column_exprs.count(); i++) { + file_column_exprs.at(i)->set_evaluated_flag(ctx); + } + for (int i = 0; OB_SUCC(ret) && i < column_exprs_.count(); i++) { + ObExpr *column_expr = column_exprs_.at(i); + ObExpr *column_convert_expr = scan_param_->ext_column_convert_exprs_->at(i); + OZ (column_convert_expr->eval_batch(ctx, *bit_vector_cache_, returned_row_cnt)); + if (OB_SUCC(ret)) { + MEMCPY(column_expr->locate_batch_datums(ctx), + column_convert_expr->locate_batch_datums(ctx), sizeof(ObDatum) * returned_row_cnt); + column_expr->set_evaluated_flag(ctx); + } + } + if (OB_SUCC(ret)) { + count = returned_row_cnt; + } + } + } + return ret; +} + +int ObODPSTableRowIterator::get_next_row() +{ + int ret = OB_SUCCESS; + if (state_.count_ >= state_.step_ && OB_FAIL(next_task())) { + if (OB_ITER_END != ret) { + LOG_WARN("get next task failed", K(ret)); + } + } else { + if (OB_FAIL(inner_get_next_row())) { + LOG_WARN("failed to get next row inner", K(ret)); + } + } + while(OB_SUCC(ret) && get_next_task_) { // used to get next task which has data need to fetch + if (state_.count_ >= state_.step_ && OB_FAIL(next_task())) { + if (OB_ITER_END != ret) { + LOG_WARN("get next task failed", K(ret)); + } + } else { + if (OB_FAIL(inner_get_next_row())) { + LOG_WARN("failed to get next row inner", K(ret)); + } + } + } + return ret; +} + +int ObODPSTableRowIterator::inner_get_next_row() +{ + int ret = OB_SUCCESS; + ObMallocHookAttrGuard guard(mem_attr_); + ObEvalCtx &ctx = scan_param_->op_->get_eval_ctx(); + const ExprFixedArray &file_column_exprs = *(scan_param_->ext_file_column_exprs_); + get_next_task_ = false; + try { + if (!(state_.record_reader_handle_->Read(*record_))) { + if (INT64_MAX == state_.step_ || state_.count_ == state_.step_) { + get_next_task_ = true; // goto get next task + state_.step_ = state_.count_; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected end", K(total_count_), K(state_), K(ret)); + } + } else { + ++state_.count_; + ++total_count_; + } + } catch (apsara::odps::sdk::OdpsTunnelException& ex) { + if (OB_SUCC(ret)) { + std::string ex_msg = ex.what(); + if (std::string::npos != ex_msg.find("EOF")) { // EOF + LOG_TRACE("odps eof", K(ret), K(total_count_), K(state_), K(ex.what())); + if (INT64_MAX == state_.step_ || state_.count_ == state_.step_) { + get_next_task_ = true; // goto get next task + state_.step_ = state_.count_; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected end", K(total_count_), K(state_), K(ret)); + } + } else { + ret = OB_ODPS_ERROR; + LOG_WARN("odps exception occured when calling Read or Close method", K(ret), K(total_count_), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } + } catch (const std::exception& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("odps exception occured when calling Read or Close method", K(ret), K(total_count_), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (...) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("odps exception occured when calling Read or Close method", K(ret)); + } + } + if (OB_FAIL(ret)) { + // do nothing + } else if (get_next_task_) { + // do nothing + } else { + for (int64_t column_idx = 0; OB_SUCC(ret) && column_idx < target_column_id_list_.count(); ++column_idx) { + uint32_t target_idx = target_column_id_list_.at(column_idx); + ObExpr &expr = *file_column_exprs.at(column_idx); // do not check null ptr + ObDatum &datum = expr.locate_datum_for_write(ctx); + ObObjType type = expr.obj_meta_.get_type(); + if (expr.type_ == T_PSEUDO_PARTITION_LIST_COL) { + int64_t loc_idx = file_column_exprs.at(column_idx)->extra_ - 1; + if (OB_UNLIKELY(loc_idx < 0 || loc_idx >= state_.part_list_val_.get_count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted loc_idx", K(ret), K(loc_idx), K(state_.part_list_val_.get_count()), KP(&state_.part_list_val_)); + } else if (state_.part_list_val_.get_cell(loc_idx).is_null()) { + datum.set_null(); + } else { + CK (OB_NOT_NULL(datum.ptr_)); + OZ (datum.from_obj(state_.part_list_val_.get_cell(loc_idx))); + } + } else { + apsara::odps::sdk::ODPSColumnType odps_type = column_list_.at(target_idx).type_info_.mType; + try { + switch(odps_type) + { + case apsara::odps::sdk::ODPS_BOOLEAN: + { + if ((ObTinyIntType == type || + ObSmallIntType == type || + ObMediumIntType == type || + ObInt32Type == type || + ObIntType == type) && !is_oracle_mode()) { + const bool* v = record_->GetBoolValue(target_idx); + if (v == NULL) { + datum.set_null(); + } else { + datum.set_int(*v); + } + } else if (ObNumberType == type && is_oracle_mode()) { + const bool* v = record_->GetBoolValue(target_idx); + if (v == NULL) { + datum.set_null(); + } else { + int64_t in_val = *v; + ObNumStackOnceAlloc tmp_alloc; + number::ObNumber nmb; + OZ(ObOdpsDataTypeCastUtil::common_int_number_wrap(expr, in_val, tmp_alloc, nmb), in_val); + if (OB_FAIL(ret)) { + LOG_WARN("failed to cast int to number", K(ret), K(column_idx)); + } + } + } else if (ObDecimalIntType == type && is_oracle_mode()) { + const bool* v = record_->GetBoolValue(target_idx); + if (v == NULL) { + datum.set_null(); + } else { + int64_t in_val = *v; + ObDecimalInt *decint = nullptr; + int32_t int_bytes = 0; + ObDecimalIntBuilder tmp_alloc; + ObScale out_scale = expr.datum_meta_.scale_; + ObScale in_scale = 0; + ObPrecision out_prec = expr.datum_meta_.precision_; + ObPrecision in_prec = + ObAccuracy::MAX_ACCURACY2[lib::is_oracle_mode()][type] + .get_precision(); + const static int64_t DECINT64_MAX = get_scale_factor(MAX_PRECISION_DECIMAL_INT_64); + if (in_prec > MAX_PRECISION_DECIMAL_INT_64 && in_val < DECINT64_MAX) { + in_prec = MAX_PRECISION_DECIMAL_INT_64; + } + if (OB_FAIL(wide::from_integer(in_val, tmp_alloc, decint, int_bytes, in_prec))) { + LOG_WARN("from_integer failed", K(ret), K(in_val), K(column_idx)); + } else if (ObDatumCast::need_scale_decimalint(in_scale, in_prec, out_scale, out_prec)) { + ObDecimalIntBuilder res_val; + if (OB_FAIL(ObDatumCast::common_scale_decimalint(decint, int_bytes, in_scale, out_scale, + out_prec, expr.extra_, res_val))) { + LOG_WARN("scale decimal int failed", K(ret), K(column_idx)); + } else { + datum.set_decimal_int(res_val.get_decimal_int(), res_val.get_int_bytes()); + } + } else { + datum.set_decimal_int(decint, int_bytes); + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(type), K(column_idx)); + } + break; + } + case apsara::odps::sdk::ODPS_TINYINT: + case apsara::odps::sdk::ODPS_SMALLINT: + case apsara::odps::sdk::ODPS_INTEGER: + case apsara::odps::sdk::ODPS_BIGINT: + { + if ((ObTinyIntType == type || + ObSmallIntType == type || + ObMediumIntType == type || + ObInt32Type == type || + ObIntType == type) && !is_oracle_mode()) { + const int64_t* v = record_->GetIntValue(target_idx, odps_type); + if (v == NULL) { + datum.set_null(); + } else { + datum.set_int(*v); + } + } else if (ObNumberType == type && is_oracle_mode()) { + const int64_t* v = record_->GetIntValue(target_idx, odps_type); + if (v == NULL) { + datum.set_null(); + } else { + int64_t in_val = *v; + ObNumStackOnceAlloc tmp_alloc; + number::ObNumber nmb; + OZ(ObOdpsDataTypeCastUtil::common_int_number_wrap(expr, in_val, tmp_alloc, nmb), in_val); + if (OB_FAIL(ret)) { + LOG_WARN("failed to cast int to number", K(ret), K(column_idx)); + } + } + } else if (ObDecimalIntType == type && is_oracle_mode()) { + const int64_t* v = record_->GetIntValue(target_idx, odps_type); + if (v == NULL) { + datum.set_null(); + } else { + int64_t in_val = *v; + ObDecimalInt *decint = nullptr; + int32_t int_bytes = 0; + ObDecimalIntBuilder tmp_alloc; + ObScale out_scale = expr.datum_meta_.scale_; + ObScale in_scale = 0; + ObPrecision out_prec = expr.datum_meta_.precision_; + ObPrecision in_prec = + ObAccuracy::MAX_ACCURACY2[lib::is_oracle_mode()][type] + .get_precision(); + const static int64_t DECINT64_MAX = get_scale_factor(MAX_PRECISION_DECIMAL_INT_64); + if (in_prec > MAX_PRECISION_DECIMAL_INT_64 && in_val < DECINT64_MAX) { + in_prec = MAX_PRECISION_DECIMAL_INT_64; + } + if (OB_FAIL(wide::from_integer(in_val, tmp_alloc, decint, int_bytes, in_prec))) { + LOG_WARN("from_integer failed", K(ret), K(in_val), K(column_idx)); + } else if (ObDatumCast::need_scale_decimalint(in_scale, in_prec, out_scale, out_prec)) { + ObDecimalIntBuilder res_val; + if (OB_FAIL(ObDatumCast::common_scale_decimalint(decint, int_bytes, in_scale, out_scale, + out_prec, expr.extra_, res_val))) { + LOG_WARN("scale decimal int failed", K(ret), K(column_idx)); + } else { + datum.set_decimal_int(res_val.get_decimal_int(), res_val.get_int_bytes()); + } + } else { + datum.set_decimal_int(decint, int_bytes); + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(type), K(column_idx)); + } + break; + } + case apsara::odps::sdk::ODPS_FLOAT: + { + if (ObFloatType == type) { + const float* v = record_->GetFloatValue(target_idx); + if (v == NULL) { + datum.set_null(); + } else { + datum.set_float(*v); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(type), K(column_idx)); + } + break; + } + case apsara::odps::sdk::ODPS_DOUBLE: + { + if (ObDoubleType == type) { + const double* v = record_->GetDoubleValue(target_idx); + if (v == NULL) { + datum.set_null(); + } else { + datum.set_double(*v); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(type), K(column_idx)); + } + break; + } + case apsara::odps::sdk::ODPS_DECIMAL: + { + if (ObDecimalIntType == type) { + uint32_t len; + const char* v = record_->GetDecimalValue(target_idx, len); + if (v == NULL || len == 0) { + datum.set_null(); + } else { + ObString in_str(len, v); + ObDecimalIntBuilder res_val; + if (OB_FAIL(ObOdpsDataTypeCastUtil::common_string_decimalint_wrap(expr, in_str, ctx.exec_ctx_.get_user_logging_ctx(), res_val))) { + LOG_WARN("cast string to decimal int failed", K(ret), K(column_idx)); + } else { + datum.set_decimal_int(res_val.get_decimal_int(), res_val.get_int_bytes()); + } + } + } else if (ObNumberType == type) { + uint32_t len; + const char* v = record_->GetDecimalValue(target_idx, len); + if (v == NULL || len == 0) { + datum.set_null(); + } else { + ObString in_str(len, v); + number::ObNumber nmb; + ObNumStackOnceAlloc tmp_alloc; + if (OB_FAIL(ObOdpsDataTypeCastUtil::common_string_number_wrap(expr, in_str, tmp_alloc, nmb))) { + LOG_WARN("cast string to number failed", K(ret), K(column_idx)); + } else { + datum.set_number(nmb); + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(type), K(column_idx)); + } + break; + } + case apsara::odps::sdk::ODPS_CHAR: + { + if (ObCharType == type) { + uint32_t len; + const char* v = record_->GetStringValue(target_idx, len, apsara::odps::sdk::ODPS_CHAR); + if (v == NULL || (0 == len && lib::is_oracle_mode())) { + datum.set_null(); + } else { + ObObjType in_type = ObCharType; + ObObjType out_type = ObCharType; + ObCollationType in_cs_type = CS_TYPE_UTF8MB4_BIN; // odps's collation + ObCollationType out_cs_type = expr.datum_meta_.cs_type_; + ObString in_str(len, v); + bool has_set_res = false; + ObCharsetType out_charset = common::ObCharset::charset_type_by_coll(out_cs_type); + if (CHARSET_UTF8MB4 == out_charset || CHARSET_BINARY == out_charset) { + datum.set_string(in_str); + } else if (OB_FAIL(oceanbase::sql::ObOdpsDataTypeCastUtil::common_string_string_wrap(expr, in_type, in_cs_type, out_type, + out_cs_type, in_str, ctx, datum, has_set_res))) { + LOG_WARN("cast string to string failed", K(ret), K(column_idx)); + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(type), K(column_idx)); + } + break; + } + case apsara::odps::sdk::ODPS_VARCHAR: + { + if (ObVarcharType == type) { + uint32_t len; + const char* v = record_->GetStringValue(target_idx, len, apsara::odps::sdk::ODPS_VARCHAR); + if (v == NULL || (0 == len && lib::is_oracle_mode())) { + datum.set_null(); + } else { + ObObjType in_type = ObVarcharType; + ObObjType out_type = ObVarcharType; + ObCollationType in_cs_type = CS_TYPE_UTF8MB4_BIN; // odps's collation + ObCollationType out_cs_type = expr.datum_meta_.cs_type_; + ObString in_str(len, v); + bool has_set_res = false; + ObCharsetType out_charset = common::ObCharset::charset_type_by_coll(out_cs_type); + if (CHARSET_UTF8MB4 == out_charset || CHARSET_BINARY == out_charset) { + datum.set_string(in_str); + } else if (OB_FAIL(ObOdpsDataTypeCastUtil::common_string_string_wrap(expr, in_type, in_cs_type, out_type, + out_cs_type, in_str, ctx, datum, has_set_res))) { + LOG_WARN("cast string to string failed", K(ret), K(column_idx)); + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(type), K(column_idx)); + } + break; + } + case apsara::odps::sdk::ODPS_STRING: + case apsara::odps::sdk::ODPS_BINARY: + { + if (ObVarcharType == type) { + uint32_t len; + const char* v = record_->GetStringValue(target_idx, len, odps_type); + if (v == NULL || (0 == len && lib::is_oracle_mode())) { + datum.set_null(); + } else if (len > expr.max_length_) { + ret = OB_EXTERNAL_ODPS_COLUMN_TYPE_MISMATCH; + LOG_WARN("unexpected data length", K(ret), + K(len), + K(expr.max_length_), + K(column_list_.at(target_idx)), + K(type)); + print_type_map_user_info(column_list_.at(target_idx).type_info_, &expr); + } else { + ObObjType in_type = ObVarcharType; + ObObjType out_type = ObVarcharType; + ObCollationType in_cs_type = apsara::odps::sdk::ODPS_STRING == odps_type ? CS_TYPE_UTF8MB4_BIN : CS_TYPE_BINARY; + ObCollationType out_cs_type = expr.datum_meta_.cs_type_; + ObString in_str(len, v); + bool has_set_res = false; + ObCharsetType out_charset = common::ObCharset::charset_type_by_coll(out_cs_type); + if (CHARSET_UTF8MB4 == out_charset || CHARSET_BINARY == out_charset) { + datum.set_string(in_str); + } else if (OB_FAIL(ObOdpsDataTypeCastUtil::common_string_string_wrap(expr, in_type, in_cs_type, out_type, + out_cs_type, in_str, ctx, datum, has_set_res))) { + LOG_WARN("cast string to string failed", K(ret), K(column_idx)); + } + } + } else if (ObTinyTextType == type || + ObTextType == type || + ObLongTextType == type || + ObMediumTextType == type) { + uint32_t len; + const char* v = record_->GetStringValue(target_idx, len, odps_type); + if (v == NULL || (0 == len && lib::is_oracle_mode())) { + datum.set_null(); + } else if (!text_type_length_is_valid_at_runtime(type, len)) { + ret = OB_EXTERNAL_ODPS_COLUMN_TYPE_MISMATCH; + LOG_WARN("unexpected data length", K(ret), + K(len), + K(expr.max_length_), + K(column_list_.at(target_idx)), + K(type)); + print_type_map_user_info(column_list_.at(target_idx).type_info_, &expr); + } else { + ObString in_str(len, v); + ObObjType in_type = ObVarcharType; // lcqlog todo ObHexStringType ? + ObCollationType in_cs_type = apsara::odps::sdk::ODPS_STRING == odps_type ? CS_TYPE_UTF8MB4_BIN : CS_TYPE_BINARY; + if (OB_FAIL(ObOdpsDataTypeCastUtil::common_string_text_wrap(expr, in_str, ctx, NULL, datum, in_type, in_cs_type))) { + LOG_WARN("cast string to text failed", K(ret), K(column_idx)); + } + } + } else if (ObRawType == type) { + uint32_t len; + const char* v = record_->GetStringValue(target_idx, len, odps_type); + if (v == NULL || (0 == len && lib::is_oracle_mode())) { + datum.set_null(); + } else { + ObString in_str(len, v); + bool has_set_res = false; + if (OB_FAIL(ObDatumHexUtils::hextoraw_string(expr, in_str, ctx, datum, has_set_res))) { + LOG_WARN("cast string to raw failed", K(ret), K(column_idx)); + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(type), K(column_idx)); + } + break; + } + case apsara::odps::sdk::ODPS_TIMESTAMP: + { + if (ObTimestampType == type && !is_oracle_mode()) { + const apsara::odps::sdk::TimeStamp* v = record_->GetTimestampValue(target_idx); + if (v == NULL) { + datum.set_null(); + } else { + int64_t datetime = v->GetSecond() * USECS_PER_SEC + (v->GetNano() + 500) / 1000; // suppose odps's timezone is same to oceanbase + datum.set_datetime(datetime); + } + } else if (false && ObTimestampLTZType == type && is_oracle_mode()) { + const apsara::odps::sdk::TimeStamp* v = record_->GetTimestampValue(target_idx); + if (v == NULL) { + datum.set_null(); + } else { + + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(type), K(column_idx)); + } + break; + } + case apsara::odps::sdk::ODPS_TIMESTAMP_NTZ: + { + if (ObDateTimeType == type && !is_oracle_mode()) { + const apsara::odps::sdk::TimeStamp* v = record_->GetTimestampNTZValue(target_idx); + if (v == NULL) { + datum.set_null(); + } else { + int64_t datetime = v->GetSecond() * USECS_PER_SEC + (v->GetNano() + 500) / 1000; + datum.set_datetime(datetime); + } + } else if (false && ObTimestampNanoType == type && is_oracle_mode()) { + const apsara::odps::sdk::TimeStamp* v = record_->GetTimestampNTZValue(target_idx); + if (v == NULL) { + datum.set_null(); + } else { + + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(type), K(column_idx)); + } + break; + } + case apsara::odps::sdk::ODPS_DATE: + { + if (ObDateType == type && !is_oracle_mode()) { + const int64_t* v = record_->GetDateValue(target_idx); + if (v == NULL) { + datum.set_null(); + } else { + int32_t date = *v; + datum.set_date(date); + } + } else if (false && ObDateTimeType == type && is_oracle_mode()) { + const int64_t* v = record_->GetDateValue(target_idx); + if (v == NULL) { + datum.set_null(); + } else { + + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(type), K(column_idx)); + } + break; + } + case apsara::odps::sdk::ODPS_DATETIME: + { + if (ObDateTimeType == type && !is_oracle_mode()) { + const int64_t* v = record_->GetDatetimeValue(target_idx); + int32_t tmp_offset = 0; + int64_t res_offset = 0; + if (v == NULL) { + datum.set_null(); + } else if (OB_FAIL(ctx.exec_ctx_.get_my_session()->get_timezone_info()->get_timezone_offset(0, tmp_offset))) { + LOG_WARN("failed to get timezone offset", K(ret)); + } else { + res_offset = SEC_TO_USEC(tmp_offset); + int64_t datetime = *v * 1000 + res_offset; + datum.set_datetime(datetime); + } + } else if (false && ObTimestampNanoType == type && is_oracle_mode()) { + const int64_t* v = record_->GetDatetimeValue(target_idx); + if (v == NULL) { + datum.set_null(); + } else { + + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(type), K(column_idx)); + } + break; + } + case apsara::odps::sdk::ODPS_JSON: + { + uint32_t len; + const char* v = record_->GetJsonValue(target_idx, len); + if (v == NULL || (0 == len && lib::is_oracle_mode())) { + datum.set_null(); + } else { + datum.set_string(v, len); + } + break; + } + default: + { + ret = OB_NOT_SUPPORTED; + LOG_WARN("odps not support type", K(ret)); + } + } + } catch (apsara::odps::sdk::OdpsTunnelException& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("odps exception occured when calling OpenReader method", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (const std::exception& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("odps exception occured when calling OpenReader method", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (...) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("odps exception occured when calling OpenReader method", K(ret)); + } + } + } + } + for (int i = 0; OB_SUCC(ret) && i < file_column_exprs.count(); i++) { + file_column_exprs.at(i)->set_evaluated_flag(ctx); + } + for (int i = 0; OB_SUCC(ret) && i < column_exprs_.count(); i++) { + ObExpr *column_expr = column_exprs_.at(i); + ObExpr *column_convert_expr = scan_param_->ext_column_convert_exprs_->at(i); + ObDatum *convert_datum = NULL; + OZ (column_convert_expr->eval(ctx, convert_datum)); + if (OB_SUCC(ret)) { + column_expr->locate_datum_for_write(ctx) = *convert_datum; + column_expr->set_evaluated_flag(ctx); + } + } + } + return ret; +} + +int ObOdpsPartitionDownloaderMgr::init_downloader(common::ObArray &external_table_files, const ObString &properties) +{ + int ret = OB_SUCCESS; + int64_t partition_cnt = external_table_files.count(); + sql::ObExternalFileFormat external_odps_format; + if (inited_) { + // do nothing + } else if (0 == partition_cnt || properties.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected args", K(partition_cnt), K(properties), K(ret)); + } else if (!odps_mgr_map_.created() && + OB_FAIL(odps_mgr_map_.create(partition_cnt, + "OdpsTable", + "OdpsTableReader"))) { + LOG_WARN("create hash table failed", K(ret), K(partition_cnt)); + } else if (OB_FAIL(external_odps_format.load_from_string(properties, arena_alloc_))) { + LOG_WARN("failed to init external_odps_format", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < external_table_files.count(); ++i) { + share::ObExternalFileInfo &odps_partition = external_table_files.at(i); + OdpsPartitionDownloader *downloader = NULL; + if (0 != odps_partition.file_id_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected file id", K(ret), K(i), K(odps_partition.file_id_), K(odps_partition.part_id_)); + } else if (OB_ISNULL(downloader = static_cast( + arena_alloc_.alloc(sizeof(OdpsPartitionDownloader))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate memory", K(ret), K(sizeof(OdpsPartitionDownloader))); + } else if (FALSE_IT(new(downloader)OdpsPartitionDownloader())) { + } else if (OB_FAIL(downloader->odps_driver_.init_tunnel(external_odps_format.odps_format_))) { + LOG_WARN("failed to init tunnel", K(ret), K(odps_partition.part_id_), K(properties)); + //odps_partition.file_url_ is odps partition specification, which is a literal string value + } else if (OB_FAIL(downloader->odps_driver_.create_downloader(odps_partition.file_url_, + downloader->odps_partition_downloader_))) { + LOG_WARN("failed create odps partition downloader", K(ret), K(i), K(odps_partition.part_id_), K(odps_partition.file_url_)); + } else if (OB_FAIL(odps_mgr_map_.set_refactored(odps_partition.part_id_, + reinterpret_cast(downloader)))) { + downloader->reset(); + LOG_WARN("failed to set refactored", K(ret), K(odps_partition.part_id_)); + } + } + if (OB_SUCC(ret)) { + inited_ = true; + LOG_TRACE("succ to init odps downloader", K(ret)); + } + } + return ret; +} + +int ObOdpsPartitionDownloaderMgr::create_upload_session(const sql::ObODPSGeneralFormat &odps_format, + const ObString &external_partition, + bool is_overwrite, + apsara::odps::sdk::IUploadPtr &upload) +{ + int ret = OB_SUCCESS; + apsara::odps::sdk::Configuration conf; + apsara::odps::sdk::OdpsTunnel tunnel; + const char* account_type = ""; + try { + conf.SetEndpoint(std::string(odps_format.endpoint_.ptr(), odps_format.endpoint_.length())); + conf.SetUserAgent("OB_ACCESS_ODPS"); + conf.SetTunnelQuotaName(std::string(odps_format.quota_.ptr(), odps_format.quota_.length())); + if (0 == odps_format.compression_code_.case_compare("zlib")) { + conf.SetCompressOption(apsara::odps::sdk::CompressOption::ZLIB_COMPRESS); + } else if (0 == odps_format.compression_code_.case_compare("zstd")) { + conf.SetCompressOption(apsara::odps::sdk::CompressOption::ZSTD_COMPRESS); + } else if (0 == odps_format.compression_code_.case_compare("lz4")) { + conf.SetCompressOption(apsara::odps::sdk::CompressOption::LZ4_COMPRESS); + } else if (0 == odps_format.compression_code_.case_compare("odps_lz4")) { + conf.SetCompressOption(apsara::odps::sdk::CompressOption::ODPS_LZ4_COMPRESS); + } else { + conf.SetCompressOption(apsara::odps::sdk::CompressOption::NO_COMPRESS); + } + if (0 == odps_format.access_type_.case_compare("aliyun") || + odps_format.access_type_.empty()) { + account_type = apsara::odps::sdk::ACCOUNT_ALIYUN; + } else if (0 == odps_format.access_type_.case_compare("sts")) { + account_type = apsara::odps::sdk::ACCOUNT_STS; + } else if (0 == odps_format.access_type_.case_compare("app")) { + account_type = apsara::odps::sdk::ACCOUNT_APPLICATION; + } else if (0 == odps_format.access_type_.case_compare("token") + || 0 == odps_format.access_type_.case_compare("domain") + || 0 == odps_format.access_type_.case_compare("taobao")) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("unsupported access type", K(ret), K(odps_format.access_type_)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "this ODPS access type"); + } else { + ret = OB_NOT_SUPPORTED; + LOG_WARN("unsupported access type", K(ret), K(odps_format.access_type_)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "this ODPS access type"); + } + if (OB_SUCC(ret)) { + apsara::odps::sdk::Account account(std::string(account_type), + std::string(odps_format.access_id_.ptr(), odps_format.access_id_.length()), + std::string(odps_format.access_key_.ptr(), odps_format.access_key_.length())); + conf.SetAccount(account); + tunnel.Init(conf); + if (OB_UNLIKELY(!(upload = tunnel.CreateUpload( + std::string(odps_format.project_.ptr(), odps_format.project_.length()), + std::string(odps_format.table_.ptr(), odps_format.table_.length()), + std::string(external_partition.ptr(), external_partition.length()), + "", + is_overwrite, + std::string(odps_format.schema_.ptr(), odps_format.schema_.length()))))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted null ptr", K(ret)); + } + } + } catch (apsara::odps::sdk::OdpsException& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when create odps upload session", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (const std::exception& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when create odps upload session", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (...) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when create odps upload session", K(ret)); + } + } + return ret; +} + +int ObOdpsPartitionDownloaderMgr::init_uploader(const ObString &properties, + const ObString &external_partition, + bool is_overwrite, + int64_t parallel) +{ + int ret = OB_SUCCESS; + sql::ObExternalFileFormat external_properties; + apsara::odps::sdk::IUploadPtr upload; + apsara::odps::sdk::IRecordWriterPtr record_writer; + void *ptr; + OdpsUploader *uploader; + if (inited_) { + // do nothing + } else if (properties.empty()) { + // do nothing + } else if (OB_FAIL(external_properties.load_from_string(properties, arena_alloc_))) { + LOG_WARN("failed to init external_odps_format", K(ret)); + } else if (sql::ObExternalFileFormat::ODPS_FORMAT != external_properties.format_type_) { + // do nothing + } else if (!odps_mgr_map_.created() && + OB_FAIL(odps_mgr_map_.create(parallel, "IntoOdps"))) { + LOG_WARN("create hash table failed", K(ret), K(parallel)); + } else if (OB_FAIL(external_properties.odps_format_.decrypt())) { + LOG_WARN("failed to decrypt odps format", K(ret)); + } else { + ObMallocHookAttrGuard guard(ObMemAttr(MTL_ID(), "IntoOdps")); + try { + if (OB_FAIL(create_upload_session(external_properties.odps_format_, + external_partition, + is_overwrite, + upload))) { + LOG_WARN("failed to create upload session", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < parallel; i++) { + if (OB_UNLIKELY(!(record_writer = upload->OpenWriter(i, true)))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted null ptr", K(ret)); + } else if (OB_ISNULL(ptr = arena_alloc_.alloc(sizeof(OdpsUploader)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate uploader", K(ret), K(sizeof(OdpsUploader))); + } else { + uploader = new(ptr) OdpsUploader(); + uploader->record_writer_ = record_writer; + uploader->upload_ = upload; + } + if (OB_SUCC(ret) + && OB_FAIL(odps_mgr_map_.set_refactored(i, reinterpret_cast(uploader)))) { + LOG_WARN("failed to set refactored", K(ret), K(i)); + } + } + } + } catch (apsara::odps::sdk::OdpsException& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when init odps tunnel", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (const std::exception& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when init odps tunnel", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (...) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when init odps tunnel", K(ret)); + } + } + } + if (OB_SUCC(ret)) { + inited_ = true; + is_download_ = false; + ATOMIC_STORE(&ref_, parallel); + LOG_TRACE("succ to init odps uploader", K(ret), K(ref_)); + } + return ret; +} + +int ObOdpsPartitionDownloaderMgr::get_odps_downloader(int64_t part_id, apsara::odps::sdk::IDownloadPtr &downloader) +{ + int ret = OB_SUCCESS; + int64_t value = 0; + OdpsPartitionDownloader *odps_downloader = NULL; + if (OB_FAIL(odps_mgr_map_.get_refactored(part_id, value))) { + LOG_WARN("failed to get downloader", K(ret), K(part_id)); + } else if (OB_ISNULL(odps_downloader = reinterpret_cast(value))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected value", K(ret), K(part_id), K(value)); + } else if (OB_ISNULL(odps_downloader->odps_partition_downloader_.get())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected value", K(ret), K(part_id), K(value)); + } else { + downloader = odps_downloader->odps_partition_downloader_; + } + return ret; +} + +int ObOdpsPartitionDownloaderMgr::get_odps_uploader(int64_t task_id, + apsara::odps::sdk::IUploadPtr &upload, + apsara::odps::sdk::IRecordWriterPtr &record_writer) +{ + int ret = OB_SUCCESS; + int64_t value = 0; + OdpsUploader *uploader = NULL; + if (OB_FAIL(odps_mgr_map_.get_refactored(task_id, value))) { + LOG_WARN("failed to get uploader", K(ret), K(task_id)); + } else if (OB_ISNULL(uploader = reinterpret_cast(value))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected value", K(ret), K(value)); + } else if (OB_UNLIKELY(!uploader->upload_ || !uploader->record_writer_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted null ptr", K(ret)); + } else { + upload = uploader->upload_; + record_writer = uploader->record_writer_; + } + return ret; +} + +int ObOdpsPartitionDownloaderMgr::commit_upload() +{ + int ret = OB_SUCCESS; + std::vector blocks; + uint32_t block_id = 0; + uint32_t task_count = static_cast(odps_mgr_map_.size()); + OdpsUploader *uploader = NULL; + LOG_TRACE("debug select into commit upload begin"); + try { + for (common::hash::ObHashMap::iterator iter = odps_mgr_map_.begin(); + OB_SUCC(ret) && iter != odps_mgr_map_.end(); iter++) { + if (OB_ISNULL(uploader = reinterpret_cast(iter->second)) + || OB_UNLIKELY(!uploader->record_writer_ || !uploader->upload_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted null ptr", K(ret)); + } else { + uploader->record_writer_->Close(); + blocks.push_back(block_id); + block_id++; + // 所有线程都成功才commit + if (block_id == task_count && true == ATOMIC_LOAD(&need_commit_)) { + uploader->upload_->Commit(blocks); + } + uploader->~OdpsUploader(); + } + } + } catch (apsara::odps::sdk::OdpsException& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when commit", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (const std::exception& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when commit", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (...) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("caught exception when commit", K(ret)); + } + } + LOG_TRACE("debug select into commit upload end"); + return ret; +} + +int ObOdpsPartitionDownloaderMgr::reset() +{ + int ret = OB_SUCCESS; + DeleteDownloaderFunc delete_func; + if (!inited_) { + // do nothing + } else if (is_download_ && OB_FAIL(odps_mgr_map_.foreach_refactored(delete_func))) { + LOG_WARN("failed to do foreach", K(ret)); + } else { + odps_mgr_map_.destroy(); + } + inited_ = false; + is_download_ = true; + return ret; +} + +int ObOdpsPartitionDownloaderMgr::DeleteDownloaderFunc::operator()(common::hash::HashMapPair &kv) +{ + int ret = OB_SUCCESS; + int64_t part_id = kv.first; + int64_t value = kv.second; + OdpsPartitionDownloader *downloader = reinterpret_cast(value); + if (OB_ISNULL(downloader)) { + // ignore ret + LOG_WARN("unexpected null ptr", K(ret), K(value), K(part_id));// ret is still OB_SUCCESS + } else { + downloader->reset(); + } + return ret; +} + +int ObOdpsPartitionDownloaderMgr::OdpsPartitionDownloader::reset() +{ + int ret = OB_SUCCESS; + try { + if (OB_ISNULL(odps_partition_downloader_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null ptr", K(ret)); + } else { + odps_partition_downloader_->Complete(); + } + } catch (const apsara::odps::sdk::OdpsTunnelException& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("odps exception occured when calling Complete method", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (const std::exception& ex) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("odps exception occured when calling Complete method", K(ret), K(ex.what())); + LOG_USER_ERROR(OB_ODPS_ERROR, ex.what()); + } + } catch (...) { + if (OB_SUCC(ret)) { + ret = OB_ODPS_ERROR; + LOG_WARN("odps exception occured when calling Complete method", K(ret)); + } + } + return ret; +} + +} // sql +} // oceanbase +#endif \ No newline at end of file diff --git a/src/sql/engine/table/ob_odps_table_row_iter.h b/src/sql/engine/table/ob_odps_table_row_iter.h new file mode 100644 index 000000000..18628fd56 --- /dev/null +++ b/src/sql/engine/table/ob_odps_table_row_iter.h @@ -0,0 +1,286 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef __SQL_OB_ODPS_TABLE_ROW_ITER_H__ +#define __SQL_OB_ODPS_TABLE_ROW_ITER_H__ +#ifdef OB_BUILD_CPP_ODPS +#include +#include +#include "sql/engine/table/ob_external_table_access_service.h" +#include "sql/engine/cmd/ob_load_data_parser.h" +#include "lib/container/ob_se_array.h" +#include "lib/ob_errno.h" +#include "lib/hash/ob_hashmap.h" + +namespace oceanbase { +namespace sql { + +class ObODPSTableRowIterator : public ObExternalTableRowIterator { +public: + static const int64_t READER_HASH_MAP_BUCKET_NUM = 1 << 7; + static const int64_t ODPS_BLOCK_DOWNLOAD_SIZE = 1 << 18; +public: + struct StateValues { + StateValues() : + task_idx_(-1), + part_id_(0), + start_(0), + step_(0), + count_(0), + is_from_gi_pump_(false), + download_handle_(NULL), + record_reader_handle_(NULL) {} + int reuse(); + TO_STRING_KV(K(task_idx_), + K(part_id_), + K(start_), + K(step_), + K(count_), + K(is_from_gi_pump_), + K(ObString(part_spec_.c_str())), + K(ObString(download_id_.c_str()))); + int64_t task_idx_; + int64_t part_id_; + int64_t start_; + int64_t step_; + int64_t count_; + bool is_from_gi_pump_; + apsara::odps::sdk::IDownloadPtr download_handle_; + apsara::odps::sdk::IRecordReaderPtr record_reader_handle_; + std::string part_spec_; + std::string download_id_; + ObNewRow part_list_val_; + }; + struct OdpsPartition { + OdpsPartition() : + name_(""), + download_handle_(NULL), + download_id_(""), + record_count_(-1) + { + } + OdpsPartition(const std::string &name) : + name_(name), + download_handle_(NULL), + download_id_(""), + record_count_(-1) + { + } + OdpsPartition(const std::string &name, + apsara::odps::sdk::IDownloadPtr download_handle, + const std::string download_id, + int64_t &record_count) : + name_(name), + download_handle_(download_handle), + download_id_(download_id), + record_count_(record_count) + { + } + ~OdpsPartition() { + reset(); + } + int reset(); + TO_STRING_KV(K(ObString(name_.c_str())), K(record_count_)); + std::string name_; + apsara::odps::sdk::IDownloadPtr download_handle_; + std::string download_id_; + int64_t record_count_; + }; + + struct OdpsColumn { + OdpsColumn() {} + OdpsColumn(std::string name, apsara::odps::sdk::ODPSColumnTypeInfo type_info) : + name_(name), + type_info_(type_info) + { + } + std::string name_; + apsara::odps::sdk::ODPSColumnTypeInfo type_info_; + TO_STRING_KV(K(ObString(name_.c_str())), K(type_info_.mType), K(type_info_.mPrecision), K(type_info_.mScale), K(type_info_.mSpecifiedLength)); + }; +public: + ObODPSTableRowIterator() : + odps_format_(), + account_(), + conf_(), + tunnel_(), + odps_(NULL), + table_handle_(NULL), + state_(), + is_part_table_(false), + total_count_(0), + bit_vector_cache_(NULL), + record_(NULL), + records_(NULL), + batch_size_(-1), + get_next_task_(false) + { + mem_attr_ = ObMemAttr(MTL_ID(), "odpsrowiter"); + malloc_alloc_.set_attr(mem_attr_); + } + virtual ~ObODPSTableRowIterator() { + if (NULL != bit_vector_cache_) { + malloc_alloc_.free(bit_vector_cache_); + } + for (int64_t i = 0; i < batch_size_; ++i) { + records_[i].reset(); + } + if (NULL != records_) { + malloc_alloc_.free(records_); + } + record_.reset(); + records_ = NULL; + batch_size_ = -1; + get_next_task_ = false; + reset(); + } + virtual int init(const storage::ObTableScanParam *scan_param) override; + virtual int get_next_row() override; + virtual int get_next_rows(int64_t &count, int64_t capacity) override; + virtual int get_next_row(ObNewRow *&row) override { + UNUSED(row); + return common::OB_ERR_UNEXPECTED; + } + virtual void reset() override; + int init_tunnel(const sql::ObODPSGeneralFormat &odps_format); + int create_downloader(ObString &part_spec, apsara::odps::sdk::IDownloadPtr &downloader); + int pull_partition_info(); + inline ObIArray& get_partition_info() { return partition_list_; } + inline bool is_part_table() { return is_part_table_; } + static int check_type_static(const apsara::odps::sdk::ODPSColumnType odps_type, + const int32_t odps_type_length, + const int32_t odps_type_precision, + const int32_t odps_type_scale, + const ObObjType ob_type, + const int32_t ob_type_length, + const int32_t ob_type_precision, + const int32_t ob_type_scale); +private: + int inner_get_next_row(); + int prepare_expr(); + int pull_column(); + int next_task(); + int print_type_map_user_info(apsara::odps::sdk::ODPSColumnTypeInfo odps_type_info, + const ObExpr *ob_type_expr); + int check_type_static(apsara::odps::sdk::ODPSColumnTypeInfo odps_type_info, + const ObExpr *ob_type_expr); + inline bool text_type_length_is_valid_at_runtime(ObObjType type, int64_t odps_data_length) { + bool is_valid = false; + if (ObTinyTextType == type && odps_data_length < OB_MAX_TINYTEXT_LENGTH) { + is_valid = true; + } else if (ObTextType == type && odps_data_length < OB_MAX_TEXT_LENGTH) { + is_valid = true; + } else if (ObMediumTextType == type && odps_data_length < OB_MAX_MEDIUMTEXT_LENGTH) { + is_valid = true; + } else if (ObLongTextType == type && odps_data_length < OB_MAX_LONGTEXT_LENGTH) { + is_valid = true; + } + return is_valid; + } +private: + ObODPSGeneralFormat odps_format_; + apsara::odps::sdk::Account account_; + apsara::odps::sdk::Configuration conf_; + apsara::odps::sdk::OdpsTunnel tunnel_; + apsara::odps::sdk::IODPSPtr odps_; + apsara::odps::sdk::IODPSTablePtr table_handle_; + ObSEArray partition_list_; + ObSEArray column_list_; + ObSEArray target_column_id_list_; + StateValues state_; + bool is_part_table_; + int64_t total_count_; + ObBitVector *bit_vector_cache_; + apsara::odps::sdk::ODPSTableRecordPtr record_; + apsara::odps::sdk::ODPSTableRecordPtr *records_; + int64_t batch_size_; // -1 means not inited, 0 means call get_next_row(), > 0 means call get_next_rows() + bool get_next_task_; // only used for get next task and recall inner_get_next_row() when curren task was iter end. + common::ObMalloc malloc_alloc_; + common::ObArenaAllocator arena_alloc_; + common::ObMemAttr mem_attr_; +}; + +class ObOdpsPartitionDownloaderMgr +{ +public: + struct OdpsPartitionDownloader { + OdpsPartitionDownloader() : + odps_driver_(), + odps_partition_downloader_(NULL) + {} + ~OdpsPartitionDownloader() { + reset(); + } + int reset(); + ObODPSTableRowIterator odps_driver_; + apsara::odps::sdk::IDownloadPtr odps_partition_downloader_; + }; + class DeleteDownloaderFunc + { + public: + DeleteDownloaderFunc() {} + virtual ~DeleteDownloaderFunc() = default; + int operator()(common::hash::HashMapPair &kv); + }; + struct OdpsUploader { + OdpsUploader() : upload_(NULL), record_writer_(NULL) {} + ~OdpsUploader() { + upload_.reset(); + record_writer_.reset(); + } + apsara::odps::sdk::IUploadPtr upload_; + apsara::odps::sdk::IRecordWriterPtr record_writer_; + }; + ObOdpsPartitionDownloaderMgr() : inited_(false), is_download_(true), ref_(0), need_commit_(true) {} + int init_downloader(common::ObArray &external_table_files, + const ObString &properties); + int init_uploader(const ObString &properties, + const ObString &external_partition, + bool is_overwrite, + int64_t parallel); + static int create_upload_session(const sql::ObODPSGeneralFormat &odps_format, + const ObString &external_partition, + bool is_overwrite, + apsara::odps::sdk::IUploadPtr &upload); + int get_odps_downloader(int64_t part_id, apsara::odps::sdk::IDownloadPtr &downloader); + int get_odps_uploader(int64_t block_id, + apsara::odps::sdk::IUploadPtr &upload, + apsara::odps::sdk::IRecordWriterPtr &record_writer); + int commit_upload(); + int reset(); + OB_INLINE bool is_download_mgr_inited() { return inited_ && is_download_; } + inline int64_t inc_ref() + { + return ATOMIC_FAA(&ref_, 1); + } + inline int64_t dec_ref() + { + return ATOMIC_SAF(&ref_, 1); + } + inline void set_fail() + { + ATOMIC_STORE(&need_commit_, false); + } +private: + bool inited_; + bool is_download_; + common::hash::ObHashMap odps_mgr_map_; + common::ObArenaAllocator arena_alloc_; + int64_t ref_; + bool need_commit_; +}; + +} // sql +} // oceanbase + +#endif +#endif // __SQL_OB_ODPS_TABLE_ROW_ITER_H__ \ No newline at end of file diff --git a/src/sql/ob_sql_utils.cpp b/src/sql/ob_sql_utils.cpp index 2a11cc596..299fb7fea 100644 --- a/src/sql/ob_sql_utils.cpp +++ b/src/sql/ob_sql_utils.cpp @@ -53,6 +53,7 @@ #include "lib/charset/ob_charset.h" #include "pl/ob_pl_user_type.h" #include "sql/engine/expr/ob_expr_lob_utils.h" +#include "sql/engine/cmd/ob_load_data_parser.h" #ifdef OB_BUILD_SPM #include "sql/spm/ob_spm_controller.h" #endif @@ -1404,6 +1405,50 @@ int ObSQLUtils::check_and_copy_column_alias_name(const ObCollationType cs_type, return ret; } +int ObSQLUtils::extract_odps_part_spec(const ObString &all_part_spec, ObIArray &part_spec_list) +{ + int ret = OB_SUCCESS; + if (all_part_spec.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected empty odps part spec", K(ret)); + } else { + const char* start = all_part_spec.ptr(); + const char* end = start + all_part_spec.length(); + const char* ptr = NULL; + while (start < end && OB_SUCC(ret)) { + if (ptr == NULL && *start == '\'') { + ptr = start; + } else if (ptr != NULL && *start == '\'') { + int64_t len = start - ptr - 1; + if (0 == len) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected part spec", K(ret), K(all_part_spec)); + } else if (OB_FAIL(part_spec_list.push_back(ObString(len, ptr + 1)))) { + LOG_WARN("failed to push back part_spec", K(ret)); + } + ptr = NULL; + } + ++start; + } + } + return ret; +} + +int ObSQLUtils::is_external_odps_table(const ObString &properties, ObIAllocator &allocator, bool &is_odps) +{ + int ret = OB_SUCCESS; + is_odps = false; + ObExternalFileFormat format; + if (properties.empty()) { + // do nothing + } else if (OB_FAIL(format.load_from_string(properties, allocator))) { + LOG_WARN("fail to load from properties string", K(ret), K(properties)); + } else { + is_odps = ObExternalFileFormat::FormatType::ODPS_FORMAT == format.format_type_; + } + return ret; +} + int ObSQLUtils::check_ident_name(const ObCollationType cs_type, ObString &name, const bool check_for_path_char, const int64_t max_ident_len) { @@ -5417,7 +5462,7 @@ void ObSQLUtils::adjust_time_by_ntp_offset(int64_t &dst_timeout_ts) bool ObSQLUtils::is_external_files_on_local_disk(const ObString &url) { - return url.prefix_match_ci(OB_FILE_PREFIX); + return url.empty() ? false : url.prefix_match_ci(OB_FILE_PREFIX); } int ObSQLUtils::split_remote_object_storage_url(ObString &url, ObBackupStorageInfo &storage_info) diff --git a/src/sql/ob_sql_utils.h b/src/sql/ob_sql_utils.h index 52618c7cf..f79a794de 100644 --- a/src/sql/ob_sql_utils.h +++ b/src/sql/ob_sql_utils.h @@ -715,6 +715,8 @@ public: static int64_t combine_server_id(int64_t ts, uint64_t server_id) { return (ts & ((1LL << 43) - 1LL)) | ((server_id & 0xFFFF) << 48); } + static int extract_odps_part_spec(const ObString &all_part_spec, ObIArray &part_spec_list); + static int is_external_odps_table(const ObString &properties, ObIAllocator &allocator, bool &is_odps); static int check_ident_name(const common::ObCollationType cs_type, common::ObString &name, const bool check_for_path_char, const int64_t max_ident_len); diff --git a/src/sql/optimizer/ob_insert_log_plan.cpp b/src/sql/optimizer/ob_insert_log_plan.cpp index c912be791..4ee1c71bb 100644 --- a/src/sql/optimizer/ob_insert_log_plan.cpp +++ b/src/sql/optimizer/ob_insert_log_plan.cpp @@ -13,6 +13,7 @@ #define USING_LOG_PREFIX SQL_OPT #include "sql/resolver/dml/ob_insert_stmt.h" #include "sql/optimizer/ob_log_insert.h" +#include "sql/optimizer/ob_log_select_into.h" #include "sql/optimizer/ob_insert_log_plan.h" #include "sql/optimizer/ob_log_operator_factory.h" #include "sql/optimizer/ob_log_plan_factory.h" @@ -126,21 +127,32 @@ int ObInsertLogPlan::generate_normal_raw_plan() osg_info->online_sample_rate_ = online_sample_percent; } } - if (OB_SUCC(ret)) { - if (OB_FAIL(prepare_dml_infos())) { - LOG_WARN("failed to prepare dml infos", K(ret)); - } else if (use_pdml()) { - if (OB_FAIL(candi_allocate_pdml_insert(osg_info))) { - LOG_WARN("failed to allocate pdml insert", K(ret)); + TableItem *insert_table = NULL; + if (OB_ISNULL(insert_table = insert_stmt->get_table_item_by_id(insert_stmt->get_insert_table_info().table_id_))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("insert target table is unexpected null", K(ret)); + } else if (schema::EXTERNAL_TABLE == insert_table->table_type_) { + if (OB_FAIL(candi_allocate_select_into_for_insert())) { + LOG_WARN("failed to allocate select into op", K(ret)); } else { - LOG_TRACE("succeed to allocate pdml insert operator", - K(candidates_.candidate_plans_.count())); + LOG_TRACE("succeed to allocate select into clause", K(candidates_.candidate_plans_.count())); } - } else if (OB_FAIL(candi_allocate_insert(osg_info))) { - LOG_WARN("failed to allocate insert operator", K(ret)); } else { - LOG_TRACE("succeed to allocate insert operator", K(candidates_.candidate_plans_.count())); + if (OB_FAIL(prepare_dml_infos())) { + LOG_WARN("failed to prepare dml infos", K(ret)); + } else if (use_pdml()) { + if (OB_FAIL(candi_allocate_pdml_insert(osg_info))) { + LOG_WARN("failed to allocate pdml insert", K(ret)); + } else { + LOG_TRACE("succeed to allocate pdml insert operator", + K(candidates_.candidate_plans_.count())); + } + } else if (OB_FAIL(candi_allocate_insert(osg_info))) { + LOG_WARN("failed to allocate insert operator", K(ret)); + } else { + LOG_TRACE("succeed to allocate insert operator", K(candidates_.candidate_plans_.count())); + } } } if (OB_SUCC(ret) && insert_stmt->get_returning_aggr_item_size() > 0) { @@ -1774,4 +1786,96 @@ int ObInsertLogPlan::get_online_estimate_percent(double &percent) LOG_WARN("failed to get sys online estimate percent", K(ret)); } return ret; -} \ No newline at end of file +} +int ObInsertLogPlan::candi_allocate_select_into_for_insert() +{ + int ret = OB_SUCCESS; + ObExchangeInfo exch_info; + CandidatePlan candidate_plan; + ObSEArray select_into_plans; + int64_t dml_parallel = ObGlobalHint::UNSET_PARALLEL; + if (OB_FAIL(get_parallel_info_from_candidate_plans(dml_parallel))) { + LOG_WARN("failed to get parallel info from candidate plans", K(ret)); + } else if (dml_parallel > 1) { + exch_info.dist_method_ = ObPQDistributeMethod::RANDOM; + } + for (int64_t i = 0 ; OB_SUCC(ret) && i < candidates_.candidate_plans_.count(); ++i) { + candidate_plan = candidates_.candidate_plans_.at(i); + if (OB_ISNULL(candidate_plan.plan_tree_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (candidate_plan.plan_tree_->is_sharding() + && OB_FAIL((allocate_exchange_as_top(candidate_plan.plan_tree_, exch_info)))) { + LOG_WARN("failed to allocate exchange as top", K(ret)); + } else if (OB_FAIL(allocate_select_into_as_top_for_insert(candidate_plan.plan_tree_))) { + LOG_WARN("failed to allocate select into", K(ret)); + } else if (OB_FAIL(select_into_plans.push_back(candidate_plan))) { + LOG_WARN("failed to push back candidate plan", K(ret)); + } else { /*do nothing*/ } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(prune_and_keep_best_plans(select_into_plans))) { + LOG_WARN("failed to prune and keep best plans", K(ret)); + } else { /*do nothing*/ } + } + return ret; +} + +int ObInsertLogPlan::allocate_select_into_as_top_for_insert(ObLogicalOperator *&old_top) +{ + int ret = OB_SUCCESS; + ObLogSelectInto *select_into = NULL; + ObSchemaGetterGuard *schema_guard = NULL; + const ObTableSchema *table_schema = NULL; + ObSQLSessionInfo *session_info = NULL; + const ObInsertStmt *stmt = get_stmt(); + if (OB_ISNULL(old_top) || OB_ISNULL(stmt) + || OB_ISNULL(schema_guard = get_optimizer_context().get_schema_guard()) + || OB_ISNULL(session_info = get_optimizer_context().get_session_info()) + || stmt->get_table_items().count() != 2 + || OB_ISNULL(stmt->get_table_item(0)) || OB_ISNULL(stmt->get_table_item(1))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Get unexpected null", K(ret), K(old_top), K(schema_guard), K(session_info), K(stmt)); + } else if (OB_FAIL(schema_guard->get_table_schema(session_info->get_effective_tenant_id(), + stmt->get_insert_table_info().ref_table_id_, + table_schema))) { + LOG_WARN("get table schema from schema guard failed", K(ret)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (OB_ISNULL(select_into = static_cast( + get_log_op_factory().allocate(*this, LOG_SELECT_INTO)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate memory for ObLogSelectInto failed", K(ret)); + } else { + ObString external_properties; + const ObString &format_or_properties = table_schema->get_external_file_format().empty() + ? table_schema->get_external_properties() + : table_schema->get_external_file_format(); + const ObInsertTableInfo& table_info = stmt->get_insert_table_info(); + if (format_or_properties.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("external properties is empty", K(ret)); + } else if (table_schema->get_external_properties().empty()) { //目前只支持写odps外表 其他类型暂不支持 + ret = OB_NOT_SUPPORTED; + LOG_WARN("not support to insert into external table which is not in odps", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "insert into external table which is not in odps"); + } else if (OB_FAIL(ob_write_string(get_allocator(), format_or_properties, external_properties))) { + LOG_WARN("failed to append string", K(ret)); + } else if (OB_FAIL(select_into->get_select_exprs().assign(table_info.column_conv_exprs_))) { + LOG_WARN("failed to get select exprs", K(ret)); + } else { + select_into->set_is_overwrite(stmt->is_overwrite()); + select_into->set_external_properties(external_properties); + select_into->set_external_partition(stmt->get_table_item(0)->external_table_partition_); + select_into->set_child(ObLogicalOperator::first_child, old_top); + // compute property + if (OB_FAIL(select_into->compute_property())) { + LOG_WARN("failed to compute equal set", K(ret)); + } else { + old_top = select_into; + } + } + } + return ret; +} diff --git a/src/sql/optimizer/ob_insert_log_plan.h b/src/sql/optimizer/ob_insert_log_plan.h index 207ae3182..0d68fada8 100644 --- a/src/sql/optimizer/ob_insert_log_plan.h +++ b/src/sql/optimizer/ob_insert_log_plan.h @@ -74,6 +74,10 @@ protected: int candi_allocate_pdml_insert(OSGShareInfo *osg_info); int candi_allocate_optimizer_stats_merge(OSGShareInfo *osg_info); + /** @brief Allocate SELECTINTO on top of plan candidates when insert into external table*/ + int candi_allocate_select_into_for_insert(); + int allocate_select_into_as_top_for_insert(ObLogicalOperator *&old_top); + int get_osg_type(bool is_multi_part_dml, ObShardingInfo *insert_table_sharding, int64_t distributed_method, diff --git a/src/sql/optimizer/ob_log_plan.cpp b/src/sql/optimizer/ob_log_plan.cpp index 11723a8f3..19e49531f 100644 --- a/src/sql/optimizer/ob_log_plan.cpp +++ b/src/sql/optimizer/ob_log_plan.cpp @@ -142,7 +142,8 @@ ObLogPlan::ObLogPlan(ObOptimizerContext &ctx, const ObDMLStmt *stmt) selectivity_ctx_(ctx, this, stmt), alloc_sfu_list_(), onetime_copier_(NULL), - nonrecursive_plan_for_fake_cte_(NULL) + nonrecursive_plan_for_fake_cte_(NULL), + has_allocated_range_shuffle_(false) { } @@ -6957,19 +6958,22 @@ int ObLogPlan::allocate_sort_and_exchange_as_top(ObLogicalOperator *&top, bool has_select_into = false; bool is_single = true; bool has_order_by = false; + ObRawExpr* partition_expr = NULL; if (OB_ISNULL(top) || OB_ISNULL(get_stmt())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret)); - } else if (OB_FAIL(check_select_into(has_select_into, is_single, has_order_by))) { + } else if (OB_FAIL(check_select_into(has_select_into, is_single, has_order_by, partition_expr))) { LOG_WARN("failed to check select into", K(ret)); } else if (exch_info.is_pq_local() && NULL == topn_expr && has_select_into && !is_single - && has_order_by) { + && has_order_by && (NULL == partition_expr || partition_expr->is_const_expr())) { if (OB_FAIL(allocate_dist_range_sort_for_select_into(top, sort_keys, need_sort, is_local_order))) { LOG_WARN("failed to allocate dist range sort as top", K(ret)); - } else { /*do nothing*/ } + } else { + has_allocated_range_shuffle_ = true; + } } else if (exch_info.is_pq_local() && NULL == topn_expr && GCONF._enable_px_ordered_coord) { if (OB_FAIL(allocate_dist_range_sort_as_top(top, sort_keys, need_sort, is_local_order))) { LOG_WARN("failed to allocate dist range sort as top", K(ret)); @@ -7576,11 +7580,16 @@ int ObLogPlan::allocate_limit_as_top(ObLogicalOperator *&old_top, return ret; } -int ObLogPlan::check_select_into(bool &has_select_into, bool &is_single, bool &has_order_by){ +int ObLogPlan::check_select_into(bool &has_select_into, + bool &is_single, + bool &has_order_by, + ObRawExpr *&file_partition_expr) +{ int ret = OB_SUCCESS; has_select_into = false; is_single = true; has_order_by = false; + file_partition_expr = NULL; ObSelectIntoItem *into_item = NULL; if (OB_ISNULL(get_stmt())) { ret = OB_ERR_UNEXPECTED; @@ -7590,11 +7599,11 @@ int ObLogPlan::check_select_into(bool &has_select_into, bool &is_single, bool &h } else { const ObSelectStmt *stmt = static_cast(get_stmt()); has_select_into = stmt->has_select_into(); - into_item = stmt->get_select_into(); - if (NULL != into_item && !into_item->is_single_) { - is_single = false; - } has_order_by = stmt->has_order_by(); + if (NULL != (into_item = stmt->get_select_into())) { + is_single = into_item->is_single_; + file_partition_expr = into_item->file_partition_expr_; + } } return ret; } @@ -7606,12 +7615,16 @@ int ObLogPlan::candi_allocate_select_into() bool has_select_into = false; bool is_single = true; bool has_order_by = false; + ObRawExpr* partition_expr = NULL; + ObSEArray partition_exprs; CandidatePlan candidate_plan; ObSEArray select_into_plans; - if (OB_FAIL(check_select_into(has_select_into, is_single, has_order_by))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret)); - } else if (!is_single && !has_order_by) { + if (OB_FAIL(check_select_into(has_select_into, is_single, has_order_by, partition_expr))) { + LOG_WARN("failed to check select into", K(ret)); + } else if (partition_expr != NULL && !partition_expr->is_const_expr() + && OB_FAIL(partition_exprs.push_back(partition_expr))) { + LOG_WARN("failed to push back partition expr", K(ret)); + } else if (!is_single && !has_order_by && partition_exprs.count() == 0) { exch_info.dist_method_ = ObPQDistributeMethod::RANDOM; } for (int64_t i = 0 ; OB_SUCC(ret) && i < candidates_.candidate_plans_.count(); ++i) { @@ -7619,7 +7632,12 @@ int ObLogPlan::candi_allocate_select_into() if (OB_ISNULL(candidate_plan.plan_tree_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret)); - } else if (!has_order_by && candidate_plan.plan_tree_->is_sharding() + } else if (!is_single && !has_order_by && partition_exprs.count() != 0 + && OB_FAIL(get_grouping_style_exchange_info(partition_exprs, + candidate_plan.plan_tree_->get_output_equal_sets(), + exch_info))) { + LOG_WARN("failed to get grouping style exchange info", K(ret)); + } else if (!has_allocated_range_shuffle_ && candidate_plan.plan_tree_->is_sharding() && OB_FAIL((allocate_exchange_as_top(candidate_plan.plan_tree_, exch_info)))) { LOG_WARN("failed to allocate exchange as top", K(ret)); } else if (OB_FAIL(allocate_select_into_as_top(candidate_plan.plan_tree_))) { @@ -7647,11 +7665,10 @@ int ObLogPlan::allocate_select_into_as_top(ObLogicalOperator *&old_top) } else if (OB_ISNULL(select_into = static_cast( get_log_op_factory().allocate(*this, LOG_SELECT_INTO)))) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_ERROR("allocate memory for ObLogSelectInto failed", K(ret)); + LOG_WARN("allocate memory for ObLogSelectInto failed", K(ret)); } else { ObSelectIntoItem *into_item = stmt->get_select_into(); ObSEArray select_exprs; - ObRawExpr *to_outfile_expr = NULL; if (OB_ISNULL(into_item)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("into item is null", K(ret)); @@ -7669,9 +7686,11 @@ int ObLogPlan::allocate_select_into_as_top(ObLogicalOperator *&old_top) select_into->set_closed_cht(into_item->closed_cht_); select_into->set_is_single(into_item->is_single_); select_into->set_max_file_size(into_item->max_file_size_); + select_into->set_buffer_size(into_item->buffer_size_); select_into->set_escaped_cht(into_item->escaped_cht_); select_into->set_cs_type(into_item->cs_type_); select_into->set_child(ObLogicalOperator::first_child, old_top); + select_into->set_file_partition_expr(into_item->file_partition_expr_); // compute property if (OB_FAIL(select_into->compute_property())) { LOG_WARN("failed to compute equal set", K(ret)); diff --git a/src/sql/optimizer/ob_log_plan.h b/src/sql/optimizer/ob_log_plan.h index 8a8aabdaa..bcb7f820d 100644 --- a/src/sql/optimizer/ob_log_plan.h +++ b/src/sql/optimizer/ob_log_plan.h @@ -989,7 +989,10 @@ public: int allocate_select_into_as_top(ObLogicalOperator *&old_top); - int check_select_into(bool &has_select_into, bool &is_single, bool &has_order_by); + int check_select_into(bool &has_select_into, + bool &is_single, + bool &has_order_by, + ObRawExpr *&file_partition_expr); int allocate_expr_values_as_top(ObLogicalOperator *&top, const ObIArray *filter_exprs = NULL); @@ -1889,6 +1892,46 @@ private: common::ObSEArray new_or_quals_; ObSelectLogPlan *nonrecursive_plan_for_fake_cte_; + + // has_allocated_range_shuffle_ is a flag for select into + // when flag = true, logical plan is like + // select into + // | + // sort + // | + // exchange in distr + // | + // exchange out distr(range) + // condition: partition expr is null or const expr, single is false, has order by without limit + // + // when flag = false, logical plan is like + // select into + // | + // exchange in distr + // | + // exchange out distr(random) + // condition: single is false, no order by, partition expr is null or const expr + // + // or + // + // select into + // | + // exchange in distr + // | + // exchange out distr(hash) + // condition: single is false, no order by, partition expr is not const expr + // + // or + // + // select into + // | + // px coordinator + // | + // exchange out distr + // condition: single is true / parallel = 1 / has limit / has order by and partition by + // + // 为select into分配了range shuffle后, 在分配select into算子时不应再分配exchange算子 + bool has_allocated_range_shuffle_; DISALLOW_COPY_AND_ASSIGN(ObLogPlan); }; diff --git a/src/sql/optimizer/ob_log_select_into.cpp b/src/sql/optimizer/ob_log_select_into.cpp index 5e6065b76..31224bc76 100644 --- a/src/sql/optimizer/ob_log_select_into.cpp +++ b/src/sql/optimizer/ob_log_select_into.cpp @@ -62,6 +62,8 @@ int ObLogSelectInto::get_op_exprs(ObIArray &all_exprs) int ret = OB_SUCCESS; if (OB_FAIL(append(all_exprs, select_exprs_))) { LOG_WARN("failed to push back select exprs", K(ret)); + } else if (file_partition_expr_ != NULL && OB_FAIL(all_exprs.push_back(file_partition_expr_))) { + LOG_WARN("failed to push back file partition expr", K(ret)); } else if (OB_FAIL(ObLogicalOperator::get_op_exprs(all_exprs))) { LOG_WARN("failed to get op exprs", K(ret)); } else { /*do nothing*/ } diff --git a/src/sql/optimizer/ob_log_select_into.h b/src/sql/optimizer/ob_log_select_into.h index 5fb32f45a..22db6b80b 100644 --- a/src/sql/optimizer/ob_log_select_into.h +++ b/src/sql/optimizer/ob_log_select_into.h @@ -16,6 +16,7 @@ #include "sql/optimizer/ob_logical_operator.h" #include "sql/optimizer/ob_log_operator_factory.h" #include "objit/common/ob_item_type.h" +#include "sql/engine/cmd/ob_load_data_parser.h" namespace oceanbase { @@ -36,7 +37,12 @@ public: is_optional_(true), is_single_(true), max_file_size_(DEFAULT_MAX_FILE_SIZE), - escaped_cht_() + buffer_size_(DEFAULT_BUFFER_SIZE), + escaped_cht_(), + file_partition_expr_(NULL), + is_overwrite_(false), + external_properties_(), + external_partition_() { cs_type_ = ObCharset::get_system_collation(); } @@ -78,18 +84,38 @@ public: { max_file_size_ = max_file_size; } - inline void set_closed_cht(common::ObObj closed_cht) + inline void set_buffer_size(int64_t buffer_size) + { + buffer_size_ = buffer_size; + } + inline void set_closed_cht(common::ObObj &closed_cht) { closed_cht_ = closed_cht; } - inline void set_escaped_cht(common::ObObj escaped_cht) + inline void set_escaped_cht(common::ObObj &escaped_cht) { escaped_cht_ = escaped_cht; } - inline void set_cs_type(common::ObCollationType cs_type) + inline void set_cs_type(common::ObCollationType &cs_type) { cs_type_ = cs_type; } + inline void set_file_partition_expr(sql::ObRawExpr* file_partition_expr) + { + file_partition_expr_ = file_partition_expr; + } + inline void set_is_overwrite(bool is_overwrite) + { + is_overwrite_ = is_overwrite; + } + inline void set_external_properties(const common::ObString &external_properties) + { + external_properties_.assign_ptr(external_properties.ptr(), external_properties.length()); + } + inline void set_external_partition(const common::ObString &external_partition) + { + external_partition_.assign_ptr(external_partition.ptr(), external_partition.length()); + } inline ObItemType get_into_type() const { return into_type_; @@ -122,6 +148,10 @@ public: { return max_file_size_; } + inline int64_t get_buffer_size() const + { + return buffer_size_; + } inline common::ObObj get_closed_cht() const { return closed_cht_; @@ -134,6 +164,22 @@ public: { return cs_type_; } + inline sql::ObRawExpr* get_file_partition_expr() const + { + return file_partition_expr_; + } + inline bool get_is_overwrite() const + { + return is_overwrite_; + } + inline common::ObString get_external_properties() const + { + return external_properties_; + } + inline common::ObString get_external_partition() const + { + return external_partition_; + } const common::ObIArray &get_select_exprs() const { return select_exprs_; } common::ObIArray &get_select_exprs() { return select_exprs_; } virtual int est_cost() override; @@ -141,6 +187,7 @@ public: virtual int get_op_exprs(ObIArray &all_exprs) override; virtual int inner_replace_op_exprs(ObRawExprReplacer &replacer); static const int64_t DEFAULT_MAX_FILE_SIZE = 256*1024*1024; + static const int64_t DEFAULT_BUFFER_SIZE = 1*1024*1024; private: ObItemType into_type_; common::ObObj outfile_name_; @@ -152,8 +199,13 @@ private: bool is_optional_; bool is_single_; int64_t max_file_size_; + int64_t buffer_size_; common::ObObj escaped_cht_; common::ObCollationType cs_type_; + sql::ObRawExpr* file_partition_expr_; + bool is_overwrite_; + common::ObString external_properties_; + common::ObString external_partition_; }; } } diff --git a/src/sql/parser/non_reserved_keywords_mysql_mode.c b/src/sql/parser/non_reserved_keywords_mysql_mode.c index 78eec77c6..3bc066704 100644 --- a/src/sql/parser/non_reserved_keywords_mysql_mode.c +++ b/src/sql/parser/non_reserved_keywords_mysql_mode.c @@ -101,6 +101,7 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] = {"buckets", BUCKETS}, {"backup_copies", BACKUP_COPIES}, {"badfile", BADFILE}, + {"buffer_size", BUFFER_SIZE}, {"cache", CACHE}, {"calibration", CALIBRATION}, {"calibration_info", CALIBRATION_INFO}, @@ -594,6 +595,13 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] = {"obconfig_url", OBCONFIG_URL}, {"object", OBJECT}, {"object_id", OBJECT_ID}, + {"accessid", ACCESSID}, + {"accesskey", ACCESSKEY}, + {"accesstype", ACCESSTYPE}, + {"endpoint", ENDPOINT}, + {"project_name", PROJECT_NAME}, + {"quota_name", QUOTA_NAME}, + {"compression_code", COMPRESSION_CODE}, {"of", OF}, {"off", OFF}, {"offset", OFFSET}, @@ -668,6 +676,7 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] = {"profiles", PROFILES}, {"progressive_merge_num", PROGRESSIVE_MERGE_NUM}, {"protection", PROTECTION}, + {"properties", PROPERTIES}, {"proxy", PROXY}, {"public", PUBLIC}, {"purge", PURGE}, @@ -852,6 +861,7 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] = {"ssl", SSL}, {"stacked", STACKED}, {"standby", STANDBY}, + {"ststoken", STSTOKEN}, {"start", START}, {"starts", STARTS}, {"starting", STARTING}, diff --git a/src/sql/parser/sql_parser_mysql_mode.y b/src/sql/parser/sql_parser_mysql_mode.y index 4f2035dd4..41146b340 100644 --- a/src/sql/parser/sql_parser_mysql_mode.y +++ b/src/sql/parser/sql_parser_mysql_mode.y @@ -261,7 +261,7 @@ END_P SET_VAR DELIMITER //-----------------------------reserved keyword end------------------------------------------------- %token //-----------------------------non_reserved keyword begin------------------------------------------- - ACCESS ACCOUNT ACTION ACTIVE ADDDATE AFTER AGAINST AGGREGATE ALGORITHM ALL_META ALL_USER ALWAYS ALLOW ANALYSE ANY + ACCESS ACCESSID ACCESSKEY ACCESSTYPE ACCOUNT ACTION ACTIVE ADDDATE AFTER AGAINST AGGREGATE ALGORITHM ALL_META ALL_USER ALWAYS ALLOW ANALYSE ANY APPROX_COUNT_DISTINCT APPROX_COUNT_DISTINCT_SYNOPSIS APPROX_COUNT_DISTINCT_SYNOPSIS_MERGE ARBITRATION ARRAY ASCII ASIS AT AUTHORS AUTO AUTOEXTEND_SIZE AUTO_INCREMENT AUTO_INCREMENT_MODE AUTO_INCREMENT_CACHE_SIZE AVG AVG_ROW_LENGTH ACTIVATE AVAILABILITY ARCHIVELOG ASYNCHRONOUS AUDIT ADMIN AUTO_REFRESH @@ -269,12 +269,12 @@ END_P SET_VAR DELIMITER BACKUP BACKUP_COPIES BALANCE BANDWIDTH BASE BASELINE BASELINE_ID BASIC BEGI BINDING SHARDING BINLOG BIT BIT_AND BIT_OR BIT_XOR BLOCK BLOCK_INDEX BLOCK_SIZE BLOOM_FILTER BOOL BOOLEAN BOOTSTRAP BTREE BYTE BREADTH BUCKETS BISON_LIST BACKUPSET BACKED BACKUPPIECE BACKUP_BACKUP_DEST BACKUPROUND - BADFILE + BADFILE BUFFER_SIZE CACHE CALIBRATION CALIBRATION_INFO CANCEL CASCADED CAST CATALOG_NAME CHAIN CHANGED CHARSET CHECKSUM CHECKPOINT CHUNK CIPHER CLASS_ORIGIN CLEAN CLEAR CLIENT CLONE CLOG CLOSE CLUSTER CLUSTER_ID CLUSTER_NAME COALESCE COLUMN_STAT CODE COLLATION COLUMN_FORMAT COLUMN_NAME COLUMNS COMMENT COMMIT COMMITTED COMPACT COMPLETION COMPLETE - COMPRESSED COMPRESSION COMPUTATION COMPUTE CONCURRENT CONDENSED CONDITIONAL CONNECTION CONSISTENT CONSISTENT_MODE CONSTRAINT_CATALOG + COMPRESSED COMPRESSION COMPRESSION_CODE COMPUTATION COMPUTE CONCURRENT CONDENSED CONDITIONAL CONNECTION CONSISTENT CONSISTENT_MODE CONSTRAINT_CATALOG CONSTRAINT_NAME CONSTRAINT_SCHEMA CONTAINS CONTEXT CONTRIBUTORS COPY COUNT CPU CREATE_TIMESTAMP CTXCAT CTX_ID CUBE CURDATE CURRENT STACKED CURTIME CURSOR_NAME CUME_DIST CYCLE CALC_PARTITION_ID CONNECT @@ -283,7 +283,7 @@ END_P SET_VAR DELIMITER DIRECTORY DISABLE DISALLOW DISCARD DISK DISKGROUP DO DOT DUMP DUMPFILE DUPLICATE DUPLICATE_SCOPE DYNAMIC DATABASE_ID DEFAULT_TABLEGROUP DISCONNECT DEMAND - EFFECTIVE EMPTY ENABLE ENABLE_ARBITRATION_SERVICE ENABLE_EXTENDED_ROWID ENCRYPTED ENCRYPTION END ENDS ENFORCED ENGINE_ ENGINES ENUM ENTITY ERROR_CODE ERROR_P ERRORS ESTIMATE + EFFECTIVE EMPTY ENABLE ENABLE_ARBITRATION_SERVICE ENABLE_EXTENDED_ROWID ENCRYPTED ENCRYPTION END ENDPOINT ENDS ENFORCED ENGINE_ ENGINES ENUM ENTITY ERROR_CODE ERROR_P ERRORS ESTIMATE ESCAPE EVENT EVENTS EVERY EXCHANGE EXCLUDING EXECUTE EXPANSION EXPIRE EXPIRE_INFO EXPORT OUTLINE EXTENDED EXTENDED_NOADDR EXTENT_SIZE EXTRACT EXCEPT EXPIRED ENCODING EMPTY_FIELD_AS_NULL EXTERNAL @@ -330,11 +330,11 @@ END_P SET_VAR DELIMITER PACK_KEYS PAGE PARALLEL PARAMETERS PARSER PARTIAL PARTITION_ID PARTITIONING PARTITIONS PASSWORD PATH PAUSE PAXOS_REPLICA_NUM PERCENTAGE PERCENT_RANK PHASE PLAN PHYSICAL PLANREGRESS PLUGIN PLUGIN_DIR PLUGINS POINT POLYGON PERFORMANCE - PROTECTION PRIORITY PL POLICY POOL PORT POSITION PREPARE PRESERVE PRETTY PRETTY_COLOR PREV PRIMARY_ZONE PRIVILEGES PROCESS - PROCESSLIST PROFILE PROFILES PROXY PRECEDING PCTFREE P_ENTITY P_CHUNK + PROTECTION PROJECT_NAME PRIORITY PL POLICY POOL PORT POSITION PREPARE PRESERVE PRETTY PRETTY_COLOR PREV PRIMARY_ZONE PRIVILEGES PROCESS + PROCESSLIST PROFILE PROFILES PROPERTIES PROXY PRECEDING PCTFREE P_ENTITY P_CHUNK PUBLIC PROGRESSIVE_MERGE_NUM PREVIEW PS PLUS PATTERN PARTITION_TYPE - QUARTER QUERY QUERY_RESPONSE_TIME QUEUE_TIME QUICK + QUARTER QUERY QUERY_RESPONSE_TIME QUEUE_TIME QUICK QUOTA_NAME RB_AND_AGG RB_BUILD_AGG RB_OR_AGG REBUILD RECOVER RECOVERY_WINDOW RECYCLE REDO_BUFFER_SIZE REDOFILE REDUNDANCY REDUNDANT REFRESH REGION RELAY RELAYLOG RELAY_LOG_FILE RELAY_LOG_POS RELAY_THREAD RELOAD REMAP REMOVE REORGANIZE REPAIR REPEATABLE REPLICA @@ -350,7 +350,7 @@ END_P SET_VAR DELIMITER SOURCE SPFILE SPLIT SQL_AFTER_GTIDS SQL_AFTER_MTS_GAPS SQL_BEFORE_GTIDS SQL_BUFFER_RESULT SQL_CACHE SQL_NO_CACHE SQL_ID SCHEMA_ID SQL_THREAD SQL_TSI_DAY SQL_TSI_HOUR SQL_TSI_MINUTE SQL_TSI_MONTH SQL_TSI_QUARTER SQL_TSI_SECOND SQL_TSI_WEEK SQL_TSI_YEAR SRID STANDBY _ST_ASMVT STAT START STARTS STATS_AUTO_RECALC - STATS_PERSISTENT STATS_SAMPLE_PAGES STATUS STATEMENTS STATISTICS STD STDDEV STDDEV_POP STDDEV_SAMP STRONG + STATS_PERSISTENT STATS_SAMPLE_PAGES STATUS STATEMENTS STATISTICS STD STDDEV STDDEV_POP STDDEV_SAMP STRONG STSTOKEN SYNCHRONIZATION SYNCHRONOUS STOP STORAGE STORAGE_FORMAT_VERSION STORE STORING STRING SUBCLASS_ORIGIN SUBDATE SUBJECT SUBPARTITION SUBPARTITIONS SUBSTR SUBSTRING SUCCESSFUL SUM SUPER SUSPEND SWAPS SWITCH SWITCHES SWITCHOVER SYSTEM SYSTEM_USER SYSDATE SESSION_ALIAS @@ -501,7 +501,7 @@ END_P SET_VAR DELIMITER %type lock_tables_stmt unlock_tables_stmt lock_type lock_table_list lock_table opt_local %type flashback_stmt purge_stmt opt_flashback_rename_table opt_flashback_rename_database opt_flashback_rename_tenant %type tenant_name_list opt_tenant_list tenant_list_tuple cache_type flush_scope opt_zone_list -%type into_opt into_clause field_opt field_term field_term_list line_opt line_term line_term_list into_var_list into_var file_opt file_option_list file_option file_size_const +%type into_opt into_clause field_opt field_term field_term_list line_opt line_term line_term_list into_var_list into_var file_partition_opt file_opt file_option_list file_option file_size_const %type string_list text_string string_val_list %type balance_task_type opt_balance_task_type %type list_expr list_partition_element list_partition_expr list_partition_list list_partition_option opt_list_partition_list opt_list_subpartition_list list_subpartition_list list_subpartition_element drop_partition_name_list @@ -537,8 +537,9 @@ END_P SET_VAR DELIMITER %type opt_storage_name opt_calibration_list calibration_info_list %type switchover_tenant_stmt switchover_clause opt_verify %type recover_tenant_stmt recover_point_clause -%type external_file_format_list external_file_format external_table_partition_option +%type external_file_format_list external_file_format external_properties_list external_properties external_table_partition_option %type dynamic_sampling_hint add_external_table_partition_actions add_external_table_partition_action +%type external_table_partitions external_table_partition %type skip_index_type opt_skip_index_type_list %type opt_rebuild_column_store %type json_table_expr mock_jt_on_error_on_empty jt_column_list json_table_column_def @@ -7248,6 +7249,11 @@ TABLE_MODE opt_equal_mark STRING_VALUE (void)($2) ; /* make bison mute */ merge_nodes($$, result, T_EXTERNAL_FILE_FORMAT, $4); } +| PROPERTIES opt_equal_mark '(' external_properties_list ')' +{ + (void)($2) ; /* make bison mute */ + merge_nodes($$, result, T_EXTERNAL_PROPERTIES, $4); +} | PATTERN opt_equal_mark STRING_VALUE { (void)($2) ; /* make bison mute */ @@ -8320,6 +8326,73 @@ REDUNDANT } ; +external_properties_list: +external_properties +{ + $$ = $1; +} +| external_properties_list opt_comma external_properties +{ + (void) ($2); + malloc_non_terminal_node($$, result->malloc_pool_, T_LINK_NODE, 2, $1, $3); +} +; + +external_properties: +TYPE COMP_EQ STRING_VALUE +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_EXTERNAL_FILE_FORMAT_TYPE, 1, $3); +} +| ACCESSTYPE COMP_EQ STRING_VALUE +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_ACCESSTYPE, 1, $3); +} +| ACCESSID COMP_EQ STRING_VALUE +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_ACCESSID, 1, $3); + $3->stmt_loc_.first_column_ = @3.first_column - 1; + $3->stmt_loc_.last_column_ = @3.last_column - 1; + result->contain_sensitive_data_ = true; +} +| ACCESSKEY COMP_EQ STRING_VALUE +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_ACCESSKEY, 1, $3); + $3->stmt_loc_.first_column_ = @3.first_column - 1; + $3->stmt_loc_.last_column_ = @3.last_column - 1; + result->contain_sensitive_data_ = true; +} +| STSTOKEN COMP_EQ STRING_VALUE +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_STSTOKEN, 1, $3); + $3->stmt_loc_.first_column_ = @3.first_column - 1; + $3->stmt_loc_.last_column_ = @3.last_column - 1; + result->contain_sensitive_data_ = true; +} +| ENDPOINT COMP_EQ STRING_VALUE +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_ENDPOINT, 1, $3); +} +| PROJECT_NAME COMP_EQ STRING_VALUE +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_PROJECT, 1, $3); +} +| SCHEMA_NAME COMP_EQ STRING_VALUE +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_SCHEMA, 1, $3); +} +| QUOTA_NAME COMP_EQ STRING_VALUE +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_QUOTA, 1, $3); +} +| COMPRESSION_CODE COMP_EQ STRING_VALUE +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_COMPRESSION_CODE, 1, $3); +} +| TABLE_NAME COMP_EQ STRING_VALUE +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_TABLE, 1, $3); +}; + external_file_format_list: external_file_format { @@ -10429,9 +10502,9 @@ LIMIT limit_expr OFFSET limit_expr ; into_clause: -INTO OUTFILE STRING_VALUE opt_charset field_opt line_opt file_opt +INTO OUTFILE STRING_VALUE file_partition_opt opt_charset field_opt line_opt file_opt { - malloc_non_terminal_node($$, result->malloc_pool_, T_INTO_OUTFILE, 5, $3, $4, $5, $6, $7); + malloc_non_terminal_node($$, result->malloc_pool_, T_INTO_OUTFILE, 6, $3, $4, $5, $6, $7, $8); } | INTO DUMPFILE STRING_VALUE { @@ -10560,8 +10633,24 @@ SINGLE opt_equal_mark BOOL_VALUE (void)($2); malloc_non_terminal_node($$, result->malloc_pool_, T_MAX_FILE_SIZE, 1, $3); } +| BUFFER_SIZE opt_equal_mark file_size_const +{ + (void)($2); + malloc_non_terminal_node($$, result->malloc_pool_, T_BUFFER_SIZE, 1, $3); +} ; +file_partition_opt: +/*empty*/ +{ + $$ = NULL; +} +| PARTITION BY bit_expr +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_PARTITION_EXPR, 1, $3); + dup_expr_string($$, result, @3.first_column, @3.last_column); +} + file_size_const: INTNUM { @@ -12844,6 +12933,31 @@ PARTITION '(' name_list ')' merge_nodes(name_list, result, T_NAME_LIST, $3); malloc_non_terminal_node($$, result->malloc_pool_, T_USE_PARTITION, 1, name_list); } +| PARTITION '(' external_table_partitions ')' +{ + ParseNode *partition_value = NULL; + merge_nodes(partition_value, result, T_EXTERNAL_TABLE_PARTITION, $3); + dup_expr_string(partition_value, result, @3.first_column, @3.last_column); + malloc_non_terminal_node($$, result->malloc_pool_, T_USE_PARTITION, 1, partition_value); +} + +external_table_partitions: +external_table_partition +{ + $$ = $1; +} +| external_table_partitions ',' external_table_partition +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_LINK_NODE, 2, $1, $3); +} +; + +external_table_partition: +relation_name COMP_EQ expr_const +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_PARTITION_LIST_ELEMENT, 2, $1, $3); +} +; use_flashback: AS OF SNAPSHOT bit_expr %prec LOWER_PARENS @@ -22963,7 +23077,10 @@ unreserved_keyword_normal { $$=$1;} ; unreserved_keyword_normal: -ACCOUNT +ACCESSID +| ACCESSKEY +| ACCESSTYPE +| ACCOUNT | ACTION | ACTIVE | ADDDATE @@ -23027,6 +23144,7 @@ ACCOUNT | BYTE | BREADTH | BUCKETS +| BUFFER_SIZE | CACHE | CALIBRATION | CALIBRATION_INFO @@ -23069,6 +23187,7 @@ ACCOUNT | COMPLETION | COMPRESSED | COMPRESSION +| COMPRESSION_CODE | COMPUTATION | COMPUTE | CONCURRENT @@ -23146,6 +23265,7 @@ ACCOUNT | ENCRYPTED | ENCRYPTION | END +| ENDPOINT | ENDS | ENFORCED | ENGINE_ @@ -23381,6 +23501,7 @@ ACCOUNT | NVARCHAR | OBJECT | OCCUR +| QUOTA_NAME | OF | OFF | OFFSET @@ -23443,6 +23564,8 @@ ACCOUNT | PROFILE | PROFILES | PROGRESSIVE_MERGE_NUM +| PROJECT_NAME +| PROPERTIES | PS | PUBLIC | PCTFREE @@ -23599,6 +23722,7 @@ ACCOUNT | STORING | STRONG | STRING +| STSTOKEN | SUBCLASS_ORIGIN | SUBDATE | SUBJECT diff --git a/src/sql/plan_cache/ob_sql_parameterization.cpp b/src/sql/plan_cache/ob_sql_parameterization.cpp index dbb232b1f..4472fc9a7 100644 --- a/src/sql/plan_cache/ob_sql_parameterization.cpp +++ b/src/sql/plan_cache/ob_sql_parameterization.cpp @@ -360,6 +360,8 @@ bool ObSqlParameterization::is_tree_not_param(const ParseNode *tree) ret_bool = true; } else if (T_INTO_FILE_LIST == tree->type_) { ret_bool = true; + } else if (T_EXTERNAL_TABLE_PARTITION == tree->type_) { + ret_bool = true; } else if (T_PIVOT_IN_LIST == tree->type_) { ret_bool = true; } else if (T_CHAR_CHARSET == tree->type_) { diff --git a/src/sql/resolver/ddl/ob_alter_table_resolver.cpp b/src/sql/resolver/ddl/ob_alter_table_resolver.cpp index fe6744afd..d83a7646d 100644 --- a/src/sql/resolver/ddl/ob_alter_table_resolver.cpp +++ b/src/sql/resolver/ddl/ob_alter_table_resolver.cpp @@ -219,6 +219,7 @@ int ObAlterTableResolver::resolve(const ParseNode &parse_tree) ObTableSchema &alter_schema = alter_table_stmt->get_alter_table_schema(); alter_schema.set_table_type(table_schema_->get_table_type()); OZ (alter_schema.set_external_file_format(table_schema_->get_external_file_format())); + OZ (alter_schema.set_external_properties(table_schema_->get_external_properties())); OZ (alter_schema.set_external_file_location(table_schema_->get_external_file_location())); OZ (alter_schema.set_external_file_location_access_info(table_schema_->get_external_file_location_access_info())); OZ (alter_schema.set_external_file_pattern(table_schema_->get_external_file_pattern())); diff --git a/src/sql/resolver/ddl/ob_create_table_resolver.cpp b/src/sql/resolver/ddl/ob_create_table_resolver.cpp index c0cf73701..f64136e88 100644 --- a/src/sql/resolver/ddl/ob_create_table_resolver.cpp +++ b/src/sql/resolver/ddl/ob_create_table_resolver.cpp @@ -715,6 +715,13 @@ int ObCreateTableResolver::resolve(const ParseNode &parse_tree) if (create_table_stmt->get_create_table_arg().schema_.get_part_level() == ObPartitionLevel::PARTITION_LEVEL_ONE) { OZ (create_default_partition_for_table(create_table_stmt->get_create_table_arg().schema_)); } + /* + if (OB_FAIL(ret)) { + } else if (ObExternalFileFormat::FormatType::ODPS_FORMAT != external_table_format_type_ && + OB_FAIL(add_hidden_external_table_pk_col())) { + LOG_WARN("fail to add hidden pk col for external table", K(ret)); + } + */ if (OB_FAIL(ret)) { } else if (OB_FAIL(add_hidden_external_table_pk_col())) { LOG_WARN("fail to add hidden pk col for external table", K(ret)); @@ -1045,6 +1052,8 @@ int ObCreateTableResolver::check_external_table_generated_partition_column_sanit LOG_WARN("user specified partition col expr contains no external partition pseudo column is not supported", K(ret)); } } + } else if (table_schema.is_odps_external_table()) { + // lcqlog to do check } else { bool found = false; for (int64_t i = 0; OB_SUCC(ret) && i < col_exprs.count(); i++) { @@ -3137,7 +3146,7 @@ int ObCreateTableResolver::resolve_external_table_format_early(const ParseNode * int32_t num = node->num_child_; for (int32_t i = 0; OB_SUCC(ret) && i < num; ++i) { option_node = node->children_[i]; - if (OB_NOT_NULL(option_node) && T_EXTERNAL_FILE_FORMAT == option_node->type_) { + if (OB_NOT_NULL(option_node) && (T_EXTERNAL_FILE_FORMAT == option_node->type_ || T_EXTERNAL_PROPERTIES == option_node->type_)) { ObExternalFileFormat format; for (int32_t j = 0; OB_SUCC(ret) && j < option_node->num_child_; ++j) { if (OB_NOT_NULL(option_node->children_[j]) @@ -3153,12 +3162,6 @@ int ObCreateTableResolver::resolve_external_table_format_early(const ParseNode * } } } - if (OB_SUCC(ret) && external_table_format_type_ >= ObExternalFileFormat::PARQUET_FORMAT) { - uint64_t data_version = 0; - CK (OB_NOT_NULL(session_info_)); - OZ (GET_MIN_DATA_VERSION(session_info_->get_effective_tenant_id(), data_version)); - OV (DATA_VERSION_4_3_2_0 <= data_version, OB_NOT_SUPPORTED, data_version); - } return ret; } diff --git a/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp b/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp index 002debb91..fcbcf9475 100644 --- a/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp +++ b/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp @@ -397,12 +397,12 @@ int ObCreateTableResolverBase::set_table_option_to_schema(ObTableSchema &table_s table_schema.set_lob_inrow_threshold(lob_inrow_threshold_); } } - if (OB_SUCC(ret) && table_schema.is_external_table()) { - if (table_schema.get_external_file_format().empty() - || table_schema.get_external_file_location().empty()) { + if ((table_schema.get_external_file_format().empty() + || table_schema.get_external_file_location().empty()) && + table_schema.get_external_properties().empty()) { ret = OB_NOT_SUPPORTED; - LOG_USER_ERROR(OB_NOT_SUPPORTED, "Default format or location option for external table"); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "Default properties or format or location option for external table"); } } if (OB_SUCC(ret) && auto_increment_cache_size_ != 0) { diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.cpp b/src/sql/resolver/ddl/ob_ddl_resolver.cpp index a6e495ab2..30a55ebeb 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.cpp +++ b/src/sql/resolver/ddl/ob_ddl_resolver.cpp @@ -2592,8 +2592,22 @@ int ObDDLResolver::resolve_table_option(const ParseNode *option_node, const bool } break; } + case T_EXTERNAL_PROPERTIES: case T_EXTERNAL_FILE_FORMAT: { - if (stmt::T_CREATE_TABLE != stmt_->get_stmt_type()) { + uint64_t data_version = 0; + uint64_t tenant_id = OB_INVALID_ID; + if (OB_ISNULL(session_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted null ptr", K(ret)); + } else if (FALSE_IT(tenant_id = session_info_->get_effective_tenant_id())) { + } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, data_version))) { + LOG_WARN("failed to get data version", K(ret)); + } else if (T_EXTERNAL_PROPERTIES == option_node->type_ && + data_version < DATA_VERSION_4_3_2_1) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not support odps external table under CLUSTER_VERSION_4_3_2_1", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "odps external table"); + } else if (stmt::T_CREATE_TABLE != stmt_->get_stmt_type()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid file format option", K(ret)); } else { @@ -2623,7 +2637,9 @@ int ObDDLResolver::resolve_table_option(const ParseNode *option_node, const bool LOG_USER_ERROR(OB_NOT_SUPPORTED, "format"); } // 2. resolve other format value + ObString masked_sql = params_.session_info_->get_current_query_string(); // that's create table operation stmt which has properties for (int i = 0; OB_SUCC(ret) && i < option_node->num_child_; ++i) { + ObString temp_masked_sql; if (OB_ISNULL(option_node->children_[i])) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed. get unexpected NULL ptr", K(ret), K(option_node->num_child_)); @@ -2631,11 +2647,17 @@ int ObDDLResolver::resolve_table_option(const ParseNode *option_node, const bool T_CHARSET == option_node->children_[i]->type_) { } else if (OB_FAIL(resolve_file_format(option_node->children_[i], format))) { LOG_WARN("fail to resolve file format", K(ret)); + } else if (OB_FAIL(mask_properties_sensitive_info(option_node->children_[i], masked_sql, temp_masked_sql))) { + LOG_WARN("failed to mask properties sensitive info", K(ret), K(i), K(option_node->num_child_)); + } else if (!temp_masked_sql.empty()) { + masked_sql = temp_masked_sql; } } if (OB_SUCC(ret)) { bool is_valid = true; - if (OB_FAIL(check_format_valid(format, is_valid))) { + if (ObExternalFileFormat::ODPS_FORMAT == format.format_type_ && OB_FAIL(format.odps_format_.encrypt())) { + LOG_WARN("failed to encrypt odps format", K(ret)); + } else if (OB_FAIL(check_format_valid(format, is_valid))) { LOG_WARN("check format valid failed", K(ret)); } else if (!is_valid) { ret = OB_NOT_SUPPORTED; @@ -2654,9 +2676,21 @@ int ObDDLResolver::resolve_table_option(const ParseNode *option_node, const bool } } while (OB_SUCC(ret) && pos >= buf_len); if (OB_SUCC(ret)) { - arg.schema_.set_external_file_format(ObString(pos, buf)); - LOG_DEBUG("debug external file format", - K(arg.schema_.get_external_file_format())); + if (ObExternalFileFormat::ODPS_FORMAT == format.format_type_) { + ObCreateTableStmt *create_table_stmt = static_cast(stmt_); + if (OB_ISNULL(create_table_stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted null ptr", K(ret)); + } else { + create_table_stmt->set_masked_sql(masked_sql); + arg.schema_.set_external_properties(ObString(pos, buf)); + } + + } else { + arg.schema_.set_external_file_format(ObString(pos, buf)); + LOG_DEBUG("debug external file format", + K(arg.schema_.get_external_file_format())); + } } } } @@ -2875,6 +2909,46 @@ int ObDDLResolver::resolve_file_format(const ParseNode *node, ObExternalFileForm LOG_WARN("invalid parse node", K(ret)); } else { switch (node->type_) { + case ObItemType::T_ACCESSTYPE: { + format.odps_format_.access_type_ = ObString(node->children_[0]->str_len_, node->children_[0]->str_value_).trim_space_only(); + break; + } + case ObItemType::T_ACCESSID: { + format.odps_format_.access_id_ = ObString(node->children_[0]->str_len_, node->children_[0]->str_value_).trim_space_only(); + break; + } + case ObItemType::T_ACCESSKEY: { + format.odps_format_.access_key_ = ObString(node->children_[0]->str_len_, node->children_[0]->str_value_).trim_space_only(); + break; + } + case ObItemType::T_STSTOKEN: { + format.odps_format_.sts_token_ = ObString(node->children_[0]->str_len_, node->children_[0]->str_value_).trim_space_only(); + break; + } + case ObItemType::T_ENDPOINT: { + format.odps_format_.endpoint_ = ObString(node->children_[0]->str_len_, node->children_[0]->str_value_).trim_space_only(); + break; + } + case ObItemType::T_PROJECT: { + format.odps_format_.project_ = ObString(node->children_[0]->str_len_, node->children_[0]->str_value_).trim_space_only(); + break; + } + case ObItemType::T_SCHEMA: { + format.odps_format_.schema_ = ObString(node->children_[0]->str_len_, node->children_[0]->str_value_).trim_space_only(); + break; + } + case ObItemType::T_TABLE: { + format.odps_format_.table_ = ObString(node->children_[0]->str_len_, node->children_[0]->str_value_).trim_space_only(); + break; + } + case ObItemType::T_QUOTA: { + format.odps_format_.quota_ = ObString(node->children_[0]->str_len_, node->children_[0]->str_value_).trim_space_only(); + break; + } + case ObItemType::T_COMPRESSION_CODE: { + format.odps_format_.compression_code_ = ObString(node->children_[0]->str_len_, node->children_[0]->str_value_).trim_space_only(); + break; + } case T_EXTERNAL_FILE_FORMAT_TYPE: { ObString string_v = ObString(node->children_[0]->str_len_, node->children_[0]->str_value_).trim_space_only(); for (int i = 0; i < ObExternalFileFormat::MAX_FORMAT; i++) { @@ -3030,6 +3104,31 @@ int ObDDLResolver::resolve_file_format(const ParseNode *node, ObExternalFileForm return ret; } +int ObDDLResolver::mask_properties_sensitive_info(const ParseNode *node, ObString &ddl_sql, ObString &masked_sql) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(node) || node->num_child_ != 1 || OB_ISNULL(node->children_[0]) || + OB_ISNULL(params_.session_info_) || OB_ISNULL(params_.expr_factory_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid parse node", K(ret)); + } else { + switch (node->type_) { + case ObItemType::T_ENDPOINT: + case ObItemType::T_STSTOKEN: + case ObItemType::T_ACCESSKEY: + case ObItemType::T_ACCESSID: { + if (OB_FAIL(ObDCLResolver::mask_password_for_passwd_node(params_.allocator_, ddl_sql, node->children_[0], masked_sql, true))) { + LOG_WARN("fail to gen masked sql", K(ret)); + } + break; + } + default: { + // do nothing + } + } + } + return ret; +} int ObDDLResolver::resolve_column_definition_ref(ObColumnSchemaV2 &column, ParseNode *node /* column_definition_def */, @@ -3088,30 +3187,34 @@ int ObDDLResolver::resolve_column_definition_ref(ObColumnSchemaV2 &column, int ObDDLResolver::check_format_valid(const ObExternalFileFormat &format, bool &is_valid) { int ret = OB_SUCCESS; - if (!format.csv_format_.line_term_str_.empty() && !format.csv_format_.field_term_str_.empty()) { - if (0 == MEMCMP(format.csv_format_.field_term_str_.ptr(), - format.csv_format_.line_term_str_.ptr(), - std::min(format.csv_format_.field_term_str_.length(), - format.csv_format_.line_term_str_.length()))) { - is_valid = false; - LOG_USER_WARN(OB_NOT_SUPPORTED, - "LINE_DELIMITER or FIELD_DELIMITER cannot be a substring of the delimiter for the other"); - LOG_WARN("LINE_DELIMITER or FIELD_DELIMITER cann't be a substring of the other's", K(ret), - K(format.csv_format_.line_term_str_), K(format.csv_format_.field_term_str_)); + if (ObExternalFileFormat::ODPS_FORMAT == format.format_type_) { + is_valid = true; + } else { + if (!format.csv_format_.line_term_str_.empty() && !format.csv_format_.field_term_str_.empty()) { + if (0 == MEMCMP(format.csv_format_.field_term_str_.ptr(), + format.csv_format_.line_term_str_.ptr(), + std::min(format.csv_format_.field_term_str_.length(), + format.csv_format_.line_term_str_.length()))) { + is_valid = false; + LOG_USER_WARN(OB_NOT_SUPPORTED, + "LINE_DELIMITER or FIELD_DELIMITER cannot be a substring of the delimiter for the other"); + LOG_WARN("LINE_DELIMITER or FIELD_DELIMITER cann't be a substring of the other's", K(ret), + K(format.csv_format_.line_term_str_), K(format.csv_format_.field_term_str_)); + } + } + if (OB_SUCC(ret)) { + if (!format.csv_format_.line_term_str_.empty() + && (format.csv_format_.line_term_str_[0] == format.csv_format_.field_escaped_char_ + || format.csv_format_.line_term_str_[0] == format.csv_format_.field_enclosed_char_)) { + ret = OB_WRONG_FIELD_TERMINATORS; + LOG_WARN("invalid line terminator", K(ret)); + } else if (!format.csv_format_.field_term_str_.empty() + && (format.csv_format_.field_term_str_[0] == format.csv_format_.field_escaped_char_ + || format.csv_format_.field_term_str_[0] == format.csv_format_.field_enclosed_char_)) { + ret = OB_WRONG_FIELD_TERMINATORS; + LOG_WARN("invalid field terminator", K(ret)); + } } - } - if (OB_SUCC(ret)) { - if (!format.csv_format_.line_term_str_.empty() - && (format.csv_format_.line_term_str_[0] == format.csv_format_.field_escaped_char_ - || format.csv_format_.line_term_str_[0] == format.csv_format_.field_enclosed_char_)) { - ret = OB_WRONG_FIELD_TERMINATORS; - LOG_WARN("invalid line terminator", K(ret)); - } else if (!format.csv_format_.field_term_str_.empty() - && (format.csv_format_.field_term_str_[0] == format.csv_format_.field_escaped_char_ - || format.csv_format_.field_term_str_[0] == format.csv_format_.field_enclosed_char_)) { - ret = OB_WRONG_FIELD_TERMINATORS; - LOG_WARN("invalid field terminator", K(ret)); - } } return ret; } diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.h b/src/sql/resolver/ddl/ob_ddl_resolver.h index 09abb31eb..9ee2861e6 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.h +++ b/src/sql/resolver/ddl/ob_ddl_resolver.h @@ -959,6 +959,7 @@ protected: share::schema::ObTableSchema &table_schema); int resolve_file_format(const ParseNode *node, ObExternalFileFormat &format); + int mask_properties_sensitive_info(const ParseNode *node, ObString &ddl_sql, ObString &masked_sql); int check_format_valid(const ObExternalFileFormat &format, bool &is_valid); diff --git a/src/sql/resolver/dml/ob_default_value_utils.cpp b/src/sql/resolver/dml/ob_default_value_utils.cpp index eb8cc3d0f..d1dc046af 100644 --- a/src/sql/resolver/dml/ob_default_value_utils.cpp +++ b/src/sql/resolver/dml/ob_default_value_utils.cpp @@ -1104,12 +1104,19 @@ int ObDefaultValueUtils::build_nullable_expr(const ColumnItem *column, ObRawExpr int ObDefaultValueUtils::build_default_expr_for_generated_column(const ColumnItem &column, ObRawExpr *&expr) { int ret = OB_SUCCESS; - if (OB_ISNULL(column.expr_) || OB_ISNULL(stmt_) || OB_ISNULL(params_) || OB_ISNULL(params_->expr_factory_)) { + bool contain = false; + if (OB_ISNULL(column.expr_) || OB_ISNULL(stmt_) || OB_ISNULL(params_) + || OB_ISNULL(params_->expr_factory_) || OB_ISNULL(column.expr_->get_dependant_expr())) { ret = OB_NOT_INIT; LOG_WARN("column expr is null", K_(column.expr), K_(stmt)); - } else if (OB_FAIL(ObDMLResolver::copy_schema_expr(*params_->expr_factory_, - column.expr_->get_dependant_expr(), - expr))) { + } else if (OB_FAIL(ObResolverUtils::cnt_external_pseudo_column(*column.expr_->get_dependant_expr(), contain))) { + LOG_WARN("failed to check if contain external pseudo column", K(ret)); + // 外表生成列包含伪列 默认值为null + } else if (contain && OB_FAIL(ObRawExprUtils::build_null_expr(*params_->expr_factory_, expr))) { + LOG_WARN("fail to build null expr", K(ret)); + } else if (!contain && OB_FAIL(ObDMLResolver::copy_schema_expr(*params_->expr_factory_, + column.expr_->get_dependant_expr(), + expr))) { LOG_WARN("failed to copy dependant expr", K(ret)); } return ret; diff --git a/src/sql/resolver/dml/ob_del_upd_resolver.cpp b/src/sql/resolver/dml/ob_del_upd_resolver.cpp index f7c423de4..746e69c51 100644 --- a/src/sql/resolver/dml/ob_del_upd_resolver.cpp +++ b/src/sql/resolver/dml/ob_del_upd_resolver.cpp @@ -25,6 +25,7 @@ #include "pl/ob_pl_resolver.h" #include "sql/parser/parse_malloc.h" #include "sql/resolver/dml/ob_merge_resolver.h" +#include "share/external_table/ob_external_table_utils.h" namespace oceanbase { @@ -1858,6 +1859,8 @@ int ObDelUpdResolver::add_all_columns_to_stmt(const TableItem &table_item, if (OB_ISNULL(column)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid column schema", K(column)); + } else if (schema::EXTERNAL_TABLE == base_table_item.table_type_ + && ObExternalTableUtils::is_skipped_insert_column(*column)) { } else if (OB_FAIL(add_column_to_stmt(table_item, *column, column_exprs))) { LOG_WARN("add column item to stmt failed", K(ret)); } @@ -1955,6 +1958,11 @@ int ObDelUpdResolver::add_all_rowkey_columns_to_stmt(const TableItem &table_item LOG_WARN("get rowkey info failed", K(ret), K(i), K(rowkey_info)); } else if (OB_FAIL(get_column_schema(base_table_item.ref_id_, rowkey_column_id, column_schema, true, base_table_item.is_link_table()))) { LOG_WARN("get column schema failed", K(rowkey_column_id)); + } else if (OB_ISNULL(column_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid column schema", K(column_schema)); + } else if (schema::EXTERNAL_TABLE == base_table_item.table_type_ + && ObExternalTableUtils::is_skipped_insert_column(*column_schema)) { } else if (OB_FAIL(add_column_to_stmt(table_item, *column_schema, column_exprs))) { LOG_WARN("add column to stmt failed", K(ret), K(table_item)); } @@ -3215,6 +3223,7 @@ int ObDelUpdResolver::resolve_insert_columns(const ParseNode *node, int ret = OB_SUCCESS; TableItem *table_item = NULL; ObDelUpdStmt *del_upd_stmt = get_del_upd_stmt(); + bool is_ext_part_column = false; if (OB_ISNULL(del_upd_stmt) || OB_ISNULL(session_info_) || OB_ISNULL(schema_checker_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid insert stmt", K(del_upd_stmt), K_(session_info), K_(schema_checker)); @@ -3269,6 +3278,13 @@ int ObDelUpdResolver::resolve_insert_columns(const ParseNode *node, } else if (!session_info_->get_ddl_info().is_ddl() && OB_HIDDEN_SESS_CREATE_TIME_COLUMN_ID == column_expr->get_column_id()) { ret = OB_NOT_SUPPORTED; LOG_USER_ERROR(OB_NOT_SUPPORTED, "specify __sess_create_time value"); + } else if (schema::EXTERNAL_TABLE == table_item->table_type_ + && OB_FAIL(is_external_table_partition_column(*table_item, column_expr->get_column_id(), is_ext_part_column))) { + LOG_WARN("failed to check external table column", K(ret)); + } else if (is_ext_part_column) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("column list should not contain external table partition column", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "contain external table partition column in insert column list"); } else if (OB_FAIL(mock_values_column_ref(column_expr))) { LOG_WARN("mock values column reference failed", K(ret)); } @@ -3310,7 +3326,10 @@ int ObDelUpdResolver::resolve_insert_columns(const ParseNode *node, } else if (is_duplicate) { ret = OB_ERR_FIELD_SPECIFIED_TWICE; LOG_USER_ERROR(OB_ERR_FIELD_SPECIFIED_TWICE, to_cstring(column_items.at(i).column_name_)); - } else if (OB_FAIL(mock_values_column_ref(column_items.at(i).expr_))) { + } else if (schema::EXTERNAL_TABLE == table_item->table_type_ + && OB_FAIL(is_external_table_partition_column(*table_item, column_id, is_ext_part_column))) { + LOG_WARN("failed to check external table column", K(ret)); + } else if (!is_ext_part_column && OB_FAIL(mock_values_column_ref(column_items.at(i).expr_))) { LOG_WARN("mock values column reference failed", K(ret)); } } @@ -3344,7 +3363,31 @@ int ObDelUpdResolver::resolve_insert_columns(const ParseNode *node, } return ret; } - +int ObDelUpdResolver::is_external_table_partition_column(const TableItem &table_item, + uint64_t column_id, + bool &is_part_column) +{ + int ret = OB_SUCCESS; + const ObTableSchema* table_schema = NULL; + const ObColumnSchemaV2 *column_schema = NULL; + is_part_column = false; + if (OB_ISNULL(session_info_) || OB_ISNULL(schema_checker_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (OB_FAIL(schema_checker_->get_table_schema(session_info_->get_effective_tenant_id(), + table_item.ref_id_, + table_schema, + table_item.is_link_table()))) { + LOG_WARN("fail to get table schema", K(ret), K(table_item.ref_id_)); + } else if (OB_ISNULL(table_schema) + || OB_ISNULL(column_schema = table_schema->get_column_schema(column_id))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column schema is null", K(ret)); + } else { + is_part_column = column_schema->is_tbl_part_key_column(); + } + return ret; +} int ObDelUpdResolver::resolve_insert_values(const ParseNode *node, ObInsertTableInfo& table_info, ObIArray &label_se_columns) @@ -3356,6 +3399,7 @@ int ObDelUpdResolver::resolve_insert_values(const ParseNode *node, uint64_t value_count = OB_INVALID_ID; bool is_all_default = false; bool is_update_view = false; + TableItem* table_item = NULL; if (OB_ISNULL(del_upd_stmt) || OB_ISNULL(node) || OB_ISNULL(session_info_) || T_VALUE_LIST != node->type_ || OB_ISNULL(node->children_) || OB_ISNULL(del_upd_stmt->get_query_ctx())) { ret = OB_INVALID_ARGUMENT; @@ -3376,7 +3420,6 @@ int ObDelUpdResolver::resolve_insert_values(const ParseNode *node, LOG_WARN("reserve memory fail", K(ret)); } if (OB_SUCC(ret)) { - TableItem* table_item = NULL; if (OB_FAIL(check_need_match_all_params(table_info.values_desc_, del_upd_stmt->get_query_ctx()->need_match_all_params_))) { LOG_WARN("check need match all params failed", K(ret)); @@ -3491,7 +3534,7 @@ int ObDelUpdResolver::resolve_insert_values(const ParseNode *node, } else if (OB_FAIL(check_basic_column_generated(column_expr, del_upd_stmt, is_generated_column))) { LOG_WARN("check column generated failed", K(ret)); - } else if (is_generated_column) { + } else if (is_generated_column && schema::EXTERNAL_TABLE != table_item->table_type_) { ret = OB_NON_DEFAULT_VALUE_FOR_GENERATED_COLUMN; if (!is_oracle_mode()) { ColumnItem *orig_col_item = NULL; diff --git a/src/sql/resolver/dml/ob_del_upd_resolver.h b/src/sql/resolver/dml/ob_del_upd_resolver.h index aada09ed1..32c86305a 100644 --- a/src/sql/resolver/dml/ob_del_upd_resolver.h +++ b/src/sql/resolver/dml/ob_del_upd_resolver.h @@ -284,6 +284,9 @@ protected: int mark_json_partial_update_flag(const ObColumnRefRawExpr *ref_expr, ObRawExpr *expr, int depth, bool &allow_json_partial_update); int add_select_item_func(ObSelectStmt &select_stmt, ColumnItem &col); int select_items_is_pk(const ObSelectStmt& select_stmt, bool &has_pk); + int is_external_table_partition_column(const TableItem &table_item, + uint64_t column_id, + bool &is_part_column); private: common::hash::ObPlacementHashSet insert_column_ids_; diff --git a/src/sql/resolver/dml/ob_dml_resolver.cpp b/src/sql/resolver/dml/ob_dml_resolver.cpp index 618b9852f..af4b54439 100755 --- a/src/sql/resolver/dml/ob_dml_resolver.cpp +++ b/src/sql/resolver/dml/ob_dml_resolver.cpp @@ -7622,46 +7622,52 @@ int ObDMLResolver::resolve_partitions(const ParseNode *part_node, int ret = OB_SUCCESS; if (NULL != part_node) { OB_ASSERT(1 == part_node->num_child_ && part_node->children_[0]->num_child_ > 0); - const ParseNode *name_list = part_node->children_[0]; - ObString partition_name; - ObSEArray part_ids; - ObSEArray part_names; - for (int i = 0; OB_SUCC(ret) && i < name_list->num_child_; i++) { - ObSEArray partition_ids; - partition_name.assign_ptr(name_list->children_[i]->str_value_, - static_cast(name_list->children_[i]->str_len_)); - //here just conver partition_name to its lowercase - ObCharset::casedn(CS_TYPE_UTF8MB4_GENERAL_CI, partition_name); - ObPartGetter part_getter(table_schema); - if (T_USE_PARTITION == part_node->type_) { - if (OB_FAIL(part_getter.get_part_ids(partition_name, partition_ids))) { - LOG_WARN("failed to get part ids", K(ret), K(partition_name)); - if (OB_UNKNOWN_PARTITION == ret && lib::is_mysql_mode()) { - LOG_USER_ERROR(OB_UNKNOWN_PARTITION, partition_name.length(), partition_name.ptr(), - table_schema.get_table_name_str().length(), - table_schema.get_table_name_str().ptr()); + if (T_NAME_LIST == part_node->children_[0]->type_) { + const ParseNode *name_list = part_node->children_[0]; + ObString partition_name; + ObSEArray part_ids; + ObSEArray part_names; + for (int i = 0; OB_SUCC(ret) && i < name_list->num_child_; i++) { + ObSEArray partition_ids; + partition_name.assign_ptr(name_list->children_[i]->str_value_, + static_cast(name_list->children_[i]->str_len_)); + //here just conver partition_name to its lowercase + ObCharset::casedn(CS_TYPE_UTF8MB4_GENERAL_CI, partition_name); + ObPartGetter part_getter(table_schema); + if (T_USE_PARTITION == part_node->type_) { + if (OB_FAIL(part_getter.get_part_ids(partition_name, partition_ids))) { + LOG_WARN("failed to get part ids", K(ret), K(partition_name)); + if (OB_UNKNOWN_PARTITION == ret && lib::is_mysql_mode()) { + LOG_USER_ERROR(OB_UNKNOWN_PARTITION, partition_name.length(), partition_name.ptr(), + table_schema.get_table_name_str().length(), + table_schema.get_table_name_str().ptr()); + } + } + } else if (OB_FAIL(part_getter.get_subpart_ids(partition_name, partition_ids))) { + LOG_WARN("failed to get subpart ids", K(ret), K(partition_name)); + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(append_array_no_dup(part_ids, partition_ids))) { + LOG_WARN("Push partition id error", K(ret)); + } else if (OB_FAIL(part_names.push_back(partition_name))) { + LOG_WARN("failed to push back partition name", K(ret)); + } else { + LOG_INFO("part ids", K(partition_name), K(partition_ids)); } } - } else if (OB_FAIL(part_getter.get_subpart_ids(partition_name, partition_ids))) { - LOG_WARN("failed to get subpart ids", K(ret), K(partition_name)); } - if (OB_SUCC(ret)) { - if (OB_FAIL(append_array_no_dup(part_ids, partition_ids))) { - LOG_WARN("Push partition id error", K(ret)); - } else if (OB_FAIL(part_names.push_back(partition_name))) { - LOG_WARN("failed to push back partition name", K(ret)); - } else { - LOG_INFO("part ids", K(partition_name), K(partition_ids)); + if (OB_FAIL(table_item.part_ids_.assign(part_ids))) { + LOG_WARN("failed to assign part ids", K(ret)); + } else if (OB_FAIL(table_item.part_names_.assign(part_names))) { + LOG_WARN("failed to assign part names", K(ret)); } } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(table_item.part_ids_.assign(part_ids))) { - LOG_WARN("failed to assign part ids", K(ret)); - } else if (OB_FAIL(table_item.part_names_.assign(part_names))) { - LOG_WARN("failed to assign part names", K(ret)); - } + } else if (schema::EXTERNAL_TABLE == table_item.table_type_ + && T_EXTERNAL_TABLE_PARTITION == part_node->children_[0]->type_) { + table_item.external_table_partition_.assign_ptr(part_node->children_[0]->str_value_, + part_node->children_[0]->str_len_); } } return ret; @@ -8245,7 +8251,10 @@ int ObDMLResolver::resolve_external_table_generated_column( } } else { ObExternalFileFormat format; - if (OB_FAIL(format.load_from_string(table_schema->get_external_file_format(), *params_.allocator_))) { + const ObString &format_or_properties = table_schema->get_external_file_format().empty() ? + table_schema->get_external_properties() : + table_schema->get_external_file_format(); + if (OB_FAIL(format.load_from_string(format_or_properties, *params_.allocator_))) { LOG_WARN("load from string failed", K(ret)); } else if (format.format_type_ == ObExternalFileFormat::ORC_FORMAT && lib::is_oracle_mode()) { ret = OB_NOT_SUPPORTED; @@ -8274,6 +8283,19 @@ int ObDMLResolver::resolve_external_table_generated_column( LOG_WARN("fail to build external table file column expr", K(ret)); } } + } else if (ObExternalFileFormat::ODPS_FORMAT == format.format_type_) { + if (OB_FAIL(ObResolverUtils::calc_file_column_idx(col.col_name_, file_column_idx))) { + LOG_WARN("fail to calc file column idx", K(ret)); + } else if (nullptr == (real_ref_expr = ObResolverUtils::find_file_column_expr( + pseudo_external_file_col_exprs_, table_item.table_id_, file_column_idx, col.col_name_))) { + if (OB_FAIL(ObResolverUtils::build_file_column_expr_for_odps( + *params_.expr_factory_, *params_.session_info_, + table_item.table_id_, table_item.table_name_, + col.col_name_, file_column_idx, column_schema, + real_ref_expr))) { + LOG_WARN("fail to build external table file column expr", K(ret)); + } + } } else if (ObExternalFileFormat::PARQUET_FORMAT == format.format_type_ || ObExternalFileFormat::ORC_FORMAT == format.format_type_ ) { ObRawExpr *cast_expr = NULL; @@ -8528,7 +8550,8 @@ int ObDMLResolver::resolve_generated_column_expr(const ObString &expr_str, LOG_WARN("build padding expr for self failed", K(ret)); } else if (OB_FAIL(ref_expr->formalize_with_local_vars(session_info, &local_vars, var_array_idx))) { LOG_WARN("formailize column reference expr failed", K(ret)); - } else if (ObRawExprUtils::need_column_conv(column.get_result_type(), *ref_expr, true)) { + } else if (table_schema->is_external_table() + || ObRawExprUtils::need_column_conv(column.get_result_type(), *ref_expr, true)) { if (OB_FAIL(ObRawExprUtils::build_column_conv_expr(*expr_factory, *allocator_, column, ref_expr, session_info, used_for_generated_column, diff --git a/src/sql/resolver/dml/ob_dml_stmt.cpp b/src/sql/resolver/dml/ob_dml_stmt.cpp index 27ddf8255..9eef0e01b 100644 --- a/src/sql/resolver/dml/ob_dml_stmt.cpp +++ b/src/sql/resolver/dml/ob_dml_stmt.cpp @@ -270,6 +270,8 @@ int TableItem::deep_copy(ObIRawExprCopier &expr_copier, ddl_schema_version_ = other.ddl_schema_version_; ddl_table_id_ = other.ddl_table_id_; ref_query_ = other.ref_query_; + //external table + external_table_partition_ = other.external_table_partition_; SampleInfo *buf = NULL; if (is_json_table() && OB_FAIL(deep_copy_json_table_def(*other.json_table_def_, expr_copier, allocator))) { diff --git a/src/sql/resolver/dml/ob_dml_stmt.h b/src/sql/resolver/dml/ob_dml_stmt.h index bbe640f41..2757506d0 100644 --- a/src/sql/resolver/dml/ob_dml_stmt.h +++ b/src/sql/resolver/dml/ob_dml_stmt.h @@ -290,10 +290,8 @@ struct TableItem K_(is_view_table), K_(part_ids), K_(part_names), K_(cte_type), KPC_(function_table_expr), K_(flashback_query_type), KPC_(flashback_query_expr), K_(table_type), - K_(exec_params), - KPC_(sample_info), - K_(mview_id), - K_(need_expand_rt_mv)); + K_(exec_params), KPC_(sample_info), K_(mview_id), K_(need_expand_rt_mv), + K_(external_table_partition)); enum TableType { @@ -307,7 +305,6 @@ struct TableItem TEMP_TABLE, LINK_TABLE, JSON_TABLE, - EXTERNAL_TABLE, VALUES_TABLE, LATERAL_TABLE, }; @@ -442,6 +439,8 @@ struct TableItem ObJsonTableDef *json_table_def_; // values table ObValuesTableDef *values_table_def_; + // external table + common::ObString external_table_partition_; // sample scan infos SampleInfo *sample_info_; }; diff --git a/src/sql/resolver/dml/ob_insert_resolver.cpp b/src/sql/resolver/dml/ob_insert_resolver.cpp index a6ad48e3c..573d8a8d1 100644 --- a/src/sql/resolver/dml/ob_insert_resolver.cpp +++ b/src/sql/resolver/dml/ob_insert_resolver.cpp @@ -181,6 +181,9 @@ int ObInsertResolver::resolve(const ParseNode &parse_tree) LOG_WARN("view not insertable", K(ret)); } } + if (OB_SUCC(ret) && OB_FAIL(check_insert_into_external_table())) { + LOG_WARN("check insert into external table failed", K(ret)); + } return ret; } @@ -980,7 +983,11 @@ int ObInsertResolver::check_insert_select_field(ObInsertStmt &insert_stmt, bool is_generated_column = false; const ObIArray &values_desc = insert_stmt.get_values_desc(); ObSelectStmt *ref_stmt = NULL; - if (OB_ISNULL(session_info_)) { + TableItem *insert_table = NULL; + if (OB_ISNULL(insert_table = insert_stmt.get_table_item_by_id(insert_stmt.get_insert_table_info().table_id_))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("insert target table is unexpected null", K(ret)); + } else if (OB_ISNULL(session_info_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid session_info_", K(ret)); } else if (values_desc.count() != select_stmt.get_select_item_size()) { @@ -1006,7 +1013,7 @@ int ObInsertResolver::check_insert_select_field(ObInsertStmt &insert_stmt, &insert_stmt, is_generated_column))) { LOG_WARN("check basic column generated failed", K(ret)); - } else if (is_generated_column) { + } else if (is_generated_column && schema::EXTERNAL_TABLE != insert_table->table_type_) { if (select_stmt.get_table_size() == 1 && select_stmt.get_table_item(0) != NULL && select_stmt.get_table_item(0)->is_values_table()) { @@ -1167,6 +1174,31 @@ int ObInsertResolver::check_returning_validity() return ret; } +int ObInsertResolver::check_insert_into_external_table() +{ + int ret = OB_SUCCESS; + ObInsertStmt *insert_stmt = get_insert_stmt(); + TableItem *table = NULL; + if (OB_ISNULL(insert_stmt) || insert_stmt->get_table_items().empty() + || OB_ISNULL(table = insert_stmt->get_table_item(0))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid stmt", K(ret), K(insert_stmt)); + } else if (schema::EXTERNAL_TABLE != table->table_type_) { + // do nothing + } else if (GET_MIN_CLUSTER_VERSION() < CLUSTER_VERSION_4_3_2_1) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not support to insert into external table during updating", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "insert into external table during updating"); + } else if (!insert_stmt->value_from_select() || insert_stmt->is_replace() + || insert_stmt->is_ignore() || insert_stmt->is_returning() + || insert_stmt->is_insert_up()) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not support insert into external table with values, replace, ignore, returning, update", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "insert into external table with values, replace, ignore, returning, update"); + } + return ret; +} + int ObInsertResolver::resolve_column_ref_expr(const ObQualifiedName &q_name, ObRawExpr *&real_ref_expr) { int ret = OB_SUCCESS; diff --git a/src/sql/resolver/dml/ob_insert_resolver.h b/src/sql/resolver/dml/ob_insert_resolver.h index 801fc9b9b..a74f39ef4 100644 --- a/src/sql/resolver/dml/ob_insert_resolver.h +++ b/src/sql/resolver/dml/ob_insert_resolver.h @@ -113,6 +113,7 @@ private: return sub_select_resolver_; } int try_expand_returning_exprs(); + int check_insert_into_external_table(); DISALLOW_COPY_AND_ASSIGN(ObInsertResolver); private: int64_t row_count_; diff --git a/src/sql/resolver/dml/ob_select_resolver.cpp b/src/sql/resolver/dml/ob_select_resolver.cpp index 838df5e76..604b93d1f 100644 --- a/src/sql/resolver/dml/ob_select_resolver.cpp +++ b/src/sql/resolver/dml/ob_select_resolver.cpp @@ -5122,9 +5122,17 @@ int ObSelectResolver::resolve_into_file_node(const ParseNode *list_node, ObSelec into_item.is_single_ = node->children_[0]->value_; } } else if (T_MAX_FILE_SIZE == node->type_) { - if (OB_FAIL(resolve_max_file_size_node(node, into_item))) { + if (OB_FAIL(resolve_size_node(node, into_item))) { LOG_WARN("failed to resolve max file size", K(ret)); } + } else if (T_BUFFER_SIZE == node->type_) { + if (GET_MIN_CLUSTER_VERSION() < CLUSTER_VERSION_4_3_2_1) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not support to set buffer size during updating", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "set buffer size during updating"); + } else if (OB_FAIL(resolve_size_node(node, into_item))) { + LOG_WARN("failed to resolve buffer size", K(ret)); + } } else { ret = OB_ERR_PARSE_SQL; LOG_WARN("child of into file node has wrong type", K(ret)); @@ -5134,13 +5142,48 @@ int ObSelectResolver::resolve_into_file_node(const ParseNode *list_node, ObSelec return ret; } -int ObSelectResolver::resolve_max_file_size_node(const ParseNode *file_size_node, ObSelectIntoItem &into_item) +int ObSelectResolver::resolve_file_partition_node(const ParseNode *node, ObSelectIntoItem &into_item) +{ + int ret = OB_SUCCESS; + ObRawExpr *expr = NULL; + ObSQLSessionInfo *session_info = params_.session_info_; + ObArray columns; + if (OB_ISNULL(node) || OB_ISNULL(session_info) || T_PARTITION_EXPR != node->type_ + || node->num_child_ != 1 || OB_ISNULL(node->children_[0])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("partition node is null", K(ret)); + } else if (OB_FAIL(resolve_sql_expr(*(node->children_[0]), expr, &columns))) { + LOG_WARN("fail to resolve const expr", K(ret)); + } else if (OB_ISNULL(expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expr is null", K(ret), K(node->children_[0]->type_)); + } else if (OB_FAIL(expr->formalize(session_info))) { + LOG_WARN("failed to formalize expr", K(ret), K(*expr)); + } else if (expr->has_flag(CNT_SUB_QUERY) || expr->has_flag(CNT_AGG) + || expr->has_flag(CNT_WINDOW_FUNC)|| expr->has_flag(CNT_PL_UDF) + || expr->has_flag(CNT_SO_UDF) || expr->has_flag(CNT_MATCH_EXPR)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("partition expr should not contain subquery, aggregate, udf or match expr", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "partition expr contains subquery, aggregate, udf or match expr"); + } else if (ObVarcharType != expr->get_result_type().get_type() + && ObCharType != expr->get_result_type().get_type()) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("partition expr should be char or varchar", K(ret), K(expr->get_result_type().get_type())); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "partition by expr whose result type is not char or varchar"); + } else { + into_item.file_partition_expr_ = expr; + } + return ret; +} + +int ObSelectResolver::resolve_size_node(const ParseNode *file_size_node, ObSelectIntoItem &into_item) { int ret = OB_SUCCESS; ParseNode *child = NULL; int64_t parse_int_value = 0; - if (OB_ISNULL(file_size_node) || T_MAX_FILE_SIZE != file_size_node->type_ - || file_size_node->num_child_ != 1 || OB_ISNULL(child = file_size_node->children_[0])) { + if (OB_ISNULL(file_size_node) || file_size_node->num_child_ != 1 + || OB_ISNULL(child = file_size_node->children_[0]) + || (T_MAX_FILE_SIZE != file_size_node->type_ && T_BUFFER_SIZE != file_size_node->type_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected file size node", K(ret)); } else if (T_INT == child->type_) { @@ -5154,11 +5197,13 @@ int ObSelectResolver::resolve_max_file_size_node(const ParseNode *file_size_node LOG_WARN("child of max file size node has wrong type", K(ret)); } if (OB_FAIL(ret)) { - } else if (OB_UNLIKELY(parse_int_value <= 0)) { + } else if (OB_UNLIKELY(parse_int_value < 0)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("file size value should be positive", K(ret), K(parse_int_value)); - } else { + LOG_WARN("size should not be smaller than 0", K(ret), K(parse_int_value)); + } else if (T_MAX_FILE_SIZE == file_size_node->type_) { into_item.max_file_size_ = parse_int_value; + } else if (T_BUFFER_SIZE == file_size_node->type_) { + into_item.buffer_size_ = parse_int_value; } return ret; } @@ -5212,11 +5257,22 @@ int ObSelectResolver::resolve_into_clause(const ParseNode *node) new(into_item) ObSelectIntoItem(); into_item->into_type_ = node->type_; if (T_INTO_OUTFILE == node->type_) { // into outfile - if (OB_FAIL(resolve_into_const_node(node->children_[0], into_item->outfile_name_))) { //name + if (node->num_child_ > 0 + && OB_FAIL(resolve_into_const_node(node->children_[0], into_item->outfile_name_))) { //name LOG_WARN("resolve into outfile name failed", K(ret)); - } else if (NULL != node->children_[1]) { // charset + } + if (OB_SUCC(ret) && node->num_child_ > 1 && NULL != node->children_[1]) { // partition by + if (GET_MIN_CLUSTER_VERSION() < CLUSTER_VERSION_4_3_2_1) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not support to use file partition option during updating", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "use file partition option during updating"); + } else if (OB_FAIL(resolve_file_partition_node(node->children_[1], *into_item))) { + LOG_WARN("resolve file partition node failed", K(ret)); + } + } + if (OB_SUCC(ret) && node->num_child_ > 2 && NULL != node->children_[2]) { // charset ObCharsetType charset_type = CHARSET_INVALID; - ObString charset(node->children_[1]->str_len_, node->children_[1]->str_value_); + ObString charset(node->children_[2]->str_len_, node->children_[2]->str_value_); if (CHARSET_INVALID == (charset_type = ObCharset::charset_type(charset.trim()))) { ret = OB_ERR_UNKNOWN_CHARSET; LOG_USER_ERROR(OB_ERR_UNKNOWN_CHARSET, charset.length(), charset.ptr()); @@ -5228,25 +5284,31 @@ int ObSelectResolver::resolve_into_clause(const ParseNode *node) into_item->cs_type_ = ObCharset::get_default_collation(charset_type); } } - if (OB_SUCC(ret) && NULL != node->children_[2]) { //field - if (OB_FAIL(resolve_into_field_node(node->children_[2], *into_item))) { + if (OB_SUCC(ret) && node->num_child_ > 3 && NULL != node->children_[3]) { //field + if (OB_FAIL(resolve_into_field_node(node->children_[3], *into_item))) { LOG_WARN("reosolve into field node failed", K(ret)); } } - if (OB_SUCC(ret) && NULL != node->children_[3]) { // line - if (OB_FAIL(resolve_into_line_node(node->children_[3], *into_item))) { + if (OB_SUCC(ret) && node->num_child_ > 4 && NULL != node->children_[4]) { // line + if (OB_FAIL(resolve_into_line_node(node->children_[4], *into_item))) { LOG_WARN("reosolve into line node failed", K(ret)); } } - if (OB_SUCC(ret) && NULL != node->children_[4]) { // file: single & max_file_size + // file: single, max_file_size, buffer_size + if (OB_SUCC(ret) && node->num_child_ > 5 && NULL != node->children_[5]) { if (GET_MIN_CLUSTER_VERSION() < CLUSTER_VERSION_4_3_1_0) { ret = OB_NOT_SUPPORTED; LOG_WARN("not support to use file option during updating", K(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "use file option during updating"); - } else if (OB_FAIL(resolve_into_file_node(node->children_[4], *into_item))) { + } else if (OB_FAIL(resolve_into_file_node(node->children_[5], *into_item))) { LOG_WARN("reosolve into line node failed", K(ret)); } } + if (OB_SUCC(ret) && into_item->is_single_ && NULL != into_item->file_partition_expr_) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not support to use file partition option when single is true", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "use file partition option when single is true"); + } } else if (T_INTO_DUMPFILE == node->type_) { // into dumpfile if (OB_FAIL(resolve_into_const_node(node->children_[0], into_item->outfile_name_))) { LOG_WARN("resolve into outfile name failed", K(ret)); diff --git a/src/sql/resolver/dml/ob_select_resolver.h b/src/sql/resolver/dml/ob_select_resolver.h index e467507dd..7c9ac026c 100644 --- a/src/sql/resolver/dml/ob_select_resolver.h +++ b/src/sql/resolver/dml/ob_select_resolver.h @@ -202,7 +202,8 @@ protected: int resolve_into_line_node(const ParseNode *node, ObSelectIntoItem &into_item); int resolve_into_variable_node(const ParseNode *node, ObSelectIntoItem &into_item); int resolve_into_file_node(const ParseNode *node, ObSelectIntoItem &into_item); - int resolve_max_file_size_node(const ParseNode *file_size_node, ObSelectIntoItem &into_item); + int resolve_file_partition_node(const ParseNode *node, ObSelectIntoItem &into_item); + int resolve_size_node(const ParseNode *file_size_node, ObSelectIntoItem &into_item); int resolve_varchar_file_size(const ParseNode *child, int64_t &parse_int_value) const; // resolve_star related functions int resolve_star_for_table_groups(ObStarExpansionInfo &star_expansion_info); diff --git a/src/sql/resolver/dml/ob_select_stmt.cpp b/src/sql/resolver/dml/ob_select_stmt.cpp index 375a627c6..6f9947985 100644 --- a/src/sql/resolver/dml/ob_select_stmt.cpp +++ b/src/sql/resolver/dml/ob_select_stmt.cpp @@ -52,12 +52,43 @@ int SelectItem::deep_copy(ObIRawExprCopier &expr_copier, return ret; } +int ObSelectIntoItem::deep_copy(ObIRawExprCopier &copier, + const ObSelectIntoItem &other) +{ + int ret = OB_SUCCESS; + into_type_ = other.into_type_; + outfile_name_ = other.outfile_name_; + field_str_ = other.field_str_; + line_str_ = other.line_str_; + closed_cht_ = other.closed_cht_; + is_optional_ = other.is_optional_; + is_single_ = other.is_single_; + max_file_size_ = other.max_file_size_; + escaped_cht_ = other.escaped_cht_; + cs_type_ = other.cs_type_; + file_partition_expr_ = other.file_partition_expr_; + buffer_size_ = other.buffer_size_; + user_vars_.assign(other.user_vars_); + if (OB_FAIL(copier.copy(other.file_partition_expr_, file_partition_expr_))) { + LOG_WARN("deep copy file partition expr failed", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < other.pl_vars_.count(); ++i) { + ObRawExpr* pl_var; + if (OB_FAIL(copier.copy(other.pl_vars_.at(i), pl_var))) { + LOG_WARN("failed to copy exprs", K(ret)); + } else if (OB_FAIL(pl_vars_.push_back(pl_var))) { + LOG_WARN("failed to push back group by expr", K(ret)); + } + } + return ret; +} const char* const ObSelectIntoItem::DEFAULT_LINE_TERM_STR = "\n"; const char* const ObSelectIntoItem::DEFAULT_FIELD_TERM_STR = "\t"; const char ObSelectIntoItem::DEFAULT_FIELD_ENCLOSED_CHAR = 0; const bool ObSelectIntoItem::DEFAULT_OPTIONAL_ENCLOSED = false; const bool ObSelectIntoItem::DEFAULT_SINGLE_OPT = true; const int64_t ObSelectIntoItem::DEFAULT_MAX_FILE_SIZE = 256 * 1024 * 1024; +const int64_t ObSelectIntoItem::DEFAULT_BUFFER_SIZE = 1 * 1024 * 1024; const char ObSelectIntoItem::DEFAULT_FIELD_ESCAPED_CHAR = '\\'; //对于select .. for update 也认为是被更改 @@ -336,7 +367,7 @@ int ObSelectStmt::deep_copy_stmt_struct(ObIAllocator &allocator, LOG_WARN("failed to allocate select into item", K(ret)); } else { temp_into_item = new(ptr) ObSelectIntoItem(); - if (OB_FAIL(temp_into_item->assign(*other.into_item_))) { + if (OB_FAIL(temp_into_item->deep_copy(expr_copier, *other.into_item_))) { LOG_WARN("deep copy into item failed", K(ret)); } else { into_item_ = temp_into_item; @@ -473,6 +504,10 @@ int ObSelectStmt::iterate_stmt_expr(ObStmtExprVisitor &visitor) LOG_WARN("failed to visit select into", K(ret)); } } + if (OB_SUCC(ret) && into_item_->file_partition_expr_ != NULL + && OB_FAIL(visitor.visit(into_item_->file_partition_expr_, SCOPE_SELECT))) { + LOG_WARN("failed to visit select into", K(ret)); + } } return ret; } diff --git a/src/sql/resolver/dml/ob_select_stmt.h b/src/sql/resolver/dml/ob_select_stmt.h index 88b28e588..e2ffcaff7 100644 --- a/src/sql/resolver/dml/ob_select_stmt.h +++ b/src/sql/resolver/dml/ob_select_stmt.h @@ -126,7 +126,9 @@ struct ObSelectIntoItem is_optional_(DEFAULT_OPTIONAL_ENCLOSED), is_single_(DEFAULT_SINGLE_OPT), max_file_size_(DEFAULT_MAX_FILE_SIZE), - escaped_cht_() + escaped_cht_(), + file_partition_expr_(NULL), + buffer_size_(DEFAULT_BUFFER_SIZE) { field_str_.set_varchar(DEFAULT_FIELD_TERM_STR); field_str_.set_collation_type(ObCharset::get_system_collation()); @@ -151,8 +153,12 @@ struct ObSelectIntoItem max_file_size_ = other.max_file_size_; escaped_cht_ = other.escaped_cht_; cs_type_ = other.cs_type_; + file_partition_expr_ = other.file_partition_expr_; + buffer_size_ = other.buffer_size_; return user_vars_.assign(other.user_vars_); } + int deep_copy(ObIRawExprCopier &copier, + const ObSelectIntoItem &other); TO_STRING_KV(K_(into_type), K_(outfile_name), K_(field_str), @@ -162,7 +168,8 @@ struct ObSelectIntoItem K_(is_single), K_(max_file_size), K_(escaped_cht), - K_(cs_type)); + K_(cs_type), + N_EXPR, file_partition_expr_); ObItemType into_type_; common::ObObj outfile_name_; common::ObObj field_str_; // field terminated str @@ -175,6 +182,9 @@ struct ObSelectIntoItem int64_t max_file_size_; common::ObObj escaped_cht_; common::ObCollationType cs_type_; + sql::ObRawExpr* file_partition_expr_; + int64_t buffer_size_; + ObPQDistributeMethod::Type dist_method_; static const char* const DEFAULT_FIELD_TERM_STR; static const char* const DEFAULT_LINE_TERM_STR; @@ -182,6 +192,7 @@ struct ObSelectIntoItem static const bool DEFAULT_OPTIONAL_ENCLOSED; static const bool DEFAULT_SINGLE_OPT; static const int64_t DEFAULT_MAX_FILE_SIZE; + static const int64_t DEFAULT_BUFFER_SIZE; static const char DEFAULT_FIELD_ESCAPED_CHAR; }; diff --git a/src/sql/resolver/ob_resolver.cpp b/src/sql/resolver/ob_resolver.cpp index 049d8fff8..7ecfcde11 100644 --- a/src/sql/resolver/ob_resolver.cpp +++ b/src/sql/resolver/ob_resolver.cpp @@ -1291,7 +1291,8 @@ int ObResolver::resolve(IsPrepared if_prepared, const ParseNode &parse_tree, ObS } } // end switch - if (OB_SUCC(ret) && stmt->is_dml_stmt()) { + // 外表写只放开insert + if (OB_SUCC(ret) && stmt->is_dml_stmt() && !stmt->is_insert_stmt()) { OZ( (static_cast(stmt)->disable_writing_external_table()) ); } diff --git a/src/sql/resolver/ob_resolver_utils.cpp b/src/sql/resolver/ob_resolver_utils.cpp index ee0d1aabc..b41934502 100644 --- a/src/sql/resolver/ob_resolver_utils.cpp +++ b/src/sql/resolver/ob_resolver_utils.cpp @@ -4783,7 +4783,8 @@ int ObResolverUtils::calc_file_column_idx(const ObString &column_name, uint64_t int32_t PREFIX_LEN = column_name.prefix_match_ci(N_PARTITION_LIST_COL) ? str_length(N_PARTITION_LIST_COL) : column_name.prefix_match_ci(N_EXTERNAL_FILE_COLUMN_PREFIX)? - str_length(N_EXTERNAL_FILE_COLUMN_PREFIX) : -1; + str_length(N_EXTERNAL_FILE_COLUMN_PREFIX) : column_name.prefix_match_ci(N_EXTERNAL_TABLE_COLUMN_PREFIX) ? + str_length(N_EXTERNAL_TABLE_COLUMN_PREFIX) : -1; if (column_name.length() <= PREFIX_LEN) { ret = OB_ERR_UNEXPECTED; } else { @@ -4863,6 +4864,16 @@ int ObResolverUtils::resolve_external_table_column_def(ObRawExprFactory &expr_fa LOG_WARN("fail to build external table file column expr", K(ret)); } } + } else if (ObExternalFileFormat::ODPS_FORMAT == ObResolverUtils::resolve_external_file_column_type(q_name.col_name_)) { + if (OB_FAIL(ObResolverUtils::calc_file_column_idx(q_name.col_name_, file_column_idx))) { + LOG_WARN("fail to calc file column idx", K(ret)); + } else if (nullptr == (file_column_expr = ObResolverUtils::find_file_column_expr( + real_exprs, OB_INVALID_ID, file_column_idx, q_name.col_name_))) { + if (OB_FAIL(ObResolverUtils::build_file_column_expr_for_odps(expr_factory, session_info, + OB_INVALID_ID, ObString(), q_name.col_name_, file_column_idx, gen_col_schema, file_column_expr))) { + LOG_WARN("fail to build external table file column expr", K(ret)); + } + } } else { if (OB_FAIL(ObResolverUtils::build_file_row_expr_for_parquet(expr_factory, session_info, OB_INVALID_ID, ObString(), @@ -4904,6 +4915,8 @@ ObExternalFileFormat::FormatType ObResolverUtils::resolve_external_file_column_t ObExternalFileFormat::FormatType type = ObExternalFileFormat::INVALID_FORMAT; if (name.prefix_match_ci(N_EXTERNAL_FILE_COLUMN_PREFIX)) { type = ObExternalFileFormat::CSV_FORMAT; + } else if (name.prefix_match_ci(N_EXTERNAL_TABLE_COLUMN_PREFIX)) { + type = ObExternalFileFormat::ODPS_FORMAT; } else if (0 == name.case_compare(N_EXTERNAL_FILE_ROW)) { type = ObExternalFileFormat::PARQUET_FORMAT; } @@ -5048,6 +5061,46 @@ int ObResolverUtils::build_file_row_expr_for_parquet( return ret; } +int ObResolverUtils::build_file_column_expr_for_odps(ObRawExprFactory &expr_factory, + const ObSQLSessionInfo &session_info, + const uint64_t table_id, + const ObString &table_name, + const ObString &column_name, + int64_t column_idx, + const ObColumnSchemaV2 *column_schema, + ObRawExpr *&expr) +{ + int ret = OB_SUCCESS; + ObPseudoColumnRawExpr *file_column_expr = nullptr; + ObItemType type = T_PSEUDO_EXTERNAL_FILE_COL; + uint64_t extra = column_idx; + if (OB_ISNULL(column_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted null ptr", K(ret)); + } else if (OB_FAIL(expr_factory.create_raw_expr(type, file_column_expr))) { + LOG_WARN("create nextval failed", K(ret)); + } else if (OB_ISNULL(file_column_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expr is null", K(ret)); + } else { + file_column_expr->set_expr_name(column_name); + file_column_expr->set_table_name(table_name); + file_column_expr->set_table_id(table_id); + file_column_expr->set_explicited_reference(); + file_column_expr->set_extra(extra); + file_column_expr->set_meta_type(column_schema->get_meta_type()); + file_column_expr->set_collation_level(CS_LEVEL_IMPLICIT); + file_column_expr->set_accuracy(column_schema->get_accuracy()); + if (OB_FAIL(file_column_expr->formalize(&session_info))) { + LOG_WARN("failed to extract info", K(ret)); + } else { + expr = file_column_expr; + } + } + + return ret; +} + int ObResolverUtils::build_file_column_expr_for_csv(ObRawExprFactory &expr_factory, const ObSQLSessionInfo &session_info, const uint64_t table_id, @@ -9715,5 +9768,36 @@ int ObResolverUtils::check_schema_valid_for_mview(const ObTableSchema &table_sch return ret; } +bool ObResolverUtils::is_external_pseudo_column(const ObRawExpr &expr) +{ + bool ret = false; + if (T_PSEUDO_EXTERNAL_FILE_COL == expr.get_expr_type() + || T_PSEUDO_EXTERNAL_FILE_URL == expr.get_expr_type() + || T_PSEUDO_EXTERNAL_FILE_ROW == expr.get_expr_type()) { + ret = true; + } + return ret; +} + +int ObResolverUtils::cnt_external_pseudo_column(const ObRawExpr &expr, bool &contain) +{ + int ret = OB_SUCCESS; + contain = false; + if (is_external_pseudo_column(expr)) { + contain = true; + } else { + for (int64_t i = 0; !contain && i < expr.get_children_count(); i++) { + const ObRawExpr *child = expr.get_param_expr(i); + if (OB_ISNULL(child)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid child", K(child)); + } else if (OB_FAIL(SMART_CALL(cnt_external_pseudo_column(*child, contain)))) { + LOG_WARN("failed to check if need to check col duplicate", K(ret)); + } + } + } + return ret; +} + } // namespace sql } // namespace oceanbase diff --git a/src/sql/resolver/ob_resolver_utils.h b/src/sql/resolver/ob_resolver_utils.h index e8675ae23..0ebb450cc 100644 --- a/src/sql/resolver/ob_resolver_utils.h +++ b/src/sql/resolver/ob_resolver_utils.h @@ -774,6 +774,15 @@ public: int64_t column_idx, const ObString &expr_name); static int calc_file_column_idx(const ObString &column_name, uint64_t &file_column_idx); + static int build_file_column_expr_for_odps( + ObRawExprFactory &expr_factory, + const ObSQLSessionInfo &session_info, + const uint64_t table_id, + const common::ObString &table_name, + const common::ObString &column_name, + int64_t column_idx, + const ObColumnSchemaV2 *column_schema, + ObRawExpr *&expr); static int build_file_column_expr_for_csv( ObRawExprFactory &expr_factory, const ObSQLSessionInfo &session_info, @@ -879,6 +888,8 @@ public: static int64_t get_mysql_max_partition_num(const uint64_t tenant_id); static int check_schema_valid_for_mview(const share::schema::ObTableSchema &table_schema); + static bool is_external_pseudo_column(const ObRawExpr &expr); + static int cnt_external_pseudo_column(const ObRawExpr &expr, bool &contain); private: static int try_convert_to_unsiged(const ObExprResType restype, ObRawExpr& src_expr, diff --git a/unittest/sql/engine/dml/CMakeLists.txt b/unittest/sql/engine/dml/CMakeLists.txt index 051fbbd93..5b3f1f828 100644 --- a/unittest/sql/engine/dml/CMakeLists.txt +++ b/unittest/sql/engine/dml/CMakeLists.txt @@ -1,2 +1,3 @@ #ob_unittest(test_table_insert) #ob_unittest(test_insert_up) +#ob_unittest(test_odps) diff --git a/unittest/sql/engine/dml/test_odps.cpp b/unittest/sql/engine/dml/test_odps.cpp new file mode 100644 index 000000000..6ba706e38 --- /dev/null +++ b/unittest/sql/engine/dml/test_odps.cpp @@ -0,0 +1,98 @@ +#ifdef OB_BUILD_CPP_ODPS +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include +#include +#include +#include + +#include + +using namespace std; +using namespace apsara; +using namespace apsara::odps; +using namespace apsara::odps::sdk; + +namespace test +{ + +class TestODPS: public ::testing::Test +{ +public: + TestODPS() {} + ~TestODPS() {} +}; + +TEST_F(TestODPS, test_odps) +{ + string ep = ""; + string odpsEndpoint = ""; + string project = "ailing_test"; + string table = "odps_table_jim"; + uint32_t blockId = 0; + + Account account(ACCOUNT_ALIYUN, "", ""); + + Configuration conf; + conf.SetAccount(account); + conf.SetTunnelEndpoint(ep); + conf.SetEndpoint(odpsEndpoint); + conf.SetUserAgent("UPLOAD_EXAMPLE"); + + OdpsTunnel dt; + + try + { + std::vector blocks; + dt.Init(conf); + IUploadPtr upload = dt.CreateUpload(project, table, "ds = 2, other = 'b'", "", true); + for (blockId = 0; blockId < 2; blockId++) { + std::cout << upload->GetStatus() << std::endl; + IRecordWriterPtr wr = upload->OpenWriter(blockId); + ODPSTableRecordPtr _r = upload->CreateBufferRecord(); + std::cout << _r->GetSchema()->GetColumnCount() << std::endl; + ODPSTableRecord& r = *_r; + + for (size_t i = 0; i < 10; i++) + { + if (i % 10 == 0) + { + r.SetNullValue(0); + } + else + { + //r.SetBigIntValue(0, i); + r.SetStringValue(0, std::string("hello") + std::to_string(i)); + } + wr->Write(r); + } + wr->Close(); + std::cout << upload->GetStatus() << std::endl; + blocks.push_back(blockId); + } + upload->Commit(blocks); + std::cout << upload->GetStatus() << std::endl; + } + catch(OdpsException& e) + { + std::cerr << "OdpsTunnelException:\n" << e.what() << std::endl; + } +} +} + +int main(int argc, char **argv) +{ + ::testing::InitGoogleTest(&argc,argv); + return RUN_ALL_TESTS(); +} +#endif \ No newline at end of file diff --git a/unittest/sql/test_sql_utils.h b/unittest/sql/test_sql_utils.h index d3d855760..bbe25e0b8 100644 --- a/unittest/sql/test_sql_utils.h +++ b/unittest/sql/test_sql_utils.h @@ -135,7 +135,7 @@ public: ObSchemaGetterGuard &get_schema_guard() { return schema_guard_; } public: //table id - hash::ObHashMap next_user_table_id_map_; + oceanbase::common::hash::ObHashMap next_user_table_id_map_; //user_id uint64_t sys_user_id_; uint64_t next_user_id_; From e273c377e4ff88a931ab28f68dc9e23d75ff3c9f Mon Sep 17 00:00:00 2001 From: yaojing624 Date: Tue, 27 Aug 2024 14:16:07 +0000 Subject: [PATCH 239/249] Fix: memory leak in sql audit --- src/observer/mysql/ob_dl_queue.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/observer/mysql/ob_dl_queue.cpp b/src/observer/mysql/ob_dl_queue.cpp index a0df0de38..6f19029e5 100644 --- a/src/observer/mysql/ob_dl_queue.cpp +++ b/src/observer/mysql/ob_dl_queue.cpp @@ -75,11 +75,13 @@ int ObDlQueue::construct_leaf_queue() { LOG_WARN("fail to allocate memory", K(ret)); } else if (OB_FAIL(cur_lf_queue->init(ObModIds::OB_MYSQL_REQUEST_RECORD, LEAF_QUEUE_SIZE, tenant_id_))) { cur_lf_queue->destroy(); + ob_free(cur_lf_queue); cur_lf_queue = NULL; LOG_WARN("fail to init leaf queue", K(ret), K(tenant_id_)); } else if (OB_FAIL(rq_.push_with_imme_seq(cur_lf_queue, root_push_idx))) { SERVER_LOG(WARN, "Failed to push leaf queue", K(ret)); cur_lf_queue->destroy(); + ob_free(cur_lf_queue); cur_lf_queue = NULL; } From c0b2e36c0fb75446328d0dd52403d5bb77b85618 Mon Sep 17 00:00:00 2001 From: Tsunaou <895254752@qq.com> Date: Tue, 27 Aug 2024 14:58:12 +0000 Subject: [PATCH 240/249] fix ls locality concurrent bug && fix ddl slice store sort bug --- .../ob_compaction_locality_cache.cpp | 46 +++++++++++++++++++ .../compaction/ob_compaction_locality_cache.h | 9 +++- .../compaction/ob_tenant_medium_checker.cpp | 25 +++++++--- .../compaction/ob_tenant_medium_checker.h | 3 +- .../ddl/ob_direct_insert_sstable_ctx_new.cpp | 35 +++++++------- 5 files changed, 92 insertions(+), 26 deletions(-) diff --git a/src/share/compaction/ob_compaction_locality_cache.cpp b/src/share/compaction/ob_compaction_locality_cache.cpp index 27a24a061..12e53711a 100644 --- a/src/share/compaction/ob_compaction_locality_cache.cpp +++ b/src/share/compaction/ob_compaction_locality_cache.cpp @@ -174,6 +174,52 @@ int ObLSColumnReplicaCache::add_cs_replica(const ObLSReplicaUniItem &ls_item) return ret; } +int ObLSColumnReplicaCache::assign(const ObLSColumnReplicaCache &other) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", K(ret)); + } else if (OB_FAIL(other.deep_fetch(ls_id_set_, ls_replica_set_, ls_infos_))) { + LOG_WARN("fail to assign", K(ret), K(other)); + } + return ret; +} + +int ObLSColumnReplicaCache::deep_fetch( + hash::ObHashSet &target_ls_id_set, + hash::ObHashSet &target_ls_replica_set, + common::ObIArray &target_ls_infos) const +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", K(ret)); + } else { + target_ls_id_set.reuse(); + target_ls_replica_set.reuse(); + target_ls_infos.reuse(); + + for (hash::ObHashSet::const_iterator it = ls_id_set_.begin(); OB_SUCC(ret) && it != ls_id_set_.end(); ++it) { + if (OB_FAIL(target_ls_id_set.set_refactored(it->first))) { + LOG_WARN("fail to add ls id", K(ret)); + } + } + for (hash::ObHashSet::const_iterator it = ls_replica_set_.begin(); OB_SUCC(ret) && it != ls_replica_set_.end(); ++it) { + if (OB_FAIL(target_ls_replica_set.set_refactored(it->first))) { + LOG_WARN("fail to add ls replica", K(ret)); + } + } + for (int64_t idx = 0; OB_SUCC(ret) && idx < ls_infos_.count(); ++idx) { + if (OB_FAIL(target_ls_infos.push_back(ls_infos_.at(idx)))) { + LOG_WARN("fail to push back ls info", K(ret), K(idx), K_(ls_infos)); + } + } + } + return ret; +} + + int ObLSColumnReplicaCache::update(const ObLSID &ls_id) { int ret = OB_SUCCESS; diff --git a/src/share/compaction/ob_compaction_locality_cache.h b/src/share/compaction/ob_compaction_locality_cache.h index 363543df1..0c4a2babe 100644 --- a/src/share/compaction/ob_compaction_locality_cache.h +++ b/src/share/compaction/ob_compaction_locality_cache.h @@ -54,6 +54,11 @@ public: int init(); void destroy(); void reuse(); + int assign(const ObLSColumnReplicaCache &other); + int deep_fetch( + hash::ObHashSet &target_ls_id_set, + hash::ObHashSet &target_ls_replica_set, + common::ObIArray &target_ls_infos) const; int update(const ObLSID &ls_id); int update_with_ls_info(const ObLSInfo &ls_info); int check_is_cs_replica(const ObLSReplicaUniItem &ls_item, bool &is_cs_replica) const; @@ -68,8 +73,8 @@ private: const static int64_t BUCKET_NUM_OF_LS_REPLICA_SET = 31; private: bool is_inited_; - hash::ObHashSet ls_id_set_; // record looped ls id - hash::ObHashSet ls_replica_set_; // cs-prelica ls + hash::ObHashSet ls_id_set_; // record looped ls id + hash::ObHashSet ls_replica_set_; // cs-prelica ls common::ObSEArray ls_infos_; // used for check member list and learner list }; diff --git a/src/storage/compaction/ob_tenant_medium_checker.cpp b/src/storage/compaction/ob_tenant_medium_checker.cpp index 3b05db2ff..102879a27 100644 --- a/src/storage/compaction/ob_tenant_medium_checker.cpp +++ b/src/storage/compaction/ob_tenant_medium_checker.cpp @@ -214,14 +214,24 @@ int ObTenantMediumChecker::check_medium_finish_schedule() { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; + ObLSColumnReplicaCache cs_replica_cache; // a copy one from ls_locality_cache_ if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ObTenantMediumChecker is not inited", K(ret)); - } else if (OB_FAIL(ls_locality_cache_.refresh_ls_locality(false /*force_refresh*/))) { - LOG_WARN("failed to refresh ls locality"); - ADD_COMMON_SUSPECT_INFO(MEDIUM_MERGE, share::ObDiagnoseTabletType::TYPE_MEDIUM_MERGE, - SUSPECT_FAILED_TO_REFRESH_LS_LOCALITY, ret); + } else if (OB_FAIL(cs_replica_cache.init())) { + LOG_WARN("failed to init cs_replica_cache", K(ret)); } else { + lib::ObMutexGuard guard(lock_); + if (OB_FAIL(ls_locality_cache_.refresh_ls_locality(false /*force_refresh*/))) { + LOG_WARN("failed to refresh ls locality"); + ADD_COMMON_SUSPECT_INFO(MEDIUM_MERGE, share::ObDiagnoseTabletType::TYPE_MEDIUM_MERGE, + SUSPECT_FAILED_TO_REFRESH_LS_LOCALITY, ret); + } else if (OB_FAIL(cs_replica_cache.assign(ls_locality_cache_.get_cs_replica_cache()))) { + LOG_WARN("failed to assign cs_replica_cache", K(ret)); + } + } + + if (OB_SUCC(ret)) { DEL_SUSPECT_INFO(MEDIUM_MERGE, UNKNOW_LS_ID, UNKNOW_TABLET_ID, ObDiagnoseTabletType::TYPE_MEDIUM_MERGE); TabletLSArray tablet_ls_infos; tablet_ls_infos.set_attr(ObMemAttr(MTL_ID(), "CheckInfos")); @@ -260,7 +270,7 @@ int ObTenantMediumChecker::check_medium_finish_schedule() int64_t cost_ts = ObTimeUtility::fast_current_time(); ObBatchFinishCheckStat stat; while (start_idx < end_idx) { - if (OB_TMP_FAIL(check_medium_finish(tablet_ls_infos, start_idx, end_idx, batch_tablet_ls_infos, finish_tablet_ls_infos, stat))) { + if (OB_TMP_FAIL(check_medium_finish(tablet_ls_infos, start_idx, end_idx, batch_tablet_ls_infos, finish_tablet_ls_infos, stat, cs_replica_cache))) { LOG_WARN("failed to check medium finish", K(tmp_ret)); } start_idx = end_idx; @@ -283,7 +293,8 @@ int ObTenantMediumChecker::check_medium_finish( int64_t end_idx, ObIArray &check_tablet_ls_infos, ObIArray &finish_tablet_ls_infos, - ObBatchFinishCheckStat &stat) + ObBatchFinishCheckStat &stat, + const share::ObLSColumnReplicaCache &ls_cs_replica_cache) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -313,7 +324,7 @@ int ObTenantMediumChecker::check_medium_finish( ObCompactionScheduleTimeGuard time_guard; stat.filter_cnt_ += (end_idx - start_idx - check_tablet_ls_infos.count()); if (FAILEDx(ObMediumCompactionScheduleFunc::batch_check_medium_finish( - ls_info_map_, finish_tablet_ls_infos, check_tablet_ls_infos, time_guard, ls_locality_cache_.get_cs_replica_cache()))) { + ls_info_map_, finish_tablet_ls_infos, check_tablet_ls_infos, time_guard, ls_cs_replica_cache))) { LOG_WARN("failed to batch check medium finish", K(ret), K(tablet_ls_infos.count()), K(check_tablet_ls_infos.count()), K(tablet_ls_infos), K(check_tablet_ls_infos)); stat.fail_cnt_ += check_tablet_ls_infos.count(); diff --git a/src/storage/compaction/ob_tenant_medium_checker.h b/src/storage/compaction/ob_tenant_medium_checker.h index 12cc6f2f9..342d67875 100644 --- a/src/storage/compaction/ob_tenant_medium_checker.h +++ b/src/storage/compaction/ob_tenant_medium_checker.h @@ -95,7 +95,8 @@ public: int64_t end_idx, ObIArray &check_tablet_ls_infos, ObIArray &finish_tablet_ls_infos, - ObBatchFinishCheckStat &stat); + ObBatchFinishCheckStat &stat, + const share::ObLSColumnReplicaCache &ls_cs_replica_cache); int add_tablet_ls(const ObTabletID &tablet_id, const share::ObLSID &ls_id, const int64_t medium_scn); bool locality_cache_empty(); TO_STRING_KV(K_(is_inited), K_(ls_locality_cache)); diff --git a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp index bae9673fc..664be5fd3 100644 --- a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp +++ b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp @@ -1801,28 +1801,32 @@ int ObTabletDirectLoadMgr::notify_all() struct CSSliceEndkeyCompareFunctor { public: - CSSliceEndkeyCompareFunctor(const ObStorageDatumUtils &datum_utils) : datum_utils_(datum_utils), ret_code_(OB_SUCCESS) {} + CSSliceEndkeyCompareFunctor(const ObStorageDatumUtils &datum_utils, int &ret_code) : datum_utils_(datum_utils), ret_code_(ret_code) {} bool operator ()(const ObDirectLoadSliceWriter *left, const ObDirectLoadSliceWriter *right) { bool bret = false; int ret = ret_code_; if (OB_FAIL(ret)) { - } else if (OB_ISNULL(left) || OB_ISNULL(right) || !left->need_column_store() - || !right->need_column_store() || left->get_writer_type() != right->get_writer_type()) { + } else if (OB_ISNULL(left) || OB_ISNULL(right)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), KPC(left), KPC(right)); } else if (!left->is_empty() && !right->is_empty()) { - const ObChunkSliceStore *left_slice_store = left->is_cs_replica_write() - ? static_cast(left->get_slice_store())->get_column_slice_store() - : static_cast(left->get_slice_store()); - const ObChunkSliceStore *right_slice_store = right->is_cs_replica_write() - ? static_cast(right->get_slice_store())->get_column_slice_store() - : static_cast(right->get_slice_store()); - int cmp_ret = 0; - if (OB_FAIL(left_slice_store->endkey_.compare(right_slice_store->endkey_, datum_utils_, cmp_ret))) { - LOG_WARN("endkey compare failed", K(ret)); + if (!left->need_column_store() || !right->need_column_store() || left->get_writer_type() != right->get_writer_type()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid cs slice writer", K(ret), KPC(left), KPC(right)); } else { - bret = cmp_ret < 0; + const ObChunkSliceStore *left_slice_store = left->is_cs_replica_write() + ? static_cast(left->get_slice_store())->get_column_slice_store() + : static_cast(left->get_slice_store()); + const ObChunkSliceStore *right_slice_store = right->is_cs_replica_write() + ? static_cast(right->get_slice_store())->get_column_slice_store() + : static_cast(right->get_slice_store()); + int cmp_ret = 0; + if (OB_FAIL(left_slice_store->endkey_.compare(right_slice_store->endkey_, datum_utils_, cmp_ret))) { + LOG_WARN("endkey compare failed", K(ret)); + } else { + bret = cmp_ret < 0; + } } } else if (left->is_empty() && right->is_empty()) { // both empty, compare pointer @@ -1836,7 +1840,7 @@ public: } public: const ObStorageDatumUtils &datum_utils_; - int ret_code_; + int &ret_code_; // is not use reference, the ret_code_ will lose when use ob_sort }; int ObTabletDirectLoadMgr::calc_range(const int64_t thread_cnt) @@ -1874,9 +1878,8 @@ int ObTabletDirectLoadMgr::calc_range(const int64_t thread_cnt) } } if (OB_SUCC(ret)) { - CSSliceEndkeyCompareFunctor cmp(tablet_handle.get_obj()->get_rowkey_read_info().get_datum_utils()); + CSSliceEndkeyCompareFunctor cmp(tablet_handle.get_obj()->get_rowkey_read_info().get_datum_utils(), ret); lib::ob_sort(sorted_slices.begin(), sorted_slices.end(), cmp); - ret = cmp.ret_code_; if (OB_FAIL(ret)) { LOG_WARN("sort slice failed", K(ret), K(sorted_slices)); } From b6b2364a32e84c07afa8a28cd4b29c21289c88f6 Mon Sep 17 00:00:00 2001 From: YangEfei Date: Tue, 27 Aug 2024 15:11:19 +0000 Subject: [PATCH 241/249] [TABLELOCK] refine struct of ObLockFuncContext, check ObSQLSessionInfo and ObExecContext is not null before use them --- .../tablelock/ob_lock_func_executor.cpp | 292 ++++++++++-------- src/storage/tablelock/ob_lock_func_executor.h | 11 +- 2 files changed, 162 insertions(+), 141 deletions(-) diff --git a/src/storage/tablelock/ob_lock_func_executor.cpp b/src/storage/tablelock/ob_lock_func_executor.cpp index 95822a95f..097b85fb9 100644 --- a/src/storage/tablelock/ob_lock_func_executor.cpp +++ b/src/storage/tablelock/ob_lock_func_executor.cpp @@ -42,97 +42,106 @@ namespace transaction namespace tablelock { -int ObLockFuncContext::init(ObSQLSessionInfo &session_info, - ObExecContext &ctx, +int ObLockFuncContext::init(ObExecContext &ctx, const int64_t timeout_us) { int ret = OB_SUCCESS; - int64_t query_start_time = session_info.get_query_start_time(); - // use smaller timeout if we specified the lock timeout us. - if (timeout_us > 0 - && (ObTimeUtility::current_time() + timeout_us) < THIS_WORKER.get_timeout_ts()) { - OX (old_worker_timeout_ts_ = THIS_WORKER.get_timeout_ts()); - OX (THIS_WORKER.set_timeout_ts(ObTimeUtility::current_time() + timeout_us)); - if (OB_SUCC(ret) && OB_NOT_NULL(ctx.get_physical_plan_ctx())) { - old_phy_plan_timeout_ts_ = ctx.get_physical_plan_ctx()->get_timeout_timestamp(); - ctx.get_physical_plan_ctx() - ->set_timeout_timestamp(ObTimeUtility::current_time() + timeout_us); - } - } - if (OB_SUCC(ret)) { - if (session_info.get_local_autocommit()) { - OX (reset_autocommit_ = true); - OZ (session_info.set_autocommit(false)); - } - has_inner_dml_write_ = session_info.has_exec_inner_dml(); - last_insert_id_ = session_info.get_local_last_insert_id(); - session_info.set_has_exec_inner_dml(false); + ObSQLSessionInfo *session_info = nullptr; - ObTransID parent_tx_id; - parent_tx_id = session_info.get_tx_id(); - OZ (session_info.begin_autonomous_session(saved_session_)); - OX (have_saved_session_ = true); - OZ (ObSqlTransControl::explicit_start_trans(ctx, false)); + if (OB_ISNULL(session_info = ctx.get_my_session())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("session_info is null in ObExecContext", K(ret)); + } else { + // use smaller timeout if we specified the lock timeout us. + if (timeout_us > 0 + && (ObTimeUtility::current_time() + timeout_us) < THIS_WORKER.get_timeout_ts()) { + OX (old_worker_timeout_ts_ = THIS_WORKER.get_timeout_ts()); + OX (THIS_WORKER.set_timeout_ts(ObTimeUtility::current_time() + timeout_us)); + if (OB_SUCC(ret) && OB_NOT_NULL(ctx.get_physical_plan_ctx())) { + old_phy_plan_timeout_ts_ = ctx.get_physical_plan_ctx()->get_timeout_timestamp(); + ctx.get_physical_plan_ctx() + ->set_timeout_timestamp(ObTimeUtility::current_time() + timeout_us); + } + } if (OB_SUCC(ret)) { - has_autonomous_tx_ = true; - } - if (OB_SUCC(ret) && parent_tx_id.is_valid()) { - (void) register_for_deadlock_(session_info, parent_tx_id); + if (session_info->get_local_autocommit()) { + OX (reset_autocommit_ = true); + OZ (session_info->set_autocommit(false)); + } + has_inner_dml_write_ = session_info->has_exec_inner_dml(); + last_insert_id_ = session_info->get_local_last_insert_id(); + session_info->set_has_exec_inner_dml(false); + + ObTransID parent_tx_id; + parent_tx_id = session_info->get_tx_id(); + OZ (session_info->begin_autonomous_session(saved_session_)); + OX (have_saved_session_ = true); + OZ (ObSqlTransControl::explicit_start_trans(ctx, false)); + if (OB_SUCC(ret)) { + has_autonomous_tx_ = true; + } + if (OB_SUCC(ret) && parent_tx_id.is_valid()) { + (void) register_for_deadlock_(*session_info, parent_tx_id); + } } + OX (my_exec_ctx_ = &ctx); + OZ (open_inner_conn_()); } - OX (session_info_ = &session_info); - OX (my_exec_ctx_ = &ctx); - OZ (open_inner_conn_()); return ret; } int ObLockFuncContext::destroy(ObExecContext &ctx, - ObSQLSessionInfo &session_info, bool is_rollback) { int tmp_ret = OB_SUCCESS; int ret = OB_SUCCESS; + ObSQLSessionInfo *session_info = nullptr; - if (has_autonomous_tx_) { - if (OB_TMP_FAIL(implicit_end_trans_(session_info, ctx, is_rollback))) { - LOG_WARN("failed to rollback trans", K(tmp_ret)); + if (OB_ISNULL(session_info = ctx.get_my_session())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("session_info is null in ObExecContext", K(ret)); + } else { + if (has_autonomous_tx_) { + if (OB_TMP_FAIL(implicit_end_trans_(*session_info, ctx, is_rollback))) { + LOG_WARN("failed to rollback trans", K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + } + if (OB_TMP_FAIL(close_inner_conn_())) { + LOG_WARN("close inner connection failed", K(tmp_ret)); ret = COVER_SUCC(tmp_ret); } - } - if (OB_TMP_FAIL(close_inner_conn_())) { - LOG_WARN("close inner connection failed", K(tmp_ret)); - ret = COVER_SUCC(tmp_ret); - } - if (have_saved_session_) { - if (OB_TMP_FAIL(session_info.end_autonomous_session(saved_session_))) { - LOG_WARN("failed to switch trans", K(tmp_ret)); - ret = COVER_SUCC(tmp_ret); + if (have_saved_session_) { + if (OB_TMP_FAIL(session_info->end_autonomous_session(saved_session_))) { + LOG_WARN("failed to switch trans", K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } } - } - // WHY WE NEED THIS - uint64_t cur_last_insert_id = session_info.get_local_last_insert_id(); - if (cur_last_insert_id != last_insert_id_) { - ObObj last_insert_id; - last_insert_id.set_uint64(last_insert_id_); - tmp_ret = session_info.update_sys_variable(SYS_VAR_LAST_INSERT_ID, last_insert_id); - if (OB_SUCCESS == tmp_ret && - OB_TMP_FAIL(session_info.update_sys_variable(SYS_VAR_IDENTITY, last_insert_id))) { - LOG_WARN("succ update last_insert_id, but fail to update identity", K(tmp_ret)); - } - ret = COVER_SUCC(tmp_ret); - } - session_info.set_has_exec_inner_dml(has_inner_dml_write_); - if (old_worker_timeout_ts_ != 0) { - THIS_WORKER.set_timeout_ts(old_worker_timeout_ts_); - if (OB_NOT_NULL(ctx.get_physical_plan_ctx())) { - ctx.get_physical_plan_ctx()->set_timeout_timestamp(old_phy_plan_timeout_ts_); - } - } - if (reset_autocommit_) { - if (OB_TMP_FAIL(session_info.set_autocommit(true))) { + // WHY WE NEED THIS + uint64_t cur_last_insert_id = session_info->get_local_last_insert_id(); + if (cur_last_insert_id != last_insert_id_) { + ObObj last_insert_id; + last_insert_id.set_uint64(last_insert_id_); + tmp_ret = session_info->update_sys_variable(SYS_VAR_LAST_INSERT_ID, last_insert_id); + if (OB_SUCCESS == tmp_ret && + OB_TMP_FAIL(session_info->update_sys_variable(SYS_VAR_IDENTITY, last_insert_id))) { + LOG_WARN("succ update last_insert_id, but fail to update identity", K(tmp_ret)); + } ret = COVER_SUCC(tmp_ret); - LOG_ERROR("restore autocommit value failed", K(tmp_ret), K(ret)); + } + session_info->set_has_exec_inner_dml(has_inner_dml_write_); + if (old_worker_timeout_ts_ != 0) { + THIS_WORKER.set_timeout_ts(old_worker_timeout_ts_); + if (OB_NOT_NULL(ctx.get_physical_plan_ctx())) { + ctx.get_physical_plan_ctx()->set_timeout_timestamp(old_phy_plan_timeout_ts_); + } + } + if (reset_autocommit_) { + if (OB_TMP_FAIL(session_info->set_autocommit(true))) { + ret = COVER_SUCC(tmp_ret); + LOG_ERROR("restore autocommit value failed", K(tmp_ret), K(ret)); + } } } return ret; @@ -208,18 +217,22 @@ void ObLockFuncContext::register_for_deadlock_(ObSQLSessionInfo &session_info, int ObLockFuncContext::open_inner_conn_() { int ret = OB_SUCCESS; - ObSQLSessionInfo *session = my_exec_ctx_->get_my_session(); + ObSQLSessionInfo *session = nullptr; + common::ObMySQLProxy *sql_proxy = nullptr; observer::ObInnerSQLConnection *inner_conn = nullptr; - if (OB_ISNULL(session) || OB_ISNULL(sql_proxy_ = my_exec_ctx_->get_sql_proxy())) { + if (OB_ISNULL(my_exec_ctx_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ObExecContext in ObLockFuncContext is null", K(ret)); + } else if (OB_ISNULL(session = my_exec_ctx_->get_my_session()) || OB_ISNULL(sql_proxy = my_exec_ctx_->get_sql_proxy())) { ret = OB_NOT_INIT; - LOG_WARN("session or sql_proxy is NULL", K(ret), KP(session), KP(sql_proxy_)); + LOG_WARN("session or sql_proxy in ObExecContext is NULL", K(ret), KP(session), KP(sql_proxy)); } else if (OB_NOT_NULL(inner_conn_) || OB_NOT_NULL(store_inner_conn_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("inner_conn_ or store_inner_conn_ should be null", K(ret), KP(inner_conn_), KP(store_inner_conn_)); } else if (FALSE_IT(store_inner_conn_ = static_cast(session->get_inner_conn()))) { } else if (FALSE_IT(session->set_inner_conn(nullptr))) { - } else if (OB_FAIL(ObInnerConnectionLockUtil::create_inner_conn(session, sql_proxy_, inner_conn))) { + } else if (OB_FAIL(ObInnerConnectionLockUtil::create_inner_conn(session, sql_proxy, inner_conn))) { LOG_WARN("create inner connection failed", K(ret), KPC(session)); } else if (OB_ISNULL(inner_conn)) { ret = OB_ERR_UNEXPECTED; @@ -242,26 +255,31 @@ int ObLockFuncContext::open_inner_conn_() int ObLockFuncContext::close_inner_conn_() { int ret = OB_SUCCESS; - ObSQLSessionInfo *session = my_exec_ctx_->get_my_session(); + ObSQLSessionInfo *session = nullptr; + common::ObMySQLProxy *sql_proxy = nullptr; - if (OB_ISNULL(sql_proxy_) || OB_ISNULL(inner_conn_)) { - ret = OB_NOT_INIT; - LOG_WARN("sql_proxy or inner_conn of session is NULL", K(ret), KP(sql_proxy_), KP(session), KP(inner_conn_)); - } else { - OZ (sql_proxy_->close(inner_conn_, true)); - } - if (OB_ISNULL(session)) { + if (OB_ISNULL(my_exec_ctx_)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("session is NULL", K(ret), KP(session)); - } else if (OB_NOT_NULL(inner_conn_) || OB_NOT_NULL(store_inner_conn_)) { - // 1. if inner_conn_ is not null, means that we have created inner_conn successfully before, so we must have already - // set store_inner_conn_ successfully, just restore it to the session. - // 2. if inner_conn_ is null, it's uncertain whether store_inner_conn_ has been set before. If store_inner_conn_ - // is not null, it must have been set. Otherwise, the inner_conn on the session may be null, or it may have existed - // with an error code before store_inner_conn_ being set. At this case, we do not set inner_conn on the session. - session->set_inner_conn(store_inner_conn_); + LOG_WARN("ObExecContext in ObLockFuncContext is null", K(ret)); + } else { + if (OB_ISNULL(sql_proxy = my_exec_ctx_->get_sql_proxy()) || OB_ISNULL(inner_conn_)) { + ret = OB_NOT_INIT; + LOG_WARN("sql_proxy or inner_conn of session is NULL", K(ret), KP(sql_proxy), KP(session), KP(inner_conn_)); + } else { + OZ (sql_proxy->close(inner_conn_, true)); + } + if (OB_ISNULL(session = my_exec_ctx_->get_my_session())) { + ret = OB_NOT_INIT; + LOG_WARN("session is NULL", K(ret), KP(session)); + } else if (OB_NOT_NULL(inner_conn_) || OB_NOT_NULL(store_inner_conn_)) { + // 1. if inner_conn_ is not null, means that we have created inner_conn successfully before, so we must have already + // set store_inner_conn_ successfully, just restore it to the session. + // 2. if inner_conn_ is null, it's uncertain whether store_inner_conn_ has been set before. If store_inner_conn_ + // is not null, it must have been set. Otherwise, the inner_conn on the session may be null, or it may have existed + // with an error code before store_inner_conn_ being set. At this case, we do not set inner_conn on the session. + session->set_inner_conn(store_inner_conn_); + } } - sql_proxy_ = nullptr; inner_conn_ = nullptr; store_inner_conn_ = nullptr; return ret; @@ -480,6 +498,7 @@ int ObLockFuncExecutor::remove_session_record_(ObLockFuncContext &ctx, ObTableLockOwnerID lock_owner; ObSqlString delete_sql; int64_t affected_rows = 0; + ObSQLSessionInfo *session = nullptr; OZ (check_owner_exist_(ctx, client_session_id, client_session_create_ts, owner_exist)); if (OB_SUCC(ret) && !owner_exist) { @@ -490,7 +509,9 @@ int ObLockFuncExecutor::remove_session_record_(ObLockFuncContext &ctx, table_name, client_session_id)); OZ (ctx.execute_write(delete_sql, affected_rows)); - OX (mark_lock_session_(ctx.session_info_, false)); + OV (OB_NOT_NULL(ctx.my_exec_ctx_), OB_INVALID_ARGUMENT); + OV (OB_NOT_NULL(session = ctx.my_exec_ctx_->get_my_session()), OB_INVALID_ARGUMENT); + OX (mark_lock_session_(session, false)); } return ret; } @@ -686,11 +707,11 @@ int ObGetLockExecutor::execute(ObExecContext &ctx, if (OB_SUCC(ret)) { SMART_VAR(ObLockFuncContext, stack_ctx1) { - OZ (stack_ctx1.init(*sess, ctx, timeout_us)); + OZ (stack_ctx1.init(ctx, timeout_us)); OZ (generate_lock_id_(stack_ctx1, lock_name, timeout_us, lock_id)); is_rollback = (OB_SUCCESS != ret); - if (OB_TMP_FAIL(stack_ctx1.destroy(ctx, *(ctx.get_my_session()), is_rollback))) { + if (OB_TMP_FAIL(stack_ctx1.destroy(ctx, is_rollback))) { LOG_WARN("stack ctx destroy failed", K(tmp_ret)); COVER_SUCC(tmp_ret); } @@ -698,7 +719,7 @@ int ObGetLockExecutor::execute(ObExecContext &ctx, } if (OB_SUCC(ret)) { SMART_VAR(ObLockFuncContext, stack_ctx2) { - OZ (stack_ctx2.init(*sess, ctx, timeout_us)); + OZ (stack_ctx2.init(ctx, timeout_us)); // only when connect by proxy should check client_session if (sess->is_obproxy_mode()) { OZ (check_client_ssid_(stack_ctx2, client_session_id, client_session_create_ts)); @@ -706,7 +727,7 @@ int ObGetLockExecutor::execute(ObExecContext &ctx, ret = OB_SUCCESS; // there're no same client_session_id records, continue } else { // TODO(yangyifei.yyf): some SQL can not redo, reroute may cause errors, so skip this step temporarily - // OZ (check_need_reroute_(stack_ctx1, sess, client_session_id, client_session_create_ts)); + // OZ (check_need_reroute_(stack_ctx1, client_session_id, client_session_create_ts)); } } OZ (update_session_table_(stack_ctx2, @@ -718,7 +739,7 @@ int ObGetLockExecutor::execute(ObExecContext &ctx, OX (mark_lock_session_(sess, true)); is_rollback = (OB_SUCCESS != ret); - if (OB_TMP_FAIL(stack_ctx2.destroy(ctx, *(ctx.get_my_session()), is_rollback))) { + if (OB_TMP_FAIL(stack_ctx2.destroy(ctx, is_rollback))) { LOG_WARN("stack ctx destroy failed", K(tmp_ret)); COVER_SUCC(tmp_ret); } @@ -869,7 +890,6 @@ int ObGetLockExecutor::update_session_table_(ObLockFuncContext &ctx, } int ObGetLockExecutor::check_need_reroute_(ObLockFuncContext &ctx, - sql::ObSQLSessionInfo *session, const uint32_t client_session_id, const uint64_t client_session_create_ts) { @@ -878,36 +898,44 @@ int ObGetLockExecutor::check_need_reroute_(ObLockFuncContext &ctx, ObAddr lock_session_addr; uint32_t lock_session_id = 0; int32_t sql_port = 0; - ObSqlCtx *sql_ctx = ctx.my_exec_ctx_->get_sql_ctx(); + ObSqlCtx *sql_ctx = nullptr; + ObSQLSessionInfo *session = nullptr; - if (!session->is_lock_session()) { - OZ (get_lock_session_(ctx, client_session_id, client_session_create_ts, lock_session_addr, lock_session_id)); - // no lock_session in this client, continue - if (OB_EMPTY_RESULT == ret) { - ret = OB_SUCCESS; - } else { - OV (lock_session_addr == GCTX.self_addr(), OB_ERR_PROXY_REROUTE, lock_session_addr, GCTX.self_addr()); - // can not reroute in one observer, so just return OB_SUCCESS - OV (lock_session_id == session->get_sessid(), OB_SUCCESS, lock_session_id, session->get_sessid()); - // to avoid this session wasn't marked as lock_session before - if (lock_session_addr == GCTX.self_addr() && lock_session_id == session->get_sessid()) { - mark_lock_session_(session, true); + OV (OB_NOT_NULL(ctx.my_exec_ctx_), OB_INVALID_ARGUMENT); + OV (OB_NOT_NULL(sql_ctx = ctx.my_exec_ctx_->get_sql_ctx()) && + OB_NOT_NULL(session = ctx.my_exec_ctx_->get_my_session()), + OB_NOT_INIT); + OX ( + if (!session->is_lock_session()) { + OZ (get_lock_session_(ctx, client_session_id, client_session_create_ts, lock_session_addr, lock_session_id)); + // no lock_session in this client, continue + if (OB_EMPTY_RESULT == ret) { + ret = OB_SUCCESS; + // get lock_session successfully, compare with current session + } else if (OB_SUCCESS == ret) { + OV (lock_session_addr == GCTX.self_addr(), OB_ERR_PROXY_REROUTE, lock_session_addr, GCTX.self_addr()); + // can not reroute in one observer, so just return OB_SUCCESS + OV (lock_session_id == session->get_sessid(), OB_SUCCESS, lock_session_id, session->get_sessid()); + // to avoid this session wasn't marked as lock_session before + if (lock_session_addr == GCTX.self_addr() && lock_session_id == session->get_sessid()) { + mark_lock_session_(session, true); + } + } + + if (OB_ERR_PROXY_REROUTE == ret) { + if (OB_TMP_FAIL(get_sql_port_(ctx, lock_session_addr, sql_port))) { + LOG_WARN("can not get sql_port, reroute to this server this time", K(tmp_ret), K(lock_session_addr), K(GCTX.self_addr())); + sql_ctx->get_or_create_reroute_info()->server_ = GCTX.self_addr(); + sql_ctx->get_reroute_info()->server_.set_port(GCONF.mysql_port); + sql_ctx->get_reroute_info()->for_session_reroute_ = true; + } else { + sql_ctx->get_or_create_reroute_info()->server_ = lock_session_addr; + sql_ctx->get_reroute_info()->server_.set_port(sql_port); + sql_ctx->get_reroute_info()->for_session_reroute_ = true; + } } } - - if (OB_ERR_PROXY_REROUTE == ret) { - if (OB_TMP_FAIL(get_sql_port_(ctx, lock_session_addr, sql_port))) { - LOG_WARN("can not get sql_port, reroute to this server this time", K(tmp_ret), K(lock_session_addr), K(GCTX.self_addr())); - sql_ctx->get_or_create_reroute_info()->server_ = GCTX.self_addr(); - sql_ctx->get_reroute_info()->server_.set_port(GCONF.mysql_port); - sql_ctx->get_reroute_info()->for_session_reroute_ = true; - } else { - sql_ctx->get_or_create_reroute_info()->server_ = lock_session_addr; - sql_ctx->get_reroute_info()->server_.set_port(sql_port); - sql_ctx->get_reroute_info()->for_session_reroute_ = true; - } - } - } + ); return ret; } @@ -1014,7 +1042,7 @@ int ObReleaseLockExecutor::execute(ObExecContext &ctx, SMART_VAR(ObLockFuncContext, stack_ctx) { client_session_id = ctx.get_my_session()->get_client_sessid(); client_session_create_ts = ctx.get_my_session()->get_client_create_time(); - OZ (stack_ctx.init(*(ctx.get_my_session()), ctx)); + OZ (stack_ctx.init(ctx)); if (OB_SUCC(ret)) { ObSQLSessionInfo *session = GET_MY_SESSION(ctx); ObTxDesc *tx_desc = session->get_tx_desc(); @@ -1069,7 +1097,7 @@ int ObReleaseLockExecutor::execute(ObExecContext &ctx, } } is_rollback = (OB_SUCCESS != ret); - if (OB_TMP_FAIL(stack_ctx.destroy(ctx, *(ctx.get_my_session()), is_rollback))) { + if (OB_TMP_FAIL(stack_ctx.destroy(ctx, is_rollback))) { LOG_WARN("stack ctx destroy failed", K(tmp_ret)); COVER_SUCC(tmp_ret); } @@ -1149,12 +1177,12 @@ int ObReleaseAllLockExecutor::execute_(ObExecContext &ctx, OZ (ObLockFuncContext::valid_execute_context(ctx)); if (OB_SUCC(ret)) { SMART_VAR(ObLockFuncContext, stack_ctx) { - OZ (stack_ctx.init(*(ctx.get_my_session()), ctx)); + OZ (stack_ctx.init(ctx)); if (OB_SUCC(ret)) { ObSQLSessionInfo *session = GET_MY_SESSION(ctx); ObTxDesc *tx_desc = session->get_tx_desc(); ObTxParam tx_param; - if (ctx.get_my_session()->is_obproxy_mode()) { + if (OB_NOT_NULL(session) && session->is_obproxy_mode()) { OZ (check_client_ssid_(stack_ctx, client_session_id, client_session_create_ts)); if (OB_EMPTY_RESULT == ret) { release_cnt = LOCK_NOT_OWN_RELEASE_CNT; @@ -1170,7 +1198,7 @@ int ObReleaseAllLockExecutor::execute_(ObExecContext &ctx, OZ (remove_session_record_(stack_ctx, client_session_id, client_session_create_ts)); } is_rollback = (OB_SUCCESS != ret); - if (OB_TMP_FAIL(stack_ctx.destroy(ctx, *(ctx.get_my_session()), is_rollback))) { + if (OB_TMP_FAIL(stack_ctx.destroy(ctx, is_rollback))) { LOG_WARN("stack ctx destroy failed", K(tmp_ret)); COVER_SUCC(tmp_ret); } @@ -1197,7 +1225,7 @@ int ObReleaseAllLockExecutor::execute_(ObExecContext &ctx, OZ (ObLockFuncContext::valid_execute_context(ctx)); if (OB_SUCC(ret)) { SMART_VAR(ObLockFuncContext, stack_ctx) { - OZ (stack_ctx.init(*(ctx.get_my_session()), ctx)); + OZ (stack_ctx.init(ctx)); if (OB_SUCC(ret)) { ObSQLSessionInfo *session = GET_MY_SESSION(ctx); ObTxDesc *tx_desc = session->get_tx_desc(); @@ -1207,7 +1235,7 @@ int ObReleaseAllLockExecutor::execute_(ObExecContext &ctx, OZ (remove_session_record_(stack_ctx, client_session_id, 0)); } is_rollback = (OB_SUCCESS != ret); - if (OB_TMP_FAIL(stack_ctx.destroy(ctx, *(ctx.get_my_session()), is_rollback))) { + if (OB_TMP_FAIL(stack_ctx.destroy(ctx, is_rollback))) { LOG_WARN("stack ctx destroy failed", K(tmp_ret)); COVER_SUCC(tmp_ret); } diff --git a/src/storage/tablelock/ob_lock_func_executor.h b/src/storage/tablelock/ob_lock_func_executor.h index 9a7843e33..86c9570b3 100644 --- a/src/storage/tablelock/ob_lock_func_executor.h +++ b/src/storage/tablelock/ob_lock_func_executor.h @@ -64,21 +64,17 @@ public: tenant_id_ = 0; database_id_ = OB_INVALID_ID; database_name_.reset(); - sql_proxy_ = nullptr; inner_conn_ = nullptr; store_inner_conn_ = nullptr; - session_info_ = nullptr; my_exec_ctx_ = nullptr; saved_session_.reset(); } - int init(sql::ObSQLSessionInfo &session_info, - sql::ObExecContext &ctx, + int init(sql::ObExecContext &ctx, const int64_t timeout_us = 0); int destroy(sql::ObExecContext &ctx, - sql::ObSQLSessionInfo &session_info, bool is_rollback); - bool is_inited() { return session_info_ != NULL; } + bool is_inited() { return my_exec_ctx_ != NULL; } static int valid_execute_context(sql::ObExecContext &ctx); int execute_write(const ObSqlString &sql, int64_t &affected_rows); @@ -105,10 +101,8 @@ private: uint64_t tenant_id_; uint64_t database_id_; ObSqlString database_name_; - common::ObMySQLProxy *sql_proxy_; observer::ObInnerSQLConnection *inner_conn_; observer::ObInnerSQLConnection *store_inner_conn_; - sql::ObSQLSessionInfo *session_info_; sql::ObExecContext *my_exec_ctx_; //my exec context sql::ObBasicSessionInfo::TransSavedValue saved_session_; }; @@ -190,7 +184,6 @@ private: const uint64_t client_session_create_ts, const uint32_t server_session_id); int check_need_reroute_(ObLockFuncContext &ctx, - sql::ObSQLSessionInfo *session, const uint32_t client_session_id, const uint64_t client_session_create_ts); int get_lock_session_(ObLockFuncContext &ctx, From 3303c6860617ebd35ad8e6b56e899197a60a1058 Mon Sep 17 00:00:00 2001 From: ZenoWang Date: Tue, 27 Aug 2024 15:17:27 +0000 Subject: [PATCH 242/249] Add debug log for memtable destroy --- src/storage/memtable/ob_memtable.cpp | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/storage/memtable/ob_memtable.cpp b/src/storage/memtable/ob_memtable.cpp index 668857797..62992121d 100644 --- a/src/storage/memtable/ob_memtable.cpp +++ b/src/storage/memtable/ob_memtable.cpp @@ -249,10 +249,12 @@ void ObMemtable::destroy() ObTimeGuard time_guard("ObMemtable::destroy()", 100 * 1000); int ret = OB_SUCCESS; if (is_inited_) { - const int64_t cost_time = ObTimeUtility::current_time() - mt_stat_.release_time_; - if (cost_time > 1 * 1000 * 1000) { - STORAGE_LOG(WARN, "it costs too much time from release to destroy", K(cost_time), KP(this)); + // check release to destroy + const int64_t release_to_destroy_time = ObTimeUtility::current_time() - mt_stat_.release_time_; + if (release_to_destroy_time > 1LL * 1000LL * 1000LL /* 1 second */) { + STORAGE_LOG(WARN, "it costs too much time from release to destroy", K(release_to_destroy_time), KP(this)); } + set_allow_freeze(true); STORAGE_LOG(INFO, "memtable destroyed", K(*this)); time_guard.click(); @@ -279,14 +281,18 @@ void ObMemtable::destroy() int ObMemtable::safe_to_destroy(bool &is_safe) { int ret = OB_SUCCESS; + + // check frozen to flush + const int64_t frozen_to_flush_time = mt_stat_.create_flush_dag_time_ - mt_stat_.ready_for_flush_time_; + if (frozen_to_flush_time > 60LL * 1000LL * 1000LL /* 60 seconds */) { + STORAGE_LOG(WARN, "it costs too much time from forzen to flush", K(frozen_to_flush_time), KP(this)); + } + int64_t ref_cnt = get_ref(); int64_t write_ref_cnt = get_write_ref(); int64_t unsubmitted_cnt = get_unsubmitted_cnt(); - is_safe = (0 == ref_cnt && 0 == write_ref_cnt); - if (is_safe) { - is_safe = (0 == unsubmitted_cnt); - } + is_safe = (0 == ref_cnt && 0 == write_ref_cnt && 0 == unsubmitted_cnt); if (is_safe) { // In scenarios where the memtable is forcefully remove (such as when the From f7e933449022679c9fc02ee6329a1117f582c96c Mon Sep 17 00:00:00 2001 From: obdev Date: Tue, 27 Aug 2024 15:23:32 +0000 Subject: [PATCH 243/249] optimize dag scheduler thread allocation --- .../oblib/src/lib/utility/ob_tracepoint_def.h | 1 + .../scheduler/ob_tenant_dag_scheduler.cpp | 70 +++++++++++++++---- src/share/scheduler/ob_tenant_dag_scheduler.h | 23 +++++- 3 files changed, 78 insertions(+), 16 deletions(-) diff --git a/deps/oblib/src/lib/utility/ob_tracepoint_def.h b/deps/oblib/src/lib/utility/ob_tracepoint_def.h index 6dd6c2767..63e094b6a 100644 --- a/deps/oblib/src/lib/utility/ob_tracepoint_def.h +++ b/deps/oblib/src/lib/utility/ob_tracepoint_def.h @@ -439,6 +439,7 @@ GLOBAL_ERRSIM_POINT_DEF(748, EN_COMPACTION_ITER_SET_BATCH_CNT, ""); // please add new trace point after 750 GLOBAL_ERRSIM_POINT_DEF(751, EN_SESSION_LEAK_COUNT_THRESHOLD, "used to control the threshold of report session leak ERROR"); GLOBAL_ERRSIM_POINT_DEF(760, EN_DISABLE_TABLET_MINOR_MERGE, "used to stop scheduling minor merge"); +GLOBAL_ERRSIM_POINT_DEF(761, EN_FAST_RECLAIM_THREAD, "used to speed up reclaiming thread"); GLOBAL_ERRSIM_POINT_DEF(800, EN_END_PARTICIPANT, ""); // compaction 801 - 899 diff --git a/src/share/scheduler/ob_tenant_dag_scheduler.cpp b/src/share/scheduler/ob_tenant_dag_scheduler.cpp index 120b53d9a..2b5f1eaec 100644 --- a/src/share/scheduler/ob_tenant_dag_scheduler.cpp +++ b/src/share/scheduler/ob_tenant_dag_scheduler.cpp @@ -3900,7 +3900,6 @@ ObTenantDagScheduler::ObTenantDagScheduler() loop_waiting_dag_list_period_(0), total_worker_cnt_(0), work_thread_num_(0), - default_work_thread_num_(0), total_running_task_cnt_(0), scheduled_task_cnt_(0), scheduler_sync_(), @@ -3989,7 +3988,7 @@ int ObTenantDagScheduler::init( loop_waiting_dag_list_period_ = loop_waiting_list_period; dag_limit_ = dag_limit; compaction_dag_limit_ = dag_limit; - work_thread_num_ = default_work_thread_num_ = 0; + work_thread_num_ = 0; MEMSET(dag_cnts_, 0, sizeof(dag_cnts_)); MEMSET(running_dag_cnts_, 0, sizeof(running_dag_cnts_)); MEMSET(added_dag_cnts_, 0, sizeof(added_dag_cnts_)); @@ -4005,14 +4004,6 @@ int ObTenantDagScheduler::init( COMMON_LOG(WARN, "failed to init prio_sche_", K(ret), K(dag_limit)); } else { work_thread_num_ += prio_sche_[i].get_limit(); - default_work_thread_num_ += prio_sche_[i].get_limit(); - } - } - - // create dag workers - for (int64_t i = 0; OB_SUCC(ret) && i < work_thread_num_; ++i) { - if (OB_FAIL(create_worker())) { - COMMON_LOG(WARN, "failed to create worker", K(ret)); } } @@ -4021,7 +4012,7 @@ int ObTenantDagScheduler::init( } else { is_inited_ = true; dump_dag_status(); - COMMON_LOG(INFO, "ObTenantDagScheduler is inited", K(ret), K_(work_thread_num), K_(default_work_thread_num)); + COMMON_LOG(INFO, "ObTenantDagScheduler is inited", K(ret), K(work_thread_num_)); } if (!is_inited_) { @@ -4077,6 +4068,7 @@ void ObTenantDagScheduler::reset() compaction_dag_limit_ = 0; total_worker_cnt_ = 0; work_thread_num_ = 0; + reclaim_util_.reset(); total_running_task_cnt_ = 0; scheduled_task_cnt_ = 0; MEMSET(dag_cnts_, 0, sizeof(dag_cnts_)); @@ -4574,6 +4566,54 @@ void ObTenantDagScheduler::run1() } } +void ObReclaimUtil::reset() +{ + total_periodic_running_worker_cnt_ = 0; + check_worker_loop_times_ = 0; +} + +int64_t ObReclaimUtil::compute_expected_reclaim_worker_cnt( + const int64_t total_running_task_cnt, + const int64_t free_worker_cnt, + const int64_t total_worker_cnt) +{ + int64_t expected_reclaim_worker_cnt = 0; + if (free_worker_cnt > 0) { + total_periodic_running_worker_cnt_ = total_periodic_running_worker_cnt_ + total_running_task_cnt; + ++check_worker_loop_times_; + bool is_always_triggered = false; +#ifdef ERRSIM + #define ADAPTIVE_RECLAIM_ERRSIM(tracepoint) \ + int ret = OB_SUCCESS; \ + do { \ + if (OB_SUCC(ret)) { \ + ret = OB_E((EventTable::tracepoint)) OB_SUCCESS; \ + if (OB_FAIL(ret)) { \ + ret = OB_SUCCESS; \ + is_always_triggered = true; \ + } \ + } \ + } while(0); + ADAPTIVE_RECLAIM_ERRSIM(EN_FAST_RECLAIM_THREAD); + #undef ADAPTIVE_RECLAIM_ERRSIM +#endif + if (REACH_TENANT_TIME_INTERVAL(CHECK_USING_WOKRER_INTERVAL) || is_always_triggered) { + const int64_t avg_periodic_running_worker_cnt = total_periodic_running_worker_cnt_ / check_worker_loop_times_; + if (total_running_task_cnt < avg_periodic_running_worker_cnt) { + // reclaim one fifth of the workers each time + expected_reclaim_worker_cnt = MAX(free_worker_cnt * 0.2 , 1); + } else if (total_running_task_cnt == avg_periodic_running_worker_cnt) { + if (0 == total_running_task_cnt) { + expected_reclaim_worker_cnt = MAX(free_worker_cnt * 0.2 , 1);; + } + } + reset(); + } + } + return expected_reclaim_worker_cnt; +} + + void ObTenantDagScheduler::notify() { ObThreadCondGuard cond_guard(scheduler_sync_); @@ -4797,14 +4837,15 @@ int ObTenantDagScheduler::try_reclaim_threads() int ret = OB_SUCCESS; ObTenantDagWorker *worker2delete = NULL; int32_t free_cnt = 0; - while (total_worker_cnt_ > work_thread_num_ && !free_workers_.is_empty()) { + int64_t expected_reclaim_worker_cnt = reclaim_util_.compute_expected_reclaim_worker_cnt(total_running_task_cnt_, free_workers_.get_size(), total_worker_cnt_); + while (free_cnt < expected_reclaim_worker_cnt) { worker2delete = free_workers_.remove_first(); ob_delete(worker2delete); --total_worker_cnt_; ++free_cnt; } if (free_cnt > 0) { - COMMON_LOG(INFO, "reclaim threads", K(free_cnt), K_(total_worker_cnt), K_(work_thread_num)); + COMMON_LOG(INFO, "reclaim threads", K(free_cnt), K_(total_worker_cnt), K(work_thread_num_)); } return ret; } @@ -4880,8 +4921,7 @@ int ObTenantDagScheduler::set_thread_score(const int64_t priority, const int64_t } else { COMMON_LOG(INFO, "set thread score successfully", K(score), "prio", OB_DAG_PRIOS[priority].dag_prio_str_, - "limits_", new_val, K_(work_thread_num), - K_(default_work_thread_num)); + "limits_", new_val, K_(work_thread_num)); } } } diff --git a/src/share/scheduler/ob_tenant_dag_scheduler.h b/src/share/scheduler/ob_tenant_dag_scheduler.h index 1343f8c67..3917eb66d 100644 --- a/src/share/scheduler/ob_tenant_dag_scheduler.h +++ b/src/share/scheduler/ob_tenant_dag_scheduler.h @@ -852,6 +852,27 @@ private: int64_t dag_net_cnts_[ObDagNetType::DAG_NET_TYPE_MAX]; // lock by dag_net_map_lock_ }; +class ObReclaimUtil +{ + +public: + ObReclaimUtil() + : total_periodic_running_worker_cnt_(0), + check_worker_loop_times_(0) + {} + ~ObReclaimUtil(){} + int64_t compute_expected_reclaim_worker_cnt( + const int64_t total_running_task_cnt, + const int64_t free_worker_cnt, + const int64_t total_worker_cnt); + void reset(); + +public: + int64_t total_periodic_running_worker_cnt_; + int64_t check_worker_loop_times_; + static const int64_t CHECK_USING_WOKRER_INTERVAL = 60 * 1000L* 1000L; // 1min +}; + class ObDagPrioScheduler { public: @@ -1203,7 +1224,6 @@ private: int64_t loop_waiting_dag_list_period_; // only set in init/destroy int64_t total_worker_cnt_; // lock by scheduler_sync_ int64_t work_thread_num_; // lock by scheduler_sync_ - int64_t default_work_thread_num_; // only set in init/destroy int64_t total_running_task_cnt_; // atomic value int64_t scheduled_task_cnt_; // atomic value // interval scheduled task count int64_t dag_cnts_[ObDagType::DAG_TYPE_MAX]; // just for showing // atomic value @@ -1212,6 +1232,7 @@ private: int64_t scheduled_dag_cnts_[ObDagType::DAG_TYPE_MAX]; // atomic value // interval scheduled dag count int64_t scheduled_task_cnts_[ObDagType::DAG_TYPE_MAX]; // atomic value // interval scheduled task count int64_t scheduled_data_size_[ObDagType::DAG_TYPE_MAX]; // atomic value // interval scheduled data size + ObReclaimUtil reclaim_util_; // util to help adaptively reclaim worker common::ObThreadCond scheduler_sync_; // Make sure the lock is inside if there are nested locks lib::MemoryContext mem_context_; lib::MemoryContext ha_mem_context_; From b10924a9cd9dd8acdb8044cad4b101c438585b03 Mon Sep 17 00:00:00 2001 From: obdev Date: Tue, 27 Aug 2024 15:36:06 +0000 Subject: [PATCH 244/249] fix cache miss count of tablet_table_cache and storage_meta_cache in __all_virtual_kvcache_info --- src/observer/virtual_table/ob_information_kvcache_table.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/observer/virtual_table/ob_information_kvcache_table.cpp b/src/observer/virtual_table/ob_information_kvcache_table.cpp index 8a20d2fc3..50717148a 100644 --- a/src/observer/virtual_table/ob_information_kvcache_table.cpp +++ b/src/observer/virtual_table/ob_information_kvcache_table.cpp @@ -199,6 +199,10 @@ int ObInfoSchemaKvCacheTable::set_diagnose_info(ObKVCacheInst *inst, ObDiagnoseT inst->status_.total_miss_cnt_ = GLOBAL_EVENT_GET(ObStatEventIds::OPT_DS_STAT_CACHE_MISS); } else if (0 == strcmp(inst->status_.config_->cache_name_,"log_kv_cache")) { inst->status_.total_miss_cnt_ = GLOBAL_EVENT_GET(ObStatEventIds::LOG_KV_CACHE_MISS); + } else if (0 == strcmp(inst->status_.config_->cache_name_,"tablet_table_cache")) { + inst->status_.total_miss_cnt_ = GLOBAL_EVENT_GET(ObStatEventIds::TABLET_CACHE_MISS); + } else if (0 == strcmp(inst->status_.config_->cache_name_,"storage_meta_cache")) { + inst->status_.total_miss_cnt_ = GLOBAL_EVENT_GET(ObStatEventIds::STORAGE_META_CACHE_MISS); } return ret; From c3c60dd20c9b35848589b9b25fe5b3bcc9228368 Mon Sep 17 00:00:00 2001 From: chinaxing Date: Tue, 27 Aug 2024 16:17:07 +0000 Subject: [PATCH 245/249] [master][tx-route] keep tx-state and op_sn will not go backward --- src/storage/tx/ob_tx_free_route_state.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/storage/tx/ob_tx_free_route_state.cpp b/src/storage/tx/ob_tx_free_route_state.cpp index 4e0e65aff..c7d8f2321 100644 --- a/src/storage/tx/ob_tx_free_route_state.cpp +++ b/src/storage/tx/ob_tx_free_route_state.cpp @@ -36,11 +36,21 @@ namespace transaction { // recover flags which should not be overwriten #define PRE_DYNAMIC_DECODE \ FLAG save_flags = flags_; \ - int save_abort_cause = abort_cause_; + int save_abort_cause = abort_cause_; \ + State save_state = state_; \ + uint64_t save_op_sn = op_sn_; + // for txn start node, if current abort_cause was set, use current +// keep the state and op_sn will not go backwards #define POST_DYNAMIC_DECODE \ flags_ = save_flags.update_with(flags_); \ - abort_cause_ = save_abort_cause ?: abort_cause_; + abort_cause_ = save_abort_cause ?: abort_cause_; \ + if (save_state > state_) { \ + state_ = save_state; \ + } \ + if (save_op_sn > op_sn_) { \ + op_sn_ = save_op_sn; \ + } #define PRE_EXTRA_DECODE #define POST_EXTRA_DECODE \ From 478d5d29de9593bddff604eec8b625b712657a8d Mon Sep 17 00:00:00 2001 From: ZenoWang Date: Tue, 27 Aug 2024 16:23:19 +0000 Subject: [PATCH 246/249] fix test_transfer_tx_data --- .../test_transfer_with_smaller_tx_data.cpp | 11 +++++--- src/storage/ls/ob_ls.h | 1 + src/storage/ls/ob_ls_transfer_status.cpp | 3 ++- src/storage/tx/ob_keep_alive_ls_handler.cpp | 11 ++++++++ src/storage/tx/ob_keep_alive_ls_handler.h | 1 + src/storage/tx_table/ob_tx_data_table.cpp | 7 ------ src/storage/tx_table/ob_tx_data_table.h | 2 -- src/storage/tx_table/ob_tx_table.cpp | 25 ++++++++++++++++++- src/storage/tx_table/ob_tx_table.h | 5 ++++ 9 files changed, 51 insertions(+), 15 deletions(-) diff --git a/mittest/simple_server/test_transfer_with_smaller_tx_data.cpp b/mittest/simple_server/test_transfer_with_smaller_tx_data.cpp index b5b079ca1..707e4d3de 100644 --- a/mittest/simple_server/test_transfer_with_smaller_tx_data.cpp +++ b/mittest/simple_server/test_transfer_with_smaller_tx_data.cpp @@ -30,6 +30,7 @@ namespace storage { int64_t ObTxTable::UPDATE_MIN_START_SCN_INTERVAL = 0; + int ObTransferHandler::wait_src_ls_advance_weak_read_ts_( const share::ObTransferTaskInfo &task_info, ObTimeoutCtx &timeout_ctx) @@ -82,7 +83,6 @@ public: WRITE_SQL_BY_CONN(connection, "set GLOBAL ob_query_timeout = 10000000000"); WRITE_SQL_BY_CONN(connection, "alter system set enable_early_lock_release = False;"); WRITE_SQL_BY_CONN(connection, "alter system set undo_retention = 1800;"); - WRITE_SQL_BY_CONN(connection, "alter system set partition_balance_schedule_interval = '10s';"); sleep(5); } @@ -164,12 +164,15 @@ public: MTL_SWITCH(tenant_id) { TRANS_LOG(INFO, "worker to do partition_balance"); auto b_svr = MTL(rootserver::ObTenantBalanceService*); + b_svr->stop(); b_svr->reset(); int64_t job_cnt = 0; int64_t start_time = OB_INVALID_TIMESTAMP, finish_time = OB_INVALID_TIMESTAMP; ObBalanceJob job; if (OB_FAIL(b_svr->gather_stat_())) { TRANS_LOG(WARN, "failed to gather stat", KR(ret)); + } else if (OB_FAIL(b_svr->gather_ls_status_stat(tenant_id, b_svr->ls_array_))) { + TRANS_LOG(WARN, "failed to gather ls stat", KR(ret)); } else if (OB_FAIL(ObBalanceJobTableOperator::get_balance_job( tenant_id, false, *GCTX.sql_proxy_, job, start_time, finish_time))) { if (OB_ENTRY_NOT_EXIST == ret) { @@ -408,7 +411,7 @@ TEST_F(ObTransferWithSmallerStartSCN, smaller_start_scn) fprintf(stdout, "end update upper info the second time %lu\n", second_min_start_scn); TRANS_LOG(INFO, "end update upper info the second time"); - ASSERT_EQ(true, first_min_start_scn > second_min_start_scn); + ASSERT_GT(first_min_start_scn, second_min_start_scn); min_start_scn_in_tx_data.set_max(); fprintf(stdout, "start get min start in tx data table second time\n"); @@ -418,8 +421,8 @@ TEST_F(ObTransferWithSmallerStartSCN, smaller_start_scn) fprintf(stdout, "end get min start in tx data table second time, %lu\n", min_start_scn_in_tx_data.val_); TRANS_LOG(INFO, "end get min start in tx data table second time"); - ASSERT_EQ(true, first_min_start_scn > second_min_start_scn); - ASSERT_EQ(true, first_min_start_scn_in_tx_data > second_min_start_scn_in_tx_data); + ASSERT_GT(first_min_start_scn, second_min_start_scn); + ASSERT_GT(first_min_start_scn_in_tx_data, second_min_start_scn_in_tx_data); inject_tx_fault_helper.release(); th.join(); diff --git a/src/storage/ls/ob_ls.h b/src/storage/ls/ob_ls.h index 3f93d1c64..403093325 100644 --- a/src/storage/ls/ob_ls.h +++ b/src/storage/ls/ob_ls.h @@ -926,6 +926,7 @@ public: // ObDataCheckpoint interface: DELEGATE_WITH_RET(data_checkpoint_, get_freezecheckpoint_info, int); DELEGATE_WITH_RET(keep_alive_ls_handler_, get_min_start_scn, void); + DELEGATE_WITH_RET(keep_alive_ls_handler_, clear_keep_alive_smaller_scn_info, void); // update tablet table store here do not using Macro because need lock ls and tablet // update table store for tablet diff --git a/src/storage/ls/ob_ls_transfer_status.cpp b/src/storage/ls/ob_ls_transfer_status.cpp index 95df82e60..e673bf1ae 100644 --- a/src/storage/ls/ob_ls_transfer_status.cpp +++ b/src/storage/ls/ob_ls_transfer_status.cpp @@ -245,7 +245,8 @@ int ObLSTransferStatus::enable_upper_trans_calculation_(const share::SCN op_scn) ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "tx data table in tx table is nullptr.", K(ret)); } else { - tx_table->enable_upper_trans_calculation(op_scn); + (void)ls_->clear_keep_alive_smaller_scn_info(); + (void)tx_table->enable_upper_trans_calculation(op_scn); TRANS_LOG(INFO, "enable upper trans calculation", KPC(ls_), K(guard), KPC(this)); } diff --git a/src/storage/tx/ob_keep_alive_ls_handler.cpp b/src/storage/tx/ob_keep_alive_ls_handler.cpp index f61c6c31c..b6ea4d199 100644 --- a/src/storage/tx/ob_keep_alive_ls_handler.cpp +++ b/src/storage/tx/ob_keep_alive_ls_handler.cpp @@ -14,6 +14,8 @@ #include "logservice/ob_log_handler.h" #include "storage/tx/ob_ts_mgr.h" +#define USING_LOG_PREFIX TRANS + namespace oceanbase { @@ -94,6 +96,14 @@ void ObKeepAliveLSHandler::reset() stat_info_.reset(); } +void ObKeepAliveLSHandler::clear_keep_alive_smaller_scn_info() +{ + SpinWLockGuard guard(lock_); + FLOG_INFO("[Keep Alive] clear keep alive ls info", K(ls_id_), K(tmp_keep_alive_info_), K(durable_keep_alive_info_)); + tmp_keep_alive_info_.reset(); + durable_keep_alive_info_.reset(); +} + int ObKeepAliveLSHandler::try_submit_log(const SCN &min_start_scn, MinStartScnStatus min_start_status) { int ret = OB_SUCCESS; @@ -150,6 +160,7 @@ int ObKeepAliveLSHandler::on_success() durable_keep_alive_info_.replace(tmp_keep_alive_info_); stat_info_.stat_keepalive_info_ = durable_keep_alive_info_; + ATOMIC_STORE(&is_busy_,false); return ret; diff --git a/src/storage/tx/ob_keep_alive_ls_handler.h b/src/storage/tx/ob_keep_alive_ls_handler.h index e9414b38f..bed743249 100644 --- a/src/storage/tx/ob_keep_alive_ls_handler.h +++ b/src/storage/tx/ob_keep_alive_ls_handler.h @@ -154,6 +154,7 @@ public: void reset(); int try_submit_log(const share::SCN &min_start_scn, MinStartScnStatus status); + void clear_keep_alive_smaller_scn_info(); void print_stat_info(); public: diff --git a/src/storage/tx_table/ob_tx_data_table.cpp b/src/storage/tx_table/ob_tx_data_table.cpp index 297c08431..eb017eeb5 100644 --- a/src/storage/tx_table/ob_tx_data_table.cpp +++ b/src/storage/tx_table/ob_tx_data_table.cpp @@ -69,7 +69,6 @@ int ObTxDataTable::init(ObLS *ls, ObTxCtxTable *tx_ctx_table) memtable_mgr_ = static_cast(memtable_mgr_handle.get_memtable_mgr()); tx_ctx_table_ = tx_ctx_table; tablet_id_ = LS_TX_DATA_TABLET; - calc_upper_trans_is_disabled_ = false; latest_transfer_scn_.reset(); is_inited_ = true; @@ -181,7 +180,6 @@ void ObTxDataTable::reset() tx_ctx_table_ = nullptr; calc_upper_trans_version_cache_.reset(); memtables_cache_.reuse(); - calc_upper_trans_is_disabled_ = false; latest_transfer_scn_.reset(); is_started_ = false; is_inited_ = false; @@ -236,7 +234,6 @@ int ObTxDataTable::online() calc_upper_trans_version_cache_.reset(); } latest_transfer_scn_.reset(); - ATOMIC_STORE(&calc_upper_trans_is_disabled_, false); is_started_ = true; } @@ -774,8 +771,6 @@ int ObTxDataTable::get_upper_trans_version_before_given_scn(const SCN sstable_en if (IS_NOT_INIT) { ret = OB_NOT_INIT; STORAGE_LOG(WARN, "The tx data table is not inited.", KR(ret)); - } else if (ATOMIC_LOAD(&calc_upper_trans_is_disabled_)) { - skip_calc = true; } else if (true == (skip_calc = skip_this_sstable_end_scn_(sstable_end_scn))) { // there is a start_scn of running transactions is smaller than the sstable_end_scn } else { @@ -1234,7 +1229,6 @@ int ObTxDataTable::get_start_tx_scn(SCN &start_tx_scn) void ObTxDataTable::disable_upper_trans_calculation() { - ATOMIC_STORE(&calc_upper_trans_is_disabled_, true); { TCWLockGuard lock_guard(calc_upper_trans_version_cache_.lock_); calc_upper_trans_version_cache_.reset(); @@ -1252,7 +1246,6 @@ void ObTxDataTable::enable_upper_trans_calculation(const share::SCN latest_trans } else { latest_transfer_scn_ = latest_transfer_scn; } - ATOMIC_STORE(&calc_upper_trans_is_disabled_, false); } int ObTxDataTable::dump_single_tx_data_2_text(const int64_t tx_id_int, FILE *fd) diff --git a/src/storage/tx_table/ob_tx_data_table.h b/src/storage/tx_table/ob_tx_data_table.h index aa49f6e29..c7b097590 100644 --- a/src/storage/tx_table/ob_tx_data_table.h +++ b/src/storage/tx_table/ob_tx_data_table.h @@ -108,7 +108,6 @@ public: // ObTxDataTable ObTxDataTable() : is_inited_(false), is_started_(false), - calc_upper_trans_is_disabled_(false), latest_transfer_scn_(), ls_id_(), tablet_id_(0), @@ -316,7 +315,6 @@ private: static const int64_t LS_TX_DATA_SCHEMA_COLUMN_CNT = 5; bool is_inited_; bool is_started_; - bool calc_upper_trans_is_disabled_; share::SCN latest_transfer_scn_; share::ObLSID ls_id_; ObTabletID tablet_id_; diff --git a/src/storage/tx_table/ob_tx_table.cpp b/src/storage/tx_table/ob_tx_table.cpp index 1eb52de0c..d3b258877 100644 --- a/src/storage/tx_table/ob_tx_table.cpp +++ b/src/storage/tx_table/ob_tx_table.cpp @@ -66,6 +66,7 @@ int ObTxTable::init(ObLS *ls) read_tx_data_table_cnt_ = 0; recycle_scn_cache_.reset(); LOG_INFO("init tx table successfully", K(ret), K(ls->get_ls_id())); + calc_upper_trans_is_disabled_ = false; is_inited_ = true; } if (OB_FAIL(ret)) { @@ -182,6 +183,7 @@ int ObTxTable::online() recycle_scn_cache_.reset(); (void)reset_ctx_min_start_scn_info_(); ATOMIC_STORE(&state_, ObTxTable::ONLINE); + ATOMIC_STORE(&calc_upper_trans_is_disabled_, false); LOG_INFO("tx table online succeed", K(ls_id_), KPC(this)); } @@ -1000,6 +1002,12 @@ int ObTxTable::get_uncommitted_tx_min_start_scn(share::SCN &min_start_scn, share void ObTxTable::update_min_start_scn_info(const SCN &max_decided_scn) { + if (true == ATOMIC_LOAD(&calc_upper_trans_is_disabled_)) { + // quit updating if calculate upper trans versions disabled + STORAGE_LOG(INFO, "skip update min start scn", K(max_decided_scn), KPC(this)); + return; + } + int64_t cur_ts = ObClockGenerator::getClock(); SpinWLockGuard lock_guard(ctx_min_start_scn_info_.lock_); @@ -1037,23 +1045,38 @@ void ObTxTable::update_min_start_scn_info(const SCN &max_decided_scn) } } } + + STORAGE_LOG(INFO, "finish update min start scn", K(max_decided_scn), K(ctx_min_start_scn_info_)); } int ObTxTable::get_upper_trans_version_before_given_scn(const SCN sstable_end_scn, SCN &upper_trans_version) { - return tx_data_table_.get_upper_trans_version_before_given_scn(sstable_end_scn, upper_trans_version); + int ret = OB_SUCCESS; + if (ATOMIC_LOAD(&calc_upper_trans_is_disabled_)) { + // cannot calculate upper trans version right now + if (REACH_TIME_INTERVAL(1LL * 1000LL * 1000LL)) { + STORAGE_LOG(INFO, "calc upper trans version is disabled", K(calc_upper_trans_is_disabled_), K(sstable_end_scn)); + } + } else { + ret = tx_data_table_.get_upper_trans_version_before_given_scn(sstable_end_scn, upper_trans_version); + } + return ret; } void ObTxTable::disable_upper_trans_calculation() { + ATOMIC_STORE(&calc_upper_trans_is_disabled_, true); (void)tx_data_table_.disable_upper_trans_calculation(); reset_ctx_min_start_scn_info_(); + FLOG_INFO("disable upper trans version calculation", KPC(this)); } void ObTxTable::enable_upper_trans_calculation(const share::SCN latest_transfer_scn) { reset_ctx_min_start_scn_info_(); (void)tx_data_table_.enable_upper_trans_calculation(latest_transfer_scn); + ATOMIC_STORE(&calc_upper_trans_is_disabled_, false); + FLOG_INFO("enable upper trans version calculation", KPC(this)); } int ObTxTable::get_start_tx_scn(SCN &start_tx_scn) diff --git a/src/storage/tx_table/ob_tx_table.h b/src/storage/tx_table/ob_tx_table.h index 6376f7b5b..f30756b58 100644 --- a/src/storage/tx_table/ob_tx_table.h +++ b/src/storage/tx_table/ob_tx_table.h @@ -102,6 +102,7 @@ public: public: ObTxTable() : is_inited_(false), + calc_upper_trans_is_disabled_(false), epoch_(INVALID_READ_EPOCH), state_(OFFLINE), ls_id_(), @@ -115,6 +116,7 @@ public: ObTxTable(ObTxDataTable &tx_data_table) : is_inited_(false), + calc_upper_trans_is_disabled_(false), epoch_(INVALID_READ_EPOCH), state_(OFFLINE), ls_id_(), @@ -310,7 +312,9 @@ public: void enable_upper_trans_calculation(const share::SCN latest_transfer_scn); TO_STRING_KV(KP(this), + K_(ls_id), K_(is_inited), + K_(calc_upper_trans_is_disabled), K_(epoch), "state", get_state_string(state_), KP_(ls), @@ -368,6 +372,7 @@ private: static const int64_t LS_TX_CTX_SCHEMA_ROWKEY_CNT = 1; static const int64_t LS_TX_CTX_SCHEMA_COLUMN_CNT = 3; bool is_inited_; + bool calc_upper_trans_is_disabled_; int64_t epoch_ CACHE_ALIGNED; TxTableState state_ CACHE_ALIGNED; share::ObLSID ls_id_; From 27fca33fd066d3b232d560416744c2eb65628122 Mon Sep 17 00:00:00 2001 From: obdev Date: Tue, 27 Aug 2024 16:29:36 +0000 Subject: [PATCH 247/249] fix bug of applying memory for fast heap mode --- .../table_load/ob_table_load_coordinator.cpp | 51 +++++++++++-------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/src/observer/table_load/ob_table_load_coordinator.cpp b/src/observer/table_load/ob_table_load_coordinator.cpp index 26564f7ad..d547a9faa 100644 --- a/src/observer/table_load/ob_table_load_coordinator.cpp +++ b/src/observer/table_load/ob_table_load_coordinator.cpp @@ -298,6 +298,7 @@ int ObTableLoadCoordinator::gen_apply_arg(ObDirectLoadResourceApplyArg &apply_ar ObArray partitions; int64_t store_server_count = all_leader_info_array.count(); int64_t coordinator_session_count = 0; + int64_t write_session_count = 0; int64_t min_session_count = MAX(ctx_->param_.parallel_, 2); int64_t max_session_count = (int64_t)tenant->unit_max_cpu() * 2; int64_t total_session_count = MIN(ctx_->param_.parallel_, max_session_count * store_server_count); @@ -351,27 +352,38 @@ int ObTableLoadCoordinator::gen_apply_arg(ObDirectLoadResourceApplyArg &apply_ar } } } - } - for (int64_t i = 0; OB_SUCC(ret) && i < store_server_count; i++) { - ObDirectLoadResourceUnit &unit = apply_arg.apply_array_[i]; - if (all_leader_info_array[i].addr_ == coordinator_addr) { - coordinator_session_count = unit.thread_count_; - } - min_session_count = MIN(min_session_count, unit.thread_count_); - int64_t min_unsort_memory = MACROBLOCK_BUFFER_SIZE * partitions[i] * unit.thread_count_; - if (ctx_->schema_.is_heap_table_) { - if (min_unsort_memory <= memory_limit) { - need_sort = false; - unit.memory_size_ = min_unsort_memory; - } else { - need_sort = ctx_->param_.need_sort_; // allow forced non-sorting - unit.memory_size_ = MIN(ObTableLoadAssignedMemoryManager::MIN_SORT_MEMORY_PER_TASK, memory_limit); + for (int64_t i = 0; i < store_server_count; i++) { + ObDirectLoadResourceUnit &unit = apply_arg.apply_array_[i]; + if (all_leader_info_array[i].addr_ == coordinator_addr) { + coordinator_session_count = unit.thread_count_; } + min_session_count = MIN(min_session_count, unit.thread_count_); + } + if (include_cur_addr && ctx_->param_.load_mode_ == ObDirectLoadMode::LOAD_DATA) { + // 协调节点和数据节点都在同一个节点上，对于load data模式，分出一半的线程用于解析数据，一半的线程用于存储数据 + write_session_count = MIN(min_session_count, (coordinator_session_count + 1) / 2); } else { - if (need_sort) { - unit.memory_size_ = MIN(ObTableLoadAssignedMemoryManager::MIN_SORT_MEMORY_PER_TASK, memory_limit); + write_session_count = min_session_count; + } + for (int64_t i = 0; i < store_server_count; i++) { + ObDirectLoadResourceUnit &unit = apply_arg.apply_array_[i]; + int64_t min_unsort_memory = 0; + if (ctx_->schema_.is_heap_table_) { + min_unsort_memory = MACROBLOCK_BUFFER_SIZE * partitions[i] * write_session_count; + if (min_unsort_memory <= memory_limit) { + need_sort = false; + unit.memory_size_ = min_unsort_memory; + } else { + need_sort = ctx_->param_.need_sort_; // allow forced non-sorting + unit.memory_size_ = MIN(ObTableLoadAssignedMemoryManager::MIN_SORT_MEMORY_PER_TASK, memory_limit); + } } else { - unit.memory_size_ = MIN(min_unsort_memory, memory_limit); + min_unsort_memory = MACROBLOCK_BUFFER_SIZE * partitions[i] * unit.thread_count_; + if (need_sort) { + unit.memory_size_ = MIN(ObTableLoadAssignedMemoryManager::MIN_SORT_MEMORY_PER_TASK, memory_limit); + } else { + unit.memory_size_ = MIN(min_unsort_memory, memory_limit); + } } } } @@ -390,8 +402,7 @@ int ObTableLoadCoordinator::gen_apply_arg(ObDirectLoadResourceApplyArg &apply_ar } else { ctx_->param_.need_sort_ = need_sort; ctx_->param_.session_count_ = coordinator_session_count; - ctx_->param_.write_session_count_ = - (include_cur_addr ? MIN(min_session_count, (coordinator_session_count + 1) / 2) : min_session_count); + ctx_->param_.write_session_count_ = write_session_count; ctx_->param_.exe_mode_ = (ctx_->schema_.is_heap_table_ ? (need_sort ? ObTableLoadExeMode::MULTIPLE_HEAP_TABLE_COMPACT : ObTableLoadExeMode::FAST_HEAP_TABLE) : (need_sort ? ObTableLoadExeMode::MEM_COMPACT : ObTableLoadExeMode::GENERAL_TABLE_COMPACT)); From 80f57afdf13ea664aa898d4ff8fc6504acd72260 Mon Sep 17 00:00:00 2001 From: zhjc1124 Date: Tue, 27 Aug 2024 16:36:21 +0000 Subject: [PATCH 248/249] [CP] raise SIGKILL when observer start failed --- src/observer/ob_server.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/observer/ob_server.cpp b/src/observer/ob_server.cpp index 2e3b7db91..804d98e3f 100644 --- a/src/observer/ob_server.cpp +++ b/src/observer/ob_server.cpp @@ -548,13 +548,14 @@ int ObServer::init(const ObServerOptions &opts, const ObPLogWriterCfg &log_cfg) } if (OB_FAIL(ret)) { - set_stop(); - destroy(); LOG_ERROR("[OBSERVER_NOTICE] fail to init observer", KR(ret)); LOG_DBA_FORCE_PRINT(DBA_ERROR, OB_SERVER_INIT_FAIL, ret, DBA_STEP_INC_INFO(server_start), "observer init fail. " "you may find solutions in previous error logs or seek help from official technicians."); + raise(SIGKILL); + set_stop(); + destroy(); } else { FLOG_INFO("[OBSERVER_NOTICE] success to init observer", "cluster_id", obrpc::ObRpcNetHandler::CLUSTER_ID, "lib::g_runtime_enabled", lib::g_runtime_enabled); @@ -1163,6 +1164,7 @@ int ObServer::start() DBA_STEP_INC_INFO(server_start), "observer start fail, the stop status is ", stop_, ". " "you may find solutions in previous error logs or seek help from official technicians."); + raise(SIGKILL); set_stop(); wait(); } else if (!stop_) { From 4f66a31a19f8ee0223d273338d0a62f871ddd513 Mon Sep 17 00:00:00 2001 From: Charles0429 Date: Tue, 27 Aug 2024 16:55:14 +0000 Subject: [PATCH 249/249] fix create hidden table hold too much memory --- .../ddl_task/ob_build_mview_task.cpp | 11 ---- src/rootserver/ddl_task/ob_build_mview_task.h | 2 - .../ddl_task/ob_column_redefinition_task.cpp | 48 ---------------- .../ddl_task/ob_column_redefinition_task.h | 2 - .../ddl_task/ob_constraint_task.cpp | 36 ------------ src/rootserver/ddl_task/ob_constraint_task.h | 2 - .../ddl_task/ob_ddl_redefinition_task.h | 2 - src/rootserver/ddl_task/ob_ddl_retry_task.cpp | 30 ---------- src/rootserver/ddl_task/ob_ddl_retry_task.h | 2 - src/rootserver/ddl_task/ob_ddl_task.cpp | 4 +- src/rootserver/ddl_task/ob_ddl_task.h | 9 +-- .../ddl_task/ob_drop_fts_index_task.cpp | 10 ---- .../ddl_task/ob_drop_fts_index_task.h | 2 - .../ddl_task/ob_drop_index_task.cpp | 48 ---------------- src/rootserver/ddl_task/ob_drop_index_task.h | 2 - .../ddl_task/ob_drop_primary_key_task.cpp | 51 ----------------- .../ddl_task/ob_drop_primary_key_task.h | 2 - .../ddl_task/ob_fts_index_build_task.cpp | 8 --- .../ddl_task/ob_fts_index_build_task.h | 2 - .../ddl_task/ob_index_build_task.cpp | 46 --------------- src/rootserver/ddl_task/ob_index_build_task.h | 2 - .../ddl_task/ob_modify_autoinc_task.cpp | 36 ------------ .../ddl_task/ob_modify_autoinc_task.h | 2 - .../ddl_task/ob_table_redefinition_task.cpp | 56 ------------------- .../ddl_task/ob_table_redefinition_task.h | 2 - src/share/ob_ddl_common.cpp | 28 ++-------- 26 files changed, 12 insertions(+), 433 deletions(-) diff --git a/src/rootserver/ddl_task/ob_build_mview_task.cpp b/src/rootserver/ddl_task/ob_build_mview_task.cpp index 49838c141..ee6efa60c 100644 --- a/src/rootserver/ddl_task/ob_build_mview_task.cpp +++ b/src/rootserver/ddl_task/ob_build_mview_task.cpp @@ -300,17 +300,6 @@ int ObBuildMViewTask::cleanup_impl() return ret; } -void ObBuildMViewTask::flt_set_task_span_tag() const -{ - LOG_INFO("flt_set_task_span_tag begin"); -} - -void ObBuildMViewTask::flt_set_status_span_tag() const -{ - LOG_INFO("flt_set_status_span_tag begin"); -} - - int ObBuildMViewTask::mview_complete_refresh(obrpc::ObMViewCompleteRefreshRes &res) { int ret = OB_SUCCESS; diff --git a/src/rootserver/ddl_task/ob_build_mview_task.h b/src/rootserver/ddl_task/ob_build_mview_task.h index 6a50fac81..6fb766d58 100644 --- a/src/rootserver/ddl_task/ob_build_mview_task.h +++ b/src/rootserver/ddl_task/ob_build_mview_task.h @@ -37,8 +37,6 @@ public: const int64_t snapshot_version = 0); virtual int process() override; virtual int cleanup_impl() override; - virtual void flt_set_task_span_tag() const override; - virtual void flt_set_status_span_tag() const override; virtual int serialize_params_to_message(char *buf, const int64_t buf_size, int64_t &pos) const override; diff --git a/src/rootserver/ddl_task/ob_column_redefinition_task.cpp b/src/rootserver/ddl_task/ob_column_redefinition_task.cpp index 04f5eda5a..56d597eb4 100644 --- a/src/rootserver/ddl_task/ob_column_redefinition_task.cpp +++ b/src/rootserver/ddl_task/ob_column_redefinition_task.cpp @@ -798,51 +798,3 @@ int ObColumnRedefinitionTask::collect_longops_stat(ObLongopsValue &value) return ret; } - -void ObColumnRedefinitionTask::flt_set_task_span_tag() const -{ - FLT_SET_TAG(ddl_task_id, task_id_, ddl_parent_task_id, parent_task_id_, - ddl_data_table_id, object_id_, ddl_schema_version, schema_version_, - ddl_snapshot_version, snapshot_version_, ddl_ret_code, ret_code_); -} - -void ObColumnRedefinitionTask::flt_set_status_span_tag() const -{ - switch (task_status_) { - case ObDDLTaskStatus::PREPARE: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::WAIT_TRANS_END: { - FLT_SET_TAG(ddl_data_table_id, object_id_, ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::OBTAIN_SNAPSHOT: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::REDEFINITION: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::COPY_TABLE_DEPENDENT_OBJECTS: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::TAKE_EFFECT: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::FAIL: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::SUCCESS: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - default: { - break; - } - } -} \ No newline at end of file diff --git a/src/rootserver/ddl_task/ob_column_redefinition_task.h b/src/rootserver/ddl_task/ob_column_redefinition_task.h index d065e7788..13f7abc2e 100644 --- a/src/rootserver/ddl_task/ob_column_redefinition_task.h +++ b/src/rootserver/ddl_task/ob_column_redefinition_task.h @@ -52,8 +52,6 @@ public: const ObDDLTaskInfo &addition_info) override; virtual int collect_longops_stat(share::ObLongopsValue &value) override; virtual bool support_longops_monitoring() const { return true; } - virtual void flt_set_task_span_tag() const override; - virtual void flt_set_status_span_tag() const override; virtual int serialize_params_to_message(char *buf, const int64_t buf_len, int64_t &pos) const override; virtual int deserialize_params_from_message(const uint64_t tenant_id, const char *buf, const int64_t data_len, int64_t &pos) override; virtual int64_t get_serialize_param_size() const override; diff --git a/src/rootserver/ddl_task/ob_constraint_task.cpp b/src/rootserver/ddl_task/ob_constraint_task.cpp index ac5135337..365149222 100755 --- a/src/rootserver/ddl_task/ob_constraint_task.cpp +++ b/src/rootserver/ddl_task/ob_constraint_task.cpp @@ -2054,39 +2054,3 @@ int64_t ObConstraintTask::get_serialize_param_size() const { return alter_table_arg_.get_serialize_size() + ObDDLTask::get_serialize_param_size(); } -void ObConstraintTask::flt_set_task_span_tag() const -{ - FLT_SET_TAG(ddl_task_id, task_id_, ddl_parent_task_id, parent_task_id_, - ddl_data_table_id, object_id_, ddl_schema_version, schema_version_, - ddl_snapshot_version, snapshot_version_); -} - -void ObConstraintTask::flt_set_status_span_tag() const -{ - switch (task_status_) { - case ObDDLTaskStatus::WAIT_TRANS_END: { - FLT_SET_TAG(ddl_data_table_id, object_id_, ddl_schema_version, schema_version_, - ddl_snapshot_version, snapshot_version_, ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::CHECK_CONSTRAINT_VALID: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::SET_CONSTRAINT_VALIDATE: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::FAIL: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::SUCCESS: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - default: { - break; - } - } -} diff --git a/src/rootserver/ddl_task/ob_constraint_task.h b/src/rootserver/ddl_task/ob_constraint_task.h index 75b69572e..5d431b4e9 100644 --- a/src/rootserver/ddl_task/ob_constraint_task.h +++ b/src/rootserver/ddl_task/ob_constraint_task.h @@ -108,8 +108,6 @@ public: virtual int serialize_params_to_message(char *buf, const int64_t buf_size, int64_t &pos) const override; virtual int deserialize_params_from_message(const uint64_t tenant_id, const char *buf, const int64_t buf_size, int64_t &pos) override; virtual int64_t get_serialize_param_size() const override; - virtual void flt_set_task_span_tag() const override; - virtual void flt_set_status_span_tag() const override; virtual int cleanup_impl() override; private: int hold_snapshot(const int64_t snapshot_version); diff --git a/src/rootserver/ddl_task/ob_ddl_redefinition_task.h b/src/rootserver/ddl_task/ob_ddl_redefinition_task.h index 2be058bab..ed98b672c 100644 --- a/src/rootserver/ddl_task/ob_ddl_redefinition_task.h +++ b/src/rootserver/ddl_task/ob_ddl_redefinition_task.h @@ -148,8 +148,6 @@ public: const uint64_t child_task_key, const int ret_code) override; int notify_update_autoinc_finish(const uint64_t autoinc_val, const int ret_code); - virtual void flt_set_task_span_tag() const = 0; - virtual void flt_set_status_span_tag() const = 0; virtual int cleanup_impl() override; int reap_old_replica_build_task(bool &need_exec_new_inner_sql); INHERIT_TO_STRING_KV("ObDDLTask", ObDDLTask, diff --git a/src/rootserver/ddl_task/ob_ddl_retry_task.cpp b/src/rootserver/ddl_task/ob_ddl_retry_task.cpp index 07587e545..b4212f7fb 100644 --- a/src/rootserver/ddl_task/ob_ddl_retry_task.cpp +++ b/src/rootserver/ddl_task/ob_ddl_retry_task.cpp @@ -734,33 +734,3 @@ int ObDDLRetryTask::update_task_status_wait_child_task_finish( } return ret; } - -void ObDDLRetryTask::flt_set_task_span_tag() const -{ - FLT_SET_TAG(ddl_task_id, task_id_, ddl_parent_task_id, parent_task_id_, ddl_data_table_id, object_id_); -} - -void ObDDLRetryTask::flt_set_status_span_tag() const -{ - switch (task_status_) { - case ObDDLTaskStatus::PREPARE: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::DROP_SCHEMA: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::FAIL: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::SUCCESS: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - default: { - break; - } - } -} diff --git a/src/rootserver/ddl_task/ob_ddl_retry_task.h b/src/rootserver/ddl_task/ob_ddl_retry_task.h index 13b8fd643..922746c4a 100644 --- a/src/rootserver/ddl_task/ob_ddl_retry_task.h +++ b/src/rootserver/ddl_task/ob_ddl_retry_task.h @@ -47,8 +47,6 @@ public: common::ObMySQLTransaction &trans, const uint64_t tenant_id, const int64_t task_id); - virtual void flt_set_task_span_tag() const override; - virtual void flt_set_status_span_tag() const override; virtual int cleanup_impl() override; private: int check_health(); diff --git a/src/rootserver/ddl_task/ob_ddl_task.cpp b/src/rootserver/ddl_task/ob_ddl_task.cpp index fc2a4943b..d35853ef6 100644 --- a/src/rootserver/ddl_task/ob_ddl_task.cpp +++ b/src/rootserver/ddl_task/ob_ddl_task.cpp @@ -348,7 +348,7 @@ void ObDDLTracing::end_task_span() int ret = OB_SUCCESS; FLT_RESTORE_DDL_TRACE_CTX(trace_ctx_); FLT_SET_AUTO_FLUSH(true); - task_->flt_set_task_span_tag(); + // task_->flt_set_task_span_tag(); // TODO hanxuan fix MTL_SWITCH MTL_SWITCH(OB_SYS_TENANT_ID) { FLT_END_SPAN(task_span_); @@ -513,7 +513,7 @@ void ObDDLTracing::end_status_span() int ret = OB_SUCCESS; FLT_RESTORE_DDL_TRACE_CTX(trace_ctx_); FLT_SET_AUTO_FLUSH(true); - task_->flt_set_status_span_tag(); + //task_->flt_set_status_span_tag(); // TODO hanxuan fix MTL_SWITCH MTL_SWITCH(OB_SYS_TENANT_ID) { FLT_END_SPAN(status_span_); diff --git a/src/rootserver/ddl_task/ob_ddl_task.h b/src/rootserver/ddl_task/ob_ddl_task.h index 06d1fa771..cc39af92e 100755 --- a/src/rootserver/ddl_task/ob_ddl_task.h +++ b/src/rootserver/ddl_task/ob_ddl_task.h @@ -544,8 +544,11 @@ public: longops_stat_(nullptr), gmt_create_(0), stat_info_(), delay_schedule_time_(0), next_schedule_ts_(0), execution_id_(-1), start_time_(0), data_format_version_(0), is_pre_split_(false), wait_trans_ctx_() {} + ObDDLTask(): + ObDDLTask(share::DDL_INVALID) + {} virtual ~ObDDLTask() {} - virtual int process() = 0; + virtual int process() { return OB_NOT_SUPPORTED; } virtual int on_child_task_finish(const uint64_t child_task_key, const int ret_code) { return common::OB_NOT_SUPPORTED; } virtual bool is_valid() const { return is_inited_; } typedef common::ObCurTraceId::TraceId TraceId; @@ -627,9 +630,7 @@ public: static bool check_is_load_data(share::ObDDLType task_type); virtual bool support_longops_monitoring() const { return false; } int cleanup(); - virtual int cleanup_impl() = 0; - virtual void flt_set_task_span_tag() const = 0; - virtual void flt_set_status_span_tag() const = 0; + virtual int cleanup_impl() { return OB_NOT_SUPPORTED; } int update_task_record_status_and_msg(common::ObISQLClient &proxy, const share::ObDDLTaskStatus real_new_status); #ifdef ERRSIM diff --git a/src/rootserver/ddl_task/ob_drop_fts_index_task.cpp b/src/rootserver/ddl_task/ob_drop_fts_index_task.cpp index e101decde..cf33c287b 100644 --- a/src/rootserver/ddl_task/ob_drop_fts_index_task.cpp +++ b/src/rootserver/ddl_task/ob_drop_fts_index_task.cpp @@ -235,16 +235,6 @@ int64_t ObDropFTSIndexTask::get_serialize_param_size() const + fts_doc_word_.get_serialize_size(); } -void ObDropFTSIndexTask::flt_set_task_span_tag() const -{ - // TODO: @hanxuan, add me for tracing. -} - -void ObDropFTSIndexTask::flt_set_status_span_tag() const -{ - // TODO: @hanxuan, add me for tracing. -} - int ObDropFTSIndexTask::check_switch_succ() { int ret = OB_SUCCESS; diff --git a/src/rootserver/ddl_task/ob_drop_fts_index_task.h b/src/rootserver/ddl_task/ob_drop_fts_index_task.h index 883e72fa0..710fff73b 100644 --- a/src/rootserver/ddl_task/ob_drop_fts_index_task.h +++ b/src/rootserver/ddl_task/ob_drop_fts_index_task.h @@ -63,8 +63,6 @@ public: int64_t &pos) override; virtual int64_t get_serialize_param_size() const override; - virtual void flt_set_task_span_tag() const override; - virtual void flt_set_status_span_tag() const override; virtual int on_child_task_finish(const uint64_t child_task_key, const int ret_code) override { return OB_SUCCESS; } INHERIT_TO_STRING_KV("ObDDLTask", ObDDLTask, K_(rowkey_doc), K_(doc_rowkey), K_(domain_index), K_(fts_doc_word)); diff --git a/src/rootserver/ddl_task/ob_drop_index_task.cpp b/src/rootserver/ddl_task/ob_drop_index_task.cpp index 3195dd8b2..dc492d726 100644 --- a/src/rootserver/ddl_task/ob_drop_index_task.cpp +++ b/src/rootserver/ddl_task/ob_drop_index_task.cpp @@ -481,51 +481,3 @@ int64_t ObDropIndexTask::get_serialize_param_size() const { return drop_index_arg_.get_serialize_size() + ObDDLTask::get_serialize_param_size(); } - -void ObDropIndexTask::flt_set_task_span_tag() const -{ - FLT_SET_TAG(ddl_task_id, task_id_, ddl_parent_task_id, parent_task_id_, - ddl_data_table_id, object_id_, ddl_index_table_id, target_object_id_, - ddl_schema_version, schema_version_); -} - -void ObDropIndexTask::flt_set_status_span_tag() const -{ - switch (task_status_) { - case ObDDLTaskStatus::PREPARE: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::SET_WRITE_ONLY: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::WAIT_TRANS_END_FOR_WRITE_ONLY: { - FLT_SET_TAG(ddl_data_table_id, object_id_, ddl_snapshot_version, snapshot_version_, ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::SET_UNUSABLE: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::WAIT_TRANS_END_FOR_UNUSABLE: { - FLT_SET_TAG(ddl_data_table_id, object_id_, ddl_snapshot_version, snapshot_version_, ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::DROP_SCHEMA: { - FLT_SET_TAG(ddl_index_table_id, target_object_id_, ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::FAIL: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::SUCCESS: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - default: { - break; - } - } -} diff --git a/src/rootserver/ddl_task/ob_drop_index_task.h b/src/rootserver/ddl_task/ob_drop_index_task.h index fd3329ea2..4fa1da4b9 100644 --- a/src/rootserver/ddl_task/ob_drop_index_task.h +++ b/src/rootserver/ddl_task/ob_drop_index_task.h @@ -43,8 +43,6 @@ public: virtual int deserialize_params_from_message(const uint64_t tenant_id, const char *buf, const int64_t buf_size, int64_t &pos) override; virtual int64_t get_serialize_param_size() const override; INHERIT_TO_STRING_KV("ObDDLTask", ObDDLTask, KP_(root_service)); - virtual void flt_set_task_span_tag() const override; - virtual void flt_set_status_span_tag() const override; virtual int cleanup_impl() override; private: int check_switch_succ(); diff --git a/src/rootserver/ddl_task/ob_drop_primary_key_task.cpp b/src/rootserver/ddl_task/ob_drop_primary_key_task.cpp index 40fc6ad99..5d9c4fc61 100644 --- a/src/rootserver/ddl_task/ob_drop_primary_key_task.cpp +++ b/src/rootserver/ddl_task/ob_drop_primary_key_task.cpp @@ -127,54 +127,3 @@ int ObDropPrimaryKeyTask::process() } return ret; } -void ObDropPrimaryKeyTask::flt_set_task_span_tag() const -{ - FLT_SET_TAG(ddl_task_id, task_id_, ddl_parent_task_id, parent_task_id_, - ddl_data_table_id, object_id_, ddl_schema_version, schema_version_); -} - -void ObDropPrimaryKeyTask::flt_set_status_span_tag() const -{ - switch (task_status_) { - case ObDDLTaskStatus::PREPARE: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::OBTAIN_SNAPSHOT: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::WAIT_TRANS_END: { - FLT_SET_TAG(ddl_data_table_id, object_id_, ddl_schema_version, schema_version_, - ddl_snapshot_version, snapshot_version_, ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::REDEFINITION: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::COPY_TABLE_DEPENDENT_OBJECTS: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::MODIFY_AUTOINC: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::TAKE_EFFECT: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::FAIL: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::SUCCESS: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - default: { - break; - } - } -} diff --git a/src/rootserver/ddl_task/ob_drop_primary_key_task.h b/src/rootserver/ddl_task/ob_drop_primary_key_task.h index 8862fac7f..f1e8f93fd 100644 --- a/src/rootserver/ddl_task/ob_drop_primary_key_task.h +++ b/src/rootserver/ddl_task/ob_drop_primary_key_task.h @@ -40,8 +40,6 @@ public: const int64_t task_status = share::ObDDLTaskStatus::PREPARE, const int64_t snapshot_version = 0); virtual int process() override; - virtual void flt_set_task_span_tag() const override; - virtual void flt_set_status_span_tag() const override; private: static const int64_t OB_DROP_PRIMARY_KEY_TASK_VERSION = 1L; }; diff --git a/src/rootserver/ddl_task/ob_fts_index_build_task.cpp b/src/rootserver/ddl_task/ob_fts_index_build_task.cpp index 88728fd21..56195498a 100644 --- a/src/rootserver/ddl_task/ob_fts_index_build_task.cpp +++ b/src/rootserver/ddl_task/ob_fts_index_build_task.cpp @@ -1655,13 +1655,5 @@ int ObFtsIndexBuildTask::cleanup_impl() return ret; } -void ObFtsIndexBuildTask::flt_set_task_span_tag() const -{ -} - -void ObFtsIndexBuildTask::flt_set_status_span_tag() const -{ -} - } // end namespace rootserver } // end namespace oceanbase diff --git a/src/rootserver/ddl_task/ob_fts_index_build_task.h b/src/rootserver/ddl_task/ob_fts_index_build_task.h index 6a9c7949a..254ecb077 100644 --- a/src/rootserver/ddl_task/ob_fts_index_build_task.h +++ b/src/rootserver/ddl_task/ob_fts_index_build_task.h @@ -39,8 +39,6 @@ public: const int64_t snapshot_version = 0); int init(const ObDDLTaskRecord &task_record); virtual int process() override; - virtual void flt_set_task_span_tag() const override; - virtual void flt_set_status_span_tag() const override; virtual int cleanup_impl() override; virtual bool is_valid() const override; virtual int collect_longops_stat(share::ObLongopsValue &value) override; diff --git a/src/rootserver/ddl_task/ob_index_build_task.cpp b/src/rootserver/ddl_task/ob_index_build_task.cpp index b0ee65b10..f96504cc6 100755 --- a/src/rootserver/ddl_task/ob_index_build_task.cpp +++ b/src/rootserver/ddl_task/ob_index_build_task.cpp @@ -346,52 +346,6 @@ int ObIndexBuildTask::process() return ret; } -void ObIndexBuildTask::flt_set_task_span_tag() const -{ - FLT_SET_TAG(ddl_task_id, task_id_, ddl_parent_task_id, parent_task_id_, - ddl_data_table_id, object_id_, ddl_index_table_id, index_table_id_, - ddl_is_unique_index, is_unique_index_, ddl_is_global_index, is_global_index_, - ddl_schema_version, schema_version_); -} - -void ObIndexBuildTask::flt_set_status_span_tag() const -{ - switch (task_status_) { - case ObDDLTaskStatus::PREPARE: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::WAIT_TRANS_END: { - FLT_SET_TAG(ddl_data_table_id, object_id_, ddl_index_table_id, index_table_id_, ddl_schema_version, schema_version_, - ddl_snapshot_version, snapshot_version_, ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::REDEFINITION: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::VALIDATE_CHECKSUM: { - FLT_SET_TAG(ddl_check_unique_snapshot, check_unique_snapshot_, ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::TAKE_EFFECT: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::FAIL: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::SUCCESS: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - default: { - break; - } - } -} - int ObIndexBuildTask::init( const uint64_t tenant_id, const int64_t task_id, diff --git a/src/rootserver/ddl_task/ob_index_build_task.h b/src/rootserver/ddl_task/ob_index_build_task.h index 3dbd3f097..f54f1048b 100644 --- a/src/rootserver/ddl_task/ob_index_build_task.h +++ b/src/rootserver/ddl_task/ob_index_build_task.h @@ -116,8 +116,6 @@ public: const int ret_code, const ObDDLTaskInfo &addition_info); virtual int process() override; - virtual void flt_set_task_span_tag() const override; - virtual void flt_set_status_span_tag() const override; virtual int cleanup_impl() override; virtual bool is_valid() const override; virtual int collect_longops_stat(share::ObLongopsValue &value) override; diff --git a/src/rootserver/ddl_task/ob_modify_autoinc_task.cpp b/src/rootserver/ddl_task/ob_modify_autoinc_task.cpp index 8c4ec8985..324b2a33f 100644 --- a/src/rootserver/ddl_task/ob_modify_autoinc_task.cpp +++ b/src/rootserver/ddl_task/ob_modify_autoinc_task.cpp @@ -673,39 +673,3 @@ int64_t ObModifyAutoincTask::get_serialize_param_size() const { return alter_table_arg_.get_serialize_size() + ObDDLTask::get_serialize_param_size(); } - -void ObModifyAutoincTask::flt_set_task_span_tag() const -{ - FLT_SET_TAG(ddl_task_id, task_id_, ddl_parent_task_id, parent_task_id_, - ddl_data_table_id, object_id_, ddl_schema_version, schema_version_, - ddl_snapshot_version, snapshot_version_, ddl_ret_code, ret_code_); -} - -void ObModifyAutoincTask::flt_set_status_span_tag() const -{ - switch (task_status_) { - case ObDDLTaskStatus::WAIT_TRANS_END: { - FLT_SET_TAG(ddl_data_table_id, object_id_, ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::OBTAIN_SNAPSHOT: { - FLT_SET_TAG(ddl_data_table_id, object_id_, ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::MODIFY_AUTOINC: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::FAIL: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::SUCCESS: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - default: { - break; - } - } -} diff --git a/src/rootserver/ddl_task/ob_modify_autoinc_task.h b/src/rootserver/ddl_task/ob_modify_autoinc_task.h index 37900a883..f810e495e 100644 --- a/src/rootserver/ddl_task/ob_modify_autoinc_task.h +++ b/src/rootserver/ddl_task/ob_modify_autoinc_task.h @@ -69,8 +69,6 @@ public: virtual int deserialize_params_from_message(const uint64_t tenant_id, const char *buf, const int64_t buf_size, int64_t &pos) override; virtual int64_t get_serialize_param_size() const override; int notify_update_autoinc_finish(const uint64_t autoinc_val, const int ret_code); - virtual void flt_set_task_span_tag() const override; - virtual void flt_set_status_span_tag() const override; virtual int cleanup_impl() override; private: int unlock_table(); diff --git a/src/rootserver/ddl_task/ob_table_redefinition_task.cpp b/src/rootserver/ddl_task/ob_table_redefinition_task.cpp index 36e761f3a..e3c74d5db 100755 --- a/src/rootserver/ddl_task/ob_table_redefinition_task.cpp +++ b/src/rootserver/ddl_task/ob_table_redefinition_task.cpp @@ -1443,62 +1443,6 @@ int ObTableRedefinitionTask::collect_longops_stat(ObLongopsValue &value) return ret; } -void ObTableRedefinitionTask::flt_set_task_span_tag() const -{ - FLT_SET_TAG(ddl_task_id, task_id_, ddl_parent_task_id, parent_task_id_, - ddl_data_table_id, object_id_, ddl_schema_version, schema_version_, - ddl_ret_code, ret_code_); -} - -void ObTableRedefinitionTask::flt_set_status_span_tag() const -{ - switch (task_status_) { - case ObDDLTaskStatus::PREPARE: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::WAIT_TRANS_END: { - FLT_SET_TAG(ddl_data_table_id, object_id_, ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::OBTAIN_SNAPSHOT: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::CHECK_TABLE_EMPTY: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::REDEFINITION: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::COPY_TABLE_DEPENDENT_OBJECTS: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::MODIFY_AUTOINC: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::TAKE_EFFECT: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::FAIL: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - case ObDDLTaskStatus::SUCCESS: { - FLT_SET_TAG(ddl_ret_code, ret_code_); - break; - } - default: { - break; - } - } -} - int ObTableRedefinitionTask::get_direct_load_job_stat(common::ObArenaAllocator &allocator, sql::ObLoadDataStat &job_stat) { diff --git a/src/rootserver/ddl_task/ob_table_redefinition_task.h b/src/rootserver/ddl_task/ob_table_redefinition_task.h index c6799462f..4377f8a2a 100644 --- a/src/rootserver/ddl_task/ob_table_redefinition_task.h +++ b/src/rootserver/ddl_task/ob_table_redefinition_task.h @@ -65,8 +65,6 @@ public: int assign(const ObTableRedefinitionTask *table_redef_task); virtual int collect_longops_stat(share::ObLongopsValue &value) override; virtual bool support_longops_monitoring() const override { return true; } - virtual void flt_set_task_span_tag() const override; - virtual void flt_set_status_span_tag() const override; static bool check_task_status_is_pending(const share::ObDDLTaskStatus task_status); INHERIT_TO_STRING_KV("ObDDLRedefinitionTask", ObDDLRedefinitionTask, K(has_rebuild_index_), K(has_rebuild_constraint_), K(has_rebuild_foreign_key_), diff --git a/src/share/ob_ddl_common.cpp b/src/share/ob_ddl_common.cpp index d8d87150c..ad5e8e93b 100644 --- a/src/share/ob_ddl_common.cpp +++ b/src/share/ob_ddl_common.cpp @@ -1942,29 +1942,11 @@ int ObDDLUtil::get_data_information( task_status = static_cast(cur_task_status); if (OB_SUCC(ret)) { - if (is_create_index(ddl_type)) { - SMART_VAR(rootserver::ObIndexBuildTask, task) { - if (OB_FAIL(task.deserialize_params_from_message(tenant_id, task_message.ptr(), task_message.length(), pos))) { - LOG_WARN("deserialize from msg failed", K(ret)); - } else { - data_format_version = task.get_data_format_version(); - } - } - } else if (is_complement_data_relying_on_dag(ddl_type)) { - SMART_VAR(rootserver::ObColumnRedefinitionTask, task) { - if (OB_FAIL(task.deserialize_params_from_message(tenant_id, task_message.ptr(), task_message.length(), pos))) { - LOG_WARN("deserialize from msg failed", K(ret)); - } else { - data_format_version = task.get_data_format_version(); - } - } - } else { - SMART_VAR(rootserver::ObTableRedefinitionTask, task) { - if (OB_FAIL(task.deserialize_params_from_message(tenant_id, task_message.ptr(), task_message.length(), pos))) { - LOG_WARN("deserialize from msg failed", K(ret)); - } else { - data_format_version = task.get_data_format_version(); - } + SMART_VAR(rootserver::ObDDLTask, task) { + if (OB_FAIL(task.deserialize_params_from_message(tenant_id, task_message.ptr(), task_message.length(), pos))) { + LOG_WARN("deserialize from msg failed", K(ret)); + } else { + data_format_version = task.get_data_format_version(); } } }